@effect-uai/core 0.15.0 → 0.17.0

package/src/domain/Transcript.test.ts

@@ -0,0 +1,98 @@
+import { describe, it } from "@effect/vitest"
+import { Cause, Effect, Queue, Ref, Stream } from "effect"
+import { TestClock } from "effect/testing"
+import { expect } from "vitest"
+import { accumulatePartials, type TranscriptEvent } from "./Transcript.js"
+
+const collecting = (stream: Stream.Stream<TranscriptEvent, never>) =>
+  Effect.gen(function* () {
+    const seen = yield* Ref.make<ReadonlyArray<TranscriptEvent>>([])
+    yield* Effect.forkChild(Stream.runForEach(stream, (e) => Ref.update(seen, (xs) => [...xs, e])))
+    return seen
+  })
+
+/** Deltas arriving faster than any settle window. */
+const speak = (
+  queue: Queue.Queue<TranscriptEvent, Cause.Done>,
+  tokens: ReadonlyArray<string>,
+): Effect.Effect<void> =>
+  Effect.forEach(
+    tokens,
+    (text) =>
+      Effect.andThen(Queue.offer(queue, { _tag: "partial", text }), TestClock.adjust("100 millis")),
+    { discard: true },
+  )
+
+const opened = Effect.gen(function* () {
+  const queue = yield* Queue.unbounded<TranscriptEvent, Cause.Done>()
+  const seen = yield* collecting(
+    Stream.fromQueue(queue).pipe(accumulatePartials({ silence: "1 second" })),
+  )
+  return { queue, seen }
+})
+
+describe("accumulatePartials", () => {
+  it.effect("joins deltas into the running hypothesis, forwarding each as it arrives", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Hi", ",", " how"])
+
+      // Three in, three out, each the whole sentence so far, none delayed.
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Hi" },
+        { _tag: "partial", text: "Hi," },
+        { _tag: "partial", text: "Hi, how" },
+      ])
+    }),
+  )
+
+  it.effect("commits on silence and starts the next utterance from empty", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Hi", " there"])
+      yield* TestClock.adjust("3 seconds")
+      yield* speak(queue, [" Bye"])
+      yield* TestClock.adjust("3 seconds")
+
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Hi" },
+        { _tag: "partial", text: "Hi there" },
+        { _tag: "final", text: "Hi there" },
+        { _tag: "partial", text: "Bye" },
+        { _tag: "final", text: "Bye" },
+      ])
+    }),
+  )
+
+  it.effect("a provider's own final resets the accumulator, so silence adds no duplicate", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Hi"])
+      yield* Queue.offer(queue, { _tag: "final", text: "Hi." })
+      yield* TestClock.adjust("3 seconds")
+
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Hi" },
+        { _tag: "final", text: "Hi." },
+      ])
+    }),
+  )
+
+  it.effect("commits the pending text when the stream ends mid-utterance", () =>
+    Effect.gen(function* () {
+      const { queue, seen } = yield* opened
+
+      yield* speak(queue, [" Half a sen"])
+      yield* Queue.end(queue)
+      yield* TestClock.adjust("10 millis")
+
+      expect(yield* Ref.get(seen)).toEqual([
+        { _tag: "partial", text: "Half a sen" },
+        { _tag: "final", text: "Half a sen" },
+      ])
+    }),
+  )
+})

package/src/domain/Transcript.ts

@@ -1,4 +1,5 @@
-import type { Duration } from "effect"
+import { type Duration, Effect, Match, Option, Ref, Stream } from "effect"
+import * as Settle from "../streaming/Settle.js"
 
 /**
  * Per-word timing + metadata. `confidence` and `speakerId` are optional
@@ -87,3 +88,54 @@ export const isMetadata = (
 ): e is Extract<TranscriptEvent, { _tag: "metadata" }> => e._tag === "metadata"
 export const isError = (e: TranscriptEvent): e is Extract<TranscriptEvent, { _tag: "error" }> =>
   e._tag === "error"
+
+export type AccumulatePartialsOptions = {
+  /** Silence after the last `partial` before the text so far is committed. */
+  readonly silence?: Duration.Input
+}
+
+const DEFAULT_SILENCE = "700 millis"
+
+const finalOf = (text: string): Option.Option<TranscriptEvent> =>
+  text.trim().length === 0 ? Option.none() : Option.some({ _tag: "final", text: text.trim() })
+
+/**
+ * Join fragment `partial`s into the running hypothesis and commit it as a
+ * `final` after `silence`. For providers that stream token-sized deltas
+ * (OpenAI Realtime) or never segment; piping one that already accumulates
+ * repeats its text.
+ *
+ * Partials are forwarded as they arrive; only the synthetic `final` waits.
+ * A real `final` resets the accumulator, and the stream end flushes it.
+ */
+export const accumulatePartials =
+  (options?: AccumulatePartialsOptions) =>
+  <E, R>(self: Stream.Stream<TranscriptEvent, E, R>): Stream.Stream<TranscriptEvent, E, R> =>
+    Stream.unwrap(
+      Effect.gen(function* () {
+        const acc = yield* Ref.make("")
+        const flush = Effect.map(Ref.getAndSet(acc, ""), finalOf)
+
+        const rewrite = (event: TranscriptEvent): Effect.Effect<TranscriptEvent> =>
+          Match.value(event).pipe(
+            Match.when({ _tag: "partial" }, (e) =>
+              Effect.map(
+                Ref.updateAndGet(acc, (text) => text + e.text),
+                (text): TranscriptEvent => ({ ...e, text: text.trim() }),
+              ),
+            ),
+            Match.when({ _tag: "final" }, (e) => Effect.as(Ref.set(acc, ""), e as TranscriptEvent)),
+            Match.orElse((e) => Effect.succeed(e as TranscriptEvent)),
+          )
+
+        return self.pipe(
+          Stream.mapEffect(rewrite),
+          Settle.onQuiet(options?.silence ?? DEFAULT_SILENCE, flush),
+          Stream.concat(
+            Stream.unwrap(
+              Effect.map(flush, Option.match({ onNone: () => Stream.empty, onSome: Stream.make })),
+            ),
+          ),
+        )
+      }),
+    )

package/src/index.ts

@@ -6,6 +6,8 @@ export * as BrowserTool from "./browser/BrowserTool.js"
 export * as Capabilities from "./capabilities/Capabilities.js"
 export * as Chunker from "./chunker/Chunker.js"
 export * as Citation from "./domain/Citation.js"
+export * as Decision from "./decision-model/Decision.js"
+export * as DecisionModel from "./decision-model/DecisionModel.js"
 export * as DeepResearch from "./research/DeepResearch.js"
 export * as Image from "./domain/Image.js"
 export * as Items from "./domain/Items.js"
@@ -13,6 +15,8 @@ export * as Job from "./job/Job.js"
 export * as Media from "./domain/Media.js"
 export * as Music from "./domain/Music.js"
 export * as Research from "./domain/Research.js"
+export * as Realtime from "./domain/Realtime.js"
+export * as RealtimeSession from "./realtime/RealtimeSession.js"
 export * as Transcript from "./domain/Transcript.js"
 export * as Turn from "./domain/Turn.js"
 export * as Embedding from "./embedding-model/Embedding.js"
@@ -35,9 +39,11 @@ export * as WebRead from "./web-read/WebRead.js"
 export * as WebReadTool from "./web-read/WebReadTool.js"
 export * as WebSearch from "./web-search/WebSearch.js"
 export * as WebSearchTool from "./web-search/WebSearchTool.js"
+export * as Probability from "./math/Probability.js"
 export * as Vector from "./math/Vector.js"
 export * as Tokenizer from "./tokenizer/Tokenizer.js"
-export * as Inbox from "./inbox/Inbox.js"
+export * as Settle from "./streaming/Settle.js"
+export * as WebSocketSession from "./streaming/WebSocketSession.js"
 export * as Loop from "./loop/Loop.js"
 export * as Retry from "./retry/Retry.js"
 export * as Tool from "./tool/Tool.js"

package/src/math/Probability.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from "vitest"
+import { confidence, margin } from "./Probability.js"
+
+describe("confidence", () => {
+  it("is 1 when all mass sits on one outcome", () => {
+    expect(confidence([1, 0, 0])).toBeCloseTo(1, 10)
+  })
+
+  it("is 0 when mass is spread evenly, at any length", () => {
+    expect(confidence([0.5, 0.5])).toBeCloseTo(0, 10)
+    expect(confidence([0.25, 0.25, 0.25, 0.25])).toBeCloseTo(0, 10)
+  })
+
+  it("is 1 for fewer than two outcomes, which have nothing to be uncertain between", () => {
+    expect(confidence([])).toBe(1)
+    expect(confidence([1])).toBe(1)
+  })
+
+  it("rises as mass concentrates", () => {
+    const spread = confidence([0.4, 0.35, 0.25])
+    const leaning = confidence([0.6, 0.25, 0.15])
+    const decided = confidence([0.9, 0.07, 0.03])
+    expect(spread).toBeLessThan(leaning)
+    expect(leaning).toBeLessThan(decided)
+  })
+
+  it("is conservative on two outcomes, so a threshold does not transfer across lengths", () => {
+    expect(confidence([0.94, 0.06])).toBeCloseTo(0.67, 2)
+  })
+})
+
+describe("margin", () => {
+  it("is the gap between the top two, whatever order they arrive in", () => {
+    expect(margin([0.6, 0.3, 0.1])).toBeCloseTo(0.3, 10)
+    expect(margin([0.1, 0.6, 0.3])).toBeCloseTo(0.3, 10)
+  })
+
+  it("is 0 on a tie and 0 when empty", () => {
+    expect(margin([0.5, 0.5])).toBeCloseTo(0, 10)
+    expect(margin([])).toBe(0)
+  })
+
+  it("yields the lone probability for a single outcome", () => {
+    expect(margin([0.7])).toBeCloseTo(0.7, 10)
+  })
+})
+
+describe("confidence and margin disagree, which is why both exist", () => {
+  // Two plausible answers versus no idea. Same winner, same top mass.
+  const closeSecond = [0.45, 0.42, 0.13]
+  const flatSpread = [0.45, 0.11, 0.11, 0.11, 0.11, 0.11]
+
+  it("confidence cannot separate them", () => {
+    expect(Math.abs(confidence(closeSecond) - confidence(flatSpread))).toBeLessThan(0.05)
+  })
+
+  it("margin separates them sharply", () => {
+    expect(margin(closeSecond)).toBeCloseTo(0.03, 10)
+    expect(margin(flatSpread)).toBeCloseTo(0.34, 10)
+  })
+})

package/src/math/Probability.ts

@@ -0,0 +1,44 @@
+/** Certainty measures over a probability vector. */
+import { Array as A, Number as N, Option, Order, pipe } from "effect"
+
+const descending = Order.flip(Order.Number)
+
+/**
+ * 1 minus normalized Shannon entropy: 1 when all mass sits on one outcome, 0
+ * when spread evenly. Fewer than two outcomes yields 1.
+ *
+ * Conservative on two outcomes, where `[0.94, 0.06]` scores ~0.67, so a
+ * threshold tuned on a long vector does not transfer. {@link margin} is the
+ * more stable gate across vectors of different length.
+ */
+export const confidence = (probabilities: ReadonlyArray<number>): number =>
+  probabilities.length < 2
+    ? 1
+    : 1 -
+      pipe(
+        probabilities,
+        A.filter((p) => p > 0),
+        A.map((p) => -p * Math.log(p)),
+        N.sumAll,
+      ) /
+        Math.log(probabilities.length)
+
+/**
+ * Top mass minus runner-up mass, which unlike {@link confidence} separates a
+ * close second from a flat spread. A single outcome yields its own
+ * probability; empty yields 0.
+ */
+export const margin = (probabilities: ReadonlyArray<number>): number => {
+  const top = pipe(probabilities, A.sort(descending), A.take(2))
+  return pipe(
+    A.get(top, 0),
+    Option.map((first) =>
+      pipe(
+        A.get(top, 1),
+        Option.getOrElse(() => 0),
+        (second) => first - second,
+      ),
+    ),
+    Option.getOrElse(() => 0),
+  )
+}

package/src/realtime/RealtimeSession.test.ts

@@ -0,0 +1,100 @@
+import { describe, it } from "@effect/vitest"
+import { Effect, Stream } from "effect"
+import { expect, expectTypeOf } from "vitest"
+import type * as AiError from "../domain/AiError.js"
+import { imageBase64 } from "../domain/Image.js"
+import { type CommonSessionRequest, RealtimeEvent, RealtimeInput } from "../domain/Realtime.js"
+import * as MockRealtimeSession from "../testing/MockRealtimeSession.js"
+import { RealtimeVideoInput, open, sendVideoFrame, toolCalls } from "./RealtimeSession.js"
+
+const pcm = { container: "raw", encoding: "pcm_s16le", sampleRate: 24000, channels: 1 } as const
+
+const request: CommonSessionRequest = {
+  model: "gpt-realtime-2.1",
+  inputFormat: pcm,
+  outputFormat: pcm,
+}
+
+const call = {
+  type: "function_call",
+  call_id: "call_1",
+  name: "get_time",
+  arguments: "{}",
+} as const
+
+describe("RealtimeSession against a scripted session", () => {
+  it.effect("answers each input and ends `events` when the scope closes", () =>
+    Effect.gen(function* () {
+      const mock = MockRealtimeSession.layer({
+        initial: [RealtimeEvent.ResponseStarted({ responseId: "r1" })],
+        onInput: (input) =>
+          RealtimeInput.$is("Audio")(input)
+            ? [RealtimeEvent.AudioDelta({ responseId: "r1", bytes: new Uint8Array([7]) })]
+            : [RealtimeEvent.ResponseDone({ responseId: "r1", reason: "interrupted" })],
+      })
+
+      const handle = yield* Effect.scoped(
+        Effect.gen(function* () {
+          const handle = yield* open(request)
+          yield* handle.send(RealtimeInput.Audio({ bytes: new Uint8Array([1, 2]) }))
+          yield* handle.send(RealtimeInput.Interrupt())
+          return handle
+        }),
+      ).pipe(Effect.provide(mock.layer))
+
+      // The scope is closed, so the stream drains what is queued and ends
+      // rather than hanging.
+      const events = yield* Stream.runCollect(handle.events)
+      expect(events).toEqual([
+        RealtimeEvent.ResponseStarted({ responseId: "r1" }),
+        RealtimeEvent.AudioDelta({ responseId: "r1", bytes: new Uint8Array([7]) }),
+        RealtimeEvent.ResponseDone({ responseId: "r1", reason: "interrupted" }),
+      ])
+
+      const recorder = yield* mock.recorder
+      expect(recorder.openCalls.map((r) => r.model)).toEqual(["gpt-realtime-2.1"])
+      expect(recorder.sent.map((i) => i._tag)).toEqual(["Audio", "Interrupt"])
+    }),
+  )
+
+  it.effect("narrows `toolCalls` to the calls the model made", () =>
+    Effect.gen(function* () {
+      const mock = MockRealtimeSession.layer({
+        initial: [
+          RealtimeEvent.ResponseStarted({ responseId: "r1" }),
+          RealtimeEvent.ToolCall({ responseId: "r1", call }),
+          RealtimeEvent.OutputTranscriptDelta({ responseId: "r1", text: "one moment" }),
+        ],
+      })
+
+      const handle = yield* Effect.scoped(open(request)).pipe(Effect.provide(mock.layer))
+      const calls = yield* Stream.runCollect(toolCalls(handle.events))
+
+      expect(calls.map((e) => e.call.call_id)).toEqual(["call_1"])
+    }),
+  )
+})
+
+describe("RealtimeVideoInput marker", () => {
+  const frame = imageBase64("aW1n", "image/jpeg")
+
+  it("leaves the marker unsatisfied against an audio-only Layer", () => {
+    const mock = MockRealtimeSession.layerAudioOnly({})
+    const program = Effect.scoped(
+      Effect.flatMap(open(request), (handle) => sendVideoFrame(handle, frame)),
+    ).pipe(Effect.provide(mock.layer))
+
+    // `RealtimeSession` is provided, `RealtimeVideoInput` is not, so running
+    // this would be a type error.
+    expectTypeOf(program).toEqualTypeOf<Effect.Effect<void, AiError.AiError, RealtimeVideoInput>>()
+  })
+
+  it("clears R to never against a Layer that ships the marker", () => {
+    const mock = MockRealtimeSession.layer({})
+    const program = Effect.scoped(
+      Effect.flatMap(open(request), (handle) => sendVideoFrame(handle, frame)),
+    ).pipe(Effect.provide(mock.layer))
+
+    expectTypeOf(program).toEqualTypeOf<Effect.Effect<void, AiError.AiError, never>>()
+  })
+})

package/src/realtime/RealtimeSession.ts

@@ -0,0 +1,77 @@
+/**
+ * A duplex speech-to-speech session. The server drives turn-taking, so this is
+ * its own primitive rather than a variant of `Loop`.
+ *
+ * The session is deliberately low level: tool execution, playback, barge-in
+ * bookkeeping and reconnection are the caller's, built from `send` and
+ * `events`. There is no agent wrapper, no hidden queue and no automatic
+ * reconnect here.
+ */
+import { Context, Effect, type Scope, Stream } from "effect"
+import type * as AiError from "../domain/AiError.js"
+import type { ImageSource } from "../domain/Image.js"
+import { type CommonSessionRequest, RealtimeInput, RealtimeEvent } from "../domain/Realtime.js"
+
+/**
+ * The live connection. `events` ends when the socket closes cleanly; a close
+ * during a response ends it with `AiError.IncompleteTurn` instead, and never
+ * synthesizes a `ResponseDone`. `send` after close fails `Unavailable`.
+ */
+export type RealtimeSessionHandle = {
+  readonly send: (input: RealtimeInput) => Effect.Effect<void, AiError.AiError>
+  readonly events: Stream.Stream<RealtimeEvent, AiError.AiError>
+}
+
+export type RealtimeSessionService = {
+  /**
+   * Connect and complete the provider handshake before succeeding. Closing the
+   * `Scope` closes the socket.
+   */
+  readonly open: (
+    request: CommonSessionRequest,
+  ) => Effect.Effect<RealtimeSessionHandle, AiError.AiError, Scope.Scope>
+}
+
+export class RealtimeSession extends Context.Service<RealtimeSession, RealtimeSessionService>()(
+  "@betalyra/effect-uai/RealtimeSession",
+) {}
+
+/**
+ * Capability marker for sessions that accept video frames. Phantom: providers
+ * register with `Layer.succeed(RealtimeVideoInput, undefined)`. Only Gemini
+ * ships it, so `sendVideoFrame` against an OpenAI-only Layer is a type error.
+ */
+export class RealtimeVideoInput extends Context.Service<RealtimeVideoInput, void>()(
+  "@betalyra/effect-uai/capability/RealtimeVideoInput",
+) {}
+
+/** Open a session for the lifetime of the surrounding `Scope`. */
+export const open = (
+  request: CommonSessionRequest,
+): Effect.Effect<RealtimeSessionHandle, AiError.AiError, RealtimeSession | Scope.Scope> =>
+  Effect.flatMap(RealtimeSession, (s) => s.open(request))
+
+/**
+ * Push one video frame. Requires `RealtimeVideoInput`, so a provider without
+ * video fails at `Effect.provide` rather than at runtime.
+ */
+export const sendVideoFrame = (
+  handle: RealtimeSessionHandle,
+  frame: ImageSource,
+): Effect.Effect<void, AiError.AiError, RealtimeVideoInput> =>
+  Effect.flatMap(RealtimeVideoInput, () => handle.send(RealtimeInput.VideoFrame({ frame })))
+
+const isAudioDelta = RealtimeEvent.$is("AudioDelta")
+const isToolCall = RealtimeEvent.$is("ToolCall")
+
+/** Just the audio, for wiring straight to playback. */
+export const audioDeltas = <E, R>(
+  events: Stream.Stream<RealtimeEvent, E, R>,
+): Stream.Stream<Extract<RealtimeEvent, { readonly _tag: "AudioDelta" }>, E, R> =>
+  Stream.filter(events, isAudioDelta)
+
+/** Just the tool calls. Answer each with a `ToolResult`; the adapter resumes. */
+export const toolCalls = <E, R>(
+  events: Stream.Stream<RealtimeEvent, E, R>,
+): Stream.Stream<Extract<RealtimeEvent, { readonly _tag: "ToolCall" }>, E, R> =>
+  Stream.filter(events, isToolCall)