@effect-uai/core 0.6.0 → 0.8.0

package/src/domain/Music.ts

@@ -1,10 +1,11 @@
-import type { AudioBlob, AudioFormat } from "./Audio.js"
+import type { Duration } from "effect"
+import type { AudioBlob, AudioChunk, AudioFormat } from "./Audio.js"
 
 /**
  * Prompt fragment with a relative weight. Native to Lyria RealTime
- * (`{ text, weight }` pairs blended in the model). Single-prompt
- * providers (Suno, Mureka, MiniMax) flatten to text at the adapter
- * layer.
+ * (`{ text, weight }` pairs blended in the model) and Riffusion compose.
+ * Single-prompt providers (the majority) treat a 1-element array with
+ * `weight: 1` as a plain prompt.
  */
 export type WeightedPrompt = {
   readonly text: string
@@ -13,109 +14,182 @@ export type WeightedPrompt = {
 }
 
 /**
- * Cross-provider music-generation request. Provider-specific extras
- * (Lyria `mode`, ElevenLabs `composition_plan`, Suno custom-mode `title`,
- * MiniMax `lyrics_optimizer`) live on each provider's typed request
- * which extends this and narrows `model`.
+ * Cross-provider music-generation request. Trimmed to fields the
+ * majority of music providers honor structurally; provider-specific
+ * extras (Lyria `bpm` / `scale`, ElevenLabs `compositionPlan` /
+ * `signWithC2pa`, Suno `style` / `personaId`, MiniMax `lyricsOptimizer`,
+ * `instrumental` on providers that ship a wire toggle) live on each
+ * provider's typed request which extends this and narrows `model`.
+ *
+ * Music providers are one-shot text-to-audio, not conversational. No
+ * `messages[]` shape applies.
  */
 export type CommonGenerateMusicRequest = {
   /** Model identifier. Each provider narrows. */
   readonly model: string
-  /** Single prompt string or weighted-prompt list (blended where supported). */
-  readonly prompts: string | ReadonlyArray<WeightedPrompt>
+  /** Single prompt string. Weighted blends are provider-typed extras. */
+  readonly prompt: string
   /**
-   * Lyrics text, optionally with section tags like `[Verse]` / `[Chorus]` /
-   * `[Bridge]` / `[Outro]`. Ignored for instrumental-only providers or
-   * when `instrumental: true`.
+   * Lyrics text, optionally with section tags like `[Verse]` /
+   * `[Chorus]` / `[Bridge]` / `[Outro]`. Routed to a structured wire
+   * field on providers that have one (ElevenLabs `lines`, MiniMax
+   * `lyrics`, Mureka `lyrics`, Suno custom-mode `prompt`); embedded in
+   * the prompt with a warning on providers without one (Lyria 3,
+   * Riffusion).
    */
   readonly lyrics?: string
-  /** Target duration in seconds. Provider may treat as a hint or hard limit. */
-  readonly durationSeconds?: number
-  /** Beats per minute (60–200 typical). */
-  readonly bpm?: number
+  /** Target duration. Provider may treat as a hint or hard limit. */
+  readonly duration?: Duration.Duration
   /**
-   * Musical key/mode hint. Provider-specific vocabulary (e.g. Lyria
-   * RealTime uses enum values like `"C_MAJOR"`, `"A_MINOR"`).
+   * Reproducibility seed. Tuning hint (bucket 3): honored where the
+   * provider takes one (Lyria 2 Vertex, Lyria RealTime, ElevenLabs
+   * with `compositionPlan`, Stable Audio, MusicGen); silently ignored
+   * elsewhere.
    */
-  readonly scale?: string
-  /** Skip vocals / lyrics. */
-  readonly instrumental?: boolean
+  readonly seed?: number
   /** Preferred output format. Provider may override. */
   readonly outputFormat?: AudioFormat
 }
 
 /**
- * Streamed-output request. Same shape as the sync request — the
+ * Streamed-output request. Same shape as the sync request: the
  * streaming variant only differs in how the response is delivered.
  */
 export type CommonStreamGenerateMusicRequest = CommonGenerateMusicRequest
 
 /**
- * Bidirectional-session input. The user pushes one of these per
- * change: a new prompt blend, a config delta, or a playback control.
- * Lyria RealTime is the only provider currently surfacing these.
+ * Playback-control vocabulary that converges across interactive-media
+ * protocols (Web Audio, MIDI sequencer control, music SDK surfaces).
+ * Provider-specific control actions (Lyria's `reset_context` and any
+ * future provider's extras) live on the provider-typed session-input
+ * union.
+ */
+export type MusicSessionControl = "play" | "pause" | "stop" | "reset"
+
+/**
+ * Bidirectional-session input on the cross-provider surface. Two
+ * load-bearing actions every interactive music session needs:
+ *
+ * - `prompts` — steer the active generation with a (single or
+ *   weighted) prompt blend.
+ * - `control` — play / pause / stop / reset.
+ *
+ * Provider-typed services extend this union with their own `config`
+ * variant for model-specific knobs (Lyria RealTime exposes density /
+ * brightness / mute-stems / BPM / scale / etc. via its own
+ * `LyriaRealtimeSessionInput`).
  */
 export type MusicSessionInput =
   | { readonly _tag: "prompts"; readonly prompts: ReadonlyArray<WeightedPrompt> }
-  | {
-      readonly _tag: "config"
-      readonly config: {
-        readonly bpm?: number
-        readonly scale?: string
-        readonly density?: number
-        readonly brightness?: number
-        readonly guidance?: number
-        readonly temperature?: number
-        readonly topK?: number
-        readonly seed?: number
-        readonly muteBass?: boolean
-        readonly muteDrums?: boolean
-        readonly onlyBassAndDrums?: boolean
-      }
-    }
-  | { readonly _tag: "control"; readonly action: "play" | "pause" | "stop" | "reset_context" }
+  | { readonly _tag: "control"; readonly action: MusicSessionControl }
 
 export const promptsInput = (prompts: ReadonlyArray<WeightedPrompt>): MusicSessionInput => ({
   _tag: "prompts",
   prompts,
 })
 
-export const configInput = (
-  config: (MusicSessionInput & { _tag: "config" })["config"],
-): MusicSessionInput => ({ _tag: "config", config })
+export const controlInput = (action: MusicSessionControl): MusicSessionInput => ({
+  _tag: "control",
+  action,
+})
+
+/**
+ * Watermark kind on the response. SynthID is mandatory on Lyria; C2PA
+ * is opt-in on ElevenLabs. No provider exposes additional metadata
+ * about the watermark itself, so a bare literal-union is enough.
+ */
+export type Watermark = "synthid" | "c2pa" | (string & {})
 
-export const controlInput = (
-  action: (MusicSessionInput & { _tag: "control" })["action"],
-): MusicSessionInput => ({ _tag: "control", action })
+/**
+ * Labelled section with absolute timing. Populated when the provider
+ * returns structured section metadata (Lyria text part, ElevenLabs
+ * `/v1/music/detailed` with `with_timestamps`).
+ */
+export type MusicSection = {
+  readonly label: string
+  readonly startSeconds: number
+  readonly endSeconds: number
+}
 
 /**
- * Sync-generation result. Extends `AudioBlob` with provider-side
- * metadata that's common across music providers:
- *
- * - `songId` — Suno task id, ElevenLabs `song_id`, etc. Used for
- *   back-reference (re-download, stem export, follow-up edits).
- * - `lyrics` — generated lyrics when the model returned them (Lyria
- *   text part, Mureka, Suno).
- * - `sections` — structured section markers (Lyria optional JSON
- *   structure response).
- * - `watermark` — presence marker (Lyria SynthID is always set).
+ * Single generated track. Composes `AudioBlob` rather than extending
+ * it — `result.audio` is the raw blob, `result.songId` / `lyrics` /
+ * `sections` / `watermark` are provider-side metadata when returned.
  */
-export type MusicResult = AudioBlob & {
+export type MusicResult = {
+  readonly audio: AudioBlob
+  /**
+   * Provider identifier, useful when routing through multiple
+   * providers in a single pipeline (`"lyria"`, `"elevenlabs-music"`,
+   * etc.).
+   */
+  readonly provider?: string
+  /** Provider-side ID for back-reference (re-download, extension, stems). */
   readonly songId?: string
+  /** Generated or transcribed lyrics when the provider returned them. */
   readonly lyrics?: string
-  readonly sections?: ReadonlyArray<{
-    readonly label: string
-    readonly startSeconds: number
-    readonly endSeconds: number
-  }>
-  readonly watermark?: { readonly kind: string }
+  /** Structured section markers when the provider returned them. */
+  readonly sections?: ReadonlyArray<MusicSection>
+  /** Watermark presence indicator when the provider applied one. */
+  readonly watermark?: Watermark
 }
 
+/**
+ * Result of a `generate` call. Suno and Mureka always return 2 tracks
+ * per request; every other provider returns 1. `primary` is the
+ * caller-friendly convenience; `variants` carries every track the
+ * provider produced (length ≥ 1; `primary === variants[0]`).
+ */
+export type GenerateResult = {
+  readonly primary: MusicResult
+  readonly variants: ReadonlyArray<MusicResult>
+}
+
+/**
+ * In-band event on the bidirectional `streamGenerationFrom` output
+ * stream. Audio chunks flow as `audio`; server-side warnings
+ * (`warning`) and prompt-filter rejections (`filteredPrompt`) flow
+ * alongside rather than going to a side-channel log.
+ */
+export type MusicStreamEvent =
+  | { readonly _tag: "audio"; readonly chunk: AudioChunk }
+  | { readonly _tag: "warning"; readonly message: string }
+  | { readonly _tag: "filteredPrompt"; readonly prompt: string; readonly reason: string }
+
 export const isPromptsInput = (
   i: MusicSessionInput,
 ): i is MusicSessionInput & { _tag: "prompts" } => i._tag === "prompts"
-export const isConfigInput = (i: MusicSessionInput): i is MusicSessionInput & { _tag: "config" } =>
-  i._tag === "config"
 export const isControlInput = (
   i: MusicSessionInput,
 ): i is MusicSessionInput & { _tag: "control" } => i._tag === "control"
+
+export const audioEvent = (chunk: AudioChunk): MusicStreamEvent => ({ _tag: "audio", chunk })
+export const warningEvent = (message: string): MusicStreamEvent => ({
+  _tag: "warning",
+  message,
+})
+export const filteredPromptEvent = (prompt: string, reason: string): MusicStreamEvent => ({
+  _tag: "filteredPrompt",
+  prompt,
+  reason,
+})
+
+export const isAudioEvent = (e: MusicStreamEvent): e is MusicStreamEvent & { _tag: "audio" } =>
+  e._tag === "audio"
+export const isWarningEvent = (e: MusicStreamEvent): e is MusicStreamEvent & { _tag: "warning" } =>
+  e._tag === "warning"
+export const isFilteredPromptEvent = (
+  e: MusicStreamEvent,
+): e is MusicStreamEvent & { _tag: "filteredPrompt" } => e._tag === "filteredPrompt"
+
+/**
+ * Build a single-variant `GenerateResult` from a `MusicResult`.
+ * Convenience for providers that always return exactly one track
+ * (Lyria, ElevenLabs, MiniMax, Stable Audio, MusicGen, Riffusion,
+ * Tencent). Providers that return multiple tracks (Suno, Mureka)
+ * construct the `GenerateResult` directly.
+ */
+export const singleVariant = (result: MusicResult): GenerateResult => ({
+  primary: result,
+  variants: [result],
+})

package/src/domain/Transcript.ts

@@ -1,7 +1,12 @@
+import type { Duration } from "effect"
+
 /**
  * Per-word timing + metadata. `confidence` and `speakerId` are optional
  * because providers vary widely in what they emit and when (some only on
  * final, some only with diarization enabled, some not at all).
+ *
+ * `startSeconds` / `endSeconds` stay raw `number` offsets (not `Duration`)
+ * — they're positions in the audio, not durations.
  */
 export type WordTimestamp = {
   readonly text: string
@@ -20,7 +25,8 @@ export type WordTimestamp = {
 export type TranscriptResult = {
   readonly text: string
   readonly languageCode?: string
-  readonly durationSeconds?: number
+  /** Total audio duration. (Word offsets stay raw seconds on `WordTimestamp`.) */
+  readonly duration?: Duration.Duration
   readonly words?: ReadonlyArray<WordTimestamp>
   readonly raw?: unknown
 }

package/src/embedding-model/Embedding.ts

@@ -107,6 +107,17 @@ export const isBinary = (e: Embedding): e is BinaryEmbedding => e._tag === "bina
 export const isSparse = (e: Embedding): e is SparseEmbedding => e._tag === "sparse"
 export const isMultivector = (e: Embedding): e is MultivectorEmbedding => e._tag === "multivector"
 
+/**
+ * Domain of `EmbeddingFor` and the response type parameter on
+ * `EmbedResponse` / `EmbedManyResponse`: every encoding a provider can
+ * *return*, including Jina's widened `sparse` / `multivector`. Distinct
+ * from `EmbedEncoding` (the cross-provider *request* set, which is dense
+ * quantization only). Providers whose typed request widens beyond the
+ * dense set (Jina) parameterize their responses with their own encoding
+ * union, which is a subset of this.
+ */
+export type ResponseEncoding = "float32" | "int8" | "binary" | "sparse" | "multivector"
+
 /**
  * Maps an `encoding` request field to the corresponding response embedding
  * variant. `undefined` (no encoding requested) defaults to `Float32Embedding`,

package/src/embedding-model/EmbeddingModel.test.ts

@@ -27,7 +27,7 @@ describe("EmbedResponse<E> conditional narrowing", () => {
     }>()
   })
 
-  it("maps each EmbedEncoding literal to the matching Embedding variant", () => {
+  it("maps each encoding literal to the matching Embedding variant", () => {
     expectTypeOf<EmbedResponse<"float32">["embedding"]>().toEqualTypeOf<Float32Embedding>()
     expectTypeOf<EmbedResponse<"int8">["embedding"]>().toEqualTypeOf<Int8Embedding>()
     expectTypeOf<EmbedResponse<"binary">["embedding"]>().toEqualTypeOf<BinaryEmbedding>()
@@ -69,13 +69,24 @@ describe("embed / embedMany free exports preserve E in their return type", () =>
     >()
   })
 
-  it("embedMany with encoding: multivector returns EmbedManyResponse<multivector>", () => {
-    const result = embedMany({ inputs: ["x"], model: "m", encoding: "multivector" })
+  it("embedMany with encoding: binary returns EmbedManyResponse<binary>", () => {
+    const result = embedMany({ inputs: ["x"], model: "m", encoding: "binary" })
     expectTypeOf(result).toEqualTypeOf<
-      Effect.Effect<EmbedManyResponse<"multivector">, AiError.AiError, EmbeddingModel>
+      Effect.Effect<EmbedManyResponse<"binary">, AiError.AiError, EmbeddingModel>
     >()
   })
 
+  it("rejects non-dense encodings on the generic path (trimmed EmbedEncoding)", () => {
+    // `sparse` / `multivector` are Jina-only representation kinds, not part
+    // of the cross-provider request set. Requesting them on the generic
+    // helper is a compile error; they're reachable only via the typed
+    // `JinaEmbedding` tag.
+    // @ts-expect-error "multivector" is not an EmbedEncoding
+    embedMany({ inputs: ["x"], model: "m", encoding: "multivector" })
+    // @ts-expect-error "sparse" is not an EmbedEncoding
+    embed({ input: "x", model: "m", encoding: "sparse" })
+  })
+
   it("when the request is widened to CommonEmbedRequest, the response is the open union", () => {
     // Demonstrates the documented case: annotating the request type erases
     // the literal `encoding` and the response falls back to `Embedding`.

package/src/embedding-model/EmbeddingModel.ts

@@ -1,29 +1,29 @@
 import { Context, Effect } from "effect"
 import * as AiError from "../domain/AiError.js"
-import type { EmbeddingFor, EmbedInput, Usage } from "./Embedding.js"
+import type { EmbeddingFor, EmbedInput, ResponseEncoding, Usage } from "./Embedding.js"
 
 /**
- * Output representation requested from the provider.
+ * Output representation requested from the provider. Cross-provider dense
+ * quantization only, the same vector at different storage cost:
+ * - `float32`: universal default.
+ * - `int8`: ~4x smaller, minimal recall loss on most benchmarks. Supported
+ *   by Cohere, Voyage, Jina (binary only), Mixedbread.
+ * - `binary`: ~32x smaller, meaningful recall loss but pairs well with a
+ *   float32 reranker pass over a small candidate set.
  *
- * Dense quantizations - same vector at different storage cost:
- * - `float32` — universal default.
- * - `int8` — ~4x smaller; minimal recall loss on most benchmarks.
- * - `binary` — ~32x smaller; meaningful recall loss but pairs well with
- *   a float32 reranker pass over a small candidate set.
+ * Non-dense representations (`sparse`, `multivector`) are a different vector
+ * *structure*, not a storage option, and are produced by Jina only. They
+ * live on `JinaEncoding`, not here (the same widening pattern `task` uses).
+ * The response `Embedding` union still carries all arms, see
+ * {@link ResponseEncoding}.
  *
- * Non-dense representations:
- * - `sparse` — learned sparse vector for hybrid (dense + lexical) search.
- *   Currently Jina ELSER only on hosted APIs.
- * - `multivector` — one vector per token for late-interaction (ColBERT-
- *   style) scoring via `Vector.maxSim`. Currently Jina v4 only.
- *
- * Each provider's typed request narrows this to its supported set at
- * compile time (e.g. `JinaEmbedEncoding = "float32" | "binary" | "sparse" |
- * "multivector"`). On the generic `EmbeddingModel` path, callers can
- * pass any `EmbedEncoding` and the provider's API will reject mismatches at
- * runtime.
+ * Each provider's typed request narrows this to its supported set. On the
+ * generic `EmbeddingModel` path, a provider that emits float32 only (OpenAI,
+ * Gemini) rejects a non-float32 encoding with `AiError.Unsupported`
+ * (bucket 1) via {@link assertEncoding}, rather than silently returning a
+ * mislabeled float32 vector.
  */
-export type EmbedEncoding = "float32" | "int8" | "binary" | "sparse" | "multivector"
+export type EmbedEncoding = "float32" | "int8" | "binary"
 
 /**
  * Cross-provider single-embed request. Mirrors the shape of
@@ -80,13 +80,13 @@ export type CommonEmbedManyRequest = Omit<CommonEmbedRequest, "input"> & {
  * narrowing. Defaults to `undefined` for back-compat with consumers that
  * use the bare `EmbedResponse` name.
  */
-export type EmbedResponse<E extends EmbedEncoding | undefined = undefined> = {
+export type EmbedResponse<E extends ResponseEncoding | undefined = undefined> = {
   readonly embedding: EmbeddingFor<E>
   readonly usage: Usage
 }
 
 /** Batch-embed response. Same encoding rule as {@link EmbedResponse}. */
-export type EmbedManyResponse<E extends EmbedEncoding | undefined = undefined> = {
+export type EmbedManyResponse<E extends ResponseEncoding | undefined = undefined> = {
   readonly embeddings: ReadonlyArray<EmbeddingFor<E>>
   readonly usage: Usage
 }
@@ -115,3 +115,30 @@ export const embedMany = <E extends EmbedEncoding | undefined = undefined>(
   request: Omit<CommonEmbedManyRequest, "encoding"> & { readonly encoding?: E },
 ): Effect.Effect<EmbedManyResponse<E>, AiError.AiError, EmbeddingModel> =>
   Effect.flatMap(EmbeddingModel.asEffect(), (m) => m.embedMany(request))
+
+/**
+ * Guard a requested `encoding` against a provider's supported set. Returns
+ * `void` when the encoding is unset or supported, and fails
+ * `AiError.Unsupported` (bucket 1) otherwise.
+ *
+ * Used by the generic-tag registration of providers that emit a strict
+ * subset of `EmbedEncoding`: OpenAI / Gemini pass `["float32"]`, Jina passes
+ * `["float32", "binary"]`. Without this guard, a non-supported encoding on
+ * the generic path silently returns a float32 vector mislabeled as the
+ * requested type (the `as`-cast lie), which breaks downstream storage and
+ * vector math.
+ */
+export const assertEncoding = (
+  encoding: EmbedEncoding | undefined,
+  supported: ReadonlyArray<EmbedEncoding>,
+  provider: string,
+): Effect.Effect<void, AiError.AiError> =>
+  encoding === undefined || supported.includes(encoding)
+    ? Effect.void
+    : Effect.fail(
+        new AiError.Unsupported({
+          provider,
+          capability: "encoding",
+          reason: `${provider} emits ${supported.join(" / ")} only; encoding="${encoding}" is unavailable.`,
+        }),
+      )

package/src/index.ts

@@ -1,5 +1,6 @@
 export * as AiError from "./domain/AiError.js"
 export * as Audio from "./domain/Audio.js"
+export * as Capabilities from "./capabilities/Capabilities.js"
 export * as Image from "./domain/Image.js"
 export * as Items from "./domain/Items.js"
 export * as Media from "./domain/Media.js"
@@ -17,6 +18,8 @@ export * as SandboxImage from "./sandbox/SandboxImage.js"
 export * as SandboxNetwork from "./sandbox/SandboxNetwork.js"
 export * as SpeechSynthesizer from "./speech-synthesizer/SpeechSynthesizer.js"
 export * as Transcriber from "./transcriber/Transcriber.js"
+export * as WebSearch from "./web-search/WebSearch.js"
+export * as WebSearchTool from "./web-search/WebSearchTool.js"
 export * as Vector from "./math/Vector.js"
 export * as Loop from "./loop/Loop.js"
 export * as Retry from "./retry/Retry.js"