@effect-uai/core 0.2.0 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effect-uai/core",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Low-level primitives (loop, conversation, items, tools, streaming codecs) for building AI agents with Effect.",
   "keywords": [
     "agents",
@@ -20,6 +20,12 @@
     "url": "https://github.com/betalyra/effect-uai",
     "directory": "packages/core"
   },
+  "files": [
+    "dist",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
   "type": "module",
   "main": "./dist/index.mjs",
   "types": "./dist/index.d.mts",
@@ -32,25 +38,65 @@
       "types": "./dist/domain/AiError.d.mts",
       "import": "./dist/domain/AiError.mjs"
     },
+    "./Audio": {
+      "types": "./dist/domain/Audio.d.mts",
+      "import": "./dist/domain/Audio.mjs"
+    },
+    "./Image": {
+      "types": "./dist/domain/Image.d.mts",
+      "import": "./dist/domain/Image.mjs"
+    },
     "./Items": {
       "types": "./dist/domain/Items.d.mts",
       "import": "./dist/domain/Items.mjs"
     },
+    "./Media": {
+      "types": "./dist/domain/Media.d.mts",
+      "import": "./dist/domain/Media.mjs"
+    },
+    "./Music": {
+      "types": "./dist/domain/Music.d.mts",
+      "import": "./dist/domain/Music.mjs"
+    },
+    "./Transcript": {
+      "types": "./dist/domain/Transcript.d.mts",
+      "import": "./dist/domain/Transcript.mjs"
+    },
     "./Turn": {
       "types": "./dist/domain/Turn.d.mts",
       "import": "./dist/domain/Turn.mjs"
     },
+    "./Embedding": {
+      "types": "./dist/embedding-model/Embedding.d.mts",
+      "import": "./dist/embedding-model/Embedding.mjs"
+    },
+    "./EmbeddingModel": {
+      "types": "./dist/embedding-model/EmbeddingModel.d.mts",
+      "import": "./dist/embedding-model/EmbeddingModel.mjs"
+    },
     "./LanguageModel": {
       "types": "./dist/language-model/LanguageModel.d.mts",
       "import": "./dist/language-model/LanguageModel.mjs"
     },
+    "./MusicGenerator": {
+      "types": "./dist/music-generator/MusicGenerator.d.mts",
+      "import": "./dist/music-generator/MusicGenerator.mjs"
+    },
+    "./SpeechSynthesizer": {
+      "types": "./dist/speech-synthesizer/SpeechSynthesizer.d.mts",
+      "import": "./dist/speech-synthesizer/SpeechSynthesizer.mjs"
+    },
+    "./Transcriber": {
+      "types": "./dist/transcriber/Transcriber.d.mts",
+      "import": "./dist/transcriber/Transcriber.mjs"
+    },
     "./Loop": {
       "types": "./dist/loop/Loop.d.mts",
       "import": "./dist/loop/Loop.mjs"
     },
-    "./Match": {
-      "types": "./dist/match/Match.d.mts",
-      "import": "./dist/match/Match.mjs"
+    "./Vector": {
+      "types": "./dist/math/Vector.d.mts",
+      "import": "./dist/math/Vector.mjs"
     },
     "./Tool": {
       "types": "./dist/tool/Tool.d.mts",
@@ -99,27 +145,33 @@
     "./testing/MockProvider": {
       "types": "./dist/testing/MockProvider.d.mts",
       "import": "./dist/testing/MockProvider.mjs"
+    },
+    "./testing/MockMusicGenerator": {
+      "types": "./dist/testing/MockMusicGenerator.d.mts",
+      "import": "./dist/testing/MockMusicGenerator.mjs"
+    },
+    "./testing/MockSpeechSynthesizer": {
+      "types": "./dist/testing/MockSpeechSynthesizer.d.mts",
+      "import": "./dist/testing/MockSpeechSynthesizer.mjs"
+    },
+    "./testing/MockTranscriber": {
+      "types": "./dist/testing/MockTranscriber.d.mts",
+      "import": "./dist/testing/MockTranscriber.mjs"
     }
   },
-  "files": [
-    "dist",
-    "src",
-    "README.md",
-    "LICENSE"
-  ],
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
     "@standard-schema/spec": "^1.1.0"
   },
-  "peerDependencies": {
-    "effect": "4.0.0-beta.57"
-  },
   "devDependencies": {
     "effect": "4.0.0-beta.57",
     "typescript": "^6.0.3"
   },
+  "peerDependencies": {
+    "effect": "4.0.0-beta.57"
+  },
   "scripts": {
     "build": "tsdown",
     "typecheck": "tsc --noEmit"

package/src/domain/AiError.ts

@@ -80,6 +80,26 @@ export class IncompleteTurn extends Data.TaggedError("IncompleteTurn")<{
   raw?: unknown
 }> {}
 
+/**
+ * The provider does not implement the requested capability for this
+ * specific request. Distinct from `InvalidRequest` (the request shape is
+ * malformed) and `AuthFailed` (the request was rejected).
+ *
+ * Reserved for request-data-dependent gaps where the provider supports
+ * the method in general but not for these inputs — e.g. Google's
+ * `streamSynthesisFrom` works only for Chirp 3 HD voices; calling it
+ * with a Neural2 voice ID fails `Unsupported`.
+ *
+ * Blanket provider-level gaps (e.g. OpenAI has no incremental-text-in
+ * TTS at all) are gated at compile time via capability marker tags
+ * (`TtsIncrementalText`, `SttStreaming`) on the R channel instead.
+ */
+export class Unsupported extends Data.TaggedError("Unsupported")<{
+  provider: string
+  capability: string
+  reason?: string
+}> {}
+
 export type AiError =
   | RateLimited
   | Unavailable
@@ -91,3 +111,4 @@ export type AiError =
   | Cancelled
   | IncompleteTurn
   | GenerationFailed
+  | Unsupported

package/src/domain/Audio.ts

@@ -0,0 +1,88 @@
+import type { MediaBase64, MediaBytes, MediaUrl } from "./Media.js"
+
+/**
+ * MIME types we care about across STT input and TTS output. Container-
+ * level only — sample rate / encoding flavours live on `AudioFormat`.
+ *
+ * Per-provider request types narrow this further. The `(string & {})`
+ * tail keeps autocomplete on the literals while still accepting any
+ * string, so unusual formats work without an SDK update.
+ */
+export type AudioMimeType =
+  | "audio/mpeg"
+  | "audio/wav"
+  | "audio/x-wav"
+  | "audio/ogg"
+  | "audio/opus"
+  | "audio/flac"
+  | "audio/aac"
+  | "audio/mp4"
+  | "audio/webm"
+  | "audio/L16"
+  | "audio/pcm"
+  | "audio/mulaw"
+  | "audio/alaw"
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  | (string & {})
+
+/**
+ * Audio at rest — instantiates `MediaSource` with the audio MIME union.
+ * Used for sync STT input.
+ *
+ * URL variant is best-effort: some providers (OpenAI, Cartesia, Azure
+ * short-audio) reject URL ingestion and the adapter must upload via the
+ * `bytes` or `base64` variant instead. Adapter layers reject unsupported
+ * shapes up front with `AiError.InvalidRequest`.
+ */
+export type AudioSource =
+  | MediaUrl<AudioMimeType>
+  | MediaBase64<AudioMimeType>
+  | MediaBytes<AudioMimeType>
+
+export const isAudioUrl = (s: AudioSource): s is MediaUrl<AudioMimeType> => s._tag === "url"
+export const isAudioBase64 = (s: AudioSource): s is MediaBase64<AudioMimeType> =>
+  s._tag === "base64"
+export const isAudioBytes = (s: AudioSource): s is MediaBytes<AudioMimeType> => s._tag === "bytes"
+
+/**
+ * Structural audio format. Used both as TTS output spec and as STT
+ * streaming-input declaration. Providers that use compound slugs
+ * (`mp3_44100_128`, `audio-16khz-128kbitrate-mono-mp3`,
+ * `aura-2-thalia-en`) are encoded at the adapter layer.
+ */
+export type AudioFormat = {
+  readonly container: "mp3" | "wav" | "ogg" | "opus" | "flac" | "aac" | "webm" | "raw"
+  readonly encoding:
+    | "pcm_s16le"
+    | "pcm_f32le"
+    | "pcm_mulaw"
+    | "pcm_alaw"
+    | "mp3"
+    | "opus"
+    | "vorbis"
+    | "flac"
+    | "aac"
+  readonly sampleRate: 8000 | 16000 | 22050 | 24000 | 32000 | 44100 | 48000
+  /** mp3 / opus only. */
+  readonly bitRate?: number
+  readonly channels?: 1 | 2
+}
+
+/**
+ * Streamed audio chunk. `bytes` carries the codec-encoded payload as
+ * declared on the stream's `AudioFormat`. No per-chunk timestamp here —
+ * providers that emit timing do so via `TranscriptEvent.words[]`.
+ */
+export type AudioChunk = {
+  readonly bytes: Uint8Array
+}
+
+/**
+ * Full audio result for sync TTS. Format mirrors the request; provider
+ * layers normalize.
+ */
+export type AudioBlob = {
+  readonly format: AudioFormat
+  readonly bytes: Uint8Array
+  readonly durationSeconds?: number
+}

package/src/domain/Image.ts

@@ -0,0 +1,75 @@
+import { Schema } from "effect"
+import type { MediaBase64, MediaBytes, MediaSource, MediaUrl } from "./Media.js"
+
+/**
+ * Image MIME types AI providers typically accept. The first four are the
+ * universal subset (Cohere v4, Voyage multimodal, Jina v4, Google
+ * `gemini-embedding-2`); HEIC / HEIF are Google-specific. The
+ * `(string & {})` tail keeps autocomplete on the literals while still
+ * accepting any string, so a newly-supported format works without an
+ * SDK update.
+ */
+export type ImageMimeType =
+  | "image/png"
+  | "image/jpeg"
+  | "image/webp"
+  | "image/gif"
+  | "image/heic"
+  | "image/heif"
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  | (string & {})
+
+const ImageMimeTypeSchema = Schema.String as unknown as Schema.Schema<ImageMimeType>
+
+export type ImageUrlSource = MediaUrl<ImageMimeType>
+export type ImageBase64Source = MediaBase64<ImageMimeType>
+export type ImageBytesSource = MediaBytes<ImageMimeType>
+
+/**
+ * Where an image lives. Provider layers normalize across these:
+ * `bytes` becomes a base64 data URI for OpenAI / Anthropic, an
+ * `inlineData` part for Gemini, and a separate field for Cohere /
+ * Voyage. URL constraints (must be HTTPS, must be public, …) are
+ * provider-specific and validated at the layer, not in the type.
+ */
+export type ImageSource = MediaSource<ImageMimeType>
+
+export const ImageUrlSource = Schema.TaggedStruct("url", {
+  url: Schema.String,
+  mimeType: Schema.optional(ImageMimeTypeSchema),
+})
+
+export const ImageBase64Source = Schema.TaggedStruct("base64", {
+  base64: Schema.String,
+  mimeType: ImageMimeTypeSchema,
+})
+
+export const ImageBytesSource = Schema.TaggedStruct("bytes", {
+  bytes: Schema.Uint8Array,
+  mimeType: ImageMimeTypeSchema,
+})
+
+export const ImageSource: Schema.Schema<ImageSource> = Schema.Union([
+  ImageUrlSource,
+  ImageBase64Source,
+  ImageBytesSource,
+]) as unknown as Schema.Schema<ImageSource>
+
+export const imageUrl = (url: string, mimeType?: ImageMimeType): ImageUrlSource =>
+  mimeType !== undefined ? { _tag: "url", url, mimeType } : { _tag: "url", url }
+
+export const imageBase64 = (base64: string, mimeType: ImageMimeType): ImageBase64Source => ({
+  _tag: "base64",
+  base64,
+  mimeType,
+})
+
+export const imageBytes = (bytes: Uint8Array, mimeType: ImageMimeType): ImageBytesSource => ({
+  _tag: "bytes",
+  bytes,
+  mimeType,
+})
+
+export const isImageUrl = Schema.is(ImageUrlSource)
+export const isImageBase64 = Schema.is(ImageBase64Source)
+export const isImageBytes = Schema.is(ImageBytesSource)

package/src/domain/Items.ts

@@ -1,4 +1,5 @@
 import { Schema } from "effect"
+import { ImageSource } from "./Image.js"
 
 // ---------------------------------------------------------------------------
 // Content blocks (inside Message.content)
@@ -10,39 +11,13 @@ export const InputText = Schema.Struct({
 })
 export type InputText = typeof InputText.Type
 
-/**
- * Where an image lives. `url` covers HTTP(S) URLs (the model fetches
- * them); `base64` covers inline bytes embedded in the request. Provider
- * encoders dispatch on `_tag`. File-id / uploaded-asset references are
- * provider-specific and stay out of this union for now.
- */
-export const ImageUrlSource = Schema.Struct({
-  _tag: Schema.Literal("url"),
-  url: Schema.String,
-})
-export type ImageUrlSource = typeof ImageUrlSource.Type
-
-/**
- * Inline image bytes. `data` is **already base64-encoded** (matches what
- * the wire formats expect; no double-encoding needed downstream).
- * `media_type` is the MIME type, e.g. `"image/png"`.
- */
-export const ImageBase64Source = Schema.Struct({
-  _tag: Schema.Literal("base64"),
-  media_type: Schema.String,
-  data: Schema.String,
-})
-export type ImageBase64Source = typeof ImageBase64Source.Type
-
-export const ImageSource = Schema.Union([ImageUrlSource, ImageBase64Source])
-export type ImageSource = typeof ImageSource.Type
-
-export const isImageUrlSource = (s: ImageSource): s is ImageUrlSource => s._tag === "url"
-export const isImageBase64Source = (s: ImageSource): s is ImageBase64Source => s._tag === "base64"
-
 /**
  * User-provided image content block. Pair with `InputText` inside a
  * `Message.content` array to ask "what's in this image?" style questions.
+ *
+ * `source` is the cross-modality `ImageSource` from `domain/Image.ts` -
+ * url, base64, or raw bytes. Provider codecs encode bytes to whatever
+ * wire format the provider wants.
  */
 export const InputImage = Schema.Struct({
   type: Schema.Literal("input_image"),
@@ -91,11 +66,10 @@ export type FilePath = typeof FilePath.Type
 export const Annotation = Schema.Union([UrlCitation, FileCitation, ContainerFileCitation, FilePath])
 export type Annotation = typeof Annotation.Type
 
-export const isUrlCitation = (a: Annotation): a is UrlCitation => a.type === "url_citation"
-export const isFileCitation = (a: Annotation): a is FileCitation => a.type === "file_citation"
-export const isContainerFileCitation = (a: Annotation): a is ContainerFileCitation =>
-  a.type === "container_file_citation"
-export const isFilePath = (a: Annotation): a is FilePath => a.type === "file_path"
+export const isUrlCitation = Schema.is(UrlCitation)
+export const isFileCitation = Schema.is(FileCitation)
+export const isContainerFileCitation = Schema.is(ContainerFileCitation)
+export const isFilePath = Schema.is(FilePath)
 
 export const OutputText = Schema.Struct({
   type: Schema.Literal("output_text"),
@@ -183,18 +157,15 @@ export type Item = typeof Item.Type
 // Type guards
 // ---------------------------------------------------------------------------
 
-export const isInputText = (block: ContentBlock): block is InputText => block.type === "input_text"
-export const isInputImage = (block: ContentBlock): block is InputImage =>
-  block.type === "input_image"
-export const isOutputText = (block: ContentBlock): block is OutputText =>
-  block.type === "output_text"
-export const isRefusal = (block: ContentBlock): block is Refusal => block.type === "refusal"
-
-export const isMessage = (item: Item): item is Message => item.type === "message"
-export const isFunctionCall = (item: Item): item is FunctionCall => item.type === "function_call"
-export const isFunctionCallOutput = (item: Item): item is FunctionCallOutput =>
-  item.type === "function_call_output"
-export const isReasoning = (item: Item): item is Reasoning => item.type === "reasoning"
+export const isInputText = Schema.is(InputText)
+export const isInputImage = Schema.is(InputImage)
+export const isOutputText = Schema.is(OutputText)
+export const isRefusal = Schema.is(Refusal)
+
+export const isMessage = Schema.is(Message)
+export const isFunctionCall = Schema.is(FunctionCall)
+export const isFunctionCallOutput = Schema.is(FunctionCallOutput)
+export const isReasoning = Schema.is(Reasoning)
 
 // ---------------------------------------------------------------------------
 // Usage and stop reason

package/src/domain/Media.ts

@@ -0,0 +1,61 @@
+/**
+ * Cross-modality media reference shape.
+ *
+ * Every "media at rest" reference - image, audio, video, document - is one
+ * of three variants:
+ *
+ *   - `url`    : a remote address (HTTP, GCS, etc.). The model fetches it.
+ *               `mimeType` is optional - servers usually set Content-Type.
+ *               Some providers (Gemini `fileData`) want it explicit.
+ *
+ *   - `base64` : an inline base64-encoded payload. Always carries a
+ *               `mimeType` so the consumer knows how to decode.
+ *
+ *   - `bytes`  : raw `Uint8Array`. Provider layers normalize to base64 or
+ *               multipart upload at the wire boundary - users don't need
+ *               to encode themselves.
+ *
+ * Per-modality files (`Image.ts`, future `Audio.ts` / `Video.ts` /
+ * `Document.ts`) instantiate this shape with their typed MIME union to
+ * get autocomplete on common formats while keeping the structural type
+ * uniform across modalities.
+ *
+ * Streaming media (live mic feed, streaming TTS playback) is *not*
+ * modeled here. Streams carry effect parameters (`Stream<A, E, R>`) and
+ * lifecycle (Scope, cancellation) that don't apply to media at rest. The
+ * complementary type lives alongside this one as `*Stream` in each
+ * per-modality file when those modalities land.
+ *
+ * Provider-uploaded asset references (OpenAI Files `file_id`, Gemini
+ * Files API URIs, Anthropic file IDs) are also out of scope here -
+ * they're a separate union (`FileRef`) added when needed.
+ */
+
+export type MediaUrl<M extends string = string> = {
+  readonly _tag: "url"
+  readonly url: string
+  readonly mimeType?: M
+}
+
+export type MediaBase64<M extends string = string> = {
+  readonly _tag: "base64"
+  readonly base64: string
+  readonly mimeType: M
+}
+
+export type MediaBytes<M extends string = string> = {
+  readonly _tag: "bytes"
+  readonly bytes: Uint8Array
+  readonly mimeType: M
+}
+
+export type MediaSource<M extends string = string> = MediaUrl<M> | MediaBase64<M> | MediaBytes<M>
+
+export const isMediaUrl = <M extends string>(s: MediaSource<M>): s is MediaUrl<M> =>
+  s._tag === "url"
+
+export const isMediaBase64 = <M extends string>(s: MediaSource<M>): s is MediaBase64<M> =>
+  s._tag === "base64"
+
+export const isMediaBytes = <M extends string>(s: MediaSource<M>): s is MediaBytes<M> =>
+  s._tag === "bytes"

package/src/domain/Music.ts

@@ -0,0 +1,121 @@
+import type { AudioBlob, AudioFormat } from "./Audio.js"
+
+/**
+ * Prompt fragment with a relative weight. Native to Lyria RealTime
+ * (`{ text, weight }` pairs blended in the model). Single-prompt
+ * providers (Suno, Mureka, MiniMax) flatten to text at the adapter
+ * layer.
+ */
+export type WeightedPrompt = {
+  readonly text: string
+  /** Default `1.0`. Range typically `[0, 1]`; provider-dependent. */
+  readonly weight?: number
+}
+
+/**
+ * Cross-provider music-generation request. Provider-specific extras
+ * (Lyria `mode`, ElevenLabs `composition_plan`, Suno custom-mode `title`,
+ * MiniMax `lyrics_optimizer`) live on each provider's typed request
+ * which extends this and narrows `model`.
+ */
+export type CommonGenerateMusicRequest = {
+  /** Model identifier. Each provider narrows. */
+  readonly model: string
+  /** Single prompt string or weighted-prompt list (blended where supported). */
+  readonly prompts: string | ReadonlyArray<WeightedPrompt>
+  /**
+   * Lyrics text, optionally with section tags like `[Verse]` / `[Chorus]` /
+   * `[Bridge]` / `[Outro]`. Ignored for instrumental-only providers or
+   * when `instrumental: true`.
+   */
+  readonly lyrics?: string
+  /** Target duration in seconds. Provider may treat as a hint or hard limit. */
+  readonly durationSeconds?: number
+  /** Beats per minute (60–200 typical). */
+  readonly bpm?: number
+  /**
+   * Musical key/mode hint. Provider-specific vocabulary (e.g. Lyria
+   * RealTime uses enum values like `"C_MAJOR"`, `"A_MINOR"`).
+   */
+  readonly scale?: string
+  /** Skip vocals / lyrics. */
+  readonly instrumental?: boolean
+  /** Preferred output format. Provider may override. */
+  readonly outputFormat?: AudioFormat
+}
+
+/**
+ * Streamed-output request. Same shape as the sync request — the
+ * streaming variant only differs in how the response is delivered.
+ */
+export type CommonStreamGenerateMusicRequest = CommonGenerateMusicRequest
+
+/**
+ * Bidirectional-session input. The user pushes one of these per
+ * change: a new prompt blend, a config delta, or a playback control.
+ * Lyria RealTime is the only provider currently surfacing these.
+ */
+export type MusicSessionInput =
+  | { readonly _tag: "prompts"; readonly prompts: ReadonlyArray<WeightedPrompt> }
+  | {
+      readonly _tag: "config"
+      readonly config: {
+        readonly bpm?: number
+        readonly scale?: string
+        readonly density?: number
+        readonly brightness?: number
+        readonly guidance?: number
+        readonly temperature?: number
+        readonly topK?: number
+        readonly seed?: number
+        readonly muteBass?: boolean
+        readonly muteDrums?: boolean
+        readonly onlyBassAndDrums?: boolean
+      }
+    }
+  | { readonly _tag: "control"; readonly action: "play" | "pause" | "stop" | "reset_context" }
+
+export const promptsInput = (prompts: ReadonlyArray<WeightedPrompt>): MusicSessionInput => ({
+  _tag: "prompts",
+  prompts,
+})
+
+export const configInput = (
+  config: (MusicSessionInput & { _tag: "config" })["config"],
+): MusicSessionInput => ({ _tag: "config", config })
+
+export const controlInput = (
+  action: (MusicSessionInput & { _tag: "control" })["action"],
+): MusicSessionInput => ({ _tag: "control", action })
+
+/**
+ * Sync-generation result. Extends `AudioBlob` with provider-side
+ * metadata that's common across music providers:
+ *
+ * - `songId` — Suno task id, ElevenLabs `song_id`, etc. Used for
+ *   back-reference (re-download, stem export, follow-up edits).
+ * - `lyrics` — generated lyrics when the model returned them (Lyria
+ *   text part, Mureka, Suno).
+ * - `sections` — structured section markers (Lyria optional JSON
+ *   structure response).
+ * - `watermark` — presence marker (Lyria SynthID is always set).
+ */
+export type MusicResult = AudioBlob & {
+  readonly songId?: string
+  readonly lyrics?: string
+  readonly sections?: ReadonlyArray<{
+    readonly label: string
+    readonly startSeconds: number
+    readonly endSeconds: number
+  }>
+  readonly watermark?: { readonly kind: string }
+}
+
+export const isPromptsInput = (
+  i: MusicSessionInput,
+): i is MusicSessionInput & { _tag: "prompts" } => i._tag === "prompts"
+export const isConfigInput = (i: MusicSessionInput): i is MusicSessionInput & { _tag: "config" } =>
+  i._tag === "config"
+export const isControlInput = (
+  i: MusicSessionInput,
+): i is MusicSessionInput & { _tag: "control" } => i._tag === "control"