@effect-uai/mistral 0.8.0

package/src/MistralTranscriber.ts

@@ -0,0 +1,224 @@
+import { Array as Arr, Context, Effect, Layer, Redacted, Schema, Stream } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import * as AiError from "@effect-uai/core/AiError"
+import * as Capabilities from "@effect-uai/core/Capabilities"
+import { isAudioUrl } from "@effect-uai/core/Audio"
+import type { TranscriptEvent, TranscriptResult, WordTimestamp } from "@effect-uai/core/Transcript"
+import {
+  type CommonStreamTranscribeRequest,
+  type CommonTranscribeRequest,
+  Transcriber,
+  type TranscriberService,
+} from "@effect-uai/core/Transcriber"
+import { audioToBlob, defaultFileName } from "./audioCodec.js"
+import { httpStatusError, transportFailure } from "./http.js"
+import type { MistralTranscribeModel } from "./models.js"
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+/**
+ * Mistral-typed transcribe request. `model` narrows to the typed
+ * `MistralTranscribeModel` union; `fileName` overrides the multipart upload's
+ * filename. Voxtral's batch endpoint supports diarization, word timestamps,
+ * and context biasing.
+ */
+export type MistralTranscribeRequest = Omit<CommonTranscribeRequest, "model"> & {
+  readonly model: MistralTranscribeModel
+  readonly temperature?: number
+  readonly fileName?: string
+}
+
+export type MistralTranscriberService = {
+  readonly transcribe: (
+    request: MistralTranscribeRequest,
+  ) => Effect.Effect<TranscriptResult, AiError.AiError>
+  readonly streamTranscriptionFrom: <E, R>(
+    audioIn: Stream.Stream<Uint8Array, E, R>,
+    request: CommonStreamTranscribeRequest,
+  ) => Stream.Stream<TranscriptEvent, AiError.AiError | E, R>
+}
+
+export class MistralTranscriber extends Context.Service<
+  MistralTranscriber,
+  MistralTranscriberService
+>()("@betalyra/effect-uai/providers/mistral/MistralTranscriber") {}
+
+export type Config = {
+  readonly apiKey: Redacted.Redacted
+  readonly baseUrl?: string
+}
+
+// ---------------------------------------------------------------------------
+// Codec — request → FormData
+//
+// Each field is described as a data `Part`, then the whole list is folded into
+// one FormData. A `Part` is either a scalar field or a file field (with an
+// upload filename).
+// ---------------------------------------------------------------------------
+
+type Part =
+  | readonly [name: string, value: string]
+  | readonly [name: string, file: Blob, filename: string]
+
+const when = (condition: boolean, part: Part): ReadonlyArray<Part> => (condition ? [part] : [])
+
+const appendPart = (fd: FormData, part: Part): FormData => {
+  if (part.length === 3) fd.append(part[0], part[1], part[2])
+  else fd.append(part[0], part[1])
+  return fd
+}
+
+const buildFormData = (
+  request: MistralTranscribeRequest,
+): Effect.Effect<FormData, AiError.AiError> =>
+  Effect.gen(function* () {
+    // Voxtral transcription has no Whisper-style prose `prompt` field.
+    yield* Capabilities.warnDroppedWhen(request.prompt, {
+      provider: "mistral",
+      capability: "prompt",
+      field: "prompt",
+      reason:
+        "Voxtral transcription has no prose prompt field; use `biasingTerms` for vocabulary hints.",
+    })
+    // URL audio rides the `file_url` field; inline bytes/base64 upload directly.
+    const filePart: Part = isAudioUrl(request.audio)
+      ? ["file_url", request.audio.url]
+      : yield* audioToBlob(request.audio).pipe(
+          Effect.map(
+            (blob): Part => ["file", blob, request.fileName ?? defaultFileName(blob.type)],
+          ),
+        )
+    const parts: ReadonlyArray<Part> = [
+      filePart,
+      ["model", request.model],
+      ...when(request.language !== undefined, ["language", request.language ?? ""]),
+      ...when(request.diarization === true, ["diarize", "true"]),
+      ...when(request.wordTimestamps === true, ["timestamp_granularities[]", "word"]),
+      ...when(request.temperature !== undefined, ["temperature", String(request.temperature)]),
+      // Voxtral biases on a `context_bias` term list; one entry per term.
+      ...(request.biasingTerms ?? []).map((term): Part => ["context_bias", term]),
+    ]
+    return Arr.reduce(parts, new FormData(), appendPart)
+  })
+
+// ---------------------------------------------------------------------------
+// Codec — response → TranscriptResult
+// ---------------------------------------------------------------------------
+
+const WireWord = Schema.Struct({
+  text: Schema.optional(Schema.String),
+  word: Schema.optional(Schema.String),
+  start: Schema.Number,
+  end: Schema.Number,
+  speaker: Schema.optional(Schema.NullOr(Schema.String)),
+})
+
+const WireResponse = Schema.Struct({
+  text: Schema.String,
+  language: Schema.optional(Schema.NullOr(Schema.String)),
+  words: Schema.optional(Schema.NullOr(Schema.Array(WireWord))),
+})
+
+const decodeResponse = Schema.decodeUnknownEffect(WireResponse)
+
+const wireWordToCommon = (w: typeof WireWord.Type): WordTimestamp => ({
+  text: w.text ?? w.word ?? "",
+  startSeconds: w.start,
+  endSeconds: w.end,
+  ...(w.speaker != null && { speakerId: w.speaker }),
+})
+
+const toResult = (raw: unknown, decoded: typeof WireResponse.Type): TranscriptResult => ({
+  text: decoded.text,
+  ...(decoded.language != null && { languageCode: decoded.language }),
+  ...(decoded.words != null && { words: decoded.words.map(wireWordToCommon) }),
+  raw,
+})
+
+// ---------------------------------------------------------------------------
+// HTTP plumbing
+// ---------------------------------------------------------------------------
+
+/** Exported for reuse by `MistralRealtimeTranscriber` (same sync path). */
+export const transcribeImpl =
+  (cfg: Config) =>
+  (
+    request: MistralTranscribeRequest,
+  ): Effect.Effect<TranscriptResult, AiError.AiError, HttpClient.HttpClient> =>
+    Effect.gen(function* () {
+      const client = yield* HttpClient.HttpClient
+      const baseUrl = cfg.baseUrl ?? "https://api.mistral.ai"
+      const formData = yield* buildFormData(request)
+      const httpRequest = HttpClientRequest.post(`${baseUrl}/v1/audio/transcriptions`).pipe(
+        HttpClientRequest.bearerToken(cfg.apiKey),
+        HttpClientRequest.bodyFormData(formData),
+      )
+      const response = yield* client.execute(httpRequest).pipe(Effect.mapError(transportFailure))
+      if (response.status >= 400) {
+        const text = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
+        return yield* Effect.fail(httpStatusError(response.status, text))
+      }
+      const json = yield* response.json.pipe(Effect.mapError(transportFailure))
+      const decoded = yield* decodeResponse(json).pipe(Effect.mapError(transportFailure))
+      return toResult(json, decoded)
+    })
+
+// ---------------------------------------------------------------------------
+// streamTranscriptionFrom — Unsupported on the sync-only Layer
+// ---------------------------------------------------------------------------
+
+/**
+ * Sync-only Layer's streaming impl. For live transcription import the
+ * `MistralRealtimeTranscriber` subpath, which registers `SttStreaming` and
+ * wires the Voxtral Realtime WebSocket endpoint.
+ */
+const streamUnsupported = <E, R>(
+  _audioIn: Stream.Stream<Uint8Array, E, R>,
+  _request: CommonStreamTranscribeRequest,
+): Stream.Stream<TranscriptEvent, AiError.AiError | E, R> =>
+  Stream.fail(
+    new AiError.Unsupported({
+      provider: "mistral",
+      capability: "streamTranscriptionFrom",
+      reason:
+        "This Layer is sync-only. Use `@effect-uai/mistral/MistralRealtimeTranscriber` for live Voxtral transcription.",
+    }),
+  )
+
+// ---------------------------------------------------------------------------
+// Constructors
+// ---------------------------------------------------------------------------
+
+export const make = (
+  cfg: Config,
+): Effect.Effect<MistralTranscriberService, never, HttpClient.HttpClient> =>
+  Effect.map(HttpClient.HttpClient.asEffect(), (client) => ({
+    transcribe: (request) =>
+      transcribeImpl(cfg)(request).pipe(Effect.provideService(HttpClient.HttpClient, client)),
+    streamTranscriptionFrom: streamUnsupported,
+  }))
+
+/**
+ * Sync-only Layer. Registers `MistralTranscriber` + the generic `Transcriber`
+ * tag. Does NOT register `SttStreaming` — for live transcription use
+ * `@effect-uai/mistral/MistralRealtimeTranscriber`.
+ */
+export const layer = (
+  cfg: Config,
+): Layer.Layer<MistralTranscriber | Transcriber, never, HttpClient.HttpClient> =>
+  Layer.merge(
+    Layer.effect(MistralTranscriber, make(cfg)),
+    Layer.effect(
+      Transcriber,
+      Effect.map(
+        make(cfg),
+        (s): TranscriberService => ({
+          transcribe: (req: CommonTranscribeRequest) =>
+            s.transcribe(req as MistralTranscribeRequest),
+          streamTranscriptionFrom: s.streamTranscriptionFrom,
+        }),
+      ),
+    ),
+  )

package/src/audioCodec.ts

@@ -0,0 +1,123 @@
+import { Array as Arr, Effect, Encoding, Match, Option, Result } from "effect"
+import * as AiError from "@effect-uai/core/AiError"
+import type { AudioFormat, AudioSource } from "@effect-uai/core/Audio"
+
+// ---------------------------------------------------------------------------
+// AudioSource → Blob (for multipart upload to /audio/transcriptions)
+// ---------------------------------------------------------------------------
+
+const decodeBase64ToBytes = (b64: string): Effect.Effect<Uint8Array, AiError.AiError> =>
+  Result.match(Encoding.decodeBase64(b64), {
+    onSuccess: Effect.succeed,
+    onFailure: (cause) =>
+      Effect.fail(new AiError.InvalidRequest({ provider: "mistral", param: "audio", raw: cause })),
+  })
+
+/**
+ * TS 6's `Blob` constructor wants `Uint8Array<ArrayBuffer>`; our domain carries
+ * the broader `Uint8Array<ArrayBufferLike>`. The cast is sound: `Blob()` does
+ * not mutate the buffer and the runtime accepts both.
+ */
+const bytesToBlob = (bytes: Uint8Array, mimeType: string): Blob =>
+  new Blob([bytes as Uint8Array<ArrayBuffer>], { type: mimeType })
+
+/**
+ * Build a `Blob` from any `AudioSource` variant. `url` variants pass through
+ * to the `file_url` field upstream, so this only handles inline bytes/base64.
+ */
+export const audioToBlob: (audio: AudioSource) => Effect.Effect<Blob, AiError.AiError> =
+  Match.type<AudioSource>().pipe(
+    Match.tag("bytes", (a) => Effect.succeed(bytesToBlob(a.bytes, a.mimeType))),
+    Match.tag("base64", (a) =>
+      decodeBase64ToBytes(a.base64).pipe(Effect.map((bytes) => bytesToBlob(bytes, a.mimeType))),
+    ),
+    Match.tag("url", (a) =>
+      Effect.fail(
+        new AiError.InvalidRequest({
+          provider: "mistral",
+          param: "audio",
+          raw: `Pass URL audio via the request's url source; got ${a.url} where inline bytes were expected.`,
+        }),
+      ),
+    ),
+    Match.exhaustive,
+  )
+
+const EXTENSION_BY_MIME: ReadonlyArray<readonly [pattern: string, ext: string]> = [
+  ["mpeg", "mp3"],
+  ["mp3", "mp3"],
+  ["wav", "wav"],
+  ["ogg", "ogg"],
+  ["opus", "ogg"],
+  ["flac", "flac"],
+  ["aac", "aac"],
+  ["mp4", "m4a"],
+  ["m4a", "m4a"],
+  ["webm", "webm"],
+]
+
+/** Derive a default upload filename from a MIME type, e.g. `audio/mpeg` -> `audio.mp3`. */
+export const defaultFileName = (mimeType: string): string =>
+  Arr.findFirst(EXTENSION_BY_MIME, ([pattern]) => mimeType.includes(pattern)).pipe(
+    Option.match({ onNone: () => "audio", onSome: ([, ext]) => `audio.${ext}` }),
+  )
+
+// ---------------------------------------------------------------------------
+// AudioFormat → Voxtral TTS `response_format`
+// ---------------------------------------------------------------------------
+
+export type MistralTtsFormat = "mp3" | "wav" | "pcm" | "flac" | "opus"
+
+/**
+ * Map an `AudioFormat.container` to Voxtral TTS's `response_format`. Voxtral
+ * produces mp3 / wav / pcm / flac / opus; aac / ogg / webm are unsupported.
+ */
+export const containerToTtsFormat: (
+  container: AudioFormat["container"],
+) => Effect.Effect<MistralTtsFormat, AiError.AiError> = Match.type<AudioFormat["container"]>().pipe(
+  Match.when("mp3", () => Effect.succeed<MistralTtsFormat>("mp3")),
+  Match.when("wav", () => Effect.succeed<MistralTtsFormat>("wav")),
+  Match.when("raw", () => Effect.succeed<MistralTtsFormat>("pcm")),
+  Match.when("flac", () => Effect.succeed<MistralTtsFormat>("flac")),
+  Match.when("opus", () => Effect.succeed<MistralTtsFormat>("opus")),
+  Match.whenOr("aac", "ogg", "webm", (c) =>
+    Effect.fail(
+      new AiError.Unsupported({
+        provider: "mistral",
+        capability: "outputFormat",
+        reason: `Voxtral TTS does not produce ${c} output; supported: mp3 | wav | pcm (raw) | flac | opus.`,
+      }),
+    ),
+  ),
+  Match.exhaustive,
+)
+
+/**
+ * The on-wire `AudioFormat` Voxtral actually returns for each
+ * `response_format`. Voxtral's `pcm` is float32 LE at 24 kHz; the caller's
+ * requested sample rate is not honoured for raw output.
+ */
+export const realizedTtsFormat: (rf: MistralTtsFormat) => AudioFormat =
+  Match.type<MistralTtsFormat>().pipe(
+    Match.when(
+      "mp3",
+      (): AudioFormat => ({ container: "mp3", encoding: "mp3", sampleRate: 24000 }),
+    ),
+    Match.when(
+      "wav",
+      (): AudioFormat => ({ container: "wav", encoding: "pcm_s16le", sampleRate: 24000 }),
+    ),
+    Match.when(
+      "pcm",
+      (): AudioFormat => ({ container: "raw", encoding: "pcm_f32le", sampleRate: 24000 }),
+    ),
+    Match.when(
+      "flac",
+      (): AudioFormat => ({ container: "flac", encoding: "flac", sampleRate: 24000 }),
+    ),
+    Match.when(
+      "opus",
+      (): AudioFormat => ({ container: "opus", encoding: "opus", sampleRate: 24000 }),
+    ),
+    Match.exhaustive,
+  )

package/src/codec.test.ts

@@ -0,0 +1,121 @@
+import type { HistoryItem } from "@effect-uai/core/Items"
+import { describe, expect, it } from "vitest"
+import {
+  type Accumulator,
+  accumulatorToTurn,
+  applyChunk,
+  emptyAccumulator,
+  itemsToMessages,
+  toolChoiceWire,
+  type WireChunk,
+} from "./codec.js"
+
+describe("itemsToMessages", () => {
+  // The interesting case: our flat history (assistant message, then separate
+  // function_call items) must fold into the OpenAI/Mistral shape where the
+  // tool calls ride the assistant message and each result is a `tool` message.
+  it("folds tool calls onto the assistant message and emits tool results", () => {
+    const history: ReadonlyArray<HistoryItem> = [
+      { type: "message", role: "assistant", content: [{ type: "output_text", text: "checking" }] },
+      { type: "function_call", call_id: "c1", name: "a", arguments: "{}" },
+      { type: "function_call", call_id: "c2", name: "b", arguments: "{}" },
+      { type: "function_call_output", call_id: "c1", output: "ra" },
+    ]
+    expect(itemsToMessages(history)).toEqual([
+      {
+        role: "assistant",
+        content: "checking",
+        tool_calls: [
+          { id: "c1", type: "function", function: { name: "a", arguments: "{}" } },
+          { id: "c2", type: "function", function: { name: "b", arguments: "{}" } },
+        ],
+      },
+      { role: "tool", tool_call_id: "c1", content: "ra" },
+    ])
+  })
+
+  it("encodes a message with an image as multimodal parts", () => {
+    const history: ReadonlyArray<HistoryItem> = [
+      {
+        type: "message",
+        role: "user",
+        content: [
+          { type: "input_text", text: "what is this" },
+          { type: "input_image", source: { _tag: "url", url: "https://x/y.png" } },
+        ],
+      },
+    ]
+    expect(itemsToMessages(history)).toEqual([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "what is this" },
+          { type: "image_url", image_url: "https://x/y.png" },
+        ],
+      },
+    ])
+  })
+})
+
+it("toolChoiceWire maps 'required' to Mistral's 'any'", () => {
+  // Mistral diverges from OpenAI here; the rest pass through unchanged.
+  expect(toolChoiceWire("required")).toBe("any")
+})
+
+// Run a sequence of chunks through the accumulator, collecting event tags.
+const runChunks = (
+  chunks: ReadonlyArray<WireChunk>,
+): { acc: Accumulator; tags: ReadonlyArray<string> } => {
+  let acc = emptyAccumulator
+  const tags: Array<string> = []
+  for (const chunk of chunks) {
+    const [next, events] = applyChunk(acc, chunk)
+    acc = next
+    for (const e of events) tags.push(e._tag)
+  }
+  return { acc, tags }
+}
+
+describe("streaming accumulation", () => {
+  it("accumulates text deltas + usage into an assistant turn", () => {
+    const { acc, tags } = runChunks([
+      { choices: [{ delta: { content: "Hel" } }] },
+      { choices: [{ delta: { content: "lo" } }] },
+      { choices: [{ delta: {}, finish_reason: "stop" }] },
+      { usage: { prompt_tokens: 3, completion_tokens: 2, total_tokens: 5 } },
+    ])
+    expect(tags).toEqual(["TextDelta", "TextDelta", "UsageUpdate"])
+    const turn = accumulatorToTurn(acc)
+    expect(turn.items).toEqual([
+      { type: "message", role: "assistant", content: [{ type: "output_text", text: "Hello" }] },
+    ])
+    expect(turn.usage).toEqual({ input_tokens: 3, output_tokens: 2, total_tokens: 5 })
+    expect(turn.stop_reason).toBe("stop")
+  })
+
+  it("stitches a tool call split across chunks", () => {
+    const { acc, tags } = runChunks([
+      {
+        choices: [
+          {
+            delta: {
+              tool_calls: [
+                { index: 0, id: "c1", function: { name: "get_weather", arguments: '{"ci' } },
+              ],
+            },
+          },
+        ],
+      },
+      {
+        choices: [{ delta: { tool_calls: [{ index: 0, function: { arguments: 'ty":"NYC"}' } }] } }],
+      },
+      { choices: [{ delta: {}, finish_reason: "tool_calls" }] },
+    ])
+    expect(tags).toEqual(["ToolCallStart", "ToolCallArgsDelta", "ToolCallArgsDelta"])
+    const turn = accumulatorToTurn(acc)
+    expect(turn.items).toEqual([
+      { type: "function_call", call_id: "c1", name: "get_weather", arguments: '{"city":"NYC"}' },
+    ])
+    expect(turn.stop_reason).toBe("tool_calls")
+  })
+})