@xsai/audio 0.0.0-stage → 1.0.0-beta.1

package/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Moeru AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,26 @@
-# Temporary Holding Version
+# @xsai/audio
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+<!-- automd:file src="/docs/snippets/audio.md" -->
+
+```sh
+pnpm add @xsai/audio
+```
+
+```ts
+import { writeFile } from 'node:fs/promises'
+
+import { generateSpeech, speech } from '@xsai/audio'
+
+const model = speech({
+  apiKey: process.env.OPENAI_API_KEY,
+  baseURL: 'https://api.openai.com/v1/',
+  model: 'YOUR_SPEECH_MODEL_ID',
+})
+
+const audio = await generateSpeech(model, { input: 'Welcome to the forest.', voice: 'alloy' })
+await writeFile('speech.mp3', new Uint8Array(await audio.arrayBuffer()))
+```
+
+<!-- /automd -->
+
+Read the [documentation](https://xsai.js.org/audio).

package/dist/index.d.ts

@@ -0,0 +1,114 @@
+import { Promisable, HttpOptions } from '@xsai/shared';
+
+type SpeechModel = (options: SpeechModelOptions) => Promisable<Response>;
+interface SpeechModelOptions {
+    input: string;
+    providerOptions?: SpeechModelProviderOptions;
+    signal?: AbortSignal;
+    voice: string;
+}
+interface SpeechModelProviderOptions {
+}
+
+declare const generateSpeech: (model: SpeechModel, options: SpeechModelOptions) => Promise<Blob>;
+
+declare module '@xsai/audio' {
+    interface SpeechModelProviderOptions {
+        speech?: {
+            instructions?: string;
+            outputFormat?: 'aac' | 'flac' | 'mp3' | 'opus' | 'wav';
+            speed?: number;
+        };
+    }
+}
+declare const speech: (options: HttpOptions) => SpeechModel;
+
+declare const streamSpeech: (model: SpeechModel, options: SpeechModelOptions) => Promise<Response>;
+
+interface TranscriptionEndEvent extends TranscriptionResult {
+    type: 'transcription.end';
+}
+type TranscriptionEvent = TranscriptionEndEvent | TranscriptionStartEvent | TranscriptionTextDeltaEvent | TranscriptionTextSegmentEvent;
+interface TranscriptionResult {
+    durationInSeconds?: number;
+    language?: string;
+    segments?: TranscriptionSegment[];
+    text: string;
+    words?: TranscriptionWord[];
+}
+interface TranscriptionSegment {
+    endSecond: number;
+    id?: string;
+    providerMetadata?: TranscriptionSegmentProviderMetadata;
+    startSecond: number;
+    text: string;
+}
+interface TranscriptionSegmentProviderMetadata {
+}
+interface TranscriptionStartEvent {
+    type: 'transcription.start';
+}
+interface TranscriptionTextDeltaEvent {
+    delta: string;
+    segmentId?: string;
+    type: 'transcription.text.delta';
+}
+interface TranscriptionTextSegmentEvent extends TranscriptionSegment {
+    type: 'transcription.text.segment';
+}
+interface TranscriptionWord {
+    endSecond: number;
+    providerMetadata?: TranscriptionWordProviderMetadata;
+    startSecond: number;
+    text: string;
+}
+interface TranscriptionWordProviderMetadata {
+}
+
+type TranscriptionModel = (options: TranscriptionModelOptions) => Promisable<ReadableStream<TranscriptionEvent>>;
+interface TranscriptionModelOptions {
+    audio: Blob;
+    fileName?: string;
+    language?: string;
+    providerOptions?: TranscriptionModelProviderOptions;
+    signal?: AbortSignal;
+}
+interface TranscriptionModelProviderOptions {
+}
+
+declare const generateTranscription: (model: TranscriptionModel, options: TranscriptionModelOptions) => Promise<TranscriptionResult>;
+
+declare module '@xsai/audio' {
+    interface TranscriptionModelProviderOptions {
+        transcriptions?: {
+            chunkingStrategy?: 'auto';
+            prompt?: string;
+            responseFormat?: 'diarized_json' | 'json' | 'verbose_json';
+            temperature?: number;
+            timestampGranularities?: ('segment' | 'word')[];
+        };
+    }
+    interface TranscriptionSegmentProviderMetadata {
+        transcriptions?: {
+            avgLogprob?: number;
+            compressionRatio?: number;
+            noSpeechProb?: number;
+            seek?: number;
+            speaker?: string;
+            temperature?: number;
+            tokens?: number[];
+        };
+    }
+    interface TranscriptionWordProviderMetadata {
+        transcriptions?: {
+            probability?: number;
+        };
+    }
+}
+declare const transcriptions: (options: HttpOptions) => TranscriptionModel;
+declare const transcriptionsNonStreaming: (options: HttpOptions) => TranscriptionModel;
+
+declare const streamTranscription: (model: TranscriptionModel, options: TranscriptionModelOptions) => Promise<ReadableStream<TranscriptionEvent>>;
+
+export { generateSpeech, generateTranscription, speech, streamSpeech, streamTranscription, transcriptions, transcriptionsNonStreaming };
+export type { SpeechModel, SpeechModelOptions, SpeechModelProviderOptions, TranscriptionEndEvent, TranscriptionEvent, TranscriptionModel, TranscriptionModelOptions, TranscriptionModelProviderOptions, TranscriptionResult, TranscriptionSegment, TranscriptionSegmentProviderMetadata, TranscriptionStartEvent, TranscriptionTextDeltaEvent, TranscriptionTextSegmentEvent, TranscriptionWord, TranscriptionWordProviderMetadata };

package/dist/index.js

@@ -0,0 +1,193 @@
+import { sendRequest, XSAIError, EventSourceParserStream } from '@xsai/shared';
+
+const streamSpeech = async (model, options) => model(options);
+
+const generateSpeech = async (model, options) => streamSpeech(model, options).then(async (r) => r.blob());
+
+const formats = {
+  aac: ["audio/aac"],
+  flac: ["audio/flac", "audio/x-flac"],
+  mp3: ["audio/mpeg", "audio/mp3"],
+  opus: ["audio/ogg"],
+  wav: ["audio/wav", "audio/x-wav", "audio/wave", "audio/vnd.wave"]
+};
+const speech = (options) => async (modelOptions) => {
+  const format = modelOptions.providerOptions?.speech?.outputFormat ?? "mp3";
+  const response = await sendRequest({
+    body: {
+      input: modelOptions.input,
+      instructions: modelOptions.providerOptions?.speech?.instructions,
+      model: options.model,
+      response_format: format,
+      speed: modelOptions.providerOptions?.speech?.speed,
+      stream_format: "audio",
+      voice: modelOptions.voice
+    },
+    path: "audio/speech",
+    signal: modelOptions.signal
+  }, options);
+  const contentType = response.headers.get("Content-Type");
+  const mime = contentType?.split(";", 1)[0]?.trim().toLowerCase();
+  const mediaTypes = formats[format];
+  if (mime == null || mime === "application/octet-stream") {
+    const headers = new Headers(response.headers);
+    headers.set("Content-Type", mediaTypes[0]);
+    return new Response(response.body, { headers, status: response.status, statusText: response.statusText });
+  }
+  if (!mediaTypes.includes(mime)) {
+    const error = new XSAIError("invalid-response", `Unexpected speech Content-Type: ${contentType}`);
+    await response.body.cancel(error).catch(() => {
+    });
+    throw error;
+  }
+  return response;
+};
+
+const generateTranscription = async (model, options) => {
+  let result;
+  for await (const event of await model(options)) {
+    if (event.type === "transcription.end") {
+      const { type: _, ...value } = event;
+      result = value;
+    }
+  }
+  if (result == null)
+    throw new XSAIError("truncated-stream", "model stream ended without a transcription.end event");
+  return result;
+};
+
+const transcriptionBody = (model, options, streaming = false) => {
+  const wireOptions = options.providerOptions?.transcriptions;
+  const body = new FormData();
+  body.append("file", options.audio, options.fileName);
+  for (const [key, value] of Object.entries({
+    chunking_strategy: wireOptions?.chunkingStrategy,
+    language: options.language,
+    model,
+    prompt: wireOptions?.prompt,
+    response_format: wireOptions?.responseFormat ?? "json",
+    stream: streaming ? true : void 0,
+    temperature: wireOptions?.temperature
+  })) {
+    if (value != null)
+      body.append(key, String(value));
+  }
+  for (const granularity of wireOptions?.timestampGranularities ?? (wireOptions?.responseFormat === "verbose_json" ? ["segment"] : []))
+    body.append("timestamp_granularities[]", granularity);
+  return body;
+};
+const transcriptionSegment = (value) => {
+  const metadata = {
+    avgLogprob: value.avg_logprob,
+    compressionRatio: value.compression_ratio,
+    noSpeechProb: value.no_speech_prob,
+    seek: value.seek,
+    speaker: value.speaker,
+    temperature: value.temperature,
+    tokens: value.tokens
+  };
+  return {
+    endSecond: value.end,
+    id: value.id?.toString(),
+    providerMetadata: Object.values(metadata).some((value2) => value2 != null) ? { transcriptions: metadata } : void 0,
+    startSecond: value.start,
+    text: value.text
+  };
+};
+const transcriptionResult = (value) => {
+  if (value == null || typeof value !== "object" || !("text" in value) || typeof value.text !== "string")
+    throw new XSAIError("invalid-response", "Expected transcription text to be a string", { cause: value });
+  const result = value;
+  return {
+    durationInSeconds: result.duration,
+    language: result.language,
+    segments: result.segments?.map(transcriptionSegment),
+    text: result.text,
+    words: result.words?.map((word) => ({
+      endSecond: word.end,
+      providerMetadata: word.probability == null ? void 0 : { transcriptions: { probability: word.probability } },
+      startSecond: word.start,
+      text: word.word
+    }))
+  };
+};
+class TranscriptionEventStream extends TransformStream {
+  constructor() {
+    const segments = [];
+    super({
+      flush: () => {
+        throw new XSAIError("truncated-stream", "transcription stream ended without transcript.text.done");
+      },
+      start: (output) => output.enqueue({ type: "transcription.start" }),
+      transform: ({ data }, output) => {
+        if (data === "" || data === "[DONE]")
+          return;
+        const event = JSON.parse(data);
+        if (event == null || typeof event !== "object")
+          throw new XSAIError("invalid-response", "Expected a transcription event object", { cause: event });
+        if (event.type === "error" || "error" in event && event.error != null)
+          throw new XSAIError("invalid-response", "Transcription provider reported an error", { cause: "error" in event ? event.error : event });
+        if (event.type === "transcript.text.delta") {
+          output.enqueue({
+            delta: event.delta,
+            segmentId: event.segment_id,
+            type: "transcription.text.delta"
+          });
+        } else if (event.type === "transcript.text.segment") {
+          const segment = transcriptionSegment(event);
+          segments.push(segment);
+          output.enqueue({ ...segment, type: "transcription.text.segment" });
+        } else if (event.type === "transcript.text.done") {
+          const result = transcriptionResult(event);
+          output.enqueue({
+            ...result,
+            segments: segments.length > 0 ? segments : result.segments,
+            type: "transcription.end"
+          });
+          output.terminate();
+        }
+      }
+    });
+  }
+}
+class TranscriptionResultStream extends TransformStream {
+  constructor() {
+    let text = "";
+    super({
+      flush: (output) => {
+        output.enqueue({ ...transcriptionResult(JSON.parse(text)), type: "transcription.end" });
+      },
+      start: (output) => output.enqueue({ type: "transcription.start" }),
+      transform: (chunk) => {
+        text += chunk;
+      }
+    });
+  }
+}
+const transcriptions = (options) => async (modelOptions) => {
+  const response = await sendRequest({
+    body: transcriptionBody(options.model, modelOptions, true),
+    path: "audio/transcriptions",
+    signal: modelOptions.signal
+  }, options);
+  const mime = response.headers.get("Content-Type")?.split(";", 1)[0]?.trim().toLowerCase();
+  if (mime !== "text/event-stream") {
+    const error = new XSAIError("invalid-response", "Expected an SSE transcription response");
+    await response.body.cancel(error).catch(() => {
+    });
+    throw error;
+  }
+  return response.body.pipeThrough(new TextDecoderStream()).pipeThrough(new EventSourceParserStream()).pipeThrough(new TranscriptionEventStream(), { signal: modelOptions.signal });
+};
+const transcriptionsNonStreaming = (options) => async (modelOptions) => {
+  const response = await sendRequest({
+    body: transcriptionBody(options.model, modelOptions),
+    path: "audio/transcriptions",
+    signal: modelOptions.signal
+  }, options);
+  return response.body.pipeThrough(new TextDecoderStream()).pipeThrough(new TranscriptionResultStream(), { signal: modelOptions.signal });
+};
+
+const streamTranscription = async (model, options) => model(options);
+
+export { generateSpeech, generateTranscription, speech, streamSpeech, streamTranscription, transcriptions, transcriptionsNonStreaming };

package/package.json

@@ -1,6 +1,46 @@
 {
   "name": "@xsai/audio",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "type": "module",
+  "version": "1.0.0-beta.1",
+  "description": "extra-small AI SDK: speech generation and transcription.",
+  "author": "Moeru AI",
+  "license": "MIT",
+  "homepage": "https://xsai.js.org",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/moeru-ai/xsai.git",
+    "directory": "packages/audio"
+  },
+  "bugs": "https://github.com/moeru-ai/xsai/issues",
+  "keywords": [
+    "xsai",
+    "ai",
+    "openai",
+    "speech",
+    "tts",
+    "transcription",
+    "stt",
+    "audio"
+  ],
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@xsai/shared": "1.0.0-beta.1"
+  },
+  "scripts": {
+    "build": "pkgroll --clean-dist",
+    "test": "vitest run --exclude e2e/**",
+    "test:watch": "vitest"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts"
 }