@speech-sdk/core 0.6.2 → 0.7.0

package/dist/stt-providers/openai/index.js

@@ -0,0 +1,184 @@
+import { parseMediaTypeParam, wrapPcm16Mono } from "../../audio-utils.js";
+import { handleErrorResponse, resolveApiKey, SDK_USER_AGENT, } from "../../provider-utils.js";
+// OpenAI Whisper is advertised as 50+ languages; we list the ISO-639-1 codes
+// the API's `language` parameter accepts. Matches the TTS provider's list.
+const OPENAI_STT_LANGUAGES = [
+    "af",
+    "ar",
+    "az",
+    "be",
+    "bg",
+    "bn",
+    "bs",
+    "ca",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "el",
+    "en",
+    "es",
+    "et",
+    "fa",
+    "fi",
+    "fr",
+    "gl",
+    "he",
+    "hi",
+    "hr",
+    "hu",
+    "hy",
+    "id",
+    "is",
+    "it",
+    "ja",
+    "kk",
+    "kn",
+    "ko",
+    "lt",
+    "lv",
+    "mi",
+    "mk",
+    "mr",
+    "ms",
+    "ne",
+    "nl",
+    "no",
+    "pl",
+    "pt",
+    "ro",
+    "ru",
+    "sk",
+    "sl",
+    "sr",
+    "sv",
+    "sw",
+    "ta",
+    "th",
+    "tl",
+    "tr",
+    "uk",
+    "ur",
+    "vi",
+    "zh",
+];
+/**
+ * OpenAI Whisper / gpt-4o-transcribe adapter for the SDK's derived-timestamps
+ * path. Uses `/v1/audio/transcriptions` with `timestamp_granularities: ["word"]`
+ * and `response_format: "verbose_json"`.
+ *
+ * Note: `gpt-4o-transcribe-diarize` is intentionally not listed — that
+ * variant does not support `timestamp_granularities`.
+ */
+export class OpenAISpeechToTextProvider {
+    id = "openai";
+    defaultModel = "whisper-1";
+    // Only whisper-1 supports word-level timestamps. The newer
+    // gpt-4o-transcribe / gpt-4o-mini-transcribe models accept `json` /
+    // `text` response formats only and don't expose `timestamp_granularities`,
+    // so they can't satisfy this provider's contract.
+    models = [
+        {
+            id: "whisper-1",
+            releaseDate: "2023-03-01",
+            languages: OPENAI_STT_LANGUAGES,
+        },
+    ];
+    apiKey;
+    baseURL;
+    fetchFn;
+    constructor(config = {}) {
+        this.apiKey = config.apiKey;
+        this.baseURL = config.baseURL ?? "https://api.openai.com/v1";
+        this.fetchFn = config.fetch ?? globalThis.fetch.bind(globalThis);
+    }
+    async transcribe(options) {
+        const { audio, mediaType } = await normalizeAudioForOpenAI(options.audio, options.mediaType);
+        const form = new FormData();
+        const filename = `audio.${mediaTypeToExtension(mediaType)}`;
+        // Cast via BlobPart: TS narrowing of Uint8Array<ArrayBufferLike> vs
+        // Blob's required ArrayBuffer-backed view is stricter than runtime.
+        form.append("file", new Blob([audio], { type: mediaType }), filename);
+        form.append("model", options.modelId);
+        form.append("response_format", "verbose_json");
+        form.append("timestamp_granularities[]", "word");
+        if (options.language) {
+            form.append("language", options.language);
+        }
+        const response = await this.fetchFn(`${this.baseURL}/audio/transcriptions`, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${resolveApiKey(this.apiKey, "OPENAI_API_KEY", "OpenAI")}`,
+                "X-User-Agent": SDK_USER_AGENT,
+                ...options.headers,
+            },
+            body: form,
+            signal: options.abortSignal,
+        });
+        await handleErrorResponse(response, `openai/${options.modelId}`);
+        const data = (await response.json());
+        const timestamps = (data.words ?? []).map((w) => ({
+            text: w.word,
+            start: w.start,
+            end: w.end,
+        }));
+        return {
+            timestamps,
+            text: data.text,
+        };
+    }
+}
+export function createOpenAISTT(config = {}) {
+    const provider = new OpenAISpeechToTextProvider(config);
+    return function openaiSTT(modelId) {
+        return {
+            provider,
+            modelId: modelId ?? provider.defaultModel,
+        };
+    };
+}
+// OpenAI transcription accepts mp3/mp4/mpeg/mpga/m4a/wav/webm/flac/ogg/opus
+// but rejects raw PCM. When a TTS provider hands us raw little-endian PCM
+// (stitch mode), we wrap it with a WAV header so the STT endpoint will
+// parse it. `audio/l16` is intentionally NOT handled: RFC 2586 defines it
+// as big-endian and `wrapPcm16Mono` writes little-endian — silently mis-
+// wrapping would corrupt audio. No current provider emits L16; add an
+// explicit byte-swap branch here if one does.
+async function normalizeAudioForOpenAI(audio, mediaType) {
+    if (mediaTypeBase(mediaType) === "audio/pcm") {
+        const sampleRate = parseMediaTypeParam(mediaType, "rate") ?? 24_000;
+        return {
+            audio: await wrapPcm16Mono(audio, sampleRate),
+            mediaType: "audio/wav",
+        };
+    }
+    return { audio, mediaType };
+}
+function mediaTypeBase(mediaType) {
+    return mediaType.split(";")[0]?.trim().toLowerCase() ?? "";
+}
+function mediaTypeToExtension(mediaType) {
+    switch (mediaTypeBase(mediaType)) {
+        case "audio/mpeg":
+        case "audio/mp3":
+            return "mp3";
+        case "audio/wav":
+        case "audio/x-wav":
+            return "wav";
+        case "audio/ogg":
+            return "ogg";
+        case "audio/opus":
+            return "opus";
+        case "audio/flac":
+            return "flac";
+        case "audio/webm":
+            return "webm";
+        case "audio/mp4":
+        case "audio/m4a":
+        case "audio/x-m4a":
+            return "m4a";
+        default:
+            return "mp3";
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/stt-providers/openai/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/stt-providers/openai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,cAAc,GACf,MAAM,yBAAyB,CAAC;AAajC,6EAA6E;AAC7E,2EAA2E;AAC3E,MAAM,oBAAoB,GAAG;IAC3B,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;CACI,CAAC;AAEX;;;;;;;GAOG;AACH,MAAM,OAAO,0BAA0B;IAC5B,EAAE,GAAG,QAAQ,CAAC;IACd,YAAY,GAAG,WAAW,CAAC;IAEpC,2DAA2D;IAC3D,oEAAoE;IACpE,2EAA2E;IAC3E,kDAAkD;IACzC,MAAM,GAAG;QAChB;YACE,EAAE,EAAE,WAAW;YACf,WAAW,EAAE,YAAY;YACzB,SAAS,EAAE,oBAAoB;SAChC;KACO,CAAC;IAEM,MAAM,CAAqB;IAC3B,OAAO,CAAS;IAChB,OAAO,CAA0B;IAElD,YAAY,SAA2C,EAAE;QACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,2BAA2B,CAAC;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAOhB;QAKC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,uBAAuB,CACxD,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,SAAS,CAClB,CAAC;QAEF,MAAM,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,SAAS,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5D,oEAAoE;QACpE,oEAAoE;QACpE,IAAI,CAAC,MAAM,CACT,MAAM,EACN,IAAI,IAAI,CAAC,CAAC,KAAiB,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAClD,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CACjC,GAAG,IAAI,CAAC,OAAO,uBAAuB,EACtC;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,CAAC,EAAE;gBACjF,cAAc,EAAE,cAAc;gBAC9B,GAAG,OAAO,CAAC,OAAO;aACnB;YACD,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,OAAO,CAAC,WAAW;SAC5B,CACF,CAAC;QAEF,MAAM,mBAAmB,CAAC,QAAQ,EAAE,UAAU,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAEjE,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAGlC,CAAC;QAEF,MAAM,UAAU,GAAoB,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACjE,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,EAAE,CAAC,CAAC,GAAG;SACX,CAAC,CAAC,CAAC;QAEJ,OAAO;YACL,UAAU;YACV,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,eAAe,CAAC,SAA2C,EAAE;IAC3E,MAAM,QAAQ,GAAG,IAAI,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAExD,OAAO,SAAS,SAAS,CAAC,OAAgB;QACxC,OAAO;YACL,QAAQ;YACR,OAAO,EAAE,OAAO,IAAI,QAAQ,CAAC,YAAY;SAC1C,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,0EAA0E;AAC1E,uEAAuE;AACvE,0EAA0E;AAC1E,yEAAyE;AACzE,sEAAsE;AACtE,8CAA8C;AAC9C,KAAK,UAAU,uBAAuB,CACpC,KAAiB,EACjB,SAAiB;IAEjB,IAAI,aAAa,CAAC,SAAS,CAAC,KAAK,WAAW,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC;QACpE,OAAO;YACL,KAAK,EAAE,MAAM,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC;YAC7C,SAAS,EAAE,WAAW;SACvB,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,aAAa,CAAC,SAAiB;IACtC,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;AAC7D,CAAC;AAED,SAAS,oBAAoB,CAAC,SAAiB;IAC7C,QAAQ,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;QACjC,KAAK,YAAY,CAAC;QAClB,KAAK,WAAW;YACd,OAAO,KAAK,CAAC;QACf,KAAK,WAAW,CAAC;QACjB,KAAK,aAAa;YAChB,OAAO,KAAK,CAAC;QACf,KAAK,WAAW;YACd,OAAO,KAAK,CAAC;QACf,KAAK,YAAY;YACf,OAAO,MAAM,CAAC;QAChB,KAAK,YAAY;YACf,OAAO,MAAM,CAAC;QAChB,KAAK,YAAY;YACf,OAAO,MAAM,CAAC;QAChB,KAAK,WAAW,CAAC;QACjB,KAAK,WAAW,CAAC;QACjB,KAAK,aAAa;YAChB,OAAO,KAAK,CAAC;QACf;YACE,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC"}

package/dist/timestamps.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Word-granularity alignment data. Timestamps are always in seconds from
+ * the start of the generated audio. Providers that natively return character
+ * or phoneme granularity are aggregated to words internally.
+ */
+export interface WordTimestamp {
+    readonly end: number;
+    readonly start: number;
+    readonly text: string;
+}
+/**
+ * Controls whether `generateSpeech()` returns word timestamps.
+ *
+ * - `"auto"` (default): return timestamps only if the TTS provider supplies
+ *   them natively. Free, no extra API calls.
+ * - `"on"`: always return timestamps. Uses native data when available;
+ *   otherwise falls back to a speech-to-text round-trip of the synthesized
+ *   audio (cost + latency implications).
+ * - `"off"`: never return timestamps, even when the provider would give them
+ *   away for free.
+ */
+export type TimestampMode = "on" | "auto" | "off";
+//# sourceMappingURL=timestamps.d.ts.map

package/dist/timestamps.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timestamps.d.ts","sourceRoot":"","sources":["../src/timestamps.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,aAAa,GAAG,IAAI,GAAG,MAAM,GAAG,KAAK,CAAC"}

package/dist/timestamps.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=timestamps.js.map

package/dist/timestamps.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"timestamps.js","sourceRoot":"","sources":["../src/timestamps.ts"],"names":[],"mappings":""}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@speech-sdk/core",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "description": "Universal, cross-platform text-to-speech SDK with multi-provider support.",
   "type": "module",
   "main": "./dist/index.js",
@@ -69,6 +69,10 @@
     "./xai": {
       "types": "./dist/providers/xai/index.d.ts",
       "default": "./dist/providers/xai/index.js"
+    },
+    "./stt/openai": {
+      "types": "./dist/stt-providers/openai/index.d.ts",
+      "default": "./dist/stt-providers/openai/index.js"
     }
   },
   "files": [
@@ -84,7 +88,7 @@
     "inworld",
     "ai"
   ],
-  "license": "MIT",
+  "license": "Apache-2.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/Jellypod-Inc/speech-sdk"