@rimelabs/sdk 0.1.0-alpha.10 → 0.1.0-alpha.13

package/dist/stt/transport.d.ts

@@ -0,0 +1,13 @@
+import * as grpc from "@grpc/grpc-js";
+import * as schema from "@rimelabs/api";
+import { BidiCall } from "../grpc.js";
+import type { TranscriptionMode } from "./types.js";
+export declare function requireSchema(): void;
+export declare const transport: {
+    makeClient: (target: string) => grpc.Client;
+};
+export declare class TranscriptionCall extends BidiCall<schema.StreamingTranscriptionRequest, schema.StreamingTranscriptionResponse> {
+    constructor(client: grpc.Client, auth: grpc.Metadata, signal: AbortSignal);
+    start(language: string, mode: TranscriptionMode, contextTerms: readonly string[]): Promise<void>;
+    writeAudio(audio: Uint8Array): Promise<void>;
+}

package/dist/stt/transport.js

@@ -0,0 +1,45 @@
+import * as grpc from "@grpc/grpc-js";
+import { create } from "@bufbuild/protobuf";
+import * as schema from "@rimelabs/api";
+import { BidiCall, serializer, deserializer } from "../grpc.js";
+import { RimeInputError } from "../errors.js";
+import { policy } from "./policy.js";
+export function requireSchema() {
+    if (!schema.SpeechToText)
+        throw new RimeInputError("STT requires @rimelabs/api with SpeechToText definitions");
+}
+export const transport = {
+    makeClient: (target) => new grpc.Client(target, grpc.credentials.createSsl(), {
+        "grpc.enable_retries": 0,
+        "grpc.max_receive_message_length": policy.receiveBytes,
+        "grpc.max_send_message_length": 131072,
+    }),
+};
+export class TranscriptionCall extends BidiCall {
+    constructor(client, auth, signal) {
+        requireSchema();
+        const service = schema.SpeechToText;
+        const method = service.method.transcribeStreaming;
+        super(client.makeBidiStreamRequest(`/${service.typeName}/${method.name}`, serializer(method.input), deserializer(method.output), auth), signal);
+    }
+    start(language, mode, contextTerms) {
+        return this.write(create(schema.StreamingTranscriptionRequestSchema, {
+            payload: {
+                case: "config",
+                value: {
+                    language,
+                    mode: mode === "verbatim"
+                        ? schema.TranscriptionMode.VERBATIM
+                        : schema.TranscriptionMode.WRITTEN,
+                    contextTerms: [...contextTerms],
+                    outputContract: schema.StreamingOutputContract.REVISED_HYPOTHESES,
+                },
+            },
+        }));
+    }
+    writeAudio(audio) {
+        return this.write(create(schema.StreamingTranscriptionRequestSchema, {
+            payload: { case: "audio", value: audio },
+        }));
+    }
+}

package/dist/stt/types.d.ts

@@ -0,0 +1,34 @@
+import type { PCMFormat } from "../pcm.js";
+/** Formatting intent, for example "flight 247" or "flight two four seven". */
+export type TranscriptionMode = "written" | "verbatim";
+/** One caller-ended utterance of headerless signed PCM16 little-endian audio. */
+export type AudioSource = AsyncIterable<Uint8Array>;
+export interface TranscriptionOptions {
+    /** Required BCP-47 tag, passed unchanged to the service (for example "en"). */
+    language: string;
+    /** Formatting intent; defaults to written and does not guarantee exact tokens. */
+    mode?: TranscriptionMode;
+    /** Recognition hints passed unchanged and in order, for example ["Super-G"]. */
+    contextTerms?: readonly string[];
+    /** Defaults to PCM16 little-endian mono 16 kHz; fixed for the utterance. */
+    inputFormat?: PCMFormat;
+    /** Optional overall deadline in seconds, including source and consumer pauses. */
+    timeout?: number | null;
+    /** Cancels only this utterance, including a pending connection. */
+    signal?: AbortSignal;
+}
+export interface TranscriptionPartial {
+    /** A revisable replacement snapshot. */
+    readonly kind: "partial";
+    /** Complete current text; replace previous text instead of appending it. */
+    readonly text: string;
+}
+export interface TranscriptionFinal {
+    /** Successful protocol and transport completion. */
+    readonly kind: "final";
+    /** Complete final text; silence can produce an empty string. */
+    readonly text: string;
+    /** Canonical BCP-47 tag selected by the service, for example "en". */
+    readonly language: string;
+}
+export type TranscriptionUpdate = TranscriptionPartial | TranscriptionFinal;

package/dist/stt/types.js

@@ -0,0 +1 @@
+export {};

package/dist/tts/client.d.ts

@@ -1,4 +1,5 @@
 import { AudioFormat } from "./audio.js";
+import { type PronunciationEntry } from "./lexicon.js";
 import { AudioStream, type TextSource } from "./stream.js";
 import { Credentials } from "../auth.js";
 import { policy } from "./policy.js";
@@ -6,6 +7,10 @@ export interface SynthesisOptions {
     voice?: string | null;
     language?: string;
     audioFormat?: AudioFormat | null;
+    /** Request final word timestamps. Supported only with model="mistv3". */
+    timestamps?: boolean;
+    /** Request-wide pronunciation overrides; requires a phoneme-capable Coda deployment. */
+    customLexicon?: readonly PronunciationEntry[];
     timeout?: number | null;
 }
 export interface DiscoveryOptions {
@@ -30,6 +35,9 @@ export declare class TTS {
     };
     constructor(credentials: Credentials, deployment: Readonly<typeof policy>, defaultTimeout: number | null, checkOpen: () => void);
     private prepare;
+    /** Synthesize complete text and return an audio stream. Do not await this call. */
+    synthesize(text: string, options?: SynthesisOptions): AudioStream;
+    /** Stream text chunks into synthesis and return an audio stream. Do not await this call. */
     stream(text: TextSource, options?: SynthesisOptions): AudioStream;
     private discover;
     close(): Promise<void>;

package/dist/tts/client.js

@@ -1,4 +1,5 @@
 import { AudioFormat } from "./audio.js";
+import { snapshot } from "./lexicon.js";
 import { AudioStream, constructionKey } from "./stream.js";
 import { policy, timeout, nonempty } from "./policy.js";
 import { abortable } from "../cancellation.js";
@@ -38,14 +39,30 @@ export class TTS {
         await connect(this.client, signal);
         return { client: this.client, metadata: metadata(auth) };
     }
+    /** Synthesize complete text and return an audio stream. Do not await this call. */
+    synthesize(text, options = {}) {
+        this.checkOpen();
+        if (typeof text !== "string")
+            throw new RimeInputError("text must be a string; use stream() for an async text source");
+        if (!text.trim())
+            throw new RimeInputError("Text must contain non-whitespace characters");
+        return this.stream((async function* () {
+            yield text;
+        })(), options);
+    }
+    /** Stream text chunks into synthesis and return an audio stream. Do not await this call. */
     stream(text, options = {}) {
         this.checkOpen();
-        if (typeof text === "string") {
-            if (!text.trim())
-                throw new RimeInputError("Text must contain non-whitespace characters");
-        }
-        else if (!text || typeof text[Symbol.asyncIterator] !== "function")
-            throw new RimeInputError("text must be a string or an async iterable of strings");
+        if (typeof text === "string" ||
+            !text ||
+            typeof text[Symbol.asyncIterator] !== "function")
+            throw new RimeInputError("text must be an async iterable of strings; use synthesize() for a string");
+        const lexicon = snapshot(options.customLexicon);
+        const timestamps = options.timestamps === undefined ? false : options.timestamps;
+        if (typeof timestamps !== "boolean")
+            throw new RimeInputError("timestamps must be a boolean");
+        if (timestamps && this.deployment.model !== "mistv3")
+            throw new RimeInputError("Word timestamps are supported only with model='mistv3'");
         const voice = options.voice === undefined || options.voice === null
             ? this.deployment.defaultVoice
             : nonempty(options.voice, "voice");
@@ -57,7 +74,7 @@ export class TTS {
             prepare: (s) => this.prepare(s),
             checkOpen: () => this.checkOpen(),
             forget: (s) => this.streams.delete(s),
-        }, text, voice, language, format, timeout(options.timeout, this.defaultTimeout));
+        }, text, voice, language, format, timeout(options.timeout, this.defaultTimeout), timestamps, lexicon);
         this.streams.add(audio);
         return audio;
     }

package/dist/tts/lexicon.d.ts

@@ -0,0 +1,7 @@
+/** A word or phrase and its space-separated X-SAMPA pronunciation. */
+export interface PronunciationEntry {
+    readonly spelling: string;
+    readonly pronunciation: string;
+}
+/** Copy request options now; model and linguistic validation belong to the service. */
+export declare function snapshot(entries?: readonly PronunciationEntry[]): readonly PronunciationEntry[];

package/dist/tts/lexicon.js

@@ -0,0 +1,23 @@
+import { RimeInputError } from "../errors.js";
+// Unicode mode treats valid surrogate pairs as one code point outside this range.
+const unpairedSurrogate = /[\uD800-\uDFFF]/u;
+/** Copy request options now; model and linguistic validation belong to the service. */
+export function snapshot(entries = []) {
+    if (!Array.isArray(entries))
+        throw new RimeInputError("customLexicon must be an array of pronunciation entries");
+    // Array.from also visits holes, which are invalid entries.
+    return Object.freeze(Array.from(entries, (entry) => {
+        if (!entry ||
+            typeof entry !== "object" ||
+            typeof entry.spelling !== "string" ||
+            typeof entry.pronunciation !== "string")
+            throw new RimeInputError("Lexicon entries must have string spelling and pronunciation fields");
+        if (unpairedSurrogate.test(entry.spelling) ||
+            unpairedSurrogate.test(entry.pronunciation))
+            throw new RimeInputError("Lexicon spelling and pronunciation must contain valid Unicode");
+        return Object.freeze({
+            spelling: entry.spelling,
+            pronunciation: entry.pronunciation,
+        });
+    }));
+}

package/dist/tts/policy.d.ts

@@ -1,9 +1,10 @@
+export { timeout } from "../validation.js";
 export declare const policy: {
+    model: string;
     target: string;
     audience: string;
     defaultVoice: string;
     exchangeUrl: string;
-    sentenceBytes: number;
     sourceChars: number;
     outputBytes: number;
     outputChunkBytes: number;
@@ -16,5 +17,4 @@ export declare const policy: {
     cleanupTimeout: number;
 };
 export declare function resolve(model: string, endpoint?: string | null): Readonly<typeof policy>;
-export declare function timeout(value: unknown, inherited?: number | null): number | null;
 export declare function nonempty(value: unknown, name: string): string;

package/dist/tts/policy.js

@@ -1,11 +1,13 @@
+import { endpointAddress } from "../validation.js";
+export { timeout } from "../validation.js";
 import { RimeInputError } from "../errors.js";
 const codaHostname = "coda.api.rime.ai";
 export const policy = {
+    model: "coda",
     target: `${codaHostname}:443`,
     audience: codaHostname,
     defaultVoice: "clementine",
     exchangeUrl: "https://themis.api.rime.ai/v1/token",
-    sentenceBytes: 65536,
     sourceChars: 1024,
     outputBytes: 96000,
     outputChunkBytes: 9600,
@@ -18,7 +20,7 @@ export const policy = {
     cleanupTimeout: 2,
 };
 export function resolve(model, endpoint) {
-    const deployment = { ...policy };
+    const deployment = { ...policy, model };
     if (model === "mistv3") {
         const hostname = "mist.api.rime.ai";
         deployment.target = `${hostname}:443`;
@@ -30,32 +32,7 @@ export function resolve(model, endpoint) {
     }
     if (endpoint === undefined || endpoint === null)
         return deployment;
-    const error = "endpoint must be a hostname with an optional port (1-65535), without a scheme or path";
-    if (typeof endpoint !== "string" || /\s/.test(endpoint))
-        throw new RimeInputError(error);
-    const [host, port, extra] = endpoint.split(":");
-    if (!host ||
-        host.length > 253 ||
-        extra !== undefined ||
-        host
-            .split(".")
-            .some((label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(label)) ||
-        (port !== undefined &&
-            (!/^[0-9]{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65535)))
-        throw new RimeInputError(error);
-    return {
-        ...deployment,
-        target: `${host.toLowerCase()}:${port === undefined ? 443 : Number(port)}`,
-        audience: host.toLowerCase(),
-    };
-}
-export function timeout(value, inherited = null) {
-    if (value === undefined)
-        return inherited;
-    if (value !== null &&
-        (typeof value !== "number" || !Number.isFinite(value) || value <= 0))
-        throw new RimeInputError("timeout must be finite positive seconds or null");
-    return value;
+    return { ...deployment, ...endpointAddress(endpoint) };
 }
 export function nonempty(value, name) {
     if (typeof value !== "string" || !value.trim())

package/dist/tts/stream.d.ts

@@ -1,11 +1,13 @@
 import { AudioFormat } from "./audio.js";
+import type { TimestampResult } from "./timestamps.js";
+import type { PronunciationEntry } from "./lexicon.js";
 import { type PreparedConnection } from "./transport.js";
 export interface StreamOwner {
     prepare(signal: AbortSignal): Promise<PreparedConnection>;
     checkOpen(): void;
     forget(stream: AudioStream): void;
 }
-export type TextSource = string | AsyncIterable<string>;
+export type TextSource = AsyncIterable<string>;
 export declare const constructionKey: unique symbol;
 export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
     private owner;
@@ -14,6 +16,8 @@ export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
     private language;
     private formatValue;
     private timeout;
+    private timestampsRequested;
+    private customLexicon;
     private readonly controller;
     private readonly queue;
     private worker;
@@ -31,9 +35,14 @@ export declare class AudioStream implements AsyncIterableIterator<Uint8Array> {
     private readonly operationId;
     private bytesReceived;
     private bytesDelivered;
-    constructor(key: symbol, owner: StreamOwner, source: TextSource | null, voice: string, language: string, formatValue: AudioFormat, timeout: number | null);
+    constructor(key: symbol, owner: StreamOwner, source: TextSource | null, voice: string, language: string, formatValue: AudioFormat, timeout: number | null, timestampsRequested?: boolean, customLexicon?: readonly PronunciationEntry[]);
     get format(): AudioFormat;
     get requestId(): string | null;
+    /** Read requested timestamps after consuming all audio; never drains audio.
+     * Calling before iteration completes rejects with RimeInputError. Synthesis
+     * errors reject here too; alignment failures are returned in status.
+     */
+    timestamps(): Promise<TimestampResult>;
     private start;
     private fail;
     private produce;

package/dist/tts/stream.js

@@ -5,7 +5,6 @@ import { Converter } from "./audio.js";
 import { policy } from "./policy.js";
 import { abortable } from "../cancellation.js";
 import { ByteQueue } from "./queue.js";
-import { SentenceBuffer, ready } from "./sentences.js";
 import { SynthesisCall } from "./transport.js";
 import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "../errors.js";
 export const constructionKey = Symbol("private stream constructor");
@@ -16,6 +15,8 @@ export class AudioStream {
     language;
     formatValue;
     timeout;
+    timestampsRequested;
+    customLexicon;
     controller = new AbortController();
     queue = new ByteQueue(policy.outputBytes, policy.outputChunkBytes);
     worker = null;
@@ -33,15 +34,17 @@ export class AudioStream {
     operationId = randomUUID();
     bytesReceived = 0;
     bytesDelivered = 0;
-    constructor(key, owner, source, voice, language, formatValue, timeout) {
+    constructor(key, owner, source, voice, language, formatValue, timeout, timestampsRequested = false, customLexicon = []) {
         this.owner = owner;
         this.source = source;
         this.voice = voice;
         this.language = language;
         this.formatValue = formatValue;
         this.timeout = timeout;
+        this.timestampsRequested = timestampsRequested;
+        this.customLexicon = customLexicon;
         if (key !== constructionKey)
-            throw new TypeError("AudioStream is returned by client.tts.stream()");
+            throw new TypeError("AudioStream is returned by client.tts.synthesize() or client.tts.stream()");
     }
     get format() {
         return this.formatValue;
@@ -49,6 +52,19 @@ export class AudioStream {
     get requestId() {
         return this.call?.requestId ?? null;
     }
+    /** Read requested timestamps after consuming all audio; never drains audio.
+     * Calling before iteration completes rejects with RimeInputError. Synthesis
+     * errors reject here too; alignment failures are returned in status.
+     */
+    async timestamps() {
+        if (!this.timestampsRequested)
+            throw new RimeInputError("Enable timestamps: true when creating the stream");
+        if (this.failure)
+            throw this.failure;
+        if (!this.finished)
+            throw new RimeInputError("Consume all audio before reading timestamps");
+        return this.call.timestampResult();
+    }
     start() {
         if (this.failure)
             throw this.failure;
@@ -96,15 +112,11 @@ export class AudioStream {
     async produce() {
         const source = this.source;
         let iterator;
-        const buffer = new SentenceBuffer(policy.sentenceBytes);
+        let highSurrogate = "";
         let meaningful = false;
         try {
             try {
-                iterator = (typeof source === "string"
-                    ? (async function* () {
-                        yield source;
-                    })()
-                    : source)[Symbol.asyncIterator]();
+                iterator = source[Symbol.asyncIterator]();
             }
             catch (error) {
                 if (this.controller.signal.aborted)
@@ -131,35 +143,39 @@ export class AudioStream {
                 }
                 if (result.done)
                     break;
-                const chunk = result.value;
-                if (typeof chunk !== "string")
+                if (typeof result.value !== "string")
                     throw new RimeInputError("The text source must yield strings");
+                let chunk = highSurrogate + result.value;
+                highSurrogate = "";
+                // A JavaScript source may split one Unicode character across yields.
+                if (/[\uD800-\uDBFF]$/.test(chunk)) {
+                    highSurrogate = chunk.slice(-1);
+                    chunk = chunk.slice(0, -1);
+                }
                 if (chunk.trim())
                     meaningful = true;
-                // Never split a surrogate pair when limiting native detector input size.
+                // Bound wire messages without splitting a Unicode character.
                 for (let offset = 0; offset < chunk.length;) {
                     let end = Math.min(chunk.length, offset + policy.sourceChars);
                     if (end < chunk.length && /[\uD800-\uDBFF]/.test(chunk[end - 1]))
                         end++;
-                    for (const sentence of buffer.feed(chunk.slice(offset, end))) {
-                        if (!this.submitted)
-                            this.progressAt = performance.now();
-                        this.submitted = true;
-                        await this.call.write(sentence);
-                    }
+                    const part = chunk.slice(offset, end);
+                    if (/[\uD800-\uDFFF]/u.test(part))
+                        throw new RimeInputError("Text must contain valid Unicode");
+                    if (!this.submitted)
+                        this.progressAt = performance.now();
+                    this.submitted = true;
+                    await this.call.write(part);
+                    this.progressAt = performance.now();
                     offset = end;
                 }
             }
+            if (highSurrogate)
+                throw new RimeInputError("Text must contain valid Unicode");
             if (!meaningful)
                 throw new RimeInputError("The text source contained no meaningful text");
-            for (const sentence of buffer.feed("", true)) {
-                if (!this.submitted)
-                    this.progressAt = performance.now();
-                this.submitted = true;
-                await this.call.write(sentence);
-            }
-            this.call.finishInput();
             this.inputDone = true;
+            this.call.finishInput();
         }
         catch (error) {
             if (error instanceof RimeError || this.controller.signal.aborted)
@@ -200,10 +216,9 @@ export class AudioStream {
     async run() {
         let tasks = [];
         try {
-            await abortable(ready, this.controller.signal);
             const prepared = await this.owner.prepare(this.controller.signal);
             this.call = new SynthesisCall(prepared, this.controller.signal);
-            await this.call.start(this.voice, this.language);
+            await this.call.start(this.voice, this.language, this.timestampsRequested, this.customLexicon);
             tasks = [this.produce(), this.read()];
             await Promise.all(tasks);
             this.queue.finish();

package/dist/tts/timestamps.d.ts

@@ -0,0 +1,24 @@
+import type { SynthesisResponseTrailer } from "@rimelabs/api";
+/** A spoken, normalized word with start/end seconds from synthesis start. */
+export interface WordTimestamp {
+    readonly text: string;
+    readonly start: number;
+    readonly end: number;
+}
+/** Alignment's google.rpc.Code (0 means OK) and service explanation. */
+export interface TimestampStatus {
+    readonly code: number;
+    readonly message: string;
+}
+/** Final alignment status and words. Non-OK statuses carry no spans. */
+export interface TimestampResult {
+    readonly status: TimestampStatus;
+    readonly spans: readonly WordTimestamp[];
+}
+export declare class TimestampTrailer {
+    seen: boolean;
+    invalid: boolean;
+    private value;
+    accept(trailer: SynthesisResponseTrailer): void;
+    result(requestId: string | null): TimestampResult;
+}

package/dist/tts/timestamps.js

@@ -0,0 +1,46 @@
+import { RimeStreamError } from "../errors.js";
+export class TimestampTrailer {
+    seen = false;
+    invalid = false;
+    value;
+    accept(trailer) {
+        if (this.seen)
+            this.invalid = true;
+        else {
+            this.seen = true;
+            this.value = trailer.timestamps;
+        }
+    }
+    result(requestId) {
+        const malformed = () => new RimeStreamError("Invalid timestamp trailer", requestId);
+        if (this.invalid)
+            throw malformed();
+        const value = this.value;
+        if (!value)
+            throw new RimeStreamError("The service did not return requested timestamps", requestId);
+        if (!value.status || (value.status.code !== 0 && value.spans.length))
+            throw malformed();
+        const seconds = (duration) => {
+            if (!duration ||
+                duration.seconds < 0n ||
+                duration.seconds > 315576000000n ||
+                duration.nanos < 0 ||
+                duration.nanos >= 1000000000)
+                throw malformed();
+            return Number(duration.seconds) + duration.nanos / 1e9;
+        };
+        const spans = value.spans.map((span) => {
+            const start = seconds(span.start), end = seconds(span.end);
+            if (!span.text.trim() || end < start)
+                throw malformed();
+            return Object.freeze({ text: span.text, start, end });
+        });
+        return Object.freeze({
+            status: Object.freeze({
+                code: value.status.code,
+                message: value.status.message,
+            }),
+            spans: Object.freeze(spans),
+        });
+    }
+}

package/dist/tts/transport.d.ts

@@ -1,33 +1,29 @@
+export { metadata } from "../grpc.js";
+export { rpcError } from "../grpc.js";
 import * as grpc from "@grpc/grpc-js";
-import * as errors from "../errors.js";
+import type { PronunciationEntry } from "./lexicon.js";
 export type Connection = grpc.Client;
 export interface PreparedConnection {
     client: Connection;
     metadata: grpc.Metadata;
 }
 export type DiscoveryKind = "voices" | "languages";
-export declare function rpcError(code: number, requestId?: string | null): errors.RimeError;
 export declare const transport: {
     makeClient: (target: string) => grpc.Client;
 };
 export declare function connect(client: grpc.Client, signal: AbortSignal): Promise<void>;
-export declare function metadata(value: string): grpc.Metadata;
 export declare function discover(prepared: PreparedConnection, kind: DiscoveryKind, language: string | null | undefined, signal: AbortSignal): Promise<string[]>;
 export declare class SynthesisCall {
-    private signal;
-    private readonly call;
-    private readonly metadataPromise;
-    private readonly statusPromise;
-    private status;
-    private id;
+    private timestampsRequested;
+    private readonly timestampTrailer;
+    private readonly rpc;
     constructor(prepared: PreparedConnection, signal: AbortSignal);
     get requestId(): string | null;
     get done(): boolean;
     cancel(): void;
-    private error;
-    private writeMessage;
-    start(voice: string, language: string): Promise<void>;
-    write(sentence: string): Promise<void>;
+    start(voice: string, language: string, timestamps?: boolean, customLexicon?: readonly PronunciationEntry[]): Promise<void>;
+    timestampResult(): import("./timestamps.js").TimestampResult;
+    write(text: string): Promise<void>;
     finishInput(): void;
     private checkFormat;
     audio(): AsyncGenerator<Uint8Array>;