@rimelabs/sdk 0.1.0-alpha.2 → 0.1.0-alpha.21

package/dist/{stream.js → tts/stream.js}

@@ -1,12 +1,12 @@
 import { randomUUID } from "node:crypto";
 import { debuglog } from "node:util";
 const log = debuglog("rime-sdk");
-import { Converter } from "./audio.js";
-import { policy, abortable } from "./policy.js";
+import { AudioFormat, Converter, wireFormat } from "./audio.js";
+import { policy } from "./policy.js";
+import { abortable } from "../cancellation.js";
 import { ByteQueue } from "./queue.js";
-import { SentenceBuffer, ready } from "./sentences.js";
 import { SynthesisCall } from "./transport.js";
-import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "./errors.js";
+import { RimeError, RimeInputError, RimeCancelledError, RimeTimeoutError, RimeStreamError, } from "../errors.js";
 export const constructionKey = Symbol("private stream constructor");
 export class AudioStream {
     owner;
@@ -15,6 +15,10 @@ export class AudioStream {
     language;
     formatValue;
     timeout;
+    timestampsRequested;
+    customLexicon;
+    sampleRateValue;
+    timeScaleFactor;
     controller = new AbortController();
     queue = new ByteQueue(policy.outputBytes, policy.outputChunkBytes);
     worker = null;
@@ -32,22 +36,43 @@ export class AudioStream {
     operationId = randomUUID();
     bytesReceived = 0;
     bytesDelivered = 0;
-    constructor(key, owner, source, voice, language, formatValue, timeout) {
+    constructor(key, owner, source, voice, language, formatValue, timeout, timestampsRequested = false, customLexicon = [], sampleRateValue = 24000, timeScaleFactor) {
         this.owner = owner;
         this.source = source;
         this.voice = voice;
         this.language = language;
         this.formatValue = formatValue;
         this.timeout = timeout;
+        this.timestampsRequested = timestampsRequested;
+        this.customLexicon = customLexicon;
+        this.sampleRateValue = sampleRateValue;
+        this.timeScaleFactor = timeScaleFactor;
         if (key !== constructionKey)
-            throw new TypeError("AudioStream is returned by client.tts.stream()");
+            throw new TypeError("AudioStream is returned by client.tts.synthesize() or client.tts.stream()");
     }
     get format() {
         return this.formatValue;
     }
+    /** Requested output rate. MP3 headers describe the encoded rate. */
+    get sampleRate() {
+        return this.sampleRateValue;
+    }
     get requestId() {
         return this.call?.requestId ?? null;
     }
+    /** Read requested timestamps after consuming all audio; never drains audio.
+     * Calling before iteration completes rejects with RimeInputError. Synthesis
+     * errors reject here too; alignment failures are returned in status.
+     */
+    async timestamps() {
+        if (!this.timestampsRequested)
+            throw new RimeInputError("Enable timestamps: true when creating the stream");
+        if (this.failure)
+            throw this.failure;
+        if (!this.finished)
+            throw new RimeInputError("Consume all audio before reading timestamps");
+        return this.call.timestampResult();
+    }
     start() {
         if (this.failure)
             throw this.failure;
@@ -95,15 +120,11 @@ export class AudioStream {
     async produce() {
         const source = this.source;
         let iterator;
-        const buffer = new SentenceBuffer(policy.sentenceBytes);
+        let highSurrogate = "";
         let meaningful = false;
         try {
             try {
-                iterator = (typeof source === "string"
-                    ? (async function* () {
-                        yield source;
-                    })()
-                    : source)[Symbol.asyncIterator]();
+                iterator = source[Symbol.asyncIterator]();
             }
             catch (error) {
                 if (this.controller.signal.aborted)
@@ -130,35 +151,39 @@ export class AudioStream {
                 }
                 if (result.done)
                     break;
-                const chunk = result.value;
-                if (typeof chunk !== "string")
+                if (typeof result.value !== "string")
                     throw new RimeInputError("The text source must yield strings");
+                let chunk = highSurrogate + result.value;
+                highSurrogate = "";
+                // A JavaScript source may split one Unicode character across yields.
+                if (/[\uD800-\uDBFF]$/.test(chunk)) {
+                    highSurrogate = chunk.slice(-1);
+                    chunk = chunk.slice(0, -1);
+                }
                 if (chunk.trim())
                     meaningful = true;
-                // Never split a surrogate pair when limiting native detector input size.
+                // Bound wire messages without splitting a Unicode character.
                 for (let offset = 0; offset < chunk.length;) {
                     let end = Math.min(chunk.length, offset + policy.sourceChars);
                     if (end < chunk.length && /[\uD800-\uDBFF]/.test(chunk[end - 1]))
                         end++;
-                    for (const sentence of buffer.feed(chunk.slice(offset, end))) {
-                        if (!this.submitted)
-                            this.progressAt = performance.now();
-                        this.submitted = true;
-                        await this.call.write(sentence);
-                    }
+                    const part = chunk.slice(offset, end);
+                    if (/[\uD800-\uDFFF]/u.test(part))
+                        throw new RimeInputError("Text must contain valid Unicode");
+                    if (!this.submitted)
+                        this.progressAt = performance.now();
+                    this.submitted = true;
+                    await this.call.write(part);
+                    this.progressAt = performance.now();
                     offset = end;
                 }
             }
+            if (highSurrogate)
+                throw new RimeInputError("Text must contain valid Unicode");
             if (!meaningful)
                 throw new RimeInputError("The text source contained no meaningful text");
-            for (const sentence of buffer.feed("", true)) {
-                if (!this.submitted)
-                    this.progressAt = performance.now();
-                this.submitted = true;
-                await this.call.write(sentence);
-            }
-            this.call.finishInput();
             this.inputDone = true;
+            this.call.finishInput();
         }
         catch (error) {
             if (error instanceof RimeError || this.controller.signal.aborted)
@@ -199,10 +224,9 @@ export class AudioStream {
     async run() {
         let tasks = [];
         try {
-            await abortable(ready, this.controller.signal);
             const prepared = await this.owner.prepare(this.controller.signal);
             this.call = new SynthesisCall(prepared, this.controller.signal);
-            await this.call.start(this.voice, this.language);
+            await this.call.start(this.voice, this.language, this.timestampsRequested, this.customLexicon, wireFormat(this.format), this.format === AudioFormat.MULAW_8000 ? 24000 : this.sampleRate, this.timeScaleFactor);
             tasks = [this.produce(), this.read()];
             await Promise.all(tasks);
             this.queue.finish();

package/dist/tts/timestamps.d.ts

@@ -0,0 +1,24 @@
+import type { SynthesisResponseTrailer } from "@rimelabs/api";
+/** A spoken, normalized word with start/end seconds from synthesis start. */
+export interface WordTimestamp {
+    readonly text: string;
+    readonly start: number;
+    readonly end: number;
+}
+/** Alignment's google.rpc.Code (0 means OK) and service explanation. */
+export interface TimestampStatus {
+    readonly code: number;
+    readonly message: string;
+}
+/** Final alignment status and words. Non-OK statuses carry no spans. */
+export interface TimestampResult {
+    readonly status: TimestampStatus;
+    readonly spans: readonly WordTimestamp[];
+}
+export declare class TimestampTrailer {
+    seen: boolean;
+    invalid: boolean;
+    private value;
+    accept(trailer: SynthesisResponseTrailer): void;
+    result(requestId: string | null): TimestampResult;
+}

package/dist/tts/timestamps.js

@@ -0,0 +1,46 @@
+import { RimeStreamError } from "../errors.js";
+export class TimestampTrailer {
+    seen = false;
+    invalid = false;
+    value;
+    accept(trailer) {
+        if (this.seen)
+            this.invalid = true;
+        else {
+            this.seen = true;
+            this.value = trailer.timestamps;
+        }
+    }
+    result(requestId) {
+        const malformed = () => new RimeStreamError("Invalid timestamp trailer", requestId);
+        if (this.invalid)
+            throw malformed();
+        const value = this.value;
+        if (!value)
+            throw new RimeStreamError("The service did not return requested timestamps", requestId);
+        if (!value.status || (value.status.code !== 0 && value.spans.length))
+            throw malformed();
+        const seconds = (duration) => {
+            if (!duration ||
+                duration.seconds < 0n ||
+                duration.seconds > 315576000000n ||
+                duration.nanos < 0 ||
+                duration.nanos >= 1000000000)
+                throw malformed();
+            return Number(duration.seconds) + duration.nanos / 1e9;
+        };
+        const spans = value.spans.map((span) => {
+            const start = seconds(span.start), end = seconds(span.end);
+            if (!span.text.trim() || end < start)
+                throw malformed();
+            return Object.freeze({ text: span.text, start, end });
+        });
+        return Object.freeze({
+            status: Object.freeze({
+                code: value.status.code,
+                message: value.status.message,
+            }),
+            spans: Object.freeze(spans),
+        });
+    }
+}

package/dist/{transport.d.ts → tts/transport.d.ts}

@@ -1,33 +1,30 @@
+export { metadata } from "../grpc.js";
+export { rpcError } from "../grpc.js";
 import * as grpc from "@grpc/grpc-js";
-import * as errors from "./errors.js";
+import type { PronunciationEntry } from "./lexicon.js";
 export type Connection = grpc.Client;
 export interface PreparedConnection {
     client: Connection;
     metadata: grpc.Metadata;
 }
 export type DiscoveryKind = "voices" | "languages";
-export declare function rpcError(code: number, requestId?: string | null): errors.RimeError;
 export declare const transport: {
     makeClient: (target: string) => grpc.Client;
 };
 export declare function connect(client: grpc.Client, signal: AbortSignal): Promise<void>;
-export declare function metadata(value: string): grpc.Metadata;
 export declare function discover(prepared: PreparedConnection, kind: DiscoveryKind, language: string | null | undefined, signal: AbortSignal): Promise<string[]>;
 export declare class SynthesisCall {
-    private signal;
-    private readonly call;
-    private readonly metadataPromise;
-    private readonly statusPromise;
-    private status;
-    private id;
+    private audioFormat;
+    private timestampsRequested;
+    private readonly timestampTrailer;
+    private readonly rpc;
     constructor(prepared: PreparedConnection, signal: AbortSignal);
     get requestId(): string | null;
     get done(): boolean;
     cancel(): void;
-    private error;
-    private writeMessage;
-    start(voice: string, language: string): Promise<void>;
-    write(sentence: string): Promise<void>;
+    start(voice: string, language: string, timestamps?: boolean, customLexicon?: readonly PronunciationEntry[], audioFormat?: string, sampleRate?: number, timeScaleFactor?: number): Promise<void>;
+    timestampResult(): import("./timestamps.js").TimestampResult;
+    write(text: string): Promise<void>;
     finishInput(): void;
     private checkFormat;
     audio(): AsyncGenerator<Uint8Array>;

package/dist/tts/transport.js

@@ -0,0 +1,133 @@
+import { connect as connectGrpc } from "../grpc.js";
+export { metadata } from "../grpc.js";
+import { BidiCall, serializer, deserializer, requestId, nativeError, } from "../grpc.js";
+export { rpcError } from "../grpc.js";
+import * as grpc from "@grpc/grpc-js";
+import { create, } from "@bufbuild/protobuf";
+import * as schema from "@rimelabs/api";
+import { policy } from "./policy.js";
+import { abortable } from "../cancellation.js";
+import * as errors from "../errors.js";
+import { TimestampTrailer } from "./timestamps.js";
+export const transport = {
+    makeClient: (target) => new grpc.Client(target, grpc.credentials.createSsl(), {
+        "grpc.enable_retries": 0,
+        "grpc.max_receive_message_length": policy.receiveBytes,
+        "grpc.max_send_message_length": 131072,
+    }),
+};
+export function connect(client, signal) {
+    return connectGrpc(client, signal, policy.connectionTimeout);
+}
+function openStream(client, auth) {
+    return client.makeBidiStreamRequest("/rime.TextToSpeech/SynthesizeStreaming", serializer(schema.StreamingSynthesisRequestSchema), deserializer(schema.SynthesisResponseStreamSchema), auth);
+}
+function header(voice, language, timestamps, customLexicon, audioFormat, samplingRate, timeScaleFactor) {
+    return create(schema.StreamingSynthesisRequestSchema, {
+        payload: {
+            case: "header",
+            value: create(schema.SynthesisRequestSchema, {
+                speaker: voice,
+                language,
+                splitStrategy: schema.SplitStrategy.ACCUMULATE_SENTENCE,
+                audioParameters: { audioFormat, samplingRate, timeScaleFactor },
+                timestamps: timestamps ? { enable: true } : undefined,
+                customLexicon: [...customLexicon],
+            }),
+        },
+    });
+}
+function textMessage(text) {
+    return create(schema.StreamingSynthesisRequestSchema, {
+        payload: { case: "textChunk", value: text },
+    });
+}
+async function unary(prepared, name, input, output, request, signal) {
+    let call;
+    let id = null;
+    try {
+        signal.throwIfAborted();
+        const operation = new Promise((resolve, reject) => {
+            call = prepared.client.makeUnaryRequest("/rime.TextToSpeech/" + name, serializer(input), deserializer(output), request, prepared.metadata, (error, result) => (error ? reject(error) : resolve(result)));
+            call.once("metadata", (metadata) => {
+                id = requestId(metadata);
+            });
+        });
+        return await abortable(operation, signal);
+    }
+    catch (error) {
+        call?.cancel();
+        if (signal.aborted) {
+            if (signal.reason instanceof errors.RimeTimeoutError)
+                throw new errors.RimeTimeoutError(signal.reason.message, id, {
+                    cause: signal.reason.cause,
+                });
+            throw signal.reason;
+        }
+        throw nativeError(error, id);
+    }
+}
+export async function discover(prepared, kind, language, signal) {
+    // Exactly one attempt. The client owns retries and the shared deadline.
+    if (kind === "voices") {
+        const method = schema.TextToSpeech.method.getSupportedSpeakers;
+        const response = await unary(prepared, method.name, method.input, method.output, create(method.input, { language: language ?? undefined }), signal);
+        return response.speakers;
+    }
+    const method = schema.TextToSpeech.method.getSupportedLanguages;
+    const response = await unary(prepared, method.name, method.input, method.output, create(method.input), signal);
+    return response.languages;
+}
+export class SynthesisCall {
+    audioFormat = "audio/pcm";
+    timestampsRequested = false;
+    timestampTrailer = new TimestampTrailer();
+    rpc;
+    constructor(prepared, signal) {
+        this.rpc = new BidiCall(openStream(prepared.client, prepared.metadata), signal);
+    }
+    get requestId() {
+        return this.rpc.requestId;
+    }
+    get done() {
+        return this.rpc.done;
+    }
+    cancel() {
+        this.rpc.cancel();
+    }
+    start(voice, language, timestamps = false, customLexicon = [], audioFormat = "audio/pcm", sampleRate = 24000, timeScaleFactor) {
+        this.timestampsRequested = timestamps;
+        this.audioFormat = audioFormat;
+        const message = header(voice, language, timestamps, customLexicon, audioFormat, sampleRate, timeScaleFactor);
+        return this.rpc.write(message);
+    }
+    timestampResult() {
+        return this.timestampTrailer.result(this.requestId);
+    }
+    write(text) {
+        return this.rpc.write(textMessage(text));
+    }
+    finishInput() {
+        this.rpc.finishInput();
+    }
+    checkFormat(contentType) {
+        if (contentType !== this.audioFormat)
+            throw new errors.RimeAudioFormatError(`Expected ${this.audioFormat} from the service`, this.requestId);
+    }
+    async *audio() {
+        const headers = await this.rpc.headers();
+        const contentType = headers.get("x-rime-audio-content-type")[0];
+        for await (const response of this.rpc.responses()) {
+            if (response.payload.case === "trailer" && this.timestampsRequested)
+                this.timestampTrailer.accept(response.payload.value);
+            if (response.payload.case !== "audio")
+                continue;
+            if (this.timestampTrailer.seen)
+                this.timestampTrailer.invalid = true;
+            if (response.payload.value.length)
+                this.checkFormat(contentType);
+            yield response.payload.value;
+        }
+        this.checkFormat(contentType);
+    }
+}

package/dist/validation.d.ts

@@ -0,0 +1,5 @@
+export declare function endpointAddress(endpoint: string): {
+    target: string;
+    audience: string;
+};
+export declare function timeout(value: unknown, inherited?: number | null): number | null;

package/dist/validation.js

@@ -0,0 +1,28 @@
+import { RimeInputError } from "./errors.js";
+export function endpointAddress(endpoint) {
+    const error = "endpoint must be a hostname with an optional port (1-65535), without a scheme or path";
+    if (typeof endpoint !== "string" || /\s/.test(endpoint))
+        throw new RimeInputError(error);
+    const [host, port, extra] = endpoint.split(":");
+    if (!host ||
+        host.length > 253 ||
+        extra !== undefined ||
+        host
+            .split(".")
+            .some((label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(label)) ||
+        (port !== undefined &&
+            (!/^[0-9]{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65535)))
+        throw new RimeInputError(error);
+    return {
+        target: `${host.toLowerCase()}:${port === undefined ? 443 : Number(port)}`,
+        audience: host.toLowerCase(),
+    };
+}
+export function timeout(value, inherited = null) {
+    if (value === undefined)
+        return inherited;
+    if (value !== null &&
+        (typeof value !== "number" || !Number.isFinite(value) || value <= 0))
+        throw new RimeInputError("timeout must be finite positive seconds or null");
+    return value;
+}