@elizaos/app-core 2.0.0-beta.3 → 2.0.11-beta.6

package/services/inference-abort.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Per-runtime in-flight inference AbortController registry.
+ *
+ * Wave 3C's `useAppLifecycleEvents` hook (`packages/ui/src/state/
+ * useAppLifecycleEvents.ts`) fires `chatAbortRef.current?.abort()` on
+ * `APP_PAUSE_EVENT` to cancel UI-side streams before iOS suspends the
+ * WKWebView. That covers the UI's fetch streams. This module covers the
+ * runtime side: inference paths internal to the agent (the AOSP llama FFI
+ * adapter, the cloud-fallback wrapper, and local model calls) can register
+ * their `AbortController` here so a single hook can
+ * abort ALL of them at once on pause / shutdown / account switch.
+ *
+ * Contract:
+ *  - `trackInflight(runtime, ctrl)` returns a disposer. Callers MUST call
+ *    the disposer in their `finally` block so completed calls don't keep
+ *    references alive.
+ *  - `abortInflightInference(runtime)` calls `.abort()` on every tracked
+ *    controller for the runtime and clears the set. Returns the count so
+ *    the caller can log how many were canceled.
+ *  - WeakMap-keyed by runtime so per-account or test-runtime instances
+ *    don't leak across each other.
+ */
+import type { IAgentRuntime } from "@elizaos/core";
+/**
+ * Register a fresh `AbortController` with the runtime's in-flight set.
+ * Returns a disposer that removes the controller from the set; callers
+ * MUST invoke it in the finally block so completed calls are GC'd.
+ */
+export declare function trackInflight(runtime: IAgentRuntime, controller: AbortController): () => void;
+/**
+ * Abort every in-flight inference controller for the runtime. Called by
+ * the UI on `APP_PAUSE_EVENT` and by other shutdown paths (account
+ * switch, hard logout, runtime teardown).
+ *
+ * Returns `{aborted}` so the caller can emit a structured log line.
+ * Idempotent — calling on a runtime with no in-flight work returns
+ * `{aborted: 0}` and does nothing.
+ */
+export declare function abortInflightInference(runtime: IAgentRuntime): {
+    aborted: number;
+};
+/**
+ * Inspect the current in-flight count without aborting. Used by
+ * diagnostics endpoints (e.g. `/api/health` extension) and tests.
+ */
+export declare function getInflightInferenceCount(runtime: IAgentRuntime): number;
+//# sourceMappingURL=inference-abort.d.ts.map

package/services/inference-abort.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference-abort.d.ts","sourceRoot":"","sources":["../../../../../src/services/inference-abort.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAInD;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,aAAa,EACtB,UAAU,EAAE,eAAe,GAC1B,MAAM,IAAI,CAUZ;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,aAAa,GAAG;IAC9D,OAAO,EAAE,MAAM,CAAC;CACjB,CAWA;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAExE"}

package/services/inference-abort.js

@@ -0,0 +1,76 @@
+/**
+ * Per-runtime in-flight inference AbortController registry.
+ *
+ * Wave 3C's `useAppLifecycleEvents` hook (`packages/ui/src/state/
+ * useAppLifecycleEvents.ts`) fires `chatAbortRef.current?.abort()` on
+ * `APP_PAUSE_EVENT` to cancel UI-side streams before iOS suspends the
+ * WKWebView. That covers the UI's fetch streams. This module covers the
+ * runtime side: inference paths internal to the agent (the AOSP llama FFI
+ * adapter, the cloud-fallback wrapper, and local model calls) can register
+ * their `AbortController` here so a single hook can
+ * abort ALL of them at once on pause / shutdown / account switch.
+ *
+ * Contract:
+ *  - `trackInflight(runtime, ctrl)` returns a disposer. Callers MUST call
+ *    the disposer in their `finally` block so completed calls don't keep
+ *    references alive.
+ *  - `abortInflightInference(runtime)` calls `.abort()` on every tracked
+ *    controller for the runtime and clears the set. Returns the count so
+ *    the caller can log how many were canceled.
+ *  - WeakMap-keyed by runtime so per-account or test-runtime instances
+ *    don't leak across each other.
+ */
+const trackers = new WeakMap();
+/**
+ * Register a fresh `AbortController` with the runtime's in-flight set.
+ * Returns a disposer that removes the controller from the set; callers
+ * MUST invoke it in the finally block so completed calls are GC'd.
+ */
+export function trackInflight(runtime, controller) {
+    let set = trackers.get(runtime);
+    if (!set) {
+        set = new Set();
+        trackers.set(runtime, set);
+    }
+    set.add(controller);
+    return () => {
+        set.delete(controller);
+    };
+}
+/**
+ * Abort every in-flight inference controller for the runtime. Called by
+ * the UI on `APP_PAUSE_EVENT` and by other shutdown paths (account
+ * switch, hard logout, runtime teardown).
+ *
+ * Returns `{aborted}` so the caller can emit a structured log line.
+ * Idempotent — calling on a runtime with no in-flight work returns
+ * `{aborted: 0}` and does nothing.
+ */
+export function abortInflightInference(runtime) {
+    const set = trackers.get(runtime);
+    if (!set || set.size === 0) {
+        return { aborted: 0 };
+    }
+    const count = set.size;
+    for (const controller of set) {
+        controller.abort();
+    }
+    set.clear();
+    return { aborted: count };
+}
+/**
+ * Inspect the current in-flight count without aborting. Used by
+ * diagnostics endpoints (e.g. `/api/health` extension) and tests.
+ */
+export function getInflightInferenceCount(runtime) {
+    return trackers.get(runtime)?.size ?? 0;
+}
+/**
+ * Test-only reset. Wipes the runtime's tracker entirely. Do NOT call
+ * from production code.
+ *
+ * @internal
+ */
+export function __resetInflightInferenceForTests(runtime) {
+    trackers.delete(runtime);
+}

package/services/persistence.d.ts

@@ -10,9 +10,8 @@
  * Preferences when it isn't (e.g. older mobile shells that haven't shipped
  * the SQLite plugin yet, or web preview).
  *
- * Token storage MUST go through Keychain on iOS — handled by a separate
- * follow-up identified in the prior security review and intentionally NOT
- * touched here.
+ * Token storage MUST go through Keychain on iOS — handled separately by
+ * `platform-secure-store-node.ts`.
  */
 export interface PersistenceAdapter {
     readonly kind: "ios-preferences" | "ios-sqlite" | "web-localstorage";

package/services/persistence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../../../../src/services/persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAQH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,GAAG,YAAY,GAAG,kBAAkB,CAAC;IACrE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACzC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,0DAA0D;IAC1D,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAC3B;AAmFD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,iBAAiB,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1C,iBAAiB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACrD,yBAAyB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAC9D;AAeD;;;;GAIG;AACH,wBAAsB,4BAA4B,CAChD,IAAI,GAAE,cAA+B,GACpC,OAAO,CAAC,kBAAkB,CAAC,CAK7B;AAED;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAC9C,WAAW,EAAE,kBAAkB,EAC/B,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,MAAM,CAAC,CAoBjB"}
+{"version":3,"file":"persistence.d.ts","sourceRoot":"","sources":["../../../../../src/services/persistence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAQH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,GAAG,YAAY,GAAG,kBAAkB,CAAC;IACrE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACzC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,0DAA0D;IAC1D,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAC3B;AAmFD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,iBAAiB,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1C,iBAAiB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACrD,yBAAyB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAC9D;AAeD;;;;GAIG;AACH,wBAAsB,4BAA4B,CAChD,IAAI,GAAE,cAA+B,GACpC,OAAO,CAAC,kBAAkB,CAAC,CAK7B;AAED;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAC9C,WAAW,EAAE,kBAAkB,EAC/B,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,MAAM,CAAC,CAoBjB"}

package/services/persistence.js

@@ -10,9 +10,8 @@
  * Preferences when it isn't (e.g. older mobile shells that haven't shipped
  * the SQLite plugin yet, or web preview).
  *
- * Token storage MUST go through Keychain on iOS — handled by a separate
- * follow-up identified in the prior security review and intentionally NOT
- * touched here.
+ * Token storage MUST go through Keychain on iOS — handled separately by
+ * `platform-secure-store-node.ts`.
  */
 import { isSqliteAvailable, openDatabase, } from "../connectors/capacitor-sqlite.js";
 class IosPreferencesAdapter {

package/services/phrase-chunked-tts.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Stream-driven phrase-chunked TTS adapter.
+ *
+ * Wraps any TTS handler (Edge TTS, ElevenLabs, OpenAI, omnivoice — anything
+ * that takes a string and returns audio) so that a streaming LLM response
+ * can be spoken out progressively: the first sentence/clause hits TTS as
+ * soon as the chunker accepts a boundary, then subsequent phrases follow
+ * while the LLM is still emitting tokens.
+ *
+ * Why this lives in app-core: the canonical phrase boundary policy
+ * (`PhraseChunker` — commas/semicolons/colons + 30-word cap + 700 ms time
+ * budget) is owned by the local-inference voice subsystem. This adapter
+ * makes that policy reusable from the REMOTE chat→TTS path (Discord,
+ * dashboard SSE→synthesis, phone) where the LLM streams text but the TTS
+ * backend only accepts complete strings.
+ *
+ * Latency contract:
+ *   - First TTS call fires on the first punctuation boundary OR on the
+ *     max-token cap (default 30 words), whichever lands first.
+ *   - If the producer stalls between tokens for >= the time-budget window
+ *     (default 700 ms), the in-flight phrase is force-flushed so audio
+ *     never waits on a slow upstream.
+ *   - `finish()` drains the tail phrase exactly once.
+ *
+ * This module is intentionally side-effect free: callers supply the TTS
+ * function and the audio sink. We don't decode, we don't queue audio —
+ * we just hand sentence-sized strings to the TTS function in order.
+ */
+export interface AcceptedToken {
+    index: number;
+    text: string;
+    id?: number;
+    acceptedAt: number;
+}
+export interface Phrase {
+    id: number;
+    text: string;
+    fromIndex: number;
+    toIndex: number;
+    terminator: "punctuation" | "max-cap" | "phoneme-stream";
+}
+export interface PhraseChunkerConfig {
+    maxTokensPerPhrase?: number;
+    sentenceTerminators?: ReadonlySet<string>;
+    chunkOn?: "punctuation" | "phoneme-stream";
+    phonemesPerChunk?: number;
+    maxAccumulationMs?: number;
+}
+export interface PhraseChunkedTtsOptions {
+    /** Phrase chunker configuration. See `PhraseChunkerConfig`. */
+    chunker?: PhraseChunkerConfig;
+    /**
+     * Optional clock override for tests. Defaults to `performance.now()`.
+     */
+    clock?: () => number;
+    /**
+     * Called once per phrase that's been handed to the TTS handler, after
+     * the handler resolves. Receives the phrase + TTS output. Use this to
+     * push audio into your sink (HTTP chunked transfer, MediaSource, Discord
+     * AudioPlayer, etc.).
+     */
+    onAudio?: (phrase: Phrase, audio: unknown) => void | Promise<void>;
+    /**
+     * Called when the chunker emits a phrase but BEFORE the TTS handler is
+     * invoked. Use for tracing — e.g. `markVoiceLatency(roomId, "phrase-1-to-tts")`.
+     */
+    onPhraseEmit?: (phrase: Phrase) => void;
+    /**
+     * Called when a TTS handler call rejects. Default: rethrow on
+     * `finish()`. Set to `"swallow"` to log + continue (next phrase still
+     * goes to TTS). The handler error itself is passed for logging.
+     */
+    onTtsError?: (phrase: Phrase, err: unknown) => "swallow" | "fail";
+}
+export type TtsHandler = (text: string) => Promise<unknown> | unknown;
+/**
+ * A phrase-chunked TTS pipeline. Push raw streaming chunks via `push()`;
+ * call `finish()` after the LLM stream ends to flush the tail phrase and
+ * await all in-flight TTS calls.
+ *
+ * Usage:
+ *   const pipe = new PhraseChunkedTts(ttsHandler, { onAudio: writeToSink });
+ *   for await (const chunk of llmStream) pipe.push(chunk);
+ *   await pipe.finish();
+ */
+export declare class PhraseChunkedTts {
+    private readonly chunker;
+    private readonly clock;
+    private readonly tts;
+    private readonly opts;
+    private tokenIndex;
+    /** In-flight TTS promises so `finish()` can await playback ordering. */
+    private readonly inflight;
+    private timeBudgetTimer;
+    private firstError;
+    private closed;
+    constructor(tts: TtsHandler, opts?: PhraseChunkedTtsOptions);
+    /**
+     * Pre-load the PhraseChunker class from plugin-local-inference. Must be
+     * called (and awaited) before the first `new PhraseChunkedTts()` when the
+     * module has not been statically imported. Idempotent — safe to call
+     * multiple times concurrently.
+     */
+    static load(): Promise<void>;
+    /**
+     * Push a streaming LLM chunk. May span multiple words. Phrase boundaries
+     * are detected by the chunker; when one fires, the phrase text is handed
+     * to the TTS handler immediately (its returned promise is tracked).
+     */
+    push(chunk: string): void;
+    /**
+     * Drain the tail phrase (if any) and await all in-flight TTS calls.
+     * Returns when every dispatched phrase has resolved or rejected. After
+     * `finish()` the pipeline cannot be reused.
+     */
+    finish(): Promise<void>;
+    /** Cancel without awaiting in-flight TTS. Safe to call after `finish()`. */
+    cancel(): void;
+    private dispatchPhrase;
+    /**
+     * Re-arm the time-budget timer. The chunker has an internal time-budget
+     * flush, but it only checks at `push()` time — a producer that goes
+     * silent before pushing the next token would stall an in-flight phrase
+     * forever. The timer polls the chunker once the budget window elapses
+     * with no new pushes.
+     */
+    private scheduleTimeBudget;
+}
+/**
+ * One-shot helper: drive an async iterable of text chunks through
+ * `PhraseChunkedTts`. Returns when the iterable is exhausted and all TTS
+ * calls have settled. Exists so SSE/streaming callers don't have to write
+ * the for-await boilerplate.
+ */
+export declare function speakStreamingText(source: AsyncIterable<string>, tts: TtsHandler, opts?: PhraseChunkedTtsOptions): Promise<void>;
+//# sourceMappingURL=phrase-chunked-tts.d.ts.map

package/services/phrase-chunked-tts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"phrase-chunked-tts.d.ts","sourceRoot":"","sources":["../../../../../src/services/phrase-chunked-tts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,aAAa,GAAG,SAAS,GAAG,gBAAgB,CAAC;CAC1D;AAED,MAAM,WAAW,mBAAmB;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,mBAAmB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1C,OAAO,CAAC,EAAE,aAAa,GAAG,gBAAgB,CAAC;IAC3C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AA0CD,MAAM,WAAW,uBAAuB;IACtC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;IACrB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE;;;OAGG;IACH,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,KAAK,SAAS,GAAG,MAAM,CAAC;CACnE;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;AAEtE;;;;;;;;;GASG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAa;IACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0B;IAC/C,OAAO,CAAC,UAAU,CAAK;IACvB,wEAAwE;IACxE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0B;IACnD,OAAO,CAAC,eAAe,CAA8C;IACrE,OAAO,CAAC,UAAU,CAAiB;IACnC,OAAO,CAAC,MAAM,CAAS;gBAEX,GAAG,EAAE,UAAU,EAAE,IAAI,GAAE,uBAA4B;IAY/D;;;;;OAKG;WACU,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAIlC;;;;OAIG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAiBzB;;;;OAIG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAe7B,4EAA4E;IAC5E,MAAM,IAAI,IAAI;IASd,OAAO,CAAC,cAAc;IAmBtB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;CAyB3B;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAC7B,GAAG,EAAE,UAAU,EACf,IAAI,GAAE,uBAA4B,GACjC,OAAO,CAAC,IAAI,CAAC,CAQf"}

package/services/phrase-chunked-tts.js

@@ -0,0 +1,208 @@
+/**
+ * Stream-driven phrase-chunked TTS adapter.
+ *
+ * Wraps any TTS handler (Edge TTS, ElevenLabs, OpenAI, omnivoice — anything
+ * that takes a string and returns audio) so that a streaming LLM response
+ * can be spoken out progressively: the first sentence/clause hits TTS as
+ * soon as the chunker accepts a boundary, then subsequent phrases follow
+ * while the LLM is still emitting tokens.
+ *
+ * Why this lives in app-core: the canonical phrase boundary policy
+ * (`PhraseChunker` — commas/semicolons/colons + 30-word cap + 700 ms time
+ * budget) is owned by the local-inference voice subsystem. This adapter
+ * makes that policy reusable from the REMOTE chat→TTS path (Discord,
+ * dashboard SSE→synthesis, phone) where the LLM streams text but the TTS
+ * backend only accepts complete strings.
+ *
+ * Latency contract:
+ *   - First TTS call fires on the first punctuation boundary OR on the
+ *     max-token cap (default 30 words), whichever lands first.
+ *   - If the producer stalls between tokens for >= the time-budget window
+ *     (default 700 ms), the in-flight phrase is force-flushed so audio
+ *     never waits on a slow upstream.
+ *   - `finish()` drains the tail phrase exactly once.
+ *
+ * This module is intentionally side-effect free: callers supply the TTS
+ * function and the audio sink. We don't decode, we don't queue audio —
+ * we just hand sentence-sized strings to the TTS function in order.
+ */
+// Lazy module reference — avoids a static boundary violation while preserving
+// synchronous construction semantics for callers that supply the PhraseChunker
+// class via `PhraseChunkedTtsOptions.chunkerClass` (used in tests).
+let _PhraseChunkerClass;
+let _servicesImport;
+function requirePhraseChunker() {
+    if (_PhraseChunkerClass)
+        return _PhraseChunkerClass;
+    throw new Error("PhraseChunker class unavailable — call await PhraseChunkedTts.load() before constructing instances, or provide chunkerClass in options.");
+}
+/** Pre-warm the PhraseChunker class. Called automatically when the first
+ *  PhraseChunkedTts is instantiated if the class is not already loaded. */
+async function ensurePhraseChunkerLoaded() {
+    if (_PhraseChunkerClass)
+        return;
+    _servicesImport ??= import("@elizaos/plugin-local-inference/services");
+    const mod = await _servicesImport;
+    _PhraseChunkerClass = mod.PhraseChunker;
+}
+/**
+ * A phrase-chunked TTS pipeline. Push raw streaming chunks via `push()`;
+ * call `finish()` after the LLM stream ends to flush the tail phrase and
+ * await all in-flight TTS calls.
+ *
+ * Usage:
+ *   const pipe = new PhraseChunkedTts(ttsHandler, { onAudio: writeToSink });
+ *   for await (const chunk of llmStream) pipe.push(chunk);
+ *   await pipe.finish();
+ */
+export class PhraseChunkedTts {
+    chunker;
+    clock;
+    tts;
+    opts;
+    tokenIndex = 0;
+    /** In-flight TTS promises so `finish()` can await playback ordering. */
+    inflight = [];
+    timeBudgetTimer = null;
+    firstError = null;
+    closed = false;
+    constructor(tts, opts = {}) {
+        this.tts = tts;
+        this.opts = opts;
+        this.clock = opts.clock ?? (() => globalThis.performance.now());
+        const Chunker = requirePhraseChunker();
+        this.chunker = new Chunker(opts.chunker ?? { chunkOn: "punctuation" }, null, this.clock);
+    }
+    /**
+     * Pre-load the PhraseChunker class from plugin-local-inference. Must be
+     * called (and awaited) before the first `new PhraseChunkedTts()` when the
+     * module has not been statically imported. Idempotent — safe to call
+     * multiple times concurrently.
+     */
+    static async load() {
+        await ensurePhraseChunkerLoaded();
+    }
+    /**
+     * Push a streaming LLM chunk. May span multiple words. Phrase boundaries
+     * are detected by the chunker; when one fires, the phrase text is handed
+     * to the TTS handler immediately (its returned promise is tracked).
+     */
+    push(chunk) {
+        if (this.closed) {
+            throw new Error("PhraseChunkedTts.push() called after finish()");
+        }
+        if (!chunk)
+            return;
+        const token = {
+            index: this.tokenIndex++,
+            text: chunk,
+            acceptedAt: this.clock(),
+        };
+        const phrase = this.chunker.push(token);
+        if (phrase) {
+            this.dispatchPhrase(phrase);
+        }
+        this.scheduleTimeBudget();
+    }
+    /**
+     * Drain the tail phrase (if any) and await all in-flight TTS calls.
+     * Returns when every dispatched phrase has resolved or rejected. After
+     * `finish()` the pipeline cannot be reused.
+     */
+    async finish() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        if (this.timeBudgetTimer) {
+            clearTimeout(this.timeBudgetTimer);
+            this.timeBudgetTimer = null;
+        }
+        const tail = this.chunker.flushPending();
+        if (tail)
+            this.dispatchPhrase(tail);
+        // Settle every TTS call; rethrow the first error if any (unless caller
+        // chose to swallow).
+        await Promise.all(this.inflight.map((p) => p.catch(() => undefined)));
+        if (this.firstError !== null)
+            throw this.firstError;
+    }
+    /** Cancel without awaiting in-flight TTS. Safe to call after `finish()`. */
+    cancel() {
+        this.closed = true;
+        if (this.timeBudgetTimer) {
+            clearTimeout(this.timeBudgetTimer);
+            this.timeBudgetTimer = null;
+        }
+        this.chunker.reset();
+    }
+    dispatchPhrase(phrase) {
+        this.opts.onPhraseEmit?.(phrase);
+        const ttsCall = Promise.resolve()
+            .then(() => this.tts(phrase.text))
+            .then(async (audio) => {
+            if (this.opts.onAudio) {
+                await this.opts.onAudio(phrase, audio);
+            }
+            return audio;
+        })
+            .catch((err) => {
+            const mode = this.opts.onTtsError?.(phrase, err) ?? "fail";
+            if (mode === "swallow")
+                return undefined;
+            if (this.firstError === null)
+                this.firstError = err;
+            return undefined;
+        });
+        this.inflight.push(ttsCall);
+    }
+    /**
+     * Re-arm the time-budget timer. The chunker has an internal time-budget
+     * flush, but it only checks at `push()` time — a producer that goes
+     * silent before pushing the next token would stall an in-flight phrase
+     * forever. The timer polls the chunker once the budget window elapses
+     * with no new pushes.
+     */
+    scheduleTimeBudget() {
+        if (this.closed)
+            return;
+        if (this.timeBudgetTimer) {
+            clearTimeout(this.timeBudgetTimer);
+            this.timeBudgetTimer = null;
+        }
+        const msUntil = this.chunker.msUntilTimeBudget();
+        if (!Number.isFinite(msUntil) || msUntil <= 0) {
+            // Already past the budget — flush now.
+            const flushed = this.chunker.flushIfTimeBudgetExceeded();
+            if (flushed)
+                this.dispatchPhrase(flushed);
+            return;
+        }
+        this.timeBudgetTimer = setTimeout(() => {
+            this.timeBudgetTimer = null;
+            const flushed = this.chunker.flushIfTimeBudgetExceeded();
+            if (flushed)
+                this.dispatchPhrase(flushed);
+        }, Math.max(1, Math.ceil(msUntil)));
+        // Don't keep the event loop alive for a stalled chunker.
+        const t = this.timeBudgetTimer;
+        if (t?.unref)
+            t.unref();
+    }
+}
+/**
+ * One-shot helper: drive an async iterable of text chunks through
+ * `PhraseChunkedTts`. Returns when the iterable is exhausted and all TTS
+ * calls have settled. Exists so SSE/streaming callers don't have to write
+ * the for-await boilerplate.
+ */
+export async function speakStreamingText(source, tts, opts = {}) {
+    await ensurePhraseChunkerLoaded();
+    const pipe = new PhraseChunkedTts(tts, opts);
+    try {
+        for await (const chunk of source)
+            pipe.push(chunk);
+    }
+    finally {
+        await pipe.finish();
+    }
+}

package/services/sandbox-registry.d.ts

@@ -0,0 +1,78 @@
+/**
+ * SandboxRegistry — self-registers the sandbox container in the shared Redis
+ * so the multi-tenant gateways (`gateway-discord`, `gateway-webhook`) can
+ * resolve `agent_id -> server URL` and forward inbound platform messages here.
+ *
+ * This is a deliberate duplicate of `packages/agent/src/runtime/sandbox-registry.ts`:
+ * the published cloud image runs the agent runtime entrypoint, which cannot
+ * import `@elizaos/app-core` without an `agent -> app-core -> agent` cycle, so
+ * the agent copy is the one that executes in a provisioned container. Keep the
+ * two copies in lock-step.
+ *
+ * Two Redis keys are written with a short TTL; a periodic heartbeat refreshes
+ * the TTL while the sandbox is alive, and `unregister()` deletes them on
+ * graceful shutdown if they still point at this sandbox. If the container
+ * crashes, the keys expire naturally and the gateways stop routing to a dead
+ * address.
+ *
+ *   server:<serverName>:url = <serverUrl>   (resolver address)
+ *   agent:<agentId>:server  = <serverName>  (agent -> server pointer)
+ *
+ * Two transports are supported, selected by the URL scheme so the same
+ * registry works before and after the managed Redis is migrated off Upstash:
+ *   - `http(s)://` — Upstash REST API via `fetch` (the pipeline endpoint
+ *     applies both SET-with-EX commands atomically server-side).
+ *   - `redis(s)://` — native RESP over a TCP socket (e.g. a Railway Redis
+ *     public proxy). Auth is carried inline in the URL, so no separate token
+ *     is required.
+ */
+export interface SandboxRegistryConfig {
+    redisUrl: string;
+    /**
+     * Bearer token for the Upstash REST transport. Not required (and ignored)
+     * for a `redis://` / `rediss://` URL, which carries auth inline.
+     */
+    redisToken?: string;
+    agentId: string;
+    serverName: string;
+    serverUrl: string;
+    /** TTL for both Redis keys in seconds. Keep this at least 3x the heartbeat interval so one missed tick does not expire a healthy sandbox. */
+    ttlSeconds: number;
+}
+export declare class SandboxRegistry {
+    private readonly config;
+    private heartbeatTimer;
+    private readonly tcp;
+    constructor(config: SandboxRegistryConfig);
+    register(): Promise<void>;
+    refresh(): Promise<void>;
+    unregister(): Promise<void>;
+    startHeartbeat(intervalMs: number): void;
+    stopHeartbeat(): void;
+    /**
+     * Atomic two-key write. Both keys must succeed together — partial state
+     * would let gateways resolve `agent:X:server` to a stale `server:Y:url`
+     * value or miss a routing entry whose other half was just renewed. REST uses
+     * the Upstash pipeline endpoint; TCP pipelines both commands on one socket.
+     */
+    private writeKeys;
+    private get;
+    private command;
+    private pipeline;
+    /**
+     * Execute one or more commands over a native RESP/TCP connection and return
+     * the per-command replies (AUTH/SELECT preamble replies are stripped). One
+     * short-lived connection per call keeps the lifecycle trivial — the registry
+     * only writes twice per heartbeat, so connection churn is negligible.
+     */
+    private tcpExec;
+}
+/**
+ * Reads the SANDBOX_REGISTRY_* and SANDBOX_* env vars and returns a fully
+ * wired `SandboxRegistry`, or `null` if the sandbox context is not
+ * configured (e.g. local dev, non-Hetzner deployment). Caller must call
+ * `register()` and `startHeartbeat(...)` after a successful boot. A `redis://`
+ * URL needs no token (auth is inline); a `http(s)://` Upstash URL requires one.
+ */
+export declare function buildSandboxRegistryFromEnv(env?: NodeJS.ProcessEnv, ttlSeconds?: number): SandboxRegistry | null;
+//# sourceMappingURL=sandbox-registry.d.ts.map

package/services/sandbox-registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sandbox-registry.d.ts","sourceRoot":"","sources":["../../../../../src/services/sandbox-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAiBH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,6IAA6I;IAC7I,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,eAAe;IAId,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHnC,OAAO,CAAC,cAAc,CAA+C;IACrE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAU;gBAED,MAAM,EAAE,qBAAqB;IAIpD,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAOzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAIxB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBjC,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAmBxC,aAAa,IAAI,IAAI;IAOrB;;;;;OAKG;YACW,SAAS;YAST,GAAG;YAKH,OAAO;YAuBP,QAAQ;IA0BtB;;;;;OAKG;YACW,OAAO;CA6DtB;AAoFD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CACzC,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,UAAU,SAAK,GACd,eAAe,GAAG,IAAI,CA0BxB"}