@estuary-ai/sdk 0.4.1 → 0.5.0

package/README.md

@@ -46,6 +46,37 @@ client.sendText('What do you remember about me?');
 client.sendText('Just respond in text', true); // textOnly mode
 ```
 
+### Scripted Lines
+
+Make the character speak exact, prewritten lines (straight to TTS, skipping the LLM):
+
+```typescript
+client.sayLine('Welcome back, traveler.');          // TTS audio
+client.sayLine('System: connection restored.', true); // text only, no audio
+```
+
+For a sequence, use `playScript()` — it paces the lines so each finishes before the next is
+sent. (`say_line` interrupts any in-progress speech, so firing several at once would make each
+line cut off the previous one.)
+
+```typescript
+const script = client.playScript(
+  ['Welcome to my shop, traveler!', 'I have wares, if you have coin.', 'Come back anytime.'],
+  { textOnly: false, lineGapMs: 250 },
+);
+
+client.on('scriptLineStarted', ({ index, text }) => console.log(`line ${index}: ${text}`));
+await script.done; // resolves when all lines have been spoken
+
+script.pause();
+script.resume();
+script.next(); // skip ahead
+script.stop(); // halt + interrupt
+```
+
+`sayLines(lines, opts)` is a convenience alias of `playScript(lines, opts)`. Lines may be plain
+strings or per-line overrides: `{ text: '(a silent stage direction)', textOnly: true }`.
+
 ### Voice (WebSocket)
 
 ```typescript
@@ -182,6 +213,10 @@ client.on('interrupt', (data) => { /* response interrupted */ });
 client.on('characterAction', (action) => { /* parsed action from bot response */ });
 client.on('cameraCaptureRequest', (request) => { /* server requests a camera image */ });
 
+// Scripted lines (playScript / sayLines)
+client.on('scriptLineStarted', ({ index, text, messageId }) => { /* a scripted line began */ });
+client.on('scriptComplete', ({ reason }) => { /* 'finished' | 'stopped' | 'disconnected' | 'interrupted' */ });
+
 // Voice
 client.on('voiceStarted', () => { /* voice session began */ });
 client.on('voiceStopped', () => { /* voice session ended */ });

package/dist/index.d.mts

@@ -156,6 +156,46 @@ interface CharacterAction {
     /** Message ID of the bot response that contained this action */
     messageId: string;
 }
+/** A scripted line: plain text (uses the script's default textOnly) or an explicit override. */
+type ScriptLine = string | {
+    text: string;
+    textOnly?: boolean;
+};
+interface ScriptOptions {
+    /** Default for plain-string lines: false = TTS audio (default), true = text-only. */
+    textOnly?: boolean;
+    /** Pause inserted after each line completes, in ms (default 0). */
+    lineGapMs?: number;
+    /** Begin speaking immediately on creation (default true). If false, call play(). */
+    autoStart?: boolean;
+    /** Repeat from the first line after the last (default false). */
+    loop?: boolean;
+    /** Force-advance a line if no completion signal arrives within this many ms (default 30000). */
+    lineTimeoutMs?: number;
+}
+type ScriptEndReason = 'finished' | 'stopped' | 'disconnected' | 'interrupted';
+type ScriptState = 'idle' | 'playing' | 'paused' | 'done';
+interface ScriptLineStartedInfo {
+    index: number;
+    text: string;
+    messageId: string;
+}
+/** Handle returned by EstuaryClient.playScript() / sayLines(). */
+interface ScriptController {
+    readonly length: number;
+    /** Index of the current / most-recently-started line (-1 before the first line starts). */
+    readonly index: number;
+    readonly state: ScriptState;
+    /** Resolves (never rejects) when the script ends, with the reason. Awaitable. */
+    readonly done: Promise<{
+        reason: ScriptEndReason;
+    }>;
+    play(): void;
+    pause(): void;
+    resume(): void;
+    next(): void;
+    stop(): void;
+}
 type EstuaryEventMap = {
     connected: (session: SessionInfo) => void;
     disconnected: (reason: string) => void;
@@ -179,6 +219,10 @@ type EstuaryEventMap = {
     /** Bot audio level 0.0–1.0, emitted during playback for both transports. */
     botAudioLevel: (level: number) => void;
     memoryUpdated: (event: MemoryUpdatedEvent) => void;
+    scriptLineStarted: (info: ScriptLineStartedInfo) => void;
+    scriptComplete: (info: {
+        reason: ScriptEndReason;
+    }) => void;
 };
 interface VoiceManager {
     start(): Promise<void>;
@@ -360,6 +404,7 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
     private _hasAutoInterrupted;
     private _autoInterruptGraceTimer;
     private _isLiveKitSpeaking;
+    private _activeScript;
     constructor(config: EstuaryConfig);
     /** Memory API client for querying memories, graphs, and facts */
     get memory(): MemoryClient;
@@ -385,6 +430,16 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
     sendText(text: string, textOnly?: boolean): void;
     /** Script the character to say a specific prewritten line. Defaults to TTS enabled (textOnly=false). */
     sayLine(text: string, textOnly?: boolean): void;
+    /**
+     * Script a sequence of prewritten lines. Lines are paced so each finishes before the next
+     * is sent — required because say_line interrupts any in-progress response server-side, so
+     * unpaced lines would stomp each other. Returns a controller (play/pause/resume/next/stop +
+     * an awaitable `done`). Starting a new script stops any currently-active one.
+     */
+    playScript(lines: ScriptLine[], opts?: ScriptOptions): ScriptController;
+    /** Convenience alias of playScript() for fire-and-forget scripted sequences. */
+    sayLines(lines: ScriptLine[], opts?: ScriptOptions): ScriptController;
+    private createScriptHost;
     /** Interrupt the current bot response */
     interrupt(messageId?: string): void;
     /** Send a camera image for vision processing */
@@ -470,4 +525,4 @@ declare class CharacterClient {
     dispose(): void;
 }
 
-export { type BotResponse, type BotVoice, type CameraCaptureRequest, type CharacterAction, CharacterClient, type CharacterInfo, ConnectionState, type CoreFact, type CoreFactsResponse, ErrorCode, EstuaryClient, type EstuaryConfig, EstuaryError, type EstuaryEventMap, type InterruptData, type LiveKitTokenResponse, MemoryClient, type MemoryData, type MemoryGraphEdge, type MemoryGraphNode, type MemoryGraphOptions, type MemoryGraphResponse, type MemoryListOptions, type MemoryListResponse, type MemorySearchOptions, type MemorySearchResponse, type MemoryStatsResponse, type MemoryTimelineOptions, type MemoryTimelineResponse, type MemoryUpdatedEvent, type ParsedAction, type QuotaExceededData, type SessionCapabilities, type SessionInfo, type ShareOpenResponse, type SttResponse, type VoiceManager, type VoiceTransport, parseActions };
+export { type BotResponse, type BotVoice, type CameraCaptureRequest, type CharacterAction, CharacterClient, type CharacterInfo, ConnectionState, type CoreFact, type CoreFactsResponse, ErrorCode, EstuaryClient, type EstuaryConfig, EstuaryError, type EstuaryEventMap, type InterruptData, type LiveKitTokenResponse, MemoryClient, type MemoryData, type MemoryGraphEdge, type MemoryGraphNode, type MemoryGraphOptions, type MemoryGraphResponse, type MemoryListOptions, type MemoryListResponse, type MemorySearchOptions, type MemorySearchResponse, type MemoryStatsResponse, type MemoryTimelineOptions, type MemoryTimelineResponse, type MemoryUpdatedEvent, type ParsedAction, type QuotaExceededData, type ScriptController, type ScriptEndReason, type ScriptLine, type ScriptLineStartedInfo, type ScriptOptions, type ScriptState, type SessionCapabilities, type SessionInfo, type ShareOpenResponse, type SttResponse, type VoiceManager, type VoiceTransport, parseActions };

package/dist/index.d.ts

@@ -156,6 +156,46 @@ interface CharacterAction {
     /** Message ID of the bot response that contained this action */
     messageId: string;
 }
+/** A scripted line: plain text (uses the script's default textOnly) or an explicit override. */
+type ScriptLine = string | {
+    text: string;
+    textOnly?: boolean;
+};
+interface ScriptOptions {
+    /** Default for plain-string lines: false = TTS audio (default), true = text-only. */
+    textOnly?: boolean;
+    /** Pause inserted after each line completes, in ms (default 0). */
+    lineGapMs?: number;
+    /** Begin speaking immediately on creation (default true). If false, call play(). */
+    autoStart?: boolean;
+    /** Repeat from the first line after the last (default false). */
+    loop?: boolean;
+    /** Force-advance a line if no completion signal arrives within this many ms (default 30000). */
+    lineTimeoutMs?: number;
+}
+type ScriptEndReason = 'finished' | 'stopped' | 'disconnected' | 'interrupted';
+type ScriptState = 'idle' | 'playing' | 'paused' | 'done';
+interface ScriptLineStartedInfo {
+    index: number;
+    text: string;
+    messageId: string;
+}
+/** Handle returned by EstuaryClient.playScript() / sayLines(). */
+interface ScriptController {
+    readonly length: number;
+    /** Index of the current / most-recently-started line (-1 before the first line starts). */
+    readonly index: number;
+    readonly state: ScriptState;
+    /** Resolves (never rejects) when the script ends, with the reason. Awaitable. */
+    readonly done: Promise<{
+        reason: ScriptEndReason;
+    }>;
+    play(): void;
+    pause(): void;
+    resume(): void;
+    next(): void;
+    stop(): void;
+}
 type EstuaryEventMap = {
     connected: (session: SessionInfo) => void;
     disconnected: (reason: string) => void;
@@ -179,6 +219,10 @@ type EstuaryEventMap = {
     /** Bot audio level 0.0–1.0, emitted during playback for both transports. */
     botAudioLevel: (level: number) => void;
     memoryUpdated: (event: MemoryUpdatedEvent) => void;
+    scriptLineStarted: (info: ScriptLineStartedInfo) => void;
+    scriptComplete: (info: {
+        reason: ScriptEndReason;
+    }) => void;
 };
 interface VoiceManager {
     start(): Promise<void>;
@@ -360,6 +404,7 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
     private _hasAutoInterrupted;
     private _autoInterruptGraceTimer;
     private _isLiveKitSpeaking;
+    private _activeScript;
     constructor(config: EstuaryConfig);
     /** Memory API client for querying memories, graphs, and facts */
     get memory(): MemoryClient;
@@ -385,6 +430,16 @@ declare class EstuaryClient extends TypedEventEmitter<EstuaryEventMap> {
     sendText(text: string, textOnly?: boolean): void;
     /** Script the character to say a specific prewritten line. Defaults to TTS enabled (textOnly=false). */
     sayLine(text: string, textOnly?: boolean): void;
+    /**
+     * Script a sequence of prewritten lines. Lines are paced so each finishes before the next
+     * is sent — required because say_line interrupts any in-progress response server-side, so
+     * unpaced lines would stomp each other. Returns a controller (play/pause/resume/next/stop +
+     * an awaitable `done`). Starting a new script stops any currently-active one.
+     */
+    playScript(lines: ScriptLine[], opts?: ScriptOptions): ScriptController;
+    /** Convenience alias of playScript() for fire-and-forget scripted sequences. */
+    sayLines(lines: ScriptLine[], opts?: ScriptOptions): ScriptController;
+    private createScriptHost;
     /** Interrupt the current bot response */
     interrupt(messageId?: string): void;
     /** Send a camera image for vision processing */
@@ -470,4 +525,4 @@ declare class CharacterClient {
     dispose(): void;
 }
 
-export { type BotResponse, type BotVoice, type CameraCaptureRequest, type CharacterAction, CharacterClient, type CharacterInfo, ConnectionState, type CoreFact, type CoreFactsResponse, ErrorCode, EstuaryClient, type EstuaryConfig, EstuaryError, type EstuaryEventMap, type InterruptData, type LiveKitTokenResponse, MemoryClient, type MemoryData, type MemoryGraphEdge, type MemoryGraphNode, type MemoryGraphOptions, type MemoryGraphResponse, type MemoryListOptions, type MemoryListResponse, type MemorySearchOptions, type MemorySearchResponse, type MemoryStatsResponse, type MemoryTimelineOptions, type MemoryTimelineResponse, type MemoryUpdatedEvent, type ParsedAction, type QuotaExceededData, type SessionCapabilities, type SessionInfo, type ShareOpenResponse, type SttResponse, type VoiceManager, type VoiceTransport, parseActions };
+export { type BotResponse, type BotVoice, type CameraCaptureRequest, type CharacterAction, CharacterClient, type CharacterInfo, ConnectionState, type CoreFact, type CoreFactsResponse, ErrorCode, EstuaryClient, type EstuaryConfig, EstuaryError, type EstuaryEventMap, type InterruptData, type LiveKitTokenResponse, MemoryClient, type MemoryData, type MemoryGraphEdge, type MemoryGraphNode, type MemoryGraphOptions, type MemoryGraphResponse, type MemoryListOptions, type MemoryListResponse, type MemorySearchOptions, type MemorySearchResponse, type MemoryStatsResponse, type MemoryTimelineOptions, type MemoryTimelineResponse, type MemoryUpdatedEvent, type ParsedAction, type QuotaExceededData, type ScriptController, type ScriptEndReason, type ScriptLine, type ScriptLineStartedInfo, type ScriptOptions, type ScriptState, type SessionCapabilities, type SessionInfo, type ShareOpenResponse, type SttResponse, type VoiceManager, type VoiceTransport, parseActions };

package/dist/index.js

@@ -9624,6 +9624,231 @@ function parseActions(text) {
   return parser.parse(text);
 }
 
+// src/scripting/script-player.ts
+var DEFAULT_LINE_TIMEOUT_MS = 3e4;
+var ScriptPlayer = class {
+  host;
+  lines;
+  lineGapMs;
+  loop;
+  lineTimeoutMs;
+  _index = -1;
+  _state = "idle";
+  currentMessageId = null;
+  lineAcked = false;
+  retiredIds = /* @__PURE__ */ new Set();
+  selfInterruptPending = false;
+  pauseRequested = false;
+  // Number of stray bot_responses still expected from text-only lines that were superseded
+  // (via next()/stop()) BEFORE they were acked. Their server messageId isn't known yet, so we
+  // can't retire them by id — instead we swallow that many fresh responses by arrival order.
+  pendingStaleResponses = 0;
+  lineTimer = null;
+  gapTimer = null;
+  resolveDone;
+  done;
+  onBotResponse = (r) => this.handleBotResponse(r);
+  onAudioComplete = (messageId) => this.handleAudioComplete(messageId);
+  onInterrupt = (data) => this.handleExternalInterrupt(data);
+  onDisconnected = () => this.finish("disconnected");
+  constructor(host, lines, opts = {}) {
+    this.host = host;
+    const defaultTextOnly = opts.textOnly ?? false;
+    this.lines = lines.map(
+      (line) => typeof line === "string" ? { text: line, textOnly: defaultTextOnly } : { text: line.text, textOnly: line.textOnly ?? defaultTextOnly }
+    );
+    this.lineGapMs = Math.max(0, opts.lineGapMs ?? 0);
+    this.loop = opts.loop ?? false;
+    this.lineTimeoutMs = opts.lineTimeoutMs ?? DEFAULT_LINE_TIMEOUT_MS;
+    this.done = new Promise((resolve) => {
+      this.resolveDone = resolve;
+    });
+    this.host.on("botResponse", this.onBotResponse);
+    this.host.on("audioPlaybackComplete", this.onAudioComplete);
+    this.host.on("interrupt", this.onInterrupt);
+    this.host.on("disconnected", this.onDisconnected);
+    if (opts.autoStart ?? true) {
+      queueMicrotask(() => this.play());
+    }
+  }
+  get length() {
+    return this.lines.length;
+  }
+  get index() {
+    return this._index;
+  }
+  get state() {
+    return this._state;
+  }
+  play() {
+    if (this._state === "done" || this._state === "playing") return;
+    this.pauseRequested = false;
+    this._state = "playing";
+    this.startNextLine();
+  }
+  resume() {
+    this.play();
+  }
+  pause() {
+    if (this._state === "done") return;
+    this.pauseRequested = true;
+  }
+  next() {
+    if (this._state === "done") return;
+    if (this._state === "idle") {
+      this.play();
+      return;
+    }
+    const expectInterrupt = this._state === "playing";
+    this.pauseRequested = false;
+    this._state = "playing";
+    this.clearTimers();
+    this.supersedeCurrentLine(expectInterrupt);
+    this.startNextLine();
+  }
+  stop() {
+    this.finish("stopped", true);
+  }
+  // ─── internals ────────────────────────────────────────────────
+  hasSpeakableLine() {
+    return this.lines.some((l) => l.text.trim().length > 0);
+  }
+  startNextLine() {
+    if (this._state !== "playing") return;
+    if (!this.hasSpeakableLine()) {
+      this.finish("finished");
+      return;
+    }
+    let next = this._index + 1;
+    for (; ; ) {
+      if (next >= this.lines.length) {
+        if (this.loop) {
+          next = 0;
+        } else {
+          this.finish("finished");
+          return;
+        }
+      }
+      if (this.lines[next].text.trim()) break;
+      next++;
+    }
+    this._index = next;
+    this.currentMessageId = null;
+    this.lineAcked = false;
+    if (!this.host.isConnected) {
+      this.finish("disconnected");
+      return;
+    }
+    const line = this.lines[next];
+    this.host.sayLine(line.text, line.textOnly);
+    this.lineTimer = setTimeout(() => this.handleLineTimeout(), this.lineTimeoutMs);
+  }
+  handleBotResponse(r) {
+    if (this._state === "done" || this._index < 0) return;
+    if (r.messageId && this.retiredIds.has(r.messageId)) return;
+    if (r.isInterjection) return;
+    if (!this.lineAcked) {
+      if (this.pendingStaleResponses > 0) {
+        this.pendingStaleResponses--;
+        if (r.messageId) this.retiredIds.add(r.messageId);
+        return;
+      }
+      this.lineAcked = true;
+      this.currentMessageId = r.messageId ?? null;
+      this.selfInterruptPending = false;
+      this.host.emitScriptLineStarted({
+        index: this._index,
+        text: this.lines[this._index].text,
+        messageId: r.messageId
+      });
+    }
+    if (this.currentMessageId && r.messageId && r.messageId !== this.currentMessageId) return;
+    const line = this.lines[this._index];
+    const completesOnResponse = line.textOnly || !this.host.willPlayScriptedAudio;
+    if (completesOnResponse && r.isFinal) {
+      this.onLineComplete();
+    }
+  }
+  handleAudioComplete(messageId) {
+    if (this._state === "done" || this._index < 0) return;
+    if (messageId && this.retiredIds.has(messageId)) return;
+    const line = this.lines[this._index];
+    if (line.textOnly || !this.host.willPlayScriptedAudio) return;
+    if (this.currentMessageId && messageId && messageId !== this.currentMessageId) return;
+    this.onLineComplete();
+  }
+  handleLineTimeout() {
+    if (this._state === "done") return;
+    this.host.log(`script line ${this._index} timed out after ${this.lineTimeoutMs}ms; advancing`);
+    this.onLineComplete();
+  }
+  onLineComplete() {
+    this.clearTimers();
+    if (this.lineGapMs > 0) {
+      this.gapTimer = setTimeout(() => this.afterGap(), this.lineGapMs);
+    } else {
+      this.afterGap();
+    }
+  }
+  afterGap() {
+    this.gapTimer = null;
+    if (this._state === "done") return;
+    if (this.pauseRequested) {
+      this._state = "paused";
+      return;
+    }
+    this.startNextLine();
+  }
+  handleExternalInterrupt(data) {
+    if (this._state === "done") return;
+    if (data?.messageId && this.retiredIds.has(data.messageId)) return;
+    if (this.selfInterruptPending) {
+      this.selfInterruptPending = false;
+      return;
+    }
+    this.finish("interrupted");
+  }
+  /**
+   * Retire the current line because we're moving off it.
+   * @param expectInterrupt true when the line is still in-flight (next()/stop() mid-line), so the
+   *   server will emit a self-induced `interrupt` we must not treat as external. false when the
+   *   line already completed (e.g. next() while paused), where no interrupt is coming.
+   */
+  supersedeCurrentLine(expectInterrupt) {
+    if (this.lineAcked && this.currentMessageId) {
+      this.retiredIds.add(this.currentMessageId);
+    } else if (expectInterrupt && this.lines[this._index]?.textOnly) {
+      this.pendingStaleResponses++;
+    }
+    if (expectInterrupt) this.selfInterruptPending = true;
+  }
+  clearTimers() {
+    if (this.lineTimer) {
+      clearTimeout(this.lineTimer);
+      this.lineTimer = null;
+    }
+    if (this.gapTimer) {
+      clearTimeout(this.gapTimer);
+      this.gapTimer = null;
+    }
+  }
+  finish(reason, interrupt = false) {
+    if (this._state === "done") return;
+    this.clearTimers();
+    if (interrupt) {
+      this.selfInterruptPending = true;
+      if (this.host.isConnected) this.host.interrupt();
+    }
+    this._state = "done";
+    this.host.off("botResponse", this.onBotResponse);
+    this.host.off("audioPlaybackComplete", this.onAudioComplete);
+    this.host.off("interrupt", this.onInterrupt);
+    this.host.off("disconnected", this.onDisconnected);
+    this.host.emitScriptComplete({ reason });
+    this.resolveDone({ reason });
+  }
+};
+
 // src/client.ts
 var DEFAULT_SAMPLE_RATE = 24e3;
 var REST_UNAVAILABLE_MESSAGE = "REST API not available with session token auth. Use a server-side proxy for REST calls.";
@@ -9640,6 +9865,7 @@ var EstuaryClient = class extends TypedEventEmitter {
   _hasAutoInterrupted = false;
   _autoInterruptGraceTimer = null;
   _isLiveKitSpeaking = false;
+  _activeScript = null;
   constructor(config) {
     super();
     if (!config.apiKey && !config.sessionToken) {
@@ -9714,6 +9940,10 @@ var EstuaryClient = class extends TypedEventEmitter {
   /** Disconnect from the server */
   async disconnect() {
     this.logger.info("Disconnecting...");
+    if (this._activeScript && this._activeScript.state !== "done") {
+      this._activeScript.stop();
+    }
+    this._activeScript = null;
     if (this._autoInterruptGraceTimer) {
       clearTimeout(this._autoInterruptGraceTimer);
       this._autoInterruptGraceTimer = null;
@@ -9735,6 +9965,59 @@ var EstuaryClient = class extends TypedEventEmitter {
     this.ensureConnected();
     this.socketManager.emitEvent("say_line", { text, text_only: textOnly });
   }
+  /**
+   * Script a sequence of prewritten lines. Lines are paced so each finishes before the next
+   * is sent — required because say_line interrupts any in-progress response server-side, so
+   * unpaced lines would stomp each other. Returns a controller (play/pause/resume/next/stop +
+   * an awaitable `done`). Starting a new script stops any currently-active one.
+   */
+  playScript(lines, opts) {
+    this.ensureConnected();
+    if (this._activeScript && this._activeScript.state !== "done") {
+      this._activeScript.stop();
+    }
+    const player = new ScriptPlayer(this.createScriptHost(), lines, opts);
+    this._activeScript = player;
+    return player;
+  }
+  /** Convenience alias of playScript() for fire-and-forget scripted sequences. */
+  sayLines(lines, opts) {
+    return this.playScript(lines, opts);
+  }
+  createScriptHost() {
+    const self = this;
+    return {
+      sayLine(text, textOnly) {
+        self.sayLine(text, textOnly);
+      },
+      interrupt() {
+        self.interrupt();
+      },
+      on(event, listener) {
+        self.on(event, listener);
+        return self;
+      },
+      off(event, listener) {
+        self.off(event, listener);
+        return self;
+      },
+      get isConnected() {
+        return self.isConnected;
+      },
+      get willPlayScriptedAudio() {
+        return self.audioPlayer != null;
+      },
+      emitScriptLineStarted(info) {
+        self.emit("scriptLineStarted", info);
+      },
+      emitScriptComplete(info) {
+        self.emit("scriptComplete", info);
+      },
+      log(msg) {
+        self.logger.debug(msg);
+      }
+    };
+  }
   /** Interrupt the current bot response */
   interrupt(messageId) {
     this.ensureConnected();