agent.libx.js 0.92.6 → 0.92.8

package/dist/index.d.ts

@@ -602,6 +602,12 @@ declare class DuplexAgentOptions {
     progressUpdates: boolean;
     /** Min ms between progress re-voices per task. */
     progressIntervalMs: number;
+    /** Relay worker questions (AskUserQuestion + permission asks via parkQuestion) through the VOICE:
+     *  the question re-voices as '[task <id> asks] …', the user answers conversationally, and the
+     *  voice model resolves it with the AnswerTask tool. Off → host.ask passthrough (text menus). */
+    askRelay: boolean;
+    /** Parked questions auto-resolve empty after this long (callers map '' to deny/best-judgment). */
+    askTimeoutMs: number;
     /** Host overrides for QuickLook lookups (keyed by `what`). The engine's defaults go through the
      *  (possibly jailed) fs — e.g. `.git/**` is deny-listed, so the CLI supplies 'branch' itself. */
     quickLook?: Record<string, (path?: string) => string | Promise<string>>;
@@ -621,6 +627,11 @@ declare class DuplexAgent {
     private seq;
     private pendingEvents;
     private flushQueued;
+    /** Parked worker questions awaiting a (voice-relayed) user answer, keyed by ask id. */
+    readonly pendingAsks: Map<string, {
+        question: string;
+        resolve: (answer: string) => void;
+    }>;
     constructor(options?: Partial<DuplexAgentOptions>);
     /** One user turn: the voice agent streams the reply (and may Delegate). Serialized with re-voice turns. */
     send(content: MessageContent): Promise<RunResult>;
@@ -640,6 +651,12 @@ declare class DuplexAgent {
      *  (pre records the in-flight call; a self-cleaning timer narrates "still inside Bash — 70s").
      *  Completion supersedes: nothing is emitted once the task has settled. */
     private progressReporter;
+    /** Park a question under `askId` (a task id, or any unique key for permission asks): re-voices
+     *  '[task <id> asks] …' and resolves with the user's answer via AnswerTask — or '' on timeout/
+     *  task settle (callers map '' to deny / best-judgment). Workers never block forever. */
+    parkQuestion(askId: string, question: string): Promise<string>;
+    /** Resolve any question a settling/cancelled task left parked (its answer can no longer matter). */
+    private dropAsk;
     private onWorkerSettled;
     private onWorkerFailed;
     private failTask;
@@ -649,6 +666,7 @@ declare class DuplexAgent {
      *  fs-only (no shell; the engine is VFS-abstracted): time, git branch (.git/HEAD read), ls, file
      *  head. Output is hard-capped so a lookup can never bloat the skinny voice context. */
     private quickLookTool;
+    private answerTaskTool;
     private cancelTaskTool;
 }
 
@@ -745,6 +763,11 @@ interface AudioSink {
     playedMs(): number;
     /** stop playback NOW (barge-in primitive) */
     kill(): void;
+    /** optional exact-sample pause/resume — enables the overlap trail-off tier (web: AudioContext
+     *  suspend/resume; CLI AEC helper: control frames). Sinks without it degrade to interrupt-only
+     *  turn-taking. Nothing is lost across a pause; playedMs/drainMs must exclude paused time. */
+    pause?(): void;
+    resume?(): void;
 }
 /** Static key (server/CLI) or an async getter (browser: fetch a short-lived token from YOUR
  *  backend). Getters are invoked on EVERY (re)connect — temp tokens expire, so a reconnect
@@ -794,6 +817,18 @@ declare class VoiceEngineOptions {
     /** heuristic (non-AEC) energy barge-in tuning */
     bargeRmsMult: number;
     bargeRmsFloor: number;
+    /** Overlap turn-taking (AEC tier, needs player.pause/resume) — human phone-call model:
+     *  onset → PAUSE (exact-sample hold, nothing lost); sustained overlap → cede (interrupt; the LLM
+     *  re-enters). Brief overlaps that die out (backchannels — "mm-hm", decided by DURATION, not
+     *  vocabulary) resume from the precise sample and are dropped. false disables. */
+    overlapPause: boolean;
+    /** sustained overlap ≥ this → cede the turn */
+    overlapSustainMs: number;
+    /** quiet for this long while paused → resume, drop the interjection */
+    overlapResumeMs: number;
+    /** energy floor for "overlap candidate" — must sit ABOVE typical room ambient (~110 rms measured;
+     *  ungated ambient re-arming the resume timer forever was a live wedge). User speech ≫ 300. */
+    overlapRms: number;
 }
 declare class VoiceEngine {
     options: VoiceEngineOptions;
@@ -817,6 +852,10 @@ declare class VoiceEngine {
     private pendingUtt;
     private pendingTimer;
     private lastInterrupted;
+    private pausedAt;
+    private overlapLoud;
+    private overlapLastLoudAt;
+    private resumeTimer;
     constructor(options?: Partial<VoiceEngineOptions>);
     start(): Promise<void>;
     get usingAec(): boolean;
@@ -852,6 +891,13 @@ declare class VoiceEngine {
     private handlePartial;
     private handleUtterance;
     private flushUtterance;
+    private get overlapCapable();
+    /** Overlap turn-taking (AEC tier): onset → pause (exact-sample hold); sustained → cede; died out
+     *  → resume. No vocabulary anywhere — duration and persistence decide (backchannels are short
+     *  and stop). Nothing is lost across a pause, so a false positive costs only a brief hold. */
+    private handleOverlap;
+    private armResume;
+    private resetOverlap;
     /** energy two-stage barge-in (heuristic tier only): spike over echo baseline → pause + confirm via STT */
     private handleLevel;
 }

package/dist/index.js

@@ -3545,11 +3545,17 @@ var DuplexAgentOptions = class {
   progressUpdates = false;
   /** Min ms between progress re-voices per task. */
   progressIntervalMs = 25e3;
+  /** Relay worker questions (AskUserQuestion + permission asks via parkQuestion) through the VOICE:
+   *  the question re-voices as '[task <id> asks] …', the user answers conversationally, and the
+   *  voice model resolves it with the AnswerTask tool. Off → host.ask passthrough (text menus). */
+  askRelay = false;
+  /** Parked questions auto-resolve empty after this long (callers map '' to deny/best-judgment). */
+  askTimeoutMs = 12e4;
   /** Host overrides for QuickLook lookups (keyed by `what`). The engine's defaults go through the
    *  (possibly jailed) fs — e.g. `.git/**` is deny-listed, so the CLI supplies 'branch' itself. */
   quickLook;
 };
-var VOICE_SYSTEM_PROMPT = 'You are a spoken voice assistant \u2014 the user HEARS everything you say. Use short sentences. One idea per sentence. No markdown, no bullet lists, no code blocks, no headings, no emoji.\nKeep turns SHORT \u2014 one to three sentences, then stop. Never lecture, enumerate cases, or add caveats unprompted. Conversation is a fast exchange: give the one thing asked, and let the user pull more if they want it.\nYou work in a pair: you talk, and a background worker with FULL access to the user\'s environment (files, shell, web) does the hands-on work. You can find out or do ANYTHING by calling `Delegate` with a clear, self-contained brief \u2014 so NEVER tell the user you can\'t see, access, or do something. Delegate and find out. When the user mentions their project, folder, files, or environment ("this project", "the current folder", "my code"), delegate IMMEDIATELY \u2014 do not ask for paths or details the worker can discover itself. Never pretend to have done the work or invent results \u2014 the worker\'s report is your only source.\nAfter calling Delegate, tell the user you are on it in one short sentence, then end your turn. Do not wait for the result.\nResults arrive later as events like "[task t1 completed] \u2026" or "[task t1 failed] \u2026". When one arrives, summarize it for the ear in one or two short sentences. "[task t1 progress] \u2026" events are interim status, NOT results \u2014 give at most a half-sentence aside ("still on it \u2014 running tests now") and end your turn. Never present progress as a finished result.\nNever read raw file paths, diffs, or code aloud verbatim.\nDo not fire a second Delegate for work already in flight \u2014 check `TaskStatus` first. Use `CancelTask` when the user asks to stop something.\nPRIORITY: when the user says goodbye or wants to end/finish/wrap up the session ("ok bye", "that\'s all", "let\'s finish", "let\'s end", "goodnight", "exit", "wrap up"), call `ExitSession` IMMEDIATELY \u2014 do not delegate, do not check status, just exit.\nFor TRIVIAL instant lookups only \u2014 current time, git branch, listing a folder, peeking at a small file \u2014 use `QuickLook` (instant, no task). Anything requiring searching, reasoning, running commands, or editing still goes through `Delegate`.\nNEVER claim to have stored, saved, or remembered something durably \u2014 you cannot. Anything the user wants persisted (their name, preferences, notes) must be Delegated so a worker writes it to memory.\nUser messages may arrive via speech-to-text and can carry transcription artifacts \u2014 odd words, cut-offs, homophones ("for you" vs "folder"). Read for INTENT, not surface text. If a message seems garbled or surprising, briefly confirm what they meant ("did you mean\u2026?") instead of answering the literal words.';
+var VOICE_SYSTEM_PROMPT = 'You are a spoken voice assistant \u2014 the user HEARS everything you say. Use short sentences. One idea per sentence. No markdown, no bullet lists, no code blocks, no headings, no emoji.\nKeep turns SHORT \u2014 one to three sentences, then stop. Never lecture, enumerate cases, or add caveats unprompted. Conversation is a fast exchange: give the one thing asked, and let the user pull more if they want it.\nYou work in a pair: you talk, and a background worker with FULL access to the user\'s environment (files, shell, web) does the hands-on work. You can find out or do ANYTHING by calling `Delegate` with a clear, self-contained brief \u2014 so NEVER tell the user you can\'t see, access, or do something. Delegate and find out. When the user mentions their project, folder, files, or environment ("this project", "the current folder", "my code"), delegate IMMEDIATELY \u2014 do not ask for paths or details the worker can discover itself. Never pretend to have done the work or invent results \u2014 the worker\'s report is your only source.\nAfter calling Delegate, tell the user you are on it in one short sentence, then end your turn. Do not wait for the result.\nResults arrive later as events like "[task t1 completed] \u2026" or "[task t1 failed] \u2026". When one arrives, summarize it for the ear in one or two short sentences. "[task t1 progress] \u2026" events are interim status, NOT results \u2014 give at most a half-sentence aside ("still on it \u2014 running tests now") and end your turn. Never present progress as a finished result.\nNever read raw file paths, diffs, or code aloud verbatim.\n"[task t1 asks] \u2026" events are QUESTIONS from a background task \u2014 relay to the user in your own words, short, then end your turn. When the user answers, call `AnswerTask` with that id and their answer. NEVER answer on the user\'s behalf for permissions or risky operations; if their reply is ambiguous, confirm first.\nDo not fire a second Delegate for work already in flight \u2014 check `TaskStatus` first. Use `CancelTask` when the user asks to stop something.\nPRIORITY: when the user says goodbye or wants to end/finish/wrap up the session ("ok bye", "that\'s all", "let\'s finish", "let\'s end", "goodnight", "exit", "wrap up"), call `ExitSession` IMMEDIATELY \u2014 do not delegate, do not check status, just exit.\nFor TRIVIAL instant lookups only \u2014 current time, git branch, listing a folder, peeking at a small file \u2014 use `QuickLook` (instant, no task). Anything requiring searching, reasoning, running commands, or editing still goes through `Delegate`.\nNEVER claim to have stored, saved, or remembered something durably \u2014 you cannot. Anything the user wants persisted (their name, preferences, notes) must be Delegated so a worker writes it to memory.\nUser messages may arrive via speech-to-text and can carry transcription artifacts \u2014 odd words, cut-offs, homophones ("for you" vs "folder"). Read for INTENT, not surface text. If a message seems garbled or surprising, briefly confirm what they meant ("did you mean\u2026?") instead of answering the literal words.';
 var VOICE_STYLE_CONVERSATIONAL = `Speak like a person in a live conversation, not an assistant reading a script. React first, then deliver: a quick impulsive beat ("oh nice", "hmm, hold on", "ah, got it") before the substance. Use contractions always. Vary sentence length \u2014 some very short. Light fillers and backchannels are fine ("mm-hm", "right", "let's see") but at most one per reply \u2014 never stack them. When you delegate, say it like a human would ("hang on, let me actually dig into that \u2014 gimme a minute") instead of announcing a task. When a result comes back, react to it like you just found out ("okay so \u2014 turns out\u2026"). Match the user's energy: a quick question gets a quick answer \u2014 a few words is a perfectly good turn. Prefer a short answer plus an offer ("want the details?") over covering everything. Never narrate your own mechanics (no "I will now delegate", no task ids out loud).`;
 var DuplexAgent = class {
   options;
@@ -3559,6 +3565,8 @@ var DuplexAgent = class {
   seq = 0;
   pendingEvents = [];
   flushQueued = false;
+  /** Parked worker questions awaiting a (voice-relayed) user answer, keyed by ask id. */
+  pendingAsks = /* @__PURE__ */ new Map();
   constructor(options) {
     this.options = { ...new DuplexAgentOptions(), ...options };
     const o = this.options;
@@ -3581,7 +3589,7 @@ Today's date: ${(/* @__PURE__ */ new Date()).toDateString()}.`,
       // no defaultTools() — the voice can only Delegate, never touch files itself. Set AFTER the
       // voiceOptions spread (addTools() would be clobbered by the first prepare()); extra voice
       // tools come in via voiceOptions.tools and are merged here.
-      tools: [...o.voiceOptions?.tools ?? [], this.delegateTool(), this.taskStatusTool(), this.cancelTaskTool(), this.quickLookTool()]
+      tools: [...o.voiceOptions?.tools ?? [], this.delegateTool(), this.taskStatusTool(), this.cancelTaskTool(), this.quickLookTool(), this.answerTaskTool()]
     });
   }
   /** One user turn: the voice agent streams the reply (and may Delegate). Serialized with re-voice turns. */
@@ -3652,12 +3660,19 @@ ${recent}` : brief;
         report.output(chunk);
       }
     } : base;
+    const relayAsk = async (q) => {
+      const opts = q.options?.length ? ` Options: ${q.options.map((x) => x.label).join(", ")}.` : "";
+      const a = await this.parkQuestion(id, `${q.question}${opts}`);
+      return a || "(no answer from the user \u2014 use your best judgment and note the assumption)";
+    };
+    const workerHost = o.askRelay ? { ask: relayAsk } : o.host?.ask ? { ask: (q) => o.host.ask(q) } : void 0;
     const worker = new Agent({
       ai: o.ai,
       fs: o.fs,
       model: o.workerModel,
       ...o.workerOptions,
       // may override ai/fs/model/tools/… —
+      ...workerHost ? { host: workerHost } : {},
       ...hooks ? { hooks } : {},
       signal: controller.signal
       // …but never the per-task cancellation signal
@@ -3706,7 +3721,35 @@ ${recent}` : brief;
       }
     };
   }
+  /** Park a question under `askId` (a task id, or any unique key for permission asks): re-voices
+   *  '[task <id> asks] …' and resolves with the user's answer via AnswerTask — or '' on timeout/
+   *  task settle (callers map '' to deny / best-judgment). Workers never block forever. */
+  parkQuestion(askId, question) {
+    return new Promise((resolve) => {
+      let settled = false;
+      const finish = (answer) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        this.pendingAsks.delete(askId);
+        resolve(answer);
+      };
+      const timer = setTimeout(() => {
+        this.notify("task_ask_timeout", `task ${askId}: question timed out \u2014 proceeding without an answer`);
+        finish("");
+      }, this.options.askTimeoutMs);
+      this.pendingAsks.set(askId, { question, resolve: finish });
+      this.notify("task_ask", `task ${askId} asks: ${question}`, { id: askId, question });
+      this.queueRevoice(`[task ${askId} asks] ${question}
+(Relay this to the user in your own words. When they answer, call AnswerTask with id "${askId}" and their answer.)`);
+    });
+  }
+  /** Resolve any question a settling/cancelled task left parked (its answer can no longer matter). */
+  dropAsk(id) {
+    this.pendingAsks.get(id)?.resolve("");
+  }
   onWorkerSettled(id, res) {
+    this.dropAsk(id);
     const rec = this.tasks.get(id);
     if (res.finishReason === "aborted" || rec.status === "cancelled") {
       rec.status = "cancelled";
@@ -3726,6 +3769,7 @@ ${recent}` : brief;
     this.failTask(this.tasks.get(id), err instanceof Error ? err.message : String(err));
   }
   failTask(rec, msg) {
+    this.dropAsk(rec.id);
     rec.status = "error";
     log7.warn(`task ${rec.id} failed: ${msg}`);
     this.notify("task_error", `task ${rec.id} (${rec.label}) failed: ${msg}`);
@@ -3818,6 +3862,23 @@ ${recent}` : brief;
       }
     };
   }
+  answerTaskTool() {
+    return {
+      name: "AnswerTask",
+      description: `Relay the user's answer to a pending question from a background task (the "[task <id> asks]" events). Pass the id from the event and the user's answer.`,
+      parameters: {
+        type: "object",
+        required: ["id", "answer"],
+        properties: { id: { type: "string" }, answer: { type: "string", description: "the user's answer, verbatim or faithfully summarized" } }
+      },
+      run: async ({ id, answer }) => {
+        const ask = this.pendingAsks.get(String(id));
+        if (!ask) return `No pending question for '${id}' \u2014 it may have been answered already or timed out.`;
+        ask.resolve(String(answer ?? ""));
+        return `Answer relayed \u2014 task ${id} resumes.`;
+      }
+    };
+  }
   cancelTaskTool() {
     return {
       name: "CancelTask",
@@ -4000,6 +4061,18 @@ var VoiceEngineOptions = class {
   /** heuristic (non-AEC) energy barge-in tuning */
   bargeRmsMult = 2;
   bargeRmsFloor = 500;
+  /** Overlap turn-taking (AEC tier, needs player.pause/resume) — human phone-call model:
+   *  onset → PAUSE (exact-sample hold, nothing lost); sustained overlap → cede (interrupt; the LLM
+   *  re-enters). Brief overlaps that die out (backchannels — "mm-hm", decided by DURATION, not
+   *  vocabulary) resume from the precise sample and are dropped. false disables. */
+  overlapPause = true;
+  /** sustained overlap ≥ this → cede the turn */
+  overlapSustainMs = 350;
+  /** quiet for this long while paused → resume, drop the interjection */
+  overlapResumeMs = 700;
+  /** energy floor for "overlap candidate" — must sit ABOVE typical room ambient (~110 rms measured;
+   *  ungated ambient re-arming the resume timer forever was a live wedge). User speech ≫ 300. */
+  overlapRms = 300;
 };
 var VoiceEngine = class {
   options;
@@ -4030,6 +4103,13 @@ var VoiceEngine = class {
   // endpointed text held for the merge window
   pendingTimer = null;
   lastInterrupted = null;
+  // overlap (pause) tier state — AEC + pause-capable sinks only
+  pausedAt = 0;
+  overlapLoud = 0;
+  // loud chunks since pause (sustain must be real sound, not two clicks)
+  overlapLastLoudAt = 0;
+  // continuity guard: a gap re-arms the onset (sparse noise ≠ sustained speech)
+  resumeTimer = null;
   constructor(options) {
     this.options = { ...new VoiceEngineOptions(), ...options };
     const o = this.options;
@@ -4077,6 +4157,7 @@ var VoiceEngine = class {
       this.drainTimer = null;
     }
     this.interrupted = false;
+    this.resetOverlap(true);
     if (!this.speaking) this.player.markTurn();
     this.speaking = true;
     this.ctxOpen = true;
@@ -4111,6 +4192,10 @@ var VoiceEngine = class {
         this.drainTimer = null;
         return;
       }
+      if (this.pausedAt) {
+        this.drainTimer = setTimeout(settle, 250);
+        return;
+      }
       this.drainTimer = null;
       this.speaking = false;
       this.echoUntil = now() + 2500;
@@ -4142,6 +4227,7 @@ var VoiceEngine = class {
       clearTimeout(this.drainTimer);
       this.drainTimer = null;
     }
+    this.resetOverlap(false);
     const heardChars = Math.round(Math.max(0, this.player.playedMs()) / 1e3 * 15);
     if (this.reply) this.lastInterrupted = { full: this.reply, heard: this.reply.slice(0, heardChars) };
     this.speaking = false;
@@ -4156,6 +4242,7 @@ var VoiceEngine = class {
     this.setState("listening");
   }
   stop() {
+    if (this.resumeTimer) clearTimeout(this.resumeTimer);
     if (this.pendingTimer) clearTimeout(this.pendingTimer);
     if (this.drainTimer) clearTimeout(this.drainTimer);
     this.stt.stop();
@@ -4181,12 +4268,11 @@ var VoiceEngine = class {
   genuine(text) {
     const total = this.words(text).length;
     const novel = this.novelWords(text).length;
-    if (!novel) return false;
-    return novel >= 2 || novel / Math.max(1, total) > 0.5;
+    return novel > 0 && novel / Math.max(1, total) > 0.5;
   }
   handlePartial(text) {
     if (this.speaking) {
-      const barge = this.usingAec ? this.genuine(text) : this.novelWords(text).length >= (this.suspectUntil ? 1 : 2);
+      const barge = this.overlapCapable ? false : this.usingAec ? this.genuine(text) : this.novelWords(text).length >= (this.suspectUntil ? 1 : 2);
       if (barge) {
         const phase = this.ctxOpen ? "speaking" : "drain";
         this.interrupt();
@@ -4203,6 +4289,10 @@ var VoiceEngine = class {
     if (!this.echoActive() || (this.usingAec ? this.genuine(text) : this.novelWords(text).length >= 1)) this.options.onPartial(text);
   }
   handleUtterance(text) {
+    if (this.speaking && this.ctxOpen && this.overlapCapable) {
+      this.stt.reset();
+      return;
+    }
     if (this.echoActive() && (this.usingAec ? !this.genuine(text) : this.novelWords(text).length < 2)) {
       this.stt.reset();
       return;
@@ -4226,9 +4316,63 @@ var VoiceEngine = class {
     this.pendingUtt = "";
     if (text) this.options.onUtterance(text);
   }
+  get overlapCapable() {
+    return this.usingAec && this.options.overlapPause && !!this.player.pause && !!this.player.resume;
+  }
+  /** Overlap turn-taking (AEC tier): onset → pause (exact-sample hold); sustained → cede; died out
+   *  → resume. No vocabulary anywhere — duration and persistence decide (backchannels are short
+   *  and stop). Nothing is lost across a pause, so a false positive costs only a brief hold. */
+  handleOverlap(rms) {
+    const o = this.options;
+    if (!this.speaking || !this.overlapCapable) return;
+    if (rms < o.overlapRms) return;
+    const t = now();
+    if (!this.pausedAt) {
+      this.overlapLoud = t - this.overlapLastLoudAt <= 60 ? this.overlapLoud + 1 : 1;
+      this.overlapLastLoudAt = t;
+      if (this.overlapLoud < 3) return;
+      this.pausedAt = t;
+      this.player.pause();
+      this.armResume();
+      return;
+    }
+    if (t - this.overlapLastLoudAt > 300) {
+      this.pausedAt = t;
+      this.overlapLoud = 1;
+      this.overlapLastLoudAt = t;
+      this.armResume();
+      return;
+    }
+    this.overlapLastLoudAt = t;
+    this.overlapLoud++;
+    if (t - this.pausedAt >= o.overlapSustainMs && this.overlapLoud >= 4) {
+      const phase = this.ctxOpen ? "speaking" : "drain";
+      this.interrupt();
+      this.options.onBargeIn(phase);
+      return;
+    }
+    this.armResume();
+  }
+  armResume() {
+    if (this.resumeTimer) clearTimeout(this.resumeTimer);
+    this.resumeTimer = setTimeout(() => {
+      this.resumeTimer = null;
+      if (!this.pausedAt) return;
+      this.resetOverlap(true);
+    }, this.options.overlapResumeMs);
+  }
+  resetOverlap(resume) {
+    if (this.resumeTimer) {
+      clearTimeout(this.resumeTimer);
+      this.resumeTimer = null;
+    }
+    if (this.pausedAt && resume) this.player.resume?.();
+    this.pausedAt = 0;
+    this.overlapLoud = 0;
+  }
   /** energy two-stage barge-in (heuristic tier only): spike over echo baseline → pause + confirm via STT */
   handleLevel(rms) {
-    if (this.usingAec) return;
+    if (this.usingAec) return this.handleOverlap(rms);
     if (!this.speaking) {
       this.baseline = 0;
       this.hot = 0;