@cairnvibe/sdk 0.2.13 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/agent-loop.d.ts +113 -0
  2. package/dist/agent-loop.js +128 -0
  3. package/dist/cairn-widget.js +14 -9
  4. package/dist/cursor-overlay.d.ts +19 -0
  5. package/dist/cursor-overlay.js +126 -0
  6. package/dist/element-ladder.d.ts +71 -0
  7. package/dist/element-ladder.js +168 -0
  8. package/dist/index.d.ts +79 -1
  9. package/dist/index.js +886 -96
  10. package/dist/key-rotator.d.ts +28 -0
  11. package/dist/key-rotator.js +57 -3
  12. package/dist/memory-sqlite.d.ts +86 -0
  13. package/dist/memory-sqlite.js +230 -0
  14. package/dist/realtime-cli.js +22 -1
  15. package/dist/realtime-server.d.ts +83 -2
  16. package/dist/realtime-server.js +561 -121
  17. package/dist/server.d.ts +266 -5
  18. package/dist/server.js +1013 -83
  19. package/dist/skill-store.d.ts +17 -0
  20. package/dist/skill-store.js +78 -0
  21. package/dist/tts-stream.d.ts +25 -0
  22. package/dist/tts-stream.js +32 -0
  23. package/dist/vad.d.ts +27 -0
  24. package/dist/vad.js +128 -0
  25. package/dist/verb-executor.d.ts +32 -11
  26. package/dist/verb-executor.js +315 -39
  27. package/dist/webmcp-client.d.ts +14 -1
  28. package/dist/webmcp-client.js +22 -1
  29. package/package.json +3 -1
  30. package/src/agent-loop.ts +222 -0
  31. package/src/cursor-overlay.ts +130 -0
  32. package/src/element-ladder.ts +170 -0
  33. package/src/index.tsx +935 -100
  34. package/src/key-rotator.ts +57 -2
  35. package/src/memory-sqlite.ts +283 -0
  36. package/src/realtime-cli.ts +24 -1
  37. package/src/realtime-server.ts +669 -123
  38. package/src/server.ts +1119 -83
  39. package/src/skill-store.ts +88 -0
  40. package/src/tts-stream.ts +30 -0
  41. package/src/vad.ts +153 -0
  42. package/src/verb-executor.ts +329 -42
  43. package/src/web-component.ts +97 -24
  44. package/src/webmcp-client.ts +30 -2
@@ -27,9 +27,28 @@
27
27
 
28
28
  import http from "node:http";
29
29
  import { WebSocket, WebSocketServer } from "ws";
30
- import { TERMINAL_VERBS, type HistoryTurn, type LiveElement, type Manifest, type VerbResponse, type WebMcpTool } from "@cairnvibe/core";
31
- import { buildSystemPrompt, createVerbLLM, resolveVerb, type CapabilityTier, type CreateCopilotHandlerOptions } from "./server";
30
+ import { classifyUiPattern, deriveStructureSignals, type AgentEvent, type HistoryTurn, type LiveElement, type Manifest, type Plan, type ProgressLedger, type VerbResponse, type WebMcpTool } from "@cairnvibe/core";
31
+ import { driveAgentLoop, looksMultiStep, MAX_HISTORY_TURNS, summarizeVerbForHistory } from "./agent-loop";
32
+ import {
33
+ buildSystemPrompt,
34
+ compileSkill,
35
+ createCriticLLM,
36
+ createPlanLLM,
37
+ createVerbLLM,
38
+ fallbackPlan,
39
+ matchSkillByGoal,
40
+ renderRegisteredActions,
41
+ renderSkillSummaries,
42
+ resolveCritic,
43
+ resolvePlan,
44
+ resolveVerb,
45
+ KeyRotator,
46
+ type CapabilityTier,
47
+ type CreateCopilotHandlerOptions,
48
+ } from "./server";
32
49
  import { DeepgramSpeakStream } from "./tts-stream";
50
+ import { formatArchivedFacts, formatRememberedFacts, seedHistoryFromMemory, type MemoryStore } from "./memory-sqlite";
51
+ import type { SkillStore } from "./skill-store";
33
52
 
34
53
  const DEEPGRAM_LIVE_URL = "wss://api.deepgram.com/v1/listen";
35
54
  const DEFAULT_STT_MODEL = "nova-2";
@@ -39,35 +58,141 @@ const DEFAULT_TTS_VOICE = "aura-2-thalia-en";
39
58
  // something within about a second instead of dead air while the real
40
59
  // multi-step work runs. A short rotating set, not one fixed line, so it
41
60
  // doesn't read as a canned bot phrase on every multi-step question.
42
- const ACK_PHRASES = ["Let me check that for you.", "One moment, let me look into that.", "Give me a second to check.", "Let me take a look."];
61
+ //
62
+ // Phase 2 step 3 — rewritten from the original set (kept below in spirit
63
+ // but not verbatim: "Let me check that for you." / "One moment, let me
64
+ // look into that." / "Give me a second to check." / "Let me take a
65
+ // look.") for two real, testable reasons, not a vibe change: (1) the
66
+ // plan's own bar is "feels like a person coordinating a team ('give me
67
+ // a sec, sorting that out'), never generic-corporate" — the original
68
+ // set's formal, service-desk phrasing ("One moment, let me look into
69
+ // that") is closer to a phone-tree script than a coworker; (2) "kept
70
+ // short on purpose (a long ack costs real latency budget)" — the
71
+ // original set averaged 6 words; this one averages under 4, a real,
72
+ // measurable reduction in synthesis time before the ack is even
73
+ // audible, on top of sounding more like a person. Graded on this now,
74
+ // not eyeballed — see judge.ts's new `persona` dimension and
75
+ // realtime-server.ts's own "ack" message, which exposes the actual
76
+ // spoken text to packages/evals' trace capture for the first time.
77
+ const ACK_PHRASES = ["Give me a sec.", "One sec, checking.", "Hang on, let me look.", "On it, one sec.", "Just a sec here.", "Let me check real quick."];
43
78
  // Not constrained by any telephony 8kHz requirement — this is just "what
44
79
  // quality does Deepgram render at" for browser playback, and the Web Audio
45
80
  // API resamples an AudioBuffer at any declared rate transparently.
46
81
  const TTS_SAMPLE_RATE = 24000;
47
82
 
83
+ /**
84
+ * Phase 5 step 2 — explicit fact-remembering (Track B's own "remember is
85
+ * an explicit act, never automatic" pattern — step 1 built the automatic
86
+ * turn-recording half; this is the deliberate half). Modeled as a
87
+ * SYNTHETIC WebMCP tool the model can call_tool, not a new verb — reuses
88
+ * the existing call_tool grammar/validation the model already knows
89
+ * ("a tool name from this turn's webMcpTools list") instead of inventing
90
+ * a new one. Handled entirely SERVER-SIDE (see executeStep below) —
91
+ * the client never learns this step happened at all (see onStep below
92
+ * for why that's not just an optimization: it's what keeps this safe).
93
+ */
94
+ const REMEMBER_FACT_TOOL_NAME = "remember_fact";
95
+ const REMEMBER_FACT_TOOL: WebMcpTool = {
96
+ name: REMEMBER_FACT_TOOL_NAME,
97
+ description:
98
+ "Remember something worth recalling in a FUTURE conversation with this same user — a stated preference, a known pitfall, anything that would help next time. Not for facts only relevant to answering right now. Call this AT MOST ONCE per turn, for one real fact. Once it returns, the fact is already saved — immediately give your final spoken answer (e.g. explain) confirming that to the user; do not call this again in the same turn.",
99
+ inputSchema: {
100
+ type: "object",
101
+ properties: {
102
+ key: { type: "string", description: "A short, stable name for this fact, e.g. \"preferredUnits\" or \"flakySelectorNote\"." },
103
+ value: { type: "string", description: "The real fact to remember, in plain language." },
104
+ },
105
+ required: ["key", "value"],
106
+ },
107
+ };
108
+
109
+ async function handleRememberFactTool(memory: MemoryStore, scopeId: string, args: Record<string, unknown> | undefined): Promise<string> {
110
+ const key = typeof args?.key === "string" ? args.key.trim() : "";
111
+ const value = typeof args?.value === "string" ? args.value.trim() : "";
112
+ if (!key || !value) return "Could not remember that — a key and a value are both required.";
113
+ memory.rememberFact(scopeId, key, value);
114
+ return `Remembered: ${key} = ${value}`;
115
+ }
116
+
48
117
  export interface CreateRealtimeServerOptions extends CreateCopilotHandlerOptions {
49
118
  manifest: Manifest;
50
119
  deepgramApiKey: string;
51
120
  sttModel?: string;
52
121
  ttsVoice?: string;
122
+ /** Phase 5 — real cross-session memory (packages/sdk/src/memory-sqlite.ts,
123
+ * or any store implementing the same interface). Optional — omitting it
124
+ * keeps every connection exactly as memory-less as before this existed.
125
+ * Scoped by whatever `scopeId` string a connection's own client sends in
126
+ * its "context" message (see ConnectionDeps' own doc comment) — this SDK
127
+ * invents no identity of its own. */
128
+ memory?: MemoryStore;
129
+ /** Architecture Pillar 3 (Skill half) — see ConnectionDeps' own doc
130
+ * comment. Optional; omitting it keeps every connection exactly as it
131
+ * was before this existed. */
132
+ skills?: SkillStore;
133
+ /** See ConnectionDeps' own doc comment. Defaults to "default" when `skills` is set but this is omitted. */
134
+ skillsScopeId?: string;
53
135
  }
54
136
 
55
137
  type ServerMessage =
56
138
  | { type: "interim"; text: string }
57
- | { type: "final"; text: string }
58
- | { type: "verb"; verb: VerbResponse }
59
- | { type: "speaking_start" }
139
+ /** `generation` (and on every other message below that carries one) is
140
+ * the server's own barge-in generation counter at the moment THIS
141
+ * message was produced — see `triggerServerBargeIn`'s `generation`
142
+ * variable. Real, live-found bug this closes: the client's own local
143
+ * barge-in (VAD-triggered, entirely independent of the server) can
144
+ * start a brand-new turn's "final" before an EARLIER turn's own verb/
145
+ * audio — already in flight on the wire when the server processed the
146
+ * barge-in — actually arrives. WebSocket delivers messages in order,
147
+ * but "in order" isn't "still relevant": without a way to tell an
148
+ * older turn's message apart from the current one, the client applied
149
+ * it anyway, misattributing a stale answer to whatever question was
150
+ * now current — the exact "one question, but a different, unrelated-
151
+ * sounding answer showed up later" bug found live. The client tracks
152
+ * the generation of the most recent "final" it's processed and drops
153
+ * any later verb/speaking/audio message whose generation is older. */
154
+ | { type: "final"; text: string; generation: number }
155
+ | { type: "verb"; verb: VerbResponse; generation: number }
156
+ | { type: "speaking_start"; generation: number }
60
157
  /** One chunk of raw linear16 PCM audio, base64-encoded, as it's rendered — never the whole clip at once. */
61
- | { type: "audio_chunk"; audio: string; sampleRate: number }
158
+ | { type: "audio_chunk"; audio: string; sampleRate: number; generation: number }
62
159
  /** No more audio chunks are coming for this turn. The client may still be mid-playback of what it already has. */
63
- | { type: "speaking_end" }
64
- | { type: "turn_complete" }
160
+ | { type: "speaking_end"; generation: number }
161
+ | { type: "turn_complete"; generation: number }
162
+ /** Phase 2 step 3 — the ack phrase's text, sent alongside the audio
163
+ * that speaks it. Purely informational (see emitEvent's own "inj"
164
+ * case) — mainly so packages/evals' voiceFrames capture has something
165
+ * readable to grade the Talker's persona against. */
166
+ | { type: "ack"; text: string }
65
167
  | { type: "error"; message: string };
66
168
 
169
+ // seedHistoryFromMemory/formatRememberedFacts moved to memory-sqlite.ts
170
+ // (Phase 5 step 4) — the SAME shared, storage-agnostic logic both the
171
+ // realtime relay and the typed/HTTP transport need. Re-exported here
172
+ // (not just imported) so every existing import from "./realtime-server"
173
+ // keeps working unchanged.
174
+ export { seedHistoryFromMemory, formatRememberedFacts };
175
+
67
176
  export function createRealtimeServer(options: CreateRealtimeServerOptions): http.Server {
68
177
  const registeredActions = options.registeredActions ?? [];
69
178
  const capability = options.capability ?? "act";
70
- const llm = createVerbLLM(options);
179
+ // One shared rotator across all three LLM roles — see
180
+ // CreateCopilotHandlerOptions.keyRotator's own doc comment for the real
181
+ // gap this closes (a key one role confirmed dead used to stay invisible
182
+ // to the other two, which kept rediscovering it fresh on every call).
183
+ // Only built for groq — anthropic's createXLLM calls ignore keyRotator
184
+ // entirely, so building one for it would be dead work. Respects a
185
+ // caller-supplied options.keyRotator (e.g. shared with the typed/HTTP
186
+ // transport in the same process) instead of always building a fresh one.
187
+ const sharedOptions: CreateRealtimeServerOptions =
188
+ options.provider === "groq" && !options.keyRotator
189
+ ? { ...options, keyRotator: options.apiKeys ? new KeyRotator(options.apiKeys) : options.apiKey ? new KeyRotator([options.apiKey]) : KeyRotator.fromEnvList(process.env.GROQ_API_KEYS) ?? undefined }
190
+ : options;
191
+ const llm = createVerbLLM(sharedOptions);
192
+ // Phase 3 steps 2-3 — real, separately-configured Planner/Critic LLMs.
193
+ // See finalizeTurn's own doc comment for how they're actually used.
194
+ const planLLM = createPlanLLM(sharedOptions);
195
+ const criticLLM = createCriticLLM(sharedOptions);
71
196
  // "text" is optional on highlight/open/navigate/do in the base prompt —
72
197
  // fine for the typed/HTTP path, which always has a visible answer area,
73
198
  // but silence reads as broken in a live voice conversation (the client
@@ -75,8 +200,9 @@ export function createRealtimeServer(options: CreateRealtimeServerOptions): http
75
200
  // instruction asks for a confirmation grounded in what was actually
76
201
  // done, not filler — generic phrasing here is what made replies feel
77
202
  // "unrelated" to the question that was just asked.
203
+ const actionDescriptions = options.actionDescriptions ?? {};
78
204
  const systemPrompt =
79
- buildSystemPrompt(options.manifest, registeredActions, options.persona) +
205
+ buildSystemPrompt(options.manifest, registeredActions, options.persona, actionDescriptions) +
80
206
  `\n\nYou are in a live voice conversation right now — the user is speaking out loud and may not be looking at the screen. For highlight/open/navigate/do, include a short spoken "text" that names the specific thing you're pointing at or the specific place you're sending them (e.g. "Highlighting the New Invoice button" or "Taking you to Invoices"), not a generic filler phrase — so they hear a confirmation that's actually about their question.`;
81
207
  const sttModel = options.sttModel ?? process.env.DEEPGRAM_MODEL ?? DEFAULT_STT_MODEL;
82
208
  const ttsVoice = options.ttsVoice ?? process.env.DEEPGRAM_VOICE ?? DEFAULT_TTS_VOICE;
@@ -88,10 +214,46 @@ export function createRealtimeServer(options: CreateRealtimeServerOptions): http
88
214
  });
89
215
  const wss = new WebSocketServer({ server: httpServer });
90
216
 
217
+ // Real, server-side visibility into how many browser tabs/connections
218
+ // are actually live at once — added specifically to answer, with real
219
+ // data instead of a guess, a live-raised concern: could a page reload
220
+ // (or several in quick succession) leave more than one realtime
221
+ // connection open at the same time, each independently running its own
222
+ // Deepgram STT/TTS and LLM calls for the same user? Every connection
223
+ // gets a short id, logged on open and close, alongside a live count —
224
+ // if that count is ever more than 1 during normal single-tab use, THAT
225
+ // is the real, direct evidence of a genuine duplicate-connection bug;
226
+ // if it always reads 1, duplication server-side is ruled out with real
227
+ // proof, not assumed away.
228
+ let nextConnectionId = 1;
229
+ let activeConnections = 0;
230
+
91
231
  wss.on("connection", (client) => {
92
- handleConnection(client, { deepgramApiKey, sttModel, ttsVoice, llm, systemPrompt, manifest: options.manifest, registeredActions, capability }).catch(
232
+ const connectionId = nextConnectionId++;
233
+ activeConnections++;
234
+ console.log(`[cairn realtime] connection ${connectionId} opened — ${activeConnections} active`);
235
+ client.on("close", () => {
236
+ activeConnections--;
237
+ console.log(`[cairn realtime] connection ${connectionId} closed — ${activeConnections} active`);
238
+ });
239
+ handleConnection(client, {
240
+ deepgramApiKey,
241
+ sttModel,
242
+ ttsVoice,
243
+ llm,
244
+ planLLM,
245
+ criticLLM,
246
+ systemPrompt,
247
+ manifest: options.manifest,
248
+ registeredActions,
249
+ actionDescriptions,
250
+ capability,
251
+ memory: options.memory,
252
+ skills: options.skills,
253
+ skillsScopeId: options.skillsScopeId,
254
+ }).catch(
93
255
  (err) => {
94
- console.error("[cairn realtime] connection error:", err);
256
+ console.error(`[cairn realtime] connection ${connectionId} error:`, err);
95
257
  safeSend(client, { type: "error", message: "internal error" });
96
258
  client.close();
97
259
  },
@@ -106,15 +268,47 @@ export interface ConnectionDeps {
106
268
  sttModel: string;
107
269
  ttsVoice: string;
108
270
  llm: ReturnType<typeof createVerbLLM>;
271
+ /** Phase 3 step 2 — a separately-configured Planner LLM, called on the
272
+ * first continuing step of a turn (see finalizeTurn). Optional so
273
+ * existing ConnectionDeps construction (and every existing test) keeps
274
+ * working unchanged; absent means no Planner call happens at all. */
275
+ planLLM?: ReturnType<typeof createPlanLLM>;
276
+ /** Phase 3 step 3 — a separately-configured Critic LLM. Only engages
277
+ * (task-advancement/replan/give-up actually driving the loop, not just
278
+ * logging) when BOTH this and planLLM are present — the Critic needs a
279
+ * real Plan's current task to check against. Optional for the same
280
+ * backward-compatibility reason as planLLM. */
281
+ criticLLM?: ReturnType<typeof createCriticLLM>;
109
282
  systemPrompt: string;
110
283
  manifest: Manifest;
111
284
  registeredActions: string[];
285
+ /** Phase 4 step 4 — real descriptions for registeredActions ids, same
286
+ * shape/purpose as CreateCopilotHandlerOptions.actionDescriptions.
287
+ * Optional, defaults to {} — an existing ConnectionDeps construction
288
+ * (own or a test's) keeps working with every action rendered bare. */
289
+ actionDescriptions?: Record<string, string>;
112
290
  capability: CapabilityTier;
291
+ /** Phase 5 — see CreateRealtimeServerOptions' own doc comment. Optional,
292
+ * same backward-compatibility reason as every other addition here:
293
+ * absent means no memory read/write happens for any connection, ever
294
+ * — today's exact behavior. */
295
+ memory?: MemoryStore;
296
+ /**
297
+ * Architecture Pillar 3 (Skill half) — real, persistent storage for
298
+ * self-authored Skills (skill-store.ts). A DIFFERENT axis of scope than
299
+ * `memory` above: Skills are meant to be shared across every user who
300
+ * talks to this deployment (the same scope `ui-manifest.json` itself
301
+ * already has), never per-user — see skill-store.ts's own doc comment.
302
+ * Optional; absent means no Skill retrieval/saving happens at all, zero
303
+ * overhead, today's exact behavior.
304
+ */
305
+ skills?: SkillStore;
306
+ /** The deployment-wide scope Skills are stored/looked up under when
307
+ * `skills` is configured. Defaults to "default" — a single-deployment
308
+ * setup, today's only real usage — when omitted. */
309
+ skillsScopeId?: string;
113
310
  }
114
311
 
115
- const MAX_HISTORY_TURNS = 8; // 4 exchanges — enough for "the first one"/"do that instead" without growing the prompt unbounded over a long call
116
- const MAX_LOOP_ITERATIONS = 6; // a hard cap on one turn's agent-loop steps, not a target — see finalizeTurn
117
-
118
312
  async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promise<void> {
119
313
  // liveElements/webMcpTools refresh on every "context" resend (the client
120
314
  // sends one on route changes and each time it's about to start listening
@@ -131,6 +325,21 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
131
325
  // one WebSocket per call — so this is accumulated here directly rather
132
326
  // than round-tripped through the client.
133
327
  const history: HistoryTurn[] = [];
328
+ // Phase 5 — real cross-session memory. `scopeId` is set from the FIRST
329
+ // "context" message that carries one (see the "context" handler below)
330
+ // and never changed again for the life of this connection — a real,
331
+ // deliberate v1 simplification (no attempt to handle a scopeId that
332
+ // legitimately changes mid-connection, e.g. a mid-session login) rather
333
+ // than guessed-at complexity. `historySeededFromMemory` guards the
334
+ // ONE-TIME load of this scope's prior turns into `history` — a later
335
+ // "context" resend (route changes send fresh ones routinely) must never
336
+ // re-seed and duplicate them.
337
+ let scopeId: string | null = null;
338
+ let historySeededFromMemory = false;
339
+ function recordMemoryTurn(role: "user" | "assistant", text: string): void {
340
+ if (!deps.memory || !scopeId) return;
341
+ deps.memory.recordTurn(scopeId, role, text);
342
+ }
134
343
  // Resolves the agent loop's in-flight waitForToolResult() call once the
135
344
  // client reports back what a click/fill/read/call_tool step actually
136
345
  // did — same "a mutable pending-callback slot, resolved when the right
@@ -140,15 +349,75 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
140
349
  const dgUrl =
141
350
  `${DEEPGRAM_LIVE_URL}?model=${encodeURIComponent(deps.sttModel)}` +
142
351
  `&encoding=linear16&sample_rate=16000&channels=1&interim_results=true&endpointing=300&utterance_end_ms=1000`;
143
- const dg = new WebSocket(dgUrl, { headers: { Authorization: `Token ${deps.deepgramApiKey}` } });
144
352
 
353
+ // Real, live-reported bug this closes: "status says Listening but nothing
354
+ // happens" — the client keeps looking and sounding fine (mic still
355
+ // capturing, WS still open, no error ever shown), because the REAL
356
+ // failure is silent and one layer deeper: Deepgram's own STT connection
357
+ // can close mid-session (an idle timeout, a network blip, Deepgram's own
358
+ // connection lifetime limit) and this code never noticed — there was no
359
+ // `dg.on("close", ...)` handler at all, `dgOpen` never got reset to
360
+ // false, and every subsequent mic frame kept calling `dg.send(buf)` on an
361
+ // already-CLOSED socket with no callback to catch the failure. The client
362
+ // never heard about any of this, because nothing here ever sent it an
363
+ // "error" message — from the outside it looks exactly like "listening,
364
+ // but the mic just isn't picking anything up."
365
+ //
366
+ // Fixed by making the STT connection self-healing instead of a single
367
+ // fire-and-forget WebSocket: `dg` is now reassignable, and a real close
368
+ // triggers a bounded number of automatic reconnects (fresh handshake,
369
+ // same handlers) before finally giving up and telling the client — so a
370
+ // transient Deepgram-side drop recovers on its own instead of silently
371
+ // bricking the rest of the call.
372
+ let dg: WebSocket;
145
373
  let dgOpen = false;
374
+ let dgReconnectAttempts = 0;
375
+ const MAX_DG_RECONNECT_ATTEMPTS = 3;
146
376
  const pendingAudio: Buffer[] = [];
377
+ // Accumulates Deepgram "Results" transcript segments across one utterance
378
+ // — see handleDeepgramMessage for why this can't just react to every
379
+ // is_final. Declared before connectDeepgramStt so its own "message"
380
+ // handler closes over an already-initialized binding, not just a
381
+ // same-scope one that happens to be safe only because WS events are
382
+ // always async.
383
+ const turnState = { buffer: "" };
147
384
 
148
- dg.on("open", () => {
149
- dgOpen = true;
150
- for (const chunk of pendingAudio.splice(0)) dg.send(chunk);
151
- });
385
+ function connectDeepgramStt(): void {
386
+ const socket = new WebSocket(dgUrl, { headers: { Authorization: `Token ${deps.deepgramApiKey}` } });
387
+ dg = socket;
388
+
389
+ socket.on("open", () => {
390
+ dgOpen = true;
391
+ dgReconnectAttempts = 0;
392
+ for (const chunk of pendingAudio.splice(0)) socket.send(chunk);
393
+ });
394
+
395
+ socket.on("message", (data) => {
396
+ void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history, turnState, () => generation, waitForToolResult, recordMemoryTurn, () => scopeId, () => {
397
+ generation++;
398
+ });
399
+ });
400
+
401
+ socket.on("error", (err) => {
402
+ console.error("[cairn realtime] Deepgram STT connection error:", err);
403
+ });
404
+
405
+ socket.on("close", (code, reason) => {
406
+ dgOpen = false;
407
+ console.log(`[cairn realtime] Deepgram STT connection closed (code ${code}${reason ? `, ${reason}` : ""})`);
408
+ if (client.readyState !== WebSocket.OPEN) return; // the whole call already ended — nothing to reconnect for
409
+ if (dgReconnectAttempts >= MAX_DG_RECONNECT_ATTEMPTS) {
410
+ console.error(`[cairn realtime] Deepgram STT gave up reconnecting after ${MAX_DG_RECONNECT_ATTEMPTS} attempts`);
411
+ safeSend(client, { type: "error", message: "Speech recognition connection was lost and couldn't be restored — try starting the call again." });
412
+ return;
413
+ }
414
+ dgReconnectAttempts++;
415
+ console.log(`[cairn realtime] reconnecting to Deepgram STT (attempt ${dgReconnectAttempts}/${MAX_DG_RECONNECT_ATTEMPTS})`);
416
+ connectDeepgramStt();
417
+ });
418
+ }
419
+
420
+ connectDeepgramStt();
152
421
 
153
422
  // ONE Speak connection reused for every turn in this session — a fresh
154
423
  // handshake per turn is a real, measurable chunk of the latency this
@@ -184,10 +453,22 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
184
453
  return { stream: speakStream, ready: speakStreamReady! };
185
454
  }
186
455
 
187
- // Discards whatever the current turn is still synthesizing/sending, and
188
- // unsticks a pending speakStreamed() call if one is in flight — Deepgram's
189
- // "Clear" isn't guaranteed to itself trigger a "Flushed" confirmation, so
190
- // without this the interrupted call's promise would hang forever.
456
+ // A real, live-reported bug in what used to live here: a "confirm-or-
457
+ // reverse" grace window that, on ANY barge-in with no confirming STT
458
+ // transcript arriving within 600ms, concluded it was a false positive
459
+ // and RE-SPOKE THE SAME TEXT FROM THE TOP. Live symptom, reported
460
+ // directly: saying "stop" cut the agent off, paused for about a
461
+ // second, then the exact same answer started playing again from the
462
+ // beginning — because Deepgram's own transcript for "stop" routinely
463
+ // arrived a little later than the 600ms window, so every clean,
464
+ // deliberate interruption looked exactly like an unconfirmed false
465
+ // alarm and got "resumed." Direct user instruction: there should be no
466
+ // such system at all — a barge-in should behave like it does in any
467
+ // normal voice assistant, an immediate, permanent stop, never a guess
468
+ // at whether to talk over the user again. `triggerServerBargeIn` is
469
+ // now exactly that: bump generation (drops any audio/verb already in
470
+ // flight), clear the TTS stream, unstick a pending speakStreamed()
471
+ // call — and nothing else.
191
472
  function triggerServerBargeIn(): void {
192
473
  generation++;
193
474
  speakStream?.clear();
@@ -200,24 +481,24 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
200
481
 
201
482
  stream.setAudioHandler((chunk) => {
202
483
  if (myGeneration !== generation) return; // stale — dropped by barge-in
203
- safeSend(client, { type: "audio_chunk", audio: chunk.toString("base64"), sampleRate: TTS_SAMPLE_RATE });
484
+ safeSend(client, { type: "audio_chunk", audio: chunk.toString("base64"), sampleRate: TTS_SAMPLE_RATE, generation: myGeneration });
204
485
  });
205
486
 
206
487
  await ready;
207
488
  if (!speakStream || myGeneration !== generation) {
208
489
  // Reconnect failed, or barge-in happened before the stream connected —
209
490
  // either way, only degrade to turn_complete if this is still current.
210
- if (myGeneration === generation) safeSend(client, { type: "turn_complete" });
491
+ if (myGeneration === generation) safeSend(client, { type: "turn_complete", generation: myGeneration });
211
492
  return;
212
493
  }
213
494
 
214
495
  await new Promise<void>((resolve) => {
215
496
  onCurrentTurnFlushed = () => {
216
497
  onCurrentTurnFlushed = null;
217
- if (myGeneration === generation) safeSend(client, { type: "speaking_end" });
498
+ if (myGeneration === generation) safeSend(client, { type: "speaking_end", generation: myGeneration });
218
499
  resolve();
219
500
  };
220
- safeSend(client, { type: "speaking_start" });
501
+ safeSend(client, { type: "speaking_start", generation: myGeneration });
221
502
  stream.sendText(text);
222
503
  stream.flush();
223
504
  });
@@ -243,24 +524,16 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
243
524
  });
244
525
  }
245
526
 
246
- // Accumulates Deepgram "Results" transcript segments across one utterance
247
- // — see handleDeepgramMessage for why this can't just react to every
248
- // is_final.
249
- const turnState = { buffer: "" };
250
-
251
- dg.on("message", (data) => {
252
- void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history, turnState, () => generation, waitForToolResult);
253
- });
254
-
255
- dg.on("error", (err) => {
256
- console.error("[cairn realtime] Deepgram STT connection error:", err);
257
- safeSend(client, { type: "error", message: "speech recognition unavailable" });
258
- });
259
-
260
527
  client.on("message", (data, isBinary) => {
261
528
  if (isBinary) {
262
529
  const buf = Buffer.isBuffer(data) ? data : Buffer.from(data as ArrayBuffer);
263
- if (dgOpen) dg.send(buf);
530
+ // The readyState check (not just dgOpen) is real, defensive belt-and-
531
+ // suspenders: dgOpen is reset to false the instant "close" fires, but
532
+ // a mic frame arriving in the same tick as a not-yet-processed close
533
+ // event should never risk calling .send() on a socket that's already
534
+ // gone — that used to be exactly how a dead connection kept silently
535
+ // swallowing audio with no error ever surfacing.
536
+ if (dgOpen && dg.readyState === WebSocket.OPEN) dg.send(buf);
264
537
  else pendingAudio.push(buf);
265
538
  return;
266
539
  }
@@ -273,6 +546,32 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
273
546
  liveElements: parseLiveElements(msg.liveElements),
274
547
  webMcpTools: parseWebMcpTools(msg.webMcpTools),
275
548
  };
549
+ // Phase 5 — `scopeId` is whatever opaque id the CUSTOMER's own
550
+ // client code chooses to send (their own end-user id if they have
551
+ // login, anything else stable otherwise) — this SDK never invents
552
+ // one. Only the first real scopeId this connection ever sees is
553
+ // used; a later "context" resend's scopeId (route changes send
554
+ // these routinely) is ignored, and the one-time prior-turn load
555
+ // below never repeats.
556
+ if (!scopeId && typeof msg.scopeId === "string" && msg.scopeId) {
557
+ const newScopeId: string = msg.scopeId;
558
+ scopeId = newScopeId;
559
+ if (deps.memory && !historySeededFromMemory) {
560
+ historySeededFromMemory = true;
561
+ const priorTurns = deps.memory.recentTurns(newScopeId);
562
+ const seeded = seedHistoryFromMemory(history, priorTurns, MAX_HISTORY_TURNS);
563
+ history.length = 0;
564
+ history.push(...seeded);
565
+
566
+ // Prepended AFTER the cap above, deliberately exempt from
567
+ // it — a remembered fact ("prefers metric units") should
568
+ // stay in context for the WHOLE connection, not age out the
569
+ // same way an ordinary conversation turn does once enough
570
+ // new turns accumulate.
571
+ const factsSummary = formatRememberedFacts(deps.memory.recallFacts(newScopeId));
572
+ if (factsSummary) history.unshift({ role: "assistant", text: factsSummary });
573
+ }
574
+ }
276
575
  } else if (msg.type === "tool_result" && typeof msg.observation === "string") {
277
576
  // The client finished executing a click/fill/read/call_tool step
278
577
  // the agent loop sent it — this is what finalizeTurn's
@@ -330,6 +629,36 @@ export async function handleDeepgramMessage(
330
629
  turnState: { buffer: string },
331
630
  getGeneration: () => number,
332
631
  waitForToolResult: () => Promise<string>,
632
+ /** Phase 5 — called with each real (role, text) turn as it's finalized,
633
+ * right alongside the same-shaped `history.push`. Optional and a no-op
634
+ * by default so every existing call site keeps working unchanged. The
635
+ * realtime connection's own recordMemoryTurn writes it to durable
636
+ * storage when memory + a scopeId are both configured for this
637
+ * connection — see ConnectionDeps.memory's own doc comment. */
638
+ recordMemoryTurn?: (role: "user" | "assistant", text: string) => void,
639
+ /** Phase 5 step 2 — see finalizeTurn's own doc comment. Threaded
640
+ * through here purely to reach finalizeTurn's two call sites below. */
641
+ getScopeId?: () => string | null,
642
+ /** Real, live-found gap this closes: `generation` (getGeneration/
643
+ * triggerServerBargeIn) previously only ever bumped on an EXPLICIT
644
+ * barge-in — two ordinary, sequential turns with no interruption
645
+ * between them shared the exact same generation number. That was
646
+ * fine for what `generation` was originally built for (dropping
647
+ * audio/verbs abandoned mid-turn by a real interruption), but it
648
+ * left the CLIENT's own generation-based staleness check (added for
649
+ * that same reason, in index.tsx) with no way to tell a merely SLOW
650
+ * turn's late-arriving reply apart from the current one — nothing
651
+ * had bumped, so the late reply's generation still matched. Found
652
+ * live: a "hello" reply that took long enough to arrive AFTER the
653
+ * next question's own "final" had already fired, landing on the
654
+ * wrong caption because both were tagged the same generation.
655
+ * Called once per genuinely NEW turn (both call sites below), so
656
+ * every real "final" gets its own fresh generation — a turn is now
657
+ * "superseded" the instant a newer one starts, not only when an
658
+ * explicit interruption says so. Optional and a no-op by default so
659
+ * every existing call site (own or a test's) that doesn't pass this
660
+ * keeps behaving exactly as before. */
661
+ bumpGeneration?: () => void,
333
662
  ): Promise<void> {
334
663
  let msg: any;
335
664
  try {
@@ -343,7 +672,10 @@ export async function handleDeepgramMessage(
343
672
  // after utterance_end_ms of silence — a safety net for the rare case a
344
673
  // Results message never carries speech_final:true, so a turn can't get
345
674
  // permanently stuck with real transcript sitting in the buffer forever.
346
- if (turnState.buffer) await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration, waitForToolResult);
675
+ if (turnState.buffer) {
676
+ bumpGeneration?.();
677
+ await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration, waitForToolResult, recordMemoryTurn, getScopeId);
678
+ }
347
679
  return;
348
680
  }
349
681
 
@@ -372,7 +704,8 @@ export async function handleDeepgramMessage(
372
704
  return;
373
705
  }
374
706
 
375
- await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration, waitForToolResult);
707
+ bumpGeneration?.();
708
+ await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration, waitForToolResult, recordMemoryTurn, getScopeId);
376
709
  }
377
710
 
378
711
  /**
@@ -396,12 +729,13 @@ export async function handleDeepgramMessage(
396
729
  * ones aren't) doesn't end the turn here: the server can't execute a DOM
397
730
  * action itself, so it sends the step to the client, awaits its real
398
731
  * result over waitForToolResult(), folds that into a *local* working copy
399
- * of history, and calls resolveVerb again — repeat up to
400
- * MAX_LOOP_ITERATIONS. The connection's real `history` only gets the
401
- * user's real question plus the turn's final answer, committed once at
402
- * the end a turn that hits the cap mid-loop doesn't leave partial tool
403
- * noise in the conversation's real memory, same discipline the HTTP
404
- * path's runTypedAgentLoop (index.tsx) follows.
732
+ * of history, and calls resolveVerb again — repeat up to the iteration cap
733
+ * (driveAgentLoop's default of 6, in agent-loop.ts — the shared skeleton
734
+ * this function and the HTTP path's runTypedAgentLoop (index.tsx) both
735
+ * drive). The connection's real `history` only gets the user's real
736
+ * question plus the turn's final answer, committed once at the end here —
737
+ * a turn that hits the cap mid-loop doesn't leave partial tool noise in
738
+ * the conversation's real memory.
405
739
  */
406
740
  async function finalizeTurn(
407
741
  turnState: { buffer: string },
@@ -412,15 +746,22 @@ async function finalizeTurn(
412
746
  history: HistoryTurn[],
413
747
  getGeneration: () => number,
414
748
  waitForToolResult: () => Promise<string>,
749
+ recordMemoryTurn?: (role: "user" | "assistant", text: string) => void,
750
+ /** Phase 5 step 2 — this connection's real scopeId, if it has one yet
751
+ * (see the "context" message handler). A getter, same pattern as
752
+ * getContext/getGeneration, since it can be set AFTER this turn
753
+ * already started (a scopeId only ever arrives via a "context"
754
+ * message, and a turn can begin before one has). Optional; absent or
755
+ * returning null both mean "no memory-backed tools offered." */
756
+ getScopeId?: () => string | null,
415
757
  ): Promise<void> {
416
758
  const transcript = turnState.buffer;
417
759
  turnState.buffer = "";
418
760
  const myGeneration = getGeneration();
419
- safeSend(client, { type: "final", text: transcript });
761
+ safeSend(client, { type: "final", text: transcript, generation: myGeneration });
420
762
 
421
- let loopHistory = history;
422
763
  // The Talker: set once, the first time a turn turns out to need more
423
- // than one step (see the loop below) — a real, in-flight speakStreamed()
764
+ // than one step (see onStep below) — a real, in-flight speakStreamed()
424
765
  // call, never awaited until we're actually ready to speak the real
425
766
  // answer. Deliberately not re-triggered per step: the Speak connection
426
767
  // (speakStreamed) only ever handles one utterance at a time, so a second
@@ -428,28 +769,138 @@ async function finalizeTurn(
428
769
  // handling instead of queuing cleanly.
429
770
  let ackPromise: Promise<void> | null = null;
430
771
 
772
+ // Phase 3 step 5 — the Talker's real event stream ("Revisable by
773
+ // Design"'s pattern): a pure, fire-and-forget consumer, never awaited
774
+ // by driveAgentLoop, never able to affect its control flow. This
775
+ // realtime transport's own Talker projection is intentionally small —
776
+ // the only event type it currently DOES anything with is "inj" (the
777
+ // ack phrase), which it turns into the same real speakStreamed() call
778
+ // as before, just reached through a real event instead of an inline
779
+ // side effect inside onStep. "act"/"obs"/"thk" events flow through the
780
+ // same stream (driveAgentLoop already emits act/obs on its own; the
781
+ // Critic below emits a real "thk" with its own reasoning) but aren't
782
+ // consumed for anything yet — logged, not narrated, a real seam for a
783
+ // future richer Talker to attach to without touching the loop again.
784
+ function emitEvent(event: AgentEvent): void {
785
+ switch (event.type) {
786
+ case "inj":
787
+ // Phase 2 step 3 — the ack phrase's audio was already the only
788
+ // thing the user hears; this text-bearing sibling message makes
789
+ // WHAT was said visible in the wire protocol too — today the
790
+ // only way to know (packages/evals' voiceFrames capture full
791
+ // frames, but an "inj" event never otherwise reaches the client
792
+ // as readable text, only as synthesized audio). Purely
793
+ // informational — a client that ignores unknown message types
794
+ // loses nothing.
795
+ safeSend(client, { type: "ack", text: event.text });
796
+ ackPromise = speakStreamed(event.text);
797
+ return;
798
+ case "act":
799
+ console.log("[cairn talker] act:", summarizeVerbForHistory(event.verb));
800
+ return;
801
+ case "obs":
802
+ console.log("[cairn talker] obs:", event.observation);
803
+ return;
804
+ case "thk":
805
+ console.log("[cairn talker] thk:", event.text);
806
+ return;
807
+ }
808
+ }
809
+
810
+ // Architecture Pillar 4 — real Plan/Progress state the Critic (below)
811
+ // actually acts on, not just observability. Started EAGERLY, before the
812
+ // first real step even runs, when looksMultiStep(transcript) already
813
+ // flags this as a probable compound goal — replacing the old lazy gate
814
+ // (kicked off only once a turn had already revealed a non-terminal
815
+ // first step, one full model round trip later than it needed to be).
816
+ // A false-negative heuristic miss still falls back to that same lazy
817
+ // path below (`if (planLLM && !planPromise)`), so nothing regresses —
818
+ // this only ever makes planning START EARLIER, never skips it. Only
819
+ // the realtime transport had this Plan/Progress wiring until now — see
820
+ // index.tsx's runTypedAgentLoop for the typed/HTTP transport's own
821
+ // version, added in the same pass.
822
+ let planPromise: Promise<Plan> | null = null;
823
+ let plan: Plan | null = null;
824
+ let progress: ProgressLedger | null = null;
825
+ const STALL_THRESHOLD = 3; // Magentic-One-sized bounded budget before the harness itself escalates to give_up, rather than trusting the Critic alone to notice it's stalling
826
+
827
+ // Architecture Pillar 3 (Skill half) — real, per-deployment Skills
828
+ // (skill-store.ts), a DIFFERENT scope axis than `memory` (per-user).
829
+ // Computed once, up front, since both the Planner (retrieval) and the
830
+ // Critic (accumulating what gets saved after this turn) need it.
831
+ const skillsScopeId = deps.skillsScopeId ?? "default";
832
+ const skillSummaries = deps.skills ? deps.skills.listSkillSummaries(skillsScopeId) : [];
833
+ const matchedSkillSummary = skillSummaries.length ? matchSkillByGoal(skillSummaries, transcript) : null;
834
+ const skillsPayload = skillSummaries.length
835
+ ? {
836
+ summariesText: renderSkillSummaries(skillSummaries) || undefined,
837
+ suggestedInstructions: matchedSkillSummary ? (deps.skills!.getSkill(skillsScopeId, matchedSkillSummary.id)?.instructions ?? undefined) : undefined,
838
+ }
839
+ : undefined;
840
+ // Every real, Critic-verified learnedFact from this turn's steps — the
841
+ // Formulator (compileSkill) compiles whatever's here into a real Skill
842
+ // once the turn concludes, below. Empty is the common case, not a gap.
843
+ const learnedFacts: string[] = [];
844
+
845
+ // Architecture Pillar 5 — the Archive tier, checked once per turn
846
+ // (never always-injected the way Core facts are — those are seeded
847
+ // once per CONNECTION, in the "context" message handler above). Added
848
+ // only to THIS turn's own ephemeral working history, never persisted
849
+ // into the connection's real `history` array below — a fact resurfaced
850
+ // because it happened to relate to this one question shouldn't linger
851
+ // in context for the rest of the conversation the way a Core fact
852
+ // deliberately does.
853
+ const archiveScopeId = getScopeId?.() ?? null;
854
+ const archivedSummary = deps.memory && archiveScopeId ? formatArchivedFacts(deps.memory.recallArchivedFacts(archiveScopeId, transcript)) : null;
855
+ const historyForThisTurn = archivedSummary ? [...history, { role: "assistant" as const, text: archivedSummary }] : history;
856
+
431
857
  try {
432
- for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
433
- const { route, visible, liveElements, webMcpTools } = getContext();
434
- const verb = await resolveVerb(deps.llm, deps.systemPrompt, deps.manifest, deps.registeredActions, deps.capability, {
435
- route,
436
- question: transcript,
437
- visible,
438
- liveElements,
439
- webMcpTools,
440
- history: loopHistory,
441
- });
858
+ const planLLM = deps.planLLM;
859
+ const criticLLM = deps.criticLLM;
860
+
861
+ if (planLLM && looksMultiStep(transcript)) {
862
+ planPromise = resolvePlan(planLLM, transcript, 1, deps.manifest, renderRegisteredActions(deps.registeredActions, deps.actionDescriptions), skillsPayload);
863
+ }
442
864
 
443
- if (myGeneration !== getGeneration()) return; // superseded by a barge-in while this turn was resolving
865
+ const result = await driveAgentLoop(historyForThisTurn, {
866
+ async getNextStep(loopHistory) {
867
+ const { route, visible, liveElements, webMcpTools } = getContext();
868
+ // Phase 5 step 2 — offered only when there's somewhere real to
869
+ // write it (memory configured AND this connection has a real
870
+ // scopeId) — never a tool the model can call into a void.
871
+ const availableTools = deps.memory && getScopeId?.() ? [...webMcpTools, REMEMBER_FACT_TOOL] : webMcpTools;
872
+ return resolveVerb(deps.llm, deps.systemPrompt, deps.manifest, deps.registeredActions, deps.capability, {
873
+ route,
874
+ question: transcript,
875
+ visible,
876
+ liveElements,
877
+ webMcpTools: availableTools,
878
+ history: loopHistory,
879
+ });
880
+ },
881
+ onStep({ verb, iteration, terminal }) {
882
+ if (myGeneration !== getGeneration()) return true; // superseded by a barge-in while this turn was resolving
444
883
 
445
- // Sent immediatelybefore speech synthesis even starts — so
446
- // highlight/navigate/do execute in the browser right away instead of
447
- // waiting on audio. The agent visibly acts while it's still about to
448
- // speak, not after.
449
- safeSend(client, { type: "verb", verb });
884
+ // Phase 5 step 2 a remember_fact call is handled entirely
885
+ // server-side (see executeStep below) and must NEVER be sent to
886
+ // the client: the client would try to look it up in its own
887
+ // real WebMCP tool registry, fail to find it (it's synthetic,
888
+ // server-only), and report an error tool_result back — landing
889
+ // on whatever's THEN occupying the single-slot
890
+ // pendingToolResultResolve, which by then could easily belong
891
+ // to a genuinely later, unrelated step. Suppressing this send
892
+ // is not an optimization, it's what keeps that real race from
893
+ // ever being possible.
894
+ const isRememberFactCall = verb.verb === "call_tool" && verb.name === REMEMBER_FACT_TOOL_NAME;
895
+ if (!isRememberFactCall) {
896
+ // Sent immediately — before speech synthesis even starts — so
897
+ // highlight/navigate/do execute in the browser right away instead
898
+ // of waiting on audio. The agent visibly acts while it's still
899
+ // about to speak, not after.
900
+ safeSend(client, { type: "verb", verb, generation: myGeneration });
901
+ }
450
902
 
451
- if (!TERMINAL_VERBS.has(verb.verb)) {
452
- if (i === 0) {
903
+ if (!terminal && iteration === 0) {
453
904
  // This turn just revealed it needs more than one step — speak a
454
905
  // quick, cheap acknowledgment *now*, in parallel with the rest
455
906
  // of the loop's own real work below (not awaited here), so the
@@ -457,22 +908,136 @@ async function finalizeTurn(
457
908
  // air for however long the real multi-step answer takes.
458
909
  // Single-step turns (the common case) never reach this branch
459
910
  // at all, so they keep today's latency exactly as it is.
460
- ackPromise = speakStreamed(ACK_PHRASES[Math.floor(Math.random() * ACK_PHRASES.length)]);
911
+ // Emitted as a real "inj" event now (step 5), consumed by
912
+ // emitEvent above — same real speakStreamed() call, reached
913
+ // through the event stream instead of an inline side effect.
914
+ emitEvent({ type: "inj", text: ACK_PHRASES[Math.floor(Math.random() * ACK_PHRASES.length)], at: Date.now() });
915
+ // The lazy fallback — only fires when looksMultiStep missed
916
+ // (planPromise is still null): a real Plan is still guaranteed
917
+ // before the Critic needs one, just one round trip later than
918
+ // the eager path above.
919
+ if (planLLM && !planPromise) planPromise = resolvePlan(planLLM, transcript, 1, deps.manifest, renderRegisteredActions(deps.registeredActions, deps.actionDescriptions), skillsPayload);
461
920
  }
462
- // A continuing step itself stays silent (keeps the loop fast; the
463
- // client still shows it visually) — wait for its real result and
464
- // go around again instead of ending the turn.
465
- const observation = await waitForToolResult();
466
- if (myGeneration !== getGeneration()) return;
467
- loopHistory = [
468
- ...loopHistory,
469
- { role: "assistant" as const, text: `${summarizeVerbForHistory(verb)}. Result: ${observation}` },
470
- ].slice(-MAX_HISTORY_TURNS);
471
- continue;
472
- }
921
+ return false;
922
+ },
923
+ // A continuing step itself stays silent (keeps the loop fast; the
924
+ // client still shows it visually) — wait for its real result and go
925
+ // around again instead of ending the turn.
926
+ executeStep: (verb) => {
927
+ // Phase 5 step 2 — resolved entirely in-process, never routed
928
+ // through the client's real tool-execution round trip (see
929
+ // onStep's own doc comment for why the client is never even
930
+ // told this step happened).
931
+ const currentScopeId = getScopeId?.() ?? null;
932
+ if (verb.verb === "call_tool" && verb.name === REMEMBER_FACT_TOOL_NAME && deps.memory && currentScopeId) {
933
+ return handleRememberFactTool(deps.memory, currentScopeId, verb.args);
934
+ }
935
+ return waitForToolResult();
936
+ },
937
+ onStepResult: () => myGeneration !== getGeneration(),
938
+ onEvent: emitEvent,
939
+ runCritic:
940
+ planLLM && criticLLM
941
+ ? async ({ verb, observation }) => {
942
+ // Real state, not the Executor's self-report — see
943
+ // resolveCritic's own doc comment for why this is a
944
+ // genuinely separate pass, mirroring judge.ts's own
945
+ // precedent. Awaited here (not just logged) on its FIRST
946
+ // use — by now at least one real tool round trip has
947
+ // already happened, so the Planner call kicked off above
948
+ // has likely already resolved in parallel; this is not a
949
+ // NEW blocking wait so much as picking up work already in
950
+ // flight.
951
+ if (!plan) {
952
+ plan = planPromise ? await planPromise : fallbackPlan(transcript, 1);
953
+ progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
954
+ }
955
+ const currentProgress = progress!;
956
+ const currentTask = plan.tasks[currentProgress.currentTaskIndex];
957
+ const verdict = await resolveCritic(criticLLM, currentTask, transcript, verb, observation);
958
+ // A real "thk" event — the Critic's own reasoning, narrated
959
+ // onto the same event stream the ack/act/obs events already
960
+ // flow through (not spoken today, just carried — see
961
+ // emitEvent's own doc comment on why that's a deliberate,
962
+ // small v1 scope).
963
+ emitEvent({ type: "thk", text: verdict.reasoning, at: Date.now() });
964
+ // Architecture Pillar 3 (Skill half) — accumulate whatever
965
+ // this step's real, Critic-verified fact was; the
966
+ // Formulator compiles whatever's here into a real Skill
967
+ // once the turn concludes, below. The common case adds
968
+ // nothing here at all.
969
+ if (verdict.learnedFact) learnedFacts.push(verdict.learnedFact);
970
+
971
+ if (verdict.verdict === "task_complete") {
972
+ currentTask.status = "done";
973
+ if (currentProgress.currentTaskIndex < plan.tasks.length - 1) {
974
+ // More tasks remain — advance and keep looping instead
975
+ // of ending the turn here.
976
+ currentProgress.currentTaskIndex++;
977
+ plan.tasks[currentProgress.currentTaskIndex].status = "in_progress";
978
+ currentProgress.stallCount = 0;
979
+ return { ...verdict, verdict: "continue" };
980
+ }
981
+ // The real bug fix: the LAST task is genuinely done —
982
+ // end the loop right here instead of asking the model
983
+ // again and hoping it notices its own success.
984
+ return verdict;
985
+ }
986
+
987
+ if (verdict.verdict === "replan") {
988
+ // A fresh Planner call, a real new version — never a
989
+ // silent patch to the existing plan.
990
+ plan = await resolvePlan(planLLM, transcript, plan.version + 1, deps.manifest, renderRegisteredActions(deps.registeredActions, deps.actionDescriptions), skillsPayload);
991
+ progress = { planVersion: plan.version, currentTaskIndex: 0, stallCount: 0 };
992
+ return { ...verdict, verdict: "continue" };
993
+ }
994
+
995
+ if (verdict.verdict === "give_up") return verdict;
473
996
 
997
+ // "continue" — a harness-enforced fail-safe on top of the
998
+ // Critic's own judgment: crossing a bounded stall budget
999
+ // escalates to give_up itself, rather than trusting the
1000
+ // Critic alone to eventually notice it's stuck (Magentic-One's
1001
+ // own two-tier tolerance pattern).
1002
+ currentProgress.stallCount++;
1003
+ if (currentProgress.stallCount >= STALL_THRESHOLD) {
1004
+ return {
1005
+ verdict: "give_up",
1006
+ reasoning: `Stuck after ${currentProgress.stallCount} steps with no confirmed progress on "${currentTask.description}" — ${verdict.reasoning}`,
1007
+ };
1008
+ }
1009
+ return verdict;
1010
+ }
1011
+ : undefined,
1012
+ });
1013
+
1014
+ if (result.outcome === "aborted") return;
1015
+
1016
+ // Architecture Pillar 3 (Skill half) — the Formulator, run once per
1017
+ // turn (not per-step — cheap on purpose). Saves nothing when nothing
1018
+ // was learned (the common case) or no SkillStore is configured (zero
1019
+ // overhead, today's exact behavior). Classified from whatever this
1020
+ // exact moment's live context reports — a best-effort snapshot, not
1021
+ // necessarily the exact page a given fact was learned on, which is
1022
+ // an acceptable trade for a Skill meant to be a general per-platform
1023
+ // note rather than a per-page one.
1024
+ if (deps.skills && learnedFacts.length > 0) {
1025
+ const patternMatches = classifyUiPattern(deriveStructureSignals(getContext().liveElements));
1026
+ const skill = compileSkill(transcript, learnedFacts, patternMatches[0]?.pattern);
1027
+ if (skill) deps.skills.saveSkill(skillsScopeId, skill);
1028
+ }
1029
+
1030
+ if (result.outcome === "terminal" || result.outcome === "unparseable" || result.outcome === "critic-complete") {
1031
+ const verb: VerbResponse =
1032
+ result.outcome === "terminal"
1033
+ ? result.finalVerb
1034
+ : result.outcome === "critic-complete"
1035
+ ? { verb: "explain", text: result.verdict.reasoning }
1036
+ : { verb: "explain", text: "I'm not sure how to help with that." };
474
1037
  history.push({ role: "user", text: transcript }, { role: "assistant", text: summarizeVerbForHistory(verb) });
475
1038
  history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
1039
+ recordMemoryTurn?.("user", transcript);
1040
+ recordMemoryTurn?.("assistant", summarizeVerbForHistory(verb));
476
1041
 
477
1042
  if (ackPromise) {
478
1043
  // Never start a second speakStreamed call before the first (the
@@ -488,29 +1053,39 @@ async function finalizeTurn(
488
1053
  // A verb with no spoken text (highlight/navigate/do often have none)
489
1054
  // still needs to unstick the client's "thinking" state and let the mic
490
1055
  // resume — turn_complete covers that with no audio path involved.
491
- if ("text" in verb && verb.text) {
492
- await speakStreamed(verb.text);
1056
+ const textToSpeak = "text" in verb ? (verb.text ?? undefined) : undefined;
1057
+
1058
+ if (textToSpeak) {
1059
+ await speakStreamed(textToSpeak);
493
1060
  } else {
494
- safeSend(client, { type: "turn_complete" });
1061
+ safeSend(client, { type: "turn_complete", generation: myGeneration });
495
1062
  }
496
1063
  return;
497
1064
  }
498
1065
 
499
- // Iteration cap hit with no terminal verb degrade honestly instead
500
- // of leaving the client waiting forever.
501
- history.push({ role: "user", text: transcript }, { role: "assistant", text: "(gave up after too many steps)" });
1066
+ // Iteration cap hit with no terminal verb, OR the Critic/stall
1067
+ // fail-safe gave up — degrade honestly instead of leaving the client
1068
+ // waiting forever. A real Critic give-up carries its own specific
1069
+ // reasoning, which is a genuinely better message than the generic
1070
+ // fallback below — use it when there is one.
1071
+ const giveUpText =
1072
+ result.outcome === "critic-give-up" ? result.verdict.reasoning : "I wasn't able to finish that — try asking again or breaking it into smaller steps.";
1073
+ const gaveUpSummary = result.outcome === "critic-give-up" ? giveUpText : "(gave up after too many steps)";
1074
+ history.push({ role: "user", text: transcript }, { role: "assistant", text: gaveUpSummary });
502
1075
  history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
503
- safeSend(client, { type: "verb", verb: { verb: "explain", text: "I wasn't able to finish that — try asking again or breaking it into smaller steps." } });
1076
+ recordMemoryTurn?.("user", transcript);
1077
+ recordMemoryTurn?.("assistant", gaveUpSummary);
1078
+ safeSend(client, { type: "verb", verb: { verb: "explain", text: giveUpText }, generation: myGeneration });
504
1079
  if (ackPromise) {
505
1080
  await ackPromise;
506
1081
  if (myGeneration !== getGeneration()) return;
507
1082
  }
508
- await speakStreamed("I wasn't able to finish that — try asking again or breaking it into smaller steps.");
1083
+ await speakStreamed(giveUpText);
509
1084
  } catch (err) {
510
1085
  console.error("[cairn realtime] failed to resolve/speak this turn:", err);
511
1086
  if (myGeneration === getGeneration()) {
512
1087
  safeSend(client, { type: "error", message: "Something went wrong answering that — try again." });
513
- safeSend(client, { type: "turn_complete" });
1088
+ safeSend(client, { type: "turn_complete", generation: myGeneration });
514
1089
  }
515
1090
  }
516
1091
  }
@@ -556,32 +1131,3 @@ function parseWebMcpTools(raw: unknown): WebMcpTool[] {
556
1131
  return tools;
557
1132
  }
558
1133
 
559
- /** A short text form of any verb for the history log — not shown to the
560
- * user, just fed back to the model on later turns so it knows what it
561
- * already did/said. */
562
- function summarizeVerbForHistory(verb: VerbResponse): string {
563
- if ("text" in verb && verb.text) return verb.text;
564
- switch (verb.verb) {
565
- case "highlight":
566
- case "open":
567
- return `(highlighted ${verb.target})`;
568
- case "navigate":
569
- return `(navigated to ${verb.route})`;
570
- case "do":
571
- return `(ran ${verb.action}${verb.target ? ` on ${verb.target}` : ""})`;
572
- case "tour":
573
- return verb.steps.map((s) => s.text).join(" ");
574
- case "click":
575
- return `(clicked ${verb.target})`;
576
- case "fill":
577
- return `(typed "${verb.value}" into ${verb.target})`;
578
- case "read":
579
- return `(read ${verb.target})`;
580
- case "call_tool":
581
- return `(called ${verb.name})`;
582
- case "batch":
583
- return `(${verb.actions.length} steps: ${verb.actions.map((a) => a.verb).join(", ")})`;
584
- default:
585
- return "(no response)";
586
- }
587
- }