@lattices/cli 0.4.1 → 0.4.5

package/bin/handsoff-infer.ts

@@ -8,10 +8,13 @@
  * All logging goes to stderr so it doesn't pollute the JSON output.
  */
 
+import {
+  buildAssistantContextMessage,
+  buildAssistantSystemPrompt,
+  normalizeAssistantPlan,
+  tryLocalAssistantPlan,
+} from "./assistant-intelligence.ts";
 import { inferJSON } from "../lib/infer.ts";
-import { readFileSync } from "fs";
-import { join, dirname } from "path";
-import { homedir } from "os";
 
 // ── Read input from stdin ──────────────────────────────────────────
 
@@ -30,213 +33,15 @@ const req = JSON.parse(input) as {
   history?: Array<{ role: "user" | "assistant"; content: string }>;
 };
 
-// ── Load system prompt from file ───────────────────────────────────
+const transcript = req.transcript ?? "";
+const systemPrompt = buildAssistantSystemPrompt();
+const userMessage = buildAssistantContextMessage(transcript, req.snapshot ?? {});
 
-const promptDir = join(dirname(import.meta.dir), "docs", "prompts");
-let systemPrompt: string;
-try {
-  systemPrompt = readFileSync(join(promptDir, "hands-off-system.md"), "utf-8")
-    .split("\n")
-    .filter((l) => !l.startsWith("# "))
-    .join("\n")
-    .trim();
-} catch {
-  systemPrompt = "You are a workspace assistant. Respond with JSON: {actions, spoken}.";
-}
-
-// Replace {{intent_catalog}} with the actual tiling reference
-const intentCatalog = `
-tile_window: Tile a window to a screen position
-  Slots:
-    position (required): Named position or grid:CxR:C,R syntax.
-      Halves: left, right, top, bottom
-      Quarters (2x2): top-left, top-right, bottom-left, bottom-right
-      Thirds (3x1): left-third, center-third, right-third
-      Sixths (3x2): top-left-third, top-center-third, top-right-third, bottom-left-third, bottom-center-third, bottom-right-third
-      Fourths (4x1): first-fourth, second-fourth, third-fourth, last-fourth
-      Eighths (4x2): top-first-fourth, top-second-fourth, top-third-fourth, top-last-fourth, bottom-first-fourth, bottom-second-fourth, bottom-third-fourth, bottom-last-fourth
-      Special: maximize (full screen), center (centered floating)
-      Grid syntax: grid:CxR:C,R (e.g. grid:5x3:2,1 = center cell of 5x3 grid)
-    app (optional): Target app name — match loosely (e.g. "chrome" matches "Google Chrome")
-    wid (optional): Target window ID (from snapshot)
-    session (optional): Tmux session name
-  If no app/wid/session given, tiles the frontmost window.
-  "quarter" = 2x2 cell (top-left etc.), NOT a 4x1 fourth.
-  "top quarter" = top-left or top-right (2x2). "top third" = top-left-third (3x2).
-  Examples: "tile chrome left" → {intent:"tile_window", slots:{app:"chrome", position:"left"}}
-
-focus: Focus a window, app, or session
-  Slots:
-    app (optional): App name to focus
-    session (optional): Session name to focus
-    wid (optional): Window ID to focus
-
-distribute: Arrange windows in an even grid — with optional app filter and region constraint
-  Slots:
-    app (optional): Filter to windows of this app (e.g. "iTerm2", "Google Chrome"). Without this, distributes ALL visible windows.
-    region (optional): Constrain the grid to a screen region. Uses the same position names as tile_window:
-      Halves: left, right, top, bottom
-      Quarters: top-left, top-right, bottom-left, bottom-right
-      Thirds: left-third, center-third, right-third
-      Without this, uses the full screen.
-  Examples:
-    "grid the terminals on the right" → {intent:"distribute", slots:{app:"iTerm2", region:"right"}}
-    "organize my chrome windows in the bottom half" → {intent:"distribute", slots:{app:"Google Chrome", region:"bottom"}}
-    "spread everything out" → {intent:"distribute", slots:{}}
-    "tile all terminals" → {intent:"distribute", slots:{app:"iTerm2"}}
-
-swap: Swap the positions of two windows
-  Slots:
-    wid_a (required): Window ID of the first window (from snapshot)
-    wid_b (required): Window ID of the second window (from snapshot)
-  Examples:
-    "swap Chrome and iTerm" → {intent:"swap", slots:{wid_a:12345, wid_b:67890}}
-
-hide: Hide or minimize a window or app
-  Slots:
-    app (optional): App name to hide (hides the entire app)
-    wid (optional): Window ID to minimize (minimizes just that window)
-  Use app to hide all windows of an app. Use wid to minimize a single window.
-  Examples:
-    "hide Slack" → {intent:"hide", slots:{app:"Slack"}}
-    "minimize that" → {intent:"hide", slots:{wid:12345}}
-
-highlight: Flash a window's border to identify it visually
-  Slots:
-    wid (optional): Window ID to highlight (from snapshot)
-    app (optional): App name to highlight
-  Use when the user asks "which one is that?" or wants to visually identify a window.
-  Examples:
-    "show me the lattices terminal" → {intent:"highlight", slots:{wid:12345}}
-    "which one is Chrome?" → {intent:"highlight", slots:{app:"Google Chrome"}}
-
-move_to_display: Move a window to another monitor/display
-  Slots:
-    display (required): Target display index (0 = main/primary, 1 = second, etc.)
-    wid (optional): Window ID to move (from snapshot)
-    app (optional): App name to move
-    position (optional): Tile position on the target display (e.g. "left", "maximize")
-  If no wid/app given, moves the frontmost window.
-  Examples:
-    "put this on my second monitor" → {intent:"move_to_display", slots:{wid:12345, display:1}}
-    "move Chrome to the main screen" → {intent:"move_to_display", slots:{app:"Google Chrome", display:0}}
-    "send iTerm to the other monitor, left half" → {intent:"move_to_display", slots:{app:"iTerm2", display:1, position:"left"}}
-
-undo: Undo the last window move — restore windows to their previous positions
-  No slots needed.
-  Examples:
-    "put it back" → {intent:"undo"}
-    "undo that" → {intent:"undo"}
-
-search: Search windows by text
-  Slots:
-    query (required): Search text
-  Examples:
-    "find the error message" → {intent:"search", slots:{query:"error"}}
-    "find all terminal windows" → {intent:"search", slots:{query:"terminal"}}
-
-list_windows: List all visible windows
-  No slots needed. Use when the user asks "what's on screen?" or "what windows do I have?"
-
-list_sessions: List active terminal sessions
-  No slots needed. Use when the user asks "what sessions are running?" or "show my projects."
-
-switch_layer: Switch to a workspace layer
-  Slots:
-    layer (required): Layer name or index
-  Examples:
-    "switch to the web layer" → {intent:"switch_layer", slots:{layer:"web"}}
-    "go to layer 2" → {intent:"switch_layer", slots:{layer:"2"}}
-
-create_layer: Save current window arrangement as a named layer
-  Slots:
-    name (required): Layer name
-  Examples:
-    "save this layout as review" → {intent:"create_layer", slots:{name:"review"}}
-
-launch: Launch a project session
-  Slots:
-    project (required): Project name or path
-  Examples:
-    "open my frontend project" → {intent:"launch", slots:{project:"frontend"}}
-    "start working on lattices" → {intent:"launch", slots:{project:"lattices"}}
-
-kill: Kill a terminal session
-  Slots:
-    session (required): Session name or project name
-  Examples:
-    "stop the frontend session" → {intent:"kill", slots:{session:"frontend"}}
-
-scan: Trigger an immediate screen text scan (OCR)
-  No slots needed. Use when the user asks you to read or scan screen content.
-
-CHOOSING THE RIGHT INTENT:
-  Positioning:
-    tile_window = position ONE specific window at a specific spot. Use for 1-6 named windows.
-    distribute = auto-grid MANY windows. Use when the user says "all", "my terminals", "everything", or names more windows than the 6-action limit.
-    distribute with app+region is the most powerful combo: "grid my terminals on the right" → distribute(app:"iTerm2", region:"right")
-  Rearranging:
-    swap = exchange positions of exactly two windows. "swap Chrome and iTerm"
-    move_to_display = move a window to a different monitor. "put this on my other screen"
-  Visibility:
-    hide = hide an app or minimize a window. "hide Slack", "minimize that"
-    highlight = flash a window's border to identify it. "which one is the lattices terminal?"
-    focus = bring a window to the front. "focus Slack", "show me Chrome"
-  Recovery:
-    undo = restore previous positions after a move. "put it back", "undo that"
-  Information:
-    list_windows, list_sessions, search = answer questions about the desktop. NO actions needed for pure questions.
-  Session lifecycle:
-    launch = start a project session. "open my frontend project"
-    kill = stop a session. "kill the API"
-
-TILING PRESETS (use multiple tile_window actions):
-  "split screen" / "side by side" → left + right
-  "thirds" → left-third, center-third, right-third
-  "main + sidebar" → main app left (or maximize), others stacked right
-  "stack" → top + bottom
-  "corners" / "quadrants" → top-left, top-right, bottom-left, bottom-right
-  "six-up" / "3 by 2" → 3x2 grid using sixth positions
-  "eight-up" / "4 by 2" → 4x2 grid using eighth positions
-
-TILING PRESETS (use distribute intent):
-  "mosaic" / "grid" / "spread out" → distribute (all windows, full screen)
-  "grid the terminals" → distribute with app:"iTerm2"
-  "terminals on the right" → distribute with app:"iTerm2", region:"right"
-  "organize chrome on the left" → distribute with app:"Google Chrome", region:"left"
-`;
-
-systemPrompt = systemPrompt.replace("{{intent_catalog}}", intentCatalog);
-
-// ── Build the per-turn message ─────────────────────────────────────
-
-let userMessage = `USER: "${req.transcript}"\n\n`;
-userMessage += "--- DESKTOP SNAPSHOT ---\n";
-
-const snap = req.snapshot;
-if (snap.stageManager) {
-  userMessage += `Stage Manager: ON (grouping: ${snap.smGrouping ?? "all-at-once"})\n\n`;
-  userMessage += `Active stage (${snap.activeStage?.length ?? 0} windows):\n`;
-  for (const w of snap.activeStage ?? []) {
-    userMessage += `  [${w.wid}] ${w.app}: "${w.title}" — ${w.frame}\n`;
-  }
-  userMessage += `\nStrip: ${snap.stripApps?.join(", ") ?? "none"}\n`;
-  userMessage += `Other stages: ${snap.hiddenApps?.join(", ") ?? "none"}\n`;
-} else {
-  userMessage += "Stage Manager: OFF\n";
-  userMessage += `Visible windows (${snap.activeStage?.length ?? 0}):\n`;
-  for (const w of snap.activeStage ?? []) {
-    userMessage += `  [${w.wid}] ${w.app}: "${w.title}" — ${w.frame}\n`;
-  }
-}
-
-if (snap.currentLayer) {
-  userMessage += `\nCurrent layer: ${snap.currentLayer}\n`;
-}
-if (snap.screen) {
-  userMessage += `Screen: ${snap.screen}\n`;
+const localPlan = tryLocalAssistantPlan(transcript, req.snapshot ?? {});
+if (localPlan) {
+  console.log(JSON.stringify(localPlan));
+  process.exit(0);
 }
-userMessage += "--- END SNAPSHOT ---\n";
 
 // ── Call inference ──────────────────────────────────────────────────
 
@@ -257,9 +62,11 @@ try {
   });
 
   // Output result as JSON to stdout
+  const plan = normalizeAssistantPlan(data, transcript);
   const output = {
-    ...data,
+    ...plan,
     _meta: {
+      ...plan._meta,
       provider: raw.provider,
       model: raw.model,
       durationMs: raw.durationMs,

package/bin/handsoff-worker.ts

@@ -16,7 +16,14 @@
  *   {"ok":false,"error":"..."}
  */
 
-import { infer, inferJSON } from "../lib/infer.ts";
+import {
+  assistantPromptPath,
+  buildAssistantContextMessage,
+  buildAssistantSystemPrompt,
+  normalizeAssistantPlan,
+  tryLocalAssistantPlan,
+} from "./assistant-intelligence.ts";
+import { infer } from "../lib/infer.ts";
 
 const INFER_TIMEOUT_MS = 15_000;
 
@@ -55,7 +62,7 @@ async function inferSmart(prompt: string, options: any): Promise<{ data: any; ra
   };
 }
 import { readFileSync } from "fs";
-import { join, dirname } from "path";
+import { join } from "path";
 import { spawn } from "child_process";
 
 // ── Streaming TTS via OpenAI API → ffplay ──────────────────────────
@@ -275,100 +282,6 @@ function playConfirm(intent: string): Promise<number> {
   return playCached(map[intent] ?? "Done.");
 }
 
-// ── Fast path: local intent matching (no LLM needed) ──────────────
-
-interface FastMatch {
-  actions: Array<{ intent: string; slots: Record<string, string> }>;
-  confirm: string; // which confirmation to play
-}
-
-function tryFastMatch(transcript: string, snapshot: any): FastMatch | null {
-  const t = transcript.toLowerCase().trim();
-  const activeApps = (snapshot.activeStage ?? []).map((w: any) => ({
-    app: w.app as string,
-    wid: w.wid as number,
-  }));
-
-  // Tile patterns
-  const tileMatch = t.match(
-    /(?:tile|snap|put|move)\s+(\w+)\s+(?:to\s+)?(?:the\s+)?(left|right|top|bottom|maximize|center|top.?left|top.?right|bottom.?left|bottom.?right|left.?third|center.?third|right.?third)/
-  );
-  if (tileMatch) {
-    const app = tileMatch[1];
-    const pos = tileMatch[2].replace(/\s+/g, "-");
-    return {
-      actions: [{ intent: "tile_window", slots: { app, position: pos } }],
-      confirm: "tile_window",
-    };
-  }
-
-  // Split screen: "split X and Y" or "X left Y right"
-  const splitMatch = t.match(/split\s+(\w+)\s+(?:and|&)\s+(\w+)/);
-  if (splitMatch) {
-    return {
-      actions: [
-        { intent: "tile_window", slots: { app: splitMatch[1], position: "left" } },
-        { intent: "tile_window", slots: { app: splitMatch[2], position: "right" } },
-      ],
-      confirm: "tile_window",
-    };
-  }
-
-  // Focus: "focus X" / "focus on X" / "switch to X" / "go to X"
-  const focusMatch = t.match(/(?:focus(?:\s+on)?|switch\s+to|go\s+to|show)\s+(?:the\s+)?(?:on\s+)?(\w+)/);
-  if (focusMatch && !t.includes("tile") && !t.includes("split")) {
-    const app = focusMatch[1];
-    if (app && app !== "on" && app !== "the") {
-      return {
-        actions: [{ intent: "focus", slots: { app } }],
-        confirm: "focus",
-      };
-    }
-  }
-
-  // Maximize: "maximize" / "full screen" / "make it big"
-  if (/maximize|full\s*screen|make\s+it\s+big/.test(t)) {
-    return {
-      actions: [{ intent: "tile_window", slots: { position: "maximize" } }],
-      confirm: "tile_window",
-    };
-  }
-
-  // Distribute: "grid" / "mosaic" / "distribute" / "even"
-  if (/grid|mosaic|distribute|even\s+(?:out|grid)|arrange/.test(t)) {
-    return {
-      actions: [{ intent: "distribute", slots: {} }],
-      confirm: "distribute",
-    };
-  }
-
-  // Corners: "quadrants" / "four corners"
-  if (/quadrants?|four\s+corners?|corners/.test(t) && activeApps.length >= 4) {
-    const positions = ["top-left", "top-right", "bottom-left", "bottom-right"];
-    return {
-      actions: activeApps.slice(0, 4).map((a: any, i: number) => ({
-        intent: "tile_window",
-        slots: { app: a.app, position: positions[i] },
-      })),
-      confirm: "tile_window",
-    };
-  }
-
-  // Thirds: "thirds"
-  if (/thirds/.test(t) && activeApps.length >= 3) {
-    const positions = ["left-third", "center-third", "right-third"];
-    return {
-      actions: activeApps.slice(0, 3).map((a: any, i: number) => ({
-        intent: "tile_window",
-        slots: { app: a.app, position: positions[i] },
-      })),
-      confirm: "tile_window",
-    };
-  }
-
-  return null; // No fast match — fall through to LLM
-}
-
 // Warm up cache on startup
 ensureVoiceCache().then(() => log("voice cache ready"));
 
@@ -376,70 +289,14 @@ log("worker started, streaming TTS ready");
 
 // ── Load system prompt once ────────────────────────────────────────
 
-const promptDir = join(dirname(import.meta.dir), "docs", "prompts");
-let systemPrompt: string;
-try {
-  systemPrompt = readFileSync(join(promptDir, "hands-off-system.md"), "utf-8")
-    .split("\n")
-    .filter((l) => !l.startsWith("# "))
-    .join("\n")
-    .trim();
-} catch {
-  systemPrompt = "You are a workspace assistant. Respond with JSON: {actions, spoken}.";
-}
-
-const intentCatalog = `
-tile_window: Tile a window to a screen position
-  Slots:
-    position (required): Named position or grid:CxR:C,R syntax.
-      Halves: left, right, top, bottom
-      Quarters (2x2): top-left, top-right, bottom-left, bottom-right
-      Thirds (3x1): left-third, center-third, right-third
-      Sixths (3x2): top-left-third, top-center-third, top-right-third, bottom-left-third, bottom-center-third, bottom-right-third
-      Fourths (4x1): first-fourth, second-fourth, third-fourth, last-fourth
-      Eighths (4x2): top-first-fourth, top-second-fourth, top-third-fourth, top-last-fourth, bottom-first-fourth, bottom-second-fourth, bottom-third-fourth, bottom-last-fourth
-      Special: maximize (full screen), center (centered floating)
-      Grid syntax: grid:CxR:C,R (e.g. grid:5x3:2,1 = center cell of 5x3 grid)
-    app (optional): Target app name — match loosely (e.g. "chrome" matches "Google Chrome")
-    wid (optional): Target window ID (from snapshot)
-    session (optional): Tmux session name
-  If no app/wid/session given, tiles the frontmost window.
-  "quarter" = 2x2 cell (top-left etc.), NOT a 4x1 fourth.
-  "top quarter" = top-left or top-right (2x2). "top third" = top-left-third (3x2).
-
-focus: Focus a window, app, or session
-  Slots: app, session, or wid (at least one)
-
-distribute: Arrange all visible windows in an even grid. No slots.
-
-search: Search windows by text
-  Slots: query (required)
-
-list_windows: List all visible windows. No slots.
-
-switch_layer: Switch to a workspace layer
-  Slots: layer (required) — name or index
-
-create_layer: Save current arrangement as a named layer
-  Slots: name (required)
-
-TILING PRESETS (use multiple tile_window actions):
-  "split screen" → left + right
-  "thirds" → left-third, center-third, right-third
-  "mosaic"/"grid" → use distribute
-  "corners"/"quadrants" → top-left, top-right, bottom-left, bottom-right
-  "stack" → top + bottom
-  "six-up"/"3 by 2" → 3x2 grid using the sixth positions
-  "eight-up"/"4 by 2" → 4x2 grid using the eighth positions
-`;
-
-systemPrompt = systemPrompt.replace("{{intent_catalog}}", intentCatalog);
+const systemPrompt = buildAssistantSystemPrompt();
 log("system prompt loaded");
 
 // ── Auto-restart on file changes ───────────────────────────────────
 
 const watchFiles = [
-  join(promptDir, "hands-off-system.md"),
+  assistantPromptPath,
+  join(import.meta.dir, "assistant-intelligence.ts"),
   import.meta.path, // this script itself
 ];
 
@@ -458,89 +315,6 @@ for (const f of watchFiles) {
   } catch {}
 }
 
-// ── Build context message from snapshot ─────────────────────────────
-
-function buildContextMessage(transcript: string, snap: any): string {
-  let msg = `USER: "${transcript}"\n\n`;
-  msg += "--- DESKTOP SNAPSHOT ---\n";
-
-  // Screens
-  const screens = snap.screens ?? [];
-  if (screens.length > 1) {
-    msg += `Displays: ${screens.map((s: any) => `${s.width}x${s.height}${s.isMain ? " (main)" : ""}`).join(", ")}\n`;
-  } else if (screens.length === 1) {
-    msg += `Screen: ${screens[0].width}x${screens[0].height}\n`;
-  }
-
-  // Stage Manager
-  if (snap.stageManager) {
-    msg += `Stage Manager: ON (grouping: ${snap.smGrouping ?? "all-at-once"})\n`;
-  }
-
-  // All windows — full inventory, ordered front-to-back (zIndex 0 = frontmost)
-  const windows = snap.windows ?? snap.activeStage ?? [];
-  const onScreen = windows.filter((w: any) => w.onScreen !== false);
-  const offScreen = windows.filter((w: any) => w.onScreen === false);
-
-  msg += `\nVisible windows (${onScreen.length}, front-to-back order):\n`;
-  for (const w of onScreen) {
-    const flags: string[] = [];
-    if (w.zIndex === 0) flags.push("FRONTMOST");
-    if (w.session) flags.push(`session:${w.session}`);
-    const flagStr = flags.length ? ` [${flags.join(", ")}]` : "";
-    msg += `  wid:${w.wid} ${w.app}: "${w.title}" — ${w.frame}${flagStr}\n`;
-  }
-
-  if (offScreen.length > 0) {
-    // Summarize hidden windows by app instead of listing all
-    const hiddenByApp: Record<string, number> = {};
-    for (const w of offScreen) {
-      const app = w.app;
-      hiddenByApp[app] = (hiddenByApp[app] || 0) + 1;
-    }
-    const summary = Object.entries(hiddenByApp)
-      .filter(([app]) => !["WindowManager", "Spotlight", "CursorUIViewService", "AutoFill", "coreautha", "loginwindow", "Open and Save Panel Service"].includes(app))
-      .map(([app, count]) => `${app}(${count})`)
-      .join(", ");
-    if (summary) {
-      msg += `\nHidden windows: ${summary}\n`;
-    }
-  }
-
-  // Terminals — cwd, running commands, claude, tmux
-  const terminals = snap.terminals ?? [];
-  if (terminals.length > 0) {
-    msg += `\nTerminal tabs (${terminals.length}):\n`;
-    for (const t of terminals) {
-      const flags: string[] = [];
-      if (t.hasClaude) flags.push("Claude Code");
-      if (t.tmuxSession) flags.push(`tmux:${t.tmuxSession}`);
-      if (!t.isActiveTab) flags.push("background tab");
-      const flagStr = flags.length ? ` [${flags.join(", ")}]` : "";
-      const cwd = t.cwd ? ` cwd:${t.cwd.replace(/^\/Users\/\w+\//, "~/")}` : "";
-      const cmds = (t.runningCommands ?? []).map((c: any) => c.command).join(", ");
-      const cmdStr = cmds ? ` running:${cmds}` : "";
-      msg += `  ${t.displayName}${cwd}${cmdStr}${flagStr}`;
-      if (t.windowId) msg += ` (wid:${t.windowId})`;
-      msg += "\n";
-    }
-  }
-
-  // Tmux sessions
-  const tmux = snap.tmuxSessions ?? [];
-  if (tmux.length > 0) {
-    msg += `\nTmux sessions: ${tmux.map((s: any) => `${s.name} (${s.windows} windows${s.attached ? ", attached" : ""})`).join(", ")}\n`;
-  }
-
-  // Layer
-  if (snap.currentLayer) {
-    msg += `\nCurrent layer: ${snap.currentLayer.name} (index: ${snap.currentLayer.index})\n`;
-  }
-
-  msg += "--- END SNAPSHOT ---\n";
-  return msg;
-}
-
 // ── Command loop ───────────────────────────────────────────────────
 
 const decoder = new TextDecoder();
@@ -588,7 +362,13 @@ async function processLine(line: string) {
 
     case "infer":
       try {
-        const userMessage = buildContextMessage(cmd.transcript, cmd.snapshot ?? {});
+        const localPlan = tryLocalAssistantPlan(cmd.transcript, cmd.snapshot ?? {});
+        if (localPlan) {
+          respond({ ok: true, data: localPlan });
+          break;
+        }
+
+        const userMessage = buildAssistantContextMessage(cmd.transcript, cmd.snapshot ?? {});
 
         const messages = (cmd.history ?? []).map((h: any) => ({
           role: h.role as "user" | "assistant",
@@ -605,11 +385,13 @@ async function processLine(line: string) {
           tag: "hands-off",
         });
 
+        const plan = normalizeAssistantPlan(data, cmd.transcript);
         respond({
           ok: true,
           data: {
-            ...data,
+            ...plan,
             _meta: {
+              ...plan._meta,
               provider: raw.provider,
               model: raw.model,
               durationMs: raw.durationMs,
@@ -649,30 +431,35 @@ async function processLine(line: string) {
       // Fire cached ack sound + inference in PARALLEL
       const ackPromise = playAck().catch((e) => log(`ack error: ${e.message}`));
 
-      // Build full context message from snapshot
-      const userMessage = buildContextMessage(transcript, snap);
-
       const messages = history.map((h: any) => ({
         role: h.role as "user" | "assistant",
         content: typeof h.content === "string" ? h.content : JSON.stringify(h.content),
       })).filter((m: any) => m.content && m.content.length > 0);
 
       let inferResult: any = null;
-      try {
-        const { data, raw } = await inferSmart(userMessage, {
-          provider: "xai",
-          model: "grok-4.20-beta-0309-non-reasoning",
-          system: systemPrompt,
-          messages,
-          temperature: 0.2,
-          maxTokens: 512,
-          tag: "hands-off",
-        });
-        inferResult = { ...data, _meta: { provider: raw.provider, model: raw.model, durationMs: raw.durationMs, tokens: raw.usage?.totalTokens } };
-        log(`⏱ inference done in ${raw.durationMs}ms`);
-      } catch (err: any) {
-        log(`⏱ inference error: ${err.message}`);
-        inferResult = { actions: [], spoken: "Sorry, I had trouble with that.", _meta: { error: err.message } };
+      const localPlan = tryLocalAssistantPlan(transcript, snap);
+      if (localPlan) {
+        inferResult = localPlan;
+        log("local planner matched");
+      } else {
+        const userMessage = buildAssistantContextMessage(transcript, snap);
+        try {
+          const { data, raw } = await inferSmart(userMessage, {
+            provider: "xai",
+            model: "grok-4.20-beta-0309-non-reasoning",
+            system: systemPrompt,
+            messages,
+            temperature: 0.2,
+            maxTokens: 512,
+            tag: "hands-off",
+          });
+          const plan = normalizeAssistantPlan(data, transcript);
+          inferResult = { ...plan, _meta: { ...plan._meta, provider: raw.provider, model: raw.model, durationMs: raw.durationMs, tokens: raw.usage?.totalTokens } };
+          log(`⏱ inference done in ${raw.durationMs}ms`);
+        } catch (err: any) {
+          log(`⏱ inference error: ${err.message}`);
+          inferResult = { actions: [], spoken: "Sorry, I had trouble with that.", _meta: { error: err.message } };
+        }
       }
 
       // Wait for ack to finish before narrating (don't overlap speech)