@idl3/claude-control 0.1.20 → 0.1.22

package/lib/transcribe.js

@@ -0,0 +1,156 @@
+/**
+ * lib/transcribe.js — local speech-to-text via ffmpeg + whisper.cpp.
+ *
+ * No API key, no cloud: transcodes the uploaded audio to 16kHz mono WAV with
+ * ffmpeg, then runs the whisper-cli binary (brew install whisper-cpp) against a
+ * local ggml model. Works for any browser that can record audio (incl. iOS
+ * Safari), which the Web Speech API does not.
+ *
+ * Exports:
+ *  - resolveFfmpeg() / resolveWhisperBin() / resolveWhisperModel() → string | null
+ *  - cleanTranscript(raw) → string   (pure; strips timestamps/blank markers)
+ *  - transcribe(inputPath, { lang }) → Promise<string>
+ *
+ * Binary/model resolution is overridable via env (FFMPEG_BIN, WHISPER_BIN,
+ * WHISPER_MODEL) and defaults to Homebrew + ~/.claude-control/models.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { spawn, execFileSync } from 'node:child_process';
+
+const MODELS_DIR = path.join(os.homedir(), '.claude-control', 'models');
+
+/** Resolve a binary: env override → `which` → known fallbacks. */
+function resolveBin(name, envVar, fallbacks) {
+  const e = process.env[envVar];
+  if (e && e.trim() && fs.existsSync(e.trim())) return e.trim();
+  try {
+    const w = execFileSync('which', [name], { encoding: 'utf8' }).trim();
+    if (w && fs.existsSync(w)) return w;
+  } catch {
+    /* not on PATH */
+  }
+  for (const f of fallbacks) if (fs.existsSync(f)) return f;
+  return null;
+}
+
+/** @returns {string | null} */
+export function resolveFfmpeg() {
+  return resolveBin('ffmpeg', 'FFMPEG_BIN', [
+    '/opt/homebrew/bin/ffmpeg',
+    '/usr/local/bin/ffmpeg',
+    '/usr/bin/ffmpeg',
+  ]);
+}
+
+/** @returns {string | null} */
+export function resolveWhisperBin() {
+  return resolveBin('whisper-cli', 'WHISPER_BIN', [
+    '/opt/homebrew/bin/whisper-cli',
+    '/usr/local/bin/whisper-cli',
+  ]);
+}
+
+/**
+ * Resolve the ggml model: WHISPER_MODEL env → preferred names in the models
+ * dir → any `ggml-*.bin` there.
+ * @returns {string | null}
+ */
+export function resolveWhisperModel() {
+  const e = process.env.WHISPER_MODEL;
+  if (e && e.trim() && fs.existsSync(e.trim())) return e.trim();
+  const prefs = [
+    'ggml-base.en.bin',
+    'ggml-small.en.bin',
+    'ggml-base.bin',
+    'ggml-small.bin',
+    'ggml-tiny.en.bin',
+  ];
+  for (const m of prefs) {
+    const p = path.join(MODELS_DIR, m);
+    if (fs.existsSync(p)) return p;
+  }
+  try {
+    const found = fs.readdirSync(MODELS_DIR).find((n) => /^ggml-.*\.bin$/.test(n));
+    if (found) return path.join(MODELS_DIR, found);
+  } catch {
+    /* dir missing */
+  }
+  return null;
+}
+
+/**
+ * Clean whisper-cli stdout into a single line: drop blank lines, drop
+ * bracketed-only markers ([BLANK_AUDIO], (silence)), collapse whitespace.
+ *
+ * @param {string} raw
+ * @returns {string}
+ */
+export function cleanTranscript(raw) {
+  return String(raw)
+    .split('\n')
+    .map((l) => l.trim())
+    .filter(Boolean)
+    .filter((l) => !/^[[(][^\])]*[\])]$/.test(l))
+    .join(' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+/** Spawn a binary, capture stdout/stderr, resolve on exit 0. */
+function run(bin, args) {
+  return new Promise((resolve, reject) => {
+    const p = spawn(bin, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    let out = '';
+    let err = '';
+    p.stdout.on('data', (d) => {
+      out += d;
+    });
+    p.stderr.on('data', (d) => {
+      err += d;
+    });
+    p.on('error', reject);
+    p.on('close', (code) =>
+      code === 0
+        ? resolve({ stdout: out, stderr: err })
+        : reject(new Error(`${path.basename(bin)} exited ${code}: ${err.slice(0, 500)}`)),
+    );
+  });
+}
+
+/**
+ * Transcribe an audio file (any ffmpeg-readable format) to text.
+ *
+ * @param {string} inputPath - path to the recorded audio file.
+ * @param {{ lang?: string }} [opts]
+ * @returns {Promise<string>}
+ */
+export async function transcribe(inputPath, { lang = 'en' } = {}) {
+  const ffmpeg = resolveFfmpeg();
+  const whisper = resolveWhisperBin();
+  const model = resolveWhisperModel();
+  if (!ffmpeg) throw new Error('ffmpeg not found (brew install ffmpeg)');
+  if (!whisper) throw new Error('whisper-cli not found (brew install whisper-cpp)');
+  if (!model) throw new Error(`no whisper model found in ${MODELS_DIR}`);
+
+  const wav = path.join(
+    os.tmpdir(),
+    `cc-stt-${Date.now()}-${process.pid}.wav`,
+  );
+  try {
+    await run(ffmpeg, [
+      '-nostdin', '-y',
+      '-i', inputPath,
+      '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le',
+      '-f', 'wav', wav,
+    ]);
+    const { stdout } = await run(whisper, [
+      '-m', model, '-f', wav, '-np', '-nt', '-l', lang,
+    ]);
+    return cleanTranscript(stdout);
+  } finally {
+    fs.promises.unlink(wav).catch(() => {});
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@idl3/claude-control",
-  "version": "0.1.20",
+  "version": "0.1.22",
   "type": "module",
   "description": "Local web UI to watch and drive your Claude Code sessions running in tmux — live transcripts, reply, answer AskUserQuestion, attach files, from a browser or phone.",
   "keywords": [

package/server.js

@@ -20,11 +20,23 @@ import { parsePanePrompt } from './lib/prompt.js';
 import { SessionRegistry, listRecentTranscripts } from './lib/sessions.js';
 import { loadPins, savePins, validateTranscriptPath, pinKey } from './lib/pins.js';
 import { ResourceMonitor } from './lib/resources.js';
-import { buildAnswerProgram } from './lib/answer.js';
+import { buildAnswerProgram, parsePicker, planStep } from './lib/answer.js';
 import { sweepUploads, resolveUploadPath } from './lib/uploads.js';
 import { getVersionInfo, currentVersion } from './lib/version.js';
 import * as push from './lib/push.js';
 import { readConfig, writeConfig } from './lib/config.js';
+import { optimizePrompt, rulesOptimize } from './lib/optimize.js';
+import { complete as claudeCliComplete } from './lib/claude-cli.js';
+import * as mlx from './lib/mlx.js';
+import {
+  MLX_MODELS,
+  CLAUDE_MODELS,
+  detectMachine,
+  recommendMlxModel,
+  recommendClaudeModel,
+} from './lib/models.js';
+import { transcribe } from './lib/transcribe.js';
+import { listSkills } from './lib/skills.js';
 // Note: the client offers [WS_PROTOCOL, token] as subprotocols; the `ws`
 // library auto-selects the FIRST offered one (the non-secret WS_PROTOCOL label)
 // and echoes it, so we never reflect the raw token back and need no custom
@@ -172,6 +184,10 @@ const server = http.createServer((req, res) => {
     if (!checkToken(req)) return endJson(res, 401, { error: 'unauthorized' });
     return endJson(res, 200, { sessions: registry.getSessions() });
   }
+  if (u.pathname === '/api/skills') {
+    if (!checkToken(req)) return endJson(res, 401, { error: 'unauthorized' });
+    return endJson(res, 200, { skills: listSkills() });
+  }
   if (u.pathname === '/api/health') {
     if (!checkToken(req)) return endJson(res, 401, { error: 'unauthorized' });
     return endJson(res, 200, { ok: true, snapshot: resources.snapshot() });
@@ -242,6 +258,29 @@ const server = http.createServer((req, res) => {
     if (req.method === 'POST') return handleConfigSave(req, res);
     return endJson(res, 405, { error: 'method not allowed' });
   }
+  if (u.pathname === '/api/optimize') {
+    if (req.method !== 'POST') return endJson(res, 405, { error: 'method not allowed' });
+    if (!checkToken(req)) return endJson(res, 401, { error: 'unauthorized' });
+    return handleOptimize(req, res);
+  }
+  if (u.pathname === '/api/models') {
+    if (!checkToken(req)) return endJson(res, 401, { error: 'unauthorized' });
+    const machine = detectMachine();
+    return endJson(res, 200, {
+      machine,
+      // Mark which MLX models are already in the local HF cache so the UI can
+      // show downloaded vs. will-download (avoids a surprise multi-GB fetch).
+      mlxModels: MLX_MODELS.map((m) => ({ ...m, installed: mlx.isModelCached(m.id) })),
+      claudeModels: CLAUDE_MODELS,
+      recommendedMlxModel: recommendMlxModel(machine.ramGB),
+      recommendedClaudeModel: recommendClaudeModel(),
+    });
+  }
+  if (u.pathname === '/api/transcribe') {
+    if (req.method !== 'POST') return endJson(res, 405, { error: 'method not allowed' });
+    if (!checkToken(req)) return endJson(res, 401, { error: 'unauthorized' });
+    return handleTranscribe(req, res, u);
+  }
   if (u.pathname === '/api/session/new') {
     if (req.method !== 'POST') return endJson(res, 405, { error: 'method not allowed' });
     if (!checkToken(req)) return endJson(res, 401, { error: 'unauthorized' });
@@ -456,12 +495,110 @@ async function handleConfigSave(req, res) {
   }
   try {
     const saved = writeConfig(body);
+    // If the MLX backend is active, (re)warm the selected model now — this
+    // restarts the local server with the new model and starts any needed
+    // download in the background, so the user doesn't hit a cold stall (or a
+    // wrong-model hang) on their next ✨ enhance.
+    if (saved.optimizeBackend === 'mlx' && mlx.resolveMlxPython()) {
+      mlx.warm();
+    }
     return endJson(res, 200, saved);
   } catch (err) {
     return endJson(res, 400, { error: String(err?.message || err) });
   }
 }
 
+// POST /api/optimize — token-gated prompt optimiser. Accepts { text, intent }
+// and returns { optimized, rationale, changes, mode } from optimizePrompt.
+// Falls back to rules-based optimization when the Claude CLI is unavailable.
+async function handleOptimize(req, res) {
+  let body;
+  try {
+    body = await readJsonBody(req);
+  } catch (err) {
+    return endJson(res, 400, { error: 'invalid JSON body' });
+  }
+  const text = typeof body.text === 'string' ? body.text : '';
+  if (!text.trim()) return endJson(res, 400, { error: 'text required' });
+  if (text.length > 8000) return endJson(res, 400, { error: 'text exceeds 8000 character limit' });
+  const intent = typeof body.intent === 'string' ? body.intent : undefined;
+  try {
+    const result = await runOptimize(text, intent);
+    return endJson(res, 200, result);
+  } catch (err) {
+    return endJson(res, 500, { error: String(err?.message || err) });
+  }
+}
+
+// Run the enhancer through the configured backend chain, recording WHICH backend
+// actually produced the result so the UI can label it accurately:
+//  - 'mlx'    → try local MLX, then claude -p, then rules.
+//  - 'claude' → try claude -p, then rules.
+//  - 'rules'  → deterministic rules optimiser only.
+// optimizePrompt returns mode:'rules' when its injected complete() fails, so a
+// non-'llm' mode means that backend fell through → try the next.
+async function runOptimize(text, intent) {
+  const cfg = readConfig();
+  const backend = cfg.optimizeBackend;
+  if (backend === 'rules') {
+    return { ...rulesOptimize(text), backend: 'rules' };
+  }
+  const order = backend === 'claude' ? ['claude'] : ['mlx', 'claude'];
+  for (const b of order) {
+    const complete = b === 'mlx' ? (p) => mlx.complete(p) : claudeCliComplete;
+    const r = await optimizePrompt(text, { complete, intent });
+    if (r.mode === 'llm') {
+      return { ...r, backend: b, model: b === 'mlx' ? cfg.mlxModel : cfg.optimizeModel };
+    }
+  }
+  return { ...rulesOptimize(text), backend: 'rules' };
+}
+
+// POST /api/transcribe — local speech-to-text. Accepts a raw audio body (the
+// MediaRecorder blob from the voice dialog; ?ext=webm|mp4|wav names the format),
+// caps the size, writes it to a temp file, and runs ffmpeg→whisper.cpp via
+// lib/transcribe. Returns { ok, text }. No key, no cloud — fully local.
+function handleTranscribe(req, res, u) {
+  const maxBytes = CONFIG.maxUploadMB * 1024 * 1024;
+  const ext =
+    (u.searchParams.get('ext') || 'webm').toLowerCase().replace(/[^a-z0-9]/g, '').slice(0, 5) ||
+    'webm';
+  const chunks = [];
+  let size = 0;
+  let aborted = false;
+
+  req.on('data', (c) => {
+    if (aborted) return;
+    size += c.length;
+    if (size > maxBytes) {
+      aborted = true;
+      endJson(res, 413, { error: `audio exceeds ${CONFIG.maxUploadMB} MB limit` });
+      req.destroy();
+      return;
+    }
+    chunks.push(c);
+  });
+
+  req.on('end', async () => {
+    if (aborted) return;
+    if (size === 0) return endJson(res, 400, { error: 'empty audio' });
+    const tmp = path.join(os.tmpdir(), `cc-stt-in-${Date.now()}-${process.pid}.${ext}`);
+    try {
+      await fs.promises.writeFile(tmp, Buffer.concat(chunks), { mode: 0o600 });
+      const text = await transcribe(tmp);
+      endJson(res, 200, { ok: true, text });
+    } catch (err) {
+      endJson(res, 500, { error: String(err?.message || err) });
+    } finally {
+      fs.promises.unlink(tmp).catch(() => {});
+    }
+  });
+
+  req.on('error', () => {
+    if (!aborted) endJson(res, 400, { error: 'audio stream error' });
+  });
+}
+
 // POST /api/session/new — create a new tmux window in the configured (or
 // body-overridden) cwd, then type the launch command into it via send-keys so
 // the interactive shell resolves aliases. Security: the command is operator
@@ -1044,25 +1181,212 @@ async function handleClientMessage(ws, msg) {
       if (msg.toolUseId !== pending.toolUseId) {
         throw new Error('stale question (already answered or changed)');
       }
-      const keys = buildAnswerProgram(pending, msg.selections || []);
-      // Log the resolved key program so a failure to drive the picker is
-      // diagnosable from ~/.claude-control/logs/out.log (no logging existed before).
-      console.log(
-        `[answer] toolUseId=${msg.toolUseId} target=${session.target} keys=${JSON.stringify(keys)}`,
-      );
+
+      // ── Capture-driven path ──────────────────────────────────────────────
+      // Attempt to navigate by parsing the live picker render. Falls back to
+      // the static buildAnswerProgram on ANY parse failure, unknown label, or
+      // post-send verification mismatch — so it can NEVER regress the working path.
+      //
+      // Constants:
+      const SETTLE_MS = 300;   // ms to wait after sending keys before re-capture
+      const MAX_RETRIES = 1;   // retry attempts per question on verification failure
+
+      let usedDynamic = false;
+      // Tracks whether the dynamic path has injected ANY keystroke. Once true,
+      // the picker is in a partial/unknown state and the from-scratch static
+      // fallback would corrupt it — so a later failure must fail loud, not retry.
+      let sentAny = false;
       try {
-        // Sequenced (with delays) so the picker's re-render settles between keys.
-        await tmux.sendRawKeysSequenced(session.target, keys);
-      } catch (err) {
-        // Surface the failure to the log and re-throw so the outer handler nacks
-        // (ok:false) — never let an "answer sent" ack imply success when the keys
-        // never landed in the pane.
+        const questions = pending?.questions || [];
+        const selections = msg.selections || [];
+
+        if (questions.length > 0) {
+          let dynamicOk = true; // will be set false to fall back
+
+          for (let qi = 0; qi < questions.length && dynamicOk; qi += 1) {
+            const question = questions[qi];
+            const selectedLabels = selections[qi] || [];
+
+            let attempt = 0;
+            let stepOk = false;
+
+            while (attempt <= MAX_RETRIES && !stepOk) {
+              // 1. Capture current picker state.
+              let capture;
+              try {
+                capture = await tmux.capturePane(session.target);
+              } catch (captureErr) {
+                console.log(`[answer/dynamic] capture failed q${qi}: ${captureErr?.message}`);
+                dynamicOk = false;
+                break;
+              }
+
+              // 2. Parse.
+              const parsed = parsePicker(capture);
+              if (parsed.confidence !== 'ok') {
+                console.log(`[answer/dynamic] low confidence on q${qi} — falling back`);
+                dynamicOk = false;
+                break;
+              }
+
+              // 3. Handle the review screen (multi-question final step).
+              if (parsed.isReview) {
+                // We expect to be here only after the last question's action Enter.
+                // Send Enter to confirm "Submit answers".
+                console.log(`[answer/dynamic] review screen — sending Enter`);
+                sentAny = true;
+                await tmux.sendRawKeysSequenced(session.target, ['Enter'], SETTLE_MS);
+                await new Promise((r) => setTimeout(r, SETTLE_MS));
+                // Verify: the review screen should be gone.
+                const afterReview = await tmux.capturePane(session.target);
+                const reparse = parsePicker(afterReview);
+                if (reparse.isReview) {
+                  console.log(`[answer/dynamic] review screen still up after Enter — falling back`);
+                  dynamicOk = false;
+                }
+                // Whether verified or not, we break out of the question loop —
+                // we've processed all questions.
+                break;
+              }
+
+              // 4. Plan keystrokes for this question.
+              const keys = planStep(parsed, question, selectedLabels);
+              if (!keys) {
+                console.log(`[answer/dynamic] planStep null on q${qi} — falling back`);
+                dynamicOk = false;
+                break;
+              }
+
+              console.log(
+                `[answer/dynamic] q${qi} attempt=${attempt} keys=${JSON.stringify(keys)}`,
+              );
+
+              // 5. Send keys.
+              sentAny = true;
+              await tmux.sendRawKeysSequenced(session.target, keys, SETTLE_MS);
+
+              // 6. Settle then verify.
+              await new Promise((r) => setTimeout(r, SETTLE_MS));
+              let afterCapture;
+              try {
+                afterCapture = await tmux.capturePane(session.target);
+              } catch (captureErr) {
+                console.log(`[answer/dynamic] post-send capture failed q${qi}: ${captureErr?.message}`);
+                dynamicOk = false;
+                break;
+              }
+
+              const afterParsed = parsePicker(afterCapture);
+
+              if (question.multiSelect) {
+                // Verify: all intended labels are now checked in the re-parsed picker.
+                // If we advanced (Next/Submit pressed), the screen changes — that's
+                // also acceptable (confidence goes low = we moved on).
+                if (afterParsed.confidence === 'ok' && !afterParsed.isReview) {
+                  const uncheckedTargets = selectedLabels.filter((label) =>
+                    afterParsed.rows.some(
+                      (r) => r.kind === 'option' && r.label === label && !r.checked,
+                    ),
+                  );
+                  if (uncheckedTargets.length > 0) {
+                    console.log(
+                      `[answer/dynamic] verify failed q${qi}: still unchecked=${JSON.stringify(uncheckedTargets)} attempt=${attempt}`,
+                    );
+                    attempt += 1;
+                    continue; // retry
+                  }
+                }
+                // Either confidence is low (screen advanced) or all checked — either
+                // way, treat the step as done and move to the next question.
+                stepOk = true;
+              } else {
+                // Single-select: after Enter, picker should advance (screen changes).
+                // If the exact same option is still shown as selected (cursor on it),
+                // something went wrong. Accept any screen change as advancement.
+                if (
+                  afterParsed.confidence === 'ok' &&
+                  !afterParsed.isReview &&
+                  afterParsed.rows.some(
+                    (r) => r.cursor && r.kind === 'option' && r.label === selectedLabels[0],
+                  )
+                ) {
+                  console.log(`[answer/dynamic] single-select stuck on q${qi} attempt=${attempt}`);
+                  attempt += 1;
+                  continue;
+                }
+                stepOk = true;
+              }
+            }
+
+            if (!stepOk && attempt > MAX_RETRIES) {
+              console.log(`[answer/dynamic] max retries exceeded on q${qi} — falling back`);
+              dynamicOk = false;
+            }
+          }
+
+          // After processing all questions via dynamic path, check if we need to
+          // handle the review screen (multi-question pickers).
+          if (dynamicOk && questions.length > 1) {
+            // Capture and check: we may already be on the review screen (handled
+            // in the loop above) or may need to check.
+            try {
+              const finalCapture = await tmux.capturePane(session.target);
+              const finalParsed = parsePicker(finalCapture);
+              if (finalParsed.isReview && finalParsed.confidence === 'ok') {
+                // Submit the review screen.
+                console.log(`[answer/dynamic] post-loop review screen — sending Enter`);
+                sentAny = true;
+                await tmux.sendRawKeysSequenced(session.target, ['Enter'], SETTLE_MS);
+              }
+            } catch (captureErr) {
+              // Non-fatal: we already sent the question answers; review Enter is best-effort.
+              console.log(`[answer/dynamic] final review capture failed: ${captureErr?.message}`);
+            }
+          }
+
+          if (dynamicOk) {
+            usedDynamic = true;
+          }
+        }
+      } catch (dynamicErr) {
+        // Any unexpected error in the dynamic path — log and fall back.
+        console.log(`[answer/dynamic] unexpected error: ${dynamicErr?.message} — falling back`);
+      }
+
+      // ── Static fallback ──────────────────────────────────────────────────
+      // Only safe when the dynamic path sent NOTHING (picker still pristine). If
+      // dynamic already injected keys then failed, the picker is in a partial
+      // state — replaying the from-scratch static program would mis-navigate a
+      // dirty picker and corrupt the answer. Fail loud so the user can retry.
+      if (!usedDynamic && sentAny) {
         console.error(
-          `[answer] FAILED toolUseId=${msg.toolUseId} target=${session.target}: ${String(err?.message || err)}`,
+          `[answer] dynamic path failed AFTER sending keys; NOT running static fallback (picker dirty) toolUseId=${msg.toolUseId}`,
         );
-        throw err;
+        return send(ws, {
+          type: 'ack',
+          op: 'answer',
+          ok: false,
+          error: 'answer injection failed mid-picker — please retry',
+        });
       }
-      console.log(`[answer] sent toolUseId=${msg.toolUseId} (${keys.length} keys)`);
+      if (!usedDynamic) {
+        const keys = buildAnswerProgram(pending, msg.selections || []);
+        console.log(
+          `[answer] toolUseId=${msg.toolUseId} target=${session.target} keys=${JSON.stringify(keys)} (static fallback)`,
+        );
+        try {
+          await tmux.sendRawKeysSequenced(session.target, keys);
+        } catch (err) {
+          console.error(
+            `[answer] FAILED toolUseId=${msg.toolUseId} target=${session.target}: ${String(err?.message || err)}`,
+          );
+          throw err;
+        }
+        console.log(`[answer] sent toolUseId=${msg.toolUseId} (${keys.length} keys)`);
+      } else {
+        console.log(`[answer] sent toolUseId=${msg.toolUseId} via dynamic path`);
+      }
+
       return send(ws, { type: 'ack', op: 'answer', ok: true });
     }
     case 'capture': {
@@ -1182,6 +1506,16 @@ async function main() {
     } else {
       console.log('   (no COCKPIT_TOKEN set — relying on 127.0.0.1 bind. This UI can type into your sessions.)');
     }
+    // Pre-warm the local MLX enhancer so the first ✨ enhance is fast (best-effort;
+    // only when that backend is selected and an mlx python is available).
+    try {
+      if (readConfig().optimizeBackend === 'mlx' && mlx.resolveMlxPython()) {
+        mlx.warm();
+        console.log('   (pre-warming local MLX enhancer model…)');
+      }
+    } catch {
+      /* best-effort */
+    }
   });
 }