create-walle 0.9.30 → 0.9.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-walle",
-  "version": "0.9.30",
+  "version": "0.9.31",
   "description": "CTM + Wall-E — AI coding dashboard and personal digital twin agent. Multi-agent terminal for Claude Code, Codex, Gemini, Aider, OpenCode, and more, plus prompt editor, task queue, remote phone and tablet access, code/doc review, and an agent that learns from Slack, email & calendar.",
   "bin": {
     "create-walle": "bin/create-walle.js"

package/template/claude-task-manager/api-prompts.js

@@ -28,6 +28,7 @@ const { queryPromptExecutions } = require('./lib/prompt-executions-query');
 const { createIngestCooldown } = require('./lib/ingest-cooldown');
 const { claudeFileSessionId, _usageMetadata } = require('./lib/jsonl-conversation-parser');
 const structuredCapture = require('./lib/structured-capture');
+const { coopSchedulerEnabled } = require('./lib/db-owner-cooperative-scheduler');
 // AI search uses direct HTTP calls to Claude API (supports Portkey proxy)
 
 let dbMaintenanceRunner = null;
@@ -1852,10 +1853,9 @@ function _readResetChunkedRouteStats() {
 
 function _chunkedImportFlags(sessionId, messages, fileSize, reason) {
   // Gated on the coop scheduler: deferring rows only helps when the cooperative scheduler is driving
-  // bounded slices (and the content-rows-backfill coop/legacy job fills the deferred rows). With the
-  // flag OFF the import behaves byte-for-byte as legacy (rows written inline) — so merging this branch
-  // is a true no-op on the primary until CTM_COOP_SCHEDULER=1 is set.
-  if (process.env.CTM_COOP_SCHEDULER !== '1') return { skipMessageRows: false, forceBlobWrite: false };
+  // bounded slices (and the content-rows-backfill coop/legacy job fills the deferred rows). The feature
+  // is ON by default; set CTM_COOP_SCHEDULER=0 to fall back to byte-for-byte legacy (rows written inline).
+  if (!coopSchedulerEnabled()) return { skipMessageRows: false, forceBlobWrite: false };
   const minMsgs = Math.max(1, Number(process.env.CTM_IMPORT_CHUNK_MIN_MSGS || 1000));
   // ALSO trigger on file size: a giant rollout is tail-bounded at parse time (transcriptMaxBytes),
   // so `messages.length` here can be small even though the session is huge and its inline row write

package/template/claude-task-manager/api-reviews.js

@@ -23,6 +23,56 @@ function isValidProjectPath(p) {
     return true;
   } catch { return false; }
 }
+// Resolve `filePath` within `project` and enforce containment AFTER following
+// symlinks. A lexical `resolved.startsWith(root + sep)` check is not enough: a
+// symlink that lives inside the repo can point at /etc/passwd (or ~/.ssh) and the
+// lexical path still looks in-bounds. realpath canonicalizes the link target, so
+// containment is checked against where the file ACTUALLY is.
+// Returns the safe absolute path to read, or null if it escapes the project.
+// A path that doesn't exist yet (ENOENT) can't be a symlink, so the lexical check
+// already guarantees safety — return it so the caller can produce its own 404.
+// Best-effort fallback when an html path doesn't resolve at the given root — e.g. a path
+// relative to the repo root while the session cwd is a subdir. Searches tracked + untracked
+// .html within the repo and returns the best suffix/basename match (stays inside the root).
+function findHtmlInRepo(root, relPath) {
+  try {
+    const { execFileSync } = require('child_process');
+    const want = String(relPath || '').replace(/^\.?\//, '');
+    const base = path.basename(want);
+    const out = execFileSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard', '*.html', '*.htm'],
+      { cwd: root, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024 }).split('\0').filter(Boolean);
+    let hit = out.find(f => f === want || f.endsWith('/' + want));
+    if (!hit) hit = out.find(f => path.basename(f) === base);
+    return hit ? path.join(root, hit) : null;
+  } catch { return null; }
+}
+
+function htmlNotFoundPage(filePath) {
+  const safe = String(filePath || '').replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
+  return `<!DOCTYPE html><html><head><meta charset="utf-8"><title>File not found</title>
+<style>body{font-family:-apple-system,system-ui,sans-serif;background:#1a1b26;color:#c0caf5;display:grid;place-items:center;height:100vh;margin:0}
+.box{text-align:center;max-width:34rem;padding:2rem}h1{font-size:1.1rem;color:#f7768e}code{background:rgba(122,162,247,.12);padding:.15rem .4rem;border-radius:4px;font-size:.85em;word-break:break-all}p{color:#8b94bd;line-height:1.6;font-size:.9rem}</style>
+</head><body><div class="box"><h1>File not found</h1>
+<p><code>${safe}</code> isn't in this checkout. It may live in a different worktree or branch than the one this session is running in.</p></div></body></html>`;
+}
+
+function resolveWithinProject(project, filePath) {
+  if (!project || typeof filePath !== 'string') return null;
+  const root = path.resolve(project);
+  const resolved = path.resolve(root, filePath);
+  if (resolved !== root && !resolved.startsWith(root + path.sep)) return null; // lexical ../ escape
+  let realRoot;
+  try { realRoot = fs.realpathSync(root); }
+  catch { return null; }
+  let realFile;
+  try { realFile = fs.realpathSync(resolved); }
+  catch (e) {
+    if (e && e.code === 'ENOENT') return resolved; // lexically in-bounds, not yet on disk
+    return null;
+  }
+  if (realFile !== realRoot && !realFile.startsWith(realRoot + path.sep)) return null; // symlink escape
+  return realFile;
+}
 function jsonResponse(res, code, data) {
   const body = JSON.stringify(data);
   res.writeHead(code, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) });
@@ -364,9 +414,9 @@ function handleReviewApi(req, res, url) {
     const ext = path.extname(filePath).toLowerCase();
     const imageExts = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', '.svg': 'image/svg+xml', '.webp': 'image/webp', '.ico': 'image/x-icon', '.bmp': 'image/bmp' };
     if (!imageExts[ext]) return jsonResponse(res, 400, { error: 'Only image files are supported' });
-    // Prevent path traversal
-    const resolved = path.resolve(project, filePath);
-    if (!resolved.startsWith(path.resolve(project) + path.sep)) return jsonResponse(res, 403, { error: 'Invalid file path' });
+    // Prevent path traversal (incl. symlinks escaping the project — see resolveWithinProject)
+    const resolved = resolveWithinProject(project, filePath);
+    if (!resolved) return jsonResponse(res, 403, { error: 'Invalid file path' });
     try {
       const data = fs.readFileSync(resolved);
       res.writeHead(200, { 'Content-Type': imageExts[ext], 'Content-Length': data.length, 'Cache-Control': 'no-cache' });
@@ -462,6 +512,105 @@ function handleReviewApi(req, res, url) {
     return jsonResponse(res, 200, { prompt });
   }
 
+  // GET /api/reviews/documents?project=... — list reviewable markdown/text files in a
+  // project (powers the Review Hub "Documents" discovery column). Tracked + untracked-
+  // not-ignored, bounded, sorted most-recently-modified first.
+  if (p === '/api/reviews/documents' && m === 'GET') {
+    const project = url.searchParams.get('project');
+    if (!project) return jsonResponse(res, 400, { error: 'project required' });
+    if (!isValidProjectPath(project)) return jsonResponse(res, 403, { error: 'Invalid project path' });
+    try {
+      const { execFileSync } = require('child_process');
+      const root = path.resolve(project);
+      const globs = ['*.md', '*.markdown', '*.mdown', '*.txt'];
+      // -z → NUL-separated so paths with spaces/specials survive (no core.quotePath issues).
+      const run = (args) => {
+        try {
+          return execFileSync('git', args, { cwd: root, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024 })
+            .split('\0').filter(Boolean);
+        } catch { return []; }
+      };
+      const set = new Set([
+        ...run(['ls-files', '-z', ...globs]),
+        ...run(['ls-files', '-z', '--others', '--exclude-standard', ...globs]),
+      ]);
+      let items = [...set].slice(0, 2000).map(rel => {
+        let mtime = 0;
+        try { mtime = fs.statSync(path.join(root, rel)).mtimeMs; } catch {}
+        return { path: rel, name: path.basename(rel), dir: path.dirname(rel), mtime };
+      });
+      items.sort((a, b) => b.mtime - a.mtime);
+      return jsonResponse(res, 200, { documents: items.slice(0, 300), total: set.size });
+    } catch (e) {
+      return jsonResponse(res, 500, { error: e.message });
+    }
+  }
+
+  // GET /api/reviews/serve-html?project=...&path=... — serve a project .html file so the
+  // file-router can open it in a new browser tab. Path-validated; no traversal outside project.
+  if (p === '/api/reviews/serve-html' && m === 'GET') {
+    const project = url.searchParams.get('project');
+    const filePath = url.searchParams.get('path');
+    if (!project || !filePath) return jsonResponse(res, 400, { error: 'project and path required' });
+    if (!isValidProjectPath(project)) return jsonResponse(res, 403, { error: 'Invalid project path' });
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext !== '.html' && ext !== '.htm') return jsonResponse(res, 400, { error: 'Only .html files are supported' });
+    let resolved = resolveWithinProject(project, filePath);
+    if (!resolved) return jsonResponse(res, 403, { error: 'Invalid file path' });
+    // Fallback: the path may be relative to the repo root while the session cwd is a
+    // subdir/worktree — search the repo for the file before giving up.
+    if (!fs.existsSync(resolved)) {
+      const found = findHtmlInRepo(path.resolve(project), filePath);
+      if (found) resolved = found;
+    }
+    // Sandbox the served file: arbitrary repo HTML rendered on the CTM origin (which holds
+    // the auth cookie). A bare `sandbox` blocks scripting AND assigns an opaque origin, so it
+    // can't run JS, read CTM storage, or make cookie-authed calls — inline CSS/images still render.
+    const htmlHeaders = (len) => ({
+      'Content-Type': 'text/html; charset=utf-8',
+      'Content-Length': len,
+      'Cache-Control': 'no-cache',
+      'X-Content-Type-Options': 'nosniff',
+      'Content-Security-Policy': 'sandbox',
+    });
+    let data;
+    try { data = fs.readFileSync(resolved); }
+    catch {
+      // Friendly in-tab page instead of a scary "unreachable file" / bare text 404.
+      const body = Buffer.from(htmlNotFoundPage(filePath));
+      res.writeHead(404, htmlHeaders(body.length));
+      res.end(body);
+      return true;
+    }
+    res.writeHead(200, htmlHeaders(data.length));
+    res.end(data);
+    return true;
+  }
+
+  // POST /api/reviews/reveal {project, path} — reveal a project file in Finder (macOS
+  // `open -R`). Path-validated; best-effort (the file-router menu's "Reveal" action).
+  if (p === '/api/reviews/reveal' && m === 'POST') {
+    readBody(req).then(body => {
+      const project = String(body.project || '');
+      const filePath = String(body.path || '');
+      if (!project || !filePath) return jsonResponse(res, 400, { error: 'project and path required' });
+      if (!isValidProjectPath(project)) return jsonResponse(res, 403, { error: 'Invalid project path' });
+      const resolved = resolveWithinProject(project, filePath);
+      if (!resolved) return jsonResponse(res, 403, { error: 'Invalid file path' });
+      if (!fs.existsSync(resolved)) return jsonResponse(res, 404, { error: 'File not found' });
+      try {
+        const { execFile } = require('child_process');
+        const cmd = process.platform === 'darwin' ? 'open' : 'xdg-open';
+        const args = process.platform === 'darwin' ? ['-R', resolved] : [path.dirname(resolved)];
+        execFile(cmd, args, () => {});
+        jsonResponse(res, 200, { ok: true });
+      } catch (e) {
+        jsonResponse(res, 500, { error: e.message });
+      }
+    }).catch(e => jsonResponse(res, 400, { error: e.message }));
+    return true;
+  }
+
   return false; // Not handled
 }
 
@@ -508,4 +657,5 @@ module.exports = {
   _sanitizeReviewCommentInput: sanitizeReviewCommentInput,
   _sanitizeReviewCommentUpdate: sanitizeReviewCommentUpdate,
   _documentReview: documentReview,
+  _resolveWithinProject: resolveWithinProject,
 };

package/template/claude-task-manager/approval-agent.js

@@ -250,6 +250,24 @@ function getApproveKeystroke(context, options = {}) {
   return context.hasAllowAll ? '2' : '1';
 }
 
+// Fallback tool-name derivation when no tool header is recognized above the
+// prompt: read it from the durable grant option ("Yes, and don't ask again for
+// <X> commands in <dir>" / "Yes, allow reading from <dir>") so an unrecognized
+// tool is still targetable by a heuristic/learned rule instead of collapsing to
+// 'Unknown'. `lines` is the trimmed, non-empty line array; `proceedIdx` the
+// "Do you want to…?" line. Pure.
+function _toolNameFromDurableOption(lines, proceedIdx) {
+  for (let i = proceedIdx + 1; i < Math.min(proceedIdx + 8, lines.length); i++) {
+    const l = String(lines[i] || '');
+    const m = l.match(/don'?t ask again for\s+(.+?)\s+commands?\b/i);
+    if (m && m[1]) return m[1].trim();
+    if (/allow\s+reading\s+from\b/i.test(l)) return 'Read';
+    const a = l.match(/allow\s+all\s+(\w+)\b/i);
+    if (a && a[1]) return a[1].trim();
+  }
+  return '';
+}
+
 function _parseGenericApprovalContext(cleanText, providerId) {
   const lines = String(cleanText || '').split('\n').map(l => l.trim()).filter(Boolean);
   if (!lines.length) return null;
@@ -284,12 +302,16 @@ function _parseGenericApprovalContext(cleanText, providerId) {
   const contextLines = [];
   for (let i = proceedIdx - 1; i >= Math.max(0, proceedIdx - 30); i--) {
     const line = lines[i];
-    if (/^[⏺●]?\s*(Bash command|Bash|Edit|Write|Read|Glob|Grep|Fetch|WebFetch|NotebookEdit|TodoWrite|Agent|MCP)\b/i.test(line)) {
+    if (/^[⏺●]?\s*(Bash command|Bash|Edit|Write|Read|Glob|Grep|Fetch|WebFetch|Web ?Search|NotebookEdit|TodoWrite|Agent|MCP)\b/i.test(line)) {
       toolName = line.trim();
       break;
     }
     contextLines.unshift(line);
   }
+  if (toolName === 'Generic approval') {
+    const durable = _toolNameFromDurableOption(lines, proceedIdx);
+    if (durable) toolName = durable;
+  }
 
   const fullContext = lines.slice(Math.max(0, proceedIdx - 20), Math.min(lines.length, proceedIdx + 12)).join('\n');
   return {
@@ -373,7 +395,7 @@ function parseApprovalContext(cleanText, providerId) {
   for (let i = endIdx - 1; i >= Math.max(0, endIdx - 30); i--) {
     const line = lines[i];
     // Detect tool headers (Claude Code shows "Bash command", "⏺ Bash(...)", "Edit", etc.)
-    if (/^[⏺●]?\s*(Bash command|Bash|Edit|Write|Read|Glob|Grep|Fetch|WebFetch|NotebookEdit|TodoWrite|Agent)\b/.test(line)) {
+    if (/^[⏺●]?\s*(Bash command|Bash|Edit|Write|Read|Glob|Grep|Fetch|WebFetch|Web ?Search|NotebookEdit|TodoWrite|Agent)\b/.test(line)) {
       toolName = line.trim();
       break;
     }
@@ -403,6 +425,10 @@ function parseApprovalContext(cleanText, providerId) {
     if (fileOp) { toolName = fileOp.toolName; fileOpCommand = fileOp.command; }
   }
 
+  // Still no header → derive a name from the durable grant option so the tool is
+  // targetable instead of 'Unknown' (mirror of the claude-code provider path).
+  if (!toolName) toolName = _toolNameFromDurableOption(lines, proceedIdx);
+
   // Bash approvals render the command followed by one dimmed prose description
   // line; drop it from the command so titles/signatures stay command-shaped
   // (mirror of the claude-code provider parse path).
@@ -1150,8 +1176,14 @@ function reviewWithHeuristics(context) {
   // "mcp__…" / "(mcp)" command text.
   const isMcp = /^(?:plugin:|mcp__|mcp\b)/i.test(tool) || /\(mcp\)/i.test(cmd);
   if (isMcp) {
-    const readOnlyMcp = /\b(navigate(?:_back)?|snapshot|take_screenshot|screenshot|read|list|get|query|search|console_messages|network_requests?|wait_for|hover|tabs|resolve[-_ ]library[-_ ]id|get[-_ ]library[-_ ]docs|browser_snapshot|browser_navigate)\b/i;
-    const mutatingMcp = /\b(click|type|fill|drag|drop|file_upload|upload|run_code|evaluate|press_key|select_option|handle_dialog|write|create|delete|remove|update|send|post|install|deploy|exec)\b/i;
+    // Boundaries are `(?<![a-z0-9]) … (?![a-z0-9])` rather than `\b`: snake_case
+    // tool ids (mcp__server__get_thread) glue the verb to the noun with `_`, which
+    // IS a regex word char, so `\bget\b` never fired and every read-only snake_case
+    // MCP op (Gmail/Calendar/Drive/wall-e) fell through to the medium verifier path.
+    // The class treats `_`, `-`, space, `:`, and start/end as separators while still
+    // rejecting alnum-glued false positives (`get` in "target", `draft` in "drafts").
+    const readOnlyMcp = /(?<![a-z0-9])(navigate(?:_back)?|snapshot|take_screenshot|screenshot|read|list|get|query|search|fetch|download|find|view|suggest|stats|status|brief|ask|console_messages|network_requests?|wait_for|hover|tabs|resolve[-_ ]?library[-_ ]?id|get[-_ ]?library[-_ ]?docs|browser_snapshot|browser_navigate)(?![a-z0-9])/i;
+    const mutatingMcp = /(?<![a-z0-9])(click|type|fill|drag|drop|file_upload|upload|run_code|evaluate|press_key|select_option|handle_dialog|write|create|delete|remove|update|send|post|install|deploy|exec|label|unlabel|respond|copy|move|add|set|insert|draft|authenticate|complete|reconnect|ingest|rebuild|export|remember|put|patch)(?![a-z0-9])/i;
     if (readOnlyMcp.test(tool) && !mutatingMcp.test(tool)) {
       return { decision: 'approve', reasoning: 'Read-only MCP operation (heuristic)', riskLevel: 'low',
         ruleLabel: context.toolName || 'MCP read-only operation',
@@ -1169,7 +1201,7 @@ function reviewWithHeuristics(context) {
   // because Edit/Write diffs may contain code with "drop table" or "rm -rf" as
   // string literals — those are code content, not dangerous operations).
   const lowRisk = [
-    /^(read|glob|grep|webfetch|notebookedit)/, // Read-only tools
+    /^(read|glob|grep|webfetch|web ?search|notebookedit)/, // Read-only tools (incl. Web Search — read-only, no side effects, same tier as WebFetch)
     /^(edit|write)\b/, // Edit/Write to project files — normal dev workflow
   ];
   for (const re of lowRisk) {