create-walle 0.9.11 → 0.9.13

package/template/wall-e/tools/local-tools.js

@@ -5,49 +5,12 @@ const fs = require('fs');
 const path = require('path');
 const execFileAsync = promisify(execFile);
 const execAsync = promisify(exec);
+const { SHELL_ALLOWLIST, SHELL_DENYLIST } = require('./shell-policy');
 
 const HOME = process.env.HOME || '/tmp';
 
 // ── Shell execution (expanded allowlist) ──
 
-const SHELL_ALLOWLIST = new Set([
-  // File ops
-  'ls', 'cat', 'head', 'tail', 'wc', 'find', 'file', 'stat', 'du', 'df',
-  'mkdir', 'cp', 'mv', 'touch', 'tee', 'readlink', 'realpath',
-  // Path utilities
-  'cd', 'pwd', 'basename', 'dirname', 'which', 'type', 'command',
-  // Search
-  'grep', 'rg', 'ag', 'fzf',
-  // Text processing
-  'sort', 'uniq', 'cut', 'tr', 'awk', 'sed', 'jq', 'yq', 'diff', 'comm',
-  'xargs', 'tee', 'fmt', 'column',
-  // Network
-  'curl', 'wget', 'ping', 'dig', 'host', 'nslookup',
-  // Dev tools
-  'git', 'node', 'npm', 'npx', 'python3', 'pip3', 'bun', 'deno',
-  'make', 'cargo', 'go', 'ruby', 'perl',
-  // Cloud / infra
-  'fly', 'docker', 'kubectl',
-  // System info
-  'date', 'echo', 'env', 'whoami', 'hostname', 'uname', 'uptime', 'ps', 'top',
-  'id', 'groups', 'printenv', 'locale', 'lsof',
-  // macOS specific
-  'open', 'pbcopy', 'pbpaste', 'say', 'defaults', 'mdfind', 'mdls',
-  'osascript', 'screencapture', 'sw_vers', 'system_profiler',
-  // Archive
-  'tar', 'zip', 'unzip', 'gzip', 'gunzip', 'xz', 'bzip2',
-  // Misc utilities
-  'less', 'more', 'true', 'false', 'yes', 'test', 'expr', 'seq', 'sleep',
-  'md5', 'shasum', 'base64', 'xxd',
-]);
-
-// Commands that are NEVER allowed regardless of context
-const SHELL_DENYLIST = new Set([
-  'rm', 'rmdir', 'kill', 'killall', 'shutdown', 'reboot',
-  'sudo', 'su', 'chmod', 'chown', 'chgrp',
-  'dd', 'mkfs', 'fdisk', 'diskutil',
-]);
-
 /**
  * Shell escape a single argument for safe interpolation.
  */
@@ -375,20 +338,41 @@ async function readFile(filePath, { max_bytes, offset, limit, sessionId, project
     content: numbered.join('\n') + '\n\n' + footer,
     truncated: more || byteCut,
     total_size: stat.size,
+    ...loadReadInstructionMetadata(resolved, { sessionId, projectRoot }),
   };
 }
 
+function loadReadInstructionMetadata(filePath, { sessionId, projectRoot } = {}) {
+  if (!sessionId || !projectRoot) return {};
+  try {
+    const { InstructionService } = require('../coding/instruction-service');
+    const result = new InstructionService().discoverForRead(filePath, {
+      projectRoot,
+      sessionId,
+    });
+    if (!result.instructions.length) return {};
+    return {
+      instruction_paths: result.loadedPaths,
+      instructions: result.instructions,
+    };
+  } catch {
+    return {};
+  }
+}
+
 async function writeFile(filePath, content, { sessionId, projectRoot } = {}) {
   const resolved = resolveToolPath(filePath, projectRoot);
   // Assert file unchanged since last read (stale edit prevention)
+  let FileTracker = null;
   if (sessionId) {
-    const FileTracker = require('./file-tracker');
+    FileTracker = require('./file-tracker');
     FileTracker.assertUnchanged(resolved, sessionId);
   }
   // Create parent dirs if needed
   const dir = path.dirname(resolved);
   if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
   fs.writeFileSync(resolved, content, 'utf8');
+  if (FileTracker) FileTracker.recordCurrent(resolved, sessionId);
   return { written: true, path: resolved, bytes: Buffer.byteLength(content) };
 }
 
@@ -1376,8 +1360,8 @@ async function executeLocalTool(name, input) {
     case 'list_directory': return listDirectory(input);
     case 'claude_code': return runClaudeCode(input);
     case 'edit_file': return editFile(input.file_path, input.old_string, input.new_string, input.replace_all, { sessionId: input.sessionId, projectRoot: input.projectRoot });
-    case 'apply_patch': return applyPatchTool(input.patch_text, { projectRoot: input.projectRoot });
-    case 'multi_edit': return multiEditTool(input.file_path, input.edits, { projectRoot: input.projectRoot });
+    case 'apply_patch': return applyPatchTool(input.patch_text, { sessionId: input.sessionId, projectRoot: input.projectRoot });
+    case 'multi_edit': return multiEditTool(input.file_path, input.edits, { sessionId: input.sessionId, projectRoot: input.projectRoot });
     default: return null; // not a local tool
   }
 }
@@ -1393,34 +1377,78 @@ async function editFile(filePath, oldString, newString, replaceAll = false, { se
   if (oldString === newString) throw new Error('old_string and new_string must differ');
   const resolved = resolveToolPath(filePath, projectRoot);
   // Assert file unchanged since last read (stale edit prevention)
+  let FileTracker = null;
   if (sessionId) {
-    const FileTracker = require('./file-tracker');
+    FileTracker = require('./file-tracker');
     FileTracker.assertUnchanged(resolved, sessionId);
   }
   const content = fs.readFileSync(resolved, 'utf8');
-  const updated = fuzzyReplace(content, oldString, newString, replaceAll);
+  let updated;
+  try {
+    updated = fuzzyReplace(content, oldString, newString, replaceAll);
+  } catch (err) {
+    if (isAlreadyAppliedEdit(content, oldString, newString)) {
+      if (FileTracker) FileTracker.recordCurrent(resolved, sessionId);
+      return { edited: false, unchanged: true, already_applied: true, path: resolved, bytes: Buffer.byteLength(content) };
+    }
+    throw err;
+  }
   // Defense in depth: replace() should throw on no-match, but if any future
   // code path returns content unchanged, refuse to silently "succeed" with
   // a no-op. The agent must see the failure and try again rather than
   // believing the file was edited when it wasn't.
   if (updated === content) {
+    if (isAlreadyAppliedEdit(content, oldString, newString)) {
+      if (FileTracker) FileTracker.recordCurrent(resolved, sessionId);
+      return { edited: false, unchanged: true, already_applied: true, path: resolved, bytes: Buffer.byteLength(content) };
+    }
     throw new Error(
       'edit_file no-op: applying old_string -> new_string left the file byte-identical. The match likely missed; provide more unique old_string context.'
     );
   }
   fs.writeFileSync(resolved, updated, 'utf8');
+  if (FileTracker) FileTracker.recordCurrent(resolved, sessionId);
   return { edited: true, path: resolved, bytes: Buffer.byteLength(updated) };
 }
 
-async function applyPatchTool(patchText, { projectRoot } = {}) {
+function isAlreadyAppliedEdit(content, oldString, newString) {
+  if (!newString || typeof newString !== 'string') return false;
+  if (content.includes(oldString)) return false;
+  return content.includes(newString);
+}
+
+async function applyPatchTool(patchText, { sessionId, projectRoot } = {}) {
   if (!patchText) throw new Error('patch_text is required');
-  return applyPatch(patchText, { baseDir: projectRoot });
+  const result = await applyPatch(patchText, { baseDir: projectRoot });
+  if (sessionId) {
+    const FileTracker = require('./file-tracker');
+    const trackingWarnings = [];
+    for (const filePath of [...(result.added || []), ...(result.modified || [])]) {
+      try {
+        FileTracker.recordCurrent(filePath, sessionId);
+      } catch (err) {
+        trackingWarnings.push(`Failed to refresh file tracker for ${filePath}: ${err.message}`);
+      }
+    }
+    for (const filePath of result.deleted || []) {
+      FileTracker.forgetFile(filePath, sessionId);
+    }
+    if (trackingWarnings.length > 0) result.tracking_warnings = trackingWarnings;
+  }
+  return result;
 }
 
-async function multiEditTool(filePath, edits, { projectRoot } = {}) {
+async function multiEditTool(filePath, edits, { sessionId, projectRoot } = {}) {
   if (!filePath || !edits || !Array.isArray(edits)) throw new Error('file_path and edits array are required');
   const resolved = resolveToolPath(filePath, projectRoot);
-  return multiEdit(resolved, edits);
+  let FileTracker = null;
+  if (sessionId) {
+    FileTracker = require('./file-tracker');
+    FileTracker.assertUnchanged(resolved, sessionId);
+  }
+  const result = multiEdit(resolved, edits);
+  if (FileTracker && result.edits_applied > 0) FileTracker.recordCurrent(resolved, sessionId);
+  return result;
 }
 
 // ── Enhanced Glob (Phase 9b) ──

package/template/wall-e/tools/session-sharing.js

@@ -5,6 +5,9 @@
 'use strict';
 
 const crypto = require('node:crypto');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
 
 /**
  * Session sharing — export coding sessions as JSON and import them back.
@@ -88,6 +91,8 @@ function exportSession(sessionId, brain) {
     }
   }
 
+  const transcript = _readWalleTranscriptMetadata(sessionId);
+
   // Store the share token in kv_store for later verification
   if (typeof brain.setKv === 'function') {
     brain.setKv(`share:${sessionId}`, JSON.stringify({
@@ -103,6 +108,10 @@ function exportSession(sessionId, brain) {
     plan,
     subtasks,
     events,
+    parts: transcript.parts,
+    compactions: transcript.compactions,
+    snapshots: transcript.snapshots,
+    transcriptPath: transcript.transcriptPath,
     diffs,
     createdAt,
     exportedAt: new Date().toISOString(),
@@ -228,4 +237,62 @@ function _tableExists(db, tableName) {
   }
 }
 
-module.exports = { exportSession, importSession, verifyShareToken, EXPORT_VERSION };
+function _readWalleTranscriptMetadata(sessionId) {
+  const transcriptPath = _findWalleTranscriptPath(sessionId);
+  if (!transcriptPath) return { transcriptPath: '', parts: [], compactions: [], snapshots: [] };
+
+  const parts = [];
+  let raw = '';
+  try { raw = fs.readFileSync(transcriptPath, 'utf8'); } catch { raw = ''; }
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue;
+    let row;
+    try { row = JSON.parse(line); } catch { continue; }
+    if (row?.type !== 'walle_part') continue;
+    const part = {
+      uuid: row.uuid || '',
+      partType: row.partType || 'event',
+      timestamp: row.timestamp || '',
+      data: row.data || {},
+    };
+    parts.push(part);
+  }
+  return {
+    transcriptPath,
+    parts,
+    compactions: parts.filter((part) => part.partType === 'compaction'),
+    snapshots: parts.filter((part) => part.partType === 'snapshot'),
+  };
+}
+
+function _findWalleTranscriptPath(sessionId) {
+  if (!sessionId) return '';
+  for (const root of _candidateWalleSessionRoots()) {
+    const candidate = path.join(root, `${sessionId}.jsonl`);
+    try {
+      if (fs.existsSync(candidate)) return candidate;
+    } catch { /* ignore inaccessible roots */ }
+  }
+  return '';
+}
+
+function _candidateWalleSessionRoots() {
+  const roots = [
+    process.env.WALLE_SESSIONS_DIR,
+    process.env.WALL_E_SESSIONS_DIR,
+    process.env.WALLE_DEV_DIR ? path.join(process.env.WALLE_DEV_DIR, 'sessions') : '',
+    process.env.WALL_E_DATA_DIR ? path.join(process.env.WALL_E_DATA_DIR, 'sessions') : '',
+    process.env.CTM_DATA_DIR ? path.join(process.env.CTM_DATA_DIR, 'sessions') : '',
+    path.join(os.homedir(), '.walle', 'sessions'),
+  ].filter(Boolean);
+  return [...new Set(roots)];
+}
+
+module.exports = {
+  exportSession,
+  importSession,
+  verifyShareToken,
+  EXPORT_VERSION,
+  _readWalleTranscriptMetadata,
+  _findWalleTranscriptPath,
+};

package/template/wall-e/tools/shell-analyzer.js

@@ -365,7 +365,7 @@ async function analyzeShellCommand(commandStr, cwd) {
   const tree = Parser.parse(commandStr);
   if (!tree) return result;
 
-  const { SHELL_ALLOWLIST, SHELL_DENYLIST } = require('./local-tools');
+  const { SHELL_ALLOWLIST, SHELL_DENYLIST } = require('./shell-policy');
   const commandNodes = collectCommandNodes(tree.rootNode);
 
   for (const node of commandNodes) {

package/template/wall-e/tools/shell-policy.js

@@ -0,0 +1,47 @@
+'use strict';
+
+// Shell execution policy shared by the local tool runner and static analyzer.
+// Keep this module dependency-free so shell analysis never imports local-tools.
+
+const SHELL_ALLOWLIST = new Set([
+  // File ops
+  'ls', 'cat', 'head', 'tail', 'wc', 'find', 'file', 'stat', 'du', 'df',
+  'mkdir', 'cp', 'mv', 'touch', 'tee', 'readlink', 'realpath',
+  // Path utilities
+  'cd', 'pwd', 'basename', 'dirname', 'which', 'type', 'command',
+  // Search
+  'grep', 'rg', 'ag', 'fzf',
+  // Text processing
+  'sort', 'uniq', 'cut', 'tr', 'awk', 'sed', 'jq', 'yq', 'diff', 'comm',
+  'xargs', 'tee', 'fmt', 'column',
+  // Network
+  'curl', 'wget', 'ping', 'dig', 'host', 'nslookup',
+  // Dev tools
+  'git', 'node', 'npm', 'npx', 'python3', 'pip3', 'bun', 'deno',
+  'make', 'cargo', 'go', 'ruby', 'perl', 'tsc',
+  // Cloud / infra
+  'fly', 'docker', 'kubectl',
+  // System info
+  'date', 'echo', 'env', 'whoami', 'hostname', 'uname', 'uptime', 'ps', 'top',
+  'id', 'groups', 'printenv', 'locale', 'lsof',
+  // macOS specific
+  'open', 'pbcopy', 'pbpaste', 'say', 'defaults', 'mdfind', 'mdls',
+  'osascript', 'screencapture', 'sw_vers', 'system_profiler',
+  // Archive
+  'tar', 'zip', 'unzip', 'gzip', 'gunzip', 'xz', 'bzip2',
+  // Misc utilities
+  'less', 'more', 'true', 'false', 'yes', 'test', 'expr', 'seq', 'sleep',
+  'md5', 'shasum', 'base64', 'xxd',
+]);
+
+// Commands that are NEVER allowed regardless of context.
+const SHELL_DENYLIST = new Set([
+  'rm', 'rmdir', 'kill', 'killall', 'shutdown', 'reboot',
+  'sudo', 'su', 'chmod', 'chown', 'chgrp',
+  'dd', 'mkfs', 'fdisk', 'diskutil',
+]);
+
+module.exports = {
+  SHELL_ALLOWLIST,
+  SHELL_DENYLIST,
+};

package/template/wall-e/tools/snapshot.js

@@ -171,6 +171,48 @@ class SnapshotManager {
     return [...this.snapshots.keys()];
   }
 
+  /**
+   * Return current snapshot stack depths by file. SnapshotService uses this as
+   * a cheap boundary marker before a coding step starts.
+   *
+   * @returns {Record<string, number>}
+   */
+  getStackDepths() {
+    const depths = {};
+    for (const [filePath, stack] of this.snapshots) {
+      depths[filePath] = stack.length;
+    }
+    return depths;
+  }
+
+  /**
+   * Restore files to the state captured immediately after the provided stack
+   * depths. If a file was first captured after the boundary, depth 0 restores
+   * the pre-boundary file content (or deletes it when it was new).
+   *
+   * @param {Record<string, number>} depths
+   * @returns {string[]} restored file paths
+   */
+  restoreToStackDepths(depths = {}) {
+    const restored = [];
+    for (const [filePath, stack] of this.snapshots) {
+      const depth = Number.isFinite(depths[filePath]) ? depths[filePath] : 0;
+      if (stack.length <= depth) continue;
+      const targetIndex = Math.min(depth, stack.length - 1);
+      const { content } = stack[targetIndex];
+      if (content === null) {
+        try { fs.unlinkSync(filePath); } catch { /* already gone */ }
+      } else if (Buffer.isBuffer(content)) {
+        fs.writeFileSync(filePath, content);
+      } else {
+        fs.writeFileSync(filePath, content, 'utf8');
+      }
+      stack.length = targetIndex + 1;
+      restored.push(filePath);
+    }
+    return restored;
+  }
+
   /**
    * Return a unified diff string between the last snapshot and the current file content.
    * Uses a simple line-by-line diff algorithm (no external dependencies).

package/template/wall-e/training/harvester.js

@@ -5,6 +5,17 @@ const path = require('path');
 const { createHash } = require('crypto');
 const { execSync } = require('child_process');
 
+let claudeDesktopSessions = null;
+function getClaudeDesktopSessions() {
+  if (claudeDesktopSessions) return claudeDesktopSessions;
+  try {
+    claudeDesktopSessions = require('../../claude-task-manager/lib/claude-desktop-sessions');
+  } catch {
+    return null;
+  }
+  return claudeDesktopSessions;
+}
+
 // --- Task type classification ---
 
 function classifyTaskType(content) {
@@ -86,6 +97,46 @@ async function harvestClaudeCodeSessions(since) {
   return samples;
 }
 
+// --- Claude Desktop Session Harvesting ---
+
+async function harvestClaudeDesktopSessions(since) {
+  const reader = getClaudeDesktopSessions();
+  if (!reader) return [];
+
+  const sessions = reader.listSessions();
+  const samples = [];
+
+  for (const session of sessions) {
+    if (since && session.updatedAt && session.updatedAt <= since) continue;
+    const messages = Array.isArray(session.messages) ? session.messages : [];
+    for (let i = 0; i < messages.length - 1; i++) {
+      const userMsg = messages[i];
+      const assistantMsg = messages[i + 1];
+      if (userMsg.role !== 'user' || assistantMsg.role !== 'assistant') continue;
+      const userContent = userMsg.text || '';
+      const assistantContent = assistantMsg.text || '';
+      if (!userContent || userContent.length < 20) continue;
+      if (!assistantContent || assistantContent.length < 20) continue;
+
+      samples.push({
+        id: contentHash('claude-desktop', `${session.uuid}:${i}:${userContent}`),
+        source: 'claude-desktop',
+        session_id: session.uuid,
+        timestamp: userMsg.timestamp || session.updatedAt || session.createdAt || new Date().toISOString(),
+        task_type: classifyTaskType(userContent),
+        prompt: userContent,
+        response: assistantContent,
+        tool_calls: [],
+        outcome: 'unknown',
+        outcome_signal: { git_committed: false, git_diff: null, task_status: null, user_corrected: false },
+        model: session.model || 'unknown',
+        quality_label: 0.5,
+      });
+    }
+  }
+  return samples;
+}
+
 // --- Codex Session Harvesting ---
 
 async function harvestCodexSessions(since) {
@@ -144,12 +195,13 @@ async function harvestCodexSessions(since) {
 
 // --- CTM Session Harvesting ---
 
-async function harvestCtmSessions(since) {
-  const dataDir = process.env.WALL_E_DATA_DIR || path.join(process.env.HOME, '.walle', 'data');
+async function harvestCtmSessions(since, dataDirOverride = null) {
+  const dataDir = dataDirOverride || process.env.WALL_E_DATA_DIR || path.join(process.env.HOME, '.walle', 'data');
   const ctmDbPath = path.join(dataDir, 'task-manager.db');
   if (!fs.existsSync(ctmDbPath)) return [];
 
-  const Database = require('better-sqlite3');
+  let Database;
+  try { Database = require('better-sqlite3'); } catch { return []; }
   let ctmDb;
   try {
     ctmDb = new Database(ctmDbPath, { readonly: true, fileMustExist: true });
@@ -405,12 +457,13 @@ async function runHarvest({ incremental = true, brain, dataDir } = {}) {
 
   // Harvest from each source
   const claudeSamples = await harvestClaudeCodeSessions(getSince('claude-code'));
+  const claudeDesktopSamples = await harvestClaudeDesktopSessions(getSince('claude-desktop'));
   const codexSamples = await harvestCodexSessions(getSince('codex'));
   const chatSamples = await harvestWalleChat(brain, getSince('walle-chat'));
   const taskSamples = await harvestWalleTasks(brain, getSince('walle-task'));
-  const ctmSamples = await harvestCtmSessions(getSince('ctm-sessions'));
+  const ctmSamples = await harvestCtmSessions(getSince('ctm-sessions'), dataDir);
 
-  allSamples.push(...claudeSamples, ...codexSamples, ...chatSamples, ...taskSamples, ...ctmSamples);
+  allSamples.push(...claudeSamples, ...claudeDesktopSamples, ...codexSamples, ...chatSamples, ...taskSamples, ...ctmSamples);
 
   // Deduplicate by content hash
   const seen = new Set();
@@ -440,6 +493,9 @@ async function runHarvest({ incremental = true, brain, dataDir } = {}) {
   if (claudeSamples.length > 0) {
     brain.updateHarvestState('claude-code', { lastProcessedAt: now, totalHarvested: claudeSamples.length });
   }
+  if (claudeDesktopSamples.length > 0) {
+    brain.updateHarvestState('claude-desktop', { lastProcessedAt: now, totalHarvested: claudeDesktopSamples.length });
+  }
   if (codexSamples.length > 0) {
     brain.updateHarvestState('codex', { lastProcessedAt: now, totalHarvested: codexSamples.length });
   }
@@ -460,6 +516,7 @@ module.exports = {
   classifyTaskType,
   contentHash,
   harvestClaudeCodeSessions,
+  harvestClaudeDesktopSessions,
   harvestCodexSessions,
   harvestCtmSessions,
   harvestWalleChat,