create-walle 0.9.21 → 0.9.23

package/template/wall-e/scripts/db-optimize/migrate.js

@@ -0,0 +1,162 @@
+'use strict';
+// Brain-DB space optimization core + CLI. Idempotent, safe to run repeatedly.
+//   1. Embeddings: convert embedding_vec_ollama float32 -> int8 IN PLACE (same table name)
+//   2. memories_fts -> external-content + maintenance triggers
+//   3. content_raw: NULL where byte-identical to content (readers use content_raw ?? content)
+//   4. lost_and_found: drop (orphaned .recover residue) — only when opts.dropLostAndFound
+//   5. VACUUM (opts.vacuum)
+//
+// Exposed as a function so the deferred upgrader loop (loops/brain-optimize.js) and the CLI
+// share one implementation. The caller must have sqlite-vec loaded on `db` (the daemon's
+// connection already does; the CLI loads it below).
+//
+// CLI: <node25> migrate.js <dbPath> [--no-vacuum] [--keep-lost-and-found]
+
+function declaredVecType(db, t) {
+  const r = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?").get(t);
+  if (!r || !r.sql) return null;
+  if (/\bint8\s*\[/.test(r.sql)) return 'int8';
+  if (/\bfloat\s*\[/.test(r.sql)) return 'float';
+  return '?';
+}
+function has(db, name) { return !!db.prepare("SELECT 1 FROM sqlite_master WHERE name=?").get(name); }
+
+// True if any optimization is still pending (used to skip no-op work / gate the loop).
+function needsOptimization(db) {
+  if (has(db, 'embedding_vec_ollama') && declaredVecType(db, 'embedding_vec_ollama') === 'float') return true;
+  const fts = db.prepare("SELECT sql FROM sqlite_master WHERE name='memories_fts'").get();
+  if (fts && !/content\s*=\s*'memories'/.test(fts.sql)) return true;
+  if (has(db, 'memories')) {
+    const red = db.prepare("SELECT 1 FROM memories WHERE content_raw IS NOT NULL AND content_raw=content LIMIT 1").get();
+    if (red) return true;
+  }
+  return false;
+}
+function embeddingCount(db) {
+  try { return db.prepare("SELECT COUNT(*) c FROM embedding_vec_ollama").get().c; } catch { return 0; }
+}
+
+function _migrateEmbeddings(db, log) {
+  const T = 'embedding_vec_ollama';
+  if (!has(db, T)) { log('embeddings: no', T, '- skip'); return; }
+  const type = declaredVecType(db, T);
+  if (type === 'int8') { log('embeddings: already int8 - skip'); return; }
+  if (type !== 'float') { log('embeddings: unexpected type', type, '- skip'); return; }
+  const dims = (db.prepare("SELECT sql FROM sqlite_master WHERE name=?").get(T).sql.match(/\[(\d+)\]/) || [])[1] || '768';
+  const n = db.prepare(`SELECT COUNT(*) c FROM ${T}`).get().c;
+  const tmp = T + '_i8mig';
+  log(`embeddings: quantizing ${n} float32[${dims}] vectors -> int8...`);
+  db.exec('BEGIN');
+  db.exec(`DROP TABLE IF EXISTS "${tmp}"`);
+  db.exec(`CREATE VIRTUAL TABLE "${tmp}" USING vec0(embedding int8[${dims}])`);
+  db.exec(`INSERT INTO "${tmp}"(rowid, embedding) SELECT rowid, vec_quantize_int8(embedding,'unit') FROM "${T}"`);
+  db.exec(`DROP TABLE "${T}"`);
+  db.exec(`CREATE VIRTUAL TABLE "${T}" USING vec0(embedding int8[${dims}])`);
+  db.exec(`INSERT INTO "${T}"(rowid, embedding) SELECT rowid, embedding FROM "${tmp}"`);
+  db.exec(`DROP TABLE "${tmp}"`);
+  db.exec('COMMIT');
+  const after = db.prepare(`SELECT COUNT(*) c FROM ${T}`).get().c;
+  if (after !== n) throw new Error(`embedding row count changed ${n} -> ${after}`);
+  log(`embeddings: done, ${after} int8 vectors`);
+}
+
+function _ftsTriggers(db) {
+  db.exec(`CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories BEGIN
+    INSERT INTO memories_fts(rowid, content, source, participants, source_channel)
+    VALUES (NEW.rowid, NEW.content, NEW.source, COALESCE(NEW.participants,''), COALESCE(NEW.source_channel,''));
+  END`);
+  db.exec(`CREATE TRIGGER memories_fts_delete AFTER DELETE ON memories BEGIN
+    INSERT INTO memories_fts(memories_fts, rowid, content, source, participants, source_channel)
+    VALUES('delete', OLD.rowid, OLD.content, OLD.source, COALESCE(OLD.participants,''), COALESCE(OLD.source_channel,''));
+  END`);
+  db.exec(`CREATE TRIGGER memories_fts_update AFTER UPDATE ON memories BEGIN
+    INSERT INTO memories_fts(memories_fts, rowid, content, source, participants, source_channel)
+    VALUES('delete', OLD.rowid, OLD.content, OLD.source, COALESCE(OLD.participants,''), COALESCE(OLD.source_channel,''));
+    INSERT INTO memories_fts(rowid, content, source, participants, source_channel)
+    VALUES (NEW.rowid, NEW.content, NEW.source, COALESCE(NEW.participants,''), COALESCE(NEW.source_channel,''));
+  END`);
+}
+function _migrateFts(db, log) {
+  const cur = db.prepare("SELECT sql FROM sqlite_master WHERE name='memories_fts'").get();
+  if (!cur) { log('fts: no memories_fts - skip'); return; }
+  if (/content\s*=\s*'memories'/.test(cur.sql)) { log('fts: already external-content - skip'); return; }
+  log('fts: converting to external-content...');
+  db.exec('BEGIN');
+  db.exec('DROP TRIGGER IF EXISTS memories_fts_insert');
+  db.exec('DROP TRIGGER IF EXISTS memories_fts_delete');
+  db.exec('DROP TRIGGER IF EXISTS memories_fts_update');
+  db.exec('DROP TABLE memories_fts');
+  db.exec(`CREATE VIRTUAL TABLE memories_fts USING fts5(
+    content, source, participants, source_channel,
+    content='memories', content_rowid='rowid', tokenize='porter unicode61')`);
+  db.exec(`INSERT INTO memories_fts(memories_fts) VALUES('rebuild')`);
+  _ftsTriggers(db);
+  db.exec('COMMIT');
+  log('fts: external-content + triggers installed, rebuilt');
+}
+
+function _migrateContentRaw(db, log) {
+  if (!has(db, 'memories')) return;
+  const n = db.prepare("SELECT COUNT(*) c FROM memories WHERE content_raw IS NOT NULL AND content_raw=content").get().c;
+  if (n === 0) { log('content_raw: nothing redundant - skip'); return; }
+  const hadTrig = !!db.prepare("SELECT 1 FROM sqlite_master WHERE name='memories_fts_update'").get();
+  db.exec('BEGIN');
+  if (hadTrig) db.exec('DROP TRIGGER IF EXISTS memories_fts_update'); // content unchanged -> FTS unaffected
+  const r = db.prepare("UPDATE memories SET content_raw=NULL WHERE content_raw IS NOT NULL AND content_raw=content").run();
+  if (hadTrig) db.exec(`CREATE TRIGGER memories_fts_update AFTER UPDATE ON memories BEGIN
+    INSERT INTO memories_fts(memories_fts, rowid, content, source, participants, source_channel)
+    VALUES('delete', OLD.rowid, OLD.content, OLD.source, COALESCE(OLD.participants,''), COALESCE(OLD.source_channel,''));
+    INSERT INTO memories_fts(rowid, content, source, participants, source_channel)
+    VALUES (NEW.rowid, NEW.content, NEW.source, COALESCE(NEW.participants,''), COALESCE(NEW.source_channel,''));
+  END`);
+  db.exec('COMMIT');
+  log(`content_raw: nulled ${r.changes} redundant rows`);
+}
+
+function _dropLostAndFound(db, log) {
+  if (!has(db, 'lost_and_found')) { log('lost_and_found: absent - skip'); return; }
+  db.exec('DROP TABLE lost_and_found');
+  log('lost_and_found: dropped');
+}
+
+/**
+ * Run the space optimization on an open better-sqlite3 `db` that already has sqlite-vec loaded.
+ * opts: { vacuum=true, dropLostAndFound=false, log=console.log }
+ * Each phase is idempotent and transactional; the float32 vectors are never dropped until the
+ * int8 copy is built and verified in the same transaction (a crash rolls back to float32).
+ */
+function optimizeBrain(db, opts = {}) {
+  const { vacuum = true, dropLostAndFound = false, log = () => {} } = opts;
+  _migrateEmbeddings(db, log);
+  _migrateFts(db, log);
+  _migrateContentRaw(db, log);
+  if (dropLostAndFound) _dropLostAndFound(db, log);
+  if (vacuum) { log('vacuuming...'); db.exec('VACUUM'); try { db.pragma('wal_checkpoint(TRUNCATE)'); } catch {} }
+}
+
+module.exports = { optimizeBrain, needsOptimization, embeddingCount, declaredVecType };
+
+// ---- CLI ----
+if (require.main === module) {
+  const path = require('path');
+  const WE = path.resolve(__dirname, '..', '..');
+  const Database = require(path.join(WE, 'node_modules', 'better-sqlite3'));
+  const vec = require(path.join(WE, 'node_modules', 'sqlite-vec'));
+  const DB = process.argv[2];
+  if (!DB) { console.error('usage: migrate.js <dbPath> [--no-vacuum] [--keep-lost-and-found]'); process.exit(1); }
+  const db = new Database(DB);
+  vec.load(db);
+  db.pragma('busy_timeout = 60000');
+  const MB = b => (b / 1048576).toFixed(0);
+  const before = db.prepare('SELECT SUM(pgsize) b FROM dbstat').get().b;
+  const t0 = Date.now();
+  optimizeBrain(db, {
+    vacuum: !process.argv.includes('--no-vacuum'),
+    dropLostAndFound: !process.argv.includes('--keep-lost-and-found'),
+    log: (...a) => console.log('[migrate]', ...a),
+  });
+  const after = db.prepare('SELECT SUM(pgsize) b FROM dbstat').get().b;
+  console.log(`[migrate] logical size ${MB(before)} -> ${MB(after)} MB in ${((Date.now()-t0)/1000).toFixed(1)}s`);
+  db.close();
+  console.log('[migrate] DONE');
+}

package/template/wall-e/scripts/db-optimize/recall-eval.js

@@ -0,0 +1,117 @@
+'use strict';
+// Phase 1a — embedding quantization recall eval (decision gate).
+// Runs ONLY on a copy. Builds int8 + binary shadow vec tables from the existing
+// float32 vectors, then measures recall@k by using stored vectors as query probes
+// (this isolates *quantization distortion* — how much the quantized index reorders
+// the same neighbors — without needing ollama to embed fresh queries).
+//
+// Usage: <node25> recall-eval.js [copyPath] [sampleN]
+const path = require('path');
+const WE = path.resolve(__dirname, '..', '..');
+const Database = require(path.join(WE, 'node_modules', 'better-sqlite3'));
+const vec = require(path.join(WE, 'node_modules', 'sqlite-vec'));
+
+const COPY = process.argv[2] || path.join(process.env.HOME || process.cwd(), '.walle', 'opt-work', 'wall-e-brain.copy.db');
+const SAMPLE = parseInt(process.argv[3] || '80', 10);
+const KMAX = 30;
+
+const db = new Database(COPY);
+vec.load(db);
+db.pragma('cache_size = -200000'); // 200MB page cache for the KNN scans
+
+const SRC = 'embedding_vec_ollama';      // float[768]
+const I8 = 'embedding_vec_ollama_i8';    // int8[768]
+const BIT = 'embedding_vec_ollama_bit';  // bit[768]
+
+function tableBytes(name) {
+  // sum pgsize across the vec0 shadow tables for this logical table
+  try {
+    const r = db.prepare(
+      `SELECT COALESCE(SUM(pgsize),0) AS b FROM dbstat WHERE name LIKE ? OR name LIKE ?`
+    ).get(name + '%', 'sqlite_autoindex_' + name + '%');
+    return r.b;
+  } catch (e) { return -1; }
+}
+
+function buildQuant(target, type, expr) {
+  const exists = db.prepare("SELECT 1 FROM sqlite_master WHERE name=?").get(target);
+  if (exists) { db.exec(`DROP TABLE ${target}`); }
+  db.exec(`CREATE VIRTUAL TABLE ${target} USING vec0(embedding ${type})`);
+  const t0 = Date.now();
+  try {
+    db.exec(`INSERT INTO ${target}(rowid, embedding) SELECT rowid, ${expr} FROM ${SRC}`);
+  } catch (e) {
+    // fallback: row-by-row
+    const sel = db.prepare(`SELECT rowid, embedding FROM ${SRC}`);
+    const ins = db.prepare(`INSERT INTO ${target}(rowid, embedding) VALUES (?, ${expr.replace('embedding', '?')})`);
+    const tx = db.transaction(() => { for (const r of sel.iterate()) ins.run(r.rowid, r.embedding); });
+    tx();
+  }
+  return Date.now() - t0;
+}
+
+console.log(`# Embedding quantization recall eval`);
+console.log(`copy: ${COPY}`);
+const srcN = db.prepare(`SELECT COUNT(*) n FROM ${SRC}`).get().n;
+console.log(`float32 vectors: ${srcN}, sample queries: ${SAMPLE}, k up to ${KMAX}\n`);
+
+console.log('building int8 shadow table...');
+const t8 = buildQuant(I8, 'int8[768]', "vec_quantize_int8(embedding,'unit')");
+console.log(`  built in ${t8}ms`);
+console.log('building binary(bit) shadow table...');
+const tb = buildQuant(BIT, 'bit[768]', 'vec_quantize_binary(embedding)');
+console.log(`  built in ${tb}ms\n`);
+
+const bytesF = tableBytes(SRC), bytes8 = tableBytes(I8), bytesB = tableBytes(BIT);
+const MB = b => (b / 1048576).toFixed(1);
+console.log(`size  float32: ${MB(bytesF)} MB   int8: ${MB(bytes8)} MB (${(bytesF/bytes8).toFixed(1)}x)   bit: ${MB(bytesB)} MB (${(bytesF/bytesB).toFixed(1)}x)\n`);
+
+// sample rowids
+const allIds = db.prepare(`SELECT rowid FROM ${SRC}`).all().map(r => r.rowid);
+function pick(n) {
+  const out = []; const used = new Set(); let seed = 1234567;
+  const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; };
+  while (out.length < n && used.size < allIds.length) {
+    const i = Math.floor(rnd() * allIds.length);
+    if (!used.has(i)) { used.add(i); out.push(allIds[i]); }
+  }
+  return out;
+}
+const sample = pick(SAMPLE);
+
+const getVec = db.prepare(`SELECT embedding FROM ${SRC} WHERE rowid=?`);
+const knnF = db.prepare(`SELECT rowid FROM ${SRC} WHERE embedding MATCH ? AND k=? ORDER BY distance`);
+const knn8 = db.prepare(`SELECT rowid FROM ${I8} WHERE embedding MATCH vec_quantize_int8(?,'unit') AND k=? ORDER BY distance`);
+const knnB = db.prepare(`SELECT rowid FROM ${BIT} WHERE embedding MATCH vec_quantize_binary(?) AND k=? ORDER BY distance`);
+
+function topk(stmt, qv, k, self) {
+  const ids = stmt.all(qv, k + 1).map(r => r.rowid).filter(id => id !== self);
+  return ids.slice(0, k);
+}
+function recallAt(gt, cand, k) {
+  const g = new Set(gt.slice(0, k));
+  let hit = 0; for (const id of cand.slice(0, k)) if (g.has(id)) hit++;
+  return hit / k;
+}
+
+let sum8_10 = 0, sum8_30 = 0, sumB_10 = 0, sumB_30 = 0;
+let tF = 0, t8t = 0, tBt = 0, worst8_10 = 1;
+for (const rid of sample) {
+  const qv = getVec.get(rid).embedding;
+  let s = Date.now(); const gt = topk(knnF, qv, KMAX, rid); tF += Date.now() - s;
+  s = Date.now(); const c8 = topk(knn8, qv, KMAX, rid); t8t += Date.now() - s;
+  s = Date.now(); const cB = topk(knnB, qv, KMAX, rid); tBt += Date.now() - s;
+  const r8_10 = recallAt(gt, c8, 10); const r8_30 = recallAt(gt, c8, 30);
+  sum8_10 += r8_10; sum8_30 += r8_30;
+  sumB_10 += recallAt(gt, cB, 10); sumB_30 += recallAt(gt, cB, 30);
+  if (r8_10 < worst8_10) worst8_10 = r8_10;
+}
+const N = sample.length;
+const pct = x => (100 * x / N).toFixed(1) + '%';
+console.log('## Recall (quantized vs float32 ground truth), averaged over', N, 'real query probes');
+console.log(`int8   recall@10: ${pct(sum8_10)}   recall@30: ${pct(sum8_30)}   worst@10: ${(100*worst8_10).toFixed(0)}%`);
+console.log(`binary recall@10: ${pct(sumB_10)}   recall@30: ${pct(sumB_30)}   (binary normally needs a float rerank pass)`);
+console.log();
+console.log('## KNN latency (avg ms/query, exhaustive over', srcN, 'vectors)');
+console.log(`float32: ${(tF/N).toFixed(1)}ms   int8: ${(t8t/N).toFixed(1)}ms   bit: ${(tBt/N).toFixed(1)}ms`);
+db.close();

package/template/wall-e/server.js

@@ -1,7 +1,10 @@
 'use strict';
+process.env.WALL_E_PROCESS_ROLE = process.env.WALL_E_PROCESS_ROLE || 'walle-daemon';
+
 const http = require('http');
 const { handleWalleApi } = require('./api-walle');
 const { getAllowedOrigin, isAuthorizedRequest } = require('./http/auth');
+const { startPortkeyModelSyncLoop } = require('./llm/portkey-sync');
 
 const PORT = parseInt(process.env.WALL_E_PORT) || 3457;
 const HOST = process.env.WALL_E_HOST || '127.0.0.1';
@@ -155,6 +158,13 @@ function startServer(options = {}) {
       readyResolve();
       if (typeof setImmediate === 'function') setImmediate(autoConfigureWallEMcp);
       else setTimeout(autoConfigureWallEMcp, 0);
+      if (process.env.WALLE_DISABLE_PORTKEY_SYNC !== '1') {
+        try {
+          server.stopPortkeyModelSync = startPortkeyModelSyncLoop();
+        } catch (err) {
+          console.warn(`[wall-e] Portkey model sync loop failed to start: ${err.message}`);
+        }
+      }
     });
   }
 
@@ -177,6 +187,11 @@ function startServer(options = {}) {
       readyReject(err);
     }
   });
+  server.on('close', () => {
+    try {
+      if (typeof server.stopPortkeyModelSync === 'function') server.stopPortkeyModelSync();
+    } catch {}
+  });
 
   tryListen();
   return server;

package/template/wall-e/session-files.js

@@ -116,6 +116,15 @@ class WallESessionRecorder {
     }));
   }
 
+  appendFrame(frame, extra = {}) {
+    if (!this.enabled) return null;
+    this.ensureStarted();
+    return this._appendRecord(this._baseRecord('frame_update', {
+      ...extra,
+      data: normalizeContent(frame),
+    }));
+  }
+
   appendError(error, extra = {}) {
     if (!this.enabled) return null;
     this.ensureStarted();

package/template/wall-e/skills/_bundled/google-calendar/run.js

@@ -141,7 +141,7 @@ function storeToBrain(events) {
     ).all(windowStart, windowEnd);
     for (const row of dbEvents) {
       if (!seenSourceIds.has(row.source_id)) {
-        brain.getDb().prepare('DELETE FROM memories WHERE id = ?').run(row.id);
+        brain.deleteMemory(row.id); // cascades to memory_index + embeddings (no orphan leak)
         cleaned++;
       }
     }

package/template/wall-e/skills/_bundled/gws-workspace/run.js

@@ -194,7 +194,7 @@ function syncCalendar(brain, token, profileId) {
     ).all(timeMin, timeMax);
     for (const row of dbEvents) {
       if (!seenSourceIds.has(row.source_id)) {
-        brain.getDb().prepare('DELETE FROM memories WHERE id = ?').run(row.id);
+        brain.deleteMemory(row.id); // cascades to memory_index + embeddings (no orphan leak)
         cleaned++;
       }
     }

package/template/wall-e/skills/_bundled/slack-mentions/run.js

@@ -25,6 +25,9 @@ const {
 const {
   parseDsmlToolCallBlocks,
 } = require(path.resolve(__dirname, '..', '..', '..', 'llm', 'text-tool-calls'));
+const ctmOperationalContext = require(path.resolve(__dirname, '..', '..', '..', 'memory', 'ctm-operational-context'));
+let telemetry;
+try { telemetry = require(path.resolve(__dirname, '..', '..', '..', 'telemetry')); } catch { telemetry = { track() {} }; }
 
 const WATERMARK_FILE = path.join(__dirname, '.watermark.json');
 const OWNER_HANDLE = process.env.SLACK_OWNER_HANDLE || '';
@@ -540,6 +543,62 @@ function buildSlackChatOptions(overrides = {}) {
   };
 }
 
+async function buildSlackCtmContextBlock(text, opts = {}) {
+  const classification = ctmOperationalContext.classifyCtmOperationalQuery(text);
+  if (!classification.required) {
+    return { required: false, ok: true, block: '', result: null };
+  }
+
+  let result;
+  try {
+    result = await ctmOperationalContext.lookupCtmOperationalContext({
+      query: text,
+      limit: opts.limit || 5,
+      force: true,
+    }, {
+      timeoutMs: opts.timeoutMs || 6000,
+      cooldownMs: opts.cooldownMs ?? 0,
+    });
+  } catch (err) {
+    result = {
+      required: true,
+      ok: false,
+      must_not_guess: true,
+      reason: err.message,
+      context_text: 'CTM operational context lookup failed. Do not guess; reply ONLY NO_REPLY.',
+      sources: {
+        remote_access: { searched: true, ok: false, reason: err.message },
+        ctm_sessions: { searched: true, ok: false, reason: err.message },
+      },
+    };
+  }
+
+  telemetry.track('slack_ctm_context_preflight', {
+    required: true,
+    ok: Boolean(result.ok),
+    remote_ok: Boolean(result.sources?.remote_access?.ok),
+    session_ok: Boolean(result.sources?.ctm_sessions?.ok),
+    session_count: Number(result.sources?.ctm_sessions?.count || 0),
+  });
+
+  if (!result.ok) {
+    console.warn(`[slack-mentions] Required CTM context unavailable; suppressing guess (${result.sources?.remote_access?.reason || result.sources?.ctm_sessions?.reason || result.reason || 'unknown'})`);
+    return {
+      required: true,
+      ok: false,
+      block: '\n\nRequired CTM operational/session-memory context was unavailable. Reply ONLY NO_REPLY. Do not guess from generic memories, DDEV hosts, localhost, or local IP heuristics.',
+      result,
+    };
+  }
+
+  return {
+    required: true,
+    ok: true,
+    block: `\n\n${result.context_text}`,
+    result,
+  };
+}
+
 function selectThreadMessagesToProcess(replies, thread, opts = {}) {
   const lastSeenTs = thread?.last_seen_ts || thread?.thread_ts || null;
   const ownerReplies = (replies || []).filter(reply => isOwnerReply(reply, opts));
@@ -577,7 +636,7 @@ async function deliverSlackReply({ channelId, threadTs, message, deliveryId }) {
       conversationId: channelId,
       threadId: threadTs,
       source: 'wall_e_outbound',
-      extractMessageId: extractSlackMessageTs,
+      extractMessageId: (result) => extractSlackMessageTs(result),
       metadata: { slackChannelId: channelId, slackThreadTs: threadTs },
     },
   });
@@ -605,20 +664,25 @@ async function generateAndSendReply(msg, threadTs, contextBlock, kind, opts = {}
   try {
     const today = new Date().toISOString().split('T')[0];
     const instruction = slackResponseInstruction(question);
+    const ctmContext = await buildSlackCtmContextBlock(question);
+    if (ctmContext.required && !ctmContext.ok) return null;
+    const effectiveContextBlock = `${contextBlock || ''}${ctmContext.block || ''}`;
     const locationAware = shouldUseLocationForMessage(question);
     const explicitCalendar = isExplicitCalendarRequest(question);
     const locationInstruction = locationAware
       ? 'For physical location/local weather questions, use Wall-E current-location context only. Do not infer location from calendar, timezone, Slack history, or memories.'
+      : ctmContext.required
+      ? 'Use the CTM operational context block as authoritative. Do not answer from generic memories if they conflict. Do not guess DDEV, localhost, or local-IP answers unless the CTM context explicitly says so.'
       : 'Search your memories for relevant info.';
     // Build a context-aware prompt: greetings/welcomes get a warmer, more proactive prompt
     let prompt;
     if (isGreeting) {
-      prompt = `[SLACK GREETING] ${actorLabel(msg)} said in Slack:\n\n"${question}"${contextBlock}\n\nToday is ${today}. This appears to be a greeting or welcome message. Search your memories (e.g. lookup_person) to learn about the people mentioned. Respond warmly — welcome them, share any relevant context you know about them, and proactively offer to help. Be genuine and concise (2-4 sentences). Do NOT mention any tool failures or technical issues in your response.`;
+      prompt = `[SLACK GREETING] ${actorLabel(msg)} said in Slack:\n\n"${question}"${effectiveContextBlock}\n\nToday is ${today}. This appears to be a greeting or welcome message. Search your memories (e.g. lookup_person) to learn about the people mentioned. Respond warmly — welcome them, share any relevant context you know about them, and proactively offer to help. Be genuine and concise (2-4 sentences). Do NOT mention any tool failures or technical issues in your response.`;
     } else {
-      prompt = `[SLACK ${kind.toUpperCase()}] ${actorLabel(msg)} asked in Slack:\n\n${question}${contextBlock}\n\nToday is ${today}. ${locationInstruction} For schedule/calendar questions, search with the specific date (e.g. "calendar 2026-04-13") since calendar events are stored as memories with source=calendar. Use mcp_call only if memories don't have what you need. ${instruction}`;
+      prompt = `[SLACK ${kind.toUpperCase()}] ${actorLabel(msg)} asked in Slack:\n\n${question}${effectiveContextBlock}\n\nToday is ${today}. ${locationInstruction} For schedule/calendar questions, search with the specific date (e.g. "calendar 2026-04-13") since calendar events are stored as memories with source=calendar. Use ctm_context for CTM operational/session-memory questions. Use mcp_call only if memories don't have what you need. ${instruction}`;
     }
 
-    const allowedTools = ['search_memories', 'think', 'lookup_person', 'mcp_call', 'calendar_events', 'mail_messages', 'mail_read', 'mail_search']
+    const allowedTools = ['search_memories', 'ctm_context', 'ctm_remote_access_status', 'ctm_session_search', 'think', 'lookup_person', 'mcp_call', 'calendar_events', 'mail_messages', 'mail_read', 'mail_search']
       .filter(tool => !(locationAware && !explicitCalendar && tool === 'calendar_events'));
     const result = await chatModule.chat(prompt, buildSlackChatOptions({
       channel: 'task',
@@ -675,14 +739,19 @@ async function generateAndSendFollowUp(combinedMessage, thread, threadContext, o
     const today = new Date().toISOString().split('T')[0];
     const contextBlock = threadContext ? `\n\nFull Slack thread context:\n${threadContext}` : '';
     const instruction = slackResponseInstruction(combinedMessage);
+    const ctmContext = await buildSlackCtmContextBlock(`${combinedMessage}\n${threadContext || ''}`);
+    if (ctmContext.required && !ctmContext.ok) return null;
+    const effectiveContextBlock = `${contextBlock}${ctmContext.block || ''}`;
     const locationAware = shouldUseLocationForMessage(combinedMessage);
     const explicitCalendar = isExplicitCalendarRequest(combinedMessage);
     const locationInstruction = locationAware
       ? 'For physical location/local weather questions, use Wall-E current-location context only. Do not infer location from calendar, timezone, Slack history, or memories.'
+      : ctmContext.required
+      ? 'Use the CTM operational context block as authoritative. Do not answer from generic memories if they conflict. Do not guess DDEV, localhost, or local-IP answers unless the CTM context explicitly says so.'
       : 'Search your memories for relevant info.';
-    const prompt = `[SLACK FOLLOW-UP] ${getConfiguredOwnerName() || 'The owner'} replied in a Slack thread:\n\n${combinedMessage}${contextBlock}\n\nToday is ${today}. ${locationInstruction} ${instruction}`;
+    const prompt = `[SLACK FOLLOW-UP] ${getConfiguredOwnerName() || 'The owner'} replied in a Slack thread:\n\n${combinedMessage}${effectiveContextBlock}\n\nToday is ${today}. ${locationInstruction} Use ctm_context for CTM operational/session-memory questions. ${instruction}`;
 
-    const allowedTools = ['search_memories', 'think', 'lookup_person', 'mcp_call', 'calendar_events', 'mail_messages', 'mail_read', 'mail_search']
+    const allowedTools = ['search_memories', 'ctm_context', 'ctm_remote_access_status', 'ctm_session_search', 'think', 'lookup_person', 'mcp_call', 'calendar_events', 'mail_messages', 'mail_read', 'mail_search']
       .filter(tool => !(locationAware && !explicitCalendar && tool === 'calendar_events'));
     const result = await chatModule.chat(prompt, buildSlackChatOptions({
       channel: 'task',
@@ -1455,6 +1524,7 @@ module.exports = {
   looksLikeToolControlMarkup,
   validateSlackAnswer,
   buildSlackChatOptions,
+  buildSlackCtmContextBlock,
   generateAndSendReply,
   parseChannelMessages,
   parseDmMessages,

package/template/wall-e/skills/claude-code-reader.js

@@ -368,10 +368,14 @@ function syncSkillReferences() {
       fs.existsSync(skill.path) ? fs.readFileSync(skill.path, 'utf8') : content
     ).digest('hex') : null;
 
-    // Check if this reference already exists
+    // Check if this reference already exists.
+    // NOTE: filter on `source` too — `WHERE source_id = ?` alone can't use the compound
+    // index idx_memories_source(source, source_id) and full-scans the (multi-GB) memories
+    // table on EVERY skill, which made boot take ~3min. Skill refs are always stored with
+    // source='skill_reference', so this is both correct and index-eligible (SEARCH not SCAN).
     try {
       const existing = brain.getDb().prepare(
-        'SELECT id, content FROM memories WHERE source_id = ? LIMIT 1'
+        "SELECT id, content FROM memories WHERE source = 'skill_reference' AND source_id = ? LIMIT 1"
       ).get(sourceId);
 
       if (existing && existing.content === content) continue; // no change
@@ -411,7 +415,7 @@ function syncSkillReferences() {
     ).all();
     for (const row of existing) {
       if (!allIds.has(row.source_id)) {
-        brain.getDb().prepare('DELETE FROM memories WHERE id = ?').run(row.id);
+        brain.deleteMemory(row.id); // cascades to memory_index + embeddings (no orphan leak)
       }
     }
   } catch {}

package/template/wall-e/skills/script-skill-runner.js

@@ -30,6 +30,16 @@ function runScriptSkill(skill, task = {}, opts = {}) {
       WALL_E_SKILL_CONFIG: JSON.stringify(skillConfig),
       WALL_E_SKILL_DIR: skill.dir,
       WALL_E_SRC_DIR: WALL_E_DIR,
+      // Script skills run as dedicated short-lived subprocesses that open their own
+      // brain.db connection (brain.initDb()). The daemon's brain lock defaults to a 0ms
+      // fail-fast (correct for the daemon event loop, which must never Atomics.wait), but
+      // inherited by a subprocess that means it ABORTS the instant the daemon holds the
+      // lock for a single statement — observed as ~272 "SQLite write lock busy" aborts
+      // (slack-mentions Exit code 1, dropped calendar/email syncs). A subprocess can
+      // safely block-wait briefly and retry, so give it a bounded timeout. In practice
+      // the lock frees in <100ms; 5s is just the ceiling. Caller env still wins.
+      WALL_E_SQLITE_WRITE_LOCK_TIMEOUT_MS:
+        process.env.WALL_E_SQLITE_WRITE_LOCK_TIMEOUT_MS || '5000',
       ...(opts.env || {}),
     };
     if (task.checkpoint) env.WALL_E_CHECKPOINT = task.checkpoint;

package/template/wall-e/skills/skill-planner.js

@@ -154,6 +154,43 @@ function clearResolvedSlackHealthAlerts({ authenticated = false, ownerConfigured
   return { cleared: before.length - after.length };
 }
 
+function _googleWorkspaceServiceKey(email, profileId) {
+  const normalizedEmail = String(email || '').trim().toLowerCase();
+  if (!normalizedEmail) return '';
+  const normalizedProfile = String(profileId || 'default').trim().toLowerCase() || 'default';
+  return `google_workspace:${normalizedProfile}:${normalizedEmail}`;
+}
+
+function _sameGoogleWorkspaceAlertAccount(alert, email, profileId) {
+  const normalizedEmail = String(email || '').trim().toLowerCase();
+  if (!normalizedEmail || !alert) return false;
+  const normalizedProfile = String(profileId || 'default').trim().toLowerCase() || 'default';
+  const service = String(alert.service || '').trim().toLowerCase();
+  if (service === _googleWorkspaceServiceKey(normalizedEmail, normalizedProfile)) return true;
+  const alertEmail = String(alert.account_email || alert.email || '').trim().toLowerCase();
+  if (alertEmail !== normalizedEmail) return false;
+  const alertProfile = String(alert.profile_id || alert.profileId || 'default').trim().toLowerCase() || 'default';
+  return alertProfile === normalizedProfile;
+}
+
+function clearResolvedGoogleWorkspaceHealthAlerts({ email = '', profileId = null, authenticated = false } = {}) {
+  if (!authenticated || !email) return { cleared: 0 };
+  const before = getServiceAlerts();
+  const after = before.filter((a) => {
+    if (!_sameGoogleWorkspaceAlertAccount(a, email, profileId)) return true;
+    const action = String(a.action || '');
+    const type = String(a.type || '');
+    const googleWorkspaceAuthAlert =
+      action === 'gws_reauth'
+      || type === 'auth_expired';
+    return !googleWorkspaceAuthAlert;
+  });
+  if (after.length !== before.length) {
+    brain.setKv('service_alerts', JSON.stringify(after));
+  }
+  return { cleared: before.length - after.length };
+}
+
 function _hasMissingSlackOwner(validation) {
   return !!(validation && Array.isArray(validation.missing)
     && validation.missing.some(m => m.type === 'env' && m.name === SLACK_OWNER_ENV_KEY));
@@ -873,6 +910,7 @@ module.exports = {
   dismissServiceAlert,
   clearServiceAlerts,
   clearResolvedSlackHealthAlerts,
+  clearResolvedGoogleWorkspaceHealthAlerts,
   _test: {
     isAuthError,
     getProviderAuthBlock,

package/template/wall-e/tools/builtin-middleware.js

@@ -99,8 +99,17 @@ function registerBuiltinMiddleware(mw, opts = {}) {
     const mode = ctx.mode || opts.mode || 'build';
     const sections = [];
 
-    sections.push(`You are an expert software engineer executing a coding task.
+    // When the orchestrator already built the full agent prompt
+    // (buildAgentSystemPrompt), keep it as the base and only append the
+    // sections it doesn't cover. Replacing it here silently dropped the
+    // workflow/quality/frontend/memory discipline from every loop turn.
+    const baseSystem = typeof ctx.system === 'string' && ctx.system.trim() ? ctx.system : '';
+    if (baseSystem) {
+      sections.push(baseSystem);
+    } else {
+      sections.push(`You are an expert software engineer executing a coding task.
 Work in: ${cwd}`);
+    }
 
     // Mode enforcement (belt + suspenders with tool filtering)
     if (mode === 'plan') {
@@ -111,9 +120,10 @@ You may ONLY read, search, and analyze. This overrides all other instructions.`)
 Do not make any edits. Review the diff and assess quality.`);
     }
 
-    // Project conventions
-    if (opts.claudeMd || ctx.claudeMd) {
-      sections.push(`## Project Conventions (CLAUDE.md)\n${opts.claudeMd || ctx.claudeMd}`);
+    // Project conventions (skip if the base prompt already carries them)
+    const claudeMd = opts.claudeMd || ctx.claudeMd;
+    if (claudeMd && !baseSystem.includes(claudeMd)) {
+      sections.push(`## Project Conventions (CLAUDE.md)\n${claudeMd}`);
     }
 
     // Quality rules (ported from OpenCode's prompt patterns)
@@ -143,7 +153,7 @@ Do not make any edits. Review the diff and assess quality.`);
     if (workflowDocs) sections.push(workflowDocs);
 
     const capabilityContext = buildCapabilityContext(ctx.promptCapabilities || opts.promptCapabilities);
-    if (capabilityContext) sections.push(capabilityContext);
+    if (capabilityContext && !baseSystem.includes(capabilityContext)) sections.push(capabilityContext);
 
     ctx.system = sections.join('\n\n');
   });
@@ -185,11 +195,11 @@ Do not make any edits. Review the diff and assess quality.`);
 
     // Prioritize tools based on detected coding phase
     const PHASE_PREFERRED = {
-      'coding:exploration': new Set(['read_file', 'glob', 'grep_files', 'list_directory', 'lsp_symbols']),
-      'coding:generation': new Set(['edit_file', 'write_file', 'apply_patch', 'multi_edit', 'run_shell']),
+      'coding:exploration': new Set(['read_file', 'glob', 'grep_files', 'list_directory', 'pdf_info', 'pdf_read_pages', 'pdf_render_pages', 'lsp_symbols']),
+      'coding:generation': new Set(['edit_file', 'write_file', 'apply_patch', 'multi_edit', 'run_shell', 'make_pdf']),
       'coding:debugging': new Set(['read_file', 'run_shell', 'grep_files', 'lsp_diagnostics', 'edit_file']),
-      'coding:testing': new Set(['run_shell', 'read_file', 'edit_file']),
-      'coding:review': new Set(['read_file', 'grep_files', 'lsp_diagnostics', 'lsp_symbols']),
+      'coding:testing': new Set(['run_shell', 'read_file', 'edit_file', 'pdf_info', 'pdf_render_pages']),
+      'coding:review': new Set(['read_file', 'grep_files', 'pdf_info', 'pdf_read_pages', 'pdf_render_pages', 'lsp_diagnostics', 'lsp_symbols']),
     };
 
     const preferred = PHASE_PREFERRED[phase];