create-walle 0.9.21 → 0.9.23

package/template/wall-e/api-walle.js

@@ -12,6 +12,7 @@ const {
   parseLimit,
   parseOffset,
   readBody,
+  BODY_LIMIT_LARGE,
 } = require('./http/api-utils');
 const authFlows = require('./auth/flow-manager');
 const {
@@ -20,7 +21,6 @@ const {
 } = require('./auth/provider-flows');
 
 const DATA_DIR = process.env.WALL_E_DATA_DIR || path.join(process.env.HOME, '.walle', 'data');
-const BRAIN_DB_PATH = path.join(DATA_DIR, 'wall-e-brain.db');
 const TELEMETRY_DB_PATH = path.join(DATA_DIR, 'telemetry.db');
 const FEEDBACK_RATE_LIMIT_PER_MINUTE = 2;
 const FEEDBACK_RATE_LIMIT_WINDOW_MS = 60 * 1000;
@@ -49,6 +49,8 @@ const TELEMETRY_DIAGNOSTIC_EVENTS = new Set([
   'upgrade',
   'upgrade_prompt',
   'funnel',
+  'session_integrity_issue',
+  'session_integrity_issue_summary',
 ]);
 const TELEMETRY_DIAGNOSTIC_PREFIXES = ['ctm_update_'];
 const TELEMETRY_REDACTED_TEXT = '[redacted after retention]';
@@ -556,6 +558,11 @@ function telemetryCleanupCandidateStats(tdb, policy = telemetryCleanupPolicy(),
   };
 }
 
+function telemetryCleanupCheckpointMode() {
+  const raw = String(process.env.WALLE_TELEMETRY_CLEANUP_CHECKPOINT_MODE || 'PASSIVE').toUpperCase();
+  return new Set(['PASSIVE', 'FULL', 'RESTART', 'TRUNCATE']).has(raw) ? raw : 'PASSIVE';
+}
+
 function telemetryAggregateArchivedEvents(tdb, since) {
   return telemetrySafeCount(
     tdb,
@@ -584,6 +591,103 @@ function telemetryEventsByTypeSince(tdb, since) {
   `).all(since, String(since || '').slice(0, 10));
 }
 
+function telemetrySetIntersectionSize(left, right) {
+  let count = 0;
+  for (const value of left || []) {
+    if (right && right.has(value)) count += 1;
+  }
+  return count;
+}
+
+function telemetrySetDifferenceSize(left, right) {
+  let count = 0;
+  for (const value of left || []) {
+    if (!right || !right.has(value)) count += 1;
+  }
+  return count;
+}
+
+function telemetryStableIdentityFromRow(row) {
+  return row && row.machine_bucket ? `m:${row.machine_bucket}` : `i:${row?.install_id || ''}`;
+}
+
+function telemetryMapCountsToRows(map, countKey = 'install_count') {
+  return Array.from(map.entries())
+    .map(([key, value]) => ({ key, [countKey]: value.size || value }))
+    .sort((a, b) => b[countKey] - a[countKey] || String(a.key).localeCompare(String(b.key)));
+}
+
+function telemetryEmbeddingFeatureAdoption(tdb, since, activeInstalls) {
+  const rows = tdb.prepare(`
+    SELECT te.event, te.install_id, ti.machine_bucket, te.meta, te.received_at
+    FROM telemetry_events te
+    LEFT JOIN telemetry_installs ti ON ti.install_id = te.install_id
+    WHERE te.event IN ('embedding_status', 'embedding_provider_switch', 'embedding_usage')
+      AND te.received_at >= ?
+    ORDER BY te.received_at ASC, te.id ASC
+  `).all(since);
+  const latestStatusByIdentity = new Map();
+  const setupView = new Set();
+  const switchSuccess = new Set();
+  const switchFailed = new Set();
+  const usage = new Set();
+  const usageByOperation = new Map();
+
+  for (const row of rows) {
+    const identity = telemetryStableIdentityFromRow(row);
+    if (!identity || identity === 'i:') continue;
+    let meta = {};
+    try { meta = row.meta ? JSON.parse(row.meta) : {}; } catch {}
+
+    if (row.event === 'embedding_status') {
+      latestStatusByIdentity.set(identity, meta);
+      if (meta.reason === 'setup_view') setupView.add(identity);
+      continue;
+    }
+    if (row.event === 'embedding_provider_switch') {
+      if (meta.ok) switchSuccess.add(identity);
+      else switchFailed.add(identity);
+      continue;
+    }
+    if (row.event === 'embedding_usage') {
+      usage.add(identity);
+      const operation = String(meta.operation || 'unknown').slice(0, 80) || 'unknown';
+      if (!usageByOperation.has(operation)) usageByOperation.set(operation, new Set());
+      usageByOperation.get(operation).add(identity);
+    }
+  }
+
+  const enabled = new Set();
+  const enabledByProvider = new Map();
+  const latestStateCounts = new Map();
+  for (const [identity, meta] of latestStatusByIdentity.entries()) {
+    const state = String(meta.state || 'unknown').slice(0, 80) || 'unknown';
+    latestStateCounts.set(state, (latestStateCounts.get(state) || 0) + 1);
+    if (meta.enabled || meta.state === 'ready') {
+      enabled.add(identity);
+      const provider = String(meta.provider || 'unknown').slice(0, 80) || 'unknown';
+      if (!enabledByProvider.has(provider)) enabledByProvider.set(provider, new Set());
+      enabledByProvider.get(provider).add(identity);
+    }
+  }
+
+  const denominator = activeInstalls || 0;
+  const pct = (num) => denominator > 0 ? Math.round(num / denominator * 100) / 100 : null;
+  return {
+    reporting_installs: latestStatusByIdentity.size,
+    enabled_installs: enabled.size,
+    usage_installs: usage.size,
+    setup_view_installs: setupView.size,
+    switch_success_installs: switchSuccess.size,
+    switch_failed_installs: switchFailed.size,
+    enabled_pct_active: pct(enabled.size),
+    usage_pct_active: pct(usage.size),
+    latest_state: telemetryMapCountsToRows(latestStateCounts, 'install_count'),
+    enabled_by_provider: telemetryMapCountsToRows(enabledByProvider),
+    usage_by_operation: telemetryMapCountsToRows(usageByOperation),
+  };
+}
+
 function archiveAndDeleteTelemetryEvents(tdb, bucket, batchSize) {
   const selectRows = tdb.prepare(`
     SELECT id, date(received_at) as day, event, COALESCE(version, '') as version,
@@ -803,8 +907,9 @@ function runTelemetryCleanup(options = {}) {
 
     let checkpoint = null;
     try {
-      const rows = tdb.pragma('wal_checkpoint(TRUNCATE)');
-      checkpoint = { ok: true, rows };
+      const mode = telemetryCleanupCheckpointMode();
+      const rows = tdb.pragma(`wal_checkpoint(${mode})`);
+      checkpoint = { ok: true, mode, rows };
     } catch (e) {
       checkpoint = { ok: false, error: e.message };
     }
@@ -1064,18 +1169,16 @@ function buildSkillMd(body) {
   return lines.join('\n');
 }
 
-let readDb = null;
+let readDbOverride = null;
 
 function getReadDb() {
-  if (!readDb) {
-    try {
-      readDb = new Database(BRAIN_DB_PATH, { readonly: true, fileMustExist: true });
-      readDb.pragma('busy_timeout = 5000');
-    } catch (err) {
-      return null;
-    }
+  if (readDbOverride) return readDbOverride;
+  if (!brain || !ensureBrainInit()) return null;
+  try {
+    return brain.getDb();
+  } catch {
+    return null;
   }
-  return readDb;
 }
 
 let brain = null;
@@ -1380,7 +1483,14 @@ function getQuestionsList(params) {
   if (!db) return [];
 
   const status = params.get('status') || 'pending';
-  return db.prepare('SELECT * FROM pending_questions WHERE status = ? ORDER BY created_at DESC').all(status);
+  // Paginate: this endpoint had no LIMIT, so it returned every pending question (15k+ on
+  // a busy brain) as one unbounded blob. Default to a page of 100; callers can override
+  // with ?limit= and page with ?offset=.
+  const limit = parseLimit(params, 100);
+  const offset = Math.max(0, parseInt(params.get('offset') || '0', 10) || 0);
+  return db.prepare(
+    'SELECT * FROM pending_questions WHERE status = ? ORDER BY created_at DESC LIMIT ? OFFSET ?'
+  ).all(status, limit, offset);
 }
 
 function getStats() {
@@ -1421,22 +1531,73 @@ function handleWalleApi(req, res, url) {
   const schedulerHandled = handleSchedulerApi(req, res, url);
   if (schedulerHandled !== false) return schedulerHandled;
 
-  // GET /api/wall-e/health — simple liveness check
+  // GET /api/wall-e/health — liveness check. Includes event-loop delay stats: if this
+  // endpoint responds at all the loop is not currently wedged, and `event_loop.peak_ms` /
+  // `last_warning` show whether it has stalled recently (a synchronous DB query starving
+  // the loop). A blocked loop can't answer this, so a tight-timeout probe of it is a
+  // truer readiness signal than "the process is running".
   if (p === '/api/wall-e/health' && m === 'GET') {
-    return jsonResponse(res, { status: 'ok', uptime: process.uptime(), version: '0.1.0' }), true;
+    let eventLoop = { enabled: false };
+    try { eventLoop = require('./lib/event-loop-monitor').stats(); } catch {}
+    return jsonResponse(res, { status: 'ok', uptime: process.uptime(), version: '0.1.0', event_loop: eventLoop }), true;
   }
 
   // GET /api/wall-e/alerts — service alerts (auth failures, skill disabled, update available)
   if (p === '/api/wall-e/alerts' && m === 'GET') {
     try {
+      res.setHeader('Cache-Control', 'no-store, max-age=0');
       const { getServiceAlerts } = require('./skills/skill-planner');
+      const { buildServiceHealth } = require('./lib/service-health');
       const alerts = getServiceAlerts({ includeVersionUpdate: true });
-      return jsonResponse(res, { alerts }), true;
+      let activeProvider = process.env.WALLE_PROVIDER || '';
+      let activeModel = process.env.WALLE_MODEL || '';
+      try {
+        activeProvider = brain?.getKv?.('walle_provider') || activeProvider;
+        activeModel = brain?.getKv?.('walle_model') || activeModel;
+      } catch {}
+      const summary = buildServiceHealth(alerts, {
+        activeProvider,
+        activeModel,
+      });
+      return jsonResponse(res, { alerts, summary }), true;
     } catch (e) {
       return jsonResponse(res, { alerts: [], error: e.message }), true;
     }
   }
 
+  // GET /api/wall-e/coding/boundaries?session_id=… — persisted snapshot
+  // boundaries (newest first) for the rewind UI.
+  if (p === '/api/wall-e/coding/boundaries' && m === 'GET') {
+    try {
+      const sessionId = String(url.searchParams.get('session_id') || '').slice(0, 200);
+      if (!sessionId) return jsonResponse(res, { ok: false, error: 'session_id required' }, 400), true;
+      const { listSessionBoundaries } = require('./coding/snapshot-service');
+      return jsonResponse(res, { ok: true, boundaries: listSessionBoundaries(sessionId, { brain }) }), true;
+    } catch (e) {
+      return jsonResponse(res, { ok: false, error: e.message }, 500), true;
+    }
+  }
+
+  // POST /api/wall-e/coding/rewind — restore the worktree to a snapshot
+  // boundary. Restores tracked snapshot content; never deletes files the
+  // snapshot system has not seen.
+  if (p === '/api/wall-e/coding/rewind' && m === 'POST') {
+    return readBody(req, { maxBytes: 16 * 1024 }).then(body => {
+      try {
+        const sessionId = String(body.session_id || body.sessionId || '').slice(0, 200);
+        const boundaryId = String(body.boundary_id || body.boundaryId || '').slice(0, 200);
+        if (!sessionId || !boundaryId) {
+          return jsonResponse(res, { ok: false, error: 'session_id and boundary_id required' }, 400);
+        }
+        const { revertSessionBoundary } = require('./coding/snapshot-service');
+        const result = revertSessionBoundary({ sessionId, boundaryId, brain });
+        return jsonResponse(res, { ok: true, ...result });
+      } catch (e) {
+        return jsonResponse(res, { ok: false, error: e.message }, 400);
+      }
+    }), true;
+  }
+
   // POST /api/wall-e/feedback/ingest — explicit CTM feedback handoff.
   // This route stores curated feedback as memory/knowledge. It does not ingest
   // screenshots or full transcripts unless CTM included consented previews.
@@ -1624,6 +1785,13 @@ function handleWalleApi(req, res, url) {
     }
   }
 
+  // GET /api/wall-e/boot-profile — per-phase startup timing (instrumentation)
+  if (p === '/api/wall-e/boot-profile' && m === 'GET') {
+    let profile = null;
+    try { profile = require('./lib/boot-profile').getProfile(); } catch (_) {}
+    return jsonResponse(res, { profile }), true;
+  }
+
   // GET /api/wall-e/scheduler — scheduler observability
   if (p === '/api/wall-e/scheduler' && m === 'GET') {
     const { Scheduler } = require('./lib/scheduler');
@@ -1782,7 +1950,10 @@ function handleWalleApi(req, res, url) {
   if (p === '/api/wall-e/gws/accounts' && m === 'GET') {
     (async () => {
       try {
+        res.setHeader('Cache-Control', 'no-store, max-age=0');
         const gwsSetup = require('./skills/_bundled/gws-workspace/setup');
+        let skillPlanner = null;
+        try { skillPlanner = require('./skills/skill-planner'); } catch {}
         const accounts = gwsSetup.listAccounts();
         const gwsProfiles = typeof gwsSetup.getGwsProfiles === 'function' ? gwsSetup.getGwsProfiles() : [];
         const hasClientSecret = gwsProfiles.length
@@ -1849,6 +2020,15 @@ function handleWalleApi(req, res, url) {
               detail.reauthRequired = false;
               detail.authIssue = null;
               detail.syncMode = 'auto';
+              if (skillPlanner && typeof skillPlanner.clearResolvedGoogleWorkspaceHealthAlerts === 'function') {
+                try {
+                  skillPlanner.clearResolvedGoogleWorkspaceHealthAlerts({
+                    email: acct.email,
+                    profileId: acct.profileId || null,
+                    authenticated: true,
+                  });
+                } catch {}
+              }
               const apiResults = await gwsSetup.testApis(creds.access_token);
               for (const r of apiResults) {
                 detail.apis[r.name.toLowerCase()] = r.enabled;
@@ -2323,9 +2503,15 @@ function handleWalleApi(req, res, url) {
     if (!hasApiKey && (walleProvider === 'ollama' || walleProvider === 'mlx')) hasApiKey = true;
 
     let serviceAlerts = [];
+    let serviceAlertSummary = null;
     try {
       const { getServiceAlerts } = require('./skills/skill-planner');
+      const { buildServiceHealth } = require('./lib/service-health');
       serviceAlerts = getServiceAlerts({ includeVersionUpdate: true });
+      serviceAlertSummary = buildServiceHealth(serviceAlerts, {
+        activeProvider: walleProvider,
+        activeModel: walleModel,
+      });
     } catch {}
 
     let slackSocketMode = null;
@@ -2350,6 +2536,7 @@ function handleWalleApi(req, res, url) {
         walle_model: walleModel,
         has_api_key: hasApiKey,
         service_alerts: serviceAlerts,
+        service_alert_summary: serviceAlertSummary,
         slack_socket_mode: slackSocketMode,
       }
     }), true;
@@ -2360,6 +2547,7 @@ function handleWalleApi(req, res, url) {
     try {
       const embeddingsModule = require('./embeddings');
       if (!brain || !ensureBrainInit()) return jsonResponse(res, { providers: [], stats: {}, totalMemories: 0, error: 'Brain module not available' }, 503), true;
+      const auto = embeddingsModule.autoSelectProvider?.('setup_view') || null;
       const providers = embeddingsModule.getProviderStatus();
       const stats = embeddingsModule.getAllEmbeddingStats();
       const totalMemories = brain.getDb().prepare("SELECT count(*) as c FROM memories WHERE content IS NOT NULL AND length(content) > 10").get().c;
@@ -2394,13 +2582,28 @@ function handleWalleApi(req, res, url) {
           etaMinutes: recentRate > 0 ? Math.ceil(remaining / recentRate) : null,
         };
       }
-      jsonResponse(res, { providers, stats, totalMemories, backfillProgress });
+      const setup = embeddingsModule.getEmbeddingSetupState?.() || {};
+      jsonResponse(res, { providers, stats, totalMemories, backfillProgress, setup, auto });
+      try { embeddingsModule.trackEmbeddingSetupView?.(); } catch {}
     } catch (e) {
       jsonResponse(res, { providers: [], stats: {}, totalMemories: 0, error: e.message });
     }
     return true;
   }
 
+  if (p === '/api/wall-e/setup/embeddings/policy' && m === 'POST') {
+    readBody(req).then((body) => {
+      try {
+        const embeddingsModule = require('./embeddings');
+        const result = embeddingsModule.setEmbeddingPolicy(body.policy);
+        jsonResponse(res, result, result.ok ? 200 : 400);
+      } catch (e) {
+        jsonResponse(res, { ok: false, message: e.message }, 400);
+      }
+    }).catch((e) => jsonResponse(res, { ok: false, message: e.message }, 400));
+    return true;
+  }
+
   if (p === '/api/wall-e/setup/embeddings/switch' && m === 'POST') {
     readBody(req).then((body) => {
       try {
@@ -2496,13 +2699,17 @@ function handleWalleApi(req, res, url) {
         const type = sanitizeSetupProviderType(body.type);
         if (!type) return jsonResponse(res, { ok: false, error: 'Invalid provider type' }, 400);
         const model = sanitizeSetupModel(body.model);
+        const hasBaseUrl = Object.prototype.hasOwnProperty.call(body, 'base_url')
+          || Object.prototype.hasOwnProperty.call(body, 'baseUrl');
+        const hasCustomHeaders = Object.prototype.hasOwnProperty.call(body, 'custom_headers')
+          || Object.prototype.hasOwnProperty.call(body, 'customHeaders');
         const result = brain.saveSetupProvider({
           id: body.id || `${type}-default`,
           name: body.name || SETUP_PROVIDER_NAMES[type] || type,
           type,
-          baseUrl: body.base_url || body.baseUrl || null,
+          baseUrl: hasBaseUrl ? (body.base_url || body.baseUrl || null) : undefined,
           apiKeyEncrypted: typeof body.api_key === 'string' ? body.api_key.replace(/[\r\n\s]/g, '').slice(0, 200) : (body.apiKeyEncrypted || null),
-          customHeaders: body.custom_headers || body.customHeaders || null,
+          customHeaders: hasCustomHeaders ? (body.custom_headers || body.customHeaders || null) : undefined,
           enabled: body.enabled !== false,
           model,
           setDefault: Boolean(body.set_default),
@@ -2880,19 +3087,73 @@ function handleWalleApi(req, res, url) {
     try {
       const backups = brain.listBackups();
       const dbPath = brain.getDbPath();
-      const dbSize = require('fs').statSync(dbPath).size;
-      jsonResponse(res, { data: { backups, db_path: dbPath, backup_dir: brain.BACKUP_DIR, db_size: dbSize } });
+      let dbSize = 0;
+      try { dbSize = require('fs').statSync(dbPath).size; } catch {}
+      const info = typeof brain.getBackupDirInfo === 'function'
+        ? brain.getBackupDirInfo()
+        : { backup_dir: brain.BACKUP_DIR };
+      jsonResponse(res, {
+        data: {
+          backups,
+          db_path: dbPath,
+          data_dir: path.dirname(dbPath),
+          backup_dir: info.backup_dir || brain.BACKUP_DIR,
+          default_backup_dir: info.default_backup_dir || '',
+          configured_backup_dir: info.configured_backup_dir || '',
+          backup_dir_source: info.source || 'default',
+          db_size: dbSize,
+        },
+      });
+    } catch (e) { jsonResponse(res, { error: e.message }, 500); }
+    return true;
+  }
+
+  // GET /api/wall-e/storage — current Wall-E data and backup locations
+  if (p === '/api/wall-e/storage' && m === 'GET') {
+    if (!brain) return jsonResponse(res, { error: 'Brain not available' }, 503), true;
+    try {
+      if (ensureBrainInit) ensureBrainInit();
+      const dbPath = brain.getDbPath();
+      const info = typeof brain.getBackupDirInfo === 'function'
+        ? brain.getBackupDirInfo()
+        : { backup_dir: brain.BACKUP_DIR };
+      jsonResponse(res, {
+        data: {
+          db_path: dbPath,
+          data_dir: path.dirname(dbPath),
+          backup_dir: info.backup_dir || brain.BACKUP_DIR,
+          default_backup_dir: info.default_backup_dir || '',
+          configured_backup_dir: info.configured_backup_dir || '',
+          backup_dir_source: info.source || 'default',
+        },
+      });
     } catch (e) { jsonResponse(res, { error: e.message }, 500); }
     return true;
   }
 
+  // PUT /api/wall-e/storage/backup-dir — update Wall-E-owned backup directory
+  if (p === '/api/wall-e/storage/backup-dir' && m === 'PUT') {
+    if (!brain || !ensureBrainInit()) return jsonResponse(res, { error: 'Brain not available' }, 503), true;
+    readBody(req, 32 * 1024)
+      .then((body) => {
+        const result = brain.setBackupDir(body.backup_dir || '', {
+          persist: true,
+          moveExisting: !!body.move_existing,
+        });
+        jsonResponse(res, { data: result });
+      })
+      .catch((e) => jsonResponse(res, { error: e.message }, 400));
+    return true;
+  }
+
   // POST /api/wall-e/backups — create brain backup
   if (p === '/api/wall-e/backups' && m === 'POST') {
     if (!brain || !ensureBrainInit()) return jsonResponse(res, { error: 'Brain not available' }, 503), true;
-    try {
-      const result = brain.createBackup('manual');
-      jsonResponse(res, { data: result });
-    } catch (e) { jsonResponse(res, { error: e.message }, 500); }
+    // createBackup is async (non-blocking incremental backup) — resolve before responding.
+    Promise.resolve()
+      .then(() => brain.createBackup('manual'))
+      .then((result) => jsonResponse(res, { data: result }))
+      .catch((e) => jsonResponse(res, { error: e.message }, 500));
     return true;
   }
 
@@ -3349,6 +3610,67 @@ function handleWalleApi(req, res, url) {
     return true;
   }
 
+  // GET /api/wall-e/diagnostics/write-lock — brain.db write-lock probes (mirrors CTM's
+  // /api/diagnostics/write-lock). Reports THIS process's (the daemon's) acquire/busy/hold/
+  // wait stats by label. Cross-process contention from skill subprocesses shows up in the
+  // `[sqlite-write-lock] busy fail-fast … pid=… holder=… caller=…` log lines. ?reset=1
+  // snapshots and clears (for windowed deltas).
+  if (p === '/api/wall-e/diagnostics/write-lock' && m === 'GET') {
+    try {
+      const b = brain || require('./brain');
+      const reset = params.get('reset') === '1' || params.get('reset') === 'true';
+      const stats = reset ? b.resetWriteLockStats() : b.getWriteLockStats();
+      jsonResponse(res, { ok: true, generatedAt: Date.now(), reset, stats });
+    } catch (e) {
+      jsonResponse(res, { ok: false, error: e.message }, 500);
+    }
+    return true;
+  }
+
+  // GET/POST /api/wall-e/questions/digest/settings — opt-in digest push channels. The CTM
+  // Wall-E tab is always on (no toggle); Slack DM + email are off by default and enabled
+  // here. Stored as kv flags read by the daily digest job.
+  if (p === '/api/wall-e/questions/digest/settings' && m === 'GET') {
+    try {
+      const b = brain || require('./brain');
+      const { KV } = require('./loops/question-digest');
+      jsonResponse(res, { ok: true, settings: {
+        slack_enabled: String(b.getKv(KV.slackEnabled) || '').toLowerCase() === 'true',
+        slack_channel: b.getKv(KV.slackChannel) || '',
+        email_enabled: String(b.getKv(KV.emailEnabled) || '').toLowerCase() === 'true',
+        email_to: b.getKv(KV.emailTo) || '',
+      } });
+    } catch (e) { jsonResponse(res, { ok: false, error: e.message }, 500); }
+    return true;
+  }
+  if (p === '/api/wall-e/questions/digest/settings' && m === 'POST') {
+    if (!brain) return jsonResponse(res, { ok: false, error: 'Brain not available' }, 503), true;
+    readBody(req).then((body) => {
+      body = body || {};
+      const { KV } = require('./loops/question-digest');
+      if ('slack_enabled' in body) brain.setKv(KV.slackEnabled, String(!!body.slack_enabled));
+      if ('slack_channel' in body) brain.setKv(KV.slackChannel, String(body.slack_channel || ''));
+      if ('email_enabled' in body) brain.setKv(KV.emailEnabled, String(!!body.email_enabled));
+      if ('email_to' in body) brain.setKv(KV.emailTo, String(body.email_to || ''));
+      jsonResponse(res, { ok: true });
+    }).catch((e) => jsonResponse(res, { ok: false, error: e.message }, 400));
+    return true;
+  }
+
+  // GET /api/wall-e/questions/digest — current undelivered digest questions (inspection +
+  // the daily digest job). The CTM Wall-E tab itself answers via the existing
+  // GET /api/wall-e/questions + POST /api/wall-e/questions/:id/answer routes above.
+  if (p === '/api/wall-e/questions/digest' && m === 'GET') {
+    try {
+      const { buildDigest } = require('./loops/question-digest');
+      const digest = buildDigest({ max: parseInt(params.get('max') || '10', 10) || 10 });
+      jsonResponse(res, { ok: true, ...digest });
+    } catch (e) {
+      jsonResponse(res, { ok: false, error: e.message }, 500);
+    }
+    return true;
+  }
+
   // GET /api/wall-e/workspace/status — Item H of prompt-catalog deep-dive.
   // Returns which workspace prompt files exist + which templates are
   // available to install. Used by the setup page to render the
@@ -4120,7 +4442,105 @@ function handleWalleApi(req, res, url) {
     return true;
   }
 
-  // POST /api/wall-e/coding — start a coding task
+  // POST /api/wall-e/coding/turn — execute one Wall-E coding turn.
+  //
+  // This is the native coding-run boundary for CTM/phone sessions. It differs
+  // from /api/wall-e/chat?stream=1 by running the coding orchestrator directly
+  // and returning a typed completion outcome: changed, read-only, blocked, or
+  // failed. Chat can still discuss code, but coding sessions need this stronger
+  // contract so prose alone cannot masquerade as completed implementation work.
+  if (p === '/api/wall-e/coding/turn' && m === 'POST') {
+    const wantsStream = params.get('stream') === '1' || (req.headers.accept || '').includes('text/event-stream');
+    // Coding messages can carry pasted image attachments (base64) — accept the
+    // same 16MB cap as /api/wall-e/chat, not the 1MB text default (a >1MB image
+    // otherwise fails with "Body too large (>1MB)").
+    readBody(req, { maxBytes: BODY_LIMIT_LARGE }).then(async body => {
+      let requestFinished = false;
+      const abortController = new AbortController();
+      if (typeof res.on === 'function') {
+        res.on('close', () => {
+          if (!requestFinished) abortController.abort();
+        });
+      }
+      const sendJsonError = (err, status = 500) => {
+        jsonResponse(res, { error: err.message || String(err), code: err.code || 'WALLE_CODING_TURN_ERROR' }, status);
+      };
+      const sendStreamEvent = wantsStream
+        ? (event) => {
+          try { res.write('data: ' + JSON.stringify(event) + '\n\n'); } catch {}
+        }
+        : () => {};
+      try {
+        if (!body.message && !body.request && !body.prompt) {
+          if (wantsStream) {
+            res.writeHead(400, {
+              'Content-Type': 'text/event-stream',
+              'Cache-Control': 'no-cache',
+              'Connection': 'keep-alive',
+            });
+            sendStreamEvent({ type: 'error', error: 'message is required', code: 'BAD_REQUEST' });
+            requestFinished = true;
+            res.end();
+          } else {
+            sendJsonError(new Error('message is required'), 400);
+          }
+          return;
+        }
+        if (wantsStream) {
+          res.writeHead(200, {
+            'Content-Type': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+            'Connection': 'keep-alive',
+          });
+        }
+        const { CodingRunController } = require('./coding/coding-run-controller');
+        const controller = new CodingRunController();
+        const result = await controller.runTurn(body, {
+          signal: abortController.signal,
+          onEvent: sendStreamEvent,
+        });
+        requestFinished = true;
+        if (wantsStream) {
+          sendStreamEvent({ type: 'done', reply: result.reply, data: result });
+          res.end();
+        } else {
+          jsonResponse(res, { data: result });
+        }
+      } catch (err) {
+        requestFinished = true;
+        const isUserCancelled = err?.code === 'WALLE_CANCELLED'
+          || err?.message === 'Cancelled'
+          || (err?.name === 'AbortError' && abortController.signal.aborted);
+        if (wantsStream) {
+          if (!res.headersSent) {
+            res.writeHead(isUserCancelled ? 499 : 500, {
+              'Content-Type': 'text/event-stream',
+              'Cache-Control': 'no-cache',
+              'Connection': 'keep-alive',
+            });
+          }
+          sendStreamEvent({
+            type: isUserCancelled ? 'cancelled' : 'error',
+            error: isUserCancelled ? 'Cancelled' : (err.message || String(err)),
+            code: isUserCancelled ? 'WALLE_CANCELLED' : (err.code || 'WALLE_CODING_TURN_ERROR'),
+          });
+          try { res.end(); } catch {}
+        } else {
+          sendJsonError(err, isUserCancelled ? 499 : 500);
+        }
+      }
+    }).catch(e => {
+      if (wantsStream && res.headersSent) {
+        try { res.write('data: ' + JSON.stringify({ type: 'error', error: e.message }) + '\n\n'); } catch {}
+        try { res.end(); } catch {}
+      } else {
+        jsonResponse(res, { error: e.message }, 400);
+      }
+    });
+    return true;
+  }
+
+  // POST /api/wall-e/coding — start a queued coding task
   if (p === '/api/wall-e/coding' && m === 'POST') {
     if (!brain || !ensureBrainInit()) return jsonResponse(res, { error: 'Brain module not available' }, 503), true;
     readBody(req).then(body => {
@@ -4148,7 +4568,7 @@ function handleWalleApi(req, res, url) {
       } catch (e) {
         jsonResponse(res, { error: e.message }, 500);
       }
-    }).catch(e => jsonResponse(res, { error: e.message }, 500));
+    }).catch(e => jsonResponse(res, { error: e.message, code: e.code || '' }, e.code === 'BODY_TOO_LARGE' ? 413 : 500));
     return true;
   }
 
@@ -4424,6 +4844,7 @@ function handleWalleApi(req, res, url) {
 
   // GET /api/wall-e/shadow/stats — shadow training pipeline stats
   if (p === '/api/wall-e/shadow/stats' && m === 'GET') {
+    (async () => {
     try {
       const db = brain.getDb();
       const totalResults = db.prepare('SELECT COUNT(*) AS count FROM shadow_results').get().count;
@@ -4458,41 +4879,22 @@ function handleWalleApi(req, res, url) {
       // Harvest progress — total CTM sessions vs processed
       let harvestProgress = null;
       try {
-        const dataDir = process.env.WALL_E_DATA_DIR || require('path').join(process.env.HOME, '.walle', 'data');
-        const ctmDbPath = require('path').join(dataDir, 'task-manager.db');
-        const fs = require('fs');
-        if (fs.existsSync(ctmDbPath)) {
-          const Database = require('better-sqlite3');
-          const ctmDb = new Database(ctmDbPath, { readonly: true, fileMustExist: true });
-          const totalSessions = ctmDb.prepare('SELECT COUNT(DISTINCT session_id) AS count FROM prompt_executions').get().count;
-          const totalPairs = ctmDb.prepare(`
-            SELECT COUNT(*) AS count FROM prompt_executions u
-            JOIN prompt_executions a ON a.session_id = u.session_id AND a.message_index = u.message_index + 1
-            WHERE u.role = 'user' AND a.role = 'assistant'
-              AND length(u.message_text) >= 20 AND length(a.message_text) >= 20
-          `).get().count;
-          const multiTurnSessions = ctmDb.prepare(`
-            SELECT COUNT(*) AS count FROM (
-              SELECT session_id FROM prompt_executions WHERE role = 'user'
-              GROUP BY session_id HAVING COUNT(*) >= 2
-            )
-          `).get().count;
-          ctmDb.close();
-
-          const singleReplayed = (replayStats['ctm-replay']?.totalProcessed) || 0;
-          const multiReplayed = (replayStats['ctm-replay-mt']?.totalProcessed) || 0;
-
-          harvestProgress = {
-            totalSessions,
-            totalPairs,
-            multiTurnSessions,
-            singleTurnReplayed: singleReplayed,
-            multiTurnReplayed: multiReplayed,
-            totalReplayed: singleReplayed + multiReplayed,
-          };
-        }
+        const { getPromptExecutionStats } = require('./memory/ctm-prompt-executions-client');
+        const stats = await getPromptExecutionStats({ timeoutMs: 1000 });
+        const singleReplayed = (replayStats['ctm-replay']?.totalProcessed) || 0;
+        const multiReplayed = (replayStats['ctm-replay-mt']?.totalProcessed) || 0;
+
+        harvestProgress = {
+          totalSessions: stats.totalSessions || 0,
+          totalPairs: stats.totalPairs || 0,
+          multiTurnSessions: stats.multiTurnSessions || 0,
+          singleTurnReplayed: singleReplayed,
+          multiTurnReplayed: multiReplayed,
+          totalReplayed: singleReplayed + multiReplayed,
+          source: 'ctm-api',
+        };
       } catch (e) {
-        // CTM DB not available — skip
+        harvestProgress = { source: 'ctm-api', unavailable: true, error: e.message };
       }
 
       // Recent results — last 5 shadow results for sample browser
@@ -4509,19 +4911,21 @@ function handleWalleApi(req, res, url) {
         LIMIT 5
       `).all();
 
-      return jsonResponse(res, { data: {
+      jsonResponse(res, { data: {
         totalResults, evaluatedResults,
         avgScore: avgScore != null ? Math.round(avgScore * 1000) / 1000 : null,
         byModel, byTaskType, promotionCandidates, modelScores,
         replayStats, harvestProgress, recentResults,
-      } }), true;
+      } });
     } catch (e) {
-      return jsonResponse(res, { data: {
+      jsonResponse(res, { data: {
         totalResults: 0, evaluatedResults: 0, avgScore: null,
         byModel: [], byTaskType: [], promotionCandidates: [], modelScores: [],
         replayStats: {}, harvestProgress: null, recentResults: [],
-      } }), true;
+      } });
     }
+    })();
+    return true;
   }
 
   // GET /api/wall-e/shadow/comparisons — paginated side-by-side comparison viewer
@@ -5578,6 +5982,28 @@ function handleWalleApi(req, res, url) {
       summary.recent_feedback = [];
     }
 
+    try {
+      summary.feature_adoption = {
+        memory_search: telemetryEmbeddingFeatureAdoption(tdb, since, summary.active_installs || 0),
+      };
+    } catch {
+      summary.feature_adoption = {
+        memory_search: {
+          reporting_installs: 0,
+          enabled_installs: 0,
+          usage_installs: 0,
+          setup_view_installs: 0,
+          switch_success_installs: 0,
+          switch_failed_installs: 0,
+          enabled_pct_active: null,
+          usage_pct_active: null,
+          latest_state: [],
+          enabled_by_provider: [],
+          usage_by_operation: [],
+        },
+      };
+    }
+
     // Compat lifecycle data (Phase 4)
     try {
       // Version distribution with last_seen
@@ -5612,8 +6038,28 @@ function handleWalleApi(req, res, url) {
       // Safe to remove: deprecated features with 0 usage
       try {
         const compat = require('./compat');
-        summary.safe_to_remove = compat.getRemovalCandidates(featureCounts)
-          .filter(c => c.safeToRemove)
+        const registry = typeof compat.getRegistry === 'function' ? compat.getRegistry() : {};
+        summary.compat_removal_status = Object.entries(registry).map(([key, info]) => {
+          const telemetryKey = info.telemetryKey || key;
+          const activeInstalls = featureCounts[telemetryKey] || 0;
+          const deprecated = Boolean(info.deprecatedIn);
+          const safeToRemove = deprecated && activeInstalls === 0;
+          const blockedReason = safeToRemove
+            ? ''
+            : (!deprecated ? 'not_deprecated' : (activeInstalls > 0 ? 'active_usage' : 'not_scheduled'));
+          return {
+            key,
+            telemetry_key: telemetryKey,
+            deprecated,
+            deprecated_in: info.deprecatedIn || null,
+            remove_after: info.removeAfter || null,
+            active_installs: activeInstalls,
+            safe_to_remove: safeToRemove,
+            blocked_reason: blockedReason,
+          };
+        });
+        summary.safe_to_remove = summary.compat_removal_status
+          .filter(c => c.safe_to_remove)
           .map(c => c.key);
       } catch {}
 
@@ -5650,6 +6096,7 @@ function handleWalleApi(req, res, url) {
         apply_clicked: new Set(),
         apply_started: new Set(),
         completed: new Set(),
+        first_tracked_completed: new Set(),
       };
       for (const row of upgradeRows) {
         const identity = row.machine_bucket ? `m:${row.machine_bucket}` : `i:${row.install_id}`;
@@ -5657,6 +6104,7 @@ function handleWalleApi(req, res, url) {
         try { meta = row.meta ? JSON.parse(row.meta) : {}; } catch {}
         if (row.event === 'upgrade') {
           upgradeSets.completed.add(identity);
+          if (meta.first_tracked) upgradeSets.first_tracked_completed.add(identity);
           continue;
         }
         if (row.event === 'funnel') {
@@ -5683,17 +6131,23 @@ function handleWalleApi(req, res, url) {
         }
         if (row.event === 'ctm_update_apply_started') upgradeSets.apply_started.add(identity);
       }
+      const completedAfterApply = telemetrySetIntersectionSize(upgradeSets.completed, upgradeSets.apply_started);
+      const externalCompleted = telemetrySetDifferenceSize(upgradeSets.completed, upgradeSets.apply_started);
       summary.upgrade_funnel = {
         available: upgradeSets.available.size,
         prompted: upgradeSets.prompted.size,
         dismissed: upgradeSets.dismissed.size,
         apply_clicked: upgradeSets.apply_clicked.size,
         apply_started: upgradeSets.apply_started.size,
-        completed: upgradeSets.completed.size,
+        completed: completedAfterApply,
+        completed_after_apply: completedAfterApply,
+        completed_observed: upgradeSets.completed.size,
+        external_completed: externalCompleted,
+        first_tracked_completed: upgradeSets.first_tracked_completed.size,
         prompt_rate: pct(upgradeSets.prompted.size, upgradeSets.available.size),
         apply_rate: pct(upgradeSets.apply_clicked.size, upgradeSets.prompted.size),
         start_rate: pct(upgradeSets.apply_started.size, upgradeSets.apply_clicked.size),
-        completion_rate: pct(upgradeSets.completed.size, upgradeSets.apply_started.size),
+        completion_rate: pct(completedAfterApply, upgradeSets.apply_started.size),
       };
     } catch {}
 
@@ -6230,7 +6684,7 @@ function handleWalleApi(req, res, url) {
 
 // Allow overriding the read DB (for testing)
 function _setReadDb(db) {
-  readDb = db;
+  readDbOverride = db;
 }
 
 function _setBrain(b) {