create-walle 0.9.26 → 0.9.28

package/template/claude-task-manager/workers/read-pool-worker.js

@@ -261,6 +261,16 @@ async function _runOp(op, payload = {}) {
       return { __transfer: true, buffer: Buffer.from(JSON.stringify(obj), 'utf8') };
     }
 
+    case 'buildSessionMessagesFullResponse': {
+      // Stage C: the NON-paginated full read — project (merge + exclusions + image-refs) AND
+      // serialize the WHOLE conversation in the worker, returning BYTES (zero-copy). Removes the
+      // on-main projection that froze the loop while reading a large conversation. Returns either a
+      // bare array or { messages, ...extra } per payload.bareArray (mirrors server.js sendMessages).
+      const proj = require('../lib/session-messages-projection');
+      const obj = proj.buildFullMessagesResponse(payload);
+      return { __transfer: true, buffer: Buffer.from(JSON.stringify(obj), 'utf8') };
+    }
+
     case 'parseJsonlTail': {
       // Bounded (≤~1MB) tail read+parse of a live agent JSONL OFF the main loop — the
       // apiSessionMessages durable-tail reconcile (lagging importer on an actively-written

package/template/package.json

@@ -1,6 +1,6 @@
 {
   "name": "walle",
-  "version": "0.9.26",
+  "version": "0.9.28",
   "private": true,
   "description": "Wall-E — your personal digital twin",
   "scripts": {

package/template/wall-e/agent.js

@@ -147,6 +147,48 @@ function clampTimeoutMs(value, fallback) {
   return Math.max(1000, Math.min(15 * 60 * 1000, Math.trunc(ms)));
 }
 
+function restartCatchupMax(env = process.env) {
+  const raw = env.WALL_E_RESTART_CATCHUP_MAX ?? env.WALLE_RESTART_CATCHUP_MAX;
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return 1;
+  return Math.max(0, Math.min(5, Math.trunc(n)));
+}
+
+function persistEventLoopWarning(warning = {}) {
+  try {
+    const { runtimeEventLedgerEnabled } = require('./lib/diagnostics-flags');
+    if (!runtimeEventLedgerEnabled() || shuttingDown) return;
+    const handle = setImmediate(() => {
+      try {
+        const append = typeof brain.appendRuntimeEvent === 'function' ? brain.appendRuntimeEvent : null;
+        const enqueue = typeof brain.enqueueOwnerWrite === 'function' ? brain.enqueueOwnerWrite : null;
+        if (!append || !enqueue) return;
+        const maxMs = Number(warning.maxMs) || 0;
+        const contextSummary = String(warning.contextSummary || '').slice(0, 360);
+        enqueue('runtime_event:event_loop', (d) => append({
+          source: 'wall-e.event-loop',
+          event_type: 'event_loop_blocked',
+          status: 'warning',
+          duration_ms: maxMs,
+          reason: 'event-loop-blocked',
+          summary: `Event loop blocked ${Math.round(maxMs)}ms${contextSummary ? `; ${contextSummary}` : ''}`,
+          detail: { kind: 'runtime-diagnostics', phase: 'event-loop-monitor' },
+          metrics: {
+            max_ms: maxMs,
+            mean_ms: Number(warning.meanMs) || 0,
+            p99_ms: Number(warning.p99Ms) || 0,
+          },
+        }, d)).catch((err) => {
+          console.warn('[wall-e] Could not persist event-loop warning:', err.message);
+        });
+      } catch (e) {
+        console.warn('[wall-e] Could not persist event-loop warning:', e.message);
+      }
+    });
+    handle.unref?.();
+  } catch {}
+}
+
 function bootstrapSkills() {
   const existing = brain.listSkills({});
   if (existing.length > 0) return; // Already bootstrapped
@@ -352,6 +394,7 @@ async function main() {
           try { workerPool = require('./lib/runtime-worker-pool').getRuntimeWorkerPoolStatus(); } catch {}
           return { scheduler, runtime_active: runtime?.active || [], worker_pool: workerPool };
         },
+        onWarn: persistEventLoopWarning,
       });
     } catch (e) { console.warn('[wall-e] event-loop monitor unavailable:', e.message); }
   }
@@ -557,6 +600,8 @@ async function main() {
         }, {
           timeoutMs: options.timeoutMs || 0,
           heavy: options.heavy !== false,
+          lane: options.lane,
+          priority: options.priority,
           terminateOnTimeout: (options.timeoutMs || 0) > 0,
         });
       } catch (err) {
@@ -691,6 +736,14 @@ async function main() {
     process.env.WALL_E_SCHEDULER_WORKER_TIMEOUT_MS ?? process.env.WALLE_SCHEDULER_WORKER_TIMEOUT_MS,
     180000
   );
+  const schedulerTasksWorkerTimeoutMs = clampTimeoutMs(
+    process.env.WALL_E_SCHEDULER_TASKS_WORKER_TIMEOUT_MS ?? process.env.WALLE_SCHEDULER_TASKS_WORKER_TIMEOUT_MS,
+    300000
+  );
+  const schedulerSkillsWorkerTimeoutMs = clampTimeoutMs(
+    process.env.WALL_E_SCHEDULER_SKILLS_WORKER_TIMEOUT_MS ?? process.env.WALLE_SCHEDULER_SKILLS_WORKER_TIMEOUT_MS,
+    420000
+  );
   const scheduler = new Scheduler({
     tickMs: 5000,
     pools: { llm: 2, ollama: 1, embedding: 1, io: 3 },
@@ -734,6 +787,7 @@ async function main() {
     run: () => ingest.runOnce(adapters),
     intervalMs: clampInterval(config.intervals?.ingest_ms, 60000),
     pool: 'io',
+    lane: 0,
     priority: 1,
     startDelayMs: 2000,
     onResult: (r) => {
@@ -750,6 +804,7 @@ async function main() {
     run: () => think.runOnce(),
     intervalMs: clampInterval(config.intervals?.think_ms, 120000),
     pool: 'llm',
+    lane: 0,
     priority: 2,
     startDelayMs: 5000,
     dependsOn: ['ingest'],
@@ -770,8 +825,10 @@ async function main() {
     run: () => runDueTasks(),
     intervalMs: clampInterval(config.intervals?.tasks_ms, 30000),
     pool: 'llm',
+    lane: 0,
     priority: 3,
     startDelayMs: 8000,
+    workerTimeoutMs: schedulerTasksWorkerTimeoutMs,
     onResult: (r) => {
       if (r.processed > 0) {
         console.log(`[wall-e] Tasks: ${r.processed} completed`);
@@ -793,8 +850,10 @@ async function main() {
     run: ({ modelOverride } = {}) => runDueSkills({ modelOverride }),
     intervalMs: clampInterval(config.intervals?.skills_ms, 300000),
     pool: 'llm',
+    lane: 1,
     priority: 4,
     startDelayMs: 11000,
+    workerTimeoutMs: schedulerSkillsWorkerTimeoutMs,
     onResult: (r) => {
       if (r.executed > 0) {
         console.log(`[wall-e] Skills: ${r.executed} executed, ${r.memoriesCreated} memories`);
@@ -842,7 +901,9 @@ async function main() {
     },
     intervalMs: clampInterval(config.intervals?.training_ms, 300000),
     pool: 'ollama',
+    lane: 2,
     priority: 5,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 20000,
     shouldRun: () => getShadowConfig().enabled,
     onError: (err) => console.error('[eval] Loop error:', err.message),
@@ -865,7 +926,9 @@ async function main() {
     },
     intervalMs: clampInterval(config.intervals?.replay_ms, 300000),
     pool: 'ollama',
+    lane: 2,
     priority: 5,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 15000,
     shouldRun: () => getShadowConfig().enabled,
     onError: (err) => console.error('[replay] Loop error:', err.message),
@@ -879,6 +942,7 @@ async function main() {
     },
     intervalMs: clampInterval(config.intervals?.initiative_ms, 300000),
     pool: 'llm',
+    lane: 1,
     priority: 6,
     startDelayMs: 25000,
     debounceMs: 30000,
@@ -899,7 +963,9 @@ async function main() {
     run: () => reflect.runOnce(),
     intervalMs: clampInterval(config.intervals?.reflect_ms, 3600000),
     pool: 'llm',
+    lane: 2,
     priority: 7,
+    restartCatchupPolicy: 'defer',
     dependsOn: ['think'],
     startDelayMs: 30000,
     onResult: (r) => {
@@ -924,7 +990,9 @@ async function main() {
     },
     intervalMs: 600000,
     pool: 'io',
+    lane: 2,
     priority: 8,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 30000,
     onError: (err) => console.error('[harvest] Loop error:', err.message),
   });
@@ -947,7 +1015,9 @@ async function main() {
     },
     intervalMs: 30000,
     pool: 'embedding',
+    lane: 2,
     priority: 9,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 35000,
     timeoutMs: 90000,
     onError: (err) => {
@@ -961,7 +1031,9 @@ async function main() {
     run: async () => require('./loops/brain-optimize').runOnce(),
     intervalMs: 3600000,    // hourly; self-gates to a one-shot via kv (brain:optimize:v1)
     pool: 'io',
+    lane: 2,
     priority: 4,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 180000,   // 3 min after boot — well past the boot storm
     timeoutMs: 120000,
     onError: (err) => console.error('[brain-optimize] Loop error:', err.message),
@@ -975,7 +1047,9 @@ async function main() {
     },
     intervalMs: 24 * 60 * 60 * 1000, // daily
     pool: 'embedding',
+    lane: 2,
     priority: 10,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 120000, // wait 2min after startup
     onResult: (r) => {
       if (r.archived > 0) console.log(`[wall-e] Dedup: archived ${r.archived} duplicates`);
@@ -996,8 +1070,10 @@ async function main() {
     },
     intervalMs: 24 * 60 * 60 * 1000, // daily
     pool: 'io',
+    lane: 2,
     priority: 11,
-    startDelayMs: 180000, // 3min after startup — well off the boot/readiness path
+    restartCatchupPolicy: 'defer',
+    startDelayMs: 185000, // just after 3min — well off the boot/readiness path and unique in the IO pool
     onError: (err) => {
       console.error('[wall-e] Daily backup error:', err.message);
       telemetry.trackError('daily-backup', err);
@@ -1011,7 +1087,9 @@ async function main() {
     run: async () => brain.runBrainRetention(),
     intervalMs: 60 * 60 * 1000,
     pool: 'io',
+    lane: 2,
     priority: 11,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 90000,
     onResult: (r) => {
       const nonZero = Object.entries(r || {}).filter(([, v]) => typeof v === 'number' && v > 0);
@@ -1031,7 +1109,9 @@ async function main() {
     },
     intervalMs: 60 * 60 * 1000,
     pool: 'io',
+    lane: 2,
     priority: 11,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 100000,
     onResult: (r) => {
       if (r && r.sent) console.log(`[wall-e] Question digest sent (${r.count}): ${JSON.stringify(r.results)}`);
@@ -1048,6 +1128,7 @@ async function main() {
     },
     intervalMs: 300000,
     pool: 'io',
+    lane: 1,
     priority: 7,
     startDelayMs: 12000,
     shouldRun: () => {
@@ -1072,7 +1153,9 @@ async function main() {
     },
     intervalMs: 24 * 60 * 60 * 1000,
     pool: 'io',
+    lane: 2,
     priority: 10,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 300000,
     onResult: (r) => {
       if (r.cleaned > 0) console.log(`[wall-e] Cleaned ${r.cleaned} old recordings`);
@@ -1096,7 +1179,9 @@ async function main() {
     },
     intervalMs: 24 * 60 * 60 * 1000,
     pool: 'io',
+    lane: 2,
     priority: 10,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 295000,
     shouldRun: () => process.env.WALLE_TELEMETRY_SERVER === '1',
     onResult: (r) => {
@@ -1140,7 +1225,9 @@ async function main() {
     },
     intervalMs: 7 * 24 * 60 * 60 * 1000, // Weekly
     pool: 'llm',
+    lane: 2,
     priority: 10,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 300000, // 5min delay on startup
     shouldRun: () => {
       // Only run if we have coding-agent benchmarks
@@ -1171,7 +1258,9 @@ async function main() {
     },
     intervalMs: 7 * 24 * 60 * 60 * 1000, // Weekly
     pool: 'llm',
+    lane: 2,
     priority: 5,
+    restartCatchupPolicy: 'defer',
     startDelayMs: 600000, // 10min delay on startup
     shouldRun: () => {
       // Run on Sundays only
@@ -1226,31 +1315,48 @@ async function main() {
     telemetry.trackError('boot_check', e);
   }
 
-  // Restart catch-up — replay jobs whose persisted nextEligibleAt fell
-  // while the daemon was down. Cap of 5 immediate runs (5s stagger);
-  // anything beyond that is deferred to the periodic tick. Borrowed
-  // from OpenClaw's runMissedJobs (src/cron/service/timer.ts:962-1040).
-  const restartCatchupOp = runtimeHealth.beginOperation('agent.restartCatchup');
-  try {
-    const catchup = await scheduler.runMissedJobs();
-    if (catchup.ran > 0 || catchup.deferred > 0) {
-      console.log(
-        `[wall-e] Restart catch-up: ran ${catchup.ran}, deferred ${catchup.deferred}`
-      );
-      telemetry.track('restart_catchup', {
-        ran: catchup.ran,
-        deferred: catchup.deferred,
-      });
-    }
-    restartCatchupOp.end({ ok: true, meta: { ran: catchup.ran || 0, deferred: catchup.deferred || 0 } });
-  } catch (e) {
-    restartCatchupOp.end({ ok: false, error: e });
-    console.warn('[wall-e] runMissedJobs threw (non-fatal):', e.message);
-  }
-
-  // Start the scheduler
+  // Start the scheduler before restart catch-up. Missed-job replay is useful,
+  // but it must never be startup-critical or block the daemon from serving
+  // chat/MCP requests after a restart.
   scheduler.start();
 
+  // Restart catch-up — replay only small foreground jobs immediately; defer
+  // expensive harvest/eval/maintenance work and persist that decision to the
+  // runtime work-item queue for observability. This keeps the main process
+  // responsive even after long downtime with many missed jobs.
+  setTimeout(() => {
+    (async () => {
+      const restartCatchupOp = runtimeHealth.beginOperation('agent.restartCatchup');
+      try {
+        const catchup = await scheduler.runMissedJobs({
+          nonBlocking: true,
+          max: restartCatchupMax(),
+          staggerMs: 0,
+          maxWorkMs: 5000,
+          deferDelayMs: 120000,
+          enqueueWorkItem: brain.enqueueRuntimeWorkItem,
+        });
+        if (catchup.ran > 0 || catchup.deferred > 0) {
+          console.log(
+            `[wall-e] Restart catch-up: dispatched ${catchup.ran}, deferred ${catchup.deferred}`
+          );
+          telemetry.track('restart_catchup', {
+            ran: catchup.ran,
+            deferred: catchup.deferred,
+            non_blocking: 1,
+          });
+        }
+        restartCatchupOp.end({
+          ok: true,
+          meta: { ran: catchup.ran || 0, deferred: catchup.deferred || 0, non_blocking: 1 },
+        });
+      } catch (e) {
+        restartCatchupOp.end({ ok: false, error: e });
+        console.warn('[wall-e] runMissedJobs threw (non-fatal):', e.message);
+      }
+    })().catch((e) => console.warn('[wall-e] Restart catch-up failed (non-fatal):', e.message));
+  }, 1000).unref?.();
+
   // Hot-reload config on file changes
   const { ConfigWatcher } = require('./lib/config-watcher');
   const configWatcher = new ConfigWatcher();

package/template/wall-e/api-walle.js

@@ -45,7 +45,10 @@ const TELEMETRY_DIAGNOSTIC_EVENTS = new Set([
   'error',
   'skill_fallback_attempt',
   'initiative_provider_cooldown',
+  'sqlite_write_retry',
   'compat_usage',
+  'cli_update_started',
+  'cli_update_completed',
   'upgrade',
   'upgrade_prompt',
   'funnel',
@@ -6210,7 +6213,7 @@ function handleWalleApi(req, res, url) {
         `SELECT te.event, te.install_id, ti.machine_bucket, te.meta
          FROM telemetry_events te
          LEFT JOIN telemetry_installs ti ON ti.install_id = te.install_id
-         WHERE te.event IN ('funnel', 'upgrade', 'upgrade_prompt')
+         WHERE te.event IN ('funnel', 'upgrade', 'upgrade_prompt', 'cli_update_started', 'cli_update_completed')
            AND te.received_at >= ?
          UNION ALL
          SELECT te.event, te.install_id, ti.machine_bucket, te.meta
@@ -6253,6 +6256,14 @@ function handleWalleApi(req, res, url) {
           if (meta.action === 'apply') upgradeSets.apply_clicked.add(identity);
           continue;
         }
+        if (row.event === 'cli_update_started') {
+          upgradeSets.apply_started.add(identity);
+          continue;
+        }
+        if (row.event === 'cli_update_completed') {
+          upgradeSets.completed.add(identity);
+          continue;
+        }
         if (row.event === 'ctm_update_update_available') upgradeSets.available.add(identity);
         if (row.event === 'ctm_update_ui_prompt_shown') upgradeSets.prompted.add(identity);
         if (row.event === 'ctm_update_ui_action') {