create-walle 0.9.21 → 0.9.23

package/template/claude-task-manager/bin/restart-ctm.sh

@@ -1,21 +1,33 @@
 #!/bin/bash
 # Graceful CTM restart via launchd.
-# Tries the API first (graceful shutdown + launchd auto-restart),
-# falls back to launchd kickstart if CTM is unresponsive.
-# NO bash watchers or nohup — launchd KeepAlive handles respawn.
+# Tries the API first (graceful shutdown + launchd auto-respawn). If launchd does NOT
+# bring CTM back -- it can park CTM's deliberate graceful exit (code 42) instead of
+# respawning -- this RECOVERS via a launchd kickstart and verifies again, so an API
+# restart never silently leaves CTM down (the old script just exit 1'd here).
+# A slow restore-storm boot is NOT force-restarted: we only kickstart when launchd
+# reports no running process for the job. NO bash watchers or nohup -- launchd handles respawn.
 CTM_PORT="${CTM_PORT:-3456}"
 LABEL="com.walle.server"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
-# Try the API restart first (works when CTM is running)
-RESULT=$(curl -s -X POST "http://localhost:${CTM_PORT}/api/restart/ctm?force=true" 2>/dev/null)
-if [[ $? -eq 0 && "$RESULT" == *'"ok":true'* ]]; then
-  echo "CTM server restarting via API (launchd will respawn)..."
-else
-  echo "API unreachable. Attempting launchd restart..."
-  SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-  if command -v node >/dev/null 2>&1; then
-    echo "Checking SQLite native driver before launchd restart..."
-    node "$SCRIPT_DIR/check-sqlite-driver.js" >/tmp/ctm-sqlite-driver-check.log 2>&1 || {
+_health_ok() { curl -sf "http://localhost:${CTM_PORT}/api/health" >/dev/null 2>&1; }
+
+# True iff launchd reports a live (non-zero) pid for the job -- i.e. it respawned CTM,
+# even if CTM is still booting and not yet answering /api/health.
+_launchd_running() {
+  launchctl print "gui/$(id -u)/$LABEL" 2>/dev/null | grep -qE 'pid = [1-9][0-9]*'
+}
+
+_kickstart_via_launchd() {
+  echo "Recovering via launchd kickstart..."
+  local REPO_ROOT NODE_BIN
+  REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+  # Run the preflight under the PINNED node (not ambient PATH) so it can never rebuild
+  # the native driver to a mismatched ABI. Falls back to PATH node in non-owner installs.
+  NODE_BIN="$(bash "$REPO_ROOT/bin/node-bin.sh" 2>/dev/null || command -v node 2>/dev/null || true)"
+  if [[ -n "$NODE_BIN" ]]; then
+    echo "Checking SQLite native driver before launchd restart (node: $NODE_BIN)..."
+    "$NODE_BIN" "$SCRIPT_DIR/check-sqlite-driver.js" >/tmp/ctm-sqlite-driver-check.log 2>&1 || {
       echo "SQLite driver preflight failed; launchd startup will retry self-repair."
       tail -20 /tmp/ctm-sqlite-driver-check.log 2>/dev/null || true
     }
@@ -27,18 +39,47 @@ else
   else
     echo "ERROR: launchd service '$LABEL' not installed."
     echo "Install: launchctl load ~/Library/LaunchAgents/com.walle.server.plist"
-    exit 1
+    return 1
   fi
+}
+
+# Wait up to $1 seconds for /api/health. After a startup grace, if launchd shows no
+# running pid for 5 consecutive seconds, conclude it parked the exit and stop early so
+# the caller can kickstart. Returns: 0 = healthy, 2 = parked (needs kickstart), 1 = timeout-while-running.
+_await_health() {
+  local deadline="${1:-120}" grace=8 parked=0 i
+  for i in $(seq 1 "$deadline"); do
+    _health_ok && return 0
+    if (( i > grace )); then
+      if _launchd_running; then parked=0; else parked=$(( parked + 1 )); fi
+      (( parked >= 5 )) && return 2
+    fi
+    sleep 1
+  done
+  return 1
+}
+
+# 1) Try the graceful API restart first (works when CTM is running).
+RESULT="$(curl -s -X POST "http://localhost:${CTM_PORT}/api/restart/ctm?force=true" 2>/dev/null)"
+if [[ $? -eq 0 && "$RESULT" == *'"ok":true'* ]]; then
+  echo "CTM restarting via API (launchd should respawn)..."
+  sleep 2
+  _await_health 120
+  case $? in
+    0) echo "CTM server is back up."; exit 0 ;;
+    1) echo "CTM is running but slow to answer /api/health (heavy boot) -- not force-restarting a live boot."; exit 1 ;;
+    2) echo "launchd did NOT respawn CTM (parked the graceful exit). Recovering..." ;;  # fall through to kickstart
+  esac
+else
+  echo "API unreachable. Attempting launchd restart..."
 fi
 
-# Wait for it to come back
+# 2) Recover via launchd kickstart (parked-exit path, or API unreachable).
+_kickstart_via_launchd || exit 1
 sleep 2
-for i in $(seq 1 15); do
-  if curl -sf "http://localhost:${CTM_PORT}/api/health" > /dev/null 2>&1; then
-    echo "CTM server is back up."
-    exit 0
-  fi
-  sleep 1
-done
-echo "CTM server did not come back within 15 seconds."
+if _await_health 120; then
+  echo "CTM server is back up."
+  exit 0
+fi
+echo "CTM server did not come back after kickstart recovery (within timeout)."
 exit 1

package/template/claude-task-manager/bin/storage-migration-supervisor.js

@@ -0,0 +1,338 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+const { spawn, execFileSync } = require('child_process');
+const Database = require('better-sqlite3');
+const {
+  atomicWriteJson,
+  ensureDir,
+  migrationPaths,
+  safeReadJson,
+} = require('../lib/storage-migration');
+
+function arg(name) {
+  const i = process.argv.indexOf(name);
+  return i >= 0 ? process.argv[i + 1] : '';
+}
+
+const journalPath = arg('--journal');
+const dryRun = process.argv.includes('--dry-run');
+if (!journalPath) {
+  console.error('Usage: storage-migration-supervisor.js --journal <path> [--dry-run]');
+  process.exit(2);
+}
+
+const repoRoot = path.resolve(__dirname, '..', '..');
+const envPath = path.join(repoRoot, '.env');
+const plan = safeReadJson(journalPath);
+if (!plan || !plan.id) {
+  console.error(`Invalid migration journal: ${journalPath}`);
+  process.exit(2);
+}
+
+const paths = migrationPaths(process.env);
+let journal = {
+  ...plan,
+  state: dryRun ? 'dry_run' : 'running',
+  supervisor_pid: process.pid,
+  started_at: new Date().toISOString(),
+  phases: Array.isArray(plan.phases) ? plan.phases : [],
+};
+let heartbeatTimer = null;
+let originalEnv = '';
+let rollbackNeeded = false;
+
+function writeJournal(extra = {}) {
+  journal = { ...journal, ...extra, updated_at: new Date().toISOString() };
+  atomicWriteJson(journalPath, journal);
+  atomicWriteJson(paths.activeJournalPath, journal);
+}
+
+function writeLock(state = journal.state || 'running') {
+  atomicWriteJson(paths.lockPath, {
+    id: plan.id,
+    pid: process.pid,
+    state,
+    allow_start: journal.allow_start === true,
+    journal_path: journalPath,
+    started_at: journal.started_at,
+    heartbeat_at: new Date().toISOString(),
+  });
+}
+
+function phase(name, detail = {}) {
+  journal.phases.push({ name, at: new Date().toISOString(), ...detail });
+  writeJournal();
+  writeLock();
+}
+
+function startHeartbeat() {
+  ensureDir(paths.controlDir);
+  writeLock();
+  heartbeatTimer = setInterval(() => writeLock(), 10_000);
+  heartbeatTimer.unref();
+}
+
+function stopHeartbeat(state) {
+  if (heartbeatTimer) clearInterval(heartbeatTimer);
+  writeLock(state);
+}
+
+function requestJson(port, pathname, { method = 'GET', body = null, timeoutMs = 5000 } = {}) {
+  return new Promise((resolve, reject) => {
+    const payload = body == null ? null : JSON.stringify(body);
+    const req = http.request({
+      host: '127.0.0.1',
+      port,
+      path: pathname,
+      method,
+      timeout: timeoutMs,
+      headers: payload ? {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(payload),
+      } : {},
+    }, (res) => {
+      const chunks = [];
+      res.on('data', c => chunks.push(c));
+      res.on('end', () => {
+        const raw = Buffer.concat(chunks).toString();
+        let json = null;
+        try { json = raw ? JSON.parse(raw) : null; } catch {}
+        resolve({ status: res.statusCode || 0, body: raw, json });
+      });
+    });
+    req.on('timeout', () => req.destroy(new Error('timeout')));
+    req.on('error', reject);
+    if (payload) req.write(payload);
+    req.end();
+  });
+}
+
+function isPortReady(port) {
+  return requestJson(port, '/api/health', { timeoutMs: 1000 })
+    .then(r => r.status >= 200 && r.status < 500)
+    .catch(() => false);
+}
+
+async function waitForPort(port, desired, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const ready = await isPortReady(port);
+    if (ready === desired) return true;
+    await new Promise(r => setTimeout(r, 500));
+  }
+  return false;
+}
+
+function copyDirContents(src, dst) {
+  if (!src || !fs.existsSync(src)) return;
+  const source = path.resolve(src);
+  const target = path.resolve(dst);
+  if (target === source || target.startsWith(source + path.sep)) {
+    throw new Error(`Target data directory must not be inside source directory: ${target}`);
+  }
+  ensureDir(target);
+  for (const name of fs.readdirSync(source)) {
+    const from = path.join(source, name);
+    const to = path.join(target, name);
+    fs.cpSync(from, to, {
+      recursive: true,
+      force: false,
+      errorOnExist: true,
+      preserveTimestamps: true,
+    });
+  }
+}
+
+function copyFileAcrossDevices(src, dst) {
+  ensureDir(path.dirname(dst));
+  if (fs.existsSync(dst)) {
+    const fromStat = fs.statSync(src);
+    const toStat = fs.statSync(dst);
+    if (fromStat.size === toStat.size) return;
+    throw new Error(`Backup target already exists with different size: ${dst}`);
+  }
+  fs.copyFileSync(src, dst);
+  try { fs.utimesSync(dst, fs.statSync(src).atime, fs.statSync(src).mtime); } catch {}
+}
+
+function isBackupFile(service, name) {
+  if (service.name === 'ctm') return /^(task-manager-.+\.db(?:\.gz)?|images-.+\.tar\.gz)$/i.test(String(name || ''));
+  if (service.name === 'walle') return /^wall-e-brain-.+\.db$/i.test(String(name || ''));
+  return false;
+}
+
+function copyBackupFiles(service) {
+  if (!service || !service.backup_dir_changed || !service.move_backups) return;
+  const source = path.resolve(service.current_backup_dir || '');
+  const target = path.resolve(service.target_backup_dir || '');
+  if (!source || !target || source === target) return;
+  if (!fs.existsSync(source)) return;
+  if (target.startsWith(source + path.sep)) {
+    throw new Error(`Target backup directory must not be inside source directory: ${target}`);
+  }
+  ensureDir(target);
+  for (const name of fs.readdirSync(source)) {
+    if (!isBackupFile(service, name)) continue;
+    copyFileAcrossDevices(path.join(source, name), path.join(target, name));
+  }
+}
+
+function verifySqlite(dbPath) {
+  const d = new Database(dbPath, { readonly: true, fileMustExist: true, timeout: 1000 });
+  try {
+    const result = d.pragma('quick_check', { simple: true });
+    if (result !== 'ok') throw new Error(`quick_check returned ${result}`);
+  } finally {
+    d.close();
+  }
+}
+
+function readEnvFile() {
+  try { return fs.readFileSync(envPath, 'utf8'); } catch { return '# Wall-E configuration\n'; }
+}
+
+function updateEnvFile(services) {
+  originalEnv = readEnvFile();
+  const removeKeys = new Set(['CTM_DATA_DIR', 'WALL_E_DATA_DIR', 'CTM_BACKUP_DIR', 'WALL_E_BACKUP_DIR']);
+  const lines = originalEnv.split('\n').filter(line => {
+    const m = line.match(/^\s*#?\s*([A-Z0-9_]+)\s*=/);
+    return !m || !removeKeys.has(m[1]);
+  });
+  while (lines.length && !lines[lines.length - 1].trim()) lines.pop();
+  lines.push('');
+  if (services.ctm?.target_data_dir) lines.push(`CTM_DATA_DIR=${services.ctm.target_data_dir}`);
+  if (services.walle?.target_data_dir) lines.push(`WALL_E_DATA_DIR=${services.walle.target_data_dir}`);
+  if (services.ctm?.target_backup_dir) lines.push(`CTM_BACKUP_DIR=${services.ctm.target_backup_dir}`);
+  if (services.walle?.target_backup_dir) lines.push(`WALL_E_BACKUP_DIR=${services.walle.target_backup_dir}`);
+  const next = lines.join('\n') + '\n';
+  const tmp = `${envPath}.${process.pid}.tmp`;
+  fs.writeFileSync(tmp, next, { mode: 0o600 });
+  fs.renameSync(tmp, envPath);
+}
+
+function rollbackEnv() {
+  if (!originalEnv) return;
+  const tmp = `${envPath}.${process.pid}.rollback.tmp`;
+  fs.writeFileSync(tmp, originalEnv, { mode: 0o600 });
+  fs.renameSync(tmp, envPath);
+}
+
+function stopCtm(pid) {
+  const n = Number(pid);
+  if (!Number.isInteger(n) || n <= 0 || n === process.pid) return;
+  try { process.kill(n, 'SIGTERM'); } catch {}
+  setTimeout(() => { try { process.kill(n, 'SIGKILL'); } catch {} }, 5000).unref();
+}
+
+function startCtm() {
+  if (process.env.CTM_MANAGED_BY_LAUNCHD === '1') {
+    const rawLabel = process.env.CTM_LAUNCHD_LABEL || 'com.walle.server';
+    const label = /^[A-Za-z0-9._-]+$/.test(rawLabel) ? rawLabel : 'com.walle.server';
+    const uid = String(process.getuid() >>> 0);
+    try { execFileSync('launchctl', ['kickstart', '-k', `gui/${uid}/${label}`], { stdio: 'ignore', timeout: 8000 }); return; } catch {}
+  }
+  const child = spawn(path.join(repoRoot, 'bin', 'ctm-launch.sh'), [], {
+    cwd: repoRoot,
+    detached: true,
+    stdio: 'ignore',
+    env: { ...process.env, CTM_STORAGE_MIGRATION_BYPASS: '1' },
+  });
+  child.unref();
+}
+
+async function stopWallE() {
+  try { await requestJson(Number(process.env.CTM_PORT || 3456), '/api/stop/walle', { method: 'POST', timeoutMs: 8000 }); } catch {}
+}
+
+function verifyPlanPaths(services) {
+  for (const service of Object.values(services)) {
+    if (!service) continue;
+    if (!service.data_dir_changed && !service.backup_dir_changed) continue;
+    if (service.data_dir_changed && !service.target_data_dir) throw new Error(`${service.name} target data dir is required`);
+    if (service.backup_dir_changed && !service.target_backup_dir) throw new Error(`${service.name} target backup dir is required`);
+    if (service.data_dir_changed) ensureDir(service.target_data_dir);
+    const backupInsideTargetData = service.data_dir_changed
+      && service.target_backup_dir
+      && path.resolve(service.target_backup_dir).startsWith(path.resolve(service.target_data_dir) + path.sep);
+    if (service.backup_dir_changed && !backupInsideTargetData) ensureDir(service.target_backup_dir);
+    if (service.data_dir_changed && fs.existsSync(service.target_db_path)) {
+      throw new Error(`${service.label || service.name} target database already exists: ${service.target_db_path}`);
+    }
+  }
+}
+
+async function apply() {
+  startHeartbeat();
+  writeJournal({ state: dryRun ? 'dry_run' : 'running' });
+  phase('validate');
+  verifyPlanPaths(plan.services || {});
+  if (dryRun) {
+    writeJournal({ state: 'completed', completed_at: new Date().toISOString(), dry_run: true });
+    stopHeartbeat('completed');
+    return;
+  }
+
+  rollbackNeeded = true;
+  phase('stop_wall_e');
+  await stopWallE();
+
+  phase('stop_ctm');
+  stopCtm(plan.ctm_pid);
+  await waitForPort(Number(process.env.CTM_PORT || 3456), false, 12_000);
+
+  phase('copy_live_data');
+  for (const service of Object.values(plan.services || {})) {
+    if (!service.data_dir_changed) continue;
+    copyDirContents(service.current_data_dir, service.target_data_dir);
+  }
+
+  phase('verify_sqlite');
+  for (const service of Object.values(plan.services || {})) {
+    if (!service.data_dir_changed) continue;
+    verifySqlite(service.target_db_path);
+  }
+
+  phase('copy_backup_snapshots');
+  for (const service of Object.values(plan.services || {})) {
+    copyBackupFiles(service);
+  }
+
+  phase('update_env');
+  updateEnvFile(plan.services || {});
+
+  writeJournal({ allow_start: true });
+  phase('restart_ctm');
+  startCtm();
+  const up = await waitForPort(Number(process.env.CTM_PORT || 3456), true, 60_000);
+  if (!up) throw new Error('CTM did not become healthy after migration');
+
+  phase('restart_wall_e');
+  try { await requestJson(Number(process.env.CTM_PORT || 3456), '/api/start/walle', { method: 'POST', timeoutMs: 20_000 }); } catch {}
+
+  rollbackNeeded = false;
+  writeJournal({ state: 'completed', completed_at: new Date().toISOString() });
+  stopHeartbeat('completed');
+}
+
+apply().catch(async (err) => {
+  try {
+    phase('error', { error: err.message });
+    if (rollbackNeeded) {
+      phase('rollback_env');
+      rollbackEnv();
+      writeJournal({ allow_start: true });
+      phase('restart_ctm_after_rollback');
+      startCtm();
+    }
+    writeJournal({ state: 'reverted', error: err.message, completed_at: new Date().toISOString() });
+    stopHeartbeat('reverted');
+  } catch (rollbackErr) {
+    writeJournal({ state: 'failed', error: err.message, rollback_error: rollbackErr.message, completed_at: new Date().toISOString() });
+    stopHeartbeat('failed');
+  }
+  process.exit(1);
+});