create-walle 0.9.21 → 0.9.23

package/template/claude-task-manager/lib/jsonl-range-reader.js

@@ -0,0 +1,112 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+function _isCodexPath(fp) {
+  return String(fp || '').includes(`${path.sep}.codex${path.sep}sessions${path.sep}`);
+}
+
+// Lossless hydration for the row-store read path.
+//
+// transcript_events rows store a LIGHTWEIGHT, truncated, text-only projection of each
+// message (good for search + fast pagination) plus a byte offset (jsonl_path, byte_start,
+// byte_end) into the source JSONL. To render a session WITHOUT losing content (tool calls,
+// tool results, images, >50KB messages), we use the rows only to LOCATE the page's bytes,
+// then read that bounded span from the JSONL and parse it with the SAME parser the legacy
+// blob/JSONL read path uses (_parseJsonlIntoMessages) — so the hydrated messages are
+// byte-identical to what the blob path renders. The rows are an index; the JSONL stays the
+// source of truth. The read is bounded to the page's span (not the whole multi-GB file), so
+// there is no freeze.
+
+function readByteRange(filePath, start, length) {
+  const len = Math.max(0, Number(length) || 0);
+  if (!filePath || len === 0) return '';
+  const fd = fs.openSync(filePath, 'r');
+  try {
+    const buf = Buffer.alloc(len);
+    const n = fs.readSync(fd, buf, 0, len, Math.max(0, Number(start) || 0));
+    return buf.subarray(0, n).toString('utf8');
+  } finally {
+    try { fs.closeSync(fd); } catch { /* ignore */ }
+  }
+}
+
+// rows: page rows with { jsonl_path, byte_start, byte_end }.
+// lightweightFallback: the row-text messages to serve if NOTHING can be hydrated
+// (every source file moved/truncated) — so a missing file degrades to text, never throws.
+// Returns FULL-fidelity messages for the page, chronologically sorted.
+function hydratePageMessages(rows, lightweightFallback = []) {
+  let parser;
+  try { parser = require('./jsonl-conversation-parser'); } catch { return lightweightFallback; }
+  const claudeParse = parser._parseJsonlIntoMessages;
+  if (typeof claudeParse !== 'function') return lightweightFallback;
+  // The Codex content parser lives in session-history (jsonl-conversation-parser imports but
+  // does NOT re-export it). Without it, Codex content parses as 0/garbage.
+  let codexParse = null;
+  try {
+    const sh = require('./session-history');
+    if (typeof sh.parseCodexJsonlIntoMessages === 'function') codexParse = sh.parseCodexJsonlIntoMessages;
+  } catch { /* optional */ }
+
+  const list = Array.isArray(rows) ? rows : [];
+  if (list.length === 0) return lightweightFallback;
+  const liteOf = (r) => ({ role: r && r.role, text: (r && (r.text != null ? r.text : r.content)) || '', timestamp: (r && r.timestamp) || '' });
+
+  // Group the page's rows per source file and decide PER FILE how to hydrate:
+  //  - Contiguous span (Claude: timestamps monotonic with byte → the page's rows sit in a
+  //    small byte range). Reading that whole span and parsing it captures the tool-use /
+  //    tool-result lines BETWEEN the message rows — which the transcript rows omit but the
+  //    blob/JSONL render shows. This is what makes Claude render-identical to the blob path.
+  //  - Row-by-row (Codex: timestamps are NOT monotonic with byte, so the page's rows scatter
+  //    across the whole multi-GB file; a span read would cover ~the entire file = wrong content
+  //    + a near-freeze). Reading each row's own line stays bounded and exact.
+  // The span-size threshold distinguishes the two automatically from the actual byte layout.
+  const MAX_SPAN_BYTES = 16 * 1024 * 1024;
+  const byFile = new Map();
+  for (const r of list) {
+    const fp = r && r.jsonl_path;
+    if (!fp || !(Number(r.byte_end) > Number(r.byte_start))) { byFile.set('__lite__:' + (byFile.size), { lite: liteOf(r) }); continue; }
+    const g = byFile.get(fp) || { fp, rows: [], min: Infinity, max: 0 };
+    g.rows.push(r); g.min = Math.min(g.min, Number(r.byte_start)); g.max = Math.max(g.max, Number(r.byte_end));
+    byFile.set(fp, g);
+  }
+
+  const out = [];
+  let hydratedAny = false;
+  const parseInto = (content, fp, sink) => {
+    if (_isCodexPath(fp) && typeof codexParse === 'function') codexParse(content, sink, {});
+    else claudeParse(content, sink, { sourceFile: fp });
+  };
+  for (const [, g] of byFile) {
+    if (g.lite) { out.push(g.lite); continue; }
+    const spanBytes = g.max - g.min;
+    // Span-read only for Claude/generic (rows contiguous in byte; captures interleaved tool
+    // lines). Codex ALWAYS goes row-by-row: its rows already exclude tool/event payloads, and
+    // its timestamps aren't byte-monotonic so even a small span can mis-span. Provider, not
+    // span size, is the right discriminator (small-span Codex regressed under a size-only rule).
+    if (!_isCodexPath(g.fp) && spanBytes > 0 && spanBytes <= MAX_SPAN_BYTES) {
+      let content = '';
+      try { content = readByteRange(g.fp, g.min, spanBytes); } catch { content = ''; }
+      const parsed = [];
+      if (content) { try { parseInto(content, g.fp, parsed); } catch { /* */ } }
+      if (parsed.length) { for (const m of parsed) out.push(m); hydratedAny = true; continue; }
+      // fall through to row-by-row if the span produced nothing
+    }
+    for (const r of g.rows) {
+      let content = '';
+      try { content = readByteRange(g.fp, Number(r.byte_start), Number(r.byte_end) - Number(r.byte_start)); } catch { content = ''; }
+      const parsed = [];
+      if (content) { try { parseInto(content, g.fp, parsed); } catch { /* */ } }
+      if (parsed.length) { for (const m of parsed) out.push(m); hydratedAny = true; }
+      else out.push(liteOf(r));
+    }
+  }
+  if (!hydratedAny) return lightweightFallback;
+  out.forEach(m => { if (m) { delete m._sourceFile; delete m._sourceLine; delete m._entrySessionId; } });
+  // Order chronologically to match the blob path's sortTimelineMessagesInPlace.
+  out.sort((a, b) => String(a.timestamp || '').localeCompare(String(b.timestamp || '')));
+  return out;
+}
+
+module.exports = { hydratePageMessages, readByteRange };

package/template/claude-task-manager/lib/main-db-census.js

@@ -0,0 +1,216 @@
+'use strict';
+
+// Main-thread DB census + budget guard.
+//
+// THE PROBLEM IT SOLVES: CTM's freeze is never really "addApprovalObservation" or
+// "backfillRewrite" — those are just WHICH of the dozens of synchronous main-thread DB
+// callers happened to hit cold Dropbox pages first this boot. The existing
+// [perf-lag]/[freeze-probe] diagnostics surface the single worst block, so we fix one
+// driver, the next surfaces, forever (whack-a-mole). This module turns the hunt into a
+// CENSUS: instrument the main-thread DB handle so a single cold boot yields the COMPLETE
+// ranked list of every main-thread DB op (by cumulative + max cost, with a stack for the
+// slow ones) — and a definition of done: zero UNBOUNDED main-thread ops on the cold path.
+//
+// THE INVARIANT IT ENFORCES: "no unbounded synchronous DB op on the main thread." A
+// single-row indexed .get() is fine even cold (one page); a full scan, a SELECT * on a
+// blob column, or a multi-row write freezes the loop. When an op exceeds the budget we
+// log `[db-budget]` with the SQL + stack so a NEW violation shows up the moment it's
+// written, not three restarts later.
+//
+// SAFETY: this is debug instrumentation, OFF by default. When disabled, maybeInstrument()
+// returns the raw better-sqlite3 handle untouched — zero overhead, zero behavior change.
+// HARD RULE: it adds per-DB-call overhead, so it must NEVER run on the primary (doing so
+// caused a real production slowdown). It only arms on a dev/staging instance (which sets
+// DEV_CTM_PORT) with CTM_DB_CENSUS=1. There is no primary-arming path.
+
+
+// Capture a stack for an op only when it is at least this slow (ms) — keeps per-call
+// overhead negligible for the thousands of fast ops while still attributing every freeze.
+const STACK_MS = clampInt(process.env.CTM_DB_CENSUS_STACK_MS, 25, 1, 60000);
+// Emit a `[db-budget]` warning when a single main-thread op exceeds this (ms). This is the
+// invariant tripwire — an unbounded op (scan / blob / big write) on the loop.
+const BUDGET_MS = clampInt(process.env.CTM_DB_CENSUS_BUDGET_MS, 100, 1, 600000);
+// Cap distinct SQL keys retained, so a pathological dynamic-SQL generator can't grow the
+// census map without bound. The hottest keys (by sumMs) are always kept.
+const MAX_KEYS = clampInt(process.env.CTM_DB_CENSUS_MAX_KEYS, 2000, 16, 100000);
+
+function clampInt(raw, dflt, lo, hi) {
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return dflt;
+  return Math.max(lo, Math.min(hi, Math.trunc(n)));
+}
+
+let _enabled = false;
+let _armedReason = '';
+const _census = new Map(); // sqlKey → stats
+let _startedAt = 0;
+let _budgetWarnings = 0;
+
+// Decide whether to arm. HARD RULE: this census wraps the live DB handle and adds per-call
+// overhead — it must NEVER run on the primary (it caused a real production slowdown there).
+// It only arms on a dev/staging instance, which sets DEV_CTM_PORT. There is deliberately no
+// primary-arming path (no sentinel) — staging only.
+function _shouldArm() {
+  if (!process.env.DEV_CTM_PORT) return ''; // primary: never arm, full stop
+  if (process.env.CTM_DB_CENSUS === '1') return 'env';
+  return '';
+}
+
+function isEnabled() { return _enabled; }
+function armedReason() { return _armedReason; }
+
+function _shortSql(sql) {
+  const s = String(sql || '').replace(/\s+/g, ' ').trim();
+  return s.length > 200 ? s.slice(0, 200) + '…' : s;
+}
+
+// A trimmed stack: drop this module's frames + the node internals, keep the CTM call site.
+function _captureStack() {
+  const raw = new Error().stack || '';
+  return raw
+    .split('\n')
+    .slice(1)
+    .filter((l) => !l.includes('main-db-census.js') && !l.includes('node:internal'))
+    .slice(0, 6)
+    .map((l) => l.trim())
+    .join(' <- ');
+}
+
+function _record(sqlKey, op, ms, rows) {
+  let st = _census.get(sqlKey);
+  if (!st) {
+    if (_census.size >= MAX_KEYS) return; // bounded; hottest keys already retained
+    st = { sql: _shortSql(sqlKey), op, count: 0, sumMs: 0, maxMs: 0, maxRows: 0, slowStack: '', slowMs: 0 };
+    _census.set(sqlKey, st);
+  }
+  st.count += 1;
+  st.sumMs += ms;
+  if (ms > st.maxMs) st.maxMs = ms;
+  if (rows > st.maxRows) st.maxRows = rows;
+  if (ms >= STACK_MS && ms > st.slowMs) {
+    st.slowMs = ms;
+    st.slowStack = _captureStack();
+  }
+  if (ms >= BUDGET_MS) {
+    _budgetWarnings += 1;
+    // The invariant tripwire. Logged (not thrown) in prod so it can never break a request;
+    // a dev harness can choose to throw on these.
+    console.warn(`[db-budget] main-thread ${op} took ${Math.round(ms)}ms rows=${rows} sql="${st.sql}"`
+      + (st.slowStack ? `\n           ${st.slowStack}` : ''));
+  }
+}
+
+// Wrap a better-sqlite3 Statement with timed get/all/run/iterate. Toggle/chaining methods
+// (pluck/expand/raw/bind/safeIntegers) return `this` in better-sqlite3, so our versions
+// return the WRAPPER to keep the proxy across chains (e.g. `.pluck().get()` stays timed).
+function _wrapStatement(stmt) {
+  const sqlKey = stmt.source || '(unknown sql)';
+  const timed = (op) => function (...args) {
+    const t0 = process.hrtime.bigint();
+    const out = stmt[op](...args);
+    const ms = Number(process.hrtime.bigint() - t0) / 1e6;
+    let rows = 0;
+    if (op === 'all' && Array.isArray(out)) rows = out.length;
+    else if (op === 'get') rows = out ? 1 : 0;
+    else if (op === 'run' && out && typeof out.changes === 'number') rows = out.changes;
+    _record(sqlKey, op, ms, rows);
+    return out;
+  };
+  const wrapper = {
+    get: timed('get'),
+    all: timed('all'),
+    run: timed('run'),
+    iterate: timed('iterate'),
+  };
+  // Chaining toggles → delegate, but return the wrapper so timing survives the chain.
+  for (const m of ['pluck', 'expand', 'raw', 'bind', 'safeIntegers']) {
+    if (typeof stmt[m] === 'function') {
+      wrapper[m] = function (...args) { stmt[m](...args); return wrapper; };
+    }
+  }
+  // Pass-through for metadata methods/props that callers read.
+  wrapper.columns = function (...args) { return stmt.columns(...args); };
+  Object.defineProperties(wrapper, {
+    source: { get: () => stmt.source },
+    reader: { get: () => stmt.reader },
+    readonly: { get: () => stmt.readonly },
+    busy: { get: () => stmt.busy },
+    database: { get: () => stmt.database },
+    _rawStatement: { value: stmt },
+  });
+  return wrapper;
+}
+
+// Wrap the connection so .prepare() returns wrapped statements and .exec/.pragma are timed.
+// Everything else delegates to the raw handle unchanged (transaction, backup, pragma config,
+// close, etc.), so behavior is identical — only the timed paths are observed.
+function _wrapConnection(rawDb) {
+  return new Proxy(rawDb, {
+    get(target, prop, receiver) {
+      if (prop === 'prepare') {
+        return function (sql) {
+          const stmt = target.prepare(sql);
+          try { return _wrapStatement(stmt); } catch { return stmt; }
+        };
+      }
+      if (prop === 'exec') {
+        return function (sql) {
+          const t0 = process.hrtime.bigint();
+          const out = target.exec(sql);
+          const ms = Number(process.hrtime.bigint() - t0) / 1e6;
+          _record(`exec: ${_shortSql(sql)}`, 'exec', ms, 0);
+          return out;
+        };
+      }
+      if (prop === 'pragma') {
+        return function (source, options) {
+          const t0 = process.hrtime.bigint();
+          const out = target.pragma(source, options);
+          const ms = Number(process.hrtime.bigint() - t0) / 1e6;
+          _record(`pragma: ${_shortSql(source)}`, 'pragma', ms, 0);
+          return out;
+        };
+      }
+      const val = Reflect.get(target, prop, receiver);
+      return typeof val === 'function' ? val.bind(target) : val;
+    },
+  });
+}
+
+// Wrap the handle IF armed; otherwise return it untouched. Called once per initDb/connectDb
+// right after the connection is opened, BEFORE the restore storm runs.
+function maybeInstrument(rawDb) {
+  if (!rawDb) return rawDb;
+  if (!_enabled) {
+    const reason = _shouldArm();
+    if (!reason) return rawDb; // default path: raw handle, zero overhead
+    _enabled = true;
+    _armedReason = reason;
+    _startedAt = Date.now();
+    console.warn(`[db-census] ARMED (${reason}) — instrumenting main-thread DB handle; `
+      + `budget=${BUDGET_MS}ms stackThreshold=${STACK_MS}ms. GET /api/diagnostics/db-census to read.`);
+  }
+  try { return _wrapConnection(rawDb); } catch { return rawDb; }
+}
+
+// Ranked snapshot (by cumulative main-thread ms — the cold-boot cost of each query).
+function getCensus({ limit = 100 } = {}) {
+  const rows = [..._census.values()]
+    .map((s) => ({ ...s, avgMs: s.count ? +(s.sumMs / s.count).toFixed(2) : 0,
+      sumMs: +s.sumMs.toFixed(1), maxMs: +s.maxMs.toFixed(1) }))
+    .sort((a, b) => b.sumMs - a.sumMs);
+  return {
+    enabled: _enabled,
+    armedReason: _armedReason,
+    sinceMs: _startedAt ? Date.now() - _startedAt : 0,
+    budgetMs: BUDGET_MS,
+    stackThresholdMs: STACK_MS,
+    budgetWarnings: _budgetWarnings,
+    distinctSql: _census.size,
+    top: rows.slice(0, Math.max(1, limit)),
+  };
+}
+
+function reset() { _census.clear(); _budgetWarnings = 0; _startedAt = Date.now(); }
+
+module.exports = { maybeInstrument, getCensus, reset, isEnabled, armedReason };

package/template/claude-task-manager/lib/message-pagination.js

@@ -17,6 +17,15 @@
 const MIN_PAGE_SIZE = 1;
 const MAX_PAGE_SIZE = 1000;
 
+function _pageSize(limit) {
+  const rawLimit = Number.isFinite(limit) ? Math.floor(limit) : 0;
+  return Math.max(MIN_PAGE_SIZE, Math.min(MAX_PAGE_SIZE, rawLimit || MIN_PAGE_SIZE));
+}
+
+function _loadedOffset(offset) {
+  return Math.max(0, Number.isFinite(offset) ? Math.floor(offset) : 0);
+}
+
 /**
  * @param {Array} all         — chronologically sorted messages
  * @param {number} offset     — how many most-recent messages are already loaded
@@ -29,9 +38,8 @@ function paginateMessages(all, offset, limit) {
   if (total === 0) {
     return { messages: [], total: 0, has_more: false, next_offset: 0 };
   }
-  const alreadyLoaded = Math.max(0, Number.isFinite(offset) ? Math.floor(offset) : 0);
-  const rawLimit = Number.isFinite(limit) ? Math.floor(limit) : 0;
-  const pageSize = Math.max(MIN_PAGE_SIZE, Math.min(MAX_PAGE_SIZE, rawLimit || MIN_PAGE_SIZE));
+  const alreadyLoaded = _loadedOffset(offset);
+  const pageSize = _pageSize(limit);
   const remaining = Math.max(0, total - alreadyLoaded);
   const take = Math.min(pageSize, remaining);
   const endIdx = total - alreadyLoaded;        // exclusive
@@ -44,4 +52,98 @@ function paginateMessages(all, offset, limit) {
   };
 }
 
-module.exports = { paginateMessages, MIN_PAGE_SIZE, MAX_PAGE_SIZE };
+function defaultIsTurnStart(message) {
+  return !!message
+    && String(message.role || '').trim() === 'user'
+    && String(message.text || message.content || message.message || '').trim().length > 0;
+}
+
+function splitMessagesIntoTurns(all, options = {}) {
+  const arr = Array.isArray(all) ? all : [];
+  const isTurnStart = typeof options.isTurnStart === 'function'
+    ? options.isTurnStart
+    : defaultIsTurnStart;
+  const setup = [];
+  const turns = [];
+  let current = null;
+
+  arr.forEach((message, index) => {
+    if (isTurnStart(message, index, arr)) {
+      current = {
+        type: 'turn',
+        start_index: index,
+        end_index: index,
+        messages: [message],
+      };
+      turns.push(current);
+      return;
+    }
+    if (current) {
+      current.messages.push(message);
+      current.end_index = index;
+      return;
+    }
+    setup.push(message);
+  });
+
+  if (setup.length > 0) {
+    turns.unshift({
+      type: 'setup',
+      start_index: 0,
+      end_index: setup.length - 1,
+      messages: setup,
+    });
+  }
+  return turns;
+}
+
+/**
+ * Paginates by complete semantic turns instead of raw rows.
+ *
+ * Review/conversation UIs group raw provider rows after fetching. Raw-message
+ * pagination can begin in the middle of a long tool-heavy turn, which leaves
+ * the renderer without the owning user prompt and makes it collapse everything
+ * into a fake "setup" block. This helper keeps the same newest-first pagination
+ * shape as paginateMessages, but `offset` and `limit` count complete turns.
+ */
+function paginateMessageTurns(all, offset, limit, options = {}) {
+  const turns = splitMessagesIntoTurns(all, options);
+  const total = turns.length;
+  if (total === 0) {
+    return {
+      messages: [],
+      total: 0,
+      has_more: false,
+      next_offset: 0,
+      page_kind: 'turns',
+      turn_count: 0,
+    };
+  }
+
+  const alreadyLoaded = _loadedOffset(offset);
+  const pageSize = _pageSize(limit);
+  const remaining = Math.max(0, total - alreadyLoaded);
+  const take = Math.min(pageSize, remaining);
+  const endIdx = total - alreadyLoaded;
+  const startIdx = Math.max(0, endIdx - take);
+  const pageTurns = turns.slice(startIdx, endIdx);
+
+  return {
+    messages: pageTurns.flatMap(turn => turn.messages),
+    total,
+    has_more: startIdx > 0,
+    next_offset: alreadyLoaded + take,
+    page_kind: 'turns',
+    turn_count: pageTurns.length,
+    first_turn_index: pageTurns.length ? startIdx : -1,
+    last_turn_index: pageTurns.length ? endIdx - 1 : -1,
+  };
+}
+
+module.exports = {
+  paginateMessages,
+  paginateMessageTurns,
+  splitMessagesIntoTurns,
+  MIN_PAGE_SIZE,
+  MAX_PAGE_SIZE,
+};