create-walle 0.9.21 → 0.9.23

package/template/claude-task-manager/lib/transcript-store.js

@@ -5,7 +5,9 @@ const fs = require('fs');
 const path = require('path');
 const { StringDecoder } = require('string_decoder');
 
-const { codexMessageFromEntry, codexRolloutIdFromPath, codexUserKey } = require('./session-history');
+const { codexMessageFromEntry, codexRolloutIdFromPath, createCodexUserDeduper } = require('./session-history');
+const { isProviderGeneratedUserContextText } = require('./provider-user-context');
+const { sortTimelineMessages } = require('./timeline-order');
 
 const PARSER_VERSION = 1;
 const DEFAULT_CHUNK_BYTES = 1024 * 1024;
@@ -16,6 +18,16 @@ function sha1(value) {
   return crypto.createHash('sha1').update(String(value || '')).digest('hex');
 }
 
+function parseMetadataJson(value) {
+  if (!value) return {};
+  try {
+    const parsed = JSON.parse(String(value));
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
 function normalizeProvider(provider, filePath = '') {
   const raw = String(provider || '').trim().toLowerCase();
   if (raw) return raw;
@@ -28,6 +40,48 @@ function sourceIdFromPath(filePath) {
     || path.basename(filePath || '').replace(/\.jsonl(\.bak)?$/, '');
 }
 
+function isSubagentTranscriptPath(filePath) {
+  return String(filePath || '').replace(/\\/g, '/').includes('/subagents/');
+}
+
+function loadedTranscriptMessageFromRow(row) {
+  const metadata = parseMetadataJson(row.metadata_json);
+  const text = String(row.text || '');
+  let role = row.role;
+  let agentLabel = metadata.agentLabel || metadata.roleLabel || '';
+  let roleLabel = metadata.roleLabel || metadata.agentLabel || '';
+  const sourceKind = metadata.sourceKind || (isSubagentTranscriptPath(row.jsonl_path) ? 'subagent' : '');
+
+  if (role === 'user' && isProviderGeneratedUserContextText(text)) return null;
+
+  if (sourceKind === 'subagent') {
+    metadata.sourceKind = 'subagent';
+    if (role === 'user') {
+      metadata.originalRole = metadata.originalRole || 'user';
+      metadata.promptKind = metadata.promptKind || 'subagent';
+      role = 'system';
+      agentLabel = agentLabel || 'Subagent prompt';
+      roleLabel = roleLabel || 'Subagent prompt';
+    } else if (!agentLabel && role === 'assistant') {
+      agentLabel = 'Subagent';
+      roleLabel = roleLabel || 'Subagent';
+    }
+  }
+
+  return {
+    role,
+    text,
+    timestamp: row.timestamp || '',
+    metadata,
+    source: 'transcript-events',
+    provider: row.provider || '',
+    agentSessionId: row.agent_session_id,
+    ctmSessionId: row.ctm_session_id || '',
+    agentLabel: agentLabel || undefined,
+    roleLabel: roleLabel || undefined,
+  };
+}
+
 function ensureTranscriptTables(db, options = {}) {
   db.exec(`
     CREATE TABLE IF NOT EXISTS transcript_files (
@@ -76,6 +130,33 @@ function ensureTranscriptTables(db, options = {}) {
       ON transcript_events(timestamp);
   `);
 
+  // Phase 1 of the blob→rows migration: enrich transcript_events so it can be the
+  // PRIMARY conversation store (not just a search overlay). Additive, idempotent —
+  // SQLite has no ADD COLUMN IF NOT EXISTS, so each ALTER is guarded individually.
+  //   turn_ordinal  – turn group (a user prompt + its answer/tool calls); assigned at
+  //                   ingest, drives turn-mode pagination + render grouping.
+  //   token_count   – cheap per-row token estimate; summed for the session total.
+  //   uuid/parent_uuid – Claude message identity (promoted from metadata_json) for
+  //                   streamed-partial dedup and parentUuid collapse parity.
+  //   model/usage_json – per-message model + provider usage when present.
+  //   content_json  – optional inline rich content for recent rows; empty rows
+  //                   hydrate on demand from (jsonl_path, byte_start, byte_end).
+  for (const ddl of [
+    'ALTER TABLE transcript_events ADD COLUMN turn_ordinal INTEGER DEFAULT 0',
+    'ALTER TABLE transcript_events ADD COLUMN token_count INTEGER DEFAULT 0',
+    "ALTER TABLE transcript_events ADD COLUMN uuid TEXT DEFAULT ''",
+    "ALTER TABLE transcript_events ADD COLUMN parent_uuid TEXT DEFAULT ''",
+    "ALTER TABLE transcript_events ADD COLUMN model TEXT DEFAULT ''",
+    "ALTER TABLE transcript_events ADD COLUMN usage_json TEXT DEFAULT ''",
+    "ALTER TABLE transcript_events ADD COLUMN content_json TEXT DEFAULT ''",
+  ]) {
+    try { db.exec(ddl); } catch { /* column already exists */ }
+  }
+  try {
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_transcript_events_ctm_turn
+      ON transcript_events(ctm_session_id, turn_ordinal, seq);`);
+  } catch { /* index already exists / column missing on an old build */ }
+
   try {
     db.exec(`
       CREATE VIRTUAL TABLE IF NOT EXISTS transcript_events_fts USING fts5(
@@ -129,6 +210,13 @@ function truncateVisibleText(text) {
   return `${value.slice(0, MAX_VISIBLE_TEXT)}...truncated (${value.length} chars)`;
 }
 
+// Cheap per-row token estimate (~4 chars/token). Feeds an incremental session
+// total; exact provider-authoritative counts are recomputed on the live path.
+function estimateRowTokens(text) {
+  const len = String(text || '').length;
+  return len ? Math.max(1, Math.ceil(len / 4)) : 0;
+}
+
 function messageFromEntry(entry, provider, state = {}) {
   if (!entry || typeof entry !== 'object') return null;
 
@@ -145,9 +233,8 @@ function messageFromEntry(entry, provider, state = {}) {
     const msg = codexMessageFromEntry(entry);
     if (!msg) return null;
     if (msg.role === 'user') {
-      const key = codexUserKey(msg.text);
-      if (!key || state.codexSeenUsers?.has(key)) return null;
-      if (state.codexSeenUsers) state.codexSeenUsers.add(key);
+      if (!state.codexUserDeduper) state.codexUserDeduper = createCodexUserDeduper();
+      if (!state.codexUserDeduper.remember(msg.text, msg.timestamp)) return null;
     }
     return { role: msg.role, text: truncateVisibleText(msg.text), timestamp: msg.timestamp || '', metadata: {} };
   }
@@ -167,6 +254,8 @@ function messageFromEntry(entry, provider, state = {}) {
     let text = '';
     if (entry.partType === 'tool_call') text = `[Tool: ${data.name || 'tool'}]`;
     else if (entry.partType === 'tool_result') text = `[Tool result: ${data.name || 'tool'}]`;
+    else if (entry.partType === 'artifact') text = artifactText(data);
+    else if (entry.partType === 'capability_routed') text = capabilityRouteText(data);
     else if (entry.partType === 'error') text = `[Wall-E error] ${data.message || ''}`.trim();
     else if (entry.partType === 'cancelled') text = '[Wall-E cancelled]';
     return text ? { role: 'system', text, timestamp: entry.timestamp || '', metadata: { partType: entry.partType || '' } } : null;
@@ -179,11 +268,18 @@ function messageFromEntry(entry, provider, state = {}) {
     text = isToolResult ? text : cleanClaudeUserText(text);
     text = truncateVisibleText(text);
     if (!text) return null;
+    if (!isToolResult && isProviderGeneratedUserContextText(text)) return null;
+    const isSubagentTask = !isToolResult && state.sourceKind === 'subagent';
     return {
-      role: isToolResult ? 'system' : 'user',
+      role: isToolResult || isSubagentTask ? 'system' : 'user',
       text,
       timestamp: entry.timestamp || '',
-      metadata: { uuid: entry.uuid || '', cwd: entry.cwd || '', gitBranch: entry.gitBranch || '' },
+      metadata: {
+        uuid: entry.uuid || '',
+        cwd: entry.cwd || '',
+        gitBranch: entry.gitBranch || '',
+        ...(isSubagentTask ? { sourceKind: 'subagent', originalRole: 'user', promptKind: 'subagent', roleLabel: 'Subagent prompt', agentLabel: 'Subagent prompt' } : {}),
+      },
     };
   }
 
@@ -201,6 +297,24 @@ function messageFromEntry(entry, provider, state = {}) {
   return null;
 }
 
+function artifactText(data = {}) {
+  const artifact = data.artifact || {};
+  const kind = artifact.kind || data.type || 'artifact';
+  const label = artifact.path || artifact.originalPath || artifact.artifactId || '';
+  const bits = [`[Artifact: ${kind}]`];
+  if (artifact.bytes) bits.push(`${artifact.bytes} bytes`);
+  if (artifact.metadata?.page_count) bits.push(`${artifact.metadata.page_count} pages`);
+  if (label) bits.push(label);
+  return bits.join(' ');
+}
+
+function capabilityRouteText(data = {}) {
+  const capabilities = Array.isArray(data.capabilities) ? data.capabilities : [];
+  if (!capabilities.length) return '';
+  const names = capabilities.map((capability) => capability.id || capability.label).filter(Boolean);
+  return names.length ? `[Wall-E capability routes] ${names.join(', ')}` : '';
+}
+
 function readCompleteLines(filePath, { startOffset, maxBytes, partialLine, trimLeadingPartial }) {
   const fd = fs.openSync(filePath, 'r');
   const decoder = new StringDecoder('utf8');
@@ -349,7 +463,11 @@ function ingestJsonlFile(db, options = {}) {
     trimLeadingPartial: cursor.trimLeadingPartial,
   });
 
-  const state = { codexSeenUsers: new Set() };
+  const state = {
+    codexUserDeduper: createCodexUserDeduper(),
+    sourceFile: filePath,
+    sourceKind: isSubagentTranscriptPath(filePath) ? 'subagent' : '',
+  };
   const messages = [];
   for (const lineInfo of read.lines) {
     let entry;
@@ -359,13 +477,17 @@ function ingestJsonlFile(db, options = {}) {
     messages.push({ ...message, byteStart: lineInfo.byteStart, byteEnd: lineInfo.byteEnd });
   }
 
-  const nextSeqRow = db.prepare('SELECT COALESCE(MAX(seq) + 1, 0) AS nextSeq FROM transcript_events WHERE agent_session_id = ?').get(agentSessionId);
+  // Seed seq AND turn_ordinal from the durable max so a restart mid-session never
+  // resets either (crash-safe resume — turns must stay monotonic across boots).
+  const nextSeqRow = db.prepare('SELECT COALESCE(MAX(seq) + 1, 0) AS nextSeq, COALESCE(MAX(turn_ordinal), 0) AS maxTurn FROM transcript_events WHERE agent_session_id = ?').get(agentSessionId);
   let seq = Number(nextSeqRow?.nextSeq || 0);
+  let turn = Number(nextSeqRow?.maxTurn || 0);
   let inserted = 0;
   const insertEvent = db.prepare(`
     INSERT OR IGNORE INTO transcript_events
-      (ctm_session_id, agent_session_id, provider, jsonl_path, seq, role, timestamp, content, byte_start, byte_end, event_hash, metadata_json)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      (ctm_session_id, agent_session_id, provider, jsonl_path, seq, role, timestamp, content, byte_start, byte_end, event_hash, metadata_json,
+       turn_ordinal, token_count, uuid, parent_uuid, model, usage_json, content_json)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
   const insertEventFts = hasTranscriptEventsFts(db)
     ? db.prepare('INSERT INTO transcript_events_fts(rowid, content) VALUES (?, ?)')
@@ -390,8 +512,24 @@ function ingestJsonlFile(db, options = {}) {
       updated_at=datetime('now')
   `);
 
-  const txn = db.transaction(() => {
-    for (const message of messages) {
+  // Insert in bounded sub-transactions so a large catch-up ingest never holds the SQLite
+  // write lock for the whole file. One whole-file db.transaction() was observed holding the
+  // lock ~20s on the primary, bloating the WAL (no checkpoint can run mid-transaction) →
+  // reader/read-pool requests time out → callers fall back to the main thread (the bursty
+  // freezes). Each chunk is its own transaction = its own lock acquire/release, so
+  // checkpoints and readers interleave and the WAL stays bounded. INSERT OR IGNORE
+  // (event_hash) keeps re-runs idempotent, so the file cursor is advanced once at the end —
+  // a crash mid-ingest simply re-processes from the old offset without duplicating rows.
+  const CHUNK = Math.max(25, Math.min(5000, Number(options.ingestChunkSize || process.env.CTM_TRANSCRIPT_INGEST_CHUNK || 250)));
+  const insertChunk = db.transaction((slice) => {
+    for (const message of slice) {
+      // A turn = one real user prompt + the assistant answer(s)/tool calls that
+      // follow it. messageFromEntry already maps tool-results and subagent prompts
+      // to role 'system', so role==='user' is a genuine prompt → start a new turn.
+      // Assistant/system rows inherit the current turn. (Read-side turn grouping in
+      // Phase 3 keys on this; it mirrors the renderer's user-message turn boundary.)
+      if (message.role === 'user') turn++;
+      const meta = message.metadata || {};
       const hash = eventHash(agentSessionId, message);
       const info = insertEvent.run(
         ctmSessionId,
@@ -405,7 +543,14 @@ function ingestJsonlFile(db, options = {}) {
         message.byteStart || 0,
         message.byteEnd || 0,
         hash,
-        JSON.stringify(message.metadata || {})
+        JSON.stringify(meta),
+        turn,
+        estimateRowTokens(message.text),
+        String(meta.uuid || ''),
+        String(meta.parentUuid || ''),
+        String(meta.model || ''),
+        '',  // usage_json: enriched later; empty keeps the row small
+        ''   // content_json: rich content hydrated on demand from byte offsets
       );
       if (info.changes > 0) {
         if (insertEventFts) {
@@ -415,21 +560,25 @@ function ingestJsonlFile(db, options = {}) {
         seq++;
       }
     }
-    upsertFile.run(
-      filePath,
-      agentSessionId,
-      ctmSessionId,
-      provider,
-      String(stat.dev ?? ''),
-      String(stat.ino ?? ''),
-      stat.size,
-      stat.mtimeMs,
-      read.nextOffset,
-      read.partialLine || '',
-      PARSER_VERSION
-    );
   });
-  txn();
+  for (let i = 0; i < messages.length; i += CHUNK) {
+    insertChunk(messages.slice(i, i + CHUNK));
+  }
+  // Advance the file cursor only after all events are committed. INSERT OR IGNORE makes a
+  // re-run from the old offset idempotent, so a crash between chunks loses no data.
+  upsertFile.run(
+    filePath,
+    agentSessionId,
+    ctmSessionId,
+    provider,
+    String(stat.dev ?? ''),
+    String(stat.ino ?? ''),
+    stat.size,
+    stat.mtimeMs,
+    read.nextOffset,
+    read.partialLine || '',
+    PARSER_VERSION
+  );
 
   return {
     inserted,
@@ -509,7 +658,7 @@ function loadTranscriptMessages(db, ids = []) {
   if (!cleanIds.length) return [];
   const placeholders = cleanIds.map(() => '?').join(',');
   return db.prepare(`
-    SELECT ctm_session_id, agent_session_id, provider, role, content AS text, timestamp, seq, jsonl_path
+    SELECT ctm_session_id, agent_session_id, provider, role, content AS text, timestamp, seq, jsonl_path, metadata_json
       FROM transcript_events
      WHERE ctm_session_id IN (${placeholders})
         OR agent_session_id IN (${placeholders})
@@ -518,14 +667,166 @@ function loadTranscriptMessages(db, ids = []) {
        timestamp,
        agent_session_id,
        seq
-  `).all(...cleanIds, ...cleanIds).map(row => ({
-    role: row.role,
-    text: row.text,
-    timestamp: row.timestamp || '',
-    source: 'transcript-events',
-    provider: row.provider || '',
-    agentSessionId: row.agent_session_id,
-  }));
+  `).all(...cleanIds, ...cleanIds)
+    .map(loadedTranscriptMessageFromRow)
+    .filter(Boolean);
+}
+
+// ---------------------------------------------------------------------------
+// Phase 3: keyset/offset-paginated reads over transcript_events, so a session's
+// conversation is served from per-message ROWS (≤ limit materialized) instead of
+// JSON.parse-ing a multi-GB blob. Reproduces lib/message-pagination.js semantics
+// (newest-first offset; chronological within a page) so the API/response shape and
+// the renderer are unchanged. Page reads are O(limit) on the (ctm_session_id, seq)
+// / (ctm_session_id, turn_ordinal) indexes — the 28-33s blob-parse freeze is gone.
+// ---------------------------------------------------------------------------
+
+function _resolveSessionRowIds(db, ids) {
+  const list = Array.isArray(ids) ? ids : [ids];
+  const seed = list.find(Boolean) || '';
+  return transcriptSourceIds(db, seed, list);
+}
+
+// True only when the row store can serve this session: the migrated columns exist
+// AND at least one row is present for the id set. Callers fall back to the legacy
+// blob/JSONL ladder otherwise, so a half-migrated session never breaks.
+function sessionMessagesRowsAvailable(db, ids) {
+  try {
+    ensureTranscriptTables(db, { skipFtsBackfill: true });
+    const cols = db.prepare("PRAGMA table_info(transcript_events)").all().map(c => c.name);
+    if (!cols.includes('turn_ordinal')) return false;
+    const sids = _resolveSessionRowIds(db, ids);
+    if (!sids.length) return false;
+    const ph = sids.map(() => '?').join(',');
+    const row = db.prepare(`SELECT 1 FROM transcript_events WHERE ctm_session_id IN (${ph}) OR agent_session_id IN (${ph}) LIMIT 1`).get(...sids, ...sids);
+    if (!row) return false;
+    // FRESHNESS GATE: the rows must be CAUGHT UP to every source file. Stat the LIVE file
+    // (not transcript_files.size, which is itself only as fresh as the last ingest): if the
+    // file has grown past the durable ingest cursor (offset), the rows are missing its recent
+    // tail — serving them would show a stale conversation (caught a real case on a giant Codex
+    // session whose rows lagged the file by ~10h). When stale, return false so the caller uses
+    // the legacy blob/JSONL path, which reads the live file and is always complete. 64KB
+    // tolerates the trailing partial line; anything more is real lag.
+    const files = db.prepare(
+      `SELECT jsonl_path, offset FROM transcript_files WHERE ctm_session_id IN (${ph}) OR agent_session_id IN (${ph})`
+    ).all(...sids, ...sids);
+    if (!files.length) return false;
+    for (const f of files) {
+      let size = 0;
+      try { size = fs.statSync(f.jsonl_path).size; } catch { return false; } // file gone → use legacy path
+      if (size - Number(f.offset || 0) > 65536) return false;
+    }
+    return true;
+  } catch { return false; }
+}
+
+function _whereIds(sids) {
+  const ph = sids.map(() => '?').join(',');
+  return { clause: `(ctm_session_id IN (${ph}) OR agent_session_id IN (${ph}))`, args: [...sids, ...sids] };
+}
+
+const _ROW_ORDER = `CASE WHEN timestamp IS NULL OR timestamp = '' THEN 1 ELSE 0 END, timestamp, agent_session_id, seq`;
+const _ROW_COLS = `ctm_session_id, agent_session_id, provider, role, content AS text, timestamp, seq, jsonl_path, byte_start, byte_end, metadata_json, turn_ordinal`;
+
+function countSessionMessages(db, ids) {
+  const sids = _resolveSessionRowIds(db, ids);
+  if (!sids.length) return 0;
+  const w = _whereIds(sids);
+  return Number(db.prepare(`SELECT COUNT(*) AS c FROM transcript_events WHERE ${w.clause}`).get(...w.args)?.c || 0);
+}
+
+// Newest-first offset (offset 0 = the most recent `limit`), chronological within the
+// page — matches lib/message-pagination.js paginateMessages().
+function getSessionMessagesPage(db, ids, { offset = 0, limit = 200 } = {}) {
+  ensureTranscriptTables(db, { skipFtsBackfill: true });
+  const sids = _resolveSessionRowIds(db, ids);
+  if (!sids.length) return { messages: [], total: 0, has_more: false, next_offset: Number(offset) || 0 };
+  const w = _whereIds(sids);
+  const total = Number(db.prepare(`SELECT COUNT(*) AS c FROM transcript_events WHERE ${w.clause}`).get(...w.args)?.c || 0);
+  const lim = Math.max(1, Math.min(1000, Number(limit) || 200));
+  const off = Math.max(0, Number(offset) || 0);
+  const start = Math.max(0, total - off - lim);
+  const take = Math.max(0, Math.min(lim, total - off - start));
+  const raw = take === 0 ? [] : db.prepare(
+    `SELECT ${_ROW_COLS} FROM transcript_events WHERE ${w.clause} ORDER BY ${_ROW_ORDER} LIMIT ? OFFSET ?`
+  ).all(...w.args, take, start);
+  const messages = raw.map(loadedTranscriptMessageFromRow).filter(Boolean);
+  return { messages, total, has_more: start > 0, next_offset: off + take, _rows: raw };
+}
+
+// Newest-first by turn (offset 0 = the most recent `limit` turns). Returns all rows
+// belonging to the page's turns in chronological order — matches paginateMessageTurns().
+function getSessionTurnsPage(db, ids, { offset = 0, limit = 50 } = {}) {
+  ensureTranscriptTables(db, { skipFtsBackfill: true });
+  const sids = _resolveSessionRowIds(db, ids);
+  const empty = { messages: [], total: 0, has_more: false, next_offset: Number(offset) || 0, page_kind: 'turns', turn_count: 0 };
+  if (!sids.length) return empty;
+  const w = _whereIds(sids);
+  const turns = db.prepare(`SELECT DISTINCT turn_ordinal FROM transcript_events WHERE ${w.clause} ORDER BY turn_ordinal`).all(...w.args).map(r => Number(r.turn_ordinal));
+  const totalTurns = turns.length;
+  const lim = Math.max(1, Math.min(1000, Number(limit) || 50));
+  const off = Math.max(0, Number(offset) || 0);
+  const startIdx = Math.max(0, totalTurns - off - lim);
+  const endIdx = Math.max(startIdx, totalTurns - off);
+  const pageTurns = turns.slice(startIdx, endIdx);
+  if (!pageTurns.length) return { ...empty, total: totalTurns, has_more: startIdx > 0 };
+  const minT = pageTurns[0], maxT = pageTurns[pageTurns.length - 1];
+  const raw = db.prepare(
+    `SELECT ${_ROW_COLS} FROM transcript_events WHERE ${w.clause} AND turn_ordinal BETWEEN ? AND ? ORDER BY ${_ROW_ORDER}`
+  ).all(...w.args, minT, maxT);
+  const messages = raw.map(loadedTranscriptMessageFromRow).filter(Boolean);
+  return {
+    messages, total: totalTurns, has_more: startIdx > 0, next_offset: off + pageTurns.length,
+    page_kind: 'turns', turn_count: pageTurns.length, first_turn_index: minT, last_turn_index: maxT, _rows: raw,
+  };
+}
+
+// Phase 4 (existing-user backfill): enrich rows that were ingested BEFORE Phase 2 and
+// therefore have turn_ordinal=0 / token_count=0. Walks one agent session's rows in order
+// and assigns turn_ordinal (++ on each real user prompt) + a token estimate, in bounded
+// UPDATE transactions. Idempotent: only runs for a session that has user rows but no turns
+// yet (after it runs, MAX(turn_ordinal) > 0, so it never repeats). Returns rows updated.
+function enrichTranscriptRowsForSession(db, agentSessionId) {
+  ensureTranscriptTables(db, { skipFtsBackfill: true });
+  const aid = String(agentSessionId || '').trim();
+  if (!aid) return 0;
+  const stat = db.prepare(`SELECT MAX(turn_ordinal) AS maxTurn,
+    SUM(CASE WHEN role = 'user' THEN 1 ELSE 0 END) AS users
+    FROM transcript_events WHERE agent_session_id = ?`).get(aid);
+  if (!stat || Number(stat.maxTurn || 0) > 0 || Number(stat.users || 0) === 0) return 0;
+  const rows = db.prepare('SELECT id, role, content FROM transcript_events WHERE agent_session_id = ? ORDER BY seq').all(aid);
+  const upd = db.prepare('UPDATE transcript_events SET turn_ordinal = ?, token_count = ? WHERE id = ?');
+  let turn = 0, n = 0;
+  const tx = db.transaction((batch) => {
+    for (const r of batch) {
+      if (r.role === 'user') turn++;
+      upd.run(turn, estimateRowTokens(r.content), r.id);
+      n++;
+    }
+  });
+  const CHUNK = 500;
+  for (let i = 0; i < rows.length; i += CHUNK) tx(rows.slice(i, i + CHUNK));
+  return n;
+}
+
+// Find agent sessions whose rows still need enrichment (have user rows, no turns yet),
+// bounded — for a background sweep that migrates existing users without a big-bang.
+function findUnenrichedSessions(db, limit = 25) {
+  ensureTranscriptTables(db, { skipFtsBackfill: true });
+  try {
+    return db.prepare(`SELECT agent_session_id FROM transcript_events
+      GROUP BY agent_session_id
+      HAVING MAX(turn_ordinal) = 0 AND SUM(CASE WHEN role = 'user' THEN 1 ELSE 0 END) > 0
+      LIMIT ?`).all(Math.max(1, Math.min(500, Number(limit) || 25))).map(r => r.agent_session_id);
+  } catch { return []; }
+}
+
+// Session token total from per-row estimates (no full-array rebuild).
+function getSessionTokenSum(db, ids) {
+  const sids = _resolveSessionRowIds(db, ids);
+  if (!sids.length) return 0;
+  const w = _whereIds(sids);
+  return Number(db.prepare(`SELECT COALESCE(SUM(token_count), 0) AS t FROM transcript_events WHERE ${w.clause}`).get(...w.args)?.t || 0);
 }
 
 function mergeMessageLists(baseMessages, overlayMessages) {
@@ -543,13 +844,7 @@ function mergeMessageLists(baseMessages, overlayMessages) {
   };
   for (const message of Array.isArray(baseMessages) ? baseMessages : []) add(message);
   for (const message of Array.isArray(overlayMessages) ? overlayMessages : []) add(message);
-  merged.sort((a, b) => {
-    const at = String(a.timestamp || '');
-    const bt = String(b.timestamp || '');
-    if (at !== bt) return at.localeCompare(bt);
-    return String(a.role || '').localeCompare(String(b.role || ''));
-  });
-  return merged;
+  return sortTimelineMessages(merged);
 }
 
 function catchUpTranscriptFiles(db, options = {}) {
@@ -641,7 +936,17 @@ module.exports = {
   markTranscriptFileError,
   mergeMessageLists,
   messageFromEntry,
+  isSubagentTranscriptPath,
   normalizeProvider,
   sourceIdFromPath,
   transcriptSourceIds,
+  // Phase 3 row-store readers
+  sessionMessagesRowsAvailable,
+  countSessionMessages,
+  getSessionMessagesPage,
+  getSessionTurnsPage,
+  getSessionTokenSum,
+  // Phase 4 existing-user enrichment backfill
+  enrichTranscriptRowsForSession,
+  findUnenrichedSessions,
 };

package/template/claude-task-manager/lib/transport-security.js

@@ -94,11 +94,59 @@ function trustedForwardedHost(req) {
   );
 }
 
+function cleanIpHeaderValue(value) {
+  let raw = stripHeaderQuotes(firstHeaderValue(value));
+  if (!raw || raw.toLowerCase() === 'unknown') return '';
+  if (raw.startsWith('[')) {
+    const end = raw.indexOf(']');
+    raw = end >= 0 ? raw.slice(1, end) : stripIpv6Brackets(raw);
+  } else if (net.isIP(raw) === 0) {
+    const colonCount = (raw.match(/:/g) || []).length;
+    if (colonCount === 1 && /^\d+\.\d+\.\d+\.\d+:\d+$/.test(raw)) {
+      raw = raw.replace(/:\d+$/, '');
+    }
+  }
+  raw = stripIpv6Brackets(raw);
+  return net.isIP(raw) ? raw : '';
+}
+
+function forwardedClientIp(req) {
+  const forwarded = parseForwardedHeader(req?.headers?.forwarded || '');
+  for (const value of [
+    req?.headers?.['x-forwarded-for'],
+    req?.headers?.['x-real-ip'],
+    req?.headers?.['cf-connecting-ip'],
+    forwarded.for,
+  ]) {
+    const ip = cleanIpHeaderValue(value);
+    if (ip) return ip;
+  }
+  return '';
+}
+
+function requestClientIp(req) {
+  const direct = stripIpv6Brackets(req?.socket?.remoteAddress || '');
+  if (!isLoopbackAddress(direct)) return direct;
+  const tunnelHost = trustedForwardedHost(req) || cleanHostHeaderValue(req?.headers?.host);
+  if (!isManagedHttpsTunnelHost(tunnelHost)) return direct;
+  const forwardedIp = forwardedClientIp(req);
+  if (forwardedIp && !isLoopbackAddress(forwardedIp)) return forwardedIp;
+  const tunnelName = hostHeaderName(tunnelHost);
+  return tunnelName ? `tunnel:${tunnelName}` : direct;
+}
+
 function isLoopbackAddress(address) {
   const h = stripIpv6Brackets(address).toLowerCase();
   return LOOPBACK_HOSTS.has(h) || h === '::ffff:127.0.0.1';
 }
 
+function primaryProcessIpLockoutExemptions(primaryHost) {
+  const out = new Set(['127.0.0.1', '::1']);
+  const host = stripIpv6Brackets(primaryHost).trim();
+  if (host && !isLoopbackHost(host)) out.add(host);
+  return Array.from(out);
+}
+
 function hasNonLoopbackBrowserOrigin(req) {
   for (const value of [req?.headers?.origin, req?.headers?.referer, req?.headers?.referrer]) {
     const raw = firstHeaderValue(value);
@@ -112,12 +160,45 @@ function hasNonLoopbackBrowserOrigin(req) {
   return false;
 }
 
+// Proxy/tunnel forwarding headers a genuinely local browser or CLI never sends.
+// A devtunnel (or any reverse proxy) forwards remote traffic to CTM over the
+// loopback interface, so socket.remoteAddress alone cannot tell local apart from
+// tunneled. Azure Dev Tunnels and every standard proxy stamp X-Forwarded-* /
+// Forwarded on the request; their presence on a loopback peer means the request
+// originated off-box and must NOT be trusted as loopback admin.
+const PROXY_FORWARDING_HEADERS = Object.freeze([
+  'x-forwarded-for',
+  'x-forwarded-host',
+  'x-forwarded-proto',
+  'forwarded',
+  'x-real-ip',
+  'x-ms-original-host',
+  'x-original-host',
+  'cf-connecting-ip',
+]);
+
+function hasProxyForwardingMarkers(req) {
+  const headers = req?.headers || {};
+  for (const name of PROXY_FORWARDING_HEADERS) {
+    const value = headers[name];
+    if (value !== undefined && value !== null && String(value).trim() !== '') return true;
+  }
+  return false;
+}
+
+// SECURITY INVARIANT: loopback === fully-trusted local admin, but ONLY for
+// requests that are provably on-box. Trust requires ALL of: (1) a loopback socket
+// peer, (2) no proxy/tunnel forwarding markers, (3) no non-loopback browser
+// Origin/Referer, and (4) a loopback (or absent) Host. A managed tunnel forwards
+// remote clients to 127.0.0.1, so any signal that the request originated off-box
+// means it must authenticate with a device token like any other remote client.
 function isLoopbackRequest(req) {
   if (!isLoopbackAddress(req?.socket?.remoteAddress || '')) return false;
+  if (hasProxyForwardingMarkers(req)) return false;
   const forwardedHost = trustedForwardedHost(req);
   if (forwardedHost && !isLoopbackHost(hostHeaderName(forwardedHost))) return false;
-  const host = hostHeaderName(req?.headers?.host || '');
   if (hasNonLoopbackBrowserOrigin(req)) return false;
+  const host = hostHeaderName(req?.headers?.host || '');
   return !host || isLoopbackHost(host);
 }
 
@@ -296,6 +377,8 @@ module.exports = {
   normalizeBindHost,
   originAllowed,
   parseAllowedOrigins,
+  primaryProcessIpLockoutExemptions,
+  requestClientIp,
   requestBrowserOrigin,
   requestOrigin,
   requestProtocol,