@hncode/hncode 0.4.1 → 0.5.0

package/src/session.js

@@ -92,6 +92,13 @@ export function convRowFor(m) {
 // `"messages"` key, so a session whose updatedAt sits after the transcript
 // would report updatedAt = 0 and the session list / --continue would sort
 // wrongly (or pick the wrong session). Key order here is authoritative.
+//
+// `transcript` goes LAST — after `messages`, not merely after the metadata block.
+// It is the one value here with no size limit: every persisted reasoning block is
+// stored whole, and the head reader gives up after META_MAX_BYTES. Written ahead of
+// `messages` a multi-megabyte `transcript` pushed the key out of reach and the session
+// became unlistable, which is what made intact conversations vanish from /sessions.
+// Behind `messages` the reader never has to cross it.
 function serializableSession(session) {
   const out = {};
   out.id = session.id;
@@ -100,15 +107,16 @@ function serializableSession(session) {
   out.model = session.model || '';
   out.createdAt = session.createdAt || Date.now();
   out.updatedAt = Date.now();
-  // Everything else (messages, rounds, steps, plan, focus, swarm, effort, …)
-  // is copied AFTER updatedAt. `messages` in particular must come after the
-  // metadata block.
+  // Everything else (rounds, steps, plan, focus, swarm, effort, …) is copied AFTER
+  // updatedAt. `messages` and `transcript` in particular must come after the metadata
+  // block, and they are appended below in that order.
   for (const k of Object.keys(session)) {
     if (k in out) continue;
-    if (k === 'messages') continue;   // handled below, last
+    if (k === 'messages' || k === 'transcript') continue;
     out[k] = session[k];
   }
   out.messages = Array.isArray(session.messages) ? session.messages : [];
+  if (Array.isArray(session.transcript)) out.transcript = session.transcript;
   return out;
 }
 
@@ -122,23 +130,43 @@ export function saveSession(session, storeDir) {
     // FIX: unique temp name so two concurrent saves of the same session cannot
     // clobber each other's temp file mid-write.
     const tmp = `${target}.${process.pid}.${crypto.randomBytes(3).toString('hex')}.tmp`;
-    const json = JSON.stringify(serializableSession(session), null, 2);
+    // Compact JSON, no indentation. `null, 2` made the file ~30% larger for a
+    // human who never reads it (the reader is JSON.parse), and the pretty-printer
+    // itself costs real time on a large transcript — measured ~22ms for 5.7MB, all
+    // of it spent inserting whitespace. Saving is on the per-turn path.
+    const json = JSON.stringify(serializableSession(session));
     fs.writeFileSync(tmp, json, 'utf8');
     // Atomic replace. If this throws (EXDEV etc.) the temp is cleaned up below.
     fs.renameSync(tmp, target);
-    // The saved transcript changed, so its cached metadata head is stale. Drop
-    // just this entry; the next scan re-reads the file (cheap, one file).
     cacheFor(storeDir, false)?.delete(sessionFile(session.id, storeDir));
+    // A save that worked clears any earlier failure, so /save-report speaks about the
+    // CURRENT state rather than a problem the user already resolved by being told.
+    saveFailure = null;
   } catch (e) {
     // FIX: never fail silently. Previously an empty catch {} in the TUI meant a
     // serialization error (circular ref, disk full, permissions) lost the turn
     // with no signal at all.
-    logSessionError('save', `${session.id}: ${e && e.message ? e.message : String(e)}`);
+    const detail = e && e.message ? e.message : String(e);
+    logSessionError('save', `${session.id}: ${detail}`);
+    // The log alone was not enough. A session whose transcript had grown past the V8
+    // string limit failed EVERY save for an hour and a half — the turn was lost each
+    // time and all the file recorded was `Invalid string length`, in a log nobody opens.
+    // The flag is what lets the TUI say so out loud; this module never reaches into the UI.
+    if (!saveFailure) saveFailure = { at: Date.now(), id: session.id, detail };
     return session;
   }
   return session;
 }
 
+// The most recent save failure, or null. Set once per failure so a caller can report it
+// (the TUI shows a warning) without this module reaching into the UI.
+let saveFailure = null;
+
+/** The last save failure `{ at, id, detail }`, or null when saving has been clean. */
+export function lastSaveFailure() { return saveFailure; }
+/** Clear the recorded failure, after a save has succeeded. */
+export function clearSaveFailure() { saveFailure = null; }
+
 export function loadSession(id, storeDir) {
   const file = sessionFile(id, storeDir);
   let text;
@@ -288,6 +316,21 @@ function _readMetaOnce(file) {
         delete m.messages;
         return { meta: m, hasMessages: nonEmpty };
       } catch {
+        // Neither parse can work here, and the reason is structural rather than
+        // corrupt: the cap was hit before the `"messages"` key, so `text` stops
+        // inside some earlier value. That is what a session carrying a large
+        // `transcript` looked like while `serializableSession` wrote it ahead of
+        // `messages` — and returning null for it made the file UNLISTABLE, so a
+        // perfectly intact session silently disappeared from /sessions and from
+        // `--continue`. The metadata is all at the front and every field a listing
+        // needs is a top-level scalar, so read those directly instead of giving up.
+        const meta = scanMetaHead(text);
+        // `hasMessages` is taken as true here on purpose. Reaching the cap without
+        // seeing the `"messages"` key means some earlier value was megabytes long, and
+        // the only value that grows like that is a transcript — which a session only
+        // accumulates by having had a conversation. Reading it as empty instead would
+        // make `--continue --skipEmpty` pass over a session full of real work.
+        if (meta) return { meta, hasMessages: true };
         return null;
       }
     }
@@ -311,6 +354,133 @@ function findMessagesValueStart(text) {
   return i;   // may point at `[` (or be end-of-text; caller handles both)
 }
 
+// The metadata a LISTING needs, read out of a head that stopped mid-value.
+//
+// Only top-level keys are taken, and only while the scanner is at depth 1: a `"title"`
+// nested inside a todo or a transcript row must never be mistaken for the session's own.
+// The values pulled out are strings and numbers, so the scan can step over an oversized
+// value with `skipJsonValue` instead of trying to read it.
+const HEAD_SCALAR_KEYS = new Set([
+  'id', 'title', 'workspace', 'model', 'createdAt', 'updatedAt',
+  'rounds', 'steps', 'lastTurnMs', 'forkedFrom',
+]);
+
+function scanMetaHead(text) {
+  const out = {};
+  let depth = 0;
+  let i = 0;
+  while (i < text.length) {
+    const c = text[i];
+    if (c === '{' || c === '[') { depth++; i++; continue; }
+    if (c === '}' || c === ']') { depth--; i++; continue; }
+    if (c !== '"') { i++; continue; }
+    const key = readJsonString(text, i);
+    if (!key) return Object.keys(out).length ? out : null;
+    i = key.end;
+    // Only a KEY of the root object counts, and only when a colon follows.
+    if (depth !== 1) {
+      const skip = skipJsonValue(text, i);
+      if (skip < 0) return Object.keys(out).length ? out : null;
+      i = skip;
+      continue;
+    }
+    let j = i;
+    while (j < text.length && /\s/.test(text[j])) j++;
+    if (text[j] !== ':') {
+      const skip = skipJsonValue(text, i);
+      if (skip < 0) return Object.keys(out).length ? out : null;
+      i = skip;
+      continue;
+    }
+    j++;
+    while (j < text.length && /\s/.test(text[j])) j++;
+    if (HEAD_SCALAR_KEYS.has(key.value)) {
+      if (text[j] === '"') {
+        const s = readJsonString(text, j);
+        if (!s) return Object.keys(out).length ? out : null;
+        out[key.value] = s.value;
+        i = s.end;
+      } else {
+        let k = j;
+        while (k < text.length && !',}]\n'.includes(text[k])) k++;
+        const n = Number(text.slice(j, k).trim());
+        if (Number.isFinite(n)) out[key.value] = n;
+        i = k;
+      }
+      continue;
+    }
+    const skip = skipJsonValue(text, j);
+    if (skip < 0) return Object.keys(out).length ? out : null;
+    i = skip;
+  }
+  return Object.keys(out).length ? out : null;
+}
+
+// A JSON string literal starting at `at` (which must be the opening quote), decoded.
+// Returns { value, end } with `end` past the closing quote, or null when the literal
+// runs off the end of `text` — the normal case for a value the head cap cut in half.
+function readJsonString(text, at) {
+  let i = at + 1;
+  let raw = '';
+  while (i < text.length) {
+    const c = text[i];
+    if (c === '\\') {
+      const n = text[i + 1];
+      if (n === undefined) return null;
+      if (n === 'u') {
+        const hex = text.slice(i + 2, i + 6);
+        if (hex.length < 4 || !/^[0-9a-fA-F]{4}$/.test(hex)) return null;
+        raw += String.fromCharCode(parseInt(hex, 16));
+        i += 6;
+        continue;
+      }
+      raw += n === 'n' ? '\n' : n === 't' ? '\t' : n === 'r' ? '\r' : n;
+      i += 2;
+      continue;
+    }
+    if (c === '"') return { value: raw, end: i + 1 };
+    raw += c;
+    i++;
+  }
+  return null;
+}
+
+// Step over one JSON value starting at `at`, returning the index just past it, or -1
+// when the value is not complete in `text`. Strings and containers are both handled, so
+// an oversized array is crossed without building it.
+function skipJsonValue(text, at) {
+  let i = at;
+  while (i < text.length && /\s/.test(text[i])) i++;
+  if (i >= text.length) return -1;
+  const c = text[i];
+  if (c === '"') {
+    const s = readJsonString(text, i);
+    return s ? s.end : -1;
+  }
+  if (c === '{' || c === '[') {
+    let depth = 0;
+    for (; i < text.length; i++) {
+      const ch = text[i];
+      if (ch === '"') {
+        const s = readJsonString(text, i);
+        if (!s) return -1;
+        i = s.end - 1;
+        continue;
+      }
+      if (ch === '{' || ch === '[') depth++;
+      else if (ch === '}' || ch === ']') {
+        depth--;
+        if (depth === 0) return i + 1;
+      }
+    }
+    return -1;
+  }
+  // A literal (number, true/false/null): up to the next delimiter.
+  let k = i;
+  while (k < text.length && !',}]'.includes(text[k])) k++;
+  return k > i ? k : -1;
+}
+
 // Non-empty detection now lives in _readMetaOnce (it reads the file head once
 // for both the metadata and the emptiness rule), so the old separate
 // hasMessages() scan is gone — latestSession no longer reopens each candidate.