@bongos/core 1.19.317 → 1.19.319

package/scripts/gds/go-live.js

@@ -76,8 +76,9 @@ function isSafeToken(v) {
 // The fields a target must carry, and the ones it may. Unknown keys are reported
 // rather than ignored: a typo'd `healthURL` that silently disables the health gate
 // is precisely the class of quiet failure this pipeline keeps being bitten by.
-const REQUIRED_FIELDS = ['ssh', 'instanceDir', 'service', 'healthUrl', 'versionUrl', 'deployTimer'];
-const OPTIONAL_FIELDS = ['deployService', 'npmEnvFile', 'runAs', 'backupDb', 'backupDir', 'env', 'registryPackage'];
+// `ssh` is NOT required: a target with no ssh host is LOCAL (the box driving itself).
+const REQUIRED_FIELDS = ['instanceDir', 'service', 'healthUrl', 'versionUrl', 'deployTimer'];
+const OPTIONAL_FIELDS = ['ssh', 'local', 'deployService', 'npmEnvFile', 'runAs', 'backupDb', 'backupDir', 'env', 'registryPackage'];
 
 function validateTarget(name, t) {
   const errors = [];
@@ -100,6 +101,10 @@ function validateTarget(name, t) {
       continue;
     }
     if (!known.has(k)) continue;
+    if (k === 'local') { // a flag, never interpolated into a command
+      if (typeof v !== 'boolean') errors.push(`target "${name}".local must be true or false`);
+      continue;
+    }
     if (!isSafeToken(v)) errors.push(`target "${name}".${k} contains characters that are unsafe in a shell (got ${JSON.stringify(v)})`);
   }
   return errors;
@@ -166,8 +171,32 @@ function selectTarget(config, name) {
 // the remote login shell). Values are validated above; this is the second layer.
 // ---------------------------------------------------------------------------
 
+// Privilege preamble for any phase that touches systemd.
+//
+// Over ssh these phases run as ROOT, so `systemctl stop …` just works — which is what
+// they originally assumed. Run LOCALLY by the nightly lane they run as the instance
+// OWNER, where the same line fails with "Interactive authentication required" and the
+// timer is never stopped: the pull-deploy `git reset --hard` then stays armed for the
+// whole upgrade, which is the exact hazard the stop exists to remove. Deciding at RUN
+// time keeps one script correct for both callers.
+const SUDO_PREAMBLE = 'SUDO=""; [ "$(id -u)" -eq 0 ] || SUDO="sudo -n"';
+
+// A target is LOCAL when it names no ssh host: the phases run through a local shell
+// instead of a remote one. Everything else — the phases, the halts, the reporting —
+// is byte-identical, which is the point: one deployment engine, reachable from a
+// laptop (remote) or from the box itself (local, how the nightly lane uses it).
+// Without this the unattended path would need its own second implementation, and the
+// two would drift — which is exactly how the nightly deploy ended up with weaker
+// checks than the manual one (task 1003219).
+function isLocalTarget(target) {
+  return !target || !target.ssh;
+}
+
 function remote(target, script, { run = spawnSync, timeoutMs = 900000 } = {}) {
-  const res = run('ssh', ['-o', 'ConnectTimeout=20', '-o', 'BatchMode=yes', target.ssh, 'bash', '-s'], {
+  const [cmd, args] = isLocalTarget(target)
+    ? ['bash', ['-s']]
+    : ['ssh', ['-o', 'ConnectTimeout=20', '-o', 'BatchMode=yes', target.ssh, 'bash', '-s']];
+  const res = run(cmd, args, {
     input: script, encoding: 'utf8', timeout: timeoutMs, maxBuffer: 32 * 1024 * 1024,
   });
   if (!res || res.error) return { ok: false, status: null, stdout: '', stderr: res && res.error ? res.error.message : 'spawn failed' };
@@ -177,36 +206,47 @@ function remote(target, script, { run = spawnSync, timeoutMs = 900000 } = {}) {
 // `sudo -u <user> -H env K=V … <cmd>` — the shape every runbook entry uses. Returns
 // a string safe to embed because every component was allowlist-validated.
 //
-// `forward` names variables that exist in the calling root shell and must reach the
-// child — NPM_TOKEN above all. sudo's env_reset drops it, and an explicit `env K=V`
-// list that omits it drops it too, so `npm install` 404s the PRIVATE package and the
-// upgrade auto-rolls-back, which reads as "the release is broken" when it is only
-// "the token never arrived". Sourcing the env file in the OUTER shell is necessary
-// and NOT sufficient.
+// A privileged child needs npm auth, and there is NO WAY to hand it a secret through
+// sudo without logging it (task 1003218, corrected).
+//
+//   `env NPM_TOKEN="$NPM_TOKEN" …`     → sudo logs it in COMMAND=
+//   `sudo --preserve-env=NPM_TOKEN …`  → sudo logs it in ENV=
 //
-// A FORWARDED VARIABLE IS A SECRET, SO IT MUST NOT ENTER argv (task 1003218).
-// `env K="$K"` keeps the value out of the script TEXT, but the shell EXPANDS it
-// before sudo runs — and sudo logs the full argv. That is how 26 lines of
-// `COMMAND=... NPM_TOKEN=npm_<real token> ...` ended up in the droplet's PERSISTENT
-// journal between 2026-08-17 and 2026-08-23. Passing by reference is NOT the same as
-// keeping a secret out of argv; only `--preserve-env` does that, handing the variable
-// over through the environment where nothing logs it. Verified side by side on the
-// box: both forms reach the child, only this one leaves no value in the journal.
+// The first was the original bug; the second was my "fix", which only MOVED the value
+// to a different field of the same log line. Both put a live credential in a
+// persistent journal. What matters is that sudo logs its whole invocation, so the
+// secret must never be part of it.
+//
+// So the child DERIVES the token itself, from a file it can already read. Only the
+// `sed` appears in argv; the value never does. This is the shape docs/recipes/
+// core-release-pipeline.md already documents as the safe one. The guard matters as
+// much as the derivation: an EMPTY token silently sends npm to the public registry,
+// which 404s the private package, which auto-rolls-back "cleanly" having taught you
+// nothing — so an empty read aborts loudly instead of half-running.
+//
+// (The nightly lane never takes this path: it runs under systemd as the owner with an
+// EnvironmentFile, no sudo, and leaks nothing. This is the manual/agent path only.)
+const NPM_AUTH_PREAMBLE = [
+  'export NPM_TOKEN="${NPM_TOKEN:-$(sed -n \'s/.*_authToken=//p\' "$HOME/.npmrc" 2>/dev/null | head -1)}"',
+  '[ -n "$NPM_TOKEN" ] || { echo "go-live: EMPTY NPM_TOKEN — npm would 404 the private core and roll back; aborting" >&2; exit 12; }',
+].join('; ');
+
+// `sudo -u <user> -H env K=V … <cmd>` — the shape every runbook entry uses. Returns
+// a string safe to embed because every component was allowlist-validated.
 //
-// The plain `env K=V` list is still used for NON-secret values (PGDATABASE, HOME) —
-// those are configuration, and seeing them in a log is useful rather than harmful.
-function asOwner(target, cmd, { forward = [] } = {}) {
+// `needsNpm` wraps the command so it derives its own npm auth (see above) instead of
+// having a secret handed across the privilege boundary. Only NON-secret configuration
+// (PGDATABASE, HOME) travels in the plain `env K=V` list — seeing those in a log is
+// useful rather than harmful.
+function asOwner(target, cmd, { needsNpm = false } = {}) {
+  const inner = needsNpm ? `sh -c ${shq(`${NPM_AUTH_PREAMBLE}; ${cmd}`)}` : cmd;
   const envPairs = Object.entries(target.env || {}).map(([k, v]) => `${k}=${v}`);
-  const preserve = forward.filter((k) => SAFE_ENV_KEY_RE.test(k)); // never interpolate an unvetted name
   if (!target.runAs) {
-    // No privilege drop: the child already inherits our environment, so a forwarded
-    // secret needs no mention at all.
     const joined = envPairs.join(' ');
-    return `${joined ? `env ${joined} ` : ''}${cmd}`.trim();
+    return `${joined ? `env ${joined} ` : ''}${inner}`.trim();
   }
-  const keep = preserve.length ? `--preserve-env=${preserve.join(',')} ` : '';
   const joined = [...envPairs, `HOME=/home/${target.runAs}`].join(' ');
-  return `sudo ${keep}-u ${target.runAs} -H env ${joined} ${cmd}`.trim();
+  return `sudo -u ${target.runAs} -H env ${joined} ${inner}`.trim();
 }
 
 // POSIX single-quote a value for embedding in a shell command.
@@ -264,6 +304,12 @@ function parseMarkerHits(raw) {
   return Number(first);
 }
 
+// Does this process sit inside a git repo that knows the given ref?
+function gitRefExists(ref, { run = spawnSync } = {}) {
+  const r = run('git', ['rev-parse', '--verify', '--quiet', ref], { cwd: ROOT, encoding: 'utf8' });
+  return !!r && !r.error && r.status === 0;
+}
+
 // Refresh origin/main before reading anything from it.
 //
 // deploy-divergence reads CORE_VERSION at the local `origin/main` REF, and a ref is
@@ -283,13 +329,6 @@ function refreshMain({ run = spawnSync } = {}) {
   return { ok: true };
 }
 
-// Which variables an npm-running command needs forwarded. Only when the target
-// declares an env file to source them from — otherwise `K="$K"` would expand to
-// empty and quietly do nothing.
-function npmForward(target) {
-  return target.npmEnvFile ? ['NPM_TOKEN'] : [];
-}
-
 // Parse a `key=value` marker into its parts. Kept pure + exported: the marker is
 // the answer to the trap that the release current when your fix merges usually
 // does NOT contain your fix (ADR 0161 carriers bump on their own cadence).
@@ -435,13 +474,13 @@ function probeRemote(target, targetVersion, marker, opts = {}) {
 set -uo pipefail
 ${src}
 echo "##REGISTRY##"
-${asOwner(target, `npm view ${pkg}@${targetVersion} version`, { forward: npmForward(target) })} 2>&1 | tail -1
+${asOwner(target, `npm view ${pkg}@${targetVersion} version`, { needsNpm: true })} 2>&1 | tail -1
 echo "##TREE##"
 ( cd ${target.instanceDir} && ${target.runAs ? `sudo -u ${target.runAs} -H ` : ''}git status --porcelain | head -20 )
 echo "##INSTALLED##"
 ${asOwner(target, `node -e "console.log(require('${target.instanceDir}/node_modules/${pkg}/package.json').version)"`)} 2>&1 | tail -1
 ${marker ? `echo "##MARKER##"
-${asOwner(target, `sh -c ${shq(markerProbe(pkg, targetVersion, marker))}`, { forward: npmForward(target) })}` : ''}
+${asOwner(target, `sh -c ${shq(markerProbe(pkg, targetVersion, marker))}`, { needsNpm: true })}` : ''}
 echo "##END##"
 `;
   const res = remote(target, script, opts);
@@ -479,7 +518,8 @@ function applyGoLive(target, targetName, targetVersion, from, opts = {}) {
     : 'echo "(no backupDb configured — skipping)"';
   const phaseA = remote(target, `
 set -euo pipefail
-echo "##TIMER##"; systemctl stop ${target.deployTimer} && systemctl is-active ${target.deployTimer} || true
+${SUDO_PREAMBLE}
+echo "##TIMER##"; $SUDO systemctl stop ${target.deployTimer} && $SUDO systemctl is-active ${target.deployTimer} || true
 echo "##BACKUP##"; ${backup}
 echo "##END##"
 `, opts);
@@ -499,7 +539,7 @@ echo "##END##"
   const phaseB = remote(target, `
 set -uo pipefail
 ${src}
-${asOwner(target, `bash -c "cd ${target.instanceDir} && ${upgrade}"`, { forward: npmForward(target) })}
+${asOwner(target, `bash -c "cd ${target.instanceDir} && ${upgrade}"`, { needsNpm: true })}
 `, opts);
   const rolledBack = /rolled back/i.test(phaseB.stdout + phaseB.stderr);
   steps.push({
@@ -510,7 +550,7 @@ ${asOwner(target, `bash -c "cd ${target.instanceDir} && ${upgrade}"`, { forward:
   if (!phaseB.ok || rolledBack) {
     // Restart the timer even on failure — leaving it stopped silently disables
     // the instance's pull-deploy until someone notices.
-    remote(target, `systemctl start ${target.deployTimer} || true`, opts);
+    remote(target, `${SUDO_PREAMBLE}\n$SUDO systemctl start ${target.deployTimer} || true`, opts);
     return { ok: false, steps, from, to: targetVersion, targetName, failedAt: 'UPGRADE', rolledBack, raw: phaseB };
   }
 
@@ -519,10 +559,11 @@ ${asOwner(target, `bash -c "cd ${target.instanceDir} && ${upgrade}"`, { forward:
   // not evidence; 127.0.0.1 is.
   const phaseC = remote(target, `
 set -uo pipefail
-echo "##TIMERSTART##"; systemctl start ${target.deployTimer}; systemctl is-active ${target.deployTimer}
+${SUDO_PREAMBLE}
+echo "##TIMERSTART##"; $SUDO systemctl start ${target.deployTimer}; $SUDO systemctl is-active ${target.deployTimer}
 echo "##PIN##"; ( cd ${target.instanceDir} && ${target.runAs ? `sudo -u ${target.runAs} -H ` : ''}git status --porcelain -- package.json package-lock.json )
 echo "##HEAD##"; ( cd ${target.instanceDir} && ${target.runAs ? `sudo -u ${target.runAs} -H ` : ''}git log -1 --format='%h %s' )
-${target.deployService ? `echo "##TICK##"; systemctl start ${target.deployService} || true; journalctl -u ${target.deployService} -n 20 --no-pager | tail -8` : ''}
+${target.deployService ? `echo "##TICK##"; $SUDO systemctl start ${target.deployService} || true; $SUDO journalctl -u ${target.deployService} -n 20 --no-pager | tail -8` : ''}
 echo "##VERSION##"; curl -s --max-time 20 ${target.versionUrl} | head -c 400
 echo ""
 echo "##INSTALLED##"; ${asOwner(target, `node -e "console.log(require('${target.instanceDir}/node_modules/${pkg}/package.json').version)"`)} 2>&1 | tail -1
@@ -563,6 +604,8 @@ go-live — move an enrolled instance's core pin, as one command.
   node scripts/gds/go-live.js --config <path>     default: ${DEFAULT_CONFIG}
   node scripts/gds/go-live.js --allow-unreleased  downgrade UNRELEASED_ON_MAIN to a warning
   node scripts/gds/go-live.js --no-fetch          skip the origin/main refresh (default: fetch)
+  node scripts/gds/go-live.js --target-json '{…}' pass a target inline instead of a config file
+  node scripts/gds/go-live.js --no-divergence     skip the git-derived checks (needs --to)
   node scripts/gds/go-live.js --json              machine-readable
 
 Host topology is NOT baked into the core. ${DEFAULT_CONFIG} (host-owned, not published):
@@ -578,6 +621,24 @@ async function main() {
   const apply = hasFlag('--apply');
   const allowUnreleased = hasFlag('--allow-unreleased');
 
+  // An inline target lets a CALLER drive this without inventing a second host-owned
+  // config file beside the one it already has. The nightly lane already holds a roster
+  // entry; making it also maintain config/go-live.json would be two sources of truth
+  // for one box, which is how they drift.
+  const inline = arg('--target-json');
+  if (inline) {
+    let parsed;
+    try { parsed = JSON.parse(inline); }
+    catch (e) { console.error(`  go-live: --target-json is not valid JSON — ${e.message}`); return 2; }
+    const errors = validateTarget('inline', parsed);
+    if (errors.length) {
+      if (json) { console.log(JSON.stringify({ ok: false, halt: 'CONFIG_INVALID', detail: errors.join('; ') }, null, 2)); return 1; }
+      console.error(`\n  go-live: CONFIG_INVALID — ${errors.join('; ')}\n`);
+      return 1;
+    }
+    return runGoLive({ target: parsed, targetName: arg('--target') || 'inline', configPath: '--target-json', json, apply, allowUnreleased });
+  }
+
   const cfg = loadConfig({ configPath: arg('--config') });
   if (cfg.error) {
     if (json) { console.log(JSON.stringify({ ok: false, halt: cfg.error, path: cfg.path, detail: cfg.detail }, null, 2)); return 1; }
@@ -595,7 +656,10 @@ async function main() {
     return 1;
   }
   const { target, name: targetName } = sel;
+  return runGoLive({ target, targetName, configPath: cfg.path, json, apply, allowUnreleased });
+}
 
+async function runGoLive({ target, targetName, configPath, json, apply, allowUnreleased }) {
   const marker = parseMarker(arg('--verify-marker'));
   if (marker && marker.error) { console.error(`  go-live: ${marker.error}`); return 2; }
 
@@ -603,18 +667,40 @@ async function main() {
   let targetVersion = arg('--to');
   if (targetVersion && !SEMVER_RE.test(targetVersion)) { console.error(`  go-live: --to "${targetVersion}" is not a version`); return 2; }
 
-  const fetched = hasFlag('--no-fetch') ? { ok: false, skipped: true } : refreshMain();
-  const div = await divergence.check({ base: arg('--url') });
+  // Is there a core GIT CHECKOUT to reason about at all?
+  //
+  // Every git-derived signal — the target read from origin/main, the UNRELEASED halt,
+  // the dirty-checkout halt — presumes this process is running inside a core repo.
+  // The nightly lane is NOT: it runs on the box, against an installed package, and it
+  // resolves its target from the REGISTRY under a channel policy instead. Running the
+  // git checks there would halt every sweep on a condition that does not apply.
+  //
+  // So the git legs are SKIPPED only when both are true: there is no resolvable
+  // origin/main, AND the caller named the version explicitly. A missing checkout with
+  // no --to is still an error — silently inventing a target is exactly the class of
+  // guess this script exists to refuse.
+  // "Am I in a SOURCE checkout?" is `.git` AT OUR ROOT — not "is there a git repo
+  // somewhere above me". The installed core package lives inside the INSTANCE's repo
+  // and ships its own src/module-api.js, so a naive `git rev-parse HEAD` succeeds
+  // there and CORE_VERSION resolves — against the instance repo's main, which is a
+  // meaningless answer delivered confidently. That is the failure this check avoids.
+  const hasCheckout = fs.existsSync(path.join(ROOT, '.git'));
+  const gitless = (!hasCheckout || hasFlag('--no-divergence')) && !!targetVersion;
+
+  const fetched = (gitless || hasFlag('--no-fetch')) ? { ok: false, skipped: true } : refreshMain();
+  const div = gitless ? null : await divergence.check({ base: arg('--url') });
   if (!targetVersion) targetVersion = div && div.onMain;
-  if (targetVersion && !SEMVER_RE.test(targetVersion)) {
-    console.error(`  go-live: could not resolve a target version (read ${JSON.stringify(targetVersion)} from main)`);
+  if (!targetVersion || !SEMVER_RE.test(targetVersion)) {
+    console.error(`  go-live: could not resolve a target version${hasCheckout ? ` (read ${JSON.stringify(targetVersion)} from main)` : ' — this is not a core source checkout, so pass --to <version>'}`);
     return 1;
   }
 
-  const dirtyRes = spawnSync('git', ['status', '--porcelain'], { cwd: ROOT, encoding: 'utf8' });
+  const dirtyRes = gitless ? { status: 1 } : spawnSync('git', ['status', '--porcelain'], { cwd: ROOT, encoding: 'utf8' });
   const dirtyCheckout = dirtyRes.status === 0 && (dirtyRes.stdout || '').trim() ? (dirtyRes.stdout || '').trim().split('\n').slice(0, 5).join(', ') : null;
 
-  const plan = planGoLive({ div, target, targetVersion, dirtyCheckout, allowUnreleased });
+  const plan = gitless
+    ? { halts: [], notes: [{ code: 'NO_CORE_CHECKOUT', message: `no core git checkout here — target ${targetVersion} was given explicitly, so the git-derived checks (unreleased-on-main, dirty tree) are skipped` }], from: null, targetVersion, target }
+    : planGoLive({ div, target, targetVersion, dirtyCheckout, allowUnreleased });
   plan.marker = marker;
   if (!fetched.ok && !fetched.skipped) {
     // Say so rather than silently planning off a ref of unknown age.
@@ -626,6 +712,14 @@ async function main() {
   if (!plan.halts.length) {
     const probe = probeRemote(target, targetVersion, marker);
     plan.registry = probe.registry;
+    // With no source checkout there is no git-derived `from`; the installed version on
+    // the box IS the truth, and the probe just read it.
+    if (gitless && probe.installed) {
+      plan.from = probe.installed;
+      if (probe.installed === targetVersion) {
+        plan.halts.push({ code: 'ALREADY_LIVE', message: HALTS.ALREADY_LIVE, detail: `installed and target are both ${targetVersion}` });
+      }
+    }
     if (!probe.reachable) {
       plan.halts.push({ code: 'BOX_UNREACHABLE', message: `could not reach ${target.ssh}`, detail: (probe.raw.stderr || '').trim().split('\n').pop() });
     } else {
@@ -656,7 +750,7 @@ async function main() {
 
   if (!apply || plan.halts.length) {
     if (json) { console.log(JSON.stringify({ ok: !plan.halts.length, mode: apply ? 'halted' : 'plan', targetName, ...plan }, null, 2)); }
-    else { console.log(renderPlan(plan, { targetName, configPath: cfg.path })); }
+    else { console.log(renderPlan(plan, { targetName, configPath })); }
     return plan.halts.length ? 1 : 0;
   }
 
@@ -675,6 +769,6 @@ if (require.main === module) {
 
 module.exports = {
   loadConfig, selectTarget, validateTarget, planGoLive, parseMarker, parseMarkerHits,
-  renderPlan, renderReport, failureExcerpt, asOwner, isSafeToken, npmForward, refreshMain,
+  renderPlan, renderReport, failureExcerpt, asOwner, isSafeToken, NPM_AUTH_PREAMBLE, refreshMain, isLocalTarget, gitRefExists, SUDO_PREAMBLE,
   CONFIG_TEMPLATE, HALTS, SEMVER_RE,
 };

package/scripts/gds/run-unit-tests.js

@@ -104,7 +104,14 @@ const INTEGRATION = new Set([
   'government_board_membership_db',
   'government_fold_bench', 'government_live_proof', 'linkify_refs', 'memory_isolation',
   'memory_pull', 'needs_rebase', 'parallel-load', 'profile_rollup_consent', 'reward_mode_cost_plus_only',
-  'scouting_rollup_fanout', 'session_records', 'setup_smoke',
+  'scouting_rollup_fanout', 'session_records',
+  // task 1003217: the cost-row RAISE against real SQL. Every claim in it IS the
+  // SQL — an ON CONFLICT DO UPDATE guarded to move a row only upward, and a
+  // revisit predicate comparing cost_log.recorded_at to the digest upload. The
+  // DB-free siblings (session_records, session_cost) cover the transcript
+  // de-duplication half.
+  'session_cost_row_raise',
+  'setup_smoke',
   'status-probe',
 ]);
 

package/scripts/gds/session-cost.js

@@ -146,9 +146,19 @@ function roundTiers(t) {
 }
 
 // Sum usage per model across the transcript, then price each model's total.
+//
+// task 1003217 — usage is deduped by `message.id`. Claude Code writes one
+// transcript line per content block of an assistant message (thinking / text /
+// tool_use) and repeats the SAME `message.usage` on every one of them, so
+// summing per line priced a single API call two or three times (measured 2.22x
+// inflation over 12 real transcripts). The evidence — including the cache-chain
+// identity that proves a line-group is one call — is written up above
+// buildSessionDigest in ./session-digest-build.js. A transcript whose messages
+// carry no `id` degrades to the old per-line behaviour.
 function priceTranscript(transcriptPath) {
   const raw = fs.readFileSync(transcriptPath, 'utf8');
   const perModel = new Map(); // model -> {input, output, read, c5, c1h}
+  const seenMessageIds = new Set();
   for (const line of raw.split('\n')) {
     if (!line.trim()) continue;
     let o;
@@ -159,6 +169,12 @@ function priceTranscript(transcriptPath) {
     // generated messages (compaction, interrupts) — no real API call, usage is
     // a zero placeholder. Pricing them would mislabel them as an unknown model.
     if (!usage || !msg.model || msg.model === '<synthetic>') continue;
+    // One API call, many lines — price it once (task 1003217).
+    const msgId = typeof msg.id === 'string' && msg.id ? msg.id : null;
+    if (msgId !== null) {
+      if (seenMessageIds.has(msgId)) continue;
+      seenMessageIds.add(msgId);
+    }
     const m = perModel.get(msg.model) || { input: 0, output: 0, read: 0, c5: 0, c1h: 0 };
     m.input += Number(usage.input_tokens) || 0;
     m.output += Number(usage.output_tokens) || 0;

package/scripts/gds/session-digest-build.js

@@ -0,0 +1,261 @@
+// scripts/gds/session-digest-build.js — the PURE transcript→digest builder.
+//
+// Extracted from scripts/gds/ship.js (task 1003217, serving the de-monolith goal
+// of task 1003203: nothing over 1,500 lines). ship.js was 4,646 lines and this
+// block — snippetOf + buildSessionDigest and their two constants — is pure,
+// separately tested, and shared: ship.js uploads a digest mid-ship and
+// session-digest.js uploads the final one on the SessionEnd seam.
+//
+// It has NO dependency on ship.js, which is what makes the split safe: ship.js
+// imports and re-exports this, so its public surface (and tests/session_records.mjs)
+// is unchanged, and the existing session-digest.js -> ship.js import cannot become
+// a cycle.
+//
+// NB the prose above deliberately says "imports", not the r-e-q-u-i-r-e call
+// spelled with './ship' — the fitness require-graph (scripts/gds/fitness-ratchets.js
+// requireGraph) scans raw file text and does NOT strip comments, so naming that
+// call here invented a phantom session-digest-build <-> ship cycle and tripped the
+// require_cycle_count ratchet. Same trap the curated-set parser in run-unit-tests.js
+// warns about for quoted words in ITS comments.
+//
+// The transcript-grade secret scrubber + its ADR-0022 internals live in
+// modules/security/secret-scrub.js — extracted (task 1211 / GDS-V4 R72) so the
+// public-mirror export pipeline can reuse the ONE copy of the secret shapes
+// without requiring ship.js (whose grader / Discord / sound / session deps are
+// absent in a bare CI job). See that module for the recall-first design (a missed
+// secret is a compromise; over-redaction is cosmetic).
+const { scrubSecrets } = require('../../modules/security/secret-scrub');
+
+// snippetOf — short, single-line, scrubbed excerpt for footgun context.
+function snippetOf(text, extraSecrets, max = 160) {
+  if (typeof text !== 'string' || text.length === 0) return undefined;
+  // Drop NULs first: Postgres rejects a literal NUL in jsonb, so ONE of them anywhere in
+  // ONE snippet made the whole digest unstorable — POST /sessions answered 500 and
+  // the session never paid. UTF-16LE tool output (`wsl --list` is the observed
+  // case) is full of them, and \s below does NOT match NUL, so it must go here.
+  const flat = text.replace(/\0/g, '').replace(/\s+/g, ' ').trim();
+  if (!flat) return undefined;
+  const cut = flat.length > max ? flat.slice(0, max) : flat;
+  const scrubbed = scrubSecrets(cut, extraSecrets);
+  return scrubbed || undefined;
+}
+
+// buildSessionDigest — turn parsed Claude Code transcript entries into
+// { startedAt, endedAt, durationSeconds, totalTokens, metrics, detail }.
+// Pure (no I/O) + exported for unit tests. `detail` is per-turn metadata + short
+// scrubbed snippets, capped at maxTurns (head+tail kept when over).
+// Harness guardrail retries (task 959) — an is_error tool_result that is NOT real
+// friction but the agent harness rejecting a tool call: an Edit/Write before the
+// file was Read ("File has not been read yet"), or a foreground sleep ("Blocked:
+// sleep"). These were counted in tool_error_count, inflating it and skewing the
+// BFG's order=tool_errors ranking (top sessions ranked on benign retries). They
+// now go into a separate harness_retry_count so tool_error_count stays NET — and
+// because the sessions/search ordering reads metrics->>'tool_error_count'
+// (src/bongos/db.js), that ordering automatically uses the net figure with no route
+// change. (ADR 0027 lineage — the 6D session evaluator.)
+const HARNESS_RETRY_RE =
+  /File has not been read yet|Read it first before (?:writing|editing)|Blocked:\s*sleep|sleep is blocked in (?:the )?foreground/i;
+
+// task 1676 — the family bucket unlabeled token-bearing turns are folded into so
+// the cost-plus reward (ADR 0054) prices the real spend instead of rounding to $0.
+// A concrete, priceable model id (llm-pricing.js maps it to the opus family by
+// substring) — NOT a fabricated dollar figure; it carries the REAL token counts
+// ship.js already parsed, priced server-side like any other bucket.
+const UNATTRIBUTED_MODEL = 'claude-opus-4-8';
+
+// task 1003217 — ONE API CALL IS MANY TRANSCRIPT LINES.
+//
+// Claude Code writes a separate transcript line per content block of a single
+// assistant message (thinking / text / tool_use), and EVERY one of those lines
+// repeats the SAME `message.usage` object. Summing usage per line therefore
+// counted one API call's tokens two or three times: measured 2.22x inflation
+// across 12 real transcripts in this project (2,357 usage-bearing lines,
+// naive cache_read 551,095,738 vs deduped 248,555,962).
+//
+// `message.id` is the API call's identity — the usage tuples are byte-identical
+// across lines sharing an id. The proof that a line-GROUP is one call is the
+// cache chain: cache_read[n] == cache_read[n-1] + cache_creation[n-1] closes on
+// 99.1% of links after de-duplication (the residue is compaction boundaries)
+// and holds for none of them before it.
+//
+// This mattered because ADR 0054 pays round(true_cost_usd x 1.20) from these
+// very numbers, so the inflation flowed into cost-plus rewards.
+//
+// `assistant_turns` is deliberately left as the LINE count (existing consumers
+// order on it); `api_calls` is the new honest step count. A transcript whose
+// messages carry no `id` degrades to exactly the old behaviour.
+function buildSessionDigest(entries, opts = {}) {
+  const extra = opts.extraSecrets || [];
+  const MAX_TURNS = opts.maxTurns || 4000;
+  const seenMessageIds = new Set();
+  const m = {
+    assistant_turns: 0,
+    api_calls: 0, // distinct message.id — the real number of API round-trips
+    user_turns: 0,
+    tool_use_count: 0,
+    tool_result_count: 0,
+    tool_error_count: 0, // NET real errors (harness guardrail retries excluded)
+    harness_retry_count: 0, // benign harness guardrail retries (task 959)
+    tokens: { input: 0, output: 0, cache_read: 0, cache_creation: 0 },
+    model_usage: {}, // per-model token buckets for the cost-plus reward (task 1005)
+    // task 1676: tokens from turns with NO model id (not '<synthetic>'). Folded
+    // into a priceable bucket after the loop, then stripped from the digest.
+    _unlabeled: { input: 0, output: 0, cache_read: 0, cache_creation: 0 },
+  };
+  const detail = [];
+  let firstTs = null;
+  let lastTs = null;
+  let i = 0;
+  for (const e of Array.isArray(entries) ? entries : []) {
+    if (!e || typeof e !== 'object') continue;
+    const ts = typeof e.timestamp === 'string' ? e.timestamp : null;
+    if (ts) {
+      if (!firstTs) firstTs = ts;
+      lastTs = ts;
+    }
+    const msg = e.message;
+    if (e.type === 'assistant' && msg) {
+      m.assistant_turns++;
+      // Dedup usage per API call (see the note above buildSessionDigest). A
+      // repeat line contributes ZERO tokens but still gets its detail row, so
+      // the per-turn record stays faithful and summing detail stays correct.
+      const msgId = typeof msg.id === 'string' && msg.id ? msg.id : null;
+      const dupUsage = msgId !== null && seenMessageIds.has(msgId);
+      if (msgId !== null) seenMessageIds.add(msgId);
+      if (!dupUsage) m.api_calls++;
+      const u = dupUsage ? {} : (msg.usage || {});
+      const tin = u.input_tokens || 0;
+      const tout = u.output_tokens || 0;
+      const tcr = u.cache_read_input_tokens || 0;
+      const tcc = u.cache_creation_input_tokens || 0;
+      m.tokens.input += tin;
+      m.tokens.output += tout;
+      m.tokens.cache_read += tcr;
+      m.tokens.cache_creation += tcc;
+      // task 1005: per-model bucket for the cost-plus token reward, reusing the
+      // tin/tout/tcr/tcc already parsed above (no second pass). Skip synthetic
+      // turns (a grader 429 etc. emits no real model id AND no real spend).
+      const model = msg.model;
+      if (model === '<synthetic>') {
+        // synthetic: no real spend — exclude from BOTH model_usage and the
+        // task-1676 unlabeled-remainder fold below (don't price phantom tokens).
+      } else if (model) {
+        const mu = m.model_usage[model] || (m.model_usage[model] =
+          { input: 0, output: 0, cache_read: 0, cache_creation: 0 });
+        mu.input += tin; mu.output += tout; mu.cache_read += tcr; mu.cache_creation += tcc;
+      } else {
+        // task 1676: a token-bearing turn with NO model id. Previously dropped
+        // from model_usage (→ total_tokens>0 but priced $0 on the dev-box/CI
+        // path; ADR 0097). Accumulate the unlabeled tokens so the digest can
+        // attribute them to a real, priceable family bucket after the loop.
+        m._unlabeled.input += tin; m._unlabeled.output += tout;
+        m._unlabeled.cache_read += tcr; m._unlabeled.cache_creation += tcc;
+      }
+      const tools = [];
+      let textLen = 0;
+      let firstText = '';
+      if (Array.isArray(msg.content)) {
+        for (const c of msg.content) {
+          if (c && c.type === 'tool_use') {
+            m.tool_use_count++;
+            tools.push(String(c.name || '?'));
+          } else if (c && c.type === 'text' && typeof c.text === 'string') {
+            textLen += c.text.length;
+            if (!firstText) firstText = c.text;
+          }
+        }
+      }
+      const row = { i, role: 'assistant', ts, in: tin, out: tout, cache_read: tcr, cache_creation: tcc };
+      // Marks a line that shares its API call with an earlier one, so a reader
+      // can tell a genuinely-zero-usage turn from a de-duplicated repeat.
+      if (dupUsage) row.dup = 1;
+      if (tools.length) row.tools = tools;
+      if (textLen) row.text_len = textLen;
+      const snip = snippetOf(firstText, extra);
+      if (snip) row.snippet = snip;
+      detail.push(row);
+      i++;
+    } else if (e.type === 'user' && msg) {
+      m.user_turns++;
+      let results = 0;
+      let errors = 0;
+      let harnessRetries = 0;
+      let firstResult = '';
+      if (Array.isArray(msg.content)) {
+        for (const c of msg.content) {
+          if (c && c.type === 'tool_result') {
+            results++;
+            m.tool_result_count++;
+            const text = typeof c.content === 'string'
+              ? c.content
+              : Array.isArray(c.content)
+                ? c.content.map((p) => (p && typeof p.text === 'string' ? p.text : '')).join(' ')
+                : '';
+            if (c.is_error) {
+              // Split benign harness guardrail retries out of the net error count
+              // so order=tool_errors ranks on real friction (task 959).
+              if (HARNESS_RETRY_RE.test(text)) {
+                harnessRetries++;
+                m.harness_retry_count++;
+              } else {
+                errors++;
+                m.tool_error_count++;
+              }
+            }
+            if (!firstResult) firstResult = text;
+          }
+        }
+      }
+      const row = { i, role: 'user', ts };
+      if (results) row.tool_results = results;
+      if (errors) row.tool_errors = errors;
+      if (harnessRetries) row.harness_retries = harnessRetries;
+      const snip = snippetOf(firstResult, extra);
+      if (snip) row.snippet = snip;
+      detail.push(row);
+      i++;
+    }
+  }
+  m.total_tokens = m.tokens.input + m.tokens.output + m.tokens.cache_read + m.tokens.cache_creation;
+  // task 1676 — cost-plus reward zero-out fix. The cost-plus session reward
+  // (ADR 0054) prices `model_usage` server-side. model_usage is keyed on msg.model
+  // and SKIPS any token-bearing turn whose model id is missing. On the dev-box/CI
+  // agent path that gap was the difference between total_tokens>0 (so the digest
+  // passes the route's empty-session guard + gets stored) and model_usage={} (so
+  // priceModelUsage returns $0 → 0 drachmae) — ADR 0097's "upload no priced model
+  // usage" symptom. Fold the unlabeled remainder (NOT synthetic turns) into a real,
+  // priceable family bucket so the cost-plus reward reflects the true spend rather
+  // than silently dropping to $0. opus is the conservative family for an unlabeled
+  // turn (matches llm-pricing.js's unknown-model fallback) — this instance runs
+  // opus by default, and pricing an unlabeled turn as opus never UNDER-pays.
+  const un = m._unlabeled;
+  if (un.input + un.output + un.cache_read + un.cache_creation > 0) {
+    const fb = m.model_usage[UNATTRIBUTED_MODEL] || (m.model_usage[UNATTRIBUTED_MODEL] =
+      { input: 0, output: 0, cache_read: 0, cache_creation: 0 });
+    fb.input += un.input; fb.output += un.output;
+    fb.cache_read += un.cache_read; fb.cache_creation += un.cache_creation;
+  }
+  delete m._unlabeled; // scratch — never persisted to the digest
+  let boundedDetail = detail;
+  if (detail.length > MAX_TURNS) {
+    const head = Math.floor(MAX_TURNS * 0.7);
+    const tail = MAX_TURNS - head;
+    boundedDetail = detail.slice(0, head).concat(detail.slice(detail.length - tail));
+    m.detail_truncated = true;
+    m.detail_total_turns = detail.length;
+  }
+  let durationSeconds = null;
+  if (firstTs && lastTs) {
+    const d = (Date.parse(lastTs) - Date.parse(firstTs)) / 1000;
+    if (Number.isFinite(d) && d >= 0) durationSeconds = Math.round(d);
+  }
+  return {
+    startedAt: firstTs,
+    endedAt: lastTs,
+    durationSeconds,
+    totalTokens: m.total_tokens,
+    metrics: m,
+    detail: boundedDetail,
+  };
+}
+module.exports = { snippetOf, buildSessionDigest, HARNESS_RETRY_RE, UNATTRIBUTED_MODEL };