@bongos/core 1.21.174 → 1.21.176

package/scripts/gds/backfill-sessions.js

@@ -1,411 +1,7 @@
 #!/usr/bin/env node
+// scripts/gds/backfill-sessions.js — @bongos-forwarder (task 1004848). Generated by
+// scripts/bongos/codemod-rename-scripts-gds.js; do not edit.
+// The real script is scripts/bongos/backfill-sessions.js. This forwarder keeps callers that still name
+// the old path working, and warns once so they get repointed. Task 1003706 removes it.
 'use strict';
-
-// Backfill the BFG session corpus (session_records) from this machine's local
-// Claude Code transcripts (ADR 0027 / cluster 6D). 6D.1 only began uploading
-// session digests on /builder-ship from 2026-06-02; every session before that —
-// the V1 game build, the PMS builds, the art pipeline, early GDS-V3 — never made
-// it into the corpus the BFG searches. The raw transcripts still live under
-// ~/.claude/projects/<encoded-cwd>/<session-id>.jsonl, one file per session.
-//
-// This script reuses the EXACT digest path the live upload uses
-// (buildSessionDigest from ship.js — same scrubbing, same metrics, same caps),
-// so a backfilled row is indistinguishable from one written at ship time. The
-// only things we reconstruct rather than read from the environment are:
-//   • session_id  — the transcript filename (Claude Code names it that).
-//   • task_ids    — extracted from high-precision in-transcript ACTION markers
-//                   (a task that was claimed/shipped/released IN this session),
-//                   never generic "#N" links (CLAUDE.md is injected every session
-//                   and is full of them). The server's filterTaskIdsClaimedBy is
-//                   a final safety net — it drops any id the caller never claimed.
-//
-// SAFETY / IDEMPOTENCY: the POST /sessions upsert ACCUMULATES drachmae_earned on
-// conflict (each ship is expected to upload exactly once). So re-uploading a
-// session already in the corpus would double-count its denormalised drachmae.
-// We therefore SKIP any session_id already present. The dedup set is the
-// authoritative list of existing session_records.session_id (pull it from prod;
-// the read API caps /sessions/mine at 100 and has no by-session_id lookup, so a
-// file is the reliable source). Pass it via --existing=<file> (one id per line).
-//
-// Owner is forced server-side to the authenticated caller, so this can only ever
-// backfill the CALLER'S OWN sessions — exactly right here, since every local
-// transcript on this machine is the operator's. Other builders backfill their
-// own machines.
-//
-// Usage:
-//   node scripts/gds/backfill-sessions.js                 # dry-run (default)
-//   node scripts/gds/backfill-sessions.js --existing=/tmp/ids.txt --apply
-//   node scripts/gds/backfill-sessions.js --apply --limit=50   # cap uploads
-//
-// Backs the GDS task that claimed this work (no change ships without a claim).
-
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-
-const { cliClient, loadSession } = require('./cli-lib');
-const { buildSessionDigest, encodeProjectPath } = require('./ship');
-const { looksLikeGraderTranscript } = require('./session-cost');
-
-// ---- args ----------------------------------------------------------------
-const ARGV = process.argv.slice(2);
-const APPLY = ARGV.includes('--apply');
-const argVal = (name) => {
-  const hit = ARGV.find((a) => a.startsWith(`--${name}=`));
-  return hit ? hit.slice(name.length + 3) : null;
-};
-const EXISTING_FILE = argVal('existing') || '/tmp/otb-existing-session-ids.txt';
-const LIMIT = argVal('limit') ? parseInt(argVal('limit'), 10) : Infinity;
-// The /api/bongos write limiter is per-IP 30 writes / 60s (the binding constraint;
-// per-session is 60/60s). 2100ms spacing keeps us safely under 30/min so the
-// limiter never trips during a long backfill. Override with --pace=<ms>.
-const PACE_MS = argVal('pace') ? parseInt(argVal('pace'), 10) : 2100;
-// On a 429 (limiter tripped — e.g. a concurrent session also writing), wait a
-// full window before retrying the SAME item, since apiCall doesn't surface the
-// Retry-After header.
-const RATE_LIMIT_BACKOFF_MS = 61 * 1000;
-const PROJECTS_DIR = path.join(os.homedir(), '.claude', 'projects');
-
-// Project-path families this project has lived under (the cwd is encoded into
-// the dir name by Claude Code; the project was renamed over its life). A dir is
-// in-scope if its name contains any of these tokens.
-//
-// The first token is DERIVED from this checkout, not hardcoded: the legacy names
-// below are the owner's old machine, so on every other clone inScope() matched
-// nothing and the script reported "0 transcripts" while exiting 0 — the failure
-// looked like success. encodeProjectPath is the same encoder Claude Code (and
-// resolveTranscriptPath) uses, so the derived token IS the real dir name, and a
-// per-task worktree dir (<encoded-root>--claude-worktrees-task-N) still matches
-// on the prefix. The legacy tokens stay so the owner's older corpus keeps working.
-const REPO_ROOT = path.resolve(__dirname, '..', '..');
-const PROJECT_TOKENS = [
-  encodeProjectPath(REPO_ROOT),
-  'Offtheboats',
-  'Claude-OTB',
-  'LarsCode-Example',
-  'Claude-LarsCode-Primea',
-];
-
-// ---- task-id extraction (high precision) ---------------------------------
-// Only markers that mean "work on task N HAPPENED in this session": a ship
-// invocation, a claim/release slash command, or the builder-claim skill's
-// "Claimed #N" confirmation. NOT markdown "#N" links, NOT API list responses
-// (those carry many incidental task_id fields), NOT "/tasks/N/..." paths.
-// Ids are 7 digits on a live instance (1002471); the original {1,4} matched none
-// of them, so every modern session extracted zero task_ids and could never pay.
-const TASK_MARKERS = [
-  /\bship\.js\s+(\d{1,7})\b/g, // node scripts/gds/ship.js 1002471 [--flags]
-  /\bbuilder-claim\s+(\d{1,7})\b/g, // /builder-claim 1002471
-  /\bbuilder-release\s+(\d{1,7})\b/g, // /builder-release 1002471
-  /\bClaimed\s+(?:task\s+)?#(\d{1,7})\b/g, // "Claimed #1002471" / "Claimed task #1002471"
-];
-
-// Decode the human-and-tool text of one transcript entry: assistant text +
-// tool_use inputs + tool_result content. This is where the action markers live.
-function entryText(entry) {
-  const msg = entry && entry.message;
-  if (!msg) return '';
-  const c = msg.content;
-  if (typeof c === 'string') return c;
-  if (!Array.isArray(c)) return '';
-  const out = [];
-  for (const p of c) {
-    if (!p || typeof p !== 'object') continue;
-    if (p.type === 'text' && typeof p.text === 'string') out.push(p.text);
-    else if (p.type === 'tool_use' && p.input) out.push(JSON.stringify(p.input));
-    else if (p.type === 'tool_result') {
-      const tc = p.content;
-      if (typeof tc === 'string') out.push(tc);
-      else if (Array.isArray(tc)) for (const q of tc) if (q && typeof q.text === 'string') out.push(q.text);
-    }
-  }
-  return out.join('\n');
-}
-
-function extractTaskIds(entries) {
-  const found = new Set();
-  for (const e of entries) {
-    const t = entryText(e);
-    if (!t) continue;
-    for (const re of TASK_MARKERS) {
-      re.lastIndex = 0;
-      let m;
-      while ((m = re.exec(t)) !== null) {
-        const n = parseInt(m[1], 10);
-        // No upper clamp: the old `n < 10000` dropped every 7-digit id a SECOND
-        // time, so widening TASK_MARKERS alone would still have extracted nothing.
-        //
-        // Over-capture cannot pay for work you did not do. The id set here is a
-        // REQUEST; the server intersects it with the caller's own claims before
-        // storing — modules/sessions/routes/sessions.js calls
-        // db.filterTaskIdsClaimedBy(req.builder.id, requestedTaskIds), which is
-        // `SELECT DISTINCT task_id FROM claims WHERE builder_id = $1 AND task_id =
-        // ANY($2)` (modules/sessions/db.js). builder_id comes from the authenticated
-        // caller, never the body, so a forged transcript naming an unclaimed task
-        // yields an EMPTY task_ids — and the cost-plus reward is gated on a stored
-        // id reaching status='shipped' (ADR 0120), so it can never book.
-        // Confirmed empirically on the 2026-07-28 recovery: ids 2, 5, 600, 2615 and
-        // 999999 were extracted from quoted docs and every one was dropped server-side.
-        if (n > 0) found.add(n);
-      }
-    }
-  }
-  return [...found].sort((a, b) => a - b);
-}
-
-// buildUploadPayload — the POST /sessions body for one planned session. Pure +
-// exported so the field set is unit-tested; the money rides on `model_usage`.
-//
-// model_usage MUST be top-level. The route promotes it to its own column and
-// deletes it from the metrics jsonb (modules/sessions/routes/sessions.js), so it
-// reads body.model_usage and never looks inside metrics. Omitting it left every
-// backfilled reward on the task-1676 fallback, which prices the whole token total
-// at the opus cache_read rate — deliberately conservative, i.e. an under-payment.
-// Sent exactly as ship.js sends it so a backfilled row prices identically to one
-// written at ship time.
-function buildUploadPayload(p) {
-  const digest = (p && p.digest) || {};
-  const metrics = digest.metrics || {};
-  return {
-    session_id: p.sessionId,
-    task_ids: p.taskIds,
-    total_tokens: digest.totalTokens,
-    duration_seconds: digest.durationSeconds,
-    started_at: digest.startedAt,
-    ended_at: digest.endedAt,
-    detail: digest.detail,
-    metrics,
-    model_usage: metrics.model_usage || {},
-  };
-}
-
-// ---- transcript discovery + parse ----------------------------------------
-function inScope(dirName) {
-  return PROJECT_TOKENS.some((tok) => dirName.includes(tok));
-}
-
-function listTranscripts() {
-  let dirs;
-  try {
-    dirs = fs.readdirSync(PROJECTS_DIR, { withFileTypes: true });
-  } catch (e) {
-    console.error(`Cannot read ${PROJECTS_DIR}: ${e.message}`);
-    process.exit(1);
-  }
-  const files = [];
-  for (const d of dirs) {
-    if (!d.isDirectory() || !inScope(d.name)) continue;
-    const abs = path.join(PROJECTS_DIR, d.name);
-    let inner;
-    try {
-      inner = fs.readdirSync(abs);
-    } catch (_) {
-      continue;
-    }
-    for (const f of inner) {
-      if (f.endsWith('.jsonl')) files.push(path.join(abs, f));
-    }
-  }
-  return files;
-}
-
-function parseEntries(file) {
-  const raw = fs.readFileSync(file, 'utf8');
-  const entries = [];
-  for (const line of raw.split('\n')) {
-    const t = line.trim();
-    if (!t) continue;
-    try {
-      entries.push(JSON.parse(t));
-    } catch (_) {
-      /* skip malformed line — best-effort, same as ship.js */
-    }
-  }
-  return entries;
-}
-
-function loadExisting() {
-  if (!fs.existsSync(EXISTING_FILE)) {
-    console.error(
-      `Dedup file not found: ${EXISTING_FILE}\n` +
-        `Pull it first (authoritative source — the read API caps at 100):\n` +
-        `  ssh lars@<droplet> 'psql -d example -tAc "SELECT session_id FROM session_records;"' > ${EXISTING_FILE}`
-    );
-    process.exit(1);
-  }
-  const set = new Set();
-  for (const line of fs.readFileSync(EXISTING_FILE, 'utf8').split('\n')) {
-    const t = line.trim();
-    if (t) set.add(t);
-  }
-  return set;
-}
-
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-
-async function main() {
-  const existing = loadExisting();
-  const files = listTranscripts();
-  console.log(
-    `Found ${files.length} transcripts in scope; ${existing.size} session_ids already in corpus.`
-  );
-
-  // Dynamic secrets to scrub from snippets (same as ship.js): live GDS token +
-  // Anthropic key. Defense in depth — snippets are already content-light.
-  const extraSecrets = [];
-  const s = await loadSession();
-  if (s && s.token) extraSecrets.push(s.token);
-  if (process.env.ANTHROPIC_API_KEY) extraSecrets.push(process.env.ANTHROPIC_API_KEY);
-
-  // Never backfill the session we're RUNNING IN, nor any session still active:
-  // those will ship themselves via /builder-ship, and POST /sessions ACCUMULATES
-  // drachmae on conflict — so a backfilled-then-shipped session double-counts its
-  // drachmae facet. The running session is also a poisoned source (its transcript
-  // may quote other sessions' "Claimed #N" output, e.g. while developing this
-  // very script). We can reliably exclude the current session; truly-concurrent
-  // OTHER sessions are the operator's call (run the backfill when idle for a
-  // complete, conflict-free sweep).
-  const currentSessionId = process.env.CLAUDE_CODE_SESSION_ID || null;
-
-  const plan = [];
-  let skippedPresent = 0;
-  let skippedEmpty = 0;
-  let skippedActive = 0;
-  let skippedGrader = 0;
-  for (const file of files) {
-    const sessionId = path.basename(file, '.jsonl');
-    if (!/^[A-Za-z0-9_-]+$/.test(sessionId)) continue; // server format guard
-    if (currentSessionId && sessionId === currentSessionId) {
-      skippedActive++;
-      continue;
-    }
-    if (existing.has(sessionId)) {
-      skippedPresent++;
-      continue;
-    }
-    // The ship-time subagent grader panel persists each worker as its OWN
-    // top-level transcript in this same project dir (they are the MAJORITY of
-    // files here — 36 of 52 in one real main dir, 146 of 151 across worktrees).
-    // A worker reads the diff, so it quotes the very ship markers above and would
-    // be uploaded as a session that shipped a real task — earning a cost-plus
-    // reward the live ship path never pays, for spend already captured by the
-    // worker cost envelope. Same exclusion, same reason as session-cost.js's
-    // ADR 0028 non-double-count invariant; reuse its detector.
-    if (looksLikeGraderTranscript(file)) {
-      skippedGrader++;
-      continue;
-    }
-    const entries = parseEntries(file);
-    const digest = buildSessionDigest(entries, { extraSecrets });
-    // Skip sessions with no real LLM work. A 0-token digest is either an aborted
-    // start OR a failed/empty subagent spawn — e.g. a grader worker that 429'd
-    // writes ONE synthetic assistant turn ("You've hit your limit") with no
-    // usage, so the assistant_turns guard alone let 98 of them into the corpus.
-    // total_tokens === 0 catches both; the server enforces the same rule (POST
-    // /sessions returns {skipped:'empty'}), this just avoids the wasted request.
-    if (!digest.metrics || !(digest.totalTokens > 0)) {
-      skippedEmpty++;
-      continue;
-    }
-    const taskIds = extractTaskIds(entries);
-    plan.push({ sessionId, file, digest, taskIds });
-  }
-
-  // Largest / most task-laden first is irrelevant; sort by start time for a
-  // readable, chronological log.
-  plan.sort((a, b) => String(a.digest.startedAt || '').localeCompare(String(b.digest.startedAt || '')));
-
-  const totalTasks = plan.reduce((n, p) => n + p.taskIds.length, 0);
-  console.log(
-    `\nPlan: ${plan.length} new sessions to upload ` +
-      `(${skippedPresent} already present, ${skippedEmpty} empty/no-turns skipped, ` +
-      `${skippedGrader} grader-worker transcripts skipped` +
-      `${skippedActive ? `, ${skippedActive} current-session skipped` : ''}).`
-  );
-  console.log(`Task-id associations extracted: ${totalTasks} across ${plan.filter((p) => p.taskIds.length).length} sessions.\n`);
-
-  if (!APPLY) {
-    // Dry-run: show a sample so the operator can eyeball before applying.
-    const sample = plan.slice(0, 12);
-    for (const p of sample) {
-      console.log(
-        `  ${p.digest.startedAt ? p.digest.startedAt.slice(0, 10) : '????-??-??'} ` +
-          `${p.sessionId.slice(0, 8)}  turns=${p.digest.metrics.assistant_turns}  ` +
-          `tok=${p.digest.totalTokens}  errs=${p.digest.metrics.tool_error_count}  ` +
-          `tasks=[${p.taskIds.join(',')}]`
-      );
-    }
-    if (plan.length > sample.length) console.log(`  … and ${plan.length - sample.length} more.`);
-    console.log(`\nDRY RUN — no uploads. Re-run with --apply to write to the corpus.`);
-    return;
-  }
-
-  // Apply: serial POSTs paced under the per-IP write limiter (30/60s). On a 429
-  // we back off a full window and retry the SAME item, so a concurrent writer
-  // tripping the limiter costs time, never a dropped session.
-  let ok = 0;
-  let failed = 0;
-  const failures = [];
-  const capped = Math.min(plan.length, LIMIT);
-  const api = await cliClient();
-  for (let i = 0; i < capped; i++) {
-    const p = plan[i];
-    const payload = buildUploadPayload(p);
-    let attempts = 0;
-    for (;;) {
-      attempts++;
-      let r;
-      try {
-        r = await api.sessions.postSessions({ body: payload });
-      } catch (e) {
-        failed++;
-        failures.push({ sessionId: p.sessionId, err: e.message });
-        break;
-      }
-      if (r.ok) {
-        ok++;
-        if (ok % 50 === 0) console.log(`  … ${ok} uploaded`);
-        break;
-      }
-      if (r.status === 401) {
-        console.error(`  401 unauthenticated — re-auth with /builder-reauth and retry. Stopping.`);
-        failed++;
-        failures.push({ sessionId: p.sessionId, status: 401 });
-        return finish(ok, failed, failures);
-      }
-      if (r.status === 429 && attempts <= 5) {
-        console.log(`  429 rate-limited at #${i} — backing off ${RATE_LIMIT_BACKOFF_MS / 1000}s (attempt ${attempts}).`);
-        await sleep(RATE_LIMIT_BACKOFF_MS);
-        continue; // retry SAME item
-      }
-      // Any other non-OK (or 429 past retry budget): record and move on.
-      failed++;
-      failures.push({ sessionId: p.sessionId, status: r.status, err: r.data && r.data.error });
-      break;
-    }
-    await sleep(PACE_MS);
-  }
-
-  return finish(ok, failed, failures);
-}
-
-function finish(ok, failed, failures) {
-  console.log(`\nDone. Uploaded ${ok}, failed ${failed}.`);
-  if (failures.length) {
-    console.log('Failures (first 15):');
-    for (const f of failures.slice(0, 15)) console.log('  ', JSON.stringify(f));
-  }
-}
-
-// Pure helpers are exported for unit testing (matches the ship.js / session-eval
-// convention). The script only executes when run directly, not when required.
-module.exports = {
-  entryText, extractTaskIds, inScope, buildUploadPayload, TASK_MARKERS, PROJECT_TOKENS,
-};
-
-if (require.main === module) {
-  main().catch((e) => {
-    console.error('backfill-sessions fatal:', e);
-    process.exit(1);
-  });
-}
+require('../bongos/forwarder.js').forward(module, 'backfill-sessions.js');