@bongos/core 1.21.175 → 1.21.177

package/scripts/bongos/onboarding-analyze.js

@@ -0,0 +1,226 @@
+#!/usr/bin/env node
+// Passive onboarding-friction analysis (V3.R79 / #298, criterion C1).
+//
+// Reads each spooled onboarding transcript, asks an Opus subagent to flag
+// moments of confusion (long unresolved responses, repeated prompts, error
+// recovery), and files one idea_inbox row per flagged session — tagged with
+// onboarding.ONBOARDING_FRICTION_TAG so triage can find them. "Passive" means
+// no human in the loop: it's kicked detached from the SessionStart hook.
+//
+// The pure brains (parse, signals, prompt, validate, payload) live in
+// src/bongos/onboarding.js; the subagent subprocess is reused from src/bongos/grader.js
+// (same `claude -p --model opus` invocation that the quality grader uses).
+//
+// Usage:
+//   node scripts/bongos/onboarding-analyze.js [--dry-run] [--only-flagged]
+//                                          [--max N] [--force] [--quiet]
+//
+//   --dry-run       parse + compute signals + print; no subagent, no POST.
+//   --only-flagged  only spend a subagent call on transcripts whose
+//                   deterministic signals already flagged (cost control).
+//   --max N         cap transcripts analyzed this run (default 5).
+//   --force         ignore the consent/window gate (testing).
+//
+// Bounded by design: at most --max per run, a lockfile prevents concurrent
+// runs, and it no-ops instantly when the builder hasn't consented.
+
+const fs = require('node:fs');
+const zlib = require('node:zlib');
+const path = require('node:path');
+const cfg = require('./onboarding-config');
+// The friction analyzer carved into modules/onboarding/ (BV1.R81 / ADR 0093 §1);
+// scripts are outside the kernel boundary, so this CLI requires the module file
+// directly (the grader.js / session-eval.js precedent — the line below already
+// does the same for grading).
+const onboarding = require('../../modules/onboarding/onboarding');
+const grader = require('../../modules/grading/grader');
+const llmCache = require('../../src/bongos/llm-cache'); // System 5 (task 1360): re-analyzing an immutable session is 100% idempotent
+const { apiCall, cliClient, requireSession, arg } = require('./cli-lib');
+llmCache.setApiFallback(apiCall); // task 1381: runs on a builder machine (no DB pool) → reach the cache via the GDS API
+
+const DRY_RUN = process.argv.includes('--dry-run');
+const ONLY_FLAGGED = process.argv.includes('--only-flagged');
+const FORCE = process.argv.includes('--force');
+const QUIET = process.argv.includes('--quiet');
+const MAX_PER_RUN = Math.max(1, Number(arg('--max')) || 5);
+const LOCK_PATH = path.join(cfg.SPOOL_DIR, '.analyze.lock');
+const LOCK_STALE_MS = 30 * 60 * 1000;
+
+function log(...a) {
+  if (!QUIET) console.log(...a);
+}
+
+// Best-effort single-writer lock. Returns true if we hold it.
+function acquireLock() {
+  cfg.ensureDir(cfg.SPOOL_DIR);
+  try {
+    fs.writeFileSync(LOCK_PATH, String(process.pid), { flag: 'wx' });
+    return true;
+  } catch (err) {
+    if (err.code !== 'EEXIST') throw err;
+    // Steal a stale lock (a crashed prior run).
+    try {
+      const st = fs.statSync(LOCK_PATH);
+      if (Date.now() - st.mtimeMs > LOCK_STALE_MS) {
+        fs.writeFileSync(LOCK_PATH, String(process.pid));
+        return true;
+      }
+    } catch (_) {
+      /* fall through */
+    }
+    return false;
+  }
+}
+function releaseLock() {
+  try {
+    fs.unlinkSync(LOCK_PATH);
+  } catch (_) {
+    /* ignore */
+  }
+}
+
+async function analyzeOne(name, entry) {
+  let jsonl;
+  try {
+    jsonl = zlib.gunzipSync(fs.readFileSync(entry.gz)).toString('utf8');
+  } catch (err) {
+    return { name, status: 'unreadable', error: err.message };
+  }
+
+  const { turns, meta } = onboarding.parseTranscript(jsonl);
+  const signals = onboarding.detectFrictionSignals(turns);
+
+  if (DRY_RUN) {
+    return { name, status: 'dry-run', meta, signals, turns: turns.length };
+  }
+
+  if (ONLY_FLAGGED && !signals.flagged) {
+    return { name, status: 'skipped-clean', signals };
+  }
+
+  // Run the Opus subagent on the transcript.
+  const prompt = onboarding.buildAnalysisPrompt({
+    sessionMeta: meta,
+    signals,
+    turns,
+  });
+  let analysis;
+  try {
+    // System 5 (task 1360): a session transcript is immutable, so re-analysis is
+    // 100% idempotent — cache on the session id so a re-run replays at $0.
+    const cache = llmCache.makeCacheSpec({
+      prompt, model: grader.DEFAULT_GRADER_MODEL, task_type: 'onboarding-analyze', scope_sha: String(name),
+    });
+    const { stdout } = await grader.runSubagentCached({
+      prompt,
+      opts: { cwd: process.cwd() },
+      cache,
+    });
+    const result = grader.extractAssistantResult(stdout);
+    analysis = onboarding.validateAnalysis(result);
+  } catch (err) {
+    // Subagent failed. Fall back to deterministic signals: if they flagged
+    // strongly, still file with a low-confidence note; otherwise leave it.
+    if (signals.flagged) {
+      analysis = {
+        ok: false,
+        flagged: true,
+        confidence: 0.3,
+        summary: `Subagent unavailable (${err.code || err.message}); filed from deterministic signals only.`,
+        friction_moments: [],
+      };
+    } else {
+      return { name, status: 'subagent-failed-clean', error: err.code || err.message, signals };
+    }
+  }
+
+  if (!analysis.flagged) {
+    return { name, status: 'not-flagged', signals, summary: analysis.summary };
+  }
+
+  const payload = onboarding.buildIdeaPayload({ sessionMeta: meta, analysis, signals });
+  const api = await cliClient();
+  const resp = await api.inbox.postInbox({
+    body: {
+      title: payload.title,
+      body_md: payload.body_md,
+      kind: payload.kind,
+    },
+  });
+  if (!resp.ok) {
+    return { name, status: 'file-failed', http: resp.status, body: resp.data };
+  }
+  return {
+    name,
+    status: 'filed',
+    idea_id: resp.data?.idea?.id ?? null,
+    moments: analysis.friction_moments.length,
+    confidence: analysis.confidence,
+  };
+}
+
+(async () => {
+  const consent = cfg.readConsent();
+  if (!FORCE && !cfg.onboardingActive(consent)) {
+    process.exit(0); // silent no-op
+  }
+  if (!DRY_RUN) {
+    await requireSession(); // need a token to POST /inbox
+  }
+
+  if (!acquireLock()) {
+    log('[onboarding] another analysis run holds the lock — skipping.');
+    process.exit(0);
+  }
+
+  try {
+    const manifest = cfg.readManifest();
+    const pending = Object.entries(manifest.entries || {})
+      .filter(([, e]) => e && e.gz && !e.analyzed && fs.existsSync(e.gz));
+
+    if (pending.length === 0) {
+      log('[onboarding] nothing to analyze.');
+      return;
+    }
+
+    let analyzed = 0;
+    for (const [name, entry] of pending) {
+      if (analyzed >= MAX_PER_RUN) {
+        log(`[onboarding] hit --max ${MAX_PER_RUN}; ${pending.length - analyzed} left for next run.`);
+        break;
+      }
+      const result = await analyzeOne(name, entry);
+      analyzed += 1;
+
+      if (DRY_RUN) {
+        log(`[onboarding] DRY ${name}: turns=${result.turns} flagged(det)=${result.signals?.flagged} ` +
+          `signals=${JSON.stringify(result.signals && {
+            longThenCorrection: result.signals.longThenCorrection,
+            repeatedSimilarPrompts: result.signals.repeatedSimilarPrompts,
+            errorRecoveryAttempts: result.signals.errorRecoveryAttempts,
+          })}`);
+        continue; // don't mutate manifest in dry-run
+      }
+
+      // Record terminal outcome so we never re-analyze the same snapshot.
+      manifest.entries[name] = {
+        ...entry,
+        analyzed: true,
+        analyzed_at: new Date().toISOString(),
+        outcome: result.status,
+        idea_id: result.idea_id ?? null,
+      };
+      cfg.writeManifest(manifest);
+      log(`[onboarding] ${name}: ${result.status}` +
+        (result.idea_id ? ` → idea #${result.idea_id} (${result.moments} moment(s))` : '') +
+        (result.error ? ` (${result.error})` : ''));
+    }
+  } finally {
+    releaseLock();
+  }
+  process.exit(0);
+})().catch((err) => {
+  releaseLock();
+  console.error('[onboarding] fatal:', err.message);
+  process.exit(1);
+});

package/scripts/bongos/onboarding-config.js

@@ -0,0 +1,179 @@
+// Local config + spool I/O for the onboarding-friction pipeline (V3.R79 / #298).
+//
+// Three on-disk artifacts, all under ~/.config/otb/ (same dir as the GDS
+// session token), so they survive worktree churn:
+//
+//   onboarding.json          — the builder's consent decision + onboarding window
+//   onboarding-spool/        — gzipped copies of session transcripts to analyze
+//   onboarding-spool/manifest.json — what's been spooled + analyzed (dedupe)
+//
+// This module is deliberately pure I/O + small predicates; the analysis brains
+// live in src/bongos/onboarding.js. Both onboarding-ship.js and onboarding-analyze.js
+// (and scripts/bongos/setup.js for the consent write) import from here so the
+// paths and window math have a single source of truth.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const os = require('node:os');
+const ic = require('../../src/instance-config');
+
+// The instance config dir (OTB → ~/.config/otb, unchanged; a fresh instance →
+// its neutral dir). R61 / task 1200.
+const CONFIG_DIR = ic.configHome();
+const CONSENT_PATH = path.join(CONFIG_DIR, 'onboarding.json');
+const SPOOL_DIR = path.join(CONFIG_DIR, 'onboarding-spool');
+const MANIFEST_PATH = path.join(SPOOL_DIR, 'manifest.json');
+
+// How long the onboarding window stays open after the consent decision.
+// Friction-mining is for the first stretch of a builder's life; after that
+// they're not "onboarding" and shipping their transcripts would be noise (and
+// a privacy overreach). Override for tests via <PREFIX>_ONBOARDING_WINDOW_DAYS
+// (BONGOS_ on a vanilla instance; the legacy OTB_ spelling is still read).
+const DEFAULT_WINDOW_DAYS = 14;
+
+function windowDays() {
+  const env = Number(ic.resolveEnv('ONBOARDING_WINDOW_DAYS'));
+  return Number.isFinite(env) && env > 0 ? env : DEFAULT_WINDOW_DAYS;
+}
+
+function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+// -------------------------------------------------------------------------
+// Consent
+// -------------------------------------------------------------------------
+
+function readConsent() {
+  try {
+    return JSON.parse(fs.readFileSync(CONSENT_PATH, 'utf8'));
+  } catch (_) {
+    return null;
+  }
+}
+
+// Persist the consent decision. consent=false is recorded too (so we never
+// re-prompt and the pipeline reliably no-ops). Window starts at decision time.
+function writeConsent({ consent }) {
+  ensureDir(CONFIG_DIR);
+  const now = new Date().toISOString();
+  const record = {
+    consent: consent === true,
+    decided_at: now,
+    window_started_at: now,
+    window_days: windowDays(),
+  };
+  fs.writeFileSync(CONSENT_PATH, JSON.stringify(record, null, 2), { mode: 0o600 });
+  return record;
+}
+
+function windowExpiry(record) {
+  if (!record?.window_started_at) return null;
+  const start = new Date(record.window_started_at).getTime();
+  if (!Number.isFinite(start)) return null;
+  const days = Number(record.window_days) || windowDays();
+  return start + days * 24 * 60 * 60 * 1000;
+}
+
+// The single gate the whole pipeline checks: consent given AND still inside
+// the onboarding window. Everything no-ops to false otherwise.
+function onboardingActive(record = readConsent(), now = Date.now()) {
+  if (!record || record.consent !== true) return false;
+  const expiry = windowExpiry(record);
+  if (expiry == null) return false;
+  return now < expiry;
+}
+
+// -------------------------------------------------------------------------
+// Transcript discovery
+//
+// Claude Code stores session transcripts as JSONL under
+//   ~/.claude/projects/<encoded-cwd>/<session-uuid>.jsonl
+// where <encoded-cwd> is the absolute project path with every non-alphanumeric
+// char replaced by '-' (e.g. "/Users/x/Desktop/Claude - OTB" becomes
+// "-Users-x-Desktop-Claude---OTB"). We derive the dir from a given cwd so the
+// pipeline works in any worktree without needing the hook's stdin payload.
+// -------------------------------------------------------------------------
+
+function encodeProjectPath(absPath) {
+  return String(absPath).replace(/[^a-zA-Z0-9]/g, '-');
+}
+
+function transcriptDirForCwd(cwd) {
+  return path.join(os.homedir(), '.claude', 'projects', encodeProjectPath(cwd));
+}
+
+// List transcript files (newest first by mtime) for a project dir.
+function listTranscripts(dir) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir);
+  } catch (_) {
+    return [];
+  }
+  const files = [];
+  for (const name of entries) {
+    if (!name.endsWith('.jsonl')) continue;
+    const full = path.join(dir, name);
+    let st;
+    try {
+      st = fs.statSync(full);
+    } catch (_) {
+      continue;
+    }
+    if (!st.isFile()) continue;
+    files.push({ name, path: full, size: st.size, mtimeMs: st.mtimeMs });
+  }
+  files.sort((a, b) => b.mtimeMs - a.mtimeMs);
+  return files;
+}
+
+// -------------------------------------------------------------------------
+// Spool manifest
+//
+// Keyed by transcript basename. We record the size+mtime we spooled so an
+// updated transcript (same session continued in a later run) re-spools and
+// re-analyzes. `analyzed` flips once the analyzer has filed (or decided not to
+// file) for that snapshot.
+// -------------------------------------------------------------------------
+
+function readManifest() {
+  try {
+    return JSON.parse(fs.readFileSync(MANIFEST_PATH, 'utf8'));
+  } catch (_) {
+    return { entries: {} };
+  }
+}
+
+function writeManifest(manifest) {
+  ensureDir(SPOOL_DIR);
+  fs.writeFileSync(MANIFEST_PATH, JSON.stringify(manifest, null, 2), { mode: 0o600 });
+}
+
+// A transcript snapshot is "fresh" (needs re-spooling) when we haven't seen it
+// or its size changed since we last spooled it.
+function needsSpool(manifest, file) {
+  const e = manifest.entries[file.name];
+  if (!e) return true;
+  return e.size !== file.size;
+}
+
+module.exports = {
+  CONFIG_DIR,
+  CONSENT_PATH,
+  SPOOL_DIR,
+  MANIFEST_PATH,
+  DEFAULT_WINDOW_DAYS,
+  windowDays,
+  ensureDir,
+  readConsent,
+  writeConsent,
+  windowExpiry,
+  onboardingActive,
+  encodeProjectPath,
+  transcriptDirForCwd,
+  listTranscripts,
+  readManifest,
+  writeManifest,
+  needsSpool,
+};

package/scripts/bongos/onboarding-ship.js

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+// Stage onboarding session transcripts for friction analysis (V3.R79 / #298).
+//
+// Runs at SessionStart (spawned by .claude/hooks/session-start.js, which
+// absorbed the former onboarding-ship-logs.sh shim). No-ops instantly unless
+// the builder consented and is still inside the onboarding window — so for
+// everyone else (and after the window closes) this costs one small file read
+// and exits 0.
+//
+// "Ships the session transcript (compressed) to server" per the done-when is
+// realized as a privacy-preserving two-step: this script gzips completed
+// transcripts into a local spool, and onboarding-analyze.js then ships only
+// the *distilled friction signal* (not the raw transcript) to the shared GDS
+// via POST /api/bongos/inbox. The raw transcript never leaves the machine — a
+// deliberate privacy choice given transcripts can contain anything the builder
+// typed. A dedicated upload endpoint would need a new route + storage table
+// (migration), both outside this task's touches[].
+//
+// Usage:
+//   node scripts/bongos/onboarding-ship.js [--cwd PATH] [--force] [--quiet]
+//
+// Exit codes: always 0 on a clean no-op or success; 0 even when there's
+// nothing to do — this is a hook, it must never break session start.
+
+const fs = require('node:fs');
+const zlib = require('node:zlib');
+const path = require('node:path');
+const cfg = require('./onboarding-config');
+const { arg } = require('./cli-lib');
+
+const QUIET = process.argv.includes('--quiet');
+const FORCE = process.argv.includes('--force');
+function log(...a) {
+  if (!QUIET) console.log(...a);
+}
+
+// A transcript still being written by the live session shouldn't be spooled
+// mid-stream. Skip the most-recently-touched file when it was modified within
+// this guard window (the active session). --force overrides for testing.
+const ACTIVE_GUARD_MS = 90_000;
+
+(function main() {
+  const consent = cfg.readConsent();
+  if (!FORCE && !cfg.onboardingActive(consent)) {
+    // Silent no-op: not consented, or window closed. This is the common path.
+    process.exit(0);
+  }
+
+  const cwd = arg('--cwd') || process.cwd();
+  const dir = cfg.transcriptDirForCwd(cwd);
+  const files = cfg.listTranscripts(dir);
+  if (files.length === 0) {
+    log('[onboarding] no transcripts found for this project yet.');
+    process.exit(0);
+  }
+
+  const manifest = cfg.readManifest();
+  const now = Date.now();
+  let staged = 0;
+
+  for (const file of files) {
+    // Don't spool the file the live session is actively writing.
+    if (!FORCE && now - file.mtimeMs < ACTIVE_GUARD_MS) continue;
+    if (!cfg.needsSpool(manifest, file)) continue;
+
+    try {
+      const raw = fs.readFileSync(file.path);
+      const gz = zlib.gzipSync(raw);
+      cfg.ensureDir(cfg.SPOOL_DIR);
+      const dest = path.join(cfg.SPOOL_DIR, `${file.name}.gz`);
+      fs.writeFileSync(dest, gz, { mode: 0o600 });
+      manifest.entries[file.name] = {
+        size: file.size,
+        mtimeMs: file.mtimeMs,
+        spooled_at: new Date().toISOString(),
+        gz: dest,
+        analyzed: false,
+        // size is the snapshot key; if the session grows, needsSpool() trips
+        // and we re-spool + clear analyzed below.
+      };
+      staged += 1;
+    } catch (err) {
+      log(`[onboarding] could not spool ${file.name}: ${err.message}`);
+    }
+  }
+
+  if (staged > 0) {
+    cfg.writeManifest(manifest);
+    log(`[onboarding] staged ${staged} transcript(s) for friction analysis.`);
+  }
+  process.exit(0);
+})();

package/scripts/bongos/pack-env-prefix.js

@@ -0,0 +1,73 @@
+'use strict';
+
+// scripts/bongos/pack-env-prefix.js — which env-var names a hosted project reads, and
+// whether its own pack agrees with what the provisioning runner writes (task 1004573).
+//
+// WHY. The runner writes every hosted project's /etc/<slug>/web.env under ONE prefix
+// (provisionedEnvPrefix). A project ADOPTED from an existing repo brings its own
+// config/branding.json, and mercury's said envPrefix CLOUDBONGOS while web.env carried
+// BONGOS_* — so the project read none of its sign-in credentials, its address or any
+// setting, and the only symptom anyone saw was "did not report sign-in ready".
+//
+// Its own file because both provision-units.js (web.env) and provision-repo.js (the
+// adopt scaffold) need it, and provision-units already requires provision-repo — a
+// helper living in either would make a require cycle. Requires only provision-config.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { REPO_ROOT } = require('./provision-config.js');
+
+// The pack whose envPrefix a provisioned instance inherits unless its own pack says
+// otherwise (see the long note on ENV_PREFIX_PACK's history in provision-units.js).
+const ENV_PREFIX_PACK = 'config/branding.neutral.json';
+
+// The env-var prefix a provisioned instance actually RESOLVES — i.e. the envPrefix
+// of the pack it loads (ENV_PREFIX_PACK). src/instance-config.resolveEnv
+// reads <envPrefix>_<SUFFIX>, and src/branding.resolveBranding applies
+// <envPrefix>_{PUBLIC,BUILDERS,…}_ORIGIN overrides — so a web.env var only takes
+// effect under THIS prefix. Read from the pack (fail-soft to the vanilla default)
+// so it stays in lockstep instead of a magic string. (The prior slug-based prefix
+// was a latent bug: it never matched the neutral pack's CLOUDBONGOS envPrefix, so
+// the OAuth creds were silently unreadable — task 1972.)
+function provisionedEnvPrefix() {
+  const { FALLBACK_ENV_PREFIX } = require('../../src/instance-config'); // eslint-disable-line global-require
+  try {
+    const pack = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, ENV_PREFIX_PACK), 'utf8'));
+    return pack.envPrefix || FALLBACK_ENV_PREFIX;
+  } catch { return FALLBACK_ENV_PREFIX; }
+}
+
+// The envPrefix a pack's text names, or null (no pack, not JSON, no envPrefix).
+function packEnvPrefix(packText) {
+  try {
+    const pack = JSON.parse(String(packText || ''));
+    return pack && typeof pack.envPrefix === 'string' ? pack.envPrefix : null;
+  } catch { return null; }
+}
+
+// Does the instance's OWN pack read the names this runner writes? PURE: give it the
+// pack's text (or null when it could not be read) and the prefix written; it returns
+// a sentence naming the mismatch, or null when the names agree or the pack says nothing.
+function envPrefixMismatch(packText, written = provisionedEnvPrefix()) {
+  const own = packEnvPrefix(packText);
+  if (!own || own === written) return null;
+  return `the project's config/branding.json reads settings named ${own}_*, but the platform wrote ${written}_* — set its envPrefix to "${written}"`;
+}
+
+// The adopt scaffold's half of the fix: rewrite ONLY the envPrefix value in an adopted
+// repo's config/branding.json so the project reads the names this runner writes,
+// leaving every other byte of the owner's file as they wrote it (a JSON round-trip
+// would reflow it). PURE. Returns { text, from } when it changed something, or null
+// when the names already agree, the pack names no prefix, or it is not JSON.
+function normalizePackEnvPrefix(packText, written = provisionedEnvPrefix()) {
+  const raw = String(packText || '');
+  const own = packEnvPrefix(raw);
+  if (!own || own === written) return null;
+  const re = /("envPrefix"\s*:\s*)"[^"]*"/;
+  if (!re.test(raw)) return null;
+  const text = raw.replace(re, (_m, key) => `${key}"${written}"`);
+  if (packEnvPrefix(text) !== written) return null;
+  return { text, from: own };
+}
+
+module.exports = { ENV_PREFIX_PACK, provisionedEnvPrefix, envPrefixMismatch, normalizePackEnvPrefix };