@bglocation/tune-context 2.3.1 → 2.5.0

package/bin/tune-context.mjs

@@ -30,6 +30,15 @@ import {
   mergePermissions,
 } from '../cli/generate.mjs';
 import { staleDoctrineEntries, verify } from '../cli/verify.mjs';
+import {
+  ExperimentError,
+  experimentPaths,
+  experimentRestore,
+  experimentSet,
+  observeArm,
+  readLedger,
+  readState,
+} from '../cli/experiment.mjs';
 
 const HOOK_EVENTS = [
   { event: 'PreCompact', scriptName: 'pre-compact-snapshot.mjs' },
@@ -58,6 +67,9 @@ function usage() {
 Usage:
   tune-context init [--cwd <dir>] [--force-claude-md]
   tune-context verify [--cwd <dir>]
+  tune-context experiment status [--cwd <dir>]
+  tune-context experiment set --tc <on|off> --rag <on|off> [--note "..."] [--cwd <dir>]
+  tune-context experiment restore
   tune-context --help
   tune-context --version
 
@@ -66,12 +78,76 @@ Usage:
                       match this package. Run it after upgrading: an upgrade
                       does not refresh copies an earlier init made.
 
+  experiment          A/B switch for measuring the doctrine against its own
+                      absence (EPIC-009). \`set\` disables what the arm asks for
+                      and records how to put it back; \`restore\` puts it back
+                      byte for byte, or refuses if a touched file changed since.
+                      Both --tc and --rag are required for \`set\`: an arm that
+                      was only half-stated is an arm nobody can reproduce.
+
   --force-claude-md   Regenerate a project CLAUDE.md whose hand-written content
                       still sits inside the managed block. Discards that content.
                       Without it, such a file is left untouched.
 `;
 }
 
+// Flags that consume the token after them. Listed once so the positional
+// scanner below cannot mistake a flag's VALUE for a subcommand — `experiment
+// --cwd /tmp/x status` must find `status`, not `/tmp/x`.
+const VALUE_FLAGS = new Set(['--cwd', '--tc', '--rag', '--note']);
+
+/**
+ * @returns {string|{error: string}} the value after `name`, or undefined if the
+ *   flag is absent. Undefined (last arg) and another flag both count as "no
+ *   value given" — resolve(undefined) used to throw a raw ERR_INVALID_ARG_TYPE
+ *   with a Node stacktrace instead of a usage message.
+ */
+function flagValue(args, name) {
+  const idx = args.indexOf(name);
+  if (idx === -1) return undefined;
+  const value = args[idx + 1];
+  if (value === undefined || value.startsWith('-')) return { error: `Missing value for ${name}` };
+  return value;
+}
+
+function positionals(args) {
+  const out = [];
+  for (let i = 0; i < args.length; i += 1) {
+    if (VALUE_FLAGS.has(args[i])) {
+      i += 1;
+      continue;
+    }
+    if (!args[i].startsWith('-')) out.push(args[i]);
+  }
+  return out;
+}
+
+const EXPERIMENT_SUBCOMMANDS = ['status', 'set', 'restore'];
+
+/**
+ * `experiment` argument shape. `set` demands BOTH --tc and --rag: an arm stated
+ * by halves is an arm nobody can reproduce from the ledger, and inferring the
+ * unstated half from the current config would write a value nobody chose.
+ */
+function parseExperiment(rest, cwd) {
+  const [sub] = positionals(rest);
+  if (sub === undefined) return { mode: 'error', message: `Missing experiment subcommand: ${EXPERIMENT_SUBCOMMANDS.join(', ')}` };
+  if (!EXPERIMENT_SUBCOMMANDS.includes(sub)) return { mode: 'error', message: `Unknown experiment subcommand: ${sub}` };
+  if (sub !== 'set') return { mode: 'experiment', sub, cwd };
+
+  const values = {};
+  for (const flag of ['--tc', '--rag', '--note']) {
+    const value = flagValue(rest, flag);
+    if (value?.error) return { mode: 'error', message: value.error };
+    values[flag] = value;
+  }
+  for (const flag of ['--tc', '--rag']) {
+    if (values[flag] === undefined) return { mode: 'error', message: `experiment set requires ${flag} <on|off>` };
+    if (values[flag] !== 'on' && values[flag] !== 'off') return { mode: 'error', message: `${flag} must be on or off, got: ${values[flag]}` };
+  }
+  return { mode: 'experiment', sub, cwd, tc: values['--tc'], rag: values['--rag'], note: values['--note'] ?? '' };
+}
+
 // Pure so it is unit-testable without spawning a process. `init` is the ONLY
 // accepted command — no bare-args shortcut (PO decision 2026-07-25, TASK-078):
 // README never once documents calling this without it, and accepting anything
@@ -86,32 +162,48 @@ export function parseArgs(argv) {
   // `verify` joins `init` as an accepted command (TASK-081). It writes nothing,
   // which is the point: init syncs before it verifies, so init can never
   // observe the drift an upgrade leaves behind — only a check that runs on its
-  // own can. The TASK-078 rule still holds: an unknown command fails loudly.
-  if (command !== 'init' && command !== 'verify') {
+  // own can. `experiment` joins them as the A/B switch (TASK-087). The TASK-078
+  // rule still holds: an unknown command fails loudly.
+  const commands = ['init', 'verify', 'experiment'];
+  if (!commands.includes(command)) {
     return {
       mode: 'error',
-      message: command === undefined ? 'Missing command: init or verify' : `Unknown command: ${command}`,
+      message: command === undefined ? `Missing command: ${commands.join(', ')}` : `Unknown command: ${command}`,
     };
   }
+
+  const cwdValue = flagValue(rest, '--cwd');
+  if (cwdValue?.error) return { mode: 'error', message: cwdValue.error };
+  const cwd = cwdValue === undefined ? process.cwd() : resolve(cwdValue);
+
+  if (command === 'experiment') return parseExperiment(rest, cwd);
   // Opt-in only, and never inferred: this is the one flag that can destroy
   // content the tool otherwise refuses to touch (TASK-082).
-  const forceClaudeMd = rest.includes('--force-claude-md');
-  const cwdFlagIdx = rest.indexOf('--cwd');
-  if (cwdFlagIdx === -1) return { mode: command, cwd: process.cwd(), forceClaudeMd };
-  const value = rest[cwdFlagIdx + 1];
-  // Undefined (last arg) or another flag (e.g. `--cwd --foo`) both mean "no
-  // value was actually given" — resolve(undefined) used to throw a raw
-  // ERR_INVALID_ARG_TYPE with a Node stacktrace instead of a usage message.
-  if (value === undefined || value.startsWith('-')) {
-    return { mode: 'error', message: 'Missing value for --cwd' };
-  }
-  return { mode: command, cwd: resolve(value), forceClaudeMd };
+  return { mode: command, cwd, forceClaudeMd: rest.includes('--force-claude-md') };
 }
 
 function runInit(cwd, { forceClaudeMd = false } = {}) {
   const templatesDir = join(pkgRoot, 'skills', 'tune-context', 'templates');
   const { claudeDir } = claudeDirs();
 
+  // An experiment arm is the one state in which init must NOT run. It would
+  // re-install everything the arm parked — turning the config back on while the
+  // ledger still says off (a mislabelled measurement), and putting content back
+  // where `restore` then finds a fingerprint it did not write, so the only way
+  // out would be refusal. Blocking here keeps both the arm and the undo intact.
+  const state = readState(claudeDir);
+  if (state) {
+    process.stderr.write(`tune-context init refused — experiment arm tc=${state.arm.tc} rag=${state.arm.rag} has been set (since ${state.startedAt}).
+
+  init would re-install what the arm parked, and \`experiment restore\` would then
+  refuse, because the files no longer match what it saved.
+
+  Run \`tune-context experiment restore\` first, then init.
+`);
+    process.exitCode = 1;
+    return;
+  }
+
   const stack = detectStack(cwd);
   const existing = detectExistingConfig(cwd);
 
@@ -226,6 +318,11 @@ function runVerify(cwd) {
   const detection = { stack: detectStack(cwd), mcpServers: detectMcpServers(cwd) };
   const permissionsAllow = buildPermissionsAllow(detection);
 
+  // The arm comes from the ledger state, never from guessing at the config:
+  // "the doctrine is missing" and "the doctrine was switched off on purpose"
+  // look identical on disk, and only one of them is a broken install.
+  const arm = readState(claudeDir)?.arm ?? null;
+
   const result = verify({
     claudeDir,
     userSettingsPath: existing.userSettingsJson,
@@ -233,9 +330,14 @@ function runVerify(cwd) {
     expectedHookCommands: HOOK_EVENTS,
     expectedPermissions: permissionsAllow,
     pkgRoot,
+    arm,
   });
 
   console.log(`tune-context verify — ${cwd}\n`);
+  if (arm) {
+    console.log(`Experiment arm: tc=${arm.tc} rag=${arm.rag} — the checks below are read against that arm, not against a full install.
+  This is the pre-run check from COST_EVALUATION_PROCEDURE §4: it must say the arm you meant to set, BEFORE the first prompt.\n`);
+  }
   console.log(`Verify: ${result.ok ? 'OK' : 'FAILED'}`);
   for (const check of result.checks) {
     const mark = check.unknown ? '?' : check.ok ? '✓' : '✗';
@@ -245,7 +347,129 @@ function runVerify(cwd) {
   // never ran init, this prints a screen of ✗ and, without the line below,
   // nothing that says the whole list is one command away from green.
   if (!result.ok) {
-    console.log('\nNext: `tune-context init` writes what is missing and refreshes what drifted. It never edits content outside the managed block.');
+    console.log(
+      arm
+        ? '\nNext: `tune-context experiment restore` — while an arm is set, init is refused, and a red check here means the arm did not fully land.'
+        : '\nNext: `tune-context init` writes what is missing and refreshes what drifted. It never edits content outside the managed block.',
+    );
+    process.exitCode = 1;
+  }
+}
+
+// Said the same way everywhere: an arm is a property of the SESSION, not of the
+// prompt. settings.json and MCP servers are read when Claude Code starts, so a
+// switch flipped mid-session produces a mixed session — the one sample class
+// EPIC-009 throws away rather than assigns.
+const RESTART_NOTE = `The arm applies from the NEXT Claude Code start. settings.json and MCP servers are read at
+  startup, so a session already running is unaffected — close it before the measured run.`;
+
+const MCP_SCOPE_NOTE = `MCP scope: only this project's .mcp.json is switched. The USER scope (~/.claude.json) is
+  rewritten by Claude Code while it runs, so this switch never touches it — a user-scope
+  semantic-search server stays available in a \`rag off\` arm. Check with \`claude mcp list\`
+  before the run, or the arm is mislabelled.`;
+
+function armLine(observed) {
+  return `tc=${observed.tc} rag=${observed.rag}`;
+}
+
+function runExperimentStatus(cwd) {
+  const { claudeDir } = claudeDirs();
+  const { statePath, ledgerPath } = experimentPaths(claudeDir);
+  const state = readState(claudeDir);
+  const ledger = readLedger(claudeDir);
+  const observed = observeArm({ claudeDir, cwd, pkgRoot });
+
+  console.log(`tune-context experiment status — ${cwd}\n`);
+
+  const last = ledger[ledger.length - 1];
+  console.log('Ledger (source of truth for assigning a session to an arm):');
+  console.log(`  ${ledgerPath}`);
+  if (!last) {
+    console.log('  no entries yet — no run has been labelled through this switch');
+  } else {
+    console.log(`  last: ${last.event ?? 'set'} tc=${last.tc} rag=${last.rag} at ${last.ts}${last.note ? ` — "${last.note}"` : ''} (${ledger.length} entr${ledger.length === 1 ? 'y' : 'ies'})`);
+  }
+
+  console.log('\nActive switch state:');
+  console.log(
+    state
+      ? `  arm tc=${state.arm.tc} rag=${state.arm.rag} set at ${state.startedAt}; ${state.entries.length} path(s) parked, restorable from ${statePath}`
+      : '  none — this switch has nothing parked, so `restore` has nothing to put back',
+  );
+
+  console.log('\nObserved in the config (cross-check, NOT the source of truth):');
+  console.log(`  ${armLine(observed)}`);
+  for (const c of observed.components) {
+    const mark = c.state === 'on' ? '✓' : c.state === 'off' ? '·' : '!';
+    console.log(`    ${mark} ${c.label} — ${c.detail}`);
+  }
+  console.log(
+    `    ${observed.mcp.servers.length ? '✓' : '·'} ${observed.mcp.path} — ${observed.mcp.servers.length ? `${observed.mcp.servers.length} server(s): ${observed.mcp.servers.join(', ')}` : 'absent or empty'}`,
+  );
+
+  // The two must agree before a measured run: the ledger is what the analyzer
+  // reads, the config is what the session actually gets.
+  if (state && (state.arm.tc !== observed.tc || state.arm.rag !== observed.rag)) {
+    console.log(`\n! The set arm (tc=${state.arm.tc} rag=${state.arm.rag}) and the observed config (${armLine(observed)}) DISAGREE.
+  Something changed the config after \`set\` — do not measure in this state; \`restore\` and set again.`);
+  }
+
+  console.log(`\n${MCP_SCOPE_NOTE}`);
+  console.log(`\n${RESTART_NOTE}`);
+}
+
+function runExperimentSet(cwd, { tc, rag, note }) {
+  const { claudeDir } = claudeDirs();
+  const result = experimentSet({ claudeDir, cwd, pkgRoot, tc, rag, note });
+
+  console.log(`tune-context experiment set — tc=${tc} rag=${rag}${note ? ` — "${note}"` : ''}\n`);
+  if (result.alreadySet) {
+    console.log('Config unchanged: this arm was already set. The ledger entry is written anyway — a run whose\n  ledger is silent at its start cannot be assigned to an arm at all.');
+  } else if (result.changes.length === 0) {
+    console.log('Config unchanged: nothing had to be switched off for this arm. Ledger entry written.');
+  } else {
+    console.log(`Switched off (${result.changes.length}):`);
+    for (const change of result.changes) console.log(`  - ${change}`);
+    console.log(`\nUndo: \`tune-context experiment restore\` — byte for byte from ${result.statePath},
+  or a refusal naming the file if one of those paths changed in the meantime.`);
+  }
+
+  for (const warning of result.warnings) console.log(`\n! ${warning}`);
+
+  console.log(`\n${RESTART_NOTE}`);
+  console.log(`\n${MCP_SCOPE_NOTE}`);
+  console.log('\nNext: `tune-context verify` — confirm the arm BEFORE the first prompt, not after the run.');
+  if (result.warnings.length) process.exitCode = 1;
+}
+
+function runExperimentRestore() {
+  const { claudeDir } = claudeDirs();
+  const result = experimentRestore({ claudeDir });
+  console.log('tune-context experiment restore\n');
+  if (result.parked) {
+    console.log(`Restored ${result.restored.length} path(s), verified against the fingerprints taken before the change:`);
+    for (const r of result.restored) console.log(`  - ${r.path}`);
+  } else {
+    // The `on` arm parks nothing, so this is its normal ending — and not an
+    // empty one: the ledger entry is what tells the analyzer the run is over,
+    // instead of letting the last `set` label every later session (TASK-088).
+    console.log('Nothing had been parked — this arm changed no files. The ledger entry below is the point of\n  the command: it closes the measurement window, so work done from now on is not read as part of the run.');
+  }
+  console.log(`\nLedger: arm back to tc=${result.armBefore.tc} rag=${result.armBefore.rag} at ${result.ledgerEntry.ts}.`);
+  console.log(`\n${RESTART_NOTE}`);
+}
+
+function runExperiment(parsed) {
+  try {
+    if (parsed.sub === 'status') runExperimentStatus(parsed.cwd);
+    else if (parsed.sub === 'set') runExperimentSet(parsed.cwd, parsed);
+    else runExperimentRestore();
+  } catch (error) {
+    // Operator-facing conditions (arm already set, nothing to restore, a file
+    // changed under us) are answers, not crashes: message + next step + exit 1,
+    // never a stacktrace. Anything else is a real bug and is rethrown.
+    if (!(error instanceof ExperimentError)) throw error;
+    process.stderr.write(`tune-context experiment ${parsed.sub}: ${error.message}\n${error.hint ? `\n  ${error.hint}\n` : ''}`);
     process.exitCode = 1;
   }
 }
@@ -265,6 +489,8 @@ export function main() {
     process.exitCode = 1;
   } else if (parsed.mode === 'verify') {
     runVerify(parsed.cwd);
+  } else if (parsed.mode === 'experiment') {
+    runExperiment(parsed);
   } else {
     runInit(parsed.cwd, { forceClaudeMd: parsed.forceClaudeMd });
   }