@remits/remits-cli 0.1.156 → 0.1.158

package/index.js

@@ -12,10 +12,10 @@
 - L4591  Auth And Component Staging
 - L5223  Component Summaries, Status, And Sync Gates
 - L7547  Branches, Promotion, Commit, And Test Runs
-- L9676  Tokens, Tools, Verification, And Config
-- L11103  Service Dashboard And WebSocket Listener
-- L13602  Agent And Ticket Workflows
-- L17778  Help, Auto Update, And Command Dispatch
+- L9790  Tokens, Tools, Verification, And Config
+- L11307  Service Dashboard And WebSocket Listener
+- L13806  Agent And Ticket Workflows
+- L17982  Help, Auto Update, And Command Dispatch
 */
 
 /*
@@ -112,7 +112,7 @@ let LOCAL_ACTOR_CONTEXT = null;
 
 // Flags that are legitimately repeatable accumulate into an array instead of last-wins. Every other
 // flag keeps last-wins so existing callers are unaffected.
-const REPEATABLE_FLAGS = new Set(['expected-removed', 'expectedRemoved', 'names', 'tag', 'tags', 'key', 'keys', 'case-key']);
+const REPEATABLE_FLAGS = new Set(['expected-removed', 'expectedRemoved', 'names', 'tag', 'tags', 'key', 'keys', 'case-key', 'score', 'scores', 'lower-is-better', 'param', 'params']);
 
 // `-m "msg"` is how git spells a commit message, and agents type it from habit. Before this the parser took a
 // single-dash token as the VALUE of whatever flag preceded it: `--json -m "x"` silently turned JSON mode
@@ -9156,7 +9156,9 @@ function watchWebsocket(baseUrl, topicId, taskId, options = {}) {
         const data = JSON.parse(msg.body);
         if (data.type === 'TestSuite') {
           const payload = data.payload || {};
-          if (payload.taskId && taskId && payload.taskId !== taskId) {
+          // A parallel run is followed by every shard's task id.
+          const followed = Array.isArray(taskId) ? taskId : (taskId ? [taskId] : []);
+          if (payload.taskId && followed.length && !followed.includes(payload.taskId)) {
             return;
           }
           if (payload.tests || payload.name || payload.total != null) {
@@ -9179,6 +9181,82 @@ function watchWebsocket(baseUrl, topicId, taskId, options = {}) {
   };
 }
 
+// `--param name=value` (repeatable) -> the map a suite reads with runParams(). A value that is a JSON number,
+// boolean, null, array or object is passed as that JSON; anything else is the literal text, so `--param model=x`
+// needs no quoting. A name without `=` is refused rather than guessed at.
+function parseRunParams(raw) {
+  const values = raw === undefined || raw === null || raw === true ? [] : (Array.isArray(raw) ? raw : [raw]);
+  const params = {};
+  values.forEach((entry) => {
+    const text = String(entry);
+    const eq = text.indexOf('=');
+    if (eq <= 0) throw new Error('--param expects name=value (got "' + text + '")');
+    const name = text.slice(0, eq).trim();
+    const value = text.slice(eq + 1);
+    let parsed = value;
+    const trimmed = value.trim();
+    if (/^(-?\d+(\.\d+)?([eE][+-]?\d+)?|true|false|null)$/.test(trimmed) || /^(\[[\s\S]*\]|\{[\s\S]*\})$/.test(trimmed)) {
+      try {
+        parsed = JSON.parse(trimmed);
+      } catch (err) {
+        parsed = value;
+      }
+    }
+    params[name] = parsed;
+  });
+  return params;
+}
+
+// How `test run` labels, parameterises, repeats and shards a run, from its flags. Validation of the values is the
+// platform's (Test.normalizeRunOptions); this only shapes the request and checks what the CLI itself acts on.
+function testRunOptionsFromFlags(flags = {}) {
+  const label = flags.label === undefined || flags.label === true ? undefined : String(flags.label);
+  const params = parseRunParams(flags.param !== undefined ? flags.param : flags.params);
+  const repeat = flags.repeat === undefined ? undefined : parsePositiveInt(flags.repeat, NaN);
+  if (flags.repeat !== undefined && !Number.isFinite(repeat)) throw new Error('--repeat expects a whole number of runs per case (got "' + flags.repeat + '")');
+  const parallel = flags.parallel === undefined ? 1 : parsePositiveInt(flags.parallel, NaN);
+  if (!Number.isFinite(parallel)) throw new Error('--parallel expects a whole number of shards (got "' + flags.parallel + '")');
+  return {
+    label,
+    params: Object.keys(params).length ? params : undefined,
+    repeat: repeat && repeat > 1 ? repeat : undefined,
+    parallel
+  };
+}
+
+// How long a later shard of a parallel run waits for a free suite slot before the launch gives up.
+const SHARD_ADMISSION_WAIT_MS = 30 * 60 * 1000;
+
+// A start the platform refused because every suite slot on the instance is taken (safe to repeat: no work began).
+function shardStartBusy(err) {
+  const response = err && err.response;
+  return !!(response && response.status === 429 && response.data && response.data.code === 'WORKLOAD_BUSY');
+}
+
+// One start request per shard of a `--parallel N` run, all under one group id: shard i of N runs every Nth
+// selected case. The platform reads the group id as ONE run in status, corpus runs/compare/gate.
+function shardRunPayloads(runOptions = {}, groupId) {
+  const count = Math.max(1, Number(runOptions.parallel || 1));
+  const common = { label: runOptions.label, params: runOptions.params, repeat: runOptions.repeat };
+  if (count <= 1) return [common];
+  return Array.from({ length: count }, (_, i) => Object.assign({}, common, {
+    runGroupId: groupId,
+    shard: { index: i + 1, count }
+  }));
+}
+
+// What a launch asked for beyond the suite itself: label, params, repetitions and shards.
+function testRunLaunchLines(start = {}, runOptions = {}) {
+  const lines = [];
+  if (runOptions.label) lines.push('Label: ' + runOptions.label);
+  if (runOptions.params) lines.push('Params: ' + Object.keys(runOptions.params).map((k) => k + '=' + JSON.stringify(runOptions.params[k])).join(' '));
+  if (runOptions.repeat) lines.push('Repeat: every selected case runs ' + runOptions.repeat + ' times');
+  if (start.runGroupId) {
+    lines.push('Parallel: ' + (start.shardTaskIds || []).length + ' shards (' + (start.shardTaskIds || []).join(', ') + '), read as ONE run by ' + start.runGroupId);
+  }
+  return lines;
+}
+
 async function testCommand(flags) {
   const cwd = process.cwd();
   ensureLocalState(cwd);
@@ -9207,6 +9285,9 @@ async function testCommand(flags) {
     ? []
     : (Array.isArray(flags.names) ? flags.names : [flags.names]);
   const names = parseTestCaseSelectors(rawNames);
+  // --label / --param / --repeat / --parallel: what this run varies, and how it is executed.
+  const runOptions = testRunOptionsFromFlags(flags);
+  const runGroupId = runOptions.parallel > 1 ? 'grp-' + crypto.randomUUID() : undefined;
 
   const api = buildAxios(baseUrl, session.token);
   // --as-account runs the Test AS a (descendant) subscriber account, so that account's relationship
@@ -9243,7 +9324,7 @@ async function testCommand(flags) {
     });
   });
 
-  const start = await loggedPost(api, cwd, '/cli/test', {
+  const startRequest = {
     token: session.token,
     accountId,
     asAccountId,
@@ -9255,11 +9336,43 @@ async function testCommand(flags) {
     testId: /^\d+$/.test(String(testRef)) ? Number(testRef) : undefined,
     testName: /^\d+$/.test(String(testRef)) ? undefined : String(testRef),
     tests: names
-  }).then((r) => r.data);
-
-  if (!start.success) {
-    throw new Error('Failed to start test run');
+  };
+  // Shards are started one after another (each start returns at once) and then run side by side on the platform.
+  const shardStarts = [];
+  // The platform admits only a few suites per instance at once and refuses the rest as WORKLOAD_BUSY. Once one
+  // of our shards is running it holds a slot it will give back, so a later shard keeps waiting for one (bounded)
+  // instead of giving up after the usual retries and leaving a group that can never finish.
+  const shardWaitDeadline = Date.now() + SHARD_ADMISSION_WAIT_MS;
+  for (const shardOptions of shardRunPayloads(runOptions, runGroupId)) {
+    let shardStart;
+    for (;;) {
+      try {
+        shardStart = await loggedPost(api, cwd, '/cli/test', Object.assign({}, startRequest, shardOptions)).then((r) => r.data);
+        break;
+      } catch (err) {
+        const data = err && err.response && err.response.data;
+        if (shardStarts.length && shardStartBusy(err) && Date.now() < shardWaitDeadline) {
+          console.error('Shard ' + (shardStarts.length + 1) + ' of ' + runOptions.parallel + ' is waiting for a free suite slot ' +
+            '(this platform instance runs a limited number of suites at once; the shards already running free them).');
+          await new Promise((resolve) => setTimeout(resolve, 10000));
+          continue;
+        }
+        const started = shardStarts.length ? ' (shard ' + (shardStarts.length + 1) + ' of ' + runOptions.parallel +
+          ' never started; the shards already started keep running: ' + shardStarts.map((st) => st.taskId).join(', ') + ')' : '';
+        if (data && data.message) throw new Error('Failed to start test run: ' + data.message + started);
+        throw err;
+      }
+    }
+    if (!shardStart.success) {
+      throw new Error('Failed to start test run' + (shardStarts.length ? ' (shard ' + (shardStarts.length + 1) + ' of ' + runOptions.parallel +
+        '; the shards already started keep running: ' + shardStarts.map((st) => st.taskId).join(', ') + ')' : ''));
+    }
+    shardStarts.push(shardStart);
   }
+  // One run to follow: the single run, or the group whose status the platform merges from every shard.
+  const start = runGroupId
+    ? Object.assign({}, shardStarts[0], { taskId: runGroupId, runGroupId, shardTaskIds: shardStarts.map((s) => s.taskId) })
+    : shardStarts[0];
 
   withStdoutRoutedToStderr(jsonOutput, () => {
     printSessionResolutionWarning(sessionContext);
@@ -9281,6 +9394,7 @@ async function testCommand(flags) {
       gitHead: safeGitValue(cwd, 'git rev-parse HEAD')
     }).forEach((line) => console.log(line));
     console.log('Test run started:', start.taskId);
+    testRunLaunchLines(start, runOptions).forEach((line) => console.log(line));
   });
   runtimeState.currentTestTaskId = start.taskId;
 
@@ -9330,7 +9444,7 @@ async function testCommand(flags) {
   if (flags.watch !== 'false') {
     const topic = session.websocketTopic || start.websocketTopic || (session.user && String(session.user.uuid || '').replace(/-/g, ''));
     if (topic) {
-      stopWs = watchWebsocket(baseUrl, topic, start.taskId, { stderr: jsonOutput, verbose: flagEnabled(flags.verbose) });
+      stopWs = watchWebsocket(baseUrl, topic, start.shardTaskIds || start.taskId, { stderr: jsonOutput, verbose: flagEnabled(flags.verbose) });
     }
   }
 
@@ -10343,6 +10457,21 @@ function corpusAiLabel(row = {}) {
   return 'no AI';
 }
 
+function corpusScoreSummary(scores = {}) {
+  return Object.keys(scores).map((k) => k + ' ' + (scores[k] && scores[k].mean !== undefined ? scores[k].mean : scores[k]) +
+    (scores[k] && scores[k].stderr !== undefined ? ' \u00b1' + scores[k].stderr : '') +
+    (scores[k] && scores[k].cases !== undefined ? '/' + scores[k].cases : '')).join(', ');
+}
+
+function corpusScoreDeltas(scores = {}) {
+  return Object.keys(scores).map((k) => {
+    const s = scores[k] || {};
+    const delta = s.delta !== undefined && s.delta !== null ? ' (' + (Number(s.delta) >= 0 ? '+' : '') + s.delta + ')' : '';
+    const noise = s.stderr !== undefined ? ' [se ' + s.stderr + (s.withinNoise ? ', within noise' : '') + ']' : '';
+    return k + ' ' + (s.base !== undefined ? s.base : '?') + ' -> ' + (s.head !== undefined ? s.head : '?') + delta + noise + (s.regressed ? ' REGRESSED' : '');
+  }).join(', ');
+}
+
 function corpusLines(sub, data = {}) {
   const lines = ['Corpus: ' + (data.corpus || '?') + ' (owner account ' + (data.ownerAccountId || '?') + ')'];
   if (sub === 'cases') {
@@ -10351,13 +10480,25 @@ function corpusLines(sub, data = {}) {
     cases.forEach((c) => lines.push('- ' + c.caseKey + (c.split ? '  [' + c.split + ']' : '') + (c.tags && c.tags.length ? '  tags ' + c.tags.join(',') : '') +
       '  expected ' + (c.hasExpected ? (c.expectedStatus || 'present') : 'none') + '  artifacts ' + (c.artifacts || []).map((a) => a.name + ':' + a.bytes + 'b').join(' ')));
   } else if (sub === 'results') {
-    (data.results || []).forEach((r) => lines.push('- ' + r.caseKey + '  ' + (r.outcome || '?') + (r.score !== undefined ? '  score ' + r.score : '') +
-      '  run ' + r.taskId + '  ' + corpusAiLabel(r) + (r.durationMs ? '  ' + formatDurationMs(r.durationMs) : '') +
+    (data.results || []).forEach((r) => lines.push('- ' + r.caseKey + (r.repeatIndex ? ' #' + r.repeatIndex : '') + '  ' + (r.outcome || '?') + (r.score !== undefined ? '  score ' + r.score : '') +
+      '  run ' + (r.runGroupId || r.taskId) + (r.runLabel ? ' (' + r.runLabel + ')' : '') + (r.split ? '  [' + r.split + ']' : '') + '  ' + corpusAiLabel(r) + (r.durationMs ? '  ' + formatDurationMs(r.durationMs) : '') +
       '  metrics ' + String(r.metricsHash || '').slice(0, 10) + '  ' + formatTime(r.recordedAt)));
   } else if (sub === 'runs') {
-    (data.runs || []).forEach((r) => lines.push('- ' + r.taskId + '  ' + r.cases + ' case(s)  ' + Object.keys(r.outcomes || {}).map((k) => k + ' ' + r.outcomes[k]).join(', ') +
+    (data.runs || []).forEach((r) => lines.push('- ' + r.taskId + (r.label ? '  "' + r.label + '"' : '') + '  ' + r.cases + ' case(s)' +
+      (r.repeats ? ' x' + r.repeats + ' repeats' : '') + (r.shards ? '  ' + r.shards + ' shards' : '') + '  ' +
+      Object.keys(r.outcomes || {}).map((k) => k + ' ' + r.outcomes[k]).join(', ') + (r.passRate !== undefined ? '  pass rate ' + r.passRate : '') +
       '  ' + corpusAiLabel(r) + '  ' + (r.dataMode || '?') + '  ' + (r.branchName || '?') + (r.workspace ? ' [ws:' + r.workspace + ']' : '') +
       (r.laneContentHash ? '  lane ' + String(r.laneContentHash).slice(0, 12) : '') + '  ' + formatTime(r.recordedAt)));
+    (data.runs || []).forEach((r) => {
+      const extra = [];
+      if (r.params && Object.keys(r.params).length) extra.push('params ' + Object.keys(r.params).map((k) => k + '=' + JSON.stringify(r.params[k])).join(' '));
+      if (r.scores && Object.keys(r.scores).length) extra.push('scores ' + corpusScoreSummary(r.scores));
+      if (r.bySplit) extra.push('by split ' + Object.keys(r.bySplit).map((sp) => sp + ': ' + r.bySplit[sp].cases + ' case(s)' +
+        (r.bySplit[sp].passRate !== undefined ? ' pass ' + r.bySplit[sp].passRate : '') +
+        (r.bySplit[sp].scores ? ' ' + corpusScoreSummary(r.bySplit[sp].scores) : '')).join('; '));
+      if (r.review) extra.push('graded ' + r.review.graded + ', spot-checked ' + r.review.reviewed + (r.review.agreement !== undefined ? ' (agreement ' + r.review.agreement + ')' : ''));
+      if (extra.length) lines.push('    ' + r.taskId + ': ' + extra.join('  '));
+    });
     if ((data.runs || []).some((r) => r.aiMode === 'mocked')) {
       lines.push('Runs marked AI MOCKED replayed every AI turn from aiMock - they measure the harness, not the model.');
     }
@@ -10366,10 +10507,13 @@ function corpusLines(sub, data = {}) {
     lines.push('Base ' + cmp.baseTaskId + ' -> head ' + cmp.headTaskId + ': ' + Object.keys(cmp.counts || {}).map((k) => k + ' ' + cmp.counts[k]).join(', '));
     lines.push('AI mode: base ' + (cmp.baseAiMode || '?') + ', head ' + (cmp.headAiMode || '?'));
     if (cmp.aiModeChanged) lines.push('WARNING: ' + cmp.aiModeChanged);
-    (cmp.cases || []).filter((c) => c.direction !== 'same').forEach((c) => {
+    if (cmp.scores && Object.keys(cmp.scores).length) lines.push('Scores (cases both runs measured): ' + corpusScoreDeltas(cmp.scores));
+    (cmp.cases || []).filter((c) => c.direction !== 'same' || c.changedScores).forEach((c) => {
       lines.push('- ' + c.caseKey + '  ' + c.direction + (c.base ? '  ' + c.base.outcome : '') + (c.head ? ' -> ' + c.head.outcome : '') +
+        (c.base && c.base.passRate !== undefined ? '  pass ' + c.base.passRate + ' -> ' + (c.head && c.head.passRate !== undefined ? c.head.passRate : '?') : '') +
         (c.expectedChanged ? '  (EXPECTED ANSWER CHANGED between runs)' : ''));
       (c.changedMetrics || []).slice(0, 8).forEach((m) => lines.push('    ' + m.metric + ': ' + JSON.stringify(m.base) + ' -> ' + JSON.stringify(m.head)));
+      Object.keys(c.changedScores || {}).slice(0, 8).forEach((k) => lines.push('    score ' + k + ': ' + c.changedScores[k].base + ' -> ' + c.changedScores[k].head));
       if (c.head && c.head.outcomeReason && c.head.outcome !== 'passed') lines.push('    reason: ' + String(c.head.outcomeReason).slice(0, 300));
     });
   } else if (sub === 'retire') {
@@ -10380,17 +10524,41 @@ function corpusLines(sub, data = {}) {
     if ((data.cases || []).some((c) => c.status === 'retired' || c.status === 'restored')) {
       lines.push('Retired cases stay readable in results and compare; they are just no longer handed to runs. `corpus cases --include-retired` lists them, `corpus retire --restore` puts them back.');
     }
+  } else if (sub === 'gate') {
+    const gate = data.gate || {};
+    lines.push('Gate ' + (gate.baseTaskId || '?') + ' -> ' + (gate.headTaskId || '?') + ': ' + (gate.passed ? 'PASSED' : 'DID NOT PASS') +
+      (gate.tolerance !== undefined ? '  (tolerance ' + gate.tolerance + ')' : '') + (gate.noiseSigma !== undefined ? '  (noise sigma ' + gate.noiseSigma + ')' : '') +
+      (gate.split ? '  split ' + gate.split : ''));
+    if (gate.scores && Object.keys(gate.scores).length) lines.push('Scores: ' + corpusScoreDeltas(gate.scores));
+    (gate.reasons || []).forEach((r) => lines.push('- ' + r));
+    (gate.notes || []).forEach((n) => lines.push('Note: ' + n));
+    if (gate.expectedChanged && gate.expectedChanged.length) lines.push('Left out (expected answer changed between runs): ' + gate.expectedChanged.join(', '));
+    (gate.changedCases || []).slice(0, 15).forEach((c) => lines.push('  ' + c.caseKey + '  ' + c.direction +
+      Object.keys(c.changedScores || {}).map((k) => '  ' + k + ' ' + c.changedScores[k].base + '->' + c.changedScores[k].head).join('')));
+  } else if (sub === 'review') {
+    if (data.review) {
+      lines.push('Recorded: ' + data.review.caseKey + ' in run ' + data.review.taskId + ': ' + data.review.verdict + (data.review.score !== undefined && data.review.score !== null ? ' (score ' + data.review.score + ')' : ''));
+    } else {
+      const sample = data.sample || [];
+      lines.push('Spot-check sample: ' + sample.length + ' graded case(s), unreviewed first');
+      sample.forEach((c) => {
+        const g = c.grade || {};
+        lines.push('- ' + c.caseKey + '  grade ' + (g.verdict || '?') + ' ' + (g.score !== undefined ? g.score : '') + (c.review ? '  reviewed: ' + c.review.verdict : '  NOT REVIEWED'));
+        if (g.reasons) lines.push('    ' + String(g.reasons).slice(0, 300));
+      });
+      if (sample.length) lines.push('Record a check: remits-cli corpus review --corpus ' + (data.corpus || '<name>') + ' --task-id <taskId> --case <caseKey> --verdict agree|disagree [--score n] [--note "..."]');
+    }
   } else if (sub === 'consistency') {
     const con = data.consistency || {};
     lines.push('Case ' + con.caseKey + ': ' + con.runs + ' run(s), ' + (con.stable ? 'STABLE (identical metrics every run)' : (con.variants || []).length + ' distinct metric results') +
       (con.aiModes && con.aiModes.length ? '  AI ' + con.aiModes.join('+') : ''));
     if (con.note) lines.push('WARNING: ' + con.note);
-    (con.variants || []).forEach((v) => lines.push('- metrics ' + String(v.metricsHash).slice(0, 10) + '  runs ' + (v.runs || []).map((r) => r.taskId + ':' + r.outcome + (r.aiMode && r.aiMode !== 'none' ? '/' + r.aiMode : '')).join(', ')));
+    (con.variants || []).forEach((v) => lines.push('- metrics ' + String(v.metricsHash).slice(0, 10) + '  runs ' + (v.runs || []).map((r) => r.taskId + (r.repeatIndex ? '#' + r.repeatIndex : '') + ':' + r.outcome + (r.aiMode && r.aiMode !== 'none' ? '/' + r.aiMode : '')).join(', ')));
   }
   return lines;
 }
 
-// `corpus import|cases|results|runs|compare|consistency`: test corpora through the platform's own corpus(name).
+// `corpus import|cases|results|runs|compare|consistency|retire|gate|review`: test corpora through the platform's own corpus(name).
 async function corpusCommand(flags, subcommand) {
   const cwd = process.cwd();
   ensureLocalState(cwd);
@@ -10495,12 +10663,16 @@ async function corpusCommand(flags, subcommand) {
     const keys = parseCorpusFilterValues(flags, ['key', 'keys', 'case', 'case-key']);
     data = await post({ command: 'cases', corpus: corpusName, split: flags.split, tags: tags.length ? tags : undefined, keys: keys.length ? keys : undefined, includeValues: flagEnabled(flags['include-values']), includeRetired: flagEnabled(flags['include-retired']), limit: flags.limit });
   } else if (sub === 'results') {
-    data = await post({ command: 'results', corpus: corpusName, caseKey: flags.case || flags['case-key'], taskId: flags['task-id'], limit: flags.limit || 50 });
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
+    data = await post({ command: 'results', corpus: corpusName, caseKey: flags.case || flags['case-key'], taskId: flags['task-id'], label: flags.label,
+      split: flags.split, tags: tags.length ? tags : undefined, limit: flags.limit || 50 });
   } else if (sub === 'runs') {
-    data = await post({ command: 'runs', corpus: corpusName, limit: flags.limit || 20 });
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
+    data = await post({ command: 'runs', corpus: corpusName, label: flags.label, split: flags.split, tags: tags.length ? tags : undefined, limit: flags.limit || 20 });
   } else if (sub === 'compare') {
     if (!flags.base || !flags.head) throw new Error('corpus compare requires --base <taskId> --head <taskId>');
-    data = await post({ command: 'compare', corpus: corpusName, base: flags.base, head: flags.head });
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
+    data = await post({ command: 'compare', corpus: corpusName, base: flags.base, head: flags.head, split: flags.split, tags: tags.length ? tags : undefined });
   } else if (sub === 'consistency') {
     const caseKey = flags.case || flags['case-key'];
     if (!caseKey) throw new Error('corpus consistency requires --case <caseKey>');
@@ -10510,8 +10682,40 @@ async function corpusCommand(flags, subcommand) {
     if (!retireKeys.length) throw new Error('corpus retire requires --case <caseKey> (repeatable) or --keys a,b,c');
     data = await post({ command: 'retire', corpus: corpusName, keys: retireKeys, restore: flagEnabled(flags.restore) });
     if (data && data.success === false) process.exitCode = 1;
+  } else if (sub === 'gate') {
+    const scores = parseCorpusFilterValues(flags, ['score', 'scores']);
+    const lowerIsBetter = parseCorpusFilterValues(flags, ['lower-is-better']);
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
+    data = await post({ command: 'gate', corpus: corpusName, base: flags.base, head: flags.head, tolerance: flags.tolerance,
+      scores: scores.length ? scores : undefined,
+      lowerIsBetter: lowerIsBetter.length ? lowerIsBetter : undefined,
+      allowModeChange: flagEnabled(flags['allow-mode-change']),
+      baseLabel: flags['base-label'], headLabel: flags['head-label'], split: flags.split, tags: tags.length ? tags : undefined,
+      minCases: flags['min-cases'], noiseSigma: flags['noise-sigma'] });
+    const gate = (data && data.gate) || {};
+    if (!gate.passed) process.exitCode = 1;
+    await appendVerificationPacket(api, cwd, session, accountId, flags, {
+      type: 'corpus_gate',
+      success: !!gate.passed,
+      claim: 'Corpus ' + corpusName + ' gate ' + (gate.baseTaskId || '?') + ' -> ' + (gate.headTaskId || '?') + ': ' + (gate.passed ? 'passed' : 'did not pass'),
+      world: Object.assign({}, (data && data.world) || {}, { host: normalizeBaseUrl(baseUrl), branchName, workspace, dataMode }),
+      revision: collectVerificationSource(cwd, flags),
+      evidenceCategories: ['corpus_gate'],
+      corpusGate: { corpus: corpusName, baseTaskId: gate.baseTaskId, headTaskId: gate.headTaskId, passed: !!gate.passed, scores: gate.scores, reasons: gate.reasons,
+        split: gate.split, noiseSigma: gate.noiseSigma, minCases: gate.minCases, notes: gate.notes }
+    }, { quiet: jsonOutput });
+  } else if (sub === 'review') {
+    const taskId = flags['task-id'];
+    if (!taskId) throw new Error('corpus review requires --task-id <taskId> (and --case <caseKey> --verdict agree|disagree to record a check)');
+    if (flags.verdict) {
+      const caseKey = flags.case || flags['case-key'];
+      if (!caseKey) throw new Error('corpus review --verdict requires --case <caseKey>');
+      data = await post({ command: 'review', corpus: corpusName, taskId, caseKey, verdict: flags.verdict, score: flags.score, note: flags.note, repeatIndex: flags.repeat });
+    } else {
+      data = await post({ command: 'review', corpus: corpusName, taskId, size: flags.sample || flags.size });
+    }
   } else {
-    throw new Error('Unknown corpus subcommand: ' + sub + ' (use import, cases, results, runs, compare, consistency or retire)');
+    throw new Error('Unknown corpus subcommand: ' + sub + ' (use import, cases, results, runs, compare, consistency, retire, gate or review)');
   }
   if (jsonOutput) {
     console.log(JSON.stringify(data, null, 2));
@@ -18108,7 +18312,7 @@ function printComponentsHelp(subcommand) {
 }
 
 function printTestHelp() {
-  console.log('Usage: remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--json]');
+  console.log('Usage: remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--label TEXT] [--param name=value ...] [--repeat N] [--parallel N] [--json]');
   console.log('       remits-cli test status --task-id <taskId> [--base-url URL] [--account-id ID] [--branch stagingScope] [--data-mode test|prod] [--json]');
   console.log('       remits-cli test runs [--test <id|name>] [--limit 20] [--compare] [--json]   # durable run history');
   console.log('       remits-cli test compare --base <taskId> --head <taskId> [--json]            # per-case outcome/cost deltas');
@@ -18139,11 +18343,14 @@ function printTestHelp() {
 function printCorpusHelp() {
   console.log('Usage: remits-cli corpus import --manifest corpus-manifest.json [--corpus <name>] [--as-account ID] [--data-mode test|prod --confirm-prod] [--json]');
   console.log('       remits-cli corpus cases --corpus <name> [--split dev] [--tag name] [--key caseKey] [--include-values] [--include-retired] [--json]');
-  console.log('       remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId>] [--limit 50] [--json]');
-  console.log('       remits-cli corpus runs --corpus <name> [--limit 20] [--json]');
-  console.log('       remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--json]');
+  console.log('       remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId|groupId>] [--label L] [--split S] [--tag T] [--limit 50] [--json]');
+  console.log('       remits-cli corpus runs --corpus <name> [--label L] [--split S] [--tag T] [--limit 20] [--json]');
+  console.log('       remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--split S] [--tag T] [--json]');
   console.log('       remits-cli corpus consistency --corpus <name> --case <caseKey> [--limit 20] [--json]');
   console.log('       remits-cli corpus retire --corpus <name> --case <caseKey> [--case ...] [--restore] [--json]');
+  console.log('       remits-cli corpus gate --corpus <name> [--base <taskId> | --base-label L] [--head <taskId> | --head-label L] [--split holdout] [--tag T]');
+  console.log('              [--score answer --score passRate] [--tolerance 0.02] [--lower-is-better <name>] [--min-cases N] [--noise-sigma 2] [--allow-mode-change] [--json]');
+  console.log('       remits-cli corpus review --corpus <name> --task-id <taskId> [--sample 5] | --case <caseKey> [--repeat N] --verdict agree|disagree [--score n] [--note "..."]');
   console.log('');
   console.log('A corpus is a named, durable set of cases an evaluation-style Test runs again and again (front stage:');
   console.log('corpus(name)). Cases live in a platform table owned by one account and visible to the accounts beneath');
@@ -18404,7 +18611,7 @@ async function main() {
     console.log('  remits-cli components branch <name> --subscribe <accountId> [--dry-run] [--confirm-primary-edge]');
     console.log('  remits-cli components branch <name> --unsubscribe <accountId>    # return that account to trunk');
     console.log('  remits-cli components branch <name> --retire [--force]           # delete the branch\'s overlays');
-    console.log('  remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--json]');
+    console.log('  remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--label TEXT] [--param name=value ...] [--repeat N] [--parallel N] [--json]');
     console.log('  remits-cli test runs [--test <id|name>] [--compare]                 # durable run history; compare latest two');
     console.log('  remits-cli test compare --base <taskId> --head <taskId>');
     console.log('  remits-cli corpus import --manifest corpus-manifest.json             # seed a test corpus: cases + artifacts');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remits/remits-cli",
-  "version": "0.1.156",
+  "version": "0.1.158",
   "description": "Local CLI for auth, component sync, and live test execution against Remits",
   "license": "MIT",
   "private": false,

package/skills/remits-cli/SKILL.md

@@ -46,6 +46,8 @@ reference below, load it before you act, not after something surprises you.
 | work from a non-trunk checkout, or promote a branch | `branch-variants.md` | what a variant is, which world your checkout resolves, `--as-account` vs `--variant-branch`, tombstones, the promotion loop and its phases, subscribe/retire |
 | touch a support ticket | `support-tickets.md` | the ticket record and its verbs, routed vs claimed vs owned, autonomous workers, the repository edit lease, ask/answer/resume, delivery |
 | register this session as an agent, or ask why a routed ticket never started | `agent-sessions.md` | `serve` vs `register`, what registration does, worker spawning per agent kind, routing order, the control center, multi-session auth |
+| tune, improve or "fine-tune" a workflow with evals, or compare two variants of one | `development-loop.md` → *Evaluation runs* | `--label`/`--param`/`--repeat`/`--parallel`, the baseline → candidate → gate loop, the holdout rule; the full playbook is `guides/test-components.md` → *Tuning A Workflow With Evals* |
+| build or evaluate cited Q&A, review learnings, or investigate answer quality | `ai-quality.md` | front-stage guide routing, interaction/session pivots, baseline vs proposal measurements, human checks and explicit corpus gates |
 | investigate live behavior | `investigation.md` | which tool reads which record, correlation keys, reading a record's `content`, HTTP audits, AI activity, node/`localMode`, the production support flows |
 | call any `mcp_*` tool | `tool-reference.md` | every tool's parameters, semantics, and traps — read the tool's entry before building its input |
 | run, describe, poll, or interrupt an Action from the CLI | `tool-reference.md` → `mcp_run_action` | Action execution through `remits-cli tool --name mcp_run_action`; there is no separate top-level `remits-cli action` / `run action` wrapper |
@@ -158,12 +160,11 @@ reference named after it.
   (`component-resolution.md`, `command-reference.md`)
 - **Host and data mode are two independent decisions.** `--base-url` picks the Remits host,
   `--data-mode` picks the data segment on it. Neither implies the other. (`command-reference.md`)
-- **Localhost being down is not a verification blocker.** `remits-cli` can talk to any reachable Remits
-  host, including the deployed production host, by passing `--base-url`. If `http://localhost:8080`
-  refuses a connection, retry the same stage/test/tool/status command against the intended deployed host
-  with the same account, branch, workspace and data-mode facts; do not stop unless no reachable host or
-  valid session exists. A deployed host can still run test-lane verification: `--base-url` selects the
-  server, while `test run` still defaults to `--data-mode test`. (`command-reference.md`)
+- **Use a reachable host that has the implementation you are verifying.** For component work against
+  existing capabilities, localhost being down need not block verification: pass the intended deployed
+  host with `--base-url`, preserving account, branch, workspace and data-mode facts. For unreleased
+  platform capabilities, a deployed old binary is not a substitute; use the updated local/test runtime
+  and follow that repository's startup/restart instructions. (`command-reference.md`, `ai-quality.md`)
 - **`test run` ignores the stored session data lane.** It defaults to `test` even when `whoami` shows
   the session parked on prod; production Test runs require explicit prod provenance (`--data-mode prod`)
   or Test source declaring `dataMode 'prod'` / `[dataMode:'prod']`. Check returned `dataModeSource`

package/skills/remits-cli/references/ai-quality.md

@@ -0,0 +1,128 @@
+# Answer Quality And Reviewed Learning
+
+Load this when building cited Q&A, investigating a wrong answer, changing retrieval or prompts,
+or measuring a proposed learning before review. Component APIs belong in the account's front-stage
+guides; this reference owns the CLI investigation and measurement workflow.
+
+## Table of Contents
+
+- [Find the right guide](#find-the-right-guide)
+- [Investigate one answer](#investigate-one-answer)
+- [Measure a change](#measure-a-change)
+- [Review and gate](#review-and-gate)
+- [Keep the regression](#keep-the-regression)
+
+## Find the right guide
+
+| Building or changing | Read in the account repository |
+|---|---|
+| Search Tool → citations → feedback/review page | `guides/features/ai-interactions.md` → *Wiring a question-to-review workflow* |
+| Learning proposals, review, retirement, Agent memory | `guides/features/agent-memory.md` → *Reviewing learnings* |
+| Rewrite, learned wording, rerank, answer spans, retrieval misses | `guides/features/schema-vertex-ai-extension.md` → *Retrieval quality* |
+| Named retrieval/answer scores, rubric grader, baseline/candidate design | `guides/test-components.md` → *Measuring a proposed learning* and *Human Review And Release Gates* |
+| Per-question guidance and timeout recovery | `guides/agent-components.md` → *ai.turnContext* and *Wiring tools to an agent* |
+
+Use the built-in interaction and learning stores. `interactions()` and `memories()` are component DSL
+builders, not top-level CLI commands. Call them from an application's Action/review UI or a Test;
+do not bypass their scope and review behavior with SQL. The CLI has `corpus review` and `corpus gate`.
+
+## Investigate one answer
+
+Start from the answer's `interactionId`, its sources and feedback. `interactions().trace(id)` gives
+its provider rounds and tool calls; `sessionId` and `traceId` pivot into `mcp_ai_session_search`
+(`investigation.md`, then that tool's entry in `tool-reference.md`). Inspect what the model actually
+received before changing a prompt. For normalization, `unstructured()` also returns `aiSessionId`
+when a model was called, plus `traceId`.
+
+Decide what failed before choosing a remedy:
+
+- Expected source missing: inspect the actual retrieval and `index_explain` diagnostics in the
+  authorized data lane. Test documents are not indexed. Diagnose eligibility/indexing before tuning
+  wording; `not_reached` describes a bounded search window, not proof that a document is irrelevant.
+- Source retrieved but answer wrong: inspect the offered passages, per-turn instructions and answer
+  trace. A citation is not proof that the source supports the claim.
+- Coverage absent: include a should-decline corpus case; a grounded decline can be the correct result.
+- Terminology mismatch: propose `kind: 'phrasing'` with indexed wording in `details` and a source
+  document id in `subjectRef`, then measure it. Feedback alone does not alter retrieval.
+
+`quality.rerank.applied: false` is fallback, not successful reranking. `indexSearchMock` bypasses
+rewriting, learned retrieval and ranking; it proves how the caller uses supplied search results.
+Use live-provider evidence for claims about those steps. Ranking charges are outside chat-token budgets.
+
+## Measure a change
+
+Use the existing account Test/corpus workflow. Keep durable regression components in the account
+repository, and stage the workset in a dedicated workspace. Establish host, execution account, source
+layer and data lane before running (`components status`). Compare the same cases, expected answers,
+source inputs and model settings. Retrieve and score through the actual application path; a parallel
+search written just for the evaluator may measure something the answering Agent never saw.
+
+1. Capture a baseline with approved memories only. Keep each question in a fresh Agent session.
+2. Make the source/prompt change, or enable `includeProposed` in the Test's Agent/retrieval path.
+   Preview includes all relevant proposals in scope, not one selected id. Keep the proposal set
+   stable. Learning records are shared across data lanes; a workspace does not isolate them.
+3. Re-stage and run the candidate under an explicit `withAiBudget` in the Test, with the same flags as
+   the baseline and a different `--label`. Keep per-case `scores: [retrieval: ..., answer: ...]`, grader
+   results and retrieval diagnostics.
+
+```bash
+remits-cli components stage --workset --account-id 1 --base-url http://localhost:8080 --data-mode test
+remits-cli test run --test "Support Answer Evaluation" --label baseline --repeat 3 --parallel 2 --account-id 1 --base-url http://localhost:8080 --data-mode test --json
+# make the change (or pass it with --param name=value), re-stage, then:
+remits-cli test run --test "Support Answer Evaluation" --label candidate --repeat 3 --parallel 2 --account-id 1 --base-url http://localhost:8080 --data-mode test --json
+```
+
+`--repeat 3` runs each question three times. Live answers and grades vary from run to run, and the repeats are
+what give each score its `±` noise estimate. `--parallel 2` runs the cases in two shards under one `grp-…` id.
+
+The suite name and account above are examples: create/use your application's suite and resolved account.
+Repeat after the candidate edit and stage. `--test` selects a suite; `--names` selects exact case names
+(or a `|`-separated list), not substrings or a regex. Check `total` and outcomes; a zero-case run proves
+nothing. `--wait false` only launches; finish with `test status --task-id <id>`.
+
+Use deterministic search/AI mocks for protocol regressions and live models against pinned corpus
+artifacts for answer quality. Real index measurements need an explicitly authorized production data
+lane and a read-only application path; changing the lane of a workflow with writes is not a shortcut.
+The intended host must also have the implementation being verified: a deployed old binary cannot
+verify unreleased platform changes, even with the same account and test lane.
+
+## Review and gate
+
+With the same explicit host/account/data-mode options as the runs:
+
+```bash
+remits-cli corpus runs --corpus support-golden --limit 2                     # each run's id (grp-...), label, scores ± noise
+remits-cli corpus compare --corpus support-golden --base BASE_TASK_ID --head CANDIDATE_TASK_ID
+remits-cli corpus review --corpus support-golden --task-id CANDIDATE_TASK_ID --sample 5
+remits-cli corpus review --corpus support-golden --task-id CANDIDATE_TASK_ID --case CASE_KEY --verdict agree --note "Checked against the source"
+remits-cli corpus gate --corpus support-golden --base-label baseline --head-label candidate --split holdout --score retrieval --score answer --noise-sigma 2
+remits-cli evidence
+```
+
+Replace uppercase ids with the ids `corpus runs` prints. A parallel run's `grp-…` id is accepted wherever a task
+id is. The gate can select runs by label instead of id. Run it on `holdout`, the split you did not tune against. Each repeated `--score` is a required metric;
+`--scores retrieval,answer` is equivalent. Name the baseline and candidate by id or by label; never rely on the gate's "latest two runs" default.
+The gate exits nonzero for regressions, lost/absent measurements or an unaccepted mocked/live mode
+change. It records `corpus_gate` evidence. Inspect measured case counts and excluded changed expected
+answers; also assess the application's minimum accuracy and the human spot checks. A non-regression
+gate does not establish an absolute quality target. With `--noise-sigma 2` (both runs launched with `--repeat`),
+a move only counts as a regression when it is larger than the run-to-run noise. Without repeats, the gate's notes
+say that a small difference may be noise.
+
+Stop a release workflow on a failing gate. Approval and component sync do not automatically invoke it.
+Human review preserves the model grade; it does not replace it. Approve/reject through the application's
+`memories()` review path after the decision, and keep the relevant task ids in the review note. Editing
+an approved memory affects later answers immediately; use a proposal when a replacement needs measurement.
+
+## Keep the regression
+
+Use `auxiliary: false` for durable Tests and their helper components. Set `regression: true` for suites
+the account expects in its normal regression, including explicitly budgeted provider suites when
+their cost/dependency is intentional. Otherwise leave costly/exploratory measurements opt-in.
+Describe the proof boundary in the sidecar and README: mocked protocol, live provider smoke,
+measured corpus quality, or UI journey. Retire/reject synthetic memories in cleanup.
+
+Read `component-integrity.md` before landing. Source files alone do not update the platform's durable
+regression selection: use normal account commit/push/sync when authorized, or stage for local runs.
+Read `evidence` before reporting results; distinguish checked-in/staged source, measured behavior,
+human approval and deployment.

package/skills/remits-cli/references/command-reference.md

@@ -65,8 +65,10 @@ Do not assume `--data-mode prod` implies the deployed prod host, or that `--data
 localhost. Likewise, do not assume localhost is required because the current checkout is a back-stage
 repo or because a previous command used localhost. If host matters, read `~/.remits-cli/sessions.json`,
 `remits-cli whoami`, or the recent `remits-cli evidence` world blocks, then pass `--base-url` explicitly.
-Only call verification blocked after trying the intended reachable host and finding that no authenticated
-session or usable network path exists.
+For existing capabilities, try the intended reachable host before declaring a connection blocker.
+For unreleased platform capabilities, also verify that the host has the required implementation.
+Staged component source cannot add missing platform APIs to an older deployed binary; use the updated
+local/test runtime and follow that platform repository's startup/restart instructions.
 
 ### Tool Execution Lifecycle
 
@@ -257,15 +259,18 @@ remits-cli components branch <name> --copy-to <newBranch> [--dry-run] [--force]
 remits-cli components branch <name> --subscribe <accountId> [--parent-account <id>] [--domain <host>] [--dry-run] [--confirm-primary-edge]   # make an account resolve this branch
 remits-cli components branch <name> --unsubscribe <accountId>                          # return that account to trunk
 remits-cli components branch <name> --retire [--force]                                 # delete the branch's overlays
-remits-cli test run --test <id|name> [--branch <stagingScope>] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account <ID>] [--variant-branch <name|none>] [--json]
-remits-cli test status --task-id <taskId> [--branch <stagingScope>] [--data-mode test|prod] [--json]   # falls back to the DURABLE run record once the live status expires
+remits-cli test run --test <id|name> [--branch <stagingScope>] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account <ID>] [--variant-branch <name|none>] [--label <text>] [--param name=value ...] [--repeat N] [--parallel N] [--json]
+remits-cli test status --task-id <taskId|groupId> [--branch <stagingScope>] [--data-mode test|prod] [--json]   # falls back to the DURABLE run record once the live status expires
 remits-cli test runs [--test <id|name>] [--limit 20] [--compare] [--all-lanes] [--as-account <ID>] [--json]  # durable run history: pass counts, live AI cost, lane, content hash
 remits-cli test compare --base <taskId> --head <taskId> [--json]                                            # per-case improved / regressed / changed, cost and world deltas
 remits-cli corpus import --manifest corpus-manifest.json [--corpus <name>] [--as-account <ID>] [--data-mode test|prod --confirm-prod] [--json]   # seed an evaluation corpus: cases + immutable artifacts; idempotent by caseKey
 remits-cli corpus cases --corpus <name> [--split S] [--tag T|--tags T,U] [--key K|--keys K,L] [--include-values] [--include-retired] [--limit N] [--json]
-remits-cli corpus runs --corpus <name> [--limit 20] [--json]                                   # runs that measured the corpus: outcomes, AI mode (live|MOCKED|none), cost, world
-remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId>] [--json]
-remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--json]              # per-case direction, metric deltas, expectedChanged
+remits-cli corpus runs --corpus <name> [--label L] [--split S] [--tag T] [--limit 20] [--json] # runs that measured the corpus: label, repeats, shards, pass rate, scores ± noise, per split, AI mode (live|MOCKED|none), cost, world
+remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId|groupId>] [--label L] [--split S] [--tag T] [--json]
+remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--split S] [--tag T] [--json]   # per-case direction, pass rate, metric deltas, expectedChanged
+remits-cli corpus gate --corpus <name> [--base <taskId> | --base-label L] [--head <taskId> | --head-label L] [--split holdout] [--tag T] [--score retrieval --score passRate] [--lower-is-better <name>] [--tolerance 0.02] [--min-cases N] [--noise-sigma 2] [--allow-mode-change] [--json]
+remits-cli corpus review --corpus <name> --task-id <taskId> [--sample 5] [--json]
+remits-cli corpus review --corpus <name> --task-id <taskId> --case <key> [--repeat N] --verdict agree|disagree [--score n] [--note "..."]
 remits-cli corpus consistency --corpus <name> --case <caseKey> [--json]                        # did the case produce the same metrics every run?
 remits-cli corpus retire --corpus <name> --case <caseKey> [--case ...] [--restore] [--json]    # drop a case from future runs; past measurements stay readable
 ```
@@ -318,6 +323,20 @@ of the same **shape** - same data lane, branch, workspace and case count - and s
 skipped; it refuses rather than comparing across worlds. `test status --data-mode prod` on a run recorded in
 the test lane returns the run's real lane and now says that is what happened.
 
+For the baseline → proposed learning → human review workflow, see `ai-quality.md` and
+`guides/test-components.md` → *Measuring a proposed learning*. `--score` is repeatable;
+`--score retrieval --score answer` requires both metrics (`--scores retrieval,answer` is equivalent).
+`interactions()` and `memories()` are component DSL APIs, not top-level CLI subcommands.
+
+`corpus gate` exits nonzero for a score regression, missing measurements, too few shared cases
+(`--min-cases`), a missing noise estimate (`--noise-sigma` without `--repeat` on both runs), or an unaccepted
+AI-mode change, and records `corpus_gate` evidence. `passRate`, `liveAiCost`, `liveAiCalls` and `durationMs` are
+built-in scores every case carries; the last three are lower-is-better without `--lower-is-better`.
+`--base-label`/`--head-label` select the latest run with that `test run --label`. Every corpus command accepts a
+parallel run's `grp-…` id wherever it accepts a task id. Run it explicitly before sync or approving a learning;
+neither operation invokes it automatically. Prefer explicit run ids and check their world.
+`corpus review` samples graded cases or records a human spot check; it preserves the model grade.
+
 A run labelled **`AI MOCKED`** replayed every AI turn from `aiMock`: it produces the same outcomes, scores and
 metrics a measured run would, so read that label before treating green as evidence. `compare` warns when base and
 head used different AI modes; `consistency` warns before calling a case unstable when its runs mixed modes.

package/skills/remits-cli/references/development-loop.md

@@ -20,6 +20,7 @@
       - [Stage your workset, not the whole repo](#stage-your-workset-not-the-whole-repo)
       - [Three numbers, three questions](#three-numbers-three-questions)
     - [Step 4: Verify the Change](#step-4-verify-the-change)
+      - [Evaluation runs](#evaluation-runs)
       - [Many failures are usually few causes](#many-failures-are-usually-few-causes)
     - [Step 5: Iterate If Needed](#step-5-iterate-if-needed)
     - [Step 6: Update Documentation](#step-6-update-documentation)
@@ -68,6 +69,10 @@ The right model is:
 
 Never treat "it looks right in prod data inspection" as sufficient proof that a code change is verified.
 
+For cited answers, retrieval changes or reviewed learnings, load `ai-quality.md`. It connects the
+component workflow to measured baseline/candidate runs, human checks and `corpus gate`; a passing
+mocked Test alone cannot establish improved answer quality.
+
 ## Verification Envelopes
 
 For a concrete user workflow, browser-facing change, branch-variant release, support ticket, or any loop
@@ -426,11 +431,38 @@ about a different world.
 
 The platform launch is asynchronous. By default `remits-cli test run` waits for its task to finish, and any
 cases selected by `--names "a|b"` execute sequentially inside that suite task. If several slow cases are
-independent, stage once, then start separate `test run --names "<case>" --wait false` commands from the same
-lane and keep the task ids they print. Complete each proof with
-`remits-cli test status --task-id <id>`; that terminal status read records and attaches the final `test_run`
-evidence. Do not parallelize cases that mutate the same fixture, rely on shared suite setup state, or make
-undeclared live AI/provider calls.
+independent, use `--parallel N`. It splits the selected cases across N shards that run side by side, and the run
+keeps ONE id (`grp-…`) for `test status` and every corpus command. An instance runs only a few suites at once
+(two by default), so shards beyond that wait for a slot. Ask for 2–4. Starting separate
+`test run --names "<case>" --wait false` commands also works for one-off proofs, but every such run is a separate
+run: a corpus cannot compare or gate them as one. Do not parallelize cases that mutate the same fixture, rely on
+shared suite setup state, or make undeclared live AI/provider calls.
+
+##### Evaluation runs
+
+When the job is to improve a workflow ("tune it", "make it more accurate", "compare these two prompts"), the unit
+of evidence is a measured corpus run, not a green suite. Follow this loop:
+
+```bash
+remits-cli test run --test "<eval suite>" --label baseline --repeat 3 --parallel 4      # live, inside withAiBudget
+remits-cli corpus runs --corpus <name> --split dev                                      # pass rate, scores ± noise
+remits-cli corpus results --corpus <name> --label baseline --split dev                  # failing cases -> trace ids
+# change ONE thing (or pass the variant with --param name=value), re-stage, then:
+remits-cli test run --test "<eval suite>" --label candidate --repeat 3 --parallel 4
+remits-cli corpus runs --corpus <name> --limit 2                                         # the two runs' grp-ids
+remits-cli corpus compare --corpus <name> --base <baseline grp-id> --head <candidate grp-id>
+remits-cli corpus gate --corpus <name> --base-label baseline --head-label candidate --split holdout --noise-sigma 2
+```
+
+Rules:
+
+- Tune against `dev`, and gate on `holdout` only once you are done.
+- Keep the lane, the AI mode and the corpus identical between the two runs.
+- Read a failing case's AI session before you change a prompt.
+- Report the gate's numbers, its `±` and its notes, not "it passed".
+
+The full playbook, `runParams()` and `compareFields(...)` are in `guides/test-components.md` →
+*Tuning A Workflow With Evals*.
 
 If cases fail, read the printed summary and pivots first: each case has an `outcome`
 (`failed`, `error`, `budget_exceeded`, `provider_unavailable`, …) with its reason, timing, trace id, AI usage

package/skills/remits-cli/references/investigation.md

@@ -57,6 +57,8 @@ different lanes (see `platform-overview.md` → *Test and Production Data Lanes*
 - **`sessionKey`** — salted user-activity session key. Pivot `mcp_user_activity` `sessions` → `story`;
   each beat then carries a `traceId` / `threadGroupingId` for trace and log investigation.
 - **`sessionId`** — pivots into persisted AI activity via `mcp_ai_session_search`.
+- **`interactionId`** — one Agent answer, including feedback and citations. In component code,
+  `interactions().get(id)` / `.trace(id)` open it; use its `sessionId` / `traceId` for deeper forensics.
 
 ### Reading a record's `content` — persisted context, not a memory dump
 
@@ -109,7 +111,13 @@ embeddable patterns: `features/http-audits.md` (`mcp_get_guide`).
 
 ### AI activity
 
-Every `ai()` call and Agent turn is persisted. Two ways in:
+For a rated or corrected Agent answer, start with `interactions().trace(interactionId)` from an
+Action or Test using the component DSL. `correlation: request_ids` identifies its recorded rounds;
+legacy `time_window` matches may overlap adjacent answers. Empty rounds can mean logging was off or
+telemetry expired. There is no top-level CLI `interactions` command: use your review Action, or pivot
+its session/trace into the tools below. `ai-quality.md` routes the full correction/evaluation workflow.
+
+AI request logging provides two deeper ways in:
 
 - **`mcp_ai_session_search`** — search groupings and open their detail. A grouping spans every session
   sharing one grouping id: the agent turns plus its guardrail and internal `ai()` calls. Use the