@remits/remits-cli 0.1.157 → 0.1.158

package/index.js

@@ -12,10 +12,10 @@
 - L4591  Auth And Component Staging
 - L5223  Component Summaries, Status, And Sync Gates
 - L7547  Branches, Promotion, Commit, And Test Runs
-- L9676  Tokens, Tools, Verification, And Config
-- L11174  Service Dashboard And WebSocket Listener
-- L13673  Agent And Ticket Workflows
-- L17849  Help, Auto Update, And Command Dispatch
+- L9790  Tokens, Tools, Verification, And Config
+- L11307  Service Dashboard And WebSocket Listener
+- L13806  Agent And Ticket Workflows
+- L17982  Help, Auto Update, And Command Dispatch
 */
 
 /*
@@ -112,7 +112,7 @@ let LOCAL_ACTOR_CONTEXT = null;
 
 // Flags that are legitimately repeatable accumulate into an array instead of last-wins. Every other
 // flag keeps last-wins so existing callers are unaffected.
-const REPEATABLE_FLAGS = new Set(['expected-removed', 'expectedRemoved', 'names', 'tag', 'tags', 'key', 'keys', 'case-key', 'score', 'scores', 'lower-is-better']);
+const REPEATABLE_FLAGS = new Set(['expected-removed', 'expectedRemoved', 'names', 'tag', 'tags', 'key', 'keys', 'case-key', 'score', 'scores', 'lower-is-better', 'param', 'params']);
 
 // `-m "msg"` is how git spells a commit message, and agents type it from habit. Before this the parser took a
 // single-dash token as the VALUE of whatever flag preceded it: `--json -m "x"` silently turned JSON mode
@@ -9156,7 +9156,9 @@ function watchWebsocket(baseUrl, topicId, taskId, options = {}) {
         const data = JSON.parse(msg.body);
         if (data.type === 'TestSuite') {
           const payload = data.payload || {};
-          if (payload.taskId && taskId && payload.taskId !== taskId) {
+          // A parallel run is followed by every shard's task id.
+          const followed = Array.isArray(taskId) ? taskId : (taskId ? [taskId] : []);
+          if (payload.taskId && followed.length && !followed.includes(payload.taskId)) {
             return;
           }
           if (payload.tests || payload.name || payload.total != null) {
@@ -9179,6 +9181,82 @@ function watchWebsocket(baseUrl, topicId, taskId, options = {}) {
   };
 }
 
+// `--param name=value` (repeatable) -> the map a suite reads with runParams(). A value that is a JSON number,
+// boolean, null, array or object is passed as that JSON; anything else is the literal text, so `--param model=x`
+// needs no quoting. A name without `=` is refused rather than guessed at.
+function parseRunParams(raw) {
+  const values = raw === undefined || raw === null || raw === true ? [] : (Array.isArray(raw) ? raw : [raw]);
+  const params = {};
+  values.forEach((entry) => {
+    const text = String(entry);
+    const eq = text.indexOf('=');
+    if (eq <= 0) throw new Error('--param expects name=value (got "' + text + '")');
+    const name = text.slice(0, eq).trim();
+    const value = text.slice(eq + 1);
+    let parsed = value;
+    const trimmed = value.trim();
+    if (/^(-?\d+(\.\d+)?([eE][+-]?\d+)?|true|false|null)$/.test(trimmed) || /^(\[[\s\S]*\]|\{[\s\S]*\})$/.test(trimmed)) {
+      try {
+        parsed = JSON.parse(trimmed);
+      } catch (err) {
+        parsed = value;
+      }
+    }
+    params[name] = parsed;
+  });
+  return params;
+}
+
+// How `test run` labels, parameterises, repeats and shards a run, from its flags. Validation of the values is the
+// platform's (Test.normalizeRunOptions); this only shapes the request and checks what the CLI itself acts on.
+function testRunOptionsFromFlags(flags = {}) {
+  const label = flags.label === undefined || flags.label === true ? undefined : String(flags.label);
+  const params = parseRunParams(flags.param !== undefined ? flags.param : flags.params);
+  const repeat = flags.repeat === undefined ? undefined : parsePositiveInt(flags.repeat, NaN);
+  if (flags.repeat !== undefined && !Number.isFinite(repeat)) throw new Error('--repeat expects a whole number of runs per case (got "' + flags.repeat + '")');
+  const parallel = flags.parallel === undefined ? 1 : parsePositiveInt(flags.parallel, NaN);
+  if (!Number.isFinite(parallel)) throw new Error('--parallel expects a whole number of shards (got "' + flags.parallel + '")');
+  return {
+    label,
+    params: Object.keys(params).length ? params : undefined,
+    repeat: repeat && repeat > 1 ? repeat : undefined,
+    parallel
+  };
+}
+
+// How long a later shard of a parallel run waits for a free suite slot before the launch gives up.
+const SHARD_ADMISSION_WAIT_MS = 30 * 60 * 1000;
+
+// A start the platform refused because every suite slot on the instance is taken (safe to repeat: no work began).
+function shardStartBusy(err) {
+  const response = err && err.response;
+  return !!(response && response.status === 429 && response.data && response.data.code === 'WORKLOAD_BUSY');
+}
+
+// One start request per shard of a `--parallel N` run, all under one group id: shard i of N runs every Nth
+// selected case. The platform reads the group id as ONE run in status, corpus runs/compare/gate.
+function shardRunPayloads(runOptions = {}, groupId) {
+  const count = Math.max(1, Number(runOptions.parallel || 1));
+  const common = { label: runOptions.label, params: runOptions.params, repeat: runOptions.repeat };
+  if (count <= 1) return [common];
+  return Array.from({ length: count }, (_, i) => Object.assign({}, common, {
+    runGroupId: groupId,
+    shard: { index: i + 1, count }
+  }));
+}
+
+// What a launch asked for beyond the suite itself: label, params, repetitions and shards.
+function testRunLaunchLines(start = {}, runOptions = {}) {
+  const lines = [];
+  if (runOptions.label) lines.push('Label: ' + runOptions.label);
+  if (runOptions.params) lines.push('Params: ' + Object.keys(runOptions.params).map((k) => k + '=' + JSON.stringify(runOptions.params[k])).join(' '));
+  if (runOptions.repeat) lines.push('Repeat: every selected case runs ' + runOptions.repeat + ' times');
+  if (start.runGroupId) {
+    lines.push('Parallel: ' + (start.shardTaskIds || []).length + ' shards (' + (start.shardTaskIds || []).join(', ') + '), read as ONE run by ' + start.runGroupId);
+  }
+  return lines;
+}
+
 async function testCommand(flags) {
   const cwd = process.cwd();
   ensureLocalState(cwd);
@@ -9207,6 +9285,9 @@ async function testCommand(flags) {
     ? []
     : (Array.isArray(flags.names) ? flags.names : [flags.names]);
   const names = parseTestCaseSelectors(rawNames);
+  // --label / --param / --repeat / --parallel: what this run varies, and how it is executed.
+  const runOptions = testRunOptionsFromFlags(flags);
+  const runGroupId = runOptions.parallel > 1 ? 'grp-' + crypto.randomUUID() : undefined;
 
   const api = buildAxios(baseUrl, session.token);
   // --as-account runs the Test AS a (descendant) subscriber account, so that account's relationship
@@ -9243,7 +9324,7 @@ async function testCommand(flags) {
     });
   });
 
-  const start = await loggedPost(api, cwd, '/cli/test', {
+  const startRequest = {
     token: session.token,
     accountId,
     asAccountId,
@@ -9255,11 +9336,43 @@ async function testCommand(flags) {
     testId: /^\d+$/.test(String(testRef)) ? Number(testRef) : undefined,
     testName: /^\d+$/.test(String(testRef)) ? undefined : String(testRef),
     tests: names
-  }).then((r) => r.data);
-
-  if (!start.success) {
-    throw new Error('Failed to start test run');
+  };
+  // Shards are started one after another (each start returns at once) and then run side by side on the platform.
+  const shardStarts = [];
+  // The platform admits only a few suites per instance at once and refuses the rest as WORKLOAD_BUSY. Once one
+  // of our shards is running it holds a slot it will give back, so a later shard keeps waiting for one (bounded)
+  // instead of giving up after the usual retries and leaving a group that can never finish.
+  const shardWaitDeadline = Date.now() + SHARD_ADMISSION_WAIT_MS;
+  for (const shardOptions of shardRunPayloads(runOptions, runGroupId)) {
+    let shardStart;
+    for (;;) {
+      try {
+        shardStart = await loggedPost(api, cwd, '/cli/test', Object.assign({}, startRequest, shardOptions)).then((r) => r.data);
+        break;
+      } catch (err) {
+        const data = err && err.response && err.response.data;
+        if (shardStarts.length && shardStartBusy(err) && Date.now() < shardWaitDeadline) {
+          console.error('Shard ' + (shardStarts.length + 1) + ' of ' + runOptions.parallel + ' is waiting for a free suite slot ' +
+            '(this platform instance runs a limited number of suites at once; the shards already running free them).');
+          await new Promise((resolve) => setTimeout(resolve, 10000));
+          continue;
+        }
+        const started = shardStarts.length ? ' (shard ' + (shardStarts.length + 1) + ' of ' + runOptions.parallel +
+          ' never started; the shards already started keep running: ' + shardStarts.map((st) => st.taskId).join(', ') + ')' : '';
+        if (data && data.message) throw new Error('Failed to start test run: ' + data.message + started);
+        throw err;
+      }
+    }
+    if (!shardStart.success) {
+      throw new Error('Failed to start test run' + (shardStarts.length ? ' (shard ' + (shardStarts.length + 1) + ' of ' + runOptions.parallel +
+        '; the shards already started keep running: ' + shardStarts.map((st) => st.taskId).join(', ') + ')' : ''));
+    }
+    shardStarts.push(shardStart);
   }
+  // One run to follow: the single run, or the group whose status the platform merges from every shard.
+  const start = runGroupId
+    ? Object.assign({}, shardStarts[0], { taskId: runGroupId, runGroupId, shardTaskIds: shardStarts.map((s) => s.taskId) })
+    : shardStarts[0];
 
   withStdoutRoutedToStderr(jsonOutput, () => {
     printSessionResolutionWarning(sessionContext);
@@ -9281,6 +9394,7 @@ async function testCommand(flags) {
       gitHead: safeGitValue(cwd, 'git rev-parse HEAD')
     }).forEach((line) => console.log(line));
     console.log('Test run started:', start.taskId);
+    testRunLaunchLines(start, runOptions).forEach((line) => console.log(line));
   });
   runtimeState.currentTestTaskId = start.taskId;
 
@@ -9330,7 +9444,7 @@ async function testCommand(flags) {
   if (flags.watch !== 'false') {
     const topic = session.websocketTopic || start.websocketTopic || (session.user && String(session.user.uuid || '').replace(/-/g, ''));
     if (topic) {
-      stopWs = watchWebsocket(baseUrl, topic, start.taskId, { stderr: jsonOutput, verbose: flagEnabled(flags.verbose) });
+      stopWs = watchWebsocket(baseUrl, topic, start.shardTaskIds || start.taskId, { stderr: jsonOutput, verbose: flagEnabled(flags.verbose) });
     }
   }
 
@@ -10345,6 +10459,7 @@ function corpusAiLabel(row = {}) {
 
 function corpusScoreSummary(scores = {}) {
   return Object.keys(scores).map((k) => k + ' ' + (scores[k] && scores[k].mean !== undefined ? scores[k].mean : scores[k]) +
+    (scores[k] && scores[k].stderr !== undefined ? ' \u00b1' + scores[k].stderr : '') +
     (scores[k] && scores[k].cases !== undefined ? '/' + scores[k].cases : '')).join(', ');
 }
 
@@ -10352,7 +10467,8 @@ function corpusScoreDeltas(scores = {}) {
   return Object.keys(scores).map((k) => {
     const s = scores[k] || {};
     const delta = s.delta !== undefined && s.delta !== null ? ' (' + (Number(s.delta) >= 0 ? '+' : '') + s.delta + ')' : '';
-    return k + ' ' + (s.base !== undefined ? s.base : '?') + ' -> ' + (s.head !== undefined ? s.head : '?') + delta + (s.regressed ? ' REGRESSED' : '');
+    const noise = s.stderr !== undefined ? ' [se ' + s.stderr + (s.withinNoise ? ', within noise' : '') + ']' : '';
+    return k + ' ' + (s.base !== undefined ? s.base : '?') + ' -> ' + (s.head !== undefined ? s.head : '?') + delta + noise + (s.regressed ? ' REGRESSED' : '');
   }).join(', ');
 }
 
@@ -10364,16 +10480,22 @@ function corpusLines(sub, data = {}) {
     cases.forEach((c) => lines.push('- ' + c.caseKey + (c.split ? '  [' + c.split + ']' : '') + (c.tags && c.tags.length ? '  tags ' + c.tags.join(',') : '') +
       '  expected ' + (c.hasExpected ? (c.expectedStatus || 'present') : 'none') + '  artifacts ' + (c.artifacts || []).map((a) => a.name + ':' + a.bytes + 'b').join(' ')));
   } else if (sub === 'results') {
-    (data.results || []).forEach((r) => lines.push('- ' + r.caseKey + '  ' + (r.outcome || '?') + (r.score !== undefined ? '  score ' + r.score : '') +
-      '  run ' + r.taskId + '  ' + corpusAiLabel(r) + (r.durationMs ? '  ' + formatDurationMs(r.durationMs) : '') +
+    (data.results || []).forEach((r) => lines.push('- ' + r.caseKey + (r.repeatIndex ? ' #' + r.repeatIndex : '') + '  ' + (r.outcome || '?') + (r.score !== undefined ? '  score ' + r.score : '') +
+      '  run ' + (r.runGroupId || r.taskId) + (r.runLabel ? ' (' + r.runLabel + ')' : '') + (r.split ? '  [' + r.split + ']' : '') + '  ' + corpusAiLabel(r) + (r.durationMs ? '  ' + formatDurationMs(r.durationMs) : '') +
       '  metrics ' + String(r.metricsHash || '').slice(0, 10) + '  ' + formatTime(r.recordedAt)));
   } else if (sub === 'runs') {
-    (data.runs || []).forEach((r) => lines.push('- ' + r.taskId + '  ' + r.cases + ' case(s)  ' + Object.keys(r.outcomes || {}).map((k) => k + ' ' + r.outcomes[k]).join(', ') +
+    (data.runs || []).forEach((r) => lines.push('- ' + r.taskId + (r.label ? '  "' + r.label + '"' : '') + '  ' + r.cases + ' case(s)' +
+      (r.repeats ? ' x' + r.repeats + ' repeats' : '') + (r.shards ? '  ' + r.shards + ' shards' : '') + '  ' +
+      Object.keys(r.outcomes || {}).map((k) => k + ' ' + r.outcomes[k]).join(', ') + (r.passRate !== undefined ? '  pass rate ' + r.passRate : '') +
       '  ' + corpusAiLabel(r) + '  ' + (r.dataMode || '?') + '  ' + (r.branchName || '?') + (r.workspace ? ' [ws:' + r.workspace + ']' : '') +
       (r.laneContentHash ? '  lane ' + String(r.laneContentHash).slice(0, 12) : '') + '  ' + formatTime(r.recordedAt)));
     (data.runs || []).forEach((r) => {
       const extra = [];
+      if (r.params && Object.keys(r.params).length) extra.push('params ' + Object.keys(r.params).map((k) => k + '=' + JSON.stringify(r.params[k])).join(' '));
       if (r.scores && Object.keys(r.scores).length) extra.push('scores ' + corpusScoreSummary(r.scores));
+      if (r.bySplit) extra.push('by split ' + Object.keys(r.bySplit).map((sp) => sp + ': ' + r.bySplit[sp].cases + ' case(s)' +
+        (r.bySplit[sp].passRate !== undefined ? ' pass ' + r.bySplit[sp].passRate : '') +
+        (r.bySplit[sp].scores ? ' ' + corpusScoreSummary(r.bySplit[sp].scores) : '')).join('; '));
       if (r.review) extra.push('graded ' + r.review.graded + ', spot-checked ' + r.review.reviewed + (r.review.agreement !== undefined ? ' (agreement ' + r.review.agreement + ')' : ''));
       if (extra.length) lines.push('    ' + r.taskId + ': ' + extra.join('  '));
     });
@@ -10388,6 +10510,7 @@ function corpusLines(sub, data = {}) {
     if (cmp.scores && Object.keys(cmp.scores).length) lines.push('Scores (cases both runs measured): ' + corpusScoreDeltas(cmp.scores));
     (cmp.cases || []).filter((c) => c.direction !== 'same' || c.changedScores).forEach((c) => {
       lines.push('- ' + c.caseKey + '  ' + c.direction + (c.base ? '  ' + c.base.outcome : '') + (c.head ? ' -> ' + c.head.outcome : '') +
+        (c.base && c.base.passRate !== undefined ? '  pass ' + c.base.passRate + ' -> ' + (c.head && c.head.passRate !== undefined ? c.head.passRate : '?') : '') +
         (c.expectedChanged ? '  (EXPECTED ANSWER CHANGED between runs)' : ''));
       (c.changedMetrics || []).slice(0, 8).forEach((m) => lines.push('    ' + m.metric + ': ' + JSON.stringify(m.base) + ' -> ' + JSON.stringify(m.head)));
       Object.keys(c.changedScores || {}).slice(0, 8).forEach((k) => lines.push('    score ' + k + ': ' + c.changedScores[k].base + ' -> ' + c.changedScores[k].head));
@@ -10404,9 +10527,11 @@ function corpusLines(sub, data = {}) {
   } else if (sub === 'gate') {
     const gate = data.gate || {};
     lines.push('Gate ' + (gate.baseTaskId || '?') + ' -> ' + (gate.headTaskId || '?') + ': ' + (gate.passed ? 'PASSED' : 'DID NOT PASS') +
-      (gate.tolerance !== undefined ? '  (tolerance ' + gate.tolerance + ')' : ''));
+      (gate.tolerance !== undefined ? '  (tolerance ' + gate.tolerance + ')' : '') + (gate.noiseSigma !== undefined ? '  (noise sigma ' + gate.noiseSigma + ')' : '') +
+      (gate.split ? '  split ' + gate.split : ''));
     if (gate.scores && Object.keys(gate.scores).length) lines.push('Scores: ' + corpusScoreDeltas(gate.scores));
     (gate.reasons || []).forEach((r) => lines.push('- ' + r));
+    (gate.notes || []).forEach((n) => lines.push('Note: ' + n));
     if (gate.expectedChanged && gate.expectedChanged.length) lines.push('Left out (expected answer changed between runs): ' + gate.expectedChanged.join(', '));
     (gate.changedCases || []).slice(0, 15).forEach((c) => lines.push('  ' + c.caseKey + '  ' + c.direction +
       Object.keys(c.changedScores || {}).map((k) => '  ' + k + ' ' + c.changedScores[k].base + '->' + c.changedScores[k].head).join('')));
@@ -10428,7 +10553,7 @@ function corpusLines(sub, data = {}) {
     lines.push('Case ' + con.caseKey + ': ' + con.runs + ' run(s), ' + (con.stable ? 'STABLE (identical metrics every run)' : (con.variants || []).length + ' distinct metric results') +
       (con.aiModes && con.aiModes.length ? '  AI ' + con.aiModes.join('+') : ''));
     if (con.note) lines.push('WARNING: ' + con.note);
-    (con.variants || []).forEach((v) => lines.push('- metrics ' + String(v.metricsHash).slice(0, 10) + '  runs ' + (v.runs || []).map((r) => r.taskId + ':' + r.outcome + (r.aiMode && r.aiMode !== 'none' ? '/' + r.aiMode : '')).join(', ')));
+    (con.variants || []).forEach((v) => lines.push('- metrics ' + String(v.metricsHash).slice(0, 10) + '  runs ' + (v.runs || []).map((r) => r.taskId + (r.repeatIndex ? '#' + r.repeatIndex : '') + ':' + r.outcome + (r.aiMode && r.aiMode !== 'none' ? '/' + r.aiMode : '')).join(', ')));
   }
   return lines;
 }
@@ -10538,12 +10663,16 @@ async function corpusCommand(flags, subcommand) {
     const keys = parseCorpusFilterValues(flags, ['key', 'keys', 'case', 'case-key']);
     data = await post({ command: 'cases', corpus: corpusName, split: flags.split, tags: tags.length ? tags : undefined, keys: keys.length ? keys : undefined, includeValues: flagEnabled(flags['include-values']), includeRetired: flagEnabled(flags['include-retired']), limit: flags.limit });
   } else if (sub === 'results') {
-    data = await post({ command: 'results', corpus: corpusName, caseKey: flags.case || flags['case-key'], taskId: flags['task-id'], limit: flags.limit || 50 });
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
+    data = await post({ command: 'results', corpus: corpusName, caseKey: flags.case || flags['case-key'], taskId: flags['task-id'], label: flags.label,
+      split: flags.split, tags: tags.length ? tags : undefined, limit: flags.limit || 50 });
   } else if (sub === 'runs') {
-    data = await post({ command: 'runs', corpus: corpusName, limit: flags.limit || 20 });
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
+    data = await post({ command: 'runs', corpus: corpusName, label: flags.label, split: flags.split, tags: tags.length ? tags : undefined, limit: flags.limit || 20 });
   } else if (sub === 'compare') {
     if (!flags.base || !flags.head) throw new Error('corpus compare requires --base <taskId> --head <taskId>');
-    data = await post({ command: 'compare', corpus: corpusName, base: flags.base, head: flags.head });
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
+    data = await post({ command: 'compare', corpus: corpusName, base: flags.base, head: flags.head, split: flags.split, tags: tags.length ? tags : undefined });
   } else if (sub === 'consistency') {
     const caseKey = flags.case || flags['case-key'];
     if (!caseKey) throw new Error('corpus consistency requires --case <caseKey>');
@@ -10556,10 +10685,13 @@ async function corpusCommand(flags, subcommand) {
   } else if (sub === 'gate') {
     const scores = parseCorpusFilterValues(flags, ['score', 'scores']);
     const lowerIsBetter = parseCorpusFilterValues(flags, ['lower-is-better']);
+    const tags = parseCorpusFilterValues(flags, ['tag', 'tags']);
     data = await post({ command: 'gate', corpus: corpusName, base: flags.base, head: flags.head, tolerance: flags.tolerance,
       scores: scores.length ? scores : undefined,
       lowerIsBetter: lowerIsBetter.length ? lowerIsBetter : undefined,
-      allowModeChange: flagEnabled(flags['allow-mode-change']) });
+      allowModeChange: flagEnabled(flags['allow-mode-change']),
+      baseLabel: flags['base-label'], headLabel: flags['head-label'], split: flags.split, tags: tags.length ? tags : undefined,
+      minCases: flags['min-cases'], noiseSigma: flags['noise-sigma'] });
     const gate = (data && data.gate) || {};
     if (!gate.passed) process.exitCode = 1;
     await appendVerificationPacket(api, cwd, session, accountId, flags, {
@@ -10569,7 +10701,8 @@ async function corpusCommand(flags, subcommand) {
       world: Object.assign({}, (data && data.world) || {}, { host: normalizeBaseUrl(baseUrl), branchName, workspace, dataMode }),
       revision: collectVerificationSource(cwd, flags),
       evidenceCategories: ['corpus_gate'],
-      corpusGate: { corpus: corpusName, baseTaskId: gate.baseTaskId, headTaskId: gate.headTaskId, passed: !!gate.passed, scores: gate.scores, reasons: gate.reasons }
+      corpusGate: { corpus: corpusName, baseTaskId: gate.baseTaskId, headTaskId: gate.headTaskId, passed: !!gate.passed, scores: gate.scores, reasons: gate.reasons,
+        split: gate.split, noiseSigma: gate.noiseSigma, minCases: gate.minCases, notes: gate.notes }
     }, { quiet: jsonOutput });
   } else if (sub === 'review') {
     const taskId = flags['task-id'];
@@ -10577,7 +10710,7 @@ async function corpusCommand(flags, subcommand) {
     if (flags.verdict) {
       const caseKey = flags.case || flags['case-key'];
       if (!caseKey) throw new Error('corpus review --verdict requires --case <caseKey>');
-      data = await post({ command: 'review', corpus: corpusName, taskId, caseKey, verdict: flags.verdict, score: flags.score, note: flags.note });
+      data = await post({ command: 'review', corpus: corpusName, taskId, caseKey, verdict: flags.verdict, score: flags.score, note: flags.note, repeatIndex: flags.repeat });
     } else {
       data = await post({ command: 'review', corpus: corpusName, taskId, size: flags.sample || flags.size });
     }
@@ -18179,7 +18312,7 @@ function printComponentsHelp(subcommand) {
 }
 
 function printTestHelp() {
-  console.log('Usage: remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--json]');
+  console.log('Usage: remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--label TEXT] [--param name=value ...] [--repeat N] [--parallel N] [--json]');
   console.log('       remits-cli test status --task-id <taskId> [--base-url URL] [--account-id ID] [--branch stagingScope] [--data-mode test|prod] [--json]');
   console.log('       remits-cli test runs [--test <id|name>] [--limit 20] [--compare] [--json]   # durable run history');
   console.log('       remits-cli test compare --base <taskId> --head <taskId> [--json]            # per-case outcome/cost deltas');
@@ -18210,13 +18343,14 @@ function printTestHelp() {
 function printCorpusHelp() {
   console.log('Usage: remits-cli corpus import --manifest corpus-manifest.json [--corpus <name>] [--as-account ID] [--data-mode test|prod --confirm-prod] [--json]');
   console.log('       remits-cli corpus cases --corpus <name> [--split dev] [--tag name] [--key caseKey] [--include-values] [--include-retired] [--json]');
-  console.log('       remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId>] [--limit 50] [--json]');
-  console.log('       remits-cli corpus runs --corpus <name> [--limit 20] [--json]');
-  console.log('       remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--json]');
+  console.log('       remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId|groupId>] [--label L] [--split S] [--tag T] [--limit 50] [--json]');
+  console.log('       remits-cli corpus runs --corpus <name> [--label L] [--split S] [--tag T] [--limit 20] [--json]');
+  console.log('       remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--split S] [--tag T] [--json]');
   console.log('       remits-cli corpus consistency --corpus <name> --case <caseKey> [--limit 20] [--json]');
   console.log('       remits-cli corpus retire --corpus <name> --case <caseKey> [--case ...] [--restore] [--json]');
-  console.log('       remits-cli corpus gate --corpus <name> [--base <taskId>] [--head <taskId>] [--tolerance 0.02] [--score answer] [--lower-is-better <name>] [--allow-mode-change] [--json]');
-  console.log('       remits-cli corpus review --corpus <name> --task-id <taskId> [--sample 5] | --case <caseKey> --verdict agree|disagree [--score n] [--note "..."]');
+  console.log('       remits-cli corpus gate --corpus <name> [--base <taskId> | --base-label L] [--head <taskId> | --head-label L] [--split holdout] [--tag T]');
+  console.log('              [--score answer --score passRate] [--tolerance 0.02] [--lower-is-better <name>] [--min-cases N] [--noise-sigma 2] [--allow-mode-change] [--json]');
+  console.log('       remits-cli corpus review --corpus <name> --task-id <taskId> [--sample 5] | --case <caseKey> [--repeat N] --verdict agree|disagree [--score n] [--note "..."]');
   console.log('');
   console.log('A corpus is a named, durable set of cases an evaluation-style Test runs again and again (front stage:');
   console.log('corpus(name)). Cases live in a platform table owned by one account and visible to the accounts beneath');
@@ -18477,7 +18611,7 @@ async function main() {
     console.log('  remits-cli components branch <name> --subscribe <accountId> [--dry-run] [--confirm-primary-edge]');
     console.log('  remits-cli components branch <name> --unsubscribe <accountId>    # return that account to trunk');
     console.log('  remits-cli components branch <name> --retire [--force]           # delete the branch\'s overlays');
-    console.log('  remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--json]');
+    console.log('  remits-cli test run --test <id|name> [--base-url URL] [--branch stagingScope] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account ID] [--variant-branch NAME|none] [--label TEXT] [--param name=value ...] [--repeat N] [--parallel N] [--json]');
     console.log('  remits-cli test runs [--test <id|name>] [--compare]                 # durable run history; compare latest two');
     console.log('  remits-cli test compare --base <taskId> --head <taskId>');
     console.log('  remits-cli corpus import --manifest corpus-manifest.json             # seed a test corpus: cases + artifacts');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remits/remits-cli",
-  "version": "0.1.157",
+  "version": "0.1.158",
   "description": "Local CLI for auth, component sync, and live test execution against Remits",
   "license": "MIT",
   "private": false,

package/skills/remits-cli/SKILL.md

@@ -46,6 +46,7 @@ reference below, load it before you act, not after something surprises you.
 | work from a non-trunk checkout, or promote a branch | `branch-variants.md` | what a variant is, which world your checkout resolves, `--as-account` vs `--variant-branch`, tombstones, the promotion loop and its phases, subscribe/retire |
 | touch a support ticket | `support-tickets.md` | the ticket record and its verbs, routed vs claimed vs owned, autonomous workers, the repository edit lease, ask/answer/resume, delivery |
 | register this session as an agent, or ask why a routed ticket never started | `agent-sessions.md` | `serve` vs `register`, what registration does, worker spawning per agent kind, routing order, the control center, multi-session auth |
+| tune, improve or "fine-tune" a workflow with evals, or compare two variants of one | `development-loop.md` → *Evaluation runs* | `--label`/`--param`/`--repeat`/`--parallel`, the baseline → candidate → gate loop, the holdout rule; the full playbook is `guides/test-components.md` → *Tuning A Workflow With Evals* |
 | build or evaluate cited Q&A, review learnings, or investigate answer quality | `ai-quality.md` | front-stage guide routing, interaction/session pivots, baseline vs proposal measurements, human checks and explicit corpus gates |
 | investigate live behavior | `investigation.md` | which tool reads which record, correlation keys, reading a record's `content`, HTTP audits, AI activity, node/`localMode`, the production support flows |
 | call any `mcp_*` tool | `tool-reference.md` | every tool's parameters, semantics, and traps — read the tool's entry before building its input |

package/skills/remits-cli/references/ai-quality.md

@@ -61,14 +61,20 @@ search written just for the evaluator may measure something the answering Agent
 2. Make the source/prompt change, or enable `includeProposed` in the Test's Agent/retrieval path.
    Preview includes all relevant proposals in scope, not one selected id. Keep the proposal set
    stable. Learning records are shared across data lanes; a workspace does not isolate them.
-3. Re-stage and run the candidate under an explicit `withAiBudget` in the Test. Retain both task ids,
-   per-case `scores: [retrieval: ..., answer: ...]`, grader results and retrieval diagnostics.
+3. Re-stage and run the candidate under an explicit `withAiBudget` in the Test, with the same flags as
+   the baseline and a different `--label`. Keep per-case `scores: [retrieval: ..., answer: ...]`, grader
+   results and retrieval diagnostics.
 
 ```bash
 remits-cli components stage --workset --account-id 1 --base-url http://localhost:8080 --data-mode test
-remits-cli test run --test "Support Answer Evaluation" --account-id 1 --base-url http://localhost:8080 --data-mode test --json
+remits-cli test run --test "Support Answer Evaluation" --label baseline --repeat 3 --parallel 2 --account-id 1 --base-url http://localhost:8080 --data-mode test --json
+# make the change (or pass it with --param name=value), re-stage, then:
+remits-cli test run --test "Support Answer Evaluation" --label candidate --repeat 3 --parallel 2 --account-id 1 --base-url http://localhost:8080 --data-mode test --json
 ```
 
+`--repeat 3` runs each question three times. Live answers and grades vary from run to run, and the repeats are
+what give each score its `±` noise estimate. `--parallel 2` runs the cases in two shards under one `grp-…` id.
+
 The suite name and account above are examples: create/use your application's suite and resolved account.
 Repeat after the candidate edit and stage. `--test` selects a suite; `--names` selects exact case names
 (or a `|`-separated list), not substrings or a regex. Check `total` and outcomes; a zero-case run proves
@@ -85,19 +91,23 @@ verify unreleased platform changes, even with the same account and test lane.
 With the same explicit host/account/data-mode options as the runs:
 
 ```bash
+remits-cli corpus runs --corpus support-golden --limit 2                     # each run's id (grp-...), label, scores ± noise
 remits-cli corpus compare --corpus support-golden --base BASE_TASK_ID --head CANDIDATE_TASK_ID
 remits-cli corpus review --corpus support-golden --task-id CANDIDATE_TASK_ID --sample 5
 remits-cli corpus review --corpus support-golden --task-id CANDIDATE_TASK_ID --case CASE_KEY --verdict agree --note "Checked against the source"
-remits-cli corpus gate --corpus support-golden --base BASE_TASK_ID --head CANDIDATE_TASK_ID --score retrieval --score answer
+remits-cli corpus gate --corpus support-golden --base-label baseline --head-label candidate --split holdout --score retrieval --score answer --noise-sigma 2
 remits-cli evidence
 ```
 
-Replace uppercase ids with the returned task/case ids. Each repeated `--score` is a required metric;
-`--scores retrieval,answer` is equivalent. Pin baseline and candidate instead of relying on latest runs.
+Replace uppercase ids with the ids `corpus runs` prints. A parallel run's `grp-…` id is accepted wherever a task
+id is. The gate can select runs by label instead of id. Run it on `holdout`, the split you did not tune against. Each repeated `--score` is a required metric;
+`--scores retrieval,answer` is equivalent. Name the baseline and candidate by id or by label; never rely on the gate's "latest two runs" default.
 The gate exits nonzero for regressions, lost/absent measurements or an unaccepted mocked/live mode
 change. It records `corpus_gate` evidence. Inspect measured case counts and excluded changed expected
 answers; also assess the application's minimum accuracy and the human spot checks. A non-regression
-gate alone does not establish an absolute quality target or statistical significance.
+gate does not establish an absolute quality target. With `--noise-sigma 2` (both runs launched with `--repeat`),
+a move only counts as a regression when it is larger than the run-to-run noise. Without repeats, the gate's notes
+say that a small difference may be noise.
 
 Stop a release workflow on a failing gate. Approval and component sync do not automatically invoke it.
 Human review preserves the model grade; it does not replace it. Approve/reject through the application's

package/skills/remits-cli/references/command-reference.md

@@ -259,18 +259,18 @@ remits-cli components branch <name> --copy-to <newBranch> [--dry-run] [--force]
 remits-cli components branch <name> --subscribe <accountId> [--parent-account <id>] [--domain <host>] [--dry-run] [--confirm-primary-edge]   # make an account resolve this branch
 remits-cli components branch <name> --unsubscribe <accountId>                          # return that account to trunk
 remits-cli components branch <name> --retire [--force]                                 # delete the branch's overlays
-remits-cli test run --test <id|name> [--branch <stagingScope>] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account <ID>] [--variant-branch <name|none>] [--json]
-remits-cli test status --task-id <taskId> [--branch <stagingScope>] [--data-mode test|prod] [--json]   # falls back to the DURABLE run record once the live status expires
+remits-cli test run --test <id|name> [--branch <stagingScope>] [--names "a|b"] [--watch true|false] [--wait true|false] [--data-mode test|prod] [--as-account <ID>] [--variant-branch <name|none>] [--label <text>] [--param name=value ...] [--repeat N] [--parallel N] [--json]
+remits-cli test status --task-id <taskId|groupId> [--branch <stagingScope>] [--data-mode test|prod] [--json]   # falls back to the DURABLE run record once the live status expires
 remits-cli test runs [--test <id|name>] [--limit 20] [--compare] [--all-lanes] [--as-account <ID>] [--json]  # durable run history: pass counts, live AI cost, lane, content hash
 remits-cli test compare --base <taskId> --head <taskId> [--json]                                            # per-case improved / regressed / changed, cost and world deltas
 remits-cli corpus import --manifest corpus-manifest.json [--corpus <name>] [--as-account <ID>] [--data-mode test|prod --confirm-prod] [--json]   # seed an evaluation corpus: cases + immutable artifacts; idempotent by caseKey
 remits-cli corpus cases --corpus <name> [--split S] [--tag T|--tags T,U] [--key K|--keys K,L] [--include-values] [--include-retired] [--limit N] [--json]
-remits-cli corpus runs --corpus <name> [--limit 20] [--json]                                   # runs that measured the corpus: outcomes, AI mode (live|MOCKED|none), cost, world
-remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId>] [--json]
-remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--json]              # per-case direction, metric deltas, expectedChanged
-remits-cli corpus gate --corpus <name> --base <taskId> --head <taskId> [--score retrieval --score answer] [--lower-is-better <name>] [--tolerance 0.02] [--allow-mode-change] [--json]
+remits-cli corpus runs --corpus <name> [--label L] [--split S] [--tag T] [--limit 20] [--json] # runs that measured the corpus: label, repeats, shards, pass rate, scores ± noise, per split, AI mode (live|MOCKED|none), cost, world
+remits-cli corpus results --corpus <name> [--case <caseKey>] [--task-id <taskId|groupId>] [--label L] [--split S] [--tag T] [--json]
+remits-cli corpus compare --corpus <name> --base <taskId> --head <taskId> [--split S] [--tag T] [--json]   # per-case direction, pass rate, metric deltas, expectedChanged
+remits-cli corpus gate --corpus <name> [--base <taskId> | --base-label L] [--head <taskId> | --head-label L] [--split holdout] [--tag T] [--score retrieval --score passRate] [--lower-is-better <name>] [--tolerance 0.02] [--min-cases N] [--noise-sigma 2] [--allow-mode-change] [--json]
 remits-cli corpus review --corpus <name> --task-id <taskId> [--sample 5] [--json]
-remits-cli corpus review --corpus <name> --task-id <taskId> --case <key> --verdict agree|disagree [--score n] [--note "..."]
+remits-cli corpus review --corpus <name> --task-id <taskId> --case <key> [--repeat N] --verdict agree|disagree [--score n] [--note "..."]
 remits-cli corpus consistency --corpus <name> --case <caseKey> [--json]                        # did the case produce the same metrics every run?
 remits-cli corpus retire --corpus <name> --case <caseKey> [--case ...] [--restore] [--json]    # drop a case from future runs; past measurements stay readable
 ```
@@ -328,8 +328,12 @@ For the baseline → proposed learning → human review workflow, see `ai-qualit
 `--score retrieval --score answer` requires both metrics (`--scores retrieval,answer` is equivalent).
 `interactions()` and `memories()` are component DSL APIs, not top-level CLI subcommands.
 
-`corpus gate` exits nonzero for a score regression, missing measurements, or an unaccepted AI-mode
-change, and records `corpus_gate` evidence. Run it explicitly before sync or approving a learning;
+`corpus gate` exits nonzero for a score regression, missing measurements, too few shared cases
+(`--min-cases`), a missing noise estimate (`--noise-sigma` without `--repeat` on both runs), or an unaccepted
+AI-mode change, and records `corpus_gate` evidence. `passRate`, `liveAiCost`, `liveAiCalls` and `durationMs` are
+built-in scores every case carries; the last three are lower-is-better without `--lower-is-better`.
+`--base-label`/`--head-label` select the latest run with that `test run --label`. Every corpus command accepts a
+parallel run's `grp-…` id wherever it accepts a task id. Run it explicitly before sync or approving a learning;
 neither operation invokes it automatically. Prefer explicit run ids and check their world.
 `corpus review` samples graded cases or records a human spot check; it preserves the model grade.
 

package/skills/remits-cli/references/development-loop.md

@@ -20,6 +20,7 @@
       - [Stage your workset, not the whole repo](#stage-your-workset-not-the-whole-repo)
       - [Three numbers, three questions](#three-numbers-three-questions)
     - [Step 4: Verify the Change](#step-4-verify-the-change)
+      - [Evaluation runs](#evaluation-runs)
       - [Many failures are usually few causes](#many-failures-are-usually-few-causes)
     - [Step 5: Iterate If Needed](#step-5-iterate-if-needed)
     - [Step 6: Update Documentation](#step-6-update-documentation)
@@ -430,11 +431,38 @@ about a different world.
 
 The platform launch is asynchronous. By default `remits-cli test run` waits for its task to finish, and any
 cases selected by `--names "a|b"` execute sequentially inside that suite task. If several slow cases are
-independent, stage once, then start separate `test run --names "<case>" --wait false` commands from the same
-lane and keep the task ids they print. Complete each proof with
-`remits-cli test status --task-id <id>`; that terminal status read records and attaches the final `test_run`
-evidence. Do not parallelize cases that mutate the same fixture, rely on shared suite setup state, or make
-undeclared live AI/provider calls.
+independent, use `--parallel N`. It splits the selected cases across N shards that run side by side, and the run
+keeps ONE id (`grp-…`) for `test status` and every corpus command. An instance runs only a few suites at once
+(two by default), so shards beyond that wait for a slot. Ask for 2–4. Starting separate
+`test run --names "<case>" --wait false` commands also works for one-off proofs, but every such run is a separate
+run: a corpus cannot compare or gate them as one. Do not parallelize cases that mutate the same fixture, rely on
+shared suite setup state, or make undeclared live AI/provider calls.
+
+##### Evaluation runs
+
+When the job is to improve a workflow ("tune it", "make it more accurate", "compare these two prompts"), the unit
+of evidence is a measured corpus run, not a green suite. Follow this loop:
+
+```bash
+remits-cli test run --test "<eval suite>" --label baseline --repeat 3 --parallel 4      # live, inside withAiBudget
+remits-cli corpus runs --corpus <name> --split dev                                      # pass rate, scores ± noise
+remits-cli corpus results --corpus <name> --label baseline --split dev                  # failing cases -> trace ids
+# change ONE thing (or pass the variant with --param name=value), re-stage, then:
+remits-cli test run --test "<eval suite>" --label candidate --repeat 3 --parallel 4
+remits-cli corpus runs --corpus <name> --limit 2                                         # the two runs' grp-ids
+remits-cli corpus compare --corpus <name> --base <baseline grp-id> --head <candidate grp-id>
+remits-cli corpus gate --corpus <name> --base-label baseline --head-label candidate --split holdout --noise-sigma 2
+```
+
+Rules:
+
+- Tune against `dev`, and gate on `holdout` only once you are done.
+- Keep the lane, the AI mode and the corpus identical between the two runs.
+- Read a failing case's AI session before you change a prompt.
+- Report the gate's numbers, its `±` and its notes, not "it passed".
+
+The full playbook, `runParams()` and `compareFields(...)` are in `guides/test-components.md` →
+*Tuning A Workflow With Evals*.
 
 If cases fail, read the printed summary and pivots first: each case has an `outcome`
 (`failed`, `error`, `budget_exceeded`, `provider_unavailable`, …) with its reason, timing, trace id, AI usage