@ngockhoale/ukit 3.3.3 → 3.4.0

package/scripts/bench/subagent-orchestrator-corpus.mjs

@@ -0,0 +1,275 @@
+#!/usr/bin/env node
+// TASK-C89-001 — frozen subagent-orchestrator golden corpus (SPEC §3 FR-01,
+// §9 FR-07, §12 comparability contract).
+//
+// This module is the machine-checkable half of the baseline freeze: a
+// deterministic, immutable fixture manifest consumed by TASK-C89-007's eval
+// harness. Fixture metadata only — no task bodies, no runtime behavior.
+//
+//   loadSubagentCorpus() → {revision, baselineSha, rubricVersion, arms,
+//                           fixtures[], rubrics}
+//   checkComparability(run) → {comparable:boolean, reason:string|null}
+//   computeSavings({baseline, candidate}) → {reportable:boolean, savingsPct,
+//                                            reason}
+//
+// Frozen constants:
+//   CORPUS_REVISION — bump on ANY fixture/rubric/arm change; a run recorded
+//     against an older revision is non-comparable (stale_corpus_revision).
+//   BASELINE_SHA    — git SHA the corpus was frozen against; a run on a
+//     different tree is non-comparable (stale_baseline_sha).
+//   RUBRIC_VERSION  — scoring rubric version; absent/mismatched →
+//     missing_rubric_version.
+//
+// Provenance rule (SPEC FR-01): savings may be reported only when BOTH runs
+// carry provider-measured usage. 'estimated'/'host-measured'/'unknown' sources
+// yield savingsPct: null — never 0%, never a fabricated number.
+
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const CORPUS_REVISION = 'so-corpus-r1';
+export const RUBRIC_VERSION = 'so-rubric-v1';
+// Git SHA this corpus was frozen against (cycle C89 worktree HEAD at freeze).
+export const BASELINE_SHA = 'd86c0be7e9974415c28a212a48ac4247c26d1a0c';
+
+// Frozen arms (SPEC §9): the eval compares these — never invented per run.
+export const ARMS = Object.freeze([
+  'A-current-handoff',   // today's inline handoff/orchestration behavior
+  'B-single-strong',     // one strong agent, no delegation
+  'C-no-pruning',        // candidate path with context pruning disabled
+]);
+
+// Per-class scoring rubric. Every rubric scores the same three axes
+// (SPEC §9: correctness / evidence / completeness) and a critical defect is
+// an independent blocker — savings never overrides quality.
+const RUBRICS = Object.freeze({
+  trivial: Object.freeze({
+    dimensions: Object.freeze(['correctness', 'evidence', 'completeness']),
+    criticalFailureBlocker: true,
+    notes: 'deterministic fast path — delegate=false is the expected route',
+  }),
+  retrieval: Object.freeze({
+    dimensions: Object.freeze(['correctness', 'evidence', 'completeness']),
+    criticalFailureBlocker: true,
+    notes: 'evidence refs must resolve to the real source span',
+  }),
+  debug: Object.freeze({
+    dimensions: Object.freeze(['correctness', 'evidence', 'completeness']),
+    criticalFailureBlocker: true,
+    notes: 'root cause must be evidenced, not symptom suppression',
+  }),
+  review: Object.freeze({
+    dimensions: Object.freeze(['correctness', 'evidence', 'completeness']),
+    criticalFailureBlocker: true,
+    notes: 'seeded defect must be found; missing it is a critical failure',
+  }),
+  continuation: Object.freeze({
+    dimensions: Object.freeze(['correctness', 'evidence', 'completeness']),
+    criticalFailureBlocker: true,
+    notes: 'long productive continuation — no premature stop, no rabbit hole',
+  }),
+  constraint: Object.freeze({
+    dimensions: Object.freeze(['correctness', 'evidence', 'completeness']),
+    criticalFailureBlocker: true,
+    notes: 'declared constraints are hard; a violated constraint is critical',
+  }),
+  failure: Object.freeze({
+    dimensions: Object.freeze(['correctness', 'evidence', 'completeness']),
+    criticalFailureBlocker: true,
+    notes: 'expected behavior is a typed refusal/fallback, never a fabricated result',
+  }),
+});
+
+const FIXTURES = Object.freeze([
+  // --- mandatory classes ---
+  Object.freeze({
+    id: 'SO-F01-trivial-rename',
+    taskClass: 'trivial',
+    language: 'en',
+    rubric: 'trivial',
+    summary: 'rename a local identifier in one file; deterministic fast path',
+  }),
+  Object.freeze({
+    id: 'SO-F02-retrieval-symbol',
+    taskClass: 'retrieval',
+    language: 'en',
+    rubric: 'retrieval',
+    summary: 'locate a symbol definition and its consumers; answer carries refs',
+  }),
+  Object.freeze({
+    id: 'SO-F03-debug-failing-test',
+    taskClass: 'debug',
+    language: 'en',
+    rubric: 'debug',
+    summary: 'reproduce a failing vitest case and name the root cause',
+  }),
+  Object.freeze({
+    id: 'SO-F04-review-seeded-defect',
+    taskClass: 'review',
+    language: 'en',
+    rubric: 'review',
+    summary: 'review a diff containing one seeded defect; defect must surface',
+  }),
+  Object.freeze({
+    id: 'SO-F05-continuation-long',
+    taskClass: 'continuation',
+    language: 'en',
+    rubric: 'continuation',
+    summary: 'multi-step productive continuation from a mid-run state',
+  }),
+  Object.freeze({
+    id: 'SO-F06-constraint-no-api-change',
+    taskClass: 'constraint',
+    language: 'en',
+    rubric: 'constraint',
+    summary: 'refactor under an explicit "no public API change" constraint',
+  }),
+  // --- VI language coverage (SPEC §9: EN/VI cases) ---
+  Object.freeze({
+    id: 'SO-F07-retrieval-vi',
+    taskClass: 'retrieval',
+    language: 'vi',
+    rubric: 'retrieval',
+    summary: 'câu hỏi truy xuất: tìm định nghĩa và nơi dùng của hàm',
+  }),
+  Object.freeze({
+    id: 'SO-F08-debug-vi',
+    taskClass: 'debug',
+    language: 'vi',
+    rubric: 'debug',
+    summary: 'tái hiện lỗi và chỉ ra nguyên nhân gốc của test hỏng',
+  }),
+  Object.freeze({
+    id: 'SO-F09-continuation-vi',
+    taskClass: 'continuation',
+    language: 'vi',
+    rubric: 'continuation',
+    summary: 'tiếp tục công việc dở dang từ trạng thái giữa chừng',
+  }),
+  // --- failure cases (typed fallbacks, never fabricated results) ---
+  Object.freeze({
+    id: 'SO-F10-failure-auth-reject',
+    taskClass: 'failure',
+    language: 'en',
+    rubric: 'failure',
+    failureKind: 'auth_reject',
+    summary: 'decision endpoint rejects auth → deterministic route, unavailable receipt',
+  }),
+  Object.freeze({
+    id: 'SO-F11-failure-stale-ref',
+    taskClass: 'failure',
+    language: 'en',
+    rubric: 'failure',
+    failureKind: 'stale_reference',
+    summary: 'evidence ref points at a moved/absent source → typed partial, never complete',
+  }),
+  Object.freeze({
+    id: 'SO-F12-failure-host-no-fork',
+    taskClass: 'failure',
+    language: 'en',
+    rubric: 'failure',
+    failureKind: 'host_fork_unsupported',
+    summary: 'host cannot fork context → fork_unavailable, curated fallback',
+  }),
+]);
+
+const CORPUS = Object.freeze({
+  schema: 1,
+  revision: CORPUS_REVISION,
+  baselineSha: BASELINE_SHA,
+  rubricVersion: RUBRIC_VERSION,
+  arms: ARMS,
+  rubrics: RUBRICS,
+  fixtures: FIXTURES,
+});
+
+/** @returns the frozen corpus manifest (immutable — mutation throws). */
+export function loadSubagentCorpus() {
+  return CORPUS;
+}
+
+/**
+ * Gate a run's eligibility for baseline comparison.
+ * @param {object} run recorded run identity
+ * @param {string} [run.corpusRevision]
+ * @param {string} [run.baselineSha]
+ * @param {string} [run.rubricVersion]
+ * @returns {{comparable:boolean, reason:string|null}} typed reason when not
+ *   comparable — 'stale_corpus_revision' | 'stale_baseline_sha' |
+ *   'missing_rubric_version'
+ */
+export function checkComparability(run = {}) {
+  if (run.corpusRevision !== CORPUS_REVISION) {
+    return { comparable: false, reason: 'stale_corpus_revision' };
+  }
+  if (run.baselineSha !== BASELINE_SHA) {
+    return { comparable: false, reason: 'stale_baseline_sha' };
+  }
+  if (run.rubricVersion !== RUBRIC_VERSION) {
+    return { comparable: false, reason: 'missing_rubric_version' };
+  }
+  return { comparable: true, reason: null };
+}
+
+/**
+ * Provenance-gated savings between two runs. A percentage is emitted ONLY
+ * when both sides carry provider-measured usage ('provider'); every other
+ * source — 'host-measured', 'estimated', 'unknown', absent — returns
+ * reportable:false with savingsPct:null. An unknown source is never 0%:
+ * absence of measurement is not evidence of no cost (SPEC FR-01).
+ * @param {{usageSource?:string, tokens?:number|null}} baseline
+ * @param {{usageSource?:string, tokens?:number|null}} candidate
+ * @returns {{reportable:boolean, savingsPct:number|null, reason:string|null}}
+ */
+export function computeSavings({ baseline = {}, candidate = {} } = {}) {
+  for (const [label, run] of [['baseline', baseline], ['candidate', candidate]]) {
+    const source = run.usageSource;
+    if (source !== 'provider') {
+      const reason = source === 'unknown' || source == null
+        ? 'usage_source_unknown'
+        : 'usage_source_not_provider';
+      return {
+        reportable: false,
+        savingsPct: null,
+        reason: `${reason}:${label}`,
+      };
+    }
+  }
+  const b = baseline.tokens;
+  const c = candidate.tokens;
+  if (typeof b !== 'number' || typeof c !== 'number' || b <= 0) {
+    return { reportable: false, savingsPct: null, reason: 'usage_tokens_missing' };
+  }
+  return {
+    reportable: true,
+    savingsPct: ((b - c) / b) * 100,
+    reason: null,
+  };
+}
+
+// ---------- CLI ----------
+
+const USAGE = 'Usage: node scripts/bench/subagent-orchestrator-corpus.mjs --json\n'
+  + '  --json   print the frozen corpus manifest to stdout (byte-deterministic)\n'
+  + 'Exit codes: 0 = ok, 1 = usage error, 2 = crash.';
+
+function main(argv) {
+  const args = argv.slice(2);
+  if (args.length !== 1 || args[0] !== '--json') {
+    process.stderr.write(USAGE + '\n');
+    return 1;
+  }
+  process.stdout.write(JSON.stringify(CORPUS, null, 2) + '\n');
+  return 0;
+}
+
+const isMain = process.argv[1]
+  && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  try {
+    process.exit(main(process.argv));
+  } catch (error) {
+    process.stderr.write(`corpus: ${error?.stack ?? error}\n`);
+    process.exit(2);
+  }
+}