@event4u/agent-config 8.8.0 → 8.9.0

package/src/scripts/orchestration_record.ts

@@ -0,0 +1,140 @@
+/**
+ * Orchestration telemetry recorder — CLI.
+ *
+ * Appends ONE validated audit-log-v1 line (with an `orchestration` object) to
+ * the monthly audit log, from the counts the orchestrating agent has after an
+ * auto-dispatch. One command instead of hand-authored JSON — the reliable,
+ * lock-respecting (2026-06-30 no-hook decision) capture path.
+ *
+ * Usage:
+ *   ./scripts-run src/scripts/orchestration_record \
+ *     --spawn-count 1 --token-delta -72000 --provenance measured \
+ *     --tier-chosen lite --tier-source inferred --task-class read-only-fanout \
+ *     [--tiers sonnet,opus] [--wall-clock-ms 18500] [--dispatch-outcome DONE] \
+ *     [--dir <audit-dir>] [--dry-run]
+ *
+ * Read by `src/scripts/orchestration_savings_report.ts`.
+ */
+
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import {
+    buildOrchestrationLine,
+    type Provenance,
+    type TierChosen,
+    type TierSource,
+    type DispatchOutcome,
+    type VerifyMode,
+    type Band,
+    type LinePhase,
+    type LineOutcome,
+    type RecordInput,
+} from './_lib/orchestration_record.js';
+
+const DEFAULT_DIR = 'agents/runtime/state/audit';
+
+type Flags = Record<string, string | boolean>;
+
+function parseFlags(argv: string[]): Flags {
+    const flags: Flags = {};
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === undefined || !a.startsWith('--')) continue;
+        const eq = a.indexOf('=');
+        if (eq !== -1) {
+            flags[a.slice(2, eq)] = a.slice(eq + 1);
+            continue;
+        }
+        const key = a.slice(2);
+        const next = argv[i + 1];
+        if (next !== undefined && !next.startsWith('--')) {
+            flags[key] = next;
+            i++;
+        } else {
+            flags[key] = true; // boolean flag (e.g. --dry-run)
+        }
+    }
+    return flags;
+}
+
+function str(flags: Flags, key: string): string | undefined {
+    const v = flags[key];
+    return typeof v === 'string' ? v : undefined;
+}
+
+function int(flags: Flags, key: string): number | undefined {
+    const v = str(flags, key);
+    if (v === undefined) return undefined;
+    const n = Number(v);
+    return Number.isFinite(n) ? n : NaN; // NaN → caught by the lib's isInt validation
+}
+
+/** Deterministic content-hash id (contract allows "ULID or content hash"). */
+function contentHashId(input: RecordInput): string {
+    const canonical = JSON.stringify({
+        ts: input.ts,
+        spawn_count: input.spawn_count,
+        token_delta: input.token_delta,
+        tier_chosen: input.tier_chosen ?? null,
+        task_class: input.task_class ?? null,
+    });
+    return crypto.createHash('sha256').update(canonical).digest('hex').slice(0, 26).toUpperCase();
+}
+
+export function main(argv: string[] = process.argv.slice(2)): number {
+    const flags = parseFlags(argv);
+    const dir = str(flags, 'dir') ?? DEFAULT_DIR;
+    const dryRun = flags['dry-run'] === true;
+    const ts = new Date().toISOString();
+
+    const partial: Omit<RecordInput, 'id'> = {
+        spawn_count: int(flags, 'spawn-count') ?? NaN,
+        token_delta: int(flags, 'token-delta') ?? NaN,
+        token_delta_provenance: str(flags, 'provenance') as Provenance | undefined,
+        tiers: str(flags, 'tiers')?.split(',').map((s) => s.trim()).filter(Boolean),
+        tier_chosen: str(flags, 'tier-chosen') as TierChosen | undefined,
+        tier_source: str(flags, 'tier-source') as TierSource | undefined,
+        task_class: str(flags, 'task-class'),
+        task_size_estimate: int(flags, 'task-size-estimate'),
+        wall_clock_ms: int(flags, 'wall-clock-ms'),
+        dispatch_tokens: int(flags, 'dispatch-tokens'),
+        session_tier: str(flags, 'session-tier'),
+        dispatch_outcome: str(flags, 'dispatch-outcome') as DispatchOutcome | undefined,
+        verify_mode: str(flags, 'verify-mode') as VerifyMode | undefined,
+        phase: str(flags, 'phase') as LinePhase | undefined,
+        outcome: str(flags, 'outcome') as LineOutcome | undefined,
+        confidence_band: str(flags, 'confidence-band') as Band | undefined,
+        risk_class: str(flags, 'risk-class') as Band | undefined,
+        persona: str(flags, 'persona'),
+        work_id: str(flags, 'work-id'),
+        ts,
+    };
+    const input: RecordInput = { ...partial, id: contentHashId({ ...partial, id: '' }) };
+
+    const { line, errors } = buildOrchestrationLine(input);
+    if (errors.length || line === null) {
+        process.stderr.write('orchestration_record: refused — invalid input:\n');
+        for (const e of errors) process.stderr.write(`  - ${e}\n`);
+        return 1;
+    }
+
+    const serialized = JSON.stringify(line);
+    if (dryRun) {
+        process.stdout.write(serialized + '\n');
+        return 0;
+    }
+
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `${ts.slice(0, 7)}.jsonl`);
+    fs.appendFileSync(file, serialized + '\n', 'utf8');
+    process.stdout.write(`orchestration telemetry recorded → ${file} (id ${line.id})\n`);
+    return 0;
+}
+
+if (process.argv[1] !== undefined) {
+    const invokedUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+    if (import.meta.url === invokedUrl) process.exit(main());
+}

package/src/scripts/orchestration_savings_report.ts

@@ -6,10 +6,12 @@
  * mutation. Handles a missing/empty audit dir gracefully.
  *
  * Usage:
- *   ./scripts-run src/scripts/orchestration_savings_report [--dir <path>] [--format text|json]
+ *   ./scripts-run src/scripts/orchestration_savings_report \
+ *     [--dir <path>] [--format text|json] [--weights lite=1,medium=5,high=15]
  *
  * Default dir: agents/runtime/state/audit (the audit-log-v1 path where the
- * orchestration layer writes telemetry lines). See
+ * orchestration layer writes telemetry lines). `--weights` tunes the
+ * provider-neutral tier weights behind the MODELED cost-%. See
  * `src/agent-src/contexts/execution/orchestration-telemetry.md`.
  */
 
@@ -17,17 +19,30 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { pathToFileURL } from 'node:url';
 
-import { aggregateOrchestrationSavings, type AuditLine, type SavingsReport } from './_lib/orchestration_savings.js';
+import { aggregateOrchestrationSavings, DEFAULT_TIER_WEIGHTS, type AuditLine, type SavingsReport, type TierWeights } from './_lib/orchestration_savings.js';
 
 const DEFAULT_DIR = 'agents/runtime/state/audit';
 
 interface Options {
     dir: string;
     format: 'text' | 'json';
+    weights: TierWeights;
+}
+
+/** Parse `lite=1,medium=5,high=15` → weight map. Returns undefined if nothing valid parsed. */
+function parseWeights(spec: string | undefined): TierWeights | undefined {
+    if (!spec) return undefined;
+    const w: TierWeights = {};
+    for (const pair of spec.split(',')) {
+        const [k, v] = pair.split('=');
+        const n = Number(v);
+        if (k && Number.isFinite(n)) w[k.trim()] = n;
+    }
+    return Object.keys(w).length ? w : undefined;
 }
 
 function parseArgs(argv: string[]): Options {
-    const opts: Options = { dir: DEFAULT_DIR, format: 'text' };
+    const opts: Options = { dir: DEFAULT_DIR, format: 'text', weights: DEFAULT_TIER_WEIGHTS };
     for (let i = 0; i < argv.length; i++) {
         const a = argv[i];
         if (a === undefined) continue;
@@ -35,6 +50,8 @@ function parseArgs(argv: string[]): Options {
         else if (a.startsWith('--dir=')) opts.dir = a.slice('--dir='.length);
         else if (a === '--format') opts.format = (argv[++i] as Options['format']) ?? opts.format;
         else if (a.startsWith('--format=')) opts.format = a.slice('--format='.length) as Options['format'];
+        else if (a === '--weights') opts.weights = parseWeights(argv[++i]) ?? opts.weights;
+        else if (a.startsWith('--weights=')) opts.weights = parseWeights(a.slice('--weights='.length)) ?? opts.weights;
     }
     if (opts.format !== 'text' && opts.format !== 'json') opts.format = 'text';
     return opts;
@@ -77,6 +94,13 @@ function renderText(r: SavingsReport, dir: string): string {
         out.push('  by task_class (Δ tokens):');
         for (const [cls, d] of Object.entries(r.by_task_class).sort()) out.push(`    ${cls}: ${d}`);
     }
+    const mc = r.modeled_cost;
+    if (mc.cost_reduction_pct !== null) {
+        out.push(`  MODELED cost reduction: ${(mc.cost_reduction_pct * 100).toFixed(0)}%  (over ${mc.covered_dispatches}/${r.dispatches} dispatch(es) with tier data)`);
+        out.push(`    weights ${JSON.stringify(mc.weights)} · baseline ${mc.baseline_cost_units} → delegated ${mc.delegated_cost_units} cost-units · MODEL, not measured $`);
+    } else {
+        out.push('  MODELED cost reduction: n/a (needs dispatch_tokens + session_tier + tier_chosen on a dispatch)');
+    }
     out.push('  notes:');
     for (const n of r.notes) out.push(`    - ${n}`);
     return out.join('\n');
@@ -85,7 +109,7 @@ function renderText(r: SavingsReport, dir: string): string {
 export function main(argv: string[] = process.argv.slice(2)): number {
     const opts = parseArgs(argv);
     const lines = readAuditLines(opts.dir);
-    const report = aggregateOrchestrationSavings(lines);
+    const report = aggregateOrchestrationSavings(lines, opts.weights);
     if (opts.format === 'json') {
         process.stdout.write(JSON.stringify({ ...report, source: opts.dir }, null, 2) + '\n');
     } else {

package/src/scripts/schemas/skill.schema.json

@@ -13,6 +13,10 @@
       "pattern": "^[a-z][a-z0-9-]*$",
       "description": "Lowercase, hyphenated slug; must match the parent directory name."
     },
+    "user-invocable": {
+      "type": "boolean",
+      "description": "Claude Code Agent-Skills extension: false = the skill claims no /name slash command (only the model can load it). REQUIRED (false) when name equals a Claude Code built-in command or bundled skill (see scripts/_lib/claude_builtin_names.ts) so the suite never shadows the host's own /name. Other tools ignore the key."
+    },
     "description": {
       "type": "string",
       "minLength": 1,

package/src/scripts/score_domain_truth.ts

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+/**
+ * Deterministic domain-truth scorer (road-to-domain-soundness Phase 3).
+ *
+ * Pure, LLM-free scoring for the `deterministic` half of the domain-truth
+ * fixtures (`skills/<skill>/evals/domain-truth.json`). Given a skill's run
+ * output for a case and the case's `expected` + `tolerance`, it extracts the
+ * numeric answer and returns pass/fail. It does NOT invoke any model — the
+ * expensive part (running the skill against the scenario on a fixed host) is
+ * the maintainer's Phase-3 gate; this only scores a captured output.
+ *
+ * Scoring contract (kept simple + gameable-input-resistant):
+ *   - Prefer an explicit `ANSWER: <n>` line (case-insensitive) — the run
+ *     prompt instructs the skill to end with one, so the echoed scenario
+ *     numbers cannot be mistaken for the answer.
+ *   - Fallback: the last numeric token in the output.
+ *   - Normalization: strip `$`, thousands separators, surrounding whitespace,
+ *     and a trailing unit token (`%`, `months`, `mo`, `x`, or stray letters).
+ *     Scenarios fix the unit (e.g. "state in $M"), so no magnitude conversion
+ *     is applied — a unit-instruction miss is a real fail, not a scorer bug.
+ *   - Pass iff |extracted - expected| <= tolerance.
+ *
+ * Rubric cases are NOT scored here (they need a pinned judge, recorded as a
+ * known-limit) — this scorer skips any case whose `check.kind !== "deterministic"`.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+export interface DeterministicCheck {
+    kind: 'deterministic';
+    expected: number | string;
+    tolerance?: number;
+    rationale: string;
+}
+
+export interface CaseResult {
+    id: string;
+    pass: boolean;
+    expected: number;
+    tolerance: number;
+    extracted: number | null;
+    reason: string;
+}
+
+/** Pull a single numeric answer out of a skill's run output. */
+export function extractAnswer(output: string): number | null {
+    if (typeof output !== 'string' || output.trim() === '') return null;
+    // Prefer an explicit ANSWER: line (last one wins if repeated).
+    const answerLines = output.match(/ANSWER:\s*([^\n\r]+)/gi);
+    if (answerLines && answerLines.length > 0) {
+        const last = answerLines[answerLines.length - 1] as string;
+        const n = _firstNumber(last.replace(/ANSWER:\s*/i, ''));
+        if (n !== null) return n;
+    }
+    // Fallback: the last numeric token anywhere in the output.
+    return _lastNumber(output);
+}
+
+/** Normalize a fragment ("$1,350M", "7.5 months", "32%") to a bare number. */
+function _firstNumber(fragment: string): number | null {
+    const cleaned = fragment.replace(/[$,]/g, '');
+    const m = cleaned.match(/-?\d+(?:\.\d+)?/);
+    if (!m) return null;
+    const v = Number(m[0]);
+    return Number.isFinite(v) ? v : null;
+}
+
+function _lastNumber(text: string): number | null {
+    const cleaned = text.replace(/[$,]/g, '');
+    const all = cleaned.match(/-?\d+(?:\.\d+)?/g);
+    if (!all || all.length === 0) return null;
+    const v = Number(all[all.length - 1]);
+    return Number.isFinite(v) ? v : null;
+}
+
+/** Score one deterministic case against a captured output string. */
+export function scoreDeterministic(output: string, check: DeterministicCheck): {
+    pass: boolean;
+    extracted: number | null;
+    expected: number;
+    tolerance: number;
+    reason: string;
+} {
+    const expected = typeof check.expected === 'number' ? check.expected : Number(check.expected);
+    const tolerance = check.tolerance ?? 0;
+    const extracted = extractAnswer(output);
+    if (extracted === null || !Number.isFinite(expected)) {
+        return {
+            pass: false,
+            extracted,
+            expected,
+            tolerance,
+            reason: extracted === null ? 'no numeric answer found in output' : 'expected is not numeric',
+        };
+    }
+    const delta = Math.abs(extracted - expected);
+    const pass = delta <= tolerance;
+    return {
+        pass,
+        extracted,
+        expected,
+        tolerance,
+        reason: pass
+            ? `|${extracted} - ${expected}| = ${_round(delta)} <= ${tolerance}`
+            : `|${extracted} - ${expected}| = ${_round(delta)} > ${tolerance}`,
+    };
+}
+
+function _round(n: number): number {
+    return Math.round(n * 1e6) / 1e6;
+}
+
+/**
+ * Score a whole fixture against a map of {caseId -> captured output}. Rubric
+ * cases are reported as `skipped` (kind !== deterministic), never silently
+ * passed.
+ */
+export function scoreFixture(
+    fixture: { skill: string; cases: Array<{ id: string; check: { kind: string; [k: string]: unknown } }> },
+    outputs: Record<string, string>,
+): { skill: string; results: CaseResult[]; skipped: string[] } {
+    const results: CaseResult[] = [];
+    const skipped: string[] = [];
+    for (const c of fixture.cases) {
+        if (c.check.kind !== 'deterministic') {
+            skipped.push(c.id);
+            continue;
+        }
+        const output = outputs[c.id] ?? '';
+        const r = scoreDeterministic(output, c.check as unknown as DeterministicCheck);
+        results.push({
+            id: c.id,
+            pass: r.pass,
+            expected: r.expected,
+            tolerance: r.tolerance,
+            extracted: r.extracted,
+            reason: output === '' ? 'no captured output for this case' : r.reason,
+        });
+    }
+    return { skill: fixture.skill, results, skipped };
+}
+
+const _HERE = fileURLToPath(import.meta.url);
+export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+
+/**
+ * CLI: `score_domain_truth <skill> <outputs.json>` where outputs.json is a
+ * `{ "<caseId>": "<captured skill output>" }` map. Prints a per-case table and
+ * exits 0 iff every deterministic case passes (1 otherwise).
+ */
+function _runCli(argv: string[]): number {
+    const [skill, outputsPath] = argv;
+    if (!skill || !outputsPath) {
+        process.stderr.write('usage: score_domain_truth <skill> <outputs.json>\n');
+        return 2;
+    }
+    const fixturePath = path.join(REPO_ROOT, 'src', 'skills', skill, 'evals', 'domain-truth.json');
+    if (!fs.existsSync(fixturePath)) {
+        process.stderr.write(`error: no domain-truth.json for skill '${skill}' at ${fixturePath}\n`);
+        return 1;
+    }
+    const fixture = JSON.parse(fs.readFileSync(fixturePath, 'utf-8'));
+    const outputs = JSON.parse(fs.readFileSync(path.resolve(outputsPath), 'utf-8')) as Record<string, string>;
+    const { results, skipped } = scoreFixture(fixture, outputs);
+    process.stdout.write(`domain-truth run — ${skill}\n`);
+    for (const r of results) {
+        const mark = r.pass ? '✅' : '❌';
+        process.stdout.write(`  ${mark} ${r.id}: extracted=${r.extracted} expected=${r.expected} (±${r.tolerance}) — ${r.reason}\n`);
+    }
+    if (skipped.length > 0) {
+        process.stdout.write(`  · skipped (rubric, needs pinned judge): ${skipped.join(', ')}\n`);
+    }
+    const passed = results.filter((r) => r.pass).length;
+    process.stdout.write(`  ${passed}/${results.length} deterministic cases pass\n`);
+    return passed === results.length && results.length > 0 ? 0 : 1;
+}
+
+function _isCliEntry(): boolean {
+    if (process.argv[1] === undefined) return false;
+    const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+    if (import.meta.url === argvUrl) return true;
+    try {
+        return fs.realpathSync(fileURLToPath(import.meta.url)) === fs.realpathSync(path.resolve(process.argv[1]));
+    } catch {
+        return false;
+    }
+}
+
+if (_isCliEntry()) {
+    process.exit(_runCli(process.argv.slice(2)));
+}