cool-workflow 0.1.80 → 0.1.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +42 -2
- package/apps/architecture-review/app.json +1 -1
- package/apps/architecture-review-fast/app.json +1 -1
- package/apps/end-to-end-golden-path/app.json +1 -1
- package/apps/pr-review-fix-ci/app.json +1 -1
- package/apps/release-cut/app.json +1 -1
- package/apps/research-synthesis/app.json +1 -1
- package/dist/agent-config.js +21 -7
- package/dist/candidate-scoring.js +62 -48
- package/dist/capability-core.js +152 -96
- package/dist/capability-registry.js +160 -174
- package/dist/cli.js +90 -100
- package/dist/collaboration.js +5 -6
- package/dist/commit.js +232 -209
- package/dist/compare.js +18 -0
- package/dist/coordinator/classify.js +45 -0
- package/dist/coordinator/paths.js +42 -0
- package/dist/coordinator/util.js +126 -0
- package/dist/coordinator.js +127 -300
- package/dist/dispatch.js +46 -3
- package/dist/drive.js +7 -7
- package/dist/error-feedback.js +8 -4
- package/dist/evidence-reasoning.js +5 -2
- package/dist/execution-backend/agent.js +294 -0
- package/dist/execution-backend/probes.js +96 -0
- package/dist/execution-backend/util.js +47 -0
- package/dist/execution-backend.js +77 -450
- package/dist/gates.js +48 -0
- package/dist/mcp-server.js +34 -173
- package/dist/multi-agent/graph.js +84 -0
- package/dist/multi-agent/helpers.js +141 -0
- package/dist/multi-agent/ids.js +20 -0
- package/dist/multi-agent/paths.js +22 -0
- package/dist/multi-agent-eval/format.js +194 -0
- package/dist/multi-agent-eval/normalize.js +51 -0
- package/dist/multi-agent-eval.js +63 -242
- package/dist/multi-agent-host.js +53 -40
- package/dist/multi-agent-operator-ux.js +2 -1
- package/dist/multi-agent-trust.js +5 -5
- package/dist/multi-agent.js +125 -314
- package/dist/node-projection.js +59 -0
- package/dist/node-snapshot.js +11 -21
- package/dist/observability/format.js +61 -0
- package/dist/observability/intake.js +98 -0
- package/dist/observability.js +14 -160
- package/dist/operator-ux/format.js +364 -0
- package/dist/operator-ux.js +22 -363
- package/dist/orchestrator/lifecycle-operations.js +22 -1
- package/dist/orchestrator/report.js +8 -0
- package/dist/orchestrator.js +41 -10
- package/dist/reclamation.js +32 -55
- package/dist/run-export.js +138 -14
- package/dist/run-registry/derive.js +172 -0
- package/dist/run-registry/format.js +124 -0
- package/dist/run-registry/gc.js +251 -0
- package/dist/run-registry/policy.js +16 -0
- package/dist/run-registry/queue.js +116 -0
- package/dist/run-registry.js +78 -593
- package/dist/run-state-schema.js +1 -0
- package/dist/sandbox-profile.js +43 -2
- package/dist/scheduler.js +34 -4
- package/dist/state-explosion/format.js +159 -0
- package/dist/state-explosion/helpers.js +82 -0
- package/dist/state-explosion.js +65 -283
- package/dist/state-node.js +19 -4
- package/dist/telemetry-attestation.js +55 -0
- package/dist/telemetry-demo.js +15 -3
- package/dist/telemetry-ledger.js +60 -15
- package/dist/topology.js +50 -12
- package/dist/triggers.js +33 -14
- package/dist/trust-audit.js +215 -71
- package/dist/validation.js +328 -0
- package/dist/version.js +1 -1
- package/dist/worker-isolation/helpers.js +51 -0
- package/dist/worker-isolation/paths.js +46 -0
- package/dist/worker-isolation.js +182 -173
- package/docs/agent-delegation-drive.7.md +14 -0
- package/docs/cli-mcp-parity.7.md +5 -0
- package/docs/contract-migration-tooling.7.md +3 -0
- package/docs/control-plane-scheduling.7.md +3 -0
- package/docs/dogfood/resume-drive-real-agent-2026-06-14.md +40 -0
- package/docs/durable-state-and-locking.7.md +5 -0
- package/docs/evidence-adoption-reasoning-chain.7.md +3 -0
- package/docs/execution-backends.7.md +3 -0
- package/docs/index.md +1 -0
- package/docs/launch/launch-kit.md +46 -23
- package/docs/launch/pre-launch-checklist.md +14 -14
- package/docs/multi-agent-cli-mcp-surface.7.md +5 -0
- package/docs/multi-agent-eval-replay-harness.7.md +3 -0
- package/docs/multi-agent-operator-ux.7.md +3 -0
- package/docs/multi-agent-trust-policy-audit.7.md +27 -0
- package/docs/node-snapshot-diff-replay.7.md +3 -0
- package/docs/observability-cost-accounting.7.md +3 -0
- package/docs/project-index.md +23 -6
- package/docs/real-execution-backends.7.md +3 -0
- package/docs/release-and-migration.7.md +5 -0
- package/docs/release-tooling.7.md +35 -2
- package/docs/run-registry-control-plane.7.md +55 -8
- package/docs/run-retention-reclamation.7.md +5 -0
- package/docs/state-explosion-management.7.md +3 -0
- package/docs/team-collaboration.7.md +3 -0
- package/docs/trust-model.md +267 -0
- package/docs/vendor-manifest-loadability.7.md +43 -0
- package/docs/web-desktop-workbench.7.md +3 -0
- package/manifest/plugin.manifest.json +1 -1
- package/manifest/source-context-profiles.json +1 -1
- package/package.json +4 -2
- package/scripts/agents/builtin-templates.json +7 -0
- package/scripts/bump-version.js +5 -11
- package/scripts/canonical-apps-list.js +64 -0
- package/scripts/canonical-apps.js +19 -4
- package/scripts/children/batch-delegate-child.js +58 -0
- package/scripts/children/http-delegate-child.js +39 -0
- package/scripts/dogfood-release.js +1 -1
- package/scripts/golden-path.js +4 -4
- package/scripts/parity-check.js +5 -0
- package/scripts/release-check.js +5 -1
- package/scripts/release-flow.js +181 -5
- package/scripts/version-sync-check.js +5 -8
- package/dist/capability-dispatcher.js +0 -86
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.formatMultiAgentEval = formatMultiAgentEval;
|
|
7
|
+
// Human formatting for the multi-agent eval replay layer (CLI-only; never affects
|
|
8
|
+
// --json / MCP payloads). Pure functions — a result object in, a string out —
|
|
9
|
+
// carved out of multi-agent-eval.ts (FreeBSD-audit god-module split) so the eval
|
|
10
|
+
// router no longer bundles the rendering layer. The runtime-discriminating type
|
|
11
|
+
// guards travel with the renderer that is their only consumer. Re-exported from
|
|
12
|
+
// multi-agent-eval.ts to keep the public surface byte-unchanged.
|
|
13
|
+
//
|
|
14
|
+
// Types are imported type-only from the parent module: `import type` is fully
|
|
15
|
+
// erased at runtime, so there is no import cycle despite the parent re-exporting
|
|
16
|
+
// formatMultiAgentEval from here.
|
|
17
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
18
|
+
function formatMultiAgentEval(value) {
|
|
19
|
+
if (isGate(value)) {
|
|
20
|
+
return [
|
|
21
|
+
"Eval Suite",
|
|
22
|
+
` ${value.suiteId}`,
|
|
23
|
+
"",
|
|
24
|
+
"Replay Status",
|
|
25
|
+
` ${value.status} (${value.score}/${value.maxScore})`,
|
|
26
|
+
"",
|
|
27
|
+
"Regression Findings",
|
|
28
|
+
...(value.findings.length ? value.findings.map((entry) => ` ${entry.severity} ${entry.category}: ${entry.reason}`) : [" none"]),
|
|
29
|
+
"",
|
|
30
|
+
"Final Verdict",
|
|
31
|
+
` ${value.verdict}`,
|
|
32
|
+
"",
|
|
33
|
+
"Next Action",
|
|
34
|
+
` ${value.nextAction}`
|
|
35
|
+
].join("\n");
|
|
36
|
+
}
|
|
37
|
+
if (isScore(value)) {
|
|
38
|
+
return [
|
|
39
|
+
"Eval Suite",
|
|
40
|
+
` ${node_path_1.default.basename(value.paths.suiteDir)}`,
|
|
41
|
+
"",
|
|
42
|
+
"Replay Status",
|
|
43
|
+
` ${value.status} (${value.score}/${value.maxScore})`,
|
|
44
|
+
"",
|
|
45
|
+
"Graph Comparison",
|
|
46
|
+
` ${metricStatus(value, "replay_completed")}; ${metricStatus(value, "graph_parity")}; ${metricStatus(value, "role_parity")}; ${metricStatus(value, "group_parity")}; ${metricStatus(value, "membership_parity")}; ${metricStatus(value, "fanout_parity")}; ${metricStatus(value, "fanin_parity")}; ${metricStatus(value, "dependency_parity")}; ${metricStatus(value, "failure_parity")}`,
|
|
47
|
+
"",
|
|
48
|
+
"Evidence Comparison",
|
|
49
|
+
` ${metricStatus(value, "blackboard_record_parity")}; ${metricStatus(value, "evidence_adoption_parity")}; ${metricStatus(value, "blackboard_provenance_parity")}`,
|
|
50
|
+
"",
|
|
51
|
+
"Trust / Policy / Audit Comparison",
|
|
52
|
+
` ${metricStatus(value, "trust_audit_parity")}; ${metricStatus(value, "role_policy_parity")}; ${metricStatus(value, "permission_decision_parity")}; ${metricStatus(value, "policy_violation_parity")}; ${metricStatus(value, "judge_rationale_parity")}; ${metricStatus(value, "panel_decision_parity")}`,
|
|
53
|
+
"",
|
|
54
|
+
"Candidate Score Comparison",
|
|
55
|
+
` ${metricStatus(value, "candidate_score_parity")}`,
|
|
56
|
+
"",
|
|
57
|
+
"Selection / Commit Gate",
|
|
58
|
+
` ${metricStatus(value, "selection_parity")}; ${metricStatus(value, "verifier_commit_gate_parity")}`,
|
|
59
|
+
"",
|
|
60
|
+
"State Explosion Summaries",
|
|
61
|
+
` ${metricStatus(value, "summary_freshness")}; ${metricStatus(value, "compact_graph_parity")}; ${metricStatus(value, "blackboard_digest_parity")}; ${metricStatus(value, "critical_path_parity")}; ${metricStatus(value, "evidence_digest_parity")}; ${metricStatus(value, "expansion_ref_integrity")}`,
|
|
62
|
+
"",
|
|
63
|
+
"Regression Findings",
|
|
64
|
+
...(value.findings.length ? value.findings.map((entry) => ` ${entry.severity} ${entry.category}: ${entry.reason}`) : [" none"]),
|
|
65
|
+
"",
|
|
66
|
+
"Final Verdict",
|
|
67
|
+
` ${value.status}`,
|
|
68
|
+
"",
|
|
69
|
+
"Next Action",
|
|
70
|
+
` ${value.status === "pass" ? "Run eval gate or include report path as evidence." : "Review findings before release."}`
|
|
71
|
+
].join("\n");
|
|
72
|
+
}
|
|
73
|
+
if (isComparison(value)) {
|
|
74
|
+
return [
|
|
75
|
+
"Eval Suite",
|
|
76
|
+
` ${node_path_1.default.basename(value.paths.suiteDir)}`,
|
|
77
|
+
"",
|
|
78
|
+
"Replay Status",
|
|
79
|
+
` ${value.status}`,
|
|
80
|
+
"",
|
|
81
|
+
"Graph Comparison",
|
|
82
|
+
` ${sectionStatus(value, "workflow")}; ${sectionStatus(value, "topologyShape")}; ${sectionStatus(value, "roles")}; ${sectionStatus(value, "groups")}; ${sectionStatus(value, "memberships")}; ${sectionStatus(value, "fanouts")}; ${sectionStatus(value, "fanins")}; ${sectionStatus(value, "dependencyEdges")}; ${sectionStatus(value, "failures")}`,
|
|
83
|
+
"",
|
|
84
|
+
"Evidence Comparison",
|
|
85
|
+
` ${sectionStatus(value, "blackboardRecords")}; ${sectionStatus(value, "evidenceAdoption")}; ${sectionStatus(value, "messageProvenance")}`,
|
|
86
|
+
"",
|
|
87
|
+
"Trust / Policy / Audit Comparison",
|
|
88
|
+
` ${sectionStatus(value, "blackboardWriteAudit")}; ${sectionStatus(value, "rolePolicies")}; ${sectionStatus(value, "permissionDecisions")}; ${sectionStatus(value, "policyViolations")}; ${sectionStatus(value, "judgeRationales")}; ${sectionStatus(value, "panelDecisions")}`,
|
|
89
|
+
"",
|
|
90
|
+
"Candidate Score Comparison",
|
|
91
|
+
` ${sectionStatus(value, "candidateScores")}`,
|
|
92
|
+
"",
|
|
93
|
+
"Selection / Commit Gate",
|
|
94
|
+
` ${sectionStatus(value, "selectedCandidates")}; ${sectionStatus(value, "verifierCommitGate")}`,
|
|
95
|
+
"",
|
|
96
|
+
"Regression Findings",
|
|
97
|
+
...(value.findings.length ? value.findings.map((entry) => ` ${entry.severity} ${entry.category}: ${entry.reason}`) : [" none"]),
|
|
98
|
+
"",
|
|
99
|
+
"Final Verdict",
|
|
100
|
+
` ${value.status}`,
|
|
101
|
+
"",
|
|
102
|
+
"Next Action",
|
|
103
|
+
" Score the replay or run the eval gate."
|
|
104
|
+
].join("\n");
|
|
105
|
+
}
|
|
106
|
+
if (isReplay(value)) {
|
|
107
|
+
return [
|
|
108
|
+
"Eval Suite",
|
|
109
|
+
` ${node_path_1.default.basename(value.paths.suiteDir)}`,
|
|
110
|
+
"",
|
|
111
|
+
"Replay Status",
|
|
112
|
+
` ${value.status}`,
|
|
113
|
+
` replay=${value.paths.replayRunPath}`,
|
|
114
|
+
"",
|
|
115
|
+
"Next Action",
|
|
116
|
+
` node scripts/cw.js eval compare ${value.paths.snapshotPath} ${value.paths.replayRunPath}`
|
|
117
|
+
].join("\n");
|
|
118
|
+
}
|
|
119
|
+
if (isSnapshot(value)) {
|
|
120
|
+
return [
|
|
121
|
+
"Eval Suite",
|
|
122
|
+
` ${value.id}`,
|
|
123
|
+
"",
|
|
124
|
+
"Replay Status",
|
|
125
|
+
" snapshot captured",
|
|
126
|
+
` snapshot=${value.paths.snapshotPath}`,
|
|
127
|
+
"",
|
|
128
|
+
"Graph Comparison",
|
|
129
|
+
` topology records=${value.normalized.topologyShape.length}`,
|
|
130
|
+
"",
|
|
131
|
+
"Evidence Comparison",
|
|
132
|
+
` evidence records=${value.normalized.evidenceAdoption.length}`,
|
|
133
|
+
"",
|
|
134
|
+
"Trust / Policy / Audit Comparison",
|
|
135
|
+
` audit records=${value.normalized.blackboardWriteAudit.length + value.normalized.messageProvenance.length}`,
|
|
136
|
+
"",
|
|
137
|
+
"Candidate Score Comparison",
|
|
138
|
+
` score records=${value.normalized.candidateScores.length}`,
|
|
139
|
+
"",
|
|
140
|
+
"Selection / Commit Gate",
|
|
141
|
+
` selected=${value.normalized.selectedCandidates.length}; commit gates=${value.normalized.verifierCommitGate.length}`,
|
|
142
|
+
"",
|
|
143
|
+
"Regression Findings",
|
|
144
|
+
" none",
|
|
145
|
+
"",
|
|
146
|
+
"Final Verdict",
|
|
147
|
+
" snapshot-ready",
|
|
148
|
+
"",
|
|
149
|
+
"Next Action",
|
|
150
|
+
` node scripts/cw.js eval replay ${value.paths.snapshotPath}`
|
|
151
|
+
].join("\n");
|
|
152
|
+
}
|
|
153
|
+
if (isReport(value)) {
|
|
154
|
+
return [
|
|
155
|
+
"Eval Suite",
|
|
156
|
+
` ${node_path_1.default.dirname(value.reportPath)}`,
|
|
157
|
+
"",
|
|
158
|
+
"Replay Status",
|
|
159
|
+
` ${value.status} (${value.score}/${value.maxScore})`,
|
|
160
|
+
"",
|
|
161
|
+
"Final Verdict",
|
|
162
|
+
` report written: ${value.reportPath}`,
|
|
163
|
+
"",
|
|
164
|
+
"Next Action",
|
|
165
|
+
" Run eval gate if this is release evidence."
|
|
166
|
+
].join("\n");
|
|
167
|
+
}
|
|
168
|
+
return JSON.stringify(value, null, 2);
|
|
169
|
+
}
|
|
170
|
+
function metricStatus(score, id) {
|
|
171
|
+
const metric = score.metrics.find((entry) => entry.id === id);
|
|
172
|
+
return `${id}=${metric?.status || "missing"}`;
|
|
173
|
+
}
|
|
174
|
+
function sectionStatus(comparison, id) {
|
|
175
|
+
return `${id}=${comparison.sections[id]?.status || "missing"}`;
|
|
176
|
+
}
|
|
177
|
+
function isSnapshot(value) {
|
|
178
|
+
return Boolean(value && typeof value === "object" && value.kind === "multi-agent-replay-snapshot");
|
|
179
|
+
}
|
|
180
|
+
function isReplay(value) {
|
|
181
|
+
return Boolean(value && typeof value === "object" && value.kind === "multi-agent-replay-run");
|
|
182
|
+
}
|
|
183
|
+
function isComparison(value) {
|
|
184
|
+
return Boolean(value && typeof value === "object" && "sections" in value && "findings" in value);
|
|
185
|
+
}
|
|
186
|
+
function isScore(value) {
|
|
187
|
+
return Boolean(value && typeof value === "object" && "metrics" in value && "score" in value);
|
|
188
|
+
}
|
|
189
|
+
function isGate(value) {
|
|
190
|
+
return Boolean(value && typeof value === "object" && "verdict" in value && "requiredArtifacts" in value);
|
|
191
|
+
}
|
|
192
|
+
function isReport(value) {
|
|
193
|
+
return Boolean(value && typeof value === "object" && "reportPath" in value && !("verdict" in value));
|
|
194
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Pure, stateless normalization helpers for the multi-agent eval replay layer —
|
|
3
|
+
// timestamp/path scrubbing, recursive value normalization, and stable
|
|
4
|
+
// stringification. Carved out of multi-agent-eval.ts (FreeBSD-audit god-module
|
|
5
|
+
// split) so the eval router no longer bundles the deterministic-normalization
|
|
6
|
+
// primitives. Nothing here touches state; everything is a pure function of its
|
|
7
|
+
// arguments. Re-exported verbatim from multi-agent-eval.ts so every importer
|
|
8
|
+
// (including node-snapshot.ts and reclamation.ts) stays byte-unchanged.
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.normalizeValue = normalizeValue;
|
|
11
|
+
exports.lines = lines;
|
|
12
|
+
exports.replayStableStringify = replayStableStringify;
|
|
13
|
+
function normalizeValue(value) {
|
|
14
|
+
if (Array.isArray(value))
|
|
15
|
+
return value.map(normalizeValue);
|
|
16
|
+
if (!value || typeof value !== "object") {
|
|
17
|
+
if (typeof value === "string")
|
|
18
|
+
return normalizeString(value);
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
const record = value;
|
|
22
|
+
const normalized = {};
|
|
23
|
+
for (const key of Object.keys(record).sort()) {
|
|
24
|
+
if (["createdAt", "updatedAt", "recordedAt", "selectedAt", "replayedAt", "generatedAt"].includes(key))
|
|
25
|
+
continue;
|
|
26
|
+
if (key.endsWith("Path") || key === "path" || key === "cwd" || key === "runDir" || key.endsWith("Dir")) {
|
|
27
|
+
normalized[key] = normalizeString(String(record[key]));
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
normalized[key] = normalizeValue(record[key]);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return normalized;
|
|
34
|
+
}
|
|
35
|
+
function normalizeString(value) {
|
|
36
|
+
return value
|
|
37
|
+
.replace(/[0-9]{8}T[0-9]{6}Z/g, "<timestamp>")
|
|
38
|
+
.replace(/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z/g, "<timestamp>")
|
|
39
|
+
.replace(/\/[^"\s]+\/\.cw\/runs\/[^"\s/]+/g, "<run-dir>")
|
|
40
|
+
.replace(/\/[^"\s]+\/\.cw\/evals\/[^"\s/]+/g, "<eval-dir>")
|
|
41
|
+
.replace(/\/var\/folders\/[^"\s]+|\/tmp\/[^"\s]+|\/private\/tmp\/[^"\s]+/g, "<tmp>");
|
|
42
|
+
}
|
|
43
|
+
function lines(value) {
|
|
44
|
+
const normalized = normalizeValue(value);
|
|
45
|
+
if (Array.isArray(normalized))
|
|
46
|
+
return normalized.map((entry) => replayStableStringify(entry)).sort();
|
|
47
|
+
return [replayStableStringify(normalized)].sort();
|
|
48
|
+
}
|
|
49
|
+
function replayStableStringify(value) {
|
|
50
|
+
return JSON.stringify(normalizeValue(value));
|
|
51
|
+
}
|
package/dist/multi-agent-eval.js
CHANGED
|
@@ -3,16 +3,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.formatMultiAgentEval = exports.replayStableStringify = exports.normalizeValue = exports.lines = void 0;
|
|
6
7
|
exports.createMultiAgentReplaySnapshot = createMultiAgentReplaySnapshot;
|
|
7
8
|
exports.replayMultiAgentSnapshot = replayMultiAgentSnapshot;
|
|
8
9
|
exports.compareMultiAgentReplay = compareMultiAgentReplay;
|
|
9
10
|
exports.scoreMultiAgentReplay = scoreMultiAgentReplay;
|
|
10
11
|
exports.gateMultiAgentEval = gateMultiAgentEval;
|
|
11
12
|
exports.reportMultiAgentEval = reportMultiAgentEval;
|
|
12
|
-
exports.formatMultiAgentEval = formatMultiAgentEval;
|
|
13
|
-
exports.normalizeValue = normalizeValue;
|
|
14
|
-
exports.lines = lines;
|
|
15
|
-
exports.stableStringify = stableStringify;
|
|
16
13
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
17
14
|
const node_path_1 = __importDefault(require("node:path"));
|
|
18
15
|
const multi_agent_1 = require("./multi-agent");
|
|
@@ -24,6 +21,16 @@ const trust_audit_1 = require("./trust-audit");
|
|
|
24
21
|
const state_explosion_1 = require("./state-explosion");
|
|
25
22
|
const evidence_reasoning_1 = require("./evidence-reasoning");
|
|
26
23
|
const state_1 = require("./state");
|
|
24
|
+
const normalize_1 = require("./multi-agent-eval/normalize");
|
|
25
|
+
// Pure normalization primitives carved into ./multi-agent-eval/normalize.ts;
|
|
26
|
+
// re-exported verbatim so every external importer stays byte-unchanged.
|
|
27
|
+
var normalize_2 = require("./multi-agent-eval/normalize");
|
|
28
|
+
Object.defineProperty(exports, "lines", { enumerable: true, get: function () { return normalize_2.lines; } });
|
|
29
|
+
Object.defineProperty(exports, "normalizeValue", { enumerable: true, get: function () { return normalize_2.normalizeValue; } });
|
|
30
|
+
Object.defineProperty(exports, "replayStableStringify", { enumerable: true, get: function () { return normalize_2.replayStableStringify; } });
|
|
31
|
+
// Human formatter (CLI-only renderer) carved into ./multi-agent-eval/format.ts.
|
|
32
|
+
var format_1 = require("./multi-agent-eval/format");
|
|
33
|
+
Object.defineProperty(exports, "formatMultiAgentEval", { enumerable: true, get: function () { return format_1.formatMultiAgentEval; } });
|
|
27
34
|
const METRIC_SECTIONS = [
|
|
28
35
|
{ metric: "replay_completed", section: "workflow", title: "Replay completed" },
|
|
29
36
|
{ metric: "graph_parity", section: "topologyShape", title: "Topology graph parity" },
|
|
@@ -84,7 +91,7 @@ function createMultiAgentReplaySnapshot(run, options = {}) {
|
|
|
84
91
|
appVersion: run.workflow.app?.version,
|
|
85
92
|
title: run.workflow.title
|
|
86
93
|
},
|
|
87
|
-
inputs: normalizeValue(run.inputs),
|
|
94
|
+
inputs: (0, normalize_1.normalizeValue)(run.inputs),
|
|
88
95
|
paths: {
|
|
89
96
|
suiteDir,
|
|
90
97
|
snapshotPath,
|
|
@@ -117,6 +124,14 @@ function replayMultiAgentSnapshot(target, options = {}) {
|
|
|
117
124
|
const replayDir = node_path_1.default.join(suiteDir, "replay");
|
|
118
125
|
const replayRunPath = node_path_1.default.join(suiteDir, "replay-run.json");
|
|
119
126
|
node_fs_1.default.mkdirSync(replayDir, { recursive: true });
|
|
127
|
+
// RE-DERIVE the projection from the raw captured state instead of copying
|
|
128
|
+
// snapshot.normalized. Copying the baseline would make compareMultiAgentReplay
|
|
129
|
+
// diff the baseline against a byte-copy of itself, so a projection-determinism
|
|
130
|
+
// regression in normalizeRun could never be caught (the determinism moat would
|
|
131
|
+
// be false-green). We reconstruct the run from the captured raw state and run
|
|
132
|
+
// the SAME normalizeRun pipeline used at capture time, producing an INDEPENDENT
|
|
133
|
+
// re-projection the comparison can pit against the baseline.
|
|
134
|
+
const replayed = rederiveNormalizedFromSnapshot(snapshot);
|
|
120
135
|
const replay = {
|
|
121
136
|
schemaVersion: 1,
|
|
122
137
|
kind: "multi-agent-replay-run",
|
|
@@ -132,7 +147,7 @@ function replayMultiAgentSnapshot(target, options = {}) {
|
|
|
132
147
|
replayRunPath,
|
|
133
148
|
snapshotPath: snapshot.paths.snapshotPath
|
|
134
149
|
},
|
|
135
|
-
replay:
|
|
150
|
+
replay: replayed,
|
|
136
151
|
errors: []
|
|
137
152
|
};
|
|
138
153
|
(0, state_1.writeJson)(replayRunPath, replay);
|
|
@@ -152,7 +167,7 @@ function compareMultiAgentReplay(baselineTarget, replayTarget) {
|
|
|
152
167
|
const findings = [];
|
|
153
168
|
for (const spec of ALL_METRIC_SECTIONS) {
|
|
154
169
|
const { baselineValue, replayValue } = comparisonValues(spec.metric, spec.section, baseline.normalized, replay);
|
|
155
|
-
const equal =
|
|
170
|
+
const equal = (0, normalize_1.replayStableStringify)(baselineValue) === (0, normalize_1.replayStableStringify)(replayValue);
|
|
156
171
|
const id = String(spec.section);
|
|
157
172
|
sections[id] = {
|
|
158
173
|
id,
|
|
@@ -391,157 +406,23 @@ function loadScoreForTarget(target, scorePath) {
|
|
|
391
406
|
}
|
|
392
407
|
return scoreMultiAgentReplay(target);
|
|
393
408
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
"",
|
|
406
|
-
"Final Verdict",
|
|
407
|
-
` ${value.verdict}`,
|
|
408
|
-
"",
|
|
409
|
-
"Next Action",
|
|
410
|
-
` ${value.nextAction}`
|
|
411
|
-
].join("\n");
|
|
409
|
+
// Re-derive snapshot.normalized INDEPENDENTLY at replay time. The raw captured
|
|
410
|
+
// state is the baseline run state file (snapshot.paths.baselineStatePath) — the
|
|
411
|
+
// same WorkflowRun captureRun/normalizeRun projected at capture time. We re-load
|
|
412
|
+
// it and re-run the IDENTICAL normalizeRun pipeline, so the result is a genuine
|
|
413
|
+
// re-projection rather than a copy of the baseline. Fail-closed: if the raw
|
|
414
|
+
// state cannot be reconstructed we throw (never silently fall back to the
|
|
415
|
+
// baseline copy — that is exactly the false-green this guards against).
|
|
416
|
+
function rederiveNormalizedFromSnapshot(snapshot) {
|
|
417
|
+
const statePath = snapshot.paths.baselineStatePath;
|
|
418
|
+
if (!statePath || !node_fs_1.default.existsSync(statePath)) {
|
|
419
|
+
throw new Error(`Cannot re-derive replay projection: baseline run state missing at ${statePath || "<unset>"}; re-snapshot from a live run before replaying.`);
|
|
412
420
|
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
` ${node_path_1.default.basename(value.paths.suiteDir)}`,
|
|
417
|
-
"",
|
|
418
|
-
"Replay Status",
|
|
419
|
-
` ${value.status} (${value.score}/${value.maxScore})`,
|
|
420
|
-
"",
|
|
421
|
-
"Graph Comparison",
|
|
422
|
-
` ${metricStatus(value, "replay_completed")}; ${metricStatus(value, "graph_parity")}; ${metricStatus(value, "role_parity")}; ${metricStatus(value, "group_parity")}; ${metricStatus(value, "membership_parity")}; ${metricStatus(value, "fanout_parity")}; ${metricStatus(value, "fanin_parity")}; ${metricStatus(value, "dependency_parity")}; ${metricStatus(value, "failure_parity")}`,
|
|
423
|
-
"",
|
|
424
|
-
"Evidence Comparison",
|
|
425
|
-
` ${metricStatus(value, "blackboard_record_parity")}; ${metricStatus(value, "evidence_adoption_parity")}; ${metricStatus(value, "blackboard_provenance_parity")}`,
|
|
426
|
-
"",
|
|
427
|
-
"Trust / Policy / Audit Comparison",
|
|
428
|
-
` ${metricStatus(value, "trust_audit_parity")}; ${metricStatus(value, "role_policy_parity")}; ${metricStatus(value, "permission_decision_parity")}; ${metricStatus(value, "policy_violation_parity")}; ${metricStatus(value, "judge_rationale_parity")}; ${metricStatus(value, "panel_decision_parity")}`,
|
|
429
|
-
"",
|
|
430
|
-
"Candidate Score Comparison",
|
|
431
|
-
` ${metricStatus(value, "candidate_score_parity")}`,
|
|
432
|
-
"",
|
|
433
|
-
"Selection / Commit Gate",
|
|
434
|
-
` ${metricStatus(value, "selection_parity")}; ${metricStatus(value, "verifier_commit_gate_parity")}`,
|
|
435
|
-
"",
|
|
436
|
-
"State Explosion Summaries",
|
|
437
|
-
` ${metricStatus(value, "summary_freshness")}; ${metricStatus(value, "compact_graph_parity")}; ${metricStatus(value, "blackboard_digest_parity")}; ${metricStatus(value, "critical_path_parity")}; ${metricStatus(value, "evidence_digest_parity")}; ${metricStatus(value, "expansion_ref_integrity")}`,
|
|
438
|
-
"",
|
|
439
|
-
"Regression Findings",
|
|
440
|
-
...(value.findings.length ? value.findings.map((entry) => ` ${entry.severity} ${entry.category}: ${entry.reason}`) : [" none"]),
|
|
441
|
-
"",
|
|
442
|
-
"Final Verdict",
|
|
443
|
-
` ${value.status}`,
|
|
444
|
-
"",
|
|
445
|
-
"Next Action",
|
|
446
|
-
` ${value.status === "pass" ? "Run eval gate or include report path as evidence." : "Review findings before release."}`
|
|
447
|
-
].join("\n");
|
|
421
|
+
const result = (0, state_1.loadRunStateFile)(statePath, { dryRun: true });
|
|
422
|
+
if (result.report.status === "unsupported") {
|
|
423
|
+
throw new Error(`Cannot re-derive replay projection: baseline run state at ${statePath} is unsupported: ${result.report.errors.join("; ")}`);
|
|
448
424
|
}
|
|
449
|
-
|
|
450
|
-
return [
|
|
451
|
-
"Eval Suite",
|
|
452
|
-
` ${node_path_1.default.basename(value.paths.suiteDir)}`,
|
|
453
|
-
"",
|
|
454
|
-
"Replay Status",
|
|
455
|
-
` ${value.status}`,
|
|
456
|
-
"",
|
|
457
|
-
"Graph Comparison",
|
|
458
|
-
` ${sectionStatus(value, "workflow")}; ${sectionStatus(value, "topologyShape")}; ${sectionStatus(value, "roles")}; ${sectionStatus(value, "groups")}; ${sectionStatus(value, "memberships")}; ${sectionStatus(value, "fanouts")}; ${sectionStatus(value, "fanins")}; ${sectionStatus(value, "dependencyEdges")}; ${sectionStatus(value, "failures")}`,
|
|
459
|
-
"",
|
|
460
|
-
"Evidence Comparison",
|
|
461
|
-
` ${sectionStatus(value, "blackboardRecords")}; ${sectionStatus(value, "evidenceAdoption")}; ${sectionStatus(value, "messageProvenance")}`,
|
|
462
|
-
"",
|
|
463
|
-
"Trust / Policy / Audit Comparison",
|
|
464
|
-
` ${sectionStatus(value, "blackboardWriteAudit")}; ${sectionStatus(value, "rolePolicies")}; ${sectionStatus(value, "permissionDecisions")}; ${sectionStatus(value, "policyViolations")}; ${sectionStatus(value, "judgeRationales")}; ${sectionStatus(value, "panelDecisions")}`,
|
|
465
|
-
"",
|
|
466
|
-
"Candidate Score Comparison",
|
|
467
|
-
` ${sectionStatus(value, "candidateScores")}`,
|
|
468
|
-
"",
|
|
469
|
-
"Selection / Commit Gate",
|
|
470
|
-
` ${sectionStatus(value, "selectedCandidates")}; ${sectionStatus(value, "verifierCommitGate")}`,
|
|
471
|
-
"",
|
|
472
|
-
"Regression Findings",
|
|
473
|
-
...(value.findings.length ? value.findings.map((entry) => ` ${entry.severity} ${entry.category}: ${entry.reason}`) : [" none"]),
|
|
474
|
-
"",
|
|
475
|
-
"Final Verdict",
|
|
476
|
-
` ${value.status}`,
|
|
477
|
-
"",
|
|
478
|
-
"Next Action",
|
|
479
|
-
" Score the replay or run the eval gate."
|
|
480
|
-
].join("\n");
|
|
481
|
-
}
|
|
482
|
-
if (isReplay(value)) {
|
|
483
|
-
return [
|
|
484
|
-
"Eval Suite",
|
|
485
|
-
` ${node_path_1.default.basename(value.paths.suiteDir)}`,
|
|
486
|
-
"",
|
|
487
|
-
"Replay Status",
|
|
488
|
-
` ${value.status}`,
|
|
489
|
-
` replay=${value.paths.replayRunPath}`,
|
|
490
|
-
"",
|
|
491
|
-
"Next Action",
|
|
492
|
-
` node scripts/cw.js eval compare ${value.paths.snapshotPath} ${value.paths.replayRunPath}`
|
|
493
|
-
].join("\n");
|
|
494
|
-
}
|
|
495
|
-
if (isSnapshot(value)) {
|
|
496
|
-
return [
|
|
497
|
-
"Eval Suite",
|
|
498
|
-
` ${value.id}`,
|
|
499
|
-
"",
|
|
500
|
-
"Replay Status",
|
|
501
|
-
" snapshot captured",
|
|
502
|
-
` snapshot=${value.paths.snapshotPath}`,
|
|
503
|
-
"",
|
|
504
|
-
"Graph Comparison",
|
|
505
|
-
` topology records=${value.normalized.topologyShape.length}`,
|
|
506
|
-
"",
|
|
507
|
-
"Evidence Comparison",
|
|
508
|
-
` evidence records=${value.normalized.evidenceAdoption.length}`,
|
|
509
|
-
"",
|
|
510
|
-
"Trust / Policy / Audit Comparison",
|
|
511
|
-
` audit records=${value.normalized.blackboardWriteAudit.length + value.normalized.messageProvenance.length}`,
|
|
512
|
-
"",
|
|
513
|
-
"Candidate Score Comparison",
|
|
514
|
-
` score records=${value.normalized.candidateScores.length}`,
|
|
515
|
-
"",
|
|
516
|
-
"Selection / Commit Gate",
|
|
517
|
-
` selected=${value.normalized.selectedCandidates.length}; commit gates=${value.normalized.verifierCommitGate.length}`,
|
|
518
|
-
"",
|
|
519
|
-
"Regression Findings",
|
|
520
|
-
" none",
|
|
521
|
-
"",
|
|
522
|
-
"Final Verdict",
|
|
523
|
-
" snapshot-ready",
|
|
524
|
-
"",
|
|
525
|
-
"Next Action",
|
|
526
|
-
` node scripts/cw.js eval replay ${value.paths.snapshotPath}`
|
|
527
|
-
].join("\n");
|
|
528
|
-
}
|
|
529
|
-
if (isReport(value)) {
|
|
530
|
-
return [
|
|
531
|
-
"Eval Suite",
|
|
532
|
-
` ${node_path_1.default.dirname(value.reportPath)}`,
|
|
533
|
-
"",
|
|
534
|
-
"Replay Status",
|
|
535
|
-
` ${value.status} (${value.score}/${value.maxScore})`,
|
|
536
|
-
"",
|
|
537
|
-
"Final Verdict",
|
|
538
|
-
` report written: ${value.reportPath}`,
|
|
539
|
-
"",
|
|
540
|
-
"Next Action",
|
|
541
|
-
" Run eval gate if this is release evidence."
|
|
542
|
-
].join("\n");
|
|
543
|
-
}
|
|
544
|
-
return JSON.stringify(value, null, 2);
|
|
425
|
+
return normalizeRun(result.run);
|
|
545
426
|
}
|
|
546
427
|
function captureRun(run) {
|
|
547
428
|
return {
|
|
@@ -566,13 +447,13 @@ function normalizeRun(run) {
|
|
|
566
447
|
const topologies = (0, topology_1.summarizeTopologies)(run);
|
|
567
448
|
const multiAgent = (0, multi_agent_1.summarizeMultiAgent)(run);
|
|
568
449
|
return {
|
|
569
|
-
workflow: normalizeValue({
|
|
450
|
+
workflow: (0, normalize_1.normalizeValue)({
|
|
570
451
|
id: run.workflow.id,
|
|
571
452
|
appId: run.workflow.app?.id,
|
|
572
453
|
appVersion: run.workflow.app?.version,
|
|
573
454
|
taskCount: run.tasks.length
|
|
574
455
|
}),
|
|
575
|
-
topologyShape: lines([
|
|
456
|
+
topologyShape: (0, normalize_1.lines)([
|
|
576
457
|
topologies.active.map((entry) => ({
|
|
577
458
|
topologyId: entry.topologyId,
|
|
578
459
|
status: entry.status,
|
|
@@ -583,22 +464,22 @@ function normalizeRun(run) {
|
|
|
583
464
|
})),
|
|
584
465
|
multiAgent.groupsDetail
|
|
585
466
|
]),
|
|
586
|
-
roles: lines(run.multiAgent?.roles || []),
|
|
587
|
-
groups: lines(run.multiAgent?.groups || []),
|
|
588
|
-
memberships: lines(run.multiAgent?.memberships || []),
|
|
589
|
-
fanouts: lines(run.multiAgent?.fanouts || []),
|
|
590
|
-
fanins: lines(run.multiAgent?.fanins || []),
|
|
591
|
-
dependencyEdges: lines(operator.dependencies.map((entry) => ({ from: entry.from, to: entry.to, label: entry.label, status: entry.status }))),
|
|
592
|
-
failures: lines(operator.failures.map((entry) => ({ kind: entry.kind, status: entry.status, owner: entry.owner, reason: entry.reason }))),
|
|
593
|
-
blackboardRecords: lines([blackboard.boards, blackboard.topics, blackboard.messages, blackboard.contexts, blackboard.artifacts, blackboard.snapshots, blackboard.decisions]),
|
|
594
|
-
messageProvenance: lines(trust.messageProvenance || []),
|
|
595
|
-
rolePolicies: lines(trust.rolePolicies || []),
|
|
596
|
-
permissionDecisions: lines(trust.permissionDecisions || []),
|
|
597
|
-
blackboardWriteAudit: lines(trust.blackboardWrites || []),
|
|
598
|
-
judgeRationales: lines(trust.judgeRationales || []),
|
|
599
|
-
panelDecisions: lines(trust.panelDecisions || []),
|
|
600
|
-
policyViolations: lines(trust.policyViolations || []),
|
|
601
|
-
evidenceAdoption: lines(operator.evidence.map((entry) => ({
|
|
467
|
+
roles: (0, normalize_1.lines)(run.multiAgent?.roles || []),
|
|
468
|
+
groups: (0, normalize_1.lines)(run.multiAgent?.groups || []),
|
|
469
|
+
memberships: (0, normalize_1.lines)(run.multiAgent?.memberships || []),
|
|
470
|
+
fanouts: (0, normalize_1.lines)(run.multiAgent?.fanouts || []),
|
|
471
|
+
fanins: (0, normalize_1.lines)(run.multiAgent?.fanins || []),
|
|
472
|
+
dependencyEdges: (0, normalize_1.lines)(operator.dependencies.map((entry) => ({ from: entry.from, to: entry.to, label: entry.label, status: entry.status }))),
|
|
473
|
+
failures: (0, normalize_1.lines)(operator.failures.map((entry) => ({ kind: entry.kind, status: entry.status, owner: entry.owner, reason: entry.reason }))),
|
|
474
|
+
blackboardRecords: (0, normalize_1.lines)([blackboard.boards, blackboard.topics, blackboard.messages, blackboard.contexts, blackboard.artifacts, blackboard.snapshots, blackboard.decisions]),
|
|
475
|
+
messageProvenance: (0, normalize_1.lines)(trust.messageProvenance || []),
|
|
476
|
+
rolePolicies: (0, normalize_1.lines)(trust.rolePolicies || []),
|
|
477
|
+
permissionDecisions: (0, normalize_1.lines)(trust.permissionDecisions || []),
|
|
478
|
+
blackboardWriteAudit: (0, normalize_1.lines)(trust.blackboardWrites || []),
|
|
479
|
+
judgeRationales: (0, normalize_1.lines)(trust.judgeRationales || []),
|
|
480
|
+
panelDecisions: (0, normalize_1.lines)(trust.panelDecisions || []),
|
|
481
|
+
policyViolations: (0, normalize_1.lines)(trust.policyViolations || []),
|
|
482
|
+
evidenceAdoption: (0, normalize_1.lines)(operator.evidence.map((entry) => ({
|
|
602
483
|
ref: entry.ref || entry.id,
|
|
603
484
|
status: entry.status,
|
|
604
485
|
adoptedBy: entry.adoptedBy,
|
|
@@ -606,15 +487,15 @@ function normalizeRun(run) {
|
|
|
606
487
|
selectionIds: entry.selectionIds,
|
|
607
488
|
commitIds: entry.commitIds
|
|
608
489
|
}))),
|
|
609
|
-
candidateScores: lines(collectCandidateScores(run)),
|
|
610
|
-
selectedCandidates: lines((run.candidateSelections || []).map((entry) => ({
|
|
490
|
+
candidateScores: (0, normalize_1.lines)(collectCandidateScores(run)),
|
|
491
|
+
selectedCandidates: (0, normalize_1.lines)((run.candidateSelections || []).map((entry) => ({
|
|
611
492
|
candidateId: entry.candidateId,
|
|
612
493
|
scoreId: entry.scoreId,
|
|
613
494
|
verifierNodeId: entry.verifierNodeId,
|
|
614
495
|
reason: entry.reason,
|
|
615
496
|
evidenceCount: entry.evidence.length
|
|
616
497
|
}))),
|
|
617
|
-
verifierCommitGate: lines((run.commits || []).map((entry) => ({
|
|
498
|
+
verifierCommitGate: (0, normalize_1.lines)((run.commits || []).map((entry) => ({
|
|
618
499
|
verifierGated: Boolean(entry.verifierGated),
|
|
619
500
|
checkpoint: Boolean(entry.checkpoint),
|
|
620
501
|
candidateId: entry.candidateId,
|
|
@@ -631,7 +512,11 @@ function collectCandidateScores(run) {
|
|
|
631
512
|
const scores = [];
|
|
632
513
|
for (const candidate of run.candidates || []) {
|
|
633
514
|
for (const scoreId of candidate.scores || []) {
|
|
634
|
-
|
|
515
|
+
// Canonical nested score path — MUST match the writers (candidate-scoring.ts
|
|
516
|
+
// persistScore, commit.ts): candidates/<candidateId>/scores/<scoreId>.json.
|
|
517
|
+
// The old flat `<id>.<scoreId>.score.json` path was written by nobody, so the
|
|
518
|
+
// candidate_score_parity eval metric silently scored empty placeholders.
|
|
519
|
+
const scorePath = node_path_1.default.join(run.paths.candidatesDir || node_path_1.default.join(run.paths.runDir, "candidates"), (0, state_1.safeFileName)(candidate.id), "scores", `${(0, state_1.safeFileName)(scoreId)}.json`);
|
|
635
520
|
if (node_fs_1.default.existsSync(scorePath)) {
|
|
636
521
|
const score = (0, state_1.readJson)(scorePath);
|
|
637
522
|
scores.push({
|
|
@@ -784,45 +669,6 @@ function loadSuiteFromDir(suiteDir) {
|
|
|
784
669
|
paths: { suiteDir, snapshotPath: node_path_1.default.join(suiteDir, "snapshot.json") }
|
|
785
670
|
};
|
|
786
671
|
}
|
|
787
|
-
function normalizeValue(value) {
|
|
788
|
-
if (Array.isArray(value))
|
|
789
|
-
return value.map(normalizeValue);
|
|
790
|
-
if (!value || typeof value !== "object") {
|
|
791
|
-
if (typeof value === "string")
|
|
792
|
-
return normalizeString(value);
|
|
793
|
-
return value;
|
|
794
|
-
}
|
|
795
|
-
const record = value;
|
|
796
|
-
const normalized = {};
|
|
797
|
-
for (const key of Object.keys(record).sort()) {
|
|
798
|
-
if (["createdAt", "updatedAt", "recordedAt", "selectedAt", "replayedAt", "generatedAt"].includes(key))
|
|
799
|
-
continue;
|
|
800
|
-
if (key.endsWith("Path") || key === "path" || key === "cwd" || key === "runDir" || key.endsWith("Dir")) {
|
|
801
|
-
normalized[key] = normalizeString(String(record[key]));
|
|
802
|
-
}
|
|
803
|
-
else {
|
|
804
|
-
normalized[key] = normalizeValue(record[key]);
|
|
805
|
-
}
|
|
806
|
-
}
|
|
807
|
-
return normalized;
|
|
808
|
-
}
|
|
809
|
-
function normalizeString(value) {
|
|
810
|
-
return value
|
|
811
|
-
.replace(/[0-9]{8}T[0-9]{6}Z/g, "<timestamp>")
|
|
812
|
-
.replace(/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z/g, "<timestamp>")
|
|
813
|
-
.replace(/\/[^"\s]+\/\.cw\/runs\/[^"\s/]+/g, "<run-dir>")
|
|
814
|
-
.replace(/\/[^"\s]+\/\.cw\/evals\/[^"\s/]+/g, "<eval-dir>")
|
|
815
|
-
.replace(/\/var\/folders\/[^"\s]+|\/tmp\/[^"\s]+|\/private\/tmp\/[^"\s]+/g, "<tmp>");
|
|
816
|
-
}
|
|
817
|
-
function lines(value) {
|
|
818
|
-
const normalized = normalizeValue(value);
|
|
819
|
-
if (Array.isArray(normalized))
|
|
820
|
-
return normalized.map((entry) => stableStringify(entry)).sort();
|
|
821
|
-
return [stableStringify(normalized)].sort();
|
|
822
|
-
}
|
|
823
|
-
function stableStringify(value) {
|
|
824
|
-
return JSON.stringify(normalizeValue(value));
|
|
825
|
-
}
|
|
826
672
|
function now() {
|
|
827
673
|
return new Date().toISOString();
|
|
828
674
|
}
|
|
@@ -830,28 +676,3 @@ function metricLine(score, id) {
|
|
|
830
676
|
const metric = score.metrics.find((entry) => entry.id === id);
|
|
831
677
|
return `- ${id}: ${metric?.status || "missing"} - ${metric?.reason || "metric missing"}`;
|
|
832
678
|
}
|
|
833
|
-
function metricStatus(score, id) {
|
|
834
|
-
const metric = score.metrics.find((entry) => entry.id === id);
|
|
835
|
-
return `${id}=${metric?.status || "missing"}`;
|
|
836
|
-
}
|
|
837
|
-
function sectionStatus(comparison, id) {
|
|
838
|
-
return `${id}=${comparison.sections[id]?.status || "missing"}`;
|
|
839
|
-
}
|
|
840
|
-
function isSnapshot(value) {
|
|
841
|
-
return Boolean(value && typeof value === "object" && value.kind === "multi-agent-replay-snapshot");
|
|
842
|
-
}
|
|
843
|
-
function isReplay(value) {
|
|
844
|
-
return Boolean(value && typeof value === "object" && value.kind === "multi-agent-replay-run");
|
|
845
|
-
}
|
|
846
|
-
function isComparison(value) {
|
|
847
|
-
return Boolean(value && typeof value === "object" && "sections" in value && "findings" in value);
|
|
848
|
-
}
|
|
849
|
-
function isScore(value) {
|
|
850
|
-
return Boolean(value && typeof value === "object" && "metrics" in value && "score" in value);
|
|
851
|
-
}
|
|
852
|
-
function isGate(value) {
|
|
853
|
-
return Boolean(value && typeof value === "object" && "verdict" in value && "requiredArtifacts" in value);
|
|
854
|
-
}
|
|
855
|
-
function isReport(value) {
|
|
856
|
-
return Boolean(value && typeof value === "object" && "reportPath" in value && !("verdict" in value));
|
|
857
|
-
}
|