cool-workflow 0.1.98 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +11 -2
- package/apps/architecture-review/app.json +1 -1
- package/apps/architecture-review-fast/app.json +1 -1
- package/apps/end-to-end-golden-path/app.json +1 -1
- package/apps/pr-review-fix-ci/app.json +1 -1
- package/apps/release-cut/app.json +1 -1
- package/apps/research-synthesis/app.json +1 -1
- package/dist/cli/dispatch.js +236 -0
- package/dist/cli/entry.js +120 -0
- package/dist/cli/io.js +21 -4
- package/dist/cli/parseargv.js +157 -0
- package/dist/cli.js +6 -33
- package/dist/core/capability-table.js +3534 -0
- package/dist/core/format/help.js +314 -0
- package/dist/{state-explosion/format.js → core/format/state-explosion-text.js} +10 -1
- package/dist/core/hash.js +137 -0
- package/dist/core/multi-agent/candidate-scoring.js +219 -0
- package/dist/core/multi-agent/collaboration.js +481 -0
- package/dist/core/multi-agent/coordinator.js +515 -0
- package/dist/core/multi-agent/eval-replay.js +306 -0
- package/dist/core/multi-agent/runtime.js +929 -0
- package/dist/core/multi-agent/topology.js +298 -0
- package/dist/core/multi-agent/trust-policy.js +197 -0
- package/dist/core/pipeline/commit-gate.js +320 -0
- package/dist/{pipeline-contract.js → core/pipeline/contract.js} +24 -37
- package/dist/core/pipeline/dispatch.js +103 -0
- package/dist/core/pipeline/drive-decide.js +227 -0
- package/dist/core/pipeline/error-feedback.js +161 -0
- package/dist/core/pipeline/loop-expansion.js +124 -0
- package/dist/{result-normalize.js → core/pipeline/result-normalize.js} +14 -37
- package/dist/core/pipeline/runner.js +230 -0
- package/dist/{contract-migration.js → core/state/contract-migration.js} +68 -92
- package/dist/{state-migrations.js → core/state/migrations.js} +103 -96
- package/dist/core/state/node-projection.js +95 -0
- package/dist/core/state/node-snapshot.js +230 -0
- package/dist/core/state/run-paths.js +93 -0
- package/dist/{schema-validate.js → core/state/schema-validate.js} +35 -28
- package/dist/core/state/schema.js +50 -0
- package/dist/core/state/state-explosion/digest.js +243 -0
- package/dist/core/state/state-explosion/graph.js +527 -0
- package/dist/{state-explosion → core/state/state-explosion}/helpers.js +34 -3
- package/dist/core/state/state-explosion/report.js +187 -0
- package/dist/{state-explosion → core/state/state-explosion}/size.js +25 -7
- package/dist/{state-node.js → core/state/state-node.js} +90 -113
- package/dist/core/state/types.js +19 -0
- package/dist/{validation.js → core/state/validation.js} +64 -131
- package/dist/core/trust/evidence-grounding.js +134 -0
- package/dist/core/trust/ledger.js +199 -0
- package/dist/{telemetry-attestation.js → core/trust/telemetry-attestation.js} +94 -68
- package/dist/core/trust/telemetry-ledger.js +127 -0
- package/dist/core/types.js +20 -0
- package/dist/core/version.js +16 -0
- package/dist/{workflow-app-framework.js → core/workflow-apps/app-schema.js} +337 -426
- package/dist/mcp/dispatch.js +104 -0
- package/dist/mcp/server.js +144 -0
- package/dist/mcp-server.js +6 -84
- package/dist/{agent-config.js → shell/agent-config.js} +118 -71
- package/dist/shell/app-run-cli.js +88 -0
- package/dist/shell/audit-cli.js +259 -0
- package/dist/shell/audit-provenance.js +83 -0
- package/dist/shell/candidate-scoring-io.js +459 -0
- package/dist/shell/collaboration-io.js +264 -0
- package/dist/shell/commit-summary.js +104 -0
- package/dist/shell/commit.js +290 -0
- package/dist/shell/coordinator-io.js +476 -0
- package/dist/shell/demo-cli.js +19 -0
- package/dist/shell/dispatch.js +162 -0
- package/dist/{doctor.js → shell/doctor.js} +123 -56
- package/dist/shell/drive.js +873 -0
- package/dist/shell/error-feedback-io.js +305 -0
- package/dist/shell/eval-io.js +473 -0
- package/dist/{multi-agent-eval/format.js → shell/eval-text.js} +78 -49
- package/dist/{evidence-reasoning.js → shell/evidence-reasoning.js} +124 -255
- package/dist/shell/exec-backend-cli.js +88 -0
- package/dist/shell/execution-backend/agent.js +475 -0
- package/dist/shell/execution-backend/ci.js +15 -0
- package/dist/shell/execution-backend/container.js +69 -0
- package/dist/shell/execution-backend/envelopes.js +55 -0
- package/dist/shell/execution-backend/local.js +113 -0
- package/dist/shell/execution-backend/probes.js +175 -0
- package/dist/shell/execution-backend/registry.js +402 -0
- package/dist/shell/execution-backend/remote.js +128 -0
- package/dist/shell/execution-backend/types.js +11 -0
- package/dist/shell/feedback-cli.js +81 -0
- package/dist/shell/feedback-operations.js +48 -0
- package/dist/shell/fs-atomic.js +276 -0
- package/dist/shell/harness.js +98 -0
- package/dist/shell/ledger-cli.js +212 -0
- package/dist/shell/ledger-io.js +169 -0
- package/dist/shell/man-cli.js +89 -0
- package/dist/shell/metrics-cli.js +98 -0
- package/dist/shell/multi-agent-cli.js +1002 -0
- package/dist/shell/multi-agent-host.js +563 -0
- package/dist/shell/multi-agent-io.js +387 -0
- package/dist/{multi-agent-operator-ux.js → shell/multi-agent-operator-ux.js} +234 -191
- package/dist/shell/node-store.js +124 -0
- package/dist/{observability/format.js → shell/observability-format.js} +7 -1
- package/dist/{observability/intake.js → shell/observability-intake.js} +79 -56
- package/dist/{observability.js → shell/observability.js} +159 -332
- package/dist/{onramp.js → shell/onramp.js} +11 -0
- package/dist/{operator-ux/format.js → shell/operator-ux-text.js} +250 -239
- package/dist/shell/operator-ux.js +431 -0
- package/dist/shell/orchestrator.js +231 -0
- package/dist/shell/pipeline-cli.js +667 -0
- package/dist/shell/pipeline.js +217 -0
- package/dist/shell/reclamation-io.js +1366 -0
- package/dist/shell/registry-cli.js +329 -0
- package/dist/{remote-source.js → shell/remote-source.js} +2 -2
- package/dist/shell/report-cli.js +101 -0
- package/dist/shell/report-view-cli.js +117 -0
- package/dist/{orchestrator → shell}/report.js +289 -282
- package/dist/shell/reporter.js +62 -0
- package/dist/shell/run-export-cli.js +106 -0
- package/dist/shell/run-export.js +680 -0
- package/dist/shell/run-registry-io.js +1014 -0
- package/dist/shell/run-store.js +164 -0
- package/dist/{sandbox-profile.js → shell/sandbox-profile.js} +134 -95
- package/dist/{scheduler.js → shell/scheduler-io.js} +248 -48
- package/dist/shell/scheduling-io.js +311 -0
- package/dist/shell/state-cli.js +181 -0
- package/dist/shell/state-explosion-cli.js +197 -0
- package/dist/shell/telemetry-cli.js +85 -0
- package/dist/{telemetry-demo.js → shell/telemetry-demo.js} +149 -119
- package/dist/shell/telemetry-ledger-io.js +132 -0
- package/dist/{term.js → shell/term.js} +36 -46
- package/dist/shell/topology-io.js +361 -0
- package/dist/shell/trust-audit.js +471 -0
- package/dist/{multi-agent-trust.js → shell/trust-policy-io.js} +67 -205
- package/dist/shell/verifier.js +48 -0
- package/dist/shell/workbench-host.js +250 -0
- package/dist/shell/workbench-text.js +18 -0
- package/dist/shell/workbench.js +175 -0
- package/dist/shell/worker-cli.js +124 -0
- package/dist/shell/worker-isolation.js +852 -0
- package/dist/shell/workflow-app-loader.js +650 -0
- package/docs/agent-delegation-drive.7.md +2 -0
- package/docs/cli-mcp-parity.7.md +280 -219
- package/docs/contract-migration-tooling.7.md +2 -0
- package/docs/control-plane-scheduling.7.md +2 -0
- package/docs/durable-state-and-locking.7.md +2 -0
- package/docs/evidence-adoption-reasoning-chain.7.md +2 -0
- package/docs/execution-backends.7.md +2 -0
- package/docs/multi-agent-cli-mcp-surface.7.md +2 -0
- package/docs/multi-agent-eval-replay-harness.7.md +2 -0
- package/docs/multi-agent-operator-ux.7.md +2 -0
- package/docs/node-snapshot-diff-replay.7.md +2 -0
- package/docs/observability-cost-accounting.7.md +2 -0
- package/docs/project-index.md +132 -71
- package/docs/real-execution-backends.7.md +2 -0
- package/docs/release-and-migration.7.md +2 -0
- package/docs/release-tooling.7.md +2 -0
- package/docs/run-registry-control-plane.7.md +2 -0
- package/docs/run-retention-reclamation.7.md +23 -0
- package/docs/state-explosion-management.7.md +2 -0
- package/docs/team-collaboration.7.md +2 -0
- package/docs/web-desktop-workbench.7.md +2 -0
- package/manifest/plugin.manifest.json +1 -1
- package/manifest/source-context-profiles.json +9 -13
- package/package.json +1 -1
- package/scripts/agents/cw-attest-wrap.js +2 -2
- package/scripts/bump-version.js +4 -3
- package/scripts/canonical-apps.js +4 -4
- package/scripts/dogfood-architecture-review.js +2 -3
- package/scripts/dogfood-release.js +3 -3
- package/scripts/gen-parity-doc.js +15 -2
- package/scripts/golden-path.js +4 -4
- package/scripts/onramp-check.js +1 -1
- package/scripts/parity-check.js +38 -21
- package/scripts/release-flow.js +2 -2
- package/scripts/sync-project-index.js +51 -27
- package/scripts/validate-run-state-schema.js +2 -2
- package/scripts/version-sync-check.js +30 -30
- package/dist/candidate-scoring.js +0 -729
- package/dist/capability-core.js +0 -1189
- package/dist/capability-registry.js +0 -885
- package/dist/cli/command-surface.js +0 -494
- package/dist/cli/format.js +0 -56
- package/dist/cli/handlers/audit.js +0 -82
- package/dist/cli/handlers/blackboard.js +0 -81
- package/dist/cli/handlers/candidate.js +0 -40
- package/dist/cli/handlers/clones.js +0 -34
- package/dist/cli/handlers/collaboration.js +0 -61
- package/dist/cli/handlers/eval.js +0 -40
- package/dist/cli/handlers/ledger.js +0 -169
- package/dist/cli/handlers/maintenance.js +0 -107
- package/dist/cli/handlers/multi-agent.js +0 -165
- package/dist/cli/handlers/node.js +0 -41
- package/dist/cli/handlers/operational.js +0 -155
- package/dist/cli/handlers/operator.js +0 -146
- package/dist/cli/handlers/registry.js +0 -68
- package/dist/cli/handlers/run.js +0 -153
- package/dist/cli/handlers/scheduling.js +0 -132
- package/dist/cli/handlers/workbench.js +0 -41
- package/dist/cli/handlers/worker.js +0 -45
- package/dist/cli/run-summary.js +0 -45
- package/dist/clones.js +0 -162
- package/dist/collaboration.js +0 -726
- package/dist/commit.js +0 -592
- package/dist/compare.js +0 -18
- package/dist/coordinator/classify.js +0 -45
- package/dist/coordinator/paths.js +0 -42
- package/dist/coordinator/util.js +0 -126
- package/dist/coordinator.js +0 -990
- package/dist/daemon.js +0 -44
- package/dist/dispatch.js +0 -250
- package/dist/drive.js +0 -864
- package/dist/error-feedback.js +0 -419
- package/dist/evidence-grounding.js +0 -184
- package/dist/execution-backend/agent.js +0 -354
- package/dist/execution-backend/probes.js +0 -112
- package/dist/execution-backend/util.js +0 -47
- package/dist/execution-backend.js +0 -961
- package/dist/gates.js +0 -48
- package/dist/harness.js +0 -61
- package/dist/ledger.js +0 -313
- package/dist/loop-expansion.js +0 -60
- package/dist/mcp/tool-call.js +0 -470
- package/dist/mcp/tool-definitions.js +0 -1066
- package/dist/mcp-surface.js +0 -30
- package/dist/multi-agent/graph.js +0 -84
- package/dist/multi-agent/helpers.js +0 -141
- package/dist/multi-agent/ids.js +0 -20
- package/dist/multi-agent/paths.js +0 -22
- package/dist/multi-agent-eval/normalize.js +0 -51
- package/dist/multi-agent-eval.js +0 -678
- package/dist/multi-agent-host.js +0 -777
- package/dist/multi-agent.js +0 -984
- package/dist/node-projection.js +0 -59
- package/dist/node-snapshot.js +0 -260
- package/dist/operator-ux.js +0 -631
- package/dist/orchestrator/app-operations.js +0 -211
- package/dist/orchestrator/audit-operations.js +0 -182
- package/dist/orchestrator/candidate-operations.js +0 -117
- package/dist/orchestrator/cli-options.js +0 -294
- package/dist/orchestrator/collaboration-operations.js +0 -86
- package/dist/orchestrator/feedback-operations.js +0 -81
- package/dist/orchestrator/host-operations.js +0 -78
- package/dist/orchestrator/lifecycle-operations.js +0 -650
- package/dist/orchestrator/migration-operations.js +0 -44
- package/dist/orchestrator/multi-agent-operations.js +0 -362
- package/dist/orchestrator/topology-operations.js +0 -84
- package/dist/orchestrator.js +0 -925
- package/dist/pipeline-runner.js +0 -285
- package/dist/reclamation/hash.js +0 -72
- package/dist/reclamation.js +0 -812
- package/dist/reporter.js +0 -67
- package/dist/run-export.js +0 -815
- package/dist/run-registry/derive.js +0 -175
- package/dist/run-registry/format.js +0 -124
- package/dist/run-registry/gc.js +0 -251
- package/dist/run-registry/policy.js +0 -16
- package/dist/run-registry/queue.js +0 -115
- package/dist/run-registry.js +0 -850
- package/dist/run-state-schema.js +0 -68
- package/dist/scheduling.js +0 -184
- package/dist/state-explosion.js +0 -1014
- package/dist/state.js +0 -367
- package/dist/telemetry-ledger.js +0 -196
- package/dist/topology.js +0 -565
- package/dist/triggers.js +0 -184
- package/dist/trust-audit.js +0 -644
- package/dist/types/blackboard.js +0 -2
- package/dist/types/candidate.js +0 -2
- package/dist/types/collaboration.js +0 -2
- package/dist/types/core.js +0 -2
- package/dist/types/drive.js +0 -10
- package/dist/types/error-feedback.js +0 -2
- package/dist/types/evidence-reasoning.js +0 -2
- package/dist/types/execution-backend.js +0 -2
- package/dist/types/multi-agent.js +0 -2
- package/dist/types/observability.js +0 -2
- package/dist/types/pipeline.js +0 -2
- package/dist/types/reclamation.js +0 -8
- package/dist/types/report-bundle.js +0 -6
- package/dist/types/result.js +0 -2
- package/dist/types/run-registry.js +0 -2
- package/dist/types/run.js +0 -2
- package/dist/types/sandbox.js +0 -2
- package/dist/types/schedule.js +0 -2
- package/dist/types/state-node.js +0 -2
- package/dist/types/topology.js +0 -2
- package/dist/types/trust.js +0 -2
- package/dist/types/workbench.js +0 -2
- package/dist/types/worker.js +0 -2
- package/dist/types/workflow-app.js +0 -2
- package/dist/types.js +0 -44
- package/dist/util/fingerprint.js +0 -19
- package/dist/util/fingerprint.test.js +0 -27
- package/dist/verifier.js +0 -78
- package/dist/version.js +0 -8
- package/dist/workbench-host.js +0 -192
- package/dist/workbench.js +0 -192
- package/dist/worker-accept/acceptance.js +0 -114
- package/dist/worker-accept/blackboard-fanout.js +0 -80
- package/dist/worker-accept/blackboard-linkage.js +0 -19
- package/dist/worker-accept/context.js +0 -2
- package/dist/worker-accept/telemetry-ledger.js +0 -126
- package/dist/worker-accept/validation.js +0 -77
- package/dist/worker-accept/verifier-completion.js +0 -73
- package/dist/worker-isolation/helpers.js +0 -51
- package/dist/worker-isolation/paths.js +0 -46
- package/dist/worker-isolation.js +0 -656
- package/dist/workflow-api.js +0 -131
- /package/dist/{types → core/types}/boundary.js +0 -0
|
@@ -1,38 +1,52 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
//
|
|
2
|
+
// shell/observability.ts — DERIVED metrics, ATTESTED cost. `cw metrics
|
|
3
|
+
// show`/`cw metrics summary`.
|
|
3
4
|
//
|
|
4
|
-
//
|
|
5
|
+
// MILESTONE 11 (reporting/observability). Byte-exact port of the old
|
|
6
|
+
// build's src/observability.ts — every number here is a PROJECTION of one
|
|
7
|
+
// run's durable state.json (timestamps -> durations, verifier nodes ->
|
|
8
|
+
// pass rate, candidates -> acceptance rate, failed workers/feedback ->
|
|
9
|
+
// failure rate). No metrics database, no collector daemon. Cost is
|
|
10
|
+
// ATTESTED, never fabricated: absent usage is `unreported`, never 0; a
|
|
11
|
+
// rate over zero samples is `n/a`, never 0%/100%.
|
|
5
12
|
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
13
|
+
// Evidence: SPEC/reporting-ux.md "cw metrics show / summary", invariant 4
|
|
14
|
+
// (metrics are derived and honest), invariant 5 (freshness is fail-
|
|
15
|
+
// closed), invariant 6 (metricsShow never touches state.json); plugins/
|
|
16
|
+
// cool-workflow/src/observability.ts:1-822 (byte-exact source).
|
|
17
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
18
|
+
if (k2 === undefined) k2 = k;
|
|
19
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
20
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
21
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
22
|
+
}
|
|
23
|
+
Object.defineProperty(o, k2, desc);
|
|
24
|
+
}) : (function(o, m, k, k2) {
|
|
25
|
+
if (k2 === undefined) k2 = k;
|
|
26
|
+
o[k2] = m[k];
|
|
27
|
+
}));
|
|
28
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
29
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
30
|
+
}) : function(o, v) {
|
|
31
|
+
o["default"] = v;
|
|
32
|
+
});
|
|
33
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
34
|
+
var ownKeys = function(o) {
|
|
35
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
36
|
+
var ar = [];
|
|
37
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
38
|
+
return ar;
|
|
39
|
+
};
|
|
40
|
+
return ownKeys(o);
|
|
41
|
+
};
|
|
42
|
+
return function (mod) {
|
|
43
|
+
if (mod && mod.__esModule) return mod;
|
|
44
|
+
var result = {};
|
|
45
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
46
|
+
__setModuleDefault(result, mod);
|
|
47
|
+
return result;
|
|
48
|
+
};
|
|
49
|
+
})();
|
|
36
50
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
37
51
|
exports.formatMetricsSummary = exports.formatMetricsReport = exports.parseUsageFromArgs = exports.loadCostPolicy = exports.METRICS_SCHEMA_VERSION = void 0;
|
|
38
52
|
exports.fingerprintMetricsSource = fingerprintMetricsSource;
|
|
@@ -42,40 +56,31 @@ exports.deriveCost = deriveCost;
|
|
|
42
56
|
exports.deriveFailureRate = deriveFailureRate;
|
|
43
57
|
exports.deriveVerifierPassRate = deriveVerifierPassRate;
|
|
44
58
|
exports.deriveCandidateAcceptanceRate = deriveCandidateAcceptanceRate;
|
|
45
|
-
exports.deriveMetricsReport = deriveMetricsReport;
|
|
46
59
|
exports.deriveCollaborationMetrics = deriveCollaborationMetrics;
|
|
60
|
+
exports.deriveMetricsReport = deriveMetricsReport;
|
|
47
61
|
exports.metricsDir = metricsDir;
|
|
48
62
|
exports.loadPersistedMetricsFingerprint = loadPersistedMetricsFingerprint;
|
|
49
63
|
exports.showMetricsReport = showMetricsReport;
|
|
50
64
|
exports.deriveMetricsSummary = deriveMetricsSummary;
|
|
51
|
-
const
|
|
52
|
-
const
|
|
53
|
-
const
|
|
54
|
-
const
|
|
55
|
-
const telemetry_ledger_1 = require("./telemetry-ledger");
|
|
65
|
+
const fs = __importStar(require("node:fs"));
|
|
66
|
+
const path = __importStar(require("node:path"));
|
|
67
|
+
const fs_atomic_1 = require("./fs-atomic");
|
|
68
|
+
const telemetry_ledger_io_1 = require("./telemetry-ledger-io");
|
|
56
69
|
exports.METRICS_SCHEMA_VERSION = 1;
|
|
57
|
-
// Verifier-gate decision classes (derived, never invented).
|
|
58
70
|
const VERIFIER_PASS_STATUSES = new Set(["verified", "completed", "committed"]);
|
|
59
71
|
const VERIFIER_FAIL_STATUSES = new Set(["failed", "rejected", "blocked"]);
|
|
60
|
-
// Candidate acceptance classes.
|
|
61
72
|
const CANDIDATE_ACCEPTED_STATUSES = new Set(["selected", "verified"]);
|
|
62
|
-
// ---------------------------------------------------------------------------
|
|
63
|
-
// Deterministic helpers
|
|
64
|
-
// ---------------------------------------------------------------------------
|
|
65
73
|
function fingerprintStrings(values) {
|
|
66
|
-
const
|
|
74
|
+
const { createHash } = require("node:crypto");
|
|
75
|
+
const hash = createHash("sha256");
|
|
67
76
|
hash.update(JSON.stringify([...values].sort()));
|
|
68
77
|
return `sha256:${hash.digest("hex").slice(0, 32)}`;
|
|
69
78
|
}
|
|
70
|
-
/** Round to a fixed precision deterministically (no locale, no float drift in
|
|
71
|
-
* the serialized form). */
|
|
72
79
|
function round(value, decimals) {
|
|
73
80
|
const factor = 10 ** decimals;
|
|
74
81
|
return Math.round(value * factor) / factor;
|
|
75
82
|
}
|
|
76
|
-
|
|
77
|
-
* the result would be negative (clock skew ⇒ untrustworthy ⇒ null, not a lie). */
|
|
78
|
-
function durationMs(startedAt, endedAt) {
|
|
83
|
+
function durationMsBetween(startedAt, endedAt) {
|
|
79
84
|
if (!startedAt || !endedAt)
|
|
80
85
|
return null;
|
|
81
86
|
const start = Date.parse(startedAt);
|
|
@@ -86,32 +91,21 @@ function durationMs(startedAt, endedAt) {
|
|
|
86
91
|
return delta >= 0 ? delta : null;
|
|
87
92
|
}
|
|
88
93
|
function duration(startedAt, endedAt) {
|
|
89
|
-
|
|
90
|
-
return {
|
|
91
|
-
startedAt,
|
|
92
|
-
endedAt,
|
|
93
|
-
wallClockMs,
|
|
94
|
-
inFlight: !endedAt
|
|
95
|
-
};
|
|
94
|
+
return { startedAt, endedAt, wallClockMs: durationMsBetween(startedAt, endedAt), inFlight: !endedAt };
|
|
96
95
|
}
|
|
97
|
-
/** Build a fail-closed rate. total === 0 ⇒ `n/a` (count/rate null). */
|
|
98
96
|
function rate(metric, count, total, buckets) {
|
|
99
|
-
if (total <= 0)
|
|
97
|
+
if (total <= 0)
|
|
100
98
|
return { metric, state: "n/a", count: null, total: 0, rate: null, buckets };
|
|
101
|
-
}
|
|
102
99
|
return { metric, state: "ok", count, total, rate: round(count / total, 6), buckets };
|
|
103
100
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
101
|
+
function usageKey(usage) {
|
|
102
|
+
if (!usage)
|
|
103
|
+
return "unreported";
|
|
104
|
+
return `${usage.source}:${usage.model || "-"}:${usage.inputTokens ?? "-"}:${usage.outputTokens ?? "-"}`;
|
|
105
|
+
}
|
|
107
106
|
function fingerprintMetricsSource(run) {
|
|
108
|
-
const parts = [
|
|
109
|
-
|
|
110
|
-
`createdAt:${run.createdAt}`,
|
|
111
|
-
`updatedAt:${run.updatedAt}`,
|
|
112
|
-
`app:${run.workflow.app?.id || run.workflow.id}`
|
|
113
|
-
];
|
|
114
|
-
for (const task of [...(run.tasks || [])].sort((a, b) => a.id.localeCompare(b.id))) {
|
|
107
|
+
const parts = [`id:${run.id}`, `createdAt:${run.createdAt}`, `updatedAt:${run.updatedAt}`, `app:${run.workflow.app?.id || run.workflow.id}`];
|
|
108
|
+
for (const task of [...run.tasks].sort((a, b) => a.id.localeCompare(b.id))) {
|
|
115
109
|
parts.push(`task:${task.id}:${task.status}:${task.dispatchedAt || "-"}:${task.completedAt || "-"}:${usageKey(task.usage)}:${task.backendId || "-"}`);
|
|
116
110
|
}
|
|
117
111
|
for (const worker of [...(run.workers || [])].sort((a, b) => a.id.localeCompare(b.id))) {
|
|
@@ -131,40 +125,25 @@ function fingerprintMetricsSource(run) {
|
|
|
131
125
|
}
|
|
132
126
|
return fingerprintStrings(parts);
|
|
133
127
|
}
|
|
134
|
-
function usageKey(usage) {
|
|
135
|
-
if (!usage)
|
|
136
|
-
return "unreported";
|
|
137
|
-
return `${usage.source}:${usage.model || "-"}:${usage.inputTokens ?? "-"}:${usage.outputTokens ?? "-"}`;
|
|
138
|
-
}
|
|
139
|
-
// ---------------------------------------------------------------------------
|
|
140
|
-
// Worker / unit helpers
|
|
141
|
-
// ---------------------------------------------------------------------------
|
|
142
|
-
/** A worker's recorded end timestamp, or undefined when still in-flight. */
|
|
143
128
|
function workerEndAt(worker) {
|
|
144
|
-
|
|
145
|
-
|
|
129
|
+
const output = worker.output;
|
|
130
|
+
if (output?.recordedAt)
|
|
131
|
+
return output.recordedAt;
|
|
146
132
|
if (["completed", "verified", "failed", "rejected"].includes(worker.status))
|
|
147
133
|
return worker.updatedAt;
|
|
148
134
|
return undefined;
|
|
149
135
|
}
|
|
150
|
-
// ---------------------------------------------------------------------------
|
|
151
|
-
// Usage + cost
|
|
152
|
-
// ---------------------------------------------------------------------------
|
|
153
|
-
/** The work units that COULD carry attested usage: every worker that produced
|
|
154
|
-
* output, plus every completed task NOT already represented by such a worker (a
|
|
155
|
-
* worker-output-backed task's usage rides on the worker, so it is never
|
|
156
|
-
* double-counted; a task completed directly via `cw result` is its own unit
|
|
157
|
-
* even if a worker was allocated but never recorded output). */
|
|
158
136
|
function usageUnits(run) {
|
|
159
137
|
const units = [];
|
|
160
138
|
const outputTaskIds = new Set();
|
|
161
139
|
for (const worker of run.workers || []) {
|
|
162
140
|
if (worker.output) {
|
|
163
|
-
|
|
141
|
+
const output = worker.output;
|
|
142
|
+
outputTaskIds.add(output.taskId || worker.taskId);
|
|
164
143
|
units.push({ unit: worker.id, kind: "worker", usage: worker.usage });
|
|
165
144
|
}
|
|
166
145
|
}
|
|
167
|
-
for (const task of run.tasks
|
|
146
|
+
for (const task of run.tasks) {
|
|
168
147
|
if (task.status === "completed" && !outputTaskIds.has(task.id)) {
|
|
169
148
|
units.push({ unit: task.id, kind: "task", usage: task.usage });
|
|
170
149
|
}
|
|
@@ -204,16 +183,10 @@ function deriveUsageTotals(run) {
|
|
|
204
183
|
inputTokens,
|
|
205
184
|
outputTokens,
|
|
206
185
|
totalTokens,
|
|
207
|
-
models: [...models].sort()
|
|
186
|
+
models: [...models].sort(),
|
|
208
187
|
};
|
|
209
188
|
return { totals, rows };
|
|
210
189
|
}
|
|
211
|
-
/** Track 1 cryptographic attestation coverage over a run's work units, plus the
|
|
212
|
-
* tamper-evident ledger state. DIFFERENT axis from UsageTotals.coverage: that
|
|
213
|
-
* counts units that merely carry a usage record; here `attested` counts units
|
|
214
|
-
* whose reported usage VERIFIED against the operator trust key. Deterministic —
|
|
215
|
-
* reads recorded `usage.attestation` and recomputes the ledger chain; no
|
|
216
|
-
* now-derived field, so it stays byte-stable for CLI<->MCP parity. */
|
|
217
190
|
function deriveAttestationCoverage(run) {
|
|
218
191
|
const units = usageUnits(run);
|
|
219
192
|
let attested = 0;
|
|
@@ -234,12 +207,11 @@ function deriveAttestationCoverage(run) {
|
|
|
234
207
|
absent++;
|
|
235
208
|
break;
|
|
236
209
|
default:
|
|
237
|
-
// A usage record with no verdict (operator-recorded / legacy intake).
|
|
238
210
|
unverified++;
|
|
239
211
|
}
|
|
240
212
|
}
|
|
241
213
|
const unitCount = units.length;
|
|
242
|
-
const ledger = (0,
|
|
214
|
+
const ledger = (0, telemetry_ledger_io_1.verifyTelemetryLedger)(run);
|
|
243
215
|
return {
|
|
244
216
|
units: unitCount,
|
|
245
217
|
attested,
|
|
@@ -247,36 +219,23 @@ function deriveAttestationCoverage(run) {
|
|
|
247
219
|
absent,
|
|
248
220
|
unverified,
|
|
249
221
|
verifiedCoverage: unitCount > 0 ? round(attested / unitCount, 6) : null,
|
|
250
|
-
ledger: { present: ledger.present, verified: ledger.verified, records: ledger.records.length }
|
|
222
|
+
ledger: { present: ledger.present, verified: ledger.verified, records: ledger.records.length },
|
|
251
223
|
};
|
|
252
224
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
225
|
+
function priceUsage(usage, price) {
|
|
226
|
+
const input = ((usage.inputTokens || 0) / 1_000_000) * price.inputPerMillion;
|
|
227
|
+
const output = ((usage.outputTokens || 0) / 1_000_000) * price.outputPerMillion;
|
|
228
|
+
const cacheRead = ((usage.cacheReadTokens || 0) / 1_000_000) * (price.cacheReadPerMillion || 0);
|
|
229
|
+
const cacheWrite = ((usage.cacheWriteTokens || 0) / 1_000_000) * (price.cacheWritePerMillion || 0);
|
|
230
|
+
return input + output + cacheRead + cacheWrite;
|
|
231
|
+
}
|
|
256
232
|
function deriveCost(rows, policy) {
|
|
257
233
|
const currency = policy?.currency || "USD";
|
|
258
234
|
if (rows.length === 0) {
|
|
259
|
-
return {
|
|
260
|
-
state: "unreported",
|
|
261
|
-
currency,
|
|
262
|
-
attestedUsd: null,
|
|
263
|
-
estimatedUsd: null,
|
|
264
|
-
policyId: policy?.id,
|
|
265
|
-
unpricedModels: [],
|
|
266
|
-
pricedCoverage: null,
|
|
267
|
-
notes: ["No attested usage on this run; cost is unreported, not zero."]
|
|
268
|
-
};
|
|
235
|
+
return { state: "unreported", currency, attestedUsd: null, estimatedUsd: null, policyId: policy?.id, unpricedModels: [], pricedCoverage: null, notes: ["No attested usage on this run; cost is unreported, not zero."] };
|
|
269
236
|
}
|
|
270
237
|
if (!policy) {
|
|
271
|
-
return {
|
|
272
|
-
state: "unpriced",
|
|
273
|
-
currency,
|
|
274
|
-
attestedUsd: null,
|
|
275
|
-
estimatedUsd: null,
|
|
276
|
-
unpricedModels: [],
|
|
277
|
-
pricedCoverage: null,
|
|
278
|
-
notes: ["Attested usage present but no pricing policy supplied; pass --pricing <path> to price it."]
|
|
279
|
-
};
|
|
238
|
+
return { state: "unpriced", currency, attestedUsd: null, estimatedUsd: null, unpricedModels: [], pricedCoverage: null, notes: ["Attested usage present but no pricing policy supplied; pass --pricing <path> to price it."] };
|
|
280
239
|
}
|
|
281
240
|
const byModel = new Map();
|
|
282
241
|
for (const m of policy.models || [])
|
|
@@ -288,19 +247,18 @@ function deriveCost(rows, policy) {
|
|
|
288
247
|
let usedDefault = false;
|
|
289
248
|
let usedExact = false;
|
|
290
249
|
const unpriced = new Set();
|
|
291
|
-
for (const
|
|
292
|
-
const
|
|
293
|
-
const tokens = tokenTotal(usage);
|
|
250
|
+
for (const rowUsage of rows.map((r) => r.usage)) {
|
|
251
|
+
const tokens = tokenTotal(rowUsage);
|
|
294
252
|
attestedTokens += tokens;
|
|
295
|
-
const model =
|
|
253
|
+
const model = rowUsage.model;
|
|
296
254
|
const exact = model ? byModel.get(model) : undefined;
|
|
297
255
|
const price = exact || policy.defaultPrice;
|
|
298
256
|
if (!price) {
|
|
299
257
|
if (model)
|
|
300
258
|
unpriced.add(model);
|
|
301
|
-
continue;
|
|
259
|
+
continue;
|
|
302
260
|
}
|
|
303
|
-
const cost = priceUsage(
|
|
261
|
+
const cost = priceUsage(rowUsage, price);
|
|
304
262
|
pricedTokens += tokens;
|
|
305
263
|
if (exact) {
|
|
306
264
|
attestedUsd += cost;
|
|
@@ -335,26 +293,14 @@ function deriveCost(rows, policy) {
|
|
|
335
293
|
policyId: policy.id,
|
|
336
294
|
unpricedModels: [...unpriced].sort(),
|
|
337
295
|
pricedCoverage: attestedTokens > 0 ? round(pricedTokens / attestedTokens, 6) : null,
|
|
338
|
-
notes
|
|
296
|
+
notes,
|
|
339
297
|
};
|
|
340
298
|
}
|
|
341
|
-
function priceUsage(usage, price) {
|
|
342
|
-
const input = ((usage.inputTokens || 0) / 1_000_000) * price.inputPerMillion;
|
|
343
|
-
const output = ((usage.outputTokens || 0) / 1_000_000) * price.outputPerMillion;
|
|
344
|
-
const cacheRead = ((usage.cacheReadTokens || 0) / 1_000_000) * (price.cacheReadPerMillion || 0);
|
|
345
|
-
const cacheWrite = ((usage.cacheWriteTokens || 0) / 1_000_000) * (price.cacheWritePerMillion || 0);
|
|
346
|
-
return input + output + cacheRead + cacheWrite;
|
|
347
|
-
}
|
|
348
|
-
// ---------------------------------------------------------------------------
|
|
349
|
-
// Rates (each a fail-closed RateMetric with explicit sample counts)
|
|
350
|
-
// ---------------------------------------------------------------------------
|
|
351
299
|
function deriveFailureRate(run) {
|
|
352
300
|
const workers = run.workers || [];
|
|
353
301
|
const memberships = run.multiAgent?.memberships || [];
|
|
354
302
|
const feedback = run.feedback || [];
|
|
355
|
-
|
|
356
|
-
// worker), counted only when dispatched (an attempt actually happened).
|
|
357
|
-
const tasks = (run.tasks || []).filter((t) => !t.workerId && t.dispatchedAt);
|
|
303
|
+
const tasks = run.tasks.filter((t) => !t.workerId && t.dispatchedAt);
|
|
358
304
|
const workersFailed = workers.filter((w) => w.status === "failed" || w.status === "rejected").length;
|
|
359
305
|
const tasksFailed = tasks.filter((t) => t.status === "failed").length;
|
|
360
306
|
const membershipsFailed = memberships.filter((m) => m.status === "failed").length;
|
|
@@ -369,7 +315,7 @@ function deriveFailureRate(run) {
|
|
|
369
315
|
memberships: memberships.length,
|
|
370
316
|
membershipsFailed,
|
|
371
317
|
feedback: feedback.length,
|
|
372
|
-
feedbackUnresolved
|
|
318
|
+
feedbackUnresolved,
|
|
373
319
|
});
|
|
374
320
|
}
|
|
375
321
|
function deriveVerifierPassRate(run) {
|
|
@@ -386,7 +332,6 @@ function deriveVerifierPassRate(run) {
|
|
|
386
332
|
else if (VERIFIER_FAIL_STATUSES.has(gate.status)) {
|
|
387
333
|
decided++;
|
|
388
334
|
}
|
|
389
|
-
// pending/running gates are undecided ⇒ excluded from the denominator.
|
|
390
335
|
}
|
|
391
336
|
return rate("verifier-pass", passed, decided, buckets);
|
|
392
337
|
}
|
|
@@ -401,41 +346,56 @@ function deriveCandidateAcceptanceRate(run) {
|
|
|
401
346
|
}
|
|
402
347
|
return rate("candidate-acceptance", accepted, candidates.length, buckets);
|
|
403
348
|
}
|
|
404
|
-
// ---------------------------------------------------------------------------
|
|
405
|
-
// Time / duration
|
|
406
|
-
// ---------------------------------------------------------------------------
|
|
407
349
|
function taskRows(run) {
|
|
408
|
-
return
|
|
409
|
-
.map((task) => ({
|
|
410
|
-
id: task.id,
|
|
411
|
-
kind: "task",
|
|
412
|
-
status: task.status,
|
|
413
|
-
duration: duration(task.dispatchedAt, task.completedAt)
|
|
414
|
-
}))
|
|
350
|
+
return run.tasks
|
|
351
|
+
.map((task) => ({ id: task.id, kind: "task", status: task.status, duration: duration(task.dispatchedAt, task.completedAt) }))
|
|
415
352
|
.sort((a, b) => a.id.localeCompare(b.id));
|
|
416
353
|
}
|
|
417
354
|
function workerRows(run) {
|
|
418
355
|
return (run.workers || [])
|
|
419
|
-
.map((worker) => ({
|
|
420
|
-
id: worker.id,
|
|
421
|
-
kind: "worker",
|
|
422
|
-
status: worker.status,
|
|
423
|
-
duration: duration(worker.createdAt, workerEndAt(worker))
|
|
424
|
-
}))
|
|
356
|
+
.map((worker) => ({ id: worker.id, kind: "worker", status: worker.status, duration: duration(worker.createdAt, workerEndAt(worker)) }))
|
|
425
357
|
.sort((a, b) => a.id.localeCompare(b.id));
|
|
426
358
|
}
|
|
359
|
+
function targetCreatedAt(run, target) {
|
|
360
|
+
if (target.kind === "candidate")
|
|
361
|
+
return (run.candidates || []).find((e) => e.id === target.id)?.createdAt;
|
|
362
|
+
if (target.kind === "commit")
|
|
363
|
+
return (run.commits || []).find((e) => e.id === target.id)?.createdAt;
|
|
364
|
+
if (target.kind === "selection")
|
|
365
|
+
return (run.candidateSelections || []).find((e) => e.id === target.id)?.selectedAt;
|
|
366
|
+
return undefined;
|
|
367
|
+
}
|
|
368
|
+
function deriveCollaborationMetrics(run) {
|
|
369
|
+
const collab = run.collaboration;
|
|
370
|
+
const approvalRecords = (collab?.approvals || []).filter((r) => r.decision === "approve");
|
|
371
|
+
const rejectionRecords = (collab?.approvals || []).filter((r) => r.decision === "reject");
|
|
372
|
+
const reviewers = new Set(approvalRecords.map((r) => r.actor?.id).filter((id) => id && id !== "unattributed"));
|
|
373
|
+
const samples = [];
|
|
374
|
+
for (const record of approvalRecords) {
|
|
375
|
+
const createdAt = targetCreatedAt(run, record.target);
|
|
376
|
+
const ms = durationMsBetween(createdAt, record.createdAt);
|
|
377
|
+
if (ms !== null)
|
|
378
|
+
samples.push(ms);
|
|
379
|
+
}
|
|
380
|
+
const meanMs = samples.length ? Math.round(samples.reduce((acc, ms) => acc + ms, 0) / samples.length) : null;
|
|
381
|
+
const maxMs = samples.length ? Math.max(...samples) : null;
|
|
382
|
+
return {
|
|
383
|
+
approvals: approvalRecords.length,
|
|
384
|
+
rejections: rejectionRecords.length,
|
|
385
|
+
comments: (collab?.comments || []).length,
|
|
386
|
+
handoffs: (collab?.handoffs || []).length,
|
|
387
|
+
reviewers: reviewers.size,
|
|
388
|
+
approvalRate: rate("approval", approvalRecords.length, approvalRecords.length + rejectionRecords.length, { approve: approvalRecords.length, reject: rejectionRecords.length }),
|
|
389
|
+
timeToApproval: { samples: samples.length, meanMs, maxMs },
|
|
390
|
+
};
|
|
391
|
+
}
|
|
427
392
|
function deriveMetricsReport(run, options) {
|
|
428
393
|
const tasks = taskRows(run);
|
|
429
394
|
const workers = workerRows(run);
|
|
430
395
|
const activeTaskMs = tasks.reduce((acc, row) => acc + (row.duration.wallClockMs || 0), 0);
|
|
431
396
|
const inFlight = tasks.filter((t) => t.duration.inFlight).length + workers.filter((w) => w.duration.inFlight).length;
|
|
432
|
-
const pendingOrRunning =
|
|
433
|
-
const runDuration = {
|
|
434
|
-
startedAt: run.createdAt,
|
|
435
|
-
endedAt: run.updatedAt,
|
|
436
|
-
wallClockMs: durationMs(run.createdAt, run.updatedAt),
|
|
437
|
-
inFlight: pendingOrRunning > 0
|
|
438
|
-
};
|
|
397
|
+
const pendingOrRunning = run.tasks.filter((t) => t.status === "pending" || t.status === "running").length;
|
|
398
|
+
const runDuration = { startedAt: run.createdAt, endedAt: run.updatedAt, wallClockMs: durationMsBetween(run.createdAt, run.updatedAt), inFlight: pendingOrRunning > 0 };
|
|
439
399
|
const { totals, rows } = deriveUsageTotals(run);
|
|
440
400
|
const cost = deriveCost(rows, options.policy);
|
|
441
401
|
const currentFingerprint = fingerprintMetricsSource(run);
|
|
@@ -447,39 +407,22 @@ function deriveMetricsReport(run, options) {
|
|
|
447
407
|
else
|
|
448
408
|
status = "stale";
|
|
449
409
|
const backendIds = new Set();
|
|
450
|
-
for (const task of run.tasks
|
|
410
|
+
for (const task of run.tasks)
|
|
451
411
|
if (task.backendId)
|
|
452
412
|
backendIds.add(task.backendId);
|
|
453
413
|
for (const worker of run.workers || [])
|
|
454
414
|
if (worker.backendId)
|
|
455
415
|
backendIds.add(worker.backendId);
|
|
456
|
-
|
|
416
|
+
return {
|
|
457
417
|
schemaVersion: exports.METRICS_SCHEMA_VERSION,
|
|
458
418
|
surface: "metrics",
|
|
459
419
|
runId: run.id,
|
|
460
420
|
generatedAt: options.now,
|
|
461
421
|
sourceFingerprint: currentFingerprint,
|
|
462
|
-
freshness: {
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
},
|
|
467
|
-
scope: {
|
|
468
|
-
app: run.workflow.app?.id || run.workflow.id,
|
|
469
|
-
backendIds: [...backendIds].sort()
|
|
470
|
-
},
|
|
471
|
-
time: {
|
|
472
|
-
run: runDuration,
|
|
473
|
-
activeTaskMs,
|
|
474
|
-
inFlight,
|
|
475
|
-
tasks,
|
|
476
|
-
workers
|
|
477
|
-
},
|
|
478
|
-
rates: {
|
|
479
|
-
failure: deriveFailureRate(run),
|
|
480
|
-
verifierPass: deriveVerifierPassRate(run),
|
|
481
|
-
candidateAcceptance: deriveCandidateAcceptanceRate(run)
|
|
482
|
-
},
|
|
422
|
+
freshness: { status, persistedFingerprint: options.persistedFingerprint, currentFingerprint },
|
|
423
|
+
scope: { app: run.workflow.app?.id || run.workflow.id, backendIds: [...backendIds].sort() },
|
|
424
|
+
time: { run: runDuration, activeTaskMs, inFlight, tasks, workers },
|
|
425
|
+
rates: { failure: deriveFailureRate(run), verifierPass: deriveVerifierPassRate(run), candidateAcceptance: deriveCandidateAcceptanceRate(run) },
|
|
483
426
|
usage: totals,
|
|
484
427
|
cost,
|
|
485
428
|
attestedUsage: rows,
|
|
@@ -487,94 +430,37 @@ function deriveMetricsReport(run, options) {
|
|
|
487
430
|
collaboration: deriveCollaborationMetrics(run),
|
|
488
431
|
nextAction: totals.unreportedUnits > 0 && totals.attestedUnits === 0
|
|
489
432
|
? "No attested usage yet — record host usage on result/worker intake (cw result ... --usage-input-tokens N --usage-output-tokens M --usage-model ID)."
|
|
490
|
-
: `node scripts/cw.js metrics show ${run.id} --json
|
|
433
|
+
: `node scripts/cw.js metrics show ${run.id} --json`,
|
|
491
434
|
};
|
|
492
|
-
return report;
|
|
493
435
|
}
|
|
494
|
-
/** v0.1.32 collaboration metrics — counts, approval rate, and time-to-approval,
|
|
495
|
-
* all from append-only records + recorded timestamps. The ONLY now-derived
|
|
496
|
-
* field anywhere in the report remains `generatedAt`; these are byte-stable. */
|
|
497
|
-
function deriveCollaborationMetrics(run) {
|
|
498
|
-
const collab = run.collaboration;
|
|
499
|
-
const approvalRecords = (collab?.approvals || []).filter((record) => record.decision === "approve");
|
|
500
|
-
const rejectionRecords = (collab?.approvals || []).filter((record) => record.decision === "reject");
|
|
501
|
-
const reviewers = new Set(approvalRecords.map((record) => record.actor?.id).filter((id) => id && id !== "unattributed"));
|
|
502
|
-
const samples = [];
|
|
503
|
-
for (const record of approvalRecords) {
|
|
504
|
-
const createdAt = targetCreatedAt(run, record.target);
|
|
505
|
-
const ms = durationMs(createdAt, record.createdAt);
|
|
506
|
-
if (ms !== null)
|
|
507
|
-
samples.push(ms);
|
|
508
|
-
}
|
|
509
|
-
const meanMs = samples.length ? Math.round(samples.reduce((acc, ms) => acc + ms, 0) / samples.length) : null;
|
|
510
|
-
const maxMs = samples.length ? Math.max(...samples) : null;
|
|
511
|
-
return {
|
|
512
|
-
approvals: approvalRecords.length,
|
|
513
|
-
rejections: rejectionRecords.length,
|
|
514
|
-
comments: (collab?.comments || []).length,
|
|
515
|
-
handoffs: (collab?.handoffs || []).length,
|
|
516
|
-
reviewers: reviewers.size,
|
|
517
|
-
approvalRate: rate("approval", approvalRecords.length, approvalRecords.length + rejectionRecords.length, {
|
|
518
|
-
approve: approvalRecords.length,
|
|
519
|
-
reject: rejectionRecords.length
|
|
520
|
-
}),
|
|
521
|
-
timeToApproval: { samples: samples.length, meanMs, maxMs }
|
|
522
|
-
};
|
|
523
|
-
}
|
|
524
|
-
/** Recorded creation timestamp of an approval's target (for time-to-approval). */
|
|
525
|
-
function targetCreatedAt(run, target) {
|
|
526
|
-
if (target.kind === "candidate")
|
|
527
|
-
return (run.candidates || []).find((entry) => entry.id === target.id)?.createdAt;
|
|
528
|
-
if (target.kind === "commit")
|
|
529
|
-
return (run.commits || []).find((entry) => entry.id === target.id)?.createdAt;
|
|
530
|
-
if (target.kind === "selection")
|
|
531
|
-
return (run.candidateSelections || []).find((entry) => entry.id === target.id)?.selectedAt;
|
|
532
|
-
return undefined;
|
|
533
|
-
}
|
|
534
|
-
// ---------------------------------------------------------------------------
|
|
535
|
-
// Persistence — a rebuildable, fingerprinted snapshot (fail-closed freshness).
|
|
536
|
-
// ---------------------------------------------------------------------------
|
|
537
436
|
function metricsDir(run) {
|
|
538
|
-
return
|
|
437
|
+
return path.join(run.paths.runDir, "metrics");
|
|
539
438
|
}
|
|
540
439
|
function metricsReportPath(run) {
|
|
541
|
-
return
|
|
440
|
+
return path.join(metricsDir(run), "metrics-report.json");
|
|
542
441
|
}
|
|
543
|
-
/** Read the persisted source fingerprint for this run, if any (never throws). */
|
|
544
442
|
function loadPersistedMetricsFingerprint(run) {
|
|
545
443
|
const file = metricsReportPath(run);
|
|
546
|
-
if (!
|
|
444
|
+
if (!fs.existsSync(file))
|
|
547
445
|
return undefined;
|
|
548
446
|
try {
|
|
549
|
-
|
|
550
|
-
return parsed.sourceFingerprint;
|
|
447
|
+
return (0, fs_atomic_1.readJson)(file).sourceFingerprint;
|
|
551
448
|
}
|
|
552
449
|
catch {
|
|
553
450
|
return undefined;
|
|
554
451
|
}
|
|
555
452
|
}
|
|
556
|
-
/** Derive + persist the per-run report. The RETURNED payload is
|
|
557
|
-
*
|
|
558
|
-
*
|
|
559
|
-
*
|
|
453
|
+
/** Derive + persist the per-run report. The RETURNED payload is pinned to
|
|
454
|
+
* `freshness: "valid"` (persistedFingerprint === itself) so `cw metrics
|
|
455
|
+
* show --json` and the MCP tool are byte-identical; NEVER touches
|
|
456
|
+
* state.json (invariant 6). */
|
|
560
457
|
function showMetricsReport(run, options) {
|
|
561
458
|
const live = deriveMetricsReport(run, { now: options.now, policy: options.policy });
|
|
562
|
-
const report = {
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
status: "valid",
|
|
566
|
-
persistedFingerprint: live.sourceFingerprint,
|
|
567
|
-
currentFingerprint: live.sourceFingerprint
|
|
568
|
-
}
|
|
569
|
-
};
|
|
570
|
-
node_fs_1.default.mkdirSync(metricsDir(run), { recursive: true });
|
|
571
|
-
(0, state_1.writeJson)(metricsReportPath(run), report);
|
|
459
|
+
const report = { ...live, freshness: { status: "valid", persistedFingerprint: live.sourceFingerprint, currentFingerprint: live.sourceFingerprint } };
|
|
460
|
+
fs.mkdirSync(metricsDir(run), { recursive: true });
|
|
461
|
+
(0, fs_atomic_1.writeJson)(metricsReportPath(run), report);
|
|
572
462
|
return report;
|
|
573
463
|
}
|
|
574
|
-
// ---------------------------------------------------------------------------
|
|
575
|
-
// Cross-repo rollup (pool samples; sum attested usage/cost with coverage).
|
|
576
|
-
// ---------------------------------------------------------------------------
|
|
577
|
-
/** Pool a list of RateMetrics into one (insufficient-data when no samples). */
|
|
578
464
|
function poolRates(metric, rates) {
|
|
579
465
|
let count = 0;
|
|
580
466
|
let total = 0;
|
|
@@ -587,7 +473,6 @@ function poolRates(metric, rates) {
|
|
|
587
473
|
}
|
|
588
474
|
return rate(metric, count, total, buckets);
|
|
589
475
|
}
|
|
590
|
-
/** Sum a list of UsageTotals (coverage recomputed over pooled units). */
|
|
591
476
|
function poolUsage(list) {
|
|
592
477
|
let units = 0;
|
|
593
478
|
let attestedUnits = 0;
|
|
@@ -604,19 +489,8 @@ function poolUsage(list) {
|
|
|
604
489
|
for (const m of u.models)
|
|
605
490
|
models.add(m);
|
|
606
491
|
}
|
|
607
|
-
return {
|
|
608
|
-
units,
|
|
609
|
-
attestedUnits,
|
|
610
|
-
unreportedUnits: units - attestedUnits,
|
|
611
|
-
coverage: units > 0 ? round(attestedUnits / units, 6) : null,
|
|
612
|
-
inputTokens,
|
|
613
|
-
outputTokens,
|
|
614
|
-
totalTokens,
|
|
615
|
-
models: [...models].sort()
|
|
616
|
-
};
|
|
492
|
+
return { units, attestedUnits, unreportedUnits: units - attestedUnits, coverage: units > 0 ? round(attestedUnits / units, 6) : null, inputTokens, outputTokens, totalTokens, models: [...models].sort() };
|
|
617
493
|
}
|
|
618
|
-
/** Sum a list of CostMetrics (attested + estimated kept separate; states merge
|
|
619
|
-
* conservatively so attested+estimated never collapses into one figure). */
|
|
620
494
|
function poolCost(list) {
|
|
621
495
|
const currency = list.find((c) => c.currency)?.currency || "USD";
|
|
622
496
|
let attestedUsd = null;
|
|
@@ -659,41 +533,13 @@ function poolCost(list) {
|
|
|
659
533
|
const notes = [];
|
|
660
534
|
if (anyAttested && anyEstimated)
|
|
661
535
|
notes.push("Totals mix exact-priced (attested) and default-priced (estimated) runs; the two USD figures are kept separate.");
|
|
662
|
-
return {
|
|
663
|
-
state,
|
|
664
|
-
currency,
|
|
665
|
-
attestedUsd,
|
|
666
|
-
estimatedUsd,
|
|
667
|
-
policyId,
|
|
668
|
-
unpricedModels: [...unpriced].sort(),
|
|
669
|
-
pricedCoverage: null,
|
|
670
|
-
notes
|
|
671
|
-
};
|
|
536
|
+
return { state, currency, attestedUsd, estimatedUsd, policyId, unpricedModels: [...unpriced].sort(), pricedCoverage: null, notes };
|
|
672
537
|
}
|
|
673
|
-
/** Build the cross-repo rollup from already-loaded runs. PURE over its inputs +
|
|
674
|
-
* injected `now`. `unreadableRuns` counts runs whose source could not be loaded
|
|
675
|
-
* (the caller passes the count); they are surfaced, never silently dropped. */
|
|
676
538
|
function deriveMetricsSummary(inputs, options) {
|
|
677
539
|
const perRun = [];
|
|
678
540
|
for (const input of inputs) {
|
|
679
|
-
const report = deriveMetricsReport(input.run, {
|
|
680
|
-
|
|
681
|
-
policy: options.policy,
|
|
682
|
-
persistedFingerprint: input.persistedFingerprint
|
|
683
|
-
});
|
|
684
|
-
perRun.push({
|
|
685
|
-
report,
|
|
686
|
-
ref: {
|
|
687
|
-
runId: report.runId,
|
|
688
|
-
repo: input.repo,
|
|
689
|
-
app: report.scope.app,
|
|
690
|
-
backendIds: report.scope.backendIds,
|
|
691
|
-
freshness: report.freshness.status,
|
|
692
|
-
rates: report.rates,
|
|
693
|
-
usage: report.usage,
|
|
694
|
-
cost: report.cost
|
|
695
|
-
}
|
|
696
|
-
});
|
|
541
|
+
const report = deriveMetricsReport(input.run, { now: options.now, policy: options.policy, persistedFingerprint: input.persistedFingerprint });
|
|
542
|
+
perRun.push({ report, ref: { runId: report.runId, repo: input.repo, app: report.scope.app, backendIds: report.scope.backendIds, freshness: report.freshness.status, rates: report.rates, usage: report.usage, cost: report.cost } });
|
|
697
543
|
}
|
|
698
544
|
perRun.sort((a, b) => a.report.runId.localeCompare(b.report.runId));
|
|
699
545
|
const groupBy = (keyOf) => {
|
|
@@ -710,13 +556,9 @@ function deriveMetricsSummary(inputs, options) {
|
|
|
710
556
|
.map(([key, reports]) => ({
|
|
711
557
|
key,
|
|
712
558
|
runCount: reports.length,
|
|
713
|
-
rates: {
|
|
714
|
-
failure: poolRates("failure", reports.map((r) => r.rates.failure)),
|
|
715
|
-
verifierPass: poolRates("verifier-pass", reports.map((r) => r.rates.verifierPass)),
|
|
716
|
-
candidateAcceptance: poolRates("candidate-acceptance", reports.map((r) => r.rates.candidateAcceptance))
|
|
717
|
-
},
|
|
559
|
+
rates: { failure: poolRates("failure", reports.map((r) => r.rates.failure)), verifierPass: poolRates("verifier-pass", reports.map((r) => r.rates.verifierPass)), candidateAcceptance: poolRates("candidate-acceptance", reports.map((r) => r.rates.candidateAcceptance)) },
|
|
718
560
|
usage: poolUsage(reports.map((r) => r.usage)),
|
|
719
|
-
cost: poolCost(reports.map((r) => r.cost))
|
|
561
|
+
cost: poolCost(reports.map((r) => r.cost)),
|
|
720
562
|
}));
|
|
721
563
|
};
|
|
722
564
|
const allReports = perRun.map((p) => p.report);
|
|
@@ -742,11 +584,7 @@ function deriveMetricsSummary(inputs, options) {
|
|
|
742
584
|
generatedAt: options.now,
|
|
743
585
|
runCount: perRun.length,
|
|
744
586
|
unreadableRuns: options.unreadableRuns || 0,
|
|
745
|
-
rates: {
|
|
746
|
-
failure: poolRates("failure", allReports.map((r) => r.rates.failure)),
|
|
747
|
-
verifierPass: poolRates("verifier-pass", allReports.map((r) => r.rates.verifierPass)),
|
|
748
|
-
candidateAcceptance: poolRates("candidate-acceptance", allReports.map((r) => r.rates.candidateAcceptance))
|
|
749
|
-
},
|
|
587
|
+
rates: { failure: poolRates("failure", allReports.map((r) => r.rates.failure)), verifierPass: poolRates("verifier-pass", allReports.map((r) => r.rates.verifierPass)), candidateAcceptance: poolRates("candidate-acceptance", allReports.map((r) => r.rates.candidateAcceptance)) },
|
|
750
588
|
usage: poolUsage(allReports.map((r) => r.usage)),
|
|
751
589
|
cost: poolCost(allReports.map((r) => r.cost)),
|
|
752
590
|
totalOutputBytes,
|
|
@@ -754,23 +592,12 @@ function deriveMetricsSummary(inputs, options) {
|
|
|
754
592
|
byApp: groupBy((r) => [r.scope.app || "unknown"]),
|
|
755
593
|
byBackend: groupBy((r) => (r.scope.backendIds.length ? r.scope.backendIds : ["unreported"])),
|
|
756
594
|
runs: perRun.map((p) => p.ref),
|
|
757
|
-
nextAction: perRun.length === 0 ? "No indexed runs; run a workflow, then `cw metrics summary`." : "Per-run detail: cw metrics show <run-id> --json"
|
|
595
|
+
nextAction: perRun.length === 0 ? "No indexed runs; run a workflow, then `cw metrics summary`." : "Per-run detail: cw metrics show <run-id> --json",
|
|
758
596
|
};
|
|
759
597
|
}
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
Object.defineProperty(exports, "loadCostPolicy", { enumerable: true, get: function () { return intake_1.loadCostPolicy; } });
|
|
767
|
-
Object.defineProperty(exports, "parseUsageFromArgs", { enumerable: true, get: function () { return intake_1.parseUsageFromArgs; } });
|
|
768
|
-
// ---------------------------------------------------------------------------
|
|
769
|
-
// Human formatters (CLI default text; --json emits the canonical payload) now
|
|
770
|
-
// live in ./observability/format.ts (god-module carve, mirrors
|
|
771
|
-
// run-registry/format.ts). Re-exported so importers of "./observability" keep
|
|
772
|
-
// the unchanged surface.
|
|
773
|
-
// ---------------------------------------------------------------------------
|
|
774
|
-
var format_1 = require("./observability/format");
|
|
775
|
-
Object.defineProperty(exports, "formatMetricsReport", { enumerable: true, get: function () { return format_1.formatMetricsReport; } });
|
|
776
|
-
Object.defineProperty(exports, "formatMetricsSummary", { enumerable: true, get: function () { return format_1.formatMetricsSummary; } });
|
|
598
|
+
var observability_intake_1 = require("./observability-intake");
|
|
599
|
+
Object.defineProperty(exports, "loadCostPolicy", { enumerable: true, get: function () { return observability_intake_1.loadCostPolicy; } });
|
|
600
|
+
Object.defineProperty(exports, "parseUsageFromArgs", { enumerable: true, get: function () { return observability_intake_1.parseUsageFromArgs; } });
|
|
601
|
+
var observability_format_1 = require("./observability-format");
|
|
602
|
+
Object.defineProperty(exports, "formatMetricsReport", { enumerable: true, get: function () { return observability_format_1.formatMetricsReport; } });
|
|
603
|
+
Object.defineProperty(exports, "formatMetricsSummary", { enumerable: true, get: function () { return observability_format_1.formatMetricsSummary; } });
|