crysk 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -22
- package/dist/SKILL-1q3658n0.md +74 -0
- package/dist/SKILL-25j4q9b8.md +143 -0
- package/dist/SKILL-32cmme3j.md +143 -0
- package/dist/SKILL-4747bhne.md +143 -0
- package/dist/SKILL-4jc3sv23.md +74 -0
- package/dist/SKILL-c0492rch.md +540 -0
- package/dist/SKILL-d1nzkhhr.md +74 -0
- package/dist/SKILL-f8p7kvx0.md +143 -0
- package/dist/SKILL-gxd45g38.md +553 -0
- package/dist/SKILL-rec41991.md +66 -0
- package/dist/SKILL-zze8zj5g.md +536 -0
- package/dist/adjudicator-eckqh14e.md +78 -0
- package/dist/ci-admission-qn797a6c.md +128 -0
- package/dist/cli.js +18608 -18523
- package/dist/consolidate-4xw1pkg7.md +24 -0
- package/dist/deep-review-logic-dgr112y1.md +196 -0
- package/dist/deep-review-logic-fck5z4xe.md +197 -0
- package/dist/deep-review-logic-h1fpamfh.md +192 -0
- package/dist/deep-review-logic-m4vk5683.md +196 -0
- package/dist/deep-review-logic-z63kmgmc.md +157 -0
- package/dist/deep-review-parity-7x7s5d81.md +148 -0
- package/dist/deep-review-parity-9wxmynsc.md +115 -0
- package/dist/deep-review-parity-bwwe7cx6.md +152 -0
- package/dist/deep-review-parity-e50pw3pk.md +117 -0
- package/dist/deep-review-parity-ennmm9m1.md +151 -0
- package/{prompts/default/deep-review-parity.md → dist/deep-review-parity-h7y6y2he.md} +14 -16
- package/dist/deep-review-parity-xg6wy69c.md +151 -0
- package/dist/deep-review-reliability-6h3c4gmw.md +125 -0
- package/dist/deep-review-reliability-dcjab69x.md +122 -0
- package/dist/deep-review-reliability-ec7b5pbk.md +159 -0
- package/dist/deep-review-reliability-hvasvmt9.md +153 -0
- package/dist/deep-review-reliability-mjnayxmt.md +158 -0
- package/dist/deep-review-reliability-mm7fnayf.md +124 -0
- package/dist/deep-review-reliability-xk4xvc2w.md +158 -0
- package/dist/deep-review-resilience-0esd5qeh.md +148 -0
- package/dist/deep-review-resilience-envgcmpy.md +115 -0
- package/dist/deep-review-resilience-fgx8ythf.md +151 -0
- package/dist/deep-review-resilience-mq5qa6ad.md +152 -0
- package/dist/deep-review-resilience-pwh1gawp.md +151 -0
- package/dist/deep-review-resilience-s9n21dkx.md +117 -0
- package/dist/deep-review-resilience-vz4qskea.md +119 -0
- package/dist/deep-review-security-1vwjdc9q.md +205 -0
- package/dist/deep-review-security-x1cc1z00.md +205 -0
- package/dist/index.js +590 -485
- package/dist/opencode-ci-jc8mf4wn.md +106 -0
- package/dist/opencode-ci-qrcv7a99.md +106 -0
- package/dist/opencode-ci-rdpcc7xc.md +106 -0
- package/dist/opencode-ci-yt0k74t0.md +106 -0
- package/dist/review-refuter-7qffwtzz.md +77 -0
- package/dist/verifier-tyh8r45v.md +40 -0
- package/dist/verifier-xf4xj701.md +40 -0
- package/dist/workflow-andv2dvm.yml +158 -0
- package/package.json +1 -1
- package/prompts/consolidate.md +1 -1
- package/prompts/default/PROVENANCE.md +6 -0
- package/prompts/default/deep-review-logic.md +56 -15
- package/prompts/default/deep-review-reliability.md +64 -30
- package/prompts/default/deep-review-resilience.md +71 -37
- package/prompts/default/deep-review-security.md +205 -0
- package/prompts/default/review-refuter.md +1 -1
- package/prompts/verifier.md +40 -0
- package/skills/crysk/SKILL.md +6 -6
- package/skills/crysk-ci-setup/SKILL.md +32 -33
- package/skills/crysk-ci-setup/assets/workflow.yml +5 -10
- package/skills/crysk-ci-setup/references/ci-admission.md +5 -5
- package/skills/crysk-ci-setup/references/opencode-ci.md +6 -6
- package/skills/crysk-loop/SKILL.md +37 -54
- package/skills/crysk-loop/adjudicator.md +10 -18
- package/skills/crysk-pr/SKILL.md +1 -1
- package/src/asset-manifest.ts +6 -6
- package/src/assets.ts +4 -4
- package/src/brand.ts +20 -30
- package/src/ci/gates.ts +155 -192
- package/src/ci/outcome.ts +4 -31
- package/src/ci/platform-limits.ts +135 -0
- package/src/ci/platform-report.ts +212 -126
- package/src/ci/platform.ts +28 -26
- package/src/ci/reporter.ts +25 -26
- package/src/ci/review-admission.ts +1 -2
- package/src/ci/review-levels.ts +12 -53
- package/src/ci/setup.ts +96 -66
- package/src/ci/trace-bundle.ts +11 -27
- package/src/ci/trace-transcripts.ts +4 -6
- package/src/ci/trigger.ts +3 -5
- package/src/cli.ts +6 -5
- package/src/commands/ci-setup-questions.ts +2 -55
- package/src/commands/ci-setup.ts +2 -26
- package/src/commands/triage.ts +131 -38
- package/src/commands/usage.ts +0 -1
- package/src/compare/compare.ts +3 -3
- package/src/compare/floor-test.ts +30 -201
- package/src/compare/hunter-exam.ts +631 -0
- package/src/config/config.ts +6 -30
- package/src/diversity/accounting.ts +0 -26
- package/src/diversity/benchmark.ts +1 -17
- package/src/diversity/identity.ts +0 -14
- package/src/diversity/pipeline-integration.ts +0 -29
- package/src/doctor.ts +3 -6
- package/src/execution/claude-session-trace.ts +7 -2
- package/src/execution/contracts.ts +15 -9
- package/src/execution/harness.ts +165 -187
- package/src/execution/spend-limiter.ts +39 -131
- package/src/execution/usage-facts.ts +397 -0
- package/src/execution/usage-normalized.ts +33 -86
- package/src/harness/materiality-judge.ts +0 -5
- package/src/harness/pipeline-plan.ts +0 -3
- package/src/harness/replay-rows.ts +0 -5
- package/src/harness/route-runner.ts +2 -2
- package/src/harness/single-step.ts +3 -9
- package/src/index.ts +1 -1
- package/src/mcp/preflight.ts +2 -2
- package/src/model/catalog.ts +1 -3
- package/src/model/free-discovery.ts +11 -78
- package/src/model/model-name.ts +16 -5
- package/src/model/profiles/claude.ts +1 -2
- package/src/model/profiles/openai.ts +1 -2
- package/src/model/profiles.ts +2 -3
- package/src/model/provider-capabilities.ts +16 -55
- package/src/model/routing.ts +39 -36
- package/src/pr/admission.ts +6 -6
- package/src/pr/answered-findings.ts +8 -4
- package/src/pr/ci-publish.ts +0 -2
- package/src/pr/discovery.ts +4 -25
- package/src/pr/plan.ts +40 -121
- package/src/pr/pr-status.ts +14 -2
- package/src/pr/pr.ts +1 -1
- package/src/pr/preflight.ts +44 -15
- package/src/pr/publish-outcome.ts +1 -19
- package/src/pr/review-pr.ts +15 -47
- package/src/pr/target.ts +76 -82
- package/src/production-runtime.ts +52 -114
- package/src/progress.ts +0 -54
- package/src/rereview/prepare.ts +4 -1
- package/src/rereview/state.ts +1 -1
- package/src/rereview/verify.ts +15 -9
- package/src/review/boundary.ts +5 -8
- package/src/review/drafts.ts +3 -3
- package/src/review/findings-conformance.ts +1 -1
- package/src/review/findings.ts +0 -4
- package/src/review/pipeline.ts +194 -407
- package/src/review/preflight.ts +175 -201
- package/src/review/prompt-set.ts +1 -2
- package/src/review/report.ts +2 -162
- package/src/review/review.ts +25 -74
- package/src/review/route-preflight.ts +19 -35
- package/src/review/run.ts +37 -73
- package/src/review/size-gate.ts +3 -3
- package/src/review/spec.ts +6 -11
- package/src/review/step-runner.ts +4 -4
- package/src/runner-authority.ts +10 -9
- package/src/security/credential-broker.ts +5 -6
- package/src/server/preflight.ts +0 -1
- package/src/server/server.ts +17 -6
- package/src/store/activity.ts +1 -2
- package/src/store/backfill-preflight.ts +0 -3
- package/src/store/backfill.ts +0 -1
- package/src/store/metrics-preflight.ts +4 -14
- package/src/store/metrics.ts +5 -5
- package/src/store/preflight.ts +0 -13
- package/src/store/store.ts +11 -12
- package/src/transport-registry.ts +6 -30
- package/src/transports/claude-code-cli.ts +149 -215
- package/src/transports/opencode-client.ts +4 -24
- package/src/transports/opencode-sdk.ts +17 -112
- package/src/triage/plain-reply.ts +37 -0
- package/src/ui/activity.ts +1 -6
- package/src/ui/config-edit.ts +14 -67
- package/src/ui/config.ts +3 -8
- package/src/ui/plan.ts +11 -198
- package/src/ui/progress.ts +3 -49
- package/src/ui/result.ts +18 -77
- package/src/ui/review-menu.ts +3 -12
- package/src/ui/select.ts +9 -79
- package/src/updater.ts +1 -1
- package/src/usage.ts +1 -7
- package/prompts/scout.md +0 -140
- package/src/ci/platform-estimate.ts +0 -111
- package/src/review/scout.ts +0 -323
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// The limits the Crysk platform sends with a run (platform-owned-cost L55). The
|
|
2
|
+
// connect step declares the `limits` capability in `X-Crysk-Capabilities`; only
|
|
3
|
+
// then does the `/runs/token` response carry `limits`, and the step hands it to
|
|
4
|
+
// the run step as `CRYSK_PLATFORM_LIMITS`, where the CI limits gate reads it
|
|
5
|
+
// (gates.ts, `evaluatePlatformLimits`).
|
|
6
|
+
//
|
|
7
|
+
// The platform owns the settings, the defaults and the month's spend; the
|
|
8
|
+
// engine only compares, because only it knows whether the run bills an API key.
|
|
9
|
+
// The field is absent when the dispatch was forced (`/crysk review`, `recheck`),
|
|
10
|
+
// and a forced run is therefore never skipped for cost.
|
|
11
|
+
//
|
|
12
|
+
// The platform already checked the numbers and the engine checks again: a field
|
|
13
|
+
// that is not a usable number is dropped, and with nothing left the run is
|
|
14
|
+
// treated as having no limits. Limits are advice for a skip, never a reason to
|
|
15
|
+
// fail the exchange.
|
|
16
|
+
//
|
|
17
|
+
// This module imports nothing, so the connect step (which runs before
|
|
18
|
+
// `bun install`) can load it.
|
|
19
|
+
|
|
20
|
+
export interface PlatformLimits {
|
|
21
|
+
// The per-review limit in effect: repository override, else organization
|
|
22
|
+
// setting, else the platform's default. Always above 0 when present.
|
|
23
|
+
readonly reviewUsd?: number;
|
|
24
|
+
// What the organization's month limit leaves after this calendar month's API
|
|
25
|
+
// spend (UTC), floored at 0. Absent when no month limit is set.
|
|
26
|
+
readonly monthLeftUsd?: number;
|
|
27
|
+
// The platform's estimate for this review, its high side. Absent when the
|
|
28
|
+
// platform cannot price the run.
|
|
29
|
+
readonly estimateUsd?: number;
|
|
30
|
+
// `estimateUsd` split per model role (`hunter.logic`, `refuter`, ...): the
|
|
31
|
+
// platform's setup estimate priced line by line. Present only when the
|
|
32
|
+
// platform priced every line. It lets the engine compare the share of the
|
|
33
|
+
// estimate that runs on an API key instead of the whole of it (S17).
|
|
34
|
+
readonly estimateByRoleUsd?: Readonly<Record<string, number>>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Which limit a skip names (gates.ts, `evaluatePlatformLimits`): the month's
|
|
38
|
+
// limit is used up, the estimate is over the per-review limit, or the estimate
|
|
39
|
+
// is over what is left of the month.
|
|
40
|
+
export type PlatformLimitHit = "month-used-up" | "review-limit" | "month-left";
|
|
41
|
+
|
|
42
|
+
export const PLATFORM_LIMITS_CAPABILITY = "limits";
|
|
43
|
+
|
|
44
|
+
const finite = (value: unknown): value is number =>
|
|
45
|
+
typeof value === "number" && Number.isFinite(value);
|
|
46
|
+
|
|
47
|
+
// Role keys are the engine's `models` keys (`hunter.<specialty>` at most 64
|
|
48
|
+
// characters after the prefix); the cap keeps a hostile body small.
|
|
49
|
+
const MAX_ROLE_KEY_LENGTH = 72;
|
|
50
|
+
const MAX_ROLES = 32;
|
|
51
|
+
|
|
52
|
+
// The split as sent, or undefined. One bad entry drops the whole field, never
|
|
53
|
+
// the entry: a dropped role would read as free, and the engine's fallback for
|
|
54
|
+
// no split (the whole estimate) is the conservative one.
|
|
55
|
+
function parseEstimateByRole(
|
|
56
|
+
value: unknown,
|
|
57
|
+
): Readonly<Record<string, number>> | undefined {
|
|
58
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
const entries = Object.entries(value);
|
|
62
|
+
if (entries.length === 0 || entries.length > MAX_ROLES) return undefined;
|
|
63
|
+
for (const [role, usd] of entries) {
|
|
64
|
+
if (role.length === 0 || role.length > MAX_ROLE_KEY_LENGTH) {
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
if (!finite(usd) || usd < 0) return undefined;
|
|
68
|
+
}
|
|
69
|
+
return Object.fromEntries(entries) as Record<string, number>;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// The `/runs/token` field, `{ review_usd?, month_left_usd?, estimate_usd?,
|
|
73
|
+
// estimate_by_role_usd? }`, or undefined when it is absent or holds no usable
|
|
74
|
+
// number. `review_usd` must be above 0 (the platform omits it when there is no
|
|
75
|
+
// limit); the other numbers may be 0.
|
|
76
|
+
export function parsePlatformLimits(
|
|
77
|
+
value: unknown,
|
|
78
|
+
): PlatformLimits | undefined {
|
|
79
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
const {
|
|
83
|
+
review_usd: review,
|
|
84
|
+
month_left_usd: monthLeft,
|
|
85
|
+
estimate_usd: estimate,
|
|
86
|
+
estimate_by_role_usd: byRole,
|
|
87
|
+
} = value as Record<string, unknown>;
|
|
88
|
+
// The split only means something beside the estimate it splits.
|
|
89
|
+
const split =
|
|
90
|
+
finite(estimate) && estimate >= 0 ? parseEstimateByRole(byRole) : undefined;
|
|
91
|
+
const limits: {
|
|
92
|
+
reviewUsd?: number;
|
|
93
|
+
monthLeftUsd?: number;
|
|
94
|
+
estimateUsd?: number;
|
|
95
|
+
estimateByRoleUsd?: Readonly<Record<string, number>>;
|
|
96
|
+
} = {
|
|
97
|
+
...(finite(review) && review > 0 ? { reviewUsd: review } : {}),
|
|
98
|
+
...(finite(monthLeft) && monthLeft >= 0 ? { monthLeftUsd: monthLeft } : {}),
|
|
99
|
+
...(finite(estimate) && estimate >= 0 ? { estimateUsd: estimate } : {}),
|
|
100
|
+
...(split === undefined ? {} : { estimateByRoleUsd: split }),
|
|
101
|
+
};
|
|
102
|
+
return Object.keys(limits).length === 0 ? undefined : limits;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// What the connect step writes to its `limits` output: one line of JSON in the
|
|
106
|
+
// response's own field names, numbers only (the split is an object of numbers
|
|
107
|
+
// keyed by role).
|
|
108
|
+
export function serializePlatformLimits(limits: PlatformLimits): string {
|
|
109
|
+
return JSON.stringify({
|
|
110
|
+
...(limits.reviewUsd === undefined ? {} : { review_usd: limits.reviewUsd }),
|
|
111
|
+
...(limits.monthLeftUsd === undefined
|
|
112
|
+
? {}
|
|
113
|
+
: { month_left_usd: limits.monthLeftUsd }),
|
|
114
|
+
...(limits.estimateUsd === undefined
|
|
115
|
+
? {}
|
|
116
|
+
: { estimate_usd: limits.estimateUsd }),
|
|
117
|
+
...(limits.estimateByRoleUsd === undefined
|
|
118
|
+
? {}
|
|
119
|
+
: { estimate_by_role_usd: limits.estimateByRoleUsd }),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// The run step's read of `CRYSK_PLATFORM_LIMITS`. Empty (a standalone run, a
|
|
124
|
+
// forced run, an older platform) or malformed reads as no limits.
|
|
125
|
+
export function readPlatformLimits(
|
|
126
|
+
env: Readonly<Record<string, string | undefined>>,
|
|
127
|
+
): PlatformLimits | undefined {
|
|
128
|
+
const text = env.CRYSK_PLATFORM_LIMITS;
|
|
129
|
+
if (text === undefined || text === "") return undefined;
|
|
130
|
+
try {
|
|
131
|
+
return parsePlatformLimits(JSON.parse(text));
|
|
132
|
+
} catch {
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// The action's report step: tell the Crysk platform how a run ended, so every
|
|
2
|
-
// run is recorded (#157) with its
|
|
2
|
+
// run is recorded (#157) with its usage attributed (#139), then hand it the
|
|
3
3
|
// run's findings so the dashboard can show them.
|
|
4
4
|
//
|
|
5
5
|
// It runs only in platform mode (action.yml gates the step on
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// after the publish step. Standalone use never reaches it.
|
|
8
8
|
//
|
|
9
9
|
// WHY THE REPORT IS METADATA ONLY: the body carries counts, token totals,
|
|
10
|
-
//
|
|
10
|
+
// model names and shas. It never carries a finding, a claim, a summary,
|
|
11
11
|
// a hop trail or any other text that can quote the customer's code. The
|
|
12
12
|
// builder below COPIES the numeric fields it names out of the run directory
|
|
13
13
|
// instead of forwarding a document, so a field added to findings.json tomorrow
|
|
@@ -81,11 +81,8 @@ interface ReportTokens {
|
|
|
81
81
|
output?: number;
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
-
// Added beside the legacy fields, never replacing them: `cost_usd` stays the cash cost (0 on a
|
|
85
|
-
// subscription), and `list_cost_usd` is what the same tokens cost at list price whatever paid for them.
|
|
86
84
|
interface ReportUsageSplit {
|
|
87
85
|
tokens?: ReportTokens;
|
|
88
|
-
list_cost_usd?: number;
|
|
89
86
|
}
|
|
90
87
|
|
|
91
88
|
export interface RunReportAgent extends ReportUsageSplit {
|
|
@@ -93,7 +90,6 @@ export interface RunReportAgent extends ReportUsageSplit {
|
|
|
93
90
|
model?: string;
|
|
94
91
|
tokens_in: number;
|
|
95
92
|
tokens_out: number;
|
|
96
|
-
cost_usd: number;
|
|
97
93
|
duration_ms?: number;
|
|
98
94
|
attempts?: number;
|
|
99
95
|
status?: string;
|
|
@@ -120,16 +116,31 @@ export interface RunReportBody {
|
|
|
120
116
|
usage: ReportUsageSplit & {
|
|
121
117
|
tokens_in: number;
|
|
122
118
|
tokens_out: number;
|
|
123
|
-
cost_usd: number;
|
|
124
|
-
cost_is_floor?: boolean;
|
|
125
119
|
wall_ms?: number;
|
|
126
120
|
index_ms?: number;
|
|
127
121
|
sync_ms?: number;
|
|
128
|
-
billing?: "subscription" | "metered";
|
|
129
122
|
};
|
|
130
123
|
agents: RunReportAgent[];
|
|
131
124
|
// Which findings stay open, under ids that last across runs. Absent when the run recorded none.
|
|
132
125
|
finding_states?: FindingStateRecord;
|
|
126
|
+
// What the run measured per model request: the platform prices the run from it, so the body
|
|
127
|
+
// carries no dollar figure anywhere.
|
|
128
|
+
usage_facts: ReportUsageFacts;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export type ReportCredential = "api" | "subscription" | "free";
|
|
132
|
+
|
|
133
|
+
// odd/tasks/platform-owned-cost-contract.md section 3. Indexes point into `steps` and `routes`.
|
|
134
|
+
export interface ReportUsageFacts {
|
|
135
|
+
v: 1;
|
|
136
|
+
steps: { key: string; role: string; agent?: string }[];
|
|
137
|
+
routes: { provider: string; model: string; credential: ReportCredential }[];
|
|
138
|
+
// [step, attempt, route, uncached, cacheRead, cacheWrite, cacheWrite1h, output, reasoning]
|
|
139
|
+
requests: number[][];
|
|
140
|
+
// [step, route, uncached, cacheRead, cacheWrite, cacheWrite1h, output, reasoning]
|
|
141
|
+
remainders?: number[][];
|
|
142
|
+
// [step, attempt, reason]
|
|
143
|
+
gaps?: [number, number, string][];
|
|
133
144
|
}
|
|
134
145
|
|
|
135
146
|
// What the builder knows about the run directory. Either part is absent when
|
|
@@ -177,8 +188,10 @@ export function buildRunReport(
|
|
|
177
188
|
pr_number: input.prNumber,
|
|
178
189
|
head_sha: headSha,
|
|
179
190
|
counts: { blocking: 0, advisory: 0 },
|
|
180
|
-
usage: { tokens_in: 0, tokens_out: 0
|
|
191
|
+
usage: { tokens_in: 0, tokens_out: 0 },
|
|
181
192
|
agents: [],
|
|
193
|
+
// A run that wrote no findings can still have spent tokens before it died.
|
|
194
|
+
usage_facts: buildUsageFacts(input.run.pipeline, new Map()),
|
|
182
195
|
};
|
|
183
196
|
}
|
|
184
197
|
|
|
@@ -189,6 +202,14 @@ export function buildRunReport(
|
|
|
189
202
|
if (finding.tier === "blocking") blocking += 1;
|
|
190
203
|
else if (finding.tier === "advisory") advisory += 1;
|
|
191
204
|
}
|
|
205
|
+
const steps = pipelineSteps(input.run.pipeline);
|
|
206
|
+
const rows = agentRows(doc);
|
|
207
|
+
// Owners are decided over every row, the capped ones too, so a step of a row
|
|
208
|
+
// past the cap is dropped with it instead of landing on another row.
|
|
209
|
+
const owned = stepsByOwner(
|
|
210
|
+
steps,
|
|
211
|
+
rows.map(([key]) => key),
|
|
212
|
+
);
|
|
192
213
|
|
|
193
214
|
return {
|
|
194
215
|
outcome: input.outcome === "" ? "error" : input.outcome,
|
|
@@ -212,10 +233,6 @@ export function buildRunReport(
|
|
|
212
233
|
usage: {
|
|
213
234
|
tokens_in: count(telemetry.tokens_in),
|
|
214
235
|
tokens_out: count(telemetry.tokens_out),
|
|
215
|
-
cost_usd: cost(telemetry.cost_usd_est),
|
|
216
|
-
...(telemetry.cost_usd_est_is_floor === true
|
|
217
|
-
? { cost_is_floor: true }
|
|
218
|
-
: {}),
|
|
219
236
|
wall_ms: count(telemetry.wall_ms),
|
|
220
237
|
// Zero means no index was built, so there is no time to report.
|
|
221
238
|
...(count(telemetry.index_ms) > 0
|
|
@@ -226,8 +243,12 @@ export function buildRunReport(
|
|
|
226
243
|
: { sync_ms: count(telemetry.sync_ms) }),
|
|
227
244
|
...runUsageSplit(input.run.pipeline),
|
|
228
245
|
},
|
|
229
|
-
agents: buildAgents(
|
|
246
|
+
agents: buildAgents(rows, steps, owned),
|
|
230
247
|
...findingStates(input.run.findingStates),
|
|
248
|
+
usage_facts: buildUsageFacts(
|
|
249
|
+
input.run.pipeline,
|
|
250
|
+
ownerOfStep(owned, rows.slice(0, MAX_AGENTS)),
|
|
251
|
+
),
|
|
231
252
|
};
|
|
232
253
|
}
|
|
233
254
|
|
|
@@ -332,36 +353,29 @@ function count(value: unknown): number {
|
|
|
332
353
|
: 0;
|
|
333
354
|
}
|
|
334
355
|
|
|
335
|
-
function cost(value: unknown): number {
|
|
336
|
-
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
337
|
-
? value
|
|
338
|
-
: 0;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
356
|
// `telemetry.per_agent` is declared with the lab-shared minimum (tokens_total +
|
|
342
357
|
// duration_ms) but this engine writes the richer PerAgentUsage, so each field
|
|
343
358
|
// is read through a runtime check, as review/report.ts does. `detail` is never
|
|
344
359
|
// copied: it is the provider's own words and can echo prompt text.
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
360
|
+
type AgentRow = [string, Record<string, unknown>];
|
|
361
|
+
|
|
362
|
+
// validateFindingsDocument only checks that `telemetry` is an object, so a
|
|
363
|
+
// row that is not one is skipped instead of read.
|
|
364
|
+
function agentRows(doc: FindingsDocument): AgentRow[] {
|
|
350
365
|
const perAgent = doc.telemetry.per_agent ?? {};
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
const rows = Object.entries(perAgent as Record<string, unknown>).filter(
|
|
354
|
-
(entry): entry is [string, Record<string, unknown>] =>
|
|
366
|
+
return Object.entries(perAgent as Record<string, unknown>).filter(
|
|
367
|
+
(entry): entry is AgentRow =>
|
|
355
368
|
typeof entry[1] === "object" &&
|
|
356
369
|
entry[1] !== null &&
|
|
357
370
|
!Array.isArray(entry[1]),
|
|
358
371
|
);
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function buildAgents(
|
|
375
|
+
rows: ReadonlyArray<AgentRow>,
|
|
376
|
+
steps: ReadonlyArray<PipelineStep>,
|
|
377
|
+
owned: ReadonlyMap<string, PipelineStep[]>,
|
|
378
|
+
): RunReportAgent[] {
|
|
365
379
|
return rows.slice(0, MAX_AGENTS).map(([key, row]) => {
|
|
366
380
|
const model = modelOf(steps, key);
|
|
367
381
|
return {
|
|
@@ -369,7 +383,6 @@ function buildAgents(
|
|
|
369
383
|
...(model === undefined ? {} : { model }),
|
|
370
384
|
tokens_in: count(row.tokens_in),
|
|
371
385
|
tokens_out: count(row.tokens_out),
|
|
372
|
-
cost_usd: cost(row.cost_usd_est),
|
|
373
386
|
duration_ms: count(row.duration_ms),
|
|
374
387
|
...(typeof row.attempts === "number" && row.attempts > 0
|
|
375
388
|
? { attempts: count(row.attempts) }
|
|
@@ -380,14 +393,14 @@ function buildAgents(
|
|
|
380
393
|
});
|
|
381
394
|
}
|
|
382
395
|
|
|
383
|
-
// pipeline.json's `steps[]` rows are `{ name, model, usage_v2,
|
|
384
|
-
// Only what this file reads is kept, and `usage_v2`
|
|
385
|
-
//
|
|
396
|
+
// pipeline.json's `steps[]` rows are `{ name, model, usage_v2, usage_facts, ... }`.
|
|
397
|
+
// Only what this file reads is kept, and `usage_v2` and `usage_facts` stay
|
|
398
|
+
// unread until the code that sends them checks each of their fields.
|
|
386
399
|
interface PipelineStep {
|
|
387
400
|
name: string;
|
|
388
401
|
model?: string;
|
|
389
402
|
usage?: unknown;
|
|
390
|
-
|
|
403
|
+
usageFacts?: unknown;
|
|
391
404
|
}
|
|
392
405
|
|
|
393
406
|
function pipelineSteps(pipeline: unknown): PipelineStep[] {
|
|
@@ -397,7 +410,7 @@ function pipelineSteps(pipeline: unknown): PipelineStep[] {
|
|
|
397
410
|
const rows: PipelineStep[] = [];
|
|
398
411
|
for (const step of steps) {
|
|
399
412
|
if (typeof step !== "object" || step === null) continue;
|
|
400
|
-
const { name, model, usage_v2,
|
|
413
|
+
const { name, model, usage_v2, usage_facts } = step as Record<
|
|
401
414
|
string,
|
|
402
415
|
unknown
|
|
403
416
|
>;
|
|
@@ -406,13 +419,7 @@ function pipelineSteps(pipeline: unknown): PipelineStep[] {
|
|
|
406
419
|
name,
|
|
407
420
|
...(typeof model === "string" && model !== "" ? { model } : {}),
|
|
408
421
|
usage: usage_v2,
|
|
409
|
-
...(
|
|
410
|
-
? {
|
|
411
|
-
modelUsages: model_attempts.map(
|
|
412
|
-
(entry) => (entry as { usage_v2?: unknown } | null)?.usage_v2,
|
|
413
|
-
),
|
|
414
|
-
}
|
|
415
|
-
: {}),
|
|
422
|
+
...(usage_facts === undefined ? {} : { usageFacts: usage_facts }),
|
|
416
423
|
});
|
|
417
424
|
}
|
|
418
425
|
return rows;
|
|
@@ -527,85 +534,14 @@ function tokenSplit(usage: UsageRecord | undefined): ReportTokens {
|
|
|
527
534
|
);
|
|
528
535
|
}
|
|
529
536
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
// `costFor`): `notionalCostUsd` on a subscription, which charges nothing, and
|
|
533
|
-
// `cashCostUsd` on a per-token key, where list price is the charge. Any other
|
|
534
|
-
// billing (free, or unknown because attempts disagreed) names no field that
|
|
535
|
-
// holds a list price, so it has none rather than a guessed one.
|
|
536
|
-
function listCost(usage: UsageRecord | undefined): number | undefined {
|
|
537
|
-
switch (usage?.billingMode) {
|
|
538
|
-
case "subscription":
|
|
539
|
-
return amount(usage.notionalCostUsd);
|
|
540
|
-
case "metered":
|
|
541
|
-
return amount(usage.cashCostUsd);
|
|
542
|
-
default:
|
|
543
|
-
return undefined;
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
// A step whose models billed differently reads `unknown` once summed, so its
|
|
548
|
-
// list cost is each model's own (`model_attempts[].usage_v2`, kept for exactly
|
|
549
|
-
// this). Unknown when any model that reported usage cannot be priced.
|
|
550
|
-
function stepListCost(step: PipelineStep): number | undefined {
|
|
551
|
-
const usage = usageRecord(step.usage);
|
|
552
|
-
const own = listCost(usage);
|
|
553
|
-
if (own !== undefined || step.modelUsages === undefined) return own;
|
|
554
|
-
return sumAll(
|
|
555
|
-
step.modelUsages.map(usageRecord).filter((entry) => entry !== undefined),
|
|
556
|
-
listCost,
|
|
557
|
-
);
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
// A sum only when every record that reported usage has a value: one unpriced
|
|
561
|
-
// part would make the total look smaller than what was spent.
|
|
562
|
-
function sumAll<T>(
|
|
563
|
-
items: readonly T[],
|
|
564
|
-
value: (item: T) => number | undefined,
|
|
565
|
-
): number | undefined {
|
|
566
|
-
if (items.length === 0) return undefined;
|
|
567
|
-
let total = 0;
|
|
568
|
-
for (const item of items) {
|
|
569
|
-
const part = value(item);
|
|
570
|
-
if (part === undefined) return undefined;
|
|
571
|
-
total += part;
|
|
572
|
-
}
|
|
573
|
-
return total;
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
function splitFields(
|
|
577
|
-
tokens: ReportTokens,
|
|
578
|
-
listUsd: number | undefined,
|
|
579
|
-
): ReportUsageSplit {
|
|
580
|
-
return {
|
|
581
|
-
...(Object.keys(tokens).length === 0 ? {} : { tokens }),
|
|
582
|
-
...(listUsd === undefined ? {} : { list_cost_usd: listUsd }),
|
|
583
|
-
};
|
|
537
|
+
function splitFields(tokens: ReportTokens): ReportUsageSplit {
|
|
538
|
+
return Object.keys(tokens).length === 0 ? {} : { tokens };
|
|
584
539
|
}
|
|
585
540
|
|
|
586
|
-
// The run's
|
|
587
|
-
|
|
588
|
-
// counts only the steps that filed one while the tokens count them all, so a step
|
|
589
|
-
// with no list price drops the run's list cost rather than leaving it short. A
|
|
590
|
-
// pipeline with no per-step usage falls back to the run's own figure.
|
|
591
|
-
function runUsageSplit(
|
|
592
|
-
pipeline: unknown,
|
|
593
|
-
): ReportUsageSplit & { billing?: "subscription" | "metered" } {
|
|
541
|
+
// The run's token split comes from its own `usage_v2`.
|
|
542
|
+
function runUsageSplit(pipeline: unknown): ReportUsageSplit {
|
|
594
543
|
const usage = usageRecord(usageRecord(pipeline)?.usage_v2);
|
|
595
|
-
|
|
596
|
-
const billing = usage.billingMode;
|
|
597
|
-
// Every step that filed usage, a malformed record included: one that cannot be
|
|
598
|
-
// read cannot be priced, so it drops the sum rather than shrinking it.
|
|
599
|
-
const priced = pipelineSteps(pipeline).filter(
|
|
600
|
-
(step) => step.usage !== undefined,
|
|
601
|
-
);
|
|
602
|
-
return {
|
|
603
|
-
...splitFields(
|
|
604
|
-
tokenSplit(usage),
|
|
605
|
-
priced.length > 0 ? sumAll(priced, stepListCost) : listCost(usage),
|
|
606
|
-
),
|
|
607
|
-
...(billing === "subscription" || billing === "metered" ? { billing } : {}),
|
|
608
|
-
};
|
|
544
|
+
return usage === undefined ? {} : splitFields(tokenSplit(usage));
|
|
609
545
|
}
|
|
610
546
|
|
|
611
547
|
// A row's split is the sum of the steps it owns, refuter fan-out included.
|
|
@@ -626,7 +562,157 @@ function agentUsageSplit(steps: ReadonlyArray<PipelineStep>): ReportUsageSplit {
|
|
|
626
562
|
return total === undefined ? [] : [[key, total]];
|
|
627
563
|
}),
|
|
628
564
|
);
|
|
629
|
-
return splitFields(tokens
|
|
565
|
+
return splitFields(tokens);
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// Which `per_agent` row owns each step, among the rows the report sends, so a
|
|
569
|
+
// usage step names the `run_agents` row it belongs to by the same rule the
|
|
570
|
+
// rows' own usage follows (stepsByOwner), never a second one.
|
|
571
|
+
function ownerOfStep(
|
|
572
|
+
owned: ReadonlyMap<string, PipelineStep[]>,
|
|
573
|
+
sent: ReadonlyArray<AgentRow>,
|
|
574
|
+
): Map<string, string> {
|
|
575
|
+
const owners = new Map<string, string>();
|
|
576
|
+
for (const [key] of sent) {
|
|
577
|
+
for (const step of owned.get(key) ?? []) owners.set(step.name, key);
|
|
578
|
+
}
|
|
579
|
+
return owners;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// The route plan's role for a step, else the step name's first word. Also the
|
|
583
|
+
// trace step's role (src/ci/trace-bundle.ts), so both name a step alike.
|
|
584
|
+
export function stepRole(pipeline: unknown, name: string): string {
|
|
585
|
+
const plan = usageRecord(usageRecord(pipeline)?.route_plan);
|
|
586
|
+
const planned = Array.isArray(plan?.steps)
|
|
587
|
+
? plan.steps
|
|
588
|
+
.map(usageRecord)
|
|
589
|
+
.find((step) => step !== undefined && step.stepKey === name)
|
|
590
|
+
: undefined;
|
|
591
|
+
const role = planned?.role;
|
|
592
|
+
if (typeof role === "string" && role !== "") return role;
|
|
593
|
+
return name.split(/[-.]/)[0] ?? name;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
const CREDENTIALS = new Set<string>(["api", "subscription", "free"]);
|
|
597
|
+
const GAP_REASONS = new Set<string>([
|
|
598
|
+
"no_session",
|
|
599
|
+
"line_omitted",
|
|
600
|
+
"truncated",
|
|
601
|
+
"no_readback",
|
|
602
|
+
"no_summary",
|
|
603
|
+
]);
|
|
604
|
+
// The platform's own bound on a label (apps/api/src/runs/usage-facts.ts).
|
|
605
|
+
const MAX_LABEL = 200;
|
|
606
|
+
|
|
607
|
+
function label(value: unknown): value is string {
|
|
608
|
+
return (
|
|
609
|
+
typeof value === "string" &&
|
|
610
|
+
value.length > 0 &&
|
|
611
|
+
value.length <= MAX_LABEL &&
|
|
612
|
+
!value.includes("\u0000")
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
function tokenCount(value: unknown): value is number {
|
|
617
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
// One [provider, model, ...six counts] entry of a pipeline.json attempt, as the
|
|
621
|
+
// harness wrote it (src/execution/usage-facts.ts), or undefined when it is not one.
|
|
622
|
+
function factEntry(
|
|
623
|
+
value: unknown,
|
|
624
|
+
): { provider: string; model: string; counts: number[] } | undefined {
|
|
625
|
+
if (!Array.isArray(value) || value.length !== 8) return undefined;
|
|
626
|
+
const [provider, model, ...counts] = value as unknown[];
|
|
627
|
+
return label(provider) && label(model) && counts.every(tokenCount)
|
|
628
|
+
? { provider, model, counts: counts as number[] }
|
|
629
|
+
: undefined;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// The platform's `usage_facts` (odd/tasks/platform-owned-cost-contract.md,
|
|
633
|
+
// section 3) from each step's `usage_facts` in pipeline.json. This only copies:
|
|
634
|
+
// the harness read every number at the attempt's settlement. Identifiers and
|
|
635
|
+
// counts only, so it is safe in metadata-only mode too. An entry that does not
|
|
636
|
+
// check is left out on its own; the platform counts nothing for it.
|
|
637
|
+
function buildUsageFacts(
|
|
638
|
+
pipeline: unknown,
|
|
639
|
+
owners: ReadonlyMap<string, string>,
|
|
640
|
+
): ReportUsageFacts {
|
|
641
|
+
const facts: ReportUsageFacts = {
|
|
642
|
+
v: 1,
|
|
643
|
+
steps: [],
|
|
644
|
+
routes: [],
|
|
645
|
+
requests: [],
|
|
646
|
+
};
|
|
647
|
+
const remainders = new Map<string, number[]>();
|
|
648
|
+
const gaps: [number, number, string][] = [];
|
|
649
|
+
const routeIndex = new Map<string, number>();
|
|
650
|
+
const routeOf = (
|
|
651
|
+
provider: string,
|
|
652
|
+
model: string,
|
|
653
|
+
credential: ReportCredential,
|
|
654
|
+
): number => {
|
|
655
|
+
const key = JSON.stringify([provider, model, credential]);
|
|
656
|
+
let index = routeIndex.get(key);
|
|
657
|
+
if (index === undefined) {
|
|
658
|
+
index = facts.routes.push({ provider, model, credential }) - 1;
|
|
659
|
+
routeIndex.set(key, index);
|
|
660
|
+
}
|
|
661
|
+
return index;
|
|
662
|
+
};
|
|
663
|
+
for (const step of pipelineSteps(pipeline)) {
|
|
664
|
+
if (!Array.isArray(step.usageFacts) || !label(step.name)) continue;
|
|
665
|
+
const attempts = step.usageFacts
|
|
666
|
+
.map(usageRecord)
|
|
667
|
+
.filter(
|
|
668
|
+
(a): a is UsageRecord =>
|
|
669
|
+
a !== undefined &&
|
|
670
|
+
tokenCount(a.attempt) &&
|
|
671
|
+
a.attempt >= 1 &&
|
|
672
|
+
typeof a.credential === "string" &&
|
|
673
|
+
CREDENTIALS.has(a.credential),
|
|
674
|
+
);
|
|
675
|
+
if (attempts.length === 0) continue;
|
|
676
|
+
const owner = owners.get(step.name);
|
|
677
|
+
const index =
|
|
678
|
+
facts.steps.push({
|
|
679
|
+
key: step.name,
|
|
680
|
+
role: stepRole(pipeline, step.name),
|
|
681
|
+
...(owner === undefined ? {} : { agent: owner }),
|
|
682
|
+
}) - 1;
|
|
683
|
+
for (const a of attempts) {
|
|
684
|
+
const attempt = a.attempt as number;
|
|
685
|
+
const credential = a.credential as ReportCredential;
|
|
686
|
+
for (const value of Array.isArray(a.requests) ? a.requests : []) {
|
|
687
|
+
const request = factEntry(value);
|
|
688
|
+
if (request === undefined) continue;
|
|
689
|
+
const route = routeOf(request.provider, request.model, credential);
|
|
690
|
+
facts.requests.push([index, attempt, route, ...request.counts]);
|
|
691
|
+
}
|
|
692
|
+
// The platform keys a remainder by step and route, so the step's attempts are summed.
|
|
693
|
+
for (const value of Array.isArray(a.remainders) ? a.remainders : []) {
|
|
694
|
+
const remainder = factEntry(value);
|
|
695
|
+
if (remainder === undefined) continue;
|
|
696
|
+
const route = routeOf(remainder.provider, remainder.model, credential);
|
|
697
|
+
const key = `${index}:${route}`;
|
|
698
|
+
const sum = remainders.get(key) ?? [index, route, 0, 0, 0, 0, 0, 0];
|
|
699
|
+
remainder.counts.forEach((n, i) => {
|
|
700
|
+
sum[i + 2] = (sum[i + 2] ?? 0) + n;
|
|
701
|
+
});
|
|
702
|
+
remainders.set(key, sum);
|
|
703
|
+
}
|
|
704
|
+
for (const reason of Array.isArray(a.gaps) ? a.gaps : []) {
|
|
705
|
+
if (typeof reason === "string" && GAP_REASONS.has(reason)) {
|
|
706
|
+
gaps.push([index, attempt, reason]);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
return {
|
|
712
|
+
...facts,
|
|
713
|
+
...(remainders.size === 0 ? {} : { remainders: [...remainders.values()] }),
|
|
714
|
+
...(gaps.length === 0 ? {} : { gaps }),
|
|
715
|
+
};
|
|
630
716
|
}
|
|
631
717
|
|
|
632
718
|
// Reads findings.json and pipeline.json from a run directory. A missing or
|