@agent-finops/core 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/actionPlanner.d.ts +140 -0
- package/dist/actionPlanner.js +938 -0
- package/dist/actionVerification.d.ts +1240 -0
- package/dist/actionVerification.js +1028 -0
- package/dist/activitySnapshot.d.ts +101 -9
- package/dist/activitySnapshot.js +145 -6
- package/dist/activitySnapshotCache.d.ts +8 -1
- package/dist/activitySnapshotCache.js +103 -7
- package/dist/agentEconomicsReceipt.d.ts +58 -58
- package/dist/analyze.js +3 -1
- package/dist/cutList.js +1 -1
- package/dist/glance.d.ts +30 -2
- package/dist/glance.js +265 -84
- package/dist/index.d.ts +11 -2
- package/dist/index.js +10 -1
- package/dist/insights.js +3 -1
- package/dist/localAgentFormats/gemini.js +2 -2
- package/dist/localAgentFormats/registry.js +6 -2
- package/dist/localAgentFormats/runtimeRegistry.js +5 -2
- package/dist/localAgentFormats/types.d.ts +2 -1
- package/dist/localAgentLogs.d.ts +362 -3
- package/dist/localAgentLogs.js +1964 -165
- package/dist/modelPricing.d.ts +1 -1
- package/dist/modelPricing.js +4 -1
- package/dist/planMath.js +12 -7
- package/dist/projectEconomics.d.ts +617 -0
- package/dist/projectEconomics.js +620 -0
- package/dist/projectEconomicsBuilder.d.ts +89 -0
- package/dist/projectEconomicsBuilder.js +473 -0
- package/dist/projectIndexStore.d.ts +545 -0
- package/dist/projectIndexStore.js +606 -0
- package/dist/providerConnectors.d.ts +59 -1
- package/dist/providerConnectors.js +192 -12
- package/dist/qualitativeIndexCache.d.ts +494 -0
- package/dist/qualitativeIndexCache.js +930 -0
- package/dist/resultCard.d.ts +350 -0
- package/dist/resultCard.js +604 -0
- package/dist/runtimeCommands.d.ts +21 -0
- package/dist/runtimeCommands.js +27 -0
- package/dist/scanGuard.d.ts +3 -1
- package/dist/scanGuard.js +164 -4
- package/dist/schema.d.ts +31 -31
- package/dist/sessionVitals.d.ts +145 -0
- package/dist/sessionVitals.js +521 -0
- package/dist/sourceRegistry.js +90 -52
- package/dist/toolInvocations.d.ts +40 -1
- package/dist/toolInvocations.js +101 -20
- package/package.json +1 -1
|
@@ -0,0 +1,1028 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { sourceValidationCoverageValues } from "./sourceStatus.js";
|
|
4
|
+
export const WASTE_FINDING_V0_KIND = "aibill.waste_finding";
|
|
5
|
+
export const WASTE_FINDING_V0_VERSION = "0.1.0";
|
|
6
|
+
export const TOKEN_REDUCTION_EXPERIMENT_V0_KIND = "aibill.token_reduction_experiment";
|
|
7
|
+
export const TOKEN_REDUCTION_EXPERIMENT_V0_VERSION = "0.1.0";
|
|
8
|
+
export const MAX_WASTE_FINDING_EVIDENCE_REFS_V0 = 256;
|
|
9
|
+
export const MAX_TOKEN_EXPERIMENT_SESSIONS_PER_PHASE_V0 = 256;
|
|
10
|
+
export const wasteFindingTypeValues = [
|
|
11
|
+
"configured_not_observed",
|
|
12
|
+
"repeated_context_read",
|
|
13
|
+
"compaction_pressure",
|
|
14
|
+
"high_context_relative_to_baseline",
|
|
15
|
+
"cumulative_context_exposure"
|
|
16
|
+
];
|
|
17
|
+
export const wasteCandidateActionValues = [
|
|
18
|
+
"inspect_scope",
|
|
19
|
+
"lazy_load",
|
|
20
|
+
"disable",
|
|
21
|
+
"remove",
|
|
22
|
+
"start_fresh",
|
|
23
|
+
"reduce_repeated_reads",
|
|
24
|
+
"trim_context"
|
|
25
|
+
];
|
|
26
|
+
export const wasteFindingCaveatValues = [
|
|
27
|
+
"signal_not_cause",
|
|
28
|
+
"no_cash_claim",
|
|
29
|
+
"missing_outcome_evidence"
|
|
30
|
+
];
|
|
31
|
+
export const actionVerificationEvidenceValues = [
|
|
32
|
+
"verified",
|
|
33
|
+
"observed",
|
|
34
|
+
"calculated",
|
|
35
|
+
"user_declared",
|
|
36
|
+
"modeled",
|
|
37
|
+
"missing"
|
|
38
|
+
];
|
|
39
|
+
export const actionVerificationEvidenceSchema = z.enum(actionVerificationEvidenceValues);
|
|
40
|
+
export const tokenExperimentLifecycleValues = [
|
|
41
|
+
"draft",
|
|
42
|
+
"baseline_ready",
|
|
43
|
+
"applied",
|
|
44
|
+
"collecting",
|
|
45
|
+
"complete",
|
|
46
|
+
"rolled_back",
|
|
47
|
+
"invalidated"
|
|
48
|
+
];
|
|
49
|
+
export const tokenExperimentResultValues = [
|
|
50
|
+
"not_evaluated",
|
|
51
|
+
"collecting",
|
|
52
|
+
"measured_token_reduction",
|
|
53
|
+
"no_measured_change",
|
|
54
|
+
"regressed",
|
|
55
|
+
"inconclusive"
|
|
56
|
+
];
|
|
57
|
+
export const tokenExperimentExclusionReasonValues = [
|
|
58
|
+
"duplicate_session",
|
|
59
|
+
"reused_across_phases",
|
|
60
|
+
"intervention_not_applied",
|
|
61
|
+
"crosses_intervention_boundary",
|
|
62
|
+
"wrong_side_of_intervention",
|
|
63
|
+
"missing_total_tokens",
|
|
64
|
+
"inconsistent_total_tokens",
|
|
65
|
+
"agent_mismatch",
|
|
66
|
+
"provider_mismatch",
|
|
67
|
+
"model_mismatch",
|
|
68
|
+
"project_mismatch",
|
|
69
|
+
"session_type_mismatch",
|
|
70
|
+
"work_type_mismatch",
|
|
71
|
+
"missing_work_unit",
|
|
72
|
+
"source_version_mismatch"
|
|
73
|
+
];
|
|
74
|
+
const finiteNonnegativeSchema = z.number().finite().nonnegative();
|
|
75
|
+
const tokenCountSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
|
|
76
|
+
const nullableTokenCountSchema = tokenCountSchema.nullable();
|
|
77
|
+
const utcTimestampSchema = z.string()
|
|
78
|
+
.datetime({ offset: true })
|
|
79
|
+
.transform((value) => new Date(value).toISOString());
|
|
80
|
+
const safeIdentifierSchema = z.string()
|
|
81
|
+
.min(1)
|
|
82
|
+
.max(128)
|
|
83
|
+
.regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/, "Expected a path-free, control-free identifier.")
|
|
84
|
+
.superRefine((value, context) => {
|
|
85
|
+
if (/^(?:sk-|sk_|gh[pousr]_|github_pat_|npm_|AIza|xox[baprs]-|glpat-|AKIA)/i.test(value) ||
|
|
86
|
+
/^(?:env|keychain|secret|credential):/i.test(value)) {
|
|
87
|
+
context.addIssue({
|
|
88
|
+
code: "custom",
|
|
89
|
+
message: "Credential-like values are not action-verification identifiers."
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
const actionVerificationReferenceSchema = z.string()
|
|
94
|
+
.regex(/^avref_[a-f0-9]{64}$/);
|
|
95
|
+
const wasteFindingIdSchema = z.string().regex(/^wf_v0_[a-f0-9]{64}$/);
|
|
96
|
+
const wasteCandidateKeySchema = z.string().regex(/^wfc_v0_[a-f0-9]{64}$/);
|
|
97
|
+
const tokenExperimentIdSchema = z.string().regex(/^tre_v0_[a-f0-9]{64}$/);
|
|
98
|
+
const tokenExperimentRevisionIdSchema = z.string().regex(/^trev_v0_[a-f0-9]{64}$/);
|
|
99
|
+
const timeWindowSchema = z.object({
|
|
100
|
+
start: utcTimestampSchema,
|
|
101
|
+
end: utcTimestampSchema
|
|
102
|
+
}).strict().superRefine((window, context) => {
|
|
103
|
+
if (Date.parse(window.start) >= Date.parse(window.end)) {
|
|
104
|
+
context.addIssue({
|
|
105
|
+
code: "custom",
|
|
106
|
+
path: ["end"],
|
|
107
|
+
message: "The window end must follow its start."
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
const findingScopeSchema = z.object({
|
|
112
|
+
agent: safeIdentifierSchema,
|
|
113
|
+
provider: safeIdentifierSchema,
|
|
114
|
+
model: safeIdentifierSchema.optional(),
|
|
115
|
+
projectRef: actionVerificationReferenceSchema.optional()
|
|
116
|
+
}).strict();
|
|
117
|
+
const findingMetricSchema = z.object({
|
|
118
|
+
name: z.enum([
|
|
119
|
+
"input_context_tokens",
|
|
120
|
+
"total_tokens",
|
|
121
|
+
"compaction_events",
|
|
122
|
+
"repeated_read_events",
|
|
123
|
+
"configured_items"
|
|
124
|
+
]),
|
|
125
|
+
unit: z.enum(["tokens", "events", "items", "ratio"]),
|
|
126
|
+
value: finiteNonnegativeSchema.nullable(),
|
|
127
|
+
sampleCount: z.number().int().nonnegative(),
|
|
128
|
+
evidence: z.enum(["observed", "calculated", "missing"])
|
|
129
|
+
}).strict().superRefine((metric, context) => {
|
|
130
|
+
if ((metric.value === null) !== (metric.evidence === "missing")) {
|
|
131
|
+
context.addIssue({
|
|
132
|
+
code: "custom",
|
|
133
|
+
path: ["value"],
|
|
134
|
+
message: "A missing finding metric must use null, and a present metric must not."
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
const wasteFindingBodySchema = z.object({
|
|
139
|
+
kind: z.literal(WASTE_FINDING_V0_KIND),
|
|
140
|
+
schemaVersion: z.literal(WASTE_FINDING_V0_VERSION),
|
|
141
|
+
generatedAt: utcTimestampSchema,
|
|
142
|
+
window: timeWindowSchema,
|
|
143
|
+
findingType: z.enum(wasteFindingTypeValues),
|
|
144
|
+
objective: z.object({
|
|
145
|
+
metric: z.literal("total_tokens_per_matched_session"),
|
|
146
|
+
direction: z.literal("reduce"),
|
|
147
|
+
guard: z.literal("user_declared_quality_must_hold")
|
|
148
|
+
}).strict(),
|
|
149
|
+
caveats: z.array(z.enum(wasteFindingCaveatValues)).length(wasteFindingCaveatValues.length),
|
|
150
|
+
candidateAction: z.object({
|
|
151
|
+
kind: z.enum(wasteCandidateActionValues),
|
|
152
|
+
provider: safeIdentifierSchema,
|
|
153
|
+
surface: z.enum([
|
|
154
|
+
"local_agent_configuration",
|
|
155
|
+
"session_workflow",
|
|
156
|
+
"provider_workload_configuration"
|
|
157
|
+
]),
|
|
158
|
+
reversible: z.literal(true),
|
|
159
|
+
canaryRequired: z.literal(true),
|
|
160
|
+
rollbackRequired: z.literal(true)
|
|
161
|
+
}).strict(),
|
|
162
|
+
target: z.object({
|
|
163
|
+
kind: z.enum(["session", "repeated_read_file", "configured_item"]),
|
|
164
|
+
ref: actionVerificationReferenceSchema
|
|
165
|
+
}).strict(),
|
|
166
|
+
scope: findingScopeSchema,
|
|
167
|
+
source: z.object({
|
|
168
|
+
id: safeIdentifierSchema,
|
|
169
|
+
validationCoverage: z.enum(sourceValidationCoverageValues),
|
|
170
|
+
freshness: z.enum(["fresh", "stale", "not_checked"])
|
|
171
|
+
}).strict(),
|
|
172
|
+
metric: findingMetricSchema,
|
|
173
|
+
evidenceRefs: z.array(actionVerificationReferenceSchema)
|
|
174
|
+
.min(1)
|
|
175
|
+
.max(MAX_WASTE_FINDING_EVIDENCE_REFS_V0),
|
|
176
|
+
causalStatus: z.literal("unproven"),
|
|
177
|
+
actionability: z.literal("inspect_only"),
|
|
178
|
+
approvalRequired: z.literal(true)
|
|
179
|
+
}).strict().superRefine((finding, context) => {
|
|
180
|
+
if (Date.parse(finding.generatedAt) < Date.parse(finding.window.end)) {
|
|
181
|
+
context.addIssue({
|
|
182
|
+
code: "custom",
|
|
183
|
+
path: ["generatedAt"],
|
|
184
|
+
message: "A finding cannot be generated before its evidence window ends."
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
if (finding.candidateAction.provider !== finding.scope.provider) {
|
|
188
|
+
context.addIssue({
|
|
189
|
+
code: "custom",
|
|
190
|
+
path: ["candidateAction", "provider"],
|
|
191
|
+
message: "The action provider must remain inside the finding's provider scope."
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
const caveats = new Set(finding.caveats);
|
|
195
|
+
for (const caveat of wasteFindingCaveatValues) {
|
|
196
|
+
if (!caveats.has(caveat)) {
|
|
197
|
+
context.addIssue({
|
|
198
|
+
code: "custom",
|
|
199
|
+
path: ["caveats"],
|
|
200
|
+
message: `WasteFindingV0 requires the ${caveat} caveat.`
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
const wasteFindingObjectSchema = wasteFindingBodySchema.extend({
|
|
206
|
+
id: wasteFindingIdSchema,
|
|
207
|
+
candidateKey: wasteCandidateKeySchema
|
|
208
|
+
});
|
|
209
|
+
const tokenComponentsSchema = z.object({
|
|
210
|
+
uncachedInputTokens: nullableTokenCountSchema,
|
|
211
|
+
cacheReadTokens: nullableTokenCountSchema,
|
|
212
|
+
cacheWriteTokens: nullableTokenCountSchema,
|
|
213
|
+
toolTokens: nullableTokenCountSchema,
|
|
214
|
+
outputTokens: nullableTokenCountSchema,
|
|
215
|
+
thoughtTokens: nullableTokenCountSchema,
|
|
216
|
+
calculatedTotalTokens: nullableTokenCountSchema,
|
|
217
|
+
reportedTotalTokens: nullableTokenCountSchema,
|
|
218
|
+
componentEvidence: z.object({
|
|
219
|
+
uncachedInputTokens: z.enum(["observed", "not_separately_reported"]),
|
|
220
|
+
cacheReadTokens: z.enum(["observed", "not_separately_reported"]),
|
|
221
|
+
cacheWriteTokens: z.enum(["observed", "partial", "not_separately_reported"]),
|
|
222
|
+
toolTokens: z.enum(["observed", "not_separately_reported"]),
|
|
223
|
+
outputTokens: z.enum(["observed", "not_separately_reported"]),
|
|
224
|
+
thoughtTokens: z.enum(["observed", "not_separately_reported"]),
|
|
225
|
+
calculatedTotalTokens: z.enum([
|
|
226
|
+
"calculated_complete",
|
|
227
|
+
"calculated_partial",
|
|
228
|
+
"missing"
|
|
229
|
+
]),
|
|
230
|
+
reportedTotalTokens: z.enum(["provider_reported", "not_reported"])
|
|
231
|
+
}).strict()
|
|
232
|
+
}).strict().superRefine((tokens, context) => {
|
|
233
|
+
const requireObservedValue = (key, evidence) => {
|
|
234
|
+
const present = tokens[key] !== null;
|
|
235
|
+
if ((evidence === "observed") !== present) {
|
|
236
|
+
context.addIssue({
|
|
237
|
+
code: "custom",
|
|
238
|
+
path: [key],
|
|
239
|
+
message: `${key} must be present exactly when its component evidence is observed.`
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
};
|
|
243
|
+
requireObservedValue("uncachedInputTokens", tokens.componentEvidence.uncachedInputTokens);
|
|
244
|
+
requireObservedValue("cacheReadTokens", tokens.componentEvidence.cacheReadTokens);
|
|
245
|
+
if (tokens.componentEvidence.cacheWriteTokens !== "partial") {
|
|
246
|
+
requireObservedValue("cacheWriteTokens", tokens.componentEvidence.cacheWriteTokens);
|
|
247
|
+
}
|
|
248
|
+
requireObservedValue("toolTokens", tokens.componentEvidence.toolTokens);
|
|
249
|
+
requireObservedValue("outputTokens", tokens.componentEvidence.outputTokens);
|
|
250
|
+
requireObservedValue("thoughtTokens", tokens.componentEvidence.thoughtTokens);
|
|
251
|
+
if ((tokens.calculatedTotalTokens !== null) !==
|
|
252
|
+
(tokens.componentEvidence.calculatedTotalTokens !== "missing")) {
|
|
253
|
+
context.addIssue({
|
|
254
|
+
code: "custom",
|
|
255
|
+
path: ["calculatedTotalTokens"],
|
|
256
|
+
message: "Calculated component evidence must match presence of the calculated total."
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
if ((tokens.reportedTotalTokens !== null) !==
|
|
260
|
+
(tokens.componentEvidence.reportedTotalTokens === "provider_reported")) {
|
|
261
|
+
context.addIssue({
|
|
262
|
+
code: "custom",
|
|
263
|
+
path: ["reportedTotalTokens"],
|
|
264
|
+
message: "Provider-reported total evidence must match presence of the reported total."
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
});
|
|
268
|
+
const qualityObservationSchema = z.object({
|
|
269
|
+
status: z.enum(["passed", "failed", "missing"]),
|
|
270
|
+
evidence: z.enum(["verified", "observed", "user_declared", "missing"]),
|
|
271
|
+
evidenceRef: actionVerificationReferenceSchema.optional()
|
|
272
|
+
}).strict().superRefine((quality, context) => {
|
|
273
|
+
if ((quality.status === "missing") !== (quality.evidence === "missing")) {
|
|
274
|
+
context.addIssue({
|
|
275
|
+
code: "custom",
|
|
276
|
+
path: ["evidence"],
|
|
277
|
+
message: "Missing quality must use missing evidence, and present quality must not."
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
if ((quality.evidence === "observed" || quality.evidence === "verified") &&
|
|
281
|
+
!quality.evidenceRef) {
|
|
282
|
+
context.addIssue({
|
|
283
|
+
code: "custom",
|
|
284
|
+
path: ["evidenceRef"],
|
|
285
|
+
message: "Observed and verified quality require an opaque evidence reference."
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
const tokenSessionSchema = z.object({
|
|
290
|
+
sessionRef: actionVerificationReferenceSchema,
|
|
291
|
+
startedAt: utcTimestampSchema,
|
|
292
|
+
endedAt: utcTimestampSchema,
|
|
293
|
+
agent: safeIdentifierSchema,
|
|
294
|
+
provider: safeIdentifierSchema,
|
|
295
|
+
model: safeIdentifierSchema,
|
|
296
|
+
projectRef: actionVerificationReferenceSchema.optional(),
|
|
297
|
+
sessionType: z.enum(["parent", "subagent", "unknown"]),
|
|
298
|
+
workTypeRef: actionVerificationReferenceSchema.optional(),
|
|
299
|
+
workUnitRef: actionVerificationReferenceSchema.optional(),
|
|
300
|
+
workUnitEvidence: z.enum(["verified", "observed", "user_declared"]).optional(),
|
|
301
|
+
sourceVersionRef: actionVerificationReferenceSchema.optional(),
|
|
302
|
+
sourceValidationCoverage: z.enum(sourceValidationCoverageValues),
|
|
303
|
+
tokens: tokenComponentsSchema,
|
|
304
|
+
quality: qualityObservationSchema
|
|
305
|
+
}).strict().superRefine((session, context) => {
|
|
306
|
+
if (Date.parse(session.startedAt) > Date.parse(session.endedAt)) {
|
|
307
|
+
context.addIssue({
|
|
308
|
+
code: "custom",
|
|
309
|
+
path: ["endedAt"],
|
|
310
|
+
message: "A session cannot end before it starts."
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
if (Boolean(session.workUnitRef) !== Boolean(session.workUnitEvidence)) {
|
|
314
|
+
context.addIssue({
|
|
315
|
+
code: "custom",
|
|
316
|
+
path: ["workUnitEvidence"],
|
|
317
|
+
message: "A work-unit reference and its evidence label must be present together."
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
});
|
|
321
|
+
const cohortSchema = z.object({
|
|
322
|
+
agent: safeIdentifierSchema,
|
|
323
|
+
provider: safeIdentifierSchema,
|
|
324
|
+
model: safeIdentifierSchema,
|
|
325
|
+
projectRef: actionVerificationReferenceSchema,
|
|
326
|
+
sessionType: z.enum(["parent", "subagent"]),
|
|
327
|
+
workTypeRef: actionVerificationReferenceSchema,
|
|
328
|
+
workTypeEvidence: z.enum(["verified", "observed", "user_declared"]),
|
|
329
|
+
sourceVersionRef: actionVerificationReferenceSchema.optional()
|
|
330
|
+
}).strict();
|
|
331
|
+
const matchingPolicySchema = z.object({
|
|
332
|
+
basis: z.enum(["session_cohort", "accepted_work_unit"]),
|
|
333
|
+
minimumBaselineSessions: z.number().int().min(3).max(100).default(3),
|
|
334
|
+
minimumPostSessions: z.number().int().min(3).max(100).default(3),
|
|
335
|
+
requireExactSourceVersion: z.boolean().default(true)
|
|
336
|
+
}).strict();
|
|
337
|
+
const qualityGuardSchema = z.object({
|
|
338
|
+
required: z.literal(true),
|
|
339
|
+
minimumEvidence: z.enum(["user_declared", "observed", "verified"]),
|
|
340
|
+
rollbackOnRegression: z.literal(true)
|
|
341
|
+
}).strict();
|
|
342
|
+
const interventionSchema = z.object({
|
|
343
|
+
approval: z.object({
|
|
344
|
+
status: z.enum(["pending", "explicit"]),
|
|
345
|
+
evidence: z.enum(["missing", "user_declared", "verified"]),
|
|
346
|
+
approvedAt: utcTimestampSchema.optional(),
|
|
347
|
+
approvalRef: actionVerificationReferenceSchema.optional()
|
|
348
|
+
}).strict(),
|
|
349
|
+
appliedAt: utcTimestampSchema.optional(),
|
|
350
|
+
changeRef: actionVerificationReferenceSchema.optional(),
|
|
351
|
+
rollbackRef: actionVerificationReferenceSchema.optional(),
|
|
352
|
+
canary: qualityObservationSchema.optional(),
|
|
353
|
+
rolledBackAt: utcTimestampSchema.optional()
|
|
354
|
+
}).strict().superRefine((intervention, context) => {
|
|
355
|
+
const explicit = intervention.approval.status === "explicit";
|
|
356
|
+
if (explicit !== (intervention.approval.evidence !== "missing")) {
|
|
357
|
+
context.addIssue({
|
|
358
|
+
code: "custom",
|
|
359
|
+
path: ["approval", "evidence"],
|
|
360
|
+
message: "Explicit approval requires evidence; pending approval must remain missing."
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
if (explicit !== Boolean(intervention.approval.approvedAt)) {
|
|
364
|
+
context.addIssue({
|
|
365
|
+
code: "custom",
|
|
366
|
+
path: ["approval", "approvedAt"],
|
|
367
|
+
message: "Explicit approval requires an approval timestamp."
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
if (intervention.approval.evidence === "verified" &&
|
|
371
|
+
!intervention.approval.approvalRef) {
|
|
372
|
+
context.addIssue({
|
|
373
|
+
code: "custom",
|
|
374
|
+
path: ["approval", "approvalRef"],
|
|
375
|
+
message: "Verified approval requires an opaque approval reference."
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
if (intervention.appliedAt) {
|
|
379
|
+
if (!explicit || !intervention.changeRef || !intervention.rollbackRef ||
|
|
380
|
+
!intervention.canary || intervention.canary.status === "missing" ||
|
|
381
|
+
!intervention.canary.evidenceRef) {
|
|
382
|
+
context.addIssue({
|
|
383
|
+
code: "custom",
|
|
384
|
+
path: ["appliedAt"],
|
|
385
|
+
message: "An applied intervention requires explicit approval plus opaque change, rollback, and canary evidence references."
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
else if (intervention.changeRef || intervention.rollbackRef || intervention.canary ||
|
|
390
|
+
intervention.rolledBackAt) {
|
|
391
|
+
context.addIssue({
|
|
392
|
+
code: "custom",
|
|
393
|
+
path: ["appliedAt"],
|
|
394
|
+
message: "Change, canary, and rollback evidence require an applied intervention."
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
if (intervention.appliedAt && intervention.approval.approvedAt &&
|
|
398
|
+
Date.parse(intervention.appliedAt) <= Date.parse(intervention.approval.approvedAt)) {
|
|
399
|
+
context.addIssue({
|
|
400
|
+
code: "custom",
|
|
401
|
+
path: ["appliedAt"],
|
|
402
|
+
message: "An intervention must be applied after approval."
|
|
403
|
+
});
|
|
404
|
+
}
|
|
405
|
+
if (intervention.rolledBackAt && intervention.appliedAt &&
|
|
406
|
+
Date.parse(intervention.rolledBackAt) <= Date.parse(intervention.appliedAt)) {
|
|
407
|
+
context.addIssue({
|
|
408
|
+
code: "custom",
|
|
409
|
+
path: ["rolledBackAt"],
|
|
410
|
+
message: "Rollback must follow application."
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
});
|
|
414
|
+
const invalidationSchema = z.object({
|
|
415
|
+
reason: z.enum([
|
|
416
|
+
"scope_changed",
|
|
417
|
+
"source_semantics_changed",
|
|
418
|
+
"concurrent_change",
|
|
419
|
+
"manual"
|
|
420
|
+
]),
|
|
421
|
+
invalidatedAt: utcTimestampSchema
|
|
422
|
+
}).strict();
|
|
423
|
+
const experimentBodySchema = z.object({
|
|
424
|
+
kind: z.literal(TOKEN_REDUCTION_EXPERIMENT_V0_KIND),
|
|
425
|
+
schemaVersion: z.literal(TOKEN_REDUCTION_EXPERIMENT_V0_VERSION),
|
|
426
|
+
createdAt: utcTimestampSchema,
|
|
427
|
+
finding: wasteFindingObjectSchema,
|
|
428
|
+
cohort: cohortSchema,
|
|
429
|
+
matchingPolicy: matchingPolicySchema,
|
|
430
|
+
qualityGuard: qualityGuardSchema,
|
|
431
|
+
baselineSessions: z.array(tokenSessionSchema)
|
|
432
|
+
.max(MAX_TOKEN_EXPERIMENT_SESSIONS_PER_PHASE_V0),
|
|
433
|
+
intervention: interventionSchema,
|
|
434
|
+
postSessions: z.array(tokenSessionSchema)
|
|
435
|
+
.max(MAX_TOKEN_EXPERIMENT_SESSIONS_PER_PHASE_V0),
|
|
436
|
+
invalidation: invalidationSchema.optional()
|
|
437
|
+
}).strict().superRefine((experiment, context) => {
|
|
438
|
+
if (Date.parse(experiment.createdAt) < Date.parse(experiment.finding.generatedAt)) {
|
|
439
|
+
context.addIssue({
|
|
440
|
+
code: "custom",
|
|
441
|
+
path: ["createdAt"],
|
|
442
|
+
message: "An experiment cannot be created before its finding."
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
for (const [index, session] of experiment.baselineSessions.entries()) {
|
|
446
|
+
if (Date.parse(session.endedAt) > Date.parse(experiment.createdAt)) {
|
|
447
|
+
context.addIssue({
|
|
448
|
+
code: "custom",
|
|
449
|
+
path: ["baselineSessions", index, "endedAt"],
|
|
450
|
+
message: "Frozen baseline evidence cannot end after experiment creation."
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
if (experiment.finding.scope.agent !== experiment.cohort.agent ||
|
|
455
|
+
experiment.finding.scope.provider !== experiment.cohort.provider) {
|
|
456
|
+
context.addIssue({
|
|
457
|
+
code: "custom",
|
|
458
|
+
path: ["cohort"],
|
|
459
|
+
message: "The experiment cohort must remain inside the finding's agent/provider scope."
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
if (experiment.finding.scope.model &&
|
|
463
|
+
experiment.finding.scope.model !== experiment.cohort.model) {
|
|
464
|
+
context.addIssue({
|
|
465
|
+
code: "custom",
|
|
466
|
+
path: ["cohort", "model"],
|
|
467
|
+
message: "The experiment model must match the finding model."
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
if (experiment.finding.scope.projectRef &&
|
|
471
|
+
experiment.finding.scope.projectRef !== experiment.cohort.projectRef) {
|
|
472
|
+
context.addIssue({
|
|
473
|
+
code: "custom",
|
|
474
|
+
path: ["cohort", "projectRef"],
|
|
475
|
+
message: "The experiment project must match the finding project."
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
if (experiment.matchingPolicy.requireExactSourceVersion !==
|
|
479
|
+
Boolean(experiment.cohort.sourceVersionRef)) {
|
|
480
|
+
context.addIssue({
|
|
481
|
+
code: "custom",
|
|
482
|
+
path: ["matchingPolicy", "requireExactSourceVersion"],
|
|
483
|
+
message: "Exact source-version matching requires observed cohort version evidence; missing versions must remain explicitly unmatched."
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
if (experiment.intervention.appliedAt &&
|
|
487
|
+
Date.parse(experiment.intervention.appliedAt) < Date.parse(experiment.createdAt)) {
|
|
488
|
+
context.addIssue({
|
|
489
|
+
code: "custom",
|
|
490
|
+
path: ["intervention", "appliedAt"],
|
|
491
|
+
message: "An intervention cannot be applied before its experiment exists."
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
if (experiment.intervention.approval.approvedAt &&
|
|
495
|
+
Date.parse(experiment.intervention.approval.approvedAt) <
|
|
496
|
+
Date.parse(experiment.createdAt)) {
|
|
497
|
+
context.addIssue({
|
|
498
|
+
code: "custom",
|
|
499
|
+
path: ["intervention", "approval", "approvedAt"],
|
|
500
|
+
message: "A token-test plan cannot be approved before its experiment exists."
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
if (experiment.invalidation &&
|
|
504
|
+
Date.parse(experiment.invalidation.invalidatedAt) < Date.parse(experiment.createdAt)) {
|
|
505
|
+
context.addIssue({
|
|
506
|
+
code: "custom",
|
|
507
|
+
path: ["invalidation", "invalidatedAt"],
|
|
508
|
+
message: "Invalidation cannot precede experiment creation."
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
if (experiment.invalidation && experiment.intervention.appliedAt) {
|
|
512
|
+
context.addIssue({
|
|
513
|
+
code: "custom",
|
|
514
|
+
path: ["invalidation"],
|
|
515
|
+
message: "An applied intervention cannot be invalidated; it must be rolled back."
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
if (experiment.intervention.rolledBackAt) {
|
|
519
|
+
const rolledBackAt = Date.parse(experiment.intervention.rolledBackAt);
|
|
520
|
+
for (const [index, session] of experiment.postSessions.entries()) {
|
|
521
|
+
if (Date.parse(session.endedAt) > rolledBackAt) {
|
|
522
|
+
context.addIssue({
|
|
523
|
+
code: "custom",
|
|
524
|
+
path: ["postSessions", index, "endedAt"],
|
|
525
|
+
message: "Post-change evidence cannot end after the rollback boundary."
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
if (experiment.intervention.canary?.status === "failed" &&
|
|
531
|
+
experiment.postSessions.length > 0) {
|
|
532
|
+
context.addIssue({
|
|
533
|
+
code: "custom",
|
|
534
|
+
path: ["postSessions"],
|
|
535
|
+
message: "A failed canary cannot collect or expose post-change token evidence."
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
});
|
|
539
|
+
const exclusionSchema = z.object({
|
|
540
|
+
phase: z.enum(["baseline", "post_change"]),
|
|
541
|
+
sessionRef: actionVerificationReferenceSchema,
|
|
542
|
+
reasons: z.array(z.enum(tokenExperimentExclusionReasonValues)).min(1)
|
|
543
|
+
}).strict();
|
|
544
|
+
const componentMedianSchema = z.object({
|
|
545
|
+
uncachedInputTokens: finiteNonnegativeSchema.nullable(),
|
|
546
|
+
cacheReadTokens: finiteNonnegativeSchema.nullable(),
|
|
547
|
+
cacheWriteTokens: finiteNonnegativeSchema.nullable(),
|
|
548
|
+
toolTokens: finiteNonnegativeSchema.nullable(),
|
|
549
|
+
outputTokens: finiteNonnegativeSchema.nullable(),
|
|
550
|
+
thoughtTokens: finiteNonnegativeSchema.nullable()
|
|
551
|
+
}).strict();
|
|
552
|
+
const evaluationSchema = z.object({
|
|
553
|
+
status: z.enum(tokenExperimentResultValues),
|
|
554
|
+
metricEvidence: z.enum(["calculated", "missing"]),
|
|
555
|
+
matchingEvidence: z.enum(["verified", "observed", "user_declared", "missing"]),
|
|
556
|
+
qualityStatus: z.enum(["held", "regressed", "insufficient"]),
|
|
557
|
+
qualityEvidence: z.enum(["verified", "observed", "user_declared", "missing"]),
|
|
558
|
+
baseline: z.object({
|
|
559
|
+
includedSessions: z.number().int().nonnegative(),
|
|
560
|
+
medianTotalTokens: finiteNonnegativeSchema.nullable(),
|
|
561
|
+
componentMedians: componentMedianSchema
|
|
562
|
+
}).strict(),
|
|
563
|
+
postChange: z.object({
|
|
564
|
+
includedSessions: z.number().int().nonnegative(),
|
|
565
|
+
medianTotalTokens: finiteNonnegativeSchema.nullable(),
|
|
566
|
+
componentMedians: componentMedianSchema
|
|
567
|
+
}).strict(),
|
|
568
|
+
reductionPercent: z.number().finite().nullable(),
|
|
569
|
+
exclusions: z.array(exclusionSchema).max(512),
|
|
570
|
+
rollbackRecommended: z.boolean()
|
|
571
|
+
}).strict();
|
|
572
|
+
const experimentObjectSchema = experimentBodySchema.extend({
|
|
573
|
+
id: tokenExperimentIdSchema,
|
|
574
|
+
revisionId: tokenExperimentRevisionIdSchema,
|
|
575
|
+
lifecycle: z.enum(tokenExperimentLifecycleValues),
|
|
576
|
+
evaluation: evaluationSchema
|
|
577
|
+
});
|
|
578
|
+
/**
|
|
579
|
+
* Hash a source-native value into the only reference form accepted by action
|
|
580
|
+
* verification contracts. The original value is never returned or persisted.
|
|
581
|
+
*/
|
|
582
|
+
export function createActionVerificationReference(namespace, sourceNativeValue) {
|
|
583
|
+
const safeNamespace = safeIdentifierSchema.parse(namespace);
|
|
584
|
+
if (sourceNativeValue.length < 1 || sourceNativeValue.length > 4_096 ||
|
|
585
|
+
hasUnpairedSurrogate(sourceNativeValue)) {
|
|
586
|
+
throw new TypeError("Action verification references require a bounded valid Unicode source value.");
|
|
587
|
+
}
|
|
588
|
+
return `avref_${createHash("sha256")
|
|
589
|
+
.update(`${safeNamespace}\u0000${sourceNativeValue}`)
|
|
590
|
+
.digest("hex")}`;
|
|
591
|
+
}
|
|
592
|
+
/** Create a canonical, content-addressed waste finding with no raw paths or prose. */
|
|
593
|
+
export function createWasteFindingV0(input) {
|
|
594
|
+
const body = canonicalWasteFindingBody(wasteFindingBodySchema.parse(input));
|
|
595
|
+
return wasteFindingObjectSchema.parse({
|
|
596
|
+
...body,
|
|
597
|
+
id: wasteFindingDigest(body),
|
|
598
|
+
candidateKey: wasteCandidateDigest(body)
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
/** Parse a serialized finding and reject a stale or forged content digest. */
|
|
602
|
+
export function parseWasteFindingV0(value) {
|
|
603
|
+
const finding = wasteFindingObjectSchema.parse(value);
|
|
604
|
+
const { id, candidateKey, ...bodyInput } = finding;
|
|
605
|
+
const body = canonicalWasteFindingBody(wasteFindingBodySchema.parse(bodyInput));
|
|
606
|
+
if (id !== wasteFindingDigest(body) || candidateKey !== wasteCandidateDigest(body)) {
|
|
607
|
+
throw new TypeError("Waste finding ID or candidate key does not match its canonical body.");
|
|
608
|
+
}
|
|
609
|
+
return wasteFindingObjectSchema.parse({ ...body, id, candidateKey });
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Build or advance one experiment. Evaluation is pure: every lifecycle label,
|
|
613
|
+
* median, exclusion, percentage, and rollback recommendation is derived from
|
|
614
|
+
* the supplied immutable evidence.
|
|
615
|
+
*/
|
|
616
|
+
export function createTokenReductionExperimentV0(input) {
|
|
617
|
+
const body = canonicalExperimentBody(experimentBodySchema.parse(input));
|
|
618
|
+
return experimentObjectSchema.parse({
|
|
619
|
+
...body,
|
|
620
|
+
id: experimentDigest(body),
|
|
621
|
+
revisionId: experimentRevisionDigest(body),
|
|
622
|
+
...evaluateExperimentBody(body)
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
/** Parse and recompute an experiment, rejecting tampered derived fields. */
|
|
626
|
+
export function parseTokenReductionExperimentV0(value) {
|
|
627
|
+
const experiment = experimentObjectSchema.parse(value);
|
|
628
|
+
const { id, revisionId, lifecycle, evaluation, ...bodyInput } = experiment;
|
|
629
|
+
const body = canonicalExperimentBody(experimentBodySchema.parse(bodyInput));
|
|
630
|
+
const expectedId = experimentDigest(body);
|
|
631
|
+
const expectedRevisionId = experimentRevisionDigest(body);
|
|
632
|
+
const expected = evaluateExperimentBody(body);
|
|
633
|
+
if (id !== expectedId || revisionId !== expectedRevisionId ||
|
|
634
|
+
canonicalJson({ lifecycle, evaluation }) !== canonicalJson(expected)) {
|
|
635
|
+
throw new TypeError("Token reduction experiment identity, revision, or derived evaluation is invalid.");
|
|
636
|
+
}
|
|
637
|
+
return experimentObjectSchema.parse({ ...body, id, revisionId, ...expected });
|
|
638
|
+
}
|
|
639
|
+
/** Pure evaluator for callers that already have a validated experiment body. */
|
|
640
|
+
export function evaluateTokenReductionExperimentV0(input) {
|
|
641
|
+
return evaluateExperimentBody(canonicalExperimentBody(experimentBodySchema.parse(input)));
|
|
642
|
+
}
|
|
643
|
+
function evaluateExperimentBody(experiment) {
|
|
644
|
+
const referenceCounts = sessionReferenceCounts(experiment);
|
|
645
|
+
const baseline = evaluatePhase(experiment, "baseline", referenceCounts);
|
|
646
|
+
const postChange = evaluatePhase(experiment, "post_change", referenceCounts);
|
|
647
|
+
const baselineEnough = baseline.included.length >= experiment.matchingPolicy.minimumBaselineSessions;
|
|
648
|
+
const postEnough = postChange.included.length >= experiment.matchingPolicy.minimumPostSessions;
|
|
649
|
+
const baselineMedian = median(baseline.included.map((entry) => entry.totalTokens));
|
|
650
|
+
const postMedian = median(postChange.included.map((entry) => entry.totalTokens));
|
|
651
|
+
const calculatedReductionPercent = baselineEnough && postEnough && baselineMedian !== null &&
|
|
652
|
+
postMedian !== null && baselineMedian > 0
|
|
653
|
+
? roundPercent(100 * (baselineMedian - postMedian) / baselineMedian)
|
|
654
|
+
: null;
|
|
655
|
+
const quality = evaluateQuality(experiment, baseline.included, postChange.included);
|
|
656
|
+
const matchingEvidence = evaluateMatchingEvidence(experiment, baseline.included, postChange.included);
|
|
657
|
+
let lifecycle;
|
|
658
|
+
if (experiment.invalidation)
|
|
659
|
+
lifecycle = "invalidated";
|
|
660
|
+
else if (experiment.intervention.rolledBackAt)
|
|
661
|
+
lifecycle = "rolled_back";
|
|
662
|
+
else if (!experiment.intervention.appliedAt) {
|
|
663
|
+
lifecycle = baselineEnough ? "baseline_ready" : "draft";
|
|
664
|
+
}
|
|
665
|
+
else if (postChange.included.length === 0)
|
|
666
|
+
lifecycle = "applied";
|
|
667
|
+
else if (!postEnough || quality.status === "insufficient")
|
|
668
|
+
lifecycle = "collecting";
|
|
669
|
+
else
|
|
670
|
+
lifecycle = "complete";
|
|
671
|
+
// The arithmetic is not a public result until comparable post evidence and
|
|
672
|
+
// its required quality guard are both complete. Terminal/cancelled attempts
|
|
673
|
+
// retain their audit evidence without continuing to expose a reduction.
|
|
674
|
+
const reductionPercent = lifecycle === "complete" && quality.status === "held"
|
|
675
|
+
? calculatedReductionPercent
|
|
676
|
+
: null;
|
|
677
|
+
let status;
|
|
678
|
+
if (lifecycle === "draft" || lifecycle === "baseline_ready")
|
|
679
|
+
status = "not_evaluated";
|
|
680
|
+
else if (experiment.intervention.canary?.status === "failed" ||
|
|
681
|
+
lifecycle === "invalidated" || lifecycle === "rolled_back") {
|
|
682
|
+
status = "inconclusive";
|
|
683
|
+
}
|
|
684
|
+
else if (!postEnough)
|
|
685
|
+
status = "collecting";
|
|
686
|
+
else if (calculatedReductionPercent === null || quality.status === "insufficient") {
|
|
687
|
+
status = "inconclusive";
|
|
688
|
+
}
|
|
689
|
+
else if (quality.status === "regressed" || calculatedReductionPercent < 0) {
|
|
690
|
+
status = "regressed";
|
|
691
|
+
}
|
|
692
|
+
else if (calculatedReductionPercent === 0)
|
|
693
|
+
status = "no_measured_change";
|
|
694
|
+
else
|
|
695
|
+
status = "measured_token_reduction";
|
|
696
|
+
const exclusions = [...baseline.exclusions, ...postChange.exclusions]
|
|
697
|
+
.sort(compareExclusions);
|
|
698
|
+
return {
|
|
699
|
+
lifecycle,
|
|
700
|
+
evaluation: {
|
|
701
|
+
status,
|
|
702
|
+
metricEvidence: reductionPercent === null ? "missing" : "calculated",
|
|
703
|
+
matchingEvidence,
|
|
704
|
+
qualityStatus: quality.status,
|
|
705
|
+
qualityEvidence: quality.evidence,
|
|
706
|
+
baseline: {
|
|
707
|
+
includedSessions: baseline.included.length,
|
|
708
|
+
medianTotalTokens: baselineMedian,
|
|
709
|
+
componentMedians: componentMedians(baseline.included)
|
|
710
|
+
},
|
|
711
|
+
postChange: {
|
|
712
|
+
includedSessions: postChange.included.length,
|
|
713
|
+
medianTotalTokens: postMedian,
|
|
714
|
+
componentMedians: componentMedians(postChange.included)
|
|
715
|
+
},
|
|
716
|
+
reductionPercent,
|
|
717
|
+
exclusions,
|
|
718
|
+
rollbackRecommended: experiment.intervention.canary?.status === "failed" ||
|
|
719
|
+
quality.status === "regressed" ||
|
|
720
|
+
(calculatedReductionPercent !== null && calculatedReductionPercent < 0)
|
|
721
|
+
}
|
|
722
|
+
};
|
|
723
|
+
}
|
|
724
|
+
function evaluateMatchingEvidence(experiment, baseline, post) {
|
|
725
|
+
if (baseline.length === 0 || post.length === 0)
|
|
726
|
+
return "missing";
|
|
727
|
+
const labels = [
|
|
728
|
+
experiment.cohort.workTypeEvidence
|
|
729
|
+
];
|
|
730
|
+
if (experiment.matchingPolicy.basis === "accepted_work_unit") {
|
|
731
|
+
for (const entry of [...baseline, ...post]) {
|
|
732
|
+
if (!entry.session.workUnitEvidence)
|
|
733
|
+
return "missing";
|
|
734
|
+
labels.push(entry.session.workUnitEvidence);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
else {
|
|
738
|
+
// Agent/project/model/session metadata is locally observed, so a session
|
|
739
|
+
// cohort can never become verified solely from an asserted work-type label.
|
|
740
|
+
labels.push("observed");
|
|
741
|
+
}
|
|
742
|
+
return labels.sort((left, right) => matchingEvidenceRank(left) - matchingEvidenceRank(right))[0];
|
|
743
|
+
}
|
|
744
|
+
function matchingEvidenceRank(evidence) {
|
|
745
|
+
switch (evidence) {
|
|
746
|
+
case "verified": return 3;
|
|
747
|
+
case "observed": return 2;
|
|
748
|
+
case "user_declared": return 1;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
function evaluatePhase(experiment, phase, referenceCounts) {
|
|
752
|
+
const sessions = phase === "baseline"
|
|
753
|
+
? experiment.baselineSessions
|
|
754
|
+
: experiment.postSessions;
|
|
755
|
+
const included = [];
|
|
756
|
+
const exclusions = [];
|
|
757
|
+
for (const session of sessions) {
|
|
758
|
+
const reasons = exclusionReasons(experiment, session, phase, referenceCounts);
|
|
759
|
+
const total = totalTokens(session.tokens);
|
|
760
|
+
if (total.reason)
|
|
761
|
+
reasons.push(total.reason);
|
|
762
|
+
const uniqueReasons = [...new Set(reasons)].sort();
|
|
763
|
+
if (uniqueReasons.length > 0 || total.value === null) {
|
|
764
|
+
exclusions.push({
|
|
765
|
+
phase,
|
|
766
|
+
sessionRef: session.sessionRef,
|
|
767
|
+
reasons: uniqueReasons
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
else {
|
|
771
|
+
included.push({ session, totalTokens: total.value });
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
return { included, exclusions };
|
|
775
|
+
}
|
|
776
|
+
function exclusionReasons(experiment, session, phase, counts) {
|
|
777
|
+
const reasons = [];
|
|
778
|
+
const count = counts.get(session.sessionRef);
|
|
779
|
+
if (count && count.baseline > 0 && count.post > 0)
|
|
780
|
+
reasons.push("reused_across_phases");
|
|
781
|
+
else if ((phase === "baseline" ? count?.baseline : count?.post) !== 1) {
|
|
782
|
+
reasons.push("duplicate_session");
|
|
783
|
+
}
|
|
784
|
+
const cohort = experiment.cohort;
|
|
785
|
+
if (session.agent !== cohort.agent)
|
|
786
|
+
reasons.push("agent_mismatch");
|
|
787
|
+
if (session.provider !== cohort.provider)
|
|
788
|
+
reasons.push("provider_mismatch");
|
|
789
|
+
if (session.model !== cohort.model)
|
|
790
|
+
reasons.push("model_mismatch");
|
|
791
|
+
if (session.projectRef !== cohort.projectRef)
|
|
792
|
+
reasons.push("project_mismatch");
|
|
793
|
+
if (session.sessionType !== cohort.sessionType)
|
|
794
|
+
reasons.push("session_type_mismatch");
|
|
795
|
+
if (session.workTypeRef !== cohort.workTypeRef)
|
|
796
|
+
reasons.push("work_type_mismatch");
|
|
797
|
+
if (experiment.matchingPolicy.basis === "accepted_work_unit" && !session.workUnitRef) {
|
|
798
|
+
reasons.push("missing_work_unit");
|
|
799
|
+
}
|
|
800
|
+
if (!experiment.matchingPolicy.requireExactSourceVersion ||
|
|
801
|
+
!cohort.sourceVersionRef ||
|
|
802
|
+
session.sourceVersionRef !== cohort.sourceVersionRef) {
|
|
803
|
+
reasons.push("source_version_mismatch");
|
|
804
|
+
}
|
|
805
|
+
const appliedAt = experiment.intervention.appliedAt;
|
|
806
|
+
if (phase === "post_change" && !appliedAt) {
|
|
807
|
+
reasons.push("intervention_not_applied");
|
|
808
|
+
}
|
|
809
|
+
else if (appliedAt) {
|
|
810
|
+
const applied = Date.parse(appliedAt);
|
|
811
|
+
const started = Date.parse(session.startedAt);
|
|
812
|
+
const ended = Date.parse(session.endedAt);
|
|
813
|
+
if (started < applied && ended > applied)
|
|
814
|
+
reasons.push("crosses_intervention_boundary");
|
|
815
|
+
else if (phase === "baseline" && ended > applied)
|
|
816
|
+
reasons.push("wrong_side_of_intervention");
|
|
817
|
+
else if (phase === "post_change" && started < applied)
|
|
818
|
+
reasons.push("wrong_side_of_intervention");
|
|
819
|
+
}
|
|
820
|
+
return reasons;
|
|
821
|
+
}
|
|
822
|
+
function totalTokens(tokens) {
|
|
823
|
+
const components = [
|
|
824
|
+
tokens.uncachedInputTokens,
|
|
825
|
+
tokens.cacheReadTokens,
|
|
826
|
+
tokens.cacheWriteTokens,
|
|
827
|
+
tokens.toolTokens,
|
|
828
|
+
tokens.outputTokens,
|
|
829
|
+
tokens.thoughtTokens
|
|
830
|
+
];
|
|
831
|
+
const calculated = tokens.calculatedTotalTokens;
|
|
832
|
+
const calculatedComplete = tokens.componentEvidence.calculatedTotalTokens === "calculated_complete";
|
|
833
|
+
const componentBreakdownObserved = tokens.componentEvidence.uncachedInputTokens === "observed" &&
|
|
834
|
+
tokens.componentEvidence.cacheReadTokens === "observed" &&
|
|
835
|
+
tokens.componentEvidence.cacheWriteTokens === "observed" &&
|
|
836
|
+
tokens.componentEvidence.toolTokens === "observed" &&
|
|
837
|
+
tokens.componentEvidence.outputTokens === "observed" &&
|
|
838
|
+
tokens.componentEvidence.thoughtTokens === "observed";
|
|
839
|
+
if (components.every((value) => value !== null)) {
|
|
840
|
+
const componentTotal = components.reduce((sum, value) => sum + value, 0);
|
|
841
|
+
if (!Number.isSafeInteger(componentTotal)) {
|
|
842
|
+
return { value: null, reason: "inconsistent_total_tokens" };
|
|
843
|
+
}
|
|
844
|
+
if (calculated !== null && calculated !== componentTotal) {
|
|
845
|
+
return { value: null, reason: "inconsistent_total_tokens" };
|
|
846
|
+
}
|
|
847
|
+
if (componentBreakdownObserved && tokens.reportedTotalTokens !== null &&
|
|
848
|
+
tokens.reportedTotalTokens !== componentTotal) {
|
|
849
|
+
return { value: null, reason: "inconsistent_total_tokens" };
|
|
850
|
+
}
|
|
851
|
+
if (tokens.reportedTotalTokens !== null)
|
|
852
|
+
return { value: tokens.reportedTotalTokens };
|
|
853
|
+
return calculatedComplete
|
|
854
|
+
? { value: componentTotal }
|
|
855
|
+
: { value: null, reason: "missing_total_tokens" };
|
|
856
|
+
}
|
|
857
|
+
if (calculatedComplete && calculated !== null && tokens.reportedTotalTokens !== null &&
|
|
858
|
+
calculated !== tokens.reportedTotalTokens) {
|
|
859
|
+
return { value: null, reason: "inconsistent_total_tokens" };
|
|
860
|
+
}
|
|
861
|
+
if (tokens.reportedTotalTokens !== null)
|
|
862
|
+
return { value: tokens.reportedTotalTokens };
|
|
863
|
+
if (calculatedComplete && calculated !== null)
|
|
864
|
+
return { value: calculated };
|
|
865
|
+
return { value: null, reason: "missing_total_tokens" };
|
|
866
|
+
}
|
|
867
|
+
function evaluateQuality(experiment, baseline, post) {
|
|
868
|
+
const all = [...baseline, ...post].map((entry) => entry.session.quality);
|
|
869
|
+
if (post.some((entry) => entry.session.quality.status === "failed")) {
|
|
870
|
+
return { status: "regressed", evidence: weakestQualityEvidence(all) };
|
|
871
|
+
}
|
|
872
|
+
const minimum = qualityEvidenceRank(experiment.qualityGuard.minimumEvidence);
|
|
873
|
+
if (all.length === 0 || all.some((quality) => quality.status !== "passed" || qualityEvidenceRank(quality.evidence) < minimum)) {
|
|
874
|
+
return { status: "insufficient", evidence: weakestQualityEvidence(all) };
|
|
875
|
+
}
|
|
876
|
+
return { status: "held", evidence: weakestQualityEvidence(all) };
|
|
877
|
+
}
|
|
878
|
+
function qualityEvidenceRank(evidence) {
|
|
879
|
+
switch (evidence) {
|
|
880
|
+
case "verified": return 3;
|
|
881
|
+
case "observed": return 2;
|
|
882
|
+
case "user_declared": return 1;
|
|
883
|
+
case "missing": return 0;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
function weakestQualityEvidence(qualities) {
|
|
887
|
+
if (qualities.length === 0)
|
|
888
|
+
return "missing";
|
|
889
|
+
return qualities
|
|
890
|
+
.map((quality) => quality.evidence)
|
|
891
|
+
.sort((left, right) => qualityEvidenceRank(left) - qualityEvidenceRank(right))[0];
|
|
892
|
+
}
|
|
893
|
+
function componentMedians(included) {
|
|
894
|
+
return {
|
|
895
|
+
uncachedInputTokens: completeComponentMedian(included, "uncachedInputTokens"),
|
|
896
|
+
cacheReadTokens: completeComponentMedian(included, "cacheReadTokens"),
|
|
897
|
+
cacheWriteTokens: completeComponentMedian(included, "cacheWriteTokens"),
|
|
898
|
+
toolTokens: completeComponentMedian(included, "toolTokens"),
|
|
899
|
+
outputTokens: completeComponentMedian(included, "outputTokens"),
|
|
900
|
+
thoughtTokens: completeComponentMedian(included, "thoughtTokens")
|
|
901
|
+
};
|
|
902
|
+
}
|
|
903
|
+
function completeComponentMedian(included, key) {
|
|
904
|
+
const values = included.map((entry) => entry.session.tokens[key]);
|
|
905
|
+
return values.length > 0 && values.every((value) => value !== null)
|
|
906
|
+
? median(values)
|
|
907
|
+
: null;
|
|
908
|
+
}
|
|
909
|
+
function median(values) {
|
|
910
|
+
if (values.length === 0)
|
|
911
|
+
return null;
|
|
912
|
+
const ordered = [...values].sort((left, right) => left - right);
|
|
913
|
+
const middle = Math.floor(ordered.length / 2);
|
|
914
|
+
return ordered.length % 2 === 1
|
|
915
|
+
? ordered[middle]
|
|
916
|
+
: (ordered[middle - 1] + ordered[middle]) / 2;
|
|
917
|
+
}
|
|
918
|
+
function roundPercent(value) {
|
|
919
|
+
const rounded = Math.round((value + Number.EPSILON) * 100) / 100;
|
|
920
|
+
return Object.is(rounded, -0) ? 0 : rounded;
|
|
921
|
+
}
|
|
922
|
+
function sessionReferenceCounts(experiment) {
|
|
923
|
+
const counts = new Map();
|
|
924
|
+
for (const session of experiment.baselineSessions) {
|
|
925
|
+
const count = counts.get(session.sessionRef) ?? { baseline: 0, post: 0 };
|
|
926
|
+
count.baseline += 1;
|
|
927
|
+
counts.set(session.sessionRef, count);
|
|
928
|
+
}
|
|
929
|
+
for (const session of experiment.postSessions) {
|
|
930
|
+
const count = counts.get(session.sessionRef) ?? { baseline: 0, post: 0 };
|
|
931
|
+
count.post += 1;
|
|
932
|
+
counts.set(session.sessionRef, count);
|
|
933
|
+
}
|
|
934
|
+
return counts;
|
|
935
|
+
}
|
|
936
|
+
function canonicalWasteFindingBody(finding) {
|
|
937
|
+
return {
|
|
938
|
+
...finding,
|
|
939
|
+
caveats: [...finding.caveats].sort(compareText),
|
|
940
|
+
evidenceRefs: [...finding.evidenceRefs].sort(compareText)
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
function canonicalExperimentBody(experiment) {
|
|
944
|
+
return {
|
|
945
|
+
...experiment,
|
|
946
|
+
finding: parseWasteFindingV0(experiment.finding),
|
|
947
|
+
baselineSessions: canonicalSessions(experiment.baselineSessions),
|
|
948
|
+
postSessions: canonicalSessions(experiment.postSessions)
|
|
949
|
+
};
|
|
950
|
+
}
|
|
951
|
+
function canonicalSessions(sessions) {
|
|
952
|
+
return [...sessions].sort((left, right) => {
|
|
953
|
+
const byRef = compareText(left.sessionRef, right.sessionRef);
|
|
954
|
+
return byRef !== 0 ? byRef : compareText(canonicalJson(left), canonicalJson(right));
|
|
955
|
+
});
|
|
956
|
+
}
|
|
957
|
+
function wasteFindingDigest(finding) {
|
|
958
|
+
return `wf_v0_${createHash("sha256").update(canonicalJson(finding)).digest("hex")}`;
|
|
959
|
+
}
|
|
960
|
+
function wasteCandidateDigest(finding) {
|
|
961
|
+
const identity = {
|
|
962
|
+
findingType: finding.findingType,
|
|
963
|
+
scope: finding.scope,
|
|
964
|
+
candidateAction: finding.candidateAction,
|
|
965
|
+
target: finding.target
|
|
966
|
+
};
|
|
967
|
+
return `wfc_v0_${createHash("sha256").update(canonicalJson(identity)).digest("hex")}`;
|
|
968
|
+
}
|
|
969
|
+
function experimentDigest(experiment) {
|
|
970
|
+
// Experiment identity deliberately excludes post-change evidence so the same
|
|
971
|
+
// ID survives lifecycle updates. Baseline references and policy remain fixed.
|
|
972
|
+
const identity = {
|
|
973
|
+
kind: experiment.kind,
|
|
974
|
+
schemaVersion: experiment.schemaVersion,
|
|
975
|
+
createdAt: experiment.createdAt,
|
|
976
|
+
findingId: experiment.finding.id,
|
|
977
|
+
cohort: experiment.cohort,
|
|
978
|
+
matchingPolicy: experiment.matchingPolicy,
|
|
979
|
+
qualityGuard: experiment.qualityGuard,
|
|
980
|
+
baselineSessionRefs: experiment.baselineSessions.map((session) => session.sessionRef)
|
|
981
|
+
};
|
|
982
|
+
return `tre_v0_${createHash("sha256").update(canonicalJson(identity)).digest("hex")}`;
|
|
983
|
+
}
|
|
984
|
+
function experimentRevisionDigest(experiment) {
|
|
985
|
+
return `trev_v0_${createHash("sha256")
|
|
986
|
+
.update(canonicalJson(experiment))
|
|
987
|
+
.digest("hex")}`;
|
|
988
|
+
}
|
|
989
|
+
function compareExclusions(left, right) {
|
|
990
|
+
return compareText(`${left.phase}\u0000${left.sessionRef}`, `${right.phase}\u0000${right.sessionRef}`);
|
|
991
|
+
}
|
|
992
|
+
function compareText(left, right) {
|
|
993
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
994
|
+
}
|
|
995
|
+
function canonicalJson(value) {
|
|
996
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") {
|
|
997
|
+
return JSON.stringify(value);
|
|
998
|
+
}
|
|
999
|
+
if (typeof value === "number") {
|
|
1000
|
+
if (!Number.isFinite(value))
|
|
1001
|
+
throw new TypeError("Canonical values must be finite.");
|
|
1002
|
+
return JSON.stringify(Object.is(value, -0) ? 0 : value);
|
|
1003
|
+
}
|
|
1004
|
+
if (Array.isArray(value)) {
|
|
1005
|
+
return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
|
|
1006
|
+
}
|
|
1007
|
+
if (typeof value === "object" && value) {
|
|
1008
|
+
const object = value;
|
|
1009
|
+
return `{${Object.keys(object).filter((key) => object[key] !== undefined).sort()
|
|
1010
|
+
.map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`).join(",")}}`;
|
|
1011
|
+
}
|
|
1012
|
+
throw new TypeError("Canonical values must be JSON data.");
|
|
1013
|
+
}
|
|
1014
|
+
function hasUnpairedSurrogate(value) {
|
|
1015
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
1016
|
+
const code = value.charCodeAt(index);
|
|
1017
|
+
if (code >= 0xd800 && code <= 0xdbff) {
|
|
1018
|
+
const next = value.charCodeAt(index + 1);
|
|
1019
|
+
if (!(next >= 0xdc00 && next <= 0xdfff))
|
|
1020
|
+
return true;
|
|
1021
|
+
index += 1;
|
|
1022
|
+
}
|
|
1023
|
+
else if (code >= 0xdc00 && code <= 0xdfff)
|
|
1024
|
+
return true;
|
|
1025
|
+
}
|
|
1026
|
+
return false;
|
|
1027
|
+
}
|
|
1028
|
+
//# sourceMappingURL=actionVerification.js.map
|