@tangle-network/agent-eval 0.109.1 → 0.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/dist/analyst/index.d.ts +10 -12
  2. package/dist/analyst/index.js +8 -11
  3. package/dist/analyst/index.js.map +1 -1
  4. package/dist/{analyze-runs-DJYpep3L.d.ts → analyze-runs-Dmz6LA9e.d.ts} +4 -4
  5. package/dist/{baseline-Bbid3WoO.d.ts → baseline-DsNteOgR.d.ts} +32 -2
  6. package/dist/belief-state/index.d.ts +6 -6
  7. package/dist/benchmarks/index.d.ts +4 -4
  8. package/dist/benchmarks/index.js +7 -8
  9. package/dist/builder-eval/index.d.ts +4 -4
  10. package/dist/builder-eval/index.js +1 -2
  11. package/dist/builder-eval/index.js.map +1 -1
  12. package/dist/{calibration-BPmzuVPk.d.ts → calibration-Dz8TQV4y.d.ts} +2 -2
  13. package/dist/campaign/index.d.ts +16 -19
  14. package/dist/campaign/index.js +7 -8
  15. package/dist/{chunk-LOJ2QVCE.js → chunk-2IY4ILP4.js} +2 -2
  16. package/dist/{chunk-LIEJUH2I.js → chunk-6PL5MGDL.js} +9 -9
  17. package/dist/{chunk-2OGPXHOB.js → chunk-7NX6ZSBG.js} +36 -7
  18. package/dist/chunk-7NX6ZSBG.js.map +1 -0
  19. package/dist/{chunk-OVPVM4JC.js → chunk-GTERJI6Q.js} +4 -4
  20. package/dist/{chunk-YEHAEDUD.js → chunk-IMWDSFUM.js} +604 -2
  21. package/dist/chunk-IMWDSFUM.js.map +1 -0
  22. package/dist/{chunk-JZXGWLK5.js → chunk-MHNQWM4I.js} +62 -6
  23. package/dist/chunk-MHNQWM4I.js.map +1 -0
  24. package/dist/{chunk-QRVS7MX4.js → chunk-OW47B5WA.js} +3 -5
  25. package/dist/{chunk-QRVS7MX4.js.map → chunk-OW47B5WA.js.map} +1 -1
  26. package/dist/{chunk-DBDRR6GF.js → chunk-PLOMR3HP.js} +48 -2
  27. package/dist/chunk-PLOMR3HP.js.map +1 -0
  28. package/dist/{chunk-GDZAWO2I.js → chunk-QFGTU7MT.js} +2 -2
  29. package/dist/{chunk-R6D7NEYJ.js → chunk-RNB2NICW.js} +11 -11
  30. package/dist/{chunk-V7HNA47Z.js → chunk-RSVSSZKF.js} +5 -5
  31. package/dist/{chunk-5PK3626Q.js → chunk-XRGOKCMO.js} +88 -17
  32. package/dist/chunk-XRGOKCMO.js.map +1 -0
  33. package/dist/{code-agent-session-rnJKlqmT.d.ts → code-agent-session-yitf9I-F.d.ts} +1 -1
  34. package/dist/contract/index.d.ts +20 -23
  35. package/dist/contract/index.js +11 -13
  36. package/dist/contract/index.js.map +1 -1
  37. package/dist/{control-B8UthSBL.d.ts → control-U8LBKUES.d.ts} +5 -6
  38. package/dist/control.d.ts +8 -9
  39. package/dist/control.js +6 -8
  40. package/dist/{dataset-DS7ytHZU.d.ts → dataset-NENEzRgk.d.ts} +1 -1
  41. package/dist/{default-registry-BswHCXnU.d.ts → default-registry-Bcf1uKVI.d.ts} +1 -2
  42. package/dist/{emitter-C2rqGH_l.d.ts → emitter-BRchAAAx.d.ts} +2 -2
  43. package/dist/{failure-cluster-DH9Flgcf.d.ts → failure-cluster-C48PiReX.d.ts} +2 -2
  44. package/dist/feedback-trajectory-pDcz1lQ1.d.ts +348 -0
  45. package/dist/{gepa-B3x5Ulcv.d.ts → gepa-BUNP3606.d.ts} +143 -2
  46. package/dist/hosted/index.d.ts +7 -7
  47. package/dist/{index-pPtfoIJO.d.ts → index-Dc3VLGhp.d.ts} +2 -2
  48. package/dist/index.d.ts +645 -61
  49. package/dist/index.js +1282 -190
  50. package/dist/index.js.map +1 -1
  51. package/dist/{insight-report-B4xrdwEK.d.ts → insight-report-D4cXFsLt.d.ts} +1 -1
  52. package/dist/{integrity-DqGZg3st.d.ts → integrity-qemeBAyx.d.ts} +1 -1
  53. package/dist/{types-D1ytG0Yg.d.ts → kind-factory-20hcaYpf.d.ts} +169 -2
  54. package/dist/meta-eval/index.d.ts +5 -5
  55. package/dist/meta-eval/index.js +1 -2
  56. package/dist/meta-eval/index.js.map +1 -1
  57. package/dist/{multi-layer-verifier-CI4jdX-q.d.ts → multi-layer-verifier-BsqKuLyN.d.ts} +1 -1
  58. package/dist/multishot/index.d.ts +3 -3
  59. package/dist/openapi.json +1 -1
  60. package/dist/pipelines/index.d.ts +6 -7
  61. package/dist/pipelines/index.js +3 -6
  62. package/dist/pipelines/index.js.map +1 -1
  63. package/dist/{policy-edit-DQUXYMDm.d.ts → policy-edit-D2bBDZDf.d.ts} +2 -2
  64. package/dist/{pre-registration-BUhVPzE7.d.ts → pre-registration-BepVVa6P.d.ts} +3 -3
  65. package/dist/{provenance-DdDhf6cg.d.ts → provenance-DMvsfknv.d.ts} +3 -5
  66. package/dist/{query-0aTmbmQe.d.ts → query-Ck190MOd.d.ts} +2 -2
  67. package/dist/{release-report-DeJpsBiA.d.ts → release-report-oBfOz8ku.d.ts} +3 -3
  68. package/dist/reporting.d.ts +8 -8
  69. package/dist/{researcher-Wc7dx6GM.d.ts → researcher-CaH0CwFC.d.ts} +6 -6
  70. package/dist/rl.d.ts +568 -15
  71. package/dist/rl.js +4 -4
  72. package/dist/{rubric-predictive-validity-DPnyG-CE.d.ts → rubric-predictive-validity-C-fMteAW.d.ts} +1 -1
  73. package/dist/{run-record-I-Z3JNvO.d.ts → run-record-DksGsfgv.d.ts} +1 -1
  74. package/dist/{runtime-trajectory-iW9IhV3e.d.ts → runtime-trajectory-h5i0SZUj.d.ts} +1 -1
  75. package/dist/{schema-m0gsnbt3.d.ts → schema-SGWcK9wa.d.ts} +1 -1
  76. package/dist/{semantic-concept-judge-BmNZPB_j.d.ts → semantic-concept-judge-D7z6JCLZ.d.ts} +57 -4
  77. package/dist/{store-BcFXE6LG.d.ts → store-BsVi7ncX.d.ts} +1 -1
  78. package/dist/storyboard/index.d.ts +1 -1
  79. package/dist/{summary-report-QMZVe3P-.d.ts → summary-report-Bz-0-t8v.d.ts} +2 -2
  80. package/dist/{test-graded-scenario-DeODGLra.d.ts → test-graded-scenario-mzYBKspu.d.ts} +3 -3
  81. package/dist/traces.d.ts +54 -11
  82. package/dist/traces.js +25 -27
  83. package/dist/{types-BdIv5dvA.d.ts → types-v--ctu-b.d.ts} +2 -2
  84. package/dist/wire/index.d.ts +5 -6
  85. package/package.json +1 -71
  86. package/dist/adapters/http.d.ts +0 -142
  87. package/dist/adapters/http.js +0 -203
  88. package/dist/adapters/http.js.map +0 -1
  89. package/dist/adapters/langchain.d.ts +0 -95
  90. package/dist/adapters/langchain.js +0 -34
  91. package/dist/adapters/langchain.js.map +0 -1
  92. package/dist/adapters/otel.d.ts +0 -112
  93. package/dist/adapters/otel.js +0 -110
  94. package/dist/adapters/otel.js.map +0 -1
  95. package/dist/chunk-2OGPXHOB.js.map +0 -1
  96. package/dist/chunk-45EEMHTC.js +0 -35
  97. package/dist/chunk-45EEMHTC.js.map +0 -1
  98. package/dist/chunk-5BKGXME7.js +0 -65
  99. package/dist/chunk-5BKGXME7.js.map +0 -1
  100. package/dist/chunk-5PK3626Q.js.map +0 -1
  101. package/dist/chunk-6SK5VFYK.js +0 -100
  102. package/dist/chunk-6SK5VFYK.js.map +0 -1
  103. package/dist/chunk-DBDRR6GF.js.map +0 -1
  104. package/dist/chunk-DJWX3GVS.js +0 -81
  105. package/dist/chunk-DJWX3GVS.js.map +0 -1
  106. package/dist/chunk-FOUG2VVS.js +0 -855
  107. package/dist/chunk-FOUG2VVS.js.map +0 -1
  108. package/dist/chunk-JZXGWLK5.js.map +0 -1
  109. package/dist/chunk-K7QEIHHJ.js +0 -613
  110. package/dist/chunk-K7QEIHHJ.js.map +0 -1
  111. package/dist/chunk-KKHDIONI.js +0 -414
  112. package/dist/chunk-KKHDIONI.js.map +0 -1
  113. package/dist/chunk-KMPRBJK4.js +0 -74
  114. package/dist/chunk-KMPRBJK4.js.map +0 -1
  115. package/dist/chunk-Q2JRAWRI.js +0 -196
  116. package/dist/chunk-Q2JRAWRI.js.map +0 -1
  117. package/dist/chunk-RZTMDUO7.js +0 -49
  118. package/dist/chunk-RZTMDUO7.js.map +0 -1
  119. package/dist/chunk-STGVSCDH.js +0 -202
  120. package/dist/chunk-STGVSCDH.js.map +0 -1
  121. package/dist/chunk-YEHAEDUD.js.map +0 -1
  122. package/dist/control-runtime-Acf9CGhw.d.ts +0 -182
  123. package/dist/corpus-eBVwhCp1.d.ts +0 -560
  124. package/dist/counterfactual-DlOz8PBx.d.ts +0 -85
  125. package/dist/diagnose.d.ts +0 -252
  126. package/dist/diagnose.js +0 -382
  127. package/dist/diagnose.js.map +0 -1
  128. package/dist/feedback-trajectory-C9KCo8ag.d.ts +0 -169
  129. package/dist/governance/index.d.ts +0 -135
  130. package/dist/governance/index.js +0 -18
  131. package/dist/governance/index.js.map +0 -1
  132. package/dist/groundedness/index.d.ts +0 -112
  133. package/dist/groundedness/index.js +0 -77
  134. package/dist/groundedness/index.js.map +0 -1
  135. package/dist/harness-optimizer-mOl9XX_O.d.ts +0 -106
  136. package/dist/kind-factory-DvIGo_cP.d.ts +0 -171
  137. package/dist/knowledge/index.d.ts +0 -103
  138. package/dist/knowledge/index.js +0 -18
  139. package/dist/knowledge/index.js.map +0 -1
  140. package/dist/pareto-E-pembql.d.ts +0 -81
  141. package/dist/perf/index.d.ts +0 -123
  142. package/dist/perf/index.js +0 -18
  143. package/dist/perf/index.js.map +0 -1
  144. package/dist/prm/index.d.ts +0 -104
  145. package/dist/prm/index.js +0 -265
  146. package/dist/prm/index.js.map +0 -1
  147. package/dist/product-benchmark/index.d.ts +0 -247
  148. package/dist/product-benchmark/index.js +0 -37
  149. package/dist/product-benchmark/index.js.map +0 -1
  150. package/dist/red-team-KmmiqBlY.d.ts +0 -63
  151. package/dist/redact-B40YG2M_.d.ts +0 -45
  152. package/dist/rubric-Cc6UHvUb.d.ts +0 -73
  153. package/dist/run-critic-CmMf05uV.d.ts +0 -56
  154. package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
  155. package/dist/telemetry/file.d.ts +0 -19
  156. package/dist/telemetry/file.js +0 -45
  157. package/dist/telemetry/file.js.map +0 -1
  158. package/dist/telemetry/index.d.ts +0 -38
  159. package/dist/telemetry/index.js +0 -130
  160. package/dist/telemetry/index.js.map +0 -1
  161. package/dist/testing-C21CHsq2.d.ts +0 -20
  162. package/dist/testing.d.ts +0 -1
  163. package/dist/testing.js +0 -8
  164. package/dist/testing.js.map +0 -1
  165. package/dist/trajectory-2TkpSEVh.d.ts +0 -33
  166. package/dist/workflow/index.d.ts +0 -496
  167. package/dist/workflow/index.js +0 -2178
  168. package/dist/workflow/index.js.map +0 -1
  169. /package/dist/{chunk-LOJ2QVCE.js.map → chunk-2IY4ILP4.js.map} +0 -0
  170. /package/dist/{chunk-LIEJUH2I.js.map → chunk-6PL5MGDL.js.map} +0 -0
  171. /package/dist/{chunk-OVPVM4JC.js.map → chunk-GTERJI6Q.js.map} +0 -0
  172. /package/dist/{chunk-GDZAWO2I.js.map → chunk-QFGTU7MT.js.map} +0 -0
  173. /package/dist/{chunk-R6D7NEYJ.js.map → chunk-RNB2NICW.js.map} +0 -0
  174. /package/dist/{chunk-V7HNA47Z.js.map → chunk-RSVSSZKF.js.map} +0 -0
@@ -1,74 +0,0 @@
1
- import {
2
- buildTrajectory
3
- } from "./chunk-RZTMDUO7.js";
4
- import {
5
- isLlmSpan,
6
- isToolSpan
7
- } from "./chunk-5BKGXME7.js";
8
-
9
- // src/prm/training-export.ts
10
- async function exportTrainingData(store, graded, options = {}) {
11
- const window = options.contextWindow ?? 5;
12
- const out = [];
13
- for (const g of graded) {
14
- const trajectory = await buildTrajectory(store, g.runId);
15
- const spanById = new Map(trajectory.steps.map((s) => [s.span.spanId, s]));
16
- for (const gs of g.steps) {
17
- const node = spanById.get(gs.spanId);
18
- if (!node) continue;
19
- const idx = trajectory.steps.indexOf(node);
20
- const priorSpans = trajectory.steps.slice(Math.max(0, idx - window), idx).map((s) => s.span);
21
- out.push({
22
- runId: g.runId,
23
- spanId: gs.spanId,
24
- rubricId: gs.rubricId,
25
- score: gs.score,
26
- context: {
27
- priorTurns: priorSpans.map(spanToTurn).filter((t) => t !== null),
28
- step: { kind: node.span.kind, text: spanToText(node.span) }
29
- },
30
- rationale: gs.rationale,
31
- evidence: gs.evidence
32
- });
33
- }
34
- }
35
- return out;
36
- }
37
- function toNdjson(samples) {
38
- return `${samples.map((s) => JSON.stringify(s)).join("\n")}
39
- `;
40
- }
41
- function spanToTurn(span) {
42
- if (isLlmSpan(span)) {
43
- const text = span.output ?? span.messages.map((m) => `${m.role}: ${m.content}`).join("\n");
44
- return { role: "assistant", content: text };
45
- }
46
- if (isToolSpan(span)) {
47
- return {
48
- role: "tool",
49
- content: `${span.toolName}(${safeStringify(span.args)}) \u2192 ${safeStringify(span.result)}`
50
- };
51
- }
52
- return null;
53
- }
54
- function spanToText(span) {
55
- if (isLlmSpan(span)) return span.output ?? "";
56
- if (isToolSpan(span))
57
- return `${span.toolName}(${safeStringify(span.args)}) \u2192 ${safeStringify(span.result)}`;
58
- return span.name;
59
- }
60
- function safeStringify(v) {
61
- if (v === null || v === void 0) return "";
62
- if (typeof v === "string") return v;
63
- try {
64
- return JSON.stringify(v);
65
- } catch {
66
- return String(v);
67
- }
68
- }
69
-
70
- export {
71
- exportTrainingData,
72
- toNdjson
73
- };
74
- //# sourceMappingURL=chunk-KMPRBJK4.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/prm/training-export.ts"],"sourcesContent":["/**\n * Export PRM-graded traces as training data for downstream reward-model\n * fine-tuning. Canonical format is NDJSON of\n * `{ trajectory_text, step_index, rubric, score }` so a small model can\n * learn to predict step rewards from step context.\n *\n * The framework doesn't train the model — we emit the data; callers\n * plug it into their preferred trainer (TRL, Unsloth, custom).\n */\n\nimport type { LlmSpan, Span } from '../trace/schema'\nimport { isLlmSpan, isToolSpan } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory } from '../trajectory'\nimport type { PrmGradedTrace } from './rubric'\n\nexport interface PrmTrainingSample {\n runId: string\n spanId: string\n rubricId: string\n score: number\n /** Serialized step context — step + surrounding conversation. */\n context: {\n priorTurns: Array<{ role: string; content: string }>\n step: { kind: Span['kind']; text: string }\n }\n /** Optional evidence + rationale for auditability. */\n rationale?: string\n evidence?: string\n}\n\nexport async function exportTrainingData(\n store: TraceStore,\n graded: PrmGradedTrace[],\n options: { contextWindow?: number } = {},\n): Promise<PrmTrainingSample[]> {\n const window = options.contextWindow ?? 5\n const out: PrmTrainingSample[] = []\n for (const g of graded) {\n const trajectory = await buildTrajectory(store, g.runId)\n const spanById = new Map(trajectory.steps.map((s) => [s.span.spanId, s]))\n for (const gs of g.steps) {\n const node = spanById.get(gs.spanId)\n if (!node) continue\n const idx = trajectory.steps.indexOf(node)\n const priorSpans = trajectory.steps.slice(Math.max(0, idx - window), idx).map((s) => s.span)\n out.push({\n runId: g.runId,\n spanId: gs.spanId,\n rubricId: gs.rubricId,\n score: gs.score,\n context: {\n priorTurns: priorSpans\n .map(spanToTurn)\n .filter((t): t is { role: string; content: string } => t !== null),\n step: { kind: node.span.kind, text: spanToText(node.span) },\n },\n rationale: gs.rationale,\n evidence: gs.evidence,\n })\n }\n }\n return out\n}\n\n/** NDJSON serialization — write to file or stream directly to a trainer. */\nexport function toNdjson(samples: PrmTrainingSample[]): string {\n return `${samples.map((s) => JSON.stringify(s)).join('\\n')}\\n`\n}\n\nfunction spanToTurn(span: Span): { role: string; content: string } | null {\n if (isLlmSpan(span)) {\n const text = span.output ?? span.messages.map((m) => `${m.role}: ${m.content}`).join('\\n')\n return { role: 'assistant', content: text }\n }\n if (isToolSpan(span)) {\n return {\n role: 'tool',\n content: `${span.toolName}(${safeStringify(span.args)}) → ${safeStringify(span.result)}`,\n }\n }\n return null\n}\n\nfunction spanToText(span: Span): string {\n if (isLlmSpan(span)) return (span as LlmSpan).output ?? ''\n if (isToolSpan(span))\n return `${span.toolName}(${safeStringify(span.args)}) → ${safeStringify(span.result)}`\n return span.name\n}\n\nfunction safeStringify(v: unknown): string {\n if (v === null || v === undefined) return ''\n if (typeof v === 'string') return v\n try {\n return JSON.stringify(v)\n } catch {\n return String(v)\n }\n}\n"],"mappings":";;;;;;;;;AA+BA,eAAsB,mBACpB,OACA,QACA,UAAsC,CAAC,GACT;AAC9B,QAAM,SAAS,QAAQ,iBAAiB;AACxC,QAAM,MAA2B,CAAC;AAClC,aAAW,KAAK,QAAQ;AACtB,UAAM,aAAa,MAAM,gBAAgB,OAAO,EAAE,KAAK;AACvD,UAAM,WAAW,IAAI,IAAI,WAAW,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC;AACxE,eAAW,MAAM,EAAE,OAAO;AACxB,YAAM,OAAO,SAAS,IAAI,GAAG,MAAM;AACnC,UAAI,CAAC,KAAM;AACX,YAAM,MAAM,WAAW,MAAM,QAAQ,IAAI;AACzC,YAAM,aAAa,WAAW,MAAM,MAAM,KAAK,IAAI,GAAG,MAAM,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;AAC3F,UAAI,KAAK;AAAA,QACP,OAAO,EAAE;AAAA,QACT,QAAQ,GAAG;AAAA,QACX,UAAU,GAAG;AAAA,QACb,OAAO,GAAG;AAAA,QACV,SAAS;AAAA,UACP,YAAY,WACT,IAAI,UAAU,EACd,OAAO,CAAC,MAA8C,MAAM,IAAI;AAAA,UACnE,MAAM,EAAE,MAAM,KAAK,KAAK,MAAM,MAAM,WAAW,KAAK,IAAI,EAAE;AAAA,QAC5D;AAAA,QACA,WAAW,GAAG;AAAA,QACd,UAAU,GAAG;AAAA,MACf,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT;AAGO,SAAS,SAAS,SAAsC;AAC7D,SAAO,GAAG,QAAQ,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAC5D;AAEA,SAAS,WAAW,MAAsD;AACxE,MAAI,UAAU,IAAI,GAAG;AACnB,UAAM,OAAO,KAAK,UAAU,KAAK,SAAS,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,KAAK,EAAE,OAAO,EAAE,EAAE,KAAK,IAAI;AACzF,WAAO,EAAE,MAAM,aAAa,SAAS,KAAK;AAAA,EAC5C;AACA,MAAI,WAAW,IAAI,GAAG;AACpB,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS,GAAG,KAAK,QAAQ,IAAI,cAAc,KAAK,IAAI,CAAC,YAAO,cAAc,KAAK,MAAM,CAAC;AAAA,IACxF;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAoB;AACtC,MAAI,UAAU,IAAI,EAAG,QAAQ,KAAiB,UAAU;AACxD,MAAI,WAAW,IAAI;AACjB,WAAO,GAAG,KAAK,QAAQ,IAAI,cAAc,KAAK,IAAI,CAAC,YAAO,cAAc,KAAK,MAAM,CAAC;AACtF,SAAO,KAAK;AACd;AAEA,SAAS,cAAc,GAAoB;AACzC,MAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,MAAI,OAAO,MAAM,SAAU,QAAO;AAClC,MAAI;AACF,WAAO,KAAK,UAAU,CAAC;AAAA,EACzB,QAAQ;AACN,WAAO,OAAO,CAAC;AAAA,EACjB;AACF;","names":[]}
@@ -1,196 +0,0 @@
1
- import {
2
- objectiveEval
3
- } from "./chunk-YEHAEDUD.js";
4
-
5
- // src/knowledge/readiness.ts
6
- function scoreKnowledgeReadiness(options) {
7
- const now = options.now ?? /* @__PURE__ */ new Date();
8
- const requirements = options.requirements.map(normalizeRequirement);
9
- const missing = requirements.filter((requirement) => isRequirementMissing(requirement, now));
10
- const blockingMissingRequirements = missing.filter(isBlockingGap);
11
- const nonBlockingGaps = missing.filter((requirement) => !isBlockingGap(requirement));
12
- const readinessScore = weightedReadinessAt(requirements, now);
13
- const bundle = {
14
- taskId: options.taskId,
15
- requirements,
16
- evidenceIds: unique([
17
- ...options.evidenceIds ?? [],
18
- ...requirements.flatMap((r) => r.evidenceIds)
19
- ]),
20
- claimIds: unique(options.claimIds ?? []),
21
- wikiPageIds: unique(options.wikiPageIds ?? []),
22
- userAnswers: options.userAnswers ?? {},
23
- missing,
24
- readinessScore,
25
- metadata: options.metadata
26
- };
27
- const recommendedAction = chooseRecommendedAction(blockingMissingRequirements, nonBlockingGaps);
28
- const severity = blockingMissingRequirements.length > 0 ? "critical" : nonBlockingGaps.some((gap) => gap.importance === "high") ? "warning" : "info";
29
- const reason = blockingMissingRequirements.length > 0 ? `${blockingMissingRequirements.length} blocking knowledge requirement(s) are missing.` : nonBlockingGaps.length > 0 ? `${nonBlockingGaps.length} non-blocking knowledge gap(s) remain.` : "All declared knowledge requirements are ready.";
30
- return {
31
- taskId: options.taskId,
32
- readinessScore,
33
- blockingMissingRequirements,
34
- nonBlockingGaps,
35
- recommendedAction,
36
- bundle,
37
- severity,
38
- reason
39
- };
40
- }
41
- function blockingKnowledgeEval(report, options = {}) {
42
- const minimumScore = options.minimumScore ?? 0.7;
43
- const passed = report.blockingMissingRequirements.length === 0 && report.readinessScore >= minimumScore;
44
- if (options.emitter) {
45
- void options.emitter.emit({
46
- kind: "custom",
47
- payload: knowledgeReadinessTracePayload(report, { passed, minimumScore })
48
- }).catch(() => void 0);
49
- }
50
- return objectiveEval({
51
- id: options.id ?? "knowledge-ready",
52
- passed,
53
- score: report.readinessScore,
54
- severity: passed ? "info" : report.severity,
55
- detail: report.reason,
56
- evidence: report.blockingMissingRequirements.map((r) => r.id).join(", ") || void 0,
57
- metadata: { knowledgeReadiness: report }
58
- });
59
- }
60
- function knowledgeReadinessTracePayload(report, options = {}) {
61
- return {
62
- kind: "readiness_scored",
63
- taskId: report.taskId,
64
- passed: options.passed ?? report.blockingMissingRequirements.length === 0,
65
- readinessScore: report.readinessScore,
66
- minimumScore: options.minimumScore,
67
- blockingRequirementIds: report.blockingMissingRequirements.map((r) => r.id),
68
- nonBlockingRequirementIds: report.nonBlockingGaps.map((r) => r.id),
69
- recommendedAction: report.recommendedAction,
70
- severity: report.severity,
71
- reason: report.reason
72
- };
73
- }
74
- function userQuestionsForKnowledgeGaps(gaps) {
75
- return gaps.filter((gap) => gap.acquisitionMode === "ask_user" || gap.fallbackPolicy === "ask").map((gap) => ({
76
- id: `question_${gap.id}`,
77
- question: `Please provide: ${gap.description}`,
78
- reason: `Required for ${gap.requiredFor.join(", ") || "the task"}.`,
79
- requirementId: gap.id,
80
- importance: gap.importance,
81
- answerType: gap.sensitivity === "secret" ? "credential" : "free_text",
82
- impactIfUnknown: impactFor(gap)
83
- }));
84
- }
85
- function acquisitionPlansForKnowledgeGaps(gaps) {
86
- const byMode = /* @__PURE__ */ new Map();
87
- for (const gap of gaps) {
88
- const mode = planMode(gap.acquisitionMode);
89
- if (!mode) continue;
90
- const bucket = byMode.get(mode) ?? [];
91
- bucket.push(gap);
92
- byMode.set(mode, bucket);
93
- }
94
- return [...byMode.entries()].map(([mode, requirements]) => ({
95
- id: `acquire_${mode}`,
96
- requirementIds: requirements.map((r) => r.id),
97
- mode,
98
- description: descriptionForPlan(mode, requirements),
99
- priority: maxImportance(requirements.map((r) => r.importance)),
100
- questions: mode === "ask_user" ? userQuestionsForKnowledgeGaps(requirements) : void 0
101
- }));
102
- }
103
- function normalizeRequirement(requirement) {
104
- return {
105
- ...requirement,
106
- confidenceNeeded: clamp01(requirement.confidenceNeeded),
107
- currentConfidence: clamp01(requirement.currentConfidence),
108
- evidenceIds: unique(requirement.evidenceIds)
109
- };
110
- }
111
- function weightedReadinessAt(requirements, now) {
112
- if (requirements.length === 0) return 1;
113
- let weightSum = 0;
114
- let scoreSum = 0;
115
- for (const requirement of requirements) {
116
- const weight = importanceWeight(requirement.importance);
117
- const score = isExpired(requirement, now) ? 0 : requirement.confidenceNeeded <= 0 ? 1 : Math.min(1, requirement.currentConfidence / requirement.confidenceNeeded);
118
- weightSum += weight;
119
- scoreSum += weight * score;
120
- }
121
- return clamp01(scoreSum / weightSum);
122
- }
123
- function isRequirementMissing(requirement, now) {
124
- return isExpired(requirement, now) || requirement.currentConfidence < requirement.confidenceNeeded;
125
- }
126
- function isExpired(requirement, now) {
127
- if (!requirement.validUntil) return false;
128
- const deadline = Date.parse(requirement.validUntil);
129
- if (!Number.isFinite(deadline)) return true;
130
- return deadline <= now.getTime();
131
- }
132
- function isBlockingGap(requirement) {
133
- return requirement.importance === "blocking" || requirement.fallbackPolicy === "block" || requirement.sensitivity === "secret";
134
- }
135
- function chooseRecommendedAction(blocking, nonBlocking) {
136
- const gaps = blocking.length > 0 ? blocking : nonBlocking;
137
- if (gaps.length === 0) return "run_agent";
138
- if (gaps.some((gap) => gap.acquisitionMode === "ask_user" || gap.fallbackPolicy === "ask"))
139
- return "ask_user";
140
- if (gaps.some((gap) => gap.acquisitionMode === "query_connector")) return "query_connectors";
141
- if (gaps.some(
142
- (gap) => gap.acquisitionMode === "inspect_repo" || gap.acquisitionMode === "run_command"
143
- ))
144
- return "inspect_repo";
145
- if (gaps.some((gap) => gap.acquisitionMode === "search_web")) return "collect_web_data";
146
- if (gaps.some((gap) => gap.acquisitionMode === "not_available")) return "abort_or_rescope";
147
- if (nonBlocking.some((gap) => gap.importance === "high")) return "build_domain_wiki";
148
- return "continue_with_caveat";
149
- }
150
- function planMode(mode) {
151
- if (mode === "infer_low_confidence" || mode === "not_available") return null;
152
- return mode;
153
- }
154
- function descriptionForPlan(mode, requirements) {
155
- const labels = requirements.map((r) => r.description).join("; ");
156
- if (mode === "ask_user") return `Ask the user for: ${labels}`;
157
- if (mode === "search_web") return `Search web or documentation sources for: ${labels}`;
158
- if (mode === "query_connector") return `Query configured connectors for: ${labels}`;
159
- if (mode === "inspect_repo") return `Inspect repository context for: ${labels}`;
160
- if (mode === "run_command") return `Run local commands to collect: ${labels}`;
161
- return `Build domain wiki evidence for: ${labels}`;
162
- }
163
- function impactFor(requirement) {
164
- if (requirement.fallbackPolicy === "block") return "The agent should not run until this is known.";
165
- if (requirement.fallbackPolicy === "continue_with_caveat")
166
- return "The agent may continue, but must disclose uncertainty.";
167
- if (requirement.fallbackPolicy === "use_default")
168
- return "The agent will use the configured default if skipped.";
169
- return "The agent should ask before continuing.";
170
- }
171
- function maxImportance(values) {
172
- const order = ["blocking", "high", "medium", "low"];
173
- return order.find((value) => values.includes(value)) ?? "low";
174
- }
175
- function importanceWeight(importance) {
176
- if (importance === "blocking") return 8;
177
- if (importance === "high") return 4;
178
- if (importance === "medium") return 2;
179
- return 1;
180
- }
181
- function clamp01(value) {
182
- if (!Number.isFinite(value)) return 0;
183
- return Math.max(0, Math.min(1, value));
184
- }
185
- function unique(items) {
186
- return [...new Set(items)];
187
- }
188
-
189
- export {
190
- scoreKnowledgeReadiness,
191
- blockingKnowledgeEval,
192
- knowledgeReadinessTracePayload,
193
- userQuestionsForKnowledgeGaps,
194
- acquisitionPlansForKnowledgeGaps
195
- };
196
- //# sourceMappingURL=chunk-Q2JRAWRI.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/knowledge/readiness.ts"],"sourcesContent":["import { type ControlEvalResult, objectiveEval } from '../control-runtime'\nimport type { TraceEmitter } from '../trace/emitter'\nimport type {\n DataAcquisitionPlan,\n KnowledgeAcquisitionMode,\n KnowledgeBundle,\n KnowledgeImportance,\n KnowledgeReadinessReport,\n KnowledgeRecommendedAction,\n KnowledgeRequirement,\n UserQuestion,\n} from './types'\n\nexport interface ScoreKnowledgeReadinessOptions {\n taskId: string\n requirements: KnowledgeRequirement[]\n evidenceIds?: string[]\n claimIds?: string[]\n wikiPageIds?: string[]\n userAnswers?: Record<string, string>\n metadata?: Record<string, unknown>\n now?: Date\n}\n\nexport function scoreKnowledgeReadiness(\n options: ScoreKnowledgeReadinessOptions,\n): KnowledgeReadinessReport {\n const now = options.now ?? new Date()\n const requirements = options.requirements.map(normalizeRequirement)\n const missing = requirements.filter((requirement) => isRequirementMissing(requirement, now))\n const blockingMissingRequirements = missing.filter(isBlockingGap)\n const nonBlockingGaps = missing.filter((requirement) => !isBlockingGap(requirement))\n const readinessScore = weightedReadinessAt(requirements, now)\n const bundle: KnowledgeBundle = {\n taskId: options.taskId,\n requirements,\n evidenceIds: unique([\n ...(options.evidenceIds ?? []),\n ...requirements.flatMap((r) => r.evidenceIds),\n ]),\n claimIds: unique(options.claimIds ?? []),\n wikiPageIds: unique(options.wikiPageIds ?? []),\n userAnswers: options.userAnswers ?? {},\n missing,\n readinessScore,\n metadata: options.metadata,\n }\n const recommendedAction = chooseRecommendedAction(blockingMissingRequirements, nonBlockingGaps)\n const severity =\n blockingMissingRequirements.length > 0\n ? 'critical'\n : nonBlockingGaps.some((gap) => gap.importance === 'high')\n ? 'warning'\n : 'info'\n const reason =\n blockingMissingRequirements.length > 0\n ? `${blockingMissingRequirements.length} blocking knowledge requirement(s) are missing.`\n : nonBlockingGaps.length > 0\n ? `${nonBlockingGaps.length} non-blocking knowledge gap(s) remain.`\n : 'All declared knowledge requirements are ready.'\n\n return {\n taskId: options.taskId,\n readinessScore,\n blockingMissingRequirements,\n nonBlockingGaps,\n recommendedAction,\n bundle,\n severity,\n reason,\n }\n}\n\nexport function blockingKnowledgeEval(\n report: KnowledgeReadinessReport,\n options: { id?: string; minimumScore?: number; emitter?: TraceEmitter } = {},\n): ControlEvalResult {\n const minimumScore = options.minimumScore ?? 0.7\n const passed =\n report.blockingMissingRequirements.length === 0 && report.readinessScore >= minimumScore\n if (options.emitter) {\n void options.emitter\n .emit({\n kind: 'custom',\n payload: knowledgeReadinessTracePayload(report, { passed, minimumScore }),\n })\n .catch(() => undefined)\n }\n return objectiveEval({\n id: options.id ?? 'knowledge-ready',\n passed,\n score: report.readinessScore,\n severity: passed ? 'info' : report.severity,\n detail: report.reason,\n evidence: report.blockingMissingRequirements.map((r) => r.id).join(', ') || undefined,\n metadata: { knowledgeReadiness: report },\n })\n}\n\nexport function knowledgeReadinessTracePayload(\n report: KnowledgeReadinessReport,\n options: { passed?: boolean; minimumScore?: number } = {},\n): Record<string, unknown> {\n return {\n kind: 'readiness_scored',\n taskId: report.taskId,\n passed: options.passed ?? report.blockingMissingRequirements.length === 0,\n readinessScore: report.readinessScore,\n minimumScore: options.minimumScore,\n blockingRequirementIds: report.blockingMissingRequirements.map((r) => r.id),\n nonBlockingRequirementIds: report.nonBlockingGaps.map((r) => r.id),\n recommendedAction: report.recommendedAction,\n severity: report.severity,\n reason: report.reason,\n }\n}\n\nexport function userQuestionsForKnowledgeGaps(gaps: KnowledgeRequirement[]): UserQuestion[] {\n return gaps\n .filter((gap) => gap.acquisitionMode === 'ask_user' || gap.fallbackPolicy === 'ask')\n .map((gap) => ({\n id: `question_${gap.id}`,\n question: `Please provide: ${gap.description}`,\n reason: `Required for ${gap.requiredFor.join(', ') || 'the task'}.`,\n requirementId: gap.id,\n importance: gap.importance,\n answerType: gap.sensitivity === 'secret' ? 'credential' : 'free_text',\n impactIfUnknown: impactFor(gap),\n }))\n}\n\nexport function acquisitionPlansForKnowledgeGaps(\n gaps: KnowledgeRequirement[],\n): DataAcquisitionPlan[] {\n const byMode = new Map<string, KnowledgeRequirement[]>()\n for (const gap of gaps) {\n const mode = planMode(gap.acquisitionMode)\n if (!mode) continue\n const bucket = byMode.get(mode) ?? []\n bucket.push(gap)\n byMode.set(mode, bucket)\n }\n return [...byMode.entries()].map(([mode, requirements]) => ({\n id: `acquire_${mode}`,\n requirementIds: requirements.map((r) => r.id),\n mode: mode as DataAcquisitionPlan['mode'],\n description: descriptionForPlan(mode as DataAcquisitionPlan['mode'], requirements),\n priority: maxImportance(requirements.map((r) => r.importance)),\n questions: mode === 'ask_user' ? userQuestionsForKnowledgeGaps(requirements) : undefined,\n }))\n}\n\nfunction normalizeRequirement(requirement: KnowledgeRequirement): KnowledgeRequirement {\n return {\n ...requirement,\n confidenceNeeded: clamp01(requirement.confidenceNeeded),\n currentConfidence: clamp01(requirement.currentConfidence),\n evidenceIds: unique(requirement.evidenceIds),\n }\n}\n\nfunction weightedReadinessAt(requirements: KnowledgeRequirement[], now: Date): number {\n if (requirements.length === 0) return 1\n let weightSum = 0\n let scoreSum = 0\n for (const requirement of requirements) {\n const weight = importanceWeight(requirement.importance)\n const score = isExpired(requirement, now)\n ? 0\n : requirement.confidenceNeeded <= 0\n ? 1\n : Math.min(1, requirement.currentConfidence / requirement.confidenceNeeded)\n weightSum += weight\n scoreSum += weight * score\n }\n return clamp01(scoreSum / weightSum)\n}\n\nfunction isRequirementMissing(requirement: KnowledgeRequirement, now: Date): boolean {\n return isExpired(requirement, now) || requirement.currentConfidence < requirement.confidenceNeeded\n}\n\nfunction isExpired(requirement: KnowledgeRequirement, now: Date): boolean {\n if (!requirement.validUntil) return false\n const deadline = Date.parse(requirement.validUntil)\n if (!Number.isFinite(deadline)) return true\n return deadline <= now.getTime()\n}\n\nfunction isBlockingGap(requirement: KnowledgeRequirement): boolean {\n return (\n requirement.importance === 'blocking' ||\n requirement.fallbackPolicy === 'block' ||\n requirement.sensitivity === 'secret'\n )\n}\n\nfunction chooseRecommendedAction(\n blocking: KnowledgeRequirement[],\n nonBlocking: KnowledgeRequirement[],\n): KnowledgeRecommendedAction {\n const gaps = blocking.length > 0 ? blocking : nonBlocking\n if (gaps.length === 0) return 'run_agent'\n if (gaps.some((gap) => gap.acquisitionMode === 'ask_user' || gap.fallbackPolicy === 'ask'))\n return 'ask_user'\n if (gaps.some((gap) => gap.acquisitionMode === 'query_connector')) return 'query_connectors'\n if (\n gaps.some(\n (gap) => gap.acquisitionMode === 'inspect_repo' || gap.acquisitionMode === 'run_command',\n )\n )\n return 'inspect_repo'\n if (gaps.some((gap) => gap.acquisitionMode === 'search_web')) return 'collect_web_data'\n if (gaps.some((gap) => gap.acquisitionMode === 'not_available')) return 'abort_or_rescope'\n if (nonBlocking.some((gap) => gap.importance === 'high')) return 'build_domain_wiki'\n return 'continue_with_caveat'\n}\n\nfunction planMode(mode: KnowledgeAcquisitionMode): DataAcquisitionPlan['mode'] | null {\n if (mode === 'infer_low_confidence' || mode === 'not_available') return null\n return mode\n}\n\nfunction descriptionForPlan(\n mode: DataAcquisitionPlan['mode'],\n requirements: KnowledgeRequirement[],\n): string {\n const labels = requirements.map((r) => r.description).join('; ')\n if (mode === 'ask_user') return `Ask the user for: ${labels}`\n if (mode === 'search_web') return `Search web or documentation sources for: ${labels}`\n if (mode === 'query_connector') return `Query configured connectors for: ${labels}`\n if (mode === 'inspect_repo') return `Inspect repository context for: ${labels}`\n if (mode === 'run_command') return `Run local commands to collect: ${labels}`\n return `Build domain wiki evidence for: ${labels}`\n}\n\nfunction impactFor(requirement: KnowledgeRequirement): string {\n if (requirement.fallbackPolicy === 'block') return 'The agent should not run until this is known.'\n if (requirement.fallbackPolicy === 'continue_with_caveat')\n return 'The agent may continue, but must disclose uncertainty.'\n if (requirement.fallbackPolicy === 'use_default')\n return 'The agent will use the configured default if skipped.'\n return 'The agent should ask before continuing.'\n}\n\nfunction maxImportance(values: KnowledgeImportance[]): KnowledgeImportance {\n const order: KnowledgeImportance[] = ['blocking', 'high', 'medium', 'low']\n return order.find((value) => values.includes(value)) ?? 'low'\n}\n\nfunction importanceWeight(importance: KnowledgeImportance): number {\n if (importance === 'blocking') return 8\n if (importance === 'high') return 4\n if (importance === 'medium') return 2\n return 1\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n return Math.max(0, Math.min(1, value))\n}\n\nfunction unique<T>(items: T[]): T[] {\n return [...new Set(items)]\n}\n"],"mappings":";;;;;AAwBO,SAAS,wBACd,SAC0B;AAC1B,QAAM,MAAM,QAAQ,OAAO,oBAAI,KAAK;AACpC,QAAM,eAAe,QAAQ,aAAa,IAAI,oBAAoB;AAClE,QAAM,UAAU,aAAa,OAAO,CAAC,gBAAgB,qBAAqB,aAAa,GAAG,CAAC;AAC3F,QAAM,8BAA8B,QAAQ,OAAO,aAAa;AAChE,QAAM,kBAAkB,QAAQ,OAAO,CAAC,gBAAgB,CAAC,cAAc,WAAW,CAAC;AACnF,QAAM,iBAAiB,oBAAoB,cAAc,GAAG;AAC5D,QAAM,SAA0B;AAAA,IAC9B,QAAQ,QAAQ;AAAA,IAChB;AAAA,IACA,aAAa,OAAO;AAAA,MAClB,GAAI,QAAQ,eAAe,CAAC;AAAA,MAC5B,GAAG,aAAa,QAAQ,CAAC,MAAM,EAAE,WAAW;AAAA,IAC9C,CAAC;AAAA,IACD,UAAU,OAAO,QAAQ,YAAY,CAAC,CAAC;AAAA,IACvC,aAAa,OAAO,QAAQ,eAAe,CAAC,CAAC;AAAA,IAC7C,aAAa,QAAQ,eAAe,CAAC;AAAA,IACrC;AAAA,IACA;AAAA,IACA,UAAU,QAAQ;AAAA,EACpB;AACA,QAAM,oBAAoB,wBAAwB,6BAA6B,eAAe;AAC9F,QAAM,WACJ,4BAA4B,SAAS,IACjC,aACA,gBAAgB,KAAK,CAAC,QAAQ,IAAI,eAAe,MAAM,IACrD,YACA;AACR,QAAM,SACJ,4BAA4B,SAAS,IACjC,GAAG,4BAA4B,MAAM,oDACrC,gBAAgB,SAAS,IACvB,GAAG,gBAAgB,MAAM,2CACzB;AAER,SAAO;AAAA,IACL,QAAQ,QAAQ;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,sBACd,QACA,UAA0E,CAAC,GACxD;AACnB,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,SACJ,OAAO,4BAA4B,WAAW,KAAK,OAAO,kBAAkB;AAC9E,MAAI,QAAQ,SAAS;AACnB,SAAK,QAAQ,QACV,KAAK;AAAA,MACJ,MAAM;AAAA,MACN,SAAS,+BAA+B,QAAQ,EAAE,QAAQ,aAAa,CAAC;AAAA,IAC1E,CAAC,EACA,MAAM,MAAM,MAAS;AAAA,EAC1B;AACA,SAAO,cAAc;AAAA,IACnB,IAAI,QAAQ,MAAM;AAAA,IAClB;AAAA,IACA,OAAO,OAAO;AAAA,IACd,UAAU,SAAS,SAAS,OAAO;AAAA,IACnC,QAAQ,OAAO;AAAA,IACf,UAAU,OAAO,4BAA4B,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,IAAI,KAAK;AAAA,IAC5E,UAAU,EAAE,oBAAoB,OAAO;AAAA,EACzC,CAAC;AACH;AAEO,SAAS,+BACd,QACA,UAAuD,CAAC,GAC/B;AACzB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,QAAQ,OAAO;AAAA,IACf,QAAQ,QAAQ,UAAU,OAAO,4BAA4B,WAAW;AAAA,IACxE,gBAAgB,OAAO;AAAA,IACvB,cAAc,QAAQ;AAAA,IACtB,wBAAwB,OAAO,4BAA4B,IAAI,CAAC,MAAM,EAAE,EAAE;AAAA,IAC1E,2BAA2B,OAAO,gBAAgB,IAAI,CAAC,MAAM,EAAE,EAAE;AAAA,IACjE,mBAAmB,OAAO;AAAA,IAC1B,UAAU,OAAO;AAAA,IACjB,QAAQ,OAAO;AAAA,EACjB;AACF;AAEO,SAAS,8BAA8B,MAA8C;AAC1F,SAAO,KACJ,OAAO,CAAC,QAAQ,IAAI,oBAAoB,cAAc,IAAI,mBAAmB,KAAK,EAClF,IAAI,CAAC,SAAS;AAAA,IACb,IAAI,YAAY,IAAI,EAAE;AAAA,IACtB,UAAU,mBAAmB,IAAI,WAAW;AAAA,IAC5C,QAAQ,gBAAgB,IAAI,YAAY,KAAK,IAAI,KAAK,UAAU;AAAA,IAChE,eAAe,IAAI;AAAA,IACnB,YAAY,IAAI;AAAA,IAChB,YAAY,IAAI,gBAAgB,WAAW,eAAe;AAAA,IAC1D,iBAAiB,UAAU,GAAG;AAAA,EAChC,EAAE;AACN;AAEO,SAAS,iCACd,MACuB;AACvB,QAAM,SAAS,oBAAI,IAAoC;AACvD,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,SAAS,IAAI,eAAe;AACzC,QAAI,CAAC,KAAM;AACX,UAAM,SAAS,OAAO,IAAI,IAAI,KAAK,CAAC;AACpC,WAAO,KAAK,GAAG;AACf,WAAO,IAAI,MAAM,MAAM;AAAA,EACzB;AACA,SAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,YAAY,OAAO;AAAA,IAC1D,IAAI,WAAW,IAAI;AAAA,IACnB,gBAAgB,aAAa,IAAI,CAAC,MAAM,EAAE,EAAE;AAAA,IAC5C;AAAA,IACA,aAAa,mBAAmB,MAAqC,YAAY;AAAA,IACjF,UAAU,cAAc,aAAa,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC;AAAA,IAC7D,WAAW,SAAS,aAAa,8BAA8B,YAAY,IAAI;AAAA,EACjF,EAAE;AACJ;AAEA,SAAS,qBAAqB,aAAyD;AACrF,SAAO;AAAA,IACL,GAAG;AAAA,IACH,kBAAkB,QAAQ,YAAY,gBAAgB;AAAA,IACtD,mBAAmB,QAAQ,YAAY,iBAAiB;AAAA,IACxD,aAAa,OAAO,YAAY,WAAW;AAAA,EAC7C;AACF;AAEA,SAAS,oBAAoB,cAAsC,KAAmB;AACpF,MAAI,aAAa,WAAW,EAAG,QAAO;AACtC,MAAI,YAAY;AAChB,MAAI,WAAW;AACf,aAAW,eAAe,cAAc;AACtC,UAAM,SAAS,iBAAiB,YAAY,UAAU;AACtD,UAAM,QAAQ,UAAU,aAAa,GAAG,IACpC,IACA,YAAY,oBAAoB,IAC9B,IACA,KAAK,IAAI,GAAG,YAAY,oBAAoB,YAAY,gBAAgB;AAC9E,iBAAa;AACb,gBAAY,SAAS;AAAA,EACvB;AACA,SAAO,QAAQ,WAAW,SAAS;AACrC;AAEA,SAAS,qBAAqB,aAAmC,KAAoB;AACnF,SAAO,UAAU,aAAa,GAAG,KAAK,YAAY,oBAAoB,YAAY;AACpF;AAEA,SAAS,UAAU,aAAmC,KAAoB;AACxE,MAAI,CAAC,YAAY,WAAY,QAAO;AACpC,QAAM,WAAW,KAAK,MAAM,YAAY,UAAU;AAClD,MAAI,CAAC,OAAO,SAAS,QAAQ,EAAG,QAAO;AACvC,SAAO,YAAY,IAAI,QAAQ;AACjC;AAEA,SAAS,cAAc,aAA4C;AACjE,SACE,YAAY,eAAe,cAC3B,YAAY,mBAAmB,WAC/B,YAAY,gBAAgB;AAEhC;AAEA,SAAS,wBACP,UACA,aAC4B;AAC5B,QAAM,OAAO,SAAS,SAAS,IAAI,WAAW;AAC9C,MAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,MAAI,KAAK,KAAK,CAAC,QAAQ,IAAI,oBAAoB,cAAc,IAAI,mBAAmB,KAAK;AACvF,WAAO;AACT,MAAI,KAAK,KAAK,CAAC,QAAQ,IAAI,oBAAoB,iBAAiB,EAAG,QAAO;AAC1E,MACE,KAAK;AAAA,IACH,CAAC,QAAQ,IAAI,oBAAoB,kBAAkB,IAAI,oBAAoB;AAAA,EAC7E;AAEA,WAAO;AACT,MAAI,KAAK,KAAK,CAAC,QAAQ,IAAI,oBAAoB,YAAY,EAAG,QAAO;AACrE,MAAI,KAAK,KAAK,CAAC,QAAQ,IAAI,oBAAoB,eAAe,EAAG,QAAO;AACxE,MAAI,YAAY,KAAK,CAAC,QAAQ,IAAI,eAAe,MAAM,EAAG,QAAO;AACjE,SAAO;AACT;AAEA,SAAS,SAAS,MAAoE;AACpF,MAAI,SAAS,0BAA0B,SAAS,gBAAiB,QAAO;AACxE,SAAO;AACT;AAEA,SAAS,mBACP,MACA,cACQ;AACR,QAAM,SAAS,aAAa,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,IAAI;AAC/D,MAAI,SAAS,WAAY,QAAO,qBAAqB,MAAM;AAC3D,MAAI,SAAS,aAAc,QAAO,4CAA4C,MAAM;AACpF,MAAI,SAAS,kBAAmB,QAAO,oCAAoC,MAAM;AACjF,MAAI,SAAS,eAAgB,QAAO,mCAAmC,MAAM;AAC7E,MAAI,SAAS,cAAe,QAAO,kCAAkC,MAAM;AAC3E,SAAO,mCAAmC,MAAM;AAClD;AAEA,SAAS,UAAU,aAA2C;AAC5D,MAAI,YAAY,mBAAmB,QAAS,QAAO;AACnD,MAAI,YAAY,mBAAmB;AACjC,WAAO;AACT,MAAI,YAAY,mBAAmB;AACjC,WAAO;AACT,SAAO;AACT;AAEA,SAAS,cAAc,QAAoD;AACzE,QAAM,QAA+B,CAAC,YAAY,QAAQ,UAAU,KAAK;AACzE,SAAO,MAAM,KAAK,CAAC,UAAU,OAAO,SAAS,KAAK,CAAC,KAAK;AAC1D;AAEA,SAAS,iBAAiB,YAAyC;AACjE,MAAI,eAAe,WAAY,QAAO;AACtC,MAAI,eAAe,OAAQ,QAAO;AAClC,MAAI,eAAe,SAAU,QAAO;AACpC,SAAO;AACT;AAEA,SAAS,QAAQ,OAAuB;AACtC,MAAI,CAAC,OAAO,SAAS,KAAK,EAAG,QAAO;AACpC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAEA,SAAS,OAAU,OAAiB;AAClC,SAAO,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC;AAC3B;","names":[]}
@@ -1,49 +0,0 @@
1
- // src/trajectory.ts
2
- async function buildTrajectory(store, runId) {
3
- const spans = await store.spans({ runId });
4
- const events = await store.events({ runId });
5
- const childrenOf = /* @__PURE__ */ new Map();
6
- for (const s of spans) {
7
- const arr = childrenOf.get(s.parentSpanId) ?? [];
8
- arr.push(s);
9
- childrenOf.set(s.parentSpanId, arr);
10
- }
11
- for (const arr of childrenOf.values()) arr.sort((a, b) => a.startedAt - b.startedAt);
12
- const eventsBySpan = /* @__PURE__ */ new Map();
13
- for (const e of events) {
14
- if (!e.spanId) continue;
15
- const arr = eventsBySpan.get(e.spanId) ?? [];
16
- arr.push(e);
17
- eventsBySpan.set(e.spanId, arr);
18
- }
19
- const steps = [];
20
- const walk = (spanId, depth) => {
21
- const kids = childrenOf.get(spanId) ?? [];
22
- for (const child of kids) {
23
- steps.push({
24
- index: steps.length,
25
- span: child,
26
- depth,
27
- events: eventsBySpan.get(child.spanId) ?? []
28
- });
29
- walk(child.spanId, depth + 1);
30
- }
31
- };
32
- walk(void 0, 0);
33
- const llmTurns = steps.filter((s) => s.span.kind === "llm").length;
34
- const toolCalls = steps.filter((s) => s.span.kind === "tool").length;
35
- const judgeVerdicts = steps.filter((s) => s.span.kind === "judge").length;
36
- const retrievals = steps.filter((s) => s.span.kind === "retrieval").length;
37
- let totalDurationMs = 0;
38
- if (steps.length > 0) {
39
- const starts = spans.map((s) => s.startedAt);
40
- const ends = spans.map((s) => s.endedAt ?? s.startedAt);
41
- totalDurationMs = Math.max(...ends) - Math.min(...starts);
42
- }
43
- return { runId, steps, llmTurns, toolCalls, judgeVerdicts, retrievals, totalDurationMs };
44
- }
45
-
46
- export {
47
- buildTrajectory
48
- };
49
- //# sourceMappingURL=chunk-RZTMDUO7.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/trajectory.ts"],"sourcesContent":["/**\n * Trajectory — ordered, structured view over a run's spans.\n *\n * A pure function `buildTrajectory(store, runId) → Trajectory` returns\n * a topologically ordered list of `TrajectoryStep` with parent-child\n * grouping collapsed into a single line-of-agent-work. Separate\n * analyzers (stuck-loop detection, waste ratio) live in\n * `pipelines/` and consume the trajectory.\n */\n\nimport type { Span, TraceEvent } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface TrajectoryStep {\n index: number\n span: Span\n /** Depth in the span tree from the root. 0 = top-level. */\n depth: number\n /** Events attached to this span. */\n events: TraceEvent[]\n}\n\nexport interface Trajectory {\n runId: string\n steps: TrajectoryStep[]\n llmTurns: number\n toolCalls: number\n judgeVerdicts: number\n retrievals: number\n totalDurationMs: number\n}\n\nexport async function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory> {\n const spans = await store.spans({ runId })\n const events = await store.events({ runId })\n const childrenOf = new Map<string | undefined, Span[]>()\n for (const s of spans) {\n const arr = childrenOf.get(s.parentSpanId) ?? []\n arr.push(s)\n childrenOf.set(s.parentSpanId, arr)\n }\n // Sort children by startedAt so DFS yields chronological order within siblings.\n for (const arr of childrenOf.values()) arr.sort((a, b) => a.startedAt - b.startedAt)\n\n const eventsBySpan = new Map<string, TraceEvent[]>()\n for (const e of events) {\n if (!e.spanId) continue\n const arr = eventsBySpan.get(e.spanId) ?? []\n arr.push(e)\n eventsBySpan.set(e.spanId, arr)\n }\n\n const steps: TrajectoryStep[] = []\n const walk = (spanId: string | undefined, depth: number): void => {\n const kids = childrenOf.get(spanId) ?? []\n for (const child of kids) {\n steps.push({\n index: steps.length,\n span: child,\n depth,\n events: eventsBySpan.get(child.spanId) ?? [],\n })\n walk(child.spanId, depth + 1)\n }\n }\n walk(undefined, 0)\n\n const llmTurns = steps.filter((s) => s.span.kind === 'llm').length\n const toolCalls = steps.filter((s) => s.span.kind === 'tool').length\n const judgeVerdicts = steps.filter((s) => s.span.kind === 'judge').length\n const retrievals = steps.filter((s) => s.span.kind === 'retrieval').length\n\n let totalDurationMs = 0\n if (steps.length > 0) {\n const starts = spans.map((s) => s.startedAt)\n const ends = spans.map((s) => s.endedAt ?? s.startedAt)\n totalDurationMs = Math.max(...ends) - Math.min(...starts)\n }\n\n return { runId, steps, llmTurns, toolCalls, judgeVerdicts, retrievals, totalDurationMs }\n}\n\n// Re-export core types for convenience so consumers don't import from two paths.\nexport type { Span, TraceEvent } from './trace/schema'\nexport type { TraceStore } from './trace/store'\n"],"mappings":";AAgCA,eAAsB,gBAAgB,OAAmB,OAAoC;AAC3F,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,CAAC;AACzC,QAAM,SAAS,MAAM,MAAM,OAAO,EAAE,MAAM,CAAC;AAC3C,QAAM,aAAa,oBAAI,IAAgC;AACvD,aAAW,KAAK,OAAO;AACrB,UAAM,MAAM,WAAW,IAAI,EAAE,YAAY,KAAK,CAAC;AAC/C,QAAI,KAAK,CAAC;AACV,eAAW,IAAI,EAAE,cAAc,GAAG;AAAA,EACpC;AAEA,aAAW,OAAO,WAAW,OAAO,EAAG,KAAI,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAEnF,QAAM,eAAe,oBAAI,IAA0B;AACnD,aAAW,KAAK,QAAQ;AACtB,QAAI,CAAC,EAAE,OAAQ;AACf,UAAM,MAAM,aAAa,IAAI,EAAE,MAAM,KAAK,CAAC;AAC3C,QAAI,KAAK,CAAC;AACV,iBAAa,IAAI,EAAE,QAAQ,GAAG;AAAA,EAChC;AAEA,QAAM,QAA0B,CAAC;AACjC,QAAM,OAAO,CAAC,QAA4B,UAAwB;AAChE,UAAM,OAAO,WAAW,IAAI,MAAM,KAAK,CAAC;AACxC,eAAW,SAAS,MAAM;AACxB,YAAM,KAAK;AAAA,QACT,OAAO,MAAM;AAAA,QACb,MAAM;AAAA,QACN;AAAA,QACA,QAAQ,aAAa,IAAI,MAAM,MAAM,KAAK,CAAC;AAAA,MAC7C,CAAC;AACD,WAAK,MAAM,QAAQ,QAAQ,CAAC;AAAA,IAC9B;AAAA,EACF;AACA,OAAK,QAAW,CAAC;AAEjB,QAAM,WAAW,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,KAAK,EAAE;AAC5D,QAAM,YAAY,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,MAAM,EAAE;AAC9D,QAAM,gBAAgB,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,OAAO,EAAE;AACnE,QAAM,aAAa,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,WAAW,EAAE;AAEpE,MAAI,kBAAkB;AACtB,MAAI,MAAM,SAAS,GAAG;AACpB,UAAM,SAAS,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS;AAC3C,UAAM,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS;AACtD,sBAAkB,KAAK,IAAI,GAAG,IAAI,IAAI,KAAK,IAAI,GAAG,MAAM;AAAA,EAC1D;AAEA,SAAO,EAAE,OAAO,OAAO,UAAU,WAAW,eAAe,YAAY,gBAAgB;AACzF;","names":[]}
@@ -1,202 +0,0 @@
1
- // src/perf/integrity.ts
2
- function isMissing(value) {
3
- return value === null || value === void 0;
4
- }
5
- function checkRecordIntegrity(records, resolveJourney) {
6
- const violations = [];
7
- for (const [recordIndex, record] of records.entries()) {
8
- if (record.pass !== true) continue;
9
- const journey = resolveJourney(record);
10
- if (journey === null) continue;
11
- for (const field of journey.requiredFields) {
12
- if (isMissing(record[field])) {
13
- violations.push({
14
- recordIndex,
15
- journeyId: journey.id,
16
- field,
17
- reason: "null-required-field",
18
- detail: `required field '${field}' is ${record[field] === null ? "null" : "undefined"} on a passing '${journey.id}' record`
19
- });
20
- }
21
- }
22
- for (const field of journey.phaseFields ?? []) {
23
- if (isMissing(record[field])) {
24
- violations.push({
25
- recordIndex,
26
- journeyId: journey.id,
27
- field,
28
- reason: "null-required-field",
29
- detail: `phase field '${field}' is ${record[field] === null ? "null" : "undefined"} on a passing '${journey.id}' record`
30
- });
31
- }
32
- }
33
- for (const { field, min } of journey.minimums ?? []) {
34
- const value = record[field];
35
- if (isMissing(value)) continue;
36
- if (typeof value !== "number" || Number.isNaN(value)) {
37
- violations.push({
38
- recordIndex,
39
- journeyId: journey.id,
40
- field,
41
- reason: "below-minimum",
42
- detail: `field '${field}' has non-numeric value ${JSON.stringify(value)} on a passing '${journey.id}' record (minimum ${min})`
43
- });
44
- continue;
45
- }
46
- if (value < min) {
47
- violations.push({
48
- recordIndex,
49
- journeyId: journey.id,
50
- field,
51
- reason: "below-minimum",
52
- detail: `field '${field}' is ${value}, below minimum ${min} on a passing '${journey.id}' record`
53
- });
54
- }
55
- }
56
- }
57
- return { succeeded: violations.length === 0, violations };
58
- }
59
- function assertRecordIntegrity(records, resolveJourney) {
60
- const result = checkRecordIntegrity(records, resolveJourney);
61
- if (result.succeeded) return;
62
- const lines = result.violations.map(
63
- (v) => ` [record ${v.recordIndex}] ${v.journeyId}.${v.field} (${v.reason}): ${v.detail}`
64
- );
65
- throw new Error(
66
- `Record integrity check failed with ${result.violations.length} violation(s):
67
- ${lines.join("\n")}`
68
- );
69
- }
70
-
71
- // src/perf/journey.ts
72
- function scenarioKey(journeyId, axes) {
73
- const parts = Object.keys(axes).sort().map((dim) => `${dim}=${axes[dim]}`);
74
- return [journeyId, ...parts].join("|");
75
- }
76
- function expandMatrix(journeys, axes, filter) {
77
- const dims = Object.keys(axes).sort();
78
- let combos = [{}];
79
- for (const dim of dims) {
80
- const values = axes[dim];
81
- const next = [];
82
- for (const combo of combos) {
83
- for (const value of values) {
84
- next.push({ ...combo, [dim]: value });
85
- }
86
- }
87
- combos = next;
88
- }
89
- const scenarios = [];
90
- for (const journey of journeys) {
91
- for (const combo of combos) {
92
- if (filter && !filter(journey.id, combo)) continue;
93
- scenarios.push({ key: scenarioKey(journey.id, combo), journey, axes: combo });
94
- }
95
- }
96
- return scenarios;
97
- }
98
-
99
- // src/perf/ratchet.ts
100
- function nearestRank(sorted, p) {
101
- if (sorted.length === 0) throw new Error("nearestRank requires at least one sample");
102
- const rank = Math.max(1, Math.ceil(p / 100 * sorted.length));
103
- return sorted[rank - 1];
104
- }
105
- function summarizeRecords(records, keyOf, metricFields) {
106
- const samples = /* @__PURE__ */ new Map();
107
- for (const record of records) {
108
- const key = keyOf(record);
109
- if (key === null) continue;
110
- let byField = samples.get(key);
111
- if (!byField) {
112
- byField = /* @__PURE__ */ new Map();
113
- samples.set(key, byField);
114
- }
115
- for (const field of metricFields) {
116
- const value = record[field];
117
- if (typeof value !== "number" || Number.isNaN(value)) continue;
118
- let values = byField.get(field);
119
- if (!values) {
120
- values = [];
121
- byField.set(field, values);
122
- }
123
- values.push(value);
124
- }
125
- }
126
- const scenarios = {};
127
- for (const [key, byField] of samples) {
128
- const stats = {};
129
- for (const [field, values] of byField) {
130
- if (values.length === 0) continue;
131
- const sorted = [...values].sort((a, b) => a - b);
132
- stats[field] = {
133
- p50: nearestRank(sorted, 50),
134
- p90: nearestRank(sorted, 90),
135
- n: sorted.length
136
- };
137
- }
138
- scenarios[key] = stats;
139
- }
140
- return { version: 1, scenarios };
141
- }
142
- function pctOver(currentValue, baselineValue) {
143
- if (baselineValue === 0) {
144
- return currentValue === 0 ? 0 : Number.POSITIVE_INFINITY;
145
- }
146
- return (currentValue - baselineValue) / baselineValue * 100;
147
- }
148
- function gatePerf(current, baseline, options) {
149
- const tolerancePct = options?.tolerancePct ?? 10;
150
- const minSamples = options?.minSamples ?? 3;
151
- const regressions = [];
152
- const improvements = [];
153
- const missing = /* @__PURE__ */ new Set();
154
- for (const [scenarioKey2, baselineStats] of Object.entries(baseline.scenarios)) {
155
- const currentStats = current.scenarios[scenarioKey2];
156
- if (!currentStats) {
157
- missing.add(scenarioKey2);
158
- continue;
159
- }
160
- for (const [field, baselineStat] of Object.entries(baselineStats)) {
161
- const currentStat = currentStats[field];
162
- if (!currentStat || currentStat.n < minSamples) {
163
- missing.add(scenarioKey2);
164
- continue;
165
- }
166
- const overBy = {
167
- p50Pct: pctOver(currentStat.p50, baselineStat.p50),
168
- p90Pct: pctOver(currentStat.p90, baselineStat.p90)
169
- };
170
- const entry = {
171
- scenarioKey: scenarioKey2,
172
- field,
173
- baseline: baselineStat,
174
- current: currentStat,
175
- overBy
176
- };
177
- if (overBy.p50Pct > tolerancePct || overBy.p90Pct > tolerancePct) {
178
- regressions.push(entry);
179
- } else if (overBy.p50Pct < 0 && overBy.p90Pct < 0) {
180
- improvements.push(entry);
181
- }
182
- }
183
- }
184
- const newScenarios = Object.keys(current.scenarios).filter((key) => !(key in baseline.scenarios));
185
- return {
186
- succeeded: regressions.length === 0,
187
- regressions,
188
- improvements,
189
- missingScenarios: [...missing],
190
- newScenarios
191
- };
192
- }
193
-
194
- export {
195
- checkRecordIntegrity,
196
- assertRecordIntegrity,
197
- scenarioKey,
198
- expandMatrix,
199
- summarizeRecords,
200
- gatePerf
201
- };
202
- //# sourceMappingURL=chunk-STGVSCDH.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/perf/integrity.ts","../src/perf/journey.ts","../src/perf/ratchet.ts"],"sourcesContent":["/**\n * Record-integrity contracts for perf metric records.\n *\n * A record that claims `pass === true` must actually carry the journey's\n * required measurements — a \"passing\" provision run with a null\n * `total_ms` is a lying record, not a pass. Failed records are exempt:\n * a run that errored mid-flight legitimately has nulls.\n */\n\nimport type { JourneySpec } from './journey'\n\nexport interface IntegrityViolation {\n recordIndex: number\n journeyId: string\n field: string\n reason: 'null-required-field' | 'below-minimum'\n detail: string\n}\n\nexport interface IntegrityResult {\n succeeded: boolean\n violations: IntegrityViolation[]\n}\n\nfunction isMissing(value: unknown): boolean {\n return value === null || value === undefined\n}\n\n/**\n * Validates flat metric records (Record<string, unknown> with a boolean\n * `pass` field) against their journey contract. Only records with\n * pass === true are checked — a failed record may legitimately have nulls.\n * resolveJourney maps a record to its JourneySpec (or null to skip).\n */\nexport function checkRecordIntegrity(\n records: ReadonlyArray<Record<string, unknown>>,\n resolveJourney: (record: Record<string, unknown>) => JourneySpec | null,\n): IntegrityResult {\n const violations: IntegrityViolation[] = []\n for (const [recordIndex, record] of records.entries()) {\n if (record.pass !== true) continue\n const journey = resolveJourney(record)\n if (journey === null) continue\n for (const field of journey.requiredFields) {\n if (isMissing(record[field])) {\n violations.push({\n recordIndex,\n journeyId: journey.id,\n field,\n reason: 'null-required-field',\n detail: `required field '${field}' is ${record[field] === null ? 'null' : 'undefined'} on a passing '${journey.id}' record`,\n })\n }\n }\n for (const field of journey.phaseFields ?? []) {\n if (isMissing(record[field])) {\n violations.push({\n recordIndex,\n journeyId: journey.id,\n field,\n reason: 'null-required-field',\n detail: `phase field '${field}' is ${record[field] === null ? 'null' : 'undefined'} on a passing '${journey.id}' record`,\n })\n }\n }\n for (const { field, min } of journey.minimums ?? []) {\n const value = record[field]\n if (isMissing(value)) continue // null-ness is the required/phase fields' contract\n if (typeof value !== 'number' || Number.isNaN(value)) {\n violations.push({\n recordIndex,\n journeyId: journey.id,\n field,\n reason: 'below-minimum',\n detail: `field '${field}' has non-numeric value ${JSON.stringify(value)} on a passing '${journey.id}' record (minimum ${min})`,\n })\n continue\n }\n if (value < min) {\n violations.push({\n recordIndex,\n journeyId: journey.id,\n field,\n reason: 'below-minimum',\n detail: `field '${field}' is ${value}, below minimum ${min} on a passing '${journey.id}' record`,\n })\n }\n }\n }\n return { succeeded: violations.length === 0, violations }\n}\n\n/** Throws an Error listing every violation when the result fails. */\nexport function assertRecordIntegrity(\n records: ReadonlyArray<Record<string, unknown>>,\n resolveJourney: (record: Record<string, unknown>) => JourneySpec | null,\n): void {\n const result = checkRecordIntegrity(records, resolveJourney)\n if (result.succeeded) return\n const lines = result.violations.map(\n (v) => ` [record ${v.recordIndex}] ${v.journeyId}.${v.field} (${v.reason}): ${v.detail}`,\n )\n throw new Error(\n `Record integrity check failed with ${result.violations.length} violation(s):\\n${lines.join('\\n')}`,\n )\n}\n","/**\n * Journey × axes matrix for infra performance benchmarks.\n *\n * A journey is one measurable user path (\"provision.cold\", \"chat.ttft\");\n * axes are free-form scenario dimensions (driver, region, image…). The\n * matrix expansion is pure bookkeeping — running the scenarios and\n * recording metrics is the caller's job. This module complements the\n * judge-panel `BenchmarkRunner` (src/benchmark.ts): that one scores\n * QUALITY via judges, this one structures LATENCY / RELIABILITY runs\n * over flat metric records.\n */\n\n/** One measurable user journey (e.g. \"provision.cold\", \"chat.ttft\"). */\nexport interface JourneySpec {\n id: string\n description: string\n /** Needs a real LLM call — schedule nightly, not per-PR. */\n requiresLLM: boolean\n /**\n * Fields that MUST be non-null on a passing record of this journey.\n * A \"passing\" record missing one is an integrity violation, not a pass.\n */\n requiredFields: ReadonlyArray<string>\n /** Numeric floors, e.g. {field: 'event_count', min: 1} for streaming. */\n minimums?: ReadonlyArray<{ field: string; min: number }>\n /** Per-phase breakdown fields expected non-null (subset of requiredFields semantics, reported separately). */\n phaseFields?: ReadonlyArray<string>\n}\n\nexport interface ScenarioAxes {\n /** e.g. driver: ['docker','firecracker'] — every key is a free-form dimension. */\n [dimension: string]: ReadonlyArray<string>\n}\n\nexport interface PerfScenario {\n /** `${journeyId}|${dim1}=${v1}|${dim2}=${v2}` (dims sorted). */\n key: string\n journey: JourneySpec\n axes: Record<string, string>\n}\n\n/** Stable scenario key: journey id then `dim=value` pairs in sorted-dim order. */\nexport function scenarioKey(journeyId: string, axes: Record<string, string>): string {\n const parts = Object.keys(axes)\n .sort()\n .map((dim) => `${dim}=${axes[dim]}`)\n return [journeyId, ...parts].join('|')\n}\n\n/** Cartesian expansion; `filter` lets callers drop invalid combos (e.g. firecracker×resume). */\nexport function expandMatrix(\n journeys: ReadonlyArray<JourneySpec>,\n axes: ScenarioAxes,\n filter?: (journeyId: string, combo: Record<string, string>) => boolean,\n): PerfScenario[] {\n const dims = Object.keys(axes).sort()\n let combos: Record<string, string>[] = [{}]\n for (const dim of dims) {\n const values = axes[dim] as ReadonlyArray<string>\n const next: Record<string, string>[] = []\n for (const combo of combos) {\n for (const value of values) {\n next.push({ ...combo, [dim]: value })\n }\n }\n combos = next\n }\n const scenarios: PerfScenario[] = []\n for (const journey of journeys) {\n for (const combo of combos) {\n if (filter && !filter(journey.id, combo)) continue\n scenarios.push({ key: scenarioKey(journey.id, combo), journey, axes: combo })\n }\n }\n return scenarios\n}\n","/**\n * Percentile ratchet over perf metric records.\n *\n * `summarizeRecords` folds flat records into per-scenario p50/p90 stats;\n * `gatePerf` compares a current summary against a committed baseline and\n * trips on regressions beyond tolerance. Percentiles use nearest-rank on\n * sorted values. Null / non-numeric metric values are excluded from the\n * stat (n reflects only real samples); a field with zero samples is\n * omitted entirely — no fake zeros.\n */\n\nexport interface PerfStat {\n p50: number\n p90: number\n n: number\n}\n\nexport interface PerfBaseline {\n version: 1\n /** key → metric field → stat. */\n scenarios: Record<string, Record<string, PerfStat>>\n}\n\nexport interface PerfRegression {\n scenarioKey: string\n field: string\n baseline: PerfStat\n current: PerfStat\n /** percent over baseline p50 / p90, whichever tripped. */\n overBy: { p50Pct: number; p90Pct: number }\n}\n\nexport interface PerfGateResult {\n succeeded: boolean\n regressions: PerfRegression[]\n /** Negative overBy: strictly better than baseline on both percentiles. */\n improvements: PerfRegression[]\n /** In baseline but absent (or under-sampled, n < minSamples) in current. */\n missingScenarios: string[]\n /** In records but absent from baseline. */\n newScenarios: string[]\n}\n\n/** Nearest-rank percentile over a non-empty ascending-sorted array (p in (0, 100]). */\nfunction nearestRank(sorted: ReadonlyArray<number>, p: number): number {\n if (sorted.length === 0) throw new Error('nearestRank requires at least one sample')\n const rank = Math.max(1, Math.ceil((p / 100) * sorted.length))\n return sorted[rank - 1] as number\n}\n\nexport function summarizeRecords(\n records: ReadonlyArray<Record<string, unknown>>,\n keyOf: (record: Record<string, unknown>) => string | null,\n metricFields: ReadonlyArray<string>,\n): PerfBaseline {\n const samples = new Map<string, Map<string, number[]>>()\n for (const record of records) {\n const key = keyOf(record)\n if (key === null) continue\n let byField = samples.get(key)\n if (!byField) {\n byField = new Map()\n samples.set(key, byField)\n }\n for (const field of metricFields) {\n const value = record[field]\n if (typeof value !== 'number' || Number.isNaN(value)) continue\n let values = byField.get(field)\n if (!values) {\n values = []\n byField.set(field, values)\n }\n values.push(value)\n }\n }\n const scenarios: Record<string, Record<string, PerfStat>> = {}\n for (const [key, byField] of samples) {\n const stats: Record<string, PerfStat> = {}\n for (const [field, values] of byField) {\n if (values.length === 0) continue\n const sorted = [...values].sort((a, b) => a - b)\n stats[field] = {\n p50: nearestRank(sorted, 50),\n p90: nearestRank(sorted, 90),\n n: sorted.length,\n }\n }\n scenarios[key] = stats\n }\n return { version: 1, scenarios }\n}\n\n/** Percent over baseline; baseline 0 → 0% when equal, Infinity when current grew. */\nfunction pctOver(currentValue: number, baselineValue: number): number {\n if (baselineValue === 0) {\n return currentValue === 0 ? 0 : Number.POSITIVE_INFINITY\n }\n return ((currentValue - baselineValue) / baselineValue) * 100\n}\n\nexport function gatePerf(\n current: PerfBaseline,\n baseline: PerfBaseline,\n options?: { tolerancePct?: number; minSamples?: number },\n): PerfGateResult {\n const tolerancePct = options?.tolerancePct ?? 10\n const minSamples = options?.minSamples ?? 3\n const regressions: PerfRegression[] = []\n const improvements: PerfRegression[] = []\n const missing = new Set<string>()\n\n for (const [scenarioKey, baselineStats] of Object.entries(baseline.scenarios)) {\n const currentStats = current.scenarios[scenarioKey]\n if (!currentStats) {\n missing.add(scenarioKey)\n continue\n }\n for (const [field, baselineStat] of Object.entries(baselineStats)) {\n const currentStat = currentStats[field]\n if (!currentStat || currentStat.n < minSamples) {\n // Under-sampled current data cannot gate — surface it instead of\n // pretending the scenario passed.\n missing.add(scenarioKey)\n continue\n }\n const overBy = {\n p50Pct: pctOver(currentStat.p50, baselineStat.p50),\n p90Pct: pctOver(currentStat.p90, baselineStat.p90),\n }\n const entry: PerfRegression = {\n scenarioKey,\n field,\n baseline: baselineStat,\n current: currentStat,\n overBy,\n }\n if (overBy.p50Pct > tolerancePct || overBy.p90Pct > tolerancePct) {\n regressions.push(entry)\n } else if (overBy.p50Pct < 0 && overBy.p90Pct < 0) {\n improvements.push(entry)\n }\n }\n }\n\n const newScenarios = Object.keys(current.scenarios).filter((key) => !(key in baseline.scenarios))\n\n return {\n succeeded: regressions.length === 0,\n regressions,\n improvements,\n missingScenarios: [...missing],\n newScenarios,\n }\n}\n"],"mappings":";AAwBA,SAAS,UAAU,OAAyB;AAC1C,SAAO,UAAU,QAAQ,UAAU;AACrC;AAQO,SAAS,qBACd,SACA,gBACiB;AACjB,QAAM,aAAmC,CAAC;AAC1C,aAAW,CAAC,aAAa,MAAM,KAAK,QAAQ,QAAQ,GAAG;AACrD,QAAI,OAAO,SAAS,KAAM;AAC1B,UAAM,UAAU,eAAe,MAAM;AACrC,QAAI,YAAY,KAAM;AACtB,eAAW,SAAS,QAAQ,gBAAgB;AAC1C,UAAI,UAAU,OAAO,KAAK,CAAC,GAAG;AAC5B,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,mBAAmB,KAAK,QAAQ,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,kBAAkB,QAAQ,EAAE;AAAA,QACnH,CAAC;AAAA,MACH;AAAA,IACF;AACA,eAAW,SAAS,QAAQ,eAAe,CAAC,GAAG;AAC7C,UAAI,UAAU,OAAO,KAAK,CAAC,GAAG;AAC5B,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,gBAAgB,KAAK,QAAQ,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,kBAAkB,QAAQ,EAAE;AAAA,QAChH,CAAC;AAAA,MACH;AAAA,IACF;AACA,eAAW,EAAE,OAAO,IAAI,KAAK,QAAQ,YAAY,CAAC,GAAG;AACnD,YAAM,QAAQ,OAAO,KAAK;AAC1B,UAAI,UAAU,KAAK,EAAG;AACtB,UAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,GAAG;AACpD,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,UAAU,KAAK,2BAA2B,KAAK,UAAU,KAAK,CAAC,kBAAkB,QAAQ,EAAE,qBAAqB,GAAG;AAAA,QAC7H,CAAC;AACD;AAAA,MACF;AACA,UAAI,QAAQ,KAAK;AACf,mBAAW,KAAK;AAAA,UACd;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,QAAQ;AAAA,UACR,QAAQ,UAAU,KAAK,QAAQ,KAAK,mBAAmB,GAAG,kBAAkB,QAAQ,EAAE;AAAA,QACxF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO,EAAE,WAAW,WAAW,WAAW,GAAG,WAAW;AAC1D;AAGO,SAAS,sBACd,SACA,gBACM;AACN,QAAM,SAAS,qBAAqB,SAAS,cAAc;AAC3D,MAAI,OAAO,UAAW;AACtB,QAAM,QAAQ,OAAO,WAAW;AAAA,IAC9B,CAAC,MAAM,aAAa,EAAE,WAAW,KAAK,EAAE,SAAS,IAAI,EAAE,KAAK,KAAK,EAAE,MAAM,MAAM,EAAE,MAAM;AAAA,EACzF;AACA,QAAM,IAAI;AAAA,IACR,sCAAsC,OAAO,WAAW,MAAM;AAAA,EAAmB,MAAM,KAAK,IAAI,CAAC;AAAA,EACnG;AACF;;;AC/DO,SAAS,YAAY,WAAmB,MAAsC;AACnF,QAAM,QAAQ,OAAO,KAAK,IAAI,EAC3B,KAAK,EACL,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,KAAK,GAAG,CAAC,EAAE;AACrC,SAAO,CAAC,WAAW,GAAG,KAAK,EAAE,KAAK,GAAG;AACvC;AAGO,SAAS,aACd,UACA,MACA,QACgB;AAChB,QAAM,OAAO,OAAO,KAAK,IAAI,EAAE,KAAK;AACpC,MAAI,SAAmC,CAAC,CAAC,CAAC;AAC1C,aAAW,OAAO,MAAM;AACtB,UAAM,SAAS,KAAK,GAAG;AACvB,UAAM,OAAiC,CAAC;AACxC,eAAW,SAAS,QAAQ;AAC1B,iBAAW,SAAS,QAAQ;AAC1B,aAAK,KAAK,EAAE,GAAG,OAAO,CAAC,GAAG,GAAG,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AACA,aAAS;AAAA,EACX;AACA,QAAM,YAA4B,CAAC;AACnC,aAAW,WAAW,UAAU;AAC9B,eAAW,SAAS,QAAQ;AAC1B,UAAI,UAAU,CAAC,OAAO,QAAQ,IAAI,KAAK,EAAG;AAC1C,gBAAU,KAAK,EAAE,KAAK,YAAY,QAAQ,IAAI,KAAK,GAAG,SAAS,MAAM,MAAM,CAAC;AAAA,IAC9E;AAAA,EACF;AACA,SAAO;AACT;;;AC/BA,SAAS,YAAY,QAA+B,GAAmB;AACrE,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,0CAA0C;AACnF,QAAM,OAAO,KAAK,IAAI,GAAG,KAAK,KAAM,IAAI,MAAO,OAAO,MAAM,CAAC;AAC7D,SAAO,OAAO,OAAO,CAAC;AACxB;AAEO,SAAS,iBACd,SACA,OACA,cACc;AACd,QAAM,UAAU,oBAAI,IAAmC;AACvD,aAAW,UAAU,SAAS;AAC5B,UAAM,MAAM,MAAM,MAAM;AACxB,QAAI,QAAQ,KAAM;AAClB,QAAI,UAAU,QAAQ,IAAI,GAAG;AAC7B,QAAI,CAAC,SAAS;AACZ,gBAAU,oBAAI,IAAI;AAClB,cAAQ,IAAI,KAAK,OAAO;AAAA,IAC1B;AACA,eAAW,SAAS,cAAc;AAChC,YAAM,QAAQ,OAAO,KAAK;AAC1B,UAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,EAAG;AACtD,UAAI,SAAS,QAAQ,IAAI,KAAK;AAC9B,UAAI,CAAC,QAAQ;AACX,iBAAS,CAAC;AACV,gBAAQ,IAAI,OAAO,MAAM;AAAA,MAC3B;AACA,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,QAAM,YAAsD,CAAC;AAC7D,aAAW,CAAC,KAAK,OAAO,KAAK,SAAS;AACpC,UAAM,QAAkC,CAAC;AACzC,eAAW,CAAC,OAAO,MAAM,KAAK,SAAS;AACrC,UAAI,OAAO,WAAW,EAAG;AACzB,YAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,YAAM,KAAK,IAAI;AAAA,QACb,KAAK,YAAY,QAAQ,EAAE;AAAA,QAC3B,KAAK,YAAY,QAAQ,EAAE;AAAA,QAC3B,GAAG,OAAO;AAAA,MACZ;AAAA,IACF;AACA,cAAU,GAAG,IAAI;AAAA,EACnB;AACA,SAAO,EAAE,SAAS,GAAG,UAAU;AACjC;AAGA,SAAS,QAAQ,cAAsB,eAA+B;AACpE,MAAI,kBAAkB,GAAG;AACvB,WAAO,iBAAiB,IAAI,IAAI,OAAO;AAAA,EACzC;AACA,UAAS,eAAe,iBAAiB,gBAAiB;AAC5D;AAEO,SAAS,SACd,SACA,UACA,SACgB;AAChB,QAAM,eAAe,SAAS,gBAAgB;AAC9C,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAiC,CAAC;AACxC,QAAM,UAAU,oBAAI,IAAY;AAEhC,aAAW,CAACA,cAAa,aAAa,KAAK,OAAO,QAAQ,SAAS,SAAS,GAAG;AAC7E,UAAM,eAAe,QAAQ,UAAUA,YAAW;AAClD,QAAI,CAAC,cAAc;AACjB,cAAQ,IAAIA,YAAW;AACvB;AAAA,IACF;AACA,eAAW,CAAC,OAAO,YAAY,KAAK,OAAO,QAAQ,aAAa,GAAG;AACjE,YAAM,cAAc,aAAa,KAAK;AACtC,UAAI,CAAC,eAAe,YAAY,IAAI,YAAY;AAG9C,gBAAQ,IAAIA,YAAW;AACvB;AAAA,MACF;AACA,YAAM,SAAS;AAAA,QACb,QAAQ,QAAQ,YAAY,KAAK,aAAa,GAAG;AAAA,QACjD,QAAQ,QAAQ,YAAY,KAAK,aAAa,GAAG;AAAA,MACnD;AACA,YAAM,QAAwB;AAAA,QAC5B,aAAAA;AAAA,QACA;AAAA,QACA,UAAU;AAAA,QACV,SAAS;AAAA,QACT;AAAA,MACF;AACA,UAAI,OAAO,SAAS,gBAAgB,OAAO,SAAS,cAAc;AAChE,oBAAY,KAAK,KAAK;AAAA,MACxB,WAAW,OAAO,SAAS,KAAK,OAAO,SAAS,GAAG;AACjD,qBAAa,KAAK,KAAK;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,KAAK,QAAQ,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,SAAS,UAAU;AAEhG,SAAO;AAAA,IACL,WAAW,YAAY,WAAW;AAAA,IAClC;AAAA,IACA;AAAA,IACA,kBAAkB,CAAC,GAAG,OAAO;AAAA,IAC7B;AAAA,EACF;AACF;","names":["scenarioKey"]}