@tangle-network/agent-eval 0.109.1 → 0.110.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/CHANGELOG.md +9 -0
  2. package/dist/analyst/index.d.ts +10 -12
  3. package/dist/analyst/index.js +8 -11
  4. package/dist/analyst/index.js.map +1 -1
  5. package/dist/{analyze-runs-DJYpep3L.d.ts → analyze-runs-Dmz6LA9e.d.ts} +4 -4
  6. package/dist/{baseline-Bbid3WoO.d.ts → baseline-DsNteOgR.d.ts} +32 -2
  7. package/dist/belief-state/index.d.ts +6 -6
  8. package/dist/benchmarks/index.d.ts +4 -4
  9. package/dist/benchmarks/index.js +7 -8
  10. package/dist/builder-eval/index.d.ts +4 -4
  11. package/dist/builder-eval/index.js +1 -2
  12. package/dist/builder-eval/index.js.map +1 -1
  13. package/dist/{calibration-BPmzuVPk.d.ts → calibration-Dz8TQV4y.d.ts} +2 -2
  14. package/dist/campaign/index.d.ts +62 -20
  15. package/dist/campaign/index.js +9 -8
  16. package/dist/{chunk-LOJ2QVCE.js → chunk-2IY4ILP4.js} +2 -2
  17. package/dist/{chunk-LIEJUH2I.js → chunk-6PL5MGDL.js} +9 -9
  18. package/dist/{chunk-2OGPXHOB.js → chunk-7NX6ZSBG.js} +36 -7
  19. package/dist/chunk-7NX6ZSBG.js.map +1 -0
  20. package/dist/{chunk-R6D7NEYJ.js → chunk-GBI5J5DB.js} +81 -11
  21. package/dist/chunk-GBI5J5DB.js.map +1 -0
  22. package/dist/{chunk-YEHAEDUD.js → chunk-IMWDSFUM.js} +604 -2
  23. package/dist/chunk-IMWDSFUM.js.map +1 -0
  24. package/dist/{chunk-OVPVM4JC.js → chunk-J4AKLZEV.js} +15 -4
  25. package/dist/{chunk-OVPVM4JC.js.map → chunk-J4AKLZEV.js.map} +1 -1
  26. package/dist/{chunk-JZXGWLK5.js → chunk-MHNQWM4I.js} +62 -6
  27. package/dist/chunk-MHNQWM4I.js.map +1 -0
  28. package/dist/{chunk-QRVS7MX4.js → chunk-OW47B5WA.js} +3 -5
  29. package/dist/{chunk-QRVS7MX4.js.map → chunk-OW47B5WA.js.map} +1 -1
  30. package/dist/{chunk-DBDRR6GF.js → chunk-PLOMR3HP.js} +48 -2
  31. package/dist/chunk-PLOMR3HP.js.map +1 -0
  32. package/dist/{chunk-GDZAWO2I.js → chunk-QFGTU7MT.js} +2 -2
  33. package/dist/{chunk-V7HNA47Z.js → chunk-RSVSSZKF.js} +5 -5
  34. package/dist/{chunk-5PK3626Q.js → chunk-XRGOKCMO.js} +88 -17
  35. package/dist/chunk-XRGOKCMO.js.map +1 -0
  36. package/dist/{code-agent-session-rnJKlqmT.d.ts → code-agent-session-yitf9I-F.d.ts} +1 -1
  37. package/dist/contract/index.d.ts +26 -28
  38. package/dist/contract/index.js +11 -13
  39. package/dist/contract/index.js.map +1 -1
  40. package/dist/{control-B8UthSBL.d.ts → control-U8LBKUES.d.ts} +5 -6
  41. package/dist/control.d.ts +8 -9
  42. package/dist/control.js +6 -8
  43. package/dist/{dataset-DS7ytHZU.d.ts → dataset-NENEzRgk.d.ts} +1 -1
  44. package/dist/{default-registry-BswHCXnU.d.ts → default-registry-Bcf1uKVI.d.ts} +1 -2
  45. package/dist/{emitter-C2rqGH_l.d.ts → emitter-BRchAAAx.d.ts} +2 -2
  46. package/dist/{failure-cluster-DH9Flgcf.d.ts → failure-cluster-C48PiReX.d.ts} +2 -2
  47. package/dist/feedback-trajectory-pDcz1lQ1.d.ts +348 -0
  48. package/dist/{gepa-B3x5Ulcv.d.ts → gepa-T8T215nw.d.ts} +149 -6
  49. package/dist/hosted/index.d.ts +7 -7
  50. package/dist/{index-pPtfoIJO.d.ts → index-Dc3VLGhp.d.ts} +2 -2
  51. package/dist/index.d.ts +645 -61
  52. package/dist/index.js +1282 -190
  53. package/dist/index.js.map +1 -1
  54. package/dist/{insight-report-B4xrdwEK.d.ts → insight-report-D4cXFsLt.d.ts} +1 -1
  55. package/dist/{integrity-DqGZg3st.d.ts → integrity-qemeBAyx.d.ts} +1 -1
  56. package/dist/{types-D1ytG0Yg.d.ts → kind-factory-20hcaYpf.d.ts} +169 -2
  57. package/dist/meta-eval/index.d.ts +5 -5
  58. package/dist/meta-eval/index.js +1 -2
  59. package/dist/meta-eval/index.js.map +1 -1
  60. package/dist/{multi-layer-verifier-CI4jdX-q.d.ts → multi-layer-verifier-BsqKuLyN.d.ts} +1 -1
  61. package/dist/multishot/index.d.ts +3 -3
  62. package/dist/openapi.json +1 -1
  63. package/dist/pipelines/index.d.ts +6 -7
  64. package/dist/pipelines/index.js +3 -6
  65. package/dist/pipelines/index.js.map +1 -1
  66. package/dist/{policy-edit-DQUXYMDm.d.ts → policy-edit-D2bBDZDf.d.ts} +2 -2
  67. package/dist/{pre-registration-BUhVPzE7.d.ts → pre-registration-BepVVa6P.d.ts} +3 -3
  68. package/dist/{provenance-DdDhf6cg.d.ts → provenance-CyxkvEi9.d.ts} +3 -5
  69. package/dist/{query-0aTmbmQe.d.ts → query-Ck190MOd.d.ts} +2 -2
  70. package/dist/{release-report-DeJpsBiA.d.ts → release-report-oBfOz8ku.d.ts} +3 -3
  71. package/dist/reporting.d.ts +8 -8
  72. package/dist/{researcher-Wc7dx6GM.d.ts → researcher-CaH0CwFC.d.ts} +6 -6
  73. package/dist/rl.d.ts +568 -15
  74. package/dist/rl.js +4 -4
  75. package/dist/{rubric-predictive-validity-DPnyG-CE.d.ts → rubric-predictive-validity-C-fMteAW.d.ts} +1 -1
  76. package/dist/{run-record-I-Z3JNvO.d.ts → run-record-DksGsfgv.d.ts} +1 -1
  77. package/dist/{runtime-trajectory-iW9IhV3e.d.ts → runtime-trajectory-h5i0SZUj.d.ts} +1 -1
  78. package/dist/{schema-m0gsnbt3.d.ts → schema-SGWcK9wa.d.ts} +1 -1
  79. package/dist/{semantic-concept-judge-BmNZPB_j.d.ts → semantic-concept-judge-D7z6JCLZ.d.ts} +57 -4
  80. package/dist/{store-BcFXE6LG.d.ts → store-BsVi7ncX.d.ts} +1 -1
  81. package/dist/storyboard/index.d.ts +1 -1
  82. package/dist/{summary-report-QMZVe3P-.d.ts → summary-report-Bz-0-t8v.d.ts} +2 -2
  83. package/dist/{test-graded-scenario-DeODGLra.d.ts → test-graded-scenario-mzYBKspu.d.ts} +3 -3
  84. package/dist/traces.d.ts +54 -11
  85. package/dist/traces.js +25 -27
  86. package/dist/{types-BdIv5dvA.d.ts → types-v--ctu-b.d.ts} +2 -2
  87. package/dist/wire/index.d.ts +5 -6
  88. package/docs/improvement-glossary.md +14 -13
  89. package/package.json +1 -71
  90. package/dist/adapters/http.d.ts +0 -142
  91. package/dist/adapters/http.js +0 -203
  92. package/dist/adapters/http.js.map +0 -1
  93. package/dist/adapters/langchain.d.ts +0 -95
  94. package/dist/adapters/langchain.js +0 -34
  95. package/dist/adapters/langchain.js.map +0 -1
  96. package/dist/adapters/otel.d.ts +0 -112
  97. package/dist/adapters/otel.js +0 -110
  98. package/dist/adapters/otel.js.map +0 -1
  99. package/dist/chunk-2OGPXHOB.js.map +0 -1
  100. package/dist/chunk-45EEMHTC.js +0 -35
  101. package/dist/chunk-45EEMHTC.js.map +0 -1
  102. package/dist/chunk-5BKGXME7.js +0 -65
  103. package/dist/chunk-5BKGXME7.js.map +0 -1
  104. package/dist/chunk-5PK3626Q.js.map +0 -1
  105. package/dist/chunk-6SK5VFYK.js +0 -100
  106. package/dist/chunk-6SK5VFYK.js.map +0 -1
  107. package/dist/chunk-DBDRR6GF.js.map +0 -1
  108. package/dist/chunk-DJWX3GVS.js +0 -81
  109. package/dist/chunk-DJWX3GVS.js.map +0 -1
  110. package/dist/chunk-FOUG2VVS.js +0 -855
  111. package/dist/chunk-FOUG2VVS.js.map +0 -1
  112. package/dist/chunk-JZXGWLK5.js.map +0 -1
  113. package/dist/chunk-K7QEIHHJ.js +0 -613
  114. package/dist/chunk-K7QEIHHJ.js.map +0 -1
  115. package/dist/chunk-KKHDIONI.js +0 -414
  116. package/dist/chunk-KKHDIONI.js.map +0 -1
  117. package/dist/chunk-KMPRBJK4.js +0 -74
  118. package/dist/chunk-KMPRBJK4.js.map +0 -1
  119. package/dist/chunk-Q2JRAWRI.js +0 -196
  120. package/dist/chunk-Q2JRAWRI.js.map +0 -1
  121. package/dist/chunk-R6D7NEYJ.js.map +0 -1
  122. package/dist/chunk-RZTMDUO7.js +0 -49
  123. package/dist/chunk-RZTMDUO7.js.map +0 -1
  124. package/dist/chunk-STGVSCDH.js +0 -202
  125. package/dist/chunk-STGVSCDH.js.map +0 -1
  126. package/dist/chunk-YEHAEDUD.js.map +0 -1
  127. package/dist/control-runtime-Acf9CGhw.d.ts +0 -182
  128. package/dist/corpus-eBVwhCp1.d.ts +0 -560
  129. package/dist/counterfactual-DlOz8PBx.d.ts +0 -85
  130. package/dist/diagnose.d.ts +0 -252
  131. package/dist/diagnose.js +0 -382
  132. package/dist/diagnose.js.map +0 -1
  133. package/dist/feedback-trajectory-C9KCo8ag.d.ts +0 -169
  134. package/dist/governance/index.d.ts +0 -135
  135. package/dist/governance/index.js +0 -18
  136. package/dist/governance/index.js.map +0 -1
  137. package/dist/groundedness/index.d.ts +0 -112
  138. package/dist/groundedness/index.js +0 -77
  139. package/dist/groundedness/index.js.map +0 -1
  140. package/dist/harness-optimizer-mOl9XX_O.d.ts +0 -106
  141. package/dist/kind-factory-DvIGo_cP.d.ts +0 -171
  142. package/dist/knowledge/index.d.ts +0 -103
  143. package/dist/knowledge/index.js +0 -18
  144. package/dist/knowledge/index.js.map +0 -1
  145. package/dist/pareto-E-pembql.d.ts +0 -81
  146. package/dist/perf/index.d.ts +0 -123
  147. package/dist/perf/index.js +0 -18
  148. package/dist/perf/index.js.map +0 -1
  149. package/dist/prm/index.d.ts +0 -104
  150. package/dist/prm/index.js +0 -265
  151. package/dist/prm/index.js.map +0 -1
  152. package/dist/product-benchmark/index.d.ts +0 -247
  153. package/dist/product-benchmark/index.js +0 -37
  154. package/dist/product-benchmark/index.js.map +0 -1
  155. package/dist/red-team-KmmiqBlY.d.ts +0 -63
  156. package/dist/redact-B40YG2M_.d.ts +0 -45
  157. package/dist/rubric-Cc6UHvUb.d.ts +0 -73
  158. package/dist/run-critic-CmMf05uV.d.ts +0 -56
  159. package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
  160. package/dist/telemetry/file.d.ts +0 -19
  161. package/dist/telemetry/file.js +0 -45
  162. package/dist/telemetry/file.js.map +0 -1
  163. package/dist/telemetry/index.d.ts +0 -38
  164. package/dist/telemetry/index.js +0 -130
  165. package/dist/telemetry/index.js.map +0 -1
  166. package/dist/testing-C21CHsq2.d.ts +0 -20
  167. package/dist/testing.d.ts +0 -1
  168. package/dist/testing.js +0 -8
  169. package/dist/testing.js.map +0 -1
  170. package/dist/trajectory-2TkpSEVh.d.ts +0 -33
  171. package/dist/workflow/index.d.ts +0 -496
  172. package/dist/workflow/index.js +0 -2178
  173. package/dist/workflow/index.js.map +0 -1
  174. /package/dist/{chunk-LOJ2QVCE.js.map → chunk-2IY4ILP4.js.map} +0 -0
  175. /package/dist/{chunk-LIEJUH2I.js.map → chunk-6PL5MGDL.js.map} +0 -0
  176. /package/dist/{chunk-GDZAWO2I.js.map → chunk-QFGTU7MT.js.map} +0 -0
  177. /package/dist/{chunk-V7HNA47Z.js.map → chunk-RSVSSZKF.js.map} +0 -0
package/dist/testing.js DELETED
@@ -1,8 +0,0 @@
1
- import {
2
- resetLockedAppendersForTesting
3
- } from "./chunk-DJWX3GVS.js";
4
- import "./chunk-PZ5AY32C.js";
5
- export {
6
- resetLockedAppendersForTesting
7
- };
8
- //# sourceMappingURL=testing.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
@@ -1,33 +0,0 @@
1
- import { S as Span, a as TraceEvent } from './schema-m0gsnbt3.js';
2
- import { T as TraceStore } from './store-BcFXE6LG.js';
3
-
4
- /**
5
- * Trajectory — ordered, structured view over a run's spans.
6
- *
7
- * A pure function `buildTrajectory(store, runId) → Trajectory` returns
8
- * a topologically ordered list of `TrajectoryStep` with parent-child
9
- * grouping collapsed into a single line-of-agent-work. Separate
10
- * analyzers (stuck-loop detection, waste ratio) live in
11
- * `pipelines/` and consume the trajectory.
12
- */
13
-
14
- interface TrajectoryStep {
15
- index: number;
16
- span: Span;
17
- /** Depth in the span tree from the root. 0 = top-level. */
18
- depth: number;
19
- /** Events attached to this span. */
20
- events: TraceEvent[];
21
- }
22
- interface Trajectory {
23
- runId: string;
24
- steps: TrajectoryStep[];
25
- llmTurns: number;
26
- toolCalls: number;
27
- judgeVerdicts: number;
28
- retrievals: number;
29
- totalDurationMs: number;
30
- }
31
- declare function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory>;
32
-
33
- export { type Trajectory as T, type TrajectoryStep as a, buildTrajectory as b };
@@ -1,496 +0,0 @@
1
- import { W as WorkflowTopology } from '../harness-optimizer-mOl9XX_O.js';
2
- import { b as RunSplitTag, c as RunTokenUsage, R as RunRecord } from '../run-record-I-Z3JNvO.js';
3
- import { A as AnalystFinding, h as AnalystSeverity, E as EvidenceRef } from '../types-D1ytG0Yg.js';
4
- import { F as FailureClusterInsight } from '../insight-report-B4xrdwEK.js';
5
- import { a as VerificationReport, L as LayerResult } from '../multi-layer-verifier-CI4jdX-q.js';
6
- import { F as FailureClusterReport } from '../failure-cluster-DH9Flgcf.js';
7
- import { R as RedactionRule, a as RedactionReport } from '../redact-B40YG2M_.js';
8
- import { D as DatasetSplit } from '../dataset-DS7ytHZU.js';
9
- import { a as FeedbackTrajectory } from '../feedback-trajectory-C9KCo8ag.js';
10
- import { a as PairedBootstrapResult } from '../statistics-D88peojY.js';
11
- import '../pareto-E-pembql.js';
12
- import '../run-critic-CmMf05uV.js';
13
- import '../schema-m0gsnbt3.js';
14
- import '../store-BcFXE6LG.js';
15
- import '@tangle-network/agent-interface';
16
- import '../errors-oeQrLqXC.js';
17
- import '../store-C1YxJDEK.js';
18
- import '../types-C7DGg5ex.js';
19
- import '@tangle-network/tcloud';
20
- import '../llm-client-DyqEH4jH.js';
21
- import '../raw-provider-sink-C46HDghv.js';
22
- import '../summary-report-QMZVe3P-.js';
23
- import '../judge-calibration-7C-IDmKr.js';
24
- import '../verdict-C9MlYujm.js';
25
- import '../control-runtime-Acf9CGhw.js';
26
- import '../emitter-C2rqGH_l.js';
27
-
28
- type WorkflowTraceVersion = 'workflow-trace-v1';
29
- type WorkflowTraceEventKind = 'workflow.started' | 'workflow.phase' | 'workflow.log' | 'workflow.parallel.started' | 'workflow.parallel.ended' | 'workflow.pipeline.started' | 'workflow.pipeline.ended' | 'workflow.branch.started' | 'workflow.branch.ended' | 'workflow.branch.failed' | 'workflow.agent.started' | 'workflow.agent.ended' | 'workflow.agent.failed' | 'workflow.loop.started' | 'workflow.loop.ended' | 'workflow.loop.failed' | 'workflow.verifier.started' | 'workflow.verifier.ended' | 'workflow.verifier.failed' | 'workflow.analyst.started' | 'workflow.analyst.ended' | 'workflow.analyst.failed' | 'workflow.reviewer.started' | 'workflow.reviewer.ended' | 'workflow.reviewer.failed' | 'workflow.failed' | 'workflow.ended';
30
- interface WorkflowTraceEvent {
31
- kind: WorkflowTraceEventKind;
32
- runId: string;
33
- timestamp: number;
34
- payload: Record<string, unknown>;
35
- }
36
- interface WorkflowTraceArtifact {
37
- kind: string;
38
- uri: string;
39
- contentType?: string;
40
- sha256?: string;
41
- metadata?: Record<string, unknown>;
42
- }
43
- interface WorkflowTraceExportLinks {
44
- traceArtifactUri?: string;
45
- exportBundleUri?: string;
46
- partnerReportUri?: string;
47
- intelligenceRunUri?: string;
48
- }
49
- interface WorkflowTraceEnvelope {
50
- traceVersion: WorkflowTraceVersion;
51
- runId: string;
52
- topology?: WorkflowTopology;
53
- events: WorkflowTraceEvent[];
54
- artifacts?: WorkflowTraceArtifact[];
55
- metadata?: Record<string, unknown>;
56
- }
57
- interface WorkflowTraceSummary {
58
- runId: string;
59
- startedAt?: number;
60
- endedAt?: number;
61
- durationMs: number;
62
- costUsd: number;
63
- tokenUsage: RunTokenUsage;
64
- phaseCount: number;
65
- branchCount: number;
66
- failedBranchCount: number;
67
- agentCalls: number;
68
- loopCalls: number;
69
- verifierCalls: number;
70
- analystCalls: number;
71
- reviewerCalls: number;
72
- agentFailures: number;
73
- loopFailures: number;
74
- verifierFailures: number;
75
- analystFailures: number;
76
- reviewerFailures: number;
77
- eventCount: number;
78
- failed: boolean;
79
- failureMessage?: string;
80
- }
81
- interface WorkflowTraceProjectionMetadata {
82
- experimentId: string;
83
- candidateId: string;
84
- seed: number;
85
- model: string;
86
- promptHash: string;
87
- configHash: string;
88
- commitSha: string;
89
- splitTag: RunSplitTag;
90
- scenarioId?: string;
91
- }
92
-
93
- declare const WORKFLOW_TRACE_EVENT_KINDS: readonly WorkflowTraceEventKind[];
94
- declare function validateWorkflowTraceEventKind(kind: string): WorkflowTraceEventKind;
95
- declare function validateWorkflowTraceEventPayload(kind: WorkflowTraceEventKind, payload: Record<string, unknown>): Record<string, unknown>;
96
-
97
- type WorkflowFeedbackPackVersion = 'workflow-feedback-pack-v1';
98
- type WorkflowFeedbackSeverity = AnalystSeverity;
99
- interface WorkflowToolUsageSummary {
100
- totalCalls: number;
101
- erroredCalls: number;
102
- byTool: Record<string, {
103
- calls: number;
104
- errors: number;
105
- }>;
106
- }
107
- interface WorkflowVerifierFindingSummary {
108
- severity: WorkflowFeedbackSeverity;
109
- message: string;
110
- evidence?: string;
111
- detail?: Record<string, unknown>;
112
- }
113
- interface WorkflowVerifierLayerSummary {
114
- layer: string;
115
- status: LayerResult['status'];
116
- score?: number;
117
- durationMs: number;
118
- reason?: string;
119
- findings: WorkflowVerifierFindingSummary[];
120
- diagnostics?: Record<string, number | null>;
121
- }
122
- interface WorkflowVerifierSummary {
123
- allPass: boolean;
124
- blendedScore: number;
125
- durationMs: number;
126
- failedLayers: string[];
127
- layers: WorkflowVerifierLayerSummary[];
128
- }
129
- interface WorkflowFailureClusterInput {
130
- id: string;
131
- name: string;
132
- share?: number;
133
- runCount?: number;
134
- exemplars?: readonly string[];
135
- suggestedFix?: string;
136
- metadata?: Record<string, unknown>;
137
- }
138
- interface WorkflowFailureClusterSummary {
139
- id: string;
140
- name: string;
141
- share: number;
142
- runCount?: number;
143
- exemplars: string[];
144
- suggestedFix?: string;
145
- source: 'failure-cluster-view' | 'insight-report' | 'custom';
146
- metadata?: Record<string, unknown>;
147
- }
148
- interface WorkflowAnalystFindingSummary {
149
- findingId: string;
150
- analystId: string;
151
- severity: WorkflowFeedbackSeverity;
152
- area: string;
153
- claim: string;
154
- confidence: number;
155
- subject?: string;
156
- recommendedAction?: string;
157
- evidenceRefs: EvidenceRef[];
158
- }
159
- interface WorkflowAnalystFeedbackPack {
160
- schemaVersion: WorkflowFeedbackPackVersion;
161
- runId: string;
162
- generatedAt: string;
163
- summary: WorkflowTraceSummary;
164
- verifier?: WorkflowVerifierSummary;
165
- toolUsage: WorkflowToolUsageSummary;
166
- failureClusters: WorkflowFailureClusterSummary[];
167
- findings: WorkflowAnalystFindingSummary[];
168
- recommendations: string[];
169
- driverContextLines: string[];
170
- }
171
- interface WorkflowFeedbackPackLimits {
172
- findings?: number;
173
- clusters?: number;
174
- layerFindings?: number;
175
- recommendations?: number;
176
- contextLines?: number;
177
- }
178
- interface BuildWorkflowAnalystFeedbackPackOptions {
179
- envelope: WorkflowTraceEnvelope | unknown;
180
- verifier?: VerificationReport;
181
- analystFindings?: readonly AnalystFinding[];
182
- failureClusters?: FailureClusterReport | FailureClusterInsight | readonly WorkflowFailureClusterInput[];
183
- generatedAt?: string;
184
- limits?: WorkflowFeedbackPackLimits;
185
- }
186
- declare function buildWorkflowAnalystFeedbackPack(options: BuildWorkflowAnalystFeedbackPackOptions): WorkflowAnalystFeedbackPack;
187
- declare function renderWorkflowFeedbackPack(pack: WorkflowAnalystFeedbackPack, options?: {
188
- maxChars?: number;
189
- }): string;
190
-
191
- interface WorkflowTraceSanitizationReport extends RedactionReport {
192
- hashedArgs: number;
193
- truncatedStrings: number;
194
- droppedPayloadKeys: Record<string, number>;
195
- droppedArtifactContents: number;
196
- }
197
- interface SanitizeWorkflowTraceEnvelopeOptions {
198
- rules?: readonly RedactionRule[];
199
- maxStringLength?: number;
200
- hashSalt?: string;
201
- approvedArtifactUris?: readonly string[];
202
- approvedArtifactKinds?: readonly string[];
203
- }
204
- interface SanitizedWorkflowTraceEnvelopeResult {
205
- envelope: WorkflowTraceEnvelope;
206
- report: WorkflowTraceSanitizationReport;
207
- }
208
- declare function sanitizeWorkflowTraceEnvelope(input: WorkflowTraceEnvelope | unknown, options?: SanitizeWorkflowTraceEnvelopeOptions): SanitizedWorkflowTraceEnvelopeResult;
209
-
210
- interface WorkflowPhaseGraphNode {
211
- id: string;
212
- title: string;
213
- startedAt?: number;
214
- endedAt?: number;
215
- eventCount: number;
216
- branchCount: number;
217
- failedBranchCount: number;
218
- agentCalls: number;
219
- loopCalls: number;
220
- verifierCalls: number;
221
- analystCalls: number;
222
- reviewerCalls: number;
223
- agentFailures: number;
224
- loopFailures: number;
225
- verifierFailures: number;
226
- analystFailures: number;
227
- reviewerFailures: number;
228
- costUsd: number;
229
- tokenUsage: RunTokenUsage;
230
- }
231
- interface WorkflowPhaseGraphBranch {
232
- id: string;
233
- operation: string;
234
- branchIndex: number;
235
- phase: string | null;
236
- status: 'started' | 'ended' | 'failed';
237
- startedAt?: number;
238
- endedAt?: number;
239
- durationMs?: number;
240
- stageCount?: number;
241
- stageIndex?: number;
242
- message?: string;
243
- code?: string;
244
- }
245
- interface WorkflowPhaseGraph {
246
- nodes: WorkflowPhaseGraphNode[];
247
- branches: WorkflowPhaseGraphBranch[];
248
- }
249
- declare function workflowPhaseGraph(events: readonly WorkflowTraceEvent[]): WorkflowPhaseGraph;
250
-
251
- interface WorkflowDelegateTraceSummary {
252
- index: number | null;
253
- label: string | null;
254
- phase: string | null;
255
- costUsd: number | null;
256
- tokenUsage: RunTokenUsage | null;
257
- trace: unknown;
258
- }
259
- interface WorkflowDelegateFailureSummary {
260
- index: number | null;
261
- label: string | null;
262
- phase: string | null;
263
- durationMs: number | null;
264
- message: string | null;
265
- code: string | null;
266
- }
267
- interface WorkflowCheckpointTraceSummary extends WorkflowDelegateTraceSummary {
268
- output: unknown;
269
- }
270
- interface WorkflowExecutionSummary extends WorkflowTraceSummary {
271
- source?: string;
272
- eventKinds: Record<string, number>;
273
- phases: string[];
274
- phaseGraph: WorkflowPhaseGraph;
275
- agentRuns: WorkflowDelegateTraceSummary[];
276
- loopRuns: WorkflowDelegateTraceSummary[];
277
- verifierOutputs: WorkflowCheckpointTraceSummary[];
278
- analystOutputs: WorkflowCheckpointTraceSummary[];
279
- reviewerOutputs: WorkflowCheckpointTraceSummary[];
280
- agentFailureDetails: WorkflowDelegateFailureSummary[];
281
- loopFailureDetails: WorkflowDelegateFailureSummary[];
282
- verifierFailureDetails: WorkflowDelegateFailureSummary[];
283
- analystFailureDetails: WorkflowDelegateFailureSummary[];
284
- reviewerFailureDetails: WorkflowDelegateFailureSummary[];
285
- }
286
- interface SummarizeWorkflowExecutionOptions {
287
- source?: string;
288
- }
289
- declare function summarizeWorkflowExecution(input: WorkflowTraceEnvelope | unknown, options?: SummarizeWorkflowExecutionOptions): WorkflowExecutionSummary;
290
-
291
- type WorkflowTraceIntelligenceEnvelopeVersion = 'workflow-trace-intelligence-envelope-v1';
292
- type WorkflowTraceExportGrantScope = 'workflow-trace:export' | 'workflow-trace:read' | '*';
293
- type WorkflowTraceExportGrantSubject = 'product' | 'partner' | 'tenant';
294
- interface WorkflowTraceExportGrant {
295
- grantId: string;
296
- subject: WorkflowTraceExportGrantSubject;
297
- subjectId: string;
298
- scopes: readonly WorkflowTraceExportGrantScope[];
299
- grantedAt?: string;
300
- expiresAt?: string;
301
- metadata?: Record<string, unknown>;
302
- }
303
- interface WorkflowTraceHashEvidence {
304
- path: string;
305
- sha256: string;
306
- shape?: unknown;
307
- }
308
- interface WorkflowTraceArtifactEvidence {
309
- kind: string;
310
- uri: string;
311
- contentType?: string;
312
- sha256?: string;
313
- }
314
- interface WorkflowTraceCompactEvidence {
315
- eventKinds: Record<string, number>;
316
- phases: string[];
317
- toolNames: string[];
318
- redactedHashes: WorkflowTraceHashEvidence[];
319
- artifacts: WorkflowTraceArtifactEvidence[];
320
- failureMessage?: string;
321
- }
322
- interface WorkflowTraceIntelligenceEnvelope {
323
- schemaVersion: WorkflowTraceIntelligenceEnvelopeVersion;
324
- destination: string;
325
- generatedAt: string;
326
- productId: string;
327
- partnerId?: string;
328
- runId: string;
329
- grantIds: string[];
330
- traceEnvelope: WorkflowTraceEnvelope;
331
- summary: WorkflowExecutionSummary;
332
- compactEvidence: WorkflowTraceCompactEvidence;
333
- sanitization: SanitizedWorkflowTraceEnvelopeResult['report'];
334
- links?: WorkflowTraceExportLinks;
335
- }
336
- interface BuildWorkflowTraceIntelligenceEnvelopeOptions {
337
- envelope: WorkflowTraceEnvelope | unknown;
338
- productId: string;
339
- partnerId?: string;
340
- grants: readonly WorkflowTraceExportGrant[];
341
- generatedAt?: string;
342
- destination?: string;
343
- sanitize?: SanitizeWorkflowTraceEnvelopeOptions;
344
- links?: WorkflowTraceExportLinks;
345
- metadata?: Record<string, unknown>;
346
- }
347
- declare function buildWorkflowTraceIntelligenceEnvelope(options: BuildWorkflowTraceIntelligenceEnvelopeOptions): WorkflowTraceIntelligenceEnvelope;
348
- declare function validateWorkflowTraceIntelligenceEnvelope(input: unknown): WorkflowTraceIntelligenceEnvelope;
349
-
350
- interface WorkflowTraceRunRecordOptions extends WorkflowTraceProjectionMetadata {
351
- runId?: string;
352
- score?: number;
353
- raw?: Record<string, number>;
354
- failureMode?: string;
355
- judgeMetadata?: RunRecord['judgeMetadata'];
356
- agentProfile?: RunRecord['agentProfile'];
357
- }
358
- declare function workflowTraceToRunRecord(input: WorkflowTraceEnvelope | unknown, options: WorkflowTraceRunRecordOptions): RunRecord;
359
-
360
- interface WorkflowTraceTrajectoryOptions {
361
- projectId?: string;
362
- scenarioId?: string;
363
- task: string;
364
- split?: DatasetSplit;
365
- tags?: Record<string, string>;
366
- score?: number;
367
- success?: boolean;
368
- metadata?: Record<string, unknown>;
369
- }
370
- declare function workflowTraceToFeedbackTrajectory(input: WorkflowTraceEnvelope | unknown, options: WorkflowTraceTrajectoryOptions): FeedbackTrajectory;
371
-
372
- type WorkflowPartnerReportVersion = 'workflow-partner-report-v1';
373
- interface WorkflowPartnerFinding {
374
- source: 'analyst' | 'verifier' | 'failure-cluster';
375
- severity: AnalystSeverity;
376
- area: string;
377
- claim: string;
378
- evidence: EvidenceRef[];
379
- recommendedAction?: string;
380
- metadata?: Record<string, unknown>;
381
- }
382
- interface WorkflowPartnerReport {
383
- schemaVersion: WorkflowPartnerReportVersion;
384
- runId: string;
385
- generatedAt: string;
386
- summary: WorkflowTraceSummary;
387
- docsApiGaps: WorkflowPartnerFinding[];
388
- prReadyFindings: WorkflowPartnerFinding[];
389
- failureClusters: ReturnType<typeof buildWorkflowAnalystFeedbackPack>['failureClusters'];
390
- recommendations: string[];
391
- traceArtifacts: WorkflowTraceEnvelope['artifacts'];
392
- links?: WorkflowTraceExportLinks;
393
- exportBundle: {
394
- traceEnvelope: WorkflowTraceEnvelope;
395
- sanitization: SanitizedWorkflowTraceEnvelopeResult['report'];
396
- feedbackPack: ReturnType<typeof buildWorkflowAnalystFeedbackPack>;
397
- trajectory: ReturnType<typeof workflowTraceToFeedbackTrajectory>;
398
- runRecord?: RunRecord;
399
- };
400
- }
401
- interface BuildWorkflowPartnerReportOptions extends Omit<BuildWorkflowAnalystFeedbackPackOptions, 'envelope'> {
402
- envelope: WorkflowTraceEnvelope | unknown;
403
- sanitize?: SanitizeWorkflowTraceEnvelopeOptions;
404
- trajectory: WorkflowTraceTrajectoryOptions;
405
- runRecord?: WorkflowTraceRunRecordOptions;
406
- links?: WorkflowTraceExportLinks;
407
- }
408
- declare function buildWorkflowPartnerReport(options: BuildWorkflowPartnerReportOptions): WorkflowPartnerReport;
409
- declare function validateWorkflowPartnerReport(input: unknown): WorkflowPartnerReport;
410
- declare function renderWorkflowPartnerReport(report: WorkflowPartnerReport, options?: {
411
- maxFindings?: number;
412
- }): string;
413
-
414
- type WorkflowDriverPromotionDecisionVersion = 'workflow-driver-promotion-v1';
415
- type WorkflowDriverPromotionRejectionCode = 'missing_baseline_records' | 'missing_candidate_records' | 'missing_holdout_pairs' | 'few_pairs' | 'insufficient_lift' | 'cost_ceiling';
416
- interface WorkflowDriverPromotionPair {
417
- key: string;
418
- scenarioId: string;
419
- seed: number;
420
- baselineRunId: string;
421
- candidateRunId: string;
422
- baselineScore: number;
423
- candidateScore: number;
424
- delta: number;
425
- }
426
- interface WorkflowDriverPromotionEvidence {
427
- pairedRuns: number;
428
- expectedScenarioIds: string[];
429
- pairedScenarioIds: string[];
430
- missingScenarioIds: string[];
431
- baselineMean: number;
432
- candidateMean: number;
433
- lift: number;
434
- liftCi: {
435
- low: number;
436
- high: number;
437
- };
438
- bootstrap: PairedBootstrapResult;
439
- confidence: number;
440
- resamples: number;
441
- statistic: 'mean' | 'median';
442
- deltaThreshold: number;
443
- baselineMedianCostUsd: number;
444
- candidateMedianCostUsd: number;
445
- pairs: WorkflowDriverPromotionPair[];
446
- }
447
- interface WorkflowDriverPromotionDecision {
448
- schemaVersion: WorkflowDriverPromotionDecisionVersion;
449
- generatedAt: string;
450
- baselineStrategyId: string;
451
- candidateStrategyId: string;
452
- promote: boolean;
453
- rejectionCode: WorkflowDriverPromotionRejectionCode | null;
454
- reason: string;
455
- evidence: WorkflowDriverPromotionEvidence;
456
- }
457
- interface DecideWorkflowDriverPromotionOptions {
458
- records: readonly RunRecord[] | readonly unknown[];
459
- baselineStrategyId?: string;
460
- candidateStrategyId?: string;
461
- expectedScenarioIds?: readonly string[];
462
- minPairedHoldoutRuns?: number;
463
- deltaThreshold?: number;
464
- confidence?: number;
465
- resamples?: number;
466
- seed?: number;
467
- statistic?: 'mean' | 'median';
468
- costPerRunCeiling?: number;
469
- generatedAt?: string;
470
- }
471
- declare function decideWorkflowDriverPromotion(options: DecideWorkflowDriverPromotionOptions): WorkflowDriverPromotionDecision;
472
-
473
- interface WorkflowTraceEnvelopeFromEventsOptions {
474
- runId?: string;
475
- topology?: WorkflowTopology;
476
- artifacts?: readonly WorkflowTraceArtifact[];
477
- metadata?: Record<string, unknown>;
478
- }
479
- interface WorkflowRuntimeResultLike {
480
- runId?: string;
481
- meta?: unknown;
482
- output?: unknown;
483
- events: readonly unknown[];
484
- }
485
- interface WorkflowRuntimeResultToTraceEnvelopeOptions extends Omit<WorkflowTraceEnvelopeFromEventsOptions, 'runId'> {
486
- runId?: string;
487
- includeOutputInMetadata?: boolean;
488
- }
489
- declare function workflowEventsToTraceEnvelope(events: readonly unknown[], options?: WorkflowTraceEnvelopeFromEventsOptions): WorkflowTraceEnvelope;
490
- declare function workflowRuntimeResultToTraceEnvelope(result: WorkflowRuntimeResultLike, options?: WorkflowRuntimeResultToTraceEnvelopeOptions): WorkflowTraceEnvelope;
491
-
492
- declare function validateWorkflowTraceEvent(input: unknown): WorkflowTraceEvent;
493
- declare function validateWorkflowTraceEnvelope(input: unknown): WorkflowTraceEnvelope;
494
- declare function summarizeWorkflowTrace(input: WorkflowTraceEnvelope | unknown): WorkflowTraceSummary;
495
-
496
- export { type BuildWorkflowAnalystFeedbackPackOptions, type BuildWorkflowPartnerReportOptions, type BuildWorkflowTraceIntelligenceEnvelopeOptions, type DecideWorkflowDriverPromotionOptions, type SanitizeWorkflowTraceEnvelopeOptions, type SanitizedWorkflowTraceEnvelopeResult, type SummarizeWorkflowExecutionOptions, WORKFLOW_TRACE_EVENT_KINDS, type WorkflowAnalystFeedbackPack, type WorkflowAnalystFindingSummary, type WorkflowCheckpointTraceSummary, type WorkflowDelegateTraceSummary, type WorkflowDriverPromotionDecision, type WorkflowDriverPromotionDecisionVersion, type WorkflowDriverPromotionEvidence, type WorkflowDriverPromotionPair, type WorkflowDriverPromotionRejectionCode, type WorkflowExecutionSummary, type WorkflowFailureClusterInput, type WorkflowFailureClusterSummary, type WorkflowFeedbackPackLimits, type WorkflowFeedbackPackVersion, type WorkflowFeedbackSeverity, type WorkflowPartnerFinding, type WorkflowPartnerReport, type WorkflowPartnerReportVersion, type WorkflowPhaseGraph, type WorkflowPhaseGraphBranch, type WorkflowPhaseGraphNode, type WorkflowRuntimeResultLike, type WorkflowRuntimeResultToTraceEnvelopeOptions, type WorkflowToolUsageSummary, WorkflowTopology, type WorkflowTraceArtifact, type WorkflowTraceArtifactEvidence, type WorkflowTraceCompactEvidence, type WorkflowTraceEnvelope, type WorkflowTraceEnvelopeFromEventsOptions, type WorkflowTraceEvent, type WorkflowTraceEventKind, type WorkflowTraceExportGrant, type WorkflowTraceExportGrantScope, type WorkflowTraceExportGrantSubject, type WorkflowTraceExportLinks, type WorkflowTraceHashEvidence, type WorkflowTraceIntelligenceEnvelope, type WorkflowTraceIntelligenceEnvelopeVersion, type WorkflowTraceProjectionMetadata, type WorkflowTraceRunRecordOptions, type WorkflowTraceSanitizationReport, type WorkflowTraceSummary, type WorkflowTraceTrajectoryOptions, type WorkflowTraceVersion, type WorkflowVerifierFindingSummary, type WorkflowVerifierLayerSummary, type WorkflowVerifierSummary, buildWorkflowAnalystFeedbackPack, buildWorkflowPartnerReport, buildWorkflowTraceIntelligenceEnvelope, decideWorkflowDriverPromotion, renderWorkflowFeedbackPack, renderWorkflowPartnerReport, sanitizeWorkflowTraceEnvelope, summarizeWorkflowExecution, summarizeWorkflowTrace, validateWorkflowPartnerReport, validateWorkflowTraceEnvelope, validateWorkflowTraceEvent, validateWorkflowTraceEventKind, validateWorkflowTraceEventPayload, validateWorkflowTraceIntelligenceEnvelope, workflowEventsToTraceEnvelope, workflowPhaseGraph, workflowRuntimeResultToTraceEnvelope, workflowTraceToFeedbackTrajectory, workflowTraceToRunRecord };