@tangle-network/agent-eval 0.108.1 → 0.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/analyst/index.d.ts +10 -12
  2. package/dist/analyst/index.js +8 -11
  3. package/dist/analyst/index.js.map +1 -1
  4. package/dist/{analyze-runs-DJYpep3L.d.ts → analyze-runs-Dmz6LA9e.d.ts} +4 -4
  5. package/dist/{baseline-Bbid3WoO.d.ts → baseline-DsNteOgR.d.ts} +32 -2
  6. package/dist/belief-state/index.d.ts +6 -6
  7. package/dist/benchmarks/index.d.ts +4 -4
  8. package/dist/benchmarks/index.js +7 -8
  9. package/dist/builder-eval/index.d.ts +4 -4
  10. package/dist/builder-eval/index.js +1 -2
  11. package/dist/builder-eval/index.js.map +1 -1
  12. package/dist/{calibration-BPmzuVPk.d.ts → calibration-Dz8TQV4y.d.ts} +2 -2
  13. package/dist/campaign/index.d.ts +161 -20
  14. package/dist/campaign/index.js +15 -8
  15. package/dist/{chunk-LOJ2QVCE.js → chunk-2IY4ILP4.js} +2 -2
  16. package/dist/{chunk-LIEJUH2I.js → chunk-6PL5MGDL.js} +9 -9
  17. package/dist/{chunk-2OGPXHOB.js → chunk-7NX6ZSBG.js} +36 -7
  18. package/dist/chunk-7NX6ZSBG.js.map +1 -0
  19. package/dist/{chunk-OVPVM4JC.js → chunk-GTERJI6Q.js} +4 -4
  20. package/dist/{chunk-YEHAEDUD.js → chunk-IMWDSFUM.js} +604 -2
  21. package/dist/chunk-IMWDSFUM.js.map +1 -0
  22. package/dist/{chunk-JZXGWLK5.js → chunk-MHNQWM4I.js} +62 -6
  23. package/dist/chunk-MHNQWM4I.js.map +1 -0
  24. package/dist/{chunk-QRVS7MX4.js → chunk-OW47B5WA.js} +3 -5
  25. package/dist/{chunk-QRVS7MX4.js.map → chunk-OW47B5WA.js.map} +1 -1
  26. package/dist/{chunk-DBDRR6GF.js → chunk-PLOMR3HP.js} +48 -2
  27. package/dist/chunk-PLOMR3HP.js.map +1 -0
  28. package/dist/{chunk-GDZAWO2I.js → chunk-QFGTU7MT.js} +2 -2
  29. package/dist/{chunk-6SKVFBTR.js → chunk-RNB2NICW.js} +115 -13
  30. package/dist/chunk-RNB2NICW.js.map +1 -0
  31. package/dist/{chunk-V7HNA47Z.js → chunk-RSVSSZKF.js} +5 -5
  32. package/dist/{chunk-5PK3626Q.js → chunk-XRGOKCMO.js} +88 -17
  33. package/dist/chunk-XRGOKCMO.js.map +1 -0
  34. package/dist/{code-agent-session-rnJKlqmT.d.ts → code-agent-session-yitf9I-F.d.ts} +1 -1
  35. package/dist/contract/index.d.ts +20 -23
  36. package/dist/contract/index.js +11 -13
  37. package/dist/contract/index.js.map +1 -1
  38. package/dist/{control-B8UthSBL.d.ts → control-U8LBKUES.d.ts} +5 -6
  39. package/dist/control.d.ts +8 -9
  40. package/dist/control.js +6 -8
  41. package/dist/{dataset-DS7ytHZU.d.ts → dataset-NENEzRgk.d.ts} +1 -1
  42. package/dist/{default-registry-BswHCXnU.d.ts → default-registry-Bcf1uKVI.d.ts} +1 -2
  43. package/dist/{emitter-C2rqGH_l.d.ts → emitter-BRchAAAx.d.ts} +2 -2
  44. package/dist/{failure-cluster-DH9Flgcf.d.ts → failure-cluster-C48PiReX.d.ts} +2 -2
  45. package/dist/feedback-trajectory-pDcz1lQ1.d.ts +348 -0
  46. package/dist/{gepa-B3x5Ulcv.d.ts → gepa-BUNP3606.d.ts} +143 -2
  47. package/dist/hosted/index.d.ts +7 -7
  48. package/dist/{index-pPtfoIJO.d.ts → index-Dc3VLGhp.d.ts} +2 -2
  49. package/dist/index.d.ts +645 -61
  50. package/dist/index.js +1282 -190
  51. package/dist/index.js.map +1 -1
  52. package/dist/{insight-report-B4xrdwEK.d.ts → insight-report-D4cXFsLt.d.ts} +1 -1
  53. package/dist/{integrity-DqGZg3st.d.ts → integrity-qemeBAyx.d.ts} +1 -1
  54. package/dist/{types-D1ytG0Yg.d.ts → kind-factory-20hcaYpf.d.ts} +169 -2
  55. package/dist/meta-eval/index.d.ts +5 -5
  56. package/dist/meta-eval/index.js +1 -2
  57. package/dist/meta-eval/index.js.map +1 -1
  58. package/dist/{multi-layer-verifier-CI4jdX-q.d.ts → multi-layer-verifier-BsqKuLyN.d.ts} +1 -1
  59. package/dist/multishot/index.d.ts +3 -3
  60. package/dist/openapi.json +1 -1
  61. package/dist/pipelines/index.d.ts +6 -7
  62. package/dist/pipelines/index.js +3 -6
  63. package/dist/pipelines/index.js.map +1 -1
  64. package/dist/{policy-edit-DQUXYMDm.d.ts → policy-edit-D2bBDZDf.d.ts} +2 -2
  65. package/dist/{pre-registration-BUhVPzE7.d.ts → pre-registration-BepVVa6P.d.ts} +3 -3
  66. package/dist/{provenance-DdDhf6cg.d.ts → provenance-DMvsfknv.d.ts} +3 -5
  67. package/dist/{query-0aTmbmQe.d.ts → query-Ck190MOd.d.ts} +2 -2
  68. package/dist/{release-report-DeJpsBiA.d.ts → release-report-oBfOz8ku.d.ts} +3 -3
  69. package/dist/reporting.d.ts +8 -8
  70. package/dist/{researcher-Wc7dx6GM.d.ts → researcher-CaH0CwFC.d.ts} +6 -6
  71. package/dist/rl.d.ts +568 -15
  72. package/dist/rl.js +4 -4
  73. package/dist/{rubric-predictive-validity-DPnyG-CE.d.ts → rubric-predictive-validity-C-fMteAW.d.ts} +1 -1
  74. package/dist/{run-record-I-Z3JNvO.d.ts → run-record-DksGsfgv.d.ts} +1 -1
  75. package/dist/{runtime-trajectory-iW9IhV3e.d.ts → runtime-trajectory-h5i0SZUj.d.ts} +1 -1
  76. package/dist/{schema-m0gsnbt3.d.ts → schema-SGWcK9wa.d.ts} +1 -1
  77. package/dist/{semantic-concept-judge-BmNZPB_j.d.ts → semantic-concept-judge-D7z6JCLZ.d.ts} +57 -4
  78. package/dist/{store-BcFXE6LG.d.ts → store-BsVi7ncX.d.ts} +1 -1
  79. package/dist/storyboard/index.d.ts +1 -1
  80. package/dist/{summary-report-QMZVe3P-.d.ts → summary-report-Bz-0-t8v.d.ts} +2 -2
  81. package/dist/{test-graded-scenario-DeODGLra.d.ts → test-graded-scenario-mzYBKspu.d.ts} +3 -3
  82. package/dist/traces.d.ts +54 -11
  83. package/dist/traces.js +25 -27
  84. package/dist/{types-BdIv5dvA.d.ts → types-v--ctu-b.d.ts} +2 -2
  85. package/dist/wire/index.d.ts +5 -6
  86. package/package.json +1 -71
  87. package/dist/adapters/http.d.ts +0 -142
  88. package/dist/adapters/http.js +0 -203
  89. package/dist/adapters/http.js.map +0 -1
  90. package/dist/adapters/langchain.d.ts +0 -95
  91. package/dist/adapters/langchain.js +0 -34
  92. package/dist/adapters/langchain.js.map +0 -1
  93. package/dist/adapters/otel.d.ts +0 -112
  94. package/dist/adapters/otel.js +0 -110
  95. package/dist/adapters/otel.js.map +0 -1
  96. package/dist/chunk-2OGPXHOB.js.map +0 -1
  97. package/dist/chunk-45EEMHTC.js +0 -35
  98. package/dist/chunk-45EEMHTC.js.map +0 -1
  99. package/dist/chunk-5BKGXME7.js +0 -65
  100. package/dist/chunk-5BKGXME7.js.map +0 -1
  101. package/dist/chunk-5PK3626Q.js.map +0 -1
  102. package/dist/chunk-6SK5VFYK.js +0 -100
  103. package/dist/chunk-6SK5VFYK.js.map +0 -1
  104. package/dist/chunk-6SKVFBTR.js.map +0 -1
  105. package/dist/chunk-DBDRR6GF.js.map +0 -1
  106. package/dist/chunk-DJWX3GVS.js +0 -81
  107. package/dist/chunk-DJWX3GVS.js.map +0 -1
  108. package/dist/chunk-FOUG2VVS.js +0 -855
  109. package/dist/chunk-FOUG2VVS.js.map +0 -1
  110. package/dist/chunk-JZXGWLK5.js.map +0 -1
  111. package/dist/chunk-K7QEIHHJ.js +0 -613
  112. package/dist/chunk-K7QEIHHJ.js.map +0 -1
  113. package/dist/chunk-KKHDIONI.js +0 -414
  114. package/dist/chunk-KKHDIONI.js.map +0 -1
  115. package/dist/chunk-KMPRBJK4.js +0 -74
  116. package/dist/chunk-KMPRBJK4.js.map +0 -1
  117. package/dist/chunk-Q2JRAWRI.js +0 -196
  118. package/dist/chunk-Q2JRAWRI.js.map +0 -1
  119. package/dist/chunk-RZTMDUO7.js +0 -49
  120. package/dist/chunk-RZTMDUO7.js.map +0 -1
  121. package/dist/chunk-STGVSCDH.js +0 -202
  122. package/dist/chunk-STGVSCDH.js.map +0 -1
  123. package/dist/chunk-YEHAEDUD.js.map +0 -1
  124. package/dist/control-runtime-Acf9CGhw.d.ts +0 -182
  125. package/dist/corpus-eBVwhCp1.d.ts +0 -560
  126. package/dist/counterfactual-DlOz8PBx.d.ts +0 -85
  127. package/dist/diagnose.d.ts +0 -252
  128. package/dist/diagnose.js +0 -382
  129. package/dist/diagnose.js.map +0 -1
  130. package/dist/feedback-trajectory-C9KCo8ag.d.ts +0 -169
  131. package/dist/governance/index.d.ts +0 -135
  132. package/dist/governance/index.js +0 -18
  133. package/dist/governance/index.js.map +0 -1
  134. package/dist/groundedness/index.d.ts +0 -112
  135. package/dist/groundedness/index.js +0 -77
  136. package/dist/groundedness/index.js.map +0 -1
  137. package/dist/harness-optimizer-mOl9XX_O.d.ts +0 -106
  138. package/dist/kind-factory-DvIGo_cP.d.ts +0 -171
  139. package/dist/knowledge/index.d.ts +0 -103
  140. package/dist/knowledge/index.js +0 -18
  141. package/dist/knowledge/index.js.map +0 -1
  142. package/dist/pareto-E-pembql.d.ts +0 -81
  143. package/dist/perf/index.d.ts +0 -123
  144. package/dist/perf/index.js +0 -18
  145. package/dist/perf/index.js.map +0 -1
  146. package/dist/prm/index.d.ts +0 -104
  147. package/dist/prm/index.js +0 -265
  148. package/dist/prm/index.js.map +0 -1
  149. package/dist/product-benchmark/index.d.ts +0 -247
  150. package/dist/product-benchmark/index.js +0 -37
  151. package/dist/product-benchmark/index.js.map +0 -1
  152. package/dist/red-team-KmmiqBlY.d.ts +0 -63
  153. package/dist/redact-B40YG2M_.d.ts +0 -45
  154. package/dist/rubric-Cc6UHvUb.d.ts +0 -73
  155. package/dist/run-critic-CmMf05uV.d.ts +0 -56
  156. package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
  157. package/dist/telemetry/file.d.ts +0 -19
  158. package/dist/telemetry/file.js +0 -45
  159. package/dist/telemetry/file.js.map +0 -1
  160. package/dist/telemetry/index.d.ts +0 -38
  161. package/dist/telemetry/index.js +0 -130
  162. package/dist/telemetry/index.js.map +0 -1
  163. package/dist/testing-C21CHsq2.d.ts +0 -20
  164. package/dist/testing.d.ts +0 -1
  165. package/dist/testing.js +0 -8
  166. package/dist/testing.js.map +0 -1
  167. package/dist/trajectory-2TkpSEVh.d.ts +0 -33
  168. package/dist/workflow/index.d.ts +0 -496
  169. package/dist/workflow/index.js +0 -2178
  170. package/dist/workflow/index.js.map +0 -1
  171. /package/dist/{chunk-LOJ2QVCE.js.map → chunk-2IY4ILP4.js.map} +0 -0
  172. /package/dist/{chunk-LIEJUH2I.js.map → chunk-6PL5MGDL.js.map} +0 -0
  173. /package/dist/{chunk-OVPVM4JC.js.map → chunk-GTERJI6Q.js.map} +0 -0
  174. /package/dist/{chunk-GDZAWO2I.js.map → chunk-QFGTU7MT.js.map} +0 -0
  175. /package/dist/{chunk-V7HNA47Z.js.map → chunk-RSVSSZKF.js.map} +0 -0
package/dist/testing.js DELETED
@@ -1,8 +0,0 @@
1
- import {
2
- resetLockedAppendersForTesting
3
- } from "./chunk-DJWX3GVS.js";
4
- import "./chunk-PZ5AY32C.js";
5
- export {
6
- resetLockedAppendersForTesting
7
- };
8
- //# sourceMappingURL=testing.js.map
@@ -1 +0,0 @@
1
- {"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
@@ -1,33 +0,0 @@
1
- import { S as Span, a as TraceEvent } from './schema-m0gsnbt3.js';
2
- import { T as TraceStore } from './store-BcFXE6LG.js';
3
-
4
- /**
5
- * Trajectory — ordered, structured view over a run's spans.
6
- *
7
- * A pure function `buildTrajectory(store, runId) → Trajectory` returns
8
- * a topologically ordered list of `TrajectoryStep` with parent-child
9
- * grouping collapsed into a single line-of-agent-work. Separate
10
- * analyzers (stuck-loop detection, waste ratio) live in
11
- * `pipelines/` and consume the trajectory.
12
- */
13
-
14
- interface TrajectoryStep {
15
- index: number;
16
- span: Span;
17
- /** Depth in the span tree from the root. 0 = top-level. */
18
- depth: number;
19
- /** Events attached to this span. */
20
- events: TraceEvent[];
21
- }
22
- interface Trajectory {
23
- runId: string;
24
- steps: TrajectoryStep[];
25
- llmTurns: number;
26
- toolCalls: number;
27
- judgeVerdicts: number;
28
- retrievals: number;
29
- totalDurationMs: number;
30
- }
31
- declare function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory>;
32
-
33
- export { type Trajectory as T, type TrajectoryStep as a, buildTrajectory as b };
@@ -1,496 +0,0 @@
1
- import { W as WorkflowTopology } from '../harness-optimizer-mOl9XX_O.js';
2
- import { b as RunSplitTag, c as RunTokenUsage, R as RunRecord } from '../run-record-I-Z3JNvO.js';
3
- import { A as AnalystFinding, h as AnalystSeverity, E as EvidenceRef } from '../types-D1ytG0Yg.js';
4
- import { F as FailureClusterInsight } from '../insight-report-B4xrdwEK.js';
5
- import { a as VerificationReport, L as LayerResult } from '../multi-layer-verifier-CI4jdX-q.js';
6
- import { F as FailureClusterReport } from '../failure-cluster-DH9Flgcf.js';
7
- import { R as RedactionRule, a as RedactionReport } from '../redact-B40YG2M_.js';
8
- import { D as DatasetSplit } from '../dataset-DS7ytHZU.js';
9
- import { a as FeedbackTrajectory } from '../feedback-trajectory-C9KCo8ag.js';
10
- import { a as PairedBootstrapResult } from '../statistics-D88peojY.js';
11
- import '../pareto-E-pembql.js';
12
- import '../run-critic-CmMf05uV.js';
13
- import '../schema-m0gsnbt3.js';
14
- import '../store-BcFXE6LG.js';
15
- import '@tangle-network/agent-interface';
16
- import '../errors-oeQrLqXC.js';
17
- import '../store-C1YxJDEK.js';
18
- import '../types-C7DGg5ex.js';
19
- import '@tangle-network/tcloud';
20
- import '../llm-client-DyqEH4jH.js';
21
- import '../raw-provider-sink-C46HDghv.js';
22
- import '../summary-report-QMZVe3P-.js';
23
- import '../judge-calibration-7C-IDmKr.js';
24
- import '../verdict-C9MlYujm.js';
25
- import '../control-runtime-Acf9CGhw.js';
26
- import '../emitter-C2rqGH_l.js';
27
-
28
- type WorkflowTraceVersion = 'workflow-trace-v1';
29
- type WorkflowTraceEventKind = 'workflow.started' | 'workflow.phase' | 'workflow.log' | 'workflow.parallel.started' | 'workflow.parallel.ended' | 'workflow.pipeline.started' | 'workflow.pipeline.ended' | 'workflow.branch.started' | 'workflow.branch.ended' | 'workflow.branch.failed' | 'workflow.agent.started' | 'workflow.agent.ended' | 'workflow.agent.failed' | 'workflow.loop.started' | 'workflow.loop.ended' | 'workflow.loop.failed' | 'workflow.verifier.started' | 'workflow.verifier.ended' | 'workflow.verifier.failed' | 'workflow.analyst.started' | 'workflow.analyst.ended' | 'workflow.analyst.failed' | 'workflow.reviewer.started' | 'workflow.reviewer.ended' | 'workflow.reviewer.failed' | 'workflow.failed' | 'workflow.ended';
30
- interface WorkflowTraceEvent {
31
- kind: WorkflowTraceEventKind;
32
- runId: string;
33
- timestamp: number;
34
- payload: Record<string, unknown>;
35
- }
36
- interface WorkflowTraceArtifact {
37
- kind: string;
38
- uri: string;
39
- contentType?: string;
40
- sha256?: string;
41
- metadata?: Record<string, unknown>;
42
- }
43
- interface WorkflowTraceExportLinks {
44
- traceArtifactUri?: string;
45
- exportBundleUri?: string;
46
- partnerReportUri?: string;
47
- intelligenceRunUri?: string;
48
- }
49
- interface WorkflowTraceEnvelope {
50
- traceVersion: WorkflowTraceVersion;
51
- runId: string;
52
- topology?: WorkflowTopology;
53
- events: WorkflowTraceEvent[];
54
- artifacts?: WorkflowTraceArtifact[];
55
- metadata?: Record<string, unknown>;
56
- }
57
- interface WorkflowTraceSummary {
58
- runId: string;
59
- startedAt?: number;
60
- endedAt?: number;
61
- durationMs: number;
62
- costUsd: number;
63
- tokenUsage: RunTokenUsage;
64
- phaseCount: number;
65
- branchCount: number;
66
- failedBranchCount: number;
67
- agentCalls: number;
68
- loopCalls: number;
69
- verifierCalls: number;
70
- analystCalls: number;
71
- reviewerCalls: number;
72
- agentFailures: number;
73
- loopFailures: number;
74
- verifierFailures: number;
75
- analystFailures: number;
76
- reviewerFailures: number;
77
- eventCount: number;
78
- failed: boolean;
79
- failureMessage?: string;
80
- }
81
- interface WorkflowTraceProjectionMetadata {
82
- experimentId: string;
83
- candidateId: string;
84
- seed: number;
85
- model: string;
86
- promptHash: string;
87
- configHash: string;
88
- commitSha: string;
89
- splitTag: RunSplitTag;
90
- scenarioId?: string;
91
- }
92
-
93
- declare const WORKFLOW_TRACE_EVENT_KINDS: readonly WorkflowTraceEventKind[];
94
- declare function validateWorkflowTraceEventKind(kind: string): WorkflowTraceEventKind;
95
- declare function validateWorkflowTraceEventPayload(kind: WorkflowTraceEventKind, payload: Record<string, unknown>): Record<string, unknown>;
96
-
97
- type WorkflowFeedbackPackVersion = 'workflow-feedback-pack-v1';
98
- type WorkflowFeedbackSeverity = AnalystSeverity;
99
- interface WorkflowToolUsageSummary {
100
- totalCalls: number;
101
- erroredCalls: number;
102
- byTool: Record<string, {
103
- calls: number;
104
- errors: number;
105
- }>;
106
- }
107
- interface WorkflowVerifierFindingSummary {
108
- severity: WorkflowFeedbackSeverity;
109
- message: string;
110
- evidence?: string;
111
- detail?: Record<string, unknown>;
112
- }
113
- interface WorkflowVerifierLayerSummary {
114
- layer: string;
115
- status: LayerResult['status'];
116
- score?: number;
117
- durationMs: number;
118
- reason?: string;
119
- findings: WorkflowVerifierFindingSummary[];
120
- diagnostics?: Record<string, number | null>;
121
- }
122
- interface WorkflowVerifierSummary {
123
- allPass: boolean;
124
- blendedScore: number;
125
- durationMs: number;
126
- failedLayers: string[];
127
- layers: WorkflowVerifierLayerSummary[];
128
- }
129
- interface WorkflowFailureClusterInput {
130
- id: string;
131
- name: string;
132
- share?: number;
133
- runCount?: number;
134
- exemplars?: readonly string[];
135
- suggestedFix?: string;
136
- metadata?: Record<string, unknown>;
137
- }
138
- interface WorkflowFailureClusterSummary {
139
- id: string;
140
- name: string;
141
- share: number;
142
- runCount?: number;
143
- exemplars: string[];
144
- suggestedFix?: string;
145
- source: 'failure-cluster-view' | 'insight-report' | 'custom';
146
- metadata?: Record<string, unknown>;
147
- }
148
- interface WorkflowAnalystFindingSummary {
149
- findingId: string;
150
- analystId: string;
151
- severity: WorkflowFeedbackSeverity;
152
- area: string;
153
- claim: string;
154
- confidence: number;
155
- subject?: string;
156
- recommendedAction?: string;
157
- evidenceRefs: EvidenceRef[];
158
- }
159
- interface WorkflowAnalystFeedbackPack {
160
- schemaVersion: WorkflowFeedbackPackVersion;
161
- runId: string;
162
- generatedAt: string;
163
- summary: WorkflowTraceSummary;
164
- verifier?: WorkflowVerifierSummary;
165
- toolUsage: WorkflowToolUsageSummary;
166
- failureClusters: WorkflowFailureClusterSummary[];
167
- findings: WorkflowAnalystFindingSummary[];
168
- recommendations: string[];
169
- driverContextLines: string[];
170
- }
171
- interface WorkflowFeedbackPackLimits {
172
- findings?: number;
173
- clusters?: number;
174
- layerFindings?: number;
175
- recommendations?: number;
176
- contextLines?: number;
177
- }
178
- interface BuildWorkflowAnalystFeedbackPackOptions {
179
- envelope: WorkflowTraceEnvelope | unknown;
180
- verifier?: VerificationReport;
181
- analystFindings?: readonly AnalystFinding[];
182
- failureClusters?: FailureClusterReport | FailureClusterInsight | readonly WorkflowFailureClusterInput[];
183
- generatedAt?: string;
184
- limits?: WorkflowFeedbackPackLimits;
185
- }
186
- declare function buildWorkflowAnalystFeedbackPack(options: BuildWorkflowAnalystFeedbackPackOptions): WorkflowAnalystFeedbackPack;
187
- declare function renderWorkflowFeedbackPack(pack: WorkflowAnalystFeedbackPack, options?: {
188
- maxChars?: number;
189
- }): string;
190
-
191
- interface WorkflowTraceSanitizationReport extends RedactionReport {
192
- hashedArgs: number;
193
- truncatedStrings: number;
194
- droppedPayloadKeys: Record<string, number>;
195
- droppedArtifactContents: number;
196
- }
197
- interface SanitizeWorkflowTraceEnvelopeOptions {
198
- rules?: readonly RedactionRule[];
199
- maxStringLength?: number;
200
- hashSalt?: string;
201
- approvedArtifactUris?: readonly string[];
202
- approvedArtifactKinds?: readonly string[];
203
- }
204
- interface SanitizedWorkflowTraceEnvelopeResult {
205
- envelope: WorkflowTraceEnvelope;
206
- report: WorkflowTraceSanitizationReport;
207
- }
208
- declare function sanitizeWorkflowTraceEnvelope(input: WorkflowTraceEnvelope | unknown, options?: SanitizeWorkflowTraceEnvelopeOptions): SanitizedWorkflowTraceEnvelopeResult;
209
-
210
- interface WorkflowPhaseGraphNode {
211
- id: string;
212
- title: string;
213
- startedAt?: number;
214
- endedAt?: number;
215
- eventCount: number;
216
- branchCount: number;
217
- failedBranchCount: number;
218
- agentCalls: number;
219
- loopCalls: number;
220
- verifierCalls: number;
221
- analystCalls: number;
222
- reviewerCalls: number;
223
- agentFailures: number;
224
- loopFailures: number;
225
- verifierFailures: number;
226
- analystFailures: number;
227
- reviewerFailures: number;
228
- costUsd: number;
229
- tokenUsage: RunTokenUsage;
230
- }
231
- interface WorkflowPhaseGraphBranch {
232
- id: string;
233
- operation: string;
234
- branchIndex: number;
235
- phase: string | null;
236
- status: 'started' | 'ended' | 'failed';
237
- startedAt?: number;
238
- endedAt?: number;
239
- durationMs?: number;
240
- stageCount?: number;
241
- stageIndex?: number;
242
- message?: string;
243
- code?: string;
244
- }
245
- interface WorkflowPhaseGraph {
246
- nodes: WorkflowPhaseGraphNode[];
247
- branches: WorkflowPhaseGraphBranch[];
248
- }
249
- declare function workflowPhaseGraph(events: readonly WorkflowTraceEvent[]): WorkflowPhaseGraph;
250
-
251
- interface WorkflowDelegateTraceSummary {
252
- index: number | null;
253
- label: string | null;
254
- phase: string | null;
255
- costUsd: number | null;
256
- tokenUsage: RunTokenUsage | null;
257
- trace: unknown;
258
- }
259
- interface WorkflowDelegateFailureSummary {
260
- index: number | null;
261
- label: string | null;
262
- phase: string | null;
263
- durationMs: number | null;
264
- message: string | null;
265
- code: string | null;
266
- }
267
- interface WorkflowCheckpointTraceSummary extends WorkflowDelegateTraceSummary {
268
- output: unknown;
269
- }
270
- interface WorkflowExecutionSummary extends WorkflowTraceSummary {
271
- source?: string;
272
- eventKinds: Record<string, number>;
273
- phases: string[];
274
- phaseGraph: WorkflowPhaseGraph;
275
- agentRuns: WorkflowDelegateTraceSummary[];
276
- loopRuns: WorkflowDelegateTraceSummary[];
277
- verifierOutputs: WorkflowCheckpointTraceSummary[];
278
- analystOutputs: WorkflowCheckpointTraceSummary[];
279
- reviewerOutputs: WorkflowCheckpointTraceSummary[];
280
- agentFailureDetails: WorkflowDelegateFailureSummary[];
281
- loopFailureDetails: WorkflowDelegateFailureSummary[];
282
- verifierFailureDetails: WorkflowDelegateFailureSummary[];
283
- analystFailureDetails: WorkflowDelegateFailureSummary[];
284
- reviewerFailureDetails: WorkflowDelegateFailureSummary[];
285
- }
286
- interface SummarizeWorkflowExecutionOptions {
287
- source?: string;
288
- }
289
- declare function summarizeWorkflowExecution(input: WorkflowTraceEnvelope | unknown, options?: SummarizeWorkflowExecutionOptions): WorkflowExecutionSummary;
290
-
291
- type WorkflowTraceIntelligenceEnvelopeVersion = 'workflow-trace-intelligence-envelope-v1';
292
- type WorkflowTraceExportGrantScope = 'workflow-trace:export' | 'workflow-trace:read' | '*';
293
- type WorkflowTraceExportGrantSubject = 'product' | 'partner' | 'tenant';
294
- interface WorkflowTraceExportGrant {
295
- grantId: string;
296
- subject: WorkflowTraceExportGrantSubject;
297
- subjectId: string;
298
- scopes: readonly WorkflowTraceExportGrantScope[];
299
- grantedAt?: string;
300
- expiresAt?: string;
301
- metadata?: Record<string, unknown>;
302
- }
303
- interface WorkflowTraceHashEvidence {
304
- path: string;
305
- sha256: string;
306
- shape?: unknown;
307
- }
308
- interface WorkflowTraceArtifactEvidence {
309
- kind: string;
310
- uri: string;
311
- contentType?: string;
312
- sha256?: string;
313
- }
314
- interface WorkflowTraceCompactEvidence {
315
- eventKinds: Record<string, number>;
316
- phases: string[];
317
- toolNames: string[];
318
- redactedHashes: WorkflowTraceHashEvidence[];
319
- artifacts: WorkflowTraceArtifactEvidence[];
320
- failureMessage?: string;
321
- }
322
- interface WorkflowTraceIntelligenceEnvelope {
323
- schemaVersion: WorkflowTraceIntelligenceEnvelopeVersion;
324
- destination: string;
325
- generatedAt: string;
326
- productId: string;
327
- partnerId?: string;
328
- runId: string;
329
- grantIds: string[];
330
- traceEnvelope: WorkflowTraceEnvelope;
331
- summary: WorkflowExecutionSummary;
332
- compactEvidence: WorkflowTraceCompactEvidence;
333
- sanitization: SanitizedWorkflowTraceEnvelopeResult['report'];
334
- links?: WorkflowTraceExportLinks;
335
- }
336
- interface BuildWorkflowTraceIntelligenceEnvelopeOptions {
337
- envelope: WorkflowTraceEnvelope | unknown;
338
- productId: string;
339
- partnerId?: string;
340
- grants: readonly WorkflowTraceExportGrant[];
341
- generatedAt?: string;
342
- destination?: string;
343
- sanitize?: SanitizeWorkflowTraceEnvelopeOptions;
344
- links?: WorkflowTraceExportLinks;
345
- metadata?: Record<string, unknown>;
346
- }
347
- declare function buildWorkflowTraceIntelligenceEnvelope(options: BuildWorkflowTraceIntelligenceEnvelopeOptions): WorkflowTraceIntelligenceEnvelope;
348
- declare function validateWorkflowTraceIntelligenceEnvelope(input: unknown): WorkflowTraceIntelligenceEnvelope;
349
-
350
- interface WorkflowTraceRunRecordOptions extends WorkflowTraceProjectionMetadata {
351
- runId?: string;
352
- score?: number;
353
- raw?: Record<string, number>;
354
- failureMode?: string;
355
- judgeMetadata?: RunRecord['judgeMetadata'];
356
- agentProfile?: RunRecord['agentProfile'];
357
- }
358
- declare function workflowTraceToRunRecord(input: WorkflowTraceEnvelope | unknown, options: WorkflowTraceRunRecordOptions): RunRecord;
359
-
360
- interface WorkflowTraceTrajectoryOptions {
361
- projectId?: string;
362
- scenarioId?: string;
363
- task: string;
364
- split?: DatasetSplit;
365
- tags?: Record<string, string>;
366
- score?: number;
367
- success?: boolean;
368
- metadata?: Record<string, unknown>;
369
- }
370
- declare function workflowTraceToFeedbackTrajectory(input: WorkflowTraceEnvelope | unknown, options: WorkflowTraceTrajectoryOptions): FeedbackTrajectory;
371
-
372
- type WorkflowPartnerReportVersion = 'workflow-partner-report-v1';
373
- interface WorkflowPartnerFinding {
374
- source: 'analyst' | 'verifier' | 'failure-cluster';
375
- severity: AnalystSeverity;
376
- area: string;
377
- claim: string;
378
- evidence: EvidenceRef[];
379
- recommendedAction?: string;
380
- metadata?: Record<string, unknown>;
381
- }
382
- interface WorkflowPartnerReport {
383
- schemaVersion: WorkflowPartnerReportVersion;
384
- runId: string;
385
- generatedAt: string;
386
- summary: WorkflowTraceSummary;
387
- docsApiGaps: WorkflowPartnerFinding[];
388
- prReadyFindings: WorkflowPartnerFinding[];
389
- failureClusters: ReturnType<typeof buildWorkflowAnalystFeedbackPack>['failureClusters'];
390
- recommendations: string[];
391
- traceArtifacts: WorkflowTraceEnvelope['artifacts'];
392
- links?: WorkflowTraceExportLinks;
393
- exportBundle: {
394
- traceEnvelope: WorkflowTraceEnvelope;
395
- sanitization: SanitizedWorkflowTraceEnvelopeResult['report'];
396
- feedbackPack: ReturnType<typeof buildWorkflowAnalystFeedbackPack>;
397
- trajectory: ReturnType<typeof workflowTraceToFeedbackTrajectory>;
398
- runRecord?: RunRecord;
399
- };
400
- }
401
- interface BuildWorkflowPartnerReportOptions extends Omit<BuildWorkflowAnalystFeedbackPackOptions, 'envelope'> {
402
- envelope: WorkflowTraceEnvelope | unknown;
403
- sanitize?: SanitizeWorkflowTraceEnvelopeOptions;
404
- trajectory: WorkflowTraceTrajectoryOptions;
405
- runRecord?: WorkflowTraceRunRecordOptions;
406
- links?: WorkflowTraceExportLinks;
407
- }
408
- declare function buildWorkflowPartnerReport(options: BuildWorkflowPartnerReportOptions): WorkflowPartnerReport;
409
- declare function validateWorkflowPartnerReport(input: unknown): WorkflowPartnerReport;
410
- declare function renderWorkflowPartnerReport(report: WorkflowPartnerReport, options?: {
411
- maxFindings?: number;
412
- }): string;
413
-
414
- type WorkflowDriverPromotionDecisionVersion = 'workflow-driver-promotion-v1';
415
- type WorkflowDriverPromotionRejectionCode = 'missing_baseline_records' | 'missing_candidate_records' | 'missing_holdout_pairs' | 'few_pairs' | 'insufficient_lift' | 'cost_ceiling';
416
- interface WorkflowDriverPromotionPair {
417
- key: string;
418
- scenarioId: string;
419
- seed: number;
420
- baselineRunId: string;
421
- candidateRunId: string;
422
- baselineScore: number;
423
- candidateScore: number;
424
- delta: number;
425
- }
426
- interface WorkflowDriverPromotionEvidence {
427
- pairedRuns: number;
428
- expectedScenarioIds: string[];
429
- pairedScenarioIds: string[];
430
- missingScenarioIds: string[];
431
- baselineMean: number;
432
- candidateMean: number;
433
- lift: number;
434
- liftCi: {
435
- low: number;
436
- high: number;
437
- };
438
- bootstrap: PairedBootstrapResult;
439
- confidence: number;
440
- resamples: number;
441
- statistic: 'mean' | 'median';
442
- deltaThreshold: number;
443
- baselineMedianCostUsd: number;
444
- candidateMedianCostUsd: number;
445
- pairs: WorkflowDriverPromotionPair[];
446
- }
447
- interface WorkflowDriverPromotionDecision {
448
- schemaVersion: WorkflowDriverPromotionDecisionVersion;
449
- generatedAt: string;
450
- baselineStrategyId: string;
451
- candidateStrategyId: string;
452
- promote: boolean;
453
- rejectionCode: WorkflowDriverPromotionRejectionCode | null;
454
- reason: string;
455
- evidence: WorkflowDriverPromotionEvidence;
456
- }
457
- interface DecideWorkflowDriverPromotionOptions {
458
- records: readonly RunRecord[] | readonly unknown[];
459
- baselineStrategyId?: string;
460
- candidateStrategyId?: string;
461
- expectedScenarioIds?: readonly string[];
462
- minPairedHoldoutRuns?: number;
463
- deltaThreshold?: number;
464
- confidence?: number;
465
- resamples?: number;
466
- seed?: number;
467
- statistic?: 'mean' | 'median';
468
- costPerRunCeiling?: number;
469
- generatedAt?: string;
470
- }
471
- declare function decideWorkflowDriverPromotion(options: DecideWorkflowDriverPromotionOptions): WorkflowDriverPromotionDecision;
472
-
473
- interface WorkflowTraceEnvelopeFromEventsOptions {
474
- runId?: string;
475
- topology?: WorkflowTopology;
476
- artifacts?: readonly WorkflowTraceArtifact[];
477
- metadata?: Record<string, unknown>;
478
- }
479
- interface WorkflowRuntimeResultLike {
480
- runId?: string;
481
- meta?: unknown;
482
- output?: unknown;
483
- events: readonly unknown[];
484
- }
485
- interface WorkflowRuntimeResultToTraceEnvelopeOptions extends Omit<WorkflowTraceEnvelopeFromEventsOptions, 'runId'> {
486
- runId?: string;
487
- includeOutputInMetadata?: boolean;
488
- }
489
- declare function workflowEventsToTraceEnvelope(events: readonly unknown[], options?: WorkflowTraceEnvelopeFromEventsOptions): WorkflowTraceEnvelope;
490
- declare function workflowRuntimeResultToTraceEnvelope(result: WorkflowRuntimeResultLike, options?: WorkflowRuntimeResultToTraceEnvelopeOptions): WorkflowTraceEnvelope;
491
-
492
- declare function validateWorkflowTraceEvent(input: unknown): WorkflowTraceEvent;
493
- declare function validateWorkflowTraceEnvelope(input: unknown): WorkflowTraceEnvelope;
494
- declare function summarizeWorkflowTrace(input: WorkflowTraceEnvelope | unknown): WorkflowTraceSummary;
495
-
496
- export { type BuildWorkflowAnalystFeedbackPackOptions, type BuildWorkflowPartnerReportOptions, type BuildWorkflowTraceIntelligenceEnvelopeOptions, type DecideWorkflowDriverPromotionOptions, type SanitizeWorkflowTraceEnvelopeOptions, type SanitizedWorkflowTraceEnvelopeResult, type SummarizeWorkflowExecutionOptions, WORKFLOW_TRACE_EVENT_KINDS, type WorkflowAnalystFeedbackPack, type WorkflowAnalystFindingSummary, type WorkflowCheckpointTraceSummary, type WorkflowDelegateTraceSummary, type WorkflowDriverPromotionDecision, type WorkflowDriverPromotionDecisionVersion, type WorkflowDriverPromotionEvidence, type WorkflowDriverPromotionPair, type WorkflowDriverPromotionRejectionCode, type WorkflowExecutionSummary, type WorkflowFailureClusterInput, type WorkflowFailureClusterSummary, type WorkflowFeedbackPackLimits, type WorkflowFeedbackPackVersion, type WorkflowFeedbackSeverity, type WorkflowPartnerFinding, type WorkflowPartnerReport, type WorkflowPartnerReportVersion, type WorkflowPhaseGraph, type WorkflowPhaseGraphBranch, type WorkflowPhaseGraphNode, type WorkflowRuntimeResultLike, type WorkflowRuntimeResultToTraceEnvelopeOptions, type WorkflowToolUsageSummary, WorkflowTopology, type WorkflowTraceArtifact, type WorkflowTraceArtifactEvidence, type WorkflowTraceCompactEvidence, type WorkflowTraceEnvelope, type WorkflowTraceEnvelopeFromEventsOptions, type WorkflowTraceEvent, type WorkflowTraceEventKind, type WorkflowTraceExportGrant, type WorkflowTraceExportGrantScope, type WorkflowTraceExportGrantSubject, type WorkflowTraceExportLinks, type WorkflowTraceHashEvidence, type WorkflowTraceIntelligenceEnvelope, type WorkflowTraceIntelligenceEnvelopeVersion, type WorkflowTraceProjectionMetadata, type WorkflowTraceRunRecordOptions, type WorkflowTraceSanitizationReport, type WorkflowTraceSummary, type WorkflowTraceTrajectoryOptions, type WorkflowTraceVersion, type WorkflowVerifierFindingSummary, type WorkflowVerifierLayerSummary, type WorkflowVerifierSummary, buildWorkflowAnalystFeedbackPack, buildWorkflowPartnerReport, buildWorkflowTraceIntelligenceEnvelope, decideWorkflowDriverPromotion, renderWorkflowFeedbackPack, renderWorkflowPartnerReport, sanitizeWorkflowTraceEnvelope, summarizeWorkflowExecution, summarizeWorkflowTrace, validateWorkflowPartnerReport, validateWorkflowTraceEnvelope, validateWorkflowTraceEvent, validateWorkflowTraceEventKind, validateWorkflowTraceEventPayload, validateWorkflowTraceIntelligenceEnvelope, workflowEventsToTraceEnvelope, workflowPhaseGraph, workflowRuntimeResultToTraceEnvelope, workflowTraceToFeedbackTrajectory, workflowTraceToRunRecord };