@tangle-network/agent-eval 0.190.1 → 0.192.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/{agent-profile-DNV_U6QR.d.ts → agent-profile-DDfIjdyS.d.ts} +3 -3
  4. package/dist/{agent-profile-DNV_U6QR.d.ts.map → agent-profile-DDfIjdyS.d.ts.map} +1 -1
  5. package/dist/analyst/index.d.ts +11 -11
  6. package/dist/analyst/index.js +5 -5
  7. package/dist/{insight-report-CvPkEIcC.d.ts → analyze-runs-CPiKLJFs.d.ts} +73 -3
  8. package/dist/analyze-runs-CPiKLJFs.d.ts.map +1 -0
  9. package/dist/{attribute-vocabulary-DLJ6303h.d.ts → attribute-vocabulary-CcXqHODT.d.ts} +23 -12
  10. package/dist/attribute-vocabulary-CcXqHODT.d.ts.map +1 -0
  11. package/dist/{benchmark-command-BXIr_k8D.js → benchmark-command-C458yEFK.js} +9 -9
  12. package/dist/{benchmark-command-BXIr_k8D.js.map → benchmark-command-C458yEFK.js.map} +1 -1
  13. package/dist/benchmarks/index.d.ts +3 -3
  14. package/dist/benchmarks/index.js +3 -3
  15. package/dist/campaign/index.d.ts +6 -6
  16. package/dist/campaign/index.js +8 -7
  17. package/dist/{campaign-B7q74uIh.js → campaign-BVQVBEfH.js} +13 -325
  18. package/dist/campaign-BVQVBEfH.js.map +1 -0
  19. package/dist/{campaign-evidence-D40-LHIL.js → campaign-evidence-DcQiYhJI.js} +18 -154
  20. package/dist/campaign-evidence-DcQiYhJI.js.map +1 -0
  21. package/dist/claim-V9Zqbq0J.js +54 -0
  22. package/dist/claim-V9Zqbq0J.js.map +1 -0
  23. package/dist/cli.js +99 -2
  24. package/dist/cli.js.map +1 -1
  25. package/dist/client-wFyDisxi.d.ts +236 -0
  26. package/dist/client-wFyDisxi.d.ts.map +1 -0
  27. package/dist/contract/index.d.ts +10 -11
  28. package/dist/contract/index.js +10 -140
  29. package/dist/contract/index.js.map +1 -1
  30. package/dist/{default-registry-B5HrwrPM.d.ts → default-registry-DdqA_-_o.d.ts} +6 -6
  31. package/dist/{default-registry-B5HrwrPM.d.ts.map → default-registry-DdqA_-_o.d.ts.map} +1 -1
  32. package/dist/{default-registry-16I-Bxlf.js → default-registry-xMLX160E.js} +3 -3
  33. package/dist/{default-registry-16I-Bxlf.js.map → default-registry-xMLX160E.js.map} +1 -1
  34. package/dist/{define-agent-eval-DPizhHfy.js → define-agent-eval-DQuos3Aw.js} +30 -113
  35. package/dist/define-agent-eval-DQuos3Aw.js.map +1 -0
  36. package/dist/{define-agent-eval-DblGedQk.d.ts → define-agent-eval-DcludNJS.d.ts} +25 -38
  37. package/dist/define-agent-eval-DcludNJS.d.ts.map +1 -0
  38. package/dist/diagnosis/index.d.ts +2 -2
  39. package/dist/diagnosis/index.js +1 -1
  40. package/dist/{dspy-rlm-engine-Cs5LzJ-f.js → dspy-rlm-engine-BX34vPVY.js} +3 -3
  41. package/dist/{dspy-rlm-engine-Cs5LzJ-f.js.map → dspy-rlm-engine-BX34vPVY.js.map} +1 -1
  42. package/dist/{engine-BkLCnZQX.d.ts → engine-C1DIAEpn.d.ts} +4 -4
  43. package/dist/{engine-BkLCnZQX.d.ts.map → engine-C1DIAEpn.d.ts.map} +1 -1
  44. package/dist/evaluation.d.ts +2 -2
  45. package/dist/{exact-types-CfpP2A4Z.d.ts → exact-types-KGxBWH4b.d.ts} +2 -2
  46. package/dist/{exact-types-CfpP2A4Z.d.ts.map → exact-types-KGxBWH4b.d.ts.map} +1 -1
  47. package/dist/experiment/index.d.ts +1 -1
  48. package/dist/experiment/index.js +2 -1
  49. package/dist/experiment/index.js.map +1 -1
  50. package/dist/{external-optimizer-process-BnSmrkgL.js → external-optimizer-process-D-Sab4v3.js} +3 -62
  51. package/dist/external-optimizer-process-D-Sab4v3.js.map +1 -0
  52. package/dist/{external-optimizer-subprocess-kbcPu5po.js → external-optimizer-subprocess-kbqC-iWf.js} +65 -27
  53. package/dist/external-optimizer-subprocess-kbqC-iWf.js.map +1 -0
  54. package/dist/{extract-usage-BrQ8mCLX.js → extract-usage-DFATEuWW.js} +2 -2
  55. package/dist/{extract-usage-BrQ8mCLX.js.map → extract-usage-DFATEuWW.js.map} +1 -1
  56. package/dist/{feedback-trajectory-BYjh7NEM.d.ts → feedback-trajectory-BXCGTRkz.d.ts} +3 -3
  57. package/dist/{feedback-trajectory-BYjh7NEM.d.ts.map → feedback-trajectory-BXCGTRkz.d.ts.map} +1 -1
  58. package/dist/hosted/index.d.ts +66 -6
  59. package/dist/hosted/index.d.ts.map +1 -1
  60. package/dist/hosted/index.js +2 -2
  61. package/dist/{index-I4UJGRno.d.ts → index-Bj96W0jZ.d.ts} +28 -2
  62. package/dist/{index-I4UJGRno.d.ts.map → index-Bj96W0jZ.d.ts.map} +1 -1
  63. package/dist/{index-2e_MTznj.d.ts → index-DBoXFaz0.d.ts} +6 -6
  64. package/dist/{index-2e_MTznj.d.ts.map → index-DBoXFaz0.d.ts.map} +1 -1
  65. package/dist/{index-CWoIJHmy.d.ts → index-DNVzu9Vv.d.ts} +10 -104
  66. package/dist/index-DNVzu9Vv.d.ts.map +1 -0
  67. package/dist/{index-CJv56rhX.d.ts → index-DwQxdWve.d.ts} +351 -135
  68. package/dist/index-DwQxdWve.d.ts.map +1 -0
  69. package/dist/index.d.ts +16 -17
  70. package/dist/index.d.ts.map +1 -1
  71. package/dist/index.js +11 -11
  72. package/dist/jev.d.ts +2 -2
  73. package/dist/{journal-c3qY0SDP.js → journal-DvKCvjty.js} +94 -2
  74. package/dist/{journal-c3qY0SDP.js.map → journal-DvKCvjty.js.map} +1 -1
  75. package/dist/{kind-factory-ByZeXDqF.js → kind-factory-CdPPHtGZ.js} +10 -26
  76. package/dist/kind-factory-CdPPHtGZ.js.map +1 -0
  77. package/dist/ledger-core/index.d.ts +2 -2
  78. package/dist/ledger-core/index.js +2 -2
  79. package/dist/{llm-judge-BnxAv7ed.js → llm-judge-Z6MkTkd0.js} +1805 -2012
  80. package/dist/llm-judge-Z6MkTkd0.js.map +1 -0
  81. package/dist/{matrix-CNypzP-v.d.ts → matrix-CQMMrEQ5.d.ts} +2 -2
  82. package/dist/{matrix-CNypzP-v.d.ts.map → matrix-CQMMrEQ5.d.ts.map} +1 -1
  83. package/dist/multishot/golden/index.d.ts +1 -1
  84. package/dist/multishot/index.d.ts +2 -2
  85. package/dist/openapi.json +1 -1
  86. package/dist/{otel-spans-CQY1pNiN.js → otel-spans-DS7t1t1J.js} +8 -19
  87. package/dist/otel-spans-DS7t1t1J.js.map +1 -0
  88. package/dist/otlp-attributes-D9ptWaIJ.js +53 -0
  89. package/dist/otlp-attributes-D9ptWaIJ.js.map +1 -0
  90. package/dist/{prime-protocol-Fwvctmhm.d.ts → prime-protocol-T0i6xcAs.d.ts} +2 -2
  91. package/dist/{prime-protocol-Fwvctmhm.d.ts.map → prime-protocol-T0i6xcAs.d.ts.map} +1 -1
  92. package/dist/{produced-state-BXjXJHJT.js → produced-state-CVmCPQDJ.js} +3 -3
  93. package/dist/{produced-state-BXjXJHJT.js.map → produced-state-CVmCPQDJ.js.map} +1 -1
  94. package/dist/{registry-mAQVtuLx.d.ts → registry-Bu3lNole.d.ts} +3 -3
  95. package/dist/{registry-mAQVtuLx.d.ts.map → registry-Bu3lNole.d.ts.map} +1 -1
  96. package/dist/rl.d.ts +1 -1
  97. package/dist/rl.js +1 -1
  98. package/dist/{run-record-DnMFyg_g.js → run-record-tNnlQIrD.js} +2 -10
  99. package/dist/{run-record-DnMFyg_g.js.map → run-record-tNnlQIrD.js.map} +1 -1
  100. package/dist/search-ledger-Ck7tok0h.js +1517 -0
  101. package/dist/search-ledger-Ck7tok0h.js.map +1 -0
  102. package/dist/search-ledger-file-DY0AXX3W.js +27 -0
  103. package/dist/search-ledger-file-DY0AXX3W.js.map +1 -0
  104. package/dist/search-shipper-BG182p-U.js +781 -0
  105. package/dist/search-shipper-BG182p-U.js.map +1 -0
  106. package/dist/{semantic-concept-judge-ClB73cei.js → semantic-concept-judge-Bg-560H0.js} +4 -4
  107. package/dist/{semantic-concept-judge-ClB73cei.js.map → semantic-concept-judge-Bg-560H0.js.map} +1 -1
  108. package/dist/{skillopt-optimization-method-l2vDBtKp.js → skillopt-optimization-method-DoayuM6D.js} +6 -5
  109. package/dist/{skillopt-optimization-method-l2vDBtKp.js.map → skillopt-optimization-method-DoayuM6D.js.map} +1 -1
  110. package/dist/{statistical-heldout-CLpjYffJ.d.ts → statistical-heldout-C1Ej5Ce8.d.ts} +2 -2
  111. package/dist/{statistical-heldout-CLpjYffJ.d.ts.map → statistical-heldout-C1Ej5Ce8.d.ts.map} +1 -1
  112. package/dist/{store-otlp-84DH0OEH.js → store-otlp-BPowgPm6.js} +2 -2
  113. package/dist/{store-otlp-84DH0OEH.js.map → store-otlp-BPowgPm6.js.map} +1 -1
  114. package/dist/{store-tool-spans-DUiqyHvR.js → store-tool-spans-BcC_5ef-.js} +5 -5
  115. package/dist/{store-tool-spans-DUiqyHvR.js.map → store-tool-spans-BcC_5ef-.js.map} +1 -1
  116. package/dist/{store-tool-spans-BBXs8hDP.d.ts → store-tool-spans-ByLdU26j.d.ts} +5 -9
  117. package/dist/store-tool-spans-ByLdU26j.d.ts.map +1 -0
  118. package/dist/{task-failure-attributes-B34TiMBe.js → task-failure-attributes-CEqEfuIY.js} +17 -9
  119. package/dist/task-failure-attributes-CEqEfuIY.js.map +1 -0
  120. package/dist/{tool-groups-W7MJobX_.d.ts → tool-groups-DiBi16ns.d.ts} +3 -3
  121. package/dist/tool-groups-DiBi16ns.d.ts.map +1 -0
  122. package/dist/trace-attributes.d.ts +1 -1
  123. package/dist/trace-attributes.js +23 -75
  124. package/dist/trace-attributes.js.map +1 -1
  125. package/dist/trace-repair/index.d.ts +1 -1
  126. package/dist/traces.d.ts +11 -18
  127. package/dist/traces.d.ts.map +1 -1
  128. package/dist/traces.js +16 -16
  129. package/dist/traces.js.map +1 -1
  130. package/dist/{types-Cvqy7asY.d.ts → types-ByZslIlr.d.ts} +2 -2
  131. package/dist/{types-Cvqy7asY.d.ts.map → types-ByZslIlr.d.ts.map} +1 -1
  132. package/dist/{types-BvmWNjjs.d.ts → types-C1CAD70_.d.ts} +15 -16
  133. package/dist/types-C1CAD70_.d.ts.map +1 -0
  134. package/dist/{types-Bay1dh2o.d.ts → types-CeDfCRXp.d.ts} +11 -16
  135. package/dist/types-CeDfCRXp.d.ts.map +1 -0
  136. package/dist/wire/index.d.ts +1 -1
  137. package/docs/campaign-proposers.md +14 -16
  138. package/docs/hosted-ingest-spec.md +92 -133
  139. package/docs/public-api.md +46 -35
  140. package/docs/search-ledger.md +79 -1
  141. package/package.json +2 -2
  142. package/dist/analyze-runs-BvK4-wSs.d.ts +0 -75
  143. package/dist/analyze-runs-BvK4-wSs.d.ts.map +0 -1
  144. package/dist/attribute-vocabulary-DLJ6303h.d.ts.map +0 -1
  145. package/dist/campaign-B7q74uIh.js.map +0 -1
  146. package/dist/campaign-evidence-D40-LHIL.js.map +0 -1
  147. package/dist/client-BhEyAdfe.d.ts +0 -202
  148. package/dist/client-BhEyAdfe.d.ts.map +0 -1
  149. package/dist/client-QwJpsNYU.js +0 -655
  150. package/dist/client-QwJpsNYU.js.map +0 -1
  151. package/dist/define-agent-eval-DPizhHfy.js.map +0 -1
  152. package/dist/define-agent-eval-DblGedQk.d.ts.map +0 -1
  153. package/dist/external-optimizer-process-BnSmrkgL.js.map +0 -1
  154. package/dist/external-optimizer-subprocess-kbcPu5po.js.map +0 -1
  155. package/dist/index-CJv56rhX.d.ts.map +0 -1
  156. package/dist/index-CWoIJHmy.d.ts.map +0 -1
  157. package/dist/insight-report-CvPkEIcC.d.ts.map +0 -1
  158. package/dist/kind-factory-ByZeXDqF.js.map +0 -1
  159. package/dist/llm-judge-BnxAv7ed.js.map +0 -1
  160. package/dist/otel-spans-CQY1pNiN.js.map +0 -1
  161. package/dist/otlp-attributes-BpC_blLF.js +0 -83
  162. package/dist/otlp-attributes-BpC_blLF.js.map +0 -1
  163. package/dist/store-tool-spans-BBXs8hDP.d.ts.map +0 -1
  164. package/dist/task-failure-attributes-B34TiMBe.js.map +0 -1
  165. package/dist/tool-groups-W7MJobX_.d.ts.map +0 -1
  166. package/dist/types-Bay1dh2o.d.ts.map +0 -1
  167. package/dist/types-BvmWNjjs.d.ts.map +0 -1
@@ -1 +0,0 @@
1
- {"version":3,"file":"client-BhEyAdfe.d.ts","names":[],"sources":["../src/hosted/types.ts","../src/hosted/client.ts"],"mappings":";;;;cAgCa;KACD,2BAA2B;;UAKtB;;EAEf;;EAEA;;EAEA,yBAAyB;;EAEzB;;;KAMU;UAQK;;EAEf;;EAEA;;EAEA;;EAEA,YAAY,eAAe;;EAE3B,iBAAiB;;EAEjB;;EAEA;;UAGe;;EAEf;;;EAGA;;;EAGA,UAAU;;EAEV,OAAO;;EAEP;;EAEA;;EAEA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,QAAQ;;EAER,WAAW;;EAEX,aAAa;;EAEb,eAAe;;EAEf;;EAEA;;EAEA;;EAEA;;;;;;EAMA,gBAAgB;;;;;;KASN;;;;;;UAOK;EACf;EACA;EACA;EACA;EACA,mBAAmB;EACnB,iBAAiB;EACjB,YAAY;EACZ,SAAS;IACP,cAAc;IACd;IACA,aAAa;;EAEf;IAAW;IAAgC;;;EAE3C;;EAEA;;EAEA;;EAEA;;UAKe;EACf,aAAa;EACb,QAAQ;;UAGO;EACf,aAAa;EACb,OAAO;;UAGQ;;EAEf;;EAEA,UAAU;IAAQ;IAAe;;;;;UC1JlB;;EAEf;;EAEA;;EAEA;;EAEA,mBAAmB;;EAEnB;;EAEA;;UAGe;EACf,cAAc,OAAO,cAAc,0BAA0B,QAAQ;EACrE,eAAe,QAAQ,gBAAgB,0BAA0B,QAAQ;EACzE,aAAa,OAAO,kBAAkB,0BAA0B,QAAQ;WAC/D,QAAQ;WACR,aAAa;;iBAyGR,mBAAmB,QAAQ,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAsE1C,oBACd,YAAW,QAAQ;EAAkB,MAAM;IAC1C;iBAiBa,oBACd,YAAW,QAAQ;EAAkB,MAAM;IAC1C"}
@@ -1,655 +0,0 @@
1
- import { t as FAILURE_CLASSES } from "./schema-CSf6qWgZ.js";
2
- import { z } from "zod";
3
- //#region src/hosted/types.ts
4
- const HOSTED_WIRE_VERSION = "2026-07-24.v1";
5
- //#endregion
6
- //#region src/hosted/schemas.ts
7
- const finiteNumber = z.number().finite();
8
- const nonNegativeNumber = finiteNumber.nonnegative();
9
- const nonNegativeInteger = z.number().int().nonnegative();
10
- const nonEmptyString = z.string().min(1).refine((value) => value.trim().length > 0, "must not be blank");
11
- const attributeValue = z.union([
12
- z.string(),
13
- finiteNumber,
14
- z.boolean()
15
- ]);
16
- const attributes = z.record(z.string(), attributeValue);
17
- const FailureClassSchema = z.custom((value) => typeof value === "string" && FAILURE_CLASSES.includes(value), `expected one of ${FAILURE_CLASSES.join(", ")}`);
18
- const UINT64_MAX = 18446744073709551615n;
19
- const UnixNanoTimestampSchema = z.string().regex(/^(0|[1-9][0-9]*)$/, "expected an unsigned base-10 integer string").refine((value) => BigInt(value) <= UINT64_MAX, "must fit in an unsigned 64-bit integer");
20
- const GainDistributionBinSchema = z.object({
21
- lo: finiteNumber,
22
- hi: finiteNumber,
23
- count: nonNegativeInteger
24
- }).strict();
25
- const ScalarDistributionSchema = z.object({
26
- n: nonNegativeInteger,
27
- mean: finiteNumber.nullable(),
28
- p50: finiteNumber.nullable(),
29
- p95: finiteNumber.nullable(),
30
- stddev: nonNegativeNumber.nullable(),
31
- min: finiteNumber.nullable(),
32
- max: finiteNumber.nullable(),
33
- histogram: z.array(GainDistributionBinSchema),
34
- tailRuns: z.array(z.object({
35
- runId: nonEmptyString,
36
- score: finiteNumber
37
- }).strict()).optional()
38
- }).strict().superRefine((distribution, ctx) => {
39
- const values = [
40
- distribution.mean,
41
- distribution.p50,
42
- distribution.p95,
43
- distribution.stddev,
44
- distribution.min,
45
- distribution.max
46
- ];
47
- if (distribution.n === 0 && values.some((value) => value !== null)) ctx.addIssue({
48
- code: "custom",
49
- message: "distribution values must be null when n is 0"
50
- });
51
- if (distribution.n > 0 && values.some((value) => value === null)) ctx.addIssue({
52
- code: "custom",
53
- message: "distribution values must be numbers when n is greater than 0"
54
- });
55
- if (distribution.min !== null && distribution.max !== null && distribution.min > distribution.max) ctx.addIssue({
56
- code: "custom",
57
- path: ["min"],
58
- message: "min must not exceed max"
59
- });
60
- });
61
- const TokenUsageInsightSchema = z.object({
62
- input: ScalarDistributionSchema,
63
- output: ScalarDistributionSchema,
64
- reasoning: ScalarDistributionSchema,
65
- cached: ScalarDistributionSchema,
66
- cacheWrite: ScalarDistributionSchema,
67
- totals: z.object({
68
- input: nonNegativeNumber,
69
- output: nonNegativeNumber,
70
- reasoning: nonNegativeNumber,
71
- cached: nonNegativeNumber,
72
- cacheWrite: nonNegativeNumber
73
- }).strict()
74
- }).strict();
75
- const ExecutionErrorOutcomeCellSchema = z.object({
76
- withErrors: nonNegativeInteger,
77
- withoutErrors: nonNegativeInteger,
78
- unreported: nonNegativeInteger
79
- }).strict();
80
- const ExecutionInsightSchema = z.object({
81
- durationMs: ScalarDistributionSchema,
82
- queueMs: ScalarDistributionSchema,
83
- tokenUsage: TokenUsageInsightSchema,
84
- aggregateUsage: z.object({
85
- runs: nonNegativeInteger,
86
- tokenUsage: TokenUsageInsightSchema,
87
- costUsd: ScalarDistributionSchema,
88
- totalCostUsd: nonNegativeNumber
89
- }).strict(),
90
- models: z.array(z.object({
91
- model: nonEmptyString,
92
- runs: nonNegativeInteger
93
- }).strict()),
94
- modelCalls: z.object({
95
- runs: nonNegativeInteger,
96
- events: nonNegativeInteger,
97
- reportingRuns: nonNegativeInteger
98
- }).strict(),
99
- executionErrors: z.object({
100
- runs: nonNegativeInteger,
101
- fraction: finiteNumber.min(0).max(1).nullable(),
102
- events: nonNegativeInteger,
103
- reportingRuns: nonNegativeInteger,
104
- errorSpanEvents: nonNegativeInteger,
105
- errorSpanReportingRuns: nonNegativeInteger,
106
- byTerminalOutcome: z.object({
107
- succeeded: ExecutionErrorOutcomeCellSchema,
108
- failed: ExecutionErrorOutcomeCellSchema,
109
- cancelled: ExecutionErrorOutcomeCellSchema,
110
- incomplete: ExecutionErrorOutcomeCellSchema,
111
- unknown: ExecutionErrorOutcomeCellSchema
112
- }).strict()
113
- }).strict(),
114
- terminalOutcomes: z.object({
115
- succeeded: nonNegativeInteger,
116
- failed: nonNegativeInteger,
117
- cancelled: nonNegativeInteger,
118
- incomplete: nonNegativeInteger,
119
- unknown: nonNegativeInteger
120
- }).strict()
121
- }).strict();
122
- const CostProvenanceSummarySchema = z.object({
123
- observed: z.object({
124
- n: nonNegativeInteger,
125
- totalUsd: nonNegativeNumber
126
- }).strict(),
127
- estimated: z.object({
128
- n: nonNegativeInteger,
129
- totalUsd: nonNegativeNumber
130
- }).strict(),
131
- lowerBound: z.object({
132
- n: nonNegativeInteger,
133
- floorUsd: nonNegativeNumber
134
- }).strict(),
135
- uncaptured: z.object({ n: nonNegativeInteger }).strict(),
136
- knownFraction: finiteNumber.min(0).max(1)
137
- }).strict();
138
- const ParetoFigureSpecSchema = z.object({
139
- kind: z.literal("pareto-cost-quality"),
140
- split: z.enum(["search", "holdout"]),
141
- points: z.array(z.object({
142
- candidateId: nonEmptyString,
143
- cost: nonNegativeNumber,
144
- quality: finiteNumber,
145
- n: nonNegativeInteger,
146
- onFrontier: z.boolean(),
147
- gate: z.enum(["promote", "reject"]).optional()
148
- }).strict()),
149
- axes: z.object({
150
- x: z.literal("costUsd"),
151
- y: z.literal("score")
152
- }).strict()
153
- }).strict();
154
- const ContinuousAgreementSchema = z.object({
155
- weightedKappa: finiteNumber,
156
- icc: finiteNumber,
157
- pearson: finiteNumber,
158
- spearman: finiteNumber,
159
- ci: z.object({
160
- icc: z.tuple([finiteNumber, finiteNumber]),
161
- weightedKappa: z.tuple([finiteNumber, finiteNumber])
162
- }).strict(),
163
- n: nonNegativeInteger,
164
- raters: nonNegativeInteger
165
- }).strict();
166
- const JudgeInsightSchema = z.object({
167
- n: nonNegativeInteger,
168
- meanScore: finiteNumber,
169
- calibration: ContinuousAgreementSchema.optional(),
170
- positionalBias: finiteNumber.optional(),
171
- selfPreference: finiteNumber.optional(),
172
- verbosityBias: finiteNumber.optional()
173
- }).strict();
174
- const InterRaterInsightSchema = z.object({
175
- raters: nonNegativeInteger,
176
- jointlyRated: nonNegativeInteger,
177
- kappa: finiteNumber,
178
- icc: finiteNumber,
179
- pearson: finiteNumber,
180
- spearman: finiteNumber,
181
- perPair: z.record(z.string(), finiteNumber),
182
- disagreementCases: z.array(z.object({
183
- runId: nonEmptyString,
184
- ratings: z.array(z.object({
185
- rater: nonEmptyString,
186
- score: finiteNumber
187
- }).strict()),
188
- range: nonNegativeNumber
189
- }).strict())
190
- }).strict();
191
- const LiftInsightSchema = z.object({
192
- baselineMean: finiteNumber,
193
- candidateMean: finiteNumber,
194
- delta: finiteNumber,
195
- ci95: z.tuple([finiteNumber, finiteNumber]),
196
- pValue: finiteNumber.min(0).max(1).nullable(),
197
- n: nonNegativeInteger,
198
- pairedRunN: nonNegativeInteger.optional(),
199
- independentUnitIds: z.array(nonEmptyString).optional(),
200
- minimumRequired: z.number().int().positive(),
201
- decisionEligible: z.boolean(),
202
- unpairedBaseline: nonNegativeInteger,
203
- unpairedCandidate: nonNegativeInteger,
204
- cohensD: finiteNumber.nullable(),
205
- mde: nonNegativeNumber,
206
- requiredN: nonNegativeInteger.nullable()
207
- }).strict();
208
- const FailureClusterInsightSchema = z.object({
209
- clusters: z.array(z.object({
210
- id: nonEmptyString,
211
- name: nonEmptyString,
212
- share: finiteNumber.min(0).max(1),
213
- exemplars: z.array(nonEmptyString).max(5),
214
- suggestedFix: nonEmptyString.optional()
215
- }).strict()),
216
- totalFailures: nonNegativeInteger
217
- }).strict();
218
- const ContaminationInsightSchema = z.object({
219
- leaks: nonNegativeInteger,
220
- holdoutAuditPassed: z.boolean(),
221
- details: z.array(z.object({
222
- runId: nonEmptyString,
223
- canary: nonEmptyString,
224
- matched: nonEmptyString
225
- }).strict()).optional()
226
- }).strict();
227
- const OutcomeCorrelationInsightSchema = z.object({
228
- metric: nonEmptyString,
229
- n: nonNegativeInteger,
230
- pearson: finiteNumber,
231
- spearman: finiteNumber,
232
- rewardModel: z.object({
233
- intercept: finiteNumber,
234
- slope: finiteNumber,
235
- r2: finiteNumber
236
- }).strict().optional()
237
- }).strict();
238
- const ReleaseSummarySchema = z.object({
239
- status: z.enum([
240
- "pass",
241
- "warn",
242
- "fail"
243
- ]),
244
- axes: z.array(z.object({
245
- name: z.enum([
246
- "quality-lift",
247
- "contamination",
248
- "composite-distribution"
249
- ]),
250
- status: z.enum([
251
- "pass",
252
- "warn",
253
- "fail",
254
- "not_evaluated"
255
- ]),
256
- detail: nonEmptyString
257
- }).strict()),
258
- issues: z.array(z.string())
259
- }).strict();
260
- const MetricDeltaBaseSchema = z.object({
261
- current: finiteNumber,
262
- baseline: finiteNumber,
263
- delta: finiteNumber,
264
- baselineN: nonNegativeInteger,
265
- currentN: nonNegativeInteger
266
- }).strict();
267
- const MetricDeltaSchema = z.discriminatedUnion("status", [MetricDeltaBaseSchema.extend({
268
- status: z.literal("ok"),
269
- ci95: z.tuple([finiteNumber, finiteNumber]),
270
- pValue: finiteNumber.min(0).max(1),
271
- cohensD: finiteNumber,
272
- significant: z.boolean()
273
- }), MetricDeltaBaseSchema.extend({
274
- status: z.enum(["insufficient-sample", "zero-variance"]),
275
- ci95: z.null(),
276
- pValue: z.null(),
277
- cohensD: z.null(),
278
- significant: z.literal(false)
279
- })]);
280
- const PriorPeriodComparisonSchema = z.object({
281
- baselineN: nonNegativeInteger,
282
- currentN: nonNegativeInteger,
283
- windowLabel: nonEmptyString.optional(),
284
- metrics: z.record(z.string(), MetricDeltaSchema),
285
- regressedMetrics: z.array(nonEmptyString),
286
- improvedMetrics: z.array(nonEmptyString),
287
- inconclusiveMetrics: z.array(nonEmptyString)
288
- }).strict();
289
- const RecommendationSchema = z.object({
290
- priority: z.enum([
291
- "critical",
292
- "high",
293
- "medium",
294
- "low"
295
- ]),
296
- kind: z.enum([
297
- "ship",
298
- "hold",
299
- "investigate",
300
- "fix",
301
- "recalibrate",
302
- "expand-corpus"
303
- ]),
304
- title: nonEmptyString,
305
- detail: nonEmptyString,
306
- evidencePath: nonEmptyString.optional()
307
- }).strict();
308
- const InsightReportSchema = z.object({
309
- n: nonNegativeInteger,
310
- execution: ExecutionInsightSchema,
311
- composite: ScalarDistributionSchema,
312
- perDimension: z.record(z.string(), ScalarDistributionSchema),
313
- costQuality: z.object({
314
- cost: ScalarDistributionSchema,
315
- pareto: ParetoFigureSpecSchema,
316
- provenance: CostProvenanceSummarySchema.optional(),
317
- degraded: z.object({
318
- cost: nonEmptyString.optional(),
319
- pareto: nonEmptyString.optional()
320
- }).strict().optional()
321
- }).strict(),
322
- judges: z.record(z.string(), JudgeInsightSchema),
323
- interRater: InterRaterInsightSchema.optional(),
324
- lift: LiftInsightSchema.optional(),
325
- failureClusters: FailureClusterInsightSchema.optional(),
326
- contamination: ContaminationInsightSchema.optional(),
327
- outcomeCorrelation: OutcomeCorrelationInsightSchema.optional(),
328
- release: ReleaseSummarySchema,
329
- priorPeriodComparison: PriorPeriodComparisonSchema.optional(),
330
- failureClasses: z.array(z.object({
331
- failureClass: FailureClassSchema,
332
- count: nonNegativeInteger,
333
- share: finiteNumber.min(0).max(1)
334
- }).strict()).optional(),
335
- recommendations: z.array(RecommendationSchema)
336
- }).strict();
337
- const sha256Digest = z.custom((value) => typeof value === "string" && /^sha256:[0-9a-f]{64}$/.test(value), "expected sha256:<64 lowercase hex characters>");
338
- const MutableSurfaceSchema = z.union([
339
- z.string(),
340
- z.object({
341
- kind: z.literal("components"),
342
- components: z.record(z.string(), z.string())
343
- }).strict(),
344
- z.object({
345
- kind: z.literal("code"),
346
- worktreeRef: nonEmptyString,
347
- baseRef: nonEmptyString,
348
- baseCommit: nonEmptyString,
349
- baseTree: nonEmptyString,
350
- candidateCommit: nonEmptyString,
351
- candidateTree: nonEmptyString,
352
- patch: z.object({
353
- format: z.literal("git-diff-binary"),
354
- sha256: sha256Digest,
355
- byteLength: nonNegativeInteger
356
- }).strict(),
357
- summary: z.string().optional()
358
- }).strict()
359
- ]);
360
- const RunTerminalOutcomeSchema = z.enum([
361
- "succeeded",
362
- "failed",
363
- "cancelled",
364
- "incomplete",
365
- "unknown"
366
- ]);
367
- const EvalRunCellScoreSchema = z.object({
368
- scenarioId: nonEmptyString,
369
- rep: nonNegativeInteger,
370
- compositeMean: finiteNumber.nullable(),
371
- dimensions: z.record(z.string(), z.record(z.string(), finiteNumber)),
372
- terminalOutcome: RunTerminalOutcomeSchema,
373
- executionErrorCount: nonNegativeInteger.nullable(),
374
- errorMessage: z.string().optional()
375
- }).strict();
376
- const EvalRunGenerationSnapshotSchema = z.object({
377
- index: nonNegativeInteger,
378
- surfaceHash: nonEmptyString,
379
- surface: MutableSurfaceSchema.optional(),
380
- cells: z.array(EvalRunCellScoreSchema),
381
- compositeMean: finiteNumber.nullable(),
382
- costUsd: nonNegativeNumber,
383
- durationMs: nonNegativeNumber
384
- }).strict();
385
- const EvalRunStatusSchema = z.enum([
386
- "started",
387
- "baseline-complete",
388
- "generation-complete",
389
- "gate-decided",
390
- "finished",
391
- "errored"
392
- ]);
393
- const GateDecisionSchema = z.enum([
394
- "ship",
395
- "hold",
396
- "need_more_work",
397
- "model_ceiling",
398
- "arch_ceiling"
399
- ]);
400
- const EvalRunEventSchema = z.object({
401
- runId: nonEmptyString,
402
- runDir: nonEmptyString,
403
- timestamp: z.string().datetime({ offset: true }),
404
- status: EvalRunStatusSchema,
405
- labels: z.record(z.string(), z.string()),
406
- baseline: EvalRunGenerationSnapshotSchema.optional(),
407
- generations: z.array(EvalRunGenerationSnapshotSchema),
408
- gateDecision: GateDecisionSchema.optional(),
409
- holdoutLift: finiteNumber.optional(),
410
- totalCostUsd: nonNegativeNumber,
411
- totalDurationMs: nonNegativeNumber,
412
- errorMessage: z.string().optional(),
413
- insightReport: InsightReportSchema.optional()
414
- }).strict().superRefine((event, ctx) => {
415
- if (event.baseline && event.baseline.index !== 0) ctx.addIssue({
416
- code: "custom",
417
- path: ["baseline", "index"],
418
- message: "baseline index must be 0"
419
- });
420
- const seen = /* @__PURE__ */ new Set();
421
- for (let i = 0; i < event.generations.length; i++) {
422
- const index = event.generations[i].index;
423
- if (seen.has(index)) ctx.addIssue({
424
- code: "custom",
425
- path: [
426
- "generations",
427
- i,
428
- "index"
429
- ],
430
- message: `duplicate generation index ${index}`
431
- });
432
- seen.add(index);
433
- }
434
- if (event.status === "errored" && !event.errorMessage?.trim()) ctx.addIssue({
435
- code: "custom",
436
- path: ["errorMessage"],
437
- message: "errorMessage is required when status is errored"
438
- });
439
- });
440
- const TraceSpanEventEntrySchema = z.object({
441
- timeUnixNano: UnixNanoTimestampSchema,
442
- name: nonEmptyString,
443
- attributes: attributes.optional()
444
- }).strict();
445
- const TraceSpanEventSchema = z.object({
446
- traceId: nonEmptyString,
447
- spanId: nonEmptyString,
448
- parentSpanId: nonEmptyString.optional(),
449
- name: nonEmptyString,
450
- startTimeUnixNano: UnixNanoTimestampSchema,
451
- endTimeUnixNano: UnixNanoTimestampSchema,
452
- attributes,
453
- events: z.array(TraceSpanEventEntrySchema).optional(),
454
- status: z.object({
455
- code: z.enum([
456
- "OK",
457
- "ERROR",
458
- "UNSET"
459
- ]),
460
- message: z.string().optional()
461
- }).strict().optional(),
462
- "tangle.runId": nonEmptyString.optional(),
463
- "tangle.generation": nonNegativeInteger.optional(),
464
- "tangle.cellId": nonEmptyString.optional(),
465
- "tangle.scenarioId": nonEmptyString.optional()
466
- }).strict().refine((span) => BigInt(span.endTimeUnixNano) >= BigInt(span.startTimeUnixNano), {
467
- path: ["endTimeUnixNano"],
468
- message: "endTimeUnixNano must be greater than or equal to startTimeUnixNano"
469
- });
470
- z.object({
471
- wireVersion: z.literal(HOSTED_WIRE_VERSION),
472
- events: z.array(z.unknown())
473
- }).strict();
474
- z.object({
475
- wireVersion: z.literal(HOSTED_WIRE_VERSION),
476
- spans: z.array(z.unknown())
477
- }).strict();
478
- const IngestEvalRunsRequestSchema = z.object({
479
- wireVersion: z.literal(HOSTED_WIRE_VERSION),
480
- events: z.array(EvalRunEventSchema)
481
- }).strict();
482
- const IngestTracesRequestSchema = z.object({
483
- wireVersion: z.literal(HOSTED_WIRE_VERSION),
484
- spans: z.array(TraceSpanEventSchema)
485
- }).strict();
486
- const IngestResponseSchema = z.object({
487
- accepted: nonNegativeInteger,
488
- rejected: z.array(z.object({
489
- index: nonNegativeInteger,
490
- reason: nonEmptyString
491
- }).strict())
492
- }).strict();
493
- //#endregion
494
- //#region src/hosted/client.ts
495
- /**
496
- * # Hosted-tier ingest client.
497
- *
498
- * Ships eval-run events + trace spans to any orchestrator (ours, a
499
- * partner's self-hosted one, or a future open implementation) that
500
- * speaks the wire format in `./types.ts`.
501
- *
502
- * Three modes:
503
- * - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —
504
- * the client appends the versioned `/v1/ingest/...` path itself; a trailing
505
- * `/v1` on the endpoint is tolerated and normalized away). We handle ingest
506
- * + storage + dashboard.
507
- * - **Self-hosted:** point at whatever URL runs the reference receiver
508
- * from `examples/hosted-ingest-server/`.
509
- * - **Off (default):** when `hostedTenant` is unset, nothing is sent.
510
- * Everything stays local.
511
- */
512
- const MAX_IDEMPOTENCY_KEY_LENGTH = 256;
513
- function sleep(ms) {
514
- return new Promise((resolve) => setTimeout(resolve, ms));
515
- }
516
- function normalizeHostedBase(endpoint) {
517
- return endpoint.trim().replace(/\/+$/, "").replace(/\/v1$/, "");
518
- }
519
- function resolveIdempotencyKey(key) {
520
- const resolved = key ?? globalThis.crypto.randomUUID();
521
- if (resolved.trim().length === 0) throw new Error("idempotency key must not be blank");
522
- if (resolved.length > MAX_IDEMPOTENCY_KEY_LENGTH) throw new Error(`idempotency key must be at most ${MAX_IDEMPOTENCY_KEY_LENGTH} characters`);
523
- return resolved;
524
- }
525
- function responseValidationReason(error) {
526
- return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.map(String).join(".") : "value"}: ${issue.message}`).join("; ");
527
- }
528
- async function post(tenant, path, body, opts = {}) {
529
- const timeoutMs = tenant.timeoutMs ?? 3e4;
530
- const maxRetries = tenant.retries ?? 2;
531
- const f = tenant.fetchImpl ?? ((...args) => fetch(...args));
532
- const url = `${normalizeHostedBase(tenant.endpoint)}${path}`;
533
- const idempotencyKey = resolveIdempotencyKey(opts.idempotencyKey);
534
- const headers = {
535
- "content-type": "application/json",
536
- authorization: `Bearer ${tenant.apiKey}`,
537
- "x-tangle-tenant-id": tenant.tenantId,
538
- "x-tangle-wire-version": HOSTED_WIRE_VERSION,
539
- "idempotency-key": idempotencyKey
540
- };
541
- let lastError;
542
- for (let attempt = 0; attempt <= maxRetries; attempt++) {
543
- const ourTimeout = AbortSignal.timeout(timeoutMs);
544
- const combinedSignal = opts.signal ? AbortSignal.any([opts.signal, ourTimeout]) : ourTimeout;
545
- let res;
546
- try {
547
- res = await f(url, {
548
- method: "POST",
549
- headers,
550
- body: JSON.stringify(body),
551
- signal: combinedSignal
552
- });
553
- } catch (err) {
554
- if (opts.signal?.aborted) throw err;
555
- lastError = err;
556
- if (attempt === maxRetries) throw err;
557
- await sleep(2 ** attempt * 200 + Math.random() * 200);
558
- continue;
559
- }
560
- if (!res.ok) {
561
- const text = await res.text().catch(() => "");
562
- const error = /* @__PURE__ */ new Error(`hosted ingest ${url} failed (${res.status}): ${text.slice(0, 500)}`);
563
- if (!(res.status >= 500 || res.status === 408 || res.status === 429) || attempt === maxRetries) throw error;
564
- lastError = error;
565
- await sleep(2 ** attempt * 200 + Math.random() * 200);
566
- continue;
567
- }
568
- let rawResponse;
569
- try {
570
- rawResponse = await res.json();
571
- } catch (error) {
572
- throw new Error(`hosted ingest ${url} returned invalid JSON`, { cause: error });
573
- }
574
- const parsed = IngestResponseSchema.safeParse(rawResponse);
575
- if (!parsed.success) throw new Error(`hosted ingest ${url} returned an invalid response: ${responseValidationReason(parsed.error)}`);
576
- return parsed.data;
577
- }
578
- throw lastError ?? /* @__PURE__ */ new Error("hosted ingest exhausted retries");
579
- }
580
- function createHostedClient(tenant) {
581
- if (normalizeHostedBase(tenant.endpoint).length === 0) throw new Error("endpoint is required");
582
- if (tenant.apiKey.trim().length === 0) throw new Error("apiKey is required");
583
- if (tenant.tenantId.trim().length === 0) throw new Error("tenantId is required");
584
- if (tenant.timeoutMs !== void 0 && (!Number.isFinite(tenant.timeoutMs) || tenant.timeoutMs <= 0)) throw new Error("timeoutMs must be greater than 0");
585
- if (tenant.retries !== void 0 && (!Number.isInteger(tenant.retries) || tenant.retries < 0)) throw new Error("retries must be a non-negative integer");
586
- return {
587
- tenant,
588
- wireVersion: HOSTED_WIRE_VERSION,
589
- async ingestEvalRun(event, idempotencyKey) {
590
- return this.ingestEvalRuns([event], idempotencyKey);
591
- },
592
- async ingestEvalRuns(events, idempotencyKey) {
593
- return post(tenant, "/v1/ingest/eval-runs", IngestEvalRunsRequestSchema.parse({
594
- wireVersion: HOSTED_WIRE_VERSION,
595
- events
596
- }), { idempotencyKey });
597
- },
598
- async ingestTraces(spans, idempotencyKey) {
599
- return post(tenant, "/v1/ingest/traces", IngestTracesRequestSchema.parse({
600
- wireVersion: HOSTED_WIRE_VERSION,
601
- spans
602
- }), { idempotencyKey });
603
- }
604
- };
605
- }
606
- /**
607
- * Build a `HostedClient` from environment, or `undefined` when ingest is not
608
- * configured — the canonical, fail-soft wiring every product uses so eval-run +
609
- * trace provenance lands in the Intelligence dashboard with ONE call:
610
- *
611
- * const hosted = hostedClientFromEnv()
612
- * // ...run the loop...
613
- * await emitLoopProvenance({ ..., hostedClient: hosted }) // no-op if undefined
614
- *
615
- * Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is
616
- * missing — so a product wires the ship call unconditionally and it stays a
617
- * no-op until the env is set. Env precedence:
618
- * - endpoint: `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`
619
- * - apiKey: `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`
620
- * - tenantId: `TANGLE_TENANT_ID`
621
- * A trailing slash on the endpoint is stripped. Pass `overrides` to supply any
622
- * field directly (e.g. a fixed `tenantId` per product) — overrides win over env.
623
- */
624
- /**
625
- * Build a {@link HostedTenant} config from env — the input `selfImprove`'s
626
- * `hostedTenant` and `emitLoopProvenance` take. Same env precedence + overrides
627
- * as {@link hostedClientFromEnv}; returns `undefined` (not an error) when any of
628
- * endpoint / apiKey / tenantId is missing, so a product wires
629
- * `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally
630
- * and it stays off until the env is set.
631
- */
632
- function hostedTenantFromEnv(overrides = {}) {
633
- const env = overrides.env ?? process.env;
634
- const endpoint = (overrides.endpoint ?? env.TANGLE_INGEST_URL ?? env.TANGLE_ORCHESTRATOR_URL)?.trim();
635
- const apiKey = (overrides.apiKey ?? env.TANGLE_INGEST_API_KEY ?? env.TANGLE_API_KEY)?.trim();
636
- const tenantId = (overrides.tenantId ?? env.TANGLE_TENANT_ID)?.trim();
637
- if (!endpoint || !apiKey || !tenantId) return void 0;
638
- const tenant = {
639
- endpoint: endpoint.replace(/\/+$/, ""),
640
- apiKey,
641
- tenantId
642
- };
643
- if (overrides.fetchImpl) tenant.fetchImpl = overrides.fetchImpl;
644
- if (overrides.timeoutMs !== void 0) tenant.timeoutMs = overrides.timeoutMs;
645
- if (overrides.retries !== void 0) tenant.retries = overrides.retries;
646
- return tenant;
647
- }
648
- function hostedClientFromEnv(overrides = {}) {
649
- const tenant = hostedTenantFromEnv(overrides);
650
- return tenant ? createHostedClient(tenant) : void 0;
651
- }
652
- //#endregion
653
- export { IngestEvalRunsRequestSchema as a, InsightReportSchema as c, HOSTED_WIRE_VERSION as d, EvalRunEventSchema as i, TraceSpanEventSchema as l, hostedClientFromEnv as n, IngestResponseSchema as o, hostedTenantFromEnv as r, IngestTracesRequestSchema as s, createHostedClient as t, UnixNanoTimestampSchema as u };
654
-
655
- //# sourceMappingURL=client-QwJpsNYU.js.map