@tangle-network/agent-eval 0.120.0 → 0.120.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/package.json +1 -1
- package/dist/analyst/index.d.ts +0 -3111
- package/dist/analyst/index.js +0 -403
- package/dist/analyst/index.js.map +0 -1
- package/dist/authenticity/index.d.ts +0 -161
- package/dist/authenticity/index.js +0 -215
- package/dist/authenticity/index.js.map +0 -1
- package/dist/belief-state/index.d.ts +0 -1301
- package/dist/belief-state/index.js +0 -2152
- package/dist/belief-state/index.js.map +0 -1
- package/dist/benchmarks/index.d.ts +0 -974
- package/dist/benchmarks/index.js +0 -60
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/builder-eval/index.d.ts +0 -695
- package/dist/builder-eval/index.js +0 -366
- package/dist/builder-eval/index.js.map +0 -1
- package/dist/campaign/index.d.ts +0 -7454
- package/dist/campaign/index.js +0 -272
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-32BZXMSO.js +0 -3878
- package/dist/chunk-32BZXMSO.js.map +0 -1
- package/dist/chunk-3A246TSA.js +0 -998
- package/dist/chunk-3A246TSA.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-3YYRZDON.js +0 -45
- package/dist/chunk-3YYRZDON.js.map +0 -1
- package/dist/chunk-4I2E3LLO.js +0 -1030
- package/dist/chunk-4I2E3LLO.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js +0 -312
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js +0 -880
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-DTJ6QUQB.js +0 -131
- package/dist/chunk-DTJ6QUQB.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-H5UD2323.js +0 -286
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-HKUCJ437.js +0 -787
- package/dist/chunk-HKUCJ437.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-JHOJHHU7.js +0 -867
- package/dist/chunk-JHOJHHU7.js.map +0 -1
- package/dist/chunk-JM2SKQMS.js +0 -750
- package/dist/chunk-JM2SKQMS.js.map +0 -1
- package/dist/chunk-JN2FCO5W.js +0 -7958
- package/dist/chunk-JN2FCO5W.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js +0 -577
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-NJC7U437.js +0 -626
- package/dist/chunk-NJC7U437.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-OYZAPX5G.js +0 -1526
- package/dist/chunk-OYZAPX5G.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PICTDURQ.js +0 -766
- package/dist/chunk-PICTDURQ.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js +0 -1182
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-PXD6ZFNY.js +0 -1107
- package/dist/chunk-PXD6ZFNY.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-QBRSJK47.js +0 -622
- package/dist/chunk-QBRSJK47.js.map +0 -1
- package/dist/chunk-QWMPPZ3X.js +0 -550
- package/dist/chunk-QWMPPZ3X.js.map +0 -1
- package/dist/chunk-S3UZOQ5Y.js +0 -328
- package/dist/chunk-S3UZOQ5Y.js.map +0 -1
- package/dist/chunk-S5TT5R3L.js +0 -2668
- package/dist/chunk-S5TT5R3L.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-U5CHZ5M3.js +0 -357
- package/dist/chunk-U5CHZ5M3.js.map +0 -1
- package/dist/chunk-ULOKLHIQ.js +0 -1937
- package/dist/chunk-ULOKLHIQ.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VSMTAMNK.js +0 -53
- package/dist/chunk-VSMTAMNK.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WW2A73HW.js +0 -159
- package/dist/chunk-WW2A73HW.js.map +0 -1
- package/dist/chunk-X4UCIOTZ.js +0 -136
- package/dist/chunk-X4UCIOTZ.js.map +0 -1
- package/dist/chunk-XDIRG3TO.js +0 -1266
- package/dist/chunk-XDIRG3TO.js.map +0 -1
- package/dist/chunk-XJYR7XFV.js +0 -317
- package/dist/chunk-XJYR7XFV.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- package/dist/cli.d.ts +0 -1
- package/dist/cli.js +0 -112
- package/dist/cli.js.map +0 -1
- package/dist/contract/index.d.ts +0 -4969
- package/dist/contract/index.js +0 -1653
- package/dist/contract/index.js.map +0 -1
- package/dist/control.d.ts +0 -1013
- package/dist/control.js +0 -34
- package/dist/control.js.map +0 -1
- package/dist/fuzz.d.ts +0 -759
- package/dist/fuzz.js +0 -714
- package/dist/fuzz.js.map +0 -1
- package/dist/hosted/index.d.ts +0 -730
- package/dist/hosted/index.js +0 -14
- package/dist/hosted/index.js.map +0 -1
- package/dist/index.d.ts +0 -16780
- package/dist/index.js +0 -12168
- package/dist/index.js.map +0 -1
- package/dist/matrix/index.d.ts +0 -155
- package/dist/matrix/index.js +0 -8
- package/dist/matrix/index.js.map +0 -1
- package/dist/meta-eval/index.d.ts +0 -1030
- package/dist/meta-eval/index.js +0 -417
- package/dist/meta-eval/index.js.map +0 -1
- package/dist/multishot/index.d.ts +0 -579
- package/dist/multishot/index.js +0 -589
- package/dist/multishot/index.js.map +0 -1
- package/dist/openapi.json +0 -992
- package/dist/pipelines/index.d.ts +0 -567
- package/dist/pipelines/index.js +0 -515
- package/dist/pipelines/index.js.map +0 -1
- package/dist/reporting.d.ts +0 -1277
- package/dist/reporting.js +0 -48
- package/dist/reporting.js.map +0 -1
- package/dist/rl.d.ts +0 -4092
- package/dist/rl.js +0 -1724
- package/dist/rl.js.map +0 -1
- package/dist/run-campaign-HNFPJET4.js +0 -14
- package/dist/run-campaign-HNFPJET4.js.map +0 -1
- package/dist/storyboard/index.d.ts +0 -279
- package/dist/storyboard/index.js +0 -767
- package/dist/storyboard/index.js.map +0 -1
- package/dist/trace-attributes.d.ts +0 -52
- package/dist/trace-attributes.js +0 -62
- package/dist/trace-attributes.js.map +0 -1
- package/dist/traces.d.ts +0 -2343
- package/dist/traces.js +0 -249
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.d.ts +0 -1252
- package/dist/wire/index.js +0 -81
- package/dist/wire/index.js.map +0 -1
package/dist/chunk-ZET2UAYW.js
DELETED
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
isJudgeSpan,
|
|
3
|
-
isLlmSpan,
|
|
4
|
-
isToolSpan
|
|
5
|
-
} from "./chunk-MA6HLL3S.js";
|
|
6
|
-
|
|
7
|
-
// src/trace/query.ts
|
|
8
|
-
async function runsForScenario(store, scenarioId) {
|
|
9
|
-
return store.listRuns({ scenarioId });
|
|
10
|
-
}
|
|
11
|
-
async function llmSpans(store, runId) {
|
|
12
|
-
const spans = await store.spans({ runId, kind: "llm" });
|
|
13
|
-
return spans.filter(isLlmSpan);
|
|
14
|
-
}
|
|
15
|
-
async function toolSpans(store, runId, toolName) {
|
|
16
|
-
const spans = await store.spans({ runId, kind: "tool", toolName });
|
|
17
|
-
return spans.filter(isToolSpan);
|
|
18
|
-
}
|
|
19
|
-
async function judgeSpans(store, runId) {
|
|
20
|
-
const spans = await store.spans({ runId, kind: "judge" });
|
|
21
|
-
return spans.filter(isJudgeSpan);
|
|
22
|
-
}
|
|
23
|
-
function groupBy(items, key) {
|
|
24
|
-
const map = /* @__PURE__ */ new Map();
|
|
25
|
-
for (const item of items) {
|
|
26
|
-
const k = key(item);
|
|
27
|
-
let bucket = map.get(k);
|
|
28
|
-
if (!bucket) {
|
|
29
|
-
bucket = [];
|
|
30
|
-
map.set(k, bucket);
|
|
31
|
-
}
|
|
32
|
-
bucket.push(item);
|
|
33
|
-
}
|
|
34
|
-
return map;
|
|
35
|
-
}
|
|
36
|
-
function argHash(args) {
|
|
37
|
-
return stableStringify(args);
|
|
38
|
-
}
|
|
39
|
-
function hasCapturedToolArgs(span) {
|
|
40
|
-
return span.argsCaptured !== false;
|
|
41
|
-
}
|
|
42
|
-
function stableStringify(value) {
|
|
43
|
-
if (value === void 0) return "undefined";
|
|
44
|
-
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "undefined";
|
|
45
|
-
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
|
46
|
-
const obj = value;
|
|
47
|
-
const keys = Object.keys(obj).filter((k) => obj[k] !== void 0).sort();
|
|
48
|
-
const parts = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`);
|
|
49
|
-
return `{${parts.join(",")}}`;
|
|
50
|
-
}
|
|
51
|
-
function aggregateLlm(spans) {
|
|
52
|
-
return spans.reduce(
|
|
53
|
-
(acc, s) => ({
|
|
54
|
-
inputTokens: acc.inputTokens + (s.inputTokens ?? 0),
|
|
55
|
-
outputTokens: acc.outputTokens + (s.outputTokens ?? 0),
|
|
56
|
-
cachedTokens: acc.cachedTokens + (s.cachedTokens ?? 0),
|
|
57
|
-
cacheWriteTokens: acc.cacheWriteTokens + (s.cacheWriteTokens ?? 0),
|
|
58
|
-
reasoningTokens: acc.reasoningTokens + (s.reasoningTokens ?? 0),
|
|
59
|
-
costUsd: acc.costUsd + (s.costUsd ?? 0)
|
|
60
|
-
}),
|
|
61
|
-
{
|
|
62
|
-
inputTokens: 0,
|
|
63
|
-
outputTokens: 0,
|
|
64
|
-
cachedTokens: 0,
|
|
65
|
-
cacheWriteTokens: 0,
|
|
66
|
-
reasoningTokens: 0,
|
|
67
|
-
costUsd: 0
|
|
68
|
-
}
|
|
69
|
-
);
|
|
70
|
-
}
|
|
71
|
-
function runFailureClass(run) {
|
|
72
|
-
if (run.outcome?.failureClass) return run.outcome.failureClass;
|
|
73
|
-
if (run.status === "completed" && run.outcome?.pass !== false) return "success";
|
|
74
|
-
if (run.status === "aborted") return "budget_exceeded";
|
|
75
|
-
return "unknown";
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
export {
|
|
79
|
-
runsForScenario,
|
|
80
|
-
llmSpans,
|
|
81
|
-
toolSpans,
|
|
82
|
-
judgeSpans,
|
|
83
|
-
groupBy,
|
|
84
|
-
argHash,
|
|
85
|
-
hasCapturedToolArgs,
|
|
86
|
-
aggregateLlm,
|
|
87
|
-
runFailureClass
|
|
88
|
-
};
|
|
89
|
-
//# sourceMappingURL=chunk-ZET2UAYW.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/trace/query.ts"],"sourcesContent":["/**\n * Typed query helpers over TraceStore.\n *\n * Not a full SQL engine — a minimal, composable set of operators that\n * cover the canned-pipeline use cases. For ad-hoc analytics, persist to\n * NDJSON and point DuckDB at it; the schema is stable so external SQL\n * tooling works out of the box.\n */\n\nimport type { FailureClass, JudgeSpan, LlmSpan, Run, ToolSpan } from './schema'\nimport { isJudgeSpan, isLlmSpan, isToolSpan } from './schema'\nimport type { TraceStore } from './store'\n\nexport async function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]> {\n return store.listRuns({ scenarioId })\n}\n\nexport async function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]> {\n const spans = await store.spans({ runId, kind: 'llm' })\n return spans.filter(isLlmSpan)\n}\n\nexport async function toolSpans(\n store: TraceStore,\n runId?: string,\n toolName?: string,\n): Promise<ToolSpan[]> {\n const spans = await store.spans({ runId, kind: 'tool', toolName })\n return spans.filter(isToolSpan)\n}\n\n/** Query judge-kind spans from the trace store, optionally scoped to a single run. */\nexport async function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]> {\n const spans = await store.spans({ runId, kind: 'judge' })\n return spans.filter(isJudgeSpan)\n}\n\n/** Group spans by any key selector. */\nexport function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]> {\n const map = new Map<K, T[]>()\n for (const item of items) {\n const k = key(item)\n let bucket = map.get(k)\n if (!bucket) {\n bucket = []\n map.set(k, bucket)\n }\n bucket.push(item)\n }\n return map\n}\n\n/** Hash tool arguments to an orderless-key-stable string for de-duplication. */\nexport function argHash(args: unknown): string {\n return stableStringify(args)\n}\n\n/** Whether argument-based comparisons are valid for this tool call. */\nexport function hasCapturedToolArgs(span: ToolSpan): boolean {\n return span.argsCaptured !== false\n}\n\nfunction stableStringify(value: unknown): string {\n // Must ALWAYS return a string: JSON.stringify(undefined) — and stringify of\n // functions/symbols — yields the JS value `undefined`, which would make\n // argHash return a non-string and silently break the de-dup keys behind\n // stuck-loop and failure-cluster.\n if (value === undefined) return 'undefined'\n if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'undefined'\n if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n // Drop undefined-valued keys to match JSON.stringify semantics, so\n // `{a:1}` and `{a:1,b:undefined}` hash identically.\n const obj = value as Record<string, unknown>\n const keys = Object.keys(obj)\n .filter((k) => obj[k] !== undefined)\n .sort()\n const parts = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`)\n return `{${parts.join(',')}}`\n}\n\n/** Sum an LLM-span array into aggregate token + cost. */\nexport function aggregateLlm(spans: LlmSpan[]): {\n inputTokens: number\n outputTokens: number\n cachedTokens: number\n cacheWriteTokens: number\n reasoningTokens: number\n costUsd: number\n} {\n return spans.reduce(\n (acc, s) => ({\n inputTokens: acc.inputTokens + (s.inputTokens ?? 0),\n outputTokens: acc.outputTokens + (s.outputTokens ?? 0),\n cachedTokens: acc.cachedTokens + (s.cachedTokens ?? 0),\n cacheWriteTokens: acc.cacheWriteTokens + (s.cacheWriteTokens ?? 0),\n reasoningTokens: acc.reasoningTokens + (s.reasoningTokens ?? 0),\n costUsd: acc.costUsd + (s.costUsd ?? 0),\n }),\n {\n inputTokens: 0,\n outputTokens: 0,\n cachedTokens: 0,\n cacheWriteTokens: 0,\n reasoningTokens: 0,\n costUsd: 0,\n },\n )\n}\n\n/** Pick the outcome's failure class when present, else derive 'success' from run status. */\nexport function runFailureClass(run: Run): FailureClass {\n if (run.outcome?.failureClass) return run.outcome.failureClass\n if (run.status === 'completed' && run.outcome?.pass !== false) return 'success'\n if (run.status === 'aborted') return 'budget_exceeded'\n return 'unknown'\n}\n"],"mappings":";;;;;;;AAaA,eAAsB,gBAAgB,OAAmB,YAAoC;AAC3F,SAAO,MAAM,SAAS,EAAE,WAAW,CAAC;AACtC;AAEA,eAAsB,SAAS,OAAmB,OAAoC;AACpF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,MAAM,CAAC;AACtD,SAAO,MAAM,OAAO,SAAS;AAC/B;AAEA,eAAsB,UACpB,OACA,OACA,UACqB;AACrB,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,SAAS,CAAC;AACjE,SAAO,MAAM,OAAO,UAAU;AAChC;AAGA,eAAsB,WAAW,OAAmB,OAAsC;AACxF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,CAAC;AACxD,SAAO,MAAM,OAAO,WAAW;AACjC;AAGO,SAAS,QAAsC,OAAY,KAA+B;AAC/F,QAAM,MAAM,oBAAI,IAAY;AAC5B,aAAW,QAAQ,OAAO;AACxB,UAAM,IAAI,IAAI,IAAI;AAClB,QAAI,SAAS,IAAI,IAAI,CAAC;AACtB,QAAI,CAAC,QAAQ;AACX,eAAS,CAAC;AACV,UAAI,IAAI,GAAG,MAAM;AAAA,IACnB;AACA,WAAO,KAAK,IAAI;AAAA,EAClB;AACA,SAAO;AACT;AAGO,SAAS,QAAQ,MAAuB;AAC7C,SAAO,gBAAgB,IAAI;AAC7B;AAGO,SAAS,oBAAoB,MAAyB;AAC3D,SAAO,KAAK,iBAAiB;AAC/B;AAEA,SAAS,gBAAgB,OAAwB;AAK/C,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK,KAAK;AACjF,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AAGzE,QAAM,MAAM;AACZ,QAAM,OAAO,OAAO,KAAK,GAAG,EACzB,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,MAAS,EAClC,KAAK;AACR,QAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,CAAC,CAAC,EAAE;AAC/E,SAAO,IAAI,MAAM,KAAK,GAAG,CAAC;AAC5B;AAGO,SAAS,aAAa,OAO3B;AACA,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,OAAO;AAAA,MACX,aAAa,IAAI,eAAe,EAAE,eAAe;AAAA,MACjD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA,MACpD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA,MACpD,kBAAkB,IAAI,oBAAoB,EAAE,oBAAoB;AAAA,MAChE,iBAAiB,IAAI,mBAAmB,EAAE,mBAAmB;AAAA,MAC7D,SAAS,IAAI,WAAW,EAAE,WAAW;AAAA,IACvC;AAAA,IACA;AAAA,MACE,aAAa;AAAA,MACb,cAAc;AAAA,MACd,cAAc;AAAA,MACd,kBAAkB;AAAA,MAClB,iBAAiB;AAAA,MACjB,SAAS;AAAA,IACX;AAAA,EACF;AACF;AAGO,SAAS,gBAAgB,KAAwB;AACtD,MAAI,IAAI,SAAS,aAAc,QAAO,IAAI,QAAQ;AAClD,MAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,MAAO,QAAO;AACtE,MAAI,IAAI,WAAW,UAAW,QAAO;AACrC,SAAO;AACT;","names":[]}
|
package/dist/chunk-ZZUXHH3R.js
DELETED
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
// src/hosted/types.ts
|
|
2
|
-
var HOSTED_WIRE_VERSION = "2026-05-26.v1";
|
|
3
|
-
|
|
4
|
-
// src/hosted/client.ts
|
|
5
|
-
function sleep(ms) {
|
|
6
|
-
return new Promise((resolve) => {
|
|
7
|
-
const t = setTimeout(resolve, ms);
|
|
8
|
-
if (typeof t.unref === "function")
|
|
9
|
-
t.unref();
|
|
10
|
-
});
|
|
11
|
-
}
|
|
12
|
-
async function post(tenant, path, body, opts = {}) {
|
|
13
|
-
const timeoutMs = tenant.timeoutMs ?? 3e4;
|
|
14
|
-
const maxRetries = tenant.retries ?? 2;
|
|
15
|
-
const f = tenant.fetchImpl ?? ((...args) => fetch(...args));
|
|
16
|
-
const base = tenant.endpoint.replace(/\/+$/, "").replace(/\/v1$/, "");
|
|
17
|
-
const url = `${base}${path}`;
|
|
18
|
-
let lastError;
|
|
19
|
-
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
20
|
-
const ourTimeout = AbortSignal.timeout(timeoutMs);
|
|
21
|
-
const combinedSignal = opts.signal ? AbortSignal.any([opts.signal, ourTimeout]) : ourTimeout;
|
|
22
|
-
try {
|
|
23
|
-
const headers = {
|
|
24
|
-
"content-type": "application/json",
|
|
25
|
-
authorization: `Bearer ${tenant.apiKey}`,
|
|
26
|
-
"x-tangle-tenant-id": tenant.tenantId,
|
|
27
|
-
"x-tangle-wire-version": HOSTED_WIRE_VERSION
|
|
28
|
-
};
|
|
29
|
-
if (opts.idempotencyKey) headers["idempotency-key"] = opts.idempotencyKey;
|
|
30
|
-
const res = await f(url, {
|
|
31
|
-
method: "POST",
|
|
32
|
-
headers,
|
|
33
|
-
body: JSON.stringify(body),
|
|
34
|
-
signal: combinedSignal
|
|
35
|
-
});
|
|
36
|
-
if (!res.ok) {
|
|
37
|
-
const retryable = res.status >= 500 || res.status === 408 || res.status === 429;
|
|
38
|
-
if (!retryable || attempt === maxRetries) {
|
|
39
|
-
const text = await res.text().catch(() => "");
|
|
40
|
-
throw new Error(`hosted ingest ${url} failed (${res.status}): ${text.slice(0, 500)}`);
|
|
41
|
-
}
|
|
42
|
-
await sleep(2 ** attempt * 200 + Math.random() * 200);
|
|
43
|
-
continue;
|
|
44
|
-
}
|
|
45
|
-
return await res.json();
|
|
46
|
-
} catch (err) {
|
|
47
|
-
if (opts.signal?.aborted) throw err;
|
|
48
|
-
lastError = err;
|
|
49
|
-
if (attempt === maxRetries) throw err;
|
|
50
|
-
await sleep(2 ** attempt * 200 + Math.random() * 200);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
throw lastError ?? new Error("hosted ingest exhausted retries");
|
|
54
|
-
}
|
|
55
|
-
function createHostedClient(tenant) {
|
|
56
|
-
return {
|
|
57
|
-
tenant,
|
|
58
|
-
wireVersion: HOSTED_WIRE_VERSION,
|
|
59
|
-
async ingestEvalRun(event, idempotencyKey) {
|
|
60
|
-
return this.ingestEvalRuns([event], idempotencyKey);
|
|
61
|
-
},
|
|
62
|
-
async ingestEvalRuns(events, idempotencyKey) {
|
|
63
|
-
const body = { wireVersion: HOSTED_WIRE_VERSION, events };
|
|
64
|
-
return post(tenant, "/v1/ingest/eval-runs", body, {
|
|
65
|
-
idempotencyKey
|
|
66
|
-
});
|
|
67
|
-
},
|
|
68
|
-
async ingestTraces(spans, idempotencyKey) {
|
|
69
|
-
const body = { wireVersion: HOSTED_WIRE_VERSION, spans };
|
|
70
|
-
return post(tenant, "/v1/ingest/traces", body, {
|
|
71
|
-
idempotencyKey
|
|
72
|
-
});
|
|
73
|
-
}
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
function hostedTenantFromEnv(overrides = {}) {
|
|
77
|
-
const env = overrides.env ?? process.env;
|
|
78
|
-
const endpoint = (overrides.endpoint ?? env.TANGLE_INGEST_URL ?? env.TANGLE_ORCHESTRATOR_URL)?.trim();
|
|
79
|
-
const apiKey = (overrides.apiKey ?? env.TANGLE_INGEST_API_KEY ?? env.TANGLE_API_KEY)?.trim();
|
|
80
|
-
const tenantId = (overrides.tenantId ?? env.TANGLE_TENANT_ID)?.trim();
|
|
81
|
-
if (!endpoint || !apiKey || !tenantId) return void 0;
|
|
82
|
-
const tenant = { endpoint: endpoint.replace(/\/+$/, ""), apiKey, tenantId };
|
|
83
|
-
if (overrides.fetchImpl) tenant.fetchImpl = overrides.fetchImpl;
|
|
84
|
-
if (overrides.timeoutMs !== void 0) tenant.timeoutMs = overrides.timeoutMs;
|
|
85
|
-
if (overrides.retries !== void 0) tenant.retries = overrides.retries;
|
|
86
|
-
return tenant;
|
|
87
|
-
}
|
|
88
|
-
function hostedClientFromEnv(overrides = {}) {
|
|
89
|
-
const tenant = hostedTenantFromEnv(overrides);
|
|
90
|
-
return tenant ? createHostedClient(tenant) : void 0;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
export {
|
|
94
|
-
HOSTED_WIRE_VERSION,
|
|
95
|
-
createHostedClient,
|
|
96
|
-
hostedTenantFromEnv,
|
|
97
|
-
hostedClientFromEnv
|
|
98
|
-
};
|
|
99
|
-
//# sourceMappingURL=chunk-ZZUXHH3R.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/hosted/types.ts","../src/hosted/client.ts"],"sourcesContent":["/**\n * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,\n * a partner's self-hosted one, a future open implementation) must accept.\n *\n * **Stability:** every type in this file is committed under semver. New\n * minors only ADD optional fields. Breaking changes mean a major bump\n * (`HostedWireVersion` literal increment).\n *\n * The wire format is two event streams in one transport:\n *\n * 1. **Eval-run events** (`POST /v1/ingest/eval-runs`). Posted when a\n * campaign / improvement-loop completes (or per-generation if\n * streaming). Carries the structured result + per-cell scores +\n * surface diffs the orchestrator stores for the dashboard.\n *\n * 2. **Trace spans** (`POST /v1/ingest/traces`). Standard OTLP-shaped\n * spans with a few additional attributes so the orchestrator can\n * pivot from eval-run → underlying execution. Compatible with any\n * OTel collector.\n *\n * Both endpoints are authenticated with a bearer token + a tenant id\n * header. Tenants isolate everything downstream of ingest; no tenant\n * ever sees another tenant's data.\n */\n\nimport type { GateDecision, MutableSurface } from '../campaign/types'\nimport type { InsightReport } from '../contract/insight-report'\n\n// re-export so wire-format consumers can import the optional payload type\n// from `@tangle-network/agent-eval/hosted` without reaching into /contract.\nexport type { InsightReport } from '../contract/insight-report'\n\nexport const HOSTED_WIRE_VERSION = '2026-05-26.v1' as const\nexport type HostedWireVersion = typeof HOSTED_WIRE_VERSION\n\n// ── Transport headers ───────────────────────────────────────────────\n\n/** Every ingest request carries these. */\nexport interface HostedIngestHeaders {\n /** Bearer token. The orchestrator validates against the tenant key. */\n authorization: `Bearer ${string}`\n /** Stable tenant id (the orchestrator-side primary key for the tenant). */\n 'x-tangle-tenant-id': string\n /** Wire-version pin so the server can reject incompatible payloads. */\n 'x-tangle-wire-version': HostedWireVersion\n /** Optional idempotency key for retry-safe ingest. */\n 'idempotency-key'?: string\n}\n\n// ── Eval-run event ──────────────────────────────────────────────────\n\n/** Lifecycle stages of an eval-run as the substrate reports them. */\nexport type EvalRunStatus =\n | 'started'\n | 'baseline-complete'\n | 'generation-complete'\n | 'gate-decided'\n | 'finished'\n | 'errored'\n\nexport interface EvalRunCellScore {\n /** Stable scenario id from the consumer's scenario set. */\n scenarioId: string\n /** Repetition index when reps > 1; 0 for the default. */\n rep: number\n /** Composite score across all judges + dimensions for this cell. */\n compositeMean: number\n /** Per-judge → per-dimension scores; null where the judge did not run. */\n dimensions: Record<string, Record<string, number>>\n /** Per-cell error message if the dispatch threw. Null on success. */\n errorMessage?: string\n}\n\nexport interface EvalRunGenerationSnapshot {\n /** Generation index. 0 is baseline. */\n index: number\n /** Candidate surface fingerprint (stable hash) — pivot key into the\n * trace stream to fetch the underlying execution. */\n surfaceHash: string\n /** The candidate surface itself. May be omitted to avoid PII when the\n * consumer prefers not to ship verbatim prompts. */\n surface?: MutableSurface\n /** Per-cell scores for this generation. */\n cells: EvalRunCellScore[]\n /** Aggregate composite mean across all cells in this generation. */\n compositeMean: number\n /** Total $ spent across this generation. */\n costUsd: number\n /** Wall-clock duration of this generation. */\n durationMs: number\n}\n\n/**\n * The top-level eval-run event. One ingest call per logical eval-run;\n * generations stream in incrementally via repeated calls with the same\n * `runId`. The orchestrator deduplicates by `(runId, generation.index)`.\n */\nexport interface EvalRunEvent {\n /** Stable run id (the substrate's `runId`). UUID or substrate-generated. */\n runId: string\n /** Where this run was happening — derived from `RunCampaignOptions.runDir`. */\n runDir: string\n /** ISO-8601 timestamp the substrate recorded the event. */\n timestamp: string\n /** Lifecycle stage this event represents. */\n status: EvalRunStatus\n /** Free-form consumer tags (env, branch, model id, etc.). Searchable. */\n labels: Record<string, string>\n /** Baseline campaign snapshot. Present when status >= baseline-complete. */\n baseline?: EvalRunGenerationSnapshot\n /** Per-generation snapshots. Streams in; orchestrator appends. */\n generations: EvalRunGenerationSnapshot[]\n /** Final gate decision. Present when status >= gate-decided. */\n gateDecision?: GateDecision\n /** Held-out lift = winner-on-holdout - baseline-on-holdout. */\n holdoutLift?: number\n /** Total $ spent across baseline + every generation. */\n totalCostUsd: number\n /** Total wall-clock duration. */\n totalDurationMs: number\n /** Error message if status === 'errored'. */\n errorMessage?: string\n /** Rigor packet emitted alongside the run — distributional summary,\n * paired-bootstrap lift CI, judge stats, inter-rater agreement,\n * contamination check, failure clusters (when an analyst is wired),\n * outcome correlation (when downstream signal is supplied), and the\n * recommendations the dashboard surfaces verbatim. Additive; older\n * clients that don't know about this field continue to work. */\n insightReport?: InsightReport\n}\n\n// ── Trace span event ────────────────────────────────────────────────\n\n/**\n * OTel-shape span with a few additional attributes for eval-run pivoting.\n * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,\n * `startTimeUnixNano`, `endTimeUnixNano`, `attributes` are stock OTel.\n */\nexport interface TraceSpanEvent {\n traceId: string\n spanId: string\n parentSpanId?: string\n name: string\n startTimeUnixNano: number\n endTimeUnixNano: number\n attributes: Record<string, string | number | boolean>\n events?: Array<{\n timeUnixNano: number\n name: string\n attributes?: Record<string, string | number | boolean>\n }>\n status?: { code: 'OK' | 'ERROR' | 'UNSET'; message?: string }\n /** Pivot back into the eval-run stream. */\n 'tangle.runId'?: string\n /** Pivot to the specific generation. */\n 'tangle.generation'?: number\n /** Pivot to the specific cell. */\n 'tangle.cellId'?: string\n /** Pivot to the specific scenario. */\n 'tangle.scenarioId'?: string\n}\n\n// ── Ingest request bodies ───────────────────────────────────────────\n\nexport interface IngestEvalRunsRequest {\n wireVersion: HostedWireVersion\n events: EvalRunEvent[]\n}\n\nexport interface IngestTracesRequest {\n wireVersion: HostedWireVersion\n spans: TraceSpanEvent[]\n}\n\nexport interface IngestResponse {\n /** Accepted events / spans count. */\n accepted: number\n /** Rejected events with reasons (validation failures, dup idempotency key, etc.). */\n rejected: Array<{ index: number; reason: string }>\n}\n","/**\n * # Hosted-tier ingest client.\n *\n * Ships eval-run events + trace spans to any orchestrator (ours, a\n * partner's self-hosted one, or a future open implementation) that\n * speaks the wire format in `./types.ts`.\n *\n * Three modes:\n * - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —\n * the client appends the versioned `/v1/ingest/...` path itself; a trailing\n * `/v1` on the endpoint is tolerated and normalized away). We handle ingest\n * + storage + dashboard.\n * - **Self-hosted:** point at whatever URL runs the reference receiver\n * from `examples/hosted-ingest-server/`.\n * - **Off (default):** when `hostedTenant` is unset, nothing is sent.\n * Everything stays local.\n */\n\nimport {\n type EvalRunEvent,\n HOSTED_WIRE_VERSION,\n type HostedWireVersion,\n type IngestEvalRunsRequest,\n type IngestResponse,\n type IngestTracesRequest,\n type TraceSpanEvent,\n} from './types'\n\nexport interface HostedTenant {\n /** Orchestrator endpoint base URL (no trailing slash). Required. */\n endpoint: string\n /** Bearer token issued by the orchestrator. Required. */\n apiKey: string\n /** Tenant id — the orchestrator's primary key for this consumer. Required. */\n tenantId: string\n /** Optional `fetch` override (auth wrappers, custom agent, test mocks). */\n fetchImpl?: typeof fetch\n /** Per-call timeout in ms. Default 30s. */\n timeoutMs?: number\n /** Retries on 5xx / network errors. Default 2. */\n retries?: number\n}\n\nexport interface HostedClient {\n ingestEvalRun(event: EvalRunEvent, idempotencyKey?: string): Promise<IngestResponse>\n ingestEvalRuns(events: EvalRunEvent[], idempotencyKey?: string): Promise<IngestResponse>\n ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<IngestResponse>\n readonly tenant: HostedTenant\n readonly wireVersion: HostedWireVersion\n}\n\ninterface RequestOptions {\n idempotencyKey?: string\n signal?: AbortSignal\n}\n\nfunction sleep(ms: number): Promise<void> {\n return new Promise((resolve) => {\n const t = setTimeout(resolve, ms)\n if (typeof (t as { unref?: () => void }).unref === 'function')\n (t as { unref: () => void }).unref()\n })\n}\n\nasync function post<TReq, TRes>(\n tenant: HostedTenant,\n path: string,\n body: TReq,\n opts: RequestOptions = {},\n): Promise<TRes> {\n const timeoutMs = tenant.timeoutMs ?? 30_000\n const maxRetries = tenant.retries ?? 2\n const f: typeof fetch = tenant.fetchImpl ?? ((...args) => fetch(...args))\n // `path` already carries the `/v1` version prefix (e.g. `/v1/ingest/eval-runs`).\n // Strip a trailing slash AND a trailing `/v1` from the endpoint so a base of\n // either `https://host` or `https://host/v1` resolves to the same correct URL\n // — callers routinely pass the versioned base and would otherwise hit\n // `/v1/v1/ingest/...` (404).\n const base = tenant.endpoint.replace(/\\/+$/, '').replace(/\\/v1$/, '')\n const url = `${base}${path}`\n\n let lastError: unknown\n for (let attempt = 0; attempt <= maxRetries; attempt++) {\n const ourTimeout = AbortSignal.timeout(timeoutMs)\n const combinedSignal = opts.signal ? AbortSignal.any([opts.signal, ourTimeout]) : ourTimeout\n try {\n const headers: Record<string, string> = {\n 'content-type': 'application/json',\n authorization: `Bearer ${tenant.apiKey}`,\n 'x-tangle-tenant-id': tenant.tenantId,\n 'x-tangle-wire-version': HOSTED_WIRE_VERSION,\n }\n if (opts.idempotencyKey) headers['idempotency-key'] = opts.idempotencyKey\n\n const res = await f(url, {\n method: 'POST',\n headers,\n body: JSON.stringify(body),\n signal: combinedSignal,\n })\n if (!res.ok) {\n const retryable = res.status >= 500 || res.status === 408 || res.status === 429\n if (!retryable || attempt === maxRetries) {\n const text = await res.text().catch(() => '')\n throw new Error(`hosted ingest ${url} failed (${res.status}): ${text.slice(0, 500)}`)\n }\n await sleep(2 ** attempt * 200 + Math.random() * 200)\n continue\n }\n return (await res.json()) as TRes\n } catch (err) {\n if (opts.signal?.aborted) throw err\n lastError = err\n if (attempt === maxRetries) throw err\n await sleep(2 ** attempt * 200 + Math.random() * 200)\n }\n }\n throw lastError ?? new Error('hosted ingest exhausted retries')\n}\n\nexport function createHostedClient(tenant: HostedTenant): HostedClient {\n return {\n tenant,\n wireVersion: HOSTED_WIRE_VERSION,\n\n async ingestEvalRun(event, idempotencyKey) {\n return this.ingestEvalRuns([event], idempotencyKey)\n },\n\n async ingestEvalRuns(events, idempotencyKey) {\n const body: IngestEvalRunsRequest = { wireVersion: HOSTED_WIRE_VERSION, events }\n return post<IngestEvalRunsRequest, IngestResponse>(tenant, '/v1/ingest/eval-runs', body, {\n idempotencyKey,\n })\n },\n\n async ingestTraces(spans, idempotencyKey) {\n const body: IngestTracesRequest = { wireVersion: HOSTED_WIRE_VERSION, spans }\n return post<IngestTracesRequest, IngestResponse>(tenant, '/v1/ingest/traces', body, {\n idempotencyKey,\n })\n },\n }\n}\n\n/**\n * Build a `HostedClient` from environment, or `undefined` when ingest is not\n * configured — the canonical, fail-soft wiring every product uses so eval-run +\n * trace provenance lands in the Intelligence dashboard with ONE call:\n *\n * const hosted = hostedClientFromEnv()\n * // ...run the loop...\n * await emitLoopProvenance({ ..., hostedClient: hosted }) // no-op if undefined\n *\n * Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is\n * missing — so a product wires the ship call unconditionally and it stays a\n * no-op until the env is set. Env precedence:\n * - endpoint: `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`\n * - apiKey: `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`\n * - tenantId: `TANGLE_TENANT_ID`\n * A trailing slash on the endpoint is stripped. Pass `overrides` to supply any\n * field directly (e.g. a fixed `tenantId` per product) — overrides win over env.\n */\n/**\n * Build a {@link HostedTenant} config from env — the input `selfImprove`'s\n * `hostedTenant` and `emitLoopProvenance` take. Same env precedence + overrides\n * as {@link hostedClientFromEnv}; returns `undefined` (not an error) when any of\n * endpoint / apiKey / tenantId is missing, so a product wires\n * `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally\n * and it stays off until the env is set.\n */\nexport function hostedTenantFromEnv(\n overrides: Partial<HostedTenant> & { env?: Record<string, string | undefined> } = {},\n): HostedTenant | undefined {\n const env = overrides.env ?? process.env\n const endpoint = (\n overrides.endpoint ??\n env.TANGLE_INGEST_URL ??\n env.TANGLE_ORCHESTRATOR_URL\n )?.trim()\n const apiKey = (overrides.apiKey ?? env.TANGLE_INGEST_API_KEY ?? env.TANGLE_API_KEY)?.trim()\n const tenantId = (overrides.tenantId ?? env.TANGLE_TENANT_ID)?.trim()\n if (!endpoint || !apiKey || !tenantId) return undefined\n const tenant: HostedTenant = { endpoint: endpoint.replace(/\\/+$/, ''), apiKey, tenantId }\n if (overrides.fetchImpl) tenant.fetchImpl = overrides.fetchImpl\n if (overrides.timeoutMs !== undefined) tenant.timeoutMs = overrides.timeoutMs\n if (overrides.retries !== undefined) tenant.retries = overrides.retries\n return tenant\n}\n\nexport function hostedClientFromEnv(\n overrides: Partial<HostedTenant> & { env?: Record<string, string | undefined> } = {},\n): HostedClient | undefined {\n const tenant = hostedTenantFromEnv(overrides)\n return tenant ? createHostedClient(tenant) : undefined\n}\n"],"mappings":";AAgCO,IAAM,sBAAsB;;;ACwBnC,SAAS,MAAM,IAA2B;AACxC,SAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,UAAM,IAAI,WAAW,SAAS,EAAE;AAChC,QAAI,OAAQ,EAA6B,UAAU;AACjD,MAAC,EAA4B,MAAM;AAAA,EACvC,CAAC;AACH;AAEA,eAAe,KACb,QACA,MACA,MACA,OAAuB,CAAC,GACT;AACf,QAAM,YAAY,OAAO,aAAa;AACtC,QAAM,aAAa,OAAO,WAAW;AACrC,QAAM,IAAkB,OAAO,cAAc,IAAI,SAAS,MAAM,GAAG,IAAI;AAMvE,QAAM,OAAO,OAAO,SAAS,QAAQ,QAAQ,EAAE,EAAE,QAAQ,SAAS,EAAE;AACpE,QAAM,MAAM,GAAG,IAAI,GAAG,IAAI;AAE1B,MAAI;AACJ,WAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,UAAM,aAAa,YAAY,QAAQ,SAAS;AAChD,UAAM,iBAAiB,KAAK,SAAS,YAAY,IAAI,CAAC,KAAK,QAAQ,UAAU,CAAC,IAAI;AAClF,QAAI;AACF,YAAM,UAAkC;AAAA,QACtC,gBAAgB;AAAA,QAChB,eAAe,UAAU,OAAO,MAAM;AAAA,QACtC,sBAAsB,OAAO;AAAA,QAC7B,yBAAyB;AAAA,MAC3B;AACA,UAAI,KAAK,eAAgB,SAAQ,iBAAiB,IAAI,KAAK;AAE3D,YAAM,MAAM,MAAM,EAAE,KAAK;AAAA,QACvB,QAAQ;AAAA,QACR;AAAA,QACA,MAAM,KAAK,UAAU,IAAI;AAAA,QACzB,QAAQ;AAAA,MACV,CAAC;AACD,UAAI,CAAC,IAAI,IAAI;AACX,cAAM,YAAY,IAAI,UAAU,OAAO,IAAI,WAAW,OAAO,IAAI,WAAW;AAC5E,YAAI,CAAC,aAAa,YAAY,YAAY;AACxC,gBAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,gBAAM,IAAI,MAAM,iBAAiB,GAAG,YAAY,IAAI,MAAM,MAAM,KAAK,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,QACtF;AACA,cAAM,MAAM,KAAK,UAAU,MAAM,KAAK,OAAO,IAAI,GAAG;AACpD;AAAA,MACF;AACA,aAAQ,MAAM,IAAI,KAAK;AAAA,IACzB,SAAS,KAAK;AACZ,UAAI,KAAK,QAAQ,QAAS,OAAM;AAChC,kBAAY;AACZ,UAAI,YAAY,WAAY,OAAM;AAClC,YAAM,MAAM,KAAK,UAAU,MAAM,KAAK,OAAO,IAAI,GAAG;AAAA,IACtD;AAAA,EACF;AACA,QAAM,aAAa,IAAI,MAAM,iCAAiC;AAChE;AAEO,SAAS,mBAAmB,QAAoC;AACrE,SAAO;AAAA,IACL;AAAA,IACA,aAAa;AAAA,IAEb,MAAM,cAAc,OAAO,gBAAgB;AACzC,aAAO,KAAK,eAAe,CAAC,KAAK,GAAG,cAAc;AAAA,IACpD;AAAA,IAEA,MAAM,eAAe,QAAQ,gBAAgB;AAC3C,YAAM,OAA8B,EAAE,aAAa,qBAAqB,OAAO;AAC/E,aAAO,KAA4C,QAAQ,wBAAwB,MAAM;AAAA,QACvF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,aAAa,OAAO,gBAAgB;AACxC,YAAM,OAA4B,EAAE,aAAa,qBAAqB,MAAM;AAC5E,aAAO,KAA0C,QAAQ,qBAAqB,MAAM;AAAA,QAClF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;AA4BO,SAAS,oBACd,YAAkF,CAAC,GACzD;AAC1B,QAAM,MAAM,UAAU,OAAO,QAAQ;AACrC,QAAM,YACJ,UAAU,YACV,IAAI,qBACJ,IAAI,0BACH,KAAK;AACR,QAAM,UAAU,UAAU,UAAU,IAAI,yBAAyB,IAAI,iBAAiB,KAAK;AAC3F,QAAM,YAAY,UAAU,YAAY,IAAI,mBAAmB,KAAK;AACpE,MAAI,CAAC,YAAY,CAAC,UAAU,CAAC,SAAU,QAAO;AAC9C,QAAM,SAAuB,EAAE,UAAU,SAAS,QAAQ,QAAQ,EAAE,GAAG,QAAQ,SAAS;AACxF,MAAI,UAAU,UAAW,QAAO,YAAY,UAAU;AACtD,MAAI,UAAU,cAAc,OAAW,QAAO,YAAY,UAAU;AACpE,MAAI,UAAU,YAAY,OAAW,QAAO,UAAU,UAAU;AAChE,SAAO;AACT;AAEO,SAAS,oBACd,YAAkF,CAAC,GACzD;AAC1B,QAAM,SAAS,oBAAoB,SAAS;AAC5C,SAAO,SAAS,mBAAmB,MAAM,IAAI;AAC/C;","names":[]}
|
package/dist/cli.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
package/dist/cli.js
DELETED
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
buildOpenApi,
|
|
4
|
-
handleVersion,
|
|
5
|
-
runRpcBatch,
|
|
6
|
-
runRpcOnce,
|
|
7
|
-
startServer
|
|
8
|
-
} from "./chunk-4I2E3LLO.js";
|
|
9
|
-
import "./chunk-NJC7U437.js";
|
|
10
|
-
import "./chunk-JHOJHHU7.js";
|
|
11
|
-
import "./chunk-VI2UW6B6.js";
|
|
12
|
-
import "./chunk-PC4UYEBM.js";
|
|
13
|
-
import "./chunk-ONWEPEDO.js";
|
|
14
|
-
import "./chunk-PZ5AY32C.js";
|
|
15
|
-
|
|
16
|
-
// src/cli.ts
|
|
17
|
-
import { writeFileSync } from "fs";
|
|
18
|
-
function parseArgs(argv) {
|
|
19
|
-
const [command, ...rest] = argv;
|
|
20
|
-
const positional = [];
|
|
21
|
-
const flags = {};
|
|
22
|
-
for (let i = 0; i < rest.length; i++) {
|
|
23
|
-
const tok = rest[i];
|
|
24
|
-
if (tok.startsWith("--")) {
|
|
25
|
-
const key = tok.slice(2);
|
|
26
|
-
const next = rest[i + 1];
|
|
27
|
-
if (next != null && !next.startsWith("--")) {
|
|
28
|
-
flags[key] = next;
|
|
29
|
-
i++;
|
|
30
|
-
} else {
|
|
31
|
-
flags[key] = "true";
|
|
32
|
-
}
|
|
33
|
-
} else {
|
|
34
|
-
positional.push(tok);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
return { command: command ?? "help", positional, flags };
|
|
38
|
-
}
|
|
39
|
-
var HELP = `agent-eval \u2014 wire-protocol entry point.
|
|
40
|
-
|
|
41
|
-
Commands:
|
|
42
|
-
serve [--port 5005] [--host 127.0.0.1]
|
|
43
|
-
Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.
|
|
44
|
-
rpc <method>
|
|
45
|
-
Read one JSON object from stdin (the params for <method>), write one
|
|
46
|
-
JSON object to stdout. Method \u2208 {judge, listRubrics, version}.
|
|
47
|
-
rpc-batch <method>
|
|
48
|
-
Like 'rpc' but JSONL in / JSONL out.
|
|
49
|
-
openapi [--out openapi.json]
|
|
50
|
-
Write the OpenAPI 3.1 spec.
|
|
51
|
-
version
|
|
52
|
-
Print server + wire-protocol version JSON.
|
|
53
|
-
|
|
54
|
-
Without arguments, prints this help.`;
|
|
55
|
-
async function main() {
|
|
56
|
-
const { command, positional, flags } = parseArgs(process.argv.slice(2));
|
|
57
|
-
switch (command) {
|
|
58
|
-
case "serve": {
|
|
59
|
-
const port = Number(flags.port ?? 5005);
|
|
60
|
-
const host = flags.host ?? "127.0.0.1";
|
|
61
|
-
const server = startServer({ port, host });
|
|
62
|
-
const shutdown = (sig) => {
|
|
63
|
-
console.log(`[agent-eval] received ${sig}, shutting down`);
|
|
64
|
-
server.close(() => process.exit(0));
|
|
65
|
-
setTimeout(() => process.exit(1), 5e3).unref();
|
|
66
|
-
};
|
|
67
|
-
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
68
|
-
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
69
|
-
await new Promise(() => {
|
|
70
|
-
});
|
|
71
|
-
return 0;
|
|
72
|
-
}
|
|
73
|
-
case "rpc": {
|
|
74
|
-
const [method] = positional;
|
|
75
|
-
return await runRpcOnce(method);
|
|
76
|
-
}
|
|
77
|
-
case "rpc-batch": {
|
|
78
|
-
const [method] = positional;
|
|
79
|
-
return await runRpcBatch(method);
|
|
80
|
-
}
|
|
81
|
-
case "openapi": {
|
|
82
|
-
const out = flags.out ?? "openapi.json";
|
|
83
|
-
const spec = buildOpenApi(handleVersion().version);
|
|
84
|
-
writeFileSync(out, `${JSON.stringify(spec, null, 2)}
|
|
85
|
-
`, "utf-8");
|
|
86
|
-
console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`);
|
|
87
|
-
return 0;
|
|
88
|
-
}
|
|
89
|
-
case "version": {
|
|
90
|
-
process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}
|
|
91
|
-
`);
|
|
92
|
-
return 0;
|
|
93
|
-
}
|
|
94
|
-
case "help":
|
|
95
|
-
case "--help":
|
|
96
|
-
case "-h":
|
|
97
|
-
case "":
|
|
98
|
-
process.stdout.write(`${HELP}
|
|
99
|
-
`);
|
|
100
|
-
return 0;
|
|
101
|
-
default:
|
|
102
|
-
process.stderr.write(`unknown command: ${command}
|
|
103
|
-
${HELP}
|
|
104
|
-
`);
|
|
105
|
-
return 1;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
main().then((code) => process.exit(code)).catch((err) => {
|
|
109
|
-
console.error("[agent-eval] cli error:", err);
|
|
110
|
-
process.exit(1);
|
|
111
|
-
});
|
|
112
|
-
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * agent-eval CLI.\n *\n * agent-eval serve [--port 5005] [--host 127.0.0.1]\n * agent-eval rpc <method> # one request from stdin → one response on stdout\n * agent-eval rpc-batch <method> # JSONL stdin → JSONL stdout\n * agent-eval openapi [--out path] # write OpenAPI spec\n * agent-eval version\n *\n * <method> is one of: judge, listRubrics, version. When omitted, the\n * stdin payload must be a full {method, params} envelope.\n */\nimport { writeFileSync } from 'node:fs'\nimport { handleVersion } from './wire/handlers'\nimport { buildOpenApi } from './wire/openapi'\nimport { runRpcBatch, runRpcOnce } from './wire/rpc'\nimport { startServer } from './wire/server'\n\ninterface Args {\n command: string\n positional: string[]\n flags: Record<string, string>\n}\n\nfunction parseArgs(argv: string[]): Args {\n const [command, ...rest] = argv\n const positional: string[] = []\n const flags: Record<string, string> = {}\n for (let i = 0; i < rest.length; i++) {\n const tok = rest[i]!\n if (tok.startsWith('--')) {\n const key = tok.slice(2)\n const next = rest[i + 1]\n if (next != null && !next.startsWith('--')) {\n flags[key] = next\n i++\n } else {\n flags[key] = 'true'\n }\n } else {\n positional.push(tok)\n }\n }\n return { command: command ?? 'help', positional, flags }\n}\n\nconst HELP = `agent-eval — wire-protocol entry point.\n\nCommands:\n serve [--port 5005] [--host 127.0.0.1]\n Start the HTTP server. POST /v1/judge, GET /v1/rubrics, GET /v1/version, GET /openapi.json.\n rpc <method>\n Read one JSON object from stdin (the params for <method>), write one\n JSON object to stdout. Method ∈ {judge, listRubrics, version}.\n rpc-batch <method>\n Like 'rpc' but JSONL in / JSONL out.\n openapi [--out openapi.json]\n Write the OpenAPI 3.1 spec.\n version\n Print server + wire-protocol version JSON.\n\nWithout arguments, prints this help.`\n\nasync function main(): Promise<number> {\n const { command, positional, flags } = parseArgs(process.argv.slice(2))\n\n switch (command) {\n case 'serve': {\n const port = Number(flags.port ?? 5005)\n const host = flags.host ?? '127.0.0.1'\n const server = startServer({ port, host })\n // Keep process alive on SIGINT/SIGTERM\n const shutdown = (sig: string) => {\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] received ${sig}, shutting down`)\n server.close(() => process.exit(0))\n // Force exit after 5s if close hangs\n setTimeout(() => process.exit(1), 5000).unref()\n }\n process.on('SIGINT', () => shutdown('SIGINT'))\n process.on('SIGTERM', () => shutdown('SIGTERM'))\n // Block forever\n await new Promise(() => {})\n return 0\n }\n case 'rpc': {\n const [method] = positional\n return await runRpcOnce(method)\n }\n case 'rpc-batch': {\n const [method] = positional\n return await runRpcBatch(method)\n }\n case 'openapi': {\n const out = flags.out ?? 'openapi.json'\n const spec = buildOpenApi(handleVersion().version)\n writeFileSync(out, `${JSON.stringify(spec, null, 2)}\\n`, 'utf-8')\n // eslint-disable-next-line no-console\n console.log(`[agent-eval] wrote OpenAPI 3.1 spec to ${out}`)\n return 0\n }\n case 'version': {\n process.stdout.write(`${JSON.stringify(handleVersion(), null, 2)}\\n`)\n return 0\n }\n case 'help':\n case '--help':\n case '-h':\n case '':\n process.stdout.write(`${HELP}\\n`)\n return 0\n default:\n process.stderr.write(`unknown command: ${command}\\n${HELP}\\n`)\n return 1\n }\n}\n\nmain()\n .then((code) => process.exit(code))\n .catch((err) => {\n // eslint-disable-next-line no-console\n console.error('[agent-eval] cli error:', err)\n process.exit(1)\n })\n"],"mappings":";;;;;;;;;;;;;;;;AAaA,SAAS,qBAAqB;AAY9B,SAAS,UAAU,MAAsB;AACvC,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAC3B,QAAM,aAAuB,CAAC;AAC9B,QAAM,QAAgC,CAAC;AACvC,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,IAAI,WAAW,IAAI,GAAG;AACxB,YAAM,MAAM,IAAI,MAAM,CAAC;AACvB,YAAM,OAAO,KAAK,IAAI,CAAC;AACvB,UAAI,QAAQ,QAAQ,CAAC,KAAK,WAAW,IAAI,GAAG;AAC1C,cAAM,GAAG,IAAI;AACb;AAAA,MACF,OAAO;AACL,cAAM,GAAG,IAAI;AAAA,MACf;AAAA,IACF,OAAO;AACL,iBAAW,KAAK,GAAG;AAAA,IACrB;AAAA,EACF;AACA,SAAO,EAAE,SAAS,WAAW,QAAQ,YAAY,MAAM;AACzD;AAEA,IAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBb,eAAe,OAAwB;AACrC,QAAM,EAAE,SAAS,YAAY,MAAM,IAAI,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AAEtE,UAAQ,SAAS;AAAA,IACf,KAAK,SAAS;AACZ,YAAM,OAAO,OAAO,MAAM,QAAQ,IAAI;AACtC,YAAM,OAAO,MAAM,QAAQ;AAC3B,YAAM,SAAS,YAAY,EAAE,MAAM,KAAK,CAAC;AAEzC,YAAM,WAAW,CAAC,QAAgB;AAEhC,gBAAQ,IAAI,yBAAyB,GAAG,iBAAiB;AACzD,eAAO,MAAM,MAAM,QAAQ,KAAK,CAAC,CAAC;AAElC,mBAAW,MAAM,QAAQ,KAAK,CAAC,GAAG,GAAI,EAAE,MAAM;AAAA,MAChD;AACA,cAAQ,GAAG,UAAU,MAAM,SAAS,QAAQ,CAAC;AAC7C,cAAQ,GAAG,WAAW,MAAM,SAAS,SAAS,CAAC;AAE/C,YAAM,IAAI,QAAQ,MAAM;AAAA,MAAC,CAAC;AAC1B,aAAO;AAAA,IACT;AAAA,IACA,KAAK,OAAO;AACV,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,WAAW,MAAM;AAAA,IAChC;AAAA,IACA,KAAK,aAAa;AAChB,YAAM,CAAC,MAAM,IAAI;AACjB,aAAO,MAAM,YAAY,MAAM;AAAA,IACjC;AAAA,IACA,KAAK,WAAW;AACd,YAAM,MAAM,MAAM,OAAO;AACzB,YAAM,OAAO,aAAa,cAAc,EAAE,OAAO;AACjD,oBAAc,KAAK,GAAG,KAAK,UAAU,MAAM,MAAM,CAAC,CAAC;AAAA,GAAM,OAAO;AAEhE,cAAQ,IAAI,0CAA0C,GAAG,EAAE;AAC3D,aAAO;AAAA,IACT;AAAA,IACA,KAAK,WAAW;AACd,cAAQ,OAAO,MAAM,GAAG,KAAK,UAAU,cAAc,GAAG,MAAM,CAAC,CAAC;AAAA,CAAI;AACpE,aAAO;AAAA,IACT;AAAA,IACA,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AACH,cAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAChC,aAAO;AAAA,IACT;AACE,cAAQ,OAAO,MAAM,oBAAoB,OAAO;AAAA,EAAK,IAAI;AAAA,CAAI;AAC7D,aAAO;AAAA,EACX;AACF;AAEA,KAAK,EACF,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AAEd,UAAQ,MAAM,2BAA2B,GAAG;AAC5C,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
|