@tangle-network/agent-eval 0.36.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-HIO4UIS5.js → chunk-L5UNCDAJ.js} +207 -1
- package/dist/chunk-L5UNCDAJ.js.map +1 -0
- package/dist/chunk-QWV226SL.js +276 -0
- package/dist/chunk-QWV226SL.js.map +1 -0
- package/dist/index.d.ts +111 -3
- package/dist/index.js +204 -1
- package/dist/index.js.map +1 -1
- package/dist/matrix/index.d.ts +2 -109
- package/dist/matrix/index.js +5 -270
- package/dist/matrix/index.js.map +1 -1
- package/dist/multishot/index.d.ts +276 -0
- package/dist/multishot/index.js +467 -0
- package/dist/multishot/index.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/traces.d.ts +81 -2
- package/dist/traces.js +7 -1
- package/dist/types-DHqkLwEU.d.ts +110 -0
- package/package.json +6 -1
- package/dist/chunk-HIO4UIS5.js.map +0 -1
|
@@ -1627,6 +1627,209 @@ function runToTraceId(run) {
|
|
|
1627
1627
|
return cleaned.slice(0, 32).padEnd(32, "0");
|
|
1628
1628
|
}
|
|
1629
1629
|
|
|
1630
|
+
// src/trace/otel-bridge.ts
|
|
1631
|
+
function otelRunCompleteHook(exporter) {
|
|
1632
|
+
return async (ctx) => {
|
|
1633
|
+
const spans = await ctx.store.spans({ runId: ctx.runId });
|
|
1634
|
+
for (const span of spans) {
|
|
1635
|
+
if (span.endedAt) {
|
|
1636
|
+
exporter.exportSpan(storeSpanToExportable(span, ctx.runId));
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
await exporter.flush();
|
|
1640
|
+
};
|
|
1641
|
+
}
|
|
1642
|
+
function createOtelTracingStore(inner, exporter, traceId) {
|
|
1643
|
+
return {
|
|
1644
|
+
async appendRun(run) {
|
|
1645
|
+
return inner.appendRun(run);
|
|
1646
|
+
},
|
|
1647
|
+
async updateRun(runId, patch) {
|
|
1648
|
+
return inner.updateRun(runId, patch);
|
|
1649
|
+
},
|
|
1650
|
+
async appendSpan(span) {
|
|
1651
|
+
if (span.endedAt) {
|
|
1652
|
+
exporter.exportSpan(storeSpanToExportable(span, traceId));
|
|
1653
|
+
}
|
|
1654
|
+
return inner.appendSpan(span);
|
|
1655
|
+
},
|
|
1656
|
+
async updateSpan(spanId, patch) {
|
|
1657
|
+
await inner.updateSpan(spanId, patch);
|
|
1658
|
+
if (patch.endedAt) {
|
|
1659
|
+
const spans = await inner.spans({ runId: traceId });
|
|
1660
|
+
const found = spans.find((s) => s.spanId === spanId);
|
|
1661
|
+
if (found) {
|
|
1662
|
+
exporter.exportSpan(storeSpanToExportable(found, traceId));
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
},
|
|
1666
|
+
async appendEvent(event) {
|
|
1667
|
+
return inner.appendEvent(event);
|
|
1668
|
+
},
|
|
1669
|
+
async appendBudgetEntry(entry) {
|
|
1670
|
+
return inner.appendBudgetEntry(entry);
|
|
1671
|
+
},
|
|
1672
|
+
async appendArtifact(artifact) {
|
|
1673
|
+
return inner.appendArtifact(artifact);
|
|
1674
|
+
},
|
|
1675
|
+
getRun: inner.getRun.bind(inner),
|
|
1676
|
+
listRuns: inner.listRuns.bind(inner),
|
|
1677
|
+
spans: inner.spans.bind(inner),
|
|
1678
|
+
events: inner.events.bind(inner),
|
|
1679
|
+
budget: inner.budget.bind(inner),
|
|
1680
|
+
artifacts: inner.artifacts.bind(inner)
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
function storeSpanToExportable(span, traceId) {
|
|
1684
|
+
const llm = span.kind === "llm" ? span : void 0;
|
|
1685
|
+
return {
|
|
1686
|
+
traceId,
|
|
1687
|
+
spanId: span.spanId,
|
|
1688
|
+
parentSpanId: span.parentSpanId,
|
|
1689
|
+
name: span.name,
|
|
1690
|
+
kind: span.kind,
|
|
1691
|
+
startedAt: span.startedAt,
|
|
1692
|
+
endedAt: span.endedAt,
|
|
1693
|
+
status: span.status,
|
|
1694
|
+
error: span.error,
|
|
1695
|
+
model: llm?.model,
|
|
1696
|
+
inputTokens: llm?.inputTokens,
|
|
1697
|
+
outputTokens: llm?.outputTokens,
|
|
1698
|
+
costUsd: llm?.costUsd,
|
|
1699
|
+
attributes: span.attributes
|
|
1700
|
+
};
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
// src/trace/otel-export.ts
|
|
1704
|
+
function createOtelExporter(config) {
|
|
1705
|
+
const resolvedEndpoint = config?.endpoint ?? (typeof process !== "undefined" ? process.env.OTEL_EXPORTER_OTLP_ENDPOINT : void 0);
|
|
1706
|
+
if (!resolvedEndpoint) return void 0;
|
|
1707
|
+
const endpoint = resolvedEndpoint;
|
|
1708
|
+
const headers = config?.headers ?? parseHeadersFromEnv();
|
|
1709
|
+
const batchSize = config?.batchSize ?? 64;
|
|
1710
|
+
const flushIntervalMs = config?.flushIntervalMs ?? 5e3;
|
|
1711
|
+
const serviceName = config?.serviceName ?? "agent-eval";
|
|
1712
|
+
const resourceAttrs = config?.resourceAttributes ?? {};
|
|
1713
|
+
const pending = [];
|
|
1714
|
+
let timer;
|
|
1715
|
+
let stopped = false;
|
|
1716
|
+
const exporter = {
|
|
1717
|
+
exportSpan(span) {
|
|
1718
|
+
if (stopped) return;
|
|
1719
|
+
pending.push(toOtlpSpan(span));
|
|
1720
|
+
if (pending.length >= batchSize) {
|
|
1721
|
+
void doFlush();
|
|
1722
|
+
}
|
|
1723
|
+
},
|
|
1724
|
+
async flush() {
|
|
1725
|
+
await doFlush();
|
|
1726
|
+
},
|
|
1727
|
+
async shutdown() {
|
|
1728
|
+
stopped = true;
|
|
1729
|
+
if (timer !== void 0) {
|
|
1730
|
+
clearInterval(timer);
|
|
1731
|
+
timer = void 0;
|
|
1732
|
+
}
|
|
1733
|
+
await doFlush();
|
|
1734
|
+
}
|
|
1735
|
+
};
|
|
1736
|
+
timer = setInterval(() => {
|
|
1737
|
+
if (pending.length > 0) void doFlush();
|
|
1738
|
+
}, flushIntervalMs);
|
|
1739
|
+
if (typeof timer === "object" && "unref" in timer) {
|
|
1740
|
+
;
|
|
1741
|
+
timer.unref();
|
|
1742
|
+
}
|
|
1743
|
+
async function doFlush() {
|
|
1744
|
+
if (pending.length === 0) return;
|
|
1745
|
+
const batch = pending.splice(0);
|
|
1746
|
+
const body = {
|
|
1747
|
+
resourceSpans: [
|
|
1748
|
+
{
|
|
1749
|
+
resource: {
|
|
1750
|
+
attributes: toAttributes2({
|
|
1751
|
+
"service.name": serviceName,
|
|
1752
|
+
...resourceAttrs
|
|
1753
|
+
})
|
|
1754
|
+
},
|
|
1755
|
+
scopeSpans: [{ scope: OTEL_AGENT_EVAL_SCOPE, spans: batch }]
|
|
1756
|
+
}
|
|
1757
|
+
]
|
|
1758
|
+
};
|
|
1759
|
+
const url = endpoint.replace(/\/+$/, "") + "/v1/traces";
|
|
1760
|
+
try {
|
|
1761
|
+
await fetch(url, {
|
|
1762
|
+
method: "POST",
|
|
1763
|
+
headers: {
|
|
1764
|
+
"content-type": "application/json",
|
|
1765
|
+
...headers
|
|
1766
|
+
},
|
|
1767
|
+
body: JSON.stringify(body)
|
|
1768
|
+
});
|
|
1769
|
+
} catch {
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
return exporter;
|
|
1773
|
+
}
|
|
1774
|
+
function parseHeadersFromEnv() {
|
|
1775
|
+
if (typeof process === "undefined") return {};
|
|
1776
|
+
const raw = process.env.OTEL_EXPORTER_OTLP_HEADERS;
|
|
1777
|
+
if (!raw) return {};
|
|
1778
|
+
const out = {};
|
|
1779
|
+
for (const pair of raw.split(",")) {
|
|
1780
|
+
const eq = pair.indexOf("=");
|
|
1781
|
+
if (eq < 0) continue;
|
|
1782
|
+
const key = pair.slice(0, eq).trim();
|
|
1783
|
+
const value = pair.slice(eq + 1).trim();
|
|
1784
|
+
if (key) out[key] = value;
|
|
1785
|
+
}
|
|
1786
|
+
return out;
|
|
1787
|
+
}
|
|
1788
|
+
function toOtlpSpan(span) {
|
|
1789
|
+
const endedAt = span.endedAt ?? span.startedAt;
|
|
1790
|
+
const attrs = {
|
|
1791
|
+
"span.kind": span.kind
|
|
1792
|
+
};
|
|
1793
|
+
if (span.model) attrs["llm.model"] = span.model;
|
|
1794
|
+
if (span.inputTokens !== void 0) attrs["llm.input_tokens"] = span.inputTokens;
|
|
1795
|
+
if (span.outputTokens !== void 0) attrs["llm.output_tokens"] = span.outputTokens;
|
|
1796
|
+
if (span.costUsd !== void 0) attrs["llm.cost_usd"] = span.costUsd;
|
|
1797
|
+
if (span.attributes) {
|
|
1798
|
+
for (const [k, v] of Object.entries(span.attributes)) {
|
|
1799
|
+
if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") attrs[k] = v;
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
return {
|
|
1803
|
+
traceId: padTraceId(span.traceId),
|
|
1804
|
+
spanId: padSpanId2(span.spanId),
|
|
1805
|
+
parentSpanId: span.parentSpanId ? padSpanId2(span.parentSpanId) : void 0,
|
|
1806
|
+
name: span.name,
|
|
1807
|
+
kind: 1,
|
|
1808
|
+
// SPAN_KIND_INTERNAL
|
|
1809
|
+
startTimeUnixNano: msToNs2(span.startedAt),
|
|
1810
|
+
endTimeUnixNano: msToNs2(endedAt),
|
|
1811
|
+
attributes: toAttributes2(attrs),
|
|
1812
|
+
status: span.status === "error" ? { code: 2, message: span.error } : { code: 1 }
|
|
1813
|
+
};
|
|
1814
|
+
}
|
|
1815
|
+
function toAttributes2(record) {
|
|
1816
|
+
return Object.entries(record).map(([key, value]) => ({
|
|
1817
|
+
key,
|
|
1818
|
+
value: typeof value === "number" ? Number.isInteger(value) ? { intValue: value.toString() } : { doubleValue: value } : typeof value === "boolean" ? { boolValue: value } : { stringValue: value }
|
|
1819
|
+
}));
|
|
1820
|
+
}
|
|
1821
|
+
function msToNs2(ms) {
|
|
1822
|
+
return (BigInt(Math.floor(ms)) * 1000000n).toString();
|
|
1823
|
+
}
|
|
1824
|
+
function padSpanId2(id) {
|
|
1825
|
+
const cleaned = id.replace(/-/g, "");
|
|
1826
|
+
return cleaned.slice(0, 16).padEnd(16, "0");
|
|
1827
|
+
}
|
|
1828
|
+
function padTraceId(id) {
|
|
1829
|
+
const cleaned = id.replace(/-/g, "");
|
|
1830
|
+
return cleaned.slice(0, 32).padEnd(32, "0");
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1630
1833
|
// src/trace/redact.ts
|
|
1631
1834
|
var DEFAULT_REDACTION_RULES = [
|
|
1632
1835
|
{ id: "email", pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi },
|
|
@@ -1855,6 +2058,9 @@ export {
|
|
|
1855
2058
|
FileSystemTraceStore,
|
|
1856
2059
|
OTEL_AGENT_EVAL_SCOPE,
|
|
1857
2060
|
exportRunAsOtlp,
|
|
2061
|
+
otelRunCompleteHook,
|
|
2062
|
+
createOtelTracingStore,
|
|
2063
|
+
createOtelExporter,
|
|
1858
2064
|
DEFAULT_REDACTION_RULES,
|
|
1859
2065
|
REDACTION_VERSION,
|
|
1860
2066
|
redactString,
|
|
@@ -1864,4 +2070,4 @@ export {
|
|
|
1864
2070
|
createReplayFetch,
|
|
1865
2071
|
iterateRawCalls
|
|
1866
2072
|
};
|
|
1867
|
-
//# sourceMappingURL=chunk-
|
|
2073
|
+
//# sourceMappingURL=chunk-L5UNCDAJ.js.map
|