@graphorin/observability 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/LICENSE +21 -0
- package/README.md +115 -0
- package/dist/cost/cost-tracker.d.ts +28 -0
- package/dist/cost/cost-tracker.d.ts.map +1 -0
- package/dist/cost/cost-tracker.js +169 -0
- package/dist/cost/cost-tracker.js.map +1 -0
- package/dist/cost/index.d.ts +3 -0
- package/dist/cost/index.js +3 -0
- package/dist/cost/types.d.ts +104 -0
- package/dist/cost/types.d.ts.map +1 -0
- package/dist/eval/index.d.ts +3 -0
- package/dist/eval/index.js +3 -0
- package/dist/eval/runner.d.ts +11 -0
- package/dist/eval/runner.d.ts.map +1 -0
- package/dist/eval/runner.js +113 -0
- package/dist/eval/runner.js.map +1 -0
- package/dist/eval/types.d.ts +108 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/exporters/console.d.ts +27 -0
- package/dist/exporters/console.d.ts.map +1 -0
- package/dist/exporters/console.js +47 -0
- package/dist/exporters/console.js.map +1 -0
- package/dist/exporters/index.d.ts +6 -0
- package/dist/exporters/index.js +7 -0
- package/dist/exporters/jsonl.d.ts +57 -0
- package/dist/exporters/jsonl.d.ts.map +1 -0
- package/dist/exporters/jsonl.js +95 -0
- package/dist/exporters/jsonl.js.map +1 -0
- package/dist/exporters/otlp-http.d.ts +38 -0
- package/dist/exporters/otlp-http.d.ts.map +1 -0
- package/dist/exporters/otlp-http.js +115 -0
- package/dist/exporters/otlp-http.js.map +1 -0
- package/dist/exporters/types.d.ts +72 -0
- package/dist/exporters/types.d.ts.map +1 -0
- package/dist/exporters/types.js +13 -0
- package/dist/exporters/types.js.map +1 -0
- package/dist/exporters/with-validation.d.ts +45 -0
- package/dist/exporters/with-validation.d.ts.map +1 -0
- package/dist/exporters/with-validation.js +99 -0
- package/dist/exporters/with-validation.js.map +1 -0
- package/dist/gen-ai/emit.d.ts +27 -0
- package/dist/gen-ai/emit.d.ts.map +1 -0
- package/dist/gen-ai/emit.js +88 -0
- package/dist/gen-ai/emit.js.map +1 -0
- package/dist/gen-ai/index.d.ts +5 -0
- package/dist/gen-ai/index.js +5 -0
- package/dist/gen-ai/operation-mapping.d.ts +24 -0
- package/dist/gen-ai/operation-mapping.d.ts.map +1 -0
- package/dist/gen-ai/operation-mapping.js +67 -0
- package/dist/gen-ai/operation-mapping.js.map +1 -0
- package/dist/gen-ai/system-derivation.d.ts +41 -0
- package/dist/gen-ai/system-derivation.d.ts.map +1 -0
- package/dist/gen-ai/system-derivation.js +67 -0
- package/dist/gen-ai/system-derivation.js.map +1 -0
- package/dist/gen-ai/types.d.ts +87 -0
- package/dist/gen-ai/types.d.ts.map +1 -0
- package/dist/index.d.ts +58 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +60 -0
- package/dist/index.js.map +1 -0
- package/dist/logger/index.d.ts +2 -0
- package/dist/logger/index.js +3 -0
- package/dist/logger/logger.d.ts +53 -0
- package/dist/logger/logger.d.ts.map +1 -0
- package/dist/logger/logger.js +145 -0
- package/dist/logger/logger.js.map +1 -0
- package/dist/openinference/index.d.ts +45 -0
- package/dist/openinference/index.d.ts.map +1 -0
- package/dist/openinference/index.js +92 -0
- package/dist/openinference/index.js.map +1 -0
- package/dist/redaction/config.d.ts +39 -0
- package/dist/redaction/config.d.ts.map +1 -0
- package/dist/redaction/config.js +15 -0
- package/dist/redaction/config.js.map +1 -0
- package/dist/redaction/errors.d.ts +36 -0
- package/dist/redaction/errors.d.ts.map +1 -0
- package/dist/redaction/errors.js +42 -0
- package/dist/redaction/errors.js.map +1 -0
- package/dist/redaction/imperative-patterns.d.ts +108 -0
- package/dist/redaction/imperative-patterns.d.ts.map +1 -0
- package/dist/redaction/imperative-patterns.js +187 -0
- package/dist/redaction/imperative-patterns.js.map +1 -0
- package/dist/redaction/index.d.ts +7 -0
- package/dist/redaction/index.js +7 -0
- package/dist/redaction/patterns.d.ts +85 -0
- package/dist/redaction/patterns.d.ts.map +1 -0
- package/dist/redaction/patterns.js +172 -0
- package/dist/redaction/patterns.js.map +1 -0
- package/dist/redaction/types.d.ts +103 -0
- package/dist/redaction/types.d.ts.map +1 -0
- package/dist/redaction/validator.d.ts +26 -0
- package/dist/redaction/validator.d.ts.map +1 -0
- package/dist/redaction/validator.js +195 -0
- package/dist/redaction/validator.js.map +1 -0
- package/dist/replay/config.d.ts +56 -0
- package/dist/replay/config.d.ts.map +1 -0
- package/dist/replay/config.js +18 -0
- package/dist/replay/config.js.map +1 -0
- package/dist/replay/index.d.ts +5 -0
- package/dist/replay/index.js +5 -0
- package/dist/replay/log.d.ts +39 -0
- package/dist/replay/log.d.ts.map +1 -0
- package/dist/replay/log.js +59 -0
- package/dist/replay/log.js.map +1 -0
- package/dist/replay/replay.d.ts +25 -0
- package/dist/replay/replay.d.ts.map +1 -0
- package/dist/replay/replay.js +173 -0
- package/dist/replay/replay.js.map +1 -0
- package/dist/replay/types.d.ts +110 -0
- package/dist/replay/types.d.ts.map +1 -0
- package/dist/telemetry/index.d.ts +63 -0
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/index.js +49 -0
- package/dist/telemetry/index.js.map +1 -0
- package/dist/tracer/ids.d.ts +23 -0
- package/dist/tracer/ids.d.ts.map +1 -0
- package/dist/tracer/ids.js +30 -0
- package/dist/tracer/ids.js.map +1 -0
- package/dist/tracer/index.d.ts +6 -0
- package/dist/tracer/index.js +6 -0
- package/dist/tracer/sampling.d.ts +77 -0
- package/dist/tracer/sampling.d.ts.map +1 -0
- package/dist/tracer/sampling.js +67 -0
- package/dist/tracer/sampling.js.map +1 -0
- package/dist/tracer/span-names.d.ts +11 -0
- package/dist/tracer/span-names.d.ts.map +1 -0
- package/dist/tracer/span-names.js +11 -0
- package/dist/tracer/span-names.js.map +1 -0
- package/dist/tracer/span.d.ts +27 -0
- package/dist/tracer/span.d.ts.map +1 -0
- package/dist/tracer/span.js +80 -0
- package/dist/tracer/span.js.map +1 -0
- package/dist/tracer/tracer.d.ts +101 -0
- package/dist/tracer/tracer.d.ts.map +1 -0
- package/dist/tracer/tracer.js +150 -0
- package/dist/tracer/tracer.js.map +1 -0
- package/package.json +138 -0
package/CHANGELOG.md
ADDED
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Oleksiy Stepurenko
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# @graphorin/observability
|
|
2
|
+
|
|
3
|
+
> Observability primitives for the [Graphorin](https://github.com/o-stepper/graphorin) framework.
|
|
4
|
+
|
|
5
|
+
`@graphorin/observability` ships every cross-cutting tracing,
|
|
6
|
+
redaction, replay, cost, and structured-logging primitive every other
|
|
7
|
+
`@graphorin/*` package builds on. The package is intentionally
|
|
8
|
+
opinionated: every exporter must be wrapped through
|
|
9
|
+
`withValidation(...)` before it can ship a single span, the validator
|
|
10
|
+
defaults to **default-deny non-public**, and the framework makes zero
|
|
11
|
+
outbound network calls without an explicit user action.
|
|
12
|
+
|
|
13
|
+
## Highlights
|
|
14
|
+
|
|
15
|
+
- **Typed `AISpan<T>` tracer.** `createTracer({...})` returns a
|
|
16
|
+
`GraphorinTracer` that emits `AISpan<SpanType>` records compatible
|
|
17
|
+
with OpenTelemetry. Sampling rules + per-event sampling cover noisy
|
|
18
|
+
span kinds (`memory.embed`, `tool.execute.partial`, …).
|
|
19
|
+
- **Mandatory `withValidation(...)` wrapper.** Every exporter passed to
|
|
20
|
+
`createTracer({ exporters })` is wrapped through the configured
|
|
21
|
+
`RedactionValidator`. Registering a raw exporter while
|
|
22
|
+
`validation: 'off'` triggers `UnvalidatedExporterError` at startup —
|
|
23
|
+
there is no silent path.
|
|
24
|
+
- **`RedactionValidator` with 14 built-in patterns.** API key / JWT /
|
|
25
|
+
PEM private key / GitHub PAT / AWS access key / Graphorin token /
|
|
26
|
+
bearer header / basic-auth header / email / credit card / US SSN /
|
|
27
|
+
E.164 phone / IBAN — all on by default. Three additional patterns
|
|
28
|
+
(IPv4, IPv6, GCP service account) are opt-in.
|
|
29
|
+
- **OpenTelemetry GenAI semantic-conventions conformance.**
|
|
30
|
+
`emitGenAIAttributes(span, {...})`, `emitGenAIMessageEvents(span, [...])`,
|
|
31
|
+
and `deriveGenAISystem(...)` ship the canonical
|
|
32
|
+
`gen_ai.*` attribute family alongside the existing
|
|
33
|
+
`graphorin.*` attributes — additive, never replacing.
|
|
34
|
+
- **OpenInference span-kind layer.** `emitOpenInferenceKind(span)` emits
|
|
35
|
+
the `openinference.span.kind` attribute via the canonical
|
|
36
|
+
per-`SpanType` mapping (`agent.*` → `AGENT`,
|
|
37
|
+
`provider.*` → `LLM`, `tool.execute` → `TOOL`,
|
|
38
|
+
`memory.*` → `RETRIEVER` / `EMBEDDING`, `workflow.*` → `CHAIN`,
|
|
39
|
+
`agent.evaluator.iteration` → `EVALUATOR`).
|
|
40
|
+
- **`ConsoleExporter`, `JSONLExporter`, `OTLPHttpExporter`.** The
|
|
41
|
+
built-in exporters cover dev (console), replay (append-only JSONL
|
|
42
|
+
with `0700` directories + `0600` files), and remote OTLP collectors
|
|
43
|
+
(`fetch`-based reference implementation, swappable `fetchImpl`).
|
|
44
|
+
- **Sanitized-by-default `Replay`.** `createReplay({...})` exposes a
|
|
45
|
+
`run(...)` async iterator that yields `replay.start` /
|
|
46
|
+
`replay.event` / `replay.skipped` / `replay.end` markers. Raw mode
|
|
47
|
+
requires the `canReadRaw` callback to return `true` and emits an
|
|
48
|
+
audit-bridge entry on every invocation.
|
|
49
|
+
- **Hierarchical `CostTracker`.** `createCostTracker({...})` rolls up
|
|
50
|
+
tokens + cost across parent-child spans and supports per-run /
|
|
51
|
+
per-session / per-agent / per-user budgets with an `onExceed`
|
|
52
|
+
callback.
|
|
53
|
+
- **Structured logger.** `createLogger({...})` writes JSON or pretty
|
|
54
|
+
records, automatically correlates with the current span via
|
|
55
|
+
`withCurrentSpan(...)`, and pipes every field through the validator.
|
|
56
|
+
- **Zero-default telemetry.** `getTelemetryStatus()` and
|
|
57
|
+
`announceTelemetryPosture()` expose the v0.1 promise: the framework
|
|
58
|
+
performs zero outbound network calls, no version pings, no crash
|
|
59
|
+
reports, no auto-updates. The `GRAPHORIN_TELEMETRY` and
|
|
60
|
+
`GRAPHORIN_NO_PHONE_HOME` environment variables are reserved for
|
|
61
|
+
forward compatibility and are acknowledged at startup.
|
|
62
|
+
|
|
63
|
+
## Installation
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pnpm add @graphorin/observability
|
|
67
|
+
# Optional peer deps for OTLP export — install only when you need them:
|
|
68
|
+
pnpm add @opentelemetry/api @opentelemetry/sdk-node @opentelemetry/exporter-trace-otlp-http
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Quick start
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
import {
|
|
75
|
+
createTracer,
|
|
76
|
+
createConsoleExporter,
|
|
77
|
+
createJSONLExporter,
|
|
78
|
+
withValidation,
|
|
79
|
+
} from '@graphorin/observability';
|
|
80
|
+
|
|
81
|
+
const tracer = createTracer({
|
|
82
|
+
serviceName: 'my-assistant',
|
|
83
|
+
exporters: [
|
|
84
|
+
// Auto-wrapped via the tracer-managed validator.
|
|
85
|
+
createConsoleExporter({ pretty: true }),
|
|
86
|
+
// Manually wrapped — useful when each exporter needs its own policy.
|
|
87
|
+
withValidation(createJSONLExporter({ path: './traces' }), {
|
|
88
|
+
minTier: 'internal',
|
|
89
|
+
}),
|
|
90
|
+
],
|
|
91
|
+
validation: { minTier: 'public', failOnUnredactedSensitive: false },
|
|
92
|
+
sampling: {
|
|
93
|
+
rate: 1.0,
|
|
94
|
+
rules: [{ type: 'memory.embed', rate: 0.1 }],
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
await tracer.span(
|
|
99
|
+
{ type: 'agent.run', attrs: { 'graphorin.agent.id': 'support-bot' } },
|
|
100
|
+
async (span) => {
|
|
101
|
+
span.setAttributes({ 'graphorin.session.id': 'session-123' });
|
|
102
|
+
return runAgent();
|
|
103
|
+
},
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
await tracer.shutdown();
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## License
|
|
110
|
+
|
|
111
|
+
MIT © 2026 [Oleksiy Stepurenko](https://github.com/o-stepper).
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
**Project Graphorin** · v0.5.0 · MIT License · © 2026 Oleksiy Stepurenko · <https://github.com/o-stepper/graphorin>
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { CostRecordInput, CostScope, CostSnapshot, CostTrackerOptions } from "./types.js";
|
|
2
|
+
|
|
3
|
+
//#region src/cost/cost-tracker.d.ts
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @stable
|
|
7
|
+
*/
|
|
8
|
+
interface CostTracker {
|
|
9
|
+
/** Record a single LLM-call usage / cost figure. */
|
|
10
|
+
record(input: CostRecordInput): void;
|
|
11
|
+
/** Snapshot for a given scope id. Returns zero figures when unknown. */
|
|
12
|
+
usage(scope: CostScope, id: string): CostSnapshot;
|
|
13
|
+
/** Snapshot for a single span id (carries nested attributions). */
|
|
14
|
+
usageForSpan(spanId: string): CostSnapshot;
|
|
15
|
+
/** Reset every counter back to zero. */
|
|
16
|
+
reset(): void;
|
|
17
|
+
/** Subscribe to per-scope rollup notifications. Returns an unsubscribe. */
|
|
18
|
+
onRollup(listener: (input: CostRecordInput) => void): () => void;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Build a {@link CostTracker} configured with the supplied budgets.
|
|
22
|
+
*
|
|
23
|
+
* @stable
|
|
24
|
+
*/
|
|
25
|
+
declare function createCostTracker(opts?: CostTrackerOptions): CostTracker;
|
|
26
|
+
//#endregion
|
|
27
|
+
export { CostTracker, createCostTracker };
|
|
28
|
+
//# sourceMappingURL=cost-tracker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-tracker.d.ts","names":[],"sources":["../../src/cost/cost-tracker.ts"],"sourcesContent":[],"mappings":";;;;;;;UA2BiB,WAAA;;gBAED;;eAED,wBAAwB;;gCAEP;;;;6BAIH;;;;;;;iBAmGb,iBAAA,QAAwB,qBAA0B"}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
//#region src/cost/cost-tracker.ts
|
|
2
|
+
function freshBucket() {
|
|
3
|
+
return {
|
|
4
|
+
promptTokens: 0,
|
|
5
|
+
completionTokens: 0,
|
|
6
|
+
reasoningTokens: 0,
|
|
7
|
+
callCount: 0,
|
|
8
|
+
costAmount: 0,
|
|
9
|
+
costCurrency: null,
|
|
10
|
+
mixedCurrency: false,
|
|
11
|
+
byModel: /* @__PURE__ */ new Map(),
|
|
12
|
+
exceeded: false
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
function addCost(bucket, cost) {
|
|
16
|
+
if (bucket.costCurrency !== null && bucket.costCurrency !== cost.currency) bucket.mixedCurrency = true;
|
|
17
|
+
else if (bucket.costCurrency === null) bucket.costCurrency = cost.currency;
|
|
18
|
+
bucket.costAmount += cost.amount;
|
|
19
|
+
}
|
|
20
|
+
function combine(target, input) {
|
|
21
|
+
target.promptTokens += input.promptTokens;
|
|
22
|
+
target.completionTokens += input.completionTokens;
|
|
23
|
+
target.reasoningTokens += input.reasoningTokens ?? 0;
|
|
24
|
+
target.callCount += 1;
|
|
25
|
+
if (input.cost !== void 0) addCost(target, input.cost);
|
|
26
|
+
const modelBucket = target.byModel.get(input.model) ?? freshBucket();
|
|
27
|
+
modelBucket.promptTokens += input.promptTokens;
|
|
28
|
+
modelBucket.completionTokens += input.completionTokens;
|
|
29
|
+
modelBucket.reasoningTokens += input.reasoningTokens ?? 0;
|
|
30
|
+
modelBucket.callCount += 1;
|
|
31
|
+
if (input.cost !== void 0) addCost(modelBucket, input.cost);
|
|
32
|
+
target.byModel.set(input.model, modelBucket);
|
|
33
|
+
}
|
|
34
|
+
function snapshotOf(bucket) {
|
|
35
|
+
const cost = bucket.costCurrency === null ? null : {
|
|
36
|
+
amount: bucket.costAmount,
|
|
37
|
+
currency: bucket.costCurrency
|
|
38
|
+
};
|
|
39
|
+
return {
|
|
40
|
+
promptTokens: bucket.promptTokens,
|
|
41
|
+
completionTokens: bucket.completionTokens,
|
|
42
|
+
reasoningTokens: bucket.reasoningTokens,
|
|
43
|
+
totalTokens: bucket.promptTokens + bucket.completionTokens + bucket.reasoningTokens,
|
|
44
|
+
callCount: bucket.callCount,
|
|
45
|
+
cost,
|
|
46
|
+
mixedCurrency: bucket.mixedCurrency,
|
|
47
|
+
byModel: [...bucket.byModel.entries()].map(([model, b]) => ({
|
|
48
|
+
model,
|
|
49
|
+
promptTokens: b.promptTokens,
|
|
50
|
+
completionTokens: b.completionTokens,
|
|
51
|
+
reasoningTokens: b.reasoningTokens,
|
|
52
|
+
callCount: b.callCount,
|
|
53
|
+
cost: b.costCurrency === null ? null : {
|
|
54
|
+
amount: b.costAmount,
|
|
55
|
+
currency: b.costCurrency
|
|
56
|
+
},
|
|
57
|
+
mixedCurrency: b.mixedCurrency
|
|
58
|
+
}))
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
const ZERO = Object.freeze({
|
|
62
|
+
promptTokens: 0,
|
|
63
|
+
completionTokens: 0,
|
|
64
|
+
reasoningTokens: 0,
|
|
65
|
+
totalTokens: 0,
|
|
66
|
+
callCount: 0,
|
|
67
|
+
cost: null,
|
|
68
|
+
mixedCurrency: false,
|
|
69
|
+
byModel: []
|
|
70
|
+
});
|
|
71
|
+
/**
|
|
72
|
+
* Build a {@link CostTracker} configured with the supplied budgets.
|
|
73
|
+
*
|
|
74
|
+
* @stable
|
|
75
|
+
*/
|
|
76
|
+
function createCostTracker(opts = {}) {
|
|
77
|
+
const budgets = opts.budgets ?? {};
|
|
78
|
+
const currency = budgets.currency ?? "USD";
|
|
79
|
+
const onExceed = opts.onExceed;
|
|
80
|
+
const bySpan = /* @__PURE__ */ new Map();
|
|
81
|
+
const byScope = {
|
|
82
|
+
run: /* @__PURE__ */ new Map(),
|
|
83
|
+
session: /* @__PURE__ */ new Map(),
|
|
84
|
+
agent: /* @__PURE__ */ new Map(),
|
|
85
|
+
user: /* @__PURE__ */ new Map()
|
|
86
|
+
};
|
|
87
|
+
const parents = /* @__PURE__ */ new Map();
|
|
88
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
89
|
+
function bump(scope, id, input) {
|
|
90
|
+
if (id === void 0) return;
|
|
91
|
+
const map = byScope[scope];
|
|
92
|
+
const bucket = map.get(id) ?? freshBucket();
|
|
93
|
+
combine(bucket, input);
|
|
94
|
+
map.set(id, bucket);
|
|
95
|
+
enforceBudget(scope, id, bucket, onExceed);
|
|
96
|
+
}
|
|
97
|
+
function enforceBudget(scope, id, bucket, cb) {
|
|
98
|
+
const limit = pickBudget(budgets, scope);
|
|
99
|
+
if (limit === void 0) return;
|
|
100
|
+
if (bucket.exceeded) return;
|
|
101
|
+
if (bucket.costAmount > limit) {
|
|
102
|
+
bucket.exceeded = true;
|
|
103
|
+
cb?.({
|
|
104
|
+
scope,
|
|
105
|
+
id,
|
|
106
|
+
budget: limit,
|
|
107
|
+
actual: bucket.costAmount,
|
|
108
|
+
currency: bucket.costCurrency ?? currency
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
record(input) {
|
|
114
|
+
const spanBucket = bySpan.get(input.spanId) ?? freshBucket();
|
|
115
|
+
combine(spanBucket, input);
|
|
116
|
+
bySpan.set(input.spanId, spanBucket);
|
|
117
|
+
if (input.parentSpanId !== void 0) parents.set(input.spanId, input.parentSpanId);
|
|
118
|
+
const seen = /* @__PURE__ */ new Set();
|
|
119
|
+
let ancestor = parents.get(input.spanId);
|
|
120
|
+
while (ancestor !== void 0 && !seen.has(ancestor)) {
|
|
121
|
+
seen.add(ancestor);
|
|
122
|
+
const ancestorBucket = bySpan.get(ancestor) ?? freshBucket();
|
|
123
|
+
combine(ancestorBucket, input);
|
|
124
|
+
bySpan.set(ancestor, ancestorBucket);
|
|
125
|
+
ancestor = parents.get(ancestor);
|
|
126
|
+
}
|
|
127
|
+
bump("run", input.runId, input);
|
|
128
|
+
bump("session", input.sessionId, input);
|
|
129
|
+
bump("agent", input.agentId, input);
|
|
130
|
+
bump("user", input.userId, input);
|
|
131
|
+
for (const listener of listeners) try {
|
|
132
|
+
listener(input);
|
|
133
|
+
} catch {}
|
|
134
|
+
},
|
|
135
|
+
usage(scope, id) {
|
|
136
|
+
const bucket = byScope[scope].get(id);
|
|
137
|
+
if (bucket === void 0) return ZERO;
|
|
138
|
+
return snapshotOf(bucket);
|
|
139
|
+
},
|
|
140
|
+
usageForSpan(spanId) {
|
|
141
|
+
const bucket = bySpan.get(spanId);
|
|
142
|
+
if (bucket === void 0) return ZERO;
|
|
143
|
+
return snapshotOf(bucket);
|
|
144
|
+
},
|
|
145
|
+
reset() {
|
|
146
|
+
bySpan.clear();
|
|
147
|
+
for (const map of Object.values(byScope)) map.clear();
|
|
148
|
+
parents.clear();
|
|
149
|
+
},
|
|
150
|
+
onRollup(listener) {
|
|
151
|
+
listeners.add(listener);
|
|
152
|
+
return () => {
|
|
153
|
+
listeners.delete(listener);
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
function pickBudget(budgets, scope) {
|
|
159
|
+
switch (scope) {
|
|
160
|
+
case "session": return budgets.perSession;
|
|
161
|
+
case "user": return budgets.perUser;
|
|
162
|
+
case "agent": return budgets.perAgent;
|
|
163
|
+
case "run": return budgets.perRun;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
//#endregion
|
|
168
|
+
export { createCostTracker };
|
|
169
|
+
//# sourceMappingURL=cost-tracker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cost-tracker.js","names":["cost: Cost | null","ZERO: CostSnapshot","budgets: CostBudgets","byScope: Record<CostScope, Map<string, AggregateBucket>>"],"sources":["../../src/cost/cost-tracker.ts"],"sourcesContent":["/**\n * `createCostTracker(...)` — aggregate token + cost figures across\n * spans and provide per-scope queries.\n *\n * The tracker preserves parent-child relationships: when\n * `record({ parentSpanId })` is supplied, the recorded usage is also\n * attributed to the ancestor span via the configured spans-by-id\n * lookup. Aggregations across runs / sessions / agents / users are\n * computed on demand.\n *\n * @packageDocumentation\n */\n\nimport type { Cost } from '@graphorin/core';\n\nimport type {\n CostBudgetExceededCallback,\n CostBudgets,\n CostRecordInput,\n CostScope,\n CostSnapshot,\n CostTrackerOptions,\n} from './types.js';\n\n/**\n * @stable\n */\nexport interface CostTracker {\n /** Record a single LLM-call usage / cost figure. */\n record(input: CostRecordInput): void;\n /** Snapshot for a given scope id. Returns zero figures when unknown. */\n usage(scope: CostScope, id: string): CostSnapshot;\n /** Snapshot for a single span id (carries nested attributions). */\n usageForSpan(spanId: string): CostSnapshot;\n /** Reset every counter back to zero. */\n reset(): void;\n /** Subscribe to per-scope rollup notifications. Returns an unsubscribe. */\n onRollup(listener: (input: CostRecordInput) => void): () => void;\n}\n\ninterface AggregateBucket {\n promptTokens: number;\n completionTokens: number;\n reasoningTokens: number;\n callCount: number;\n costAmount: number;\n costCurrency: string | null;\n /** RP-22: set when records carrying differing currencies are aggregated. */\n mixedCurrency: boolean;\n byModel: Map<string, AggregateBucket>;\n exceeded: boolean;\n}\n\nfunction freshBucket(): AggregateBucket {\n return {\n promptTokens: 0,\n completionTokens: 0,\n reasoningTokens: 0,\n callCount: 0,\n costAmount: 0,\n costCurrency: null,\n mixedCurrency: false,\n byModel: new Map(),\n exceeded: false,\n };\n}\n\n// RP-22: accumulate a cost without silently overwriting the currency. The\n// first currency seen is kept; a record in a different currency flags the\n// bucket as `mixedCurrency` so a USD + EUR total is never reported as one\n// clean figure.\nfunction addCost(bucket: AggregateBucket, cost: Cost): void {\n if (bucket.costCurrency !== null && bucket.costCurrency !== cost.currency) {\n bucket.mixedCurrency = true;\n } else if (bucket.costCurrency === null) {\n bucket.costCurrency = cost.currency;\n }\n bucket.costAmount += cost.amount;\n}\n\nfunction combine(target: AggregateBucket, input: CostRecordInput): void {\n target.promptTokens += input.promptTokens;\n target.completionTokens += input.completionTokens;\n target.reasoningTokens += input.reasoningTokens ?? 0;\n target.callCount += 1;\n if (input.cost !== undefined) addCost(target, input.cost);\n const modelBucket = target.byModel.get(input.model) ?? freshBucket();\n modelBucket.promptTokens += input.promptTokens;\n modelBucket.completionTokens += input.completionTokens;\n modelBucket.reasoningTokens += input.reasoningTokens ?? 0;\n modelBucket.callCount += 1;\n if (input.cost !== undefined) addCost(modelBucket, input.cost);\n target.byModel.set(input.model, modelBucket);\n}\n\nfunction snapshotOf(bucket: AggregateBucket): CostSnapshot {\n const cost: Cost | null =\n bucket.costCurrency === null\n ? null\n : { amount: bucket.costAmount, currency: bucket.costCurrency };\n return {\n promptTokens: bucket.promptTokens,\n completionTokens: bucket.completionTokens,\n reasoningTokens: bucket.reasoningTokens,\n totalTokens: bucket.promptTokens + bucket.completionTokens + bucket.reasoningTokens,\n callCount: bucket.callCount,\n cost,\n mixedCurrency: bucket.mixedCurrency,\n byModel: [...bucket.byModel.entries()].map(([model, b]) => ({\n model,\n promptTokens: b.promptTokens,\n completionTokens: b.completionTokens,\n reasoningTokens: b.reasoningTokens,\n callCount: b.callCount,\n cost: b.costCurrency === null ? null : { amount: b.costAmount, currency: b.costCurrency },\n mixedCurrency: b.mixedCurrency,\n })),\n };\n}\n\nconst ZERO: CostSnapshot = Object.freeze({\n promptTokens: 0,\n completionTokens: 0,\n reasoningTokens: 0,\n totalTokens: 0,\n callCount: 0,\n cost: null,\n mixedCurrency: false,\n byModel: [],\n});\n\n/**\n * Build a {@link CostTracker} configured with the supplied budgets.\n *\n * @stable\n */\nexport function createCostTracker(opts: CostTrackerOptions = {}): CostTracker {\n const budgets: CostBudgets = opts.budgets ?? {};\n const currency = budgets.currency ?? 'USD';\n const onExceed = opts.onExceed;\n\n const bySpan = new Map<string, AggregateBucket>();\n const byScope: Record<CostScope, Map<string, AggregateBucket>> = {\n run: new Map(),\n session: new Map(),\n agent: new Map(),\n user: new Map(),\n };\n\n // Parent index: child span -> parent span. Used to roll attributions\n // up the ancestor chain on every record call.\n const parents = new Map<string, string>();\n const listeners = new Set<(input: CostRecordInput) => void>();\n\n function bump(scope: CostScope, id: string | undefined, input: CostRecordInput): void {\n if (id === undefined) return;\n const map = byScope[scope];\n const bucket = map.get(id) ?? freshBucket();\n combine(bucket, input);\n map.set(id, bucket);\n enforceBudget(scope, id, bucket, onExceed);\n }\n\n function enforceBudget(\n scope: CostScope,\n id: string,\n bucket: AggregateBucket,\n cb?: CostBudgetExceededCallback,\n ): void {\n const limit = pickBudget(budgets, scope);\n if (limit === undefined) return;\n if (bucket.exceeded) return;\n if (bucket.costAmount > limit) {\n bucket.exceeded = true;\n cb?.({\n scope,\n id,\n budget: limit,\n actual: bucket.costAmount,\n currency: bucket.costCurrency ?? currency,\n });\n }\n }\n\n return {\n record(input: CostRecordInput): void {\n // Record the raw span figures.\n const spanBucket = bySpan.get(input.spanId) ?? freshBucket();\n combine(spanBucket, input);\n bySpan.set(input.spanId, spanBucket);\n\n // Walk the ancestor chain so that the figures roll up to every\n // parent span. The walker breaks if it encounters a cycle.\n if (input.parentSpanId !== undefined) {\n parents.set(input.spanId, input.parentSpanId);\n }\n const seen = new Set<string>();\n let ancestor = parents.get(input.spanId);\n while (ancestor !== undefined && !seen.has(ancestor)) {\n seen.add(ancestor);\n const ancestorBucket = bySpan.get(ancestor) ?? freshBucket();\n combine(ancestorBucket, input);\n bySpan.set(ancestor, ancestorBucket);\n ancestor = parents.get(ancestor);\n }\n\n // Roll up to the requested aggregation scopes.\n bump('run', input.runId, input);\n bump('session', input.sessionId, input);\n bump('agent', input.agentId, input);\n bump('user', input.userId, input);\n\n for (const listener of listeners) {\n try {\n listener(input);\n } catch {\n // Listeners must never break the tracker.\n }\n }\n },\n usage(scope: CostScope, id: string): CostSnapshot {\n const bucket = byScope[scope].get(id);\n if (bucket === undefined) return ZERO;\n return snapshotOf(bucket);\n },\n usageForSpan(spanId: string): CostSnapshot {\n const bucket = bySpan.get(spanId);\n if (bucket === undefined) return ZERO;\n return snapshotOf(bucket);\n },\n reset(): void {\n bySpan.clear();\n for (const map of Object.values(byScope)) {\n map.clear();\n }\n parents.clear();\n },\n onRollup(listener): () => void {\n listeners.add(listener);\n return () => {\n listeners.delete(listener);\n };\n },\n };\n}\n\nfunction pickBudget(budgets: CostBudgets, scope: CostScope): number | undefined {\n switch (scope) {\n case 'session':\n return budgets.perSession;\n case 'user':\n return budgets.perUser;\n case 'agent':\n return budgets.perAgent;\n case 'run':\n return budgets.perRun;\n }\n}\n"],"mappings":";AAqDA,SAAS,cAA+B;AACtC,QAAO;EACL,cAAc;EACd,kBAAkB;EAClB,iBAAiB;EACjB,WAAW;EACX,YAAY;EACZ,cAAc;EACd,eAAe;EACf,yBAAS,IAAI,KAAK;EAClB,UAAU;EACX;;AAOH,SAAS,QAAQ,QAAyB,MAAkB;AAC1D,KAAI,OAAO,iBAAiB,QAAQ,OAAO,iBAAiB,KAAK,SAC/D,QAAO,gBAAgB;UACd,OAAO,iBAAiB,KACjC,QAAO,eAAe,KAAK;AAE7B,QAAO,cAAc,KAAK;;AAG5B,SAAS,QAAQ,QAAyB,OAA8B;AACtE,QAAO,gBAAgB,MAAM;AAC7B,QAAO,oBAAoB,MAAM;AACjC,QAAO,mBAAmB,MAAM,mBAAmB;AACnD,QAAO,aAAa;AACpB,KAAI,MAAM,SAAS,OAAW,SAAQ,QAAQ,MAAM,KAAK;CACzD,MAAM,cAAc,OAAO,QAAQ,IAAI,MAAM,MAAM,IAAI,aAAa;AACpE,aAAY,gBAAgB,MAAM;AAClC,aAAY,oBAAoB,MAAM;AACtC,aAAY,mBAAmB,MAAM,mBAAmB;AACxD,aAAY,aAAa;AACzB,KAAI,MAAM,SAAS,OAAW,SAAQ,aAAa,MAAM,KAAK;AAC9D,QAAO,QAAQ,IAAI,MAAM,OAAO,YAAY;;AAG9C,SAAS,WAAW,QAAuC;CACzD,MAAMA,OACJ,OAAO,iBAAiB,OACpB,OACA;EAAE,QAAQ,OAAO;EAAY,UAAU,OAAO;EAAc;AAClE,QAAO;EACL,cAAc,OAAO;EACrB,kBAAkB,OAAO;EACzB,iBAAiB,OAAO;EACxB,aAAa,OAAO,eAAe,OAAO,mBAAmB,OAAO;EACpE,WAAW,OAAO;EAClB;EACA,eAAe,OAAO;EACtB,SAAS,CAAC,GAAG,OAAO,QAAQ,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ;GAC1D;GACA,cAAc,EAAE;GAChB,kBAAkB,EAAE;GACpB,iBAAiB,EAAE;GACnB,WAAW,EAAE;GACb,MAAM,EAAE,iBAAiB,OAAO,OAAO;IAAE,QAAQ,EAAE;IAAY,UAAU,EAAE;IAAc;GACzF,eAAe,EAAE;GAClB,EAAE;EACJ;;AAGH,MAAMC,OAAqB,OAAO,OAAO;CACvC,cAAc;CACd,kBAAkB;CAClB,iBAAiB;CACjB,aAAa;CACb,WAAW;CACX,MAAM;CACN,eAAe;CACf,SAAS,EAAE;CACZ,CAAC;;;;;;AAOF,SAAgB,kBAAkB,OAA2B,EAAE,EAAe;CAC5E,MAAMC,UAAuB,KAAK,WAAW,EAAE;CAC/C,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,KAAK;CAEtB,MAAM,yBAAS,IAAI,KAA8B;CACjD,MAAMC,UAA2D;EAC/D,qBAAK,IAAI,KAAK;EACd,yBAAS,IAAI,KAAK;EAClB,uBAAO,IAAI,KAAK;EAChB,sBAAM,IAAI,KAAK;EAChB;CAID,MAAM,0BAAU,IAAI,KAAqB;CACzC,MAAM,4BAAY,IAAI,KAAuC;CAE7D,SAAS,KAAK,OAAkB,IAAwB,OAA8B;AACpF,MAAI,OAAO,OAAW;EACtB,MAAM,MAAM,QAAQ;EACpB,MAAM,SAAS,IAAI,IAAI,GAAG,IAAI,aAAa;AAC3C,UAAQ,QAAQ,MAAM;AACtB,MAAI,IAAI,IAAI,OAAO;AACnB,gBAAc,OAAO,IAAI,QAAQ,SAAS;;CAG5C,SAAS,cACP,OACA,IACA,QACA,IACM;EACN,MAAM,QAAQ,WAAW,SAAS,MAAM;AACxC,MAAI,UAAU,OAAW;AACzB,MAAI,OAAO,SAAU;AACrB,MAAI,OAAO,aAAa,OAAO;AAC7B,UAAO,WAAW;AAClB,QAAK;IACH;IACA;IACA,QAAQ;IACR,QAAQ,OAAO;IACf,UAAU,OAAO,gBAAgB;IAClC,CAAC;;;AAIN,QAAO;EACL,OAAO,OAA8B;GAEnC,MAAM,aAAa,OAAO,IAAI,MAAM,OAAO,IAAI,aAAa;AAC5D,WAAQ,YAAY,MAAM;AAC1B,UAAO,IAAI,MAAM,QAAQ,WAAW;AAIpC,OAAI,MAAM,iBAAiB,OACzB,SAAQ,IAAI,MAAM,QAAQ,MAAM,aAAa;GAE/C,MAAM,uBAAO,IAAI,KAAa;GAC9B,IAAI,WAAW,QAAQ,IAAI,MAAM,OAAO;AACxC,UAAO,aAAa,UAAa,CAAC,KAAK,IAAI,SAAS,EAAE;AACpD,SAAK,IAAI,SAAS;IAClB,MAAM,iBAAiB,OAAO,IAAI,SAAS,IAAI,aAAa;AAC5D,YAAQ,gBAAgB,MAAM;AAC9B,WAAO,IAAI,UAAU,eAAe;AACpC,eAAW,QAAQ,IAAI,SAAS;;AAIlC,QAAK,OAAO,MAAM,OAAO,MAAM;AAC/B,QAAK,WAAW,MAAM,WAAW,MAAM;AACvC,QAAK,SAAS,MAAM,SAAS,MAAM;AACnC,QAAK,QAAQ,MAAM,QAAQ,MAAM;AAEjC,QAAK,MAAM,YAAY,UACrB,KAAI;AACF,aAAS,MAAM;WACT;;EAKZ,MAAM,OAAkB,IAA0B;GAChD,MAAM,SAAS,QAAQ,OAAO,IAAI,GAAG;AACrC,OAAI,WAAW,OAAW,QAAO;AACjC,UAAO,WAAW,OAAO;;EAE3B,aAAa,QAA8B;GACzC,MAAM,SAAS,OAAO,IAAI,OAAO;AACjC,OAAI,WAAW,OAAW,QAAO;AACjC,UAAO,WAAW,OAAO;;EAE3B,QAAc;AACZ,UAAO,OAAO;AACd,QAAK,MAAM,OAAO,OAAO,OAAO,QAAQ,CACtC,KAAI,OAAO;AAEb,WAAQ,OAAO;;EAEjB,SAAS,UAAsB;AAC7B,aAAU,IAAI,SAAS;AACvB,gBAAa;AACX,cAAU,OAAO,SAAS;;;EAG/B;;AAGH,SAAS,WAAW,SAAsB,OAAsC;AAC9E,SAAQ,OAAR;EACE,KAAK,UACH,QAAO,QAAQ;EACjB,KAAK,OACH,QAAO,QAAQ;EACjB,KAAK,QACH,QAAO,QAAQ;EACjB,KAAK,MACH,QAAO,QAAQ"}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { CostBudgetExceededCallback, CostBudgets, CostRecordInput, CostScope, CostSnapshot, CostTrackerOptions } from "./types.js";
|
|
2
|
+
import { CostTracker, createCostTracker } from "./cost-tracker.js";
|
|
3
|
+
export { type CostBudgetExceededCallback, type CostBudgets, type CostRecordInput, type CostScope, type CostSnapshot, type CostTracker, type CostTrackerOptions, createCostTracker };
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { Cost } from "@graphorin/core";
|
|
2
|
+
|
|
3
|
+
//#region src/cost/types.d.ts
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Aggregation scope used by {@link CostTracker.usage}. The framework
|
|
7
|
+
* tracks four canonical scopes; deployments needing additional
|
|
8
|
+
* dimensions can build them by registering listeners on
|
|
9
|
+
* `CostTracker.onRollup(...)`.
|
|
10
|
+
*
|
|
11
|
+
* @stable
|
|
12
|
+
*/
|
|
13
|
+
type CostScope = 'run' | 'session' | 'agent' | 'user';
|
|
14
|
+
/**
|
|
15
|
+
* Per-call usage record fed into {@link CostTracker.record}.
|
|
16
|
+
*
|
|
17
|
+
* @stable
|
|
18
|
+
*/
|
|
19
|
+
interface CostRecordInput {
|
|
20
|
+
readonly model: string;
|
|
21
|
+
readonly promptTokens: number;
|
|
22
|
+
readonly completionTokens: number;
|
|
23
|
+
readonly reasoningTokens?: number;
|
|
24
|
+
readonly cost?: Cost;
|
|
25
|
+
/** Span id; used to thread parent-child rollups. */
|
|
26
|
+
readonly spanId: string;
|
|
27
|
+
/** Optional parent span id (when the call is part of a nested run). */
|
|
28
|
+
readonly parentSpanId?: string;
|
|
29
|
+
readonly runId?: string;
|
|
30
|
+
readonly sessionId?: string;
|
|
31
|
+
readonly agentId?: string;
|
|
32
|
+
readonly userId?: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Snapshot returned by {@link CostTracker.usage}.
|
|
36
|
+
*
|
|
37
|
+
* @stable
|
|
38
|
+
*/
|
|
39
|
+
interface CostSnapshot {
|
|
40
|
+
readonly totalTokens: number;
|
|
41
|
+
readonly promptTokens: number;
|
|
42
|
+
readonly completionTokens: number;
|
|
43
|
+
readonly reasoningTokens: number;
|
|
44
|
+
readonly callCount: number;
|
|
45
|
+
readonly cost: Cost | null;
|
|
46
|
+
/**
|
|
47
|
+
* RP-22: `true` when records carrying differing currencies were aggregated
|
|
48
|
+
* into this snapshot. `cost.amount` is then a sum across currencies and must
|
|
49
|
+
* not be treated as a single clean figure.
|
|
50
|
+
*/
|
|
51
|
+
readonly mixedCurrency: boolean;
|
|
52
|
+
readonly byModel: ReadonlyArray<{
|
|
53
|
+
readonly model: string;
|
|
54
|
+
readonly promptTokens: number;
|
|
55
|
+
readonly completionTokens: number;
|
|
56
|
+
readonly reasoningTokens: number;
|
|
57
|
+
readonly callCount: number;
|
|
58
|
+
readonly cost: Cost | null;
|
|
59
|
+
readonly mixedCurrency: boolean;
|
|
60
|
+
}>;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Budget configuration shape consumed by {@link createCostTracker}.
|
|
64
|
+
*
|
|
65
|
+
* @stable
|
|
66
|
+
*/
|
|
67
|
+
interface CostBudgets {
|
|
68
|
+
/** Per-session budget in the configured currency. */
|
|
69
|
+
readonly perSession?: number;
|
|
70
|
+
/** Per-user budget. */
|
|
71
|
+
readonly perUser?: number;
|
|
72
|
+
/** Per-agent budget. */
|
|
73
|
+
readonly perAgent?: number;
|
|
74
|
+
/** Per-run budget. */
|
|
75
|
+
readonly perRun?: number;
|
|
76
|
+
/** Currency. Defaults to `'USD'`. */
|
|
77
|
+
readonly currency?: string;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Callback invoked when an aggregation scope crosses its configured
|
|
81
|
+
* budget. The handler receives a sanitized payload — the secret-free
|
|
82
|
+
* scope id + the breached numbers.
|
|
83
|
+
*
|
|
84
|
+
* @stable
|
|
85
|
+
*/
|
|
86
|
+
type CostBudgetExceededCallback = (event: {
|
|
87
|
+
readonly scope: CostScope;
|
|
88
|
+
readonly id: string;
|
|
89
|
+
readonly budget: number;
|
|
90
|
+
readonly actual: number;
|
|
91
|
+
readonly currency: string;
|
|
92
|
+
}) => void;
|
|
93
|
+
/**
|
|
94
|
+
* Configuration shape for {@link createCostTracker}.
|
|
95
|
+
*
|
|
96
|
+
* @stable
|
|
97
|
+
*/
|
|
98
|
+
interface CostTrackerOptions {
|
|
99
|
+
readonly budgets?: CostBudgets;
|
|
100
|
+
readonly onExceed?: CostBudgetExceededCallback;
|
|
101
|
+
}
|
|
102
|
+
//#endregion
|
|
103
|
+
export { CostBudgetExceededCallback, CostBudgets, CostRecordInput, CostScope, CostSnapshot, CostTrackerOptions };
|
|
104
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","names":[],"sources":["../../src/cost/types.ts"],"sourcesContent":[],"mappings":";;;;;;;;AAyEA;AAoBA;AAaA;;KA1FY,SAAA;;;;;;UAOK,eAAA;;;;;kBAKC;;;;;;;;;;;;;;;UAgBD,YAAA;;;;;;iBAMA;;;;;;;oBAOG;;;;;;mBAMD;;;;;;;;;UAUF,WAAA;;;;;;;;;;;;;;;;;;;KAoBL,0BAAA;kBACM;;;;;;;;;;;UAYD,kBAAA;qBACI;sBACC"}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { Case, Dataset, EvalCaseResult, EvalReport, RunEvalOptions, ScoreResult, Scorer } from "./types.js";
|
|
2
|
+
import { runEval } from "./runner.js";
|
|
3
|
+
export { type Case, type Dataset, type EvalCaseResult, type EvalReport, type RunEvalOptions, type ScoreResult, type Scorer, runEval };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { EvalReport, RunEvalOptions } from "./types.js";
|
|
2
|
+
|
|
3
|
+
//#region src/eval/runner.d.ts
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @stable
|
|
7
|
+
*/
|
|
8
|
+
declare function runEval<I, O>(opts: RunEvalOptions<I, O>): Promise<EvalReport<I, O>>;
|
|
9
|
+
//#endregion
|
|
10
|
+
export { runEval };
|
|
11
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","names":[],"sources":["../../src/eval/runner.ts"],"sourcesContent":[],"mappings":";;;;;;;iBAwBsB,oBAAoB,eAAe,GAAG,KAAK,QAAQ,WAAW,GAAG"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
//#region src/eval/runner.ts
|
|
2
|
+
/**
|
|
3
|
+
* @stable
|
|
4
|
+
*/
|
|
5
|
+
async function runEval(opts) {
|
|
6
|
+
const iterations = Math.max(1, opts.iterations ?? 1);
|
|
7
|
+
const signal = opts.signal;
|
|
8
|
+
const results = [];
|
|
9
|
+
for (let iter = 0; iter < iterations; iter++) for (let idx = 0; idx < opts.dataset.cases.length; idx++) {
|
|
10
|
+
throwIfAborted(signal);
|
|
11
|
+
const sampleCase = opts.dataset.cases[idx];
|
|
12
|
+
if (sampleCase === void 0) continue;
|
|
13
|
+
const baseId = sampleCase.id ?? `case-${idx}`;
|
|
14
|
+
const caseId = iterations === 1 ? baseId : `${baseId}-iter-${iter}`;
|
|
15
|
+
const startedAt = Date.now();
|
|
16
|
+
const output = await opts.agent.run(sampleCase.input);
|
|
17
|
+
const durationMs = Date.now() - startedAt;
|
|
18
|
+
const scores = [];
|
|
19
|
+
for (const scorer of opts.scorers) {
|
|
20
|
+
throwIfAborted(signal);
|
|
21
|
+
const result = await safeScore(scorer, sampleCase, output, durationMs);
|
|
22
|
+
scores.push({
|
|
23
|
+
scorer: scorer.name,
|
|
24
|
+
result
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
results.push({
|
|
28
|
+
caseId,
|
|
29
|
+
input: sampleCase.input,
|
|
30
|
+
output,
|
|
31
|
+
durationMs,
|
|
32
|
+
scores
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
return summarize(results, opts.scorers);
|
|
36
|
+
}
|
|
37
|
+
async function safeScore(scorer, c, output, durationMs) {
|
|
38
|
+
try {
|
|
39
|
+
return await scorer.score({
|
|
40
|
+
case: c,
|
|
41
|
+
output,
|
|
42
|
+
durationMs
|
|
43
|
+
});
|
|
44
|
+
} catch (err) {
|
|
45
|
+
return {
|
|
46
|
+
pass: false,
|
|
47
|
+
reason: `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function summarize(results, scorers) {
|
|
52
|
+
const total = results.length;
|
|
53
|
+
let passed = 0;
|
|
54
|
+
let failed = 0;
|
|
55
|
+
let durationSum = 0;
|
|
56
|
+
const byScorer = {};
|
|
57
|
+
for (const scorer of scorers) byScorer[scorer.name] = {
|
|
58
|
+
passed: 0,
|
|
59
|
+
failed: 0,
|
|
60
|
+
scoreSum: 0,
|
|
61
|
+
scoreCount: 0
|
|
62
|
+
};
|
|
63
|
+
for (const r of results) {
|
|
64
|
+
durationSum += r.durationMs;
|
|
65
|
+
let passEntire = true;
|
|
66
|
+
for (const { scorer, result } of r.scores) {
|
|
67
|
+
const bucket = byScorer[scorer] ?? {
|
|
68
|
+
passed: 0,
|
|
69
|
+
failed: 0,
|
|
70
|
+
scoreSum: 0,
|
|
71
|
+
scoreCount: 0
|
|
72
|
+
};
|
|
73
|
+
if (result.pass) bucket.passed += 1;
|
|
74
|
+
else {
|
|
75
|
+
bucket.failed += 1;
|
|
76
|
+
passEntire = false;
|
|
77
|
+
}
|
|
78
|
+
if (typeof result.score === "number" && Number.isFinite(result.score)) {
|
|
79
|
+
bucket.scoreSum += result.score;
|
|
80
|
+
bucket.scoreCount += 1;
|
|
81
|
+
}
|
|
82
|
+
byScorer[scorer] = bucket;
|
|
83
|
+
}
|
|
84
|
+
if (passEntire) passed += 1;
|
|
85
|
+
else failed += 1;
|
|
86
|
+
}
|
|
87
|
+
const summaryByScorer = {};
|
|
88
|
+
for (const [scorer, b] of Object.entries(byScorer)) summaryByScorer[scorer] = {
|
|
89
|
+
passed: b.passed,
|
|
90
|
+
failed: b.failed,
|
|
91
|
+
avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount
|
|
92
|
+
};
|
|
93
|
+
return {
|
|
94
|
+
results,
|
|
95
|
+
summary: {
|
|
96
|
+
total,
|
|
97
|
+
passed,
|
|
98
|
+
failed,
|
|
99
|
+
avgDurationMs: total === 0 ? 0 : durationSum / total,
|
|
100
|
+
byScorer: Object.freeze(summaryByScorer)
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function throwIfAborted(signal) {
|
|
105
|
+
if (signal?.aborted === true) {
|
|
106
|
+
const reason = signal.reason;
|
|
107
|
+
throw reason instanceof Error ? reason : /* @__PURE__ */ new Error("Eval run aborted");
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
//#endregion
|
|
112
|
+
export { runEval };
|
|
113
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","names":["results: EvalCaseResult<I, O>[]","scores: EvalCaseResult<I, O>['scores'][number][]","byScorer: Record<\n string,\n { passed: number; failed: number; scoreSum: number; scoreCount: number }\n >","summaryByScorer: Record<\n string,\n { passed: number; failed: number; avgScore: number | null }\n >"],"sources":["../../src/eval/runner.ts"],"sourcesContent":["/**\n * `runEval(...)` — minimal inline eval runner. Walks every case in\n * the dataset, calls `agent.run(case.input)`, applies every supplied\n * scorer, and returns an aggregated {@link EvalReport}.\n *\n * The runner is deliberately tiny — it has no parallelism, no\n * retries, no reporters, no dataset loaders. Production scenarios\n * that need any of those should `pnpm add @graphorin/evals` (post-MVP).\n *\n * @packageDocumentation\n */\n\nimport type {\n Case,\n EvalCaseResult,\n EvalReport,\n RunEvalOptions,\n ScoreResult,\n Scorer,\n} from './types.js';\n\n/**\n * @stable\n */\nexport async function runEval<I, O>(opts: RunEvalOptions<I, O>): Promise<EvalReport<I, O>> {\n const iterations = Math.max(1, opts.iterations ?? 1);\n const signal = opts.signal;\n const results: EvalCaseResult<I, O>[] = [];\n\n for (let iter = 0; iter < iterations; iter++) {\n for (let idx = 0; idx < opts.dataset.cases.length; idx++) {\n throwIfAborted(signal);\n const sampleCase = opts.dataset.cases[idx];\n if (sampleCase === undefined) continue;\n // EB-6: disambiguate a caller-provided id per iteration too, not just the\n // synthetic fallback — otherwise iterations>1 emits duplicate caseIds.\n const baseId = sampleCase.id ?? `case-${idx}`;\n const caseId = iterations === 1 ? baseId : `${baseId}-iter-${iter}`;\n\n const startedAt = Date.now();\n const output = await opts.agent.run(sampleCase.input);\n const durationMs = Date.now() - startedAt;\n\n const scores: EvalCaseResult<I, O>['scores'][number][] = [];\n for (const scorer of opts.scorers) {\n throwIfAborted(signal);\n const result = await safeScore(scorer, sampleCase, output, durationMs);\n scores.push({ scorer: scorer.name, result });\n }\n\n results.push({\n caseId,\n input: sampleCase.input,\n output,\n durationMs,\n scores,\n });\n }\n }\n\n return summarize(results, opts.scorers);\n}\n\nasync function safeScore<I, O>(\n scorer: Scorer<I, O>,\n c: Case<I, O>,\n output: O,\n durationMs: number,\n): Promise<ScoreResult> {\n try {\n return await scorer.score({ case: c, output, durationMs });\n } catch (err) {\n return {\n pass: false,\n reason: `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n };\n }\n}\n\nfunction summarize<I, O>(\n results: ReadonlyArray<EvalCaseResult<I, O>>,\n scorers: ReadonlyArray<Scorer<I, O>>,\n): EvalReport<I, O> {\n const total = results.length;\n let passed = 0;\n let failed = 0;\n let durationSum = 0;\n const byScorer: Record<\n string,\n { passed: number; failed: number; scoreSum: number; scoreCount: number }\n > = {};\n\n for (const scorer of scorers) {\n byScorer[scorer.name] = { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };\n }\n\n for (const r of results) {\n durationSum += r.durationMs;\n let passEntire = true;\n for (const { scorer, result } of r.scores) {\n const bucket = byScorer[scorer] ?? { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };\n if (result.pass) bucket.passed += 1;\n else {\n bucket.failed += 1;\n passEntire = false;\n }\n if (typeof result.score === 'number' && Number.isFinite(result.score)) {\n bucket.scoreSum += result.score;\n bucket.scoreCount += 1;\n }\n byScorer[scorer] = bucket;\n }\n if (passEntire) passed += 1;\n else failed += 1;\n }\n\n const summaryByScorer: Record<\n string,\n { passed: number; failed: number; avgScore: number | null }\n > = {};\n for (const [scorer, b] of Object.entries(byScorer)) {\n summaryByScorer[scorer] = {\n passed: b.passed,\n failed: b.failed,\n avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount,\n };\n }\n\n return {\n results,\n summary: {\n total,\n passed,\n failed,\n avgDurationMs: total === 0 ? 0 : durationSum / total,\n byScorer: Object.freeze(summaryByScorer),\n },\n };\n}\n\nfunction throwIfAborted(signal: AbortSignal | undefined): void {\n if (signal?.aborted === true) {\n const reason = signal.reason;\n throw reason instanceof Error ? reason : new Error('Eval run aborted');\n }\n}\n"],"mappings":";;;;AAwBA,eAAsB,QAAc,MAAuD;CACzF,MAAM,aAAa,KAAK,IAAI,GAAG,KAAK,cAAc,EAAE;CACpD,MAAM,SAAS,KAAK;CACpB,MAAMA,UAAkC,EAAE;AAE1C,MAAK,IAAI,OAAO,GAAG,OAAO,YAAY,OACpC,MAAK,IAAI,MAAM,GAAG,MAAM,KAAK,QAAQ,MAAM,QAAQ,OAAO;AACxD,iBAAe,OAAO;EACtB,MAAM,aAAa,KAAK,QAAQ,MAAM;AACtC,MAAI,eAAe,OAAW;EAG9B,MAAM,SAAS,WAAW,MAAM,QAAQ;EACxC,MAAM,SAAS,eAAe,IAAI,SAAS,GAAG,OAAO,QAAQ;EAE7D,MAAM,YAAY,KAAK,KAAK;EAC5B,MAAM,SAAS,MAAM,KAAK,MAAM,IAAI,WAAW,MAAM;EACrD,MAAM,aAAa,KAAK,KAAK,GAAG;EAEhC,MAAMC,SAAmD,EAAE;AAC3D,OAAK,MAAM,UAAU,KAAK,SAAS;AACjC,kBAAe,OAAO;GACtB,MAAM,SAAS,MAAM,UAAU,QAAQ,YAAY,QAAQ,WAAW;AACtE,UAAO,KAAK;IAAE,QAAQ,OAAO;IAAM;IAAQ,CAAC;;AAG9C,UAAQ,KAAK;GACX;GACA,OAAO,WAAW;GAClB;GACA;GACA;GACD,CAAC;;AAIN,QAAO,UAAU,SAAS,KAAK,QAAQ;;AAGzC,eAAe,UACb,QACA,GACA,QACA,YACsB;AACtB,KAAI;AACF,SAAO,MAAM,OAAO,MAAM;GAAE,MAAM;GAAG;GAAQ;GAAY,CAAC;UACnD,KAAK;AACZ,SAAO;GACL,MAAM;GACN,QAAQ,WAAW,OAAO,KAAK,WAAW,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GAC3F;;;AAIL,SAAS,UACP,SACA,SACkB;CAClB,MAAM,QAAQ,QAAQ;CACtB,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,cAAc;CAClB,MAAMC,WAGF,EAAE;AAEN,MAAK,MAAM,UAAU,QACnB,UAAS,OAAO,QAAQ;EAAE,QAAQ;EAAG,QAAQ;EAAG,UAAU;EAAG,YAAY;EAAG;AAG9E,MAAK,MAAM,KAAK,SAAS;AACvB,iBAAe,EAAE;EACjB,IAAI,aAAa;AACjB,OAAK,MAAM,EAAE,QAAQ,YAAY,EAAE,QAAQ;GACzC,MAAM,SAAS,SAAS,WAAW;IAAE,QAAQ;IAAG,QAAQ;IAAG,UAAU;IAAG,YAAY;IAAG;AACvF,OAAI,OAAO,KAAM,QAAO,UAAU;QAC7B;AACH,WAAO,UAAU;AACjB,iBAAa;;AAEf,OAAI,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,OAAO,MAAM,EAAE;AACrE,WAAO,YAAY,OAAO;AAC1B,WAAO,cAAc;;AAEvB,YAAS,UAAU;;AAErB,MAAI,WAAY,WAAU;MACrB,WAAU;;CAGjB,MAAMC,kBAGF,EAAE;AACN,MAAK,MAAM,CAAC,QAAQ,MAAM,OAAO,QAAQ,SAAS,CAChD,iBAAgB,UAAU;EACxB,QAAQ,EAAE;EACV,QAAQ,EAAE;EACV,UAAU,EAAE,eAAe,IAAI,OAAO,EAAE,WAAW,EAAE;EACtD;AAGH,QAAO;EACL;EACA,SAAS;GACP;GACA;GACA;GACA,eAAe,UAAU,IAAI,IAAI,cAAc;GAC/C,UAAU,OAAO,OAAO,gBAAgB;GACzC;EACF;;AAGH,SAAS,eAAe,QAAuC;AAC7D,KAAI,QAAQ,YAAY,MAAM;EAC5B,MAAM,SAAS,OAAO;AACtB,QAAM,kBAAkB,QAAQ,yBAAS,IAAI,MAAM,mBAAmB"}
|