@graphorin/observability 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/LICENSE +21 -0
  3. package/README.md +115 -0
  4. package/dist/cost/cost-tracker.d.ts +28 -0
  5. package/dist/cost/cost-tracker.d.ts.map +1 -0
  6. package/dist/cost/cost-tracker.js +169 -0
  7. package/dist/cost/cost-tracker.js.map +1 -0
  8. package/dist/cost/index.d.ts +3 -0
  9. package/dist/cost/index.js +3 -0
  10. package/dist/cost/types.d.ts +104 -0
  11. package/dist/cost/types.d.ts.map +1 -0
  12. package/dist/eval/index.d.ts +3 -0
  13. package/dist/eval/index.js +3 -0
  14. package/dist/eval/runner.d.ts +11 -0
  15. package/dist/eval/runner.d.ts.map +1 -0
  16. package/dist/eval/runner.js +113 -0
  17. package/dist/eval/runner.js.map +1 -0
  18. package/dist/eval/types.d.ts +108 -0
  19. package/dist/eval/types.d.ts.map +1 -0
  20. package/dist/exporters/console.d.ts +27 -0
  21. package/dist/exporters/console.d.ts.map +1 -0
  22. package/dist/exporters/console.js +47 -0
  23. package/dist/exporters/console.js.map +1 -0
  24. package/dist/exporters/index.d.ts +6 -0
  25. package/dist/exporters/index.js +7 -0
  26. package/dist/exporters/jsonl.d.ts +57 -0
  27. package/dist/exporters/jsonl.d.ts.map +1 -0
  28. package/dist/exporters/jsonl.js +95 -0
  29. package/dist/exporters/jsonl.js.map +1 -0
  30. package/dist/exporters/otlp-http.d.ts +38 -0
  31. package/dist/exporters/otlp-http.d.ts.map +1 -0
  32. package/dist/exporters/otlp-http.js +115 -0
  33. package/dist/exporters/otlp-http.js.map +1 -0
  34. package/dist/exporters/types.d.ts +72 -0
  35. package/dist/exporters/types.d.ts.map +1 -0
  36. package/dist/exporters/types.js +13 -0
  37. package/dist/exporters/types.js.map +1 -0
  38. package/dist/exporters/with-validation.d.ts +45 -0
  39. package/dist/exporters/with-validation.d.ts.map +1 -0
  40. package/dist/exporters/with-validation.js +99 -0
  41. package/dist/exporters/with-validation.js.map +1 -0
  42. package/dist/gen-ai/emit.d.ts +27 -0
  43. package/dist/gen-ai/emit.d.ts.map +1 -0
  44. package/dist/gen-ai/emit.js +88 -0
  45. package/dist/gen-ai/emit.js.map +1 -0
  46. package/dist/gen-ai/index.d.ts +5 -0
  47. package/dist/gen-ai/index.js +5 -0
  48. package/dist/gen-ai/operation-mapping.d.ts +24 -0
  49. package/dist/gen-ai/operation-mapping.d.ts.map +1 -0
  50. package/dist/gen-ai/operation-mapping.js +67 -0
  51. package/dist/gen-ai/operation-mapping.js.map +1 -0
  52. package/dist/gen-ai/system-derivation.d.ts +41 -0
  53. package/dist/gen-ai/system-derivation.d.ts.map +1 -0
  54. package/dist/gen-ai/system-derivation.js +67 -0
  55. package/dist/gen-ai/system-derivation.js.map +1 -0
  56. package/dist/gen-ai/types.d.ts +87 -0
  57. package/dist/gen-ai/types.d.ts.map +1 -0
  58. package/dist/index.d.ts +58 -0
  59. package/dist/index.d.ts.map +1 -0
  60. package/dist/index.js +60 -0
  61. package/dist/index.js.map +1 -0
  62. package/dist/logger/index.d.ts +2 -0
  63. package/dist/logger/index.js +3 -0
  64. package/dist/logger/logger.d.ts +53 -0
  65. package/dist/logger/logger.d.ts.map +1 -0
  66. package/dist/logger/logger.js +145 -0
  67. package/dist/logger/logger.js.map +1 -0
  68. package/dist/openinference/index.d.ts +45 -0
  69. package/dist/openinference/index.d.ts.map +1 -0
  70. package/dist/openinference/index.js +92 -0
  71. package/dist/openinference/index.js.map +1 -0
  72. package/dist/redaction/config.d.ts +39 -0
  73. package/dist/redaction/config.d.ts.map +1 -0
  74. package/dist/redaction/config.js +15 -0
  75. package/dist/redaction/config.js.map +1 -0
  76. package/dist/redaction/errors.d.ts +36 -0
  77. package/dist/redaction/errors.d.ts.map +1 -0
  78. package/dist/redaction/errors.js +42 -0
  79. package/dist/redaction/errors.js.map +1 -0
  80. package/dist/redaction/imperative-patterns.d.ts +108 -0
  81. package/dist/redaction/imperative-patterns.d.ts.map +1 -0
  82. package/dist/redaction/imperative-patterns.js +187 -0
  83. package/dist/redaction/imperative-patterns.js.map +1 -0
  84. package/dist/redaction/index.d.ts +7 -0
  85. package/dist/redaction/index.js +7 -0
  86. package/dist/redaction/patterns.d.ts +85 -0
  87. package/dist/redaction/patterns.d.ts.map +1 -0
  88. package/dist/redaction/patterns.js +172 -0
  89. package/dist/redaction/patterns.js.map +1 -0
  90. package/dist/redaction/types.d.ts +103 -0
  91. package/dist/redaction/types.d.ts.map +1 -0
  92. package/dist/redaction/validator.d.ts +26 -0
  93. package/dist/redaction/validator.d.ts.map +1 -0
  94. package/dist/redaction/validator.js +195 -0
  95. package/dist/redaction/validator.js.map +1 -0
  96. package/dist/replay/config.d.ts +56 -0
  97. package/dist/replay/config.d.ts.map +1 -0
  98. package/dist/replay/config.js +18 -0
  99. package/dist/replay/config.js.map +1 -0
  100. package/dist/replay/index.d.ts +5 -0
  101. package/dist/replay/index.js +5 -0
  102. package/dist/replay/log.d.ts +39 -0
  103. package/dist/replay/log.d.ts.map +1 -0
  104. package/dist/replay/log.js +59 -0
  105. package/dist/replay/log.js.map +1 -0
  106. package/dist/replay/replay.d.ts +25 -0
  107. package/dist/replay/replay.d.ts.map +1 -0
  108. package/dist/replay/replay.js +173 -0
  109. package/dist/replay/replay.js.map +1 -0
  110. package/dist/replay/types.d.ts +110 -0
  111. package/dist/replay/types.d.ts.map +1 -0
  112. package/dist/telemetry/index.d.ts +63 -0
  113. package/dist/telemetry/index.d.ts.map +1 -0
  114. package/dist/telemetry/index.js +49 -0
  115. package/dist/telemetry/index.js.map +1 -0
  116. package/dist/tracer/ids.d.ts +23 -0
  117. package/dist/tracer/ids.d.ts.map +1 -0
  118. package/dist/tracer/ids.js +30 -0
  119. package/dist/tracer/ids.js.map +1 -0
  120. package/dist/tracer/index.d.ts +6 -0
  121. package/dist/tracer/index.js +6 -0
  122. package/dist/tracer/sampling.d.ts +77 -0
  123. package/dist/tracer/sampling.d.ts.map +1 -0
  124. package/dist/tracer/sampling.js +67 -0
  125. package/dist/tracer/sampling.js.map +1 -0
  126. package/dist/tracer/span-names.d.ts +11 -0
  127. package/dist/tracer/span-names.d.ts.map +1 -0
  128. package/dist/tracer/span-names.js +11 -0
  129. package/dist/tracer/span-names.js.map +1 -0
  130. package/dist/tracer/span.d.ts +27 -0
  131. package/dist/tracer/span.d.ts.map +1 -0
  132. package/dist/tracer/span.js +80 -0
  133. package/dist/tracer/span.js.map +1 -0
  134. package/dist/tracer/tracer.d.ts +101 -0
  135. package/dist/tracer/tracer.d.ts.map +1 -0
  136. package/dist/tracer/tracer.js +150 -0
  137. package/dist/tracer/tracer.js.map +1 -0
  138. package/package.json +138 -0
package/CHANGELOG.md ADDED
@@ -0,0 +1,7 @@
1
+ # @graphorin/observability
2
+
3
+ ## 0.1.0
4
+
5
+ Initial release. See the workspace root `CHANGELOG.md` for the full
6
+ release notes; the per-package changelog is generated by Changesets
7
+ and tracks subsequent updates.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Oleksiy Stepurenko
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,115 @@
1
+ # @graphorin/observability
2
+
3
+ > Observability primitives for the [Graphorin](https://github.com/o-stepper/graphorin) framework.
4
+
5
+ `@graphorin/observability` ships every cross-cutting tracing,
6
+ redaction, replay, cost, and structured-logging primitive every other
7
+ `@graphorin/*` package builds on. The package is intentionally
8
+ opinionated: every exporter must be wrapped through
9
+ `withValidation(...)` before it can ship a single span, the validator
10
+ defaults to **default-deny non-public**, and the framework makes zero
11
+ outbound network calls without an explicit user action.
12
+
13
+ ## Highlights
14
+
15
+ - **Typed `AISpan<T>` tracer.** `createTracer({...})` returns a
16
+ `GraphorinTracer` that emits `AISpan<SpanType>` records compatible
17
+ with OpenTelemetry. Sampling rules + per-event sampling cover noisy
18
+ span kinds (`memory.embed`, `tool.execute.partial`, …).
19
+ - **Mandatory `withValidation(...)` wrapper.** Every exporter passed to
20
+ `createTracer({ exporters })` is wrapped through the configured
21
+ `RedactionValidator`. Registering a raw exporter while
22
+ `validation: 'off'` triggers `UnvalidatedExporterError` at startup —
23
+ there is no silent path.
24
+ - **`RedactionValidator` with 14 built-in patterns.** API key / JWT /
25
+ PEM private key / GitHub PAT / AWS access key / Graphorin token /
26
+ bearer header / basic-auth header / email / credit card / US SSN /
27
+ E.164 phone / IBAN — all on by default. Three additional patterns
28
+ (IPv4, IPv6, GCP service account) are opt-in.
29
+ - **OpenTelemetry GenAI semantic-conventions conformance.**
30
+ `emitGenAIAttributes(span, {...})`, `emitGenAIMessageEvents(span, [...])`,
31
+ and `deriveGenAISystem(...)` ship the canonical
32
+ `gen_ai.*` attribute family alongside the existing
33
+ `graphorin.*` attributes — additive, never replacing.
34
+ - **OpenInference span-kind layer.** `emitOpenInferenceKind(span)` emits
35
+ the `openinference.span.kind` attribute via the canonical
36
+ per-`SpanType` mapping (`agent.*` → `AGENT`,
37
+ `provider.*` → `LLM`, `tool.execute` → `TOOL`,
38
+ `memory.*` → `RETRIEVER` / `EMBEDDING`, `workflow.*` → `CHAIN`,
39
+ `agent.evaluator.iteration` → `EVALUATOR`).
40
+ - **`ConsoleExporter`, `JSONLExporter`, `OTLPHttpExporter`.** The
41
+ built-in exporters cover dev (console), replay (append-only JSONL
42
+ with `0700` directories + `0600` files), and remote OTLP collectors
43
+ (`fetch`-based reference implementation, swappable `fetchImpl`).
44
+ - **Sanitized-by-default `Replay`.** `createReplay({...})` exposes a
45
+ `run(...)` async iterator that yields `replay.start` /
46
+ `replay.event` / `replay.skipped` / `replay.end` markers. Raw mode
47
+ requires the `canReadRaw` callback to return `true` and emits an
48
+ audit-bridge entry on every invocation.
49
+ - **Hierarchical `CostTracker`.** `createCostTracker({...})` rolls up
50
+ tokens + cost across parent-child spans and supports per-run /
51
+ per-session / per-agent / per-user budgets with an `onExceed`
52
+ callback.
53
+ - **Structured logger.** `createLogger({...})` writes JSON or pretty
54
+ records, automatically correlates with the current span via
55
+ `withCurrentSpan(...)`, and pipes every field through the validator.
56
+ - **Zero-default telemetry.** `getTelemetryStatus()` and
57
+ `announceTelemetryPosture()` expose the v0.1 promise: the framework
58
+ performs zero outbound network calls, no version pings, no crash
59
+ reports, no auto-updates. The `GRAPHORIN_TELEMETRY` and
60
+ `GRAPHORIN_NO_PHONE_HOME` environment variables are reserved for
61
+ forward compatibility and are acknowledged at startup.
62
+
63
+ ## Installation
64
+
65
+ ```bash
66
+ pnpm add @graphorin/observability
67
+ # Optional peer deps for OTLP export — install only when you need them:
68
+ pnpm add @opentelemetry/api @opentelemetry/sdk-node @opentelemetry/exporter-trace-otlp-http
69
+ ```
70
+
71
+ ## Quick start
72
+
73
+ ```ts
74
+ import {
75
+ createTracer,
76
+ createConsoleExporter,
77
+ createJSONLExporter,
78
+ withValidation,
79
+ } from '@graphorin/observability';
80
+
81
+ const tracer = createTracer({
82
+ serviceName: 'my-assistant',
83
+ exporters: [
84
+ // Auto-wrapped via the tracer-managed validator.
85
+ createConsoleExporter({ pretty: true }),
86
+ // Manually wrapped — useful when each exporter needs its own policy.
87
+ withValidation(createJSONLExporter({ path: './traces' }), {
88
+ minTier: 'internal',
89
+ }),
90
+ ],
91
+ validation: { minTier: 'public', failOnUnredactedSensitive: false },
92
+ sampling: {
93
+ rate: 1.0,
94
+ rules: [{ type: 'memory.embed', rate: 0.1 }],
95
+ },
96
+ });
97
+
98
+ await tracer.span(
99
+ { type: 'agent.run', attrs: { 'graphorin.agent.id': 'support-bot' } },
100
+ async (span) => {
101
+ span.setAttributes({ 'graphorin.session.id': 'session-123' });
102
+ return runAgent();
103
+ },
104
+ );
105
+
106
+ await tracer.shutdown();
107
+ ```
108
+
109
+ ## License
110
+
111
+ MIT © 2026 [Oleksiy Stepurenko](https://github.com/o-stepper).
112
+
113
+ ---
114
+
115
+ **Project Graphorin** · v0.5.0 · MIT License · © 2026 Oleksiy Stepurenko · <https://github.com/o-stepper/graphorin>
@@ -0,0 +1,28 @@
1
+ import { CostRecordInput, CostScope, CostSnapshot, CostTrackerOptions } from "./types.js";
2
+
3
+ //#region src/cost/cost-tracker.d.ts
4
+
5
+ /**
6
+ * @stable
7
+ */
8
+ interface CostTracker {
9
+ /** Record a single LLM-call usage / cost figure. */
10
+ record(input: CostRecordInput): void;
11
+ /** Snapshot for a given scope id. Returns zero figures when unknown. */
12
+ usage(scope: CostScope, id: string): CostSnapshot;
13
+ /** Snapshot for a single span id (carries nested attributions). */
14
+ usageForSpan(spanId: string): CostSnapshot;
15
+ /** Reset every counter back to zero. */
16
+ reset(): void;
17
+ /** Subscribe to per-scope rollup notifications. Returns an unsubscribe. */
18
+ onRollup(listener: (input: CostRecordInput) => void): () => void;
19
+ }
20
+ /**
21
+ * Build a {@link CostTracker} configured with the supplied budgets.
22
+ *
23
+ * @stable
24
+ */
25
+ declare function createCostTracker(opts?: CostTrackerOptions): CostTracker;
26
+ //#endregion
27
+ export { CostTracker, createCostTracker };
28
+ //# sourceMappingURL=cost-tracker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cost-tracker.d.ts","names":[],"sources":["../../src/cost/cost-tracker.ts"],"sourcesContent":[],"mappings":";;;;;;;UA2BiB,WAAA;;gBAED;;eAED,wBAAwB;;gCAEP;;;;6BAIH;;;;;;;iBAmGb,iBAAA,QAAwB,qBAA0B"}
@@ -0,0 +1,169 @@
1
+ //#region src/cost/cost-tracker.ts
2
+ function freshBucket() {
3
+ return {
4
+ promptTokens: 0,
5
+ completionTokens: 0,
6
+ reasoningTokens: 0,
7
+ callCount: 0,
8
+ costAmount: 0,
9
+ costCurrency: null,
10
+ mixedCurrency: false,
11
+ byModel: /* @__PURE__ */ new Map(),
12
+ exceeded: false
13
+ };
14
+ }
15
+ function addCost(bucket, cost) {
16
+ if (bucket.costCurrency !== null && bucket.costCurrency !== cost.currency) bucket.mixedCurrency = true;
17
+ else if (bucket.costCurrency === null) bucket.costCurrency = cost.currency;
18
+ bucket.costAmount += cost.amount;
19
+ }
20
+ function combine(target, input) {
21
+ target.promptTokens += input.promptTokens;
22
+ target.completionTokens += input.completionTokens;
23
+ target.reasoningTokens += input.reasoningTokens ?? 0;
24
+ target.callCount += 1;
25
+ if (input.cost !== void 0) addCost(target, input.cost);
26
+ const modelBucket = target.byModel.get(input.model) ?? freshBucket();
27
+ modelBucket.promptTokens += input.promptTokens;
28
+ modelBucket.completionTokens += input.completionTokens;
29
+ modelBucket.reasoningTokens += input.reasoningTokens ?? 0;
30
+ modelBucket.callCount += 1;
31
+ if (input.cost !== void 0) addCost(modelBucket, input.cost);
32
+ target.byModel.set(input.model, modelBucket);
33
+ }
34
+ function snapshotOf(bucket) {
35
+ const cost = bucket.costCurrency === null ? null : {
36
+ amount: bucket.costAmount,
37
+ currency: bucket.costCurrency
38
+ };
39
+ return {
40
+ promptTokens: bucket.promptTokens,
41
+ completionTokens: bucket.completionTokens,
42
+ reasoningTokens: bucket.reasoningTokens,
43
+ totalTokens: bucket.promptTokens + bucket.completionTokens + bucket.reasoningTokens,
44
+ callCount: bucket.callCount,
45
+ cost,
46
+ mixedCurrency: bucket.mixedCurrency,
47
+ byModel: [...bucket.byModel.entries()].map(([model, b]) => ({
48
+ model,
49
+ promptTokens: b.promptTokens,
50
+ completionTokens: b.completionTokens,
51
+ reasoningTokens: b.reasoningTokens,
52
+ callCount: b.callCount,
53
+ cost: b.costCurrency === null ? null : {
54
+ amount: b.costAmount,
55
+ currency: b.costCurrency
56
+ },
57
+ mixedCurrency: b.mixedCurrency
58
+ }))
59
+ };
60
+ }
61
+ const ZERO = Object.freeze({
62
+ promptTokens: 0,
63
+ completionTokens: 0,
64
+ reasoningTokens: 0,
65
+ totalTokens: 0,
66
+ callCount: 0,
67
+ cost: null,
68
+ mixedCurrency: false,
69
+ byModel: []
70
+ });
71
+ /**
72
+ * Build a {@link CostTracker} configured with the supplied budgets.
73
+ *
74
+ * @stable
75
+ */
76
+ function createCostTracker(opts = {}) {
77
+ const budgets = opts.budgets ?? {};
78
+ const currency = budgets.currency ?? "USD";
79
+ const onExceed = opts.onExceed;
80
+ const bySpan = /* @__PURE__ */ new Map();
81
+ const byScope = {
82
+ run: /* @__PURE__ */ new Map(),
83
+ session: /* @__PURE__ */ new Map(),
84
+ agent: /* @__PURE__ */ new Map(),
85
+ user: /* @__PURE__ */ new Map()
86
+ };
87
+ const parents = /* @__PURE__ */ new Map();
88
+ const listeners = /* @__PURE__ */ new Set();
89
+ function bump(scope, id, input) {
90
+ if (id === void 0) return;
91
+ const map = byScope[scope];
92
+ const bucket = map.get(id) ?? freshBucket();
93
+ combine(bucket, input);
94
+ map.set(id, bucket);
95
+ enforceBudget(scope, id, bucket, onExceed);
96
+ }
97
+ function enforceBudget(scope, id, bucket, cb) {
98
+ const limit = pickBudget(budgets, scope);
99
+ if (limit === void 0) return;
100
+ if (bucket.exceeded) return;
101
+ if (bucket.costAmount > limit) {
102
+ bucket.exceeded = true;
103
+ cb?.({
104
+ scope,
105
+ id,
106
+ budget: limit,
107
+ actual: bucket.costAmount,
108
+ currency: bucket.costCurrency ?? currency
109
+ });
110
+ }
111
+ }
112
+ return {
113
+ record(input) {
114
+ const spanBucket = bySpan.get(input.spanId) ?? freshBucket();
115
+ combine(spanBucket, input);
116
+ bySpan.set(input.spanId, spanBucket);
117
+ if (input.parentSpanId !== void 0) parents.set(input.spanId, input.parentSpanId);
118
+ const seen = /* @__PURE__ */ new Set();
119
+ let ancestor = parents.get(input.spanId);
120
+ while (ancestor !== void 0 && !seen.has(ancestor)) {
121
+ seen.add(ancestor);
122
+ const ancestorBucket = bySpan.get(ancestor) ?? freshBucket();
123
+ combine(ancestorBucket, input);
124
+ bySpan.set(ancestor, ancestorBucket);
125
+ ancestor = parents.get(ancestor);
126
+ }
127
+ bump("run", input.runId, input);
128
+ bump("session", input.sessionId, input);
129
+ bump("agent", input.agentId, input);
130
+ bump("user", input.userId, input);
131
+ for (const listener of listeners) try {
132
+ listener(input);
133
+ } catch {}
134
+ },
135
+ usage(scope, id) {
136
+ const bucket = byScope[scope].get(id);
137
+ if (bucket === void 0) return ZERO;
138
+ return snapshotOf(bucket);
139
+ },
140
+ usageForSpan(spanId) {
141
+ const bucket = bySpan.get(spanId);
142
+ if (bucket === void 0) return ZERO;
143
+ return snapshotOf(bucket);
144
+ },
145
+ reset() {
146
+ bySpan.clear();
147
+ for (const map of Object.values(byScope)) map.clear();
148
+ parents.clear();
149
+ },
150
+ onRollup(listener) {
151
+ listeners.add(listener);
152
+ return () => {
153
+ listeners.delete(listener);
154
+ };
155
+ }
156
+ };
157
+ }
158
+ function pickBudget(budgets, scope) {
159
+ switch (scope) {
160
+ case "session": return budgets.perSession;
161
+ case "user": return budgets.perUser;
162
+ case "agent": return budgets.perAgent;
163
+ case "run": return budgets.perRun;
164
+ }
165
+ }
166
+
167
+ //#endregion
168
+ export { createCostTracker };
169
+ //# sourceMappingURL=cost-tracker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cost-tracker.js","names":["cost: Cost | null","ZERO: CostSnapshot","budgets: CostBudgets","byScope: Record<CostScope, Map<string, AggregateBucket>>"],"sources":["../../src/cost/cost-tracker.ts"],"sourcesContent":["/**\n * `createCostTracker(...)` — aggregate token + cost figures across\n * spans and provide per-scope queries.\n *\n * The tracker preserves parent-child relationships: when\n * `record({ parentSpanId })` is supplied, the recorded usage is also\n * attributed to the ancestor span via the configured spans-by-id\n * lookup. Aggregations across runs / sessions / agents / users are\n * computed on demand.\n *\n * @packageDocumentation\n */\n\nimport type { Cost } from '@graphorin/core';\n\nimport type {\n CostBudgetExceededCallback,\n CostBudgets,\n CostRecordInput,\n CostScope,\n CostSnapshot,\n CostTrackerOptions,\n} from './types.js';\n\n/**\n * @stable\n */\nexport interface CostTracker {\n /** Record a single LLM-call usage / cost figure. */\n record(input: CostRecordInput): void;\n /** Snapshot for a given scope id. Returns zero figures when unknown. */\n usage(scope: CostScope, id: string): CostSnapshot;\n /** Snapshot for a single span id (carries nested attributions). */\n usageForSpan(spanId: string): CostSnapshot;\n /** Reset every counter back to zero. */\n reset(): void;\n /** Subscribe to per-scope rollup notifications. Returns an unsubscribe. */\n onRollup(listener: (input: CostRecordInput) => void): () => void;\n}\n\ninterface AggregateBucket {\n promptTokens: number;\n completionTokens: number;\n reasoningTokens: number;\n callCount: number;\n costAmount: number;\n costCurrency: string | null;\n /** RP-22: set when records carrying differing currencies are aggregated. */\n mixedCurrency: boolean;\n byModel: Map<string, AggregateBucket>;\n exceeded: boolean;\n}\n\nfunction freshBucket(): AggregateBucket {\n return {\n promptTokens: 0,\n completionTokens: 0,\n reasoningTokens: 0,\n callCount: 0,\n costAmount: 0,\n costCurrency: null,\n mixedCurrency: false,\n byModel: new Map(),\n exceeded: false,\n };\n}\n\n// RP-22: accumulate a cost without silently overwriting the currency. The\n// first currency seen is kept; a record in a different currency flags the\n// bucket as `mixedCurrency` so a USD + EUR total is never reported as one\n// clean figure.\nfunction addCost(bucket: AggregateBucket, cost: Cost): void {\n if (bucket.costCurrency !== null && bucket.costCurrency !== cost.currency) {\n bucket.mixedCurrency = true;\n } else if (bucket.costCurrency === null) {\n bucket.costCurrency = cost.currency;\n }\n bucket.costAmount += cost.amount;\n}\n\nfunction combine(target: AggregateBucket, input: CostRecordInput): void {\n target.promptTokens += input.promptTokens;\n target.completionTokens += input.completionTokens;\n target.reasoningTokens += input.reasoningTokens ?? 0;\n target.callCount += 1;\n if (input.cost !== undefined) addCost(target, input.cost);\n const modelBucket = target.byModel.get(input.model) ?? freshBucket();\n modelBucket.promptTokens += input.promptTokens;\n modelBucket.completionTokens += input.completionTokens;\n modelBucket.reasoningTokens += input.reasoningTokens ?? 0;\n modelBucket.callCount += 1;\n if (input.cost !== undefined) addCost(modelBucket, input.cost);\n target.byModel.set(input.model, modelBucket);\n}\n\nfunction snapshotOf(bucket: AggregateBucket): CostSnapshot {\n const cost: Cost | null =\n bucket.costCurrency === null\n ? null\n : { amount: bucket.costAmount, currency: bucket.costCurrency };\n return {\n promptTokens: bucket.promptTokens,\n completionTokens: bucket.completionTokens,\n reasoningTokens: bucket.reasoningTokens,\n totalTokens: bucket.promptTokens + bucket.completionTokens + bucket.reasoningTokens,\n callCount: bucket.callCount,\n cost,\n mixedCurrency: bucket.mixedCurrency,\n byModel: [...bucket.byModel.entries()].map(([model, b]) => ({\n model,\n promptTokens: b.promptTokens,\n completionTokens: b.completionTokens,\n reasoningTokens: b.reasoningTokens,\n callCount: b.callCount,\n cost: b.costCurrency === null ? null : { amount: b.costAmount, currency: b.costCurrency },\n mixedCurrency: b.mixedCurrency,\n })),\n };\n}\n\nconst ZERO: CostSnapshot = Object.freeze({\n promptTokens: 0,\n completionTokens: 0,\n reasoningTokens: 0,\n totalTokens: 0,\n callCount: 0,\n cost: null,\n mixedCurrency: false,\n byModel: [],\n});\n\n/**\n * Build a {@link CostTracker} configured with the supplied budgets.\n *\n * @stable\n */\nexport function createCostTracker(opts: CostTrackerOptions = {}): CostTracker {\n const budgets: CostBudgets = opts.budgets ?? {};\n const currency = budgets.currency ?? 'USD';\n const onExceed = opts.onExceed;\n\n const bySpan = new Map<string, AggregateBucket>();\n const byScope: Record<CostScope, Map<string, AggregateBucket>> = {\n run: new Map(),\n session: new Map(),\n agent: new Map(),\n user: new Map(),\n };\n\n // Parent index: child span -> parent span. Used to roll attributions\n // up the ancestor chain on every record call.\n const parents = new Map<string, string>();\n const listeners = new Set<(input: CostRecordInput) => void>();\n\n function bump(scope: CostScope, id: string | undefined, input: CostRecordInput): void {\n if (id === undefined) return;\n const map = byScope[scope];\n const bucket = map.get(id) ?? freshBucket();\n combine(bucket, input);\n map.set(id, bucket);\n enforceBudget(scope, id, bucket, onExceed);\n }\n\n function enforceBudget(\n scope: CostScope,\n id: string,\n bucket: AggregateBucket,\n cb?: CostBudgetExceededCallback,\n ): void {\n const limit = pickBudget(budgets, scope);\n if (limit === undefined) return;\n if (bucket.exceeded) return;\n if (bucket.costAmount > limit) {\n bucket.exceeded = true;\n cb?.({\n scope,\n id,\n budget: limit,\n actual: bucket.costAmount,\n currency: bucket.costCurrency ?? currency,\n });\n }\n }\n\n return {\n record(input: CostRecordInput): void {\n // Record the raw span figures.\n const spanBucket = bySpan.get(input.spanId) ?? freshBucket();\n combine(spanBucket, input);\n bySpan.set(input.spanId, spanBucket);\n\n // Walk the ancestor chain so that the figures roll up to every\n // parent span. The walker breaks if it encounters a cycle.\n if (input.parentSpanId !== undefined) {\n parents.set(input.spanId, input.parentSpanId);\n }\n const seen = new Set<string>();\n let ancestor = parents.get(input.spanId);\n while (ancestor !== undefined && !seen.has(ancestor)) {\n seen.add(ancestor);\n const ancestorBucket = bySpan.get(ancestor) ?? freshBucket();\n combine(ancestorBucket, input);\n bySpan.set(ancestor, ancestorBucket);\n ancestor = parents.get(ancestor);\n }\n\n // Roll up to the requested aggregation scopes.\n bump('run', input.runId, input);\n bump('session', input.sessionId, input);\n bump('agent', input.agentId, input);\n bump('user', input.userId, input);\n\n for (const listener of listeners) {\n try {\n listener(input);\n } catch {\n // Listeners must never break the tracker.\n }\n }\n },\n usage(scope: CostScope, id: string): CostSnapshot {\n const bucket = byScope[scope].get(id);\n if (bucket === undefined) return ZERO;\n return snapshotOf(bucket);\n },\n usageForSpan(spanId: string): CostSnapshot {\n const bucket = bySpan.get(spanId);\n if (bucket === undefined) return ZERO;\n return snapshotOf(bucket);\n },\n reset(): void {\n bySpan.clear();\n for (const map of Object.values(byScope)) {\n map.clear();\n }\n parents.clear();\n },\n onRollup(listener): () => void {\n listeners.add(listener);\n return () => {\n listeners.delete(listener);\n };\n },\n };\n}\n\nfunction pickBudget(budgets: CostBudgets, scope: CostScope): number | undefined {\n switch (scope) {\n case 'session':\n return budgets.perSession;\n case 'user':\n return budgets.perUser;\n case 'agent':\n return budgets.perAgent;\n case 'run':\n return budgets.perRun;\n }\n}\n"],"mappings":";AAqDA,SAAS,cAA+B;AACtC,QAAO;EACL,cAAc;EACd,kBAAkB;EAClB,iBAAiB;EACjB,WAAW;EACX,YAAY;EACZ,cAAc;EACd,eAAe;EACf,yBAAS,IAAI,KAAK;EAClB,UAAU;EACX;;AAOH,SAAS,QAAQ,QAAyB,MAAkB;AAC1D,KAAI,OAAO,iBAAiB,QAAQ,OAAO,iBAAiB,KAAK,SAC/D,QAAO,gBAAgB;UACd,OAAO,iBAAiB,KACjC,QAAO,eAAe,KAAK;AAE7B,QAAO,cAAc,KAAK;;AAG5B,SAAS,QAAQ,QAAyB,OAA8B;AACtE,QAAO,gBAAgB,MAAM;AAC7B,QAAO,oBAAoB,MAAM;AACjC,QAAO,mBAAmB,MAAM,mBAAmB;AACnD,QAAO,aAAa;AACpB,KAAI,MAAM,SAAS,OAAW,SAAQ,QAAQ,MAAM,KAAK;CACzD,MAAM,cAAc,OAAO,QAAQ,IAAI,MAAM,MAAM,IAAI,aAAa;AACpE,aAAY,gBAAgB,MAAM;AAClC,aAAY,oBAAoB,MAAM;AACtC,aAAY,mBAAmB,MAAM,mBAAmB;AACxD,aAAY,aAAa;AACzB,KAAI,MAAM,SAAS,OAAW,SAAQ,aAAa,MAAM,KAAK;AAC9D,QAAO,QAAQ,IAAI,MAAM,OAAO,YAAY;;AAG9C,SAAS,WAAW,QAAuC;CACzD,MAAMA,OACJ,OAAO,iBAAiB,OACpB,OACA;EAAE,QAAQ,OAAO;EAAY,UAAU,OAAO;EAAc;AAClE,QAAO;EACL,cAAc,OAAO;EACrB,kBAAkB,OAAO;EACzB,iBAAiB,OAAO;EACxB,aAAa,OAAO,eAAe,OAAO,mBAAmB,OAAO;EACpE,WAAW,OAAO;EAClB;EACA,eAAe,OAAO;EACtB,SAAS,CAAC,GAAG,OAAO,QAAQ,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ;GAC1D;GACA,cAAc,EAAE;GAChB,kBAAkB,EAAE;GACpB,iBAAiB,EAAE;GACnB,WAAW,EAAE;GACb,MAAM,EAAE,iBAAiB,OAAO,OAAO;IAAE,QAAQ,EAAE;IAAY,UAAU,EAAE;IAAc;GACzF,eAAe,EAAE;GAClB,EAAE;EACJ;;AAGH,MAAMC,OAAqB,OAAO,OAAO;CACvC,cAAc;CACd,kBAAkB;CAClB,iBAAiB;CACjB,aAAa;CACb,WAAW;CACX,MAAM;CACN,eAAe;CACf,SAAS,EAAE;CACZ,CAAC;;;;;;AAOF,SAAgB,kBAAkB,OAA2B,EAAE,EAAe;CAC5E,MAAMC,UAAuB,KAAK,WAAW,EAAE;CAC/C,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,WAAW,KAAK;CAEtB,MAAM,yBAAS,IAAI,KAA8B;CACjD,MAAMC,UAA2D;EAC/D,qBAAK,IAAI,KAAK;EACd,yBAAS,IAAI,KAAK;EAClB,uBAAO,IAAI,KAAK;EAChB,sBAAM,IAAI,KAAK;EAChB;CAID,MAAM,0BAAU,IAAI,KAAqB;CACzC,MAAM,4BAAY,IAAI,KAAuC;CAE7D,SAAS,KAAK,OAAkB,IAAwB,OAA8B;AACpF,MAAI,OAAO,OAAW;EACtB,MAAM,MAAM,QAAQ;EACpB,MAAM,SAAS,IAAI,IAAI,GAAG,IAAI,aAAa;AAC3C,UAAQ,QAAQ,MAAM;AACtB,MAAI,IAAI,IAAI,OAAO;AACnB,gBAAc,OAAO,IAAI,QAAQ,SAAS;;CAG5C,SAAS,cACP,OACA,IACA,QACA,IACM;EACN,MAAM,QAAQ,WAAW,SAAS,MAAM;AACxC,MAAI,UAAU,OAAW;AACzB,MAAI,OAAO,SAAU;AACrB,MAAI,OAAO,aAAa,OAAO;AAC7B,UAAO,WAAW;AAClB,QAAK;IACH;IACA;IACA,QAAQ;IACR,QAAQ,OAAO;IACf,UAAU,OAAO,gBAAgB;IAClC,CAAC;;;AAIN,QAAO;EACL,OAAO,OAA8B;GAEnC,MAAM,aAAa,OAAO,IAAI,MAAM,OAAO,IAAI,aAAa;AAC5D,WAAQ,YAAY,MAAM;AAC1B,UAAO,IAAI,MAAM,QAAQ,WAAW;AAIpC,OAAI,MAAM,iBAAiB,OACzB,SAAQ,IAAI,MAAM,QAAQ,MAAM,aAAa;GAE/C,MAAM,uBAAO,IAAI,KAAa;GAC9B,IAAI,WAAW,QAAQ,IAAI,MAAM,OAAO;AACxC,UAAO,aAAa,UAAa,CAAC,KAAK,IAAI,SAAS,EAAE;AACpD,SAAK,IAAI,SAAS;IAClB,MAAM,iBAAiB,OAAO,IAAI,SAAS,IAAI,aAAa;AAC5D,YAAQ,gBAAgB,MAAM;AAC9B,WAAO,IAAI,UAAU,eAAe;AACpC,eAAW,QAAQ,IAAI,SAAS;;AAIlC,QAAK,OAAO,MAAM,OAAO,MAAM;AAC/B,QAAK,WAAW,MAAM,WAAW,MAAM;AACvC,QAAK,SAAS,MAAM,SAAS,MAAM;AACnC,QAAK,QAAQ,MAAM,QAAQ,MAAM;AAEjC,QAAK,MAAM,YAAY,UACrB,KAAI;AACF,aAAS,MAAM;WACT;;EAKZ,MAAM,OAAkB,IAA0B;GAChD,MAAM,SAAS,QAAQ,OAAO,IAAI,GAAG;AACrC,OAAI,WAAW,OAAW,QAAO;AACjC,UAAO,WAAW,OAAO;;EAE3B,aAAa,QAA8B;GACzC,MAAM,SAAS,OAAO,IAAI,OAAO;AACjC,OAAI,WAAW,OAAW,QAAO;AACjC,UAAO,WAAW,OAAO;;EAE3B,QAAc;AACZ,UAAO,OAAO;AACd,QAAK,MAAM,OAAO,OAAO,OAAO,QAAQ,CACtC,KAAI,OAAO;AAEb,WAAQ,OAAO;;EAEjB,SAAS,UAAsB;AAC7B,aAAU,IAAI,SAAS;AACvB,gBAAa;AACX,cAAU,OAAO,SAAS;;;EAG/B;;AAGH,SAAS,WAAW,SAAsB,OAAsC;AAC9E,SAAQ,OAAR;EACE,KAAK,UACH,QAAO,QAAQ;EACjB,KAAK,OACH,QAAO,QAAQ;EACjB,KAAK,QACH,QAAO,QAAQ;EACjB,KAAK,MACH,QAAO,QAAQ"}
@@ -0,0 +1,3 @@
1
+ import { CostBudgetExceededCallback, CostBudgets, CostRecordInput, CostScope, CostSnapshot, CostTrackerOptions } from "./types.js";
2
+ import { CostTracker, createCostTracker } from "./cost-tracker.js";
3
+ export { type CostBudgetExceededCallback, type CostBudgets, type CostRecordInput, type CostScope, type CostSnapshot, type CostTracker, type CostTrackerOptions, createCostTracker };
@@ -0,0 +1,3 @@
1
+ import { createCostTracker } from "./cost-tracker.js";
2
+
3
+ export { createCostTracker };
@@ -0,0 +1,104 @@
1
+ import { Cost } from "@graphorin/core";
2
+
3
+ //#region src/cost/types.d.ts
4
+
5
+ /**
6
+ * Aggregation scope used by {@link CostTracker.usage}. The framework
7
+ * tracks four canonical scopes; deployments needing additional
8
+ * dimensions can build them by registering listeners on
9
+ * `CostTracker.onRollup(...)`.
10
+ *
11
+ * @stable
12
+ */
13
+ type CostScope = 'run' | 'session' | 'agent' | 'user';
14
+ /**
15
+ * Per-call usage record fed into {@link CostTracker.record}.
16
+ *
17
+ * @stable
18
+ */
19
+ interface CostRecordInput {
20
+ readonly model: string;
21
+ readonly promptTokens: number;
22
+ readonly completionTokens: number;
23
+ readonly reasoningTokens?: number;
24
+ readonly cost?: Cost;
25
+ /** Span id; used to thread parent-child rollups. */
26
+ readonly spanId: string;
27
+ /** Optional parent span id (when the call is part of a nested run). */
28
+ readonly parentSpanId?: string;
29
+ readonly runId?: string;
30
+ readonly sessionId?: string;
31
+ readonly agentId?: string;
32
+ readonly userId?: string;
33
+ }
34
+ /**
35
+ * Snapshot returned by {@link CostTracker.usage}.
36
+ *
37
+ * @stable
38
+ */
39
+ interface CostSnapshot {
40
+ readonly totalTokens: number;
41
+ readonly promptTokens: number;
42
+ readonly completionTokens: number;
43
+ readonly reasoningTokens: number;
44
+ readonly callCount: number;
45
+ readonly cost: Cost | null;
46
+ /**
47
+ * RP-22: `true` when records carrying differing currencies were aggregated
48
+ * into this snapshot. `cost.amount` is then a sum across currencies and must
49
+ * not be treated as a single clean figure.
50
+ */
51
+ readonly mixedCurrency: boolean;
52
+ readonly byModel: ReadonlyArray<{
53
+ readonly model: string;
54
+ readonly promptTokens: number;
55
+ readonly completionTokens: number;
56
+ readonly reasoningTokens: number;
57
+ readonly callCount: number;
58
+ readonly cost: Cost | null;
59
+ readonly mixedCurrency: boolean;
60
+ }>;
61
+ }
62
+ /**
63
+ * Budget configuration shape consumed by {@link createCostTracker}.
64
+ *
65
+ * @stable
66
+ */
67
+ interface CostBudgets {
68
+ /** Per-session budget in the configured currency. */
69
+ readonly perSession?: number;
70
+ /** Per-user budget. */
71
+ readonly perUser?: number;
72
+ /** Per-agent budget. */
73
+ readonly perAgent?: number;
74
+ /** Per-run budget. */
75
+ readonly perRun?: number;
76
+ /** Currency. Defaults to `'USD'`. */
77
+ readonly currency?: string;
78
+ }
79
+ /**
80
+ * Callback invoked when an aggregation scope crosses its configured
81
+ * budget. The handler receives a sanitized payload — the secret-free
82
+ * scope id + the breached numbers.
83
+ *
84
+ * @stable
85
+ */
86
+ type CostBudgetExceededCallback = (event: {
87
+ readonly scope: CostScope;
88
+ readonly id: string;
89
+ readonly budget: number;
90
+ readonly actual: number;
91
+ readonly currency: string;
92
+ }) => void;
93
+ /**
94
+ * Configuration shape for {@link createCostTracker}.
95
+ *
96
+ * @stable
97
+ */
98
+ interface CostTrackerOptions {
99
+ readonly budgets?: CostBudgets;
100
+ readonly onExceed?: CostBudgetExceededCallback;
101
+ }
102
+ //#endregion
103
+ export { CostBudgetExceededCallback, CostBudgets, CostRecordInput, CostScope, CostSnapshot, CostTrackerOptions };
104
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","names":[],"sources":["../../src/cost/types.ts"],"sourcesContent":[],"mappings":";;;;;;;;AAyEA;AAoBA;AAaA;;KA1FY,SAAA;;;;;;UAOK,eAAA;;;;;kBAKC;;;;;;;;;;;;;;;UAgBD,YAAA;;;;;;iBAMA;;;;;;;oBAOG;;;;;;mBAMD;;;;;;;;;UAUF,WAAA;;;;;;;;;;;;;;;;;;;KAoBL,0BAAA;kBACM;;;;;;;;;;;UAYD,kBAAA;qBACI;sBACC"}
@@ -0,0 +1,3 @@
1
+ import { Case, Dataset, EvalCaseResult, EvalReport, RunEvalOptions, ScoreResult, Scorer } from "./types.js";
2
+ import { runEval } from "./runner.js";
3
+ export { type Case, type Dataset, type EvalCaseResult, type EvalReport, type RunEvalOptions, type ScoreResult, type Scorer, runEval };
@@ -0,0 +1,3 @@
1
+ import { runEval } from "./runner.js";
2
+
3
+ export { runEval };
@@ -0,0 +1,11 @@
1
+ import { EvalReport, RunEvalOptions } from "./types.js";
2
+
3
+ //#region src/eval/runner.d.ts
4
+
5
+ /**
6
+ * @stable
7
+ */
8
+ declare function runEval<I, O>(opts: RunEvalOptions<I, O>): Promise<EvalReport<I, O>>;
9
+ //#endregion
10
+ export { runEval };
11
+ //# sourceMappingURL=runner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.d.ts","names":[],"sources":["../../src/eval/runner.ts"],"sourcesContent":[],"mappings":";;;;;;;iBAwBsB,oBAAoB,eAAe,GAAG,KAAK,QAAQ,WAAW,GAAG"}
@@ -0,0 +1,113 @@
1
+ //#region src/eval/runner.ts
2
+ /**
3
+ * @stable
4
+ */
5
+ async function runEval(opts) {
6
+ const iterations = Math.max(1, opts.iterations ?? 1);
7
+ const signal = opts.signal;
8
+ const results = [];
9
+ for (let iter = 0; iter < iterations; iter++) for (let idx = 0; idx < opts.dataset.cases.length; idx++) {
10
+ throwIfAborted(signal);
11
+ const sampleCase = opts.dataset.cases[idx];
12
+ if (sampleCase === void 0) continue;
13
+ const baseId = sampleCase.id ?? `case-${idx}`;
14
+ const caseId = iterations === 1 ? baseId : `${baseId}-iter-${iter}`;
15
+ const startedAt = Date.now();
16
+ const output = await opts.agent.run(sampleCase.input);
17
+ const durationMs = Date.now() - startedAt;
18
+ const scores = [];
19
+ for (const scorer of opts.scorers) {
20
+ throwIfAborted(signal);
21
+ const result = await safeScore(scorer, sampleCase, output, durationMs);
22
+ scores.push({
23
+ scorer: scorer.name,
24
+ result
25
+ });
26
+ }
27
+ results.push({
28
+ caseId,
29
+ input: sampleCase.input,
30
+ output,
31
+ durationMs,
32
+ scores
33
+ });
34
+ }
35
+ return summarize(results, opts.scorers);
36
+ }
37
+ async function safeScore(scorer, c, output, durationMs) {
38
+ try {
39
+ return await scorer.score({
40
+ case: c,
41
+ output,
42
+ durationMs
43
+ });
44
+ } catch (err) {
45
+ return {
46
+ pass: false,
47
+ reason: `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
48
+ };
49
+ }
50
+ }
51
+ function summarize(results, scorers) {
52
+ const total = results.length;
53
+ let passed = 0;
54
+ let failed = 0;
55
+ let durationSum = 0;
56
+ const byScorer = {};
57
+ for (const scorer of scorers) byScorer[scorer.name] = {
58
+ passed: 0,
59
+ failed: 0,
60
+ scoreSum: 0,
61
+ scoreCount: 0
62
+ };
63
+ for (const r of results) {
64
+ durationSum += r.durationMs;
65
+ let passEntire = true;
66
+ for (const { scorer, result } of r.scores) {
67
+ const bucket = byScorer[scorer] ?? {
68
+ passed: 0,
69
+ failed: 0,
70
+ scoreSum: 0,
71
+ scoreCount: 0
72
+ };
73
+ if (result.pass) bucket.passed += 1;
74
+ else {
75
+ bucket.failed += 1;
76
+ passEntire = false;
77
+ }
78
+ if (typeof result.score === "number" && Number.isFinite(result.score)) {
79
+ bucket.scoreSum += result.score;
80
+ bucket.scoreCount += 1;
81
+ }
82
+ byScorer[scorer] = bucket;
83
+ }
84
+ if (passEntire) passed += 1;
85
+ else failed += 1;
86
+ }
87
+ const summaryByScorer = {};
88
+ for (const [scorer, b] of Object.entries(byScorer)) summaryByScorer[scorer] = {
89
+ passed: b.passed,
90
+ failed: b.failed,
91
+ avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount
92
+ };
93
+ return {
94
+ results,
95
+ summary: {
96
+ total,
97
+ passed,
98
+ failed,
99
+ avgDurationMs: total === 0 ? 0 : durationSum / total,
100
+ byScorer: Object.freeze(summaryByScorer)
101
+ }
102
+ };
103
+ }
104
+ function throwIfAborted(signal) {
105
+ if (signal?.aborted === true) {
106
+ const reason = signal.reason;
107
+ throw reason instanceof Error ? reason : /* @__PURE__ */ new Error("Eval run aborted");
108
+ }
109
+ }
110
+
111
+ //#endregion
112
+ export { runEval };
113
+ //# sourceMappingURL=runner.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runner.js","names":["results: EvalCaseResult<I, O>[]","scores: EvalCaseResult<I, O>['scores'][number][]","byScorer: Record<\n string,\n { passed: number; failed: number; scoreSum: number; scoreCount: number }\n >","summaryByScorer: Record<\n string,\n { passed: number; failed: number; avgScore: number | null }\n >"],"sources":["../../src/eval/runner.ts"],"sourcesContent":["/**\n * `runEval(...)` — minimal inline eval runner. Walks every case in\n * the dataset, calls `agent.run(case.input)`, applies every supplied\n * scorer, and returns an aggregated {@link EvalReport}.\n *\n * The runner is deliberately tiny — it has no parallelism, no\n * retries, no reporters, no dataset loaders. Production scenarios\n * that need any of those should `pnpm add @graphorin/evals` (post-MVP).\n *\n * @packageDocumentation\n */\n\nimport type {\n Case,\n EvalCaseResult,\n EvalReport,\n RunEvalOptions,\n ScoreResult,\n Scorer,\n} from './types.js';\n\n/**\n * @stable\n */\nexport async function runEval<I, O>(opts: RunEvalOptions<I, O>): Promise<EvalReport<I, O>> {\n const iterations = Math.max(1, opts.iterations ?? 1);\n const signal = opts.signal;\n const results: EvalCaseResult<I, O>[] = [];\n\n for (let iter = 0; iter < iterations; iter++) {\n for (let idx = 0; idx < opts.dataset.cases.length; idx++) {\n throwIfAborted(signal);\n const sampleCase = opts.dataset.cases[idx];\n if (sampleCase === undefined) continue;\n // EB-6: disambiguate a caller-provided id per iteration too, not just the\n // synthetic fallback — otherwise iterations>1 emits duplicate caseIds.\n const baseId = sampleCase.id ?? `case-${idx}`;\n const caseId = iterations === 1 ? baseId : `${baseId}-iter-${iter}`;\n\n const startedAt = Date.now();\n const output = await opts.agent.run(sampleCase.input);\n const durationMs = Date.now() - startedAt;\n\n const scores: EvalCaseResult<I, O>['scores'][number][] = [];\n for (const scorer of opts.scorers) {\n throwIfAborted(signal);\n const result = await safeScore(scorer, sampleCase, output, durationMs);\n scores.push({ scorer: scorer.name, result });\n }\n\n results.push({\n caseId,\n input: sampleCase.input,\n output,\n durationMs,\n scores,\n });\n }\n }\n\n return summarize(results, opts.scorers);\n}\n\nasync function safeScore<I, O>(\n scorer: Scorer<I, O>,\n c: Case<I, O>,\n output: O,\n durationMs: number,\n): Promise<ScoreResult> {\n try {\n return await scorer.score({ case: c, output, durationMs });\n } catch (err) {\n return {\n pass: false,\n reason: `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n };\n }\n}\n\nfunction summarize<I, O>(\n results: ReadonlyArray<EvalCaseResult<I, O>>,\n scorers: ReadonlyArray<Scorer<I, O>>,\n): EvalReport<I, O> {\n const total = results.length;\n let passed = 0;\n let failed = 0;\n let durationSum = 0;\n const byScorer: Record<\n string,\n { passed: number; failed: number; scoreSum: number; scoreCount: number }\n > = {};\n\n for (const scorer of scorers) {\n byScorer[scorer.name] = { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };\n }\n\n for (const r of results) {\n durationSum += r.durationMs;\n let passEntire = true;\n for (const { scorer, result } of r.scores) {\n const bucket = byScorer[scorer] ?? { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };\n if (result.pass) bucket.passed += 1;\n else {\n bucket.failed += 1;\n passEntire = false;\n }\n if (typeof result.score === 'number' && Number.isFinite(result.score)) {\n bucket.scoreSum += result.score;\n bucket.scoreCount += 1;\n }\n byScorer[scorer] = bucket;\n }\n if (passEntire) passed += 1;\n else failed += 1;\n }\n\n const summaryByScorer: Record<\n string,\n { passed: number; failed: number; avgScore: number | null }\n > = {};\n for (const [scorer, b] of Object.entries(byScorer)) {\n summaryByScorer[scorer] = {\n passed: b.passed,\n failed: b.failed,\n avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount,\n };\n }\n\n return {\n results,\n summary: {\n total,\n passed,\n failed,\n avgDurationMs: total === 0 ? 0 : durationSum / total,\n byScorer: Object.freeze(summaryByScorer),\n },\n };\n}\n\nfunction throwIfAborted(signal: AbortSignal | undefined): void {\n if (signal?.aborted === true) {\n const reason = signal.reason;\n throw reason instanceof Error ? reason : new Error('Eval run aborted');\n }\n}\n"],"mappings":";;;;AAwBA,eAAsB,QAAc,MAAuD;CACzF,MAAM,aAAa,KAAK,IAAI,GAAG,KAAK,cAAc,EAAE;CACpD,MAAM,SAAS,KAAK;CACpB,MAAMA,UAAkC,EAAE;AAE1C,MAAK,IAAI,OAAO,GAAG,OAAO,YAAY,OACpC,MAAK,IAAI,MAAM,GAAG,MAAM,KAAK,QAAQ,MAAM,QAAQ,OAAO;AACxD,iBAAe,OAAO;EACtB,MAAM,aAAa,KAAK,QAAQ,MAAM;AACtC,MAAI,eAAe,OAAW;EAG9B,MAAM,SAAS,WAAW,MAAM,QAAQ;EACxC,MAAM,SAAS,eAAe,IAAI,SAAS,GAAG,OAAO,QAAQ;EAE7D,MAAM,YAAY,KAAK,KAAK;EAC5B,MAAM,SAAS,MAAM,KAAK,MAAM,IAAI,WAAW,MAAM;EACrD,MAAM,aAAa,KAAK,KAAK,GAAG;EAEhC,MAAMC,SAAmD,EAAE;AAC3D,OAAK,MAAM,UAAU,KAAK,SAAS;AACjC,kBAAe,OAAO;GACtB,MAAM,SAAS,MAAM,UAAU,QAAQ,YAAY,QAAQ,WAAW;AACtE,UAAO,KAAK;IAAE,QAAQ,OAAO;IAAM;IAAQ,CAAC;;AAG9C,UAAQ,KAAK;GACX;GACA,OAAO,WAAW;GAClB;GACA;GACA;GACD,CAAC;;AAIN,QAAO,UAAU,SAAS,KAAK,QAAQ;;AAGzC,eAAe,UACb,QACA,GACA,QACA,YACsB;AACtB,KAAI;AACF,SAAO,MAAM,OAAO,MAAM;GAAE,MAAM;GAAG;GAAQ;GAAY,CAAC;UACnD,KAAK;AACZ,SAAO;GACL,MAAM;GACN,QAAQ,WAAW,OAAO,KAAK,WAAW,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;GAC3F;;;AAIL,SAAS,UACP,SACA,SACkB;CAClB,MAAM,QAAQ,QAAQ;CACtB,IAAI,SAAS;CACb,IAAI,SAAS;CACb,IAAI,cAAc;CAClB,MAAMC,WAGF,EAAE;AAEN,MAAK,MAAM,UAAU,QACnB,UAAS,OAAO,QAAQ;EAAE,QAAQ;EAAG,QAAQ;EAAG,UAAU;EAAG,YAAY;EAAG;AAG9E,MAAK,MAAM,KAAK,SAAS;AACvB,iBAAe,EAAE;EACjB,IAAI,aAAa;AACjB,OAAK,MAAM,EAAE,QAAQ,YAAY,EAAE,QAAQ;GACzC,MAAM,SAAS,SAAS,WAAW;IAAE,QAAQ;IAAG,QAAQ;IAAG,UAAU;IAAG,YAAY;IAAG;AACvF,OAAI,OAAO,KAAM,QAAO,UAAU;QAC7B;AACH,WAAO,UAAU;AACjB,iBAAa;;AAEf,OAAI,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,OAAO,MAAM,EAAE;AACrE,WAAO,YAAY,OAAO;AAC1B,WAAO,cAAc;;AAEvB,YAAS,UAAU;;AAErB,MAAI,WAAY,WAAU;MACrB,WAAU;;CAGjB,MAAMC,kBAGF,EAAE;AACN,MAAK,MAAM,CAAC,QAAQ,MAAM,OAAO,QAAQ,SAAS,CAChD,iBAAgB,UAAU;EACxB,QAAQ,EAAE;EACV,QAAQ,EAAE;EACV,UAAU,EAAE,eAAe,IAAI,OAAO,EAAE,WAAW,EAAE;EACtD;AAGH,QAAO;EACL;EACA,SAAS;GACP;GACA;GACA;GACA,eAAe,UAAU,IAAI,IAAI,cAAc;GAC/C,UAAU,OAAO,OAAO,gBAAgB;GACzC;EACF;;AAGH,SAAS,eAAe,QAAuC;AAC7D,KAAI,QAAQ,YAAY,MAAM;EAC5B,MAAM,SAAS,OAAO;AACtB,QAAM,kBAAkB,QAAQ,yBAAS,IAAI,MAAM,mBAAmB"}