@elsium-ai/observe 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -0
- package/dist/experiment.d.ts +34 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +197 -6
- package/dist/instrument.d.ts +9 -0
- package/dist/instrument.d.ts.map +1 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1171,6 +1171,125 @@ await tracer.flush()
|
|
|
1171
1171
|
|
|
1172
1172
|
---
|
|
1173
1173
|
|
|
1174
|
+
## Experiments Persistence
|
|
1175
|
+
|
|
1176
|
+
### `createFileExperimentStore`
|
|
1177
|
+
|
|
1178
|
+
Creates a file-based storage adapter for saving and loading experiment results to disk. Experiment data is serialized as JSON files in the specified directory.
|
|
1179
|
+
|
|
1180
|
+
```ts
|
|
1181
|
+
function createFileExperimentStore(dir: string): ExperimentStore
|
|
1182
|
+
```
|
|
1183
|
+
|
|
1184
|
+
| Parameter | Type | Description |
|
|
1185
|
+
|---|---|---|
|
|
1186
|
+
| `dir` | `string` | Directory path where experiment result files will be stored |
|
|
1187
|
+
|
|
1188
|
+
**Returns:** `ExperimentStore`
|
|
1189
|
+
|
|
1190
|
+
```ts
|
|
1191
|
+
interface ExperimentStore {
|
|
1192
|
+
save(experiment: ExperimentResults): Promise<void>
|
|
1193
|
+
load(experimentId: string): Promise<ExperimentResults | null>
|
|
1194
|
+
list(): Promise<string[]>
|
|
1195
|
+
}
|
|
1196
|
+
```
|
|
1197
|
+
|
|
1198
|
+
```ts
|
|
1199
|
+
import { createExperiment, createFileExperimentStore } from 'elsium-ai/observe'
|
|
1200
|
+
|
|
1201
|
+
const store = createFileExperimentStore('./experiments')
|
|
1202
|
+
|
|
1203
|
+
const experiment = createExperiment({
|
|
1204
|
+
name: 'prompt-comparison',
|
|
1205
|
+
variants: [
|
|
1206
|
+
{ name: 'concise', config: { system: 'Be brief.' } },
|
|
1207
|
+
{ name: 'detailed', config: { system: 'Be thorough.' } },
|
|
1208
|
+
],
|
|
1209
|
+
})
|
|
1210
|
+
|
|
1211
|
+
const results = await experiment.run(evaluator)
|
|
1212
|
+
|
|
1213
|
+
// Persist results to disk
|
|
1214
|
+
await store.save(results)
|
|
1215
|
+
|
|
1216
|
+
// Load results later
|
|
1217
|
+
const loaded = await store.load(results.id)
|
|
1218
|
+
```
|
|
1219
|
+
|
|
1220
|
+
---
|
|
1221
|
+
|
|
1222
|
+
## Auto-Instrumentation
|
|
1223
|
+
|
|
1224
|
+
### `instrumentComplete`
|
|
1225
|
+
|
|
1226
|
+
Wraps an LLM completion function with automatic span creation. Every call produces a span with model, token, cost, and latency metadata.
|
|
1227
|
+
|
|
1228
|
+
```ts
|
|
1229
|
+
function instrumentComplete(
|
|
1230
|
+
complete: (request: CompletionRequest) => Promise<LLMResponse>,
|
|
1231
|
+
tracer: Tracer,
|
|
1232
|
+
): (request: CompletionRequest) => Promise<LLMResponse>
|
|
1233
|
+
```
|
|
1234
|
+
|
|
1235
|
+
| Parameter | Type | Description |
|
|
1236
|
+
|---|---|---|
|
|
1237
|
+
| `complete` | `(request: CompletionRequest) => Promise<LLMResponse>` | The LLM completion function to instrument |
|
|
1238
|
+
| `tracer` | `Tracer` | The tracer instance to record spans to |
|
|
1239
|
+
|
|
1240
|
+
**Returns:** A wrapped completion function with the same signature.
|
|
1241
|
+
|
|
1242
|
+
```ts
|
|
1243
|
+
import { observe, instrumentComplete } from 'elsium-ai/observe'
|
|
1244
|
+
|
|
1245
|
+
const tracer = observe()
|
|
1246
|
+
|
|
1247
|
+
const tracedComplete = instrumentComplete(
|
|
1248
|
+
(req) => llm.complete(req),
|
|
1249
|
+
tracer,
|
|
1250
|
+
)
|
|
1251
|
+
|
|
1252
|
+
// Every call now creates an 'llm' span automatically
|
|
1253
|
+
const response = await tracedComplete({ model: 'gpt-4o', messages })
|
|
1254
|
+
```
|
|
1255
|
+
|
|
1256
|
+
### `instrumentAgent`
|
|
1257
|
+
|
|
1258
|
+
Wraps an agent's `run` method with automatic span creation. Produces an `agent` span that captures the agent name, input, output, token usage, and tool calls.
|
|
1259
|
+
|
|
1260
|
+
```ts
|
|
1261
|
+
function instrumentAgent(
|
|
1262
|
+
agent: Agent,
|
|
1263
|
+
tracer: Tracer,
|
|
1264
|
+
): Agent
|
|
1265
|
+
```
|
|
1266
|
+
|
|
1267
|
+
| Parameter | Type | Description |
|
|
1268
|
+
|---|---|---|
|
|
1269
|
+
| `agent` | `Agent` | The agent to instrument |
|
|
1270
|
+
| `tracer` | `Tracer` | The tracer instance to record spans to |
|
|
1271
|
+
|
|
1272
|
+
**Returns:** A new `Agent` with the same interface, where `run` and `chat` are automatically traced.
|
|
1273
|
+
|
|
1274
|
+
```ts
|
|
1275
|
+
import { observe, instrumentAgent } from 'elsium-ai/observe'
|
|
1276
|
+
import { defineAgent } from 'elsium-ai/agents'
|
|
1277
|
+
|
|
1278
|
+
const tracer = observe()
|
|
1279
|
+
|
|
1280
|
+
const agent = defineAgent(
|
|
1281
|
+
{ name: 'assistant', system: 'You are helpful.' },
|
|
1282
|
+
{ complete: (req) => llm.complete(req) },
|
|
1283
|
+
)
|
|
1284
|
+
|
|
1285
|
+
const tracedAgent = instrumentAgent(agent, tracer)
|
|
1286
|
+
|
|
1287
|
+
// Every run/chat call now creates an 'agent' span automatically
|
|
1288
|
+
const result = await tracedAgent.run('Hello')
|
|
1289
|
+
```
|
|
1290
|
+
|
|
1291
|
+
---
|
|
1292
|
+
|
|
1174
1293
|
## Part of ElsiumAI
|
|
1175
1294
|
|
|
1176
1295
|
This package is the observability layer of the [ElsiumAI](https://github.com/elsium-ai/elsium-ai) framework. See the [full documentation](https://github.com/elsium-ai/elsium-ai) for guides and examples.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export interface ExperimentVariant {
|
|
2
|
+
name: string;
|
|
3
|
+
weight: number;
|
|
4
|
+
config: Record<string, unknown>;
|
|
5
|
+
}
|
|
6
|
+
export interface ExperimentResults {
|
|
7
|
+
name: string;
|
|
8
|
+
totalAssignments: number;
|
|
9
|
+
variants: Record<string, {
|
|
10
|
+
assignments: number;
|
|
11
|
+
metrics: Record<string, {
|
|
12
|
+
sum: number;
|
|
13
|
+
count: number;
|
|
14
|
+
avg: number;
|
|
15
|
+
}>;
|
|
16
|
+
}>;
|
|
17
|
+
}
|
|
18
|
+
export interface Experiment {
|
|
19
|
+
assign(userId?: string): ExperimentVariant;
|
|
20
|
+
record(variant: string, metrics: Record<string, number>): void;
|
|
21
|
+
results(): ExperimentResults;
|
|
22
|
+
}
|
|
23
|
+
export interface ExperimentStore {
|
|
24
|
+
save(name: string, data: ExperimentResults): void;
|
|
25
|
+
load(name: string): ExperimentResults | null;
|
|
26
|
+
}
|
|
27
|
+
export interface ExperimentConfig {
|
|
28
|
+
name: string;
|
|
29
|
+
variants: ExperimentVariant[];
|
|
30
|
+
store?: ExperimentStore;
|
|
31
|
+
}
|
|
32
|
+
export declare function createFileExperimentStore(dir: string): ExperimentStore;
|
|
33
|
+
export declare function createExperiment(config: ExperimentConfig): Experiment;
|
|
34
|
+
//# sourceMappingURL=experiment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../src/experiment.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,gBAAgB,EAAE,MAAM,CAAA;IACxB,QAAQ,EAAE,MAAM,CACf,MAAM,EACN;QACC,WAAW,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACpE,CACD,CAAA;CACD;AAED,MAAM,WAAW,UAAU;IAC1B,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAA;IAC1C,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAA;IAC9D,OAAO,IAAI,iBAAiB,CAAA;CAC5B;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,GAAG,IAAI,CAAA;IACjD,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAAA;CAC5C;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,iBAAiB,EAAE,CAAA;IAC7B,KAAK,CAAC,EAAE,eAAe,CAAA;CACvB;AAED,wBAAgB,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CA4BtE;AAyDD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,UAAU,CAmErE"}
|
package/dist/index.d.ts
CHANGED
|
@@ -10,6 +10,10 @@ export { createAuditTrail, auditMiddleware } from './audit';
|
|
|
10
10
|
export type { AuditEventType, AuditEvent, AuditStorageAdapter, AuditQueryFilter, AuditIntegrityResult, AuditTrailConfig, AuditTrail, } from './audit';
|
|
11
11
|
export { createProvenanceTracker } from './provenance';
|
|
12
12
|
export type { ProvenanceRecord, ProvenanceTracker } from './provenance';
|
|
13
|
+
export { createExperiment, createFileExperimentStore } from './experiment';
|
|
14
|
+
export type { Experiment, ExperimentConfig, ExperimentVariant, ExperimentResults, ExperimentStore, } from './experiment';
|
|
15
|
+
export { instrumentComplete, instrumentAgent } from './instrument';
|
|
16
|
+
export type { InstrumentableAgent } from './instrument';
|
|
13
17
|
export { toOTelSpan, toOTelExportRequest, toTraceparent, parseTraceparent, injectTraceContext, extractTraceContext, createOTLPExporter, } from './otel';
|
|
14
18
|
export type { OTelSpan, OTelSpanKind, OTelStatusCode, OTelAttribute, OTelAttributeValue, OTelEvent, OTelResource, OTelExportRequest, TraceContext, OTLPExporterConfig, } from './otel';
|
|
15
19
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACnC,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAG1F,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACnE,YAAY,EACX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,EACnB,SAAS,EACT,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,cAAc,GACd,MAAM,eAAe,CAAA;AAGtB,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAClC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAG9F,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACzC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAG9D,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC3D,YAAY,EACX,cAAc,EACd,UAAU,EACV,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,gBAAgB,EAChB,UAAU,GACV,MAAM,SAAS,CAAA;AAGhB,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACtD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAGvE,OAAO,EACN,UAAU,EACV,mBAAmB,EACnB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,GAClB,MAAM,QAAQ,CAAA;AACf,YAAY,EACX,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,GAClB,MAAM,QAAQ,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACnC,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAG1F,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACnE,YAAY,EACX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,EACnB,SAAS,EACT,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,cAAc,GACd,MAAM,eAAe,CAAA;AAGtB,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAClC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAG9F,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACzC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAG9D,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC3D,YAAY,EACX,cAAc,EACd,UAAU,EACV,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,gBAAgB,EAChB,UAAU,GACV,MAAM,SAAS,CAAA;AAGhB,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACtD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAGvE,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAA;AAC1E,YAAY,EACX,UAAU,EACV,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,GACf,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAClE,YAAY,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAGvD,OAAO,EACN,UAAU,EACV,mBAAmB,EACnB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,GAClB,MAAM,QAAQ,CAAA;AACf,YAAY,EACX,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,GAClB,MAAM,QAAQ,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -140,6 +140,11 @@ function createLogger(options = {}) {
|
|
|
140
140
|
}
|
|
141
141
|
};
|
|
142
142
|
}
|
|
143
|
+
// ../core/src/schema.ts
|
|
144
|
+
var log = createLogger();
|
|
145
|
+
// ../core/src/registry.ts
|
|
146
|
+
var log2 = createLogger();
|
|
147
|
+
var BLOCKED_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
|
143
148
|
// src/span.ts
|
|
144
149
|
function createSpan(name, options = {}) {
|
|
145
150
|
const id = generateId("spn");
|
|
@@ -492,7 +497,7 @@ function createCostEngine(config = {}) {
|
|
|
492
497
|
}
|
|
493
498
|
// src/tracer.ts
|
|
494
499
|
import { writeFileSync } from "node:fs";
|
|
495
|
-
var
|
|
500
|
+
var log3 = createLogger();
|
|
496
501
|
function observe(config = {}) {
|
|
497
502
|
const {
|
|
498
503
|
output = ["console"],
|
|
@@ -515,7 +520,7 @@ function observe(config = {}) {
|
|
|
515
520
|
try {
|
|
516
521
|
writeFileSync(filename, JSON.stringify(spansToExport, null, 2));
|
|
517
522
|
} catch (err2) {
|
|
518
|
-
|
|
523
|
+
log3.error("Failed to write trace file", {
|
|
519
524
|
error: err2 instanceof Error ? err2.message : String(err2)
|
|
520
525
|
});
|
|
521
526
|
}
|
|
@@ -590,7 +595,7 @@ function observe(config = {}) {
|
|
|
590
595
|
function consoleHandler(span) {
|
|
591
596
|
const duration = span.durationMs !== undefined ? `${span.durationMs}ms` : "running";
|
|
592
597
|
const status = span.status === "error" ? "[ERROR]" : span.status === "ok" ? "[OK]" : "[...]";
|
|
593
|
-
|
|
598
|
+
log3.info("span", {
|
|
594
599
|
trace: span.traceId,
|
|
595
600
|
span: span.name,
|
|
596
601
|
kind: span.kind,
|
|
@@ -920,8 +925,190 @@ function createProvenanceTracker(options) {
|
|
|
920
925
|
}
|
|
921
926
|
};
|
|
922
927
|
}
|
|
928
|
+
// src/experiment.ts
|
|
929
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
930
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync as writeFileSync2 } from "node:fs";
|
|
931
|
+
import { join } from "node:path";
|
|
932
|
+
var log4 = createLogger();
|
|
933
|
+
function createFileExperimentStore(dir) {
|
|
934
|
+
return {
|
|
935
|
+
save(name, data) {
|
|
936
|
+
try {
|
|
937
|
+
if (!existsSync(dir)) {
|
|
938
|
+
mkdirSync(dir, { recursive: true });
|
|
939
|
+
}
|
|
940
|
+
const filePath = join(dir, `${name}.json`);
|
|
941
|
+
writeFileSync2(filePath, JSON.stringify(data, null, 2));
|
|
942
|
+
} catch (err2) {
|
|
943
|
+
log4.error("Failed to save experiment", {
|
|
944
|
+
name,
|
|
945
|
+
error: err2 instanceof Error ? err2.message : String(err2)
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
},
|
|
949
|
+
load(name) {
|
|
950
|
+
try {
|
|
951
|
+
const filePath = join(dir, `${name}.json`);
|
|
952
|
+
if (!existsSync(filePath))
|
|
953
|
+
return null;
|
|
954
|
+
const raw = readFileSync(filePath, "utf-8");
|
|
955
|
+
return JSON.parse(raw);
|
|
956
|
+
} catch {
|
|
957
|
+
return null;
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
function loadFromStore(store, name, stats) {
|
|
963
|
+
const saved = store.load(name);
|
|
964
|
+
if (!saved)
|
|
965
|
+
return;
|
|
966
|
+
for (const [vName, vData] of Object.entries(saved.variants)) {
|
|
967
|
+
if (!stats[vName])
|
|
968
|
+
continue;
|
|
969
|
+
stats[vName].assignments = vData.assignments;
|
|
970
|
+
for (const [key, m] of Object.entries(vData.metrics)) {
|
|
971
|
+
stats[vName].metrics[key] = { sum: m.sum, count: m.count };
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
log4.debug("Loaded experiment state", { name, totalAssignments: saved.totalAssignments });
|
|
975
|
+
}
|
|
976
|
+
function recordMetrics(s, metrics) {
|
|
977
|
+
for (const [key, value] of Object.entries(metrics)) {
|
|
978
|
+
if (!s.metrics[key]) {
|
|
979
|
+
s.metrics[key] = { sum: 0, count: 0 };
|
|
980
|
+
}
|
|
981
|
+
s.metrics[key].sum += value;
|
|
982
|
+
s.metrics[key].count++;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
function buildResults(name, stats) {
|
|
986
|
+
let totalAssignments = 0;
|
|
987
|
+
const variantResults = {};
|
|
988
|
+
for (const [vName, s] of Object.entries(stats)) {
|
|
989
|
+
totalAssignments += s.assignments;
|
|
990
|
+
const metricsResult = {};
|
|
991
|
+
for (const [key, m] of Object.entries(s.metrics)) {
|
|
992
|
+
metricsResult[key] = {
|
|
993
|
+
sum: m.sum,
|
|
994
|
+
count: m.count,
|
|
995
|
+
avg: m.count > 0 ? m.sum / m.count : 0
|
|
996
|
+
};
|
|
997
|
+
}
|
|
998
|
+
variantResults[vName] = {
|
|
999
|
+
assignments: s.assignments,
|
|
1000
|
+
metrics: metricsResult
|
|
1001
|
+
};
|
|
1002
|
+
}
|
|
1003
|
+
return { name, totalAssignments, variants: variantResults };
|
|
1004
|
+
}
|
|
1005
|
+
function createExperiment(config) {
|
|
1006
|
+
const { name, variants, store } = config;
|
|
1007
|
+
if (variants.length === 0) {
|
|
1008
|
+
throw new Error("Experiment must have at least one variant");
|
|
1009
|
+
}
|
|
1010
|
+
const totalWeight = variants.reduce((sum, v) => sum + v.weight, 0);
|
|
1011
|
+
const stats = {};
|
|
1012
|
+
for (const v of variants) {
|
|
1013
|
+
stats[v.name] = { assignments: 0, metrics: {} };
|
|
1014
|
+
}
|
|
1015
|
+
if (store) {
|
|
1016
|
+
loadFromStore(store, name, stats);
|
|
1017
|
+
}
|
|
1018
|
+
function hashAssign(userId) {
|
|
1019
|
+
const hash = createHash3("sha256").update(`${name}:${userId}`).digest();
|
|
1020
|
+
const value = hash.readUInt32BE(0) % 1e4 / 1e4;
|
|
1021
|
+
return pickVariant(value);
|
|
1022
|
+
}
|
|
1023
|
+
function randomAssign() {
|
|
1024
|
+
const value = Math.random();
|
|
1025
|
+
return pickVariant(value);
|
|
1026
|
+
}
|
|
1027
|
+
function pickVariant(value) {
|
|
1028
|
+
let cumulative = 0;
|
|
1029
|
+
for (const v of variants) {
|
|
1030
|
+
cumulative += v.weight / totalWeight;
|
|
1031
|
+
if (value < cumulative)
|
|
1032
|
+
return v;
|
|
1033
|
+
}
|
|
1034
|
+
return variants[variants.length - 1];
|
|
1035
|
+
}
|
|
1036
|
+
return {
|
|
1037
|
+
assign(userId) {
|
|
1038
|
+
const variant = userId ? hashAssign(userId) : randomAssign();
|
|
1039
|
+
const s = stats[variant.name];
|
|
1040
|
+
if (s)
|
|
1041
|
+
s.assignments++;
|
|
1042
|
+
log4.debug("Experiment assignment", {
|
|
1043
|
+
experiment: name,
|
|
1044
|
+
variant: variant.name,
|
|
1045
|
+
userId
|
|
1046
|
+
});
|
|
1047
|
+
return variant;
|
|
1048
|
+
},
|
|
1049
|
+
record(variant, metrics) {
|
|
1050
|
+
const s = stats[variant];
|
|
1051
|
+
if (!s)
|
|
1052
|
+
return;
|
|
1053
|
+
recordMetrics(s, metrics);
|
|
1054
|
+
if (store) {
|
|
1055
|
+
store.save(name, this.results());
|
|
1056
|
+
}
|
|
1057
|
+
},
|
|
1058
|
+
results() {
|
|
1059
|
+
return buildResults(name, stats);
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
// src/instrument.ts
|
|
1064
|
+
function instrumentComplete(complete, tracer) {
|
|
1065
|
+
return async (request) => {
|
|
1066
|
+
const span = tracer.startSpan("llm.complete", "llm");
|
|
1067
|
+
span.setMetadata("model", request.model ?? "default");
|
|
1068
|
+
span.setMetadata("messageCount", request.messages.length);
|
|
1069
|
+
try {
|
|
1070
|
+
const response = await complete(request);
|
|
1071
|
+
span.setMetadata("inputTokens", response.usage.inputTokens);
|
|
1072
|
+
span.setMetadata("outputTokens", response.usage.outputTokens);
|
|
1073
|
+
span.setMetadata("totalCost", response.cost.totalCost);
|
|
1074
|
+
span.setMetadata("provider", response.provider);
|
|
1075
|
+
span.setMetadata("latencyMs", response.latencyMs);
|
|
1076
|
+
tracer.trackLLMCall({
|
|
1077
|
+
model: response.model,
|
|
1078
|
+
inputTokens: response.usage.inputTokens,
|
|
1079
|
+
outputTokens: response.usage.outputTokens,
|
|
1080
|
+
cost: response.cost.totalCost,
|
|
1081
|
+
latencyMs: response.latencyMs
|
|
1082
|
+
});
|
|
1083
|
+
span.end({ status: "ok" });
|
|
1084
|
+
return response;
|
|
1085
|
+
} catch (error) {
|
|
1086
|
+
span.setMetadata("error", error instanceof Error ? error.message : String(error));
|
|
1087
|
+
span.end({ status: "error" });
|
|
1088
|
+
throw error;
|
|
1089
|
+
}
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
function instrumentAgent(agent, tracer) {
|
|
1093
|
+
const originalRun = agent.run.bind(agent);
|
|
1094
|
+
const instrumented = Object.create(agent);
|
|
1095
|
+
instrumented.run = async (input, options) => {
|
|
1096
|
+
const span = tracer.startSpan(`agent.${agent.name}`, "agent");
|
|
1097
|
+
span.setMetadata("agentName", agent.name);
|
|
1098
|
+
try {
|
|
1099
|
+
const result = await originalRun(input, options);
|
|
1100
|
+
span.end({ status: "ok" });
|
|
1101
|
+
return result;
|
|
1102
|
+
} catch (error) {
|
|
1103
|
+
span.setMetadata("error", error instanceof Error ? error.message : String(error));
|
|
1104
|
+
span.end({ status: "error" });
|
|
1105
|
+
throw error;
|
|
1106
|
+
}
|
|
1107
|
+
};
|
|
1108
|
+
return instrumented;
|
|
1109
|
+
}
|
|
923
1110
|
// src/otel.ts
|
|
924
|
-
var
|
|
1111
|
+
var log5 = createLogger();
|
|
925
1112
|
var SPAN_KIND_MAP = {
|
|
926
1113
|
llm: 3,
|
|
927
1114
|
tool: 1,
|
|
@@ -1062,10 +1249,10 @@ function createOTLPExporter(config) {
|
|
|
1062
1249
|
body: JSON.stringify(payload)
|
|
1063
1250
|
});
|
|
1064
1251
|
if (!response.ok) {
|
|
1065
|
-
|
|
1252
|
+
log5.error(`OTLP export failed: ${response.status} ${response.statusText}`);
|
|
1066
1253
|
}
|
|
1067
1254
|
} catch (err2) {
|
|
1068
|
-
|
|
1255
|
+
log5.error("OTLP export error", { error: err2 instanceof Error ? err2.message : String(err2) });
|
|
1069
1256
|
}
|
|
1070
1257
|
}
|
|
1071
1258
|
function startAutoFlush() {
|
|
@@ -1108,12 +1295,16 @@ export {
|
|
|
1108
1295
|
registerModelTier,
|
|
1109
1296
|
parseTraceparent,
|
|
1110
1297
|
observe,
|
|
1298
|
+
instrumentComplete,
|
|
1299
|
+
instrumentAgent,
|
|
1111
1300
|
injectTraceContext,
|
|
1112
1301
|
extractTraceContext,
|
|
1113
1302
|
createSpan,
|
|
1114
1303
|
createProvenanceTracker,
|
|
1115
1304
|
createOTLPExporter,
|
|
1116
1305
|
createMetrics,
|
|
1306
|
+
createFileExperimentStore,
|
|
1307
|
+
createExperiment,
|
|
1117
1308
|
createCostEngine,
|
|
1118
1309
|
createAuditTrail,
|
|
1119
1310
|
auditMiddleware
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { CompletionRequest, LLMResponse } from '@elsium-ai/core';
|
|
2
|
+
import type { Tracer } from './tracer';
|
|
3
|
+
export declare function instrumentComplete(complete: (request: CompletionRequest) => Promise<LLMResponse>, tracer: Tracer): (request: CompletionRequest) => Promise<LLMResponse>;
|
|
4
|
+
export interface InstrumentableAgent {
|
|
5
|
+
readonly name: string;
|
|
6
|
+
run(input: string, options?: Record<string, unknown>): Promise<unknown>;
|
|
7
|
+
}
|
|
8
|
+
export declare function instrumentAgent<T extends InstrumentableAgent>(agent: T, tracer: Tracer): T;
|
|
9
|
+
//# sourceMappingURL=instrument.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"instrument.d.ts","sourceRoot":"","sources":["../src/instrument.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAA;AAEtC,wBAAgB,kBAAkB,CACjC,QAAQ,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,WAAW,CAAC,EAC9D,MAAM,EAAE,MAAM,GACZ,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,WAAW,CAAC,CAgCtD;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CACvE;AAED,wBAAgB,eAAe,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,CAuB1F"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elsium-ai/observe",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Observability, tracing, and cost tracking for ElsiumAI",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Eric Utrera <ebutrera9103@gmail.com>",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"dev": "bun --watch src/index.ts"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@elsium-ai/core": "^0.
|
|
29
|
+
"@elsium-ai/core": "^0.4.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"typescript": "^5.7.0"
|