@elsium-ai/observe 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -0
- package/dist/experiment.d.ts +6 -0
- package/dist/experiment.d.ts.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +132 -25
- package/dist/instrument.d.ts +9 -0
- package/dist/instrument.d.ts.map +1 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1171,6 +1171,125 @@ await tracer.flush()
|
|
|
1171
1171
|
|
|
1172
1172
|
---
|
|
1173
1173
|
|
|
1174
|
+
## Experiments Persistence
|
|
1175
|
+
|
|
1176
|
+
### `createFileExperimentStore`
|
|
1177
|
+
|
|
1178
|
+
Creates a file-based storage adapter for saving and loading experiment results to disk. Experiment data is serialized as JSON files in the specified directory.
|
|
1179
|
+
|
|
1180
|
+
```ts
|
|
1181
|
+
function createFileExperimentStore(dir: string): ExperimentStore
|
|
1182
|
+
```
|
|
1183
|
+
|
|
1184
|
+
| Parameter | Type | Description |
|
|
1185
|
+
|---|---|---|
|
|
1186
|
+
| `dir` | `string` | Directory path where experiment result files will be stored |
|
|
1187
|
+
|
|
1188
|
+
**Returns:** `ExperimentStore`
|
|
1189
|
+
|
|
1190
|
+
```ts
|
|
1191
|
+
interface ExperimentStore {
|
|
1192
|
+
save(experiment: ExperimentResults): Promise<void>
|
|
1193
|
+
load(experimentId: string): Promise<ExperimentResults | null>
|
|
1194
|
+
list(): Promise<string[]>
|
|
1195
|
+
}
|
|
1196
|
+
```
|
|
1197
|
+
|
|
1198
|
+
```ts
|
|
1199
|
+
import { createExperiment, createFileExperimentStore } from 'elsium-ai/observe'
|
|
1200
|
+
|
|
1201
|
+
const store = createFileExperimentStore('./experiments')
|
|
1202
|
+
|
|
1203
|
+
const experiment = createExperiment({
|
|
1204
|
+
name: 'prompt-comparison',
|
|
1205
|
+
variants: [
|
|
1206
|
+
{ name: 'concise', config: { system: 'Be brief.' } },
|
|
1207
|
+
{ name: 'detailed', config: { system: 'Be thorough.' } },
|
|
1208
|
+
],
|
|
1209
|
+
})
|
|
1210
|
+
|
|
1211
|
+
const results = await experiment.run(evaluator)
|
|
1212
|
+
|
|
1213
|
+
// Persist results to disk
|
|
1214
|
+
await store.save(results)
|
|
1215
|
+
|
|
1216
|
+
// Load results later
|
|
1217
|
+
const loaded = await store.load(results.id)
|
|
1218
|
+
```
|
|
1219
|
+
|
|
1220
|
+
---
|
|
1221
|
+
|
|
1222
|
+
## Auto-Instrumentation
|
|
1223
|
+
|
|
1224
|
+
### `instrumentComplete`
|
|
1225
|
+
|
|
1226
|
+
Wraps an LLM completion function with automatic span creation. Every call produces a span with model, token, cost, and latency metadata.
|
|
1227
|
+
|
|
1228
|
+
```ts
|
|
1229
|
+
function instrumentComplete(
|
|
1230
|
+
complete: (request: CompletionRequest) => Promise<LLMResponse>,
|
|
1231
|
+
tracer: Tracer,
|
|
1232
|
+
): (request: CompletionRequest) => Promise<LLMResponse>
|
|
1233
|
+
```
|
|
1234
|
+
|
|
1235
|
+
| Parameter | Type | Description |
|
|
1236
|
+
|---|---|---|
|
|
1237
|
+
| `complete` | `(request: CompletionRequest) => Promise<LLMResponse>` | The LLM completion function to instrument |
|
|
1238
|
+
| `tracer` | `Tracer` | The tracer instance to record spans to |
|
|
1239
|
+
|
|
1240
|
+
**Returns:** A wrapped completion function with the same signature.
|
|
1241
|
+
|
|
1242
|
+
```ts
|
|
1243
|
+
import { observe, instrumentComplete } from 'elsium-ai/observe'
|
|
1244
|
+
|
|
1245
|
+
const tracer = observe()
|
|
1246
|
+
|
|
1247
|
+
const tracedComplete = instrumentComplete(
|
|
1248
|
+
(req) => llm.complete(req),
|
|
1249
|
+
tracer,
|
|
1250
|
+
)
|
|
1251
|
+
|
|
1252
|
+
// Every call now creates an 'llm' span automatically
|
|
1253
|
+
const response = await tracedComplete({ model: 'gpt-4o', messages })
|
|
1254
|
+
```
|
|
1255
|
+
|
|
1256
|
+
### `instrumentAgent`
|
|
1257
|
+
|
|
1258
|
+
Wraps an agent's `run` method with automatic span creation. Produces an `agent` span that captures the agent name, input, output, token usage, and tool calls.
|
|
1259
|
+
|
|
1260
|
+
```ts
|
|
1261
|
+
function instrumentAgent(
|
|
1262
|
+
agent: Agent,
|
|
1263
|
+
tracer: Tracer,
|
|
1264
|
+
): Agent
|
|
1265
|
+
```
|
|
1266
|
+
|
|
1267
|
+
| Parameter | Type | Description |
|
|
1268
|
+
|---|---|---|
|
|
1269
|
+
| `agent` | `Agent` | The agent to instrument |
|
|
1270
|
+
| `tracer` | `Tracer` | The tracer instance to record spans to |
|
|
1271
|
+
|
|
1272
|
+
**Returns:** A new `Agent` with the same interface, where `run` and `chat` are automatically traced.
|
|
1273
|
+
|
|
1274
|
+
```ts
|
|
1275
|
+
import { observe, instrumentAgent } from 'elsium-ai/observe'
|
|
1276
|
+
import { defineAgent } from 'elsium-ai/agents'
|
|
1277
|
+
|
|
1278
|
+
const tracer = observe()
|
|
1279
|
+
|
|
1280
|
+
const agent = defineAgent(
|
|
1281
|
+
{ name: 'assistant', system: 'You are helpful.' },
|
|
1282
|
+
{ complete: (req) => llm.complete(req) },
|
|
1283
|
+
)
|
|
1284
|
+
|
|
1285
|
+
const tracedAgent = instrumentAgent(agent, tracer)
|
|
1286
|
+
|
|
1287
|
+
// Every run/chat call now creates an 'agent' span automatically
|
|
1288
|
+
const result = await tracedAgent.run('Hello')
|
|
1289
|
+
```
|
|
1290
|
+
|
|
1291
|
+
---
|
|
1292
|
+
|
|
1174
1293
|
## Part of ElsiumAI
|
|
1175
1294
|
|
|
1176
1295
|
This package is the observability layer of the [ElsiumAI](https://github.com/elsium-ai/elsium-ai) framework. See the [full documentation](https://github.com/elsium-ai/elsium-ai) for guides and examples.
|
package/dist/experiment.d.ts
CHANGED
|
@@ -20,9 +20,15 @@ export interface Experiment {
|
|
|
20
20
|
record(variant: string, metrics: Record<string, number>): void;
|
|
21
21
|
results(): ExperimentResults;
|
|
22
22
|
}
|
|
23
|
+
export interface ExperimentStore {
|
|
24
|
+
save(name: string, data: ExperimentResults): void;
|
|
25
|
+
load(name: string): ExperimentResults | null;
|
|
26
|
+
}
|
|
23
27
|
export interface ExperimentConfig {
|
|
24
28
|
name: string;
|
|
25
29
|
variants: ExperimentVariant[];
|
|
30
|
+
store?: ExperimentStore;
|
|
26
31
|
}
|
|
32
|
+
export declare function createFileExperimentStore(dir: string): ExperimentStore;
|
|
27
33
|
export declare function createExperiment(config: ExperimentConfig): Experiment;
|
|
28
34
|
//# sourceMappingURL=experiment.d.ts.map
|
package/dist/experiment.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../src/experiment.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../src/experiment.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,gBAAgB,EAAE,MAAM,CAAA;IACxB,QAAQ,EAAE,MAAM,CACf,MAAM,EACN;QACC,WAAW,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACpE,CACD,CAAA;CACD;AAED,MAAM,WAAW,UAAU;IAC1B,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAA;IAC1C,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAA;IAC9D,OAAO,IAAI,iBAAiB,CAAA;CAC5B;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,GAAG,IAAI,CAAA;IACjD,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAAA;CAC5C;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,iBAAiB,EAAE,CAAA;IAC7B,KAAK,CAAC,EAAE,eAAe,CAAA;CACvB;AAED,wBAAgB,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CA4BtE;AAyDD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,GAAG,UAAU,CAmErE"}
|
package/dist/index.d.ts
CHANGED
|
@@ -10,8 +10,10 @@ export { createAuditTrail, auditMiddleware } from './audit';
|
|
|
10
10
|
export type { AuditEventType, AuditEvent, AuditStorageAdapter, AuditQueryFilter, AuditIntegrityResult, AuditTrailConfig, AuditTrail, } from './audit';
|
|
11
11
|
export { createProvenanceTracker } from './provenance';
|
|
12
12
|
export type { ProvenanceRecord, ProvenanceTracker } from './provenance';
|
|
13
|
-
export { createExperiment } from './experiment';
|
|
14
|
-
export type { Experiment, ExperimentConfig, ExperimentVariant, ExperimentResults, } from './experiment';
|
|
13
|
+
export { createExperiment, createFileExperimentStore } from './experiment';
|
|
14
|
+
export type { Experiment, ExperimentConfig, ExperimentVariant, ExperimentResults, ExperimentStore, } from './experiment';
|
|
15
|
+
export { instrumentComplete, instrumentAgent } from './instrument';
|
|
16
|
+
export type { InstrumentableAgent } from './instrument';
|
|
15
17
|
export { toOTelSpan, toOTelExportRequest, toTraceparent, parseTraceparent, injectTraceContext, extractTraceContext, createOTLPExporter, } from './otel';
|
|
16
18
|
export type { OTelSpan, OTelSpanKind, OTelStatusCode, OTelAttribute, OTelAttributeValue, OTelEvent, OTelResource, OTelExportRequest, TraceContext, OTLPExporterConfig, } from './otel';
|
|
17
19
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACnC,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAG1F,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACnE,YAAY,EACX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,EACnB,SAAS,EACT,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,cAAc,GACd,MAAM,eAAe,CAAA;AAGtB,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAClC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAG9F,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACzC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAG9D,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC3D,YAAY,EACX,cAAc,EACd,UAAU,EACV,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,gBAAgB,EAChB,UAAU,GACV,MAAM,SAAS,CAAA;AAGhB,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACtD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAGvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACnC,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAG1F,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AACnE,YAAY,EACX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,mBAAmB,EACnB,SAAS,EACT,aAAa,EACb,sBAAsB,EACtB,eAAe,EACf,cAAc,GACd,MAAM,eAAe,CAAA;AAGtB,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAClC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAG9F,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACzC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAA;AAG9D,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC3D,YAAY,EACX,cAAc,EACd,UAAU,EACV,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,gBAAgB,EAChB,UAAU,GACV,MAAM,SAAS,CAAA;AAGhB,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACtD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAGvE,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,MAAM,cAAc,CAAA;AAC1E,YAAY,EACX,UAAU,EACV,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,GACf,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAClE,YAAY,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAGvD,OAAO,EACN,UAAU,EACV,mBAAmB,EACnB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,GAClB,MAAM,QAAQ,CAAA;AACf,YAAY,EACX,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,GAClB,MAAM,QAAQ,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -927,9 +927,83 @@ function createProvenanceTracker(options) {
|
|
|
927
927
|
}
|
|
928
928
|
// src/experiment.ts
|
|
929
929
|
import { createHash as createHash3 } from "node:crypto";
|
|
930
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync as writeFileSync2 } from "node:fs";
|
|
931
|
+
import { join } from "node:path";
|
|
930
932
|
var log4 = createLogger();
|
|
933
|
+
function createFileExperimentStore(dir) {
|
|
934
|
+
return {
|
|
935
|
+
save(name, data) {
|
|
936
|
+
try {
|
|
937
|
+
if (!existsSync(dir)) {
|
|
938
|
+
mkdirSync(dir, { recursive: true });
|
|
939
|
+
}
|
|
940
|
+
const filePath = join(dir, `${name}.json`);
|
|
941
|
+
writeFileSync2(filePath, JSON.stringify(data, null, 2));
|
|
942
|
+
} catch (err2) {
|
|
943
|
+
log4.error("Failed to save experiment", {
|
|
944
|
+
name,
|
|
945
|
+
error: err2 instanceof Error ? err2.message : String(err2)
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
},
|
|
949
|
+
load(name) {
|
|
950
|
+
try {
|
|
951
|
+
const filePath = join(dir, `${name}.json`);
|
|
952
|
+
if (!existsSync(filePath))
|
|
953
|
+
return null;
|
|
954
|
+
const raw = readFileSync(filePath, "utf-8");
|
|
955
|
+
return JSON.parse(raw);
|
|
956
|
+
} catch {
|
|
957
|
+
return null;
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
function loadFromStore(store, name, stats) {
|
|
963
|
+
const saved = store.load(name);
|
|
964
|
+
if (!saved)
|
|
965
|
+
return;
|
|
966
|
+
for (const [vName, vData] of Object.entries(saved.variants)) {
|
|
967
|
+
if (!stats[vName])
|
|
968
|
+
continue;
|
|
969
|
+
stats[vName].assignments = vData.assignments;
|
|
970
|
+
for (const [key, m] of Object.entries(vData.metrics)) {
|
|
971
|
+
stats[vName].metrics[key] = { sum: m.sum, count: m.count };
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
log4.debug("Loaded experiment state", { name, totalAssignments: saved.totalAssignments });
|
|
975
|
+
}
|
|
976
|
+
function recordMetrics(s, metrics) {
|
|
977
|
+
for (const [key, value] of Object.entries(metrics)) {
|
|
978
|
+
if (!s.metrics[key]) {
|
|
979
|
+
s.metrics[key] = { sum: 0, count: 0 };
|
|
980
|
+
}
|
|
981
|
+
s.metrics[key].sum += value;
|
|
982
|
+
s.metrics[key].count++;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
function buildResults(name, stats) {
|
|
986
|
+
let totalAssignments = 0;
|
|
987
|
+
const variantResults = {};
|
|
988
|
+
for (const [vName, s] of Object.entries(stats)) {
|
|
989
|
+
totalAssignments += s.assignments;
|
|
990
|
+
const metricsResult = {};
|
|
991
|
+
for (const [key, m] of Object.entries(s.metrics)) {
|
|
992
|
+
metricsResult[key] = {
|
|
993
|
+
sum: m.sum,
|
|
994
|
+
count: m.count,
|
|
995
|
+
avg: m.count > 0 ? m.sum / m.count : 0
|
|
996
|
+
};
|
|
997
|
+
}
|
|
998
|
+
variantResults[vName] = {
|
|
999
|
+
assignments: s.assignments,
|
|
1000
|
+
metrics: metricsResult
|
|
1001
|
+
};
|
|
1002
|
+
}
|
|
1003
|
+
return { name, totalAssignments, variants: variantResults };
|
|
1004
|
+
}
|
|
931
1005
|
function createExperiment(config) {
|
|
932
|
-
const { name, variants } = config;
|
|
1006
|
+
const { name, variants, store } = config;
|
|
933
1007
|
if (variants.length === 0) {
|
|
934
1008
|
throw new Error("Experiment must have at least one variant");
|
|
935
1009
|
}
|
|
@@ -938,6 +1012,9 @@ function createExperiment(config) {
|
|
|
938
1012
|
for (const v of variants) {
|
|
939
1013
|
stats[v.name] = { assignments: 0, metrics: {} };
|
|
940
1014
|
}
|
|
1015
|
+
if (store) {
|
|
1016
|
+
loadFromStore(store, name, stats);
|
|
1017
|
+
}
|
|
941
1018
|
function hashAssign(userId) {
|
|
942
1019
|
const hash = createHash3("sha256").update(`${name}:${userId}`).digest();
|
|
943
1020
|
const value = hash.readUInt32BE(0) % 1e4 / 1e4;
|
|
@@ -973,35 +1050,62 @@ function createExperiment(config) {
|
|
|
973
1050
|
const s = stats[variant];
|
|
974
1051
|
if (!s)
|
|
975
1052
|
return;
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
}
|
|
980
|
-
s.metrics[key].sum += value;
|
|
981
|
-
s.metrics[key].count++;
|
|
1053
|
+
recordMetrics(s, metrics);
|
|
1054
|
+
if (store) {
|
|
1055
|
+
store.save(name, this.results());
|
|
982
1056
|
}
|
|
983
1057
|
},
|
|
984
1058
|
results() {
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1059
|
+
return buildResults(name, stats);
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
// src/instrument.ts
|
|
1064
|
+
function instrumentComplete(complete, tracer) {
|
|
1065
|
+
return async (request) => {
|
|
1066
|
+
const span = tracer.startSpan("llm.complete", "llm");
|
|
1067
|
+
span.setMetadata("model", request.model ?? "default");
|
|
1068
|
+
span.setMetadata("messageCount", request.messages.length);
|
|
1069
|
+
try {
|
|
1070
|
+
const response = await complete(request);
|
|
1071
|
+
span.setMetadata("inputTokens", response.usage.inputTokens);
|
|
1072
|
+
span.setMetadata("outputTokens", response.usage.outputTokens);
|
|
1073
|
+
span.setMetadata("totalCost", response.cost.totalCost);
|
|
1074
|
+
span.setMetadata("provider", response.provider);
|
|
1075
|
+
span.setMetadata("latencyMs", response.latencyMs);
|
|
1076
|
+
tracer.trackLLMCall({
|
|
1077
|
+
model: response.model,
|
|
1078
|
+
inputTokens: response.usage.inputTokens,
|
|
1079
|
+
outputTokens: response.usage.outputTokens,
|
|
1080
|
+
cost: response.cost.totalCost,
|
|
1081
|
+
latencyMs: response.latencyMs
|
|
1082
|
+
});
|
|
1083
|
+
span.end({ status: "ok" });
|
|
1084
|
+
return response;
|
|
1085
|
+
} catch (error) {
|
|
1086
|
+
span.setMetadata("error", error instanceof Error ? error.message : String(error));
|
|
1087
|
+
span.end({ status: "error" });
|
|
1088
|
+
throw error;
|
|
1089
|
+
}
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
function instrumentAgent(agent, tracer) {
|
|
1093
|
+
const originalRun = agent.run.bind(agent);
|
|
1094
|
+
const instrumented = Object.create(agent);
|
|
1095
|
+
instrumented.run = async (input, options) => {
|
|
1096
|
+
const span = tracer.startSpan(`agent.${agent.name}`, "agent");
|
|
1097
|
+
span.setMetadata("agentName", agent.name);
|
|
1098
|
+
try {
|
|
1099
|
+
const result = await originalRun(input, options);
|
|
1100
|
+
span.end({ status: "ok" });
|
|
1101
|
+
return result;
|
|
1102
|
+
} catch (error) {
|
|
1103
|
+
span.setMetadata("error", error instanceof Error ? error.message : String(error));
|
|
1104
|
+
span.end({ status: "error" });
|
|
1105
|
+
throw error;
|
|
1003
1106
|
}
|
|
1004
1107
|
};
|
|
1108
|
+
return instrumented;
|
|
1005
1109
|
}
|
|
1006
1110
|
// src/otel.ts
|
|
1007
1111
|
var log5 = createLogger();
|
|
@@ -1191,12 +1295,15 @@ export {
|
|
|
1191
1295
|
registerModelTier,
|
|
1192
1296
|
parseTraceparent,
|
|
1193
1297
|
observe,
|
|
1298
|
+
instrumentComplete,
|
|
1299
|
+
instrumentAgent,
|
|
1194
1300
|
injectTraceContext,
|
|
1195
1301
|
extractTraceContext,
|
|
1196
1302
|
createSpan,
|
|
1197
1303
|
createProvenanceTracker,
|
|
1198
1304
|
createOTLPExporter,
|
|
1199
1305
|
createMetrics,
|
|
1306
|
+
createFileExperimentStore,
|
|
1200
1307
|
createExperiment,
|
|
1201
1308
|
createCostEngine,
|
|
1202
1309
|
createAuditTrail,
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { CompletionRequest, LLMResponse } from '@elsium-ai/core';
|
|
2
|
+
import type { Tracer } from './tracer';
|
|
3
|
+
export declare function instrumentComplete(complete: (request: CompletionRequest) => Promise<LLMResponse>, tracer: Tracer): (request: CompletionRequest) => Promise<LLMResponse>;
|
|
4
|
+
export interface InstrumentableAgent {
|
|
5
|
+
readonly name: string;
|
|
6
|
+
run(input: string, options?: Record<string, unknown>): Promise<unknown>;
|
|
7
|
+
}
|
|
8
|
+
export declare function instrumentAgent<T extends InstrumentableAgent>(agent: T, tracer: Tracer): T;
|
|
9
|
+
//# sourceMappingURL=instrument.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"instrument.d.ts","sourceRoot":"","sources":["../src/instrument.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAA;AAEtC,wBAAgB,kBAAkB,CACjC,QAAQ,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,WAAW,CAAC,EAC9D,MAAM,EAAE,MAAM,GACZ,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,WAAW,CAAC,CAgCtD;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CACvE;AAED,wBAAgB,eAAe,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,CAuB1F"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elsium-ai/observe",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Observability, tracing, and cost tracking for ElsiumAI",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Eric Utrera <ebutrera9103@gmail.com>",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"dev": "bun --watch src/index.ts"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@elsium-ai/core": "^0.
|
|
29
|
+
"@elsium-ai/core": "^0.4.1"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
32
|
"typescript": "^5.7.0"
|