@launchdarkly/server-sdk-ai 0.16.8 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/index.cjs +669 -184
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +480 -45
- package/dist/index.d.ts +480 -45
- package/dist/index.js +666 -184
- package/dist/index.js.map +1 -1
- package/package.json +1 -3
package/dist/index.cjs
CHANGED
|
@@ -32,8 +32,11 @@ var src_exports = {};
|
|
|
32
32
|
__export(src_exports, {
|
|
33
33
|
AIProvider: () => AIProvider,
|
|
34
34
|
AIProviderFactory: () => AIProviderFactory,
|
|
35
|
+
AgentGraphDefinition: () => AgentGraphDefinition,
|
|
36
|
+
AgentGraphNode: () => AgentGraphNode,
|
|
35
37
|
Judge: () => Judge,
|
|
36
38
|
LDFeedbackKind: () => LDFeedbackKind,
|
|
39
|
+
LDGraphTrackerImpl: () => LDGraphTrackerImpl,
|
|
37
40
|
SUPPORTED_AI_PROVIDERS: () => SUPPORTED_AI_PROVIDERS,
|
|
38
41
|
TrackedChat: () => TrackedChat,
|
|
39
42
|
createBedrockTokenUsage: () => createBedrockTokenUsage,
|
|
@@ -45,12 +48,12 @@ module.exports = __toCommonJS(src_exports);
|
|
|
45
48
|
|
|
46
49
|
// src/LDAIClientImpl.ts
|
|
47
50
|
var import_mustache2 = __toESM(require("mustache"), 1);
|
|
51
|
+
var import_node_crypto = require("crypto");
|
|
48
52
|
|
|
49
53
|
// src/api/chat/TrackedChat.ts
|
|
50
54
|
var TrackedChat = class {
|
|
51
|
-
constructor(aiConfig,
|
|
55
|
+
constructor(aiConfig, provider, judges = {}, _logger) {
|
|
52
56
|
this.aiConfig = aiConfig;
|
|
53
|
-
this.tracker = tracker;
|
|
54
57
|
this.provider = provider;
|
|
55
58
|
this.judges = judges;
|
|
56
59
|
this._logger = _logger;
|
|
@@ -61,6 +64,7 @@ var TrackedChat = class {
|
|
|
61
64
|
* This method handles conversation management and tracking, delegating to the provider's invokeModel method.
|
|
62
65
|
*/
|
|
63
66
|
async invoke(prompt) {
|
|
67
|
+
const tracker = this.aiConfig.createTracker();
|
|
64
68
|
const userMessage = {
|
|
65
69
|
role: "user",
|
|
66
70
|
content: prompt
|
|
@@ -68,12 +72,19 @@ var TrackedChat = class {
|
|
|
68
72
|
this.messages.push(userMessage);
|
|
69
73
|
const configMessages = this.aiConfig.messages || [];
|
|
70
74
|
const allMessages = [...configMessages, ...this.messages];
|
|
71
|
-
const response = await
|
|
75
|
+
const response = await tracker.trackMetricsOf(
|
|
72
76
|
(result) => result.metrics,
|
|
73
77
|
() => this.provider.invokeModel(allMessages)
|
|
74
78
|
);
|
|
75
79
|
if (this.aiConfig.judgeConfiguration?.judges && this.aiConfig.judgeConfiguration.judges.length > 0) {
|
|
76
|
-
response.evaluations = this._evaluateWithJudges(this.messages, response)
|
|
80
|
+
response.evaluations = this._evaluateWithJudges(this.messages, response).then(
|
|
81
|
+
(evaluations) => {
|
|
82
|
+
evaluations.forEach((judgeResult) => {
|
|
83
|
+
tracker.trackJudgeResult(judgeResult);
|
|
84
|
+
});
|
|
85
|
+
return evaluations;
|
|
86
|
+
}
|
|
87
|
+
);
|
|
77
88
|
}
|
|
78
89
|
this.messages.push(response.message);
|
|
79
90
|
return response;
|
|
@@ -92,23 +103,29 @@ var TrackedChat = class {
|
|
|
92
103
|
const judge = this.judges[judgeConfig.key];
|
|
93
104
|
if (!judge) {
|
|
94
105
|
this._logger?.warn(
|
|
95
|
-
`Judge configuration is not enabled
|
|
96
|
-
this.tracker.getTrackData()
|
|
106
|
+
`Judge configuration is not enabled for ${judgeConfig.key} in ${this.aiConfig.key}`
|
|
97
107
|
);
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
);
|
|
105
|
-
if (judgeResponse && judgeResponse.success) {
|
|
106
|
-
this.tracker.trackJudgeResponse(judgeResponse);
|
|
108
|
+
const result = {
|
|
109
|
+
success: false,
|
|
110
|
+
sampled: true,
|
|
111
|
+
errorMessage: `Judge configuration is not enabled for ${judgeConfig.key}`
|
|
112
|
+
};
|
|
113
|
+
return result;
|
|
107
114
|
}
|
|
108
|
-
return
|
|
115
|
+
return judge.evaluateMessages(messages, response, judgeConfig.samplingRate);
|
|
109
116
|
});
|
|
110
117
|
const results = await Promise.allSettled(evaluationPromises);
|
|
111
|
-
return results.map((
|
|
118
|
+
return results.map((settled) => {
|
|
119
|
+
if (settled.status === "fulfilled") {
|
|
120
|
+
return settled.value;
|
|
121
|
+
}
|
|
122
|
+
const result = {
|
|
123
|
+
success: false,
|
|
124
|
+
sampled: true,
|
|
125
|
+
errorMessage: "Judge evaluation failed"
|
|
126
|
+
};
|
|
127
|
+
return result;
|
|
128
|
+
});
|
|
112
129
|
}
|
|
113
130
|
/**
|
|
114
131
|
* Get the underlying AI configuration used to initialize this TrackedChat.
|
|
@@ -116,12 +133,6 @@ var TrackedChat = class {
|
|
|
116
133
|
getConfig() {
|
|
117
134
|
return this.aiConfig;
|
|
118
135
|
}
|
|
119
|
-
/**
|
|
120
|
-
* Get the underlying AI configuration tracker used to initialize this TrackedChat.
|
|
121
|
-
*/
|
|
122
|
-
getTracker() {
|
|
123
|
-
return this.tracker;
|
|
124
|
-
}
|
|
125
136
|
/**
|
|
126
137
|
* Get the underlying AI provider instance.
|
|
127
138
|
* This provides direct access to the provider for advanced use cases.
|
|
@@ -206,20 +217,21 @@ var LDAIConfigUtils = class {
|
|
|
206
217
|
/**
|
|
207
218
|
* Converts a LaunchDarkly flag value to the appropriate AI configuration type.
|
|
208
219
|
*
|
|
220
|
+
* @param key The configuration key
|
|
209
221
|
* @param flagValue The flag value from LaunchDarkly
|
|
210
|
-
* @param
|
|
222
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
211
223
|
* @returns The appropriate AI configuration type
|
|
212
224
|
*/
|
|
213
|
-
static fromFlagValue(key, flagValue,
|
|
225
|
+
static fromFlagValue(key, flagValue, trackerFactory) {
|
|
214
226
|
const flagValueMode = flagValue._ldMeta?.mode;
|
|
215
227
|
switch (flagValueMode) {
|
|
216
228
|
case "agent":
|
|
217
|
-
return this.toAgentConfig(key, flagValue,
|
|
229
|
+
return this.toAgentConfig(key, flagValue, trackerFactory);
|
|
218
230
|
case "judge":
|
|
219
|
-
return this.toJudgeConfig(key, flagValue,
|
|
231
|
+
return this.toJudgeConfig(key, flagValue, trackerFactory);
|
|
220
232
|
case "completion":
|
|
221
233
|
default:
|
|
222
|
-
return this.toCompletionConfig(key, flagValue,
|
|
234
|
+
return this.toCompletionConfig(key, flagValue, trackerFactory);
|
|
223
235
|
}
|
|
224
236
|
}
|
|
225
237
|
/**
|
|
@@ -234,20 +246,20 @@ var LDAIConfigUtils = class {
|
|
|
234
246
|
return {
|
|
235
247
|
key,
|
|
236
248
|
enabled: false,
|
|
237
|
-
|
|
249
|
+
createTracker: void 0
|
|
238
250
|
};
|
|
239
251
|
case "judge":
|
|
240
252
|
return {
|
|
241
253
|
key,
|
|
242
254
|
enabled: false,
|
|
243
|
-
|
|
255
|
+
createTracker: void 0
|
|
244
256
|
};
|
|
245
257
|
case "completion":
|
|
246
258
|
default:
|
|
247
259
|
return {
|
|
248
260
|
key,
|
|
249
261
|
enabled: false,
|
|
250
|
-
|
|
262
|
+
createTracker: void 0
|
|
251
263
|
};
|
|
252
264
|
}
|
|
253
265
|
}
|
|
@@ -269,14 +281,15 @@ var LDAIConfigUtils = class {
|
|
|
269
281
|
/**
|
|
270
282
|
* Creates a completion config from flag value data.
|
|
271
283
|
*
|
|
284
|
+
* @param key The configuration key
|
|
272
285
|
* @param flagValue The flag value from LaunchDarkly
|
|
273
|
-
* @param
|
|
286
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
274
287
|
* @returns A completion configuration
|
|
275
288
|
*/
|
|
276
|
-
static toCompletionConfig(key, flagValue,
|
|
289
|
+
static toCompletionConfig(key, flagValue, trackerFactory) {
|
|
277
290
|
return {
|
|
278
291
|
...this._toBaseConfig(key, flagValue),
|
|
279
|
-
|
|
292
|
+
createTracker: trackerFactory,
|
|
280
293
|
messages: flagValue.messages,
|
|
281
294
|
judgeConfiguration: flagValue.judgeConfiguration
|
|
282
295
|
};
|
|
@@ -284,14 +297,15 @@ var LDAIConfigUtils = class {
|
|
|
284
297
|
/**
|
|
285
298
|
* Creates an agent config from flag value data.
|
|
286
299
|
*
|
|
300
|
+
* @param key The configuration key
|
|
287
301
|
* @param flagValue The flag value from LaunchDarkly
|
|
288
|
-
* @param
|
|
302
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
289
303
|
* @returns An agent configuration
|
|
290
304
|
*/
|
|
291
|
-
static toAgentConfig(key, flagValue,
|
|
305
|
+
static toAgentConfig(key, flagValue, trackerFactory) {
|
|
292
306
|
return {
|
|
293
307
|
...this._toBaseConfig(key, flagValue),
|
|
294
|
-
|
|
308
|
+
createTracker: trackerFactory,
|
|
295
309
|
instructions: flagValue.instructions,
|
|
296
310
|
judgeConfiguration: flagValue.judgeConfiguration
|
|
297
311
|
};
|
|
@@ -299,11 +313,12 @@ var LDAIConfigUtils = class {
|
|
|
299
313
|
/**
|
|
300
314
|
* Creates a judge config from flag value data.
|
|
301
315
|
*
|
|
316
|
+
* @param key The configuration key
|
|
302
317
|
* @param flagValue The flag value from LaunchDarkly
|
|
303
|
-
* @param
|
|
318
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
304
319
|
* @returns A judge configuration
|
|
305
320
|
*/
|
|
306
|
-
static toJudgeConfig(key, flagValue,
|
|
321
|
+
static toJudgeConfig(key, flagValue, trackerFactory) {
|
|
307
322
|
let evaluationMetricKey;
|
|
308
323
|
if (flagValue.evaluationMetricKey && flagValue.evaluationMetricKey.trim().length > 0) {
|
|
309
324
|
evaluationMetricKey = flagValue.evaluationMetricKey.trim();
|
|
@@ -315,69 +330,269 @@ var LDAIConfigUtils = class {
|
|
|
315
330
|
}
|
|
316
331
|
return {
|
|
317
332
|
...this._toBaseConfig(key, flagValue),
|
|
318
|
-
|
|
333
|
+
createTracker: trackerFactory,
|
|
319
334
|
messages: flagValue.messages,
|
|
320
335
|
evaluationMetricKey
|
|
321
336
|
};
|
|
322
337
|
}
|
|
323
338
|
};
|
|
324
339
|
|
|
325
|
-
// src/api/
|
|
326
|
-
var
|
|
340
|
+
// src/api/graph/AgentGraphNode.ts
|
|
341
|
+
var AgentGraphNode = class {
|
|
342
|
+
constructor(_key, _config, _edges) {
|
|
343
|
+
this._key = _key;
|
|
344
|
+
this._config = _config;
|
|
345
|
+
this._edges = _edges;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Returns the agent config key that identifies this node in the graph.
|
|
349
|
+
*/
|
|
350
|
+
getKey() {
|
|
351
|
+
return this._key;
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Returns the underlying AIAgentConfig for this node.
|
|
355
|
+
* Use `getConfig().tracker` to record node-level metrics.
|
|
356
|
+
*/
|
|
357
|
+
getConfig() {
|
|
358
|
+
return this._config;
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Returns the outgoing edges from this node to its children.
|
|
362
|
+
*/
|
|
363
|
+
getEdges() {
|
|
364
|
+
return this._edges;
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Returns `true` if this node has no outgoing edges (i.e., it is a terminal/leaf node).
|
|
368
|
+
*/
|
|
369
|
+
isTerminal() {
|
|
370
|
+
return this._edges.length === 0;
|
|
371
|
+
}
|
|
372
|
+
};
|
|
327
373
|
|
|
328
|
-
// src/api/
|
|
329
|
-
var
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
374
|
+
// src/api/graph/AgentGraphDefinition.ts
|
|
375
|
+
var AgentGraphDefinition = class _AgentGraphDefinition {
|
|
376
|
+
constructor(_agentGraph, _nodes, enabled, _createTracker) {
|
|
377
|
+
this._agentGraph = _agentGraph;
|
|
378
|
+
this._nodes = _nodes;
|
|
379
|
+
this.enabled = enabled;
|
|
380
|
+
this._createTracker = _createTracker;
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Builds a node map from a raw agent graph flag value and a map of pre-fetched agent configs.
|
|
384
|
+
*
|
|
385
|
+
* @param graph Raw graph flag value from LaunchDarkly.
|
|
386
|
+
* @param agentConfigs Map of agent config key to resolved LDAIAgentConfig.
|
|
387
|
+
* @returns Record mapping agent config keys to AgentGraphNode instances.
|
|
388
|
+
*/
|
|
389
|
+
static buildNodes(graph, agentConfigs) {
|
|
390
|
+
const nodes = {};
|
|
391
|
+
const allKeys = _AgentGraphDefinition.collectAllKeys(graph);
|
|
392
|
+
allKeys.forEach((key) => {
|
|
393
|
+
const config = agentConfigs[key];
|
|
394
|
+
if (!config) {
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
const outgoingEdges = graph.edges?.[key] ?? [];
|
|
398
|
+
nodes[key] = new AgentGraphNode(key, config, outgoingEdges);
|
|
399
|
+
});
|
|
400
|
+
return nodes;
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Returns the children of the node identified by `nodeKey`.
|
|
404
|
+
*
|
|
405
|
+
* @param nodeKey The agent config key of the parent node.
|
|
406
|
+
*/
|
|
407
|
+
getChildNodes(nodeKey) {
|
|
408
|
+
const node = this._nodes[nodeKey];
|
|
409
|
+
if (!node) {
|
|
410
|
+
return [];
|
|
333
411
|
}
|
|
334
|
-
return
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
412
|
+
return node.getEdges().map((edge) => this._nodes[edge.key]).filter((n) => n !== void 0);
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Returns all nodes that have a direct edge to the node identified by `nodeKey`.
|
|
416
|
+
*
|
|
417
|
+
* @param nodeKey The agent config key of the child node.
|
|
418
|
+
*/
|
|
419
|
+
getParentNodes(nodeKey) {
|
|
420
|
+
return Object.values(this._nodes).filter(
|
|
421
|
+
(node) => node.getEdges().some((edge) => edge.key === nodeKey)
|
|
422
|
+
);
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Returns all terminal nodes (nodes with no outgoing edges).
|
|
426
|
+
*/
|
|
427
|
+
terminalNodes() {
|
|
428
|
+
return Object.values(this._nodes).filter((node) => node.isTerminal());
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Returns the root node of the graph.
|
|
432
|
+
*/
|
|
433
|
+
rootNode() {
|
|
434
|
+
return this._nodes[this._agentGraph.root];
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Returns the node with the given key, or `null` if not found.
|
|
438
|
+
*
|
|
439
|
+
* @param nodeKey The agent config key to look up.
|
|
440
|
+
*/
|
|
441
|
+
getNode(nodeKey) {
|
|
442
|
+
return this._nodes[nodeKey] ?? null;
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Returns the underlying raw graph configuration from LaunchDarkly.
|
|
446
|
+
*/
|
|
447
|
+
getConfig() {
|
|
448
|
+
return this._agentGraph;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Returns a new {@link LDGraphTracker} for this graph invocation.
|
|
452
|
+
*
|
|
453
|
+
* Call this once per invocation. Each call produces a tracker with a fresh `runId`
|
|
454
|
+
* that groups all events for that invocation.
|
|
455
|
+
*/
|
|
456
|
+
createTracker() {
|
|
457
|
+
return this._createTracker();
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Traverses the graph breadth-first from the root to all terminal nodes.
|
|
461
|
+
*
|
|
462
|
+
* Nodes at the same depth are processed before advancing to the next depth.
|
|
463
|
+
* The value returned by `fn` is stored in the mutable `executionContext` under
|
|
464
|
+
* the node's key, making upstream results available to downstream nodes.
|
|
465
|
+
*
|
|
466
|
+
* Cyclic graphs are handled safely — each node is visited at most once.
|
|
467
|
+
*
|
|
468
|
+
* @param fn Callback invoked for each node. Its return value is added to
|
|
469
|
+
* `executionContext` keyed by the node's config key.
|
|
470
|
+
* @param initialExecutionContext Optional initial context to seed the traversal.
|
|
471
|
+
*/
|
|
472
|
+
traverse(fn, initialExecutionContext = {}) {
|
|
473
|
+
const root = this.rootNode();
|
|
474
|
+
if (!root) {
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
const executionContext = { ...initialExecutionContext };
|
|
478
|
+
const visited = /* @__PURE__ */ new Set();
|
|
479
|
+
const queue = [root];
|
|
480
|
+
visited.add(root.getKey());
|
|
481
|
+
while (queue.length > 0) {
|
|
482
|
+
const node = queue.shift();
|
|
483
|
+
const result = fn(node, executionContext);
|
|
484
|
+
executionContext[node.getKey()] = result;
|
|
485
|
+
node.getEdges().forEach((edge) => {
|
|
486
|
+
if (!visited.has(edge.key)) {
|
|
487
|
+
const child = this._nodes[edge.key];
|
|
488
|
+
if (child) {
|
|
489
|
+
visited.add(edge.key);
|
|
490
|
+
queue.push(child);
|
|
491
|
+
}
|
|
345
492
|
}
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
additionalProperties: false
|
|
349
|
-
};
|
|
493
|
+
});
|
|
494
|
+
}
|
|
350
495
|
}
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
496
|
+
/**
|
|
497
|
+
* Traverses the graph from terminal nodes up to the root.
|
|
498
|
+
*
|
|
499
|
+
* Uses BFS upward via parent edges so that each node is processed only after
|
|
500
|
+
* all of its reachable descendants have been processed. The root is always
|
|
501
|
+
* visited last. Cyclic graphs are handled safely — each node is visited at
|
|
502
|
+
* most once; if the graph has no terminal nodes, this method returns without
|
|
503
|
+
* invoking `fn`.
|
|
504
|
+
*
|
|
505
|
+
* **Ordering note:** Within a single BFS level (nodes at the same depth from a
|
|
506
|
+
* terminal) the visit order is not strictly guaranteed. The guarantee is only
|
|
507
|
+
* that a node is visited before any of its ancestors — not that siblings at the
|
|
508
|
+
* same depth are visited in a specific order relative to each other.
|
|
509
|
+
*
|
|
510
|
+
* The value returned by `fn` is stored in the mutable `executionContext` under
|
|
511
|
+
* the node's key.
|
|
512
|
+
*
|
|
513
|
+
* @param fn Callback invoked for each node. Its return value is added to
|
|
514
|
+
* `executionContext` keyed by the node's config key.
|
|
515
|
+
* @param initialExecutionContext Optional initial context to seed the traversal.
|
|
516
|
+
*/
|
|
517
|
+
reverseTraverse(fn, initialExecutionContext = {}) {
|
|
518
|
+
const terminals = this.terminalNodes();
|
|
519
|
+
if (terminals.length === 0) {
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
const executionContext = { ...initialExecutionContext };
|
|
523
|
+
const rootKey = this._agentGraph.root;
|
|
524
|
+
const visited = /* @__PURE__ */ new Set();
|
|
525
|
+
let queue = terminals;
|
|
526
|
+
while (queue.length > 0) {
|
|
527
|
+
const nextQueue = [];
|
|
528
|
+
queue.forEach((node) => {
|
|
529
|
+
const key = node.getKey();
|
|
530
|
+
if (visited.has(key)) {
|
|
531
|
+
return;
|
|
364
532
|
}
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
533
|
+
visited.add(key);
|
|
534
|
+
if (key === rootKey) {
|
|
535
|
+
return;
|
|
536
|
+
}
|
|
537
|
+
const result = fn(node, executionContext);
|
|
538
|
+
executionContext[key] = result;
|
|
539
|
+
this.getParentNodes(key).forEach((parent) => {
|
|
540
|
+
if (!visited.has(parent.getKey())) {
|
|
541
|
+
nextQueue.push(parent);
|
|
542
|
+
}
|
|
543
|
+
});
|
|
544
|
+
});
|
|
545
|
+
queue = nextQueue;
|
|
546
|
+
}
|
|
547
|
+
const root = this._nodes[rootKey];
|
|
548
|
+
if (root && visited.has(rootKey)) {
|
|
549
|
+
const result = fn(root, executionContext);
|
|
550
|
+
executionContext[rootKey] = result;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
/**
|
|
554
|
+
* Collects every unique node key referenced in the graph (root + all edge sources
|
|
555
|
+
* and targets).
|
|
556
|
+
*/
|
|
557
|
+
static collectAllKeys(graph) {
|
|
558
|
+
const keys = /* @__PURE__ */ new Set();
|
|
559
|
+
keys.add(graph.root);
|
|
560
|
+
if (graph.edges) {
|
|
561
|
+
Object.entries(graph.edges).forEach(([sourceKey, edges]) => {
|
|
562
|
+
keys.add(sourceKey);
|
|
563
|
+
edges.forEach((edge) => {
|
|
564
|
+
keys.add(edge.key);
|
|
565
|
+
});
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
return keys;
|
|
369
569
|
}
|
|
370
570
|
};
|
|
371
571
|
|
|
372
572
|
// src/api/judge/Judge.ts
|
|
573
|
+
var import_mustache = __toESM(require("mustache"), 1);
|
|
574
|
+
var EVALUATION_SCHEMA = {
|
|
575
|
+
type: "object",
|
|
576
|
+
properties: {
|
|
577
|
+
score: {
|
|
578
|
+
type: "number",
|
|
579
|
+
minimum: 0,
|
|
580
|
+
maximum: 1,
|
|
581
|
+
description: "Score between 0.0 and 1.0."
|
|
582
|
+
},
|
|
583
|
+
reasoning: {
|
|
584
|
+
type: "string",
|
|
585
|
+
description: "Reasoning behind the score."
|
|
586
|
+
}
|
|
587
|
+
},
|
|
588
|
+
required: ["score", "reasoning"],
|
|
589
|
+
additionalProperties: false
|
|
590
|
+
};
|
|
373
591
|
var Judge = class {
|
|
374
|
-
constructor(_aiConfig,
|
|
592
|
+
constructor(_aiConfig, _aiProvider, logger) {
|
|
375
593
|
this._aiConfig = _aiConfig;
|
|
376
|
-
this._aiConfigTracker = _aiConfigTracker;
|
|
377
594
|
this._aiProvider = _aiProvider;
|
|
378
595
|
this._logger = logger;
|
|
379
|
-
const evaluationMetricKey = this._getEvaluationMetricKey();
|
|
380
|
-
this._evaluationResponseStructure = EvaluationSchemaBuilder.build(evaluationMetricKey);
|
|
381
596
|
}
|
|
382
597
|
/**
|
|
383
598
|
* Gets the evaluation metric key, prioritizing evaluationMetricKey over evaluationMetricKeys.
|
|
@@ -403,56 +618,62 @@ var Judge = class {
|
|
|
403
618
|
* @param input The input prompt or question that was provided to the AI
|
|
404
619
|
* @param output The AI-generated response to be evaluated
|
|
405
620
|
* @param samplingRate Sampling rate (0-1) to determine if evaluation should be processed (defaults to 1)
|
|
406
|
-
* @returns Promise that resolves to evaluation results
|
|
621
|
+
* @returns Promise that resolves to evaluation results
|
|
407
622
|
*/
|
|
408
623
|
async evaluate(input, output, samplingRate = 1) {
|
|
624
|
+
const result = {
|
|
625
|
+
success: false,
|
|
626
|
+
sampled: false,
|
|
627
|
+
judgeConfigKey: this._aiConfig.key
|
|
628
|
+
};
|
|
629
|
+
const tracker = this._aiConfig.createTracker();
|
|
409
630
|
try {
|
|
410
631
|
const evaluationMetricKey = this._getEvaluationMetricKey();
|
|
411
632
|
if (!evaluationMetricKey) {
|
|
412
633
|
this._logger?.warn(
|
|
413
634
|
"Judge configuration is missing required evaluation metric key",
|
|
414
|
-
|
|
635
|
+
tracker.getTrackData()
|
|
415
636
|
);
|
|
416
|
-
|
|
637
|
+
result.sampled = true;
|
|
638
|
+
result.errorMessage = "Judge configuration is missing required evaluation metric key";
|
|
639
|
+
return result;
|
|
417
640
|
}
|
|
418
641
|
if (!this._aiConfig.messages) {
|
|
419
|
-
this._logger?.warn(
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
return void 0;
|
|
642
|
+
this._logger?.warn("Judge configuration must include messages", tracker.getTrackData());
|
|
643
|
+
result.sampled = true;
|
|
644
|
+
result.errorMessage = "Judge configuration must include messages";
|
|
645
|
+
return result;
|
|
424
646
|
}
|
|
425
647
|
if (Math.random() > samplingRate) {
|
|
426
648
|
this._logger?.debug(`Judge evaluation skipped due to sampling rate: ${samplingRate}`);
|
|
427
|
-
return
|
|
649
|
+
return result;
|
|
428
650
|
}
|
|
651
|
+
result.sampled = true;
|
|
429
652
|
const messages = this._constructEvaluationMessages(input, output);
|
|
430
|
-
const response = await
|
|
431
|
-
(
|
|
432
|
-
() => this._aiProvider.invokeStructuredModel(messages,
|
|
653
|
+
const response = await tracker.trackMetricsOf(
|
|
654
|
+
(r) => r.metrics,
|
|
655
|
+
() => this._aiProvider.invokeStructuredModel(messages, EVALUATION_SCHEMA)
|
|
433
656
|
);
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
if (!evals[evaluationMetricKey]) {
|
|
657
|
+
const evalResult = this._parseEvaluationResponse(response.data);
|
|
658
|
+
if (!evalResult) {
|
|
437
659
|
this._logger?.warn(
|
|
438
|
-
|
|
439
|
-
|
|
660
|
+
`Could not parse evaluation response: ${JSON.stringify(response.data)}`,
|
|
661
|
+
tracker.getTrackData()
|
|
440
662
|
);
|
|
441
|
-
|
|
663
|
+
return result;
|
|
442
664
|
}
|
|
443
665
|
return {
|
|
444
|
-
|
|
445
|
-
success,
|
|
446
|
-
|
|
666
|
+
...result,
|
|
667
|
+
success: response.metrics.success,
|
|
668
|
+
score: evalResult.score,
|
|
669
|
+
reasoning: evalResult.reasoning,
|
|
670
|
+
metricKey: evaluationMetricKey
|
|
447
671
|
};
|
|
448
672
|
} catch (error) {
|
|
449
673
|
this._logger?.error("Judge evaluation failed:", error);
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
454
|
-
judgeConfigKey: this._aiConfig.key
|
|
455
|
-
};
|
|
674
|
+
result.sampled = true;
|
|
675
|
+
result.errorMessage = error instanceof Error ? error.message : "Unknown error";
|
|
676
|
+
return result;
|
|
456
677
|
}
|
|
457
678
|
}
|
|
458
679
|
/**
|
|
@@ -461,7 +682,7 @@ var Judge = class {
|
|
|
461
682
|
* @param messages Array of messages representing the conversation history
|
|
462
683
|
* @param response The AI response to be evaluated
|
|
463
684
|
* @param samplingRatio Sampling ratio (0-1) to determine if evaluation should be processed (defaults to 1)
|
|
464
|
-
* @returns Promise that resolves to evaluation results
|
|
685
|
+
* @returns Promise that resolves to evaluation results
|
|
465
686
|
*/
|
|
466
687
|
async evaluateMessages(messages, response, samplingRatio = 1) {
|
|
467
688
|
const input = messages.length === 0 ? "" : messages.map((msg) => msg.content).join("\r\n");
|
|
@@ -474,12 +695,6 @@ var Judge = class {
|
|
|
474
695
|
getAIConfig() {
|
|
475
696
|
return this._aiConfig;
|
|
476
697
|
}
|
|
477
|
-
/**
|
|
478
|
-
* Returns the tracker associated with this judge.
|
|
479
|
-
*/
|
|
480
|
-
getTracker() {
|
|
481
|
-
return this._aiConfigTracker;
|
|
482
|
-
}
|
|
483
698
|
/**
|
|
484
699
|
* Returns the AI provider used by this judge.
|
|
485
700
|
*/
|
|
@@ -506,43 +721,23 @@ var Judge = class {
|
|
|
506
721
|
return import_mustache.default.render(content, variables, void 0, { escape: (item) => item });
|
|
507
722
|
}
|
|
508
723
|
/**
|
|
509
|
-
* Parses the structured evaluation response
|
|
724
|
+
* Parses the structured evaluation response. Expects top-level {score, reasoning}.
|
|
725
|
+
* Returns score and reasoning, or undefined if parsing fails.
|
|
510
726
|
*/
|
|
511
|
-
_parseEvaluationResponse(data
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
if (!data.evaluations || typeof data.evaluations !== "object") {
|
|
515
|
-
this._logger?.warn("Invalid response: missing or invalid evaluations object");
|
|
516
|
-
return results;
|
|
517
|
-
}
|
|
518
|
-
const evaluation = evaluations[evaluationMetricKey];
|
|
519
|
-
if (!evaluation || typeof evaluation !== "object") {
|
|
520
|
-
this._logger?.warn(
|
|
521
|
-
`Missing evaluation for metric key: ${evaluationMetricKey}`,
|
|
522
|
-
this._aiConfigTracker.getTrackData()
|
|
523
|
-
);
|
|
524
|
-
return results;
|
|
727
|
+
_parseEvaluationResponse(data) {
|
|
728
|
+
if (!data || typeof data !== "object" || Array.isArray(data)) {
|
|
729
|
+
return void 0;
|
|
525
730
|
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
this._logger?.warn(
|
|
529
|
-
`Invalid score evaluated for ${evaluationMetricKey}: ${evalData.score}. Score must be a number between 0 and 1 inclusive`,
|
|
530
|
-
this._aiConfigTracker.getTrackData()
|
|
531
|
-
);
|
|
532
|
-
return results;
|
|
731
|
+
if (typeof data.score !== "number" || data.score < 0 || data.score > 1) {
|
|
732
|
+
return void 0;
|
|
533
733
|
}
|
|
534
|
-
if (typeof
|
|
535
|
-
|
|
536
|
-
`Invalid reasoning evaluated for ${evaluationMetricKey}: ${evalData.reasoning}. Reasoning must be a string`,
|
|
537
|
-
this._aiConfigTracker.getTrackData()
|
|
538
|
-
);
|
|
539
|
-
return results;
|
|
734
|
+
if (typeof data.reasoning !== "string") {
|
|
735
|
+
return void 0;
|
|
540
736
|
}
|
|
541
|
-
|
|
542
|
-
score:
|
|
543
|
-
reasoning:
|
|
737
|
+
return {
|
|
738
|
+
score: data.score,
|
|
739
|
+
reasoning: data.reasoning
|
|
544
740
|
};
|
|
545
|
-
return results;
|
|
546
741
|
}
|
|
547
742
|
};
|
|
548
743
|
|
|
@@ -742,27 +937,62 @@ function createVercelAISDKTokenUsage(data) {
|
|
|
742
937
|
}
|
|
743
938
|
|
|
744
939
|
// src/LDAIConfigTrackerImpl.ts
|
|
745
|
-
var LDAIConfigTrackerImpl = class {
|
|
746
|
-
constructor(_ldClient, _configKey, _variationKey, _version, _modelName, _providerName, _context) {
|
|
940
|
+
var LDAIConfigTrackerImpl = class _LDAIConfigTrackerImpl {
|
|
941
|
+
constructor(_ldClient, _runId, _configKey, _variationKey, _version, _modelName, _providerName, _context, _graphKey) {
|
|
747
942
|
this._ldClient = _ldClient;
|
|
943
|
+
this._runId = _runId;
|
|
748
944
|
this._configKey = _configKey;
|
|
749
945
|
this._variationKey = _variationKey;
|
|
750
946
|
this._version = _version;
|
|
751
947
|
this._modelName = _modelName;
|
|
752
948
|
this._providerName = _providerName;
|
|
753
949
|
this._context = _context;
|
|
950
|
+
this._graphKey = _graphKey;
|
|
754
951
|
this._trackedMetrics = {};
|
|
755
952
|
}
|
|
756
953
|
getTrackData() {
|
|
757
954
|
return {
|
|
758
|
-
|
|
955
|
+
runId: this._runId,
|
|
759
956
|
configKey: this._configKey,
|
|
957
|
+
variationKey: this._variationKey,
|
|
760
958
|
version: this._version,
|
|
761
959
|
modelName: this._modelName,
|
|
762
|
-
providerName: this._providerName
|
|
960
|
+
providerName: this._providerName,
|
|
961
|
+
...this._graphKey !== void 0 ? { graphKey: this._graphKey } : {}
|
|
763
962
|
};
|
|
764
963
|
}
|
|
964
|
+
get resumptionToken() {
|
|
965
|
+
const json = JSON.stringify({
|
|
966
|
+
runId: this._runId,
|
|
967
|
+
configKey: this._configKey,
|
|
968
|
+
variationKey: this._variationKey,
|
|
969
|
+
version: this._version,
|
|
970
|
+
...this._graphKey !== void 0 ? { graphKey: this._graphKey } : {}
|
|
971
|
+
});
|
|
972
|
+
return Buffer.from(json).toString("base64url");
|
|
973
|
+
}
|
|
974
|
+
static fromResumptionToken(token, ldClient, context) {
|
|
975
|
+
const json = Buffer.from(token, "base64url").toString("utf8");
|
|
976
|
+
const payload = JSON.parse(json);
|
|
977
|
+
return new _LDAIConfigTrackerImpl(
|
|
978
|
+
ldClient,
|
|
979
|
+
payload.runId,
|
|
980
|
+
payload.configKey,
|
|
981
|
+
payload.variationKey ?? "",
|
|
982
|
+
payload.version,
|
|
983
|
+
"",
|
|
984
|
+
"",
|
|
985
|
+
context,
|
|
986
|
+
payload.graphKey
|
|
987
|
+
);
|
|
988
|
+
}
|
|
765
989
|
trackDuration(duration) {
|
|
990
|
+
if (this._trackedMetrics.durationMs !== void 0) {
|
|
991
|
+
this._ldClient.logger?.warn(
|
|
992
|
+
"Duration has already been tracked for this execution. Use createTracker() for a new execution."
|
|
993
|
+
);
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
766
996
|
this._trackedMetrics.durationMs = duration;
|
|
767
997
|
this._ldClient.track("$ld:ai:duration:total", this._context, this.getTrackData(), duration);
|
|
768
998
|
}
|
|
@@ -778,6 +1008,12 @@ var LDAIConfigTrackerImpl = class {
|
|
|
778
1008
|
}
|
|
779
1009
|
}
|
|
780
1010
|
trackTimeToFirstToken(timeToFirstTokenMs) {
|
|
1011
|
+
if (this._trackedMetrics.timeToFirstTokenMs !== void 0) {
|
|
1012
|
+
this._ldClient.logger?.warn(
|
|
1013
|
+
"Time to first token has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1014
|
+
);
|
|
1015
|
+
return;
|
|
1016
|
+
}
|
|
781
1017
|
this._trackedMetrics.timeToFirstTokenMs = timeToFirstTokenMs;
|
|
782
1018
|
this._ldClient.track(
|
|
783
1019
|
"$ld:ai:tokens:ttf",
|
|
@@ -786,22 +1022,30 @@ var LDAIConfigTrackerImpl = class {
|
|
|
786
1022
|
timeToFirstTokenMs
|
|
787
1023
|
);
|
|
788
1024
|
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
}
|
|
1025
|
+
trackJudgeResult(result) {
|
|
1026
|
+
if (!result.sampled || !result.success) {
|
|
1027
|
+
return;
|
|
1028
|
+
}
|
|
1029
|
+
if (result.metricKey !== void 0 && result.score !== void 0) {
|
|
1030
|
+
const trackData = result.judgeConfigKey ? { ...this.getTrackData(), judgeConfigKey: result.judgeConfigKey } : this.getTrackData();
|
|
1031
|
+
this._ldClient.track(result.metricKey, this._context, trackData, result.score);
|
|
1032
|
+
}
|
|
793
1033
|
}
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
evalScore.score
|
|
801
|
-
);
|
|
1034
|
+
trackToolCall(toolKey) {
|
|
1035
|
+
this._ldClient.track("$ld:ai:tool_call", this._context, { ...this.getTrackData(), toolKey }, 1);
|
|
1036
|
+
}
|
|
1037
|
+
trackToolCalls(toolKeys) {
|
|
1038
|
+
toolKeys.forEach((toolKey) => {
|
|
1039
|
+
this.trackToolCall(toolKey);
|
|
802
1040
|
});
|
|
803
1041
|
}
|
|
804
1042
|
trackFeedback(feedback) {
|
|
1043
|
+
if (this._trackedMetrics.feedback !== void 0) {
|
|
1044
|
+
this._ldClient.logger?.warn(
|
|
1045
|
+
"Feedback has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1046
|
+
);
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
805
1049
|
this._trackedMetrics.feedback = feedback;
|
|
806
1050
|
if (feedback.kind === "positive" /* Positive */) {
|
|
807
1051
|
this._ldClient.track("$ld:ai:feedback:user:positive", this._context, this.getTrackData(), 1);
|
|
@@ -810,10 +1054,22 @@ var LDAIConfigTrackerImpl = class {
|
|
|
810
1054
|
}
|
|
811
1055
|
}
|
|
812
1056
|
trackSuccess() {
|
|
1057
|
+
if (this._trackedMetrics.success !== void 0) {
|
|
1058
|
+
this._ldClient.logger?.warn(
|
|
1059
|
+
"Generation result has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1060
|
+
);
|
|
1061
|
+
return;
|
|
1062
|
+
}
|
|
813
1063
|
this._trackedMetrics.success = true;
|
|
814
1064
|
this._ldClient.track("$ld:ai:generation:success", this._context, this.getTrackData(), 1);
|
|
815
1065
|
}
|
|
816
1066
|
trackError() {
|
|
1067
|
+
if (this._trackedMetrics.success !== void 0) {
|
|
1068
|
+
this._ldClient.logger?.warn(
|
|
1069
|
+
"Generation result has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1070
|
+
);
|
|
1071
|
+
return;
|
|
1072
|
+
}
|
|
817
1073
|
this._trackedMetrics.success = false;
|
|
818
1074
|
this._ldClient.track("$ld:ai:generation:error", this._context, this.getTrackData(), 1);
|
|
819
1075
|
}
|
|
@@ -906,6 +1162,12 @@ var LDAIConfigTrackerImpl = class {
|
|
|
906
1162
|
}
|
|
907
1163
|
}
|
|
908
1164
|
trackTokens(tokens) {
|
|
1165
|
+
if (this._trackedMetrics.tokens !== void 0) {
|
|
1166
|
+
this._ldClient.logger?.warn(
|
|
1167
|
+
"Token usage has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1168
|
+
);
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
909
1171
|
this._trackedMetrics.tokens = tokens;
|
|
910
1172
|
const trackData = this.getTrackData();
|
|
911
1173
|
if (tokens.total > 0) {
|
|
@@ -926,9 +1188,155 @@ var LDAIConfigTrackerImpl = class {
|
|
|
926
1188
|
}
|
|
927
1189
|
};
|
|
928
1190
|
|
|
1191
|
+
// src/LDGraphTrackerImpl.ts
|
|
1192
|
+
var LDGraphTrackerImpl = class _LDGraphTrackerImpl {
|
|
1193
|
+
constructor(_ldClient, _runId, _graphKey, _variationKey, _version, _context) {
|
|
1194
|
+
this._ldClient = _ldClient;
|
|
1195
|
+
this._runId = _runId;
|
|
1196
|
+
this._graphKey = _graphKey;
|
|
1197
|
+
this._variationKey = _variationKey;
|
|
1198
|
+
this._version = _version;
|
|
1199
|
+
this._context = _context;
|
|
1200
|
+
this._summary = {};
|
|
1201
|
+
}
|
|
1202
|
+
/**
|
|
1203
|
+
* Reconstructs an {@link LDGraphTrackerImpl} from a resumption token, preserving
|
|
1204
|
+
* the original `runId` so all events continue to be correlated under the same run.
|
|
1205
|
+
*
|
|
1206
|
+
* **Security note:** The token contains the flag variation key and version.
|
|
1207
|
+
* Do not pass the raw token to untrusted clients.
|
|
1208
|
+
*
|
|
1209
|
+
* @param token URL-safe Base64-encoded token produced by {@link LDGraphTrackerImpl.resumptionToken}.
|
|
1210
|
+
* @param ldClient LaunchDarkly client instance.
|
|
1211
|
+
* @param context LDContext for the new tracker.
|
|
1212
|
+
*/
|
|
1213
|
+
static fromResumptionToken(token, ldClient, context) {
|
|
1214
|
+
const json = Buffer.from(token, "base64url").toString("utf8");
|
|
1215
|
+
const data = JSON.parse(json);
|
|
1216
|
+
return new _LDGraphTrackerImpl(
|
|
1217
|
+
ldClient,
|
|
1218
|
+
data.runId,
|
|
1219
|
+
data.graphKey,
|
|
1220
|
+
data.variationKey,
|
|
1221
|
+
data.version,
|
|
1222
|
+
context
|
|
1223
|
+
);
|
|
1224
|
+
}
|
|
1225
|
+
getTrackData() {
|
|
1226
|
+
const data = {
|
|
1227
|
+
runId: this._runId,
|
|
1228
|
+
graphKey: this._graphKey,
|
|
1229
|
+
version: this._version
|
|
1230
|
+
};
|
|
1231
|
+
if (this._variationKey !== void 0) {
|
|
1232
|
+
data.variationKey = this._variationKey;
|
|
1233
|
+
}
|
|
1234
|
+
return data;
|
|
1235
|
+
}
|
|
1236
|
+
getSummary() {
|
|
1237
|
+
return { ...this._summary };
|
|
1238
|
+
}
|
|
1239
|
+
get resumptionToken() {
|
|
1240
|
+
const parts = [
|
|
1241
|
+
`"runId":${JSON.stringify(this._runId)}`,
|
|
1242
|
+
`"graphKey":${JSON.stringify(this._graphKey)}`
|
|
1243
|
+
];
|
|
1244
|
+
if (this._variationKey !== void 0) {
|
|
1245
|
+
parts.push(`"variationKey":${JSON.stringify(this._variationKey)}`);
|
|
1246
|
+
}
|
|
1247
|
+
parts.push(`"version":${this._version}`);
|
|
1248
|
+
const json = `{${parts.join(",")}}`;
|
|
1249
|
+
return Buffer.from(json).toString("base64url");
|
|
1250
|
+
}
|
|
1251
|
+
trackInvocationSuccess() {
|
|
1252
|
+
if (this._summary.success !== void 0) {
|
|
1253
|
+
this._ldClient.logger?.warn(
|
|
1254
|
+
"LDGraphTracker: invocation success/failure already recorded for this run \u2014 dropping duplicate call."
|
|
1255
|
+
);
|
|
1256
|
+
return;
|
|
1257
|
+
}
|
|
1258
|
+
this._summary.success = true;
|
|
1259
|
+
this._ldClient.track("$ld:ai:graph:invocation_success", this._context, this.getTrackData(), 1);
|
|
1260
|
+
}
|
|
1261
|
+
trackInvocationFailure() {
|
|
1262
|
+
if (this._summary.success !== void 0) {
|
|
1263
|
+
this._ldClient.logger?.warn(
|
|
1264
|
+
"LDGraphTracker: invocation success/failure already recorded for this run \u2014 dropping duplicate call."
|
|
1265
|
+
);
|
|
1266
|
+
return;
|
|
1267
|
+
}
|
|
1268
|
+
this._summary.success = false;
|
|
1269
|
+
this._ldClient.track("$ld:ai:graph:invocation_failure", this._context, this.getTrackData(), 1);
|
|
1270
|
+
}
|
|
1271
|
+
trackDuration(durationMs) {
|
|
1272
|
+
if (this._summary.durationMs !== void 0) {
|
|
1273
|
+
this._ldClient.logger?.warn(
|
|
1274
|
+
"LDGraphTracker: trackDuration already called for this run \u2014 dropping duplicate call."
|
|
1275
|
+
);
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
this._summary.durationMs = durationMs;
|
|
1279
|
+
this._ldClient.track(
|
|
1280
|
+
"$ld:ai:graph:duration:total",
|
|
1281
|
+
this._context,
|
|
1282
|
+
this.getTrackData(),
|
|
1283
|
+
durationMs
|
|
1284
|
+
);
|
|
1285
|
+
}
|
|
1286
|
+
trackTotalTokens(tokens) {
|
|
1287
|
+
if (this._summary.tokens !== void 0) {
|
|
1288
|
+
this._ldClient.logger?.warn(
|
|
1289
|
+
"LDGraphTracker: trackTotalTokens already called for this run \u2014 dropping duplicate call."
|
|
1290
|
+
);
|
|
1291
|
+
return;
|
|
1292
|
+
}
|
|
1293
|
+
this._summary.tokens = { ...tokens };
|
|
1294
|
+
this._ldClient.track(
|
|
1295
|
+
"$ld:ai:graph:total_tokens",
|
|
1296
|
+
this._context,
|
|
1297
|
+
this.getTrackData(),
|
|
1298
|
+
tokens.total
|
|
1299
|
+
);
|
|
1300
|
+
}
|
|
1301
|
+
trackPath(path) {
|
|
1302
|
+
if (this._summary.path !== void 0) {
|
|
1303
|
+
this._ldClient.logger?.warn(
|
|
1304
|
+
"LDGraphTracker: trackPath already called for this run \u2014 dropping duplicate call."
|
|
1305
|
+
);
|
|
1306
|
+
return;
|
|
1307
|
+
}
|
|
1308
|
+
this._summary.path = [...path];
|
|
1309
|
+
this._ldClient.track("$ld:ai:graph:path", this._context, { ...this.getTrackData(), path }, 1);
|
|
1310
|
+
}
|
|
1311
|
+
trackRedirect(sourceKey, redirectedTarget) {
|
|
1312
|
+
this._ldClient.track(
|
|
1313
|
+
"$ld:ai:graph:redirect",
|
|
1314
|
+
this._context,
|
|
1315
|
+
{ ...this.getTrackData(), sourceKey, redirectedTarget },
|
|
1316
|
+
1
|
|
1317
|
+
);
|
|
1318
|
+
}
|
|
1319
|
+
trackHandoffSuccess(sourceKey, targetKey) {
|
|
1320
|
+
this._ldClient.track(
|
|
1321
|
+
"$ld:ai:graph:handoff_success",
|
|
1322
|
+
this._context,
|
|
1323
|
+
{ ...this.getTrackData(), sourceKey, targetKey },
|
|
1324
|
+
1
|
|
1325
|
+
);
|
|
1326
|
+
}
|
|
1327
|
+
trackHandoffFailure(sourceKey, targetKey) {
|
|
1328
|
+
this._ldClient.track(
|
|
1329
|
+
"$ld:ai:graph:handoff_failure",
|
|
1330
|
+
this._context,
|
|
1331
|
+
{ ...this.getTrackData(), sourceKey, targetKey },
|
|
1332
|
+
1
|
|
1333
|
+
);
|
|
1334
|
+
}
|
|
1335
|
+
};
|
|
1336
|
+
|
|
929
1337
|
// src/sdkInfo.ts
|
|
930
1338
|
var aiSdkName = "@launchdarkly/server-sdk-ai";
|
|
931
|
-
var aiSdkVersion = "0.
|
|
1339
|
+
var aiSdkVersion = "0.17.0";
|
|
932
1340
|
var aiSdkLanguage = "javascript";
|
|
933
1341
|
|
|
934
1342
|
// src/LDAIClientImpl.ts
|
|
@@ -939,6 +1347,7 @@ var TRACK_USAGE_JUDGE_CONFIG = "$ld:ai:usage:judge-config";
|
|
|
939
1347
|
var TRACK_USAGE_CREATE_JUDGE = "$ld:ai:usage:create-judge";
|
|
940
1348
|
var TRACK_USAGE_AGENT_CONFIG = "$ld:ai:usage:agent-config";
|
|
941
1349
|
var TRACK_USAGE_AGENT_CONFIGS = "$ld:ai:usage:agent-configs";
|
|
1350
|
+
var TRACK_USAGE_AGENT_GRAPH = "$ld:ai:usage:agent-graph";
|
|
942
1351
|
var INIT_TRACK_CONTEXT = {
|
|
943
1352
|
kind: "ld_ai",
|
|
944
1353
|
key: "ld-internal-tracking",
|
|
@@ -963,7 +1372,7 @@ var LDAIClientImpl = class {
|
|
|
963
1372
|
_interpolateTemplate(template, variables) {
|
|
964
1373
|
return import_mustache2.default.render(template, variables, void 0, { escape: (item) => item });
|
|
965
1374
|
}
|
|
966
|
-
async _evaluate(key, context, defaultValue, mode, variables) {
|
|
1375
|
+
async _evaluate(key, context, defaultValue, mode, variables, graphKey) {
|
|
967
1376
|
const ldFlagValue = LDAIConfigUtils.toFlagValue(defaultValue, mode);
|
|
968
1377
|
const value = await this._ldClient.variation(key, context, ldFlagValue);
|
|
969
1378
|
const flagMode = value._ldMeta?.mode ?? "completion";
|
|
@@ -973,8 +1382,9 @@ var LDAIClientImpl = class {
|
|
|
973
1382
|
);
|
|
974
1383
|
return LDAIConfigUtils.createDisabledConfig(key, mode);
|
|
975
1384
|
}
|
|
976
|
-
const
|
|
1385
|
+
const trackerFactory = () => new LDAIConfigTrackerImpl(
|
|
977
1386
|
this._ldClient,
|
|
1387
|
+
(0, import_node_crypto.randomUUID)(),
|
|
978
1388
|
key,
|
|
979
1389
|
// eslint-disable-next-line no-underscore-dangle
|
|
980
1390
|
value._ldMeta?.variationKey ?? "",
|
|
@@ -982,9 +1392,10 @@ var LDAIClientImpl = class {
|
|
|
982
1392
|
value._ldMeta?.version ?? 1,
|
|
983
1393
|
value.model?.name ?? "",
|
|
984
1394
|
value.provider?.name ?? "",
|
|
985
|
-
context
|
|
1395
|
+
context,
|
|
1396
|
+
graphKey
|
|
986
1397
|
);
|
|
987
|
-
const config = LDAIConfigUtils.fromFlagValue(key, value,
|
|
1398
|
+
const config = LDAIConfigUtils.fromFlagValue(key, value, trackerFactory);
|
|
988
1399
|
return this._applyInterpolation(config, context, variables);
|
|
989
1400
|
}
|
|
990
1401
|
_applyInterpolation(config, context, variables) {
|
|
@@ -1048,16 +1459,13 @@ var LDAIClientImpl = class {
|
|
|
1048
1459
|
this._ldClient.track(TRACK_USAGE_JUDGE_CONFIG, context, key, 1);
|
|
1049
1460
|
return this._judgeConfig(key, context, defaultValue ?? disabledAIConfig, variables);
|
|
1050
1461
|
}
|
|
1462
|
+
async _agentConfig(key, context, defaultValue, variables, graphKey) {
|
|
1463
|
+
const config = await this._evaluate(key, context, defaultValue, "agent", variables, graphKey);
|
|
1464
|
+
return config;
|
|
1465
|
+
}
|
|
1051
1466
|
async agentConfig(key, context, defaultValue, variables) {
|
|
1052
1467
|
this._ldClient.track(TRACK_USAGE_AGENT_CONFIG, context, key, 1);
|
|
1053
|
-
|
|
1054
|
-
key,
|
|
1055
|
-
context,
|
|
1056
|
-
defaultValue ?? disabledAIConfig,
|
|
1057
|
-
"agent",
|
|
1058
|
-
variables
|
|
1059
|
-
);
|
|
1060
|
-
return config;
|
|
1468
|
+
return this._agentConfig(key, context, defaultValue ?? disabledAIConfig, variables);
|
|
1061
1469
|
}
|
|
1062
1470
|
/**
|
|
1063
1471
|
* @deprecated Use `agentConfig` instead. This method will be removed in a future version.
|
|
@@ -1075,11 +1483,10 @@ var LDAIClientImpl = class {
|
|
|
1075
1483
|
const agents = {};
|
|
1076
1484
|
await Promise.all(
|
|
1077
1485
|
agentConfigs.map(async (config) => {
|
|
1078
|
-
const agent = await this.
|
|
1486
|
+
const agent = await this._agentConfig(
|
|
1079
1487
|
config.key,
|
|
1080
1488
|
context,
|
|
1081
1489
|
config.defaultValue ?? disabledAIConfig,
|
|
1082
|
-
"agent",
|
|
1083
1490
|
config.variables
|
|
1084
1491
|
);
|
|
1085
1492
|
agents[config.key] = agent;
|
|
@@ -1101,7 +1508,7 @@ var LDAIClientImpl = class {
|
|
|
1101
1508
|
defaultValue ?? disabledAIConfig,
|
|
1102
1509
|
variables
|
|
1103
1510
|
);
|
|
1104
|
-
if (!config.enabled
|
|
1511
|
+
if (!config.enabled) {
|
|
1105
1512
|
this._logger?.info(`Chat configuration is disabled: ${key}`);
|
|
1106
1513
|
return void 0;
|
|
1107
1514
|
}
|
|
@@ -1115,7 +1522,7 @@ var LDAIClientImpl = class {
|
|
|
1115
1522
|
variables,
|
|
1116
1523
|
defaultAiProvider
|
|
1117
1524
|
);
|
|
1118
|
-
return new TrackedChat(config,
|
|
1525
|
+
return new TrackedChat(config, provider, judges, this._logger);
|
|
1119
1526
|
}
|
|
1120
1527
|
async createJudge(key, context, defaultValue, variables, defaultAiProvider) {
|
|
1121
1528
|
this._ldClient.track(TRACK_USAGE_CREATE_JUDGE, context, key, 1);
|
|
@@ -1141,7 +1548,7 @@ var LDAIClientImpl = class {
|
|
|
1141
1548
|
defaultValue ?? disabledAIConfig,
|
|
1142
1549
|
extendedVariables
|
|
1143
1550
|
);
|
|
1144
|
-
if (!judgeConfig.enabled
|
|
1551
|
+
if (!judgeConfig.enabled) {
|
|
1145
1552
|
this._logger?.info(`Judge configuration is disabled: ${key}`);
|
|
1146
1553
|
return void 0;
|
|
1147
1554
|
}
|
|
@@ -1149,7 +1556,7 @@ var LDAIClientImpl = class {
|
|
|
1149
1556
|
if (!provider) {
|
|
1150
1557
|
return void 0;
|
|
1151
1558
|
}
|
|
1152
|
-
return new Judge(judgeConfig,
|
|
1559
|
+
return new Judge(judgeConfig, provider, this._logger);
|
|
1153
1560
|
} catch (error) {
|
|
1154
1561
|
this._logger?.error(`Failed to initialize judge ${key}:`, error);
|
|
1155
1562
|
return void 0;
|
|
@@ -1161,6 +1568,81 @@ var LDAIClientImpl = class {
|
|
|
1161
1568
|
async initChat(key, context, defaultValue, variables, defaultAiProvider) {
|
|
1162
1569
|
return this.createChat(key, context, defaultValue, variables, defaultAiProvider);
|
|
1163
1570
|
}
|
|
1571
|
+
createTracker(token, context) {
|
|
1572
|
+
return LDAIConfigTrackerImpl.fromResumptionToken(token, this._ldClient, context);
|
|
1573
|
+
}
|
|
1574
|
+
async agentGraph(graphKey, context, variables) {
|
|
1575
|
+
this._ldClient.track(TRACK_USAGE_AGENT_GRAPH, context, graphKey, 1);
|
|
1576
|
+
const defaultGraphValue = { root: "" };
|
|
1577
|
+
const graphFlagValue = await this._ldClient.variation(
|
|
1578
|
+
graphKey,
|
|
1579
|
+
context,
|
|
1580
|
+
defaultGraphValue
|
|
1581
|
+
);
|
|
1582
|
+
const variationKey = graphFlagValue._ldMeta?.variationKey;
|
|
1583
|
+
const version = graphFlagValue._ldMeta?.version ?? 1;
|
|
1584
|
+
const ldClient = this._ldClient;
|
|
1585
|
+
const trackerFactory = () => new LDGraphTrackerImpl(ldClient, (0, import_node_crypto.randomUUID)(), graphKey, variationKey, version, context);
|
|
1586
|
+
const disabled = new AgentGraphDefinition(graphFlagValue, {}, false, trackerFactory);
|
|
1587
|
+
if (graphFlagValue._ldMeta?.enabled === false) {
|
|
1588
|
+
this._logger?.debug(`agentGraph: graph "${graphKey}" is disabled.`);
|
|
1589
|
+
return disabled;
|
|
1590
|
+
}
|
|
1591
|
+
if (!graphFlagValue.root) {
|
|
1592
|
+
this._logger?.debug(`agentGraph: graph "${graphKey}" is not fetchable or has no root node.`);
|
|
1593
|
+
return disabled;
|
|
1594
|
+
}
|
|
1595
|
+
const allKeys = AgentGraphDefinition.collectAllKeys(graphFlagValue);
|
|
1596
|
+
const reachableKeys = this._collectReachableKeys(graphFlagValue);
|
|
1597
|
+
const unreachableKey = [...allKeys].find((key) => !reachableKeys.has(key));
|
|
1598
|
+
if (unreachableKey) {
|
|
1599
|
+
this._logger?.debug(
|
|
1600
|
+
`agentGraph: graph "${graphKey}" has unconnected node "${unreachableKey}" that is not reachable from the root.`
|
|
1601
|
+
);
|
|
1602
|
+
return disabled;
|
|
1603
|
+
}
|
|
1604
|
+
const agentConfigs = {};
|
|
1605
|
+
const fetchResults = await Promise.all(
|
|
1606
|
+
[...allKeys].map(async (key) => {
|
|
1607
|
+
const config = await this._agentConfig(key, context, disabledAIConfig, variables, graphKey);
|
|
1608
|
+
return { key, config };
|
|
1609
|
+
})
|
|
1610
|
+
);
|
|
1611
|
+
const disabledResult = fetchResults.find(({ config }) => !config.enabled);
|
|
1612
|
+
if (disabledResult) {
|
|
1613
|
+
this._logger?.debug(
|
|
1614
|
+
`agentGraph: agent config "${disabledResult.key}" in graph "${graphKey}" is not enabled or could not be fetched.`
|
|
1615
|
+
);
|
|
1616
|
+
return disabled;
|
|
1617
|
+
}
|
|
1618
|
+
fetchResults.forEach(({ key, config }) => {
|
|
1619
|
+
agentConfigs[key] = config;
|
|
1620
|
+
});
|
|
1621
|
+
const nodes = AgentGraphDefinition.buildNodes(graphFlagValue, agentConfigs);
|
|
1622
|
+
return new AgentGraphDefinition(graphFlagValue, nodes, true, trackerFactory);
|
|
1623
|
+
}
|
|
1624
|
+
createGraphTracker(token, context) {
|
|
1625
|
+
return LDGraphTrackerImpl.fromResumptionToken(token, this._ldClient, context);
|
|
1626
|
+
}
|
|
1627
|
+
/**
|
|
1628
|
+
* Returns the set of all node keys reachable from the root via BFS.
|
|
1629
|
+
*/
|
|
1630
|
+
_collectReachableKeys(graph) {
|
|
1631
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1632
|
+
const queue = [graph.root];
|
|
1633
|
+
visited.add(graph.root);
|
|
1634
|
+
while (queue.length > 0) {
|
|
1635
|
+
const key = queue.shift();
|
|
1636
|
+
const edges = graph.edges?.[key] ?? [];
|
|
1637
|
+
edges.forEach((edge) => {
|
|
1638
|
+
if (!visited.has(edge.key)) {
|
|
1639
|
+
visited.add(edge.key);
|
|
1640
|
+
queue.push(edge.key);
|
|
1641
|
+
}
|
|
1642
|
+
});
|
|
1643
|
+
}
|
|
1644
|
+
return visited;
|
|
1645
|
+
}
|
|
1164
1646
|
};
|
|
1165
1647
|
|
|
1166
1648
|
// src/index.ts
|
|
@@ -1171,8 +1653,11 @@ function initAi(ldClient) {
|
|
|
1171
1653
|
0 && (module.exports = {
|
|
1172
1654
|
AIProvider,
|
|
1173
1655
|
AIProviderFactory,
|
|
1656
|
+
AgentGraphDefinition,
|
|
1657
|
+
AgentGraphNode,
|
|
1174
1658
|
Judge,
|
|
1175
1659
|
LDFeedbackKind,
|
|
1660
|
+
LDGraphTrackerImpl,
|
|
1176
1661
|
SUPPORTED_AI_PROVIDERS,
|
|
1177
1662
|
TrackedChat,
|
|
1178
1663
|
createBedrockTokenUsage,
|