@launchdarkly/server-sdk-ai 0.16.8 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/index.cjs +669 -184
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +480 -45
- package/dist/index.d.ts +480 -45
- package/dist/index.js +666 -184
- package/dist/index.js.map +1 -1
- package/package.json +2 -4
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// src/LDAIClientImpl.ts
|
|
2
2
|
import Mustache2 from "mustache";
|
|
3
|
+
import { randomUUID } from "crypto";
|
|
3
4
|
|
|
4
5
|
// src/api/chat/TrackedChat.ts
|
|
5
6
|
var TrackedChat = class {
|
|
6
|
-
constructor(aiConfig,
|
|
7
|
+
constructor(aiConfig, provider, judges = {}, _logger) {
|
|
7
8
|
this.aiConfig = aiConfig;
|
|
8
|
-
this.tracker = tracker;
|
|
9
9
|
this.provider = provider;
|
|
10
10
|
this.judges = judges;
|
|
11
11
|
this._logger = _logger;
|
|
@@ -16,6 +16,7 @@ var TrackedChat = class {
|
|
|
16
16
|
* This method handles conversation management and tracking, delegating to the provider's invokeModel method.
|
|
17
17
|
*/
|
|
18
18
|
async invoke(prompt) {
|
|
19
|
+
const tracker = this.aiConfig.createTracker();
|
|
19
20
|
const userMessage = {
|
|
20
21
|
role: "user",
|
|
21
22
|
content: prompt
|
|
@@ -23,12 +24,19 @@ var TrackedChat = class {
|
|
|
23
24
|
this.messages.push(userMessage);
|
|
24
25
|
const configMessages = this.aiConfig.messages || [];
|
|
25
26
|
const allMessages = [...configMessages, ...this.messages];
|
|
26
|
-
const response = await
|
|
27
|
+
const response = await tracker.trackMetricsOf(
|
|
27
28
|
(result) => result.metrics,
|
|
28
29
|
() => this.provider.invokeModel(allMessages)
|
|
29
30
|
);
|
|
30
31
|
if (this.aiConfig.judgeConfiguration?.judges && this.aiConfig.judgeConfiguration.judges.length > 0) {
|
|
31
|
-
response.evaluations = this._evaluateWithJudges(this.messages, response)
|
|
32
|
+
response.evaluations = this._evaluateWithJudges(this.messages, response).then(
|
|
33
|
+
(evaluations) => {
|
|
34
|
+
evaluations.forEach((judgeResult) => {
|
|
35
|
+
tracker.trackJudgeResult(judgeResult);
|
|
36
|
+
});
|
|
37
|
+
return evaluations;
|
|
38
|
+
}
|
|
39
|
+
);
|
|
32
40
|
}
|
|
33
41
|
this.messages.push(response.message);
|
|
34
42
|
return response;
|
|
@@ -47,23 +55,29 @@ var TrackedChat = class {
|
|
|
47
55
|
const judge = this.judges[judgeConfig.key];
|
|
48
56
|
if (!judge) {
|
|
49
57
|
this._logger?.warn(
|
|
50
|
-
`Judge configuration is not enabled
|
|
51
|
-
this.tracker.getTrackData()
|
|
58
|
+
`Judge configuration is not enabled for ${judgeConfig.key} in ${this.aiConfig.key}`
|
|
52
59
|
);
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
);
|
|
60
|
-
if (judgeResponse && judgeResponse.success) {
|
|
61
|
-
this.tracker.trackJudgeResponse(judgeResponse);
|
|
60
|
+
const result = {
|
|
61
|
+
success: false,
|
|
62
|
+
sampled: true,
|
|
63
|
+
errorMessage: `Judge configuration is not enabled for ${judgeConfig.key}`
|
|
64
|
+
};
|
|
65
|
+
return result;
|
|
62
66
|
}
|
|
63
|
-
return
|
|
67
|
+
return judge.evaluateMessages(messages, response, judgeConfig.samplingRate);
|
|
64
68
|
});
|
|
65
69
|
const results = await Promise.allSettled(evaluationPromises);
|
|
66
|
-
return results.map((
|
|
70
|
+
return results.map((settled) => {
|
|
71
|
+
if (settled.status === "fulfilled") {
|
|
72
|
+
return settled.value;
|
|
73
|
+
}
|
|
74
|
+
const result = {
|
|
75
|
+
success: false,
|
|
76
|
+
sampled: true,
|
|
77
|
+
errorMessage: "Judge evaluation failed"
|
|
78
|
+
};
|
|
79
|
+
return result;
|
|
80
|
+
});
|
|
67
81
|
}
|
|
68
82
|
/**
|
|
69
83
|
* Get the underlying AI configuration used to initialize this TrackedChat.
|
|
@@ -71,12 +85,6 @@ var TrackedChat = class {
|
|
|
71
85
|
getConfig() {
|
|
72
86
|
return this.aiConfig;
|
|
73
87
|
}
|
|
74
|
-
/**
|
|
75
|
-
* Get the underlying AI configuration tracker used to initialize this TrackedChat.
|
|
76
|
-
*/
|
|
77
|
-
getTracker() {
|
|
78
|
-
return this.tracker;
|
|
79
|
-
}
|
|
80
88
|
/**
|
|
81
89
|
* Get the underlying AI provider instance.
|
|
82
90
|
* This provides direct access to the provider for advanced use cases.
|
|
@@ -161,20 +169,21 @@ var LDAIConfigUtils = class {
|
|
|
161
169
|
/**
|
|
162
170
|
* Converts a LaunchDarkly flag value to the appropriate AI configuration type.
|
|
163
171
|
*
|
|
172
|
+
* @param key The configuration key
|
|
164
173
|
* @param flagValue The flag value from LaunchDarkly
|
|
165
|
-
* @param
|
|
174
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
166
175
|
* @returns The appropriate AI configuration type
|
|
167
176
|
*/
|
|
168
|
-
static fromFlagValue(key, flagValue,
|
|
177
|
+
static fromFlagValue(key, flagValue, trackerFactory) {
|
|
169
178
|
const flagValueMode = flagValue._ldMeta?.mode;
|
|
170
179
|
switch (flagValueMode) {
|
|
171
180
|
case "agent":
|
|
172
|
-
return this.toAgentConfig(key, flagValue,
|
|
181
|
+
return this.toAgentConfig(key, flagValue, trackerFactory);
|
|
173
182
|
case "judge":
|
|
174
|
-
return this.toJudgeConfig(key, flagValue,
|
|
183
|
+
return this.toJudgeConfig(key, flagValue, trackerFactory);
|
|
175
184
|
case "completion":
|
|
176
185
|
default:
|
|
177
|
-
return this.toCompletionConfig(key, flagValue,
|
|
186
|
+
return this.toCompletionConfig(key, flagValue, trackerFactory);
|
|
178
187
|
}
|
|
179
188
|
}
|
|
180
189
|
/**
|
|
@@ -189,20 +198,20 @@ var LDAIConfigUtils = class {
|
|
|
189
198
|
return {
|
|
190
199
|
key,
|
|
191
200
|
enabled: false,
|
|
192
|
-
|
|
201
|
+
createTracker: void 0
|
|
193
202
|
};
|
|
194
203
|
case "judge":
|
|
195
204
|
return {
|
|
196
205
|
key,
|
|
197
206
|
enabled: false,
|
|
198
|
-
|
|
207
|
+
createTracker: void 0
|
|
199
208
|
};
|
|
200
209
|
case "completion":
|
|
201
210
|
default:
|
|
202
211
|
return {
|
|
203
212
|
key,
|
|
204
213
|
enabled: false,
|
|
205
|
-
|
|
214
|
+
createTracker: void 0
|
|
206
215
|
};
|
|
207
216
|
}
|
|
208
217
|
}
|
|
@@ -224,14 +233,15 @@ var LDAIConfigUtils = class {
|
|
|
224
233
|
/**
|
|
225
234
|
* Creates a completion config from flag value data.
|
|
226
235
|
*
|
|
236
|
+
* @param key The configuration key
|
|
227
237
|
* @param flagValue The flag value from LaunchDarkly
|
|
228
|
-
* @param
|
|
238
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
229
239
|
* @returns A completion configuration
|
|
230
240
|
*/
|
|
231
|
-
static toCompletionConfig(key, flagValue,
|
|
241
|
+
static toCompletionConfig(key, flagValue, trackerFactory) {
|
|
232
242
|
return {
|
|
233
243
|
...this._toBaseConfig(key, flagValue),
|
|
234
|
-
|
|
244
|
+
createTracker: trackerFactory,
|
|
235
245
|
messages: flagValue.messages,
|
|
236
246
|
judgeConfiguration: flagValue.judgeConfiguration
|
|
237
247
|
};
|
|
@@ -239,14 +249,15 @@ var LDAIConfigUtils = class {
|
|
|
239
249
|
/**
|
|
240
250
|
* Creates an agent config from flag value data.
|
|
241
251
|
*
|
|
252
|
+
* @param key The configuration key
|
|
242
253
|
* @param flagValue The flag value from LaunchDarkly
|
|
243
|
-
* @param
|
|
254
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
244
255
|
* @returns An agent configuration
|
|
245
256
|
*/
|
|
246
|
-
static toAgentConfig(key, flagValue,
|
|
257
|
+
static toAgentConfig(key, flagValue, trackerFactory) {
|
|
247
258
|
return {
|
|
248
259
|
...this._toBaseConfig(key, flagValue),
|
|
249
|
-
|
|
260
|
+
createTracker: trackerFactory,
|
|
250
261
|
instructions: flagValue.instructions,
|
|
251
262
|
judgeConfiguration: flagValue.judgeConfiguration
|
|
252
263
|
};
|
|
@@ -254,11 +265,12 @@ var LDAIConfigUtils = class {
|
|
|
254
265
|
/**
|
|
255
266
|
* Creates a judge config from flag value data.
|
|
256
267
|
*
|
|
268
|
+
* @param key The configuration key
|
|
257
269
|
* @param flagValue The flag value from LaunchDarkly
|
|
258
|
-
* @param
|
|
270
|
+
* @param trackerFactory A factory function that creates a new tracker for each execution
|
|
259
271
|
* @returns A judge configuration
|
|
260
272
|
*/
|
|
261
|
-
static toJudgeConfig(key, flagValue,
|
|
273
|
+
static toJudgeConfig(key, flagValue, trackerFactory) {
|
|
262
274
|
let evaluationMetricKey;
|
|
263
275
|
if (flagValue.evaluationMetricKey && flagValue.evaluationMetricKey.trim().length > 0) {
|
|
264
276
|
evaluationMetricKey = flagValue.evaluationMetricKey.trim();
|
|
@@ -270,69 +282,269 @@ var LDAIConfigUtils = class {
|
|
|
270
282
|
}
|
|
271
283
|
return {
|
|
272
284
|
...this._toBaseConfig(key, flagValue),
|
|
273
|
-
|
|
285
|
+
createTracker: trackerFactory,
|
|
274
286
|
messages: flagValue.messages,
|
|
275
287
|
evaluationMetricKey
|
|
276
288
|
};
|
|
277
289
|
}
|
|
278
290
|
};
|
|
279
291
|
|
|
280
|
-
// src/api/
|
|
281
|
-
|
|
292
|
+
// src/api/graph/AgentGraphNode.ts
|
|
293
|
+
var AgentGraphNode = class {
|
|
294
|
+
constructor(_key, _config, _edges) {
|
|
295
|
+
this._key = _key;
|
|
296
|
+
this._config = _config;
|
|
297
|
+
this._edges = _edges;
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Returns the agent config key that identifies this node in the graph.
|
|
301
|
+
*/
|
|
302
|
+
getKey() {
|
|
303
|
+
return this._key;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Returns the underlying AIAgentConfig for this node.
|
|
307
|
+
* Use `getConfig().tracker` to record node-level metrics.
|
|
308
|
+
*/
|
|
309
|
+
getConfig() {
|
|
310
|
+
return this._config;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Returns the outgoing edges from this node to its children.
|
|
314
|
+
*/
|
|
315
|
+
getEdges() {
|
|
316
|
+
return this._edges;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Returns `true` if this node has no outgoing edges (i.e., it is a terminal/leaf node).
|
|
320
|
+
*/
|
|
321
|
+
isTerminal() {
|
|
322
|
+
return this._edges.length === 0;
|
|
323
|
+
}
|
|
324
|
+
};
|
|
282
325
|
|
|
283
|
-
// src/api/
|
|
284
|
-
var
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
326
|
+
// src/api/graph/AgentGraphDefinition.ts
|
|
327
|
+
var AgentGraphDefinition = class _AgentGraphDefinition {
|
|
328
|
+
constructor(_agentGraph, _nodes, enabled, _createTracker) {
|
|
329
|
+
this._agentGraph = _agentGraph;
|
|
330
|
+
this._nodes = _nodes;
|
|
331
|
+
this.enabled = enabled;
|
|
332
|
+
this._createTracker = _createTracker;
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Builds a node map from a raw agent graph flag value and a map of pre-fetched agent configs.
|
|
336
|
+
*
|
|
337
|
+
* @param graph Raw graph flag value from LaunchDarkly.
|
|
338
|
+
* @param agentConfigs Map of agent config key to resolved LDAIAgentConfig.
|
|
339
|
+
* @returns Record mapping agent config keys to AgentGraphNode instances.
|
|
340
|
+
*/
|
|
341
|
+
static buildNodes(graph, agentConfigs) {
|
|
342
|
+
const nodes = {};
|
|
343
|
+
const allKeys = _AgentGraphDefinition.collectAllKeys(graph);
|
|
344
|
+
allKeys.forEach((key) => {
|
|
345
|
+
const config = agentConfigs[key];
|
|
346
|
+
if (!config) {
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
const outgoingEdges = graph.edges?.[key] ?? [];
|
|
350
|
+
nodes[key] = new AgentGraphNode(key, config, outgoingEdges);
|
|
351
|
+
});
|
|
352
|
+
return nodes;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Returns the children of the node identified by `nodeKey`.
|
|
356
|
+
*
|
|
357
|
+
* @param nodeKey The agent config key of the parent node.
|
|
358
|
+
*/
|
|
359
|
+
getChildNodes(nodeKey) {
|
|
360
|
+
const node = this._nodes[nodeKey];
|
|
361
|
+
if (!node) {
|
|
362
|
+
return [];
|
|
288
363
|
}
|
|
289
|
-
return
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
364
|
+
return node.getEdges().map((edge) => this._nodes[edge.key]).filter((n) => n !== void 0);
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Returns all nodes that have a direct edge to the node identified by `nodeKey`.
|
|
368
|
+
*
|
|
369
|
+
* @param nodeKey The agent config key of the child node.
|
|
370
|
+
*/
|
|
371
|
+
getParentNodes(nodeKey) {
|
|
372
|
+
return Object.values(this._nodes).filter(
|
|
373
|
+
(node) => node.getEdges().some((edge) => edge.key === nodeKey)
|
|
374
|
+
);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Returns all terminal nodes (nodes with no outgoing edges).
|
|
378
|
+
*/
|
|
379
|
+
terminalNodes() {
|
|
380
|
+
return Object.values(this._nodes).filter((node) => node.isTerminal());
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Returns the root node of the graph.
|
|
384
|
+
*/
|
|
385
|
+
rootNode() {
|
|
386
|
+
return this._nodes[this._agentGraph.root];
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* Returns the node with the given key, or `null` if not found.
|
|
390
|
+
*
|
|
391
|
+
* @param nodeKey The agent config key to look up.
|
|
392
|
+
*/
|
|
393
|
+
getNode(nodeKey) {
|
|
394
|
+
return this._nodes[nodeKey] ?? null;
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* Returns the underlying raw graph configuration from LaunchDarkly.
|
|
398
|
+
*/
|
|
399
|
+
getConfig() {
|
|
400
|
+
return this._agentGraph;
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Returns a new {@link LDGraphTracker} for this graph invocation.
|
|
404
|
+
*
|
|
405
|
+
* Call this once per invocation. Each call produces a tracker with a fresh `runId`
|
|
406
|
+
* that groups all events for that invocation.
|
|
407
|
+
*/
|
|
408
|
+
createTracker() {
|
|
409
|
+
return this._createTracker();
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* Traverses the graph breadth-first from the root to all terminal nodes.
|
|
413
|
+
*
|
|
414
|
+
* Nodes at the same depth are processed before advancing to the next depth.
|
|
415
|
+
* The value returned by `fn` is stored in the mutable `executionContext` under
|
|
416
|
+
* the node's key, making upstream results available to downstream nodes.
|
|
417
|
+
*
|
|
418
|
+
* Cyclic graphs are handled safely — each node is visited at most once.
|
|
419
|
+
*
|
|
420
|
+
* @param fn Callback invoked for each node. Its return value is added to
|
|
421
|
+
* `executionContext` keyed by the node's config key.
|
|
422
|
+
* @param initialExecutionContext Optional initial context to seed the traversal.
|
|
423
|
+
*/
|
|
424
|
+
traverse(fn, initialExecutionContext = {}) {
|
|
425
|
+
const root = this.rootNode();
|
|
426
|
+
if (!root) {
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
const executionContext = { ...initialExecutionContext };
|
|
430
|
+
const visited = /* @__PURE__ */ new Set();
|
|
431
|
+
const queue = [root];
|
|
432
|
+
visited.add(root.getKey());
|
|
433
|
+
while (queue.length > 0) {
|
|
434
|
+
const node = queue.shift();
|
|
435
|
+
const result = fn(node, executionContext);
|
|
436
|
+
executionContext[node.getKey()] = result;
|
|
437
|
+
node.getEdges().forEach((edge) => {
|
|
438
|
+
if (!visited.has(edge.key)) {
|
|
439
|
+
const child = this._nodes[edge.key];
|
|
440
|
+
if (child) {
|
|
441
|
+
visited.add(edge.key);
|
|
442
|
+
queue.push(child);
|
|
443
|
+
}
|
|
300
444
|
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
additionalProperties: false
|
|
304
|
-
};
|
|
445
|
+
});
|
|
446
|
+
}
|
|
305
447
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
448
|
+
/**
|
|
449
|
+
* Traverses the graph from terminal nodes up to the root.
|
|
450
|
+
*
|
|
451
|
+
* Uses BFS upward via parent edges so that each node is processed only after
|
|
452
|
+
* all of its reachable descendants have been processed. The root is always
|
|
453
|
+
* visited last. Cyclic graphs are handled safely — each node is visited at
|
|
454
|
+
* most once; if the graph has no terminal nodes, this method returns without
|
|
455
|
+
* invoking `fn`.
|
|
456
|
+
*
|
|
457
|
+
* **Ordering note:** Within a single BFS level (nodes at the same depth from a
|
|
458
|
+
* terminal) the visit order is not strictly guaranteed. The guarantee is only
|
|
459
|
+
* that a node is visited before any of its ancestors — not that siblings at the
|
|
460
|
+
* same depth are visited in a specific order relative to each other.
|
|
461
|
+
*
|
|
462
|
+
* The value returned by `fn` is stored in the mutable `executionContext` under
|
|
463
|
+
* the node's key.
|
|
464
|
+
*
|
|
465
|
+
* @param fn Callback invoked for each node. Its return value is added to
|
|
466
|
+
* `executionContext` keyed by the node's config key.
|
|
467
|
+
* @param initialExecutionContext Optional initial context to seed the traversal.
|
|
468
|
+
*/
|
|
469
|
+
reverseTraverse(fn, initialExecutionContext = {}) {
|
|
470
|
+
const terminals = this.terminalNodes();
|
|
471
|
+
if (terminals.length === 0) {
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
const executionContext = { ...initialExecutionContext };
|
|
475
|
+
const rootKey = this._agentGraph.root;
|
|
476
|
+
const visited = /* @__PURE__ */ new Set();
|
|
477
|
+
let queue = terminals;
|
|
478
|
+
while (queue.length > 0) {
|
|
479
|
+
const nextQueue = [];
|
|
480
|
+
queue.forEach((node) => {
|
|
481
|
+
const key = node.getKey();
|
|
482
|
+
if (visited.has(key)) {
|
|
483
|
+
return;
|
|
319
484
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
485
|
+
visited.add(key);
|
|
486
|
+
if (key === rootKey) {
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
const result = fn(node, executionContext);
|
|
490
|
+
executionContext[key] = result;
|
|
491
|
+
this.getParentNodes(key).forEach((parent) => {
|
|
492
|
+
if (!visited.has(parent.getKey())) {
|
|
493
|
+
nextQueue.push(parent);
|
|
494
|
+
}
|
|
495
|
+
});
|
|
496
|
+
});
|
|
497
|
+
queue = nextQueue;
|
|
498
|
+
}
|
|
499
|
+
const root = this._nodes[rootKey];
|
|
500
|
+
if (root && visited.has(rootKey)) {
|
|
501
|
+
const result = fn(root, executionContext);
|
|
502
|
+
executionContext[rootKey] = result;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* Collects every unique node key referenced in the graph (root + all edge sources
|
|
507
|
+
* and targets).
|
|
508
|
+
*/
|
|
509
|
+
static collectAllKeys(graph) {
|
|
510
|
+
const keys = /* @__PURE__ */ new Set();
|
|
511
|
+
keys.add(graph.root);
|
|
512
|
+
if (graph.edges) {
|
|
513
|
+
Object.entries(graph.edges).forEach(([sourceKey, edges]) => {
|
|
514
|
+
keys.add(sourceKey);
|
|
515
|
+
edges.forEach((edge) => {
|
|
516
|
+
keys.add(edge.key);
|
|
517
|
+
});
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
return keys;
|
|
324
521
|
}
|
|
325
522
|
};
|
|
326
523
|
|
|
327
524
|
// src/api/judge/Judge.ts
|
|
525
|
+
import Mustache from "mustache";
|
|
526
|
+
var EVALUATION_SCHEMA = {
|
|
527
|
+
type: "object",
|
|
528
|
+
properties: {
|
|
529
|
+
score: {
|
|
530
|
+
type: "number",
|
|
531
|
+
minimum: 0,
|
|
532
|
+
maximum: 1,
|
|
533
|
+
description: "Score between 0.0 and 1.0."
|
|
534
|
+
},
|
|
535
|
+
reasoning: {
|
|
536
|
+
type: "string",
|
|
537
|
+
description: "Reasoning behind the score."
|
|
538
|
+
}
|
|
539
|
+
},
|
|
540
|
+
required: ["score", "reasoning"],
|
|
541
|
+
additionalProperties: false
|
|
542
|
+
};
|
|
328
543
|
var Judge = class {
|
|
329
|
-
constructor(_aiConfig,
|
|
544
|
+
constructor(_aiConfig, _aiProvider, logger) {
|
|
330
545
|
this._aiConfig = _aiConfig;
|
|
331
|
-
this._aiConfigTracker = _aiConfigTracker;
|
|
332
546
|
this._aiProvider = _aiProvider;
|
|
333
547
|
this._logger = logger;
|
|
334
|
-
const evaluationMetricKey = this._getEvaluationMetricKey();
|
|
335
|
-
this._evaluationResponseStructure = EvaluationSchemaBuilder.build(evaluationMetricKey);
|
|
336
548
|
}
|
|
337
549
|
/**
|
|
338
550
|
* Gets the evaluation metric key, prioritizing evaluationMetricKey over evaluationMetricKeys.
|
|
@@ -358,56 +570,62 @@ var Judge = class {
|
|
|
358
570
|
* @param input The input prompt or question that was provided to the AI
|
|
359
571
|
* @param output The AI-generated response to be evaluated
|
|
360
572
|
* @param samplingRate Sampling rate (0-1) to determine if evaluation should be processed (defaults to 1)
|
|
361
|
-
* @returns Promise that resolves to evaluation results
|
|
573
|
+
* @returns Promise that resolves to evaluation results
|
|
362
574
|
*/
|
|
363
575
|
async evaluate(input, output, samplingRate = 1) {
|
|
576
|
+
const result = {
|
|
577
|
+
success: false,
|
|
578
|
+
sampled: false,
|
|
579
|
+
judgeConfigKey: this._aiConfig.key
|
|
580
|
+
};
|
|
581
|
+
const tracker = this._aiConfig.createTracker();
|
|
364
582
|
try {
|
|
365
583
|
const evaluationMetricKey = this._getEvaluationMetricKey();
|
|
366
584
|
if (!evaluationMetricKey) {
|
|
367
585
|
this._logger?.warn(
|
|
368
586
|
"Judge configuration is missing required evaluation metric key",
|
|
369
|
-
|
|
587
|
+
tracker.getTrackData()
|
|
370
588
|
);
|
|
371
|
-
|
|
589
|
+
result.sampled = true;
|
|
590
|
+
result.errorMessage = "Judge configuration is missing required evaluation metric key";
|
|
591
|
+
return result;
|
|
372
592
|
}
|
|
373
593
|
if (!this._aiConfig.messages) {
|
|
374
|
-
this._logger?.warn(
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
return void 0;
|
|
594
|
+
this._logger?.warn("Judge configuration must include messages", tracker.getTrackData());
|
|
595
|
+
result.sampled = true;
|
|
596
|
+
result.errorMessage = "Judge configuration must include messages";
|
|
597
|
+
return result;
|
|
379
598
|
}
|
|
380
599
|
if (Math.random() > samplingRate) {
|
|
381
600
|
this._logger?.debug(`Judge evaluation skipped due to sampling rate: ${samplingRate}`);
|
|
382
|
-
return
|
|
601
|
+
return result;
|
|
383
602
|
}
|
|
603
|
+
result.sampled = true;
|
|
384
604
|
const messages = this._constructEvaluationMessages(input, output);
|
|
385
|
-
const response = await
|
|
386
|
-
(
|
|
387
|
-
() => this._aiProvider.invokeStructuredModel(messages,
|
|
605
|
+
const response = await tracker.trackMetricsOf(
|
|
606
|
+
(r) => r.metrics,
|
|
607
|
+
() => this._aiProvider.invokeStructuredModel(messages, EVALUATION_SCHEMA)
|
|
388
608
|
);
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
if (!evals[evaluationMetricKey]) {
|
|
609
|
+
const evalResult = this._parseEvaluationResponse(response.data);
|
|
610
|
+
if (!evalResult) {
|
|
392
611
|
this._logger?.warn(
|
|
393
|
-
|
|
394
|
-
|
|
612
|
+
`Could not parse evaluation response: ${JSON.stringify(response.data)}`,
|
|
613
|
+
tracker.getTrackData()
|
|
395
614
|
);
|
|
396
|
-
|
|
615
|
+
return result;
|
|
397
616
|
}
|
|
398
617
|
return {
|
|
399
|
-
|
|
400
|
-
success,
|
|
401
|
-
|
|
618
|
+
...result,
|
|
619
|
+
success: response.metrics.success,
|
|
620
|
+
score: evalResult.score,
|
|
621
|
+
reasoning: evalResult.reasoning,
|
|
622
|
+
metricKey: evaluationMetricKey
|
|
402
623
|
};
|
|
403
624
|
} catch (error) {
|
|
404
625
|
this._logger?.error("Judge evaluation failed:", error);
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
error: error instanceof Error ? error.message : "Unknown error",
|
|
409
|
-
judgeConfigKey: this._aiConfig.key
|
|
410
|
-
};
|
|
626
|
+
result.sampled = true;
|
|
627
|
+
result.errorMessage = error instanceof Error ? error.message : "Unknown error";
|
|
628
|
+
return result;
|
|
411
629
|
}
|
|
412
630
|
}
|
|
413
631
|
/**
|
|
@@ -416,7 +634,7 @@ var Judge = class {
|
|
|
416
634
|
* @param messages Array of messages representing the conversation history
|
|
417
635
|
* @param response The AI response to be evaluated
|
|
418
636
|
* @param samplingRatio Sampling ratio (0-1) to determine if evaluation should be processed (defaults to 1)
|
|
419
|
-
* @returns Promise that resolves to evaluation results
|
|
637
|
+
* @returns Promise that resolves to evaluation results
|
|
420
638
|
*/
|
|
421
639
|
async evaluateMessages(messages, response, samplingRatio = 1) {
|
|
422
640
|
const input = messages.length === 0 ? "" : messages.map((msg) => msg.content).join("\r\n");
|
|
@@ -429,12 +647,6 @@ var Judge = class {
|
|
|
429
647
|
getAIConfig() {
|
|
430
648
|
return this._aiConfig;
|
|
431
649
|
}
|
|
432
|
-
/**
|
|
433
|
-
* Returns the tracker associated with this judge.
|
|
434
|
-
*/
|
|
435
|
-
getTracker() {
|
|
436
|
-
return this._aiConfigTracker;
|
|
437
|
-
}
|
|
438
650
|
/**
|
|
439
651
|
* Returns the AI provider used by this judge.
|
|
440
652
|
*/
|
|
@@ -461,43 +673,23 @@ var Judge = class {
|
|
|
461
673
|
return Mustache.render(content, variables, void 0, { escape: (item) => item });
|
|
462
674
|
}
|
|
463
675
|
/**
|
|
464
|
-
* Parses the structured evaluation response
|
|
676
|
+
* Parses the structured evaluation response. Expects top-level {score, reasoning}.
|
|
677
|
+
* Returns score and reasoning, or undefined if parsing fails.
|
|
465
678
|
*/
|
|
466
|
-
_parseEvaluationResponse(data
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
if (!data.evaluations || typeof data.evaluations !== "object") {
|
|
470
|
-
this._logger?.warn("Invalid response: missing or invalid evaluations object");
|
|
471
|
-
return results;
|
|
472
|
-
}
|
|
473
|
-
const evaluation = evaluations[evaluationMetricKey];
|
|
474
|
-
if (!evaluation || typeof evaluation !== "object") {
|
|
475
|
-
this._logger?.warn(
|
|
476
|
-
`Missing evaluation for metric key: ${evaluationMetricKey}`,
|
|
477
|
-
this._aiConfigTracker.getTrackData()
|
|
478
|
-
);
|
|
479
|
-
return results;
|
|
679
|
+
_parseEvaluationResponse(data) {
|
|
680
|
+
if (!data || typeof data !== "object" || Array.isArray(data)) {
|
|
681
|
+
return void 0;
|
|
480
682
|
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
this._logger?.warn(
|
|
484
|
-
`Invalid score evaluated for ${evaluationMetricKey}: ${evalData.score}. Score must be a number between 0 and 1 inclusive`,
|
|
485
|
-
this._aiConfigTracker.getTrackData()
|
|
486
|
-
);
|
|
487
|
-
return results;
|
|
683
|
+
if (typeof data.score !== "number" || data.score < 0 || data.score > 1) {
|
|
684
|
+
return void 0;
|
|
488
685
|
}
|
|
489
|
-
if (typeof
|
|
490
|
-
|
|
491
|
-
`Invalid reasoning evaluated for ${evaluationMetricKey}: ${evalData.reasoning}. Reasoning must be a string`,
|
|
492
|
-
this._aiConfigTracker.getTrackData()
|
|
493
|
-
);
|
|
494
|
-
return results;
|
|
686
|
+
if (typeof data.reasoning !== "string") {
|
|
687
|
+
return void 0;
|
|
495
688
|
}
|
|
496
|
-
|
|
497
|
-
score:
|
|
498
|
-
reasoning:
|
|
689
|
+
return {
|
|
690
|
+
score: data.score,
|
|
691
|
+
reasoning: data.reasoning
|
|
499
692
|
};
|
|
500
|
-
return results;
|
|
501
693
|
}
|
|
502
694
|
};
|
|
503
695
|
|
|
@@ -697,27 +889,62 @@ function createVercelAISDKTokenUsage(data) {
|
|
|
697
889
|
}
|
|
698
890
|
|
|
699
891
|
// src/LDAIConfigTrackerImpl.ts
|
|
700
|
-
var LDAIConfigTrackerImpl = class {
|
|
701
|
-
constructor(_ldClient, _configKey, _variationKey, _version, _modelName, _providerName, _context) {
|
|
892
|
+
var LDAIConfigTrackerImpl = class _LDAIConfigTrackerImpl {
|
|
893
|
+
constructor(_ldClient, _runId, _configKey, _variationKey, _version, _modelName, _providerName, _context, _graphKey) {
|
|
702
894
|
this._ldClient = _ldClient;
|
|
895
|
+
this._runId = _runId;
|
|
703
896
|
this._configKey = _configKey;
|
|
704
897
|
this._variationKey = _variationKey;
|
|
705
898
|
this._version = _version;
|
|
706
899
|
this._modelName = _modelName;
|
|
707
900
|
this._providerName = _providerName;
|
|
708
901
|
this._context = _context;
|
|
902
|
+
this._graphKey = _graphKey;
|
|
709
903
|
this._trackedMetrics = {};
|
|
710
904
|
}
|
|
711
905
|
getTrackData() {
|
|
712
906
|
return {
|
|
713
|
-
|
|
907
|
+
runId: this._runId,
|
|
714
908
|
configKey: this._configKey,
|
|
909
|
+
variationKey: this._variationKey,
|
|
715
910
|
version: this._version,
|
|
716
911
|
modelName: this._modelName,
|
|
717
|
-
providerName: this._providerName
|
|
912
|
+
providerName: this._providerName,
|
|
913
|
+
...this._graphKey !== void 0 ? { graphKey: this._graphKey } : {}
|
|
718
914
|
};
|
|
719
915
|
}
|
|
916
|
+
get resumptionToken() {
|
|
917
|
+
const json = JSON.stringify({
|
|
918
|
+
runId: this._runId,
|
|
919
|
+
configKey: this._configKey,
|
|
920
|
+
variationKey: this._variationKey,
|
|
921
|
+
version: this._version,
|
|
922
|
+
...this._graphKey !== void 0 ? { graphKey: this._graphKey } : {}
|
|
923
|
+
});
|
|
924
|
+
return Buffer.from(json).toString("base64url");
|
|
925
|
+
}
|
|
926
|
+
static fromResumptionToken(token, ldClient, context) {
|
|
927
|
+
const json = Buffer.from(token, "base64url").toString("utf8");
|
|
928
|
+
const payload = JSON.parse(json);
|
|
929
|
+
return new _LDAIConfigTrackerImpl(
|
|
930
|
+
ldClient,
|
|
931
|
+
payload.runId,
|
|
932
|
+
payload.configKey,
|
|
933
|
+
payload.variationKey ?? "",
|
|
934
|
+
payload.version,
|
|
935
|
+
"",
|
|
936
|
+
"",
|
|
937
|
+
context,
|
|
938
|
+
payload.graphKey
|
|
939
|
+
);
|
|
940
|
+
}
|
|
720
941
|
trackDuration(duration) {
|
|
942
|
+
if (this._trackedMetrics.durationMs !== void 0) {
|
|
943
|
+
this._ldClient.logger?.warn(
|
|
944
|
+
"Duration has already been tracked for this execution. Use createTracker() for a new execution."
|
|
945
|
+
);
|
|
946
|
+
return;
|
|
947
|
+
}
|
|
721
948
|
this._trackedMetrics.durationMs = duration;
|
|
722
949
|
this._ldClient.track("$ld:ai:duration:total", this._context, this.getTrackData(), duration);
|
|
723
950
|
}
|
|
@@ -733,6 +960,12 @@ var LDAIConfigTrackerImpl = class {
|
|
|
733
960
|
}
|
|
734
961
|
}
|
|
735
962
|
trackTimeToFirstToken(timeToFirstTokenMs) {
|
|
963
|
+
if (this._trackedMetrics.timeToFirstTokenMs !== void 0) {
|
|
964
|
+
this._ldClient.logger?.warn(
|
|
965
|
+
"Time to first token has already been tracked for this execution. Use createTracker() for a new execution."
|
|
966
|
+
);
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
736
969
|
this._trackedMetrics.timeToFirstTokenMs = timeToFirstTokenMs;
|
|
737
970
|
this._ldClient.track(
|
|
738
971
|
"$ld:ai:tokens:ttf",
|
|
@@ -741,22 +974,30 @@ var LDAIConfigTrackerImpl = class {
|
|
|
741
974
|
timeToFirstTokenMs
|
|
742
975
|
);
|
|
743
976
|
}
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
}
|
|
977
|
+
trackJudgeResult(result) {
|
|
978
|
+
if (!result.sampled || !result.success) {
|
|
979
|
+
return;
|
|
980
|
+
}
|
|
981
|
+
if (result.metricKey !== void 0 && result.score !== void 0) {
|
|
982
|
+
const trackData = result.judgeConfigKey ? { ...this.getTrackData(), judgeConfigKey: result.judgeConfigKey } : this.getTrackData();
|
|
983
|
+
this._ldClient.track(result.metricKey, this._context, trackData, result.score);
|
|
984
|
+
}
|
|
748
985
|
}
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
evalScore.score
|
|
756
|
-
);
|
|
986
|
+
trackToolCall(toolKey) {
|
|
987
|
+
this._ldClient.track("$ld:ai:tool_call", this._context, { ...this.getTrackData(), toolKey }, 1);
|
|
988
|
+
}
|
|
989
|
+
trackToolCalls(toolKeys) {
|
|
990
|
+
toolKeys.forEach((toolKey) => {
|
|
991
|
+
this.trackToolCall(toolKey);
|
|
757
992
|
});
|
|
758
993
|
}
|
|
759
994
|
trackFeedback(feedback) {
|
|
995
|
+
if (this._trackedMetrics.feedback !== void 0) {
|
|
996
|
+
this._ldClient.logger?.warn(
|
|
997
|
+
"Feedback has already been tracked for this execution. Use createTracker() for a new execution."
|
|
998
|
+
);
|
|
999
|
+
return;
|
|
1000
|
+
}
|
|
760
1001
|
this._trackedMetrics.feedback = feedback;
|
|
761
1002
|
if (feedback.kind === "positive" /* Positive */) {
|
|
762
1003
|
this._ldClient.track("$ld:ai:feedback:user:positive", this._context, this.getTrackData(), 1);
|
|
@@ -765,10 +1006,22 @@ var LDAIConfigTrackerImpl = class {
|
|
|
765
1006
|
}
|
|
766
1007
|
}
|
|
767
1008
|
trackSuccess() {
|
|
1009
|
+
if (this._trackedMetrics.success !== void 0) {
|
|
1010
|
+
this._ldClient.logger?.warn(
|
|
1011
|
+
"Generation result has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1012
|
+
);
|
|
1013
|
+
return;
|
|
1014
|
+
}
|
|
768
1015
|
this._trackedMetrics.success = true;
|
|
769
1016
|
this._ldClient.track("$ld:ai:generation:success", this._context, this.getTrackData(), 1);
|
|
770
1017
|
}
|
|
771
1018
|
trackError() {
|
|
1019
|
+
if (this._trackedMetrics.success !== void 0) {
|
|
1020
|
+
this._ldClient.logger?.warn(
|
|
1021
|
+
"Generation result has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1022
|
+
);
|
|
1023
|
+
return;
|
|
1024
|
+
}
|
|
772
1025
|
this._trackedMetrics.success = false;
|
|
773
1026
|
this._ldClient.track("$ld:ai:generation:error", this._context, this.getTrackData(), 1);
|
|
774
1027
|
}
|
|
@@ -861,6 +1114,12 @@ var LDAIConfigTrackerImpl = class {
|
|
|
861
1114
|
}
|
|
862
1115
|
}
|
|
863
1116
|
trackTokens(tokens) {
|
|
1117
|
+
if (this._trackedMetrics.tokens !== void 0) {
|
|
1118
|
+
this._ldClient.logger?.warn(
|
|
1119
|
+
"Token usage has already been tracked for this execution. Use createTracker() for a new execution."
|
|
1120
|
+
);
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
864
1123
|
this._trackedMetrics.tokens = tokens;
|
|
865
1124
|
const trackData = this.getTrackData();
|
|
866
1125
|
if (tokens.total > 0) {
|
|
@@ -881,9 +1140,155 @@ var LDAIConfigTrackerImpl = class {
|
|
|
881
1140
|
}
|
|
882
1141
|
};
|
|
883
1142
|
|
|
1143
|
+
// src/LDGraphTrackerImpl.ts
|
|
1144
|
+
var LDGraphTrackerImpl = class _LDGraphTrackerImpl {
|
|
1145
|
+
constructor(_ldClient, _runId, _graphKey, _variationKey, _version, _context) {
|
|
1146
|
+
this._ldClient = _ldClient;
|
|
1147
|
+
this._runId = _runId;
|
|
1148
|
+
this._graphKey = _graphKey;
|
|
1149
|
+
this._variationKey = _variationKey;
|
|
1150
|
+
this._version = _version;
|
|
1151
|
+
this._context = _context;
|
|
1152
|
+
this._summary = {};
|
|
1153
|
+
}
|
|
1154
|
+
/**
|
|
1155
|
+
* Reconstructs an {@link LDGraphTrackerImpl} from a resumption token, preserving
|
|
1156
|
+
* the original `runId` so all events continue to be correlated under the same run.
|
|
1157
|
+
*
|
|
1158
|
+
* **Security note:** The token contains the flag variation key and version.
|
|
1159
|
+
* Do not pass the raw token to untrusted clients.
|
|
1160
|
+
*
|
|
1161
|
+
* @param token URL-safe Base64-encoded token produced by {@link LDGraphTrackerImpl.resumptionToken}.
|
|
1162
|
+
* @param ldClient LaunchDarkly client instance.
|
|
1163
|
+
* @param context LDContext for the new tracker.
|
|
1164
|
+
*/
|
|
1165
|
+
static fromResumptionToken(token, ldClient, context) {
|
|
1166
|
+
const json = Buffer.from(token, "base64url").toString("utf8");
|
|
1167
|
+
const data = JSON.parse(json);
|
|
1168
|
+
return new _LDGraphTrackerImpl(
|
|
1169
|
+
ldClient,
|
|
1170
|
+
data.runId,
|
|
1171
|
+
data.graphKey,
|
|
1172
|
+
data.variationKey,
|
|
1173
|
+
data.version,
|
|
1174
|
+
context
|
|
1175
|
+
);
|
|
1176
|
+
}
|
|
1177
|
+
getTrackData() {
|
|
1178
|
+
const data = {
|
|
1179
|
+
runId: this._runId,
|
|
1180
|
+
graphKey: this._graphKey,
|
|
1181
|
+
version: this._version
|
|
1182
|
+
};
|
|
1183
|
+
if (this._variationKey !== void 0) {
|
|
1184
|
+
data.variationKey = this._variationKey;
|
|
1185
|
+
}
|
|
1186
|
+
return data;
|
|
1187
|
+
}
|
|
1188
|
+
getSummary() {
|
|
1189
|
+
return { ...this._summary };
|
|
1190
|
+
}
|
|
1191
|
+
get resumptionToken() {
|
|
1192
|
+
const parts = [
|
|
1193
|
+
`"runId":${JSON.stringify(this._runId)}`,
|
|
1194
|
+
`"graphKey":${JSON.stringify(this._graphKey)}`
|
|
1195
|
+
];
|
|
1196
|
+
if (this._variationKey !== void 0) {
|
|
1197
|
+
parts.push(`"variationKey":${JSON.stringify(this._variationKey)}`);
|
|
1198
|
+
}
|
|
1199
|
+
parts.push(`"version":${this._version}`);
|
|
1200
|
+
const json = `{${parts.join(",")}}`;
|
|
1201
|
+
return Buffer.from(json).toString("base64url");
|
|
1202
|
+
}
|
|
1203
|
+
trackInvocationSuccess() {
|
|
1204
|
+
if (this._summary.success !== void 0) {
|
|
1205
|
+
this._ldClient.logger?.warn(
|
|
1206
|
+
"LDGraphTracker: invocation success/failure already recorded for this run \u2014 dropping duplicate call."
|
|
1207
|
+
);
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
this._summary.success = true;
|
|
1211
|
+
this._ldClient.track("$ld:ai:graph:invocation_success", this._context, this.getTrackData(), 1);
|
|
1212
|
+
}
|
|
1213
|
+
trackInvocationFailure() {
|
|
1214
|
+
if (this._summary.success !== void 0) {
|
|
1215
|
+
this._ldClient.logger?.warn(
|
|
1216
|
+
"LDGraphTracker: invocation success/failure already recorded for this run \u2014 dropping duplicate call."
|
|
1217
|
+
);
|
|
1218
|
+
return;
|
|
1219
|
+
}
|
|
1220
|
+
this._summary.success = false;
|
|
1221
|
+
this._ldClient.track("$ld:ai:graph:invocation_failure", this._context, this.getTrackData(), 1);
|
|
1222
|
+
}
|
|
1223
|
+
trackDuration(durationMs) {
|
|
1224
|
+
if (this._summary.durationMs !== void 0) {
|
|
1225
|
+
this._ldClient.logger?.warn(
|
|
1226
|
+
"LDGraphTracker: trackDuration already called for this run \u2014 dropping duplicate call."
|
|
1227
|
+
);
|
|
1228
|
+
return;
|
|
1229
|
+
}
|
|
1230
|
+
this._summary.durationMs = durationMs;
|
|
1231
|
+
this._ldClient.track(
|
|
1232
|
+
"$ld:ai:graph:duration:total",
|
|
1233
|
+
this._context,
|
|
1234
|
+
this.getTrackData(),
|
|
1235
|
+
durationMs
|
|
1236
|
+
);
|
|
1237
|
+
}
|
|
1238
|
+
trackTotalTokens(tokens) {
|
|
1239
|
+
if (this._summary.tokens !== void 0) {
|
|
1240
|
+
this._ldClient.logger?.warn(
|
|
1241
|
+
"LDGraphTracker: trackTotalTokens already called for this run \u2014 dropping duplicate call."
|
|
1242
|
+
);
|
|
1243
|
+
return;
|
|
1244
|
+
}
|
|
1245
|
+
this._summary.tokens = { ...tokens };
|
|
1246
|
+
this._ldClient.track(
|
|
1247
|
+
"$ld:ai:graph:total_tokens",
|
|
1248
|
+
this._context,
|
|
1249
|
+
this.getTrackData(),
|
|
1250
|
+
tokens.total
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1253
|
+
trackPath(path) {
|
|
1254
|
+
if (this._summary.path !== void 0) {
|
|
1255
|
+
this._ldClient.logger?.warn(
|
|
1256
|
+
"LDGraphTracker: trackPath already called for this run \u2014 dropping duplicate call."
|
|
1257
|
+
);
|
|
1258
|
+
return;
|
|
1259
|
+
}
|
|
1260
|
+
this._summary.path = [...path];
|
|
1261
|
+
this._ldClient.track("$ld:ai:graph:path", this._context, { ...this.getTrackData(), path }, 1);
|
|
1262
|
+
}
|
|
1263
|
+
trackRedirect(sourceKey, redirectedTarget) {
|
|
1264
|
+
this._ldClient.track(
|
|
1265
|
+
"$ld:ai:graph:redirect",
|
|
1266
|
+
this._context,
|
|
1267
|
+
{ ...this.getTrackData(), sourceKey, redirectedTarget },
|
|
1268
|
+
1
|
|
1269
|
+
);
|
|
1270
|
+
}
|
|
1271
|
+
trackHandoffSuccess(sourceKey, targetKey) {
|
|
1272
|
+
this._ldClient.track(
|
|
1273
|
+
"$ld:ai:graph:handoff_success",
|
|
1274
|
+
this._context,
|
|
1275
|
+
{ ...this.getTrackData(), sourceKey, targetKey },
|
|
1276
|
+
1
|
|
1277
|
+
);
|
|
1278
|
+
}
|
|
1279
|
+
trackHandoffFailure(sourceKey, targetKey) {
|
|
1280
|
+
this._ldClient.track(
|
|
1281
|
+
"$ld:ai:graph:handoff_failure",
|
|
1282
|
+
this._context,
|
|
1283
|
+
{ ...this.getTrackData(), sourceKey, targetKey },
|
|
1284
|
+
1
|
|
1285
|
+
);
|
|
1286
|
+
}
|
|
1287
|
+
};
|
|
1288
|
+
|
|
884
1289
|
// src/sdkInfo.ts
|
|
885
1290
|
var aiSdkName = "@launchdarkly/server-sdk-ai";
|
|
886
|
-
var aiSdkVersion = "0.
|
|
1291
|
+
var aiSdkVersion = "0.17.1";
|
|
887
1292
|
var aiSdkLanguage = "javascript";
|
|
888
1293
|
|
|
889
1294
|
// src/LDAIClientImpl.ts
|
|
@@ -894,6 +1299,7 @@ var TRACK_USAGE_JUDGE_CONFIG = "$ld:ai:usage:judge-config";
|
|
|
894
1299
|
var TRACK_USAGE_CREATE_JUDGE = "$ld:ai:usage:create-judge";
|
|
895
1300
|
var TRACK_USAGE_AGENT_CONFIG = "$ld:ai:usage:agent-config";
|
|
896
1301
|
var TRACK_USAGE_AGENT_CONFIGS = "$ld:ai:usage:agent-configs";
|
|
1302
|
+
var TRACK_USAGE_AGENT_GRAPH = "$ld:ai:usage:agent-graph";
|
|
897
1303
|
var INIT_TRACK_CONTEXT = {
|
|
898
1304
|
kind: "ld_ai",
|
|
899
1305
|
key: "ld-internal-tracking",
|
|
@@ -918,7 +1324,7 @@ var LDAIClientImpl = class {
|
|
|
918
1324
|
_interpolateTemplate(template, variables) {
|
|
919
1325
|
return Mustache2.render(template, variables, void 0, { escape: (item) => item });
|
|
920
1326
|
}
|
|
921
|
-
async _evaluate(key, context, defaultValue, mode, variables) {
|
|
1327
|
+
async _evaluate(key, context, defaultValue, mode, variables, graphKey) {
|
|
922
1328
|
const ldFlagValue = LDAIConfigUtils.toFlagValue(defaultValue, mode);
|
|
923
1329
|
const value = await this._ldClient.variation(key, context, ldFlagValue);
|
|
924
1330
|
const flagMode = value._ldMeta?.mode ?? "completion";
|
|
@@ -928,8 +1334,9 @@ var LDAIClientImpl = class {
|
|
|
928
1334
|
);
|
|
929
1335
|
return LDAIConfigUtils.createDisabledConfig(key, mode);
|
|
930
1336
|
}
|
|
931
|
-
const
|
|
1337
|
+
const trackerFactory = () => new LDAIConfigTrackerImpl(
|
|
932
1338
|
this._ldClient,
|
|
1339
|
+
randomUUID(),
|
|
933
1340
|
key,
|
|
934
1341
|
// eslint-disable-next-line no-underscore-dangle
|
|
935
1342
|
value._ldMeta?.variationKey ?? "",
|
|
@@ -937,9 +1344,10 @@ var LDAIClientImpl = class {
|
|
|
937
1344
|
value._ldMeta?.version ?? 1,
|
|
938
1345
|
value.model?.name ?? "",
|
|
939
1346
|
value.provider?.name ?? "",
|
|
940
|
-
context
|
|
1347
|
+
context,
|
|
1348
|
+
graphKey
|
|
941
1349
|
);
|
|
942
|
-
const config = LDAIConfigUtils.fromFlagValue(key, value,
|
|
1350
|
+
const config = LDAIConfigUtils.fromFlagValue(key, value, trackerFactory);
|
|
943
1351
|
return this._applyInterpolation(config, context, variables);
|
|
944
1352
|
}
|
|
945
1353
|
_applyInterpolation(config, context, variables) {
|
|
@@ -1003,16 +1411,13 @@ var LDAIClientImpl = class {
|
|
|
1003
1411
|
this._ldClient.track(TRACK_USAGE_JUDGE_CONFIG, context, key, 1);
|
|
1004
1412
|
return this._judgeConfig(key, context, defaultValue ?? disabledAIConfig, variables);
|
|
1005
1413
|
}
|
|
1414
|
+
async _agentConfig(key, context, defaultValue, variables, graphKey) {
|
|
1415
|
+
const config = await this._evaluate(key, context, defaultValue, "agent", variables, graphKey);
|
|
1416
|
+
return config;
|
|
1417
|
+
}
|
|
1006
1418
|
async agentConfig(key, context, defaultValue, variables) {
|
|
1007
1419
|
this._ldClient.track(TRACK_USAGE_AGENT_CONFIG, context, key, 1);
|
|
1008
|
-
|
|
1009
|
-
key,
|
|
1010
|
-
context,
|
|
1011
|
-
defaultValue ?? disabledAIConfig,
|
|
1012
|
-
"agent",
|
|
1013
|
-
variables
|
|
1014
|
-
);
|
|
1015
|
-
return config;
|
|
1420
|
+
return this._agentConfig(key, context, defaultValue ?? disabledAIConfig, variables);
|
|
1016
1421
|
}
|
|
1017
1422
|
/**
|
|
1018
1423
|
* @deprecated Use `agentConfig` instead. This method will be removed in a future version.
|
|
@@ -1030,11 +1435,10 @@ var LDAIClientImpl = class {
|
|
|
1030
1435
|
const agents = {};
|
|
1031
1436
|
await Promise.all(
|
|
1032
1437
|
agentConfigs.map(async (config) => {
|
|
1033
|
-
const agent = await this.
|
|
1438
|
+
const agent = await this._agentConfig(
|
|
1034
1439
|
config.key,
|
|
1035
1440
|
context,
|
|
1036
1441
|
config.defaultValue ?? disabledAIConfig,
|
|
1037
|
-
"agent",
|
|
1038
1442
|
config.variables
|
|
1039
1443
|
);
|
|
1040
1444
|
agents[config.key] = agent;
|
|
@@ -1056,7 +1460,7 @@ var LDAIClientImpl = class {
|
|
|
1056
1460
|
defaultValue ?? disabledAIConfig,
|
|
1057
1461
|
variables
|
|
1058
1462
|
);
|
|
1059
|
-
if (!config.enabled
|
|
1463
|
+
if (!config.enabled) {
|
|
1060
1464
|
this._logger?.info(`Chat configuration is disabled: ${key}`);
|
|
1061
1465
|
return void 0;
|
|
1062
1466
|
}
|
|
@@ -1070,7 +1474,7 @@ var LDAIClientImpl = class {
|
|
|
1070
1474
|
variables,
|
|
1071
1475
|
defaultAiProvider
|
|
1072
1476
|
);
|
|
1073
|
-
return new TrackedChat(config,
|
|
1477
|
+
return new TrackedChat(config, provider, judges, this._logger);
|
|
1074
1478
|
}
|
|
1075
1479
|
async createJudge(key, context, defaultValue, variables, defaultAiProvider) {
|
|
1076
1480
|
this._ldClient.track(TRACK_USAGE_CREATE_JUDGE, context, key, 1);
|
|
@@ -1096,7 +1500,7 @@ var LDAIClientImpl = class {
|
|
|
1096
1500
|
defaultValue ?? disabledAIConfig,
|
|
1097
1501
|
extendedVariables
|
|
1098
1502
|
);
|
|
1099
|
-
if (!judgeConfig.enabled
|
|
1503
|
+
if (!judgeConfig.enabled) {
|
|
1100
1504
|
this._logger?.info(`Judge configuration is disabled: ${key}`);
|
|
1101
1505
|
return void 0;
|
|
1102
1506
|
}
|
|
@@ -1104,7 +1508,7 @@ var LDAIClientImpl = class {
|
|
|
1104
1508
|
if (!provider) {
|
|
1105
1509
|
return void 0;
|
|
1106
1510
|
}
|
|
1107
|
-
return new Judge(judgeConfig,
|
|
1511
|
+
return new Judge(judgeConfig, provider, this._logger);
|
|
1108
1512
|
} catch (error) {
|
|
1109
1513
|
this._logger?.error(`Failed to initialize judge ${key}:`, error);
|
|
1110
1514
|
return void 0;
|
|
@@ -1116,6 +1520,81 @@ var LDAIClientImpl = class {
|
|
|
1116
1520
|
async initChat(key, context, defaultValue, variables, defaultAiProvider) {
|
|
1117
1521
|
return this.createChat(key, context, defaultValue, variables, defaultAiProvider);
|
|
1118
1522
|
}
|
|
1523
|
+
createTracker(token, context) {
|
|
1524
|
+
return LDAIConfigTrackerImpl.fromResumptionToken(token, this._ldClient, context);
|
|
1525
|
+
}
|
|
1526
|
+
async agentGraph(graphKey, context, variables) {
|
|
1527
|
+
this._ldClient.track(TRACK_USAGE_AGENT_GRAPH, context, graphKey, 1);
|
|
1528
|
+
const defaultGraphValue = { root: "" };
|
|
1529
|
+
const graphFlagValue = await this._ldClient.variation(
|
|
1530
|
+
graphKey,
|
|
1531
|
+
context,
|
|
1532
|
+
defaultGraphValue
|
|
1533
|
+
);
|
|
1534
|
+
const variationKey = graphFlagValue._ldMeta?.variationKey;
|
|
1535
|
+
const version = graphFlagValue._ldMeta?.version ?? 1;
|
|
1536
|
+
const ldClient = this._ldClient;
|
|
1537
|
+
const trackerFactory = () => new LDGraphTrackerImpl(ldClient, randomUUID(), graphKey, variationKey, version, context);
|
|
1538
|
+
const disabled = new AgentGraphDefinition(graphFlagValue, {}, false, trackerFactory);
|
|
1539
|
+
if (graphFlagValue._ldMeta?.enabled === false) {
|
|
1540
|
+
this._logger?.debug(`agentGraph: graph "${graphKey}" is disabled.`);
|
|
1541
|
+
return disabled;
|
|
1542
|
+
}
|
|
1543
|
+
if (!graphFlagValue.root) {
|
|
1544
|
+
this._logger?.debug(`agentGraph: graph "${graphKey}" is not fetchable or has no root node.`);
|
|
1545
|
+
return disabled;
|
|
1546
|
+
}
|
|
1547
|
+
const allKeys = AgentGraphDefinition.collectAllKeys(graphFlagValue);
|
|
1548
|
+
const reachableKeys = this._collectReachableKeys(graphFlagValue);
|
|
1549
|
+
const unreachableKey = [...allKeys].find((key) => !reachableKeys.has(key));
|
|
1550
|
+
if (unreachableKey) {
|
|
1551
|
+
this._logger?.debug(
|
|
1552
|
+
`agentGraph: graph "${graphKey}" has unconnected node "${unreachableKey}" that is not reachable from the root.`
|
|
1553
|
+
);
|
|
1554
|
+
return disabled;
|
|
1555
|
+
}
|
|
1556
|
+
const agentConfigs = {};
|
|
1557
|
+
const fetchResults = await Promise.all(
|
|
1558
|
+
[...allKeys].map(async (key) => {
|
|
1559
|
+
const config = await this._agentConfig(key, context, disabledAIConfig, variables, graphKey);
|
|
1560
|
+
return { key, config };
|
|
1561
|
+
})
|
|
1562
|
+
);
|
|
1563
|
+
const disabledResult = fetchResults.find(({ config }) => !config.enabled);
|
|
1564
|
+
if (disabledResult) {
|
|
1565
|
+
this._logger?.debug(
|
|
1566
|
+
`agentGraph: agent config "${disabledResult.key}" in graph "${graphKey}" is not enabled or could not be fetched.`
|
|
1567
|
+
);
|
|
1568
|
+
return disabled;
|
|
1569
|
+
}
|
|
1570
|
+
fetchResults.forEach(({ key, config }) => {
|
|
1571
|
+
agentConfigs[key] = config;
|
|
1572
|
+
});
|
|
1573
|
+
const nodes = AgentGraphDefinition.buildNodes(graphFlagValue, agentConfigs);
|
|
1574
|
+
return new AgentGraphDefinition(graphFlagValue, nodes, true, trackerFactory);
|
|
1575
|
+
}
|
|
1576
|
+
createGraphTracker(token, context) {
|
|
1577
|
+
return LDGraphTrackerImpl.fromResumptionToken(token, this._ldClient, context);
|
|
1578
|
+
}
|
|
1579
|
+
/**
|
|
1580
|
+
* Returns the set of all node keys reachable from the root via BFS.
|
|
1581
|
+
*/
|
|
1582
|
+
_collectReachableKeys(graph) {
|
|
1583
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1584
|
+
const queue = [graph.root];
|
|
1585
|
+
visited.add(graph.root);
|
|
1586
|
+
while (queue.length > 0) {
|
|
1587
|
+
const key = queue.shift();
|
|
1588
|
+
const edges = graph.edges?.[key] ?? [];
|
|
1589
|
+
edges.forEach((edge) => {
|
|
1590
|
+
if (!visited.has(edge.key)) {
|
|
1591
|
+
visited.add(edge.key);
|
|
1592
|
+
queue.push(edge.key);
|
|
1593
|
+
}
|
|
1594
|
+
});
|
|
1595
|
+
}
|
|
1596
|
+
return visited;
|
|
1597
|
+
}
|
|
1119
1598
|
};
|
|
1120
1599
|
|
|
1121
1600
|
// src/index.ts
|
|
@@ -1125,8 +1604,11 @@ function initAi(ldClient) {
|
|
|
1125
1604
|
export {
|
|
1126
1605
|
AIProvider,
|
|
1127
1606
|
AIProviderFactory,
|
|
1607
|
+
AgentGraphDefinition,
|
|
1608
|
+
AgentGraphNode,
|
|
1128
1609
|
Judge,
|
|
1129
1610
|
LDFeedbackKind,
|
|
1611
|
+
LDGraphTrackerImpl,
|
|
1130
1612
|
SUPPORTED_AI_PROVIDERS,
|
|
1131
1613
|
TrackedChat,
|
|
1132
1614
|
createBedrockTokenUsage,
|