@terreno/ai 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +3 -3
- package/biome.jsonc +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +42 -8
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +10 -4
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -28
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +30 -30
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/commit.ts +24 -4
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +6 -8
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +29 -4
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -20
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +25 -28
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +497 -17
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +295 -3
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +39 -2
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +8 -8
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +63 -2
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +156 -115
- package/src/service/aiService.test.ts +22 -1
- package/src/service/aiService.ts +4 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +42 -33
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +0 -54
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +11 -4
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/tsconfig.typecheck.json +18 -0
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -60
- package/src/langfuseApp.test.ts +0 -135
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -256
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -68
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -79
- package/src/langfuseVercelAi.ts +0 -72
|
@@ -2,7 +2,7 @@ import {APIError} from "@terreno/api";
|
|
|
2
2
|
import {DateTime} from "luxon";
|
|
3
3
|
import type mongoose from "mongoose";
|
|
4
4
|
|
|
5
|
-
import type {ObsPromptVariable} from "../../types/observability";
|
|
5
|
+
import type {ObsPromptRef, ObsPromptVariable} from "../../types/observability";
|
|
6
6
|
import {compileTemplate} from "../compileTemplate";
|
|
7
7
|
import type {ModelPrice, PromptRegistry, PromptVersionRef} from "../types";
|
|
8
8
|
import {registerObsExperiment} from "./models/obsExperiment";
|
|
@@ -10,6 +10,12 @@ import {registerObsPrompt} from "./models/obsPrompt";
|
|
|
10
10
|
import {registerObsPromptLabel} from "./models/obsPromptLabel";
|
|
11
11
|
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
12
12
|
import {registerObsTrace} from "./models/obsTrace";
|
|
13
|
+
import {
|
|
14
|
+
type PromptReferenceLookup,
|
|
15
|
+
promptIncludeKey,
|
|
16
|
+
type ResolvedPromptText,
|
|
17
|
+
resolvePromptReferences,
|
|
18
|
+
} from "./promptReferences";
|
|
13
19
|
|
|
14
20
|
const PROMPT_RELATIONSHIP_LIMIT = 20;
|
|
15
21
|
|
|
@@ -45,6 +51,20 @@ export interface PromptRelationships {
|
|
|
45
51
|
};
|
|
46
52
|
}
|
|
47
53
|
|
|
54
|
+
/** Another prompt whose saved system or template contains `{{> name}}`. */
|
|
55
|
+
export interface PromptUsedBy {
|
|
56
|
+
folder: string;
|
|
57
|
+
name: string;
|
|
58
|
+
versions: number[];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Bounded used-by list, same cap as trace and experiment relationships. */
|
|
62
|
+
export interface PromptUsedByPage {
|
|
63
|
+
items: PromptUsedBy[];
|
|
64
|
+
limit: number;
|
|
65
|
+
total: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
48
68
|
export interface PromptVersionFields {
|
|
49
69
|
config?: Record<string, unknown>;
|
|
50
70
|
inputSchema?: Record<string, unknown>;
|
|
@@ -80,11 +100,36 @@ export interface MoveLabelResult {
|
|
|
80
100
|
version: number;
|
|
81
101
|
}
|
|
82
102
|
|
|
103
|
+
/** A label of the prompt being saved, as it will read after the save. */
|
|
104
|
+
interface PendingLabel {
|
|
105
|
+
body: string;
|
|
106
|
+
label: string;
|
|
107
|
+
version: number;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
interface ResolvedVersionText {
|
|
111
|
+
includes: ObsPromptRef[];
|
|
112
|
+
system?: string;
|
|
113
|
+
template?: string;
|
|
114
|
+
}
|
|
115
|
+
|
|
83
116
|
export interface CompiledMessage {
|
|
84
117
|
content: string;
|
|
85
118
|
role: "system" | "user";
|
|
86
119
|
}
|
|
87
120
|
|
|
121
|
+
/** A stored version with prompt references in `system` and `template` resolved. */
|
|
122
|
+
export interface ResolvedPromptVersion {
|
|
123
|
+
includes: ObsPromptRef[];
|
|
124
|
+
inputSchema?: Record<string, unknown>;
|
|
125
|
+
outputSchema?: Record<string, unknown>;
|
|
126
|
+
system?: string;
|
|
127
|
+
template?: string;
|
|
128
|
+
type: "chat" | "text";
|
|
129
|
+
variables: ObsPromptVariable[];
|
|
130
|
+
version: number;
|
|
131
|
+
}
|
|
132
|
+
|
|
88
133
|
export interface PlaygroundRunResult {
|
|
89
134
|
compiledMessages: CompiledMessage[];
|
|
90
135
|
costUsd?: number;
|
|
@@ -113,6 +158,11 @@ const versionBody = (system?: string, template?: string): string => {
|
|
|
113
158
|
return template ?? "";
|
|
114
159
|
};
|
|
115
160
|
|
|
161
|
+
/** Matches `{{> name}}`, `{{> name label=…}}`, and `{{> name version=n}}`. */
|
|
162
|
+
const promptReferencePattern = (name: string): RegExp => {
|
|
163
|
+
return new RegExp(`\\{\\{>\\s*${escapeRegex(name)}(\\s|\\}\\})`);
|
|
164
|
+
};
|
|
165
|
+
|
|
116
166
|
const escapeRegex = (value: string): string => {
|
|
117
167
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
118
168
|
};
|
|
@@ -146,6 +196,7 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
146
196
|
if (existing) {
|
|
147
197
|
throw new APIError({status: 409, title: `Prompt "${input.name}" already exists`});
|
|
148
198
|
}
|
|
199
|
+
await this.validateReferences(input.name, input);
|
|
149
200
|
try {
|
|
150
201
|
const prompt = await ObsPrompt.create({
|
|
151
202
|
...(input.description !== undefined ? {description: input.description} : {}),
|
|
@@ -174,6 +225,13 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
174
225
|
.sort({version: -1})
|
|
175
226
|
.limit(1);
|
|
176
227
|
const nextVersion = (latestDocs[0]?.version ?? 0) + 1;
|
|
228
|
+
await this.validateReferences(prompt.name, fields);
|
|
229
|
+
// The new version takes the `latest` label, so prompts that include `latest` must still resolve.
|
|
230
|
+
await this.validateDependents(prompt.name, {
|
|
231
|
+
body: versionBody(fields.system, fields.template),
|
|
232
|
+
label: "latest",
|
|
233
|
+
version: nextVersion,
|
|
234
|
+
});
|
|
177
235
|
const created = await this.insertVersion(prompt._id, nextVersion, fields);
|
|
178
236
|
await this.upsertLabel(prompt._id, "latest", created._id);
|
|
179
237
|
return {name: prompt.name, version: nextVersion};
|
|
@@ -198,6 +256,13 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
198
256
|
title: `Unknown version ${params.version} for prompt "${name}"`,
|
|
199
257
|
});
|
|
200
258
|
}
|
|
259
|
+
await this.validateReferences(prompt.name, versionDoc);
|
|
260
|
+
// Other prompts may include this label, so moving it can nest them too deeply.
|
|
261
|
+
await this.validateDependents(prompt.name, {
|
|
262
|
+
body: versionBody(versionDoc.system, versionDoc.template),
|
|
263
|
+
label: params.label,
|
|
264
|
+
version: versionDoc.version,
|
|
265
|
+
});
|
|
201
266
|
const outgoingVersion = await this.upsertLabel(prompt._id, params.label, versionDoc._id);
|
|
202
267
|
return {label: params.label, outgoingVersion, version: params.version};
|
|
203
268
|
}
|
|
@@ -219,8 +284,13 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
219
284
|
if (!version) {
|
|
220
285
|
return undefined;
|
|
221
286
|
}
|
|
287
|
+
const resolved = await this.resolveText(
|
|
288
|
+
prompt.name,
|
|
289
|
+
versionBody(version.system, version.template)
|
|
290
|
+
);
|
|
222
291
|
return {
|
|
223
|
-
body:
|
|
292
|
+
body: resolved.text,
|
|
293
|
+
...(resolved.includes.length > 0 ? {includes: resolved.includes} : {}),
|
|
224
294
|
label,
|
|
225
295
|
name: prompt.name,
|
|
226
296
|
sensitive: version.sensitive,
|
|
@@ -238,9 +308,13 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
238
308
|
name: string;
|
|
239
309
|
relationships: PromptRelationships;
|
|
240
310
|
tags: string[];
|
|
311
|
+
usedBy: PromptUsedByPage;
|
|
241
312
|
versions: Array<{
|
|
242
313
|
config?: Record<string, unknown>;
|
|
243
314
|
created: string;
|
|
315
|
+
inputSchema?: Record<string, unknown>;
|
|
316
|
+
outputFieldNotes?: Record<string, string>;
|
|
317
|
+
outputSchema?: Record<string, unknown>;
|
|
244
318
|
sensitive: boolean;
|
|
245
319
|
system?: string;
|
|
246
320
|
template?: string;
|
|
@@ -264,10 +338,14 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
264
338
|
name: prompt.name,
|
|
265
339
|
relationships: await this.loadRelationships(prompt.name, options?.promptVersion),
|
|
266
340
|
tags: prompt.tags,
|
|
341
|
+
usedBy: await this.loadUsedBy(prompt.name),
|
|
267
342
|
versions: versions.map((row) => {
|
|
268
343
|
return {
|
|
269
344
|
config: row.config,
|
|
270
345
|
created: DateTime.fromJSDate(row.created).toUTC().toISO() ?? "",
|
|
346
|
+
...(row.inputSchema ? {inputSchema: row.inputSchema} : {}),
|
|
347
|
+
...(row.outputFieldNotes ? {outputFieldNotes: row.outputFieldNotes} : {}),
|
|
348
|
+
...(row.outputSchema ? {outputSchema: row.outputSchema} : {}),
|
|
271
349
|
sensitive: row.sensitive,
|
|
272
350
|
system: row.system,
|
|
273
351
|
template: row.template,
|
|
@@ -361,9 +439,10 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
361
439
|
}): Promise<PlaygroundRunResult> {
|
|
362
440
|
const prompt = await this.requirePrompt(params.name);
|
|
363
441
|
const versionDoc = await this.resolvePlaygroundVersion(prompt._id, params.version);
|
|
442
|
+
const resolved = await this.resolveVersionText(prompt.name, versionDoc);
|
|
364
443
|
const compiledMessages = this.compile({
|
|
365
|
-
system:
|
|
366
|
-
template:
|
|
444
|
+
system: resolved.system,
|
|
445
|
+
template: resolved.template,
|
|
367
446
|
userPrompt: params.userPrompt,
|
|
368
447
|
variables: params.variables,
|
|
369
448
|
});
|
|
@@ -503,18 +582,7 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
503
582
|
async getVersionByLabel(
|
|
504
583
|
name: string,
|
|
505
584
|
label = "production"
|
|
506
|
-
): Promise<
|
|
507
|
-
| {
|
|
508
|
-
inputSchema?: Record<string, unknown>;
|
|
509
|
-
outputSchema?: Record<string, unknown>;
|
|
510
|
-
system?: string;
|
|
511
|
-
template?: string;
|
|
512
|
-
type: "chat" | "text";
|
|
513
|
-
variables: ObsPromptVariable[];
|
|
514
|
-
version: number;
|
|
515
|
-
}
|
|
516
|
-
| undefined
|
|
517
|
-
> {
|
|
585
|
+
): Promise<ResolvedPromptVersion | undefined> {
|
|
518
586
|
const prompt = await registerObsPrompt().findOneOrNone({name});
|
|
519
587
|
if (!prompt) {
|
|
520
588
|
return undefined;
|
|
@@ -530,11 +598,13 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
530
598
|
if (!version) {
|
|
531
599
|
return undefined;
|
|
532
600
|
}
|
|
601
|
+
const resolved = await this.resolveVersionText(prompt.name, version);
|
|
533
602
|
return {
|
|
603
|
+
includes: resolved.includes,
|
|
534
604
|
inputSchema: version.inputSchema,
|
|
535
605
|
outputSchema: version.outputSchema,
|
|
536
|
-
system:
|
|
537
|
-
template:
|
|
606
|
+
system: resolved.system,
|
|
607
|
+
template: resolved.template,
|
|
538
608
|
type: version.type,
|
|
539
609
|
variables: version.variables,
|
|
540
610
|
version: version.version,
|
|
@@ -544,18 +614,7 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
544
614
|
async getVersionByNumber(
|
|
545
615
|
name: string,
|
|
546
616
|
versionNumber: number
|
|
547
|
-
): Promise<
|
|
548
|
-
| {
|
|
549
|
-
inputSchema?: Record<string, unknown>;
|
|
550
|
-
outputSchema?: Record<string, unknown>;
|
|
551
|
-
system?: string;
|
|
552
|
-
template?: string;
|
|
553
|
-
type: "chat" | "text";
|
|
554
|
-
variables: ObsPromptVariable[];
|
|
555
|
-
version: number;
|
|
556
|
-
}
|
|
557
|
-
| undefined
|
|
558
|
-
> {
|
|
617
|
+
): Promise<ResolvedPromptVersion | undefined> {
|
|
559
618
|
const prompt = await registerObsPrompt().findOneOrNone({name});
|
|
560
619
|
if (!prompt) {
|
|
561
620
|
return undefined;
|
|
@@ -567,17 +626,60 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
567
626
|
if (!version) {
|
|
568
627
|
return undefined;
|
|
569
628
|
}
|
|
629
|
+
const resolved = await this.resolveVersionText(prompt.name, version);
|
|
570
630
|
return {
|
|
631
|
+
includes: resolved.includes,
|
|
571
632
|
inputSchema: version.inputSchema,
|
|
572
633
|
outputSchema: version.outputSchema,
|
|
573
|
-
system:
|
|
574
|
-
template:
|
|
634
|
+
system: resolved.system,
|
|
635
|
+
template: resolved.template,
|
|
575
636
|
type: version.type,
|
|
576
637
|
variables: version.variables,
|
|
577
638
|
version: version.version,
|
|
578
639
|
};
|
|
579
640
|
}
|
|
580
641
|
|
|
642
|
+
/**
|
|
643
|
+
* Prompts that mention this name in a saved `{{> name}}` tag, with the versions that do.
|
|
644
|
+
* The current prompt is omitted so a self-tag does not list itself.
|
|
645
|
+
*/
|
|
646
|
+
private async loadUsedBy(name: string): Promise<PromptUsedByPage> {
|
|
647
|
+
const versions = await registerObsPromptVersion().find({
|
|
648
|
+
$or: [{system: promptReferencePattern(name)}, {template: promptReferencePattern(name)}],
|
|
649
|
+
});
|
|
650
|
+
const versionsByPromptId = new Map<string, number[]>();
|
|
651
|
+
for (const version of versions) {
|
|
652
|
+
const promptId = String(version.promptId);
|
|
653
|
+
const current = versionsByPromptId.get(promptId) ?? [];
|
|
654
|
+
current.push(version.version);
|
|
655
|
+
versionsByPromptId.set(promptId, current);
|
|
656
|
+
}
|
|
657
|
+
const empty = {items: [], limit: PROMPT_RELATIONSHIP_LIMIT, total: 0};
|
|
658
|
+
if (versionsByPromptId.size === 0) {
|
|
659
|
+
return empty;
|
|
660
|
+
}
|
|
661
|
+
const prompts = await registerObsPrompt().find({
|
|
662
|
+
_id: {$in: [...versionsByPromptId.keys()]},
|
|
663
|
+
});
|
|
664
|
+
const items = prompts
|
|
665
|
+
.filter((prompt) => prompt.name !== name)
|
|
666
|
+
.map((prompt) => {
|
|
667
|
+
return {
|
|
668
|
+
folder: prompt.folder,
|
|
669
|
+
name: prompt.name,
|
|
670
|
+
versions: [...(versionsByPromptId.get(String(prompt._id)) ?? [])].sort(
|
|
671
|
+
(left, right) => left - right
|
|
672
|
+
),
|
|
673
|
+
};
|
|
674
|
+
})
|
|
675
|
+
.sort((left, right) => left.name.localeCompare(right.name));
|
|
676
|
+
return {
|
|
677
|
+
items: items.slice(0, PROMPT_RELATIONSHIP_LIMIT),
|
|
678
|
+
limit: PROMPT_RELATIONSHIP_LIMIT,
|
|
679
|
+
total: items.length,
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
|
|
581
683
|
private async loadRelationships(
|
|
582
684
|
promptName: string,
|
|
583
685
|
promptVersion?: number
|
|
@@ -640,6 +742,115 @@ export class LocalPromptStore implements PromptRegistry {
|
|
|
640
742
|
};
|
|
641
743
|
}
|
|
642
744
|
|
|
745
|
+
/**
|
|
746
|
+
* Loads a referenced prompt by pinned version or by label. `pending` stands in for a label
|
|
747
|
+
* of the prompt being saved, as it will read once the save lands, when checking the prompts
|
|
748
|
+
* that include it.
|
|
749
|
+
*/
|
|
750
|
+
private referenceLookup(pending?: PendingLabel & {name: string}): PromptReferenceLookup {
|
|
751
|
+
return async (reference) => {
|
|
752
|
+
if (pending && reference.name === pending.name && reference.label === pending.label) {
|
|
753
|
+
return pending;
|
|
754
|
+
}
|
|
755
|
+
const prompt = await registerObsPrompt().findOneOrNone({name: reference.name});
|
|
756
|
+
if (!prompt) {
|
|
757
|
+
return undefined;
|
|
758
|
+
}
|
|
759
|
+
const ObsPromptVersion = registerObsPromptVersion();
|
|
760
|
+
if (reference.version !== undefined) {
|
|
761
|
+
const pinned = await ObsPromptVersion.findOneOrNone({
|
|
762
|
+
promptId: prompt._id,
|
|
763
|
+
version: reference.version,
|
|
764
|
+
});
|
|
765
|
+
return pinned
|
|
766
|
+
? {body: versionBody(pinned.system, pinned.template), version: pinned.version}
|
|
767
|
+
: undefined;
|
|
768
|
+
}
|
|
769
|
+
const labelDoc = await registerObsPromptLabel().findOneOrNone({
|
|
770
|
+
label: reference.label,
|
|
771
|
+
promptId: prompt._id,
|
|
772
|
+
});
|
|
773
|
+
const labelled = labelDoc
|
|
774
|
+
? await ObsPromptVersion.findOneOrNone({_id: labelDoc.versionId})
|
|
775
|
+
: undefined;
|
|
776
|
+
return labelled
|
|
777
|
+
? {
|
|
778
|
+
body: versionBody(labelled.system, labelled.template),
|
|
779
|
+
label: reference.label,
|
|
780
|
+
version: labelled.version,
|
|
781
|
+
}
|
|
782
|
+
: undefined;
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
private async resolveText(name: string, text: string): Promise<ResolvedPromptText> {
|
|
787
|
+
return resolvePromptReferences({ancestors: [name], lookup: this.referenceLookup(), text});
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
private async resolveVersionText(
|
|
791
|
+
name: string,
|
|
792
|
+
version: {system?: string; template?: string}
|
|
793
|
+
): Promise<ResolvedVersionText> {
|
|
794
|
+
const system =
|
|
795
|
+
version.system === undefined ? undefined : await this.resolveText(name, version.system);
|
|
796
|
+
const template =
|
|
797
|
+
version.template === undefined ? undefined : await this.resolveText(name, version.template);
|
|
798
|
+
const includes = new Map<string, ObsPromptRef>();
|
|
799
|
+
for (const ref of [...(system?.includes ?? []), ...(template?.includes ?? [])]) {
|
|
800
|
+
includes.set(promptIncludeKey(ref), ref);
|
|
801
|
+
}
|
|
802
|
+
return {includes: [...includes.values()], system: system?.text, template: template?.text};
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* Rejects a malformed, unknown, cyclic, or too-deep reference before a version or label is
|
|
807
|
+
* saved, so every saved reference resolves at call time. A label must already exist on the
|
|
808
|
+
* referenced prompt: promote the shared prompt before referencing it by label.
|
|
809
|
+
*/
|
|
810
|
+
private async validateReferences(
|
|
811
|
+
name: string,
|
|
812
|
+
fields: {system?: string; template?: string},
|
|
813
|
+
lookup: PromptReferenceLookup = this.referenceLookup()
|
|
814
|
+
): Promise<void> {
|
|
815
|
+
for (const text of [fields.system, fields.template]) {
|
|
816
|
+
if (text) {
|
|
817
|
+
await resolvePromptReferences({ancestors: [name], lookup, text});
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
/**
|
|
823
|
+
* Re-resolves every version of each prompt that includes `name`, directly or through another
|
|
824
|
+
* prompt, as it will read once `pending` lands. A new body for a shared prompt can push a prompt
|
|
825
|
+
* that includes it past the nesting limit, and experiments still run unlabelled versions.
|
|
826
|
+
*/
|
|
827
|
+
private async validateDependents(name: string, pending: PendingLabel): Promise<void> {
|
|
828
|
+
const lookup = this.referenceLookup({...pending, name});
|
|
829
|
+
const ObsPrompt = registerObsPrompt();
|
|
830
|
+
const ObsPromptVersion = registerObsPromptVersion();
|
|
831
|
+
const seen = new Set([name]);
|
|
832
|
+
const queue = [name];
|
|
833
|
+
while (queue.length > 0) {
|
|
834
|
+
const target = queue.shift() as string;
|
|
835
|
+
const includesTarget = promptReferencePattern(target);
|
|
836
|
+
const promptIds = await ObsPromptVersion.distinct("promptId", {
|
|
837
|
+
$or: [{system: includesTarget}, {template: includesTarget}],
|
|
838
|
+
});
|
|
839
|
+
const dependents = await ObsPrompt.find({_id: {$in: promptIds}});
|
|
840
|
+
for (const dependent of dependents) {
|
|
841
|
+
if (seen.has(dependent.name)) {
|
|
842
|
+
continue;
|
|
843
|
+
}
|
|
844
|
+
seen.add(dependent.name);
|
|
845
|
+
queue.push(dependent.name);
|
|
846
|
+
const versions = await ObsPromptVersion.find({promptId: dependent._id});
|
|
847
|
+
for (const version of versions) {
|
|
848
|
+
await this.validateReferences(dependent.name, version, lookup);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
|
|
643
854
|
private async requirePrompt(name: string): Promise<{
|
|
644
855
|
_id: mongoose.Types.ObjectId;
|
|
645
856
|
description?: string;
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import {DateTime} from "luxon";
|
|
3
|
+
|
|
4
|
+
import type {EvaluatorAiClient} from "../evaluate";
|
|
5
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
6
|
+
import {LocalEvaluatorStore} from "./evaluatorStore";
|
|
7
|
+
import {createLocalObservabilityPlugin} from "./localPlugin";
|
|
8
|
+
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
9
|
+
import {registerObsJudgeRun} from "./models/obsJudgeRun";
|
|
10
|
+
import {registerObsPrompt} from "./models/obsPrompt";
|
|
11
|
+
import {registerObsPromptLabel} from "./models/obsPromptLabel";
|
|
12
|
+
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
13
|
+
import {registerObsReviewBatch} from "./models/obsReviewBatch";
|
|
14
|
+
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
15
|
+
import {registerObsTrace} from "./models/obsTrace";
|
|
16
|
+
import {LocalPromptStore} from "./promptStore";
|
|
17
|
+
import {LocalReviewBatchStore} from "./reviewBatchStore";
|
|
18
|
+
import {LocalReviewStore} from "./reviewStore";
|
|
19
|
+
import {LocalTraceStore, MemoryScoreSink} from "./traceStore";
|
|
20
|
+
|
|
21
|
+
const QUESTIONS = [
|
|
22
|
+
{dataType: "boolean" as const, key: "faithful", question: "Is it faithful?", required: true},
|
|
23
|
+
{dataType: "boolean" as const, key: "complete", question: "Is it complete?", required: true},
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
const exportTrace = async (params: {
|
|
27
|
+
hoursAgo: number;
|
|
28
|
+
output?: string;
|
|
29
|
+
prompt: string;
|
|
30
|
+
status?: "error" | "ok";
|
|
31
|
+
}): Promise<string> => {
|
|
32
|
+
const startedAt = DateTime.utc().minus({hours: params.hoursAgo}).toISO() ?? "";
|
|
33
|
+
const exported = await new LocalTraceStore().exportTrace({
|
|
34
|
+
id: `t-${params.hoursAgo}`,
|
|
35
|
+
input: `input ${params.hoursAgo}`,
|
|
36
|
+
name: params.prompt,
|
|
37
|
+
output: params.output,
|
|
38
|
+
prompts: [{name: params.prompt, version: 1}],
|
|
39
|
+
sensitive: false,
|
|
40
|
+
spans: [{id: "s", kind: "LLM", name: "llm", startedAt, status: params.status ?? "ok"}],
|
|
41
|
+
startedAt,
|
|
42
|
+
status: params.status ?? "ok",
|
|
43
|
+
});
|
|
44
|
+
return exported.id;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
// Judge that answers "faithful" from the output text and always says "complete".
|
|
48
|
+
const fakeJudge: EvaluatorAiClient & {modelId: string} = {
|
|
49
|
+
generateJsonObject: async <OBJECT>(options: {prompt: string}) => {
|
|
50
|
+
return {complete: true, confidence: 0.9, faithful: !options.prompt.includes("bad")} as OBJECT;
|
|
51
|
+
},
|
|
52
|
+
modelId: "fake-judge",
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
describe("LocalReviewBatchStore", () => {
|
|
56
|
+
let promptStore: LocalPromptStore;
|
|
57
|
+
let store: LocalReviewBatchStore;
|
|
58
|
+
let questionSetId: string;
|
|
59
|
+
let judgeId: string;
|
|
60
|
+
|
|
61
|
+
afterEach(() => {
|
|
62
|
+
resetObservabilityApp();
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
beforeEach(async () => {
|
|
66
|
+
const plugin = createLocalObservabilityPlugin();
|
|
67
|
+
plugin.scoreSink = new MemoryScoreSink();
|
|
68
|
+
new ObservabilityApp({plugins: [plugin]});
|
|
69
|
+
for (const model of [
|
|
70
|
+
registerObsReviewItem(),
|
|
71
|
+
registerObsReviewBatch(),
|
|
72
|
+
registerObsJudgeRun(),
|
|
73
|
+
registerObsEvaluator(),
|
|
74
|
+
registerObsTrace(),
|
|
75
|
+
registerObsPrompt(),
|
|
76
|
+
registerObsPromptVersion(),
|
|
77
|
+
registerObsPromptLabel(),
|
|
78
|
+
]) {
|
|
79
|
+
await (model as unknown as {deleteMany: (filter: object) => Promise<unknown>}).deleteMany({});
|
|
80
|
+
}
|
|
81
|
+
promptStore = new LocalPromptStore();
|
|
82
|
+
store = new LocalReviewBatchStore(promptStore);
|
|
83
|
+
await promptStore.create({
|
|
84
|
+
folder: "evaluators",
|
|
85
|
+
name: "judge-questions",
|
|
86
|
+
outputSchema: {
|
|
87
|
+
properties: {
|
|
88
|
+
complete: {type: "boolean"},
|
|
89
|
+
confidence: {type: "number"},
|
|
90
|
+
faithful: {type: "boolean"},
|
|
91
|
+
},
|
|
92
|
+
required: ["faithful", "complete"],
|
|
93
|
+
type: "object",
|
|
94
|
+
},
|
|
95
|
+
system: "Answer each question.",
|
|
96
|
+
type: "chat",
|
|
97
|
+
});
|
|
98
|
+
await promptStore.moveLabel("judge-questions", {label: "production", version: 1});
|
|
99
|
+
const evaluators = new LocalEvaluatorStore(promptStore);
|
|
100
|
+
questionSetId = (
|
|
101
|
+
await evaluators.create({
|
|
102
|
+
dimensions: QUESTIONS,
|
|
103
|
+
name: "summary-questions",
|
|
104
|
+
target: "full trace",
|
|
105
|
+
type: "human",
|
|
106
|
+
})
|
|
107
|
+
).id;
|
|
108
|
+
judgeId = (
|
|
109
|
+
await evaluators.create({
|
|
110
|
+
dimensions: QUESTIONS.map(({question: _question, ...row}) => row),
|
|
111
|
+
judgePromptName: "judge-questions",
|
|
112
|
+
name: "summary-judge",
|
|
113
|
+
target: "full trace",
|
|
114
|
+
type: "llm-judge",
|
|
115
|
+
})
|
|
116
|
+
).id;
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("builds a batch from the newest successful traces for a prompt", async () => {
|
|
120
|
+
for (let hoursAgo = 1; hoursAgo <= 12; hoursAgo += 1) {
|
|
121
|
+
await exportTrace({hoursAgo, output: `summary ${hoursAgo}`, prompt: "summarize"});
|
|
122
|
+
}
|
|
123
|
+
await exportTrace({hoursAgo: 0, prompt: "summarize", status: "error"});
|
|
124
|
+
await exportTrace({hoursAgo: 0, output: "other", prompt: "jokes"});
|
|
125
|
+
|
|
126
|
+
const batch = await store.create({evaluatorId: questionSetId, promptName: "summarize"});
|
|
127
|
+
expect(batch.name).toBe("summarize — batch 1");
|
|
128
|
+
expect(batch.counts).toEqual({done: 0, pending: 10, skipped: 0, total: 10});
|
|
129
|
+
expect(batch.items.map((item) => item.output)).toEqual(
|
|
130
|
+
Array.from({length: 10}, (_value, index) => `summary ${index + 1}`)
|
|
131
|
+
);
|
|
132
|
+
expect(batch.dimensions.map((row) => row.question)).toEqual([
|
|
133
|
+
"Is it faithful?",
|
|
134
|
+
"Is it complete?",
|
|
135
|
+
]);
|
|
136
|
+
|
|
137
|
+
// A second batch skips traces already batched for this question set.
|
|
138
|
+
const next = await store.create({evaluatorId: questionSetId, promptName: "summarize"});
|
|
139
|
+
expect(next.items.map((item) => item.output)).toEqual(["summary 11", "summary 12"]);
|
|
140
|
+
await expect(
|
|
141
|
+
store.create({evaluatorId: questionSetId, promptName: "summarize"})
|
|
142
|
+
).rejects.toThrow(/No recent "summarize" traces/);
|
|
143
|
+
|
|
144
|
+
const listed = await store.list();
|
|
145
|
+
expect(listed.map((row) => [row.name, row.counts.total, row.judgeCount])).toEqual([
|
|
146
|
+
["summarize — batch 2", 2, 0],
|
|
147
|
+
["summarize — batch 1", 10, 0],
|
|
148
|
+
]);
|
|
149
|
+
|
|
150
|
+
const pending = await new LocalReviewStore().list("pending", {batchId: batch.id});
|
|
151
|
+
expect(pending.data).toHaveLength(10);
|
|
152
|
+
expect(pending.counts.pending).toBe(10);
|
|
153
|
+
const detail = await new LocalReviewStore().getDetail(pending.data[0]?.id ?? "");
|
|
154
|
+
expect(detail.batchName).toBe("summarize — batch 1");
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("leaves no batch behind when traces are unknown or enqueue fails", async () => {
|
|
158
|
+
const trace = await exportTrace({hoursAgo: 1, output: "x", prompt: "summarize"});
|
|
159
|
+
await expect(
|
|
160
|
+
store.create({evaluatorId: questionSetId, traceIds: [trace, "64b000000000000000000000"]})
|
|
161
|
+
).rejects.toThrow(/Unknown trace ids/);
|
|
162
|
+
|
|
163
|
+
const failingReviews = new LocalReviewStore();
|
|
164
|
+
failingReviews.enqueue = async (): Promise<never> => {
|
|
165
|
+
throw new Error("enqueue failed");
|
|
166
|
+
};
|
|
167
|
+
const failingStore = new LocalReviewBatchStore(promptStore, failingReviews);
|
|
168
|
+
await expect(failingStore.create({evaluatorId: questionSetId})).rejects.toThrow(
|
|
169
|
+
"enqueue failed"
|
|
170
|
+
);
|
|
171
|
+
expect(await store.list()).toEqual([]);
|
|
172
|
+
|
|
173
|
+
// The trace was not claimed by the failed batch, so grabbing recent traces still finds it.
|
|
174
|
+
const batch = await store.create({evaluatorId: questionSetId});
|
|
175
|
+
expect(batch.items.map((item) => item.traceId)).toEqual([trace]);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it("rejects non-human question sets and out-of-range limits", async () => {
|
|
179
|
+
await expect(store.create({evaluatorId: judgeId})).rejects.toThrow(/human evaluator/);
|
|
180
|
+
await expect(store.create({evaluatorId: questionSetId, limit: 51})).rejects.toThrow(/limit/);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("compares judge answers with human answers question by question", async () => {
|
|
184
|
+
const good = await exportTrace({hoursAgo: 2, output: "good summary", prompt: "summarize"});
|
|
185
|
+
const bad = await exportTrace({hoursAgo: 1, output: "bad summary", prompt: "summarize"});
|
|
186
|
+
const batch = await store.create({
|
|
187
|
+
evaluatorId: questionSetId,
|
|
188
|
+
name: "Hand picked",
|
|
189
|
+
traceIds: [good, bad],
|
|
190
|
+
});
|
|
191
|
+
const reviews = new LocalReviewStore();
|
|
192
|
+
const [goodItem, badItem] = batch.items;
|
|
193
|
+
await reviews.submit({
|
|
194
|
+
id: goodItem?.reviewItemId ?? "",
|
|
195
|
+
scores: {complete: true, faithful: true},
|
|
196
|
+
sinks: [new MemoryScoreSink()],
|
|
197
|
+
});
|
|
198
|
+
await reviews.submit({
|
|
199
|
+
id: badItem?.reviewItemId ?? "",
|
|
200
|
+
scores: {complete: false, faithful: false},
|
|
201
|
+
sinks: [new MemoryScoreSink()],
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
const judged = await store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: judgeId});
|
|
205
|
+
expect(judged.judgeCount).toBe(1);
|
|
206
|
+
const [summary] = judged.judges;
|
|
207
|
+
expect(summary?.evaluatorName).toBe("summary-judge");
|
|
208
|
+
expect(summary?.modelId).toBe("fake-judge");
|
|
209
|
+
expect(summary?.agreement.rate).toBe(0.75);
|
|
210
|
+
expect(summary?.agreement.questions.map((row) => row.rate)).toEqual([1, 0.5]);
|
|
211
|
+
expect(summary?.agreement.disagreements).toEqual([
|
|
212
|
+
{human: false, judge: true, key: "complete", traceId: bad},
|
|
213
|
+
]);
|
|
214
|
+
expect(judged.items[1]?.judgeAnswers[judgeId]?.scores).toEqual({
|
|
215
|
+
complete: true,
|
|
216
|
+
faithful: false,
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
// Re-running keeps only the latest run per judge.
|
|
220
|
+
await store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: judgeId});
|
|
221
|
+
const listed = await store.list();
|
|
222
|
+
expect(listed[0]?.judgeCount).toBe(1);
|
|
223
|
+
expect(listed[0]?.bestJudge).toEqual({evaluatorName: "summary-judge", rate: 0.75});
|
|
224
|
+
expect(listed[0]?.counts.done).toBe(2);
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
it("refuses judges that share no questions with the batch", async () => {
|
|
228
|
+
await promptStore.create({
|
|
229
|
+
folder: "evaluators",
|
|
230
|
+
name: "judge-other",
|
|
231
|
+
outputSchema: {properties: {tone: {type: "string"}}, required: ["tone"], type: "object"},
|
|
232
|
+
type: "chat",
|
|
233
|
+
});
|
|
234
|
+
await promptStore.moveLabel("judge-other", {label: "production", version: 1});
|
|
235
|
+
const other = await new LocalEvaluatorStore(promptStore).create({
|
|
236
|
+
dimensions: [{dataType: "categorical", key: "tone", required: true}],
|
|
237
|
+
judgePromptName: "judge-other",
|
|
238
|
+
name: "tone-judge",
|
|
239
|
+
target: "full trace",
|
|
240
|
+
type: "llm-judge",
|
|
241
|
+
});
|
|
242
|
+
const trace = await exportTrace({hoursAgo: 1, output: "x", prompt: "summarize"});
|
|
243
|
+
const batch = await store.create({evaluatorId: questionSetId, traceIds: [trace]});
|
|
244
|
+
await expect(
|
|
245
|
+
store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: other.id})
|
|
246
|
+
).rejects.toThrow(/answers none of this batch's questions/);
|
|
247
|
+
await expect(
|
|
248
|
+
store.runJudge({ai: fakeJudge, batchId: batch.id, evaluatorId: questionSetId})
|
|
249
|
+
).rejects.toThrow(/Only llm-judge/);
|
|
250
|
+
});
|
|
251
|
+
});
|