@terreno/ai 58.0.0-beta.3 → 58.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -0
- package/README.md +3 -3
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +40 -3
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/package.json +8 -24
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentRecovery.test.ts +0 -3
- package/src/harness/commit.ts +24 -4
- package/src/harness/extensions.test.ts +0 -3
- package/src/harness/subagent.test.ts +27 -3
- package/src/index.test.ts +0 -17
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +0 -8
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.ts +495 -15
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +291 -0
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/experiments.test.ts +37 -0
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/traces.test.ts +61 -0
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/routes.test.ts +101 -76
- package/src/service/aiService.test.ts +21 -0
- package/src/service/aiService.ts +4 -1
- package/src/service/chatTurn.ts +42 -33
- package/src/tests/bunSetup.ts +0 -54
- package/src/types/harness.ts +5 -2
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -66
- package/src/langfuseApp.test.ts +0 -119
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -261
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -72
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -82
- package/src/langfuseVercelAi.ts +0 -72
- package/src/tests/fakeLangfusePrompt.ts +0 -10
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.addObservabilityTraceRoutes = void 0;
|
|
4
4
|
const api_1 = require("@terreno/api");
|
|
5
|
+
const traceStore_1 = require("../local/traceStore");
|
|
5
6
|
const observabilityAppRegistry_1 = require("../observabilityAppRegistry");
|
|
6
7
|
const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
|
|
7
8
|
const BASE_PATH = "/ai/observability";
|
|
@@ -25,11 +26,34 @@ const parsePromptVersion = (value, prompt) => {
|
|
|
25
26
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
26
27
|
throw new api_1.APIError({ status: 400, title: "promptVersion must be a positive integer" });
|
|
27
28
|
}
|
|
28
|
-
if (!prompt) {
|
|
29
|
+
if (!prompt || prompt.length === 0) {
|
|
29
30
|
throw new api_1.APIError({ status: 400, title: "prompt is required when promptVersion is set" });
|
|
30
31
|
}
|
|
31
32
|
return parsed;
|
|
32
33
|
};
|
|
34
|
+
const parsePrompt = (value) => {
|
|
35
|
+
if (typeof value === "string") {
|
|
36
|
+
return value;
|
|
37
|
+
}
|
|
38
|
+
if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
|
|
39
|
+
return value;
|
|
40
|
+
}
|
|
41
|
+
if (value === undefined) {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
throw new api_1.APIError({ status: 400, title: "prompt must be a string or a list of strings" });
|
|
45
|
+
};
|
|
46
|
+
const parseSort = (value) => {
|
|
47
|
+
if (value === undefined) {
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
if (typeof value !== "string") {
|
|
51
|
+
throw new api_1.APIError({ status: 400, title: "Unknown trace sort key" });
|
|
52
|
+
}
|
|
53
|
+
// Validate here so a bad key fails before the store runs any query.
|
|
54
|
+
(0, traceStore_1.parseTraceListSort)(value);
|
|
55
|
+
return value;
|
|
56
|
+
};
|
|
33
57
|
const parsePositiveInt = (value, fallback) => {
|
|
34
58
|
if (value === undefined) {
|
|
35
59
|
return fallback;
|
|
@@ -51,9 +75,16 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
51
75
|
.withQueryParameter("prompt", { type: "string" }, { required: false })
|
|
52
76
|
.withQueryParameter("promptVersion", { type: "number" }, { required: false })
|
|
53
77
|
.withQueryParameter("scope", { type: "string" }, { required: false })
|
|
78
|
+
.withQueryParameter("sort", {
|
|
79
|
+
enum: traceStore_1.TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
|
|
80
|
+
type: "string",
|
|
81
|
+
}, {
|
|
82
|
+
description: `Sort column, \`-\` prefix for descending. Defaults to \`${traceStore_1.DEFAULT_TRACE_LIST_SORT}\`.`,
|
|
83
|
+
required: false,
|
|
84
|
+
})
|
|
54
85
|
.withResponse(200, { data: { type: "array" } })
|
|
55
86
|
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
56
|
-
const prompt =
|
|
87
|
+
const prompt = parsePrompt(req.query.prompt);
|
|
57
88
|
const listed = await options.store.list({
|
|
58
89
|
flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
|
|
59
90
|
from: typeof req.query.from === "string" ? req.query.from : undefined,
|
|
@@ -65,6 +96,7 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
65
96
|
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
66
97
|
sensitive: parseBoolean(req.query.sensitive),
|
|
67
98
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
99
|
+
sort: parseSort(req.query.sort),
|
|
68
100
|
status: req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
|
|
69
101
|
to: typeof req.query.to === "string" ? req.query.to : undefined,
|
|
70
102
|
userId: typeof req.query.userId === "string" ? req.query.userId : undefined,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type AnyTerrenoAccess } from "@terreno/api";
|
|
2
2
|
export type ObservabilityCapability = "datasets" | "experiments" | "prompts" | "reviewQueue" | "scores" | "traces";
|
|
3
|
-
export type ControlPrimary = "
|
|
3
|
+
export type ControlPrimary = "local";
|
|
4
4
|
export interface ObservabilityControlConfig {
|
|
5
5
|
datasets: ControlPrimary;
|
|
6
6
|
experiments: ControlPrimary;
|
|
@@ -39,6 +39,7 @@ export interface TraceRecord {
|
|
|
39
39
|
name: string;
|
|
40
40
|
output?: unknown;
|
|
41
41
|
prompts: {
|
|
42
|
+
includedBy?: string;
|
|
42
43
|
label?: string;
|
|
43
44
|
name: string;
|
|
44
45
|
version: number;
|
|
@@ -68,7 +69,15 @@ export interface ScoreRecord {
|
|
|
68
69
|
value: boolean | number | string;
|
|
69
70
|
}
|
|
70
71
|
export interface PromptVersionRef {
|
|
72
|
+
/** `body` with every prompt reference resolved. */
|
|
71
73
|
body: string;
|
|
74
|
+
/** Prompt versions pulled in by references, recorded on the trace after this prompt. */
|
|
75
|
+
includes?: {
|
|
76
|
+
includedBy?: string;
|
|
77
|
+
label?: string;
|
|
78
|
+
name: string;
|
|
79
|
+
version: number;
|
|
80
|
+
}[];
|
|
72
81
|
label?: string;
|
|
73
82
|
name: string;
|
|
74
83
|
sensitive?: boolean;
|
|
@@ -267,7 +267,10 @@ class AIService {
|
|
|
267
267
|
name: spanName,
|
|
268
268
|
output: params.response,
|
|
269
269
|
prompts: promptRef
|
|
270
|
-
? [
|
|
270
|
+
? [
|
|
271
|
+
{ label: promptRef.label, name: promptRef.name, version: promptRef.version },
|
|
272
|
+
...(promptRef.includes ?? []),
|
|
273
|
+
]
|
|
271
274
|
: [],
|
|
272
275
|
sensitive: params.observability.sensitive,
|
|
273
276
|
sessionId: params.observability.sessionId,
|
package/dist/service/chatTurn.js
CHANGED
|
@@ -6,11 +6,11 @@ const api_1 = require("@terreno/api");
|
|
|
6
6
|
const blocks_1 = require("@terreno/blocks");
|
|
7
7
|
const ai_1 = require("ai");
|
|
8
8
|
const luxon_1 = require("luxon");
|
|
9
|
-
const langfuseVercelAi_1 = require("../langfuseVercelAi");
|
|
10
9
|
const aiRequest_1 = require("../models/aiRequest");
|
|
11
10
|
const fileAttachment_1 = require("../models/fileAttachment");
|
|
12
11
|
const gptHistory_1 = require("../models/gptHistory");
|
|
13
12
|
const project_1 = require("../models/project");
|
|
13
|
+
const observabilityAppRegistry_1 = require("../observability/observabilityAppRegistry");
|
|
14
14
|
const aiService_1 = require("./aiService");
|
|
15
15
|
const askFiles_1 = require("./askFiles");
|
|
16
16
|
const asks_1 = require("./asks");
|
|
@@ -696,8 +696,28 @@ const startTurn = async ({ aiService, askAnswer, body, options, req, userId, })
|
|
|
696
696
|
};
|
|
697
697
|
};
|
|
698
698
|
const lastUserText = (prompts) => prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
|
|
699
|
-
/** The
|
|
700
|
-
const
|
|
699
|
+
/** The route's default registry system prompt, or undefined when it cannot be loaded. */
|
|
700
|
+
const loadRegistrySystemPrompt = async (options) => {
|
|
701
|
+
const registry = (0, observabilityAppRegistry_1.getObservabilityApp)()?.promptRegistry;
|
|
702
|
+
if (!options.systemPromptName || !registry) {
|
|
703
|
+
return undefined;
|
|
704
|
+
}
|
|
705
|
+
try {
|
|
706
|
+
return await registry.get({
|
|
707
|
+
label: options.systemPromptLabel ?? "production",
|
|
708
|
+
name: options.systemPromptName,
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
catch (error) {
|
|
712
|
+
api_1.logger.warn("Registry system prompt skipped", {
|
|
713
|
+
error: error instanceof Error ? error.message : String(error),
|
|
714
|
+
promptName: options.systemPromptName,
|
|
715
|
+
});
|
|
716
|
+
return undefined;
|
|
717
|
+
}
|
|
718
|
+
};
|
|
719
|
+
/** The request's system prompt with project context and the registry prompt prepended. */
|
|
720
|
+
const buildSystemPrompt = async ({ history, projectId, registryPrompt, systemPrompt, userId, }) => {
|
|
701
721
|
const effectiveProjectId = projectId ?? history.projectId;
|
|
702
722
|
let effectiveSystemPrompt = systemPrompt;
|
|
703
723
|
if (effectiveProjectId) {
|
|
@@ -723,22 +743,10 @@ const buildSystemPrompt = async ({ history, options, projectId, systemPrompt, us
|
|
|
723
743
|
// Project loading failure should not block the request
|
|
724
744
|
}
|
|
725
745
|
}
|
|
726
|
-
if (
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
userId: userId?.toString(),
|
|
731
|
-
});
|
|
732
|
-
const langfusePrompt = typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
|
|
733
|
-
if (langfusePrompt) {
|
|
734
|
-
effectiveSystemPrompt = effectiveSystemPrompt
|
|
735
|
-
? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
|
|
736
|
-
: langfusePrompt;
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
catch (err) {
|
|
740
|
-
api_1.logger.debug(`Langfuse system prompt skipped: ${err.message}`);
|
|
741
|
-
}
|
|
746
|
+
if (registryPrompt) {
|
|
747
|
+
effectiveSystemPrompt = effectiveSystemPrompt
|
|
748
|
+
? `${registryPrompt}\n\n${effectiveSystemPrompt}`
|
|
749
|
+
: registryPrompt;
|
|
742
750
|
}
|
|
743
751
|
return effectiveSystemPrompt;
|
|
744
752
|
};
|
|
@@ -1197,20 +1205,29 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
|
|
|
1197
1205
|
const turn = await startTurn({ aiService, askAnswer, body, options, req, userId });
|
|
1198
1206
|
const { answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt } = turn;
|
|
1199
1207
|
let { history } = turn;
|
|
1208
|
+
// An admin-selected promptName replaces the system prompt, so the route default is skipped.
|
|
1209
|
+
const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
|
|
1200
1210
|
let effectiveSystemPrompt = await buildSystemPrompt({
|
|
1201
1211
|
history,
|
|
1202
|
-
options,
|
|
1203
1212
|
projectId,
|
|
1213
|
+
registryPrompt: registryPrompt?.body,
|
|
1204
1214
|
systemPrompt,
|
|
1205
1215
|
userId,
|
|
1206
1216
|
});
|
|
1207
|
-
const
|
|
1217
|
+
const resolvedObservability = await aiService.resolveGenerateObservability({
|
|
1208
1218
|
promptLabel,
|
|
1209
1219
|
promptName,
|
|
1210
1220
|
sensitive: body.sensitive === true ? true : undefined,
|
|
1211
1221
|
sessionId,
|
|
1212
1222
|
systemPrompt: effectiveSystemPrompt,
|
|
1213
1223
|
});
|
|
1224
|
+
const observability = registryPrompt
|
|
1225
|
+
? {
|
|
1226
|
+
...resolvedObservability,
|
|
1227
|
+
promptRef: registryPrompt,
|
|
1228
|
+
sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
|
|
1229
|
+
}
|
|
1230
|
+
: resolvedObservability;
|
|
1214
1231
|
effectiveSystemPrompt = observability.systemPrompt;
|
|
1215
1232
|
// Some models (e.g. gemini-2.5-flash-image) don't support tool calling
|
|
1216
1233
|
const modelId = aiService.modelId;
|
|
@@ -1440,17 +1457,7 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
|
|
|
1440
1457
|
sink.emit({ askResolved: { action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId } });
|
|
1441
1458
|
}
|
|
1442
1459
|
api_1.logger.debug("Starting streamText", { model: modelId, supportsTools });
|
|
1443
|
-
const telemetry = (0, langfuseVercelAi_1.createTelemetryConfig)({
|
|
1444
|
-
functionId: "gpt-prompt",
|
|
1445
|
-
metadata: {
|
|
1446
|
-
...(options.langfuseSystemPromptName
|
|
1447
|
-
? { langfusePromptName: options.langfuseSystemPromptName }
|
|
1448
|
-
: {}),
|
|
1449
|
-
},
|
|
1450
|
-
userId: userId?.toString(),
|
|
1451
|
-
});
|
|
1452
1460
|
const result = (0, ai_1.streamText)({
|
|
1453
|
-
experimental_telemetry: telemetry,
|
|
1454
1461
|
messages,
|
|
1455
1462
|
model: aiService.model,
|
|
1456
1463
|
providerOptions: !supportsTools
|
package/dist/types/harness.d.ts
CHANGED
|
@@ -6,9 +6,12 @@ import type { ExecutionEnv } from "../harness/executionEnv";
|
|
|
6
6
|
import type { ObsPromptRef } from "./observability";
|
|
7
7
|
/**
|
|
8
8
|
* A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
|
|
9
|
-
* from `promptRegistry.get` fits as-is; its `body` is ignored
|
|
9
|
+
* from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
|
|
10
|
+
* prompts it referenced) are recorded right after it.
|
|
10
11
|
*/
|
|
11
|
-
export
|
|
12
|
+
export interface HarnessPromptRef extends ObsPromptRef {
|
|
13
|
+
includes?: ObsPromptRef[];
|
|
14
|
+
}
|
|
12
15
|
/** A registry prompt passed as instructions: its `body` is used and its version recorded. */
|
|
13
16
|
export interface HarnessPromptInstructions extends HarnessPromptRef {
|
|
14
17
|
body: string;
|
package/dist/types/index.d.ts
CHANGED
|
@@ -388,9 +388,14 @@ export interface GptRouteOptions {
|
|
|
388
388
|
* `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
|
|
389
389
|
*/
|
|
390
390
|
uiBlocks?: boolean | UiBlocksOptions;
|
|
391
|
-
/**
|
|
392
|
-
*
|
|
393
|
-
|
|
391
|
+
/**
|
|
392
|
+
* Observability prompt registry name used as the default system prompt (prepended to the
|
|
393
|
+
* request's system prompt). The trace records the version. Skipped when no registry is registered, the
|
|
394
|
+
* label has no version, or an admin request selects `promptName`.
|
|
395
|
+
*/
|
|
396
|
+
systemPromptName?: string;
|
|
397
|
+
/** Registry label read for `systemPromptName`. Defaults to `production`. */
|
|
398
|
+
systemPromptLabel?: string;
|
|
394
399
|
/**
|
|
395
400
|
* When `false` or the function returns `false`, prompts that include attachments are rejected.
|
|
396
401
|
* Omit or pass `true` to leave uploads enabled.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { FindExactlyOnePlugin, FindOneOrNonePlugin } from "@terreno/api";
|
|
2
2
|
import type mongoose from "mongoose";
|
|
3
|
+
import type { JudgeAgreement } from "../observability/judgeAgreement";
|
|
3
4
|
export interface ObsPromptVariable {
|
|
4
5
|
key: string;
|
|
5
6
|
label?: string;
|
|
@@ -13,6 +14,8 @@ export interface ObsPromptUsage {
|
|
|
13
14
|
outputTokens?: number;
|
|
14
15
|
}
|
|
15
16
|
export interface ObsPromptRef {
|
|
17
|
+
/** Set when this version was pulled in by another prompt's reference, to that prompt's name. */
|
|
18
|
+
includedBy?: string;
|
|
16
19
|
label?: string;
|
|
17
20
|
name: string;
|
|
18
21
|
version: number;
|
|
@@ -131,6 +134,8 @@ export interface ObsScoreModel extends mongoose.Model<ObsScoreDocument>, ObsScor
|
|
|
131
134
|
export interface EvaluatorDimension {
|
|
132
135
|
dataType: "boolean" | "categorical" | "numeric";
|
|
133
136
|
key: string;
|
|
137
|
+
/** Question shown to human reviewers, e.g. "Does the summary mention every todo?" */
|
|
138
|
+
question?: string;
|
|
134
139
|
range?: string;
|
|
135
140
|
required: boolean;
|
|
136
141
|
}
|
|
@@ -163,6 +168,7 @@ export interface ObsEvaluatorModel extends mongoose.Model<ObsEvaluatorDocument>,
|
|
|
163
168
|
}
|
|
164
169
|
export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
165
170
|
assigneeId?: mongoose.Types.ObjectId;
|
|
171
|
+
batchId?: mongoose.Types.ObjectId;
|
|
166
172
|
comment?: string;
|
|
167
173
|
created: Date;
|
|
168
174
|
datasetItemId?: mongoose.Types.ObjectId;
|
|
@@ -173,13 +179,53 @@ export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.
|
|
|
173
179
|
scores?: Record<string, boolean | number | string>;
|
|
174
180
|
spanId?: mongoose.Types.ObjectId;
|
|
175
181
|
status: "done" | "in_progress" | "pending" | "skipped";
|
|
176
|
-
|
|
182
|
+
/** Trace under review. Unset when reviewers judge a dataset item that has no source trace. */
|
|
183
|
+
traceId?: mongoose.Types.ObjectId;
|
|
177
184
|
updated: Date;
|
|
178
185
|
}
|
|
179
186
|
export interface ObsReviewItemStatics extends FindExactlyOnePlugin<ObsReviewItemDocument>, FindOneOrNonePlugin<ObsReviewItemDocument> {
|
|
180
187
|
}
|
|
181
188
|
export interface ObsReviewItemModel extends mongoose.Model<ObsReviewItemDocument>, ObsReviewItemStatics {
|
|
182
189
|
}
|
|
190
|
+
export interface ObsReviewBatchDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
191
|
+
created: Date;
|
|
192
|
+
/** Dataset the batch was built from, when reviewers judge dataset items. */
|
|
193
|
+
datasetId?: mongoose.Types.ObjectId;
|
|
194
|
+
deleted: boolean;
|
|
195
|
+
description?: string;
|
|
196
|
+
evaluatorId: mongoose.Types.ObjectId;
|
|
197
|
+
name: string;
|
|
198
|
+
promptName?: string;
|
|
199
|
+
/** Question-set fingerprint when the batch was created (see `questionSetFingerprint`). */
|
|
200
|
+
questionsFingerprint?: string;
|
|
201
|
+
traceIds: mongoose.Types.ObjectId[];
|
|
202
|
+
updated: Date;
|
|
203
|
+
}
|
|
204
|
+
export interface ObsReviewBatchStatics extends FindExactlyOnePlugin<ObsReviewBatchDocument>, FindOneOrNonePlugin<ObsReviewBatchDocument> {
|
|
205
|
+
}
|
|
206
|
+
export interface ObsReviewBatchModel extends mongoose.Model<ObsReviewBatchDocument>, ObsReviewBatchStatics {
|
|
207
|
+
}
|
|
208
|
+
export interface ObsJudgeRunResult {
|
|
209
|
+
confidence?: number;
|
|
210
|
+
datasetItemId?: mongoose.Types.ObjectId;
|
|
211
|
+
error?: string;
|
|
212
|
+
scores?: Record<string, boolean | number | string>;
|
|
213
|
+
traceId?: mongoose.Types.ObjectId;
|
|
214
|
+
}
|
|
215
|
+
export interface ObsJudgeRunDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
216
|
+
batchId: mongoose.Types.ObjectId;
|
|
217
|
+
completedAt: Date;
|
|
218
|
+
created: Date;
|
|
219
|
+
deleted: boolean;
|
|
220
|
+
evaluatorId: mongoose.Types.ObjectId;
|
|
221
|
+
modelId?: string;
|
|
222
|
+
results: ObsJudgeRunResult[];
|
|
223
|
+
updated: Date;
|
|
224
|
+
}
|
|
225
|
+
export interface ObsJudgeRunStatics extends FindExactlyOnePlugin<ObsJudgeRunDocument>, FindOneOrNonePlugin<ObsJudgeRunDocument> {
|
|
226
|
+
}
|
|
227
|
+
export interface ObsJudgeRunModel extends mongoose.Model<ObsJudgeRunDocument>, ObsJudgeRunStatics {
|
|
228
|
+
}
|
|
183
229
|
export interface ObsDatasetAnnotatedBy {
|
|
184
230
|
label: string;
|
|
185
231
|
reviewItemId?: string;
|
|
@@ -199,12 +245,23 @@ export interface ObsDatasetStatics extends FindExactlyOnePlugin<ObsDatasetDocume
|
|
|
199
245
|
}
|
|
200
246
|
export interface ObsDatasetModel extends mongoose.Model<ObsDatasetDocument>, ObsDatasetStatics {
|
|
201
247
|
}
|
|
248
|
+
/** One question set's human answers for a dataset item. */
|
|
249
|
+
export interface ObsDatasetHumanAnswer {
|
|
250
|
+
answeredAt: Date;
|
|
251
|
+
/** `questionSetFingerprint` of the question set when the reviewer answered. */
|
|
252
|
+
fingerprint: string;
|
|
253
|
+
reviewItemId: string;
|
|
254
|
+
scores: Record<string, boolean | number | string>;
|
|
255
|
+
userId?: string;
|
|
256
|
+
}
|
|
202
257
|
export interface ObsDatasetItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
203
258
|
annotatedBy?: ObsDatasetAnnotatedBy;
|
|
204
259
|
created: Date;
|
|
205
260
|
datasetId: mongoose.Types.ObjectId;
|
|
206
261
|
deleted: boolean;
|
|
207
262
|
expectedOutput?: unknown;
|
|
263
|
+
/** Human answers keyed by question-set (human evaluator) id. */
|
|
264
|
+
humanAnswers?: Record<string, ObsDatasetHumanAnswer>;
|
|
208
265
|
input: unknown;
|
|
209
266
|
metadata?: Record<string, unknown>;
|
|
210
267
|
origin: "manual" | "synthetic" | "trace";
|
|
@@ -219,7 +276,8 @@ export interface ObsDatasetItemStatics extends FindExactlyOnePlugin<ObsDatasetIt
|
|
|
219
276
|
export interface ObsDatasetItemModel extends mongoose.Model<ObsDatasetItemDocument>, ObsDatasetItemStatics {
|
|
220
277
|
}
|
|
221
278
|
export interface ScoreThreshold {
|
|
222
|
-
|
|
279
|
+
/** `agreement` gates compare a judge prompt version's answers with human answers. */
|
|
280
|
+
aggregate: "agreement" | "mean" | "trueRate";
|
|
223
281
|
dimension: string;
|
|
224
282
|
evaluatorName: string;
|
|
225
283
|
op: "eq" | "gte" | "lte";
|
|
@@ -235,7 +293,10 @@ export interface ExperimentEstimate {
|
|
|
235
293
|
generations: number;
|
|
236
294
|
wallClockSeconds: number;
|
|
237
295
|
}
|
|
296
|
+
export type ExperimentKind = "judge" | "prompt";
|
|
238
297
|
export interface ExperimentAggregates {
|
|
298
|
+
/** Judge experiments: agreement with the human answers per judge prompt version. */
|
|
299
|
+
agreement?: Record<string, JudgeAgreement>;
|
|
239
300
|
gates: ExperimentGateResult[];
|
|
240
301
|
lowConfidenceItemIds: string[];
|
|
241
302
|
outlierItemIds: string[];
|
|
@@ -253,11 +314,20 @@ export interface ObsExperimentDocument extends mongoose.Document<mongoose.Types.
|
|
|
253
314
|
estimate?: ExperimentEstimate;
|
|
254
315
|
evaluatorIds: mongoose.Types.ObjectId[];
|
|
255
316
|
includeUnproofread: boolean;
|
|
317
|
+
/** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
|
|
318
|
+
judgeEvaluatorId?: mongoose.Types.ObjectId;
|
|
319
|
+
kind: ExperimentKind;
|
|
256
320
|
modelOverride?: string;
|
|
257
321
|
name: string;
|
|
258
322
|
promptName: string;
|
|
323
|
+
/** Judge experiments: question-set fingerprint of the human answers compared. */
|
|
324
|
+
questionsFingerprint?: string;
|
|
325
|
+
/** Judge experiments: the human question set whose answers are the reference. */
|
|
326
|
+
questionSetId?: mongoose.Types.ObjectId;
|
|
259
327
|
results?: ExperimentAggregates;
|
|
260
328
|
status: "completed" | "failed" | "pending" | "running";
|
|
329
|
+
/** Judge experiments: agreement rate (0–1) a version must reach to pass. */
|
|
330
|
+
targetAgreement?: number;
|
|
261
331
|
thresholds: ScoreThreshold[];
|
|
262
332
|
updated: Date;
|
|
263
333
|
versions: number[];
|
|
@@ -282,6 +352,8 @@ export interface ObsExperimentItemDocument extends mongoose.Document<mongoose.Ty
|
|
|
282
352
|
deleted: boolean;
|
|
283
353
|
experimentId: mongoose.Types.ObjectId;
|
|
284
354
|
failed: boolean;
|
|
355
|
+
/** Judge experiments: the human answers the judge versions are compared with. */
|
|
356
|
+
humanScores?: Record<string, boolean | number | string>;
|
|
285
357
|
updated: Date;
|
|
286
358
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
287
359
|
}
|
package/package.json
CHANGED
|
@@ -6,11 +6,7 @@
|
|
|
6
6
|
"dependencies": {
|
|
7
7
|
"@ai-sdk/mcp": "^2.0.57",
|
|
8
8
|
"@google-cloud/storage": "^7.15.0",
|
|
9
|
-
"@
|
|
10
|
-
"@langfuse/otel": "^5.0.1",
|
|
11
|
-
"@opentelemetry/api": "^1.9.0",
|
|
12
|
-
"@opentelemetry/sdk-node": "0.222.0",
|
|
13
|
-
"@terreno/blocks": "58.0.0-beta.3",
|
|
9
|
+
"@terreno/blocks": "58.0.0-beta.4",
|
|
14
10
|
"ai": "^6.0.97",
|
|
15
11
|
"ajv": "8.20.0",
|
|
16
12
|
"csv-parse": "^6.1.0",
|
|
@@ -22,10 +18,10 @@
|
|
|
22
18
|
"description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
|
|
23
19
|
"devDependencies": {
|
|
24
20
|
"@biomejs/biome": "^2.3.6",
|
|
25
|
-
"@terreno/admin-backend": "58.0.0-beta.
|
|
26
|
-
"@terreno/api": "58.0.0-beta.
|
|
27
|
-
"@terreno/jobs": "58.0.0-beta.
|
|
28
|
-
"@terreno/test": "58.0.0-beta.
|
|
21
|
+
"@terreno/admin-backend": "58.0.0-beta.4",
|
|
22
|
+
"@terreno/api": "58.0.0-beta.4",
|
|
23
|
+
"@terreno/jobs": "58.0.0-beta.4",
|
|
24
|
+
"@terreno/test": "58.0.0-beta.4",
|
|
29
25
|
"@types/bun": "^1.4.0",
|
|
30
26
|
"@types/express": "^5.0.6",
|
|
31
27
|
"@types/luxon": "^3.7.1",
|
|
@@ -87,18 +83,6 @@
|
|
|
87
83
|
"require": "./dist/harness/runners/jobsRunner.js",
|
|
88
84
|
"types": "./dist/harness/runners/jobsRunner.d.ts"
|
|
89
85
|
},
|
|
90
|
-
"./langfuseApp": {
|
|
91
|
-
"default": "./dist/langfuseApp.js",
|
|
92
|
-
"import": "./dist/langfuseApp.js",
|
|
93
|
-
"require": "./dist/langfuseApp.js",
|
|
94
|
-
"types": "./dist/langfuseApp.d.ts"
|
|
95
|
-
},
|
|
96
|
-
"./langfuseClient": {
|
|
97
|
-
"default": "./dist/langfuseClient.js",
|
|
98
|
-
"import": "./dist/langfuseClient.js",
|
|
99
|
-
"require": "./dist/langfuseClient.js",
|
|
100
|
-
"types": "./dist/langfuseClient.d.ts"
|
|
101
|
-
},
|
|
102
86
|
"./observability/localPlugin": {
|
|
103
87
|
"default": "./dist/observability/local/localPlugin.js",
|
|
104
88
|
"import": "./dist/observability/local/localPlugin.js",
|
|
@@ -137,8 +121,8 @@
|
|
|
137
121
|
"peerDependencies": {
|
|
138
122
|
"@ai-sdk/google-vertex": "^5.0.92",
|
|
139
123
|
"@ai-sdk/provider": "^4.0.26",
|
|
140
|
-
"@terreno/api": "58.0.0-beta.
|
|
141
|
-
"@terreno/jobs": "58.0.0-beta.
|
|
124
|
+
"@terreno/api": "58.0.0-beta.4",
|
|
125
|
+
"@terreno/jobs": "58.0.0-beta.4",
|
|
142
126
|
"mongoose": "^8.0.0 || ^9.0.0"
|
|
143
127
|
},
|
|
144
128
|
"peerDependenciesMeta": {
|
|
@@ -171,5 +155,5 @@
|
|
|
171
155
|
"typecheck": "tsc -p tsconfig.typecheck.json"
|
|
172
156
|
},
|
|
173
157
|
"types": "dist/index.d.ts",
|
|
174
|
-
"version": "58.0.0-beta.
|
|
158
|
+
"version": "58.0.0-beta.4"
|
|
175
159
|
}
|
|
@@ -178,9 +178,6 @@ const spansOf = (task: HarnessTaskDocument) =>
|
|
|
178
178
|
|
|
179
179
|
describe("Agent turns", () => {
|
|
180
180
|
beforeAll(async () => {
|
|
181
|
-
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
182
|
-
// cold first import can exceed it, so load it here first.
|
|
183
|
-
await import("../langfuseClient.js");
|
|
184
181
|
createLocalObservabilityPlugin();
|
|
185
182
|
SpanModel = registerObsSpan();
|
|
186
183
|
TraceModel = registerObsTrace();
|
|
@@ -170,9 +170,6 @@ const toolMessages = async (conversationId: string) => {
|
|
|
170
170
|
|
|
171
171
|
describe("Agent turns across a crash", () => {
|
|
172
172
|
beforeAll(async () => {
|
|
173
|
-
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
174
|
-
// cold first import can exceed it, so load it here first.
|
|
175
|
-
await import("../langfuseClient.js");
|
|
176
173
|
createLocalObservabilityPlugin();
|
|
177
174
|
SpanModel = registerObsSpan();
|
|
178
175
|
TraceModel = registerObsTrace();
|
package/src/harness/commit.ts
CHANGED
|
@@ -152,9 +152,10 @@ const findByRequestId = async ({
|
|
|
152
152
|
};
|
|
153
153
|
|
|
154
154
|
/**
|
|
155
|
-
* Prompt refs as stored on `ObsTrace.prompts` (`{label?, name, version}`; a
|
|
156
|
-
* `body` is dropped
|
|
157
|
-
* non-string label
|
|
155
|
+
* Prompt refs as stored on `ObsTrace.prompts` (`{includedBy?, label?, name, version}`; a
|
|
156
|
+
* registry `body` is dropped and its `includes` follow it), once each. Throws
|
|
157
|
+
* `invalidRequest` for a ref without a name, with a non-string label or includedBy, with
|
|
158
|
+
* non-array includes, or without a positive integer version.
|
|
158
159
|
*/
|
|
159
160
|
export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPromptRef[] => {
|
|
160
161
|
if (refs === undefined) {
|
|
@@ -164,7 +165,19 @@ export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPr
|
|
|
164
165
|
throw harnessError({detail: "prompts must be an array", kind: "invalidRequest"});
|
|
165
166
|
}
|
|
166
167
|
const unique = new Map<string, HarnessPromptRef>();
|
|
167
|
-
|
|
168
|
+
const expanded = refs.flatMap((ref) => {
|
|
169
|
+
if (ref?.includes !== undefined && !Array.isArray(ref.includes)) {
|
|
170
|
+
throw harnessError({
|
|
171
|
+
detail:
|
|
172
|
+
typeof ref.name === "string" && ref.name.trim()
|
|
173
|
+
? `Prompt ref "${ref.name}" includes must be an array`
|
|
174
|
+
: "Prompt ref includes must be an array",
|
|
175
|
+
kind: "invalidRequest",
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
return [ref, ...(ref?.includes ?? [])];
|
|
179
|
+
});
|
|
180
|
+
for (const ref of expanded) {
|
|
168
181
|
if (typeof ref?.name !== "string" || !ref.name.trim()) {
|
|
169
182
|
throw harnessError({detail: "Each prompt ref needs a name", kind: "invalidRequest"});
|
|
170
183
|
}
|
|
@@ -180,7 +193,14 @@ export const tracePromptRefs = (refs: HarnessPromptRef[] | undefined): HarnessPr
|
|
|
180
193
|
kind: "invalidRequest",
|
|
181
194
|
});
|
|
182
195
|
}
|
|
196
|
+
if (ref.includedBy !== undefined && typeof ref.includedBy !== "string") {
|
|
197
|
+
throw harnessError({
|
|
198
|
+
detail: `Prompt ref "${ref.name}" includedBy must be a string`,
|
|
199
|
+
kind: "invalidRequest",
|
|
200
|
+
});
|
|
201
|
+
}
|
|
183
202
|
const normalized = {
|
|
203
|
+
...(ref.includedBy !== undefined ? {includedBy: ref.includedBy} : {}),
|
|
184
204
|
...(ref.label !== undefined ? {label: ref.label} : {}),
|
|
185
205
|
name: ref.name,
|
|
186
206
|
version: ref.version,
|
|
@@ -198,9 +198,6 @@ const llmSpans = (task: HarnessTaskDocument) =>
|
|
|
198
198
|
|
|
199
199
|
describe("Extensions, hooks, wraps, and memos", () => {
|
|
200
200
|
beforeAll(async () => {
|
|
201
|
-
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
202
|
-
// cold first import can exceed it, so load it here first.
|
|
203
|
-
await import("../langfuseClient.js");
|
|
204
201
|
createLocalObservabilityPlugin();
|
|
205
202
|
SpanModel = registerObsSpan();
|
|
206
203
|
TraceModel = registerObsTrace();
|
|
@@ -247,9 +247,6 @@ const parentTask = ({
|
|
|
247
247
|
|
|
248
248
|
describe("rt.runAgent (subagents)", () => {
|
|
249
249
|
beforeAll(async () => {
|
|
250
|
-
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
251
|
-
// cold first import can exceed it, so load it here first.
|
|
252
|
-
await import("../langfuseClient.js");
|
|
253
250
|
createLocalObservabilityPlugin();
|
|
254
251
|
SpanModel = registerObsSpan();
|
|
255
252
|
TraceModel = registerObsTrace();
|
|
@@ -378,6 +375,16 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
378
375
|
).rejects.toThrow(
|
|
379
376
|
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" label must be a string')
|
|
380
377
|
);
|
|
378
|
+
await expect(
|
|
379
|
+
harness.createTask(parent, {}, {prompts: [{includes: {}, name: "plan", version: 1} as never]})
|
|
380
|
+
).rejects.toThrow(
|
|
381
|
+
harnessErrorMatching("invalidRequest", 'Prompt ref "plan" includes must be an array')
|
|
382
|
+
);
|
|
383
|
+
await expect(
|
|
384
|
+
harness.createTask(parent, {}, {prompts: [{includes: {}, version: 1} as never]})
|
|
385
|
+
).rejects.toThrow(
|
|
386
|
+
harnessErrorMatching("invalidRequest", "Prompt ref includes must be an array")
|
|
387
|
+
);
|
|
381
388
|
await expect(harness.createTask(parent, {}, {trace: {tags: ["ok", " "]}})).rejects.toThrow(
|
|
382
389
|
harnessErrorMatching("invalidRequest", "trace.tags must be an array of non-empty strings")
|
|
383
390
|
);
|
|
@@ -397,6 +404,23 @@ describe("rt.runAgent (subagents)", () => {
|
|
|
397
404
|
expect(plain(trace.prompts)).toEqual([plan]);
|
|
398
405
|
});
|
|
399
406
|
|
|
407
|
+
it("records a registry prompt's shared includes after it", async () => {
|
|
408
|
+
const agent = summarizerWith();
|
|
409
|
+
const parent = parentTask({agent, inputs: []});
|
|
410
|
+
const {model} = scriptedModel([]);
|
|
411
|
+
const {harness} = await openProcess({model, registry: [agent, parent]});
|
|
412
|
+
|
|
413
|
+
const include = {includedBy: "plan", label: "production", name: "base-rules", version: 4};
|
|
414
|
+
const created = await harness.createTask(
|
|
415
|
+
parent,
|
|
416
|
+
{},
|
|
417
|
+
{prompts: [{includes: [include], name: "plan", version: 2}]}
|
|
418
|
+
);
|
|
419
|
+
|
|
420
|
+
const trace = await TraceModel.findExactlyOne({_id: created.traceId});
|
|
421
|
+
expect(plain(trace.prompts)).toEqual([{name: "plan", version: 2}, include]);
|
|
422
|
+
});
|
|
423
|
+
|
|
400
424
|
it("fails the caller at once for an instructions prompt without a body or a bad prompt ref", async () => {
|
|
401
425
|
const agent = summarizerWith();
|
|
402
426
|
const noBody = parentTask({
|