@terreno/ai 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +3 -3
- package/biome.jsonc +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +42 -8
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +10 -4
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -28
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +30 -30
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/commit.ts +24 -4
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +6 -8
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +29 -4
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -20
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +25 -28
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +497 -17
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +295 -3
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +39 -2
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +8 -8
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +63 -2
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +156 -115
- package/src/service/aiService.test.ts +22 -1
- package/src/service/aiService.ts +4 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +42 -33
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +0 -54
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +11 -4
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/tsconfig.typecheck.json +18 -0
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -60
- package/src/langfuseApp.test.ts +0 -135
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -256
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -68
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -79
- package/src/langfuseVercelAi.ts +0 -72
|
@@ -13,7 +13,7 @@ import {AIService, TemperaturePresets} from "./aiService";
|
|
|
13
13
|
// Create a mock LanguageModelV2
|
|
14
14
|
const createMockModel = (responseText = "Mock response") => {
|
|
15
15
|
return {
|
|
16
|
-
doGenerate: mock(async () => ({
|
|
16
|
+
doGenerate: mock(async (_options: unknown) => ({
|
|
17
17
|
content: [{text: responseText, type: "text" as const}],
|
|
18
18
|
finishReason: "stop" as const,
|
|
19
19
|
usage: {inputTokens: 5, outputTokens: 10},
|
|
@@ -1201,6 +1201,27 @@ describe("AIService", () => {
|
|
|
1201
1201
|
expect(sink.traces[0].prompts).toEqual([{label: "production", name: "greeter", version: 2}]);
|
|
1202
1202
|
});
|
|
1203
1203
|
|
|
1204
|
+
it("records a prompt's shared includes on the trace after the prompt itself", async () => {
|
|
1205
|
+
const sink = new MemoryTraceSink();
|
|
1206
|
+
registerLocalApp({
|
|
1207
|
+
promptVersion: {
|
|
1208
|
+
...productionVersion,
|
|
1209
|
+
includes: [{includedBy: "greeter", label: "production", name: "base-rules", version: 4}],
|
|
1210
|
+
},
|
|
1211
|
+
traceSink: sink,
|
|
1212
|
+
});
|
|
1213
|
+
const model = createMockModel("Hello world");
|
|
1214
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1215
|
+
|
|
1216
|
+
await service.generateText({prompt: "User says hi", promptName: "greeter"});
|
|
1217
|
+
|
|
1218
|
+
expect(sink.traces[0].name).toBe("greeter");
|
|
1219
|
+
expect(sink.traces[0].prompts).toEqual([
|
|
1220
|
+
{label: "production", name: "greeter", version: 2},
|
|
1221
|
+
{includedBy: "greeter", label: "production", name: "base-rules", version: 4},
|
|
1222
|
+
]);
|
|
1223
|
+
});
|
|
1224
|
+
|
|
1204
1225
|
it("still writes AIRequest for a chat stream when a sink is registered", async () => {
|
|
1205
1226
|
const sink = new MemoryTraceSink();
|
|
1206
1227
|
registerLocalApp({traceSink: sink});
|
package/src/service/aiService.ts
CHANGED
|
@@ -397,7 +397,10 @@ export class AIService {
|
|
|
397
397
|
name: spanName,
|
|
398
398
|
output: params.response,
|
|
399
399
|
prompts: promptRef
|
|
400
|
-
? [
|
|
400
|
+
? [
|
|
401
|
+
{label: promptRef.label, name: promptRef.name, version: promptRef.version},
|
|
402
|
+
...(promptRef.includes ?? []),
|
|
403
|
+
]
|
|
401
404
|
: [],
|
|
402
405
|
sensitive: params.observability.sensitive,
|
|
403
406
|
sessionId: params.observability.sessionId,
|
package/src/service/asks.test.ts
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import {describe, expect, it} from "bun:test";
|
|
2
2
|
import {ASK_LIMITS, type AskKind, type AskResponse, askPromptSection} from "@terreno/blocks";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
asSchema,
|
|
5
|
+
type JSONValue,
|
|
6
|
+
jsonSchema,
|
|
7
|
+
type ModelMessage,
|
|
8
|
+
type Tool,
|
|
9
|
+
type ToolResultPart,
|
|
10
|
+
tool,
|
|
11
|
+
} from "ai";
|
|
4
12
|
|
|
5
13
|
import {
|
|
6
14
|
approvalAskInput,
|
|
@@ -24,7 +32,7 @@ const PLAN_ASK_INPUT = {
|
|
|
24
32
|
{id: "team", label: "Team"},
|
|
25
33
|
],
|
|
26
34
|
prompt: "Which plan should I set up?",
|
|
27
|
-
select: "one",
|
|
35
|
+
select: "one" as const,
|
|
28
36
|
};
|
|
29
37
|
|
|
30
38
|
const TEAM_ANSWER = {action: "accept" as const, content: {selected: ["team"]}};
|
|
@@ -57,8 +65,8 @@ const lookupResult = {
|
|
|
57
65
|
type: "tool-result" as const,
|
|
58
66
|
};
|
|
59
67
|
|
|
60
|
-
const answerResult = (toolCallId: string, value: unknown) => ({
|
|
61
|
-
output: {type: "json", value},
|
|
68
|
+
const answerResult = (toolCallId: string, value: unknown): ToolResultPart => ({
|
|
69
|
+
output: {type: "json", value: value as JSONValue},
|
|
62
70
|
toolCallId,
|
|
63
71
|
toolName: "ask_choice",
|
|
64
72
|
type: "tool-result",
|
|
@@ -76,14 +84,14 @@ describe("asks", () => {
|
|
|
76
84
|
expected: ["confirm"],
|
|
77
85
|
label: '{kinds: ["confirm"]}',
|
|
78
86
|
},
|
|
79
|
-
{asks: {kinds: []}, expected: [], label: "{kinds: []}"},
|
|
87
|
+
{asks: {kinds: [] as AskKind[]}, expected: [], label: "{kinds: []}"},
|
|
80
88
|
{
|
|
81
89
|
asks: {kinds: ["choice", "choice"] as AskKind[]},
|
|
82
90
|
expected: ["choice"],
|
|
83
91
|
label: '{kinds: ["choice", "choice"]}',
|
|
84
92
|
},
|
|
85
93
|
])("resolves $label", ({asks, expected}) => {
|
|
86
|
-
expect(resolveAskKinds(asks)).toEqual(expected);
|
|
94
|
+
expect(resolveAskKinds(asks)).toEqual([...expected]);
|
|
87
95
|
});
|
|
88
96
|
|
|
89
97
|
it("throws on an unknown kind and names the known kinds", () => {
|
package/src/service/chatTurn.ts
CHANGED
|
@@ -24,12 +24,12 @@ import type express from "express";
|
|
|
24
24
|
import {DateTime} from "luxon";
|
|
25
25
|
import type mongoose from "mongoose";
|
|
26
26
|
|
|
27
|
-
import {createTelemetryConfig, preparePromptForAI} from "../langfuseVercelAi";
|
|
28
27
|
import {AIRequest} from "../models/aiRequest";
|
|
29
28
|
import {FileAttachment} from "../models/fileAttachment";
|
|
30
29
|
import {GptHistory} from "../models/gptHistory";
|
|
31
30
|
import {getProjectModel} from "../models/project";
|
|
32
|
-
import
|
|
31
|
+
import {getObservabilityApp} from "../observability/observabilityAppRegistry";
|
|
32
|
+
import type {PromptVersionRef, SpanRecord} from "../observability/types";
|
|
33
33
|
import type {
|
|
34
34
|
AskOrigin,
|
|
35
35
|
GptHistoryAskStatus,
|
|
@@ -1140,17 +1140,39 @@ const startTurn = async ({
|
|
|
1140
1140
|
const lastUserText = (prompts: GptHistoryPrompt[]): string =>
|
|
1141
1141
|
prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
|
|
1142
1142
|
|
|
1143
|
-
/** The
|
|
1143
|
+
/** The route's default registry system prompt, or undefined when it cannot be loaded. */
|
|
1144
|
+
const loadRegistrySystemPrompt = async (
|
|
1145
|
+
options: GptRouteOptions
|
|
1146
|
+
): Promise<PromptVersionRef | undefined> => {
|
|
1147
|
+
const registry = getObservabilityApp()?.promptRegistry;
|
|
1148
|
+
if (!options.systemPromptName || !registry) {
|
|
1149
|
+
return undefined;
|
|
1150
|
+
}
|
|
1151
|
+
try {
|
|
1152
|
+
return await registry.get({
|
|
1153
|
+
label: options.systemPromptLabel ?? "production",
|
|
1154
|
+
name: options.systemPromptName,
|
|
1155
|
+
});
|
|
1156
|
+
} catch (error) {
|
|
1157
|
+
logger.warn("Registry system prompt skipped", {
|
|
1158
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1159
|
+
promptName: options.systemPromptName,
|
|
1160
|
+
});
|
|
1161
|
+
return undefined;
|
|
1162
|
+
}
|
|
1163
|
+
};
|
|
1164
|
+
|
|
1165
|
+
/** The request's system prompt with project context and the registry prompt prepended. */
|
|
1144
1166
|
const buildSystemPrompt = async ({
|
|
1145
1167
|
history,
|
|
1146
|
-
options,
|
|
1147
1168
|
projectId,
|
|
1169
|
+
registryPrompt,
|
|
1148
1170
|
systemPrompt,
|
|
1149
1171
|
userId,
|
|
1150
1172
|
}: {
|
|
1151
1173
|
history: GptHistoryDocument;
|
|
1152
|
-
options: GptRouteOptions;
|
|
1153
1174
|
projectId: unknown;
|
|
1175
|
+
registryPrompt?: string;
|
|
1154
1176
|
systemPrompt: unknown;
|
|
1155
1177
|
userId: mongoose.Types.ObjectId | undefined;
|
|
1156
1178
|
}): Promise<string | undefined> => {
|
|
@@ -1181,22 +1203,10 @@ const buildSystemPrompt = async ({
|
|
|
1181
1203
|
}
|
|
1182
1204
|
}
|
|
1183
1205
|
|
|
1184
|
-
if (
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
userId: userId?.toString(),
|
|
1189
|
-
});
|
|
1190
|
-
const langfusePrompt =
|
|
1191
|
-
typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
|
|
1192
|
-
if (langfusePrompt) {
|
|
1193
|
-
effectiveSystemPrompt = effectiveSystemPrompt
|
|
1194
|
-
? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
|
|
1195
|
-
: langfusePrompt;
|
|
1196
|
-
}
|
|
1197
|
-
} catch (err) {
|
|
1198
|
-
logger.debug(`Langfuse system prompt skipped: ${(err as Error).message}`);
|
|
1199
|
-
}
|
|
1206
|
+
if (registryPrompt) {
|
|
1207
|
+
effectiveSystemPrompt = effectiveSystemPrompt
|
|
1208
|
+
? `${registryPrompt}\n\n${effectiveSystemPrompt}`
|
|
1209
|
+
: registryPrompt;
|
|
1200
1210
|
}
|
|
1201
1211
|
return effectiveSystemPrompt;
|
|
1202
1212
|
};
|
|
@@ -1778,20 +1788,29 @@ export const runChatTurn = async ({
|
|
|
1778
1788
|
const {answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt} =
|
|
1779
1789
|
turn;
|
|
1780
1790
|
let {history} = turn;
|
|
1791
|
+
// An admin-selected promptName replaces the system prompt, so the route default is skipped.
|
|
1792
|
+
const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
|
|
1781
1793
|
let effectiveSystemPrompt = await buildSystemPrompt({
|
|
1782
1794
|
history,
|
|
1783
|
-
options,
|
|
1784
1795
|
projectId,
|
|
1796
|
+
registryPrompt: registryPrompt?.body,
|
|
1785
1797
|
systemPrompt,
|
|
1786
1798
|
userId,
|
|
1787
1799
|
});
|
|
1788
|
-
const
|
|
1800
|
+
const resolvedObservability = await aiService.resolveGenerateObservability({
|
|
1789
1801
|
promptLabel,
|
|
1790
1802
|
promptName,
|
|
1791
1803
|
sensitive: body.sensitive === true ? true : undefined,
|
|
1792
1804
|
sessionId,
|
|
1793
1805
|
systemPrompt: effectiveSystemPrompt,
|
|
1794
1806
|
});
|
|
1807
|
+
const observability = registryPrompt
|
|
1808
|
+
? {
|
|
1809
|
+
...resolvedObservability,
|
|
1810
|
+
promptRef: registryPrompt,
|
|
1811
|
+
sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
|
|
1812
|
+
}
|
|
1813
|
+
: resolvedObservability;
|
|
1795
1814
|
effectiveSystemPrompt = observability.systemPrompt;
|
|
1796
1815
|
|
|
1797
1816
|
// Some models (e.g. gemini-2.5-flash-image) don't support tool calling
|
|
@@ -2037,17 +2056,7 @@ export const runChatTurn = async ({
|
|
|
2037
2056
|
sink.emit({askResolved: {action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId}});
|
|
2038
2057
|
}
|
|
2039
2058
|
logger.debug("Starting streamText", {model: modelId, supportsTools});
|
|
2040
|
-
const telemetry = createTelemetryConfig({
|
|
2041
|
-
functionId: "gpt-prompt",
|
|
2042
|
-
metadata: {
|
|
2043
|
-
...(options.langfuseSystemPromptName
|
|
2044
|
-
? {langfusePromptName: options.langfuseSystemPromptName}
|
|
2045
|
-
: {}),
|
|
2046
|
-
},
|
|
2047
|
-
userId: userId?.toString(),
|
|
2048
|
-
});
|
|
2049
2059
|
const result = streamText({
|
|
2050
|
-
experimental_telemetry: telemetry,
|
|
2051
2060
|
messages,
|
|
2052
2061
|
model: aiService.model,
|
|
2053
2062
|
providerOptions: !supportsTools
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
2
2
|
import {registerGcsMock} from "@terreno/test";
|
|
3
3
|
import mongoose from "mongoose";
|
|
4
4
|
|
|
@@ -7,7 +7,10 @@ import {FileAttachment} from "../models/fileAttachment";
|
|
|
7
7
|
// In-memory @google-cloud/storage so the service doesn't try to authenticate
|
|
8
8
|
const gcs = registerGcsMock();
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
let FileStorageService: typeof import("./fileStorage.js").FileStorageService;
|
|
11
|
+
beforeAll(async () => {
|
|
12
|
+
({FileStorageService} = await import("./fileStorage.js"));
|
|
13
|
+
});
|
|
11
14
|
|
|
12
15
|
describe("FileStorageService", () => {
|
|
13
16
|
beforeEach(async () => {
|
|
@@ -7,12 +7,16 @@ import {
|
|
|
7
7
|
registerMCPTool,
|
|
8
8
|
type User,
|
|
9
9
|
} from "@terreno/api";
|
|
10
|
+
import type {ToolExecutionOptions} from "ai";
|
|
10
11
|
import {assert} from "chai";
|
|
11
12
|
import mongoose, {Schema} from "mongoose";
|
|
12
13
|
import {z} from "zod";
|
|
13
14
|
|
|
14
15
|
import {getMCPTools} from "./getMCPTools";
|
|
15
16
|
|
|
17
|
+
/** `execute` options as the AI SDK passes them; getMCPTools ignores them. */
|
|
18
|
+
const callOptions: ToolExecutionOptions = {messages: [], toolCallId: "call-1"};
|
|
19
|
+
|
|
16
20
|
interface NoteFields {
|
|
17
21
|
_id: mongoose.Types.ObjectId;
|
|
18
22
|
title: string;
|
|
@@ -67,7 +71,7 @@ describe("getMCPTools", () => {
|
|
|
67
71
|
|
|
68
72
|
expect(Object.keys(tools).sort()).toEqual(["notes_create", "notes_list"]);
|
|
69
73
|
|
|
70
|
-
const created = await tools.notes_create.execute?.({title: "From AI tools"});
|
|
74
|
+
const created = await tools.notes_create.execute?.({title: "From AI tools"}, callOptions);
|
|
71
75
|
expect(created).toEqual(
|
|
72
76
|
expect.objectContaining({
|
|
73
77
|
data: expect.objectContaining({title: "From AI tools"}),
|
|
@@ -88,7 +92,7 @@ describe("getMCPTools", () => {
|
|
|
88
92
|
const tools = getMCPTools();
|
|
89
93
|
assert.deepEqual(Object.keys(tools), ["echo_json"]);
|
|
90
94
|
assert.equal(tools.echo_json.description, "Echoes structured json back to the caller");
|
|
91
|
-
assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}), {echoed: "hi"});
|
|
95
|
+
assert.deepEqual(await tools.echo_json.execute?.({value: "hi"}, callOptions), {echoed: "hi"});
|
|
92
96
|
});
|
|
93
97
|
|
|
94
98
|
it("returns joined plain text when the tool response is not json", async () => {
|
|
@@ -105,7 +109,7 @@ describe("getMCPTools", () => {
|
|
|
105
109
|
});
|
|
106
110
|
|
|
107
111
|
const tools = getMCPTools();
|
|
108
|
-
assert.equal(await tools.prose.execute?.({}), "first line\nsecond line");
|
|
112
|
+
assert.equal(await tools.prose.execute?.({}, callOptions), "first line\nsecond line");
|
|
109
113
|
});
|
|
110
114
|
|
|
111
115
|
it("passes the authenticated user through to custom tool handlers", async () => {
|
|
@@ -121,8 +125,8 @@ describe("getMCPTools", () => {
|
|
|
121
125
|
});
|
|
122
126
|
|
|
123
127
|
const user = asUser();
|
|
124
|
-
await getMCPTools(user).whoami.execute?.({});
|
|
125
|
-
await getMCPTools().whoami.execute?.({});
|
|
128
|
+
await getMCPTools(user).whoami.execute?.({}, callOptions);
|
|
129
|
+
await getMCPTools().whoami.execute?.({}, callOptions);
|
|
126
130
|
assert.deepEqual(seenUserIds, [user.id, undefined]);
|
|
127
131
|
});
|
|
128
132
|
});
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
-
import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
1
|
+
import {beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
2
|
|
|
3
3
|
const clientImpl = {
|
|
4
4
|
close: mock(async () => {}),
|
|
5
5
|
tools: mock(async () => ({search: {description: "Search"}})),
|
|
6
6
|
};
|
|
7
|
-
const createMCPClientMock = mock(async () => clientImpl);
|
|
7
|
+
const createMCPClientMock = mock(async (_opts: Record<string, unknown>) => clientImpl);
|
|
8
8
|
mock.module("@ai-sdk/mcp", () => ({
|
|
9
9
|
createMCPClient: (opts: Record<string, unknown>) => createMCPClientMock(opts),
|
|
10
10
|
}));
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
let MCPService: typeof import("./mcpService.js").MCPService;
|
|
13
|
+
beforeAll(async () => {
|
|
14
|
+
({MCPService} = await import("./mcpService.js"));
|
|
15
|
+
});
|
|
13
16
|
|
|
14
17
|
const makeConfig = (overrides?: Partial<Record<string, unknown>>) => ({
|
|
15
18
|
name: "test-server",
|
package/src/tests/bunSetup.ts
CHANGED
|
@@ -6,61 +6,7 @@ import {registerBackendPreload} from "@terreno/test";
|
|
|
6
6
|
registerBackendPreload({
|
|
7
7
|
connectMongoInBeforeAll: true,
|
|
8
8
|
mongo: {baseDatabaseName: "terreno-ai-test", useReplSet: true},
|
|
9
|
-
onBeforeEach: async () => {
|
|
10
|
-
const {shutdownLangfuseClient} = await import("../langfuseClient");
|
|
11
|
-
await shutdownLangfuseClient();
|
|
12
|
-
},
|
|
13
9
|
testEnv: {
|
|
14
10
|
tokenIssuer: "terreno-ai.test",
|
|
15
11
|
},
|
|
16
12
|
});
|
|
17
|
-
|
|
18
|
-
// Mock @langfuse/client globally so the real `./langfuseClient` module runs
|
|
19
|
-
// without making network calls.
|
|
20
|
-
import {mock} from "bun:test";
|
|
21
|
-
|
|
22
|
-
mock.module("@langfuse/client", () => {
|
|
23
|
-
return {
|
|
24
|
-
LangfuseClient: class FakeLangfuseClient {
|
|
25
|
-
baseUrl: string;
|
|
26
|
-
publicKey: string;
|
|
27
|
-
secretKey: string;
|
|
28
|
-
api = {
|
|
29
|
-
prompts: {
|
|
30
|
-
list: mock(async () => ({
|
|
31
|
-
data: [],
|
|
32
|
-
meta: {limit: 20, page: 1, total: 0, totalPages: 0},
|
|
33
|
-
})),
|
|
34
|
-
},
|
|
35
|
-
trace: {
|
|
36
|
-
get: mock(async (id: string) => ({id, name: "Trace"})),
|
|
37
|
-
list: mock(async () => ({
|
|
38
|
-
data: [],
|
|
39
|
-
meta: {limit: 20, page: 1, total: 0, totalPages: 0},
|
|
40
|
-
})),
|
|
41
|
-
},
|
|
42
|
-
};
|
|
43
|
-
prompt = {
|
|
44
|
-
create: mock(async (_params: Record<string, unknown>) => undefined),
|
|
45
|
-
get: mock(async (name: string) => ({
|
|
46
|
-
config: {},
|
|
47
|
-
labels: [],
|
|
48
|
-
name,
|
|
49
|
-
prompt: "",
|
|
50
|
-
tags: [],
|
|
51
|
-
type: "text" as const,
|
|
52
|
-
version: 1,
|
|
53
|
-
})),
|
|
54
|
-
};
|
|
55
|
-
score = {create: mock(() => {})};
|
|
56
|
-
flush = mock(async () => {});
|
|
57
|
-
shutdown = mock(async () => {});
|
|
58
|
-
|
|
59
|
-
constructor(opts: {baseUrl: string; publicKey: string; secretKey: string}) {
|
|
60
|
-
this.baseUrl = opts.baseUrl;
|
|
61
|
-
this.publicKey = opts.publicKey;
|
|
62
|
-
this.secretKey = opts.secretKey;
|
|
63
|
-
}
|
|
64
|
-
},
|
|
65
|
-
};
|
|
66
|
-
});
|
package/src/tests/chatHarness.ts
CHANGED
|
@@ -11,7 +11,7 @@ import {addGptRoutes} from "../routes/gpt";
|
|
|
11
11
|
import {addGptHistoryRoutes} from "../routes/gptHistories";
|
|
12
12
|
import {AIService} from "../service/aiService";
|
|
13
13
|
import type {GptHistoryDocument, GptRouteOptions} from "../types";
|
|
14
|
-
import {type authAsUser,
|
|
14
|
+
import {type authAsUser, historyRouterOptions, terrenoUserModel} from "./helpers";
|
|
15
15
|
|
|
16
16
|
type ModelStreamPart = {type: string; [key: string]: unknown};
|
|
17
17
|
|
|
@@ -232,7 +232,7 @@ export const createScriptedModel = ({
|
|
|
232
232
|
const remaining = [...steps];
|
|
233
233
|
let callCount = 0;
|
|
234
234
|
return {
|
|
235
|
-
doGenerate: mock(async () => ({
|
|
235
|
+
doGenerate: mock(async (_options: ModelCallOptions) => ({
|
|
236
236
|
content: [{text: "Workspace setup", type: "text" as const}],
|
|
237
237
|
finishReason: "stop" as const,
|
|
238
238
|
usage: USAGE,
|
|
@@ -348,11 +348,11 @@ export const buildApp = ({
|
|
|
348
348
|
openApiOptions: options,
|
|
349
349
|
...routeOptions,
|
|
350
350
|
};
|
|
351
|
-
addGptHistoryRoutes(router, {...options, chat});
|
|
351
|
+
addGptHistoryRoutes(router, {...historyRouterOptions(options), chat});
|
|
352
352
|
addGptRoutes(router, chat);
|
|
353
353
|
},
|
|
354
354
|
skipListen: true,
|
|
355
|
-
userModel:
|
|
355
|
+
userModel: terrenoUserModel,
|
|
356
356
|
}).build();
|
|
357
357
|
|
|
358
358
|
export interface ServedApp {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A stub for `AIService.generateJsonObject` that answers every call with `answer()`, whatever
|
|
3
|
+
* schema the caller asks for. The real method is generic over the parsed object's type.
|
|
4
|
+
*/
|
|
5
|
+
export const judgeAnswering =
|
|
6
|
+
(answer: () => Record<string, unknown>) =>
|
|
7
|
+
async <OBJECT>(): Promise<OBJECT> =>
|
|
8
|
+
answer() as OBJECT;
|
package/src/tests/helpers.ts
CHANGED
|
@@ -1,10 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
type UserModel as ApiUserModel,
|
|
3
|
+
createdUpdatedPlugin,
|
|
4
|
+
type ModelRouterOptions,
|
|
5
|
+
} from "@terreno/api";
|
|
2
6
|
import {authAsUser as authAsUserWithCredentials} from "@terreno/test";
|
|
3
7
|
import type express from "express";
|
|
4
|
-
import mongoose, {type Model} from "mongoose";
|
|
8
|
+
import mongoose, {type HydratedDocument, type Model} from "mongoose";
|
|
5
9
|
import passportLocalMongoose from "passport-local-mongoose";
|
|
6
10
|
import type TestAgent from "supertest/lib/agent";
|
|
7
11
|
|
|
12
|
+
import type {GptHistoryDocument} from "../types/index";
|
|
13
|
+
|
|
8
14
|
type PasswordedUser = {setPassword: (password: string) => Promise<void>};
|
|
9
15
|
|
|
10
16
|
export interface AiTestUser {
|
|
@@ -43,6 +49,22 @@ const createTestUserModel = (): Model<AiTestUserDoc> => {
|
|
|
43
49
|
|
|
44
50
|
export const UserModel = createTestUserModel();
|
|
45
51
|
|
|
52
|
+
/**
|
|
53
|
+
* `UserModel` as `TerrenoApp({userModel})` expects it. The passport-local-mongoose plugin adds
|
|
54
|
+
* the auth statics at runtime, and `@terreno/api`'s `User._id` uses mongoose's schema `ObjectId`
|
|
55
|
+
* type, which a hydrated document's `Types.ObjectId` does not satisfy.
|
|
56
|
+
*/
|
|
57
|
+
export const terrenoUserModel = UserModel as unknown as ApiUserModel;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* `TerrenoApp`'s `configureApp` options, typed as the GptHistory model router options tests
|
|
61
|
+
* spread them into. `configureApp` types them for no particular model.
|
|
62
|
+
*/
|
|
63
|
+
export const historyRouterOptions = (
|
|
64
|
+
options: Partial<ModelRouterOptions<unknown>> | undefined
|
|
65
|
+
): Partial<ModelRouterOptions<GptHistoryDocument>> =>
|
|
66
|
+
options as Partial<ModelRouterOptions<GptHistoryDocument>>;
|
|
67
|
+
|
|
46
68
|
const STANDARD_AI_TEST_USERS = {
|
|
47
69
|
admin: {admin: true, email: "admin@example.com", name: "Admin", password: "securePassword"},
|
|
48
70
|
notAdmin: {admin: false, email: "notAdmin@example.com", name: "User", password: "password"},
|
|
@@ -52,10 +74,10 @@ export type StandardAiTestUserRole = keyof typeof STANDARD_AI_TEST_USERS;
|
|
|
52
74
|
|
|
53
75
|
export const ensureTestUsers = async (
|
|
54
76
|
users: AiTestUser[] = [STANDARD_AI_TEST_USERS.admin, STANDARD_AI_TEST_USERS.notAdmin]
|
|
55
|
-
): Promise<AiTestUserDoc[]> => {
|
|
77
|
+
): Promise<HydratedDocument<AiTestUserDoc>[]> => {
|
|
56
78
|
await UserModel.deleteMany({email: {$in: users.map((user) => user.email)}});
|
|
57
79
|
|
|
58
|
-
const createdUsers: AiTestUserDoc[] = [];
|
|
80
|
+
const createdUsers: HydratedDocument<AiTestUserDoc>[] = [];
|
|
59
81
|
for (const user of users) {
|
|
60
82
|
const doc = await UserModel.create({
|
|
61
83
|
admin: user.admin ?? false,
|
package/src/types/harness.ts
CHANGED
|
@@ -8,9 +8,12 @@ import type {ObsPromptRef} from "./observability";
|
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
|
|
11
|
-
* from `promptRegistry.get` fits as-is; its `body` is ignored
|
|
11
|
+
* from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
|
|
12
|
+
* prompts it referenced) are recorded right after it.
|
|
12
13
|
*/
|
|
13
|
-
export
|
|
14
|
+
export interface HarnessPromptRef extends ObsPromptRef {
|
|
15
|
+
includes?: ObsPromptRef[];
|
|
16
|
+
}
|
|
14
17
|
|
|
15
18
|
/** A registry prompt passed as instructions: its `body` is used and its version recorded. */
|
|
16
19
|
export interface HarnessPromptInstructions extends HarnessPromptRef {
|
|
@@ -625,9 +628,13 @@ export interface HarnessToolDefinition<Args = unknown, Result = unknown>
|
|
|
625
628
|
|
|
626
629
|
/**
|
|
627
630
|
* Any tool, whatever its argument and result types. `execute` arguments are contravariant, so
|
|
628
|
-
* `never` accepts every tool in heterogeneous lists without an explicit `any`.
|
|
631
|
+
* `never` accepts every tool in heterogeneous lists without an explicit `any`. A Zod schema's
|
|
632
|
+
* output is covariant, so `parameters` widens to `unknown` instead.
|
|
629
633
|
*/
|
|
630
|
-
export
|
|
634
|
+
export interface AnyHarnessToolDefinition
|
|
635
|
+
extends Omit<HarnessToolDefinition<never, unknown>, "parameters"> {
|
|
636
|
+
parameters: z.ZodType<unknown>;
|
|
637
|
+
}
|
|
631
638
|
|
|
632
639
|
export interface HarnessAgentDefinitionInput {
|
|
633
640
|
/** Tried in order once the primary model's retryable failures exhaust `modelRetry`. */
|
package/src/types/index.ts
CHANGED
|
@@ -448,9 +448,14 @@ export interface GptRouteOptions {
|
|
|
448
448
|
* `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
|
|
449
449
|
*/
|
|
450
450
|
uiBlocks?: boolean | UiBlocksOptions;
|
|
451
|
-
/**
|
|
452
|
-
*
|
|
453
|
-
|
|
451
|
+
/**
|
|
452
|
+
* Observability prompt registry name used as the default system prompt (prepended to the
|
|
453
|
+
* request's system prompt). The trace records the version. Skipped when no registry is registered, the
|
|
454
|
+
* label has no version, or an admin request selects `promptName`.
|
|
455
|
+
*/
|
|
456
|
+
systemPromptName?: string;
|
|
457
|
+
/** Registry label read for `systemPromptName`. Defaults to `production`. */
|
|
458
|
+
systemPromptLabel?: string;
|
|
454
459
|
/**
|
|
455
460
|
* When `false` or the function returns `false`, prompts that include attachments are rejected.
|
|
456
461
|
* Omit or pass `true` to leave uploads enabled.
|