@terreno/ai 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +3 -3
- package/biome.jsonc +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +42 -8
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +10 -4
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -28
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +30 -30
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/commit.ts +24 -4
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +6 -8
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +29 -4
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -20
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +25 -28
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +497 -17
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +295 -3
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +39 -2
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +8 -8
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +63 -2
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +156 -115
- package/src/service/aiService.test.ts +22 -1
- package/src/service/aiService.ts +4 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +42 -33
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +0 -54
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +11 -4
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/tsconfig.typecheck.json +18 -0
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -60
- package/src/langfuseApp.test.ts +0 -135
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -256
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -68
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -79
- package/src/langfuseVercelAi.ts +0 -72
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.addObservabilityTraceRoutes = void 0;
|
|
4
4
|
const api_1 = require("@terreno/api");
|
|
5
|
+
const traceStore_1 = require("../local/traceStore");
|
|
5
6
|
const observabilityAppRegistry_1 = require("../observabilityAppRegistry");
|
|
6
7
|
const observabilityRouteAccess_1 = require("./observabilityRouteAccess");
|
|
7
8
|
const BASE_PATH = "/ai/observability";
|
|
@@ -25,11 +26,34 @@ const parsePromptVersion = (value, prompt) => {
|
|
|
25
26
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
26
27
|
throw new api_1.APIError({ status: 400, title: "promptVersion must be a positive integer" });
|
|
27
28
|
}
|
|
28
|
-
if (!prompt) {
|
|
29
|
+
if (!prompt || prompt.length === 0) {
|
|
29
30
|
throw new api_1.APIError({ status: 400, title: "prompt is required when promptVersion is set" });
|
|
30
31
|
}
|
|
31
32
|
return parsed;
|
|
32
33
|
};
|
|
34
|
+
const parsePrompt = (value) => {
|
|
35
|
+
if (typeof value === "string") {
|
|
36
|
+
return value;
|
|
37
|
+
}
|
|
38
|
+
if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
|
|
39
|
+
return value;
|
|
40
|
+
}
|
|
41
|
+
if (value === undefined) {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
throw new api_1.APIError({ status: 400, title: "prompt must be a string or a list of strings" });
|
|
45
|
+
};
|
|
46
|
+
const parseSort = (value) => {
|
|
47
|
+
if (value === undefined) {
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
if (typeof value !== "string") {
|
|
51
|
+
throw new api_1.APIError({ status: 400, title: "Unknown trace sort key" });
|
|
52
|
+
}
|
|
53
|
+
// Validate here so a bad key fails before the store runs any query.
|
|
54
|
+
(0, traceStore_1.parseTraceListSort)(value);
|
|
55
|
+
return value;
|
|
56
|
+
};
|
|
33
57
|
const parsePositiveInt = (value, fallback) => {
|
|
34
58
|
if (value === undefined) {
|
|
35
59
|
return fallback;
|
|
@@ -51,9 +75,16 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
51
75
|
.withQueryParameter("prompt", { type: "string" }, { required: false })
|
|
52
76
|
.withQueryParameter("promptVersion", { type: "number" }, { required: false })
|
|
53
77
|
.withQueryParameter("scope", { type: "string" }, { required: false })
|
|
78
|
+
.withQueryParameter("sort", {
|
|
79
|
+
enum: traceStore_1.TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
|
|
80
|
+
type: "string",
|
|
81
|
+
}, {
|
|
82
|
+
description: `Sort column, \`-\` prefix for descending. Defaults to \`${traceStore_1.DEFAULT_TRACE_LIST_SORT}\`.`,
|
|
83
|
+
required: false,
|
|
84
|
+
})
|
|
54
85
|
.withResponse(200, { data: { type: "array" } })
|
|
55
86
|
.build()), (0, api_1.asyncHandler)(async (req, res) => {
|
|
56
|
-
const prompt =
|
|
87
|
+
const prompt = parsePrompt(req.query.prompt);
|
|
57
88
|
const listed = await options.store.list({
|
|
58
89
|
flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
|
|
59
90
|
from: typeof req.query.from === "string" ? req.query.from : undefined,
|
|
@@ -65,6 +96,7 @@ const addObservabilityTraceRoutes = (router, options) => {
|
|
|
65
96
|
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
66
97
|
sensitive: parseBoolean(req.query.sensitive),
|
|
67
98
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
99
|
+
sort: parseSort(req.query.sort),
|
|
68
100
|
status: req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
|
|
69
101
|
to: typeof req.query.to === "string" ? req.query.to : undefined,
|
|
70
102
|
userId: typeof req.query.userId === "string" ? req.query.userId : undefined,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type AnyTerrenoAccess } from "@terreno/api";
|
|
2
2
|
export type ObservabilityCapability = "datasets" | "experiments" | "prompts" | "reviewQueue" | "scores" | "traces";
|
|
3
|
-
export type ControlPrimary = "
|
|
3
|
+
export type ControlPrimary = "local";
|
|
4
4
|
export interface ObservabilityControlConfig {
|
|
5
5
|
datasets: ControlPrimary;
|
|
6
6
|
experiments: ControlPrimary;
|
|
@@ -39,6 +39,7 @@ export interface TraceRecord {
|
|
|
39
39
|
name: string;
|
|
40
40
|
output?: unknown;
|
|
41
41
|
prompts: {
|
|
42
|
+
includedBy?: string;
|
|
42
43
|
label?: string;
|
|
43
44
|
name: string;
|
|
44
45
|
version: number;
|
|
@@ -68,7 +69,15 @@ export interface ScoreRecord {
|
|
|
68
69
|
value: boolean | number | string;
|
|
69
70
|
}
|
|
70
71
|
export interface PromptVersionRef {
|
|
72
|
+
/** `body` with every prompt reference resolved. */
|
|
71
73
|
body: string;
|
|
74
|
+
/** Prompt versions pulled in by references, recorded on the trace after this prompt. */
|
|
75
|
+
includes?: {
|
|
76
|
+
includedBy?: string;
|
|
77
|
+
label?: string;
|
|
78
|
+
name: string;
|
|
79
|
+
version: number;
|
|
80
|
+
}[];
|
|
72
81
|
label?: string;
|
|
73
82
|
name: string;
|
|
74
83
|
sensitive?: boolean;
|
|
@@ -267,7 +267,10 @@ class AIService {
|
|
|
267
267
|
name: spanName,
|
|
268
268
|
output: params.response,
|
|
269
269
|
prompts: promptRef
|
|
270
|
-
? [
|
|
270
|
+
? [
|
|
271
|
+
{ label: promptRef.label, name: promptRef.name, version: promptRef.version },
|
|
272
|
+
...(promptRef.includes ?? []),
|
|
273
|
+
]
|
|
271
274
|
: [],
|
|
272
275
|
sensitive: params.observability.sensitive,
|
|
273
276
|
sessionId: params.observability.sessionId,
|
package/dist/service/chatTurn.js
CHANGED
|
@@ -6,11 +6,11 @@ const api_1 = require("@terreno/api");
|
|
|
6
6
|
const blocks_1 = require("@terreno/blocks");
|
|
7
7
|
const ai_1 = require("ai");
|
|
8
8
|
const luxon_1 = require("luxon");
|
|
9
|
-
const langfuseVercelAi_1 = require("../langfuseVercelAi");
|
|
10
9
|
const aiRequest_1 = require("../models/aiRequest");
|
|
11
10
|
const fileAttachment_1 = require("../models/fileAttachment");
|
|
12
11
|
const gptHistory_1 = require("../models/gptHistory");
|
|
13
12
|
const project_1 = require("../models/project");
|
|
13
|
+
const observabilityAppRegistry_1 = require("../observability/observabilityAppRegistry");
|
|
14
14
|
const aiService_1 = require("./aiService");
|
|
15
15
|
const askFiles_1 = require("./askFiles");
|
|
16
16
|
const asks_1 = require("./asks");
|
|
@@ -691,8 +691,28 @@ const startTurn = async ({ aiService, askAnswer, body, options, req, userId, })
|
|
|
691
691
|
};
|
|
692
692
|
};
|
|
693
693
|
const lastUserText = (prompts) => prompts.findLast((prompt) => prompt.type === "user")?.text ?? "";
|
|
694
|
-
/** The
|
|
695
|
-
const
|
|
694
|
+
/** The route's default registry system prompt, or undefined when it cannot be loaded. */
|
|
695
|
+
const loadRegistrySystemPrompt = async (options) => {
|
|
696
|
+
const registry = (0, observabilityAppRegistry_1.getObservabilityApp)()?.promptRegistry;
|
|
697
|
+
if (!options.systemPromptName || !registry) {
|
|
698
|
+
return undefined;
|
|
699
|
+
}
|
|
700
|
+
try {
|
|
701
|
+
return await registry.get({
|
|
702
|
+
label: options.systemPromptLabel ?? "production",
|
|
703
|
+
name: options.systemPromptName,
|
|
704
|
+
});
|
|
705
|
+
}
|
|
706
|
+
catch (error) {
|
|
707
|
+
api_1.logger.warn("Registry system prompt skipped", {
|
|
708
|
+
error: error instanceof Error ? error.message : String(error),
|
|
709
|
+
promptName: options.systemPromptName,
|
|
710
|
+
});
|
|
711
|
+
return undefined;
|
|
712
|
+
}
|
|
713
|
+
};
|
|
714
|
+
/** The request's system prompt with project context and the registry prompt prepended. */
|
|
715
|
+
const buildSystemPrompt = async ({ history, projectId, registryPrompt, systemPrompt, userId, }) => {
|
|
696
716
|
const effectiveProjectId = projectId ?? history.projectId;
|
|
697
717
|
let effectiveSystemPrompt = systemPrompt;
|
|
698
718
|
if (effectiveProjectId) {
|
|
@@ -718,22 +738,10 @@ const buildSystemPrompt = async ({ history, options, projectId, systemPrompt, us
|
|
|
718
738
|
// Project loading failure should not block the request
|
|
719
739
|
}
|
|
720
740
|
}
|
|
721
|
-
if (
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
userId: userId?.toString(),
|
|
726
|
-
});
|
|
727
|
-
const langfusePrompt = typeof langfuseResult.prompt === "string" ? langfuseResult.prompt : undefined;
|
|
728
|
-
if (langfusePrompt) {
|
|
729
|
-
effectiveSystemPrompt = effectiveSystemPrompt
|
|
730
|
-
? `${langfusePrompt}\n\n${effectiveSystemPrompt}`
|
|
731
|
-
: langfusePrompt;
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
catch (err) {
|
|
735
|
-
api_1.logger.debug(`Langfuse system prompt skipped: ${err.message}`);
|
|
736
|
-
}
|
|
741
|
+
if (registryPrompt) {
|
|
742
|
+
effectiveSystemPrompt = effectiveSystemPrompt
|
|
743
|
+
? `${registryPrompt}\n\n${effectiveSystemPrompt}`
|
|
744
|
+
: registryPrompt;
|
|
737
745
|
}
|
|
738
746
|
return effectiveSystemPrompt;
|
|
739
747
|
};
|
|
@@ -1192,20 +1200,29 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
|
|
|
1192
1200
|
const turn = await startTurn({ aiService, askAnswer, body, options, req, userId });
|
|
1193
1201
|
const { answer, isNewHistory, logPrompt, messages, replayedMessages, resolvedAsk, titlePrompt } = turn;
|
|
1194
1202
|
let { history } = turn;
|
|
1203
|
+
// An admin-selected promptName replaces the system prompt, so the route default is skipped.
|
|
1204
|
+
const registryPrompt = promptName ? undefined : await loadRegistrySystemPrompt(options);
|
|
1195
1205
|
let effectiveSystemPrompt = await buildSystemPrompt({
|
|
1196
1206
|
history,
|
|
1197
|
-
options,
|
|
1198
1207
|
projectId,
|
|
1208
|
+
registryPrompt: registryPrompt?.body,
|
|
1199
1209
|
systemPrompt,
|
|
1200
1210
|
userId,
|
|
1201
1211
|
});
|
|
1202
|
-
const
|
|
1212
|
+
const resolvedObservability = await aiService.resolveGenerateObservability({
|
|
1203
1213
|
promptLabel,
|
|
1204
1214
|
promptName,
|
|
1205
1215
|
sensitive: body.sensitive === true ? true : undefined,
|
|
1206
1216
|
sessionId,
|
|
1207
1217
|
systemPrompt: effectiveSystemPrompt,
|
|
1208
1218
|
});
|
|
1219
|
+
const observability = registryPrompt
|
|
1220
|
+
? {
|
|
1221
|
+
...resolvedObservability,
|
|
1222
|
+
promptRef: registryPrompt,
|
|
1223
|
+
sensitive: resolvedObservability.sensitive || Boolean(registryPrompt.sensitive),
|
|
1224
|
+
}
|
|
1225
|
+
: resolvedObservability;
|
|
1209
1226
|
effectiveSystemPrompt = observability.systemPrompt;
|
|
1210
1227
|
// Some models (e.g. gemini-2.5-flash-image) don't support tool calling
|
|
1211
1228
|
const modelId = aiService.modelId;
|
|
@@ -1435,17 +1452,7 @@ const runChatTurn = async ({ body, options, req, sink, }) => {
|
|
|
1435
1452
|
sink.emit({ askResolved: { action: resolvedAsk.action, toolCallId: resolvedAsk.toolCallId } });
|
|
1436
1453
|
}
|
|
1437
1454
|
api_1.logger.debug("Starting streamText", { model: modelId, supportsTools });
|
|
1438
|
-
const telemetry = (0, langfuseVercelAi_1.createTelemetryConfig)({
|
|
1439
|
-
functionId: "gpt-prompt",
|
|
1440
|
-
metadata: {
|
|
1441
|
-
...(options.langfuseSystemPromptName
|
|
1442
|
-
? { langfusePromptName: options.langfuseSystemPromptName }
|
|
1443
|
-
: {}),
|
|
1444
|
-
},
|
|
1445
|
-
userId: userId?.toString(),
|
|
1446
|
-
});
|
|
1447
1455
|
const result = (0, ai_1.streamText)({
|
|
1448
|
-
experimental_telemetry: telemetry,
|
|
1449
1456
|
messages,
|
|
1450
1457
|
model: aiService.model,
|
|
1451
1458
|
providerOptions: !supportsTools
|
package/dist/types/harness.d.ts
CHANGED
|
@@ -6,9 +6,12 @@ import type { ExecutionEnv } from "../harness/executionEnv";
|
|
|
6
6
|
import type { ObsPromptRef } from "./observability";
|
|
7
7
|
/**
|
|
8
8
|
* A prompt version a task used, recorded on its `ObsTrace.prompts`. A `PromptVersionRef`
|
|
9
|
-
* from `promptRegistry.get` fits as-is; its `body` is ignored
|
|
9
|
+
* from `promptRegistry.get` fits as-is; its `body` is ignored and its `includes` (shared
|
|
10
|
+
* prompts it referenced) are recorded right after it.
|
|
10
11
|
*/
|
|
11
|
-
export
|
|
12
|
+
export interface HarnessPromptRef extends ObsPromptRef {
|
|
13
|
+
includes?: ObsPromptRef[];
|
|
14
|
+
}
|
|
12
15
|
/** A registry prompt passed as instructions: its `body` is used and its version recorded. */
|
|
13
16
|
export interface HarnessPromptInstructions extends HarnessPromptRef {
|
|
14
17
|
body: string;
|
|
@@ -536,9 +539,12 @@ export interface HarnessToolDefinition<Args = unknown, Result = unknown> extends
|
|
|
536
539
|
}
|
|
537
540
|
/**
|
|
538
541
|
* Any tool, whatever its argument and result types. `execute` arguments are contravariant, so
|
|
539
|
-
* `never` accepts every tool in heterogeneous lists without an explicit `any`.
|
|
542
|
+
* `never` accepts every tool in heterogeneous lists without an explicit `any`. A Zod schema's
|
|
543
|
+
* output is covariant, so `parameters` widens to `unknown` instead.
|
|
540
544
|
*/
|
|
541
|
-
export
|
|
545
|
+
export interface AnyHarnessToolDefinition extends Omit<HarnessToolDefinition<never, unknown>, "parameters"> {
|
|
546
|
+
parameters: z.ZodType<unknown>;
|
|
547
|
+
}
|
|
542
548
|
export interface HarnessAgentDefinitionInput {
|
|
543
549
|
/** Tried in order once the primary model's retryable failures exhaust `modelRetry`. */
|
|
544
550
|
fallbackModels?: HarnessModelRef[];
|
package/dist/types/index.d.ts
CHANGED
|
@@ -384,9 +384,14 @@ export interface GptRouteOptions {
|
|
|
384
384
|
* `POST /gpt/actions`. `{repair: true}` runs one repair call when that check fails.
|
|
385
385
|
*/
|
|
386
386
|
uiBlocks?: boolean | UiBlocksOptions;
|
|
387
|
-
/**
|
|
388
|
-
*
|
|
389
|
-
|
|
387
|
+
/**
|
|
388
|
+
* Observability prompt registry name used as the default system prompt (prepended to the
|
|
389
|
+
* request's system prompt). The trace records the version. Skipped when no registry is registered, the
|
|
390
|
+
* label has no version, or an admin request selects `promptName`.
|
|
391
|
+
*/
|
|
392
|
+
systemPromptName?: string;
|
|
393
|
+
/** Registry label read for `systemPromptName`. Defaults to `production`. */
|
|
394
|
+
systemPromptLabel?: string;
|
|
390
395
|
/**
|
|
391
396
|
* When `false` or the function returns `false`, prompts that include attachments are rejected.
|
|
392
397
|
* Omit or pass `true` to leave uploads enabled.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { FindExactlyOnePlugin, FindOneOrNonePlugin } from "@terreno/api";
|
|
2
2
|
import type mongoose from "mongoose";
|
|
3
|
+
import type { JudgeAgreement } from "../observability/judgeAgreement";
|
|
3
4
|
export interface ObsPromptVariable {
|
|
4
5
|
key: string;
|
|
5
6
|
label?: string;
|
|
@@ -13,6 +14,8 @@ export interface ObsPromptUsage {
|
|
|
13
14
|
outputTokens?: number;
|
|
14
15
|
}
|
|
15
16
|
export interface ObsPromptRef {
|
|
17
|
+
/** Set when this version was pulled in by another prompt's reference, to that prompt's name. */
|
|
18
|
+
includedBy?: string;
|
|
16
19
|
label?: string;
|
|
17
20
|
name: string;
|
|
18
21
|
version: number;
|
|
@@ -131,6 +134,8 @@ export interface ObsScoreModel extends mongoose.Model<ObsScoreDocument>, ObsScor
|
|
|
131
134
|
export interface EvaluatorDimension {
|
|
132
135
|
dataType: "boolean" | "categorical" | "numeric";
|
|
133
136
|
key: string;
|
|
137
|
+
/** Question shown to human reviewers, e.g. "Does the summary mention every todo?" */
|
|
138
|
+
question?: string;
|
|
134
139
|
range?: string;
|
|
135
140
|
required: boolean;
|
|
136
141
|
}
|
|
@@ -163,6 +168,7 @@ export interface ObsEvaluatorModel extends mongoose.Model<ObsEvaluatorDocument>,
|
|
|
163
168
|
}
|
|
164
169
|
export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
165
170
|
assigneeId?: mongoose.Types.ObjectId;
|
|
171
|
+
batchId?: mongoose.Types.ObjectId;
|
|
166
172
|
comment?: string;
|
|
167
173
|
created: Date;
|
|
168
174
|
datasetItemId?: mongoose.Types.ObjectId;
|
|
@@ -173,13 +179,53 @@ export interface ObsReviewItemDocument extends mongoose.Document<mongoose.Types.
|
|
|
173
179
|
scores?: Record<string, boolean | number | string>;
|
|
174
180
|
spanId?: mongoose.Types.ObjectId;
|
|
175
181
|
status: "done" | "in_progress" | "pending" | "skipped";
|
|
176
|
-
|
|
182
|
+
/** Trace under review. Unset when reviewers judge a dataset item that has no source trace. */
|
|
183
|
+
traceId?: mongoose.Types.ObjectId;
|
|
177
184
|
updated: Date;
|
|
178
185
|
}
|
|
179
186
|
export interface ObsReviewItemStatics extends FindExactlyOnePlugin<ObsReviewItemDocument>, FindOneOrNonePlugin<ObsReviewItemDocument> {
|
|
180
187
|
}
|
|
181
188
|
export interface ObsReviewItemModel extends mongoose.Model<ObsReviewItemDocument>, ObsReviewItemStatics {
|
|
182
189
|
}
|
|
190
|
+
export interface ObsReviewBatchDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
191
|
+
created: Date;
|
|
192
|
+
/** Dataset the batch was built from, when reviewers judge dataset items. */
|
|
193
|
+
datasetId?: mongoose.Types.ObjectId;
|
|
194
|
+
deleted: boolean;
|
|
195
|
+
description?: string;
|
|
196
|
+
evaluatorId: mongoose.Types.ObjectId;
|
|
197
|
+
name: string;
|
|
198
|
+
promptName?: string;
|
|
199
|
+
/** Question-set fingerprint when the batch was created (see `questionSetFingerprint`). */
|
|
200
|
+
questionsFingerprint?: string;
|
|
201
|
+
traceIds: mongoose.Types.ObjectId[];
|
|
202
|
+
updated: Date;
|
|
203
|
+
}
|
|
204
|
+
export interface ObsReviewBatchStatics extends FindExactlyOnePlugin<ObsReviewBatchDocument>, FindOneOrNonePlugin<ObsReviewBatchDocument> {
|
|
205
|
+
}
|
|
206
|
+
export interface ObsReviewBatchModel extends mongoose.Model<ObsReviewBatchDocument>, ObsReviewBatchStatics {
|
|
207
|
+
}
|
|
208
|
+
export interface ObsJudgeRunResult {
|
|
209
|
+
confidence?: number;
|
|
210
|
+
datasetItemId?: mongoose.Types.ObjectId;
|
|
211
|
+
error?: string;
|
|
212
|
+
scores?: Record<string, boolean | number | string>;
|
|
213
|
+
traceId?: mongoose.Types.ObjectId;
|
|
214
|
+
}
|
|
215
|
+
export interface ObsJudgeRunDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
216
|
+
batchId: mongoose.Types.ObjectId;
|
|
217
|
+
completedAt: Date;
|
|
218
|
+
created: Date;
|
|
219
|
+
deleted: boolean;
|
|
220
|
+
evaluatorId: mongoose.Types.ObjectId;
|
|
221
|
+
modelId?: string;
|
|
222
|
+
results: ObsJudgeRunResult[];
|
|
223
|
+
updated: Date;
|
|
224
|
+
}
|
|
225
|
+
export interface ObsJudgeRunStatics extends FindExactlyOnePlugin<ObsJudgeRunDocument>, FindOneOrNonePlugin<ObsJudgeRunDocument> {
|
|
226
|
+
}
|
|
227
|
+
export interface ObsJudgeRunModel extends mongoose.Model<ObsJudgeRunDocument>, ObsJudgeRunStatics {
|
|
228
|
+
}
|
|
183
229
|
export interface ObsDatasetAnnotatedBy {
|
|
184
230
|
label: string;
|
|
185
231
|
reviewItemId?: string;
|
|
@@ -199,12 +245,23 @@ export interface ObsDatasetStatics extends FindExactlyOnePlugin<ObsDatasetDocume
|
|
|
199
245
|
}
|
|
200
246
|
export interface ObsDatasetModel extends mongoose.Model<ObsDatasetDocument>, ObsDatasetStatics {
|
|
201
247
|
}
|
|
248
|
+
/** One question set's human answers for a dataset item. */
|
|
249
|
+
export interface ObsDatasetHumanAnswer {
|
|
250
|
+
answeredAt: Date;
|
|
251
|
+
/** `questionSetFingerprint` of the question set when the reviewer answered. */
|
|
252
|
+
fingerprint: string;
|
|
253
|
+
reviewItemId: string;
|
|
254
|
+
scores: Record<string, boolean | number | string>;
|
|
255
|
+
userId?: string;
|
|
256
|
+
}
|
|
202
257
|
export interface ObsDatasetItemDocument extends mongoose.Document<mongoose.Types.ObjectId> {
|
|
203
258
|
annotatedBy?: ObsDatasetAnnotatedBy;
|
|
204
259
|
created: Date;
|
|
205
260
|
datasetId: mongoose.Types.ObjectId;
|
|
206
261
|
deleted: boolean;
|
|
207
262
|
expectedOutput?: unknown;
|
|
263
|
+
/** Human answers keyed by question-set (human evaluator) id. */
|
|
264
|
+
humanAnswers?: Record<string, ObsDatasetHumanAnswer>;
|
|
208
265
|
input: unknown;
|
|
209
266
|
metadata?: Record<string, unknown>;
|
|
210
267
|
origin: "manual" | "synthetic" | "trace";
|
|
@@ -219,7 +276,8 @@ export interface ObsDatasetItemStatics extends FindExactlyOnePlugin<ObsDatasetIt
|
|
|
219
276
|
export interface ObsDatasetItemModel extends mongoose.Model<ObsDatasetItemDocument>, ObsDatasetItemStatics {
|
|
220
277
|
}
|
|
221
278
|
export interface ScoreThreshold {
|
|
222
|
-
|
|
279
|
+
/** `agreement` gates compare a judge prompt version's answers with human answers. */
|
|
280
|
+
aggregate: "agreement" | "mean" | "trueRate";
|
|
223
281
|
dimension: string;
|
|
224
282
|
evaluatorName: string;
|
|
225
283
|
op: "eq" | "gte" | "lte";
|
|
@@ -235,7 +293,10 @@ export interface ExperimentEstimate {
|
|
|
235
293
|
generations: number;
|
|
236
294
|
wallClockSeconds: number;
|
|
237
295
|
}
|
|
296
|
+
export type ExperimentKind = "judge" | "prompt";
|
|
238
297
|
export interface ExperimentAggregates {
|
|
298
|
+
/** Judge experiments: agreement with the human answers per judge prompt version. */
|
|
299
|
+
agreement?: Record<string, JudgeAgreement>;
|
|
239
300
|
gates: ExperimentGateResult[];
|
|
240
301
|
lowConfidenceItemIds: string[];
|
|
241
302
|
outlierItemIds: string[];
|
|
@@ -253,11 +314,20 @@ export interface ObsExperimentDocument extends mongoose.Document<mongoose.Types.
|
|
|
253
314
|
estimate?: ExperimentEstimate;
|
|
254
315
|
evaluatorIds: mongoose.Types.ObjectId[];
|
|
255
316
|
includeUnproofread: boolean;
|
|
317
|
+
/** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
|
|
318
|
+
judgeEvaluatorId?: mongoose.Types.ObjectId;
|
|
319
|
+
kind: ExperimentKind;
|
|
256
320
|
modelOverride?: string;
|
|
257
321
|
name: string;
|
|
258
322
|
promptName: string;
|
|
323
|
+
/** Judge experiments: question-set fingerprint of the human answers compared. */
|
|
324
|
+
questionsFingerprint?: string;
|
|
325
|
+
/** Judge experiments: the human question set whose answers are the reference. */
|
|
326
|
+
questionSetId?: mongoose.Types.ObjectId;
|
|
259
327
|
results?: ExperimentAggregates;
|
|
260
328
|
status: "completed" | "failed" | "pending" | "running";
|
|
329
|
+
/** Judge experiments: agreement rate (0–1) a version must reach to pass. */
|
|
330
|
+
targetAgreement?: number;
|
|
261
331
|
thresholds: ScoreThreshold[];
|
|
262
332
|
updated: Date;
|
|
263
333
|
versions: number[];
|
|
@@ -282,6 +352,8 @@ export interface ObsExperimentItemDocument extends mongoose.Document<mongoose.Ty
|
|
|
282
352
|
deleted: boolean;
|
|
283
353
|
experimentId: mongoose.Types.ObjectId;
|
|
284
354
|
failed: boolean;
|
|
355
|
+
/** Judge experiments: the human answers the judge versions are compared with. */
|
|
356
|
+
humanScores?: Record<string, boolean | number | string>;
|
|
285
357
|
updated: Date;
|
|
286
358
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
287
359
|
}
|
package/e2e/smoke.ts
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Real-process smoke for @terreno/ai. Run `bun e2e/smoke.ts` after `bun run compile`.
|
|
3
|
+
*
|
|
4
|
+
* The smoke re-runs this file with `serve` as a child process: a minimal TerrenoApp from the
|
|
5
|
+
* compiled `@terreno/api` with the compiled `AiApp` registered, on a free port, against
|
|
6
|
+
* `smokeMongoUri()`. The child's `AIService` wraps a deterministic fake provider
|
|
7
|
+
* (`MockLanguageModelV3` from `ai/test`) and runs with every provider key unset. The parent then
|
|
8
|
+
* checks over real HTTP that a signed-in user's chat request streams back the fake provider's
|
|
9
|
+
* text. Exits non-zero on the first failed check.
|
|
10
|
+
*/
|
|
11
|
+
import {
|
|
12
|
+
freePort,
|
|
13
|
+
type SpawnedProcess,
|
|
14
|
+
smokeMongoUri,
|
|
15
|
+
spawnUntilReady,
|
|
16
|
+
stopSmokeMongo,
|
|
17
|
+
} from "@terreno/test";
|
|
18
|
+
import type express from "express";
|
|
19
|
+
import {DateTime} from "luxon";
|
|
20
|
+
import type {Schema} from "mongoose";
|
|
21
|
+
|
|
22
|
+
const SERVE_ARG = "serve";
|
|
23
|
+
const FAKE_REPLY = "Hello from the ai smoke's fake provider.";
|
|
24
|
+
const FAKE_TITLE = "Smoke chat";
|
|
25
|
+
|
|
26
|
+
/** Provider credentials the smoke unsets in the child, so no real provider can answer. */
|
|
27
|
+
const PROVIDER_ENV_KEYS = [
|
|
28
|
+
"ANTHROPIC_API_KEY",
|
|
29
|
+
"GEMINI_API_KEY",
|
|
30
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
31
|
+
"GOOGLE_GENERATIVE_AI_API_KEY",
|
|
32
|
+
"GOOGLE_VERTEX_PROJECT",
|
|
33
|
+
"LANGFUSE_PUBLIC_KEY",
|
|
34
|
+
"LANGFUSE_SECRET_KEY",
|
|
35
|
+
"OPENAI_API_KEY",
|
|
36
|
+
];
|
|
37
|
+
|
|
38
|
+
interface ChatEvent {
|
|
39
|
+
done?: boolean;
|
|
40
|
+
error?: string;
|
|
41
|
+
historyId?: string;
|
|
42
|
+
text?: string;
|
|
43
|
+
title?: string;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const ZERO_USAGE = {
|
|
47
|
+
inputTokens: {cacheRead: 0, cacheWrite: 0, noCache: 0, total: 0},
|
|
48
|
+
outputTokens: {reasoning: 0, text: 0, total: 0},
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Child: the compiled API with auth, the compiled AiApp on a fake model, and a health plugin.
|
|
53
|
+
* The packages are imported here, not at the top, so the parent process never loads them.
|
|
54
|
+
*/
|
|
55
|
+
const serve = async (): Promise<void> => {
|
|
56
|
+
const {createdUpdatedPlugin, logger, TerrenoApp} = await import("@terreno/api");
|
|
57
|
+
const {AIService, AiApp} = await import("@terreno/ai");
|
|
58
|
+
const {MockLanguageModelV3} = await import("ai/test");
|
|
59
|
+
const {default: mongoose} = await import("mongoose");
|
|
60
|
+
const {default: passportLocalMongoose} = await import("passport-local-mongoose");
|
|
61
|
+
|
|
62
|
+
const userSchema = new mongoose.Schema({
|
|
63
|
+
admin: {default: false, description: "Whether the user is an admin", type: Boolean},
|
|
64
|
+
email: {description: "Sign-in email", type: String},
|
|
65
|
+
name: {description: "Display name", type: String},
|
|
66
|
+
});
|
|
67
|
+
userSchema.plugin(passportLocalMongoose as unknown as (schema: Schema, options: object) => void, {
|
|
68
|
+
usernameCaseInsensitive: true,
|
|
69
|
+
usernameField: "email",
|
|
70
|
+
});
|
|
71
|
+
userSchema.plugin(createdUpdatedPlugin);
|
|
72
|
+
const User = mongoose.model("User", userSchema);
|
|
73
|
+
|
|
74
|
+
// Streams the chat reply and answers the title request, the two calls one chat turn makes.
|
|
75
|
+
const model = new MockLanguageModelV3({
|
|
76
|
+
doGenerate: async () => ({
|
|
77
|
+
content: [{text: FAKE_TITLE, type: "text"}],
|
|
78
|
+
finishReason: {raw: "stop", unified: "stop"},
|
|
79
|
+
usage: ZERO_USAGE,
|
|
80
|
+
warnings: [],
|
|
81
|
+
}),
|
|
82
|
+
doStream: async () => ({
|
|
83
|
+
stream: new ReadableStream({
|
|
84
|
+
start(controller): void {
|
|
85
|
+
controller.enqueue({type: "stream-start", warnings: []});
|
|
86
|
+
controller.enqueue({id: "text-1", type: "text-start"});
|
|
87
|
+
controller.enqueue({delta: FAKE_REPLY, id: "text-1", type: "text-delta"});
|
|
88
|
+
controller.enqueue({id: "text-1", type: "text-end"});
|
|
89
|
+
controller.enqueue({
|
|
90
|
+
finishReason: {raw: "stop", unified: "stop"},
|
|
91
|
+
type: "finish",
|
|
92
|
+
usage: ZERO_USAGE,
|
|
93
|
+
});
|
|
94
|
+
controller.close();
|
|
95
|
+
},
|
|
96
|
+
}),
|
|
97
|
+
}),
|
|
98
|
+
modelId: "ai-smoke-fake-model",
|
|
99
|
+
provider: "ai-smoke-fake-provider",
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const healthPlugin = {
|
|
103
|
+
register: (app: express.Application): void => {
|
|
104
|
+
app.get("/health", (_req, res) => {
|
|
105
|
+
res.json({healthy: mongoose.connection.readyState === 1});
|
|
106
|
+
});
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
await mongoose.connect(process.env.MONGO_URI as string);
|
|
111
|
+
const app = new TerrenoApp({
|
|
112
|
+
loggingOptions: {logRequests: false},
|
|
113
|
+
userModel: User as unknown as ConstructorParameters<typeof TerrenoApp>[0]["userModel"],
|
|
114
|
+
})
|
|
115
|
+
.register(new AiApp({aiService: new AIService({model})}))
|
|
116
|
+
.register(healthPlugin);
|
|
117
|
+
app.start();
|
|
118
|
+
await app.whenReady();
|
|
119
|
+
logger.info(`[aiSmoke] serving on port ${process.env.PORT}`);
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
const expectStatus = async (res: Response, status: number, step: string): Promise<string> => {
|
|
123
|
+
const body = await res.text();
|
|
124
|
+
if (res.status !== status) {
|
|
125
|
+
throw new Error(`${step}: expected ${status}, got ${res.status}: ${body}`);
|
|
126
|
+
}
|
|
127
|
+
return body;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/** Splits a finished `text/event-stream` body into its JSON `data:` events. */
|
|
131
|
+
const parseEvents = (body: string): ChatEvent[] =>
|
|
132
|
+
body
|
|
133
|
+
.split("\n\n")
|
|
134
|
+
.map((chunk) => chunk.trim())
|
|
135
|
+
.filter((chunk) => chunk.startsWith("data: "))
|
|
136
|
+
.map((chunk) => JSON.parse(chunk.slice("data: ".length)) as ChatEvent);
|
|
137
|
+
|
|
138
|
+
/** Parent: boots the child and sends one chat request over HTTP. */
|
|
139
|
+
const run = async (): Promise<void> => {
|
|
140
|
+
const startedAt = DateTime.now();
|
|
141
|
+
const port = await freePort();
|
|
142
|
+
const baseUrl = `http://127.0.0.1:${port}`;
|
|
143
|
+
let server: SpawnedProcess | undefined;
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
server = await spawnUntilReady({
|
|
147
|
+
cmd: [process.execPath, __filename, SERVE_ARG],
|
|
148
|
+
env: {
|
|
149
|
+
...Object.fromEntries(PROVIDER_ENV_KEYS.map((key) => [key, undefined])),
|
|
150
|
+
MONGO_URI: await smokeMongoUri({databaseName: "aiSmoke"}),
|
|
151
|
+
PORT: String(port),
|
|
152
|
+
REFRESH_TOKEN_SECRET: "ai-smoke-refresh-secret",
|
|
153
|
+
SESSION_SECRET: "ai-smoke-session-secret",
|
|
154
|
+
TOKEN_ISSUER: "ai-smoke",
|
|
155
|
+
TOKEN_SECRET: "ai-smoke-token-secret",
|
|
156
|
+
},
|
|
157
|
+
timeoutMs: 30000,
|
|
158
|
+
url: `${baseUrl}/health`,
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
const signup = JSON.parse(
|
|
162
|
+
await expectStatus(
|
|
163
|
+
await fetch(`${baseUrl}/auth/signup`, {
|
|
164
|
+
body: JSON.stringify({
|
|
165
|
+
email: `ai-smoke-${startedAt.toMillis()}@example.com`,
|
|
166
|
+
password: "ai-smoke-password-1",
|
|
167
|
+
}),
|
|
168
|
+
headers: {"Content-Type": "application/json"},
|
|
169
|
+
method: "POST",
|
|
170
|
+
}),
|
|
171
|
+
200,
|
|
172
|
+
"POST /auth/signup"
|
|
173
|
+
)
|
|
174
|
+
) as {data: {token: string}};
|
|
175
|
+
console.info("✅ POST /auth/signup signs in a user");
|
|
176
|
+
|
|
177
|
+
const chat = await fetch(`${baseUrl}/gpt/prompt`, {
|
|
178
|
+
body: JSON.stringify({prompt: "Say hello."}),
|
|
179
|
+
headers: {
|
|
180
|
+
Authorization: `Bearer ${signup.data.token}`,
|
|
181
|
+
"Content-Type": "application/json",
|
|
182
|
+
},
|
|
183
|
+
method: "POST",
|
|
184
|
+
});
|
|
185
|
+
const events = parseEvents(await expectStatus(chat, 200, "POST /gpt/prompt"));
|
|
186
|
+
const failed = events.find((event) => event.error);
|
|
187
|
+
if (failed) {
|
|
188
|
+
throw new Error(`POST /gpt/prompt: the stream reported an error: ${failed.error}`);
|
|
189
|
+
}
|
|
190
|
+
const reply = events.map((event) => event.text ?? "").join("");
|
|
191
|
+
if (reply !== FAKE_REPLY) {
|
|
192
|
+
throw new Error(
|
|
193
|
+
`POST /gpt/prompt: expected the fake provider's text, got ${JSON.stringify(events)}`
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
const done = events.find((event) => event.done);
|
|
197
|
+
if (!done?.historyId || done.title !== FAKE_TITLE) {
|
|
198
|
+
throw new Error(
|
|
199
|
+
`POST /gpt/prompt: expected a done event with a history and the fake title, got ${JSON.stringify(done)}`
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
console.info("✅ POST /gpt/prompt streams the fake provider's reply");
|
|
203
|
+
} catch (error: unknown) {
|
|
204
|
+
if (server) {
|
|
205
|
+
console.error(`--- server output ---\n${server.output()}`);
|
|
206
|
+
}
|
|
207
|
+
throw error;
|
|
208
|
+
} finally {
|
|
209
|
+
await server?.stop();
|
|
210
|
+
await stopSmokeMongo();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
console.info(`ai smoke passed in ${DateTime.now().diff(startedAt).toMillis()} ms`);
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
const main = process.argv[2] === SERVE_ARG ? serve : run;
|
|
217
|
+
main().then(
|
|
218
|
+
() => {
|
|
219
|
+
if (main === run) {
|
|
220
|
+
process.exit(0);
|
|
221
|
+
}
|
|
222
|
+
},
|
|
223
|
+
(error: unknown) => {
|
|
224
|
+
console.error(`❌ ${error instanceof Error ? error.message : String(error)}`);
|
|
225
|
+
process.exit(1);
|
|
226
|
+
}
|
|
227
|
+
);
|