@terreno/ai 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +3 -3
- package/biome.jsonc +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/harness/commit.d.ts +4 -3
- package/dist/harness/commit.js +23 -4
- package/dist/index.d.ts +6 -9
- package/dist/index.js +14 -25
- package/dist/observability/adminScreens.js +5 -1
- package/dist/observability/evaluate.d.ts +8 -1
- package/dist/observability/evaluate.js +10 -2
- package/dist/observability/judgeAgreement.d.ts +53 -0
- package/dist/observability/judgeAgreement.js +123 -0
- package/dist/observability/local/datasetStore.d.ts +5 -0
- package/dist/observability/local/datasetStore.js +7 -2
- package/dist/observability/local/evaluatorStore.d.ts +17 -0
- package/dist/observability/local/evaluatorStore.js +73 -2
- package/dist/observability/local/experimentRunner.d.ts +42 -8
- package/dist/observability/local/experimentRunner.js +390 -11
- package/dist/observability/local/humanAnswers.d.ts +57 -0
- package/dist/observability/local/humanAnswers.js +174 -0
- package/dist/observability/local/judgeRun.d.ts +56 -0
- package/dist/observability/local/judgeRun.js +111 -0
- package/dist/observability/local/localPlugin.js +4 -0
- package/dist/observability/local/models/obsDatasetItem.js +4 -0
- package/dist/observability/local/models/obsEvaluator.js +4 -0
- package/dist/observability/local/models/obsExperiment.js +28 -0
- package/dist/observability/local/models/obsExperimentItem.js +4 -0
- package/dist/observability/local/models/obsJudgeRun.d.ts +2 -0
- package/dist/observability/local/models/obsJudgeRun.js +60 -0
- package/dist/observability/local/models/obsReviewBatch.d.ts +2 -0
- package/dist/observability/local/models/obsReviewBatch.js +48 -0
- package/dist/observability/local/models/obsReviewItem.js +10 -2
- package/dist/observability/local/models/obsTrace.js +4 -0
- package/dist/observability/local/promptReferences.d.ts +31 -0
- package/dist/observability/local/promptReferences.js +114 -0
- package/dist/observability/local/promptStore.d.ts +55 -19
- package/dist/observability/local/promptStore.js +171 -7
- package/dist/observability/local/reviewBatchStore.d.ts +106 -0
- package/dist/observability/local/reviewBatchStore.js +517 -0
- package/dist/observability/local/reviewStore.d.ts +24 -3
- package/dist/observability/local/reviewStore.js +98 -22
- package/dist/observability/local/traceStore.d.ts +14 -1
- package/dist/observability/local/traceStore.js +144 -11
- package/dist/observability/observabilityApp.js +10 -1
- package/dist/observability/questionFingerprint.d.ts +7 -0
- package/dist/observability/questionFingerprint.js +23 -0
- package/dist/observability/routes/experiments.js +12 -1
- package/dist/observability/routes/review.js +3 -1
- package/dist/observability/routes/reviewBatches.d.ts +11 -0
- package/dist/observability/routes/reviewBatches.js +78 -0
- package/dist/observability/routes/traces.d.ts +1 -1
- package/dist/observability/routes/traces.js +34 -2
- package/dist/observability/types.d.ts +10 -1
- package/dist/service/aiService.js +4 -1
- package/dist/service/chatTurn.js +38 -31
- package/dist/types/harness.d.ts +10 -4
- package/dist/types/index.d.ts +8 -3
- package/dist/types/observability.d.ts +74 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -28
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +0 -3
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +30 -30
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/commit.ts +24 -4
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +6 -8
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +29 -4
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -20
- package/src/index.ts +23 -26
- package/src/isolated/gptStream.isolated.ts +25 -28
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/adminScreens.test.ts +7 -0
- package/src/observability/adminScreens.ts +6 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/evaluate.ts +12 -2
- package/src/observability/judgeAgreement.test.ts +111 -0
- package/src/observability/judgeAgreement.ts +184 -0
- package/src/observability/local/datasetStore.ts +30 -11
- package/src/observability/local/evaluatorStore.ts +96 -2
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +497 -17
- package/src/observability/local/humanAnswers.ts +233 -0
- package/src/observability/local/judgePipeline.test.ts +444 -0
- package/src/observability/local/judgeRun.test.ts +173 -0
- package/src/observability/local/judgeRun.ts +184 -0
- package/src/observability/local/localPlugin.ts +4 -0
- package/src/observability/local/models/obsDatasetItem.ts +5 -0
- package/src/observability/local/models/obsEvaluator.ts +4 -0
- package/src/observability/local/models/obsExperiment.ts +29 -0
- package/src/observability/local/models/obsExperimentItem.ts +4 -0
- package/src/observability/local/models/obsJudgeRun.ts +61 -0
- package/src/observability/local/models/obsReviewBatch.ts +52 -0
- package/src/observability/local/models/obsReviewItem.ts +11 -2
- package/src/observability/local/models/obsTrace.ts +5 -0
- package/src/observability/local/obsModels.test.ts +13 -0
- package/src/observability/local/promptReferences.test.ts +128 -0
- package/src/observability/local/promptReferences.ts +149 -0
- package/src/observability/local/promptStore.test.ts +295 -3
- package/src/observability/local/promptStore.ts +243 -32
- package/src/observability/local/reviewBatchStore.test.ts +251 -0
- package/src/observability/local/reviewBatchStore.ts +656 -0
- package/src/observability/local/reviewStore.ts +122 -25
- package/src/observability/local/traceStore.test.ts +139 -1
- package/src/observability/local/traceStore.ts +162 -11
- package/src/observability/observabilityApp.test.ts +13 -10
- package/src/observability/observabilityApp.ts +10 -1
- package/src/observability/questionFingerprint.test.ts +33 -0
- package/src/observability/questionFingerprint.ts +24 -0
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +39 -2
- package/src/observability/routes/experiments.ts +13 -1
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/review.ts +3 -1
- package/src/observability/routes/reviewBatches.test.ts +209 -0
- package/src/observability/routes/reviewBatches.ts +138 -0
- package/src/observability/routes/status.test.ts +8 -8
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +63 -2
- package/src/observability/routes/traces.ts +46 -4
- package/src/observability/types.ts +5 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +156 -115
- package/src/service/aiService.test.ts +22 -1
- package/src/service/aiService.ts +4 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +42 -33
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +0 -54
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +11 -4
- package/src/types/index.ts +8 -3
- package/src/types/observability.ts +86 -2
- package/tsconfig.typecheck.json +18 -0
- package/dist/langfuseApp.d.ts +0 -9
- package/dist/langfuseApp.js +0 -53
- package/dist/langfuseCache.d.ts +0 -15
- package/dist/langfuseCache.js +0 -54
- package/dist/langfuseClient.d.ts +0 -6
- package/dist/langfuseClient.js +0 -34
- package/dist/langfusePrompts.d.ts +0 -12
- package/dist/langfusePrompts.js +0 -89
- package/dist/langfuseRoutesEvaluations.d.ts +0 -3
- package/dist/langfuseRoutesEvaluations.js +0 -32
- package/dist/langfuseRoutesMiddleware.d.ts +0 -2
- package/dist/langfuseRoutesMiddleware.js +0 -13
- package/dist/langfuseRoutesPlayground.d.ts +0 -2
- package/dist/langfuseRoutesPlayground.js +0 -36
- package/dist/langfuseRoutesPrompts.d.ts +0 -2
- package/dist/langfuseRoutesPrompts.js +0 -56
- package/dist/langfuseRoutesTraces.d.ts +0 -2
- package/dist/langfuseRoutesTraces.js +0 -30
- package/dist/langfuseTracing.d.ts +0 -4
- package/dist/langfuseTracing.js +0 -32
- package/dist/langfuseTypes.d.ts +0 -96
- package/dist/langfuseTypes.js +0 -2
- package/dist/langfuseVercelAi.d.ts +0 -12
- package/dist/langfuseVercelAi.js +0 -49
- package/src/isolated/langfuseClient.isolated.ts +0 -60
- package/src/langfuseApp.test.ts +0 -135
- package/src/langfuseApp.ts +0 -62
- package/src/langfuseCache.test.ts +0 -75
- package/src/langfuseCache.ts +0 -72
- package/src/langfuseClient.test.ts +0 -69
- package/src/langfuseClient.ts +0 -34
- package/src/langfusePrompts.test.ts +0 -188
- package/src/langfusePrompts.ts +0 -121
- package/src/langfuseRoutes.test.ts +0 -256
- package/src/langfuseRoutesEvaluations.ts +0 -48
- package/src/langfuseRoutesMiddleware.test.ts +0 -57
- package/src/langfuseRoutesMiddleware.ts +0 -12
- package/src/langfuseRoutesPlayground.ts +0 -50
- package/src/langfuseRoutesPrompts.ts +0 -87
- package/src/langfuseRoutesTraces.ts +0 -40
- package/src/langfuseTracing.test.ts +0 -68
- package/src/langfuseTracing.ts +0 -33
- package/src/langfuseTypes.ts +0 -103
- package/src/langfuseVercelAi.test.ts +0 -79
- package/src/langfuseVercelAi.ts +0 -72
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
|
|
2
|
+
import type express from "express";
|
|
3
|
+
|
|
4
|
+
import type {LocalReviewBatchStore} from "../local/reviewBatchStore";
|
|
5
|
+
import type {ObservabilityGenerateClient, ObservabilityRequestAiServiceFactory} from "../types";
|
|
6
|
+
import {
|
|
7
|
+
type ObservabilityRouteAccessOptions,
|
|
8
|
+
observabilityRouteMiddleware,
|
|
9
|
+
} from "./observabilityRouteAccess";
|
|
10
|
+
import {resolveRequestAiService} from "./requestAiService";
|
|
11
|
+
|
|
12
|
+
const BASE_PATH = "/ai/observability";
|
|
13
|
+
|
|
14
|
+
export interface ObservabilityReviewBatchRouteOptions extends ObservabilityRouteAccessOptions {
|
|
15
|
+
aiService?: ObservabilityGenerateClient;
|
|
16
|
+
requestAiServiceFactory?: ObservabilityRequestAiServiceFactory;
|
|
17
|
+
store: LocalReviewBatchStore;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Register before the review item routes so `/review/batches` is not read as an item id. */
|
|
21
|
+
export const addObservabilityReviewBatchRoutes = (
|
|
22
|
+
router: express.Application,
|
|
23
|
+
options: ObservabilityReviewBatchRouteOptions
|
|
24
|
+
): void => {
|
|
25
|
+
const openApiOptions = options.openApi ? {openApi: options.openApi} : {};
|
|
26
|
+
const builder = (): ReturnType<typeof createOpenApiBuilder> => {
|
|
27
|
+
return createOpenApiBuilder(openApiOptions as Parameters<typeof createOpenApiBuilder>[0]);
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
router.get(
|
|
31
|
+
`${BASE_PATH}/review/batches`,
|
|
32
|
+
observabilityRouteMiddleware(
|
|
33
|
+
options.accessControl,
|
|
34
|
+
{action: "list", resource: "aiReview"},
|
|
35
|
+
builder()
|
|
36
|
+
.withTags(["observability"])
|
|
37
|
+
.withSummary("List human review batches with judge agreement")
|
|
38
|
+
.withResponse(200, {data: {type: "array"}})
|
|
39
|
+
.build()
|
|
40
|
+
),
|
|
41
|
+
asyncHandler(async (_req, res) => {
|
|
42
|
+
return res.json({data: await options.store.list(), more: false});
|
|
43
|
+
})
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
router.post(
|
|
47
|
+
`${BASE_PATH}/review/batches`,
|
|
48
|
+
observabilityRouteMiddleware(
|
|
49
|
+
options.accessControl,
|
|
50
|
+
{action: "assign", resource: "aiReview"},
|
|
51
|
+
builder()
|
|
52
|
+
.withTags(["observability"])
|
|
53
|
+
.withSummary(
|
|
54
|
+
"Create a review batch from traces, the most recent traces, or a dataset's unanswered items"
|
|
55
|
+
)
|
|
56
|
+
.withResponse(201, {data: {type: "object"}})
|
|
57
|
+
.build()
|
|
58
|
+
),
|
|
59
|
+
asyncHandler(async (req, res) => {
|
|
60
|
+
const body = req.body as {
|
|
61
|
+
datasetId?: string;
|
|
62
|
+
description?: string;
|
|
63
|
+
evaluatorId?: string;
|
|
64
|
+
limit?: number;
|
|
65
|
+
name?: string;
|
|
66
|
+
promptName?: string;
|
|
67
|
+
traceIds?: string[];
|
|
68
|
+
};
|
|
69
|
+
if (!body.evaluatorId) {
|
|
70
|
+
throw new APIError({status: 400, title: "evaluatorId is required"});
|
|
71
|
+
}
|
|
72
|
+
if (body.traceIds !== undefined && !Array.isArray(body.traceIds)) {
|
|
73
|
+
throw new APIError({status: 400, title: "traceIds must be an array"});
|
|
74
|
+
}
|
|
75
|
+
if (body.datasetId !== undefined && typeof body.datasetId !== "string") {
|
|
76
|
+
throw new APIError({status: 400, title: "datasetId must be a string"});
|
|
77
|
+
}
|
|
78
|
+
const data = await options.store.create({
|
|
79
|
+
datasetId: body.datasetId || undefined,
|
|
80
|
+
description: body.description,
|
|
81
|
+
evaluatorId: body.evaluatorId,
|
|
82
|
+
limit: body.limit,
|
|
83
|
+
name: body.name,
|
|
84
|
+
promptName: body.promptName || undefined,
|
|
85
|
+
traceIds: body.traceIds,
|
|
86
|
+
});
|
|
87
|
+
return res.status(201).json({data});
|
|
88
|
+
})
|
|
89
|
+
);
|
|
90
|
+
|
|
91
|
+
router.get(
|
|
92
|
+
`${BASE_PATH}/review/batches/:id`,
|
|
93
|
+
observabilityRouteMiddleware(
|
|
94
|
+
options.accessControl,
|
|
95
|
+
{action: "read", resource: "aiReview"},
|
|
96
|
+
builder()
|
|
97
|
+
.withTags(["observability"])
|
|
98
|
+
.withSummary("Get a review batch with human answers and judge agreement")
|
|
99
|
+
.withPathParameter("id", {type: "string"})
|
|
100
|
+
.withResponse(200, {data: {type: "object"}})
|
|
101
|
+
.build()
|
|
102
|
+
),
|
|
103
|
+
asyncHandler(async (req, res) => {
|
|
104
|
+
return res.json({data: await options.store.get(req.params.id)});
|
|
105
|
+
})
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
router.post(
|
|
109
|
+
`${BASE_PATH}/review/batches/:id/judgeRuns`,
|
|
110
|
+
observabilityRouteMiddleware(
|
|
111
|
+
options.accessControl,
|
|
112
|
+
{action: "assign", resource: "aiReview"},
|
|
113
|
+
builder()
|
|
114
|
+
.withTags(["observability"])
|
|
115
|
+
.withSummary("Run an LLM judge over a review batch")
|
|
116
|
+
.withPathParameter("id", {type: "string"})
|
|
117
|
+
.withResponse(201, {data: {type: "object"}})
|
|
118
|
+
.build()
|
|
119
|
+
),
|
|
120
|
+
asyncHandler(async (req, res) => {
|
|
121
|
+
const body = req.body as {evaluatorId?: string};
|
|
122
|
+
if (!body.evaluatorId) {
|
|
123
|
+
throw new APIError({status: 400, title: "evaluatorId is required"});
|
|
124
|
+
}
|
|
125
|
+
const ai = resolveRequestAiService({
|
|
126
|
+
aiService: options.aiService,
|
|
127
|
+
req,
|
|
128
|
+
requestAiServiceFactory: options.requestAiServiceFactory,
|
|
129
|
+
});
|
|
130
|
+
const data = await options.store.runJudge({
|
|
131
|
+
ai,
|
|
132
|
+
batchId: req.params.id,
|
|
133
|
+
evaluatorId: body.evaluatorId,
|
|
134
|
+
});
|
|
135
|
+
return res.status(201).json({data});
|
|
136
|
+
})
|
|
137
|
+
);
|
|
138
|
+
};
|
|
@@ -4,17 +4,17 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {AIService} from "../../service/aiService";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
9
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
10
10
|
import type {ObservabilityPlugin} from "../types";
|
|
11
11
|
|
|
12
|
-
const
|
|
12
|
+
const createRemotePlugin = (): ObservabilityPlugin => {
|
|
13
13
|
return {
|
|
14
14
|
capabilities: new Set(["datasets", "experiments", "prompts", "scores", "traces"]),
|
|
15
15
|
datasetStore: {},
|
|
16
16
|
experimentRunner: {},
|
|
17
|
-
id: "
|
|
17
|
+
id: "remote",
|
|
18
18
|
promptRegistry: {get: async () => undefined},
|
|
19
19
|
scoreSink: {export: async () => {}},
|
|
20
20
|
traceSink: {export: async () => {}},
|
|
@@ -33,10 +33,10 @@ describe("observability status routes", () => {
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
beforeEach(() => {
|
|
36
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
36
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
37
37
|
.register(
|
|
38
38
|
new ObservabilityApp({
|
|
39
|
-
plugins: [createLocalObservabilityPlugin(),
|
|
39
|
+
plugins: [createLocalObservabilityPlugin(), createRemotePlugin()],
|
|
40
40
|
})
|
|
41
41
|
)
|
|
42
42
|
.build();
|
|
@@ -55,7 +55,7 @@ describe("observability status routes", () => {
|
|
|
55
55
|
});
|
|
56
56
|
expect(response.body.data.plugins.map((plugin: {id: string}) => plugin.id)).toEqual([
|
|
57
57
|
"local",
|
|
58
|
-
"
|
|
58
|
+
"remote",
|
|
59
59
|
]);
|
|
60
60
|
expect(response.body.data.plugins[0].capabilities).toContain("reviewQueue");
|
|
61
61
|
});
|
|
@@ -89,7 +89,7 @@ describe("observability status routes", () => {
|
|
|
89
89
|
specificationVersion: "v2" as const,
|
|
90
90
|
supportedUrls: {},
|
|
91
91
|
};
|
|
92
|
-
const serverApp = new TerrenoApp({skipListen: true, userModel:
|
|
92
|
+
const serverApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
93
93
|
.register(
|
|
94
94
|
new ObservabilityApp({
|
|
95
95
|
aiService: new AIService({model: model as unknown as LanguageModel}),
|
|
@@ -101,7 +101,7 @@ describe("observability status routes", () => {
|
|
|
101
101
|
const serverStatus = await serverAgent.get("/ai/observability/status");
|
|
102
102
|
expect(serverStatus.body.data.playgroundAi).toEqual({source: "server"});
|
|
103
103
|
|
|
104
|
-
const requestKeyApp = new TerrenoApp({skipListen: true, userModel:
|
|
104
|
+
const requestKeyApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
105
105
|
.register(
|
|
106
106
|
new ObservabilityApp({
|
|
107
107
|
plugins: [createLocalObservabilityPlugin()],
|
|
@@ -3,7 +3,7 @@ import {TerrenoApp} from "@terreno/api";
|
|
|
3
3
|
import {assert} from "chai";
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
|
-
import {authAsUser, ensureTestUsers,
|
|
6
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
7
7
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
8
8
|
import {registerObsSpan} from "../local/models/obsSpan";
|
|
9
9
|
import {registerObsTrace} from "../local/models/obsTrace";
|
|
@@ -86,7 +86,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
86
86
|
await registerObsSpan().deleteMany({});
|
|
87
87
|
const plugin = createLocalObservabilityPlugin();
|
|
88
88
|
fakeAiService = createFakeAiService();
|
|
89
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
89
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
90
90
|
.register(new ObservabilityApp({aiService: fakeAiService, plugins: [plugin]}))
|
|
91
91
|
.build();
|
|
92
92
|
});
|
|
@@ -125,7 +125,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
125
125
|
|
|
126
126
|
it("returns 503 when no server or request AI service is available", async () => {
|
|
127
127
|
const plugin = createLocalObservabilityPlugin();
|
|
128
|
-
const noAiApp = new TerrenoApp({skipListen: true, userModel:
|
|
128
|
+
const noAiApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
129
129
|
.register(new ObservabilityApp({plugins: [plugin]}))
|
|
130
130
|
.build();
|
|
131
131
|
const agent = await authAsUser(noAiApp, "admin");
|
|
@@ -141,7 +141,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
141
141
|
return apiKey === "request-key" ? createFakeAiService() : undefined;
|
|
142
142
|
});
|
|
143
143
|
const plugin = createLocalObservabilityPlugin();
|
|
144
|
-
const requestKeyApp = new TerrenoApp({skipListen: true, userModel:
|
|
144
|
+
const requestKeyApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
145
145
|
.register(new ObservabilityApp({plugins: [plugin], requestAiServiceFactory}))
|
|
146
146
|
.build();
|
|
147
147
|
const agent = await authAsUser(requestKeyApp, "admin");
|
|
@@ -159,7 +159,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
159
159
|
it("exports an error trace and rethrows when a child LLM stage fails", async () => {
|
|
160
160
|
const failingService = createFakeAiService({failOn: OBS_TEST_MULTI_STAGE_CALL_2_SCHEMA_NAME});
|
|
161
161
|
const plugin = createLocalObservabilityPlugin();
|
|
162
|
-
const failingApp = new TerrenoApp({skipListen: true, userModel:
|
|
162
|
+
const failingApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
163
163
|
.register(new ObservabilityApp({aiService: failingService, plugins: [plugin]}))
|
|
164
164
|
.build();
|
|
165
165
|
const agent = await authAsUser(failingApp, "admin");
|
|
@@ -186,7 +186,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
186
186
|
it("exports the completed child stages when final synthesis fails", async () => {
|
|
187
187
|
const failingService = createFakeAiService({failOn: OBS_TEST_MULTI_STAGE_FINAL_SCHEMA_NAME});
|
|
188
188
|
const plugin = createLocalObservabilityPlugin();
|
|
189
|
-
const failingApp = new TerrenoApp({skipListen: true, userModel:
|
|
189
|
+
const failingApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
190
190
|
.register(new ObservabilityApp({aiService: failingService, plugins: [plugin]}))
|
|
191
191
|
.build();
|
|
192
192
|
const agent = await authAsUser(failingApp, "admin");
|
|
@@ -3,7 +3,7 @@ import {TerrenoApp} from "@terreno/api";
|
|
|
3
3
|
import type express from "express";
|
|
4
4
|
import {DateTime} from "luxon";
|
|
5
5
|
|
|
6
|
-
import {authAsUser, ensureTestUsers,
|
|
6
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
7
7
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
8
8
|
import {registerObsScore} from "../local/models/obsScore";
|
|
9
9
|
import {registerObsSpan} from "../local/models/obsSpan";
|
|
@@ -32,7 +32,7 @@ describe("observability trace routes", () => {
|
|
|
32
32
|
plugin.traceSink && "store" in plugin.traceSink
|
|
33
33
|
? (plugin.traceSink as {store: LocalTraceStore}).store
|
|
34
34
|
: new LocalTraceStore();
|
|
35
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
35
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
36
36
|
.register(new ObservabilityApp({plugins: [plugin]}))
|
|
37
37
|
.build();
|
|
38
38
|
});
|
|
@@ -109,6 +109,67 @@ describe("observability trace routes", () => {
|
|
|
109
109
|
).toEqual([[inScope.id, "ws-1"]]);
|
|
110
110
|
});
|
|
111
111
|
|
|
112
|
+
it("sorts the trace list across pages and rejects unknown sort keys", async () => {
|
|
113
|
+
const base = DateTime.utc().minus({hours: 1});
|
|
114
|
+
const exportAt = async (
|
|
115
|
+
name: string,
|
|
116
|
+
offsetMs: number,
|
|
117
|
+
latencyMs: number,
|
|
118
|
+
costUsd: number,
|
|
119
|
+
inputTokens: number
|
|
120
|
+
): Promise<string> => {
|
|
121
|
+
const exported = await store.exportTrace({
|
|
122
|
+
endedAt: base.plus({milliseconds: offsetMs + latencyMs}).toISO() ?? "",
|
|
123
|
+
id: name,
|
|
124
|
+
name,
|
|
125
|
+
prompts: [{name, version: 1}],
|
|
126
|
+
sensitive: false,
|
|
127
|
+
spans: [],
|
|
128
|
+
startedAt: base.plus({milliseconds: offsetMs}).toISO() ?? "",
|
|
129
|
+
status: "ok",
|
|
130
|
+
usage: {costUsd, inputTokens, outputTokens: 0},
|
|
131
|
+
});
|
|
132
|
+
return exported.id;
|
|
133
|
+
};
|
|
134
|
+
const cheapSlow = await exportAt("cheap-slow", 0, 900, 0.0001, 500);
|
|
135
|
+
const priceyFast = await exportAt("pricey-fast", 1000, 50, 0.05, 20);
|
|
136
|
+
const midMid = await exportAt("mid-mid", 2000, 300, 0.001, 12_345);
|
|
137
|
+
const admin = await authAsUser(app, "admin");
|
|
138
|
+
|
|
139
|
+
const pageIds = async (query: string): Promise<string[]> => {
|
|
140
|
+
const first = await admin.get(`/ai/observability/traces?${query}&limit=2&page=1`);
|
|
141
|
+
const second = await admin.get(`/ai/observability/traces?${query}&limit=2&page=2`);
|
|
142
|
+
expect(first.status).toBe(200);
|
|
143
|
+
expect(first.body.more).toBe(true);
|
|
144
|
+
expect(second.body.more).toBe(false);
|
|
145
|
+
return [...first.body.data, ...second.body.data].map((row: {id: string}) => row.id);
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
expect(await pageIds("sort=-cost")).toEqual([priceyFast, midMid, cheapSlow]);
|
|
149
|
+
expect(await pageIds("sort=cost")).toEqual([cheapSlow, midMid, priceyFast]);
|
|
150
|
+
expect(await pageIds("sort=-latency")).toEqual([cheapSlow, midMid, priceyFast]);
|
|
151
|
+
expect(await pageIds("sort=latency")).toEqual([priceyFast, midMid, cheapSlow]);
|
|
152
|
+
expect(await pageIds("sort=-tokens")).toEqual([midMid, cheapSlow, priceyFast]);
|
|
153
|
+
expect(await pageIds("sort=tokens")).toEqual([priceyFast, cheapSlow, midMid]);
|
|
154
|
+
expect(await pageIds("sort=startedAt")).toEqual([cheapSlow, priceyFast, midMid]);
|
|
155
|
+
expect(await pageIds("sort=-name")).toEqual([priceyFast, midMid, cheapSlow]);
|
|
156
|
+
// Default stays newest first.
|
|
157
|
+
expect(await pageIds("status=ok")).toEqual([midMid, priceyFast, cheapSlow]);
|
|
158
|
+
|
|
159
|
+
const multiPrompt = await admin.get(
|
|
160
|
+
"/ai/observability/traces?prompt[0]=cheap-slow&prompt[1]=mid-mid&sort=name"
|
|
161
|
+
);
|
|
162
|
+
expect(multiPrompt.status).toBe(200);
|
|
163
|
+
expect(multiPrompt.body.data.map((row: {id: string}) => row.id)).toEqual([cheapSlow, midMid]);
|
|
164
|
+
|
|
165
|
+
const unknown = await admin.get("/ai/observability/traces?sort=created");
|
|
166
|
+
expect(unknown.status).toBe(400);
|
|
167
|
+
const listSort = await admin.get("/ai/observability/traces?sort[0]=cost");
|
|
168
|
+
expect(listSort.status).toBe(400);
|
|
169
|
+
const objectPrompt = await admin.get("/ai/observability/traces?prompt[name]=x");
|
|
170
|
+
expect(objectPrompt.status).toBe(400);
|
|
171
|
+
});
|
|
172
|
+
|
|
112
173
|
it("forbids non-admins from listing traces", async () => {
|
|
113
174
|
const agent = await authAsUser(app, "notAdmin");
|
|
114
175
|
const res = await agent.get("/ai/observability/traces");
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import {APIError, asyncHandler, createOpenApiBuilder} from "@terreno/api";
|
|
2
2
|
import type express from "express";
|
|
3
|
-
import
|
|
3
|
+
import {
|
|
4
|
+
DEFAULT_TRACE_LIST_SORT,
|
|
5
|
+
type LocalTraceStore,
|
|
6
|
+
parseTraceListSort,
|
|
7
|
+
TRACE_LIST_SORT_KEYS,
|
|
8
|
+
} from "../local/traceStore";
|
|
4
9
|
import {getObservabilityApp} from "../observabilityAppRegistry";
|
|
5
10
|
import type {ScoreRecord, ScoreSink} from "../types";
|
|
6
11
|
import {
|
|
@@ -27,7 +32,7 @@ const parseBoolean = (value: unknown): boolean | undefined => {
|
|
|
27
32
|
throw new APIError({status: 400, title: "boolean query must be true or false"});
|
|
28
33
|
};
|
|
29
34
|
|
|
30
|
-
const parsePromptVersion = (value: unknown, prompt?: string): number | undefined => {
|
|
35
|
+
const parsePromptVersion = (value: unknown, prompt?: string | string[]): number | undefined => {
|
|
31
36
|
if (value === undefined) {
|
|
32
37
|
return undefined;
|
|
33
38
|
}
|
|
@@ -35,12 +40,37 @@ const parsePromptVersion = (value: unknown, prompt?: string): number | undefined
|
|
|
35
40
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
36
41
|
throw new APIError({status: 400, title: "promptVersion must be a positive integer"});
|
|
37
42
|
}
|
|
38
|
-
if (!prompt) {
|
|
43
|
+
if (!prompt || prompt.length === 0) {
|
|
39
44
|
throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
|
|
40
45
|
}
|
|
41
46
|
return parsed;
|
|
42
47
|
};
|
|
43
48
|
|
|
49
|
+
const parsePrompt = (value: unknown): string | string[] | undefined => {
|
|
50
|
+
if (typeof value === "string") {
|
|
51
|
+
return value;
|
|
52
|
+
}
|
|
53
|
+
if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) {
|
|
54
|
+
return value as string[];
|
|
55
|
+
}
|
|
56
|
+
if (value === undefined) {
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
throw new APIError({status: 400, title: "prompt must be a string or a list of strings"});
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const parseSort = (value: unknown): string | undefined => {
|
|
63
|
+
if (value === undefined) {
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
if (typeof value !== "string") {
|
|
67
|
+
throw new APIError({status: 400, title: "Unknown trace sort key"});
|
|
68
|
+
}
|
|
69
|
+
// Validate here so a bad key fails before the store runs any query.
|
|
70
|
+
parseTraceListSort(value);
|
|
71
|
+
return value;
|
|
72
|
+
};
|
|
73
|
+
|
|
44
74
|
const parsePositiveInt = (value: unknown, fallback: number): number => {
|
|
45
75
|
if (value === undefined) {
|
|
46
76
|
return fallback;
|
|
@@ -72,11 +102,22 @@ export const addObservabilityTraceRoutes = (
|
|
|
72
102
|
.withQueryParameter("prompt", {type: "string"}, {required: false})
|
|
73
103
|
.withQueryParameter("promptVersion", {type: "number"}, {required: false})
|
|
74
104
|
.withQueryParameter("scope", {type: "string"}, {required: false})
|
|
105
|
+
.withQueryParameter(
|
|
106
|
+
"sort",
|
|
107
|
+
{
|
|
108
|
+
enum: TRACE_LIST_SORT_KEYS.flatMap((key) => [key, `-${key}`]),
|
|
109
|
+
type: "string",
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
description: `Sort column, \`-\` prefix for descending. Defaults to \`${DEFAULT_TRACE_LIST_SORT}\`.`,
|
|
113
|
+
required: false,
|
|
114
|
+
}
|
|
115
|
+
)
|
|
75
116
|
.withResponse(200, {data: {type: "array"}})
|
|
76
117
|
.build()
|
|
77
118
|
),
|
|
78
119
|
asyncHandler(async (req, res) => {
|
|
79
|
-
const prompt =
|
|
120
|
+
const prompt = parsePrompt(req.query.prompt);
|
|
80
121
|
const listed = await options.store.list({
|
|
81
122
|
flaggedForDataset: parseBoolean(req.query.flaggedForDataset),
|
|
82
123
|
from: typeof req.query.from === "string" ? req.query.from : undefined,
|
|
@@ -88,6 +129,7 @@ export const addObservabilityTraceRoutes = (
|
|
|
88
129
|
scope: typeof req.query.scope === "string" ? req.query.scope : undefined,
|
|
89
130
|
sensitive: parseBoolean(req.query.sensitive),
|
|
90
131
|
sessionId: typeof req.query.sessionId === "string" ? req.query.sessionId : undefined,
|
|
132
|
+
sort: parseSort(req.query.sort),
|
|
91
133
|
status:
|
|
92
134
|
req.query.status === "ok" || req.query.status === "error" ? req.query.status : undefined,
|
|
93
135
|
to: typeof req.query.to === "string" ? req.query.to : undefined,
|
|
@@ -8,7 +8,7 @@ export type ObservabilityCapability =
|
|
|
8
8
|
| "scores"
|
|
9
9
|
| "traces";
|
|
10
10
|
|
|
11
|
-
export type ControlPrimary = "
|
|
11
|
+
export type ControlPrimary = "local";
|
|
12
12
|
|
|
13
13
|
export interface ObservabilityControlConfig {
|
|
14
14
|
datasets: ControlPrimary;
|
|
@@ -50,7 +50,7 @@ export interface TraceRecord {
|
|
|
50
50
|
metadata?: Record<string, unknown>;
|
|
51
51
|
name: string;
|
|
52
52
|
output?: unknown;
|
|
53
|
-
prompts: {label?: string; name: string; version: number}[];
|
|
53
|
+
prompts: {includedBy?: string; label?: string; name: string; version: number}[];
|
|
54
54
|
sensitive: boolean;
|
|
55
55
|
sessionId?: string;
|
|
56
56
|
spans: SpanRecord[];
|
|
@@ -78,7 +78,10 @@ export interface ScoreRecord {
|
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
export interface PromptVersionRef {
|
|
81
|
+
/** `body` with every prompt reference resolved. */
|
|
81
82
|
body: string;
|
|
83
|
+
/** Prompt versions pulled in by references, recorded on the trace after this prompt. */
|
|
84
|
+
includes?: {includedBy?: string; label?: string; name: string; version: number}[];
|
|
82
85
|
label?: string;
|
|
83
86
|
name: string;
|
|
84
87
|
sensitive?: boolean;
|
package/src/routes/files.test.ts
CHANGED
|
@@ -6,7 +6,7 @@ import supertest from "supertest";
|
|
|
6
6
|
|
|
7
7
|
import {FileAttachment} from "../models/fileAttachment";
|
|
8
8
|
import type {FileStorageService, UploadFileParams, UploadFileResult} from "../service/fileStorage";
|
|
9
|
-
import {authAsUser, ensureTestUsers, UserModel} from "../tests/helpers";
|
|
9
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
|
|
10
10
|
import {addFileRoutes} from "./files";
|
|
11
11
|
|
|
12
12
|
type MockFileStorageService = Pick<FileStorageService, "upload" | "delete" | "getSignedUrl">;
|
|
@@ -45,11 +45,12 @@ describe("File Routes", () => {
|
|
|
45
45
|
configureApp: (router, options) => {
|
|
46
46
|
addFileRoutes(router, {
|
|
47
47
|
fileStorageService: fileStorageService as FileStorageService,
|
|
48
|
+
gcsBucket: "test-bucket",
|
|
48
49
|
openApiOptions: options,
|
|
49
50
|
});
|
|
50
51
|
},
|
|
51
52
|
skipListen: true,
|
|
52
|
-
userModel:
|
|
53
|
+
userModel: terrenoUserModel,
|
|
53
54
|
}).build();
|
|
54
55
|
});
|
|
55
56
|
|
|
@@ -76,11 +77,12 @@ describe("File Routes", () => {
|
|
|
76
77
|
addFileRoutes(router, {
|
|
77
78
|
fileStorageService: fileStorageService as FileStorageService,
|
|
78
79
|
fileUploadsEnabled: async () => false,
|
|
80
|
+
gcsBucket: "test-bucket",
|
|
79
81
|
openApiOptions: options,
|
|
80
82
|
});
|
|
81
83
|
},
|
|
82
84
|
skipListen: true,
|
|
83
|
-
userModel:
|
|
85
|
+
userModel: terrenoUserModel,
|
|
84
86
|
}).build();
|
|
85
87
|
const agent = await authAsUser(gated, "notAdmin");
|
|
86
88
|
const res = await agent
|
package/src/routes/gpt.test.ts
CHANGED
|
@@ -51,7 +51,13 @@ import {
|
|
|
51
51
|
toolNamesOf,
|
|
52
52
|
USER_PROMPT,
|
|
53
53
|
} from "../tests/chatHarness";
|
|
54
|
-
import {
|
|
54
|
+
import {
|
|
55
|
+
authAsUser,
|
|
56
|
+
ensureTestUsers,
|
|
57
|
+
historyRouterOptions,
|
|
58
|
+
terrenoUserModel,
|
|
59
|
+
UserModel,
|
|
60
|
+
} from "../tests/helpers";
|
|
55
61
|
import type {GptHistoryDocument} from "../types";
|
|
56
62
|
import {addGptRoutes} from "./gpt";
|
|
57
63
|
import {addGptHistoryRoutes} from "./gptHistories";
|
|
@@ -2840,7 +2846,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2840
2846
|
asks: true,
|
|
2841
2847
|
};
|
|
2842
2848
|
addGptHistoryRoutes(router, {
|
|
2843
|
-
...options,
|
|
2849
|
+
...historyRouterOptions(options),
|
|
2844
2850
|
chat,
|
|
2845
2851
|
responseHandler: async (value) => {
|
|
2846
2852
|
const history = (value as GptHistoryDocument).toObject();
|
|
@@ -2850,7 +2856,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2850
2856
|
addGptRoutes(router, chat);
|
|
2851
2857
|
},
|
|
2852
2858
|
skipListen: true,
|
|
2853
|
-
userModel:
|
|
2859
|
+
userModel: terrenoUserModel,
|
|
2854
2860
|
}).build();
|
|
2855
2861
|
const agent = await authAsUser(app, "notAdmin");
|
|
2856
2862
|
const historyId = await pauseOnPlanAsk(agent);
|
|
@@ -2894,7 +2900,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2894
2900
|
const app = new TerrenoApp({
|
|
2895
2901
|
configureApp: (router, options) => {
|
|
2896
2902
|
addGptHistoryRoutes(router, {
|
|
2897
|
-
...options,
|
|
2903
|
+
...historyRouterOptions(options),
|
|
2898
2904
|
openApiOverwrite: {
|
|
2899
2905
|
create: {summary: "Start a conversation"},
|
|
2900
2906
|
update: {requestBody: {description: "The fields to change"}},
|
|
@@ -2902,7 +2908,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2902
2908
|
});
|
|
2903
2909
|
},
|
|
2904
2910
|
skipListen: true,
|
|
2905
|
-
userModel:
|
|
2911
|
+
userModel: terrenoUserModel,
|
|
2906
2912
|
}).build();
|
|
2907
2913
|
const agent = await authAsUser(app, "notAdmin");
|
|
2908
2914
|
|
|
@@ -2920,12 +2926,12 @@ describe("/gpt/prompt asks", () => {
|
|
|
2920
2926
|
const app = new TerrenoApp({
|
|
2921
2927
|
configureApp: (router, options) => {
|
|
2922
2928
|
addGptHistoryRoutes(router, {
|
|
2923
|
-
...options,
|
|
2929
|
+
...historyRouterOptions(options),
|
|
2924
2930
|
preUpdate: (body) => ({...body, title: `${body.title} (edited)`}) as GptHistoryDocument,
|
|
2925
2931
|
});
|
|
2926
2932
|
},
|
|
2927
2933
|
skipListen: true,
|
|
2928
|
-
userModel:
|
|
2934
|
+
userModel: terrenoUserModel,
|
|
2929
2935
|
}).build();
|
|
2930
2936
|
const agent = await authAsUser(app, "notAdmin");
|
|
2931
2937
|
const created = await agent
|
|
@@ -2980,13 +2986,14 @@ const STEPPER_HOST_ACTIONS = {
|
|
|
2980
2986
|
scaleStepper: {handles: "stepper" as const},
|
|
2981
2987
|
};
|
|
2982
2988
|
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2989
|
+
interface BlocksEvent {
|
|
2990
|
+
errors: {code: string; path: string}[];
|
|
2991
|
+
ok: boolean;
|
|
2992
|
+
warnings?: unknown[];
|
|
2993
|
+
}
|
|
2994
|
+
|
|
2995
|
+
const blocksEventOf = (events: Record<string, unknown>[]): BlocksEvent =>
|
|
2996
|
+
events.find((event) => "blocks" in event)?.blocks as BlocksEvent;
|
|
2990
2997
|
|
|
2991
2998
|
describe("/gpt/prompt uiBlocks", () => {
|
|
2992
2999
|
beforeAll(async () => {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import {afterEach, describe, expect, it} from "bun:test";
|
|
2
2
|
import {z} from "@terreno/api";
|
|
3
|
-
import type mongoose from "mongoose";
|
|
4
3
|
|
|
5
4
|
import {AIRequest} from "../models/aiRequest";
|
|
6
5
|
import {GptHistory} from "../models/gptHistory";
|
|
@@ -47,7 +46,7 @@ describe("POST /gpt/actions", () => {
|
|
|
47
46
|
|
|
48
47
|
it("rejects an unknown name, a bad payload, and a non-owner", async () => {
|
|
49
48
|
const created = await ensureTestUsers();
|
|
50
|
-
const userId =
|
|
49
|
+
const userId = created[1]._id;
|
|
51
50
|
const history = await GptHistory.create({
|
|
52
51
|
prompts: [{text: "Hello", type: "user"}],
|
|
53
52
|
userId,
|
|
@@ -89,7 +88,7 @@ describe("POST /gpt/actions", () => {
|
|
|
89
88
|
|
|
90
89
|
it("returns the handler document and logs ui_action", async () => {
|
|
91
90
|
const created = await ensureTestUsers();
|
|
92
|
-
const userId =
|
|
91
|
+
const userId = created[1]._id;
|
|
93
92
|
const history = await GptHistory.create({
|
|
94
93
|
prompts: [{text: "Hello", type: "user"}],
|
|
95
94
|
userId,
|
|
@@ -135,7 +134,7 @@ describe("POST /gpt/actions", () => {
|
|
|
135
134
|
|
|
136
135
|
it("returns 504 and logs the timeout", async () => {
|
|
137
136
|
const created = await ensureTestUsers();
|
|
138
|
-
const userId =
|
|
137
|
+
const userId = created[1]._id;
|
|
139
138
|
const history = await GptHistory.create({
|
|
140
139
|
prompts: [{text: "Hello", type: "user"}],
|
|
141
140
|
userId,
|
|
@@ -166,7 +165,7 @@ describe("POST /gpt/actions", () => {
|
|
|
166
165
|
|
|
167
166
|
it("returns 500 when the handler's document is invalid", async () => {
|
|
168
167
|
const created = await ensureTestUsers();
|
|
169
|
-
const userId =
|
|
168
|
+
const userId = created[1]._id;
|
|
170
169
|
const history = await GptHistory.create({
|
|
171
170
|
prompts: [{text: "Hello", type: "user"}],
|
|
172
171
|
userId,
|
|
@@ -230,7 +229,7 @@ describe("POST /gpt/actions", () => {
|
|
|
230
229
|
|
|
231
230
|
it("rejects a replace value the client does not understand", async () => {
|
|
232
231
|
const created = await ensureTestUsers();
|
|
233
|
-
const userId =
|
|
232
|
+
const userId = created[1]._id;
|
|
234
233
|
const history = await GptHistory.create({
|
|
235
234
|
prompts: [{text: "Hello", type: "user"}],
|
|
236
235
|
userId,
|
|
@@ -291,7 +290,7 @@ describe("POST /gpt/actions with scaleStepperHostAction", () => {
|
|
|
291
290
|
assistantPrompts: string[]
|
|
292
291
|
): Promise<{agent: Awaited<ReturnType<typeof authAsUser>>; historyId: string}> => {
|
|
293
292
|
const created = await ensureTestUsers();
|
|
294
|
-
const userId =
|
|
293
|
+
const userId = created[1]._id;
|
|
295
294
|
const history = await GptHistory.create({
|
|
296
295
|
prompts: [
|
|
297
296
|
{text: "Plan a roast", type: "user"},
|
|
@@ -403,7 +402,7 @@ describe("POST /gpt/actions with toggleChecklistHostAction", () => {
|
|
|
403
402
|
assistantPrompts: string[]
|
|
404
403
|
): Promise<{agent: Awaited<ReturnType<typeof authAsUser>>; historyId: string}> => {
|
|
405
404
|
const created = await ensureTestUsers();
|
|
406
|
-
const userId =
|
|
405
|
+
const userId = created[1]._id;
|
|
407
406
|
const history = await GptHistory.create({
|
|
408
407
|
prompts: [
|
|
409
408
|
{text: "Plan a roast", type: "user"},
|
|
@@ -39,7 +39,7 @@ describe("GET /gpt/datasets/:id", () => {
|
|
|
39
39
|
it("buckets by week, downsamples with LTTB, and paginates tables", async () => {
|
|
40
40
|
await ensureTestUsers();
|
|
41
41
|
const created = await ensureTestUsers();
|
|
42
|
-
const userId =
|
|
42
|
+
const userId = created[1]._id;
|
|
43
43
|
const rows = hourlyRows(12_480);
|
|
44
44
|
const registered = await registerAiDataset({
|
|
45
45
|
columns,
|
|
@@ -279,7 +279,7 @@ describe("/gpt/prompt files asks", () => {
|
|
|
279
279
|
},
|
|
280
280
|
]);
|
|
281
281
|
|
|
282
|
-
const output = receiptResultOf(model, 1) as {value: {text?: string}[]};
|
|
282
|
+
const output = receiptResultOf(model, 1) as {value: {text?: string; type?: string}[]};
|
|
283
283
|
expect(output.value.slice(1)).toEqual([
|
|
284
284
|
{text: "File 1 of 1: ledger.csv (text/csv, 100051 bytes)", type: "text"},
|
|
285
285
|
{text: "a".repeat(99_999), type: "text"},
|