@terreno/ai 57.19.2 → 57.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/CLAUDE.local.md +1 -1
- package/.cursor/rules/00-root.mdc +1 -1
- package/.github/copilot-instructions.md +1 -1
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +443 -0
- package/biome.jsonc +1 -0
- package/dist/harness/agentLoop.js +1 -0
- package/dist/observability/local/experimentRunner.d.ts +2 -5
- package/dist/types/harness.d.ts +5 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +14 -12
- package/src/aiApp.test.ts +22 -17
- package/src/harness/agentLoop.test.ts +1 -1
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +31 -28
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +7 -6
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +3 -2
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -3
- package/src/isolated/gptStream.isolated.ts +33 -28
- package/src/isolated/langfuseClient.isolated.ts +9 -3
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/langfuseApp.test.ts +4 -20
- package/src/langfuseRoutes.test.ts +20 -15
- package/src/langfuseTracing.test.ts +6 -2
- package/src/langfuseVercelAi.test.ts +12 -9
- package/src/models/gptHistory.test.ts +10 -6
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +2 -2
- package/src/observability/local/promptStore.test.ts +4 -3
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +2 -2
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +2 -2
- package/src/routes/files.test.ts +5 -3
- package/src/routes/gpt.test.ts +21 -14
- package/src/routes/gptActions.test.ts +7 -8
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +1 -1
- package/src/routes/gptHistories.test.ts +15 -9
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +58 -42
- package/src/service/aiService.test.ts +1 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/tests/bunSetup.ts +1 -1
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/fakeLangfusePrompt.ts +10 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +6 -2
- package/tsconfig.typecheck.json +18 -0
|
@@ -3,16 +3,19 @@ import {beforeEach, describe, expect, it, mock} from "bun:test";
|
|
|
3
3
|
import {LangfuseCache} from "./langfuseCache";
|
|
4
4
|
import {getLangfuseClient, initLangfuseClient} from "./langfuseClient";
|
|
5
5
|
import {createTelemetryConfig, preparePromptForAI} from "./langfuseVercelAi";
|
|
6
|
+
import type {FakeLangfusePrompt} from "./tests/fakeLangfusePrompt";
|
|
6
7
|
|
|
7
|
-
const mockPromptGet = mock(
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
8
|
+
const mockPromptGet = mock(
|
|
9
|
+
async (_name: string, _options?: Record<string, unknown>): Promise<FakeLangfusePrompt> => ({
|
|
10
|
+
config: {temperature: 0.5},
|
|
11
|
+
labels: [],
|
|
12
|
+
name: "welcome",
|
|
13
|
+
prompt: "Hi {{name}}",
|
|
14
|
+
tags: [],
|
|
15
|
+
type: "text" as const,
|
|
16
|
+
version: 3,
|
|
17
|
+
})
|
|
18
|
+
);
|
|
16
19
|
|
|
17
20
|
describe("langfuseVercelAi", () => {
|
|
18
21
|
beforeEach(async () => {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
2
|
import mongoose from "mongoose";
|
|
3
3
|
|
|
4
|
+
import type {GptHistoryDocument, GptHistoryPendingAsk, GptHistoryPrompt} from "../types/index";
|
|
4
5
|
import {GptHistory} from "./gptHistory";
|
|
5
6
|
|
|
6
7
|
describe("GptHistory Model", () => {
|
|
@@ -217,9 +218,9 @@ describe("GptHistory Model", () => {
|
|
|
217
218
|
{id: "team", label: "Team"},
|
|
218
219
|
],
|
|
219
220
|
prompt: "Which plan should I set up?",
|
|
220
|
-
select: "one",
|
|
221
|
+
select: "one" as const,
|
|
221
222
|
};
|
|
222
|
-
const pendingAsk = {
|
|
223
|
+
const pendingAsk: GptHistoryPendingAsk = {
|
|
223
224
|
created: new Date("2026-09-27T12:00:00.000Z"),
|
|
224
225
|
input: planAsk,
|
|
225
226
|
kind: "choice",
|
|
@@ -256,7 +257,10 @@ describe("GptHistory Model", () => {
|
|
|
256
257
|
},
|
|
257
258
|
toolCallId: "call_plan",
|
|
258
259
|
};
|
|
259
|
-
|
|
260
|
+
/** Create from deliberately invalid input, so the schema rather than the compiler rejects it. */
|
|
261
|
+
const createUnchecked = (doc: Record<string, unknown>) =>
|
|
262
|
+
GptHistory.create(doc as Partial<GptHistoryDocument>);
|
|
263
|
+
const askRow: GptHistoryPrompt = {
|
|
260
264
|
args: planAsk,
|
|
261
265
|
ask: {kind: "choice", status: "pending"},
|
|
262
266
|
text: "Tool call: ask_choice",
|
|
@@ -309,7 +313,7 @@ describe("GptHistory Model", () => {
|
|
|
309
313
|
const userId = new mongoose.Types.ObjectId();
|
|
310
314
|
const history = await GptHistory.create({prompts: [], userId});
|
|
311
315
|
|
|
312
|
-
history.pendingAsk = pendingAsk
|
|
316
|
+
history.pendingAsk = pendingAsk;
|
|
313
317
|
await history.save();
|
|
314
318
|
|
|
315
319
|
const stored = await GptHistory.collection.findOne({_id: history._id});
|
|
@@ -334,7 +338,7 @@ describe("GptHistory Model", () => {
|
|
|
334
338
|
},
|
|
335
339
|
])("rejects an unknown key in $field", async ({expected, history}) => {
|
|
336
340
|
await expect(
|
|
337
|
-
|
|
341
|
+
createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
|
|
338
342
|
).rejects.toThrow(expected);
|
|
339
343
|
});
|
|
340
344
|
|
|
@@ -361,7 +365,7 @@ describe("GptHistory Model", () => {
|
|
|
361
365
|
},
|
|
362
366
|
])("rejects $label", async ({expected, history}) => {
|
|
363
367
|
await expect(
|
|
364
|
-
|
|
368
|
+
createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
|
|
365
369
|
).rejects.toThrow(expected);
|
|
366
370
|
});
|
|
367
371
|
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, it} from "bun:test";
|
|
2
2
|
import {assert} from "chai";
|
|
3
|
+
import {judgeAnswering} from "../tests/fakeJudge";
|
|
3
4
|
import {
|
|
4
5
|
judgeSchemaMissingDimensions,
|
|
5
6
|
runEvaluator,
|
|
@@ -95,9 +96,7 @@ describe("evaluate", () => {
|
|
|
95
96
|
output: "4",
|
|
96
97
|
},
|
|
97
98
|
{
|
|
98
|
-
generateJsonObject:
|
|
99
|
-
return {confidence: 0.95, correct: true};
|
|
100
|
-
},
|
|
99
|
+
generateJsonObject: judgeAnswering(() => ({confidence: 0.95, correct: true})),
|
|
101
100
|
},
|
|
102
101
|
{
|
|
103
102
|
properties: {
|
|
@@ -152,7 +151,7 @@ describe("evaluate", () => {
|
|
|
152
151
|
const base = {
|
|
153
152
|
evaluator: {
|
|
154
153
|
assertion: {constraint: "exists", path: "answer"},
|
|
155
|
-
dimensions: [{dataType: "boolean", key: "matches", required: true}],
|
|
154
|
+
dimensions: [{dataType: "boolean" as const, key: "matches", required: true}],
|
|
156
155
|
name: "path-assert",
|
|
157
156
|
type: "json-assert" as const,
|
|
158
157
|
},
|
|
@@ -389,7 +388,7 @@ describe("evaluate", () => {
|
|
|
389
388
|
input: {},
|
|
390
389
|
output: "4",
|
|
391
390
|
},
|
|
392
|
-
{generateJsonObject:
|
|
391
|
+
{generateJsonObject: judgeAnswering(() => ({correct: true}))},
|
|
393
392
|
{properties: {correct: {type: "boolean"}}, type: "object"}
|
|
394
393
|
);
|
|
395
394
|
assert.equal(missingJudgePrompt.error, "llm-judge evaluator is missing judgePromptName");
|
|
@@ -408,7 +407,7 @@ describe("evaluate", () => {
|
|
|
408
407
|
},
|
|
409
408
|
{
|
|
410
409
|
ai: {
|
|
411
|
-
generateJsonObject:
|
|
410
|
+
generateJsonObject: judgeAnswering(() => ({confidence: "not-a-number", correct: 1})),
|
|
412
411
|
},
|
|
413
412
|
judgeOutputSchema: {
|
|
414
413
|
properties: {
|
|
@@ -461,9 +460,12 @@ describe("evaluate", () => {
|
|
|
461
460
|
output: "4",
|
|
462
461
|
},
|
|
463
462
|
{
|
|
464
|
-
generateJsonObject:
|
|
465
|
-
|
|
466
|
-
|
|
463
|
+
generateJsonObject: judgeAnswering(() => ({
|
|
464
|
+
confidence: "0.8",
|
|
465
|
+
correct: "true",
|
|
466
|
+
helpfulness: "not-a-number",
|
|
467
|
+
tone: 1,
|
|
468
|
+
})),
|
|
467
469
|
},
|
|
468
470
|
{
|
|
469
471
|
properties: {
|
|
@@ -489,7 +491,7 @@ describe("LocalEvaluatorStore phase 2", () => {
|
|
|
489
491
|
beforeEach(async () => {
|
|
490
492
|
new ObservabilityApp({plugins: [createLocalObservabilityBundle().plugin]});
|
|
491
493
|
promptStore = new LocalPromptStore();
|
|
492
|
-
store = new (await import("./local/evaluatorStore")).LocalEvaluatorStore(promptStore);
|
|
494
|
+
store = new (await import("./local/evaluatorStore.js")).LocalEvaluatorStore(promptStore);
|
|
493
495
|
await registerObsEvaluator().deleteMany({});
|
|
494
496
|
await clearObsPrompts();
|
|
495
497
|
await promptStore.create({
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, it} from "bun:test";
|
|
2
|
-
import {BackgroundTask} from "@terreno/api";
|
|
2
|
+
import {BackgroundTask, type BackgroundTaskDocument, type FindExactlyOnePlugin} from "@terreno/api";
|
|
3
3
|
import {assert} from "chai";
|
|
4
4
|
import mongoose from "mongoose";
|
|
5
|
-
|
|
5
|
+
import {judgeAnswering} from "../../tests/fakeJudge";
|
|
6
6
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
7
7
|
import {LocalDatasetStore} from "./datasetStore";
|
|
8
8
|
import {LocalEvaluatorStore} from "./evaluatorStore";
|
|
@@ -37,9 +37,7 @@ const createTestRunner = (params: {
|
|
|
37
37
|
});
|
|
38
38
|
runner.configureAi({
|
|
39
39
|
aiService: {
|
|
40
|
-
generateJsonObject:
|
|
41
|
-
return {correct: true};
|
|
42
|
-
},
|
|
40
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
43
41
|
generateText: params.generateText,
|
|
44
42
|
modelId: "mock-model",
|
|
45
43
|
},
|
|
@@ -340,9 +338,7 @@ describe("LocalExperimentRunner", () => {
|
|
|
340
338
|
});
|
|
341
339
|
defaultRunner.configureAi({
|
|
342
340
|
aiService: {
|
|
343
|
-
generateJsonObject:
|
|
344
|
-
return {correct: true};
|
|
345
|
-
},
|
|
341
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
346
342
|
generateText: async () => {
|
|
347
343
|
return JSON.stringify({answer: "ok"});
|
|
348
344
|
},
|
|
@@ -399,9 +395,7 @@ describe("LocalExperimentRunner", () => {
|
|
|
399
395
|
});
|
|
400
396
|
failingRunner.configureAi({
|
|
401
397
|
aiService: {
|
|
402
|
-
generateJsonObject:
|
|
403
|
-
return {correct: true};
|
|
404
|
-
},
|
|
398
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
405
399
|
generateText: async () => {
|
|
406
400
|
throw new Error("generation failed");
|
|
407
401
|
},
|
|
@@ -427,7 +421,10 @@ describe("LocalExperimentRunner", () => {
|
|
|
427
421
|
});
|
|
428
422
|
detail = await failingRunner.get(experiment.id);
|
|
429
423
|
}
|
|
430
|
-
|
|
424
|
+
// @terreno/api applies findExactlyOne to BackgroundTask, but its model type does not declare it.
|
|
425
|
+
const tasks = BackgroundTask as typeof BackgroundTask &
|
|
426
|
+
FindExactlyOnePlugin<BackgroundTaskDocument>;
|
|
427
|
+
const task = await tasks.findExactlyOne({_id: detail.backgroundTaskId});
|
|
431
428
|
assert.equal(task.status, "failed");
|
|
432
429
|
assert.include(task.error ?? "", "generation failed");
|
|
433
430
|
});
|
|
@@ -22,7 +22,7 @@ import type {LocalDatasetStore} from "./datasetStore";
|
|
|
22
22
|
import type {LocalEvaluatorStore} from "./evaluatorStore";
|
|
23
23
|
import {registerObsExperiment} from "./models/obsExperiment";
|
|
24
24
|
import {registerObsExperimentItem} from "./models/obsExperimentItem";
|
|
25
|
-
import type {LocalPromptStore} from "./promptStore";
|
|
25
|
+
import type {LocalPromptStore, MoveLabelResult} from "./promptStore";
|
|
26
26
|
|
|
27
27
|
export interface ExperimentCreateInput {
|
|
28
28
|
datasetId: string;
|
|
@@ -354,7 +354,7 @@ export class LocalExperimentRunner {
|
|
|
354
354
|
};
|
|
355
355
|
}
|
|
356
356
|
|
|
357
|
-
async promote(id: string, version: number): Promise<
|
|
357
|
+
async promote(id: string, version: number): Promise<MoveLabelResult> {
|
|
358
358
|
const experiment = await this.get(id);
|
|
359
359
|
if (!experiment.results) {
|
|
360
360
|
throw new APIError({status: 409, title: "Experiment has not finished running"});
|
|
@@ -10,7 +10,7 @@ import {registerObsPrompt} from "./models/obsPrompt";
|
|
|
10
10
|
import {registerObsPromptLabel} from "./models/obsPromptLabel";
|
|
11
11
|
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
12
12
|
import {registerObsTrace} from "./models/obsTrace";
|
|
13
|
-
import {LocalPromptStore} from "./promptStore";
|
|
13
|
+
import {type CreatePromptInput, LocalPromptStore} from "./promptStore";
|
|
14
14
|
|
|
15
15
|
describe("LocalPromptStore", () => {
|
|
16
16
|
let store: LocalPromptStore;
|
|
@@ -253,7 +253,7 @@ describe("LocalPromptStore", () => {
|
|
|
253
253
|
|
|
254
254
|
it("validates create/moveLabel inputs and resolves versions by label or number", async () => {
|
|
255
255
|
try {
|
|
256
|
-
await store.create({folder: "", name: "bad"});
|
|
256
|
+
await store.create({folder: "", name: "bad"} as CreatePromptInput);
|
|
257
257
|
expect.unreachable();
|
|
258
258
|
} catch (error) {
|
|
259
259
|
expect(String(error)).toMatch(/folder and name/);
|
|
@@ -334,9 +334,10 @@ describe("LocalPromptStore", () => {
|
|
|
334
334
|
const latest = await store.runPlayground({
|
|
335
335
|
generator: {
|
|
336
336
|
generate: async () => {
|
|
337
|
-
return {output: "latest"};
|
|
337
|
+
return {latencyMs: 0, output: "latest"};
|
|
338
338
|
},
|
|
339
339
|
},
|
|
340
|
+
modelId: "mock-model",
|
|
340
341
|
name: "playground",
|
|
341
342
|
variables: {name: "Grace"},
|
|
342
343
|
});
|
|
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
4
|
|
|
5
|
-
import {authAsUser, ensureTestUsers,
|
|
5
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
6
6
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
7
7
|
import {registerObsDataset} from "../local/models/obsDataset";
|
|
8
8
|
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
@@ -32,7 +32,7 @@ describe("observability dataset routes", () => {
|
|
|
32
32
|
await registerObsPromptLabel().deleteMany({});
|
|
33
33
|
promptStore = new LocalPromptStore();
|
|
34
34
|
const plugin = createLocalObservabilityPlugin();
|
|
35
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
35
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
36
36
|
.register(new ObservabilityApp({plugins: [plugin]}))
|
|
37
37
|
.build();
|
|
38
38
|
});
|
|
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
4
|
|
|
5
|
-
import {authAsUser, ensureTestUsers,
|
|
5
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
6
6
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
7
7
|
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
8
8
|
import {registerObsPrompt} from "../local/models/obsPrompt";
|
|
@@ -40,7 +40,7 @@ describe("observability evaluator routes", () => {
|
|
|
40
40
|
type: "text",
|
|
41
41
|
});
|
|
42
42
|
await promptStore.moveLabel("eval-judge-correctness", {label: "production", version: 1});
|
|
43
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
43
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
44
44
|
.register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
|
|
45
45
|
.build();
|
|
46
46
|
});
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {AIService} from "../../service/aiService";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
9
|
import {registerObsDataset} from "../local/models/obsDataset";
|
|
10
10
|
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
@@ -70,7 +70,7 @@ describe("observability experiment routes", () => {
|
|
|
70
70
|
},
|
|
71
71
|
},
|
|
72
72
|
});
|
|
73
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
73
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
74
74
|
.register(
|
|
75
75
|
new ObservabilityApp({
|
|
76
76
|
aiService,
|
|
@@ -8,7 +8,7 @@ import mongoose from "mongoose";
|
|
|
8
8
|
|
|
9
9
|
import {AIRequest} from "../../models/aiRequest";
|
|
10
10
|
import {AIService} from "../../service/aiService";
|
|
11
|
-
import {authAsUser, ensureTestUsers,
|
|
11
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
12
12
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
13
13
|
import {registerObsExperiment} from "../local/models/obsExperiment";
|
|
14
14
|
import {registerObsPrompt} from "../local/models/obsPrompt";
|
|
@@ -61,7 +61,7 @@ describe("observability prompt routes", () => {
|
|
|
61
61
|
const model = createMockModel();
|
|
62
62
|
doGenerate = model.doGenerate;
|
|
63
63
|
aiService = new AIService({model: model as unknown as LanguageModel});
|
|
64
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
64
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
65
65
|
.register(
|
|
66
66
|
new ObservabilityApp({
|
|
67
67
|
aiService,
|
|
@@ -134,7 +134,7 @@ describe("observability prompt routes", () => {
|
|
|
134
134
|
const requestAiServiceFactory = mock(({apiKey}: {apiKey?: string}) => {
|
|
135
135
|
return apiKey === "request-key" ? aiService : undefined;
|
|
136
136
|
});
|
|
137
|
-
const requestServiceApp = new TerrenoApp({skipListen: true, userModel:
|
|
137
|
+
const requestServiceApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
138
138
|
.register(
|
|
139
139
|
new ObservabilityApp({
|
|
140
140
|
plugins: [createLocalObservabilityPlugin()],
|
|
@@ -4,7 +4,7 @@ import {assert} from "chai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
import {DateTime} from "luxon";
|
|
6
6
|
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {LocalEvaluatorStore} from "../local/evaluatorStore";
|
|
9
9
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
10
10
|
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
@@ -28,7 +28,7 @@ describe("observability review routes", () => {
|
|
|
28
28
|
await registerObsReviewItem().deleteMany({});
|
|
29
29
|
await registerObsEvaluator().deleteMany({});
|
|
30
30
|
await registerObsTrace().deleteMany({});
|
|
31
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
31
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
32
32
|
.register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
|
|
33
33
|
.build();
|
|
34
34
|
});
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {AIService} from "../../service/aiService";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
9
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
10
10
|
import type {ObservabilityPlugin} from "../types";
|
|
@@ -33,7 +33,7 @@ describe("observability status routes", () => {
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
beforeEach(() => {
|
|
36
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
36
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
37
37
|
.register(
|
|
38
38
|
new ObservabilityApp({
|
|
39
39
|
plugins: [createLocalObservabilityPlugin(), createLangfusePlugin()],
|
|
@@ -89,7 +89,7 @@ describe("observability status routes", () => {
|
|
|
89
89
|
specificationVersion: "v2" as const,
|
|
90
90
|
supportedUrls: {},
|
|
91
91
|
};
|
|
92
|
-
const serverApp = new TerrenoApp({skipListen: true, userModel:
|
|
92
|
+
const serverApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
93
93
|
.register(
|
|
94
94
|
new ObservabilityApp({
|
|
95
95
|
aiService: new AIService({model: model as unknown as LanguageModel}),
|
|
@@ -101,7 +101,7 @@ describe("observability status routes", () => {
|
|
|
101
101
|
const serverStatus = await serverAgent.get("/ai/observability/status");
|
|
102
102
|
expect(serverStatus.body.data.playgroundAi).toEqual({source: "server"});
|
|
103
103
|
|
|
104
|
-
const requestKeyApp = new TerrenoApp({skipListen: true, userModel:
|
|
104
|
+
const requestKeyApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
105
105
|
.register(
|
|
106
106
|
new ObservabilityApp({
|
|
107
107
|
plugins: [createLocalObservabilityPlugin()],
|
|
@@ -3,7 +3,7 @@ import {TerrenoApp} from "@terreno/api";
|
|
|
3
3
|
import {assert} from "chai";
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
|
-
import {authAsUser, ensureTestUsers,
|
|
6
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
7
7
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
8
8
|
import {registerObsSpan} from "../local/models/obsSpan";
|
|
9
9
|
import {registerObsTrace} from "../local/models/obsTrace";
|
|
@@ -86,7 +86,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
86
86
|
await registerObsSpan().deleteMany({});
|
|
87
87
|
const plugin = createLocalObservabilityPlugin();
|
|
88
88
|
fakeAiService = createFakeAiService();
|
|
89
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
89
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
90
90
|
.register(new ObservabilityApp({aiService: fakeAiService, plugins: [plugin]}))
|
|
91
91
|
.build();
|
|
92
92
|
});
|
|
@@ -125,7 +125,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
125
125
|
|
|
126
126
|
it("returns 503 when no server or request AI service is available", async () => {
|
|
127
127
|
const plugin = createLocalObservabilityPlugin();
|
|
128
|
-
const noAiApp = new TerrenoApp({skipListen: true, userModel:
|
|
128
|
+
const noAiApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
129
129
|
.register(new ObservabilityApp({plugins: [plugin]}))
|
|
130
130
|
.build();
|
|
131
131
|
const agent = await authAsUser(noAiApp, "admin");
|
|
@@ -141,7 +141,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
141
141
|
return apiKey === "request-key" ? createFakeAiService() : undefined;
|
|
142
142
|
});
|
|
143
143
|
const plugin = createLocalObservabilityPlugin();
|
|
144
|
-
const requestKeyApp = new TerrenoApp({skipListen: true, userModel:
|
|
144
|
+
const requestKeyApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
145
145
|
.register(new ObservabilityApp({plugins: [plugin], requestAiServiceFactory}))
|
|
146
146
|
.build();
|
|
147
147
|
const agent = await authAsUser(requestKeyApp, "admin");
|
|
@@ -159,7 +159,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
159
159
|
it("exports an error trace and rethrows when a child LLM stage fails", async () => {
|
|
160
160
|
const failingService = createFakeAiService({failOn: OBS_TEST_MULTI_STAGE_CALL_2_SCHEMA_NAME});
|
|
161
161
|
const plugin = createLocalObservabilityPlugin();
|
|
162
|
-
const failingApp = new TerrenoApp({skipListen: true, userModel:
|
|
162
|
+
const failingApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
163
163
|
.register(new ObservabilityApp({aiService: failingService, plugins: [plugin]}))
|
|
164
164
|
.build();
|
|
165
165
|
const agent = await authAsUser(failingApp, "admin");
|
|
@@ -186,7 +186,7 @@ describe("observability test-multi-stage route", () => {
|
|
|
186
186
|
it("exports the completed child stages when final synthesis fails", async () => {
|
|
187
187
|
const failingService = createFakeAiService({failOn: OBS_TEST_MULTI_STAGE_FINAL_SCHEMA_NAME});
|
|
188
188
|
const plugin = createLocalObservabilityPlugin();
|
|
189
|
-
const failingApp = new TerrenoApp({skipListen: true, userModel:
|
|
189
|
+
const failingApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
190
190
|
.register(new ObservabilityApp({aiService: failingService, plugins: [plugin]}))
|
|
191
191
|
.build();
|
|
192
192
|
const agent = await authAsUser(failingApp, "admin");
|
|
@@ -3,7 +3,7 @@ import {TerrenoApp} from "@terreno/api";
|
|
|
3
3
|
import type express from "express";
|
|
4
4
|
import {DateTime} from "luxon";
|
|
5
5
|
|
|
6
|
-
import {authAsUser, ensureTestUsers,
|
|
6
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
7
7
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
8
8
|
import {registerObsScore} from "../local/models/obsScore";
|
|
9
9
|
import {registerObsSpan} from "../local/models/obsSpan";
|
|
@@ -32,7 +32,7 @@ describe("observability trace routes", () => {
|
|
|
32
32
|
plugin.traceSink && "store" in plugin.traceSink
|
|
33
33
|
? (plugin.traceSink as {store: LocalTraceStore}).store
|
|
34
34
|
: new LocalTraceStore();
|
|
35
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
35
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
36
36
|
.register(new ObservabilityApp({plugins: [plugin]}))
|
|
37
37
|
.build();
|
|
38
38
|
});
|
package/src/routes/files.test.ts
CHANGED
|
@@ -6,7 +6,7 @@ import supertest from "supertest";
|
|
|
6
6
|
|
|
7
7
|
import {FileAttachment} from "../models/fileAttachment";
|
|
8
8
|
import type {FileStorageService, UploadFileParams, UploadFileResult} from "../service/fileStorage";
|
|
9
|
-
import {authAsUser, ensureTestUsers, UserModel} from "../tests/helpers";
|
|
9
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel, UserModel} from "../tests/helpers";
|
|
10
10
|
import {addFileRoutes} from "./files";
|
|
11
11
|
|
|
12
12
|
type MockFileStorageService = Pick<FileStorageService, "upload" | "delete" | "getSignedUrl">;
|
|
@@ -45,11 +45,12 @@ describe("File Routes", () => {
|
|
|
45
45
|
configureApp: (router, options) => {
|
|
46
46
|
addFileRoutes(router, {
|
|
47
47
|
fileStorageService: fileStorageService as FileStorageService,
|
|
48
|
+
gcsBucket: "test-bucket",
|
|
48
49
|
openApiOptions: options,
|
|
49
50
|
});
|
|
50
51
|
},
|
|
51
52
|
skipListen: true,
|
|
52
|
-
userModel:
|
|
53
|
+
userModel: terrenoUserModel,
|
|
53
54
|
}).build();
|
|
54
55
|
});
|
|
55
56
|
|
|
@@ -76,11 +77,12 @@ describe("File Routes", () => {
|
|
|
76
77
|
addFileRoutes(router, {
|
|
77
78
|
fileStorageService: fileStorageService as FileStorageService,
|
|
78
79
|
fileUploadsEnabled: async () => false,
|
|
80
|
+
gcsBucket: "test-bucket",
|
|
79
81
|
openApiOptions: options,
|
|
80
82
|
});
|
|
81
83
|
},
|
|
82
84
|
skipListen: true,
|
|
83
|
-
userModel:
|
|
85
|
+
userModel: terrenoUserModel,
|
|
84
86
|
}).build();
|
|
85
87
|
const agent = await authAsUser(gated, "notAdmin");
|
|
86
88
|
const res = await agent
|
package/src/routes/gpt.test.ts
CHANGED
|
@@ -51,7 +51,13 @@ import {
|
|
|
51
51
|
toolNamesOf,
|
|
52
52
|
USER_PROMPT,
|
|
53
53
|
} from "../tests/chatHarness";
|
|
54
|
-
import {
|
|
54
|
+
import {
|
|
55
|
+
authAsUser,
|
|
56
|
+
ensureTestUsers,
|
|
57
|
+
historyRouterOptions,
|
|
58
|
+
terrenoUserModel,
|
|
59
|
+
UserModel,
|
|
60
|
+
} from "../tests/helpers";
|
|
55
61
|
import type {GptHistoryDocument} from "../types";
|
|
56
62
|
import {addGptRoutes} from "./gpt";
|
|
57
63
|
import {addGptHistoryRoutes} from "./gptHistories";
|
|
@@ -2840,7 +2846,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2840
2846
|
asks: true,
|
|
2841
2847
|
};
|
|
2842
2848
|
addGptHistoryRoutes(router, {
|
|
2843
|
-
...options,
|
|
2849
|
+
...historyRouterOptions(options),
|
|
2844
2850
|
chat,
|
|
2845
2851
|
responseHandler: async (value) => {
|
|
2846
2852
|
const history = (value as GptHistoryDocument).toObject();
|
|
@@ -2850,7 +2856,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2850
2856
|
addGptRoutes(router, chat);
|
|
2851
2857
|
},
|
|
2852
2858
|
skipListen: true,
|
|
2853
|
-
userModel:
|
|
2859
|
+
userModel: terrenoUserModel,
|
|
2854
2860
|
}).build();
|
|
2855
2861
|
const agent = await authAsUser(app, "notAdmin");
|
|
2856
2862
|
const historyId = await pauseOnPlanAsk(agent);
|
|
@@ -2894,7 +2900,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2894
2900
|
const app = new TerrenoApp({
|
|
2895
2901
|
configureApp: (router, options) => {
|
|
2896
2902
|
addGptHistoryRoutes(router, {
|
|
2897
|
-
...options,
|
|
2903
|
+
...historyRouterOptions(options),
|
|
2898
2904
|
openApiOverwrite: {
|
|
2899
2905
|
create: {summary: "Start a conversation"},
|
|
2900
2906
|
update: {requestBody: {description: "The fields to change"}},
|
|
@@ -2902,7 +2908,7 @@ describe("/gpt/prompt asks", () => {
|
|
|
2902
2908
|
});
|
|
2903
2909
|
},
|
|
2904
2910
|
skipListen: true,
|
|
2905
|
-
userModel:
|
|
2911
|
+
userModel: terrenoUserModel,
|
|
2906
2912
|
}).build();
|
|
2907
2913
|
const agent = await authAsUser(app, "notAdmin");
|
|
2908
2914
|
|
|
@@ -2920,12 +2926,12 @@ describe("/gpt/prompt asks", () => {
|
|
|
2920
2926
|
const app = new TerrenoApp({
|
|
2921
2927
|
configureApp: (router, options) => {
|
|
2922
2928
|
addGptHistoryRoutes(router, {
|
|
2923
|
-
...options,
|
|
2929
|
+
...historyRouterOptions(options),
|
|
2924
2930
|
preUpdate: (body) => ({...body, title: `${body.title} (edited)`}) as GptHistoryDocument,
|
|
2925
2931
|
});
|
|
2926
2932
|
},
|
|
2927
2933
|
skipListen: true,
|
|
2928
|
-
userModel:
|
|
2934
|
+
userModel: terrenoUserModel,
|
|
2929
2935
|
}).build();
|
|
2930
2936
|
const agent = await authAsUser(app, "notAdmin");
|
|
2931
2937
|
const created = await agent
|
|
@@ -2980,13 +2986,14 @@ const STEPPER_HOST_ACTIONS = {
|
|
|
2980
2986
|
scaleStepper: {handles: "stepper" as const},
|
|
2981
2987
|
};
|
|
2982
2988
|
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2989
|
+
interface BlocksEvent {
|
|
2990
|
+
errors: {code: string; path: string}[];
|
|
2991
|
+
ok: boolean;
|
|
2992
|
+
warnings?: unknown[];
|
|
2993
|
+
}
|
|
2994
|
+
|
|
2995
|
+
const blocksEventOf = (events: Record<string, unknown>[]): BlocksEvent =>
|
|
2996
|
+
events.find((event) => "blocks" in event)?.blocks as BlocksEvent;
|
|
2990
2997
|
|
|
2991
2998
|
describe("/gpt/prompt uiBlocks", () => {
|
|
2992
2999
|
beforeAll(async () => {
|