@terreno/ai 57.20.0 → 58.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +20 -18
- package/CHANGELOG.md +93 -0
- package/biome.jsonc +3 -2
- package/dist/aiApp.d.ts +5 -2
- package/dist/aiApp.js +3 -2
- package/dist/harness/agentLoop.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/migrations/gptHistoryStringIds.d.ts +41 -0
- package/dist/migrations/gptHistoryStringIds.js +94 -0
- package/dist/models/aiDataset.js +2 -2
- package/dist/models/gptHistory.js +11 -0
- package/dist/observability/local/experimentRunner.d.ts +2 -5
- package/dist/routes/files.d.ts +7 -1
- package/dist/routes/files.js +62 -13
- package/dist/routes/gptActions.js +0 -7
- package/dist/routes/gptHistories.js +8 -2
- package/dist/service/aiDatasets.js +2 -1
- package/dist/service/chatTurn.js +4 -0
- package/dist/service/vertex.d.ts +5 -2
- package/dist/types/harness.d.ts +5 -2
- package/dist/types/index.d.ts +5 -2
- package/e2e/smoke.ts +227 -0
- package/package.json +19 -16
- package/src/aiApp.test.ts +43 -18
- package/src/aiApp.ts +6 -3
- package/src/harness/agentLoop.test.ts +1 -1
- package/src/harness/agentLoop.ts +2 -1
- package/src/harness/agentRecovery.test.ts +31 -28
- package/src/harness/approvals.test.ts +11 -9
- package/src/harness/events.test.ts +10 -6
- package/src/harness/extensions.test.ts +7 -6
- package/src/harness/ownership.test.ts +20 -9
- package/src/harness/retries.test.ts +6 -3
- package/src/harness/routes/conversations.test.ts +2 -2
- package/src/harness/routes/tasks.test.ts +2 -2
- package/src/harness/runners/jobsRunner.test.ts +1 -1
- package/src/harness/subagent.test.ts +3 -2
- package/src/harness/waits.test.ts +6 -4
- package/src/index.test.ts +3 -3
- package/src/index.ts +7 -1
- package/src/isolated/gptStream.isolated.ts +35 -30
- package/src/isolated/langfuseClient.isolated.ts +9 -3
- package/src/isolated/lazyProjectModel.isolated.ts +2 -2
- package/src/langfuseApp.test.ts +4 -20
- package/src/langfuseRoutes.test.ts +20 -15
- package/src/langfuseTracing.test.ts +6 -2
- package/src/langfuseVercelAi.test.ts +12 -9
- package/src/migrations/gptHistoryStringIds.test.ts +231 -0
- package/src/migrations/gptHistoryStringIds.ts +150 -0
- package/src/models/aiDataset.test.ts +1 -1
- package/src/models/aiDataset.ts +2 -2
- package/src/models/gptHistory.test.ts +22 -11
- package/src/models/gptHistory.ts +19 -1
- package/src/observability/evaluate.test.ts +12 -10
- package/src/observability/local/experimentRunner.test.ts +9 -12
- package/src/observability/local/experimentRunner.ts +2 -2
- package/src/observability/local/promptStore.test.ts +4 -3
- package/src/observability/routes/datasets.test.ts +2 -2
- package/src/observability/routes/evaluators.test.ts +2 -2
- package/src/observability/routes/experiments.test.ts +2 -2
- package/src/observability/routes/prompts.test.ts +3 -3
- package/src/observability/routes/review.test.ts +2 -2
- package/src/observability/routes/status.test.ts +4 -4
- package/src/observability/routes/testMultiStage.test.ts +6 -6
- package/src/observability/routes/traces.test.ts +2 -2
- package/src/routes/files.test.ts +126 -3
- package/src/routes/files.ts +90 -15
- package/src/routes/gpt.test.ts +23 -16
- package/src/routes/gptActions.test.ts +15 -15
- package/src/routes/gptActions.ts +1 -4
- package/src/routes/gptApprovals.test.ts +1 -1
- package/src/routes/gptDatasets.test.ts +1 -1
- package/src/routes/gptFiles.test.ts +2 -2
- package/src/routes/gptHistories.test.ts +17 -11
- package/src/routes/gptHistories.ts +59 -55
- package/src/routes/gptHistoriesSync.test.ts +141 -0
- package/src/routes/mcp.test.ts +3 -3
- package/src/routes/projects.test.ts +5 -3
- package/src/routes/routes.test.ts +64 -47
- package/src/service/aiDatasets.ts +2 -1
- package/src/service/aiService.test.ts +1 -1
- package/src/service/asks.test.ts +14 -6
- package/src/service/chatTurn.ts +4 -0
- package/src/service/fileStorage.test.ts +5 -2
- package/src/service/getMCPTools.test.ts +9 -5
- package/src/service/mcpService.test.ts +6 -3
- package/src/service/vertex.test.ts +1 -1
- package/src/service/vertex.ts +7 -4
- package/src/tests/bunSetup.ts +1 -1
- package/src/tests/chatHarness.ts +4 -4
- package/src/tests/fakeJudge.ts +8 -0
- package/src/tests/fakeLangfusePrompt.ts +10 -0
- package/src/tests/helpers.ts +26 -4
- package/src/types/harness.ts +6 -2
- package/src/types/index.ts +5 -2
- package/tsconfig.typecheck.json +18 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Data migration for Terreno 58: GptHistory became a synced collection with String `_id`s.
|
|
3
|
+
* Histories saved before 58 have ObjectId `_id`s, which sync clients and the String-typed
|
|
4
|
+
* schema cannot address. This migration re-inserts each one with `_id` set to the same id's
|
|
5
|
+
* hex string, so URLs and client caches that hold the hex id keep working, rewrites
|
|
6
|
+
* `AIDataset.historyId` refs to that hex string, and then deletes the ObjectId originals.
|
|
7
|
+
*
|
|
8
|
+
* It writes through the native driver: `syncPlugin` forbids the hard deletes and
|
|
9
|
+
* multi-document writes a move needs, and the copies keep every stored field as-is.
|
|
10
|
+
* Running it again is a no-op.
|
|
11
|
+
*/
|
|
12
|
+
import type {MigrationContext, MigrationLogger} from "@terreno/api";
|
|
13
|
+
import type mongoose from "mongoose";
|
|
14
|
+
|
|
15
|
+
import {AIDataset} from "../models/aiDataset";
|
|
16
|
+
import {GptHistory} from "../models/gptHistory";
|
|
17
|
+
|
|
18
|
+
/** Slug for the timestamped file an app adds to its `migrations/` directory. */
|
|
19
|
+
export const GPT_HISTORY_STRING_IDS_MIGRATION = "gpt-history-string-ids";
|
|
20
|
+
|
|
21
|
+
const DEFAULT_BATCH_SIZE = 500;
|
|
22
|
+
|
|
23
|
+
const DUPLICATE_KEY_CODE = 11000;
|
|
24
|
+
|
|
25
|
+
/** Only ObjectId `_id`s still need a move; String ids are already migrated or new. */
|
|
26
|
+
const OBJECT_ID_FILTER = {_id: {$type: "objectId" as const}};
|
|
27
|
+
|
|
28
|
+
/** Server-side ObjectId → hex string, so refs move without reading each dataset. */
|
|
29
|
+
const HISTORY_ID_TO_STRING = [{$set: {historyId: {$toString: "$historyId"}}}];
|
|
30
|
+
|
|
31
|
+
export interface GptHistoryStringIdsResult {
|
|
32
|
+
/** AIDataset rows whose ObjectId `historyId` was rewritten (or would be, in a dry run). */
|
|
33
|
+
datasetsUpdated: number;
|
|
34
|
+
/** Histories moved to a String `_id` (or that would move, in a dry run). */
|
|
35
|
+
historiesMigrated: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface WriteErrorLike {
|
|
39
|
+
code?: number;
|
|
40
|
+
writeErrors?: Array<{code?: number}>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** An insert that failed only because the hex-id copy exists from an interrupted earlier run. */
|
|
44
|
+
const isOnlyDuplicateKeyError = (error: unknown): boolean => {
|
|
45
|
+
const writeError = error as WriteErrorLike;
|
|
46
|
+
if (writeError.writeErrors?.length) {
|
|
47
|
+
return writeError.writeErrors.every((item) => item.code === DUPLICATE_KEY_CODE);
|
|
48
|
+
}
|
|
49
|
+
return writeError.code === DUPLICATE_KEY_CODE;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const insertCopies = async ({
|
|
53
|
+
collection,
|
|
54
|
+
copies,
|
|
55
|
+
}: {
|
|
56
|
+
collection: mongoose.mongo.Collection;
|
|
57
|
+
copies: mongoose.mongo.Document[];
|
|
58
|
+
}): Promise<void> => {
|
|
59
|
+
try {
|
|
60
|
+
await collection.insertMany(copies, {ordered: false});
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (!isOnlyDuplicateKeyError(error)) {
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Moves every ObjectId-id GptHistory to its hex-string id in batches and rewrites the
|
|
70
|
+
* AIDataset refs. In a dry run it only counts what it would change.
|
|
71
|
+
*/
|
|
72
|
+
export const migrateGptHistoryStringIds = async ({
|
|
73
|
+
batchSize = DEFAULT_BATCH_SIZE,
|
|
74
|
+
checkCancellation,
|
|
75
|
+
connection,
|
|
76
|
+
dryRun,
|
|
77
|
+
logger,
|
|
78
|
+
}: {
|
|
79
|
+
batchSize?: number;
|
|
80
|
+
checkCancellation?: () => Promise<void>;
|
|
81
|
+
connection: mongoose.Connection;
|
|
82
|
+
dryRun: boolean;
|
|
83
|
+
logger?: MigrationLogger;
|
|
84
|
+
}): Promise<GptHistoryStringIdsResult> => {
|
|
85
|
+
const histories = connection.collection(GptHistory.collection.collectionName);
|
|
86
|
+
const datasets = connection.collection(AIDataset.collection.collectionName);
|
|
87
|
+
const datasetFilter = {historyId: {$type: "objectId" as const}};
|
|
88
|
+
|
|
89
|
+
if (dryRun) {
|
|
90
|
+
const result = {
|
|
91
|
+
datasetsUpdated: await datasets.countDocuments(datasetFilter),
|
|
92
|
+
historiesMigrated: await histories.countDocuments(OBJECT_ID_FILTER),
|
|
93
|
+
};
|
|
94
|
+
logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] dry run`, result);
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let historiesMigrated = 0;
|
|
99
|
+
let datasetsUpdated = 0;
|
|
100
|
+
while (true) {
|
|
101
|
+
await checkCancellation?.();
|
|
102
|
+
const batch = await histories.find(OBJECT_ID_FILTER).limit(batchSize).toArray();
|
|
103
|
+
if (batch.length === 0) {
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
const originalIds = batch.map((doc) => doc._id);
|
|
107
|
+
// Copy first and delete second, so an interrupted run never loses a history: the next
|
|
108
|
+
// run finds the original again and the copy's duplicate key is ignored.
|
|
109
|
+
await insertCopies({
|
|
110
|
+
collection: histories,
|
|
111
|
+
copies: batch.map((doc) => ({...doc, _id: String(doc._id)})),
|
|
112
|
+
});
|
|
113
|
+
const moved = await datasets.updateMany({historyId: {$in: originalIds}}, HISTORY_ID_TO_STRING);
|
|
114
|
+
datasetsUpdated += moved.modifiedCount;
|
|
115
|
+
await histories.deleteMany({_id: {$in: originalIds}});
|
|
116
|
+
historiesMigrated += batch.length;
|
|
117
|
+
logger?.info(
|
|
118
|
+
`[${GPT_HISTORY_STRING_IDS_MIGRATION}] moved ${historiesMigrated} histories to String ids`
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Refs whose history was moved by an interrupted run, or never existed, still hold ObjectIds.
|
|
123
|
+
const remaining = await datasets.updateMany(datasetFilter, HISTORY_ID_TO_STRING);
|
|
124
|
+
datasetsUpdated += remaining.modifiedCount;
|
|
125
|
+
const result = {datasetsUpdated, historiesMigrated};
|
|
126
|
+
logger?.info(`[${GPT_HISTORY_STRING_IDS_MIGRATION}] done`, result);
|
|
127
|
+
return result;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Ready-made `up` for an app's `migrations/<YYYYMMDDHHmmss>-gpt-history-string-ids.ts` file.
|
|
132
|
+
* Run it before deploying Terreno 58 frontends, which address histories by String id. It has
|
|
133
|
+
* no `down`: new histories get UUIDs, which have no ObjectId to go back to.
|
|
134
|
+
*/
|
|
135
|
+
export const gptHistoryStringIdsMigration = {
|
|
136
|
+
slug: GPT_HISTORY_STRING_IDS_MIGRATION,
|
|
137
|
+
up: async (ctx: MigrationContext): Promise<void> => {
|
|
138
|
+
const result = await migrateGptHistoryStringIds({
|
|
139
|
+
checkCancellation: ctx.checkCancellation,
|
|
140
|
+
connection: ctx.mongoose.connection,
|
|
141
|
+
dryRun: ctx.dryRun,
|
|
142
|
+
logger: ctx.logger,
|
|
143
|
+
});
|
|
144
|
+
await ctx.addLog?.(
|
|
145
|
+
"info",
|
|
146
|
+
`[${GPT_HISTORY_STRING_IDS_MIGRATION}] ${ctx.dryRun ? "would move" : "moved"} ` +
|
|
147
|
+
`${result.historiesMigrated} histories and ${result.datasetsUpdated} dataset refs`
|
|
148
|
+
);
|
|
149
|
+
},
|
|
150
|
+
};
|
|
@@ -12,7 +12,7 @@ describe("AIDataset model", () => {
|
|
|
12
12
|
const userId = new mongoose.Types.ObjectId();
|
|
13
13
|
const dataset = await AIDataset.create({
|
|
14
14
|
columns: [{name: "count", type: "number"}],
|
|
15
|
-
historyId: new mongoose.Types.ObjectId(),
|
|
15
|
+
historyId: new mongoose.Types.ObjectId().toString(),
|
|
16
16
|
rowCount: 1,
|
|
17
17
|
rows: [[3]],
|
|
18
18
|
userId,
|
package/src/models/aiDataset.ts
CHANGED
|
@@ -32,11 +32,11 @@ const aiDatasetSchema: AIDatasetSchema = new mongoose.Schema(
|
|
|
32
32
|
type: Date,
|
|
33
33
|
},
|
|
34
34
|
historyId: {
|
|
35
|
-
description: "The conversation whose tool stored these rows",
|
|
35
|
+
description: "The conversation whose tool stored these rows (a GptHistory String id)",
|
|
36
36
|
index: true,
|
|
37
37
|
ref: "GptHistory",
|
|
38
38
|
required: true,
|
|
39
|
-
type:
|
|
39
|
+
type: String,
|
|
40
40
|
},
|
|
41
41
|
rowCount: {
|
|
42
42
|
description: "How many rows were stored",
|
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
2
|
import mongoose from "mongoose";
|
|
3
3
|
|
|
4
|
+
import type {GptHistoryDocument, GptHistoryPendingAsk, GptHistoryPrompt} from "../types/index";
|
|
4
5
|
import {GptHistory} from "./gptHistory";
|
|
5
6
|
|
|
7
|
+
// GptHistory ids are strings, but the driver types the raw collection with ObjectId ids.
|
|
8
|
+
const stringIdCollection = (): mongoose.mongo.Collection<{_id: string} & mongoose.mongo.Document> =>
|
|
9
|
+
GptHistory.collection as unknown as mongoose.mongo.Collection<
|
|
10
|
+
{_id: string} & mongoose.mongo.Document
|
|
11
|
+
>;
|
|
12
|
+
|
|
6
13
|
describe("GptHistory Model", () => {
|
|
7
14
|
beforeEach(async () => {
|
|
8
|
-
await GptHistory.deleteMany({});
|
|
15
|
+
await GptHistory.collection.deleteMany({});
|
|
9
16
|
});
|
|
10
17
|
|
|
11
18
|
afterEach(async () => {
|
|
12
|
-
await GptHistory.deleteMany({});
|
|
19
|
+
await GptHistory.collection.deleteMany({});
|
|
13
20
|
});
|
|
14
21
|
|
|
15
22
|
describe("schema", () => {
|
|
@@ -217,9 +224,9 @@ describe("GptHistory Model", () => {
|
|
|
217
224
|
{id: "team", label: "Team"},
|
|
218
225
|
],
|
|
219
226
|
prompt: "Which plan should I set up?",
|
|
220
|
-
select: "one",
|
|
227
|
+
select: "one" as const,
|
|
221
228
|
};
|
|
222
|
-
const pendingAsk = {
|
|
229
|
+
const pendingAsk: GptHistoryPendingAsk = {
|
|
223
230
|
created: new Date("2026-09-27T12:00:00.000Z"),
|
|
224
231
|
input: planAsk,
|
|
225
232
|
kind: "choice",
|
|
@@ -256,7 +263,10 @@ describe("GptHistory Model", () => {
|
|
|
256
263
|
},
|
|
257
264
|
toolCallId: "call_plan",
|
|
258
265
|
};
|
|
259
|
-
|
|
266
|
+
/** Create from deliberately invalid input, so the schema rather than the compiler rejects it. */
|
|
267
|
+
const createUnchecked = (doc: Record<string, unknown>) =>
|
|
268
|
+
GptHistory.create(doc as Partial<GptHistoryDocument>);
|
|
269
|
+
const askRow: GptHistoryPrompt = {
|
|
260
270
|
args: planAsk,
|
|
261
271
|
ask: {kind: "choice", status: "pending"},
|
|
262
272
|
text: "Tool call: ask_choice",
|
|
@@ -267,7 +277,8 @@ describe("GptHistory Model", () => {
|
|
|
267
277
|
|
|
268
278
|
it("loads a history saved before asks existed", async () => {
|
|
269
279
|
const userId = new mongoose.Types.ObjectId();
|
|
270
|
-
const {insertedId} = await
|
|
280
|
+
const {insertedId} = await stringIdCollection().insertOne({
|
|
281
|
+
_id: new mongoose.Types.ObjectId().toHexString(),
|
|
271
282
|
created: new Date("2026-01-05T09:00:00.000Z"),
|
|
272
283
|
deleted: false,
|
|
273
284
|
prompts: [
|
|
@@ -299,7 +310,7 @@ describe("GptHistory Model", () => {
|
|
|
299
310
|
userId,
|
|
300
311
|
});
|
|
301
312
|
|
|
302
|
-
const stored = await
|
|
313
|
+
const stored = await stringIdCollection().findOne({_id: history._id});
|
|
303
314
|
|
|
304
315
|
expect(stored?.pendingAsk).toEqual(pendingAsk);
|
|
305
316
|
expect(stored?.prompts[1].ask).toEqual({kind: "choice", status: "pending"});
|
|
@@ -309,10 +320,10 @@ describe("GptHistory Model", () => {
|
|
|
309
320
|
const userId = new mongoose.Types.ObjectId();
|
|
310
321
|
const history = await GptHistory.create({prompts: [], userId});
|
|
311
322
|
|
|
312
|
-
history.pendingAsk = pendingAsk
|
|
323
|
+
history.pendingAsk = pendingAsk;
|
|
313
324
|
await history.save();
|
|
314
325
|
|
|
315
|
-
const stored = await
|
|
326
|
+
const stored = await stringIdCollection().findOne({_id: history._id});
|
|
316
327
|
expect(stored?.pendingAsk.responseMessages[0].content[0]).toEqual({
|
|
317
328
|
input: {},
|
|
318
329
|
toolCallId: "call_lookup",
|
|
@@ -334,7 +345,7 @@ describe("GptHistory Model", () => {
|
|
|
334
345
|
},
|
|
335
346
|
])("rejects an unknown key in $field", async ({expected, history}) => {
|
|
336
347
|
await expect(
|
|
337
|
-
|
|
348
|
+
createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
|
|
338
349
|
).rejects.toThrow(expected);
|
|
339
350
|
});
|
|
340
351
|
|
|
@@ -361,7 +372,7 @@ describe("GptHistory Model", () => {
|
|
|
361
372
|
},
|
|
362
373
|
])("rejects $label", async ({expected, history}) => {
|
|
363
374
|
await expect(
|
|
364
|
-
|
|
375
|
+
createUnchecked({...history, userId: new mongoose.Types.ObjectId()})
|
|
365
376
|
).rejects.toThrow(expected);
|
|
366
377
|
});
|
|
367
378
|
});
|
package/src/models/gptHistory.ts
CHANGED
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {randomUUID} from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
createdUpdatedPlugin,
|
|
5
|
+
findExactlyOne,
|
|
6
|
+
findOneOrNone,
|
|
7
|
+
isDeletedPlugin,
|
|
8
|
+
syncPlugin,
|
|
9
|
+
} from "@terreno/api";
|
|
2
10
|
import {ASK_KINDS} from "@terreno/blocks";
|
|
3
11
|
import mongoose from "mongoose";
|
|
4
12
|
|
|
@@ -153,6 +161,14 @@ const gptHistoryPromptSchema = new mongoose.Schema(
|
|
|
153
161
|
|
|
154
162
|
const gptHistorySchema = new mongoose.Schema<GptHistoryDocument, GptHistoryModel>(
|
|
155
163
|
{
|
|
164
|
+
_id: {
|
|
165
|
+
// Synced through @terreno/syncdb: offline clients mint entity ids locally and the sync
|
|
166
|
+
// mutation channel writes them through as _id. Histories created before Terreno 58 keep
|
|
167
|
+
// their ObjectId hex string after the gpt-history-string-ids migration.
|
|
168
|
+
default: (): string => randomUUID(),
|
|
169
|
+
description: "The conversation id (a String so offline sync clients can mint ids)",
|
|
170
|
+
type: String,
|
|
171
|
+
},
|
|
156
172
|
pendingAsk: {
|
|
157
173
|
description:
|
|
158
174
|
"The ask this conversation is waiting on; cleared when the user answers or the ask is cancelled",
|
|
@@ -185,6 +201,8 @@ gptHistorySchema.plugin(createdUpdatedPlugin);
|
|
|
185
201
|
gptHistorySchema.plugin(isDeletedPlugin);
|
|
186
202
|
gptHistorySchema.plugin(findOneOrNone);
|
|
187
203
|
gptHistorySchema.plugin(findExactlyOne);
|
|
204
|
+
// Stamps a per-stream _syncSeq on every write; required by the /gpt/histories sync config.
|
|
205
|
+
gptHistorySchema.plugin(syncPlugin);
|
|
188
206
|
|
|
189
207
|
// Virtual ownerId alias so Permissions.IsOwner works with userId field
|
|
190
208
|
gptHistorySchema.virtual("ownerId").get(function () {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, it} from "bun:test";
|
|
2
2
|
import {assert} from "chai";
|
|
3
|
+
import {judgeAnswering} from "../tests/fakeJudge";
|
|
3
4
|
import {
|
|
4
5
|
judgeSchemaMissingDimensions,
|
|
5
6
|
runEvaluator,
|
|
@@ -95,9 +96,7 @@ describe("evaluate", () => {
|
|
|
95
96
|
output: "4",
|
|
96
97
|
},
|
|
97
98
|
{
|
|
98
|
-
generateJsonObject:
|
|
99
|
-
return {confidence: 0.95, correct: true};
|
|
100
|
-
},
|
|
99
|
+
generateJsonObject: judgeAnswering(() => ({confidence: 0.95, correct: true})),
|
|
101
100
|
},
|
|
102
101
|
{
|
|
103
102
|
properties: {
|
|
@@ -152,7 +151,7 @@ describe("evaluate", () => {
|
|
|
152
151
|
const base = {
|
|
153
152
|
evaluator: {
|
|
154
153
|
assertion: {constraint: "exists", path: "answer"},
|
|
155
|
-
dimensions: [{dataType: "boolean", key: "matches", required: true}],
|
|
154
|
+
dimensions: [{dataType: "boolean" as const, key: "matches", required: true}],
|
|
156
155
|
name: "path-assert",
|
|
157
156
|
type: "json-assert" as const,
|
|
158
157
|
},
|
|
@@ -389,7 +388,7 @@ describe("evaluate", () => {
|
|
|
389
388
|
input: {},
|
|
390
389
|
output: "4",
|
|
391
390
|
},
|
|
392
|
-
{generateJsonObject:
|
|
391
|
+
{generateJsonObject: judgeAnswering(() => ({correct: true}))},
|
|
393
392
|
{properties: {correct: {type: "boolean"}}, type: "object"}
|
|
394
393
|
);
|
|
395
394
|
assert.equal(missingJudgePrompt.error, "llm-judge evaluator is missing judgePromptName");
|
|
@@ -408,7 +407,7 @@ describe("evaluate", () => {
|
|
|
408
407
|
},
|
|
409
408
|
{
|
|
410
409
|
ai: {
|
|
411
|
-
generateJsonObject:
|
|
410
|
+
generateJsonObject: judgeAnswering(() => ({confidence: "not-a-number", correct: 1})),
|
|
412
411
|
},
|
|
413
412
|
judgeOutputSchema: {
|
|
414
413
|
properties: {
|
|
@@ -461,9 +460,12 @@ describe("evaluate", () => {
|
|
|
461
460
|
output: "4",
|
|
462
461
|
},
|
|
463
462
|
{
|
|
464
|
-
generateJsonObject:
|
|
465
|
-
|
|
466
|
-
|
|
463
|
+
generateJsonObject: judgeAnswering(() => ({
|
|
464
|
+
confidence: "0.8",
|
|
465
|
+
correct: "true",
|
|
466
|
+
helpfulness: "not-a-number",
|
|
467
|
+
tone: 1,
|
|
468
|
+
})),
|
|
467
469
|
},
|
|
468
470
|
{
|
|
469
471
|
properties: {
|
|
@@ -489,7 +491,7 @@ describe("LocalEvaluatorStore phase 2", () => {
|
|
|
489
491
|
beforeEach(async () => {
|
|
490
492
|
new ObservabilityApp({plugins: [createLocalObservabilityBundle().plugin]});
|
|
491
493
|
promptStore = new LocalPromptStore();
|
|
492
|
-
store = new (await import("./local/evaluatorStore")).LocalEvaluatorStore(promptStore);
|
|
494
|
+
store = new (await import("./local/evaluatorStore.js")).LocalEvaluatorStore(promptStore);
|
|
493
495
|
await registerObsEvaluator().deleteMany({});
|
|
494
496
|
await clearObsPrompts();
|
|
495
497
|
await promptStore.create({
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, it} from "bun:test";
|
|
2
|
-
import {BackgroundTask} from "@terreno/api";
|
|
2
|
+
import {BackgroundTask, type BackgroundTaskDocument, type FindExactlyOnePlugin} from "@terreno/api";
|
|
3
3
|
import {assert} from "chai";
|
|
4
4
|
import mongoose from "mongoose";
|
|
5
|
-
|
|
5
|
+
import {judgeAnswering} from "../../tests/fakeJudge";
|
|
6
6
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
7
7
|
import {LocalDatasetStore} from "./datasetStore";
|
|
8
8
|
import {LocalEvaluatorStore} from "./evaluatorStore";
|
|
@@ -37,9 +37,7 @@ const createTestRunner = (params: {
|
|
|
37
37
|
});
|
|
38
38
|
runner.configureAi({
|
|
39
39
|
aiService: {
|
|
40
|
-
generateJsonObject:
|
|
41
|
-
return {correct: true};
|
|
42
|
-
},
|
|
40
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
43
41
|
generateText: params.generateText,
|
|
44
42
|
modelId: "mock-model",
|
|
45
43
|
},
|
|
@@ -340,9 +338,7 @@ describe("LocalExperimentRunner", () => {
|
|
|
340
338
|
});
|
|
341
339
|
defaultRunner.configureAi({
|
|
342
340
|
aiService: {
|
|
343
|
-
generateJsonObject:
|
|
344
|
-
return {correct: true};
|
|
345
|
-
},
|
|
341
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
346
342
|
generateText: async () => {
|
|
347
343
|
return JSON.stringify({answer: "ok"});
|
|
348
344
|
},
|
|
@@ -399,9 +395,7 @@ describe("LocalExperimentRunner", () => {
|
|
|
399
395
|
});
|
|
400
396
|
failingRunner.configureAi({
|
|
401
397
|
aiService: {
|
|
402
|
-
generateJsonObject:
|
|
403
|
-
return {correct: true};
|
|
404
|
-
},
|
|
398
|
+
generateJsonObject: judgeAnswering(() => ({correct: true})),
|
|
405
399
|
generateText: async () => {
|
|
406
400
|
throw new Error("generation failed");
|
|
407
401
|
},
|
|
@@ -427,7 +421,10 @@ describe("LocalExperimentRunner", () => {
|
|
|
427
421
|
});
|
|
428
422
|
detail = await failingRunner.get(experiment.id);
|
|
429
423
|
}
|
|
430
|
-
|
|
424
|
+
// @terreno/api applies findExactlyOne to BackgroundTask, but its model type does not declare it.
|
|
425
|
+
const tasks = BackgroundTask as typeof BackgroundTask &
|
|
426
|
+
FindExactlyOnePlugin<BackgroundTaskDocument>;
|
|
427
|
+
const task = await tasks.findExactlyOne({_id: detail.backgroundTaskId});
|
|
431
428
|
assert.equal(task.status, "failed");
|
|
432
429
|
assert.include(task.error ?? "", "generation failed");
|
|
433
430
|
});
|
|
@@ -22,7 +22,7 @@ import type {LocalDatasetStore} from "./datasetStore";
|
|
|
22
22
|
import type {LocalEvaluatorStore} from "./evaluatorStore";
|
|
23
23
|
import {registerObsExperiment} from "./models/obsExperiment";
|
|
24
24
|
import {registerObsExperimentItem} from "./models/obsExperimentItem";
|
|
25
|
-
import type {LocalPromptStore} from "./promptStore";
|
|
25
|
+
import type {LocalPromptStore, MoveLabelResult} from "./promptStore";
|
|
26
26
|
|
|
27
27
|
export interface ExperimentCreateInput {
|
|
28
28
|
datasetId: string;
|
|
@@ -354,7 +354,7 @@ export class LocalExperimentRunner {
|
|
|
354
354
|
};
|
|
355
355
|
}
|
|
356
356
|
|
|
357
|
-
async promote(id: string, version: number): Promise<
|
|
357
|
+
async promote(id: string, version: number): Promise<MoveLabelResult> {
|
|
358
358
|
const experiment = await this.get(id);
|
|
359
359
|
if (!experiment.results) {
|
|
360
360
|
throw new APIError({status: 409, title: "Experiment has not finished running"});
|
|
@@ -10,7 +10,7 @@ import {registerObsPrompt} from "./models/obsPrompt";
|
|
|
10
10
|
import {registerObsPromptLabel} from "./models/obsPromptLabel";
|
|
11
11
|
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
12
12
|
import {registerObsTrace} from "./models/obsTrace";
|
|
13
|
-
import {LocalPromptStore} from "./promptStore";
|
|
13
|
+
import {type CreatePromptInput, LocalPromptStore} from "./promptStore";
|
|
14
14
|
|
|
15
15
|
describe("LocalPromptStore", () => {
|
|
16
16
|
let store: LocalPromptStore;
|
|
@@ -253,7 +253,7 @@ describe("LocalPromptStore", () => {
|
|
|
253
253
|
|
|
254
254
|
it("validates create/moveLabel inputs and resolves versions by label or number", async () => {
|
|
255
255
|
try {
|
|
256
|
-
await store.create({folder: "", name: "bad"});
|
|
256
|
+
await store.create({folder: "", name: "bad"} as CreatePromptInput);
|
|
257
257
|
expect.unreachable();
|
|
258
258
|
} catch (error) {
|
|
259
259
|
expect(String(error)).toMatch(/folder and name/);
|
|
@@ -334,9 +334,10 @@ describe("LocalPromptStore", () => {
|
|
|
334
334
|
const latest = await store.runPlayground({
|
|
335
335
|
generator: {
|
|
336
336
|
generate: async () => {
|
|
337
|
-
return {output: "latest"};
|
|
337
|
+
return {latencyMs: 0, output: "latest"};
|
|
338
338
|
},
|
|
339
339
|
},
|
|
340
|
+
modelId: "mock-model",
|
|
340
341
|
name: "playground",
|
|
341
342
|
variables: {name: "Grace"},
|
|
342
343
|
});
|
|
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
4
|
|
|
5
|
-
import {authAsUser, ensureTestUsers,
|
|
5
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
6
6
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
7
7
|
import {registerObsDataset} from "../local/models/obsDataset";
|
|
8
8
|
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
@@ -32,7 +32,7 @@ describe("observability dataset routes", () => {
|
|
|
32
32
|
await registerObsPromptLabel().deleteMany({});
|
|
33
33
|
promptStore = new LocalPromptStore();
|
|
34
34
|
const plugin = createLocalObservabilityPlugin();
|
|
35
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
35
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
36
36
|
.register(new ObservabilityApp({plugins: [plugin]}))
|
|
37
37
|
.build();
|
|
38
38
|
});
|
|
@@ -2,7 +2,7 @@ import {afterEach, beforeAll, beforeEach, describe, expect, it} from "bun:test";
|
|
|
2
2
|
import {TerrenoApp} from "@terreno/api";
|
|
3
3
|
import type express from "express";
|
|
4
4
|
|
|
5
|
-
import {authAsUser, ensureTestUsers,
|
|
5
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
6
6
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
7
7
|
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
8
8
|
import {registerObsPrompt} from "../local/models/obsPrompt";
|
|
@@ -40,7 +40,7 @@ describe("observability evaluator routes", () => {
|
|
|
40
40
|
type: "text",
|
|
41
41
|
});
|
|
42
42
|
await promptStore.moveLabel("eval-judge-correctness", {label: "production", version: 1});
|
|
43
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
43
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
44
44
|
.register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
|
|
45
45
|
.build();
|
|
46
46
|
});
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {AIService} from "../../service/aiService";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
9
|
import {registerObsDataset} from "../local/models/obsDataset";
|
|
10
10
|
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
@@ -70,7 +70,7 @@ describe("observability experiment routes", () => {
|
|
|
70
70
|
},
|
|
71
71
|
},
|
|
72
72
|
});
|
|
73
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
73
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
74
74
|
.register(
|
|
75
75
|
new ObservabilityApp({
|
|
76
76
|
aiService,
|
|
@@ -8,7 +8,7 @@ import mongoose from "mongoose";
|
|
|
8
8
|
|
|
9
9
|
import {AIRequest} from "../../models/aiRequest";
|
|
10
10
|
import {AIService} from "../../service/aiService";
|
|
11
|
-
import {authAsUser, ensureTestUsers,
|
|
11
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
12
12
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
13
13
|
import {registerObsExperiment} from "../local/models/obsExperiment";
|
|
14
14
|
import {registerObsPrompt} from "../local/models/obsPrompt";
|
|
@@ -61,7 +61,7 @@ describe("observability prompt routes", () => {
|
|
|
61
61
|
const model = createMockModel();
|
|
62
62
|
doGenerate = model.doGenerate;
|
|
63
63
|
aiService = new AIService({model: model as unknown as LanguageModel});
|
|
64
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
64
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
65
65
|
.register(
|
|
66
66
|
new ObservabilityApp({
|
|
67
67
|
aiService,
|
|
@@ -134,7 +134,7 @@ describe("observability prompt routes", () => {
|
|
|
134
134
|
const requestAiServiceFactory = mock(({apiKey}: {apiKey?: string}) => {
|
|
135
135
|
return apiKey === "request-key" ? aiService : undefined;
|
|
136
136
|
});
|
|
137
|
-
const requestServiceApp = new TerrenoApp({skipListen: true, userModel:
|
|
137
|
+
const requestServiceApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
138
138
|
.register(
|
|
139
139
|
new ObservabilityApp({
|
|
140
140
|
plugins: [createLocalObservabilityPlugin()],
|
|
@@ -4,7 +4,7 @@ import {assert} from "chai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
import {DateTime} from "luxon";
|
|
6
6
|
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {LocalEvaluatorStore} from "../local/evaluatorStore";
|
|
9
9
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
10
10
|
import {registerObsEvaluator} from "../local/models/obsEvaluator";
|
|
@@ -28,7 +28,7 @@ describe("observability review routes", () => {
|
|
|
28
28
|
await registerObsReviewItem().deleteMany({});
|
|
29
29
|
await registerObsEvaluator().deleteMany({});
|
|
30
30
|
await registerObsTrace().deleteMany({});
|
|
31
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
31
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
32
32
|
.register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
|
|
33
33
|
.build();
|
|
34
34
|
});
|
|
@@ -4,7 +4,7 @@ import type {LanguageModel} from "ai";
|
|
|
4
4
|
import type express from "express";
|
|
5
5
|
|
|
6
6
|
import {AIService} from "../../service/aiService";
|
|
7
|
-
import {authAsUser, ensureTestUsers,
|
|
7
|
+
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
8
8
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
9
9
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
10
10
|
import type {ObservabilityPlugin} from "../types";
|
|
@@ -33,7 +33,7 @@ describe("observability status routes", () => {
|
|
|
33
33
|
});
|
|
34
34
|
|
|
35
35
|
beforeEach(() => {
|
|
36
|
-
app = new TerrenoApp({skipListen: true, userModel:
|
|
36
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
37
37
|
.register(
|
|
38
38
|
new ObservabilityApp({
|
|
39
39
|
plugins: [createLocalObservabilityPlugin(), createLangfusePlugin()],
|
|
@@ -89,7 +89,7 @@ describe("observability status routes", () => {
|
|
|
89
89
|
specificationVersion: "v2" as const,
|
|
90
90
|
supportedUrls: {},
|
|
91
91
|
};
|
|
92
|
-
const serverApp = new TerrenoApp({skipListen: true, userModel:
|
|
92
|
+
const serverApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
93
93
|
.register(
|
|
94
94
|
new ObservabilityApp({
|
|
95
95
|
aiService: new AIService({model: model as unknown as LanguageModel}),
|
|
@@ -101,7 +101,7 @@ describe("observability status routes", () => {
|
|
|
101
101
|
const serverStatus = await serverAgent.get("/ai/observability/status");
|
|
102
102
|
expect(serverStatus.body.data.playgroundAi).toEqual({source: "server"});
|
|
103
103
|
|
|
104
|
-
const requestKeyApp = new TerrenoApp({skipListen: true, userModel:
|
|
104
|
+
const requestKeyApp = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
105
105
|
.register(
|
|
106
106
|
new ObservabilityApp({
|
|
107
107
|
plugins: [createLocalObservabilityPlugin()],
|