@terreno/ai 58.0.5 → 58.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/models/gptHistory.js +4 -0
- package/dist/observability/feedback.d.ts +31 -0
- package/dist/observability/feedback.js +218 -0
- package/dist/observability/local/datasetStore.d.ts +7 -0
- package/dist/observability/local/datasetStore.js +48 -2
- package/dist/observability/local/models/obsReviewItem.js +6 -0
- package/dist/observability/local/models/obsScore.js +12 -1
- package/dist/observability/local/promptStore.d.ts +1 -0
- package/dist/observability/local/promptStore.js +1 -0
- package/dist/observability/local/reviewStore.d.ts +5 -0
- package/dist/observability/local/reviewStore.js +17 -0
- package/dist/observability/local/traceStore.d.ts +23 -0
- package/dist/observability/local/traceStore.js +121 -10
- package/dist/observability/observabilityApp.d.ts +1 -0
- package/dist/observability/observabilityApp.js +2 -0
- package/dist/observability/observabilityAppRegistry.d.ts +1 -0
- package/dist/observability/routes/traces.js +72 -0
- package/dist/observability/types.d.ts +8 -1
- package/dist/routes/gpt.js +42 -2
- package/dist/routes/gptHistories.js +42 -2
- package/dist/service/aiService.d.ts +4 -1
- package/dist/service/aiService.js +17 -2
- package/dist/service/chatTurn.js +10 -1
- package/dist/types/index.d.ts +4 -0
- package/dist/types/observability.d.ts +1 -1
- package/package.json +9 -9
- package/src/index.ts +8 -1
- package/src/models/gptHistory.ts +5 -0
- package/src/observability/feedback.test.ts +688 -0
- package/src/observability/feedback.ts +263 -0
- package/src/observability/local/datasetStore.ts +57 -2
- package/src/observability/local/models/obsReviewItem.ts +9 -0
- package/src/observability/local/models/obsScore.ts +18 -1
- package/src/observability/local/obsModels.test.ts +10 -2
- package/src/observability/local/promptStore.test.ts +2 -0
- package/src/observability/local/promptStore.ts +2 -0
- package/src/observability/local/reviewStore.test.ts +55 -0
- package/src/observability/local/reviewStore.ts +21 -0
- package/src/observability/local/traceStore.test.ts +239 -1
- package/src/observability/local/traceStore.ts +140 -9
- package/src/observability/observabilityApp.ts +2 -0
- package/src/observability/observabilityAppRegistry.ts +1 -0
- package/src/observability/routes/datasets.test.ts +253 -0
- package/src/observability/routes/prompts.test.ts +1 -0
- package/src/observability/routes/traces.test.ts +581 -3
- package/src/observability/routes/traces.ts +87 -1
- package/src/observability/types.ts +6 -1
- package/src/routes/gpt.test.ts +263 -1
- package/src/routes/gpt.ts +43 -2
- package/src/routes/gptHistories.ts +55 -2
- package/src/service/aiService.observability.test.ts +244 -0
- package/src/service/aiService.ts +15 -2
- package/src/service/chatTurn.ts +11 -2
- package/src/types/index.ts +4 -0
- package/src/types/observability.ts +1 -1
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import {afterEach, beforeEach, describe, expect, it, mock, spyOn} from "bun:test";
|
|
2
|
+
import {APIError, logger} from "@terreno/api";
|
|
3
|
+
import {jsonSchema, type LanguageModel} from "ai";
|
|
4
|
+
import mongoose from "mongoose";
|
|
5
|
+
|
|
6
|
+
import {createTraceId} from "../index";
|
|
7
|
+
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
8
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
9
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observability/observabilityApp";
|
|
10
|
+
import {AIService} from "./aiService";
|
|
11
|
+
|
|
12
|
+
const createMockModel = (responseText = "Mock response") => ({
|
|
13
|
+
doGenerate: mock(async (_options: unknown) => ({
|
|
14
|
+
content: [{text: responseText, type: "text" as const}],
|
|
15
|
+
finishReason: "stop" as const,
|
|
16
|
+
usage: {inputTokens: 5, outputTokens: 10},
|
|
17
|
+
})),
|
|
18
|
+
doStream: mock(async () => ({
|
|
19
|
+
stream: new ReadableStream({
|
|
20
|
+
start(controller) {
|
|
21
|
+
controller.enqueue({id: "t1", type: "text-start" as const});
|
|
22
|
+
controller.enqueue({delta: responseText, id: "t1", type: "text-delta" as const});
|
|
23
|
+
controller.enqueue({id: "t1", type: "text-end" as const});
|
|
24
|
+
controller.enqueue({
|
|
25
|
+
finishReason: "stop" as const,
|
|
26
|
+
type: "finish" as const,
|
|
27
|
+
usage: {inputTokens: 5, outputTokens: 10},
|
|
28
|
+
});
|
|
29
|
+
controller.close();
|
|
30
|
+
},
|
|
31
|
+
}),
|
|
32
|
+
})),
|
|
33
|
+
modelId: "mock-model",
|
|
34
|
+
provider: "mock-provider",
|
|
35
|
+
specificationVersion: "v2" as const,
|
|
36
|
+
supportedUrls: {},
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const drain = async (stream: AsyncGenerator<string>): Promise<string> => {
|
|
40
|
+
let out = "";
|
|
41
|
+
for await (const chunk of stream) {
|
|
42
|
+
out += chunk;
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const expectBadTraceId = async (run: () => Promise<unknown>): Promise<void> => {
|
|
48
|
+
let caught: unknown;
|
|
49
|
+
try {
|
|
50
|
+
await run();
|
|
51
|
+
} catch (error) {
|
|
52
|
+
caught = error;
|
|
53
|
+
}
|
|
54
|
+
expect(caught).toBeInstanceOf(APIError);
|
|
55
|
+
expect((caught as APIError).status).toBe(400);
|
|
56
|
+
expect((caught as APIError).title).toBe("traceId must be an object id");
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
describe("AIService caller-supplied trace ids", () => {
|
|
60
|
+
const ObsTrace = (): ReturnType<typeof registerObsTrace> => registerObsTrace();
|
|
61
|
+
|
|
62
|
+
beforeEach(async () => {
|
|
63
|
+
new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]});
|
|
64
|
+
await ObsTrace().deleteMany({});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
afterEach(() => {
|
|
68
|
+
resetObservabilityApp();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("createTraceId returns a 24-character hex string, unique per call", () => {
|
|
72
|
+
const first = createTraceId();
|
|
73
|
+
expect(first).toMatch(/^[0-9a-f]{24}$/);
|
|
74
|
+
expect(createTraceId()).not.toBe(first);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("generateText writes the ObsTrace under the given traceId with that userId", async () => {
|
|
78
|
+
const service = new AIService({model: createMockModel("Hello") as unknown as LanguageModel});
|
|
79
|
+
const traceId = createTraceId();
|
|
80
|
+
const userId = new mongoose.Types.ObjectId();
|
|
81
|
+
|
|
82
|
+
const text = await service.generateText({prompt: "hi", traceId, userId});
|
|
83
|
+
|
|
84
|
+
expect(text).toBe("Hello");
|
|
85
|
+
const trace = await ObsTrace().findById(traceId);
|
|
86
|
+
expect(trace).not.toBeNull();
|
|
87
|
+
expect(String(trace?.userId)).toBe(userId.toString());
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("generateText without a traceId still writes a trace with a generated _id", async () => {
|
|
91
|
+
const service = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
92
|
+
|
|
93
|
+
await service.generateText({prompt: "hi"});
|
|
94
|
+
|
|
95
|
+
const traces = await ObsTrace().find({});
|
|
96
|
+
expect(traces.length).toBe(1);
|
|
97
|
+
expect(String(traces[0]?._id)).toMatch(/^[0-9a-f]{24}$/);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("rejects a non-ObjectId traceId with 400 before calling the model", async () => {
|
|
101
|
+
const model = createMockModel();
|
|
102
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
103
|
+
|
|
104
|
+
await expectBadTraceId(() => service.generateText({prompt: "hi", traceId: "abc"}));
|
|
105
|
+
|
|
106
|
+
expect(model.doGenerate.mock.calls.length).toBe(0);
|
|
107
|
+
expect(await ObsTrace().countDocuments({})).toBe(0);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("rejects an invalid traceId on streaming methods before calling the model", async () => {
|
|
111
|
+
const model = createMockModel();
|
|
112
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
113
|
+
|
|
114
|
+
await expectBadTraceId(() => drain(service.generateTextStream({prompt: "hi", traceId: "abc"})));
|
|
115
|
+
await expectBadTraceId(() =>
|
|
116
|
+
drain(
|
|
117
|
+
service.generateChatStream({
|
|
118
|
+
messages: [{content: "hi", role: "user"}],
|
|
119
|
+
traceId: "abc",
|
|
120
|
+
})
|
|
121
|
+
)
|
|
122
|
+
);
|
|
123
|
+
|
|
124
|
+
expect(model.doStream.mock.calls.length).toBe(0);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("rejects a 24-char non-hex traceId with 400", async () => {
|
|
128
|
+
const model = createMockModel();
|
|
129
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
130
|
+
|
|
131
|
+
await expectBadTraceId(() => service.generateText({prompt: "hi", traceId: "z".repeat(24)}));
|
|
132
|
+
|
|
133
|
+
expect(model.doGenerate.mock.calls.length).toBe(0);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("a duplicate traceId returns its text, keeps one ObsTrace, and logs one error", async () => {
|
|
137
|
+
const service = new AIService({model: createMockModel("Again") as unknown as LanguageModel});
|
|
138
|
+
const traceId = createTraceId();
|
|
139
|
+
await service.generateText({prompt: "first", traceId});
|
|
140
|
+
const errorSpy = spyOn(logger, "error").mockImplementation(() => logger);
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
const text = await service.generateText({prompt: "second", traceId});
|
|
144
|
+
|
|
145
|
+
expect(text).toBe("Again");
|
|
146
|
+
expect(await ObsTrace().countDocuments({})).toBe(1);
|
|
147
|
+
expect(errorSpy).toHaveBeenCalledTimes(1);
|
|
148
|
+
} finally {
|
|
149
|
+
errorSpy.mockRestore();
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it("skipTrace with a traceId writes no trace and does not throw", async () => {
|
|
154
|
+
const service = new AIService({model: createMockModel("Quiet") as unknown as LanguageModel});
|
|
155
|
+
|
|
156
|
+
const text = await service.generateText({
|
|
157
|
+
prompt: "hi",
|
|
158
|
+
skipTrace: true,
|
|
159
|
+
traceId: createTraceId(),
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
expect(text).toBe("Quiet");
|
|
163
|
+
expect(await ObsTrace().countDocuments({})).toBe(0);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it("resolveGenerateObservability({traceId}) then recordGenerate writes that _id", async () => {
|
|
167
|
+
const service = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
168
|
+
const traceId = createTraceId();
|
|
169
|
+
const observability = await service.resolveGenerateObservability({traceId});
|
|
170
|
+
|
|
171
|
+
await service.recordGenerate({
|
|
172
|
+
observability,
|
|
173
|
+
prompt: "p",
|
|
174
|
+
requestType: "general",
|
|
175
|
+
response: "r",
|
|
176
|
+
responseTime: 5,
|
|
177
|
+
startTime: Date.now() - 5,
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
expect(await ObsTrace().findById(traceId)).not.toBeNull();
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("resolveGenerateObservability rejects an invalid traceId with 400", async () => {
|
|
184
|
+
const service = new AIService({model: createMockModel() as unknown as LanguageModel});
|
|
185
|
+
|
|
186
|
+
await expectBadTraceId(() => service.resolveGenerateObservability({traceId: "abc"}));
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
describe("every generate method passes traceId through", () => {
|
|
190
|
+
const cases: Array<{
|
|
191
|
+
name: string;
|
|
192
|
+
run: (service: AIService, traceId: string) => Promise<unknown>;
|
|
193
|
+
}> = [
|
|
194
|
+
{
|
|
195
|
+
name: "generateTextStream",
|
|
196
|
+
run: (service, traceId) => drain(service.generateTextStream({prompt: "hi", traceId})),
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
name: "generateChatStream",
|
|
200
|
+
run: (service, traceId) =>
|
|
201
|
+
drain(service.generateChatStream({messages: [{content: "hi", role: "user"}], traceId})),
|
|
202
|
+
},
|
|
203
|
+
{
|
|
204
|
+
name: "generateSummary",
|
|
205
|
+
run: (service, traceId) => service.generateSummary({text: "long text", traceId}),
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
name: "generateRemix",
|
|
209
|
+
run: (service, traceId) => service.generateRemix({text: "some text", traceId}),
|
|
210
|
+
},
|
|
211
|
+
{
|
|
212
|
+
name: "translateText",
|
|
213
|
+
run: (service, traceId) =>
|
|
214
|
+
service.translateText({targetLanguage: "French", text: "hello", traceId}),
|
|
215
|
+
},
|
|
216
|
+
{
|
|
217
|
+
name: "generateJsonObject",
|
|
218
|
+
run: (service, traceId) =>
|
|
219
|
+
service.generateJsonObject({
|
|
220
|
+
prompt: "give json",
|
|
221
|
+
schema: jsonSchema<{a: number}>({
|
|
222
|
+
properties: {a: {type: "number"}},
|
|
223
|
+
required: ["a"],
|
|
224
|
+
type: "object",
|
|
225
|
+
}),
|
|
226
|
+
traceId,
|
|
227
|
+
}),
|
|
228
|
+
},
|
|
229
|
+
];
|
|
230
|
+
|
|
231
|
+
for (const {name, run} of cases) {
|
|
232
|
+
it(`${name} writes the ObsTrace under the given traceId`, async () => {
|
|
233
|
+
const service = new AIService({
|
|
234
|
+
model: createMockModel('{"a":1}') as unknown as LanguageModel,
|
|
235
|
+
});
|
|
236
|
+
const traceId = createTraceId();
|
|
237
|
+
|
|
238
|
+
await run(service, traceId);
|
|
239
|
+
|
|
240
|
+
expect(await ObsTrace().findById(traceId)).not.toBeNull();
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
});
|
package/src/service/aiService.ts
CHANGED
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
streamText,
|
|
19
19
|
} from "ai";
|
|
20
20
|
import {DateTime} from "luxon";
|
|
21
|
-
import
|
|
21
|
+
import mongoose from "mongoose";
|
|
22
22
|
|
|
23
23
|
import {AIRequest} from "../models/aiRequest";
|
|
24
24
|
import {getObservabilityApp} from "../observability/observabilityAppRegistry";
|
|
@@ -138,8 +138,16 @@ interface ResolvedObservability {
|
|
|
138
138
|
sessionId?: string;
|
|
139
139
|
skipTrace: boolean;
|
|
140
140
|
systemPrompt?: string;
|
|
141
|
+
traceId?: string;
|
|
141
142
|
}
|
|
142
143
|
|
|
144
|
+
const TRACE_ID_PATTERN = /^[0-9a-f]{24}$/i;
|
|
145
|
+
|
|
146
|
+
/** Mints an id a caller can pass as `traceId` before a generate call and return to its client. */
|
|
147
|
+
export const createTraceId = (): string => {
|
|
148
|
+
return new mongoose.Types.ObjectId().toHexString();
|
|
149
|
+
};
|
|
150
|
+
|
|
143
151
|
export class AIService {
|
|
144
152
|
readonly model: LanguageModel;
|
|
145
153
|
readonly defaultTemperature: number;
|
|
@@ -265,6 +273,9 @@ export class AIService {
|
|
|
265
273
|
private async resolveObservability(
|
|
266
274
|
options: GenerateObservabilityOptions & {systemPrompt?: string}
|
|
267
275
|
): Promise<ResolvedObservability> {
|
|
276
|
+
if (options.traceId !== undefined && !TRACE_ID_PATTERN.test(options.traceId)) {
|
|
277
|
+
throw new APIError({status: 400, title: "traceId must be an object id"});
|
|
278
|
+
}
|
|
268
279
|
let promptRef: PromptVersionRef | undefined;
|
|
269
280
|
let systemPrompt = options.systemPrompt;
|
|
270
281
|
let sensitive = options.sensitive ?? false;
|
|
@@ -296,6 +307,7 @@ export class AIService {
|
|
|
296
307
|
sessionId: options.sessionId,
|
|
297
308
|
skipTrace: options.skipTrace === true,
|
|
298
309
|
systemPrompt,
|
|
310
|
+
traceId: options.traceId,
|
|
299
311
|
};
|
|
300
312
|
}
|
|
301
313
|
|
|
@@ -307,6 +319,7 @@ export class AIService {
|
|
|
307
319
|
sensitive: options.sensitive,
|
|
308
320
|
sessionId: options.sessionId,
|
|
309
321
|
skipTrace: options.skipTrace,
|
|
322
|
+
traceId: options.traceId,
|
|
310
323
|
};
|
|
311
324
|
}
|
|
312
325
|
|
|
@@ -392,7 +405,7 @@ export class AIService {
|
|
|
392
405
|
const promptRef = params.observability.promptRef;
|
|
393
406
|
return {
|
|
394
407
|
endedAt,
|
|
395
|
-
id: randomUUID(),
|
|
408
|
+
id: params.observability.traceId ?? randomUUID(),
|
|
396
409
|
input: params.prompt,
|
|
397
410
|
name: spanName,
|
|
398
411
|
output: params.response,
|
package/src/service/chatTurn.ts
CHANGED
|
@@ -41,7 +41,7 @@ import type {
|
|
|
41
41
|
MessageContentPart,
|
|
42
42
|
UiBlocksOptions,
|
|
43
43
|
} from "../types";
|
|
44
|
-
import {AIService} from "./aiService";
|
|
44
|
+
import {AIService, createTraceId} from "./aiService";
|
|
45
45
|
import {
|
|
46
46
|
askFilesModelOutput,
|
|
47
47
|
invalidAskResponseError,
|
|
@@ -1834,12 +1834,15 @@ export const runChatTurn = async ({
|
|
|
1834
1834
|
systemPrompt,
|
|
1835
1835
|
userId,
|
|
1836
1836
|
});
|
|
1837
|
+
// Minted up front so the assistant row carries it and feedback on the reply can find the trace.
|
|
1838
|
+
const traceId = getObservabilityApp() ? createTraceId() : undefined;
|
|
1837
1839
|
const resolvedObservability = await aiService.resolveGenerateObservability({
|
|
1838
1840
|
promptLabel,
|
|
1839
1841
|
promptName,
|
|
1840
1842
|
sensitive: body.sensitive === true ? true : undefined,
|
|
1841
1843
|
sessionId,
|
|
1842
1844
|
systemPrompt: effectiveSystemPrompt,
|
|
1845
|
+
traceId,
|
|
1843
1846
|
});
|
|
1844
1847
|
const observability = registryPrompt
|
|
1845
1848
|
? {
|
|
@@ -1954,6 +1957,7 @@ export const runChatTurn = async ({
|
|
|
1954
1957
|
status: "streaming",
|
|
1955
1958
|
streamId,
|
|
1956
1959
|
text: "",
|
|
1960
|
+
...(traceId ? {traceId} : {}),
|
|
1957
1961
|
type: "assistant",
|
|
1958
1962
|
},
|
|
1959
1963
|
]);
|
|
@@ -2192,7 +2196,12 @@ export const runChatTurn = async ({
|
|
|
2192
2196
|
fullResponse: storedResponse,
|
|
2193
2197
|
generatedImages,
|
|
2194
2198
|
modelId: aiService.modelId,
|
|
2195
|
-
}).map((row) => ({
|
|
2199
|
+
}).map((row) => ({
|
|
2200
|
+
...row,
|
|
2201
|
+
status: "complete" as const,
|
|
2202
|
+
streamId,
|
|
2203
|
+
...(traceId && row.type === "assistant" ? {traceId} : {}),
|
|
2204
|
+
})),
|
|
2196
2205
|
...(ask
|
|
2197
2206
|
? [
|
|
2198
2207
|
askCallRow(ask, "pending"),
|
package/src/types/index.ts
CHANGED
|
@@ -122,6 +122,8 @@ export type GptHistoryPromptStatus = "streaming" | "complete" | "error";
|
|
|
122
122
|
export interface GptHistoryPrompt {
|
|
123
123
|
model?: string;
|
|
124
124
|
rating?: "up" | "down";
|
|
125
|
+
/** Observability trace id of an assistant reply (written by the chat turn). */
|
|
126
|
+
traceId?: string;
|
|
125
127
|
/** Set on assistant replies produced by /gpt/prompt. "streaming" while partial text is persisted. */
|
|
126
128
|
status?: GptHistoryPromptStatus;
|
|
127
129
|
/** Identifies one /gpt/prompt reply so resume clients can follow it. */
|
|
@@ -219,6 +221,8 @@ export interface GenerateObservabilityOptions {
|
|
|
219
221
|
sensitive?: boolean;
|
|
220
222
|
sessionId?: string;
|
|
221
223
|
skipTrace?: boolean;
|
|
224
|
+
/** Id to store the trace under (from `createTraceId()`), so the caller can return it for feedback. */
|
|
225
|
+
traceId?: string;
|
|
222
226
|
}
|
|
223
227
|
|
|
224
228
|
export interface GenerateTextOptions extends GenerateObservabilityOptions {
|
|
@@ -145,7 +145,7 @@ export interface ObsScoreDocument extends mongoose.Document<mongoose.Types.Objec
|
|
|
145
145
|
deleted: boolean;
|
|
146
146
|
evaluatorId?: mongoose.Types.ObjectId;
|
|
147
147
|
name: string;
|
|
148
|
-
source: "code" | "human" | "llm-judge" | "user-feedback";
|
|
148
|
+
source: "code" | "human" | "llm-judge" | "user-feedback" | "user-signal";
|
|
149
149
|
spanId?: mongoose.Types.ObjectId;
|
|
150
150
|
traceId: mongoose.Types.ObjectId;
|
|
151
151
|
updated: Date;
|