@terreno/ai 58.0.6 → 58.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/dist/index.d.ts +2 -1
  3. package/dist/index.js +6 -2
  4. package/dist/models/gptHistory.js +4 -0
  5. package/dist/observability/feedback.d.ts +31 -0
  6. package/dist/observability/feedback.js +218 -0
  7. package/dist/observability/local/datasetStore.d.ts +7 -0
  8. package/dist/observability/local/datasetStore.js +48 -2
  9. package/dist/observability/local/models/obsReviewItem.js +6 -0
  10. package/dist/observability/local/models/obsScore.js +12 -1
  11. package/dist/observability/local/promptStore.d.ts +1 -0
  12. package/dist/observability/local/promptStore.js +1 -0
  13. package/dist/observability/local/reviewStore.d.ts +5 -0
  14. package/dist/observability/local/reviewStore.js +17 -0
  15. package/dist/observability/local/traceStore.d.ts +23 -0
  16. package/dist/observability/local/traceStore.js +121 -10
  17. package/dist/observability/observabilityApp.d.ts +1 -0
  18. package/dist/observability/observabilityApp.js +2 -0
  19. package/dist/observability/observabilityAppRegistry.d.ts +1 -0
  20. package/dist/observability/routes/traces.js +72 -0
  21. package/dist/observability/types.d.ts +8 -1
  22. package/dist/routes/gpt.js +42 -2
  23. package/dist/routes/gptHistories.js +42 -2
  24. package/dist/service/aiService.d.ts +4 -1
  25. package/dist/service/aiService.js +17 -2
  26. package/dist/service/chatTurn.js +10 -1
  27. package/dist/types/index.d.ts +4 -0
  28. package/dist/types/observability.d.ts +1 -1
  29. package/package.json +8 -8
  30. package/src/index.ts +8 -1
  31. package/src/models/gptHistory.ts +5 -0
  32. package/src/observability/feedback.test.ts +688 -0
  33. package/src/observability/feedback.ts +263 -0
  34. package/src/observability/local/datasetStore.ts +57 -2
  35. package/src/observability/local/models/obsReviewItem.ts +9 -0
  36. package/src/observability/local/models/obsScore.ts +18 -1
  37. package/src/observability/local/obsModels.test.ts +10 -2
  38. package/src/observability/local/promptStore.test.ts +2 -0
  39. package/src/observability/local/promptStore.ts +2 -0
  40. package/src/observability/local/reviewStore.test.ts +55 -0
  41. package/src/observability/local/reviewStore.ts +21 -0
  42. package/src/observability/local/traceStore.test.ts +239 -1
  43. package/src/observability/local/traceStore.ts +140 -9
  44. package/src/observability/observabilityApp.ts +2 -0
  45. package/src/observability/observabilityAppRegistry.ts +1 -0
  46. package/src/observability/routes/datasets.test.ts +253 -0
  47. package/src/observability/routes/prompts.test.ts +1 -0
  48. package/src/observability/routes/traces.test.ts +581 -3
  49. package/src/observability/routes/traces.ts +87 -1
  50. package/src/observability/types.ts +6 -1
  51. package/src/routes/gpt.test.ts +263 -1
  52. package/src/routes/gpt.ts +43 -2
  53. package/src/routes/gptHistories.ts +55 -2
  54. package/src/service/aiService.observability.test.ts +244 -0
  55. package/src/service/aiService.ts +15 -2
  56. package/src/service/chatTurn.ts +11 -2
  57. package/src/types/index.ts +4 -0
  58. package/src/types/observability.ts +1 -1
@@ -0,0 +1,244 @@
1
+ import {afterEach, beforeEach, describe, expect, it, mock, spyOn} from "bun:test";
2
+ import {APIError, logger} from "@terreno/api";
3
+ import {jsonSchema, type LanguageModel} from "ai";
4
+ import mongoose from "mongoose";
5
+
6
+ import {createTraceId} from "../index";
7
+ import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
8
+ import {registerObsTrace} from "../observability/local/models/obsTrace";
9
+ import {ObservabilityApp, resetObservabilityApp} from "../observability/observabilityApp";
10
+ import {AIService} from "./aiService";
11
+
12
+ const createMockModel = (responseText = "Mock response") => ({
13
+ doGenerate: mock(async (_options: unknown) => ({
14
+ content: [{text: responseText, type: "text" as const}],
15
+ finishReason: "stop" as const,
16
+ usage: {inputTokens: 5, outputTokens: 10},
17
+ })),
18
+ doStream: mock(async () => ({
19
+ stream: new ReadableStream({
20
+ start(controller) {
21
+ controller.enqueue({id: "t1", type: "text-start" as const});
22
+ controller.enqueue({delta: responseText, id: "t1", type: "text-delta" as const});
23
+ controller.enqueue({id: "t1", type: "text-end" as const});
24
+ controller.enqueue({
25
+ finishReason: "stop" as const,
26
+ type: "finish" as const,
27
+ usage: {inputTokens: 5, outputTokens: 10},
28
+ });
29
+ controller.close();
30
+ },
31
+ }),
32
+ })),
33
+ modelId: "mock-model",
34
+ provider: "mock-provider",
35
+ specificationVersion: "v2" as const,
36
+ supportedUrls: {},
37
+ });
38
+
39
+ const drain = async (stream: AsyncGenerator<string>): Promise<string> => {
40
+ let out = "";
41
+ for await (const chunk of stream) {
42
+ out += chunk;
43
+ }
44
+ return out;
45
+ };
46
+
47
+ const expectBadTraceId = async (run: () => Promise<unknown>): Promise<void> => {
48
+ let caught: unknown;
49
+ try {
50
+ await run();
51
+ } catch (error) {
52
+ caught = error;
53
+ }
54
+ expect(caught).toBeInstanceOf(APIError);
55
+ expect((caught as APIError).status).toBe(400);
56
+ expect((caught as APIError).title).toBe("traceId must be an object id");
57
+ };
58
+
59
+ describe("AIService caller-supplied trace ids", () => {
60
+ const ObsTrace = (): ReturnType<typeof registerObsTrace> => registerObsTrace();
61
+
62
+ beforeEach(async () => {
63
+ new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]});
64
+ await ObsTrace().deleteMany({});
65
+ });
66
+
67
+ afterEach(() => {
68
+ resetObservabilityApp();
69
+ });
70
+
71
+ it("createTraceId returns a 24-character hex string, unique per call", () => {
72
+ const first = createTraceId();
73
+ expect(first).toMatch(/^[0-9a-f]{24}$/);
74
+ expect(createTraceId()).not.toBe(first);
75
+ });
76
+
77
+ it("generateText writes the ObsTrace under the given traceId with that userId", async () => {
78
+ const service = new AIService({model: createMockModel("Hello") as unknown as LanguageModel});
79
+ const traceId = createTraceId();
80
+ const userId = new mongoose.Types.ObjectId();
81
+
82
+ const text = await service.generateText({prompt: "hi", traceId, userId});
83
+
84
+ expect(text).toBe("Hello");
85
+ const trace = await ObsTrace().findById(traceId);
86
+ expect(trace).not.toBeNull();
87
+ expect(String(trace?.userId)).toBe(userId.toString());
88
+ });
89
+
90
+ it("generateText without a traceId still writes a trace with a generated _id", async () => {
91
+ const service = new AIService({model: createMockModel() as unknown as LanguageModel});
92
+
93
+ await service.generateText({prompt: "hi"});
94
+
95
+ const traces = await ObsTrace().find({});
96
+ expect(traces.length).toBe(1);
97
+ expect(String(traces[0]?._id)).toMatch(/^[0-9a-f]{24}$/);
98
+ });
99
+
100
+ it("rejects a non-ObjectId traceId with 400 before calling the model", async () => {
101
+ const model = createMockModel();
102
+ const service = new AIService({model: model as unknown as LanguageModel});
103
+
104
+ await expectBadTraceId(() => service.generateText({prompt: "hi", traceId: "abc"}));
105
+
106
+ expect(model.doGenerate.mock.calls.length).toBe(0);
107
+ expect(await ObsTrace().countDocuments({})).toBe(0);
108
+ });
109
+
110
+ it("rejects an invalid traceId on streaming methods before calling the model", async () => {
111
+ const model = createMockModel();
112
+ const service = new AIService({model: model as unknown as LanguageModel});
113
+
114
+ await expectBadTraceId(() => drain(service.generateTextStream({prompt: "hi", traceId: "abc"})));
115
+ await expectBadTraceId(() =>
116
+ drain(
117
+ service.generateChatStream({
118
+ messages: [{content: "hi", role: "user"}],
119
+ traceId: "abc",
120
+ })
121
+ )
122
+ );
123
+
124
+ expect(model.doStream.mock.calls.length).toBe(0);
125
+ });
126
+
127
+ it("rejects a 24-char non-hex traceId with 400", async () => {
128
+ const model = createMockModel();
129
+ const service = new AIService({model: model as unknown as LanguageModel});
130
+
131
+ await expectBadTraceId(() => service.generateText({prompt: "hi", traceId: "z".repeat(24)}));
132
+
133
+ expect(model.doGenerate.mock.calls.length).toBe(0);
134
+ });
135
+
136
+ it("a duplicate traceId returns its text, keeps one ObsTrace, and logs one error", async () => {
137
+ const service = new AIService({model: createMockModel("Again") as unknown as LanguageModel});
138
+ const traceId = createTraceId();
139
+ await service.generateText({prompt: "first", traceId});
140
+ const errorSpy = spyOn(logger, "error").mockImplementation(() => logger);
141
+
142
+ try {
143
+ const text = await service.generateText({prompt: "second", traceId});
144
+
145
+ expect(text).toBe("Again");
146
+ expect(await ObsTrace().countDocuments({})).toBe(1);
147
+ expect(errorSpy).toHaveBeenCalledTimes(1);
148
+ } finally {
149
+ errorSpy.mockRestore();
150
+ }
151
+ });
152
+
153
+ it("skipTrace with a traceId writes no trace and does not throw", async () => {
154
+ const service = new AIService({model: createMockModel("Quiet") as unknown as LanguageModel});
155
+
156
+ const text = await service.generateText({
157
+ prompt: "hi",
158
+ skipTrace: true,
159
+ traceId: createTraceId(),
160
+ });
161
+
162
+ expect(text).toBe("Quiet");
163
+ expect(await ObsTrace().countDocuments({})).toBe(0);
164
+ });
165
+
166
+ it("resolveGenerateObservability({traceId}) then recordGenerate writes that _id", async () => {
167
+ const service = new AIService({model: createMockModel() as unknown as LanguageModel});
168
+ const traceId = createTraceId();
169
+ const observability = await service.resolveGenerateObservability({traceId});
170
+
171
+ await service.recordGenerate({
172
+ observability,
173
+ prompt: "p",
174
+ requestType: "general",
175
+ response: "r",
176
+ responseTime: 5,
177
+ startTime: Date.now() - 5,
178
+ });
179
+
180
+ expect(await ObsTrace().findById(traceId)).not.toBeNull();
181
+ });
182
+
183
+ it("resolveGenerateObservability rejects an invalid traceId with 400", async () => {
184
+ const service = new AIService({model: createMockModel() as unknown as LanguageModel});
185
+
186
+ await expectBadTraceId(() => service.resolveGenerateObservability({traceId: "abc"}));
187
+ });
188
+
189
+ describe("every generate method passes traceId through", () => {
190
+ const cases: Array<{
191
+ name: string;
192
+ run: (service: AIService, traceId: string) => Promise<unknown>;
193
+ }> = [
194
+ {
195
+ name: "generateTextStream",
196
+ run: (service, traceId) => drain(service.generateTextStream({prompt: "hi", traceId})),
197
+ },
198
+ {
199
+ name: "generateChatStream",
200
+ run: (service, traceId) =>
201
+ drain(service.generateChatStream({messages: [{content: "hi", role: "user"}], traceId})),
202
+ },
203
+ {
204
+ name: "generateSummary",
205
+ run: (service, traceId) => service.generateSummary({text: "long text", traceId}),
206
+ },
207
+ {
208
+ name: "generateRemix",
209
+ run: (service, traceId) => service.generateRemix({text: "some text", traceId}),
210
+ },
211
+ {
212
+ name: "translateText",
213
+ run: (service, traceId) =>
214
+ service.translateText({targetLanguage: "French", text: "hello", traceId}),
215
+ },
216
+ {
217
+ name: "generateJsonObject",
218
+ run: (service, traceId) =>
219
+ service.generateJsonObject({
220
+ prompt: "give json",
221
+ schema: jsonSchema<{a: number}>({
222
+ properties: {a: {type: "number"}},
223
+ required: ["a"],
224
+ type: "object",
225
+ }),
226
+ traceId,
227
+ }),
228
+ },
229
+ ];
230
+
231
+ for (const {name, run} of cases) {
232
+ it(`${name} writes the ObsTrace under the given traceId`, async () => {
233
+ const service = new AIService({
234
+ model: createMockModel('{"a":1}') as unknown as LanguageModel,
235
+ });
236
+ const traceId = createTraceId();
237
+
238
+ await run(service, traceId);
239
+
240
+ expect(await ObsTrace().findById(traceId)).not.toBeNull();
241
+ });
242
+ }
243
+ });
244
+ });
@@ -18,7 +18,7 @@ import {
18
18
  streamText,
19
19
  } from "ai";
20
20
  import {DateTime} from "luxon";
21
- import type mongoose from "mongoose";
21
+ import mongoose from "mongoose";
22
22
 
23
23
  import {AIRequest} from "../models/aiRequest";
24
24
  import {getObservabilityApp} from "../observability/observabilityAppRegistry";
@@ -138,8 +138,16 @@ interface ResolvedObservability {
138
138
  sessionId?: string;
139
139
  skipTrace: boolean;
140
140
  systemPrompt?: string;
141
+ traceId?: string;
141
142
  }
142
143
 
144
+ const TRACE_ID_PATTERN = /^[0-9a-f]{24}$/i;
145
+
146
+ /** Mints an id a caller can pass as `traceId` before a generate call and return to its client. */
147
+ export const createTraceId = (): string => {
148
+ return new mongoose.Types.ObjectId().toHexString();
149
+ };
150
+
143
151
  export class AIService {
144
152
  readonly model: LanguageModel;
145
153
  readonly defaultTemperature: number;
@@ -265,6 +273,9 @@ export class AIService {
265
273
  private async resolveObservability(
266
274
  options: GenerateObservabilityOptions & {systemPrompt?: string}
267
275
  ): Promise<ResolvedObservability> {
276
+ if (options.traceId !== undefined && !TRACE_ID_PATTERN.test(options.traceId)) {
277
+ throw new APIError({status: 400, title: "traceId must be an object id"});
278
+ }
268
279
  let promptRef: PromptVersionRef | undefined;
269
280
  let systemPrompt = options.systemPrompt;
270
281
  let sensitive = options.sensitive ?? false;
@@ -296,6 +307,7 @@ export class AIService {
296
307
  sessionId: options.sessionId,
297
308
  skipTrace: options.skipTrace === true,
298
309
  systemPrompt,
310
+ traceId: options.traceId,
299
311
  };
300
312
  }
301
313
 
@@ -307,6 +319,7 @@ export class AIService {
307
319
  sensitive: options.sensitive,
308
320
  sessionId: options.sessionId,
309
321
  skipTrace: options.skipTrace,
322
+ traceId: options.traceId,
310
323
  };
311
324
  }
312
325
 
@@ -392,7 +405,7 @@ export class AIService {
392
405
  const promptRef = params.observability.promptRef;
393
406
  return {
394
407
  endedAt,
395
- id: randomUUID(),
408
+ id: params.observability.traceId ?? randomUUID(),
396
409
  input: params.prompt,
397
410
  name: spanName,
398
411
  output: params.response,
@@ -41,7 +41,7 @@ import type {
41
41
  MessageContentPart,
42
42
  UiBlocksOptions,
43
43
  } from "../types";
44
- import {AIService} from "./aiService";
44
+ import {AIService, createTraceId} from "./aiService";
45
45
  import {
46
46
  askFilesModelOutput,
47
47
  invalidAskResponseError,
@@ -1834,12 +1834,15 @@ export const runChatTurn = async ({
1834
1834
  systemPrompt,
1835
1835
  userId,
1836
1836
  });
1837
+ // Minted up front so the assistant row carries it and feedback on the reply can find the trace.
1838
+ const traceId = getObservabilityApp() ? createTraceId() : undefined;
1837
1839
  const resolvedObservability = await aiService.resolveGenerateObservability({
1838
1840
  promptLabel,
1839
1841
  promptName,
1840
1842
  sensitive: body.sensitive === true ? true : undefined,
1841
1843
  sessionId,
1842
1844
  systemPrompt: effectiveSystemPrompt,
1845
+ traceId,
1843
1846
  });
1844
1847
  const observability = registryPrompt
1845
1848
  ? {
@@ -1954,6 +1957,7 @@ export const runChatTurn = async ({
1954
1957
  status: "streaming",
1955
1958
  streamId,
1956
1959
  text: "",
1960
+ ...(traceId ? {traceId} : {}),
1957
1961
  type: "assistant",
1958
1962
  },
1959
1963
  ]);
@@ -2192,7 +2196,12 @@ export const runChatTurn = async ({
2192
2196
  fullResponse: storedResponse,
2193
2197
  generatedImages,
2194
2198
  modelId: aiService.modelId,
2195
- }).map((row) => ({...row, status: "complete" as const, streamId})),
2199
+ }).map((row) => ({
2200
+ ...row,
2201
+ status: "complete" as const,
2202
+ streamId,
2203
+ ...(traceId && row.type === "assistant" ? {traceId} : {}),
2204
+ })),
2196
2205
  ...(ask
2197
2206
  ? [
2198
2207
  askCallRow(ask, "pending"),
@@ -122,6 +122,8 @@ export type GptHistoryPromptStatus = "streaming" | "complete" | "error";
122
122
  export interface GptHistoryPrompt {
123
123
  model?: string;
124
124
  rating?: "up" | "down";
125
+ /** Observability trace id of an assistant reply (written by the chat turn). */
126
+ traceId?: string;
125
127
  /** Set on assistant replies produced by /gpt/prompt. "streaming" while partial text is persisted. */
126
128
  status?: GptHistoryPromptStatus;
127
129
  /** Identifies one /gpt/prompt reply so resume clients can follow it. */
@@ -219,6 +221,8 @@ export interface GenerateObservabilityOptions {
219
221
  sensitive?: boolean;
220
222
  sessionId?: string;
221
223
  skipTrace?: boolean;
224
+ /** Id to store the trace under (from `createTraceId()`), so the caller can return it for feedback. */
225
+ traceId?: string;
222
226
  }
223
227
 
224
228
  export interface GenerateTextOptions extends GenerateObservabilityOptions {
@@ -145,7 +145,7 @@ export interface ObsScoreDocument extends mongoose.Document<mongoose.Types.Objec
145
145
  deleted: boolean;
146
146
  evaluatorId?: mongoose.Types.ObjectId;
147
147
  name: string;
148
- source: "code" | "human" | "llm-judge" | "user-feedback";
148
+ source: "code" | "human" | "llm-judge" | "user-feedback" | "user-signal";
149
149
  spanId?: mongoose.Types.ObjectId;
150
150
  traceId: mongoose.Types.ObjectId;
151
151
  updated: Date;