@terreno/ai 58.0.6 → 58.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/models/gptHistory.js +4 -0
- package/dist/observability/feedback.d.ts +31 -0
- package/dist/observability/feedback.js +218 -0
- package/dist/observability/local/datasetStore.d.ts +7 -0
- package/dist/observability/local/datasetStore.js +48 -2
- package/dist/observability/local/models/obsReviewItem.js +6 -0
- package/dist/observability/local/models/obsScore.js +12 -1
- package/dist/observability/local/promptStore.d.ts +1 -0
- package/dist/observability/local/promptStore.js +1 -0
- package/dist/observability/local/reviewStore.d.ts +5 -0
- package/dist/observability/local/reviewStore.js +17 -0
- package/dist/observability/local/traceStore.d.ts +23 -0
- package/dist/observability/local/traceStore.js +121 -10
- package/dist/observability/observabilityApp.d.ts +1 -0
- package/dist/observability/observabilityApp.js +2 -0
- package/dist/observability/observabilityAppRegistry.d.ts +1 -0
- package/dist/observability/routes/traces.js +72 -0
- package/dist/observability/types.d.ts +8 -1
- package/dist/routes/gpt.js +42 -2
- package/dist/routes/gptHistories.js +42 -2
- package/dist/service/aiService.d.ts +4 -1
- package/dist/service/aiService.js +17 -2
- package/dist/service/chatTurn.js +10 -1
- package/dist/types/index.d.ts +4 -0
- package/dist/types/observability.d.ts +1 -1
- package/package.json +8 -8
- package/src/index.ts +8 -1
- package/src/models/gptHistory.ts +5 -0
- package/src/observability/feedback.test.ts +688 -0
- package/src/observability/feedback.ts +263 -0
- package/src/observability/local/datasetStore.ts +57 -2
- package/src/observability/local/models/obsReviewItem.ts +9 -0
- package/src/observability/local/models/obsScore.ts +18 -1
- package/src/observability/local/obsModels.test.ts +10 -2
- package/src/observability/local/promptStore.test.ts +2 -0
- package/src/observability/local/promptStore.ts +2 -0
- package/src/observability/local/reviewStore.test.ts +55 -0
- package/src/observability/local/reviewStore.ts +21 -0
- package/src/observability/local/traceStore.test.ts +239 -1
- package/src/observability/local/traceStore.ts +140 -9
- package/src/observability/observabilityApp.ts +2 -0
- package/src/observability/observabilityAppRegistry.ts +1 -0
- package/src/observability/routes/datasets.test.ts +253 -0
- package/src/observability/routes/prompts.test.ts +1 -0
- package/src/observability/routes/traces.test.ts +581 -3
- package/src/observability/routes/traces.ts +87 -1
- package/src/observability/types.ts +6 -1
- package/src/routes/gpt.test.ts +263 -1
- package/src/routes/gpt.ts +43 -2
- package/src/routes/gptHistories.ts +55 -2
- package/src/service/aiService.observability.test.ts +244 -0
- package/src/service/aiService.ts +15 -2
- package/src/service/chatTurn.ts +11 -2
- package/src/types/index.ts +4 -0
- package/src/types/observability.ts +1 -1
|
@@ -3,14 +3,19 @@ import {TerrenoApp} from "@terreno/api";
|
|
|
3
3
|
import type express from "express";
|
|
4
4
|
|
|
5
5
|
import {authAsUser, ensureTestUsers, terrenoUserModel} from "../../tests/helpers";
|
|
6
|
+
import {LocalDatasetStore} from "../local/datasetStore";
|
|
6
7
|
import {createLocalObservabilityPlugin} from "../local/localPlugin";
|
|
7
8
|
import {registerObsDataset} from "../local/models/obsDataset";
|
|
8
9
|
import {registerObsDatasetItem} from "../local/models/obsDatasetItem";
|
|
9
10
|
import {registerObsPrompt} from "../local/models/obsPrompt";
|
|
10
11
|
import {registerObsPromptLabel} from "../local/models/obsPromptLabel";
|
|
11
12
|
import {registerObsPromptVersion} from "../local/models/obsPromptVersion";
|
|
13
|
+
import {registerObsScore} from "../local/models/obsScore";
|
|
14
|
+
import {registerObsTrace} from "../local/models/obsTrace";
|
|
12
15
|
import {LocalPromptStore} from "../local/promptStore";
|
|
16
|
+
import {LocalTraceStore} from "../local/traceStore";
|
|
13
17
|
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
18
|
+
import type {ScoreRecord} from "../types";
|
|
14
19
|
|
|
15
20
|
describe("observability dataset routes", () => {
|
|
16
21
|
let app: express.Application;
|
|
@@ -174,3 +179,251 @@ describe("observability dataset routes", () => {
|
|
|
174
179
|
expect(deleted.status).toBe(204);
|
|
175
180
|
});
|
|
176
181
|
});
|
|
182
|
+
|
|
183
|
+
describe("add-to-dataset carries user feedback", () => {
|
|
184
|
+
let app: express.Application;
|
|
185
|
+
const traces = new LocalTraceStore();
|
|
186
|
+
|
|
187
|
+
beforeAll(async () => {
|
|
188
|
+
await ensureTestUsers();
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
afterEach(() => {
|
|
192
|
+
resetObservabilityApp();
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
beforeEach(async () => {
|
|
196
|
+
await registerObsDataset().deleteMany({});
|
|
197
|
+
await registerObsDatasetItem().deleteMany({});
|
|
198
|
+
await registerObsTrace().deleteMany({});
|
|
199
|
+
await registerObsScore().deleteMany({});
|
|
200
|
+
app = new TerrenoApp({skipListen: true, userModel: terrenoUserModel})
|
|
201
|
+
.register(new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]}))
|
|
202
|
+
.build();
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
const makeTrace = async (sensitive = false): Promise<string> => {
|
|
206
|
+
const exported = await traces.exportTrace({
|
|
207
|
+
endedAt: "2026-01-01T00:00:01.000Z",
|
|
208
|
+
id: "unused",
|
|
209
|
+
input: {text: "hello"},
|
|
210
|
+
name: "generate",
|
|
211
|
+
output: {answer: "world"},
|
|
212
|
+
prompts: [],
|
|
213
|
+
sensitive,
|
|
214
|
+
spans: [],
|
|
215
|
+
startedAt: "2026-01-01T00:00:00.000Z",
|
|
216
|
+
status: "ok",
|
|
217
|
+
});
|
|
218
|
+
return exported.id;
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
const score = async (
|
|
222
|
+
traceId: string,
|
|
223
|
+
fields: Pick<ScoreRecord, "dataType" | "name" | "source" | "value"> & {comment?: string}
|
|
224
|
+
): Promise<void> => {
|
|
225
|
+
await traces.exportScore({traceId, ...fields});
|
|
226
|
+
};
|
|
227
|
+
|
|
228
|
+
// Item typed instead of `any` to satisfy the noExplicitAny lint rule.
|
|
229
|
+
interface AddedItem {
|
|
230
|
+
metadata: {userFeedback?: {[key: string]: unknown}};
|
|
231
|
+
proofread: boolean;
|
|
232
|
+
tags: string[];
|
|
233
|
+
[key: string]: unknown;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const addToDataset = async (traceId: string): Promise<{item: AddedItem; datasetId: string}> => {
|
|
237
|
+
const admin = await authAsUser(app, "admin");
|
|
238
|
+
const created = await admin.post("/ai/observability/datasets").send({name: "fb-set"});
|
|
239
|
+
const datasetId = created.body.data.id as string;
|
|
240
|
+
const res = await admin
|
|
241
|
+
.post("/ai/observability/traces/add-to-dataset")
|
|
242
|
+
.send({datasetId, traceIds: [traceId]});
|
|
243
|
+
expect(res.status).toBe(201);
|
|
244
|
+
return {datasetId, item: res.body.data[0]};
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
it("thumbs down with a comment lands unproofread, tagged, with userFeedback", async () => {
|
|
248
|
+
const traceId = await makeTrace();
|
|
249
|
+
await score(traceId, {
|
|
250
|
+
comment: "Too long",
|
|
251
|
+
dataType: "boolean",
|
|
252
|
+
name: "thumbs_up",
|
|
253
|
+
source: "user-feedback",
|
|
254
|
+
value: false,
|
|
255
|
+
});
|
|
256
|
+
const {item} = await addToDataset(traceId);
|
|
257
|
+
expect(item.proofread).toBe(false);
|
|
258
|
+
expect(item.tags).toContain("user:thumbs-down");
|
|
259
|
+
expect(item.metadata.userFeedback).toEqual({
|
|
260
|
+
comment: "Too long",
|
|
261
|
+
signals: [],
|
|
262
|
+
thumbs: "down",
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
it("thumbs up plus copied stays proofread with no user tags", async () => {
|
|
267
|
+
const traceId = await makeTrace();
|
|
268
|
+
await score(traceId, {
|
|
269
|
+
dataType: "boolean",
|
|
270
|
+
name: "thumbs_up",
|
|
271
|
+
source: "user-feedback",
|
|
272
|
+
value: true,
|
|
273
|
+
});
|
|
274
|
+
await score(traceId, {
|
|
275
|
+
dataType: "boolean",
|
|
276
|
+
name: "copied",
|
|
277
|
+
source: "user-signal",
|
|
278
|
+
value: true,
|
|
279
|
+
});
|
|
280
|
+
const {item} = await addToDataset(traceId);
|
|
281
|
+
expect(item.proofread).toBe(true);
|
|
282
|
+
expect(item.tags.filter((tag: string) => tag.startsWith("user:"))).toEqual([]);
|
|
283
|
+
expect(item.metadata.userFeedback).toEqual({signals: ["copied"], thumbs: "up"});
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
it("dismissed alone lands unproofread with user:dismissed", async () => {
|
|
287
|
+
const traceId = await makeTrace();
|
|
288
|
+
await score(traceId, {
|
|
289
|
+
dataType: "boolean",
|
|
290
|
+
name: "dismissed",
|
|
291
|
+
source: "user-signal",
|
|
292
|
+
value: true,
|
|
293
|
+
});
|
|
294
|
+
const {item} = await addToDataset(traceId);
|
|
295
|
+
expect(item.proofread).toBe(false);
|
|
296
|
+
expect(item.tags).toContain("user:dismissed");
|
|
297
|
+
expect(item.metadata.userFeedback).toEqual({signals: ["dismissed"]});
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
it("outcome fp tags user:false-positive and is unproofread", async () => {
|
|
301
|
+
const traceId = await makeTrace();
|
|
302
|
+
await score(traceId, {
|
|
303
|
+
dataType: "categorical",
|
|
304
|
+
name: "outcome",
|
|
305
|
+
source: "user-feedback",
|
|
306
|
+
value: "fp",
|
|
307
|
+
});
|
|
308
|
+
const {item} = await addToDataset(traceId);
|
|
309
|
+
expect(item.proofread).toBe(false);
|
|
310
|
+
expect(item.tags).toContain("user:false-positive");
|
|
311
|
+
expect(item.metadata.userFeedback).toEqual({outcome: "fp", signals: []});
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
it("outcome fn tags user:false-negative and is unproofread", async () => {
|
|
315
|
+
const traceId = await makeTrace();
|
|
316
|
+
await score(traceId, {
|
|
317
|
+
dataType: "categorical",
|
|
318
|
+
name: "outcome",
|
|
319
|
+
source: "user-feedback",
|
|
320
|
+
value: "fn",
|
|
321
|
+
});
|
|
322
|
+
const {item} = await addToDataset(traceId);
|
|
323
|
+
expect(item.proofread).toBe(false);
|
|
324
|
+
expect(item.tags).toContain("user:false-negative");
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
it("outcome tp stays proofread and carries the outcome", async () => {
|
|
328
|
+
const traceId = await makeTrace();
|
|
329
|
+
await score(traceId, {
|
|
330
|
+
dataType: "categorical",
|
|
331
|
+
name: "outcome",
|
|
332
|
+
source: "user-feedback",
|
|
333
|
+
value: "tp",
|
|
334
|
+
});
|
|
335
|
+
const {item} = await addToDataset(traceId);
|
|
336
|
+
expect(item.proofread).toBe(true);
|
|
337
|
+
expect(item.tags).toEqual([]);
|
|
338
|
+
expect(item.metadata.userFeedback).toEqual({outcome: "tp", signals: []});
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
it("applies every matching tag when thumbs down and dismissed both exist", async () => {
|
|
342
|
+
const traceId = await makeTrace();
|
|
343
|
+
await score(traceId, {
|
|
344
|
+
dataType: "boolean",
|
|
345
|
+
name: "thumbs_up",
|
|
346
|
+
source: "user-feedback",
|
|
347
|
+
value: false,
|
|
348
|
+
});
|
|
349
|
+
await score(traceId, {
|
|
350
|
+
dataType: "boolean",
|
|
351
|
+
name: "dismissed",
|
|
352
|
+
source: "user-signal",
|
|
353
|
+
value: true,
|
|
354
|
+
});
|
|
355
|
+
const {item} = await addToDataset(traceId);
|
|
356
|
+
expect(item.tags).toContain("user:thumbs-down");
|
|
357
|
+
expect(item.tags).toContain("user:dismissed");
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
it("a trace with no scores behaves as before", async () => {
|
|
361
|
+
const traceId = await makeTrace();
|
|
362
|
+
const {item} = await addToDataset(traceId);
|
|
363
|
+
expect(item.proofread).toBe(true);
|
|
364
|
+
expect(item.tags).toEqual([]);
|
|
365
|
+
expect(item.metadata?.userFeedback).toBeUndefined();
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
it("a deselected thumbs down does not count", async () => {
|
|
369
|
+
const traceId = await makeTrace();
|
|
370
|
+
await score(traceId, {
|
|
371
|
+
dataType: "boolean",
|
|
372
|
+
name: "thumbs_up",
|
|
373
|
+
source: "user-feedback",
|
|
374
|
+
value: false,
|
|
375
|
+
});
|
|
376
|
+
await traces.deleteFeedbackScore({name: "thumbs_up", source: "user-feedback", traceId});
|
|
377
|
+
const {item} = await addToDataset(traceId);
|
|
378
|
+
expect(item.proofread).toBe(true);
|
|
379
|
+
expect(item.tags).toEqual([]);
|
|
380
|
+
expect(item.metadata?.userFeedback).toBeUndefined();
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
it("a changed vote uses the latest thumbs value", async () => {
|
|
384
|
+
const traceId = await makeTrace();
|
|
385
|
+
await score(traceId, {
|
|
386
|
+
dataType: "boolean",
|
|
387
|
+
name: "thumbs_up",
|
|
388
|
+
source: "user-feedback",
|
|
389
|
+
value: false,
|
|
390
|
+
});
|
|
391
|
+
await score(traceId, {
|
|
392
|
+
dataType: "boolean",
|
|
393
|
+
name: "thumbs_up",
|
|
394
|
+
source: "user-feedback",
|
|
395
|
+
value: true,
|
|
396
|
+
});
|
|
397
|
+
const {item} = await addToDataset(traceId);
|
|
398
|
+
expect(item.proofread).toBe(true);
|
|
399
|
+
expect(item.metadata.userFeedback).toEqual({signals: [], thumbs: "up"});
|
|
400
|
+
});
|
|
401
|
+
|
|
402
|
+
it("sensitive trace with thumbs up stays unproofread and keeps the comment", async () => {
|
|
403
|
+
const traceId = await makeTrace(true);
|
|
404
|
+
await score(traceId, {
|
|
405
|
+
comment: "Nice",
|
|
406
|
+
dataType: "boolean",
|
|
407
|
+
name: "thumbs_up",
|
|
408
|
+
source: "user-feedback",
|
|
409
|
+
value: true,
|
|
410
|
+
});
|
|
411
|
+
const {item} = await addToDataset(traceId);
|
|
412
|
+
expect(item.proofread).toBe(false);
|
|
413
|
+
expect(item.metadata.userFeedback).toEqual({comment: "Nice", signals: [], thumbs: "up"});
|
|
414
|
+
});
|
|
415
|
+
|
|
416
|
+
it("an experiment without includeUnproofread skips the thumbs-down item", async () => {
|
|
417
|
+
const traceId = await makeTrace();
|
|
418
|
+
await score(traceId, {
|
|
419
|
+
dataType: "boolean",
|
|
420
|
+
name: "thumbs_up",
|
|
421
|
+
source: "user-feedback",
|
|
422
|
+
value: false,
|
|
423
|
+
});
|
|
424
|
+
const {datasetId} = await addToDataset(traceId);
|
|
425
|
+
const store = new LocalDatasetStore(new LocalPromptStore());
|
|
426
|
+
expect(await store.listExperimentItems(datasetId, false)).toHaveLength(0);
|
|
427
|
+
expect(await store.listExperimentItems(datasetId, true)).toHaveLength(1);
|
|
428
|
+
});
|
|
429
|
+
});
|