@terreno/ai 58.0.6 → 58.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -2
- package/dist/models/gptHistory.js +4 -0
- package/dist/observability/feedback.d.ts +31 -0
- package/dist/observability/feedback.js +218 -0
- package/dist/observability/local/datasetStore.d.ts +7 -0
- package/dist/observability/local/datasetStore.js +48 -2
- package/dist/observability/local/models/obsReviewItem.js +6 -0
- package/dist/observability/local/models/obsScore.js +12 -1
- package/dist/observability/local/promptStore.d.ts +1 -0
- package/dist/observability/local/promptStore.js +1 -0
- package/dist/observability/local/reviewStore.d.ts +5 -0
- package/dist/observability/local/reviewStore.js +17 -0
- package/dist/observability/local/traceStore.d.ts +23 -0
- package/dist/observability/local/traceStore.js +121 -10
- package/dist/observability/observabilityApp.d.ts +1 -0
- package/dist/observability/observabilityApp.js +2 -0
- package/dist/observability/observabilityAppRegistry.d.ts +1 -0
- package/dist/observability/routes/traces.js +72 -0
- package/dist/observability/types.d.ts +8 -1
- package/dist/routes/gpt.js +42 -2
- package/dist/routes/gptHistories.js +42 -2
- package/dist/service/aiService.d.ts +4 -1
- package/dist/service/aiService.js +17 -2
- package/dist/service/chatTurn.js +10 -1
- package/dist/types/index.d.ts +4 -0
- package/dist/types/observability.d.ts +1 -1
- package/package.json +8 -8
- package/src/index.ts +8 -1
- package/src/models/gptHistory.ts +5 -0
- package/src/observability/feedback.test.ts +688 -0
- package/src/observability/feedback.ts +263 -0
- package/src/observability/local/datasetStore.ts +57 -2
- package/src/observability/local/models/obsReviewItem.ts +9 -0
- package/src/observability/local/models/obsScore.ts +18 -1
- package/src/observability/local/obsModels.test.ts +10 -2
- package/src/observability/local/promptStore.test.ts +2 -0
- package/src/observability/local/promptStore.ts +2 -0
- package/src/observability/local/reviewStore.test.ts +55 -0
- package/src/observability/local/reviewStore.ts +21 -0
- package/src/observability/local/traceStore.test.ts +239 -1
- package/src/observability/local/traceStore.ts +140 -9
- package/src/observability/observabilityApp.ts +2 -0
- package/src/observability/observabilityAppRegistry.ts +1 -0
- package/src/observability/routes/datasets.test.ts +253 -0
- package/src/observability/routes/prompts.test.ts +1 -0
- package/src/observability/routes/traces.test.ts +581 -3
- package/src/observability/routes/traces.ts +87 -1
- package/src/observability/types.ts +6 -1
- package/src/routes/gpt.test.ts +263 -1
- package/src/routes/gpt.ts +43 -2
- package/src/routes/gptHistories.ts +55 -2
- package/src/service/aiService.observability.test.ts +244 -0
- package/src/service/aiService.ts +15 -2
- package/src/service/chatTurn.ts +11 -2
- package/src/types/index.ts +4 -0
- package/src/types/observability.ts +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -20,6 +20,58 @@ Upgrade notes for consumer action live in [`mcp-server/src/docs/upgrades/`](mcp-
|
|
|
20
20
|
|
|
21
21
|
Unreleased changes live in [`changelog/unreleased/`](changelog/unreleased/). Add one Markdown file per feature (see that directory's README) instead of editing this section.
|
|
22
22
|
|
|
23
|
+
## [58.1.0] - 2026-11-21
|
|
24
|
+
|
|
25
|
+
Upgrade note: [`mcp-server/src/docs/upgrades/58.1.0.md`](mcp-server/src/docs/upgrades/58.1.0.md).
|
|
26
|
+
|
|
27
|
+
### Breaking
|
|
28
|
+
|
|
29
|
+
- **@terreno/ui:** `GPTChat` now reads as a transcript: user messages in a light bubble, assistant text on the page in a 760px column, and tool calls as one expandable row capped at 240px. Breaking changes for apps and e2e suites that select these elements:
|
|
30
|
+
|
|
31
|
+
- `gpt-copy-image` is removed. Use `gpt-copy-msg-<index>` for copy. It copies the image for an image-only reply and the text when a reply has both.
|
|
32
|
+
- Tool rows are labeled `Tool: <name>` instead of `Result: <name>`.
|
|
33
|
+
- **Tools** (`gpt-tools-button`) appears beside the composer only when `mcpTools` has entries.
|
|
34
|
+
- A user or assistant message with no text, content parts, or blocks renders nothing, including its action row.
|
|
35
|
+
|
|
36
|
+
Download for an image reply is `gpt-download-image-<index>`. The **Tools** modal hides arguments until **Show arguments** is checked, and it resets to hidden when closed.
|
|
37
|
+
- **Upgrade:** switching an app with connected clients from the default `mcp` scope to `grants` ends access for live `mcp`-scoped tokens (users reconnect), and confidential clients stored with `scope: "mcp"` get `invalid_scope` at `/oauth/token` until their stored scope is updated. Custom tools from `registerMCPTool` default to `write`, so add `access: "read"` to read-only tools or read-only grants will not see them. Bearer-mode web apps add the consent path to `oauthHandoff.callbackPaths`. See [Enable MCP OAuth](docs/how-to/enable-mcp-oauth.md).
|
|
38
|
+
- **Behavior change, even without `grants`:** a confidential client whose stored `scope` lists only scopes the server no longer supports now gets `invalid_scope` from `client_credentials`, where it used to get a token. Update the client's `scope`, or leave it unset to receive every supported scope.
|
|
39
|
+
|
|
40
|
+
### Added
|
|
41
|
+
|
|
42
|
+
- **@terreno/admin-frontend:** AI Prompts nest by folder. A slash in a prompt name is a folder level: each folder is a table row you open, and an up row returns to the parent. Tag and search filters match prompts in every folder. The list filters by one tag.
|
|
43
|
+
- **@terreno/ai:** `GET /ai/observability/prompts` returns each prompt's `tags`, which the admin list filters on.
|
|
44
|
+
- **@terreno/admin-frontend:** dataset items open on their own page. Expected output is a form built from `expectedOutputSchema`, or from the stored JSON when no schema is set. The expected output is the golden label, and **Save and next** moves through the Needs review queue in list order.
|
|
45
|
+
- **@terreno/admin-frontend:** `AdminRolesList` takes `isActionVisible({resource, action})`. Return false to hide a statement action from the role list and the role editor. Hidden actions stay on a saved role and are written back unchanged. The predicate receives the resource, so the same verb can be hidden on one model and shown on another.
|
|
46
|
+
- **@terreno/ai:** end users can give feedback on any AI output. `POST /ai/observability/traces/:id/feedback` (trace owner only) records thumbs with an optional comment and outcome class as `user-feedback` scores, passive signals (`accepted`, `dismissed`, `copied`, `edited`, `regenerated`) as the new `user-signal` score source, and `flagDataset`. `recordTraceFeedback` exposes the same logic to server code.
|
|
47
|
+
- **@terreno/ai:** `createTraceId()` and the `traceId` generate option let a route return an output's trace id. `/gpt/prompt` stores `traceId` on each assistant reply, and `PATCH /gpt/histories/:id/rating` accepts a `comment` and records the rating on the reply's trace.
|
|
48
|
+
- **@terreno/ai:** `ObservabilityApp({feedbackReview: {evaluatorName}})` queues every thumbs-down trace in the review queue with reason `feedback`.
|
|
49
|
+
- **@terreno/ai:** `GET /ai/observability/traces` filters by `thumbs`, `signal`, and `hasComment`. Adding a trace the user rejected to a dataset now lands it `proofread: false` with a `user:` tag and `metadata.userFeedback`, so known-bad outputs never become gold labels.
|
|
50
|
+
- **@terreno/ui:** `ThumbsUpDownFeedback` takes `onCommentSubmit` for an optional comment (new `FeedbackCommentField`), and `GPTChat` takes `onRateComment` and `onMessageSignal`.
|
|
51
|
+
- **@terreno/admin-frontend:** AI Observability Traces adds **Thumbs**, **Signal**, and **Has comment** filters, shows each score's source, and shows user comments (collapsed on sensitive traces).
|
|
52
|
+
- **@terreno/admin-frontend:** dataset items open with review steps and an explanation of the golden label. **Save and next** on a Needs review item opens the next unproofread item in list order (wrapping), and the last one returns to the Needs review tab. The dataset shows a golden-label note and a Needs review empty state, and `?tab=needsReview` opens that tab.
|
|
53
|
+
- **@terreno/cli:** every MCP tool is now a `terreno` command, so an agent with a shell needs no MCP configuration. New commands: `terreno syncdb action` and `terreno syncdb snapshot`, `terreno storage read|write`, `terreno dev list|run`, `terreno dispatch`, `terreno docs notes search|get|ask`, and `terreno validate blocks`. `terreno tool list|schema|call` reaches any tool by its MCP name.
|
|
54
|
+
- **@terreno/cli:** `terreno browser <action>` drives one web page across separate commands. The page and SyncDB snapshots live in a background CLI session (`terreno session start|stop|status`) that those commands start themselves.
|
|
55
|
+
- **@terreno/cli:** `terreno login` signs in with Better Auth or JWT and saves a token that `terreno api` sends to the same API origin. `terreno logout` removes it.
|
|
56
|
+
- **@terreno/mcp:** new exports `@terreno/mcp/project-root` (`resolveTerrenoProjectRoot`) and `@terreno/mcp/local-tool-names` (`LOCAL_TOOL_NAMES`), and `checkUiBlocks` on `@terreno/mcp/tools`, which reports whether a block document is valid along with the `terreno-blocks validate` text.
|
|
57
|
+
- **@terreno/api:** `mcpOAuth.grants` enforces MCP OAuth scopes. Pass `"readOnly"` (`mcp:read`), `"readWrite"` (`mcp:read`, `mcp:write`), or your own `McpOAuthGrant[]` (`{scope, label, description?, access: "read" | "write"}`). Every MCP tool is `read` or `write` (CRUD `list`/`read` and GET actions are `read`; the rest are `write`), and an OAuth token sees and can call only the tools its grants allow. Write implies read. `MCPConfig.scopes`, an action's `mcp.scope`, and `registerMCPTool({access, scope})` tag tools; `canUseMcpTool` and `MCP_OAUTH_GRANT_PRESETS` are exported. Without `grants`, nothing changes.
|
|
58
|
+
- **@terreno/api:** `POST /oauth/authorize/:request/approve` accepts an optional `{scope}` so the user can approve a subset of the requested scopes, and `GET /oauth/authorize/:request` returns `grants` with each grant's label and description. `client_credentials` now intersects a client's stored scope with the supported scopes and answers `invalid_scope` when nothing is left.
|
|
59
|
+
- **@terreno/syncdb:** `TerrenoOAuthConsentScreen` is a drop-in MCP OAuth consent page: `TerrenoLoginScreen` inline while signed out, then the client, redirect host, a checkbox per grant, and Approve and Deny. `useMcpOAuthConsent` is the hook behind it. `TerrenoLoginScreen` and `useLoginController` take `methods` to show only some of the server's sign-in methods.
|
|
60
|
+
- **@terreno/api:** `modelRouter` takes an opt-in `bulk` option (`true` or `{maxItems, methods}`) that mounts `POST /<route>/bulk`. One request carries up to 100 create, update, and delete operations, each run through the router's permissions, hooks, validation, audit, and `responseHandler`. By default the batch runs in one MongoDB transaction and commits or rolls back together (replica set required); `"atomic": false` runs each operation on its own and returns a status per operation. See `docs/reference/api.md` ("modelRouter bulk writes").
|
|
61
|
+
- **@terreno/api:** `validateModelRouterBody`, which validates one create or update body the way `POST /` and `PATCH /:id` do.
|
|
62
|
+
- Role permissions can carry an optional description. `createAccess({ statementDescriptions })` merges app labels over the framework labels, and `GET /rbac/statements` returns them as `data.statementDescriptions`. The admin roles screen shows each label beside its action badge. Pass `showActionBadges={false}` to hide the badges and stack one label per line.
|
|
63
|
+
|
|
64
|
+
### Fixed
|
|
65
|
+
|
|
66
|
+
- Admin shell content padding uses longhand values. react-native-web drops a multi-value `padding` string, so admin pages sat flush against the sidebar.
|
|
67
|
+
- Data table alternate rows in light mode use `neutral100` (`#E6E6E6`) instead of `neutralLight` (`#CDCDCD`).
|
|
68
|
+
- **@terreno/ui:** `DraggableList` accepts `variableHeight`. Each row is as tall as its content, and the next row starts below it. During a drag, rows use their measured heights to pick the drop slot. `itemHeight` is the placeholder until a row is measured.
|
|
69
|
+
- **@terreno/admin-frontend:** arrays of subdocuments, such as feature-flag rules and variants, use `variableHeight`. A rule with a long user list no longer covers the Type and Variants fields below it.
|
|
70
|
+
- **@terreno/admin-frontend:** reordering an array of subdocuments by drag keeps the new order. Before, the rows snapped back to their original order after release.
|
|
71
|
+
- `@terreno/test`: `registerBackendPreload` and `registerSimpleMongoPreload` apply `testEnv` when they are called, not only in `beforeEach`. An app built in a test file's `beforeAll` used to miss it, so the first file in a run built its app without `TOKEN_SECRET`: `TerrenoApp` skipped the JWT strategy, and anonymous requests got 500 `Unknown authentication strategy "jwt"`.
|
|
72
|
+
- `@terreno/test`: new `testRequest(app)`, a drop-in for `supertest(app)` that binds 127.0.0.1. Under Bun, `supertest(app)` listens on the IPv6 wildcard only and sends to `127.0.0.1`, so on macOS another local process holding that port could answer, failing random tests with its 404 or 401. Bun also left each supertest server open after its request; these now close. `authAsUser` and `authWithToken` use the same server. `createLoopbackServer(app)` returns it for your own use.
|
|
73
|
+
- Web spell check is on again for prose `TextField` and `TextArea` values; browser autocorrect stays off on web so controlled inputs do not oscillate between two spellings.
|
|
74
|
+
|
|
23
75
|
## [58.0.6] - 2026-11-14
|
|
24
76
|
|
|
25
77
|
### Fixed
|
package/dist/index.d.ts
CHANGED
|
@@ -11,6 +11,7 @@ export { FileAttachment } from "./models/fileAttachment";
|
|
|
11
11
|
export { GptHistory } from "./models/gptHistory";
|
|
12
12
|
export { getProjectModel, Project } from "./models/project";
|
|
13
13
|
export { AI_OBSERVABILITY_GROUP, observabilityAdminScreens } from "./observability/adminScreens";
|
|
14
|
+
export { FEEDBACK_SIGNALS, type FeedbackSignal, recordTraceFeedback, type TraceFeedbackInput, type TraceFeedbackResult, } from "./observability/feedback";
|
|
14
15
|
export { answersAgree, computeJudgeAgreement, type JudgeAgreement, } from "./observability/judgeAgreement";
|
|
15
16
|
export { LocalDatasetStore } from "./observability/local/datasetStore";
|
|
16
17
|
export { type EvaluatorCalibration, LocalEvaluatorStore } from "./observability/local/evaluatorStore";
|
|
@@ -36,7 +37,7 @@ export { addProjectRoutes } from "./routes/projects";
|
|
|
36
37
|
export { findAgentBlock } from "./service/agentBlocks";
|
|
37
38
|
export type { RegisteredDataset } from "./service/aiDatasets";
|
|
38
39
|
export { configureAiDatasets, registerAiDataset } from "./service/aiDatasets";
|
|
39
|
-
export { AIService, TemperaturePresets } from "./service/aiService";
|
|
40
|
+
export { AIService, createTraceId, TemperaturePresets } from "./service/aiService";
|
|
40
41
|
export { createAskTools } from "./service/asks";
|
|
41
42
|
export { runBufferedChatTurn } from "./service/chatTurn";
|
|
42
43
|
export { FileStorageService } from "./service/fileStorage";
|
package/dist/index.js
CHANGED
|
@@ -14,8 +14,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.
|
|
18
|
-
exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.AIService = void 0;
|
|
17
|
+
exports.findAgentBlock = exports.addProjectRoutes = exports.addMcpRoutes = exports.addGptHistoryRoutes = exports.addGptRoutes = exports.addFileRoutes = exports.addAiRequestsExplorerRoutes = exports.validateObservabilityConfig = exports.resolveObservabilityControl = exports.DEFAULT_OBSERVABILITY_CONTROL = exports.isLocalObservabilityPluginOn = exports.buildPlaygroundAiStatus = exports.buildObservabilityStatus = exports.questionSetFingerprint = exports.resetObservabilityApp = exports.ObservabilityApp = exports.getObservabilityApp = exports.MemoryTraceSink = exports.MemoryScoreSink = exports.LocalTraceStore = exports.LocalTraceSink = exports.LocalScoreSink = exports.LocalReviewStore = exports.MAX_REVIEW_BATCH_SIZE = exports.LocalReviewBatchStore = exports.DEFAULT_REVIEW_BATCH_SIZE = exports.LocalPromptStore = exports.createLocalObservabilityPlugin = exports.createLocalObservabilityBundle = exports.loadCurrentHumanAnswers = exports.LocalExperimentRunner = exports.DEFAULT_TARGET_AGREEMENT = exports.LocalEvaluatorStore = exports.LocalDatasetStore = exports.computeJudgeAgreement = exports.answersAgree = exports.recordTraceFeedback = exports.FEEDBACK_SIGNALS = exports.observabilityAdminScreens = exports.AI_OBSERVABILITY_GROUP = exports.Project = exports.getProjectModel = exports.GptHistory = exports.FileAttachment = exports.AIRequest = exports.AIDataset = exports.AiApp = exports.AIAdminApp = exports.Output = exports.jsonSchema = void 0;
|
|
18
|
+
exports.verifyVertexModelsEnabled = exports.normalizeVertexModelId = exports.listEnabledVertexModels = exports.isVertexModelAllowed = exports.DEFAULT_VERTEX_LOCATION = exports.createVertexProvider = exports.assertVertexModelsEnabled = exports.toggleChecklistHostAction = exports.scaleStepperHostAction = exports.TRANSLATION_PROMPT = exports.TITLE_GENERATION_PROMPT = exports.TERRENO_ASKS_SYSTEM_PROMPT = exports.REMIX_PROMPT = exports.JSON_VALUE_SYSTEM_PROMPT = exports.DEFAULT_GPT_MEMORY = exports.CONTENT_SUMMARY_PROMPT = exports.COMPACT_SURFACE_SYSTEM_PROMPT = exports.parseAiJson = exports.normalizeLlmJsonTextForStructuredOutput = exports.MCPService = exports.getMCPTools = exports.normalizeGeminiModelId = exports.listGeminiApiModels = exports.GEMINI_API_BASE_URL = exports.FileStorageService = exports.runBufferedChatTurn = exports.createAskTools = exports.TemperaturePresets = exports.createTraceId = exports.AIService = exports.registerAiDataset = exports.configureAiDatasets = void 0;
|
|
19
19
|
var ai_1 = require("ai");
|
|
20
20
|
Object.defineProperty(exports, "jsonSchema", { enumerable: true, get: function () { return ai_1.jsonSchema; } });
|
|
21
21
|
Object.defineProperty(exports, "Output", { enumerable: true, get: function () { return ai_1.Output; } });
|
|
@@ -37,6 +37,9 @@ Object.defineProperty(exports, "Project", { enumerable: true, get: function () {
|
|
|
37
37
|
var adminScreens_1 = require("./observability/adminScreens");
|
|
38
38
|
Object.defineProperty(exports, "AI_OBSERVABILITY_GROUP", { enumerable: true, get: function () { return adminScreens_1.AI_OBSERVABILITY_GROUP; } });
|
|
39
39
|
Object.defineProperty(exports, "observabilityAdminScreens", { enumerable: true, get: function () { return adminScreens_1.observabilityAdminScreens; } });
|
|
40
|
+
var feedback_1 = require("./observability/feedback");
|
|
41
|
+
Object.defineProperty(exports, "FEEDBACK_SIGNALS", { enumerable: true, get: function () { return feedback_1.FEEDBACK_SIGNALS; } });
|
|
42
|
+
Object.defineProperty(exports, "recordTraceFeedback", { enumerable: true, get: function () { return feedback_1.recordTraceFeedback; } });
|
|
40
43
|
var judgeAgreement_1 = require("./observability/judgeAgreement");
|
|
41
44
|
Object.defineProperty(exports, "answersAgree", { enumerable: true, get: function () { return judgeAgreement_1.answersAgree; } });
|
|
42
45
|
Object.defineProperty(exports, "computeJudgeAgreement", { enumerable: true, get: function () { return judgeAgreement_1.computeJudgeAgreement; } });
|
|
@@ -99,6 +102,7 @@ Object.defineProperty(exports, "configureAiDatasets", { enumerable: true, get: f
|
|
|
99
102
|
Object.defineProperty(exports, "registerAiDataset", { enumerable: true, get: function () { return aiDatasets_1.registerAiDataset; } });
|
|
100
103
|
var aiService_1 = require("./service/aiService");
|
|
101
104
|
Object.defineProperty(exports, "AIService", { enumerable: true, get: function () { return aiService_1.AIService; } });
|
|
105
|
+
Object.defineProperty(exports, "createTraceId", { enumerable: true, get: function () { return aiService_1.createTraceId; } });
|
|
102
106
|
Object.defineProperty(exports, "TemperaturePresets", { enumerable: true, get: function () { return aiService_1.TemperaturePresets; } });
|
|
103
107
|
var asks_1 = require("./service/asks");
|
|
104
108
|
Object.defineProperty(exports, "createAskTools", { enumerable: true, get: function () { return asks_1.createAskTools; } });
|
|
@@ -124,6 +124,10 @@ const gptHistoryPromptSchema = new mongoose_1.default.Schema({
|
|
|
124
124
|
type: String,
|
|
125
125
|
},
|
|
126
126
|
toolName: { description: "Name of the tool that was invoked", type: String },
|
|
127
|
+
traceId: {
|
|
128
|
+
description: "Observability trace id of this assistant reply; feedback on the reply is recorded against it",
|
|
129
|
+
type: String,
|
|
130
|
+
},
|
|
127
131
|
type: {
|
|
128
132
|
description: "Role of this message in the conversation",
|
|
129
133
|
enum: ["user", "assistant", "system", "tool-call", "tool-result"],
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ScoreRecord } from "./types";
|
|
2
|
+
/** Passive signals a product can record about what a user did with an AI output. */
|
|
3
|
+
export declare const FEEDBACK_SIGNALS: readonly ["accepted", "dismissed", "copied", "edited", "regenerated"];
|
|
4
|
+
export type FeedbackSignal = (typeof FEEDBACK_SIGNALS)[number];
|
|
5
|
+
export declare const FEEDBACK_COMMENT_MAX_LENGTH = 2000;
|
|
6
|
+
export interface TraceFeedbackInput {
|
|
7
|
+
comment?: string;
|
|
8
|
+
flagDataset?: boolean;
|
|
9
|
+
outcome?: "fn" | "fp" | "tn" | "tp";
|
|
10
|
+
signal?: FeedbackSignal;
|
|
11
|
+
thumbs?: "down" | "up" | null;
|
|
12
|
+
}
|
|
13
|
+
export interface TraceFeedbackResult {
|
|
14
|
+
flaggedForDataset: boolean;
|
|
15
|
+
reviewItemId?: string;
|
|
16
|
+
scores: ScoreRecord[];
|
|
17
|
+
traceId: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Records an end user's feedback on a trace they own. Only the trace's own user may call it:
|
|
21
|
+
* admins score traces through `POST /traces/:id/scores`, so `user-feedback` always means the
|
|
22
|
+
* user said it.
|
|
23
|
+
*/
|
|
24
|
+
export declare const recordTraceFeedback: (args: {
|
|
25
|
+
input: TraceFeedbackInput;
|
|
26
|
+
traceId: string;
|
|
27
|
+
user: {
|
|
28
|
+
admin?: boolean;
|
|
29
|
+
id: string;
|
|
30
|
+
};
|
|
31
|
+
}) => Promise<TraceFeedbackResult>;
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.recordTraceFeedback = exports.FEEDBACK_COMMENT_MAX_LENGTH = exports.FEEDBACK_SIGNALS = void 0;
|
|
4
|
+
const api_1 = require("@terreno/api");
|
|
5
|
+
const evaluatorStore_1 = require("./local/evaluatorStore");
|
|
6
|
+
const obsReviewItem_1 = require("./local/models/obsReviewItem");
|
|
7
|
+
const obsTrace_1 = require("./local/models/obsTrace");
|
|
8
|
+
const reviewStore_1 = require("./local/reviewStore");
|
|
9
|
+
const traceStore_1 = require("./local/traceStore");
|
|
10
|
+
const observabilityAppRegistry_1 = require("./observabilityAppRegistry");
|
|
11
|
+
/** Passive signals a product can record about what a user did with an AI output. */
|
|
12
|
+
exports.FEEDBACK_SIGNALS = [
|
|
13
|
+
"accepted",
|
|
14
|
+
"dismissed",
|
|
15
|
+
"copied",
|
|
16
|
+
"edited",
|
|
17
|
+
"regenerated",
|
|
18
|
+
];
|
|
19
|
+
exports.FEEDBACK_COMMENT_MAX_LENGTH = 2000;
|
|
20
|
+
/** Score name for explicit thumbs; `true` is up. */
|
|
21
|
+
const THUMBS_SCORE_NAME = "thumbs_up";
|
|
22
|
+
const TRACE_ID_PATTERN = /^[0-9a-f]{24}$/i;
|
|
23
|
+
const KNOWN_FIELDS = new Set(["comment", "flagDataset", "outcome", "signal", "thumbs"]);
|
|
24
|
+
const OUTCOMES = ["tp", "fp", "tn", "fn"];
|
|
25
|
+
/** Score name for outcome class feedback (`tp` / `fp` / `tn` / `fn`). */
|
|
26
|
+
const OUTCOME_SCORE_NAME = "outcome";
|
|
27
|
+
const badRequest = (title) => {
|
|
28
|
+
return new api_1.APIError({ status: 400, title });
|
|
29
|
+
};
|
|
30
|
+
/** Validates the request body; the comment comes back trimmed. */
|
|
31
|
+
const parseInput = (input) => {
|
|
32
|
+
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
|
33
|
+
throw badRequest("feedback body must be an object");
|
|
34
|
+
}
|
|
35
|
+
const body = input;
|
|
36
|
+
const keys = Object.keys(body).filter((key) => body[key] !== undefined);
|
|
37
|
+
if (keys.length === 0) {
|
|
38
|
+
throw badRequest("feedback needs thumbs, comment, outcome, signal, or flagDataset");
|
|
39
|
+
}
|
|
40
|
+
const unknownKey = keys.find((key) => !KNOWN_FIELDS.has(key));
|
|
41
|
+
if (unknownKey) {
|
|
42
|
+
throw badRequest(`Unknown feedback field "${unknownKey}"`);
|
|
43
|
+
}
|
|
44
|
+
const { flagDataset, outcome, signal, thumbs } = body;
|
|
45
|
+
if (thumbs !== undefined && thumbs !== null && thumbs !== "up" && thumbs !== "down") {
|
|
46
|
+
throw badRequest("thumbs must be 'up', 'down', or null");
|
|
47
|
+
}
|
|
48
|
+
if (outcome !== undefined && !OUTCOMES.includes(outcome)) {
|
|
49
|
+
throw badRequest(`outcome must be one of ${OUTCOMES.join(", ")}`);
|
|
50
|
+
}
|
|
51
|
+
if (signal !== undefined && !exports.FEEDBACK_SIGNALS.includes(signal)) {
|
|
52
|
+
throw badRequest(`signal must be one of ${exports.FEEDBACK_SIGNALS.join(", ")}`);
|
|
53
|
+
}
|
|
54
|
+
if (flagDataset !== undefined && typeof flagDataset !== "boolean") {
|
|
55
|
+
throw badRequest("flagDataset must be a boolean");
|
|
56
|
+
}
|
|
57
|
+
const parsed = {
|
|
58
|
+
flagDataset: flagDataset,
|
|
59
|
+
outcome: outcome,
|
|
60
|
+
signal: signal,
|
|
61
|
+
thumbs: thumbs,
|
|
62
|
+
};
|
|
63
|
+
if (body.comment === undefined) {
|
|
64
|
+
return parsed;
|
|
65
|
+
}
|
|
66
|
+
if (typeof body.comment !== "string") {
|
|
67
|
+
throw badRequest("comment must be a string");
|
|
68
|
+
}
|
|
69
|
+
const comment = body.comment.trim();
|
|
70
|
+
if (!comment) {
|
|
71
|
+
throw badRequest("comment must not be empty");
|
|
72
|
+
}
|
|
73
|
+
if (comment.length > exports.FEEDBACK_COMMENT_MAX_LENGTH) {
|
|
74
|
+
throw badRequest(`comment must be at most ${exports.FEEDBACK_COMMENT_MAX_LENGTH} characters`);
|
|
75
|
+
}
|
|
76
|
+
if (thumbs !== "up" && thumbs !== "down" && parsed.outcome === undefined) {
|
|
77
|
+
throw badRequest("comment needs thumbs or outcome");
|
|
78
|
+
}
|
|
79
|
+
return { ...parsed, comment };
|
|
80
|
+
};
|
|
81
|
+
/** Sends a score to every registered sink; the local sink upserts feedback scores. */
|
|
82
|
+
const exportToSinks = async (score, store) => {
|
|
83
|
+
const sinks = (0, observabilityAppRegistry_1.getObservabilityApp)()?.scoreSinks ?? [];
|
|
84
|
+
if (sinks.length === 0) {
|
|
85
|
+
await store.exportScore(score);
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
const results = await Promise.allSettled(sinks.map((sink) => {
|
|
89
|
+
return sink.export(score);
|
|
90
|
+
}));
|
|
91
|
+
// The local store is the record the app reads, so its failure fails the request. Other sinks
|
|
92
|
+
// are best-effort, like trace export: a vendor outage must not stop review queueing.
|
|
93
|
+
results.forEach((result, index) => {
|
|
94
|
+
if (result.status === "fulfilled") {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (sinks[index] instanceof traceStore_1.LocalScoreSink) {
|
|
98
|
+
throw result.reason;
|
|
99
|
+
}
|
|
100
|
+
api_1.logger.error("Feedback ScoreSink.export failed", { error: result.reason, name: score.name });
|
|
101
|
+
});
|
|
102
|
+
};
|
|
103
|
+
/**
|
|
104
|
+
* Queues a thumbs-down trace for human review against `feedbackReview.evaluatorName`, reusing an
|
|
105
|
+
* open item. Never fails the feedback request: a bad evaluator is logged and skipped.
|
|
106
|
+
*/
|
|
107
|
+
const enqueueFeedbackReview = async (traceId) => {
|
|
108
|
+
const evaluatorName = (0, observabilityAppRegistry_1.getObservabilityApp)()?.feedbackReview?.evaluatorName;
|
|
109
|
+
if (!evaluatorName) {
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
const evaluator = await new evaluatorStore_1.LocalEvaluatorStore().getByName(evaluatorName);
|
|
114
|
+
if (!evaluator) {
|
|
115
|
+
api_1.logger.error("feedbackReview evaluator not found; thumbs down not queued", { evaluatorName });
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
const reviewStore = new reviewStore_1.LocalReviewStore();
|
|
119
|
+
const open = await reviewStore.findOpen({ evaluatorId: evaluator.id, traceId });
|
|
120
|
+
if (open) {
|
|
121
|
+
return open.id;
|
|
122
|
+
}
|
|
123
|
+
// The pending-feedback unique index must exist before the first insert can race.
|
|
124
|
+
await (0, obsReviewItem_1.registerObsReviewItem)().init();
|
|
125
|
+
try {
|
|
126
|
+
const [item] = await reviewStore.enqueue({
|
|
127
|
+
evaluatorId: evaluator.id,
|
|
128
|
+
reason: "feedback",
|
|
129
|
+
traceIds: [traceId],
|
|
130
|
+
});
|
|
131
|
+
return item?.id;
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
// A concurrent thumbs down queued it first; return that item.
|
|
135
|
+
if (error.code !== 11000) {
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
return (await reviewStore.findOpen({ evaluatorId: evaluator.id, traceId }))?.id;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
api_1.logger.error("Failed to queue thumbs-down trace for review", { error, evaluatorName, traceId });
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
/**
|
|
147
|
+
* Records an end user's feedback on a trace they own. Only the trace's own user may call it:
|
|
148
|
+
* admins score traces through `POST /traces/:id/scores`, so `user-feedback` always means the
|
|
149
|
+
* user said it.
|
|
150
|
+
*/
|
|
151
|
+
const recordTraceFeedback = async (args) => {
|
|
152
|
+
if (!TRACE_ID_PATTERN.test(args.traceId)) {
|
|
153
|
+
throw badRequest("trace id must be an object id");
|
|
154
|
+
}
|
|
155
|
+
const input = parseInput(args.input);
|
|
156
|
+
const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: args.traceId });
|
|
157
|
+
if (!trace) {
|
|
158
|
+
throw new api_1.APIError({ status: 404, title: "Trace not found" });
|
|
159
|
+
}
|
|
160
|
+
if (!trace.userId || String(trace.userId) !== args.user.id) {
|
|
161
|
+
throw new api_1.APIError({ status: 403, title: "Only the trace's user can give feedback on it" });
|
|
162
|
+
}
|
|
163
|
+
const store = new traceStore_1.LocalTraceStore();
|
|
164
|
+
const scores = [];
|
|
165
|
+
const commentFields = input.comment ? { comment: input.comment } : {};
|
|
166
|
+
if (input.thumbs === null) {
|
|
167
|
+
await store.deleteFeedbackScore({
|
|
168
|
+
name: THUMBS_SCORE_NAME,
|
|
169
|
+
source: "user-feedback",
|
|
170
|
+
traceId: args.traceId,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
else if (input.thumbs) {
|
|
174
|
+
scores.push({
|
|
175
|
+
dataType: "boolean",
|
|
176
|
+
name: THUMBS_SCORE_NAME,
|
|
177
|
+
source: "user-feedback",
|
|
178
|
+
traceId: args.traceId,
|
|
179
|
+
value: input.thumbs === "up",
|
|
180
|
+
...commentFields,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
if (input.outcome) {
|
|
184
|
+
scores.push({
|
|
185
|
+
dataType: "categorical",
|
|
186
|
+
name: OUTCOME_SCORE_NAME,
|
|
187
|
+
source: "user-feedback",
|
|
188
|
+
traceId: args.traceId,
|
|
189
|
+
value: input.outcome,
|
|
190
|
+
...commentFields,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
if (input.signal) {
|
|
194
|
+
scores.push({
|
|
195
|
+
dataType: "boolean",
|
|
196
|
+
name: input.signal,
|
|
197
|
+
source: "user-signal",
|
|
198
|
+
traceId: args.traceId,
|
|
199
|
+
value: true,
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
for (const score of scores) {
|
|
203
|
+
await exportToSinks(score, store);
|
|
204
|
+
}
|
|
205
|
+
let flaggedForDataset = Boolean(trace.flaggedForDataset);
|
|
206
|
+
if (input.flagDataset !== undefined) {
|
|
207
|
+
await (0, obsTrace_1.registerObsTrace)().updateOne({ _id: trace._id }, { $set: { flaggedForDataset: input.flagDataset } });
|
|
208
|
+
flaggedForDataset = input.flagDataset;
|
|
209
|
+
}
|
|
210
|
+
const reviewItemId = input.thumbs === "down" ? await enqueueFeedbackReview(args.traceId) : undefined;
|
|
211
|
+
return {
|
|
212
|
+
flaggedForDataset,
|
|
213
|
+
...(reviewItemId ? { reviewItemId } : {}),
|
|
214
|
+
scores,
|
|
215
|
+
traceId: args.traceId,
|
|
216
|
+
};
|
|
217
|
+
};
|
|
218
|
+
exports.recordTraceFeedback = recordTraceFeedback;
|
|
@@ -64,6 +64,13 @@ export interface DatasetImportResult {
|
|
|
64
64
|
}>;
|
|
65
65
|
}
|
|
66
66
|
export declare const throwOnDatasetImportErrors: (result: DatasetImportResult) => void;
|
|
67
|
+
/** What the trace's user said and did, copied onto dataset items added from the trace. */
|
|
68
|
+
export interface DatasetItemUserFeedback {
|
|
69
|
+
comment?: string;
|
|
70
|
+
outcome?: string;
|
|
71
|
+
signals: string[];
|
|
72
|
+
thumbs?: "down" | "up";
|
|
73
|
+
}
|
|
67
74
|
export declare class LocalDatasetStore {
|
|
68
75
|
private readonly promptStore;
|
|
69
76
|
constructor(promptStore?: LocalPromptStore);
|
|
@@ -12,6 +12,7 @@ const schemaValidation_1 = require("../schemaValidation");
|
|
|
12
12
|
const humanAnswers_1 = require("./humanAnswers");
|
|
13
13
|
const obsDataset_1 = require("./models/obsDataset");
|
|
14
14
|
const obsDatasetItem_1 = require("./models/obsDatasetItem");
|
|
15
|
+
const obsScore_1 = require("./models/obsScore");
|
|
15
16
|
const obsTrace_1 = require("./models/obsTrace");
|
|
16
17
|
const promptStore_1 = require("./promptStore");
|
|
17
18
|
const throwOnDatasetImportErrors = (result) => {
|
|
@@ -79,6 +80,48 @@ const computeDatasetCounts = async (datasetId) => {
|
|
|
79
80
|
total,
|
|
80
81
|
};
|
|
81
82
|
};
|
|
83
|
+
/**
|
|
84
|
+
* Reads a trace's live end-user scores. Negative feedback means the trace output is a known-bad
|
|
85
|
+
* answer, so the item must not be treated as a gold label until a human labels it.
|
|
86
|
+
*/
|
|
87
|
+
const readUserFeedback = async (traceId) => {
|
|
88
|
+
const scores = await (0, obsScore_1.registerObsScore)()
|
|
89
|
+
.find({ source: { $in: ["user-feedback", "user-signal"] }, traceId })
|
|
90
|
+
.sort({ created: 1 });
|
|
91
|
+
if (scores.length === 0) {
|
|
92
|
+
return { negativeTags: [] };
|
|
93
|
+
}
|
|
94
|
+
const feedback = { signals: [] };
|
|
95
|
+
const negativeTags = [];
|
|
96
|
+
for (const score of scores) {
|
|
97
|
+
if (score.source === "user-signal") {
|
|
98
|
+
feedback.signals.push(score.name);
|
|
99
|
+
if (score.name === "dismissed") {
|
|
100
|
+
negativeTags.push("user:dismissed");
|
|
101
|
+
}
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
if (score.name === "thumbs_up") {
|
|
105
|
+
feedback.thumbs = score.value === true ? "up" : "down";
|
|
106
|
+
if (score.value === false) {
|
|
107
|
+
negativeTags.push("user:thumbs-down");
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
else if (score.name === "outcome") {
|
|
111
|
+
feedback.outcome = String(score.value);
|
|
112
|
+
if (score.value === "fp") {
|
|
113
|
+
negativeTags.push("user:false-positive");
|
|
114
|
+
}
|
|
115
|
+
else if (score.value === "fn") {
|
|
116
|
+
negativeTags.push("user:false-negative");
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (score.comment && !feedback.comment) {
|
|
120
|
+
feedback.comment = score.comment;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return { feedback, negativeTags };
|
|
124
|
+
};
|
|
82
125
|
class LocalDatasetStore {
|
|
83
126
|
promptStore;
|
|
84
127
|
constructor(promptStore = new promptStore_1.LocalPromptStore()) {
|
|
@@ -280,14 +323,17 @@ class LocalDatasetStore {
|
|
|
280
323
|
}
|
|
281
324
|
const input = trace.input ?? {};
|
|
282
325
|
await this.validateItemInput(dataset, input, 1);
|
|
326
|
+
const { feedback, negativeTags } = await readUserFeedback(trace._id);
|
|
283
327
|
const item = await (0, obsDatasetItem_1.registerObsDatasetItem)().create({
|
|
284
328
|
datasetId: dataset._id,
|
|
285
329
|
expectedOutput: trace.output,
|
|
286
330
|
input,
|
|
287
331
|
origin: "trace",
|
|
288
|
-
|
|
332
|
+
// A user-rejected output is not a gold label: it waits for a human to relabel it.
|
|
333
|
+
proofread: !trace.sensitive && negativeTags.length === 0,
|
|
289
334
|
sourceTraceId: trace._id,
|
|
290
|
-
tags:
|
|
335
|
+
tags: negativeTags,
|
|
336
|
+
...(feedback ? { metadata: { userFeedback: feedback } } : {}),
|
|
291
337
|
});
|
|
292
338
|
created.push(toItemView(item));
|
|
293
339
|
}
|
|
@@ -74,6 +74,12 @@ obsReviewItemSchema.index(Object.fromEntries(reviewIndexSpec));
|
|
|
74
74
|
obsReviewItemSchema.index({ batchId: 1, status: 1 });
|
|
75
75
|
// Dataset batches skip items that already wait in an open review for the same question set.
|
|
76
76
|
obsReviewItemSchema.index({ datasetItemId: 1, evaluatorId: 1 });
|
|
77
|
+
// One pending user-feedback item per trace and evaluator, so concurrent thumbs downs queue once.
|
|
78
|
+
obsReviewItemSchema.index({ evaluatorId: 1, traceId: 1 }, {
|
|
79
|
+
name: "pending_feedback_unique",
|
|
80
|
+
partialFilterExpression: { deleted: false, reason: "feedback", status: "pending" },
|
|
81
|
+
unique: true,
|
|
82
|
+
});
|
|
77
83
|
const registerObsReviewItem = () => {
|
|
78
84
|
if (mongoose_1.default.models.ObsReviewItem) {
|
|
79
85
|
return mongoose_1.default.models.ObsReviewItem;
|
|
@@ -23,7 +23,7 @@ const obsScoreSchema = new mongoose_1.default.Schema({
|
|
|
23
23
|
name: { description: "Score dimension or name", required: true, type: String },
|
|
24
24
|
source: {
|
|
25
25
|
description: "Who wrote the score",
|
|
26
|
-
enum: ["code", "human", "llm-judge", "user-feedback"],
|
|
26
|
+
enum: ["code", "human", "llm-judge", "user-feedback", "user-signal"],
|
|
27
27
|
required: true,
|
|
28
28
|
type: String,
|
|
29
29
|
},
|
|
@@ -49,6 +49,17 @@ obsScoreSchema.plugin(api_1.isDeletedPlugin);
|
|
|
49
49
|
obsScoreSchema.plugin(api_1.findOneOrNone);
|
|
50
50
|
obsScoreSchema.plugin(api_1.findExactlyOne);
|
|
51
51
|
obsScoreSchema.index({ traceId: 1 });
|
|
52
|
+
// One live end-user score per (trace, source, name); a changed vote replaces the old one.
|
|
53
|
+
obsScoreSchema.index({ name: 1, source: 1, traceId: 1 }, {
|
|
54
|
+
name: "user_feedback_unique",
|
|
55
|
+
partialFilterExpression: { deleted: false, source: "user-feedback" },
|
|
56
|
+
unique: true,
|
|
57
|
+
});
|
|
58
|
+
obsScoreSchema.index({ name: 1, source: 1, traceId: 1 }, {
|
|
59
|
+
name: "user_signal_unique",
|
|
60
|
+
partialFilterExpression: { deleted: false, source: "user-signal" },
|
|
61
|
+
unique: true,
|
|
62
|
+
});
|
|
52
63
|
const registerObsScore = () => {
|
|
53
64
|
if (mongoose_1.default.models.ObsScore) {
|
|
54
65
|
return mongoose_1.default.models.ObsScore;
|
|
@@ -207,6 +207,7 @@ class LocalPromptStore {
|
|
|
207
207
|
latestVersion: latest?.version ?? 0,
|
|
208
208
|
name: prompt.name,
|
|
209
209
|
production,
|
|
210
|
+
tags: prompt.tags ?? [],
|
|
210
211
|
type: latest?.type ?? "text",
|
|
211
212
|
...(usageByName ? { usage7d: usageByName.get(prompt.name) ?? { calls: 0 } } : {}),
|
|
212
213
|
});
|
|
@@ -25,6 +25,11 @@ export declare class LocalReviewStore {
|
|
|
25
25
|
spanId?: string;
|
|
26
26
|
traceIds: string[];
|
|
27
27
|
}): Promise<ReviewListItem[]>;
|
|
28
|
+
/** The pending or in-progress item for a trace and evaluator, if one exists. */
|
|
29
|
+
findOpen(params: {
|
|
30
|
+
evaluatorId: string;
|
|
31
|
+
traceId: string;
|
|
32
|
+
}): Promise<ReviewListItem | undefined>;
|
|
28
33
|
/** Queue dataset items for a human question set; items with a source trace show that trace. */
|
|
29
34
|
enqueueDatasetItems(params: {
|
|
30
35
|
batchId: string;
|
|
@@ -66,6 +66,23 @@ class LocalReviewStore {
|
|
|
66
66
|
}
|
|
67
67
|
return created;
|
|
68
68
|
}
|
|
69
|
+
/** The pending or in-progress item for a trace and evaluator, if one exists. */
|
|
70
|
+
async findOpen(params) {
|
|
71
|
+
// Oldest open item: an operator may already have queued the same trace more than once.
|
|
72
|
+
const [item] = await (0, obsReviewItem_1.registerObsReviewItem)()
|
|
73
|
+
.find({
|
|
74
|
+
evaluatorId: objectId(params.evaluatorId, "evaluatorId"),
|
|
75
|
+
status: { $in: ["pending", "in_progress"] },
|
|
76
|
+
traceId: objectId(params.traceId, "traceId"),
|
|
77
|
+
})
|
|
78
|
+
.sort({ enqueuedAt: 1 })
|
|
79
|
+
.limit(1);
|
|
80
|
+
if (!item) {
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
const trace = await (0, obsTrace_1.registerObsTrace)().findOneOrNone({ _id: item.traceId });
|
|
84
|
+
return this.toListItem(item, trace?.name ?? "", trace?.prompts[0]?.name);
|
|
85
|
+
}
|
|
69
86
|
/** Queue dataset items for a human question set; items with a source trace show that trace. */
|
|
70
87
|
async enqueueDatasetItems(params) {
|
|
71
88
|
const created = [];
|
|
@@ -23,6 +23,12 @@ export interface TraceListQuery {
|
|
|
23
23
|
/** One of `TRACE_LIST_SORT_KEYS`, optionally prefixed with `-` for descending. */
|
|
24
24
|
sort?: string;
|
|
25
25
|
status?: "error" | "ok";
|
|
26
|
+
/** Traces with a live end-user `thumbs_up` score in that direction. */
|
|
27
|
+
thumbs?: "down" | "up";
|
|
28
|
+
/** Traces with a live `user-signal` score of this name. */
|
|
29
|
+
signal?: string;
|
|
30
|
+
/** Traces with (or without) a live `user-feedback` score carrying a non-empty comment. */
|
|
31
|
+
hasComment?: boolean;
|
|
26
32
|
to?: string;
|
|
27
33
|
userId?: string;
|
|
28
34
|
}
|
|
@@ -82,11 +88,28 @@ export interface TraceListItem {
|
|
|
82
88
|
userId?: string;
|
|
83
89
|
}
|
|
84
90
|
export declare const parseTraceListSort: (value?: string) => TraceListSort;
|
|
91
|
+
/**
|
|
92
|
+
* Forgets the index build so a test can simulate a failed build.
|
|
93
|
+
* @internal Test seam. Production builds the indexes once per process.
|
|
94
|
+
*/
|
|
95
|
+
export declare const resetObsScoreIndexBuild: () => void;
|
|
85
96
|
export declare class LocalTraceStore {
|
|
86
97
|
exportTrace(trace: TraceRecord): Promise<{
|
|
87
98
|
id: string;
|
|
88
99
|
}>;
|
|
89
100
|
exportScore(score: ScoreRecord): Promise<void>;
|
|
101
|
+
/**
|
|
102
|
+
* Keeps one live score per (trace, source, name) for end-user sources, so a changed vote
|
|
103
|
+
* replaces the old one. One atomic upsert backed by a partial unique index, so concurrent
|
|
104
|
+
* taps cannot leave two live scores.
|
|
105
|
+
*/
|
|
106
|
+
private upsertUserScore;
|
|
107
|
+
/** Soft-deletes the live end-user score for (trace, source, name), e.g. a cleared thumbs vote. */
|
|
108
|
+
deleteFeedbackScore(params: {
|
|
109
|
+
name: string;
|
|
110
|
+
source: ScoreRecord["source"];
|
|
111
|
+
traceId: string;
|
|
112
|
+
}): Promise<void>;
|
|
90
113
|
list(query: TraceListQuery): Promise<{
|
|
91
114
|
data: TraceListItem[];
|
|
92
115
|
meta: {
|