@terreno/ai 58.0.1 → 58.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/observability/judgeAgreement.js +3 -1
- package/dist/observability/local/datasetStore.js +7 -1
- package/dist/observability/local/promptReferences.js +2 -1
- package/dist/observability/local/reviewBatchStore.js +8 -2
- package/dist/observability/local/reviewStore.js +9 -1
- package/dist/observability/local/traceStore.js +37 -9
- package/dist/observability/schemaValidation.js +2 -1
- package/dist/routes/aiRequestsExplorer.js +9 -3
- package/dist/service/parseAiJson.js +5 -3
- package/package.json +11 -11
- package/src/harness/extensions.test.ts +8 -0
- package/src/observability/judgeAgreement.test.ts +10 -0
- package/src/observability/judgeAgreement.ts +4 -1
- package/src/observability/local/datasetStore.test.ts +9 -1
- package/src/observability/local/datasetStore.ts +7 -1
- package/src/observability/local/promptReferences.test.ts +12 -0
- package/src/observability/local/promptReferences.ts +2 -1
- package/src/observability/local/reviewBatchStore.test.ts +3 -0
- package/src/observability/local/reviewBatchStore.ts +7 -2
- package/src/observability/local/reviewStore.test.ts +7 -1
- package/src/observability/local/reviewStore.ts +10 -1
- package/src/observability/local/traceStore.test.ts +10 -0
- package/src/observability/local/traceStore.ts +41 -10
- package/src/observability/schemaValidation.test.ts +29 -0
- package/src/observability/schemaValidation.ts +2 -1
- package/src/routes/aiRequestsExplorer.ts +8 -3
- package/src/routes/routes.test.ts +7 -0
- package/src/service/parseAiJson.test.ts +17 -0
- package/src/service/parseAiJson.ts +6 -5
package/CHANGELOG.md
CHANGED
|
@@ -20,6 +20,37 @@ Upgrade notes for consumer action live in [`mcp-server/src/docs/upgrades/`](mcp-
|
|
|
20
20
|
|
|
21
21
|
Unreleased changes live in [`changelog/unreleased/`](changelog/unreleased/). Add one Markdown file per feature (see that directory's README) instead of editing this section.
|
|
22
22
|
|
|
23
|
+
## [58.0.3] - 2026-11-11
|
|
24
|
+
|
|
25
|
+
Upgrade note: [`mcp-server/src/docs/upgrades/58.0.3.md`](mcp-server/src/docs/upgrades/58.0.3.md).
|
|
26
|
+
|
|
27
|
+
### Added
|
|
28
|
+
|
|
29
|
+
- `fix-code-scanning-alerts` skill: clears every open CodeQL alert on one PR. Fixes close on the master rescan; reviewed false positives go in `.github/code-scanning/dismissals.json` and a workflow dismisses them after merge. CodeQL now also scans pull requests and ignores test files. See `docs/how-to/fix-code-scanning-alerts.md`.
|
|
30
|
+
- **@terreno/syncdb:** `registerTerrenoDevClientDispatch`, `registerTerrenoDevStorage`, and `registerTerrenoDevCommand` for `terreno-mcp-local` dev tooling.
|
|
31
|
+
- **@terreno/mcp-server:** `read_logs` sources `network` and `native`; `dispatch_client_action`, storage read/write, and dev command list/invoke; CDP network capture with header redaction.
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
|
|
35
|
+
- Package tests run as concurrent `bun test --shard` processes through `scripts/test-shards.ts` (half the cores by default, `TERRENO_TEST_SHARDS` to override), and `test:coverage` merges the shards' LCOV. CircleCI now pins Bun 1.4.4, the same version as the local toolchain.
|
|
36
|
+
|
|
37
|
+
### Fixed
|
|
38
|
+
|
|
39
|
+
- `Avatar` no longer freezes release builds when its image fails to load. The image error handler passed the native error event to `console.warn`, and with Sentry enabled the app hung while turning that event into a breadcrumb. It now logs only the error message.
|
|
40
|
+
- Mongo filters built from request values reject non-string input with 400 instead of running it as a query operator: AI request explorer `model`, observability review `status`, review batch `promptName`, experiment `datasetId`, trace list filters, push-token `platform`, and `loadDocOr404` ids (404). Sync conflict reloads and tombstone lookups match `_id` with `$eq`.
|
|
41
|
+
- Nested-path helpers in `populate`, RBAC field views, and MCP `excludeFields` skip `__proto__`, `constructor`, and `prototype`, so a crafted path can no longer touch `Object.prototype`.
|
|
42
|
+
- Removed polynomial-time regexes from bearer-header parsing, trailing-slash trimming (syncdb, ui `ListDetailLayout`, jobs Cloud Tasks runner, admin route paths), prompt references, judge score ranges, AI JSON fence stripping, and the MCP import parser.
|
|
43
|
+
- Dataset schema validation stops at the first error (`allErrors` off) to bound work on untrusted input.
|
|
44
|
+
- `isTestUser` in `@terreno/ui` matches `nang.io` and `example.com` as the email domain or a subdomain, not as a substring, and always returns a boolean.
|
|
45
|
+
- MCP OAuth metadata responses set an explicit JSON content type.
|
|
46
|
+
- The example app's AI session id comes from `expo-crypto` `randomUUID` instead of `Math.random`.
|
|
47
|
+
|
|
48
|
+
## [58.0.2] - 2026-11-09
|
|
49
|
+
|
|
50
|
+
### Fixed
|
|
51
|
+
|
|
52
|
+
- Patch the remaining critical and high security advisories in Terreno's dependency tree. `@terreno/ui` now asks for `@react-navigation/native` `^7.5.1`, whose `@react-navigation/core` no longer pulls in the vulnerable `decode-uri-component` (#1693). The docs site moves to tinypool 2.2.0 (critical) and serialize-javascript 7.1.4 (#1690, #1691), Metro's stale nested copy no longer brings in image-size 1.x (#1694), and rulesync 43.8.0 drops its pinned zod 4.6.5 (#1695). Dependabot now opens one security PR per advisory instead of one grouped PR per package block (#1689). Expo tooling still carries zod 3.25.76, which has no fixed 3.x release.
|
|
53
|
+
|
|
23
54
|
## [58.0.1] - 2026-11-09
|
|
24
55
|
|
|
25
56
|
### Added
|
|
@@ -4,7 +4,9 @@ exports.computeJudgeAgreement = exports.agreementAnswerKey = exports.answersAgre
|
|
|
4
4
|
// Numeric answers agree when they land within this share of the dimension's range.
|
|
5
5
|
const NUMERIC_TOLERANCE = 0.1;
|
|
6
6
|
const numericSpan = (range) => {
|
|
7
|
-
const matches = range?.match(
|
|
7
|
+
const matches = range?.match(
|
|
8
|
+
// Bounded quantifiers keep the unanchored scan linear on long digit runs.
|
|
9
|
+
/(-?\d{1,15}(?:\.\d{1,15})?)\s{0,10}(?:-|\.\.)\s{0,10}(-?\d{1,15}(?:\.\d{1,15})?)/);
|
|
8
10
|
if (!matches) {
|
|
9
11
|
return 1;
|
|
10
12
|
}
|
|
@@ -294,7 +294,13 @@ class LocalDatasetStore {
|
|
|
294
294
|
return created;
|
|
295
295
|
}
|
|
296
296
|
async listExperimentItems(datasetId, includeUnproofread) {
|
|
297
|
-
|
|
297
|
+
// Estimate passes the request body through, so reject operator objects before querying.
|
|
298
|
+
if (typeof datasetId !== "string" || !mongoose_1.default.Types.ObjectId.isValid(datasetId)) {
|
|
299
|
+
throw new api_1.APIError({ status: 400, title: "datasetId must be an object id" });
|
|
300
|
+
}
|
|
301
|
+
const filter = {
|
|
302
|
+
datasetId: new mongoose_1.default.Types.ObjectId(datasetId),
|
|
303
|
+
};
|
|
298
304
|
if (!includeUnproofread) {
|
|
299
305
|
filter.proofread = true;
|
|
300
306
|
}
|
|
@@ -6,7 +6,8 @@ const api_1 = require("@terreno/api");
|
|
|
6
6
|
* A reference to another registry prompt, written like a template partial:
|
|
7
7
|
* `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}`.
|
|
8
8
|
*/
|
|
9
|
-
|
|
9
|
+
// Excluding `{` from the body keeps the scan linear: a match attempt stops at the next tag start.
|
|
10
|
+
const PROMPT_REFERENCE = /\{\{>([^{}]*)\}\}/g;
|
|
10
11
|
const MAX_PROMPT_REFERENCE_DEPTH = 5;
|
|
11
12
|
const DEFAULT_PROMPT_REFERENCE_LABEL = "production";
|
|
12
13
|
/** Identity of one recorded include, for deduping. */
|
|
@@ -503,8 +503,14 @@ class LocalReviewBatchStore {
|
|
|
503
503
|
output: { $exists: true, $ne: null },
|
|
504
504
|
status: "ok",
|
|
505
505
|
};
|
|
506
|
-
|
|
507
|
-
|
|
506
|
+
const promptName = params.promptName;
|
|
507
|
+
if (typeof promptName === "string") {
|
|
508
|
+
if (promptName) {
|
|
509
|
+
filter["prompts.name"] = { $eq: promptName };
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
else if (promptName !== undefined) {
|
|
513
|
+
throw new api_1.APIError({ status: 400, title: "promptName must be a string" });
|
|
508
514
|
}
|
|
509
515
|
const traces = await (0, obsTrace_1.registerObsTrace)()
|
|
510
516
|
.find(filter)
|
|
@@ -19,6 +19,7 @@ const obsReviewBatch_1 = require("./models/obsReviewBatch");
|
|
|
19
19
|
const obsReviewItem_1 = require("./models/obsReviewItem");
|
|
20
20
|
const obsTrace_1 = require("./models/obsTrace");
|
|
21
21
|
const traceStore_1 = require("./traceStore");
|
|
22
|
+
const REVIEW_STATUSES = ["done", "in_progress", "pending", "skipped"];
|
|
22
23
|
const DATASET_ITEM_NAME = "Dataset item";
|
|
23
24
|
const objectId = (value, label) => {
|
|
24
25
|
if (!mongoose_1.default.Types.ObjectId.isValid(value)) {
|
|
@@ -83,6 +84,13 @@ class LocalReviewStore {
|
|
|
83
84
|
return created;
|
|
84
85
|
}
|
|
85
86
|
async list(status, options = {}) {
|
|
87
|
+
// The route forwards the raw query value, so only the four known statuses reach the filter.
|
|
88
|
+
if (status !== undefined && !REVIEW_STATUSES.includes(status)) {
|
|
89
|
+
throw new api_1.APIError({
|
|
90
|
+
status: 400,
|
|
91
|
+
title: "status must be done, in_progress, pending, or skipped",
|
|
92
|
+
});
|
|
93
|
+
}
|
|
86
94
|
const ObsReviewItem = (0, obsReviewItem_1.registerObsReviewItem)();
|
|
87
95
|
const scope = options.batchId ? { batchId: objectId(options.batchId, "batchId") } : {};
|
|
88
96
|
const counts = {
|
|
@@ -91,7 +99,7 @@ class LocalReviewStore {
|
|
|
91
99
|
pending: await ObsReviewItem.countDocuments({ ...scope, status: "pending" }),
|
|
92
100
|
skipped: await ObsReviewItem.countDocuments({ ...scope, status: "skipped" }),
|
|
93
101
|
};
|
|
94
|
-
const filter = status ? { ...scope, status } : scope;
|
|
102
|
+
const filter = status ? { ...scope, status: { $eq: status } } : scope;
|
|
95
103
|
// Batch items keep their batch order (enqueued together, newest trace first).
|
|
96
104
|
const rows = await ObsReviewItem.find(filter).sort({ _id: 1, enqueuedAt: 1 });
|
|
97
105
|
const data = [];
|
|
@@ -81,9 +81,29 @@ const parseTraceListSort = (value) => {
|
|
|
81
81
|
return { direction: descending ? -1 : 1, key };
|
|
82
82
|
};
|
|
83
83
|
exports.parseTraceListSort = parseTraceListSort;
|
|
84
|
+
/** Rejects non-string query values so an operator object never reaches a Mongo filter. */
|
|
85
|
+
const optionalString = (value, label) => {
|
|
86
|
+
if (value === undefined) {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
if (typeof value !== "string") {
|
|
90
|
+
throw new api_1.APIError({ status: 400, title: `${label} must be a string` });
|
|
91
|
+
}
|
|
92
|
+
return value;
|
|
93
|
+
};
|
|
84
94
|
const promptNames = (prompt) => {
|
|
85
95
|
const values = Array.isArray(prompt) ? prompt : prompt === undefined ? [] : [prompt];
|
|
86
|
-
|
|
96
|
+
const names = [];
|
|
97
|
+
for (const entry of values) {
|
|
98
|
+
if (typeof entry !== "string") {
|
|
99
|
+
throw new api_1.APIError({ status: 400, title: "prompt must be a string or a list of strings" });
|
|
100
|
+
}
|
|
101
|
+
const trimmed = entry.trim();
|
|
102
|
+
if (trimmed !== "") {
|
|
103
|
+
names.push(trimmed);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return names;
|
|
87
107
|
};
|
|
88
108
|
/** Count rows in `from` whose traceId matches the trace, for span and score sorts. */
|
|
89
109
|
const countLookupStages = (from) => {
|
|
@@ -246,17 +266,25 @@ class LocalTraceStore {
|
|
|
246
266
|
const page = query.page && query.page > 0 ? query.page : 1;
|
|
247
267
|
const limit = query.limit && query.limit > 0 ? Math.min(query.limit, 100) : 20;
|
|
248
268
|
const filter = {};
|
|
249
|
-
|
|
250
|
-
|
|
269
|
+
// LocalTraceStore is exported, so library callers get the same guards the route applies.
|
|
270
|
+
const status = optionalString(query.status, "status");
|
|
271
|
+
if (status) {
|
|
272
|
+
filter.status = status;
|
|
251
273
|
}
|
|
252
|
-
|
|
253
|
-
|
|
274
|
+
const sessionId = optionalString(query.sessionId, "sessionId");
|
|
275
|
+
if (sessionId) {
|
|
276
|
+
filter.sessionId = sessionId;
|
|
254
277
|
}
|
|
255
|
-
|
|
256
|
-
|
|
278
|
+
const scope = optionalString(query.scope, "scope");
|
|
279
|
+
if (scope) {
|
|
280
|
+
filter.scope = scope;
|
|
257
281
|
}
|
|
258
282
|
const sort = (0, exports.parseTraceListSort)(query.sort);
|
|
259
283
|
const prompts = promptNames(query.prompt);
|
|
284
|
+
if (query.promptVersion !== undefined &&
|
|
285
|
+
(typeof query.promptVersion !== "number" || !Number.isInteger(query.promptVersion))) {
|
|
286
|
+
throw new api_1.APIError({ status: 400, title: "promptVersion must be an integer" });
|
|
287
|
+
}
|
|
260
288
|
if (query.promptVersion !== undefined && prompts.length === 0) {
|
|
261
289
|
throw new api_1.APIError({ status: 400, title: "prompt is required when promptVersion is set" });
|
|
262
290
|
}
|
|
@@ -275,10 +303,10 @@ class LocalTraceStore {
|
|
|
275
303
|
filter["prompts.name"] = { $in: prompts };
|
|
276
304
|
}
|
|
277
305
|
if (query.sensitive !== undefined) {
|
|
278
|
-
filter.sensitive = query.sensitive;
|
|
306
|
+
filter.sensitive = query.sensitive === true;
|
|
279
307
|
}
|
|
280
308
|
if (query.flaggedForDataset !== undefined) {
|
|
281
|
-
filter.flaggedForDataset = query.flaggedForDataset;
|
|
309
|
+
filter.flaggedForDataset = query.flaggedForDataset === true;
|
|
282
310
|
}
|
|
283
311
|
const userId = objectIdOrUndefined(query.userId);
|
|
284
312
|
if (query.userId) {
|
|
@@ -5,7 +5,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.validateAgainstSchema = void 0;
|
|
7
7
|
const ajv_1 = __importDefault(require("ajv"));
|
|
8
|
-
|
|
8
|
+
// Stop at the first error: allErrors walks every node of untrusted input (CodeQL js/resource-exhaustion).
|
|
9
|
+
const ajv = new ajv_1.default({ strict: false });
|
|
9
10
|
const validateAgainstSchema = (schema, value) => {
|
|
10
11
|
if (!schema) {
|
|
11
12
|
return [];
|
|
@@ -49,13 +49,19 @@ const addAiRequestsExplorerRoutes = (router, options) => {
|
|
|
49
49
|
};
|
|
50
50
|
const requestTypes = parseRequestTypes(req.query.requestType);
|
|
51
51
|
if (requestTypes.length === 1) {
|
|
52
|
-
match.requestType = requestTypes[0];
|
|
52
|
+
match.requestType = { $eq: requestTypes[0] };
|
|
53
53
|
}
|
|
54
54
|
else if (requestTypes.length > 1) {
|
|
55
55
|
match.requestType = { $in: requestTypes };
|
|
56
56
|
}
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
const model = req.query.model;
|
|
58
|
+
if (typeof model === "string") {
|
|
59
|
+
if (model) {
|
|
60
|
+
match.aiModel = { $eq: model };
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
else if (model !== undefined) {
|
|
64
|
+
throw new api_1.APIError({ status: 400, title: "model must be a string" });
|
|
59
65
|
}
|
|
60
66
|
if (req.query.startDate || req.query.endDate) {
|
|
61
67
|
match.created = {};
|
|
@@ -13,15 +13,17 @@
|
|
|
13
13
|
*/
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.normalizeLlmJsonTextForStructuredOutput = exports.parseAiJson = void 0;
|
|
16
|
+
const FENCE_LANGUAGE = /^(?:json|javascript|js|typescript|ts)/i;
|
|
16
17
|
/**
|
|
17
18
|
* Strip leading/trailing markdown code fences when the whole payload is fenced,
|
|
18
19
|
* otherwise strip a common opening ```lang line and trailing ``` block.
|
|
19
20
|
*/
|
|
20
21
|
const stripFences = (input) => {
|
|
21
22
|
const trimmed = input.trim();
|
|
22
|
-
|
|
23
|
-
if (
|
|
24
|
-
|
|
23
|
+
// String ops instead of one fenced regex: `\s*` and a lazy body made that regex quadratic.
|
|
24
|
+
if (trimmed.length >= 6 && trimmed.startsWith("```") && trimmed.endsWith("```")) {
|
|
25
|
+
const inner = trimmed.slice(3, -3);
|
|
26
|
+
return inner.replace(FENCE_LANGUAGE, "").trim();
|
|
25
27
|
}
|
|
26
28
|
let t = trimmed;
|
|
27
29
|
t = t.replace(/^```[a-zA-Z0-9_-]*\s*:?\s*\n?/, "");
|
package/package.json
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
"dependencies": {
|
|
7
7
|
"@ai-sdk/mcp": "^2.0.57",
|
|
8
8
|
"@google-cloud/storage": "^7.15.0",
|
|
9
|
-
"@terreno/blocks": "58.0.
|
|
9
|
+
"@terreno/blocks": "58.0.3",
|
|
10
10
|
"ai": "^6.0.97",
|
|
11
11
|
"ajv": "8.20.0",
|
|
12
12
|
"csv-parse": "^7.0.3",
|
|
@@ -18,10 +18,10 @@
|
|
|
18
18
|
"description": "Provider-agnostic AI service with streaming chat, request logging, and admin tools — part of Terreno, the batteries-included TypeScript framework for universal apps.",
|
|
19
19
|
"devDependencies": {
|
|
20
20
|
"@biomejs/biome": "2.5.13",
|
|
21
|
-
"@terreno/admin-backend": "58.0.
|
|
22
|
-
"@terreno/api": "58.0.
|
|
23
|
-
"@terreno/jobs": "58.0.
|
|
24
|
-
"@terreno/test": "58.0.
|
|
21
|
+
"@terreno/admin-backend": "58.0.3",
|
|
22
|
+
"@terreno/api": "58.0.3",
|
|
23
|
+
"@terreno/jobs": "58.0.3",
|
|
24
|
+
"@terreno/test": "58.0.3",
|
|
25
25
|
"@types/bun": "^1.4.0",
|
|
26
26
|
"@types/express": "^5.0.6",
|
|
27
27
|
"@types/luxon": "^3.7.1",
|
|
@@ -121,8 +121,8 @@
|
|
|
121
121
|
"peerDependencies": {
|
|
122
122
|
"@ai-sdk/google-vertex": "^5.0.92",
|
|
123
123
|
"@ai-sdk/provider": "^4.0.26",
|
|
124
|
-
"@terreno/api": "58.0.
|
|
125
|
-
"@terreno/jobs": "58.0.
|
|
124
|
+
"@terreno/api": "58.0.3",
|
|
125
|
+
"@terreno/jobs": "58.0.3",
|
|
126
126
|
"mongoose": "^8.0.0 || ^9.0.0"
|
|
127
127
|
},
|
|
128
128
|
"peerDependenciesMeta": {
|
|
@@ -148,12 +148,12 @@
|
|
|
148
148
|
"lint:fix": "biome check --write ./src ./e2e",
|
|
149
149
|
"lint:unsafefix": "biome check --fix --unsafe ./src ./e2e",
|
|
150
150
|
"smoke": "bun e2e/smoke.ts",
|
|
151
|
-
"test": "bun test
|
|
152
|
-
"test:agent": "AGENT=1 bun test
|
|
153
|
-
"test:ci": "bun test
|
|
151
|
+
"test": "bun ../scripts/test-shards.ts --isolated -- --preload ./src/tests/bunSetup.ts",
|
|
152
|
+
"test:agent": "AGENT=1 bun ../scripts/test-shards.ts --isolated -- --only-failures --preload ./src/tests/bunSetup.ts",
|
|
153
|
+
"test:ci": "bun ../scripts/test-shards.ts --isolated -- --preload ./src/tests/bunSetup.ts",
|
|
154
154
|
"test:coverage": "bun run ../scripts/check-coverage.ts",
|
|
155
155
|
"typecheck": "tsc -p tsconfig.typecheck.json"
|
|
156
156
|
},
|
|
157
157
|
"types": "dist/index.d.ts",
|
|
158
|
-
"version": "58.0.
|
|
158
|
+
"version": "58.0.3"
|
|
159
159
|
}
|
|
@@ -994,6 +994,12 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
994
994
|
it("rejects blank keys, non-JSON values, and writes after the phase committed", async () => {
|
|
995
995
|
const errors: string[] = [];
|
|
996
996
|
const lateRead: unknown[] = [];
|
|
997
|
+
// The task reads as completed from the commit on, before the phase function returns.
|
|
998
|
+
// Wait for the function itself, or the post-commit calls race the assertions below.
|
|
999
|
+
let markPhaseReturned = (): void => {};
|
|
1000
|
+
const phaseReturned = new Promise<void>((resolve) => {
|
|
1001
|
+
markPhaseReturned = resolve;
|
|
1002
|
+
});
|
|
997
1003
|
const capture = async (work: () => Promise<unknown>): Promise<void> => {
|
|
998
1004
|
try {
|
|
999
1005
|
await work();
|
|
@@ -1013,6 +1019,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1013
1019
|
await rt.commit({terminal: {status: "completed"}});
|
|
1014
1020
|
await capture(() => rt.memo("late", 1));
|
|
1015
1021
|
lateRead.push(await rt.memo("late"));
|
|
1022
|
+
markPhaseReturned();
|
|
1016
1023
|
},
|
|
1017
1024
|
},
|
|
1018
1025
|
},
|
|
@@ -1021,6 +1028,7 @@ describe("Extensions, hooks, wraps, and memos", () => {
|
|
|
1021
1028
|
const harness = await openHarness({registry: [task]});
|
|
1022
1029
|
const created = await harness.createTask(task, {});
|
|
1023
1030
|
await harness.waitForTask(created._id, {timeout: {seconds: 10}});
|
|
1031
|
+
await phaseReturned;
|
|
1024
1032
|
expect(errors).toEqual([
|
|
1025
1033
|
"memo requires a non-empty key",
|
|
1026
1034
|
'memo "big": value is not JSON-serializable (JSON.stringify cannot serialize BigInt.)',
|
|
@@ -24,6 +24,16 @@ describe("answersAgree", () => {
|
|
|
24
24
|
expect(answersAgree({...tone, range: undefined}, 0.5, 0.55)).toBe(true);
|
|
25
25
|
});
|
|
26
26
|
|
|
27
|
+
it("reads a numeric range in linear time on a long digit run", () => {
|
|
28
|
+
const tone = QUESTIONS[2] as EvaluatorDimension;
|
|
29
|
+
const started = performance.now();
|
|
30
|
+
// No separator, so the span falls back to 1.
|
|
31
|
+
expect(answersAgree({...tone, range: "0".repeat(100_000)}, 0.5, 0.55)).toBe(true);
|
|
32
|
+
expect(performance.now() - started).toBeLessThan(500);
|
|
33
|
+
expect(answersAgree({...tone, range: "1 .. 5"}, 2, 2.4)).toBe(true);
|
|
34
|
+
expect(answersAgree({...tone, range: "1 .. 5"}, 2, 2.5)).toBe(false);
|
|
35
|
+
});
|
|
36
|
+
|
|
27
37
|
it("matches categorical answers case-insensitively", () => {
|
|
28
38
|
const dimension: EvaluatorDimension = {dataType: "categorical", key: "c", required: true};
|
|
29
39
|
expect(answersAgree(dimension, "Warm", " warm ")).toBe(true);
|
|
@@ -48,7 +48,10 @@ export interface JudgeAgreement {
|
|
|
48
48
|
const NUMERIC_TOLERANCE = 0.1;
|
|
49
49
|
|
|
50
50
|
const numericSpan = (range?: string): number => {
|
|
51
|
-
const matches = range?.match(
|
|
51
|
+
const matches = range?.match(
|
|
52
|
+
// Bounded quantifiers keep the unanchored scan linear on long digit runs.
|
|
53
|
+
/(-?\d{1,15}(?:\.\d{1,15})?)\s{0,10}(?:-|\.\.)\s{0,10}(-?\d{1,15}(?:\.\d{1,15})?)/
|
|
54
|
+
);
|
|
52
55
|
if (!matches) {
|
|
53
56
|
return 1;
|
|
54
57
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {afterEach, beforeEach, describe, it} from "bun:test";
|
|
1
|
+
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
2
|
import {assert} from "chai";
|
|
3
3
|
import {DateTime} from "luxon";
|
|
4
4
|
|
|
@@ -227,4 +227,12 @@ describe("LocalDatasetStore", () => {
|
|
|
227
227
|
});
|
|
228
228
|
assert.equal(single.sourceTraceId, String(trace._id));
|
|
229
229
|
});
|
|
230
|
+
|
|
231
|
+
it("rejects an operator object as the experiment dataset id", async () => {
|
|
232
|
+
const operator = {$ne: null} as unknown as string;
|
|
233
|
+
await expect(store.listExperimentItems(operator, true)).rejects.toMatchObject({status: 400});
|
|
234
|
+
await expect(store.listExperimentItems("not-an-id", true)).rejects.toMatchObject({
|
|
235
|
+
status: 400,
|
|
236
|
+
});
|
|
237
|
+
});
|
|
230
238
|
});
|
|
@@ -426,7 +426,13 @@ export class LocalDatasetStore {
|
|
|
426
426
|
datasetId: string,
|
|
427
427
|
includeUnproofread: boolean
|
|
428
428
|
): Promise<DatasetItemView[]> {
|
|
429
|
-
|
|
429
|
+
// Estimate passes the request body through, so reject operator objects before querying.
|
|
430
|
+
if (typeof datasetId !== "string" || !mongoose.Types.ObjectId.isValid(datasetId)) {
|
|
431
|
+
throw new APIError({status: 400, title: "datasetId must be an object id"});
|
|
432
|
+
}
|
|
433
|
+
const filter: Record<string, unknown> = {
|
|
434
|
+
datasetId: new mongoose.Types.ObjectId(datasetId),
|
|
435
|
+
};
|
|
430
436
|
if (!includeUnproofread) {
|
|
431
437
|
filter.proofread = true;
|
|
432
438
|
}
|
|
@@ -125,4 +125,16 @@ describe("resolvePromptReferences", () => {
|
|
|
125
125
|
resolvePromptReferences({ancestors: ["p0"], lookup: lookupFrom(bodies), text: tag("p1")})
|
|
126
126
|
).rejects.toThrow("Prompt references nested too deeply");
|
|
127
127
|
});
|
|
128
|
+
|
|
129
|
+
it("scans a long run of unclosed tag starts in linear time", async () => {
|
|
130
|
+
const text = "{{{{>|".repeat(50_000);
|
|
131
|
+
const started = performance.now();
|
|
132
|
+
const resolved = await resolvePromptReferences({
|
|
133
|
+
ancestors: ["caller"],
|
|
134
|
+
lookup: lookupFrom({}),
|
|
135
|
+
text,
|
|
136
|
+
});
|
|
137
|
+
expect(performance.now() - started).toBeLessThan(500);
|
|
138
|
+
expect(resolved).toEqual({includes: [], text});
|
|
139
|
+
});
|
|
128
140
|
});
|
|
@@ -6,7 +6,8 @@ import type {ObsPromptRef} from "../../types/observability";
|
|
|
6
6
|
* A reference to another registry prompt, written like a template partial:
|
|
7
7
|
* `{{> base-rules}}`, `{{> base-rules label=staging}}`, or `{{> base-rules version=3}}`.
|
|
8
8
|
*/
|
|
9
|
-
|
|
9
|
+
// Excluding `{` from the body keeps the scan linear: a match attempt stops at the next tag start.
|
|
10
|
+
const PROMPT_REFERENCE = /\{\{>([^{}]*)\}\}/g;
|
|
10
11
|
|
|
11
12
|
const MAX_PROMPT_REFERENCE_DEPTH = 5;
|
|
12
13
|
|
|
@@ -140,6 +140,9 @@ describe("LocalReviewBatchStore", () => {
|
|
|
140
140
|
await expect(
|
|
141
141
|
store.create({evaluatorId: questionSetId, promptName: "summarize"})
|
|
142
142
|
).rejects.toThrow(/No recent "summarize" traces/);
|
|
143
|
+
await expect(
|
|
144
|
+
store.create({evaluatorId: questionSetId, promptName: {$ne: null} as unknown as string})
|
|
145
|
+
).rejects.toMatchObject({status: 400, title: "promptName must be a string"});
|
|
143
146
|
|
|
144
147
|
const listed = await store.list();
|
|
145
148
|
expect(listed.map((row) => [row.name, row.counts.total, row.judgeCount])).toEqual([
|
|
@@ -643,8 +643,13 @@ export class LocalReviewBatchStore {
|
|
|
643
643
|
output: {$exists: true, $ne: null},
|
|
644
644
|
status: "ok",
|
|
645
645
|
};
|
|
646
|
-
|
|
647
|
-
|
|
646
|
+
const promptName: unknown = params.promptName;
|
|
647
|
+
if (typeof promptName === "string") {
|
|
648
|
+
if (promptName) {
|
|
649
|
+
filter["prompts.name"] = {$eq: promptName};
|
|
650
|
+
}
|
|
651
|
+
} else if (promptName !== undefined) {
|
|
652
|
+
throw new APIError({status: 400, title: "promptName must be a string"});
|
|
648
653
|
}
|
|
649
654
|
const traces = await registerObsTrace()
|
|
650
655
|
.find(filter)
|
|
@@ -8,7 +8,7 @@ import {createLocalObservabilityPlugin} from "./localPlugin";
|
|
|
8
8
|
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
9
9
|
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
10
10
|
import {registerObsTrace} from "./models/obsTrace";
|
|
11
|
-
import {LocalReviewStore} from "./reviewStore";
|
|
11
|
+
import {LocalReviewStore, type ReviewStatus} from "./reviewStore";
|
|
12
12
|
import {LocalTraceStore, MemoryScoreSink} from "./traceStore";
|
|
13
13
|
|
|
14
14
|
describe("LocalReviewStore", () => {
|
|
@@ -161,4 +161,10 @@ describe("LocalReviewStore", () => {
|
|
|
161
161
|
assert.lengthOf(trace.scores, 1);
|
|
162
162
|
assert.equal((await reviewStore.getDetail(item.id)).status, "done");
|
|
163
163
|
});
|
|
164
|
+
|
|
165
|
+
it("rejects a status that is not one of the review statuses", async () => {
|
|
166
|
+
const operator = {$ne: "done"} as unknown as ReviewStatus;
|
|
167
|
+
await expect(reviewStore.list(operator)).rejects.toMatchObject({status: 400});
|
|
168
|
+
await expect(reviewStore.list("pending")).resolves.toHaveProperty("counts");
|
|
169
|
+
});
|
|
164
170
|
});
|
|
@@ -19,6 +19,8 @@ import {LocalTraceStore} from "./traceStore";
|
|
|
19
19
|
|
|
20
20
|
export type ReviewStatus = "done" | "in_progress" | "pending" | "skipped";
|
|
21
21
|
|
|
22
|
+
const REVIEW_STATUSES: readonly unknown[] = ["done", "in_progress", "pending", "skipped"];
|
|
23
|
+
|
|
22
24
|
export interface ReviewListItem {
|
|
23
25
|
assigneeId?: string;
|
|
24
26
|
batchId?: string;
|
|
@@ -120,6 +122,13 @@ export class LocalReviewStore {
|
|
|
120
122
|
counts: Record<ReviewStatus, number>;
|
|
121
123
|
data: ReviewListItem[];
|
|
122
124
|
}> {
|
|
125
|
+
// The route forwards the raw query value, so only the four known statuses reach the filter.
|
|
126
|
+
if (status !== undefined && !REVIEW_STATUSES.includes(status)) {
|
|
127
|
+
throw new APIError({
|
|
128
|
+
status: 400,
|
|
129
|
+
title: "status must be done, in_progress, pending, or skipped",
|
|
130
|
+
});
|
|
131
|
+
}
|
|
123
132
|
const ObsReviewItem = registerObsReviewItem();
|
|
124
133
|
const scope = options.batchId ? {batchId: objectId(options.batchId, "batchId")} : {};
|
|
125
134
|
const counts = {
|
|
@@ -128,7 +137,7 @@ export class LocalReviewStore {
|
|
|
128
137
|
pending: await ObsReviewItem.countDocuments({...scope, status: "pending"}),
|
|
129
138
|
skipped: await ObsReviewItem.countDocuments({...scope, status: "skipped"}),
|
|
130
139
|
};
|
|
131
|
-
const filter = status ? {...scope, status} : scope;
|
|
140
|
+
const filter = status ? {...scope, status: {$eq: status}} : scope;
|
|
132
141
|
// Batch items keep their batch order (enqueued together, newest trace first).
|
|
133
142
|
const rows = await ObsReviewItem.find(filter).sort({_id: 1, enqueuedAt: 1});
|
|
134
143
|
const data: ReviewListItem[] = [];
|
|
@@ -406,4 +406,14 @@ describe("LocalTraceStore", () => {
|
|
|
406
406
|
expect(traceSink.traces).toEqual([trace]);
|
|
407
407
|
expect(scoreSink.scores).toEqual([score]);
|
|
408
408
|
});
|
|
409
|
+
|
|
410
|
+
it("rejects operator objects in string filters", async () => {
|
|
411
|
+
const operator = {$ne: "x"} as unknown as string;
|
|
412
|
+
await expect(store.list({sessionId: operator})).rejects.toMatchObject({status: 400});
|
|
413
|
+
await expect(store.list({scope: operator})).rejects.toMatchObject({status: 400});
|
|
414
|
+
await expect(store.list({status: {$ne: "ok"} as unknown as "ok"})).rejects.toMatchObject({
|
|
415
|
+
status: 400,
|
|
416
|
+
});
|
|
417
|
+
await expect(store.list({prompt: [operator]})).rejects.toMatchObject({status: 400});
|
|
418
|
+
});
|
|
409
419
|
});
|
|
@@ -172,9 +172,30 @@ export const parseTraceListSort = (value?: string): TraceListSort => {
|
|
|
172
172
|
return {direction: descending ? -1 : 1, key};
|
|
173
173
|
};
|
|
174
174
|
|
|
175
|
+
/** Rejects non-string query values so an operator object never reaches a Mongo filter. */
|
|
176
|
+
const optionalString = (value: unknown, label: string): string | undefined => {
|
|
177
|
+
if (value === undefined) {
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
if (typeof value !== "string") {
|
|
181
|
+
throw new APIError({status: 400, title: `${label} must be a string`});
|
|
182
|
+
}
|
|
183
|
+
return value;
|
|
184
|
+
};
|
|
185
|
+
|
|
175
186
|
const promptNames = (prompt?: string | string[]): string[] => {
|
|
176
|
-
const values = Array.isArray(prompt) ? prompt : prompt === undefined ? [] : [prompt];
|
|
177
|
-
|
|
187
|
+
const values: unknown[] = Array.isArray(prompt) ? prompt : prompt === undefined ? [] : [prompt];
|
|
188
|
+
const names: string[] = [];
|
|
189
|
+
for (const entry of values) {
|
|
190
|
+
if (typeof entry !== "string") {
|
|
191
|
+
throw new APIError({status: 400, title: "prompt must be a string or a list of strings"});
|
|
192
|
+
}
|
|
193
|
+
const trimmed = entry.trim();
|
|
194
|
+
if (trimmed !== "") {
|
|
195
|
+
names.push(trimmed);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return names;
|
|
178
199
|
};
|
|
179
200
|
|
|
180
201
|
/** Count rows in `from` whose traceId matches the trace, for span and score sorts. */
|
|
@@ -351,17 +372,27 @@ export class LocalTraceStore {
|
|
|
351
372
|
const page = query.page && query.page > 0 ? query.page : 1;
|
|
352
373
|
const limit = query.limit && query.limit > 0 ? Math.min(query.limit, 100) : 20;
|
|
353
374
|
const filter: Record<string, unknown> = {};
|
|
354
|
-
|
|
355
|
-
|
|
375
|
+
// LocalTraceStore is exported, so library callers get the same guards the route applies.
|
|
376
|
+
const status = optionalString(query.status, "status");
|
|
377
|
+
if (status) {
|
|
378
|
+
filter.status = status;
|
|
356
379
|
}
|
|
357
|
-
|
|
358
|
-
|
|
380
|
+
const sessionId = optionalString(query.sessionId, "sessionId");
|
|
381
|
+
if (sessionId) {
|
|
382
|
+
filter.sessionId = sessionId;
|
|
359
383
|
}
|
|
360
|
-
|
|
361
|
-
|
|
384
|
+
const scope = optionalString(query.scope, "scope");
|
|
385
|
+
if (scope) {
|
|
386
|
+
filter.scope = scope;
|
|
362
387
|
}
|
|
363
388
|
const sort = parseTraceListSort(query.sort);
|
|
364
389
|
const prompts = promptNames(query.prompt);
|
|
390
|
+
if (
|
|
391
|
+
query.promptVersion !== undefined &&
|
|
392
|
+
(typeof query.promptVersion !== "number" || !Number.isInteger(query.promptVersion))
|
|
393
|
+
) {
|
|
394
|
+
throw new APIError({status: 400, title: "promptVersion must be an integer"});
|
|
395
|
+
}
|
|
365
396
|
if (query.promptVersion !== undefined && prompts.length === 0) {
|
|
366
397
|
throw new APIError({status: 400, title: "prompt is required when promptVersion is set"});
|
|
367
398
|
}
|
|
@@ -378,10 +409,10 @@ export class LocalTraceStore {
|
|
|
378
409
|
filter["prompts.name"] = {$in: prompts};
|
|
379
410
|
}
|
|
380
411
|
if (query.sensitive !== undefined) {
|
|
381
|
-
filter.sensitive = query.sensitive;
|
|
412
|
+
filter.sensitive = query.sensitive === true;
|
|
382
413
|
}
|
|
383
414
|
if (query.flaggedForDataset !== undefined) {
|
|
384
|
-
filter.flaggedForDataset = query.flaggedForDataset;
|
|
415
|
+
filter.flaggedForDataset = query.flaggedForDataset === true;
|
|
385
416
|
}
|
|
386
417
|
const userId = objectIdOrUndefined(query.userId);
|
|
387
418
|
if (query.userId) {
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {validateAgainstSchema} from "./schemaValidation";
|
|
4
|
+
|
|
5
|
+
const SCHEMA = {
|
|
6
|
+
properties: {
|
|
7
|
+
age: {type: "number"},
|
|
8
|
+
name: {type: "string"},
|
|
9
|
+
},
|
|
10
|
+
required: ["age", "name"],
|
|
11
|
+
type: "object",
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
describe("validateAgainstSchema", () => {
|
|
15
|
+
it("accepts a matching value and skips a missing schema", () => {
|
|
16
|
+
expect(validateAgainstSchema(SCHEMA, {age: 3, name: "a"})).toEqual([]);
|
|
17
|
+
expect(validateAgainstSchema(undefined, {anything: true})).toEqual([]);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it("stops at the first error instead of walking the whole value", () => {
|
|
21
|
+
const errors = validateAgainstSchema(SCHEMA, {age: "old", name: 7});
|
|
22
|
+
expect(errors).toHaveLength(1);
|
|
23
|
+
expect(errors[0]?.path).toBe("/age");
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it("reports a root-level mismatch at /", () => {
|
|
27
|
+
expect(validateAgainstSchema(SCHEMA, "nope")).toEqual([{message: "must be object", path: "/"}]);
|
|
28
|
+
});
|
|
29
|
+
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import Ajv, {type ErrorObject} from "ajv";
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
// Stop at the first error: allErrors walks every node of untrusted input (CodeQL js/resource-exhaustion).
|
|
4
|
+
const ajv = new Ajv({strict: false});
|
|
4
5
|
|
|
5
6
|
export interface SchemaValidationError {
|
|
6
7
|
message: string;
|
|
@@ -63,12 +63,17 @@ export const addAiRequestsExplorerRoutes = (
|
|
|
63
63
|
|
|
64
64
|
const requestTypes = parseRequestTypes(req.query.requestType);
|
|
65
65
|
if (requestTypes.length === 1) {
|
|
66
|
-
match.requestType = requestTypes[0];
|
|
66
|
+
match.requestType = {$eq: requestTypes[0]};
|
|
67
67
|
} else if (requestTypes.length > 1) {
|
|
68
68
|
match.requestType = {$in: requestTypes};
|
|
69
69
|
}
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
const model: unknown = req.query.model;
|
|
71
|
+
if (typeof model === "string") {
|
|
72
|
+
if (model) {
|
|
73
|
+
match.aiModel = {$eq: model};
|
|
74
|
+
}
|
|
75
|
+
} else if (model !== undefined) {
|
|
76
|
+
throw new APIError({status: 400, title: "model must be a string"});
|
|
72
77
|
}
|
|
73
78
|
if (req.query.startDate || req.query.endDate) {
|
|
74
79
|
match.created = {};
|
|
@@ -2495,6 +2495,13 @@ describe("AI Routes", () => {
|
|
|
2495
2495
|
expect(res.body.data[0].aiModel).toBe("claude-3");
|
|
2496
2496
|
});
|
|
2497
2497
|
|
|
2498
|
+
it("rejects an operator object as the model filter", async () => {
|
|
2499
|
+
const agent = await authAsUser(app, "admin");
|
|
2500
|
+
const res = await agent.get("/aiRequestsExplorer?model[$ne]=claude-3");
|
|
2501
|
+
|
|
2502
|
+
expect(res.status).toBe(400);
|
|
2503
|
+
});
|
|
2504
|
+
|
|
2498
2505
|
it("filters by startDate query parameter", async () => {
|
|
2499
2506
|
const old = await AIRequest.create({
|
|
2500
2507
|
aiModel: "gpt-4",
|
|
@@ -247,6 +247,23 @@ describe("parseAiJson", () => {
|
|
|
247
247
|
});
|
|
248
248
|
});
|
|
249
249
|
|
|
250
|
+
describe("parseAiJson fences", () => {
|
|
251
|
+
it("strips a fence with a language tag and surrounding blank lines", () => {
|
|
252
|
+
const r = parseAiJson<{a: number}>('```JSON\n\n{"a":1}\n```');
|
|
253
|
+
expect(r.success).toBe(true);
|
|
254
|
+
if (r.success) {
|
|
255
|
+
expect(r.data).toEqual({a: 1});
|
|
256
|
+
}
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
it("fails fast on an opening fence followed by a long run of spaces", () => {
|
|
260
|
+
const started = performance.now();
|
|
261
|
+
const r = parseAiJson(`\`\`\`${" ".repeat(50_000)}x`);
|
|
262
|
+
expect(performance.now() - started).toBeLessThan(500);
|
|
263
|
+
expect(r.success).toBe(false);
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
|
|
250
267
|
describe("normalizeLlmJsonTextForStructuredOutput", () => {
|
|
251
268
|
it("returns JSON.stringify output on success", () => {
|
|
252
269
|
expect(normalizeLlmJsonTextForStructuredOutput(`Yep: [1,2]`)).toBe("[1,2]");
|
|
@@ -25,17 +25,18 @@ export interface ParseFailure {
|
|
|
25
25
|
|
|
26
26
|
export type ParseResult<T> = ParseFailure | ParseSuccess<T>;
|
|
27
27
|
|
|
28
|
+
const FENCE_LANGUAGE = /^(?:json|javascript|js|typescript|ts)/i;
|
|
29
|
+
|
|
28
30
|
/**
|
|
29
31
|
* Strip leading/trailing markdown code fences when the whole payload is fenced,
|
|
30
32
|
* otherwise strip a common opening ```lang line and trailing ``` block.
|
|
31
33
|
*/
|
|
32
34
|
const stripFences = (input: string): string => {
|
|
33
35
|
const trimmed = input.trim();
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
return fullyFenced[1].trim();
|
|
36
|
+
// String ops instead of one fenced regex: `\s*` and a lazy body made that regex quadratic.
|
|
37
|
+
if (trimmed.length >= 6 && trimmed.startsWith("```") && trimmed.endsWith("```")) {
|
|
38
|
+
const inner = trimmed.slice(3, -3);
|
|
39
|
+
return inner.replace(FENCE_LANGUAGE, "").trim();
|
|
39
40
|
}
|
|
40
41
|
|
|
41
42
|
let t = trimmed;
|