@agenr/openclaw-plugin 1.5.0 → 2026.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -661
- package/README.md +1 -0
- package/dist/build-before-turn-artifact-NPUHVWFE.js +71 -0
- package/dist/build-recall-artifact-F3LS3PZX.js +62 -0
- package/dist/chunk-5AXMFBHR.js +14 -0
- package/dist/chunk-5AYIXQRF.js +4452 -0
- package/dist/chunk-5TIP2EPP.js +6944 -0
- package/dist/chunk-GAERET5Q.js +2070 -0
- package/dist/chunk-GF3PX3VM.js +41 -0
- package/dist/chunk-GKZQ5AG5.js +44 -0
- package/dist/chunk-LDJN7CVU.js +3231 -0
- package/dist/chunk-MC3C2XM5.js +148 -0
- package/dist/chunk-NBS62ES5.js +3012 -0
- package/dist/chunk-NSLTJBUC.js +270 -0
- package/dist/chunk-OJSIZDZD.js +9 -0
- package/dist/chunk-OWGQWQUP.js +45 -0
- package/dist/chunk-Q5UTJXHZ.js +1069 -0
- package/dist/chunk-SOQW7356.js +2416 -0
- package/dist/chunk-VBPYU7GO.js +597 -0
- package/dist/chunk-VTHBPXDQ.js +1750 -0
- package/dist/chunk-XFJ4S4G2.js +1679 -0
- package/dist/chunk-Y5NB3FTH.js +106 -0
- package/dist/chunk-ZX55JBV2.js +4451 -0
- package/dist/index.js +1905 -15298
- package/dist/lifecycle-checkpoint-IAC5FCQU.js +154 -0
- package/dist/scan-6JKPOQHD.js +6 -0
- package/dist/service-EKFACEN6.js +15 -0
- package/dist/service-RHNB5AEQ.js +861 -0
- package/dist/sink-AUAAWC5O.js +8 -0
- package/openclaw.plugin.json +148 -11
- package/package.json +7 -5
- package/dist/anthropic-RE4XNAKE.js +0 -5515
- package/dist/azure-openai-responses-IQLXOCZS.js +0 -190
- package/dist/chunk-6DQXEU2A.js +0 -32306
- package/dist/chunk-EAQYK3U2.js +0 -41
- package/dist/chunk-HNWLZUWE.js +0 -31
- package/dist/chunk-JRUUYSFL.js +0 -262
- package/dist/chunk-OLOUBEE5.js +0 -14022
- package/dist/chunk-P5HNPYGQ.js +0 -174
- package/dist/chunk-RD7BUOBD.js +0 -416
- package/dist/chunk-RWWH2U4W.js +0 -7056
- package/dist/chunk-SEOMNQGB.js +0 -86
- package/dist/chunk-SQLXP7LT.js +0 -4792
- package/dist/chunk-URGOKODJ.js +0 -17
- package/dist/dist-R6ESEJ6P.js +0 -1244
- package/dist/google-NAVXTQLO.js +0 -371
- package/dist/google-gemini-cli-NKYJWHX2.js +0 -712
- package/dist/google-vertex-ZBJ2EDRH.js +0 -414
- package/dist/mistral-SBQYC4J5.js +0 -38407
- package/dist/multipart-parser-DV373IRF.js +0 -371
- package/dist/openai-codex-responses-XN3T3DEN.js +0 -712
- package/dist/openai-completions-75ZFOFU6.js +0 -657
- package/dist/openai-responses-DCK4BVNT.js +0 -198
- package/dist/src-T5RRS2HN.js +0 -1408
|
@@ -0,0 +1,4451 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BEFORE_TURN_DEBUG_ARTIFACT_DEFAULT_TOP_K,
|
|
3
|
+
BEFORE_TURN_DEBUG_ARTIFACT_MAX_TOP_K,
|
|
4
|
+
RECALL_DEBUG_ARTIFACT_DEFAULT_TOP_K,
|
|
5
|
+
RECALL_DEBUG_ARTIFACT_MAX_TOP_K
|
|
6
|
+
} from "./chunk-5AXMFBHR.js";
|
|
7
|
+
import {
|
|
8
|
+
applyDefaultClaimKeyLifecycle,
|
|
9
|
+
composeProcedureRecallText,
|
|
10
|
+
computeProcedureRevisionHash,
|
|
11
|
+
computeProcedureSourceHash
|
|
12
|
+
} from "./chunk-MC3C2XM5.js";
|
|
13
|
+
import {
|
|
14
|
+
createSessionStartRepository,
|
|
15
|
+
formatAgenrBeforeTurnRecall,
|
|
16
|
+
listActiveAbstainDirectives,
|
|
17
|
+
listActiveSessionStartProactiveDirectives,
|
|
18
|
+
listActiveTopicProactiveDirectives,
|
|
19
|
+
runBeforeTurn,
|
|
20
|
+
runSessionStart
|
|
21
|
+
} from "./chunk-XFJ4S4G2.js";
|
|
22
|
+
import {
|
|
23
|
+
DREAM_STAGES,
|
|
24
|
+
DREAM_TIERS,
|
|
25
|
+
attachCrossEncoderPort,
|
|
26
|
+
completeDreamRun,
|
|
27
|
+
createDatabase,
|
|
28
|
+
createDreamRun,
|
|
29
|
+
createEmbeddingClient,
|
|
30
|
+
createOpenAICrossEncoder,
|
|
31
|
+
createProfileSnapshot,
|
|
32
|
+
createRecallAdapter,
|
|
33
|
+
mapRunRow,
|
|
34
|
+
normalizeProcedureDefinition,
|
|
35
|
+
projectClaimCentricRecallEntry,
|
|
36
|
+
resolveCrossEncoderApiKey,
|
|
37
|
+
resolveEmbeddingApiKey,
|
|
38
|
+
resolveEmbeddingModel,
|
|
39
|
+
resolveModel,
|
|
40
|
+
runUnifiedRecall,
|
|
41
|
+
updateDreamState
|
|
42
|
+
} from "./chunk-5TIP2EPP.js";
|
|
43
|
+
import {
|
|
44
|
+
recall
|
|
45
|
+
} from "./chunk-GAERET5Q.js";
|
|
46
|
+
import {
|
|
47
|
+
deriveDreamEfficiencySummary,
|
|
48
|
+
estimateProfileInjectionTokens
|
|
49
|
+
} from "./chunk-GF3PX3VM.js";
|
|
50
|
+
import {
|
|
51
|
+
CLAIM_KEY_SOURCES,
|
|
52
|
+
CLAIM_KEY_STATUSES,
|
|
53
|
+
CLAIM_SUPPORT_MODES,
|
|
54
|
+
DURABLE_KINDS,
|
|
55
|
+
EXPIRY_LEVELS,
|
|
56
|
+
composeEmbeddingText,
|
|
57
|
+
isRecord,
|
|
58
|
+
parseDirectivePolarity,
|
|
59
|
+
parseDirectiveTrigger,
|
|
60
|
+
parseOptionalBoolean,
|
|
61
|
+
parseOptionalIntegerInRange,
|
|
62
|
+
parseOptionalTimestampString,
|
|
63
|
+
parseOptionalTrimmedString,
|
|
64
|
+
parseRequiredTrimmedString,
|
|
65
|
+
pushIssue,
|
|
66
|
+
pushUnexpectedFields,
|
|
67
|
+
readConfig
|
|
68
|
+
} from "./chunk-VTHBPXDQ.js";
|
|
69
|
+
|
|
70
|
+
// src/internal-eval-server.ts
|
|
71
|
+
import process from "process";
|
|
72
|
+
|
|
73
|
+
// src/adapters/api/internal-eval-server.ts
|
|
74
|
+
import { createServer } from "http";
|
|
75
|
+
|
|
76
|
+
// src/app/evals/ablation-arm.ts
|
|
77
|
+
var ABLATION_ARMS = /* @__PURE__ */ new Set(["memory-off", "store-only", "dreaming-on"]);
|
|
78
|
+
function resolveAblationConfig(sandbox) {
|
|
79
|
+
const arm = parseAblationArm(sandbox?.ablationArm);
|
|
80
|
+
const now = sandbox?.now?.trim();
|
|
81
|
+
const profileSnapshot = sandbox?.profileSnapshot;
|
|
82
|
+
return {
|
|
83
|
+
...arm ? { arm } : {},
|
|
84
|
+
...now ? { now } : {},
|
|
85
|
+
...profileSnapshot ? { profileSnapshot } : {}
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
function isMemoryOffArm(config) {
|
|
89
|
+
return config.arm === "memory-off";
|
|
90
|
+
}
|
|
91
|
+
function shouldProvisionProfileSnapshot(config) {
|
|
92
|
+
return config.arm === "dreaming-on" && config.profileSnapshot !== void 0;
|
|
93
|
+
}
|
|
94
|
+
function parseAblationArm(value) {
|
|
95
|
+
if (value === void 0) {
|
|
96
|
+
return void 0;
|
|
97
|
+
}
|
|
98
|
+
if (typeof value !== "string" || !ABLATION_ARMS.has(value)) {
|
|
99
|
+
throw new Error(`sandbox.ablationArm must be one of: ${[...ABLATION_ARMS].join(", ")}.`);
|
|
100
|
+
}
|
|
101
|
+
return value;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// src/app/evals/eval-embedding.ts
|
|
105
|
+
function createEvalEmbeddingResolver() {
|
|
106
|
+
let sharedEmbeddingPort;
|
|
107
|
+
let sharedEmbeddingError;
|
|
108
|
+
const getSupport = () => {
|
|
109
|
+
if (sharedEmbeddingPort) {
|
|
110
|
+
return {
|
|
111
|
+
available: true,
|
|
112
|
+
port: sharedEmbeddingPort
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
if (sharedEmbeddingError) {
|
|
116
|
+
return {
|
|
117
|
+
available: false,
|
|
118
|
+
error: sharedEmbeddingError
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
const config = readConfig();
|
|
122
|
+
try {
|
|
123
|
+
sharedEmbeddingPort = createEmbeddingClient(resolveEmbeddingApiKey(config), resolveEmbeddingModel(config));
|
|
124
|
+
return {
|
|
125
|
+
available: true,
|
|
126
|
+
port: sharedEmbeddingPort
|
|
127
|
+
};
|
|
128
|
+
} catch (error) {
|
|
129
|
+
sharedEmbeddingError = error instanceof Error ? error.message : String(error);
|
|
130
|
+
return {
|
|
131
|
+
available: false,
|
|
132
|
+
error: sharedEmbeddingError
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
const requirePort = () => {
|
|
137
|
+
const support = getSupport();
|
|
138
|
+
if (!support.port) {
|
|
139
|
+
throw new Error(support.error ?? "Embeddings are unavailable.");
|
|
140
|
+
}
|
|
141
|
+
return support.port;
|
|
142
|
+
};
|
|
143
|
+
const portOrUnavailable = () => {
|
|
144
|
+
const support = getSupport();
|
|
145
|
+
return support.port ?? createUnavailableEmbeddingPort(support.error ?? "Embeddings are unavailable.");
|
|
146
|
+
};
|
|
147
|
+
return {
|
|
148
|
+
getSupport,
|
|
149
|
+
requirePort,
|
|
150
|
+
portOrUnavailable
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function createUnavailableEmbeddingPort(message) {
|
|
154
|
+
return {
|
|
155
|
+
async embed() {
|
|
156
|
+
throw new Error(message);
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/app/evals/eval-clock.ts
|
|
162
|
+
function parseEvalNow(nowIso) {
|
|
163
|
+
if (!nowIso) {
|
|
164
|
+
return void 0;
|
|
165
|
+
}
|
|
166
|
+
const parsed = new Date(nowIso);
|
|
167
|
+
if (!Number.isFinite(parsed.getTime())) {
|
|
168
|
+
throw new Error(`sandbox.now must be a parseable ISO timestamp. Received: ${nowIso}`);
|
|
169
|
+
}
|
|
170
|
+
return parsed;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// src/app/evals/recall/provision-fixtures.ts
|
|
174
|
+
import { createHash } from "crypto";
|
|
175
|
+
var DEFAULT_IMPORTANCE = 6;
|
|
176
|
+
var DEFAULT_EXPIRY = "permanent";
|
|
177
|
+
var DEFAULT_QUALITY_SCORE = 0.5;
|
|
178
|
+
async function provisionRecallEvalFixtures(params) {
|
|
179
|
+
const preparedBatch = prepareFixtures(params.caseId, params.memoryPool, params.provisionedAt);
|
|
180
|
+
if (preparedBatch.insertionOrder.length === 0) {
|
|
181
|
+
return {
|
|
182
|
+
provisionedCount: 0,
|
|
183
|
+
providedIdCount: 0,
|
|
184
|
+
generatedIdCount: 0,
|
|
185
|
+
staleCount: 0,
|
|
186
|
+
supersededCount: 0,
|
|
187
|
+
createdAtDefaultedCount: 0,
|
|
188
|
+
updatedAtDefaultedCount: 0,
|
|
189
|
+
seededEntries: []
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const embeddings = await params.embedding.embed(preparedBatch.insertionOrder.map((fixture) => fixture.embeddingText));
|
|
193
|
+
if (embeddings.length !== preparedBatch.insertionOrder.length) {
|
|
194
|
+
throw new Error(`Fixture embedding count mismatch: expected ${preparedBatch.insertionOrder.length}, received ${embeddings.length}.`);
|
|
195
|
+
}
|
|
196
|
+
await params.store.withTransaction(async (store) => {
|
|
197
|
+
for (const [index, fixture] of preparedBatch.insertionOrder.entries()) {
|
|
198
|
+
await store.insertDurable(fixture.entry, embeddings[index] ?? [], fixture.contentHash);
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
return {
|
|
202
|
+
provisionedCount: preparedBatch.insertionOrder.length,
|
|
203
|
+
providedIdCount: preparedBatch.providedIdCount,
|
|
204
|
+
generatedIdCount: preparedBatch.generatedIdCount,
|
|
205
|
+
staleCount: preparedBatch.staleCount,
|
|
206
|
+
supersededCount: preparedBatch.supersededCount,
|
|
207
|
+
createdAtDefaultedCount: preparedBatch.createdAtDefaultedCount,
|
|
208
|
+
updatedAtDefaultedCount: preparedBatch.updatedAtDefaultedCount,
|
|
209
|
+
seededEntries: preparedBatch.seededEntries
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
function prepareFixtures(caseId, fixtures, provisionedAt) {
|
|
213
|
+
const resolvedIds = fixtures.map((fixture, index) => fixture.id ?? createFixtureId(caseId, index, fixture));
|
|
214
|
+
const duplicateIds = findDuplicateIds(resolvedIds);
|
|
215
|
+
if (duplicateIds.length > 0) {
|
|
216
|
+
throw new Error(`Fixture IDs must be unique. Duplicate IDs: ${duplicateIds.join(", ")}.`);
|
|
217
|
+
}
|
|
218
|
+
const knownIds = new Set(resolvedIds);
|
|
219
|
+
const prepared = fixtures.map((fixture, index) => {
|
|
220
|
+
const supersededBy = fixture.superseded_by;
|
|
221
|
+
if (supersededBy && !knownIds.has(supersededBy)) {
|
|
222
|
+
throw new Error(`memoryPool[${index}].superseded_by references unknown fixture id "${supersededBy}".`);
|
|
223
|
+
}
|
|
224
|
+
const entry = buildEntry(fixture, resolvedIds[index] ?? "", provisionedAt);
|
|
225
|
+
return {
|
|
226
|
+
fixtureIndex: index,
|
|
227
|
+
entry,
|
|
228
|
+
contentHash: hashText(`${entry.type}
|
|
229
|
+
${entry.subject}
|
|
230
|
+
${entry.content}`),
|
|
231
|
+
embeddingText: composeEmbeddingText(entry)
|
|
232
|
+
};
|
|
233
|
+
});
|
|
234
|
+
return {
|
|
235
|
+
insertionOrder: topologicallySortFixtures(prepared),
|
|
236
|
+
providedIdCount: fixtures.filter((fixture) => fixture.id !== void 0).length,
|
|
237
|
+
generatedIdCount: fixtures.filter((fixture) => fixture.id === void 0).length,
|
|
238
|
+
staleCount: prepared.filter((fixture) => fixture.entry.valid_to !== void 0).length,
|
|
239
|
+
supersededCount: prepared.filter((fixture) => fixture.entry.superseded_by !== void 0).length,
|
|
240
|
+
createdAtDefaultedCount: fixtures.filter((fixture) => fixture.created_at === void 0).length,
|
|
241
|
+
updatedAtDefaultedCount: fixtures.filter((fixture) => fixture.updated_at === void 0).length,
|
|
242
|
+
seededEntries: prepared.map((fixture) => summarizePreparedFixture(fixture.entry))
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
function buildEntry(fixture, id, provisionedAt) {
|
|
246
|
+
const createdAt = fixture.created_at ?? provisionedAt;
|
|
247
|
+
const updatedAt = fixture.updated_at ?? createdAt;
|
|
248
|
+
return applyDefaultClaimKeyLifecycle(
|
|
249
|
+
{
|
|
250
|
+
id,
|
|
251
|
+
type: fixture.type,
|
|
252
|
+
subject: fixture.subject,
|
|
253
|
+
content: fixture.content,
|
|
254
|
+
importance: fixture.importance ?? DEFAULT_IMPORTANCE,
|
|
255
|
+
expiry: fixture.expiry ?? DEFAULT_EXPIRY,
|
|
256
|
+
tags: fixture.tags ?? [],
|
|
257
|
+
source_file: fixture.source_file,
|
|
258
|
+
source_context: fixture.source_context,
|
|
259
|
+
quality_score: DEFAULT_QUALITY_SCORE,
|
|
260
|
+
recall_count: 0,
|
|
261
|
+
superseded_by: fixture.superseded_by,
|
|
262
|
+
claim_key: fixture.claim_key,
|
|
263
|
+
claim_key_status: fixture.claim_key_status,
|
|
264
|
+
claim_key_source: fixture.claim_key_source,
|
|
265
|
+
claim_support_source_kind: fixture.claim_support_source_kind,
|
|
266
|
+
claim_support_locator: fixture.claim_support_locator,
|
|
267
|
+
claim_support_observed_at: fixture.claim_support_observed_at,
|
|
268
|
+
claim_support_mode: fixture.claim_support_mode,
|
|
269
|
+
valid_from: fixture.valid_from,
|
|
270
|
+
valid_to: fixture.valid_to,
|
|
271
|
+
supersession_kind: fixture.supersession_kind,
|
|
272
|
+
supersession_reason: fixture.supersession_reason,
|
|
273
|
+
directive_polarity: fixture.directive_polarity,
|
|
274
|
+
directive_trigger: fixture.directive_trigger,
|
|
275
|
+
created_at: createdAt,
|
|
276
|
+
updated_at: updatedAt
|
|
277
|
+
},
|
|
278
|
+
"eval fixture"
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
function summarizePreparedFixture(entry) {
|
|
282
|
+
return {
|
|
283
|
+
id: entry.id,
|
|
284
|
+
created_at: entry.created_at,
|
|
285
|
+
updated_at: entry.updated_at,
|
|
286
|
+
superseded_by: entry.superseded_by,
|
|
287
|
+
claim_key: entry.claim_key,
|
|
288
|
+
claim_key_status: entry.claim_key_status,
|
|
289
|
+
valid_from: entry.valid_from,
|
|
290
|
+
valid_to: entry.valid_to
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
function createFixtureId(caseId, index, fixture) {
|
|
294
|
+
const digest = createHash("sha256").update(caseId).update(":").update(String(index)).update(":").update(fixture.type).update(":").update(fixture.subject).update(":").update(fixture.content).digest("hex");
|
|
295
|
+
return `eval-${digest.slice(0, 24)}`;
|
|
296
|
+
}
|
|
297
|
+
function findDuplicateIds(ids) {
|
|
298
|
+
const seen = /* @__PURE__ */ new Set();
|
|
299
|
+
const duplicates = [];
|
|
300
|
+
for (const id of ids) {
|
|
301
|
+
if (seen.has(id)) {
|
|
302
|
+
if (!duplicates.includes(id)) {
|
|
303
|
+
duplicates.push(id);
|
|
304
|
+
}
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
seen.add(id);
|
|
308
|
+
}
|
|
309
|
+
return duplicates;
|
|
310
|
+
}
|
|
311
|
+
function topologicallySortFixtures(fixtures) {
|
|
312
|
+
const indegree = new Map(fixtures.map((fixture) => [fixture.entry.id, 0]));
|
|
313
|
+
const dependents = /* @__PURE__ */ new Map();
|
|
314
|
+
for (const fixture of fixtures) {
|
|
315
|
+
const successorId = fixture.entry.superseded_by;
|
|
316
|
+
if (!successorId) {
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
indegree.set(fixture.entry.id, (indegree.get(fixture.entry.id) ?? 0) + 1);
|
|
320
|
+
const successorDependents = dependents.get(successorId) ?? [];
|
|
321
|
+
successorDependents.push(fixture);
|
|
322
|
+
dependents.set(successorId, successorDependents);
|
|
323
|
+
}
|
|
324
|
+
const ready = fixtures.filter((fixture) => (indegree.get(fixture.entry.id) ?? 0) === 0).sort((left, right) => left.fixtureIndex - right.fixtureIndex);
|
|
325
|
+
const sorted = [];
|
|
326
|
+
while (ready.length > 0) {
|
|
327
|
+
const current = ready.shift();
|
|
328
|
+
if (!current) {
|
|
329
|
+
break;
|
|
330
|
+
}
|
|
331
|
+
sorted.push(current);
|
|
332
|
+
const currentDependents = (dependents.get(current.entry.id) ?? []).sort((left, right) => left.fixtureIndex - right.fixtureIndex);
|
|
333
|
+
for (const dependent of currentDependents) {
|
|
334
|
+
const remaining = (indegree.get(dependent.entry.id) ?? 0) - 1;
|
|
335
|
+
indegree.set(dependent.entry.id, remaining);
|
|
336
|
+
if (remaining === 0) {
|
|
337
|
+
ready.push(dependent);
|
|
338
|
+
ready.sort((left, right) => left.fixtureIndex - right.fixtureIndex);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
if (sorted.length !== fixtures.length) {
|
|
343
|
+
const unresolved = fixtures.filter((fixture) => !sorted.includes(fixture)).map((fixture) => fixture.entry.id);
|
|
344
|
+
throw new Error(`Fixture supersession metadata contains a cycle: ${unresolved.join(", ")}.`);
|
|
345
|
+
}
|
|
346
|
+
return sorted;
|
|
347
|
+
}
|
|
348
|
+
function hashText(value) {
|
|
349
|
+
return createHash("sha256").update(value).digest("hex");
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// src/app/evals/recall/provision-procedure-fixtures.ts
|
|
353
|
+
import { createHash as createHash2 } from "crypto";
|
|
354
|
+
async function provisionRecallEvalProcedureFixtures(params) {
|
|
355
|
+
const procedures = prepareProcedures(params.caseId, params.procedurePool, params.provisionedAt);
|
|
356
|
+
if (procedures.length === 0) {
|
|
357
|
+
return {
|
|
358
|
+
provisionedCount: 0
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
await params.store.withTransaction(async (store) => {
|
|
362
|
+
for (const procedure of procedures) {
|
|
363
|
+
await store.insertProcedure(procedure);
|
|
364
|
+
}
|
|
365
|
+
});
|
|
366
|
+
return {
|
|
367
|
+
provisionedCount: procedures.length
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
function prepareProcedures(caseId, fixtures, provisionedAt) {
|
|
371
|
+
const resolvedIds = fixtures.map((fixture, index) => fixture.id ?? createFixtureId2(caseId, index, fixture));
|
|
372
|
+
const duplicateIds = findDuplicateIds2(resolvedIds);
|
|
373
|
+
if (duplicateIds.length > 0) {
|
|
374
|
+
throw new Error(`Procedure fixture IDs must be unique. Duplicate IDs: ${duplicateIds.join(", ")}.`);
|
|
375
|
+
}
|
|
376
|
+
const knownIds = new Set(resolvedIds);
|
|
377
|
+
return fixtures.map((fixture, index) => {
|
|
378
|
+
if (fixture.superseded_by && !knownIds.has(fixture.superseded_by)) {
|
|
379
|
+
throw new Error(`procedurePool[${index}].superseded_by references unknown fixture id "${fixture.superseded_by}".`);
|
|
380
|
+
}
|
|
381
|
+
const normalizedBody = normalizeProcedureDefinition(
|
|
382
|
+
{
|
|
383
|
+
procedure_key: fixture.procedure_key,
|
|
384
|
+
title: fixture.title,
|
|
385
|
+
goal: fixture.goal,
|
|
386
|
+
when_to_use: fixture.when_to_use ?? [],
|
|
387
|
+
when_not_to_use: fixture.when_not_to_use ?? [],
|
|
388
|
+
prerequisites: fixture.prerequisites ?? [],
|
|
389
|
+
steps: fixture.steps,
|
|
390
|
+
verification: fixture.verification ?? [],
|
|
391
|
+
failure_modes: fixture.failure_modes ?? [],
|
|
392
|
+
sources: fixture.sources ?? [{ kind: "manual", label: "recall eval fixture" }]
|
|
393
|
+
},
|
|
394
|
+
`procedurePool[${index}]`
|
|
395
|
+
);
|
|
396
|
+
const createdAt = fixture.created_at ?? provisionedAt;
|
|
397
|
+
const updatedAt = fixture.updated_at ?? createdAt;
|
|
398
|
+
return {
|
|
399
|
+
id: resolvedIds[index] ?? "",
|
|
400
|
+
...normalizedBody,
|
|
401
|
+
source_file: fixture.source_file,
|
|
402
|
+
recall_text: composeProcedureRecallText(normalizedBody),
|
|
403
|
+
revision_hash: computeProcedureRevisionHash(normalizedBody),
|
|
404
|
+
source_hash: computeProcedureSourceHash(JSON.stringify(normalizedBody)),
|
|
405
|
+
valid_from: fixture.valid_from,
|
|
406
|
+
valid_to: fixture.valid_to,
|
|
407
|
+
supersession_kind: fixture.supersession_kind,
|
|
408
|
+
supersession_reason: fixture.supersession_reason,
|
|
409
|
+
superseded_by: fixture.superseded_by,
|
|
410
|
+
created_at: createdAt,
|
|
411
|
+
updated_at: updatedAt
|
|
412
|
+
};
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
function createFixtureId2(caseId, index, fixture) {
|
|
416
|
+
const digest = createHash2("sha256").update(caseId).update(":").update(String(index)).update(":").update(fixture.procedure_key).update(":").update(fixture.title).update(":").update(fixture.goal).digest("hex");
|
|
417
|
+
return `eval-procedure-${digest.slice(0, 24)}`;
|
|
418
|
+
}
|
|
419
|
+
function findDuplicateIds2(ids) {
|
|
420
|
+
const seen = /* @__PURE__ */ new Set();
|
|
421
|
+
const duplicates = [];
|
|
422
|
+
for (const id of ids) {
|
|
423
|
+
if (seen.has(id)) {
|
|
424
|
+
if (!duplicates.includes(id)) {
|
|
425
|
+
duplicates.push(id);
|
|
426
|
+
}
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
seen.add(id);
|
|
430
|
+
}
|
|
431
|
+
return duplicates;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// src/app/evals/provision-sandbox.ts
|
|
435
|
+
async function provisionEvalSandbox(params) {
|
|
436
|
+
let entryProvisionResult;
|
|
437
|
+
if (params.memoryPool.length > 0) {
|
|
438
|
+
if (!params.embedding) {
|
|
439
|
+
throw new Error("Embeddings are unavailable.");
|
|
440
|
+
}
|
|
441
|
+
entryProvisionResult = await provisionRecallEvalFixtures({
|
|
442
|
+
caseId: params.caseId,
|
|
443
|
+
memoryPool: params.memoryPool,
|
|
444
|
+
store: params.sandbox.fixtureStore,
|
|
445
|
+
embedding: params.embedding,
|
|
446
|
+
provisionedAt: params.provisionedAt
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
if ((params.procedurePool?.length ?? 0) > 0) {
|
|
450
|
+
await provisionRecallEvalProcedureFixtures({
|
|
451
|
+
caseId: params.caseId,
|
|
452
|
+
procedurePool: params.procedurePool ?? [],
|
|
453
|
+
store: params.sandbox.fixtureStore,
|
|
454
|
+
provisionedAt: params.provisionedAt
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
const profileSnapshot = params.profileSnapshot ? await params.sandbox.provisionProfileSnapshot(params.profileSnapshot, params.provisionedAt) : void 0;
|
|
458
|
+
return {
|
|
459
|
+
...entryProvisionResult ? { entryProvisionResult } : {},
|
|
460
|
+
...profileSnapshot ? { profileSnapshotId: profileSnapshot.snapshotId } : {}
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// src/app/evals/recall/sandbox.ts
|
|
465
|
+
import { access, copyFile, mkdir, mkdtemp, rm } from "fs/promises";
|
|
466
|
+
import { tmpdir } from "os";
|
|
467
|
+
import path from "path";
|
|
468
|
+
|
|
469
|
+
// src/adapters/db/eval-dream-run-store.ts
|
|
470
|
+
function createEvalDreamRunStore(executor) {
|
|
471
|
+
return {
|
|
472
|
+
provisionDreamRun: (fixture) => provisionEvalDreamRun(executor, fixture),
|
|
473
|
+
getDreamRun: (runId) => getEvalDreamRun(executor, runId)
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
async function provisionEvalDreamRun(executor, fixture) {
|
|
477
|
+
const completedAt = fixture.completedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
478
|
+
const runId = await createDreamRun(executor, {
|
|
479
|
+
tier: fixture.tier,
|
|
480
|
+
dryRun: false,
|
|
481
|
+
startedAt: completedAt
|
|
482
|
+
});
|
|
483
|
+
const efficiency = fixture.summaryJson.efficiency;
|
|
484
|
+
const estimatedCostUsd = fixture.estimatedCostUsd ?? (efficiency?.costPerSynthesizedDurableUsd !== void 0 && efficiency?.costPerSynthesizedDurableUsd !== null && efficiency.synthesizedDurableMutations > 0 ? efficiency.costPerSynthesizedDurableUsd * efficiency.synthesizedDurableMutations : 0);
|
|
485
|
+
await completeDreamRun(executor, runId, {
|
|
486
|
+
status: "completed",
|
|
487
|
+
inputTokens: 0,
|
|
488
|
+
outputTokens: 0,
|
|
489
|
+
estimatedCostUsd,
|
|
490
|
+
actionsTaken: fixture.summaryJson.actions_taken ?? 0,
|
|
491
|
+
actionsSkipped: 0,
|
|
492
|
+
durablesStaled: fixture.summaryJson.prune?.durablesStaled ?? 0,
|
|
493
|
+
summaryJson: fixture.summaryJson,
|
|
494
|
+
completedAt
|
|
495
|
+
});
|
|
496
|
+
return { runId };
|
|
497
|
+
}
|
|
498
|
+
async function getEvalDreamRun(executor, runId) {
|
|
499
|
+
const result = await executor.execute({
|
|
500
|
+
sql: `
|
|
501
|
+
SELECT
|
|
502
|
+
id,
|
|
503
|
+
tier,
|
|
504
|
+
project,
|
|
505
|
+
started_at,
|
|
506
|
+
completed_at,
|
|
507
|
+
status,
|
|
508
|
+
input_tokens,
|
|
509
|
+
output_tokens,
|
|
510
|
+
estimated_cost_usd,
|
|
511
|
+
model,
|
|
512
|
+
actions_taken,
|
|
513
|
+
actions_skipped,
|
|
514
|
+
durables_staled,
|
|
515
|
+
summary_json,
|
|
516
|
+
error,
|
|
517
|
+
dry_run,
|
|
518
|
+
config_json
|
|
519
|
+
FROM dream_runs
|
|
520
|
+
WHERE id = ?
|
|
521
|
+
LIMIT 1
|
|
522
|
+
`,
|
|
523
|
+
args: [runId.trim()]
|
|
524
|
+
});
|
|
525
|
+
const row = result.rows[0];
|
|
526
|
+
if (!row) {
|
|
527
|
+
return null;
|
|
528
|
+
}
|
|
529
|
+
const run = mapRunRow(row);
|
|
530
|
+
return {
|
|
531
|
+
runId: run.id,
|
|
532
|
+
status: run.status,
|
|
533
|
+
completedAt: run.completedAt,
|
|
534
|
+
estimatedCostUsd: run.estimatedCostUsd,
|
|
535
|
+
summaryJson: run.summaryJson
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// src/adapters/db/eval-fixture-store.ts
|
|
540
|
+
function createRecallEvalFixtureStore(database) {
|
|
541
|
+
return {
|
|
542
|
+
insertDurable: async (entry, embedding, contentHash) => database.insertDurable(entry, embedding, contentHash),
|
|
543
|
+
insertProcedure: async (procedure) => database.upsertProcedure(procedure),
|
|
544
|
+
withTransaction: async (fn) => database.withTransaction(async (transaction) => fn(createRecallEvalFixtureStore(transaction)))
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// src/adapters/db/eval-profile-snapshot-store.ts
|
|
549
|
+
import { createHash as createHash3, randomUUID } from "crypto";
|
|
550
|
+
async function ensureEvalDreamRunExists(executor, runId, provisionedAt) {
|
|
551
|
+
const normalizedRunId = runId.trim();
|
|
552
|
+
if (normalizedRunId.length === 0) {
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
const existing = await executor.execute({
|
|
556
|
+
sql: `
|
|
557
|
+
SELECT id
|
|
558
|
+
FROM dream_runs
|
|
559
|
+
WHERE id = ?
|
|
560
|
+
LIMIT 1
|
|
561
|
+
`,
|
|
562
|
+
args: [normalizedRunId]
|
|
563
|
+
});
|
|
564
|
+
if (existing.rows[0]) {
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
await executor.execute({
|
|
568
|
+
sql: `
|
|
569
|
+
INSERT INTO dream_runs (
|
|
570
|
+
id,
|
|
571
|
+
tier,
|
|
572
|
+
started_at,
|
|
573
|
+
completed_at,
|
|
574
|
+
status,
|
|
575
|
+
dry_run
|
|
576
|
+
)
|
|
577
|
+
VALUES (?, 'light', ?, ?, 'completed', 0)
|
|
578
|
+
`,
|
|
579
|
+
args: [normalizedRunId, provisionedAt, provisionedAt]
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
async function provisionEvalProfileSnapshot(executor, fixture, provisionedAt) {
|
|
583
|
+
const snapshotId = fixture.id?.trim() || `eval-profile-${randomUUID()}`;
|
|
584
|
+
const durableIds = fixture.durableIds.map((id) => id.trim()).filter((id) => id.length > 0);
|
|
585
|
+
const directiveIds = (fixture.directiveIds ?? []).map((id) => id.trim()).filter((id) => id.length > 0);
|
|
586
|
+
const createdAt = fixture.createdAt ?? provisionedAt;
|
|
587
|
+
const asOf = fixture.asOf ?? createdAt;
|
|
588
|
+
const contentHash = createHash3("sha256").update(JSON.stringify({ durableIds, directiveIds, asOf })).digest("hex");
|
|
589
|
+
const runId = fixture.runId?.trim();
|
|
590
|
+
if (runId) {
|
|
591
|
+
await ensureEvalDreamRunExists(executor, runId, createdAt);
|
|
592
|
+
}
|
|
593
|
+
await createProfileSnapshot(executor, {
|
|
594
|
+
id: snapshotId,
|
|
595
|
+
durableIds,
|
|
596
|
+
directiveIds,
|
|
597
|
+
asOf,
|
|
598
|
+
contentHash,
|
|
599
|
+
runId: runId ?? null,
|
|
600
|
+
createdAt
|
|
601
|
+
});
|
|
602
|
+
await updateDreamState(executor, {
|
|
603
|
+
activeProfileSnapshotId: snapshotId,
|
|
604
|
+
updatedAt: createdAt
|
|
605
|
+
});
|
|
606
|
+
return { snapshotId };
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
// src/app/evals/recall/sandbox.ts
|
|
610
|
+
var SANDBOX_DB_FILENAME = "knowledge.db";
|
|
611
|
+
var SANDBOX_DIR_PREFIX = "agenr-recall-eval-";
|
|
612
|
+
async function setupRecallEvalSandbox(request) {
|
|
613
|
+
const suppliedRoot = request?.root !== void 0;
|
|
614
|
+
const preserved = request?.preserve === true;
|
|
615
|
+
const root = suppliedRoot ? path.resolve(request.root ?? "") : await mkdtemp(path.join(tmpdir(), SANDBOX_DIR_PREFIX));
|
|
616
|
+
const snapshotSeed = request?.corpusSeed?.mode === "snapshot_copy" ? request.corpusSeed : void 0;
|
|
617
|
+
let database;
|
|
618
|
+
const dbPath = path.join(root, SANDBOX_DB_FILENAME);
|
|
619
|
+
try {
|
|
620
|
+
if (suppliedRoot) {
|
|
621
|
+
await mkdir(root, { recursive: true });
|
|
622
|
+
}
|
|
623
|
+
await removeDatabaseFiles(dbPath);
|
|
624
|
+
if (snapshotSeed !== void 0) {
|
|
625
|
+
await seedSandboxFromSnapshot(snapshotSeed, dbPath);
|
|
626
|
+
}
|
|
627
|
+
database = await createDatabase(dbPath);
|
|
628
|
+
const openDatabase = database;
|
|
629
|
+
const snapshot = snapshotSeed ? buildSnapshotMetadata(snapshotSeed) : void 0;
|
|
630
|
+
return {
|
|
631
|
+
root,
|
|
632
|
+
dbPath,
|
|
633
|
+
preserved,
|
|
634
|
+
fixtureStore: createRecallEvalFixtureStore(openDatabase),
|
|
635
|
+
episodeDatabase: openDatabase,
|
|
636
|
+
procedureDatabase: openDatabase,
|
|
637
|
+
sessionStartRepository: createSessionStartRepository(openDatabase),
|
|
638
|
+
listActiveAbstainDirectives: (now) => listActiveAbstainDirectives(openDatabase, now),
|
|
639
|
+
listActiveSessionStartProactiveDirectives: (now) => listActiveSessionStartProactiveDirectives(openDatabase, now),
|
|
640
|
+
listActiveTopicProactiveDirectives: (now) => listActiveTopicProactiveDirectives(openDatabase, now),
|
|
641
|
+
provisionProfileSnapshot: (fixture, provisionedAt) => provisionEvalProfileSnapshot(openDatabase, fixture, provisionedAt),
|
|
642
|
+
dreamRunStore: createEvalDreamRunStore(openDatabase),
|
|
643
|
+
...snapshot ? { snapshot } : {},
|
|
644
|
+
createRecallPorts: (embedding) => createRecallAdapter(openDatabase, embedding),
|
|
645
|
+
cleanup: async () => {
|
|
646
|
+
await openDatabase.close().catch(() => void 0);
|
|
647
|
+
if (preserved) {
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
650
|
+
if (suppliedRoot) {
|
|
651
|
+
await removeDatabaseFiles(dbPath);
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
await rm(root, { recursive: true, force: true });
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
} catch (error) {
|
|
658
|
+
await database?.close().catch(() => void 0);
|
|
659
|
+
if (!preserved) {
|
|
660
|
+
if (suppliedRoot) {
|
|
661
|
+
await removeDatabaseFiles(dbPath).catch(() => void 0);
|
|
662
|
+
} else {
|
|
663
|
+
await rm(root, { recursive: true, force: true }).catch(() => void 0);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
throw error;
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
async function seedSandboxFromSnapshot(seed, dbPath) {
|
|
670
|
+
const rawPath = seed.snapshotDbPath.trim();
|
|
671
|
+
if (rawPath.length === 0) {
|
|
672
|
+
throw new Error("Snapshot database path must not be empty.");
|
|
673
|
+
}
|
|
674
|
+
const sourcePath = path.resolve(rawPath);
|
|
675
|
+
const targetPath = path.resolve(dbPath);
|
|
676
|
+
if (sourcePath === targetPath) {
|
|
677
|
+
throw new Error("Snapshot database path must not point at the sandbox database path.");
|
|
678
|
+
}
|
|
679
|
+
try {
|
|
680
|
+
await access(sourcePath);
|
|
681
|
+
} catch (error) {
|
|
682
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
683
|
+
throw new Error(`Snapshot database file is not accessible at ${sourcePath}: ${cause}`, {
|
|
684
|
+
cause: error
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
await copyFile(sourcePath, dbPath);
|
|
688
|
+
}
|
|
689
|
+
function buildSnapshotMetadata(seed) {
|
|
690
|
+
return {
|
|
691
|
+
...seed.snapshotId !== void 0 ? { id: seed.snapshotId } : {},
|
|
692
|
+
...seed.snapshotLabel !== void 0 ? { label: seed.snapshotLabel } : {},
|
|
693
|
+
dbPathBasename: path.basename(seed.snapshotDbPath),
|
|
694
|
+
allowedTelemetryWrites: seed.allowTelemetryWrites === true
|
|
695
|
+
};
|
|
696
|
+
}
|
|
697
|
+
async function removeDatabaseFiles(dbPath) {
|
|
698
|
+
await Promise.all([rm(dbPath, { force: true }), rm(`${dbPath}-wal`, { force: true }), rm(`${dbPath}-shm`, { force: true })]);
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// src/app/evals/recall/telemetry-write-gate.ts
|
|
702
|
+
function applyTelemetryWriteGate(ports, sandbox) {
|
|
703
|
+
const snapshot = sandbox.snapshot;
|
|
704
|
+
if (snapshot === void 0 || snapshot.allowedTelemetryWrites) {
|
|
705
|
+
return ports;
|
|
706
|
+
}
|
|
707
|
+
return {
|
|
708
|
+
async embed(text) {
|
|
709
|
+
return ports.embed(text);
|
|
710
|
+
},
|
|
711
|
+
async vectorSearch(params) {
|
|
712
|
+
return ports.vectorSearch(params);
|
|
713
|
+
},
|
|
714
|
+
async ftsSearch(params) {
|
|
715
|
+
return ports.ftsSearch(params);
|
|
716
|
+
},
|
|
717
|
+
...ports.expandNeighborhood ? {
|
|
718
|
+
async expandNeighborhood(request) {
|
|
719
|
+
return ports.expandNeighborhood(request);
|
|
720
|
+
}
|
|
721
|
+
} : {},
|
|
722
|
+
...ports.crossEncoder ? {
|
|
723
|
+
crossEncoder: ports.crossEncoder
|
|
724
|
+
} : {},
|
|
725
|
+
async hydrateEntries(ids) {
|
|
726
|
+
return ports.hydrateEntries(ids);
|
|
727
|
+
},
|
|
728
|
+
async recordRecallEvents() {
|
|
729
|
+
return void 0;
|
|
730
|
+
}
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// src/app/evals/before-turn/build-debug-artifact.ts
|
|
735
|
+
function buildBeforeTurnDebugArtifact(params) {
|
|
736
|
+
const { request, patch, sandbox } = params;
|
|
737
|
+
const diagnostics = patch.diagnostics;
|
|
738
|
+
const topK = resolveTopK(request.options?.topKCandidates);
|
|
739
|
+
const durableTopCandidates = buildDurableCandidates(patch, topK);
|
|
740
|
+
const procedureTopCandidates = buildProcedureCandidates(patch, topK);
|
|
741
|
+
const trigger = request.beforeTurnInput.trigger ?? "unspecified";
|
|
742
|
+
return {
|
|
743
|
+
schemaVersion: "before-turn-debug-artifact.v1",
|
|
744
|
+
caseId: request.caseId,
|
|
745
|
+
...sandbox.snapshot ? { snapshot: buildSnapshot(sandbox.snapshot) } : {},
|
|
746
|
+
input: {
|
|
747
|
+
trigger,
|
|
748
|
+
currentTurnText: request.beforeTurnInput.currentTurnText
|
|
749
|
+
},
|
|
750
|
+
...diagnostics.queryPolicy ? { queryPolicy: diagnostics.queryPolicy } : {},
|
|
751
|
+
...diagnostics.queryVariants.length > 0 ? { queryVariants: [...diagnostics.queryVariants] } : {},
|
|
752
|
+
...diagnostics.abstentionReasons.length > 0 ? { abstentionReasons: [...diagnostics.abstentionReasons] } : {},
|
|
753
|
+
selectedEntryIds: patch.durableMemory.map((item) => item.entry.id),
|
|
754
|
+
selectedProcedureKey: patch.procedure?.procedure.procedure_key ?? null,
|
|
755
|
+
...durableTopCandidates.length > 0 ? { durableRecallTopCandidates: durableTopCandidates } : {},
|
|
756
|
+
...procedureTopCandidates.length > 0 ? { procedureTopCandidates } : {}
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
function buildSnapshot(snapshot) {
|
|
760
|
+
return {
|
|
761
|
+
...snapshot.id !== void 0 ? { id: snapshot.id } : {},
|
|
762
|
+
...snapshot.label !== void 0 ? { label: snapshot.label } : {},
|
|
763
|
+
dbPathBasename: snapshot.dbPathBasename
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
function resolveTopK(requested) {
|
|
767
|
+
if (requested === void 0) {
|
|
768
|
+
return BEFORE_TURN_DEBUG_ARTIFACT_DEFAULT_TOP_K;
|
|
769
|
+
}
|
|
770
|
+
if (!Number.isFinite(requested) || !Number.isInteger(requested)) {
|
|
771
|
+
return BEFORE_TURN_DEBUG_ARTIFACT_DEFAULT_TOP_K;
|
|
772
|
+
}
|
|
773
|
+
if (requested < 1) {
|
|
774
|
+
return 1;
|
|
775
|
+
}
|
|
776
|
+
if (requested > BEFORE_TURN_DEBUG_ARTIFACT_MAX_TOP_K) {
|
|
777
|
+
return BEFORE_TURN_DEBUG_ARTIFACT_MAX_TOP_K;
|
|
778
|
+
}
|
|
779
|
+
return requested;
|
|
780
|
+
}
|
|
781
|
+
function buildDurableCandidates(patch, topK) {
|
|
782
|
+
return patch.durableMemory.slice(0, topK).map((item) => {
|
|
783
|
+
const reasons = item.whySurfaced.reasons.length > 0 ? [...item.whySurfaced.reasons] : void 0;
|
|
784
|
+
return {
|
|
785
|
+
id: item.entry.id,
|
|
786
|
+
score: item.score,
|
|
787
|
+
...reasons ? { reasons } : {}
|
|
788
|
+
};
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
function buildProcedureCandidates(patch, topK) {
|
|
792
|
+
if (!patch.procedure) {
|
|
793
|
+
return [];
|
|
794
|
+
}
|
|
795
|
+
if (topK < 1) {
|
|
796
|
+
return [];
|
|
797
|
+
}
|
|
798
|
+
const reasons = patch.procedure.whySurfaced.reasons.length > 0 ? [...patch.procedure.whySurfaced.reasons] : void 0;
|
|
799
|
+
return [
|
|
800
|
+
{
|
|
801
|
+
procedureKey: patch.procedure.procedure.procedure_key,
|
|
802
|
+
score: patch.procedure.score,
|
|
803
|
+
...reasons ? { reasons } : {}
|
|
804
|
+
}
|
|
805
|
+
];
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
// src/app/evals/before-turn/normalize-response.ts
|
|
809
|
+
function buildBeforeTurnEvalSuccessResponse(params) {
|
|
810
|
+
const output = buildOutput(params.patch, params.renderedPatchText);
|
|
811
|
+
const debugArtifact = params.request.options?.includeDebugArtifact === true ? buildBeforeTurnDebugArtifact({ request: params.request, patch: params.patch, sandbox: params.sandbox }) : void 0;
|
|
812
|
+
return {
|
|
813
|
+
status: "ok",
|
|
814
|
+
caseId: params.request.caseId,
|
|
815
|
+
output,
|
|
816
|
+
diagnostics: params.request.options?.includeDiagnostics === true ? params.patch.diagnostics : void 0,
|
|
817
|
+
timings: params.timings,
|
|
818
|
+
sandbox: buildSandboxResult(params.sandbox),
|
|
819
|
+
...debugArtifact ? { debugArtifact } : {}
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
function buildBeforeTurnEvalErrorResponse(params) {
|
|
823
|
+
return {
|
|
824
|
+
status: "error",
|
|
825
|
+
caseId: params.request.caseId,
|
|
826
|
+
error: {
|
|
827
|
+
code: params.code,
|
|
828
|
+
message: params.message,
|
|
829
|
+
details: params.details
|
|
830
|
+
},
|
|
831
|
+
timings: params.timings,
|
|
832
|
+
sandbox: params.sandbox ? buildSandboxResult(params.sandbox) : void 0
|
|
833
|
+
};
|
|
834
|
+
}
|
|
835
|
+
function maybeRenderBeforeTurnPatch(request, patch) {
|
|
836
|
+
if (request.options?.includeRenderedPatch !== true) {
|
|
837
|
+
return void 0;
|
|
838
|
+
}
|
|
839
|
+
return formatAgenrBeforeTurnRecall(patch);
|
|
840
|
+
}
|
|
841
|
+
function buildSandboxResult(sandbox) {
|
|
842
|
+
return {
|
|
843
|
+
root: sandbox.root,
|
|
844
|
+
dbPath: sandbox.dbPath,
|
|
845
|
+
preserved: sandbox.preserved,
|
|
846
|
+
...sandbox.snapshot ? { snapshot: sandbox.snapshot } : {}
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
function buildOutput(patch, renderedPatchText) {
|
|
850
|
+
return {
|
|
851
|
+
abstained: patch.diagnostics.abstained,
|
|
852
|
+
selectedEntryIds: patch.durableMemory.map((item) => item.entry.id),
|
|
853
|
+
selectedProcedureKey: patch.procedure?.procedure.procedure_key ?? null,
|
|
854
|
+
patch: normalizePatchForEvalOutput(patch),
|
|
855
|
+
...renderedPatchText !== void 0 ? { renderedPatchText } : {}
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
function normalizePatchForEvalOutput(patch) {
|
|
859
|
+
return {
|
|
860
|
+
...patch,
|
|
861
|
+
durableMemory: patch.durableMemory.map((item) => ({
|
|
862
|
+
...item,
|
|
863
|
+
entry: {
|
|
864
|
+
...item.entry,
|
|
865
|
+
...typeof item.entry.claim_key === "string" ? { claimKey: item.entry.claim_key } : {}
|
|
866
|
+
}
|
|
867
|
+
}))
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
// src/app/evals/before-turn/run-before-turn-eval-case.ts
|
|
872
|
+
async function runBeforeTurnEvalCase(request, dependencies = {}) {
|
|
873
|
+
const startedAt = Date.now();
|
|
874
|
+
const provisionedAt = new Date(startedAt).toISOString();
|
|
875
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
876
|
+
const evalNow = parseEvalNow(ablation.now);
|
|
877
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
878
|
+
let sandbox;
|
|
879
|
+
let timings;
|
|
880
|
+
try {
|
|
881
|
+
const sandboxStartedAt = Date.now();
|
|
882
|
+
try {
|
|
883
|
+
sandbox = await setupRecallEvalSandbox(request.sandbox);
|
|
884
|
+
timings = {
|
|
885
|
+
...timings,
|
|
886
|
+
sandboxSetupMs: elapsedMs(sandboxStartedAt)
|
|
887
|
+
};
|
|
888
|
+
} catch (error) {
|
|
889
|
+
timings = {
|
|
890
|
+
...timings,
|
|
891
|
+
totalMs: elapsedMs(startedAt),
|
|
892
|
+
sandboxSetupMs: elapsedMs(sandboxStartedAt)
|
|
893
|
+
};
|
|
894
|
+
return buildBeforeTurnEvalErrorResponse({
|
|
895
|
+
request,
|
|
896
|
+
code: "sandbox_setup_failed",
|
|
897
|
+
message: "Failed to create isolated before-turn eval sandbox.",
|
|
898
|
+
details: toErrorDetails(error),
|
|
899
|
+
timings: request.options?.includeTimings === true ? timings : void 0
|
|
900
|
+
});
|
|
901
|
+
}
|
|
902
|
+
if (request.memoryPool.length > 0 || (request.procedurePool?.length ?? 0) > 0 || shouldProvisionProfileSnapshot(ablation)) {
|
|
903
|
+
const provisionStartedAt = Date.now();
|
|
904
|
+
try {
|
|
905
|
+
await provisionEvalSandbox({
|
|
906
|
+
caseId: request.caseId,
|
|
907
|
+
sandbox,
|
|
908
|
+
memoryPool: request.memoryPool,
|
|
909
|
+
procedurePool: request.procedurePool,
|
|
910
|
+
profileSnapshot: shouldProvisionProfileSnapshot(ablation) ? ablation.profileSnapshot : void 0,
|
|
911
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.portOrUnavailable() : void 0,
|
|
912
|
+
provisionedAt
|
|
913
|
+
});
|
|
914
|
+
timings = {
|
|
915
|
+
...timings,
|
|
916
|
+
fixtureProvisionMs: elapsedMs(provisionStartedAt)
|
|
917
|
+
};
|
|
918
|
+
} catch (error) {
|
|
919
|
+
timings = {
|
|
920
|
+
...timings,
|
|
921
|
+
totalMs: elapsedMs(startedAt),
|
|
922
|
+
fixtureProvisionMs: elapsedMs(provisionStartedAt)
|
|
923
|
+
};
|
|
924
|
+
return buildBeforeTurnEvalErrorResponse({
|
|
925
|
+
request,
|
|
926
|
+
code: "fixture_provision_failed",
|
|
927
|
+
message: "Failed to provision before-turn eval fixtures into isolated storage.",
|
|
928
|
+
details: toErrorDetails(error),
|
|
929
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
930
|
+
sandbox
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
const beforeTurnStartedAt = Date.now();
|
|
935
|
+
try {
|
|
936
|
+
if (isMemoryOffArm(ablation)) {
|
|
937
|
+
timings = {
|
|
938
|
+
...timings,
|
|
939
|
+
beforeTurnMs: elapsedMs(beforeTurnStartedAt),
|
|
940
|
+
totalMs: elapsedMs(startedAt)
|
|
941
|
+
};
|
|
942
|
+
return buildBeforeTurnEvalSuccessResponse({
|
|
943
|
+
request,
|
|
944
|
+
patch: {
|
|
945
|
+
durableMemory: [],
|
|
946
|
+
diagnostics: {
|
|
947
|
+
abstained: true,
|
|
948
|
+
abstentionReasons: ["memory_off_ablation"],
|
|
949
|
+
queryVariants: [],
|
|
950
|
+
recentTurnCount: 0,
|
|
951
|
+
turnSignalLabels: [],
|
|
952
|
+
durableRecallUsed: false,
|
|
953
|
+
durableRecallCandidateCount: 0,
|
|
954
|
+
procedureRecallUsed: false,
|
|
955
|
+
procedureCandidateCount: 0,
|
|
956
|
+
notices: ["memory-off ablation arm stubbed before-turn injection."]
|
|
957
|
+
}
|
|
958
|
+
},
|
|
959
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
960
|
+
sandbox
|
|
961
|
+
});
|
|
962
|
+
}
|
|
963
|
+
const activeSandbox = sandbox;
|
|
964
|
+
if (!activeSandbox) {
|
|
965
|
+
throw new Error("Before-turn eval sandbox was not initialized.");
|
|
966
|
+
}
|
|
967
|
+
const embeddingSupport = embeddingResolver.getSupport();
|
|
968
|
+
const sandboxRecallPorts = activeSandbox.createRecallPorts(
|
|
969
|
+
embeddingSupport.port ?? createUnavailableEmbeddingPort(embeddingSupport.error ?? "Embeddings are unavailable.")
|
|
970
|
+
);
|
|
971
|
+
const recallPorts = applyTelemetryWriteGate(attachCrossEncoderPort(sandboxRecallPorts, dependencies.crossEncoder), activeSandbox);
|
|
972
|
+
const patch = await runBeforeTurn(request.beforeTurnInput, {
|
|
973
|
+
recall: recallPorts,
|
|
974
|
+
procedures: activeSandbox.procedureDatabase,
|
|
975
|
+
...evalNow ? { now: evalNow } : {},
|
|
976
|
+
listActiveAbstainDirectives: () => activeSandbox.listActiveAbstainDirectives(evalNow),
|
|
977
|
+
listActiveTopicProactiveDirectives: () => activeSandbox.listActiveTopicProactiveDirectives(evalNow),
|
|
978
|
+
embedQuery: embeddingSupport.port ? async (text) => {
|
|
979
|
+
const vectors = await embeddingSupport.port.embed([text]);
|
|
980
|
+
return vectors[0] ?? [];
|
|
981
|
+
} : void 0
|
|
982
|
+
});
|
|
983
|
+
timings = {
|
|
984
|
+
...timings,
|
|
985
|
+
beforeTurnMs: elapsedMs(beforeTurnStartedAt)
|
|
986
|
+
};
|
|
987
|
+
let renderedPatchText;
|
|
988
|
+
if (request.options?.includeRenderedPatch === true) {
|
|
989
|
+
const renderStartedAt = Date.now();
|
|
990
|
+
renderedPatchText = maybeRenderBeforeTurnPatch(request, patch);
|
|
991
|
+
timings = {
|
|
992
|
+
...timings,
|
|
993
|
+
renderPatchMs: elapsedMs(renderStartedAt)
|
|
994
|
+
};
|
|
995
|
+
}
|
|
996
|
+
timings = {
|
|
997
|
+
...timings,
|
|
998
|
+
totalMs: elapsedMs(startedAt)
|
|
999
|
+
};
|
|
1000
|
+
return buildBeforeTurnEvalSuccessResponse({
|
|
1001
|
+
request,
|
|
1002
|
+
patch,
|
|
1003
|
+
renderedPatchText,
|
|
1004
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
1005
|
+
sandbox: activeSandbox
|
|
1006
|
+
});
|
|
1007
|
+
} catch (error) {
|
|
1008
|
+
timings = {
|
|
1009
|
+
...timings,
|
|
1010
|
+
totalMs: elapsedMs(startedAt),
|
|
1011
|
+
beforeTurnMs: elapsedMs(beforeTurnStartedAt)
|
|
1012
|
+
};
|
|
1013
|
+
return buildBeforeTurnEvalErrorResponse({
|
|
1014
|
+
request,
|
|
1015
|
+
code: "before_turn_execution_failed",
|
|
1016
|
+
message: "Failed to execute real before-turn selection against isolated eval state.",
|
|
1017
|
+
details: toErrorDetails(error),
|
|
1018
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
1019
|
+
sandbox
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
} catch (error) {
|
|
1023
|
+
return buildBeforeTurnEvalErrorResponse({
|
|
1024
|
+
request,
|
|
1025
|
+
code: "internal_error",
|
|
1026
|
+
message: "Before-turn eval execution failed unexpectedly.",
|
|
1027
|
+
details: toErrorDetails(error),
|
|
1028
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs(startedAt) } : void 0,
|
|
1029
|
+
sandbox
|
|
1030
|
+
});
|
|
1031
|
+
} finally {
|
|
1032
|
+
await sandbox?.cleanup().catch(() => void 0);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
function toErrorDetails(error) {
|
|
1036
|
+
if (error instanceof Error) {
|
|
1037
|
+
return {
|
|
1038
|
+
cause: error.message
|
|
1039
|
+
};
|
|
1040
|
+
}
|
|
1041
|
+
return {
|
|
1042
|
+
cause: String(error)
|
|
1043
|
+
};
|
|
1044
|
+
}
|
|
1045
|
+
function elapsedMs(startedAt) {
|
|
1046
|
+
return Date.now() - startedAt;
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
// src/adapters/api/validation/internal-eval-shared.ts
|
|
1050
|
+
var SANDBOX_REQUEST_KEYS = /* @__PURE__ */ new Set(["root", "preserve", "corpusSeed", "ablationArm", "now", "profileSnapshot"]);
|
|
1051
|
+
var PROFILE_SNAPSHOT_KEYS = /* @__PURE__ */ new Set(["id", "durableIds", "directiveIds", "asOf", "runId", "createdAt"]);
|
|
1052
|
+
var ABLATION_ARMS2 = ["memory-off", "store-only", "dreaming-on"];
|
|
1053
|
+
var CORPUS_SEED_MODES = ["fixture", "snapshot_copy"];
|
|
1054
|
+
var FIXTURE_CORPUS_SEED_KEYS = /* @__PURE__ */ new Set(["mode"]);
|
|
1055
|
+
var SNAPSHOT_COPY_CORPUS_SEED_KEYS = /* @__PURE__ */ new Set(["mode", "snapshotDbPath", "snapshotId", "snapshotLabel", "allowTelemetryWrites"]);
|
|
1056
|
+
var FIXTURE_ENTRY_KEYS = /* @__PURE__ */ new Set([
|
|
1057
|
+
"id",
|
|
1058
|
+
"type",
|
|
1059
|
+
"subject",
|
|
1060
|
+
"content",
|
|
1061
|
+
"importance",
|
|
1062
|
+
"expiry",
|
|
1063
|
+
"tags",
|
|
1064
|
+
"source_file",
|
|
1065
|
+
"source_context",
|
|
1066
|
+
"created_at",
|
|
1067
|
+
"updated_at",
|
|
1068
|
+
"superseded_by",
|
|
1069
|
+
"claim_key",
|
|
1070
|
+
"claim_key_status",
|
|
1071
|
+
"claim_key_source",
|
|
1072
|
+
"claim_support_source_kind",
|
|
1073
|
+
"claim_support_locator",
|
|
1074
|
+
"claim_support_observed_at",
|
|
1075
|
+
"claim_support_mode",
|
|
1076
|
+
"valid_from",
|
|
1077
|
+
"valid_to",
|
|
1078
|
+
"supersession_kind",
|
|
1079
|
+
"supersession_reason",
|
|
1080
|
+
"directive_polarity",
|
|
1081
|
+
"directive_trigger"
|
|
1082
|
+
]);
|
|
1083
|
+
var FIXTURE_PROCEDURE_KEYS = /* @__PURE__ */ new Set([
|
|
1084
|
+
"id",
|
|
1085
|
+
"procedure_key",
|
|
1086
|
+
"title",
|
|
1087
|
+
"goal",
|
|
1088
|
+
"when_to_use",
|
|
1089
|
+
"when_not_to_use",
|
|
1090
|
+
"prerequisites",
|
|
1091
|
+
"steps",
|
|
1092
|
+
"verification",
|
|
1093
|
+
"failure_modes",
|
|
1094
|
+
"sources",
|
|
1095
|
+
"source_file",
|
|
1096
|
+
"valid_from",
|
|
1097
|
+
"valid_to",
|
|
1098
|
+
"supersession_kind",
|
|
1099
|
+
"supersession_reason",
|
|
1100
|
+
"superseded_by",
|
|
1101
|
+
"created_at",
|
|
1102
|
+
"updated_at"
|
|
1103
|
+
]);
|
|
1104
|
+
function extractParseableCaseId(value) {
|
|
1105
|
+
if (!isRecord(value) || typeof value.caseId !== "string") {
|
|
1106
|
+
return void 0;
|
|
1107
|
+
}
|
|
1108
|
+
const normalized = value.caseId.trim();
|
|
1109
|
+
return normalized.length > 0 ? normalized : void 0;
|
|
1110
|
+
}
|
|
1111
|
+
function parseObject(value, path2, issues) {
|
|
1112
|
+
if (!isRecord(value)) {
|
|
1113
|
+
pushIssue(issues, path2, "Expected an object.");
|
|
1114
|
+
return void 0;
|
|
1115
|
+
}
|
|
1116
|
+
return value;
|
|
1117
|
+
}
|
|
1118
|
+
function parseSandbox(value, issues) {
|
|
1119
|
+
if (value === void 0) {
|
|
1120
|
+
return void 0;
|
|
1121
|
+
}
|
|
1122
|
+
const sandbox = parseObject(value, "sandbox", issues);
|
|
1123
|
+
if (sandbox === void 0) {
|
|
1124
|
+
return void 0;
|
|
1125
|
+
}
|
|
1126
|
+
pushUnexpectedFields(sandbox, SANDBOX_REQUEST_KEYS, "sandbox", issues);
|
|
1127
|
+
return {
|
|
1128
|
+
root: parseOptionalTrimmedString(sandbox.root, "sandbox.root", issues),
|
|
1129
|
+
preserve: parseOptionalBoolean(sandbox.preserve, "sandbox.preserve", issues),
|
|
1130
|
+
corpusSeed: parseCorpusSeed(sandbox.corpusSeed, issues),
|
|
1131
|
+
ablationArm: parseOptionalAblationArm(sandbox.ablationArm, "sandbox.ablationArm", issues),
|
|
1132
|
+
now: parseOptionalTimestampString(sandbox.now, "sandbox.now", issues),
|
|
1133
|
+
profileSnapshot: parseProfileSnapshot(sandbox.profileSnapshot, issues)
|
|
1134
|
+
};
|
|
1135
|
+
}
|
|
1136
|
+
function parseCorpusSeed(value, issues) {
|
|
1137
|
+
if (value === void 0) {
|
|
1138
|
+
return void 0;
|
|
1139
|
+
}
|
|
1140
|
+
const seed = parseObject(value, "sandbox.corpusSeed", issues);
|
|
1141
|
+
if (seed === void 0) {
|
|
1142
|
+
return void 0;
|
|
1143
|
+
}
|
|
1144
|
+
const mode = parseCorpusSeedMode(seed.mode, "sandbox.corpusSeed.mode", issues);
|
|
1145
|
+
if (mode === void 0) {
|
|
1146
|
+
return void 0;
|
|
1147
|
+
}
|
|
1148
|
+
if (mode === "fixture") {
|
|
1149
|
+
pushUnexpectedFields(seed, FIXTURE_CORPUS_SEED_KEYS, "sandbox.corpusSeed", issues);
|
|
1150
|
+
return { mode: "fixture" };
|
|
1151
|
+
}
|
|
1152
|
+
pushUnexpectedFields(seed, SNAPSHOT_COPY_CORPUS_SEED_KEYS, "sandbox.corpusSeed", issues);
|
|
1153
|
+
const snapshotDbPath = parseRequiredTrimmedString(seed.snapshotDbPath, "sandbox.corpusSeed.snapshotDbPath", issues);
|
|
1154
|
+
const snapshotId = parseOptionalTrimmedString(seed.snapshotId, "sandbox.corpusSeed.snapshotId", issues);
|
|
1155
|
+
const snapshotLabel = parseOptionalTrimmedString(seed.snapshotLabel, "sandbox.corpusSeed.snapshotLabel", issues);
|
|
1156
|
+
const allowTelemetryWrites = parseOptionalBoolean(seed.allowTelemetryWrites, "sandbox.corpusSeed.allowTelemetryWrites", issues);
|
|
1157
|
+
if (snapshotDbPath === void 0) {
|
|
1158
|
+
return void 0;
|
|
1159
|
+
}
|
|
1160
|
+
return {
|
|
1161
|
+
mode: "snapshot_copy",
|
|
1162
|
+
snapshotDbPath,
|
|
1163
|
+
...snapshotId !== void 0 ? { snapshotId } : {},
|
|
1164
|
+
...snapshotLabel !== void 0 ? { snapshotLabel } : {},
|
|
1165
|
+
...allowTelemetryWrites !== void 0 ? { allowTelemetryWrites } : {}
|
|
1166
|
+
};
|
|
1167
|
+
}
|
|
1168
|
+
function parseCorpusSeedMode(value, path2, issues) {
|
|
1169
|
+
if (typeof value !== "string" || !CORPUS_SEED_MODES.includes(value)) {
|
|
1170
|
+
pushIssue(issues, path2, `Expected one of: ${CORPUS_SEED_MODES.join(", ")}.`);
|
|
1171
|
+
return void 0;
|
|
1172
|
+
}
|
|
1173
|
+
return value;
|
|
1174
|
+
}
|
|
1175
|
+
function parseMemoryPool(value, issues) {
|
|
1176
|
+
if (!Array.isArray(value)) {
|
|
1177
|
+
pushIssue(issues, "memoryPool", "Expected an array of fixture entries.");
|
|
1178
|
+
return void 0;
|
|
1179
|
+
}
|
|
1180
|
+
return value.flatMap((entry, index) => {
|
|
1181
|
+
const parsed = parseFixtureEntry(entry, index, issues);
|
|
1182
|
+
return parsed ? [parsed] : [];
|
|
1183
|
+
});
|
|
1184
|
+
}
|
|
1185
|
+
function parseProcedurePool(value, issues) {
|
|
1186
|
+
if (value === void 0) {
|
|
1187
|
+
return void 0;
|
|
1188
|
+
}
|
|
1189
|
+
if (!Array.isArray(value)) {
|
|
1190
|
+
pushIssue(issues, "procedurePool", "Expected an array of fixture procedures.");
|
|
1191
|
+
return void 0;
|
|
1192
|
+
}
|
|
1193
|
+
return value.flatMap((procedure, index) => {
|
|
1194
|
+
const parsed = parseFixtureProcedure(procedure, index, issues);
|
|
1195
|
+
return parsed ? [parsed] : [];
|
|
1196
|
+
});
|
|
1197
|
+
}
|
|
1198
|
+
function mapSandboxRequestDto(dto) {
|
|
1199
|
+
if (dto === void 0) {
|
|
1200
|
+
return void 0;
|
|
1201
|
+
}
|
|
1202
|
+
return {
|
|
1203
|
+
root: dto.root,
|
|
1204
|
+
preserve: dto.preserve,
|
|
1205
|
+
corpusSeed: dto.corpusSeed,
|
|
1206
|
+
ablationArm: dto.ablationArm,
|
|
1207
|
+
now: dto.now,
|
|
1208
|
+
profileSnapshot: dto.profileSnapshot
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
function mapFixtureEntryDto(dto) {
|
|
1212
|
+
return {
|
|
1213
|
+
id: dto.id,
|
|
1214
|
+
type: dto.type,
|
|
1215
|
+
subject: dto.subject,
|
|
1216
|
+
content: dto.content,
|
|
1217
|
+
importance: dto.importance,
|
|
1218
|
+
expiry: dto.expiry,
|
|
1219
|
+
tags: dto.tags,
|
|
1220
|
+
source_file: dto.source_file,
|
|
1221
|
+
source_context: dto.source_context,
|
|
1222
|
+
created_at: dto.created_at,
|
|
1223
|
+
updated_at: dto.updated_at,
|
|
1224
|
+
superseded_by: dto.superseded_by,
|
|
1225
|
+
claim_key: dto.claim_key,
|
|
1226
|
+
claim_key_status: dto.claim_key_status,
|
|
1227
|
+
claim_key_source: dto.claim_key_source,
|
|
1228
|
+
claim_support_source_kind: dto.claim_support_source_kind,
|
|
1229
|
+
claim_support_locator: dto.claim_support_locator,
|
|
1230
|
+
claim_support_observed_at: dto.claim_support_observed_at,
|
|
1231
|
+
claim_support_mode: dto.claim_support_mode,
|
|
1232
|
+
valid_from: dto.valid_from,
|
|
1233
|
+
valid_to: dto.valid_to,
|
|
1234
|
+
supersession_kind: dto.supersession_kind,
|
|
1235
|
+
supersession_reason: dto.supersession_reason,
|
|
1236
|
+
directive_polarity: dto.directive_polarity,
|
|
1237
|
+
directive_trigger: dto.directive_trigger
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
function mapFixtureProcedureDto(dto) {
|
|
1241
|
+
return {
|
|
1242
|
+
id: dto.id,
|
|
1243
|
+
procedure_key: dto.procedure_key,
|
|
1244
|
+
title: dto.title,
|
|
1245
|
+
goal: dto.goal,
|
|
1246
|
+
when_to_use: dto.when_to_use,
|
|
1247
|
+
when_not_to_use: dto.when_not_to_use,
|
|
1248
|
+
prerequisites: dto.prerequisites,
|
|
1249
|
+
steps: dto.steps,
|
|
1250
|
+
verification: dto.verification,
|
|
1251
|
+
failure_modes: dto.failure_modes,
|
|
1252
|
+
sources: dto.sources,
|
|
1253
|
+
source_file: dto.source_file,
|
|
1254
|
+
valid_from: dto.valid_from,
|
|
1255
|
+
valid_to: dto.valid_to,
|
|
1256
|
+
supersession_kind: dto.supersession_kind,
|
|
1257
|
+
supersession_reason: dto.supersession_reason,
|
|
1258
|
+
superseded_by: dto.superseded_by,
|
|
1259
|
+
created_at: dto.created_at,
|
|
1260
|
+
updated_at: dto.updated_at
|
|
1261
|
+
};
|
|
1262
|
+
}
|
|
1263
|
+
function parseOptionalStringArray(value, path2, issues) {
|
|
1264
|
+
if (value === void 0) {
|
|
1265
|
+
return void 0;
|
|
1266
|
+
}
|
|
1267
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
1268
|
+
pushIssue(issues, path2, "Expected an array of strings.");
|
|
1269
|
+
return void 0;
|
|
1270
|
+
}
|
|
1271
|
+
return value.map((item) => item.trim()).filter((item) => item.length > 0);
|
|
1272
|
+
}
|
|
1273
|
+
function parseOptionalThreshold(value, path2, issues) {
|
|
1274
|
+
if (value === void 0) {
|
|
1275
|
+
return void 0;
|
|
1276
|
+
}
|
|
1277
|
+
if (typeof value !== "number" || Number.isNaN(value) || value < 0 || value > 1) {
|
|
1278
|
+
pushIssue(issues, path2, "Expected a number from 0 to 1.");
|
|
1279
|
+
return void 0;
|
|
1280
|
+
}
|
|
1281
|
+
return value;
|
|
1282
|
+
}
|
|
1283
|
+
function parseRequiredString(value, path2, issues) {
|
|
1284
|
+
if (typeof value !== "string") {
|
|
1285
|
+
pushIssue(issues, path2, "Expected a string.");
|
|
1286
|
+
return void 0;
|
|
1287
|
+
}
|
|
1288
|
+
return value;
|
|
1289
|
+
}
|
|
1290
|
+
function parseRecentTurnRole(value, path2, issues) {
|
|
1291
|
+
if (typeof value !== "string") {
|
|
1292
|
+
pushIssue(issues, path2, 'Expected "user" or "assistant".');
|
|
1293
|
+
return void 0;
|
|
1294
|
+
}
|
|
1295
|
+
const normalized = value.trim();
|
|
1296
|
+
if (normalized !== "user" && normalized !== "assistant") {
|
|
1297
|
+
pushIssue(issues, path2, 'Expected "user" or "assistant".');
|
|
1298
|
+
return void 0;
|
|
1299
|
+
}
|
|
1300
|
+
return normalized;
|
|
1301
|
+
}
|
|
1302
|
+
function parseFixtureEntry(value, index, issues) {
|
|
1303
|
+
const basePath = `memoryPool[${index}]`;
|
|
1304
|
+
const fixture = parseObject(value, basePath, issues);
|
|
1305
|
+
if (fixture === void 0) {
|
|
1306
|
+
return void 0;
|
|
1307
|
+
}
|
|
1308
|
+
pushUnexpectedFields(fixture, FIXTURE_ENTRY_KEYS, basePath, issues);
|
|
1309
|
+
const type = parseDurableKind(fixture.type, `${basePath}.type`, issues);
|
|
1310
|
+
const subject = parseRequiredTrimmedString(fixture.subject, `${basePath}.subject`, issues);
|
|
1311
|
+
const content = parseRequiredTrimmedString(fixture.content, `${basePath}.content`, issues);
|
|
1312
|
+
if (type === void 0 || subject === void 0 || content === void 0) {
|
|
1313
|
+
return void 0;
|
|
1314
|
+
}
|
|
1315
|
+
return {
|
|
1316
|
+
id: parseOptionalTrimmedString(fixture.id, `${basePath}.id`, issues),
|
|
1317
|
+
type,
|
|
1318
|
+
subject,
|
|
1319
|
+
content,
|
|
1320
|
+
importance: parseOptionalIntegerInRange(fixture.importance, `${basePath}.importance`, issues, {
|
|
1321
|
+
min: 1,
|
|
1322
|
+
max: 10
|
|
1323
|
+
}),
|
|
1324
|
+
expiry: parseOptionalExpiry(fixture.expiry, `${basePath}.expiry`, issues),
|
|
1325
|
+
tags: parseOptionalStringArray(fixture.tags, `${basePath}.tags`, issues),
|
|
1326
|
+
source_file: parseOptionalTrimmedString(fixture.source_file, `${basePath}.source_file`, issues),
|
|
1327
|
+
source_context: parseOptionalTrimmedString(fixture.source_context, `${basePath}.source_context`, issues),
|
|
1328
|
+
created_at: parseOptionalTimestampString(fixture.created_at, `${basePath}.created_at`, issues),
|
|
1329
|
+
updated_at: parseOptionalTimestampString(fixture.updated_at, `${basePath}.updated_at`, issues),
|
|
1330
|
+
superseded_by: parseOptionalTrimmedString(fixture.superseded_by, `${basePath}.superseded_by`, issues),
|
|
1331
|
+
claim_key: parseOptionalTrimmedString(fixture.claim_key, `${basePath}.claim_key`, issues),
|
|
1332
|
+
claim_key_status: parseOptionalClaimKeyStatus(fixture.claim_key_status, `${basePath}.claim_key_status`, issues),
|
|
1333
|
+
claim_key_source: parseOptionalClaimKeySource(fixture.claim_key_source, `${basePath}.claim_key_source`, issues),
|
|
1334
|
+
claim_support_source_kind: parseOptionalTrimmedString(fixture.claim_support_source_kind, `${basePath}.claim_support_source_kind`, issues),
|
|
1335
|
+
claim_support_locator: parseOptionalTrimmedString(fixture.claim_support_locator, `${basePath}.claim_support_locator`, issues),
|
|
1336
|
+
claim_support_observed_at: parseOptionalTimestampString(fixture.claim_support_observed_at, `${basePath}.claim_support_observed_at`, issues),
|
|
1337
|
+
claim_support_mode: parseOptionalClaimSupportMode(fixture.claim_support_mode, `${basePath}.claim_support_mode`, issues),
|
|
1338
|
+
valid_from: parseOptionalTimestampString(fixture.valid_from, `${basePath}.valid_from`, issues),
|
|
1339
|
+
valid_to: parseOptionalTimestampString(fixture.valid_to, `${basePath}.valid_to`, issues),
|
|
1340
|
+
supersession_kind: parseOptionalTrimmedString(fixture.supersession_kind, `${basePath}.supersession_kind`, issues),
|
|
1341
|
+
supersession_reason: parseOptionalTrimmedString(fixture.supersession_reason, `${basePath}.supersession_reason`, issues),
|
|
1342
|
+
directive_polarity: parseOptionalDirectivePolarity(fixture.directive_polarity, `${basePath}.directive_polarity`, issues),
|
|
1343
|
+
directive_trigger: parseOptionalDirectiveTrigger(fixture.directive_trigger, `${basePath}.directive_trigger`, issues)
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
function parseOptionalAblationArm(value, path2, issues) {
|
|
1347
|
+
if (value === void 0) {
|
|
1348
|
+
return void 0;
|
|
1349
|
+
}
|
|
1350
|
+
if (typeof value !== "string" || !ABLATION_ARMS2.includes(value)) {
|
|
1351
|
+
pushIssue(issues, path2, `Expected one of: ${ABLATION_ARMS2.join(", ")}.`);
|
|
1352
|
+
return void 0;
|
|
1353
|
+
}
|
|
1354
|
+
return value;
|
|
1355
|
+
}
|
|
1356
|
+
function parseProfileSnapshot(value, issues) {
|
|
1357
|
+
if (value === void 0) {
|
|
1358
|
+
return void 0;
|
|
1359
|
+
}
|
|
1360
|
+
const snapshot = parseObject(value, "sandbox.profileSnapshot", issues);
|
|
1361
|
+
if (snapshot === void 0) {
|
|
1362
|
+
return void 0;
|
|
1363
|
+
}
|
|
1364
|
+
pushUnexpectedFields(snapshot, PROFILE_SNAPSHOT_KEYS, "sandbox.profileSnapshot", issues);
|
|
1365
|
+
const durableIds = parseRequiredStringArray(snapshot.durableIds, "sandbox.profileSnapshot.durableIds", issues);
|
|
1366
|
+
if (durableIds === void 0) {
|
|
1367
|
+
return void 0;
|
|
1368
|
+
}
|
|
1369
|
+
return {
|
|
1370
|
+
id: parseOptionalTrimmedString(snapshot.id, "sandbox.profileSnapshot.id", issues),
|
|
1371
|
+
durableIds,
|
|
1372
|
+
directiveIds: parseOptionalStringArray(snapshot.directiveIds, "sandbox.profileSnapshot.directiveIds", issues),
|
|
1373
|
+
asOf: parseOptionalTimestampString(snapshot.asOf, "sandbox.profileSnapshot.asOf", issues),
|
|
1374
|
+
runId: parseOptionalTrimmedString(snapshot.runId, "sandbox.profileSnapshot.runId", issues),
|
|
1375
|
+
createdAt: parseOptionalTimestampString(snapshot.createdAt, "sandbox.profileSnapshot.createdAt", issues)
|
|
1376
|
+
};
|
|
1377
|
+
}
|
|
1378
|
+
function parseRequiredStringArray(value, path2, issues) {
|
|
1379
|
+
if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== "string" || item.trim().length === 0)) {
|
|
1380
|
+
pushIssue(issues, path2, "Expected a non-empty array of strings.");
|
|
1381
|
+
return void 0;
|
|
1382
|
+
}
|
|
1383
|
+
return value.map((item) => item.trim());
|
|
1384
|
+
}
|
|
1385
|
+
function parseOptionalDirectivePolarity(value, path2, issues) {
|
|
1386
|
+
if (value === void 0) {
|
|
1387
|
+
return void 0;
|
|
1388
|
+
}
|
|
1389
|
+
if (typeof value !== "string") {
|
|
1390
|
+
pushIssue(issues, path2, "Expected a string.");
|
|
1391
|
+
return void 0;
|
|
1392
|
+
}
|
|
1393
|
+
const polarity = parseDirectivePolarity(value);
|
|
1394
|
+
if (!polarity) {
|
|
1395
|
+
pushIssue(issues, path2, "Expected abstain or proactive.");
|
|
1396
|
+
return void 0;
|
|
1397
|
+
}
|
|
1398
|
+
return polarity;
|
|
1399
|
+
}
|
|
1400
|
+
function parseOptionalDirectiveTrigger(value, path2, issues) {
|
|
1401
|
+
if (value === void 0) {
|
|
1402
|
+
return void 0;
|
|
1403
|
+
}
|
|
1404
|
+
if (typeof value !== "string") {
|
|
1405
|
+
pushIssue(issues, path2, "Expected a string.");
|
|
1406
|
+
return void 0;
|
|
1407
|
+
}
|
|
1408
|
+
const trigger = parseDirectiveTrigger(value);
|
|
1409
|
+
if (!trigger) {
|
|
1410
|
+
pushIssue(issues, path2, "Expected session_start, always, or topic:<term>.");
|
|
1411
|
+
return void 0;
|
|
1412
|
+
}
|
|
1413
|
+
return trigger;
|
|
1414
|
+
}
|
|
1415
|
+
function parseFixtureProcedure(value, index, issues) {
|
|
1416
|
+
const basePath = `procedurePool[${index}]`;
|
|
1417
|
+
const fixture = parseObject(value, basePath, issues);
|
|
1418
|
+
if (fixture === void 0) {
|
|
1419
|
+
return void 0;
|
|
1420
|
+
}
|
|
1421
|
+
pushUnexpectedFields(fixture, FIXTURE_PROCEDURE_KEYS, basePath, issues);
|
|
1422
|
+
const procedureKey = parseRequiredTrimmedString(fixture.procedure_key, `${basePath}.procedure_key`, issues);
|
|
1423
|
+
const title = parseRequiredTrimmedString(fixture.title, `${basePath}.title`, issues);
|
|
1424
|
+
const goal = parseRequiredTrimmedString(fixture.goal, `${basePath}.goal`, issues);
|
|
1425
|
+
const whenToUse = parseOptionalStringArray(fixture.when_to_use, `${basePath}.when_to_use`, issues);
|
|
1426
|
+
const whenNotToUse = parseOptionalStringArray(fixture.when_not_to_use, `${basePath}.when_not_to_use`, issues);
|
|
1427
|
+
const prerequisites = parseOptionalStringArray(fixture.prerequisites, `${basePath}.prerequisites`, issues);
|
|
1428
|
+
const verification = parseOptionalStringArray(fixture.verification, `${basePath}.verification`, issues);
|
|
1429
|
+
const failureModes = parseOptionalStringArray(fixture.failure_modes, `${basePath}.failure_modes`, issues);
|
|
1430
|
+
if (procedureKey === void 0 || title === void 0 || goal === void 0) {
|
|
1431
|
+
return void 0;
|
|
1432
|
+
}
|
|
1433
|
+
try {
|
|
1434
|
+
const normalized = normalizeProcedureDefinition(
|
|
1435
|
+
{
|
|
1436
|
+
procedure_key: procedureKey,
|
|
1437
|
+
title,
|
|
1438
|
+
goal,
|
|
1439
|
+
when_to_use: whenToUse ?? [],
|
|
1440
|
+
when_not_to_use: whenNotToUse ?? [],
|
|
1441
|
+
prerequisites: prerequisites ?? [],
|
|
1442
|
+
steps: fixture.steps,
|
|
1443
|
+
verification: verification ?? [],
|
|
1444
|
+
failure_modes: failureModes ?? [],
|
|
1445
|
+
sources: fixture.sources ?? [{ kind: "manual", label: "recall eval fixture" }]
|
|
1446
|
+
},
|
|
1447
|
+
basePath
|
|
1448
|
+
);
|
|
1449
|
+
return {
|
|
1450
|
+
id: parseOptionalTrimmedString(fixture.id, `${basePath}.id`, issues),
|
|
1451
|
+
procedure_key: normalized.procedure_key,
|
|
1452
|
+
title: normalized.title,
|
|
1453
|
+
goal: normalized.goal,
|
|
1454
|
+
when_to_use: normalized.when_to_use,
|
|
1455
|
+
when_not_to_use: normalized.when_not_to_use,
|
|
1456
|
+
prerequisites: normalized.prerequisites,
|
|
1457
|
+
steps: normalized.steps,
|
|
1458
|
+
verification: normalized.verification,
|
|
1459
|
+
failure_modes: normalized.failure_modes,
|
|
1460
|
+
sources: normalized.sources,
|
|
1461
|
+
source_file: parseOptionalTrimmedString(fixture.source_file, `${basePath}.source_file`, issues),
|
|
1462
|
+
valid_from: parseOptionalTimestampString(fixture.valid_from, `${basePath}.valid_from`, issues),
|
|
1463
|
+
valid_to: parseOptionalTimestampString(fixture.valid_to, `${basePath}.valid_to`, issues),
|
|
1464
|
+
supersession_kind: parseOptionalTrimmedString(fixture.supersession_kind, `${basePath}.supersession_kind`, issues),
|
|
1465
|
+
supersession_reason: parseOptionalTrimmedString(fixture.supersession_reason, `${basePath}.supersession_reason`, issues),
|
|
1466
|
+
superseded_by: parseOptionalTrimmedString(fixture.superseded_by, `${basePath}.superseded_by`, issues),
|
|
1467
|
+
created_at: parseOptionalTimestampString(fixture.created_at, `${basePath}.created_at`, issues),
|
|
1468
|
+
updated_at: parseOptionalTimestampString(fixture.updated_at, `${basePath}.updated_at`, issues)
|
|
1469
|
+
};
|
|
1470
|
+
} catch (error) {
|
|
1471
|
+
pushIssue(issues, basePath, error instanceof Error ? error.message : String(error));
|
|
1472
|
+
return void 0;
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
function parseDurableKind(value, path2, issues) {
|
|
1476
|
+
if (typeof value !== "string" || !DURABLE_KINDS.includes(value)) {
|
|
1477
|
+
pushIssue(issues, path2, `Expected one of: ${DURABLE_KINDS.join(", ")}.`);
|
|
1478
|
+
return void 0;
|
|
1479
|
+
}
|
|
1480
|
+
return value;
|
|
1481
|
+
}
|
|
1482
|
+
function parseOptionalExpiry(value, path2, issues) {
|
|
1483
|
+
if (value === void 0) {
|
|
1484
|
+
return void 0;
|
|
1485
|
+
}
|
|
1486
|
+
if (typeof value !== "string" || !EXPIRY_LEVELS.includes(value)) {
|
|
1487
|
+
pushIssue(issues, path2, `Expected one of: ${EXPIRY_LEVELS.join(", ")}.`);
|
|
1488
|
+
return void 0;
|
|
1489
|
+
}
|
|
1490
|
+
return value;
|
|
1491
|
+
}
|
|
1492
|
+
function parseOptionalClaimKeyStatus(value, path2, issues) {
|
|
1493
|
+
if (value === void 0) {
|
|
1494
|
+
return void 0;
|
|
1495
|
+
}
|
|
1496
|
+
if (typeof value !== "string" || !CLAIM_KEY_STATUSES.includes(value)) {
|
|
1497
|
+
pushIssue(issues, path2, `Expected one of: ${CLAIM_KEY_STATUSES.join(", ")}.`);
|
|
1498
|
+
return void 0;
|
|
1499
|
+
}
|
|
1500
|
+
return value;
|
|
1501
|
+
}
|
|
1502
|
+
function parseOptionalClaimKeySource(value, path2, issues) {
|
|
1503
|
+
if (value === void 0) {
|
|
1504
|
+
return void 0;
|
|
1505
|
+
}
|
|
1506
|
+
if (typeof value !== "string" || !CLAIM_KEY_SOURCES.includes(value)) {
|
|
1507
|
+
pushIssue(issues, path2, `Expected one of: ${CLAIM_KEY_SOURCES.join(", ")}.`);
|
|
1508
|
+
return void 0;
|
|
1509
|
+
}
|
|
1510
|
+
return value;
|
|
1511
|
+
}
|
|
1512
|
+
function parseOptionalClaimSupportMode(value, path2, issues) {
|
|
1513
|
+
if (value === void 0) {
|
|
1514
|
+
return void 0;
|
|
1515
|
+
}
|
|
1516
|
+
if (typeof value !== "string" || !CLAIM_SUPPORT_MODES.includes(value)) {
|
|
1517
|
+
pushIssue(issues, path2, `Expected one of: ${CLAIM_SUPPORT_MODES.join(", ")}.`);
|
|
1518
|
+
return void 0;
|
|
1519
|
+
}
|
|
1520
|
+
return value;
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
// src/adapters/api/validation/before-turn-eval-request.ts
|
|
1524
|
+
var ROOT_REQUEST_KEYS = /* @__PURE__ */ new Set(["caseId", "description", "sandbox", "memoryPool", "procedurePool", "beforeTurnInput", "options"]);
|
|
1525
|
+
var BEFORE_TURN_INPUT_KEYS = /* @__PURE__ */ new Set(["sessionKey", "currentTurnText", "recentTurns", "trigger", "policy"]);
|
|
1526
|
+
var BEFORE_TURN_RECENT_TURN_KEYS = /* @__PURE__ */ new Set(["role", "text"]);
|
|
1527
|
+
var BEFORE_TURN_POLICY_KEYS = /* @__PURE__ */ new Set([
|
|
1528
|
+
"enableDurableRecall",
|
|
1529
|
+
"enableProcedureSuggestion",
|
|
1530
|
+
"maxRecentTurns",
|
|
1531
|
+
"maxQueryChars",
|
|
1532
|
+
"maxDurableEntries",
|
|
1533
|
+
"maxHighConfidenceDurableEntries",
|
|
1534
|
+
"maxProcedureCandidates",
|
|
1535
|
+
"recallThreshold",
|
|
1536
|
+
"highConfidenceRecallThreshold",
|
|
1537
|
+
"procedureThreshold",
|
|
1538
|
+
"skipTrivialTurns",
|
|
1539
|
+
"requireTurnSignal"
|
|
1540
|
+
]);
|
|
1541
|
+
var OPTIONS_KEYS = /* @__PURE__ */ new Set(["includeDiagnostics", "includeRenderedPatch", "includeTimings", "includeDebugArtifact", "topKCandidates"]);
|
|
1542
|
+
var BeforeTurnEvalRequestValidationError = class extends Error {
|
|
1543
|
+
/** Parseable case identifier echoed for invalid request correlation when available. */
|
|
1544
|
+
caseId;
|
|
1545
|
+
/** Structured list of request validation issues. */
|
|
1546
|
+
issues;
|
|
1547
|
+
/**
|
|
1548
|
+
* Creates a request validation error with stable issue details.
|
|
1549
|
+
*
|
|
1550
|
+
* @param issues - Structured validation issues collected during parsing.
|
|
1551
|
+
* @param caseId - Parseable request case identifier when available.
|
|
1552
|
+
*/
|
|
1553
|
+
constructor(issues, caseId) {
|
|
1554
|
+
super("Invalid before-turn eval request.");
|
|
1555
|
+
this.name = "BeforeTurnEvalRequestValidationError";
|
|
1556
|
+
this.issues = issues;
|
|
1557
|
+
this.caseId = caseId;
|
|
1558
|
+
}
|
|
1559
|
+
};
|
|
1560
|
+
function parseBeforeTurnEvalCaseRequest(input) {
|
|
1561
|
+
const caseId = extractParseableCaseId(input);
|
|
1562
|
+
if (!isRecord(input)) {
|
|
1563
|
+
throw new BeforeTurnEvalRequestValidationError(
|
|
1564
|
+
[
|
|
1565
|
+
{
|
|
1566
|
+
path: "$",
|
|
1567
|
+
message: "Request body must be a JSON object."
|
|
1568
|
+
}
|
|
1569
|
+
],
|
|
1570
|
+
caseId
|
|
1571
|
+
);
|
|
1572
|
+
}
|
|
1573
|
+
const issues = [];
|
|
1574
|
+
pushUnexpectedFields(input, ROOT_REQUEST_KEYS, "", issues);
|
|
1575
|
+
const parsedCaseId = parseRequiredTrimmedString(input.caseId, "caseId", issues);
|
|
1576
|
+
const description = parseOptionalTrimmedString(input.description, "description", issues);
|
|
1577
|
+
const sandbox = parseSandbox(input.sandbox, issues);
|
|
1578
|
+
const memoryPool = parseMemoryPool(input.memoryPool, issues);
|
|
1579
|
+
const procedurePool = parseProcedurePool(input.procedurePool, issues);
|
|
1580
|
+
const beforeTurnInput = parseBeforeTurnInput(input.beforeTurnInput, issues);
|
|
1581
|
+
const options = parseOptions(input.options, issues);
|
|
1582
|
+
if (issues.length > 0 || parsedCaseId === void 0 || memoryPool === void 0 || beforeTurnInput === void 0) {
|
|
1583
|
+
throw new BeforeTurnEvalRequestValidationError(issues, caseId);
|
|
1584
|
+
}
|
|
1585
|
+
return {
|
|
1586
|
+
caseId: parsedCaseId,
|
|
1587
|
+
description,
|
|
1588
|
+
sandbox,
|
|
1589
|
+
memoryPool,
|
|
1590
|
+
procedurePool,
|
|
1591
|
+
beforeTurnInput,
|
|
1592
|
+
options
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
function mapBeforeTurnEvalCaseRequestDto(dto) {
|
|
1596
|
+
return {
|
|
1597
|
+
caseId: dto.caseId,
|
|
1598
|
+
description: dto.description,
|
|
1599
|
+
sandbox: mapSandboxRequestDto(dto.sandbox),
|
|
1600
|
+
memoryPool: dto.memoryPool.map((entry) => mapFixtureEntryDto(entry)),
|
|
1601
|
+
procedurePool: dto.procedurePool?.map((procedure) => mapFixtureProcedureDto(procedure)),
|
|
1602
|
+
beforeTurnInput: mapBeforeTurnInputDto(dto.beforeTurnInput),
|
|
1603
|
+
options: mapCaseOptionsDto(dto.options)
|
|
1604
|
+
};
|
|
1605
|
+
}
|
|
1606
|
+
function parseBeforeTurnInput(value, issues) {
|
|
1607
|
+
const beforeTurnInput = parseObject(value, "beforeTurnInput", issues);
|
|
1608
|
+
if (beforeTurnInput === void 0) {
|
|
1609
|
+
return void 0;
|
|
1610
|
+
}
|
|
1611
|
+
pushUnexpectedFields(beforeTurnInput, BEFORE_TURN_INPUT_KEYS, "beforeTurnInput", issues);
|
|
1612
|
+
const currentTurnText = parseRequiredString(beforeTurnInput.currentTurnText, "beforeTurnInput.currentTurnText", issues);
|
|
1613
|
+
if (currentTurnText === void 0) {
|
|
1614
|
+
return void 0;
|
|
1615
|
+
}
|
|
1616
|
+
return {
|
|
1617
|
+
sessionKey: parseOptionalTrimmedString(beforeTurnInput.sessionKey, "beforeTurnInput.sessionKey", issues),
|
|
1618
|
+
currentTurnText,
|
|
1619
|
+
recentTurns: parseRecentTurns(beforeTurnInput.recentTurns, issues),
|
|
1620
|
+
trigger: parseOptionalTrimmedString(beforeTurnInput.trigger, "beforeTurnInput.trigger", issues),
|
|
1621
|
+
policy: parseBeforeTurnPolicy(beforeTurnInput.policy, issues)
|
|
1622
|
+
};
|
|
1623
|
+
}
|
|
1624
|
+
function parseRecentTurns(value, issues) {
|
|
1625
|
+
if (value === void 0) {
|
|
1626
|
+
return void 0;
|
|
1627
|
+
}
|
|
1628
|
+
if (!Array.isArray(value)) {
|
|
1629
|
+
issues.push({
|
|
1630
|
+
path: "beforeTurnInput.recentTurns",
|
|
1631
|
+
message: "Expected an array of recent turns."
|
|
1632
|
+
});
|
|
1633
|
+
return void 0;
|
|
1634
|
+
}
|
|
1635
|
+
return value.flatMap((turn, index) => {
|
|
1636
|
+
const basePath = `beforeTurnInput.recentTurns[${index}]`;
|
|
1637
|
+
const record = parseObject(turn, basePath, issues);
|
|
1638
|
+
if (record === void 0) {
|
|
1639
|
+
return [];
|
|
1640
|
+
}
|
|
1641
|
+
pushUnexpectedFields(record, BEFORE_TURN_RECENT_TURN_KEYS, basePath, issues);
|
|
1642
|
+
const role = parseRecentTurnRole(record.role, `${basePath}.role`, issues);
|
|
1643
|
+
const text = parseRequiredString(record.text, `${basePath}.text`, issues);
|
|
1644
|
+
if (role === void 0 || text === void 0) {
|
|
1645
|
+
return [];
|
|
1646
|
+
}
|
|
1647
|
+
return [{ role, text }];
|
|
1648
|
+
});
|
|
1649
|
+
}
|
|
1650
|
+
function parseBeforeTurnPolicy(value, issues) {
|
|
1651
|
+
if (value === void 0) {
|
|
1652
|
+
return void 0;
|
|
1653
|
+
}
|
|
1654
|
+
const policy = parseObject(value, "beforeTurnInput.policy", issues);
|
|
1655
|
+
if (policy === void 0) {
|
|
1656
|
+
return void 0;
|
|
1657
|
+
}
|
|
1658
|
+
pushUnexpectedFields(policy, BEFORE_TURN_POLICY_KEYS, "beforeTurnInput.policy", issues);
|
|
1659
|
+
return {
|
|
1660
|
+
enableDurableRecall: parseOptionalBoolean(policy.enableDurableRecall, "beforeTurnInput.policy.enableDurableRecall", issues),
|
|
1661
|
+
enableProcedureSuggestion: parseOptionalBoolean(policy.enableProcedureSuggestion, "beforeTurnInput.policy.enableProcedureSuggestion", issues),
|
|
1662
|
+
maxRecentTurns: parseOptionalIntegerInRange(policy.maxRecentTurns, "beforeTurnInput.policy.maxRecentTurns", issues, {
|
|
1663
|
+
min: 0
|
|
1664
|
+
}),
|
|
1665
|
+
maxQueryChars: parseOptionalIntegerInRange(policy.maxQueryChars, "beforeTurnInput.policy.maxQueryChars", issues, {
|
|
1666
|
+
min: 0
|
|
1667
|
+
}),
|
|
1668
|
+
maxDurableEntries: parseOptionalIntegerInRange(policy.maxDurableEntries, "beforeTurnInput.policy.maxDurableEntries", issues, {
|
|
1669
|
+
min: 0
|
|
1670
|
+
}),
|
|
1671
|
+
maxHighConfidenceDurableEntries: parseOptionalIntegerInRange(
|
|
1672
|
+
policy.maxHighConfidenceDurableEntries,
|
|
1673
|
+
"beforeTurnInput.policy.maxHighConfidenceDurableEntries",
|
|
1674
|
+
issues,
|
|
1675
|
+
{
|
|
1676
|
+
min: 0
|
|
1677
|
+
}
|
|
1678
|
+
),
|
|
1679
|
+
maxProcedureCandidates: parseOptionalIntegerInRange(policy.maxProcedureCandidates, "beforeTurnInput.policy.maxProcedureCandidates", issues, {
|
|
1680
|
+
min: 0
|
|
1681
|
+
}),
|
|
1682
|
+
recallThreshold: parseOptionalThreshold(policy.recallThreshold, "beforeTurnInput.policy.recallThreshold", issues),
|
|
1683
|
+
highConfidenceRecallThreshold: parseOptionalThreshold(policy.highConfidenceRecallThreshold, "beforeTurnInput.policy.highConfidenceRecallThreshold", issues),
|
|
1684
|
+
procedureThreshold: parseOptionalThreshold(policy.procedureThreshold, "beforeTurnInput.policy.procedureThreshold", issues),
|
|
1685
|
+
skipTrivialTurns: parseOptionalBoolean(policy.skipTrivialTurns, "beforeTurnInput.policy.skipTrivialTurns", issues),
|
|
1686
|
+
requireTurnSignal: parseOptionalBoolean(policy.requireTurnSignal, "beforeTurnInput.policy.requireTurnSignal", issues)
|
|
1687
|
+
};
|
|
1688
|
+
}
|
|
1689
|
+
function parseOptions(value, issues) {
|
|
1690
|
+
if (value === void 0) {
|
|
1691
|
+
return void 0;
|
|
1692
|
+
}
|
|
1693
|
+
const options = parseObject(value, "options", issues);
|
|
1694
|
+
if (options === void 0) {
|
|
1695
|
+
return void 0;
|
|
1696
|
+
}
|
|
1697
|
+
pushUnexpectedFields(options, OPTIONS_KEYS, "options", issues);
|
|
1698
|
+
return {
|
|
1699
|
+
includeDiagnostics: parseOptionalBoolean(options.includeDiagnostics, "options.includeDiagnostics", issues),
|
|
1700
|
+
includeRenderedPatch: parseOptionalBoolean(options.includeRenderedPatch, "options.includeRenderedPatch", issues),
|
|
1701
|
+
includeTimings: parseOptionalBoolean(options.includeTimings, "options.includeTimings", issues),
|
|
1702
|
+
includeDebugArtifact: parseOptionalBoolean(options.includeDebugArtifact, "options.includeDebugArtifact", issues),
|
|
1703
|
+
topKCandidates: parseOptionalIntegerInRange(options.topKCandidates, "options.topKCandidates", issues, {
|
|
1704
|
+
min: 1,
|
|
1705
|
+
max: BEFORE_TURN_DEBUG_ARTIFACT_MAX_TOP_K
|
|
1706
|
+
})
|
|
1707
|
+
};
|
|
1708
|
+
}
|
|
1709
|
+
function mapBeforeTurnInputDto(dto) {
|
|
1710
|
+
return {
|
|
1711
|
+
sessionKey: dto.sessionKey,
|
|
1712
|
+
currentTurnText: dto.currentTurnText,
|
|
1713
|
+
recentTurns: dto.recentTurns?.map((turn) => ({
|
|
1714
|
+
role: turn.role,
|
|
1715
|
+
text: turn.text
|
|
1716
|
+
})),
|
|
1717
|
+
trigger: dto.trigger,
|
|
1718
|
+
policy: mapBeforeTurnPolicyDto(dto.policy)
|
|
1719
|
+
};
|
|
1720
|
+
}
|
|
1721
|
+
function mapBeforeTurnPolicyDto(dto) {
|
|
1722
|
+
if (dto === void 0) {
|
|
1723
|
+
return void 0;
|
|
1724
|
+
}
|
|
1725
|
+
return {
|
|
1726
|
+
enableDurableRecall: dto.enableDurableRecall,
|
|
1727
|
+
enableProcedureSuggestion: dto.enableProcedureSuggestion,
|
|
1728
|
+
maxRecentTurns: dto.maxRecentTurns,
|
|
1729
|
+
maxQueryChars: dto.maxQueryChars,
|
|
1730
|
+
maxDurableEntries: dto.maxDurableEntries,
|
|
1731
|
+
maxHighConfidenceDurableEntries: dto.maxHighConfidenceDurableEntries,
|
|
1732
|
+
maxProcedureCandidates: dto.maxProcedureCandidates,
|
|
1733
|
+
recallThreshold: dto.recallThreshold,
|
|
1734
|
+
highConfidenceRecallThreshold: dto.highConfidenceRecallThreshold,
|
|
1735
|
+
procedureThreshold: dto.procedureThreshold,
|
|
1736
|
+
skipTrivialTurns: dto.skipTrivialTurns,
|
|
1737
|
+
requireTurnSignal: dto.requireTurnSignal
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
function mapCaseOptionsDto(dto) {
|
|
1741
|
+
if (dto === void 0) {
|
|
1742
|
+
return void 0;
|
|
1743
|
+
}
|
|
1744
|
+
return {
|
|
1745
|
+
includeDiagnostics: dto.includeDiagnostics,
|
|
1746
|
+
includeRenderedPatch: dto.includeRenderedPatch,
|
|
1747
|
+
includeTimings: dto.includeTimings,
|
|
1748
|
+
includeDebugArtifact: dto.includeDebugArtifact,
|
|
1749
|
+
topKCandidates: dto.topKCandidates
|
|
1750
|
+
};
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
// src/adapters/api/routes/internal-before-turn-eval.ts
|
|
1754
|
+
var INTERNAL_BEFORE_TURN_EVAL_ROUTE_PATH = "/internal/evals/before-turn/run";
|
|
1755
|
+
function createInternalBeforeTurnEvalRoute(optionsOrRunner = {}) {
|
|
1756
|
+
const options = typeof optionsOrRunner === "function" ? { runner: optionsOrRunner } : optionsOrRunner;
|
|
1757
|
+
const crossEncoder = options.crossEncoder;
|
|
1758
|
+
const runner = options.runner ?? ((request) => runBeforeTurnEvalCase(request, { crossEncoder }));
|
|
1759
|
+
return {
|
|
1760
|
+
method: "POST",
|
|
1761
|
+
path: INTERNAL_BEFORE_TURN_EVAL_ROUTE_PATH,
|
|
1762
|
+
handler: async (request) => {
|
|
1763
|
+
let validatedRequest;
|
|
1764
|
+
try {
|
|
1765
|
+
validatedRequest = await parseValidatedRequest(request);
|
|
1766
|
+
const result = await runner(validatedRequest);
|
|
1767
|
+
return jsonResponse(result, 200);
|
|
1768
|
+
} catch (error) {
|
|
1769
|
+
if (error instanceof BeforeTurnEvalRequestValidationError) {
|
|
1770
|
+
return jsonResponse(
|
|
1771
|
+
{
|
|
1772
|
+
status: "error",
|
|
1773
|
+
caseId: error.caseId,
|
|
1774
|
+
error: {
|
|
1775
|
+
code: "invalid_request",
|
|
1776
|
+
message: error.message,
|
|
1777
|
+
details: error.issues
|
|
1778
|
+
}
|
|
1779
|
+
},
|
|
1780
|
+
400
|
|
1781
|
+
);
|
|
1782
|
+
}
|
|
1783
|
+
return jsonResponse(
|
|
1784
|
+
{
|
|
1785
|
+
status: "error",
|
|
1786
|
+
caseId: validatedRequest?.caseId,
|
|
1787
|
+
error: {
|
|
1788
|
+
code: "internal_error",
|
|
1789
|
+
message: "Internal before-turn eval adapter error."
|
|
1790
|
+
}
|
|
1791
|
+
},
|
|
1792
|
+
500
|
|
1793
|
+
);
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
};
|
|
1797
|
+
}
|
|
1798
|
+
var parseJsonBody = async (request) => {
|
|
1799
|
+
try {
|
|
1800
|
+
return await request.json();
|
|
1801
|
+
} catch {
|
|
1802
|
+
throw new BeforeTurnEvalRequestValidationError([
|
|
1803
|
+
{
|
|
1804
|
+
path: "$",
|
|
1805
|
+
message: "Request body must be valid JSON."
|
|
1806
|
+
}
|
|
1807
|
+
]);
|
|
1808
|
+
}
|
|
1809
|
+
};
|
|
1810
|
+
var parseValidatedRequest = async (request) => {
|
|
1811
|
+
const payload = await parseJsonBody(request);
|
|
1812
|
+
const requestDto = parseBeforeTurnEvalCaseRequest(payload);
|
|
1813
|
+
return mapBeforeTurnEvalCaseRequestDto(requestDto);
|
|
1814
|
+
};
|
|
1815
|
+
var jsonResponse = (body, status) => new Response(JSON.stringify(body), {
|
|
1816
|
+
status,
|
|
1817
|
+
headers: {
|
|
1818
|
+
"content-type": "application/json; charset=utf-8"
|
|
1819
|
+
}
|
|
1820
|
+
});
|
|
1821
|
+
|
|
1822
|
+
// src/app/evals/dreaming-efficiency/normalize-response.ts
|
|
1823
|
+
function buildDreamingEfficiencyEvalSuccessResponse(params) {
|
|
1824
|
+
return {
|
|
1825
|
+
status: "ok",
|
|
1826
|
+
caseId: params.request.caseId,
|
|
1827
|
+
efficiency: params.efficiency,
|
|
1828
|
+
profileInjectionTokenEstimate: params.profileInjectionTokenEstimate,
|
|
1829
|
+
...params.storeOnlyEquivalentTokenEstimate !== void 0 ? { storeOnlyEquivalentTokenEstimate: params.storeOnlyEquivalentTokenEstimate } : {},
|
|
1830
|
+
...params.timings ? { timings: params.timings } : {},
|
|
1831
|
+
sandbox: buildSandboxResult2(params.sandbox)
|
|
1832
|
+
};
|
|
1833
|
+
}
|
|
1834
|
+
function buildDreamingEfficiencyEvalErrorResponse(params) {
|
|
1835
|
+
return {
|
|
1836
|
+
status: "error",
|
|
1837
|
+
caseId: params.request.caseId,
|
|
1838
|
+
error: {
|
|
1839
|
+
code: params.code,
|
|
1840
|
+
message: params.message,
|
|
1841
|
+
...params.details !== void 0 ? { details: params.details } : {}
|
|
1842
|
+
},
|
|
1843
|
+
...params.timings ? { timings: params.timings } : {},
|
|
1844
|
+
...params.sandbox ? { sandbox: buildSandboxResult2(params.sandbox) } : {}
|
|
1845
|
+
};
|
|
1846
|
+
}
|
|
1847
|
+
function buildSandboxResult2(sandbox) {
|
|
1848
|
+
return {
|
|
1849
|
+
root: sandbox.root,
|
|
1850
|
+
dbPath: sandbox.dbPath,
|
|
1851
|
+
preserved: sandbox.preserved,
|
|
1852
|
+
...sandbox.snapshot ? { snapshot: sandbox.snapshot } : {}
|
|
1853
|
+
};
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
// src/app/evals/dreaming-efficiency/run-dreaming-efficiency-eval-case.ts
|
|
1857
|
+
async function runDreamingEfficiencyEvalCase(request) {
|
|
1858
|
+
const startedAt = Date.now();
|
|
1859
|
+
const provisionedAt = new Date(startedAt).toISOString();
|
|
1860
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
1861
|
+
const shouldProvisionProfile = shouldProvisionProfileSnapshot(ablation);
|
|
1862
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
1863
|
+
let sandbox;
|
|
1864
|
+
let timings;
|
|
1865
|
+
let dreamRunId;
|
|
1866
|
+
try {
|
|
1867
|
+
const sandboxStartedAt = Date.now();
|
|
1868
|
+
try {
|
|
1869
|
+
sandbox = await setupRecallEvalSandbox(request.sandbox);
|
|
1870
|
+
timings = { sandboxSetupMs: elapsedMs2(sandboxStartedAt) };
|
|
1871
|
+
} catch (error) {
|
|
1872
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1873
|
+
request,
|
|
1874
|
+
code: "sandbox_setup_failed",
|
|
1875
|
+
message: "Failed to create isolated dreaming-efficiency eval sandbox.",
|
|
1876
|
+
details: toErrorDetails2(error),
|
|
1877
|
+
timings: request.options?.includeTimings === true ? { totalMs: elapsedMs2(startedAt), sandboxSetupMs: elapsedMs2(sandboxStartedAt) } : void 0
|
|
1878
|
+
});
|
|
1879
|
+
}
|
|
1880
|
+
const provisionStartedAt = Date.now();
|
|
1881
|
+
try {
|
|
1882
|
+
if (request.memoryPool.length > 0) {
|
|
1883
|
+
await provisionEvalSandbox({
|
|
1884
|
+
caseId: request.caseId,
|
|
1885
|
+
sandbox,
|
|
1886
|
+
memoryPool: request.memoryPool,
|
|
1887
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.portOrUnavailable() : void 0,
|
|
1888
|
+
provisionedAt
|
|
1889
|
+
});
|
|
1890
|
+
}
|
|
1891
|
+
const seededRun = await sandbox.dreamRunStore.provisionDreamRun(request.dreamRunFixture);
|
|
1892
|
+
dreamRunId = seededRun.runId;
|
|
1893
|
+
if (shouldProvisionProfile) {
|
|
1894
|
+
await sandbox.provisionProfileSnapshot(
|
|
1895
|
+
{
|
|
1896
|
+
...ablation.profileSnapshot,
|
|
1897
|
+
runId: dreamRunId
|
|
1898
|
+
},
|
|
1899
|
+
provisionedAt
|
|
1900
|
+
);
|
|
1901
|
+
}
|
|
1902
|
+
timings = { ...timings, fixtureProvisionMs: elapsedMs2(provisionStartedAt) };
|
|
1903
|
+
} catch (error) {
|
|
1904
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1905
|
+
request,
|
|
1906
|
+
code: "fixture_provision_failed",
|
|
1907
|
+
message: "Failed to provision dreaming-efficiency eval fixtures into isolated storage.",
|
|
1908
|
+
details: toErrorDetails2(error),
|
|
1909
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt), fixtureProvisionMs: elapsedMs2(provisionStartedAt) } : void 0,
|
|
1910
|
+
sandbox
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
try {
|
|
1914
|
+
const persistedRun = dreamRunId ? await sandbox.dreamRunStore.getDreamRun(dreamRunId) : null;
|
|
1915
|
+
if (!persistedRun?.summaryJson) {
|
|
1916
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1917
|
+
request,
|
|
1918
|
+
code: "efficiency_resolution_failed",
|
|
1919
|
+
message: "Persisted dreaming run summary is required for dreaming-efficiency eval cases.",
|
|
1920
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1921
|
+
sandbox
|
|
1922
|
+
});
|
|
1923
|
+
}
|
|
1924
|
+
const efficiency = deriveDreamEfficiencySummary(persistedRun.summaryJson, persistedRun.estimatedCostUsd);
|
|
1925
|
+
if (!efficiency) {
|
|
1926
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1927
|
+
request,
|
|
1928
|
+
code: "efficiency_resolution_failed",
|
|
1929
|
+
message: "Persisted dreaming run summary must include scan and project counters for efficiency derivation.",
|
|
1930
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1931
|
+
sandbox
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1934
|
+
const directiveCount = request.memoryPool.filter((entry) => entry.type === "directive").length;
|
|
1935
|
+
const storeOnlyEquivalentTokenEstimate = request.memoryPool.length > 0 ? estimateProfileInjectionTokens(request.memoryPool.length, directiveCount) : void 0;
|
|
1936
|
+
timings = {
|
|
1937
|
+
...timings,
|
|
1938
|
+
totalMs: elapsedMs2(startedAt)
|
|
1939
|
+
};
|
|
1940
|
+
return buildDreamingEfficiencyEvalSuccessResponse({
|
|
1941
|
+
request,
|
|
1942
|
+
efficiency,
|
|
1943
|
+
profileInjectionTokenEstimate: efficiency.profileInjectionTokenEstimate,
|
|
1944
|
+
...storeOnlyEquivalentTokenEstimate !== void 0 ? { storeOnlyEquivalentTokenEstimate } : {},
|
|
1945
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
1946
|
+
sandbox
|
|
1947
|
+
});
|
|
1948
|
+
} catch (error) {
|
|
1949
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1950
|
+
request,
|
|
1951
|
+
code: "efficiency_resolution_failed",
|
|
1952
|
+
message: "Failed to resolve dreaming-efficiency telemetry from isolated eval state.",
|
|
1953
|
+
details: toErrorDetails2(error),
|
|
1954
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1955
|
+
sandbox
|
|
1956
|
+
});
|
|
1957
|
+
}
|
|
1958
|
+
} catch (error) {
|
|
1959
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1960
|
+
request,
|
|
1961
|
+
code: "internal_error",
|
|
1962
|
+
message: "Dreaming-efficiency eval execution failed unexpectedly.",
|
|
1963
|
+
details: toErrorDetails2(error),
|
|
1964
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1965
|
+
sandbox
|
|
1966
|
+
});
|
|
1967
|
+
} finally {
|
|
1968
|
+
await sandbox?.cleanup().catch(() => void 0);
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
function toErrorDetails2(error) {
|
|
1972
|
+
if (error instanceof Error) {
|
|
1973
|
+
return { cause: error.message };
|
|
1974
|
+
}
|
|
1975
|
+
return { cause: String(error) };
|
|
1976
|
+
}
|
|
1977
|
+
function elapsedMs2(startedAt) {
|
|
1978
|
+
return Date.now() - startedAt;
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
// src/adapters/api/validation/parse-dream-completion-summary.ts
|
|
1982
|
+
var DREAM_COMPLETION_SUMMARY_KEYS = /* @__PURE__ */ new Set([
|
|
1983
|
+
"actions_taken",
|
|
1984
|
+
"backupSkipped",
|
|
1985
|
+
"stages_skipped",
|
|
1986
|
+
"durables_skipped",
|
|
1987
|
+
"observations",
|
|
1988
|
+
"recommendations",
|
|
1989
|
+
"scan",
|
|
1990
|
+
"extract",
|
|
1991
|
+
"reconcile",
|
|
1992
|
+
"temporalize",
|
|
1993
|
+
"project",
|
|
1994
|
+
"prune",
|
|
1995
|
+
"efficiency"
|
|
1996
|
+
]);
|
|
1997
|
+
var DREAM_SCAN_KEYS = /* @__PURE__ */ new Set([
|
|
1998
|
+
"episodesSinceLastRun",
|
|
1999
|
+
"ingestFilesSinceLastRun",
|
|
2000
|
+
"durablesCreatedSinceLastRun",
|
|
2001
|
+
"evidenceRefs",
|
|
2002
|
+
"unsynthesizedImportanceSum"
|
|
2003
|
+
]);
|
|
2004
|
+
var DREAM_EVIDENCE_REF_KEYS = /* @__PURE__ */ new Set(["kind", "locator", "observedAt"]);
|
|
2005
|
+
var DREAM_EXTRACT_KEYS = /* @__PURE__ */ new Set([
|
|
2006
|
+
"episodesScanned",
|
|
2007
|
+
"candidatesEmitted",
|
|
2008
|
+
"newCandidates",
|
|
2009
|
+
"refineCandidates",
|
|
2010
|
+
"knownCandidates",
|
|
2011
|
+
"durablesInserted"
|
|
2012
|
+
]);
|
|
2013
|
+
var DREAM_TEMPORALIZE_KEYS = /* @__PURE__ */ new Set(["revisionsIdentified", "revisionsApplied", "revisionsSkipped"]);
|
|
2014
|
+
var DREAM_PROJECT_KEYS = /* @__PURE__ */ new Set(["profileDurableCount", "directiveCount", "snapshotId", "applied"]);
|
|
2015
|
+
var DREAM_PRUNE_KEYS = /* @__PURE__ */ new Set(["durablesScanned", "candidatesIdentified", "candidatesProtected", "candidatesRetirable", "durablesStaled", "dryRun"]);
|
|
2016
|
+
var DREAM_EFFICIENCY_KEYS = /* @__PURE__ */ new Set([
|
|
2017
|
+
"evidenceItemsRead",
|
|
2018
|
+
"synthesizedDurableMutations",
|
|
2019
|
+
"costPerSynthesizedDurableUsd",
|
|
2020
|
+
"profileInjectionTokenEstimate",
|
|
2021
|
+
"recomputeRatio"
|
|
2022
|
+
]);
|
|
2023
|
+
var DURABLE_SKIP_KEYS = /* @__PURE__ */ new Set(["durable_id", "reason"]);
|
|
2024
|
+
var DREAM_STAGE_SKIP_KEYS = /* @__PURE__ */ new Set(["stage", "reason"]);
|
|
2025
|
+
var DREAM_EVIDENCE_KINDS = ["episode", "ingest_log", "durable", "transcript"];
|
|
2026
|
+
var DREAM_STAGE_SKIP_REASONS = ["light_tier"];
|
|
2027
|
+
function parseDreamCompletionSummary(value, path2, issues) {
|
|
2028
|
+
const summary = parseObject(value, path2, issues);
|
|
2029
|
+
if (summary === void 0) {
|
|
2030
|
+
return void 0;
|
|
2031
|
+
}
|
|
2032
|
+
pushUnexpectedFields(summary, DREAM_COMPLETION_SUMMARY_KEYS, path2, issues);
|
|
2033
|
+
const actionsTaken = parseRequiredNonNegativeInteger(summary.actions_taken, `${path2}.actions_taken`, issues);
|
|
2034
|
+
const backupSkipped = parseOptionalBoolean2(summary.backupSkipped, `${path2}.backupSkipped`, issues);
|
|
2035
|
+
const stagesSkipped = parseDreamStageSkips(summary.stages_skipped, `${path2}.stages_skipped`, issues);
|
|
2036
|
+
const durablesSkipped = parseDurablesSkipped(summary.durables_skipped, `${path2}.durables_skipped`, issues);
|
|
2037
|
+
const observations = parseRequiredStringArray2(summary.observations, `${path2}.observations`, issues);
|
|
2038
|
+
const recommendations = parseRequiredStringArray2(summary.recommendations, `${path2}.recommendations`, issues);
|
|
2039
|
+
const scan = parseDreamScanSummary(summary.scan, `${path2}.scan`, issues);
|
|
2040
|
+
const project = parseDreamProjectSummary(summary.project, `${path2}.project`, issues);
|
|
2041
|
+
parseOptionalObject(summary.reconcile, `${path2}.reconcile`, issues);
|
|
2042
|
+
return {
|
|
2043
|
+
actions_taken: actionsTaken ?? 0,
|
|
2044
|
+
...backupSkipped !== void 0 ? { backupSkipped } : {},
|
|
2045
|
+
...stagesSkipped !== void 0 ? { stages_skipped: stagesSkipped } : {},
|
|
2046
|
+
durables_skipped: durablesSkipped ?? [],
|
|
2047
|
+
observations: observations ?? [],
|
|
2048
|
+
recommendations: recommendations ?? [],
|
|
2049
|
+
...scan ? { scan } : {},
|
|
2050
|
+
...parseDreamExtractSummary(summary.extract, `${path2}.extract`, issues) ?? {},
|
|
2051
|
+
...parseDreamTemporalizeSummary(summary.temporalize, `${path2}.temporalize`, issues) ?? {},
|
|
2052
|
+
...project ? { project } : {},
|
|
2053
|
+
...parseDreamPruneSummary(summary.prune, `${path2}.prune`, issues) ?? {},
|
|
2054
|
+
...parseDreamEfficiencySummary(summary.efficiency, `${path2}.efficiency`, issues) ?? {}
|
|
2055
|
+
};
|
|
2056
|
+
}
|
|
2057
|
+
function parseDreamStageSkips(value, path2, issues) {
|
|
2058
|
+
if (value === void 0) {
|
|
2059
|
+
return void 0;
|
|
2060
|
+
}
|
|
2061
|
+
if (!Array.isArray(value)) {
|
|
2062
|
+
pushIssue(issues, path2, "Expected an array.");
|
|
2063
|
+
return void 0;
|
|
2064
|
+
}
|
|
2065
|
+
return value.flatMap((item, index) => {
|
|
2066
|
+
const itemPath = `${path2}[${index}]`;
|
|
2067
|
+
const record = parseObject(item, itemPath, issues);
|
|
2068
|
+
if (record === void 0) {
|
|
2069
|
+
return [];
|
|
2070
|
+
}
|
|
2071
|
+
pushUnexpectedFields(record, DREAM_STAGE_SKIP_KEYS, itemPath, issues);
|
|
2072
|
+
const stage = parseDreamStage(record.stage, `${itemPath}.stage`, issues);
|
|
2073
|
+
const reason = parseDreamStageSkipReason(record.reason, `${itemPath}.reason`, issues);
|
|
2074
|
+
return [
|
|
2075
|
+
{
|
|
2076
|
+
stage: stage ?? "scan",
|
|
2077
|
+
reason: reason ?? "light_tier"
|
|
2078
|
+
}
|
|
2079
|
+
];
|
|
2080
|
+
});
|
|
2081
|
+
}
|
|
2082
|
+
function parseDreamScanSummary(value, path2, issues) {
|
|
2083
|
+
const scan = parseRequiredObject(value, path2, issues);
|
|
2084
|
+
if (scan === void 0) {
|
|
2085
|
+
return void 0;
|
|
2086
|
+
}
|
|
2087
|
+
pushUnexpectedFields(scan, DREAM_SCAN_KEYS, path2, issues);
|
|
2088
|
+
return {
|
|
2089
|
+
episodesSinceLastRun: parseRequiredNonNegativeInteger(scan.episodesSinceLastRun, `${path2}.episodesSinceLastRun`, issues) ?? 0,
|
|
2090
|
+
ingestFilesSinceLastRun: parseRequiredNonNegativeInteger(scan.ingestFilesSinceLastRun, `${path2}.ingestFilesSinceLastRun`, issues) ?? 0,
|
|
2091
|
+
durablesCreatedSinceLastRun: parseRequiredNonNegativeInteger(scan.durablesCreatedSinceLastRun, `${path2}.durablesCreatedSinceLastRun`, issues) ?? 0,
|
|
2092
|
+
evidenceRefs: parseEvidenceRefs(scan.evidenceRefs, `${path2}.evidenceRefs`, issues) ?? [],
|
|
2093
|
+
unsynthesizedImportanceSum: parseRequiredNonNegativeNumber(scan.unsynthesizedImportanceSum, `${path2}.unsynthesizedImportanceSum`, issues) ?? 0
|
|
2094
|
+
};
|
|
2095
|
+
}
|
|
2096
|
+
function parseDreamExtractSummary(value, path2, issues) {
|
|
2097
|
+
if (value === void 0) {
|
|
2098
|
+
return void 0;
|
|
2099
|
+
}
|
|
2100
|
+
const extract = parseObject(value, path2, issues);
|
|
2101
|
+
if (extract === void 0) {
|
|
2102
|
+
return void 0;
|
|
2103
|
+
}
|
|
2104
|
+
pushUnexpectedFields(extract, DREAM_EXTRACT_KEYS, path2, issues);
|
|
2105
|
+
return {
|
|
2106
|
+
extract: {
|
|
2107
|
+
episodesScanned: parseRequiredNonNegativeInteger(extract.episodesScanned, `${path2}.episodesScanned`, issues) ?? 0,
|
|
2108
|
+
candidatesEmitted: parseRequiredNonNegativeInteger(extract.candidatesEmitted, `${path2}.candidatesEmitted`, issues) ?? 0,
|
|
2109
|
+
newCandidates: parseRequiredNonNegativeInteger(extract.newCandidates, `${path2}.newCandidates`, issues) ?? 0,
|
|
2110
|
+
refineCandidates: parseRequiredNonNegativeInteger(extract.refineCandidates, `${path2}.refineCandidates`, issues) ?? 0,
|
|
2111
|
+
knownCandidates: parseRequiredNonNegativeInteger(extract.knownCandidates, `${path2}.knownCandidates`, issues) ?? 0,
|
|
2112
|
+
durablesInserted: parseRequiredNonNegativeInteger(extract.durablesInserted, `${path2}.durablesInserted`, issues) ?? 0
|
|
2113
|
+
}
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
function parseDreamTemporalizeSummary(value, path2, issues) {
|
|
2117
|
+
if (value === void 0) {
|
|
2118
|
+
return void 0;
|
|
2119
|
+
}
|
|
2120
|
+
const temporalize = parseObject(value, path2, issues);
|
|
2121
|
+
if (temporalize === void 0) {
|
|
2122
|
+
return void 0;
|
|
2123
|
+
}
|
|
2124
|
+
pushUnexpectedFields(temporalize, DREAM_TEMPORALIZE_KEYS, path2, issues);
|
|
2125
|
+
return {
|
|
2126
|
+
temporalize: {
|
|
2127
|
+
revisionsIdentified: parseRequiredNonNegativeInteger(temporalize.revisionsIdentified, `${path2}.revisionsIdentified`, issues) ?? 0,
|
|
2128
|
+
revisionsApplied: parseRequiredNonNegativeInteger(temporalize.revisionsApplied, `${path2}.revisionsApplied`, issues) ?? 0,
|
|
2129
|
+
revisionsSkipped: parseRequiredNonNegativeInteger(temporalize.revisionsSkipped, `${path2}.revisionsSkipped`, issues) ?? 0
|
|
2130
|
+
}
|
|
2131
|
+
};
|
|
2132
|
+
}
|
|
2133
|
+
function parseDreamProjectSummary(value, path2, issues) {
|
|
2134
|
+
const project = parseRequiredObject(value, path2, issues);
|
|
2135
|
+
if (project === void 0) {
|
|
2136
|
+
return void 0;
|
|
2137
|
+
}
|
|
2138
|
+
pushUnexpectedFields(project, DREAM_PROJECT_KEYS, path2, issues);
|
|
2139
|
+
return {
|
|
2140
|
+
profileDurableCount: parseRequiredNonNegativeInteger(project.profileDurableCount, `${path2}.profileDurableCount`, issues) ?? 0,
|
|
2141
|
+
directiveCount: parseRequiredNonNegativeInteger(project.directiveCount, `${path2}.directiveCount`, issues) ?? 0,
|
|
2142
|
+
snapshotId: parseOptionalStringOrNull(project.snapshotId, `${path2}.snapshotId`, issues),
|
|
2143
|
+
applied: parseRequiredBoolean(project.applied, `${path2}.applied`, issues) ?? false
|
|
2144
|
+
};
|
|
2145
|
+
}
|
|
2146
|
+
function parseDreamPruneSummary(value, path2, issues) {
|
|
2147
|
+
if (value === void 0) {
|
|
2148
|
+
return void 0;
|
|
2149
|
+
}
|
|
2150
|
+
const prune = parseObject(value, path2, issues);
|
|
2151
|
+
if (prune === void 0) {
|
|
2152
|
+
return void 0;
|
|
2153
|
+
}
|
|
2154
|
+
pushUnexpectedFields(prune, DREAM_PRUNE_KEYS, path2, issues);
|
|
2155
|
+
return {
|
|
2156
|
+
prune: {
|
|
2157
|
+
durablesScanned: parseRequiredNonNegativeInteger(prune.durablesScanned, `${path2}.durablesScanned`, issues) ?? 0,
|
|
2158
|
+
candidatesIdentified: parseRequiredNonNegativeInteger(prune.candidatesIdentified, `${path2}.candidatesIdentified`, issues) ?? 0,
|
|
2159
|
+
candidatesProtected: parseRequiredNonNegativeInteger(prune.candidatesProtected, `${path2}.candidatesProtected`, issues) ?? 0,
|
|
2160
|
+
candidatesRetirable: parseRequiredNonNegativeInteger(prune.candidatesRetirable, `${path2}.candidatesRetirable`, issues) ?? 0,
|
|
2161
|
+
durablesStaled: parseRequiredNonNegativeInteger(prune.durablesStaled, `${path2}.durablesStaled`, issues) ?? 0,
|
|
2162
|
+
dryRun: parseRequiredBoolean(prune.dryRun, `${path2}.dryRun`, issues) ?? false
|
|
2163
|
+
}
|
|
2164
|
+
};
|
|
2165
|
+
}
|
|
2166
|
+
function parseDreamEfficiencySummary(value, path2, issues) {
|
|
2167
|
+
if (value === void 0) {
|
|
2168
|
+
return void 0;
|
|
2169
|
+
}
|
|
2170
|
+
const efficiency = parseObject(value, path2, issues);
|
|
2171
|
+
if (efficiency === void 0) {
|
|
2172
|
+
return void 0;
|
|
2173
|
+
}
|
|
2174
|
+
pushUnexpectedFields(efficiency, DREAM_EFFICIENCY_KEYS, path2, issues);
|
|
2175
|
+
return {
|
|
2176
|
+
efficiency: {
|
|
2177
|
+
evidenceItemsRead: parseRequiredNonNegativeInteger(efficiency.evidenceItemsRead, `${path2}.evidenceItemsRead`, issues) ?? 0,
|
|
2178
|
+
synthesizedDurableMutations: parseRequiredNonNegativeInteger(efficiency.synthesizedDurableMutations, `${path2}.synthesizedDurableMutations`, issues) ?? 0,
|
|
2179
|
+
costPerSynthesizedDurableUsd: parseOptionalNumberOrNull(efficiency.costPerSynthesizedDurableUsd, `${path2}.costPerSynthesizedDurableUsd`, issues),
|
|
2180
|
+
profileInjectionTokenEstimate: parseRequiredNonNegativeInteger(efficiency.profileInjectionTokenEstimate, `${path2}.profileInjectionTokenEstimate`, issues) ?? 0,
|
|
2181
|
+
recomputeRatio: parseRequiredNonNegativeNumber(efficiency.recomputeRatio, `${path2}.recomputeRatio`, issues) ?? 0
|
|
2182
|
+
}
|
|
2183
|
+
};
|
|
2184
|
+
}
|
|
2185
|
+
function parseDurablesSkipped(value, path2, issues) {
|
|
2186
|
+
if (!Array.isArray(value)) {
|
|
2187
|
+
pushIssue(issues, path2, "Expected an array.");
|
|
2188
|
+
return void 0;
|
|
2189
|
+
}
|
|
2190
|
+
return value.flatMap((item, index) => {
|
|
2191
|
+
const itemPath = `${path2}[${index}]`;
|
|
2192
|
+
const record = parseObject(item, itemPath, issues);
|
|
2193
|
+
if (record === void 0) {
|
|
2194
|
+
return [];
|
|
2195
|
+
}
|
|
2196
|
+
pushUnexpectedFields(record, DURABLE_SKIP_KEYS, itemPath, issues);
|
|
2197
|
+
const reason = parseRequiredTrimmedString(record.reason, `${itemPath}.reason`, issues);
|
|
2198
|
+
const durableId = parseOptionalTrimmedString(record.durable_id, `${itemPath}.durable_id`, issues);
|
|
2199
|
+
return [
|
|
2200
|
+
{
|
|
2201
|
+
...durableId ? { durable_id: durableId } : {},
|
|
2202
|
+
reason: reason ?? ""
|
|
2203
|
+
}
|
|
2204
|
+
];
|
|
2205
|
+
});
|
|
2206
|
+
}
|
|
2207
|
+
function parseEvidenceRefs(value, path2, issues) {
|
|
2208
|
+
if (!Array.isArray(value)) {
|
|
2209
|
+
pushIssue(issues, path2, "Expected an array.");
|
|
2210
|
+
return void 0;
|
|
2211
|
+
}
|
|
2212
|
+
return value.flatMap((item, index) => {
|
|
2213
|
+
const itemPath = `${path2}[${index}]`;
|
|
2214
|
+
const record = parseObject(item, itemPath, issues);
|
|
2215
|
+
if (record === void 0) {
|
|
2216
|
+
return [];
|
|
2217
|
+
}
|
|
2218
|
+
pushUnexpectedFields(record, DREAM_EVIDENCE_REF_KEYS, itemPath, issues);
|
|
2219
|
+
const kind = parseEvidenceKind(record.kind, `${itemPath}.kind`, issues);
|
|
2220
|
+
const locator = parseRequiredTrimmedString(record.locator, `${itemPath}.locator`, issues);
|
|
2221
|
+
const observedAt = parseOptionalTrimmedString(record.observedAt, `${itemPath}.observedAt`, issues);
|
|
2222
|
+
return [
|
|
2223
|
+
{
|
|
2224
|
+
kind: kind ?? "episode",
|
|
2225
|
+
locator: locator ?? "",
|
|
2226
|
+
...observedAt ? { observedAt } : {}
|
|
2227
|
+
}
|
|
2228
|
+
];
|
|
2229
|
+
});
|
|
2230
|
+
}
|
|
2231
|
+
function parseEvidenceKind(value, path2, issues) {
|
|
2232
|
+
if (typeof value !== "string" || !DREAM_EVIDENCE_KINDS.includes(value)) {
|
|
2233
|
+
pushIssue(issues, path2, `kind must be one of: ${DREAM_EVIDENCE_KINDS.join(", ")}.`);
|
|
2234
|
+
return void 0;
|
|
2235
|
+
}
|
|
2236
|
+
return value;
|
|
2237
|
+
}
|
|
2238
|
+
function parseDreamStage(value, path2, issues) {
|
|
2239
|
+
if (typeof value !== "string" || !DREAM_STAGES.includes(value)) {
|
|
2240
|
+
pushIssue(issues, path2, `stage must be one of: ${DREAM_STAGES.join(", ")}.`);
|
|
2241
|
+
return void 0;
|
|
2242
|
+
}
|
|
2243
|
+
return value;
|
|
2244
|
+
}
|
|
2245
|
+
function parseDreamStageSkipReason(value, path2, issues) {
|
|
2246
|
+
if (typeof value !== "string" || !DREAM_STAGE_SKIP_REASONS.includes(value)) {
|
|
2247
|
+
pushIssue(issues, path2, `reason must be one of: ${DREAM_STAGE_SKIP_REASONS.join(", ")}.`);
|
|
2248
|
+
return void 0;
|
|
2249
|
+
}
|
|
2250
|
+
return value;
|
|
2251
|
+
}
|
|
2252
|
+
function parseOptionalObject(value, path2, issues) {
|
|
2253
|
+
if (value === void 0) {
|
|
2254
|
+
return void 0;
|
|
2255
|
+
}
|
|
2256
|
+
return parseObject(value, path2, issues);
|
|
2257
|
+
}
|
|
2258
|
+
function parseRequiredObject(value, path2, issues) {
|
|
2259
|
+
if (value === void 0) {
|
|
2260
|
+
pushIssue(issues, path2, "Expected an object.");
|
|
2261
|
+
return void 0;
|
|
2262
|
+
}
|
|
2263
|
+
return parseObject(value, path2, issues);
|
|
2264
|
+
}
|
|
2265
|
+
function parseRequiredNonNegativeInteger(value, path2, issues) {
|
|
2266
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
2267
|
+
pushIssue(issues, path2, "Expected a non-negative integer.");
|
|
2268
|
+
return void 0;
|
|
2269
|
+
}
|
|
2270
|
+
return value;
|
|
2271
|
+
}
|
|
2272
|
+
function parseRequiredNonNegativeNumber(value, path2, issues) {
|
|
2273
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
2274
|
+
pushIssue(issues, path2, "Expected a non-negative number.");
|
|
2275
|
+
return void 0;
|
|
2276
|
+
}
|
|
2277
|
+
return value;
|
|
2278
|
+
}
|
|
2279
|
+
function parseOptionalBoolean2(value, path2, issues) {
|
|
2280
|
+
if (value === void 0) {
|
|
2281
|
+
return void 0;
|
|
2282
|
+
}
|
|
2283
|
+
return parseRequiredBoolean(value, path2, issues);
|
|
2284
|
+
}
|
|
2285
|
+
function parseRequiredBoolean(value, path2, issues) {
|
|
2286
|
+
if (typeof value !== "boolean") {
|
|
2287
|
+
pushIssue(issues, path2, "Expected a boolean.");
|
|
2288
|
+
return void 0;
|
|
2289
|
+
}
|
|
2290
|
+
return value;
|
|
2291
|
+
}
|
|
2292
|
+
function parseRequiredStringArray2(value, path2, issues) {
|
|
2293
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
2294
|
+
pushIssue(issues, path2, "Expected an array of strings.");
|
|
2295
|
+
return void 0;
|
|
2296
|
+
}
|
|
2297
|
+
return value;
|
|
2298
|
+
}
|
|
2299
|
+
function parseOptionalStringOrNull(value, path2, issues) {
|
|
2300
|
+
if (value === null || value === void 0) {
|
|
2301
|
+
return null;
|
|
2302
|
+
}
|
|
2303
|
+
return parseOptionalTrimmedString(value, path2, issues) ?? null;
|
|
2304
|
+
}
|
|
2305
|
+
function parseOptionalNumberOrNull(value, path2, issues) {
|
|
2306
|
+
if (value === null) {
|
|
2307
|
+
return null;
|
|
2308
|
+
}
|
|
2309
|
+
return parseRequiredNonNegativeNumber(value, path2, issues) ?? null;
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
// src/adapters/api/validation/dreaming-efficiency-eval-request.ts
|
|
2313
|
+
var ROOT_REQUEST_KEYS2 = /* @__PURE__ */ new Set(["caseId", "description", "sandbox", "memoryPool", "dreamRunFixture", "options"]);
|
|
2314
|
+
var DREAM_RUN_FIXTURE_KEYS = /* @__PURE__ */ new Set(["tier", "summaryJson", "estimatedCostUsd", "completedAt"]);
|
|
2315
|
+
var OPTIONS_KEYS2 = /* @__PURE__ */ new Set(["includeTimings"]);
|
|
2316
|
+
var DreamingEfficiencyEvalRequestValidationError = class extends Error {
|
|
2317
|
+
caseId;
|
|
2318
|
+
issues;
|
|
2319
|
+
/**
|
|
2320
|
+
* Creates one validation error with structured request issues.
|
|
2321
|
+
*
|
|
2322
|
+
* @param issues - Validation failures found at the HTTP boundary.
|
|
2323
|
+
* @param caseId - Optional case identifier extracted before validation failed.
|
|
2324
|
+
*/
|
|
2325
|
+
constructor(issues, caseId) {
|
|
2326
|
+
super("Invalid dreaming-efficiency eval request.");
|
|
2327
|
+
this.name = "DreamingEfficiencyEvalRequestValidationError";
|
|
2328
|
+
this.issues = issues;
|
|
2329
|
+
this.caseId = caseId;
|
|
2330
|
+
}
|
|
2331
|
+
};
|
|
2332
|
+
function parseDreamingEfficiencyEvalCaseRequest(payload) {
|
|
2333
|
+
const issues = [];
|
|
2334
|
+
const input = parseObject(payload, "$", issues);
|
|
2335
|
+
if (input === void 0) {
|
|
2336
|
+
throw new DreamingEfficiencyEvalRequestValidationError(issues);
|
|
2337
|
+
}
|
|
2338
|
+
pushUnexpectedFields(input, ROOT_REQUEST_KEYS2, "$", issues);
|
|
2339
|
+
const caseId = parseRequiredTrimmedString(input.caseId, "caseId", issues);
|
|
2340
|
+
const memoryPool = parseMemoryPool(input.memoryPool, issues);
|
|
2341
|
+
if (memoryPool === void 0) {
|
|
2342
|
+
pushUnexpectedFields({}, /* @__PURE__ */ new Set(["memoryPool"]), "$", issues);
|
|
2343
|
+
}
|
|
2344
|
+
const description = parseOptionalTrimmedString(input.description, "description", issues);
|
|
2345
|
+
const sandbox = parseSandbox(input.sandbox, issues);
|
|
2346
|
+
const dreamRunFixture = parseRequiredDreamRunFixture(input.dreamRunFixture, issues);
|
|
2347
|
+
const options = parseOptions2(input.options, issues);
|
|
2348
|
+
if (issues.length > 0) {
|
|
2349
|
+
throw new DreamingEfficiencyEvalRequestValidationError(issues, caseId);
|
|
2350
|
+
}
|
|
2351
|
+
return {
|
|
2352
|
+
caseId: caseId ?? "",
|
|
2353
|
+
memoryPool: memoryPool ?? [],
|
|
2354
|
+
...description ? { description } : {},
|
|
2355
|
+
...sandbox ? { sandbox } : {},
|
|
2356
|
+
dreamRunFixture,
|
|
2357
|
+
...options ? { options } : {}
|
|
2358
|
+
};
|
|
2359
|
+
}
|
|
2360
|
+
function mapDreamingEfficiencyEvalCaseRequestDto(dto) {
|
|
2361
|
+
return {
|
|
2362
|
+
caseId: dto.caseId,
|
|
2363
|
+
...dto.description ? { description: dto.description } : {},
|
|
2364
|
+
...dto.sandbox ? { sandbox: mapSandboxRequestDto(dto.sandbox) } : {},
|
|
2365
|
+
memoryPool: dto.memoryPool.map((entry) => mapFixtureEntryDto(entry)),
|
|
2366
|
+
dreamRunFixture: mapDreamRunFixtureDto(dto.dreamRunFixture),
|
|
2367
|
+
...dto.options ? { options: mapCaseOptionsDto2(dto.options) } : {}
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2370
|
+
function parseRequiredDreamRunFixture(value, issues) {
|
|
2371
|
+
if (value === void 0) {
|
|
2372
|
+
pushIssue(issues, "dreamRunFixture", "Expected an object.");
|
|
2373
|
+
return void 0;
|
|
2374
|
+
}
|
|
2375
|
+
const fixture = parseObject(value, "dreamRunFixture", issues);
|
|
2376
|
+
if (fixture === void 0) {
|
|
2377
|
+
return void 0;
|
|
2378
|
+
}
|
|
2379
|
+
pushUnexpectedFields(fixture, DREAM_RUN_FIXTURE_KEYS, "dreamRunFixture", issues);
|
|
2380
|
+
const tier = parseDreamTier(fixture.tier, "dreamRunFixture.tier", issues);
|
|
2381
|
+
const summaryJson = parseDreamCompletionSummary(fixture.summaryJson, "dreamRunFixture.summaryJson", issues);
|
|
2382
|
+
if (summaryJson === void 0) {
|
|
2383
|
+
return void 0;
|
|
2384
|
+
}
|
|
2385
|
+
return {
|
|
2386
|
+
tier: tier ?? "standard",
|
|
2387
|
+
summaryJson,
|
|
2388
|
+
estimatedCostUsd: parseOptionalNonNegativeNumber(fixture.estimatedCostUsd, "dreamRunFixture.estimatedCostUsd", issues),
|
|
2389
|
+
completedAt: parseOptionalTrimmedString(fixture.completedAt, "dreamRunFixture.completedAt", issues)
|
|
2390
|
+
};
|
|
2391
|
+
}
|
|
2392
|
+
function parseDreamTier(value, path2, issues) {
|
|
2393
|
+
if (value === void 0) {
|
|
2394
|
+
pushIssue(issues, path2, "Expected a dreaming tier.");
|
|
2395
|
+
return void 0;
|
|
2396
|
+
}
|
|
2397
|
+
if (typeof value !== "string" || !DREAM_TIERS.includes(value)) {
|
|
2398
|
+
issues.push({ path: path2, message: `tier must be one of: ${DREAM_TIERS.join(", ")}.` });
|
|
2399
|
+
return void 0;
|
|
2400
|
+
}
|
|
2401
|
+
return value;
|
|
2402
|
+
}
|
|
2403
|
+
function parseOptionalNonNegativeNumber(value, path2, issues) {
|
|
2404
|
+
if (value === void 0) {
|
|
2405
|
+
return void 0;
|
|
2406
|
+
}
|
|
2407
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
2408
|
+
pushIssue(issues, path2, "Expected a non-negative number.");
|
|
2409
|
+
return void 0;
|
|
2410
|
+
}
|
|
2411
|
+
return value;
|
|
2412
|
+
}
|
|
2413
|
+
function parseOptions2(value, issues) {
|
|
2414
|
+
if (value === void 0) {
|
|
2415
|
+
return void 0;
|
|
2416
|
+
}
|
|
2417
|
+
const options = parseObject(value, "options", issues);
|
|
2418
|
+
if (options === void 0) {
|
|
2419
|
+
return void 0;
|
|
2420
|
+
}
|
|
2421
|
+
pushUnexpectedFields(options, OPTIONS_KEYS2, "options", issues);
|
|
2422
|
+
return {
|
|
2423
|
+
includeTimings: parseOptionalBoolean(options.includeTimings, "options.includeTimings", issues)
|
|
2424
|
+
};
|
|
2425
|
+
}
|
|
2426
|
+
function mapDreamRunFixtureDto(dto) {
|
|
2427
|
+
return {
|
|
2428
|
+
tier: dto.tier,
|
|
2429
|
+
summaryJson: dto.summaryJson,
|
|
2430
|
+
...dto.estimatedCostUsd !== void 0 ? { estimatedCostUsd: dto.estimatedCostUsd } : {},
|
|
2431
|
+
...dto.completedAt ? { completedAt: dto.completedAt } : {}
|
|
2432
|
+
};
|
|
2433
|
+
}
|
|
2434
|
+
function mapCaseOptionsDto2(dto) {
|
|
2435
|
+
return {
|
|
2436
|
+
includeTimings: dto.includeTimings
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
// src/adapters/api/routes/internal-dreaming-efficiency-eval.ts
|
|
2441
|
+
var INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE_PATH = "/internal/evals/dreaming-efficiency/run";
|
|
2442
|
+
var INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE = {
|
|
2443
|
+
method: "POST",
|
|
2444
|
+
path: INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE_PATH
|
|
2445
|
+
};
|
|
2446
|
+
function createInternalDreamingEfficiencyEvalRoute(runner = runDreamingEfficiencyEvalCase) {
|
|
2447
|
+
return {
|
|
2448
|
+
...INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE,
|
|
2449
|
+
handler: async (request) => {
|
|
2450
|
+
let validatedRequest;
|
|
2451
|
+
try {
|
|
2452
|
+
validatedRequest = await parseValidatedRequest2(request);
|
|
2453
|
+
const result = await runner(validatedRequest);
|
|
2454
|
+
return jsonResponse2(result, 200);
|
|
2455
|
+
} catch (error) {
|
|
2456
|
+
if (error instanceof DreamingEfficiencyEvalRequestValidationError) {
|
|
2457
|
+
return jsonResponse2(
|
|
2458
|
+
{
|
|
2459
|
+
status: "error",
|
|
2460
|
+
caseId: error.caseId,
|
|
2461
|
+
error: {
|
|
2462
|
+
code: "invalid_request",
|
|
2463
|
+
message: error.message,
|
|
2464
|
+
details: error.issues
|
|
2465
|
+
}
|
|
2466
|
+
},
|
|
2467
|
+
400
|
|
2468
|
+
);
|
|
2469
|
+
}
|
|
2470
|
+
return jsonResponse2(
|
|
2471
|
+
{
|
|
2472
|
+
status: "error",
|
|
2473
|
+
caseId: validatedRequest?.caseId,
|
|
2474
|
+
error: {
|
|
2475
|
+
code: "internal_error",
|
|
2476
|
+
message: "Internal dreaming-efficiency eval adapter error."
|
|
2477
|
+
}
|
|
2478
|
+
},
|
|
2479
|
+
500
|
|
2480
|
+
);
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
};
|
|
2484
|
+
}
|
|
2485
|
+
var parseJsonBody2 = async (request) => {
|
|
2486
|
+
try {
|
|
2487
|
+
return await request.json();
|
|
2488
|
+
} catch {
|
|
2489
|
+
throw new DreamingEfficiencyEvalRequestValidationError([
|
|
2490
|
+
{
|
|
2491
|
+
path: "$",
|
|
2492
|
+
message: "Request body must be valid JSON."
|
|
2493
|
+
}
|
|
2494
|
+
]);
|
|
2495
|
+
}
|
|
2496
|
+
};
|
|
2497
|
+
var parseValidatedRequest2 = async (request) => {
|
|
2498
|
+
const payload = await parseJsonBody2(request);
|
|
2499
|
+
const requestDto = parseDreamingEfficiencyEvalCaseRequest(payload);
|
|
2500
|
+
return mapDreamingEfficiencyEvalCaseRequestDto(requestDto);
|
|
2501
|
+
};
|
|
2502
|
+
var jsonResponse2 = (body, status) => new Response(JSON.stringify(body), {
|
|
2503
|
+
status,
|
|
2504
|
+
headers: {
|
|
2505
|
+
"content-type": "application/json; charset=utf-8"
|
|
2506
|
+
}
|
|
2507
|
+
});
|
|
2508
|
+
|
|
2509
|
+
// src/app/evals/recall/collect-diagnostics.ts
|
|
2510
|
+
function createRecallEvalDiagnosticsCollector(request) {
|
|
2511
|
+
const diagnosticsRequested = wantsRecallEvalDiagnostics(request);
|
|
2512
|
+
const timingsRequested = request.options?.includeTimings === true;
|
|
2513
|
+
const debugArtifactRequested = request.options?.includeDebugArtifact === true;
|
|
2514
|
+
const observationEnabled = diagnosticsRequested || timingsRequested || debugArtifactRequested;
|
|
2515
|
+
const execution = {
|
|
2516
|
+
mode: "isolated-case",
|
|
2517
|
+
provisioning: "exact-fixture-seed",
|
|
2518
|
+
recallPath: request.recallPath ?? "core",
|
|
2519
|
+
memoryPoolCount: request.memoryPool.length,
|
|
2520
|
+
provisionedCount: 0,
|
|
2521
|
+
requestedDiagnostics: request.options?.includeDiagnostics === true,
|
|
2522
|
+
requestedCandidates: request.options?.includeCandidates === true
|
|
2523
|
+
};
|
|
2524
|
+
const stageTimings = {
|
|
2525
|
+
sandboxSetupMs: 0,
|
|
2526
|
+
fixtureProvisionMs: 0,
|
|
2527
|
+
recallMs: 0,
|
|
2528
|
+
queryEmbeddingMs: 0,
|
|
2529
|
+
vectorSearchMs: 0,
|
|
2530
|
+
lexicalSearchMs: 0,
|
|
2531
|
+
mergeCandidatesMs: 0,
|
|
2532
|
+
scoreCandidatesMs: 0,
|
|
2533
|
+
thresholdMs: 0,
|
|
2534
|
+
budgetMs: 0,
|
|
2535
|
+
hydrateEntriesMs: 0,
|
|
2536
|
+
shapeResultsMs: 0,
|
|
2537
|
+
recordRecallEventsMs: 0
|
|
2538
|
+
};
|
|
2539
|
+
const retrieval = {
|
|
2540
|
+
queryEmbeddingDimensions: 0,
|
|
2541
|
+
vectorSearchLimit: 0,
|
|
2542
|
+
lexicalSearchLimit: 0
|
|
2543
|
+
};
|
|
2544
|
+
const candidateCounts = {
|
|
2545
|
+
vectorRetrieved: 0,
|
|
2546
|
+
lexicalRetrieved: 0,
|
|
2547
|
+
merged: 0,
|
|
2548
|
+
thresholdQualified: 0,
|
|
2549
|
+
budgetAccepted: 0,
|
|
2550
|
+
finalRanked: 0,
|
|
2551
|
+
hydrated: 0,
|
|
2552
|
+
returned: 0,
|
|
2553
|
+
telemetryAttempted: 0
|
|
2554
|
+
};
|
|
2555
|
+
let provision;
|
|
2556
|
+
let ranking;
|
|
2557
|
+
let filtering;
|
|
2558
|
+
let claimKey;
|
|
2559
|
+
let rrf;
|
|
2560
|
+
let neighborhood;
|
|
2561
|
+
let mmr;
|
|
2562
|
+
let crossEncoder;
|
|
2563
|
+
let degraded;
|
|
2564
|
+
let provisionObserved = false;
|
|
2565
|
+
let retrievalObserved = false;
|
|
2566
|
+
let traceObserved = false;
|
|
2567
|
+
const traceSink = {
|
|
2568
|
+
reportSummary(summary) {
|
|
2569
|
+
traceObserved = true;
|
|
2570
|
+
ranking = {
|
|
2571
|
+
limit: summary.ranking.limit,
|
|
2572
|
+
threshold: summary.ranking.threshold,
|
|
2573
|
+
budget: summary.ranking.budget,
|
|
2574
|
+
noResultReason: summary.ranking.noResultReason
|
|
2575
|
+
};
|
|
2576
|
+
filtering = {
|
|
2577
|
+
types: [...summary.filtering.types],
|
|
2578
|
+
tags: [...summary.filtering.tags],
|
|
2579
|
+
since: summary.filtering.since,
|
|
2580
|
+
until: summary.filtering.until,
|
|
2581
|
+
around: summary.filtering.around ? {
|
|
2582
|
+
source: summary.filtering.around.source,
|
|
2583
|
+
anchor: summary.filtering.around.anchor,
|
|
2584
|
+
radiusDays: summary.filtering.around.radiusDays
|
|
2585
|
+
} : void 0
|
|
2586
|
+
};
|
|
2587
|
+
candidateCounts.merged = summary.candidateCounts.merged;
|
|
2588
|
+
candidateCounts.thresholdQualified = summary.candidateCounts.thresholdQualified;
|
|
2589
|
+
candidateCounts.budgetAccepted = summary.candidateCounts.budgetAccepted;
|
|
2590
|
+
candidateCounts.finalRanked = summary.candidateCounts.finalRanked;
|
|
2591
|
+
candidateCounts.returned = summary.candidateCounts.returned;
|
|
2592
|
+
claimKey = {
|
|
2593
|
+
historicalBoosted: summary.claimKey.historicalBoosted,
|
|
2594
|
+
tentativeLineageSuppressed: summary.claimKey.tentativeLineageSuppressed,
|
|
2595
|
+
trustPenalized: summary.claimKey.trustPenalized,
|
|
2596
|
+
redundancyPenalized: summary.claimKey.redundancyPenalized
|
|
2597
|
+
};
|
|
2598
|
+
rrf = {
|
|
2599
|
+
applied: summary.rrf.applied,
|
|
2600
|
+
channelCount: summary.rrf.channelCount,
|
|
2601
|
+
rankConstant: summary.rrf.rankConstant,
|
|
2602
|
+
fusedCandidateCount: summary.rrf.fusedCandidateCount,
|
|
2603
|
+
maxFusedScore: summary.rrf.maxFusedScore
|
|
2604
|
+
};
|
|
2605
|
+
neighborhood = {
|
|
2606
|
+
expansionRequested: summary.neighborhood.expansionRequested,
|
|
2607
|
+
expansionAvailable: summary.neighborhood.expansionAvailable,
|
|
2608
|
+
familiesRequested: [...summary.neighborhood.familiesRequested],
|
|
2609
|
+
includeHistorical: summary.neighborhood.includeHistorical,
|
|
2610
|
+
seedIds: [...summary.neighborhood.seedIds],
|
|
2611
|
+
expansionCandidates: summary.neighborhood.expansionCandidates,
|
|
2612
|
+
strongSeedIds: [...summary.neighborhood.strongSeedIds],
|
|
2613
|
+
rerankBoostedIds: [...summary.neighborhood.rerankBoostedIds]
|
|
2614
|
+
};
|
|
2615
|
+
mmr = {
|
|
2616
|
+
applied: summary.mmr.applied,
|
|
2617
|
+
lambda: summary.mmr.lambda,
|
|
2618
|
+
droppedDuplicateCount: summary.mmr.droppedDuplicateCount,
|
|
2619
|
+
reorderedIds: [...summary.mmr.reorderedIds]
|
|
2620
|
+
};
|
|
2621
|
+
crossEncoder = {
|
|
2622
|
+
applied: summary.crossEncoder.applied,
|
|
2623
|
+
k: summary.crossEncoder.k,
|
|
2624
|
+
alpha: summary.crossEncoder.alpha,
|
|
2625
|
+
latencyMs: summary.crossEncoder.latencyMs,
|
|
2626
|
+
rescoredIds: [...summary.crossEncoder.rescoredIds],
|
|
2627
|
+
...summary.crossEncoder.degradedReason ? { degradedReason: summary.crossEncoder.degradedReason } : {}
|
|
2628
|
+
};
|
|
2629
|
+
degraded = {
|
|
2630
|
+
active: summary.degraded.active,
|
|
2631
|
+
reasons: [...summary.degraded.reasons],
|
|
2632
|
+
lexicalOnly: summary.degraded.lexicalOnly,
|
|
2633
|
+
notices: [...summary.degraded.notices]
|
|
2634
|
+
};
|
|
2635
|
+
stageTimings.mergeCandidatesMs = summary.timings.mergeCandidatesMs;
|
|
2636
|
+
stageTimings.scoreCandidatesMs = summary.timings.scoreCandidatesMs;
|
|
2637
|
+
stageTimings.thresholdMs = summary.timings.thresholdMs;
|
|
2638
|
+
stageTimings.budgetMs = summary.timings.budgetMs;
|
|
2639
|
+
stageTimings.shapeResultsMs = summary.timings.shapeResultsMs;
|
|
2640
|
+
}
|
|
2641
|
+
};
|
|
2642
|
+
return {
|
|
2643
|
+
traceSink,
|
|
2644
|
+
isObservationEnabled() {
|
|
2645
|
+
return observationEnabled;
|
|
2646
|
+
},
|
|
2647
|
+
buildObservedArtifactFacts() {
|
|
2648
|
+
return {
|
|
2649
|
+
candidateCounts: { ...candidateCounts },
|
|
2650
|
+
ranking: traceObserved && ranking ? { ...ranking } : void 0,
|
|
2651
|
+
degraded: traceObserved && degraded ? {
|
|
2652
|
+
active: degraded.active,
|
|
2653
|
+
reasons: [...degraded.reasons],
|
|
2654
|
+
lexicalOnly: degraded.lexicalOnly,
|
|
2655
|
+
notices: [...degraded.notices]
|
|
2656
|
+
} : void 0,
|
|
2657
|
+
traceObserved
|
|
2658
|
+
};
|
|
2659
|
+
},
|
|
2660
|
+
recordSandboxSetup(durationMs) {
|
|
2661
|
+
stageTimings.sandboxSetupMs = durationMs;
|
|
2662
|
+
},
|
|
2663
|
+
recordFixtureProvisionTiming(durationMs) {
|
|
2664
|
+
stageTimings.fixtureProvisionMs = durationMs;
|
|
2665
|
+
},
|
|
2666
|
+
recordProvision(result, durationMs) {
|
|
2667
|
+
provisionObserved = true;
|
|
2668
|
+
execution.provisionedCount = result.provisionedCount;
|
|
2669
|
+
stageTimings.fixtureProvisionMs = durationMs;
|
|
2670
|
+
provision = {
|
|
2671
|
+
requestedCount: request.memoryPool.length,
|
|
2672
|
+
provisionedCount: result.provisionedCount,
|
|
2673
|
+
providedIdCount: result.providedIdCount,
|
|
2674
|
+
generatedIdCount: result.generatedIdCount,
|
|
2675
|
+
staleCount: result.staleCount,
|
|
2676
|
+
supersededCount: result.supersededCount,
|
|
2677
|
+
createdAtDefaultedCount: result.createdAtDefaultedCount,
|
|
2678
|
+
updatedAtDefaultedCount: result.updatedAtDefaultedCount,
|
|
2679
|
+
seededEntries: result.seededEntries.map((entry) => ({
|
|
2680
|
+
id: entry.id,
|
|
2681
|
+
created_at: entry.created_at,
|
|
2682
|
+
updated_at: entry.updated_at,
|
|
2683
|
+
superseded_by: entry.superseded_by,
|
|
2684
|
+
claim_key: entry.claim_key,
|
|
2685
|
+
claim_key_status: entry.claim_key_status,
|
|
2686
|
+
valid_from: entry.valid_from,
|
|
2687
|
+
valid_to: entry.valid_to
|
|
2688
|
+
}))
|
|
2689
|
+
};
|
|
2690
|
+
},
|
|
2691
|
+
recordRecall(durationMs) {
|
|
2692
|
+
stageTimings.recallMs = durationMs;
|
|
2693
|
+
},
|
|
2694
|
+
recordQueryEmbedding(params) {
|
|
2695
|
+
retrievalObserved = true;
|
|
2696
|
+
stageTimings.queryEmbeddingMs = params.durationMs;
|
|
2697
|
+
retrieval.queryEmbeddingDimensions = params.dimensions;
|
|
2698
|
+
},
|
|
2699
|
+
recordVectorSearch(params) {
|
|
2700
|
+
retrievalObserved = true;
|
|
2701
|
+
stageTimings.vectorSearchMs = params.durationMs;
|
|
2702
|
+
retrieval.vectorSearchLimit = params.limit;
|
|
2703
|
+
candidateCounts.vectorRetrieved = params.count;
|
|
2704
|
+
},
|
|
2705
|
+
recordLexicalSearch(params) {
|
|
2706
|
+
retrievalObserved = true;
|
|
2707
|
+
stageTimings.lexicalSearchMs = params.durationMs;
|
|
2708
|
+
retrieval.lexicalSearchLimit = params.limit;
|
|
2709
|
+
candidateCounts.lexicalRetrieved = params.count;
|
|
2710
|
+
},
|
|
2711
|
+
recordHydrateEntries(params) {
|
|
2712
|
+
retrievalObserved = true;
|
|
2713
|
+
stageTimings.hydrateEntriesMs = params.durationMs;
|
|
2714
|
+
candidateCounts.hydrated = params.count;
|
|
2715
|
+
},
|
|
2716
|
+
recordRecallTelemetry(params) {
|
|
2717
|
+
retrievalObserved = true;
|
|
2718
|
+
stageTimings.recordRecallEventsMs = params.durationMs;
|
|
2719
|
+
candidateCounts.telemetryAttempted = params.entryCount;
|
|
2720
|
+
},
|
|
2721
|
+
buildDiagnostics() {
|
|
2722
|
+
if (!diagnosticsRequested) {
|
|
2723
|
+
return void 0;
|
|
2724
|
+
}
|
|
2725
|
+
return {
|
|
2726
|
+
execution,
|
|
2727
|
+
provision: provisionObserved ? provision : void 0,
|
|
2728
|
+
retrieval: retrievalObserved ? retrieval : void 0,
|
|
2729
|
+
ranking: traceObserved ? ranking : void 0,
|
|
2730
|
+
filtering: traceObserved ? filtering : void 0,
|
|
2731
|
+
claimKey: traceObserved ? claimKey : void 0,
|
|
2732
|
+
rrf: traceObserved ? rrf : void 0,
|
|
2733
|
+
neighborhood: traceObserved ? neighborhood : void 0,
|
|
2734
|
+
mmr: traceObserved ? mmr : void 0,
|
|
2735
|
+
crossEncoder: traceObserved ? crossEncoder : void 0,
|
|
2736
|
+
degraded: traceObserved ? degraded : void 0,
|
|
2737
|
+
candidateCounts
|
|
2738
|
+
};
|
|
2739
|
+
},
|
|
2740
|
+
buildTimings(totalMs) {
|
|
2741
|
+
if (!timingsRequested) {
|
|
2742
|
+
return void 0;
|
|
2743
|
+
}
|
|
2744
|
+
return {
|
|
2745
|
+
totalMs,
|
|
2746
|
+
sandboxSetupMs: stageTimings.sandboxSetupMs,
|
|
2747
|
+
fixtureProvisionMs: stageTimings.fixtureProvisionMs,
|
|
2748
|
+
recallMs: stageTimings.recallMs,
|
|
2749
|
+
queryEmbeddingMs: stageTimings.queryEmbeddingMs,
|
|
2750
|
+
vectorSearchMs: stageTimings.vectorSearchMs,
|
|
2751
|
+
lexicalSearchMs: stageTimings.lexicalSearchMs,
|
|
2752
|
+
mergeCandidatesMs: stageTimings.mergeCandidatesMs,
|
|
2753
|
+
scoreCandidatesMs: stageTimings.scoreCandidatesMs,
|
|
2754
|
+
thresholdMs: stageTimings.thresholdMs,
|
|
2755
|
+
budgetMs: stageTimings.budgetMs,
|
|
2756
|
+
hydrateEntriesMs: stageTimings.hydrateEntriesMs,
|
|
2757
|
+
shapeResultsMs: stageTimings.shapeResultsMs,
|
|
2758
|
+
recordRecallEventsMs: stageTimings.recordRecallEventsMs
|
|
2759
|
+
};
|
|
2760
|
+
}
|
|
2761
|
+
};
|
|
2762
|
+
}
|
|
2763
|
+
function wantsRecallEvalDiagnostics(request) {
|
|
2764
|
+
return request.options?.includeDiagnostics === true || request.options?.includeCandidates === true;
|
|
2765
|
+
}
|
|
2766
|
+
|
|
2767
|
+
// src/app/evals/recall/instrumented-recall-ports.ts
|
|
2768
|
+
function createInstrumentedRecallPorts(ports, observer) {
|
|
2769
|
+
return {
|
|
2770
|
+
async embed(text) {
|
|
2771
|
+
const startedAt = Date.now();
|
|
2772
|
+
try {
|
|
2773
|
+
const embedding = await ports.embed(text);
|
|
2774
|
+
observer.recordQueryEmbedding({
|
|
2775
|
+
durationMs: elapsedMs3(startedAt),
|
|
2776
|
+
dimensions: embedding.length
|
|
2777
|
+
});
|
|
2778
|
+
return embedding;
|
|
2779
|
+
} catch (error) {
|
|
2780
|
+
observer.recordQueryEmbedding({
|
|
2781
|
+
durationMs: elapsedMs3(startedAt),
|
|
2782
|
+
dimensions: 0
|
|
2783
|
+
});
|
|
2784
|
+
throw error;
|
|
2785
|
+
}
|
|
2786
|
+
},
|
|
2787
|
+
async vectorSearch(params) {
|
|
2788
|
+
const startedAt = Date.now();
|
|
2789
|
+
try {
|
|
2790
|
+
const results = await ports.vectorSearch(params);
|
|
2791
|
+
observer.recordVectorSearch({
|
|
2792
|
+
durationMs: elapsedMs3(startedAt),
|
|
2793
|
+
count: results.length,
|
|
2794
|
+
limit: params.limit
|
|
2795
|
+
});
|
|
2796
|
+
return results;
|
|
2797
|
+
} catch (error) {
|
|
2798
|
+
observer.recordVectorSearch({
|
|
2799
|
+
durationMs: elapsedMs3(startedAt),
|
|
2800
|
+
count: 0,
|
|
2801
|
+
limit: params.limit
|
|
2802
|
+
});
|
|
2803
|
+
throw error;
|
|
2804
|
+
}
|
|
2805
|
+
},
|
|
2806
|
+
async ftsSearch(params) {
|
|
2807
|
+
const startedAt = Date.now();
|
|
2808
|
+
try {
|
|
2809
|
+
const results = await ports.ftsSearch(params);
|
|
2810
|
+
observer.recordLexicalSearch({
|
|
2811
|
+
durationMs: elapsedMs3(startedAt),
|
|
2812
|
+
count: results.length,
|
|
2813
|
+
limit: params.limit
|
|
2814
|
+
});
|
|
2815
|
+
return results;
|
|
2816
|
+
} catch (error) {
|
|
2817
|
+
observer.recordLexicalSearch({
|
|
2818
|
+
durationMs: elapsedMs3(startedAt),
|
|
2819
|
+
count: 0,
|
|
2820
|
+
limit: params.limit
|
|
2821
|
+
});
|
|
2822
|
+
throw error;
|
|
2823
|
+
}
|
|
2824
|
+
},
|
|
2825
|
+
...ports.expandNeighborhood ? {
|
|
2826
|
+
async expandNeighborhood(request) {
|
|
2827
|
+
return ports.expandNeighborhood(request);
|
|
2828
|
+
}
|
|
2829
|
+
} : {},
|
|
2830
|
+
// Cross-encoder is an optional port: proxy it when available so the
|
|
2831
|
+
// core recall pipeline sees the same rerank surface as in production,
|
|
2832
|
+
// and the diagnostics collector can report the `crossEncoder` trace
|
|
2833
|
+
// branch exactly as the core emits it. Dropping the proxy here would
|
|
2834
|
+
// silently convert rerank-aware eval cases into rerank-disabled runs.
|
|
2835
|
+
...ports.crossEncoder ? {
|
|
2836
|
+
crossEncoder: ports.crossEncoder
|
|
2837
|
+
} : {},
|
|
2838
|
+
async hydrateEntries(ids) {
|
|
2839
|
+
const startedAt = Date.now();
|
|
2840
|
+
try {
|
|
2841
|
+
const entries = await ports.hydrateEntries(ids);
|
|
2842
|
+
observer.recordHydrateEntries({
|
|
2843
|
+
durationMs: elapsedMs3(startedAt),
|
|
2844
|
+
count: entries.length
|
|
2845
|
+
});
|
|
2846
|
+
return entries;
|
|
2847
|
+
} catch (error) {
|
|
2848
|
+
observer.recordHydrateEntries({
|
|
2849
|
+
durationMs: elapsedMs3(startedAt),
|
|
2850
|
+
count: 0
|
|
2851
|
+
});
|
|
2852
|
+
throw error;
|
|
2853
|
+
}
|
|
2854
|
+
},
|
|
2855
|
+
async recordRecallEvents(params) {
|
|
2856
|
+
const startedAt = Date.now();
|
|
2857
|
+
try {
|
|
2858
|
+
await ports.recordRecallEvents(params);
|
|
2859
|
+
} finally {
|
|
2860
|
+
observer.recordRecallTelemetry({
|
|
2861
|
+
durationMs: elapsedMs3(startedAt),
|
|
2862
|
+
entryCount: params.entryIds.length
|
|
2863
|
+
});
|
|
2864
|
+
}
|
|
2865
|
+
}
|
|
2866
|
+
};
|
|
2867
|
+
}
|
|
2868
|
+
function elapsedMs3(startedAt) {
|
|
2869
|
+
return Math.max(0, Date.now() - startedAt);
|
|
2870
|
+
}
|
|
2871
|
+
|
|
2872
|
+
// src/app/evals/recall/build-debug-artifact.ts
|
|
2873
|
+
function buildRecallDebugArtifact(params) {
|
|
2874
|
+
const { request, results, projectedEntries, sandbox, observed } = params;
|
|
2875
|
+
const recallPath = request.recallPath ?? "core";
|
|
2876
|
+
const entryResults = Array.isArray(results) ? results : results.entries;
|
|
2877
|
+
const selectedEntryIds = entryResults.map((result) => result.entry.id);
|
|
2878
|
+
const topK = resolveTopK2(request.options?.topKCandidates);
|
|
2879
|
+
const reasonsByEntryId = /* @__PURE__ */ new Map();
|
|
2880
|
+
for (const entry of projectedEntries) {
|
|
2881
|
+
if (entry.whySurfaced.reasons.length > 0) {
|
|
2882
|
+
reasonsByEntryId.set(entry.entryId, [...entry.whySurfaced.reasons]);
|
|
2883
|
+
}
|
|
2884
|
+
}
|
|
2885
|
+
const topCandidates = buildTopCandidates(entryResults, reasonsByEntryId, topK);
|
|
2886
|
+
const artifact = {
|
|
2887
|
+
schemaVersion: "recall-debug-artifact.v1",
|
|
2888
|
+
caseId: request.caseId,
|
|
2889
|
+
...sandbox.snapshot ? { snapshot: buildSnapshot2(sandbox.snapshot) } : {},
|
|
2890
|
+
request: {
|
|
2891
|
+
recallPath,
|
|
2892
|
+
query: request.recallRequest.text
|
|
2893
|
+
},
|
|
2894
|
+
...Array.isArray(results) ? {} : { routing: results.routing },
|
|
2895
|
+
...observed.traceObserved ? { candidateCounts: observed.candidateCounts } : {},
|
|
2896
|
+
...observed.ranking ? { ranking: observed.ranking } : {},
|
|
2897
|
+
...observed.degraded ? { degraded: observed.degraded } : {},
|
|
2898
|
+
selectedEntryIds,
|
|
2899
|
+
...topCandidates.length > 0 ? { topCandidates } : {}
|
|
2900
|
+
};
|
|
2901
|
+
return artifact;
|
|
2902
|
+
}
|
|
2903
|
+
function buildSnapshot2(snapshot) {
|
|
2904
|
+
return {
|
|
2905
|
+
...snapshot.id !== void 0 ? { id: snapshot.id } : {},
|
|
2906
|
+
...snapshot.label !== void 0 ? { label: snapshot.label } : {},
|
|
2907
|
+
dbPathBasename: snapshot.dbPathBasename
|
|
2908
|
+
};
|
|
2909
|
+
}
|
|
2910
|
+
function resolveTopK2(requested) {
|
|
2911
|
+
if (requested === void 0) {
|
|
2912
|
+
return RECALL_DEBUG_ARTIFACT_DEFAULT_TOP_K;
|
|
2913
|
+
}
|
|
2914
|
+
if (!Number.isFinite(requested) || !Number.isInteger(requested)) {
|
|
2915
|
+
return RECALL_DEBUG_ARTIFACT_DEFAULT_TOP_K;
|
|
2916
|
+
}
|
|
2917
|
+
if (requested < 1) {
|
|
2918
|
+
return 1;
|
|
2919
|
+
}
|
|
2920
|
+
if (requested > RECALL_DEBUG_ARTIFACT_MAX_TOP_K) {
|
|
2921
|
+
return RECALL_DEBUG_ARTIFACT_MAX_TOP_K;
|
|
2922
|
+
}
|
|
2923
|
+
return requested;
|
|
2924
|
+
}
|
|
2925
|
+
function buildTopCandidates(entryResults, reasonsByEntryId, topK) {
|
|
2926
|
+
const sliced = entryResults.slice(0, topK);
|
|
2927
|
+
return sliced.map((result) => {
|
|
2928
|
+
const reasons = reasonsByEntryId.get(result.entry.id);
|
|
2929
|
+
return {
|
|
2930
|
+
id: result.entry.id,
|
|
2931
|
+
score: result.score,
|
|
2932
|
+
lexicalScore: result.scores.lexical,
|
|
2933
|
+
vectorScore: result.scores.vector,
|
|
2934
|
+
recencyScore: result.scores.recency,
|
|
2935
|
+
importanceScore: result.scores.importance,
|
|
2936
|
+
...reasons && reasons.length > 0 ? { reasons } : {}
|
|
2937
|
+
};
|
|
2938
|
+
});
|
|
2939
|
+
}
|
|
2940
|
+
|
|
2941
|
+
// src/app/evals/recall/normalize-response.ts
|
|
2942
|
+
function buildRecallEvalSuccessResponse(params) {
|
|
2943
|
+
const entryResults = Array.isArray(params.results) ? params.results : params.results.entries;
|
|
2944
|
+
const projectedEntries = Array.isArray(params.results) ? entryResults.map((result) => projectClaimCentricRecallEntry(result, { asOf: params.request.recallRequest.asOf })) : params.results.projectedEntries;
|
|
2945
|
+
const metadata = buildMetadata(params.request, params.results, projectedEntries);
|
|
2946
|
+
const debugArtifact = params.request.options?.includeDebugArtifact === true && params.observedArtifactFacts ? buildRecallDebugArtifact({
|
|
2947
|
+
request: params.request,
|
|
2948
|
+
results: params.results,
|
|
2949
|
+
projectedEntries,
|
|
2950
|
+
sandbox: params.sandbox,
|
|
2951
|
+
observed: params.observedArtifactFacts
|
|
2952
|
+
}) : void 0;
|
|
2953
|
+
return {
|
|
2954
|
+
status: "ok",
|
|
2955
|
+
caseId: params.request.caseId,
|
|
2956
|
+
result: {
|
|
2957
|
+
entries: entryResults.map((result, index) => ({
|
|
2958
|
+
id: result.entry.id,
|
|
2959
|
+
subject: result.entry.subject,
|
|
2960
|
+
content: result.entry.content,
|
|
2961
|
+
type: result.entry.type,
|
|
2962
|
+
importance: result.entry.importance,
|
|
2963
|
+
expiry: result.entry.expiry,
|
|
2964
|
+
tags: result.entry.tags,
|
|
2965
|
+
created_at: result.entry.created_at,
|
|
2966
|
+
score: result.score,
|
|
2967
|
+
scores: result.scores,
|
|
2968
|
+
claim: {
|
|
2969
|
+
familyKey: projectedEntries[index]?.familyKey ?? `entry:${result.entry.id}`,
|
|
2970
|
+
claimKey: projectedEntries[index]?.claimKey,
|
|
2971
|
+
slotPolicy: projectedEntries[index]?.slotPolicy ?? "exclusive",
|
|
2972
|
+
memoryState: projectedEntries[index]?.memoryState ?? "current",
|
|
2973
|
+
claimStatus: projectedEntries[index]?.claimStatus ?? "no_key",
|
|
2974
|
+
freshness: projectedEntries[index]?.freshness ?? {
|
|
2975
|
+
createdAt: result.entry.created_at,
|
|
2976
|
+
isCurrent: true,
|
|
2977
|
+
label: `created ${result.entry.created_at} | current state`
|
|
2978
|
+
},
|
|
2979
|
+
provenance: projectedEntries[index]?.provenance ?? {},
|
|
2980
|
+
whySurfaced: projectedEntries[index]?.whySurfaced ?? {
|
|
2981
|
+
summary: `ranked score ${result.score.toFixed(2)}`,
|
|
2982
|
+
reasons: []
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2985
|
+
})),
|
|
2986
|
+
entryIds: entryResults.map((result) => result.entry.id)
|
|
2987
|
+
},
|
|
2988
|
+
metadata,
|
|
2989
|
+
diagnostics: params.diagnostics,
|
|
2990
|
+
timings: params.timings,
|
|
2991
|
+
sandbox: buildSandboxResult3(params.sandbox),
|
|
2992
|
+
...debugArtifact ? { debugArtifact } : {}
|
|
2993
|
+
};
|
|
2994
|
+
}
|
|
2995
|
+
function buildRecallEvalErrorResponse(params) {
|
|
2996
|
+
return {
|
|
2997
|
+
status: "error",
|
|
2998
|
+
caseId: params.request.caseId,
|
|
2999
|
+
error: {
|
|
3000
|
+
code: params.code,
|
|
3001
|
+
message: params.message,
|
|
3002
|
+
details: params.details
|
|
3003
|
+
},
|
|
3004
|
+
diagnostics: params.diagnostics,
|
|
3005
|
+
timings: params.timings,
|
|
3006
|
+
sandbox: params.sandbox ? buildSandboxResult3(params.sandbox) : void 0
|
|
3007
|
+
};
|
|
3008
|
+
}
|
|
3009
|
+
function buildSandboxResult3(sandbox) {
|
|
3010
|
+
return {
|
|
3011
|
+
root: sandbox.root,
|
|
3012
|
+
dbPath: sandbox.dbPath,
|
|
3013
|
+
preserved: sandbox.preserved,
|
|
3014
|
+
...sandbox.snapshot ? { snapshot: sandbox.snapshot } : {}
|
|
3015
|
+
};
|
|
3016
|
+
}
|
|
3017
|
+
function buildMetadata(request, results, projectedEntries) {
|
|
3018
|
+
if (Array.isArray(results)) {
|
|
3019
|
+
return {
|
|
3020
|
+
path: request.recallPath ?? "core",
|
|
3021
|
+
claim: {
|
|
3022
|
+
projectedEntries: projectedEntries.map(buildProjectedEntryMetadata)
|
|
3023
|
+
}
|
|
3024
|
+
};
|
|
3025
|
+
}
|
|
3026
|
+
return {
|
|
3027
|
+
path: "unified",
|
|
3028
|
+
claim: {
|
|
3029
|
+
projectedEntries: projectedEntries.map(buildProjectedEntryMetadata),
|
|
3030
|
+
entryFamilies: results.entryFamilies.map(buildClaimFamilyMetadata),
|
|
3031
|
+
transitions: results.claimTransitions
|
|
3032
|
+
},
|
|
3033
|
+
unified: {
|
|
3034
|
+
routing: results.routing,
|
|
3035
|
+
timeWindow: results.timeWindow,
|
|
3036
|
+
asOf: results.asOf,
|
|
3037
|
+
procedure: results.procedure ? {
|
|
3038
|
+
id: results.procedure.id,
|
|
3039
|
+
procedureKey: results.procedure.procedure_key,
|
|
3040
|
+
title: results.procedure.title,
|
|
3041
|
+
goal: results.procedure.goal
|
|
3042
|
+
} : void 0,
|
|
3043
|
+
procedureCandidates: results.procedureCandidates.map((candidate) => ({
|
|
3044
|
+
id: candidate.procedure.id,
|
|
3045
|
+
procedureKey: candidate.procedure.procedure_key,
|
|
3046
|
+
title: candidate.procedure.title,
|
|
3047
|
+
score: candidate.score,
|
|
3048
|
+
lexicalScore: candidate.scores.lexical,
|
|
3049
|
+
vectorScore: candidate.scores.vector
|
|
3050
|
+
})),
|
|
3051
|
+
procedureNotices: results.procedureNotices,
|
|
3052
|
+
notices: results.notices,
|
|
3053
|
+
episodeCount: results.episodes.length
|
|
3054
|
+
}
|
|
3055
|
+
};
|
|
3056
|
+
}
|
|
3057
|
+
function buildProjectedEntryMetadata(entry) {
|
|
3058
|
+
return {
|
|
3059
|
+
entryId: entry.entryId,
|
|
3060
|
+
familyKey: entry.familyKey,
|
|
3061
|
+
claimKey: entry.claimKey,
|
|
3062
|
+
slotPolicy: entry.slotPolicy,
|
|
3063
|
+
memoryState: entry.memoryState,
|
|
3064
|
+
claimStatus: entry.claimStatus,
|
|
3065
|
+
freshness: entry.freshness,
|
|
3066
|
+
provenance: entry.provenance,
|
|
3067
|
+
whySurfaced: entry.whySurfaced
|
|
3068
|
+
};
|
|
3069
|
+
}
|
|
3070
|
+
function buildClaimFamilyMetadata(family) {
|
|
3071
|
+
return {
|
|
3072
|
+
familyKey: family.familyKey,
|
|
3073
|
+
claimKey: family.claimKey,
|
|
3074
|
+
slotPolicy: family.slotPolicy,
|
|
3075
|
+
subject: family.subject,
|
|
3076
|
+
primaryEntryId: family.primary.entryId,
|
|
3077
|
+
entries: family.entries.map((entry) => ({
|
|
3078
|
+
id: entry.entryId,
|
|
3079
|
+
memoryState: entry.memoryState,
|
|
3080
|
+
claimStatus: entry.claimStatus
|
|
3081
|
+
}))
|
|
3082
|
+
};
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
// src/app/evals/recall/run-recall-eval-case.ts
|
|
3086
|
+
async function runRecallEvalCase(request, dependencies = {}) {
|
|
3087
|
+
const startedAt = Date.now();
|
|
3088
|
+
const provisionedAt = new Date(startedAt).toISOString();
|
|
3089
|
+
const diagnostics = createRecallEvalDiagnosticsCollector(request);
|
|
3090
|
+
const recallPath = request.recallPath ?? "core";
|
|
3091
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
3092
|
+
const evalNow = parseEvalNow(ablation.now);
|
|
3093
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
3094
|
+
let sandbox;
|
|
3095
|
+
try {
|
|
3096
|
+
const sandboxStartedAt = Date.now();
|
|
3097
|
+
try {
|
|
3098
|
+
sandbox = await setupRecallEvalSandbox(request.sandbox);
|
|
3099
|
+
diagnostics.recordSandboxSetup(elapsedMs4(sandboxStartedAt));
|
|
3100
|
+
} catch (error) {
|
|
3101
|
+
diagnostics.recordSandboxSetup(elapsedMs4(sandboxStartedAt));
|
|
3102
|
+
return buildRecallEvalErrorResponse({
|
|
3103
|
+
request,
|
|
3104
|
+
code: "sandbox_setup_failed",
|
|
3105
|
+
message: "Failed to create isolated recall eval sandbox.",
|
|
3106
|
+
details: toErrorDetails3(error),
|
|
3107
|
+
diagnostics: diagnostics.buildDiagnostics(),
|
|
3108
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt))
|
|
3109
|
+
});
|
|
3110
|
+
}
|
|
3111
|
+
if (request.memoryPool.length > 0 || (request.procedurePool?.length ?? 0) > 0 || shouldProvisionProfileSnapshot(ablation)) {
|
|
3112
|
+
const provisionStartedAt = Date.now();
|
|
3113
|
+
try {
|
|
3114
|
+
const provisionResult = await provisionEvalSandbox({
|
|
3115
|
+
caseId: request.caseId,
|
|
3116
|
+
sandbox,
|
|
3117
|
+
memoryPool: request.memoryPool,
|
|
3118
|
+
procedurePool: request.procedurePool,
|
|
3119
|
+
profileSnapshot: shouldProvisionProfileSnapshot(ablation) ? ablation.profileSnapshot : void 0,
|
|
3120
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.requirePort() : void 0,
|
|
3121
|
+
provisionedAt
|
|
3122
|
+
});
|
|
3123
|
+
if (provisionResult.entryProvisionResult) {
|
|
3124
|
+
diagnostics.recordProvision(provisionResult.entryProvisionResult, elapsedMs4(provisionStartedAt));
|
|
3125
|
+
} else {
|
|
3126
|
+
diagnostics.recordFixtureProvisionTiming(elapsedMs4(provisionStartedAt));
|
|
3127
|
+
}
|
|
3128
|
+
} catch (error) {
|
|
3129
|
+
diagnostics.recordFixtureProvisionTiming(elapsedMs4(provisionStartedAt));
|
|
3130
|
+
return buildRecallEvalErrorResponse({
|
|
3131
|
+
request,
|
|
3132
|
+
code: "fixture_provision_failed",
|
|
3133
|
+
message: "Failed to provision recall eval fixtures into isolated storage.",
|
|
3134
|
+
details: toErrorDetails3(error),
|
|
3135
|
+
diagnostics: diagnostics.buildDiagnostics(),
|
|
3136
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
3137
|
+
sandbox
|
|
3138
|
+
});
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
const recallStartedAt = Date.now();
|
|
3142
|
+
try {
|
|
3143
|
+
if (isMemoryOffArm(ablation)) {
|
|
3144
|
+
diagnostics.recordRecall(elapsedMs4(recallStartedAt));
|
|
3145
|
+
return buildRecallEvalSuccessResponse({
|
|
3146
|
+
request,
|
|
3147
|
+
results: [],
|
|
3148
|
+
diagnostics: diagnostics.buildDiagnostics(),
|
|
3149
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
3150
|
+
sandbox
|
|
3151
|
+
});
|
|
3152
|
+
}
|
|
3153
|
+
const activeSandbox = sandbox;
|
|
3154
|
+
if (!activeSandbox) {
|
|
3155
|
+
throw new Error("Recall eval sandbox was not initialized.");
|
|
3156
|
+
}
|
|
3157
|
+
const embeddingSupport = embeddingResolver.getSupport();
|
|
3158
|
+
const recallEmbeddingPort = request.options?.faultInjection?.queryEmbeddingFailure === true ? createUnavailableEmbeddingPort("Injected recall eval query embedding failure.") : embeddingSupport.port ?? createUnavailableEmbeddingPort(embeddingSupport.error ?? "Embeddings are unavailable.");
|
|
3159
|
+
const sandboxPorts = activeSandbox.createRecallPorts(recallEmbeddingPort);
|
|
3160
|
+
const portsWithCrossEncoder = attachCrossEncoderPort(sandboxPorts, dependencies.crossEncoder);
|
|
3161
|
+
const telemetryGatedPorts = applyTelemetryWriteGate(portsWithCrossEncoder, activeSandbox);
|
|
3162
|
+
const basePorts = applyRecallEvalFaultInjection(telemetryGatedPorts, request);
|
|
3163
|
+
const recallPorts = diagnostics.isObservationEnabled() ? createInstrumentedRecallPorts(basePorts, diagnostics) : basePorts;
|
|
3164
|
+
const slotPolicyConfig = request.unified?.memoryPolicy?.slotPolicies;
|
|
3165
|
+
const rankingPolicy = request.recallRequest.rankingPolicy;
|
|
3166
|
+
const unifiedRecallOptions = {
|
|
3167
|
+
...slotPolicyConfig ? { slotPolicyConfig } : {},
|
|
3168
|
+
...rankingPolicy ? { rankingPolicy } : {},
|
|
3169
|
+
...diagnostics.isObservationEnabled() ? { trace: diagnostics.traceSink } : {}
|
|
3170
|
+
};
|
|
3171
|
+
const coreRecallOptions = {
|
|
3172
|
+
...rankingPolicy ? { rankingPolicy } : {},
|
|
3173
|
+
...diagnostics.isObservationEnabled() ? { trace: diagnostics.traceSink } : {}
|
|
3174
|
+
};
|
|
3175
|
+
const results = recallPath === "unified" ? await runUnifiedRecall(
|
|
3176
|
+
{
|
|
3177
|
+
text: request.recallRequest.text,
|
|
3178
|
+
...request.unified?.mode ? { mode: request.unified.mode } : {},
|
|
3179
|
+
...request.recallRequest.limit !== void 0 ? { limit: request.recallRequest.limit } : {},
|
|
3180
|
+
...request.recallRequest.threshold !== void 0 ? { threshold: request.recallRequest.threshold } : {},
|
|
3181
|
+
...request.recallRequest.types && request.recallRequest.types.length > 0 ? { types: request.recallRequest.types } : {},
|
|
3182
|
+
...request.recallRequest.tags && request.recallRequest.tags.length > 0 ? { tags: request.recallRequest.tags } : {},
|
|
3183
|
+
...request.recallRequest.asOf ? { asOf: request.recallRequest.asOf } : {},
|
|
3184
|
+
...request.unified?.sessionKey ? { sessionKey: request.unified.sessionKey } : {}
|
|
3185
|
+
},
|
|
3186
|
+
{
|
|
3187
|
+
database: activeSandbox.episodeDatabase,
|
|
3188
|
+
procedures: activeSandbox.procedureDatabase,
|
|
3189
|
+
recall: recallPorts,
|
|
3190
|
+
embeddingAvailable: embeddingSupport.available,
|
|
3191
|
+
...embeddingSupport.error ? { embeddingError: embeddingSupport.error } : {},
|
|
3192
|
+
...slotPolicyConfig ? { claimSlotPolicyConfig: slotPolicyConfig } : {},
|
|
3193
|
+
...evalNow ? { now: evalNow } : {},
|
|
3194
|
+
...embeddingSupport.available ? {
|
|
3195
|
+
embedQuery: async (text) => {
|
|
3196
|
+
const vectors = await recallEmbeddingPort.embed([text]);
|
|
3197
|
+
return vectors[0] ?? [];
|
|
3198
|
+
}
|
|
3199
|
+
} : {},
|
|
3200
|
+
...Object.keys(unifiedRecallOptions).length > 0 ? { recallOptions: unifiedRecallOptions } : {}
|
|
3201
|
+
}
|
|
3202
|
+
) : await recall(request.recallRequest, recallPorts, {
|
|
3203
|
+
...Object.keys(coreRecallOptions).length > 0 ? coreRecallOptions : {},
|
|
3204
|
+
...evalNow ? { now: evalNow } : {}
|
|
3205
|
+
});
|
|
3206
|
+
diagnostics.recordRecall(elapsedMs4(recallStartedAt));
|
|
3207
|
+
return buildRecallEvalSuccessResponse({
|
|
3208
|
+
request,
|
|
3209
|
+
results,
|
|
3210
|
+
diagnostics: diagnostics.buildDiagnostics(),
|
|
3211
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
3212
|
+
sandbox: activeSandbox,
|
|
3213
|
+
observedArtifactFacts: request.options?.includeDebugArtifact === true ? diagnostics.buildObservedArtifactFacts() : void 0
|
|
3214
|
+
});
|
|
3215
|
+
} catch (error) {
|
|
3216
|
+
diagnostics.recordRecall(elapsedMs4(recallStartedAt));
|
|
3217
|
+
return buildRecallEvalErrorResponse({
|
|
3218
|
+
request,
|
|
3219
|
+
code: "recall_execution_failed",
|
|
3220
|
+
message: "Failed to execute real recall against isolated eval state.",
|
|
3221
|
+
details: toErrorDetails3(error),
|
|
3222
|
+
diagnostics: diagnostics.buildDiagnostics(),
|
|
3223
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
3224
|
+
sandbox
|
|
3225
|
+
});
|
|
3226
|
+
}
|
|
3227
|
+
} catch (error) {
|
|
3228
|
+
return buildRecallEvalErrorResponse({
|
|
3229
|
+
request,
|
|
3230
|
+
code: "internal_error",
|
|
3231
|
+
message: "Recall eval execution failed unexpectedly.",
|
|
3232
|
+
details: toErrorDetails3(error),
|
|
3233
|
+
diagnostics: diagnostics.buildDiagnostics(),
|
|
3234
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
3235
|
+
sandbox
|
|
3236
|
+
});
|
|
3237
|
+
} finally {
|
|
3238
|
+
await sandbox?.cleanup().catch(() => void 0);
|
|
3239
|
+
}
|
|
3240
|
+
}
|
|
3241
|
+
function toErrorDetails3(error) {
|
|
3242
|
+
if (error instanceof Error) {
|
|
3243
|
+
return {
|
|
3244
|
+
cause: error.message
|
|
3245
|
+
};
|
|
3246
|
+
}
|
|
3247
|
+
return {
|
|
3248
|
+
cause: String(error)
|
|
3249
|
+
};
|
|
3250
|
+
}
|
|
3251
|
+
function applyRecallEvalFaultInjection(ports, request) {
|
|
3252
|
+
if (request.options?.faultInjection?.vectorSearchFailure !== true) {
|
|
3253
|
+
return ports;
|
|
3254
|
+
}
|
|
3255
|
+
return {
|
|
3256
|
+
async embed(text) {
|
|
3257
|
+
return ports.embed(text);
|
|
3258
|
+
},
|
|
3259
|
+
async vectorSearch() {
|
|
3260
|
+
throw new Error("Injected recall eval vector search failure.");
|
|
3261
|
+
},
|
|
3262
|
+
async ftsSearch(params) {
|
|
3263
|
+
return ports.ftsSearch(params);
|
|
3264
|
+
},
|
|
3265
|
+
...ports.expandNeighborhood ? {
|
|
3266
|
+
async expandNeighborhood(request2) {
|
|
3267
|
+
return ports.expandNeighborhood(request2);
|
|
3268
|
+
}
|
|
3269
|
+
} : {},
|
|
3270
|
+
// Cross-encoder remains wired during fault injection so rerank-aware
|
|
3271
|
+
// cases can still observe the rerank stage running after the core
|
|
3272
|
+
// recall pipeline falls back to lexical-only retrieval. Preserving
|
|
3273
|
+
// the port is safe because the core helper fails closed on adapter
|
|
3274
|
+
// errors and honors the ranking policy kill switch.
|
|
3275
|
+
...ports.crossEncoder ? {
|
|
3276
|
+
crossEncoder: ports.crossEncoder
|
|
3277
|
+
} : {},
|
|
3278
|
+
async hydrateEntries(ids) {
|
|
3279
|
+
return ports.hydrateEntries(ids);
|
|
3280
|
+
},
|
|
3281
|
+
async recordRecallEvents(params) {
|
|
3282
|
+
return ports.recordRecallEvents(params);
|
|
3283
|
+
}
|
|
3284
|
+
};
|
|
3285
|
+
}
|
|
3286
|
+
function elapsedMs4(startedAt) {
|
|
3287
|
+
return Math.max(0, Date.now() - startedAt);
|
|
3288
|
+
}
|
|
3289
|
+
|
|
3290
|
+
// src/adapters/api/validation/recall-eval-request.ts
|
|
3291
|
+
var ROOT_REQUEST_KEYS3 = /* @__PURE__ */ new Set(["caseId", "description", "recallPath", "sandbox", "memoryPool", "recallRequest", "unified", "options"]);
|
|
3292
|
+
var RECALL_REQUEST_KEYS = /* @__PURE__ */ new Set([
|
|
3293
|
+
"text",
|
|
3294
|
+
"limit",
|
|
3295
|
+
"threshold",
|
|
3296
|
+
"budget",
|
|
3297
|
+
"types",
|
|
3298
|
+
"tags",
|
|
3299
|
+
"since",
|
|
3300
|
+
"until",
|
|
3301
|
+
"around",
|
|
3302
|
+
"aroundRadius",
|
|
3303
|
+
"asOf",
|
|
3304
|
+
"rankingProfile",
|
|
3305
|
+
"rankingPolicy"
|
|
3306
|
+
]);
|
|
3307
|
+
var RANKING_POLICY_KEYS = /* @__PURE__ */ new Set([
|
|
3308
|
+
"rrf",
|
|
3309
|
+
"rrfRankConstant",
|
|
3310
|
+
"rrfSmallPoolRankConstant",
|
|
3311
|
+
"neighborhood",
|
|
3312
|
+
"mmr",
|
|
3313
|
+
"mmrLambda",
|
|
3314
|
+
"mmrMinPoolSize",
|
|
3315
|
+
"crossEncoder",
|
|
3316
|
+
"crossEncoderTopK",
|
|
3317
|
+
"crossEncoderAlpha"
|
|
3318
|
+
]);
|
|
3319
|
+
var UNIFIED_REQUEST_KEYS = /* @__PURE__ */ new Set(["mode", "sessionKey", "memoryPolicy"]);
|
|
3320
|
+
var UNIFIED_MEMORY_POLICY_KEYS = /* @__PURE__ */ new Set(["slotPolicies"]);
|
|
3321
|
+
var SLOT_POLICY_KEYS = /* @__PURE__ */ new Set(["attributeHeads"]);
|
|
3322
|
+
var OPTIONS_KEYS3 = /* @__PURE__ */ new Set(["includeDiagnostics", "includeCandidates", "includeTimings", "includeDebugArtifact", "topKCandidates", "faultInjection"]);
|
|
3323
|
+
var FAULT_INJECTION_KEYS = /* @__PURE__ */ new Set(["queryEmbeddingFailure", "vectorSearchFailure"]);
|
|
3324
|
+
var RECALL_PATHS = ["core", "unified"];
|
|
3325
|
+
var RECALL_RANKING_PROFILES = ["historical_state"];
|
|
3326
|
+
var RANKING_POLICY_TOGGLES = ["enabled", "disabled"];
|
|
3327
|
+
var UNIFIED_RECALL_MODES = ["auto", "durables", "episodes"];
|
|
3328
|
+
var CLAIM_SLOT_POLICIES = ["exclusive", "multivalued"];
|
|
3329
|
+
var RecallEvalRequestValidationError = class extends Error {
|
|
3330
|
+
/** Parseable case identifier echoed for invalid request correlation when available. */
|
|
3331
|
+
caseId;
|
|
3332
|
+
/** Structured list of request validation issues. */
|
|
3333
|
+
issues;
|
|
3334
|
+
/**
|
|
3335
|
+
* Creates a request validation error with stable issue details.
|
|
3336
|
+
*
|
|
3337
|
+
* @param issues - Structured validation issues collected during parsing.
|
|
3338
|
+
* @param caseId - Parseable request case identifier when available.
|
|
3339
|
+
*/
|
|
3340
|
+
constructor(issues, caseId) {
|
|
3341
|
+
super("Invalid recall eval request.");
|
|
3342
|
+
this.name = "RecallEvalRequestValidationError";
|
|
3343
|
+
this.issues = issues;
|
|
3344
|
+
this.caseId = caseId;
|
|
3345
|
+
}
|
|
3346
|
+
};
|
|
3347
|
+
function parseRecallEvalCaseRequest(input) {
|
|
3348
|
+
const caseId = extractParseableCaseId(input);
|
|
3349
|
+
if (!isRecord(input)) {
|
|
3350
|
+
throw new RecallEvalRequestValidationError(
|
|
3351
|
+
[
|
|
3352
|
+
{
|
|
3353
|
+
path: "$",
|
|
3354
|
+
message: "Request body must be a JSON object."
|
|
3355
|
+
}
|
|
3356
|
+
],
|
|
3357
|
+
caseId
|
|
3358
|
+
);
|
|
3359
|
+
}
|
|
3360
|
+
const issues = [];
|
|
3361
|
+
pushUnexpectedFields(input, ROOT_REQUEST_KEYS3, "", issues);
|
|
3362
|
+
const parsedCaseId = parseRequiredTrimmedString(input.caseId, "caseId", issues);
|
|
3363
|
+
const description = parseOptionalTrimmedString(input.description, "description", issues);
|
|
3364
|
+
const recallPath = parseOptionalRecallPath(input.recallPath, "recallPath", issues);
|
|
3365
|
+
const sandbox = parseSandbox(input.sandbox, issues);
|
|
3366
|
+
const memoryPool = parseMemoryPool(input.memoryPool, issues);
|
|
3367
|
+
const recallRequest = parseRecallRequest(input.recallRequest, issues);
|
|
3368
|
+
const unified = parseUnifiedRequest(input.unified, issues);
|
|
3369
|
+
const options = parseOptions3(input.options, issues);
|
|
3370
|
+
validatePathSpecificRequest(recallPath, recallRequest, unified, issues);
|
|
3371
|
+
if (issues.length > 0 || parsedCaseId === void 0 || memoryPool === void 0 || recallRequest === void 0) {
|
|
3372
|
+
throw new RecallEvalRequestValidationError(issues, caseId);
|
|
3373
|
+
}
|
|
3374
|
+
return {
|
|
3375
|
+
caseId: parsedCaseId,
|
|
3376
|
+
description,
|
|
3377
|
+
recallPath,
|
|
3378
|
+
sandbox,
|
|
3379
|
+
memoryPool,
|
|
3380
|
+
recallRequest,
|
|
3381
|
+
unified,
|
|
3382
|
+
options
|
|
3383
|
+
};
|
|
3384
|
+
}
|
|
3385
|
+
function mapRecallEvalCaseRequestDto(dto) {
|
|
3386
|
+
return {
|
|
3387
|
+
caseId: dto.caseId,
|
|
3388
|
+
description: dto.description,
|
|
3389
|
+
recallPath: dto.recallPath,
|
|
3390
|
+
sandbox: mapSandboxRequestDto2(dto.sandbox),
|
|
3391
|
+
memoryPool: dto.memoryPool.map(mapFixtureEntryDto2),
|
|
3392
|
+
recallRequest: mapRecallRequestDto(dto.recallRequest),
|
|
3393
|
+
unified: mapUnifiedRequestDto(dto.unified),
|
|
3394
|
+
options: mapCaseOptionsDto3(dto.options)
|
|
3395
|
+
};
|
|
3396
|
+
}
|
|
3397
|
+
function parseRecallRequest(value, issues) {
|
|
3398
|
+
const recallRequest = parseObject(value, "recallRequest", issues);
|
|
3399
|
+
if (recallRequest === void 0) {
|
|
3400
|
+
return void 0;
|
|
3401
|
+
}
|
|
3402
|
+
pushUnexpectedFields(recallRequest, RECALL_REQUEST_KEYS, "recallRequest", issues);
|
|
3403
|
+
const text = parseRequiredTrimmedString(recallRequest.text, "recallRequest.text", issues);
|
|
3404
|
+
if (text === void 0) {
|
|
3405
|
+
return void 0;
|
|
3406
|
+
}
|
|
3407
|
+
return {
|
|
3408
|
+
text,
|
|
3409
|
+
limit: parseOptionalIntegerInRange(recallRequest.limit, "recallRequest.limit", issues, {
|
|
3410
|
+
min: 0
|
|
3411
|
+
}),
|
|
3412
|
+
threshold: parseOptionalThreshold(recallRequest.threshold, "recallRequest.threshold", issues),
|
|
3413
|
+
budget: parseOptionalIntegerInRange(recallRequest.budget, "recallRequest.budget", issues, {
|
|
3414
|
+
min: 0
|
|
3415
|
+
}),
|
|
3416
|
+
types: parseOptionalDurableKindArray(recallRequest.types, "recallRequest.types", issues),
|
|
3417
|
+
tags: parseOptionalStringArray(recallRequest.tags, "recallRequest.tags", issues),
|
|
3418
|
+
since: parseOptionalTrimmedString(recallRequest.since, "recallRequest.since", issues),
|
|
3419
|
+
until: parseOptionalTrimmedString(recallRequest.until, "recallRequest.until", issues),
|
|
3420
|
+
around: parseOptionalTrimmedString(recallRequest.around, "recallRequest.around", issues),
|
|
3421
|
+
aroundRadius: parseOptionalIntegerInRange(recallRequest.aroundRadius, "recallRequest.aroundRadius", issues, {
|
|
3422
|
+
min: 1
|
|
3423
|
+
}),
|
|
3424
|
+
asOf: parseOptionalTrimmedString(recallRequest.asOf, "recallRequest.asOf", issues),
|
|
3425
|
+
rankingProfile: parseOptionalRankingProfile(recallRequest.rankingProfile, "recallRequest.rankingProfile", issues),
|
|
3426
|
+
rankingPolicy: parseOptionalRankingPolicy(recallRequest.rankingPolicy, "recallRequest.rankingPolicy", issues)
|
|
3427
|
+
};
|
|
3428
|
+
}
|
|
3429
|
+
function parseOptionalRankingPolicy(value, path2, issues) {
|
|
3430
|
+
if (value === void 0) {
|
|
3431
|
+
return void 0;
|
|
3432
|
+
}
|
|
3433
|
+
const policy = parseObject(value, path2, issues);
|
|
3434
|
+
if (policy === void 0) {
|
|
3435
|
+
return void 0;
|
|
3436
|
+
}
|
|
3437
|
+
pushUnexpectedFields(policy, RANKING_POLICY_KEYS, path2, issues);
|
|
3438
|
+
const parsed = {};
|
|
3439
|
+
const rrf = parseOptionalStageToggle(policy.rrf, `${path2}.rrf`, issues);
|
|
3440
|
+
if (rrf !== void 0) {
|
|
3441
|
+
parsed.rrf = rrf;
|
|
3442
|
+
}
|
|
3443
|
+
const rrfRankConstant = parseOptionalIntegerInRange(policy.rrfRankConstant, `${path2}.rrfRankConstant`, issues, {
|
|
3444
|
+
min: 1
|
|
3445
|
+
});
|
|
3446
|
+
if (rrfRankConstant !== void 0) {
|
|
3447
|
+
parsed.rrfRankConstant = rrfRankConstant;
|
|
3448
|
+
}
|
|
3449
|
+
const rrfSmallPoolRankConstant = parseOptionalIntegerInRange(policy.rrfSmallPoolRankConstant, `${path2}.rrfSmallPoolRankConstant`, issues, { min: 1 });
|
|
3450
|
+
if (rrfSmallPoolRankConstant !== void 0) {
|
|
3451
|
+
parsed.rrfSmallPoolRankConstant = rrfSmallPoolRankConstant;
|
|
3452
|
+
}
|
|
3453
|
+
const neighborhood = parseOptionalStageToggle(policy.neighborhood, `${path2}.neighborhood`, issues);
|
|
3454
|
+
if (neighborhood !== void 0) {
|
|
3455
|
+
parsed.neighborhood = neighborhood;
|
|
3456
|
+
}
|
|
3457
|
+
const mmr = parseOptionalStageToggle(policy.mmr, `${path2}.mmr`, issues);
|
|
3458
|
+
if (mmr !== void 0) {
|
|
3459
|
+
parsed.mmr = mmr;
|
|
3460
|
+
}
|
|
3461
|
+
const mmrLambda = parseOptionalUnitInterval(policy.mmrLambda, `${path2}.mmrLambda`, issues);
|
|
3462
|
+
if (mmrLambda !== void 0) {
|
|
3463
|
+
parsed.mmrLambda = mmrLambda;
|
|
3464
|
+
}
|
|
3465
|
+
const mmrMinPoolSize = parseOptionalIntegerInRange(policy.mmrMinPoolSize, `${path2}.mmrMinPoolSize`, issues, {
|
|
3466
|
+
min: 0
|
|
3467
|
+
});
|
|
3468
|
+
if (mmrMinPoolSize !== void 0) {
|
|
3469
|
+
parsed.mmrMinPoolSize = mmrMinPoolSize;
|
|
3470
|
+
}
|
|
3471
|
+
const crossEncoder = parseOptionalStageToggle(policy.crossEncoder, `${path2}.crossEncoder`, issues);
|
|
3472
|
+
if (crossEncoder !== void 0) {
|
|
3473
|
+
parsed.crossEncoder = crossEncoder;
|
|
3474
|
+
}
|
|
3475
|
+
const crossEncoderTopK = parseOptionalIntegerInRange(policy.crossEncoderTopK, `${path2}.crossEncoderTopK`, issues, {
|
|
3476
|
+
min: 1
|
|
3477
|
+
});
|
|
3478
|
+
if (crossEncoderTopK !== void 0) {
|
|
3479
|
+
parsed.crossEncoderTopK = crossEncoderTopK;
|
|
3480
|
+
}
|
|
3481
|
+
const crossEncoderAlpha = parseOptionalUnitInterval(policy.crossEncoderAlpha, `${path2}.crossEncoderAlpha`, issues);
|
|
3482
|
+
if (crossEncoderAlpha !== void 0) {
|
|
3483
|
+
parsed.crossEncoderAlpha = crossEncoderAlpha;
|
|
3484
|
+
}
|
|
3485
|
+
return Object.keys(parsed).length > 0 ? parsed : void 0;
|
|
3486
|
+
}
|
|
3487
|
+
function parseOptionalStageToggle(value, path2, issues) {
|
|
3488
|
+
if (value === void 0) {
|
|
3489
|
+
return void 0;
|
|
3490
|
+
}
|
|
3491
|
+
if (typeof value !== "string" || !RANKING_POLICY_TOGGLES.includes(value)) {
|
|
3492
|
+
pushIssue(issues, path2, `Expected one of: ${RANKING_POLICY_TOGGLES.join(", ")}.`);
|
|
3493
|
+
return void 0;
|
|
3494
|
+
}
|
|
3495
|
+
return value;
|
|
3496
|
+
}
|
|
3497
|
+
function parseOptionalUnitInterval(value, path2, issues) {
|
|
3498
|
+
if (value === void 0) {
|
|
3499
|
+
return void 0;
|
|
3500
|
+
}
|
|
3501
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
|
|
3502
|
+
pushIssue(issues, path2, "Expected a number from 0 to 1.");
|
|
3503
|
+
return void 0;
|
|
3504
|
+
}
|
|
3505
|
+
return value;
|
|
3506
|
+
}
|
|
3507
|
+
function parseUnifiedRequest(value, issues) {
|
|
3508
|
+
if (value === void 0) {
|
|
3509
|
+
return void 0;
|
|
3510
|
+
}
|
|
3511
|
+
const unified = parseObject(value, "unified", issues);
|
|
3512
|
+
if (unified === void 0) {
|
|
3513
|
+
return void 0;
|
|
3514
|
+
}
|
|
3515
|
+
pushUnexpectedFields(unified, UNIFIED_REQUEST_KEYS, "unified", issues);
|
|
3516
|
+
return {
|
|
3517
|
+
mode: parseOptionalUnifiedRecallMode(unified.mode, "unified.mode", issues),
|
|
3518
|
+
sessionKey: parseOptionalTrimmedString(unified.sessionKey, "unified.sessionKey", issues),
|
|
3519
|
+
memoryPolicy: parseUnifiedMemoryPolicy(unified.memoryPolicy, issues)
|
|
3520
|
+
};
|
|
3521
|
+
}
|
|
3522
|
+
function parseOptions3(value, issues) {
|
|
3523
|
+
if (value === void 0) {
|
|
3524
|
+
return void 0;
|
|
3525
|
+
}
|
|
3526
|
+
const options = parseObject(value, "options", issues);
|
|
3527
|
+
if (options === void 0) {
|
|
3528
|
+
return void 0;
|
|
3529
|
+
}
|
|
3530
|
+
pushUnexpectedFields(options, OPTIONS_KEYS3, "options", issues);
|
|
3531
|
+
return {
|
|
3532
|
+
includeDiagnostics: parseOptionalBoolean(options.includeDiagnostics, "options.includeDiagnostics", issues),
|
|
3533
|
+
includeCandidates: parseOptionalBoolean(options.includeCandidates, "options.includeCandidates", issues),
|
|
3534
|
+
includeTimings: parseOptionalBoolean(options.includeTimings, "options.includeTimings", issues),
|
|
3535
|
+
includeDebugArtifact: parseOptionalBoolean(options.includeDebugArtifact, "options.includeDebugArtifact", issues),
|
|
3536
|
+
topKCandidates: parseOptionalIntegerInRange(options.topKCandidates, "options.topKCandidates", issues, {
|
|
3537
|
+
min: 1,
|
|
3538
|
+
max: RECALL_DEBUG_ARTIFACT_MAX_TOP_K
|
|
3539
|
+
}),
|
|
3540
|
+
faultInjection: parseFaultInjection(options.faultInjection, issues)
|
|
3541
|
+
};
|
|
3542
|
+
}
|
|
3543
|
+
function parseFaultInjection(value, issues) {
|
|
3544
|
+
if (value === void 0) {
|
|
3545
|
+
return void 0;
|
|
3546
|
+
}
|
|
3547
|
+
const faultInjection = parseObject(value, "options.faultInjection", issues);
|
|
3548
|
+
if (faultInjection === void 0) {
|
|
3549
|
+
return void 0;
|
|
3550
|
+
}
|
|
3551
|
+
pushUnexpectedFields(faultInjection, FAULT_INJECTION_KEYS, "options.faultInjection", issues);
|
|
3552
|
+
return {
|
|
3553
|
+
queryEmbeddingFailure: parseOptionalBoolean(faultInjection.queryEmbeddingFailure, "options.faultInjection.queryEmbeddingFailure", issues),
|
|
3554
|
+
vectorSearchFailure: parseOptionalBoolean(faultInjection.vectorSearchFailure, "options.faultInjection.vectorSearchFailure", issues)
|
|
3555
|
+
};
|
|
3556
|
+
}
|
|
3557
|
+
function parseUnifiedMemoryPolicy(value, issues) {
|
|
3558
|
+
if (value === void 0) {
|
|
3559
|
+
return void 0;
|
|
3560
|
+
}
|
|
3561
|
+
const memoryPolicy = parseObject(value, "unified.memoryPolicy", issues);
|
|
3562
|
+
if (memoryPolicy === void 0) {
|
|
3563
|
+
return void 0;
|
|
3564
|
+
}
|
|
3565
|
+
pushUnexpectedFields(memoryPolicy, UNIFIED_MEMORY_POLICY_KEYS, "unified.memoryPolicy", issues);
|
|
3566
|
+
return {
|
|
3567
|
+
slotPolicies: parseClaimSlotPolicyConfig(memoryPolicy.slotPolicies, "unified.memoryPolicy.slotPolicies", issues)
|
|
3568
|
+
};
|
|
3569
|
+
}
|
|
3570
|
+
function parseDurableKind2(value, path2, issues) {
|
|
3571
|
+
if (typeof value !== "string" || !DURABLE_KINDS.includes(value)) {
|
|
3572
|
+
pushIssue(issues, path2, `Expected one of: ${DURABLE_KINDS.join(", ")}.`);
|
|
3573
|
+
return void 0;
|
|
3574
|
+
}
|
|
3575
|
+
return value;
|
|
3576
|
+
}
|
|
3577
|
+
function parseOptionalRecallPath(value, path2, issues) {
|
|
3578
|
+
if (value === void 0) {
|
|
3579
|
+
return void 0;
|
|
3580
|
+
}
|
|
3581
|
+
if (typeof value !== "string" || !RECALL_PATHS.includes(value)) {
|
|
3582
|
+
pushIssue(issues, path2, `Expected one of: ${RECALL_PATHS.join(", ")}.`);
|
|
3583
|
+
return void 0;
|
|
3584
|
+
}
|
|
3585
|
+
return value;
|
|
3586
|
+
}
|
|
3587
|
+
function parseOptionalUnifiedRecallMode(value, path2, issues) {
|
|
3588
|
+
if (value === void 0) {
|
|
3589
|
+
return void 0;
|
|
3590
|
+
}
|
|
3591
|
+
if (typeof value !== "string" || !UNIFIED_RECALL_MODES.includes(value)) {
|
|
3592
|
+
pushIssue(issues, path2, `Expected one of: ${UNIFIED_RECALL_MODES.join(", ")}.`);
|
|
3593
|
+
return void 0;
|
|
3594
|
+
}
|
|
3595
|
+
return value;
|
|
3596
|
+
}
|
|
3597
|
+
function parseOptionalRankingProfile(value, path2, issues) {
|
|
3598
|
+
if (value === void 0) {
|
|
3599
|
+
return void 0;
|
|
3600
|
+
}
|
|
3601
|
+
if (typeof value !== "string" || !RECALL_RANKING_PROFILES.includes(value)) {
|
|
3602
|
+
pushIssue(issues, path2, `Expected one of: ${RECALL_RANKING_PROFILES.join(", ")}.`);
|
|
3603
|
+
return void 0;
|
|
3604
|
+
}
|
|
3605
|
+
return value;
|
|
3606
|
+
}
|
|
3607
|
+
function parseOptionalDurableKindArray(value, path2, issues) {
|
|
3608
|
+
if (value === void 0) {
|
|
3609
|
+
return void 0;
|
|
3610
|
+
}
|
|
3611
|
+
if (!Array.isArray(value)) {
|
|
3612
|
+
pushIssue(issues, path2, "Expected an array.");
|
|
3613
|
+
return void 0;
|
|
3614
|
+
}
|
|
3615
|
+
const parsed = [];
|
|
3616
|
+
for (const [index, item] of value.entries()) {
|
|
3617
|
+
const entryType = parseDurableKind2(item, `${path2}[${index}]`, issues);
|
|
3618
|
+
if (entryType !== void 0) {
|
|
3619
|
+
parsed.push(entryType);
|
|
3620
|
+
}
|
|
3621
|
+
}
|
|
3622
|
+
return parsed;
|
|
3623
|
+
}
|
|
3624
|
+
function parseClaimSlotPolicyConfig(value, path2, issues) {
|
|
3625
|
+
if (value === void 0) {
|
|
3626
|
+
return void 0;
|
|
3627
|
+
}
|
|
3628
|
+
const config = parseObject(value, path2, issues);
|
|
3629
|
+
if (config === void 0) {
|
|
3630
|
+
return void 0;
|
|
3631
|
+
}
|
|
3632
|
+
pushUnexpectedFields(config, SLOT_POLICY_KEYS, path2, issues);
|
|
3633
|
+
const attributeHeads = parseClaimSlotPolicyAttributeHeads(config.attributeHeads, `${path2}.attributeHeads`, issues);
|
|
3634
|
+
return attributeHeads ? { attributeHeads } : void 0;
|
|
3635
|
+
}
|
|
3636
|
+
function parseClaimSlotPolicyAttributeHeads(value, path2, issues) {
|
|
3637
|
+
if (value === void 0) {
|
|
3638
|
+
return void 0;
|
|
3639
|
+
}
|
|
3640
|
+
if (!isRecord(value)) {
|
|
3641
|
+
pushIssue(issues, path2, "Expected an object.");
|
|
3642
|
+
return void 0;
|
|
3643
|
+
}
|
|
3644
|
+
const normalized = {};
|
|
3645
|
+
for (const [rawKey, rawValue] of Object.entries(value)) {
|
|
3646
|
+
const attributeHead = rawKey.trim().toLowerCase();
|
|
3647
|
+
if (!/^[a-z0-9][a-z0-9_-]*$/.test(attributeHead)) {
|
|
3648
|
+
pushIssue(issues, `${path2}.${rawKey}`, "Expected a canonical attribute-head label.");
|
|
3649
|
+
continue;
|
|
3650
|
+
}
|
|
3651
|
+
if (typeof rawValue !== "string" || !CLAIM_SLOT_POLICIES.includes(rawValue)) {
|
|
3652
|
+
pushIssue(issues, `${path2}.${attributeHead}`, `Expected one of: ${CLAIM_SLOT_POLICIES.join(", ")}.`);
|
|
3653
|
+
continue;
|
|
3654
|
+
}
|
|
3655
|
+
normalized[attributeHead] = rawValue;
|
|
3656
|
+
}
|
|
3657
|
+
return Object.keys(normalized).length > 0 ? normalized : void 0;
|
|
3658
|
+
}
|
|
3659
|
+
function validatePathSpecificRequest(recallPath, recallRequest, unified, issues) {
|
|
3660
|
+
const effectivePath = recallPath ?? "core";
|
|
3661
|
+
if (effectivePath !== "unified") {
|
|
3662
|
+
if (unified !== void 0) {
|
|
3663
|
+
pushIssue(issues, "unified", 'The "unified" block is only allowed when recallPath is "unified".');
|
|
3664
|
+
}
|
|
3665
|
+
return;
|
|
3666
|
+
}
|
|
3667
|
+
if (recallRequest === void 0) {
|
|
3668
|
+
return;
|
|
3669
|
+
}
|
|
3670
|
+
if (recallRequest.budget !== void 0) {
|
|
3671
|
+
pushIssue(issues, "recallRequest.budget", 'This field is only supported when recallPath is "core".');
|
|
3672
|
+
}
|
|
3673
|
+
if (recallRequest.since !== void 0) {
|
|
3674
|
+
pushIssue(issues, "recallRequest.since", 'This field is only supported when recallPath is "core".');
|
|
3675
|
+
}
|
|
3676
|
+
if (recallRequest.until !== void 0) {
|
|
3677
|
+
pushIssue(issues, "recallRequest.until", 'This field is only supported when recallPath is "core".');
|
|
3678
|
+
}
|
|
3679
|
+
if (recallRequest.around !== void 0) {
|
|
3680
|
+
pushIssue(issues, "recallRequest.around", 'This field is only supported when recallPath is "core".');
|
|
3681
|
+
}
|
|
3682
|
+
if (recallRequest.aroundRadius !== void 0) {
|
|
3683
|
+
pushIssue(issues, "recallRequest.aroundRadius", 'This field is only supported when recallPath is "core".');
|
|
3684
|
+
}
|
|
3685
|
+
if (recallRequest.rankingProfile !== void 0) {
|
|
3686
|
+
pushIssue(issues, "recallRequest.rankingProfile", 'This field is derived by unified recall and cannot be supplied when recallPath is "unified".');
|
|
3687
|
+
}
|
|
3688
|
+
}
|
|
3689
|
+
function mapSandboxRequestDto2(dto) {
|
|
3690
|
+
if (dto === void 0) {
|
|
3691
|
+
return void 0;
|
|
3692
|
+
}
|
|
3693
|
+
return {
|
|
3694
|
+
root: dto.root,
|
|
3695
|
+
preserve: dto.preserve,
|
|
3696
|
+
corpusSeed: dto.corpusSeed,
|
|
3697
|
+
ablationArm: dto.ablationArm,
|
|
3698
|
+
now: dto.now,
|
|
3699
|
+
profileSnapshot: dto.profileSnapshot
|
|
3700
|
+
};
|
|
3701
|
+
}
|
|
3702
|
+
function mapFixtureEntryDto2(dto) {
|
|
3703
|
+
return mapFixtureEntryDto(dto);
|
|
3704
|
+
}
|
|
3705
|
+
function mapRecallRequestDto(dto) {
|
|
3706
|
+
return {
|
|
3707
|
+
text: dto.text,
|
|
3708
|
+
limit: dto.limit,
|
|
3709
|
+
threshold: dto.threshold,
|
|
3710
|
+
budget: dto.budget,
|
|
3711
|
+
types: dto.types,
|
|
3712
|
+
tags: dto.tags,
|
|
3713
|
+
since: dto.since,
|
|
3714
|
+
until: dto.until,
|
|
3715
|
+
around: dto.around,
|
|
3716
|
+
aroundRadius: dto.aroundRadius,
|
|
3717
|
+
asOf: dto.asOf,
|
|
3718
|
+
rankingProfile: dto.rankingProfile,
|
|
3719
|
+
rankingPolicy: dto.rankingPolicy
|
|
3720
|
+
};
|
|
3721
|
+
}
|
|
3722
|
+
function mapUnifiedRequestDto(dto) {
|
|
3723
|
+
if (dto === void 0) {
|
|
3724
|
+
return void 0;
|
|
3725
|
+
}
|
|
3726
|
+
return {
|
|
3727
|
+
mode: dto.mode,
|
|
3728
|
+
sessionKey: dto.sessionKey,
|
|
3729
|
+
memoryPolicy: dto.memoryPolicy !== void 0 ? {
|
|
3730
|
+
slotPolicies: dto.memoryPolicy.slotPolicies
|
|
3731
|
+
} : void 0
|
|
3732
|
+
};
|
|
3733
|
+
}
|
|
3734
|
+
function mapCaseOptionsDto3(dto) {
|
|
3735
|
+
if (dto === void 0) {
|
|
3736
|
+
return void 0;
|
|
3737
|
+
}
|
|
3738
|
+
return {
|
|
3739
|
+
includeDiagnostics: dto.includeDiagnostics,
|
|
3740
|
+
includeCandidates: dto.includeCandidates,
|
|
3741
|
+
includeTimings: dto.includeTimings,
|
|
3742
|
+
includeDebugArtifact: dto.includeDebugArtifact,
|
|
3743
|
+
topKCandidates: dto.topKCandidates,
|
|
3744
|
+
faultInjection: dto.faultInjection
|
|
3745
|
+
};
|
|
3746
|
+
}
|
|
3747
|
+
|
|
3748
|
+
// src/adapters/api/routes/internal-recall-eval.ts
|
|
3749
|
+
var INTERNAL_RECALL_EVAL_ROUTE_PATH = "/internal/evals/recall/run";
|
|
3750
|
+
var INTERNAL_RECALL_EVAL_ROUTE = {
|
|
3751
|
+
method: "POST",
|
|
3752
|
+
path: INTERNAL_RECALL_EVAL_ROUTE_PATH
|
|
3753
|
+
};
|
|
3754
|
+
function createInternalRecallEvalRoute(optionsOrRunner = {}) {
|
|
3755
|
+
const options = typeof optionsOrRunner === "function" ? { runner: optionsOrRunner } : optionsOrRunner;
|
|
3756
|
+
const crossEncoder = options.crossEncoder;
|
|
3757
|
+
const runner = options.runner ?? ((request) => runRecallEvalCase(request, { crossEncoder }));
|
|
3758
|
+
return {
|
|
3759
|
+
...INTERNAL_RECALL_EVAL_ROUTE,
|
|
3760
|
+
handler: async (request) => {
|
|
3761
|
+
let validatedRequest;
|
|
3762
|
+
try {
|
|
3763
|
+
validatedRequest = await parseValidatedRequest3(request);
|
|
3764
|
+
const result = await runner(validatedRequest);
|
|
3765
|
+
return jsonResponse3(result, 200);
|
|
3766
|
+
} catch (error) {
|
|
3767
|
+
if (error instanceof RecallEvalRequestValidationError) {
|
|
3768
|
+
return jsonResponse3(
|
|
3769
|
+
{
|
|
3770
|
+
status: "error",
|
|
3771
|
+
caseId: error.caseId,
|
|
3772
|
+
error: {
|
|
3773
|
+
code: "invalid_request",
|
|
3774
|
+
message: error.message,
|
|
3775
|
+
details: error.issues
|
|
3776
|
+
}
|
|
3777
|
+
},
|
|
3778
|
+
400
|
|
3779
|
+
);
|
|
3780
|
+
}
|
|
3781
|
+
return jsonResponse3(
|
|
3782
|
+
{
|
|
3783
|
+
status: "error",
|
|
3784
|
+
caseId: validatedRequest?.caseId,
|
|
3785
|
+
error: {
|
|
3786
|
+
code: "internal_error",
|
|
3787
|
+
message: "Internal recall eval adapter error."
|
|
3788
|
+
}
|
|
3789
|
+
},
|
|
3790
|
+
500
|
|
3791
|
+
);
|
|
3792
|
+
}
|
|
3793
|
+
}
|
|
3794
|
+
};
|
|
3795
|
+
}
|
|
3796
|
+
var parseJsonBody3 = async (request) => {
|
|
3797
|
+
try {
|
|
3798
|
+
return await request.json();
|
|
3799
|
+
} catch {
|
|
3800
|
+
throw new RecallEvalRequestValidationError([
|
|
3801
|
+
{
|
|
3802
|
+
path: "$",
|
|
3803
|
+
message: "Request body must be valid JSON."
|
|
3804
|
+
}
|
|
3805
|
+
]);
|
|
3806
|
+
}
|
|
3807
|
+
};
|
|
3808
|
+
var parseValidatedRequest3 = async (request) => {
|
|
3809
|
+
const payload = await parseJsonBody3(request);
|
|
3810
|
+
const requestDto = parseRecallEvalCaseRequest(payload);
|
|
3811
|
+
return mapRecallEvalCaseRequestDto(requestDto);
|
|
3812
|
+
};
|
|
3813
|
+
var jsonResponse3 = (body, status) => new Response(JSON.stringify(body), {
|
|
3814
|
+
status,
|
|
3815
|
+
headers: {
|
|
3816
|
+
"content-type": "application/json; charset=utf-8"
|
|
3817
|
+
}
|
|
3818
|
+
});
|
|
3819
|
+
|
|
3820
|
+
// src/app/evals/session-start/normalize-response.ts
|
|
3821
|
+
function buildSessionStartEvalSuccessResponse(params) {
|
|
3822
|
+
return {
|
|
3823
|
+
status: "ok",
|
|
3824
|
+
caseId: params.request.caseId,
|
|
3825
|
+
output: buildOutput2(params.patch),
|
|
3826
|
+
...params.request.options?.includeDiagnostics === true ? { diagnostics: params.patch.diagnostics } : {},
|
|
3827
|
+
...params.timings ? { timings: params.timings } : {},
|
|
3828
|
+
sandbox: buildSandboxResult4(params.sandbox)
|
|
3829
|
+
};
|
|
3830
|
+
}
|
|
3831
|
+
function buildSessionStartEvalErrorResponse(params) {
|
|
3832
|
+
return {
|
|
3833
|
+
status: "error",
|
|
3834
|
+
caseId: params.request.caseId,
|
|
3835
|
+
error: {
|
|
3836
|
+
code: params.code,
|
|
3837
|
+
message: params.message,
|
|
3838
|
+
...params.details !== void 0 ? { details: params.details } : {}
|
|
3839
|
+
},
|
|
3840
|
+
...params.timings ? { timings: params.timings } : {},
|
|
3841
|
+
...params.sandbox ? { sandbox: buildSandboxResult4(params.sandbox) } : {}
|
|
3842
|
+
};
|
|
3843
|
+
}
|
|
3844
|
+
function buildOutput2(patch) {
|
|
3845
|
+
const sourceKindsByEntryId = {};
|
|
3846
|
+
for (const item of patch.durableMemory) {
|
|
3847
|
+
sourceKindsByEntryId[item.entry.id] = item.sourceKind;
|
|
3848
|
+
}
|
|
3849
|
+
return {
|
|
3850
|
+
selectedEntryIds: patch.durableMemory.map((item) => item.entry.id),
|
|
3851
|
+
sourceKindsByEntryId
|
|
3852
|
+
};
|
|
3853
|
+
}
|
|
3854
|
+
function buildSandboxResult4(sandbox) {
|
|
3855
|
+
return {
|
|
3856
|
+
root: sandbox.root,
|
|
3857
|
+
dbPath: sandbox.dbPath,
|
|
3858
|
+
preserved: sandbox.preserved,
|
|
3859
|
+
...sandbox.snapshot ? { snapshot: sandbox.snapshot } : {}
|
|
3860
|
+
};
|
|
3861
|
+
}
|
|
3862
|
+
|
|
3863
|
+
// src/app/evals/session-start/run-session-start-eval-case.ts
|
|
3864
|
+
async function runSessionStartEvalCase(request) {
|
|
3865
|
+
const startedAt = Date.now();
|
|
3866
|
+
const provisionedAt = new Date(startedAt).toISOString();
|
|
3867
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
3868
|
+
const evalNow = parseEvalNow(ablation.now);
|
|
3869
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
3870
|
+
let sandbox;
|
|
3871
|
+
let timings;
|
|
3872
|
+
try {
|
|
3873
|
+
const sandboxStartedAt = Date.now();
|
|
3874
|
+
try {
|
|
3875
|
+
sandbox = await setupRecallEvalSandbox(request.sandbox);
|
|
3876
|
+
timings = { sandboxSetupMs: elapsedMs5(sandboxStartedAt) };
|
|
3877
|
+
} catch (error) {
|
|
3878
|
+
return buildSessionStartEvalErrorResponse({
|
|
3879
|
+
request,
|
|
3880
|
+
code: "sandbox_setup_failed",
|
|
3881
|
+
message: "Failed to create isolated session-start eval sandbox.",
|
|
3882
|
+
details: toErrorDetails4(error),
|
|
3883
|
+
timings: request.options?.includeTimings === true ? { totalMs: elapsedMs5(startedAt), sandboxSetupMs: elapsedMs5(sandboxStartedAt) } : void 0
|
|
3884
|
+
});
|
|
3885
|
+
}
|
|
3886
|
+
if (request.memoryPool.length > 0 || shouldProvisionProfileSnapshot(ablation)) {
|
|
3887
|
+
const provisionStartedAt = Date.now();
|
|
3888
|
+
try {
|
|
3889
|
+
await provisionEvalSandbox({
|
|
3890
|
+
caseId: request.caseId,
|
|
3891
|
+
sandbox,
|
|
3892
|
+
memoryPool: request.memoryPool,
|
|
3893
|
+
profileSnapshot: shouldProvisionProfileSnapshot(ablation) ? ablation.profileSnapshot : void 0,
|
|
3894
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.portOrUnavailable() : void 0,
|
|
3895
|
+
provisionedAt
|
|
3896
|
+
});
|
|
3897
|
+
timings = { ...timings, fixtureProvisionMs: elapsedMs5(provisionStartedAt) };
|
|
3898
|
+
} catch (error) {
|
|
3899
|
+
return buildSessionStartEvalErrorResponse({
|
|
3900
|
+
request,
|
|
3901
|
+
code: "fixture_provision_failed",
|
|
3902
|
+
message: "Failed to provision session-start eval fixtures into isolated storage.",
|
|
3903
|
+
details: toErrorDetails4(error),
|
|
3904
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs5(startedAt), fixtureProvisionMs: elapsedMs5(provisionStartedAt) } : void 0,
|
|
3905
|
+
sandbox
|
|
3906
|
+
});
|
|
3907
|
+
}
|
|
3908
|
+
}
|
|
3909
|
+
const sessionStartStartedAt = Date.now();
|
|
3910
|
+
try {
|
|
3911
|
+
if (isMemoryOffArm(ablation)) {
|
|
3912
|
+
timings = {
|
|
3913
|
+
...timings,
|
|
3914
|
+
sessionStartMs: elapsedMs5(sessionStartStartedAt),
|
|
3915
|
+
totalMs: elapsedMs5(startedAt)
|
|
3916
|
+
};
|
|
3917
|
+
return buildSessionStartEvalSuccessResponse({
|
|
3918
|
+
request,
|
|
3919
|
+
patch: {
|
|
3920
|
+
durableMemory: [],
|
|
3921
|
+
diagnostics: {
|
|
3922
|
+
coreCandidateCount: 0,
|
|
3923
|
+
profileCandidateCount: 0,
|
|
3924
|
+
proactiveDirectiveCandidateCount: 0,
|
|
3925
|
+
artifactRecallCandidateCount: 0,
|
|
3926
|
+
artifactRecallUsed: false,
|
|
3927
|
+
notices: ["memory-off ablation arm stubbed session-start injection."]
|
|
3928
|
+
}
|
|
3929
|
+
},
|
|
3930
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
3931
|
+
sandbox
|
|
3932
|
+
});
|
|
3933
|
+
}
|
|
3934
|
+
const activeSandbox = sandbox;
|
|
3935
|
+
if (!activeSandbox) {
|
|
3936
|
+
throw new Error("Session-start eval sandbox was not initialized.");
|
|
3937
|
+
}
|
|
3938
|
+
const embeddingSupport = embeddingResolver.getSupport();
|
|
3939
|
+
const recallPorts = activeSandbox.createRecallPorts(
|
|
3940
|
+
embeddingSupport.port ?? createUnavailableEmbeddingPort(embeddingSupport.error ?? "Embeddings are unavailable.")
|
|
3941
|
+
);
|
|
3942
|
+
const patch = await runSessionStart(request.sessionStartInput, {
|
|
3943
|
+
repository: activeSandbox.sessionStartRepository,
|
|
3944
|
+
recall: recallPorts,
|
|
3945
|
+
...evalNow ? { now: evalNow } : {},
|
|
3946
|
+
listActiveAbstainDirectives: () => activeSandbox.listActiveAbstainDirectives(evalNow),
|
|
3947
|
+
listActiveProactiveDirectives: () => activeSandbox.listActiveSessionStartProactiveDirectives(evalNow)
|
|
3948
|
+
});
|
|
3949
|
+
timings = {
|
|
3950
|
+
...timings,
|
|
3951
|
+
sessionStartMs: elapsedMs5(sessionStartStartedAt),
|
|
3952
|
+
totalMs: elapsedMs5(startedAt)
|
|
3953
|
+
};
|
|
3954
|
+
return buildSessionStartEvalSuccessResponse({
|
|
3955
|
+
request,
|
|
3956
|
+
patch,
|
|
3957
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
3958
|
+
sandbox: activeSandbox
|
|
3959
|
+
});
|
|
3960
|
+
} catch (error) {
|
|
3961
|
+
return buildSessionStartEvalErrorResponse({
|
|
3962
|
+
request,
|
|
3963
|
+
code: "session_start_execution_failed",
|
|
3964
|
+
message: "Failed to execute real session-start selection against isolated eval state.",
|
|
3965
|
+
details: toErrorDetails4(error),
|
|
3966
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs5(startedAt), sessionStartMs: elapsedMs5(sessionStartStartedAt) } : void 0,
|
|
3967
|
+
sandbox
|
|
3968
|
+
});
|
|
3969
|
+
}
|
|
3970
|
+
} catch (error) {
|
|
3971
|
+
return buildSessionStartEvalErrorResponse({
|
|
3972
|
+
request,
|
|
3973
|
+
code: "internal_error",
|
|
3974
|
+
message: "Session-start eval execution failed unexpectedly.",
|
|
3975
|
+
details: toErrorDetails4(error),
|
|
3976
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs5(startedAt) } : void 0,
|
|
3977
|
+
sandbox
|
|
3978
|
+
});
|
|
3979
|
+
} finally {
|
|
3980
|
+
await sandbox?.cleanup().catch(() => void 0);
|
|
3981
|
+
}
|
|
3982
|
+
}
|
|
3983
|
+
function toErrorDetails4(error) {
|
|
3984
|
+
if (error instanceof Error) {
|
|
3985
|
+
return { cause: error.message };
|
|
3986
|
+
}
|
|
3987
|
+
return { cause: String(error) };
|
|
3988
|
+
}
|
|
3989
|
+
function elapsedMs5(startedAt) {
|
|
3990
|
+
return Date.now() - startedAt;
|
|
3991
|
+
}
|
|
3992
|
+
|
|
3993
|
+
// src/adapters/api/validation/session-start-eval-request.ts
|
|
3994
|
+
var ROOT_REQUEST_KEYS4 = /* @__PURE__ */ new Set(["caseId", "description", "sandbox", "memoryPool", "sessionStartInput", "options"]);
|
|
3995
|
+
var SESSION_START_INPUT_KEYS = /* @__PURE__ */ new Set(["sessionKey", "policy"]);
|
|
3996
|
+
var SESSION_START_POLICY_KEYS = /* @__PURE__ */ new Set([
|
|
3997
|
+
"maxCoreEntries",
|
|
3998
|
+
"enableArtifactRecall",
|
|
3999
|
+
"maxArtifactRecallEntries",
|
|
4000
|
+
"maxDurableEntries",
|
|
4001
|
+
"maxArtifactChars",
|
|
4002
|
+
"recallThreshold",
|
|
4003
|
+
"maxProfileSnapshotAgeHours"
|
|
4004
|
+
]);
|
|
4005
|
+
var OPTIONS_KEYS4 = /* @__PURE__ */ new Set(["includeDiagnostics", "includeTimings"]);
|
|
4006
|
+
var SessionStartEvalRequestValidationError = class extends Error {
|
|
4007
|
+
caseId;
|
|
4008
|
+
issues;
|
|
4009
|
+
/** Creates a validation error with optional safely parsed case id. */
|
|
4010
|
+
constructor(issues, caseId) {
|
|
4011
|
+
super("Invalid session-start eval request.");
|
|
4012
|
+
this.name = "SessionStartEvalRequestValidationError";
|
|
4013
|
+
this.issues = issues;
|
|
4014
|
+
this.caseId = caseId;
|
|
4015
|
+
}
|
|
4016
|
+
};
|
|
4017
|
+
function parseSessionStartEvalCaseRequest(payload) {
|
|
4018
|
+
const issues = [];
|
|
4019
|
+
const input = parseObject(payload, "$", issues);
|
|
4020
|
+
if (input === void 0) {
|
|
4021
|
+
throw new SessionStartEvalRequestValidationError(issues);
|
|
4022
|
+
}
|
|
4023
|
+
pushUnexpectedFields(input, ROOT_REQUEST_KEYS4, "$", issues);
|
|
4024
|
+
const caseId = parseRequiredTrimmedString(input.caseId, "caseId", issues);
|
|
4025
|
+
const memoryPool = parseMemoryPool(input.memoryPool, issues);
|
|
4026
|
+
const sessionStartInput = parseSessionStartInput(input.sessionStartInput, issues);
|
|
4027
|
+
if (caseId === void 0 || memoryPool === void 0 || sessionStartInput === void 0) {
|
|
4028
|
+
throw new SessionStartEvalRequestValidationError(issues, extractParseableCaseId(input));
|
|
4029
|
+
}
|
|
4030
|
+
const dto = {
|
|
4031
|
+
caseId,
|
|
4032
|
+
memoryPool,
|
|
4033
|
+
sessionStartInput,
|
|
4034
|
+
description: parseOptionalTrimmedString(input.description, "description", issues),
|
|
4035
|
+
sandbox: parseSandbox(input.sandbox, issues),
|
|
4036
|
+
options: parseOptions4(input.options, issues)
|
|
4037
|
+
};
|
|
4038
|
+
if (issues.length > 0) {
|
|
4039
|
+
throw new SessionStartEvalRequestValidationError(issues, caseId);
|
|
4040
|
+
}
|
|
4041
|
+
return dto;
|
|
4042
|
+
}
|
|
4043
|
+
function mapSessionStartEvalCaseRequestDto(dto) {
|
|
4044
|
+
return {
|
|
4045
|
+
caseId: dto.caseId,
|
|
4046
|
+
...dto.description ? { description: dto.description } : {},
|
|
4047
|
+
sandbox: mapSandboxRequestDto(dto.sandbox),
|
|
4048
|
+
memoryPool: dto.memoryPool.map((entry) => mapFixtureEntryDto(entry)),
|
|
4049
|
+
sessionStartInput: mapSessionStartInputDto(dto.sessionStartInput),
|
|
4050
|
+
...dto.options ? { options: mapCaseOptionsDto4(dto.options) } : {}
|
|
4051
|
+
};
|
|
4052
|
+
}
|
|
4053
|
+
function parseSessionStartInput(value, issues) {
|
|
4054
|
+
const sessionStartInput = parseObject(value, "sessionStartInput", issues);
|
|
4055
|
+
if (sessionStartInput === void 0) {
|
|
4056
|
+
return void 0;
|
|
4057
|
+
}
|
|
4058
|
+
pushUnexpectedFields(sessionStartInput, SESSION_START_INPUT_KEYS, "sessionStartInput", issues);
|
|
4059
|
+
return {
|
|
4060
|
+
sessionKey: parseOptionalTrimmedString(sessionStartInput.sessionKey, "sessionStartInput.sessionKey", issues),
|
|
4061
|
+
policy: parseSessionStartPolicy(sessionStartInput.policy, issues)
|
|
4062
|
+
};
|
|
4063
|
+
}
|
|
4064
|
+
function parseSessionStartPolicy(value, issues) {
|
|
4065
|
+
if (value === void 0) {
|
|
4066
|
+
return void 0;
|
|
4067
|
+
}
|
|
4068
|
+
const policy = parseObject(value, "sessionStartInput.policy", issues);
|
|
4069
|
+
if (policy === void 0) {
|
|
4070
|
+
return void 0;
|
|
4071
|
+
}
|
|
4072
|
+
pushUnexpectedFields(policy, SESSION_START_POLICY_KEYS, "sessionStartInput.policy", issues);
|
|
4073
|
+
return {
|
|
4074
|
+
maxCoreEntries: parseOptionalIntegerInRange(policy.maxCoreEntries, "sessionStartInput.policy.maxCoreEntries", issues, { min: 0 }),
|
|
4075
|
+
enableArtifactRecall: parseOptionalBoolean(policy.enableArtifactRecall, "sessionStartInput.policy.enableArtifactRecall", issues),
|
|
4076
|
+
maxArtifactRecallEntries: parseOptionalIntegerInRange(policy.maxArtifactRecallEntries, "sessionStartInput.policy.maxArtifactRecallEntries", issues, {
|
|
4077
|
+
min: 0
|
|
4078
|
+
}),
|
|
4079
|
+
maxDurableEntries: parseOptionalIntegerInRange(policy.maxDurableEntries, "sessionStartInput.policy.maxDurableEntries", issues, { min: 0 }),
|
|
4080
|
+
maxArtifactChars: parseOptionalIntegerInRange(policy.maxArtifactChars, "sessionStartInput.policy.maxArtifactChars", issues, { min: 0 }),
|
|
4081
|
+
recallThreshold: parseOptionalThreshold(policy.recallThreshold, "sessionStartInput.policy.recallThreshold", issues),
|
|
4082
|
+
maxProfileSnapshotAgeHours: parseOptionalIntegerInRange(policy.maxProfileSnapshotAgeHours, "sessionStartInput.policy.maxProfileSnapshotAgeHours", issues, {
|
|
4083
|
+
min: 0
|
|
4084
|
+
})
|
|
4085
|
+
};
|
|
4086
|
+
}
|
|
4087
|
+
function parseOptions4(value, issues) {
|
|
4088
|
+
if (value === void 0) {
|
|
4089
|
+
return void 0;
|
|
4090
|
+
}
|
|
4091
|
+
const options = parseObject(value, "options", issues);
|
|
4092
|
+
if (options === void 0) {
|
|
4093
|
+
return void 0;
|
|
4094
|
+
}
|
|
4095
|
+
pushUnexpectedFields(options, OPTIONS_KEYS4, "options", issues);
|
|
4096
|
+
return {
|
|
4097
|
+
includeDiagnostics: parseOptionalBoolean(options.includeDiagnostics, "options.includeDiagnostics", issues),
|
|
4098
|
+
includeTimings: parseOptionalBoolean(options.includeTimings, "options.includeTimings", issues)
|
|
4099
|
+
};
|
|
4100
|
+
}
|
|
4101
|
+
function mapSessionStartInputDto(dto) {
|
|
4102
|
+
return {
|
|
4103
|
+
...dto.sessionKey ? { sessionKey: dto.sessionKey } : {},
|
|
4104
|
+
...dto.policy ? { policy: mapSessionStartPolicyDto(dto.policy) } : {}
|
|
4105
|
+
};
|
|
4106
|
+
}
|
|
4107
|
+
function mapSessionStartPolicyDto(dto) {
|
|
4108
|
+
return {
|
|
4109
|
+
maxCoreEntries: dto.maxCoreEntries,
|
|
4110
|
+
enableArtifactRecall: dto.enableArtifactRecall,
|
|
4111
|
+
maxArtifactRecallEntries: dto.maxArtifactRecallEntries,
|
|
4112
|
+
maxDurableEntries: dto.maxDurableEntries,
|
|
4113
|
+
maxArtifactChars: dto.maxArtifactChars,
|
|
4114
|
+
recallThreshold: dto.recallThreshold,
|
|
4115
|
+
maxProfileSnapshotAgeHours: dto.maxProfileSnapshotAgeHours
|
|
4116
|
+
};
|
|
4117
|
+
}
|
|
4118
|
+
function mapCaseOptionsDto4(dto) {
|
|
4119
|
+
return {
|
|
4120
|
+
includeDiagnostics: dto.includeDiagnostics,
|
|
4121
|
+
includeTimings: dto.includeTimings
|
|
4122
|
+
};
|
|
4123
|
+
}
|
|
4124
|
+
|
|
4125
|
+
// src/adapters/api/routes/internal-session-start-eval.ts
|
|
4126
|
+
var INTERNAL_SESSION_START_EVAL_ROUTE_PATH = "/internal/evals/session-start/run";
|
|
4127
|
+
var INTERNAL_SESSION_START_EVAL_ROUTE = {
|
|
4128
|
+
method: "POST",
|
|
4129
|
+
path: INTERNAL_SESSION_START_EVAL_ROUTE_PATH
|
|
4130
|
+
};
|
|
4131
|
+
function createInternalSessionStartEvalRoute(runner = runSessionStartEvalCase) {
|
|
4132
|
+
return {
|
|
4133
|
+
...INTERNAL_SESSION_START_EVAL_ROUTE,
|
|
4134
|
+
handler: async (request) => {
|
|
4135
|
+
let validatedRequest;
|
|
4136
|
+
try {
|
|
4137
|
+
validatedRequest = await parseValidatedRequest4(request);
|
|
4138
|
+
const result = await runner(validatedRequest);
|
|
4139
|
+
return jsonResponse4(result, 200);
|
|
4140
|
+
} catch (error) {
|
|
4141
|
+
if (error instanceof SessionStartEvalRequestValidationError) {
|
|
4142
|
+
return jsonResponse4(
|
|
4143
|
+
{
|
|
4144
|
+
status: "error",
|
|
4145
|
+
caseId: error.caseId,
|
|
4146
|
+
error: {
|
|
4147
|
+
code: "invalid_request",
|
|
4148
|
+
message: error.message,
|
|
4149
|
+
details: error.issues
|
|
4150
|
+
}
|
|
4151
|
+
},
|
|
4152
|
+
400
|
|
4153
|
+
);
|
|
4154
|
+
}
|
|
4155
|
+
return jsonResponse4(
|
|
4156
|
+
{
|
|
4157
|
+
status: "error",
|
|
4158
|
+
caseId: validatedRequest?.caseId,
|
|
4159
|
+
error: {
|
|
4160
|
+
code: "internal_error",
|
|
4161
|
+
message: "Internal session-start eval adapter error."
|
|
4162
|
+
}
|
|
4163
|
+
},
|
|
4164
|
+
500
|
|
4165
|
+
);
|
|
4166
|
+
}
|
|
4167
|
+
}
|
|
4168
|
+
};
|
|
4169
|
+
}
|
|
4170
|
+
var parseJsonBody4 = async (request) => {
|
|
4171
|
+
try {
|
|
4172
|
+
return await request.json();
|
|
4173
|
+
} catch {
|
|
4174
|
+
throw new SessionStartEvalRequestValidationError([
|
|
4175
|
+
{
|
|
4176
|
+
path: "$",
|
|
4177
|
+
message: "Request body must be valid JSON."
|
|
4178
|
+
}
|
|
4179
|
+
]);
|
|
4180
|
+
}
|
|
4181
|
+
};
|
|
4182
|
+
var parseValidatedRequest4 = async (request) => {
|
|
4183
|
+
const payload = await parseJsonBody4(request);
|
|
4184
|
+
const requestDto = parseSessionStartEvalCaseRequest(payload);
|
|
4185
|
+
return mapSessionStartEvalCaseRequestDto(requestDto);
|
|
4186
|
+
};
|
|
4187
|
+
var jsonResponse4 = (body, status) => new Response(JSON.stringify(body), {
|
|
4188
|
+
status,
|
|
4189
|
+
headers: {
|
|
4190
|
+
"content-type": "application/json; charset=utf-8"
|
|
4191
|
+
}
|
|
4192
|
+
});
|
|
4193
|
+
|
|
4194
|
+
// src/adapters/api/internal-eval-routes.ts
|
|
4195
|
+
function createInternalEvalRoutes(options = {}) {
|
|
4196
|
+
return [
|
|
4197
|
+
createInternalRecallEvalRoute({ crossEncoder: options.crossEncoder }),
|
|
4198
|
+
createInternalBeforeTurnEvalRoute({ crossEncoder: options.crossEncoder }),
|
|
4199
|
+
createInternalSessionStartEvalRoute(),
|
|
4200
|
+
createInternalDreamingEfficiencyEvalRoute()
|
|
4201
|
+
];
|
|
4202
|
+
}
|
|
4203
|
+
|
|
4204
|
+
// src/adapters/api/internal-eval-server.ts
|
|
4205
|
+
var DEFAULT_INTERNAL_EVAL_HOST = "127.0.0.1";
|
|
4206
|
+
var DEFAULT_INTERNAL_EVAL_PORT = 4010;
|
|
4207
|
+
async function startInternalEvalServer(options = {}) {
|
|
4208
|
+
const host2 = options.host ?? DEFAULT_INTERNAL_EVAL_HOST;
|
|
4209
|
+
const port2 = options.port ?? DEFAULT_INTERNAL_EVAL_PORT;
|
|
4210
|
+
const crossEncoderResolution = resolveCrossEncoderPort(options);
|
|
4211
|
+
const routes = options.routes ?? createInternalEvalRoutes({ crossEncoder: crossEncoderResolution.port });
|
|
4212
|
+
const server2 = createServer((request, response) => {
|
|
4213
|
+
void handleRequest(request, response, routes, host2, port2).catch(() => {
|
|
4214
|
+
if (response.headersSent !== true) {
|
|
4215
|
+
writeTextResponse(response, 500, "Internal server error.\n");
|
|
4216
|
+
return;
|
|
4217
|
+
}
|
|
4218
|
+
response.destroy();
|
|
4219
|
+
});
|
|
4220
|
+
});
|
|
4221
|
+
await listen(server2, port2, host2);
|
|
4222
|
+
const address = server2.address();
|
|
4223
|
+
if (address === null || typeof address === "string") {
|
|
4224
|
+
await closeServer(server2);
|
|
4225
|
+
throw new Error("Internal eval server did not expose a TCP address.");
|
|
4226
|
+
}
|
|
4227
|
+
return {
|
|
4228
|
+
host: host2,
|
|
4229
|
+
port: address.port,
|
|
4230
|
+
routePaths: routes.map((route) => route.path),
|
|
4231
|
+
baseUrl: `http://${formatHostForUrl(host2)}:${address.port}`,
|
|
4232
|
+
crossEncoder: crossEncoderResolution.result,
|
|
4233
|
+
close: async () => {
|
|
4234
|
+
await closeServer(server2);
|
|
4235
|
+
}
|
|
4236
|
+
};
|
|
4237
|
+
}
|
|
4238
|
+
function resolveCrossEncoderPort(options) {
|
|
4239
|
+
if (options.crossEncoder) {
|
|
4240
|
+
return {
|
|
4241
|
+
port: options.crossEncoder,
|
|
4242
|
+
result: { status: "configured" }
|
|
4243
|
+
};
|
|
4244
|
+
}
|
|
4245
|
+
if (options.autoResolveCrossEncoder === false) {
|
|
4246
|
+
return {
|
|
4247
|
+
port: void 0,
|
|
4248
|
+
result: {
|
|
4249
|
+
status: "not_configured",
|
|
4250
|
+
reason: "Auto-resolution disabled by caller."
|
|
4251
|
+
}
|
|
4252
|
+
};
|
|
4253
|
+
}
|
|
4254
|
+
let config;
|
|
4255
|
+
try {
|
|
4256
|
+
config = readConfig();
|
|
4257
|
+
} catch (error) {
|
|
4258
|
+
return {
|
|
4259
|
+
port: void 0,
|
|
4260
|
+
result: {
|
|
4261
|
+
status: "error",
|
|
4262
|
+
reason: toReason(error, "Failed to read agenr config for cross-encoder resolution.")
|
|
4263
|
+
}
|
|
4264
|
+
};
|
|
4265
|
+
}
|
|
4266
|
+
let apiKey;
|
|
4267
|
+
try {
|
|
4268
|
+
apiKey = resolveCrossEncoderApiKey(config);
|
|
4269
|
+
} catch (error) {
|
|
4270
|
+
return {
|
|
4271
|
+
port: void 0,
|
|
4272
|
+
result: {
|
|
4273
|
+
status: "not_configured",
|
|
4274
|
+
reason: toReason(error, "OPENAI_API_KEY not configured.")
|
|
4275
|
+
}
|
|
4276
|
+
};
|
|
4277
|
+
}
|
|
4278
|
+
try {
|
|
4279
|
+
const { modelId } = resolveModel(config, "cross_encoder");
|
|
4280
|
+
return {
|
|
4281
|
+
port: createOpenAICrossEncoder({ apiKey, model: modelId }),
|
|
4282
|
+
result: { status: "configured" }
|
|
4283
|
+
};
|
|
4284
|
+
} catch (error) {
|
|
4285
|
+
return {
|
|
4286
|
+
port: void 0,
|
|
4287
|
+
result: {
|
|
4288
|
+
status: "error",
|
|
4289
|
+
reason: toReason(error, "Failed to construct OpenAI cross-encoder adapter.")
|
|
4290
|
+
}
|
|
4291
|
+
};
|
|
4292
|
+
}
|
|
4293
|
+
}
|
|
4294
|
+
function toReason(error, fallback) {
|
|
4295
|
+
if (error instanceof Error && error.message.length > 0) {
|
|
4296
|
+
return error.message;
|
|
4297
|
+
}
|
|
4298
|
+
return fallback;
|
|
4299
|
+
}
|
|
4300
|
+
var handleRequest = async (request, response, routes, fallbackHost, fallbackPort) => {
|
|
4301
|
+
const requestUrl = new URL(request.url ?? "/", `http://${formatHostForUrl(fallbackHost)}:${request.socket.localPort ?? fallbackPort}`);
|
|
4302
|
+
const route = routes.find((candidate) => candidate.path === requestUrl.pathname);
|
|
4303
|
+
if (!route) {
|
|
4304
|
+
writeTextResponse(response, 404, "Not found.\n");
|
|
4305
|
+
return;
|
|
4306
|
+
}
|
|
4307
|
+
if (request.method !== route.method) {
|
|
4308
|
+
response.statusCode = 405;
|
|
4309
|
+
response.setHeader("allow", route.method);
|
|
4310
|
+
response.end();
|
|
4311
|
+
return;
|
|
4312
|
+
}
|
|
4313
|
+
const body = await readBody(request);
|
|
4314
|
+
const routeRequest = new Request(requestUrl, {
|
|
4315
|
+
method: route.method,
|
|
4316
|
+
headers: toHeaders(request),
|
|
4317
|
+
body: body.length > 0 ? body : void 0
|
|
4318
|
+
});
|
|
4319
|
+
const routeResponse = await route.handler(routeRequest);
|
|
4320
|
+
await writeRouteResponse(response, routeResponse);
|
|
4321
|
+
};
|
|
4322
|
+
var readBody = async (request) => {
|
|
4323
|
+
const chunks = [];
|
|
4324
|
+
for await (const chunk of request) {
|
|
4325
|
+
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
|
4326
|
+
}
|
|
4327
|
+
return Buffer.concat(chunks);
|
|
4328
|
+
};
|
|
4329
|
+
var toHeaders = (request) => {
|
|
4330
|
+
const headers = new Headers();
|
|
4331
|
+
for (const [name, value] of Object.entries(request.headers)) {
|
|
4332
|
+
if (value === void 0) {
|
|
4333
|
+
continue;
|
|
4334
|
+
}
|
|
4335
|
+
if (Array.isArray(value)) {
|
|
4336
|
+
for (const item of value) {
|
|
4337
|
+
headers.append(name, item);
|
|
4338
|
+
}
|
|
4339
|
+
continue;
|
|
4340
|
+
}
|
|
4341
|
+
headers.set(name, value);
|
|
4342
|
+
}
|
|
4343
|
+
return headers;
|
|
4344
|
+
};
|
|
4345
|
+
var writeRouteResponse = async (response, routeResponse) => {
|
|
4346
|
+
response.statusCode = routeResponse.status;
|
|
4347
|
+
for (const [name, value] of routeResponse.headers) {
|
|
4348
|
+
response.setHeader(name, value);
|
|
4349
|
+
}
|
|
4350
|
+
const body = Buffer.from(await routeResponse.arrayBuffer());
|
|
4351
|
+
response.end(body);
|
|
4352
|
+
};
|
|
4353
|
+
var writeTextResponse = (response, status, body) => {
|
|
4354
|
+
response.statusCode = status;
|
|
4355
|
+
response.setHeader("content-type", "text/plain; charset=utf-8");
|
|
4356
|
+
response.end(body);
|
|
4357
|
+
};
|
|
4358
|
+
var listen = async (server2, port2, host2) => {
|
|
4359
|
+
await new Promise((resolve, reject) => {
|
|
4360
|
+
const onError = (error) => {
|
|
4361
|
+
server2.off("listening", onListening);
|
|
4362
|
+
reject(error);
|
|
4363
|
+
};
|
|
4364
|
+
const onListening = () => {
|
|
4365
|
+
server2.off("error", onError);
|
|
4366
|
+
resolve();
|
|
4367
|
+
};
|
|
4368
|
+
server2.once("error", onError);
|
|
4369
|
+
server2.once("listening", onListening);
|
|
4370
|
+
server2.listen(port2, host2);
|
|
4371
|
+
});
|
|
4372
|
+
};
|
|
4373
|
+
var closeServer = async (server2) => {
|
|
4374
|
+
if (server2.listening !== true) {
|
|
4375
|
+
return;
|
|
4376
|
+
}
|
|
4377
|
+
await new Promise((resolve, reject) => {
|
|
4378
|
+
server2.close((error) => {
|
|
4379
|
+
if (error) {
|
|
4380
|
+
reject(error);
|
|
4381
|
+
return;
|
|
4382
|
+
}
|
|
4383
|
+
resolve();
|
|
4384
|
+
});
|
|
4385
|
+
});
|
|
4386
|
+
};
|
|
4387
|
+
var formatHostForUrl = (host2) => {
|
|
4388
|
+
if (host2.includes(":") && host2.startsWith("[") !== true) {
|
|
4389
|
+
return `[${host2}]`;
|
|
4390
|
+
}
|
|
4391
|
+
return host2;
|
|
4392
|
+
};
|
|
4393
|
+
|
|
4394
|
+
// src/internal-eval-server.ts
|
|
4395
|
+
var HOST_ENV_NAME = "AGENR_INTERNAL_EVAL_HOST";
|
|
4396
|
+
var PORT_ENV_NAME = "AGENR_INTERNAL_EVAL_PORT";
|
|
4397
|
+
var LEGACY_HOST_ENV_NAME = "AGENR_INTERNAL_RECALL_EVAL_HOST";
|
|
4398
|
+
var LEGACY_PORT_ENV_NAME = "AGENR_INTERNAL_RECALL_EVAL_PORT";
|
|
4399
|
+
var host = resolveHost(process.env[HOST_ENV_NAME], process.env[LEGACY_HOST_ENV_NAME]);
|
|
4400
|
+
var port = resolvePort(process.env[PORT_ENV_NAME], process.env[LEGACY_PORT_ENV_NAME]);
|
|
4401
|
+
var server = await startInternalEvalServer({ host, port });
|
|
4402
|
+
console.log(`Internal eval dev server listening at ${server.baseUrl}`);
|
|
4403
|
+
console.log(`Serving routes: ${server.routePaths.join(", ")}`);
|
|
4404
|
+
if (server.crossEncoder.status === "configured") {
|
|
4405
|
+
console.log("Cross-encoder enabled: OpenAI credential resolved at startup.");
|
|
4406
|
+
} else if (server.crossEncoder.status === "not_configured") {
|
|
4407
|
+
console.log("Cross-encoder disabled: OPENAI_API_KEY not configured.");
|
|
4408
|
+
} else {
|
|
4409
|
+
console.warn(`Cross-encoder unavailable: ${server.crossEncoder.reason ?? "construction failed"}.`);
|
|
4410
|
+
}
|
|
4411
|
+
installSignalHandler("SIGINT");
|
|
4412
|
+
installSignalHandler("SIGTERM");
|
|
4413
|
+
function installSignalHandler(signal) {
|
|
4414
|
+
process.once(signal, () => {
|
|
4415
|
+
void shutdown(signal);
|
|
4416
|
+
});
|
|
4417
|
+
}
|
|
4418
|
+
var shuttingDown = false;
|
|
4419
|
+
async function shutdown(signal) {
|
|
4420
|
+
if (shuttingDown === true) {
|
|
4421
|
+
return;
|
|
4422
|
+
}
|
|
4423
|
+
shuttingDown = true;
|
|
4424
|
+
console.log(`Received ${signal}. Shutting down internal eval dev server.`);
|
|
4425
|
+
try {
|
|
4426
|
+
await server.close();
|
|
4427
|
+
} finally {
|
|
4428
|
+
process.exit(0);
|
|
4429
|
+
}
|
|
4430
|
+
}
|
|
4431
|
+
function resolveHost(value, fallbackValue) {
|
|
4432
|
+
const trimmed = value?.trim() || fallbackValue?.trim();
|
|
4433
|
+
if (!trimmed) {
|
|
4434
|
+
return DEFAULT_INTERNAL_EVAL_HOST;
|
|
4435
|
+
}
|
|
4436
|
+
return trimmed;
|
|
4437
|
+
}
|
|
4438
|
+
function resolvePort(value, fallbackValue) {
|
|
4439
|
+
const trimmed = value?.trim() || fallbackValue?.trim();
|
|
4440
|
+
if (!trimmed) {
|
|
4441
|
+
return DEFAULT_INTERNAL_EVAL_PORT;
|
|
4442
|
+
}
|
|
4443
|
+
if (/^\d+$/u.test(trimmed) !== true) {
|
|
4444
|
+
throw new Error(`${PORT_ENV_NAME} must be an integer between 0 and 65535.`);
|
|
4445
|
+
}
|
|
4446
|
+
const parsed = Number.parseInt(trimmed, 10);
|
|
4447
|
+
if (!Number.isInteger(parsed) || parsed < 0 || parsed > 65535) {
|
|
4448
|
+
throw new Error(`${PORT_ENV_NAME} must be an integer between 0 and 65535.`);
|
|
4449
|
+
}
|
|
4450
|
+
return parsed;
|
|
4451
|
+
}
|