@agenr/skeln-plugin 3.3.0 → 2026.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-before-turn-artifact-NPUHVWFE.js +71 -0
- package/dist/build-recall-artifact-F3LS3PZX.js +62 -0
- package/dist/chunk-5AXMFBHR.js +14 -0
- package/dist/chunk-5AYIXQRF.js +4452 -0
- package/dist/{chunk-Z5X7T4QZ.js → chunk-5TIP2EPP.js} +1519 -2565
- package/dist/{chunk-5LADPJ4C.js → chunk-GAERET5Q.js} +138 -504
- package/dist/chunk-GF3PX3VM.js +41 -0
- package/dist/chunk-GKZQ5AG5.js +44 -0
- package/dist/chunk-IBPS64W3.js +1069 -0
- package/dist/{chunk-ZYADFKX3.js → chunk-MC3C2XM5.js} +34 -1
- package/dist/chunk-NSLTJBUC.js +270 -0
- package/dist/chunk-OJSIZDZD.js +9 -0
- package/dist/chunk-OWGQWQUP.js +45 -0
- package/dist/chunk-SIY3JA7T.js +3062 -0
- package/dist/{chunk-M5M65AYP.js → chunk-SOQW7356.js} +271 -1934
- package/dist/chunk-U74RE3L7.js +3233 -0
- package/dist/chunk-VBPYU7GO.js +597 -0
- package/dist/chunk-VTHBPXDQ.js +1750 -0
- package/dist/{chunk-KH52KJSJ.js → chunk-XFJ4S4G2.js} +844 -39
- package/dist/chunk-Y5NB3FTH.js +106 -0
- package/dist/{chunk-RYMSM3OS.js → chunk-ZX55JBV2.js} +1710 -322
- package/dist/claim-slot-policy-CdrW_1l4.d.ts +13 -0
- package/dist/index.d.ts +630 -51
- package/dist/index.js +852 -4683
- package/dist/lifecycle-checkpoint-IAC5FCQU.js +154 -0
- package/dist/{claim-slot-policy-CQ-h0GaV.d.ts → ports-C4QkwDBS.d.ts} +168 -78
- package/dist/scan-6JKPOQHD.js +6 -0
- package/dist/service-EKFACEN6.js +15 -0
- package/dist/service-RHNB5AEQ.js +861 -0
- package/dist/sink-AUAAWC5O.js +8 -0
- package/package.json +1 -1
- package/dist/cli.d.ts +0 -1
- package/dist/internal-eval-server.d.ts +0 -1
- package/dist/internal-recall-eval-server.d.ts +0 -1
|
@@ -1,44 +1,71 @@
|
|
|
1
1
|
import {
|
|
2
|
+
BEFORE_TURN_DEBUG_ARTIFACT_DEFAULT_TOP_K,
|
|
3
|
+
BEFORE_TURN_DEBUG_ARTIFACT_MAX_TOP_K,
|
|
4
|
+
RECALL_DEBUG_ARTIFACT_DEFAULT_TOP_K,
|
|
5
|
+
RECALL_DEBUG_ARTIFACT_MAX_TOP_K
|
|
6
|
+
} from "./chunk-5AXMFBHR.js";
|
|
7
|
+
import {
|
|
8
|
+
applyDefaultClaimKeyLifecycle,
|
|
2
9
|
composeProcedureRecallText,
|
|
3
10
|
computeProcedureRevisionHash,
|
|
4
11
|
computeProcedureSourceHash
|
|
5
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-MC3C2XM5.js";
|
|
6
13
|
import {
|
|
14
|
+
createSessionStartRepository,
|
|
7
15
|
formatAgenrBeforeTurnRecall,
|
|
8
|
-
|
|
9
|
-
|
|
16
|
+
listActiveAbstainDirectives,
|
|
17
|
+
listActiveSessionStartProactiveDirectives,
|
|
18
|
+
listActiveTopicProactiveDirectives,
|
|
19
|
+
runBeforeTurn,
|
|
20
|
+
runSessionStart
|
|
21
|
+
} from "./chunk-XFJ4S4G2.js";
|
|
10
22
|
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
CLAIM_SUPPORT_MODES,
|
|
14
|
-
ENTRY_TYPES,
|
|
15
|
-
EXPIRY_LEVELS,
|
|
23
|
+
DREAM_STAGES,
|
|
24
|
+
DREAM_TIERS,
|
|
16
25
|
attachCrossEncoderPort,
|
|
17
|
-
|
|
26
|
+
completeDreamRun,
|
|
18
27
|
createDatabase,
|
|
28
|
+
createDreamRun,
|
|
19
29
|
createEmbeddingClient,
|
|
20
30
|
createOpenAICrossEncoder,
|
|
31
|
+
createProfileSnapshot,
|
|
21
32
|
createRecallAdapter,
|
|
22
|
-
|
|
33
|
+
mapRunRow,
|
|
23
34
|
normalizeProcedureDefinition,
|
|
24
|
-
parseOptionalBoolean,
|
|
25
|
-
parseOptionalIntegerInRange,
|
|
26
|
-
parseOptionalTimestampString,
|
|
27
|
-
parseOptionalTrimmedString,
|
|
28
|
-
parseRequiredTrimmedString,
|
|
29
35
|
projectClaimCentricRecallEntry,
|
|
30
|
-
pushIssue,
|
|
31
|
-
pushUnexpectedFields,
|
|
32
|
-
readConfig,
|
|
33
36
|
resolveCrossEncoderApiKey,
|
|
34
37
|
resolveEmbeddingApiKey,
|
|
35
38
|
resolveEmbeddingModel,
|
|
36
39
|
resolveModel,
|
|
37
|
-
runUnifiedRecall
|
|
38
|
-
|
|
40
|
+
runUnifiedRecall,
|
|
41
|
+
updateDreamState
|
|
42
|
+
} from "./chunk-5TIP2EPP.js";
|
|
39
43
|
import {
|
|
40
44
|
recall
|
|
41
|
-
} from "./chunk-
|
|
45
|
+
} from "./chunk-GAERET5Q.js";
|
|
46
|
+
import {
|
|
47
|
+
deriveDreamEfficiencySummary,
|
|
48
|
+
estimateProfileInjectionTokens
|
|
49
|
+
} from "./chunk-GF3PX3VM.js";
|
|
50
|
+
import {
|
|
51
|
+
CLAIM_KEY_SOURCES,
|
|
52
|
+
CLAIM_KEY_STATUSES,
|
|
53
|
+
CLAIM_SUPPORT_MODES,
|
|
54
|
+
DURABLE_KINDS,
|
|
55
|
+
EXPIRY_LEVELS,
|
|
56
|
+
composeEmbeddingText,
|
|
57
|
+
isRecord,
|
|
58
|
+
parseDirectivePolarity,
|
|
59
|
+
parseDirectiveTrigger,
|
|
60
|
+
parseOptionalBoolean,
|
|
61
|
+
parseOptionalIntegerInRange,
|
|
62
|
+
parseOptionalTimestampString,
|
|
63
|
+
parseOptionalTrimmedString,
|
|
64
|
+
parseRequiredTrimmedString,
|
|
65
|
+
pushIssue,
|
|
66
|
+
pushUnexpectedFields,
|
|
67
|
+
readConfig
|
|
68
|
+
} from "./chunk-VTHBPXDQ.js";
|
|
42
69
|
|
|
43
70
|
// src/internal-eval-server.ts
|
|
44
71
|
import process from "process";
|
|
@@ -46,6 +73,103 @@ import process from "process";
|
|
|
46
73
|
// src/adapters/api/internal-eval-server.ts
|
|
47
74
|
import { createServer } from "http";
|
|
48
75
|
|
|
76
|
+
// src/app/evals/ablation-arm.ts
|
|
77
|
+
var ABLATION_ARMS = /* @__PURE__ */ new Set(["memory-off", "store-only", "dreaming-on"]);
|
|
78
|
+
function resolveAblationConfig(sandbox) {
|
|
79
|
+
const arm = parseAblationArm(sandbox?.ablationArm);
|
|
80
|
+
const now = sandbox?.now?.trim();
|
|
81
|
+
const profileSnapshot = sandbox?.profileSnapshot;
|
|
82
|
+
return {
|
|
83
|
+
...arm ? { arm } : {},
|
|
84
|
+
...now ? { now } : {},
|
|
85
|
+
...profileSnapshot ? { profileSnapshot } : {}
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
function isMemoryOffArm(config) {
|
|
89
|
+
return config.arm === "memory-off";
|
|
90
|
+
}
|
|
91
|
+
function shouldProvisionProfileSnapshot(config) {
|
|
92
|
+
return config.arm === "dreaming-on" && config.profileSnapshot !== void 0;
|
|
93
|
+
}
|
|
94
|
+
function parseAblationArm(value) {
|
|
95
|
+
if (value === void 0) {
|
|
96
|
+
return void 0;
|
|
97
|
+
}
|
|
98
|
+
if (typeof value !== "string" || !ABLATION_ARMS.has(value)) {
|
|
99
|
+
throw new Error(`sandbox.ablationArm must be one of: ${[...ABLATION_ARMS].join(", ")}.`);
|
|
100
|
+
}
|
|
101
|
+
return value;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// src/app/evals/eval-embedding.ts
|
|
105
|
+
function createEvalEmbeddingResolver() {
|
|
106
|
+
let sharedEmbeddingPort;
|
|
107
|
+
let sharedEmbeddingError;
|
|
108
|
+
const getSupport = () => {
|
|
109
|
+
if (sharedEmbeddingPort) {
|
|
110
|
+
return {
|
|
111
|
+
available: true,
|
|
112
|
+
port: sharedEmbeddingPort
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
if (sharedEmbeddingError) {
|
|
116
|
+
return {
|
|
117
|
+
available: false,
|
|
118
|
+
error: sharedEmbeddingError
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
const config = readConfig();
|
|
122
|
+
try {
|
|
123
|
+
sharedEmbeddingPort = createEmbeddingClient(resolveEmbeddingApiKey(config), resolveEmbeddingModel(config));
|
|
124
|
+
return {
|
|
125
|
+
available: true,
|
|
126
|
+
port: sharedEmbeddingPort
|
|
127
|
+
};
|
|
128
|
+
} catch (error) {
|
|
129
|
+
sharedEmbeddingError = error instanceof Error ? error.message : String(error);
|
|
130
|
+
return {
|
|
131
|
+
available: false,
|
|
132
|
+
error: sharedEmbeddingError
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
const requirePort = () => {
|
|
137
|
+
const support = getSupport();
|
|
138
|
+
if (!support.port) {
|
|
139
|
+
throw new Error(support.error ?? "Embeddings are unavailable.");
|
|
140
|
+
}
|
|
141
|
+
return support.port;
|
|
142
|
+
};
|
|
143
|
+
const portOrUnavailable = () => {
|
|
144
|
+
const support = getSupport();
|
|
145
|
+
return support.port ?? createUnavailableEmbeddingPort(support.error ?? "Embeddings are unavailable.");
|
|
146
|
+
};
|
|
147
|
+
return {
|
|
148
|
+
getSupport,
|
|
149
|
+
requirePort,
|
|
150
|
+
portOrUnavailable
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function createUnavailableEmbeddingPort(message) {
|
|
154
|
+
return {
|
|
155
|
+
async embed() {
|
|
156
|
+
throw new Error(message);
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/app/evals/eval-clock.ts
|
|
162
|
+
function parseEvalNow(nowIso) {
|
|
163
|
+
if (!nowIso) {
|
|
164
|
+
return void 0;
|
|
165
|
+
}
|
|
166
|
+
const parsed = new Date(nowIso);
|
|
167
|
+
if (!Number.isFinite(parsed.getTime())) {
|
|
168
|
+
throw new Error(`sandbox.now must be a parseable ISO timestamp. Received: ${nowIso}`);
|
|
169
|
+
}
|
|
170
|
+
return parsed;
|
|
171
|
+
}
|
|
172
|
+
|
|
49
173
|
// src/app/evals/recall/provision-fixtures.ts
|
|
50
174
|
import { createHash } from "crypto";
|
|
51
175
|
var DEFAULT_IMPORTANCE = 6;
|
|
@@ -58,7 +182,7 @@ async function provisionRecallEvalFixtures(params) {
|
|
|
58
182
|
provisionedCount: 0,
|
|
59
183
|
providedIdCount: 0,
|
|
60
184
|
generatedIdCount: 0,
|
|
61
|
-
|
|
185
|
+
staleCount: 0,
|
|
62
186
|
supersededCount: 0,
|
|
63
187
|
createdAtDefaultedCount: 0,
|
|
64
188
|
updatedAtDefaultedCount: 0,
|
|
@@ -71,14 +195,14 @@ async function provisionRecallEvalFixtures(params) {
|
|
|
71
195
|
}
|
|
72
196
|
await params.store.withTransaction(async (store) => {
|
|
73
197
|
for (const [index, fixture] of preparedBatch.insertionOrder.entries()) {
|
|
74
|
-
await store.
|
|
198
|
+
await store.insertDurable(fixture.entry, embeddings[index] ?? [], fixture.contentHash);
|
|
75
199
|
}
|
|
76
200
|
});
|
|
77
201
|
return {
|
|
78
202
|
provisionedCount: preparedBatch.insertionOrder.length,
|
|
79
203
|
providedIdCount: preparedBatch.providedIdCount,
|
|
80
204
|
generatedIdCount: preparedBatch.generatedIdCount,
|
|
81
|
-
|
|
205
|
+
staleCount: preparedBatch.staleCount,
|
|
82
206
|
supersededCount: preparedBatch.supersededCount,
|
|
83
207
|
createdAtDefaultedCount: preparedBatch.createdAtDefaultedCount,
|
|
84
208
|
updatedAtDefaultedCount: preparedBatch.updatedAtDefaultedCount,
|
|
@@ -111,7 +235,7 @@ ${entry.content}`),
|
|
|
111
235
|
insertionOrder: topologicallySortFixtures(prepared),
|
|
112
236
|
providedIdCount: fixtures.filter((fixture) => fixture.id !== void 0).length,
|
|
113
237
|
generatedIdCount: fixtures.filter((fixture) => fixture.id === void 0).length,
|
|
114
|
-
|
|
238
|
+
staleCount: prepared.filter((fixture) => fixture.entry.valid_to !== void 0).length,
|
|
115
239
|
supersededCount: prepared.filter((fixture) => fixture.entry.superseded_by !== void 0).length,
|
|
116
240
|
createdAtDefaultedCount: fixtures.filter((fixture) => fixture.created_at === void 0).length,
|
|
117
241
|
updatedAtDefaultedCount: fixtures.filter((fixture) => fixture.updated_at === void 0).length,
|
|
@@ -121,43 +245,44 @@ ${entry.content}`),
|
|
|
121
245
|
function buildEntry(fixture, id, provisionedAt) {
|
|
122
246
|
const createdAt = fixture.created_at ?? provisionedAt;
|
|
123
247
|
const updatedAt = fixture.updated_at ?? createdAt;
|
|
124
|
-
return
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
248
|
+
return applyDefaultClaimKeyLifecycle(
|
|
249
|
+
{
|
|
250
|
+
id,
|
|
251
|
+
type: fixture.type,
|
|
252
|
+
subject: fixture.subject,
|
|
253
|
+
content: fixture.content,
|
|
254
|
+
importance: fixture.importance ?? DEFAULT_IMPORTANCE,
|
|
255
|
+
expiry: fixture.expiry ?? DEFAULT_EXPIRY,
|
|
256
|
+
tags: fixture.tags ?? [],
|
|
257
|
+
source_file: fixture.source_file,
|
|
258
|
+
source_context: fixture.source_context,
|
|
259
|
+
quality_score: DEFAULT_QUALITY_SCORE,
|
|
260
|
+
recall_count: 0,
|
|
261
|
+
superseded_by: fixture.superseded_by,
|
|
262
|
+
claim_key: fixture.claim_key,
|
|
263
|
+
claim_key_status: fixture.claim_key_status,
|
|
264
|
+
claim_key_source: fixture.claim_key_source,
|
|
265
|
+
claim_support_source_kind: fixture.claim_support_source_kind,
|
|
266
|
+
claim_support_locator: fixture.claim_support_locator,
|
|
267
|
+
claim_support_observed_at: fixture.claim_support_observed_at,
|
|
268
|
+
claim_support_mode: fixture.claim_support_mode,
|
|
269
|
+
valid_from: fixture.valid_from,
|
|
270
|
+
valid_to: fixture.valid_to,
|
|
271
|
+
supersession_kind: fixture.supersession_kind,
|
|
272
|
+
supersession_reason: fixture.supersession_reason,
|
|
273
|
+
directive_polarity: fixture.directive_polarity,
|
|
274
|
+
directive_trigger: fixture.directive_trigger,
|
|
275
|
+
created_at: createdAt,
|
|
276
|
+
updated_at: updatedAt
|
|
277
|
+
},
|
|
278
|
+
"eval fixture"
|
|
279
|
+
);
|
|
154
280
|
}
|
|
155
281
|
function summarizePreparedFixture(entry) {
|
|
156
282
|
return {
|
|
157
283
|
id: entry.id,
|
|
158
284
|
created_at: entry.created_at,
|
|
159
285
|
updated_at: entry.updated_at,
|
|
160
|
-
retired: entry.retired,
|
|
161
286
|
superseded_by: entry.superseded_by,
|
|
162
287
|
claim_key: entry.claim_key,
|
|
163
288
|
claim_key_status: entry.claim_key_status,
|
|
@@ -277,9 +402,10 @@ function prepareProcedures(caseId, fixtures, provisionedAt) {
|
|
|
277
402
|
recall_text: composeProcedureRecallText(normalizedBody),
|
|
278
403
|
revision_hash: computeProcedureRevisionHash(normalizedBody),
|
|
279
404
|
source_hash: computeProcedureSourceHash(JSON.stringify(normalizedBody)),
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
405
|
+
valid_from: fixture.valid_from,
|
|
406
|
+
valid_to: fixture.valid_to,
|
|
407
|
+
supersession_kind: fixture.supersession_kind,
|
|
408
|
+
supersession_reason: fixture.supersession_reason,
|
|
283
409
|
superseded_by: fixture.superseded_by,
|
|
284
410
|
created_at: createdAt,
|
|
285
411
|
updated_at: updatedAt
|
|
@@ -305,20 +431,181 @@ function findDuplicateIds2(ids) {
|
|
|
305
431
|
return duplicates;
|
|
306
432
|
}
|
|
307
433
|
|
|
434
|
+
// src/app/evals/provision-sandbox.ts
|
|
435
|
+
async function provisionEvalSandbox(params) {
|
|
436
|
+
let entryProvisionResult;
|
|
437
|
+
if (params.memoryPool.length > 0) {
|
|
438
|
+
if (!params.embedding) {
|
|
439
|
+
throw new Error("Embeddings are unavailable.");
|
|
440
|
+
}
|
|
441
|
+
entryProvisionResult = await provisionRecallEvalFixtures({
|
|
442
|
+
caseId: params.caseId,
|
|
443
|
+
memoryPool: params.memoryPool,
|
|
444
|
+
store: params.sandbox.fixtureStore,
|
|
445
|
+
embedding: params.embedding,
|
|
446
|
+
provisionedAt: params.provisionedAt
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
if ((params.procedurePool?.length ?? 0) > 0) {
|
|
450
|
+
await provisionRecallEvalProcedureFixtures({
|
|
451
|
+
caseId: params.caseId,
|
|
452
|
+
procedurePool: params.procedurePool ?? [],
|
|
453
|
+
store: params.sandbox.fixtureStore,
|
|
454
|
+
provisionedAt: params.provisionedAt
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
const profileSnapshot = params.profileSnapshot ? await params.sandbox.provisionProfileSnapshot(params.profileSnapshot, params.provisionedAt) : void 0;
|
|
458
|
+
return {
|
|
459
|
+
...entryProvisionResult ? { entryProvisionResult } : {},
|
|
460
|
+
...profileSnapshot ? { profileSnapshotId: profileSnapshot.snapshotId } : {}
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
|
|
308
464
|
// src/app/evals/recall/sandbox.ts
|
|
309
465
|
import { access, copyFile, mkdir, mkdtemp, rm } from "fs/promises";
|
|
310
466
|
import { tmpdir } from "os";
|
|
311
467
|
import path from "path";
|
|
312
468
|
|
|
469
|
+
// src/adapters/db/eval-dream-run-store.ts
|
|
470
|
+
function createEvalDreamRunStore(executor) {
|
|
471
|
+
return {
|
|
472
|
+
provisionDreamRun: (fixture) => provisionEvalDreamRun(executor, fixture),
|
|
473
|
+
getDreamRun: (runId) => getEvalDreamRun(executor, runId)
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
async function provisionEvalDreamRun(executor, fixture) {
|
|
477
|
+
const completedAt = fixture.completedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
478
|
+
const runId = await createDreamRun(executor, {
|
|
479
|
+
tier: fixture.tier,
|
|
480
|
+
dryRun: false,
|
|
481
|
+
startedAt: completedAt
|
|
482
|
+
});
|
|
483
|
+
const efficiency = fixture.summaryJson.efficiency;
|
|
484
|
+
const estimatedCostUsd = fixture.estimatedCostUsd ?? (efficiency?.costPerSynthesizedDurableUsd !== void 0 && efficiency?.costPerSynthesizedDurableUsd !== null && efficiency.synthesizedDurableMutations > 0 ? efficiency.costPerSynthesizedDurableUsd * efficiency.synthesizedDurableMutations : 0);
|
|
485
|
+
await completeDreamRun(executor, runId, {
|
|
486
|
+
status: "completed",
|
|
487
|
+
inputTokens: 0,
|
|
488
|
+
outputTokens: 0,
|
|
489
|
+
estimatedCostUsd,
|
|
490
|
+
actionsTaken: fixture.summaryJson.actions_taken ?? 0,
|
|
491
|
+
actionsSkipped: 0,
|
|
492
|
+
durablesStaled: fixture.summaryJson.prune?.durablesStaled ?? 0,
|
|
493
|
+
summaryJson: fixture.summaryJson,
|
|
494
|
+
completedAt
|
|
495
|
+
});
|
|
496
|
+
return { runId };
|
|
497
|
+
}
|
|
498
|
+
async function getEvalDreamRun(executor, runId) {
|
|
499
|
+
const result = await executor.execute({
|
|
500
|
+
sql: `
|
|
501
|
+
SELECT
|
|
502
|
+
id,
|
|
503
|
+
tier,
|
|
504
|
+
project,
|
|
505
|
+
started_at,
|
|
506
|
+
completed_at,
|
|
507
|
+
status,
|
|
508
|
+
input_tokens,
|
|
509
|
+
output_tokens,
|
|
510
|
+
estimated_cost_usd,
|
|
511
|
+
model,
|
|
512
|
+
actions_taken,
|
|
513
|
+
actions_skipped,
|
|
514
|
+
durables_staled,
|
|
515
|
+
summary_json,
|
|
516
|
+
error,
|
|
517
|
+
dry_run,
|
|
518
|
+
config_json
|
|
519
|
+
FROM dream_runs
|
|
520
|
+
WHERE id = ?
|
|
521
|
+
LIMIT 1
|
|
522
|
+
`,
|
|
523
|
+
args: [runId.trim()]
|
|
524
|
+
});
|
|
525
|
+
const row = result.rows[0];
|
|
526
|
+
if (!row) {
|
|
527
|
+
return null;
|
|
528
|
+
}
|
|
529
|
+
const run = mapRunRow(row);
|
|
530
|
+
return {
|
|
531
|
+
runId: run.id,
|
|
532
|
+
status: run.status,
|
|
533
|
+
completedAt: run.completedAt,
|
|
534
|
+
estimatedCostUsd: run.estimatedCostUsd,
|
|
535
|
+
summaryJson: run.summaryJson
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
|
|
313
539
|
// src/adapters/db/eval-fixture-store.ts
|
|
314
540
|
function createRecallEvalFixtureStore(database) {
|
|
315
541
|
return {
|
|
316
|
-
|
|
542
|
+
insertDurable: async (entry, embedding, contentHash) => database.insertDurable(entry, embedding, contentHash),
|
|
317
543
|
insertProcedure: async (procedure) => database.upsertProcedure(procedure),
|
|
318
544
|
withTransaction: async (fn) => database.withTransaction(async (transaction) => fn(createRecallEvalFixtureStore(transaction)))
|
|
319
545
|
};
|
|
320
546
|
}
|
|
321
547
|
|
|
548
|
+
// src/adapters/db/eval-profile-snapshot-store.ts
|
|
549
|
+
import { createHash as createHash3, randomUUID } from "crypto";
|
|
550
|
+
async function ensureEvalDreamRunExists(executor, runId, provisionedAt) {
|
|
551
|
+
const normalizedRunId = runId.trim();
|
|
552
|
+
if (normalizedRunId.length === 0) {
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
const existing = await executor.execute({
|
|
556
|
+
sql: `
|
|
557
|
+
SELECT id
|
|
558
|
+
FROM dream_runs
|
|
559
|
+
WHERE id = ?
|
|
560
|
+
LIMIT 1
|
|
561
|
+
`,
|
|
562
|
+
args: [normalizedRunId]
|
|
563
|
+
});
|
|
564
|
+
if (existing.rows[0]) {
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
await executor.execute({
|
|
568
|
+
sql: `
|
|
569
|
+
INSERT INTO dream_runs (
|
|
570
|
+
id,
|
|
571
|
+
tier,
|
|
572
|
+
started_at,
|
|
573
|
+
completed_at,
|
|
574
|
+
status,
|
|
575
|
+
dry_run
|
|
576
|
+
)
|
|
577
|
+
VALUES (?, 'light', ?, ?, 'completed', 0)
|
|
578
|
+
`,
|
|
579
|
+
args: [normalizedRunId, provisionedAt, provisionedAt]
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
async function provisionEvalProfileSnapshot(executor, fixture, provisionedAt) {
|
|
583
|
+
const snapshotId = fixture.id?.trim() || `eval-profile-${randomUUID()}`;
|
|
584
|
+
const durableIds = fixture.durableIds.map((id) => id.trim()).filter((id) => id.length > 0);
|
|
585
|
+
const directiveIds = (fixture.directiveIds ?? []).map((id) => id.trim()).filter((id) => id.length > 0);
|
|
586
|
+
const createdAt = fixture.createdAt ?? provisionedAt;
|
|
587
|
+
const asOf = fixture.asOf ?? createdAt;
|
|
588
|
+
const contentHash = createHash3("sha256").update(JSON.stringify({ durableIds, directiveIds, asOf })).digest("hex");
|
|
589
|
+
const runId = fixture.runId?.trim();
|
|
590
|
+
if (runId) {
|
|
591
|
+
await ensureEvalDreamRunExists(executor, runId, createdAt);
|
|
592
|
+
}
|
|
593
|
+
await createProfileSnapshot(executor, {
|
|
594
|
+
id: snapshotId,
|
|
595
|
+
durableIds,
|
|
596
|
+
directiveIds,
|
|
597
|
+
asOf,
|
|
598
|
+
contentHash,
|
|
599
|
+
runId: runId ?? null,
|
|
600
|
+
createdAt
|
|
601
|
+
});
|
|
602
|
+
await updateDreamState(executor, {
|
|
603
|
+
activeProfileSnapshotId: snapshotId,
|
|
604
|
+
updatedAt: createdAt
|
|
605
|
+
});
|
|
606
|
+
return { snapshotId };
|
|
607
|
+
}
|
|
608
|
+
|
|
322
609
|
// src/app/evals/recall/sandbox.ts
|
|
323
610
|
var SANDBOX_DB_FILENAME = "knowledge.db";
|
|
324
611
|
var SANDBOX_DIR_PREFIX = "agenr-recall-eval-";
|
|
@@ -347,6 +634,12 @@ async function setupRecallEvalSandbox(request) {
|
|
|
347
634
|
fixtureStore: createRecallEvalFixtureStore(openDatabase),
|
|
348
635
|
episodeDatabase: openDatabase,
|
|
349
636
|
procedureDatabase: openDatabase,
|
|
637
|
+
sessionStartRepository: createSessionStartRepository(openDatabase),
|
|
638
|
+
listActiveAbstainDirectives: (now) => listActiveAbstainDirectives(openDatabase, now),
|
|
639
|
+
listActiveSessionStartProactiveDirectives: (now) => listActiveSessionStartProactiveDirectives(openDatabase, now),
|
|
640
|
+
listActiveTopicProactiveDirectives: (now) => listActiveTopicProactiveDirectives(openDatabase, now),
|
|
641
|
+
provisionProfileSnapshot: (fixture, provisionedAt) => provisionEvalProfileSnapshot(openDatabase, fixture, provisionedAt),
|
|
642
|
+
dreamRunStore: createEvalDreamRunStore(openDatabase),
|
|
350
643
|
...snapshot ? { snapshot } : {},
|
|
351
644
|
createRecallPorts: (embedding) => createRecallAdapter(openDatabase, embedding),
|
|
352
645
|
cleanup: async () => {
|
|
@@ -438,14 +731,6 @@ function applyTelemetryWriteGate(ports, sandbox) {
|
|
|
438
731
|
};
|
|
439
732
|
}
|
|
440
733
|
|
|
441
|
-
// src/app/debug-artifacts/before-turn.ts
|
|
442
|
-
var BEFORE_TURN_DEBUG_ARTIFACT_DEFAULT_TOP_K = 10;
|
|
443
|
-
var BEFORE_TURN_DEBUG_ARTIFACT_MAX_TOP_K = 25;
|
|
444
|
-
|
|
445
|
-
// src/app/debug-artifacts/recall.ts
|
|
446
|
-
var RECALL_DEBUG_ARTIFACT_DEFAULT_TOP_K = 10;
|
|
447
|
-
var RECALL_DEBUG_ARTIFACT_MAX_TOP_K = 25;
|
|
448
|
-
|
|
449
734
|
// src/app/evals/before-turn/build-debug-artifact.ts
|
|
450
735
|
function buildBeforeTurnDebugArtifact(params) {
|
|
451
736
|
const { request, patch, sandbox } = params;
|
|
@@ -587,38 +872,11 @@ function normalizePatchForEvalOutput(patch) {
|
|
|
587
872
|
async function runBeforeTurnEvalCase(request, dependencies = {}) {
|
|
588
873
|
const startedAt = Date.now();
|
|
589
874
|
const provisionedAt = new Date(startedAt).toISOString();
|
|
875
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
876
|
+
const evalNow = parseEvalNow(ablation.now);
|
|
877
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
590
878
|
let sandbox;
|
|
591
|
-
let sharedEmbeddingPort;
|
|
592
|
-
let sharedEmbeddingError;
|
|
593
879
|
let timings;
|
|
594
|
-
const getEmbeddingSupport = () => {
|
|
595
|
-
if (sharedEmbeddingPort) {
|
|
596
|
-
return {
|
|
597
|
-
available: true,
|
|
598
|
-
port: sharedEmbeddingPort
|
|
599
|
-
};
|
|
600
|
-
}
|
|
601
|
-
if (sharedEmbeddingError) {
|
|
602
|
-
return {
|
|
603
|
-
available: false,
|
|
604
|
-
error: sharedEmbeddingError
|
|
605
|
-
};
|
|
606
|
-
}
|
|
607
|
-
const config = readConfig();
|
|
608
|
-
try {
|
|
609
|
-
sharedEmbeddingPort = createEmbeddingClient(resolveEmbeddingApiKey(config), resolveEmbeddingModel(config));
|
|
610
|
-
return {
|
|
611
|
-
available: true,
|
|
612
|
-
port: sharedEmbeddingPort
|
|
613
|
-
};
|
|
614
|
-
} catch (error) {
|
|
615
|
-
sharedEmbeddingError = error instanceof Error ? error.message : String(error);
|
|
616
|
-
return {
|
|
617
|
-
available: false,
|
|
618
|
-
error: sharedEmbeddingError
|
|
619
|
-
};
|
|
620
|
-
}
|
|
621
|
-
};
|
|
622
880
|
try {
|
|
623
881
|
const sandboxStartedAt = Date.now();
|
|
624
882
|
try {
|
|
@@ -641,28 +899,18 @@ async function runBeforeTurnEvalCase(request, dependencies = {}) {
|
|
|
641
899
|
timings: request.options?.includeTimings === true ? timings : void 0
|
|
642
900
|
});
|
|
643
901
|
}
|
|
644
|
-
if (request.memoryPool.length > 0 || (request.procedurePool?.length ?? 0) > 0) {
|
|
902
|
+
if (request.memoryPool.length > 0 || (request.procedurePool?.length ?? 0) > 0 || shouldProvisionProfileSnapshot(ablation)) {
|
|
645
903
|
const provisionStartedAt = Date.now();
|
|
646
904
|
try {
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
});
|
|
657
|
-
}
|
|
658
|
-
if ((request.procedurePool?.length ?? 0) > 0) {
|
|
659
|
-
await provisionRecallEvalProcedureFixtures({
|
|
660
|
-
caseId: request.caseId,
|
|
661
|
-
procedurePool: request.procedurePool ?? [],
|
|
662
|
-
store: sandbox.fixtureStore,
|
|
663
|
-
provisionedAt
|
|
664
|
-
});
|
|
665
|
-
}
|
|
905
|
+
await provisionEvalSandbox({
|
|
906
|
+
caseId: request.caseId,
|
|
907
|
+
sandbox,
|
|
908
|
+
memoryPool: request.memoryPool,
|
|
909
|
+
procedurePool: request.procedurePool,
|
|
910
|
+
profileSnapshot: shouldProvisionProfileSnapshot(ablation) ? ablation.profileSnapshot : void 0,
|
|
911
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.portOrUnavailable() : void 0,
|
|
912
|
+
provisionedAt
|
|
913
|
+
});
|
|
666
914
|
timings = {
|
|
667
915
|
...timings,
|
|
668
916
|
fixtureProvisionMs: elapsedMs(provisionStartedAt)
|
|
@@ -685,14 +933,48 @@ async function runBeforeTurnEvalCase(request, dependencies = {}) {
|
|
|
685
933
|
}
|
|
686
934
|
const beforeTurnStartedAt = Date.now();
|
|
687
935
|
try {
|
|
688
|
-
|
|
689
|
-
|
|
936
|
+
if (isMemoryOffArm(ablation)) {
|
|
937
|
+
timings = {
|
|
938
|
+
...timings,
|
|
939
|
+
beforeTurnMs: elapsedMs(beforeTurnStartedAt),
|
|
940
|
+
totalMs: elapsedMs(startedAt)
|
|
941
|
+
};
|
|
942
|
+
return buildBeforeTurnEvalSuccessResponse({
|
|
943
|
+
request,
|
|
944
|
+
patch: {
|
|
945
|
+
durableMemory: [],
|
|
946
|
+
diagnostics: {
|
|
947
|
+
abstained: true,
|
|
948
|
+
abstentionReasons: ["memory_off_ablation"],
|
|
949
|
+
queryVariants: [],
|
|
950
|
+
recentTurnCount: 0,
|
|
951
|
+
turnSignalLabels: [],
|
|
952
|
+
durableRecallUsed: false,
|
|
953
|
+
durableRecallCandidateCount: 0,
|
|
954
|
+
procedureRecallUsed: false,
|
|
955
|
+
procedureCandidateCount: 0,
|
|
956
|
+
notices: ["memory-off ablation arm stubbed before-turn injection."]
|
|
957
|
+
}
|
|
958
|
+
},
|
|
959
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
960
|
+
sandbox
|
|
961
|
+
});
|
|
962
|
+
}
|
|
963
|
+
const activeSandbox = sandbox;
|
|
964
|
+
if (!activeSandbox) {
|
|
965
|
+
throw new Error("Before-turn eval sandbox was not initialized.");
|
|
966
|
+
}
|
|
967
|
+
const embeddingSupport = embeddingResolver.getSupport();
|
|
968
|
+
const sandboxRecallPorts = activeSandbox.createRecallPorts(
|
|
690
969
|
embeddingSupport.port ?? createUnavailableEmbeddingPort(embeddingSupport.error ?? "Embeddings are unavailable.")
|
|
691
970
|
);
|
|
692
|
-
const recallPorts = applyTelemetryWriteGate(attachCrossEncoderPort(sandboxRecallPorts, dependencies.crossEncoder),
|
|
971
|
+
const recallPorts = applyTelemetryWriteGate(attachCrossEncoderPort(sandboxRecallPorts, dependencies.crossEncoder), activeSandbox);
|
|
693
972
|
const patch = await runBeforeTurn(request.beforeTurnInput, {
|
|
694
973
|
recall: recallPorts,
|
|
695
|
-
procedures:
|
|
974
|
+
procedures: activeSandbox.procedureDatabase,
|
|
975
|
+
...evalNow ? { now: evalNow } : {},
|
|
976
|
+
listActiveAbstainDirectives: () => activeSandbox.listActiveAbstainDirectives(evalNow),
|
|
977
|
+
listActiveTopicProactiveDirectives: () => activeSandbox.listActiveTopicProactiveDirectives(evalNow),
|
|
696
978
|
embedQuery: embeddingSupport.port ? async (text) => {
|
|
697
979
|
const vectors = await embeddingSupport.port.embed([text]);
|
|
698
980
|
return vectors[0] ?? [];
|
|
@@ -720,7 +1002,7 @@ async function runBeforeTurnEvalCase(request, dependencies = {}) {
|
|
|
720
1002
|
patch,
|
|
721
1003
|
renderedPatchText,
|
|
722
1004
|
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
723
|
-
sandbox
|
|
1005
|
+
sandbox: activeSandbox
|
|
724
1006
|
});
|
|
725
1007
|
} catch (error) {
|
|
726
1008
|
timings = {
|
|
@@ -760,19 +1042,14 @@ function toErrorDetails(error) {
|
|
|
760
1042
|
cause: String(error)
|
|
761
1043
|
};
|
|
762
1044
|
}
|
|
763
|
-
function createUnavailableEmbeddingPort(message) {
|
|
764
|
-
return {
|
|
765
|
-
async embed() {
|
|
766
|
-
throw new Error(message);
|
|
767
|
-
}
|
|
768
|
-
};
|
|
769
|
-
}
|
|
770
1045
|
function elapsedMs(startedAt) {
|
|
771
1046
|
return Date.now() - startedAt;
|
|
772
1047
|
}
|
|
773
1048
|
|
|
774
1049
|
// src/adapters/api/validation/internal-eval-shared.ts
|
|
775
|
-
var SANDBOX_REQUEST_KEYS = /* @__PURE__ */ new Set(["root", "preserve", "corpusSeed"]);
|
|
1050
|
+
var SANDBOX_REQUEST_KEYS = /* @__PURE__ */ new Set(["root", "preserve", "corpusSeed", "ablationArm", "now", "profileSnapshot"]);
|
|
1051
|
+
var PROFILE_SNAPSHOT_KEYS = /* @__PURE__ */ new Set(["id", "durableIds", "directiveIds", "asOf", "runId", "createdAt"]);
|
|
1052
|
+
var ABLATION_ARMS2 = ["memory-off", "store-only", "dreaming-on"];
|
|
776
1053
|
var CORPUS_SEED_MODES = ["fixture", "snapshot_copy"];
|
|
777
1054
|
var FIXTURE_CORPUS_SEED_KEYS = /* @__PURE__ */ new Set(["mode"]);
|
|
778
1055
|
var SNAPSHOT_COPY_CORPUS_SEED_KEYS = /* @__PURE__ */ new Set(["mode", "snapshotDbPath", "snapshotId", "snapshotLabel", "allowTelemetryWrites"]);
|
|
@@ -788,9 +1065,6 @@ var FIXTURE_ENTRY_KEYS = /* @__PURE__ */ new Set([
|
|
|
788
1065
|
"source_context",
|
|
789
1066
|
"created_at",
|
|
790
1067
|
"updated_at",
|
|
791
|
-
"retired",
|
|
792
|
-
"retired_at",
|
|
793
|
-
"retired_reason",
|
|
794
1068
|
"superseded_by",
|
|
795
1069
|
"claim_key",
|
|
796
1070
|
"claim_key_status",
|
|
@@ -802,7 +1076,9 @@ var FIXTURE_ENTRY_KEYS = /* @__PURE__ */ new Set([
|
|
|
802
1076
|
"valid_from",
|
|
803
1077
|
"valid_to",
|
|
804
1078
|
"supersession_kind",
|
|
805
|
-
"supersession_reason"
|
|
1079
|
+
"supersession_reason",
|
|
1080
|
+
"directive_polarity",
|
|
1081
|
+
"directive_trigger"
|
|
806
1082
|
]);
|
|
807
1083
|
var FIXTURE_PROCEDURE_KEYS = /* @__PURE__ */ new Set([
|
|
808
1084
|
"id",
|
|
@@ -817,9 +1093,10 @@ var FIXTURE_PROCEDURE_KEYS = /* @__PURE__ */ new Set([
|
|
|
817
1093
|
"failure_modes",
|
|
818
1094
|
"sources",
|
|
819
1095
|
"source_file",
|
|
820
|
-
"
|
|
821
|
-
"
|
|
822
|
-
"
|
|
1096
|
+
"valid_from",
|
|
1097
|
+
"valid_to",
|
|
1098
|
+
"supersession_kind",
|
|
1099
|
+
"supersession_reason",
|
|
823
1100
|
"superseded_by",
|
|
824
1101
|
"created_at",
|
|
825
1102
|
"updated_at"
|
|
@@ -850,7 +1127,10 @@ function parseSandbox(value, issues) {
|
|
|
850
1127
|
return {
|
|
851
1128
|
root: parseOptionalTrimmedString(sandbox.root, "sandbox.root", issues),
|
|
852
1129
|
preserve: parseOptionalBoolean(sandbox.preserve, "sandbox.preserve", issues),
|
|
853
|
-
corpusSeed: parseCorpusSeed(sandbox.corpusSeed, issues)
|
|
1130
|
+
corpusSeed: parseCorpusSeed(sandbox.corpusSeed, issues),
|
|
1131
|
+
ablationArm: parseOptionalAblationArm(sandbox.ablationArm, "sandbox.ablationArm", issues),
|
|
1132
|
+
now: parseOptionalTimestampString(sandbox.now, "sandbox.now", issues),
|
|
1133
|
+
profileSnapshot: parseProfileSnapshot(sandbox.profileSnapshot, issues)
|
|
854
1134
|
};
|
|
855
1135
|
}
|
|
856
1136
|
function parseCorpusSeed(value, issues) {
|
|
@@ -922,7 +1202,10 @@ function mapSandboxRequestDto(dto) {
|
|
|
922
1202
|
return {
|
|
923
1203
|
root: dto.root,
|
|
924
1204
|
preserve: dto.preserve,
|
|
925
|
-
corpusSeed: dto.corpusSeed
|
|
1205
|
+
corpusSeed: dto.corpusSeed,
|
|
1206
|
+
ablationArm: dto.ablationArm,
|
|
1207
|
+
now: dto.now,
|
|
1208
|
+
profileSnapshot: dto.profileSnapshot
|
|
926
1209
|
};
|
|
927
1210
|
}
|
|
928
1211
|
function mapFixtureEntryDto(dto) {
|
|
@@ -938,9 +1221,6 @@ function mapFixtureEntryDto(dto) {
|
|
|
938
1221
|
source_context: dto.source_context,
|
|
939
1222
|
created_at: dto.created_at,
|
|
940
1223
|
updated_at: dto.updated_at,
|
|
941
|
-
retired: dto.retired,
|
|
942
|
-
retired_at: dto.retired_at,
|
|
943
|
-
retired_reason: dto.retired_reason,
|
|
944
1224
|
superseded_by: dto.superseded_by,
|
|
945
1225
|
claim_key: dto.claim_key,
|
|
946
1226
|
claim_key_status: dto.claim_key_status,
|
|
@@ -952,7 +1232,9 @@ function mapFixtureEntryDto(dto) {
|
|
|
952
1232
|
valid_from: dto.valid_from,
|
|
953
1233
|
valid_to: dto.valid_to,
|
|
954
1234
|
supersession_kind: dto.supersession_kind,
|
|
955
|
-
supersession_reason: dto.supersession_reason
|
|
1235
|
+
supersession_reason: dto.supersession_reason,
|
|
1236
|
+
directive_polarity: dto.directive_polarity,
|
|
1237
|
+
directive_trigger: dto.directive_trigger
|
|
956
1238
|
};
|
|
957
1239
|
}
|
|
958
1240
|
function mapFixtureProcedureDto(dto) {
|
|
@@ -969,9 +1251,10 @@ function mapFixtureProcedureDto(dto) {
|
|
|
969
1251
|
failure_modes: dto.failure_modes,
|
|
970
1252
|
sources: dto.sources,
|
|
971
1253
|
source_file: dto.source_file,
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
1254
|
+
valid_from: dto.valid_from,
|
|
1255
|
+
valid_to: dto.valid_to,
|
|
1256
|
+
supersession_kind: dto.supersession_kind,
|
|
1257
|
+
supersession_reason: dto.supersession_reason,
|
|
975
1258
|
superseded_by: dto.superseded_by,
|
|
976
1259
|
created_at: dto.created_at,
|
|
977
1260
|
updated_at: dto.updated_at
|
|
@@ -1023,7 +1306,7 @@ function parseFixtureEntry(value, index, issues) {
|
|
|
1023
1306
|
return void 0;
|
|
1024
1307
|
}
|
|
1025
1308
|
pushUnexpectedFields(fixture, FIXTURE_ENTRY_KEYS, basePath, issues);
|
|
1026
|
-
const type =
|
|
1309
|
+
const type = parseDurableKind(fixture.type, `${basePath}.type`, issues);
|
|
1027
1310
|
const subject = parseRequiredTrimmedString(fixture.subject, `${basePath}.subject`, issues);
|
|
1028
1311
|
const content = parseRequiredTrimmedString(fixture.content, `${basePath}.content`, issues);
|
|
1029
1312
|
if (type === void 0 || subject === void 0 || content === void 0) {
|
|
@@ -1044,9 +1327,6 @@ function parseFixtureEntry(value, index, issues) {
|
|
|
1044
1327
|
source_context: parseOptionalTrimmedString(fixture.source_context, `${basePath}.source_context`, issues),
|
|
1045
1328
|
created_at: parseOptionalTimestampString(fixture.created_at, `${basePath}.created_at`, issues),
|
|
1046
1329
|
updated_at: parseOptionalTimestampString(fixture.updated_at, `${basePath}.updated_at`, issues),
|
|
1047
|
-
retired: parseOptionalBoolean(fixture.retired, `${basePath}.retired`, issues),
|
|
1048
|
-
retired_at: parseOptionalTimestampString(fixture.retired_at, `${basePath}.retired_at`, issues),
|
|
1049
|
-
retired_reason: parseOptionalTrimmedString(fixture.retired_reason, `${basePath}.retired_reason`, issues),
|
|
1050
1330
|
superseded_by: parseOptionalTrimmedString(fixture.superseded_by, `${basePath}.superseded_by`, issues),
|
|
1051
1331
|
claim_key: parseOptionalTrimmedString(fixture.claim_key, `${basePath}.claim_key`, issues),
|
|
1052
1332
|
claim_key_status: parseOptionalClaimKeyStatus(fixture.claim_key_status, `${basePath}.claim_key_status`, issues),
|
|
@@ -1058,9 +1338,80 @@ function parseFixtureEntry(value, index, issues) {
|
|
|
1058
1338
|
valid_from: parseOptionalTimestampString(fixture.valid_from, `${basePath}.valid_from`, issues),
|
|
1059
1339
|
valid_to: parseOptionalTimestampString(fixture.valid_to, `${basePath}.valid_to`, issues),
|
|
1060
1340
|
supersession_kind: parseOptionalTrimmedString(fixture.supersession_kind, `${basePath}.supersession_kind`, issues),
|
|
1061
|
-
supersession_reason: parseOptionalTrimmedString(fixture.supersession_reason, `${basePath}.supersession_reason`, issues)
|
|
1341
|
+
supersession_reason: parseOptionalTrimmedString(fixture.supersession_reason, `${basePath}.supersession_reason`, issues),
|
|
1342
|
+
directive_polarity: parseOptionalDirectivePolarity(fixture.directive_polarity, `${basePath}.directive_polarity`, issues),
|
|
1343
|
+
directive_trigger: parseOptionalDirectiveTrigger(fixture.directive_trigger, `${basePath}.directive_trigger`, issues)
|
|
1062
1344
|
};
|
|
1063
1345
|
}
|
|
1346
|
+
function parseOptionalAblationArm(value, path2, issues) {
|
|
1347
|
+
if (value === void 0) {
|
|
1348
|
+
return void 0;
|
|
1349
|
+
}
|
|
1350
|
+
if (typeof value !== "string" || !ABLATION_ARMS2.includes(value)) {
|
|
1351
|
+
pushIssue(issues, path2, `Expected one of: ${ABLATION_ARMS2.join(", ")}.`);
|
|
1352
|
+
return void 0;
|
|
1353
|
+
}
|
|
1354
|
+
return value;
|
|
1355
|
+
}
|
|
1356
|
+
function parseProfileSnapshot(value, issues) {
|
|
1357
|
+
if (value === void 0) {
|
|
1358
|
+
return void 0;
|
|
1359
|
+
}
|
|
1360
|
+
const snapshot = parseObject(value, "sandbox.profileSnapshot", issues);
|
|
1361
|
+
if (snapshot === void 0) {
|
|
1362
|
+
return void 0;
|
|
1363
|
+
}
|
|
1364
|
+
pushUnexpectedFields(snapshot, PROFILE_SNAPSHOT_KEYS, "sandbox.profileSnapshot", issues);
|
|
1365
|
+
const durableIds = parseRequiredStringArray(snapshot.durableIds, "sandbox.profileSnapshot.durableIds", issues);
|
|
1366
|
+
if (durableIds === void 0) {
|
|
1367
|
+
return void 0;
|
|
1368
|
+
}
|
|
1369
|
+
return {
|
|
1370
|
+
id: parseOptionalTrimmedString(snapshot.id, "sandbox.profileSnapshot.id", issues),
|
|
1371
|
+
durableIds,
|
|
1372
|
+
directiveIds: parseOptionalStringArray(snapshot.directiveIds, "sandbox.profileSnapshot.directiveIds", issues),
|
|
1373
|
+
asOf: parseOptionalTimestampString(snapshot.asOf, "sandbox.profileSnapshot.asOf", issues),
|
|
1374
|
+
runId: parseOptionalTrimmedString(snapshot.runId, "sandbox.profileSnapshot.runId", issues),
|
|
1375
|
+
createdAt: parseOptionalTimestampString(snapshot.createdAt, "sandbox.profileSnapshot.createdAt", issues)
|
|
1376
|
+
};
|
|
1377
|
+
}
|
|
1378
|
+
function parseRequiredStringArray(value, path2, issues) {
|
|
1379
|
+
if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== "string" || item.trim().length === 0)) {
|
|
1380
|
+
pushIssue(issues, path2, "Expected a non-empty array of strings.");
|
|
1381
|
+
return void 0;
|
|
1382
|
+
}
|
|
1383
|
+
return value.map((item) => item.trim());
|
|
1384
|
+
}
|
|
1385
|
+
function parseOptionalDirectivePolarity(value, path2, issues) {
|
|
1386
|
+
if (value === void 0) {
|
|
1387
|
+
return void 0;
|
|
1388
|
+
}
|
|
1389
|
+
if (typeof value !== "string") {
|
|
1390
|
+
pushIssue(issues, path2, "Expected a string.");
|
|
1391
|
+
return void 0;
|
|
1392
|
+
}
|
|
1393
|
+
const polarity = parseDirectivePolarity(value);
|
|
1394
|
+
if (!polarity) {
|
|
1395
|
+
pushIssue(issues, path2, "Expected abstain or proactive.");
|
|
1396
|
+
return void 0;
|
|
1397
|
+
}
|
|
1398
|
+
return polarity;
|
|
1399
|
+
}
|
|
1400
|
+
function parseOptionalDirectiveTrigger(value, path2, issues) {
|
|
1401
|
+
if (value === void 0) {
|
|
1402
|
+
return void 0;
|
|
1403
|
+
}
|
|
1404
|
+
if (typeof value !== "string") {
|
|
1405
|
+
pushIssue(issues, path2, "Expected a string.");
|
|
1406
|
+
return void 0;
|
|
1407
|
+
}
|
|
1408
|
+
const trigger = parseDirectiveTrigger(value);
|
|
1409
|
+
if (!trigger) {
|
|
1410
|
+
pushIssue(issues, path2, "Expected session_start, always, or topic:<term>.");
|
|
1411
|
+
return void 0;
|
|
1412
|
+
}
|
|
1413
|
+
return trigger;
|
|
1414
|
+
}
|
|
1064
1415
|
function parseFixtureProcedure(value, index, issues) {
|
|
1065
1416
|
const basePath = `procedurePool[${index}]`;
|
|
1066
1417
|
const fixture = parseObject(value, basePath, issues);
|
|
@@ -1108,9 +1459,10 @@ function parseFixtureProcedure(value, index, issues) {
|
|
|
1108
1459
|
failure_modes: normalized.failure_modes,
|
|
1109
1460
|
sources: normalized.sources,
|
|
1110
1461
|
source_file: parseOptionalTrimmedString(fixture.source_file, `${basePath}.source_file`, issues),
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1462
|
+
valid_from: parseOptionalTimestampString(fixture.valid_from, `${basePath}.valid_from`, issues),
|
|
1463
|
+
valid_to: parseOptionalTimestampString(fixture.valid_to, `${basePath}.valid_to`, issues),
|
|
1464
|
+
supersession_kind: parseOptionalTrimmedString(fixture.supersession_kind, `${basePath}.supersession_kind`, issues),
|
|
1465
|
+
supersession_reason: parseOptionalTrimmedString(fixture.supersession_reason, `${basePath}.supersession_reason`, issues),
|
|
1114
1466
|
superseded_by: parseOptionalTrimmedString(fixture.superseded_by, `${basePath}.superseded_by`, issues),
|
|
1115
1467
|
created_at: parseOptionalTimestampString(fixture.created_at, `${basePath}.created_at`, issues),
|
|
1116
1468
|
updated_at: parseOptionalTimestampString(fixture.updated_at, `${basePath}.updated_at`, issues)
|
|
@@ -1120,9 +1472,9 @@ function parseFixtureProcedure(value, index, issues) {
|
|
|
1120
1472
|
return void 0;
|
|
1121
1473
|
}
|
|
1122
1474
|
}
|
|
1123
|
-
function
|
|
1124
|
-
if (typeof value !== "string" || !
|
|
1125
|
-
pushIssue(issues, path2, `Expected one of: ${
|
|
1475
|
+
function parseDurableKind(value, path2, issues) {
|
|
1476
|
+
if (typeof value !== "string" || !DURABLE_KINDS.includes(value)) {
|
|
1477
|
+
pushIssue(issues, path2, `Expected one of: ${DURABLE_KINDS.join(", ")}.`);
|
|
1126
1478
|
return void 0;
|
|
1127
1479
|
}
|
|
1128
1480
|
return value;
|
|
@@ -1467,28 +1819,715 @@ var jsonResponse = (body, status) => new Response(JSON.stringify(body), {
|
|
|
1467
1819
|
}
|
|
1468
1820
|
});
|
|
1469
1821
|
|
|
1470
|
-
// src/app/evals/
|
|
1471
|
-
function
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
memoryPoolCount: request.memoryPool.length,
|
|
1481
|
-
provisionedCount: 0,
|
|
1482
|
-
requestedDiagnostics: request.options?.includeDiagnostics === true,
|
|
1483
|
-
requestedCandidates: request.options?.includeCandidates === true
|
|
1822
|
+
// src/app/evals/dreaming-efficiency/normalize-response.ts
|
|
1823
|
+
function buildDreamingEfficiencyEvalSuccessResponse(params) {
|
|
1824
|
+
return {
|
|
1825
|
+
status: "ok",
|
|
1826
|
+
caseId: params.request.caseId,
|
|
1827
|
+
efficiency: params.efficiency,
|
|
1828
|
+
profileInjectionTokenEstimate: params.profileInjectionTokenEstimate,
|
|
1829
|
+
...params.storeOnlyEquivalentTokenEstimate !== void 0 ? { storeOnlyEquivalentTokenEstimate: params.storeOnlyEquivalentTokenEstimate } : {},
|
|
1830
|
+
...params.timings ? { timings: params.timings } : {},
|
|
1831
|
+
sandbox: buildSandboxResult2(params.sandbox)
|
|
1484
1832
|
};
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1833
|
+
}
|
|
1834
|
+
function buildDreamingEfficiencyEvalErrorResponse(params) {
|
|
1835
|
+
return {
|
|
1836
|
+
status: "error",
|
|
1837
|
+
caseId: params.request.caseId,
|
|
1838
|
+
error: {
|
|
1839
|
+
code: params.code,
|
|
1840
|
+
message: params.message,
|
|
1841
|
+
...params.details !== void 0 ? { details: params.details } : {}
|
|
1842
|
+
},
|
|
1843
|
+
...params.timings ? { timings: params.timings } : {},
|
|
1844
|
+
...params.sandbox ? { sandbox: buildSandboxResult2(params.sandbox) } : {}
|
|
1845
|
+
};
|
|
1846
|
+
}
|
|
1847
|
+
function buildSandboxResult2(sandbox) {
|
|
1848
|
+
return {
|
|
1849
|
+
root: sandbox.root,
|
|
1850
|
+
dbPath: sandbox.dbPath,
|
|
1851
|
+
preserved: sandbox.preserved,
|
|
1852
|
+
...sandbox.snapshot ? { snapshot: sandbox.snapshot } : {}
|
|
1853
|
+
};
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
// src/app/evals/dreaming-efficiency/run-dreaming-efficiency-eval-case.ts
|
|
1857
|
+
async function runDreamingEfficiencyEvalCase(request) {
|
|
1858
|
+
const startedAt = Date.now();
|
|
1859
|
+
const provisionedAt = new Date(startedAt).toISOString();
|
|
1860
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
1861
|
+
const shouldProvisionProfile = shouldProvisionProfileSnapshot(ablation);
|
|
1862
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
1863
|
+
let sandbox;
|
|
1864
|
+
let timings;
|
|
1865
|
+
let dreamRunId;
|
|
1866
|
+
try {
|
|
1867
|
+
const sandboxStartedAt = Date.now();
|
|
1868
|
+
try {
|
|
1869
|
+
sandbox = await setupRecallEvalSandbox(request.sandbox);
|
|
1870
|
+
timings = { sandboxSetupMs: elapsedMs2(sandboxStartedAt) };
|
|
1871
|
+
} catch (error) {
|
|
1872
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1873
|
+
request,
|
|
1874
|
+
code: "sandbox_setup_failed",
|
|
1875
|
+
message: "Failed to create isolated dreaming-efficiency eval sandbox.",
|
|
1876
|
+
details: toErrorDetails2(error),
|
|
1877
|
+
timings: request.options?.includeTimings === true ? { totalMs: elapsedMs2(startedAt), sandboxSetupMs: elapsedMs2(sandboxStartedAt) } : void 0
|
|
1878
|
+
});
|
|
1879
|
+
}
|
|
1880
|
+
const provisionStartedAt = Date.now();
|
|
1881
|
+
try {
|
|
1882
|
+
if (request.memoryPool.length > 0) {
|
|
1883
|
+
await provisionEvalSandbox({
|
|
1884
|
+
caseId: request.caseId,
|
|
1885
|
+
sandbox,
|
|
1886
|
+
memoryPool: request.memoryPool,
|
|
1887
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.portOrUnavailable() : void 0,
|
|
1888
|
+
provisionedAt
|
|
1889
|
+
});
|
|
1890
|
+
}
|
|
1891
|
+
const seededRun = await sandbox.dreamRunStore.provisionDreamRun(request.dreamRunFixture);
|
|
1892
|
+
dreamRunId = seededRun.runId;
|
|
1893
|
+
if (shouldProvisionProfile) {
|
|
1894
|
+
await sandbox.provisionProfileSnapshot(
|
|
1895
|
+
{
|
|
1896
|
+
...ablation.profileSnapshot,
|
|
1897
|
+
runId: dreamRunId
|
|
1898
|
+
},
|
|
1899
|
+
provisionedAt
|
|
1900
|
+
);
|
|
1901
|
+
}
|
|
1902
|
+
timings = { ...timings, fixtureProvisionMs: elapsedMs2(provisionStartedAt) };
|
|
1903
|
+
} catch (error) {
|
|
1904
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1905
|
+
request,
|
|
1906
|
+
code: "fixture_provision_failed",
|
|
1907
|
+
message: "Failed to provision dreaming-efficiency eval fixtures into isolated storage.",
|
|
1908
|
+
details: toErrorDetails2(error),
|
|
1909
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt), fixtureProvisionMs: elapsedMs2(provisionStartedAt) } : void 0,
|
|
1910
|
+
sandbox
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
try {
|
|
1914
|
+
const persistedRun = dreamRunId ? await sandbox.dreamRunStore.getDreamRun(dreamRunId) : null;
|
|
1915
|
+
if (!persistedRun?.summaryJson) {
|
|
1916
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1917
|
+
request,
|
|
1918
|
+
code: "efficiency_resolution_failed",
|
|
1919
|
+
message: "Persisted dreaming run summary is required for dreaming-efficiency eval cases.",
|
|
1920
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1921
|
+
sandbox
|
|
1922
|
+
});
|
|
1923
|
+
}
|
|
1924
|
+
const efficiency = deriveDreamEfficiencySummary(persistedRun.summaryJson, persistedRun.estimatedCostUsd);
|
|
1925
|
+
if (!efficiency) {
|
|
1926
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1927
|
+
request,
|
|
1928
|
+
code: "efficiency_resolution_failed",
|
|
1929
|
+
message: "Persisted dreaming run summary must include scan and project counters for efficiency derivation.",
|
|
1930
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1931
|
+
sandbox
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1934
|
+
const directiveCount = request.memoryPool.filter((entry) => entry.type === "directive").length;
|
|
1935
|
+
const storeOnlyEquivalentTokenEstimate = request.memoryPool.length > 0 ? estimateProfileInjectionTokens(request.memoryPool.length, directiveCount) : void 0;
|
|
1936
|
+
timings = {
|
|
1937
|
+
...timings,
|
|
1938
|
+
totalMs: elapsedMs2(startedAt)
|
|
1939
|
+
};
|
|
1940
|
+
return buildDreamingEfficiencyEvalSuccessResponse({
|
|
1941
|
+
request,
|
|
1942
|
+
efficiency,
|
|
1943
|
+
profileInjectionTokenEstimate: efficiency.profileInjectionTokenEstimate,
|
|
1944
|
+
...storeOnlyEquivalentTokenEstimate !== void 0 ? { storeOnlyEquivalentTokenEstimate } : {},
|
|
1945
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
1946
|
+
sandbox
|
|
1947
|
+
});
|
|
1948
|
+
} catch (error) {
|
|
1949
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1950
|
+
request,
|
|
1951
|
+
code: "efficiency_resolution_failed",
|
|
1952
|
+
message: "Failed to resolve dreaming-efficiency telemetry from isolated eval state.",
|
|
1953
|
+
details: toErrorDetails2(error),
|
|
1954
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1955
|
+
sandbox
|
|
1956
|
+
});
|
|
1957
|
+
}
|
|
1958
|
+
} catch (error) {
|
|
1959
|
+
return buildDreamingEfficiencyEvalErrorResponse({
|
|
1960
|
+
request,
|
|
1961
|
+
code: "internal_error",
|
|
1962
|
+
message: "Dreaming-efficiency eval execution failed unexpectedly.",
|
|
1963
|
+
details: toErrorDetails2(error),
|
|
1964
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs2(startedAt) } : void 0,
|
|
1965
|
+
sandbox
|
|
1966
|
+
});
|
|
1967
|
+
} finally {
|
|
1968
|
+
await sandbox?.cleanup().catch(() => void 0);
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
function toErrorDetails2(error) {
|
|
1972
|
+
if (error instanceof Error) {
|
|
1973
|
+
return { cause: error.message };
|
|
1974
|
+
}
|
|
1975
|
+
return { cause: String(error) };
|
|
1976
|
+
}
|
|
1977
|
+
function elapsedMs2(startedAt) {
|
|
1978
|
+
return Date.now() - startedAt;
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
// src/adapters/api/validation/parse-dream-completion-summary.ts
|
|
1982
|
+
var DREAM_COMPLETION_SUMMARY_KEYS = /* @__PURE__ */ new Set([
|
|
1983
|
+
"actions_taken",
|
|
1984
|
+
"backupSkipped",
|
|
1985
|
+
"stages_skipped",
|
|
1986
|
+
"durables_skipped",
|
|
1987
|
+
"observations",
|
|
1988
|
+
"recommendations",
|
|
1989
|
+
"scan",
|
|
1990
|
+
"extract",
|
|
1991
|
+
"reconcile",
|
|
1992
|
+
"temporalize",
|
|
1993
|
+
"project",
|
|
1994
|
+
"prune",
|
|
1995
|
+
"efficiency"
|
|
1996
|
+
]);
|
|
1997
|
+
var DREAM_SCAN_KEYS = /* @__PURE__ */ new Set([
|
|
1998
|
+
"episodesSinceLastRun",
|
|
1999
|
+
"ingestFilesSinceLastRun",
|
|
2000
|
+
"durablesCreatedSinceLastRun",
|
|
2001
|
+
"evidenceRefs",
|
|
2002
|
+
"unsynthesizedImportanceSum"
|
|
2003
|
+
]);
|
|
2004
|
+
var DREAM_EVIDENCE_REF_KEYS = /* @__PURE__ */ new Set(["kind", "locator", "observedAt"]);
|
|
2005
|
+
var DREAM_EXTRACT_KEYS = /* @__PURE__ */ new Set([
|
|
2006
|
+
"episodesScanned",
|
|
2007
|
+
"candidatesEmitted",
|
|
2008
|
+
"newCandidates",
|
|
2009
|
+
"refineCandidates",
|
|
2010
|
+
"knownCandidates",
|
|
2011
|
+
"durablesInserted"
|
|
2012
|
+
]);
|
|
2013
|
+
var DREAM_TEMPORALIZE_KEYS = /* @__PURE__ */ new Set(["revisionsIdentified", "revisionsApplied", "revisionsSkipped"]);
|
|
2014
|
+
var DREAM_PROJECT_KEYS = /* @__PURE__ */ new Set(["profileDurableCount", "directiveCount", "snapshotId", "applied"]);
|
|
2015
|
+
var DREAM_PRUNE_KEYS = /* @__PURE__ */ new Set(["durablesScanned", "candidatesIdentified", "candidatesProtected", "candidatesRetirable", "durablesStaled", "dryRun"]);
|
|
2016
|
+
var DREAM_EFFICIENCY_KEYS = /* @__PURE__ */ new Set([
|
|
2017
|
+
"evidenceItemsRead",
|
|
2018
|
+
"synthesizedDurableMutations",
|
|
2019
|
+
"costPerSynthesizedDurableUsd",
|
|
2020
|
+
"profileInjectionTokenEstimate",
|
|
2021
|
+
"recomputeRatio"
|
|
2022
|
+
]);
|
|
2023
|
+
var DURABLE_SKIP_KEYS = /* @__PURE__ */ new Set(["durable_id", "reason"]);
|
|
2024
|
+
var DREAM_STAGE_SKIP_KEYS = /* @__PURE__ */ new Set(["stage", "reason"]);
|
|
2025
|
+
var DREAM_EVIDENCE_KINDS = ["episode", "ingest_log", "durable", "transcript"];
|
|
2026
|
+
var DREAM_STAGE_SKIP_REASONS = ["light_tier"];
|
|
2027
|
+
function parseDreamCompletionSummary(value, path2, issues) {
|
|
2028
|
+
const summary = parseObject(value, path2, issues);
|
|
2029
|
+
if (summary === void 0) {
|
|
2030
|
+
return void 0;
|
|
2031
|
+
}
|
|
2032
|
+
pushUnexpectedFields(summary, DREAM_COMPLETION_SUMMARY_KEYS, path2, issues);
|
|
2033
|
+
const actionsTaken = parseRequiredNonNegativeInteger(summary.actions_taken, `${path2}.actions_taken`, issues);
|
|
2034
|
+
const backupSkipped = parseOptionalBoolean2(summary.backupSkipped, `${path2}.backupSkipped`, issues);
|
|
2035
|
+
const stagesSkipped = parseDreamStageSkips(summary.stages_skipped, `${path2}.stages_skipped`, issues);
|
|
2036
|
+
const durablesSkipped = parseDurablesSkipped(summary.durables_skipped, `${path2}.durables_skipped`, issues);
|
|
2037
|
+
const observations = parseRequiredStringArray2(summary.observations, `${path2}.observations`, issues);
|
|
2038
|
+
const recommendations = parseRequiredStringArray2(summary.recommendations, `${path2}.recommendations`, issues);
|
|
2039
|
+
const scan = parseDreamScanSummary(summary.scan, `${path2}.scan`, issues);
|
|
2040
|
+
const project = parseDreamProjectSummary(summary.project, `${path2}.project`, issues);
|
|
2041
|
+
parseOptionalObject(summary.reconcile, `${path2}.reconcile`, issues);
|
|
2042
|
+
return {
|
|
2043
|
+
actions_taken: actionsTaken ?? 0,
|
|
2044
|
+
...backupSkipped !== void 0 ? { backupSkipped } : {},
|
|
2045
|
+
...stagesSkipped !== void 0 ? { stages_skipped: stagesSkipped } : {},
|
|
2046
|
+
durables_skipped: durablesSkipped ?? [],
|
|
2047
|
+
observations: observations ?? [],
|
|
2048
|
+
recommendations: recommendations ?? [],
|
|
2049
|
+
...scan ? { scan } : {},
|
|
2050
|
+
...parseDreamExtractSummary(summary.extract, `${path2}.extract`, issues) ?? {},
|
|
2051
|
+
...parseDreamTemporalizeSummary(summary.temporalize, `${path2}.temporalize`, issues) ?? {},
|
|
2052
|
+
...project ? { project } : {},
|
|
2053
|
+
...parseDreamPruneSummary(summary.prune, `${path2}.prune`, issues) ?? {},
|
|
2054
|
+
...parseDreamEfficiencySummary(summary.efficiency, `${path2}.efficiency`, issues) ?? {}
|
|
2055
|
+
};
|
|
2056
|
+
}
|
|
2057
|
+
function parseDreamStageSkips(value, path2, issues) {
|
|
2058
|
+
if (value === void 0) {
|
|
2059
|
+
return void 0;
|
|
2060
|
+
}
|
|
2061
|
+
if (!Array.isArray(value)) {
|
|
2062
|
+
pushIssue(issues, path2, "Expected an array.");
|
|
2063
|
+
return void 0;
|
|
2064
|
+
}
|
|
2065
|
+
return value.flatMap((item, index) => {
|
|
2066
|
+
const itemPath = `${path2}[${index}]`;
|
|
2067
|
+
const record = parseObject(item, itemPath, issues);
|
|
2068
|
+
if (record === void 0) {
|
|
2069
|
+
return [];
|
|
2070
|
+
}
|
|
2071
|
+
pushUnexpectedFields(record, DREAM_STAGE_SKIP_KEYS, itemPath, issues);
|
|
2072
|
+
const stage = parseDreamStage(record.stage, `${itemPath}.stage`, issues);
|
|
2073
|
+
const reason = parseDreamStageSkipReason(record.reason, `${itemPath}.reason`, issues);
|
|
2074
|
+
return [
|
|
2075
|
+
{
|
|
2076
|
+
stage: stage ?? "scan",
|
|
2077
|
+
reason: reason ?? "light_tier"
|
|
2078
|
+
}
|
|
2079
|
+
];
|
|
2080
|
+
});
|
|
2081
|
+
}
|
|
2082
|
+
function parseDreamScanSummary(value, path2, issues) {
|
|
2083
|
+
const scan = parseRequiredObject(value, path2, issues);
|
|
2084
|
+
if (scan === void 0) {
|
|
2085
|
+
return void 0;
|
|
2086
|
+
}
|
|
2087
|
+
pushUnexpectedFields(scan, DREAM_SCAN_KEYS, path2, issues);
|
|
2088
|
+
return {
|
|
2089
|
+
episodesSinceLastRun: parseRequiredNonNegativeInteger(scan.episodesSinceLastRun, `${path2}.episodesSinceLastRun`, issues) ?? 0,
|
|
2090
|
+
ingestFilesSinceLastRun: parseRequiredNonNegativeInteger(scan.ingestFilesSinceLastRun, `${path2}.ingestFilesSinceLastRun`, issues) ?? 0,
|
|
2091
|
+
durablesCreatedSinceLastRun: parseRequiredNonNegativeInteger(scan.durablesCreatedSinceLastRun, `${path2}.durablesCreatedSinceLastRun`, issues) ?? 0,
|
|
2092
|
+
evidenceRefs: parseEvidenceRefs(scan.evidenceRefs, `${path2}.evidenceRefs`, issues) ?? [],
|
|
2093
|
+
unsynthesizedImportanceSum: parseRequiredNonNegativeNumber(scan.unsynthesizedImportanceSum, `${path2}.unsynthesizedImportanceSum`, issues) ?? 0
|
|
2094
|
+
};
|
|
2095
|
+
}
|
|
2096
|
+
function parseDreamExtractSummary(value, path2, issues) {
|
|
2097
|
+
if (value === void 0) {
|
|
2098
|
+
return void 0;
|
|
2099
|
+
}
|
|
2100
|
+
const extract = parseObject(value, path2, issues);
|
|
2101
|
+
if (extract === void 0) {
|
|
2102
|
+
return void 0;
|
|
2103
|
+
}
|
|
2104
|
+
pushUnexpectedFields(extract, DREAM_EXTRACT_KEYS, path2, issues);
|
|
2105
|
+
return {
|
|
2106
|
+
extract: {
|
|
2107
|
+
episodesScanned: parseRequiredNonNegativeInteger(extract.episodesScanned, `${path2}.episodesScanned`, issues) ?? 0,
|
|
2108
|
+
candidatesEmitted: parseRequiredNonNegativeInteger(extract.candidatesEmitted, `${path2}.candidatesEmitted`, issues) ?? 0,
|
|
2109
|
+
newCandidates: parseRequiredNonNegativeInteger(extract.newCandidates, `${path2}.newCandidates`, issues) ?? 0,
|
|
2110
|
+
refineCandidates: parseRequiredNonNegativeInteger(extract.refineCandidates, `${path2}.refineCandidates`, issues) ?? 0,
|
|
2111
|
+
knownCandidates: parseRequiredNonNegativeInteger(extract.knownCandidates, `${path2}.knownCandidates`, issues) ?? 0,
|
|
2112
|
+
durablesInserted: parseRequiredNonNegativeInteger(extract.durablesInserted, `${path2}.durablesInserted`, issues) ?? 0
|
|
2113
|
+
}
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
function parseDreamTemporalizeSummary(value, path2, issues) {
|
|
2117
|
+
if (value === void 0) {
|
|
2118
|
+
return void 0;
|
|
2119
|
+
}
|
|
2120
|
+
const temporalize = parseObject(value, path2, issues);
|
|
2121
|
+
if (temporalize === void 0) {
|
|
2122
|
+
return void 0;
|
|
2123
|
+
}
|
|
2124
|
+
pushUnexpectedFields(temporalize, DREAM_TEMPORALIZE_KEYS, path2, issues);
|
|
2125
|
+
return {
|
|
2126
|
+
temporalize: {
|
|
2127
|
+
revisionsIdentified: parseRequiredNonNegativeInteger(temporalize.revisionsIdentified, `${path2}.revisionsIdentified`, issues) ?? 0,
|
|
2128
|
+
revisionsApplied: parseRequiredNonNegativeInteger(temporalize.revisionsApplied, `${path2}.revisionsApplied`, issues) ?? 0,
|
|
2129
|
+
revisionsSkipped: parseRequiredNonNegativeInteger(temporalize.revisionsSkipped, `${path2}.revisionsSkipped`, issues) ?? 0
|
|
2130
|
+
}
|
|
2131
|
+
};
|
|
2132
|
+
}
|
|
2133
|
+
function parseDreamProjectSummary(value, path2, issues) {
|
|
2134
|
+
const project = parseRequiredObject(value, path2, issues);
|
|
2135
|
+
if (project === void 0) {
|
|
2136
|
+
return void 0;
|
|
2137
|
+
}
|
|
2138
|
+
pushUnexpectedFields(project, DREAM_PROJECT_KEYS, path2, issues);
|
|
2139
|
+
return {
|
|
2140
|
+
profileDurableCount: parseRequiredNonNegativeInteger(project.profileDurableCount, `${path2}.profileDurableCount`, issues) ?? 0,
|
|
2141
|
+
directiveCount: parseRequiredNonNegativeInteger(project.directiveCount, `${path2}.directiveCount`, issues) ?? 0,
|
|
2142
|
+
snapshotId: parseOptionalStringOrNull(project.snapshotId, `${path2}.snapshotId`, issues),
|
|
2143
|
+
applied: parseRequiredBoolean(project.applied, `${path2}.applied`, issues) ?? false
|
|
2144
|
+
};
|
|
2145
|
+
}
|
|
2146
|
+
function parseDreamPruneSummary(value, path2, issues) {
|
|
2147
|
+
if (value === void 0) {
|
|
2148
|
+
return void 0;
|
|
2149
|
+
}
|
|
2150
|
+
const prune = parseObject(value, path2, issues);
|
|
2151
|
+
if (prune === void 0) {
|
|
2152
|
+
return void 0;
|
|
2153
|
+
}
|
|
2154
|
+
pushUnexpectedFields(prune, DREAM_PRUNE_KEYS, path2, issues);
|
|
2155
|
+
return {
|
|
2156
|
+
prune: {
|
|
2157
|
+
durablesScanned: parseRequiredNonNegativeInteger(prune.durablesScanned, `${path2}.durablesScanned`, issues) ?? 0,
|
|
2158
|
+
candidatesIdentified: parseRequiredNonNegativeInteger(prune.candidatesIdentified, `${path2}.candidatesIdentified`, issues) ?? 0,
|
|
2159
|
+
candidatesProtected: parseRequiredNonNegativeInteger(prune.candidatesProtected, `${path2}.candidatesProtected`, issues) ?? 0,
|
|
2160
|
+
candidatesRetirable: parseRequiredNonNegativeInteger(prune.candidatesRetirable, `${path2}.candidatesRetirable`, issues) ?? 0,
|
|
2161
|
+
durablesStaled: parseRequiredNonNegativeInteger(prune.durablesStaled, `${path2}.durablesStaled`, issues) ?? 0,
|
|
2162
|
+
dryRun: parseRequiredBoolean(prune.dryRun, `${path2}.dryRun`, issues) ?? false
|
|
2163
|
+
}
|
|
2164
|
+
};
|
|
2165
|
+
}
|
|
2166
|
+
function parseDreamEfficiencySummary(value, path2, issues) {
|
|
2167
|
+
if (value === void 0) {
|
|
2168
|
+
return void 0;
|
|
2169
|
+
}
|
|
2170
|
+
const efficiency = parseObject(value, path2, issues);
|
|
2171
|
+
if (efficiency === void 0) {
|
|
2172
|
+
return void 0;
|
|
2173
|
+
}
|
|
2174
|
+
pushUnexpectedFields(efficiency, DREAM_EFFICIENCY_KEYS, path2, issues);
|
|
2175
|
+
return {
|
|
2176
|
+
efficiency: {
|
|
2177
|
+
evidenceItemsRead: parseRequiredNonNegativeInteger(efficiency.evidenceItemsRead, `${path2}.evidenceItemsRead`, issues) ?? 0,
|
|
2178
|
+
synthesizedDurableMutations: parseRequiredNonNegativeInteger(efficiency.synthesizedDurableMutations, `${path2}.synthesizedDurableMutations`, issues) ?? 0,
|
|
2179
|
+
costPerSynthesizedDurableUsd: parseOptionalNumberOrNull(efficiency.costPerSynthesizedDurableUsd, `${path2}.costPerSynthesizedDurableUsd`, issues),
|
|
2180
|
+
profileInjectionTokenEstimate: parseRequiredNonNegativeInteger(efficiency.profileInjectionTokenEstimate, `${path2}.profileInjectionTokenEstimate`, issues) ?? 0,
|
|
2181
|
+
recomputeRatio: parseRequiredNonNegativeNumber(efficiency.recomputeRatio, `${path2}.recomputeRatio`, issues) ?? 0
|
|
2182
|
+
}
|
|
2183
|
+
};
|
|
2184
|
+
}
|
|
2185
|
+
function parseDurablesSkipped(value, path2, issues) {
|
|
2186
|
+
if (!Array.isArray(value)) {
|
|
2187
|
+
pushIssue(issues, path2, "Expected an array.");
|
|
2188
|
+
return void 0;
|
|
2189
|
+
}
|
|
2190
|
+
return value.flatMap((item, index) => {
|
|
2191
|
+
const itemPath = `${path2}[${index}]`;
|
|
2192
|
+
const record = parseObject(item, itemPath, issues);
|
|
2193
|
+
if (record === void 0) {
|
|
2194
|
+
return [];
|
|
2195
|
+
}
|
|
2196
|
+
pushUnexpectedFields(record, DURABLE_SKIP_KEYS, itemPath, issues);
|
|
2197
|
+
const reason = parseRequiredTrimmedString(record.reason, `${itemPath}.reason`, issues);
|
|
2198
|
+
const durableId = parseOptionalTrimmedString(record.durable_id, `${itemPath}.durable_id`, issues);
|
|
2199
|
+
return [
|
|
2200
|
+
{
|
|
2201
|
+
...durableId ? { durable_id: durableId } : {},
|
|
2202
|
+
reason: reason ?? ""
|
|
2203
|
+
}
|
|
2204
|
+
];
|
|
2205
|
+
});
|
|
2206
|
+
}
|
|
2207
|
+
function parseEvidenceRefs(value, path2, issues) {
|
|
2208
|
+
if (!Array.isArray(value)) {
|
|
2209
|
+
pushIssue(issues, path2, "Expected an array.");
|
|
2210
|
+
return void 0;
|
|
2211
|
+
}
|
|
2212
|
+
return value.flatMap((item, index) => {
|
|
2213
|
+
const itemPath = `${path2}[${index}]`;
|
|
2214
|
+
const record = parseObject(item, itemPath, issues);
|
|
2215
|
+
if (record === void 0) {
|
|
2216
|
+
return [];
|
|
2217
|
+
}
|
|
2218
|
+
pushUnexpectedFields(record, DREAM_EVIDENCE_REF_KEYS, itemPath, issues);
|
|
2219
|
+
const kind = parseEvidenceKind(record.kind, `${itemPath}.kind`, issues);
|
|
2220
|
+
const locator = parseRequiredTrimmedString(record.locator, `${itemPath}.locator`, issues);
|
|
2221
|
+
const observedAt = parseOptionalTrimmedString(record.observedAt, `${itemPath}.observedAt`, issues);
|
|
2222
|
+
return [
|
|
2223
|
+
{
|
|
2224
|
+
kind: kind ?? "episode",
|
|
2225
|
+
locator: locator ?? "",
|
|
2226
|
+
...observedAt ? { observedAt } : {}
|
|
2227
|
+
}
|
|
2228
|
+
];
|
|
2229
|
+
});
|
|
2230
|
+
}
|
|
2231
|
+
function parseEvidenceKind(value, path2, issues) {
|
|
2232
|
+
if (typeof value !== "string" || !DREAM_EVIDENCE_KINDS.includes(value)) {
|
|
2233
|
+
pushIssue(issues, path2, `kind must be one of: ${DREAM_EVIDENCE_KINDS.join(", ")}.`);
|
|
2234
|
+
return void 0;
|
|
2235
|
+
}
|
|
2236
|
+
return value;
|
|
2237
|
+
}
|
|
2238
|
+
function parseDreamStage(value, path2, issues) {
|
|
2239
|
+
if (typeof value !== "string" || !DREAM_STAGES.includes(value)) {
|
|
2240
|
+
pushIssue(issues, path2, `stage must be one of: ${DREAM_STAGES.join(", ")}.`);
|
|
2241
|
+
return void 0;
|
|
2242
|
+
}
|
|
2243
|
+
return value;
|
|
2244
|
+
}
|
|
2245
|
+
function parseDreamStageSkipReason(value, path2, issues) {
|
|
2246
|
+
if (typeof value !== "string" || !DREAM_STAGE_SKIP_REASONS.includes(value)) {
|
|
2247
|
+
pushIssue(issues, path2, `reason must be one of: ${DREAM_STAGE_SKIP_REASONS.join(", ")}.`);
|
|
2248
|
+
return void 0;
|
|
2249
|
+
}
|
|
2250
|
+
return value;
|
|
2251
|
+
}
|
|
2252
|
+
function parseOptionalObject(value, path2, issues) {
|
|
2253
|
+
if (value === void 0) {
|
|
2254
|
+
return void 0;
|
|
2255
|
+
}
|
|
2256
|
+
return parseObject(value, path2, issues);
|
|
2257
|
+
}
|
|
2258
|
+
function parseRequiredObject(value, path2, issues) {
|
|
2259
|
+
if (value === void 0) {
|
|
2260
|
+
pushIssue(issues, path2, "Expected an object.");
|
|
2261
|
+
return void 0;
|
|
2262
|
+
}
|
|
2263
|
+
return parseObject(value, path2, issues);
|
|
2264
|
+
}
|
|
2265
|
+
function parseRequiredNonNegativeInteger(value, path2, issues) {
|
|
2266
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
2267
|
+
pushIssue(issues, path2, "Expected a non-negative integer.");
|
|
2268
|
+
return void 0;
|
|
2269
|
+
}
|
|
2270
|
+
return value;
|
|
2271
|
+
}
|
|
2272
|
+
function parseRequiredNonNegativeNumber(value, path2, issues) {
|
|
2273
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
2274
|
+
pushIssue(issues, path2, "Expected a non-negative number.");
|
|
2275
|
+
return void 0;
|
|
2276
|
+
}
|
|
2277
|
+
return value;
|
|
2278
|
+
}
|
|
2279
|
+
function parseOptionalBoolean2(value, path2, issues) {
|
|
2280
|
+
if (value === void 0) {
|
|
2281
|
+
return void 0;
|
|
2282
|
+
}
|
|
2283
|
+
return parseRequiredBoolean(value, path2, issues);
|
|
2284
|
+
}
|
|
2285
|
+
function parseRequiredBoolean(value, path2, issues) {
|
|
2286
|
+
if (typeof value !== "boolean") {
|
|
2287
|
+
pushIssue(issues, path2, "Expected a boolean.");
|
|
2288
|
+
return void 0;
|
|
2289
|
+
}
|
|
2290
|
+
return value;
|
|
2291
|
+
}
|
|
2292
|
+
function parseRequiredStringArray2(value, path2, issues) {
|
|
2293
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
2294
|
+
pushIssue(issues, path2, "Expected an array of strings.");
|
|
2295
|
+
return void 0;
|
|
2296
|
+
}
|
|
2297
|
+
return value;
|
|
2298
|
+
}
|
|
2299
|
+
function parseOptionalStringOrNull(value, path2, issues) {
|
|
2300
|
+
if (value === null || value === void 0) {
|
|
2301
|
+
return null;
|
|
2302
|
+
}
|
|
2303
|
+
return parseOptionalTrimmedString(value, path2, issues) ?? null;
|
|
2304
|
+
}
|
|
2305
|
+
function parseOptionalNumberOrNull(value, path2, issues) {
|
|
2306
|
+
if (value === null) {
|
|
2307
|
+
return null;
|
|
2308
|
+
}
|
|
2309
|
+
return parseRequiredNonNegativeNumber(value, path2, issues) ?? null;
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
// src/adapters/api/validation/dreaming-efficiency-eval-request.ts
|
|
2313
|
+
var ROOT_REQUEST_KEYS2 = /* @__PURE__ */ new Set(["caseId", "description", "sandbox", "memoryPool", "dreamRunFixture", "options"]);
|
|
2314
|
+
var DREAM_RUN_FIXTURE_KEYS = /* @__PURE__ */ new Set(["tier", "summaryJson", "estimatedCostUsd", "completedAt"]);
|
|
2315
|
+
var OPTIONS_KEYS2 = /* @__PURE__ */ new Set(["includeTimings"]);
|
|
2316
|
+
var DreamingEfficiencyEvalRequestValidationError = class extends Error {
|
|
2317
|
+
caseId;
|
|
2318
|
+
issues;
|
|
2319
|
+
/**
|
|
2320
|
+
* Creates one validation error with structured request issues.
|
|
2321
|
+
*
|
|
2322
|
+
* @param issues - Validation failures found at the HTTP boundary.
|
|
2323
|
+
* @param caseId - Optional case identifier extracted before validation failed.
|
|
2324
|
+
*/
|
|
2325
|
+
constructor(issues, caseId) {
|
|
2326
|
+
super("Invalid dreaming-efficiency eval request.");
|
|
2327
|
+
this.name = "DreamingEfficiencyEvalRequestValidationError";
|
|
2328
|
+
this.issues = issues;
|
|
2329
|
+
this.caseId = caseId;
|
|
2330
|
+
}
|
|
2331
|
+
};
|
|
2332
|
+
function parseDreamingEfficiencyEvalCaseRequest(payload) {
|
|
2333
|
+
const issues = [];
|
|
2334
|
+
const input = parseObject(payload, "$", issues);
|
|
2335
|
+
if (input === void 0) {
|
|
2336
|
+
throw new DreamingEfficiencyEvalRequestValidationError(issues);
|
|
2337
|
+
}
|
|
2338
|
+
pushUnexpectedFields(input, ROOT_REQUEST_KEYS2, "$", issues);
|
|
2339
|
+
const caseId = parseRequiredTrimmedString(input.caseId, "caseId", issues);
|
|
2340
|
+
const memoryPool = parseMemoryPool(input.memoryPool, issues);
|
|
2341
|
+
if (memoryPool === void 0) {
|
|
2342
|
+
pushUnexpectedFields({}, /* @__PURE__ */ new Set(["memoryPool"]), "$", issues);
|
|
2343
|
+
}
|
|
2344
|
+
const description = parseOptionalTrimmedString(input.description, "description", issues);
|
|
2345
|
+
const sandbox = parseSandbox(input.sandbox, issues);
|
|
2346
|
+
const dreamRunFixture = parseRequiredDreamRunFixture(input.dreamRunFixture, issues);
|
|
2347
|
+
const options = parseOptions2(input.options, issues);
|
|
2348
|
+
if (issues.length > 0) {
|
|
2349
|
+
throw new DreamingEfficiencyEvalRequestValidationError(issues, caseId);
|
|
2350
|
+
}
|
|
2351
|
+
return {
|
|
2352
|
+
caseId: caseId ?? "",
|
|
2353
|
+
memoryPool: memoryPool ?? [],
|
|
2354
|
+
...description ? { description } : {},
|
|
2355
|
+
...sandbox ? { sandbox } : {},
|
|
2356
|
+
dreamRunFixture,
|
|
2357
|
+
...options ? { options } : {}
|
|
2358
|
+
};
|
|
2359
|
+
}
|
|
2360
|
+
function mapDreamingEfficiencyEvalCaseRequestDto(dto) {
|
|
2361
|
+
return {
|
|
2362
|
+
caseId: dto.caseId,
|
|
2363
|
+
...dto.description ? { description: dto.description } : {},
|
|
2364
|
+
...dto.sandbox ? { sandbox: mapSandboxRequestDto(dto.sandbox) } : {},
|
|
2365
|
+
memoryPool: dto.memoryPool.map((entry) => mapFixtureEntryDto(entry)),
|
|
2366
|
+
dreamRunFixture: mapDreamRunFixtureDto(dto.dreamRunFixture),
|
|
2367
|
+
...dto.options ? { options: mapCaseOptionsDto2(dto.options) } : {}
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2370
|
+
function parseRequiredDreamRunFixture(value, issues) {
|
|
2371
|
+
if (value === void 0) {
|
|
2372
|
+
pushIssue(issues, "dreamRunFixture", "Expected an object.");
|
|
2373
|
+
return void 0;
|
|
2374
|
+
}
|
|
2375
|
+
const fixture = parseObject(value, "dreamRunFixture", issues);
|
|
2376
|
+
if (fixture === void 0) {
|
|
2377
|
+
return void 0;
|
|
2378
|
+
}
|
|
2379
|
+
pushUnexpectedFields(fixture, DREAM_RUN_FIXTURE_KEYS, "dreamRunFixture", issues);
|
|
2380
|
+
const tier = parseDreamTier(fixture.tier, "dreamRunFixture.tier", issues);
|
|
2381
|
+
const summaryJson = parseDreamCompletionSummary(fixture.summaryJson, "dreamRunFixture.summaryJson", issues);
|
|
2382
|
+
if (summaryJson === void 0) {
|
|
2383
|
+
return void 0;
|
|
2384
|
+
}
|
|
2385
|
+
return {
|
|
2386
|
+
tier: tier ?? "standard",
|
|
2387
|
+
summaryJson,
|
|
2388
|
+
estimatedCostUsd: parseOptionalNonNegativeNumber(fixture.estimatedCostUsd, "dreamRunFixture.estimatedCostUsd", issues),
|
|
2389
|
+
completedAt: parseOptionalTrimmedString(fixture.completedAt, "dreamRunFixture.completedAt", issues)
|
|
2390
|
+
};
|
|
2391
|
+
}
|
|
2392
|
+
function parseDreamTier(value, path2, issues) {
|
|
2393
|
+
if (value === void 0) {
|
|
2394
|
+
pushIssue(issues, path2, "Expected a dreaming tier.");
|
|
2395
|
+
return void 0;
|
|
2396
|
+
}
|
|
2397
|
+
if (typeof value !== "string" || !DREAM_TIERS.includes(value)) {
|
|
2398
|
+
issues.push({ path: path2, message: `tier must be one of: ${DREAM_TIERS.join(", ")}.` });
|
|
2399
|
+
return void 0;
|
|
2400
|
+
}
|
|
2401
|
+
return value;
|
|
2402
|
+
}
|
|
2403
|
+
function parseOptionalNonNegativeNumber(value, path2, issues) {
|
|
2404
|
+
if (value === void 0) {
|
|
2405
|
+
return void 0;
|
|
2406
|
+
}
|
|
2407
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
2408
|
+
pushIssue(issues, path2, "Expected a non-negative number.");
|
|
2409
|
+
return void 0;
|
|
2410
|
+
}
|
|
2411
|
+
return value;
|
|
2412
|
+
}
|
|
2413
|
+
function parseOptions2(value, issues) {
|
|
2414
|
+
if (value === void 0) {
|
|
2415
|
+
return void 0;
|
|
2416
|
+
}
|
|
2417
|
+
const options = parseObject(value, "options", issues);
|
|
2418
|
+
if (options === void 0) {
|
|
2419
|
+
return void 0;
|
|
2420
|
+
}
|
|
2421
|
+
pushUnexpectedFields(options, OPTIONS_KEYS2, "options", issues);
|
|
2422
|
+
return {
|
|
2423
|
+
includeTimings: parseOptionalBoolean(options.includeTimings, "options.includeTimings", issues)
|
|
2424
|
+
};
|
|
2425
|
+
}
|
|
2426
|
+
function mapDreamRunFixtureDto(dto) {
|
|
2427
|
+
return {
|
|
2428
|
+
tier: dto.tier,
|
|
2429
|
+
summaryJson: dto.summaryJson,
|
|
2430
|
+
...dto.estimatedCostUsd !== void 0 ? { estimatedCostUsd: dto.estimatedCostUsd } : {},
|
|
2431
|
+
...dto.completedAt ? { completedAt: dto.completedAt } : {}
|
|
2432
|
+
};
|
|
2433
|
+
}
|
|
2434
|
+
function mapCaseOptionsDto2(dto) {
|
|
2435
|
+
return {
|
|
2436
|
+
includeTimings: dto.includeTimings
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
// src/adapters/api/routes/internal-dreaming-efficiency-eval.ts
|
|
2441
|
+
var INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE_PATH = "/internal/evals/dreaming-efficiency/run";
|
|
2442
|
+
var INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE = {
|
|
2443
|
+
method: "POST",
|
|
2444
|
+
path: INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE_PATH
|
|
2445
|
+
};
|
|
2446
|
+
function createInternalDreamingEfficiencyEvalRoute(runner = runDreamingEfficiencyEvalCase) {
|
|
2447
|
+
return {
|
|
2448
|
+
...INTERNAL_DREAMING_EFFICIENCY_EVAL_ROUTE,
|
|
2449
|
+
handler: async (request) => {
|
|
2450
|
+
let validatedRequest;
|
|
2451
|
+
try {
|
|
2452
|
+
validatedRequest = await parseValidatedRequest2(request);
|
|
2453
|
+
const result = await runner(validatedRequest);
|
|
2454
|
+
return jsonResponse2(result, 200);
|
|
2455
|
+
} catch (error) {
|
|
2456
|
+
if (error instanceof DreamingEfficiencyEvalRequestValidationError) {
|
|
2457
|
+
return jsonResponse2(
|
|
2458
|
+
{
|
|
2459
|
+
status: "error",
|
|
2460
|
+
caseId: error.caseId,
|
|
2461
|
+
error: {
|
|
2462
|
+
code: "invalid_request",
|
|
2463
|
+
message: error.message,
|
|
2464
|
+
details: error.issues
|
|
2465
|
+
}
|
|
2466
|
+
},
|
|
2467
|
+
400
|
|
2468
|
+
);
|
|
2469
|
+
}
|
|
2470
|
+
return jsonResponse2(
|
|
2471
|
+
{
|
|
2472
|
+
status: "error",
|
|
2473
|
+
caseId: validatedRequest?.caseId,
|
|
2474
|
+
error: {
|
|
2475
|
+
code: "internal_error",
|
|
2476
|
+
message: "Internal dreaming-efficiency eval adapter error."
|
|
2477
|
+
}
|
|
2478
|
+
},
|
|
2479
|
+
500
|
|
2480
|
+
);
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
};
|
|
2484
|
+
}
|
|
2485
|
+
var parseJsonBody2 = async (request) => {
|
|
2486
|
+
try {
|
|
2487
|
+
return await request.json();
|
|
2488
|
+
} catch {
|
|
2489
|
+
throw new DreamingEfficiencyEvalRequestValidationError([
|
|
2490
|
+
{
|
|
2491
|
+
path: "$",
|
|
2492
|
+
message: "Request body must be valid JSON."
|
|
2493
|
+
}
|
|
2494
|
+
]);
|
|
2495
|
+
}
|
|
2496
|
+
};
|
|
2497
|
+
var parseValidatedRequest2 = async (request) => {
|
|
2498
|
+
const payload = await parseJsonBody2(request);
|
|
2499
|
+
const requestDto = parseDreamingEfficiencyEvalCaseRequest(payload);
|
|
2500
|
+
return mapDreamingEfficiencyEvalCaseRequestDto(requestDto);
|
|
2501
|
+
};
|
|
2502
|
+
var jsonResponse2 = (body, status) => new Response(JSON.stringify(body), {
|
|
2503
|
+
status,
|
|
2504
|
+
headers: {
|
|
2505
|
+
"content-type": "application/json; charset=utf-8"
|
|
2506
|
+
}
|
|
2507
|
+
});
|
|
2508
|
+
|
|
2509
|
+
// src/app/evals/recall/collect-diagnostics.ts
|
|
2510
|
+
function createRecallEvalDiagnosticsCollector(request) {
|
|
2511
|
+
const diagnosticsRequested = wantsRecallEvalDiagnostics(request);
|
|
2512
|
+
const timingsRequested = request.options?.includeTimings === true;
|
|
2513
|
+
const debugArtifactRequested = request.options?.includeDebugArtifact === true;
|
|
2514
|
+
const observationEnabled = diagnosticsRequested || timingsRequested || debugArtifactRequested;
|
|
2515
|
+
const execution = {
|
|
2516
|
+
mode: "isolated-case",
|
|
2517
|
+
provisioning: "exact-fixture-seed",
|
|
2518
|
+
recallPath: request.recallPath ?? "core",
|
|
2519
|
+
memoryPoolCount: request.memoryPool.length,
|
|
2520
|
+
provisionedCount: 0,
|
|
2521
|
+
requestedDiagnostics: request.options?.includeDiagnostics === true,
|
|
2522
|
+
requestedCandidates: request.options?.includeCandidates === true
|
|
2523
|
+
};
|
|
2524
|
+
const stageTimings = {
|
|
2525
|
+
sandboxSetupMs: 0,
|
|
2526
|
+
fixtureProvisionMs: 0,
|
|
2527
|
+
recallMs: 0,
|
|
2528
|
+
queryEmbeddingMs: 0,
|
|
2529
|
+
vectorSearchMs: 0,
|
|
2530
|
+
lexicalSearchMs: 0,
|
|
1492
2531
|
mergeCandidatesMs: 0,
|
|
1493
2532
|
scoreCandidatesMs: 0,
|
|
1494
2533
|
thresholdMs: 0,
|
|
@@ -1567,7 +2606,7 @@ function createRecallEvalDiagnosticsCollector(request) {
|
|
|
1567
2606
|
expansionRequested: summary.neighborhood.expansionRequested,
|
|
1568
2607
|
expansionAvailable: summary.neighborhood.expansionAvailable,
|
|
1569
2608
|
familiesRequested: [...summary.neighborhood.familiesRequested],
|
|
1570
|
-
|
|
2609
|
+
includeHistorical: summary.neighborhood.includeHistorical,
|
|
1571
2610
|
seedIds: [...summary.neighborhood.seedIds],
|
|
1572
2611
|
expansionCandidates: summary.neighborhood.expansionCandidates,
|
|
1573
2612
|
strongSeedIds: [...summary.neighborhood.strongSeedIds],
|
|
@@ -1633,7 +2672,7 @@ function createRecallEvalDiagnosticsCollector(request) {
|
|
|
1633
2672
|
provisionedCount: result.provisionedCount,
|
|
1634
2673
|
providedIdCount: result.providedIdCount,
|
|
1635
2674
|
generatedIdCount: result.generatedIdCount,
|
|
1636
|
-
|
|
2675
|
+
staleCount: result.staleCount,
|
|
1637
2676
|
supersededCount: result.supersededCount,
|
|
1638
2677
|
createdAtDefaultedCount: result.createdAtDefaultedCount,
|
|
1639
2678
|
updatedAtDefaultedCount: result.updatedAtDefaultedCount,
|
|
@@ -1641,7 +2680,6 @@ function createRecallEvalDiagnosticsCollector(request) {
|
|
|
1641
2680
|
id: entry.id,
|
|
1642
2681
|
created_at: entry.created_at,
|
|
1643
2682
|
updated_at: entry.updated_at,
|
|
1644
|
-
retired: entry.retired,
|
|
1645
2683
|
superseded_by: entry.superseded_by,
|
|
1646
2684
|
claim_key: entry.claim_key,
|
|
1647
2685
|
claim_key_status: entry.claim_key_status,
|
|
@@ -1734,13 +2772,13 @@ function createInstrumentedRecallPorts(ports, observer) {
|
|
|
1734
2772
|
try {
|
|
1735
2773
|
const embedding = await ports.embed(text);
|
|
1736
2774
|
observer.recordQueryEmbedding({
|
|
1737
|
-
durationMs:
|
|
2775
|
+
durationMs: elapsedMs3(startedAt),
|
|
1738
2776
|
dimensions: embedding.length
|
|
1739
2777
|
});
|
|
1740
2778
|
return embedding;
|
|
1741
2779
|
} catch (error) {
|
|
1742
2780
|
observer.recordQueryEmbedding({
|
|
1743
|
-
durationMs:
|
|
2781
|
+
durationMs: elapsedMs3(startedAt),
|
|
1744
2782
|
dimensions: 0
|
|
1745
2783
|
});
|
|
1746
2784
|
throw error;
|
|
@@ -1751,14 +2789,14 @@ function createInstrumentedRecallPorts(ports, observer) {
|
|
|
1751
2789
|
try {
|
|
1752
2790
|
const results = await ports.vectorSearch(params);
|
|
1753
2791
|
observer.recordVectorSearch({
|
|
1754
|
-
durationMs:
|
|
2792
|
+
durationMs: elapsedMs3(startedAt),
|
|
1755
2793
|
count: results.length,
|
|
1756
2794
|
limit: params.limit
|
|
1757
2795
|
});
|
|
1758
2796
|
return results;
|
|
1759
2797
|
} catch (error) {
|
|
1760
2798
|
observer.recordVectorSearch({
|
|
1761
|
-
durationMs:
|
|
2799
|
+
durationMs: elapsedMs3(startedAt),
|
|
1762
2800
|
count: 0,
|
|
1763
2801
|
limit: params.limit
|
|
1764
2802
|
});
|
|
@@ -1770,14 +2808,14 @@ function createInstrumentedRecallPorts(ports, observer) {
|
|
|
1770
2808
|
try {
|
|
1771
2809
|
const results = await ports.ftsSearch(params);
|
|
1772
2810
|
observer.recordLexicalSearch({
|
|
1773
|
-
durationMs:
|
|
2811
|
+
durationMs: elapsedMs3(startedAt),
|
|
1774
2812
|
count: results.length,
|
|
1775
2813
|
limit: params.limit
|
|
1776
2814
|
});
|
|
1777
2815
|
return results;
|
|
1778
2816
|
} catch (error) {
|
|
1779
2817
|
observer.recordLexicalSearch({
|
|
1780
|
-
durationMs:
|
|
2818
|
+
durationMs: elapsedMs3(startedAt),
|
|
1781
2819
|
count: 0,
|
|
1782
2820
|
limit: params.limit
|
|
1783
2821
|
});
|
|
@@ -1802,13 +2840,13 @@ function createInstrumentedRecallPorts(ports, observer) {
|
|
|
1802
2840
|
try {
|
|
1803
2841
|
const entries = await ports.hydrateEntries(ids);
|
|
1804
2842
|
observer.recordHydrateEntries({
|
|
1805
|
-
durationMs:
|
|
2843
|
+
durationMs: elapsedMs3(startedAt),
|
|
1806
2844
|
count: entries.length
|
|
1807
2845
|
});
|
|
1808
2846
|
return entries;
|
|
1809
2847
|
} catch (error) {
|
|
1810
2848
|
observer.recordHydrateEntries({
|
|
1811
|
-
durationMs:
|
|
2849
|
+
durationMs: elapsedMs3(startedAt),
|
|
1812
2850
|
count: 0
|
|
1813
2851
|
});
|
|
1814
2852
|
throw error;
|
|
@@ -1820,14 +2858,14 @@ function createInstrumentedRecallPorts(ports, observer) {
|
|
|
1820
2858
|
await ports.recordRecallEvents(params);
|
|
1821
2859
|
} finally {
|
|
1822
2860
|
observer.recordRecallTelemetry({
|
|
1823
|
-
durationMs:
|
|
2861
|
+
durationMs: elapsedMs3(startedAt),
|
|
1824
2862
|
entryCount: params.entryIds.length
|
|
1825
2863
|
});
|
|
1826
2864
|
}
|
|
1827
2865
|
}
|
|
1828
2866
|
};
|
|
1829
2867
|
}
|
|
1830
|
-
function
|
|
2868
|
+
function elapsedMs3(startedAt) {
|
|
1831
2869
|
return Math.max(0, Date.now() - startedAt);
|
|
1832
2870
|
}
|
|
1833
2871
|
|
|
@@ -1950,7 +2988,7 @@ function buildRecallEvalSuccessResponse(params) {
|
|
|
1950
2988
|
metadata,
|
|
1951
2989
|
diagnostics: params.diagnostics,
|
|
1952
2990
|
timings: params.timings,
|
|
1953
|
-
sandbox:
|
|
2991
|
+
sandbox: buildSandboxResult3(params.sandbox),
|
|
1954
2992
|
...debugArtifact ? { debugArtifact } : {}
|
|
1955
2993
|
};
|
|
1956
2994
|
}
|
|
@@ -1965,10 +3003,10 @@ function buildRecallEvalErrorResponse(params) {
|
|
|
1965
3003
|
},
|
|
1966
3004
|
diagnostics: params.diagnostics,
|
|
1967
3005
|
timings: params.timings,
|
|
1968
|
-
sandbox: params.sandbox ?
|
|
3006
|
+
sandbox: params.sandbox ? buildSandboxResult3(params.sandbox) : void 0
|
|
1969
3007
|
};
|
|
1970
3008
|
}
|
|
1971
|
-
function
|
|
3009
|
+
function buildSandboxResult3(sandbox) {
|
|
1972
3010
|
return {
|
|
1973
3011
|
root: sandbox.root,
|
|
1974
3012
|
dbPath: sandbox.dbPath,
|
|
@@ -2038,118 +3076,89 @@ function buildClaimFamilyMetadata(family) {
|
|
|
2038
3076
|
primaryEntryId: family.primary.entryId,
|
|
2039
3077
|
entries: family.entries.map((entry) => ({
|
|
2040
3078
|
id: entry.entryId,
|
|
2041
|
-
memoryState: entry.memoryState,
|
|
2042
|
-
claimStatus: entry.claimStatus
|
|
2043
|
-
}))
|
|
2044
|
-
};
|
|
2045
|
-
}
|
|
2046
|
-
|
|
2047
|
-
// src/app/evals/recall/run-recall-eval-case.ts
|
|
2048
|
-
async function runRecallEvalCase(request, dependencies = {}) {
|
|
2049
|
-
const startedAt = Date.now();
|
|
2050
|
-
const provisionedAt = new Date(startedAt).toISOString();
|
|
2051
|
-
const diagnostics = createRecallEvalDiagnosticsCollector(request);
|
|
2052
|
-
const recallPath = request.recallPath ?? "core";
|
|
2053
|
-
let sandbox;
|
|
2054
|
-
let sharedEmbeddingPort;
|
|
2055
|
-
let sharedEmbeddingError;
|
|
2056
|
-
const getEmbeddingSupport = () => {
|
|
2057
|
-
if (sharedEmbeddingPort) {
|
|
2058
|
-
return {
|
|
2059
|
-
available: true,
|
|
2060
|
-
port: sharedEmbeddingPort
|
|
2061
|
-
};
|
|
2062
|
-
}
|
|
2063
|
-
if (sharedEmbeddingError) {
|
|
2064
|
-
return {
|
|
2065
|
-
available: false,
|
|
2066
|
-
error: sharedEmbeddingError
|
|
2067
|
-
};
|
|
2068
|
-
}
|
|
2069
|
-
const config = readConfig();
|
|
2070
|
-
try {
|
|
2071
|
-
sharedEmbeddingPort = createEmbeddingClient(resolveEmbeddingApiKey(config), resolveEmbeddingModel(config));
|
|
2072
|
-
return {
|
|
2073
|
-
available: true,
|
|
2074
|
-
port: sharedEmbeddingPort
|
|
2075
|
-
};
|
|
2076
|
-
} catch (error) {
|
|
2077
|
-
sharedEmbeddingError = error instanceof Error ? error.message : String(error);
|
|
2078
|
-
return {
|
|
2079
|
-
available: false,
|
|
2080
|
-
error: sharedEmbeddingError
|
|
2081
|
-
};
|
|
2082
|
-
}
|
|
2083
|
-
};
|
|
2084
|
-
const getEmbeddingPort = () => {
|
|
2085
|
-
const support = getEmbeddingSupport();
|
|
2086
|
-
if (!support.port) {
|
|
2087
|
-
throw new Error(support.error ?? "Embeddings are unavailable.");
|
|
2088
|
-
}
|
|
2089
|
-
return support.port;
|
|
3079
|
+
memoryState: entry.memoryState,
|
|
3080
|
+
claimStatus: entry.claimStatus
|
|
3081
|
+
}))
|
|
2090
3082
|
};
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
// src/app/evals/recall/run-recall-eval-case.ts
|
|
3086
|
+
async function runRecallEvalCase(request, dependencies = {}) {
|
|
3087
|
+
const startedAt = Date.now();
|
|
3088
|
+
const provisionedAt = new Date(startedAt).toISOString();
|
|
3089
|
+
const diagnostics = createRecallEvalDiagnosticsCollector(request);
|
|
3090
|
+
const recallPath = request.recallPath ?? "core";
|
|
3091
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
3092
|
+
const evalNow = parseEvalNow(ablation.now);
|
|
3093
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
3094
|
+
let sandbox;
|
|
2091
3095
|
try {
|
|
2092
3096
|
const sandboxStartedAt = Date.now();
|
|
2093
3097
|
try {
|
|
2094
3098
|
sandbox = await setupRecallEvalSandbox(request.sandbox);
|
|
2095
|
-
diagnostics.recordSandboxSetup(
|
|
3099
|
+
diagnostics.recordSandboxSetup(elapsedMs4(sandboxStartedAt));
|
|
2096
3100
|
} catch (error) {
|
|
2097
|
-
diagnostics.recordSandboxSetup(
|
|
3101
|
+
diagnostics.recordSandboxSetup(elapsedMs4(sandboxStartedAt));
|
|
2098
3102
|
return buildRecallEvalErrorResponse({
|
|
2099
3103
|
request,
|
|
2100
3104
|
code: "sandbox_setup_failed",
|
|
2101
3105
|
message: "Failed to create isolated recall eval sandbox.",
|
|
2102
|
-
details:
|
|
3106
|
+
details: toErrorDetails3(error),
|
|
2103
3107
|
diagnostics: diagnostics.buildDiagnostics(),
|
|
2104
|
-
timings: diagnostics.buildTimings(
|
|
3108
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt))
|
|
2105
3109
|
});
|
|
2106
3110
|
}
|
|
2107
|
-
if (request.memoryPool.length > 0 || (request.procedurePool?.length ?? 0) > 0) {
|
|
3111
|
+
if (request.memoryPool.length > 0 || (request.procedurePool?.length ?? 0) > 0 || shouldProvisionProfileSnapshot(ablation)) {
|
|
2108
3112
|
const provisionStartedAt = Date.now();
|
|
2109
3113
|
try {
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
await provisionRecallEvalProcedureFixtures({
|
|
2122
|
-
caseId: request.caseId,
|
|
2123
|
-
procedurePool: request.procedurePool ?? [],
|
|
2124
|
-
store: sandbox.fixtureStore,
|
|
2125
|
-
provisionedAt
|
|
2126
|
-
});
|
|
2127
|
-
}
|
|
2128
|
-
if (entryProvisionResult) {
|
|
2129
|
-
diagnostics.recordProvision(entryProvisionResult, elapsedMs3(provisionStartedAt));
|
|
3114
|
+
const provisionResult = await provisionEvalSandbox({
|
|
3115
|
+
caseId: request.caseId,
|
|
3116
|
+
sandbox,
|
|
3117
|
+
memoryPool: request.memoryPool,
|
|
3118
|
+
procedurePool: request.procedurePool,
|
|
3119
|
+
profileSnapshot: shouldProvisionProfileSnapshot(ablation) ? ablation.profileSnapshot : void 0,
|
|
3120
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.requirePort() : void 0,
|
|
3121
|
+
provisionedAt
|
|
3122
|
+
});
|
|
3123
|
+
if (provisionResult.entryProvisionResult) {
|
|
3124
|
+
diagnostics.recordProvision(provisionResult.entryProvisionResult, elapsedMs4(provisionStartedAt));
|
|
2130
3125
|
} else {
|
|
2131
|
-
diagnostics.recordFixtureProvisionTiming(
|
|
3126
|
+
diagnostics.recordFixtureProvisionTiming(elapsedMs4(provisionStartedAt));
|
|
2132
3127
|
}
|
|
2133
3128
|
} catch (error) {
|
|
2134
|
-
diagnostics.recordFixtureProvisionTiming(
|
|
3129
|
+
diagnostics.recordFixtureProvisionTiming(elapsedMs4(provisionStartedAt));
|
|
2135
3130
|
return buildRecallEvalErrorResponse({
|
|
2136
3131
|
request,
|
|
2137
3132
|
code: "fixture_provision_failed",
|
|
2138
3133
|
message: "Failed to provision recall eval fixtures into isolated storage.",
|
|
2139
|
-
details:
|
|
3134
|
+
details: toErrorDetails3(error),
|
|
2140
3135
|
diagnostics: diagnostics.buildDiagnostics(),
|
|
2141
|
-
timings: diagnostics.buildTimings(
|
|
3136
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
2142
3137
|
sandbox
|
|
2143
3138
|
});
|
|
2144
3139
|
}
|
|
2145
3140
|
}
|
|
2146
3141
|
const recallStartedAt = Date.now();
|
|
2147
3142
|
try {
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
3143
|
+
if (isMemoryOffArm(ablation)) {
|
|
3144
|
+
diagnostics.recordRecall(elapsedMs4(recallStartedAt));
|
|
3145
|
+
return buildRecallEvalSuccessResponse({
|
|
3146
|
+
request,
|
|
3147
|
+
results: [],
|
|
3148
|
+
diagnostics: diagnostics.buildDiagnostics(),
|
|
3149
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
3150
|
+
sandbox
|
|
3151
|
+
});
|
|
3152
|
+
}
|
|
3153
|
+
const activeSandbox = sandbox;
|
|
3154
|
+
if (!activeSandbox) {
|
|
3155
|
+
throw new Error("Recall eval sandbox was not initialized.");
|
|
3156
|
+
}
|
|
3157
|
+
const embeddingSupport = embeddingResolver.getSupport();
|
|
3158
|
+
const recallEmbeddingPort = request.options?.faultInjection?.queryEmbeddingFailure === true ? createUnavailableEmbeddingPort("Injected recall eval query embedding failure.") : embeddingSupport.port ?? createUnavailableEmbeddingPort(embeddingSupport.error ?? "Embeddings are unavailable.");
|
|
3159
|
+
const sandboxPorts = activeSandbox.createRecallPorts(recallEmbeddingPort);
|
|
2151
3160
|
const portsWithCrossEncoder = attachCrossEncoderPort(sandboxPorts, dependencies.crossEncoder);
|
|
2152
|
-
const telemetryGatedPorts = applyTelemetryWriteGate(portsWithCrossEncoder,
|
|
3161
|
+
const telemetryGatedPorts = applyTelemetryWriteGate(portsWithCrossEncoder, activeSandbox);
|
|
2153
3162
|
const basePorts = applyRecallEvalFaultInjection(telemetryGatedPorts, request);
|
|
2154
3163
|
const recallPorts = diagnostics.isObservationEnabled() ? createInstrumentedRecallPorts(basePorts, diagnostics) : basePorts;
|
|
2155
3164
|
const slotPolicyConfig = request.unified?.memoryPolicy?.slotPolicies;
|
|
@@ -2175,12 +3184,13 @@ async function runRecallEvalCase(request, dependencies = {}) {
|
|
|
2175
3184
|
...request.unified?.sessionKey ? { sessionKey: request.unified.sessionKey } : {}
|
|
2176
3185
|
},
|
|
2177
3186
|
{
|
|
2178
|
-
database:
|
|
2179
|
-
procedures:
|
|
3187
|
+
database: activeSandbox.episodeDatabase,
|
|
3188
|
+
procedures: activeSandbox.procedureDatabase,
|
|
2180
3189
|
recall: recallPorts,
|
|
2181
3190
|
embeddingAvailable: embeddingSupport.available,
|
|
2182
3191
|
...embeddingSupport.error ? { embeddingError: embeddingSupport.error } : {},
|
|
2183
3192
|
...slotPolicyConfig ? { claimSlotPolicyConfig: slotPolicyConfig } : {},
|
|
3193
|
+
...evalNow ? { now: evalNow } : {},
|
|
2184
3194
|
...embeddingSupport.available ? {
|
|
2185
3195
|
embedQuery: async (text) => {
|
|
2186
3196
|
const vectors = await recallEmbeddingPort.embed([text]);
|
|
@@ -2189,25 +3199,28 @@ async function runRecallEvalCase(request, dependencies = {}) {
|
|
|
2189
3199
|
} : {},
|
|
2190
3200
|
...Object.keys(unifiedRecallOptions).length > 0 ? { recallOptions: unifiedRecallOptions } : {}
|
|
2191
3201
|
}
|
|
2192
|
-
) : await recall(request.recallRequest, recallPorts,
|
|
2193
|
-
|
|
3202
|
+
) : await recall(request.recallRequest, recallPorts, {
|
|
3203
|
+
...Object.keys(coreRecallOptions).length > 0 ? coreRecallOptions : {},
|
|
3204
|
+
...evalNow ? { now: evalNow } : {}
|
|
3205
|
+
});
|
|
3206
|
+
diagnostics.recordRecall(elapsedMs4(recallStartedAt));
|
|
2194
3207
|
return buildRecallEvalSuccessResponse({
|
|
2195
3208
|
request,
|
|
2196
3209
|
results,
|
|
2197
3210
|
diagnostics: diagnostics.buildDiagnostics(),
|
|
2198
|
-
timings: diagnostics.buildTimings(
|
|
2199
|
-
sandbox,
|
|
3211
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
3212
|
+
sandbox: activeSandbox,
|
|
2200
3213
|
observedArtifactFacts: request.options?.includeDebugArtifact === true ? diagnostics.buildObservedArtifactFacts() : void 0
|
|
2201
3214
|
});
|
|
2202
3215
|
} catch (error) {
|
|
2203
|
-
diagnostics.recordRecall(
|
|
3216
|
+
diagnostics.recordRecall(elapsedMs4(recallStartedAt));
|
|
2204
3217
|
return buildRecallEvalErrorResponse({
|
|
2205
3218
|
request,
|
|
2206
3219
|
code: "recall_execution_failed",
|
|
2207
3220
|
message: "Failed to execute real recall against isolated eval state.",
|
|
2208
|
-
details:
|
|
3221
|
+
details: toErrorDetails3(error),
|
|
2209
3222
|
diagnostics: diagnostics.buildDiagnostics(),
|
|
2210
|
-
timings: diagnostics.buildTimings(
|
|
3223
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
2211
3224
|
sandbox
|
|
2212
3225
|
});
|
|
2213
3226
|
}
|
|
@@ -2216,16 +3229,16 @@ async function runRecallEvalCase(request, dependencies = {}) {
|
|
|
2216
3229
|
request,
|
|
2217
3230
|
code: "internal_error",
|
|
2218
3231
|
message: "Recall eval execution failed unexpectedly.",
|
|
2219
|
-
details:
|
|
3232
|
+
details: toErrorDetails3(error),
|
|
2220
3233
|
diagnostics: diagnostics.buildDiagnostics(),
|
|
2221
|
-
timings: diagnostics.buildTimings(
|
|
3234
|
+
timings: diagnostics.buildTimings(elapsedMs4(startedAt)),
|
|
2222
3235
|
sandbox
|
|
2223
3236
|
});
|
|
2224
3237
|
} finally {
|
|
2225
3238
|
await sandbox?.cleanup().catch(() => void 0);
|
|
2226
3239
|
}
|
|
2227
3240
|
}
|
|
2228
|
-
function
|
|
3241
|
+
function toErrorDetails3(error) {
|
|
2229
3242
|
if (error instanceof Error) {
|
|
2230
3243
|
return {
|
|
2231
3244
|
cause: error.message
|
|
@@ -2235,13 +3248,6 @@ function toErrorDetails2(error) {
|
|
|
2235
3248
|
cause: String(error)
|
|
2236
3249
|
};
|
|
2237
3250
|
}
|
|
2238
|
-
function createUnavailableEmbeddingPort2(message) {
|
|
2239
|
-
return {
|
|
2240
|
-
async embed() {
|
|
2241
|
-
throw new Error(message);
|
|
2242
|
-
}
|
|
2243
|
-
};
|
|
2244
|
-
}
|
|
2245
3251
|
function applyRecallEvalFaultInjection(ports, request) {
|
|
2246
3252
|
if (request.options?.faultInjection?.vectorSearchFailure !== true) {
|
|
2247
3253
|
return ports;
|
|
@@ -2277,12 +3283,12 @@ function applyRecallEvalFaultInjection(ports, request) {
|
|
|
2277
3283
|
}
|
|
2278
3284
|
};
|
|
2279
3285
|
}
|
|
2280
|
-
function
|
|
3286
|
+
function elapsedMs4(startedAt) {
|
|
2281
3287
|
return Math.max(0, Date.now() - startedAt);
|
|
2282
3288
|
}
|
|
2283
3289
|
|
|
2284
3290
|
// src/adapters/api/validation/recall-eval-request.ts
|
|
2285
|
-
var
|
|
3291
|
+
var ROOT_REQUEST_KEYS3 = /* @__PURE__ */ new Set(["caseId", "description", "recallPath", "sandbox", "memoryPool", "recallRequest", "unified", "options"]);
|
|
2286
3292
|
var RECALL_REQUEST_KEYS = /* @__PURE__ */ new Set([
|
|
2287
3293
|
"text",
|
|
2288
3294
|
"limit",
|
|
@@ -2313,12 +3319,12 @@ var RANKING_POLICY_KEYS = /* @__PURE__ */ new Set([
|
|
|
2313
3319
|
var UNIFIED_REQUEST_KEYS = /* @__PURE__ */ new Set(["mode", "sessionKey", "memoryPolicy"]);
|
|
2314
3320
|
var UNIFIED_MEMORY_POLICY_KEYS = /* @__PURE__ */ new Set(["slotPolicies"]);
|
|
2315
3321
|
var SLOT_POLICY_KEYS = /* @__PURE__ */ new Set(["attributeHeads"]);
|
|
2316
|
-
var
|
|
3322
|
+
var OPTIONS_KEYS3 = /* @__PURE__ */ new Set(["includeDiagnostics", "includeCandidates", "includeTimings", "includeDebugArtifact", "topKCandidates", "faultInjection"]);
|
|
2317
3323
|
var FAULT_INJECTION_KEYS = /* @__PURE__ */ new Set(["queryEmbeddingFailure", "vectorSearchFailure"]);
|
|
2318
3324
|
var RECALL_PATHS = ["core", "unified"];
|
|
2319
3325
|
var RECALL_RANKING_PROFILES = ["historical_state"];
|
|
2320
3326
|
var RANKING_POLICY_TOGGLES = ["enabled", "disabled"];
|
|
2321
|
-
var UNIFIED_RECALL_MODES = ["auto", "
|
|
3327
|
+
var UNIFIED_RECALL_MODES = ["auto", "durables", "episodes"];
|
|
2322
3328
|
var CLAIM_SLOT_POLICIES = ["exclusive", "multivalued"];
|
|
2323
3329
|
var RecallEvalRequestValidationError = class extends Error {
|
|
2324
3330
|
/** Parseable case identifier echoed for invalid request correlation when available. */
|
|
@@ -2352,7 +3358,7 @@ function parseRecallEvalCaseRequest(input) {
|
|
|
2352
3358
|
);
|
|
2353
3359
|
}
|
|
2354
3360
|
const issues = [];
|
|
2355
|
-
pushUnexpectedFields(input,
|
|
3361
|
+
pushUnexpectedFields(input, ROOT_REQUEST_KEYS3, "", issues);
|
|
2356
3362
|
const parsedCaseId = parseRequiredTrimmedString(input.caseId, "caseId", issues);
|
|
2357
3363
|
const description = parseOptionalTrimmedString(input.description, "description", issues);
|
|
2358
3364
|
const recallPath = parseOptionalRecallPath(input.recallPath, "recallPath", issues);
|
|
@@ -2360,7 +3366,7 @@ function parseRecallEvalCaseRequest(input) {
|
|
|
2360
3366
|
const memoryPool = parseMemoryPool(input.memoryPool, issues);
|
|
2361
3367
|
const recallRequest = parseRecallRequest(input.recallRequest, issues);
|
|
2362
3368
|
const unified = parseUnifiedRequest(input.unified, issues);
|
|
2363
|
-
const options =
|
|
3369
|
+
const options = parseOptions3(input.options, issues);
|
|
2364
3370
|
validatePathSpecificRequest(recallPath, recallRequest, unified, issues);
|
|
2365
3371
|
if (issues.length > 0 || parsedCaseId === void 0 || memoryPool === void 0 || recallRequest === void 0) {
|
|
2366
3372
|
throw new RecallEvalRequestValidationError(issues, caseId);
|
|
@@ -2385,7 +3391,7 @@ function mapRecallEvalCaseRequestDto(dto) {
|
|
|
2385
3391
|
memoryPool: dto.memoryPool.map(mapFixtureEntryDto2),
|
|
2386
3392
|
recallRequest: mapRecallRequestDto(dto.recallRequest),
|
|
2387
3393
|
unified: mapUnifiedRequestDto(dto.unified),
|
|
2388
|
-
options:
|
|
3394
|
+
options: mapCaseOptionsDto3(dto.options)
|
|
2389
3395
|
};
|
|
2390
3396
|
}
|
|
2391
3397
|
function parseRecallRequest(value, issues) {
|
|
@@ -2407,7 +3413,7 @@ function parseRecallRequest(value, issues) {
|
|
|
2407
3413
|
budget: parseOptionalIntegerInRange(recallRequest.budget, "recallRequest.budget", issues, {
|
|
2408
3414
|
min: 0
|
|
2409
3415
|
}),
|
|
2410
|
-
types:
|
|
3416
|
+
types: parseOptionalDurableKindArray(recallRequest.types, "recallRequest.types", issues),
|
|
2411
3417
|
tags: parseOptionalStringArray(recallRequest.tags, "recallRequest.tags", issues),
|
|
2412
3418
|
since: parseOptionalTrimmedString(recallRequest.since, "recallRequest.since", issues),
|
|
2413
3419
|
until: parseOptionalTrimmedString(recallRequest.until, "recallRequest.until", issues),
|
|
@@ -2513,7 +3519,7 @@ function parseUnifiedRequest(value, issues) {
|
|
|
2513
3519
|
memoryPolicy: parseUnifiedMemoryPolicy(unified.memoryPolicy, issues)
|
|
2514
3520
|
};
|
|
2515
3521
|
}
|
|
2516
|
-
function
|
|
3522
|
+
function parseOptions3(value, issues) {
|
|
2517
3523
|
if (value === void 0) {
|
|
2518
3524
|
return void 0;
|
|
2519
3525
|
}
|
|
@@ -2521,7 +3527,7 @@ function parseOptions2(value, issues) {
|
|
|
2521
3527
|
if (options === void 0) {
|
|
2522
3528
|
return void 0;
|
|
2523
3529
|
}
|
|
2524
|
-
pushUnexpectedFields(options,
|
|
3530
|
+
pushUnexpectedFields(options, OPTIONS_KEYS3, "options", issues);
|
|
2525
3531
|
return {
|
|
2526
3532
|
includeDiagnostics: parseOptionalBoolean(options.includeDiagnostics, "options.includeDiagnostics", issues),
|
|
2527
3533
|
includeCandidates: parseOptionalBoolean(options.includeCandidates, "options.includeCandidates", issues),
|
|
@@ -2561,9 +3567,9 @@ function parseUnifiedMemoryPolicy(value, issues) {
|
|
|
2561
3567
|
slotPolicies: parseClaimSlotPolicyConfig(memoryPolicy.slotPolicies, "unified.memoryPolicy.slotPolicies", issues)
|
|
2562
3568
|
};
|
|
2563
3569
|
}
|
|
2564
|
-
function
|
|
2565
|
-
if (typeof value !== "string" || !
|
|
2566
|
-
pushIssue(issues, path2, `Expected one of: ${
|
|
3570
|
+
function parseDurableKind2(value, path2, issues) {
|
|
3571
|
+
if (typeof value !== "string" || !DURABLE_KINDS.includes(value)) {
|
|
3572
|
+
pushIssue(issues, path2, `Expected one of: ${DURABLE_KINDS.join(", ")}.`);
|
|
2567
3573
|
return void 0;
|
|
2568
3574
|
}
|
|
2569
3575
|
return value;
|
|
@@ -2598,7 +3604,7 @@ function parseOptionalRankingProfile(value, path2, issues) {
|
|
|
2598
3604
|
}
|
|
2599
3605
|
return value;
|
|
2600
3606
|
}
|
|
2601
|
-
function
|
|
3607
|
+
function parseOptionalDurableKindArray(value, path2, issues) {
|
|
2602
3608
|
if (value === void 0) {
|
|
2603
3609
|
return void 0;
|
|
2604
3610
|
}
|
|
@@ -2608,7 +3614,7 @@ function parseOptionalEntryTypeArray(value, path2, issues) {
|
|
|
2608
3614
|
}
|
|
2609
3615
|
const parsed = [];
|
|
2610
3616
|
for (const [index, item] of value.entries()) {
|
|
2611
|
-
const entryType =
|
|
3617
|
+
const entryType = parseDurableKind2(item, `${path2}[${index}]`, issues);
|
|
2612
3618
|
if (entryType !== void 0) {
|
|
2613
3619
|
parsed.push(entryType);
|
|
2614
3620
|
}
|
|
@@ -2687,7 +3693,10 @@ function mapSandboxRequestDto2(dto) {
|
|
|
2687
3693
|
return {
|
|
2688
3694
|
root: dto.root,
|
|
2689
3695
|
preserve: dto.preserve,
|
|
2690
|
-
corpusSeed: dto.corpusSeed
|
|
3696
|
+
corpusSeed: dto.corpusSeed,
|
|
3697
|
+
ablationArm: dto.ablationArm,
|
|
3698
|
+
now: dto.now,
|
|
3699
|
+
profileSnapshot: dto.profileSnapshot
|
|
2691
3700
|
};
|
|
2692
3701
|
}
|
|
2693
3702
|
function mapFixtureEntryDto2(dto) {
|
|
@@ -2722,7 +3731,7 @@ function mapUnifiedRequestDto(dto) {
|
|
|
2722
3731
|
} : void 0
|
|
2723
3732
|
};
|
|
2724
3733
|
}
|
|
2725
|
-
function
|
|
3734
|
+
function mapCaseOptionsDto3(dto) {
|
|
2726
3735
|
if (dto === void 0) {
|
|
2727
3736
|
return void 0;
|
|
2728
3737
|
}
|
|
@@ -2751,12 +3760,12 @@ function createInternalRecallEvalRoute(optionsOrRunner = {}) {
|
|
|
2751
3760
|
handler: async (request) => {
|
|
2752
3761
|
let validatedRequest;
|
|
2753
3762
|
try {
|
|
2754
|
-
validatedRequest = await
|
|
3763
|
+
validatedRequest = await parseValidatedRequest3(request);
|
|
2755
3764
|
const result = await runner(validatedRequest);
|
|
2756
|
-
return
|
|
3765
|
+
return jsonResponse3(result, 200);
|
|
2757
3766
|
} catch (error) {
|
|
2758
3767
|
if (error instanceof RecallEvalRequestValidationError) {
|
|
2759
|
-
return
|
|
3768
|
+
return jsonResponse3(
|
|
2760
3769
|
{
|
|
2761
3770
|
status: "error",
|
|
2762
3771
|
caseId: error.caseId,
|
|
@@ -2769,7 +3778,7 @@ function createInternalRecallEvalRoute(optionsOrRunner = {}) {
|
|
|
2769
3778
|
400
|
|
2770
3779
|
);
|
|
2771
3780
|
}
|
|
2772
|
-
return
|
|
3781
|
+
return jsonResponse3(
|
|
2773
3782
|
{
|
|
2774
3783
|
status: "error",
|
|
2775
3784
|
caseId: validatedRequest?.caseId,
|
|
@@ -2784,7 +3793,7 @@ function createInternalRecallEvalRoute(optionsOrRunner = {}) {
|
|
|
2784
3793
|
}
|
|
2785
3794
|
};
|
|
2786
3795
|
}
|
|
2787
|
-
var
|
|
3796
|
+
var parseJsonBody3 = async (request) => {
|
|
2788
3797
|
try {
|
|
2789
3798
|
return await request.json();
|
|
2790
3799
|
} catch {
|
|
@@ -2796,12 +3805,386 @@ var parseJsonBody2 = async (request) => {
|
|
|
2796
3805
|
]);
|
|
2797
3806
|
}
|
|
2798
3807
|
};
|
|
2799
|
-
var
|
|
2800
|
-
const payload = await
|
|
3808
|
+
var parseValidatedRequest3 = async (request) => {
|
|
3809
|
+
const payload = await parseJsonBody3(request);
|
|
2801
3810
|
const requestDto = parseRecallEvalCaseRequest(payload);
|
|
2802
3811
|
return mapRecallEvalCaseRequestDto(requestDto);
|
|
2803
3812
|
};
|
|
2804
|
-
var
|
|
3813
|
+
var jsonResponse3 = (body, status) => new Response(JSON.stringify(body), {
|
|
3814
|
+
status,
|
|
3815
|
+
headers: {
|
|
3816
|
+
"content-type": "application/json; charset=utf-8"
|
|
3817
|
+
}
|
|
3818
|
+
});
|
|
3819
|
+
|
|
3820
|
+
// src/app/evals/session-start/normalize-response.ts
|
|
3821
|
+
function buildSessionStartEvalSuccessResponse(params) {
|
|
3822
|
+
return {
|
|
3823
|
+
status: "ok",
|
|
3824
|
+
caseId: params.request.caseId,
|
|
3825
|
+
output: buildOutput2(params.patch),
|
|
3826
|
+
...params.request.options?.includeDiagnostics === true ? { diagnostics: params.patch.diagnostics } : {},
|
|
3827
|
+
...params.timings ? { timings: params.timings } : {},
|
|
3828
|
+
sandbox: buildSandboxResult4(params.sandbox)
|
|
3829
|
+
};
|
|
3830
|
+
}
|
|
3831
|
+
function buildSessionStartEvalErrorResponse(params) {
|
|
3832
|
+
return {
|
|
3833
|
+
status: "error",
|
|
3834
|
+
caseId: params.request.caseId,
|
|
3835
|
+
error: {
|
|
3836
|
+
code: params.code,
|
|
3837
|
+
message: params.message,
|
|
3838
|
+
...params.details !== void 0 ? { details: params.details } : {}
|
|
3839
|
+
},
|
|
3840
|
+
...params.timings ? { timings: params.timings } : {},
|
|
3841
|
+
...params.sandbox ? { sandbox: buildSandboxResult4(params.sandbox) } : {}
|
|
3842
|
+
};
|
|
3843
|
+
}
|
|
3844
|
+
function buildOutput2(patch) {
|
|
3845
|
+
const sourceKindsByEntryId = {};
|
|
3846
|
+
for (const item of patch.durableMemory) {
|
|
3847
|
+
sourceKindsByEntryId[item.entry.id] = item.sourceKind;
|
|
3848
|
+
}
|
|
3849
|
+
return {
|
|
3850
|
+
selectedEntryIds: patch.durableMemory.map((item) => item.entry.id),
|
|
3851
|
+
sourceKindsByEntryId
|
|
3852
|
+
};
|
|
3853
|
+
}
|
|
3854
|
+
function buildSandboxResult4(sandbox) {
|
|
3855
|
+
return {
|
|
3856
|
+
root: sandbox.root,
|
|
3857
|
+
dbPath: sandbox.dbPath,
|
|
3858
|
+
preserved: sandbox.preserved,
|
|
3859
|
+
...sandbox.snapshot ? { snapshot: sandbox.snapshot } : {}
|
|
3860
|
+
};
|
|
3861
|
+
}
|
|
3862
|
+
|
|
3863
|
+
// src/app/evals/session-start/run-session-start-eval-case.ts
|
|
3864
|
+
async function runSessionStartEvalCase(request) {
|
|
3865
|
+
const startedAt = Date.now();
|
|
3866
|
+
const provisionedAt = new Date(startedAt).toISOString();
|
|
3867
|
+
const ablation = resolveAblationConfig(request.sandbox);
|
|
3868
|
+
const evalNow = parseEvalNow(ablation.now);
|
|
3869
|
+
const embeddingResolver = createEvalEmbeddingResolver();
|
|
3870
|
+
let sandbox;
|
|
3871
|
+
let timings;
|
|
3872
|
+
try {
|
|
3873
|
+
const sandboxStartedAt = Date.now();
|
|
3874
|
+
try {
|
|
3875
|
+
sandbox = await setupRecallEvalSandbox(request.sandbox);
|
|
3876
|
+
timings = { sandboxSetupMs: elapsedMs5(sandboxStartedAt) };
|
|
3877
|
+
} catch (error) {
|
|
3878
|
+
return buildSessionStartEvalErrorResponse({
|
|
3879
|
+
request,
|
|
3880
|
+
code: "sandbox_setup_failed",
|
|
3881
|
+
message: "Failed to create isolated session-start eval sandbox.",
|
|
3882
|
+
details: toErrorDetails4(error),
|
|
3883
|
+
timings: request.options?.includeTimings === true ? { totalMs: elapsedMs5(startedAt), sandboxSetupMs: elapsedMs5(sandboxStartedAt) } : void 0
|
|
3884
|
+
});
|
|
3885
|
+
}
|
|
3886
|
+
if (request.memoryPool.length > 0 || shouldProvisionProfileSnapshot(ablation)) {
|
|
3887
|
+
const provisionStartedAt = Date.now();
|
|
3888
|
+
try {
|
|
3889
|
+
await provisionEvalSandbox({
|
|
3890
|
+
caseId: request.caseId,
|
|
3891
|
+
sandbox,
|
|
3892
|
+
memoryPool: request.memoryPool,
|
|
3893
|
+
profileSnapshot: shouldProvisionProfileSnapshot(ablation) ? ablation.profileSnapshot : void 0,
|
|
3894
|
+
embedding: request.memoryPool.length > 0 ? embeddingResolver.portOrUnavailable() : void 0,
|
|
3895
|
+
provisionedAt
|
|
3896
|
+
});
|
|
3897
|
+
timings = { ...timings, fixtureProvisionMs: elapsedMs5(provisionStartedAt) };
|
|
3898
|
+
} catch (error) {
|
|
3899
|
+
return buildSessionStartEvalErrorResponse({
|
|
3900
|
+
request,
|
|
3901
|
+
code: "fixture_provision_failed",
|
|
3902
|
+
message: "Failed to provision session-start eval fixtures into isolated storage.",
|
|
3903
|
+
details: toErrorDetails4(error),
|
|
3904
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs5(startedAt), fixtureProvisionMs: elapsedMs5(provisionStartedAt) } : void 0,
|
|
3905
|
+
sandbox
|
|
3906
|
+
});
|
|
3907
|
+
}
|
|
3908
|
+
}
|
|
3909
|
+
const sessionStartStartedAt = Date.now();
|
|
3910
|
+
try {
|
|
3911
|
+
if (isMemoryOffArm(ablation)) {
|
|
3912
|
+
timings = {
|
|
3913
|
+
...timings,
|
|
3914
|
+
sessionStartMs: elapsedMs5(sessionStartStartedAt),
|
|
3915
|
+
totalMs: elapsedMs5(startedAt)
|
|
3916
|
+
};
|
|
3917
|
+
return buildSessionStartEvalSuccessResponse({
|
|
3918
|
+
request,
|
|
3919
|
+
patch: {
|
|
3920
|
+
durableMemory: [],
|
|
3921
|
+
diagnostics: {
|
|
3922
|
+
coreCandidateCount: 0,
|
|
3923
|
+
profileCandidateCount: 0,
|
|
3924
|
+
proactiveDirectiveCandidateCount: 0,
|
|
3925
|
+
artifactRecallCandidateCount: 0,
|
|
3926
|
+
artifactRecallUsed: false,
|
|
3927
|
+
notices: ["memory-off ablation arm stubbed session-start injection."]
|
|
3928
|
+
}
|
|
3929
|
+
},
|
|
3930
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
3931
|
+
sandbox
|
|
3932
|
+
});
|
|
3933
|
+
}
|
|
3934
|
+
const activeSandbox = sandbox;
|
|
3935
|
+
if (!activeSandbox) {
|
|
3936
|
+
throw new Error("Session-start eval sandbox was not initialized.");
|
|
3937
|
+
}
|
|
3938
|
+
const embeddingSupport = embeddingResolver.getSupport();
|
|
3939
|
+
const recallPorts = activeSandbox.createRecallPorts(
|
|
3940
|
+
embeddingSupport.port ?? createUnavailableEmbeddingPort(embeddingSupport.error ?? "Embeddings are unavailable.")
|
|
3941
|
+
);
|
|
3942
|
+
const patch = await runSessionStart(request.sessionStartInput, {
|
|
3943
|
+
repository: activeSandbox.sessionStartRepository,
|
|
3944
|
+
recall: recallPorts,
|
|
3945
|
+
...evalNow ? { now: evalNow } : {},
|
|
3946
|
+
listActiveAbstainDirectives: () => activeSandbox.listActiveAbstainDirectives(evalNow),
|
|
3947
|
+
listActiveProactiveDirectives: () => activeSandbox.listActiveSessionStartProactiveDirectives(evalNow)
|
|
3948
|
+
});
|
|
3949
|
+
timings = {
|
|
3950
|
+
...timings,
|
|
3951
|
+
sessionStartMs: elapsedMs5(sessionStartStartedAt),
|
|
3952
|
+
totalMs: elapsedMs5(startedAt)
|
|
3953
|
+
};
|
|
3954
|
+
return buildSessionStartEvalSuccessResponse({
|
|
3955
|
+
request,
|
|
3956
|
+
patch,
|
|
3957
|
+
timings: request.options?.includeTimings === true ? timings : void 0,
|
|
3958
|
+
sandbox: activeSandbox
|
|
3959
|
+
});
|
|
3960
|
+
} catch (error) {
|
|
3961
|
+
return buildSessionStartEvalErrorResponse({
|
|
3962
|
+
request,
|
|
3963
|
+
code: "session_start_execution_failed",
|
|
3964
|
+
message: "Failed to execute real session-start selection against isolated eval state.",
|
|
3965
|
+
details: toErrorDetails4(error),
|
|
3966
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs5(startedAt), sessionStartMs: elapsedMs5(sessionStartStartedAt) } : void 0,
|
|
3967
|
+
sandbox
|
|
3968
|
+
});
|
|
3969
|
+
}
|
|
3970
|
+
} catch (error) {
|
|
3971
|
+
return buildSessionStartEvalErrorResponse({
|
|
3972
|
+
request,
|
|
3973
|
+
code: "internal_error",
|
|
3974
|
+
message: "Session-start eval execution failed unexpectedly.",
|
|
3975
|
+
details: toErrorDetails4(error),
|
|
3976
|
+
timings: request.options?.includeTimings === true ? { ...timings, totalMs: elapsedMs5(startedAt) } : void 0,
|
|
3977
|
+
sandbox
|
|
3978
|
+
});
|
|
3979
|
+
} finally {
|
|
3980
|
+
await sandbox?.cleanup().catch(() => void 0);
|
|
3981
|
+
}
|
|
3982
|
+
}
|
|
3983
|
+
function toErrorDetails4(error) {
|
|
3984
|
+
if (error instanceof Error) {
|
|
3985
|
+
return { cause: error.message };
|
|
3986
|
+
}
|
|
3987
|
+
return { cause: String(error) };
|
|
3988
|
+
}
|
|
3989
|
+
function elapsedMs5(startedAt) {
|
|
3990
|
+
return Date.now() - startedAt;
|
|
3991
|
+
}
|
|
3992
|
+
|
|
3993
|
+
// src/adapters/api/validation/session-start-eval-request.ts
|
|
3994
|
+
var ROOT_REQUEST_KEYS4 = /* @__PURE__ */ new Set(["caseId", "description", "sandbox", "memoryPool", "sessionStartInput", "options"]);
|
|
3995
|
+
var SESSION_START_INPUT_KEYS = /* @__PURE__ */ new Set(["sessionKey", "policy"]);
|
|
3996
|
+
var SESSION_START_POLICY_KEYS = /* @__PURE__ */ new Set([
|
|
3997
|
+
"maxCoreEntries",
|
|
3998
|
+
"enableArtifactRecall",
|
|
3999
|
+
"maxArtifactRecallEntries",
|
|
4000
|
+
"maxDurableEntries",
|
|
4001
|
+
"maxArtifactChars",
|
|
4002
|
+
"recallThreshold",
|
|
4003
|
+
"maxProfileSnapshotAgeHours"
|
|
4004
|
+
]);
|
|
4005
|
+
var OPTIONS_KEYS4 = /* @__PURE__ */ new Set(["includeDiagnostics", "includeTimings"]);
|
|
4006
|
+
var SessionStartEvalRequestValidationError = class extends Error {
|
|
4007
|
+
caseId;
|
|
4008
|
+
issues;
|
|
4009
|
+
/** Creates a validation error with optional safely parsed case id. */
|
|
4010
|
+
constructor(issues, caseId) {
|
|
4011
|
+
super("Invalid session-start eval request.");
|
|
4012
|
+
this.name = "SessionStartEvalRequestValidationError";
|
|
4013
|
+
this.issues = issues;
|
|
4014
|
+
this.caseId = caseId;
|
|
4015
|
+
}
|
|
4016
|
+
};
|
|
4017
|
+
function parseSessionStartEvalCaseRequest(payload) {
|
|
4018
|
+
const issues = [];
|
|
4019
|
+
const input = parseObject(payload, "$", issues);
|
|
4020
|
+
if (input === void 0) {
|
|
4021
|
+
throw new SessionStartEvalRequestValidationError(issues);
|
|
4022
|
+
}
|
|
4023
|
+
pushUnexpectedFields(input, ROOT_REQUEST_KEYS4, "$", issues);
|
|
4024
|
+
const caseId = parseRequiredTrimmedString(input.caseId, "caseId", issues);
|
|
4025
|
+
const memoryPool = parseMemoryPool(input.memoryPool, issues);
|
|
4026
|
+
const sessionStartInput = parseSessionStartInput(input.sessionStartInput, issues);
|
|
4027
|
+
if (caseId === void 0 || memoryPool === void 0 || sessionStartInput === void 0) {
|
|
4028
|
+
throw new SessionStartEvalRequestValidationError(issues, extractParseableCaseId(input));
|
|
4029
|
+
}
|
|
4030
|
+
const dto = {
|
|
4031
|
+
caseId,
|
|
4032
|
+
memoryPool,
|
|
4033
|
+
sessionStartInput,
|
|
4034
|
+
description: parseOptionalTrimmedString(input.description, "description", issues),
|
|
4035
|
+
sandbox: parseSandbox(input.sandbox, issues),
|
|
4036
|
+
options: parseOptions4(input.options, issues)
|
|
4037
|
+
};
|
|
4038
|
+
if (issues.length > 0) {
|
|
4039
|
+
throw new SessionStartEvalRequestValidationError(issues, caseId);
|
|
4040
|
+
}
|
|
4041
|
+
return dto;
|
|
4042
|
+
}
|
|
4043
|
+
function mapSessionStartEvalCaseRequestDto(dto) {
|
|
4044
|
+
return {
|
|
4045
|
+
caseId: dto.caseId,
|
|
4046
|
+
...dto.description ? { description: dto.description } : {},
|
|
4047
|
+
sandbox: mapSandboxRequestDto(dto.sandbox),
|
|
4048
|
+
memoryPool: dto.memoryPool.map((entry) => mapFixtureEntryDto(entry)),
|
|
4049
|
+
sessionStartInput: mapSessionStartInputDto(dto.sessionStartInput),
|
|
4050
|
+
...dto.options ? { options: mapCaseOptionsDto4(dto.options) } : {}
|
|
4051
|
+
};
|
|
4052
|
+
}
|
|
4053
|
+
function parseSessionStartInput(value, issues) {
|
|
4054
|
+
const sessionStartInput = parseObject(value, "sessionStartInput", issues);
|
|
4055
|
+
if (sessionStartInput === void 0) {
|
|
4056
|
+
return void 0;
|
|
4057
|
+
}
|
|
4058
|
+
pushUnexpectedFields(sessionStartInput, SESSION_START_INPUT_KEYS, "sessionStartInput", issues);
|
|
4059
|
+
return {
|
|
4060
|
+
sessionKey: parseOptionalTrimmedString(sessionStartInput.sessionKey, "sessionStartInput.sessionKey", issues),
|
|
4061
|
+
policy: parseSessionStartPolicy(sessionStartInput.policy, issues)
|
|
4062
|
+
};
|
|
4063
|
+
}
|
|
4064
|
+
function parseSessionStartPolicy(value, issues) {
|
|
4065
|
+
if (value === void 0) {
|
|
4066
|
+
return void 0;
|
|
4067
|
+
}
|
|
4068
|
+
const policy = parseObject(value, "sessionStartInput.policy", issues);
|
|
4069
|
+
if (policy === void 0) {
|
|
4070
|
+
return void 0;
|
|
4071
|
+
}
|
|
4072
|
+
pushUnexpectedFields(policy, SESSION_START_POLICY_KEYS, "sessionStartInput.policy", issues);
|
|
4073
|
+
return {
|
|
4074
|
+
maxCoreEntries: parseOptionalIntegerInRange(policy.maxCoreEntries, "sessionStartInput.policy.maxCoreEntries", issues, { min: 0 }),
|
|
4075
|
+
enableArtifactRecall: parseOptionalBoolean(policy.enableArtifactRecall, "sessionStartInput.policy.enableArtifactRecall", issues),
|
|
4076
|
+
maxArtifactRecallEntries: parseOptionalIntegerInRange(policy.maxArtifactRecallEntries, "sessionStartInput.policy.maxArtifactRecallEntries", issues, {
|
|
4077
|
+
min: 0
|
|
4078
|
+
}),
|
|
4079
|
+
maxDurableEntries: parseOptionalIntegerInRange(policy.maxDurableEntries, "sessionStartInput.policy.maxDurableEntries", issues, { min: 0 }),
|
|
4080
|
+
maxArtifactChars: parseOptionalIntegerInRange(policy.maxArtifactChars, "sessionStartInput.policy.maxArtifactChars", issues, { min: 0 }),
|
|
4081
|
+
recallThreshold: parseOptionalThreshold(policy.recallThreshold, "sessionStartInput.policy.recallThreshold", issues),
|
|
4082
|
+
maxProfileSnapshotAgeHours: parseOptionalIntegerInRange(policy.maxProfileSnapshotAgeHours, "sessionStartInput.policy.maxProfileSnapshotAgeHours", issues, {
|
|
4083
|
+
min: 0
|
|
4084
|
+
})
|
|
4085
|
+
};
|
|
4086
|
+
}
|
|
4087
|
+
function parseOptions4(value, issues) {
|
|
4088
|
+
if (value === void 0) {
|
|
4089
|
+
return void 0;
|
|
4090
|
+
}
|
|
4091
|
+
const options = parseObject(value, "options", issues);
|
|
4092
|
+
if (options === void 0) {
|
|
4093
|
+
return void 0;
|
|
4094
|
+
}
|
|
4095
|
+
pushUnexpectedFields(options, OPTIONS_KEYS4, "options", issues);
|
|
4096
|
+
return {
|
|
4097
|
+
includeDiagnostics: parseOptionalBoolean(options.includeDiagnostics, "options.includeDiagnostics", issues),
|
|
4098
|
+
includeTimings: parseOptionalBoolean(options.includeTimings, "options.includeTimings", issues)
|
|
4099
|
+
};
|
|
4100
|
+
}
|
|
4101
|
+
function mapSessionStartInputDto(dto) {
|
|
4102
|
+
return {
|
|
4103
|
+
...dto.sessionKey ? { sessionKey: dto.sessionKey } : {},
|
|
4104
|
+
...dto.policy ? { policy: mapSessionStartPolicyDto(dto.policy) } : {}
|
|
4105
|
+
};
|
|
4106
|
+
}
|
|
4107
|
+
function mapSessionStartPolicyDto(dto) {
|
|
4108
|
+
return {
|
|
4109
|
+
maxCoreEntries: dto.maxCoreEntries,
|
|
4110
|
+
enableArtifactRecall: dto.enableArtifactRecall,
|
|
4111
|
+
maxArtifactRecallEntries: dto.maxArtifactRecallEntries,
|
|
4112
|
+
maxDurableEntries: dto.maxDurableEntries,
|
|
4113
|
+
maxArtifactChars: dto.maxArtifactChars,
|
|
4114
|
+
recallThreshold: dto.recallThreshold,
|
|
4115
|
+
maxProfileSnapshotAgeHours: dto.maxProfileSnapshotAgeHours
|
|
4116
|
+
};
|
|
4117
|
+
}
|
|
4118
|
+
function mapCaseOptionsDto4(dto) {
|
|
4119
|
+
return {
|
|
4120
|
+
includeDiagnostics: dto.includeDiagnostics,
|
|
4121
|
+
includeTimings: dto.includeTimings
|
|
4122
|
+
};
|
|
4123
|
+
}
|
|
4124
|
+
|
|
4125
|
+
// src/adapters/api/routes/internal-session-start-eval.ts
|
|
4126
|
+
var INTERNAL_SESSION_START_EVAL_ROUTE_PATH = "/internal/evals/session-start/run";
|
|
4127
|
+
var INTERNAL_SESSION_START_EVAL_ROUTE = {
|
|
4128
|
+
method: "POST",
|
|
4129
|
+
path: INTERNAL_SESSION_START_EVAL_ROUTE_PATH
|
|
4130
|
+
};
|
|
4131
|
+
function createInternalSessionStartEvalRoute(runner = runSessionStartEvalCase) {
|
|
4132
|
+
return {
|
|
4133
|
+
...INTERNAL_SESSION_START_EVAL_ROUTE,
|
|
4134
|
+
handler: async (request) => {
|
|
4135
|
+
let validatedRequest;
|
|
4136
|
+
try {
|
|
4137
|
+
validatedRequest = await parseValidatedRequest4(request);
|
|
4138
|
+
const result = await runner(validatedRequest);
|
|
4139
|
+
return jsonResponse4(result, 200);
|
|
4140
|
+
} catch (error) {
|
|
4141
|
+
if (error instanceof SessionStartEvalRequestValidationError) {
|
|
4142
|
+
return jsonResponse4(
|
|
4143
|
+
{
|
|
4144
|
+
status: "error",
|
|
4145
|
+
caseId: error.caseId,
|
|
4146
|
+
error: {
|
|
4147
|
+
code: "invalid_request",
|
|
4148
|
+
message: error.message,
|
|
4149
|
+
details: error.issues
|
|
4150
|
+
}
|
|
4151
|
+
},
|
|
4152
|
+
400
|
|
4153
|
+
);
|
|
4154
|
+
}
|
|
4155
|
+
return jsonResponse4(
|
|
4156
|
+
{
|
|
4157
|
+
status: "error",
|
|
4158
|
+
caseId: validatedRequest?.caseId,
|
|
4159
|
+
error: {
|
|
4160
|
+
code: "internal_error",
|
|
4161
|
+
message: "Internal session-start eval adapter error."
|
|
4162
|
+
}
|
|
4163
|
+
},
|
|
4164
|
+
500
|
|
4165
|
+
);
|
|
4166
|
+
}
|
|
4167
|
+
}
|
|
4168
|
+
};
|
|
4169
|
+
}
|
|
4170
|
+
var parseJsonBody4 = async (request) => {
|
|
4171
|
+
try {
|
|
4172
|
+
return await request.json();
|
|
4173
|
+
} catch {
|
|
4174
|
+
throw new SessionStartEvalRequestValidationError([
|
|
4175
|
+
{
|
|
4176
|
+
path: "$",
|
|
4177
|
+
message: "Request body must be valid JSON."
|
|
4178
|
+
}
|
|
4179
|
+
]);
|
|
4180
|
+
}
|
|
4181
|
+
};
|
|
4182
|
+
var parseValidatedRequest4 = async (request) => {
|
|
4183
|
+
const payload = await parseJsonBody4(request);
|
|
4184
|
+
const requestDto = parseSessionStartEvalCaseRequest(payload);
|
|
4185
|
+
return mapSessionStartEvalCaseRequestDto(requestDto);
|
|
4186
|
+
};
|
|
4187
|
+
var jsonResponse4 = (body, status) => new Response(JSON.stringify(body), {
|
|
2805
4188
|
status,
|
|
2806
4189
|
headers: {
|
|
2807
4190
|
"content-type": "application/json; charset=utf-8"
|
|
@@ -2810,7 +4193,12 @@ var jsonResponse2 = (body, status) => new Response(JSON.stringify(body), {
|
|
|
2810
4193
|
|
|
2811
4194
|
// src/adapters/api/internal-eval-routes.ts
|
|
2812
4195
|
function createInternalEvalRoutes(options = {}) {
|
|
2813
|
-
return [
|
|
4196
|
+
return [
|
|
4197
|
+
createInternalRecallEvalRoute({ crossEncoder: options.crossEncoder }),
|
|
4198
|
+
createInternalBeforeTurnEvalRoute({ crossEncoder: options.crossEncoder }),
|
|
4199
|
+
createInternalSessionStartEvalRoute(),
|
|
4200
|
+
createInternalDreamingEfficiencyEvalRoute()
|
|
4201
|
+
];
|
|
2814
4202
|
}
|
|
2815
4203
|
|
|
2816
4204
|
// src/adapters/api/internal-eval-server.ts
|