@agenr/skeln-plugin 3.3.0 → 2026.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-before-turn-artifact-NPUHVWFE.js +71 -0
- package/dist/build-recall-artifact-F3LS3PZX.js +62 -0
- package/dist/chunk-5AXMFBHR.js +14 -0
- package/dist/chunk-5AYIXQRF.js +4452 -0
- package/dist/{chunk-Z5X7T4QZ.js → chunk-5TIP2EPP.js} +1519 -2565
- package/dist/{chunk-5LADPJ4C.js → chunk-GAERET5Q.js} +138 -504
- package/dist/chunk-GF3PX3VM.js +41 -0
- package/dist/chunk-GKZQ5AG5.js +44 -0
- package/dist/chunk-IBPS64W3.js +1069 -0
- package/dist/{chunk-ZYADFKX3.js → chunk-MC3C2XM5.js} +34 -1
- package/dist/chunk-NSLTJBUC.js +270 -0
- package/dist/chunk-OJSIZDZD.js +9 -0
- package/dist/chunk-OWGQWQUP.js +45 -0
- package/dist/chunk-SIY3JA7T.js +3062 -0
- package/dist/{chunk-M5M65AYP.js → chunk-SOQW7356.js} +271 -1934
- package/dist/chunk-U74RE3L7.js +3233 -0
- package/dist/chunk-VBPYU7GO.js +597 -0
- package/dist/chunk-VTHBPXDQ.js +1750 -0
- package/dist/{chunk-KH52KJSJ.js → chunk-XFJ4S4G2.js} +844 -39
- package/dist/chunk-Y5NB3FTH.js +106 -0
- package/dist/{chunk-RYMSM3OS.js → chunk-ZX55JBV2.js} +1710 -322
- package/dist/claim-slot-policy-CdrW_1l4.d.ts +13 -0
- package/dist/index.d.ts +630 -51
- package/dist/index.js +852 -4683
- package/dist/lifecycle-checkpoint-IAC5FCQU.js +154 -0
- package/dist/{claim-slot-policy-CQ-h0GaV.d.ts → ports-C4QkwDBS.d.ts} +168 -78
- package/dist/scan-6JKPOQHD.js +6 -0
- package/dist/service-EKFACEN6.js +15 -0
- package/dist/service-RHNB5AEQ.js +861 -0
- package/dist/sink-AUAAWC5O.js +8 -0
- package/package.json +1 -1
- package/dist/cli.d.ts +0 -1
- package/dist/internal-eval-server.d.ts +0 -1
- package/dist/internal-recall-eval-server.d.ts +0 -1
|
@@ -1,30 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
|
-
EMBEDDING_DIMENSIONS,
|
|
3
|
-
ENTRY_SELECT_COLUMNS,
|
|
4
|
-
ENTRY_TYPES,
|
|
5
|
-
EPISODE_ACTIVITY_LEVELS,
|
|
6
|
-
EXPIRY_LEVELS,
|
|
7
|
-
VECTOR_INDEX_NAME,
|
|
8
2
|
applyClaimKeyLifecycle,
|
|
9
|
-
buildActiveEntryClause,
|
|
10
3
|
buildExtractedClaimKeyLifecycle,
|
|
11
|
-
buildInferredIngestClaimKeySupportContext
|
|
12
|
-
|
|
13
|
-
buildPrecomputedClaimKeyLifecycle,
|
|
14
|
-
composeEmbeddingText,
|
|
15
|
-
hasPrecomputedClaimKeyLifecycleFields,
|
|
16
|
-
mapEntryRow,
|
|
17
|
-
parseClaimKeyConfidence,
|
|
18
|
-
parseClaimKeySource,
|
|
19
|
-
parseClaimKeyStatus,
|
|
20
|
-
parseClaimSupportMode,
|
|
21
|
-
readNumber,
|
|
22
|
-
readOptionalString,
|
|
23
|
-
readRequiredString,
|
|
24
|
-
truncate,
|
|
25
|
-
validateTemporalValidityRange
|
|
26
|
-
} from "./chunk-Z5X7T4QZ.js";
|
|
4
|
+
buildInferredIngestClaimKeySupportContext
|
|
5
|
+
} from "./chunk-VTHBPXDQ.js";
|
|
27
6
|
import {
|
|
7
|
+
assertKeyedDurableHasLifecycle,
|
|
28
8
|
compactClaimKey,
|
|
29
9
|
describeClaimKeyNormalizationFailure,
|
|
30
10
|
describeExtractedClaimKeyRejection,
|
|
@@ -32,1211 +12,167 @@ import {
|
|
|
32
12
|
isTrustedClaimKeyForCleanup,
|
|
33
13
|
normalizeClaimKey,
|
|
34
14
|
normalizeClaimKeySegment,
|
|
35
|
-
parseRelativeDate,
|
|
36
|
-
resolveClaimSlotPolicy,
|
|
37
15
|
validateExtractedClaimKey
|
|
38
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-VBPYU7GO.js";
|
|
39
17
|
|
|
40
|
-
// src/
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
const subject = readOptionalStringParam(params, "subject");
|
|
56
|
-
const last = options.allowLast ? readBooleanParam(params, "last") : void 0;
|
|
57
|
-
const selectorCount = (id ? 1 : 0) + (subject ? 1 : 0) + (last === true ? 1 : 0);
|
|
58
|
-
const selectorDescription = options.allowLast ? "id, subject, or last" : "id or subject";
|
|
59
|
-
if (selectorCount !== 1) {
|
|
60
|
-
throw new Error(`Provide exactly one target selector: ${selectorDescription}.`);
|
|
61
|
-
}
|
|
62
|
-
if (last) {
|
|
63
|
-
const entry2 = await ports.findMostRecentEntry();
|
|
64
|
-
if (!entry2) {
|
|
65
|
-
throw new Error("No agenr entries exist yet.");
|
|
66
|
-
}
|
|
67
|
-
return entry2;
|
|
68
|
-
}
|
|
69
|
-
if (id) {
|
|
70
|
-
const entry2 = await ports.getEntryById(id);
|
|
71
|
-
if (!entry2) {
|
|
72
|
-
throw new Error(`No agenr entry found for id ${id}.`);
|
|
73
|
-
}
|
|
74
|
-
return entry2;
|
|
75
|
-
}
|
|
76
|
-
const entry = await ports.findEntryBySubject(subject ?? "");
|
|
77
|
-
if (!entry) {
|
|
78
|
-
throw new Error(`No agenr entry found for subject "${subject}".`);
|
|
79
|
-
}
|
|
80
|
-
return entry;
|
|
81
|
-
}
|
|
82
|
-
function readBooleanParam(params, key) {
|
|
83
|
-
const value = params[key];
|
|
84
|
-
if (value === void 0) {
|
|
85
|
-
return void 0;
|
|
86
|
-
}
|
|
87
|
-
if (typeof value === "boolean") {
|
|
88
|
-
return value;
|
|
89
|
-
}
|
|
90
|
-
throw new Error(`${key} must be a boolean.`);
|
|
91
|
-
}
|
|
92
|
-
function readOptionalStringParam(params, key) {
|
|
93
|
-
const value = params[key];
|
|
94
|
-
if (value === void 0 || value === null) {
|
|
95
|
-
return void 0;
|
|
96
|
-
}
|
|
97
|
-
if (typeof value !== "string") {
|
|
98
|
-
throw new Error(`${key} must be a string.`);
|
|
99
|
-
}
|
|
100
|
-
const normalized = value.trim();
|
|
101
|
-
return normalized.length > 0 ? normalized : void 0;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
// src/adapters/shared/entry-tools.ts
|
|
105
|
-
var ENTRY_TYPE_DESCRIPTION = "Knowledge type to store. Use fact for durable truth about a person, system, place, or how something works. Use decision for a standing rule, constraint, policy, or chosen approach future sessions should follow - not a progress update or completed action. Use preference for what someone likes, wants, values, or wants avoided. Use lesson for a non-obvious takeaway learned from experience that should change future behavior. Use milestone for a rare one-time event with durable future significance - not ordinary execution progress. Use relationship for a meaningful durable connection between people, groups, or systems.";
|
|
106
|
-
var EXPIRY_DESCRIPTION = "Lifetime bucket: core (always injected at session start, use sparingly), permanent (durable and recalled on demand), or temporary (short-horizon).";
|
|
107
|
-
var UPDATE_EXPIRY_DESCRIPTION = `${EXPIRY_DESCRIPTION} Accepted values: ${EXPIRY_LEVELS.join(", ")}.`;
|
|
108
|
-
var RECALL_MODES = ["auto", "entries", "episodes", "procedures"];
|
|
109
|
-
function asRecord(value) {
|
|
110
|
-
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
111
|
-
return value;
|
|
112
|
-
}
|
|
113
|
-
throw new Error("Tool parameters must be an object.");
|
|
114
|
-
}
|
|
115
|
-
function parseExpiry(value) {
|
|
116
|
-
if (value === void 0) {
|
|
117
|
-
return void 0;
|
|
118
|
-
}
|
|
119
|
-
if (EXPIRY_LEVELS.includes(value)) {
|
|
120
|
-
return value;
|
|
121
|
-
}
|
|
122
|
-
throw new Error(`Unsupported expiry "${value}".`);
|
|
123
|
-
}
|
|
124
|
-
function parseEntryTypes(values) {
|
|
125
|
-
return normalizeStringArray(values).map((value) => parseEntryType(value));
|
|
126
|
-
}
|
|
127
|
-
function parseEntryType(value) {
|
|
128
|
-
if (ENTRY_TYPES.includes(value)) {
|
|
129
|
-
return value;
|
|
130
|
-
}
|
|
131
|
-
throw new Error(`Unsupported entry type "${value}".`);
|
|
132
|
-
}
|
|
133
|
-
function parseRecallMode(value) {
|
|
134
|
-
if (value === void 0) {
|
|
135
|
-
return void 0;
|
|
136
|
-
}
|
|
137
|
-
if (RECALL_MODES.includes(value)) {
|
|
138
|
-
return value;
|
|
139
|
-
}
|
|
140
|
-
throw new Error(`Unsupported recall mode "${value}".`);
|
|
141
|
-
}
|
|
142
|
-
function normalizeStringArray(values) {
|
|
143
|
-
if (!values) {
|
|
144
|
-
return [];
|
|
145
|
-
}
|
|
146
|
-
return Array.from(new Set(values.map((value) => value.trim()).filter((value) => value.length > 0)));
|
|
147
|
-
}
|
|
148
|
-
function formatTargetSelector(id, subject, last) {
|
|
149
|
-
if (last === true) {
|
|
150
|
-
return "last";
|
|
151
|
-
}
|
|
152
|
-
if (id) {
|
|
153
|
-
return `id:${JSON.stringify(id)}`;
|
|
154
|
-
}
|
|
155
|
-
if (subject) {
|
|
156
|
-
return `subject:${JSON.stringify(subject)}`;
|
|
157
|
-
}
|
|
158
|
-
return "unknown";
|
|
159
|
-
}
|
|
160
|
-
function formatTargetSelectorFromParams(params, options = {}) {
|
|
161
|
-
const id = readTrimmedOptionalStringParam(params, "id");
|
|
162
|
-
const subject = readTrimmedOptionalStringParam(params, "subject");
|
|
163
|
-
const last = options.allowLast ? readBooleanParam(params, "last") : void 0;
|
|
164
|
-
const maxValueChars = options.maxValueChars;
|
|
165
|
-
return formatTargetSelector(
|
|
166
|
-
id && maxValueChars !== void 0 ? truncate(id, maxValueChars) : id,
|
|
167
|
-
subject && maxValueChars !== void 0 ? truncate(subject, maxValueChars) : subject,
|
|
168
|
-
last
|
|
169
|
-
);
|
|
170
|
-
}
|
|
171
|
-
function readTrimmedOptionalStringParam(params, key) {
|
|
172
|
-
const value = params[key];
|
|
173
|
-
if (value === void 0 || value === null) {
|
|
174
|
-
return void 0;
|
|
175
|
-
}
|
|
176
|
-
if (typeof value !== "string") {
|
|
177
|
-
throw new Error(`${key} must be a string.`);
|
|
178
|
-
}
|
|
179
|
-
const normalized = value.trim();
|
|
180
|
-
return normalized.length > 0 ? normalized : void 0;
|
|
181
|
-
}
|
|
182
|
-
function sanitizeUpdateToolParams(params) {
|
|
183
|
-
return {
|
|
184
|
-
...params.id ? { id: params.id } : {},
|
|
185
|
-
...params.subject ? { subject: params.subject } : {},
|
|
186
|
-
...params.importance !== void 0 ? { importance: params.importance } : {},
|
|
187
|
-
...params.expiry !== void 0 ? { expiry: params.expiry } : {},
|
|
188
|
-
...params.claimKey !== void 0 ? { hasClaimKey: true } : {},
|
|
189
|
-
...params.validFrom !== void 0 ? { hasValidFrom: true } : {},
|
|
190
|
-
...params.validTo !== void 0 ? { hasValidTo: true } : {}
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
function sanitizeFetchToolParams(params) {
|
|
194
|
-
return {
|
|
195
|
-
...params.id ? { id: params.id } : {},
|
|
196
|
-
...params.subject ? { subject: params.subject } : {}
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
// src/app/episode-ingest/activity-threshold.ts
|
|
201
|
-
function resolveEpisodeActivityEligibility(materialTurns, startedAt, endedAt, threshold) {
|
|
202
|
-
const durationMs = resolveTranscriptDurationMs(startedAt, endedAt);
|
|
203
|
-
if (materialTurns >= threshold.minMaterialTurns || durationMs >= threshold.minDurationMs) {
|
|
204
|
-
return {
|
|
205
|
-
eligible: true,
|
|
206
|
-
materialTurns,
|
|
207
|
-
durationMs
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
return {
|
|
211
|
-
eligible: false,
|
|
212
|
-
reason: "below_activity_threshold",
|
|
213
|
-
materialTurns,
|
|
214
|
-
durationMs
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
function resolveTranscriptDurationMs(startedAt, endedAt) {
|
|
218
|
-
if (!startedAt || !endedAt) {
|
|
219
|
-
return 0;
|
|
220
|
-
}
|
|
221
|
-
const started = Date.parse(startedAt);
|
|
222
|
-
const ended = Date.parse(endedAt);
|
|
223
|
-
return Number.isFinite(started) && Number.isFinite(ended) && ended > started ? ended - started : 0;
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
// src/core/episode/transcript-render.ts
|
|
227
|
-
var MIN_EPISODE_MESSAGES = 4;
|
|
228
|
-
var MAX_EPISODE_TRANSCRIPT_CHARS = 14e3;
|
|
229
|
-
function countMaterialTranscriptTurns(messages) {
|
|
230
|
-
return messages.filter((message) => message.text.trim().length > 0).length;
|
|
231
|
-
}
|
|
232
|
-
function renderTranscript(messages) {
|
|
233
|
-
return messages.map((message) => `${message.role === "user" ? "User" : "Assistant"}: ${message.text.trim()}`).join("\n");
|
|
234
|
-
}
|
|
235
|
-
function capEpisodeTranscript(transcript, maxChars) {
|
|
236
|
-
if (transcript.length <= maxChars) {
|
|
237
|
-
return transcript;
|
|
238
|
-
}
|
|
239
|
-
const omissionMarker = "\n\n[Earlier middle transcript omitted for brevity]\n\n";
|
|
240
|
-
const headBudget = Math.max(0, Math.floor((maxChars - omissionMarker.length) * 0.35));
|
|
241
|
-
const tailBudget = Math.max(0, maxChars - omissionMarker.length - headBudget);
|
|
242
|
-
const head = trimToBoundary(transcript.slice(0, headBudget), false);
|
|
243
|
-
const tail = trimToBoundary(transcript.slice(-tailBudget), true);
|
|
244
|
-
return `${head}${omissionMarker}${tail}`.trim();
|
|
245
|
-
}
|
|
246
|
-
function trimToBoundary(value, fromStart) {
|
|
247
|
-
if (value.length === 0) {
|
|
248
|
-
return value;
|
|
249
|
-
}
|
|
250
|
-
if (fromStart) {
|
|
251
|
-
const boundary = value.search(/\s/u);
|
|
252
|
-
return boundary >= 0 ? value.slice(boundary).trimStart() : value.trim();
|
|
253
|
-
}
|
|
254
|
-
const reversedBoundary = value.trimEnd().search(/\s\S*$/u);
|
|
255
|
-
return reversedBoundary >= 0 ? value.slice(0, reversedBoundary).trimEnd() : value.trim();
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
// src/app/episode-ingest/single-transcript.ts
|
|
259
|
-
function createSingleTranscriptDiscoveryPort(filePath) {
|
|
260
|
-
return {
|
|
261
|
-
async discoverFiles(_targetPath) {
|
|
262
|
-
return [filePath];
|
|
263
|
-
}
|
|
264
|
-
};
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
// src/core/episode/summary-prompt.ts
|
|
268
|
-
var EPISODE_SUMMARY_SYSTEM_PROMPT = [
|
|
269
|
-
"You write strict JSON episode summaries for historical recall.",
|
|
270
|
-
"The transcript can be about any topic - technical work, casual conversation, planning, research, creative projects, life events, or anything else.",
|
|
271
|
-
"Do not assume any particular domain.",
|
|
272
|
-
"Describe only what happened in this session.",
|
|
273
|
-
"Do not carry inherited context or open loops forward unless the session actively worked on them.",
|
|
274
|
-
"Return exactly one JSON object with this shape:",
|
|
275
|
-
'{ "summary": string, "tags": string[], "activityLevel": "substantial" | "minimal" | "none", "project": string | null }',
|
|
276
|
-
"Requirements:",
|
|
277
|
-
"- summary must be 100 to 300 words in plain prose (roughly 4 to 10 sentences)",
|
|
278
|
-
"- describe what was discussed, decided, or accomplished - not a turn-by-turn replay",
|
|
279
|
-
"- this is a narrative overview for historical recall, not a verbatim record",
|
|
280
|
-
"- preserve concrete details worth remembering: names, places, dates, specific decisions, key topics, and notable specifics that would help someone recall this session months later",
|
|
281
|
-
"- tags must be 3 to 8 short lowercase anchors drawn from the actual session content",
|
|
282
|
-
"- project should be null when no clear project scope appears",
|
|
283
|
-
"- activityLevel: use substantial when meaningful discussion or work occurred, minimal when the session was brief or lightweight, none when essentially nothing happened",
|
|
284
|
-
"- do not include Markdown fences or extra commentary"
|
|
285
|
-
].join("\n");
|
|
286
|
-
function buildEpisodeSummaryPrompt(transcript) {
|
|
287
|
-
return [
|
|
288
|
-
"Produce a historical episodic summary for this completed session.",
|
|
289
|
-
"Describe what was discussed, decided, or accomplished during this transcript window.",
|
|
290
|
-
"",
|
|
291
|
-
"Transcript:",
|
|
292
|
-
transcript
|
|
293
|
-
].join("\n");
|
|
294
|
-
}
|
|
295
|
-
function parseEpisodeSummaryResponse(value) {
|
|
296
|
-
const parsed = parseJsonObject(value);
|
|
297
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
298
|
-
return null;
|
|
299
|
-
}
|
|
300
|
-
const parsedRecord = parsed;
|
|
301
|
-
const summary = normalizeSummary(parsedRecord.summary);
|
|
302
|
-
const activityLevel = normalizeActivityLevel(parsedRecord.activityLevel);
|
|
303
|
-
if (!summary || !activityLevel) {
|
|
304
|
-
return null;
|
|
305
|
-
}
|
|
306
|
-
return {
|
|
307
|
-
summary,
|
|
308
|
-
tags: normalizeTags(parsedRecord.tags),
|
|
309
|
-
activityLevel,
|
|
310
|
-
...normalizeProject(parsedRecord.project) ? { project: normalizeProject(parsedRecord.project) } : {}
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
function normalizeSummary(value) {
|
|
314
|
-
if (typeof value !== "string") {
|
|
18
|
+
// src/app/dreaming/concurrency.ts
|
|
19
|
+
import { randomUUID } from "crypto";
|
|
20
|
+
var DREAMING_RUN_LEASE_BRAND = /* @__PURE__ */ Symbol("DreamingRunLease");
|
|
21
|
+
var inProcessRunLocks = /* @__PURE__ */ new Map();
|
|
22
|
+
var episodeWriteRefcounts = /* @__PURE__ */ new Map();
|
|
23
|
+
var DEFAULT_LOCK_HEARTBEAT_INTERVAL_MS = 5 * 60 * 1e3;
|
|
24
|
+
var DEFAULT_LOCK_WAIT_TIMEOUT_MS = 60 * 1e3;
|
|
25
|
+
var DEFAULT_LOCK_WAIT_POLL_MS = 500;
|
|
26
|
+
function resolveDreamingLockKey(dbPath) {
|
|
27
|
+
const trimmed = dbPath?.trim();
|
|
28
|
+
return trimmed && trimmed.length > 0 ? trimmed : ":memory:";
|
|
29
|
+
}
|
|
30
|
+
async function tryAcquireDreamingRunLock(port, dbPath) {
|
|
31
|
+
const lockKey = resolveDreamingLockKey(dbPath);
|
|
32
|
+
if (inProcessRunLocks.has(lockKey)) {
|
|
315
33
|
return null;
|
|
316
34
|
}
|
|
317
|
-
const
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
function normalizeActivityLevel(value) {
|
|
321
|
-
if (typeof value !== "string") {
|
|
35
|
+
const token = randomUUID();
|
|
36
|
+
const acquired = await port.tryAcquireRunLock(token);
|
|
37
|
+
if (!acquired) {
|
|
322
38
|
return null;
|
|
323
39
|
}
|
|
324
|
-
|
|
325
|
-
return
|
|
40
|
+
inProcessRunLocks.set(lockKey, token);
|
|
41
|
+
return createDreamingRunLease(port, lockKey, token);
|
|
326
42
|
}
|
|
327
|
-
function
|
|
328
|
-
|
|
329
|
-
|
|
43
|
+
async function withDreamingRunLock(port, dbPath, fn) {
|
|
44
|
+
const lease = await tryAcquireDreamingRunLock(port, dbPath);
|
|
45
|
+
if (!lease) {
|
|
46
|
+
throw new Error("Dreaming run already in progress.");
|
|
330
47
|
}
|
|
331
|
-
return
|
|
332
|
-
new Set(
|
|
333
|
-
value.filter((tag) => typeof tag === "string").map((tag) => tag.trim().toLowerCase()).filter((tag) => tag.length > 0)
|
|
334
|
-
)
|
|
335
|
-
).slice(0, 8);
|
|
48
|
+
return withHeldDreamingRunLock(lease, fn);
|
|
336
49
|
}
|
|
337
|
-
function
|
|
338
|
-
|
|
339
|
-
|
|
50
|
+
async function withHeldDreamingRunLock(lease, fn) {
|
|
51
|
+
let callbackError;
|
|
52
|
+
let result;
|
|
53
|
+
const stopHeartbeat = startDreamingRunLockHeartbeat(lease);
|
|
54
|
+
try {
|
|
55
|
+
await lease.heartbeat();
|
|
56
|
+
result = await fn(lease);
|
|
57
|
+
} catch (error) {
|
|
58
|
+
callbackError = error;
|
|
340
59
|
}
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
for (const candidate of candidates) {
|
|
347
|
-
try {
|
|
348
|
-
return JSON.parse(candidate);
|
|
349
|
-
} catch {
|
|
350
|
-
continue;
|
|
351
|
-
}
|
|
60
|
+
let cleanupError;
|
|
61
|
+
try {
|
|
62
|
+
await stopHeartbeat();
|
|
63
|
+
} catch (error) {
|
|
64
|
+
cleanupError = error;
|
|
352
65
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
if (trimmed) {
|
|
359
|
-
candidates.add(trimmed);
|
|
360
|
-
}
|
|
361
|
-
const fencedMatches = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/giu) ?? [];
|
|
362
|
-
for (const match of fencedMatches) {
|
|
363
|
-
const normalized = match.replace(/```(?:json)?/iu, "").replace(/```/gu, "").trim();
|
|
364
|
-
if (normalized) {
|
|
365
|
-
candidates.add(normalized);
|
|
66
|
+
try {
|
|
67
|
+
await lease.release();
|
|
68
|
+
} catch (error) {
|
|
69
|
+
if (cleanupError === void 0) {
|
|
70
|
+
cleanupError = error;
|
|
366
71
|
}
|
|
367
72
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
if (objectStart >= 0 && objectEnd > objectStart) {
|
|
371
|
-
candidates.add(trimmed.slice(objectStart, objectEnd + 1));
|
|
73
|
+
if (callbackError !== void 0) {
|
|
74
|
+
throw callbackError;
|
|
372
75
|
}
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
// src/app/episode-ingest/service/shared.ts
|
|
377
|
-
var CHARS_PER_TOKEN_ESTIMATE = 4;
|
|
378
|
-
function estimateInputTokens(renderedTranscript) {
|
|
379
|
-
return Math.max(1, Math.ceil(renderedTranscript.length / CHARS_PER_TOKEN_ESTIMATE));
|
|
380
|
-
}
|
|
381
|
-
function estimateEpisodeSummaryInputTokens(renderedTranscript) {
|
|
382
|
-
return estimateInputTokens(EPISODE_SUMMARY_SYSTEM_PROMPT) + estimateInputTokens(buildEpisodeSummaryPrompt(renderedTranscript));
|
|
383
|
-
}
|
|
384
|
-
async function embedEpisodeSummary(summary, ports) {
|
|
385
|
-
if (ports.embedSummary) {
|
|
386
|
-
try {
|
|
387
|
-
return normalizeEmbeddingVector(await ports.embedSummary(summary));
|
|
388
|
-
} catch {
|
|
389
|
-
return void 0;
|
|
390
|
-
}
|
|
76
|
+
if (cleanupError !== void 0) {
|
|
77
|
+
throw cleanupError;
|
|
391
78
|
}
|
|
392
|
-
return
|
|
79
|
+
return result;
|
|
393
80
|
}
|
|
394
|
-
async function
|
|
395
|
-
|
|
396
|
-
return void 0;
|
|
397
|
-
}
|
|
81
|
+
async function withEpisodeWriteGuard(input, fn) {
|
|
82
|
+
beginEpisodeWrite(input.dbPath);
|
|
398
83
|
try {
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
if (!endedAt) {
|
|
407
|
-
return void 0;
|
|
408
|
-
}
|
|
409
|
-
const parsed = new Date(endedAt);
|
|
410
|
-
return Number.isNaN(parsed.getTime()) ? void 0 : parsed;
|
|
411
|
-
}
|
|
412
|
-
function createSerializedExecutor() {
|
|
413
|
-
let pending = Promise.resolve();
|
|
414
|
-
return async (task) => {
|
|
415
|
-
const current = pending.then(task, task);
|
|
416
|
-
pending = current.then(
|
|
417
|
-
() => void 0,
|
|
418
|
-
() => void 0
|
|
419
|
-
);
|
|
420
|
-
return current;
|
|
421
|
-
};
|
|
422
|
-
}
|
|
423
|
-
function createEmptyUsageStats() {
|
|
424
|
-
return {
|
|
425
|
-
calls: 0,
|
|
426
|
-
inputTokens: 0,
|
|
427
|
-
outputTokens: 0,
|
|
428
|
-
cacheReadTokens: 0,
|
|
429
|
-
cacheWriteTokens: 0,
|
|
430
|
-
totalTokens: 0,
|
|
431
|
-
totalCost: 0
|
|
432
|
-
};
|
|
433
|
-
}
|
|
434
|
-
function cloneUsageStats(usage) {
|
|
435
|
-
return {
|
|
436
|
-
calls: usage.calls,
|
|
437
|
-
inputTokens: usage.inputTokens,
|
|
438
|
-
outputTokens: usage.outputTokens,
|
|
439
|
-
cacheReadTokens: usage.cacheReadTokens,
|
|
440
|
-
cacheWriteTokens: usage.cacheWriteTokens,
|
|
441
|
-
totalTokens: usage.totalTokens,
|
|
442
|
-
totalCost: usage.totalCost
|
|
443
|
-
};
|
|
444
|
-
}
|
|
445
|
-
function addUsageStats(total, usage) {
|
|
446
|
-
total.calls += usage.calls;
|
|
447
|
-
total.inputTokens += usage.inputTokens;
|
|
448
|
-
total.outputTokens += usage.outputTokens;
|
|
449
|
-
total.cacheReadTokens += usage.cacheReadTokens;
|
|
450
|
-
total.cacheWriteTokens += usage.cacheWriteTokens;
|
|
451
|
-
total.totalTokens += usage.totalTokens;
|
|
452
|
-
total.totalCost += usage.totalCost;
|
|
453
|
-
return total;
|
|
454
|
-
}
|
|
455
|
-
function trimOptionalString(value) {
|
|
456
|
-
const trimmed = value?.trim();
|
|
457
|
-
return trimmed ? trimmed : void 0;
|
|
458
|
-
}
|
|
459
|
-
function formatExecutionError(error) {
|
|
460
|
-
if (error instanceof Error) {
|
|
461
|
-
return error.message || error.name;
|
|
462
|
-
}
|
|
463
|
-
return String(error);
|
|
464
|
-
}
|
|
465
|
-
function compareCandidatesByEndedAt(left, right) {
|
|
466
|
-
const leftTime = left.endedAt ? new Date(left.endedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
467
|
-
const rightTime = right.endedAt ? new Date(right.endedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
468
|
-
if (leftTime !== rightTime) {
|
|
469
|
-
return rightTime - leftTime;
|
|
84
|
+
const lease = await waitForDreamingRunLock(input.port, input.dbPath, {
|
|
85
|
+
timeoutMs: input.waitTimeoutMs,
|
|
86
|
+
pollMs: input.waitPollMs
|
|
87
|
+
});
|
|
88
|
+
return await withHeldDreamingRunLock(lease, async () => fn());
|
|
89
|
+
} finally {
|
|
90
|
+
endEpisodeWrite(input.dbPath);
|
|
470
91
|
}
|
|
471
|
-
return left.filePath.localeCompare(right.filePath);
|
|
472
92
|
}
|
|
473
|
-
function
|
|
474
|
-
const
|
|
475
|
-
|
|
93
|
+
function beginEpisodeWrite(dbPath) {
|
|
94
|
+
const lockKey = resolveDreamingLockKey(dbPath);
|
|
95
|
+
episodeWriteRefcounts.set(lockKey, (episodeWriteRefcounts.get(lockKey) ?? 0) + 1);
|
|
476
96
|
}
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
}
|
|
484
|
-
if (!Number.isFinite(options.concurrency) || Math.trunc(options.concurrency) <= 0) {
|
|
485
|
-
throw new Error(`Episode embedding backfill concurrency must be a positive integer. Received: ${options.concurrency}.`);
|
|
486
|
-
}
|
|
487
|
-
const pendingEpisodes = await ports.episodes.listEpisodesWithoutEmbeddings();
|
|
488
|
-
if (pendingEpisodes.length === 0) {
|
|
489
|
-
return {
|
|
490
|
-
totalMissing: 0,
|
|
491
|
-
attempted: 0,
|
|
492
|
-
embedded: 0,
|
|
493
|
-
failed: 0,
|
|
494
|
-
estimatedInputTokens: 0
|
|
495
|
-
};
|
|
97
|
+
function endEpisodeWrite(dbPath) {
|
|
98
|
+
const lockKey = resolveDreamingLockKey(dbPath);
|
|
99
|
+
const next = (episodeWriteRefcounts.get(lockKey) ?? 0) - 1;
|
|
100
|
+
if (next <= 0) {
|
|
101
|
+
episodeWriteRefcounts.delete(lockKey);
|
|
102
|
+
return;
|
|
496
103
|
}
|
|
497
|
-
|
|
498
|
-
const workerCount = Math.min(Math.trunc(options.concurrency), pendingEpisodes.length);
|
|
499
|
-
let nextIndex = 0;
|
|
500
|
-
let completed = 0;
|
|
501
|
-
let embeddedCount = 0;
|
|
502
|
-
let failedCount = 0;
|
|
503
|
-
await Promise.all(
|
|
504
|
-
Array.from({ length: workerCount }, async () => {
|
|
505
|
-
while (true) {
|
|
506
|
-
const currentIndex = nextIndex;
|
|
507
|
-
nextIndex += 1;
|
|
508
|
-
if (currentIndex >= pendingEpisodes.length) {
|
|
509
|
-
return;
|
|
510
|
-
}
|
|
511
|
-
const episode = pendingEpisodes[currentIndex];
|
|
512
|
-
if (!episode) {
|
|
513
|
-
return;
|
|
514
|
-
}
|
|
515
|
-
let status = "failed";
|
|
516
|
-
try {
|
|
517
|
-
const vector = await embedEpisodeSummaryWithPort(episode.summary, embedding);
|
|
518
|
-
if (vector) {
|
|
519
|
-
await ports.episodes.updateEpisodeEmbedding(episode.id, vector);
|
|
520
|
-
embeddedCount += 1;
|
|
521
|
-
status = "embedded";
|
|
522
|
-
} else {
|
|
523
|
-
failedCount += 1;
|
|
524
|
-
}
|
|
525
|
-
} catch {
|
|
526
|
-
failedCount += 1;
|
|
527
|
-
}
|
|
528
|
-
completed += 1;
|
|
529
|
-
options.onProgress?.(completed, pendingEpisodes.length, episode, status);
|
|
530
|
-
}
|
|
531
|
-
})
|
|
532
|
-
);
|
|
533
|
-
return {
|
|
534
|
-
totalMissing: pendingEpisodes.length,
|
|
535
|
-
attempted: pendingEpisodes.length,
|
|
536
|
-
embedded: embeddedCount,
|
|
537
|
-
failed: failedCount,
|
|
538
|
-
estimatedInputTokens
|
|
539
|
-
};
|
|
104
|
+
episodeWriteRefcounts.set(lockKey, next);
|
|
540
105
|
}
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
async function generateEpisodeSummary(transcript, llm) {
|
|
544
|
-
const response = await llm.complete(EPISODE_SUMMARY_SYSTEM_PROMPT, buildEpisodeSummaryPrompt(transcript));
|
|
545
|
-
return parseEpisodeSummaryResponse(response);
|
|
106
|
+
function isEpisodeWriteInProgress(dbPath) {
|
|
107
|
+
return (episodeWriteRefcounts.get(resolveDreamingLockKey(dbPath)) ?? 0) > 0;
|
|
546
108
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
import path from "path";
|
|
550
|
-
var ACTIVE_SESSION_WINDOW_MS = 5 * 60 * 1e3;
|
|
551
|
-
async function prepareEpisodeIngest(targetPath, ports, options = {}) {
|
|
552
|
-
const files = await ports.files.discoverFiles(targetPath);
|
|
553
|
-
if (files.length === 0) {
|
|
554
|
-
return createEmptyPreflightResult();
|
|
555
|
-
}
|
|
556
|
-
if (ports.sessionRegistry) {
|
|
557
|
-
await ports.sessionRegistry.listSessions();
|
|
558
|
-
}
|
|
559
|
-
const requestedPreflightConcurrency = options.preflightConcurrency ?? 20;
|
|
560
|
-
const preflightConcurrency = Number.isFinite(requestedPreflightConcurrency) ? Math.max(1, Math.trunc(requestedPreflightConcurrency)) : 20;
|
|
561
|
-
const workerCount = Math.min(preflightConcurrency, files.length);
|
|
562
|
-
const skippedByIndex = new Array(files.length);
|
|
563
|
-
const invalidByIndex = new Array(files.length);
|
|
564
|
-
const candidatesByIndex = new Array(files.length);
|
|
565
|
-
const referenceNow = options.now ?? /* @__PURE__ */ new Date();
|
|
566
|
-
let nextIndex = 0;
|
|
567
|
-
let completed = 0;
|
|
568
|
-
await Promise.all(
|
|
569
|
-
Array.from({ length: workerCount }, async () => {
|
|
570
|
-
while (true) {
|
|
571
|
-
const currentIndex = nextIndex;
|
|
572
|
-
nextIndex += 1;
|
|
573
|
-
if (currentIndex >= files.length) {
|
|
574
|
-
return;
|
|
575
|
-
}
|
|
576
|
-
const filePath = files[currentIndex];
|
|
577
|
-
if (!filePath) {
|
|
578
|
-
return;
|
|
579
|
-
}
|
|
580
|
-
const result = await classifyPreflightTranscript(filePath, ports, {
|
|
581
|
-
referenceNow,
|
|
582
|
-
regenerate: options.regenerate === true
|
|
583
|
-
});
|
|
584
|
-
if (result.kind === "candidate") {
|
|
585
|
-
candidatesByIndex[currentIndex] = result.value;
|
|
586
|
-
} else if (result.kind === "skipped") {
|
|
587
|
-
skippedByIndex[currentIndex] = result.value;
|
|
588
|
-
} else {
|
|
589
|
-
invalidByIndex[currentIndex] = result.value;
|
|
590
|
-
}
|
|
591
|
-
completed += 1;
|
|
592
|
-
options.onPreflightProgress?.(completed, files.length);
|
|
593
|
-
}
|
|
594
|
-
})
|
|
595
|
-
);
|
|
596
|
-
const skipped = skippedByIndex.flatMap((entry) => entry ? [entry] : []);
|
|
597
|
-
const invalid = invalidByIndex.flatMap((entry) => entry ? [entry] : []);
|
|
598
|
-
const candidates = candidatesByIndex.flatMap((entry) => entry ? [entry] : []);
|
|
599
|
-
candidates.sort(compareCandidatesByEndedAt);
|
|
109
|
+
function createDreamingRunLease(port, lockKey, token) {
|
|
110
|
+
let released = false;
|
|
600
111
|
return {
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
discovered: files.length,
|
|
607
|
-
candidates: candidates.length,
|
|
608
|
-
skipped: skipped.length,
|
|
609
|
-
invalid: invalid.length,
|
|
610
|
-
skippedShort: skipped.filter((entry) => entry.reason === "skipped_short").length,
|
|
611
|
-
skippedActive: skipped.filter((entry) => entry.reason === "skipped_active").length,
|
|
612
|
-
skippedExists: skipped.filter((entry) => entry.reason === "skipped_exists").length
|
|
613
|
-
}
|
|
614
|
-
};
|
|
615
|
-
}
|
|
616
|
-
async function classifyPreflightTranscript(filePath, ports, options) {
|
|
617
|
-
const parsedTranscript = await ports.transcript.parseFile(filePath);
|
|
618
|
-
const cleanedMessages = parsedTranscript.messages.filter((message) => message.text.trim().length > 0);
|
|
619
|
-
const materialTurns = countMaterialTranscriptTurns(parsedTranscript.messages);
|
|
620
|
-
const parsedSessionId = parsedTranscript.metadata.sessionId?.trim() || void 0;
|
|
621
|
-
const registryMeta = parsedSessionId ? await ports.sessionRegistry?.getSessionMeta(parsedSessionId) : void 0;
|
|
622
|
-
const reconstructedMeta = registryMeta ? void 0 : {
|
|
623
|
-
surface: parsedTranscript.metadata.reconstructedSurface ?? null,
|
|
624
|
-
metadataSource: parsedTranscript.metadata.surfaceReconstructionSource ?? "none"
|
|
625
|
-
};
|
|
626
|
-
const resolvedMeta = resolveSessionMeta(filePath, parsedSessionId, registryMeta, reconstructedMeta);
|
|
627
|
-
if (!resolvedMeta.sessionId && cleanedMessages.length === 0) {
|
|
628
|
-
return {
|
|
629
|
-
kind: "invalid",
|
|
630
|
-
value: {
|
|
631
|
-
filePath,
|
|
632
|
-
sessionId: void 0,
|
|
633
|
-
transcriptHash: parsedTranscript.metadata.transcriptHash,
|
|
634
|
-
messageCount: 0,
|
|
635
|
-
metadataSource: resolvedMeta.metadataSource
|
|
636
|
-
}
|
|
637
|
-
};
|
|
638
|
-
}
|
|
639
|
-
const existingEpisode = await findExistingEpisode(ports, resolvedMeta.sessionId, parsedTranscript.metadata.transcriptHash);
|
|
640
|
-
if (existingEpisode && options.regenerate !== true) {
|
|
641
|
-
return {
|
|
642
|
-
kind: "skipped",
|
|
643
|
-
value: {
|
|
644
|
-
filePath,
|
|
645
|
-
reason: "skipped_exists",
|
|
646
|
-
sessionId: resolvedMeta.sessionId,
|
|
647
|
-
transcriptHash: parsedTranscript.metadata.transcriptHash,
|
|
648
|
-
messageCount: cleanedMessages.length,
|
|
649
|
-
startedAt: parsedTranscript.metadata.startedAt,
|
|
650
|
-
endedAt: parsedTranscript.metadata.endedAt,
|
|
651
|
-
agentId: resolvedMeta.agentId,
|
|
652
|
-
surface: resolvedMeta.surface,
|
|
653
|
-
metadataSource: resolvedMeta.metadataSource,
|
|
654
|
-
existingEpisode
|
|
112
|
+
token,
|
|
113
|
+
[DREAMING_RUN_LEASE_BRAND]: true,
|
|
114
|
+
async heartbeat() {
|
|
115
|
+
if (released) {
|
|
116
|
+
return;
|
|
655
117
|
}
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
if (cleanedMessages.length < MIN_EPISODE_MESSAGES) {
|
|
659
|
-
return {
|
|
660
|
-
kind: "skipped",
|
|
661
|
-
value: {
|
|
662
|
-
filePath,
|
|
663
|
-
reason: "skipped_short",
|
|
664
|
-
sessionId: resolvedMeta.sessionId,
|
|
665
|
-
transcriptHash: parsedTranscript.metadata.transcriptHash,
|
|
666
|
-
messageCount: cleanedMessages.length,
|
|
667
|
-
startedAt: parsedTranscript.metadata.startedAt,
|
|
668
|
-
endedAt: parsedTranscript.metadata.endedAt,
|
|
669
|
-
agentId: resolvedMeta.agentId,
|
|
670
|
-
surface: resolvedMeta.surface,
|
|
671
|
-
metadataSource: resolvedMeta.metadataSource
|
|
118
|
+
if (inProcessRunLocks.get(lockKey) !== token) {
|
|
119
|
+
throw new Error("Dreaming run lock was lost in this process.");
|
|
672
120
|
}
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
const eligibility = resolveEpisodeActivityEligibility(
|
|
677
|
-
materialTurns,
|
|
678
|
-
parsedTranscript.metadata.startedAt,
|
|
679
|
-
parsedTranscript.metadata.endedAt,
|
|
680
|
-
options.activityThreshold
|
|
681
|
-
);
|
|
682
|
-
if (!eligibility.eligible) {
|
|
683
|
-
return {
|
|
684
|
-
kind: "skipped",
|
|
685
|
-
value: {
|
|
686
|
-
filePath,
|
|
687
|
-
reason: eligibility.reason,
|
|
688
|
-
sessionId: resolvedMeta.sessionId,
|
|
689
|
-
transcriptHash: parsedTranscript.metadata.transcriptHash,
|
|
690
|
-
messageCount: eligibility.materialTurns,
|
|
691
|
-
startedAt: parsedTranscript.metadata.startedAt,
|
|
692
|
-
endedAt: parsedTranscript.metadata.endedAt,
|
|
693
|
-
agentId: resolvedMeta.agentId,
|
|
694
|
-
surface: resolvedMeta.surface,
|
|
695
|
-
metadataSource: resolvedMeta.metadataSource
|
|
696
|
-
}
|
|
697
|
-
};
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
if (options.skipActiveSessionCheck !== true && isActiveSession(parsedTranscript.metadata.endedAt, options.referenceNow)) {
|
|
701
|
-
return {
|
|
702
|
-
kind: "skipped",
|
|
703
|
-
value: {
|
|
704
|
-
filePath,
|
|
705
|
-
reason: "skipped_active",
|
|
706
|
-
sessionId: resolvedMeta.sessionId,
|
|
707
|
-
transcriptHash: parsedTranscript.metadata.transcriptHash,
|
|
708
|
-
messageCount: cleanedMessages.length,
|
|
709
|
-
startedAt: parsedTranscript.metadata.startedAt,
|
|
710
|
-
endedAt: parsedTranscript.metadata.endedAt,
|
|
711
|
-
agentId: resolvedMeta.agentId,
|
|
712
|
-
surface: resolvedMeta.surface,
|
|
713
|
-
metadataSource: resolvedMeta.metadataSource
|
|
121
|
+
const retained = await port.heartbeatRunLock(token);
|
|
122
|
+
if (!retained) {
|
|
123
|
+
throw new Error("Dreaming run lock was lost.");
|
|
714
124
|
}
|
|
715
|
-
};
|
|
716
|
-
}
|
|
717
|
-
const renderedTranscript = capEpisodeTranscript(renderTranscript(cleanedMessages), MAX_EPISODE_TRANSCRIPT_CHARS);
|
|
718
|
-
return {
|
|
719
|
-
kind: "candidate",
|
|
720
|
-
value: {
|
|
721
|
-
filePath,
|
|
722
|
-
sessionId: resolvedMeta.sessionId,
|
|
723
|
-
sourceRef: resolvedMeta.sourceRef,
|
|
724
|
-
transcriptHash: parsedTranscript.metadata.transcriptHash,
|
|
725
|
-
startedAt: parsedTranscript.metadata.startedAt,
|
|
726
|
-
endedAt: parsedTranscript.metadata.endedAt,
|
|
727
|
-
messageCount: cleanedMessages.length,
|
|
728
|
-
agentId: resolvedMeta.agentId,
|
|
729
|
-
surface: resolvedMeta.surface,
|
|
730
|
-
metadataSource: resolvedMeta.metadataSource,
|
|
731
|
-
renderedTranscript,
|
|
732
|
-
estimatedInputTokens: estimateInputTokens(renderedTranscript),
|
|
733
|
-
...existingEpisode ? { existingEpisode } : {}
|
|
734
|
-
}
|
|
735
|
-
};
|
|
736
|
-
}
|
|
737
|
-
function createEmptyPreflightResult() {
|
|
738
|
-
return {
|
|
739
|
-
files: [],
|
|
740
|
-
candidates: [],
|
|
741
|
-
skipped: [],
|
|
742
|
-
invalid: [],
|
|
743
|
-
totals: {
|
|
744
|
-
discovered: 0,
|
|
745
|
-
candidates: 0,
|
|
746
|
-
skipped: 0,
|
|
747
|
-
invalid: 0,
|
|
748
|
-
skippedShort: 0,
|
|
749
|
-
skippedActive: 0,
|
|
750
|
-
skippedExists: 0
|
|
751
|
-
}
|
|
752
|
-
};
|
|
753
|
-
}
|
|
754
|
-
function resolveSessionMeta(filePath, parsedSessionId, registryMeta, reconstructedMeta) {
|
|
755
|
-
if (registryMeta) {
|
|
756
|
-
return {
|
|
757
|
-
sessionId: parsedSessionId ?? registryMeta.sessionId,
|
|
758
|
-
sourceRef: registryMeta.sourceRef,
|
|
759
|
-
agentId: registryMeta.agentId,
|
|
760
|
-
surface: registryMeta.surface,
|
|
761
|
-
metadataSource: "registry"
|
|
762
|
-
};
|
|
763
|
-
}
|
|
764
|
-
return {
|
|
765
|
-
sessionId: parsedSessionId,
|
|
766
|
-
sourceRef: filePath,
|
|
767
|
-
agentId: deriveAgentIdFromPath(filePath),
|
|
768
|
-
surface: reconstructedMeta?.surface ?? null,
|
|
769
|
-
metadataSource: reconstructedMeta?.metadataSource ?? "none"
|
|
770
|
-
};
|
|
771
|
-
}
|
|
772
|
-
function deriveAgentIdFromPath(filePath) {
|
|
773
|
-
const resolved = path.resolve(filePath);
|
|
774
|
-
const parent = path.basename(path.dirname(resolved));
|
|
775
|
-
const grandparent = path.basename(path.dirname(path.dirname(resolved)));
|
|
776
|
-
if (parent !== "sessions") {
|
|
777
|
-
return null;
|
|
778
|
-
}
|
|
779
|
-
const candidate = grandparent.trim();
|
|
780
|
-
if (!candidate || candidate === "." || candidate === "/") {
|
|
781
|
-
return null;
|
|
782
|
-
}
|
|
783
|
-
return candidate;
|
|
784
|
-
}
|
|
785
|
-
function isActiveSession(endedAt, now) {
|
|
786
|
-
if (!endedAt) {
|
|
787
|
-
return false;
|
|
788
|
-
}
|
|
789
|
-
const endedAtDate = new Date(endedAt);
|
|
790
|
-
if (Number.isNaN(endedAtDate.getTime())) {
|
|
791
|
-
return false;
|
|
792
|
-
}
|
|
793
|
-
return endedAtDate.getTime() > now.getTime() - ACTIVE_SESSION_WINDOW_MS;
|
|
794
|
-
}
|
|
795
|
-
async function findExistingEpisode(ports, sessionId, transcriptHash) {
|
|
796
|
-
const bySourceId = sessionId ? await ports.episodes.getEpisodeBySourceId("openclaw", sessionId) : null;
|
|
797
|
-
if (bySourceId) {
|
|
798
|
-
return bySourceId;
|
|
799
|
-
}
|
|
800
|
-
return ports.episodes.getEpisodeByTranscriptHash("openclaw", transcriptHash);
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
// src/app/episode-ingest/service/execute.ts
|
|
804
|
-
async function ingestEpisodeTranscript(filePath, ports, options) {
|
|
805
|
-
const createSummaryLlm = ports.createSummaryLlm;
|
|
806
|
-
if (!createSummaryLlm) {
|
|
807
|
-
throw new Error("Episode transcript ingest requires createSummaryLlm().");
|
|
808
|
-
}
|
|
809
|
-
const classification = await classifyPreflightTranscript(filePath, ports, {
|
|
810
|
-
referenceNow: options.now ?? /* @__PURE__ */ new Date(),
|
|
811
|
-
regenerate: options.regenerate === true,
|
|
812
|
-
skipActiveSessionCheck: options.skipActiveSessionCheck === true,
|
|
813
|
-
activityThreshold: options.activityThreshold
|
|
814
|
-
});
|
|
815
|
-
if (classification.kind === "skipped") {
|
|
816
|
-
return {
|
|
817
|
-
kind: "skipped",
|
|
818
|
-
skipped: classification.value
|
|
819
|
-
};
|
|
820
|
-
}
|
|
821
|
-
if (classification.kind === "invalid") {
|
|
822
|
-
return {
|
|
823
|
-
kind: "invalid",
|
|
824
|
-
invalid: classification.value
|
|
825
|
-
};
|
|
826
|
-
}
|
|
827
|
-
const candidate = applyCandidateOverrides(classification.value, options.candidateOverrides);
|
|
828
|
-
const session = await executeEpisodeCandidate(
|
|
829
|
-
candidate,
|
|
830
|
-
createSummaryLlm,
|
|
831
|
-
ports,
|
|
832
|
-
{
|
|
833
|
-
source: options.source ?? DEFAULT_EPISODE_SOURCE,
|
|
834
|
-
genVersion: options.genVersion
|
|
835
125
|
},
|
|
836
|
-
async (
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
kind: "executed",
|
|
840
|
-
candidate,
|
|
841
|
-
session
|
|
842
|
-
};
|
|
843
|
-
}
|
|
844
|
-
async function executeEpisodeIngestPlan(plan, ports, options) {
|
|
845
|
-
const createSummaryLlm = ports.createSummaryLlm;
|
|
846
|
-
if (!createSummaryLlm) {
|
|
847
|
-
throw new Error("Episode ingest execution requires createSummaryLlm().");
|
|
848
|
-
}
|
|
849
|
-
if (!Number.isFinite(options.concurrency) || Math.trunc(options.concurrency) <= 0) {
|
|
850
|
-
throw new Error(`Episode ingest concurrency must be a positive integer. Received: ${options.concurrency}.`);
|
|
851
|
-
}
|
|
852
|
-
if (plan.candidates.length === 0) {
|
|
853
|
-
return {
|
|
854
|
-
sessions: [],
|
|
855
|
-
usage: createEmptyUsageStats(),
|
|
856
|
-
modelRef: plan.model.modelRef,
|
|
857
|
-
totals: {
|
|
858
|
-
attempted: 0,
|
|
859
|
-
written: 0,
|
|
860
|
-
updated: 0,
|
|
861
|
-
unchanged: 0,
|
|
862
|
-
failed: 0
|
|
126
|
+
async release() {
|
|
127
|
+
if (released) {
|
|
128
|
+
return;
|
|
863
129
|
}
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
let nextIndex = 0;
|
|
868
|
-
let completed = 0;
|
|
869
|
-
const workerCount = Math.min(Math.trunc(options.concurrency), plan.candidates.length);
|
|
870
|
-
const runSerializedWrite = createSerializedExecutor();
|
|
871
|
-
await Promise.all(
|
|
872
|
-
Array.from({ length: workerCount }, async () => {
|
|
873
|
-
while (true) {
|
|
874
|
-
const currentIndex = nextIndex;
|
|
875
|
-
nextIndex += 1;
|
|
876
|
-
if (currentIndex >= plan.candidates.length) {
|
|
877
|
-
return;
|
|
878
|
-
}
|
|
879
|
-
const candidate = plan.candidates[currentIndex];
|
|
880
|
-
if (!candidate) {
|
|
881
|
-
return;
|
|
882
|
-
}
|
|
883
|
-
const result = await executeEpisodeCandidate(
|
|
884
|
-
candidate,
|
|
885
|
-
createSummaryLlm,
|
|
886
|
-
ports,
|
|
887
|
-
{
|
|
888
|
-
source: options.source ?? DEFAULT_EPISODE_SOURCE,
|
|
889
|
-
genVersion: options.genVersion
|
|
890
|
-
},
|
|
891
|
-
runSerializedWrite
|
|
892
|
-
);
|
|
893
|
-
results[currentIndex] = result;
|
|
894
|
-
completed += 1;
|
|
895
|
-
options.onProgress?.(completed, plan.candidates.length, result);
|
|
130
|
+
released = true;
|
|
131
|
+
if (inProcessRunLocks.get(lockKey) === token) {
|
|
132
|
+
inProcessRunLocks.delete(lockKey);
|
|
896
133
|
}
|
|
897
|
-
|
|
898
|
-
);
|
|
899
|
-
const usage = results.reduce((total, result) => addUsageStats(total, result.usage), createEmptyUsageStats());
|
|
900
|
-
return {
|
|
901
|
-
sessions: results,
|
|
902
|
-
usage,
|
|
903
|
-
modelRef: plan.model.modelRef,
|
|
904
|
-
totals: {
|
|
905
|
-
attempted: results.length,
|
|
906
|
-
written: results.filter((result) => result.action === "written").length,
|
|
907
|
-
updated: results.filter((result) => result.action === "updated").length,
|
|
908
|
-
unchanged: results.filter((result) => result.action === "unchanged").length,
|
|
909
|
-
failed: results.filter((result) => result.action === "failed").length
|
|
134
|
+
await port.releaseRunLock(token);
|
|
910
135
|
}
|
|
911
136
|
};
|
|
912
137
|
}
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
return {
|
|
930
|
-
action: "failed",
|
|
931
|
-
filePath: candidate.filePath,
|
|
932
|
-
...candidate.sessionId ? { sessionId: candidate.sessionId } : {},
|
|
933
|
-
error: "invalid_response",
|
|
934
|
-
usage: cloneUsageStats(llm.metadata.usage)
|
|
935
|
-
};
|
|
138
|
+
function startDreamingRunLockHeartbeat(lease) {
|
|
139
|
+
let heartbeatError;
|
|
140
|
+
let pendingHeartbeat = null;
|
|
141
|
+
const timer = setInterval(() => {
|
|
142
|
+
pendingHeartbeat = lease.heartbeat().catch((error) => {
|
|
143
|
+
heartbeatError = error;
|
|
144
|
+
});
|
|
145
|
+
}, DEFAULT_LOCK_HEARTBEAT_INTERVAL_MS);
|
|
146
|
+
timer.unref?.();
|
|
147
|
+
return async () => {
|
|
148
|
+
clearInterval(timer);
|
|
149
|
+
if (pendingHeartbeat) {
|
|
150
|
+
await pendingHeartbeat;
|
|
151
|
+
}
|
|
152
|
+
if (heartbeatError !== void 0) {
|
|
153
|
+
throw heartbeatError;
|
|
936
154
|
}
|
|
937
|
-
const existingEpisode = candidate.existingEpisode;
|
|
938
|
-
const embedding = await embedEpisodeSummary(structured.summary, ports);
|
|
939
|
-
const writeResult = await runSerializedWrite(
|
|
940
|
-
async () => ports.episodes.upsertEpisode({
|
|
941
|
-
source: writeOptions.source,
|
|
942
|
-
...candidate.sessionId ? { sourceId: candidate.sessionId } : {},
|
|
943
|
-
sourceRef: candidate.metadataSource === "registry" || !existingEpisode?.sourceRef ? candidate.sourceRef : existingEpisode.sourceRef,
|
|
944
|
-
transcriptHash: candidate.transcriptHash,
|
|
945
|
-
...trimOptionalString(candidate.agentId) ?? trimOptionalString(existingEpisode?.agentId) ? { agentId: trimOptionalString(candidate.agentId) ?? trimOptionalString(existingEpisode?.agentId) } : {},
|
|
946
|
-
...trimOptionalString(candidate.surface) ?? trimOptionalString(existingEpisode?.surface) ? { surface: trimOptionalString(candidate.surface) ?? trimOptionalString(existingEpisode?.surface) } : {},
|
|
947
|
-
startedAt,
|
|
948
|
-
...endedAt ? { endedAt } : {},
|
|
949
|
-
summary: structured.summary,
|
|
950
|
-
tags: structured.tags,
|
|
951
|
-
activityLevel: structured.activityLevel,
|
|
952
|
-
...structured.project ? { project: structured.project } : {},
|
|
953
|
-
genModel: llm.metadata.modelRef,
|
|
954
|
-
genVersion: writeOptions.genVersion,
|
|
955
|
-
messageCount: candidate.messageCount,
|
|
956
|
-
...embedding ? { embedding } : {}
|
|
957
|
-
})
|
|
958
|
-
);
|
|
959
|
-
return {
|
|
960
|
-
action: mapWriteAction(writeResult.action),
|
|
961
|
-
filePath: candidate.filePath,
|
|
962
|
-
...candidate.sessionId ? { sessionId: candidate.sessionId } : {},
|
|
963
|
-
activityLevel: structured.activityLevel,
|
|
964
|
-
episodeId: writeResult.episode.id,
|
|
965
|
-
usage: cloneUsageStats(llm.metadata.usage)
|
|
966
|
-
};
|
|
967
|
-
} catch (error) {
|
|
968
|
-
return {
|
|
969
|
-
action: "failed",
|
|
970
|
-
filePath: candidate.filePath,
|
|
971
|
-
...candidate.sessionId ? { sessionId: candidate.sessionId } : {},
|
|
972
|
-
error: formatExecutionError(error),
|
|
973
|
-
usage: cloneUsageStats(llm.metadata.usage)
|
|
974
|
-
};
|
|
975
|
-
}
|
|
976
|
-
}
|
|
977
|
-
var DEFAULT_EPISODE_SOURCE = "openclaw";
|
|
978
|
-
function applyCandidateOverrides(candidate, overrides) {
|
|
979
|
-
if (!overrides) {
|
|
980
|
-
return candidate;
|
|
981
|
-
}
|
|
982
|
-
return {
|
|
983
|
-
...candidate,
|
|
984
|
-
...overrides.sessionId !== void 0 ? { sessionId: overrides.sessionId } : {},
|
|
985
|
-
...overrides.sourceRef !== void 0 ? { sourceRef: overrides.sourceRef } : {},
|
|
986
|
-
..."agentId" in overrides ? { agentId: overrides.agentId ?? null } : {},
|
|
987
|
-
..."surface" in overrides ? { surface: overrides.surface ?? null } : {},
|
|
988
|
-
...overrides.metadataSource !== void 0 ? { metadataSource: overrides.metadataSource } : {}
|
|
989
155
|
};
|
|
990
156
|
}
|
|
991
|
-
function
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
function createEpisodeIngestPlan(preflight, model, options = {}) {
|
|
1000
|
-
const cutoff = resolveRecentCutoff(options.recent, options.now);
|
|
1001
|
-
let excludedByRecent = 0;
|
|
1002
|
-
let excludedUndated = 0;
|
|
1003
|
-
const candidates = preflight.candidates.flatMap((candidate) => {
|
|
1004
|
-
const estimatedInputTokens = estimateEpisodeSummaryInputTokens(candidate.renderedTranscript);
|
|
1005
|
-
const plannedCandidate = {
|
|
1006
|
-
...candidate,
|
|
1007
|
-
estimatedInputTokens
|
|
1008
|
-
};
|
|
1009
|
-
if (!cutoff) {
|
|
1010
|
-
return [plannedCandidate];
|
|
1011
|
-
}
|
|
1012
|
-
const endedAt = parseCandidateEndedAt(candidate.endedAt);
|
|
1013
|
-
if (!endedAt) {
|
|
1014
|
-
excludedByRecent += 1;
|
|
1015
|
-
excludedUndated += 1;
|
|
1016
|
-
return [];
|
|
1017
|
-
}
|
|
1018
|
-
if (endedAt.getTime() < cutoff.getTime()) {
|
|
1019
|
-
excludedByRecent += 1;
|
|
1020
|
-
return [];
|
|
157
|
+
async function waitForDreamingRunLock(port, dbPath, options) {
|
|
158
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_LOCK_WAIT_TIMEOUT_MS;
|
|
159
|
+
const pollMs = options.pollMs ?? DEFAULT_LOCK_WAIT_POLL_MS;
|
|
160
|
+
const deadline = Date.now() + timeoutMs;
|
|
161
|
+
while (true) {
|
|
162
|
+
const lease = await tryAcquireDreamingRunLock(port, dbPath);
|
|
163
|
+
if (lease) {
|
|
164
|
+
return lease;
|
|
1021
165
|
}
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
const inputTokens = candidates.reduce((total, candidate) => total + candidate.estimatedInputTokens, 0);
|
|
1025
|
-
const outputTokens = candidates.length * 500;
|
|
1026
|
-
const estimatedCostUsd = inputTokens / 1e6 * model.pricing.input + outputTokens / 1e6 * model.pricing.output;
|
|
1027
|
-
return {
|
|
1028
|
-
candidates,
|
|
1029
|
-
model,
|
|
1030
|
-
estimate: {
|
|
1031
|
-
candidateCount: candidates.length,
|
|
1032
|
-
inputTokens,
|
|
1033
|
-
outputTokens,
|
|
1034
|
-
totalTokens: inputTokens + outputTokens,
|
|
1035
|
-
estimatedCostUsd
|
|
1036
|
-
},
|
|
1037
|
-
...options.recent?.trim() ? { recent: options.recent.trim() } : {},
|
|
1038
|
-
...cutoff ? { recentCutoff: cutoff.toISOString() } : {},
|
|
1039
|
-
totals: {
|
|
1040
|
-
preflightCandidates: preflight.candidates.length,
|
|
1041
|
-
selectedCandidates: candidates.length,
|
|
1042
|
-
excludedByRecent,
|
|
1043
|
-
excludedUndated
|
|
166
|
+
if (Date.now() >= deadline) {
|
|
167
|
+
throw new Error("Timed out waiting for dreaming run lock before episode write.");
|
|
1044
168
|
}
|
|
1045
|
-
|
|
1046
|
-
}
|
|
1047
|
-
function resolveRecentCutoff(recent, now) {
|
|
1048
|
-
const trimmedRecent = recent?.trim();
|
|
1049
|
-
if (!trimmedRecent) {
|
|
1050
|
-
return void 0;
|
|
1051
|
-
}
|
|
1052
|
-
const cutoff = parseRelativeDate(trimmedRecent, now ?? /* @__PURE__ */ new Date());
|
|
1053
|
-
if (!cutoff) {
|
|
1054
|
-
throw new Error(`Unsupported recent value "${trimmedRecent}". Use day shorthand like 30d or an ISO timestamp.`);
|
|
1055
|
-
}
|
|
1056
|
-
return cutoff;
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
// src/adapters/db/memory-repository.ts
|
|
1060
|
-
var ZERO_VECTOR = JSON.stringify(Array.from({ length: EMBEDDING_DIMENSIONS }, () => 0));
|
|
1061
|
-
function createMemoryRepository(executor, options = {}) {
|
|
1062
|
-
return {
|
|
1063
|
-
findEntryBySubject: async (subject) => findEntryBySubject(executor, subject),
|
|
1064
|
-
findMostRecentEntry: async () => findMostRecentEntry(executor),
|
|
1065
|
-
getEntryTrace: async (entryId) => getEntryTrace(executor, entryId, options.claimSlotPolicyConfig),
|
|
1066
|
-
getMemoryStatusSnapshot: async () => getMemoryStatusSnapshot(executor),
|
|
1067
|
-
probeVectorAvailability: async () => probeVectorAvailability(executor)
|
|
1068
|
-
};
|
|
1069
|
-
}
|
|
1070
|
-
async function findEntryBySubject(executor, subject) {
|
|
1071
|
-
const normalizedSubject = subject.trim();
|
|
1072
|
-
if (normalizedSubject.length === 0) {
|
|
1073
|
-
return null;
|
|
1074
|
-
}
|
|
1075
|
-
const result = await executor.execute({
|
|
1076
|
-
sql: `
|
|
1077
|
-
SELECT
|
|
1078
|
-
${ENTRY_SELECT_COLUMNS},
|
|
1079
|
-
CASE
|
|
1080
|
-
WHEN lower(subject) = lower(?) THEN 0
|
|
1081
|
-
WHEN lower(subject) LIKE lower(?) THEN 1
|
|
1082
|
-
ELSE 2
|
|
1083
|
-
END AS match_rank
|
|
1084
|
-
FROM entries
|
|
1085
|
-
WHERE lower(subject) = lower(?)
|
|
1086
|
-
OR lower(subject) LIKE lower(?)
|
|
1087
|
-
ORDER BY match_rank ASC, created_at DESC
|
|
1088
|
-
LIMIT 1
|
|
1089
|
-
`,
|
|
1090
|
-
args: [normalizedSubject, `%${normalizedSubject}%`, normalizedSubject, `%${normalizedSubject}%`]
|
|
1091
|
-
});
|
|
1092
|
-
const row = result.rows[0];
|
|
1093
|
-
return row ? mapEntryRow(row) : null;
|
|
1094
|
-
}
|
|
1095
|
-
async function findMostRecentEntry(executor) {
|
|
1096
|
-
const result = await executor.execute({
|
|
1097
|
-
sql: `
|
|
1098
|
-
SELECT
|
|
1099
|
-
${ENTRY_SELECT_COLUMNS}
|
|
1100
|
-
FROM entries
|
|
1101
|
-
ORDER BY created_at DESC
|
|
1102
|
-
LIMIT 1
|
|
1103
|
-
`
|
|
1104
|
-
});
|
|
1105
|
-
const row = result.rows[0];
|
|
1106
|
-
return row ? mapEntryRow(row) : null;
|
|
1107
|
-
}
|
|
1108
|
-
async function getEntryTrace(executor, entryId, claimSlotPolicyConfig) {
|
|
1109
|
-
const entry = await getEntryByIdIncludingInactive(executor, entryId);
|
|
1110
|
-
if (!entry) {
|
|
1111
|
-
return null;
|
|
169
|
+
await sleep(Math.min(pollMs, Math.max(1, deadline - Date.now())));
|
|
1112
170
|
}
|
|
1113
|
-
const [supersededBy, supersedes, claimFamily, recallEvents] = await Promise.all([
|
|
1114
|
-
entry.superseded_by ? getEntryByIdIncludingInactive(executor, entry.superseded_by) : Promise.resolve(null),
|
|
1115
|
-
listSupersededEntries(executor, entry.id),
|
|
1116
|
-
entry.claim_key ? getClaimFamily(executor, entry.claim_key, claimSlotPolicyConfig) : Promise.resolve(void 0),
|
|
1117
|
-
listRecallEvents(executor, entry.id)
|
|
1118
|
-
]);
|
|
1119
|
-
return {
|
|
1120
|
-
entry,
|
|
1121
|
-
...supersededBy ? { supersededBy } : {},
|
|
1122
|
-
supersedes,
|
|
1123
|
-
...claimFamily ? { claimFamily } : {},
|
|
1124
|
-
recallEvents
|
|
1125
|
-
};
|
|
1126
171
|
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
SELECT
|
|
1131
|
-
COUNT(*) AS active_entries,
|
|
1132
|
-
SUM(CASE WHEN expiry = 'core' THEN 1 ELSE 0 END) AS core_entries,
|
|
1133
|
-
COUNT(DISTINCT source_file) AS source_files
|
|
1134
|
-
FROM entries
|
|
1135
|
-
WHERE ${buildActiveEntryClause()}
|
|
1136
|
-
`
|
|
172
|
+
function sleep(ms) {
|
|
173
|
+
return new Promise((resolve) => {
|
|
174
|
+
setTimeout(resolve, ms);
|
|
1137
175
|
});
|
|
1138
|
-
const row = result.rows[0];
|
|
1139
|
-
if (!row) {
|
|
1140
|
-
return {
|
|
1141
|
-
activeEntries: 0,
|
|
1142
|
-
coreEntries: 0,
|
|
1143
|
-
sourceFiles: 0
|
|
1144
|
-
};
|
|
1145
|
-
}
|
|
1146
|
-
return {
|
|
1147
|
-
activeEntries: readNumber(row, "active_entries", 0),
|
|
1148
|
-
coreEntries: readNumber(row, "core_entries", 0),
|
|
1149
|
-
sourceFiles: readNumber(row, "source_files", 0)
|
|
1150
|
-
};
|
|
1151
|
-
}
|
|
1152
|
-
async function probeVectorAvailability(executor) {
|
|
1153
|
-
try {
|
|
1154
|
-
await executor.execute({
|
|
1155
|
-
sql: `
|
|
1156
|
-
SELECT COUNT(*) AS matches
|
|
1157
|
-
FROM vector_top_k('${VECTOR_INDEX_NAME}', vector32(?), ?) AS matches
|
|
1158
|
-
`,
|
|
1159
|
-
args: [ZERO_VECTOR, 1]
|
|
1160
|
-
});
|
|
1161
|
-
return true;
|
|
1162
|
-
} catch {
|
|
1163
|
-
return false;
|
|
1164
|
-
}
|
|
1165
|
-
}
|
|
1166
|
-
async function getEntryByIdIncludingInactive(executor, entryId) {
|
|
1167
|
-
const normalizedId = entryId.trim();
|
|
1168
|
-
if (normalizedId.length === 0) {
|
|
1169
|
-
return null;
|
|
1170
|
-
}
|
|
1171
|
-
const result = await executor.execute({
|
|
1172
|
-
sql: `
|
|
1173
|
-
SELECT
|
|
1174
|
-
${ENTRY_SELECT_COLUMNS}
|
|
1175
|
-
FROM entries
|
|
1176
|
-
WHERE id = ?
|
|
1177
|
-
LIMIT 1
|
|
1178
|
-
`,
|
|
1179
|
-
args: [normalizedId]
|
|
1180
|
-
});
|
|
1181
|
-
const row = result.rows[0];
|
|
1182
|
-
return row ? mapEntryRow(row) : null;
|
|
1183
|
-
}
|
|
1184
|
-
async function listSupersededEntries(executor, entryId) {
|
|
1185
|
-
const result = await executor.execute({
|
|
1186
|
-
sql: `
|
|
1187
|
-
SELECT
|
|
1188
|
-
${ENTRY_SELECT_COLUMNS}
|
|
1189
|
-
FROM entries
|
|
1190
|
-
WHERE superseded_by = ?
|
|
1191
|
-
ORDER BY created_at DESC
|
|
1192
|
-
`,
|
|
1193
|
-
args: [entryId]
|
|
1194
|
-
});
|
|
1195
|
-
return result.rows.map((row) => mapEntryRow(row));
|
|
1196
|
-
}
|
|
1197
|
-
async function getClaimFamily(executor, claimKey, claimSlotPolicyConfig) {
|
|
1198
|
-
const normalizedClaimKey = claimKey.trim();
|
|
1199
|
-
if (normalizedClaimKey.length === 0) {
|
|
1200
|
-
return void 0;
|
|
1201
|
-
}
|
|
1202
|
-
const result = await executor.execute({
|
|
1203
|
-
sql: `
|
|
1204
|
-
SELECT
|
|
1205
|
-
${ENTRY_SELECT_COLUMNS}
|
|
1206
|
-
FROM entries
|
|
1207
|
-
WHERE claim_key = ?
|
|
1208
|
-
ORDER BY created_at ASC, id ASC
|
|
1209
|
-
`,
|
|
1210
|
-
args: [normalizedClaimKey]
|
|
1211
|
-
});
|
|
1212
|
-
const entries = result.rows.map((row) => mapEntryRow(row));
|
|
1213
|
-
const slotPolicy = resolveClaimSlotPolicy(normalizedClaimKey, claimSlotPolicyConfig);
|
|
1214
|
-
return {
|
|
1215
|
-
claimKey: normalizedClaimKey,
|
|
1216
|
-
slotPolicy: slotPolicy.policy,
|
|
1217
|
-
slotPolicyReason: slotPolicy.reason,
|
|
1218
|
-
entries
|
|
1219
|
-
};
|
|
1220
|
-
}
|
|
1221
|
-
async function listRecallEvents(executor, entryId) {
|
|
1222
|
-
const result = await executor.execute({
|
|
1223
|
-
sql: `
|
|
1224
|
-
SELECT
|
|
1225
|
-
query,
|
|
1226
|
-
session_key,
|
|
1227
|
-
recalled_at
|
|
1228
|
-
FROM recall_events
|
|
1229
|
-
WHERE entry_id = ?
|
|
1230
|
-
ORDER BY recalled_at DESC
|
|
1231
|
-
LIMIT 10
|
|
1232
|
-
`,
|
|
1233
|
-
args: [entryId]
|
|
1234
|
-
});
|
|
1235
|
-
return result.rows.map((row) => ({
|
|
1236
|
-
query: readOptionalString(row, "query"),
|
|
1237
|
-
sessionKey: readOptionalString(row, "session_key"),
|
|
1238
|
-
recalledAt: readRequiredString(row, "recalled_at")
|
|
1239
|
-
}));
|
|
1240
176
|
}
|
|
1241
177
|
|
|
1242
178
|
// src/core/claim-key-entity-family.ts
|
|
@@ -1333,12 +269,12 @@ function detectClaimKeyEntityFamilyCandidates(entries) {
|
|
|
1333
269
|
return support?.autoSafe === true;
|
|
1334
270
|
});
|
|
1335
271
|
const componentProfiles = component.map((entity) => profiles.get(entity)).filter((profile) => Boolean(profile));
|
|
1336
|
-
const
|
|
1337
|
-
const claimKeys =
|
|
272
|
+
const durableIds = normalizeStringArray(componentProfiles.flatMap((profile) => [...profile.durableIds]));
|
|
273
|
+
const claimKeys = normalizeStringArray(componentProfiles.flatMap((profile) => [...profile.claimKeys]));
|
|
1338
274
|
const confidence = componentSupport.length > 0 ? Math.max(...componentSupport.map((support) => support.confidence)) : 0.75;
|
|
1339
275
|
families.push({
|
|
1340
276
|
entityPrefixes: [...component].sort((left, right) => left.localeCompare(right)),
|
|
1341
|
-
|
|
277
|
+
durableIds,
|
|
1342
278
|
claimKeys,
|
|
1343
279
|
canonicalEntityPrefix,
|
|
1344
280
|
canonicalSelectionReasons: canonicalSelection.reasons,
|
|
@@ -1372,6 +308,7 @@ function summarizeClaimKeyEntityPrefixStats(observations) {
|
|
|
1372
308
|
if (!inspection.normalized) {
|
|
1373
309
|
continue;
|
|
1374
310
|
}
|
|
311
|
+
assertKeyedDurableHasLifecycle(observation);
|
|
1375
312
|
const entityPrefix = inspection.normalized.entity;
|
|
1376
313
|
const existing = counts.get(entityPrefix) ?? {
|
|
1377
314
|
entityPrefix,
|
|
@@ -1379,12 +316,11 @@ function summarizeClaimKeyEntityPrefixStats(observations) {
|
|
|
1379
316
|
trustedEntryCount: 0,
|
|
1380
317
|
tentativeEntryCount: 0,
|
|
1381
318
|
unresolvedEntryCount: 0,
|
|
1382
|
-
legacyEntryCount: 0,
|
|
1383
319
|
deterministicRepairEntryCount: 0,
|
|
1384
320
|
manualEntryCount: 0,
|
|
1385
321
|
modelEntryCount: 0,
|
|
1386
322
|
jsonRetryEntryCount: 0,
|
|
1387
|
-
|
|
323
|
+
dreamingFamilyReuseDurableCount: 0
|
|
1388
324
|
};
|
|
1389
325
|
existing.activeEntryCount += 1;
|
|
1390
326
|
switch (observation.claim_key_status) {
|
|
@@ -1397,9 +333,6 @@ function summarizeClaimKeyEntityPrefixStats(observations) {
|
|
|
1397
333
|
case "unresolved":
|
|
1398
334
|
existing.unresolvedEntryCount += 1;
|
|
1399
335
|
break;
|
|
1400
|
-
default:
|
|
1401
|
-
existing.legacyEntryCount += 1;
|
|
1402
|
-
break;
|
|
1403
336
|
}
|
|
1404
337
|
switch (observation.claim_key_source) {
|
|
1405
338
|
case "deterministic_repair":
|
|
@@ -1414,8 +347,8 @@ function summarizeClaimKeyEntityPrefixStats(observations) {
|
|
|
1414
347
|
case "json_retry":
|
|
1415
348
|
existing.jsonRetryEntryCount += 1;
|
|
1416
349
|
break;
|
|
1417
|
-
case "
|
|
1418
|
-
existing.
|
|
350
|
+
case "dreaming_reconcile":
|
|
351
|
+
existing.dreamingFamilyReuseDurableCount += 1;
|
|
1419
352
|
break;
|
|
1420
353
|
default:
|
|
1421
354
|
break;
|
|
@@ -1471,7 +404,7 @@ function buildTrustedClaimKeyEntityProfiles(entries) {
|
|
|
1471
404
|
const entityPrefix = inspection.normalized.entity;
|
|
1472
405
|
const attribute = inspection.normalized.attribute;
|
|
1473
406
|
const profile = getOrCreateProfile(profiles, entityPrefix);
|
|
1474
|
-
profile.
|
|
407
|
+
profile.durableIds.add(entry.id);
|
|
1475
408
|
profile.claimKeys.add(inspection.normalized.claimKey);
|
|
1476
409
|
profile.attributeSet.add(attribute);
|
|
1477
410
|
const [attributeHead = attribute] = attribute.split("_");
|
|
@@ -1503,7 +436,7 @@ function getOrCreateProfile(profiles, entityPrefix) {
|
|
|
1503
436
|
const tokenList = entityPrefix.split("_").filter((token) => token.length > 0);
|
|
1504
437
|
const created = {
|
|
1505
438
|
entityPrefix,
|
|
1506
|
-
|
|
439
|
+
durableIds: /* @__PURE__ */ new Set(),
|
|
1507
440
|
claimKeys: /* @__PURE__ */ new Set(),
|
|
1508
441
|
attributeSet: /* @__PURE__ */ new Set(),
|
|
1509
442
|
attributeHeadSet: /* @__PURE__ */ new Set(),
|
|
@@ -1533,7 +466,7 @@ function buildPairSupport(profiles) {
|
|
|
1533
466
|
}
|
|
1534
467
|
}
|
|
1535
468
|
for (const entities of attributeBuckets.values()) {
|
|
1536
|
-
const normalizedEntities =
|
|
469
|
+
const normalizedEntities = normalizeStringArray(entities);
|
|
1537
470
|
if (normalizedEntities.length < 2 || normalizedEntities.length > MAX_ATTRIBUTE_BUCKET_SIZE) {
|
|
1538
471
|
continue;
|
|
1539
472
|
}
|
|
@@ -1624,7 +557,7 @@ function evaluateEntityFamilyPairSupport(leftProfile, rightProfile) {
|
|
|
1624
557
|
);
|
|
1625
558
|
return {
|
|
1626
559
|
entityPrefixes: [leftProfile.entityPrefix, rightProfile.entityPrefix],
|
|
1627
|
-
|
|
560
|
+
supportingDurableIds: normalizeStringArray([...leftProfile.durableIds, ...rightProfile.durableIds]),
|
|
1628
561
|
sharedAttributes,
|
|
1629
562
|
confidence,
|
|
1630
563
|
autoSafe: lexicalRelation.autoSafe && (sharedAttributes.length >= 2 || sharedAttributes.length === 1 && groundingAnchorCount >= 1 && groundingScore >= 2),
|
|
@@ -1730,7 +663,7 @@ function selectCanonicalEntityPrefix(entityPrefixes, pairSupport, profiles) {
|
|
|
1730
663
|
reasons.push(`lexical alias evidence prefers "${entityPrefix}"`);
|
|
1731
664
|
}
|
|
1732
665
|
scoreByEntity.set(entityPrefix, score);
|
|
1733
|
-
reasonsByEntity.set(entityPrefix,
|
|
666
|
+
reasonsByEntity.set(entityPrefix, normalizeStringArray(reasons));
|
|
1734
667
|
}
|
|
1735
668
|
const ranked = [...scoreByEntity.entries()].sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]));
|
|
1736
669
|
const [bestCandidate, secondCandidate] = ranked;
|
|
@@ -1974,23 +907,133 @@ function intersectSets(left, right) {
|
|
|
1974
907
|
}
|
|
1975
908
|
return intersection.sort((first, second) => first.localeCompare(second));
|
|
1976
909
|
}
|
|
1977
|
-
function
|
|
910
|
+
function normalizeStringArray(values) {
|
|
1978
911
|
return Array.from(new Set(values.map((value) => value.trim()).filter((value) => value.length > 0)));
|
|
1979
912
|
}
|
|
1980
913
|
function buildPairKey(leftEntityPrefix, rightEntityPrefix) {
|
|
1981
914
|
return [leftEntityPrefix, rightEntityPrefix].sort((left, right) => left.localeCompare(right)).join("::");
|
|
1982
915
|
}
|
|
1983
|
-
function getOrCreateSet(map, key) {
|
|
1984
|
-
const existing = map.get(key);
|
|
1985
|
-
if (existing) {
|
|
1986
|
-
return existing;
|
|
916
|
+
function getOrCreateSet(map, key) {
|
|
917
|
+
const existing = map.get(key);
|
|
918
|
+
if (existing) {
|
|
919
|
+
return existing;
|
|
920
|
+
}
|
|
921
|
+
const created = /* @__PURE__ */ new Set();
|
|
922
|
+
map.set(key, created);
|
|
923
|
+
return created;
|
|
924
|
+
}
|
|
925
|
+
function pluralize(count, noun) {
|
|
926
|
+
return count === 1 ? noun : `${noun}s`;
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
// src/core/supersession.ts
|
|
930
|
+
function validateSupersessionRules(oldEntry, newEntry) {
|
|
931
|
+
if (oldEntry.type !== newEntry.type) {
|
|
932
|
+
return {
|
|
933
|
+
ok: false,
|
|
934
|
+
reason: "type_mismatch"
|
|
935
|
+
};
|
|
936
|
+
}
|
|
937
|
+
if (oldEntry.type === "milestone") {
|
|
938
|
+
return {
|
|
939
|
+
ok: false,
|
|
940
|
+
reason: "milestone"
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
if (oldEntry.expiry === "core") {
|
|
944
|
+
return {
|
|
945
|
+
ok: false,
|
|
946
|
+
reason: "core_expiry"
|
|
947
|
+
};
|
|
948
|
+
}
|
|
949
|
+
return {
|
|
950
|
+
ok: true
|
|
951
|
+
};
|
|
952
|
+
}
|
|
953
|
+
function describeSupersessionRuleFailure(reason) {
|
|
954
|
+
switch (reason) {
|
|
955
|
+
case "type_mismatch":
|
|
956
|
+
return "Supersession requires both entries to have the same type.";
|
|
957
|
+
case "milestone":
|
|
958
|
+
return "Milestone entries are never superseded automatically.";
|
|
959
|
+
case "core_expiry":
|
|
960
|
+
return "Core-expiry entries are never superseded automatically.";
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
// src/core/store/hashing.ts
|
|
965
|
+
import { createHash } from "crypto";
|
|
966
|
+
function computeContentHash(content, sourceFile) {
|
|
967
|
+
const input = sourceFile ? `${sourceFile}
|
|
968
|
+
${content}` : content;
|
|
969
|
+
return createHash("sha256").update(input).digest("hex");
|
|
970
|
+
}
|
|
971
|
+
function computeNormContentHash(content) {
|
|
972
|
+
const normalized = content.toLowerCase().replace(/\s+/g, " ").trim().replace(/[^\w\s]/g, "");
|
|
973
|
+
return createHash("sha256").update(normalized).digest("hex");
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
// src/core/store/project-scope.ts
|
|
977
|
+
import path from "path";
|
|
978
|
+
var IGNORED_PROJECT_DIRECTORY_NAMES = /* @__PURE__ */ new Set(["", ".", "..", "users", "user", "home", "tmp", "var"]);
|
|
979
|
+
function resolveDurableProjectScope(entry, context = {}) {
|
|
980
|
+
const entryProject = normalizeOptionalString(entry.project);
|
|
981
|
+
if (entryProject) {
|
|
982
|
+
return entryProject;
|
|
983
|
+
}
|
|
984
|
+
const sessionWorkspace = normalizeOptionalString(context.sessionWorkspace ?? void 0);
|
|
985
|
+
if (sessionWorkspace) {
|
|
986
|
+
if (claimKeySuggestsProjectScope(entry.claim_key, sessionWorkspace) || entryContainsProjectSignal(entry, sessionWorkspace)) {
|
|
987
|
+
return sessionWorkspace;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
const workingDirectoryProject = deriveWorkingDirectoryProject(context.workingDirectory);
|
|
991
|
+
if (workingDirectoryProject && entryContainsProjectSignal(entry, workingDirectoryProject)) {
|
|
992
|
+
return workingDirectoryProject;
|
|
993
|
+
}
|
|
994
|
+
return void 0;
|
|
995
|
+
}
|
|
996
|
+
function claimKeySuggestsProjectScope(claimKey, project) {
|
|
997
|
+
const entity = normalizeMetadataIdentifier(claimKey?.split("/")[0]);
|
|
998
|
+
const normalizedProject = normalizeMetadataIdentifier(project);
|
|
999
|
+
if (!entity || !normalizedProject) {
|
|
1000
|
+
return false;
|
|
1001
|
+
}
|
|
1002
|
+
return entity === normalizedProject;
|
|
1003
|
+
}
|
|
1004
|
+
function deriveWorkingDirectoryProject(workingDirectory) {
|
|
1005
|
+
const normalizedWorkingDirectory = normalizeOptionalString(workingDirectory ?? void 0);
|
|
1006
|
+
if (!normalizedWorkingDirectory) {
|
|
1007
|
+
return void 0;
|
|
1008
|
+
}
|
|
1009
|
+
const candidate = normalizeMetadataIdentifier(path.basename(normalizedWorkingDirectory));
|
|
1010
|
+
if (!candidate || IGNORED_PROJECT_DIRECTORY_NAMES.has(candidate)) {
|
|
1011
|
+
return void 0;
|
|
1012
|
+
}
|
|
1013
|
+
return candidate;
|
|
1014
|
+
}
|
|
1015
|
+
function entryContainsProjectSignal(entry, project) {
|
|
1016
|
+
const projectTokens = project.split("_").filter((token) => token.length > 0);
|
|
1017
|
+
if (projectTokens.length === 0) {
|
|
1018
|
+
return false;
|
|
1987
1019
|
}
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1020
|
+
return [entry.subject, entry.source_context, ...entry.tags ?? []].some((value) => {
|
|
1021
|
+
const tokens = tokenizeText(value);
|
|
1022
|
+
return projectTokens.every((token) => tokens.has(token));
|
|
1023
|
+
});
|
|
1991
1024
|
}
|
|
1992
|
-
function
|
|
1993
|
-
|
|
1025
|
+
function normalizeMetadataIdentifier(value) {
|
|
1026
|
+
const normalized = normalizeOptionalString(value)?.toLowerCase().replace(/[^a-z0-9]+/gu, "_").replace(/^_+|_+$/gu, "");
|
|
1027
|
+
return normalized && normalized.length > 0 ? normalized : void 0;
|
|
1028
|
+
}
|
|
1029
|
+
function tokenizeText(value) {
|
|
1030
|
+
return new Set(
|
|
1031
|
+
(value ?? "").toLowerCase().split(/[^a-z0-9]+/u).map((token) => token.trim()).filter((token) => token.length > 0)
|
|
1032
|
+
);
|
|
1033
|
+
}
|
|
1034
|
+
function normalizeOptionalString(value) {
|
|
1035
|
+
const normalized = value?.trim();
|
|
1036
|
+
return normalized && normalized.length > 0 ? normalized : void 0;
|
|
1994
1037
|
}
|
|
1995
1038
|
|
|
1996
1039
|
// src/core/claim-key-support.ts
|
|
@@ -2239,7 +1282,7 @@ function evaluateClaimKeySupport(entry, targetClaimKey, trustedHints) {
|
|
|
2239
1282
|
familyReuseCount: familyReuseEntries.length,
|
|
2240
1283
|
groundedFamilyReuseCount: groundedFamilyReuseEntries.length,
|
|
2241
1284
|
relaxedStableSlotFamilyGate: promotionSupport.relaxedStableSlotFamilyGate,
|
|
2242
|
-
|
|
1285
|
+
supportingDurableIds: normalizeStringArray2([
|
|
2243
1286
|
...groundedExactReuseEntries.map((candidate) => candidate.id),
|
|
2244
1287
|
...groundedFamilyReuseEntries.map((candidate) => candidate.id),
|
|
2245
1288
|
...familyReuseEntries.filter((candidate) => candidate.id.startsWith("example:")).map((candidate) => candidate.id)
|
|
@@ -2266,7 +1309,7 @@ function createEmptyClaimKeySupportEvaluation() {
|
|
|
2266
1309
|
familyReuseCount: 0,
|
|
2267
1310
|
groundedFamilyReuseCount: 0,
|
|
2268
1311
|
relaxedStableSlotFamilyGate: false,
|
|
2269
|
-
|
|
1312
|
+
supportingDurableIds: [],
|
|
2270
1313
|
supportEvidence: [],
|
|
2271
1314
|
rationaleFragments: []
|
|
2272
1315
|
};
|
|
@@ -2295,18 +1338,18 @@ function evaluateClaimKeyCompactness(claimKey, prior) {
|
|
|
2295
1338
|
};
|
|
2296
1339
|
}
|
|
2297
1340
|
function normalizeGroundingTags(tags) {
|
|
2298
|
-
return
|
|
1341
|
+
return normalizeStringArray2((tags ?? []).map((tag) => normalizeClaimKeySegment(tag)).filter((tag) => tag.length > 0));
|
|
2299
1342
|
}
|
|
2300
1343
|
function tokenizeGroundingText(value) {
|
|
2301
1344
|
if (!value) {
|
|
2302
1345
|
return [];
|
|
2303
1346
|
}
|
|
2304
|
-
return
|
|
1347
|
+
return normalizeStringArray2(
|
|
2305
1348
|
value.split(/[^a-zA-Z0-9]+/u).map((token) => normalizeClaimKeySegment(token)).filter((token) => token.length > 2 && !GROUNDING_STOP_TOKENS.has(token))
|
|
2306
1349
|
);
|
|
2307
1350
|
}
|
|
2308
|
-
function
|
|
2309
|
-
return
|
|
1351
|
+
function buildDurableLocalLexicalTokens(entry) {
|
|
1352
|
+
return normalizeStringArray2([
|
|
2310
1353
|
...tokenizeGroundingText(entry.subject),
|
|
2311
1354
|
...tokenizeGroundingText(entry.content),
|
|
2312
1355
|
...tokenizeGroundingText(entry.source_context),
|
|
@@ -2351,7 +1394,7 @@ function inspectGroundingOverlap(entryTagSet, entrySourceTokens, trustedEntry) {
|
|
|
2351
1394
|
};
|
|
2352
1395
|
}
|
|
2353
1396
|
function inspectCandidateLexicalAlignment(entry, entity, attribute) {
|
|
2354
|
-
const lexicalTokens = new Set(
|
|
1397
|
+
const lexicalTokens = new Set(buildDurableLocalLexicalTokens(entry));
|
|
2355
1398
|
const entityTokens = entity.split("_").filter((token) => token.length > 0);
|
|
2356
1399
|
const attributeTokens = attribute.split("_").filter((token) => token.length > 0 && !GROUNDING_STOP_TOKENS.has(token));
|
|
2357
1400
|
const entityOverlapCount = countSetOverlap(lexicalTokens, entityTokens);
|
|
@@ -2414,7 +1457,7 @@ function intersects(left, right) {
|
|
|
2414
1457
|
}
|
|
2415
1458
|
return false;
|
|
2416
1459
|
}
|
|
2417
|
-
function
|
|
1460
|
+
function normalizeStringArray2(values) {
|
|
2418
1461
|
const seen = /* @__PURE__ */ new Set();
|
|
2419
1462
|
const normalized = [];
|
|
2420
1463
|
for (const value of values) {
|
|
@@ -3069,12 +2112,11 @@ function summarizeAugmentedEntityPrefixStats(entityPrefixStats, entityPrefix) {
|
|
|
3069
2112
|
trustedEntryCount: 0,
|
|
3070
2113
|
tentativeEntryCount: 1,
|
|
3071
2114
|
unresolvedEntryCount: 0,
|
|
3072
|
-
legacyEntryCount: 0,
|
|
3073
2115
|
deterministicRepairEntryCount: 1,
|
|
3074
2116
|
manualEntryCount: 0,
|
|
3075
2117
|
modelEntryCount: 0,
|
|
3076
2118
|
jsonRetryEntryCount: 0,
|
|
3077
|
-
|
|
2119
|
+
dreamingFamilyReuseDurableCount: 0
|
|
3078
2120
|
}
|
|
3079
2121
|
];
|
|
3080
2122
|
}
|
|
@@ -3349,691 +2391,6 @@ function limitUnique(values, limit) {
|
|
|
3349
2391
|
return Array.from(new Set(values.filter((value) => value.length > 0))).slice(0, limit);
|
|
3350
2392
|
}
|
|
3351
2393
|
|
|
3352
|
-
// src/core/store/pipeline.ts
|
|
3353
|
-
import { randomUUID } from "crypto";
|
|
3354
|
-
|
|
3355
|
-
// src/core/supersession.ts
|
|
3356
|
-
function validateSupersessionRules(oldEntry, newEntry) {
|
|
3357
|
-
if (oldEntry.type !== newEntry.type) {
|
|
3358
|
-
return {
|
|
3359
|
-
ok: false,
|
|
3360
|
-
reason: "type_mismatch"
|
|
3361
|
-
};
|
|
3362
|
-
}
|
|
3363
|
-
if (oldEntry.type === "milestone") {
|
|
3364
|
-
return {
|
|
3365
|
-
ok: false,
|
|
3366
|
-
reason: "milestone"
|
|
3367
|
-
};
|
|
3368
|
-
}
|
|
3369
|
-
if (oldEntry.expiry === "core") {
|
|
3370
|
-
return {
|
|
3371
|
-
ok: false,
|
|
3372
|
-
reason: "core_expiry"
|
|
3373
|
-
};
|
|
3374
|
-
}
|
|
3375
|
-
return {
|
|
3376
|
-
ok: true
|
|
3377
|
-
};
|
|
3378
|
-
}
|
|
3379
|
-
function describeSupersessionRuleFailure(reason) {
|
|
3380
|
-
switch (reason) {
|
|
3381
|
-
case "type_mismatch":
|
|
3382
|
-
return "Supersession requires both entries to have the same type.";
|
|
3383
|
-
case "milestone":
|
|
3384
|
-
return "Milestone entries are never superseded automatically.";
|
|
3385
|
-
case "core_expiry":
|
|
3386
|
-
return "Core-expiry entries are never superseded automatically.";
|
|
3387
|
-
}
|
|
3388
|
-
}
|
|
3389
|
-
|
|
3390
|
-
// src/core/store/hashing.ts
|
|
3391
|
-
import { createHash } from "crypto";
|
|
3392
|
-
function computeContentHash(content, sourceFile) {
|
|
3393
|
-
const input = sourceFile ? `${sourceFile}
|
|
3394
|
-
${content}` : content;
|
|
3395
|
-
return createHash("sha256").update(input).digest("hex");
|
|
3396
|
-
}
|
|
3397
|
-
function computeNormContentHash(content) {
|
|
3398
|
-
const normalized = content.toLowerCase().replace(/\s+/g, " ").trim().replace(/[^\w\s]/g, "");
|
|
3399
|
-
return createHash("sha256").update(normalized).digest("hex");
|
|
3400
|
-
}
|
|
3401
|
-
|
|
3402
|
-
// src/core/store/validation.ts
|
|
3403
|
-
var UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
3404
|
-
function validateEntriesWithIndexes(inputs) {
|
|
3405
|
-
const valid = [];
|
|
3406
|
-
const errors = [];
|
|
3407
|
-
const warnings = [];
|
|
3408
|
-
const rejectedInputIndexes = [];
|
|
3409
|
-
for (const [index, input] of inputs.entries()) {
|
|
3410
|
-
const subject = normalizeString(input.subject);
|
|
3411
|
-
const content = normalizeString(input.content);
|
|
3412
|
-
if (!ENTRY_TYPES.includes(input.type)) {
|
|
3413
|
-
errors.push(`Entry ${index} has an invalid type.`);
|
|
3414
|
-
rejectedInputIndexes.push(index);
|
|
3415
|
-
continue;
|
|
3416
|
-
}
|
|
3417
|
-
if (subject.length === 0) {
|
|
3418
|
-
errors.push(`Entry ${index} is missing a subject.`);
|
|
3419
|
-
rejectedInputIndexes.push(index);
|
|
3420
|
-
continue;
|
|
3421
|
-
}
|
|
3422
|
-
if (content.length === 0) {
|
|
3423
|
-
errors.push(`Entry ${index} is missing content.`);
|
|
3424
|
-
rejectedInputIndexes.push(index);
|
|
3425
|
-
continue;
|
|
3426
|
-
}
|
|
3427
|
-
if (input.expiry !== void 0 && !EXPIRY_LEVELS.includes(input.expiry)) {
|
|
3428
|
-
errors.push(`Entry ${index} has an invalid expiry.`);
|
|
3429
|
-
rejectedInputIndexes.push(index);
|
|
3430
|
-
continue;
|
|
3431
|
-
}
|
|
3432
|
-
if (input.tags !== void 0 && !areValidTags(input.tags)) {
|
|
3433
|
-
errors.push(`Entry ${index} has invalid tags.`);
|
|
3434
|
-
rejectedInputIndexes.push(index);
|
|
3435
|
-
continue;
|
|
3436
|
-
}
|
|
3437
|
-
if (input.importance !== void 0 && !Number.isFinite(input.importance)) {
|
|
3438
|
-
errors.push(`Entry ${index} has an invalid importance.`);
|
|
3439
|
-
rejectedInputIndexes.push(index);
|
|
3440
|
-
continue;
|
|
3441
|
-
}
|
|
3442
|
-
if (input.supersedes !== void 0 && !isUuid(input.supersedes)) {
|
|
3443
|
-
errors.push(`Entry ${index} has an invalid supersedes id.`);
|
|
3444
|
-
rejectedInputIndexes.push(index);
|
|
3445
|
-
continue;
|
|
3446
|
-
}
|
|
3447
|
-
const temporalValidity = validateTemporalValidityRange(input.valid_from, input.valid_to);
|
|
3448
|
-
if (!temporalValidity.ok) {
|
|
3449
|
-
errors.push(`Entry ${index} ${temporalValidity.message}`);
|
|
3450
|
-
rejectedInputIndexes.push(index);
|
|
3451
|
-
continue;
|
|
3452
|
-
}
|
|
3453
|
-
let normalizedClaimKey;
|
|
3454
|
-
if (input.claim_key !== void 0) {
|
|
3455
|
-
if (typeof input.claim_key !== "string") {
|
|
3456
|
-
warnings.push(`Entry ${index} provided a non-string claim key and it was dropped.`);
|
|
3457
|
-
} else {
|
|
3458
|
-
const claimKey = normalizeClaimKey(input.claim_key);
|
|
3459
|
-
if (claimKey.ok) {
|
|
3460
|
-
normalizedClaimKey = claimKey.value.claimKey;
|
|
3461
|
-
} else {
|
|
3462
|
-
warnings.push(
|
|
3463
|
-
`Entry ${index} provided invalid claim key ${JSON.stringify(input.claim_key)} and it was dropped: ${describeClaimKeyNormalizationFailure(claimKey.reason)}.`
|
|
3464
|
-
);
|
|
3465
|
-
}
|
|
3466
|
-
}
|
|
3467
|
-
}
|
|
3468
|
-
const claimKeyRaw = normalizedClaimKey ? normalizeOptionalString(input.claim_key_raw) : void 0;
|
|
3469
|
-
const claimKeyStatus = normalizedClaimKey ? normalizeClaimKeyStatus(input.claim_key_status, index, warnings) : void 0;
|
|
3470
|
-
const claimKeySource = normalizedClaimKey ? normalizeClaimKeySource(input.claim_key_source, index, warnings) : void 0;
|
|
3471
|
-
const claimKeyConfidence = normalizedClaimKey ? normalizeClaimKeyConfidence(input.claim_key_confidence, index, warnings) : void 0;
|
|
3472
|
-
const claimKeyRationale = normalizedClaimKey ? normalizeOptionalString(input.claim_key_rationale) : void 0;
|
|
3473
|
-
const claimSupportSourceKind = normalizedClaimKey ? normalizeOptionalString(input.claim_support_source_kind) : void 0;
|
|
3474
|
-
const claimSupportLocator = normalizedClaimKey ? normalizeOptionalString(input.claim_support_locator) : void 0;
|
|
3475
|
-
const claimSupportObservedAt = normalizedClaimKey && input.claim_support_observed_at !== void 0 ? normalizeClaimSupportObservedAt(input.claim_support_observed_at, index, warnings) : void 0;
|
|
3476
|
-
const claimSupportMode = normalizedClaimKey && input.claim_support_mode !== void 0 ? normalizeClaimSupportMode(input.claim_support_mode, index, warnings) : void 0;
|
|
3477
|
-
const hasPrecomputedLifecycleFields = hasPrecomputedClaimKeyLifecycleFields(input);
|
|
3478
|
-
const resolvedPrecomputedLifecycle = normalizedClaimKey && hasPrecomputedLifecycleFields ? buildPrecomputedClaimKeyLifecycle({
|
|
3479
|
-
claim_key: normalizedClaimKey,
|
|
3480
|
-
claim_key_raw: claimKeyRaw,
|
|
3481
|
-
claim_key_status: claimKeyStatus,
|
|
3482
|
-
claim_key_source: claimKeySource,
|
|
3483
|
-
claim_key_confidence: claimKeyConfidence,
|
|
3484
|
-
claim_key_rationale: claimKeyRationale,
|
|
3485
|
-
claim_support_source_kind: claimSupportSourceKind,
|
|
3486
|
-
claim_support_locator: claimSupportLocator,
|
|
3487
|
-
claim_support_observed_at: claimSupportObservedAt,
|
|
3488
|
-
claim_support_mode: claimSupportMode
|
|
3489
|
-
}) : void 0;
|
|
3490
|
-
if (hasPrecomputedLifecycleFields) {
|
|
3491
|
-
if (!normalizedClaimKey) {
|
|
3492
|
-
errors.push(`Entry ${index} provided claim-key lifecycle metadata without a valid claim key.`);
|
|
3493
|
-
rejectedInputIndexes.push(index);
|
|
3494
|
-
continue;
|
|
3495
|
-
}
|
|
3496
|
-
if (!resolvedPrecomputedLifecycle) {
|
|
3497
|
-
errors.push(
|
|
3498
|
-
`Entry ${index} provided partial or invalid claim-key lifecycle metadata. Complete bundles require claim_key_status, claim_key_source, claim_key_confidence, and claim_key_rationale.`
|
|
3499
|
-
);
|
|
3500
|
-
rejectedInputIndexes.push(index);
|
|
3501
|
-
continue;
|
|
3502
|
-
}
|
|
3503
|
-
}
|
|
3504
|
-
valid.push({
|
|
3505
|
-
inputIndex: index,
|
|
3506
|
-
input: {
|
|
3507
|
-
type: input.type,
|
|
3508
|
-
subject,
|
|
3509
|
-
content,
|
|
3510
|
-
importance: clampImportance(input.importance),
|
|
3511
|
-
expiry: input.expiry ?? "temporary",
|
|
3512
|
-
tags: normalizeTags2(input.tags),
|
|
3513
|
-
source_file: normalizeOptionalString(input.source_file),
|
|
3514
|
-
source_context: normalizeOptionalString(input.source_context),
|
|
3515
|
-
user_id: normalizeOptionalString(input.user_id),
|
|
3516
|
-
project: normalizeOptionalString(input.project),
|
|
3517
|
-
created_at: normalizeOptionalString(input.created_at),
|
|
3518
|
-
supersedes: normalizeOptionalString(input.supersedes),
|
|
3519
|
-
claim_key: normalizedClaimKey,
|
|
3520
|
-
claim_key_raw: resolvedPrecomputedLifecycle?.claim_key_raw ?? claimKeyRaw,
|
|
3521
|
-
claim_key_status: resolvedPrecomputedLifecycle?.claim_key_status,
|
|
3522
|
-
claim_key_source: resolvedPrecomputedLifecycle?.claim_key_source,
|
|
3523
|
-
claim_key_confidence: resolvedPrecomputedLifecycle?.claim_key_confidence,
|
|
3524
|
-
claim_key_rationale: resolvedPrecomputedLifecycle?.claim_key_rationale,
|
|
3525
|
-
claim_support_source_kind: resolvedPrecomputedLifecycle?.claim_support_source_kind ?? claimSupportSourceKind,
|
|
3526
|
-
claim_support_locator: resolvedPrecomputedLifecycle?.claim_support_locator ?? claimSupportLocator,
|
|
3527
|
-
claim_support_observed_at: resolvedPrecomputedLifecycle?.claim_support_observed_at ?? claimSupportObservedAt,
|
|
3528
|
-
claim_support_mode: resolvedPrecomputedLifecycle?.claim_support_mode ?? claimSupportMode,
|
|
3529
|
-
valid_from: temporalValidity.value.validFrom,
|
|
3530
|
-
valid_to: temporalValidity.value.validTo
|
|
3531
|
-
}
|
|
3532
|
-
});
|
|
3533
|
-
}
|
|
3534
|
-
return {
|
|
3535
|
-
valid,
|
|
3536
|
-
rejected: errors.length,
|
|
3537
|
-
rejectedInputIndexes,
|
|
3538
|
-
errors,
|
|
3539
|
-
warnings
|
|
3540
|
-
};
|
|
3541
|
-
}
|
|
3542
|
-
function clampImportance(value) {
|
|
3543
|
-
if (value === void 0) {
|
|
3544
|
-
return 7;
|
|
3545
|
-
}
|
|
3546
|
-
return Math.min(10, Math.max(1, Math.round(value)));
|
|
3547
|
-
}
|
|
3548
|
-
function normalizeString(value) {
|
|
3549
|
-
return value.trim();
|
|
3550
|
-
}
|
|
3551
|
-
function normalizeOptionalString(value) {
|
|
3552
|
-
const normalized = value?.trim();
|
|
3553
|
-
return normalized && normalized.length > 0 ? normalized : void 0;
|
|
3554
|
-
}
|
|
3555
|
-
function normalizeClaimSupportObservedAt(value, index, warnings) {
|
|
3556
|
-
const normalized = normalizeOptionalString(value);
|
|
3557
|
-
if (!normalized) {
|
|
3558
|
-
return void 0;
|
|
3559
|
-
}
|
|
3560
|
-
if (!isIsoTimestamp(normalized)) {
|
|
3561
|
-
warnings.push(`Entry ${index} provided invalid claim_support_observed_at ${JSON.stringify(value)} and it was dropped.`);
|
|
3562
|
-
return void 0;
|
|
3563
|
-
}
|
|
3564
|
-
return normalized;
|
|
3565
|
-
}
|
|
3566
|
-
function normalizeClaimKeyStatus(value, index, warnings) {
|
|
3567
|
-
const parsed = parseClaimKeyStatus(value);
|
|
3568
|
-
if (parsed) {
|
|
3569
|
-
return parsed;
|
|
3570
|
-
}
|
|
3571
|
-
if (value !== void 0) {
|
|
3572
|
-
warnings.push(`Entry ${index} provided invalid claim_key_status ${JSON.stringify(value)} and it was dropped.`);
|
|
3573
|
-
}
|
|
3574
|
-
return void 0;
|
|
3575
|
-
}
|
|
3576
|
-
function normalizeClaimKeySource(value, index, warnings) {
|
|
3577
|
-
const parsed = parseClaimKeySource(value);
|
|
3578
|
-
if (parsed) {
|
|
3579
|
-
return parsed;
|
|
3580
|
-
}
|
|
3581
|
-
if (value !== void 0) {
|
|
3582
|
-
warnings.push(`Entry ${index} provided invalid claim_key_source ${JSON.stringify(value)} and it was dropped.`);
|
|
3583
|
-
}
|
|
3584
|
-
return void 0;
|
|
3585
|
-
}
|
|
3586
|
-
function normalizeClaimKeyConfidence(value, index, warnings) {
|
|
3587
|
-
if (value === void 0) {
|
|
3588
|
-
return void 0;
|
|
3589
|
-
}
|
|
3590
|
-
const parsed = parseClaimKeyConfidence(value);
|
|
3591
|
-
if (parsed !== void 0) {
|
|
3592
|
-
return parsed;
|
|
3593
|
-
}
|
|
3594
|
-
warnings.push(`Entry ${index} provided invalid claim_key_confidence ${JSON.stringify(value)} and it was dropped.`);
|
|
3595
|
-
return void 0;
|
|
3596
|
-
}
|
|
3597
|
-
function normalizeClaimSupportMode(value, index, warnings) {
|
|
3598
|
-
const parsed = parseClaimSupportMode(value);
|
|
3599
|
-
if (parsed) {
|
|
3600
|
-
return parsed;
|
|
3601
|
-
}
|
|
3602
|
-
warnings.push(`Entry ${index} provided invalid claim_support_mode ${JSON.stringify(value)} and it was dropped.`);
|
|
3603
|
-
return void 0;
|
|
3604
|
-
}
|
|
3605
|
-
function areValidTags(value) {
|
|
3606
|
-
return Array.isArray(value) && value.every((tag) => typeof tag === "string");
|
|
3607
|
-
}
|
|
3608
|
-
function normalizeTags2(tags) {
|
|
3609
|
-
if (!tags) {
|
|
3610
|
-
return [];
|
|
3611
|
-
}
|
|
3612
|
-
return tags.map((tag) => tag.trim()).filter((tag) => tag.length > 0);
|
|
3613
|
-
}
|
|
3614
|
-
function isUuid(value) {
|
|
3615
|
-
return UUID_PATTERN.test(value.trim());
|
|
3616
|
-
}
|
|
3617
|
-
function isIsoTimestamp(value) {
|
|
3618
|
-
const normalized = value.trim();
|
|
3619
|
-
return normalized.length > 0 && normalized.includes("T") && !Number.isNaN(Date.parse(normalized));
|
|
3620
|
-
}
|
|
3621
|
-
|
|
3622
|
-
// src/core/store/pipeline.ts
|
|
3623
|
-
var AUTO_SUPERSESSION_MIN_EXTRACTED_CONFIDENCE = 0.9;
|
|
3624
|
-
var AUTO_SUPERSESSION_ELIGIBLE_SOURCES = /* @__PURE__ */ new Set(["model", "json_retry"]);
|
|
3625
|
-
async function storeEntriesDetailed(inputs, db, embedding, options = {}) {
|
|
3626
|
-
if (inputs.length === 0) {
|
|
3627
|
-
return { stored: 0, skipped: 0, rejected: 0, details: [] };
|
|
3628
|
-
}
|
|
3629
|
-
const plan = await buildStorePlan(inputs, db);
|
|
3630
|
-
for (const warning of plan.warnings) {
|
|
3631
|
-
options.onWarning?.(warning);
|
|
3632
|
-
}
|
|
3633
|
-
if (plan.pendingEntries.length === 0) {
|
|
3634
|
-
return {
|
|
3635
|
-
stored: 0,
|
|
3636
|
-
skipped: plan.skipped,
|
|
3637
|
-
rejected: plan.rejected,
|
|
3638
|
-
details: sortStoreDetails(plan.details)
|
|
3639
|
-
};
|
|
3640
|
-
}
|
|
3641
|
-
if (options.dryRun === true) {
|
|
3642
|
-
return {
|
|
3643
|
-
stored: 0,
|
|
3644
|
-
skipped: plan.skipped,
|
|
3645
|
-
rejected: plan.rejected,
|
|
3646
|
-
details: sortStoreDetails([
|
|
3647
|
-
...plan.details,
|
|
3648
|
-
...plan.pendingEntries.map((entry) => ({
|
|
3649
|
-
inputIndex: entry.inputIndex,
|
|
3650
|
-
outcome: "dry_run",
|
|
3651
|
-
reason: "dry_run"
|
|
3652
|
-
}))
|
|
3653
|
-
])
|
|
3654
|
-
};
|
|
3655
|
-
}
|
|
3656
|
-
const pendingEntries = plan.pendingEntries;
|
|
3657
|
-
const extractedClaimKeys = await maybeExtractClaimKeys(pendingEntries, options);
|
|
3658
|
-
applyExtractedClaimKeyMetadata(pendingEntries, extractedClaimKeys);
|
|
3659
|
-
const embeddings = await resolvePendingEmbeddings(inputs, pendingEntries, embedding, options.precomputedEmbeddings);
|
|
3660
|
-
await persistEntries(db, pendingEntries, embeddings, extractedClaimKeys, options.claimExtraction?.config, options.onWarning);
|
|
3661
|
-
return {
|
|
3662
|
-
stored: pendingEntries.length,
|
|
3663
|
-
skipped: plan.skipped,
|
|
3664
|
-
rejected: plan.rejected,
|
|
3665
|
-
details: sortStoreDetails([
|
|
3666
|
-
...plan.details,
|
|
3667
|
-
...pendingEntries.map((entry) => ({
|
|
3668
|
-
inputIndex: entry.inputIndex,
|
|
3669
|
-
outcome: "stored"
|
|
3670
|
-
}))
|
|
3671
|
-
])
|
|
3672
|
-
};
|
|
3673
|
-
}
|
|
3674
|
-
async function resolvePendingEmbeddings(inputs, entries, embedding, precomputedEmbeddings) {
|
|
3675
|
-
if (!precomputedEmbeddings) {
|
|
3676
|
-
return embedPendingEntries(entries, embedding);
|
|
3677
|
-
}
|
|
3678
|
-
if (precomputedEmbeddings.length !== inputs.length) {
|
|
3679
|
-
throw new Error(`Precomputed embedding length mismatch: expected ${inputs.length}, received ${precomputedEmbeddings.length}.`);
|
|
3680
|
-
}
|
|
3681
|
-
return entries.map((entry) => {
|
|
3682
|
-
const vector = precomputedEmbeddings[entry.inputIndex];
|
|
3683
|
-
if (!vector) {
|
|
3684
|
-
throw new Error(`Missing precomputed embedding for input index ${entry.inputIndex}.`);
|
|
3685
|
-
}
|
|
3686
|
-
return vector;
|
|
3687
|
-
});
|
|
3688
|
-
}
|
|
3689
|
-
async function embedPendingEntries(entries, embedding) {
|
|
3690
|
-
const texts = entries.map(({ input }) => composeEmbeddingText(input));
|
|
3691
|
-
const vectors = await embedding.embed(texts);
|
|
3692
|
-
if (vectors.length !== entries.length) {
|
|
3693
|
-
throw new Error(`Embedding length mismatch: expected ${entries.length}, received ${vectors.length}.`);
|
|
3694
|
-
}
|
|
3695
|
-
return vectors;
|
|
3696
|
-
}
|
|
3697
|
-
async function persistEntries(db, preparedEntries, embeddings, extractedClaimKeys, claimExtractionConfig, onWarning) {
|
|
3698
|
-
const writeBatch = async (targetDb) => {
|
|
3699
|
-
let stored = 0;
|
|
3700
|
-
const autoSupersessionPlans = await planAutoSupersession(targetDb, preparedEntries, extractedClaimKeys, claimExtractionConfig);
|
|
3701
|
-
const emittedWarnings = /* @__PURE__ */ new Set();
|
|
3702
|
-
for (const [index, preparedEntry] of preparedEntries.entries()) {
|
|
3703
|
-
const embedding = embeddings[index] ?? [];
|
|
3704
|
-
const entry = buildEntry(preparedEntry, embedding);
|
|
3705
|
-
const entryId = await targetDb.insertEntry(entry, embedding, preparedEntry.contentHash);
|
|
3706
|
-
const supersededEntryId = preparedEntry.input.supersedes;
|
|
3707
|
-
if (supersededEntryId) {
|
|
3708
|
-
const superseded = await targetDb.supersedeEntry(supersededEntryId, entryId, "update");
|
|
3709
|
-
if (!superseded) {
|
|
3710
|
-
onWarning?.(`Stored entry ${entryId} but could not supersede ${supersededEntryId} because the target was missing or inactive.`);
|
|
3711
|
-
}
|
|
3712
|
-
}
|
|
3713
|
-
const autoSupersessionPlan = autoSupersessionPlans.get(preparedEntry.inputIndex);
|
|
3714
|
-
if (autoSupersessionPlan?.kind === "link" && autoSupersessionPlan.oldEntryId) {
|
|
3715
|
-
const superseded = await targetDb.supersedeEntry(autoSupersessionPlan.oldEntryId, entryId, "update");
|
|
3716
|
-
if (!superseded) {
|
|
3717
|
-
onWarning?.(
|
|
3718
|
-
`Stored entry ${entryId} with claim_key "${preparedEntry.input.claim_key}" but could not auto-supersede ${autoSupersessionPlan.oldEntryId} because the target was missing or inactive.`
|
|
3719
|
-
);
|
|
3720
|
-
}
|
|
3721
|
-
}
|
|
3722
|
-
if (autoSupersessionPlan?.warning && !emittedWarnings.has(autoSupersessionPlan.warning)) {
|
|
3723
|
-
emittedWarnings.add(autoSupersessionPlan.warning);
|
|
3724
|
-
onWarning?.(autoSupersessionPlan.warning);
|
|
3725
|
-
}
|
|
3726
|
-
stored += 1;
|
|
3727
|
-
}
|
|
3728
|
-
return stored;
|
|
3729
|
-
};
|
|
3730
|
-
if (hasTransactionSupport(db) && preparedEntries.some((entry) => entry.input.supersedes !== void 0 || entry.input.claim_key !== void 0)) {
|
|
3731
|
-
return db.withTransaction(writeBatch);
|
|
3732
|
-
}
|
|
3733
|
-
if (hasTransactionSupport(db) && preparedEntries.length > 1) {
|
|
3734
|
-
return db.withTransaction(writeBatch);
|
|
3735
|
-
}
|
|
3736
|
-
return writeBatch(db);
|
|
3737
|
-
}
|
|
3738
|
-
function buildEntry(preparedEntry, embedding) {
|
|
3739
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3740
|
-
const acceptedClaimKey = preparedEntry.claimKey;
|
|
3741
|
-
return {
|
|
3742
|
-
id: randomUUID(),
|
|
3743
|
-
type: preparedEntry.input.type,
|
|
3744
|
-
subject: preparedEntry.input.subject,
|
|
3745
|
-
content: preparedEntry.input.content,
|
|
3746
|
-
importance: preparedEntry.input.importance ?? 7,
|
|
3747
|
-
expiry: preparedEntry.input.expiry ?? "temporary",
|
|
3748
|
-
tags: preparedEntry.input.tags ?? [],
|
|
3749
|
-
source_file: preparedEntry.input.source_file,
|
|
3750
|
-
source_context: preparedEntry.input.source_context,
|
|
3751
|
-
user_id: preparedEntry.input.user_id,
|
|
3752
|
-
project: preparedEntry.input.project,
|
|
3753
|
-
embedding,
|
|
3754
|
-
content_hash: preparedEntry.contentHash,
|
|
3755
|
-
norm_content_hash: preparedEntry.normContentHash,
|
|
3756
|
-
quality_score: 0.5,
|
|
3757
|
-
recall_count: 0,
|
|
3758
|
-
valid_from: preparedEntry.input.valid_from,
|
|
3759
|
-
valid_to: preparedEntry.input.valid_to,
|
|
3760
|
-
claim_key: acceptedClaimKey?.claim_key ?? preparedEntry.input.claim_key,
|
|
3761
|
-
claim_key_raw: acceptedClaimKey?.claim_key_raw,
|
|
3762
|
-
claim_key_status: acceptedClaimKey?.claim_key_status,
|
|
3763
|
-
claim_key_source: acceptedClaimKey?.claim_key_source,
|
|
3764
|
-
claim_key_confidence: acceptedClaimKey?.claim_key_confidence,
|
|
3765
|
-
claim_key_rationale: acceptedClaimKey?.claim_key_rationale,
|
|
3766
|
-
claim_support_source_kind: acceptedClaimKey?.claim_support_source_kind,
|
|
3767
|
-
claim_support_locator: acceptedClaimKey?.claim_support_locator,
|
|
3768
|
-
claim_support_observed_at: acceptedClaimKey?.claim_support_observed_at,
|
|
3769
|
-
claim_support_mode: acceptedClaimKey?.claim_support_mode,
|
|
3770
|
-
retired: false,
|
|
3771
|
-
created_at: preparedEntry.input.created_at ?? now,
|
|
3772
|
-
updated_at: now
|
|
3773
|
-
};
|
|
3774
|
-
}
|
|
3775
|
-
async function maybeExtractClaimKeys(preparedEntries, options) {
|
|
3776
|
-
const claimExtraction = options.claimExtraction;
|
|
3777
|
-
if (!claimExtraction || preparedEntries.length === 0) {
|
|
3778
|
-
return /* @__PURE__ */ new Map();
|
|
3779
|
-
}
|
|
3780
|
-
try {
|
|
3781
|
-
const extractedEntries = await runBatchClaimExtraction(
|
|
3782
|
-
[
|
|
3783
|
-
{
|
|
3784
|
-
entries: preparedEntries.map((preparedEntry) => preparedEntry.input)
|
|
3785
|
-
}
|
|
3786
|
-
],
|
|
3787
|
-
{
|
|
3788
|
-
createLlm: () => claimExtraction.llm,
|
|
3789
|
-
db: claimExtraction.db
|
|
3790
|
-
},
|
|
3791
|
-
claimExtraction.config,
|
|
3792
|
-
claimExtraction.config.concurrency ?? 10,
|
|
3793
|
-
options.onWarning,
|
|
3794
|
-
(entry, diagnostic) => {
|
|
3795
|
-
const preparedEntry = preparedEntries.find((candidate) => candidate.input === entry);
|
|
3796
|
-
if (preparedEntry) {
|
|
3797
|
-
options.onClaimExtractionDiagnostic?.(preparedEntry.inputIndex, diagnostic);
|
|
3798
|
-
}
|
|
3799
|
-
}
|
|
3800
|
-
);
|
|
3801
|
-
const extractedClaimKeys = /* @__PURE__ */ new Map();
|
|
3802
|
-
for (const preparedEntry of preparedEntries) {
|
|
3803
|
-
const extracted = extractedEntries.get(preparedEntry.input);
|
|
3804
|
-
if (extracted) {
|
|
3805
|
-
extractedClaimKeys.set(preparedEntry.inputIndex, extracted);
|
|
3806
|
-
}
|
|
3807
|
-
}
|
|
3808
|
-
return extractedClaimKeys;
|
|
3809
|
-
} catch (error) {
|
|
3810
|
-
const subject = preparedEntries[0]?.input.subject ?? "batch";
|
|
3811
|
-
options.onWarning?.(`Claim extraction failed for "${subject}": ${formatPipelineError(error)}`);
|
|
3812
|
-
return /* @__PURE__ */ new Map();
|
|
3813
|
-
}
|
|
3814
|
-
}
|
|
3815
|
-
function hasTransactionSupport(db) {
|
|
3816
|
-
return typeof db.withTransaction === "function";
|
|
3817
|
-
}
|
|
3818
|
-
function applyExtractedClaimKeyMetadata(preparedEntries, extractedClaimKeys) {
|
|
3819
|
-
for (const preparedEntry of preparedEntries) {
|
|
3820
|
-
if (preparedEntry.claimKey) {
|
|
3821
|
-
continue;
|
|
3822
|
-
}
|
|
3823
|
-
const extractedClaimKey = extractedClaimKeys.get(preparedEntry.inputIndex);
|
|
3824
|
-
const acceptedClaimKey = buildPrecomputedClaimKeyLifecycle(preparedEntry.input) ?? (extractedClaimKey ? buildExtractedClaimKeyLifecycle(extractedClaimKey, buildInferredIngestClaimKeySupportContext(preparedEntry.input)) : void 0);
|
|
3825
|
-
if (!acceptedClaimKey) {
|
|
3826
|
-
continue;
|
|
3827
|
-
}
|
|
3828
|
-
preparedEntry.claimKey = acceptedClaimKey;
|
|
3829
|
-
applyClaimKeyLifecycle(preparedEntry.input, acceptedClaimKey);
|
|
3830
|
-
}
|
|
3831
|
-
}
|
|
3832
|
-
async function planAutoSupersession(db, preparedEntries, extractedClaimKeys, claimExtractionConfig) {
|
|
3833
|
-
const plans = /* @__PURE__ */ new Map();
|
|
3834
|
-
const preparedEntriesByClaimKey = groupPreparedEntriesByClaimKey(preparedEntries);
|
|
3835
|
-
const siblingCache = /* @__PURE__ */ new Map();
|
|
3836
|
-
for (const preparedEntry of preparedEntries) {
|
|
3837
|
-
const claimKey = preparedEntry.claimKey?.claim_key ?? preparedEntry.input.claim_key;
|
|
3838
|
-
if (!claimKey || preparedEntry.input.supersedes) {
|
|
3839
|
-
continue;
|
|
3840
|
-
}
|
|
3841
|
-
const siblings = await getClaimKeySiblings(db, siblingCache, claimKey);
|
|
3842
|
-
if (siblings.length === 0) {
|
|
3843
|
-
continue;
|
|
3844
|
-
}
|
|
3845
|
-
const batchSiblingCount = preparedEntriesByClaimKey.get(claimKey)?.length ?? 0;
|
|
3846
|
-
if (batchSiblingCount > 1) {
|
|
3847
|
-
plans.set(preparedEntry.inputIndex, {
|
|
3848
|
-
kind: "skip",
|
|
3849
|
-
warning: `Skipped auto-supersession for claim_key "${claimKey}" because this store batch contains ${batchSiblingCount} entries for the same slot.`
|
|
3850
|
-
});
|
|
3851
|
-
continue;
|
|
3852
|
-
}
|
|
3853
|
-
if (siblings.length > 1) {
|
|
3854
|
-
plans.set(preparedEntry.inputIndex, {
|
|
3855
|
-
kind: "skip",
|
|
3856
|
-
warning: `Skipped auto-supersession for claim_key "${claimKey}" because ${siblings.length} active siblings already exist for that slot.`
|
|
3857
|
-
});
|
|
3858
|
-
continue;
|
|
3859
|
-
}
|
|
3860
|
-
const sibling = siblings[0];
|
|
3861
|
-
if (!sibling) {
|
|
3862
|
-
continue;
|
|
3863
|
-
}
|
|
3864
|
-
if (!isAutoSupersessionEligible(preparedEntry.claimKey, claimExtractionConfig)) {
|
|
3865
|
-
plans.set(preparedEntry.inputIndex, {
|
|
3866
|
-
kind: "skip",
|
|
3867
|
-
warning: buildAutoSupersessionEligibilityWarning(preparedEntry)
|
|
3868
|
-
});
|
|
3869
|
-
continue;
|
|
3870
|
-
}
|
|
3871
|
-
const supersessionValidation = validateSupersessionRules(sibling, {
|
|
3872
|
-
type: preparedEntry.input.type,
|
|
3873
|
-
expiry: preparedEntry.input.expiry ?? "temporary"
|
|
3874
|
-
});
|
|
3875
|
-
if (!supersessionValidation.ok) {
|
|
3876
|
-
plans.set(preparedEntry.inputIndex, {
|
|
3877
|
-
kind: "skip",
|
|
3878
|
-
warning: buildAutoSupersessionRuleWarning(preparedEntry, sibling, supersessionValidation.reason)
|
|
3879
|
-
});
|
|
3880
|
-
continue;
|
|
3881
|
-
}
|
|
3882
|
-
plans.set(preparedEntry.inputIndex, {
|
|
3883
|
-
kind: "link",
|
|
3884
|
-
oldEntryId: sibling.id
|
|
3885
|
-
});
|
|
3886
|
-
}
|
|
3887
|
-
return plans;
|
|
3888
|
-
}
|
|
3889
|
-
function groupPreparedEntriesByClaimKey(preparedEntries) {
|
|
3890
|
-
const grouped = /* @__PURE__ */ new Map();
|
|
3891
|
-
for (const preparedEntry of preparedEntries) {
|
|
3892
|
-
const claimKey = preparedEntry.claimKey?.claim_key ?? preparedEntry.input.claim_key;
|
|
3893
|
-
if (!claimKey) {
|
|
3894
|
-
continue;
|
|
3895
|
-
}
|
|
3896
|
-
const existing = grouped.get(claimKey) ?? [];
|
|
3897
|
-
existing.push(preparedEntry);
|
|
3898
|
-
grouped.set(claimKey, existing);
|
|
3899
|
-
}
|
|
3900
|
-
return grouped;
|
|
3901
|
-
}
|
|
3902
|
-
async function getClaimKeySiblings(db, cache, claimKey) {
|
|
3903
|
-
const cached = cache.get(claimKey);
|
|
3904
|
-
if (cached) {
|
|
3905
|
-
return cached;
|
|
3906
|
-
}
|
|
3907
|
-
const siblings = await db.findActiveEntriesByClaimKey(claimKey);
|
|
3908
|
-
cache.set(claimKey, siblings);
|
|
3909
|
-
return siblings;
|
|
3910
|
-
}
|
|
3911
|
-
function isAutoSupersessionEligible(claimKey, claimExtractionConfig) {
|
|
3912
|
-
if (!claimKey || claimKey.claim_key_status !== "trusted") {
|
|
3913
|
-
return false;
|
|
3914
|
-
}
|
|
3915
|
-
if (claimKey.claim_key_source === "manual") {
|
|
3916
|
-
return true;
|
|
3917
|
-
}
|
|
3918
|
-
if (!AUTO_SUPERSESSION_ELIGIBLE_SOURCES.has(claimKey.claim_key_source) || !claimExtractionConfig) {
|
|
3919
|
-
return false;
|
|
3920
|
-
}
|
|
3921
|
-
return claimKey.claim_key_confidence >= Math.max(claimExtractionConfig.confidenceThreshold, AUTO_SUPERSESSION_MIN_EXTRACTED_CONFIDENCE);
|
|
3922
|
-
}
|
|
3923
|
-
function buildAutoSupersessionEligibilityWarning(preparedEntry) {
|
|
3924
|
-
const acceptedClaimKey = preparedEntry.claimKey;
|
|
3925
|
-
const claimKey = acceptedClaimKey?.claim_key ?? preparedEntry.input.claim_key ?? "(missing)";
|
|
3926
|
-
if (!acceptedClaimKey) {
|
|
3927
|
-
return `Stored entry "${preparedEntry.input.subject}" with claim_key "${claimKey}" but skipped auto-supersession because the claim-key provenance was not explicit or a tracked high-confidence extraction.`;
|
|
3928
|
-
}
|
|
3929
|
-
if (acceptedClaimKey.claim_key_source === "manual") {
|
|
3930
|
-
return `Stored entry "${preparedEntry.input.subject}" with claim_key "${claimKey}" but skipped auto-supersession because the claim-key provenance was not eligible for automatic linking.`;
|
|
3931
|
-
}
|
|
3932
|
-
if (acceptedClaimKey.claim_key_status !== "trusted") {
|
|
3933
|
-
return `Stored entry "${preparedEntry.input.subject}" with claim_key "${claimKey}" but skipped auto-supersession because the accepted claim key is ${acceptedClaimKey.claim_key_status} from ${acceptedClaimKey.claim_key_source} at confidence ${acceptedClaimKey.claim_key_confidence.toFixed(2)}. Only explicit/manual claim keys or model-extracted keys at ${AUTO_SUPERSESSION_MIN_EXTRACTED_CONFIDENCE.toFixed(2)}+ auto-link.`;
|
|
3934
|
-
}
|
|
3935
|
-
return `Stored entry "${preparedEntry.input.subject}" with claim_key "${claimKey}" but skipped auto-supersession because the extracted claim key came from ${acceptedClaimKey.claim_key_source} at confidence ${acceptedClaimKey.claim_key_confidence.toFixed(2)}. Only explicit/manual claim keys or model-extracted keys at ${AUTO_SUPERSESSION_MIN_EXTRACTED_CONFIDENCE.toFixed(2)}+ auto-link.`;
|
|
3936
|
-
}
|
|
3937
|
-
function buildAutoSupersessionRuleWarning(preparedEntry, sibling, reason) {
|
|
3938
|
-
if (reason === "type_mismatch") {
|
|
3939
|
-
return `Stored entry "${preparedEntry.input.subject}" with claim_key "${preparedEntry.input.claim_key}" but skipped auto-supersession because the matching active entry is type "${sibling.type}" and the new entry is type "${preparedEntry.input.type}". ${describeSupersessionRuleFailure(reason)}`;
|
|
3940
|
-
}
|
|
3941
|
-
return `Stored entry "${preparedEntry.input.subject}" with claim_key "${preparedEntry.input.claim_key}" but skipped auto-supersession: ${describeSupersessionRuleFailure(reason)}`;
|
|
3942
|
-
}
|
|
3943
|
-
async function buildStorePlan(inputs, db) {
|
|
3944
|
-
const validation = validateEntriesWithIndexes(inputs);
|
|
3945
|
-
const details = validation.rejectedInputIndexes.map((inputIndex) => ({
|
|
3946
|
-
inputIndex,
|
|
3947
|
-
outcome: "rejected",
|
|
3948
|
-
reason: "validation"
|
|
3949
|
-
}));
|
|
3950
|
-
const preparedEntries = validation.valid.map(({ input, inputIndex }) => ({
|
|
3951
|
-
input,
|
|
3952
|
-
inputIndex,
|
|
3953
|
-
contentHash: computeContentHash(input.content, input.source_file),
|
|
3954
|
-
normContentHash: computeNormContentHash(input.content),
|
|
3955
|
-
claimKey: buildManualAcceptedClaimKey(inputs[inputIndex], input)
|
|
3956
|
-
}));
|
|
3957
|
-
const afterBatchContentHash = dedupePreparedEntries(preparedEntries, "contentHash", "content_hash", details);
|
|
3958
|
-
const existingHashes = await db.findExistingHashes(afterBatchContentHash.map((entry) => entry.contentHash));
|
|
3959
|
-
const afterExistingContentHash = filterExistingPreparedEntries(afterBatchContentHash, existingHashes, "contentHash", "content_hash", details);
|
|
3960
|
-
const afterBatchNormHash = dedupePreparedEntries(afterExistingContentHash, "normContentHash", "norm_content_hash", details);
|
|
3961
|
-
const existingNormHashes = await db.findExistingNormHashes(afterBatchNormHash.map((entry) => entry.normContentHash));
|
|
3962
|
-
const pendingEntries = filterExistingPreparedEntries(afterBatchNormHash, existingNormHashes, "normContentHash", "norm_content_hash", details);
|
|
3963
|
-
return {
|
|
3964
|
-
pendingEntries,
|
|
3965
|
-
skipped: details.filter((detail) => detail.outcome === "skipped").length,
|
|
3966
|
-
rejected: validation.rejected,
|
|
3967
|
-
details,
|
|
3968
|
-
warnings: validation.warnings
|
|
3969
|
-
};
|
|
3970
|
-
}
|
|
3971
|
-
function dedupePreparedEntries(entries, field, reason, details) {
|
|
3972
|
-
const seen = /* @__PURE__ */ new Set();
|
|
3973
|
-
const deduped = [];
|
|
3974
|
-
for (const entry of entries) {
|
|
3975
|
-
const key = entry[field];
|
|
3976
|
-
if (seen.has(key)) {
|
|
3977
|
-
details.push({
|
|
3978
|
-
inputIndex: entry.inputIndex,
|
|
3979
|
-
outcome: "skipped",
|
|
3980
|
-
reason
|
|
3981
|
-
});
|
|
3982
|
-
continue;
|
|
3983
|
-
}
|
|
3984
|
-
seen.add(key);
|
|
3985
|
-
deduped.push(entry);
|
|
3986
|
-
}
|
|
3987
|
-
return deduped;
|
|
3988
|
-
}
|
|
3989
|
-
function filterExistingPreparedEntries(entries, existing, field, reason, details) {
|
|
3990
|
-
return entries.filter((entry) => {
|
|
3991
|
-
if (!existing.has(entry[field])) {
|
|
3992
|
-
return true;
|
|
3993
|
-
}
|
|
3994
|
-
details.push({
|
|
3995
|
-
inputIndex: entry.inputIndex,
|
|
3996
|
-
outcome: "skipped",
|
|
3997
|
-
reason
|
|
3998
|
-
});
|
|
3999
|
-
return false;
|
|
4000
|
-
});
|
|
4001
|
-
}
|
|
4002
|
-
function formatPipelineError(error) {
|
|
4003
|
-
if (error instanceof Error) {
|
|
4004
|
-
return error.message;
|
|
4005
|
-
}
|
|
4006
|
-
return String(error);
|
|
4007
|
-
}
|
|
4008
|
-
function sortStoreDetails(details) {
|
|
4009
|
-
return [...details].sort((left, right) => left.inputIndex - right.inputIndex);
|
|
4010
|
-
}
|
|
4011
|
-
function buildManualAcceptedClaimKey(rawInput, normalizedInput) {
|
|
4012
|
-
const canonicalClaimKey = normalizedInput.claim_key;
|
|
4013
|
-
if (!canonicalClaimKey) {
|
|
4014
|
-
return void 0;
|
|
4015
|
-
}
|
|
4016
|
-
const precomputedAcceptedClaimKey = buildPrecomputedClaimKeyLifecycle(normalizedInput);
|
|
4017
|
-
if (precomputedAcceptedClaimKey) {
|
|
4018
|
-
return precomputedAcceptedClaimKey;
|
|
4019
|
-
}
|
|
4020
|
-
if (rawInput && hasPrecomputedClaimKeyLifecycleFields(rawInput)) {
|
|
4021
|
-
throw new Error("Store inputs with claim-key lifecycle metadata must provide a complete valid lifecycle bundle.");
|
|
4022
|
-
}
|
|
4023
|
-
return buildManualClaimKeyLifecycle({
|
|
4024
|
-
claimKey: canonicalClaimKey,
|
|
4025
|
-
rawClaimKey: normalizedInput.claim_key_raw ?? normalizeOptionalString2(rawInput?.claim_key),
|
|
4026
|
-
supportSourceKind: normalizedInput.claim_support_source_kind,
|
|
4027
|
-
supportLocator: normalizedInput.claim_support_locator,
|
|
4028
|
-
supportObservedAt: normalizedInput.claim_support_observed_at,
|
|
4029
|
-
supportMode: normalizedInput.claim_support_mode
|
|
4030
|
-
});
|
|
4031
|
-
}
|
|
4032
|
-
function normalizeOptionalString2(value) {
|
|
4033
|
-
const normalized = value?.trim();
|
|
4034
|
-
return normalized && normalized.length > 0 ? normalized : void 0;
|
|
4035
|
-
}
|
|
4036
|
-
|
|
4037
2394
|
export {
|
|
4038
2395
|
detectClaimKeyEntityFamilyCandidates,
|
|
4039
2396
|
detectClaimKeySingletonAliasCandidates,
|
|
@@ -4042,7 +2399,7 @@ export {
|
|
|
4042
2399
|
evaluateClaimKeyCompactness,
|
|
4043
2400
|
normalizeGroundingTags,
|
|
4044
2401
|
tokenizeGroundingText,
|
|
4045
|
-
|
|
2402
|
+
buildDurableLocalLexicalTokens,
|
|
4046
2403
|
applyClaimExtractionResultToEntry,
|
|
4047
2404
|
previewClaimKeyExtraction,
|
|
4048
2405
|
runBatchClaimExtraction,
|
|
@@ -4050,30 +2407,10 @@ export {
|
|
|
4050
2407
|
describeSupersessionRuleFailure,
|
|
4051
2408
|
computeContentHash,
|
|
4052
2409
|
computeNormContentHash,
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
EXPIRY_DESCRIPTION,
|
|
4060
|
-
UPDATE_EXPIRY_DESCRIPTION,
|
|
4061
|
-
RECALL_MODES,
|
|
4062
|
-
asRecord,
|
|
4063
|
-
parseExpiry,
|
|
4064
|
-
parseEntryTypes,
|
|
4065
|
-
parseEntryType,
|
|
4066
|
-
parseRecallMode,
|
|
4067
|
-
normalizeStringArray,
|
|
4068
|
-
formatTargetSelector,
|
|
4069
|
-
formatTargetSelectorFromParams,
|
|
4070
|
-
sanitizeUpdateToolParams,
|
|
4071
|
-
sanitizeFetchToolParams,
|
|
4072
|
-
backfillEpisodeEmbeddings,
|
|
4073
|
-
prepareEpisodeIngest,
|
|
4074
|
-
ingestEpisodeTranscript,
|
|
4075
|
-
executeEpisodeIngestPlan,
|
|
4076
|
-
createEpisodeIngestPlan,
|
|
4077
|
-
createSingleTranscriptDiscoveryPort,
|
|
4078
|
-
createMemoryRepository
|
|
2410
|
+
resolveDurableProjectScope,
|
|
2411
|
+
tryAcquireDreamingRunLock,
|
|
2412
|
+
withDreamingRunLock,
|
|
2413
|
+
withHeldDreamingRunLock,
|
|
2414
|
+
withEpisodeWriteGuard,
|
|
2415
|
+
isEpisodeWriteInProgress
|
|
4079
2416
|
};
|