@femtomc/mu-core 26.2.75 → 26.2.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/idb.js +1 -1
- package/dist/node/index.d.ts +2 -1
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +2 -1
- package/dist/node/memory_runtime.d.ts +130 -0
- package/dist/node/memory_runtime.d.ts.map +1 -0
- package/dist/node/memory_runtime.js +1918 -0
- package/dist/node/store.d.ts +4 -0
- package/dist/node/store.d.ts.map +1 -1
- package/dist/node/store.js +41 -5
- package/package.json +1 -1
|
@@ -0,0 +1,1918 @@
|
|
|
1
|
+
import { existsSync, createReadStream } from "node:fs";
|
|
2
|
+
import { mkdir, readdir, readFile, stat } from "node:fs/promises";
|
|
3
|
+
import { join, relative } from "node:path";
|
|
4
|
+
import { createInterface } from "node:readline";
|
|
5
|
+
import { Database } from "bun:sqlite";
|
|
6
|
+
import { getStorePaths } from "./store.js";
|
|
7
|
+
const DEFAULT_LIMIT = 40;
|
|
8
|
+
const MAX_LIMIT = 500;
|
|
9
|
+
const MAX_TEXT_LENGTH = 8_000;
|
|
10
|
+
const PREVIEW_LENGTH = 240;
|
|
11
|
+
const CONTEXT_INDEX_SCHEMA_VERSION = 1;
|
|
12
|
+
const CONTEXT_INDEX_FILENAME = "memory.db";
|
|
13
|
+
const INDEX_QUERY_ROW_LIMIT = 100_000;
|
|
14
|
+
const INDEX_FTS_ROW_LIMIT = 100_000;
|
|
15
|
+
const AUTO_INDEX_REBUILD_IN_FLIGHT = new Map();
|
|
16
|
+
export const CONTEXT_SOURCE_KINDS = [
|
|
17
|
+
"issues",
|
|
18
|
+
"forum",
|
|
19
|
+
"events",
|
|
20
|
+
"cp_commands",
|
|
21
|
+
"cp_outbox",
|
|
22
|
+
"cp_adapter_audit",
|
|
23
|
+
"cp_operator_turns",
|
|
24
|
+
"cp_telegram_ingress",
|
|
25
|
+
"session_flash",
|
|
26
|
+
"operator_sessions",
|
|
27
|
+
"cp_operator_sessions",
|
|
28
|
+
];
|
|
29
|
+
export class ContextQueryValidationError extends Error {
|
|
30
|
+
status = 400;
|
|
31
|
+
constructor(message) {
|
|
32
|
+
super(message);
|
|
33
|
+
this.name = "ContextQueryValidationError";
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function asRecord(value) {
|
|
37
|
+
if (typeof value !== "object" || value == null || Array.isArray(value)) {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
return value;
|
|
41
|
+
}
|
|
42
|
+
function nonEmptyString(value) {
|
|
43
|
+
if (typeof value !== "string") {
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
const trimmed = value.trim();
|
|
47
|
+
return trimmed.length > 0 ? trimmed : null;
|
|
48
|
+
}
|
|
49
|
+
function asInt(value) {
|
|
50
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
return Math.trunc(value);
|
|
54
|
+
}
|
|
55
|
+
function parseTimestamp(value) {
|
|
56
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
57
|
+
return Math.trunc(value);
|
|
58
|
+
}
|
|
59
|
+
if (typeof value === "string") {
|
|
60
|
+
const parsed = new Date(value);
|
|
61
|
+
if (!Number.isNaN(parsed.getTime())) {
|
|
62
|
+
return Math.trunc(parsed.getTime());
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (value instanceof Date) {
|
|
66
|
+
const ms = value.getTime();
|
|
67
|
+
if (!Number.isNaN(ms)) {
|
|
68
|
+
return Math.trunc(ms);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
function toSingleLine(value) {
|
|
74
|
+
return value.replace(/\s+/g, " ").trim();
|
|
75
|
+
}
|
|
76
|
+
function clampText(value, maxLength) {
|
|
77
|
+
if (value.length <= maxLength) {
|
|
78
|
+
return value;
|
|
79
|
+
}
|
|
80
|
+
if (maxLength <= 1) {
|
|
81
|
+
return value.slice(0, Math.max(0, maxLength));
|
|
82
|
+
}
|
|
83
|
+
return `${value.slice(0, maxLength - 1)}…`;
|
|
84
|
+
}
|
|
85
|
+
function buildPreview(text) {
|
|
86
|
+
const normalized = toSingleLine(text);
|
|
87
|
+
if (normalized.length === 0) {
|
|
88
|
+
return "";
|
|
89
|
+
}
|
|
90
|
+
return clampText(normalized, PREVIEW_LENGTH);
|
|
91
|
+
}
|
|
92
|
+
function textFromUnknown(value) {
|
|
93
|
+
if (typeof value === "string") {
|
|
94
|
+
return value;
|
|
95
|
+
}
|
|
96
|
+
if (value == null) {
|
|
97
|
+
return "";
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
return JSON.stringify(value);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return String(value);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function buildConversationKey(parts) {
|
|
107
|
+
if (!parts.channel || !parts.tenantId || !parts.conversationId || !parts.bindingId) {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
return `${parts.channel}:${parts.tenantId}:${parts.conversationId}:${parts.bindingId}`;
|
|
111
|
+
}
|
|
112
|
+
function conversationScopeKey(key) {
|
|
113
|
+
const parts = key.split(":");
|
|
114
|
+
if (parts.length < 3) {
|
|
115
|
+
return key;
|
|
116
|
+
}
|
|
117
|
+
return `${parts[0]}:${parts[1]}:${parts[2]}`;
|
|
118
|
+
}
|
|
119
|
+
function parseCsv(value) {
|
|
120
|
+
if (!value) {
|
|
121
|
+
return [];
|
|
122
|
+
}
|
|
123
|
+
return value
|
|
124
|
+
.split(",")
|
|
125
|
+
.map((part) => part.trim())
|
|
126
|
+
.filter((part) => part.length > 0);
|
|
127
|
+
}
|
|
128
|
+
function parseLimit(value) {
|
|
129
|
+
if (value == null || value.trim().length === 0) {
|
|
130
|
+
return DEFAULT_LIMIT;
|
|
131
|
+
}
|
|
132
|
+
const parsed = Number.parseInt(value, 10);
|
|
133
|
+
if (!Number.isFinite(parsed)) {
|
|
134
|
+
throw new ContextQueryValidationError("invalid limit: expected integer");
|
|
135
|
+
}
|
|
136
|
+
if (parsed < 1) {
|
|
137
|
+
throw new ContextQueryValidationError("invalid limit: must be >= 1");
|
|
138
|
+
}
|
|
139
|
+
return Math.min(parsed, MAX_LIMIT);
|
|
140
|
+
}
|
|
141
|
+
function parseOptionalTs(value, name) {
|
|
142
|
+
if (value == null || value.trim().length === 0) {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
const parsed = Number.parseInt(value, 10);
|
|
146
|
+
if (!Number.isFinite(parsed)) {
|
|
147
|
+
throw new ContextQueryValidationError(`invalid ${name}: expected integer epoch ms`);
|
|
148
|
+
}
|
|
149
|
+
return parsed;
|
|
150
|
+
}
|
|
151
|
+
function parseSourceFilter(value) {
|
|
152
|
+
const parts = parseCsv(value);
|
|
153
|
+
if (parts.length === 0) {
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
const out = new Set();
|
|
157
|
+
for (const part of parts) {
|
|
158
|
+
const normalized = part.trim().toLowerCase();
|
|
159
|
+
if (normalized === "all") {
|
|
160
|
+
for (const source of CONTEXT_SOURCE_KINDS) {
|
|
161
|
+
out.add(source);
|
|
162
|
+
}
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
if (CONTEXT_SOURCE_KINDS.includes(normalized)) {
|
|
166
|
+
out.add(normalized);
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
throw new ContextQueryValidationError(`unknown memory source: ${part}. valid sources: ${CONTEXT_SOURCE_KINDS.join(", ")}`);
|
|
170
|
+
}
|
|
171
|
+
return out;
|
|
172
|
+
}
|
|
173
|
+
function parseSearchFilters(url) {
|
|
174
|
+
const query = nonEmptyString(url.searchParams.get("query")) ??
|
|
175
|
+
nonEmptyString(url.searchParams.get("q")) ??
|
|
176
|
+
nonEmptyString(url.searchParams.get("contains"));
|
|
177
|
+
return {
|
|
178
|
+
query,
|
|
179
|
+
sources: parseSourceFilter(url.searchParams.get("sources") ?? url.searchParams.get("source")),
|
|
180
|
+
limit: parseLimit(url.searchParams.get("limit")),
|
|
181
|
+
sinceMs: parseOptionalTs(url.searchParams.get("since"), "since"),
|
|
182
|
+
untilMs: parseOptionalTs(url.searchParams.get("until"), "until"),
|
|
183
|
+
issueId: nonEmptyString(url.searchParams.get("issue_id")),
|
|
184
|
+
runId: nonEmptyString(url.searchParams.get("run_id")),
|
|
185
|
+
sessionId: nonEmptyString(url.searchParams.get("session_id")),
|
|
186
|
+
conversationKey: nonEmptyString(url.searchParams.get("conversation_key")),
|
|
187
|
+
channel: nonEmptyString(url.searchParams.get("channel")),
|
|
188
|
+
channelTenantId: nonEmptyString(url.searchParams.get("channel_tenant_id")),
|
|
189
|
+
channelConversationId: nonEmptyString(url.searchParams.get("channel_conversation_id")),
|
|
190
|
+
actorBindingId: nonEmptyString(url.searchParams.get("actor_binding_id")),
|
|
191
|
+
topic: nonEmptyString(url.searchParams.get("topic")),
|
|
192
|
+
author: nonEmptyString(url.searchParams.get("author")),
|
|
193
|
+
role: nonEmptyString(url.searchParams.get("role")),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
function parseTimelineFilters(url) {
|
|
197
|
+
const base = parseSearchFilters(url);
|
|
198
|
+
const orderRaw = nonEmptyString(url.searchParams.get("order"))?.toLowerCase();
|
|
199
|
+
const order = orderRaw === "desc" ? "desc" : "asc";
|
|
200
|
+
if (!base.conversationKey &&
|
|
201
|
+
!base.issueId &&
|
|
202
|
+
!base.runId &&
|
|
203
|
+
!base.sessionId &&
|
|
204
|
+
!base.topic &&
|
|
205
|
+
!base.channel) {
|
|
206
|
+
throw new ContextQueryValidationError("timeline requires one anchor filter: conversation_key, issue_id, run_id, session_id, topic, or channel");
|
|
207
|
+
}
|
|
208
|
+
return { ...base, order };
|
|
209
|
+
}
|
|
210
|
+
function matchesConversation(item, requested) {
|
|
211
|
+
const requestedTrimmed = requested.trim();
|
|
212
|
+
if (requestedTrimmed.length === 0) {
|
|
213
|
+
return true;
|
|
214
|
+
}
|
|
215
|
+
const direct = item.conversation_key ? [item.conversation_key] : [];
|
|
216
|
+
const metadataKeys = Array.isArray(item.metadata.conversation_keys)
|
|
217
|
+
? item.metadata.conversation_keys.filter((v) => typeof v === "string" && v.trim().length > 0)
|
|
218
|
+
: [];
|
|
219
|
+
const allKeys = [...direct, ...metadataKeys];
|
|
220
|
+
if (allKeys.length === 0) {
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
223
|
+
if (requestedTrimmed.includes("*")) {
|
|
224
|
+
const prefix = requestedTrimmed.replace(/\*/g, "");
|
|
225
|
+
return allKeys.some((key) => key.startsWith(prefix));
|
|
226
|
+
}
|
|
227
|
+
if (allKeys.includes(requestedTrimmed)) {
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
230
|
+
const requestedScope = conversationScopeKey(requestedTrimmed);
|
|
231
|
+
return allKeys.some((key) => conversationScopeKey(key) === requestedScope);
|
|
232
|
+
}
|
|
233
|
+
function matchSource(item, sources) {
|
|
234
|
+
if (!sources) {
|
|
235
|
+
return true;
|
|
236
|
+
}
|
|
237
|
+
return sources.has(item.source_kind);
|
|
238
|
+
}
|
|
239
|
+
function matchSearchFilters(item, filters) {
|
|
240
|
+
if (!matchSource(item, filters.sources)) {
|
|
241
|
+
return false;
|
|
242
|
+
}
|
|
243
|
+
if (filters.sinceMs != null && item.ts_ms < filters.sinceMs) {
|
|
244
|
+
return false;
|
|
245
|
+
}
|
|
246
|
+
if (filters.untilMs != null && item.ts_ms > filters.untilMs) {
|
|
247
|
+
return false;
|
|
248
|
+
}
|
|
249
|
+
if (filters.issueId && item.issue_id !== filters.issueId) {
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
if (filters.runId && item.run_id !== filters.runId) {
|
|
253
|
+
return false;
|
|
254
|
+
}
|
|
255
|
+
if (filters.sessionId && item.session_id !== filters.sessionId) {
|
|
256
|
+
return false;
|
|
257
|
+
}
|
|
258
|
+
if (filters.channel && item.channel !== filters.channel) {
|
|
259
|
+
return false;
|
|
260
|
+
}
|
|
261
|
+
if (filters.channelTenantId && item.channel_tenant_id !== filters.channelTenantId) {
|
|
262
|
+
return false;
|
|
263
|
+
}
|
|
264
|
+
if (filters.channelConversationId && item.channel_conversation_id !== filters.channelConversationId) {
|
|
265
|
+
return false;
|
|
266
|
+
}
|
|
267
|
+
if (filters.actorBindingId && item.actor_binding_id !== filters.actorBindingId) {
|
|
268
|
+
return false;
|
|
269
|
+
}
|
|
270
|
+
if (filters.topic && item.topic !== filters.topic) {
|
|
271
|
+
return false;
|
|
272
|
+
}
|
|
273
|
+
if (filters.author && item.author !== filters.author) {
|
|
274
|
+
return false;
|
|
275
|
+
}
|
|
276
|
+
if (filters.role && item.role !== filters.role) {
|
|
277
|
+
return false;
|
|
278
|
+
}
|
|
279
|
+
if (filters.conversationKey && !matchesConversation(item, filters.conversationKey)) {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
return true;
|
|
283
|
+
}
|
|
284
|
+
function tokenizeQuery(query) {
|
|
285
|
+
return query
|
|
286
|
+
.toLowerCase()
|
|
287
|
+
.split(/\s+/)
|
|
288
|
+
.map((part) => part.trim())
|
|
289
|
+
.filter((part) => part.length > 0);
|
|
290
|
+
}
|
|
291
|
+
function searchableText(item) {
|
|
292
|
+
const tags = item.tags.join(" ");
|
|
293
|
+
const fields = [
|
|
294
|
+
item.text,
|
|
295
|
+
item.preview,
|
|
296
|
+
item.source_kind,
|
|
297
|
+
item.issue_id ?? "",
|
|
298
|
+
item.run_id ?? "",
|
|
299
|
+
item.session_id ?? "",
|
|
300
|
+
item.channel ?? "",
|
|
301
|
+
item.topic ?? "",
|
|
302
|
+
item.author ?? "",
|
|
303
|
+
tags,
|
|
304
|
+
];
|
|
305
|
+
return fields.join("\n").toLowerCase();
|
|
306
|
+
}
|
|
307
|
+
function scoreItem(item, query) {
|
|
308
|
+
if (!query) {
|
|
309
|
+
return item.ts_ms;
|
|
310
|
+
}
|
|
311
|
+
const haystack = searchableText(item);
|
|
312
|
+
const needle = query.toLowerCase();
|
|
313
|
+
const tokens = tokenizeQuery(needle);
|
|
314
|
+
let score = 0;
|
|
315
|
+
if (haystack.includes(needle)) {
|
|
316
|
+
score += 100;
|
|
317
|
+
}
|
|
318
|
+
let tokenHits = 0;
|
|
319
|
+
for (const token of tokens) {
|
|
320
|
+
if (haystack.includes(token)) {
|
|
321
|
+
tokenHits += 1;
|
|
322
|
+
score += 20;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
if (tokens.length > 0 && tokenHits === 0) {
|
|
326
|
+
return -1;
|
|
327
|
+
}
|
|
328
|
+
if (item.role === "user") {
|
|
329
|
+
score += 8;
|
|
330
|
+
}
|
|
331
|
+
if (item.role === "assistant") {
|
|
332
|
+
score += 4;
|
|
333
|
+
}
|
|
334
|
+
if (item.source_kind === "cp_operator_sessions" || item.source_kind === "operator_sessions") {
|
|
335
|
+
score += 5;
|
|
336
|
+
}
|
|
337
|
+
const ageMs = Math.max(0, Date.now() - item.ts_ms);
|
|
338
|
+
const recencyBonus = Math.max(0, 24 - Math.trunc(ageMs / (1000 * 60 * 60 * 24)));
|
|
339
|
+
score += recencyBonus;
|
|
340
|
+
return score;
|
|
341
|
+
}
|
|
342
|
+
function isErrnoCode(err, code) {
|
|
343
|
+
return Boolean(err && typeof err === "object" && "code" in err && err.code === code);
|
|
344
|
+
}
|
|
345
|
+
async function readJsonlRows(path) {
|
|
346
|
+
const rows = [];
|
|
347
|
+
let stream = null;
|
|
348
|
+
let rl = null;
|
|
349
|
+
try {
|
|
350
|
+
stream = createReadStream(path, { encoding: "utf8" });
|
|
351
|
+
rl = createInterface({ input: stream, crlfDelay: Number.POSITIVE_INFINITY });
|
|
352
|
+
let line = 0;
|
|
353
|
+
for await (const raw of rl) {
|
|
354
|
+
line += 1;
|
|
355
|
+
const trimmed = raw.trim();
|
|
356
|
+
if (trimmed.length === 0) {
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
try {
|
|
360
|
+
rows.push({ line, value: JSON.parse(trimmed) });
|
|
361
|
+
}
|
|
362
|
+
catch {
|
|
363
|
+
// Keep malformed JSON rows non-fatal for retrieval.
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
return rows;
|
|
367
|
+
}
|
|
368
|
+
catch (err) {
|
|
369
|
+
if (isErrnoCode(err, "ENOENT")) {
|
|
370
|
+
return [];
|
|
371
|
+
}
|
|
372
|
+
throw err;
|
|
373
|
+
}
|
|
374
|
+
finally {
|
|
375
|
+
rl?.close();
|
|
376
|
+
stream?.close();
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
async function listJsonlFiles(dir) {
|
|
380
|
+
try {
|
|
381
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
382
|
+
return entries
|
|
383
|
+
.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
|
|
384
|
+
.map((entry) => join(dir, entry.name))
|
|
385
|
+
.sort((a, b) => a.localeCompare(b));
|
|
386
|
+
}
|
|
387
|
+
catch (err) {
|
|
388
|
+
if (isErrnoCode(err, "ENOENT")) {
|
|
389
|
+
return [];
|
|
390
|
+
}
|
|
391
|
+
throw err;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
function extractMessageText(messageRaw) {
|
|
395
|
+
if (typeof messageRaw === "string") {
|
|
396
|
+
return messageRaw;
|
|
397
|
+
}
|
|
398
|
+
const message = asRecord(messageRaw);
|
|
399
|
+
if (!message) {
|
|
400
|
+
return "";
|
|
401
|
+
}
|
|
402
|
+
if (typeof message.content === "string") {
|
|
403
|
+
return message.content;
|
|
404
|
+
}
|
|
405
|
+
if (Array.isArray(message.content)) {
|
|
406
|
+
const chunks = [];
|
|
407
|
+
for (const itemRaw of message.content) {
|
|
408
|
+
const item = asRecord(itemRaw);
|
|
409
|
+
if (!item) {
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
const text = nonEmptyString(item.text);
|
|
413
|
+
if (text) {
|
|
414
|
+
chunks.push(text);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
if (chunks.length > 0) {
|
|
418
|
+
return chunks.join("\n");
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
const text = nonEmptyString(message.text);
|
|
422
|
+
if (text) {
|
|
423
|
+
return text;
|
|
424
|
+
}
|
|
425
|
+
return "";
|
|
426
|
+
}
|
|
427
|
+
async function loadConversationBindings(path) {
|
|
428
|
+
const out = new Map();
|
|
429
|
+
let raw = "";
|
|
430
|
+
try {
|
|
431
|
+
raw = await readFile(path, "utf8");
|
|
432
|
+
}
|
|
433
|
+
catch (err) {
|
|
434
|
+
if (isErrnoCode(err, "ENOENT")) {
|
|
435
|
+
return out;
|
|
436
|
+
}
|
|
437
|
+
throw err;
|
|
438
|
+
}
|
|
439
|
+
const parsed = asRecord(JSON.parse(raw));
|
|
440
|
+
const bindings = parsed ? asRecord(parsed.bindings) : null;
|
|
441
|
+
if (!bindings) {
|
|
442
|
+
return out;
|
|
443
|
+
}
|
|
444
|
+
for (const [conversationKey, sessionIdRaw] of Object.entries(bindings)) {
|
|
445
|
+
const sessionId = nonEmptyString(sessionIdRaw);
|
|
446
|
+
if (conversationKey.trim().length === 0 || !sessionId) {
|
|
447
|
+
continue;
|
|
448
|
+
}
|
|
449
|
+
out.set(conversationKey, sessionId);
|
|
450
|
+
}
|
|
451
|
+
return out;
|
|
452
|
+
}
|
|
453
|
+
function reverseConversationBindings(bindings) {
|
|
454
|
+
const out = new Map();
|
|
455
|
+
for (const [conversationKey, sessionId] of bindings.entries()) {
|
|
456
|
+
const rows = out.get(sessionId) ?? [];
|
|
457
|
+
rows.push(conversationKey);
|
|
458
|
+
out.set(sessionId, rows);
|
|
459
|
+
}
|
|
460
|
+
for (const values of out.values()) {
|
|
461
|
+
values.sort((a, b) => a.localeCompare(b));
|
|
462
|
+
}
|
|
463
|
+
return out;
|
|
464
|
+
}
|
|
465
|
+
function normalizeRelative(repoRoot, path) {
|
|
466
|
+
return relative(repoRoot, path).replaceAll("\\", "/");
|
|
467
|
+
}
|
|
468
|
+
function pushItem(out, item) {
|
|
469
|
+
const trimmed = item.text.trim();
|
|
470
|
+
if (trimmed.length === 0) {
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
const text = clampText(trimmed, MAX_TEXT_LENGTH);
|
|
474
|
+
out.push({
|
|
475
|
+
...item,
|
|
476
|
+
text,
|
|
477
|
+
preview: buildPreview(text),
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
async function collectIssues(repoRoot, path) {
|
|
481
|
+
const out = [];
|
|
482
|
+
for (const row of await readJsonlRows(path)) {
|
|
483
|
+
const rec = asRecord(row.value);
|
|
484
|
+
if (!rec) {
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
const issueId = nonEmptyString(rec.id);
|
|
488
|
+
const title = nonEmptyString(rec.title) ?? "";
|
|
489
|
+
const body = nonEmptyString(rec.body) ?? "";
|
|
490
|
+
const status = nonEmptyString(rec.status) ?? "unknown";
|
|
491
|
+
const tags = Array.isArray(rec.tags)
|
|
492
|
+
? rec.tags.filter((tag) => typeof tag === "string" && tag.trim().length > 0)
|
|
493
|
+
: [];
|
|
494
|
+
if (!issueId) {
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
497
|
+
pushItem(out, {
|
|
498
|
+
id: `issues:${issueId}:${row.line}`,
|
|
499
|
+
ts_ms: asInt(rec.updated_at) ?? asInt(rec.created_at) ?? 0,
|
|
500
|
+
source_kind: "issues",
|
|
501
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
502
|
+
source_line: row.line,
|
|
503
|
+
repo_root: repoRoot,
|
|
504
|
+
issue_id: issueId,
|
|
505
|
+
run_id: null,
|
|
506
|
+
session_id: null,
|
|
507
|
+
channel: null,
|
|
508
|
+
channel_tenant_id: null,
|
|
509
|
+
channel_conversation_id: null,
|
|
510
|
+
actor_binding_id: null,
|
|
511
|
+
conversation_key: null,
|
|
512
|
+
topic: null,
|
|
513
|
+
author: null,
|
|
514
|
+
role: null,
|
|
515
|
+
tags: ["issue", status, ...tags],
|
|
516
|
+
metadata: {
|
|
517
|
+
status,
|
|
518
|
+
priority: asInt(rec.priority),
|
|
519
|
+
outcome: rec.outcome ?? null,
|
|
520
|
+
},
|
|
521
|
+
text: [title, body, tags.join(" ")].filter((part) => part.length > 0).join("\n"),
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
return out;
|
|
525
|
+
}
|
|
526
|
+
async function collectForum(repoRoot, path) {
|
|
527
|
+
const out = [];
|
|
528
|
+
for (const row of await readJsonlRows(path)) {
|
|
529
|
+
const rec = asRecord(row.value);
|
|
530
|
+
if (!rec) {
|
|
531
|
+
continue;
|
|
532
|
+
}
|
|
533
|
+
const topic = nonEmptyString(rec.topic);
|
|
534
|
+
const body = nonEmptyString(rec.body) ?? "";
|
|
535
|
+
const author = nonEmptyString(rec.author);
|
|
536
|
+
if (!topic) {
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
pushItem(out, {
|
|
540
|
+
id: `forum:${topic}:${row.line}`,
|
|
541
|
+
ts_ms: asInt(rec.created_at) ?? 0,
|
|
542
|
+
source_kind: "forum",
|
|
543
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
544
|
+
source_line: row.line,
|
|
545
|
+
repo_root: repoRoot,
|
|
546
|
+
issue_id: topic.startsWith("issue:") ? topic.slice("issue:".length) : null,
|
|
547
|
+
run_id: null,
|
|
548
|
+
session_id: null,
|
|
549
|
+
channel: null,
|
|
550
|
+
channel_tenant_id: null,
|
|
551
|
+
channel_conversation_id: null,
|
|
552
|
+
actor_binding_id: null,
|
|
553
|
+
conversation_key: null,
|
|
554
|
+
topic,
|
|
555
|
+
author,
|
|
556
|
+
role: null,
|
|
557
|
+
tags: ["forum", topic],
|
|
558
|
+
metadata: {},
|
|
559
|
+
text: [topic, author ?? "", body].filter((part) => part.length > 0).join("\n"),
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
return out;
|
|
563
|
+
}
|
|
564
|
+
async function collectEvents(repoRoot, path) {
|
|
565
|
+
const out = [];
|
|
566
|
+
for (const row of await readJsonlRows(path)) {
|
|
567
|
+
const rec = asRecord(row.value);
|
|
568
|
+
if (!rec) {
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
const type = nonEmptyString(rec.type) ?? "event";
|
|
572
|
+
const source = nonEmptyString(rec.source) ?? "unknown";
|
|
573
|
+
const payload = rec.payload ?? null;
|
|
574
|
+
const payloadText = textFromUnknown(payload);
|
|
575
|
+
const issueId = nonEmptyString(rec.issue_id);
|
|
576
|
+
const runId = nonEmptyString(rec.run_id);
|
|
577
|
+
pushItem(out, {
|
|
578
|
+
id: `events:${type}:${row.line}`,
|
|
579
|
+
ts_ms: asInt(rec.ts_ms) ?? 0,
|
|
580
|
+
source_kind: "events",
|
|
581
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
582
|
+
source_line: row.line,
|
|
583
|
+
repo_root: repoRoot,
|
|
584
|
+
issue_id: issueId,
|
|
585
|
+
run_id: runId,
|
|
586
|
+
session_id: null,
|
|
587
|
+
channel: null,
|
|
588
|
+
channel_tenant_id: null,
|
|
589
|
+
channel_conversation_id: null,
|
|
590
|
+
actor_binding_id: null,
|
|
591
|
+
conversation_key: null,
|
|
592
|
+
topic: null,
|
|
593
|
+
author: null,
|
|
594
|
+
role: null,
|
|
595
|
+
tags: ["event", type, source],
|
|
596
|
+
metadata: {
|
|
597
|
+
type,
|
|
598
|
+
source,
|
|
599
|
+
},
|
|
600
|
+
text: [type, source, issueId ?? "", runId ?? "", payloadText]
|
|
601
|
+
.filter((part) => part.length > 0)
|
|
602
|
+
.join("\n"),
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
return out;
|
|
606
|
+
}
|
|
607
|
+
async function collectCommandJournal(repoRoot, path) {
|
|
608
|
+
const out = [];
|
|
609
|
+
for (const row of await readJsonlRows(path)) {
|
|
610
|
+
const rec = asRecord(row.value);
|
|
611
|
+
if (!rec) {
|
|
612
|
+
continue;
|
|
613
|
+
}
|
|
614
|
+
const kind = nonEmptyString(rec.kind) ?? "unknown";
|
|
615
|
+
if (kind === "command.lifecycle") {
|
|
616
|
+
const command = asRecord(rec.command);
|
|
617
|
+
const commandId = command ? nonEmptyString(command.command_id) : null;
|
|
618
|
+
const commandText = command ? nonEmptyString(command.command_text) : null;
|
|
619
|
+
const state = command ? nonEmptyString(command.state) : null;
|
|
620
|
+
const targetType = command ? nonEmptyString(command.target_type) : null;
|
|
621
|
+
const targetId = command ? nonEmptyString(command.target_id) : null;
|
|
622
|
+
const channel = command ? nonEmptyString(command.channel) : null;
|
|
623
|
+
const channelTenantId = command ? nonEmptyString(command.channel_tenant_id) : null;
|
|
624
|
+
const channelConversationId = command ? nonEmptyString(command.channel_conversation_id) : null;
|
|
625
|
+
const actorBindingId = command ? nonEmptyString(command.actor_binding_id) : null;
|
|
626
|
+
const runId = command ? nonEmptyString(command.run_root_id) : null;
|
|
627
|
+
const sessionId = command
|
|
628
|
+
? nonEmptyString(command.operator_session_id) ?? nonEmptyString(command.meta_session_id)
|
|
629
|
+
: null;
|
|
630
|
+
const conversationKey = buildConversationKey({
|
|
631
|
+
channel,
|
|
632
|
+
tenantId: channelTenantId,
|
|
633
|
+
conversationId: channelConversationId,
|
|
634
|
+
bindingId: actorBindingId,
|
|
635
|
+
});
|
|
636
|
+
const eventType = nonEmptyString(rec.event_type) ?? "command.lifecycle";
|
|
637
|
+
pushItem(out, {
|
|
638
|
+
id: `cp_commands:lifecycle:${commandId ?? row.line}`,
|
|
639
|
+
ts_ms: asInt(rec.ts_ms) ?? (command ? asInt(command.updated_at_ms) : null) ?? 0,
|
|
640
|
+
source_kind: "cp_commands",
|
|
641
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
642
|
+
source_line: row.line,
|
|
643
|
+
repo_root: repoRoot,
|
|
644
|
+
issue_id: null,
|
|
645
|
+
run_id: runId,
|
|
646
|
+
session_id: sessionId,
|
|
647
|
+
channel,
|
|
648
|
+
channel_tenant_id: channelTenantId,
|
|
649
|
+
channel_conversation_id: channelConversationId,
|
|
650
|
+
actor_binding_id: actorBindingId,
|
|
651
|
+
conversation_key: conversationKey,
|
|
652
|
+
topic: null,
|
|
653
|
+
author: null,
|
|
654
|
+
role: null,
|
|
655
|
+
tags: ["cp", "command.lifecycle", state ?? "unknown"],
|
|
656
|
+
metadata: {
|
|
657
|
+
kind,
|
|
658
|
+
event_type: eventType,
|
|
659
|
+
command_id: commandId,
|
|
660
|
+
state,
|
|
661
|
+
target_type: targetType,
|
|
662
|
+
target_id: targetId,
|
|
663
|
+
},
|
|
664
|
+
text: [eventType, commandText ?? "", targetType ?? "", targetId ?? "", state ?? ""]
|
|
665
|
+
.filter((part) => part.length > 0)
|
|
666
|
+
.join("\n"),
|
|
667
|
+
});
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
if (kind === "domain.mutating") {
|
|
671
|
+
const correlation = asRecord(rec.correlation);
|
|
672
|
+
const eventType = nonEmptyString(rec.event_type) ?? "domain.mutating";
|
|
673
|
+
const channel = correlation ? nonEmptyString(correlation.channel) : null;
|
|
674
|
+
const channelTenantId = correlation ? nonEmptyString(correlation.channel_tenant_id) : null;
|
|
675
|
+
const channelConversationId = correlation ? nonEmptyString(correlation.channel_conversation_id) : null;
|
|
676
|
+
const actorBindingId = correlation ? nonEmptyString(correlation.actor_binding_id) : null;
|
|
677
|
+
const runId = correlation ? nonEmptyString(correlation.run_root_id) : null;
|
|
678
|
+
const sessionId = correlation
|
|
679
|
+
? nonEmptyString(correlation.operator_session_id) ?? nonEmptyString(correlation.meta_session_id)
|
|
680
|
+
: null;
|
|
681
|
+
const conversationKey = buildConversationKey({
|
|
682
|
+
channel,
|
|
683
|
+
tenantId: channelTenantId,
|
|
684
|
+
conversationId: channelConversationId,
|
|
685
|
+
bindingId: actorBindingId,
|
|
686
|
+
});
|
|
687
|
+
const payload = rec.payload ?? null;
|
|
688
|
+
pushItem(out, {
|
|
689
|
+
id: `cp_commands:mutating:${row.line}`,
|
|
690
|
+
ts_ms: asInt(rec.ts_ms) ?? 0,
|
|
691
|
+
source_kind: "cp_commands",
|
|
692
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
693
|
+
source_line: row.line,
|
|
694
|
+
repo_root: repoRoot,
|
|
695
|
+
issue_id: null,
|
|
696
|
+
run_id: runId,
|
|
697
|
+
session_id: sessionId,
|
|
698
|
+
channel,
|
|
699
|
+
channel_tenant_id: channelTenantId,
|
|
700
|
+
channel_conversation_id: channelConversationId,
|
|
701
|
+
actor_binding_id: actorBindingId,
|
|
702
|
+
conversation_key: conversationKey,
|
|
703
|
+
topic: null,
|
|
704
|
+
author: null,
|
|
705
|
+
role: null,
|
|
706
|
+
tags: ["cp", "domain.mutating", eventType],
|
|
707
|
+
metadata: {
|
|
708
|
+
kind,
|
|
709
|
+
event_type: eventType,
|
|
710
|
+
},
|
|
711
|
+
text: [eventType, textFromUnknown(payload)].filter((part) => part.length > 0).join("\n"),
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
return out;
|
|
716
|
+
}
|
|
717
|
+
async function collectOutbox(repoRoot, path) {
|
|
718
|
+
const out = [];
|
|
719
|
+
for (const row of await readJsonlRows(path)) {
|
|
720
|
+
const rec = asRecord(row.value);
|
|
721
|
+
if (!rec || nonEmptyString(rec.kind) !== "outbox.state") {
|
|
722
|
+
continue;
|
|
723
|
+
}
|
|
724
|
+
const record = asRecord(rec.record);
|
|
725
|
+
const envelope = record ? asRecord(record.envelope) : null;
|
|
726
|
+
const correlation = envelope ? asRecord(envelope.correlation) : null;
|
|
727
|
+
const outboxId = record ? nonEmptyString(record.outbox_id) : null;
|
|
728
|
+
const channel = envelope ? nonEmptyString(envelope.channel) : null;
|
|
729
|
+
const channelTenantId = envelope ? nonEmptyString(envelope.channel_tenant_id) : null;
|
|
730
|
+
const channelConversationId = envelope ? nonEmptyString(envelope.channel_conversation_id) : null;
|
|
731
|
+
const actorBindingId = correlation ? nonEmptyString(correlation.actor_binding_id) : null;
|
|
732
|
+
const conversationKey = buildConversationKey({
|
|
733
|
+
channel,
|
|
734
|
+
tenantId: channelTenantId,
|
|
735
|
+
conversationId: channelConversationId,
|
|
736
|
+
bindingId: actorBindingId,
|
|
737
|
+
});
|
|
738
|
+
const runId = correlation ? nonEmptyString(correlation.run_root_id) : null;
|
|
739
|
+
const sessionId = correlation
|
|
740
|
+
? nonEmptyString(correlation.operator_session_id) ?? nonEmptyString(correlation.meta_session_id)
|
|
741
|
+
: null;
|
|
742
|
+
const body = envelope ? nonEmptyString(envelope.body) : null;
|
|
743
|
+
const kind = envelope ? nonEmptyString(envelope.kind) : null;
|
|
744
|
+
const state = record ? nonEmptyString(record.state) : null;
|
|
745
|
+
pushItem(out, {
|
|
746
|
+
id: `cp_outbox:${outboxId ?? row.line}`,
|
|
747
|
+
ts_ms: asInt(rec.ts_ms) ?? (record ? asInt(record.updated_at_ms) : null) ?? 0,
|
|
748
|
+
source_kind: "cp_outbox",
|
|
749
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
750
|
+
source_line: row.line,
|
|
751
|
+
repo_root: repoRoot,
|
|
752
|
+
issue_id: null,
|
|
753
|
+
run_id: runId,
|
|
754
|
+
session_id: sessionId,
|
|
755
|
+
channel,
|
|
756
|
+
channel_tenant_id: channelTenantId,
|
|
757
|
+
channel_conversation_id: channelConversationId,
|
|
758
|
+
actor_binding_id: actorBindingId,
|
|
759
|
+
conversation_key: conversationKey,
|
|
760
|
+
topic: null,
|
|
761
|
+
author: null,
|
|
762
|
+
role: null,
|
|
763
|
+
tags: ["cp", "outbox", state ?? "unknown", kind ?? "unknown"],
|
|
764
|
+
metadata: {
|
|
765
|
+
outbox_id: outboxId,
|
|
766
|
+
state,
|
|
767
|
+
kind,
|
|
768
|
+
attempt_count: record ? asInt(record.attempt_count) : null,
|
|
769
|
+
max_attempts: record ? asInt(record.max_attempts) : null,
|
|
770
|
+
},
|
|
771
|
+
text: [kind ?? "", body ?? "", state ?? ""].filter((part) => part.length > 0).join("\n"),
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
return out;
|
|
775
|
+
}
|
|
776
|
+
async function collectAdapterAudit(repoRoot, path) {
|
|
777
|
+
const out = [];
|
|
778
|
+
for (const row of await readJsonlRows(path)) {
|
|
779
|
+
const rec = asRecord(row.value);
|
|
780
|
+
if (!rec || nonEmptyString(rec.kind) !== "adapter.audit") {
|
|
781
|
+
continue;
|
|
782
|
+
}
|
|
783
|
+
const channel = nonEmptyString(rec.channel);
|
|
784
|
+
const channelTenantId = nonEmptyString(rec.channel_tenant_id);
|
|
785
|
+
const channelConversationId = nonEmptyString(rec.channel_conversation_id);
|
|
786
|
+
const event = nonEmptyString(rec.event) ?? "adapter.audit";
|
|
787
|
+
const commandText = nonEmptyString(rec.command_text) ?? "";
|
|
788
|
+
const reason = nonEmptyString(rec.reason);
|
|
789
|
+
pushItem(out, {
|
|
790
|
+
id: `cp_adapter_audit:${event}:${row.line}`,
|
|
791
|
+
ts_ms: asInt(rec.ts_ms) ?? 0,
|
|
792
|
+
source_kind: "cp_adapter_audit",
|
|
793
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
794
|
+
source_line: row.line,
|
|
795
|
+
repo_root: repoRoot,
|
|
796
|
+
issue_id: null,
|
|
797
|
+
run_id: null,
|
|
798
|
+
session_id: null,
|
|
799
|
+
channel,
|
|
800
|
+
channel_tenant_id: channelTenantId,
|
|
801
|
+
channel_conversation_id: channelConversationId,
|
|
802
|
+
actor_binding_id: null,
|
|
803
|
+
conversation_key: null,
|
|
804
|
+
topic: null,
|
|
805
|
+
author: nonEmptyString(rec.actor_id),
|
|
806
|
+
role: null,
|
|
807
|
+
tags: ["cp", "adapter.audit", event],
|
|
808
|
+
metadata: {
|
|
809
|
+
request_id: nonEmptyString(rec.request_id),
|
|
810
|
+
delivery_id: nonEmptyString(rec.delivery_id),
|
|
811
|
+
reason,
|
|
812
|
+
},
|
|
813
|
+
text: [event, commandText, reason ?? ""].filter((part) => part.length > 0).join("\n"),
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
return out;
|
|
817
|
+
}
|
|
818
|
+
async function collectOperatorTurns(repoRoot, path) {
|
|
819
|
+
const out = [];
|
|
820
|
+
for (const row of await readJsonlRows(path)) {
|
|
821
|
+
const rec = asRecord(row.value);
|
|
822
|
+
if (!rec || nonEmptyString(rec.kind) !== "operator.turn") {
|
|
823
|
+
continue;
|
|
824
|
+
}
|
|
825
|
+
const outcome = nonEmptyString(rec.outcome) ?? "unknown";
|
|
826
|
+
const reason = nonEmptyString(rec.reason);
|
|
827
|
+
const sessionId = nonEmptyString(rec.session_id);
|
|
828
|
+
const commandText = textFromUnknown(rec.command);
|
|
829
|
+
const messagePreview = nonEmptyString(rec.message_preview) ?? "";
|
|
830
|
+
pushItem(out, {
|
|
831
|
+
id: `cp_operator_turns:${sessionId ?? "unknown"}:${row.line}`,
|
|
832
|
+
ts_ms: asInt(rec.ts_ms) ?? 0,
|
|
833
|
+
source_kind: "cp_operator_turns",
|
|
834
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
835
|
+
source_line: row.line,
|
|
836
|
+
repo_root: repoRoot,
|
|
837
|
+
issue_id: null,
|
|
838
|
+
run_id: null,
|
|
839
|
+
session_id: sessionId,
|
|
840
|
+
channel: nonEmptyString(rec.channel),
|
|
841
|
+
channel_tenant_id: null,
|
|
842
|
+
channel_conversation_id: null,
|
|
843
|
+
actor_binding_id: null,
|
|
844
|
+
conversation_key: null,
|
|
845
|
+
topic: null,
|
|
846
|
+
author: null,
|
|
847
|
+
role: null,
|
|
848
|
+
tags: ["cp", "operator.turn", outcome],
|
|
849
|
+
metadata: {
|
|
850
|
+
request_id: nonEmptyString(rec.request_id),
|
|
851
|
+
turn_id: nonEmptyString(rec.turn_id),
|
|
852
|
+
reason,
|
|
853
|
+
},
|
|
854
|
+
text: [outcome, reason ?? "", messagePreview, commandText].filter((part) => part.length > 0).join("\n"),
|
|
855
|
+
});
|
|
856
|
+
}
|
|
857
|
+
return out;
|
|
858
|
+
}
|
|
859
|
+
async function collectTelegramIngress(repoRoot, path) {
|
|
860
|
+
const out = [];
|
|
861
|
+
for (const row of await readJsonlRows(path)) {
|
|
862
|
+
const rec = asRecord(row.value);
|
|
863
|
+
if (!rec || nonEmptyString(rec.kind) !== "telegram.ingress.state") {
|
|
864
|
+
continue;
|
|
865
|
+
}
|
|
866
|
+
const record = asRecord(rec.record);
|
|
867
|
+
const inbound = record ? asRecord(record.inbound) : null;
|
|
868
|
+
const channel = inbound ? nonEmptyString(inbound.channel) : null;
|
|
869
|
+
const channelTenantId = inbound ? nonEmptyString(inbound.channel_tenant_id) : null;
|
|
870
|
+
const channelConversationId = inbound ? nonEmptyString(inbound.channel_conversation_id) : null;
|
|
871
|
+
const actorBindingId = inbound ? nonEmptyString(inbound.actor_binding_id) : null;
|
|
872
|
+
const conversationKey = buildConversationKey({
|
|
873
|
+
channel,
|
|
874
|
+
tenantId: channelTenantId,
|
|
875
|
+
conversationId: channelConversationId,
|
|
876
|
+
bindingId: actorBindingId,
|
|
877
|
+
});
|
|
878
|
+
const ingressId = record ? nonEmptyString(record.ingress_id) : null;
|
|
879
|
+
const state = record ? nonEmptyString(record.state) : null;
|
|
880
|
+
const commandText = inbound ? nonEmptyString(inbound.command_text) : null;
|
|
881
|
+
pushItem(out, {
|
|
882
|
+
id: `cp_telegram_ingress:${ingressId ?? row.line}`,
|
|
883
|
+
ts_ms: asInt(rec.ts_ms) ?? (record ? asInt(record.updated_at_ms) : null) ?? 0,
|
|
884
|
+
source_kind: "cp_telegram_ingress",
|
|
885
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
886
|
+
source_line: row.line,
|
|
887
|
+
repo_root: repoRoot,
|
|
888
|
+
issue_id: null,
|
|
889
|
+
run_id: null,
|
|
890
|
+
session_id: null,
|
|
891
|
+
channel,
|
|
892
|
+
channel_tenant_id: channelTenantId,
|
|
893
|
+
channel_conversation_id: channelConversationId,
|
|
894
|
+
actor_binding_id: actorBindingId,
|
|
895
|
+
conversation_key: conversationKey,
|
|
896
|
+
topic: null,
|
|
897
|
+
author: inbound ? nonEmptyString(inbound.actor_id) : null,
|
|
898
|
+
role: null,
|
|
899
|
+
tags: ["cp", "telegram.ingress", state ?? "unknown"],
|
|
900
|
+
metadata: {
|
|
901
|
+
ingress_id: ingressId,
|
|
902
|
+
state,
|
|
903
|
+
request_id: inbound ? nonEmptyString(inbound.request_id) : null,
|
|
904
|
+
},
|
|
905
|
+
text: [commandText ?? "", state ?? ""].filter((part) => part.length > 0).join("\n"),
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
return out;
|
|
909
|
+
}
|
|
910
|
+
async function collectSessionFlash(repoRoot, path) {
|
|
911
|
+
const out = [];
|
|
912
|
+
const created = new Map();
|
|
913
|
+
const delivered = new Map();
|
|
914
|
+
for (const row of await readJsonlRows(path)) {
|
|
915
|
+
const rec = asRecord(row.value);
|
|
916
|
+
if (!rec) {
|
|
917
|
+
continue;
|
|
918
|
+
}
|
|
919
|
+
const kind = nonEmptyString(rec.kind);
|
|
920
|
+
if (kind === "session_flash.create") {
|
|
921
|
+
const flashId = nonEmptyString(rec.flash_id);
|
|
922
|
+
const sessionId = nonEmptyString(rec.session_id);
|
|
923
|
+
const body = nonEmptyString(rec.body);
|
|
924
|
+
const tsMs = asInt(rec.ts_ms) ?? 0;
|
|
925
|
+
if (!flashId || !sessionId || !body) {
|
|
926
|
+
continue;
|
|
927
|
+
}
|
|
928
|
+
const contextIds = Array.isArray(rec.context_ids)
|
|
929
|
+
? rec.context_ids
|
|
930
|
+
.map((value) => nonEmptyString(value))
|
|
931
|
+
.filter((value) => value != null)
|
|
932
|
+
: [];
|
|
933
|
+
created.set(flashId, {
|
|
934
|
+
created_at_ms: tsMs,
|
|
935
|
+
flash_id: flashId,
|
|
936
|
+
session_id: sessionId,
|
|
937
|
+
session_kind: nonEmptyString(rec.session_kind),
|
|
938
|
+
body,
|
|
939
|
+
context_ids: contextIds,
|
|
940
|
+
source: nonEmptyString(rec.source),
|
|
941
|
+
metadata: asRecord(rec.metadata) ?? {},
|
|
942
|
+
source_line: row.line,
|
|
943
|
+
});
|
|
944
|
+
continue;
|
|
945
|
+
}
|
|
946
|
+
if (kind === "session_flash.delivery") {
|
|
947
|
+
const flashId = nonEmptyString(rec.flash_id);
|
|
948
|
+
if (!flashId) {
|
|
949
|
+
continue;
|
|
950
|
+
}
|
|
951
|
+
delivered.set(flashId, {
|
|
952
|
+
delivered_at_ms: asInt(rec.ts_ms) ?? 0,
|
|
953
|
+
delivered_by: nonEmptyString(rec.delivered_by),
|
|
954
|
+
note: nonEmptyString(rec.note),
|
|
955
|
+
});
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
for (const create of created.values()) {
|
|
959
|
+
const delivery = delivered.get(create.flash_id);
|
|
960
|
+
const status = delivery ? "delivered" : "pending";
|
|
961
|
+
pushItem(out, {
|
|
962
|
+
id: `session_flash:${create.flash_id}`,
|
|
963
|
+
ts_ms: create.created_at_ms,
|
|
964
|
+
source_kind: "session_flash",
|
|
965
|
+
source_path: normalizeRelative(repoRoot, path),
|
|
966
|
+
source_line: create.source_line,
|
|
967
|
+
repo_root: repoRoot,
|
|
968
|
+
issue_id: null,
|
|
969
|
+
run_id: null,
|
|
970
|
+
session_id: create.session_id,
|
|
971
|
+
channel: null,
|
|
972
|
+
channel_tenant_id: null,
|
|
973
|
+
channel_conversation_id: null,
|
|
974
|
+
actor_binding_id: null,
|
|
975
|
+
conversation_key: null,
|
|
976
|
+
topic: null,
|
|
977
|
+
author: create.source,
|
|
978
|
+
role: "user",
|
|
979
|
+
tags: ["session_flash", status, create.session_kind ?? "unknown"],
|
|
980
|
+
metadata: {
|
|
981
|
+
flash_id: create.flash_id,
|
|
982
|
+
session_kind: create.session_kind,
|
|
983
|
+
context_ids: create.context_ids,
|
|
984
|
+
delivery: delivery
|
|
985
|
+
? {
|
|
986
|
+
delivered_at_ms: delivery.delivered_at_ms,
|
|
987
|
+
delivered_by: delivery.delivered_by,
|
|
988
|
+
note: delivery.note,
|
|
989
|
+
}
|
|
990
|
+
: null,
|
|
991
|
+
...create.metadata,
|
|
992
|
+
},
|
|
993
|
+
text: [create.body, create.context_ids.join(" "), status].filter((part) => part.length > 0).join("\n"),
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
return out;
|
|
997
|
+
}
|
|
998
|
+
function sessionIdFromPath(path) {
|
|
999
|
+
const fileName = path.split(/[\\/]/).pop() ?? path;
|
|
1000
|
+
return fileName.replace(/\.jsonl$/i, "") || `session-${crypto.randomUUID()}`;
|
|
1001
|
+
}
|
|
1002
|
+
async function collectSessionMessages(opts) {
|
|
1003
|
+
const out = [];
|
|
1004
|
+
const files = await listJsonlFiles(opts.dir);
|
|
1005
|
+
for (const filePath of files) {
|
|
1006
|
+
const rows = await readJsonlRows(filePath);
|
|
1007
|
+
const fileStat = await stat(filePath).catch(() => null);
|
|
1008
|
+
let sessionId = null;
|
|
1009
|
+
for (const row of rows) {
|
|
1010
|
+
const rec = asRecord(row.value);
|
|
1011
|
+
if (!rec) {
|
|
1012
|
+
continue;
|
|
1013
|
+
}
|
|
1014
|
+
const entryType = nonEmptyString(rec.type);
|
|
1015
|
+
if (entryType === "session") {
|
|
1016
|
+
sessionId = nonEmptyString(rec.id) ?? sessionId;
|
|
1017
|
+
continue;
|
|
1018
|
+
}
|
|
1019
|
+
if (entryType !== "message") {
|
|
1020
|
+
continue;
|
|
1021
|
+
}
|
|
1022
|
+
const msg = asRecord(rec.message);
|
|
1023
|
+
if (!msg) {
|
|
1024
|
+
continue;
|
|
1025
|
+
}
|
|
1026
|
+
const role = nonEmptyString(msg.role);
|
|
1027
|
+
const text = extractMessageText(msg);
|
|
1028
|
+
if (text.trim().length === 0) {
|
|
1029
|
+
continue;
|
|
1030
|
+
}
|
|
1031
|
+
const resolvedSessionId = sessionId ?? sessionIdFromPath(filePath);
|
|
1032
|
+
const tsMs = parseTimestamp(rec.timestamp) ??
|
|
1033
|
+
Math.trunc(fileStat?.mtimeMs ?? fileStat?.ctimeMs ?? fileStat?.birthtimeMs ?? 0);
|
|
1034
|
+
const conversationKeys = opts.conversationKeysBySessionId?.get(resolvedSessionId)?.slice().sort((a, b) => a.localeCompare(b)) ?? [];
|
|
1035
|
+
pushItem(out, {
|
|
1036
|
+
id: `${opts.sourceKind}:${resolvedSessionId}:${row.line}:${normalizeRelative(opts.repoRoot, filePath)}`,
|
|
1037
|
+
ts_ms: tsMs,
|
|
1038
|
+
source_kind: opts.sourceKind,
|
|
1039
|
+
source_path: normalizeRelative(opts.repoRoot, filePath),
|
|
1040
|
+
source_line: row.line,
|
|
1041
|
+
repo_root: opts.repoRoot,
|
|
1042
|
+
issue_id: null,
|
|
1043
|
+
run_id: null,
|
|
1044
|
+
session_id: resolvedSessionId,
|
|
1045
|
+
channel: null,
|
|
1046
|
+
channel_tenant_id: null,
|
|
1047
|
+
channel_conversation_id: null,
|
|
1048
|
+
actor_binding_id: null,
|
|
1049
|
+
conversation_key: conversationKeys[0] ?? null,
|
|
1050
|
+
topic: null,
|
|
1051
|
+
author: null,
|
|
1052
|
+
role,
|
|
1053
|
+
tags: ["session", opts.sourceKind, role ?? "unknown"],
|
|
1054
|
+
metadata: {
|
|
1055
|
+
entry_id: nonEmptyString(rec.id),
|
|
1056
|
+
parent_id: nonEmptyString(rec.parentId),
|
|
1057
|
+
conversation_keys: conversationKeys,
|
|
1058
|
+
},
|
|
1059
|
+
text,
|
|
1060
|
+
});
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
return out;
|
|
1064
|
+
}
|
|
1065
|
+
function getControlPlaneMemoryPaths(repoRoot) {
|
|
1066
|
+
const store = getStorePaths(repoRoot);
|
|
1067
|
+
const controlPlaneDir = join(store.storeDir, "control-plane");
|
|
1068
|
+
return {
|
|
1069
|
+
controlPlaneDir,
|
|
1070
|
+
commandsPath: join(controlPlaneDir, "commands.jsonl"),
|
|
1071
|
+
outboxPath: join(controlPlaneDir, "outbox.jsonl"),
|
|
1072
|
+
adapterAuditPath: join(controlPlaneDir, "adapter_audit.jsonl"),
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
async function collectContextItems(repoRoot, requestedSources) {
|
|
1076
|
+
const include = (kind) => (requestedSources ? requestedSources.has(kind) : true);
|
|
1077
|
+
const paths = getStorePaths(repoRoot);
|
|
1078
|
+
const cp = getControlPlaneMemoryPaths(repoRoot);
|
|
1079
|
+
const cpDir = cp.controlPlaneDir;
|
|
1080
|
+
const conversationMap = await loadConversationBindings(join(cpDir, "operator_conversations.json")).catch(() => new Map());
|
|
1081
|
+
const conversationKeysBySessionId = reverseConversationBindings(conversationMap);
|
|
1082
|
+
const tasks = [];
|
|
1083
|
+
if (include("issues")) {
|
|
1084
|
+
tasks.push(collectIssues(repoRoot, paths.issuesPath));
|
|
1085
|
+
}
|
|
1086
|
+
if (include("forum")) {
|
|
1087
|
+
tasks.push(collectForum(repoRoot, paths.forumPath));
|
|
1088
|
+
}
|
|
1089
|
+
if (include("events")) {
|
|
1090
|
+
tasks.push(collectEvents(repoRoot, paths.eventsPath));
|
|
1091
|
+
}
|
|
1092
|
+
if (include("cp_commands")) {
|
|
1093
|
+
tasks.push(collectCommandJournal(repoRoot, cp.commandsPath));
|
|
1094
|
+
}
|
|
1095
|
+
if (include("cp_outbox")) {
|
|
1096
|
+
tasks.push(collectOutbox(repoRoot, cp.outboxPath));
|
|
1097
|
+
}
|
|
1098
|
+
if (include("cp_adapter_audit")) {
|
|
1099
|
+
tasks.push(collectAdapterAudit(repoRoot, cp.adapterAuditPath));
|
|
1100
|
+
}
|
|
1101
|
+
if (include("cp_operator_turns")) {
|
|
1102
|
+
tasks.push(collectOperatorTurns(repoRoot, join(cpDir, "operator_turns.jsonl")));
|
|
1103
|
+
}
|
|
1104
|
+
if (include("cp_telegram_ingress")) {
|
|
1105
|
+
tasks.push(collectTelegramIngress(repoRoot, join(cpDir, "telegram_ingress.jsonl")));
|
|
1106
|
+
}
|
|
1107
|
+
if (include("session_flash")) {
|
|
1108
|
+
tasks.push(collectSessionFlash(repoRoot, join(cpDir, "session_flash.jsonl")));
|
|
1109
|
+
}
|
|
1110
|
+
if (include("operator_sessions")) {
|
|
1111
|
+
tasks.push(collectSessionMessages({
|
|
1112
|
+
repoRoot,
|
|
1113
|
+
dir: join(paths.storeDir, "operator", "sessions"),
|
|
1114
|
+
sourceKind: "operator_sessions",
|
|
1115
|
+
}));
|
|
1116
|
+
}
|
|
1117
|
+
if (include("cp_operator_sessions")) {
|
|
1118
|
+
tasks.push(collectSessionMessages({
|
|
1119
|
+
repoRoot,
|
|
1120
|
+
dir: join(cpDir, "operator-sessions"),
|
|
1121
|
+
sourceKind: "cp_operator_sessions",
|
|
1122
|
+
conversationKeysBySessionId,
|
|
1123
|
+
}));
|
|
1124
|
+
}
|
|
1125
|
+
const chunks = await Promise.all(tasks);
|
|
1126
|
+
const items = chunks.flat();
|
|
1127
|
+
items.sort((a, b) => {
|
|
1128
|
+
if (a.ts_ms !== b.ts_ms) {
|
|
1129
|
+
return b.ts_ms - a.ts_ms;
|
|
1130
|
+
}
|
|
1131
|
+
return a.id.localeCompare(b.id);
|
|
1132
|
+
});
|
|
1133
|
+
return items;
|
|
1134
|
+
}
|
|
1135
|
+
function searchContext(items, filters) {
|
|
1136
|
+
const scored = [];
|
|
1137
|
+
for (const item of items) {
|
|
1138
|
+
if (!matchSearchFilters(item, filters)) {
|
|
1139
|
+
continue;
|
|
1140
|
+
}
|
|
1141
|
+
const score = scoreItem(item, filters.query);
|
|
1142
|
+
if (score < 0) {
|
|
1143
|
+
continue;
|
|
1144
|
+
}
|
|
1145
|
+
scored.push({ ...item, score });
|
|
1146
|
+
}
|
|
1147
|
+
scored.sort((a, b) => {
|
|
1148
|
+
if (a.score !== b.score) {
|
|
1149
|
+
return b.score - a.score;
|
|
1150
|
+
}
|
|
1151
|
+
if (a.ts_ms !== b.ts_ms) {
|
|
1152
|
+
return b.ts_ms - a.ts_ms;
|
|
1153
|
+
}
|
|
1154
|
+
return a.id.localeCompare(b.id);
|
|
1155
|
+
});
|
|
1156
|
+
return scored;
|
|
1157
|
+
}
|
|
1158
|
+
function timelineContext(items, filters) {
|
|
1159
|
+
const out = items.filter((item) => matchSearchFilters(item, filters));
|
|
1160
|
+
out.sort((a, b) => {
|
|
1161
|
+
if (a.ts_ms !== b.ts_ms) {
|
|
1162
|
+
return filters.order === "asc" ? a.ts_ms - b.ts_ms : b.ts_ms - a.ts_ms;
|
|
1163
|
+
}
|
|
1164
|
+
return a.id.localeCompare(b.id);
|
|
1165
|
+
});
|
|
1166
|
+
if (filters.query) {
|
|
1167
|
+
const query = filters.query.toLowerCase();
|
|
1168
|
+
const tokens = tokenizeQuery(query);
|
|
1169
|
+
return out.filter((item) => {
|
|
1170
|
+
const haystack = searchableText(item);
|
|
1171
|
+
if (haystack.includes(query)) {
|
|
1172
|
+
return true;
|
|
1173
|
+
}
|
|
1174
|
+
if (tokens.length === 0) {
|
|
1175
|
+
return true;
|
|
1176
|
+
}
|
|
1177
|
+
return tokens.every((token) => haystack.includes(token));
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
return out;
|
|
1181
|
+
}
|
|
1182
|
+
function buildSourceStats(items) {
|
|
1183
|
+
const map = new Map();
|
|
1184
|
+
for (const item of items) {
|
|
1185
|
+
const row = map.get(item.source_kind) ?? { count: 0, textBytes: 0, lastTsMs: 0 };
|
|
1186
|
+
row.count += 1;
|
|
1187
|
+
row.textBytes += item.text.length;
|
|
1188
|
+
row.lastTsMs = Math.max(row.lastTsMs, item.ts_ms);
|
|
1189
|
+
map.set(item.source_kind, row);
|
|
1190
|
+
}
|
|
1191
|
+
const out = [...map.entries()].map(([source, row]) => ({
|
|
1192
|
+
source_kind: source,
|
|
1193
|
+
count: row.count,
|
|
1194
|
+
text_bytes: row.textBytes,
|
|
1195
|
+
last_ts_ms: row.lastTsMs,
|
|
1196
|
+
}));
|
|
1197
|
+
out.sort((a, b) => {
|
|
1198
|
+
if (a.count !== b.count) {
|
|
1199
|
+
return b.count - a.count;
|
|
1200
|
+
}
|
|
1201
|
+
return a.source_kind.localeCompare(b.source_kind);
|
|
1202
|
+
});
|
|
1203
|
+
return out;
|
|
1204
|
+
}
|
|
1205
|
+
function contextIndexPath(repoRoot) {
|
|
1206
|
+
return join(getStorePaths(repoRoot).storeDir, "context", CONTEXT_INDEX_FILENAME);
|
|
1207
|
+
}
|
|
1208
|
+
function toContextSourceKind(value) {
|
|
1209
|
+
if (CONTEXT_SOURCE_KINDS.includes(value)) {
|
|
1210
|
+
return value;
|
|
1211
|
+
}
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
function contextIndexFtsText(item) {
|
|
1215
|
+
return [
|
|
1216
|
+
item.text,
|
|
1217
|
+
item.preview,
|
|
1218
|
+
item.source_kind,
|
|
1219
|
+
item.issue_id ?? "",
|
|
1220
|
+
item.run_id ?? "",
|
|
1221
|
+
item.session_id ?? "",
|
|
1222
|
+
item.channel ?? "",
|
|
1223
|
+
item.channel_tenant_id ?? "",
|
|
1224
|
+
item.channel_conversation_id ?? "",
|
|
1225
|
+
item.actor_binding_id ?? "",
|
|
1226
|
+
item.conversation_key ?? "",
|
|
1227
|
+
item.topic ?? "",
|
|
1228
|
+
item.author ?? "",
|
|
1229
|
+
item.role ?? "",
|
|
1230
|
+
item.tags.join(" "),
|
|
1231
|
+
].join("\n");
|
|
1232
|
+
}
|
|
1233
|
+
function parseStringArrayJson(value) {
|
|
1234
|
+
if (typeof value !== "string") {
|
|
1235
|
+
return [];
|
|
1236
|
+
}
|
|
1237
|
+
try {
|
|
1238
|
+
const parsed = JSON.parse(value);
|
|
1239
|
+
if (!Array.isArray(parsed)) {
|
|
1240
|
+
return [];
|
|
1241
|
+
}
|
|
1242
|
+
return parsed.filter((entry) => typeof entry === "string");
|
|
1243
|
+
}
|
|
1244
|
+
catch {
|
|
1245
|
+
return [];
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
function parseRecordJson(value) {
|
|
1249
|
+
if (typeof value !== "string") {
|
|
1250
|
+
return {};
|
|
1251
|
+
}
|
|
1252
|
+
try {
|
|
1253
|
+
const parsed = JSON.parse(value);
|
|
1254
|
+
return asRecord(parsed) ?? {};
|
|
1255
|
+
}
|
|
1256
|
+
catch {
|
|
1257
|
+
return {};
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
function contextItemFromIndexedRow(rowRaw) {
|
|
1261
|
+
const row = asRecord(rowRaw);
|
|
1262
|
+
if (!row) {
|
|
1263
|
+
return null;
|
|
1264
|
+
}
|
|
1265
|
+
const sourceKindRaw = nonEmptyString(row.source_kind);
|
|
1266
|
+
if (!sourceKindRaw) {
|
|
1267
|
+
return null;
|
|
1268
|
+
}
|
|
1269
|
+
const sourceKind = toContextSourceKind(sourceKindRaw);
|
|
1270
|
+
if (!sourceKind) {
|
|
1271
|
+
return null;
|
|
1272
|
+
}
|
|
1273
|
+
const id = nonEmptyString(row.item_id);
|
|
1274
|
+
const sourcePath = nonEmptyString(row.source_path);
|
|
1275
|
+
const repoRoot = nonEmptyString(row.repo_root);
|
|
1276
|
+
const text = nonEmptyString(row.text);
|
|
1277
|
+
const preview = nonEmptyString(row.preview);
|
|
1278
|
+
if (!id || !sourcePath || !repoRoot || !text || !preview) {
|
|
1279
|
+
return null;
|
|
1280
|
+
}
|
|
1281
|
+
return {
|
|
1282
|
+
id,
|
|
1283
|
+
ts_ms: asInt(row.ts_ms) ?? 0,
|
|
1284
|
+
source_kind: sourceKind,
|
|
1285
|
+
source_path: sourcePath,
|
|
1286
|
+
source_line: asInt(row.source_line) ?? 0,
|
|
1287
|
+
repo_root: repoRoot,
|
|
1288
|
+
text,
|
|
1289
|
+
preview,
|
|
1290
|
+
issue_id: nonEmptyString(row.issue_id),
|
|
1291
|
+
run_id: nonEmptyString(row.run_id),
|
|
1292
|
+
session_id: nonEmptyString(row.session_id),
|
|
1293
|
+
channel: nonEmptyString(row.channel),
|
|
1294
|
+
channel_tenant_id: nonEmptyString(row.channel_tenant_id),
|
|
1295
|
+
channel_conversation_id: nonEmptyString(row.channel_conversation_id),
|
|
1296
|
+
actor_binding_id: nonEmptyString(row.actor_binding_id),
|
|
1297
|
+
conversation_key: nonEmptyString(row.conversation_key),
|
|
1298
|
+
topic: nonEmptyString(row.topic),
|
|
1299
|
+
author: nonEmptyString(row.author),
|
|
1300
|
+
role: nonEmptyString(row.role),
|
|
1301
|
+
tags: parseStringArrayJson(row.tags_json),
|
|
1302
|
+
metadata: parseRecordJson(row.metadata_json),
|
|
1303
|
+
};
|
|
1304
|
+
}
|
|
1305
|
+
function sqlClauseForFilters(filters) {
|
|
1306
|
+
const clauses = [];
|
|
1307
|
+
const params = [];
|
|
1308
|
+
if (filters.sources && filters.sources.size > 0) {
|
|
1309
|
+
const values = [...filters.sources];
|
|
1310
|
+
clauses.push(`source_kind IN (${values.map(() => "?").join(",")})`);
|
|
1311
|
+
params.push(...values);
|
|
1312
|
+
}
|
|
1313
|
+
if (filters.sinceMs != null) {
|
|
1314
|
+
clauses.push("ts_ms >= ?");
|
|
1315
|
+
params.push(filters.sinceMs);
|
|
1316
|
+
}
|
|
1317
|
+
if (filters.untilMs != null) {
|
|
1318
|
+
clauses.push("ts_ms <= ?");
|
|
1319
|
+
params.push(filters.untilMs);
|
|
1320
|
+
}
|
|
1321
|
+
if (filters.issueId) {
|
|
1322
|
+
clauses.push("issue_id = ?");
|
|
1323
|
+
params.push(filters.issueId);
|
|
1324
|
+
}
|
|
1325
|
+
if (filters.runId) {
|
|
1326
|
+
clauses.push("run_id = ?");
|
|
1327
|
+
params.push(filters.runId);
|
|
1328
|
+
}
|
|
1329
|
+
if (filters.sessionId) {
|
|
1330
|
+
clauses.push("session_id = ?");
|
|
1331
|
+
params.push(filters.sessionId);
|
|
1332
|
+
}
|
|
1333
|
+
if (filters.channel) {
|
|
1334
|
+
clauses.push("channel = ?");
|
|
1335
|
+
params.push(filters.channel);
|
|
1336
|
+
}
|
|
1337
|
+
if (filters.channelTenantId) {
|
|
1338
|
+
clauses.push("channel_tenant_id = ?");
|
|
1339
|
+
params.push(filters.channelTenantId);
|
|
1340
|
+
}
|
|
1341
|
+
if (filters.channelConversationId) {
|
|
1342
|
+
clauses.push("channel_conversation_id = ?");
|
|
1343
|
+
params.push(filters.channelConversationId);
|
|
1344
|
+
}
|
|
1345
|
+
if (filters.actorBindingId) {
|
|
1346
|
+
clauses.push("actor_binding_id = ?");
|
|
1347
|
+
params.push(filters.actorBindingId);
|
|
1348
|
+
}
|
|
1349
|
+
if (filters.topic) {
|
|
1350
|
+
clauses.push("topic = ?");
|
|
1351
|
+
params.push(filters.topic);
|
|
1352
|
+
}
|
|
1353
|
+
if (filters.author) {
|
|
1354
|
+
clauses.push("author = ?");
|
|
1355
|
+
params.push(filters.author);
|
|
1356
|
+
}
|
|
1357
|
+
if (filters.role) {
|
|
1358
|
+
clauses.push("role = ?");
|
|
1359
|
+
params.push(filters.role);
|
|
1360
|
+
}
|
|
1361
|
+
const clause = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
1362
|
+
return { clause, params };
|
|
1363
|
+
}
|
|
1364
|
+
function ftsQueryFromText(query) {
|
|
1365
|
+
const tokens = tokenizeQuery(query);
|
|
1366
|
+
if (tokens.length === 0) {
|
|
1367
|
+
const escaped = query.replaceAll('"', '""');
|
|
1368
|
+
return `"${escaped}"`;
|
|
1369
|
+
}
|
|
1370
|
+
return tokens.map((token) => `"${token.replaceAll('"', '""')}"`).join(" AND ");
|
|
1371
|
+
}
|
|
1372
|
+
function lookupFtsMatches(db, query) {
|
|
1373
|
+
const ftsQuery = ftsQueryFromText(query);
|
|
1374
|
+
try {
|
|
1375
|
+
const rows = db
|
|
1376
|
+
.query("SELECT item_id FROM memory_fts WHERE memory_fts MATCH ? LIMIT ?")
|
|
1377
|
+
.all(ftsQuery, INDEX_FTS_ROW_LIMIT);
|
|
1378
|
+
if (rows.length >= INDEX_FTS_ROW_LIMIT) {
|
|
1379
|
+
return { kind: "truncated" };
|
|
1380
|
+
}
|
|
1381
|
+
const ids = new Set();
|
|
1382
|
+
for (const rowRaw of rows) {
|
|
1383
|
+
const row = asRecord(rowRaw);
|
|
1384
|
+
const id = row ? nonEmptyString(row.item_id) : null;
|
|
1385
|
+
if (id) {
|
|
1386
|
+
ids.add(id);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
return { kind: "ok", ids };
|
|
1390
|
+
}
|
|
1391
|
+
catch {
|
|
1392
|
+
return { kind: "error" };
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
function queryIndexedCandidates(db, filters) {
|
|
1396
|
+
const sqlFilter = sqlClauseForFilters(filters);
|
|
1397
|
+
const sql = [
|
|
1398
|
+
"SELECT",
|
|
1399
|
+
" item_id, ts_ms, source_kind, source_path, source_line, repo_root,",
|
|
1400
|
+
" text, preview, issue_id, run_id, session_id, channel,",
|
|
1401
|
+
" channel_tenant_id, channel_conversation_id, actor_binding_id, conversation_key,",
|
|
1402
|
+
" topic, author, role, tags_json, metadata_json",
|
|
1403
|
+
"FROM memory_items",
|
|
1404
|
+
sqlFilter.clause,
|
|
1405
|
+
"ORDER BY ts_ms DESC, item_id ASC",
|
|
1406
|
+
"LIMIT ?",
|
|
1407
|
+
].join(" ");
|
|
1408
|
+
const rows = db.query(sql).all(...sqlFilter.params, INDEX_QUERY_ROW_LIMIT);
|
|
1409
|
+
if (rows.length >= INDEX_QUERY_ROW_LIMIT) {
|
|
1410
|
+
return null;
|
|
1411
|
+
}
|
|
1412
|
+
const items = [];
|
|
1413
|
+
for (const row of rows) {
|
|
1414
|
+
const item = contextItemFromIndexedRow(row);
|
|
1415
|
+
if (item) {
|
|
1416
|
+
items.push(item);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
return items;
|
|
1420
|
+
}
|
|
1421
|
+
function readContextSearchFromIndex(opts) {
|
|
1422
|
+
const indexPath = contextIndexPath(opts.repoRoot);
|
|
1423
|
+
if (!existsSync(indexPath)) {
|
|
1424
|
+
return null;
|
|
1425
|
+
}
|
|
1426
|
+
let db = null;
|
|
1427
|
+
try {
|
|
1428
|
+
db = new Database(indexPath, { readonly: true, create: false });
|
|
1429
|
+
const baseItems = queryIndexedCandidates(db, opts.filters);
|
|
1430
|
+
if (!baseItems) {
|
|
1431
|
+
return null;
|
|
1432
|
+
}
|
|
1433
|
+
let candidates = baseItems;
|
|
1434
|
+
if (opts.filters.query) {
|
|
1435
|
+
const match = lookupFtsMatches(db, opts.filters.query);
|
|
1436
|
+
if (match.kind === "truncated") {
|
|
1437
|
+
return null;
|
|
1438
|
+
}
|
|
1439
|
+
if (match.kind === "ok") {
|
|
1440
|
+
candidates = candidates.filter((item) => match.ids.has(item.id));
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
const ranked = searchContext(candidates, opts.filters);
|
|
1444
|
+
const sliced = ranked.slice(0, opts.filters.limit);
|
|
1445
|
+
return {
|
|
1446
|
+
mode: "search",
|
|
1447
|
+
repo_root: opts.repoRoot,
|
|
1448
|
+
query: opts.filters.query,
|
|
1449
|
+
count: sliced.length,
|
|
1450
|
+
total: ranked.length,
|
|
1451
|
+
items: sliced,
|
|
1452
|
+
};
|
|
1453
|
+
}
|
|
1454
|
+
catch {
|
|
1455
|
+
return null;
|
|
1456
|
+
}
|
|
1457
|
+
finally {
|
|
1458
|
+
db?.close();
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
function readContextTimelineFromIndex(opts) {
|
|
1462
|
+
const indexPath = contextIndexPath(opts.repoRoot);
|
|
1463
|
+
if (!existsSync(indexPath)) {
|
|
1464
|
+
return null;
|
|
1465
|
+
}
|
|
1466
|
+
let db = null;
|
|
1467
|
+
try {
|
|
1468
|
+
db = new Database(indexPath, { readonly: true, create: false });
|
|
1469
|
+
const baseItems = queryIndexedCandidates(db, opts.filters);
|
|
1470
|
+
if (!baseItems) {
|
|
1471
|
+
return null;
|
|
1472
|
+
}
|
|
1473
|
+
let candidates = baseItems;
|
|
1474
|
+
if (opts.filters.query) {
|
|
1475
|
+
const match = lookupFtsMatches(db, opts.filters.query);
|
|
1476
|
+
if (match.kind === "truncated") {
|
|
1477
|
+
return null;
|
|
1478
|
+
}
|
|
1479
|
+
if (match.kind === "ok") {
|
|
1480
|
+
candidates = candidates.filter((item) => match.ids.has(item.id));
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
const timeline = timelineContext(candidates, opts.filters);
|
|
1484
|
+
const sliced = timeline.slice(0, opts.filters.limit);
|
|
1485
|
+
return {
|
|
1486
|
+
mode: "timeline",
|
|
1487
|
+
repo_root: opts.repoRoot,
|
|
1488
|
+
order: opts.filters.order,
|
|
1489
|
+
count: sliced.length,
|
|
1490
|
+
total: timeline.length,
|
|
1491
|
+
items: sliced,
|
|
1492
|
+
};
|
|
1493
|
+
}
|
|
1494
|
+
catch {
|
|
1495
|
+
return null;
|
|
1496
|
+
}
|
|
1497
|
+
finally {
|
|
1498
|
+
db?.close();
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
function readContextStatsFromIndex(opts) {
|
|
1502
|
+
const indexPath = contextIndexPath(opts.repoRoot);
|
|
1503
|
+
if (!existsSync(indexPath)) {
|
|
1504
|
+
return null;
|
|
1505
|
+
}
|
|
1506
|
+
let db = null;
|
|
1507
|
+
try {
|
|
1508
|
+
db = new Database(indexPath, { readonly: true, create: false });
|
|
1509
|
+
const items = queryIndexedCandidates(db, opts.filters);
|
|
1510
|
+
if (!items) {
|
|
1511
|
+
return null;
|
|
1512
|
+
}
|
|
1513
|
+
const filtered = items.filter((item) => matchSearchFilters(item, { ...opts.filters, query: null }));
|
|
1514
|
+
const sources = buildSourceStats(filtered);
|
|
1515
|
+
return {
|
|
1516
|
+
mode: "stats",
|
|
1517
|
+
repo_root: opts.repoRoot,
|
|
1518
|
+
total_count: filtered.length,
|
|
1519
|
+
total_text_bytes: filtered.reduce((sum, item) => sum + item.text.length, 0),
|
|
1520
|
+
sources,
|
|
1521
|
+
};
|
|
1522
|
+
}
|
|
1523
|
+
catch {
|
|
1524
|
+
return null;
|
|
1525
|
+
}
|
|
1526
|
+
finally {
|
|
1527
|
+
db?.close();
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
function parseMetaInt(db, key) {
|
|
1531
|
+
const row = db.query("SELECT value FROM memory_meta WHERE key = ?").get(key);
|
|
1532
|
+
const rec = asRecord(row);
|
|
1533
|
+
const valueRaw = rec ? nonEmptyString(rec.value) : null;
|
|
1534
|
+
if (!valueRaw) {
|
|
1535
|
+
return null;
|
|
1536
|
+
}
|
|
1537
|
+
const parsed = Number.parseInt(valueRaw, 10);
|
|
1538
|
+
if (!Number.isFinite(parsed)) {
|
|
1539
|
+
return null;
|
|
1540
|
+
}
|
|
1541
|
+
return parsed;
|
|
1542
|
+
}
|
|
1543
|
+
function ensureContextIndexSchema(db) {
|
|
1544
|
+
db.exec([
|
|
1545
|
+
"PRAGMA journal_mode = WAL;",
|
|
1546
|
+
"PRAGMA synchronous = NORMAL;",
|
|
1547
|
+
"CREATE TABLE IF NOT EXISTS memory_meta (",
|
|
1548
|
+
" key TEXT PRIMARY KEY,",
|
|
1549
|
+
" value TEXT NOT NULL",
|
|
1550
|
+
");",
|
|
1551
|
+
"CREATE TABLE IF NOT EXISTS memory_items (",
|
|
1552
|
+
" item_id TEXT PRIMARY KEY,",
|
|
1553
|
+
" ts_ms INTEGER NOT NULL,",
|
|
1554
|
+
" source_kind TEXT NOT NULL,",
|
|
1555
|
+
" source_path TEXT NOT NULL,",
|
|
1556
|
+
" source_line INTEGER NOT NULL,",
|
|
1557
|
+
" repo_root TEXT NOT NULL,",
|
|
1558
|
+
" text TEXT NOT NULL,",
|
|
1559
|
+
" preview TEXT NOT NULL,",
|
|
1560
|
+
" issue_id TEXT,",
|
|
1561
|
+
" run_id TEXT,",
|
|
1562
|
+
" session_id TEXT,",
|
|
1563
|
+
" channel TEXT,",
|
|
1564
|
+
" channel_tenant_id TEXT,",
|
|
1565
|
+
" channel_conversation_id TEXT,",
|
|
1566
|
+
" actor_binding_id TEXT,",
|
|
1567
|
+
" conversation_key TEXT,",
|
|
1568
|
+
" topic TEXT,",
|
|
1569
|
+
" author TEXT,",
|
|
1570
|
+
" role TEXT,",
|
|
1571
|
+
" tags_json TEXT NOT NULL,",
|
|
1572
|
+
" metadata_json TEXT NOT NULL",
|
|
1573
|
+
");",
|
|
1574
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_ts ON memory_items(ts_ms DESC);",
|
|
1575
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_source_ts ON memory_items(source_kind, ts_ms DESC);",
|
|
1576
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_issue ON memory_items(issue_id);",
|
|
1577
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_run ON memory_items(run_id);",
|
|
1578
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_session ON memory_items(session_id);",
|
|
1579
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_channel ON memory_items(channel);",
|
|
1580
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_topic ON memory_items(topic);",
|
|
1581
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_author ON memory_items(author);",
|
|
1582
|
+
"CREATE INDEX IF NOT EXISTS idx_memory_items_role ON memory_items(role);",
|
|
1583
|
+
"CREATE TABLE IF NOT EXISTS source_state (",
|
|
1584
|
+
" source_kind TEXT NOT NULL,",
|
|
1585
|
+
" source_path TEXT NOT NULL,",
|
|
1586
|
+
" row_count INTEGER NOT NULL,",
|
|
1587
|
+
" mtime_ms INTEGER,",
|
|
1588
|
+
" size_bytes INTEGER,",
|
|
1589
|
+
" updated_at_ms INTEGER NOT NULL,",
|
|
1590
|
+
" PRIMARY KEY(source_kind, source_path)",
|
|
1591
|
+
");",
|
|
1592
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(",
|
|
1593
|
+
" item_id UNINDEXED,",
|
|
1594
|
+
" fulltext",
|
|
1595
|
+
");",
|
|
1596
|
+
].join("\n"));
|
|
1597
|
+
}
|
|
1598
|
+
function writeMeta(db, key, value) {
|
|
1599
|
+
db
|
|
1600
|
+
.query("INSERT INTO memory_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
|
|
1601
|
+
.run(key, value);
|
|
1602
|
+
}
|
|
1603
|
+
async function buildIndexSourceStateRows(repoRoot, items) {
|
|
1604
|
+
const rowsByKey = new Map();
|
|
1605
|
+
for (const item of items) {
|
|
1606
|
+
const key = `${item.source_kind}\u0000${item.source_path}`;
|
|
1607
|
+
const row = rowsByKey.get(key);
|
|
1608
|
+
if (row) {
|
|
1609
|
+
row.row_count += 1;
|
|
1610
|
+
continue;
|
|
1611
|
+
}
|
|
1612
|
+
rowsByKey.set(key, {
|
|
1613
|
+
source_kind: item.source_kind,
|
|
1614
|
+
source_path: item.source_path,
|
|
1615
|
+
row_count: 1,
|
|
1616
|
+
});
|
|
1617
|
+
}
|
|
1618
|
+
const out = [];
|
|
1619
|
+
for (const row of rowsByKey.values()) {
|
|
1620
|
+
const absolutePath = join(repoRoot, row.source_path);
|
|
1621
|
+
const stats = await stat(absolutePath).catch(() => null);
|
|
1622
|
+
out.push({
|
|
1623
|
+
...row,
|
|
1624
|
+
mtime_ms: stats ? Math.trunc(stats.mtimeMs) : null,
|
|
1625
|
+
size_bytes: stats ? Math.trunc(stats.size) : null,
|
|
1626
|
+
});
|
|
1627
|
+
}
|
|
1628
|
+
out.sort((a, b) => {
|
|
1629
|
+
if (a.source_kind !== b.source_kind) {
|
|
1630
|
+
return a.source_kind.localeCompare(b.source_kind);
|
|
1631
|
+
}
|
|
1632
|
+
return a.source_path.localeCompare(b.source_path);
|
|
1633
|
+
});
|
|
1634
|
+
return out;
|
|
1635
|
+
}
|
|
1636
|
+
function readSourceSummariesFromIndex(db) {
|
|
1637
|
+
const rows = db
|
|
1638
|
+
.query("SELECT source_kind, COUNT(*) AS count, IFNULL(SUM(LENGTH(text)), 0) AS text_bytes, IFNULL(MAX(ts_ms), 0) AS last_ts_ms FROM memory_items GROUP BY source_kind")
|
|
1639
|
+
.all();
|
|
1640
|
+
const out = [];
|
|
1641
|
+
for (const rowRaw of rows) {
|
|
1642
|
+
const row = asRecord(rowRaw);
|
|
1643
|
+
if (!row) {
|
|
1644
|
+
continue;
|
|
1645
|
+
}
|
|
1646
|
+
const sourceKindRaw = nonEmptyString(row.source_kind);
|
|
1647
|
+
if (!sourceKindRaw) {
|
|
1648
|
+
continue;
|
|
1649
|
+
}
|
|
1650
|
+
const sourceKind = toContextSourceKind(sourceKindRaw);
|
|
1651
|
+
if (!sourceKind) {
|
|
1652
|
+
continue;
|
|
1653
|
+
}
|
|
1654
|
+
out.push({
|
|
1655
|
+
source_kind: sourceKind,
|
|
1656
|
+
count: asInt(row.count) ?? 0,
|
|
1657
|
+
text_bytes: asInt(row.text_bytes) ?? 0,
|
|
1658
|
+
last_ts_ms: asInt(row.last_ts_ms) ?? 0,
|
|
1659
|
+
});
|
|
1660
|
+
}
|
|
1661
|
+
out.sort((a, b) => {
|
|
1662
|
+
if (a.count !== b.count) {
|
|
1663
|
+
return b.count - a.count;
|
|
1664
|
+
}
|
|
1665
|
+
return a.source_kind.localeCompare(b.source_kind);
|
|
1666
|
+
});
|
|
1667
|
+
return out;
|
|
1668
|
+
}
|
|
1669
|
+
async function staleSourcePathsFromIndex(repoRoot, db) {
|
|
1670
|
+
const rows = db
|
|
1671
|
+
.query("SELECT source_path, mtime_ms, size_bytes FROM source_state ORDER BY source_path ASC")
|
|
1672
|
+
.all();
|
|
1673
|
+
const stale = [];
|
|
1674
|
+
for (const rowRaw of rows) {
|
|
1675
|
+
const row = asRecord(rowRaw);
|
|
1676
|
+
if (!row) {
|
|
1677
|
+
continue;
|
|
1678
|
+
}
|
|
1679
|
+
const sourcePath = nonEmptyString(row.source_path);
|
|
1680
|
+
if (!sourcePath) {
|
|
1681
|
+
continue;
|
|
1682
|
+
}
|
|
1683
|
+
const expectedMtime = asInt(row.mtime_ms);
|
|
1684
|
+
const expectedSize = asInt(row.size_bytes);
|
|
1685
|
+
const absolutePath = join(repoRoot, sourcePath);
|
|
1686
|
+
const stats = await stat(absolutePath).catch(() => null);
|
|
1687
|
+
const currentMtime = stats ? Math.trunc(stats.mtimeMs) : null;
|
|
1688
|
+
const currentSize = stats ? Math.trunc(stats.size) : null;
|
|
1689
|
+
if (expectedMtime !== currentMtime || expectedSize !== currentSize) {
|
|
1690
|
+
stale.push(sourcePath);
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
return stale;
|
|
1694
|
+
}
|
|
1695
|
+
export async function runContextIndexStatus(opts) {
|
|
1696
|
+
const indexPath = contextIndexPath(opts.repoRoot);
|
|
1697
|
+
if (!existsSync(indexPath)) {
|
|
1698
|
+
return {
|
|
1699
|
+
mode: "index_status",
|
|
1700
|
+
repo_root: opts.repoRoot,
|
|
1701
|
+
index_path: indexPath,
|
|
1702
|
+
exists: false,
|
|
1703
|
+
schema_version: CONTEXT_INDEX_SCHEMA_VERSION,
|
|
1704
|
+
built_at_ms: null,
|
|
1705
|
+
total_count: 0,
|
|
1706
|
+
total_text_bytes: 0,
|
|
1707
|
+
source_count: 0,
|
|
1708
|
+
stale_source_count: 0,
|
|
1709
|
+
stale_source_paths: [],
|
|
1710
|
+
sources: [],
|
|
1711
|
+
};
|
|
1712
|
+
}
|
|
1713
|
+
const db = new Database(indexPath, { readonly: true, create: false });
|
|
1714
|
+
try {
|
|
1715
|
+
const totalsRow = asRecord(db.query("SELECT COUNT(*) AS count, IFNULL(SUM(LENGTH(text)), 0) AS text_bytes FROM memory_items").get());
|
|
1716
|
+
const totalCount = totalsRow ? asInt(totalsRow.count) ?? 0 : 0;
|
|
1717
|
+
const totalTextBytes = totalsRow ? asInt(totalsRow.text_bytes) ?? 0 : 0;
|
|
1718
|
+
const sources = readSourceSummariesFromIndex(db);
|
|
1719
|
+
const staleSourcePaths = await staleSourcePathsFromIndex(opts.repoRoot, db);
|
|
1720
|
+
return {
|
|
1721
|
+
mode: "index_status",
|
|
1722
|
+
repo_root: opts.repoRoot,
|
|
1723
|
+
index_path: indexPath,
|
|
1724
|
+
exists: true,
|
|
1725
|
+
schema_version: parseMetaInt(db, "schema_version") ?? CONTEXT_INDEX_SCHEMA_VERSION,
|
|
1726
|
+
built_at_ms: parseMetaInt(db, "built_at_ms"),
|
|
1727
|
+
total_count: totalCount,
|
|
1728
|
+
total_text_bytes: totalTextBytes,
|
|
1729
|
+
source_count: sources.length,
|
|
1730
|
+
stale_source_count: staleSourcePaths.length,
|
|
1731
|
+
stale_source_paths: staleSourcePaths.slice(0, 25),
|
|
1732
|
+
sources,
|
|
1733
|
+
};
|
|
1734
|
+
}
|
|
1735
|
+
finally {
|
|
1736
|
+
db.close();
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
export async function runContextIndexRebuild(opts) {
|
|
1740
|
+
const startedAtMs = Date.now();
|
|
1741
|
+
const requestedSources = parseSourceFilter(opts.search.get("sources") ?? opts.search.get("source"));
|
|
1742
|
+
const items = await collectContextItems(opts.repoRoot, requestedSources);
|
|
1743
|
+
const sourceStateRows = await buildIndexSourceStateRows(opts.repoRoot, items);
|
|
1744
|
+
const indexPath = contextIndexPath(opts.repoRoot);
|
|
1745
|
+
await mkdir(join(getStorePaths(opts.repoRoot).storeDir, "context"), { recursive: true });
|
|
1746
|
+
const db = new Database(indexPath, { create: true });
|
|
1747
|
+
try {
|
|
1748
|
+
ensureContextIndexSchema(db);
|
|
1749
|
+
db.exec("BEGIN IMMEDIATE");
|
|
1750
|
+
try {
|
|
1751
|
+
db.exec("DELETE FROM memory_items");
|
|
1752
|
+
db.exec("DELETE FROM memory_fts");
|
|
1753
|
+
db.exec("DELETE FROM source_state");
|
|
1754
|
+
const insertItem = db.query([
|
|
1755
|
+
"INSERT INTO memory_items (",
|
|
1756
|
+
" item_id, ts_ms, source_kind, source_path, source_line, repo_root,",
|
|
1757
|
+
" text, preview, issue_id, run_id, session_id, channel,",
|
|
1758
|
+
" channel_tenant_id, channel_conversation_id, actor_binding_id, conversation_key,",
|
|
1759
|
+
" topic, author, role, tags_json, metadata_json",
|
|
1760
|
+
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
1761
|
+
].join("\n"));
|
|
1762
|
+
const insertFts = db.query("INSERT INTO memory_fts (item_id, fulltext) VALUES (?, ?)");
|
|
1763
|
+
for (const item of items) {
|
|
1764
|
+
insertItem.run(item.id, item.ts_ms, item.source_kind, item.source_path, item.source_line, item.repo_root, item.text, item.preview, item.issue_id, item.run_id, item.session_id, item.channel, item.channel_tenant_id, item.channel_conversation_id, item.actor_binding_id, item.conversation_key, item.topic, item.author, item.role, JSON.stringify(item.tags), JSON.stringify(item.metadata));
|
|
1765
|
+
insertFts.run(item.id, contextIndexFtsText(item));
|
|
1766
|
+
}
|
|
1767
|
+
const insertSourceState = db.query("INSERT INTO source_state (source_kind, source_path, row_count, mtime_ms, size_bytes, updated_at_ms) VALUES (?, ?, ?, ?, ?, ?)");
|
|
1768
|
+
const updatedAtMs = Date.now();
|
|
1769
|
+
for (const row of sourceStateRows) {
|
|
1770
|
+
insertSourceState.run(row.source_kind, row.source_path, row.row_count, row.mtime_ms, row.size_bytes, updatedAtMs);
|
|
1771
|
+
}
|
|
1772
|
+
writeMeta(db, "schema_version", String(CONTEXT_INDEX_SCHEMA_VERSION));
|
|
1773
|
+
writeMeta(db, "built_at_ms", String(Date.now()));
|
|
1774
|
+
writeMeta(db, "repo_root", opts.repoRoot);
|
|
1775
|
+
db.exec("COMMIT");
|
|
1776
|
+
}
|
|
1777
|
+
catch (err) {
|
|
1778
|
+
db.exec("ROLLBACK");
|
|
1779
|
+
throw err;
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
finally {
|
|
1783
|
+
db.close();
|
|
1784
|
+
}
|
|
1785
|
+
const status = await runContextIndexStatus({ repoRoot: opts.repoRoot });
|
|
1786
|
+
return {
|
|
1787
|
+
mode: "index_rebuild",
|
|
1788
|
+
repo_root: status.repo_root,
|
|
1789
|
+
index_path: status.index_path,
|
|
1790
|
+
exists: status.exists,
|
|
1791
|
+
schema_version: status.schema_version,
|
|
1792
|
+
built_at_ms: status.built_at_ms,
|
|
1793
|
+
total_count: status.total_count,
|
|
1794
|
+
total_text_bytes: status.total_text_bytes,
|
|
1795
|
+
source_count: status.source_count,
|
|
1796
|
+
stale_source_count: status.stale_source_count,
|
|
1797
|
+
stale_source_paths: status.stale_source_paths,
|
|
1798
|
+
sources: status.sources,
|
|
1799
|
+
indexed_count: items.length,
|
|
1800
|
+
duration_ms: Math.max(0, Date.now() - startedAtMs),
|
|
1801
|
+
requested_sources: requestedSources ? [...requestedSources].sort((a, b) => a.localeCompare(b)) : null,
|
|
1802
|
+
};
|
|
1803
|
+
}
|
|
1804
|
+
const AUTO_INDEX_REBUILD_COOLDOWN_MS = 15_000;
|
|
1805
|
+
const AUTO_INDEX_LAST_ATTEMPT_MS = new Map();
|
|
1806
|
+
function shouldAttemptAutoIndexRebuild(status, mode) {
|
|
1807
|
+
if (mode === "off") {
|
|
1808
|
+
return false;
|
|
1809
|
+
}
|
|
1810
|
+
if (!status.exists) {
|
|
1811
|
+
return true;
|
|
1812
|
+
}
|
|
1813
|
+
if (mode === "missing_or_stale" && status.stale_source_count > 0) {
|
|
1814
|
+
return true;
|
|
1815
|
+
}
|
|
1816
|
+
return false;
|
|
1817
|
+
}
|
|
1818
|
+
async function maybeAutoRebuildContextIndex(opts) {
|
|
1819
|
+
if (opts.mode === "off") {
|
|
1820
|
+
return;
|
|
1821
|
+
}
|
|
1822
|
+
const status = await runContextIndexStatus({ repoRoot: opts.repoRoot });
|
|
1823
|
+
if (!shouldAttemptAutoIndexRebuild(status, opts.mode)) {
|
|
1824
|
+
return;
|
|
1825
|
+
}
|
|
1826
|
+
const nowMs = Date.now();
|
|
1827
|
+
const lastAttemptMs = AUTO_INDEX_LAST_ATTEMPT_MS.get(opts.repoRoot) ?? 0;
|
|
1828
|
+
if (nowMs - lastAttemptMs < AUTO_INDEX_REBUILD_COOLDOWN_MS) {
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
AUTO_INDEX_LAST_ATTEMPT_MS.set(opts.repoRoot, nowMs);
|
|
1832
|
+
let inFlight = AUTO_INDEX_REBUILD_IN_FLIGHT.get(opts.repoRoot);
|
|
1833
|
+
if (!inFlight) {
|
|
1834
|
+
inFlight = (async () => {
|
|
1835
|
+
try {
|
|
1836
|
+
await runContextIndexRebuild({ repoRoot: opts.repoRoot, search: new URLSearchParams() });
|
|
1837
|
+
return true;
|
|
1838
|
+
}
|
|
1839
|
+
catch {
|
|
1840
|
+
return false;
|
|
1841
|
+
}
|
|
1842
|
+
finally {
|
|
1843
|
+
AUTO_INDEX_REBUILD_IN_FLIGHT.delete(opts.repoRoot);
|
|
1844
|
+
}
|
|
1845
|
+
})();
|
|
1846
|
+
AUTO_INDEX_REBUILD_IN_FLIGHT.set(opts.repoRoot, inFlight);
|
|
1847
|
+
}
|
|
1848
|
+
await inFlight;
|
|
1849
|
+
}
|
|
1850
|
+
function contextUrlFromSearch(search) {
|
|
1851
|
+
const query = search.toString();
|
|
1852
|
+
return new URL(`http://mu.local/context${query.length > 0 ? `?${query}` : ""}`);
|
|
1853
|
+
}
|
|
1854
|
+
export async function runContextSearch(opts) {
|
|
1855
|
+
const filters = parseSearchFilters(contextUrlFromSearch(opts.search));
|
|
1856
|
+
await maybeAutoRebuildContextIndex({
|
|
1857
|
+
repoRoot: opts.repoRoot,
|
|
1858
|
+
mode: opts.indexAutoRebuild ?? "off",
|
|
1859
|
+
});
|
|
1860
|
+
const indexed = readContextSearchFromIndex({ repoRoot: opts.repoRoot, filters });
|
|
1861
|
+
if (indexed) {
|
|
1862
|
+
return indexed;
|
|
1863
|
+
}
|
|
1864
|
+
const items = await collectContextItems(opts.repoRoot, filters.sources);
|
|
1865
|
+
const ranked = searchContext(items, filters);
|
|
1866
|
+
const sliced = ranked.slice(0, filters.limit);
|
|
1867
|
+
return {
|
|
1868
|
+
mode: "search",
|
|
1869
|
+
repo_root: opts.repoRoot,
|
|
1870
|
+
query: filters.query,
|
|
1871
|
+
count: sliced.length,
|
|
1872
|
+
total: ranked.length,
|
|
1873
|
+
items: sliced,
|
|
1874
|
+
};
|
|
1875
|
+
}
|
|
1876
|
+
export async function runContextTimeline(opts) {
|
|
1877
|
+
const filters = parseTimelineFilters(contextUrlFromSearch(opts.search));
|
|
1878
|
+
await maybeAutoRebuildContextIndex({
|
|
1879
|
+
repoRoot: opts.repoRoot,
|
|
1880
|
+
mode: opts.indexAutoRebuild ?? "off",
|
|
1881
|
+
});
|
|
1882
|
+
const indexed = readContextTimelineFromIndex({ repoRoot: opts.repoRoot, filters });
|
|
1883
|
+
if (indexed) {
|
|
1884
|
+
return indexed;
|
|
1885
|
+
}
|
|
1886
|
+
const items = await collectContextItems(opts.repoRoot, filters.sources);
|
|
1887
|
+
const timeline = timelineContext(items, filters);
|
|
1888
|
+
const sliced = timeline.slice(0, filters.limit);
|
|
1889
|
+
return {
|
|
1890
|
+
mode: "timeline",
|
|
1891
|
+
repo_root: opts.repoRoot,
|
|
1892
|
+
order: filters.order,
|
|
1893
|
+
count: sliced.length,
|
|
1894
|
+
total: timeline.length,
|
|
1895
|
+
items: sliced,
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
export async function runContextStats(opts) {
|
|
1899
|
+
const filters = parseSearchFilters(contextUrlFromSearch(opts.search));
|
|
1900
|
+
await maybeAutoRebuildContextIndex({
|
|
1901
|
+
repoRoot: opts.repoRoot,
|
|
1902
|
+
mode: opts.indexAutoRebuild ?? "off",
|
|
1903
|
+
});
|
|
1904
|
+
const indexed = readContextStatsFromIndex({ repoRoot: opts.repoRoot, filters });
|
|
1905
|
+
if (indexed) {
|
|
1906
|
+
return indexed;
|
|
1907
|
+
}
|
|
1908
|
+
const items = await collectContextItems(opts.repoRoot, filters.sources);
|
|
1909
|
+
const filtered = items.filter((item) => matchSearchFilters(item, { ...filters, query: null }));
|
|
1910
|
+
const sources = buildSourceStats(filtered);
|
|
1911
|
+
return {
|
|
1912
|
+
mode: "stats",
|
|
1913
|
+
repo_root: opts.repoRoot,
|
|
1914
|
+
total_count: filtered.length,
|
|
1915
|
+
total_text_bytes: filtered.reduce((sum, item) => sum + item.text.length, 0),
|
|
1916
|
+
sources,
|
|
1917
|
+
};
|
|
1918
|
+
}
|