@wipcomputer/memory-crystal 0.7.34-alpha.1 → 0.7.34-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cc-hook.js CHANGED
@@ -1,48 +1,2153 @@
1
1
  #!/usr/bin/env node
2
- import {
3
- createCrystal,
4
- resolveConfig
5
- } from "./chunk-2GBYLMEF.js";
6
- import {
7
- encryptJSON,
8
- loadRelayKey
9
- } from "./chunk-D3MACYZ4.js";
10
- import {
11
- ensureLdm,
12
- getAgentId,
13
- ldmPaths,
14
- resolveStatePath,
15
- stateWritePath
16
- } from "./chunk-DFQ72B7M.js";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropNames = Object.getOwnPropertyNames;
4
+ var __esm = (fn, res) => function __init() {
5
+ return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
6
+ };
7
+ var __export = (target, all) => {
8
+ for (var name in all)
9
+ __defProp(target, name, { get: all[name], enumerable: true });
10
+ };
11
+
12
+ // src/llm.ts
13
+ import { existsSync, readFileSync } from "fs";
14
+ import { join } from "path";
15
+ import { homedir } from "os";
16
+ import { execSync } from "child_process";
17
+ function dbCacheGet(key) {
18
+ if (!_cacheDb) return null;
19
+ try {
20
+ const row = _cacheDb.prepare(
21
+ "SELECT result FROM llm_cache WHERE cache_key = ? AND created_at > ?"
22
+ ).get(key, new Date(Date.now() - CACHE_TTL_DAYS * 864e5).toISOString());
23
+ if (row) {
24
+ _cacheDb.prepare("UPDATE llm_cache SET hit_count = hit_count + 1, last_hit_at = ? WHERE cache_key = ?").run((/* @__PURE__ */ new Date()).toISOString(), key);
25
+ return row.result;
26
+ }
27
+ } catch {
28
+ }
29
+ return null;
30
+ }
31
+ function dbCacheSet(key, type, query, intent, result, provider) {
32
+ if (!_cacheDb) return;
33
+ try {
34
+ _cacheDb.prepare(
35
+ "INSERT OR REPLACE INTO llm_cache (cache_key, cache_type, query, intent, result, provider, created_at, hit_count, last_hit_at) VALUES (?, ?, ?, ?, ?, ?, ?, 0, NULL)"
36
+ ).run(key, type, query, intent || null, result, provider, (/* @__PURE__ */ new Date()).toISOString());
37
+ } catch {
38
+ }
39
+ }
40
+ function getOpSecret(itemName, fieldLabel) {
41
+ try {
42
+ const saTokenPath = join(homedir(), ".openclaw/secrets/op-sa-token");
43
+ if (!existsSync(saTokenPath)) return void 0;
44
+ const saToken = readFileSync(saTokenPath, "utf-8").trim();
45
+ const result = execSync(
46
+ `OP_SERVICE_ACCOUNT_TOKEN="${saToken}" op item get "${itemName}" --vault "Agent Secrets" --fields "${fieldLabel}" --reveal`,
47
+ { encoding: "utf-8", timeout: 5e3, stdio: ["pipe", "pipe", "pipe"] }
48
+ ).trim();
49
+ return result || void 0;
50
+ } catch {
51
+ return void 0;
52
+ }
53
+ }
54
+ async function detectProvider() {
55
+ if (detectionDone && detectedProvider) return detectedProvider;
56
+ detectionDone = true;
57
+ if (samplingServer) {
58
+ detectedProvider = { provider: "sampling", baseURL: "", apiKey: "", model: "client-selected" };
59
+ process.stderr.write("[memory-crystal] LLM provider: MCP Sampling (via client)\n");
60
+ return detectedProvider;
61
+ }
62
+ try {
63
+ const resp = await fetch("http://localhost:18791/v1/models", { signal: AbortSignal.timeout(1e3) });
64
+ if (resp.ok) {
65
+ const data = await resp.json();
66
+ const model = data?.data?.[0]?.id || "default";
67
+ detectedProvider = { provider: "mlx", baseURL: "http://localhost:18791/v1", apiKey: "not-needed", model };
68
+ process.stderr.write(`[memory-crystal] LLM provider: MLX (${model})
69
+ `);
70
+ return detectedProvider;
71
+ }
72
+ } catch {
73
+ }
74
+ try {
75
+ const resp = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(1e3) });
76
+ if (resp.ok) {
77
+ const data = await resp.json();
78
+ const models = data?.models || [];
79
+ const embeddingOnly = ["nomic-embed-text", "mxbai-embed", "all-minilm", "snowflake-arctic-embed"];
80
+ const chatModel = models.find((m) => !embeddingOnly.some((e) => m.name.startsWith(e)));
81
+ if (chatModel) {
82
+ detectedProvider = { provider: "ollama", baseURL: "http://localhost:11434/v1", apiKey: "ollama", model: chatModel.name };
83
+ process.stderr.write(`[memory-crystal] LLM provider: Ollama (${chatModel.name})
84
+ `);
85
+ return detectedProvider;
86
+ }
87
+ }
88
+ } catch {
89
+ }
90
+ const openaiKey = process.env.OPENAI_API_KEY || getOpSecret("OpenAI API", "api key");
91
+ if (openaiKey) {
92
+ detectedProvider = { provider: "openai", baseURL: "https://api.openai.com/v1", apiKey: openaiKey, model: "gpt-4o-mini" };
93
+ process.stderr.write("[memory-crystal] LLM provider: OpenAI API\n");
94
+ return detectedProvider;
95
+ }
96
+ const anthropicKey = process.env.ANTHROPIC_API_KEY || getOpSecret("Anthropic Auth Token - remote bunkers", "Auth Token");
97
+ if (anthropicKey && !anthropicKey.startsWith("sk-ant-oat")) {
98
+ detectedProvider = { provider: "anthropic", baseURL: "https://api.anthropic.com", apiKey: anthropicKey, model: "claude-haiku-4-5-20251001" };
99
+ process.stderr.write("[memory-crystal] LLM provider: Anthropic API\n");
100
+ return detectedProvider;
101
+ }
102
+ detectedProvider = { provider: "none", baseURL: "", apiKey: "", model: "" };
103
+ process.stderr.write("[memory-crystal] LLM provider: none (deep search unavailable)\n");
104
+ return detectedProvider;
105
+ }
106
+ async function chatComplete(config, messages, maxTokens = 300) {
107
+ if (config.provider === "sampling") {
108
+ return samplingComplete(messages, maxTokens);
109
+ }
110
+ if (config.provider === "anthropic") {
111
+ return anthropicComplete(config, messages, maxTokens);
112
+ }
113
+ const resp = await fetch(`${config.baseURL}/chat/completions`, {
114
+ method: "POST",
115
+ headers: {
116
+ "Content-Type": "application/json",
117
+ "Authorization": `Bearer ${config.apiKey}`
118
+ },
119
+ body: JSON.stringify({
120
+ model: config.model,
121
+ messages,
122
+ max_tokens: maxTokens,
123
+ temperature: 0.7
124
+ })
125
+ });
126
+ if (!resp.ok) throw new Error(`LLM request failed: ${resp.status}`);
127
+ const data = await resp.json();
128
+ return data.choices?.[0]?.message?.content || "";
129
+ }
130
+ async function anthropicComplete(config, messages, maxTokens) {
131
+ const systemMsg = messages.find((m) => m.role === "system");
132
+ const userMessages = messages.filter((m) => m.role !== "system");
133
+ const body = {
134
+ model: config.model,
135
+ max_tokens: maxTokens,
136
+ messages: userMessages
137
+ };
138
+ if (systemMsg) body.system = systemMsg.content;
139
+ const resp = await fetch("https://api.anthropic.com/v1/messages", {
140
+ method: "POST",
141
+ headers: {
142
+ "Content-Type": "application/json",
143
+ "x-api-key": config.apiKey,
144
+ "anthropic-version": "2023-06-01"
145
+ },
146
+ body: JSON.stringify(body)
147
+ });
148
+ if (!resp.ok) throw new Error(`Anthropic request failed: ${resp.status}`);
149
+ const data = await resp.json();
150
+ return data.content?.[0]?.text || "";
151
+ }
152
+ async function samplingComplete(messages, maxTokens) {
153
+ if (!samplingServer) throw new Error("MCP sampling server not set");
154
+ const systemMsg = messages.find((m) => m.role === "system");
155
+ const userMessages = messages.filter((m) => m.role !== "system");
156
+ const result = await samplingServer.createMessage({
157
+ messages: userMessages.map((m) => ({
158
+ role: m.role,
159
+ content: { type: "text", text: m.content }
160
+ })),
161
+ systemPrompt: systemMsg?.content,
162
+ maxTokens,
163
+ modelPreferences: {
164
+ // Request cheap, fast model (Haiku-class). We don't need Opus for query expansion.
165
+ costPriority: 0.9,
166
+ speedPriority: 0.8,
167
+ intelligencePriority: 0.3,
168
+ hints: [{ name: "haiku" }]
169
+ }
170
+ });
171
+ if (result?.content?.type === "text") return result.content.text;
172
+ if (typeof result?.content === "string") return result.content;
173
+ return "";
174
+ }
175
+ async function expandQuery(query, intent) {
176
+ const cacheKey = intent ? `expand:${query}||${intent}` : `expand:${query}`;
177
+ const dbCached = dbCacheGet(cacheKey);
178
+ if (dbCached) {
179
+ try {
180
+ return JSON.parse(dbCached);
181
+ } catch {
182
+ }
183
+ }
184
+ const cached = expansionCache.get(cacheKey);
185
+ if (cached) return cached;
186
+ const config = await detectProvider();
187
+ if (config.provider === "none") return [];
188
+ try {
189
+ const intentContext = intent ? `
190
+ Query intent: ${intent}. Use this to guide your variations toward the intended domain.` : "";
191
+ const result = await chatComplete(config, [
192
+ { role: "system", content: EXPAND_PROMPT + intentContext },
193
+ { role: "user", content: query }
194
+ ], 300);
195
+ const lines = result.trim().split("\n");
196
+ const queryLower = query.toLowerCase();
197
+ const queryTerms = queryLower.replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
198
+ const hasQueryTerm = (text) => {
199
+ const lower = text.toLowerCase();
200
+ if (queryTerms.length === 0) return true;
201
+ return queryTerms.some((term) => lower.includes(term));
202
+ };
203
+ const variations = lines.map((line) => {
204
+ const colonIdx = line.indexOf(":");
205
+ if (colonIdx === -1) return null;
206
+ const type = line.slice(0, colonIdx).trim();
207
+ if (type !== "lex" && type !== "vec" && type !== "hyde") return null;
208
+ const text = line.slice(colonIdx + 1).trim();
209
+ if (!text || !hasQueryTerm(text)) return null;
210
+ return { type, text };
211
+ }).filter((v) => v !== null);
212
+ if (variations.length > 0) {
213
+ expansionCache.set(cacheKey, variations);
214
+ dbCacheSet(cacheKey, "expansion", query, intent, JSON.stringify(variations), config.provider);
215
+ return variations;
216
+ }
217
+ } catch (err) {
218
+ process.stderr.write(`[memory-crystal] Query expansion failed: ${err.message}
219
+ `);
220
+ }
221
+ const fallback = [
222
+ { type: "lex", text: query },
223
+ { type: "vec", text: query },
224
+ { type: "hyde", text: `Information about ${query}` }
225
+ ];
226
+ return fallback;
227
+ }
228
+ async function rerankResults(query, passages) {
229
+ const config = await detectProvider();
230
+ if (config.provider === "none") {
231
+ return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
232
+ }
233
+ const { createHash: createHash4 } = await import("crypto");
234
+ const contentHash = createHash4("sha256").update(passages.map((p) => p.slice(0, 200)).sort().join("|")).digest("hex").slice(0, 16);
235
+ const rerankCacheKey = `rerank:${query}||${contentHash}`;
236
+ const dbCachedRerank = dbCacheGet(rerankCacheKey);
237
+ if (dbCachedRerank) {
238
+ try {
239
+ return JSON.parse(dbCachedRerank);
240
+ } catch {
241
+ }
242
+ }
243
+ try {
244
+ const passageList = passages.map((p, i) => `[${i}] ${p.slice(0, 500)}`).join("\n\n");
245
+ const result = await chatComplete(config, [
246
+ { role: "system", content: RERANK_PROMPT },
247
+ { role: "user", content: `Query: ${query}
248
+
249
+ Passages:
250
+ ${passageList}` }
251
+ ], 200);
252
+ const results = [];
253
+ for (const line of result.trim().split("\n")) {
254
+ const match = line.match(/^(\d+):\s*([\d.]+)/);
255
+ if (match) {
256
+ results.push({ index: parseInt(match[1]), score: parseFloat(match[2]) });
257
+ }
258
+ }
259
+ const scored = new Set(results.map((r) => r.index));
260
+ for (let i = 0; i < passages.length; i++) {
261
+ if (!scored.has(i)) results.push({ index: i, score: 0 });
262
+ }
263
+ const sorted = results.sort((a, b) => b.score - a.score);
264
+ dbCacheSet(rerankCacheKey, "rerank", query, void 0, JSON.stringify(sorted), config.provider);
265
+ return sorted;
266
+ } catch (err) {
267
+ process.stderr.write(`[memory-crystal] Reranking failed: ${err.message}
268
+ `);
269
+ return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
270
+ }
271
+ }
272
+ var samplingServer, expansionCache, _cacheDb, CACHE_TTL_DAYS, detectedProvider, detectionDone, EXPAND_PROMPT, RERANK_PROMPT;
273
+ var init_llm = __esm({
274
+ "src/llm.ts"() {
275
+ "use strict";
276
+ samplingServer = null;
277
+ expansionCache = /* @__PURE__ */ new Map();
278
+ _cacheDb = null;
279
+ CACHE_TTL_DAYS = parseInt(process.env.CRYSTAL_CACHE_TTL_DAYS || "7", 10);
280
+ detectedProvider = null;
281
+ detectionDone = false;
282
+ EXPAND_PROMPT = `You are a search query expander. Given a search query, generate exactly 3 variations to improve search recall.
283
+
284
+ Output exactly 3 lines in this format (no other text):
285
+ lex: <keyword-focused variation for full-text search>
286
+ vec: <semantic variation rephrased for embedding similarity>
287
+ hyde: <hypothetical document snippet that would answer this query>
288
+
289
+ Rules:
290
+ - Each variation must contain at least one term from the original query
291
+ - Keep variations concise (under 30 words each)
292
+ - lex should use specific keywords and synonyms
293
+ - vec should rephrase the intent naturally
294
+ - hyde should be a short passage as if answering the query`;
295
+ RERANK_PROMPT = `You are a search result re-ranker. Given a query and a list of text passages, rate each passage's relevance to the query.
296
+
297
+ Output one line per passage in this exact format:
298
+ <index>: <score>
299
+
300
+ Where index is the passage number (0-based) and score is a float from 0.0 to 1.0.
301
+ - 1.0 = perfectly relevant, directly answers the query
302
+ - 0.7 = highly relevant, closely related
303
+ - 0.4 = somewhat relevant, tangentially related
304
+ - 0.1 = barely relevant
305
+ - 0.0 = not relevant at all
306
+
307
+ Rate ALL passages. Output nothing else.`;
308
+ }
309
+ });
310
+
311
+ // src/search-pipeline.ts
312
+ var search_pipeline_exports = {};
313
+ __export(search_pipeline_exports, {
314
+ deepSearch: () => deepSearch
315
+ });
316
+ async function deepSearch(crystal, query, options = {}) {
317
+ const limit = options.limit || 5;
318
+ const candidateLimit = options.candidateLimit || DEFAULT_CANDIDATE_LIMIT;
319
+ const intent = options.intent;
320
+ const filter = options.filter;
321
+ const explain = options.explain || false;
322
+ const provider = await detectProvider();
323
+ if (provider.provider === "none") {
324
+ return crystal.search(query, limit, filter);
325
+ }
326
+ const db = crystal.sqliteDb;
327
+ if (!db) return crystal.search(query, limit, filter);
328
+ const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
329
+ const untilDate = filter?.until ? crystal.parseSince(filter.until) : void 0;
330
+ const internalFilter = { ...filter, sinceDate, untilDate };
331
+ const initialFts = crystal.searchFTS(query, 20, internalFilter);
332
+ const topScore = initialFts[0]?.score ?? 0;
333
+ const secondScore = initialFts[1]?.score ?? 0;
334
+ const hasStrongSignal = !intent && initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
335
+ const expanded = hasStrongSignal ? [] : await expandQuery(query, intent);
336
+ const allResultLists = [];
337
+ if (initialFts.length > 0) allResultLists.push(initialFts);
338
+ const [queryEmbedding] = await crystal.embed([query]);
339
+ const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
340
+ if (originalVec.length > 0) allResultLists.push(originalVec);
341
+ for (const variation of expanded) {
342
+ if (variation.type === "lex") {
343
+ const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
344
+ if (ftsResults.length > 0) allResultLists.push(ftsResults);
345
+ } else {
346
+ const [embedding] = await crystal.embed([variation.text]);
347
+ const vecResults = crystal.searchVec(embedding, 20, internalFilter);
348
+ if (vecResults.length > 0) allResultLists.push(vecResults);
349
+ }
350
+ }
351
+ const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
352
+ const fused = crystal.reciprocalRankFusion(allResultLists, weights);
353
+ const candidates = fused.slice(0, candidateLimit);
354
+ if (candidates.length === 0) return [];
355
+ const ftsScoreMap = /* @__PURE__ */ new Map();
356
+ const vecScoreMap = /* @__PURE__ */ new Map();
357
+ if (explain) {
358
+ for (const r of initialFts) ftsScoreMap.set(r.text.slice(0, 200), r.score);
359
+ for (const r of originalVec) vecScoreMap.set(r.text.slice(0, 200), r.score);
360
+ }
361
+ const passages = candidates.map((c) => c.text.slice(0, 500));
362
+ const rerankQuery = intent ? `${intent}: ${query}` : query;
363
+ const reranked = await rerankResults(rerankQuery, passages);
364
+ const now = Date.now();
365
+ const blended = reranked.map((r) => {
366
+ const candidate = candidates[r.index];
367
+ if (!candidate) return null;
368
+ const rrfRank = r.index + 1;
369
+ let rrfWeight;
370
+ if (rrfRank <= 3) rrfWeight = 0.75;
371
+ else if (rrfRank <= 10) rrfWeight = 0.6;
372
+ else rrfWeight = 0.4;
373
+ const rrfScore = 1 / rrfRank;
374
+ const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
375
+ const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
376
+ const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
377
+ const finalScore = blendedScore * recency;
378
+ const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
379
+ const result = {
380
+ ...candidate,
381
+ score: finalScore,
382
+ freshness
383
+ };
384
+ if (explain) {
385
+ const dedup = candidate.text.slice(0, 200);
386
+ result.explain = {
387
+ fts_score: ftsScoreMap.get(dedup),
388
+ vec_score: vecScoreMap.get(dedup),
389
+ rrf_rank: rrfRank,
390
+ rrf_score: rrfScore,
391
+ rerank_score: r.score,
392
+ recency_weight: recency,
393
+ final_score: finalScore
394
+ };
395
+ }
396
+ return result;
397
+ }).filter((r) => r !== null);
398
+ const sorted = blended.sort((a, b) => b.score - a.score).slice(0, limit);
399
+ const topNormScore = sorted[0]?.score || 1;
400
+ return sorted.map((r) => ({ ...r, score: Math.min(r.score / topNormScore * 0.95, 0.95) }));
401
+ }
402
+ var STRONG_SIGNAL_MIN_SCORE, STRONG_SIGNAL_MIN_GAP, DEFAULT_CANDIDATE_LIMIT;
403
+ var init_search_pipeline = __esm({
404
+ "src/search-pipeline.ts"() {
405
+ "use strict";
406
+ init_llm();
407
+ STRONG_SIGNAL_MIN_SCORE = 0.85;
408
+ STRONG_SIGNAL_MIN_GAP = 0.15;
409
+ DEFAULT_CANDIDATE_LIMIT = 40;
410
+ }
411
+ });
412
+
413
+ // src/summarize.ts
414
+ var summarize_exports = {};
415
+ __export(summarize_exports, {
416
+ formatSummaryMarkdown: () => formatSummaryMarkdown,
417
+ generateSessionSummary: () => generateSessionSummary,
418
+ writeSummaryFile: () => writeSummaryFile
419
+ });
420
+ import { writeFileSync as writeFileSync2, existsSync as existsSync5, mkdirSync as mkdirSync3 } from "fs";
421
+ import { join as join4 } from "path";
422
+ import https2 from "https";
423
+ import http2 from "http";
424
+ function generateSimpleSummary(messages) {
425
+ const firstUser = messages.find((m) => m.role === "user");
426
+ const title = firstUser ? firstUser.text.slice(0, 80).replace(/\n/g, " ").trim() : "Untitled Session";
427
+ const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 50);
428
+ const preview = messages.slice(0, 10).map((m) => {
429
+ const roleLabel = m.role === "user" ? "User" : "Assistant";
430
+ const snippet = m.text.slice(0, 200).replace(/\n/g, " ").trim();
431
+ return `**${roleLabel}:** ${snippet}${m.text.length > 200 ? "..." : ""}`;
432
+ }).join("\n\n");
433
+ const date = messages[0]?.timestamp?.slice(0, 10) || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
434
+ return {
435
+ title,
436
+ slug,
437
+ summary: preview,
438
+ topics: [],
439
+ messageCount: messages.length,
440
+ date
441
+ };
442
+ }
443
+ async function generateLlmSummary(messages) {
444
+ const condensed = messages.slice(0, 30).map((m) => {
445
+ const roleLabel = m.role === "user" ? "User" : "Assistant";
446
+ const text = m.text.slice(0, 500);
447
+ return `${roleLabel}: ${text}`;
448
+ }).join("\n\n");
449
+ const prompt = `Summarize this conversation. Return JSON only, no markdown fences.
450
+
451
+ Format:
452
+ {"title": "short title", "slug": "url-safe-slug", "summary": "2-4 sentences", "topics": ["topic1", "topic2"]}
453
+
454
+ Conversation:
455
+ ${condensed}`;
456
+ const apiKey = process.env.OPENAI_API_KEY;
457
+ if (!apiKey) {
458
+ return generateSimpleSummary(messages);
459
+ }
460
+ try {
461
+ const body = JSON.stringify({
462
+ model: SUMMARY_MODEL,
463
+ messages: [{ role: "user", content: prompt }],
464
+ temperature: 0.3,
465
+ max_tokens: 300
466
+ });
467
+ const result = await httpPost("https://api.openai.com/v1/chat/completions", body, {
468
+ "Authorization": `Bearer ${apiKey}`,
469
+ "Content-Type": "application/json"
470
+ });
471
+ const parsed = JSON.parse(result);
472
+ const content = parsed.choices?.[0]?.message?.content || "";
473
+ const jsonStr = content.replace(/```json?\n?/g, "").replace(/```/g, "").trim();
474
+ const data = JSON.parse(jsonStr);
475
+ const date = messages[0]?.timestamp?.slice(0, 10) || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
476
+ return {
477
+ title: data.title || "Untitled",
478
+ slug: (data.slug || "untitled").slice(0, 50),
479
+ summary: data.summary || "",
480
+ topics: data.topics || [],
481
+ messageCount: messages.length,
482
+ date
483
+ };
484
+ } catch {
485
+ return generateSimpleSummary(messages);
486
+ }
487
+ }
488
+ function httpPost(url, body, headers) {
489
+ return new Promise((resolve, reject) => {
490
+ const parsed = new URL(url);
491
+ const client = parsed.protocol === "https:" ? https2 : http2;
492
+ const req = client.request({
493
+ hostname: parsed.hostname,
494
+ port: parsed.port,
495
+ path: parsed.pathname + parsed.search,
496
+ method: "POST",
497
+ headers: { ...headers, "Content-Length": Buffer.byteLength(body) },
498
+ timeout: 3e4
499
+ }, (res) => {
500
+ let data = "";
501
+ res.on("data", (chunk) => {
502
+ data += chunk;
503
+ });
504
+ res.on("end", () => {
505
+ if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
506
+ resolve(data);
507
+ } else {
508
+ reject(new Error(`HTTP ${res.statusCode}: ${data.slice(0, 200)}`));
509
+ }
510
+ });
511
+ });
512
+ req.on("error", reject);
513
+ req.on("timeout", () => {
514
+ req.destroy();
515
+ reject(new Error("Request timeout"));
516
+ });
517
+ req.write(body);
518
+ req.end();
519
+ });
520
+ }
521
+ async function generateSessionSummary(messages) {
522
+ if (SUMMARY_MODE === "llm") {
523
+ return generateLlmSummary(messages);
524
+ }
525
+ return generateSimpleSummary(messages);
526
+ }
527
+ function formatSummaryMarkdown(summary, sessionId) {
528
+ const lines = [];
529
+ lines.push(`# ${summary.title}`);
530
+ lines.push("");
531
+ lines.push(`**Session:** ${sessionId} **Date:** ${summary.date} **Messages:** ${summary.messageCount}`);
532
+ lines.push("");
533
+ lines.push("## Summary");
534
+ lines.push("");
535
+ lines.push(summary.summary);
536
+ if (summary.topics.length > 0) {
537
+ lines.push("");
538
+ lines.push("## Key Topics");
539
+ lines.push("");
540
+ for (const topic of summary.topics) {
541
+ lines.push(`- ${topic}`);
542
+ }
543
+ }
544
+ lines.push("");
545
+ return lines.join("\n");
546
+ }
547
+ function writeSummaryFile(sessionsDir, summary, agentId, sessionId) {
548
+ if (!existsSync5(sessionsDir)) mkdirSync3(sessionsDir, { recursive: true });
549
+ const now = /* @__PURE__ */ new Date();
550
+ const dateStr = now.toISOString().slice(0, 10);
551
+ const timeStr = now.toISOString().slice(11, 19).replace(/:/g, "-");
552
+ const filename = `${dateStr}--${timeStr}--${agentId}--${summary.slug}.md`;
553
+ const filepath = join4(sessionsDir, filename);
554
+ const content = formatSummaryMarkdown(summary, sessionId);
555
+ writeFileSync2(filepath, content);
556
+ return filepath;
557
+ }
558
+ var SUMMARY_MODE, SUMMARY_PROVIDER, SUMMARY_MODEL;
559
+ var init_summarize = __esm({
560
+ "src/summarize.ts"() {
561
+ "use strict";
562
+ SUMMARY_MODE = process.env.CRYSTAL_SUMMARY_MODE || "simple";
563
+ SUMMARY_PROVIDER = process.env.CRYSTAL_SUMMARY_PROVIDER || "openai";
564
+ SUMMARY_MODEL = process.env.CRYSTAL_SUMMARY_MODEL || "gpt-4o-mini";
565
+ }
566
+ });
567
+
568
+ // src/core.ts
569
+ import * as lancedb from "@lancedb/lancedb";
570
+ import Database from "better-sqlite3";
571
+ import * as sqliteVec from "sqlite-vec";
572
+ import { readFileSync as readFileSync2, existsSync as existsSync2, mkdirSync, readdirSync, statSync } from "fs";
573
+ import { execSync as execSync2 } from "child_process";
574
+ import { join as join2, relative, extname, basename } from "path";
575
+ import { createHash } from "crypto";
576
+ import http from "http";
577
+ import https from "https";
578
+ async function embedOpenAI(texts, apiKey, model) {
579
+ return new Promise((resolve, reject) => {
580
+ const body = JSON.stringify({ input: texts, model });
581
+ const req = https.request({
582
+ hostname: "api.openai.com",
583
+ path: "/v1/embeddings",
584
+ method: "POST",
585
+ headers: {
586
+ "Content-Type": "application/json",
587
+ "Authorization": `Bearer ${apiKey}`,
588
+ "Content-Length": Buffer.byteLength(body)
589
+ },
590
+ timeout: 3e4
591
+ }, (res) => {
592
+ let data = "";
593
+ res.on("data", (chunk) => data += chunk);
594
+ res.on("end", () => {
595
+ if (res.statusCode !== 200) {
596
+ reject(new Error(`OpenAI API error ${res.statusCode}: ${data.slice(0, 200)}`));
597
+ return;
598
+ }
599
+ const parsed = JSON.parse(data);
600
+ resolve(parsed.data.map((d) => d.embedding));
601
+ });
602
+ });
603
+ req.on("error", reject);
604
+ req.on("timeout", () => {
605
+ req.destroy();
606
+ reject(new Error("OpenAI timeout"));
607
+ });
608
+ req.write(body);
609
+ req.end();
610
+ });
611
+ }
612
+ async function embedOllama(texts, host, model) {
613
+ const results = [];
614
+ for (const text of texts) {
615
+ const result = await new Promise((resolve, reject) => {
616
+ const url = new URL("/api/embeddings", host);
617
+ const body = JSON.stringify({ model, prompt: text });
618
+ const req = http.request({
619
+ hostname: url.hostname,
620
+ port: url.port,
621
+ path: url.pathname,
622
+ method: "POST",
623
+ headers: {
624
+ "Content-Type": "application/json",
625
+ "Content-Length": Buffer.byteLength(body)
626
+ },
627
+ timeout: 15e3
628
+ }, (res) => {
629
+ let data = "";
630
+ res.on("data", (chunk) => data += chunk);
631
+ res.on("end", () => {
632
+ if (res.statusCode !== 200) {
633
+ reject(new Error(`Ollama error ${res.statusCode}: ${data.slice(0, 200)}`));
634
+ return;
635
+ }
636
+ resolve(JSON.parse(data).embedding);
637
+ });
638
+ });
639
+ req.on("error", reject);
640
+ req.on("timeout", () => {
641
+ req.destroy();
642
+ reject(new Error("Ollama timeout"));
643
+ });
644
+ req.write(body);
645
+ req.end();
646
+ });
647
+ results.push(result);
648
+ }
649
+ return results;
650
+ }
651
+ async function embedGoogle(texts, apiKey, model) {
652
+ return new Promise((resolve, reject) => {
653
+ const body = JSON.stringify({
654
+ requests: texts.map((text) => ({ model: `models/${model}`, content: { parts: [{ text }] } }))
655
+ });
656
+ const req = https.request({
657
+ hostname: "generativelanguage.googleapis.com",
658
+ path: `/v1beta/models/${model}:batchEmbedContents?key=${apiKey}`,
659
+ method: "POST",
660
+ headers: {
661
+ "Content-Type": "application/json",
662
+ "Content-Length": Buffer.byteLength(body)
663
+ },
664
+ timeout: 3e4
665
+ }, (res) => {
666
+ let data = "";
667
+ res.on("data", (chunk) => data += chunk);
668
+ res.on("end", () => {
669
+ if (res.statusCode !== 200) {
670
+ reject(new Error(`Google API error ${res.statusCode}: ${data.slice(0, 200)}`));
671
+ return;
672
+ }
673
+ const parsed = JSON.parse(data);
674
+ resolve(parsed.embeddings.map((e) => e.values));
675
+ });
676
+ });
677
+ req.on("error", reject);
678
+ req.on("timeout", () => {
679
+ req.destroy();
680
+ reject(new Error("Google timeout"));
681
+ });
682
+ req.write(body);
683
+ req.end();
684
+ });
685
+ }
686
+ var Crystal = class _Crystal {
687
+ config;
688
+ lanceDb = null;
689
+ sqliteDb = null;
690
+ chunksTable = null;
691
+ vecDimensions = null;
692
+ constructor(config) {
693
+ this.config = config;
694
+ if (!existsSync2(config.dataDir)) {
695
+ mkdirSync(config.dataDir, { recursive: true });
696
+ }
697
+ }
698
+ // ── Initialization ──
699
+ async init() {
700
+ const lanceDir = join2(this.config.dataDir, "lance");
701
+ const sqlitePath = join2(this.config.dataDir, "crystal.db");
702
+ if (!existsSync2(lanceDir)) mkdirSync(lanceDir, { recursive: true });
703
+ this.lanceDb = await lancedb.connect(lanceDir);
704
+ this.sqliteDb = new Database(sqlitePath);
705
+ this.sqliteDb.pragma("journal_mode = WAL");
706
+ sqliteVec.load(this.sqliteDb);
707
+ this.initSqliteTables();
708
+ this.initChunksTables();
709
+ await this.initLanceTables();
710
+ }
711
+ initSqliteTables() {
712
+ const db = this.sqliteDb;
713
+ db.exec(`
714
+ CREATE TABLE IF NOT EXISTS sources (
715
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
716
+ type TEXT NOT NULL,
717
+ uri TEXT NOT NULL,
718
+ title TEXT,
719
+ agent_id TEXT NOT NULL,
720
+ metadata TEXT DEFAULT '{}',
721
+ ingested_at TEXT NOT NULL,
722
+ chunk_count INTEGER DEFAULT 0
723
+ );
724
+
725
+ CREATE TABLE IF NOT EXISTS capture_state (
726
+ agent_id TEXT NOT NULL,
727
+ source_id TEXT NOT NULL,
728
+ last_message_count INTEGER DEFAULT 0,
729
+ capture_count INTEGER DEFAULT 0,
730
+ last_capture_at TEXT,
731
+ PRIMARY KEY (agent_id, source_id)
732
+ );
733
+
734
+ CREATE TABLE IF NOT EXISTS memories (
735
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
736
+ text TEXT NOT NULL,
737
+ category TEXT NOT NULL DEFAULT 'fact',
738
+ confidence REAL NOT NULL DEFAULT 1.0,
739
+ source_ids TEXT DEFAULT '[]',
740
+ status TEXT NOT NULL DEFAULT 'active',
741
+ created_at TEXT NOT NULL,
742
+ updated_at TEXT NOT NULL
743
+ );
744
+
745
+ CREATE TABLE IF NOT EXISTS entities (
746
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
747
+ name TEXT NOT NULL UNIQUE,
748
+ type TEXT NOT NULL DEFAULT 'concept',
749
+ description TEXT,
750
+ properties TEXT DEFAULT '{}',
751
+ created_at TEXT NOT NULL,
752
+ updated_at TEXT NOT NULL
753
+ );
754
+
755
+ CREATE TABLE IF NOT EXISTS relationships (
756
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
757
+ source_id INTEGER NOT NULL REFERENCES entities(id),
758
+ target_id INTEGER NOT NULL REFERENCES entities(id),
759
+ type TEXT NOT NULL,
760
+ description TEXT,
761
+ weight REAL DEFAULT 1.0,
762
+ valid_from TEXT NOT NULL,
763
+ valid_until TEXT,
764
+ created_at TEXT NOT NULL
765
+ );
766
+
767
+ CREATE INDEX IF NOT EXISTS idx_sources_agent ON sources(agent_id);
768
+ CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
769
+ CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
770
+ CREATE INDEX IF NOT EXISTS idx_relationships_source ON relationships(source_id);
771
+ CREATE INDEX IF NOT EXISTS idx_relationships_target ON relationships(target_id);
772
+
773
+ -- LLM cache (persistent expansion + reranking results)
774
+ CREATE TABLE IF NOT EXISTS llm_cache (
775
+ cache_key TEXT PRIMARY KEY,
776
+ cache_type TEXT NOT NULL,
777
+ query TEXT NOT NULL,
778
+ intent TEXT,
779
+ result TEXT NOT NULL,
780
+ provider TEXT NOT NULL,
781
+ created_at TEXT NOT NULL,
782
+ hit_count INTEGER DEFAULT 0,
783
+ last_hit_at TEXT
784
+ );
785
+ CREATE INDEX IF NOT EXISTS idx_llm_cache_type ON llm_cache(cache_type);
786
+ CREATE INDEX IF NOT EXISTS idx_llm_cache_created ON llm_cache(created_at);
787
+
788
+ -- Source file indexing (optional feature)
789
+ CREATE TABLE IF NOT EXISTS source_collections (
790
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
791
+ name TEXT NOT NULL UNIQUE,
792
+ root_path TEXT NOT NULL,
793
+ glob_patterns TEXT NOT NULL DEFAULT '["**/*"]',
794
+ ignore_patterns TEXT NOT NULL DEFAULT '[]',
795
+ file_count INTEGER DEFAULT 0,
796
+ chunk_count INTEGER DEFAULT 0,
797
+ last_sync_at TEXT,
798
+ created_at TEXT NOT NULL
799
+ );
800
+
801
+ CREATE TABLE IF NOT EXISTS source_files (
802
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
803
+ collection_id INTEGER NOT NULL REFERENCES source_collections(id) ON DELETE CASCADE,
804
+ file_path TEXT NOT NULL,
805
+ file_hash TEXT NOT NULL,
806
+ file_size INTEGER NOT NULL,
807
+ chunk_count INTEGER DEFAULT 0,
808
+ last_indexed_at TEXT NOT NULL
809
+ );
810
+
811
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_source_files_path ON source_files(collection_id, file_path);
812
+ CREATE INDEX IF NOT EXISTS idx_source_files_collection ON source_files(collection_id);
813
+ `);
814
+ }
815
+ initChunksTables() {
816
+ const db = this.sqliteDb;
817
+ db.exec(`
818
+ CREATE TABLE IF NOT EXISTS chunks (
819
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
820
+ text TEXT NOT NULL,
821
+ text_hash TEXT NOT NULL,
822
+ role TEXT,
823
+ source_type TEXT,
824
+ source_id TEXT,
825
+ agent_id TEXT,
826
+ token_count INTEGER,
827
+ created_at TEXT NOT NULL
828
+ );
829
+
830
+ CREATE INDEX IF NOT EXISTS idx_chunks_agent ON chunks(agent_id);
831
+ CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_type);
832
+ CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(text_hash);
833
+ CREATE INDEX IF NOT EXISTS idx_chunks_created ON chunks(created_at);
834
+
835
+ -- FTS5 full-text search table
836
+ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
837
+ text,
838
+ tokenize='porter unicode61'
839
+ );
840
+
841
+ -- Sync trigger: populate FTS on chunk insert
842
+ CREATE TRIGGER IF NOT EXISTS chunks_fts_insert AFTER INSERT ON chunks
843
+ BEGIN
844
+ INSERT INTO chunks_fts(rowid, text) VALUES (NEW.id, NEW.text);
845
+ END;
846
+
847
+ -- Sync trigger: clean up FTS and vec on chunk delete
848
+ CREATE TRIGGER IF NOT EXISTS chunks_cleanup AFTER DELETE ON chunks
849
+ BEGIN
850
+ DELETE FROM chunks_vec WHERE chunk_id = OLD.id;
851
+ INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', OLD.id, OLD.text);
852
+ END;
853
+ `);
854
+ const vecTable = db.prepare(
855
+ `SELECT name FROM sqlite_master WHERE type='table' AND name='chunks_vec'`
856
+ ).get();
857
+ if (vecTable) {
858
+ try {
859
+ const row = db.prepare("SELECT embedding FROM chunks_vec LIMIT 1").get();
860
+ if (row?.embedding) {
861
+ this.vecDimensions = row.embedding.length / 4;
862
+ }
863
+ } catch {
864
+ }
865
+ }
866
+ }
867
+ ensureVecTable(dimensions) {
868
+ const db = this.sqliteDb;
869
+ const existing = db.prepare(
870
+ `SELECT name FROM sqlite_master WHERE type='table' AND name='chunks_vec'`
871
+ ).get();
872
+ if (!existing) {
873
+ db.exec(`
874
+ CREATE VIRTUAL TABLE chunks_vec USING vec0(
875
+ chunk_id INTEGER PRIMARY KEY,
876
+ embedding float[${dimensions}] distance_metric=cosine
877
+ );
878
+ `);
879
+ }
880
+ this.vecDimensions = dimensions;
881
+ }
882
+ async initLanceTables() {
883
+ const db = this.lanceDb;
884
+ const tableNames = await db.tableNames();
885
+ if (tableNames.includes("chunks")) {
886
+ this.chunksTable = await db.openTable("chunks");
887
+ }
888
+ }
889
+ // ── Embedding ──
890
+ async embed(texts) {
891
+ if (texts.length === 0) return [];
892
+ const cfg = this.config;
893
+ switch (cfg.embeddingProvider) {
894
+ case "openai": {
895
+ if (!cfg.openaiApiKey) throw new Error("OpenAI API key required");
896
+ const model = cfg.openaiModel || "text-embedding-3-small";
897
+ const maxCharsPerBatch = 8e5;
898
+ const results = [];
899
+ let batch = [];
900
+ let batchChars = 0;
901
+ for (const text of texts) {
902
+ if (batchChars + text.length > maxCharsPerBatch && batch.length > 0) {
903
+ results.push(...await embedOpenAI(batch, cfg.openaiApiKey, model));
904
+ batch = [];
905
+ batchChars = 0;
906
+ }
907
+ batch.push(text);
908
+ batchChars += text.length;
909
+ }
910
+ if (batch.length > 0) {
911
+ results.push(...await embedOpenAI(batch, cfg.openaiApiKey, model));
912
+ }
913
+ return results;
914
+ }
915
+ case "ollama":
916
+ return embedOllama(texts, cfg.ollamaHost || "http://localhost:11434", cfg.ollamaModel || "nomic-embed-text");
917
+ case "google":
918
+ if (!cfg.googleApiKey) throw new Error("Google API key required");
919
+ return embedGoogle(texts, cfg.googleApiKey, cfg.googleModel || "text-embedding-004");
920
+ default:
921
+ throw new Error(`Unknown embedding provider: ${cfg.embeddingProvider}`);
922
+ }
923
+ }
924
+ // ── Chunking ──
925
+ chunkText(text, targetTokens = 400, overlapTokens = 80) {
926
+ const targetChars = targetTokens * 4;
927
+ const overlapChars = overlapTokens * 4;
928
+ const chunks = [];
929
+ let start = 0;
930
+ while (start < text.length) {
931
+ let end = Math.min(start + targetChars, text.length);
932
+ if (end < text.length) {
933
+ const minBreak = start + Math.floor(targetChars * 0.5);
934
+ const paraBreak = text.lastIndexOf("\n\n", end);
935
+ if (paraBreak > minBreak) {
936
+ end = paraBreak;
937
+ } else {
938
+ const sentBreak = text.lastIndexOf(". ", end);
939
+ if (sentBreak > minBreak) {
940
+ end = sentBreak + 1;
941
+ }
942
+ }
943
+ }
944
+ const chunk = text.slice(start, end).trim();
945
+ if (chunk.length > 0) chunks.push(chunk);
946
+ if (end >= text.length) break;
947
+ start = end - overlapChars;
948
+ if (start <= (chunks.length > 0 ? end - targetChars : 0)) {
949
+ start = end;
950
+ }
951
+ }
952
+ return chunks;
953
+ }
954
+ // ── Ingest ──
955
+ async ingest(chunks) {
956
+ if (chunks.length === 0) return 0;
957
+ const db = this.sqliteDb;
958
+ const newChunks = chunks.filter((c) => {
959
+ const hash = createHash("sha256").update(c.text).digest("hex");
960
+ return !db.prepare("SELECT 1 FROM chunks WHERE text_hash = ?").get(hash);
961
+ });
962
+ if (newChunks.length === 0) return 0;
963
+ const texts = newChunks.map((c) => c.text);
964
+ const embeddings = await this.embed(texts);
965
+ if (!this.vecDimensions && embeddings.length > 0) {
966
+ this.ensureVecTable(embeddings[0].length);
967
+ }
968
+ const insertChunk = db.prepare(`
969
+ INSERT INTO chunks (text, text_hash, role, source_type, source_id, agent_id, token_count, created_at)
970
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
971
+ `);
972
+ const insertVec = db.prepare(`
973
+ INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)
974
+ `);
975
+ const transaction = db.transaction(() => {
976
+ for (let i = 0; i < newChunks.length; i++) {
977
+ const c = newChunks[i];
978
+ const hash = createHash("sha256").update(c.text).digest("hex");
979
+ const result = insertChunk.run(
980
+ c.text,
981
+ hash,
982
+ c.role,
983
+ c.source_type,
984
+ c.source_id,
985
+ c.agent_id,
986
+ c.token_count,
987
+ c.created_at || (/* @__PURE__ */ new Date()).toISOString()
988
+ );
989
+ const chunkId = typeof result.lastInsertRowid === "bigint" ? result.lastInsertRowid : BigInt(result.lastInsertRowid);
990
+ insertVec.run(chunkId, new Float32Array(embeddings[i]));
991
+ }
992
+ });
993
+ transaction();
994
+ const records = newChunks.map((chunk, i) => ({
995
+ text: chunk.text,
996
+ vector: embeddings[i],
997
+ role: chunk.role,
998
+ source_type: chunk.source_type,
999
+ source_id: chunk.source_id,
1000
+ agent_id: chunk.agent_id,
1001
+ token_count: chunk.token_count,
1002
+ created_at: chunk.created_at || (/* @__PURE__ */ new Date()).toISOString()
1003
+ }));
1004
+ try {
1005
+ if (!this.chunksTable) {
1006
+ this.chunksTable = await this.lanceDb.createTable("chunks", records);
1007
+ } else {
1008
+ await this.chunksTable.add(records);
1009
+ }
1010
+ } catch (err) {
1011
+ console.warn("LanceDB dual-write failed (non-fatal):", err.message);
1012
+ }
1013
+ return newChunks.length;
1014
+ }
1015
+ // ── Delta Sync (export/import pre-embedded chunks) ──
1016
+ /** Export interface for delta sync payloads. */
1017
+ static DELTA_VERSION = 1;
1018
+ /** Export chunks with IDs greater than sinceId. Returns pre-embedded chunks for delta sync.
1019
+ * Core calls this to build delta payloads for Nodes. */
1020
+ exportChunksSince(sinceId) {
1021
+ const db = this.sqliteDb;
1022
+ const rows = db.prepare(`
1023
+ SELECT c.id, c.text, c.text_hash, c.role, c.source_type, c.source_id,
1024
+ c.agent_id, c.token_count, c.created_at, v.embedding
1025
+ FROM chunks c
1026
+ LEFT JOIN chunks_vec v ON v.chunk_id = c.id
1027
+ WHERE c.id > ?
1028
+ ORDER BY c.id ASC
1029
+ `).all(sinceId);
1030
+ return rows.map((row) => ({
1031
+ id: row.id,
1032
+ text: row.text,
1033
+ text_hash: row.text_hash,
1034
+ role: row.role,
1035
+ source_type: row.source_type,
1036
+ source_id: row.source_id,
1037
+ agent_id: row.agent_id,
1038
+ token_count: row.token_count,
1039
+ created_at: row.created_at,
1040
+ // Convert Float32Array buffer to number[] for JSON serialization
1041
+ embedding: row.embedding ? Array.from(new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 4)) : null
1042
+ }));
1043
+ }
1044
+ /** Get the highest chunk ID in the database. Used for watermark tracking. */
1045
+ getMaxChunkId() {
1046
+ const db = this.sqliteDb;
1047
+ const row = db.prepare("SELECT MAX(id) as maxId FROM chunks").get();
1048
+ return row.maxId || 0;
1049
+ }
1050
+ /** Import pre-embedded chunks from Core. Node calls this to apply delta payloads.
1051
+ * Skips chunks that already exist (by text_hash). Does NOT re-embed. */
1052
+ importChunks(exported) {
1053
+ if (exported.length === 0) return 0;
1054
+ const db = this.sqliteDb;
1055
+ const firstWithEmbed = exported.find((c) => c.embedding && c.embedding.length > 0);
1056
+ if (firstWithEmbed && !this.vecDimensions) {
1057
+ this.ensureVecTable(firstWithEmbed.embedding.length);
1058
+ }
1059
+ const insertChunk = db.prepare(`
1060
+ INSERT INTO chunks (text, text_hash, role, source_type, source_id, agent_id, token_count, created_at)
1061
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1062
+ `);
1063
+ const insertVec = db.prepare(`
1064
+ INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)
1065
+ `);
1066
+ const checkHash = db.prepare("SELECT 1 FROM chunks WHERE text_hash = ?");
1067
+ let imported = 0;
1068
+ const transaction = db.transaction(() => {
1069
+ for (const chunk of exported) {
1070
+ if (checkHash.get(chunk.text_hash)) continue;
1071
+ const result = insertChunk.run(
1072
+ chunk.text,
1073
+ chunk.text_hash,
1074
+ chunk.role,
1075
+ chunk.source_type,
1076
+ chunk.source_id,
1077
+ chunk.agent_id,
1078
+ chunk.token_count,
1079
+ chunk.created_at
1080
+ );
1081
+ if (chunk.embedding && chunk.embedding.length > 0) {
1082
+ const chunkId = typeof result.lastInsertRowid === "bigint" ? result.lastInsertRowid : BigInt(result.lastInsertRowid);
1083
+ insertVec.run(chunkId, new Float32Array(chunk.embedding));
1084
+ }
1085
+ imported++;
1086
+ }
1087
+ });
1088
+ transaction();
1089
+ return imported;
1090
+ }
1091
+ // ── Recency helpers ──
1092
+ recencyWeight(ageDays) {
1093
+ return Math.max(0.3, Math.exp(-ageDays * 0.1));
1094
+ }
1095
+ /** Parse relative time strings ("24h", "7d", "30d") or ISO dates into ISO date strings. */
1096
+ parseSince(since) {
1097
+ const match = since.match(/^(\d+)(h|d)$/);
1098
+ if (match) {
1099
+ const [, num, unit] = match;
1100
+ const ms = unit === "h" ? parseInt(num) * 36e5 : parseInt(num) * 864e5;
1101
+ return new Date(Date.now() - ms).toISOString();
1102
+ }
1103
+ const parsed = new Date(since);
1104
+ if (!isNaN(parsed.getTime())) return parsed.toISOString();
1105
+ return void 0;
1106
+ }
1107
+ freshnessLabel(ageDays) {
1108
+ if (ageDays < 3) return "fresh";
1109
+ if (ageDays < 7) return "recent";
1110
+ if (ageDays < 14) return "aging";
1111
+ return "stale";
1112
+ }
1113
+ // ── Search (Hybrid: BM25 + Vector + RRF fusion + Recency) ──
1114
+ async search(query, limit = 5, filter) {
1115
+ const db = this.sqliteDb;
1116
+ const sqliteChunks = db.prepare("SELECT COUNT(*) as count FROM chunks").get()?.count || 0;
1117
+ let lanceChunks = 0;
1118
+ if (this.chunksTable) {
1119
+ try {
1120
+ lanceChunks = await this.chunksTable.countRows();
1121
+ } catch {
1122
+ }
1123
+ }
1124
+ if (sqliteChunks === 0 || lanceChunks > 0 && sqliteChunks < lanceChunks * 0.5) {
1125
+ return this.searchLanceFallback(query, limit, filter);
1126
+ }
1127
+ const sinceDate = filter?.since ? this.parseSince(filter.since) : void 0;
1128
+ const untilDate = filter?.until ? this.parseSince(filter.until) : void 0;
1129
+ const [embedding] = await this.embed([query]);
1130
+ const fetchLimit = Math.max(limit * 5, 50);
1131
+ const vecResults = this.searchVec(embedding, fetchLimit, { ...filter, sinceDate, untilDate });
1132
+ const ftsResults = this.searchFTS(query, fetchLimit, { ...filter, sinceDate, untilDate });
1133
+ const fused = this.reciprocalRankFusion([ftsResults, vecResults], [2, 1]);
1134
+ const now = Date.now();
1135
+ const scored = fused.map((r) => {
1136
+ const ageDays = r.created_at ? (now - new Date(r.created_at).getTime()) / 864e5 : 0;
1137
+ const recency = r.created_at ? this.recencyWeight(ageDays) : 1;
1138
+ const rescaled = r.score * recency;
1139
+ return {
1140
+ ...r,
1141
+ score: rescaled,
1142
+ freshness: r.created_at ? this.freshnessLabel(ageDays) : void 0
1143
+ };
1144
+ });
1145
+ const sorted = scored.sort((a, b) => b.score - a.score).slice(0, limit);
1146
+ const topScore = sorted[0]?.score || 1;
1147
+ return sorted.map((r) => ({ ...r, score: Math.min(r.score / topScore * 0.95, 0.95) }));
1148
+ }
1149
+ /** Deep search: query expansion + LLM re-ranking + position-aware blending.
1150
+ * Falls back to standard search if no LLM provider is available.
1151
+ * Supports intent disambiguation, candidateLimit tuning, and explain traces. */
1152
+ async deepSearch(query, limit = 5, filter, options) {
1153
+ const { deepSearch: deepSearchFn } = await Promise.resolve().then(() => (init_search_pipeline(), search_pipeline_exports));
1154
+ return deepSearchFn(this, query, { limit, filter, ...options });
1155
+ }
1156
+ /** Structured search: pass pre-expanded queries to skip LLM expansion.
1157
+ * Each query is typed (lex, vec, hyde) and searched independently, then fused with RRF. */
1158
+ async structuredSearch(queries, limit = 5, filter) {
1159
+ const db = this.sqliteDb;
1160
+ const sinceDate = filter?.since ? this.parseSince(filter.since) : void 0;
1161
+ const untilDate = filter?.until ? this.parseSince(filter.until) : void 0;
1162
+ const internalFilter = { ...filter, sinceDate, untilDate };
1163
+ const allResultLists = [];
1164
+ for (const q of queries) {
1165
+ if (q.type === "lex") {
1166
+ const fts = this.searchFTS(q.text, Math.max(limit * 5, 50), internalFilter);
1167
+ if (fts.length > 0) allResultLists.push(fts);
1168
+ } else {
1169
+ const [embedding] = await this.embed([q.text]);
1170
+ const vec = this.searchVec(embedding, Math.max(limit * 5, 50), internalFilter);
1171
+ if (vec.length > 0) allResultLists.push(vec);
1172
+ }
1173
+ }
1174
+ const weights = allResultLists.map((_, i) => i === 0 ? 2 : 1);
1175
+ const fused = this.reciprocalRankFusion(allResultLists, weights);
1176
+ const now = Date.now();
1177
+ const scored = fused.map((r) => {
1178
+ const ageDays = r.created_at ? (now - new Date(r.created_at).getTime()) / 864e5 : 0;
1179
+ const recency = r.created_at ? this.recencyWeight(ageDays) : 1;
1180
+ return { ...r, score: r.score * recency, freshness: r.created_at ? this.freshnessLabel(ageDays) : void 0 };
1181
+ });
1182
+ const sorted = scored.sort((a, b) => b.score - a.score).slice(0, limit);
1183
+ const topScore = sorted[0]?.score || 1;
1184
+ return sorted.map((r) => ({ ...r, score: Math.min(r.score / topScore * 0.95, 0.95) }));
1185
+ }
1186
+ /** Vector search via sqlite-vec. Two-step pattern: MATCH first, then JOIN. */
1187
+ searchVec(embedding, limit, filter) {
1188
+ const db = this.sqliteDb;
1189
+ if (!this.vecDimensions) return [];
1190
+ const vecRows = db.prepare(`
1191
+ SELECT chunk_id, distance
1192
+ FROM chunks_vec
1193
+ WHERE embedding MATCH ? AND k = ?
1194
+ `).all(new Float32Array(embedding), limit);
1195
+ if (vecRows.length === 0) return [];
1196
+ const ids = vecRows.map((r) => r.chunk_id);
1197
+ const distMap = new Map(vecRows.map((r) => [r.chunk_id, r.distance]));
1198
+ const placeholders = ids.map(() => "?").join(",");
1199
+ let sql = `SELECT id, text, role, source_type, source_id, agent_id, created_at FROM chunks WHERE id IN (${placeholders})`;
1200
+ const params = [...ids];
1201
+ if (filter?.agent_id) {
1202
+ sql += " AND agent_id = ?";
1203
+ params.push(filter.agent_id);
1204
+ }
1205
+ if (filter?.source_type) {
1206
+ sql += " AND source_type = ?";
1207
+ params.push(filter.source_type);
1208
+ }
1209
+ if (filter?.sinceDate) {
1210
+ sql += " AND created_at >= ?";
1211
+ params.push(filter.sinceDate);
1212
+ }
1213
+ if (filter?.untilDate) {
1214
+ sql += " AND created_at < ?";
1215
+ params.push(filter.untilDate);
1216
+ }
1217
+ const rows = db.prepare(sql).all(...params);
1218
+ return rows.map((row) => ({
1219
+ text: row.text,
1220
+ role: row.role,
1221
+ score: 1 - (distMap.get(row.id) || 1),
1222
+ // cosine similarity from distance
1223
+ source_type: row.source_type,
1224
+ source_id: row.source_id,
1225
+ agent_id: row.agent_id,
1226
+ created_at: row.created_at
1227
+ }));
1228
+ }
1229
+ /** Full-text search via FTS5 with BM25 scoring. */
1230
+ searchFTS(query, limit, filter) {
1231
+ const db = this.sqliteDb;
1232
+ const ftsQuery = this.buildFTS5Query(query);
1233
+ if (!ftsQuery) return [];
1234
+ let sql = `
1235
+ SELECT c.id, c.text, c.role, c.source_type, c.source_id, c.agent_id, c.created_at,
1236
+ bm25(chunks_fts) as bm25_score
1237
+ FROM chunks_fts f
1238
+ JOIN chunks c ON c.id = f.rowid
1239
+ WHERE chunks_fts MATCH ?
1240
+ `;
1241
+ const params = [ftsQuery];
1242
+ if (filter?.agent_id) {
1243
+ sql += " AND c.agent_id = ?";
1244
+ params.push(filter.agent_id);
1245
+ }
1246
+ if (filter?.source_type) {
1247
+ sql += " AND c.source_type = ?";
1248
+ params.push(filter.source_type);
1249
+ }
1250
+ if (filter?.sinceDate) {
1251
+ sql += " AND c.created_at >= ?";
1252
+ params.push(filter.sinceDate);
1253
+ }
1254
+ if (filter?.untilDate) {
1255
+ sql += " AND c.created_at < ?";
1256
+ params.push(filter.untilDate);
1257
+ }
1258
+ sql += " ORDER BY bm25_score LIMIT ?";
1259
+ params.push(limit);
1260
+ const rows = db.prepare(sql).all(...params);
1261
+ return rows.map((row) => ({
1262
+ text: row.text,
1263
+ role: row.role,
1264
+ // BM25 scores are negative (lower = better). Normalize to [0..1).
1265
+ // |x| / (1 + |x|) maps: strong(-10)->0.91, medium(-2)->0.67, weak(-0.5)->0.33
1266
+ score: Math.abs(row.bm25_score) / (1 + Math.abs(row.bm25_score)),
1267
+ source_type: row.source_type,
1268
+ source_id: row.source_id,
1269
+ agent_id: row.agent_id,
1270
+ created_at: row.created_at
1271
+ }));
1272
+ }
1273
+ /** Build a safe FTS5 query from user input. */
1274
+ buildFTS5Query(query) {
1275
+ const terms = query.split(/\s+/).map((t) => t.replace(/[^\p{L}\p{N}']/gu, "").toLowerCase()).filter((t) => t.length > 0);
1276
+ if (terms.length === 0) return null;
1277
+ if (terms.length === 1) return `"${terms[0]}"*`;
1278
+ return terms.map((t) => `"${t}"*`).join(" AND ");
1279
+ }
1280
+ /**
1281
+ * Reciprocal Rank Fusion. Ported from QMD (MIT License, Tobi Lutke, 2024-2026).
1282
+ * Fuses multiple ranked result lists into one using RRF scoring.
1283
+ * Uses text content as dedup key (instead of QMD's file path).
1284
+ */
1285
+ reciprocalRankFusion(resultLists, weights = [], k = 60) {
1286
+ const scores = /* @__PURE__ */ new Map();
1287
+ for (let listIdx = 0; listIdx < resultLists.length; listIdx++) {
1288
+ const list = resultLists[listIdx];
1289
+ if (!list) continue;
1290
+ const weight = weights[listIdx] ?? 1;
1291
+ for (let rank = 0; rank < list.length; rank++) {
1292
+ const result = list[rank];
1293
+ if (!result) continue;
1294
+ const rrfContribution = weight / (k + rank + 1);
1295
+ const dedup = result.text.slice(0, 200);
1296
+ const existing = scores.get(dedup);
1297
+ if (existing) {
1298
+ existing.rrfScore += rrfContribution;
1299
+ existing.topRank = Math.min(existing.topRank, rank);
1300
+ } else {
1301
+ scores.set(dedup, {
1302
+ result,
1303
+ rrfScore: rrfContribution,
1304
+ topRank: rank
1305
+ });
1306
+ }
1307
+ }
1308
+ }
1309
+ for (const entry of scores.values()) {
1310
+ if (entry.topRank === 0) {
1311
+ entry.rrfScore += 0.05;
1312
+ } else if (entry.topRank <= 2) {
1313
+ entry.rrfScore += 0.02;
1314
+ }
1315
+ }
1316
+ return Array.from(scores.values()).sort((a, b) => b.rrfScore - a.rrfScore).map((e) => ({ ...e.result, score: e.rrfScore }));
1317
+ }
1318
+ /** LanceDB fallback for search (used when sqlite-vec tables are empty, pre-migration). */
1319
+ async searchLanceFallback(query, limit, filter) {
1320
+ if (!this.chunksTable) return [];
1321
+ const [embedding] = await this.embed([query]);
1322
+ const fetchLimit = Math.max(limit * 3, 30);
1323
+ let queryBuilder = this.chunksTable.vectorSearch(embedding).distanceType("cosine").limit(fetchLimit);
1324
+ if (filter?.agent_id) {
1325
+ queryBuilder = queryBuilder.where(`agent_id = '${filter.agent_id}'`);
1326
+ }
1327
+ if (filter?.source_type) {
1328
+ queryBuilder = queryBuilder.where(`source_type = '${filter.source_type}'`);
1329
+ }
1330
+ const results = await queryBuilder.toArray();
1331
+ const now = Date.now();
1332
+ return results.map((row) => {
1333
+ const cosine = row._distance != null ? 1 - row._distance : 0;
1334
+ const createdAt = row.created_at || "";
1335
+ const ageDays = createdAt ? (now - new Date(createdAt).getTime()) / 864e5 : 0;
1336
+ const weight = createdAt ? this.recencyWeight(ageDays) : 1;
1337
+ return {
1338
+ text: row.text,
1339
+ role: row.role,
1340
+ score: cosine * weight,
1341
+ source_type: row.source_type,
1342
+ source_id: row.source_id,
1343
+ agent_id: row.agent_id,
1344
+ created_at: createdAt,
1345
+ freshness: createdAt ? this.freshnessLabel(ageDays) : void 0
1346
+ };
1347
+ }).sort((a, b) => b.score - a.score).slice(0, limit);
1348
+ }
1349
+ // ── Remember (explicit fact storage) ──
1350
+ async remember(text, category = "fact") {
1351
+ const db = this.sqliteDb;
1352
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1353
+ const stmt = db.prepare(`
1354
+ INSERT INTO memories (text, category, confidence, source_ids, status, created_at, updated_at)
1355
+ VALUES (?, ?, 1.0, '[]', 'active', ?, ?)
1356
+ `);
1357
+ const result = stmt.run(text, category, now, now);
1358
+ await this.ingest([{
1359
+ text,
1360
+ role: "system",
1361
+ source_type: "manual",
1362
+ source_id: `memory:${result.lastInsertRowid}`,
1363
+ agent_id: "system",
1364
+ token_count: Math.ceil(text.length / 4),
1365
+ created_at: now
1366
+ }]);
1367
+ return result.lastInsertRowid;
1368
+ }
1369
+ // ── Forget (deprecate a memory) ──
1370
+ forget(memoryId) {
1371
+ const db = this.sqliteDb;
1372
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1373
+ const result = db.prepare(`
1374
+ UPDATE memories SET status = 'deprecated', updated_at = ? WHERE id = ? AND status = 'active'
1375
+ `).run(now, memoryId);
1376
+ return result.changes > 0;
1377
+ }
1378
+ // ── Status ──
1379
+ async status() {
1380
+ const db = this.sqliteDb;
1381
+ const sqliteChunks = db.prepare("SELECT COUNT(*) as count FROM chunks").get()?.count || 0;
1382
+ let lanceChunks = 0;
1383
+ if (this.chunksTable) {
1384
+ try {
1385
+ lanceChunks = await this.chunksTable.countRows();
1386
+ } catch {
1387
+ }
1388
+ }
1389
+ const chunks = Math.max(sqliteChunks, lanceChunks);
1390
+ const oldest = db.prepare("SELECT MIN(created_at) as ts FROM chunks").get()?.ts || null;
1391
+ const newest = db.prepare("SELECT MAX(created_at) as ts FROM chunks").get()?.ts || null;
1392
+ const memories = db.prepare("SELECT COUNT(*) as count FROM memories WHERE status = ?").get("active")?.count || 0;
1393
+ const sources = db.prepare("SELECT COUNT(*) as count FROM sources").get()?.count || 0;
1394
+ const chunkAgentRows = db.prepare("SELECT DISTINCT agent_id FROM chunks WHERE agent_id IS NOT NULL").all();
1395
+ const sourceAgentRows = db.prepare("SELECT DISTINCT agent_id FROM sources").all();
1396
+ const captureAgentRows = db.prepare("SELECT DISTINCT agent_id FROM capture_state").all();
1397
+ const agents = [.../* @__PURE__ */ new Set([
1398
+ ...chunkAgentRows.map((r) => r.agent_id),
1399
+ ...sourceAgentRows.map((r) => r.agent_id),
1400
+ ...captureAgentRows.map((r) => r.agent_id)
1401
+ ])];
1402
+ const captureInfo = db.prepare(
1403
+ "SELECT COUNT(*) as count, MAX(last_capture_at) as latest FROM capture_state"
1404
+ ).get();
1405
+ return {
1406
+ chunks,
1407
+ memories,
1408
+ sources,
1409
+ agents,
1410
+ oldestChunk: oldest,
1411
+ newestChunk: newest,
1412
+ embeddingProvider: this.config.embeddingProvider,
1413
+ dataDir: this.config.dataDir,
1414
+ capturedSessions: captureInfo?.count || 0,
1415
+ latestCapture: captureInfo?.latest || null
1416
+ };
1417
+ }
1418
+ // ── Capture State (for incremental ingestion) ──
1419
+ getCaptureState(agentId, sourceId) {
1420
+ const db = this.sqliteDb;
1421
+ const row = db.prepare("SELECT last_message_count, capture_count FROM capture_state WHERE agent_id = ? AND source_id = ?").get(agentId, sourceId);
1422
+ if (!row) return { lastMessageCount: 0, captureCount: 0 };
1423
+ return {
1424
+ lastMessageCount: row.last_message_count,
1425
+ captureCount: row.capture_count
1426
+ };
1427
+ }
1428
+ setCaptureState(agentId, sourceId, messageCount, captureCount) {
1429
+ const db = this.sqliteDb;
1430
+ db.prepare(`
1431
+ INSERT OR REPLACE INTO capture_state (agent_id, source_id, last_message_count, capture_count, last_capture_at)
1432
+ VALUES (?, ?, ?, ?, ?)
1433
+ `).run(agentId, sourceId, messageCount, captureCount, (/* @__PURE__ */ new Date()).toISOString());
1434
+ }
1435
+ // ── Source File Indexing (optional feature) ──
1436
+ //
1437
+ // Add directories as "collections", sync to index/re-index changed files.
1438
+ // All source chunks get source_type='file' so they're searchable alongside
1439
+ // conversations and memories. Nothing here is required... you can use MC
1440
+ // without ever touching sources.
1441
+ // Default patterns for files worth indexing
1442
+ static DEFAULT_INCLUDE = [
1443
+ "**/*.ts",
1444
+ "**/*.js",
1445
+ "**/*.tsx",
1446
+ "**/*.jsx",
1447
+ "**/*.py",
1448
+ "**/*.rs",
1449
+ "**/*.go",
1450
+ "**/*.java",
1451
+ "**/*.md",
1452
+ "**/*.txt",
1453
+ "**/*.json",
1454
+ "**/*.yaml",
1455
+ "**/*.yml",
1456
+ "**/*.toml",
1457
+ "**/*.sh",
1458
+ "**/*.bash",
1459
+ "**/*.zsh",
1460
+ "**/*.css",
1461
+ "**/*.html",
1462
+ "**/*.svg",
1463
+ "**/*.sql",
1464
+ "**/*.graphql",
1465
+ "**/*.c",
1466
+ "**/*.cpp",
1467
+ "**/*.h",
1468
+ "**/*.hpp",
1469
+ "**/*.swift",
1470
+ "**/*.kt",
1471
+ "**/*.rb",
1472
+ "**/*.env.example",
1473
+ "**/*.gitignore",
1474
+ "**/Makefile",
1475
+ "**/Dockerfile",
1476
+ "**/Cargo.toml",
1477
+ "**/package.json",
1478
+ "**/tsconfig.json"
1479
+ ];
1480
+ static DEFAULT_IGNORE = [
1481
+ "**/node_modules/**",
1482
+ "**/.git/**",
1483
+ "**/dist/**",
1484
+ "**/build/**",
1485
+ "**/.next/**",
1486
+ "**/.cache/**",
1487
+ "**/coverage/**",
1488
+ "**/__pycache__/**",
1489
+ "**/target/**",
1490
+ "**/vendor/**",
1491
+ "**/.venv/**",
1492
+ "**/*.lock",
1493
+ "**/package-lock.json",
1494
+ "**/yarn.lock",
1495
+ "**/bun.lockb",
1496
+ "**/*.min.js",
1497
+ "**/*.min.css",
1498
+ "**/*.map",
1499
+ "**/*.png",
1500
+ "**/*.jpg",
1501
+ "**/*.jpeg",
1502
+ "**/*.gif",
1503
+ "**/*.ico",
1504
+ "**/*.webp",
1505
+ "**/*.woff",
1506
+ "**/*.woff2",
1507
+ "**/*.ttf",
1508
+ "**/*.eot",
1509
+ "**/*.mp3",
1510
+ "**/*.mp4",
1511
+ "**/*.wav",
1512
+ "**/*.ogg",
1513
+ "**/*.webm",
1514
+ "**/*.zip",
1515
+ "**/*.tar",
1516
+ "**/*.gz",
1517
+ "**/*.br",
1518
+ "**/*.sqlite",
1519
+ "**/*.db",
1520
+ "**/*.lance/**",
1521
+ "**/*.jsonl",
1522
+ "**/secrets/**",
1523
+ "**/.env"
1524
+ ];
1525
+ /** Add a directory as a source collection for indexing. */
1526
+ async sourcesAdd(rootPath, name, options) {
1527
+ const db = this.sqliteDb;
1528
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1529
+ const includePatterns = JSON.stringify(options?.include || _Crystal.DEFAULT_INCLUDE);
1530
+ const ignorePatterns = JSON.stringify(options?.ignore || _Crystal.DEFAULT_IGNORE);
1531
+ const existing = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
1532
+ if (existing) {
1533
+ throw new Error(`Collection "${name}" already exists. Use sourcesSync() to update it.`);
1534
+ }
1535
+ db.prepare(`
1536
+ INSERT INTO source_collections (name, root_path, glob_patterns, ignore_patterns, created_at)
1537
+ VALUES (?, ?, ?, ?, ?)
1538
+ `).run(name, rootPath, includePatterns, ignorePatterns, now);
1539
+ const row = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
1540
+ return row;
1541
+ }
1542
+ /** Remove a source collection and its file records. Chunks remain in LanceDB. */
1543
+ sourcesRemove(name) {
1544
+ const db = this.sqliteDb;
1545
+ const col = db.prepare("SELECT id FROM source_collections WHERE name = ?").get(name);
1546
+ if (!col) return false;
1547
+ db.prepare("DELETE FROM source_files WHERE collection_id = ?").run(col.id);
1548
+ db.prepare("DELETE FROM source_collections WHERE id = ?").run(col.id);
1549
+ return true;
1550
+ }
1551
+ /** Sync a collection: scan files, detect changes, re-index what changed. */
1552
+ async sourcesSync(name, options) {
1553
+ const db = this.sqliteDb;
1554
+ const startTime = Date.now();
1555
+ const batchSize = options?.batchSize || 20;
1556
+ const col = db.prepare("SELECT * FROM source_collections WHERE name = ?").get(name);
1557
+ if (!col) throw new Error(`Collection "${name}" not found. Add it first with sourcesAdd().`);
1558
+ const includePatterns = JSON.parse(col.glob_patterns);
1559
+ const ignorePatterns = JSON.parse(col.ignore_patterns);
1560
+ const files = this.scanDirectory(col.root_path, includePatterns, ignorePatterns);
1561
+ const existingFiles = /* @__PURE__ */ new Map();
1562
+ const rows = db.prepare("SELECT id, file_path, file_hash FROM source_files WHERE collection_id = ?").all(col.id);
1563
+ for (const row of rows) {
1564
+ existingFiles.set(row.file_path, { id: row.id, file_hash: row.file_hash });
1565
+ }
1566
+ let added = 0;
1567
+ let updated = 0;
1568
+ let removed = 0;
1569
+ let chunksAdded = 0;
1570
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1571
+ const toIndex = [];
1572
+ for (const absPath of files) {
1573
+ const relPath = relative(col.root_path, absPath);
1574
+ let content;
1575
+ try {
1576
+ content = readFileSync2(absPath, "utf-8");
1577
+ } catch {
1578
+ continue;
1579
+ }
1580
+ const stat = statSync(absPath);
1581
+ if (stat.size > 500 * 1024) continue;
1582
+ const hash = createHash("sha256").update(content).digest("hex");
1583
+ const existing = existingFiles.get(relPath);
1584
+ if (existing) {
1585
+ existingFiles.delete(relPath);
1586
+ if (existing.file_hash === hash) continue;
1587
+ toIndex.push({ relPath, absPath, hash, size: stat.size, isUpdate: true });
1588
+ } else {
1589
+ toIndex.push({ relPath, absPath, hash, size: stat.size, isUpdate: false });
1590
+ }
1591
+ }
1592
+ if (options?.dryRun) {
1593
+ const newFiles = toIndex.filter((f) => !f.isUpdate).length;
1594
+ const updatedFiles = toIndex.filter((f) => f.isUpdate).length;
1595
+ return {
1596
+ collection: name,
1597
+ added: newFiles,
1598
+ updated: updatedFiles,
1599
+ removed: existingFiles.size,
1600
+ chunks_added: 0,
1601
+ duration_ms: Date.now() - startTime
1602
+ };
1603
+ }
1604
+ for (let i = 0; i < toIndex.length; i += batchSize) {
1605
+ const batch = toIndex.slice(i, i + batchSize);
1606
+ const allChunks = [];
1607
+ for (const file of batch) {
1608
+ const content = readFileSync2(file.absPath, "utf-8");
1609
+ const ext = extname(file.absPath);
1610
+ const fileName = basename(file.absPath);
1611
+ const header = `File: ${file.relPath}
1612
+
1613
+ `;
1614
+ const textChunks = this.chunkText(header + content, 400, 80);
1615
+ const fileChunks = textChunks.map((text) => ({
1616
+ text,
1617
+ role: "system",
1618
+ source_type: "file",
1619
+ source_id: `file:${name}:${file.relPath}`,
1620
+ agent_id: "system",
1621
+ token_count: Math.ceil(text.length / 4),
1622
+ created_at: now
1623
+ }));
1624
+ allChunks.push(...fileChunks);
1625
+ if (file.isUpdate) {
1626
+ db.prepare(`
1627
+ UPDATE source_files SET file_hash = ?, file_size = ?, chunk_count = ?, last_indexed_at = ?
1628
+ WHERE collection_id = ? AND file_path = ?
1629
+ `).run(file.hash, file.size, fileChunks.length, now, col.id, file.relPath);
1630
+ updated++;
1631
+ } else {
1632
+ db.prepare(`
1633
+ INSERT INTO source_files (collection_id, file_path, file_hash, file_size, chunk_count, last_indexed_at)
1634
+ VALUES (?, ?, ?, ?, ?, ?)
1635
+ `).run(col.id, file.relPath, file.hash, file.size, fileChunks.length, now);
1636
+ added++;
1637
+ }
1638
+ }
1639
+ if (allChunks.length > 0) {
1640
+ const ingested = await this.ingest(allChunks);
1641
+ chunksAdded += ingested;
1642
+ }
1643
+ }
1644
+ for (const [relPath, { id }] of existingFiles) {
1645
+ db.prepare("DELETE FROM source_files WHERE id = ?").run(id);
1646
+ removed++;
1647
+ }
1648
+ const fileCount = db.prepare("SELECT COUNT(*) as count FROM source_files WHERE collection_id = ?").get(col.id).count;
1649
+ const chunkCount = db.prepare("SELECT SUM(chunk_count) as total FROM source_files WHERE collection_id = ?").get(col.id).total || 0;
1650
+ db.prepare("UPDATE source_collections SET file_count = ?, chunk_count = ?, last_sync_at = ? WHERE id = ?").run(fileCount, chunkCount, now, col.id);
1651
+ return {
1652
+ collection: name,
1653
+ added,
1654
+ updated,
1655
+ removed,
1656
+ chunks_added: chunksAdded,
1657
+ duration_ms: Date.now() - startTime
1658
+ };
1659
+ }
1660
+ /** Get status of all source collections. */
1661
+ sourcesStatus() {
1662
+ const db = this.sqliteDb;
1663
+ const collections = db.prepare("SELECT name, root_path, file_count, chunk_count, last_sync_at FROM source_collections").all();
1664
+ const totalFiles = collections.reduce((sum, c) => sum + c.file_count, 0);
1665
+ const totalChunks = collections.reduce((sum, c) => sum + c.chunk_count, 0);
1666
+ return {
1667
+ collections: collections.map((c) => ({
1668
+ name: c.name,
1669
+ root_path: c.root_path,
1670
+ file_count: c.file_count,
1671
+ chunk_count: c.chunk_count,
1672
+ last_sync_at: c.last_sync_at
1673
+ })),
1674
+ total_files: totalFiles,
1675
+ total_chunks: totalChunks
1676
+ };
1677
+ }
1678
+ /** Scan a directory recursively, matching include/ignore patterns. */
1679
+ scanDirectory(rootPath, includePatterns, ignorePatterns) {
1680
+ const results = [];
1681
+ const allowedExtensions = /* @__PURE__ */ new Set();
1682
+ const allowedExactNames = /* @__PURE__ */ new Set();
1683
+ for (const pattern of includePatterns) {
1684
+ const extMatch = pattern.match(/\*\*\/\*(\.\w+)$/);
1685
+ if (extMatch) {
1686
+ allowedExtensions.add(extMatch[1]);
1687
+ }
1688
+ const nameMatch = pattern.match(/\*\*\/([^*]+)$/);
1689
+ if (nameMatch && !nameMatch[1].startsWith("*.")) {
1690
+ allowedExactNames.add(nameMatch[1]);
1691
+ }
1692
+ }
1693
+ const ignoreDirs = /* @__PURE__ */ new Set();
1694
+ for (const pattern of ignorePatterns) {
1695
+ const dirMatch = pattern.match(/\*\*\/([^/*]+)\/\*\*$/);
1696
+ if (dirMatch) {
1697
+ ignoreDirs.add(dirMatch[1]);
1698
+ }
1699
+ }
1700
+ const ignoreFiles = /* @__PURE__ */ new Set();
1701
+ for (const pattern of ignorePatterns) {
1702
+ const fileMatch = pattern.match(/\*\*\/\*(\.\w+)$/);
1703
+ if (fileMatch) {
1704
+ ignoreFiles.add(fileMatch[1]);
1705
+ }
1706
+ const exactMatch = pattern.match(/\*\*\/([^*]+)$/);
1707
+ if (exactMatch && !exactMatch[1].includes("/")) {
1708
+ ignoreFiles.add(exactMatch[1]);
1709
+ }
1710
+ }
1711
+ const walk = (dir) => {
1712
+ let entries;
1713
+ try {
1714
+ entries = readdirSync(dir);
1715
+ } catch {
1716
+ return;
1717
+ }
1718
+ for (const entry of entries) {
1719
+ const fullPath = join2(dir, entry);
1720
+ let stat;
1721
+ try {
1722
+ stat = statSync(fullPath);
1723
+ } catch {
1724
+ continue;
1725
+ }
1726
+ if (stat.isDirectory()) {
1727
+ if (ignoreDirs.has(entry)) continue;
1728
+ if (entry.startsWith(".")) continue;
1729
+ walk(fullPath);
1730
+ } else if (stat.isFile()) {
1731
+ const ext = extname(entry);
1732
+ if (ignoreFiles.has(ext)) continue;
1733
+ if (ignoreFiles.has(entry)) continue;
1734
+ if (allowedExtensions.has(ext) || allowedExactNames.has(entry)) {
1735
+ results.push(fullPath);
1736
+ }
1737
+ }
1738
+ }
1739
+ };
1740
+ walk(rootPath);
1741
+ return results;
1742
+ }
1743
+ // ── Orphan Cleanup ──
1744
+ /** Clean orphaned entries in chunks_vec and chunks_fts that no longer have
1745
+ * corresponding rows in the chunks table. Returns counts of what was found/cleaned. */
1746
+ cleanOrphans(options) {
1747
+ const db = this.sqliteDb;
1748
+ const dryRun = options?.dryRun ?? false;
1749
+ const orphanedVec = db.prepare(
1750
+ "SELECT COUNT(*) as cnt FROM chunks_vec WHERE chunk_id NOT IN (SELECT id FROM chunks)"
1751
+ ).get().cnt;
1752
+ const orphanedFts = db.prepare(
1753
+ "SELECT COUNT(*) as cnt FROM chunks_fts WHERE rowid NOT IN (SELECT id FROM chunks)"
1754
+ ).get().cnt;
1755
+ if (dryRun) {
1756
+ return { orphanedVec, orphanedFts, cleanedVec: 0, cleanedFts: 0, dryRun: true };
1757
+ }
1758
+ let cleanedVec = 0;
1759
+ if (orphanedVec > 0) {
1760
+ const ids = db.prepare(
1761
+ "SELECT chunk_id FROM chunks_vec WHERE chunk_id NOT IN (SELECT id FROM chunks)"
1762
+ ).all();
1763
+ const del = db.prepare("DELETE FROM chunks_vec WHERE chunk_id = ?");
1764
+ const BATCH = 1e3;
1765
+ for (let i = 0; i < ids.length; i += BATCH) {
1766
+ const batch = ids.slice(i, i + BATCH);
1767
+ db.transaction(() => {
1768
+ for (const r of batch) {
1769
+ del.run(r.chunk_id);
1770
+ cleanedVec++;
1771
+ }
1772
+ })();
1773
+ }
1774
+ }
1775
+ let cleanedFts = 0;
1776
+ if (orphanedFts > 0) {
1777
+ db.exec("DELETE FROM chunks_fts");
1778
+ db.exec("INSERT INTO chunks_fts(rowid, text) SELECT id, text FROM chunks");
1779
+ cleanedFts = orphanedFts;
1780
+ }
1781
+ return { orphanedVec, orphanedFts, cleanedVec, cleanedFts, dryRun: false };
1782
+ }
1783
+ // ── Cleanup ──
1784
+ close() {
1785
+ this.sqliteDb?.close();
1786
+ }
1787
+ };
1788
+ function resolveConfig(overrides) {
1789
+ const HOME2 = process.env.HOME || "";
1790
+ const ldmMemory = join2(HOME2, ".ldm", "memory");
1791
+ let dataDir = overrides?.dataDir || process.env.CRYSTAL_DATA_DIR;
1792
+ if (!dataDir) {
1793
+ if (existsSync2(join2(ldmMemory, "crystal.db"))) {
1794
+ dataDir = ldmMemory;
1795
+ } else {
1796
+ const legacyDir = join2(HOME2, ".openclaw", "memory-crystal");
1797
+ if (existsSync2(join2(legacyDir, "crystal.db"))) {
1798
+ dataDir = legacyDir;
1799
+ } else {
1800
+ dataDir = ldmMemory;
1801
+ }
1802
+ }
1803
+ }
1804
+ loadEnvFile(join2(dataDir, ".env"));
1805
+ const openaiApiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY || opRead("OpenAI API", "api key");
1806
+ const googleApiKey = overrides?.googleApiKey || process.env.GOOGLE_API_KEY || opRead("Google AI", "api key");
1807
+ const remoteToken = overrides?.remoteToken || process.env.CRYSTAL_REMOTE_TOKEN || opRead("Memory Crystal Remote", "token");
1808
+ return {
1809
+ dataDir,
1810
+ embeddingProvider: overrides?.embeddingProvider || process.env.CRYSTAL_EMBEDDING_PROVIDER || "openai",
1811
+ openaiApiKey,
1812
+ openaiModel: overrides?.openaiModel || process.env.CRYSTAL_OPENAI_MODEL || "text-embedding-3-small",
1813
+ ollamaHost: overrides?.ollamaHost || process.env.CRYSTAL_OLLAMA_HOST || "http://localhost:11434",
1814
+ ollamaModel: overrides?.ollamaModel || process.env.CRYSTAL_OLLAMA_MODEL || "nomic-embed-text",
1815
+ googleApiKey,
1816
+ googleModel: overrides?.googleModel || process.env.CRYSTAL_GOOGLE_MODEL || "text-embedding-004",
1817
+ remoteUrl: overrides?.remoteUrl || process.env.CRYSTAL_REMOTE_URL,
1818
+ remoteToken
1819
+ };
1820
+ }
1821
+ function loadEnvFile(path) {
1822
+ if (!existsSync2(path)) return;
1823
+ const content = readFileSync2(path, "utf8");
1824
+ for (const line of content.split("\n")) {
1825
+ const trimmed = line.trim();
1826
+ if (!trimmed || trimmed.startsWith("#")) continue;
1827
+ const eqIdx = trimmed.indexOf("=");
1828
+ if (eqIdx === -1) continue;
1829
+ const key = trimmed.slice(0, eqIdx).trim();
1830
+ let value = trimmed.slice(eqIdx + 1).trim();
1831
+ if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
1832
+ value = value.slice(1, -1);
1833
+ }
1834
+ if (key && !process.env[key]) {
1835
+ process.env[key] = value;
1836
+ }
1837
+ }
1838
+ }
1839
+ function opRead(item, field) {
1840
+ try {
1841
+ const HOME2 = process.env.HOME || "";
1842
+ let saTokenPath = join2(HOME2, ".ldm", "secrets", "op-sa-token");
1843
+ if (!existsSync2(saTokenPath)) {
1844
+ saTokenPath = join2(HOME2, ".openclaw", "secrets", "op-sa-token");
1845
+ }
1846
+ if (!existsSync2(saTokenPath)) return void 0;
1847
+ const saToken = readFileSync2(saTokenPath, "utf8").trim();
1848
+ return execSync2(`op read "op://Agent Secrets/${item}/${field}" 2>/dev/null`, {
1849
+ encoding: "utf8",
1850
+ env: { ...process.env, OP_SERVICE_ACCOUNT_TOKEN: saToken },
1851
+ timeout: 1e4
1852
+ }).trim() || void 0;
1853
+ } catch {
1854
+ return void 0;
1855
+ }
1856
+ }
1857
+ var RemoteCrystal = class {
1858
+ url;
1859
+ token;
1860
+ constructor(url, token) {
1861
+ this.url = url.replace(/\/$/, "");
1862
+ this.token = token;
1863
+ }
1864
+ async init() {
1865
+ const resp = await fetch(`${this.url}/health`);
1866
+ if (!resp.ok) {
1867
+ throw new Error(`Remote crystal unreachable: ${resp.status}`);
1868
+ }
1869
+ }
1870
+ async request(path, body) {
1871
+ const resp = await fetch(`${this.url}${path}`, {
1872
+ method: body ? "POST" : "GET",
1873
+ headers: {
1874
+ "Authorization": `Bearer ${this.token}`,
1875
+ "Content-Type": "application/json"
1876
+ },
1877
+ ...body ? { body: JSON.stringify(body) } : {}
1878
+ });
1879
+ if (!resp.ok) {
1880
+ const err = await resp.text();
1881
+ throw new Error(`Remote crystal error ${resp.status}: ${err}`);
1882
+ }
1883
+ return resp.json();
1884
+ }
1885
+ async search(query, limit = 5, filter) {
1886
+ const data = await this.request("/search", { query, limit, agent_id: filter?.agent_id });
1887
+ return data.results || [];
1888
+ }
1889
+ async ingest(chunks) {
1890
+ const data = await this.request("/ingest", { chunks });
1891
+ return data.ingested || 0;
1892
+ }
1893
+ async remember(text, category = "fact") {
1894
+ const data = await this.request("/remember", { text, category });
1895
+ return data.id;
1896
+ }
1897
+ forget(memoryId) {
1898
+ return this.request("/forget", { id: memoryId }).then((d) => d.ok);
1899
+ }
1900
+ async status() {
1901
+ const data = await this.request("/status");
1902
+ return {
1903
+ chunks: data.chunks || 0,
1904
+ memories: data.memories || 0,
1905
+ sources: 0,
1906
+ agents: data.agents || [],
1907
+ oldestChunk: data.oldestChunk,
1908
+ newestChunk: data.newestChunk,
1909
+ embeddingProvider: "remote",
1910
+ dataDir: this.url,
1911
+ capturedSessions: data.capturedSessions || 0,
1912
+ latestCapture: data.newestChunk
1913
+ };
1914
+ }
1915
+ // Expose chunkText from a local Crystal instance for cc-hook to use
1916
+ chunkText(text) {
1917
+ const targetChars = 400 * 4;
1918
+ const overlapChars = 80 * 4;
1919
+ if (text.length <= targetChars) return [text];
1920
+ const chunks = [];
1921
+ let start = 0;
1922
+ while (start < text.length) {
1923
+ let end = start + targetChars;
1924
+ if (end >= text.length) {
1925
+ chunks.push(text.slice(start));
1926
+ break;
1927
+ }
1928
+ const paraBreak = text.lastIndexOf("\n\n", end);
1929
+ if (paraBreak > start + targetChars * 0.5) end = paraBreak;
1930
+ else {
1931
+ const sentBreak = text.lastIndexOf(". ", end);
1932
+ if (sentBreak > start + targetChars * 0.5) end = sentBreak + 1;
1933
+ }
1934
+ chunks.push(text.slice(start, end));
1935
+ start = end - overlapChars;
1936
+ }
1937
+ return chunks;
1938
+ }
1939
+ };
1940
+ function createCrystal(config) {
1941
+ if (config.remoteUrl && config.remoteToken) {
1942
+ return new RemoteCrystal(config.remoteUrl, config.remoteToken);
1943
+ }
1944
+ return new Crystal(config);
1945
+ }
1946
+
1947
+ // src/crypto.ts
1948
+ import { readFileSync as readFileSync4, existsSync as existsSync4 } from "fs";
1949
+ import { createCipheriv, createDecipheriv, createHmac, randomBytes, hkdfSync } from "crypto";
1950
+
1951
+ // src/ldm.ts
1952
+ import { existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync, copyFileSync, chmodSync, readdirSync as readdirSync2 } from "fs";
1953
+ import { join as join3, dirname } from "path";
1954
+ import { execSync as execSync3 } from "child_process";
1955
+ import { fileURLToPath } from "url";
1956
+ var HOME = process.env.HOME || "";
1957
+ var LDM_ROOT = join3(HOME, ".ldm");
1958
+ function loadAgentConfig(id) {
1959
+ const cfgPath = join3(LDM_ROOT, "agents", id, "config.json");
1960
+ try {
1961
+ if (existsSync3(cfgPath)) return JSON.parse(readFileSync3(cfgPath, "utf-8"));
1962
+ } catch {
1963
+ }
1964
+ return null;
1965
+ }
1966
+ function getAgentId(harnessHint) {
1967
+ if (process.env.CRYSTAL_AGENT_ID) return process.env.CRYSTAL_AGENT_ID;
1968
+ const agentsDir = join3(LDM_ROOT, "agents");
1969
+ if (existsSync3(agentsDir)) {
1970
+ try {
1971
+ for (const d of readdirSync2(agentsDir)) {
1972
+ const cfg = loadAgentConfig(d);
1973
+ if (!cfg || !cfg.agentId) continue;
1974
+ if (!harnessHint) return cfg.agentId;
1975
+ if (harnessHint === "claude-code" && cfg.harness === "claude-code-cli") return cfg.agentId;
1976
+ if (harnessHint === "openclaw" && cfg.harness === "openclaw") return cfg.agentId;
1977
+ }
1978
+ } catch {
1979
+ }
1980
+ }
1981
+ return harnessHint === "openclaw" ? "oc-lesa-mini" : "cc-mini";
1982
+ }
1983
+ function ldmPaths(agentId) {
1984
+ const id = agentId || getAgentId();
1985
+ const agentRoot = join3(LDM_ROOT, "agents", id);
1986
+ return {
1987
+ root: LDM_ROOT,
1988
+ bin: join3(LDM_ROOT, "bin"),
1989
+ secrets: join3(LDM_ROOT, "secrets"),
1990
+ state: join3(LDM_ROOT, "state"),
1991
+ config: join3(LDM_ROOT, "config.json"),
1992
+ crystalDb: join3(LDM_ROOT, "memory", "crystal.db"),
1993
+ crystalLance: join3(LDM_ROOT, "memory", "lance"),
1994
+ agentRoot,
1995
+ transcripts: join3(agentRoot, "memory", "transcripts"),
1996
+ sessions: join3(agentRoot, "memory", "sessions"),
1997
+ daily: join3(agentRoot, "memory", "daily"),
1998
+ journals: join3(agentRoot, "memory", "journals"),
1999
+ workspace: join3(agentRoot, "memory", "workspace")
2000
+ };
2001
+ }
2002
+ function loadConfig() {
2003
+ const configPath = join3(LDM_ROOT, "config.json");
2004
+ try {
2005
+ if (existsSync3(configPath)) {
2006
+ return JSON.parse(readFileSync3(configPath, "utf-8"));
2007
+ }
2008
+ } catch {
2009
+ }
2010
+ return null;
2011
+ }
2012
+ function saveConfig(config) {
2013
+ const configPath = join3(LDM_ROOT, "config.json");
2014
+ writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n");
2015
+ }
2016
+ function scaffoldLdm(agentId) {
2017
+ const paths = ldmPaths(agentId);
2018
+ mkdirSync2(join3(paths.root, "memory"), { recursive: true });
2019
+ mkdirSync2(paths.crystalLance, { recursive: true });
2020
+ mkdirSync2(paths.bin, { recursive: true });
2021
+ mkdirSync2(paths.secrets, { recursive: true, mode: 448 });
2022
+ mkdirSync2(paths.state, { recursive: true });
2023
+ mkdirSync2(paths.transcripts, { recursive: true });
2024
+ mkdirSync2(paths.sessions, { recursive: true });
2025
+ mkdirSync2(paths.daily, { recursive: true });
2026
+ mkdirSync2(paths.journals, { recursive: true });
2027
+ mkdirSync2(paths.workspace, { recursive: true });
2028
+ const id = agentId || getAgentId();
2029
+ let config = loadConfig();
2030
+ if (!config) {
2031
+ config = {
2032
+ version: "1.0.0",
2033
+ agents: [id],
2034
+ createdAt: (/* @__PURE__ */ new Date()).toISOString(),
2035
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
2036
+ };
2037
+ } else {
2038
+ if (!config.agents.includes(id)) {
2039
+ config.agents.push(id);
2040
+ }
2041
+ config.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
2042
+ }
2043
+ saveConfig(config);
2044
+ return paths;
2045
+ }
2046
+ var LEGACY_OC_DIR = join3(HOME, ".openclaw");
2047
+ function resolveStatePath(filename) {
2048
+ const paths = ldmPaths();
2049
+ const ldmPath = join3(paths.state, filename);
2050
+ if (existsSync3(ldmPath)) return ldmPath;
2051
+ const legacyPath = join3(LEGACY_OC_DIR, "memory", filename);
2052
+ if (existsSync3(legacyPath)) return legacyPath;
2053
+ return ldmPath;
2054
+ }
2055
+ function stateWritePath(filename) {
2056
+ const paths = ldmPaths();
2057
+ const dir = paths.state;
2058
+ if (!existsSync3(dir)) mkdirSync2(dir, { recursive: true });
2059
+ return join3(dir, filename);
2060
+ }
2061
+ function resolveSecretPath(filename) {
2062
+ const paths = ldmPaths();
2063
+ const ldmPath = join3(paths.secrets, filename);
2064
+ if (existsSync3(ldmPath)) return ldmPath;
2065
+ const legacyPath = join3(LEGACY_OC_DIR, "secrets", filename);
2066
+ if (existsSync3(legacyPath)) return legacyPath;
2067
+ return ldmPath;
2068
+ }
2069
+ function ensureLdm(agentId) {
2070
+ const paths = ldmPaths(agentId);
2071
+ if (existsSync3(paths.transcripts) && existsSync3(paths.config)) {
2072
+ return paths;
2073
+ }
2074
+ return scaffoldLdm(agentId);
2075
+ }
2076
+
2077
+ // src/crypto.ts
2078
+ import { createHash as createHash2 } from "crypto";
2079
+ var KEY_PATH = process.env.CRYSTAL_RELAY_KEY_PATH || resolveSecretPath("crystal-relay-key");
2080
+ function loadRelayKey() {
2081
+ if (!existsSync4(KEY_PATH)) {
2082
+ throw new Error(
2083
+ `Relay key not found at ${KEY_PATH}
2084
+ Generate one: mkdir -p ~/.ldm/secrets && openssl rand -base64 32 > ~/.ldm/secrets/crystal-relay-key && chmod 600 ~/.ldm/secrets/crystal-relay-key
2085
+ Or run: crystal pair`
2086
+ );
2087
+ }
2088
+ const raw = readFileSync4(KEY_PATH, "utf-8").trim();
2089
+ const key = Buffer.from(raw, "base64");
2090
+ if (key.length !== 32) {
2091
+ throw new Error(`Relay key must be 32 bytes (256 bits). Got ${key.length} bytes. Regenerate with: openssl rand -base64 32`);
2092
+ }
2093
+ return key;
2094
+ }
2095
+ function deriveSigningKey(masterKey) {
2096
+ return Buffer.from(hkdfSync("sha256", masterKey, "", "crystal-relay-sign", 32));
2097
+ }
2098
+ function encrypt(plaintext, masterKey) {
2099
+ const nonce = randomBytes(12);
2100
+ const cipher = createCipheriv("aes-256-gcm", masterKey, nonce);
2101
+ const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
2102
+ const tag = cipher.getAuthTag();
2103
+ const signingKey = deriveSigningKey(masterKey);
2104
+ const hmacData = Buffer.concat([nonce, ciphertext, tag]);
2105
+ const hmac = createHmac("sha256", signingKey).update(hmacData).digest("hex");
2106
+ return {
2107
+ v: 1,
2108
+ nonce: nonce.toString("base64"),
2109
+ ciphertext: ciphertext.toString("base64"),
2110
+ tag: tag.toString("base64"),
2111
+ hmac
2112
+ };
2113
+ }
2114
+ function encryptJSON(data, masterKey) {
2115
+ const plaintext = Buffer.from(JSON.stringify(data), "utf-8");
2116
+ return encrypt(plaintext, masterKey);
2117
+ }
17
2118
 
18
2119
  // src/cc-hook.ts
19
2120
  import {
20
- readFileSync,
21
- writeFileSync,
2121
+ readFileSync as readFileSync6,
2122
+ writeFileSync as writeFileSync3,
22
2123
  appendFileSync,
23
- existsSync,
24
- mkdirSync,
25
- statSync,
2124
+ existsSync as existsSync6,
2125
+ mkdirSync as mkdirSync4,
2126
+ statSync as statSync2,
26
2127
  openSync,
27
2128
  readSync,
28
2129
  closeSync,
29
- copyFileSync
2130
+ copyFileSync as copyFileSync2,
2131
+ readdirSync as readdirSync3
30
2132
  } from "fs";
31
- import { join, basename } from "path";
2133
+ import { join as join5, basename as basename3 } from "path";
2134
+ import { createHash as createHash3 } from "crypto";
32
2135
  var CC_AGENT_ID = getAgentId("claude-code");
33
2136
  var RELAY_URL = process.env.CRYSTAL_RELAY_URL || "";
34
2137
  var RELAY_TOKEN = process.env.CRYSTAL_RELAY_TOKEN || "";
35
2138
  var PRIVATE_MODE_PATH = resolveStatePath("memory-capture-state.json");
36
2139
  var WATERMARK_PATH = resolveStatePath("cc-capture-watermark.json");
37
2140
  var CC_ENABLED_PATH = resolveStatePath("cc-capture-enabled.json");
2141
+ var MEMORY_SYNC_WATERMARK_PATH = resolveStatePath("cc-memory-sync-watermark.json");
2142
+ var CLAUDE_PROJECTS_DIR = join5(process.env.HOME || "", ".claude", "projects");
38
2143
  function getCaptureMode() {
39
2144
  if (RELAY_URL && RELAY_TOKEN) return "relay";
40
2145
  return "local";
41
2146
  }
42
2147
  function isPrivateMode() {
43
2148
  try {
44
- if (existsSync(PRIVATE_MODE_PATH)) {
45
- const state = JSON.parse(readFileSync(PRIVATE_MODE_PATH, "utf-8"));
2149
+ if (existsSync6(PRIVATE_MODE_PATH)) {
2150
+ const state = JSON.parse(readFileSync6(PRIVATE_MODE_PATH, "utf-8"));
46
2151
  return state.enabled === false;
47
2152
  }
48
2153
  } catch {
@@ -51,8 +2156,8 @@ function isPrivateMode() {
51
2156
  }
52
2157
  function isCaptureEnabled() {
53
2158
  try {
54
- if (existsSync(CC_ENABLED_PATH)) {
55
- const state = JSON.parse(readFileSync(CC_ENABLED_PATH, "utf-8"));
2159
+ if (existsSync6(CC_ENABLED_PATH)) {
2160
+ const state = JSON.parse(readFileSync6(CC_ENABLED_PATH, "utf-8"));
56
2161
  return state.enabled !== false;
57
2162
  }
58
2163
  } catch {
@@ -61,15 +2166,15 @@ function isCaptureEnabled() {
61
2166
  }
62
2167
  function setCaptureEnabled(enabled) {
63
2168
  const writePath = stateWritePath("cc-capture-enabled.json");
64
- writeFileSync(writePath, JSON.stringify({
2169
+ writeFileSync3(writePath, JSON.stringify({
65
2170
  enabled,
66
2171
  updatedAt: (/* @__PURE__ */ new Date()).toISOString()
67
2172
  }, null, 2));
68
2173
  }
69
2174
  function loadWatermark() {
70
2175
  try {
71
- if (existsSync(WATERMARK_PATH)) {
72
- return JSON.parse(readFileSync(WATERMARK_PATH, "utf-8"));
2176
+ if (existsSync6(WATERMARK_PATH)) {
2177
+ return JSON.parse(readFileSync6(WATERMARK_PATH, "utf-8"));
73
2178
  }
74
2179
  } catch {
75
2180
  }
@@ -78,10 +2183,139 @@ function loadWatermark() {
78
2183
  function saveWatermark(wm) {
79
2184
  const writePath = stateWritePath("cc-capture-watermark.json");
80
2185
  wm.lastRun = (/* @__PURE__ */ new Date()).toISOString();
81
- writeFileSync(writePath, JSON.stringify(wm, null, 2));
2186
+ writeFileSync3(writePath, JSON.stringify(wm, null, 2));
2187
+ }
2188
+ function loadMemorySyncWatermark() {
2189
+ try {
2190
+ if (existsSync6(MEMORY_SYNC_WATERMARK_PATH)) {
2191
+ return JSON.parse(readFileSync6(MEMORY_SYNC_WATERMARK_PATH, "utf-8"));
2192
+ }
2193
+ } catch {
2194
+ }
2195
+ return { files: {}, lastRun: null };
2196
+ }
2197
+ function saveMemorySyncWatermark(wm) {
2198
+ const writePath = stateWritePath("cc-memory-sync-watermark.json");
2199
+ wm.lastRun = (/* @__PURE__ */ new Date()).toISOString();
2200
+ writeFileSync3(writePath, JSON.stringify(wm, null, 2));
2201
+ }
2202
+ function hashFileContent(content) {
2203
+ return createHash3("sha256").update(content).digest("hex");
2204
+ }
2205
+ function parseFrontmatter(content) {
2206
+ const frontmatter = {};
2207
+ if (!content.startsWith("---")) {
2208
+ return { frontmatter, body: content };
2209
+ }
2210
+ const endIdx = content.indexOf("---", 3);
2211
+ if (endIdx === -1) {
2212
+ return { frontmatter, body: content };
2213
+ }
2214
+ const yamlBlock = content.slice(3, endIdx).trim();
2215
+ const body = content.slice(endIdx + 3).trim();
2216
+ for (const line of yamlBlock.split("\n")) {
2217
+ const colonIdx = line.indexOf(":");
2218
+ if (colonIdx === -1) continue;
2219
+ const key = line.slice(0, colonIdx).trim();
2220
+ const value = line.slice(colonIdx + 1).trim();
2221
+ if (key === "name") frontmatter.name = value;
2222
+ else if (key === "description") frontmatter.description = value;
2223
+ else if (key === "type") frontmatter.type = value;
2224
+ }
2225
+ return { frontmatter, body };
2226
+ }
2227
+ var VALID_CATEGORIES = /* @__PURE__ */ new Set([
2228
+ "fact",
2229
+ "preference",
2230
+ "event",
2231
+ "opinion",
2232
+ "skill",
2233
+ "user",
2234
+ "feedback",
2235
+ "project",
2236
+ "reference"
2237
+ ]);
2238
+ function discoverMemoryFiles() {
2239
+ const files = [];
2240
+ try {
2241
+ if (!existsSync6(CLAUDE_PROJECTS_DIR)) return files;
2242
+ for (const project of readdirSync3(CLAUDE_PROJECTS_DIR)) {
2243
+ const memDir = join5(CLAUDE_PROJECTS_DIR, project, "memory");
2244
+ if (!existsSync6(memDir)) continue;
2245
+ try {
2246
+ for (const file of readdirSync3(memDir)) {
2247
+ if (!file.endsWith(".md")) continue;
2248
+ files.push(join5(memDir, file));
2249
+ }
2250
+ } catch {
2251
+ }
2252
+ }
2253
+ } catch {
2254
+ }
2255
+ return files;
2256
+ }
2257
+ async function syncMemoryFiles() {
2258
+ if (isPrivateMode()) return 0;
2259
+ const files = discoverMemoryFiles();
2260
+ if (files.length === 0) return 0;
2261
+ const wm = loadMemorySyncWatermark();
2262
+ const changed = [];
2263
+ for (const filePath of files) {
2264
+ try {
2265
+ const content = readFileSync6(filePath, "utf-8");
2266
+ if (content.trim().length === 0) continue;
2267
+ const hash = hashFileContent(content);
2268
+ if (wm.files[filePath] && wm.files[filePath].hash === hash) continue;
2269
+ const { frontmatter, body } = parseFrontmatter(content);
2270
+ changed.push({ path: filePath, content, hash, frontmatter, body });
2271
+ } catch {
2272
+ }
2273
+ }
2274
+ if (changed.length === 0) {
2275
+ saveMemorySyncWatermark(wm);
2276
+ return 0;
2277
+ }
2278
+ const mode = getCaptureMode();
2279
+ if (mode === "relay") {
2280
+ process.stderr.write(`[cc-memory-sync] skipping ${changed.length} files (relay mode not yet supported)
2281
+ `);
2282
+ for (const item of changed) {
2283
+ wm.files[item.path] = { hash: item.hash, syncedAt: (/* @__PURE__ */ new Date()).toISOString() };
2284
+ }
2285
+ saveMemorySyncWatermark(wm);
2286
+ return 0;
2287
+ }
2288
+ const config = resolveConfig();
2289
+ const crystal = createCrystal(config);
2290
+ await crystal.init();
2291
+ let ingested = 0;
2292
+ for (const item of changed) {
2293
+ try {
2294
+ const rawType = item.frontmatter.type || "reference";
2295
+ const category = VALID_CATEGORIES.has(rawType) ? rawType : "reference";
2296
+ let text = "";
2297
+ if (item.frontmatter.name) {
2298
+ text += `[${item.frontmatter.name}] `;
2299
+ }
2300
+ if (item.body.length > 0) {
2301
+ text += item.body;
2302
+ } else {
2303
+ text += item.content;
2304
+ }
2305
+ if (text.trim().length < 10) continue;
2306
+ await crystal.remember(text.trim(), category);
2307
+ ingested++;
2308
+ wm.files[item.path] = { hash: item.hash, syncedAt: (/* @__PURE__ */ new Date()).toISOString() };
2309
+ } catch (err) {
2310
+ process.stderr.write(`[cc-memory-sync] error ingesting ${basename3(item.path)}: ${err.message}
2311
+ `);
2312
+ }
2313
+ }
2314
+ saveMemorySyncWatermark(wm);
2315
+ return ingested;
82
2316
  }
83
2317
  function extractMessages(filePath, lastByteOffset) {
84
- const fileSize = statSync(filePath).size;
2318
+ const fileSize = statSync2(filePath).size;
85
2319
  if (lastByteOffset >= fileSize) {
86
2320
  return { messages: [], newByteOffset: fileSize };
87
2321
  }
@@ -125,21 +2359,21 @@ function archiveTranscript(transcriptPath, agentId) {
125
2359
  try {
126
2360
  if (isPrivateMode()) return;
127
2361
  const paths = ensureLdm(agentId);
128
- const dest = join(paths.transcripts, basename(transcriptPath));
129
- if (existsSync(dest)) {
130
- const srcMtime = statSync(transcriptPath).mtimeMs;
131
- const dstMtime = statSync(dest).mtimeMs;
2362
+ const dest = join5(paths.transcripts, basename3(transcriptPath));
2363
+ if (existsSync6(dest)) {
2364
+ const srcMtime = statSync2(transcriptPath).mtimeMs;
2365
+ const dstMtime = statSync2(dest).mtimeMs;
132
2366
  if (srcMtime <= dstMtime) return;
133
2367
  }
134
- copyFileSync(transcriptPath, dest);
2368
+ copyFileSync2(transcriptPath, dest);
135
2369
  } catch {
136
2370
  }
137
2371
  }
138
2372
  function appendDailyLog(messages, agentId) {
139
2373
  try {
140
2374
  const paths = ldmPaths(agentId);
141
- if (!existsSync(paths.root)) return;
142
- if (!existsSync(paths.daily)) mkdirSync(paths.daily, { recursive: true });
2375
+ if (!existsSync6(paths.root)) return;
2376
+ if (!existsSync6(paths.daily)) mkdirSync4(paths.daily, { recursive: true });
143
2377
  const now = /* @__PURE__ */ new Date();
144
2378
  const dateStr = now.toISOString().slice(0, 10);
145
2379
  const timeStr = now.toLocaleTimeString("en-US", {
@@ -148,14 +2382,14 @@ function appendDailyLog(messages, agentId) {
148
2382
  hour12: false,
149
2383
  timeZone: "America/Los_Angeles"
150
2384
  });
151
- const logPath = join(paths.daily, `${dateStr}.md`);
2385
+ const logPath = join5(paths.daily, `${dateStr}.md`);
152
2386
  const userMsg = messages.find((m) => m.role === "user");
153
2387
  if (!userMsg) return;
154
2388
  const snippet = userMsg.text.slice(0, 120).replace(/\n/g, " ").trim();
155
2389
  const line = `- **${timeStr}** ${snippet}${userMsg.text.length > 120 ? "..." : ""}
156
2390
  `;
157
- if (!existsSync(logPath)) {
158
- writeFileSync(logPath, `# ${dateStr} - CC Daily Log
2391
+ if (!existsSync6(logPath)) {
2392
+ writeFileSync3(logPath, `# ${dateStr} - CC Daily Log
159
2393
 
160
2394
  `);
161
2395
  }
@@ -312,7 +2546,7 @@ async function main() {
312
2546
  process.exit(0);
313
2547
  }
314
2548
  const transcriptPath = hookData.transcript_path;
315
- if (!transcriptPath || !existsSync(transcriptPath)) process.exit(0);
2549
+ if (!transcriptPath || !existsSync6(transcriptPath)) process.exit(0);
316
2550
  if (isPrivateMode() || !isCaptureEnabled()) process.exit(0);
317
2551
  archiveTranscript(transcriptPath);
318
2552
  const wm = loadWatermark();
@@ -337,25 +2571,35 @@ async function main() {
337
2571
  try {
338
2572
  if (mode === "relay") {
339
2573
  const count = await dropAtRelay(messages);
340
- process.stderr.write(`[cc-memory-capture] relayed ${count} messages (${totalTokens} tokens) from ${basename(transcriptPath)}
2574
+ process.stderr.write(`[cc-memory-capture] relayed ${count} messages (${totalTokens} tokens) from ${basename3(transcriptPath)}
341
2575
  `);
342
2576
  } else {
343
2577
  const count = await ingestLocal(messages);
344
- process.stderr.write(`[cc-memory-capture] ${count} chunks (${totalTokens} tokens) from ${basename(transcriptPath)}
2578
+ process.stderr.write(`[cc-memory-capture] ${count} chunks (${totalTokens} tokens) from ${basename3(transcriptPath)}
345
2579
  `);
346
2580
  }
347
2581
  wm.files[fileKey] = { lastByteOffset: newByteOffset, lastTimestamp: (/* @__PURE__ */ new Date()).toISOString() };
348
2582
  saveWatermark(wm);
349
2583
  appendDailyLog(messages);
350
2584
  try {
351
- const { generateSessionSummary, writeSummaryFile } = await import("./summarize.js");
2585
+ const { generateSessionSummary: generateSessionSummary2, writeSummaryFile: writeSummaryFile2 } = await Promise.resolve().then(() => (init_summarize(), summarize_exports));
352
2586
  const paths = ldmPaths();
353
2587
  const summaryMsgs = messages.map((m) => ({ role: m.role, text: m.text, timestamp: m.timestamp, sessionId: m.sessionId }));
354
- const summary = await generateSessionSummary(summaryMsgs);
2588
+ const summary = await generateSessionSummary2(summaryMsgs);
355
2589
  const sessionId = messages[0]?.sessionId || "unknown";
356
- writeSummaryFile(paths.sessions, summary, CC_AGENT_ID, sessionId);
2590
+ writeSummaryFile2(paths.sessions, summary, CC_AGENT_ID, sessionId);
357
2591
  } catch {
358
2592
  }
2593
+ try {
2594
+ const syncCount = await syncMemoryFiles();
2595
+ if (syncCount > 0) {
2596
+ process.stderr.write(`[cc-memory-sync] ingested ${syncCount} changed memory file(s)
2597
+ `);
2598
+ }
2599
+ } catch (err) {
2600
+ process.stderr.write(`[cc-memory-sync] error: ${err.message}
2601
+ `);
2602
+ }
359
2603
  } catch (err) {
360
2604
  process.stderr.write(`[cc-memory-capture] error: ${err.message}
361
2605
  `);