@wipcomputer/memory-crystal 0.7.29 → 0.7.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/CHANGELOG.md +75 -0
  2. package/SKILL.md +1 -1
  3. package/package.json +1 -1
  4. package/scripts/migrate-lance-to-sqlite.mjs +2 -1
  5. package/_trash/RELEASE-NOTES-v0-7-23.md +0 -48
  6. package/_trash/RELEASE-NOTES-v0-7-25.md +0 -24
  7. package/_trash/RELEASE-NOTES-v0-7-26.md +0 -7
  8. package/_trash/RELEASE-NOTES-v0-7-28.md +0 -31
  9. package/_trash/RELEASE-NOTES-v0-7-29.md +0 -28
  10. package/_trash/RELEASE-NOTES-v0-7-4.md +0 -64
  11. package/_trash/RELEASE-NOTES-v0-7-5.md +0 -19
  12. package/dist/bridge.d.ts +0 -7
  13. package/dist/bridge.js +0 -14
  14. package/dist/bulk-copy.d.ts +0 -17
  15. package/dist/bulk-copy.js +0 -90
  16. package/dist/cc-hook.d.ts +0 -8
  17. package/dist/cc-hook.js +0 -368
  18. package/dist/cc-poller.d.ts +0 -1
  19. package/dist/cc-poller.js +0 -550
  20. package/dist/chunk-25LXQJ4Z.js +0 -110
  21. package/dist/chunk-2DRXIRQW.js +0 -97
  22. package/dist/chunk-2ZNH5F6E.js +0 -1281
  23. package/dist/chunk-3G3SFYYI.js +0 -288
  24. package/dist/chunk-3RG5ZIWI.js +0 -10
  25. package/dist/chunk-3S6TI23B.js +0 -97
  26. package/dist/chunk-3VFIJYS4.js +0 -818
  27. package/dist/chunk-52QE3YI3.js +0 -1169
  28. package/dist/chunk-57RP3DIN.js +0 -1205
  29. package/dist/chunk-5HSZ4W2P.js +0 -62
  30. package/dist/chunk-645IPXW3.js +0 -290
  31. package/dist/chunk-7A7ELD4C.js +0 -1205
  32. package/dist/chunk-7FYY4GZM.js +0 -1205
  33. package/dist/chunk-7IUE7ODU.js +0 -254
  34. package/dist/chunk-7RMLKZIS.js +0 -108
  35. package/dist/chunk-AA3OPP4Z.js +0 -432
  36. package/dist/chunk-AEWLSYPH.js +0 -72
  37. package/dist/chunk-ASSZDR6I.js +0 -108
  38. package/dist/chunk-AYRJVWUC.js +0 -1205
  39. package/dist/chunk-CCYI5O3D.js +0 -148
  40. package/dist/chunk-D3I3ZSE2.js +0 -411
  41. package/dist/chunk-DACSKLY6.js +0 -219
  42. package/dist/chunk-DW5B4BL7.js +0 -108
  43. package/dist/chunk-EKSACBTJ.js +0 -1070
  44. package/dist/chunk-EXEZZADG.js +0 -248
  45. package/dist/chunk-F3Y7EL7K.js +0 -83
  46. package/dist/chunk-FBQWSDPC.js +0 -1328
  47. package/dist/chunk-FHRZNOMW.js +0 -1205
  48. package/dist/chunk-IM7N24MT.js +0 -129
  49. package/dist/chunk-IPNYIXFK.js +0 -1178
  50. package/dist/chunk-J7MRSZIO.js +0 -167
  51. package/dist/chunk-JITKI2OI.js +0 -106
  52. package/dist/chunk-JWZXYVET.js +0 -1068
  53. package/dist/chunk-KCQUXVYT.js +0 -108
  54. package/dist/chunk-KOQ43OX6.js +0 -1281
  55. package/dist/chunk-KYVWO6ZM.js +0 -1069
  56. package/dist/chunk-L3VHARQH.js +0 -413
  57. package/dist/chunk-LBWDS6BE.js +0 -288
  58. package/dist/chunk-LOVAHSQV.js +0 -411
  59. package/dist/chunk-LQOYCAGG.js +0 -446
  60. package/dist/chunk-LWAIPJ2W.js +0 -146
  61. package/dist/chunk-M5DHKW7M.js +0 -127
  62. package/dist/chunk-MBKCIJHM.js +0 -1328
  63. package/dist/chunk-MK42FMEG.js +0 -147
  64. package/dist/chunk-MOBMYHKL.js +0 -1205
  65. package/dist/chunk-MPLTNMRG.js +0 -67
  66. package/dist/chunk-NIJCVN3O.js +0 -147
  67. package/dist/chunk-NZCFSZQ7.js +0 -1205
  68. package/dist/chunk-O2UITJGH.js +0 -465
  69. package/dist/chunk-OCRA44AZ.js +0 -108
  70. package/dist/chunk-P3KJR66H.js +0 -117
  71. package/dist/chunk-PEK6JH65.js +0 -432
  72. package/dist/chunk-PJ6FFKEX.js +0 -77
  73. package/dist/chunk-PLUBBZYR.js +0 -800
  74. package/dist/chunk-PNKVD2UK.js +0 -26
  75. package/dist/chunk-PSQZURHO.js +0 -229
  76. package/dist/chunk-SGL6ISBJ.js +0 -1061
  77. package/dist/chunk-SJABZZT5.js +0 -97
  78. package/dist/chunk-TD3P3K32.js +0 -1199
  79. package/dist/chunk-TMDZJJKV.js +0 -288
  80. package/dist/chunk-UNHVZB5G.js +0 -411
  81. package/dist/chunk-VAFTWSTE.js +0 -1061
  82. package/dist/chunk-VNFXFQBB.js +0 -217
  83. package/dist/chunk-X3GVFKSJ.js +0 -1205
  84. package/dist/chunk-XZ3S56RQ.js +0 -1061
  85. package/dist/chunk-Y72C7F6O.js +0 -148
  86. package/dist/chunk-YLICP577.js +0 -1205
  87. package/dist/chunk-YX6AXLVK.js +0 -159
  88. package/dist/chunk-ZCQYHTNU.js +0 -146
  89. package/dist/cli.d.ts +0 -1
  90. package/dist/cli.js +0 -1105
  91. package/dist/cloud-crystal.js +0 -6
  92. package/dist/core.d.ts +0 -232
  93. package/dist/core.js +0 -12
  94. package/dist/crypto.d.ts +0 -20
  95. package/dist/crypto.js +0 -27
  96. package/dist/crystal-capture.sh +0 -29
  97. package/dist/crystal-serve.d.ts +0 -4
  98. package/dist/crystal-serve.js +0 -252
  99. package/dist/dev-update-SZ2Z4WCQ.js +0 -6
  100. package/dist/discover.d.ts +0 -30
  101. package/dist/discover.js +0 -177
  102. package/dist/doctor.d.ts +0 -9
  103. package/dist/doctor.js +0 -334
  104. package/dist/dream-weaver.d.ts +0 -8
  105. package/dist/dream-weaver.js +0 -56
  106. package/dist/file-sync.d.ts +0 -48
  107. package/dist/file-sync.js +0 -18
  108. package/dist/installer.d.ts +0 -61
  109. package/dist/installer.js +0 -676
  110. package/dist/ldm-backup.sh +0 -116
  111. package/dist/ldm.d.ts +0 -50
  112. package/dist/ldm.js +0 -32
  113. package/dist/mcp-server.d.ts +0 -1
  114. package/dist/mcp-server.js +0 -265
  115. package/dist/migrate.d.ts +0 -1
  116. package/dist/migrate.js +0 -89
  117. package/dist/mirror-sync.d.ts +0 -1
  118. package/dist/mirror-sync.js +0 -159
  119. package/dist/oc-backfill.d.ts +0 -19
  120. package/dist/oc-backfill.js +0 -74
  121. package/dist/openclaw.d.ts +0 -5
  122. package/dist/openclaw.js +0 -423
  123. package/dist/pair.d.ts +0 -4
  124. package/dist/pair.js +0 -75
  125. package/dist/poller.d.ts +0 -1
  126. package/dist/poller.js +0 -634
  127. package/dist/role.d.ts +0 -24
  128. package/dist/role.js +0 -13
  129. package/dist/search-pipeline-4K4OJSSS.js +0 -255
  130. package/dist/search-pipeline-4PRS6LI7.js +0 -280
  131. package/dist/search-pipeline-7UJMXPLO.js +0 -280
  132. package/dist/search-pipeline-DQTRLGBH.js +0 -74
  133. package/dist/search-pipeline-HNG37REH.js +0 -282
  134. package/dist/search-pipeline-IZFPLBUB.js +0 -280
  135. package/dist/search-pipeline-MID6F26Q.js +0 -73
  136. package/dist/search-pipeline-N52JZFNN.js +0 -282
  137. package/dist/search-pipeline-OPB2PRQQ.js +0 -280
  138. package/dist/search-pipeline-VXTE5HAD.js +0 -262
  139. package/dist/search-pipeline-XHFKADRG.js +0 -73
  140. package/dist/staging.d.ts +0 -29
  141. package/dist/staging.js +0 -21
  142. package/dist/summarize.d.ts +0 -19
  143. package/dist/summarize.js +0 -10
  144. package/dist/worker-demo.js +0 -186
  145. package/dist/worker-mcp.js +0 -404
  146. package/dist/worker.js +0 -137
@@ -1,413 +0,0 @@
1
- // src/core.ts
2
- import * as lancedb from "@lancedb/lancedb";
3
- import Database from "better-sqlite3";
4
- import { readFileSync, existsSync, mkdirSync } from "fs";
5
- import { execSync } from "child_process";
6
- import { join } from "path";
7
- import http from "http";
8
- import https from "https";
9
- async function embedOpenAI(texts, apiKey, model) {
10
- return new Promise((resolve, reject) => {
11
- const body = JSON.stringify({ input: texts, model });
12
- const req = https.request({
13
- hostname: "api.openai.com",
14
- path: "/v1/embeddings",
15
- method: "POST",
16
- headers: {
17
- "Content-Type": "application/json",
18
- "Authorization": `Bearer ${apiKey}`,
19
- "Content-Length": Buffer.byteLength(body)
20
- },
21
- timeout: 3e4
22
- }, (res) => {
23
- let data = "";
24
- res.on("data", (chunk) => data += chunk);
25
- res.on("end", () => {
26
- if (res.statusCode !== 200) {
27
- reject(new Error(`OpenAI API error ${res.statusCode}: ${data.slice(0, 200)}`));
28
- return;
29
- }
30
- const parsed = JSON.parse(data);
31
- resolve(parsed.data.map((d) => d.embedding));
32
- });
33
- });
34
- req.on("error", reject);
35
- req.on("timeout", () => {
36
- req.destroy();
37
- reject(new Error("OpenAI timeout"));
38
- });
39
- req.write(body);
40
- req.end();
41
- });
42
- }
43
- async function embedOllama(texts, host, model) {
44
- const results = [];
45
- for (const text of texts) {
46
- const result = await new Promise((resolve, reject) => {
47
- const url = new URL("/api/embeddings", host);
48
- const body = JSON.stringify({ model, prompt: text });
49
- const req = http.request({
50
- hostname: url.hostname,
51
- port: url.port,
52
- path: url.pathname,
53
- method: "POST",
54
- headers: {
55
- "Content-Type": "application/json",
56
- "Content-Length": Buffer.byteLength(body)
57
- },
58
- timeout: 15e3
59
- }, (res) => {
60
- let data = "";
61
- res.on("data", (chunk) => data += chunk);
62
- res.on("end", () => {
63
- if (res.statusCode !== 200) {
64
- reject(new Error(`Ollama error ${res.statusCode}: ${data.slice(0, 200)}`));
65
- return;
66
- }
67
- resolve(JSON.parse(data).embedding);
68
- });
69
- });
70
- req.on("error", reject);
71
- req.on("timeout", () => {
72
- req.destroy();
73
- reject(new Error("Ollama timeout"));
74
- });
75
- req.write(body);
76
- req.end();
77
- });
78
- results.push(result);
79
- }
80
- return results;
81
- }
82
- async function embedGoogle(texts, apiKey, model) {
83
- return new Promise((resolve, reject) => {
84
- const body = JSON.stringify({
85
- requests: texts.map((text) => ({ model: `models/${model}`, content: { parts: [{ text }] } }))
86
- });
87
- const req = https.request({
88
- hostname: "generativelanguage.googleapis.com",
89
- path: `/v1beta/models/${model}:batchEmbedContents?key=${apiKey}`,
90
- method: "POST",
91
- headers: {
92
- "Content-Type": "application/json",
93
- "Content-Length": Buffer.byteLength(body)
94
- },
95
- timeout: 3e4
96
- }, (res) => {
97
- let data = "";
98
- res.on("data", (chunk) => data += chunk);
99
- res.on("end", () => {
100
- if (res.statusCode !== 200) {
101
- reject(new Error(`Google API error ${res.statusCode}: ${data.slice(0, 200)}`));
102
- return;
103
- }
104
- const parsed = JSON.parse(data);
105
- resolve(parsed.embeddings.map((e) => e.values));
106
- });
107
- });
108
- req.on("error", reject);
109
- req.on("timeout", () => {
110
- req.destroy();
111
- reject(new Error("Google timeout"));
112
- });
113
- req.write(body);
114
- req.end();
115
- });
116
- }
117
- var Crystal = class {
118
- config;
119
- lanceDb = null;
120
- sqliteDb = null;
121
- chunksTable = null;
122
- constructor(config) {
123
- this.config = config;
124
- if (!existsSync(config.dataDir)) {
125
- mkdirSync(config.dataDir, { recursive: true });
126
- }
127
- }
128
- // ── Initialization ──
129
- async init() {
130
- const lanceDir = join(this.config.dataDir, "lance");
131
- const sqlitePath = join(this.config.dataDir, "crystal.db");
132
- if (!existsSync(lanceDir)) mkdirSync(lanceDir, { recursive: true });
133
- this.lanceDb = await lancedb.connect(lanceDir);
134
- this.sqliteDb = new Database(sqlitePath);
135
- this.sqliteDb.pragma("journal_mode = WAL");
136
- this.initSqliteTables();
137
- await this.initLanceTables();
138
- }
139
- initSqliteTables() {
140
- const db = this.sqliteDb;
141
- db.exec(`
142
- CREATE TABLE IF NOT EXISTS sources (
143
- id INTEGER PRIMARY KEY AUTOINCREMENT,
144
- type TEXT NOT NULL,
145
- uri TEXT NOT NULL,
146
- title TEXT,
147
- agent_id TEXT NOT NULL,
148
- metadata TEXT DEFAULT '{}',
149
- ingested_at TEXT NOT NULL,
150
- chunk_count INTEGER DEFAULT 0
151
- );
152
-
153
- CREATE TABLE IF NOT EXISTS capture_state (
154
- agent_id TEXT NOT NULL,
155
- source_id TEXT NOT NULL,
156
- last_message_count INTEGER DEFAULT 0,
157
- capture_count INTEGER DEFAULT 0,
158
- last_capture_at TEXT,
159
- PRIMARY KEY (agent_id, source_id)
160
- );
161
-
162
- CREATE TABLE IF NOT EXISTS memories (
163
- id INTEGER PRIMARY KEY AUTOINCREMENT,
164
- text TEXT NOT NULL,
165
- category TEXT NOT NULL DEFAULT 'fact',
166
- confidence REAL NOT NULL DEFAULT 1.0,
167
- source_ids TEXT DEFAULT '[]',
168
- status TEXT NOT NULL DEFAULT 'active',
169
- created_at TEXT NOT NULL,
170
- updated_at TEXT NOT NULL
171
- );
172
-
173
- CREATE TABLE IF NOT EXISTS entities (
174
- id INTEGER PRIMARY KEY AUTOINCREMENT,
175
- name TEXT NOT NULL UNIQUE,
176
- type TEXT NOT NULL DEFAULT 'concept',
177
- description TEXT,
178
- properties TEXT DEFAULT '{}',
179
- created_at TEXT NOT NULL,
180
- updated_at TEXT NOT NULL
181
- );
182
-
183
- CREATE TABLE IF NOT EXISTS relationships (
184
- id INTEGER PRIMARY KEY AUTOINCREMENT,
185
- source_id INTEGER NOT NULL REFERENCES entities(id),
186
- target_id INTEGER NOT NULL REFERENCES entities(id),
187
- type TEXT NOT NULL,
188
- description TEXT,
189
- weight REAL DEFAULT 1.0,
190
- valid_from TEXT NOT NULL,
191
- valid_until TEXT,
192
- created_at TEXT NOT NULL
193
- );
194
-
195
- CREATE INDEX IF NOT EXISTS idx_sources_agent ON sources(agent_id);
196
- CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
197
- CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
198
- CREATE INDEX IF NOT EXISTS idx_relationships_source ON relationships(source_id);
199
- CREATE INDEX IF NOT EXISTS idx_relationships_target ON relationships(target_id);
200
- `);
201
- }
202
- async initLanceTables() {
203
- const db = this.lanceDb;
204
- const tableNames = await db.tableNames();
205
- if (tableNames.includes("chunks")) {
206
- this.chunksTable = await db.openTable("chunks");
207
- }
208
- }
209
- // ── Embedding ──
210
- async embed(texts) {
211
- if (texts.length === 0) return [];
212
- const cfg = this.config;
213
- switch (cfg.embeddingProvider) {
214
- case "openai":
215
- if (!cfg.openaiApiKey) throw new Error("OpenAI API key required");
216
- return embedOpenAI(texts, cfg.openaiApiKey, cfg.openaiModel || "text-embedding-3-small");
217
- case "ollama":
218
- return embedOllama(texts, cfg.ollamaHost || "http://localhost:11434", cfg.ollamaModel || "nomic-embed-text");
219
- case "google":
220
- if (!cfg.googleApiKey) throw new Error("Google API key required");
221
- return embedGoogle(texts, cfg.googleApiKey, cfg.googleModel || "text-embedding-004");
222
- default:
223
- throw new Error(`Unknown embedding provider: ${cfg.embeddingProvider}`);
224
- }
225
- }
226
- // ── Chunking ──
227
- chunkText(text, targetTokens = 400, overlapTokens = 80) {
228
- const targetChars = targetTokens * 4;
229
- const overlapChars = overlapTokens * 4;
230
- const chunks = [];
231
- let start = 0;
232
- while (start < text.length) {
233
- let end = Math.min(start + targetChars, text.length);
234
- if (end < text.length) {
235
- const minBreak = start + Math.floor(targetChars * 0.5);
236
- const paraBreak = text.lastIndexOf("\n\n", end);
237
- if (paraBreak > minBreak) {
238
- end = paraBreak;
239
- } else {
240
- const sentBreak = text.lastIndexOf(". ", end);
241
- if (sentBreak > minBreak) {
242
- end = sentBreak + 1;
243
- }
244
- }
245
- }
246
- const chunk = text.slice(start, end).trim();
247
- if (chunk.length > 0) chunks.push(chunk);
248
- if (end >= text.length) break;
249
- start = end - overlapChars;
250
- if (start <= (chunks.length > 0 ? end - targetChars : 0)) {
251
- start = end;
252
- }
253
- }
254
- return chunks;
255
- }
256
- // ── Ingest ──
257
- async ingest(chunks) {
258
- if (chunks.length === 0) return 0;
259
- const texts = chunks.map((c) => c.text);
260
- const embeddings = await this.embed(texts);
261
- const records = chunks.map((chunk, i) => ({
262
- text: chunk.text,
263
- vector: embeddings[i],
264
- role: chunk.role,
265
- source_type: chunk.source_type,
266
- source_id: chunk.source_id,
267
- agent_id: chunk.agent_id,
268
- token_count: chunk.token_count,
269
- created_at: chunk.created_at || (/* @__PURE__ */ new Date()).toISOString()
270
- }));
271
- if (!this.chunksTable) {
272
- this.chunksTable = await this.lanceDb.createTable("chunks", records);
273
- } else {
274
- await this.chunksTable.add(records);
275
- }
276
- return records.length;
277
- }
278
- // ── Search ──
279
- async search(query, limit = 5, filter) {
280
- if (!this.chunksTable) return [];
281
- const [embedding] = await this.embed([query]);
282
- let queryBuilder = this.chunksTable.vectorSearch(embedding).distanceType("cosine").limit(limit);
283
- if (filter?.agent_id) {
284
- queryBuilder = queryBuilder.where(`agent_id = '${filter.agent_id}'`);
285
- }
286
- if (filter?.source_type) {
287
- queryBuilder = queryBuilder.where(`source_type = '${filter.source_type}'`);
288
- }
289
- const results = await queryBuilder.toArray();
290
- return results.map((row) => ({
291
- text: row.text,
292
- role: row.role,
293
- score: row._distance != null ? 1 - row._distance : 0,
294
- source_type: row.source_type,
295
- source_id: row.source_id,
296
- agent_id: row.agent_id,
297
- created_at: row.created_at
298
- }));
299
- }
300
- // ── Remember (explicit fact storage) ──
301
- async remember(text, category = "fact") {
302
- const db = this.sqliteDb;
303
- const now = (/* @__PURE__ */ new Date()).toISOString();
304
- const stmt = db.prepare(`
305
- INSERT INTO memories (text, category, confidence, source_ids, status, created_at, updated_at)
306
- VALUES (?, ?, 1.0, '[]', 'active', ?, ?)
307
- `);
308
- const result = stmt.run(text, category, now, now);
309
- await this.ingest([{
310
- text,
311
- role: "system",
312
- source_type: "manual",
313
- source_id: `memory:${result.lastInsertRowid}`,
314
- agent_id: "system",
315
- token_count: Math.ceil(text.length / 4),
316
- created_at: now
317
- }]);
318
- return result.lastInsertRowid;
319
- }
320
- // ── Forget (deprecate a memory) ──
321
- forget(memoryId) {
322
- const db = this.sqliteDb;
323
- const now = (/* @__PURE__ */ new Date()).toISOString();
324
- const result = db.prepare(`
325
- UPDATE memories SET status = 'deprecated', updated_at = ? WHERE id = ? AND status = 'active'
326
- `).run(now, memoryId);
327
- return result.changes > 0;
328
- }
329
- // ── Status ──
330
- async status() {
331
- const db = this.sqliteDb;
332
- let chunks = 0;
333
- let oldestChunk = null;
334
- let newestChunk = null;
335
- const agents = [];
336
- if (this.chunksTable) {
337
- chunks = await this.chunksTable.countRows();
338
- try {
339
- const sample = await this.chunksTable.search([]).limit(1).toArray();
340
- } catch {
341
- }
342
- }
343
- const memories = db.prepare("SELECT COUNT(*) as count FROM memories WHERE status = ?").get("active")?.count || 0;
344
- const sources = db.prepare("SELECT COUNT(*) as count FROM sources").get()?.count || 0;
345
- const agentRows = db.prepare("SELECT DISTINCT agent_id FROM sources").all();
346
- agents.push(...agentRows.map((r) => r.agent_id));
347
- return {
348
- chunks,
349
- memories,
350
- sources,
351
- agents,
352
- oldestChunk,
353
- newestChunk,
354
- embeddingProvider: this.config.embeddingProvider,
355
- dataDir: this.config.dataDir
356
- };
357
- }
358
- // ── Capture State (for incremental ingestion) ──
359
- getCaptureState(agentId, sourceId) {
360
- const db = this.sqliteDb;
361
- const row = db.prepare("SELECT last_message_count, capture_count FROM capture_state WHERE agent_id = ? AND source_id = ?").get(agentId, sourceId);
362
- return row || { lastMessageCount: 0, captureCount: 0 };
363
- }
364
- setCaptureState(agentId, sourceId, messageCount, captureCount) {
365
- const db = this.sqliteDb;
366
- db.prepare(`
367
- INSERT OR REPLACE INTO capture_state (agent_id, source_id, last_message_count, capture_count, last_capture_at)
368
- VALUES (?, ?, ?, ?, ?)
369
- `).run(agentId, sourceId, messageCount, captureCount, (/* @__PURE__ */ new Date()).toISOString());
370
- }
371
- // ── Cleanup ──
372
- close() {
373
- this.sqliteDb?.close();
374
- }
375
- };
376
- function resolveConfig(overrides) {
377
- const openclawHome = process.env.OPENCLAW_HOME || join(process.env.HOME || "/Users/lesa", ".openclaw");
378
- const dataDir = overrides?.dataDir || join(openclawHome, "memory-crystal");
379
- const openaiApiKey = overrides?.openaiApiKey || process.env.OPENAI_API_KEY || opRead(openclawHome, "OpenAI API", "api key");
380
- const googleApiKey = overrides?.googleApiKey || process.env.GOOGLE_API_KEY || opRead(openclawHome, "Google AI", "api key");
381
- const remoteToken = overrides?.remoteToken || process.env.CRYSTAL_REMOTE_TOKEN || opRead(openclawHome, "Memory Crystal Remote", "token");
382
- return {
383
- dataDir,
384
- embeddingProvider: overrides?.embeddingProvider || process.env.CRYSTAL_EMBEDDING_PROVIDER || "openai",
385
- openaiApiKey,
386
- openaiModel: overrides?.openaiModel || process.env.CRYSTAL_OPENAI_MODEL || "text-embedding-3-small",
387
- ollamaHost: overrides?.ollamaHost || process.env.CRYSTAL_OLLAMA_HOST || "http://localhost:11434",
388
- ollamaModel: overrides?.ollamaModel || process.env.CRYSTAL_OLLAMA_MODEL || "nomic-embed-text",
389
- googleApiKey,
390
- googleModel: overrides?.googleModel || process.env.CRYSTAL_GOOGLE_MODEL || "text-embedding-004",
391
- remoteUrl: overrides?.remoteUrl || process.env.CRYSTAL_REMOTE_URL,
392
- remoteToken
393
- };
394
- }
395
- function opRead(openclawHome, item, field) {
396
- try {
397
- const saTokenPath = join(openclawHome, "secrets", "op-sa-token");
398
- if (!existsSync(saTokenPath)) return void 0;
399
- const saToken = readFileSync(saTokenPath, "utf8").trim();
400
- return execSync(`op read "op://Agent Secrets/${item}/${field}"`, {
401
- encoding: "utf8",
402
- env: { ...process.env, OP_SERVICE_ACCOUNT_TOKEN: saToken },
403
- timeout: 1e4
404
- }).trim() || void 0;
405
- } catch {
406
- return void 0;
407
- }
408
- }
409
-
410
- export {
411
- Crystal,
412
- resolveConfig
413
- };
@@ -1,288 +0,0 @@
1
- import {
2
- decrypt,
3
- decryptJSON,
4
- encrypt,
5
- encryptJSON,
6
- loadRelayKey
7
- } from "./chunk-ASSZDR6I.js";
8
- import {
9
- ldmPaths,
10
- resolveStatePath,
11
- stateWritePath
12
- } from "./chunk-DACSKLY6.js";
13
-
14
- // src/file-sync.ts
15
- import { createHash } from "crypto";
16
- import {
17
- existsSync,
18
- mkdirSync,
19
- readFileSync,
20
- writeFileSync,
21
- unlinkSync,
22
- readdirSync,
23
- statSync
24
- } from "fs";
25
- import { join, relative, dirname } from "path";
26
- var RELAY_URL = process.env.CRYSTAL_RELAY_URL || "";
27
- var RELAY_TOKEN = process.env.CRYSTAL_RELAY_TOKEN || "";
28
- var EXCLUDE_PATTERNS = [
29
- "memory/crystal.db",
30
- // DB syncs via delta chunks, not file copy
31
- "memory/crystal.db-wal",
32
- "memory/crystal.db-shm",
33
- "memory/crystal.db.bak",
34
- "memory/crystal.db.tmp",
35
- "memory/lance/",
36
- // LanceDB (deprecated, not synced)
37
- "state/",
38
- // Local state files (watermarks, etc.)
39
- "secrets/",
40
- // Encryption keys, tokens
41
- "staging/",
42
- // Staging pipeline (Core-only)
43
- "bin/",
44
- // Local scripts
45
- ".DS_Store"
46
- ];
47
- function shouldExclude(relativePath) {
48
- for (const pattern of EXCLUDE_PATTERNS) {
49
- if (relativePath === pattern || relativePath.startsWith(pattern)) return true;
50
- }
51
- const parts = relativePath.split("/");
52
- for (const part of parts) {
53
- if (part.startsWith(".") && part !== ".ldm") return true;
54
- }
55
- return false;
56
- }
57
- function scanDir(baseDir, currentDir, entries) {
58
- if (!existsSync(currentDir)) return;
59
- const items = readdirSync(currentDir);
60
- for (const item of items) {
61
- const fullPath = join(currentDir, item);
62
- const relPath = relative(baseDir, fullPath);
63
- if (shouldExclude(relPath)) continue;
64
- let stat;
65
- try {
66
- stat = statSync(fullPath);
67
- } catch {
68
- continue;
69
- }
70
- if (stat.isDirectory()) {
71
- scanDir(baseDir, fullPath, entries);
72
- } else if (stat.isFile()) {
73
- if (stat.size > 50 * 1024 * 1024) continue;
74
- const content = readFileSync(fullPath);
75
- const sha256 = createHash("sha256").update(content).digest("hex");
76
- entries.push({ path: relPath, sha256, size: stat.size });
77
- }
78
- }
79
- }
80
- function generateManifest() {
81
- const paths = ldmPaths();
82
- const entries = [];
83
- scanDir(paths.root, paths.root, entries);
84
- return {
85
- version: 1,
86
- generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
87
- fileCount: entries.length,
88
- entries
89
- };
90
- }
91
- function compareManifest(coreManifest) {
92
- const paths = ldmPaths();
93
- const upsert = [];
94
- const toDelete = [];
95
- const localManifest = generateManifest();
96
- const localMap = new Map(localManifest.entries.map((e) => [e.path, e]));
97
- const coreMap = new Map(coreManifest.entries.map((e) => [e.path, e]));
98
- for (const entry of coreManifest.entries) {
99
- const local = localMap.get(entry.path);
100
- if (!local || local.sha256 !== entry.sha256) {
101
- upsert.push({ path: entry.path, sha256: entry.sha256 });
102
- }
103
- }
104
- for (const entry of localManifest.entries) {
105
- if (!coreMap.has(entry.path)) {
106
- toDelete.push(entry.path);
107
- }
108
- }
109
- return { upsert, delete: toDelete };
110
- }
111
- function loadFileSyncState() {
112
- const statePath = resolveStatePath("file-sync-state.json");
113
- try {
114
- if (existsSync(statePath)) {
115
- return JSON.parse(readFileSync(statePath, "utf-8"));
116
- }
117
- } catch {
118
- }
119
- return { lastSync: null, lastManifestHash: null, filesTransferred: 0, filesDeleted: 0 };
120
- }
121
- function saveFileSyncState(state) {
122
- const writePath = stateWritePath("file-sync-state.json");
123
- writeFileSync(writePath, JSON.stringify(state, null, 2));
124
- }
125
- async function pushFileSync() {
126
- if (!RELAY_URL || !RELAY_TOKEN) {
127
- throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
128
- }
129
- const relayKey = loadRelayKey();
130
- const paths = ldmPaths();
131
- const manifest = generateManifest();
132
- const manifestJson = JSON.stringify(manifest.entries.map((e) => `${e.path}:${e.sha256}`));
133
- const manifestHash = createHash("sha256").update(manifestJson).digest("hex");
134
- const state = loadFileSyncState();
135
- if (state.lastManifestHash === manifestHash) {
136
- return { manifest: 0, files: 0 };
137
- }
138
- const encryptedManifest = encryptJSON(manifest, relayKey);
139
- const manifestResp = await fetch(`${RELAY_URL}/drop/files`, {
140
- method: "POST",
141
- headers: {
142
- "Authorization": `Bearer ${RELAY_TOKEN}`,
143
- "Content-Type": "application/octet-stream",
144
- "X-File-Type": "manifest"
145
- },
146
- body: JSON.stringify(encryptedManifest)
147
- });
148
- if (!manifestResp.ok) {
149
- throw new Error(`Manifest push failed: ${manifestResp.status} ${await manifestResp.text()}`);
150
- }
151
- let filesPushed = 0;
152
- for (const entry of manifest.entries) {
153
- const fullPath = join(paths.root, entry.path);
154
- if (!existsSync(fullPath)) continue;
155
- const content = readFileSync(fullPath);
156
- const encrypted = encrypt(content, relayKey);
157
- const filePayload = JSON.stringify({
158
- path: entry.path,
159
- sha256: entry.sha256,
160
- size: entry.size,
161
- data: encrypted
162
- });
163
- const fileResp = await fetch(`${RELAY_URL}/drop/files`, {
164
- method: "POST",
165
- headers: {
166
- "Authorization": `Bearer ${RELAY_TOKEN}`,
167
- "Content-Type": "application/octet-stream",
168
- "X-File-Type": "file"
169
- },
170
- body: filePayload
171
- });
172
- if (fileResp.ok) filesPushed++;
173
- }
174
- state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
175
- state.lastManifestHash = manifestHash;
176
- state.filesTransferred += filesPushed;
177
- saveFileSyncState(state);
178
- return { manifest: manifest.fileCount, files: filesPushed };
179
- }
180
- async function pullFileSync() {
181
- if (!RELAY_URL || !RELAY_TOKEN) {
182
- throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
183
- }
184
- const relayKey = loadRelayKey();
185
- const paths = ldmPaths();
186
- const listResp = await fetch(`${RELAY_URL}/pickup/files`, {
187
- headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
188
- });
189
- if (!listResp.ok) {
190
- throw new Error(`File sync list failed: ${listResp.status} ${await listResp.text()}`);
191
- }
192
- const listData = await listResp.json();
193
- if (listData.count === 0) {
194
- return { imported: 0, deleted: 0 };
195
- }
196
- let coreManifest = null;
197
- const fileBlobs = [];
198
- for (const blob of listData.blobs) {
199
- try {
200
- const blobResp = await fetch(`${RELAY_URL}/pickup/files/${blob.id}`, {
201
- headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
202
- });
203
- if (!blobResp.ok) continue;
204
- const text = await blobResp.text();
205
- const parsed = JSON.parse(text);
206
- if (parsed.v !== void 0 && parsed.nonce !== void 0) {
207
- try {
208
- const manifest = decryptJSON(parsed, relayKey);
209
- if (manifest.version && manifest.entries) {
210
- coreManifest = manifest;
211
- }
212
- } catch {
213
- }
214
- } else if (parsed.path && parsed.data) {
215
- fileBlobs.push({
216
- id: blob.id,
217
- path: parsed.path,
218
- sha256: parsed.sha256,
219
- data: parsed.data
220
- });
221
- }
222
- await fetch(`${RELAY_URL}/confirm/files/${blob.id}`, {
223
- method: "DELETE",
224
- headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
225
- });
226
- } catch (err) {
227
- process.stderr.write(`[file-sync] error processing blob ${blob.id}: ${err.message}
228
- `);
229
- }
230
- }
231
- if (!coreManifest) {
232
- process.stderr.write("[file-sync] no manifest found in relay\n");
233
- return { imported: 0, deleted: 0 };
234
- }
235
- const delta = compareManifest(coreManifest);
236
- const fileBlobMap = new Map(fileBlobs.map((f) => [f.path, f]));
237
- let imported = 0;
238
- for (const entry of delta.upsert) {
239
- const fileBlob = fileBlobMap.get(entry.path);
240
- if (!fileBlob) {
241
- continue;
242
- }
243
- try {
244
- const content = decrypt(fileBlob.data, relayKey);
245
- const actualHash = createHash("sha256").update(content).digest("hex");
246
- if (actualHash !== entry.sha256) {
247
- process.stderr.write(`[file-sync] hash mismatch for ${entry.path}, skipping
248
- `);
249
- continue;
250
- }
251
- const destPath = join(paths.root, entry.path);
252
- mkdirSync(dirname(destPath), { recursive: true });
253
- writeFileSync(destPath, content);
254
- imported++;
255
- } catch (err) {
256
- process.stderr.write(`[file-sync] failed to write ${entry.path}: ${err.message}
257
- `);
258
- }
259
- }
260
- let deleted = 0;
261
- for (const path of delta.delete) {
262
- const destPath = join(paths.root, path);
263
- if (existsSync(destPath)) {
264
- try {
265
- unlinkSync(destPath);
266
- deleted++;
267
- } catch (err) {
268
- process.stderr.write(`[file-sync] failed to delete ${path}: ${err.message}
269
- `);
270
- }
271
- }
272
- }
273
- const state = loadFileSyncState();
274
- state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
275
- state.filesTransferred += imported;
276
- state.filesDeleted += deleted;
277
- saveFileSyncState(state);
278
- return { imported, deleted };
279
- }
280
-
281
- export {
282
- generateManifest,
283
- compareManifest,
284
- loadFileSyncState,
285
- saveFileSyncState,
286
- pushFileSync,
287
- pullFileSync
288
- };