@wipcomputer/memory-crystal 0.7.34-alpha.2 → 0.7.34-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/LICENSE +1 -1
  2. package/dist/bridge.js +64 -7
  3. package/dist/bulk-copy.js +67 -16
  4. package/dist/cc-hook.js +2163 -62
  5. package/dist/cc-poller.js +1967 -70
  6. package/dist/cli.js +4538 -139
  7. package/dist/core.js +1789 -6
  8. package/dist/crypto.js +153 -14
  9. package/dist/crystal-serve.js +64 -12
  10. package/dist/doctor.js +517 -52
  11. package/dist/dream-weaver.js +1755 -7
  12. package/dist/file-sync.js +407 -9
  13. package/dist/installer.js +840 -145
  14. package/dist/ldm.js +231 -16
  15. package/dist/mcp-server.js +1882 -17
  16. package/dist/migrate.js +1707 -11
  17. package/dist/mirror-sync.js +2052 -34
  18. package/dist/openclaw.js +1895 -84
  19. package/dist/pair.js +112 -16
  20. package/dist/poller.js +2275 -80
  21. package/dist/role.js +159 -7
  22. package/dist/staging.js +235 -10
  23. package/dist/summarize.js +142 -5
  24. package/package.json +7 -4
  25. package/dist/chunk-25LXQJ4Z.js +0 -110
  26. package/dist/chunk-2DRXIRQW.js +0 -97
  27. package/dist/chunk-2GBYLMEF.js +0 -1385
  28. package/dist/chunk-2ZNH5F6E.js +0 -1281
  29. package/dist/chunk-3G3SFYYI.js +0 -288
  30. package/dist/chunk-3RG5ZIWI.js +0 -10
  31. package/dist/chunk-3S6TI23B.js +0 -97
  32. package/dist/chunk-3VFIJYS4.js +0 -818
  33. package/dist/chunk-437F27T6.js +0 -97
  34. package/dist/chunk-52QE3YI3.js +0 -1169
  35. package/dist/chunk-57RP3DIN.js +0 -1205
  36. package/dist/chunk-5HSZ4W2P.js +0 -62
  37. package/dist/chunk-5I7GMRDN.js +0 -146
  38. package/dist/chunk-645IPXW3.js +0 -290
  39. package/dist/chunk-7A7ELD4C.js +0 -1205
  40. package/dist/chunk-7FYY4GZM.js +0 -1205
  41. package/dist/chunk-7IUE7ODU.js +0 -254
  42. package/dist/chunk-7RMLKZIS.js +0 -108
  43. package/dist/chunk-AA3OPP4Z.js +0 -432
  44. package/dist/chunk-AEWLSYPH.js +0 -72
  45. package/dist/chunk-ASSZDR6I.js +0 -108
  46. package/dist/chunk-AYRJVWUC.js +0 -1205
  47. package/dist/chunk-CCYI5O3D.js +0 -148
  48. package/dist/chunk-CGIDSAJB.js +0 -288
  49. package/dist/chunk-D3I3ZSE2.js +0 -411
  50. package/dist/chunk-D3MACYZ4.js +0 -108
  51. package/dist/chunk-DACSKLY6.js +0 -219
  52. package/dist/chunk-DFQ72B7M.js +0 -248
  53. package/dist/chunk-DW5B4BL7.js +0 -108
  54. package/dist/chunk-EKSACBTJ.js +0 -1070
  55. package/dist/chunk-EXEZZADG.js +0 -248
  56. package/dist/chunk-F3Y7EL7K.js +0 -83
  57. package/dist/chunk-FBQWSDPC.js +0 -1328
  58. package/dist/chunk-FHRZNOMW.js +0 -1205
  59. package/dist/chunk-IM7N24MT.js +0 -129
  60. package/dist/chunk-IPNYIXFK.js +0 -1178
  61. package/dist/chunk-J7MRSZIO.js +0 -167
  62. package/dist/chunk-JITKI2OI.js +0 -106
  63. package/dist/chunk-JWZXYVET.js +0 -1068
  64. package/dist/chunk-KCQUXVYT.js +0 -108
  65. package/dist/chunk-KOQ43OX6.js +0 -1281
  66. package/dist/chunk-KYVWO6ZM.js +0 -1069
  67. package/dist/chunk-L3VHARQH.js +0 -413
  68. package/dist/chunk-LBWDS6BE.js +0 -288
  69. package/dist/chunk-LOVAHSQV.js +0 -411
  70. package/dist/chunk-LQOYCAGG.js +0 -446
  71. package/dist/chunk-LWAIPJ2W.js +0 -146
  72. package/dist/chunk-M5DHKW7M.js +0 -127
  73. package/dist/chunk-MBKCIJHM.js +0 -1328
  74. package/dist/chunk-MK42FMEG.js +0 -147
  75. package/dist/chunk-MOBMYHKL.js +0 -1205
  76. package/dist/chunk-MPLTNMRG.js +0 -67
  77. package/dist/chunk-NIJCVN3O.js +0 -147
  78. package/dist/chunk-NX647OM3.js +0 -310
  79. package/dist/chunk-NZCFSZQ7.js +0 -1205
  80. package/dist/chunk-O2UITJGH.js +0 -465
  81. package/dist/chunk-OCRA44AZ.js +0 -108
  82. package/dist/chunk-P3KJR66H.js +0 -117
  83. package/dist/chunk-PEK6JH65.js +0 -432
  84. package/dist/chunk-PJ6FFKEX.js +0 -77
  85. package/dist/chunk-PLUBBZYR.js +0 -800
  86. package/dist/chunk-PNKVD2UK.js +0 -26
  87. package/dist/chunk-PSQZURHO.js +0 -229
  88. package/dist/chunk-SGL6ISBJ.js +0 -1061
  89. package/dist/chunk-SJABZZT5.js +0 -97
  90. package/dist/chunk-TD3P3K32.js +0 -1199
  91. package/dist/chunk-TMDZJJKV.js +0 -288
  92. package/dist/chunk-UNHVZB5G.js +0 -411
  93. package/dist/chunk-VAFTWSTE.js +0 -1061
  94. package/dist/chunk-VNFXFQBB.js +0 -217
  95. package/dist/chunk-X3GVFKSJ.js +0 -1205
  96. package/dist/chunk-XZ3S56RQ.js +0 -1061
  97. package/dist/chunk-Y72C7F6O.js +0 -148
  98. package/dist/chunk-YLICP577.js +0 -1205
  99. package/dist/chunk-YX6AXLVK.js +0 -159
  100. package/dist/chunk-ZCQYHTNU.js +0 -146
  101. package/dist/cloud-crystal.js +0 -6
  102. package/dist/dev-update-SZ2Z4WCQ.js +0 -6
  103. package/dist/llm-XXLYPIOF.js +0 -16
  104. package/dist/mlx-setup-XKU67WCT.js +0 -289
  105. package/dist/search-pipeline-4K4OJSSS.js +0 -255
  106. package/dist/search-pipeline-4PRS6LI7.js +0 -280
  107. package/dist/search-pipeline-7UJMXPLO.js +0 -280
  108. package/dist/search-pipeline-CBV25NX7.js +0 -99
  109. package/dist/search-pipeline-DQTRLGBH.js +0 -74
  110. package/dist/search-pipeline-HNG37REH.js +0 -282
  111. package/dist/search-pipeline-IZFPLBUB.js +0 -280
  112. package/dist/search-pipeline-MID6F26Q.js +0 -73
  113. package/dist/search-pipeline-N52JZFNN.js +0 -282
  114. package/dist/search-pipeline-OPB2PRQQ.js +0 -280
  115. package/dist/search-pipeline-VXTE5HAD.js +0 -262
  116. package/dist/search-pipeline-XHFKADRG.js +0 -73
  117. package/dist/worker-demo.js +0 -186
  118. package/dist/worker-mcp.js +0 -404
  119. package/scripts/crystal-capture 2.sh +0 -29
  120. package/scripts/deploy-cloud 2.sh +0 -153
@@ -1,282 +0,0 @@
1
- import {
2
- __require
3
- } from "./chunk-3RG5ZIWI.js";
4
-
5
- // src/llm.ts
6
- var expansionCache = /* @__PURE__ */ new Map();
7
- var detectedProvider = null;
8
- var detectionDone = false;
9
- function getOpSecret(itemName, fieldLabel) {
10
- try {
11
- const saTokenPath = __require("path").join(__require("os").homedir(), ".openclaw/secrets/op-sa-token");
12
- const { existsSync, readFileSync } = __require("fs");
13
- if (!existsSync(saTokenPath)) return void 0;
14
- const saToken = readFileSync(saTokenPath, "utf-8").trim();
15
- const { execSync } = __require("child_process");
16
- const result = execSync(
17
- `OP_SERVICE_ACCOUNT_TOKEN="${saToken}" op item get "${itemName}" --vault "Agent Secrets" --fields "${fieldLabel}" --reveal`,
18
- { encoding: "utf-8", timeout: 5e3, stdio: ["pipe", "pipe", "pipe"] }
19
- ).trim();
20
- return result || void 0;
21
- } catch {
22
- return void 0;
23
- }
24
- }
25
- async function detectProvider() {
26
- if (detectionDone && detectedProvider) return detectedProvider;
27
- detectionDone = true;
28
- try {
29
- const resp = await fetch("http://localhost:8080/v1/models", { signal: AbortSignal.timeout(1e3) });
30
- if (resp.ok) {
31
- const data = await resp.json();
32
- const model = data?.data?.[0]?.id || "default";
33
- detectedProvider = { provider: "mlx", baseURL: "http://localhost:8080/v1", apiKey: "not-needed", model };
34
- process.stderr.write(`[memory-crystal] LLM provider: MLX (${model})
35
- `);
36
- return detectedProvider;
37
- }
38
- } catch {
39
- }
40
- try {
41
- const resp = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(1e3) });
42
- if (resp.ok) {
43
- const data = await resp.json();
44
- const models = data?.models || [];
45
- const embeddingOnly = ["nomic-embed-text", "mxbai-embed", "all-minilm", "snowflake-arctic-embed"];
46
- const chatModel = models.find((m) => !embeddingOnly.some((e) => m.name.startsWith(e)));
47
- if (chatModel) {
48
- detectedProvider = { provider: "ollama", baseURL: "http://localhost:11434/v1", apiKey: "ollama", model: chatModel.name };
49
- process.stderr.write(`[memory-crystal] LLM provider: Ollama (${chatModel.name})
50
- `);
51
- return detectedProvider;
52
- }
53
- }
54
- } catch {
55
- }
56
- const anthropicKey = process.env.ANTHROPIC_API_KEY || getOpSecret("Anthropic Auth Token - buglers lads", "credential");
57
- if (anthropicKey) {
58
- detectedProvider = { provider: "anthropic", baseURL: "https://api.anthropic.com", apiKey: anthropicKey, model: "claude-haiku-4-5-20251001" };
59
- process.stderr.write("[memory-crystal] LLM provider: Anthropic API\n");
60
- return detectedProvider;
61
- }
62
- const openaiKey = process.env.OPENAI_API_KEY || getOpSecret("OpenAI API", "api key");
63
- if (openaiKey) {
64
- detectedProvider = { provider: "openai", baseURL: "https://api.openai.com/v1", apiKey: openaiKey, model: "gpt-4o-mini" };
65
- process.stderr.write("[memory-crystal] LLM provider: OpenAI API\n");
66
- return detectedProvider;
67
- }
68
- detectedProvider = { provider: "none", baseURL: "", apiKey: "", model: "" };
69
- process.stderr.write("[memory-crystal] LLM provider: none (deep search unavailable)\n");
70
- return detectedProvider;
71
- }
72
- async function chatComplete(config, messages, maxTokens = 300) {
73
- if (config.provider === "anthropic") {
74
- return anthropicComplete(config, messages, maxTokens);
75
- }
76
- const resp = await fetch(`${config.baseURL}/chat/completions`, {
77
- method: "POST",
78
- headers: {
79
- "Content-Type": "application/json",
80
- "Authorization": `Bearer ${config.apiKey}`
81
- },
82
- body: JSON.stringify({
83
- model: config.model,
84
- messages,
85
- max_tokens: maxTokens,
86
- temperature: 0.7
87
- })
88
- });
89
- if (!resp.ok) throw new Error(`LLM request failed: ${resp.status}`);
90
- const data = await resp.json();
91
- return data.choices?.[0]?.message?.content || "";
92
- }
93
- async function anthropicComplete(config, messages, maxTokens) {
94
- const systemMsg = messages.find((m) => m.role === "system");
95
- const userMessages = messages.filter((m) => m.role !== "system");
96
- const body = {
97
- model: config.model,
98
- max_tokens: maxTokens,
99
- messages: userMessages
100
- };
101
- if (systemMsg) body.system = systemMsg.content;
102
- const resp = await fetch("https://api.anthropic.com/v1/messages", {
103
- method: "POST",
104
- headers: {
105
- "Content-Type": "application/json",
106
- "x-api-key": config.apiKey,
107
- "anthropic-version": "2023-06-01"
108
- },
109
- body: JSON.stringify(body)
110
- });
111
- if (!resp.ok) throw new Error(`Anthropic request failed: ${resp.status}`);
112
- const data = await resp.json();
113
- return data.content?.[0]?.text || "";
114
- }
115
- var EXPAND_PROMPT = `You are a search query expander. Given a search query, generate exactly 3 variations to improve search recall.
116
-
117
- Output exactly 3 lines in this format (no other text):
118
- lex: <keyword-focused variation for full-text search>
119
- vec: <semantic variation rephrased for embedding similarity>
120
- hyde: <hypothetical document snippet that would answer this query>
121
-
122
- Rules:
123
- - Each variation must contain at least one term from the original query
124
- - Keep variations concise (under 30 words each)
125
- - lex should use specific keywords and synonyms
126
- - vec should rephrase the intent naturally
127
- - hyde should be a short passage as if answering the query`;
128
- async function expandQuery(query) {
129
- const cached = expansionCache.get(query);
130
- if (cached) return cached;
131
- const config = await detectProvider();
132
- if (config.provider === "none") return [];
133
- try {
134
- const result = await chatComplete(config, [
135
- { role: "system", content: EXPAND_PROMPT },
136
- { role: "user", content: query }
137
- ], 300);
138
- const lines = result.trim().split("\n");
139
- const queryLower = query.toLowerCase();
140
- const queryTerms = queryLower.replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
141
- const hasQueryTerm = (text) => {
142
- const lower = text.toLowerCase();
143
- if (queryTerms.length === 0) return true;
144
- return queryTerms.some((term) => lower.includes(term));
145
- };
146
- const variations = lines.map((line) => {
147
- const colonIdx = line.indexOf(":");
148
- if (colonIdx === -1) return null;
149
- const type = line.slice(0, colonIdx).trim();
150
- if (type !== "lex" && type !== "vec" && type !== "hyde") return null;
151
- const text = line.slice(colonIdx + 1).trim();
152
- if (!text || !hasQueryTerm(text)) return null;
153
- return { type, text };
154
- }).filter((v) => v !== null);
155
- if (variations.length > 0) {
156
- expansionCache.set(query, variations);
157
- return variations;
158
- }
159
- } catch (err) {
160
- process.stderr.write(`[memory-crystal] Query expansion failed: ${err.message}
161
- `);
162
- }
163
- const fallback = [
164
- { type: "lex", text: query },
165
- { type: "vec", text: query },
166
- { type: "hyde", text: `Information about ${query}` }
167
- ];
168
- return fallback;
169
- }
170
- var RERANK_PROMPT = `You are a search result re-ranker. Given a query and a list of text passages, rate each passage's relevance to the query.
171
-
172
- Output one line per passage in this exact format:
173
- <index>: <score>
174
-
175
- Where index is the passage number (0-based) and score is a float from 0.0 to 1.0.
176
- - 1.0 = perfectly relevant, directly answers the query
177
- - 0.7 = highly relevant, closely related
178
- - 0.4 = somewhat relevant, tangentially related
179
- - 0.1 = barely relevant
180
- - 0.0 = not relevant at all
181
-
182
- Rate ALL passages. Output nothing else.`;
183
- async function rerankResults(query, passages) {
184
- const config = await detectProvider();
185
- if (config.provider === "none") {
186
- return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
187
- }
188
- try {
189
- const passageList = passages.map((p, i) => `[${i}] ${p.slice(0, 500)}`).join("\n\n");
190
- const result = await chatComplete(config, [
191
- { role: "system", content: RERANK_PROMPT },
192
- { role: "user", content: `Query: ${query}
193
-
194
- Passages:
195
- ${passageList}` }
196
- ], 200);
197
- const results = [];
198
- for (const line of result.trim().split("\n")) {
199
- const match = line.match(/^(\d+):\s*([\d.]+)/);
200
- if (match) {
201
- results.push({ index: parseInt(match[1]), score: parseFloat(match[2]) });
202
- }
203
- }
204
- const scored = new Set(results.map((r) => r.index));
205
- for (let i = 0; i < passages.length; i++) {
206
- if (!scored.has(i)) results.push({ index: i, score: 0 });
207
- }
208
- return results.sort((a, b) => b.score - a.score);
209
- } catch (err) {
210
- process.stderr.write(`[memory-crystal] Reranking failed: ${err.message}
211
- `);
212
- return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
213
- }
214
- }
215
-
216
- // src/search-pipeline.ts
217
- var STRONG_SIGNAL_MIN_SCORE = 0.85;
218
- var STRONG_SIGNAL_MIN_GAP = 0.15;
219
- var RERANK_CANDIDATE_LIMIT = 40;
220
- async function deepSearch(crystal, query, options = {}) {
221
- const limit = options.limit || 5;
222
- const filter = options.filter;
223
- const provider = await detectProvider();
224
- if (provider.provider === "none") {
225
- return crystal.search(query, limit, filter);
226
- }
227
- const db = crystal.sqliteDb;
228
- if (!db) return crystal.search(query, limit, filter);
229
- const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
230
- const internalFilter = { ...filter, sinceDate };
231
- const initialFts = crystal.searchFTS(query, 20, internalFilter);
232
- const topScore = initialFts[0]?.score ?? 0;
233
- const secondScore = initialFts[1]?.score ?? 0;
234
- const hasStrongSignal = initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
235
- const expanded = hasStrongSignal ? [] : await expandQuery(query);
236
- const allResultLists = [];
237
- if (initialFts.length > 0) allResultLists.push(initialFts);
238
- const [queryEmbedding] = await crystal.embed([query]);
239
- const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
240
- if (originalVec.length > 0) allResultLists.push(originalVec);
241
- for (const variation of expanded) {
242
- if (variation.type === "lex") {
243
- const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
244
- if (ftsResults.length > 0) allResultLists.push(ftsResults);
245
- } else {
246
- const [embedding] = await crystal.embed([variation.text]);
247
- const vecResults = crystal.searchVec(embedding, 20, internalFilter);
248
- if (vecResults.length > 0) allResultLists.push(vecResults);
249
- }
250
- }
251
- const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
252
- const fused = crystal.reciprocalRankFusion(allResultLists, weights);
253
- const candidates = fused.slice(0, RERANK_CANDIDATE_LIMIT);
254
- if (candidates.length === 0) return [];
255
- const passages = candidates.map((c) => c.text.slice(0, 500));
256
- const reranked = await rerankResults(query, passages);
257
- const now = Date.now();
258
- const blended = reranked.map((r) => {
259
- const candidate = candidates[r.index];
260
- if (!candidate) return null;
261
- const rrfRank = r.index + 1;
262
- let rrfWeight;
263
- if (rrfRank <= 3) rrfWeight = 0.75;
264
- else if (rrfRank <= 10) rrfWeight = 0.6;
265
- else rrfWeight = 0.4;
266
- const rrfScore = 1 / rrfRank;
267
- const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
268
- const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
269
- const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
270
- const finalScore = Math.min(blendedScore * recency * 8, 1);
271
- const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
272
- return {
273
- ...candidate,
274
- score: finalScore,
275
- freshness
276
- };
277
- }).filter((r) => r !== null);
278
- return blended.sort((a, b) => b.score - a.score).slice(0, limit);
279
- }
280
- export {
281
- deepSearch
282
- };
@@ -1,280 +0,0 @@
1
- // src/llm.ts
2
- import { existsSync, readFileSync } from "fs";
3
- import { join } from "path";
4
- import { homedir } from "os";
5
- import { execSync } from "child_process";
6
- var expansionCache = /* @__PURE__ */ new Map();
7
- var detectedProvider = null;
8
- var detectionDone = false;
9
- function getOpSecret(itemName, fieldLabel) {
10
- try {
11
- const saTokenPath = join(homedir(), ".openclaw/secrets/op-sa-token");
12
- if (!existsSync(saTokenPath)) return void 0;
13
- const saToken = readFileSync(saTokenPath, "utf-8").trim();
14
- const result = execSync(
15
- `OP_SERVICE_ACCOUNT_TOKEN="${saToken}" op item get "${itemName}" --vault "Agent Secrets" --fields "${fieldLabel}" --reveal`,
16
- { encoding: "utf-8", timeout: 5e3, stdio: ["pipe", "pipe", "pipe"] }
17
- ).trim();
18
- return result || void 0;
19
- } catch {
20
- return void 0;
21
- }
22
- }
23
- async function detectProvider() {
24
- if (detectionDone && detectedProvider) return detectedProvider;
25
- detectionDone = true;
26
- try {
27
- const resp = await fetch("http://localhost:8080/v1/models", { signal: AbortSignal.timeout(1e3) });
28
- if (resp.ok) {
29
- const data = await resp.json();
30
- const model = data?.data?.[0]?.id || "default";
31
- detectedProvider = { provider: "mlx", baseURL: "http://localhost:8080/v1", apiKey: "not-needed", model };
32
- process.stderr.write(`[memory-crystal] LLM provider: MLX (${model})
33
- `);
34
- return detectedProvider;
35
- }
36
- } catch {
37
- }
38
- try {
39
- const resp = await fetch("http://localhost:11434/api/tags", { signal: AbortSignal.timeout(1e3) });
40
- if (resp.ok) {
41
- const data = await resp.json();
42
- const models = data?.models || [];
43
- const embeddingOnly = ["nomic-embed-text", "mxbai-embed", "all-minilm", "snowflake-arctic-embed"];
44
- const chatModel = models.find((m) => !embeddingOnly.some((e) => m.name.startsWith(e)));
45
- if (chatModel) {
46
- detectedProvider = { provider: "ollama", baseURL: "http://localhost:11434/v1", apiKey: "ollama", model: chatModel.name };
47
- process.stderr.write(`[memory-crystal] LLM provider: Ollama (${chatModel.name})
48
- `);
49
- return detectedProvider;
50
- }
51
- }
52
- } catch {
53
- }
54
- const openaiKey = process.env.OPENAI_API_KEY || getOpSecret("OpenAI API", "api key");
55
- if (openaiKey) {
56
- detectedProvider = { provider: "openai", baseURL: "https://api.openai.com/v1", apiKey: openaiKey, model: "gpt-4o-mini" };
57
- process.stderr.write("[memory-crystal] LLM provider: OpenAI API\n");
58
- return detectedProvider;
59
- }
60
- const anthropicKey = process.env.ANTHROPIC_API_KEY;
61
- if (anthropicKey) {
62
- detectedProvider = { provider: "anthropic", baseURL: "https://api.anthropic.com", apiKey: anthropicKey, model: "claude-haiku-4-5-20251001" };
63
- process.stderr.write("[memory-crystal] LLM provider: Anthropic API\n");
64
- return detectedProvider;
65
- }
66
- detectedProvider = { provider: "none", baseURL: "", apiKey: "", model: "" };
67
- process.stderr.write("[memory-crystal] LLM provider: none (deep search unavailable)\n");
68
- return detectedProvider;
69
- }
70
- async function chatComplete(config, messages, maxTokens = 300) {
71
- if (config.provider === "anthropic") {
72
- return anthropicComplete(config, messages, maxTokens);
73
- }
74
- const resp = await fetch(`${config.baseURL}/chat/completions`, {
75
- method: "POST",
76
- headers: {
77
- "Content-Type": "application/json",
78
- "Authorization": `Bearer ${config.apiKey}`
79
- },
80
- body: JSON.stringify({
81
- model: config.model,
82
- messages,
83
- max_tokens: maxTokens,
84
- temperature: 0.7
85
- })
86
- });
87
- if (!resp.ok) throw new Error(`LLM request failed: ${resp.status}`);
88
- const data = await resp.json();
89
- return data.choices?.[0]?.message?.content || "";
90
- }
91
- async function anthropicComplete(config, messages, maxTokens) {
92
- const systemMsg = messages.find((m) => m.role === "system");
93
- const userMessages = messages.filter((m) => m.role !== "system");
94
- const body = {
95
- model: config.model,
96
- max_tokens: maxTokens,
97
- messages: userMessages
98
- };
99
- if (systemMsg) body.system = systemMsg.content;
100
- const resp = await fetch("https://api.anthropic.com/v1/messages", {
101
- method: "POST",
102
- headers: {
103
- "Content-Type": "application/json",
104
- "x-api-key": config.apiKey,
105
- "anthropic-version": "2023-06-01"
106
- },
107
- body: JSON.stringify(body)
108
- });
109
- if (!resp.ok) throw new Error(`Anthropic request failed: ${resp.status}`);
110
- const data = await resp.json();
111
- return data.content?.[0]?.text || "";
112
- }
113
- var EXPAND_PROMPT = `You are a search query expander. Given a search query, generate exactly 3 variations to improve search recall.
114
-
115
- Output exactly 3 lines in this format (no other text):
116
- lex: <keyword-focused variation for full-text search>
117
- vec: <semantic variation rephrased for embedding similarity>
118
- hyde: <hypothetical document snippet that would answer this query>
119
-
120
- Rules:
121
- - Each variation must contain at least one term from the original query
122
- - Keep variations concise (under 30 words each)
123
- - lex should use specific keywords and synonyms
124
- - vec should rephrase the intent naturally
125
- - hyde should be a short passage as if answering the query`;
126
- async function expandQuery(query) {
127
- const cached = expansionCache.get(query);
128
- if (cached) return cached;
129
- const config = await detectProvider();
130
- if (config.provider === "none") return [];
131
- try {
132
- const result = await chatComplete(config, [
133
- { role: "system", content: EXPAND_PROMPT },
134
- { role: "user", content: query }
135
- ], 300);
136
- const lines = result.trim().split("\n");
137
- const queryLower = query.toLowerCase();
138
- const queryTerms = queryLower.replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
139
- const hasQueryTerm = (text) => {
140
- const lower = text.toLowerCase();
141
- if (queryTerms.length === 0) return true;
142
- return queryTerms.some((term) => lower.includes(term));
143
- };
144
- const variations = lines.map((line) => {
145
- const colonIdx = line.indexOf(":");
146
- if (colonIdx === -1) return null;
147
- const type = line.slice(0, colonIdx).trim();
148
- if (type !== "lex" && type !== "vec" && type !== "hyde") return null;
149
- const text = line.slice(colonIdx + 1).trim();
150
- if (!text || !hasQueryTerm(text)) return null;
151
- return { type, text };
152
- }).filter((v) => v !== null);
153
- if (variations.length > 0) {
154
- expansionCache.set(query, variations);
155
- return variations;
156
- }
157
- } catch (err) {
158
- process.stderr.write(`[memory-crystal] Query expansion failed: ${err.message}
159
- `);
160
- }
161
- const fallback = [
162
- { type: "lex", text: query },
163
- { type: "vec", text: query },
164
- { type: "hyde", text: `Information about ${query}` }
165
- ];
166
- return fallback;
167
- }
168
- var RERANK_PROMPT = `You are a search result re-ranker. Given a query and a list of text passages, rate each passage's relevance to the query.
169
-
170
- Output one line per passage in this exact format:
171
- <index>: <score>
172
-
173
- Where index is the passage number (0-based) and score is a float from 0.0 to 1.0.
174
- - 1.0 = perfectly relevant, directly answers the query
175
- - 0.7 = highly relevant, closely related
176
- - 0.4 = somewhat relevant, tangentially related
177
- - 0.1 = barely relevant
178
- - 0.0 = not relevant at all
179
-
180
- Rate ALL passages. Output nothing else.`;
181
- async function rerankResults(query, passages) {
182
- const config = await detectProvider();
183
- if (config.provider === "none") {
184
- return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
185
- }
186
- try {
187
- const passageList = passages.map((p, i) => `[${i}] ${p.slice(0, 500)}`).join("\n\n");
188
- const result = await chatComplete(config, [
189
- { role: "system", content: RERANK_PROMPT },
190
- { role: "user", content: `Query: ${query}
191
-
192
- Passages:
193
- ${passageList}` }
194
- ], 200);
195
- const results = [];
196
- for (const line of result.trim().split("\n")) {
197
- const match = line.match(/^(\d+):\s*([\d.]+)/);
198
- if (match) {
199
- results.push({ index: parseInt(match[1]), score: parseFloat(match[2]) });
200
- }
201
- }
202
- const scored = new Set(results.map((r) => r.index));
203
- for (let i = 0; i < passages.length; i++) {
204
- if (!scored.has(i)) results.push({ index: i, score: 0 });
205
- }
206
- return results.sort((a, b) => b.score - a.score);
207
- } catch (err) {
208
- process.stderr.write(`[memory-crystal] Reranking failed: ${err.message}
209
- `);
210
- return passages.map((_, i) => ({ index: i, score: 1 - i * 0.01 }));
211
- }
212
- }
213
-
214
- // src/search-pipeline.ts
215
- var STRONG_SIGNAL_MIN_SCORE = 0.85;
216
- var STRONG_SIGNAL_MIN_GAP = 0.15;
217
- var RERANK_CANDIDATE_LIMIT = 40;
218
- async function deepSearch(crystal, query, options = {}) {
219
- const limit = options.limit || 5;
220
- const filter = options.filter;
221
- const provider = await detectProvider();
222
- if (provider.provider === "none") {
223
- return crystal.search(query, limit, filter);
224
- }
225
- const db = crystal.sqliteDb;
226
- if (!db) return crystal.search(query, limit, filter);
227
- const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
228
- const internalFilter = { ...filter, sinceDate };
229
- const initialFts = crystal.searchFTS(query, 20, internalFilter);
230
- const topScore = initialFts[0]?.score ?? 0;
231
- const secondScore = initialFts[1]?.score ?? 0;
232
- const hasStrongSignal = initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
233
- const expanded = hasStrongSignal ? [] : await expandQuery(query);
234
- const allResultLists = [];
235
- if (initialFts.length > 0) allResultLists.push(initialFts);
236
- const [queryEmbedding] = await crystal.embed([query]);
237
- const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
238
- if (originalVec.length > 0) allResultLists.push(originalVec);
239
- for (const variation of expanded) {
240
- if (variation.type === "lex") {
241
- const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
242
- if (ftsResults.length > 0) allResultLists.push(ftsResults);
243
- } else {
244
- const [embedding] = await crystal.embed([variation.text]);
245
- const vecResults = crystal.searchVec(embedding, 20, internalFilter);
246
- if (vecResults.length > 0) allResultLists.push(vecResults);
247
- }
248
- }
249
- const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
250
- const fused = crystal.reciprocalRankFusion(allResultLists, weights);
251
- const candidates = fused.slice(0, RERANK_CANDIDATE_LIMIT);
252
- if (candidates.length === 0) return [];
253
- const passages = candidates.map((c) => c.text.slice(0, 500));
254
- const reranked = await rerankResults(query, passages);
255
- const now = Date.now();
256
- const blended = reranked.map((r) => {
257
- const candidate = candidates[r.index];
258
- if (!candidate) return null;
259
- const rrfRank = r.index + 1;
260
- let rrfWeight;
261
- if (rrfRank <= 3) rrfWeight = 0.75;
262
- else if (rrfRank <= 10) rrfWeight = 0.6;
263
- else rrfWeight = 0.4;
264
- const rrfScore = 1 / rrfRank;
265
- const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
266
- const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
267
- const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
268
- const finalScore = Math.min(blendedScore * recency * 8, 1);
269
- const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
270
- return {
271
- ...candidate,
272
- score: finalScore,
273
- freshness
274
- };
275
- }).filter((r) => r !== null);
276
- return blended.sort((a, b) => b.score - a.score).slice(0, limit);
277
- }
278
- export {
279
- deepSearch
280
- };
@@ -1,73 +0,0 @@
1
- import {
2
- detectProvider,
3
- expandQuery,
4
- rerankResults
5
- } from "./chunk-7IUE7ODU.js";
6
-
7
- // src/search-pipeline.ts
8
- var STRONG_SIGNAL_MIN_SCORE = 0.85;
9
- var STRONG_SIGNAL_MIN_GAP = 0.15;
10
- var RERANK_CANDIDATE_LIMIT = 40;
11
- async function deepSearch(crystal, query, options = {}) {
12
- const limit = options.limit || 5;
13
- const filter = options.filter;
14
- const provider = await detectProvider();
15
- if (provider.provider === "none") {
16
- return crystal.search(query, limit, filter);
17
- }
18
- const db = crystal.sqliteDb;
19
- if (!db) return crystal.search(query, limit, filter);
20
- const sinceDate = filter?.since ? crystal.parseSince(filter.since) : void 0;
21
- const internalFilter = { ...filter, sinceDate };
22
- const initialFts = crystal.searchFTS(query, 20, internalFilter);
23
- const topScore = initialFts[0]?.score ?? 0;
24
- const secondScore = initialFts[1]?.score ?? 0;
25
- const hasStrongSignal = initialFts.length > 0 && topScore >= STRONG_SIGNAL_MIN_SCORE && topScore - secondScore >= STRONG_SIGNAL_MIN_GAP;
26
- const expanded = hasStrongSignal ? [] : await expandQuery(query);
27
- const allResultLists = [];
28
- if (initialFts.length > 0) allResultLists.push(initialFts);
29
- const [queryEmbedding] = await crystal.embed([query]);
30
- const originalVec = crystal.searchVec(queryEmbedding, 30, internalFilter);
31
- if (originalVec.length > 0) allResultLists.push(originalVec);
32
- for (const variation of expanded) {
33
- if (variation.type === "lex") {
34
- const ftsResults = crystal.searchFTS(variation.text, 20, internalFilter);
35
- if (ftsResults.length > 0) allResultLists.push(ftsResults);
36
- } else {
37
- const [embedding] = await crystal.embed([variation.text]);
38
- const vecResults = crystal.searchVec(embedding, 20, internalFilter);
39
- if (vecResults.length > 0) allResultLists.push(vecResults);
40
- }
41
- }
42
- const weights = allResultLists.map((_, i) => i < 2 ? 2 : 1);
43
- const fused = crystal.reciprocalRankFusion(allResultLists, weights);
44
- const candidates = fused.slice(0, RERANK_CANDIDATE_LIMIT);
45
- if (candidates.length === 0) return [];
46
- const passages = candidates.map((c) => c.text.slice(0, 500));
47
- const reranked = await rerankResults(query, passages);
48
- const now = Date.now();
49
- const blended = reranked.map((r) => {
50
- const candidate = candidates[r.index];
51
- if (!candidate) return null;
52
- const rrfRank = r.index + 1;
53
- let rrfWeight;
54
- if (rrfRank <= 3) rrfWeight = 0.75;
55
- else if (rrfRank <= 10) rrfWeight = 0.6;
56
- else rrfWeight = 0.4;
57
- const rrfScore = 1 / rrfRank;
58
- const blendedScore = rrfWeight * rrfScore + (1 - rrfWeight) * r.score;
59
- const ageDays = candidate.created_at ? (now - new Date(candidate.created_at).getTime()) / 864e5 : 0;
60
- const recency = candidate.created_at ? crystal.recencyWeight(ageDays) : 1;
61
- const finalScore = Math.min(blendedScore * recency * 8, 1);
62
- const freshness = candidate.created_at ? crystal.freshnessLabel(ageDays) : void 0;
63
- return {
64
- ...candidate,
65
- score: finalScore,
66
- freshness
67
- };
68
- }).filter((r) => r !== null);
69
- return blended.sort((a, b) => b.score - a.score).slice(0, limit);
70
- }
71
- export {
72
- deepSearch
73
- };