metame-cli 1.5.26 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +4 -1
- package/package.json +1 -1
- package/scripts/agent-layer.js +36 -0
- package/scripts/core/chunker.js +100 -0
- package/scripts/core/embedding.js +225 -0
- package/scripts/core/hybrid-search.js +296 -0
- package/scripts/core/wiki-db.js +545 -0
- package/scripts/core/wiki-prompt.js +88 -0
- package/scripts/core/wiki-slug.js +66 -0
- package/scripts/core/wiki-staleness.js +18 -0
- package/scripts/daemon-agent-commands.js +10 -4
- package/scripts/daemon-bridges.js +16 -0
- package/scripts/daemon-claude-engine.js +62 -8
- package/scripts/daemon-command-router.js +40 -1
- package/scripts/daemon-default.yaml +33 -3
- package/scripts/daemon-embedding.js +162 -0
- package/scripts/daemon-engine-runtime.js +1 -1
- package/scripts/daemon-health-scan.js +185 -0
- package/scripts/daemon-ops-commands.js +9 -18
- package/scripts/daemon-runtime-lifecycle.js +1 -1
- package/scripts/daemon-session-commands.js +4 -0
- package/scripts/daemon-task-scheduler.js +5 -3
- package/scripts/daemon-warm-pool.js +15 -0
- package/scripts/daemon-wiki.js +420 -0
- package/scripts/daemon.js +10 -5
- package/scripts/distill.js +1 -1
- package/scripts/docs/file-transfer.md +0 -1
- package/scripts/docs/maintenance-manual.md +2 -55
- package/scripts/docs/pointer-map.md +0 -34
- package/scripts/feishu-adapter.js +25 -0
- package/scripts/hooks/intent-file-transfer.js +1 -2
- package/scripts/memory-backfill-chunks.js +92 -0
- package/scripts/memory-search.js +49 -6
- package/scripts/memory-wiki-schema.js +255 -0
- package/scripts/memory.js +103 -3
- package/scripts/signal-capture.js +1 -1
- package/scripts/skill-evolution.js +2 -11
- package/scripts/wiki-cluster.js +121 -0
- package/scripts/wiki-extract.js +171 -0
- package/scripts/wiki-facts.js +351 -0
- package/scripts/wiki-import.js +256 -0
- package/scripts/wiki-reflect-build.js +441 -0
- package/scripts/wiki-reflect-export.js +448 -0
- package/scripts/wiki-reflect-query.js +109 -0
- package/scripts/wiki-reflect.js +338 -0
- package/scripts/wiki-synthesis.js +224 -0
package/index.js
CHANGED
|
@@ -290,7 +290,10 @@ function ensureLaunchdPlist({ daemonScript, daemonLog }) {
|
|
|
290
290
|
<key>RunAtLoad</key>
|
|
291
291
|
<true/>
|
|
292
292
|
<key>KeepAlive</key>
|
|
293
|
-
<
|
|
293
|
+
<dict>
|
|
294
|
+
<key>SuccessfulExit</key>
|
|
295
|
+
<false/>
|
|
296
|
+
</dict>
|
|
294
297
|
<key>ThrottleInterval</key>
|
|
295
298
|
<integer>30</integer>
|
|
296
299
|
<key>StandardOutPath</key>
|
package/package.json
CHANGED
package/scripts/agent-layer.js
CHANGED
|
@@ -285,6 +285,41 @@ function buildMemorySnapshotContent(sessions = [], facts = []) {
|
|
|
285
285
|
return lines.join('\n');
|
|
286
286
|
}
|
|
287
287
|
|
|
288
|
+
function selectSnapshotContext(memoryApi, {
|
|
289
|
+
projectHints = [],
|
|
290
|
+
sessionLimit = 5,
|
|
291
|
+
factLimit = 10,
|
|
292
|
+
} = {}) {
|
|
293
|
+
if (!memoryApi || typeof memoryApi.recentSessions !== 'function') {
|
|
294
|
+
return { sessions: [], facts: [], matchedProject: null, usedFallback: false };
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const fetchFacts = typeof memoryApi.recentFacts === 'function'
|
|
298
|
+
? (project) => memoryApi.recentFacts({ limit: factLimit, project: project || null })
|
|
299
|
+
: () => [];
|
|
300
|
+
|
|
301
|
+
const candidates = Array.from(new Set(
|
|
302
|
+
(Array.isArray(projectHints) ? projectHints : [projectHints])
|
|
303
|
+
.map(v => String(v || '').trim())
|
|
304
|
+
.filter(Boolean)
|
|
305
|
+
));
|
|
306
|
+
|
|
307
|
+
for (const candidate of candidates) {
|
|
308
|
+
const sessions = memoryApi.recentSessions({ limit: sessionLimit, project: candidate });
|
|
309
|
+
const facts = fetchFacts(candidate);
|
|
310
|
+
if (sessions.length > 0 || facts.length > 0) {
|
|
311
|
+
return { sessions, facts, matchedProject: candidate, usedFallback: false };
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
return {
|
|
316
|
+
sessions: memoryApi.recentSessions({ limit: sessionLimit }),
|
|
317
|
+
facts: fetchFacts(null),
|
|
318
|
+
matchedProject: null,
|
|
319
|
+
usedFallback: true,
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
|
|
288
323
|
/**
|
|
289
324
|
* Overwrite memory-snapshot.md for the given agent.
|
|
290
325
|
* Returns true on success, false if the agent directory doesn't exist yet.
|
|
@@ -318,5 +353,6 @@ module.exports = {
|
|
|
318
353
|
buildAgentContextForEngine,
|
|
319
354
|
buildAgentContextForProject,
|
|
320
355
|
buildMemorySnapshotContent,
|
|
356
|
+
selectSnapshotContext,
|
|
321
357
|
refreshMemorySnapshot,
|
|
322
358
|
};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* core/chunker.js — Recursive delimiter-aware text chunker
|
|
5
|
+
*
|
|
6
|
+
* Pure function, no I/O, no dependencies.
|
|
7
|
+
*
|
|
8
|
+
* Exports:
|
|
9
|
+
* chunkText(text, opts?) → string[]
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
// Delimiter hierarchy: paragraphs → lines → sentences → words
|
|
13
|
+
const DELIMITERS = [
|
|
14
|
+
/\n\n+/, // paragraphs
|
|
15
|
+
/\n/, // lines
|
|
16
|
+
/(?<=[.!?。!?])\s+/, // sentences
|
|
17
|
+
/\s+/, // words
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Split text into chunks of approximately targetWords size.
|
|
22
|
+
*
|
|
23
|
+
* Algorithm:
|
|
24
|
+
* 1. Split by highest-level delimiter that produces >1 segment.
|
|
25
|
+
* 2. Greedily merge consecutive segments until adding the next would exceed targetWords.
|
|
26
|
+
* 3. If a single segment exceeds targetWords, recurse with the next finer delimiter.
|
|
27
|
+
* 4. Fragments smaller than targetWords * 0.3 are merged into the previous chunk.
|
|
28
|
+
*
|
|
29
|
+
* @param {string} text
|
|
30
|
+
* @param {{ targetWords?: number }} [opts]
|
|
31
|
+
* @returns {string[]}
|
|
32
|
+
*/
|
|
33
|
+
function chunkText(text, { targetWords = 300 } = {}) {
|
|
34
|
+
if (!text || typeof text !== 'string') return [];
|
|
35
|
+
const trimmed = text.trim();
|
|
36
|
+
if (!trimmed) return [];
|
|
37
|
+
|
|
38
|
+
const wordCount = trimmed.split(/\s+/).length;
|
|
39
|
+
if (wordCount <= targetWords * 1.5) return [trimmed];
|
|
40
|
+
|
|
41
|
+
return _splitRecursive(trimmed, targetWords, 0);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @param {string} text
|
|
46
|
+
* @param {number} target
|
|
47
|
+
* @param {number} delimIdx — current position in DELIMITERS hierarchy
|
|
48
|
+
* @returns {string[]}
|
|
49
|
+
*/
|
|
50
|
+
function _splitRecursive(text, target, delimIdx) {
|
|
51
|
+
if (delimIdx >= DELIMITERS.length) return [text];
|
|
52
|
+
|
|
53
|
+
const segments = text.split(DELIMITERS[delimIdx]).filter(s => s.trim());
|
|
54
|
+
if (segments.length <= 1) {
|
|
55
|
+
return _splitRecursive(text, target, delimIdx + 1);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Greedy merge
|
|
59
|
+
const chunks = [];
|
|
60
|
+
let current = '';
|
|
61
|
+
|
|
62
|
+
for (const seg of segments) {
|
|
63
|
+
const segWords = seg.split(/\s+/).length;
|
|
64
|
+
const curWords = current ? current.split(/\s+/).length : 0;
|
|
65
|
+
|
|
66
|
+
if (current && curWords + segWords > target) {
|
|
67
|
+
chunks.push(current.trim());
|
|
68
|
+
current = seg;
|
|
69
|
+
} else {
|
|
70
|
+
current = current ? current + '\n\n' + seg : seg;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (current.trim()) chunks.push(current.trim());
|
|
74
|
+
|
|
75
|
+
// Recurse on oversized chunks
|
|
76
|
+
const result = [];
|
|
77
|
+
for (const chunk of chunks) {
|
|
78
|
+
const cw = chunk.split(/\s+/).length;
|
|
79
|
+
if (cw > target * 1.5) {
|
|
80
|
+
result.push(..._splitRecursive(chunk, target, delimIdx + 1));
|
|
81
|
+
} else {
|
|
82
|
+
result.push(chunk);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Merge tiny trailing fragments into previous chunk
|
|
87
|
+
const merged = [];
|
|
88
|
+
for (const chunk of result) {
|
|
89
|
+
const cw = chunk.split(/\s+/).length;
|
|
90
|
+
if (merged.length > 0 && cw < target * 0.3) {
|
|
91
|
+
merged[merged.length - 1] += '\n\n' + chunk;
|
|
92
|
+
} else {
|
|
93
|
+
merged.push(chunk);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return merged;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
module.exports = { chunkText };
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* core/embedding.js — Embedding adapter with two backends:
|
|
5
|
+
* 1. OpenAI text-embedding-3-small (512-dim) — requires OPENAI_API_KEY
|
|
6
|
+
* 2. Ollama bge-m3 (1024-dim, local) — fallback when OpenAI key absent
|
|
7
|
+
*
|
|
8
|
+
* Exports:
|
|
9
|
+
* getEmbedding(text) → Float32Array | null
|
|
10
|
+
* batchEmbed(texts[]) → (Float32Array | null)[]
|
|
11
|
+
* embeddingToBuffer(f32) → Buffer (for SQLite BLOB write)
|
|
12
|
+
* bufferToEmbedding(blob) → Float32Array (for SQLite BLOB read)
|
|
13
|
+
* isEmbeddingAvailable() → boolean
|
|
14
|
+
*
|
|
15
|
+
* Backend selection (automatic):
|
|
16
|
+
* OPENAI_API_KEY set → OpenAI (512-dim)
|
|
17
|
+
* ollama installed → bge-m3 via localhost:11434 (1024-dim)
|
|
18
|
+
* neither → all functions return null gracefully
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
const { existsSync } = require('node:fs');
|
|
22
|
+
|
|
23
|
+
// ── OpenAI backend ────────────────────────────────────────────────────────────
|
|
24
|
+
const OPENAI_MODEL = 'text-embedding-3-small';
|
|
25
|
+
const OPENAI_DIMENSIONS = 512;
|
|
26
|
+
const OPENAI_API_URL = 'https://api.openai.com/v1/embeddings';
|
|
27
|
+
|
|
28
|
+
// ── Ollama backend ────────────────────────────────────────────────────────────
|
|
29
|
+
const OLLAMA_MODEL = 'bge-m3';
|
|
30
|
+
const OLLAMA_DIMENSIONS = 1024;
|
|
31
|
+
const OLLAMA_API_URL = process.env.OLLAMA_HOST
|
|
32
|
+
? `${process.env.OLLAMA_HOST}/api/embed`
|
|
33
|
+
: 'http://localhost:11434/api/embed';
|
|
34
|
+
const OLLAMA_BIN_PATHS = [
|
|
35
|
+
'/usr/local/bin/ollama',
|
|
36
|
+
'/opt/homebrew/bin/ollama',
|
|
37
|
+
'/usr/bin/ollama',
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
// ── Shared constants ──────────────────────────────────────────────────────────
|
|
41
|
+
const MAX_INPUT_CHARS = 8000;
|
|
42
|
+
const MAX_RETRIES = 3;
|
|
43
|
+
const BASE_DELAY_MS = 2000;
|
|
44
|
+
const BATCH_SIZE = 100;
|
|
45
|
+
|
|
46
|
+
// Keep legacy export name for callers that read it directly
|
|
47
|
+
const MODEL = OPENAI_MODEL;
|
|
48
|
+
const DIMENSIONS = OPENAI_DIMENSIONS;
|
|
49
|
+
|
|
50
|
+
// ── Backend detection ─────────────────────────────────────────────────────────
|
|
51
|
+
|
|
52
|
+
function getApiKey() {
|
|
53
|
+
return process.env.OPENAI_API_KEY || '';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function isOllamaInstalled() {
|
|
57
|
+
return OLLAMA_BIN_PATHS.some(p => existsSync(p));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function getBackend() {
|
|
61
|
+
if (getApiKey()) return 'openai';
|
|
62
|
+
if (isOllamaInstalled()) return 'ollama';
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function isEmbeddingAvailable() {
|
|
67
|
+
return getBackend() !== null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* L2-normalize a Float32Array in-place and return it.
|
|
72
|
+
* @param {Float32Array} vec
|
|
73
|
+
* @returns {Float32Array}
|
|
74
|
+
*/
|
|
75
|
+
function l2Normalize(vec) {
|
|
76
|
+
let norm = 0;
|
|
77
|
+
for (let i = 0; i < vec.length; i++) norm += vec[i] * vec[i];
|
|
78
|
+
norm = Math.sqrt(norm);
|
|
79
|
+
if (norm > 0) {
|
|
80
|
+
for (let i = 0; i < vec.length; i++) vec[i] /= norm;
|
|
81
|
+
}
|
|
82
|
+
return vec;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ── OpenAI API call ───────────────────────────────────────────────────────────
|
|
86
|
+
|
|
87
|
+
async function callOpenAI(inputs) {
|
|
88
|
+
const apiKey = getApiKey();
|
|
89
|
+
let lastError;
|
|
90
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
91
|
+
try {
|
|
92
|
+
const resp = await fetch(OPENAI_API_URL, {
|
|
93
|
+
method: 'POST',
|
|
94
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
|
95
|
+
body: JSON.stringify({ model: OPENAI_MODEL, input: inputs, dimensions: OPENAI_DIMENSIONS }),
|
|
96
|
+
});
|
|
97
|
+
if (resp.status === 429) {
|
|
98
|
+
const retryAfter = parseInt(resp.headers.get('retry-after') || '0', 10);
|
|
99
|
+
await new Promise(r => setTimeout(r, Math.max(retryAfter * 1000, BASE_DELAY_MS * Math.pow(2, attempt))));
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (!resp.ok) {
|
|
103
|
+
const body = await resp.text().catch(() => '');
|
|
104
|
+
throw new Error(`OpenAI API ${resp.status}: ${body.slice(0, 200)}`);
|
|
105
|
+
}
|
|
106
|
+
const data = await resp.json();
|
|
107
|
+
return data.data.map(item => l2Normalize(new Float32Array(item.embedding)));
|
|
108
|
+
} catch (err) {
|
|
109
|
+
lastError = err;
|
|
110
|
+
if (attempt < MAX_RETRIES - 1) await new Promise(r => setTimeout(r, BASE_DELAY_MS * Math.pow(2, attempt)));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
throw lastError;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// ── Ollama API call (bge-m3, one text at a time — ollama /api/embed) ──────────
|
|
117
|
+
|
|
118
|
+
async function callOllama(inputs) {
|
|
119
|
+
const results = [];
|
|
120
|
+
for (const text of inputs) {
|
|
121
|
+
let lastError;
|
|
122
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
123
|
+
try {
|
|
124
|
+
const resp = await fetch(OLLAMA_API_URL, {
|
|
125
|
+
method: 'POST',
|
|
126
|
+
headers: { 'Content-Type': 'application/json' },
|
|
127
|
+
body: JSON.stringify({ model: OLLAMA_MODEL, input: text }),
|
|
128
|
+
});
|
|
129
|
+
if (!resp.ok) {
|
|
130
|
+
const body = await resp.text().catch(() => '');
|
|
131
|
+
throw new Error(`Ollama API ${resp.status}: ${body.slice(0, 200)}`);
|
|
132
|
+
}
|
|
133
|
+
const data = await resp.json();
|
|
134
|
+
const vec = data.embeddings?.[0];
|
|
135
|
+
if (!vec || !Array.isArray(vec)) throw new Error('Ollama: unexpected response shape');
|
|
136
|
+
results.push(l2Normalize(new Float32Array(vec)));
|
|
137
|
+
lastError = null;
|
|
138
|
+
break;
|
|
139
|
+
} catch (err) {
|
|
140
|
+
lastError = err;
|
|
141
|
+
if (attempt < MAX_RETRIES - 1) await new Promise(r => setTimeout(r, BASE_DELAY_MS * Math.pow(2, attempt)));
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (lastError) results.push(null);
|
|
145
|
+
}
|
|
146
|
+
return results;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ── Unified router ────────────────────────────────────────────────────────────
|
|
150
|
+
|
|
151
|
+
async function callApi(inputs) {
|
|
152
|
+
const backend = getBackend();
|
|
153
|
+
if (backend === 'openai') return callOpenAI(inputs);
|
|
154
|
+
if (backend === 'ollama') return callOllama(inputs);
|
|
155
|
+
return inputs.map(() => null);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Get embedding for a single text.
|
|
160
|
+
* @param {string} text
|
|
161
|
+
* @returns {Promise<Float32Array|null>}
|
|
162
|
+
*/
|
|
163
|
+
async function getEmbedding(text) {
|
|
164
|
+
if (!isEmbeddingAvailable()) return null;
|
|
165
|
+
if (!text || typeof text !== 'string') return null;
|
|
166
|
+
const truncated = text.slice(0, MAX_INPUT_CHARS);
|
|
167
|
+
const results = await callApi([truncated]);
|
|
168
|
+
return results[0] || null;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Get embeddings for multiple texts in batches.
|
|
173
|
+
* @param {string[]} texts
|
|
174
|
+
* @returns {Promise<(Float32Array|null)[]>}
|
|
175
|
+
*/
|
|
176
|
+
async function batchEmbed(texts) {
|
|
177
|
+
if (!isEmbeddingAvailable()) return texts.map(() => null);
|
|
178
|
+
if (!texts || texts.length === 0) return [];
|
|
179
|
+
|
|
180
|
+
const results = [];
|
|
181
|
+
for (let i = 0; i < texts.length; i += BATCH_SIZE) {
|
|
182
|
+
const batch = texts.slice(i, i + BATCH_SIZE).map(t =>
|
|
183
|
+
(typeof t === 'string' ? t : '').slice(0, MAX_INPUT_CHARS),
|
|
184
|
+
);
|
|
185
|
+
const embeddings = await callApi(batch);
|
|
186
|
+
results.push(...embeddings);
|
|
187
|
+
}
|
|
188
|
+
return results;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Convert Float32Array to Buffer for SQLite BLOB storage.
|
|
193
|
+
* @param {Float32Array} f32
|
|
194
|
+
* @returns {Buffer}
|
|
195
|
+
*/
|
|
196
|
+
function embeddingToBuffer(f32) {
|
|
197
|
+
if (!f32) return null;
|
|
198
|
+
return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Read Buffer from SQLite BLOB back to Float32Array.
|
|
203
|
+
* Dimension-agnostic: infers dim from blob length (supports both 512-dim OpenAI
|
|
204
|
+
* and 1024-dim bge-m3 embeddings stored in the same DB).
|
|
205
|
+
* @param {Buffer|Uint8Array} blob
|
|
206
|
+
* @returns {Float32Array|null}
|
|
207
|
+
*/
|
|
208
|
+
function bufferToEmbedding(blob) {
|
|
209
|
+
if (!blob || blob.length === 0 || blob.length % 4 !== 0) return null;
|
|
210
|
+
// Copy into aligned ArrayBuffer to avoid RangeError on unaligned byteOffset
|
|
211
|
+
const aligned = new ArrayBuffer(blob.length);
|
|
212
|
+
new Uint8Array(aligned).set(new Uint8Array(blob.buffer, blob.byteOffset, blob.length));
|
|
213
|
+
return new Float32Array(aligned);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
module.exports = {
|
|
217
|
+
getEmbedding,
|
|
218
|
+
batchEmbed,
|
|
219
|
+
embeddingToBuffer,
|
|
220
|
+
bufferToEmbedding,
|
|
221
|
+
isEmbeddingAvailable,
|
|
222
|
+
l2Normalize,
|
|
223
|
+
MODEL,
|
|
224
|
+
DIMENSIONS,
|
|
225
|
+
};
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* core/hybrid-search.js — Hybrid wiki search (FTS5 + Vector + RRF fusion)
|
|
5
|
+
*
|
|
6
|
+
* Exports:
|
|
7
|
+
* hybridSearchWiki(db, query, opts?) → { wikiPages: object[], facts: object[] }
|
|
8
|
+
*
|
|
9
|
+
* When vector embeddings are available:
|
|
10
|
+
* 1. FTS5 search → page candidates with rank
|
|
11
|
+
* 2. Vector cosine search on content_chunks → chunk candidates
|
|
12
|
+
* 3. Chunk → page aggregation (max score per slug, keep best chunk as excerpt)
|
|
13
|
+
* 4. RRF fusion of FTS page ranks + vector page ranks
|
|
14
|
+
* 5. Normalize scores to 0-1
|
|
15
|
+
*
|
|
16
|
+
* Degradation: no embeddings in DB → pure FTS5 (same as searchWikiAndFacts)
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const { sanitizeFts5 } = require('./wiki-slug');
|
|
20
|
+
const { bufferToEmbedding, getEmbedding, isEmbeddingAvailable } = require('./embedding');
|
|
21
|
+
|
|
22
|
+
const RRF_K = 60;
|
|
23
|
+
const STALE_THRESHOLD = 0.3;
|
|
24
|
+
const MAX_FTS_RESULTS = 10;
|
|
25
|
+
const MAX_VECTOR_RESULTS = 20;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Dot product of two Float32Arrays (assumes L2-normalized → equals cosine similarity).
|
|
29
|
+
* @param {Float32Array} a
|
|
30
|
+
* @param {Float32Array} b
|
|
31
|
+
* @returns {number}
|
|
32
|
+
*/
|
|
33
|
+
function dotProduct(a, b) {
|
|
34
|
+
let sum = 0;
|
|
35
|
+
for (let i = 0; i < a.length; i++) sum += a[i] * b[i];
|
|
36
|
+
return sum;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Top-K selection via bounded insertion (avoids full sort).
|
|
41
|
+
* @param {{ score: number }[]} items
|
|
42
|
+
* @param {number} k
|
|
43
|
+
* @returns {{ score: number }[]}
|
|
44
|
+
*/
|
|
45
|
+
function topK(items, k) {
|
|
46
|
+
if (items.length <= k) return items.slice().sort((a, b) => b.score - a.score);
|
|
47
|
+
const heap = items.slice(0, k).sort((a, b) => a.score - b.score);
|
|
48
|
+
for (let i = k; i < items.length; i++) {
|
|
49
|
+
if (items[i].score > heap[0].score) {
|
|
50
|
+
heap[0] = items[i];
|
|
51
|
+
heap.sort((a, b) => a.score - b.score);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return heap.sort((a, b) => b.score - a.score);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* FTS5 search for wiki pages.
|
|
59
|
+
* @param {object} db
|
|
60
|
+
* @param {string} safeQuery — already sanitized
|
|
61
|
+
* @returns {{ slug: string, title: string, staleness: number, excerpt: string, ftsRank: number }[]}
|
|
62
|
+
*/
|
|
63
|
+
function ftsSearch(db, safeQuery) {
|
|
64
|
+
try {
|
|
65
|
+
return db.prepare(`
|
|
66
|
+
SELECT wp.slug, wp.title, wp.staleness, wp.last_built_at,
|
|
67
|
+
snippet(wiki_pages_fts, 2, '<b>', '</b>', '...', 20) as excerpt,
|
|
68
|
+
rank as ftsRank
|
|
69
|
+
FROM wiki_pages_fts
|
|
70
|
+
JOIN wiki_pages wp ON wiki_pages_fts.rowid = wp.rowid
|
|
71
|
+
WHERE wiki_pages_fts MATCH ?
|
|
72
|
+
ORDER BY rank
|
|
73
|
+
LIMIT ?
|
|
74
|
+
`).all(safeQuery, MAX_FTS_RESULTS);
|
|
75
|
+
} catch {
|
|
76
|
+
return [];
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Vector cosine search on content_chunks.
|
|
82
|
+
* Brute-force scan with top-K heap. Only scans rows with embedding IS NOT NULL.
|
|
83
|
+
*
|
|
84
|
+
* @param {object} db
|
|
85
|
+
* @param {Float32Array} queryEmbedding
|
|
86
|
+
* @returns {{ page_slug: string, chunk_text: string, score: number }[]}
|
|
87
|
+
*/
|
|
88
|
+
function vectorSearch(db, queryEmbedding) {
|
|
89
|
+
let rows;
|
|
90
|
+
try {
|
|
91
|
+
rows = db.prepare(`
|
|
92
|
+
SELECT page_slug, chunk_text, embedding
|
|
93
|
+
FROM content_chunks
|
|
94
|
+
WHERE embedding IS NOT NULL
|
|
95
|
+
`).all();
|
|
96
|
+
} catch {
|
|
97
|
+
return [];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const scored = [];
|
|
101
|
+
for (const row of rows) {
|
|
102
|
+
const emb = bufferToEmbedding(row.embedding);
|
|
103
|
+
if (!emb) continue;
|
|
104
|
+
const score = dotProduct(queryEmbedding, emb);
|
|
105
|
+
scored.push({ page_slug: row.page_slug, chunk_text: row.chunk_text, score });
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return topK(scored, MAX_VECTOR_RESULTS);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Check if any content_chunks have stored embeddings.
|
|
113
|
+
* Avoids wasting OpenAI API calls when no embeddings exist yet.
|
|
114
|
+
*/
|
|
115
|
+
function hasStoredEmbeddings(db) {
|
|
116
|
+
try {
|
|
117
|
+
return !!db.prepare('SELECT 1 FROM content_chunks WHERE embedding IS NOT NULL LIMIT 1').get();
|
|
118
|
+
} catch { return false; }
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Aggregate chunk-level vector results to page-level.
|
|
123
|
+
* Per slug: keep max score and best chunk text as excerpt.
|
|
124
|
+
* @param {{ page_slug: string, chunk_text: string, score: number }[]} chunks
|
|
125
|
+
* @returns {Map<string, { score: number, excerpt: string }>}
|
|
126
|
+
*/
|
|
127
|
+
function aggregateChunksToPages(chunks) {
|
|
128
|
+
const pages = new Map();
|
|
129
|
+
for (const c of chunks) {
|
|
130
|
+
const existing = pages.get(c.page_slug);
|
|
131
|
+
if (!existing || c.score > existing.score) {
|
|
132
|
+
pages.set(c.page_slug, { score: c.score, excerpt: c.chunk_text.slice(0, 200) });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return pages;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* RRF fusion of two ranked lists.
|
|
140
|
+
* @param {Map<string, { ftsRank?: number, vectorRank?: number, title?: string, excerpt?: string, staleness?: number }>} merged
|
|
141
|
+
* @returns {{ slug: string, score: number, title: string, excerpt: string, staleness: number, stale: boolean, source: string }[]}
|
|
142
|
+
*/
|
|
143
|
+
function rrfFuse(merged) {
|
|
144
|
+
const results = [];
|
|
145
|
+
for (const [slug, info] of merged) {
|
|
146
|
+
let score = 0;
|
|
147
|
+
let source = '';
|
|
148
|
+
if (typeof info.ftsRank === 'number') {
|
|
149
|
+
score += 1 / (RRF_K + info.ftsRank);
|
|
150
|
+
source = 'fts';
|
|
151
|
+
}
|
|
152
|
+
if (typeof info.vectorRank === 'number') {
|
|
153
|
+
score += 1 / (RRF_K + info.vectorRank);
|
|
154
|
+
source = source ? 'hybrid' : 'vector';
|
|
155
|
+
}
|
|
156
|
+
const staleness = info.staleness || 0;
|
|
157
|
+
results.push({
|
|
158
|
+
slug,
|
|
159
|
+
score,
|
|
160
|
+
title: info.title || slug,
|
|
161
|
+
excerpt: info.excerpt || '',
|
|
162
|
+
staleness,
|
|
163
|
+
stale: staleness >= STALE_THRESHOLD,
|
|
164
|
+
source,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
results.sort((a, b) => b.score - a.score);
|
|
168
|
+
return results;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Normalize scores to 0-1 range.
|
|
173
|
+
* @param {{ score: number }[]} results
|
|
174
|
+
*/
|
|
175
|
+
function normalizeScores(results) {
|
|
176
|
+
if (results.length === 0) return;
|
|
177
|
+
const max = results[0].score;
|
|
178
|
+
const min = results[results.length - 1].score;
|
|
179
|
+
for (const r of results) {
|
|
180
|
+
r.score = max === min ? 1.0 : (r.score - min) / (max - min);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Main entry: hybrid wiki search.
|
|
186
|
+
*
|
|
187
|
+
* @param {object} db
|
|
188
|
+
* @param {string} query
|
|
189
|
+
* @param {{ ftsOnly?: boolean, trackSearch?: boolean }} [opts]
|
|
190
|
+
* @returns {{ wikiPages: object[], facts: object[] }}
|
|
191
|
+
*/
|
|
192
|
+
async function hybridSearchWiki(db, query, { ftsOnly = false, trackSearch = true } = {}) {
|
|
193
|
+
const safeQuery = sanitizeFts5(query);
|
|
194
|
+
if (!safeQuery) return { wikiPages: [], facts: [] };
|
|
195
|
+
|
|
196
|
+
// 1. FTS5 search (always)
|
|
197
|
+
const ftsResults = ftsSearch(db, safeQuery);
|
|
198
|
+
|
|
199
|
+
// 2. Vector search (if available and not forced FTS-only)
|
|
200
|
+
let vectorPages = new Map();
|
|
201
|
+
const hasEmbeddings = !ftsOnly && isEmbeddingAvailable();
|
|
202
|
+
|
|
203
|
+
if (hasEmbeddings && hasStoredEmbeddings(db)) {
|
|
204
|
+
try {
|
|
205
|
+
const queryEmb = await getEmbedding(query);
|
|
206
|
+
if (queryEmb) {
|
|
207
|
+
const chunks = vectorSearch(db, queryEmb);
|
|
208
|
+
vectorPages = aggregateChunksToPages(chunks);
|
|
209
|
+
}
|
|
210
|
+
} catch {
|
|
211
|
+
// Vector search failed — degrade gracefully
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// 3. Merge FTS + vector results into unified map
|
|
216
|
+
const merged = new Map();
|
|
217
|
+
|
|
218
|
+
for (let i = 0; i < ftsResults.length; i++) {
|
|
219
|
+
const r = ftsResults[i];
|
|
220
|
+
merged.set(r.slug, {
|
|
221
|
+
ftsRank: i + 1,
|
|
222
|
+
title: r.title,
|
|
223
|
+
excerpt: r.excerpt,
|
|
224
|
+
staleness: r.staleness,
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
for (const [slug, vInfo] of vectorPages) {
|
|
229
|
+
const existing = merged.get(slug);
|
|
230
|
+
const rank = [...vectorPages.keys()].indexOf(slug) + 1;
|
|
231
|
+
if (existing) {
|
|
232
|
+
existing.vectorRank = rank;
|
|
233
|
+
// Prefer vector excerpt if FTS didn't have a good one
|
|
234
|
+
if (vInfo.excerpt && (!existing.excerpt || existing.excerpt.length < 20)) {
|
|
235
|
+
existing.excerpt = vInfo.excerpt;
|
|
236
|
+
}
|
|
237
|
+
} else {
|
|
238
|
+
// Vector-only result — need to fetch page metadata
|
|
239
|
+
let title = slug;
|
|
240
|
+
let staleness = 0;
|
|
241
|
+
try {
|
|
242
|
+
const page = db.prepare('SELECT title, staleness FROM wiki_pages WHERE slug = ?').get(slug);
|
|
243
|
+
if (page) { title = page.title; staleness = page.staleness || 0; }
|
|
244
|
+
} catch { }
|
|
245
|
+
merged.set(slug, {
|
|
246
|
+
vectorRank: rank,
|
|
247
|
+
title,
|
|
248
|
+
excerpt: vInfo.excerpt,
|
|
249
|
+
staleness,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// 4. RRF fusion + normalize
|
|
255
|
+
const wikiPages = rrfFuse(merged);
|
|
256
|
+
normalizeScores(wikiPages);
|
|
257
|
+
|
|
258
|
+
// 5. Facts search (same as searchWikiAndFacts — FTS5 only)
|
|
259
|
+
let facts = [];
|
|
260
|
+
try {
|
|
261
|
+
facts = db.prepare(`
|
|
262
|
+
SELECT mi.id, mi.title, mi.content, mi.kind, mi.confidence,
|
|
263
|
+
snippet(memory_items_fts, 1, '<b>', '</b>', '...', 20) as excerpt,
|
|
264
|
+
rank as score
|
|
265
|
+
FROM memory_items_fts
|
|
266
|
+
JOIN memory_items mi ON memory_items_fts.rowid = mi.rowid
|
|
267
|
+
WHERE memory_items_fts MATCH ?
|
|
268
|
+
AND mi.state = 'active'
|
|
269
|
+
ORDER BY rank
|
|
270
|
+
LIMIT 10
|
|
271
|
+
`).all(safeQuery);
|
|
272
|
+
} catch {
|
|
273
|
+
facts = [];
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// 6. Track search counts on matched facts
|
|
277
|
+
if (trackSearch && facts.length > 0) {
|
|
278
|
+
const ids = facts.map(r => r.id).filter(Boolean);
|
|
279
|
+
if (ids.length > 0) {
|
|
280
|
+
const ph = ids.map(() => '?').join(', ');
|
|
281
|
+
try {
|
|
282
|
+
db.prepare(`
|
|
283
|
+
UPDATE memory_items SET search_count = search_count + 1, last_searched_at = datetime('now')
|
|
284
|
+
WHERE id IN (${ph})
|
|
285
|
+
`).run(...ids);
|
|
286
|
+
} catch { }
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
return { wikiPages: wikiPages.slice(0, 5), facts };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
module.exports = {
|
|
294
|
+
hybridSearchWiki,
|
|
295
|
+
_internal: { dotProduct, topK, ftsSearch, vectorSearch, aggregateChunksToPages, rrfFuse, normalizeScores, hasStoredEmbeddings },
|
|
296
|
+
};
|