grimoire-framework 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.grimoire/install-manifest.yaml +2 -2
- package/bin/commands/memory.js +21 -1
- package/bin/commands/rag.js +297 -0
- package/bin/commands/search.js +99 -37
- package/bin/commands/setup.js +173 -0
- package/bin/grimoire-cli.js +3 -0
- package/package.json +4 -1
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
# - SHA256 hashes for change detection
|
|
8
8
|
# - File types for categorization
|
|
9
9
|
#
|
|
10
|
-
version: 1.
|
|
11
|
-
generated_at: "2026-02-22T17:
|
|
10
|
+
version: 1.6.0
|
|
11
|
+
generated_at: "2026-02-22T17:57:17.499Z"
|
|
12
12
|
generator: scripts/generate-install-manifest.js
|
|
13
13
|
file_count: 1011
|
|
14
14
|
files:
|
package/bin/commands/memory.js
CHANGED
|
@@ -70,7 +70,7 @@ async function run(args) {
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
// ── Auto-update context (silent)
|
|
73
|
+
// ── Auto-update context + RAG index (silent) ─────────────────────────
|
|
74
74
|
function silentContextUpdate(grimoireDir) {
|
|
75
75
|
try {
|
|
76
76
|
const { doUpdate } = require('./context');
|
|
@@ -78,6 +78,24 @@ function silentContextUpdate(grimoireDir) {
|
|
|
78
78
|
} catch (_) { } // never block memory save
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
async function silentRagIndex(entry, date, grimoireDir) {
|
|
82
|
+
try {
|
|
83
|
+
const fs2 = require('fs');
|
|
84
|
+
const cfgF = require('path').join(grimoireDir, 'config.yaml');
|
|
85
|
+
const cfg = {};
|
|
86
|
+
if (fs2.existsSync(cfgF)) {
|
|
87
|
+
fs2.readFileSync(cfgF, 'utf8').split('\n').forEach(l => {
|
|
88
|
+
const m = l.match(/^(\w[\w_]*)\s*:\s*(.+)$/);
|
|
89
|
+
if (m) cfg[m[1].trim()] = m[2].trim();
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
// Only index if a real embedding provider is configured
|
|
93
|
+
if (!cfg.embedding_provider || cfg.embedding_provider === 'tfidf') return;
|
|
94
|
+
const { indexEntry } = require('./rag');
|
|
95
|
+
await indexEntry(entry, date, grimoireDir, cfg);
|
|
96
|
+
} catch (_) { }
|
|
97
|
+
}
|
|
98
|
+
|
|
81
99
|
/**
|
|
82
100
|
* Saves memory using JSONL append (O(1) complexity)
|
|
83
101
|
*/
|
|
@@ -167,6 +185,8 @@ async function saveMemory(args, memoryDir, grimoireDir) {
|
|
|
167
185
|
console.log(`✅ Memory saved${tagLabel}${storyLabel}`);
|
|
168
186
|
// Auto-update .grimoire/context/ silently
|
|
169
187
|
silentContextUpdate(grimoireDir);
|
|
188
|
+
// Auto-index for RAG (fire-and-forget)
|
|
189
|
+
silentRagIndex(entry, today, grimoireDir);
|
|
170
190
|
} catch (err) {
|
|
171
191
|
console.error(`❌ Failed to acquire lock for memory file: ${err.message}`);
|
|
172
192
|
} finally {
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* grimoire rag.js — Embedding Engine (Layered Strategy)
|
|
3
|
+
*
|
|
4
|
+
* Layer 1: @xenova/transformers (local, no server needed)
|
|
5
|
+
* Layer 2: Ollama (http://localhost:11434) — if running
|
|
6
|
+
* Layer 3: TF-IDF (always available, zero deps)
|
|
7
|
+
*
|
|
8
|
+
* Vector store: .grimoire/memory/vectors/YYYY-MM-DD.json (plain JSON arrays)
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
'use strict';
|
|
12
|
+
|
|
13
|
+
const path = require('path');
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const os = require('os');
|
|
16
|
+
|
|
17
|
+
// ── Config ────────────────────────────────────────────────────────────────────
|
|
18
|
+
const DEFAULT_XENOVA_MODEL = 'Xenova/all-MiniLM-L6-v2'; // 23MB, fast
|
|
19
|
+
const OLLAMA_URL = 'http://localhost:11434';
|
|
20
|
+
const OLLAMA_MODEL = 'nomic-embed-text';
|
|
21
|
+
const VECTORS_DIR_NAME = 'vectors';
|
|
22
|
+
|
|
23
|
+
// ── Cosine similarity ─────────────────────────────────────────────────────────
|
|
24
|
+
function cosineSimilarity(a, b) {
|
|
25
|
+
if (!a || !b || a.length !== b.length) return 0;
|
|
26
|
+
let dot = 0, normA = 0, normB = 0;
|
|
27
|
+
for (let i = 0; i < a.length; i++) {
|
|
28
|
+
dot += a[i] * b[i];
|
|
29
|
+
normA += a[i] * a[i];
|
|
30
|
+
normB += b[i] * b[i];
|
|
31
|
+
}
|
|
32
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
33
|
+
return denom === 0 ? 0 : dot / denom;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// ── TF-IDF fallback ───────────────────────────────────────────────────────────
|
|
37
|
+
function tokenize(text) {
|
|
38
|
+
return text.toLowerCase().replace(/[^a-z0-9\u00c0-\u024f\s]/g, '').split(/\s+/).filter(Boolean);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const STOP_WORDS = new Set(['de', 'do', 'da', 'em', 'um', 'uma', 'e', 'a', 'o', 'os', 'as', 'para', 'com', 'que', 'não', 'por', 'se', 'mais', 'mas', 'ou', 'no', 'na', 'ao', 'isso', 'este', 'esta', 'é', 'foi', 'são', 'tem', 'como', 'sua', 'seu', 'usar', 'usar', 'ter', 'ser', 'was', 'the', 'and', 'for', 'are', 'with', 'this', 'that', 'from', 'not', 'has', 'have', 'will', 'can', 'also', 'use', 'used', 'via']);
|
|
42
|
+
|
|
43
|
+
function buildTFIDF(allEntries, query) {
|
|
44
|
+
// Build corpus
|
|
45
|
+
const docs = allEntries.map(e => tokenize(e.content));
|
|
46
|
+
const queryTokens = tokenize(query);
|
|
47
|
+
const N = docs.length;
|
|
48
|
+
|
|
49
|
+
// Build vocabulary (union of query + docs)
|
|
50
|
+
const vocab = new Set([...queryTokens]);
|
|
51
|
+
docs.forEach(d => d.forEach(t => { if (!STOP_WORDS.has(t)) vocab.add(t); }));
|
|
52
|
+
const vocabArr = [...vocab];
|
|
53
|
+
|
|
54
|
+
// IDF
|
|
55
|
+
const idf = {};
|
|
56
|
+
vocabArr.forEach(term => {
|
|
57
|
+
const df = docs.filter(d => d.includes(term)).length;
|
|
58
|
+
idf[term] = df === 0 ? 0 : Math.log((N + 1) / (df + 1)) + 1;
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
// TF-IDF vector for a token list
|
|
62
|
+
function toVec(tokens) {
|
|
63
|
+
const tf = {};
|
|
64
|
+
tokens.forEach(t => { tf[t] = (tf[t] || 0) + 1; });
|
|
65
|
+
const maxTF = Math.max(...Object.values(tf), 1);
|
|
66
|
+
return vocabArr.map(term => ((tf[term] || 0) / maxTF) * (idf[term] || 0));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const queryVec = toVec(queryTokens);
|
|
70
|
+
return allEntries.map((entry, i) => ({
|
|
71
|
+
entry,
|
|
72
|
+
score: cosineSimilarity(queryVec, toVec(docs[i])),
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// ── Xenova embedding ──────────────────────────────────────────────────────────
|
|
77
|
+
let _xenovaExtractor = null;
|
|
78
|
+
|
|
79
|
+
async function getXenovaExtractor() {
|
|
80
|
+
if (_xenovaExtractor) return _xenovaExtractor;
|
|
81
|
+
try {
|
|
82
|
+
// Dynamic import for ESM compat
|
|
83
|
+
const { pipeline } = await import('@xenova/transformers');
|
|
84
|
+
_xenovaExtractor = await pipeline('feature-extraction', DEFAULT_XENOVA_MODEL, {
|
|
85
|
+
progress_callback: null, // silent
|
|
86
|
+
});
|
|
87
|
+
return _xenovaExtractor;
|
|
88
|
+
} catch (_) {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function embedXenova(text) {
|
|
94
|
+
const extractor = await getXenovaExtractor();
|
|
95
|
+
if (!extractor) return null;
|
|
96
|
+
const output = await extractor(text, { pooling: 'mean', normalize: true });
|
|
97
|
+
return Array.from(output.data);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// ── Ollama embedding ──────────────────────────────────────────────────────────
|
|
101
|
+
async function embedOllama(text) {
|
|
102
|
+
try {
|
|
103
|
+
const res = await fetch(`${OLLAMA_URL}/api/embeddings`, {
|
|
104
|
+
method: 'POST',
|
|
105
|
+
headers: { 'Content-Type': 'application/json' },
|
|
106
|
+
body: JSON.stringify({ model: OLLAMA_MODEL, prompt: text }),
|
|
107
|
+
signal: AbortSignal.timeout(5000),
|
|
108
|
+
});
|
|
109
|
+
const { embedding } = await res.json();
|
|
110
|
+
return Array.isArray(embedding) ? embedding : null;
|
|
111
|
+
} catch (_) {
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// ── Auto-detect provider ──────────────────────────────────────────────────────
|
|
117
|
+
async function detectProvider(config) {
|
|
118
|
+
const preferred = config?.embedding_provider || 'auto';
|
|
119
|
+
|
|
120
|
+
if (preferred === 'tfidf') return 'tfidf';
|
|
121
|
+
if (preferred === 'ollama') return 'ollama';
|
|
122
|
+
|
|
123
|
+
if (preferred === 'xenova' || preferred === 'auto') {
|
|
124
|
+
try {
|
|
125
|
+
await import('@xenova/transformers');
|
|
126
|
+
return 'xenova';
|
|
127
|
+
} catch (_) { }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (preferred === 'auto') {
|
|
131
|
+
// Try Ollama ping
|
|
132
|
+
try {
|
|
133
|
+
const r = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(1000) });
|
|
134
|
+
if (r.ok) return 'ollama';
|
|
135
|
+
} catch (_) { }
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return 'tfidf';
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// ── Public: embed a single text ───────────────────────────────────────────────
|
|
142
|
+
async function embed(text, config) {
|
|
143
|
+
const provider = await detectProvider(config);
|
|
144
|
+
|
|
145
|
+
if (provider === 'xenova') {
|
|
146
|
+
const v = await embedXenova(text);
|
|
147
|
+
if (v) return { vector: v, provider: 'xenova' };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
if (provider === 'ollama') {
|
|
151
|
+
const v = await embedOllama(text);
|
|
152
|
+
if (v) return { vector: v, provider: 'ollama' };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
return { vector: null, provider: 'tfidf' }; // TF-IDF is computed at search time
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// ── Vector store ──────────────────────────────────────────────────────────────
|
|
159
|
+
function getVectorsDir(grimoireDir) {
|
|
160
|
+
const d = path.join(grimoireDir, 'memory', VECTORS_DIR_NAME);
|
|
161
|
+
if (!fs.existsSync(d)) fs.mkdirSync(d, { recursive: true });
|
|
162
|
+
return d;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function loadVectorFile(vectorsDir, date) {
|
|
166
|
+
const f = path.join(vectorsDir, `${date}.json`);
|
|
167
|
+
if (!fs.existsSync(f)) return [];
|
|
168
|
+
try { return JSON.parse(fs.readFileSync(f, 'utf8')); } catch (_) { return []; }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function saveVectorFile(vectorsDir, date, records) {
|
|
172
|
+
fs.writeFileSync(path.join(vectorsDir, `${date}.json`), JSON.stringify(records), 'utf8');
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Index a single entry (called after memory save)
|
|
176
|
+
async function indexEntry(entry, date, grimoireDir, config) {
|
|
177
|
+
try {
|
|
178
|
+
const { vector, provider } = await embed(entry.content, config);
|
|
179
|
+
if (!vector) return; // TF-IDF: skip vector storage, computed at query time
|
|
180
|
+
|
|
181
|
+
const vectorsDir = getVectorsDir(grimoireDir);
|
|
182
|
+
const records = loadVectorFile(vectorsDir, date);
|
|
183
|
+
|
|
184
|
+
// Remove existing record with same timestamp to avoid duplicates
|
|
185
|
+
const filtered = records.filter(r => r.timestamp !== entry.timestamp);
|
|
186
|
+
filtered.push({
|
|
187
|
+
timestamp: entry.timestamp,
|
|
188
|
+
content: entry.content,
|
|
189
|
+
tag: entry.tag || null,
|
|
190
|
+
story: entry.story || null,
|
|
191
|
+
source: entry.source || 'local',
|
|
192
|
+
vector,
|
|
193
|
+
provider,
|
|
194
|
+
});
|
|
195
|
+
saveVectorFile(vectorsDir, date, filtered);
|
|
196
|
+
} catch (_) { } // never block memory save
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// ── Public: semantic search ───────────────────────────────────────────────────
|
|
200
|
+
async function search(query, grimoireDir, config, topK = 5) {
|
|
201
|
+
const provider = await detectProvider(config);
|
|
202
|
+
|
|
203
|
+
if (provider === 'xenova' || provider === 'ollama') {
|
|
204
|
+
return await vectorSearch(query, grimoireDir, config, topK, provider);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// TF-IDF fallback — load all entries from JSONL
|
|
208
|
+
return tfidfSearch(query, grimoireDir, topK);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
async function vectorSearch(query, grimoireDir, config, topK, provider) {
|
|
212
|
+
const { vector: queryVec } = await embed(query, config);
|
|
213
|
+
if (!queryVec) return tfidfSearch(query, grimoireDir, topK);
|
|
214
|
+
|
|
215
|
+
const vectorsDir = getVectorsDir(grimoireDir);
|
|
216
|
+
if (!fs.existsSync(vectorsDir)) return [];
|
|
217
|
+
|
|
218
|
+
const results = [];
|
|
219
|
+
for (const f of fs.readdirSync(vectorsDir).filter(f => f.endsWith('.json'))) {
|
|
220
|
+
const records = loadVectorFile(vectorsDir, f.replace('.json', ''));
|
|
221
|
+
for (const rec of records) {
|
|
222
|
+
if (!rec.vector) continue;
|
|
223
|
+
const score = cosineSimilarity(queryVec, rec.vector);
|
|
224
|
+
results.push({ ...rec, score, searchMode: 'semantic' });
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
return results.sort((a, b) => b.score - a.score).slice(0, topK);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function tfidfSearch(query, grimoireDir, topK) {
|
|
232
|
+
// Load all JSONL entries
|
|
233
|
+
const allEntries = [];
|
|
234
|
+
const sessionsDir = path.join(grimoireDir, 'memory', 'sessions');
|
|
235
|
+
if (!fs.existsSync(sessionsDir)) return [];
|
|
236
|
+
|
|
237
|
+
for (const f of fs.readdirSync(sessionsDir).filter(f => f.endsWith('.jsonl')).sort()) {
|
|
238
|
+
const raw = fs.readFileSync(path.join(sessionsDir, f), 'utf8');
|
|
239
|
+
for (const line of raw.split('\n').filter(l => l.trim())) {
|
|
240
|
+
try { allEntries.push({ date: f.slice(0, 10), ...JSON.parse(line) }); } catch (_) { }
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
return buildTFIDF(allEntries, query)
|
|
245
|
+
.filter(r => r.score > 0)
|
|
246
|
+
.sort((a, b) => b.score - a.score)
|
|
247
|
+
.slice(0, topK)
|
|
248
|
+
.map(r => ({ ...r.entry, score: r.score, searchMode: 'tfidf' }));
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ── Re-index all existing entries ─────────────────────────────────────────────
|
|
252
|
+
async function reindexAll(grimoireDir, config, onProgress) {
|
|
253
|
+
const sessionsDir = path.join(grimoireDir, 'memory', 'sessions');
|
|
254
|
+
if (!fs.existsSync(sessionsDir)) return 0;
|
|
255
|
+
|
|
256
|
+
let count = 0;
|
|
257
|
+
const files = fs.readdirSync(sessionsDir).filter(f => f.endsWith('.jsonl')).sort();
|
|
258
|
+
|
|
259
|
+
for (const f of files) {
|
|
260
|
+
const date = f.slice(0, 10);
|
|
261
|
+
const raw = fs.readFileSync(path.join(sessionsDir, f), 'utf8');
|
|
262
|
+
const lines = raw.split('\n').filter(l => l.trim());
|
|
263
|
+
|
|
264
|
+
for (const line of lines) {
|
|
265
|
+
try {
|
|
266
|
+
const entry = JSON.parse(line);
|
|
267
|
+
await indexEntry(entry, date, grimoireDir, config);
|
|
268
|
+
count++;
|
|
269
|
+
if (onProgress) onProgress(count);
|
|
270
|
+
} catch (_) { }
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return count;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ── Status check ──────────────────────────────────────────────────────────────
|
|
277
|
+
async function ragStatus(config) {
|
|
278
|
+
const provider = await detectProvider(config);
|
|
279
|
+
const xenovaAvail = await (async () => {
|
|
280
|
+
try { await import('@xenova/transformers'); return true; } catch (_) { return false; }
|
|
281
|
+
})();
|
|
282
|
+
const ollamaAvail = await (async () => {
|
|
283
|
+
try {
|
|
284
|
+
const r = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(1000) });
|
|
285
|
+
return r.ok;
|
|
286
|
+
} catch (_) { return false; }
|
|
287
|
+
})();
|
|
288
|
+
|
|
289
|
+
return {
|
|
290
|
+
activeProvider: provider,
|
|
291
|
+
xenova: { available: xenovaAvail, model: DEFAULT_XENOVA_MODEL },
|
|
292
|
+
ollama: { available: ollamaAvail, url: OLLAMA_URL, model: OLLAMA_MODEL },
|
|
293
|
+
tfidf: { available: true },
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
module.exports = { embed, indexEntry, search, reindexAll, ragStatus, cosineSimilarity };
|
package/bin/commands/search.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* grimoire search — Global Search
|
|
2
|
+
* grimoire search — Global Search (RAG-enhanced)
|
|
3
3
|
*
|
|
4
|
-
* grimoire search "termo" Busca em memory + stories + agents
|
|
5
|
-
* grimoire search "JWT" --memory Só memória
|
|
4
|
+
* grimoire search "termo" Busca semântica em memory + stories + agents
|
|
5
|
+
* grimoire search "JWT" --memory Só memória (semântica se RAG ativo)
|
|
6
6
|
* grimoire search "JWT" --stories Só stories
|
|
7
7
|
* grimoire search "JWT" --agents Só agentes
|
|
8
|
+
* grimoire search "JWT" --semantic Força busca semântica
|
|
9
|
+
* grimoire search "JWT" --exact Força busca por substring
|
|
8
10
|
*/
|
|
9
11
|
|
|
10
12
|
'use strict';
|
|
@@ -36,8 +38,38 @@ function hl(text, query) {
|
|
|
36
38
|
return text.replace(re, m => `\x1b[33m${m}\x1b[0m`);
|
|
37
39
|
}
|
|
38
40
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
+
function loadConfig(grimoireDir) {
|
|
42
|
+
const f = path.join(grimoireDir, 'config.yaml');
|
|
43
|
+
if (!fs.existsSync(f)) return {};
|
|
44
|
+
const cfg = {};
|
|
45
|
+
fs.readFileSync(f, 'utf8').split('\n').forEach(line => {
|
|
46
|
+
const m = line.match(/^(\w[\w_]*)\s*:\s*(.+)$/);
|
|
47
|
+
if (m) cfg[m[1].trim()] = m[2].trim();
|
|
48
|
+
});
|
|
49
|
+
return cfg;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ── RAG memory search ─────────────────────────────────────────────────────────
|
|
53
|
+
async function searchMemoryRAG(grimoireDir, query, topK = 15) {
|
|
54
|
+
try {
|
|
55
|
+
const { search } = require('./rag');
|
|
56
|
+
const config = loadConfig(grimoireDir);
|
|
57
|
+
const results = await search(query, grimoireDir, config, topK);
|
|
58
|
+
return results.map(r => ({
|
|
59
|
+
date: r.date || r.timestamp?.slice(0, 10) || '',
|
|
60
|
+
content: r.content,
|
|
61
|
+
tag: r.tag || null,
|
|
62
|
+
story: r.story || null,
|
|
63
|
+
score: r.score,
|
|
64
|
+
searchMode: r.searchMode || 'unknown',
|
|
65
|
+
}));
|
|
66
|
+
} catch (_) {
|
|
67
|
+
return searchMemoryExact(grimoireDir, query);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// ── Exact substring memory search (fallback) ─────────────────────────────────
|
|
72
|
+
function searchMemoryExact(grimoireDir, query) {
|
|
41
73
|
const sessionsDir = path.join(grimoireDir, 'memory', 'sessions');
|
|
42
74
|
if (!fs.existsSync(sessionsDir)) return [];
|
|
43
75
|
const lq = query.toLowerCase();
|
|
@@ -49,7 +81,7 @@ function searchMemory(grimoireDir, query) {
|
|
|
49
81
|
try {
|
|
50
82
|
const e = JSON.parse(line);
|
|
51
83
|
if ((e.content || '').toLowerCase().includes(lq)) {
|
|
52
|
-
results.push({ date: f.slice(0, 10), content: e.content, tag: e.tag || null, story: e.story || null });
|
|
84
|
+
results.push({ date: f.slice(0, 10), content: e.content, tag: e.tag || null, story: e.story || null, score: 1, searchMode: 'exact' });
|
|
53
85
|
}
|
|
54
86
|
} catch (_) { }
|
|
55
87
|
}
|
|
@@ -80,7 +112,6 @@ function searchAgents(agentsDir, query) {
|
|
|
80
112
|
const raw = fs.readFileSync(path.join(agentsDir, f), 'utf8').toLowerCase();
|
|
81
113
|
if (raw.includes(lq)) {
|
|
82
114
|
const id = f.replace('.md', '');
|
|
83
|
-
// Extract name from yaml block
|
|
84
115
|
let name = id;
|
|
85
116
|
const nameMatch = raw.match(/name:\s*([^\n]+)/);
|
|
86
117
|
if (nameMatch) name = nameMatch[1].trim();
|
|
@@ -92,18 +123,33 @@ function searchAgents(agentsDir, query) {
|
|
|
92
123
|
return results;
|
|
93
124
|
}
|
|
94
125
|
|
|
126
|
+
// ── Render score bar ──────────────────────────────────────────────────────────
|
|
127
|
+
function scoreBar(score) {
|
|
128
|
+
if (score === null || score === undefined) return '';
|
|
129
|
+
const pct = Math.round(score * 10);
|
|
130
|
+
const bar = '█'.repeat(pct) + '░'.repeat(10 - pct);
|
|
131
|
+
const pctStr = `${Math.round(score * 100)}%`.padStart(4);
|
|
132
|
+
return ` \x1b[2m${bar} ${pctStr}\x1b[0m`;
|
|
133
|
+
}
|
|
134
|
+
|
|
95
135
|
// ── run ───────────────────────────────────────────────────────────────────────
|
|
96
|
-
function run(args) {
|
|
136
|
+
async function run(args) {
|
|
97
137
|
const query = args.filter(a => !a.startsWith('-')).join(' ');
|
|
98
138
|
if (!query) {
|
|
99
139
|
console.log('Usage: grimoire search "termo"\n');
|
|
100
|
-
console.log('Busca em: memória · stories · agentes\n');
|
|
140
|
+
console.log('Busca semântica em: memória · stories · agentes\n');
|
|
141
|
+
console.log('Flags:');
|
|
142
|
+
console.log(' --memory Só memória');
|
|
143
|
+
console.log(' --stories Só stories');
|
|
144
|
+
console.log(' --agents Só agentes');
|
|
145
|
+
console.log(' --exact Força busca por substring (ignora RAG)\n');
|
|
101
146
|
return;
|
|
102
147
|
}
|
|
103
148
|
|
|
104
149
|
const onlyMemory = args.includes('--memory');
|
|
105
150
|
const onlyStories = args.includes('--stories');
|
|
106
151
|
const onlyAgents = args.includes('--agents');
|
|
152
|
+
const forceExact = args.includes('--exact');
|
|
107
153
|
const all = !onlyMemory && !onlyStories && !onlyAgents;
|
|
108
154
|
|
|
109
155
|
const grimoireDir = findGrimoireDir();
|
|
@@ -111,43 +157,59 @@ function run(args) {
|
|
|
111
157
|
|
|
112
158
|
let totalResults = 0;
|
|
113
159
|
|
|
114
|
-
|
|
160
|
+
// Detect search mode label
|
|
161
|
+
let searchModeLabel = '(substring)';
|
|
162
|
+
if (!forceExact && grimoireDir) {
|
|
163
|
+
const config = loadConfig(grimoireDir);
|
|
164
|
+
const provider = config.embedding_provider || 'auto';
|
|
165
|
+
if (provider !== 'tfidf') {
|
|
166
|
+
searchModeLabel = provider === 'xenova' ? '🤖 semântica (xenova)' :
|
|
167
|
+
provider === 'ollama' ? '🦙 semântica (ollama)' :
|
|
168
|
+
'📊 TF-IDF';
|
|
169
|
+
} else {
|
|
170
|
+
searchModeLabel = '📊 TF-IDF';
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
console.log(`\n🔍 Grimoire Search — "${query}" ${searchModeLabel}\n${'─'.repeat(52)}`);
|
|
115
175
|
|
|
116
176
|
// Memory
|
|
117
|
-
if (all || onlyMemory) {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
177
|
+
if ((all || onlyMemory) && grimoireDir) {
|
|
178
|
+
const memResults = forceExact
|
|
179
|
+
? searchMemoryExact(grimoireDir, query)
|
|
180
|
+
: await searchMemoryRAG(grimoireDir, query, 10);
|
|
181
|
+
|
|
182
|
+
if (memResults.length > 0) {
|
|
183
|
+
const modeIcon = memResults[0]?.searchMode === 'semantic' ? ' 🤖' :
|
|
184
|
+
memResults[0]?.searchMode === 'tfidf' ? ' 📊' : '';
|
|
185
|
+
console.log(`\n🧠 Memória${modeIcon}:`);
|
|
186
|
+
for (const r of memResults) {
|
|
187
|
+
const tag = r.tag ? ` \x1b[36m[#${r.tag}]\x1b[0m` : '';
|
|
188
|
+
const story = r.story ? ` \x1b[35m[${r.story}]\x1b[0m` : '';
|
|
189
|
+
const bar = r.searchMode !== 'exact' ? scoreBar(r.score) : '';
|
|
190
|
+
console.log(` ${r.date} ${hl(r.content, query)}${tag}${story}${bar}`);
|
|
128
191
|
}
|
|
192
|
+
totalResults += memResults.length;
|
|
129
193
|
}
|
|
130
194
|
}
|
|
131
195
|
|
|
132
|
-
// Stories
|
|
133
|
-
if (all || onlyStories) {
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
console.log(` 📝 ${hl(n, query)}`);
|
|
143
|
-
}
|
|
196
|
+
// Stories (always exact — small dataset)
|
|
197
|
+
if ((all || onlyStories) && grimoireDir) {
|
|
198
|
+
const storyResults = searchStories(grimoireDir, query);
|
|
199
|
+
if (storyResults.length > 0) {
|
|
200
|
+
console.log('\n📋 Stories:');
|
|
201
|
+
for (const s of storyResults) {
|
|
202
|
+
const icon = s.status === 'done' ? '✅' : '🔄';
|
|
203
|
+
console.log(` ${icon} [${s.id}] ${hl(s.title, query)}`);
|
|
204
|
+
for (const n of (s.notes || []).filter(n => n.toLowerCase().includes(query.toLowerCase()))) {
|
|
205
|
+
console.log(` 📝 ${hl(n, query)}`);
|
|
144
206
|
}
|
|
145
|
-
totalResults += storyResults.length;
|
|
146
207
|
}
|
|
208
|
+
totalResults += storyResults.length;
|
|
147
209
|
}
|
|
148
210
|
}
|
|
149
211
|
|
|
150
|
-
// Agents
|
|
212
|
+
// Agents (always exact)
|
|
151
213
|
if (all || onlyAgents) {
|
|
152
214
|
const agentResults = searchAgents(agentsDir, query);
|
|
153
215
|
if (agentResults.length > 0) {
|
|
@@ -161,9 +223,9 @@ function run(args) {
|
|
|
161
223
|
|
|
162
224
|
if (totalResults === 0) {
|
|
163
225
|
console.log('\n (nenhum resultado encontrado)');
|
|
164
|
-
console.log(` Dica: tente termos mais
|
|
226
|
+
console.log(` Dica: tente termos mais amplos ou "grimoire setup rag" para busca semântica\n`);
|
|
165
227
|
} else {
|
|
166
|
-
console.log(`\n${'─'.repeat(
|
|
228
|
+
console.log(`\n${'─'.repeat(52)}`);
|
|
167
229
|
console.log(` ${totalResults} resultado(s)\n`);
|
|
168
230
|
}
|
|
169
231
|
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* grimoire setup — Environment & RAG Setup
|
|
3
|
+
*
|
|
4
|
+
* grimoire setup rag Configura RAG (detecta provider + baixa modelo)
|
|
5
|
+
* grimoire setup rag --reindex Reindexar todas as entradas existentes
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
'use strict';
|
|
9
|
+
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const fs = require('fs');
|
|
12
|
+
const os = require('os');
|
|
13
|
+
const { execSync, spawn } = require('child_process');
|
|
14
|
+
|
|
15
|
+
function findGrimoireDir() {
|
|
16
|
+
const cwd = process.cwd();
|
|
17
|
+
const d1 = path.join(cwd, '.grimoire');
|
|
18
|
+
const d2 = path.join(cwd, 'grimoire', '.grimoire');
|
|
19
|
+
if (fs.existsSync(d1)) return d1;
|
|
20
|
+
if (fs.existsSync(d2)) return d2;
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function loadConfig(grimoireDir) {
|
|
25
|
+
const f = path.join(grimoireDir, 'config.yaml');
|
|
26
|
+
if (!fs.existsSync(f)) return {};
|
|
27
|
+
// Simple YAML key: value parser
|
|
28
|
+
const cfg = {};
|
|
29
|
+
fs.readFileSync(f, 'utf8').split('\n').forEach(line => {
|
|
30
|
+
const m = line.match(/^(\w[\w_]*)\s*:\s*(.+)$/);
|
|
31
|
+
if (m) cfg[m[1].trim()] = m[2].trim();
|
|
32
|
+
});
|
|
33
|
+
return cfg;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function saveConfig(grimoireDir, updates) {
|
|
37
|
+
const f = path.join(grimoireDir, 'config.yaml');
|
|
38
|
+
let content = fs.existsSync(f) ? fs.readFileSync(f, 'utf8') : '';
|
|
39
|
+
|
|
40
|
+
Object.entries(updates).forEach(([key, value]) => {
|
|
41
|
+
const re = new RegExp(`^${key}\\s*:.*$`, 'm');
|
|
42
|
+
if (re.test(content)) {
|
|
43
|
+
content = content.replace(re, `${key}: ${value}`);
|
|
44
|
+
} else {
|
|
45
|
+
content += `\n${key}: ${value}`;
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
fs.writeFileSync(f, content.trim() + '\n', 'utf8');
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ── Check if @xenova/transformers is installed ─────────────────────────────────
|
|
53
|
+
function isXenovaInstalled() {
|
|
54
|
+
try { require.resolve('@xenova/transformers'); return true; } catch (_) { return false; }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// ── Check if Ollama is running ─────────────────────────────────────────────────
|
|
58
|
+
async function isOllamaRunning() {
|
|
59
|
+
try {
|
|
60
|
+
const r = await fetch('http://localhost:11434/api/tags', { signal: AbortSignal.timeout(1500) });
|
|
61
|
+
return r.ok;
|
|
62
|
+
} catch (_) { return false; }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// ── Setup RAG ─────────────────────────────────────────────────────────────────
|
|
66
|
+
async function setupRag(args) {
|
|
67
|
+
const reindex = args.includes('--reindex');
|
|
68
|
+
const sep = '─'.repeat(52);
|
|
69
|
+
|
|
70
|
+
console.log(`\n🔮 Grimoire RAG Setup\n${sep}\n`);
|
|
71
|
+
|
|
72
|
+
const { ragStatus } = require('./rag');
|
|
73
|
+
const grimoireDir = findGrimoireDir();
|
|
74
|
+
|
|
75
|
+
if (!grimoireDir) {
|
|
76
|
+
console.error('❌ .grimoire/ not found. Run: npx grimoire-framework install\n');
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const config = loadConfig(grimoireDir);
|
|
81
|
+
|
|
82
|
+
console.log('🔍 Detectando providers disponíveis...\n');
|
|
83
|
+
|
|
84
|
+
// ── Layer 1: @xenova/transformers ──────────────────────────────────────────
|
|
85
|
+
console.log(' [1/3] @xenova/transformers (local, offline)');
|
|
86
|
+
|
|
87
|
+
if (isXenovaInstalled()) {
|
|
88
|
+
console.log(' ✅ Já instalado');
|
|
89
|
+
} else {
|
|
90
|
+
console.log(' 📦 Instalando @xenova/transformers...');
|
|
91
|
+
try {
|
|
92
|
+
execSync('npm install @xenova/transformers --save-optional --no-fund --no-audit', {
|
|
93
|
+
stdio: 'inherit',
|
|
94
|
+
cwd: path.dirname(require.main?.filename || process.cwd()),
|
|
95
|
+
});
|
|
96
|
+
console.log(' ✅ Instalado com sucesso');
|
|
97
|
+
} catch (e) {
|
|
98
|
+
console.log(' ⚠️ Falha ao instalar — tentando layer 2 (Ollama)');
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Download model on first use (xenova downloads automatically)
|
|
103
|
+
if (isXenovaInstalled()) {
|
|
104
|
+
console.log(' 🤖 Carregando modelo Xenova/all-MiniLM-L6-v2...');
|
|
105
|
+
console.log(' (primeiro carregamento pode levar 30-60s ~23MB)');
|
|
106
|
+
try {
|
|
107
|
+
const { pipeline } = await import('@xenova/transformers');
|
|
108
|
+
await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', { progress_callback: null });
|
|
109
|
+
console.log(' ✅ Modelo pronto!\n');
|
|
110
|
+
saveConfig(grimoireDir, { embedding_provider: 'xenova', embedding_model: 'Xenova/all-MiniLM-L6-v2' });
|
|
111
|
+
} catch (e) {
|
|
112
|
+
console.log(` ❌ Erro ao carregar modelo: ${e.message}\n`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// ── Layer 2: Ollama ────────────────────────────────────────────────────────
|
|
117
|
+
console.log(' [2/3] Ollama (local server, melhor qualidade)');
|
|
118
|
+
const ollamaRunning = await isOllamaRunning();
|
|
119
|
+
if (ollamaRunning) {
|
|
120
|
+
console.log(' ✅ Ollama detectado em localhost:11434');
|
|
121
|
+
if (!isXenovaInstalled()) {
|
|
122
|
+
saveConfig(grimoireDir, { embedding_provider: 'ollama', embedding_model: 'nomic-embed-text' });
|
|
123
|
+
}
|
|
124
|
+
} else {
|
|
125
|
+
console.log(' ℹ️ Ollama não detectado (opcional)');
|
|
126
|
+
console.log(' Instale em: https://ollama.com');
|
|
127
|
+
console.log(' Depois: ollama pull nomic-embed-text\n');
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// ── Layer 3: TF-IDF ────────────────────────────────────────────────────────
|
|
131
|
+
console.log(' [3/3] TF-IDF (fallback — sempre disponível)');
|
|
132
|
+
console.log(' ✅ Ativo\n');
|
|
133
|
+
|
|
134
|
+
// ── Final status ───────────────────────────────────────────────────────────
|
|
135
|
+
const status = await ragStatus(loadConfig(grimoireDir));
|
|
136
|
+
const activeIcon = { xenova: '🤖', ollama: '🦙', tfidf: '📊' };
|
|
137
|
+
|
|
138
|
+
console.log(`${sep}`);
|
|
139
|
+
console.log(` Provider ativo: ${activeIcon[status.activeProvider] || '📊'} ${status.activeProvider}`);
|
|
140
|
+
console.log(` Xenova: ${status.xenova.available ? '✅' : '❌'} ${status.xenova.model}`);
|
|
141
|
+
console.log(` Ollama: ${status.ollama.available ? '✅' : '❌'} ${status.ollama.url}`);
|
|
142
|
+
console.log(` TF-IDF: ✅ sempre disponível`);
|
|
143
|
+
console.log(`${sep}\n`);
|
|
144
|
+
|
|
145
|
+
// ── Reindex ────────────────────────────────────────────────────────────────
|
|
146
|
+
if (reindex || status.activeProvider !== 'tfidf') {
|
|
147
|
+
const { reindexAll } = require('./rag');
|
|
148
|
+
process.stdout.write(' 🔄 Indexando entradas de memória existentes...');
|
|
149
|
+
let n = 0;
|
|
150
|
+
const count = await reindexAll(grimoireDir, loadConfig(grimoireDir), (i) => {
|
|
151
|
+
n = i;
|
|
152
|
+
if (i % 10 === 0) process.stdout.write('.');
|
|
153
|
+
});
|
|
154
|
+
console.log(`\n ✅ ${count} entradas indexadas\n`);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
console.log('🎯 RAG configurado! Agora o grimoire memory search é semântico.');
|
|
158
|
+
console.log(' grimoire memory search "autenticação" ← busca por significado\n');
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// ── run ────────────────────────────────────────────────────────────────────────
|
|
162
|
+
async function run(args) {
|
|
163
|
+
const sub = args[0];
|
|
164
|
+
switch (sub) {
|
|
165
|
+
case 'rag': await setupRag(args.slice(1)); break;
|
|
166
|
+
default:
|
|
167
|
+
console.log('\nUsage:\n');
|
|
168
|
+
console.log(' grimoire setup rag Configura RAG (xenova + modelos)');
|
|
169
|
+
console.log(' grimoire setup rag --reindex Reindexar todas as entradas\n');
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
module.exports = { run };
|
package/bin/grimoire-cli.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "grimoire-framework",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.0",
|
|
4
4
|
"description": "Grimoire: AI-Orchestrated System for Full Stack Development - Core Framework",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -82,6 +82,9 @@
|
|
|
82
82
|
"prepublishOnly": "node scripts/pre-publish-check.js && npm run generate:manifest && npm run validate:manifest",
|
|
83
83
|
"prepare": "husky"
|
|
84
84
|
},
|
|
85
|
+
"optionalDependencies": {
|
|
86
|
+
"@xenova/transformers": "^2.17.2"
|
|
87
|
+
},
|
|
85
88
|
"dependencies": {
|
|
86
89
|
"@clack/prompts": "^0.11.0",
|
|
87
90
|
"@kayvan/markdown-tree-parser": "^1.5.0",
|