@timmeck/brain-core 2.36.24 → 2.36.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/active-learning/__tests__/active-learning.test.d.ts +1 -0
- package/dist/active-learning/__tests__/active-learning.test.js +132 -0
- package/dist/active-learning/__tests__/active-learning.test.js.map +1 -0
- package/dist/active-learning/active-learner.d.ts +79 -0
- package/dist/active-learning/active-learner.js +224 -0
- package/dist/active-learning/active-learner.js.map +1 -0
- package/dist/active-learning/index.d.ts +2 -0
- package/dist/active-learning/index.js +2 -0
- package/dist/active-learning/index.js.map +1 -0
- package/dist/code-health/__tests__/code-health.test.d.ts +1 -0
- package/dist/code-health/__tests__/code-health.test.js +123 -0
- package/dist/code-health/__tests__/code-health.test.js.map +1 -0
- package/dist/code-health/health-monitor.d.ts +55 -0
- package/dist/code-health/health-monitor.js +180 -0
- package/dist/code-health/health-monitor.js.map +1 -0
- package/dist/code-health/index.d.ts +2 -0
- package/dist/code-health/index.js +2 -0
- package/dist/code-health/index.js.map +1 -0
- package/dist/consensus/__tests__/consensus.test.d.ts +1 -0
- package/dist/consensus/__tests__/consensus.test.js +159 -0
- package/dist/consensus/__tests__/consensus.test.js.map +1 -0
- package/dist/consensus/consensus-engine.d.ts +81 -0
- package/dist/consensus/consensus-engine.js +237 -0
- package/dist/consensus/consensus-engine.js.map +1 -0
- package/dist/consensus/index.d.ts +2 -0
- package/dist/consensus/index.js +2 -0
- package/dist/consensus/index.js.map +1 -0
- package/dist/feedback/__tests__/feedback-engine.test.d.ts +1 -0
- package/dist/feedback/__tests__/feedback-engine.test.js +156 -0
- package/dist/feedback/__tests__/feedback-engine.test.js.map +1 -0
- package/dist/feedback/feedback-engine.d.ts +61 -0
- package/dist/feedback/feedback-engine.js +203 -0
- package/dist/feedback/feedback-engine.js.map +1 -0
- package/dist/feedback/index.d.ts +2 -0
- package/dist/feedback/index.js +2 -0
- package/dist/feedback/index.js.map +1 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.js +27 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-graph/__tests__/knowledge-graph.test.d.ts +1 -0
- package/dist/knowledge-graph/__tests__/knowledge-graph.test.js +215 -0
- package/dist/knowledge-graph/__tests__/knowledge-graph.test.js.map +1 -0
- package/dist/knowledge-graph/fact-extractor.d.ts +23 -0
- package/dist/knowledge-graph/fact-extractor.js +70 -0
- package/dist/knowledge-graph/fact-extractor.js.map +1 -0
- package/dist/knowledge-graph/graph-engine.d.ts +78 -0
- package/dist/knowledge-graph/graph-engine.js +276 -0
- package/dist/knowledge-graph/graph-engine.js.map +1 -0
- package/dist/knowledge-graph/index.d.ts +4 -0
- package/dist/knowledge-graph/index.js +3 -0
- package/dist/knowledge-graph/index.js.map +1 -0
- package/dist/proactive/__tests__/proactive-engine.test.d.ts +1 -0
- package/dist/proactive/__tests__/proactive-engine.test.js +183 -0
- package/dist/proactive/__tests__/proactive-engine.test.js.map +1 -0
- package/dist/proactive/index.d.ts +2 -0
- package/dist/proactive/index.js +2 -0
- package/dist/proactive/index.js.map +1 -0
- package/dist/proactive/proactive-engine.d.ts +86 -0
- package/dist/proactive/proactive-engine.js +252 -0
- package/dist/proactive/proactive-engine.js.map +1 -0
- package/dist/rag/__tests__/rag-engine.test.d.ts +1 -0
- package/dist/rag/__tests__/rag-engine.test.js +235 -0
- package/dist/rag/__tests__/rag-engine.test.js.map +1 -0
- package/dist/rag/index.d.ts +4 -0
- package/dist/rag/index.js +3 -0
- package/dist/rag/index.js.map +1 -0
- package/dist/rag/rag-engine.d.ts +98 -0
- package/dist/rag/rag-engine.js +310 -0
- package/dist/rag/rag-engine.js.map +1 -0
- package/dist/rag/rag-indexer.d.ts +52 -0
- package/dist/rag/rag-indexer.js +144 -0
- package/dist/rag/rag-indexer.js.map +1 -0
- package/dist/research/__tests__/semantic-compressor.test.d.ts +1 -0
- package/dist/research/__tests__/semantic-compressor.test.js +153 -0
- package/dist/research/__tests__/semantic-compressor.test.js.map +1 -0
- package/dist/research/semantic-compressor.d.ts +55 -0
- package/dist/research/semantic-compressor.js +227 -0
- package/dist/research/semantic-compressor.js.map +1 -0
- package/dist/teaching/__tests__/teaching.test.d.ts +1 -0
- package/dist/teaching/__tests__/teaching.test.js +151 -0
- package/dist/teaching/__tests__/teaching.test.js.map +1 -0
- package/dist/teaching/curriculum.d.ts +32 -0
- package/dist/teaching/curriculum.js +89 -0
- package/dist/teaching/curriculum.js.map +1 -0
- package/dist/teaching/index.d.ts +4 -0
- package/dist/teaching/index.js +3 -0
- package/dist/teaching/index.js.map +1 -0
- package/dist/teaching/teaching-protocol.d.ts +74 -0
- package/dist/teaching/teaching-protocol.js +164 -0
- package/dist/teaching/teaching-protocol.js.map +1 -0
- package/dist/tool-learning/__tests__/tool-learning.test.d.ts +1 -0
- package/dist/tool-learning/__tests__/tool-learning.test.js +187 -0
- package/dist/tool-learning/__tests__/tool-learning.test.js.map +1 -0
- package/dist/tool-learning/index.d.ts +4 -0
- package/dist/tool-learning/index.js +3 -0
- package/dist/tool-learning/index.js.map +1 -0
- package/dist/tool-learning/tool-patterns.d.ts +47 -0
- package/dist/tool-learning/tool-patterns.js +125 -0
- package/dist/tool-learning/tool-patterns.js.map +1 -0
- package/dist/tool-learning/tool-tracker.d.ts +58 -0
- package/dist/tool-learning/tool-tracker.js +135 -0
- package/dist/tool-learning/tool-tracker.js.map +1 -0
- package/dist/user-model/__tests__/user-model.test.d.ts +1 -0
- package/dist/user-model/__tests__/user-model.test.js +142 -0
- package/dist/user-model/__tests__/user-model.test.js.map +1 -0
- package/dist/user-model/adaptive-context.d.ts +18 -0
- package/dist/user-model/adaptive-context.js +46 -0
- package/dist/user-model/adaptive-context.js.map +1 -0
- package/dist/user-model/index.d.ts +4 -0
- package/dist/user-model/index.js +3 -0
- package/dist/user-model/index.js.map +1 -0
- package/dist/user-model/user-model.d.ts +53 -0
- package/dist/user-model/user-model.js +204 -0
- package/dist/user-model/user-model.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { ThoughtStream } from '../consciousness/thought-stream.js';
|
|
3
|
+
import type { BaseEmbeddingEngine } from '../embeddings/engine.js';
|
|
4
|
+
import type { LLMService } from '../llm/llm-service.js';
|
|
5
|
+
export interface RAGEngineConfig {
|
|
6
|
+
brainName: string;
|
|
7
|
+
/** Default similarity threshold for search (0-1). Default: 0.3 */
|
|
8
|
+
defaultThreshold?: number;
|
|
9
|
+
/** Default max results. Default: 10 */
|
|
10
|
+
defaultLimit?: number;
|
|
11
|
+
/** Max batch size for indexing. Default: 50 */
|
|
12
|
+
batchSize?: number;
|
|
13
|
+
}
|
|
14
|
+
export interface RAGSearchOptions {
|
|
15
|
+
/** Filter by collections */
|
|
16
|
+
collections?: string[];
|
|
17
|
+
/** Max results */
|
|
18
|
+
limit?: number;
|
|
19
|
+
/** Minimum similarity threshold */
|
|
20
|
+
threshold?: number;
|
|
21
|
+
/** Enable LLM-based reranking */
|
|
22
|
+
rerank?: boolean;
|
|
23
|
+
}
|
|
24
|
+
export interface RAGResult {
|
|
25
|
+
collection: string;
|
|
26
|
+
sourceId: number;
|
|
27
|
+
text: string;
|
|
28
|
+
similarity: number;
|
|
29
|
+
metadata?: Record<string, unknown>;
|
|
30
|
+
}
|
|
31
|
+
export interface RAGIndexStats {
|
|
32
|
+
collection: string;
|
|
33
|
+
count: number;
|
|
34
|
+
}
|
|
35
|
+
export interface RAGStatus {
|
|
36
|
+
totalVectors: number;
|
|
37
|
+
collections: RAGIndexStats[];
|
|
38
|
+
lastIndexedAt: string | null;
|
|
39
|
+
}
|
|
40
|
+
export declare function runRAGMigration(db: Database.Database): void;
|
|
41
|
+
export declare class RAGEngine {
|
|
42
|
+
private readonly db;
|
|
43
|
+
private readonly config;
|
|
44
|
+
private readonly log;
|
|
45
|
+
private ts;
|
|
46
|
+
private embedding;
|
|
47
|
+
private llm;
|
|
48
|
+
private readonly stmtUpsert;
|
|
49
|
+
private readonly stmtSearch;
|
|
50
|
+
private readonly stmtSearchAll;
|
|
51
|
+
private readonly stmtGetByHash;
|
|
52
|
+
private readonly stmtDelete;
|
|
53
|
+
private readonly stmtStats;
|
|
54
|
+
private readonly stmtTotal;
|
|
55
|
+
private readonly stmtLastIndexed;
|
|
56
|
+
private readonly stmtGetCollections;
|
|
57
|
+
constructor(db: Database.Database, config: RAGEngineConfig);
|
|
58
|
+
setThoughtStream(stream: ThoughtStream): void;
|
|
59
|
+
setEmbeddingEngine(engine: BaseEmbeddingEngine): void;
|
|
60
|
+
setLLMService(llm: LLMService): void;
|
|
61
|
+
/**
|
|
62
|
+
* Index a single item into the RAG store.
|
|
63
|
+
*/
|
|
64
|
+
index(collection: string, sourceId: number, text: string, metadata?: Record<string, unknown>): Promise<boolean>;
|
|
65
|
+
/**
|
|
66
|
+
* Batch index multiple items. Returns count of newly indexed items.
|
|
67
|
+
*/
|
|
68
|
+
indexBatch(items: Array<{
|
|
69
|
+
collection: string;
|
|
70
|
+
sourceId: number;
|
|
71
|
+
text: string;
|
|
72
|
+
metadata?: Record<string, unknown>;
|
|
73
|
+
}>): Promise<number>;
|
|
74
|
+
/**
|
|
75
|
+
* Search the RAG store for similar content.
|
|
76
|
+
*/
|
|
77
|
+
search(query: string, options?: RAGSearchOptions): Promise<RAGResult[]>;
|
|
78
|
+
/**
|
|
79
|
+
* LLM-based reranking of search results.
|
|
80
|
+
*/
|
|
81
|
+
rerank(query: string, results: RAGResult[]): Promise<RAGResult[]>;
|
|
82
|
+
/**
|
|
83
|
+
* Format search results as context for an LLM prompt.
|
|
84
|
+
*/
|
|
85
|
+
augment(query: string, context: RAGResult[]): string;
|
|
86
|
+
/**
|
|
87
|
+
* Remove an item from the RAG store.
|
|
88
|
+
*/
|
|
89
|
+
remove(collection: string, sourceId: number): void;
|
|
90
|
+
/**
|
|
91
|
+
* Get RAG engine status.
|
|
92
|
+
*/
|
|
93
|
+
getStatus(): RAGStatus;
|
|
94
|
+
/**
|
|
95
|
+
* Get list of all collections.
|
|
96
|
+
*/
|
|
97
|
+
getCollections(): string[];
|
|
98
|
+
}
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { getLogger } from '../utils/logger.js';
|
|
2
|
+
// ── Migration ────────────────────────────────────────────────
|
|
3
|
+
export function runRAGMigration(db) {
|
|
4
|
+
db.exec(`
|
|
5
|
+
CREATE TABLE IF NOT EXISTS rag_vectors (
|
|
6
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
7
|
+
collection TEXT NOT NULL,
|
|
8
|
+
source_id INTEGER NOT NULL,
|
|
9
|
+
text_hash TEXT NOT NULL,
|
|
10
|
+
text_preview TEXT,
|
|
11
|
+
embedding BLOB NOT NULL,
|
|
12
|
+
metadata TEXT,
|
|
13
|
+
created_at TEXT DEFAULT (datetime('now')),
|
|
14
|
+
UNIQUE(collection, source_id)
|
|
15
|
+
);
|
|
16
|
+
CREATE INDEX IF NOT EXISTS idx_rag_collection ON rag_vectors(collection);
|
|
17
|
+
CREATE INDEX IF NOT EXISTS idx_rag_text_hash ON rag_vectors(text_hash);
|
|
18
|
+
`);
|
|
19
|
+
}
|
|
20
|
+
// ── Engine ───────────────────────────────────────────────────
|
|
21
|
+
export class RAGEngine {
|
|
22
|
+
db;
|
|
23
|
+
config;
|
|
24
|
+
log = getLogger();
|
|
25
|
+
ts = null;
|
|
26
|
+
embedding = null;
|
|
27
|
+
llm = null;
|
|
28
|
+
// Prepared statements
|
|
29
|
+
stmtUpsert;
|
|
30
|
+
stmtSearch;
|
|
31
|
+
stmtSearchAll;
|
|
32
|
+
stmtGetByHash;
|
|
33
|
+
stmtDelete;
|
|
34
|
+
stmtStats;
|
|
35
|
+
stmtTotal;
|
|
36
|
+
stmtLastIndexed;
|
|
37
|
+
stmtGetCollections;
|
|
38
|
+
constructor(db, config) {
|
|
39
|
+
this.db = db;
|
|
40
|
+
this.config = {
|
|
41
|
+
brainName: config.brainName,
|
|
42
|
+
defaultThreshold: config.defaultThreshold ?? 0.3,
|
|
43
|
+
defaultLimit: config.defaultLimit ?? 10,
|
|
44
|
+
batchSize: config.batchSize ?? 50,
|
|
45
|
+
};
|
|
46
|
+
runRAGMigration(db);
|
|
47
|
+
this.stmtUpsert = db.prepare(`
|
|
48
|
+
INSERT INTO rag_vectors (collection, source_id, text_hash, text_preview, embedding, metadata)
|
|
49
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
50
|
+
ON CONFLICT(collection, source_id) DO UPDATE SET
|
|
51
|
+
text_hash = excluded.text_hash,
|
|
52
|
+
text_preview = excluded.text_preview,
|
|
53
|
+
embedding = excluded.embedding,
|
|
54
|
+
metadata = excluded.metadata,
|
|
55
|
+
created_at = datetime('now')
|
|
56
|
+
`);
|
|
57
|
+
this.stmtSearch = db.prepare(`
|
|
58
|
+
SELECT id, collection, source_id, text_preview, embedding, metadata, created_at
|
|
59
|
+
FROM rag_vectors WHERE collection = ?
|
|
60
|
+
`);
|
|
61
|
+
this.stmtSearchAll = db.prepare(`
|
|
62
|
+
SELECT id, collection, source_id, text_preview, embedding, metadata, created_at
|
|
63
|
+
FROM rag_vectors
|
|
64
|
+
`);
|
|
65
|
+
this.stmtGetByHash = db.prepare(`
|
|
66
|
+
SELECT id FROM rag_vectors WHERE collection = ? AND source_id = ? AND text_hash = ?
|
|
67
|
+
`);
|
|
68
|
+
this.stmtDelete = db.prepare(`DELETE FROM rag_vectors WHERE collection = ? AND source_id = ?`);
|
|
69
|
+
this.stmtStats = db.prepare(`
|
|
70
|
+
SELECT collection, COUNT(*) as count FROM rag_vectors GROUP BY collection
|
|
71
|
+
`);
|
|
72
|
+
this.stmtTotal = db.prepare(`SELECT COUNT(*) as total FROM rag_vectors`);
|
|
73
|
+
this.stmtLastIndexed = db.prepare(`
|
|
74
|
+
SELECT MAX(created_at) as last_indexed FROM rag_vectors
|
|
75
|
+
`);
|
|
76
|
+
this.stmtGetCollections = db.prepare(`SELECT DISTINCT collection FROM rag_vectors`);
|
|
77
|
+
}
|
|
78
|
+
setThoughtStream(stream) { this.ts = stream; }
|
|
79
|
+
setEmbeddingEngine(engine) { this.embedding = engine; }
|
|
80
|
+
setLLMService(llm) { this.llm = llm; }
|
|
81
|
+
/**
|
|
82
|
+
* Index a single item into the RAG store.
|
|
83
|
+
*/
|
|
84
|
+
async index(collection, sourceId, text, metadata) {
|
|
85
|
+
if (!this.embedding) {
|
|
86
|
+
this.log.warn('[RAG] No embedding engine available, skipping index');
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
const textHash = simpleHash(text);
|
|
90
|
+
// Skip if text hasn't changed
|
|
91
|
+
const existing = this.stmtGetByHash.get(collection, sourceId, textHash);
|
|
92
|
+
if (existing)
|
|
93
|
+
return false;
|
|
94
|
+
try {
|
|
95
|
+
const vector = await this.embedding.embed(text);
|
|
96
|
+
const blob = serializeEmbedding(vector);
|
|
97
|
+
const preview = text.slice(0, 200);
|
|
98
|
+
const meta = metadata ? JSON.stringify(metadata) : null;
|
|
99
|
+
this.stmtUpsert.run(collection, sourceId, textHash, preview, blob, meta);
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
this.log.error(`[RAG] Failed to index ${collection}:${sourceId}: ${err.message}`);
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Batch index multiple items. Returns count of newly indexed items.
|
|
109
|
+
*/
|
|
110
|
+
async indexBatch(items) {
|
|
111
|
+
if (!this.embedding)
|
|
112
|
+
return 0;
|
|
113
|
+
this.ts?.emit('rag', 'analyzing', `Batch indexing ${items.length} items`, 'routine');
|
|
114
|
+
let indexed = 0;
|
|
115
|
+
const batchSize = this.config.batchSize;
|
|
116
|
+
for (let i = 0; i < items.length; i += batchSize) {
|
|
117
|
+
const batch = items.slice(i, i + batchSize);
|
|
118
|
+
// Filter out unchanged items
|
|
119
|
+
const toIndex = batch.filter(item => {
|
|
120
|
+
const hash = simpleHash(item.text);
|
|
121
|
+
const existing = this.stmtGetByHash.get(item.collection, item.sourceId, hash);
|
|
122
|
+
return !existing;
|
|
123
|
+
});
|
|
124
|
+
if (toIndex.length === 0)
|
|
125
|
+
continue;
|
|
126
|
+
// Compute embeddings in batch
|
|
127
|
+
const texts = toIndex.map(item => item.text);
|
|
128
|
+
const embeddings = await this.embedding.embedBatch(texts);
|
|
129
|
+
const insertMany = this.db.transaction(() => {
|
|
130
|
+
for (let j = 0; j < toIndex.length; j++) {
|
|
131
|
+
const item = toIndex[j];
|
|
132
|
+
const vector = embeddings[j];
|
|
133
|
+
if (!vector)
|
|
134
|
+
continue;
|
|
135
|
+
const blob = serializeEmbedding(vector);
|
|
136
|
+
const preview = item.text.slice(0, 200);
|
|
137
|
+
const meta = item.metadata ? JSON.stringify(item.metadata) : null;
|
|
138
|
+
const hash = simpleHash(item.text);
|
|
139
|
+
this.stmtUpsert.run(item.collection, item.sourceId, hash, preview, blob, meta);
|
|
140
|
+
indexed++;
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
insertMany();
|
|
144
|
+
}
|
|
145
|
+
if (indexed > 0) {
|
|
146
|
+
this.ts?.emit('rag', 'discovering', `Indexed ${indexed} new vectors`, 'notable');
|
|
147
|
+
}
|
|
148
|
+
return indexed;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Search the RAG store for similar content.
|
|
152
|
+
*/
|
|
153
|
+
async search(query, options) {
|
|
154
|
+
if (!this.embedding)
|
|
155
|
+
return [];
|
|
156
|
+
const limit = options?.limit ?? this.config.defaultLimit;
|
|
157
|
+
const threshold = options?.threshold ?? this.config.defaultThreshold;
|
|
158
|
+
const collections = options?.collections;
|
|
159
|
+
this.ts?.emit('rag', 'analyzing', `Searching RAG: "${query.slice(0, 50)}..."`, 'routine');
|
|
160
|
+
let queryVector;
|
|
161
|
+
try {
|
|
162
|
+
queryVector = await this.embedding.embed(query);
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return [];
|
|
166
|
+
}
|
|
167
|
+
// Get candidates from DB
|
|
168
|
+
let rows;
|
|
169
|
+
if (collections && collections.length > 0) {
|
|
170
|
+
// Search specific collections
|
|
171
|
+
const allRows = [];
|
|
172
|
+
for (const col of collections) {
|
|
173
|
+
const colRows = this.stmtSearch.all(col);
|
|
174
|
+
allRows.push(...colRows);
|
|
175
|
+
}
|
|
176
|
+
rows = allRows;
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
rows = this.stmtSearchAll.all();
|
|
180
|
+
}
|
|
181
|
+
// Compute similarities
|
|
182
|
+
const results = [];
|
|
183
|
+
for (const row of rows) {
|
|
184
|
+
const storedVector = deserializeEmbedding(row.embedding);
|
|
185
|
+
const sim = cosineSimilarity(queryVector, storedVector);
|
|
186
|
+
if (sim >= threshold) {
|
|
187
|
+
results.push({
|
|
188
|
+
collection: row.collection,
|
|
189
|
+
sourceId: row.source_id,
|
|
190
|
+
text: row.text_preview ?? '',
|
|
191
|
+
similarity: Math.round(sim * 1000) / 1000,
|
|
192
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Sort by similarity descending
|
|
197
|
+
results.sort((a, b) => b.similarity - a.similarity);
|
|
198
|
+
// Take top-K
|
|
199
|
+
const topK = results.slice(0, limit);
|
|
200
|
+
// Optional reranking via LLM
|
|
201
|
+
if (options?.rerank && this.llm && topK.length > 1) {
|
|
202
|
+
return this.rerank(query, topK);
|
|
203
|
+
}
|
|
204
|
+
return topK;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* LLM-based reranking of search results.
|
|
208
|
+
*/
|
|
209
|
+
async rerank(query, results) {
|
|
210
|
+
if (!this.llm || results.length <= 1)
|
|
211
|
+
return results;
|
|
212
|
+
try {
|
|
213
|
+
const numbered = results.map((r, i) => `[${i + 1}] ${r.text}`).join('\n');
|
|
214
|
+
const prompt = `Given the query: "${query}"\n\nRank these results by relevance (most relevant first). Return ONLY the numbers in order, comma-separated:\n\n${numbered}`;
|
|
215
|
+
const response = await this.llm.call('custom', prompt, { maxTokens: 100 });
|
|
216
|
+
if (!response)
|
|
217
|
+
return results;
|
|
218
|
+
// Parse the ranking
|
|
219
|
+
const rankOrder = response.text
|
|
220
|
+
.replace(/[^0-9,]/g, '')
|
|
221
|
+
.split(',')
|
|
222
|
+
.map((n) => parseInt(n.trim(), 10) - 1)
|
|
223
|
+
.filter((n) => n >= 0 && n < results.length);
|
|
224
|
+
if (rankOrder.length === 0)
|
|
225
|
+
return results;
|
|
226
|
+
// Reorder results
|
|
227
|
+
const reranked = [];
|
|
228
|
+
const used = new Set();
|
|
229
|
+
for (const idx of rankOrder) {
|
|
230
|
+
if (!used.has(idx)) {
|
|
231
|
+
reranked.push(results[idx]);
|
|
232
|
+
used.add(idx);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
// Append any missed results
|
|
236
|
+
for (let i = 0; i < results.length; i++) {
|
|
237
|
+
if (!used.has(i))
|
|
238
|
+
reranked.push(results[i]);
|
|
239
|
+
}
|
|
240
|
+
return reranked;
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
return results; // Graceful fallback
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Format search results as context for an LLM prompt.
|
|
248
|
+
*/
|
|
249
|
+
augment(query, context) {
|
|
250
|
+
if (context.length === 0)
|
|
251
|
+
return query;
|
|
252
|
+
const contextBlock = context
|
|
253
|
+
.map((r, i) => `[${i + 1}] (${r.collection}, sim=${r.similarity}) ${r.text}`)
|
|
254
|
+
.join('\n');
|
|
255
|
+
return `Context from knowledge base:\n${contextBlock}\n\nQuery: ${query}`;
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Remove an item from the RAG store.
|
|
259
|
+
*/
|
|
260
|
+
remove(collection, sourceId) {
|
|
261
|
+
this.stmtDelete.run(collection, sourceId);
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Get RAG engine status.
|
|
265
|
+
*/
|
|
266
|
+
getStatus() {
|
|
267
|
+
const total = this.stmtTotal.get().total;
|
|
268
|
+
const collections = this.stmtStats.all();
|
|
269
|
+
const lastRow = this.stmtLastIndexed.get();
|
|
270
|
+
return {
|
|
271
|
+
totalVectors: total,
|
|
272
|
+
collections,
|
|
273
|
+
lastIndexedAt: lastRow.last_indexed,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Get list of all collections.
|
|
278
|
+
*/
|
|
279
|
+
getCollections() {
|
|
280
|
+
const rows = this.stmtGetCollections.all();
|
|
281
|
+
return rows.map(r => r.collection);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// ── Helpers ──────────────────────────────────────────────────
|
|
285
|
+
function simpleHash(text) {
|
|
286
|
+
let hash = 0;
|
|
287
|
+
for (let i = 0; i < text.length; i++) {
|
|
288
|
+
const char = text.charCodeAt(i);
|
|
289
|
+
hash = ((hash << 5) - hash) + char;
|
|
290
|
+
hash = hash & hash; // Convert to 32bit integer
|
|
291
|
+
}
|
|
292
|
+
return hash.toString(36);
|
|
293
|
+
}
|
|
294
|
+
function serializeEmbedding(embedding) {
|
|
295
|
+
return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
296
|
+
}
|
|
297
|
+
function deserializeEmbedding(buffer) {
|
|
298
|
+
const copy = Buffer.from(buffer);
|
|
299
|
+
return new Float32Array(copy.buffer, copy.byteOffset, copy.length / 4);
|
|
300
|
+
}
|
|
301
|
+
function cosineSimilarity(a, b) {
|
|
302
|
+
if (a.length !== b.length)
|
|
303
|
+
return 0;
|
|
304
|
+
let dot = 0;
|
|
305
|
+
for (let i = 0; i < a.length; i++) {
|
|
306
|
+
dot += a[i] * b[i];
|
|
307
|
+
}
|
|
308
|
+
return Math.max(0, Math.min(1, dot));
|
|
309
|
+
}
|
|
310
|
+
//# sourceMappingURL=rag-engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rag-engine.js","sourceRoot":"","sources":["../../src/rag/rag-engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AA+C/C,gEAAgE;AAEhE,MAAM,UAAU,eAAe,CAAC,EAAqB;IACnD,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;GAcP,CAAC,CAAC;AACL,CAAC;AAED,gEAAgE;AAEhE,MAAM,OAAO,SAAS;IACH,EAAE,CAAoB;IACtB,MAAM,CAA4B;IAClC,GAAG,GAAG,SAAS,EAAE,CAAC;IAC3B,EAAE,GAAyB,IAAI,CAAC;IAChC,SAAS,GAA+B,IAAI,CAAC;IAC7C,GAAG,GAAsB,IAAI,CAAC;IAEtC,sBAAsB;IACL,UAAU,CAAqB;IAC/B,UAAU,CAAqB;IAC/B,aAAa,CAAqB;IAClC,aAAa,CAAqB;IAClC,UAAU,CAAqB;IAC/B,SAAS,CAAqB;IAC9B,SAAS,CAAqB;IAC9B,eAAe,CAAqB;IACpC,kBAAkB,CAAqB;IAExD,YAAY,EAAqB,EAAE,MAAuB;QACxD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG;YACZ,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,GAAG;YAChD,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;YACvC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;SAClC,CAAC;QAEF,eAAe,CAAC,EAAE,CAAC,CAAC;QAEpB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;KAS5B,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC;;;KAG5B,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,OAAO,CAAC;;;KAG/B,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,OAAO,CAAC;;KAE/B,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,gEAAgE,CAAC,CAAC;QAE/F,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC;;KAE3B,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC;QAEzE,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC,OAAO,CAAC;;KAEjC,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC;IACtF,CAAC;IAED,gBAAgB,CAAC,MAAqB,IAAU,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;IACnE,kBAAkB,CAAC,MAA2B,IAAU,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC;IAClF,aAAa,CAAC,GAAe,IAAU,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;IAExD;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,UAAkB,EAAE,QAAgB,EAAE,IAAY,EAAE,QAAkC;QAChG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAElC,8BAA8B;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAA+B,CAAC;QACtG,IAAI,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE3B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAExD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACzE,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,IAAI,QAAQ,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7F,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAwG;QACvH,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,CAAC;QAE9B,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,kBAAkB,KAAK,CAAC,MAAM,QAAQ,EAAE,SAAS,CAAC,CAAC;QAErF,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAE5C,6BAA6B;YAC7B,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBAClC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAA+B,CAAC;gBAC5G,OAAO,CAAC,QAAQ,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEnC,8BAA8B;YAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE1D,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;gBAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;oBACzB,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAC7B,IAAI,CAAC,MAAM;wBAAE,SAAS;oBAEtB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;oBACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBACxC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;oBAClE,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAEnC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC/E,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,CAAC;YACH,UAAU,EAAE,CAAC;QACf,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,WAAW,OAAO,cAAc,EAAE,SAAS,CAAC,CAAC;QACnF,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAA0B;QACpD,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,CAAC;QAE/B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QACzD,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC;QACrE,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;QAEzC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,mBAAmB,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE1F,IAAI,WAAyB,CAAC;QAC9B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,yBAAyB;QACzB,IAAI,IAAoI,CAAC;QAEzI,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,8BAA8B;YAC9B,MAAM,OAAO,GAAgB,EAAE,CAAC;YAChC,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAgB,CAAC;gBACxD,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC3B,CAAC;YACD,IAAI,GAAG,OAAO,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,EAAiB,CAAC;QACjD,CAAC;QAED,uBAAuB;QACvB,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,YAAY,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,gBAAgB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;YAExD,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC;oBACX,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,QAAQ,EAAE,GAAG,CAAC,SAAS;oBACvB,IAAI,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE;oBAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI;oBACzC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAEpD,aAAa;QACb,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAErC,6BAA6B;QAC7B,IAAI,OAAO,EAAE,MAAM,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAoB;QAC9C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,OAAO,CAAC;QAErD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1E,MAAM,MAAM,GAAG,qBAAqB,KAAK,qHAAqH,QAAQ,EAAE,CAAC;YAEzK,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YAC3E,IAAI,CAAC,QAAQ;gBAAE,OAAO,OAAO,CAAC;YAE9B,oBAAoB;YACpB,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI;iBAC5B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;iBACvB,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;iBAC9C,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;YAEvD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC;YAE3C,kBAAkB;YAClB,MAAM,QAAQ,GAAgB,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAE,CAAC,CAAC;oBAC7B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;YACD,4BAA4B;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAC/C,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,OAAO,CAAC,CAAC,oBAAoB;QACtC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,KAAa,EAAE,OAAoB;QACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEvC,MAAM,YAAY,GAAG,OAAO;aACzB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,UAAU,SAAS,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5E,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,OAAO,iCAAiC,YAAY,cAAc,KAAK,EAAE,CAAC;IAC5E,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,UAAkB,EAAE,QAAgB;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS;QACP,MAAM,KAAK,GAAI,IAAI,CAAC,SAAS,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAChE,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAqB,CAAC;QAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,EAAqC,CAAC;QAE9E,OAAO;YACL,YAAY,EAAE,KAAK;YACnB,WAAW;YACX,aAAa,EAAE,OAAO,CAAC,YAAY;SACpC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAmC,CAAC;QAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;CACF;AAED,gEAAgE;AAEhE,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QACnC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,2BAA2B;IACjD,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAuB;IACjD,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAc;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAe,EAAE,CAAe;IACxD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACvB,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { RAGEngine } from './rag-engine.js';
|
|
3
|
+
export interface RAGIndexerConfig {
|
|
4
|
+
/** Collections to auto-index. Default: all known collections */
|
|
5
|
+
collections?: string[];
|
|
6
|
+
/** Batch size for indexing. Default: 50 */
|
|
7
|
+
batchSize?: number;
|
|
8
|
+
}
|
|
9
|
+
export interface IndexSource {
|
|
10
|
+
collection: string;
|
|
11
|
+
query: string;
|
|
12
|
+
textColumns: string[];
|
|
13
|
+
idColumn?: string;
|
|
14
|
+
timestampColumn?: string;
|
|
15
|
+
}
|
|
16
|
+
export interface IndexerStatus {
|
|
17
|
+
lastRun: string | null;
|
|
18
|
+
totalIndexed: number;
|
|
19
|
+
sources: Array<{
|
|
20
|
+
collection: string;
|
|
21
|
+
count: number;
|
|
22
|
+
}>;
|
|
23
|
+
}
|
|
24
|
+
export declare class RAGIndexer {
|
|
25
|
+
private readonly db;
|
|
26
|
+
private readonly config;
|
|
27
|
+
private readonly log;
|
|
28
|
+
private rag;
|
|
29
|
+
private lastRun;
|
|
30
|
+
private totalIndexed;
|
|
31
|
+
private sourceCounts;
|
|
32
|
+
private customSources;
|
|
33
|
+
constructor(db: Database.Database, config?: RAGIndexerConfig);
|
|
34
|
+
setRAGEngine(rag: RAGEngine): void;
|
|
35
|
+
/**
|
|
36
|
+
* Register a custom index source (for brain-specific tables).
|
|
37
|
+
*/
|
|
38
|
+
addSource(source: IndexSource): void;
|
|
39
|
+
/**
|
|
40
|
+
* Run full indexing over all available sources.
|
|
41
|
+
* Only indexes tables that actually exist in the database.
|
|
42
|
+
*/
|
|
43
|
+
indexAll(): Promise<number>;
|
|
44
|
+
/**
|
|
45
|
+
* Index a single source.
|
|
46
|
+
*/
|
|
47
|
+
private indexSource;
|
|
48
|
+
/**
|
|
49
|
+
* Get indexer status.
|
|
50
|
+
*/
|
|
51
|
+
getStatus(): IndexerStatus;
|
|
52
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { getLogger } from '../utils/logger.js';
|
|
2
|
+
// ── Default Sources ──────────────────────────────────────────
|
|
3
|
+
const DEFAULT_SOURCES = [
|
|
4
|
+
{
|
|
5
|
+
collection: 'insights',
|
|
6
|
+
query: `SELECT id, content, category, created_at FROM research_insights WHERE content IS NOT NULL`,
|
|
7
|
+
textColumns: ['content'],
|
|
8
|
+
timestampColumn: 'created_at',
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
collection: 'memories',
|
|
12
|
+
query: `SELECT id, key, content, category FROM memories WHERE content IS NOT NULL`,
|
|
13
|
+
textColumns: ['key', 'content'],
|
|
14
|
+
timestampColumn: undefined,
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
collection: 'principles',
|
|
18
|
+
query: `SELECT id, name, description, evidence_count FROM distilled_principles WHERE description IS NOT NULL`,
|
|
19
|
+
textColumns: ['name', 'description'],
|
|
20
|
+
timestampColumn: undefined,
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
collection: 'errors',
|
|
24
|
+
query: `SELECT id, error_type, message, context FROM errors WHERE message IS NOT NULL`,
|
|
25
|
+
textColumns: ['error_type', 'message'],
|
|
26
|
+
timestampColumn: undefined,
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
collection: 'solutions',
|
|
30
|
+
query: `SELECT id, description, steps, context FROM solutions WHERE description IS NOT NULL`,
|
|
31
|
+
textColumns: ['description', 'steps'],
|
|
32
|
+
timestampColumn: undefined,
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
collection: 'rules',
|
|
36
|
+
query: `SELECT id, condition_pattern, action, domain FROM rules WHERE condition_pattern IS NOT NULL`,
|
|
37
|
+
textColumns: ['condition_pattern', 'action'],
|
|
38
|
+
timestampColumn: undefined,
|
|
39
|
+
},
|
|
40
|
+
];
|
|
41
|
+
// ── Indexer ──────────────────────────────────────────────────
|
|
42
|
+
export class RAGIndexer {
|
|
43
|
+
db;
|
|
44
|
+
config;
|
|
45
|
+
log = getLogger();
|
|
46
|
+
rag = null;
|
|
47
|
+
lastRun = null;
|
|
48
|
+
totalIndexed = 0;
|
|
49
|
+
sourceCounts = new Map();
|
|
50
|
+
customSources = [];
|
|
51
|
+
constructor(db, config) {
|
|
52
|
+
this.db = db;
|
|
53
|
+
this.config = {
|
|
54
|
+
collections: config?.collections ?? [],
|
|
55
|
+
batchSize: config?.batchSize ?? 50,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
setRAGEngine(rag) { this.rag = rag; }
|
|
59
|
+
/**
|
|
60
|
+
* Register a custom index source (for brain-specific tables).
|
|
61
|
+
*/
|
|
62
|
+
addSource(source) {
|
|
63
|
+
this.customSources.push(source);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Run full indexing over all available sources.
|
|
67
|
+
* Only indexes tables that actually exist in the database.
|
|
68
|
+
*/
|
|
69
|
+
async indexAll() {
|
|
70
|
+
if (!this.rag) {
|
|
71
|
+
this.log.warn('[RAGIndexer] No RAG engine set, skipping');
|
|
72
|
+
return 0;
|
|
73
|
+
}
|
|
74
|
+
const allSources = [...DEFAULT_SOURCES, ...this.customSources];
|
|
75
|
+
const filteredSources = this.config.collections.length > 0
|
|
76
|
+
? allSources.filter(s => this.config.collections.includes(s.collection))
|
|
77
|
+
: allSources;
|
|
78
|
+
let totalNew = 0;
|
|
79
|
+
for (const source of filteredSources) {
|
|
80
|
+
try {
|
|
81
|
+
const count = await this.indexSource(source);
|
|
82
|
+
totalNew += count;
|
|
83
|
+
this.sourceCounts.set(source.collection, (this.sourceCounts.get(source.collection) ?? 0) + count);
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
// Table might not exist in this brain — skip gracefully
|
|
87
|
+
const msg = err.message;
|
|
88
|
+
if (msg.includes('no such table') || msg.includes('no such column')) {
|
|
89
|
+
this.log.debug(`[RAGIndexer] Skipping ${source.collection}: table not found`);
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
this.log.error(`[RAGIndexer] Error indexing ${source.collection}: ${msg}`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
this.lastRun = new Date().toISOString();
|
|
97
|
+
this.totalIndexed += totalNew;
|
|
98
|
+
if (totalNew > 0) {
|
|
99
|
+
this.log.info(`[RAGIndexer] Indexed ${totalNew} new items across ${filteredSources.length} sources`);
|
|
100
|
+
}
|
|
101
|
+
return totalNew;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Index a single source.
|
|
105
|
+
*/
|
|
106
|
+
async indexSource(source) {
|
|
107
|
+
if (!this.rag)
|
|
108
|
+
return 0;
|
|
109
|
+
const rows = this.db.prepare(source.query).all();
|
|
110
|
+
if (rows.length === 0)
|
|
111
|
+
return 0;
|
|
112
|
+
const items = rows.map(row => {
|
|
113
|
+
const id = row[source.idColumn ?? 'id'];
|
|
114
|
+
const text = source.textColumns
|
|
115
|
+
.map(col => row[col] ?? '')
|
|
116
|
+
.filter(Boolean)
|
|
117
|
+
.join(' | ');
|
|
118
|
+
// Build metadata from all non-text, non-id columns
|
|
119
|
+
const metadata = {};
|
|
120
|
+
for (const [key, value] of Object.entries(row)) {
|
|
121
|
+
if (key !== (source.idColumn ?? 'id') && !source.textColumns.includes(key) && value != null) {
|
|
122
|
+
metadata[key] = value;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return { collection: source.collection, sourceId: id, text, metadata };
|
|
126
|
+
});
|
|
127
|
+
return this.rag.indexBatch(items);
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Get indexer status.
|
|
131
|
+
*/
|
|
132
|
+
getStatus() {
|
|
133
|
+
const sources = [];
|
|
134
|
+
for (const [collection, count] of this.sourceCounts) {
|
|
135
|
+
sources.push({ collection, count });
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
lastRun: this.lastRun,
|
|
139
|
+
totalIndexed: this.totalIndexed,
|
|
140
|
+
sources,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
//# sourceMappingURL=rag-indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rag-indexer.js","sourceRoot":"","sources":["../../src/rag/rag-indexer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AA0B/C,gEAAgE;AAEhE,MAAM,eAAe,GAAkB;IACrC;QACE,UAAU,EAAE,UAAU;QACtB,KAAK,EAAE,2FAA2F;QAClG,WAAW,EAAE,CAAC,SAAS,CAAC;QACxB,eAAe,EAAE,YAAY;KAC9B;IACD;QACE,UAAU,EAAE,UAAU;QACtB,KAAK,EAAE,2EAA2E;QAClF,WAAW,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC;QAC/B,eAAe,EAAE,SAAS;KAC3B;IACD;QACE,UAAU,EAAE,YAAY;QACxB,KAAK,EAAE,sGAAsG;QAC7G,WAAW,EAAE,CAAC,MAAM,EAAE,aAAa,CAAC;QACpC,eAAe,EAAE,SAAS;KAC3B;IACD;QACE,UAAU,EAAE,QAAQ;QACpB,KAAK,EAAE,+EAA+E;QACtF,WAAW,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC;QACtC,eAAe,EAAE,SAAS;KAC3B;IACD;QACE,UAAU,EAAE,WAAW;QACvB,KAAK,EAAE,qFAAqF;QAC5F,WAAW,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC;QACrC,eAAe,EAAE,SAAS;KAC3B;IACD;QACE,UAAU,EAAE,OAAO;QACnB,KAAK,EAAE,6FAA6F;QACpG,WAAW,EAAE,CAAC,mBAAmB,EAAE,QAAQ,CAAC;QAC5C,eAAe,EAAE,SAAS;KAC3B;CACF,CAAC;AAEF,gEAAgE;AAEhE,MAAM,OAAO,UAAU;IACJ,EAAE,CAAoB;IACtB,MAAM,CAA6B;IACnC,GAAG,GAAG,SAAS,EAAE,CAAC;IAC3B,GAAG,GAAqB,IAAI,CAAC;IAC7B,OAAO,GAAkB,IAAI,CAAC;IAC9B,YAAY,GAAG,CAAC,CAAC;IACjB,YAAY,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC9C,aAAa,GAAkB,EAAE,CAAC;IAE1C,YAAY,EAAqB,EAAE,MAAyB;QAC1D,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG;YACZ,WAAW,EAAE,MAAM,EAAE,WAAW,IAAI,EAAE;YACtC,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,EAAE;SACnC,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,GAAc,IAAU,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;IAEtD;;OAEG;IACH,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;YAC1D,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;QAC/D,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;YACxD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACxE,CAAC,CAAC,UAAU,CAAC;QAEf,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC7C,QAAQ,IAAI,KAAK,CAAC;gBAClB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;YACpG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wDAAwD;gBACxD,MAAM,GAAG,GAAI,GAAa,CAAC,OAAO,CAAC;gBACnC,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBACpE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,MAAM,CAAC,UAAU,mBAAmB,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+BAA+B,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACxC,IAAI,CAAC,YAAY,IAAI,QAAQ,CAAC;QAE9B,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,QAAQ,qBAAqB,eAAe,CAAC,MAAM,UAAU,CAAC,CAAC;QACvG,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,MAAmB;QAC3C,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC;QAExB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,EAAoC,CAAC;QACnF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEhC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YAC3B,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAW,CAAC;YAClD,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW;iBAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAW,IAAI,EAAE,CAAC;iBACpC,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,KAAK,CAAC,CAAC;YAEf,mDAAmD;YACnD,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/C,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;oBAC5F,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;gBACxB,CAAC;YACH,CAAC;YAED,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACzE,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,SAAS;QACP,MAAM,OAAO,GAAiD,EAAE,CAAC;QACjE,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,OAAO;SACR,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|