@xfabric/memory 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/index.d.ts +3 -0
- package/dist/chunking/index.d.ts.map +1 -0
- package/dist/chunking/index.js +3 -0
- package/dist/chunking/index.js.map +1 -0
- package/dist/chunking/markdown.d.ts +13 -0
- package/dist/chunking/markdown.d.ts.map +1 -0
- package/dist/chunking/markdown.js +106 -0
- package/dist/chunking/markdown.js.map +1 -0
- package/dist/chunking/session.d.ts +24 -0
- package/dist/chunking/session.d.ts.map +1 -0
- package/dist/chunking/session.js +173 -0
- package/dist/chunking/session.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -0
- package/dist/memory-manager.d.ts +189 -0
- package/dist/memory-manager.d.ts.map +1 -0
- package/dist/memory-manager.js +1055 -0
- package/dist/memory-manager.js.map +1 -0
- package/dist/providers/gemini.d.ts +6 -0
- package/dist/providers/gemini.d.ts.map +1 -0
- package/dist/providers/gemini.js +73 -0
- package/dist/providers/gemini.js.map +1 -0
- package/dist/providers/index.d.ts +20 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +102 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/local.d.ts +14 -0
- package/dist/providers/local.d.ts.map +1 -0
- package/dist/providers/local.js +73 -0
- package/dist/providers/local.js.map +1 -0
- package/dist/providers/openai.d.ts +6 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +48 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/types.d.ts +62 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/search/fts.d.ts +11 -0
- package/dist/search/fts.d.ts.map +1 -0
- package/dist/search/fts.js +50 -0
- package/dist/search/fts.js.map +1 -0
- package/dist/search/hybrid.d.ts +16 -0
- package/dist/search/hybrid.d.ts.map +1 -0
- package/dist/search/hybrid.js +83 -0
- package/dist/search/hybrid.js.map +1 -0
- package/dist/search/index.d.ts +4 -0
- package/dist/search/index.d.ts.map +1 -0
- package/dist/search/index.js +4 -0
- package/dist/search/index.js.map +1 -0
- package/dist/search/vector.d.ts +25 -0
- package/dist/search/vector.d.ts.map +1 -0
- package/dist/search/vector.js +152 -0
- package/dist/search/vector.js.map +1 -0
- package/dist/storage/index.d.ts +4 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +4 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/schema.d.ts +24 -0
- package/dist/storage/schema.d.ts.map +1 -0
- package/dist/storage/schema.js +175 -0
- package/dist/storage/schema.js.map +1 -0
- package/dist/storage/sqlite-vec.d.ts +22 -0
- package/dist/storage/sqlite-vec.d.ts.map +1 -0
- package/dist/storage/sqlite-vec.js +85 -0
- package/dist/storage/sqlite-vec.js.map +1 -0
- package/dist/storage/sqlite.d.ts +206 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +352 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/sync/index.d.ts +4 -0
- package/dist/sync/index.d.ts.map +1 -0
- package/dist/sync/index.js +4 -0
- package/dist/sync/index.js.map +1 -0
- package/dist/sync/minimatch.d.ts +6 -0
- package/dist/sync/minimatch.d.ts.map +1 -0
- package/dist/sync/minimatch.js +60 -0
- package/dist/sync/minimatch.js.map +1 -0
- package/dist/sync/session-monitor.d.ts +50 -0
- package/dist/sync/session-monitor.d.ts.map +1 -0
- package/dist/sync/session-monitor.js +126 -0
- package/dist/sync/session-monitor.js.map +1 -0
- package/dist/sync/watcher.d.ts +44 -0
- package/dist/sync/watcher.d.ts.map +1 -0
- package/dist/sync/watcher.js +110 -0
- package/dist/sync/watcher.js.map +1 -0
- package/dist/tools/index.d.ts +3 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +3 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/memory-get.d.ts +32 -0
- package/dist/tools/memory-get.d.ts.map +1 -0
- package/dist/tools/memory-get.js +53 -0
- package/dist/tools/memory-get.js.map +1 -0
- package/dist/tools/memory-search.d.ts +32 -0
- package/dist/tools/memory-search.d.ts.map +1 -0
- package/dist/tools/memory-search.js +56 -0
- package/dist/tools/memory-search.js.map +1 -0
- package/dist/types.d.ts +350 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +15 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/concurrency.d.ts +25 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +59 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/hash.d.ts +9 -0
- package/dist/utils/hash.d.ts.map +1 -0
- package/dist/utils/hash.js +16 -0
- package/dist/utils/hash.js.map +1 -0
- package/dist/utils/index.d.ts +4 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +4 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/retry.d.ts +22 -0
- package/dist/utils/retry.d.ts.map +1 -0
- package/dist/utils/retry.js +48 -0
- package/dist/utils/retry.js.map +1 -0
- package/package.json +67 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build an FTS5 query from search text
|
|
3
|
+
* Extracts alphanumeric tokens and joins with AND
|
|
4
|
+
*/
|
|
5
|
+
export function buildFtsQuery(text) {
|
|
6
|
+
// Extract alphanumeric tokens (including underscores)
|
|
7
|
+
const tokens = text.match(/[A-Za-z0-9_]+/g);
|
|
8
|
+
if (!tokens || tokens.length === 0) {
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
// Quote each token and join with AND
|
|
12
|
+
return tokens.map((t) => `"${t}"`).join(" AND ");
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Convert BM25 rank to a 0-1 score
|
|
16
|
+
* Lower rank = higher relevance, so we invert
|
|
17
|
+
*/
|
|
18
|
+
export function bm25RankToScore(rank) {
|
|
19
|
+
// Handle non-finite values
|
|
20
|
+
const normalized = Number.isFinite(rank) ? Math.max(0, rank) : 999;
|
|
21
|
+
// Convert rank to score: 1/(1+rank) maps 0->1, infinity->0
|
|
22
|
+
return 1 / (1 + normalized);
|
|
23
|
+
}
|
|
24
|
+
const DEFAULT_VECTOR_WEIGHT = 0.7;
|
|
25
|
+
const DEFAULT_TEXT_WEIGHT = 0.3;
|
|
26
|
+
/**
|
|
27
|
+
* Merge vector and keyword search results with weighted scoring
|
|
28
|
+
*/
|
|
29
|
+
export function mergeHybridResults(vectorResults, keywordResults, options = {}) {
|
|
30
|
+
const vectorWeight = options.vectorWeight ?? DEFAULT_VECTOR_WEIGHT;
|
|
31
|
+
const textWeight = options.textWeight ?? DEFAULT_TEXT_WEIGHT;
|
|
32
|
+
// Build a map of all results by ID
|
|
33
|
+
const resultMap = new Map();
|
|
34
|
+
// Add vector results
|
|
35
|
+
for (const result of vectorResults) {
|
|
36
|
+
resultMap.set(result.id, {
|
|
37
|
+
path: result.path,
|
|
38
|
+
startLine: result.startLine,
|
|
39
|
+
endLine: result.endLine,
|
|
40
|
+
source: result.source,
|
|
41
|
+
vectorSnippet: result.snippet,
|
|
42
|
+
vectorScore: result.vectorScore,
|
|
43
|
+
textScore: 0,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
// Merge keyword results
|
|
47
|
+
for (const result of keywordResults) {
|
|
48
|
+
const existing = resultMap.get(result.id);
|
|
49
|
+
if (existing) {
|
|
50
|
+
existing.keywordSnippet = result.snippet;
|
|
51
|
+
existing.textScore = result.textScore;
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
resultMap.set(result.id, {
|
|
55
|
+
path: result.path,
|
|
56
|
+
startLine: result.startLine,
|
|
57
|
+
endLine: result.endLine,
|
|
58
|
+
source: result.source,
|
|
59
|
+
keywordSnippet: result.snippet,
|
|
60
|
+
vectorScore: 0,
|
|
61
|
+
textScore: result.textScore,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// Convert to final results with combined scores
|
|
66
|
+
const results = [];
|
|
67
|
+
for (const [, value] of resultMap) {
|
|
68
|
+
const combinedScore = vectorWeight * value.vectorScore + textWeight * value.textScore;
|
|
69
|
+
results.push({
|
|
70
|
+
path: value.path,
|
|
71
|
+
startLine: value.startLine,
|
|
72
|
+
endLine: value.endLine,
|
|
73
|
+
source: value.source,
|
|
74
|
+
// Prefer keyword snippet (often more contextually relevant)
|
|
75
|
+
snippet: value.keywordSnippet || value.vectorSnippet || "",
|
|
76
|
+
score: combinedScore,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
// Sort by combined score descending
|
|
80
|
+
results.sort((a, b) => b.score - a.score);
|
|
81
|
+
return results;
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=hybrid.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/search/hybrid.ts"],"names":[],"mappings":"AAOA;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,sDAAsD;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAE5C,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,qCAAqC;IACrC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACnD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,2BAA2B;IAC3B,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACnE,2DAA2D;IAC3D,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,aAAmC,EACnC,cAAqC,EACrC,UAA+B,EAAE;IAEjC,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,qBAAqB,CAAC;IACnE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,mBAAmB,CAAC;IAE7D,mCAAmC;IACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAYtB,CAAC;IAEJ,qBAAqB;IACrB,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE;YACvB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,aAAa,EAAE,MAAM,CAAC,OAAO;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,SAAS,EAAE,CAAC;SACb,CAAC,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;YACzC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE;gBACvB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,cAAc,EAAE,MAAM,CAAC,OAAO;gBAC9B,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,aAAa,GACjB,YAAY,GAAG,KAAK,CAAC,WAAW,GAAG,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC;QAElE,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,4DAA4D;YAC5D,OAAO,EAAE,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,aAAa,IAAI,EAAE;YAC1D,KAAK,EAAE,aAAa;SACrB,CAAC,CAAC;IACL,CAAC;IAED,oCAAoC;IACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAE1C,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { buildFtsQuery, bm25RankToScore, mergeHybridResults } from "./hybrid.js";
|
|
2
|
+
export { searchFts, type FtsSearchOptions } from "./fts.js";
|
|
3
|
+
export { searchVector, searchVectorWithSqliteVec, syncVectorTable, type VectorSearchOptions, } from "./vector.js";
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/search/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,KAAK,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,yBAAyB,EACzB,eAAe,EACf,KAAK,mBAAmB,GACzB,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/search/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjF,OAAO,EAAE,SAAS,EAAyB,MAAM,UAAU,CAAC;AAC5D,OAAO,EACL,YAAY,EACZ,yBAAyB,EACzB,eAAe,GAEhB,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
import type { VectorSearchResult, MemorySource } from "../types.js";
|
|
3
|
+
export type VectorSearchOptions = {
|
|
4
|
+
maxResults?: number;
|
|
5
|
+
minScore?: number;
|
|
6
|
+
source?: MemorySource;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Search using vector similarity (cosine similarity) - JS fallback
|
|
10
|
+
*/
|
|
11
|
+
export declare function searchVector(db: Database.Database, queryEmbedding: number[], options?: VectorSearchOptions): VectorSearchResult[];
|
|
12
|
+
/**
|
|
13
|
+
* Search using sqlite-vec for database-side ANN search
|
|
14
|
+
* Falls back to JS cosine similarity if vec_chunks table doesn't exist
|
|
15
|
+
*/
|
|
16
|
+
export declare function searchVectorWithSqliteVec(db: Database.Database, queryEmbedding: number[], options?: VectorSearchOptions): VectorSearchResult[];
|
|
17
|
+
/**
|
|
18
|
+
* Sync vec_chunks virtual table with chunks table
|
|
19
|
+
* Call this after bulk inserts to ensure ANN search data is up to date
|
|
20
|
+
*/
|
|
21
|
+
export declare function syncVectorTable(db: Database.Database): {
|
|
22
|
+
synced: number;
|
|
23
|
+
errors: number;
|
|
24
|
+
};
|
|
25
|
+
//# sourceMappingURL=vector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../../src/search/vector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAGpE,MAAM,MAAM,mBAAmB,GAAG;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF;;GAEG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,GAAE,mBAAwB,GAChC,kBAAkB,EAAE,CA0DtB;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,GAAE,mBAAwB,GAChC,kBAAkB,EAAE,CAmEtB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAkDzF"}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { cosineSimilarity } from "../storage/sqlite-vec.js";
|
|
2
|
+
/**
|
|
3
|
+
* Search using vector similarity (cosine similarity) - JS fallback
|
|
4
|
+
*/
|
|
5
|
+
export function searchVector(db, queryEmbedding, options = {}) {
|
|
6
|
+
const { maxResults = 50, minScore = 0, source } = options;
|
|
7
|
+
// Build query with optional source filter
|
|
8
|
+
let sql = `
|
|
9
|
+
SELECT
|
|
10
|
+
id,
|
|
11
|
+
path,
|
|
12
|
+
source,
|
|
13
|
+
start_line,
|
|
14
|
+
end_line,
|
|
15
|
+
text,
|
|
16
|
+
embedding
|
|
17
|
+
FROM chunks
|
|
18
|
+
`;
|
|
19
|
+
const params = [];
|
|
20
|
+
if (source) {
|
|
21
|
+
sql += " WHERE source = ?";
|
|
22
|
+
params.push(source);
|
|
23
|
+
}
|
|
24
|
+
const stmt = db.prepare(sql);
|
|
25
|
+
const rows = (params.length > 0 ? stmt.all(...params) : stmt.all());
|
|
26
|
+
// Calculate similarities
|
|
27
|
+
const results = [];
|
|
28
|
+
for (const row of rows) {
|
|
29
|
+
const embedding = JSON.parse(row.embedding);
|
|
30
|
+
const similarity = cosineSimilarity(queryEmbedding, embedding);
|
|
31
|
+
if (similarity >= minScore) {
|
|
32
|
+
results.push({
|
|
33
|
+
id: row.id,
|
|
34
|
+
path: row.path,
|
|
35
|
+
startLine: row.start_line,
|
|
36
|
+
endLine: row.end_line,
|
|
37
|
+
source: row.source,
|
|
38
|
+
snippet: row.text,
|
|
39
|
+
vectorScore: similarity,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
// Sort by similarity descending
|
|
44
|
+
results.sort((a, b) => b.vectorScore - a.vectorScore);
|
|
45
|
+
// Limit results
|
|
46
|
+
return results.slice(0, maxResults);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Search using sqlite-vec for database-side ANN search
|
|
50
|
+
* Falls back to JS cosine similarity if vec_chunks table doesn't exist
|
|
51
|
+
*/
|
|
52
|
+
export function searchVectorWithSqliteVec(db, queryEmbedding, options = {}) {
|
|
53
|
+
const { maxResults = 50, minScore = 0, source } = options;
|
|
54
|
+
try {
|
|
55
|
+
// Check if vec_chunks virtual table exists
|
|
56
|
+
const tableExists = db
|
|
57
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='vec_chunks'")
|
|
58
|
+
.get();
|
|
59
|
+
if (!tableExists) {
|
|
60
|
+
// Fall back to JS-based search
|
|
61
|
+
return searchVector(db, queryEmbedding, options);
|
|
62
|
+
}
|
|
63
|
+
// Convert embedding to blob format for sqlite-vec
|
|
64
|
+
const embeddingBlob = Buffer.from(new Float32Array(queryEmbedding).buffer);
|
|
65
|
+
// Build query with ANN search
|
|
66
|
+
let sql = `
|
|
67
|
+
SELECT
|
|
68
|
+
c.id,
|
|
69
|
+
c.path,
|
|
70
|
+
c.source,
|
|
71
|
+
c.start_line,
|
|
72
|
+
c.end_line,
|
|
73
|
+
c.text,
|
|
74
|
+
1 - vec_distance_cosine(v.embedding, ?) as score
|
|
75
|
+
FROM vec_chunks v
|
|
76
|
+
JOIN chunks c ON v.id = c.id
|
|
77
|
+
`;
|
|
78
|
+
const params = [embeddingBlob];
|
|
79
|
+
if (source) {
|
|
80
|
+
sql += " WHERE c.source = ?";
|
|
81
|
+
params.push(source);
|
|
82
|
+
}
|
|
83
|
+
sql += " ORDER BY score DESC LIMIT ?";
|
|
84
|
+
params.push(maxResults);
|
|
85
|
+
const rows = db.prepare(sql).all(...params);
|
|
86
|
+
// Filter by min score and map to results
|
|
87
|
+
return rows
|
|
88
|
+
.filter((row) => row.score >= minScore)
|
|
89
|
+
.map((row) => ({
|
|
90
|
+
id: row.id,
|
|
91
|
+
path: row.path,
|
|
92
|
+
startLine: row.start_line,
|
|
93
|
+
endLine: row.end_line,
|
|
94
|
+
source: row.source,
|
|
95
|
+
snippet: row.text,
|
|
96
|
+
vectorScore: row.score,
|
|
97
|
+
}));
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
// Fall back to JS-based search on any error
|
|
101
|
+
return searchVector(db, queryEmbedding, options);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Sync vec_chunks virtual table with chunks table
|
|
106
|
+
* Call this after bulk inserts to ensure ANN search data is up to date
|
|
107
|
+
*/
|
|
108
|
+
export function syncVectorTable(db) {
|
|
109
|
+
let synced = 0;
|
|
110
|
+
let errors = 0;
|
|
111
|
+
try {
|
|
112
|
+
// Check if vec_chunks exists
|
|
113
|
+
const tableExists = db
|
|
114
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='vec_chunks'")
|
|
115
|
+
.get();
|
|
116
|
+
if (!tableExists) {
|
|
117
|
+
return { synced: 0, errors: 0 };
|
|
118
|
+
}
|
|
119
|
+
// Get all chunks that don't have entries in vec_chunks
|
|
120
|
+
const missingRows = db
|
|
121
|
+
.prepare(`
|
|
122
|
+
SELECT c.id, c.embedding
|
|
123
|
+
FROM chunks c
|
|
124
|
+
LEFT JOIN vec_chunks v ON c.id = v.id
|
|
125
|
+
WHERE v.id IS NULL
|
|
126
|
+
`)
|
|
127
|
+
.all();
|
|
128
|
+
// Insert missing entries
|
|
129
|
+
const insertStmt = db.prepare("INSERT INTO vec_chunks (id, embedding) VALUES (?, ?)");
|
|
130
|
+
for (const row of missingRows) {
|
|
131
|
+
try {
|
|
132
|
+
const embedding = JSON.parse(row.embedding);
|
|
133
|
+
const embeddingBlob = Buffer.from(new Float32Array(embedding).buffer);
|
|
134
|
+
insertStmt.run(row.id, embeddingBlob);
|
|
135
|
+
synced++;
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
errors++;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// Remove orphaned entries
|
|
142
|
+
db.exec(`
|
|
143
|
+
DELETE FROM vec_chunks
|
|
144
|
+
WHERE id NOT IN (SELECT id FROM chunks)
|
|
145
|
+
`);
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
// Table doesn't exist or other error
|
|
149
|
+
}
|
|
150
|
+
return { synced, errors };
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=vector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../../src/search/vector.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAQ5D;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,EAAqB,EACrB,cAAwB,EACxB,UAA+B,EAAE;IAEjC,MAAM,EAAE,UAAU,GAAG,EAAE,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE1D,0CAA0C;IAC1C,IAAI,GAAG,GAAG;;;;;;;;;;GAUT,CAAC;IAEF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,IAAI,mBAAmB,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAQ/D,CAAC;IAEJ,yBAAyB;IACzB,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAa,CAAC;QACxD,MAAM,UAAU,GAAG,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;QAE/D,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,SAAS,EAAE,GAAG,CAAC,UAAU;gBACzB,OAAO,EAAE,GAAG,CAAC,QAAQ;gBACrB,MAAM,EAAE,GAAG,CAAC,MAAsB;gBAClC,OAAO,EAAE,GAAG,CAAC,IAAI;gBACjB,WAAW,EAAE,UAAU;aACxB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAEtD,gBAAgB;IAChB,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CACvC,EAAqB,EACrB,cAAwB,EACxB,UAA+B,EAAE;IAEjC,MAAM,EAAE,UAAU,GAAG,EAAE,EAAE,QAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE1D,IAAI,CAAC;QACH,2CAA2C;QAC3C,MAAM,WAAW,GAAG,EAAE;aACnB,OAAO,CAAC,yEAAyE,CAAC;aAClF,GAAG,EAAE,CAAC;QAET,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,+BAA+B;YAC/B,OAAO,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,kDAAkD;QAClD,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;QAE3E,8BAA8B;QAC9B,IAAI,GAAG,GAAG;;;;;;;;;;;KAWT,CAAC;QAEF,MAAM,MAAM,GAAiC,CAAC,aAAa,CAAC,CAAC;QAE7D,IAAI,MAAM,EAAE,CAAC;YACX,GAAG,IAAI,qBAAqB,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;QAED,GAAG,IAAI,8BAA8B,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAQvC,CAAC;QAEJ,yCAAyC;QACzC,OAAO,IAAI;aACR,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,IAAI,QAAQ,CAAC;aACtC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,OAAO,EAAE,GAAG,CAAC,QAAQ;YACrB,MAAM,EAAE,GAAG,CAAC,MAAsB;YAClC,OAAO,EAAE,GAAG,CAAC,IAAI;YACjB,WAAW,EAAE,GAAG,CAAC,KAAK;SACvB,CAAC,CAAC,CAAC;IACR,CAAC;IAAC,MAAM,CAAC;QACP,4CAA4C;QAC5C,OAAO,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,EAAqB;IACnD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,IAAI,CAAC;QACH,6BAA6B;QAC7B,MAAM,WAAW,GAAG,EAAE;aACnB,OAAO,CAAC,yEAAyE,CAAC;aAClF,GAAG,EAAE,CAAC;QAET,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAClC,CAAC;QAED,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE;aACnB,OAAO,CACN;;;;;OAKD,CACA;aACA,GAAG,EAAyC,CAAC;QAEhD,yBAAyB;QACzB,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC;QAEtF,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAa,CAAC;gBACxD,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC;gBACtE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;gBACtC,MAAM,EAAE,CAAC;YACX,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,EAAE,CAAC,IAAI,CAAC;;;KAGP,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,qCAAqC;IACvC,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { ensureMemoryIndexSchema, getSchemaVersion, migrateSchema, ensureColumn, SCHEMA_VERSION, type SchemaStatus, } from "./schema.js";
|
|
2
|
+
export { createDatabase, MemoryStorage, type StorageOptions, type CreateDatabaseResult, } from "./sqlite.js";
|
|
3
|
+
export { loadSqliteVec, isSqliteVecAvailable, createVectorTable, cosineSimilarity, resetSqliteVecState, } from "./sqlite-vec.js";
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAChB,aAAa,EACb,YAAY,EACZ,cAAc,EACd,KAAK,YAAY,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,cAAc,EACd,aAAa,EACb,KAAK,cAAc,EACnB,KAAK,oBAAoB,GAC1B,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { ensureMemoryIndexSchema, getSchemaVersion, migrateSchema, ensureColumn, SCHEMA_VERSION, } from "./schema.js";
|
|
2
|
+
export { createDatabase, MemoryStorage, } from "./sqlite.js";
|
|
3
|
+
export { loadSqliteVec, isSqliteVecAvailable, createVectorTable, cosineSimilarity, resetSqliteVecState, } from "./sqlite-vec.js";
|
|
4
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAChB,aAAa,EACb,YAAY,EACZ,cAAc,GAEf,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,cAAc,EACd,aAAa,GAGd,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
export declare const SCHEMA_VERSION = 2;
|
|
3
|
+
export type SchemaStatus = {
|
|
4
|
+
version: number;
|
|
5
|
+
ftsAvailable: boolean;
|
|
6
|
+
ftsError?: string;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Ensure a column exists in a table, adding it if necessary
|
|
10
|
+
*/
|
|
11
|
+
export declare function ensureColumn(db: Database.Database, table: string, column: string, definition: string): boolean;
|
|
12
|
+
/**
|
|
13
|
+
* Ensure the memory index schema exists in the database
|
|
14
|
+
*/
|
|
15
|
+
export declare function ensureMemoryIndexSchema(db: Database.Database): SchemaStatus;
|
|
16
|
+
/**
|
|
17
|
+
* Get the current schema version from the database
|
|
18
|
+
*/
|
|
19
|
+
export declare function getSchemaVersion(db: Database.Database): number;
|
|
20
|
+
/**
|
|
21
|
+
* Migrate schema from one version to another
|
|
22
|
+
*/
|
|
23
|
+
export declare function migrateSchema(db: Database.Database, fromVersion: number): SchemaStatus;
|
|
24
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/storage/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAE3C,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;GAEG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAaT;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,YAAY,CAmH3E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,MAAM,CAS9D;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,GAAG,YAAY,CAuCtF"}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
export const SCHEMA_VERSION = 2;
|
|
2
|
+
/**
|
|
3
|
+
* Ensure a column exists in a table, adding it if necessary
|
|
4
|
+
*/
|
|
5
|
+
export function ensureColumn(db, table, column, definition) {
|
|
6
|
+
try {
|
|
7
|
+
// Check if column exists using PRAGMA table_info
|
|
8
|
+
const columns = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
9
|
+
const exists = columns.some((col) => col.name === column);
|
|
10
|
+
if (!exists) {
|
|
11
|
+
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
12
|
+
}
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Ensure the memory index schema exists in the database
|
|
21
|
+
*/
|
|
22
|
+
export function ensureMemoryIndexSchema(db) {
|
|
23
|
+
const status = {
|
|
24
|
+
version: SCHEMA_VERSION,
|
|
25
|
+
ftsAvailable: false,
|
|
26
|
+
};
|
|
27
|
+
// meta table - stores configuration and version info
|
|
28
|
+
db.exec(`
|
|
29
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
30
|
+
key TEXT PRIMARY KEY,
|
|
31
|
+
value TEXT NOT NULL
|
|
32
|
+
)
|
|
33
|
+
`);
|
|
34
|
+
// files table - tracks indexed files
|
|
35
|
+
db.exec(`
|
|
36
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
37
|
+
path TEXT PRIMARY KEY,
|
|
38
|
+
source TEXT NOT NULL DEFAULT 'memory',
|
|
39
|
+
hash TEXT NOT NULL,
|
|
40
|
+
mtime INTEGER NOT NULL,
|
|
41
|
+
size INTEGER NOT NULL
|
|
42
|
+
)
|
|
43
|
+
`);
|
|
44
|
+
// Add session-specific columns to files table if needed
|
|
45
|
+
ensureColumn(db, "files", "byte_offset", "INTEGER DEFAULT 0");
|
|
46
|
+
ensureColumn(db, "files", "session_id", "TEXT");
|
|
47
|
+
// chunks table - stores text chunks with embeddings
|
|
48
|
+
db.exec(`
|
|
49
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
50
|
+
id TEXT PRIMARY KEY,
|
|
51
|
+
path TEXT NOT NULL,
|
|
52
|
+
source TEXT NOT NULL DEFAULT 'memory',
|
|
53
|
+
start_line INTEGER NOT NULL,
|
|
54
|
+
end_line INTEGER NOT NULL,
|
|
55
|
+
hash TEXT NOT NULL,
|
|
56
|
+
model TEXT NOT NULL,
|
|
57
|
+
text TEXT NOT NULL,
|
|
58
|
+
embedding TEXT NOT NULL,
|
|
59
|
+
updated_at INTEGER NOT NULL
|
|
60
|
+
)
|
|
61
|
+
`);
|
|
62
|
+
// Add binary embedding column for sqlite-vec support
|
|
63
|
+
ensureColumn(db, "chunks", "embedding_blob", "BLOB");
|
|
64
|
+
// Add session-specific columns to chunks table
|
|
65
|
+
ensureColumn(db, "chunks", "session_id", "TEXT");
|
|
66
|
+
ensureColumn(db, "chunks", "role", "TEXT");
|
|
67
|
+
ensureColumn(db, "chunks", "byte_offset", "INTEGER DEFAULT 0");
|
|
68
|
+
// embedding_cache table - caches embeddings to avoid re-computation
|
|
69
|
+
db.exec(`
|
|
70
|
+
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
71
|
+
provider TEXT NOT NULL,
|
|
72
|
+
model TEXT NOT NULL,
|
|
73
|
+
provider_key TEXT NOT NULL,
|
|
74
|
+
hash TEXT NOT NULL,
|
|
75
|
+
embedding TEXT NOT NULL,
|
|
76
|
+
dims INTEGER,
|
|
77
|
+
updated_at INTEGER NOT NULL,
|
|
78
|
+
PRIMARY KEY (provider, model, provider_key, hash)
|
|
79
|
+
)
|
|
80
|
+
`);
|
|
81
|
+
// Add binary embedding column to cache
|
|
82
|
+
ensureColumn(db, "embedding_cache", "embedding_blob", "BLOB");
|
|
83
|
+
// FTS5 virtual table for full-text search
|
|
84
|
+
try {
|
|
85
|
+
db.exec(`
|
|
86
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
87
|
+
text,
|
|
88
|
+
id UNINDEXED,
|
|
89
|
+
path UNINDEXED,
|
|
90
|
+
source UNINDEXED,
|
|
91
|
+
model UNINDEXED,
|
|
92
|
+
start_line UNINDEXED,
|
|
93
|
+
end_line UNINDEXED
|
|
94
|
+
)
|
|
95
|
+
`);
|
|
96
|
+
status.ftsAvailable = true;
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
status.ftsError = err instanceof Error ? err.message : String(err);
|
|
100
|
+
}
|
|
101
|
+
// Indexes
|
|
102
|
+
db.exec(`
|
|
103
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path)
|
|
104
|
+
`);
|
|
105
|
+
db.exec(`
|
|
106
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source)
|
|
107
|
+
`);
|
|
108
|
+
db.exec(`
|
|
109
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_session ON chunks(session_id)
|
|
110
|
+
`);
|
|
111
|
+
db.exec(`
|
|
112
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON embedding_cache(updated_at)
|
|
113
|
+
`);
|
|
114
|
+
db.exec(`
|
|
115
|
+
CREATE INDEX IF NOT EXISTS idx_files_source ON files(source)
|
|
116
|
+
`);
|
|
117
|
+
// Store schema version
|
|
118
|
+
db.prepare(`
|
|
119
|
+
INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)
|
|
120
|
+
`).run("schema_version", String(SCHEMA_VERSION));
|
|
121
|
+
return status;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Get the current schema version from the database
|
|
125
|
+
*/
|
|
126
|
+
export function getSchemaVersion(db) {
|
|
127
|
+
try {
|
|
128
|
+
const row = db.prepare("SELECT value FROM meta WHERE key = ?").get("schema_version");
|
|
129
|
+
return row ? parseInt(row.value, 10) : 0;
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
return 0;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Migrate schema from one version to another
|
|
137
|
+
*/
|
|
138
|
+
export function migrateSchema(db, fromVersion) {
|
|
139
|
+
const status = {
|
|
140
|
+
version: SCHEMA_VERSION,
|
|
141
|
+
ftsAvailable: false,
|
|
142
|
+
};
|
|
143
|
+
// Version 1 -> 2: Add binary embedding columns and session support
|
|
144
|
+
if (fromVersion < 2) {
|
|
145
|
+
ensureColumn(db, "chunks", "embedding_blob", "BLOB");
|
|
146
|
+
ensureColumn(db, "chunks", "session_id", "TEXT");
|
|
147
|
+
ensureColumn(db, "chunks", "role", "TEXT");
|
|
148
|
+
ensureColumn(db, "chunks", "byte_offset", "INTEGER DEFAULT 0");
|
|
149
|
+
ensureColumn(db, "embedding_cache", "embedding_blob", "BLOB");
|
|
150
|
+
ensureColumn(db, "files", "byte_offset", "INTEGER DEFAULT 0");
|
|
151
|
+
ensureColumn(db, "files", "session_id", "TEXT");
|
|
152
|
+
// Create new indexes
|
|
153
|
+
try {
|
|
154
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_session ON chunks(session_id)`);
|
|
155
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_files_source ON files(source)`);
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
// Indexes might already exist
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// Check FTS availability
|
|
162
|
+
try {
|
|
163
|
+
db.prepare("SELECT 1 FROM chunks_fts LIMIT 0").run();
|
|
164
|
+
status.ftsAvailable = true;
|
|
165
|
+
}
|
|
166
|
+
catch (err) {
|
|
167
|
+
status.ftsError = err instanceof Error ? err.message : String(err);
|
|
168
|
+
}
|
|
169
|
+
// Update schema version
|
|
170
|
+
db.prepare(`
|
|
171
|
+
INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)
|
|
172
|
+
`).run("schema_version", String(SCHEMA_VERSION));
|
|
173
|
+
return status;
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/storage/schema.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAQhC;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,EAAqB,EACrB,KAAa,EACb,MAAc,EACd,UAAkB;IAElB,IAAI,CAAC;QACH,iDAAiD;QACjD,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,GAAG,EAAwB,CAAC;QACtF,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAE1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,EAAE,CAAC,IAAI,CAAC,eAAe,KAAK,eAAe,MAAM,IAAI,UAAU,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,EAAqB;IAC3D,MAAM,MAAM,GAAiB;QAC3B,OAAO,EAAE,cAAc;QACvB,YAAY,EAAE,KAAK;KACpB,CAAC;IAEF,qDAAqD;IACrD,EAAE,CAAC,IAAI,CAAC;;;;;GAKP,CAAC,CAAC;IAEH,qCAAqC;IACrC,EAAE,CAAC,IAAI,CAAC;;;;;;;;GAQP,CAAC,CAAC;IAEH,wDAAwD;IACxD,YAAY,CAAC,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,mBAAmB,CAAC,CAAC;IAC9D,YAAY,CAAC,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAEhD,oDAAoD;IACpD,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;GAaP,CAAC,CAAC;IAEH,qDAAqD;IACrD,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAErD,+CAA+C;IAC/C,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACjD,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,mBAAmB,CAAC,CAAC;IAE/D,oEAAoE;IACpE,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;GAWP,CAAC,CAAC;IAEH,uCAAuC;IACvC,YAAY,CAAC,EAAE,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE9D,0CAA0C;IAC1C,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;KAUP,CAAC,CAAC;QACH,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,UAAU;IACV,EAAE,CAAC,IAAI,CAAC;;GAEP,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC;;GAEP,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC;;GAEP,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC;;GAEP,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC;;GAEP,CAAC,CAAC;IAEH,uBAAuB;IACvB,EAAE,CAAC,OAAO,CAAC;;GAEV,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;IAEjD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAqB;IACpD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAEtE,CAAC;QACd,OAAO,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,EAAqB,EAAE,WAAmB;IACtE,MAAM,MAAM,GAAiB;QAC3B,OAAO,EAAE,cAAc;QACvB,YAAY,EAAE,KAAK;KACpB,CAAC;IAEF,mEAAmE;IACnE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;QACrD,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QACjD,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,mBAAmB,CAAC,CAAC;QAC/D,YAAY,CAAC,EAAE,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAC9D,YAAY,CAAC,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,mBAAmB,CAAC,CAAC;QAC9D,YAAY,CAAC,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAEhD,qBAAqB;QACrB,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;YAC/E,EAAE,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;QAC1E,CAAC;QAAC,MAAM,CAAC;YACP,8BAA8B;QAChC,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAE,CAAC;QACrD,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,wBAAwB;IACxB,EAAE,CAAC,OAAO,CAAC;;GAEV,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;IAEjD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type Database from "better-sqlite3";
|
|
2
|
+
/**
|
|
3
|
+
* Try to load the sqlite-vec extension
|
|
4
|
+
*/
|
|
5
|
+
export declare function loadSqliteVec(db: Database.Database): boolean;
|
|
6
|
+
/**
|
|
7
|
+
* Check if sqlite-vec is available
|
|
8
|
+
*/
|
|
9
|
+
export declare function isSqliteVecAvailable(): boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Create a vec0 virtual table for vector search
|
|
12
|
+
*/
|
|
13
|
+
export declare function createVectorTable(db: Database.Database, tableName: string, dimensions: number): boolean;
|
|
14
|
+
/**
|
|
15
|
+
* Pure JS cosine similarity fallback
|
|
16
|
+
*/
|
|
17
|
+
export declare function cosineSimilarity(a: number[], b: number[]): number;
|
|
18
|
+
/**
|
|
19
|
+
* Reset extension state (for testing)
|
|
20
|
+
*/
|
|
21
|
+
export declare function resetSqliteVecState(): void;
|
|
22
|
+
//# sourceMappingURL=sqlite-vec.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite-vec.d.ts","sourceRoot":"","sources":["../../src/storage/sqlite-vec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAK3C;;GAEG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,OAAO,CAmB5D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,OAAO,CAW9C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAgBT;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAiBjE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,IAAI,CAG1C"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
let sqliteVecLoaded = false;
|
|
2
|
+
let loadError = null;
|
|
3
|
+
/**
|
|
4
|
+
* Try to load the sqlite-vec extension
|
|
5
|
+
*/
|
|
6
|
+
export function loadSqliteVec(db) {
|
|
7
|
+
if (loadError) {
|
|
8
|
+
return false;
|
|
9
|
+
}
|
|
10
|
+
if (sqliteVecLoaded) {
|
|
11
|
+
return true;
|
|
12
|
+
}
|
|
13
|
+
try {
|
|
14
|
+
// Dynamic import of sqlite-vec
|
|
15
|
+
const sqliteVec = require("sqlite-vec");
|
|
16
|
+
sqliteVec.load(db);
|
|
17
|
+
sqliteVecLoaded = true;
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
loadError = err instanceof Error ? err : new Error(String(err));
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Check if sqlite-vec is available
|
|
27
|
+
*/
|
|
28
|
+
export function isSqliteVecAvailable() {
|
|
29
|
+
if (loadError) {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
require("sqlite-vec");
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Create a vec0 virtual table for vector search
|
|
42
|
+
*/
|
|
43
|
+
export function createVectorTable(db, tableName, dimensions) {
|
|
44
|
+
if (!loadSqliteVec(db)) {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
try {
|
|
48
|
+
db.exec(`
|
|
49
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName} USING vec0(
|
|
50
|
+
id TEXT PRIMARY KEY,
|
|
51
|
+
embedding float[${dimensions}]
|
|
52
|
+
)
|
|
53
|
+
`);
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Pure JS cosine similarity fallback
|
|
62
|
+
*/
|
|
63
|
+
export function cosineSimilarity(a, b) {
|
|
64
|
+
if (a.length !== b.length || a.length === 0) {
|
|
65
|
+
return 0;
|
|
66
|
+
}
|
|
67
|
+
let dotProduct = 0;
|
|
68
|
+
let normA = 0;
|
|
69
|
+
let normB = 0;
|
|
70
|
+
for (let i = 0; i < a.length; i++) {
|
|
71
|
+
dotProduct += a[i] * b[i];
|
|
72
|
+
normA += a[i] * a[i];
|
|
73
|
+
normB += b[i] * b[i];
|
|
74
|
+
}
|
|
75
|
+
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
|
|
76
|
+
return magnitude === 0 ? 0 : dotProduct / magnitude;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Reset extension state (for testing)
|
|
80
|
+
*/
|
|
81
|
+
export function resetSqliteVecState() {
|
|
82
|
+
sqliteVecLoaded = false;
|
|
83
|
+
loadError = null;
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=sqlite-vec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite-vec.js","sourceRoot":"","sources":["../../src/storage/sqlite-vec.ts"],"names":[],"mappings":"AAEA,IAAI,eAAe,GAAG,KAAK,CAAC;AAC5B,IAAI,SAAS,GAAiB,IAAI,CAAC;AAEnC;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,EAAqB;IACjD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,+BAA+B;QAC/B,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;QACxC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnB,eAAe,GAAG,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB;IAClC,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,OAAO,CAAC,YAAY,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,EAAqB,EACrB,SAAiB,EACjB,UAAkB;IAElB,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC;2CAC+B,SAAS;;0BAE1B,UAAU;;KAE/B,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB;IACjC,eAAe,GAAG,KAAK,CAAC;IACxB,SAAS,GAAG,IAAI,CAAC;AACnB,CAAC"}
|