local-agent-memory 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +108 -2
- package/dist/cli/commands/hook.d.ts +1 -0
- package/dist/cli/commands/hook.js +66 -0
- package/dist/cli/commands/init.d.ts +5 -0
- package/dist/cli/commands/init.js +158 -0
- package/dist/cli/commands/record.d.ts +3 -0
- package/dist/cli/commands/record.js +58 -0
- package/dist/cli/commands/reindex.d.ts +1 -0
- package/dist/cli/commands/reindex.js +41 -0
- package/dist/cli/commands/status.d.ts +1 -0
- package/dist/cli/commands/status.js +88 -0
- package/dist/cli/commands/sync.d.ts +8 -0
- package/dist/cli/commands/sync.js +134 -0
- package/dist/cli/commands/template.d.ts +1 -0
- package/dist/cli/commands/template.js +116 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +140 -0
- package/dist/cli/mcp.d.ts +2 -0
- package/dist/cli/mcp.js +13 -0
- package/dist/core/embeddings.d.ts +14 -0
- package/dist/core/embeddings.js +127 -0
- package/dist/core/parser.d.ts +25 -0
- package/dist/core/parser.js +151 -0
- package/dist/core/search.d.ts +19 -0
- package/dist/core/search.js +100 -0
- package/dist/core/sync.d.ts +22 -0
- package/dist/core/sync.js +159 -0
- package/dist/db/project.d.ts +12 -0
- package/dist/db/project.js +113 -0
- package/dist/db/registry.d.ts +9 -0
- package/dist/db/registry.js +92 -0
- package/dist/mcp/middleware.d.ts +1 -0
- package/dist/mcp/middleware.js +46 -0
- package/dist/mcp/server.d.ts +2 -0
- package/dist/mcp/server.js +102 -0
- package/dist/mcp/tools.d.ts +2 -0
- package/dist/mcp/tools.js +244 -0
- package/dist/mcp/utils.d.ts +5 -0
- package/dist/mcp/utils.js +82 -0
- package/dist/setup/initGlobal.d.ts +5 -0
- package/dist/setup/initGlobal.js +167 -0
- package/package.json +72 -6
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.countTokens = countTokens;
|
|
7
|
+
exports.parseMarkdown = parseMarkdown;
|
|
8
|
+
const remark_parse_1 = __importDefault(require("remark-parse"));
|
|
9
|
+
const unified_1 = __importDefault(require("unified"));
|
|
10
|
+
/**
|
|
11
|
+
* A very simple token count heuristic (4 chars ~ 1 token)
|
|
12
|
+
* to avoid depending on tiktoken or similar heavy libraries,
|
|
13
|
+
* suitable for MiniLM approximation.
|
|
14
|
+
*/
|
|
15
|
+
function countTokens(text) {
|
|
16
|
+
return Math.ceil(text.length / 4);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Recursively splits a markdown node's text so that each chunk is under maxChars.
|
|
20
|
+
* Preserves code block delimiters (prefix/suffix) in each fragment.
|
|
21
|
+
*/
|
|
22
|
+
function splitNodeTextRecursively(text, startLine, maxChars, prefix = "", suffix = "", breadcrumb = "") {
|
|
23
|
+
const fullLength = prefix.length + text.length + suffix.length;
|
|
24
|
+
// Stop recursion if it fits or if text cannot be reasonably split further
|
|
25
|
+
if (fullLength <= maxChars || text.length <= 1) {
|
|
26
|
+
const lines = text.split("\n").length;
|
|
27
|
+
const fullText = prefix + text + suffix;
|
|
28
|
+
return [
|
|
29
|
+
{
|
|
30
|
+
text: fullText,
|
|
31
|
+
tokens: countTokens(fullText),
|
|
32
|
+
startLine,
|
|
33
|
+
endLine: startLine + lines - 1,
|
|
34
|
+
breadcrumb,
|
|
35
|
+
},
|
|
36
|
+
];
|
|
37
|
+
}
|
|
38
|
+
const half = Math.floor(text.length / 2);
|
|
39
|
+
let splitIndex = text.lastIndexOf("\n", half);
|
|
40
|
+
if (splitIndex <= 0) {
|
|
41
|
+
splitIndex = text.indexOf("\n", half);
|
|
42
|
+
}
|
|
43
|
+
if (splitIndex <= 0) {
|
|
44
|
+
splitIndex = half;
|
|
45
|
+
}
|
|
46
|
+
const leftText = text.slice(0, splitIndex);
|
|
47
|
+
const rightText = text.slice(text[splitIndex] === "\n" ? splitIndex + 1 : splitIndex);
|
|
48
|
+
const leftLinesCount = leftText.split("\n").length - 1;
|
|
49
|
+
const rightStartLine = startLine + leftLinesCount + (text[splitIndex] === "\n" ? 1 : 0);
|
|
50
|
+
return [
|
|
51
|
+
...splitNodeTextRecursively(leftText, startLine, maxChars, prefix, suffix, breadcrumb),
|
|
52
|
+
...splitNodeTextRecursively(rightText, rightStartLine, maxChars, prefix, suffix, breadcrumb),
|
|
53
|
+
];
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Parses markdown content and breaks it down into chunks of text
|
|
57
|
+
* bounded by maxTokens, aiming to preserve logical code blocks and headings.
|
|
58
|
+
* Includes simple breadcrumbs based on headings.
|
|
59
|
+
*
|
|
60
|
+
* @param content The markdown text content.
|
|
61
|
+
* @param path The file path (for metadata).
|
|
62
|
+
* @param maxTokens Maximum number of tokens per chunk.
|
|
63
|
+
* @param overlapTokens Target token overlap between chunks.
|
|
64
|
+
* @returns Array of chunk objects.
|
|
65
|
+
*/
|
|
66
|
+
async function parseMarkdown(content, path, maxTokens = 450, overlapTokens = 50) {
|
|
67
|
+
const processor = (0, unified_1.default)().use(remark_parse_1.default);
|
|
68
|
+
const ast = processor.parse(content);
|
|
69
|
+
const chunks = [];
|
|
70
|
+
let breadcrumbs = [];
|
|
71
|
+
let currentBlocks = [];
|
|
72
|
+
function getBreadcrumbString() {
|
|
73
|
+
return breadcrumbs.map((b) => b.text).join(" > ");
|
|
74
|
+
}
|
|
75
|
+
function pushChunk() {
|
|
76
|
+
if (currentBlocks.length > 0) {
|
|
77
|
+
const first = currentBlocks[0];
|
|
78
|
+
const last = currentBlocks[currentBlocks.length - 1];
|
|
79
|
+
chunks.push({
|
|
80
|
+
path,
|
|
81
|
+
text: currentBlocks.map((b) => b.text).join("\n\n"),
|
|
82
|
+
breadcrumbs: first.breadcrumb,
|
|
83
|
+
startLine: first.startLine,
|
|
84
|
+
endLine: last.endLine,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (ast?.children) {
|
|
89
|
+
for (const node of ast.children) {
|
|
90
|
+
// Update breadcrumbs based on headings
|
|
91
|
+
if (node.type === "heading") {
|
|
92
|
+
const depth = node.depth;
|
|
93
|
+
// Simple extraction of heading text
|
|
94
|
+
const text = content
|
|
95
|
+
.slice(node.position.start.offset, node.position.end.offset)
|
|
96
|
+
.replace(/^#+\s*/, "")
|
|
97
|
+
.trim();
|
|
98
|
+
breadcrumbs = breadcrumbs.filter((b) => b.depth < depth);
|
|
99
|
+
breadcrumbs.push({ depth, text });
|
|
100
|
+
}
|
|
101
|
+
const nodeText = content.slice(node.position.start.offset, node.position.end.offset);
|
|
102
|
+
const nodeTokens = countTokens(nodeText);
|
|
103
|
+
let blocksToAdd = [];
|
|
104
|
+
const MAX_CHARS = 1500;
|
|
105
|
+
if (nodeText.length > MAX_CHARS || nodeTokens > maxTokens) {
|
|
106
|
+
let contentToSplit = nodeText;
|
|
107
|
+
let prefix = "";
|
|
108
|
+
let suffix = "";
|
|
109
|
+
if (node.type === "code" && typeof node.value === "string") {
|
|
110
|
+
contentToSplit = node.value;
|
|
111
|
+
prefix = `\`\`\`${node.lang || ""}\n`;
|
|
112
|
+
suffix = "\n```";
|
|
113
|
+
}
|
|
114
|
+
blocksToAdd = splitNodeTextRecursively(contentToSplit, node.position.start.line, MAX_CHARS, prefix, suffix, getBreadcrumbString());
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
blocksToAdd.push({
|
|
118
|
+
text: nodeText,
|
|
119
|
+
tokens: nodeTokens,
|
|
120
|
+
startLine: node.position.start.line,
|
|
121
|
+
endLine: node.position.end.line,
|
|
122
|
+
breadcrumb: getBreadcrumbString(),
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
for (const block of blocksToAdd) {
|
|
126
|
+
const currentTokens = currentBlocks.reduce((acc, b) => acc + b.tokens, 0);
|
|
127
|
+
if (currentTokens + block.tokens > maxTokens &&
|
|
128
|
+
currentBlocks.length > 0) {
|
|
129
|
+
pushChunk();
|
|
130
|
+
const overlapBlocks = [];
|
|
131
|
+
let overlapTokensCount = 0;
|
|
132
|
+
for (let i = currentBlocks.length - 1; i >= 0; i--) {
|
|
133
|
+
const b = currentBlocks[i];
|
|
134
|
+
if (overlapTokensCount + b.tokens <= overlapTokens) {
|
|
135
|
+
overlapBlocks.unshift(b);
|
|
136
|
+
overlapTokensCount += b.tokens;
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
currentBlocks = overlapBlocks;
|
|
143
|
+
}
|
|
144
|
+
currentBlocks.push(block);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
// Push the last chunk if any
|
|
149
|
+
pushChunk();
|
|
150
|
+
return chunks;
|
|
151
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Database } from "sqlite3";
|
|
2
|
+
import type { Chunk } from "./parser";
|
|
3
|
+
export interface SearchResult extends Chunk {
|
|
4
|
+
id: number;
|
|
5
|
+
score: number;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Sanitizes a search query for safe use in SQLite FTS5 MATCH clause.
|
|
9
|
+
* Wrapping each term in double quotes avoids FTS5 syntax errors with reserved keywords
|
|
10
|
+
* or punctuation (like *, AND, OR, NEAR).
|
|
11
|
+
*/
|
|
12
|
+
export declare function sanitizeFtsQuery(query: string): string;
|
|
13
|
+
/**
|
|
14
|
+
* Performs a hybrid search combining FTS5 lexical search and sqlite-vec vector search.
|
|
15
|
+
* Uses Reciprocal Rank Fusion (RRF) to combine results.
|
|
16
|
+
*
|
|
17
|
+
* RRF Score = 1 / (k + rank) where k is typically 60.
|
|
18
|
+
*/
|
|
19
|
+
export declare function hybridSearch(query: string, limit: number, db: Database, filter?: string): Promise<SearchResult[]>;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.sanitizeFtsQuery = sanitizeFtsQuery;
|
|
4
|
+
exports.hybridSearch = hybridSearch;
|
|
5
|
+
const embeddings_1 = require("./embeddings");
|
|
6
|
+
const sync_1 = require("./sync");
|
|
7
|
+
/**
|
|
8
|
+
* Sanitizes a search query for safe use in SQLite FTS5 MATCH clause.
|
|
9
|
+
* Wrapping each term in double quotes avoids FTS5 syntax errors with reserved keywords
|
|
10
|
+
* or punctuation (like *, AND, OR, NEAR).
|
|
11
|
+
*/
|
|
12
|
+
function sanitizeFtsQuery(query) {
|
|
13
|
+
if (!query)
|
|
14
|
+
return "";
|
|
15
|
+
const terms = query.split(/\s+/).filter((t) => t.trim().length > 0);
|
|
16
|
+
if (terms.length === 0)
|
|
17
|
+
return "";
|
|
18
|
+
return terms.map((t) => `"${t.replace(/"/g, '""')}"`).join(" ");
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Performs a hybrid search combining FTS5 lexical search and sqlite-vec vector search.
|
|
22
|
+
* Uses Reciprocal Rank Fusion (RRF) to combine results.
|
|
23
|
+
*
|
|
24
|
+
* RRF Score = 1 / (k + rank) where k is typically 60.
|
|
25
|
+
*/
|
|
26
|
+
async function hybridSearch(query, limit, db, filter) {
|
|
27
|
+
const asyncDb = new sync_1.AsyncDatabase(db);
|
|
28
|
+
const RRF_K = 60;
|
|
29
|
+
// 1. FTS5 Search
|
|
30
|
+
const ftsResults = await new Promise((resolve, reject) => {
|
|
31
|
+
// Sanitize query to prevent FTS5 syntax errors
|
|
32
|
+
const ftsQuery = sanitizeFtsQuery(query);
|
|
33
|
+
if (!ftsQuery) {
|
|
34
|
+
return resolve([]);
|
|
35
|
+
}
|
|
36
|
+
// using bm25() for scoring in FTS5
|
|
37
|
+
db.all(`SELECT rowid as id, rank as score FROM chunks_fts WHERE chunks_fts MATCH ? ORDER BY rank LIMIT 100`, [ftsQuery], (err, rows) => {
|
|
38
|
+
if (err)
|
|
39
|
+
reject(err);
|
|
40
|
+
else
|
|
41
|
+
resolve(rows);
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
// 2. Vector Search
|
|
45
|
+
const queryEmbedding = await (0, embeddings_1.generateEmbedding)(query);
|
|
46
|
+
const vectorData = Buffer.from(new Float32Array(queryEmbedding).buffer);
|
|
47
|
+
const vecResults = await new Promise((resolve, reject) => {
|
|
48
|
+
// sqlite-vec distance function. distance is lower = better (closer)
|
|
49
|
+
// we want smallest distance first
|
|
50
|
+
db.all(`SELECT rowid as id, distance FROM chunks_vec
|
|
51
|
+
WHERE embedding MATCH ? AND k = 100
|
|
52
|
+
ORDER BY distance`, [vectorData], (err, rows) => {
|
|
53
|
+
if (err)
|
|
54
|
+
reject(err);
|
|
55
|
+
else
|
|
56
|
+
resolve(rows);
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
// 3. RRF Fusion
|
|
60
|
+
const rrfScores = new Map();
|
|
61
|
+
// Add FTS scores
|
|
62
|
+
ftsResults.forEach((res, index) => {
|
|
63
|
+
const rank = index + 1;
|
|
64
|
+
const currentScore = rrfScores.get(res.id) || 0;
|
|
65
|
+
rrfScores.set(res.id, currentScore + 1 / (RRF_K + rank));
|
|
66
|
+
});
|
|
67
|
+
// Add Vector scores
|
|
68
|
+
vecResults.forEach((res, index) => {
|
|
69
|
+
const rank = index + 1;
|
|
70
|
+
const currentScore = rrfScores.get(res.id) || 0;
|
|
71
|
+
rrfScores.set(res.id, currentScore + 1 / (RRF_K + rank));
|
|
72
|
+
});
|
|
73
|
+
// Sort by RRF score descending
|
|
74
|
+
const sortedIds = Array.from(rrfScores.entries())
|
|
75
|
+
.sort((a, b) => b[1] - a[1])
|
|
76
|
+
.slice(0, limit);
|
|
77
|
+
// Fetch actual chunk data
|
|
78
|
+
const finalResults = [];
|
|
79
|
+
for (const [id, score] of sortedIds) {
|
|
80
|
+
let row;
|
|
81
|
+
if (filter) {
|
|
82
|
+
row = await asyncDb.get("SELECT * FROM chunks WHERE id = ? AND path LIKE ?", [id, `%${filter}%`]);
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
row = await asyncDb.get("SELECT * FROM chunks WHERE id = ?", [id]);
|
|
86
|
+
}
|
|
87
|
+
if (row) {
|
|
88
|
+
finalResults.push({
|
|
89
|
+
id: row.id,
|
|
90
|
+
path: row.path,
|
|
91
|
+
text: row.text,
|
|
92
|
+
breadcrumbs: row.breadcrumbs,
|
|
93
|
+
startLine: row.startLine,
|
|
94
|
+
endLine: row.endLine,
|
|
95
|
+
score,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return finalResults;
|
|
100
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Database } from "sqlite3";
|
|
2
|
+
/**
|
|
3
|
+
* Interface to interact with the DB properly using Promises
|
|
4
|
+
*/
|
|
5
|
+
export declare class AsyncDatabase {
|
|
6
|
+
db: Database;
|
|
7
|
+
constructor(db: Database);
|
|
8
|
+
get(sql: string, params?: any[]): Promise<any>;
|
|
9
|
+
run(sql: string, params?: any[]): Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Synchronizes a single markdown file into the database.
|
|
13
|
+
* Uses incremental sync logic based on SHA256 hashes.
|
|
14
|
+
* If the file changed, processes chunks, generates embeddings,
|
|
15
|
+
* and updates DB in a transaction.
|
|
16
|
+
*/
|
|
17
|
+
export declare function syncFile(path: string, content: string, db: Database): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Removes a file and all its corresponding chunks, vectors, and FTS entries
|
|
20
|
+
* from the database atomically. Also removes the file_manifest entry.
|
|
21
|
+
*/
|
|
22
|
+
export declare function removeFile(path: string, db: Database): Promise<void>;
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.AsyncDatabase = void 0;
|
|
37
|
+
exports.syncFile = syncFile;
|
|
38
|
+
exports.removeFile = removeFile;
|
|
39
|
+
const crypto = __importStar(require("node:crypto"));
|
|
40
|
+
const embeddings_1 = require("./embeddings");
|
|
41
|
+
const parser_1 = require("./parser");
|
|
42
|
+
function getHash(content) {
|
|
43
|
+
return crypto.createHash("sha256").update(content).digest("hex");
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Interface to interact with the DB properly using Promises
|
|
47
|
+
*/
|
|
48
|
+
class AsyncDatabase {
|
|
49
|
+
db;
|
|
50
|
+
constructor(db) {
|
|
51
|
+
this.db = db;
|
|
52
|
+
}
|
|
53
|
+
get(sql, params = []) {
|
|
54
|
+
return new Promise((resolve, reject) => {
|
|
55
|
+
this.db.get(sql, params, (err, row) => {
|
|
56
|
+
if (err)
|
|
57
|
+
reject(err);
|
|
58
|
+
else
|
|
59
|
+
resolve(row);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
run(sql, params = []) {
|
|
64
|
+
return new Promise((resolve, reject) => {
|
|
65
|
+
this.db.run(sql, params, (err) => {
|
|
66
|
+
if (err)
|
|
67
|
+
reject(err);
|
|
68
|
+
else
|
|
69
|
+
resolve();
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
exports.AsyncDatabase = AsyncDatabase;
|
|
75
|
+
/**
|
|
76
|
+
* Synchronizes a single markdown file into the database.
|
|
77
|
+
* Uses incremental sync logic based on SHA256 hashes.
|
|
78
|
+
* If the file changed, processes chunks, generates embeddings,
|
|
79
|
+
* and updates DB in a transaction.
|
|
80
|
+
*/
|
|
81
|
+
async function syncFile(path, content, db) {
|
|
82
|
+
const asyncDb = new AsyncDatabase(db);
|
|
83
|
+
const hash = getHash(content);
|
|
84
|
+
try {
|
|
85
|
+
// We expect the table files_metadata to exist
|
|
86
|
+
// files_metadata (path TEXT PRIMARY KEY, hash TEXT)
|
|
87
|
+
// We expect chunks to exist (id INTEGER PRIMARY KEY, path TEXT, text TEXT, breadcrumbs TEXT, startLine INTEGER, endLine INTEGER)
|
|
88
|
+
// We expect chunks_fts to exist (FTS5)
|
|
89
|
+
// We expect chunks_vec to exist (sqlite-vec)
|
|
90
|
+
const existingFile = await asyncDb.get("SELECT hash FROM files_metadata WHERE path = ?", [path]);
|
|
91
|
+
if (existingFile && existingFile.hash === hash) {
|
|
92
|
+
// File hasn't changed, skip
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
// Begin transaction
|
|
96
|
+
await asyncDb.run("BEGIN TRANSACTION");
|
|
97
|
+
// Remove existing chunks for this path if any
|
|
98
|
+
await asyncDb.run("DELETE FROM chunks_vec WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)", [path]);
|
|
99
|
+
await asyncDb.run("DELETE FROM chunks_fts WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)", [path]);
|
|
100
|
+
await asyncDb.run("DELETE FROM chunks WHERE path = ?", [path]);
|
|
101
|
+
await asyncDb.run("DELETE FROM files_metadata WHERE path = ?", [path]);
|
|
102
|
+
// Insert new hash
|
|
103
|
+
await asyncDb.run("INSERT INTO files_metadata (path, hash) VALUES (?, ?)", [
|
|
104
|
+
path,
|
|
105
|
+
hash,
|
|
106
|
+
]);
|
|
107
|
+
const chunks = await (0, parser_1.parseMarkdown)(content, path);
|
|
108
|
+
const BATCH_SIZE = 8;
|
|
109
|
+
for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
|
|
110
|
+
const batch = chunks.slice(i, i + BATCH_SIZE);
|
|
111
|
+
// Process embeddings in batches of BATCH_SIZE to avoid blocking Event Loop too long
|
|
112
|
+
const embeddings = await Promise.all(batch.map((chunk) => (0, embeddings_1.generateEmbedding)(chunk.text)));
|
|
113
|
+
for (let j = 0; j < batch.length; j++) {
|
|
114
|
+
const chunk = batch[j];
|
|
115
|
+
const embedding = embeddings[j];
|
|
116
|
+
// Store in main table
|
|
117
|
+
await asyncDb.run("INSERT INTO chunks (path, text, breadcrumbs, startLine, endLine) VALUES (?, ?, ?, ?, ?)", [
|
|
118
|
+
chunk.path,
|
|
119
|
+
chunk.text,
|
|
120
|
+
chunk.breadcrumbs,
|
|
121
|
+
chunk.startLine,
|
|
122
|
+
chunk.endLine,
|
|
123
|
+
]);
|
|
124
|
+
const row = await asyncDb.get("SELECT last_insert_rowid() as id");
|
|
125
|
+
const chunkId = row.id;
|
|
126
|
+
// Ensure FTS5 sync
|
|
127
|
+
await asyncDb.run("INSERT INTO chunks_fts (rowid, path, text, breadcrumbs) VALUES (?, ?, ?, ?)", [chunkId, chunk.path, chunk.text, chunk.breadcrumbs]);
|
|
128
|
+
// Insert vector
|
|
129
|
+
const vectorData = Buffer.from(new Float32Array(embedding).buffer);
|
|
130
|
+
await asyncDb.run("INSERT INTO chunks_vec (rowid, embedding) VALUES (?, ?)", [chunkId, vectorData]);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
await asyncDb.run("COMMIT");
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
await asyncDb.run("ROLLBACK");
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Removes a file and all its corresponding chunks, vectors, and FTS entries
|
|
142
|
+
* from the database atomically. Also removes the file_manifest entry.
|
|
143
|
+
*/
|
|
144
|
+
async function removeFile(path, db) {
|
|
145
|
+
const asyncDb = new AsyncDatabase(db);
|
|
146
|
+
await asyncDb.run("BEGIN TRANSACTION");
|
|
147
|
+
try {
|
|
148
|
+
await asyncDb.run("DELETE FROM chunks_vec WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)", [path]);
|
|
149
|
+
await asyncDb.run("DELETE FROM chunks_fts WHERE rowid IN (SELECT id FROM chunks WHERE path = ?)", [path]);
|
|
150
|
+
await asyncDb.run("DELETE FROM chunks WHERE path = ?", [path]);
|
|
151
|
+
await asyncDb.run("DELETE FROM files_metadata WHERE path = ?", [path]);
|
|
152
|
+
await asyncDb.run("DELETE FROM file_manifest WHERE file_path = ?", [path]);
|
|
153
|
+
await asyncDb.run("COMMIT");
|
|
154
|
+
}
|
|
155
|
+
catch (err) {
|
|
156
|
+
await asyncDb.run("ROLLBACK");
|
|
157
|
+
throw err;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import sqlite3 from "sqlite3";
|
|
2
|
+
export interface FileManifestRow {
|
|
3
|
+
file_path: string;
|
|
4
|
+
mtime: number;
|
|
5
|
+
sha256: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function getLocalDbPath(projectRoot: string): string;
|
|
8
|
+
export declare function initLocalDb(dbPath: string): sqlite3.Database;
|
|
9
|
+
export declare function upsertFileManifest(dbPath: string, file_path: string, mtime: number, sha256: string): Promise<void>;
|
|
10
|
+
export declare function getFileManifest(dbPath: string, file_path: string): Promise<FileManifestRow | null>;
|
|
11
|
+
export declare function getAllManifests(dbPath: string): Promise<FileManifestRow[]>;
|
|
12
|
+
export declare function removeFileManifest(dbPath: string, file_path: string): Promise<void>;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.getLocalDbPath = getLocalDbPath;
|
|
7
|
+
exports.initLocalDb = initLocalDb;
|
|
8
|
+
exports.upsertFileManifest = upsertFileManifest;
|
|
9
|
+
exports.getFileManifest = getFileManifest;
|
|
10
|
+
exports.getAllManifests = getAllManifests;
|
|
11
|
+
exports.removeFileManifest = removeFileManifest;
|
|
12
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
13
|
+
const sqlite3_1 = __importDefault(require("sqlite3"));
|
|
14
|
+
function getLocalDbPath(projectRoot) {
|
|
15
|
+
return node_path_1.default.join(projectRoot, ".memory", "local.sqlite");
|
|
16
|
+
}
|
|
17
|
+
function initLocalDb(dbPath) {
|
|
18
|
+
const db = new sqlite3_1.default.Database(dbPath);
|
|
19
|
+
db.configure("busyTimeout", 5000);
|
|
20
|
+
db.serialize(() => {
|
|
21
|
+
db.run("PRAGMA journal_mode = WAL;");
|
|
22
|
+
db.run("PRAGMA busy_timeout = 5000;");
|
|
23
|
+
db.run(`
|
|
24
|
+
CREATE TABLE IF NOT EXISTS file_manifest (
|
|
25
|
+
file_path TEXT PRIMARY KEY,
|
|
26
|
+
mtime INTEGER NOT NULL,
|
|
27
|
+
sha256 TEXT NOT NULL,
|
|
28
|
+
last_synced DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
29
|
+
)
|
|
30
|
+
`);
|
|
31
|
+
db.run(`
|
|
32
|
+
CREATE TABLE IF NOT EXISTS files_metadata (
|
|
33
|
+
path TEXT PRIMARY KEY,
|
|
34
|
+
hash TEXT NOT NULL
|
|
35
|
+
)
|
|
36
|
+
`);
|
|
37
|
+
db.run(`
|
|
38
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
39
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
40
|
+
path TEXT NOT NULL,
|
|
41
|
+
text TEXT NOT NULL,
|
|
42
|
+
breadcrumbs TEXT,
|
|
43
|
+
startLine INTEGER,
|
|
44
|
+
endLine INTEGER
|
|
45
|
+
)
|
|
46
|
+
`);
|
|
47
|
+
// Note: for real usage, chunks_fts and chunks_vec need FTS5 and vec0 extensions enabled.
|
|
48
|
+
// Standard sqlite3 node driver has FTS5, but vec0 must be loaded.
|
|
49
|
+
// Assuming they are available or using regular tables for E2E.
|
|
50
|
+
db.run(`
|
|
51
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
52
|
+
path, text, breadcrumbs, content='chunks', content_rowid='id'
|
|
53
|
+
)
|
|
54
|
+
`);
|
|
55
|
+
// Create a regular table for chunks_vec to prevent crashes in tests or environments
|
|
56
|
+
// where the sqlite-vec extension is not loaded yet.
|
|
57
|
+
db.run(`
|
|
58
|
+
CREATE TABLE IF NOT EXISTS chunks_vec (
|
|
59
|
+
rowid INTEGER,
|
|
60
|
+
embedding BLOB
|
|
61
|
+
)
|
|
62
|
+
`);
|
|
63
|
+
});
|
|
64
|
+
return db;
|
|
65
|
+
}
|
|
66
|
+
async function upsertFileManifest(dbPath, file_path, mtime, sha256) {
|
|
67
|
+
return new Promise((resolve, reject) => {
|
|
68
|
+
const db = initLocalDb(dbPath);
|
|
69
|
+
db.run("INSERT OR REPLACE INTO file_manifest (file_path, mtime, sha256, last_synced) VALUES (?, ?, ?, CURRENT_TIMESTAMP)", [file_path, mtime, sha256], (err) => {
|
|
70
|
+
db.close();
|
|
71
|
+
if (err)
|
|
72
|
+
reject(err);
|
|
73
|
+
else
|
|
74
|
+
resolve();
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
async function getFileManifest(dbPath, file_path) {
|
|
79
|
+
return new Promise((resolve, reject) => {
|
|
80
|
+
const db = initLocalDb(dbPath);
|
|
81
|
+
db.get("SELECT file_path, mtime, sha256 FROM file_manifest WHERE file_path = ?", [file_path], (err, row) => {
|
|
82
|
+
db.close();
|
|
83
|
+
if (err)
|
|
84
|
+
reject(err);
|
|
85
|
+
else
|
|
86
|
+
resolve(row || null);
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
async function getAllManifests(dbPath) {
|
|
91
|
+
return new Promise((resolve, reject) => {
|
|
92
|
+
const db = initLocalDb(dbPath);
|
|
93
|
+
db.all("SELECT file_path, mtime, sha256 FROM file_manifest", (err, rows) => {
|
|
94
|
+
db.close();
|
|
95
|
+
if (err)
|
|
96
|
+
reject(err);
|
|
97
|
+
else
|
|
98
|
+
resolve(rows || []);
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
async function removeFileManifest(dbPath, file_path) {
|
|
103
|
+
return new Promise((resolve, reject) => {
|
|
104
|
+
const db = initLocalDb(dbPath);
|
|
105
|
+
db.run("DELETE FROM file_manifest WHERE file_path = ?", [file_path], (err) => {
|
|
106
|
+
db.close();
|
|
107
|
+
if (err)
|
|
108
|
+
reject(err);
|
|
109
|
+
else
|
|
110
|
+
resolve();
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import sqlite3 from "sqlite3";
|
|
2
|
+
export declare function getRegistryDbPath(): string;
|
|
3
|
+
export declare function initRegistryDb(dbPath?: string): sqlite3.Database;
|
|
4
|
+
export declare function registerProject(projectId: string, projectPath: string, dbPath?: string): Promise<void>;
|
|
5
|
+
export declare function getProjectByPath(projectPath: string, dbPath?: string): Promise<string | null>;
|
|
6
|
+
export declare function getProjectRootFromRegistry(activePath: string, dbPath?: string): Promise<{
|
|
7
|
+
path: string;
|
|
8
|
+
fallback: boolean;
|
|
9
|
+
} | null>;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.getRegistryDbPath = getRegistryDbPath;
|
|
7
|
+
exports.initRegistryDb = initRegistryDb;
|
|
8
|
+
exports.registerProject = registerProject;
|
|
9
|
+
exports.getProjectByPath = getProjectByPath;
|
|
10
|
+
exports.getProjectRootFromRegistry = getProjectRootFromRegistry;
|
|
11
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
12
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
13
|
+
const sqlite3_1 = __importDefault(require("sqlite3"));
|
|
14
|
+
const initGlobal_1 = require("../setup/initGlobal");
|
|
15
|
+
function getRegistryDbPath() {
|
|
16
|
+
const memDir = (0, initGlobal_1.getLocalMemoryDir)();
|
|
17
|
+
if (!node_fs_1.default.existsSync(memDir)) {
|
|
18
|
+
node_fs_1.default.mkdirSync(memDir, { recursive: true });
|
|
19
|
+
}
|
|
20
|
+
return node_path_1.default.join(memDir, "registry.sqlite");
|
|
21
|
+
}
|
|
22
|
+
function initRegistryDb(dbPath = getRegistryDbPath()) {
|
|
23
|
+
const db = new sqlite3_1.default.Database(dbPath);
|
|
24
|
+
db.configure("busyTimeout", 5000);
|
|
25
|
+
db.serialize(() => {
|
|
26
|
+
db.run("PRAGMA journal_mode = WAL;");
|
|
27
|
+
db.run("PRAGMA busy_timeout = 5000;");
|
|
28
|
+
db.run(`
|
|
29
|
+
CREATE TABLE IF NOT EXISTS projects (
|
|
30
|
+
project_id TEXT PRIMARY KEY,
|
|
31
|
+
file_path TEXT NOT NULL UNIQUE,
|
|
32
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
33
|
+
)
|
|
34
|
+
`);
|
|
35
|
+
});
|
|
36
|
+
return db;
|
|
37
|
+
}
|
|
38
|
+
async function registerProject(projectId, projectPath, dbPath = getRegistryDbPath()) {
|
|
39
|
+
return new Promise((resolve, reject) => {
|
|
40
|
+
const db = initRegistryDb(dbPath);
|
|
41
|
+
db.run("INSERT OR REPLACE INTO projects (project_id, file_path) VALUES (?, ?)", [projectId, projectPath], (err) => {
|
|
42
|
+
db.close();
|
|
43
|
+
if (err)
|
|
44
|
+
reject(err);
|
|
45
|
+
else
|
|
46
|
+
resolve();
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
async function getProjectByPath(projectPath, dbPath = getRegistryDbPath()) {
|
|
51
|
+
return new Promise((resolve, reject) => {
|
|
52
|
+
const db = initRegistryDb(dbPath);
|
|
53
|
+
db.get("SELECT project_id FROM projects WHERE file_path = ?", [projectPath], (err, row) => {
|
|
54
|
+
db.close();
|
|
55
|
+
if (err)
|
|
56
|
+
reject(err);
|
|
57
|
+
else
|
|
58
|
+
resolve(row ? row.project_id : null);
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
async function getProjectRootFromRegistry(activePath, dbPath = getRegistryDbPath()) {
|
|
63
|
+
return new Promise((resolve, reject) => {
|
|
64
|
+
const db = initRegistryDb(dbPath);
|
|
65
|
+
db.all("SELECT file_path FROM projects ORDER BY created_at DESC", [], (err, rows) => {
|
|
66
|
+
db.close();
|
|
67
|
+
if (err) {
|
|
68
|
+
return reject(err);
|
|
69
|
+
}
|
|
70
|
+
let bestMatch = null;
|
|
71
|
+
let maxLen = -1;
|
|
72
|
+
for (const row of rows) {
|
|
73
|
+
const p = row.file_path;
|
|
74
|
+
if (activePath.startsWith(p) || activePath === p) {
|
|
75
|
+
if (p.length > maxLen) {
|
|
76
|
+
maxLen = p.length;
|
|
77
|
+
bestMatch = p;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
if (bestMatch) {
|
|
82
|
+
resolve({ path: bestMatch, fallback: false });
|
|
83
|
+
}
|
|
84
|
+
else if (rows.length > 0) {
|
|
85
|
+
resolve({ path: rows[0].file_path, fallback: true });
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
resolve(null);
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function syncMiddleware(cwd?: string): Promise<void>;
|