@appland/search 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/built/build-file-index.js +24 -45
- package/built/cli.js +11 -7
- package/built/file-index.d.ts +19 -22
- package/built/file-index.js +149 -61
- package/built/snippet-index.d.ts +1 -1
- package/built/snippet-index.js +10 -7
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
@@ -1,3 +1,25 @@
|
|
1
|
+
# [@appland/search-v1.2.0](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.1.3...@appland/search-v1.2.0) (2025-02-27)
|
2
|
+
|
3
|
+
|
4
|
+
### Bug Fixes
|
5
|
+
|
6
|
+
* Finalize insert and search operations in FileIndex close method ([19fe2e8](https://github.com/getappmap/appmap-js/commit/19fe2e8574cf11e6619629331d7deb7fbdcb11c5))
|
7
|
+
* Fix file index scoring priorities ([9c798bc](https://github.com/getappmap/appmap-js/commit/9c798bca5c943e9e344b838314f9e0739082c4d9))
|
8
|
+
* Migrate from better-sqlite3 to node-sqlite3-wasm for improved compatibility ([86e0fe5](https://github.com/getappmap/appmap-js/commit/86e0fe5386286816473c0b16a91f7fa80f8706af))
|
9
|
+
|
10
|
+
|
11
|
+
### Features
|
12
|
+
|
13
|
+
* Add performance measurement to search CLI ([1fc4ef3](https://github.com/getappmap/appmap-js/commit/1fc4ef331256a861c6de3e310cbdd70b7a9aa41c))
|
14
|
+
* Cache the file index ([fa465d2](https://github.com/getappmap/appmap-js/commit/fa465d244688da939c86444ba4652feff207f378))
|
15
|
+
|
16
|
+
# [@appland/search-v1.1.3](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.1.2...@appland/search-v1.1.3) (2025-02-05)
|
17
|
+
|
18
|
+
|
19
|
+
### Bug Fixes
|
20
|
+
|
21
|
+
* Snippet paths are URI encoded ([088ac7e](https://github.com/getappmap/appmap-js/commit/088ac7eb22dceadd320ae1a162ee8d7290f88b9b))
|
22
|
+
|
1
23
|
# [@appland/search-v1.1.2](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.1.1...@appland/search-v1.1.2) (2025-01-23)
|
2
24
|
|
3
25
|
|
@@ -6,55 +6,34 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.default = buildFileIndex;
|
7
7
|
const debug_1 = __importDefault(require("debug"));
|
8
8
|
const path_1 = require("path");
|
9
|
-
const console_1 = require("console");
|
10
|
-
const types_1 = require("util/types");
|
11
9
|
const debug = (0, debug_1.default)('appmap:search:build-index');
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
debug('Tokenized file: %s', filePath);
|
23
|
-
context.fileIndex.indexFile(context.baseDirectory, filePath, symbols, words);
|
24
|
-
debug('Wrote file to index: %s', filePath);
|
10
|
+
function fileReader(contentReader, tokenizer) {
|
11
|
+
return async (filePath) => {
|
12
|
+
debug('Indexing file: %s', filePath);
|
13
|
+
const fileContents = await contentReader(filePath);
|
14
|
+
if (!fileContents)
|
15
|
+
return;
|
16
|
+
debug('Read file: %s, length: %d (%s...)', filePath, fileContents.length, fileContents.slice(0, 40));
|
17
|
+
const fileExtension = filePath.split('.').pop() ?? '';
|
18
|
+
return await tokenizer(fileContents, fileExtension);
|
19
|
+
};
|
25
20
|
}
|
26
|
-
async function
|
27
|
-
const
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
await indexFile(context, filePath);
|
40
|
-
}
|
41
|
-
catch (e) {
|
42
|
-
const message = (0, types_1.isNativeError)(e) ? e.message : String(e);
|
43
|
-
(0, console_1.warn)(`Error indexing file ${filePath}: ${message}`);
|
44
|
-
}
|
21
|
+
async function* listFiles(directories, fileFilter, listDirectory) {
|
22
|
+
for (const directory of directories) {
|
23
|
+
const dirContents = await listDirectory(directory);
|
24
|
+
if (!dirContents)
|
25
|
+
continue;
|
26
|
+
for (const dirContentItem of dirContents) {
|
27
|
+
let filePath;
|
28
|
+
if ((0, path_1.isAbsolute)(dirContentItem))
|
29
|
+
filePath = dirContentItem;
|
30
|
+
else
|
31
|
+
filePath = (0, path_1.join)(directory, dirContentItem);
|
32
|
+
if (await fileFilter(filePath))
|
33
|
+
yield { directory, filePath };
|
45
34
|
}
|
46
35
|
}
|
47
36
|
}
|
48
37
|
async function buildFileIndex(fileIndex, directories, listDirectory, fileFilter, contentReader, tokenizer) {
|
49
|
-
|
50
|
-
const context = {
|
51
|
-
fileIndex,
|
52
|
-
baseDirectory: directory,
|
53
|
-
listDirectory,
|
54
|
-
fileFilter,
|
55
|
-
contentReader,
|
56
|
-
tokenizer,
|
57
|
-
};
|
58
|
-
await indexDirectory(context, directory);
|
59
|
-
}
|
38
|
+
await fileIndex.index(listFiles(directories, fileFilter, listDirectory), fileReader(contentReader, tokenizer));
|
60
39
|
}
|
package/built/cli.js
CHANGED
@@ -5,8 +5,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
6
6
|
const yargs_1 = __importDefault(require("yargs"));
|
7
7
|
const helpers_1 = require("yargs/helpers");
|
8
|
-
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
9
8
|
const debug_1 = __importDefault(require("debug"));
|
9
|
+
const node_sqlite3_wasm_1 = __importDefault(require("node-sqlite3-wasm"));
|
10
10
|
const tokenize_1 = require("./tokenize");
|
11
11
|
const file_index_1 = __importDefault(require("./file-index"));
|
12
12
|
const build_file_index_1 = __importDefault(require("./build-file-index"));
|
@@ -24,7 +24,8 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
24
24
|
return yargs
|
25
25
|
.option('directories', {
|
26
26
|
alias: 'd',
|
27
|
-
type: '
|
27
|
+
type: 'string',
|
28
|
+
array: true,
|
28
29
|
description: 'List of directories to index',
|
29
30
|
default: ['.'],
|
30
31
|
})
|
@@ -35,9 +36,11 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
35
36
|
.positional('query', {
|
36
37
|
describe: 'Search query',
|
37
38
|
type: 'string',
|
39
|
+
demandOption: true,
|
38
40
|
})
|
39
41
|
.strict();
|
40
42
|
}, async (argv) => {
|
43
|
+
new PerformanceObserver((entries) => entries.getEntries().forEach((e) => console.warn(`${e.name}: ${e.duration.toFixed(0)} ms`))).observe({ entryTypes: ['measure'] });
|
41
44
|
const { directories, query } = argv;
|
42
45
|
let filterRE;
|
43
46
|
if (argv.fileFilter)
|
@@ -57,10 +60,11 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
57
60
|
return true;
|
58
61
|
return !filterRE.test(path);
|
59
62
|
};
|
60
|
-
const
|
61
|
-
const fileIndex = new file_index_1.default(db);
|
63
|
+
const fileIndex = await file_index_1.default.cached('file', ...directories);
|
62
64
|
const sessionId = (0, session_id_1.generateSessionId)();
|
65
|
+
performance.mark('start indexing');
|
63
66
|
await (0, build_file_index_1.default)(fileIndex, directories, project_files_1.default, fileFilter, ioutil_1.readFileSafe, tokenize_1.fileTokens);
|
67
|
+
performance.measure('indexing', 'start indexing');
|
64
68
|
const filePathAtMostThreeEntries = (filePath) => {
|
65
69
|
const parts = filePath.split('/');
|
66
70
|
if (parts.length <= 3)
|
@@ -70,13 +74,13 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
70
74
|
const printResult = (type, id, score) => console.log('%s %s %s', type, filePathAtMostThreeEntries(id), score.toPrecision(3));
|
71
75
|
console.log('File search results');
|
72
76
|
console.log('-------------------');
|
73
|
-
const fileSearchResults = fileIndex.search(
|
77
|
+
const fileSearchResults = fileIndex.search(query);
|
74
78
|
for (const result of fileSearchResults) {
|
75
79
|
const { filePath, score } = result;
|
76
80
|
printResult('file', filePath, score);
|
77
81
|
}
|
78
82
|
const splitter = splitter_1.langchainSplitter;
|
79
|
-
const snippetIndex = new snippet_index_1.default(
|
83
|
+
const snippetIndex = new snippet_index_1.default(new node_sqlite3_wasm_1.default.Database());
|
80
84
|
await (0, build_snippet_index_1.default)(snippetIndex, fileSearchResults, ioutil_1.readFileSafe, splitter, tokenize_1.fileTokens);
|
81
85
|
console.log('');
|
82
86
|
console.log('Snippet search results');
|
@@ -98,7 +102,7 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
98
102
|
const lines = content.split('\n').slice(startLine - 1, endLine);
|
99
103
|
console.log(lines.map((l) => ` > ${l}`).join('\n'));
|
100
104
|
}
|
101
|
-
|
105
|
+
fileIndex.close();
|
102
106
|
})
|
103
107
|
.help().argv;
|
104
108
|
if (cli instanceof Promise) {
|
package/built/file-index.d.ts
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
import sqlite3 from '
|
2
|
-
import { SessionId } from './session-id';
|
1
|
+
import sqlite3 from 'node-sqlite3-wasm';
|
3
2
|
export type FileSearchResult = {
|
4
3
|
directory: string;
|
5
4
|
filePath: string;
|
@@ -11,38 +10,36 @@ export type FileSearchResult = {
|
|
11
10
|
* The primary responsibilities of this class include:
|
12
11
|
* 1. Indexing files by storing their directory paths, file paths, symbols (e.g., class names, method names), and
|
13
12
|
* general words in the database. Symbols are given more weight in the search results.
|
14
|
-
* 2.
|
15
|
-
*
|
16
|
-
* influenced by both the indexed content and any associated boost factors.
|
13
|
+
* 2. Performing search queries on the indexed files using full-text search with BM25 ranking. The search results are
|
14
|
+
* influenced by the indexed content.
|
17
15
|
*
|
18
16
|
* The class uses two SQLite tables:
|
19
17
|
* - `file_content`: A virtual table that holds the file content and allows for full-text search using BM25 ranking.
|
20
|
-
* - `file_boost`: A table that stores boost factors for specific files to enhance their search relevance.
|
21
18
|
*/
|
22
19
|
export default class FileIndex {
|
23
20
|
#private;
|
24
21
|
database: sqlite3.Database;
|
25
22
|
constructor(database: sqlite3.Database);
|
26
|
-
|
23
|
+
get schemaVersion(): string | undefined;
|
24
|
+
private createSchema;
|
25
|
+
private dropAllTables;
|
26
|
+
static cached(kind: string, ...paths: string[]): Promise<FileIndex>;
|
27
|
+
get length(): number;
|
28
|
+
get path(): string;
|
29
|
+
clear(): void;
|
30
|
+
index(files: AsyncIterable<{
|
31
|
+
directory: string;
|
32
|
+
filePath: string;
|
33
|
+
}>, reader: (filePath: string) => Promise<{
|
34
|
+
symbols: string[];
|
35
|
+
words: string[];
|
36
|
+
} | undefined>): Promise<void>;
|
27
37
|
/**
|
28
|
-
*
|
29
|
-
* @param sessionId - The session identifier to associate the boost with.
|
30
|
-
* @param filePath - The path of the file to boost.
|
31
|
-
* @param boostFactor - The factor by which to boost the file's relevance.
|
32
|
-
*/
|
33
|
-
boostFile(sessionId: SessionId, filePath: string, boostFactor: number): void;
|
34
|
-
/**
|
35
|
-
* Deletes all data associated with a specific session.
|
36
|
-
* @param sessionId - The session identifier to delete data for.
|
37
|
-
*/
|
38
|
-
deleteSession(sessionId: string): void;
|
39
|
-
/**
|
40
|
-
* Searches for files matching the query, considering session-specific boosts.
|
41
|
-
* @param sessionId - The session identifier to apply during the search.
|
38
|
+
* Searches for files matching the query.
|
42
39
|
* @param query - The search query string.
|
43
40
|
* @param limit - The maximum number of results to return.
|
44
41
|
* @returns An array of search results with directory, file path, and score.
|
45
42
|
*/
|
46
|
-
search(
|
43
|
+
search(query: string, limit?: number): FileSearchResult[];
|
47
44
|
close(): void;
|
48
45
|
}
|
package/built/file-index.js
CHANGED
@@ -10,39 +10,46 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
10
10
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
11
11
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
12
12
|
};
|
13
|
-
var
|
13
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
14
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
15
|
+
};
|
16
|
+
var _FileIndex_search;
|
14
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
15
|
-
const
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
)
|
22
|
-
const
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
18
|
+
const node_assert_1 = __importDefault(require("node:assert"));
|
19
|
+
const node_console_1 = require("node:console");
|
20
|
+
const promises_1 = require("node:fs/promises");
|
21
|
+
const node_path_1 = require("node:path");
|
22
|
+
const cachedir_1 = __importDefault(require("cachedir"));
|
23
|
+
const debug_1 = __importDefault(require("debug"));
|
24
|
+
const node_sqlite3_wasm_1 = __importDefault(require("node-sqlite3-wasm"));
|
25
|
+
const debug = (0, debug_1.default)('appmap:search:file-index');
|
26
|
+
const SCHEMA = `
|
27
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS file_content USING fts5(
|
28
|
+
directory UNINDEXED,
|
29
|
+
file_path,
|
30
|
+
file_symbols,
|
31
|
+
file_words,
|
32
|
+
tokenize = 'porter unicode61'
|
33
|
+
);
|
34
|
+
CREATE TABLE IF NOT EXISTS file_metadata (
|
35
|
+
file_path TEXT PRIMARY KEY,
|
36
|
+
directory TEXT NOT NULL,
|
37
|
+
generation INTEGER NOT NULL,
|
38
|
+
modified INTEGER
|
39
|
+
);
|
40
|
+
CREATE INDEX file_metadata_generation ON file_metadata (generation);
|
41
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
42
|
+
key TEXT PRIMARY KEY,
|
43
|
+
value TEXT
|
44
|
+
);`;
|
33
45
|
const SEARCH_SQL = `SELECT
|
34
46
|
file_content.directory,
|
35
47
|
file_content.file_path,
|
36
|
-
|
37
|
-
|
48
|
+
-- directory, file_path, file_symbols, file_words
|
49
|
+
-bm25(file_content, 0, 3, 2, 1)
|
38
50
|
AS score
|
39
51
|
FROM
|
40
52
|
file_content
|
41
|
-
LEFT JOIN
|
42
|
-
file_boost
|
43
|
-
ON
|
44
|
-
file_content.file_path = file_boost.file_path
|
45
|
-
AND file_boost.session_id = ?
|
46
53
|
WHERE
|
47
54
|
file_content MATCH ?
|
48
55
|
ORDER BY
|
@@ -50,64 +57,144 @@ ORDER BY
|
|
50
57
|
LIMIT
|
51
58
|
?
|
52
59
|
`;
|
60
|
+
const SCHEMA_VERSION = '1';
|
53
61
|
/**
|
54
62
|
* The FileIndex class provides an interface to interact with the SQLite search index.
|
55
63
|
*
|
56
64
|
* The primary responsibilities of this class include:
|
57
65
|
* 1. Indexing files by storing their directory paths, file paths, symbols (e.g., class names, method names), and
|
58
66
|
* general words in the database. Symbols are given more weight in the search results.
|
59
|
-
* 2.
|
60
|
-
*
|
61
|
-
* influenced by both the indexed content and any associated boost factors.
|
67
|
+
* 2. Performing search queries on the indexed files using full-text search with BM25 ranking. The search results are
|
68
|
+
* influenced by the indexed content.
|
62
69
|
*
|
63
70
|
* The class uses two SQLite tables:
|
64
71
|
* - `file_content`: A virtual table that holds the file content and allows for full-text search using BM25 ranking.
|
65
|
-
* - `file_boost`: A table that stores boost factors for specific files to enhance their search relevance.
|
66
72
|
*/
|
67
73
|
class FileIndex {
|
68
74
|
constructor(database) {
|
69
75
|
this.database = database;
|
70
|
-
_FileIndex_insert.set(this, void 0);
|
71
|
-
_FileIndex_updateBoost.set(this, void 0);
|
72
|
-
_FileIndex_deleteSession.set(this, void 0);
|
73
76
|
_FileIndex_search.set(this, void 0);
|
74
|
-
this.
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
__classPrivateFieldSet(this, _FileIndex_updateBoost, this.database.prepare(UPDATE_BOOST_SQL), "f");
|
80
|
-
__classPrivateFieldSet(this, _FileIndex_deleteSession, this.database.prepare(DELETE_SESSION_SQL), "f");
|
77
|
+
if (this.schemaVersion !== SCHEMA_VERSION) {
|
78
|
+
debug('Schema version mismatch, recreating schema');
|
79
|
+
this.dropAllTables();
|
80
|
+
this.createSchema();
|
81
|
+
}
|
81
82
|
__classPrivateFieldSet(this, _FileIndex_search, this.database.prepare(SEARCH_SQL), "f");
|
82
83
|
}
|
83
|
-
|
84
|
-
|
84
|
+
get schemaVersion() {
|
85
|
+
try {
|
86
|
+
const version = this.database.get("SELECT value FROM metadata WHERE key = 'schema_version'");
|
87
|
+
if (!version?.value)
|
88
|
+
return undefined;
|
89
|
+
return String(version.value);
|
90
|
+
}
|
91
|
+
catch {
|
92
|
+
return;
|
93
|
+
}
|
85
94
|
}
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
* @param filePath - The path of the file to boost.
|
90
|
-
* @param boostFactor - The factor by which to boost the file's relevance.
|
91
|
-
*/
|
92
|
-
boostFile(sessionId, filePath, boostFactor) {
|
93
|
-
__classPrivateFieldGet(this, _FileIndex_updateBoost, "f").run(sessionId, filePath, boostFactor);
|
95
|
+
createSchema() {
|
96
|
+
this.database.exec(SCHEMA);
|
97
|
+
this.database.run("INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ?)", SCHEMA_VERSION);
|
94
98
|
}
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
99
|
+
dropAllTables() {
|
100
|
+
this.database.exec(`
|
101
|
+
DROP TABLE IF EXISTS file_content;
|
102
|
+
DROP TABLE IF EXISTS file_metadata;
|
103
|
+
DROP TABLE IF EXISTS metadata;
|
104
|
+
`);
|
105
|
+
}
|
106
|
+
static async cached(kind, ...paths) {
|
107
|
+
const names = paths.map((path) => (0, node_path_1.resolve)(path).replace(/[^a-z0-9]/gi, '_')).join('-');
|
108
|
+
const name = `index-${kind}-${names}.sqlite`;
|
109
|
+
const cacheDir = (0, cachedir_1.default)('appmap');
|
110
|
+
await (0, promises_1.mkdir)(cacheDir, { recursive: true });
|
111
|
+
const dbPath = (0, node_path_1.join)(cacheDir, name);
|
112
|
+
const db = new node_sqlite3_wasm_1.default.Database(dbPath);
|
113
|
+
debug('Using cached database: %s', dbPath);
|
114
|
+
return new FileIndex(db);
|
115
|
+
}
|
116
|
+
get length() {
|
117
|
+
const row = this.database.get('SELECT COUNT(*) AS count FROM file_content');
|
118
|
+
(0, node_assert_1.default)(typeof row?.count === 'number');
|
119
|
+
return row.count;
|
120
|
+
}
|
121
|
+
get path() {
|
122
|
+
const file = this.database.get('PRAGMA database_list')?.file;
|
123
|
+
(0, node_assert_1.default)(typeof file === 'string');
|
124
|
+
return file || ':memory:';
|
125
|
+
}
|
126
|
+
clear() {
|
127
|
+
this.database.exec(`
|
128
|
+
DELETE FROM file_content;
|
129
|
+
DELETE FROM file_metadata;
|
130
|
+
`);
|
131
|
+
}
|
132
|
+
async index(files, reader) {
|
133
|
+
const insert = this.database.prepare(`INSERT INTO file_content (directory, file_path, file_symbols, file_words) VALUES (?, ?, ?, ?)`);
|
134
|
+
const update = this.database.prepare(`UPDATE file_content SET file_symbols = ?, file_words = ? WHERE directory = ? AND file_path = ?`);
|
135
|
+
const setMetadata = this.database.prepare(`
|
136
|
+
INSERT OR REPLACE INTO file_metadata (file_path, directory, generation, modified)
|
137
|
+
VALUES (?, ?, ?, ?)
|
138
|
+
`);
|
139
|
+
const getMetadata = this.database.prepare(`SELECT * FROM file_metadata WHERE file_path = ?`);
|
140
|
+
this.database.exec('BEGIN TRANSACTION');
|
141
|
+
const generation = Date.now();
|
142
|
+
try {
|
143
|
+
for await (const file of files) {
|
144
|
+
const { directory, filePath } = file;
|
145
|
+
const stats = await (0, promises_1.stat)(filePath).catch(() => undefined);
|
146
|
+
const metadata = getMetadata.get(filePath);
|
147
|
+
setMetadata.run([filePath, directory, generation, stats?.mtimeMs || null]);
|
148
|
+
if (metadata) {
|
149
|
+
if (stats && metadata.modified === stats.mtimeMs) {
|
150
|
+
debug('Skipping unchanged file: %s', filePath);
|
151
|
+
continue;
|
152
|
+
}
|
153
|
+
else {
|
154
|
+
debug('Updating file: %s', filePath);
|
155
|
+
}
|
156
|
+
}
|
157
|
+
try {
|
158
|
+
const content = await reader(filePath);
|
159
|
+
if (!content)
|
160
|
+
continue;
|
161
|
+
debug('Indexing file: %s', filePath);
|
162
|
+
if (metadata) {
|
163
|
+
update.run([content.symbols.join(' '), content.words.join(' '), directory, filePath]);
|
164
|
+
}
|
165
|
+
else {
|
166
|
+
insert.run([directory, filePath, content.symbols.join(' '), content.words.join(' ')]);
|
167
|
+
}
|
168
|
+
}
|
169
|
+
catch (error) {
|
170
|
+
(0, node_console_1.warn)('Error indexing file: %s', filePath);
|
171
|
+
debug(error);
|
172
|
+
}
|
173
|
+
}
|
174
|
+
this.database.exec(`
|
175
|
+
DELETE FROM file_content WHERE file_path IN (SELECT file_path FROM file_metadata WHERE generation <> ${generation});
|
176
|
+
DELETE FROM file_metadata WHERE generation <> ${generation};
|
177
|
+
`);
|
178
|
+
this.database.exec('COMMIT');
|
179
|
+
}
|
180
|
+
catch (error) {
|
181
|
+
this.database.exec('ROLLBACK');
|
182
|
+
throw error;
|
183
|
+
}
|
184
|
+
finally {
|
185
|
+
insert.finalize();
|
186
|
+
update.finalize();
|
187
|
+
getMetadata.finalize();
|
188
|
+
}
|
101
189
|
}
|
102
190
|
/**
|
103
|
-
* Searches for files matching the query
|
104
|
-
* @param sessionId - The session identifier to apply during the search.
|
191
|
+
* Searches for files matching the query.
|
105
192
|
* @param query - The search query string.
|
106
193
|
* @param limit - The maximum number of results to return.
|
107
194
|
* @returns An array of search results with directory, file path, and score.
|
108
195
|
*/
|
109
|
-
search(
|
110
|
-
const rows = __classPrivateFieldGet(this, _FileIndex_search, "f").all(
|
196
|
+
search(query, limit = 10) {
|
197
|
+
const rows = __classPrivateFieldGet(this, _FileIndex_search, "f").all([query, limit]);
|
111
198
|
return rows.map((row) => ({
|
112
199
|
directory: row.directory,
|
113
200
|
filePath: row.file_path,
|
@@ -115,8 +202,9 @@ class FileIndex {
|
|
115
202
|
}));
|
116
203
|
}
|
117
204
|
close() {
|
205
|
+
__classPrivateFieldGet(this, _FileIndex_search, "f").finalize();
|
118
206
|
this.database.close();
|
119
207
|
}
|
120
208
|
}
|
121
|
-
|
209
|
+
_FileIndex_search = new WeakMap();
|
122
210
|
exports.default = FileIndex;
|
package/built/snippet-index.d.ts
CHANGED
package/built/snippet-index.js
CHANGED
@@ -68,7 +68,10 @@ var SnippetType;
|
|
68
68
|
function fileChunkSnippetId(filePath, startLine) {
|
69
69
|
return {
|
70
70
|
type: 'file-chunk',
|
71
|
-
id: [filePath, startLine]
|
71
|
+
id: [filePath, startLine]
|
72
|
+
.filter((t) => Boolean(t))
|
73
|
+
.map(encodeURIComponent)
|
74
|
+
.join(':'),
|
72
75
|
};
|
73
76
|
}
|
74
77
|
function parseFileChunkSnippetId(snippetId) {
|
@@ -79,7 +82,7 @@ function parseFileChunkSnippetId(snippetId) {
|
|
79
82
|
(0, assert_1.default)(filePath);
|
80
83
|
const startLine = parts.shift();
|
81
84
|
return {
|
82
|
-
filePath: filePath,
|
85
|
+
filePath: decodeURIComponent(filePath),
|
83
86
|
startLine: startLine ? parseInt(startLine, 10) : undefined,
|
84
87
|
};
|
85
88
|
}
|
@@ -105,8 +108,8 @@ class SnippetIndex {
|
|
105
108
|
_SnippetIndex_searchSnippet.set(this, void 0);
|
106
109
|
this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL);
|
107
110
|
this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL);
|
108
|
-
this.database.
|
109
|
-
this.database.
|
111
|
+
this.database.exec('PRAGMA journal_mode = OFF');
|
112
|
+
this.database.exec('PRAGMA synchronous = OFF');
|
110
113
|
__classPrivateFieldSet(this, _SnippetIndex_insertSnippet, this.database.prepare(INSERT_SNIPPET_SQL), "f");
|
111
114
|
__classPrivateFieldSet(this, _SnippetIndex_deleteSession, this.database.prepare(DELETE_SESSION_SQL), "f");
|
112
115
|
__classPrivateFieldSet(this, _SnippetIndex_updateSnippetBoost, this.database.prepare(UPDATE_SNIPPET_BOOST_SQL), "f");
|
@@ -128,7 +131,7 @@ class SnippetIndex {
|
|
128
131
|
* @param content - The actual content of the snippet.
|
129
132
|
*/
|
130
133
|
indexSnippet(snippetId, directory, symbols, words, content) {
|
131
|
-
__classPrivateFieldGet(this, _SnippetIndex_insertSnippet, "f").run(encodeSnippetId(snippetId), directory, symbols, words, content);
|
134
|
+
__classPrivateFieldGet(this, _SnippetIndex_insertSnippet, "f").run([encodeSnippetId(snippetId), directory, symbols, words, content]);
|
132
135
|
}
|
133
136
|
/**
|
134
137
|
* Boosts the relevance score of a specific snippet for a given session.
|
@@ -137,10 +140,10 @@ class SnippetIndex {
|
|
137
140
|
* @param boostFactor - The factor by which to boost the snippet's relevance.
|
138
141
|
*/
|
139
142
|
boostSnippet(sessionId, snippetId, boostFactor) {
|
140
|
-
__classPrivateFieldGet(this, _SnippetIndex_updateSnippetBoost, "f").run(sessionId, encodeSnippetId(snippetId), boostFactor);
|
143
|
+
__classPrivateFieldGet(this, _SnippetIndex_updateSnippetBoost, "f").run([sessionId, encodeSnippetId(snippetId), boostFactor]);
|
141
144
|
}
|
142
145
|
searchSnippets(sessionId, query, limit = 10) {
|
143
|
-
const rows = __classPrivateFieldGet(this, _SnippetIndex_searchSnippet, "f").all(sessionId, query, limit);
|
146
|
+
const rows = __classPrivateFieldGet(this, _SnippetIndex_searchSnippet, "f").all([sessionId, query, limit]);
|
144
147
|
return rows.map((row) => ({
|
145
148
|
directory: row.directory,
|
146
149
|
snippetId: parseSnippetId(row.snippet_id),
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@appland/search",
|
3
|
-
"version": "1.
|
3
|
+
"version": "1.2.0",
|
4
4
|
"description": "",
|
5
5
|
"bin": "built/cli.js",
|
6
6
|
"publishConfig": {
|
@@ -21,7 +21,6 @@
|
|
21
21
|
"author": "AppLand, Inc",
|
22
22
|
"license": "Commons Clause + MIT",
|
23
23
|
"devDependencies": {
|
24
|
-
"@types/better-sqlite3": "^7.6.11",
|
25
24
|
"@types/jest": "^29.5.4",
|
26
25
|
"@types/node": "^16",
|
27
26
|
"eslint": "^9",
|
@@ -39,8 +38,9 @@
|
|
39
38
|
"typescript-eslint": "^8.11.0"
|
40
39
|
},
|
41
40
|
"dependencies": {
|
42
|
-
"
|
41
|
+
"cachedir": "^2.4.0",
|
43
42
|
"isbinaryfile": "^5.0.4",
|
43
|
+
"node-sqlite3-wasm": "^0.8.34",
|
44
44
|
"yargs": "^17.7.2"
|
45
45
|
}
|
46
46
|
}
|