@appland/search 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/built/cli.js +4 -3
- package/built/file-index.d.ts +1 -2
- package/built/file-index.js +32 -40
- package/built/snippet-index.d.ts +1 -1
- package/built/snippet-index.js +5 -5
- package/package.json +3 -2
package/CHANGELOG.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
# [@appland/search-v1.2.2](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.2.1...@appland/search-v1.2.2) (2025-05-01)
|
2
|
+
|
3
|
+
|
4
|
+
### Bug Fixes
|
5
|
+
|
6
|
+
* Migrate from node-sqlite3-wasm to better-sqlite3 ([47b7698](https://github.com/getappmap/appmap-js/commit/47b769860ce7a55dc3806981da2a78a408bc0648))
|
7
|
+
|
8
|
+
# [@appland/search-v1.2.1](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.2.0...@appland/search-v1.2.1) (2025-03-11)
|
9
|
+
|
10
|
+
|
11
|
+
### Bug Fixes
|
12
|
+
|
13
|
+
* Add PerformanceObserver import ([#2258](https://github.com/getappmap/appmap-js/issues/2258)) ([595b542](https://github.com/getappmap/appmap-js/commit/595b542c33529b7fb5e8ed07126f44bc5a545137))
|
14
|
+
|
1
15
|
# [@appland/search-v1.2.0](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.1.3...@appland/search-v1.2.0) (2025-02-27)
|
2
16
|
|
3
17
|
|
package/built/cli.js
CHANGED
@@ -3,10 +3,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
4
4
|
};
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
6
|
+
const node_perf_hooks_1 = require("node:perf_hooks");
|
7
|
+
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
6
8
|
const yargs_1 = __importDefault(require("yargs"));
|
7
9
|
const helpers_1 = require("yargs/helpers");
|
8
10
|
const debug_1 = __importDefault(require("debug"));
|
9
|
-
const node_sqlite3_wasm_1 = __importDefault(require("node-sqlite3-wasm"));
|
10
11
|
const tokenize_1 = require("./tokenize");
|
11
12
|
const file_index_1 = __importDefault(require("./file-index"));
|
12
13
|
const build_file_index_1 = __importDefault(require("./build-file-index"));
|
@@ -40,7 +41,7 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
40
41
|
})
|
41
42
|
.strict();
|
42
43
|
}, async (argv) => {
|
43
|
-
new PerformanceObserver((entries) => entries.getEntries().forEach((e) => console.warn(`${e.name}: ${e.duration.toFixed(0)} ms`))).observe({ entryTypes: ['measure'] });
|
44
|
+
new node_perf_hooks_1.PerformanceObserver((entries) => entries.getEntries().forEach((e) => console.warn(`${e.name}: ${e.duration.toFixed(0)} ms`))).observe({ entryTypes: ['measure'] });
|
44
45
|
const { directories, query } = argv;
|
45
46
|
let filterRE;
|
46
47
|
if (argv.fileFilter)
|
@@ -80,7 +81,7 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
80
81
|
printResult('file', filePath, score);
|
81
82
|
}
|
82
83
|
const splitter = splitter_1.langchainSplitter;
|
83
|
-
const snippetIndex = new snippet_index_1.default(new
|
84
|
+
const snippetIndex = new snippet_index_1.default(new better_sqlite3_1.default());
|
84
85
|
await (0, build_snippet_index_1.default)(snippetIndex, fileSearchResults, ioutil_1.readFileSafe, splitter, tokenize_1.fileTokens);
|
85
86
|
console.log('');
|
86
87
|
console.log('Snippet search results');
|
package/built/file-index.d.ts
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
import sqlite3 from '
|
1
|
+
import sqlite3 from 'better-sqlite3';
|
2
2
|
export type FileSearchResult = {
|
3
3
|
directory: string;
|
4
4
|
filePath: string;
|
@@ -17,7 +17,6 @@ export type FileSearchResult = {
|
|
17
17
|
* - `file_content`: A virtual table that holds the file content and allows for full-text search using BM25 ranking.
|
18
18
|
*/
|
19
19
|
export default class FileIndex {
|
20
|
-
#private;
|
21
20
|
database: sqlite3.Database;
|
22
21
|
constructor(database: sqlite3.Database);
|
23
22
|
get schemaVersion(): string | undefined;
|
package/built/file-index.js
CHANGED
@@ -1,29 +1,19 @@
|
|
1
1
|
"use strict";
|
2
|
-
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
3
|
-
if (kind === "m") throw new TypeError("Private method is not writable");
|
4
|
-
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
5
|
-
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
6
|
-
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
7
|
-
};
|
8
|
-
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
9
|
-
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
10
|
-
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
11
|
-
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
12
|
-
};
|
13
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
14
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
15
4
|
};
|
16
|
-
var _FileIndex_search;
|
17
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
18
6
|
const node_assert_1 = __importDefault(require("node:assert"));
|
19
7
|
const node_console_1 = require("node:console");
|
20
8
|
const promises_1 = require("node:fs/promises");
|
21
9
|
const node_path_1 = require("node:path");
|
10
|
+
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
22
11
|
const cachedir_1 = __importDefault(require("cachedir"));
|
23
12
|
const debug_1 = __importDefault(require("debug"));
|
24
|
-
const node_sqlite3_wasm_1 = __importDefault(require("node-sqlite3-wasm"));
|
25
13
|
const debug = (0, debug_1.default)('appmap:search:file-index');
|
14
|
+
const SCHEMA_VERSION = '1';
|
26
15
|
const SCHEMA = `
|
16
|
+
BEGIN;
|
27
17
|
CREATE VIRTUAL TABLE IF NOT EXISTS file_content USING fts5(
|
28
18
|
directory UNINDEXED,
|
29
19
|
file_path,
|
@@ -41,7 +31,10 @@ const SCHEMA = `
|
|
41
31
|
CREATE TABLE IF NOT EXISTS metadata (
|
42
32
|
key TEXT PRIMARY KEY,
|
43
33
|
value TEXT
|
44
|
-
)
|
34
|
+
);
|
35
|
+
INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ${SCHEMA_VERSION});
|
36
|
+
COMMIT;
|
37
|
+
`;
|
45
38
|
const SEARCH_SQL = `SELECT
|
46
39
|
file_content.directory,
|
47
40
|
file_content.file_path,
|
@@ -57,7 +50,6 @@ ORDER BY
|
|
57
50
|
LIMIT
|
58
51
|
?
|
59
52
|
`;
|
60
|
-
const SCHEMA_VERSION = '1';
|
61
53
|
/**
|
62
54
|
* The FileIndex class provides an interface to interact with the SQLite search index.
|
63
55
|
*
|
@@ -73,28 +65,29 @@ const SCHEMA_VERSION = '1';
|
|
73
65
|
class FileIndex {
|
74
66
|
constructor(database) {
|
75
67
|
this.database = database;
|
76
|
-
|
77
|
-
if (
|
78
|
-
debug(
|
68
|
+
const version = this.schemaVersion;
|
69
|
+
if (version !== SCHEMA_VERSION) {
|
70
|
+
debug(`Schema version mismatch (${version}), recreating schema`);
|
79
71
|
this.dropAllTables();
|
80
72
|
this.createSchema();
|
81
73
|
}
|
82
|
-
__classPrivateFieldSet(this, _FileIndex_search, this.database.prepare(SEARCH_SQL), "f");
|
83
74
|
}
|
84
75
|
get schemaVersion() {
|
85
76
|
try {
|
86
|
-
const version = this.database
|
87
|
-
|
77
|
+
const version = this.database
|
78
|
+
.prepare("SELECT value FROM metadata WHERE key = 'schema_version'")
|
79
|
+
.get();
|
80
|
+
if (!version?.version)
|
88
81
|
return undefined;
|
89
|
-
return String(version.
|
82
|
+
return String(version.version);
|
90
83
|
}
|
91
|
-
catch {
|
84
|
+
catch (error) {
|
85
|
+
debug('Error retrieving schema version: %s', error);
|
92
86
|
return;
|
93
87
|
}
|
94
88
|
}
|
95
89
|
createSchema() {
|
96
90
|
this.database.exec(SCHEMA);
|
97
|
-
this.database.run("INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ?)", SCHEMA_VERSION);
|
98
91
|
}
|
99
92
|
dropAllTables() {
|
100
93
|
this.database.exec(`
|
@@ -109,17 +102,19 @@ class FileIndex {
|
|
109
102
|
const cacheDir = (0, cachedir_1.default)('appmap');
|
110
103
|
await (0, promises_1.mkdir)(cacheDir, { recursive: true });
|
111
104
|
const dbPath = (0, node_path_1.join)(cacheDir, name);
|
112
|
-
const db =
|
105
|
+
const db = (0, better_sqlite3_1.default)(dbPath);
|
113
106
|
debug('Using cached database: %s', dbPath);
|
114
107
|
return new FileIndex(db);
|
115
108
|
}
|
116
109
|
get length() {
|
117
|
-
const row = this.database
|
110
|
+
const row = this.database
|
111
|
+
.prepare('SELECT COUNT(*) AS count FROM file_content')
|
112
|
+
.get();
|
118
113
|
(0, node_assert_1.default)(typeof row?.count === 'number');
|
119
114
|
return row.count;
|
120
115
|
}
|
121
116
|
get path() {
|
122
|
-
const file = this.database.
|
117
|
+
const file = this.database.prepare('PRAGMA database_list').get()?.file;
|
123
118
|
(0, node_assert_1.default)(typeof file === 'string');
|
124
119
|
return file || ':memory:';
|
125
120
|
}
|
@@ -144,7 +139,7 @@ class FileIndex {
|
|
144
139
|
const { directory, filePath } = file;
|
145
140
|
const stats = await (0, promises_1.stat)(filePath).catch(() => undefined);
|
146
141
|
const metadata = getMetadata.get(filePath);
|
147
|
-
setMetadata.run(
|
142
|
+
setMetadata.run(filePath, directory, generation, stats?.mtimeMs || null);
|
148
143
|
if (metadata) {
|
149
144
|
if (stats && metadata.modified === stats.mtimeMs) {
|
150
145
|
debug('Skipping unchanged file: %s', filePath);
|
@@ -160,10 +155,10 @@ class FileIndex {
|
|
160
155
|
continue;
|
161
156
|
debug('Indexing file: %s', filePath);
|
162
157
|
if (metadata) {
|
163
|
-
update.run(
|
158
|
+
update.run(content.symbols.join(' '), content.words.join(' '), directory, filePath);
|
164
159
|
}
|
165
160
|
else {
|
166
|
-
insert.run(
|
161
|
+
insert.run(directory, filePath, content.symbols.join(' '), content.words.join(' '));
|
167
162
|
}
|
168
163
|
}
|
169
164
|
catch (error) {
|
@@ -178,14 +173,10 @@ class FileIndex {
|
|
178
173
|
this.database.exec('COMMIT');
|
179
174
|
}
|
180
175
|
catch (error) {
|
181
|
-
this.database.
|
176
|
+
if (this.database.open)
|
177
|
+
this.database.exec('ROLLBACK');
|
182
178
|
throw error;
|
183
179
|
}
|
184
|
-
finally {
|
185
|
-
insert.finalize();
|
186
|
-
update.finalize();
|
187
|
-
getMetadata.finalize();
|
188
|
-
}
|
189
180
|
}
|
190
181
|
/**
|
191
182
|
* Searches for files matching the query.
|
@@ -194,7 +185,9 @@ class FileIndex {
|
|
194
185
|
* @returns An array of search results with directory, file path, and score.
|
195
186
|
*/
|
196
187
|
search(query, limit = 10) {
|
197
|
-
const rows =
|
188
|
+
const rows = this.database
|
189
|
+
.prepare(SEARCH_SQL)
|
190
|
+
.all(query, limit);
|
198
191
|
return rows.map((row) => ({
|
199
192
|
directory: row.directory,
|
200
193
|
filePath: row.file_path,
|
@@ -202,9 +195,8 @@ class FileIndex {
|
|
202
195
|
}));
|
203
196
|
}
|
204
197
|
close() {
|
205
|
-
|
206
|
-
|
198
|
+
if (this.database.open)
|
199
|
+
this.database.close();
|
207
200
|
}
|
208
201
|
}
|
209
|
-
_FileIndex_search = new WeakMap();
|
210
202
|
exports.default = FileIndex;
|
package/built/snippet-index.d.ts
CHANGED
package/built/snippet-index.js
CHANGED
@@ -108,8 +108,8 @@ class SnippetIndex {
|
|
108
108
|
_SnippetIndex_searchSnippet.set(this, void 0);
|
109
109
|
this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL);
|
110
110
|
this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL);
|
111
|
-
this.database.
|
112
|
-
this.database.
|
111
|
+
this.database.pragma('journal_mode = OFF');
|
112
|
+
this.database.pragma('synchronous = OFF');
|
113
113
|
__classPrivateFieldSet(this, _SnippetIndex_insertSnippet, this.database.prepare(INSERT_SNIPPET_SQL), "f");
|
114
114
|
__classPrivateFieldSet(this, _SnippetIndex_deleteSession, this.database.prepare(DELETE_SESSION_SQL), "f");
|
115
115
|
__classPrivateFieldSet(this, _SnippetIndex_updateSnippetBoost, this.database.prepare(UPDATE_SNIPPET_BOOST_SQL), "f");
|
@@ -131,7 +131,7 @@ class SnippetIndex {
|
|
131
131
|
* @param content - The actual content of the snippet.
|
132
132
|
*/
|
133
133
|
indexSnippet(snippetId, directory, symbols, words, content) {
|
134
|
-
__classPrivateFieldGet(this, _SnippetIndex_insertSnippet, "f").run(
|
134
|
+
__classPrivateFieldGet(this, _SnippetIndex_insertSnippet, "f").run(encodeSnippetId(snippetId), directory, symbols, words, content);
|
135
135
|
}
|
136
136
|
/**
|
137
137
|
* Boosts the relevance score of a specific snippet for a given session.
|
@@ -140,10 +140,10 @@ class SnippetIndex {
|
|
140
140
|
* @param boostFactor - The factor by which to boost the snippet's relevance.
|
141
141
|
*/
|
142
142
|
boostSnippet(sessionId, snippetId, boostFactor) {
|
143
|
-
__classPrivateFieldGet(this, _SnippetIndex_updateSnippetBoost, "f").run(
|
143
|
+
__classPrivateFieldGet(this, _SnippetIndex_updateSnippetBoost, "f").run(sessionId, encodeSnippetId(snippetId), boostFactor);
|
144
144
|
}
|
145
145
|
searchSnippets(sessionId, query, limit = 10) {
|
146
|
-
const rows = __classPrivateFieldGet(this, _SnippetIndex_searchSnippet, "f").all(
|
146
|
+
const rows = __classPrivateFieldGet(this, _SnippetIndex_searchSnippet, "f").all(sessionId, query, limit);
|
147
147
|
return rows.map((row) => ({
|
148
148
|
directory: row.directory,
|
149
149
|
snippetId: parseSnippetId(row.snippet_id),
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@appland/search",
|
3
|
-
"version": "1.2.
|
3
|
+
"version": "1.2.2",
|
4
4
|
"description": "",
|
5
5
|
"bin": "built/cli.js",
|
6
6
|
"publishConfig": {
|
@@ -21,6 +21,7 @@
|
|
21
21
|
"author": "AppLand, Inc",
|
22
22
|
"license": "Commons Clause + MIT",
|
23
23
|
"devDependencies": {
|
24
|
+
"@types/better-sqlite3": "^7.6.13",
|
24
25
|
"@types/jest": "^29.5.4",
|
25
26
|
"@types/node": "^16",
|
26
27
|
"eslint": "^9",
|
@@ -38,9 +39,9 @@
|
|
38
39
|
"typescript-eslint": "^8.11.0"
|
39
40
|
},
|
40
41
|
"dependencies": {
|
42
|
+
"better-sqlite3": "^11.9.1",
|
41
43
|
"cachedir": "^2.4.0",
|
42
44
|
"isbinaryfile": "^5.0.4",
|
43
|
-
"node-sqlite3-wasm": "^0.8.34",
|
44
45
|
"yargs": "^17.7.2"
|
45
46
|
}
|
46
47
|
}
|