@appland/search 1.2.1 → 1.2.3

package/CHANGELOG.md

@@ -1,3 +1,19 @@
+# [@appland/search-v1.2.3](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.2.2...@appland/search-v1.2.3) (2025-07-07)
+
+
+### Bug Fixes
+
+* Improve worst case performance of file indexing ([3a8baa1](https://github.com/getappmap/appmap-js/commit/3a8baa18f7fdb7b4ecf8df91ce2f4ff27ea13d18))
+* Optimize file indexing by caching binary content by file extension ([e402f68](https://github.com/getappmap/appmap-js/commit/e402f6872ce041bab0e5b4124ff9fa059bc5b321))
+* Prevent file index from being re-created ([d59717f](https://github.com/getappmap/appmap-js/commit/d59717fd7b09d60608625d0b3ecd14cbe82b975f))
+
+# [@appland/search-v1.2.2](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.2.1...@appland/search-v1.2.2) (2025-05-01)
+
+
+### Bug Fixes
+
+* Migrate from node-sqlite3-wasm to better-sqlite3 ([47b7698](https://github.com/getappmap/appmap-js/commit/47b769860ce7a55dc3806981da2a78a408bc0648))
+
 # [@appland/search-v1.2.1](https://github.com/getappmap/appmap-js/compare/@appland/search-v1.2.0...@appland/search-v1.2.1) (2025-03-11)
 
 

package/LICENSE.txt

@@ -0,0 +1,25 @@
+The Software is provided to you by the Licensor under the License, as defined below, subject to the following condition.
+
+Without limiting other conditions in the License, the grant of rights under the License will not include,
+and the License does not grant to you, the right to Sell the Software.
+
+For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you under the License to provide to third parties,
+for a fee or other consideration (including without limitation fees for hosting or consulting/ support services related to the Software),
+a product or service whose value derives, entirely or substantially, from the functionality of the Software.
+Any license notice or attribution required by the License must also include this Commons Clause License Condition notice.
+
+Software: @appland/search
+
+License: MIT License
+
+Copyright 2025, AppLand Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/built/cli.js

@@ -4,10 +4,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const node_perf_hooks_1 = require("node:perf_hooks");
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const yargs_1 = __importDefault(require("yargs"));
 const helpers_1 = require("yargs/helpers");
 const debug_1 = __importDefault(require("debug"));
-const node_sqlite3_wasm_1 = __importDefault(require("node-sqlite3-wasm"));
 const tokenize_1 = require("./tokenize");
 const file_index_1 = __importDefault(require("./file-index"));
 const build_file_index_1 = __importDefault(require("./build-file-index"));
@@ -48,7 +48,7 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
         filterRE = new RegExp(argv.fileFilter);
     const fileFilter = async (path) => {
         debug('Filtering: %s', path);
-        if ((0, file_type_1.isBinaryFile)(path)) {
+        if (await (0, file_type_1.isBinaryFile)(path)) {
             debug('Skipping binary file: %s', path);
             return false;
         }
@@ -81,7 +81,7 @@ const cli = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
         printResult('file', filePath, score);
     }
     const splitter = splitter_1.langchainSplitter;
-    const snippetIndex = new snippet_index_1.default(new node_sqlite3_wasm_1.default.Database());
+    const snippetIndex = new snippet_index_1.default(new better_sqlite3_1.default());
     await (0, build_snippet_index_1.default)(snippetIndex, fileSearchResults, ioutil_1.readFileSafe, splitter, tokenize_1.fileTokens);
     console.log('');
     console.log('Snippet search results');

package/built/file-index.d.ts

@@ -1,4 +1,4 @@
-import sqlite3 from 'node-sqlite3-wasm';
+import sqlite3 from 'better-sqlite3';
 export type FileSearchResult = {
     directory: string;
     filePath: string;
@@ -17,7 +17,6 @@ export type FileSearchResult = {
  * - `file_content`: A virtual table that holds the file content and allows for full-text search using BM25 ranking.
  */
 export default class FileIndex {
-    #private;
     database: sqlite3.Database;
     constructor(database: sqlite3.Database);
     get schemaVersion(): string | undefined;

package/built/file-index.js

@@ -1,29 +1,19 @@
 "use strict";
-var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
-    if (kind === "m") throw new TypeError("Private method is not writable");
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
-    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
-};
-var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
-    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
-    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
-    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
-var _FileIndex_search;
 Object.defineProperty(exports, "__esModule", { value: true });
 const node_assert_1 = __importDefault(require("node:assert"));
 const node_console_1 = require("node:console");
 const promises_1 = require("node:fs/promises");
 const node_path_1 = require("node:path");
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const cachedir_1 = __importDefault(require("cachedir"));
 const debug_1 = __importDefault(require("debug"));
-const node_sqlite3_wasm_1 = __importDefault(require("node-sqlite3-wasm"));
 const debug = (0, debug_1.default)('appmap:search:file-index');
+const SCHEMA_VERSION = '1';
 const SCHEMA = `
+  BEGIN;
   CREATE VIRTUAL TABLE IF NOT EXISTS file_content USING fts5(
     directory UNINDEXED,
     file_path,
@@ -41,7 +31,10 @@ const SCHEMA = `
   CREATE TABLE IF NOT EXISTS metadata (
     key TEXT PRIMARY KEY,
     value TEXT
-  );`;
+  );
+  INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ${SCHEMA_VERSION});
+  COMMIT;
+`;
 const SEARCH_SQL = `SELECT
     file_content.directory,
     file_content.file_path,
@@ -57,7 +50,6 @@ ORDER BY
 LIMIT
   ?
 `;
-const SCHEMA_VERSION = '1';
 /**
  * The FileIndex class provides an interface to interact with the SQLite search index.
  *
@@ -73,28 +65,29 @@ const SCHEMA_VERSION = '1';
 class FileIndex {
     constructor(database) {
         this.database = database;
-        _FileIndex_search.set(this, void 0);
-        if (this.schemaVersion !== SCHEMA_VERSION) {
-            debug('Schema version mismatch, recreating schema');
+        const version = this.schemaVersion;
+        if (version !== SCHEMA_VERSION) {
+            debug(`Schema version mismatch (${version}), recreating schema`);
             this.dropAllTables();
             this.createSchema();
         }
-        __classPrivateFieldSet(this, _FileIndex_search, this.database.prepare(SEARCH_SQL), "f");
     }
     get schemaVersion() {
         try {
-            const version = this.database.get("SELECT value FROM metadata WHERE key = 'schema_version'");
+            const version = this.database
+                .prepare("SELECT value FROM metadata WHERE key = 'schema_version'")
+                .get();
             if (!version?.value)
                 return undefined;
             return String(version.value);
         }
-        catch {
+        catch (error) {
+            debug('Error retrieving schema version: %s', error);
             return;
         }
     }
     createSchema() {
         this.database.exec(SCHEMA);
-        this.database.run("INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ?)", SCHEMA_VERSION);
     }
     dropAllTables() {
         this.database.exec(`
@@ -109,17 +102,19 @@ class FileIndex {
         const cacheDir = (0, cachedir_1.default)('appmap');
         await (0, promises_1.mkdir)(cacheDir, { recursive: true });
         const dbPath = (0, node_path_1.join)(cacheDir, name);
-        const db = new node_sqlite3_wasm_1.default.Database(dbPath);
+        const db = (0, better_sqlite3_1.default)(dbPath);
         debug('Using cached database: %s', dbPath);
         return new FileIndex(db);
     }
     get length() {
-        const row = this.database.get('SELECT COUNT(*) AS count FROM file_content');
+        const row = this.database
+            .prepare('SELECT COUNT(*) AS count FROM file_content')
+            .get();
         (0, node_assert_1.default)(typeof row?.count === 'number');
         return row.count;
     }
     get path() {
-        const file = this.database.get('PRAGMA database_list')?.file;
+        const file = this.database.prepare('PRAGMA database_list').get()?.file;
         (0, node_assert_1.default)(typeof file === 'string');
         return file || ':memory:';
     }
@@ -144,7 +139,7 @@ class FileIndex {
                 const { directory, filePath } = file;
                 const stats = await (0, promises_1.stat)(filePath).catch(() => undefined);
                 const metadata = getMetadata.get(filePath);
-                setMetadata.run([filePath, directory, generation, stats?.mtimeMs || null]);
+                setMetadata.run(filePath, directory, generation, stats?.mtimeMs || null);
                 if (metadata) {
                     if (stats && metadata.modified === stats.mtimeMs) {
                         debug('Skipping unchanged file: %s', filePath);
@@ -160,10 +155,10 @@ class FileIndex {
                         continue;
                     debug('Indexing file: %s', filePath);
                     if (metadata) {
-                        update.run([content.symbols.join(' '), content.words.join(' '), directory, filePath]);
+                        update.run(content.symbols.join(' '), content.words.join(' '), directory, filePath);
                     }
                     else {
-                        insert.run([directory, filePath, content.symbols.join(' '), content.words.join(' ')]);
+                        insert.run(directory, filePath, content.symbols.join(' '), content.words.join(' '));
                     }
                 }
                 catch (error) {
@@ -178,14 +173,10 @@ class FileIndex {
             this.database.exec('COMMIT');
         }
         catch (error) {
-            this.database.exec('ROLLBACK');
+            if (this.database.open)
+                this.database.exec('ROLLBACK');
             throw error;
         }
-        finally {
-            insert.finalize();
-            update.finalize();
-            getMetadata.finalize();
-        }
     }
     /**
      * Searches for files matching the query.
@@ -194,7 +185,9 @@ class FileIndex {
      * @returns An array of search results with directory, file path, and score.
      */
     search(query, limit = 10) {
-        const rows = __classPrivateFieldGet(this, _FileIndex_search, "f").all([query, limit]);
+        const rows = this.database
+            .prepare(SEARCH_SQL)
+            .all(query, limit);
         return rows.map((row) => ({
             directory: row.directory,
             filePath: row.file_path,
@@ -202,9 +195,8 @@ class FileIndex {
         }));
     }
     close() {
-        __classPrivateFieldGet(this, _FileIndex_search, "f").finalize();
-        this.database.close();
+        if (this.database.open)
+            this.database.close();
     }
 }
-_FileIndex_search = new WeakMap();
 exports.default = FileIndex;

package/built/file-type.d.ts

@@ -1,3 +1,4 @@
 export declare const isLargeFile: (fileName: string) => Promise<boolean>;
-export declare const isBinaryFile: (filePath: string) => boolean;
+export declare const getFileExtensions: (filePath: string) => string[];
+export declare const isBinaryFile: (filePath: string) => Promise<boolean>;
 export declare const isDataFile: (fileName: string) => boolean;

package/built/file-type.js

@@ -3,78 +3,123 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isDataFile = exports.isBinaryFile = exports.isLargeFile = void 0;
+exports.isDataFile = exports.isBinaryFile = exports.getFileExtensions = exports.isLargeFile = void 0;
 const promises_1 = require("fs/promises");
 const debug_1 = __importDefault(require("debug"));
 const isbinaryfile_1 = require("isbinaryfile");
+const path_1 = require("path");
 const debug = (0, debug_1.default)('appmap:search:file-type');
-const BINARY_FILE_EXTENSIONS = [
-    '7z',
-    'aac',
-    'avi',
-    'bmp',
-    'bz2',
-    'class',
-    'dll',
-    'doc',
-    'docx',
-    'dylib',
-    'ear',
-    'exe',
-    'eot',
-    'flac',
-    'flv',
-    'gif',
-    'gz',
-    'ico',
-    'jar',
-    'jpeg',
-    'jpg',
-    'js.map',
-    'min.js',
-    'min.css',
-    'mkv',
-    'mo',
-    'mov',
-    'mp3',
-    'mp4',
-    'mpg',
-    'odt',
-    'odp',
-    'ods',
-    'ogg',
-    'otf',
-    'pdf',
-    'po',
-    'png',
-    'ppt',
-    'pptx',
-    'pyc',
-    'rar',
-    'rtf',
-    'so',
-    'svg',
-    'tar',
-    'tiff',
-    'ttf',
-    'wav',
-    'webm',
-    'webp',
-    'woff',
-    'woff2',
-    'wmv',
-    'xls',
-    'xlsx',
-    'xz',
-    'yarn.lock',
-    'zip',
-].map((ext) => '.' + ext);
+const BINARY_FILE_MEMO = new Map([
+    // Binary files
+    ['.7z', true],
+    ['.aac', true],
+    ['.avi', true],
+    ['.bin', true],
+    ['.bmp', true],
+    ['.bz2', true],
+    ['.class', true],
+    ['.data', true],
+    ['.dat', true],
+    ['.dll', true],
+    ['.doc', true],
+    ['.docx', true],
+    ['.dylib', true],
+    ['.ear', true],
+    ['.exe', true],
+    ['.eot', true],
+    ['.flac', true],
+    ['.flv', true],
+    ['.gif', true],
+    ['.gz', true],
+    ['.ico', true],
+    ['.jar', true],
+    ['.jpeg', true],
+    ['.jpg', true],
+    ['.js.map', true],
+    ['.min.js', true],
+    ['.min.css', true],
+    ['.mkv', true],
+    ['.mo', true],
+    ['.mov', true],
+    ['.mp3', true],
+    ['.mp4', true],
+    ['.mpg', true],
+    ['.odt', true],
+    ['.odp', true],
+    ['.ods', true],
+    ['.ogg', true],
+    ['.otf', true],
+    ['.pdf', true],
+    ['.po', true],
+    ['.png', true],
+    ['.ppt', true],
+    ['.pptx', true],
+    ['.pyc', true],
+    ['.rar', true],
+    ['.rtf', true],
+    ['.so', true],
+    ['.svg', true],
+    ['.tar', true],
+    ['.tiff', true],
+    ['.ttf', true],
+    ['.wav', true],
+    ['.webm', true],
+    ['.webp', true],
+    ['.woff', true],
+    ['.woff2', true],
+    ['.wmv', true],
+    ['.xls', true],
+    ['.xlsx', true],
+    ['.xz', true],
+    ['.yarn.lock', true],
+    ['.zip', true],
+    // Source code files
+    ['.cjs', false],
+    ['.mjs', false],
+    ['.coffee', false],
+    ['.dart', false],
+    ['.el', false],
+    ['.elm', false],
+    ['.js', false],
+    ['.ts', false],
+    ['.jsx', false],
+    ['.tsx', false],
+    ['.py', false],
+    ['.java', false],
+    ['.rb', false],
+    ['.php', false],
+    ['.go', false],
+    ['.rs', false],
+    ['.cpp', false],
+    ['.cc', false],
+    ['.cxx', false],
+    ['.h', false],
+    ['.hpp', false],
+    ['.cs', false],
+    ['.swift', false],
+    ['.kt', false],
+    ['.kts', false],
+    ['.scala', false],
+    ['.lua', false],
+    ['.sh', false],
+    ['.bat', false],
+    ['.pl', false],
+    ['.sql', false],
+    ['.html', false],
+    ['.htm', false],
+    ['.css', false],
+    ['.scss', false],
+    ['.sass', false],
+    ['.less', false],
+    ['.vue', false],
+]);
 const DATA_FILE_EXTENSIONS = [
     'cjs',
     'csv',
     'dat',
     'log',
     'json',
+    'toml',
     'tsv',
     'yaml',
     'yml',
@@ -108,11 +153,35 @@ const isLargeFile = async (fileName) => {
     return fileSize > largeFileThreshold();
 };
 exports.isLargeFile = isLargeFile;
-const isBinaryFile = (filePath) => {
-    if (BINARY_FILE_EXTENSIONS.some((ext) => filePath.endsWith(ext)))
-        return true;
+/* Returns unique file extensions for the given file path.
+ * E.g., for 'example.tar.gz', it returns [ '.tar.gz', '.gz'].
+ */
+const getFileExtensions = (filePath) => {
+    const fileName = (0, path_1.basename)(filePath).toLowerCase();
+    const parts = fileName.split('.');
+    if (parts.length <= 1)
+        return [];
+    const extensions = [];
+    for (let i = 1; i < parts.length; i++) {
+        const ext = '.' + parts.slice(i).join('.');
+        extensions.push(ext);
+    }
+    return extensions;
+};
+exports.getFileExtensions = getFileExtensions;
+const isBinaryFile = async (filePath) => {
+    const fileExtensions = (0, exports.getFileExtensions)(filePath);
+    for (const ext of fileExtensions) {
+        if (BINARY_FILE_MEMO.has(ext)) {
+            return BINARY_FILE_MEMO.get(ext) === true;
+        }
+    }
     try {
-        return (0, isbinaryfile_1.isBinaryFileSync)(filePath);
+        const isBinary = await (0, isbinaryfile_1.isBinaryFile)(filePath);
+        fileExtensions.forEach((ext) => {
+            BINARY_FILE_MEMO.set(ext, isBinary);
+        });
+        return isBinary;
     }
     catch (error) {
         debug(`Error reading file: %s`, filePath);

package/built/git.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Git = exports.GitCommitEnvKeys = exports.GitBranchEnvKeys = exports.GitRepositoryEnvKeys = exports.GitState = void 0;
-const child_process_1 = require("child_process");
+const node_child_process_1 = require("node:child_process");
 const util_1 = require("util");
-const exec = (0, util_1.promisify)(child_process_1.exec);
+const exec = (0, util_1.promisify)(node_child_process_1.exec);
 var GitState;
 (function (GitState) {
     GitState[GitState["NotInstalled"] = 0] = "NotInstalled";
@@ -102,7 +102,7 @@ class GitProperties {
     static async state(cwd) {
         return new Promise((resolve) => {
             try {
-                const commandProcess = (0, child_process_1.spawn)('git', ['status', '--porcelain'], {
+                const commandProcess = (0, node_child_process_1.spawn)('git', ['status', '--porcelain'], {
                     shell: true,
                     cwd: cwd?.toString(),
                     stdio: 'ignore',

package/built/snippet-index.d.ts

@@ -1,4 +1,4 @@
-import type sqlite3 from 'node-sqlite3-wasm';
+import sqlite3 from 'better-sqlite3';
 import { SessionId } from './session-id';
 export declare enum SnippetType {
     FileChunk = "file-chunk"

package/built/snippet-index.js

@@ -108,8 +108,8 @@ class SnippetIndex {
         _SnippetIndex_searchSnippet.set(this, void 0);
         this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL);
         this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL);
-        this.database.exec('PRAGMA journal_mode = OFF');
-        this.database.exec('PRAGMA synchronous = OFF');
+        this.database.pragma('journal_mode = OFF');
+        this.database.pragma('synchronous = OFF');
         __classPrivateFieldSet(this, _SnippetIndex_insertSnippet, this.database.prepare(INSERT_SNIPPET_SQL), "f");
         __classPrivateFieldSet(this, _SnippetIndex_deleteSession, this.database.prepare(DELETE_SESSION_SQL), "f");
         __classPrivateFieldSet(this, _SnippetIndex_updateSnippetBoost, this.database.prepare(UPDATE_SNIPPET_BOOST_SQL), "f");
@@ -131,7 +131,7 @@ class SnippetIndex {
      * @param content - The actual content of the snippet.
      */
     indexSnippet(snippetId, directory, symbols, words, content) {
-        __classPrivateFieldGet(this, _SnippetIndex_insertSnippet, "f").run([encodeSnippetId(snippetId), directory, symbols, words, content]);
+        __classPrivateFieldGet(this, _SnippetIndex_insertSnippet, "f").run(encodeSnippetId(snippetId), directory, symbols, words, content);
     }
     /**
      * Boosts the relevance score of a specific snippet for a given session.
@@ -140,10 +140,10 @@ class SnippetIndex {
      * @param boostFactor - The factor by which to boost the snippet's relevance.
      */
     boostSnippet(sessionId, snippetId, boostFactor) {
-        __classPrivateFieldGet(this, _SnippetIndex_updateSnippetBoost, "f").run([sessionId, encodeSnippetId(snippetId), boostFactor]);
+        __classPrivateFieldGet(this, _SnippetIndex_updateSnippetBoost, "f").run(sessionId, encodeSnippetId(snippetId), boostFactor);
     }
     searchSnippets(sessionId, query, limit = 10) {
-        const rows = __classPrivateFieldGet(this, _SnippetIndex_searchSnippet, "f").all([sessionId, query, limit]);
+        const rows = __classPrivateFieldGet(this, _SnippetIndex_searchSnippet, "f").all(sessionId, query, limit);
         return rows.map((row) => ({
             directory: row.directory,
             snippetId: parseSnippetId(row.snippet_id),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appland/search",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "description": "",
   "bin": "built/cli.js",
   "publishConfig": {
@@ -21,6 +21,7 @@
   "author": "AppLand, Inc",
   "license": "Commons Clause + MIT",
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "@types/jest": "^29.5.4",
     "@types/node": "^16",
     "eslint": "^9",
@@ -38,9 +39,9 @@
     "typescript-eslint": "^8.11.0"
   },
   "dependencies": {
+    "better-sqlite3": "^11.9.1",
     "cachedir": "^2.4.0",
     "isbinaryfile": "^5.0.4",
-    "node-sqlite3-wasm": "^0.8.34",
     "yargs": "^17.7.2"
   }
 }