directory-indexer 0.3.0 → 1.1.0

package/dist/storage.js

@@ -1,5 +1,5 @@
 import Database from "better-sqlite3";
-import { ensureDirectory } from "./utils.js";
+import { mkdirSync } from "node:fs";
 import { dirname } from "path";
 function escapeLike(value) {
   return value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
@@ -8,10 +8,11 @@ function directoryLikeClause(column, dirPath) {
   const escaped = escapeLike(dirPath);
   return {
     clause: `(${column} = ? OR ${column} LIKE ? ESCAPE '\\' OR ${column} LIKE ? ESCAPE '\\')`,
-    params: [dirPath, `${escaped}/%`, `${escaped}\\%`]
+    params: [dirPath, `${escaped}/%`, `${escaped}\\\\%`]
   };
 }
 const openStorageInstances = /* @__PURE__ */ new Set();
+const sharedStorage = /* @__PURE__ */ new Map();
 function closeAllStorage() {
   for (const instance of openStorageInstances) {
     try {
@@ -19,6 +20,7 @@ function closeAllStorage() {
     } catch {
     }
   }
+  sharedStorage.clear();
 }
 class StorageError extends Error {
   constructor(message, cause) {
@@ -31,27 +33,33 @@ class QdrantClient {
   constructor(config) {
     this.config = config;
   }
+  qdrantFetch(path, init) {
+    const headers = { "Content-Type": "application/json" };
+    if (this.config.storage.qdrantApiKey) {
+      headers["api-key"] = this.config.storage.qdrantApiKey;
+    }
+    return fetch(`${this.config.storage.qdrantEndpoint}${path}`, { ...init, headers });
+  }
   async healthCheck() {
     try {
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/healthz`);
+      const response = await this.qdrantFetch("/healthz");
       return response.ok;
     } catch {
       return false;
     }
   }
-  async createCollection() {
+  async createCollection(dimension) {
     const collectionName = this.config.storage.qdrantCollection;
     try {
-      const checkResponse = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}`);
+      const checkResponse = await this.qdrantFetch(`/collections/${collectionName}`);
       if (checkResponse.ok) {
         return;
       }
-      const createResponse = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}`, {
+      const createResponse = await this.qdrantFetch(`/collections/${collectionName}`, {
         method: "PUT",
-        headers: { "Content-Type": "application/json" },
         body: JSON.stringify({
           vectors: {
-            size: 768,
+            size: dimension,
             distance: "Cosine"
           }
         })
@@ -66,9 +74,8 @@ class QdrantClient {
   async upsertPoints(points) {
     const collectionName = this.config.storage.qdrantCollection;
     try {
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}/points`, {
+      const response = await this.qdrantFetch(`/collections/${collectionName}/points`, {
         method: "PUT",
-        headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ points })
       });
       if (!response.ok) {
@@ -90,11 +97,13 @@ class QdrantClient {
       if (filter) {
         searchBody.filter = filter;
       }
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}/points/search`, {
+      const response = await this.qdrantFetch(`/collections/${collectionName}/points/search`, {
         method: "POST",
-        headers: { "Content-Type": "application/json" },
         body: JSON.stringify(searchBody)
       });
+      if (response.status === 404) {
+        return [];
+      }
       if (!response.ok) {
         const errorBody = await response.text();
         throw new Error(`Failed to search points: ${response.statusText} - ${errorBody}`);
@@ -113,11 +122,13 @@ class QdrantClient {
   async deletePoints(ids) {
     const collectionName = this.config.storage.qdrantCollection;
     try {
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}/points/delete`, {
+      const response = await this.qdrantFetch(`/collections/${collectionName}/points/delete`, {
         method: "POST",
-        headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ points: ids })
       });
+      if (response.status === 404) {
+        return;
+      }
       if (!response.ok) {
         throw new Error(`Failed to delete points: ${response.statusText}`);
       }
@@ -128,9 +139,8 @@ class QdrantClient {
   async deletePointsByFilePath(filePath) {
     const collectionName = this.config.storage.qdrantCollection;
     try {
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}/points/delete`, {
+      const response = await this.qdrantFetch(`/collections/${collectionName}/points/delete`, {
         method: "POST",
-        headers: { "Content-Type": "application/json" },
         body: JSON.stringify({
           filter: {
             must: [
@@ -142,6 +152,9 @@ class QdrantClient {
           }
         })
       });
+      if (response.status === 404) {
+        return;
+      }
       if (!response.ok) {
         const errorText = await response.text();
         throw new Error(`Failed to delete points by file path: ${response.status} ${response.statusText} - ${errorText}`);
@@ -157,11 +170,13 @@ class QdrantClient {
       if (filter) {
         countBody.filter = filter;
       }
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}/points/count`, {
+      const response = await this.qdrantFetch(`/collections/${collectionName}/points/count`, {
         method: "POST",
-        headers: { "Content-Type": "application/json" },
         body: JSON.stringify(countBody)
       });
+      if (response.status === 404) {
+        return 0;
+      }
       if (!response.ok) {
         throw new Error(`Failed to count points: ${response.statusText}`);
       }
@@ -174,29 +189,40 @@ class QdrantClient {
   async scrollPoints(filter, limit = 1e3) {
     const collectionName = this.config.storage.qdrantCollection;
     try {
-      const scrollBody = {
-        limit,
-        with_payload: true,
-        with_vector: false
-      };
-      if (filter) {
-        scrollBody.filter = filter;
-      }
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}/points/scroll`, {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify(scrollBody)
-      });
-      if (!response.ok) {
-        throw new Error(`Failed to scroll points: ${response.statusText}`);
-      }
-      const data = await response.json();
-      return data.result.points.map((item) => ({
-        id: item.id,
-        vector: [],
-        payload: item.payload,
-        score: 0
-      }));
+      const points = [];
+      let offset = null;
+      do {
+        const scrollBody = {
+          limit,
+          with_payload: true,
+          with_vector: false
+        };
+        if (filter) {
+          scrollBody.filter = filter;
+        }
+        if (offset !== null) {
+          scrollBody.offset = offset;
+        }
+        const response = await this.qdrantFetch(`/collections/${collectionName}/points/scroll`, {
+          method: "POST",
+          body: JSON.stringify(scrollBody)
+        });
+        if (response.status === 404) {
+          return [];
+        }
+        if (!response.ok) {
+          throw new Error(`Failed to scroll points: ${response.statusText}`);
+        }
+        const data = await response.json();
+        points.push(...data.result.points.map((item) => ({
+          id: item.id,
+          vector: [],
+          payload: item.payload,
+          score: 0
+        })));
+        offset = data.result.next_page_offset ?? null;
+      } while (offset !== null);
+      return points;
     } catch (error) {
       throw new StorageError(`Failed to scroll points in Qdrant`, error);
     }
@@ -204,7 +230,7 @@ class QdrantClient {
   async getCollectionInfo() {
     const collectionName = this.config.storage.qdrantCollection;
     try {
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}`);
+      const response = await this.qdrantFetch(`/collections/${collectionName}`);
       if (!response.ok) {
         if (response.status === 404) {
           return null;
@@ -212,9 +238,14 @@ class QdrantClient {
         throw new Error(`Failed to get collection info: ${response.statusText}`);
       }
       const data = await response.json();
-      return {
+      const vectors = data.result?.config?.params?.vectors;
+      const info = {
         vectors_count: data.result?.points_count || data.result?.vectors_count || 0
       };
+      if (vectors && typeof vectors === "object" && typeof vectors.size === "number") {
+        info.vectorSize = vectors.size;
+      }
+      return info;
     } catch (error) {
       throw new StorageError(`Failed to get collection info from Qdrant`, error);
     }
@@ -222,11 +253,8 @@ class QdrantClient {
   async deleteCollection() {
     const collectionName = this.config.storage.qdrantCollection;
     try {
-      const response = await fetch(`${this.config.storage.qdrantEndpoint}/collections/${collectionName}`, {
-        method: "DELETE",
-        headers: this.config.storage.qdrantApiKey ? {
-          "api-key": this.config.storage.qdrantApiKey
-        } : {}
+      const response = await this.qdrantFetch(`/collections/${collectionName}`, {
+        method: "DELETE"
       });
       if (!response.ok && response.status !== 404) {
         throw new Error(`Failed to delete collection: ${response.statusText}`);
@@ -245,7 +273,7 @@ class SQLiteStorage {
   db;
   initializeDatabase() {
     try {
-      ensureDirectory(dirname(this.config.storage.sqlitePath));
+      mkdirSync(dirname(this.config.storage.sqlitePath), { recursive: true });
       const db = new Database(this.config.storage.sqlitePath);
       db.exec(`
         CREATE TABLE IF NOT EXISTS directories (
@@ -269,6 +297,11 @@ class SQLiteStorage {
         CREATE INDEX IF NOT EXISTS idx_files_path ON files(path);
         CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);
         CREATE INDEX IF NOT EXISTS idx_directories_path ON directories(path);
+
+        CREATE TABLE IF NOT EXISTS meta (
+          key TEXT PRIMARY KEY,
+          value TEXT
+        );
       `);
       db.pragma("journal_mode = WAL");
       db.pragma("busy_timeout = 5000");
@@ -292,6 +325,21 @@ class SQLiteStorage {
       throw new StorageError(`Failed to get directory record`, error);
     }
   }
+  async getDirectoryByCaseInsensitive(path) {
+    try {
+      const stmt = this.db.prepare("SELECT * FROM directories WHERE path = ? COLLATE NOCASE");
+      const row = stmt.get(path);
+      if (!row) return null;
+      return {
+        id: row.id,
+        path: row.path,
+        status: row.status,
+        indexedAt: new Date(row.indexed_at)
+      };
+    } catch (error) {
+      throw new StorageError(`Failed to get directory record`, error);
+    }
+  }
   async upsertDirectory(path, status) {
     try {
       const stmt = this.db.prepare(`
@@ -312,7 +360,7 @@ class SQLiteStorage {
         id: row.id,
         path: row.path,
         size: row.size,
-        modifiedTime: new Date(row.modified_time * 1e3),
+        modifiedTime: new Date(row.modified_time),
         hash: row.hash,
         parentDirs: JSON.parse(row.parent_dirs),
         chunks: row.chunks_json ? JSON.parse(row.chunks_json) : [],
@@ -362,7 +410,7 @@ class SQLiteStorage {
         id: row.id,
         path: row.path,
         size: row.size,
-        modifiedTime: new Date(row.modified_time * 1e3),
+        modifiedTime: new Date(row.modified_time),
         hash: row.hash,
         parentDirs: JSON.parse(row.parent_dirs),
         chunks: row.chunks_json ? JSON.parse(row.chunks_json) : [],
@@ -380,16 +428,52 @@ class SQLiteStorage {
     const result = this.db.prepare(`DELETE FROM files WHERE ${clause}`).run(...params);
     return result.changes;
   }
+  getMeta(key) {
+    const row = this.db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
+    return row?.value ?? null;
+  }
+  setMeta(key, value) {
+    this.db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)").run(key, value);
+  }
   close() {
     openStorageInstances.delete(this);
     this.db.close();
   }
 }
 async function initializeStorage(config) {
+  const dbPath = config.storage.sqlitePath;
+  const cached = sharedStorage.get(dbPath);
+  if (cached) {
+    return cached;
+  }
   const sqlite = new SQLiteStorage(config);
   const qdrant = new QdrantClient(config);
-  await qdrant.createCollection();
-  return { sqlite, qdrant };
+  const storage = { sqlite, qdrant };
+  sharedStorage.set(dbPath, storage);
+  return storage;
+}
+function closeSharedStorageForPath(dbPath) {
+  const shared = sharedStorage.get(dbPath);
+  if (!shared) {
+    return;
+  }
+  sharedStorage.delete(dbPath);
+  shared.sqlite.close();
+}
+async function ensureCollectionDimensions(sqlite, qdrant, config, dimension) {
+  const info = await qdrant.getCollectionInfo();
+  if (!info) {
+    await qdrant.createCollection(dimension);
+  } else if (info.vectorSize !== void 0 && info.vectorSize !== dimension) {
+    const provider = sqlite.getMeta("embedding_provider");
+    const recordedDims = sqlite.getMeta("embedding_dims");
+    const recorded = provider && recordedDims ? ` (meta table records provider '${provider}' with ${recordedDims} dimensions)` : "";
+    throw new StorageError(
+      `Qdrant collection '${config.storage.qdrantCollection}' was created with ${info.vectorSize}-dimensional vectors but the active embedding provider produces ${dimension}-dimensional vectors${recorded}. Run \`directory-indexer reset\` to delete the existing collection, then re-index.`
+    );
+  }
+  sqlite.setMeta("embedding_provider", config.embedding.provider);
+  sqlite.setMeta("embedding_dims", String(dimension));
 }
 async function initDatabase(dbPath) {
   return new Database(dbPath);
@@ -425,6 +509,7 @@ async function getResetPreview(config) {
 }
 async function clearDatabase(config) {
   try {
+    closeSharedStorageForPath(config.storage.sqlitePath);
     if (!await import("fs").then((fs) => fs.existsSync(config.storage.sqlitePath))) {
       return true;
     }
@@ -589,13 +674,12 @@ async function checkQdrantConsistency(sqlite, config) {
     }
     const collectionName = config.storage.qdrantCollection;
     try {
-      const response = await fetch(`${config.storage.qdrantEndpoint}/collections/${collectionName}`);
-      if (!response.ok) {
+      const collectionInfo = await qdrant.getCollectionInfo();
+      if (!collectionInfo) {
         issues.push(`Vector collection '${collectionName}' not found (normal during first-time setup)`);
         return { isConsistent: false, issues };
       }
-      const collectionInfo = await response.json();
-      const qdrantPointCount = collectionInfo.result?.points_count || 0;
+      const qdrantPointCount = collectionInfo.vectors_count || 0;
       const sqliteChunkCount = totalChunks.count || 0;
       if (Math.abs(qdrantPointCount - sqliteChunkCount) > 0) {
         if (qdrantPointCount > sqliteChunkCount) {
@@ -625,7 +709,7 @@ async function getIndexStatus() {
     const filesCount = filesStmt.get();
     const chunksStmt = sqlite.db.prepare("SELECT SUM(json_array_length(chunks_json)) as count FROM files WHERE chunks_json IS NOT NULL");
     const chunksCount = chunksStmt.get();
-    const lastIndexedStmt = sqlite.db.prepare("SELECT MAX(indexed_at) as last_indexed FROM directories WHERE indexed_at > 0");
+    const lastIndexedStmt = sqlite.db.prepare("SELECT MAX(indexed_at) as last_indexed FROM directories WHERE indexed_at > 0 AND status = 'completed'");
     const lastIndexedResult = lastIndexedStmt.get();
     const errorsStmt = sqlite.db.prepare("SELECT errors_json FROM files WHERE errors_json IS NOT NULL");
     const errorRows = errorsStmt.all();
@@ -638,19 +722,14 @@ async function getIndexStatus() {
         allErrors.push("Failed to parse error JSON");
       }
     });
-    const directoriesDetailStmt = sqlite.db.prepare(`
-      SELECT
-        d.path,
-        d.status,
-        d.indexed_at,
-        (SELECT COUNT(*) FROM files f WHERE f.path = d.path OR f.path LIKE REPLACE(d.path, '\\', '\\\\') || '/%' ESCAPE '\\' OR f.path LIKE REPLACE(d.path, '\\', '\\\\') || '\\%' ESCAPE '\\') as files_count,
-        (SELECT COALESCE(SUM(json_array_length(f.chunks_json)), 0) FROM files f WHERE (f.path = d.path OR f.path LIKE REPLACE(d.path, '\\', '\\\\') || '/%' ESCAPE '\\' OR f.path LIKE REPLACE(d.path, '\\', '\\\\') || '\\%' ESCAPE '\\') AND f.chunks_json IS NOT NULL) as chunks_count
-      FROM directories d
-      ORDER BY d.indexed_at DESC
-    `);
+    const directoriesDetailStmt = sqlite.db.prepare("SELECT path, status, indexed_at FROM directories ORDER BY indexed_at DESC");
     const directoryDetails = directoriesDetailStmt.all();
     const directories = directoryDetails.map((row) => {
       const { clause, params } = directoryLikeClause("path", row.path);
+      const filesByDirStmt = sqlite.db.prepare(`SELECT COUNT(*) as count FROM files WHERE ${clause}`);
+      const filesCount2 = filesByDirStmt.get(...params);
+      const chunksByDirStmt = sqlite.db.prepare(`SELECT COALESCE(SUM(json_array_length(chunks_json)), 0) as count FROM files WHERE ${clause} AND chunks_json IS NOT NULL`);
+      const chunksCount2 = chunksByDirStmt.get(...params);
       const errorsByDirStmt = sqlite.db.prepare(`
         SELECT errors_json FROM files
         WHERE ${clause} AND errors_json IS NOT NULL
@@ -668,8 +747,8 @@ async function getIndexStatus() {
       return {
         path: row.path,
         status: row.status,
-        filesCount: row.files_count,
-        chunksCount: row.chunks_count,
+        filesCount: filesCount2.count,
+        chunksCount: chunksCount2.count,
         lastIndexed: row.indexed_at && row.indexed_at > 0 ? new Date(row.indexed_at).toISOString() : null,
         errors: dirErrorsList
       };
@@ -715,6 +794,8 @@ export {
   clearDatabase,
   clearVectorCollection,
   closeAllStorage,
+  closeSharedStorageForPath,
+  ensureCollectionDimensions,
   getIndexStatus,
   getResetPreview,
   initDatabase,