localrag 0.1.0

package/dist/watcher/file-watcher.js

@@ -0,0 +1,110 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileWatcher = void 0;
+const path = __importStar(require("path"));
+const chokidar = __importStar(require("chokidar"));
+const extractor_interface_1 = require("../extractors/extractor.interface");
+/**
+ * FileWatcher wraps chokidar and routes file-system events to the Indexer.
+ *
+ * - add / change  → indexFile()
+ * - unlink        → removeFile()
+ *
+ * Events are serialised through a simple sequential queue to prevent
+ * concurrent writes to LanceDB from the same watcher process.
+ */
+class FileWatcher {
+    watcher;
+    indexer;
+    // Simple sequential queue: each pending promise is awaited before the next
+    queue = Promise.resolve();
+    constructor(indexer) {
+        this.indexer = indexer;
+    }
+    watch(folders, callbacks = {}) {
+        const { onAdd, onChange, onUnlink, onReady, onProgress, onError } = callbacks;
+        const opts = { onProgress, onError };
+        this.watcher = chokidar.watch(folders, {
+            persistent: true,
+            ignoreInitial: false, // emit 'add' for existing files on startup
+            followSymlinks: true,
+            awaitWriteFinish: { stabilityThreshold: 500, pollInterval: 100 },
+            ignored: [
+                /(^|[/\\])\../, // dotfiles / hidden
+                '**/node_modules/**',
+                '**/.git/**',
+            ],
+        });
+        this.watcher
+            .on('add', (filePath) => {
+            if (!(0, extractor_interface_1.isSupportedFile)(filePath))
+                return;
+            onAdd?.(filePath);
+            this.enqueue(() => this.indexer.indexFile(filePath, opts));
+        })
+            .on('change', (filePath) => {
+            if (!(0, extractor_interface_1.isSupportedFile)(filePath))
+                return;
+            onChange?.(filePath);
+            this.enqueue(async () => {
+                await this.indexer.removeFile(filePath, opts);
+                await this.indexer.indexFile(filePath, opts);
+            });
+        })
+            .on('unlink', (filePath) => {
+            if (!(0, extractor_interface_1.isSupportedFile)(filePath))
+                return;
+            onUnlink?.(filePath);
+            this.enqueue(() => this.indexer.removeFile(filePath, opts));
+        })
+            .on('ready', () => onReady?.())
+            .on('error', (err) => {
+            onError?.(path.join(...folders), err);
+        });
+    }
+    async close() {
+        if (this.watcher)
+            await this.watcher.close();
+    }
+    // ── Queue ────────────────────────────────────────────────────────────────
+    enqueue(task) {
+        // Chain the task onto the queue; errors are swallowed at queue level
+        // (individual tasks already call onError internally)
+        this.queue = this.queue.then(task).catch(() => { });
+    }
+}
+exports.FileWatcher = FileWatcher;
+//# sourceMappingURL=file-watcher.js.map

package/dist/watcher/file-watcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-watcher.js","sourceRoot":"","sources":["../../src/watcher/file-watcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAC7B,mDAAqC;AACrC,2EAAoE;AAYpE;;;;;;;;GAQG;AACH,MAAa,WAAW;IACd,OAAO,CAAsB;IAC7B,OAAO,CAAU;IAEzB,2EAA2E;IACnE,KAAK,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEjD,YAAY,OAAgB;QAC1B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,OAAiB,EAAE,YAA8B,EAAE;QACvD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,SAAS,CAAC;QAE9E,MAAM,IAAI,GAAmB,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QAErD,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE;YACrC,UAAU,EAAE,IAAI;YAChB,aAAa,EAAE,KAAK,EAAE,2CAA2C;YACjE,cAAc,EAAE,IAAI;YACpB,gBAAgB,EAAE,EAAE,kBAAkB,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE;YAChE,OAAO,EAAE;gBACP,cAAc,EAAE,oBAAoB;gBACpC,oBAAoB;gBACpB,YAAY;aACb;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO;aACT,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC9B,IAAI,CAAC,IAAA,qCAAe,EAAC,QAAQ,CAAC;gBAAE,OAAO;YACvC,KAAK,EAAE,CAAC,QAAQ,CAAC,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC;aACD,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YACjC,IAAI,CAAC,IAAA,qCAAe,EAAC,QAAQ,CAAC;gBAAE,OAAO;YACvC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YACrB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;gBACtB,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;aACD,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAgB,EAAE,EAAE;YACjC,IAAI,CAAC,IAAA,qCAAe,EAAC,QAAQ,CAAC;gBAAE,OAAO;YACvC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;YACrB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAC9D,CAAC,CAAC;aACD,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;aAC9B,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;YAC1B,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC/C,CAAC;IAED,4EAA4E;IAEpE,OAAO,CAAC,IAAyB;QACvC,qEAAqE;QACrE,qDAAqD;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAA0B,CAAC,CAAC,CAAC;IAC7E,CAAC;CACF;AAhED,kCAgEC"}

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "localrag",
+  "version": "0.1.0",
+  "description": "Local-first semantic document search CLI — indexes your documents and finds them by meaning, not keywords. No cloud. No API keys.",
+  "main": "dist/cli/index.js",
+  "bin": {
+    "localrag": "./dist/cli/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "rag",
+    "semantic-search",
+    "local-first",
+    "vector-search",
+    "documents",
+    "cli",
+    "lancedb",
+    "embeddings"
+  ],
+  "author": "ujjwal.shrivas",
+  "license": "MIT",
+  "files": [
+    "dist/",
+    "README.md"
+  ],
+  "dependencies": {
+    "@xenova/transformers": "^2.17.2",
+    "adm-zip": "^0.5.16",
+    "chalk": "^4.1.2",
+    "chokidar": "^3.6.0",
+    "cli-table3": "^0.6.5",
+    "commander": "^12.1.0",
+    "fast-xml-parser": "^4.5.0",
+    "mammoth": "^1.8.0",
+    "ora": "^5.4.1",
+    "pdf-parse": "^1.1.1",
+    "vectordb": "^0.4.20",
+    "xlsx": "^0.18.5"
+  },
+  "devDependencies": {
+    "@types/adm-zip": "^0.5.7",
+    "@types/node": "^22.0.0",
+    "@types/pdf-parse": "^1.1.4",
+    "typescript": "^5.5.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}