@ambicuity/kindx 0.1.0 → 1.1.0

package/dist/watcher.js

@@ -0,0 +1,243 @@
+import chokidar from "chokidar";
+import { relative as pathRelative } from "path";
+import { realpathSync } from "fs";
+const c = {
+    reset: "\x1b[0m",
+    bold: "\x1b[1m",
+    dim: "\x1b[2m",
+    red: "\x1b[31m",
+    green: "\x1b[32m",
+    yellow: "\x1b[33m",
+    cyan: "\x1b[36m",
+};
+function formatMs(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    return `${(ms / 1000).toFixed(1)}s`;
+}
+export class WatchDaemon {
+    store;
+    watchers = [];
+    eventQueue = [];
+    isProcessing = false;
+    debounceTimer = null;
+    debounceMs = 500;
+    // Track start time for freshness reporting
+    startTime = Date.now();
+    lastUpdateTs = Date.now();
+    eventCount = 0;
+    constructor(store) {
+        this.store = store;
+    }
+    /**
+     * Start watching specified collections (or all active if none specified)
+     */
+    async start(collectionNames) {
+        const { listCollections } = await import("./catalogs.js");
+        // Since we are reading from the DB/yaml config
+        const collections = listCollections();
+        // Filter if specific collections requested
+        const targetCollections = collectionNames && collectionNames.length > 0
+            ? collections.filter(c => collectionNames.includes(c.name))
+            : collections;
+        if (targetCollections.length === 0) {
+            console.log("No collections to watch.");
+            return;
+        }
+        try {
+            const { resolve } = await import("path");
+            const { homedir } = await import("os");
+            const { writeFileSync, existsSync, mkdirSync } = await import("fs");
+            const cacheDir = process.env.XDG_CACHE_HOME
+                ? resolve(process.env.XDG_CACHE_HOME, "kindx")
+                : resolve(homedir(), ".cache", "kindx");
+            if (!existsSync(cacheDir)) {
+                mkdirSync(cacheDir, { recursive: true });
+            }
+            writeFileSync(resolve(cacheDir, "watch.pid"), process.pid.toString(), "utf-8");
+        }
+        catch (err) {
+            console.error("Warning: could not write watch.pid file", err);
+        }
+        console.log(`Starting watcher daemon for ${targetCollections.length} collections...`);
+        const readyPromises = [];
+        for (const coll of targetCollections) {
+            // Build absolute watch path based on collection pwd and pattern
+            // chokidar handles globs like /path/to/collection/**/*.md
+            // We watch the whole directory and filter by pattern later to avoid chokidar glob complexity issues on some platforms
+            const watchTarget = coll.path;
+            const usePolling = process.env.CHOKIDAR_USEPOLLING === "1"
+                || process.env.CHOKIDAR_USEPOLLING === "true";
+            const pollInterval = Number(process.env.CHOKIDAR_INTERVAL || 100);
+            console.log(`- Watching [${c.bold}${coll.name}${c.reset}]: ${watchTarget} (pattern: ${coll.pattern})`);
+            const watcher = chokidar.watch(watchTarget, {
+                persistent: true,
+                ignoreInitial: true, // Don't trigger 'add' for existing files on startup
+                ignored: [/(^|[\/\\])\../, "**/node_modules/**", "dist/**"], // ignore dotfiles and common dirs
+                usePolling,
+                interval: pollInterval,
+                binaryInterval: pollInterval,
+                awaitWriteFinish: {
+                    stabilityThreshold: 300,
+                    pollInterval: 100
+                }
+            });
+            // Simple matcher for the collection pattern
+            const picomatchModule = await import("picomatch");
+            const picomatch = picomatchModule.default || picomatchModule;
+            const isMatch = picomatch(coll.pattern);
+            const processEvent = (type, absolutePath) => {
+                let canonicalPath = absolutePath;
+                try {
+                    canonicalPath = realpathSync(absolutePath);
+                }
+                catch {
+                    // The file may already be gone for unlink events; fall back to chokidar's path.
+                }
+                const relativePath = pathRelative(coll.path, canonicalPath).replace(/\\/g, "/");
+                if (relativePath === "" || relativePath === "." || relativePath.startsWith("../")) {
+                    return;
+                }
+                // Check pattern
+                if (!isMatch(relativePath))
+                    return;
+                this.enqueue(type, coll.name, relativePath, canonicalPath.replace(/\\/g, "/"));
+            };
+            watcher
+                .on("add", (path) => processEvent("add", path))
+                .on("change", (path) => processEvent("change", path))
+                .on("unlink", (path) => processEvent("unlink", path))
+                .on("error", (error) => console.error(`Watcher error for [${coll.name}]:`, error));
+            readyPromises.push(new Promise((resolve) => {
+                let settled = false;
+                watcher.once("ready", () => {
+                    settled = true;
+                    resolve();
+                });
+                watcher.once("error", () => {
+                    if (!settled)
+                        resolve();
+                });
+            }));
+            this.watchers.push(watcher);
+        }
+        await Promise.all(readyPromises);
+        console.log("Daemon active. Waiting for file system changes...");
+    }
+    /**
+     * Stop all watchers
+     */
+    async stop() {
+        console.log("Stopping watchers...");
+        await Promise.all(this.watchers.map(w => w.close()));
+        this.watchers = [];
+        if (this.debounceTimer)
+            clearTimeout(this.debounceTimer);
+        try {
+            const { resolve } = await import("path");
+            const { homedir } = await import("os");
+            const { unlinkSync, existsSync } = await import("fs");
+            const cacheDir = process.env.XDG_CACHE_HOME
+                ? resolve(process.env.XDG_CACHE_HOME, "kindx")
+                : resolve(homedir(), ".cache", "kindx");
+            const pidPath = resolve(cacheDir, "watch.pid");
+            if (existsSync(pidPath))
+                unlinkSync(pidPath);
+        }
+        catch (err) { }
+    }
+    /**
+     * Enqueue a filesystem event and trigger debounced processing
+     */
+    enqueue(type, collectionName, relativePath, absolutePath) {
+        // Deduplicate: if an event for this file already exists in queue, update it
+        const existingIdx = this.eventQueue.findIndex(e => e.collectionName === collectionName && e.relativePath === relativePath);
+        if (existingIdx >= 0) {
+            const existing = this.eventQueue[existingIdx];
+            // If we got 'add' then 'change', it's still an 'add' or 'change'. 
+            // If we got 'add' then 'unlink', it's dead.
+            // Easiest approach: just overwrite with the latest state
+            if (type === "unlink") {
+                existing.type = "unlink";
+            }
+            else if (existing.type !== "add") {
+                existing.type = "change";
+            }
+        }
+        else {
+            this.eventQueue.push({ type, collectionName, relativePath, absolutePath });
+        }
+        // Debounce processing
+        if (this.debounceTimer)
+            clearTimeout(this.debounceTimer);
+        this.debounceTimer = setTimeout(() => {
+            this.processQueue().catch(err => {
+                console.error("Error processing watch queue:", err);
+            });
+        }, this.debounceMs);
+    }
+    /**
+     * Process queued events sequentially to prevent SQLite WAL contention
+     */
+    async processQueue() {
+        if (this.isProcessing || this.eventQueue.length === 0)
+            return;
+        this.isProcessing = true;
+        // Take a snapshot of the current queue
+        const batch = [...this.eventQueue];
+        this.eventQueue = [];
+        const startMs = Date.now();
+        let processed = 0;
+        // Format current time
+        const now = new Date().toLocaleTimeString();
+        console.log(`\n[${now}] Processing ${batch.length} changed files...`);
+        try {
+            // Need to cast store to any temporarily since the methods don't exist yet
+            const store = this.store;
+            for (const event of batch) {
+                try {
+                    if (event.type === "unlink") {
+                        const removed = await store.unlinkSingleFile(event.collectionName, event.relativePath);
+                        if (removed) {
+                            console.log(`  ${c.red}[-]${c.reset} Removed: ${event.relativePath}`);
+                            processed++;
+                        }
+                    }
+                    else {
+                        // add or change
+                        const result = await store.indexSingleFile(event.collectionName, event.relativePath, event.absolutePath);
+                        if (result === "embedded") {
+                            console.log(`  ${c.green}[+]${c.reset} Re-indexed: ${event.relativePath}`);
+                            processed++;
+                        }
+                        else if (result === "unchanged") {
+                            // hash matched, no change needed
+                            // console.log(`  [=] Unchanged: ${event.relativePath}`);
+                        }
+                        else if (result === "failed") {
+                            console.log(`  ${c.red}[x]${c.reset} Failed: ${event.relativePath}`);
+                        }
+                    }
+                }
+                catch (err) {
+                    console.error(`  ${c.red}[!]${c.reset} Error processing ${event.relativePath}:`, err);
+                }
+            }
+            if (processed > 0) {
+                this.lastUpdateTs = Date.now();
+                this.eventCount += processed;
+                const elapsed = Date.now() - startMs;
+                console.log(`[${new Date().toLocaleTimeString()}] Batch complete. ${processed} files updated in ${formatMs(elapsed)}`);
+            }
+        }
+        finally {
+            this.isProcessing = false;
+            // If more events arrived while we were processing, trigger again
+            if (this.eventQueue.length > 0) {
+                setTimeout(() => {
+                    this.processQueue().catch(err => console.error(err));
+                }, 50);
+            }
+        }
+    }
+}

package/package.json

@@ -1,20 +1,32 @@
 {
   "name": "@ambicuity/kindx",
-  "version": "0.1.0",
+  "version": "1.1.0",
   "description": "KINDX - On-device hybrid search engine for markdown documents with BM25, vector embeddings, and LLM reranking",
   "type": "module",
+  "workspaces": [
+    "packages/*"
+  ],
   "bin": {
-    "kindx": "dist/kindx.js"
+    "kindx": "bin/kindx"
   },
   "files": [
+    "bin/",
     "dist/",
+    "capabilities/kindx/",
     "LICENSE",
     "CHANGELOG.md"
   ],
   "scripts": {
     "prepare": "[ -d .git ] && ./tooling/install-hooks.sh || true",
     "build": "tsc -p tsconfig.build.json && printf '#!/usr/bin/env node\n' | cat - dist/kindx.js > dist/kindx.tmp && mv dist/kindx.tmp dist/kindx.js && chmod +x dist/kindx.js",
+    "build:packages": "npm run build -w @ambicuity/kindx-schemas && npm run build -w @ambicuity/kindx-client",
+    "build:all": "npm run build && npm run build:packages",
+    "pretest": "npm run build",
+    "pretest:all": "npm run build:all",
     "test": "vitest run --reporter=verbose specs/",
+    "test:packages": "npm run test -w @ambicuity/kindx-schemas && npm run test -w @ambicuity/kindx-client",
+    "test:python": "python3 -m unittest discover -s python/kindx-langchain/tests -v",
+    "test:all": "npm run pretest:all && npm run test && npm run test:packages && npm run test:python",
     "kindx": "tsx engine/kindx.ts",
     "index": "tsx engine/kindx.ts index",
     "vector": "tsx engine/kindx.ts vector",
@@ -25,7 +37,8 @@
     "release": "./tooling/release.sh"
   },
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
   },
   "repository": {
     "type": "git",
@@ -37,20 +50,21 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.25.1",
-    "better-sqlite3": "^11.0.0",
+    "better-sqlite3": "^12.4.5",
+    "chokidar": "^5.0.0",
     "fast-glob": "^3.3.0",
     "node-llama-cpp": "^3.17.1",
     "picomatch": "^4.0.0",
-    "sqlite-vec": "^0.1.7-alpha.2",
+    "sqlite-vec": "0.1.7",
     "yaml": "^2.8.2",
     "zod": "^4.2.1"
   },
   "optionalDependencies": {
-    "sqlite-vec-darwin-arm64": "^0.1.7-alpha.2",
-    "sqlite-vec-darwin-x64": "^0.1.7-alpha.2",
-    "sqlite-vec-linux-arm64": "^0.1.7-alpha.2",
-    "sqlite-vec-linux-x64": "^0.1.7-alpha.2",
-    "sqlite-vec-windows-x64": "^0.1.7-alpha.2"
+    "sqlite-vec-darwin-arm64": "0.1.7",
+    "sqlite-vec-darwin-x64": "0.1.7",
+    "sqlite-vec-linux-arm64": "0.1.7",
+    "sqlite-vec-linux-x64": "0.1.7",
+    "sqlite-vec-windows-x64": "0.1.7"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.0",
@@ -58,7 +72,8 @@
     "@types/picomatch": "^4.0.2",
     "@types/yaml": "^1.9.6",
     "tsx": "^4.0.0",
-    "vitest": "^3.0.0"
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.2"
   },
   "peerDependencies": {
     "typescript": "^5.9.3"