@ghcrawl/api-core 0.7.0 → 0.8.0-beta.1

package/dist/vector/vectorlite-store.js

@@ -0,0 +1,124 @@
+import { createRequire } from 'node:module';
+import fs from 'node:fs';
+import path from 'node:path';
+import { openDb } from '../db/sqlite.js';
+const requireFromHere = createRequire(import.meta.url);
+const TABLE_NAME = 'thread_vectors_ann';
+const META_TABLE_NAME = 'vector_store_meta';
+const HNSW_MAX_ELEMENTS = 1_000_000;
+export class VectorliteStore {
+    options;
+    handles = new Map();
+    constructor(options = {}) {
+        this.options = options;
+    }
+    checkRuntime() {
+        try {
+            this.resolveExtensionPath();
+            const db = openDb(':memory:');
+            try {
+                db.loadExtension(this.resolveExtensionPath());
+                db.prepare('select vectorlite_info()').get();
+            }
+            finally {
+                db.close();
+            }
+            return { ok: true, error: null };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+    resetRepository(params) {
+        const handle = this.getHandle(params.storePath, params.dimensions);
+        handle.db.exec(`drop table if exists ${TABLE_NAME}`);
+        handle.db.exec(`delete from ${META_TABLE_NAME}`);
+        fs.rmSync(this.indexPath(params.storePath), { force: true });
+        handle.dimensions = null;
+        this.ensureSchema(handle, params.dimensions);
+    }
+    upsertVector(params) {
+        const handle = this.getHandle(params.storePath, params.dimensions);
+        handle.db.exec(`delete from ${TABLE_NAME} where rowid = ${Math.trunc(params.threadId)}`);
+        handle.db
+            .prepare(`insert into ${TABLE_NAME}(rowid, vec) values (?, ?)`)
+            .run(params.threadId, this.vectorBuffer(params.vector));
+    }
+    deleteVector(params) {
+        const handle = this.getHandle(params.storePath, params.dimensions);
+        handle.db.exec(`delete from ${TABLE_NAME} where rowid = ${Math.trunc(params.threadId)}`);
+    }
+    queryNearest(params) {
+        const handle = this.getHandle(params.storePath, params.dimensions);
+        const safeLimit = Math.max(1, params.limit);
+        const safeCandidateK = Math.max(safeLimit, params.candidateK ?? safeLimit);
+        const querySql = params.efSearch !== undefined
+            ? `select rowid, distance from ${TABLE_NAME} where knn_search(vec, knn_param(?, ${safeCandidateK}, ${params.efSearch}))`
+            : `select rowid, distance from ${TABLE_NAME} where knn_search(vec, knn_param(?, ${safeCandidateK}))`;
+        const rows = handle.db.prepare(querySql).all([this.vectorBuffer(params.vector)]);
+        return rows
+            .filter((row) => row.rowid !== params.excludeThreadId)
+            .slice(0, safeLimit)
+            .map((row) => ({
+            threadId: row.rowid,
+            score: this.distanceToScore(row.distance),
+        }));
+    }
+    close() {
+        for (const handle of this.handles.values()) {
+            handle.db.close();
+        }
+        this.handles.clear();
+    }
+    getHandle(storePath, dimensions) {
+        const existing = this.handles.get(storePath);
+        if (existing) {
+            this.ensureSchema(existing, dimensions);
+            return existing;
+        }
+        const db = openDb(storePath);
+        db.pragma('journal_mode = WAL');
+        db.pragma('synchronous = NORMAL');
+        db.loadExtension(this.resolveExtensionPath());
+        const handle = { db, storePath, dimensions: null };
+        this.handles.set(storePath, handle);
+        this.ensureSchema(handle, dimensions);
+        return handle;
+    }
+    ensureSchema(handle, dimensions) {
+        handle.db.exec(`create table if not exists ${META_TABLE_NAME} (id integer primary key check (id = 1), dimensions integer not null)`);
+        const meta = handle.db.prepare(`select dimensions from ${META_TABLE_NAME} where id = 1`).get();
+        const tableExists = Boolean(handle.db.prepare("select 1 from sqlite_master where type = 'table' and name = ? limit 1").get(TABLE_NAME));
+        if (!meta || meta.dimensions !== dimensions || !tableExists) {
+            handle.db.exec(`drop table if exists ${TABLE_NAME}`);
+            handle.db.exec(`delete from ${META_TABLE_NAME}`);
+            const indexPath = this.indexPath(handle.storePath);
+            handle.db.exec(`create virtual table ${TABLE_NAME} using vectorlite(vec float32[${dimensions}], hnsw(max_elements=${HNSW_MAX_ELEMENTS}), '${this.escapeSqlString(indexPath)}')`);
+            handle.db.prepare(`insert into ${META_TABLE_NAME}(id, dimensions) values (1, ?)`).run(dimensions);
+        }
+        handle.dimensions = dimensions;
+    }
+    resolveExtensionPath() {
+        if (this.options.extensionPathProvider) {
+            return this.options.extensionPathProvider();
+        }
+        const vectorlite = requireFromHere('vectorlite');
+        return vectorlite.vectorlitePath();
+    }
+    vectorBuffer(vector) {
+        return Buffer.from(Float32Array.from(vector).buffer);
+    }
+    distanceToScore(distance) {
+        return 1 - distance / 2;
+    }
+    indexPath(storePath) {
+        return path.join(path.dirname(storePath), `${path.basename(storePath, path.extname(storePath))}.hnsw`);
+    }
+    escapeSqlString(value) {
+        return value.replace(/'/g, "''");
+    }
+}
+//# sourceMappingURL=vectorlite-store.js.map

package/dist/vector/vectorlite-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectorlite-store.js","sourceRoot":"","sources":["../../src/vector/vectorlite-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,MAAM,EAAuB,MAAM,iBAAiB,CAAC;AAG9D,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACvD,MAAM,UAAU,GAAG,oBAAoB,CAAC;AACxC,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAC5C,MAAM,iBAAiB,GAAG,SAAS,CAAC;AAYpC,MAAM,OAAO,eAAe;IAIP;IAHF,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAE1D,YACmB,UAEb,EAAE;QAFW,YAAO,GAAP,OAAO,CAElB;IACL,CAAC;IAEJ,YAAY;QACV,IAAI,CAAC;YACH,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,CAAwB,CAAC;YACrD,IAAI,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC;gBAC9C,EAAE,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,GAAG,EAAE,CAAC;YAC/C,CAAC;oBAAS,CAAC;gBACT,EAAE,CAAC,KAAK,EAAE,CAAC;YACb,CAAC;YACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,eAAe,CAAC,MAAiD;QAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACnE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;QACrD,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,eAAe,EAAE,CAAC,CAAC;QACjD,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,YAAY,CAAC,MAAqF;QAChG,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACnE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,UAAU,kBAAkB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzF,MAAM,CAAC,EAAE;aACN,OAAO,CAAC,eAAe,UAAU,4BAA4B,CAAC;aAC9D,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,YAAY,CAAC,MAAmE;QAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACnE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,UAAU,kBAAkB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED,YAAY,CAAC,MAAyB;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC;QAC3E,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,KAAK,SAAS;YAC3B,CAAC,CAAC,+BAA+B,UAAU,uCAAuC,cAAc,KAAK,MAAM,CAAC,QAAQ,IAAI;YACxH,CAAC,CAAC,+BAA+B,UAAU,uCAAuC,cAAc,IAAI,CAAC;QACzG,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAA+C,CAAC;QAE/H,OAAO,IAAI;aACR,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,CAAC,eAAe,CAAC;aACrD,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC;aACnB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,QAAQ,EAAE,GAAG,CAAC,KAAK;YACnB,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC1C,CAAC,CAAC,CAAC;IACR,CAAC;IAED,KAAK;QACH,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAEO,SAAS,CAAC,SAAiB,EAAE,UAAkB;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACxC,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAwB,CAAC;QACpD,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAChC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAClC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAgB,EAAE,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAChE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACtC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,YAAY,CAAC,MAAmB,EAAE,UAAkB;QAC1D,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,8BAA8B,eAAe,uEAAuE,CAAC,CAAC;QACrI,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,0BAA0B,eAAe,eAAe,CAAC,CAAC,GAAG,EAAwC,CAAC;QACrI,MAAM,WAAW,GAAG,OAAO,CACzB,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,uEAAuE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAC3G,CAAC;QAEF,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,UAAU,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5D,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,eAAe,EAAE,CAAC,CAAC;YACjD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACnD,MAAM,CAAC,EAAE,CAAC,IAAI,CACZ,wBAAwB,UAAU,iCAAiC,UAAU,wBAAwB,iBAAiB,OAAO,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CACjK,CAAC;YACF,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,eAAe,gCAAgC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpG,CAAC;QAED,MAAM,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAEO,oBAAoB;QAC1B,IAAI,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC;QAC9C,CAAC;QACD,MAAM,UAAU,GAAG,eAAe,CAAC,YAAY,CAAqC,CAAC;QACrF,OAAO,UAAU,CAAC,cAAc,EAAE,CAAC;IACrC,CAAC;IAEO,YAAY,CAAC,MAAgB;QACnC,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IAEO,eAAe,CAAC,QAAgB;QACtC,OAAO,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEO,SAAS,CAAC,SAAiB;QACjC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC;IACzG,CAAC;IAEO,eAAe,CAAC,KAAa;QACnC,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ghcrawl/api-core",
-  "version": "0.7.0",
+  "version": "0.8.0-beta.1",
   "type": "module",
   "description": "ghcrawl core library for GitHub sync, SQLite storage, embeddings, search, and clustering",
   "author": "PwrDrvr LLC <harold@pwrdrvr.com>",
@@ -38,15 +38,16 @@
     }
   },
   "dependencies": {
-    "@shutterstock/p-map-iterable": "^1.1.2",
     "@octokit/plugin-retry": "^8.0.3",
     "@octokit/plugin-throttling": "^11.0.1",
-    "octokit": "^5.0.3",
-    "better-sqlite3": "^12.8.0",
+    "@shutterstock/p-map-iterable": "^1.1.2",
+    "better-sqlite3": "^12.2.0",
     "dotenv": "^17.2.2",
-    "openai": "^6.32.0",
+    "octokit": "^5.0.3",
+    "openai": "^6.33.0",
+    "vectorlite": "^0.2.0",
     "zod": "^4.3.6",
-    "@ghcrawl/api-contract": "0.7.0"
+    "@ghcrawl/api-contract": "0.8.0-beta.1"
   },
   "scripts": {
     "build": "tsc -p tsconfig.build.json",