@vpxa/aikit 0.1.115 → 0.1.116
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import{rm as e}from"node:fs/promises";import{homedir as t}from"node:os";import{join as n}from"node:path";import{EMBEDDING_DEFAULTS as r}from"../../core/dist/index.js";import{env as i,pipeline as a}from"@huggingface/transformers";i.cacheDir=n(t(),`.cache`,`huggingface`,`transformers-js`);var o=class{pipe=null;shutdownPromise=null;dimensions;modelId;nativeDim;queryPrefix;threadConfig;constructor(e){if(this.modelId=e?.model??r.model,this.nativeDim=e?.nativeDim??1024,this.dimensions=e?.dimensions??r.dimensions,this.dimensions>this.nativeDim)throw Error(`Configured dimensions (${this.dimensions}) exceeds model native output (${this.nativeDim}). Matryoshka truncation cannot upscale — dimensions must be <= nativeDim.`);this.queryPrefix=e?.queryPrefix??this.detectQueryPrefix(this.modelId),this.threadConfig={interOp:e?.interOpNumThreads??1,intraOp:e?.intraOpNumThreads??4}}getPipelineOptions(){let e=i.backends.onnx;e.wasm||={};let t=e.wasm;return t.numThreads=this.threadConfig.intraOp,{dtype:`q8`,session_options:{interOpNumThreads:this.threadConfig.interOp,intraOpNumThreads:this.threadConfig.intraOp}}}truncateAndRenorm(e){if(this.dimensions>=this.nativeDim)return e;let t=e.subarray(0,this.dimensions),n=0;for(let e=0;e<t.length;e++)n+=t[e]*t[e];if(n=Math.sqrt(n),n===0)return new Float32Array(this.dimensions);let r=new Float32Array(this.dimensions);for(let e=0;e<this.dimensions;e++)r[e]=t[e]/n;return r}detectQueryPrefix(e){let t=e.toLowerCase();return t.includes(`bge`)||t.includes(`mxbai-embed`)?`Represent this sentence for searching relevant passages: `:t.includes(`/e5-`)||t.includes(`multilingual-e5`)?`query: `:``}async initialize(){if(!this.pipe){this.shutdownPromise=null;try{this.pipe=await a(`feature-extraction`,this.modelId,this.getPipelineOptions())}catch(r){let o=r.message?.toLowerCase()??``;if(this.isCorruptionError(o)){let r=n(i.cacheDir??n(t(),`.cache`,`huggingface`,`transformers-js`),this.modelId);console.error(`[aikit:auto-heal] Detected corrupted model cache for "${this.modelId}". Clearing cache at ${r} and retrying download...`);try{await e(r,{recursive:!0,force:!0})}catch{}try{this.pipe=await a(`feature-extraction`,this.modelId,this.getPipelineOptions()),console.error(`[aikit:auto-heal] Model "${this.modelId}" re-downloaded successfully.`);return}catch(e){throw Error(`Failed to initialize embedding model "${this.modelId}" after auto-heal: ${e.message}`)}}throw Error(`Failed to initialize embedding model "${this.modelId}": ${r.message}`)}}}isCorruptionError(e){return[`protobuf`,`invalid model`,`invalid onnx`,`unexpected end`,`unexpected token`,`failed to load`,`checksum`,`corrupt`,`could not load`,`onnx`,`malformed`].some(t=>e.includes(t))}async shutdown(){return this.shutdownPromise||=this._doShutdown(),this.shutdownPromise}async _doShutdown(){let e=this.pipe;if(e)try{let t=e;typeof t.dispose==`function`?await t.dispose():typeof t.model?.dispose==`function`&&await t.model.dispose()}catch{}finally{this.pipe=null}}async embed(e){this.pipe||await this.initialize();let t=await this.pipe?.(e,{pooling:`mean`,normalize:!0});if(!t)throw Error(`Embedding pipeline returned no output`);let n=new Float32Array(t.data);return this.truncateAndRenorm(n)}async embedQuery(e){return this.embed(this.queryPrefix+e)}async embedBatch(e,t=64){if(e.length===0)return[];this.pipe||await this.initialize();let n=[];for(let r=0;r<e.length;r+=t){let i=e.slice(r,r+t),a=await this.pipe?.(i,{pooling:`mean`,normalize:!0});if(!a)throw Error(`Embedding pipeline returned no output`);if(i.length===1){let e=new Float32Array(a.data);n.push(this.truncateAndRenorm(e))}else for(let e=0;e<i.length;e++){let t=e*this.nativeDim,r=a.data.slice(t,t+this.nativeDim);n.push(this.truncateAndRenorm(new Float32Array(r)))}}return n}};let s=null,c=null,l=!1;const u=Number(process.env.AIKIT_EMBED_IDLE_MS),d=Number.isFinite(u)&&u>=0?u:6e4;function f(e){process.send?.(e)}function p(){c&&=(clearTimeout(c),null)}async function m(){if(!s)return;let e=s;s=null,await e.shutdown?.()}async function h(e,t){l&&process.exit(e),l=!0,p(),t?.notifyIdle&&f({type:`idle-exit`});try{await m()}finally{process.exit(e)}}function g(){p(),d!==0&&(c=setTimeout(()=>{h(0,{notifyIdle:!0})},d))}function _(e){return Array.from(e)}function v(e){return e instanceof Error?e.message:String(e)}function y(){if(!s)throw Error(`Embedder not initialized`);return s}async function b(e,t){f({type:`error`,id:e,message:v(t)})}async function x(e){p();try{switch(e.type){case`init`:await m(),s=new o(e.config),await s.initialize(),f({type:`ready`,dimensions:s.dimensions,modelId:s.modelId});break;case`embed`:{let t=await y().embed(e.text);f({type:`result`,id:e.id,data:_(t)});break}case`embedBatch`:{let t=await y().embedBatch(e.texts,e.batchSize);f({type:`batchResult`,id:e.id,data:t.map(e=>_(e))});break}case`embedQuery`:{let t=await y().embedQuery(e.text);f({type:`result`,id:e.id,data:_(t)});break}case`shutdown`:await h(0);return}}catch(t){await b(e.type===`shutdown`?`shutdown`:e.type===`init`?`init`:e.id,t),e.type===`init`&&await h(1)}g()}async function S(e,t){await b(e,t),await h(1)}process.on(`message`,e=>{x(e)}),process.on(`SIGTERM`,()=>{h(0)}),process.on(`uncaughtException`,e=>{S(`fatal`,e)}),process.on(`unhandledRejection`,e=>{S(`fatal`,e)});export{};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{i as e,r as t,t as n}from"./sqlite-vec-store-
|
|
1
|
+
import{i as e,r as t,t as n}from"./sqlite-vec-store-xl2D7Iui.js";import{existsSync as r,mkdirSync as i}from"node:fs";import{AIKIT_PATHS as a}from"../../core/dist/index.js";import{dirname as o,join as s}from"node:path";var c=class{adapter=null;reopenPromise=null;dbPath;externalAdapter;constructor(e={}){if(e.adapter)this.adapter=e.adapter,this.externalAdapter=!0,this.dbPath=``;else{let t=e.path??a.data;this.dbPath=s(t,`graph.db`),this.externalAdapter=!1}}async initialize(){if(this.externalAdapter){let e=this.getAdapter();this.createTables(e),this.migrateSchema(e),e.flush();return}let t=o(this.dbPath);r(t)||i(t,{recursive:!0}),this.adapter=await e(this.dbPath),this.configureAdapter(this.adapter),this.createTables(this.adapter),this.migrateSchema(this.adapter),this.adapter.flush()}configureAdapter(e){e.pragma(`journal_mode = WAL`),e.pragma(`foreign_keys = ON`)}createTables(e){e.exec(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS nodes (
|
|
3
3
|
id TEXT PRIMARY KEY,
|
|
4
4
|
type TEXT NOT NULL,
|
|
@@ -87,4 +87,4 @@ import{i as e,r as t,t as n}from"./sqlite-vec-store-DA6fzQiD.js";import{existsSy
|
|
|
87
87
|
VALUES (?, ?, ?, '{}', ?)`,[a,e,t,o]);for(let e=0;e<n.length;e++)this.run(`INSERT INTO process_steps (process_id, node_id, step_order) VALUES (?, ?, ?)`,[a,n[e],e]);s.exec(`COMMIT`),s.flush()}catch(e){throw s.exec(`ROLLBACK`),e}return{id:a,entryNodeId:e,label:t,properties:{},steps:n,createdAt:o}}async getProcesses(e){await this.ensureOpen();let t;t=e?this.query(`SELECT DISTINCT p.id, p.entry_node_id, p.label, p.properties, p.created_at
|
|
88
88
|
FROM processes p
|
|
89
89
|
JOIN process_steps ps ON p.id = ps.process_id
|
|
90
|
-
WHERE ps.node_id = ?`,[e]):this.query(`SELECT * FROM processes`);let n=[];for(let e of t){let t=this.query(`SELECT node_id FROM process_steps WHERE process_id = ? ORDER BY step_order`,[e.id]);n.push({id:e.id,entryNodeId:e.entry_node_id,label:e.label,properties:l(e.properties),steps:t.map(e=>e.node_id),createdAt:e.created_at})}return n}async deleteProcess(e){await this.ensureOpen();let t=this.getAdapter();t.exec(`BEGIN TRANSACTION`);try{this.run(`DELETE FROM process_steps WHERE process_id = ?`,[e]),this.run(`DELETE FROM processes WHERE id = ?`,[e]),t.exec(`COMMIT`),t.flush()}catch(e){throw t.exec(`ROLLBACK`),e}}async depthGroupedTraversal(e,t=3,n){await this.ensureOpen();let r=n?.direction??`both`,i=n?.edgeType,a=n?.limit??100,o={},s=new Set;s.add(e);let c=[e];for(let e=1;e<=t;e++){let t=[],n=[];for(let e of c){let o=await this.getNeighbors(e,{direction:r,edgeType:i,limit:a});for(let e of o.nodes)s.has(e.id)||(s.add(e.id),t.push(e.id),n.push(e))}if(n.length>0&&(o[e]=n),c=t,c.length===0||s.size>=a)break}return o}async getCohesionScore(e){await this.ensureOpen();let t=this.query(`SELECT id FROM nodes WHERE community = ?`,[e]);if(t.length===0)return 0;let n=new Set(t.map(e=>e.id)),r=t.map(()=>`?`).join(`,`),i=t.map(e=>e.id),a=this.query(`SELECT from_id, to_id FROM edges WHERE from_id IN (${r}) OR to_id IN (${r})`,[...i,...i]);if(a.length===0)return 0;let o=0;for(let e of a)n.has(e.from_id)&&n.has(e.to_id)&&o++;return o/a.length}async getSymbol360(e){let t=await this.getNode(e);if(!t)throw Error(`Node '${e}' not found`);let n=await this.findEdges({toId:e}),r=await this.findEdges({fromId:e}),i=await this.getProcesses(e);return{node:t,incoming:n,outgoing:r,community:t.community??null,processes:i}}releaseMemory(){if(this.adapter)try{this.adapter.exec(`PRAGMA shrink_memory`),this.adapter.exec(`PRAGMA wal_checkpoint(TRUNCATE)`)}catch{}}async close(){this.adapter&&=(this.externalAdapter||this.adapter.close(),null)}};function l(e){if(!e)return{};try{return JSON.parse(e)}catch{return{}}}function u(e){return{id:e.id,type:e.type,name:e.name,properties:l(e.properties),sourceRecordId:e.source_record_id??void 0,sourcePath:e.source_path??void 0,createdAt:e.created_at,community:e.community??void 0}}function d(e){return{id:e.id,fromId:e.from_id,toId:e.to_id,type:e.type,weight:e.weight??1,confidence:e.confidence??1,properties:l(e.properties)}}function f(e){return{id:e.edge_id,fromId:e.from_id,toId:e.to_id,type:e.edge_type,weight:e.weight??1,confidence:e.edge_confidence??1,properties:l(e.edge_props??`{}`)}}function p(e){return{id:e.node_id,type:e.node_type,name:e.node_name,properties:l(e.node_props??`{}`),sourceRecordId:e.node_src_rec??void 0,sourcePath:e.node_src_path??void 0,createdAt:e.node_created,community:e.node_community??void 0}}async function m(e){switch(e.backend){case`lancedb`:{let{LanceStore:t}=await import(`./lance-store-D1JolXlO.js`);return new t({path:e.path})}case`sqlite-vec`:{let{SqliteVecStore:t}=await import(`./sqlite-vec-store-
|
|
90
|
+
WHERE ps.node_id = ?`,[e]):this.query(`SELECT * FROM processes`);let n=[];for(let e of t){let t=this.query(`SELECT node_id FROM process_steps WHERE process_id = ? ORDER BY step_order`,[e.id]);n.push({id:e.id,entryNodeId:e.entry_node_id,label:e.label,properties:l(e.properties),steps:t.map(e=>e.node_id),createdAt:e.created_at})}return n}async deleteProcess(e){await this.ensureOpen();let t=this.getAdapter();t.exec(`BEGIN TRANSACTION`);try{this.run(`DELETE FROM process_steps WHERE process_id = ?`,[e]),this.run(`DELETE FROM processes WHERE id = ?`,[e]),t.exec(`COMMIT`),t.flush()}catch(e){throw t.exec(`ROLLBACK`),e}}async depthGroupedTraversal(e,t=3,n){await this.ensureOpen();let r=n?.direction??`both`,i=n?.edgeType,a=n?.limit??100,o={},s=new Set;s.add(e);let c=[e];for(let e=1;e<=t;e++){let t=[],n=[];for(let e of c){let o=await this.getNeighbors(e,{direction:r,edgeType:i,limit:a});for(let e of o.nodes)s.has(e.id)||(s.add(e.id),t.push(e.id),n.push(e))}if(n.length>0&&(o[e]=n),c=t,c.length===0||s.size>=a)break}return o}async getCohesionScore(e){await this.ensureOpen();let t=this.query(`SELECT id FROM nodes WHERE community = ?`,[e]);if(t.length===0)return 0;let n=new Set(t.map(e=>e.id)),r=t.map(()=>`?`).join(`,`),i=t.map(e=>e.id),a=this.query(`SELECT from_id, to_id FROM edges WHERE from_id IN (${r}) OR to_id IN (${r})`,[...i,...i]);if(a.length===0)return 0;let o=0;for(let e of a)n.has(e.from_id)&&n.has(e.to_id)&&o++;return o/a.length}async getSymbol360(e){let t=await this.getNode(e);if(!t)throw Error(`Node '${e}' not found`);let n=await this.findEdges({toId:e}),r=await this.findEdges({fromId:e}),i=await this.getProcesses(e);return{node:t,incoming:n,outgoing:r,community:t.community??null,processes:i}}releaseMemory(){if(this.adapter)try{this.adapter.exec(`PRAGMA shrink_memory`),this.adapter.exec(`PRAGMA wal_checkpoint(TRUNCATE)`)}catch{}}async close(){this.adapter&&=(this.externalAdapter||this.adapter.close(),null)}};function l(e){if(!e)return{};try{return JSON.parse(e)}catch{return{}}}function u(e){return{id:e.id,type:e.type,name:e.name,properties:l(e.properties),sourceRecordId:e.source_record_id??void 0,sourcePath:e.source_path??void 0,createdAt:e.created_at,community:e.community??void 0}}function d(e){return{id:e.id,fromId:e.from_id,toId:e.to_id,type:e.type,weight:e.weight??1,confidence:e.confidence??1,properties:l(e.properties)}}function f(e){return{id:e.edge_id,fromId:e.from_id,toId:e.to_id,type:e.edge_type,weight:e.weight??1,confidence:e.edge_confidence??1,properties:l(e.edge_props??`{}`)}}function p(e){return{id:e.node_id,type:e.node_type,name:e.node_name,properties:l(e.node_props??`{}`),sourceRecordId:e.node_src_rec??void 0,sourcePath:e.node_src_path??void 0,createdAt:e.node_created,community:e.node_community??void 0}}async function m(e){switch(e.backend){case`lancedb`:{let{LanceStore:t}=await import(`./lance-store-D1JolXlO.js`);return new t({path:e.path})}case`sqlite-vec`:{let{SqliteVecStore:t}=await import(`./sqlite-vec-store-xl2D7Iui.js`).then(e=>e.n);return new t({path:e.path,adapter:e.adapter,embeddingDim:e.embeddingDim})}default:{let t=e.backend;throw Error(`Unknown store backend: "${t}". Supported: lancedb, sqlite-vec`)}}}export{c as SqliteGraphStore,n as SqliteVecStore,t as createSqlJsAdapter,e as createSqliteAdapter,m as createStore};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as n,readFileSync as r,renameSync as i,unlinkSync as a,writeFileSync as o}from"node:fs";import{EMBEDDING_DEFAULTS as s,SEARCH_DEFAULTS as c,STORE_DEFAULTS as l,createLogger as u,serializeError as d,sourceTypeContentTypes as f}from"../../core/dist/index.js";import{dirname as p}from"node:path";var m=Object.defineProperty,h=(e,t)=>{let n={};for(var r in e)m(n,r,{get:e[r],enumerable:!0});return t||m(n,Symbol.toStringTag,{value:`Module`}),n},g=e(import.meta.url);const _=u(`sqlite-adapter`),v=e(import.meta.url);var y=class{type=`better-sqlite3`;vectorCapable=!1;db=null;stmtCache=new Map;async open(e){let t;try{t=v(`better-sqlite3`)}catch(e){throw Error(`better-sqlite3 native binding unavailable: ${e instanceof Error?e.message:String(e)}`)}this.db=new t(e),this.db.pragma(`journal_mode = WAL`),this.db.pragma(`foreign_keys = ON`),this.db.pragma(`synchronous = NORMAL`);try{v(`sqlite-vec`).load(this.db),this.vectorCapable=!0,_.info(`sqlite-vec extension loaded`)}catch(e){this.vectorCapable=!1,_.warn(`sqlite-vec extension failed to load; vector search disabled`,d(e))}}exec(e){this.getDb().exec(e)}pragma(e){this.getDb().pragma(e)}queryAll(e,t=[]){let n=this.prepareCached(e);return t.length>0?n.all(...t):n.all()}run(e,t=[]){let n=this.prepareCached(e);t.length>0?n.run(...t):n.run()}flush(){}close(){this.db&&=(this.stmtCache.clear(),this.db.close(),null)}getDb(){if(!this.db)throw Error(`BetterSqlite3Adapter: database not opened`);return this.db}prepareCached(e){let t=this.stmtCache.get(e);if(t)return t;let n=this.getDb().prepare(e);return this.stmtCache.set(e,n),n}};function b(e){return v.resolve(`sql.js/dist/${e}`)}var x=class{type=`sql.js`;vectorCapable=!1;db=null;dbPath=``;dirty=!1;inTransaction=!1;async open(e){this.dbPath=e;let n=(await import(`sql.js`)).default,i=await n({locateFile:e=>b(e)});if(t(e)){let t=r(e);this.db=new i.Database(t)}else this.db=new i.Database}exec(e){let t=e.trimStart().toUpperCase();this.getDb().run(e),t.startsWith(`BEGIN`)?this.inTransaction=!0:(t.startsWith(`COMMIT`)||t.startsWith(`ROLLBACK`))&&(this.inTransaction=!1),this.dirty=!0}pragma(e){this.getDb().exec(`PRAGMA ${e}`)}queryAll(e,t=[]){let n=this.getDb().prepare(e);try{t.length>0&&n.bind(t);let e=[];for(;n.step();)e.push(n.getAsObject());return e}finally{n.free()}}run(e,t=[]){let n=this.getDb(),r=e.trimStart().toUpperCase(),i=!this.inTransaction&&(r.startsWith(`INSERT`)||r.startsWith(`UPDATE`));i&&n.run(`SAVEPOINT fk_check`);try{if(t.length>0){let r=n.prepare(e);try{r.bind(t),r.step()}finally{r.free()}}else n.run(e);if(i){if(n.exec(`PRAGMA foreign_key_check`).length>0)throw n.run(`ROLLBACK TO fk_check`),n.run(`RELEASE fk_check`),Error(`FOREIGN KEY constraint failed`);n.run(`RELEASE fk_check`)}}catch(e){if(i)try{n.run(`ROLLBACK TO fk_check`),n.run(`RELEASE fk_check`)}catch{}throw e}this.dirty=!0}flush(){if(!this.dirty||!this.db)return;let e=this.db.export(),t=`${this.dbPath}.tmp`;o(t,Buffer.from(e)),i(t,this.dbPath),this.dirty=!1}close(){if(this.db){let e=this.db,t;try{this.flush()}catch(e){t=e;try{a(`${this.dbPath}.tmp`)}catch{}}try{e.close()}finally{this.db=null}if(t)throw t}}getDb(){if(!this.db)throw Error(`SqlJsAdapter: database not opened`);return this.db}};let
|
|
1
|
+
import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as n,readFileSync as r,renameSync as i,unlinkSync as a,writeFileSync as o}from"node:fs";import{EMBEDDING_DEFAULTS as s,SEARCH_DEFAULTS as c,STORE_DEFAULTS as l,createLogger as u,serializeError as d,sourceTypeContentTypes as f}from"../../core/dist/index.js";import{dirname as p}from"node:path";var m=Object.defineProperty,h=(e,t)=>{let n={};for(var r in e)m(n,r,{get:e[r],enumerable:!0});return t||m(n,Symbol.toStringTag,{value:`Module`}),n},g=e(import.meta.url);const _=u(`sqlite-adapter`),v=e(import.meta.url);var y=class{type=`better-sqlite3`;vectorCapable=!1;db=null;stmtCache=new Map;async open(e){let t;try{t=v(`better-sqlite3`)}catch(e){throw Error(`better-sqlite3 native binding unavailable: ${e instanceof Error?e.message:String(e)}`)}this.db=new t(e),this.db.pragma(`journal_mode = WAL`),this.db.pragma(`foreign_keys = ON`),this.db.pragma(`synchronous = NORMAL`);try{v(`sqlite-vec`).load(this.db),this.vectorCapable=!0,_.info(`sqlite-vec extension loaded`)}catch(e){this.vectorCapable=!1,_.warn(`sqlite-vec extension failed to load; vector search disabled`,d(e))}}exec(e){this.getDb().exec(e)}pragma(e){this.getDb().pragma(e)}queryAll(e,t=[]){let n=this.prepareCached(e);return t.length>0?n.all(...t):n.all()}run(e,t=[]){let n=this.prepareCached(e);t.length>0?n.run(...t):n.run()}flush(){}close(){this.db&&=(this.stmtCache.clear(),this.db.close(),null)}getDb(){if(!this.db)throw Error(`BetterSqlite3Adapter: database not opened`);return this.db}prepareCached(e){let t=this.stmtCache.get(e);if(t)return t;let n=this.getDb().prepare(e);return this.stmtCache.set(e,n),n}};function b(e){return v.resolve(`sql.js/dist/${e}`)}var x=class{type=`sql.js`;vectorCapable=!1;db=null;dbPath=``;dirty=!1;inTransaction=!1;async open(e){this.dbPath=e;let n=(await import(`sql.js`)).default,i=await n({locateFile:e=>b(e)});if(t(e)){let t=r(e);this.db=new i.Database(t)}else this.db=new i.Database}exec(e){let t=e.trimStart().toUpperCase();this.getDb().run(e),t.startsWith(`BEGIN`)?this.inTransaction=!0:(t.startsWith(`COMMIT`)||t.startsWith(`ROLLBACK`))&&(this.inTransaction=!1),this.dirty=!0}pragma(e){this.getDb().exec(`PRAGMA ${e}`)}queryAll(e,t=[]){let n=this.getDb().prepare(e);try{t.length>0&&n.bind(t);let e=[];for(;n.step();)e.push(n.getAsObject());return e}finally{n.free()}}run(e,t=[]){let n=this.getDb(),r=e.trimStart().toUpperCase(),i=!this.inTransaction&&(r.startsWith(`INSERT`)||r.startsWith(`UPDATE`));i&&n.run(`SAVEPOINT fk_check`);try{if(t.length>0){let r=n.prepare(e);try{r.bind(t),r.step()}finally{r.free()}}else n.run(e);if(i){if(n.exec(`PRAGMA foreign_key_check`).length>0)throw n.run(`ROLLBACK TO fk_check`),n.run(`RELEASE fk_check`),Error(`FOREIGN KEY constraint failed`);n.run(`RELEASE fk_check`)}}catch(e){if(i)try{n.run(`ROLLBACK TO fk_check`),n.run(`RELEASE fk_check`)}catch{}throw e}this.dirty=!0}flush(){if(!this.dirty||!this.db)return;let e=this.db.export(),t=`${this.dbPath}.tmp`;o(t,Buffer.from(e)),i(t,this.dbPath),this.dirty=!1}close(){if(this.db){let e=this.db,t;try{this.flush()}catch(e){t=e;try{a(`${this.dbPath}.tmp`)}catch{}}try{e.close()}finally{this.db=null}if(t)throw t}}getDb(){if(!this.db)throw Error(`SqlJsAdapter: database not opened`);return this.db}};async function S(){try{let{execSync:e}=await import(`node:child_process`),{createRequire:t}=await import(`node:module`),n=t(import.meta.url).resolve(`better-sqlite3/package.json`).replace(/[\\/]package\.json$/,``).replace(/[\\/]node_modules[\\/]better-sqlite3$/,``);return _.info(`Attempting native module rebuild for better-sqlite3`,{cwd:n}),e(`npm rebuild better-sqlite3`,{cwd:n,stdio:`pipe`,timeout:6e4}),_.info(`Native module rebuild completed successfully`),!0}catch(e){return _.warn(`Native module rebuild failed — continuing with sql.js fallback`,d(e)),!1}}let C=!1;async function w(e){let t=new y;try{return await t.open(e),t}catch(t){let n=t instanceof Error?t.message:String(t);if(/NODE_MODULE_VERSION/.test(n)&&await S()){let t=new y;try{return await t.open(e),_.info(`better-sqlite3 recovered after native module rebuild`),t}catch{}}C||(C=!0,_.warn(`[aikit] better-sqlite3 unavailable — falling back to sql.js (vector search disabled). Reinstall with prebuild support or rebuild from source to enable vector search.`,d(t)))}let n=new x;try{return await n.open(e),n}catch(e){let t=e instanceof Error?e.message:String(e);throw Error(`[aikit] SQLite adapter "sql.js" failed to load: ${t}`)}}async function T(e){let t=new x;return await t.open(e),t}var E=h({SqliteVecStore:()=>M});function D(e){let t=0;for(let n=0;n<e.length;n++){let r=e[n]<0?-e[n]:e[n];r>t&&(t=r)}let n=new Int8Array(e.length);if(t===0)return Buffer.from(n.buffer,n.byteOffset,n.byteLength);let r=127/t;for(let t=0;t<e.length;t++){let i=Math.round(e[t]*r);n[t]=i<-127?-127:i>127?127:i}return Buffer.from(n.buffer,n.byteOffset,n.byteLength)}const O=/^[\w.\-/ ]+$/,k=u(`sqlite-vec-store`);function A(e){if(!e)return[];try{let t=JSON.parse(e);return Array.isArray(t)?t:[]}catch{return[]}}function j(e,t){if(!O.test(e))throw Error(`Invalid ${t} filter value: contains disallowed characters`);return e}var M=class{adapter=null;externalAdapter;dbPath;embeddingDim;coarseDim;vectorEnabled=!1;warnedVectorDisabled=!1;_draining=!1;_priorityQueue=[];_normalQueue=[];_onCloseHooks=[];constructor(e={}){this.embeddingDim=e.embeddingDim??s.dimensions,this.coarseDim=Math.min(128,this.embeddingDim),e.adapter?(this.adapter=e.adapter,this.externalAdapter=!0,this.dbPath=``):(this.dbPath=e.path??`${l.path}/aikit.db`,this.externalAdapter=!1)}async initialize(){if(!this.adapter){let e=p(this.dbPath);t(e)||n(e,{recursive:!0}),this.adapter=await w(this.dbPath)}this.vectorEnabled=this.adapter.vectorCapable,this.createKnowledgeTable(),this.createFtsTable(),this.vectorEnabled?this.ensureVecTable():this.warnedVectorDisabled||(this.warnedVectorDisabled=!0,k.warn(`SqliteVecStore: vector search disabled (sqlite-vec extension not loaded). Hybrid search will return FTS-only results.`))}getDiagnostics(){let e=this.adapter,t=e?e.constructor.name:`unknown`,n=null,r=this.externalAdapter?``:this.dbPath;if(r)try{let{statSync:e}=g(`node:fs`);n=e(r).size}catch{n=null}return{adapterType:t,vectorSearchEnabled:this.vectorEnabled,degradedMode:!this.vectorEnabled,dbPath:r,dbSizeBytes:n,embeddingDim:this.embeddingDim,vectorDtype:`int8`}}createKnowledgeTable(){let e=this.getAdapter();e.exec(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS knowledge (
|
|
3
3
|
id TEXT PRIMARY KEY,
|
|
4
4
|
content TEXT NOT NULL,
|
|
@@ -23,7 +23,7 @@ import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as
|
|
|
23
23
|
content,
|
|
24
24
|
tokenize = 'unicode61 remove_diacritics 2'
|
|
25
25
|
)
|
|
26
|
-
`)}ensureVecTable(){let e=this.getAdapter(),t=e.queryAll(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name='vec_knowledge'`);if(t.length>0){let n=t[0].sql??``,r=n.match(/(?:float|int8)\[(\d+)\]/i),i=r?Number(r[1]):null,a=/int8\[/i.test(n);(i!==null&&i!==this.embeddingDim||!a)&&(
|
|
26
|
+
`)}ensureVecTable(){let e=this.getAdapter(),t=e.queryAll(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name='vec_knowledge'`);if(t.length>0){let n=t[0].sql??``,r=n.match(/(?:float|int8)\[(\d+)\]/i),i=r?Number(r[1]):null,a=/int8\[/i.test(n);(i!==null&&i!==this.embeddingDim||!a)&&(k.warn(`Vec table schema mismatch — dropping for recreation`,{existingDim:i,newDim:this.embeddingDim,wasInt8:a}),e.exec(`DROP TABLE vec_knowledge`))}e.exec(`
|
|
27
27
|
CREATE VIRTUAL TABLE IF NOT EXISTS vec_knowledge USING vec0(
|
|
28
28
|
embedding int8[${this.embeddingDim}] distance_metric=cosine,
|
|
29
29
|
+knowledge_id TEXT
|
|
@@ -33,7 +33,7 @@ import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as
|
|
|
33
33
|
embedding int8[${this.coarseDim}] distance_metric=cosine,
|
|
34
34
|
+knowledge_id TEXT
|
|
35
35
|
)
|
|
36
|
-
`)}enqueueWrite(e,t=!1){return new Promise((n,r)=>{let i=async()=>{try{n(await e())}catch(e){r(e)}};t?this._priorityQueue.push(i):this._normalQueue.push(i),this._drain()})}async _drain(){if(!this._draining){this._draining=!0;try{for(;this._priorityQueue.length>0||this._normalQueue.length>0;){let e=this._priorityQueue.shift()??this._normalQueue.shift();e&&await e()}}finally{this._draining=!1}}}async upsert(e,t){if(e.length!==0){if(e.length!==t.length)throw Error(`Record count (${e.length}) does not match vector count (${t.length})`);return this.enqueueWrite(()=>this._upsertImpl(e,t))}}async upsertInteractive(e,t){if(e.length!==0){if(e.length!==t.length)throw Error(`Record count (${e.length}) does not match vector count (${t.length})`);return this.enqueueWrite(()=>this._upsertImpl(e,t),!0)}}async upsertWithoutVector(e,t){return this.enqueueWrite(async()=>{let n=this.getAdapter(),r=e;n.exec(`BEGIN`);try{if(this.upsertKnowledgeRow(r),n.run(`DELETE FROM knowledge_fts WHERE id = ?`,[r.id]),n.run(`INSERT INTO knowledge_fts (id, content) VALUES (?, ?)`,[r.id,r.content]),this.vectorEnabled){let e=n.queryAll(`SELECT embedding FROM vec_knowledge WHERE knowledge_id = ?`,[t]);if(e.length>0){if(n.run(`DELETE FROM vec_knowledge WHERE knowledge_id = ?`,[r.id]),n.run(`INSERT INTO vec_knowledge (embedding, knowledge_id) VALUES (vec_int8(?), ?)`,[e[0].embedding,r.id]),this.coarseDim<this.embeddingDim){let e=n.queryAll(`SELECT embedding FROM vec_knowledge_coarse WHERE knowledge_id = ?`,[t]);e.length>0&&(n.run(`DELETE FROM vec_knowledge_coarse WHERE knowledge_id = ?`,[r.id]),n.run(`INSERT INTO vec_knowledge_coarse (embedding, knowledge_id) VALUES (vec_int8(?), ?)`,[e[0].embedding,r.id]))}}else
|
|
36
|
+
`)}enqueueWrite(e,t=!1){return new Promise((n,r)=>{let i=async()=>{try{n(await e())}catch(e){r(e)}};t?this._priorityQueue.push(i):this._normalQueue.push(i),this._drain()})}async _drain(){if(!this._draining){this._draining=!0;try{for(;this._priorityQueue.length>0||this._normalQueue.length>0;){let e=this._priorityQueue.shift()??this._normalQueue.shift();e&&await e()}}finally{this._draining=!1}}}async upsert(e,t){if(e.length!==0){if(e.length!==t.length)throw Error(`Record count (${e.length}) does not match vector count (${t.length})`);return this.enqueueWrite(()=>this._upsertImpl(e,t))}}async upsertInteractive(e,t){if(e.length!==0){if(e.length!==t.length)throw Error(`Record count (${e.length}) does not match vector count (${t.length})`);return this.enqueueWrite(()=>this._upsertImpl(e,t),!0)}}async upsertWithoutVector(e,t){return this.enqueueWrite(async()=>{let n=this.getAdapter(),r=e;n.exec(`BEGIN`);try{if(this.upsertKnowledgeRow(r),n.run(`DELETE FROM knowledge_fts WHERE id = ?`,[r.id]),n.run(`INSERT INTO knowledge_fts (id, content) VALUES (?, ?)`,[r.id,r.content]),this.vectorEnabled){let e=n.queryAll(`SELECT embedding FROM vec_knowledge WHERE knowledge_id = ?`,[t]);if(e.length>0){if(n.run(`DELETE FROM vec_knowledge WHERE knowledge_id = ?`,[r.id]),n.run(`INSERT INTO vec_knowledge (embedding, knowledge_id) VALUES (vec_int8(?), ?)`,[e[0].embedding,r.id]),this.coarseDim<this.embeddingDim){let e=n.queryAll(`SELECT embedding FROM vec_knowledge_coarse WHERE knowledge_id = ?`,[t]);e.length>0&&(n.run(`DELETE FROM vec_knowledge_coarse WHERE knowledge_id = ?`,[r.id]),n.run(`INSERT INTO vec_knowledge_coarse (embedding, knowledge_id) VALUES (vec_int8(?), ?)`,[e[0].embedding,r.id]))}}else k.warn(`upsertWithoutVector: source vector not found, record will lack vector`,{recordId:r.id,sourceRecordId:t})}n.exec(`COMMIT`)}catch(e){try{n.exec(`ROLLBACK`)}catch{}throw e}n.flush()})}upsertKnowledgeRow(e){this.getAdapter().run(`INSERT INTO knowledge (id, content, sourcePath, contentType, headingPath, chunkIndex,
|
|
37
37
|
totalChunks, startLine, endLine, fileHash, content_hash, indexedAt, origin, tags, category, version)
|
|
38
38
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
39
39
|
ON CONFLICT(id) DO UPDATE SET
|
|
@@ -51,7 +51,7 @@ import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as
|
|
|
51
51
|
origin = excluded.origin,
|
|
52
52
|
tags = excluded.tags,
|
|
53
53
|
category = excluded.category,
|
|
54
|
-
version = excluded.version`,[e.id,e.content,e.sourcePath,e.contentType,e.headingPath??``,e.chunkIndex,e.totalChunks,e.startLine,e.endLine,e.fileHash,e.contentHash??``,e.indexedAt,e.origin,JSON.stringify(e.tags??[]),e.category??``,e.version])}async _upsertImpl(e,t){let n=this.getAdapter();n.exec(`BEGIN`);try{for(let r=0;r<e.length;r++){let i=e[r];if(this.upsertKnowledgeRow(i),n.run(`DELETE FROM knowledge_fts WHERE id = ?`,[i.id]),n.run(`INSERT INTO knowledge_fts (id, content) VALUES (?, ?)`,[i.id,i.content]),this.vectorEnabled&&(n.run(`DELETE FROM vec_knowledge WHERE knowledge_id = ?`,[i.id]),n.run(`INSERT INTO vec_knowledge (embedding, knowledge_id) VALUES (vec_int8(?), ?)`,[
|
|
54
|
+
version = excluded.version`,[e.id,e.content,e.sourcePath,e.contentType,e.headingPath??``,e.chunkIndex,e.totalChunks,e.startLine,e.endLine,e.fileHash,e.contentHash??``,e.indexedAt,e.origin,JSON.stringify(e.tags??[]),e.category??``,e.version])}async _upsertImpl(e,t){let n=this.getAdapter();n.exec(`BEGIN`);try{for(let r=0;r<e.length;r++){let i=e[r];if(this.upsertKnowledgeRow(i),n.run(`DELETE FROM knowledge_fts WHERE id = ?`,[i.id]),n.run(`INSERT INTO knowledge_fts (id, content) VALUES (?, ?)`,[i.id,i.content]),this.vectorEnabled&&(n.run(`DELETE FROM vec_knowledge WHERE knowledge_id = ?`,[i.id]),n.run(`INSERT INTO vec_knowledge (embedding, knowledge_id) VALUES (vec_int8(?), ?)`,[D(t[r]),i.id]),this.coarseDim<this.embeddingDim)){let e=t[r].subarray(0,this.coarseDim);n.run(`DELETE FROM vec_knowledge_coarse WHERE knowledge_id = ?`,[i.id]),n.run(`INSERT INTO vec_knowledge_coarse (embedding, knowledge_id) VALUES (vec_int8(?), ?)`,[D(e),i.id])}}n.exec(`COMMIT`)}catch(e){try{n.exec(`ROLLBACK`)}catch{}throw e}n.flush()}async search(e,t){if(!this.vectorEnabled)return this.warnedVectorDisabled||(this.warnedVectorDisabled=!0,k.warn(`search() called but vector backend is disabled — returning []`)),[];let n=this.getAdapter(),r=t?.limit??c.maxResults,i=t?.minScore??c.minScore,a=r*4,o=`
|
|
55
55
|
SELECT k.*, v.distance AS _distance
|
|
56
56
|
FROM (
|
|
57
57
|
SELECT knowledge_id, distance
|
|
@@ -64,7 +64,7 @@ import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as
|
|
|
64
64
|
${this.buildFilterSqlSuffix(t)}
|
|
65
65
|
ORDER BY v.distance ASC
|
|
66
66
|
LIMIT ?
|
|
67
|
-
`,s;try{s=n.queryAll(o,[
|
|
67
|
+
`,s;try{s=n.queryAll(o,[D(e),a,r])}catch(e){return k.warn(`vector search failed`,d(e)),[]}return s.map(e=>({record:this.fromRow(e),score:1-(e._distance??1)})).filter(e=>e.score>=i).slice(0,r)}async coarseSearch(e,t){if(!this.vectorEnabled||this.coarseDim>=this.embeddingDim)return this.search(e,t);let n=this.getAdapter(),r=t?.limit??c.maxResults,i=t?.minScore??c.minScore,a=r*4,o=e.subarray(0,this.coarseDim),s=`
|
|
68
68
|
SELECT k.*, v.distance AS _distance
|
|
69
69
|
FROM (
|
|
70
70
|
SELECT knowledge_id, distance
|
|
@@ -77,7 +77,7 @@ import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as
|
|
|
77
77
|
${this.buildFilterSqlSuffix(t)}
|
|
78
78
|
ORDER BY v.distance ASC
|
|
79
79
|
LIMIT ?
|
|
80
|
-
`,l;try{l=n.queryAll(s,[
|
|
80
|
+
`,l;try{l=n.queryAll(s,[D(o),a,r])}catch(n){return k.warn(`coarse vector search failed, falling back to full search`,d(n)),this.search(e,t)}return l.length===0?this.search(e,t):l.map(e=>({record:this.fromRow(e),score:1-(e._distance??1)})).filter(e=>e.score>=i).slice(0,r)}async ftsSearch(e,t){if(!e||e.trim().length===0)return[];let n=this.getAdapter(),r=t?.limit??c.maxResults,i=`
|
|
81
81
|
SELECT k.*, bm25(knowledge_fts) AS _bm25
|
|
82
82
|
FROM knowledge_fts
|
|
83
83
|
JOIN knowledge k ON k.id = knowledge_fts.id
|
|
@@ -85,4 +85,4 @@ import{createRequire as e}from"node:module";import{existsSync as t,mkdirSync as
|
|
|
85
85
|
${this.buildFilterSqlSuffix(t,!0)}
|
|
86
86
|
ORDER BY _bm25 ASC
|
|
87
87
|
LIMIT ?
|
|
88
|
-
`,a;try{let t=
|
|
88
|
+
`,a;try{let t=N(e);a=n.queryAll(i,[t,r])}catch(e){return k.warn(`fts search failed`,d(e)),[]}return a.map(e=>({record:this.fromRow(e),score:P(e._bm25)}))}async getById(e){let t=this.getAdapter().queryAll(`SELECT * FROM knowledge WHERE id = ? LIMIT 1`,[e]);return t.length===0?null:this.fromRow(t[0])}async deleteBySourcePath(e){return this.enqueueWrite(()=>this._deleteBySourcePathImpl(e))}async _deleteBySourcePathImpl(e){let t=this.getAdapter(),n=t.queryAll(`SELECT id FROM knowledge WHERE sourcePath = ?`,[e]);if(n.length===0)return 0;t.exec(`BEGIN`);try{for(let{id:e}of n)this.vectorEnabled&&(t.run(`DELETE FROM vec_knowledge WHERE knowledge_id = ?`,[e]),this.coarseDim<this.embeddingDim&&t.run(`DELETE FROM vec_knowledge_coarse WHERE knowledge_id = ?`,[e])),t.run(`DELETE FROM knowledge_fts WHERE id = ?`,[e]);t.run(`DELETE FROM knowledge WHERE sourcePath = ?`,[e]),t.exec(`COMMIT`)}catch(e){try{t.exec(`ROLLBACK`)}catch{}throw e}return t.flush(),n.length}async deleteById(e){return this.enqueueWrite(()=>this._deleteByIdImpl(e))}async deleteByIdInteractive(e){return this.enqueueWrite(()=>this._deleteByIdImpl(e),!0)}async _deleteByIdImpl(e){let t=this.getAdapter();if(t.queryAll(`SELECT id FROM knowledge WHERE id = ? LIMIT 1`,[e]).length===0)return!1;t.exec(`BEGIN`);try{this.vectorEnabled&&(t.run(`DELETE FROM vec_knowledge WHERE knowledge_id = ?`,[e]),this.coarseDim<this.embeddingDim&&t.run(`DELETE FROM vec_knowledge_coarse WHERE knowledge_id = ?`,[e])),t.run(`DELETE FROM knowledge_fts WHERE id = ?`,[e]),t.run(`DELETE FROM knowledge WHERE id = ?`,[e]),t.exec(`COMMIT`)}catch(e){try{t.exec(`ROLLBACK`)}catch{}throw e}return t.flush(),!0}async getBySourcePath(e){return this.getAdapter().queryAll(`SELECT * FROM knowledge WHERE sourcePath = ? ORDER BY chunkIndex ASC`,[e]).map(e=>this.fromRow(e))}async getStats(){let e=this.getAdapter(),t=e.queryAll(`SELECT COUNT(*) AS n FROM knowledge`)[0]?.n??0;if(t===0)return{totalRecords:0,totalFiles:0,contentTypeBreakdown:{},lastIndexedAt:null,storeBackend:`sqlite-vec`,embeddingModel:s.model};let n=e.queryAll(`SELECT COUNT(DISTINCT sourcePath) AS n FROM knowledge`),r=e.queryAll(`SELECT contentType, COUNT(*) AS n FROM knowledge GROUP BY contentType`),i=e.queryAll(`SELECT MAX(indexedAt) AS ts FROM knowledge`),a={};for(let e of r)a[e.contentType]=e.n;return{totalRecords:t,totalFiles:n[0]?.n??0,contentTypeBreakdown:a,lastIndexedAt:i[0]?.ts??null,storeBackend:`sqlite-vec`,embeddingModel:s.model}}async listSourcePaths(){return this.getAdapter().queryAll(`SELECT DISTINCT sourcePath FROM knowledge ORDER BY sourcePath`).map(e=>e.sourcePath)}async createFtsIndex(){this.createFtsTable()}async dropTable(){return this.enqueueWrite(()=>this._dropTableImpl())}async _dropTableImpl(){let e=this.getAdapter();e.exec(`DROP TABLE IF EXISTS knowledge_fts`),this.vectorEnabled&&(this.coarseDim<this.embeddingDim&&e.exec(`DROP TABLE IF EXISTS vec_knowledge_coarse`),e.exec(`DROP TABLE IF EXISTS vec_knowledge`)),e.exec(`DROP TABLE IF EXISTS knowledge`),e.flush(),this.createKnowledgeTable(),this.createFtsTable(),this.vectorEnabled&&this.ensureVecTable()}releaseMemory(){if(this.adapter)try{this.adapter.exec(`PRAGMA shrink_memory`),this.adapter.exec(`PRAGMA wal_checkpoint(TRUNCATE)`)}catch{}}onBeforeClose(e){this._onCloseHooks.push(e)}async close(){for(let e of this._onCloseHooks)try{e()}catch{}for(this._onCloseHooks.length=0;this._priorityQueue.length>0||this._normalQueue.length>0||this._draining;)await new Promise(e=>setTimeout(e,5));this.adapter&&!this.externalAdapter&&this.adapter.close(),this.adapter=null}getAdapter(){if(!this.adapter)throw Error(`SqliteVecStore: not initialized — call initialize() first`);return this.adapter}buildFilterSqlSuffix(e,t=!1){if(!e)return``;let n=[];if(e.contentType&&n.push(`k.contentType = '${j(e.contentType,`contentType`)}'`),e.sourceType){let t=f(e.sourceType);if(t.length>0){let e=t.map(e=>`'${j(e,`sourceType`)}'`).join(`, `);n.push(`k.contentType IN (${e})`)}}if(e.origin&&n.push(`k.origin = '${j(e.origin,`origin`)}'`),e.category&&n.push(`k.category = '${j(e.category,`category`)}'`),e.tags&&e.tags.length>0){let t=e.tags.map(e=>`k.tags LIKE '%' || '${j(e,`tag`)}' || '%'`);n.push(`(${t.join(` OR `)})`)}if(n.length===0)return``;let r=n.join(` AND `);return t?`AND ${r}`:`WHERE ${r}`}fromRow(e){return{id:e.id,content:e.content,sourcePath:e.sourcePath,contentType:e.contentType,headingPath:e.headingPath||void 0,chunkIndex:e.chunkIndex,totalChunks:e.totalChunks,startLine:e.startLine,endLine:e.endLine,fileHash:e.fileHash,contentHash:e.content_hash||void 0,indexedAt:e.indexedAt,origin:e.origin,tags:A(e.tags),category:e.category||void 0,version:e.version}}};function N(e){let t=e.replace(/["'()*:^-]/g,` `).trim();return t?t.split(/\s+/).filter(e=>e.length>0).map(e=>`"${e}"`).join(` OR `):`""`}function P(e){return e==null||Number.isNaN(e)?0:1-Math.exp(-Math.abs(e)/5)}export{w as i,E as n,T as r,M as t};
|