@gencode/agents 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/builtin-provider-_CdSlTJE.js +71 -0
- package/dist/index.d.ts +61 -11
- package/dist/index.js +81 -78
- package/package.json +3 -1
- package/dist/builtin-provider-BX9aHHJK.js +0 -71
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gencode/agents",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
"isbinaryfile": "^5.0.6",
|
|
30
30
|
"jiti": "^2.6.0",
|
|
31
31
|
"log4js": "^6.9.1",
|
|
32
|
+
"lz-string": "^1.5.0",
|
|
32
33
|
"openai": "6.10.0",
|
|
33
34
|
"sqlite-vec": "^0.1.6",
|
|
34
35
|
"zod": "^4.3.6",
|
|
@@ -36,6 +37,7 @@
|
|
|
36
37
|
},
|
|
37
38
|
"devDependencies": {
|
|
38
39
|
"@sinclair/typebox": "^0.34.48",
|
|
40
|
+
"@types/lz-string": "^1.5.0",
|
|
39
41
|
"@types/node": "^22.0.0",
|
|
40
42
|
"@vitest/coverage-v8": "^4.0.18",
|
|
41
43
|
"typescript": "^5.9.3",
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
import{t as e}from"./rolldown-runtime-CNxR59P3.js";import{createRequire as t}from"node:module";import n from"node:fs/promises";import r,{basename as i}from"node:path";import a from"node:fs";import o,{randomUUID as s}from"node:crypto";const c=`sessions`,l=2e3,u=new Map;function d(e){let t=e?.trim()||c;if(!/^[A-Za-z0-9_-]+$/.test(t))throw Error(`Invalid session store name: ${e}. Must match /^[A-Za-z0-9_-]+$/`);return t}function f(e,t){let n=t.trim();if(!/^[A-Za-z0-9_-]+$/.test(n))throw Error(`Invalid ${e}: ${t}. Must match /^[A-Za-z0-9_-]+$/`);return n}function p(e,t){return r.join(e,`.aimax`,d(t?.storeName))}function m(e,t,n){let i=n?.subagent?f(`subagent session id`,t):t;if(n?.subagent){if(n.subagent.parentDir)return r.join(n.subagent.parentDir,`subagents`,i);let t=f(`parent session id`,n.subagent.parentSessionId);return r.join(p(e,n),t,`subagents`,i)}return r.join(p(e,n),i)}function h(e,t,n){return r.join(m(e,t,n),`transcript.jsonl`)}function g(e,t,n){return r.join(m(e,t,n),`artifacts.json`)}function _(e,t,n){return r.join(m(e,t,n),`session.json`)}function v(e,t,n){return r.join(m(e,t,n),`context.json`)}function y(e,t,n){return r.join(m(e,t,n),`session-memory.json`)}function ee(e,t,n){return r.join(m(e,t,n),`collapse-log.jsonl`)}function b(e,t,n){return r.join(m(e,t,n),`tool-results`)}function x(e,t,n){return r.join(m(e,t,n),`cron-executions.jsonl`)}const S={version:2,updatedAt:new Date(0).toISOString(),readStates:[],toolResults:[],compaction:{budgets:[],snips:[],collapseSpans:[],consecutiveAutocompactFailures:0}};async function te(e,t,i){let a=i?.sessionId??s(),o=m(e,a,i);await n.mkdir(o,{recursive:!0});let c=r.join(o,`.channel`);return await n.writeFile(c,t,`utf-8`),a}async function C(e,t,r){let i=m(e,t,r);await n.mkdir(i,{recursive:!0})}async function w(e,t,r){let i=h(e,t,r),a;try{a=await n.readFile(i,`utf-8`)}catch(e){if(e.code===`ENOENT`)return[];throw e}let o=[];for(let e of a.split(`
|
|
2
|
-
`)){let t=e.trim();if(t)try{o.push(JSON.parse(t))}catch{}}return o}async function ne(e,t,n,r){await C(e,t,r);let i=g(e,t,r);await oe(i,async()=>{let e=await T(i);await ae(i,{version:1,updatedAt:new Date().toISOString(),operations:[...e.operations,ie(n)]})})}async function re(e,t,n){return(await T(g(e,t,n))).operations}function ie(e){let t=e.content.length,n=t>l;return{...e,content:n?`${e.content.slice(0,Math.max(0,l-3))}...`:e.content,timestamp:new Date().toISOString(),truncated:n,originalChars:t}}async function T(e){let t;try{t=await n.readFile(e,`utf-8`)}catch(e){if(e.code===`ENOENT`)return E();throw e}try{let e=JSON.parse(t);return e.version!==1||!Array.isArray(e.operations)?E():{version:1,updatedAt:typeof e.updatedAt==`string`?e.updatedAt:new Date(0).toISOString(),operations:e.operations}}catch{return E()}}function E(){return{version:1,updatedAt:new Date(0).toISOString(),operations:[]}}async function ae(e,t){await n.mkdir(r.dirname(e),{recursive:!0});let i=`${e}.${process.pid}.${Date.now()}.tmp`;await n.writeFile(i,`${JSON.stringify(t,null,2)}\n`,`utf-8`),await n.rename(i,e)}async function oe(e,t){let n=(u.get(e)??Promise.resolve()).catch(()=>{}).then(t);u.set(e,n);try{await n}finally{u.get(e)===n&&u.delete(e)}}async function se(e,t,i,a){await C(e,t,a);let o=h(e,t,a),s=JSON.stringify(i)+`
|
|
3
|
-
`;await n.appendFile(o,s,`utf-8`);try{let{resolveMemoryProvider:t}=await import(`./provider-registry-CxtHsHdk.js`).then(e=>e.t),{createBuiltinMemoryProvider:n}=await Promise.resolve().then(()=>$),i=r.join(e,`.aimax`),s=t({providerId:a?.providerId,pluginId:a?.pluginId,dataDir:e,memoryDir:i,sessionStoreName:a?.storeName})?.provider??n({dataDir:e,memoryDir:i,sessionStoreName:a?.storeName});s.noteSessionUpdate&&s.noteSessionUpdate(o)}catch{}a?.onMemoryChanged&&await Promise.resolve(a.onMemoryChanged({reason:`transcript-append`,files:[r.join(d(a.storeName),t,`transcript.jsonl`).replace(/\\/g,`/`)],source:`sessions`,sessionId:t,providerId:a.providerId??a.pluginId,timestamp:new Date().toISOString()})).catch(()=>{})}async function ce(e,t,i,a){await C(e,t,a);let o=h(e,t,a),s=i(await w(e,t,a)),c=s.map(e=>JSON.stringify(e)).join(`
|
|
4
|
-
`);await n.writeFile(o,c.length>0?`${c}\n`:``,`utf-8`);try{let{resolveMemoryProvider:t}=await import(`./provider-registry-CxtHsHdk.js`).then(e=>e.t),{createBuiltinMemoryProvider:n}=await Promise.resolve().then(()=>$),i=r.join(e,`.aimax`),s=t({providerId:a?.providerId,pluginId:a?.pluginId,dataDir:e,memoryDir:i,sessionStoreName:a?.storeName})?.provider??n({dataDir:e,memoryDir:i,sessionStoreName:a?.storeName});s.noteSessionUpdate&&s.noteSessionUpdate(o)}catch{}return a?.onMemoryChanged&&await Promise.resolve(a.onMemoryChanged({reason:`transcript-rewrite`,files:[r.join(d(a.storeName),t,`transcript.jsonl`).replace(/\\/g,`/`)],source:`sessions`,sessionId:t,providerId:a.providerId??a.pluginId,timestamp:new Date().toISOString()})).catch(()=>{}),s}async function le(e,t,r,i){await C(e,t,i);let a=x(e,t,i),o=JSON.stringify(r)+`
|
|
5
|
-
`;await n.appendFile(a,o,`utf-8`)}async function ue(e,t,r){let i=x(e,t,r),a;try{a=await n.readFile(i,`utf-8`)}catch(e){if(e.code===`ENOENT`)return[];throw e}let o=[];for(let e of a.split(`
|
|
6
|
-
`)){let t=e.trim();if(t)try{o.push(JSON.parse(t))}catch{}}return o}async function D(e,t){let r=p(e,t);try{return(await n.readdir(r,{withFileTypes:!0})).filter(e=>e.isDirectory()).map(e=>e.name).sort()}catch(e){if(e.code===`ENOENT`)return[];throw e}}async function O(e,t,r){await C(e,t.id,r);let i=_(e,t.id,r);await n.writeFile(i,JSON.stringify(t,null,2),`utf-8`)}async function de(e,t,n,r){let i=await k(e,t,r);if(!i)return null;let a={...i,...n,id:i.id,createdAt:i.createdAt,updatedAt:new Date().toISOString()};return await O(e,a,r),a}async function k(e,t,i){let a=_(e,t,i);try{let o=await n.readFile(a,`utf-8`),s=JSON.parse(o);if(!s.channel){let a=r.join(m(e,t,i),`.channel`);try{s.channel=(await n.readFile(a,`utf-8`)).trim()}catch{s.channel=`WEB`}}return s}catch(e){if(e.code===`ENOENT`)return null;throw e}}async function fe(e,t,n){let r=await D(e,n),i=await Promise.all(r.map(async t=>{let r=await k(e,t,n);return{id:t,title:r?.title??t,channel:r?.channel??`WEB`,createdAt:r?.createdAt??``,updatedAt:r?.updatedAt??``}}));return t?i.filter(e=>e.channel===t):i}async function A(e,t,r){let i=v(e,t,r);try{let e=await n.readFile(i,`utf-8`),t=JSON.parse(e),r=typeof t.version==`number`?t.version:void 0;return r!==1&&r!==2?S:{version:2,updatedAt:typeof t.updatedAt==`string`?t.updatedAt:S.updatedAt,readStates:Array.isArray(t.readStates)?t.readStates:[],toolResults:Array.isArray(t.toolResults)?t.toolResults:[],compaction:{sessionMemory:t.compaction?.sessionMemory,modelUsage:t.compaction?.modelUsage,budgets:Array.isArray(t.compaction?.budgets)?t.compaction.budgets:[],snips:Array.isArray(t.compaction?.snips)?t.compaction.snips:[],collapseSpans:Array.isArray(t.compaction?.collapseSpans)?t.compaction.collapseSpans:[],consecutiveAutocompactFailures:typeof t.compaction?.consecutiveAutocompactFailures==`number`?t.compaction.consecutiveAutocompactFailures:0,lastCompactionAt:typeof t.compaction?.lastCompactionAt==`string`?t.compaction.lastCompactionAt:void 0,lastCompactionLayer:t.compaction?.lastCompactionLayer===`L1`||t.compaction?.lastCompactionLayer===`L2`||t.compaction?.lastCompactionLayer===`L3`||t.compaction?.lastCompactionLayer===`L4`||t.compaction?.lastCompactionLayer===`L5`||t.compaction?.lastCompactionLayer===`L6`?t.compaction.lastCompactionLayer:void 0}}}catch(e){if(e.code===`ENOENT`)return S;throw e}}async function pe(e,t,r){let i=m(e,t,r);try{return(await n.stat(i)).isDirectory()}catch(e){if(e.code===`ENOENT`)return!1;throw e}}async function j(e,t,n){let[r,i,a]=await Promise.all([k(e,t,n),w(e,t,n),A(e,t,n)]);return{id:t,metadata:r,transcriptPath:h(e,t,n),contextSnapshotPath:v(e,t,n),sessionMemoryPath:y(e,t,n),collapseLogPath:ee(e,t,n),toolResultsDir:b(e,t,n),transcriptEntryCount:i.length,readStateCount:a.readStates.length,toolResultRefCount:a.toolResults.length,transcriptEntries:i,context:a}}async function me(e,t,n){let r=await j(e,t,n);return{id:r.id,metadata:r.metadata,transcript:r.transcriptEntries,context:r.context,paths:{transcriptPath:r.transcriptPath,contextSnapshotPath:r.contextSnapshotPath,sessionMemoryPath:r.sessionMemoryPath,collapseLogPath:r.collapseLogPath,toolResultsDir:r.toolResultsDir}}}function M(e){return!!(e&&typeof e==`object`&&`code`in e&&e.code===`ENOENT`)}async function he(e){let t;try{t=await n.lstat(e)}catch(e){if(M(e))return{missing:!0};throw e}if(t.isSymbolicLink()||!t.isFile())throw Error(`path required`);return{missing:!1,stat:t}}function ge(e){try{a.mkdirSync(e,{recursive:!0})}catch{}return e}function _e(e){return e.trim().replace(/^[./]+/,``).replace(/\\/g,`/`)}function ve(e){let t=_e(e);return t?t===`MEMORY.md`||t===`memory.md`?!0:t.startsWith(`memory/`):!1}async function N(e,t){let i=await n.readdir(e,{withFileTypes:!0});for(let n of i){let i=r.join(e,n.name);if(!n.isSymbolicLink()){if(n.isDirectory()){await N(i,t);continue}n.isFile()&&n.name.endsWith(`.md`)&&t.push(i)}}}async function P(e){let t=[],i=r.join(e,`MEMORY.md`),a=r.join(e,`memory.md`),o=r.join(e,`memory`),s=async e=>{try{let r=await n.lstat(e);if(r.isSymbolicLink()||!r.isFile()||!e.endsWith(`.md`))return;t.push(e)}catch{}};await s(i),await s(a);try{let e=await n.lstat(o);!e.isSymbolicLink()&&e.isDirectory()&&await N(o,t)}catch{}if(t.length<=1)return t;let c=new Set,l=[];for(let e of t){let t=e;try{t=await n.realpath(e)}catch{}c.has(t)||(c.add(t),l.push(e))}return l}function F(e){return o.createHash(`sha256`).update(e).digest(`hex`)}async function ye(e,t){let i;try{i=await n.stat(e)}catch(e){if(M(e))return null;throw e}let a;try{a=await n.readFile(e,`utf-8`)}catch(e){if(M(e))return null;throw e}let o=F(a);return{path:r.relative(t,e).replace(/\\/g,`/`),absPath:e,mtimeMs:i.mtimeMs,size:i.size,hash:o}}function be(e,t){let n=e.split(`
|
|
7
|
-
`);if(n.length===0)return[];let r=Math.max(32,t.tokens*4),i=Math.max(0,t.overlap*4),a=[],o=[],s=0,c=()=>{if(o.length===0)return;let e=o[0],t=o[o.length-1];if(!e||!t)return;let n=o.map(e=>e.line).join(`
|
|
8
|
-
`),r=e.lineNo,i=t.lineNo;a.push({startLine:r,endLine:i,text:n,hash:F(n)})},l=()=>{if(i<=0||o.length===0){o=[],s=0;return}let e=0,t=[];for(let n=o.length-1;n>=0;--n){let r=o[n];if(r&&(e+=r.line.length+1,t.unshift(r),e>=i))break}o=t,s=t.reduce((e,t)=>e+t.line.length+1,0)};for(let e=0;e<n.length;e+=1){let t=n[e]??``,i=e+1,a=[];if(t.length===0)a.push(``);else for(let e=0;e<t.length;e+=r)a.push(t.slice(e,e+r));for(let e of a){let t=e.length+1;s+t>r&&o.length>0&&(c(),l()),o.push({line:e,lineNo:i}),s+=t}}return c(),a}function xe(e,t){if(!(!t||t.length===0))for(let n of e)n.startLine=t[n.startLine-1]??n.startLine,n.endLine=t[n.endLine-1]??n.endLine}function Se(e){try{let t=JSON.parse(e);return Array.isArray(t)?t:[]}catch{return[]}}function Ce(e,t){if(e.length===0||t.length===0)return 0;let n=Math.min(e.length,t.length),r=0,i=0,a=0;for(let o=0;o<n;o+=1){let n=e[o]??0,s=t[o]??0;r+=n*s,i+=n*n,a+=s*s}return i===0||a===0?0:r/(Math.sqrt(i)*Math.sqrt(a))}function we(e){return(e.toLowerCase().match(/[\p{L}\p{N}_]+/gu)??[]).filter(Boolean)}function Te(e){return o.createHash(`sha256`).update(e).digest().readUInt32BE(0)}function I(e,t=384){let n=Array(t).fill(0),r=we(e);if(r.length===0)return n;for(let e of r){let r=Te(e)%t;n[r]+=1}let i=Math.sqrt(n.reduce((e,t)=>e+t*t,0));if(i>0)for(let e=0;e<n.length;e+=1)n[e]=n[e]/i;return n}function L(e){let t=e?.model?.trim()||`mock-embedding-v1`,n=e?.dimensions??384;return{id:`external-mock`,model:t,embedQuery:async e=>I(e,n),embedBatch:async e=>e.map(e=>I(e,n))}}const R=new class{providers=new Map;byPluginId=new Map;register(e){this.providers.set(e.id,e),e.pluginId&&this.byPluginId.set(e.pluginId,e.id)}getById(e){return this.providers.get(e)}getByPluginId(e){let t=this.byPluginId.get(e);return t?this.providers.get(t):void 0}clear(){this.providers.clear(),this.byPluginId.clear()}};function Ee(e){let t=e.id?.trim()||e.pluginId;return R.register({id:t,pluginId:e.pluginId,create:e.create,config:e.config,rootDir:e.rootDir,source:e.source}),t}function z(e){let t=e.providerId?.trim(),n=e.pluginId?.trim(),r=(t?R.getById(t):void 0)??(n?R.getByPluginId(n):void 0);return r?{provider:r.create({dataDir:e.dataDir,memoryDir:e.memoryDir,pluginId:r.pluginId,config:r.config,rootDir:r.rootDir,source:r.source}),registration:r}:null}function De(){R.clear()}const B={enabled:!1,lambda:.7};function V(e){let t=e.toLowerCase().match(/[a-z0-9_]+/g)??[];return new Set(t)}function Oe(e,t){if(e.size===0&&t.size===0)return 1;if(e.size===0||t.size===0)return 0;let n=0,r=e.size<=t.size?e:t,i=e.size<=t.size?t:e;for(let e of r)i.has(e)&&n++;let a=e.size+t.size-n;return a===0?0:n/a}function ke(e,t,n){if(t.length===0)return 0;let r=0,i=n.get(e.id)??V(e.content);for(let e of t){let t=Oe(i,n.get(e.id)??V(e.content));t>r&&(r=t)}return r}function Ae(e,t,n){return n*e-(1-n)*t}function je(e,t={}){let{enabled:n=B.enabled,lambda:r=B.lambda}=t;if(!n||e.length<=1)return[...e];let i=Math.max(0,Math.min(1,r));if(i===1)return e.slice().sort((e,t)=>t.score-e.score);let a=new Map;for(let t of e)a.set(t.id,V(t.content));let o=Math.max(...e.map(e=>e.score)),s=Math.min(...e.map(e=>e.score)),c=o-s,l=e=>c===0?1:(e-s)/c,u=[],d=new Set(e);for(;d.size>0;){let e=null,t=-1/0;for(let n of d){let r=Ae(l(n.score),ke(n,u,a),i);(r>t||r===t&&n.score>(e?.score??-1/0))&&(t=r,e=n)}if(e)u.push(e),d.delete(e);else break}return u}function Me(e,t={}){if(e.length===0)return e;let n=new Map;return je(e.map((e,t)=>{let r=`${e.path}:${e.startLine}:${t}`;return n.set(r,e),{id:r,score:e.score,content:e.snippet}}),t).map(e=>n.get(e.id))}const H={enabled:!1,halfLifeDays:30},U=/(?:^|\/)(?:memory)\/(\d{4})-(\d{2})-(\d{2})\.md$/;function Ne(e){return!Number.isFinite(e)||e<=0?0:Math.LN2/e}function Pe(e){let t=Ne(e.halfLifeDays),n=Math.max(0,e.ageInDays);return t<=0||!Number.isFinite(n)?1:Math.exp(-t*n)}function Fe(e){return e.score*Pe(e)}function Ie(e){let t=e.replaceAll(`\\`,`/`).replace(/^\.\//,``),n=U.exec(t);if(!n)return null;let r=Number(n[1]),i=Number(n[2]),a=Number(n[3]);if(!Number.isInteger(r)||!Number.isInteger(i)||!Number.isInteger(a))return null;let o=Date.UTC(r,i-1,a),s=new Date(o);return s.getUTCFullYear()!==r||s.getUTCMonth()!==i-1||s.getUTCDate()!==a?null:s}function Le(e){let t=e.replaceAll(`\\`,`/`).replace(/^\.\//,``);return t===`MEMORY.md`||t===`memory.md`?!0:t.startsWith(`memory/`)?!U.test(t):!1}async function Re(e){let t=Ie(e.filePath);if(t)return t;if(e.source===`memory`&&Le(e.filePath)||!e.memoryRoot)return null;let i=r.isAbsolute(e.filePath)?e.filePath:r.resolve(e.memoryRoot,e.filePath);try{let e=await n.stat(i);return Number.isFinite(e.mtimeMs)?new Date(e.mtimeMs):null}catch{return null}}function ze(e,t){return Math.max(0,t-e.getTime())/864e5}async function Be(e){let t={...H,...e.temporalDecay};if(!t.enabled)return[...e.results];let n=e.nowMs??Date.now(),r=new Map;return Promise.all(e.results.map(async i=>{let a=`${i.source}:${i.path}`,o=r.get(a);o||(o=Re({filePath:i.path,source:i.source,memoryRoot:e.memoryRoot}),r.set(a,o));let s=await o;if(!s)return i;let c=Fe({score:i.score,ageInDays:ze(s,n),halfLifeDays:t.halfLifeDays});return{...i,score:c}}))}function Ve(e){let t=e.match(/[\p{L}\p{N}_]+/gu)?.map(e=>e.trim()).filter(Boolean)??[];return t.length===0?null:t.map(e=>`"${e.replaceAll(`"`,``)}"`).join(` AND `)}function He(e){return 1/(1+(Number.isFinite(e)?Math.max(0,e):999))}async function Ue(e){let t=new Map;for(let n of e.vector)t.set(n.id,{id:n.id,path:n.path,startLine:n.startLine,endLine:n.endLine,source:n.source,snippet:n.snippet,vectorScore:n.vectorScore,textScore:0});for(let n of e.keyword){let e=t.get(n.id);e?(e.textScore=n.textScore,n.snippet&&n.snippet.length>0&&(e.snippet=n.snippet)):t.set(n.id,{id:n.id,path:n.path,startLine:n.startLine,endLine:n.endLine,source:n.source,snippet:n.snippet,vectorScore:0,textScore:n.textScore})}let n=(await Be({results:Array.from(t.values()).map(t=>{let n=e.vectorWeight*t.vectorScore+e.textWeight*t.textScore;return{path:t.path,startLine:t.startLine,endLine:t.endLine,score:n,snippet:t.snippet,source:t.source}}),temporalDecay:{...H,...e.temporalDecay},memoryRoot:e.memoryRoot,nowMs:e.nowMs})).slice().sort((e,t)=>t.score-e.score),r={...B,...e.mmr};return r.enabled?Me(n,r):n}function We(e){e.db.exec(`
|
|
9
|
-
CREATE TABLE IF NOT EXISTS meta (
|
|
10
|
-
key TEXT PRIMARY KEY,
|
|
11
|
-
value TEXT NOT NULL
|
|
12
|
-
);
|
|
13
|
-
`),e.db.exec(`
|
|
14
|
-
CREATE TABLE IF NOT EXISTS files (
|
|
15
|
-
path TEXT PRIMARY KEY,
|
|
16
|
-
source TEXT NOT NULL DEFAULT 'memory',
|
|
17
|
-
hash TEXT NOT NULL,
|
|
18
|
-
mtime INTEGER NOT NULL,
|
|
19
|
-
size INTEGER NOT NULL
|
|
20
|
-
);
|
|
21
|
-
`),e.db.exec(`
|
|
22
|
-
CREATE TABLE IF NOT EXISTS chunks (
|
|
23
|
-
id TEXT PRIMARY KEY,
|
|
24
|
-
path TEXT NOT NULL,
|
|
25
|
-
source TEXT NOT NULL DEFAULT 'memory',
|
|
26
|
-
start_line INTEGER NOT NULL,
|
|
27
|
-
end_line INTEGER NOT NULL,
|
|
28
|
-
hash TEXT NOT NULL,
|
|
29
|
-
model TEXT NOT NULL,
|
|
30
|
-
text TEXT NOT NULL,
|
|
31
|
-
embedding TEXT NOT NULL,
|
|
32
|
-
updated_at INTEGER NOT NULL
|
|
33
|
-
);
|
|
34
|
-
`),e.db.exec(`
|
|
35
|
-
CREATE TABLE IF NOT EXISTS ${e.embeddingCacheTable} (
|
|
36
|
-
provider TEXT NOT NULL,
|
|
37
|
-
model TEXT NOT NULL,
|
|
38
|
-
provider_key TEXT NOT NULL,
|
|
39
|
-
hash TEXT NOT NULL,
|
|
40
|
-
embedding TEXT NOT NULL,
|
|
41
|
-
dims INTEGER,
|
|
42
|
-
updated_at INTEGER NOT NULL,
|
|
43
|
-
PRIMARY KEY (provider, model, provider_key, hash)
|
|
44
|
-
);
|
|
45
|
-
`),e.db.exec(`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${e.embeddingCacheTable}(updated_at);`);let t=!1,n;if(e.ftsEnabled)try{e.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${e.ftsTable} USING fts5(\n text,\n id UNINDEXED,\n path UNINDEXED,\n source UNINDEXED,\n model UNINDEXED,\n start_line UNINDEXED,\n end_line UNINDEXED\n);`),t=!0}catch(e){let r=e instanceof Error?e.message:String(e);t=!1,n=r}return W(e.db,`files`,`source`,`TEXT NOT NULL DEFAULT 'memory'`),W(e.db,`chunks`,`source`,`TEXT NOT NULL DEFAULT 'memory'`),e.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`),e.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`),{ftsAvailable:t,...n?{ftsError:n}:{}}}function W(e,t,n,r){e.prepare(`PRAGMA table_info(${t})`).all().some(e=>e.name===n)||e.exec(`ALTER TABLE ${t} ADD COLUMN ${n} ${r}`)}function G(e,t){return e.length<=t?e:e.slice(0,t)}const Ge=e=>Buffer.from(new Float32Array(e).buffer);async function Ke(e){return e.queryVec.length===0||e.limit<=0?[]:await e.ensureVectorReady(e.queryVec.length)?e.db.prepare(`SELECT c.id, c.path, c.start_line, c.end_line, c.text,
|
|
46
|
-
c.source,
|
|
47
|
-
vec_distance_cosine(v.embedding, ?) AS dist
|
|
48
|
-
FROM ${e.vectorTable} v\n JOIN chunks c ON c.id = v.id\n WHERE c.model = ?${e.sourceFilterVec.sql}\n ORDER BY dist ASC\n LIMIT ?`).all(Ge(e.queryVec),e.providerModel,...e.sourceFilterVec.params,e.limit).map(t=>({id:t.id,path:t.path,startLine:t.start_line,endLine:t.end_line,score:1-t.dist,snippet:G(t.text,e.snippetMaxChars),source:t.source})):qe({db:e.db,providerModel:e.providerModel,sourceFilter:e.sourceFilterChunks}).map(t=>({chunk:t,score:Ce(e.queryVec,t.embedding)})).filter(e=>Number.isFinite(e.score)).slice().sort((e,t)=>t.score-e.score).slice(0,e.limit).map(t=>({id:t.chunk.id,path:t.chunk.path,startLine:t.chunk.startLine,endLine:t.chunk.endLine,score:t.score,snippet:G(t.chunk.text,e.snippetMaxChars),source:t.chunk.source}))}function qe(e){return e.db.prepare(`SELECT id, path, start_line, end_line, text, embedding, source
|
|
49
|
-
FROM chunks
|
|
50
|
-
WHERE model = ?${e.sourceFilter.sql}`).all(e.providerModel,...e.sourceFilter.params).map(e=>({id:e.id,path:e.path,startLine:e.start_line,endLine:e.end_line,text:e.text,embedding:Se(e.embedding),source:e.source}))}async function Je(e){if(e.limit<=0)return[];let t=e.buildFtsQuery(e.query);if(!t)return[];let n=e.providerModel?` AND model = ?`:``,r=e.providerModel?[e.providerModel]:[];return e.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n bm25(${e.ftsTable}) AS rank\n FROM ${e.ftsTable}\n WHERE ${e.ftsTable} MATCH ?${n}${e.sourceFilter.sql}\n ORDER BY rank ASC\n LIMIT ?`).all(t,...r,...e.sourceFilter.params,e.limit).map(t=>{let n=e.bm25RankToScore(t.rank);return{id:t.id,path:t.path,startLine:t.start_line,endLine:t.end_line,score:n,textScore:n,snippet:G(t.text,e.snippetMaxChars),source:t.source}})}const Ye=t(import.meta.url);function Xe(){try{return Ye(`node:sqlite`)}catch(e){let t=e instanceof Error?e.message:String(e);throw Error(`SQLite support is unavailable in this Node runtime (missing node:sqlite). ${t}`,{cause:e})}}const K={chunkTokens:400,chunkOverlap:80,maxResults:6,minScore:.2,fallback:{enabled:!0},embedding:{},experimental:{sessionMemory:!1},sources:[`memory`],store:{sessionStoreName:void 0,vector:{enabled:!0}},hybrid:{enabled:!0,vectorWeight:.7,textWeight:.3,candidateMultiplier:4,mmr:{enabled:!1,lambda:.7},temporalDecay:{enabled:!1,halfLifeDays:30}},cache:{enabled:!0,maxEntries:5e4},sync:{onSessionStart:!0,onSearch:!0,watch:!0,watchDebounceMs:1500,sessions:{deltaBytes:1e5,deltaMessages:50}},citations:`auto`};async function Ze(e){try{let t=await import(`sqlite-vec`),n=e.extensionPath?.trim()?e.extensionPath.trim():void 0,r=n??t.getLoadablePath();return e.db.enableLoadExtension(!0),n?e.db.loadExtension(r):t.load(e.db),{ok:!0,extensionPath:r}}catch(e){return{ok:!1,error:e instanceof Error?e.message:String(e)}}}function Qe(e,t,n){let i=p(e,{storeName:n});return r.relative(i,t).replace(/\\/g,`/`)}async function $e(e,t){let i=p(e,{storeName:t});try{let e=await n.readdir(i,{withFileTypes:!0}),t=[];for(let a of e){if(!a.isDirectory())continue;let e=r.join(i,a.name,`transcript.jsonl`);try{(await n.stat(e)).isFile()&&t.push(e)}catch{}}return t}catch(e){if(M(e))return[];throw e}}function et(e){return typeof e==`string`?e:Array.isArray(e)?e.map(e=>typeof e==`string`?e:e&&typeof e==`object`&&`text`in e?String(e.text??``):``).filter(Boolean).join(` `):e?String(e):``}async function tt(e,t,i){let a;try{a=await n.stat(t)}catch(e){if(M(e))return null;throw e}let o;try{o=await n.readFile(t,`utf-8`)}catch(e){if(M(e))return null;throw e}let s=[],c=[],l=o.split(`
|
|
51
|
-
`);for(let e=0;e<l.length;e+=1){let t=l[e]?.trim();if(t)try{let n=JSON.parse(t),r=n.role??`unknown`,i=et(n.content);if(!i)continue;let a=i.split(`
|
|
52
|
-
`);for(let t of a)s.push(`[${r}] ${t}`),c.push(e+1)}catch{}}let u=s.join(`
|
|
53
|
-
`),f=F(u);return{path:r.join(d(i),Qe(e,t,i)).replace(/\\/g,`/`),absPath:t,mtimeMs:a.mtimeMs,size:a.size,hash:f,content:u,lineMap:c}}const q=`chunks_vec`,J=`chunks_fts`,Y=`embedding_cache`,X=`memory_index_meta_v1`,Z=3e4;function nt(e,t,n){let r=t.split(`
|
|
54
|
-
`);for(let t of r){let n=t.match(/^##\s+(.+)$/);if(n?.[1])return`${e}#${n[1].trim()}`}return`${e}#L${n}`}const rt=new Map;function Q(e,t){let n={...e,...t,hybrid:{...e.hybrid,...t?.hybrid??{}},cache:{...e.cache,...t?.cache??{}},sync:{...e.sync,...t?.sync??{}},experimental:{...e.experimental,...t?.experimental??{}},sources:t?.sources??e.sources,embedding:{...e.embedding,...t?.embedding??{}},store:{...e.store,...t?.store??{},vector:{...e.store.vector,...t?.store?.vector??{}}}};return n.experimental.sessionMemory&&!n.sources.includes(`sessions`)&&(n.sources=[...n.sources,`sessions`]),n}var it=class e{dataDir;memoryRoot;storePath;config;sessionStoreName;provider;providerKey;embeddingProviderId;db;dirty=!0;ftsAvailable=!1;ftsError;vector={enabled:!0,available:null,extensionPath:void 0,loadError:void 0,dims:void 0};vectorReady=null;watchers=[];watchTimer=null;sessionWatchTimer=null;sessionsDirty=!1;sessionsDirtyFiles=new Set;sessionPendingFiles=new Set;searchFallbackOnly=!1;lastRebuildSummary;sessionDeltas=new Map;static get(t,n){let i=r.join(t,`.aimax`),a=Q(K,n),o=`${i}::${d(a.store.sessionStoreName)}`,s=rt.get(o);if(s)return n&&s.applyConfig(n),s;let c=new e(t,i,a);return rt.set(o,c),c}constructor(e,t,n){this.dataDir=e,this.memoryRoot=t,this.storePath=r.join(t,`.index.sqlite`),this.config=Q(K,n),this.sessionStoreName=d(this.config.store.sessionStoreName),this.provider=L(),this.providerKey=this.computeProviderKey(),this.applyEmbeddingProvider(),this.vector.enabled=this.config.store.vector.enabled,this.vector.extensionPath=this.config.store.vector.extensionPath,this.db=this.openDatabase(),this.ensureSchema(),this.ensureWatcher(),this.ensureSessionListener(),this.dirty=!0}applyConfig(e){this.config=Q(this.config,e),this.sessionStoreName=d(this.config.store.sessionStoreName),this.applyEmbeddingProvider(),this.vector.enabled=this.config.store.vector.enabled,this.vector.extensionPath=this.config.store.vector.extensionPath,this.ensureWatcher(),this.ensureSessionListener()}warmSession(){this.config.sync.onSessionStart&&this.sync({reason:`session-start`}).catch(()=>{})}noteSessionUpdate(e){this.scheduleSessionDirty(e)}status(){let e=this.buildSourceFilter(),t=this.db.prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${e.sql}`).get(...e.params),n=this.db.prepare(`SELECT COUNT(*) as c FROM chunks WHERE 1=1${e.sql}`).get(...e.params),r=(()=>{let t=Array.from(this.config.sources);if(t.length===0)return[];let n=new Map;for(let e of t)n.set(e,{files:0,chunks:0});let r=this.db.prepare(`SELECT source, COUNT(*) as c FROM files WHERE 1=1${e.sql} GROUP BY source`).all(...e.params);for(let e of r){let t=n.get(e.source)??{files:0,chunks:0};t.files=e.c??0,n.set(e.source,t)}let i=this.db.prepare(`SELECT source, COUNT(*) as c FROM chunks WHERE 1=1${e.sql} GROUP BY source`).all(...e.params);for(let e of i){let t=n.get(e.source)??{files:0,chunks:0};t.chunks=e.c??0,n.set(e.source,t)}return t.map(e=>Object.assign({source:e},n.get(e)))})(),i=this.provider?`hybrid`:`fts-only`,a=this.config.cache.enabled?this.db.prepare(`SELECT COUNT(*) as c FROM ${Y}`).get()?.c??0:void 0;return{backend:`builtin`,files:t?.c??0,chunks:n?.c??0,dirty:this.dirty||this.sessionsDirty,workspaceDir:this.memoryRoot,dbPath:this.storePath,provider:this.provider.id,model:this.provider.model,sources:Array.from(this.config.sources),sourceCounts:r,cache:this.config.cache.enabled?{enabled:!0,entries:a,maxEntries:this.config.cache.maxEntries}:{enabled:!1,maxEntries:this.config.cache.maxEntries},fts:{enabled:this.config.hybrid.enabled,available:this.ftsAvailable,error:this.ftsError,rows:this.getFtsRowCount()},vector:{enabled:this.vector.enabled,available:this.vector.available??void 0,extensionPath:this.vector.extensionPath,loadError:this.vector.loadError,dims:this.vector.dims},custom:{searchMode:i,fallbackActive:this.searchFallbackOnly,lastRebuildSummary:this.lastRebuildSummary}}}async probeEmbeddingAvailability(){try{return await this.provider.embedBatch([`ping`]),{ok:!0}}catch(e){return{ok:!1,error:e instanceof Error?e.message:String(e)}}}async probeVectorAvailability(){return this.vector.enabled?await this.ensureVectorReady():!1}async search(e){let t=e.trim();if(!t)return[];if(this.readMeta()?this.config.sync.onSearch&&(this.dirty||this.sessionsDirty)&&this.sync({reason:`search`}).catch(()=>{this.searchFallbackOnly=!0}):await this.sync({reason:`boot`}).catch(()=>{this.searchFallbackOnly=!0}),this.searchFallbackOnly)return this.decorateCitations(await this.fileScanSearch(t));let n=this.config.hybrid,r=Math.min(200,Math.max(1,Math.floor(this.config.maxResults*n.candidateMultiplier))),i=n.enabled&&this.ftsAvailable?await this.searchKeyword(t,r).catch(()=>[]):[],a=await this.provider.embedQuery(t).catch(()=>[]),o=a.some(e=>e!==0)?await this.searchVector(a,r).catch(()=>[]):[],s=[];if(!n.enabled||!this.ftsAvailable)s=o.filter(e=>e.score>=this.config.minScore).slice(0,this.config.maxResults);else{let e=await Ue({vector:o.map(e=>({id:`${e.path}:${e.startLine}:${e.endLine}:${e.source}`,path:e.path,startLine:e.startLine,endLine:e.endLine,source:e.source,snippet:e.snippet,vectorScore:e.score})),keyword:i.map(e=>({id:`${e.path}:${e.startLine}:${e.endLine}:${e.source}`,path:e.path,startLine:e.startLine,endLine:e.endLine,source:e.source,snippet:e.snippet,textScore:e.textScore})),vectorWeight:n.vectorWeight,textWeight:n.textWeight,mmr:n.mmr,temporalDecay:n.temporalDecay,memoryRoot:this.memoryRoot}),t=e.filter(e=>e.score>=this.config.minScore);if(t.length>0||i.length===0)s=t.slice(0,this.config.maxResults);else{let t=Math.min(this.config.minScore,n.textWeight);s=e.filter(e=>e.score>=t).slice(0,this.config.maxResults)}}return s.length>0?this.decorateCitations(this.decorateEntryIds(s)):this.config.fallback.enabled?(this.searchFallbackOnly=!0,this.decorateCitations(this.decorateEntryIds(await this.fileScanSearch(t)))):[]}async readFile(e){let{absPath:t,relPath:r}=this.resolveMemoryFilePath(e.relPath);if((await he(t)).missing)return{text:``,path:r};let i;try{i=await n.readFile(t,`utf-8`)}catch(e){if(M(e))return{text:``,path:r};throw e}if(!e.from&&!e.lines)return{text:i,path:r};let a=i.split(`
|
|
55
|
-
`),o=Math.max(1,e.from??1),s=Math.max(1,e.lines??a.length);return{text:a.slice(o-1,o-1+s).join(`
|
|
56
|
-
`),path:r}}async replaceFile(e){let{absPath:t}=this.resolveMemoryFilePath(e.relPath);await n.mkdir(r.dirname(t),{recursive:!0}),await n.writeFile(t,e.content,`utf-8`),this.dirty=!0,await this.syncAndPostCheck(`replace`)}async deleteFile(e){let{absPath:t}=this.resolveMemoryFilePath(e);try{await n.unlink(t)}catch(e){if(M(e))return;throw e}this.dirty=!0,await this.syncAndPostCheck(`delete`)}async appendToMemory(e){let t=this.memoryRoot;await n.mkdir(t,{recursive:!0});let i=r.join(t,`MEMORY.md`),a=e.endsWith(`
|
|
57
|
-
`)?e:`${e}\n`;await n.appendFile(i,a,`utf-8`),this.dirty=!0,await this.syncAndPostCheck(`append`)}openDatabase(){ge(r.dirname(this.storePath));let{DatabaseSync:e}=Xe();return new e(this.storePath,{allowExtension:this.vector.enabled})}ensureSchema(){let e=We({db:this.db,embeddingCacheTable:Y,ftsTable:J,ftsEnabled:this.config.hybrid.enabled});this.ftsAvailable=e.ftsAvailable,e.ftsError&&(this.ftsError=e.ftsError)}async ensureVectorReady(e){if(!this.vector.enabled)return!1;this.vectorReady||=this.withTimeout(this.loadVectorExtension(),Z,`sqlite-vec load timed out after ${Math.round(Z/1e3)}s`);let t=!1;try{t=await this.vectorReady||!1}catch(e){let t=e instanceof Error?e.message:String(e);return this.vector.available=!1,this.vector.loadError=t,this.vectorReady=null,!1}return t&&typeof e==`number`&&e>0&&this.ensureVectorTable(e),t}async loadVectorExtension(){if(this.vector.available!==null)return this.vector.available;if(!this.vector.enabled)return this.vector.available=!1,!1;try{let e=this.vector.extensionPath?.trim()?this.vector.extensionPath.trim():void 0,t=await Ze({db:this.db,extensionPath:e});if(!t.ok)throw Error(t.error??`unknown sqlite-vec load error`);return this.vector.extensionPath=t.extensionPath,this.vector.available=!0,!0}catch(e){let t=e instanceof Error?e.message:String(e);return this.vector.available=!1,this.vector.loadError=t,!1}}ensureVectorTable(e){this.vector.dims!==e&&(this.vector.dims&&this.vector.dims!==e&&this.dropVectorTable(),this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${q} USING vec0(\n id TEXT PRIMARY KEY,\n embedding FLOAT[${e}]\n)`),this.vector.dims=e)}dropVectorTable(){try{this.db.exec(`DROP TABLE IF EXISTS ${q}`)}catch{}}async withTimeout(e,t,n){if(!Number.isFinite(t)||t<=0)return await e;let r=null,i=new Promise((e,i)=>{r=setTimeout(()=>i(Error(n)),t)});try{return await Promise.race([e,i])}finally{r&&clearTimeout(r)}}ensureWatcher(){if(!this.config.sync.watch||this.watchers.length>0)return;let e=[r.join(this.memoryRoot,`MEMORY.md`),r.join(this.memoryRoot,`memory.md`),r.join(this.memoryRoot,`memory`)],t=()=>{this.dirty=!0,this.scheduleWatchSync()};for(let n of e)try{let e=a.watch(n,{recursive:!1},t);e.on(`error`,()=>{try{e.close()}catch{}}),this.watchers.push(e)}catch{}}scheduleWatchSync(){this.config.sync.watch&&(this.watchTimer&&clearTimeout(this.watchTimer),this.watchTimer=setTimeout(()=>{this.watchTimer=null,this.sync({reason:`watch`}).catch(()=>{})},this.config.sync.watchDebounceMs))}ensureSessionListener(){if(!this.config.experimental.sessionMemory)return;let e=p(this.dataDir,{storeName:this.sessionStoreName});try{let t=a.watch(e,{recursive:!0},(t,n)=>{if(!n||!n.endsWith(`transcript.jsonl`))return;let i=r.join(e,n);this.scheduleSessionDirty(i)});this.watchers.push(t)}catch{}}scheduleSessionDirty(e){this.config.experimental.sessionMemory&&(this.sessionPendingFiles.add(e),!this.sessionWatchTimer&&(this.sessionWatchTimer=setTimeout(()=>{this.sessionWatchTimer=null,this.processSessionDeltaBatch().catch(()=>{})},5e3)))}async processSessionDeltaBatch(){if(this.sessionPendingFiles.size===0)return;let e=Array.from(this.sessionPendingFiles);this.sessionPendingFiles.clear();let t=!1;for(let n of e){let e=await this.updateSessionDelta(n);if(!e)continue;let r=e.deltaBytes,i=e.deltaMessages,a=r<=0?e.pendingBytes>0:e.pendingBytes>=r,o=i<=0?e.pendingMessages>0:e.pendingMessages>=i;!a&&!o||(this.sessionsDirtyFiles.add(n),this.sessionsDirty=!0,e.pendingBytes=r>0?Math.max(0,e.pendingBytes-r):0,e.pendingMessages=i>0?Math.max(0,e.pendingMessages-i):0,t=!0)}t&&this.sync({reason:`session-delta`}).catch(()=>{})}async updateSessionDelta(e){let t=this.config.sync.sessions;if(!t)return null;let r;try{r=await n.stat(e)}catch{return null}let i=r.size,a=this.sessionDeltas.get(e);a||(a={lastSize:0,pendingBytes:0,pendingMessages:0},this.sessionDeltas.set(e,a));let o=Math.max(0,i-a.lastSize);return o===0&&i===a.lastSize?{deltaBytes:t.deltaBytes,deltaMessages:t.deltaMessages,pendingBytes:a.pendingBytes,pendingMessages:a.pendingMessages}:(i<a.lastSize?(a.lastSize=i,a.pendingBytes+=i,t.deltaMessages>0&&(t.deltaBytes<=0||a.pendingBytes<t.deltaBytes)&&(a.pendingMessages+=await this.countNewlines(e,0,i))):(a.pendingBytes+=o,t.deltaMessages>0&&(t.deltaBytes<=0||a.pendingBytes<t.deltaBytes)&&(a.pendingMessages+=await this.countNewlines(e,a.lastSize,i)),a.lastSize=i),this.sessionDeltas.set(e,a),{deltaBytes:t.deltaBytes,deltaMessages:t.deltaMessages,pendingBytes:a.pendingBytes,pendingMessages:a.pendingMessages})}async countNewlines(e,t,r){if(r<=t)return 0;let i;try{i=await n.open(e,`r`)}catch{return 0}try{let e=t,n=0,a=Buffer.alloc(64*1024);for(;e<r;){let t=Math.min(a.length,r-e),{bytesRead:o}=await i.read(a,0,t,e);if(o<=0)break;for(let e=0;e<o;e+=1)a[e]===10&&(n+=1);e+=o}return n}finally{await i.close()}}computeProviderKey(){return F(JSON.stringify({provider:this.provider.id,model:this.provider.model}))}resolveMemoryFilePath(e){let t=e.trim();if(!t)throw Error(`path required`);let n=r.isAbsolute(t)?r.resolve(t):r.resolve(this.memoryRoot,t),i=r.relative(this.memoryRoot,n).replace(/\\/g,`/`);if(!(i.length>0&&!i.startsWith(`..`)&&!r.isAbsolute(i)&&ve(i))||!n.endsWith(`.md`))throw Error(`path required`);return{absPath:n,relPath:i}}applyEmbeddingProvider(){let e=this.config.embedding?.providerId?.trim();if(e){let t=z({providerId:e,dataDir:this.dataDir,memoryDir:this.memoryRoot});if(!t)throw Error(`Embedding provider not found: ${e}`);this.provider=t.provider,this.embeddingProviderId=e,this.providerKey=this.computeProviderKey(),this.dirty=!0;return}this.embeddingProviderId&&(this.provider=L(),this.embeddingProviderId=void 0,this.providerKey=this.computeProviderKey(),this.dirty=!0)}readMeta(){let e=this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(X);if(!e?.value)return null;try{return JSON.parse(e.value)}catch{return null}}writeMeta(e){let t=JSON.stringify(e);this.db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(X,t)}loadEmbeddingCache(e){if(!this.config.cache.enabled||e.length===0)return new Map;let t=Array.from(new Set(e.filter(Boolean)));if(t.length===0)return new Map;let n=new Map,r=[this.provider.id,this.provider.model,this.providerKey];for(let e=0;e<t.length;e+=400){let i=t.slice(e,e+400),a=i.map(()=>`?`).join(`, `),o=this.db.prepare(`SELECT hash, embedding FROM ${Y}\n WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${a})`).all(...r,...i);for(let e of o)n.set(e.hash,Se(e.embedding))}return n}upsertEmbeddingCache(e){if(!this.config.cache.enabled||e.length===0)return;let t=Date.now(),n=this.db.prepare(`INSERT INTO ${Y} (provider, model, provider_key, hash, embedding, dims, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET\n embedding=excluded.embedding,\n dims=excluded.dims,\n updated_at=excluded.updated_at`);for(let r of e){let e=r.embedding??[];n.run(this.provider.id,this.provider.model,this.providerKey,r.hash,JSON.stringify(e),e.length,t)}}async embedChunks(e){if(e.length===0)return[];let t=this.loadEmbeddingCache(e.map(e=>e.hash)),n=Array.from({length:e.length},()=>[]),r=[];for(let i=0;i<e.length;i+=1){let a=e[i],o=a?.hash?t.get(a.hash):void 0;o&&o.length>0?n[i]=o:a&&r.push({index:i,chunk:a})}if(r.length===0)return n;let i=await this.provider.embedBatch(r.map(e=>e.chunk.text)),a=[];for(let e=0;e<r.length;e+=1){let t=r[e],o=i[e]??[];n[t.index]=o,a.push({hash:t.chunk.hash,embedding:o})}return this.upsertEmbeddingCache(a),n}async indexFile(e,t){let r=be(t.content??await n.readFile(e.absPath,`utf-8`),{tokens:this.config.chunkTokens,overlap:this.config.chunkOverlap}).filter(e=>e.text.trim().length>0);t.source===`sessions`&&xe(r,t.lineMap);let i=await this.embedChunks(r),a=i.find(e=>e.length>0),o=a?await this.ensureVectorReady(a.length):!1,s=Date.now();if(o)try{this.db.prepare(`DELETE FROM ${q} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(e.path,t.source)}catch{}if(this.ftsAvailable)try{this.db.prepare(`DELETE FROM ${J} WHERE path = ? AND source = ? AND model = ?`).run(e.path,t.source,this.provider.model)}catch{}this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(e.path,t.source);for(let n=0;n<r.length;n+=1){let a=r[n],c=i[n]??[],l=F(`${t.source}:${e.path}:${a.startLine}:${a.endLine}:${a.hash}:${this.provider.model}`);if(this.db.prepare(`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
|
|
58
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
59
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
60
|
-
hash=excluded.hash,
|
|
61
|
-
model=excluded.model,
|
|
62
|
-
text=excluded.text,
|
|
63
|
-
embedding=excluded.embedding,
|
|
64
|
-
updated_at=excluded.updated_at`).run(l,e.path,t.source,a.startLine,a.endLine,a.hash,this.provider.model,a.text,JSON.stringify(c),s),o&&c.length>0){try{this.db.prepare(`DELETE FROM ${q} WHERE id = ?`).run(l)}catch{}this.db.prepare(`INSERT INTO ${q} (id, embedding) VALUES (?, ?)`).run(l,Buffer.from(new Float32Array(c).buffer))}this.ftsAvailable&&this.db.prepare(`INSERT INTO ${J} (text, id, path, source, model, start_line, end_line)\n VALUES (?, ?, ?, ?, ?, ?, ?)`).run(a.text,l,e.path,t.source,this.provider.model,a.startLine,a.endLine)}this.db.prepare(`INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
|
|
65
|
-
ON CONFLICT(path) DO UPDATE SET
|
|
66
|
-
source=excluded.source,
|
|
67
|
-
hash=excluded.hash,
|
|
68
|
-
mtime=excluded.mtime,
|
|
69
|
-
size=excluded.size`).run(e.path,t.source,e.hash,e.mtimeMs,e.size)}buildSourceFilter(){let e=this.config.sources;return e.length?{sql:` AND source IN (${e.map(()=>`?`).join(`, `)})`,params:e}:{sql:``,params:[]}}async searchVector(e,t){return await Ke({db:this.db,vectorTable:q,providerModel:this.provider.model,queryVec:e,limit:t,snippetMaxChars:700,ensureVectorReady:async e=>await this.ensureVectorReady(e),sourceFilterVec:this.buildSourceFilter(),sourceFilterChunks:this.buildSourceFilter()})}async searchKeyword(e,t){return this.ftsAvailable?await Je({db:this.db,ftsTable:J,providerModel:this.provider.model,query:e,limit:t,snippetMaxChars:700,sourceFilter:this.buildSourceFilter(),buildFtsQuery:e=>Ve(e),bm25RankToScore:He}):[]}resetIndex(){if(this.db.exec(`DELETE FROM files`),this.db.exec(`DELETE FROM chunks`),this.ftsAvailable)try{this.db.exec(`DELETE FROM ${J}`)}catch{}this.dropVectorTable(),this.vector.dims=void 0,this.sessionsDirtyFiles.clear()}async syncAndPostCheck(e){try{await this.sync({reason:`mutation-postcheck`})}catch{this.searchFallbackOnly=!0;return}this.searchFallbackOnly=!this.indexHasSearchableData()}indexHasSearchableData(){try{if((this.db.prepare(`SELECT COUNT(*) as c FROM chunks`).get()?.c??0)>0)return!0}catch{}try{if(a.readdirSync(this.memoryRoot).filter(e=>e.endsWith(`.md`)).length>0)return!1;let e=r.join(this.memoryRoot,`memory`);return a.existsSync(e)?a.readdirSync(e).filter(e=>e.endsWith(`.md`)).length===0:!0}catch{return!1}}async fileScanSearch(e){let t=e.trim().toLowerCase();if(!t)return[];let a=await P(this.memoryRoot),o=[];for(let e of a){let a=``;try{a=await n.readFile(e,`utf-8`)}catch{continue}let s=r.relative(this.memoryRoot,e).replace(/\\/g,`/`);s.startsWith(`memory/`);let c=a.split(`
|
|
70
|
-
`);for(let n=0;n<c.length;n+=1){let r=c[n]??``,a=r.toLowerCase();if(!a.includes(t))continue;let l=a===t?1:Math.max(.2,t.length/Math.max(r.length,t.length));o.push({path:s||i(e),startLine:n+1,endLine:n+1,score:Number(l.toFixed(4)),snippet:r.slice(0,700),source:`memory`})}}return o.sort((e,t)=>t.score-e.score||e.path.localeCompare(t.path)||e.startLine-t.startLine),o.slice(0,this.config.maxResults)}getFtsRowCount(){if(this.ftsAvailable)try{return this.db.prepare(`SELECT COUNT(*) as c FROM ${J}`).get()?.c??0}catch{return}}buildRebuildSummary(e){let t=this.db.prepare(`SELECT COUNT(*) as c FROM chunks`).get();return{reason:e.reason,memoryFilesIndexed:e.memoryFilesIndexed,sessionFilesIndexed:e.sessionFilesIndexed,staleMemoryFilesRemoved:e.staleMemoryFilesRemoved,staleSessionFilesRemoved:e.staleSessionFilesRemoved,chunks:t?.c??0,ftsRows:this.getFtsRowCount(),fallbackActive:this.searchFallbackOnly,needsFullReindex:e.needsFullReindex}}async rebuild(e=`manual-rebuild`){return this.resetIndex(),this.dirty=!0,this.sessionsDirty=!0,await this.sync({reason:e,forceFullReindex:!0}),this.lastRebuildSummary??this.buildRebuildSummary({reason:e,memoryFilesIndexed:0,sessionFilesIndexed:0,staleMemoryFilesRemoved:0,staleSessionFilesRemoved:0,needsFullReindex:!0})}async sync(e){let t=this.readMeta(),n=e?.forceFullReindex||!t||t.model!==this.provider.model||t.provider!==this.provider.id||t.providerKey!==this.providerKey||t.chunkTokens!==this.config.chunkTokens||t.chunkOverlap!==this.config.chunkOverlap||this.vector.available&&!t?.vectorDims;n&&this.resetIndex();let r=await P(this.memoryRoot),i=(await Promise.all(r.map(async e=>ye(e,this.memoryRoot)))).filter(e=>e!==null),a=new Set(i.map(e=>e.path)),o=0,s=i.map(e=>async()=>{let t=this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(e.path,`memory`);!n&&t?.hash===e.hash||(await this.indexFile(e,{source:`memory`}),o+=1)});for(let e of s)await e();let c=0,l=this.db.prepare(`SELECT path FROM files WHERE source = ?`).all(`memory`);for(let e of l)if(!a.has(e.path)){this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(e.path,`memory`),c+=1;try{this.db.prepare(`DELETE FROM ${q} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(e.path,`memory`)}catch{}if(this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(e.path,`memory`),this.ftsAvailable)try{this.db.prepare(`DELETE FROM ${J} WHERE path = ? AND source = ? AND model = ?`).run(e.path,`memory`,this.provider.model)}catch{}}let u={indexed:0,staleRemoved:0};this.config.experimental.sessionMemory&&(n||this.sessionsDirty)&&(u=await this.syncSessionFiles(n)),this.writeMeta({model:this.provider.model,provider:this.provider.id,providerKey:this.providerKey,chunkTokens:this.config.chunkTokens,chunkOverlap:this.config.chunkOverlap,...this.vector.available&&this.vector.dims?{vectorDims:this.vector.dims}:{}}),this.dirty=!1,this.searchFallbackOnly=!this.indexHasSearchableData(),this.lastRebuildSummary=this.buildRebuildSummary({reason:e?.reason??`sync`,memoryFilesIndexed:o,sessionFilesIndexed:u.indexed,staleMemoryFilesRemoved:c,staleSessionFilesRemoved:u.staleRemoved,needsFullReindex:!!n})}async syncSessionFiles(e){let t=await $e(this.dataDir,this.sessionStoreName),n=(await Promise.all(t.map(async e=>tt(this.dataDir,e,this.sessionStoreName)))).filter(e=>e!==null),r=new Set(n.map(e=>e.path)),i=e||this.sessionsDirtyFiles.size===0,a=0;for(let t of n){if(!i&&!this.sessionsDirtyFiles.has(t.absPath))continue;let n=this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(t.path,`sessions`);if(!e&&n?.hash===t.hash){this.resetSessionDelta(t.absPath,t.size);continue}await this.indexFile(t,{source:`sessions`,content:t.content,lineMap:t.lineMap}),a+=1,this.resetSessionDelta(t.absPath,t.size)}let o=0,s=this.db.prepare(`SELECT path FROM files WHERE source = ?`).all(`sessions`);for(let e of s)if(!r.has(e.path)){this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(e.path,`sessions`),o+=1;try{this.db.prepare(`DELETE FROM ${q} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`).run(e.path,`sessions`)}catch{}if(this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(e.path,`sessions`),this.ftsAvailable)try{this.db.prepare(`DELETE FROM ${J} WHERE path = ? AND source = ? AND model = ?`).run(e.path,`sessions`,this.provider.model)}catch{}}return this.sessionsDirty=!1,this.sessionsDirtyFiles.clear(),{indexed:a,staleRemoved:o}}resetSessionDelta(e,t){let n=this.sessionDeltas.get(e);n&&(n.lastSize=t,n.pendingBytes=0,n.pendingMessages=0)}decorateCitations(e){return this.config.citations===`off`?e:e.map(e=>{let t=`Source: ${e.path}#L${e.startLine}-L${e.endLine}`;return this.config.citations===`on`?{...e,citation:t,snippet:`${e.snippet}\n\n${t}`}:{...e,citation:t}})}decorateEntryIds(e){return e.map(e=>{if(e.id)return e;let t=nt(e.path,e.snippet,e.startLine);return{...e,id:t}})}},$=e({createBuiltinMemoryProvider:()=>at});function at(e,t){let n=it.get(e.dataDir,t?.includeSessions?{experimental:{sessionMemory:!0},sources:[`memory`,`sessions`],store:{sessionStoreName:e.sessionStoreName,vector:{enabled:!0}}}:e.sessionStoreName?{store:{sessionStoreName:e.sessionStoreName,vector:{enabled:!0}}}:void 0),i=e.memoryDir;return{id:`builtin-memory`,name:`Builtin Memory Provider`,status:()=>n.status(),search:async(e,t)=>n.search(e),getLines:async(e,t,r)=>{let i=Math.max(1,r-t+1),a=await n.readFile({relPath:e,from:t,lines:i});return a.text?a.text.split(`
|
|
71
|
-
`):[]},listFiles:async()=>P(i),append:async e=>n.appendToMemory(e),updateFile:async(e,t)=>{await n.replaceFile({relPath:e,content:t})},deleteFile:async e=>{await n.deleteFile(e)},sync:async e=>{if(e===`cli-rebuild`){await n.rebuild(e);return}await n.sync({reason:e??`provider-sync`})},noteSessionUpdate:e=>{let t=r.isAbsolute(e)?e:r.resolve(e);n.noteSessionUpdate(t)}}}export{ce as A,re as C,w as D,k as E,p as F,b as I,h as L,m as M,pe as N,_ as O,y as P,de as R,D as S,A as T,x as _,De as a,j as b,c,le as d,se as f,te as g,v as h,Ee as i,O as j,d as k,l,ee as m,at as n,z as o,g as p,it as r,P as s,$ as t,ne as u,C as v,ue as w,fe as x,me as y};
|