agent-rag-mcp 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/extensions/agent-rag.mjs +27 -0
- package/dist/src/adapters/structured-text.mjs +111 -45
- package/dist/src/cli/format.mjs +6 -0
- package/dist/src/cli/index.mjs +29 -2
- package/dist/src/ingest/pipeline.mjs +10 -9
- package/dist/src/store/canonical.mjs +65 -0
- package/dist/src/store/identity.mjs +1 -1
- package/dist/src/store/lifecycle.mjs +2 -0
- package/dist/src/store/manifest.mjs +13 -4
- package/dist/src/store/rehash.mjs +162 -0
- package/dist/src/store/schema.mjs +11 -1
- package/package.json +7 -3
- package/skills/agent-rag/SKILL.md +35 -0
- package/extensions/agent-rag.ts +0 -89
- package/extensions/pi-host.d.ts +0 -55
package/README.md
CHANGED
|
@@ -32,6 +32,8 @@ The `rag` commands available today are:
|
|
|
32
32
|
- `rag backup --to <dir>` — Backs up the manifest and object store.
|
|
33
33
|
- `rag restore --from <dir>` — Restores a backup under this root.
|
|
34
34
|
- `rag reindex` — Rebuilds the database from the manifest and objects.
|
|
35
|
+
- `rag rehash` — Previews recomputed hashes for stored versions; `rag rehash --apply` writes the changed hash facts in one transaction. Preview first, and back up with `rag backup --to <backup-dir>` before applying.
|
|
36
|
+
- `rag rehash --apply` writes `manifest.json` before committing the database transaction. An interruption in that window can leave the manifest ahead of the database; rerunning `rag rehash --apply` or running `rag reindex` converges to the requested facts.
|
|
35
37
|
|
|
36
38
|
### CLI output
|
|
37
39
|
|
|
@@ -44,7 +46,11 @@ rag ls
|
|
|
44
46
|
rag relabel <document-id> --label "replacement label"
|
|
45
47
|
```
|
|
46
48
|
|
|
47
|
-
Structured files can use `.yaml`, `.yml`, `.json`, or `.jsonl`; each structured file yields
|
|
49
|
+
Structured files can use `.yaml`, `.yml`, `.json`, or `.jsonl`; each structured file yields sections from its parsed data tree. YAML uses the YAML 1.2 core schema, so words such as `yes` and `on` remain strings and unquoted timestamps remain strings.
|
|
50
|
+
|
|
51
|
+
### Content identity
|
|
52
|
+
|
|
53
|
+
Structured JSON, JSONL, and YAML are hashed from compact canonical JSON for their parsed data tree, without a format prefix. Their `doc_version` rows use `content_kind='structured'` and `hash_version=2`; equivalent trees therefore share a content hash across formats and key order. Other formats retain the existing normalized extracted-text hash and use `content_kind=NULL` with `hash_version=1`. Existing version rows stay at version 1 with a null content kind until repaired with `rag rehash`; the repair never rewrites stored object bytes and skips structured documents that cannot be canonically hashed.
|
|
48
54
|
|
|
49
55
|
The default knowledge-base root is `%USERPROFILE%\.rag\`. Each client has its own `rag.db` under `.rag\clients\<client>\`; projects are logical within that client and are identified by `project_id`. The namespace is the pair `(client, project)`. By default, `failClosed` is `true`; without a namespace, reads are allowed, and when `failClosed` is set to `false`, writes go to the reserved `default/default` namespace. The Tier 3 human review queue is available through `rag review list`.
|
|
50
56
|
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
var w=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(r){throw n=[r],r}};var $e=w(()=>{"use strict"});import*as ye from"node:path";import{DatabaseSync as br}from"node:sqlite";function Gt(e){if(e==="."||e===".."||!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(e))throw new Te(e)}function q(e,t){return Gt(t),ye.join(e,"clients",t,"rag.db")}var Te,be=w(()=>{"use strict";$e();Te=class extends Error{code="invalid_client_name";constructor(t){super(`Invalid client name "${t}". Client names must match [A-Za-z0-9][A-Za-z0-9._-]* and cannot be "." or "..".`),this.name="InvalidClientNameError"}}});import{extractRawText as Qt}from"mammoth";function en(e){if(typeof e=="string")return/\.docx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===Se;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.docx$/i.test(r)}async function tn(e){let n=(await Qt({buffer:Buffer.from(e)})).value.trim();return{sections:n.length>0?[{ordinal:0,heading:null,level:null,text:n}]:[],versionLabel:null,mediaType:Se}}var Se,Ge,Ve=w(()=>{"use strict";Se="application/vnd.openxmlformats-officedocument.wordprocessingml.document";Ge={name:"docx",version:"1.13.0",mediaTypes:[Se],supports:en,extract:tn}});var b,M=w(()=>{"use strict";b=class extends Error{code;constructor(t,n){super(n),this.name="ResourceLimitError",this.code=t}}});import{createRequire as nn}from"node:module";import we from"tesseract.js";function on(e){if(typeof e=="string"){let o=e.split(";")[0]?.trim().toLowerCase();return Ye.has(o??"")||Je.test(e)}if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return Ye.has(n.split(";")[0]?.trim().toLowerCase()??"");let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&Je.test(r)}function sn(e){if(e.byteLength>=8&&e[0]===137&&e[1]===80&&e[2]===78&&e[3]===71&&e[4]===13&&e[5]===10&&e[6]===26&&e[7]===10)return te;if(e.byteLength>=3&&e[0]===255&&e[1]===216&&e[2]===255)return Re;throw new Error("Unsupported image data; expected PNG or JPEG bytes.")}function an(e,t){let n=new DataView(e.buffer,e.byteOffset,e.byteLength);if(t===te){if(e.byteLength<24||String.fromCharCode(...e.subarray(12,16))!=="IHDR")throw new Error("Invalid PNG image header.");return{width:n.getUint32(16),height:n.getUint32(20)}}let r=2;for(;r+3<e.byteLength;){if(e[r]!==255){r+=1;continue}for(;e[r]===255;)r+=1;let o=e[r];if(r+=1,o===217||o===218)break;if(o===216||o===1||o!==void 0&&o>=208&&o<=215)continue;if(r+2>e.byteLength)break;let i=n.getUint16(r);if(i<2||r+i>e.byteLength)break;if(o!==void 0&&(o>=192&&o<=195||o>=197&&o<=199||o>=201&&o<=203||o>=205&&o<=207)){if(i<7)break;return{height:n.getUint16(r+3),width:n.getUint16(r+5)}}r+=i}throw new Error("JPEG image dimensions could not be read.")}async function cn(e){let t=sn(e),{width:n,height:r}=an(e,t);if(n===0||r===0||n>We/r)throw new b("image_too_large",`Image exceeds MAX_IMAGE_PIXELS (${We} pixels).`);let o=await we.createWorker("eng",we.OEM.LSTM_ONLY,Ne);try{await o.setParameters({tessedit_pageseg_mode:we.PSM.SINGLE_BLOCK});let{data:i}=await o.recognize(Buffer.from(e)),s=i.text.replace(/\s+/g," ").trim();return{sections:s.length>0?[{ordinal:0,heading:null,level:null,text:s}]:[],versionLabel:null,mediaType:t}}finally{await o.terminate()}}var te,Re,Ye,Je,We,Ke,rn,Ne,ne,Ae=w(()=>{"use strict";M();te="image/png",Re="image/jpeg",Ye=new Set([te,Re]),Je=/\.(?:png|jpe?g)$/i,We=5e7,Ke=nn(import.meta.url)("@tesseract.js-data/eng"),rn=Ke.langPath,Ne={langPath:rn,cacheMethod:"none",gzip:Ke.gzip};ne={name:"image-ocr",version:"7.0.0",mediaTypes:[te,Re],supports:on,extract:cn}});var Ze=w(()=>{"use strict"});import{createCanvas as ln}from"@napi-rs/canvas";import Le from"tesseract.js";import{getDocument as un}from"pdfjs-dist/legacy/build/pdf.mjs";function fn(e){if(typeof e=="string")return/\.pdf$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===re;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.pdf$/i.test(r)}async function pn(e){let t=un({data:Uint8Array.from(e),maxImageSize:1e7,disableFontFace:!0,useWorkerFetch:!1,verbosity:0}),n=await t.promise;try{if(n.numPages>et)throw new b("too_many_pages",`PDF exceeds MAX_PDF_PAGES (${et} pages).`);let r=[];for(let i=1;i<=n.numPages;i+=1){let c=(await(await n.getPage(i)).getTextContent()).items.filter(l=>"str"in l).map(l=>"str"in l?l.str:"").join(" ").replace(/\s+/g," ").trim();c.length>0&&r.push({ordinal:r.length,heading:null,level:null,text:c})}if(r.length>0)return{sections:r,versionLabel:null,mediaType:re};if(n.numPages>100)throw new b("too_many_pages",`PDF exceeds MAX_OCR_PDF_PAGES (${100} pages) for OCR.`);let o=await Le.createWorker("eng",Le.OEM.LSTM_ONLY,Ne);try{await o.setParameters({tessedit_pageseg_mode:Le.PSM.SINGLE_BLOCK});for(let i=1;i<=n.numPages;i+=1){let s=await n.getPage(i),a=s.getViewport({scale:mn}),c=Math.ceil(a.width),l=Math.ceil(a.height);if(c===0||l===0||c>5e6/l)throw new b("image_too_large",`PDF page exceeds MAX_PDF_OCR_RENDER_PIXELS (${5e6} pixels).`);let d=ln(c,l),u=d.getContext("2d");await s.render({canvas:d,canvasContext:u,viewport:a}).promise;let{data:g}=await o.recognize(d.toBuffer("image/png")),E=g.text.replace(/\s+/g," ").trim();E.length>0&&r.push({ordinal:i-1,heading:`Page ${i}`,level:null,text:E})}}finally{await o.terminate()}return{sections:r,versionLabel:null,mediaType:re}}finally{await t.destroy()}}var re,et,mn,tt,nt=w(()=>{"use strict";Ae();Ze();M();re="application/pdf",et=2e3,mn=2;tt={name:"pdf",version:"6.4.299",mediaTypes:[re],supports:fn,extract:pn}});function rt(e){let t=e.match(/^(.*?)[-_ ]v(\d+(?:\.\d+)*)(\.[^./\\]+)?$/);return t?{prefix:t[1].replace(/[-_ ]+$/,""),version:t[2],suffix:t[3]??null}:null}var ot=w(()=>{"use strict"});function it(e,t){if(typeof e=="string")return e;if(typeof e!="object"||e===null)return null;let n=e;for(let r of t)if(typeof n[r]=="string")return n[r];return null}function En(e){let t=it(e,["mediaType","mimeType","contentType"]);if(t!==null)return gn.has(t.split(";")[0].trim().toLowerCase());let n=it(e,["filename","name","path","extension"]);return n===null?!1:/(?:^|[\\/])[^\\/]+\.(?:txt|md)$/i.test(n)||/^\.(?:txt|md)$/i.test(n)}function hn(e){let t=e.split(`
|
|
2
|
+
`),n=[],r=null,o=/^ {0,3}(#{1,6})(?:[ \t]+|$)(.*)$/;if(!t.some(s=>o.test(s.replace(/\r$/,""))))return e.length===0?[]:[{ordinal:0,heading:null,level:null,text:e}];let i=()=>{r&&n.push({ordinal:n.length,heading:r.heading,level:r.level,text:r.lines.join(`
|
|
3
|
+
`).trim()})};for(let s of t){let a=o.exec(s.replace(/\r$/,""));if(!a){r?r.lines.push(s):s.length>0&&(r={heading:null,level:null,lines:[s]});continue}i(),r={heading:a[2].replace(/[ \t]+#+[ \t]*$/,"").trim(),level:a[1].length,lines:[]}}return i(),n}async function _n(e){let t=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(e),n=t.startsWith("\uFEFF")?t.slice(1):t,r=hn(n),o=r.some(s=>s.heading!==null),i=r.find(s=>s.heading!==null&&rt(s.heading)!==null)?.heading??null;return{sections:r,versionLabel:i,mediaType:o?"text/markdown":n.length>0?"text/plain":null}}var gn,Oe,st=w(()=>{"use strict";ot();gn=new Set(["text/plain","text/markdown"]);Oe={name:"plain-text",version:"1.0.0",supports:En,extract:_n}});function xn(e,t){let n=Array.from(e,i=>i.codePointAt(0)),r=Array.from(t,i=>i.codePointAt(0)),o=Math.min(n.length,r.length);for(let i=0;i<o;i+=1){let s=n[i],a=r[i];if(s!==a)return s-a}return n.length-r.length}function Ie(e,t){if(e===null)return"null";if(typeof e=="string")return JSON.stringify(e);if(typeof e=="boolean")return e?"true":"false";if(typeof e=="number"){if(!Number.isFinite(e))throw new D;return JSON.stringify(e)}if(typeof e=="bigint")return e<=BigInt(Number.MAX_SAFE_INTEGER)&&e>=BigInt(Number.MIN_SAFE_INTEGER)?JSON.stringify(Number(e)):JSON.stringify(e.toString());if(Array.isArray(e)){if(t.has(e))throw new TypeError("Structured content must be acyclic.");t.add(e);try{return`[${e.map(n=>n===void 0?"null":Ie(n,t)).join(",")}]`}finally{t.delete(e)}}if(typeof e=="object"){let n=e;if(t.has(n))throw new TypeError("Structured content must be acyclic.");t.add(n);try{return`{${Object.keys(n).filter(o=>n[o]!==void 0).sort(xn).map(o=>`${JSON.stringify(o)}:${Ie(n[o],t)}`).join(",")}}`}finally{t.delete(n)}}return"null"}function oe(e){return Ie(e,new Set)}var D,at=w(()=>{"use strict";D=class extends Error{code="non_finite_number";constructor(){super("Canonical structured content contains a non-finite number."),this.name="NonFiniteNumberCanonicalizationError"}}});import{parseDocument as Tn}from"yaml";function ct(e,t){if(typeof e=="string")return e;if(typeof e!="object"||e===null)return null;let n=e;for(let r of t)if(typeof n[r]=="string")return n[r];return null}function yn(e,t){if(typeof e!="string"){let o=ct(e,["mediaType","mimeType","contentType"]);if(o!==null)return t.mediaTypes.includes(o.split(";")[0].trim().toLowerCase())}let n=typeof e=="string"?e:ct(e,["filename","name","path","extension"]);if(n===null)return!1;let r=n.toLowerCase();return t.extensions.some(o=>r.endsWith(o))}function dt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function ut(e){return JSON.parse(e,(t,n,r)=>{let o=r?.source;return typeof n=="number"&&!Number.isSafeInteger(n)&&o!==void 0&&/^-?(?:0|[1-9][0-9]*)$/.test(o)?BigInt(o):n})}function Pe(e){try{return{canonicalContent:oe(e)}}catch(t){if(t instanceof D)return{canonicalizationError:t.code};throw t}}function bn(e){let t;try{t=ut(e)}catch{throw new U("extraction_invalid_json")}return{sections:typeof t=="object"&&t!==null&&!Array.isArray(t)?Object.entries(t).map(([r,o],i)=>({ordinal:i,heading:r,level:1,text:dt(o)})):[{ordinal:0,heading:null,level:null,text:dt(t)}],...Pe(t)}}function Sn(e){let t=[],n=[];for(let r of e.split(/\r?\n/)){let o=r.trim();if(o.length!==0){try{n.push(ut(o))}catch{throw new U("extraction_invalid_json")}t.push({ordinal:t.length,heading:null,level:null,text:o})}}return{sections:t,...Pe(n)}}function j(e,t=new Set){if(typeof e!="object"||e===null)return e;if(t.has(e))throw new U("extraction_invalid_yaml");t.add(e);try{if(e instanceof Map){let r=Object.create(null);for(let[o,i]of e){let s=typeof o=="string"?o:oe(j(o,t));r[s]=j(i,t)}return r}if(Array.isArray(e))return e.map(r=>j(r,t));let n=Object.create(null);for(let[r,o]of Object.entries(e))n[r]=j(o,t);return n}finally{t.delete(e)}}function lt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function wn(e){let t=e.contents,n=new Map;for(let r of t?.items??[]){let o=r.key?.range?.[0];if(o===void 0)continue;let i=r.key&&Object.hasOwn(r.key,"value")?r.key.value:r.key?.toJSON?.(),s=j(i),a=typeof s=="string"?s:oe(s);n.set(a,o)}return n}function Rn(e){let t,n,r;try{let i=Tn(e,{schema:"core",uniqueKeys:!1,intAsBigInt:!0});if(i.errors.length>0)throw new Error("invalid YAML document");t=j(i.toJS({mapAsMap:!0})),n=wn(i)}catch(i){if(!(i instanceof D))throw new U("extraction_invalid_yaml");r=i.code,t=null,n=new Map}if(r!==void 0)return{sections:[{ordinal:0,heading:null,level:null,text:e.trim()}],canonicalizationError:r};let o;if(typeof t=="object"&&t!==null&&!Array.isArray(t)){let i=Object.entries(t).map(([s,a])=>({heading:s,value:a,start:n.get(s)})).filter(s=>s.start!==void 0).sort((s,a)=>s.start-a.start);o=i.map(({heading:s,start:a},c)=>({ordinal:c,heading:s,level:1,text:e.slice(a,i[c+1]?.start??e.length).trim()})),o.length===0&&(o=[{ordinal:0,heading:null,level:null,text:lt(t)}])}else o=[{ordinal:0,heading:null,level:null,text:lt(t)}];return{sections:o,...Pe(t)}}function Xe(e){return{name:e.name,version:"1.0.0",mediaTypes:e.mediaTypes,supports(t){return yn(t,e)},async extract(t){let n=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(t),r=n.startsWith("\uFEFF")?n.slice(1):n;if(r.length===0)return{sections:[],versionLabel:null,mediaType:null};let o;switch(e.kind){case"json":o=bn(r);break;case"jsonl":o=Sn(r);break;case"yaml":o=Rn(r);break}return{sections:o.sections,versionLabel:null,mediaType:e.mediaTypes[0]??null,...o.canonicalContent===void 0?{}:{canonicalContent:o.canonicalContent},...o.canonicalizationError===void 0?{}:{canonicalizationError:o.canonicalizationError}}}}}var U,mt,ft,Ce,pt=w(()=>{"use strict";at();U=class extends Error{code;constructor(t){super(t==="extraction_invalid_json"?"Structured JSON is invalid.":"Structured YAML is invalid."),this.name="StructuredTextExtractionError",this.code=t}};mt=Xe({kind:"json",name:"json",extensions:[".json"],mediaTypes:["application/json"]}),ft=Xe({kind:"jsonl",name:"jsonl",extensions:[".jsonl"],mediaTypes:["application/x-ndjson"]}),Ce=Xe({kind:"yaml",name:"yaml",extensions:[".yaml",".yml"],mediaTypes:["application/yaml"]})});import{PassThrough as Nn}from"node:stream";import{StringDecoder as An}from"node:string_decoder";import Ln from"jszip";function On(e){if(typeof e=="string")return/\.pptx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===ke;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.pptx$/i.test(r)}function In(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10);return String.fromCodePoint(r)}}})}function gt(e){return Number(e.match(/\/slide(\d+)\.xml$/)?.[1]??Number.MAX_SAFE_INTEGER)}function x(e){throw new b("archive_too_large",e)}function Pn(e,t,n,r){t>n&&x(`PPTX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&x(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}function F(e,t,n){t+8>n&&x("PPTX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&x("PPTX ZIP64 metadata exceeds safe bounds."),Number(r)}function G(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Xn(e,t){let n=F(e,t+8,t+20),r=o=>{if(!G(e,o,101075792)||o+12>t)return!1;let i=F(e,o+4,t);return i>=44&&o+12+i===t};if(r(n))return n;for(let o=t-56;o>=0;o-=1)if(r(o))return o;x("PPTX ZIP64 end record cannot be located safely.")}function Cn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),l=e.getUint16(s+2,!0),d=s+4,u=d+l;if((u>a||u>n)&&x("PPTX central-directory extra field is truncated."),c===1)return F(e,d,u);s=u}x("PPTX ZIP64 entry size is missing from its central-directory extra field.")}function kn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let p=e.byteLength-22;p>=n;p-=1)if(G(t,p,101010256)&&p+22+t.getUint16(p+20,!0)===e.byteLength){r=p;break}if(r<0)return new Map;let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=G(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&x("PPTX ZIP64 end record is missing.");let d=o,u=i,g=s,E=r;if(c){let p=Xn(t,a),R=a;d=F(t,p+32,R),u=F(t,p+40,R),g=F(t,p+48,R),E=p}d>L&&x(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>E||g>E||u+g>E)&&x("PPTX central-directory bounds cannot be checked safely.");let h=E-u-g,m=g+h,f=m+u,y=0,A=0,z=new Map;for(;m+4<=f&&G(t,m,33639248);){m+46>f&&x("PPTX central-directory bounds cannot be checked safely.");let p=t.getUint16(m+28,!0),R=t.getUint16(m+30,!0),he=t.getUint16(m+32,!0),H=m+46+p+R+he;H>f&&x("PPTX central-directory entry is truncated."),y+=1,y>L&&x(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let X=Cn(t,m,f,p,R),_e=new TextDecoder().decode(e.subarray(m+46,m+46+p));z.set(_e,X),X>O-A&&x(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=X,m=H}if(m<f&&G(t,m,84233040)&&m+6<=f){let p=t.getUint16(m+4,!0);m+6+p===f&&(m=f)}return(m!==f||y!==d)&&x("PPTX central-directory count or bounds cannot be checked safely."),z}async function vn(e,t,n){let r=[],o=new An("utf8"),i=new Nn,s=e.nodeStream("nodebuffer");s.on("error",c=>i.destroy(c)),s.pipe(i);let a=0;try{for await(let c of i){let l=Buffer.isBuffer(c)?c:Buffer.from(c);a+=l.byteLength,Pn(e.name,a,n,t+a),r.push(o.write(l))}}catch(c){throw c instanceof Error&&c.message.includes("uncompressed data size mismatch")&&x(`PPTX entry ${e.name} decompressed size differs from its declared size (${n} bytes).`),c}return r.push(o.end()),{xml:r.join(""),bytesRead:a}}async function Mn(e){let t=kn(e),r=(await Ln.loadAsync(e)).file(/^ppt\/slides\/slide\d+\.xml$/).sort((s,a)=>gt(s.name)-gt(a.name)),o=[],i=0;for(let s of r){let a=t.get(s.name);a===void 0&&x(`PPTX central-directory size for ${s.name} cannot be checked safely.`);let c=await vn(s,i,a);i+=c.bytesRead;let l=[...c.xml.matchAll(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/g)].map(d=>In(d[1]??"").trim()).filter(d=>d.length>0).join(" ");l.length>0&&o.push({ordinal:o.length,heading:null,level:null,text:l})}return{sections:o,versionLabel:null,mediaType:ke}}var ke,L,O,Et,ve=w(()=>{"use strict";M();ke="application/vnd.openxmlformats-officedocument.presentationml.presentation",L=5e3,O=128*1024*1024;Et={name:"pptx",version:"3.10.2",mediaTypes:[ke],supports:On,extract:Mn}});import{posix as ht}from"node:path";import{PassThrough as Dn}from"node:stream";import{StringDecoder as jn}from"node:string_decoder";import Un from"jszip";function $n(e){if(typeof e=="string")return/\.xlsx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===Me;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.xlsx$/i.test(r)}function T(e){throw new b("sheet_too_large",e)}function _(e){throw new b("archive_too_large",e)}function bt(e){let t=/^\$?([A-Z]{1,3})\$?(\d+)$/i.exec(e);if(!t?.[1]||!t[2])return null;let n=0;for(let o of t[1].toUpperCase())n=n*26+o.charCodeAt(0)-64;let r=Number(t[2]);return Number.isSafeInteger(r)&&r>0&&n<=Fn?{row:r,column:n}:null}function Tt(e,t){(!Number.isSafeInteger(e)||e<1||!Number.isSafeInteger(t)||t<1)&&T("Worksheet contains an invalid row or column reference."),(e>Y||e*t>ae)&&T(`Worksheet exceeds MAX_SHEET_ROWS (${Y}) or MAX_SHEET_CELLS (${ae}).`)}function S(e,t){return e.match(new RegExp(`\\b${t}\\s*=\\s*[\\"']([^\\"']*)[\\"']`,"i"))?.[1]}function $(e,t,n){t+8>n&&_("XLSX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&_("XLSX ZIP64 metadata exceeds safe bounds."),Number(r)}function V(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Bn(e,t){let n=$(e,t+8,t+20),r=i=>{if(!V(e,i,101075792)||i+12>t)return!1;let s=$(e,i+4,t);return s>=44&&i+12+s===t};if(r(n))return n;let o=0;for(let i=t-56;i>=0;i-=1)if(o>=xt&&_(`XLSX ZIP64 recovery scan exceeds MAX_ZIP64_RECOVERY_SCAN_ITERATIONS (${xt} iterations).`),o+=1,r(i))return i;_("XLSX ZIP64 end record cannot be located safely.")}function zn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),l=e.getUint16(s+2,!0),d=s+4,u=d+l;if((u>a||u>n)&&_("XLSX central-directory extra field is truncated."),c===1)return $(e,d,u);s=u}_("XLSX ZIP64 entry size is missing from its central-directory extra field.")}function Hn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let p=e.byteLength-22;p>=n;p-=1)if(V(t,p,101010256)&&p+22+t.getUint16(p+20,!0)===e.byteLength){r=p;break}r<0&&_("XLSX end-of-central-directory record is missing.");let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=V(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&_("XLSX ZIP64 end record is missing.");let d=o,u=i,g=s,E=r;if(c){let p=Bn(t,a),R=a;d=$(t,p+32,R),u=$(t,p+40,R),g=$(t,p+48,R),E=p}d>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>E||g>E||u+g>E)&&_("XLSX central-directory bounds cannot be checked safely.");let h=E-u-g,m=g+h,f=m+u,y=0,A=0,z=new Map;for(;m+4<=f&&V(t,m,33639248);){m+46>f&&_("XLSX central-directory bounds cannot be checked safely.");let p=t.getUint16(m+28,!0),R=t.getUint16(m+30,!0),he=t.getUint16(m+32,!0),H=m+46+p+R+he;H>f&&_("XLSX central-directory entry is truncated."),y+=1,y>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let X=zn(t,m,f,p,R),_e=new TextDecoder().decode(e.subarray(m+46,m+46+p));z.set(_e,X),X>O-A&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=X,m=H}if(m<f&&V(t,m,84233040)&&m+6<=f){let p=t.getUint16(m+4,!0);m+6+p===f&&(m=f)}return(m!==f||y!==d)&&_("XLSX central-directory count or bounds cannot be checked safely."),z}function Gn(e,t,n,r){t>n&&_(`XLSX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}async function St(e,t,n,r){let o=[],i=new jn("utf8"),s=new Dn,a=e.nodeStream("nodebuffer");a.on("error",d=>s.destroy(d)),a.pipe(s);let c=0;try{for await(let d of s){let u=Buffer.isBuffer(d)?d:Buffer.from(d);c+=u.byteLength,Gn(e.name,c,t,n.bytes+c);let g=i.write(u);o.push(g),r?.(g,u.byteLength)}}catch(d){throw d instanceof Error&&d.message.includes("uncompressed data size mismatch")&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),d}let l=i.end();return o.push(l),r?.(l,0),c!==t&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),n.bytes+=c,o.join("")}function Vn(e,t,n,r){n.bytes+=t,n.bytes>_t&&T(`Worksheet XML scan exceeds MAX_SHEET_SCAN_BYTES (${_t} bytes).`);let o=r.pending+e,i=0;for(r.pending="";;){let s=o.indexOf("<",i);if(s<0)break;let a=o.indexOf(">",s+1);if(a<0){r.pending=o.slice(s),Buffer.byteLength(r.pending)>ie&&T(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${ie} bytes).`);break}let c=o.slice(s,a+1);if(i=a+1,Buffer.byteLength(c)>ie&&T(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${ie} bytes).`),/^<(?:[\w.-]+:)?row\b/i.test(c)){r.rowCount+=1;let l=S(c,"r");l!==void 0&&!/^\d+$/.test(l)&&T("Worksheet contains an invalid row reference.");let d=Number(l??r.rowCount);(!Number.isSafeInteger(d)||d<1)&&T("Worksheet contains an invalid row reference."),(d>Y||r.rowCount>Y)&&T(`Worksheet exceeds MAX_SHEET_ROWS (${Y}).`),r.maxRow=Math.max(r.maxRow,d),Tt(r.maxRow,r.maxColumn)}else if(/^<(?:[\w.-]+:)?c\b/i.test(c)){r.cellCount+=1,r.cellCount>ae&&T(`Worksheet exceeds MAX_SHEET_CELLS (${ae}).`);let l=S(c,"r"),d=l===void 0?null:bt(l);l!==void 0&&!d&&T("Worksheet contains an invalid cell reference."),d&&(r.maxRow=Math.max(r.maxRow,d.row),r.maxColumn=Math.max(r.maxColumn,d.column),Tt(r.maxRow,r.maxColumn))}}}function W(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10),o=r===9||r===10||r===13||r>=32&&r<=55295||r>=57344&&r<=65533||r>=65536&&r<=1114111;return(!Number.isInteger(r)||!o)&&T("XLSX contains an invalid XML character reference."),String.fromCodePoint(r)}}})}function wt(e){return[...e.replace(/<(?:[\w.-]+:)?rPh\b[\s\S]*?<\/(?:[\w.-]+:)?rPh\s*>/gi,"").matchAll(/<(?:[\w.-]+:)?t\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?t\s*>/gi)].map(n=>W(n[1]??"")).join("")}function Yn(e){return[...e.matchAll(/<(?:[\w.-]+:)?si\b[^>]*\/>|<(?:[\w.-]+:)?si\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?si\s*>/gi)].map(t=>t[0].endsWith("/>")?"":wt(t[1]??""))}function Jn(e){let t;try{t=decodeURIComponent(W(e))}catch(o){if(o instanceof b)throw o;return null}let n=t.split(/[?#]/,1)[0]??"",r=ht.normalize(n.startsWith("/")?n.slice(1):ht.join("xl",n));return r.startsWith("xl/")?r:null}function Wn(e){let t=new Map;for(let n of e.matchAll(/<(?:[\w.-]+:)?Relationship\b[^>]*\/?>/gi)){let r=n[0];if(!/\/worksheet$/i.test(S(r,"Type")??"")||/^external$/i.test(S(r,"TargetMode")??""))continue;let o=S(r,"Id"),i=S(r,"Target"),s=i?Jn(i):null;o&&s&&t.set(o,s)}return t}function Kn(e,t){let n=[];for(let r of e.matchAll(/<(?:[\w.-]+:)?sheet\b[^>]*\/?>/gi)){let o=r[0],i=S(o,"name"),s=S(o,"r:id")??S(o,"id"),a=s?t.get(s):void 0;(i===void 0||a===void 0)&&_("XLSX workbook contains an unresolved worksheet relationship."),n.push({name:W(i),part:a})}return n}function Zn(e){let t=e.replace(/"(?:""|[^"])*"/g,"").replace(/\\./g,"").replace(/_.|\*./g,"").replace(/\[[^\]]*\]/g,""),n=/y/i.test(t)&&/m/i.test(t)&&/d/i.test(t);return{date:n,dateTime:n&&(/h|s/i.test(t)||/am\/pm|\[(?:h+|s+)\]/i.test(e))}}function qn(e){let t=new Map;for(let r of e.matchAll(/<(?:[\w.-]+:)?numFmt\b[^>]*\/?\s*>/gi)){let o=Number(S(r[0],"numFmtId")),i=S(r[0],"formatCode");Number.isSafeInteger(o)&&i!==void 0&&t.set(o,W(i))}return[...(e.match(/<(?:[\w.-]+:)?cellXfs\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?cellXfs\s*>/i)?.[1]??"").matchAll(/<(?:[\w.-]+:)?xf\b[^>]*\/?\s*>/gi)].map(r=>{let o=Number(S(r[0],"numFmtId")??"0");return[14,15,16,17].includes(o)?{date:!0,dateTime:!1}:o===22?{date:!0,dateTime:!0}:Zn(t.get(o)??"")})}function Qn(e,t){let n=Number(e);(e.trim()===""||!Number.isFinite(n))&&T("Worksheet contains an invalid date serial.");let r=Math.floor(n),o=n-r,i=r>=60?r-1:r,s=Date.UTC(1899,11,31)+i*864e5+Math.round(o*864e5),a=new Date(s);return Number.isFinite(a.getTime())||T("Worksheet date serial is outside the supported range."),t?a.toISOString():a.toISOString().slice(0,10)}function er(e,t,n){let r=e.indexOf(">"),o=r<0?e:e.slice(0,r+1),i=S(o,"t");if(i==="s"){let u=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1]?.trim();(u===void 0||!/^\d+$/.test(u))&&T("Worksheet contains an invalid shared-string index.");let g=Number(u);return(!Number.isSafeInteger(g)||t[g]===void 0)&&T("Worksheet shared-string index is outside the shared-strings table."),t[g]??""}if(i==="inlineStr"){let u=e.match(/<(?:[\w.-]+:)?is\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?is\s*>/i)?.[1]??"";return wt(u).trim()}let s=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1];if(s===void 0)return"";let a=W(s).trim(),c=S(o,"s"),l=c===void 0?0:Number(c);(!Number.isSafeInteger(l)||l<0||c!==void 0&&n[l]===void 0)&&T("Worksheet contains an invalid cell style reference.");let d=n[l];return d?.date?Qn(a,d.dateTime):a}function tr(e,t,n){let r=new Map,o=0;for(let s of e.matchAll(/<(?:[\w.-]+:)?row\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?row\s*>/gi)){o+=1;let a=s[0].slice(0,s[0].indexOf(">")+1),c=Number(S(a,"r")??o),l=1;for(let d of(s[1]??"").matchAll(/<(?:[\w.-]+:)?c\b[^>]*?(?:\/>|>([\s\S]*?)<\/(?:[\w.-]+:)?c\s*>)/gi)){let u=d[0],g=u.slice(0,u.indexOf(">")+1),E=bt(S(g,"r")??""),h=E?.column??l,m=E?.row??c;if(l=h+1,!Number.isSafeInteger(m)||m<1||!Number.isSafeInteger(h)||h<1)continue;let f=r.get(m)??new Map;f.set(h,er(u,t,n)),r.set(m,f)}}let i=[];for(let s of[...r].sort(([a],[c])=>a-c)){let a=s[1],c=0;for(let d of a.keys())c=Math.max(c,d);let l=Array.from({length:c},(d,u)=>a.get(u+1)??"");for(;l.length>0&&l[l.length-1]==="";)l.pop();l.some(d=>d.length>0)&&i.push(l.join(" "))}return i.join(`
|
|
4
|
+
`)}function J(e,t){let n=Object.values(e.files).filter(r=>!r.dir&&r.name.toLowerCase()===t.toLowerCase());return n.length>1&&_(`XLSX package contains ambiguous case variants of ${t}.`),n[0]}function nr(e){return/<(?:[\w.-]+:)?c\b[^>]*\bt\s*=\s*["']s["']/i.test(e)}async function se(e,t,n,r){let o=J(e,t);if(!o)throw new Error(`XLSX package is missing ${t}.`);let i=n.get(o.name);return i===void 0&&_(`XLSX central-directory size for ${o.name} cannot be checked safely.`),St(o,i,r)}async function yt(e,t,n,r){let o=J(e,t);if(!o)return null;let i=n.get(o.name);i===void 0&&_(`XLSX central-directory size for ${o.name} cannot be checked safely.`);let s={bytes:0},a={pending:"",rowCount:0,cellCount:0,maxRow:0,maxColumn:1};return St(o,i,r,(c,l)=>{Vn(c,l,s,a)})}async function rr(e){try{let t=Hn(e),n=await Un.loadAsync(e),r={bytes:0};if(!J(n,"xl/workbook.xml")){for(let[h,m]of Object.entries(n.files))!m.dir&&/^xl\/worksheets\/[^/]+\.xml$/i.test(h)&&await yt(n,h,t,r);T("XLSX package is missing xl/workbook.xml.")}let o=await se(n,"xl/workbook.xml",t,r),i=await se(n,"xl/_rels/workbook.xml.rels",t,r),s=Kn(o,Wn(i));s.length===0&&_("XLSX workbook has no resolvable worksheets.");let a=new Map;for(let h of s){let m=h.part.toLowerCase();if(a.has(m))continue;let f=await yt(n,h.part,t,r);f!==null&&a.set(m,f)}let c=J(n,"xl/sharedStrings.xml"),l=[...a.values()].some(nr);!c&&l&&_("XLSX worksheet references shared strings, but the shared-strings part is missing.");let d=c?Yn(await se(n,c.name,t,r)):[],u=J(n,"xl/styles.xml"),g=u?qn(await se(n,u.name,t,r)):[],E=[];for(let h of s){let m=a.get(h.part.toLowerCase());m===void 0&&_(`XLSX package is missing worksheet ${h.part}.`);let f=tr(m,d,g);f.trim().length!==0&&E.push({ordinal:E.length,heading:h.name,level:1,text:f})}return{sections:E,versionLabel:null,mediaType:Me}}catch(t){if(t instanceof b)throw t;_("XLSX package could not be parsed safely.")}}var Me,Y,ae,ie,_t,Fn,xt,Rt,Nt=w(()=>{"use strict";ve();M();Me="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",Y=1e5,ae=1e6,ie=64*1024,_t=16*1024*1024,Fn=16384,xt=1e6;Rt={name:"xlsx",version:"4.4.0",mediaTypes:[Me],supports:$n,extract:rr}});import*as je from"node:path";function At(e){let t=je.extname(e).toLowerCase(),n=or.get(t);if(!n||!n.supports({filename:e}))throw new De(e);return n}var De,or,Lt=w(()=>{"use strict";Ve();Ae();nt();st();pt();ve();Nt();De=class extends Error{code="unsupported_media_type";extension;constructor(t){let n=je.extname(t).toLowerCase()||"(none)";super(`Unsupported media type for extension "${n}".`),this.name="UnsupportedMediaTypeError",this.extension=n}},or=new Map([[".pdf",tt],[".docx",Ge],[".png",ne],[".jpg",ne],[".jpeg",ne],[".xlsx",Rt],[".pptx",Et],[".txt",Oe],[".md",Oe],[".yaml",Ce],[".yml",Ce],[".json",mt],[".jsonl",ft]])});import*as N from"node:fs/promises";import*as ce from"node:path";function sr(e){if(!/^[0-9a-f]{64}$/.test(e)||e.includes("..")||e.includes("/")||e.includes("\\"))throw new Error("Invalid SHA-256 hash: expected 64 lowercase hexadecimal characters.");return ce.join("objects",e.slice(0,2),e.slice(2,4),e)}async function Ot(e,t){return N.readFile(ce.join(e,sr(t)))}var _o,xo,ir,To,It=w(()=>{"use strict";M();_o=64*1024,xo=1440*60*1e3,ir=["EPERM","ENOTSUP","EXDEV","EACCES"],To=new Set(ir)});import*as v from"node:fs";import*as C from"node:path";var Z=class extends Error{code;path;constructor(t,n,r){let o=r instanceof Error?r.message:String(r);super(n==="config_invalid"?`Invalid JSON in config file ${t}: ${o}`:`Unable to read config file ${t}: ${o}`,{cause:r}),this.name="ConfigDiscoveryError",this.code=n,this.path=t}};function Ht(e){let t;try{t=v.readFileSync(e,"utf8")}catch(n){throw new Z(e,"config_unreadable",n)}try{return JSON.parse(t)}catch(n){throw new Z(e,"config_invalid",n)}}function xe({cwd:e,homeDir:t,argv:n,env:r}){let o={};for(let y=0;y<n.length;y+=1){let A=/^--(client|project|root)=(.*)$/.exec(n[y]);A?o[A[1]]=A[2]:/^--(client|project|root)$/.test(n[y])&&n[y+1]!==void 0&&(o[n[y].slice(2)]=n[y+1],y+=1)}let i={client:r.RAG_CLIENT,project:r.RAG_PROJECT,root:r.RAG_ROOT},s=C.join(e,"rag.config.json"),a=C.join(C.dirname(e),"rag.config.json"),c=C.join(t,".rag","config.json"),l=v.existsSync(s)?s:v.existsSync(a)?a:v.existsSync(c)?c:null,d=l===null?{}:Ht(l),u=typeof d.client=="string"?d.client:void 0,g=typeof d.project=="string"?d.project:void 0,E=o.client??i.client??u??null,h=o.project??i.project??g??null,m=E===null&&h===null?null:{client:E,project:h},f=o.client!==void 0||o.project!==void 0?"args":i.client!==void 0||i.project!==void 0?"env":u!==void 0||g!==void 0?l===s?"cwd":l===a?"parent":"home":"unscoped";return{namespace:m,source:f,configPath:f==="cwd"||f==="parent"||f==="home"?l:null,failClosed:typeof d.failClosed=="boolean"?d.failClosed:!0,root:o.root??i.root??(typeof d.root=="string"?d.root:C.join(t,".rag"))}}be();import*as ee from"node:fs";import{DatabaseSync as Vt}from"node:sqlite";var Yt=240,ze=/[\p{L}\p{M}\p{N}]+/gu;function Jt(e){return e.match(ze)??[]}function Wt(e){return e.map(t=>`"${t.replaceAll('"','""')}"`).join(" AND ")}function Be(e){return e.normalize("NFD").replace(new RegExp("\\p{M}","gu"),"").toLowerCase()}function Kt(e,t){let n=new Set(t.map(Be));for(let r of e.matchAll(ze)){let o=r[0];if(!n.has(Be(o)))continue;let i=r.index;return{start:i,end:i+o.length}}return null}function Zt(e,t,n,r){let o=Math.min(r,Math.max(0,e.length-1));if(o===0)return"";let i=Math.min(n-t,o),s=Math.max(0,Math.min(t-Math.floor((o-i)/2),e.length-o));return e.slice(s,s+o)}function qt(e,t){return e<t?-1:e>t?1:0}function Q(e,t,n){return{namespace:e,error:{code:t,message:n}}}function He(e){let{root:t,namespace:n,query:r}=e,o=e.excerptLength??Yt;if(!Number.isInteger(o)||o<1)return Q(n,"invalid_excerpt_length","excerptLength must be a positive integer.");let i=Jt(r);if(i.length===0)return{namespace:n,result:{citations:[],total:0}};let s;try{let a=q(t,n.client);if(!ee.existsSync(a)||!ee.statSync(a).isFile())return Q(n,"database_not_found",`No database exists for namespace ${n.client}/${n.project}.`);s=new Vt(a,{readOnly:!0});let c=s.prepare(`SELECT dv.doc_version_id AS docVersionId, d.document_id AS documentId,
|
|
5
|
+
e.entity_key AS entityKey, d.label AS label, dv.version_label AS versionLabel,
|
|
6
|
+
s.ordinal AS sectionOrdinal, c.ordinal AS chunkOrdinal, c.text AS text,
|
|
7
|
+
c.char_start AS charStart, c.char_end AS charEnd, bm25(chunk_fts) AS score
|
|
8
|
+
FROM chunk_fts
|
|
9
|
+
JOIN chunk AS c ON c.rowid = chunk_fts.rowid
|
|
10
|
+
JOIN doc_version AS dv ON dv.doc_version_id = c.doc_version_id
|
|
11
|
+
JOIN document AS d ON d.document_id = dv.document_id
|
|
12
|
+
JOIN entity AS e ON e.entity_id = d.entity_id
|
|
13
|
+
JOIN project AS p ON p.project_id = d.project_id
|
|
14
|
+
JOIN client AS cl ON cl.client_id = p.client_id
|
|
15
|
+
LEFT JOIN section AS s ON s.section_id = c.section_id
|
|
16
|
+
WHERE chunk_fts MATCH ? AND cl.name = ? AND p.slug = ?
|
|
17
|
+
AND d.deleted_at IS NULL AND e.deleted_at IS NULL
|
|
18
|
+
ORDER BY score ASC, dv.doc_version_id COLLATE BINARY ASC, c.ordinal ASC`).all(Wt(i),n.client,n.project),l=[];for(let d of c){let u=Kt(d.text,i);if(!u)return Q(n,"match_offsets_unavailable","A matched FTS token could not be located in its stored chunk text.");l.push({docVersionId:d.docVersionId,documentId:d.documentId,entityKey:d.entityKey,label:d.label,versionLabel:d.versionLabel,sectionOrdinal:d.sectionOrdinal,chunkOrdinal:d.chunkOrdinal,charStart:d.charStart,charEnd:d.charEnd,matchStart:u.start,matchEnd:u.end,score:d.score,excerpt:Zt(d.text,u.start,u.end,o)})}return l.sort((d,u)=>d.score-u.score||qt(d.docVersionId,u.docVersionId)||d.chunkOrdinal-u.chunkOrdinal),{namespace:n,result:{citations:l,total:l.length}}}catch(a){let c=a instanceof Error?a.message:String(a);return Q(n,"database_unavailable",c)}finally{s?.close()}}Lt();be();It();import*as de from"node:fs";import*as K from"node:path";import{DatabaseSync as ar}from"node:sqlite";function I(e,t,n){return{namespace:e,error:{code:t,message:n}}}function cr(e,t){let n=Buffer.from(e,"utf8");if(n.byteLength<=t)return{text:e,truncated:!1};let r=t;for(;r>0;)try{return{text:new TextDecoder("utf-8",{fatal:!0}).decode(n.subarray(0,r)),truncated:!0}}catch{r-=1}return{text:"",truncated:!0}}function dr(e,t){if(e.length===0)return{text:e,truncated:!1};if(t===0)return{text:"",truncated:!0};let n=1;for(let r=0;r<e.length;r+=1){if(e[r]!=="\r"&&e[r]!==`
|
|
19
|
+
`)continue;let o=r;if(e[r]==="\r"&&e[r+1]===`
|
|
20
|
+
`&&(r+=1),n===t)return{text:e.slice(0,o),truncated:!0};n+=1}return{text:e,truncated:!1}}async function Pt(e){let{root:t,namespace:n,docVersionId:r,chunkOrdinal:o,maxBytes:i,maxLines:s}=e;if(!Number.isInteger(i)||i<0||!Number.isInteger(s)||s<0)return I(n,"invalid_read_limit","maxBytes and maxLines must be non-negative integers.");let a;try{let c=q(t,n.client);if(!de.existsSync(c)||!de.statSync(c).isFile())return I(n,"database_not_found",`No database exists for namespace ${n.client}/${n.project}.`);a=new ar(c,{readOnly:!0});let l=a.prepare(`SELECT o.sha256 AS sha256, o.rel_path AS relPath, d.label AS label
|
|
21
|
+
FROM chunk AS c
|
|
22
|
+
JOIN doc_version AS dv ON dv.doc_version_id = c.doc_version_id
|
|
23
|
+
JOIN document AS d ON d.document_id = dv.document_id
|
|
24
|
+
JOIN project AS p ON p.project_id = d.project_id
|
|
25
|
+
JOIN client AS cl ON cl.client_id = p.client_id
|
|
26
|
+
LEFT JOIN object AS o ON o.sha256 = dv.sha256
|
|
27
|
+
WHERE c.doc_version_id = ? AND c.ordinal = ? AND cl.name = ? AND p.slug = ?`).get(r,o,n.client,n.project);if(!l)return I(n,"citation_not_found","The citation does not exist in the requested document version and namespace.");if(!l.sha256||!l.relPath)return I(n,"object_missing","The citation's recorded object is missing.");let d;try{let m=K.resolve(t,l.relPath),f=K.resolve(t);if(m!==f&&!m.startsWith(`${f}${K.sep}`))return I(n,"object_path_invalid","The recorded object path escapes the configured root.");d=await Ot(t,l.sha256)}catch(m){if(m.code==="ENOENT")return I(n,"object_missing","The citation's recorded object file is missing.");throw m}let g=(await At(l.label).extract(d)).sections[o]?.text;if(g===void 0)return I(n,"citation_not_found","The stored object does not contain the requested extracted section.");let E=cr(g,i),h=dr(E.text,s);return{namespace:n,result:{text:h.text,truncated:E.truncated||h.truncated}}}catch(c){let l=c instanceof Error?c.message:String(c);return I(n,"object_read_failed",l)}finally{a?.close()}}import{z as k}from"zod";var Xt=100,Ct=2e3,kt=1048576,vt=1e4,lr=10,ur=240,mr=65536,fr=200;function Mt(e){let t=new Set;for(let n of(e??"find,read").split(",").map(r=>r.trim())){if(n!=="find"&&n!=="read")throw new Error(`invalid_capability: Unknown RAG_CAPABILITIES value "${n}".`);t.add(n)}return t}var ue=k.string().min(1).refine(e=>e.trim().length>0),Ue=k.object({client:ue,project:ue}).strict();function me(e,t,n){return k.number().int().min(e).max(t).optional().transform(r=>r??n)}var Dt=k.object({namespace:Ue,query:ue,limit:me(1,Xt,lr),excerptChars:me(1,Ct,ur)}).strict(),jt=k.object({namespace:Ue,docVersionId:ue,chunkOrdinal:k.number().multipleOf(1).min(0).refine(Number.isSafeInteger),maxBytes:me(0,kt,mr),maxLines:me(0,vt,fr)}).strict();function pr(e){let t=e.find(i=>i.code==="unrecognized_keys"&&i.path.length===0)??e.find(i=>i.code==="unrecognized_keys");if(!t||t.code!=="unrecognized_keys")return null;let n=t.keys[0];if(n===void 0)return null;let r=t.path.map(String).join(".");return{code:"unknown_field",message:`Unknown field "${r.length>0?`${r}.${n}`:n}".`}}function P(e,t){return e.some(n=>n.path[0]===t)}function le(e,t,n){return{code:"invalid_limit",message:`${e} must be an integer from ${t} to ${n}.`}}function Ut(e,t){let n=pr(e);if(n)return n;if(P(e,"namespace"))return{code:"namespace_required",message:"namespace must contain non-empty client and project strings."};if(t==="find"){if(P(e,"query"))return{code:"invalid_query",message:"query must be a non-empty string."};if(P(e,"limit"))return le("limit",1,Xt);if(P(e,"excerptChars"))return le("excerptChars",1,Ct)}else{if(P(e,"docVersionId"))return{code:"invalid_input",message:"docVersionId must be a non-empty string."};if(P(e,"chunkOrdinal"))return{code:"invalid_input",message:"chunkOrdinal must be a non-negative integer."};if(P(e,"maxBytes"))return le("maxBytes",0,kt);if(P(e,"maxLines"))return le("maxLines",0,vt)}return e.some(r=>r.path.length===0)?{code:"invalid_input",message:"Tool arguments must be an object."}:{code:"invalid_input",message:"Tool arguments are invalid."}}function gr(e){let t=Ue.safeParse(e);return t.success?t.data:null}function Ft(e){let t=Dt.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:fe(e),error:Ut(t.error.issues,"find")}}function $t(e){let t=jt.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:fe(e),error:Ut(t.error.issues,"read")}}function Er(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}function fe(e){return Er(e)?gr(e.namespace):null}function Bt(e){return k.toJSONSchema(e,{io:"input"})}var pe={name:"rag_find",description:"Search a client/project knowledge base and return bounded citations.",inputSchema:Bt(Dt)},ge={name:"rag_read",description:"Read bounded text from a cited document chunk in the explicit namespace.",inputSchema:Bt(jt)};function Fe(e,t=!1){return{content:[{type:"text",text:JSON.stringify(e)}],details:e,...t?{isError:!0}:{}}}function B(e,t){return Fe({namespace:e,error:t},!0)}function Ee(e){return B(e,{code:"cancelled",message:"Tool call was cancelled."})}function zt(e,t){let n=t instanceof Error?t.message:String(t);return B(fe(e),{code:"retrieval_failed",message:n})}function Co(e){let t=Mt(process.env.RAG_CAPABILITIES);t.has("find")&&e.registerTool({name:pe.name,label:"RAG Find",description:pe.description,parameters:pe.inputSchema,async execute(n,r,o,i,s){try{let a=Ft(r);if(!a.ok)return B(a.namespace,a.error);let{namespace:c,query:l,limit:d,excerptChars:u}=a.input;if(o?.aborted)return Ee(c);let g=xe({cwd:s.cwd,homeDir:s.cwd,argv:[],env:process.env}).root,E=He({root:g,namespace:c,query:l,excerptLength:u});return o?.aborted?Ee(c):"error"in E?B(c,E.error):Fe({namespace:c,result:{...E.result,citations:E.result.citations.slice(0,d)}})}catch(a){return zt(r,a)}}}),t.has("read")&&e.registerTool({name:ge.name,label:"RAG Read",description:ge.description,parameters:ge.inputSchema,async execute(n,r,o,i,s){try{let a=$t(r);if(!a.ok)return B(a.namespace,a.error);let{namespace:c,docVersionId:l,chunkOrdinal:d,maxBytes:u,maxLines:g}=a.input;if(o?.aborted)return Ee(c);let E=xe({cwd:s.cwd,homeDir:s.cwd,argv:[],env:process.env}).root,h=await Pt({root:E,namespace:c,docVersionId:l,chunkOrdinal:d,maxBytes:u,maxLines:g});return o?.aborted?Ee(c):"error"in h?B(c,h.error):Fe({namespace:c,result:h.result})}catch(a){return zt(r,a)}}})}export{Co as default};
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { parseDocument } from "yaml";
|
|
2
|
+
import { canonicalizeStructuredContent, NonFiniteNumberCanonicalizationError } from "../store/canonical.mjs";
|
|
1
3
|
class StructuredTextExtractionError extends Error {
|
|
2
4
|
code;
|
|
3
5
|
constructor(code) {
|
|
@@ -29,70 +31,127 @@ function supportsReference(ref, options) {
|
|
|
29
31
|
}
|
|
30
32
|
function renderJsonValue(value) {
|
|
31
33
|
if (typeof value === "string") return value;
|
|
32
|
-
return JSON.stringify(value,
|
|
34
|
+
return JSON.stringify(value, (_key, child) => typeof child === "bigint" ? child.toString() : child, 2) ?? "null";
|
|
35
|
+
}
|
|
36
|
+
function parseJsonContent(content) {
|
|
37
|
+
return JSON.parse(content, (_key, value, context) => {
|
|
38
|
+
const source = context?.source;
|
|
39
|
+
if (typeof value === "number" && !Number.isSafeInteger(value) && source !== void 0 && /^-?(?:0|[1-9][0-9]*)$/.test(source)) {
|
|
40
|
+
return BigInt(source);
|
|
41
|
+
}
|
|
42
|
+
return value;
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
function canonicalFields(value) {
|
|
46
|
+
try {
|
|
47
|
+
return { canonicalContent: canonicalizeStructuredContent(value) };
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (error instanceof NonFiniteNumberCanonicalizationError) return { canonicalizationError: error.code };
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
33
52
|
}
|
|
34
53
|
function extractJson(content) {
|
|
35
54
|
let parsed;
|
|
36
55
|
try {
|
|
37
|
-
parsed =
|
|
56
|
+
parsed = parseJsonContent(content);
|
|
38
57
|
} catch {
|
|
39
58
|
throw new StructuredTextExtractionError("extraction_invalid_json");
|
|
40
59
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
49
|
-
return [{ ordinal: 0, heading: null, level: null, text: renderJsonValue(parsed) }];
|
|
60
|
+
const sections = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? Object.entries(parsed).map(([heading, value], ordinal) => ({
|
|
61
|
+
ordinal,
|
|
62
|
+
heading,
|
|
63
|
+
level: 1,
|
|
64
|
+
text: renderJsonValue(value)
|
|
65
|
+
})) : [{ ordinal: 0, heading: null, level: null, text: renderJsonValue(parsed) }];
|
|
66
|
+
return { sections, ...canonicalFields(parsed) };
|
|
50
67
|
}
|
|
51
68
|
function extractJsonl(content) {
|
|
52
69
|
const sections = [];
|
|
70
|
+
const records = [];
|
|
53
71
|
for (const line of content.split(/\r?\n/)) {
|
|
54
72
|
const text = line.trim();
|
|
55
73
|
if (text.length === 0) continue;
|
|
56
74
|
try {
|
|
57
|
-
|
|
75
|
+
records.push(parseJsonContent(text));
|
|
58
76
|
} catch {
|
|
59
77
|
throw new StructuredTextExtractionError("extraction_invalid_json");
|
|
60
78
|
}
|
|
61
79
|
sections.push({ ordinal: sections.length, heading: null, level: null, text });
|
|
62
80
|
}
|
|
63
|
-
return sections;
|
|
81
|
+
return { sections, ...canonicalFields(records) };
|
|
64
82
|
}
|
|
65
|
-
function
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
83
|
+
function normalizeYamlValue(value, ancestors = /* @__PURE__ */ new Set()) {
|
|
84
|
+
if (typeof value !== "object" || value === null) return value;
|
|
85
|
+
if (ancestors.has(value)) throw new StructuredTextExtractionError("extraction_invalid_yaml");
|
|
86
|
+
ancestors.add(value);
|
|
87
|
+
try {
|
|
88
|
+
if (value instanceof Map) {
|
|
89
|
+
const result2 = /* @__PURE__ */ Object.create(null);
|
|
90
|
+
for (const [key, child] of value) {
|
|
91
|
+
const stringKey = typeof key === "string" ? key : canonicalizeStructuredContent(normalizeYamlValue(key, ancestors));
|
|
92
|
+
result2[stringKey] = normalizeYamlValue(child, ancestors);
|
|
93
|
+
}
|
|
94
|
+
return result2;
|
|
95
|
+
}
|
|
96
|
+
if (Array.isArray(value)) return value.map((child) => normalizeYamlValue(child, ancestors));
|
|
97
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
98
|
+
for (const [key, child] of Object.entries(value)) result[key] = normalizeYamlValue(child, ancestors);
|
|
99
|
+
return result;
|
|
100
|
+
} finally {
|
|
101
|
+
ancestors.delete(value);
|
|
75
102
|
}
|
|
76
|
-
|
|
77
|
-
|
|
103
|
+
}
|
|
104
|
+
function renderYamlValue(value) {
|
|
105
|
+
if (typeof value === "string") return value;
|
|
106
|
+
return JSON.stringify(value, (_key, child) => typeof child === "bigint" ? child.toString() : child, 2) ?? "null";
|
|
107
|
+
}
|
|
108
|
+
function yamlTopLevelStarts(document) {
|
|
109
|
+
const root = document.contents;
|
|
110
|
+
const starts = /* @__PURE__ */ new Map();
|
|
111
|
+
for (const pair of root?.items ?? []) {
|
|
112
|
+
const position = pair.key?.range?.[0];
|
|
113
|
+
if (position === void 0) continue;
|
|
114
|
+
const rawKey = pair.key && Object.hasOwn(pair.key, "value") ? pair.key.value : pair.key?.toJSON?.();
|
|
115
|
+
const key = normalizeYamlValue(rawKey);
|
|
116
|
+
const stringKey = typeof key === "string" ? key : canonicalizeStructuredContent(key);
|
|
117
|
+
starts.set(stringKey, position);
|
|
78
118
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
119
|
+
return starts;
|
|
120
|
+
}
|
|
121
|
+
function extractYaml(content) {
|
|
122
|
+
let parsed;
|
|
123
|
+
let topLevelStarts;
|
|
124
|
+
let canonicalizationError;
|
|
125
|
+
try {
|
|
126
|
+
const document = parseDocument(content, { schema: "core", uniqueKeys: false, intAsBigInt: true });
|
|
127
|
+
if (document.errors.length > 0) throw new Error("invalid YAML document");
|
|
128
|
+
parsed = normalizeYamlValue(document.toJS({ mapAsMap: true }));
|
|
129
|
+
topLevelStarts = yamlTopLevelStarts(document);
|
|
130
|
+
} catch (error) {
|
|
131
|
+
if (!(error instanceof NonFiniteNumberCanonicalizationError)) {
|
|
132
|
+
throw new StructuredTextExtractionError("extraction_invalid_yaml");
|
|
87
133
|
}
|
|
88
|
-
|
|
89
|
-
|
|
134
|
+
canonicalizationError = error.code;
|
|
135
|
+
parsed = null;
|
|
136
|
+
topLevelStarts = /* @__PURE__ */ new Map();
|
|
90
137
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
138
|
+
if (canonicalizationError !== void 0) {
|
|
139
|
+
return { sections: [{ ordinal: 0, heading: null, level: null, text: content.trim() }], canonicalizationError };
|
|
140
|
+
}
|
|
141
|
+
let sections;
|
|
142
|
+
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
|
|
143
|
+
const entries = Object.entries(parsed).map(([heading, value]) => ({ heading, value, start: topLevelStarts.get(heading) })).filter((entry) => entry.start !== void 0).sort((left, right) => left.start - right.start);
|
|
144
|
+
sections = entries.map(({ heading, start }, ordinal) => ({
|
|
145
|
+
ordinal,
|
|
146
|
+
heading,
|
|
147
|
+
level: 1,
|
|
148
|
+
text: content.slice(start, entries[ordinal + 1]?.start ?? content.length).trim()
|
|
149
|
+
}));
|
|
150
|
+
if (sections.length === 0) sections = [{ ordinal: 0, heading: null, level: null, text: renderYamlValue(parsed) }];
|
|
151
|
+
} else {
|
|
152
|
+
sections = [{ ordinal: 0, heading: null, level: null, text: renderYamlValue(parsed) }];
|
|
153
|
+
}
|
|
154
|
+
return { sections, ...canonicalFields(parsed) };
|
|
96
155
|
}
|
|
97
156
|
function createStructuredTextExtractor(options) {
|
|
98
157
|
return {
|
|
@@ -108,19 +167,26 @@ function createStructuredTextExtractor(options) {
|
|
|
108
167
|
if (content.length === 0) {
|
|
109
168
|
return { sections: [], versionLabel: null, mediaType: null };
|
|
110
169
|
}
|
|
111
|
-
let
|
|
170
|
+
let extracted;
|
|
112
171
|
switch (options.kind) {
|
|
113
172
|
case "json":
|
|
114
|
-
|
|
173
|
+
extracted = extractJson(content);
|
|
115
174
|
break;
|
|
116
175
|
case "jsonl":
|
|
117
|
-
|
|
176
|
+
extracted = extractJsonl(content);
|
|
118
177
|
break;
|
|
119
178
|
case "yaml":
|
|
120
|
-
|
|
179
|
+
extracted = extractYaml(content);
|
|
121
180
|
break;
|
|
122
181
|
}
|
|
123
|
-
|
|
182
|
+
const result = {
|
|
183
|
+
sections: extracted.sections,
|
|
184
|
+
versionLabel: null,
|
|
185
|
+
mediaType: options.mediaTypes[0] ?? null,
|
|
186
|
+
...extracted.canonicalContent === void 0 ? {} : { canonicalContent: extracted.canonicalContent },
|
|
187
|
+
...extracted.canonicalizationError === void 0 ? {} : { canonicalizationError: extracted.canonicalizationError }
|
|
188
|
+
};
|
|
189
|
+
return result;
|
|
124
190
|
}
|
|
125
191
|
};
|
|
126
192
|
}
|
package/dist/src/cli/format.mjs
CHANGED
|
@@ -93,6 +93,12 @@ function renderOutput(output, env) {
|
|
|
93
93
|
return `Documents: ${result.documents} | Versions: ${result.versions} | Objects: ${result.objects}`;
|
|
94
94
|
}
|
|
95
95
|
if (typeof result.ok === "boolean" && typeof result.checked === "number") return renderDoctor(result);
|
|
96
|
+
if (typeof result.updated === "number" && typeof result.unchanged === "number" && typeof result.skipped === "number") {
|
|
97
|
+
const label = result.applied === true ? "Rehash applied" : "Rehash preview";
|
|
98
|
+
const summary = `${label}: ${result.updated} updated, ${result.unchanged} unchanged, ${result.skipped} skipped`;
|
|
99
|
+
return typeof result.backupReminder === "string" ? `${summary}
|
|
100
|
+
${oneLine(result.backupReminder)}` : summary;
|
|
101
|
+
}
|
|
96
102
|
if (Array.isArray(result.files) && record(result.summary)) return renderImport(result);
|
|
97
103
|
if (typeof result.status === "string" && typeof result.documentId === "string") return renderDocumentAction(result);
|
|
98
104
|
return JSON.stringify(output) ?? "null";
|
package/dist/src/cli/index.mjs
CHANGED
|
@@ -10,6 +10,7 @@ import { importDirectory } from "../ingest/import.mjs";
|
|
|
10
10
|
import { clientDbPath, openDatabase, validateClientName } from "../store/db.mjs";
|
|
11
11
|
import { backupRoot, reindexRoot, restoreRoot, writeRootManifest } from "../store/backup.mjs";
|
|
12
12
|
import { readObject } from "../store/objects.mjs";
|
|
13
|
+
import { rehashNamespace } from "../store/rehash.mjs";
|
|
13
14
|
import { getActiveLimits } from "../limits.mjs";
|
|
14
15
|
import { renderOutput, resolveOutputFormat } from "./format.mjs";
|
|
15
16
|
import {
|
|
@@ -37,6 +38,7 @@ Commands:
|
|
|
37
38
|
backup --to <dir> Back up the manifest and object store
|
|
38
39
|
restore --from <dir> Restore a backup into this root
|
|
39
40
|
reindex Rebuild the database from manifest and objects
|
|
41
|
+
rehash [--apply] Recompute stored-version content hashes (preview by default)
|
|
40
42
|
|
|
41
43
|
Options:
|
|
42
44
|
--root <path> Knowledge-base root
|
|
@@ -46,7 +48,12 @@ Options:
|
|
|
46
48
|
--from <dir> Backup source
|
|
47
49
|
--type <type> Human relation type for rag review link
|
|
48
50
|
--label <text> Replacement label for rag relabel
|
|
49
|
-
--
|
|
51
|
+
--apply Persist the rag rehash results
|
|
52
|
+
--help Show this help
|
|
53
|
+
|
|
54
|
+
Content identity:
|
|
55
|
+
JSON, JSONL, and YAML use canonical tree hashes (content_kind=structured, hash_version=2).
|
|
56
|
+
Other formats keep normalized-text hashes (content_kind=NULL, hash_version=1).`;
|
|
50
57
|
const HELP = `${USAGE}
|
|
51
58
|
|
|
52
59
|
Output format:
|
|
@@ -89,6 +96,7 @@ const NAMED_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
|
89
96
|
"backup_failed",
|
|
90
97
|
"destination_exists",
|
|
91
98
|
"namespace_mismatch",
|
|
99
|
+
"rehash_failed",
|
|
92
100
|
"restore_destination_not_empty",
|
|
93
101
|
"restore_failed",
|
|
94
102
|
"foreign_client_in_manifest",
|
|
@@ -159,12 +167,17 @@ function parseArgs(argv) {
|
|
|
159
167
|
const positional = [];
|
|
160
168
|
const values = {};
|
|
161
169
|
let help = false;
|
|
170
|
+
let apply = false;
|
|
162
171
|
for (let index = 0; index < argv.length; index += 1) {
|
|
163
172
|
const argument = argv[index];
|
|
164
173
|
if (argument === "--help") {
|
|
165
174
|
help = true;
|
|
166
175
|
continue;
|
|
167
176
|
}
|
|
177
|
+
if (argument === "--apply") {
|
|
178
|
+
apply = true;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
168
181
|
const assignment = /^--(to|from|type|label)=(.*)$/.exec(argument);
|
|
169
182
|
if (assignment) {
|
|
170
183
|
values[assignment[1]] = assignment[2];
|
|
@@ -191,7 +204,7 @@ function parseArgs(argv) {
|
|
|
191
204
|
if (argument.startsWith("--")) throw usageError(`Unknown option: ${argument}`);
|
|
192
205
|
positional.push(argument);
|
|
193
206
|
}
|
|
194
|
-
return { command: positional[0] ?? null, positional: positional.slice(1), help, values };
|
|
207
|
+
return { command: positional[0] ?? null, positional: positional.slice(1), help, values, apply };
|
|
195
208
|
}
|
|
196
209
|
function requireNamespace(config, mode) {
|
|
197
210
|
const { namespace } = resolveCommandNamespace(config, mode);
|
|
@@ -399,6 +412,9 @@ async function runCli(options) {
|
|
|
399
412
|
if (parsed.values.label !== void 0 && parsed.command !== "relabel") {
|
|
400
413
|
return errorOutput(config.namespace, usageError('Option --label is only accepted by "rag relabel".'));
|
|
401
414
|
}
|
|
415
|
+
if (parsed.apply && parsed.command !== "rehash") {
|
|
416
|
+
return errorOutput(config.namespace, usageError('Option --apply is only accepted by "rag rehash".'));
|
|
417
|
+
}
|
|
402
418
|
if (parsed.command === "relabel" && parsed.values.label === void 0) {
|
|
403
419
|
return errorOutput(config.namespace, usageError("Option --label requires a non-blank value.", "invalid_label"));
|
|
404
420
|
}
|
|
@@ -657,6 +673,17 @@ async function runCli(options) {
|
|
|
657
673
|
return commandErrorOutput(config.namespace, error);
|
|
658
674
|
}
|
|
659
675
|
}
|
|
676
|
+
if (parsed.command === "rehash") {
|
|
677
|
+
if (parsed.positional.length !== 0) return commandErrorOutput(config.namespace, usageError("rehash does not accept positional arguments."));
|
|
678
|
+
try {
|
|
679
|
+
const namespace = requireNamespace(config, parsed.apply ? "write" : "read");
|
|
680
|
+
if (!namespace) throw new NamespaceRequiredError();
|
|
681
|
+
const result = await rehashNamespace(path.resolve(options.cwd, config.root), namespace, parsed.apply);
|
|
682
|
+
return { exitCode: 0, output: { namespace, result } };
|
|
683
|
+
} catch (error) {
|
|
684
|
+
return commandErrorOutput(config.namespace, error);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
660
687
|
return errorOutput(config.namespace, usageError(parsed.command === null ? "Command is required." : `Unknown command: ${parsed.command}`));
|
|
661
688
|
}
|
|
662
689
|
async function main(dependencies = {}) {
|
|
@@ -6,7 +6,8 @@ import { ResourceLimitError } from "../ports/extractor.mjs";
|
|
|
6
6
|
import { clientDbPath, openDatabase } from "../store/db.mjs";
|
|
7
7
|
import { writeRootManifest } from "../store/backup.mjs";
|
|
8
8
|
import { deriveTier0Relations, normalizeEntityKey } from "../store/identity.mjs";
|
|
9
|
-
import {
|
|
9
|
+
import { computeContentHashFacts } from "../store/rehash.mjs";
|
|
10
|
+
import { fingerprintDistance, NEAR_DUPLICATE_MAX_DISTANCE } from "../store/similarity.mjs";
|
|
10
11
|
import { putObjectFromPath, readObject } from "../store/objects.mjs";
|
|
11
12
|
import { MAX_SOURCE_FILE_BYTES as SHARED_MAX_SOURCE_FILE_BYTES } from "../limits.mjs";
|
|
12
13
|
import { parseDocument, ParseHostError } from "./parse-host.mjs";
|
|
@@ -86,9 +87,8 @@ async function ingestFile(input) {
|
|
|
86
87
|
recordFailedRun(db, projectId, input.sourcePath, sha256, runStartedAt, error.message, error.code);
|
|
87
88
|
throw error;
|
|
88
89
|
}
|
|
89
|
-
const
|
|
90
|
-
const contentHash =
|
|
91
|
-
const normalizedHash = fingerprintNormalizedText(normalizedText);
|
|
90
|
+
const hashFacts = computeContentHashFacts(extracted);
|
|
91
|
+
const { contentHash, normalizedHash, contentKind, hashVersion } = hashFacts;
|
|
92
92
|
const sourceLabel = path.basename(input.sourcePath);
|
|
93
93
|
const entityKey = normalizeEntityKey(input.entityKey ?? path.parse(sourceLabel).name);
|
|
94
94
|
const label = input.label ?? sourceLabel;
|
|
@@ -125,8 +125,8 @@ async function ingestFile(input) {
|
|
|
125
125
|
db.prepare(`INSERT INTO ingest_run (run_id, project_id, started_at, status, source_uri, sha256)
|
|
126
126
|
VALUES (?, ?, ?, 'running', ?, ?)`).run(runId, projectId, now, input.sourcePath, stored.sha256);
|
|
127
127
|
db.prepare("UPDATE doc_version SET is_current = 0, superseded_at = ? WHERE document_id = ? AND is_current = 1").run(now, documentId);
|
|
128
|
-
db.prepare(`INSERT INTO doc_version (doc_version_id, document_id, sha256, version_label, content_hash, normalized_hash, extractor, extractor_version,
|
|
129
|
-
ingest_run_id, is_current, ingested_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`).run(docVersionId, documentId, stored.sha256, extracted.versionLabel, contentHash, normalizedHash, extractorName, extractorVersion, runId, now);
|
|
128
|
+
db.prepare(`INSERT INTO doc_version (doc_version_id, document_id, sha256, version_label, content_hash, normalized_hash, content_kind, hash_version, extractor, extractor_version,
|
|
129
|
+
ingest_run_id, is_current, ingested_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`).run(docVersionId, documentId, stored.sha256, extracted.versionLabel, contentHash, normalizedHash, contentKind, hashVersion, extractorName, extractorVersion, runId, now);
|
|
130
130
|
let offset = 0;
|
|
131
131
|
extracted.sections.forEach((section, ordinal) => {
|
|
132
132
|
const sectionId = randomUUID();
|
|
@@ -146,7 +146,8 @@ async function ingestFile(input) {
|
|
|
146
146
|
contentHash,
|
|
147
147
|
entityKey,
|
|
148
148
|
projectId,
|
|
149
|
-
versionLabel: extracted.versionLabel
|
|
149
|
+
versionLabel: extracted.versionLabel,
|
|
150
|
+
hashVersion
|
|
150
151
|
};
|
|
151
152
|
const insertRelation = db.prepare(`INSERT INTO doc_relation
|
|
152
153
|
(relation_id, from_doc_version_id, to_doc_version_id, relation_type, score, decided_by, decided_at, evidence)
|
|
@@ -154,7 +155,7 @@ async function ingestFile(input) {
|
|
|
154
155
|
const currentDocument = db.prepare("SELECT deleted_at FROM document WHERE document_id = ?").get(documentId);
|
|
155
156
|
if (currentDocument?.deleted_at === null) {
|
|
156
157
|
const priorVersions = db.prepare(`SELECT dv.doc_version_id AS docVersionId, dv.sha256,
|
|
157
|
-
dv.content_hash AS contentHash, dv.normalized_hash AS normalizedHash,
|
|
158
|
+
dv.content_hash AS contentHash, dv.normalized_hash AS normalizedHash, dv.hash_version AS hashVersion,
|
|
158
159
|
e.entity_key AS entityKey, d.project_id AS projectId, dv.version_label AS versionLabel
|
|
159
160
|
FROM doc_version AS dv
|
|
160
161
|
JOIN document AS d ON d.document_id = dv.document_id
|
|
@@ -173,7 +174,7 @@ async function ingestFile(input) {
|
|
|
173
174
|
JSON.stringify(relation.evidence)
|
|
174
175
|
);
|
|
175
176
|
}
|
|
176
|
-
if (prior.normalizedHash !== null) {
|
|
177
|
+
if (prior.hashVersion === hashVersion && prior.normalizedHash !== null) {
|
|
177
178
|
const distance = fingerprintDistance(normalizedHash, prior.normalizedHash);
|
|
178
179
|
if (distance <= NEAR_DUPLICATE_MAX_DISTANCE) {
|
|
179
180
|
const relationId = `rel_${createHash("sha256").update(JSON.stringify([docVersionId, prior.docVersionId, "near_duplicate"])).digest("hex")}`;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
const TEXT_HASH_VERSION = 1;
|
|
2
|
+
const STRUCTURED_HASH_VERSION = 2;
|
|
3
|
+
class NonFiniteNumberCanonicalizationError extends Error {
|
|
4
|
+
code = "non_finite_number";
|
|
5
|
+
constructor() {
|
|
6
|
+
super("Canonical structured content contains a non-finite number.");
|
|
7
|
+
this.name = "NonFiniteNumberCanonicalizationError";
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
function compareCodePoints(left, right) {
|
|
11
|
+
const leftPoints = Array.from(left, (character) => character.codePointAt(0));
|
|
12
|
+
const rightPoints = Array.from(right, (character) => character.codePointAt(0));
|
|
13
|
+
const sharedLength = Math.min(leftPoints.length, rightPoints.length);
|
|
14
|
+
for (let index = 0; index < sharedLength; index += 1) {
|
|
15
|
+
const leftPoint = leftPoints[index];
|
|
16
|
+
const rightPoint = rightPoints[index];
|
|
17
|
+
if (leftPoint !== rightPoint) return leftPoint - rightPoint;
|
|
18
|
+
}
|
|
19
|
+
return leftPoints.length - rightPoints.length;
|
|
20
|
+
}
|
|
21
|
+
function encode(value, ancestors) {
|
|
22
|
+
if (value === null) return "null";
|
|
23
|
+
if (typeof value === "string") return JSON.stringify(value);
|
|
24
|
+
if (typeof value === "boolean") return value ? "true" : "false";
|
|
25
|
+
if (typeof value === "number") {
|
|
26
|
+
if (!Number.isFinite(value)) throw new NonFiniteNumberCanonicalizationError();
|
|
27
|
+
return JSON.stringify(value);
|
|
28
|
+
}
|
|
29
|
+
if (typeof value === "bigint") {
|
|
30
|
+
if (value <= BigInt(Number.MAX_SAFE_INTEGER) && value >= BigInt(Number.MIN_SAFE_INTEGER)) {
|
|
31
|
+
return JSON.stringify(Number(value));
|
|
32
|
+
}
|
|
33
|
+
return JSON.stringify(value.toString());
|
|
34
|
+
}
|
|
35
|
+
if (Array.isArray(value)) {
|
|
36
|
+
if (ancestors.has(value)) throw new TypeError("Structured content must be acyclic.");
|
|
37
|
+
ancestors.add(value);
|
|
38
|
+
try {
|
|
39
|
+
return `[${value.map((item) => item === void 0 ? "null" : encode(item, ancestors)).join(",")}]`;
|
|
40
|
+
} finally {
|
|
41
|
+
ancestors.delete(value);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (typeof value === "object") {
|
|
45
|
+
const object = value;
|
|
46
|
+
if (ancestors.has(object)) throw new TypeError("Structured content must be acyclic.");
|
|
47
|
+
ancestors.add(object);
|
|
48
|
+
try {
|
|
49
|
+
const members = Object.keys(object).filter((key) => object[key] !== void 0).sort(compareCodePoints).map((key) => `${JSON.stringify(key)}:${encode(object[key], ancestors)}`);
|
|
50
|
+
return `{${members.join(",")}}`;
|
|
51
|
+
} finally {
|
|
52
|
+
ancestors.delete(object);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return "null";
|
|
56
|
+
}
|
|
57
|
+
function canonicalizeStructuredContent(value) {
|
|
58
|
+
return encode(value, /* @__PURE__ */ new Set());
|
|
59
|
+
}
|
|
60
|
+
export {
|
|
61
|
+
NonFiniteNumberCanonicalizationError,
|
|
62
|
+
STRUCTURED_HASH_VERSION,
|
|
63
|
+
TEXT_HASH_VERSION,
|
|
64
|
+
canonicalizeStructuredContent
|
|
65
|
+
};
|
|
@@ -57,7 +57,7 @@ function deriveTier0Relations(a, b) {
|
|
|
57
57
|
const relations = [];
|
|
58
58
|
if (a.sha256 === b.sha256) {
|
|
59
59
|
relations.push(makeRelation(a.docVersionId, b.docVersionId, "same_bytes", { sha256: a.sha256 }));
|
|
60
|
-
} else if (a.contentHash !== null && b.contentHash !== null && a.contentHash === b.contentHash) {
|
|
60
|
+
} else if (a.contentHash !== null && b.contentHash !== null && a.contentHash === b.contentHash && (a.hashVersion ?? 1) === (b.hashVersion ?? 1)) {
|
|
61
61
|
relations.push(makeRelation(a.docVersionId, b.docVersionId, "same_content", { contentHash: a.contentHash }));
|
|
62
62
|
}
|
|
63
63
|
const sameEntity = normalizeEntityKey(a.entityKey) === normalizeEntityKey(b.entityKey);
|
|
@@ -64,6 +64,7 @@ function listReviewCandidates(db, projectId) {
|
|
|
64
64
|
JOIN entity AS right_entity ON right_entity.entity_id = right_document.entity_id
|
|
65
65
|
WHERE left_document.project_id = ? AND right_document.project_id = left_document.project_id
|
|
66
66
|
AND left_version.content_hash IS NOT NULL AND left_version.content_hash = right_version.content_hash
|
|
67
|
+
AND left_version.hash_version = right_version.hash_version
|
|
67
68
|
AND left_document.deleted_at IS NULL AND right_document.deleted_at IS NULL
|
|
68
69
|
AND left_entity.deleted_at IS NULL AND right_entity.deleted_at IS NULL
|
|
69
70
|
AND NOT EXISTS (
|
|
@@ -86,6 +87,7 @@ function listReviewCandidates(db, projectId) {
|
|
|
86
87
|
JOIN entity AS to_entity ON to_entity.entity_id = to_document.entity_id
|
|
87
88
|
WHERE near_relation.relation_type = 'near_duplicate' AND near_relation.decided_by <> 'human'
|
|
88
89
|
AND from_document.project_id = ? AND to_document.project_id = from_document.project_id
|
|
90
|
+
AND from_version.hash_version = to_version.hash_version
|
|
89
91
|
AND from_document.deleted_at IS NULL AND to_document.deleted_at IS NULL
|
|
90
92
|
AND from_entity.deleted_at IS NULL AND to_entity.deleted_at IS NULL
|
|
91
93
|
AND NOT EXISTS (
|
|
@@ -21,7 +21,7 @@ function buildManifest(db) {
|
|
|
21
21
|
ingestRuns: orderedRows(db, "ingest_run"),
|
|
22
22
|
entities: db.prepare("SELECT entity_id, project_id, entity_key, kind, title, created_at, updated_at, deleted_at FROM entity ORDER BY entity_id").all(),
|
|
23
23
|
documents: db.prepare("SELECT document_id, entity_id, project_id, label, created_at, updated_at, deleted_at FROM document ORDER BY document_id").all(),
|
|
24
|
-
versions: db.prepare("SELECT doc_version_id, document_id, sha256, version_label, content_hash, normalized_hash, extractor, extractor_version, ingest_run_id, is_current, ingested_at, superseded_at FROM doc_version ORDER BY doc_version_id").all(),
|
|
24
|
+
versions: db.prepare("SELECT doc_version_id, document_id, sha256, version_label, content_hash, normalized_hash, content_kind, hash_version, extractor, extractor_version, ingest_run_id, is_current, ingested_at, superseded_at FROM doc_version ORDER BY doc_version_id").all(),
|
|
25
25
|
sections: db.prepare("SELECT section_id, doc_version_id, ordinal, heading, level, char_start, char_end FROM section ORDER BY doc_version_id, ordinal, section_id").all(),
|
|
26
26
|
chunks: db.prepare("SELECT chunk_id, doc_version_id, section_id, ordinal, text, char_start, char_end, token_count FROM chunk ORDER BY doc_version_id, ordinal, chunk_id").all(),
|
|
27
27
|
relations: db.prepare("SELECT relation_id, from_doc_version_id, to_doc_version_id, relation_type, score, decided_by, decided_at, evidence FROM doc_relation ORDER BY relation_id").all(),
|
|
@@ -38,7 +38,7 @@ const ROW_COLUMNS = {
|
|
|
38
38
|
ingestRuns: ["run_id", "project_id", "started_at", "finished_at", "status", "source_uri", "sha256", "result_code", "detail"],
|
|
39
39
|
entities: ["entity_id", "project_id", "entity_key", "kind", "title", "created_at", "updated_at", "deleted_at"],
|
|
40
40
|
documents: ["document_id", "entity_id", "project_id", "label", "created_at", "updated_at", "deleted_at"],
|
|
41
|
-
versions: ["doc_version_id", "document_id", "sha256", "version_label", "content_hash", "normalized_hash", "extractor", "extractor_version", "ingest_run_id", "is_current", "ingested_at", "superseded_at"],
|
|
41
|
+
versions: ["doc_version_id", "document_id", "sha256", "version_label", "content_hash", "normalized_hash", "content_kind", "hash_version", "extractor", "extractor_version", "ingest_run_id", "is_current", "ingested_at", "superseded_at"],
|
|
42
42
|
sections: ["section_id", "doc_version_id", "ordinal", "heading", "level", "char_start", "char_end"],
|
|
43
43
|
chunks: ["chunk_id", "doc_version_id", "section_id", "ordinal", "text", "char_start", "char_end", "token_count"],
|
|
44
44
|
relations: ["relation_id", "from_doc_version_id", "to_doc_version_id", "relation_type", "score", "decided_by", "decided_at", "evidence"],
|
|
@@ -47,8 +47,12 @@ const ROW_COLUMNS = {
|
|
|
47
47
|
};
|
|
48
48
|
function validateRowKeys(value, field) {
|
|
49
49
|
const expected = ROW_COLUMNS[field];
|
|
50
|
+
const legacyVersionColumns = field === "versions" ? expected.filter((column) => column !== "content_kind" && column !== "hash_version") : null;
|
|
50
51
|
for (const row of value) {
|
|
51
|
-
if (!isRecord(row)
|
|
52
|
+
if (!isRecord(row)) throw new ManifestError("manifest_invalid", `Manifest ${field} row keys must match expected columns: ${expected.join(", ")}.`);
|
|
53
|
+
const keys = Object.keys(row);
|
|
54
|
+
const matches = (columns) => keys.length === columns.length && columns.every((column) => Object.hasOwn(row, column));
|
|
55
|
+
if (!matches(expected) && !(legacyVersionColumns !== null && matches(legacyVersionColumns))) {
|
|
52
56
|
throw new ManifestError("manifest_invalid", `Manifest ${field} row keys must match expected columns: ${expected.join(", ")}.`);
|
|
53
57
|
}
|
|
54
58
|
}
|
|
@@ -99,7 +103,12 @@ function importManifest(db, value) {
|
|
|
99
103
|
insertRows(db, "INSERT INTO entity (entity_id, project_id, entity_key, kind, title, created_at, updated_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", value.entities, (r) => [r.entity_id, r.project_id, r.entity_key, r.kind, r.title, r.created_at, r.updated_at, r.deleted_at]);
|
|
100
104
|
insertRows(db, "INSERT INTO document (document_id, entity_id, project_id, label, created_at, updated_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?)", value.documents, (r) => [r.document_id, r.entity_id, r.project_id, r.label, r.created_at, r.updated_at, r.deleted_at]);
|
|
101
105
|
insertRows(db, "INSERT INTO object (sha256, byte_size, rel_path, media_type, created_at, last_verified_at) VALUES (?, ?, ?, ?, ?, ?)", value.objects, (r) => [r.sha256, r.byte_size, r.rel_path, r.media_type, r.created_at, r.last_verified_at]);
|
|
102
|
-
|
|
106
|
+
const versions = value.versions.map((version) => ({
|
|
107
|
+
...version,
|
|
108
|
+
content_kind: version.content_kind ?? null,
|
|
109
|
+
hash_version: version.hash_version ?? 1
|
|
110
|
+
}));
|
|
111
|
+
insertRows(db, "INSERT INTO doc_version (doc_version_id, document_id, sha256, version_label, content_hash, normalized_hash, content_kind, hash_version, extractor, extractor_version, ingest_run_id, is_current, ingested_at, superseded_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", versions, (r) => [r.doc_version_id, r.document_id, r.sha256, r.version_label, r.content_hash, r.normalized_hash, r.content_kind, r.hash_version, r.extractor, r.extractor_version, r.ingest_run_id, r.is_current, r.ingested_at, r.superseded_at]);
|
|
103
112
|
insertRows(db, "INSERT INTO section (section_id, doc_version_id, ordinal, heading, level, char_start, char_end) VALUES (?, ?, ?, ?, ?, ?, ?)", value.sections, (r) => [r.section_id, r.doc_version_id, r.ordinal, r.heading, r.level, r.char_start, r.char_end]);
|
|
104
113
|
insertRows(db, "INSERT INTO chunk (chunk_id, doc_version_id, section_id, ordinal, text, char_start, char_end, token_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", value.chunks, (r) => [r.chunk_id, r.doc_version_id, r.section_id, r.ordinal, r.text, r.char_start, r.char_end, r.token_count]);
|
|
105
114
|
db.exec("INSERT INTO chunk_fts(chunk_fts) VALUES ('rebuild');");
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs/promises";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { NonFiniteNumberCanonicalizationError, STRUCTURED_HASH_VERSION, TEXT_HASH_VERSION } from "./canonical.mjs";
|
|
5
|
+
import { clientDbPath, openDatabase } from "./db.mjs";
|
|
6
|
+
import { writeRootManifest } from "./backup.mjs";
|
|
7
|
+
import { readObject } from "./objects.mjs";
|
|
8
|
+
import { fingerprintNormalizedText } from "./similarity.mjs";
|
|
9
|
+
import { parseDocument } from "../ingest/parse-host.mjs";
|
|
10
|
+
function computeContentHashFacts(extracted) {
|
|
11
|
+
const normalizedText = extracted.sections.map((section) => section.text).join("\n").replace(/\s+/g, " ").trim().toLowerCase();
|
|
12
|
+
let contentKind;
|
|
13
|
+
let hashVersion;
|
|
14
|
+
let hashInput;
|
|
15
|
+
let skipped = false;
|
|
16
|
+
try {
|
|
17
|
+
if (extracted.canonicalizationError === "non_finite_number") throw new NonFiniteNumberCanonicalizationError();
|
|
18
|
+
contentKind = extracted.canonicalContent === void 0 ? null : "structured";
|
|
19
|
+
hashVersion = extracted.canonicalContent === void 0 ? TEXT_HASH_VERSION : STRUCTURED_HASH_VERSION;
|
|
20
|
+
hashInput = extracted.canonicalContent ?? normalizedText;
|
|
21
|
+
} catch (error) {
|
|
22
|
+
if (!(error instanceof NonFiniteNumberCanonicalizationError)) throw error;
|
|
23
|
+
contentKind = null;
|
|
24
|
+
hashVersion = TEXT_HASH_VERSION;
|
|
25
|
+
hashInput = normalizedText;
|
|
26
|
+
skipped = true;
|
|
27
|
+
}
|
|
28
|
+
return {
|
|
29
|
+
contentHash: createHash("sha256").update(hashInput).digest("hex"),
|
|
30
|
+
normalizedHash: fingerprintNormalizedText(hashInput),
|
|
31
|
+
contentKind,
|
|
32
|
+
hashVersion,
|
|
33
|
+
skipped
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function errorWithCode(code, message) {
|
|
37
|
+
return Object.assign(new Error(message), { code });
|
|
38
|
+
}
|
|
39
|
+
function mediaHint(extractor, label) {
|
|
40
|
+
const labelExtension = path.extname(label).toLowerCase();
|
|
41
|
+
switch (extractor) {
|
|
42
|
+
case "json":
|
|
43
|
+
return ".json";
|
|
44
|
+
case "jsonl":
|
|
45
|
+
return ".jsonl";
|
|
46
|
+
case "yaml":
|
|
47
|
+
return labelExtension === ".yml" ? ".yml" : ".yaml";
|
|
48
|
+
case "plain-text":
|
|
49
|
+
return labelExtension === ".md" ? ".md" : ".txt";
|
|
50
|
+
case "pdf":
|
|
51
|
+
return ".pdf";
|
|
52
|
+
case "docx":
|
|
53
|
+
return ".docx";
|
|
54
|
+
case "xlsx":
|
|
55
|
+
return ".xlsx";
|
|
56
|
+
case "pptx":
|
|
57
|
+
return ".pptx";
|
|
58
|
+
case "image-ocr":
|
|
59
|
+
return [".png", ".jpg", ".jpeg"].includes(labelExtension) ? labelExtension : ".png";
|
|
60
|
+
default:
|
|
61
|
+
throw errorWithCode("rehash_failed", `Cannot re-extract version with unknown extractor "${extractor}".`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function sameFacts(row, facts) {
|
|
65
|
+
return row.content_hash === facts.contentHash && row.normalized_hash === facts.normalizedHash && row.content_kind === facts.contentKind && row.hash_version === facts.hashVersion;
|
|
66
|
+
}
|
|
67
|
+
async function rehashNamespace(root, namespace, apply) {
|
|
68
|
+
const dbPath = clientDbPath(root, namespace.client);
|
|
69
|
+
try {
|
|
70
|
+
await fs.access(dbPath);
|
|
71
|
+
} catch (error) {
|
|
72
|
+
if (error.code === "ENOENT") {
|
|
73
|
+
throw errorWithCode("database_not_found", `Client database not found: ${dbPath}`);
|
|
74
|
+
}
|
|
75
|
+
throw errorWithCode("io_error", `Unable to inspect client database: ${dbPath}`);
|
|
76
|
+
}
|
|
77
|
+
const db = openDatabase(dbPath);
|
|
78
|
+
try {
|
|
79
|
+
const project = db.prepare(`SELECT project.project_id FROM project
|
|
80
|
+
JOIN client ON client.client_id = project.client_id WHERE client.name = ? AND project.slug = ?`).get(namespace.client, namespace.project);
|
|
81
|
+
if (!project) return { applied: apply, updated: 0, unchanged: 0, skipped: 0, ...!apply ? { backupReminder: reminder() } : {} };
|
|
82
|
+
const rows = db.prepare(`SELECT version.doc_version_id, version.sha256, version.content_hash, version.normalized_hash,
|
|
83
|
+
version.content_kind, version.hash_version, version.extractor, document.label
|
|
84
|
+
FROM doc_version AS version
|
|
85
|
+
JOIN document ON document.document_id = version.document_id
|
|
86
|
+
WHERE document.project_id = ? ORDER BY version.doc_version_id`).all(project.project_id);
|
|
87
|
+
const updates = [];
|
|
88
|
+
let unchanged = 0;
|
|
89
|
+
let skipped = 0;
|
|
90
|
+
for (const row of rows) {
|
|
91
|
+
let bytes;
|
|
92
|
+
try {
|
|
93
|
+
bytes = await readObject(root, row.sha256);
|
|
94
|
+
} catch (error) {
|
|
95
|
+
const code = error.code === "ENOENT" ? "object_missing" : "object_read_failed";
|
|
96
|
+
throw errorWithCode(code, `Cannot read object ${row.sha256} for version ${row.doc_version_id}.`);
|
|
97
|
+
}
|
|
98
|
+
const objectHash = createHash("sha256").update(bytes).digest("hex");
|
|
99
|
+
if (objectHash !== row.sha256) {
|
|
100
|
+
throw errorWithCode("object_corrupt", `Object ${row.sha256} failed SHA-256 verification for version ${row.doc_version_id}.`);
|
|
101
|
+
}
|
|
102
|
+
let extracted;
|
|
103
|
+
try {
|
|
104
|
+
extracted = await parseDocument(
|
|
105
|
+
path.join(root, "objects", row.sha256.slice(0, 2), row.sha256.slice(2, 4), row.sha256),
|
|
106
|
+
namespace,
|
|
107
|
+
void 0,
|
|
108
|
+
{ mediaHint: mediaHint(row.extractor, row.label) }
|
|
109
|
+
);
|
|
110
|
+
} catch (error) {
|
|
111
|
+
if (error.code === "rehash_failed") throw error;
|
|
112
|
+
throw errorWithCode("rehash_failed", `Cannot re-extract version ${row.doc_version_id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
113
|
+
}
|
|
114
|
+
const facts = computeContentHashFacts(extracted);
|
|
115
|
+
if (facts.skipped) {
|
|
116
|
+
skipped += 1;
|
|
117
|
+
} else if (sameFacts(row, facts)) {
|
|
118
|
+
unchanged += 1;
|
|
119
|
+
} else {
|
|
120
|
+
updates.push({
|
|
121
|
+
docVersionId: row.doc_version_id,
|
|
122
|
+
contentHash: facts.contentHash,
|
|
123
|
+
normalizedHash: facts.normalizedHash,
|
|
124
|
+
contentKind: facts.contentKind,
|
|
125
|
+
hashVersion: facts.hashVersion
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (apply && updates.length > 0) {
|
|
130
|
+
db.exec("BEGIN IMMEDIATE;");
|
|
131
|
+
try {
|
|
132
|
+
const update = db.prepare(`UPDATE doc_version SET content_hash = ?, normalized_hash = ?, content_kind = ?, hash_version = ?
|
|
133
|
+
WHERE doc_version_id = ?`);
|
|
134
|
+
for (const row of updates) update.run(row.contentHash, row.normalizedHash, row.contentKind, row.hashVersion, row.docVersionId);
|
|
135
|
+
await writeRootManifest(root, db);
|
|
136
|
+
db.exec("COMMIT;");
|
|
137
|
+
} catch (error) {
|
|
138
|
+
try {
|
|
139
|
+
db.exec("ROLLBACK;");
|
|
140
|
+
} catch {
|
|
141
|
+
}
|
|
142
|
+
throw errorWithCode("rehash_failed", `Backfill transaction failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
applied: apply,
|
|
147
|
+
updated: updates.length,
|
|
148
|
+
unchanged,
|
|
149
|
+
skipped,
|
|
150
|
+
...!apply ? { backupReminder: reminder() } : {}
|
|
151
|
+
};
|
|
152
|
+
} finally {
|
|
153
|
+
db.close();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function reminder() {
|
|
157
|
+
return "Before applying, back up this namespace with: rag backup --to <backup-dir>.";
|
|
158
|
+
}
|
|
159
|
+
export {
|
|
160
|
+
computeContentHashFacts,
|
|
161
|
+
rehashNamespace
|
|
162
|
+
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const SCHEMA_VERSION =
|
|
1
|
+
const SCHEMA_VERSION = 2;
|
|
2
2
|
const MIGRATIONS = [
|
|
3
3
|
{
|
|
4
4
|
version: 1,
|
|
@@ -170,6 +170,16 @@ BEFORE DELETE ON doc_version
|
|
|
170
170
|
BEGIN
|
|
171
171
|
SELECT RAISE(ABORT, 'doc_version history is immutable');
|
|
172
172
|
END;`
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
version: 2,
|
|
176
|
+
sql: `CREATE TABLE schema_migration (
|
|
177
|
+
version INTEGER PRIMARY KEY,
|
|
178
|
+
applied_at TEXT NOT NULL
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
ALTER TABLE doc_version ADD COLUMN content_kind TEXT;
|
|
182
|
+
ALTER TABLE doc_version ADD COLUMN hash_version INTEGER NOT NULL DEFAULT 1;`
|
|
173
183
|
}
|
|
174
184
|
];
|
|
175
185
|
export {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-rag-mcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Deterministic, namespace-isolated document retrieval with a CLI and MCP server.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -27,13 +27,16 @@
|
|
|
27
27
|
"files": [
|
|
28
28
|
"bin",
|
|
29
29
|
"dist",
|
|
30
|
-
"
|
|
30
|
+
"skills",
|
|
31
31
|
"README.md",
|
|
32
32
|
"LICENSE"
|
|
33
33
|
],
|
|
34
34
|
"pi": {
|
|
35
35
|
"extensions": [
|
|
36
|
-
"./extensions/agent-rag.
|
|
36
|
+
"./dist/extensions/agent-rag.mjs"
|
|
37
|
+
],
|
|
38
|
+
"skills": [
|
|
39
|
+
"./skills"
|
|
37
40
|
]
|
|
38
41
|
},
|
|
39
42
|
"engines": {
|
|
@@ -58,6 +61,7 @@
|
|
|
58
61
|
"mammoth": "1.13.0",
|
|
59
62
|
"pdfjs-dist": "6.4.299",
|
|
60
63
|
"tesseract.js": "7.0.0",
|
|
64
|
+
"yaml": "^2.9.1",
|
|
61
65
|
"zod": "4.6.7"
|
|
62
66
|
}
|
|
63
67
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: agent-rag
|
|
3
|
+
description: "Trigger: search or read the indexed knowledge base with explicit client/project scope and verifiable citations."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
## Choose a tool
|
|
7
|
+
|
|
8
|
+
- Use `rag_find` to search a client/project namespace and get a bounded catalog of matching citations.
|
|
9
|
+
- Use `rag_read` only when a citation needs more surrounding text; pass its `docVersionId` and `chunkOrdinal` from `rag_find`.
|
|
10
|
+
- The orchestrator only receives `rag_find`; delegate exploration when a citation needs reading.
|
|
11
|
+
|
|
12
|
+
## Evidence and scope
|
|
13
|
+
|
|
14
|
+
- The `namespace` is always explicit: pass an object with non-empty `client` and `project`; never infer it from cwd or discover another namespace.
|
|
15
|
+
- Cite source wording with a verbatim quote. Citations must be quoted, never paraphrased as source text.
|
|
16
|
+
- Keep the citation's document/version and section context with the quote; say when a search returns no matches.
|
|
17
|
+
- Treat `rag_find` excerpts as bounded evidence, not the complete source. Use `rag_read` for a bounded passage only when needed.
|
|
18
|
+
|
|
19
|
+
## Config and CLI
|
|
20
|
+
|
|
21
|
+
- CLI namespace/root options are `--client`, `--project`, `--root`; environment alternatives are `RAG_CLIENT`, `RAG_PROJECT`, `RAG_ROOT`.
|
|
22
|
+
- The CLI home is `RAG_HOME ?? os.homedir()`. The MCP server and Pi extension pass cwd as `homeDir`.
|
|
23
|
+
- The only config keys read in `src/config/discovery.mts` are `client`, `project`, `root`, and `failClosed`; unknown keys are silently ignored.
|
|
24
|
+
- Config candidates, first existing only: `<cwd>/rag.config.json`, `<dirname(cwd)>/rag.config.json` (one level up only), then `<homeDir>/.rag/config.json`.
|
|
25
|
+
- Only JSON syntax errors (`config_invalid`) or config-file I/O errors (`config_unreadable`) raise those config errors.
|
|
26
|
+
- Each field resolves independently to its first available value: client `--client` > `RAG_CLIENT` > file > null; project follows the same chain.
|
|
27
|
+
- Root precedence is `--root` > `RAG_ROOT` > file > `<homeDir>/.rag`; fields can resolve from different sources.
|
|
28
|
+
- Discovery reports its source as `args`, `env`, `cwd`, `parent`, `home`, or `unscoped`.
|
|
29
|
+
- `client` must match `/^[A-Za-z0-9][A-Za-z0-9._-]*$/`; `.` and `..` are rejected with `invalid_client_name`. Its DB is `<root>/clients/<client>/rag.db`.
|
|
30
|
+
- `project` is logical only: its slug is stored in `project.slug` rows in that client DB; it creates no directory and has no name validation.
|
|
31
|
+
- `root` defaults to `<homeDir>/.rag`; the CLI resolves a relative root against cwd for each command.
|
|
32
|
+
- Root layout: `clients/<client>/rag.db`, `objects/<aa>/<bb>/<sha256>`, and `manifest.json`.
|
|
33
|
+
- `failClosed` defaults to true and is file-only (no flag or environment variable). For writes without a namespace it throws `namespace_required`; false uses reserved `default/default`. Reads are unaffected.
|
|
34
|
+
- `rag init` does not write `rag.config.json`; it creates only the client DB, object store, and root manifest.
|
|
35
|
+
- MCP tools require `namespace: {client, project}` on every call. The server resolves only `root` from discovery; config, `RAG_CLIENT`, and `RAG_PROJECT` never provide tool identity.
|
package/extensions/agent-rag.ts
DELETED
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
import { Type } from "typebox";
|
|
2
|
-
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
3
|
-
import { discoverConfig } from "../src/config/discovery.mts";
|
|
4
|
-
import { findInNamespace } from "../src/retrieve/find.mts";
|
|
5
|
-
import { readFromCitation } from "../src/retrieve/read.mts";
|
|
6
|
-
import {
|
|
7
|
-
RAG_FIND_TOOL,
|
|
8
|
-
RAG_READ_TOOL,
|
|
9
|
-
namespaceFromToolInput,
|
|
10
|
-
parseCapabilities,
|
|
11
|
-
type RagFindInput,
|
|
12
|
-
type RagReadInput,
|
|
13
|
-
type RetrievalNamespace,
|
|
14
|
-
type ToolFailure,
|
|
15
|
-
validateRagFindInput,
|
|
16
|
-
validateRagReadInput,
|
|
17
|
-
} from "../src/tools/schema.mts";
|
|
18
|
-
|
|
19
|
-
function result(envelope: Record<string, unknown>, isError = false) {
|
|
20
|
-
return {
|
|
21
|
-
content: [{ type: "text" as const, text: JSON.stringify(envelope) }],
|
|
22
|
-
details: envelope,
|
|
23
|
-
...(isError ? { isError: true } : {}),
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
function failure(namespace: RetrievalNamespace | null, error: ToolFailure) {
|
|
28
|
-
return result({ namespace, error }, true);
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
function cancelled(namespace: RetrievalNamespace) {
|
|
32
|
-
return failure(namespace, { code: "cancelled", message: "Tool call was cancelled." });
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function retrievalFailure(params: unknown, error: unknown) {
|
|
36
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
37
|
-
return failure(namespaceFromToolInput(params), { code: "retrieval_failed", message });
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
export default function (pi: ExtensionAPI): void {
|
|
41
|
-
const capabilities = parseCapabilities(process.env.RAG_CAPABILITIES);
|
|
42
|
-
if (capabilities.has("find")) {
|
|
43
|
-
pi.registerTool({
|
|
44
|
-
name: RAG_FIND_TOOL.name,
|
|
45
|
-
label: "RAG Find",
|
|
46
|
-
description: RAG_FIND_TOOL.description,
|
|
47
|
-
parameters: Type.Unsafe<RagFindInput>(RAG_FIND_TOOL.inputSchema),
|
|
48
|
-
async execute(_id, params, signal, _onUpdate, ctx) {
|
|
49
|
-
try {
|
|
50
|
-
const check = validateRagFindInput(params);
|
|
51
|
-
if (!check.ok) return failure(check.namespace, check.error);
|
|
52
|
-
const { namespace, query, limit, excerptChars } = check.input;
|
|
53
|
-
if (signal?.aborted) return cancelled(namespace);
|
|
54
|
-
const root = discoverConfig({ cwd: ctx.cwd, homeDir: ctx.cwd, argv: [], env: process.env }).root;
|
|
55
|
-
const found = findInNamespace({ root, namespace, query, excerptLength: excerptChars });
|
|
56
|
-
if (signal?.aborted) return cancelled(namespace);
|
|
57
|
-
if ("error" in found) return failure(namespace, found.error);
|
|
58
|
-
return result({ namespace, result: { ...found.result, citations: found.result.citations.slice(0, limit) } });
|
|
59
|
-
} catch (error) {
|
|
60
|
-
return retrievalFailure(params, error);
|
|
61
|
-
}
|
|
62
|
-
},
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
if (capabilities.has("read")) {
|
|
67
|
-
pi.registerTool({
|
|
68
|
-
name: RAG_READ_TOOL.name,
|
|
69
|
-
label: "RAG Read",
|
|
70
|
-
description: RAG_READ_TOOL.description,
|
|
71
|
-
parameters: Type.Unsafe<RagReadInput>(RAG_READ_TOOL.inputSchema),
|
|
72
|
-
async execute(_id, params, signal, _onUpdate, ctx) {
|
|
73
|
-
try {
|
|
74
|
-
const check = validateRagReadInput(params);
|
|
75
|
-
if (!check.ok) return failure(check.namespace, check.error);
|
|
76
|
-
const { namespace, docVersionId, chunkOrdinal, maxBytes, maxLines } = check.input;
|
|
77
|
-
if (signal?.aborted) return cancelled(namespace);
|
|
78
|
-
const root = discoverConfig({ cwd: ctx.cwd, homeDir: ctx.cwd, argv: [], env: process.env }).root;
|
|
79
|
-
const read = await readFromCitation({ root, namespace, docVersionId, chunkOrdinal, maxBytes, maxLines });
|
|
80
|
-
if (signal?.aborted) return cancelled(namespace);
|
|
81
|
-
if ("error" in read) return failure(namespace, read.error);
|
|
82
|
-
return result({ namespace, result: read.result });
|
|
83
|
-
} catch (error) {
|
|
84
|
-
return retrievalFailure(params, error);
|
|
85
|
-
}
|
|
86
|
-
},
|
|
87
|
-
});
|
|
88
|
-
}
|
|
89
|
-
}
|
package/extensions/pi-host.d.ts
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
declare module "typebox" {
|
|
2
|
-
interface Schema {}
|
|
3
|
-
interface UnsafeSchema<Type> extends Schema {
|
|
4
|
-
"~unsafe": Type;
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
export const Type: {
|
|
8
|
-
Unsafe<Type>(schema: Schema): UnsafeSchema<Type>;
|
|
9
|
-
};
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
declare module "@earendil-works/pi-coding-agent" {
|
|
13
|
-
interface UnsafeParameters<Type> {
|
|
14
|
-
"~unsafe": Type;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
interface TextContent {
|
|
18
|
-
type: "text";
|
|
19
|
-
text: string;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
interface ImageContent {
|
|
23
|
-
type: "image";
|
|
24
|
-
data: string;
|
|
25
|
-
mimeType: string;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
interface ToolResult {
|
|
29
|
-
content: (TextContent | ImageContent)[];
|
|
30
|
-
details: unknown;
|
|
31
|
-
isError?: boolean;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
interface ToolContext {
|
|
35
|
-
cwd: string;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
interface ToolDefinition<Params> {
|
|
39
|
-
name: string;
|
|
40
|
-
label: string;
|
|
41
|
-
description: string;
|
|
42
|
-
parameters: UnsafeParameters<Params>;
|
|
43
|
-
execute(
|
|
44
|
-
toolCallId: string,
|
|
45
|
-
params: Params,
|
|
46
|
-
signal: AbortSignal | undefined,
|
|
47
|
-
onUpdate: ((partialResult: ToolResult) => void) | undefined,
|
|
48
|
-
ctx: ToolContext,
|
|
49
|
-
): Promise<ToolResult>;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export interface ExtensionAPI {
|
|
53
|
-
registerTool<Params>(tool: ToolDefinition<Params>): void;
|
|
54
|
-
}
|
|
55
|
-
}
|