agent-rag-mcp 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/extensions/agent-rag.mjs +9 -8
- package/dist/src/adapters/docx.mjs +99 -8
- package/dist/src/cli/index.mjs +4 -1
- package/dist/src/limits.mjs +3 -0
- package/dist/src/retrieve/read.mjs +38 -31
- package/dist/src/server.mjs +2 -2
- package/dist/src/tools/schema.mjs +3 -1
- package/package.json +1 -1
- package/skills/agent-rag/SKILL.md +1 -0
package/README.md
CHANGED
|
@@ -28,7 +28,7 @@ The `rag` commands available today are:
|
|
|
28
28
|
- `rag review list` — Lists version pairs awaiting a human decision.
|
|
29
29
|
- `rag review link <from> <to> --type <conflicts_with|same_entity>` — Records a human link between versions with a note.
|
|
30
30
|
- `rag status` — Shows counts for the namespace.
|
|
31
|
-
- `rag doctor` — Verifies registered-object integrity and reports missing objects, orphan blobs,
|
|
31
|
+
- `rag doctor` — Verifies registered-object integrity and reports missing objects, orphan blobs, temporary files, and stored chunks over the DOCX section cap.
|
|
32
32
|
- `rag backup --to <dir>` — Backs up the manifest and object store.
|
|
33
33
|
- `rag restore --from <dir>` — Restores a backup under this root.
|
|
34
34
|
- `rag reindex` — Rebuilds the database from the manifest and objects.
|
|
@@ -89,7 +89,7 @@ The extension adds a `rag` section to the agent system prompt describing the nam
|
|
|
89
89
|
|
|
90
90
|
The namespace travels in every `rag_find` or `rag_read` call; it does not come from the environment. For example, include `"namespace": {"client":"acme","project":"research"}` in each tool call's arguments. Do not configure `RAG_CLIENT` or `RAG_PROJECT` as a substitute for the per-call namespace.
|
|
91
91
|
|
|
92
|
-
`rag_find` returns `total`, the number of matching chunks before the `limit` slice; `citations.length` is how many citations were actually returned, so comparing them shows how many matches were cut by the limit. Its `excerptChars` input defaults to 800. Every successful `rag_read` result includes `totalChunks` and `nextChunkOrdinal`; the latter is the next zero-based chunk ordinal, or `null` when there is no following chunk. `truncated` means this chunk was not delivered whole, so re-read the same ordinal with higher limits; it does not indicate whether another chunk exists.
|
|
92
|
+
`rag_find` returns `total`, the number of matching chunks before the `limit` slice; `citations.length` is how many citations were actually returned, so comparing them shows how many matches were cut by the limit. Its `excerptChars` input defaults to 800. Every successful `rag_read` result includes `totalChunks` and `nextChunkOrdinal`; the latter is the next zero-based chunk ordinal, or `null` when there is no following chunk. `truncated` means this chunk was not delivered whole, so re-read the same ordinal with higher limits; it does not indicate whether another chunk exists. `rag_read` accepts `startLine`, the zero-based first line to return (default `0`, maximum `10,000`), and returns `returnedBytes` and `returnedLines` for the page, `chunkBytes` for the full chunk, and `truncatedBy` (`"bytes"`, `"lines"`, or `null`) alongside `truncated`. For a chunk that exceeds the line limit, ensure each page fits within `maxBytes`, then repeat calls advancing `startLine` by `returnedLines`; this covers every line without overlap or gaps, and `truncatedBy: null` marks the last page.
|
|
93
93
|
|
|
94
94
|
## Requirements
|
|
95
95
|
|
|
@@ -103,6 +103,8 @@ The namespace travels in every `rag_find` or `rag_read` call; it does not come f
|
|
|
103
103
|
|
|
104
104
|
Version 0.1.0 has no embeddings or Tier 2; scanned PDFs go through OCR. Document parsing runs in a child process with a 512 MB V8 heap cap.
|
|
105
105
|
|
|
106
|
+
At ingest, DOCX files split by heading; heading groups over 8,192 characters are subdivided at whitespace, keeping the heading at the start of the first section. Files without headings use greedy paragraph chunks capped at 8,192 characters, and oversized paragraphs split at whitespace. Already-ingested DOCX documents keep their existing single chunk until re-ingested; `rag reindex` does not re-extract source documents.
|
|
107
|
+
|
|
106
108
|
## Design documents
|
|
107
109
|
|
|
108
110
|
- [Architecture](https://github.com/Lsarmientov/agent-rag/blob/main/docs/design/0001-arquitectura.md)
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
`),n=[],r=null,o=/^ {0,3}(#{1,6})(?:[ \t]+|$)(.*)$/;if(!t.some(s=>o.test(s.replace(/\r$/,""))))return e.length===0?[]:[{ordinal:0,heading:null,level:null,text:e}];let i=()=>{r&&n.push({ordinal:n.length,heading:r.heading,level:r.level,text:r.lines.join(`
|
|
3
|
-
`).trim()})};for(let s of t){let a=o.exec(s.replace(/\r$/,""));if(!a){r?r.lines.push(s):s.length>0&&(r={heading:null,level:null,lines:[s]});continue}i(),r={heading:a[2].replace(/[ \t]+#+[ \t]*$/,"").trim(),level:a[1].length,lines:[]}}return i(),n}async function _n(e){let t=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(e),n=t.startsWith("\uFEFF")?t.slice(1):t,r=hn(n),o=r.some(s=>s.heading!==null),i=r.find(s=>s.heading!==null&&rt(s.heading)!==null)?.heading??null;return{sections:r,versionLabel:i,mediaType:o?"text/markdown":n.length>0?"text/plain":null}}var gn,Ie,st=w(()=>{"use strict";ot();gn=new Set(["text/plain","text/markdown"]);Ie={name:"plain-text",version:"1.0.0",supports:En,extract:_n}});function xn(e,t){let n=Array.from(e,i=>i.codePointAt(0)),r=Array.from(t,i=>i.codePointAt(0)),o=Math.min(n.length,r.length);for(let i=0;i<o;i+=1){let s=n[i],a=r[i];if(s!==a)return s-a}return n.length-r.length}function Pe(e,t){if(e===null)return"null";if(typeof e=="string")return JSON.stringify(e);if(typeof e=="boolean")return e?"true":"false";if(typeof e=="number"){if(!Number.isFinite(e))throw new D;return JSON.stringify(e)}if(typeof e=="bigint")return e<=BigInt(Number.MAX_SAFE_INTEGER)&&e>=BigInt(Number.MIN_SAFE_INTEGER)?JSON.stringify(Number(e)):JSON.stringify(e.toString());if(Array.isArray(e)){if(t.has(e))throw new TypeError("Structured content must be acyclic.");t.add(e);try{return`[${e.map(n=>n===void 0?"null":Pe(n,t)).join(",")}]`}finally{t.delete(e)}}if(typeof e=="object"){let n=e;if(t.has(n))throw new TypeError("Structured content must be acyclic.");t.add(n);try{return`{${Object.keys(n).filter(o=>n[o]!==void 0).sort(xn).map(o=>`${JSON.stringify(o)}:${Pe(n[o],t)}`).join(",")}}`}finally{t.delete(n)}}return"null"}function oe(e){return Pe(e,new Set)}var D,at=w(()=>{"use strict";D=class extends Error{code="non_finite_number";constructor(){super("Canonical structured content contains a non-finite number."),this.name="NonFiniteNumberCanonicalizationError"}}});import{parseDocument as Tn}from"yaml";function ct(e,t){if(typeof e=="string")return e;if(typeof e!="object"||e===null)return null;let n=e;for(let r of t)if(typeof n[r]=="string")return n[r];return null}function yn(e,t){if(typeof e!="string"){let o=ct(e,["mediaType","mimeType","contentType"]);if(o!==null)return t.mediaTypes.includes(o.split(";")[0].trim().toLowerCase())}let n=typeof e=="string"?e:ct(e,["filename","name","path","extension"]);if(n===null)return!1;let r=n.toLowerCase();return t.extensions.some(o=>r.endsWith(o))}function dt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function ut(e){return JSON.parse(e,(t,n,r)=>{let o=r?.source;return typeof n=="number"&&!Number.isSafeInteger(n)&&o!==void 0&&/^-?(?:0|[1-9][0-9]*)$/.test(o)?BigInt(o):n})}function Xe(e){try{return{canonicalContent:oe(e)}}catch(t){if(t instanceof D)return{canonicalizationError:t.code};throw t}}function bn(e){let t;try{t=ut(e)}catch{throw new U("extraction_invalid_json")}return{sections:typeof t=="object"&&t!==null&&!Array.isArray(t)?Object.entries(t).map(([r,o],i)=>({ordinal:i,heading:r,level:1,text:dt(o)})):[{ordinal:0,heading:null,level:null,text:dt(t)}],...Xe(t)}}function Sn(e){let t=[],n=[];for(let r of e.split(/\r?\n/)){let o=r.trim();if(o.length!==0){try{n.push(ut(o))}catch{throw new U("extraction_invalid_json")}t.push({ordinal:t.length,heading:null,level:null,text:o})}}return{sections:t,...Xe(n)}}function j(e,t=new Set){if(typeof e!="object"||e===null)return e;if(t.has(e))throw new U("extraction_invalid_yaml");t.add(e);try{if(e instanceof Map){let r=Object.create(null);for(let[o,i]of e){let s=typeof o=="string"?o:oe(j(o,t));r[s]=j(i,t)}return r}if(Array.isArray(e))return e.map(r=>j(r,t));let n=Object.create(null);for(let[r,o]of Object.entries(e))n[r]=j(o,t);return n}finally{t.delete(e)}}function lt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function wn(e){let t=e.contents,n=new Map;for(let r of t?.items??[]){let o=r.key?.range?.[0];if(o===void 0)continue;let i=r.key&&Object.hasOwn(r.key,"value")?r.key.value:r.key?.toJSON?.(),s=j(i),a=typeof s=="string"?s:oe(s);n.set(a,o)}return n}function Rn(e){let t,n,r;try{let i=Tn(e,{schema:"core",uniqueKeys:!1,intAsBigInt:!0});if(i.errors.length>0)throw new Error("invalid YAML document");t=j(i.toJS({mapAsMap:!0})),n=wn(i)}catch(i){if(!(i instanceof D))throw new U("extraction_invalid_yaml");r=i.code,t=null,n=new Map}if(r!==void 0)return{sections:[{ordinal:0,heading:null,level:null,text:e.trim()}],canonicalizationError:r};let o;if(typeof t=="object"&&t!==null&&!Array.isArray(t)){let i=Object.entries(t).map(([s,a])=>({heading:s,value:a,start:n.get(s)})).filter(s=>s.start!==void 0).sort((s,a)=>s.start-a.start);o=i.map(({heading:s,start:a},c)=>({ordinal:c,heading:s,level:1,text:e.slice(a,i[c+1]?.start??e.length).trim()})),o.length===0&&(o=[{ordinal:0,heading:null,level:null,text:lt(t)}])}else o=[{ordinal:0,heading:null,level:null,text:lt(t)}];return{sections:o,...Xe(t)}}function Ce(e){return{name:e.name,version:"1.0.0",mediaTypes:e.mediaTypes,supports(t){return yn(t,e)},async extract(t){let n=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(t),r=n.startsWith("\uFEFF")?n.slice(1):n;if(r.length===0)return{sections:[],versionLabel:null,mediaType:null};let o;switch(e.kind){case"json":o=bn(r);break;case"jsonl":o=Sn(r);break;case"yaml":o=Rn(r);break}return{sections:o.sections,versionLabel:null,mediaType:e.mediaTypes[0]??null,...o.canonicalContent===void 0?{}:{canonicalContent:o.canonicalContent},...o.canonicalizationError===void 0?{}:{canonicalizationError:o.canonicalizationError}}}}}var U,ft,mt,ve,pt=w(()=>{"use strict";at();U=class extends Error{code;constructor(t){super(t==="extraction_invalid_json"?"Structured JSON is invalid.":"Structured YAML is invalid."),this.name="StructuredTextExtractionError",this.code=t}};ft=Ce({kind:"json",name:"json",extensions:[".json"],mediaTypes:["application/json"]}),mt=Ce({kind:"jsonl",name:"jsonl",extensions:[".jsonl"],mediaTypes:["application/x-ndjson"]}),ve=Ce({kind:"yaml",name:"yaml",extensions:[".yaml",".yml"],mediaTypes:["application/yaml"]})});import{PassThrough as Nn}from"node:stream";import{StringDecoder as An}from"node:string_decoder";import Ln from"jszip";function On(e){if(typeof e=="string")return/\.pptx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===ke;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.pptx$/i.test(r)}function In(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10);return String.fromCodePoint(r)}}})}function gt(e){return Number(e.match(/\/slide(\d+)\.xml$/)?.[1]??Number.MAX_SAFE_INTEGER)}function T(e){throw new b("archive_too_large",e)}function Pn(e,t,n,r){t>n&&T(`PPTX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&T(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}function F(e,t,n){t+8>n&&T("PPTX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&T("PPTX ZIP64 metadata exceeds safe bounds."),Number(r)}function H(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Xn(e,t){let n=F(e,t+8,t+20),r=o=>{if(!H(e,o,101075792)||o+12>t)return!1;let i=F(e,o+4,t);return i>=44&&o+12+i===t};if(r(n))return n;for(let o=t-56;o>=0;o-=1)if(r(o))return o;T("PPTX ZIP64 end record cannot be located safely.")}function Cn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),l=e.getUint16(s+2,!0),d=s+4,u=d+l;if((u>a||u>n)&&T("PPTX central-directory extra field is truncated."),c===1)return F(e,d,u);s=u}T("PPTX ZIP64 entry size is missing from its central-directory extra field.")}function vn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let E=e.byteLength-22;E>=n;E-=1)if(H(t,E,101010256)&&E+22+t.getUint16(E+20,!0)===e.byteLength){r=E;break}if(r<0)return new Map;let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=H(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&T("PPTX ZIP64 end record is missing.");let d=o,u=i,g=s,m=r;if(c){let E=Xn(t,a),R=a;d=F(t,E+32,R),u=F(t,E+40,R),g=F(t,E+48,R),m=E}d>L&&T(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>m||g>m||u+g>m)&&T("PPTX central-directory bounds cannot be checked safely.");let h=m-u-g,f=g+h,p=f+u,x=0,A=0,z=new Map;for(;f+4<=p&&H(t,f,33639248);){f+46>p&&T("PPTX central-directory bounds cannot be checked safely.");let E=t.getUint16(f+28,!0),R=t.getUint16(f+30,!0),he=t.getUint16(f+32,!0),G=f+46+E+R+he;G>p&&T("PPTX central-directory entry is truncated."),x+=1,x>L&&T(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let X=Cn(t,f,p,E,R),_e=new TextDecoder().decode(e.subarray(f+46,f+46+E));z.set(_e,X),X>O-A&&T(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=X,f=G}if(f<p&&H(t,f,84233040)&&f+6<=p){let E=t.getUint16(f+4,!0);f+6+E===p&&(f=p)}return(f!==p||x!==d)&&T("PPTX central-directory count or bounds cannot be checked safely."),z}async function kn(e,t,n){let r=[],o=new An("utf8"),i=new Nn,s=e.nodeStream("nodebuffer");s.on("error",c=>i.destroy(c)),s.pipe(i);let a=0;try{for await(let c of i){let l=Buffer.isBuffer(c)?c:Buffer.from(c);a+=l.byteLength,Pn(e.name,a,n,t+a),r.push(o.write(l))}}catch(c){throw c instanceof Error&&c.message.includes("uncompressed data size mismatch")&&T(`PPTX entry ${e.name} decompressed size differs from its declared size (${n} bytes).`),c}return r.push(o.end()),{xml:r.join(""),bytesRead:a}}async function Mn(e){let t=vn(e),r=(await Ln.loadAsync(e)).file(/^ppt\/slides\/slide\d+\.xml$/).sort((s,a)=>gt(s.name)-gt(a.name)),o=[],i=0;for(let s of r){let a=t.get(s.name);a===void 0&&T(`PPTX central-directory size for ${s.name} cannot be checked safely.`);let c=await kn(s,i,a);i+=c.bytesRead;let l=[...c.xml.matchAll(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/g)].map(d=>In(d[1]??"").trim()).filter(d=>d.length>0).join(" ");l.length>0&&o.push({ordinal:o.length,heading:null,level:null,text:l})}return{sections:o,versionLabel:null,mediaType:ke}}var ke,L,O,Et,Me=w(()=>{"use strict";M();ke="application/vnd.openxmlformats-officedocument.presentationml.presentation",L=5e3,O=128*1024*1024;Et={name:"pptx",version:"3.10.2",mediaTypes:[ke],supports:On,extract:Mn}});import{posix as ht}from"node:path";import{PassThrough as Dn}from"node:stream";import{StringDecoder as jn}from"node:string_decoder";import Un from"jszip";function $n(e){if(typeof e=="string")return/\.xlsx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===De;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.xlsx$/i.test(r)}function y(e){throw new b("sheet_too_large",e)}function _(e){throw new b("archive_too_large",e)}function bt(e){let t=/^\$?([A-Z]{1,3})\$?(\d+)$/i.exec(e);if(!t?.[1]||!t[2])return null;let n=0;for(let o of t[1].toUpperCase())n=n*26+o.charCodeAt(0)-64;let r=Number(t[2]);return Number.isSafeInteger(r)&&r>0&&n<=Fn?{row:r,column:n}:null}function Tt(e,t){(!Number.isSafeInteger(e)||e<1||!Number.isSafeInteger(t)||t<1)&&y("Worksheet contains an invalid row or column reference."),(e>Y||e*t>ae)&&y(`Worksheet exceeds MAX_SHEET_ROWS (${Y}) or MAX_SHEET_CELLS (${ae}).`)}function S(e,t){return e.match(new RegExp(`\\b${t}\\s*=\\s*[\\"']([^\\"']*)[\\"']`,"i"))?.[1]}function $(e,t,n){t+8>n&&_("XLSX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&_("XLSX ZIP64 metadata exceeds safe bounds."),Number(r)}function V(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Bn(e,t){let n=$(e,t+8,t+20),r=i=>{if(!V(e,i,101075792)||i+12>t)return!1;let s=$(e,i+4,t);return s>=44&&i+12+s===t};if(r(n))return n;let o=0;for(let i=t-56;i>=0;i-=1)if(o>=xt&&_(`XLSX ZIP64 recovery scan exceeds MAX_ZIP64_RECOVERY_SCAN_ITERATIONS (${xt} iterations).`),o+=1,r(i))return i;_("XLSX ZIP64 end record cannot be located safely.")}function zn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),l=e.getUint16(s+2,!0),d=s+4,u=d+l;if((u>a||u>n)&&_("XLSX central-directory extra field is truncated."),c===1)return $(e,d,u);s=u}_("XLSX ZIP64 entry size is missing from its central-directory extra field.")}function Gn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let E=e.byteLength-22;E>=n;E-=1)if(V(t,E,101010256)&&E+22+t.getUint16(E+20,!0)===e.byteLength){r=E;break}r<0&&_("XLSX end-of-central-directory record is missing.");let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=V(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&_("XLSX ZIP64 end record is missing.");let d=o,u=i,g=s,m=r;if(c){let E=Bn(t,a),R=a;d=$(t,E+32,R),u=$(t,E+40,R),g=$(t,E+48,R),m=E}d>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>m||g>m||u+g>m)&&_("XLSX central-directory bounds cannot be checked safely.");let h=m-u-g,f=g+h,p=f+u,x=0,A=0,z=new Map;for(;f+4<=p&&V(t,f,33639248);){f+46>p&&_("XLSX central-directory bounds cannot be checked safely.");let E=t.getUint16(f+28,!0),R=t.getUint16(f+30,!0),he=t.getUint16(f+32,!0),G=f+46+E+R+he;G>p&&_("XLSX central-directory entry is truncated."),x+=1,x>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let X=zn(t,f,p,E,R),_e=new TextDecoder().decode(e.subarray(f+46,f+46+E));z.set(_e,X),X>O-A&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=X,f=G}if(f<p&&V(t,f,84233040)&&f+6<=p){let E=t.getUint16(f+4,!0);f+6+E===p&&(f=p)}return(f!==p||x!==d)&&_("XLSX central-directory count or bounds cannot be checked safely."),z}function Hn(e,t,n,r){t>n&&_(`XLSX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}async function St(e,t,n,r){let o=[],i=new jn("utf8"),s=new Dn,a=e.nodeStream("nodebuffer");a.on("error",d=>s.destroy(d)),a.pipe(s);let c=0;try{for await(let d of s){let u=Buffer.isBuffer(d)?d:Buffer.from(d);c+=u.byteLength,Hn(e.name,c,t,n.bytes+c);let g=i.write(u);o.push(g),r?.(g,u.byteLength)}}catch(d){throw d instanceof Error&&d.message.includes("uncompressed data size mismatch")&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),d}let l=i.end();return o.push(l),r?.(l,0),c!==t&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),n.bytes+=c,o.join("")}function Vn(e,t,n,r){n.bytes+=t,n.bytes>_t&&y(`Worksheet XML scan exceeds MAX_SHEET_SCAN_BYTES (${_t} bytes).`);let o=r.pending+e,i=0;for(r.pending="";;){let s=o.indexOf("<",i);if(s<0)break;let a=o.indexOf(">",s+1);if(a<0){r.pending=o.slice(s),Buffer.byteLength(r.pending)>ie&&y(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${ie} bytes).`);break}let c=o.slice(s,a+1);if(i=a+1,Buffer.byteLength(c)>ie&&y(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${ie} bytes).`),/^<(?:[\w.-]+:)?row\b/i.test(c)){r.rowCount+=1;let l=S(c,"r");l!==void 0&&!/^\d+$/.test(l)&&y("Worksheet contains an invalid row reference.");let d=Number(l??r.rowCount);(!Number.isSafeInteger(d)||d<1)&&y("Worksheet contains an invalid row reference."),(d>Y||r.rowCount>Y)&&y(`Worksheet exceeds MAX_SHEET_ROWS (${Y}).`),r.maxRow=Math.max(r.maxRow,d),Tt(r.maxRow,r.maxColumn)}else if(/^<(?:[\w.-]+:)?c\b/i.test(c)){r.cellCount+=1,r.cellCount>ae&&y(`Worksheet exceeds MAX_SHEET_CELLS (${ae}).`);let l=S(c,"r"),d=l===void 0?null:bt(l);l!==void 0&&!d&&y("Worksheet contains an invalid cell reference."),d&&(r.maxRow=Math.max(r.maxRow,d.row),r.maxColumn=Math.max(r.maxColumn,d.column),Tt(r.maxRow,r.maxColumn))}}}function W(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10),o=r===9||r===10||r===13||r>=32&&r<=55295||r>=57344&&r<=65533||r>=65536&&r<=1114111;return(!Number.isInteger(r)||!o)&&y("XLSX contains an invalid XML character reference."),String.fromCodePoint(r)}}})}function wt(e){return[...e.replace(/<(?:[\w.-]+:)?rPh\b[\s\S]*?<\/(?:[\w.-]+:)?rPh\s*>/gi,"").matchAll(/<(?:[\w.-]+:)?t\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?t\s*>/gi)].map(n=>W(n[1]??"")).join("")}function Yn(e){return[...e.matchAll(/<(?:[\w.-]+:)?si\b[^>]*\/>|<(?:[\w.-]+:)?si\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?si\s*>/gi)].map(t=>t[0].endsWith("/>")?"":wt(t[1]??""))}function Jn(e){let t;try{t=decodeURIComponent(W(e))}catch(o){if(o instanceof b)throw o;return null}let n=t.split(/[?#]/,1)[0]??"",r=ht.normalize(n.startsWith("/")?n.slice(1):ht.join("xl",n));return r.startsWith("xl/")?r:null}function Wn(e){let t=new Map;for(let n of e.matchAll(/<(?:[\w.-]+:)?Relationship\b[^>]*\/?>/gi)){let r=n[0];if(!/\/worksheet$/i.test(S(r,"Type")??"")||/^external$/i.test(S(r,"TargetMode")??""))continue;let o=S(r,"Id"),i=S(r,"Target"),s=i?Jn(i):null;o&&s&&t.set(o,s)}return t}function Kn(e,t){let n=[];for(let r of e.matchAll(/<(?:[\w.-]+:)?sheet\b[^>]*\/?>/gi)){let o=r[0],i=S(o,"name"),s=S(o,"r:id")??S(o,"id"),a=s?t.get(s):void 0;(i===void 0||a===void 0)&&_("XLSX workbook contains an unresolved worksheet relationship."),n.push({name:W(i),part:a})}return n}function Zn(e){let t=e.replace(/"(?:""|[^"])*"/g,"").replace(/\\./g,"").replace(/_.|\*./g,"").replace(/\[[^\]]*\]/g,""),n=/y/i.test(t)&&/m/i.test(t)&&/d/i.test(t);return{date:n,dateTime:n&&(/h|s/i.test(t)||/am\/pm|\[(?:h+|s+)\]/i.test(e))}}function qn(e){let t=new Map;for(let r of e.matchAll(/<(?:[\w.-]+:)?numFmt\b[^>]*\/?\s*>/gi)){let o=Number(S(r[0],"numFmtId")),i=S(r[0],"formatCode");Number.isSafeInteger(o)&&i!==void 0&&t.set(o,W(i))}return[...(e.match(/<(?:[\w.-]+:)?cellXfs\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?cellXfs\s*>/i)?.[1]??"").matchAll(/<(?:[\w.-]+:)?xf\b[^>]*\/?\s*>/gi)].map(r=>{let o=Number(S(r[0],"numFmtId")??"0");return[14,15,16,17].includes(o)?{date:!0,dateTime:!1}:o===22?{date:!0,dateTime:!0}:Zn(t.get(o)??"")})}function Qn(e,t){let n=Number(e);(e.trim()===""||!Number.isFinite(n))&&y("Worksheet contains an invalid date serial.");let r=Math.floor(n),o=n-r,i=r>=60?r-1:r,s=Date.UTC(1899,11,31)+i*864e5+Math.round(o*864e5),a=new Date(s);return Number.isFinite(a.getTime())||y("Worksheet date serial is outside the supported range."),t?a.toISOString():a.toISOString().slice(0,10)}function er(e,t,n){let r=e.indexOf(">"),o=r<0?e:e.slice(0,r+1),i=S(o,"t");if(i==="s"){let u=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1]?.trim();(u===void 0||!/^\d+$/.test(u))&&y("Worksheet contains an invalid shared-string index.");let g=Number(u);return(!Number.isSafeInteger(g)||t[g]===void 0)&&y("Worksheet shared-string index is outside the shared-strings table."),t[g]??""}if(i==="inlineStr"){let u=e.match(/<(?:[\w.-]+:)?is\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?is\s*>/i)?.[1]??"";return wt(u).trim()}let s=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1];if(s===void 0)return"";let a=W(s).trim(),c=S(o,"s"),l=c===void 0?0:Number(c);(!Number.isSafeInteger(l)||l<0||c!==void 0&&n[l]===void 0)&&y("Worksheet contains an invalid cell style reference.");let d=n[l];return d?.date?Qn(a,d.dateTime):a}function tr(e,t,n){let r=new Map,o=0;for(let s of e.matchAll(/<(?:[\w.-]+:)?row\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?row\s*>/gi)){o+=1;let a=s[0].slice(0,s[0].indexOf(">")+1),c=Number(S(a,"r")??o),l=1;for(let d of(s[1]??"").matchAll(/<(?:[\w.-]+:)?c\b[^>]*?(?:\/>|>([\s\S]*?)<\/(?:[\w.-]+:)?c\s*>)/gi)){let u=d[0],g=u.slice(0,u.indexOf(">")+1),m=bt(S(g,"r")??""),h=m?.column??l,f=m?.row??c;if(l=h+1,!Number.isSafeInteger(f)||f<1||!Number.isSafeInteger(h)||h<1)continue;let p=r.get(f)??new Map;p.set(h,er(u,t,n)),r.set(f,p)}}let i=[];for(let s of[...r].sort(([a],[c])=>a-c)){let a=s[1],c=0;for(let d of a.keys())c=Math.max(c,d);let l=Array.from({length:c},(d,u)=>a.get(u+1)??"");for(;l.length>0&&l[l.length-1]==="";)l.pop();l.some(d=>d.length>0)&&i.push(l.join(" "))}return i.join(`
|
|
4
|
-
`)}function J(e,t){let n=Object.values(e.files).filter(r=>!r.dir&&r.name.toLowerCase()===t.toLowerCase());return n.length>1&&_(`XLSX package contains ambiguous case variants of ${t}.`),n[0]}function nr(e){return/<(?:[\w.-]+:)?c\b[^>]*\bt\s*=\s*["']s["']/i.test(e)}async function se(e,t,n,r){let o=J(e,t);if(!o)throw new Error(`XLSX package is missing ${t}.`);let i=n.get(o.name);return i===void 0&&_(`XLSX central-directory size for ${o.name} cannot be checked safely.`),St(o,i,r)}async function yt(e,t,n,r){let o=J(e,t);if(!o)return null;let i=n.get(o.name);i===void 0&&_(`XLSX central-directory size for ${o.name} cannot be checked safely.`);let s={bytes:0},a={pending:"",rowCount:0,cellCount:0,maxRow:0,maxColumn:1};return St(o,i,r,(c,l)=>{Vn(c,l,s,a)})}async function rr(e){try{let t=Gn(e),n=await Un.loadAsync(e),r={bytes:0};if(!J(n,"xl/workbook.xml")){for(let[h,f]of Object.entries(n.files))!f.dir&&/^xl\/worksheets\/[^/]+\.xml$/i.test(h)&&await yt(n,h,t,r);y("XLSX package is missing xl/workbook.xml.")}let o=await se(n,"xl/workbook.xml",t,r),i=await se(n,"xl/_rels/workbook.xml.rels",t,r),s=Kn(o,Wn(i));s.length===0&&_("XLSX workbook has no resolvable worksheets.");let a=new Map;for(let h of s){let f=h.part.toLowerCase();if(a.has(f))continue;let p=await yt(n,h.part,t,r);p!==null&&a.set(f,p)}let c=J(n,"xl/sharedStrings.xml"),l=[...a.values()].some(nr);!c&&l&&_("XLSX worksheet references shared strings, but the shared-strings part is missing.");let d=c?Yn(await se(n,c.name,t,r)):[],u=J(n,"xl/styles.xml"),g=u?qn(await se(n,u.name,t,r)):[],m=[];for(let h of s){let f=a.get(h.part.toLowerCase());f===void 0&&_(`XLSX package is missing worksheet ${h.part}.`);let p=tr(f,d,g);p.trim().length!==0&&m.push({ordinal:m.length,heading:h.name,level:1,text:p})}return{sections:m,versionLabel:null,mediaType:De}}catch(t){if(t instanceof b)throw t;_("XLSX package could not be parsed safely.")}}var De,Y,ae,ie,_t,Fn,xt,Rt,Nt=w(()=>{"use strict";Me();M();De="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",Y=1e5,ae=1e6,ie=64*1024,_t=16*1024*1024,Fn=16384,xt=1e6;Rt={name:"xlsx",version:"4.4.0",mediaTypes:[De],supports:$n,extract:rr}});import*as Ue from"node:path";function At(e){let t=Ue.extname(e).toLowerCase(),n=or.get(t);if(!n||!n.supports({filename:e}))throw new je(e);return n}var je,or,Lt=w(()=>{"use strict";Ve();Le();nt();st();pt();Me();Nt();je=class extends Error{code="unsupported_media_type";extension;constructor(t){let n=Ue.extname(t).toLowerCase()||"(none)";super(`Unsupported media type for extension "${n}".`),this.name="UnsupportedMediaTypeError",this.extension=n}},or=new Map([[".pdf",tt],[".docx",He],[".png",ne],[".jpg",ne],[".jpeg",ne],[".xlsx",Rt],[".pptx",Et],[".txt",Ie],[".md",Ie],[".yaml",ve],[".yml",ve],[".json",ft],[".jsonl",mt]])});import*as N from"node:fs/promises";import*as ce from"node:path";function sr(e){if(!/^[0-9a-f]{64}$/.test(e)||e.includes("..")||e.includes("/")||e.includes("\\"))throw new Error("Invalid SHA-256 hash: expected 64 lowercase hexadecimal characters.");return ce.join("objects",e.slice(0,2),e.slice(2,4),e)}async function Ot(e,t){return N.readFile(ce.join(e,sr(t)))}var xo,To,ir,yo,It=w(()=>{"use strict";M();xo=64*1024,To=1440*60*1e3,ir=["EPERM","ENOTSUP","EXDEV","EACCES"],yo=new Set(ir)});import*as k from"node:fs";import*as C from"node:path";var Z=class extends Error{code;path;constructor(t,n,r){let o=r instanceof Error?r.message:String(r);super(n==="config_invalid"?`Invalid JSON in config file ${t}: ${o}`:`Unable to read config file ${t}: ${o}`,{cause:r}),this.name="ConfigDiscoveryError",this.code=n,this.path=t}};function Gt(e){let t;try{t=k.readFileSync(e,"utf8")}catch(n){throw new Z(e,"config_unreadable",n)}try{return JSON.parse(t)}catch(n){throw new Z(e,"config_invalid",n)}}function xe({cwd:e,homeDir:t,argv:n,env:r}){let o={};for(let x=0;x<n.length;x+=1){let A=/^--(client|project|root)=(.*)$/.exec(n[x]);A?o[A[1]]=A[2]:/^--(client|project|root)$/.test(n[x])&&n[x+1]!==void 0&&(o[n[x].slice(2)]=n[x+1],x+=1)}let i={client:r.RAG_CLIENT,project:r.RAG_PROJECT,root:r.RAG_ROOT},s=C.join(e,"rag.config.json"),a=C.join(C.dirname(e),"rag.config.json"),c=C.join(t,".rag","config.json"),l=k.existsSync(s)?s:k.existsSync(a)?a:k.existsSync(c)?c:null,d=l===null?{}:Gt(l),u=typeof d.client=="string"?d.client:void 0,g=typeof d.project=="string"?d.project:void 0,m=o.client??i.client??u??null,h=o.project??i.project??g??null,f=m===null&&h===null?null:{client:m,project:h},p=o.client!==void 0||o.project!==void 0?"args":i.client!==void 0||i.project!==void 0?"env":u!==void 0||g!==void 0?l===s?"cwd":l===a?"parent":"home":"unscoped";return{namespace:f,source:p,configPath:p==="cwd"||p==="parent"||p==="home"?l:null,failClosed:typeof d.failClosed=="boolean"?d.failClosed:!0,root:o.root??i.root??(typeof d.root=="string"?d.root:C.join(t,".rag"))}}be();import*as ee from"node:fs";import{DatabaseSync as Vt}from"node:sqlite";var Yt=800,Se=/[\p{L}\p{M}\p{N}]+/gu;function Jt(e){return e.match(Se)??[]}function Wt(e){return e.map(t=>`"${t.replaceAll('"','""')}"`).join(" AND ")}function ze(e){return e.normalize("NFD").replace(new RegExp("\\p{M}","gu"),"").toLowerCase()}function Kt(e,t){let n=new Set(t.map(ze));for(let r of e.matchAll(Se)){let o=r[0];if(!n.has(ze(o)))continue;let i=r.index;return{start:i,end:i+o.length}}return null}function Zt(e,t,n,r){let o=Math.min(r,e.length);if(o===0)return"";let i=Math.min(n-t,o),s=Math.max(0,Math.min(t-Math.floor((o-i)/2),e.length-o)),a=s+o,c=s,l=a,d=Array.from(e.matchAll(Se),m=>({start:m.index,end:m.index+m[0].length})),u=d.find(m=>m.start<s&&s<m.end);u&&(s=d.find(m=>m.start>=u.end)?.start??e.length);let g=d.find(m=>m.start<a&&a<m.end);if(g){let m=d.filter(h=>h.end<=g.start);a=m[m.length-1]?.end??0}return a<=s?e.slice(c,l):e.slice(s,a)}function qt(e,t){return e<t?-1:e>t?1:0}function Q(e,t,n){return{namespace:e,error:{code:t,message:n}}}function Ge(e){let{root:t,namespace:n,query:r}=e,o=e.excerptLength??Yt;if(!Number.isInteger(o)||o<1)return Q(n,"invalid_excerpt_length","excerptLength must be a positive integer.");let i=Jt(r);if(i.length===0)return{namespace:n,result:{citations:[],total:0}};let s;try{let a=q(t,n.client);if(!ee.existsSync(a)||!ee.statSync(a).isFile())return Q(n,"database_not_found",`No database exists for namespace ${n.client}/${n.project}.`);s=new Vt(a,{readOnly:!0});let c=s.prepare(`SELECT dv.doc_version_id AS docVersionId, d.document_id AS documentId,
|
|
1
|
+
import*as h from"node:fs";import*as N from"node:path";var I=class extends Error{code;path;constructor(t,n,r){let o=r instanceof Error?r.message:String(r);super(n==="config_invalid"?`Invalid JSON in config file ${t}: ${o}`:`Unable to read config file ${t}: ${o}`,{cause:r}),this.name="ConfigDiscoveryError",this.code=n,this.path=t}};function ie(e){let t;try{t=h.readFileSync(e,"utf8")}catch(n){throw new I(e,"config_unreadable",n)}try{return JSON.parse(t)}catch(n){throw new I(e,"config_invalid",n)}}function D({cwd:e,homeDir:t,argv:n,env:r}){let o={};for(let f=0;f<n.length;f+=1){let F=/^--(client|project|root)=(.*)$/.exec(n[f]);F?o[F[1]]=F[2]:/^--(client|project|root)$/.test(n[f])&&n[f+1]!==void 0&&(o[n[f].slice(2)]=n[f+1],f+=1)}let c={client:r.RAG_CLIENT,project:r.RAG_PROJECT,root:r.RAG_ROOT},d=N.join(e,"rag.config.json"),a=N.join(N.dirname(e),"rag.config.json"),u=N.join(t,".rag","config.json"),l=h.existsSync(d)?d:h.existsSync(a)?a:h.existsSync(u)?u:null,i=l===null?{}:ie(l),p=typeof i.client=="string"?i.client:void 0,E=typeof i.project=="string"?i.project:void 0,s=o.client??c.client??p??null,m=o.project??c.project??E??null,T=s===null&&m===null?null:{client:s,project:m},y=o.client!==void 0||o.project!==void 0?"args":c.client!==void 0||c.project!==void 0?"env":p!==void 0||E!==void 0?l===d?"cwd":l===a?"parent":"home":"unscoped";return{namespace:T,source:y,configPath:y==="cwd"||y==="parent"||y==="home"?l:null,failClosed:typeof i.failClosed=="boolean"?i.failClosed:!0,root:o.root??c.root??(typeof i.root=="string"?i.root:N.join(t,".rag"))}}import*as O from"node:fs";import{DatabaseSync as ae}from"node:sqlite";import*as B from"node:path";import{DatabaseSync as xe}from"node:sqlite";var M=class extends Error{code="invalid_client_name";constructor(t){super(`Invalid client name "${t}". Client names must match [A-Za-z0-9][A-Za-z0-9._-]* and cannot be "." or "..".`),this.name="InvalidClientNameError"}};function oe(e){if(e==="."||e===".."||!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(e))throw new M(e)}function v(e,t){return oe(t),B.join(e,"clients",t,"rag.db")}var se=800,P=/[\p{L}\p{M}\p{N}]+/gu;function ce(e){return e.match(P)??[]}function de(e){return e.map(t=>`"${t.replaceAll('"','""')}"`).join(" AND ")}function K(e){return e.normalize("NFD").replace(new RegExp("\\p{M}","gu"),"").toLowerCase()}function ue(e,t){let n=new Set(t.map(K));for(let r of e.matchAll(P)){let o=r[0];if(!n.has(K(o)))continue;let c=r.index;return{start:c,end:c+o.length}}return null}function le(e,t,n,r){let o=Math.min(r,e.length);if(o===0)return"";let c=Math.min(n-t,o),d=Math.max(0,Math.min(t-Math.floor((o-c)/2),e.length-o)),a=d+o,u=d,l=a,i=Array.from(e.matchAll(P),s=>({start:s.index,end:s.index+s[0].length})),p=i.find(s=>s.start<d&&d<s.end);p&&(d=i.find(s=>s.start>=p.end)?.start??e.length);let E=i.find(s=>s.start<a&&a<s.end);if(E){let s=i.filter(m=>m.end<=E.start);a=s[s.length-1]?.end??0}return a<=d?e.slice(u,l):e.slice(d,a)}function pe(e,t){return e<t?-1:e>t?1:0}function b(e,t,n){return{namespace:e,error:{code:t,message:n}}}function z(e){let{root:t,namespace:n,query:r}=e,o=e.excerptLength??se;if(!Number.isInteger(o)||o<1)return b(n,"invalid_excerpt_length","excerptLength must be a positive integer.");let c=ce(r);if(c.length===0)return{namespace:n,result:{citations:[],total:0}};let d;try{let a=v(t,n.client);if(!O.existsSync(a)||!O.statSync(a).isFile())return b(n,"database_not_found",`No database exists for namespace ${n.client}/${n.project}.`);d=new ae(a,{readOnly:!0});let u=d.prepare(`SELECT dv.doc_version_id AS docVersionId, d.document_id AS documentId,
|
|
5
2
|
e.entity_key AS entityKey, d.label AS label, dv.version_label AS versionLabel,
|
|
6
3
|
s.ordinal AS sectionOrdinal, c.ordinal AS chunkOrdinal, c.text AS text,
|
|
7
4
|
c.char_start AS charStart, c.char_end AS charEnd, bm25(chunk_fts) AS score
|
|
@@ -15,13 +12,17 @@ var w=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(r){throw n=
|
|
|
15
12
|
LEFT JOIN section AS s ON s.section_id = c.section_id
|
|
16
13
|
WHERE chunk_fts MATCH ? AND cl.name = ? AND p.slug = ?
|
|
17
14
|
AND d.deleted_at IS NULL AND e.deleted_at IS NULL
|
|
18
|
-
ORDER BY score ASC, dv.doc_version_id COLLATE BINARY ASC, c.ordinal ASC`).all(
|
|
15
|
+
ORDER BY score ASC, dv.doc_version_id COLLATE BINARY ASC, c.ordinal ASC`).all(de(c),n.client,n.project),l=[];for(let i of u){let p=ue(i.text,c);if(!p)return b(n,"match_offsets_unavailable","A matched FTS token could not be located in its stored chunk text.");l.push({docVersionId:i.docVersionId,documentId:i.documentId,entityKey:i.entityKey,label:i.label,versionLabel:i.versionLabel,sectionOrdinal:i.sectionOrdinal,chunkOrdinal:i.chunkOrdinal,charStart:i.charStart,charEnd:i.charEnd,matchStart:p.start,matchEnd:p.end,score:i.score,excerpt:le(i.text,p.start,p.end,o)})}return l.sort((i,p)=>i.score-p.score||pe(i.docVersionId,p.docVersionId)||i.chunkOrdinal-p.chunkOrdinal),{namespace:n,result:{citations:l,total:l.length}}}catch(a){let u=a instanceof Error?a.message:String(a);return b(n,"database_unavailable",u)}finally{d?.close()}}import*as x from"node:fs";import{DatabaseSync as Ee}from"node:sqlite";function C(e,t,n){return{namespace:e,error:{code:t,message:n}}}function me(e,t){let n=Buffer.from(e,"utf8");if(n.byteLength<=t)return{text:e,truncated:!1};let r=t;for(;r>0;)try{return{text:new TextDecoder("utf-8",{fatal:!0}).decode(n.subarray(0,r)),truncated:!0}}catch{r-=1}return{text:"",truncated:!0}}function _e(e,t){if(t===0)return 0;let n=0;for(let r=0;r<e.length;r+=1)if(!(e[r]!=="\r"&&e[r]!==`
|
|
16
|
+
`)&&(e[r]==="\r"&&e[r+1]===`
|
|
17
|
+
`&&(r+=1),n+=1,n===t))return r+1;return e.length}function Te(e){if(e.length===0)return 0;let t=1;for(let n=0;n<e.length;n+=1)e[n]==="\r"&&e[n+1]===`
|
|
18
|
+
`?(t+=1,n+=1):(e[n]==="\r"||e[n]===`
|
|
19
|
+
`)&&(t+=1);return t}function fe(e,t){if(e.length===0)return{text:e,truncated:!1};if(t===0)return{text:"",truncated:!0};let n=1;for(let r=0;r<e.length;r+=1){if(e[r]!=="\r"&&e[r]!==`
|
|
19
20
|
`)continue;let o=r;if(e[r]==="\r"&&e[r+1]===`
|
|
20
|
-
`&&(r+=1),n===t)return{text:e.slice(0,o),truncated:!0};n+=1}return{text:e,truncated:!1}}async function
|
|
21
|
+
`&&(r+=1),n===t)return{text:e.slice(0,o),truncated:!0};n+=1}return{text:e,truncated:!1}}async function V(e){let{root:t,namespace:n,docVersionId:r,chunkOrdinal:o,maxBytes:c,maxLines:d,startLine:a=0}=e;if(!Number.isInteger(c)||c<0||!Number.isInteger(d)||d<0||!Number.isInteger(a)||a<0)return C(n,"invalid_read_limit","maxBytes, maxLines, and startLine must be non-negative integers.");let u;try{let l=v(t,n.client);if(!x.existsSync(l)||!x.statSync(l).isFile())return C(n,"database_not_found",`No database exists for namespace ${n.client}/${n.project}.`);u=new Ee(l,{readOnly:!0});let i=u.prepare(`SELECT c.text AS text,
|
|
22
|
+
(SELECT COUNT(*) FROM chunk AS total_chunks WHERE total_chunks.doc_version_id = c.doc_version_id) AS totalChunks
|
|
21
23
|
FROM chunk AS c
|
|
22
24
|
JOIN doc_version AS dv ON dv.doc_version_id = c.doc_version_id
|
|
23
25
|
JOIN document AS d ON d.document_id = dv.document_id
|
|
24
26
|
JOIN project AS p ON p.project_id = d.project_id
|
|
25
27
|
JOIN client AS cl ON cl.client_id = p.client_id
|
|
26
|
-
|
|
27
|
-
WHERE c.doc_version_id = ? AND c.ordinal = ? AND cl.name = ? AND p.slug = ?`).get(r,o,n.client,n.project);if(!l)return I(n,"citation_not_found","The citation does not exist in the requested document version and namespace.");if(!l.sha256||!l.relPath)return I(n,"object_missing","The citation's recorded object is missing.");let d;try{let p=K.resolve(t,l.relPath),x=K.resolve(t);if(p!==x&&!p.startsWith(`${x}${K.sep}`))return I(n,"object_path_invalid","The recorded object path escapes the configured root.");d=await Ot(t,l.sha256)}catch(p){if(p.code==="ENOENT")return I(n,"object_missing","The citation's recorded object file is missing.");throw p}let u=await At(l.label).extract(d),g=u.sections[o]?.text;if(g===void 0)return I(n,"citation_not_found","The stored object does not contain the requested extracted section.");let m=cr(g,i),h=dr(m.text,s),f=u.sections.length;return{namespace:n,result:{text:h.text,truncated:m.truncated||h.truncated,totalChunks:f,nextChunkOrdinal:o+1<f?o+1:null}}}catch(c){let l=c instanceof Error?c.message:String(c);return I(n,"object_read_failed",l)}finally{a?.close()}}import{z as v}from"zod";var Xt=100,Ct=2e3,vt=1048576,kt=1e4,lr=10,ur=800,fr=65536,mr=200;function Mt(e){let t=new Set;for(let n of(e??"find,read").split(",").map(r=>r.trim())){if(n!=="find"&&n!=="read")throw new Error(`invalid_capability: Unknown RAG_CAPABILITIES value "${n}".`);t.add(n)}return t}var ue=v.string().min(1).refine(e=>e.trim().length>0),Fe=v.object({client:ue,project:ue}).strict();function fe(e,t,n){return v.number().int().min(e).max(t).optional().transform(r=>r??n)}var Dt=v.object({namespace:Fe,query:ue,limit:fe(1,Xt,lr),excerptChars:fe(1,Ct,ur)}).strict(),jt=v.object({namespace:Fe,docVersionId:ue,chunkOrdinal:v.number().multipleOf(1).min(0).refine(Number.isSafeInteger),maxBytes:fe(0,vt,fr),maxLines:fe(0,kt,mr)}).strict();function pr(e){let t=e.find(i=>i.code==="unrecognized_keys"&&i.path.length===0)??e.find(i=>i.code==="unrecognized_keys");if(!t||t.code!=="unrecognized_keys")return null;let n=t.keys[0];if(n===void 0)return null;let r=t.path.map(String).join(".");return{code:"unknown_field",message:`Unknown field "${r.length>0?`${r}.${n}`:n}".`}}function P(e,t){return e.some(n=>n.path[0]===t)}function le(e,t,n){return{code:"invalid_limit",message:`${e} must be an integer from ${t} to ${n}.`}}function Ut(e,t){let n=pr(e);if(n)return n;if(P(e,"namespace"))return{code:"namespace_required",message:"namespace must contain non-empty client and project strings."};if(t==="find"){if(P(e,"query"))return{code:"invalid_query",message:"query must be a non-empty string."};if(P(e,"limit"))return le("limit",1,Xt);if(P(e,"excerptChars"))return le("excerptChars",1,Ct)}else{if(P(e,"docVersionId"))return{code:"invalid_input",message:"docVersionId must be a non-empty string."};if(P(e,"chunkOrdinal"))return{code:"invalid_input",message:"chunkOrdinal must be a non-negative integer."};if(P(e,"maxBytes"))return le("maxBytes",0,vt);if(P(e,"maxLines"))return le("maxLines",0,kt)}return e.some(r=>r.path.length===0)?{code:"invalid_input",message:"Tool arguments must be an object."}:{code:"invalid_input",message:"Tool arguments are invalid."}}function gr(e){let t=Fe.safeParse(e);return t.success?t.data:null}function Ft(e){let t=Dt.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:me(e),error:Ut(t.error.issues,"find")}}function $t(e){let t=jt.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:me(e),error:Ut(t.error.issues,"read")}}function Er(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}function me(e){return Er(e)?gr(e.namespace):null}function Bt(e){return v.toJSONSchema(e,{io:"input"})}var pe={name:"rag_find",description:"Search a client/project knowledge base and return bounded citations.",inputSchema:Bt(Dt)},ge={name:"rag_read",description:"Read bounded text from a cited document chunk in the explicit namespace.",inputSchema:Bt(jt)};function $e(e,t=!1){return{content:[{type:"text",text:JSON.stringify(e)}],details:e,...t?{isError:!0}:{}}}function B(e,t){return $e({namespace:e,error:t},!0)}function Ee(e){return B(e,{code:"cancelled",message:"Tool call was cancelled."})}function zt(e,t){let n=t instanceof Error?t.message:String(t);return B(me(e),{code:"retrieval_failed",message:n})}var hr="The namespace travels on every call: never infer `client` or `project` from the cwd, repo name, config, env, or a placeholder like `default`. If the user has not stated it, ask before the first call.\nUse the RAG tool the role actually exposes; when only `rag_find` is available, delegate for deeper text instead of paraphrasing a catalog entry.\nWhen an excerpt is truncated, when the user asks whether the document covers something, or when a longer verbatim quote is needed, hand off to the subagent that exposes `rag_read` with the namespace and the citation's `docVersionId`; never assert absence from a catalog.";function vo(e){e.on("before_agent_start",n=>{if(process.env.RAG_PROMPT==="off")return;let r=n.systemPromptOptions.sections??(n.systemPromptOptions.sections={});r.rag=hr});let t=Mt(process.env.RAG_CAPABILITIES);t.has("find")&&e.registerTool({name:pe.name,label:"RAG Find",description:pe.description,parameters:pe.inputSchema,async execute(n,r,o,i,s){try{let a=Ft(r);if(!a.ok)return B(a.namespace,a.error);let{namespace:c,query:l,limit:d,excerptChars:u}=a.input;if(o?.aborted)return Ee(c);let g=xe({cwd:s.cwd,homeDir:s.cwd,argv:[],env:process.env}).root,m=Ge({root:g,namespace:c,query:l,excerptLength:u});return o?.aborted?Ee(c):"error"in m?B(c,m.error):$e({namespace:c,result:{...m.result,citations:m.result.citations.slice(0,d)}})}catch(a){return zt(r,a)}}}),t.has("read")&&e.registerTool({name:ge.name,label:"RAG Read",description:ge.description,parameters:ge.inputSchema,async execute(n,r,o,i,s){try{let a=$t(r);if(!a.ok)return B(a.namespace,a.error);let{namespace:c,docVersionId:l,chunkOrdinal:d,maxBytes:u,maxLines:g}=a.input;if(o?.aborted)return Ee(c);let m=xe({cwd:s.cwd,homeDir:s.cwd,argv:[],env:process.env}).root,h=await Pt({root:m,namespace:c,docVersionId:l,chunkOrdinal:d,maxBytes:u,maxLines:g});return o?.aborted?Ee(c):"error"in h?B(c,h.error):$e({namespace:c,result:h.result})}catch(a){return zt(r,a)}}})}export{vo as default};
|
|
28
|
+
WHERE c.doc_version_id = ? AND c.ordinal = ? AND cl.name = ? AND p.slug = ?`).get(r,o,n.client,n.project);if(!i)return C(n,"citation_not_found","The citation does not exist in the requested document version and namespace.");let p=i.text.slice(_e(i.text,a)),E=me(p,c),s=fe(E.text,d),m=E.truncated?"bytes":s.truncated?"lines":null,T=i.totalChunks;return{namespace:n,result:{text:s.text,truncated:m!==null,totalChunks:T,nextChunkOrdinal:o+1<T?o+1:null,chunkBytes:Buffer.byteLength(i.text,"utf8"),returnedBytes:Buffer.byteLength(s.text,"utf8"),returnedLines:Te(s.text),truncatedBy:m}}}catch(l){let i=l instanceof Error?l.message:String(l);return C(n,"object_read_failed",i)}finally{u?.close()}}import{z as g}from"zod";var $=100,q=2e3,H=1048576,S=1e4,Ne=10,ge=800,he=65536,Re=200;function J(e){let t=new Set;for(let n of(e??"find,read").split(",").map(r=>r.trim())){if(n!=="find"&&n!=="read")throw new Error(`invalid_capability: Unknown RAG_CAPABILITIES value "${n}".`);t.add(n)}return t}var k=g.string().min(1).refine(e=>e.trim().length>0),G=g.object({client:k,project:k}).strict();function A(e,t,n){return g.number().int().min(e).max(t).optional().transform(r=>r??n)}var Q=g.object({namespace:G,query:k,limit:A(1,$,Ne),excerptChars:A(1,q,ge)}).strict(),W=g.object({namespace:G,docVersionId:k,chunkOrdinal:g.number().multipleOf(1).min(0).refine(Number.isSafeInteger),maxBytes:A(0,H,he),maxLines:A(0,S,Re),startLine:A(0,S,0)}).strict();function Le(e){let t=e.find(c=>c.code==="unrecognized_keys"&&c.path.length===0)??e.find(c=>c.code==="unrecognized_keys");if(!t||t.code!=="unrecognized_keys")return null;let n=t.keys[0];if(n===void 0)return null;let r=t.path.map(String).join(".");return{code:"unknown_field",message:`Unknown field "${r.length>0?`${r}.${n}`:n}".`}}function _(e,t){return e.some(n=>n.path[0]===t)}function L(e,t,n){return{code:"invalid_limit",message:`${e} must be an integer from ${t} to ${n}.`}}function Z(e,t){let n=Le(e);if(n)return n;if(_(e,"namespace"))return{code:"namespace_required",message:"namespace must contain non-empty client and project strings."};if(t==="find"){if(_(e,"query"))return{code:"invalid_query",message:"query must be a non-empty string."};if(_(e,"limit"))return L("limit",1,$);if(_(e,"excerptChars"))return L("excerptChars",1,q)}else{if(_(e,"docVersionId"))return{code:"invalid_input",message:"docVersionId must be a non-empty string."};if(_(e,"chunkOrdinal"))return{code:"invalid_input",message:"chunkOrdinal must be a non-negative integer."};if(_(e,"maxBytes"))return L("maxBytes",0,H);if(_(e,"maxLines"))return L("maxLines",0,S);if(_(e,"startLine"))return L("startLine",0,S)}return e.some(r=>r.path.length===0)?{code:"invalid_input",message:"Tool arguments must be an object."}:{code:"invalid_input",message:"Tool arguments are invalid."}}function Ae(e){let t=G.safeParse(e);return t.success?t.data:null}function ee(e){let t=Q.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:U(e),error:Z(t.error.issues,"find")}}function ne(e){let t=W.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:U(e),error:Z(t.error.issues,"read")}}function ye(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}function U(e){return ye(e)?Ae(e.namespace):null}function te(e){return g.toJSONSchema(e,{io:"input"})}var X={name:"rag_find",description:"Search a client/project knowledge base and return bounded citations.",inputSchema:te(Q)},j={name:"rag_read",description:"Read bounded text from a cited document chunk in the explicit namespace.",inputSchema:te(W)};function Y(e,t=!1){return{content:[{type:"text",text:JSON.stringify(e)}],details:e,...t?{isError:!0}:{}}}function R(e,t){return Y({namespace:e,error:t},!0)}function w(e){return R(e,{code:"cancelled",message:"Tool call was cancelled."})}function re(e,t){let n=t instanceof Error?t.message:String(t);return R(U(e),{code:"retrieval_failed",message:n})}var Ie="The namespace travels on every call: never infer `client` or `project` from the cwd, repo name, config, env, or a placeholder like `default`. If the user has not stated it, ask before the first call.\nUse the RAG tool the role actually exposes; when only `rag_find` is available, delegate for deeper text instead of paraphrasing a catalog entry.\nWhen an excerpt is truncated, when the user asks whether the document covers something, or when a longer verbatim quote is needed, hand off to the subagent that exposes `rag_read` with the namespace and the citation's `docVersionId`; never assert absence from a catalog.";function ze(e){e.on("before_agent_start",n=>{if(process.env.RAG_PROMPT==="off")return;let r=n.systemPromptOptions.sections??(n.systemPromptOptions.sections={});r.rag=Ie});let t=J(process.env.RAG_CAPABILITIES);t.has("find")&&e.registerTool({name:X.name,label:"RAG Find",description:X.description,parameters:X.inputSchema,async execute(n,r,o,c,d){try{let a=ee(r);if(!a.ok)return R(a.namespace,a.error);let{namespace:u,query:l,limit:i,excerptChars:p}=a.input;if(o?.aborted)return w(u);let E=D({cwd:d.cwd,homeDir:d.cwd,argv:[],env:process.env}).root,s=z({root:E,namespace:u,query:l,excerptLength:p});return o?.aborted?w(u):"error"in s?R(u,s.error):Y({namespace:u,result:{...s.result,citations:s.result.citations.slice(0,i)}})}catch(a){return re(r,a)}}}),t.has("read")&&e.registerTool({name:j.name,label:"RAG Read",description:j.description,parameters:j.inputSchema,async execute(n,r,o,c,d){try{let a=ne(r);if(!a.ok)return R(a.namespace,a.error);let{namespace:u,docVersionId:l,chunkOrdinal:i,maxBytes:p,maxLines:E,startLine:s}=a.input;if(o?.aborted)return w(u);let m=D({cwd:d.cwd,homeDir:d.cwd,argv:[],env:process.env}).root,T=await V({root:m,namespace:u,docVersionId:l,chunkOrdinal:i,maxBytes:p,maxLines:E,startLine:s});return o?.aborted?w(u):"error"in T?R(u,T.error):Y({namespace:u,result:T.result})}catch(a){return re(r,a)}}})}export{ze as default};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { convertToHtml } from "mammoth";
|
|
2
|
+
import { MAX_SECTION_CHARS } from "../limits.mjs";
|
|
2
3
|
const DOCX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
3
4
|
function supports(ref) {
|
|
4
5
|
if (typeof ref === "string") return /\.docx$/i.test(ref);
|
|
@@ -9,14 +10,104 @@ function supports(ref) {
|
|
|
9
10
|
const fileName = fields.filename ?? fields.name ?? fields.path ?? fields.extension;
|
|
10
11
|
return typeof fileName === "string" && /(?:^|[\\/])[^\\/]+\.docx$/i.test(fileName);
|
|
11
12
|
}
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
function htmlText(value) {
|
|
14
|
+
const entities = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: "\xA0", quot: '"' };
|
|
15
|
+
return value.replace(/<br\b[^>]*>/gi, "\n").replace(/<[^>]*>/g, "").replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (entity, code) => {
|
|
16
|
+
if (code[0] !== "#") return entities[code.toLowerCase()] ?? entity;
|
|
17
|
+
const numeric = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
|
|
18
|
+
return Number.isInteger(numeric) && numeric >= 0 && numeric <= 1114111 ? String.fromCodePoint(numeric) : entity;
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
function readBlocks(html) {
|
|
22
|
+
const blocks = [];
|
|
23
|
+
const tags = /<(h([1-6])|p|li)\b[^>]*>([\s\S]*?)<\/\1\s*>/gi;
|
|
24
|
+
for (const match of html.matchAll(tags)) {
|
|
25
|
+
const level = match[2] === void 0 ? null : Number(match[2]);
|
|
26
|
+
blocks.push({ text: htmlText(match[3] ?? ""), heading: level === null ? null : htmlText(match[3] ?? ""), level });
|
|
27
|
+
}
|
|
28
|
+
return blocks.filter((block) => block.text.length > 0);
|
|
29
|
+
}
|
|
30
|
+
function isWhitespace(value) {
|
|
31
|
+
return value !== void 0 && /\s/u.test(value);
|
|
32
|
+
}
|
|
33
|
+
function splitLongParagraph(text) {
|
|
34
|
+
const sections = [];
|
|
35
|
+
let start = 0;
|
|
36
|
+
while (text.length - start > MAX_SECTION_CHARS) {
|
|
37
|
+
let boundary = Math.min(start + MAX_SECTION_CHARS, text.length - 1);
|
|
38
|
+
while (boundary > start && !isWhitespace(text[boundary])) boundary -= 1;
|
|
39
|
+
if (boundary === start) {
|
|
40
|
+
boundary = start + MAX_SECTION_CHARS + 1;
|
|
41
|
+
while (boundary < text.length && !isWhitespace(text[boundary])) boundary += 1;
|
|
42
|
+
if (boundary === text.length) break;
|
|
43
|
+
}
|
|
44
|
+
sections.push(text.slice(start, boundary));
|
|
45
|
+
start = boundary + 1;
|
|
46
|
+
}
|
|
47
|
+
if (start < text.length || sections.length === 0) sections.push(text.slice(start));
|
|
48
|
+
return sections;
|
|
49
|
+
}
|
|
50
|
+
function packParagraphs(paragraphs) {
|
|
51
|
+
const sections = [];
|
|
52
|
+
let current = null;
|
|
53
|
+
const flush = () => {
|
|
54
|
+
if (current !== null) sections.push(current);
|
|
55
|
+
current = null;
|
|
19
56
|
};
|
|
57
|
+
for (const paragraph of paragraphs) {
|
|
58
|
+
if (paragraph.length > MAX_SECTION_CHARS) {
|
|
59
|
+
flush();
|
|
60
|
+
const pieces = splitLongParagraph(paragraph);
|
|
61
|
+
sections.push(...pieces.slice(0, -1));
|
|
62
|
+
current = pieces.at(-1) ?? null;
|
|
63
|
+
if (current !== null && current.length > MAX_SECTION_CHARS) flush();
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
const candidate = current === null ? paragraph : `${current}
|
|
67
|
+
${paragraph}`;
|
|
68
|
+
if (candidate.length <= MAX_SECTION_CHARS) current = candidate;
|
|
69
|
+
else {
|
|
70
|
+
flush();
|
|
71
|
+
current = paragraph;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
flush();
|
|
75
|
+
return sections;
|
|
76
|
+
}
|
|
77
|
+
function groupByHeadings(blocks) {
|
|
78
|
+
const sections = [];
|
|
79
|
+
let heading = null;
|
|
80
|
+
let level = null;
|
|
81
|
+
let paragraphs = [];
|
|
82
|
+
const flush = () => {
|
|
83
|
+
if (paragraphs.length > 0) {
|
|
84
|
+
const packable = heading !== null && paragraphs.length > 1 ? [`${paragraphs[0]}
|
|
85
|
+
${paragraphs[1]}`, ...paragraphs.slice(2)] : paragraphs;
|
|
86
|
+
for (const [index, text] of packParagraphs(packable).entries()) {
|
|
87
|
+
sections.push({ ordinal: sections.length, heading: index === 0 ? heading : null, level: index === 0 ? level : null, text });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
paragraphs = [];
|
|
91
|
+
};
|
|
92
|
+
for (const block of blocks) {
|
|
93
|
+
if (block.heading !== null) {
|
|
94
|
+
flush();
|
|
95
|
+
heading = block.heading;
|
|
96
|
+
level = block.level;
|
|
97
|
+
} else if (paragraphs.length === 0) {
|
|
98
|
+
heading = null;
|
|
99
|
+
level = null;
|
|
100
|
+
}
|
|
101
|
+
paragraphs.push(block.text);
|
|
102
|
+
}
|
|
103
|
+
flush();
|
|
104
|
+
return sections;
|
|
105
|
+
}
|
|
106
|
+
async function extract(bytes) {
|
|
107
|
+
const result = await convertToHtml({ buffer: Buffer.from(bytes) });
|
|
108
|
+
const blocks = readBlocks(result.value);
|
|
109
|
+
const sections = blocks.some((block) => block.heading !== null) ? groupByHeadings(blocks) : packParagraphs(blocks.map((block) => block.text)).map((text, ordinal) => ({ ordinal, heading: null, level: null, text }));
|
|
110
|
+
return { sections, versionLabel: null, mediaType: DOCX_MEDIA_TYPE };
|
|
20
111
|
}
|
|
21
112
|
const docxExtractor = {
|
|
22
113
|
name: "docx",
|
package/dist/src/cli/index.mjs
CHANGED
|
@@ -11,7 +11,7 @@ import { clientDbPath, openDatabase, validateClientName } from "../store/db.mjs"
|
|
|
11
11
|
import { backupRoot, reindexRoot, restoreRoot, writeRootManifest } from "../store/backup.mjs";
|
|
12
12
|
import { readObject } from "../store/objects.mjs";
|
|
13
13
|
import { rehashNamespace } from "../store/rehash.mjs";
|
|
14
|
-
import { getActiveLimits } from "../limits.mjs";
|
|
14
|
+
import { getActiveLimits, MAX_SECTION_CHARS } from "../limits.mjs";
|
|
15
15
|
import { renderOutput, resolveOutputFormat } from "./format.mjs";
|
|
16
16
|
import {
|
|
17
17
|
linkReviewVersions,
|
|
@@ -327,10 +327,12 @@ async function verifyNamespaceObjects(root, namespace) {
|
|
|
327
327
|
let objects;
|
|
328
328
|
let chunks;
|
|
329
329
|
let indexedChunks;
|
|
330
|
+
let oversizedChunks;
|
|
330
331
|
try {
|
|
331
332
|
objects = db.prepare("SELECT sha256, byte_size FROM object ORDER BY sha256").all();
|
|
332
333
|
chunks = Number(db.prepare("SELECT COUNT(*) AS count FROM chunk").get()?.count ?? 0);
|
|
333
334
|
indexedChunks = Number(db.prepare("SELECT COUNT(*) AS count FROM chunk_fts_docsize").get()?.count ?? 0);
|
|
335
|
+
oversizedChunks = Number(db.prepare("SELECT COUNT(*) AS count FROM chunk WHERE length(text) > ?").get(MAX_SECTION_CHARS)?.count ?? 0);
|
|
334
336
|
} finally {
|
|
335
337
|
db.close();
|
|
336
338
|
}
|
|
@@ -364,6 +366,7 @@ async function verifyNamespaceObjects(root, namespace) {
|
|
|
364
366
|
corrupt,
|
|
365
367
|
limits: await getActiveLimits(),
|
|
366
368
|
...indexHealthy ? {} : { chunks, indexedChunks },
|
|
369
|
+
...oversizedChunks === 0 ? {} : { oversizedChunks },
|
|
367
370
|
// Keep healthy doctor envelopes stable; surface counts whenever artifacts need attention.
|
|
368
371
|
...artifacts.unreferencedObjects === 0 && artifacts.leftoverTemporaries === 0 ? {} : artifacts
|
|
369
372
|
}
|
package/dist/src/limits.mjs
CHANGED
|
@@ -4,6 +4,7 @@ const MAX_PDF_OCR_RENDER_PIXELS = 5e6;
|
|
|
4
4
|
const DEFAULT_PARSE_MAX_OLD_SPACE_SIZE_MB = 512;
|
|
5
5
|
const DEFAULT_PARSE_TIMEOUT_MS = 12e4;
|
|
6
6
|
const MAX_SOURCE_FILE_BYTES = 64 * 1024 * 1024;
|
|
7
|
+
const MAX_SECTION_CHARS = 8192;
|
|
7
8
|
const LIMIT_NAMES_BY_REJECTION = {
|
|
8
9
|
source_too_large: ["MAX_SOURCE_FILE_BYTES"],
|
|
9
10
|
too_many_pages: ["MAX_PDF_PAGES", "MAX_OCR_PDF_PAGES"],
|
|
@@ -27,6 +28,7 @@ async function getActiveLimits() {
|
|
|
27
28
|
]);
|
|
28
29
|
return [
|
|
29
30
|
{ name: "MAX_SOURCE_FILE_BYTES", value: MAX_SOURCE_FILE_BYTES2, unit: "bytes", reason: "Allows large manuals while bounding bytes held for hashing and extraction." },
|
|
31
|
+
{ name: "MAX_SECTION_CHARS", value: MAX_SECTION_CHARS, unit: "characters", reason: "Keeps DOCX sections within a predictable retrieval text window." },
|
|
30
32
|
{ name: "MAX_PDF_PAGES", value: MAX_PDF_PAGES, unit: "pages", reason: "Covers exceptionally long references while limiting page extraction from hostile PDFs." },
|
|
31
33
|
{ name: "MAX_OCR_PDF_PAGES", value: MAX_OCR_PDF_PAGES, unit: "pages", reason: "OCR is far slower than text extraction, so scanned PDFs use a separate page budget." },
|
|
32
34
|
{ name: "MAX_PDF_SOURCE_IMAGE_PIXELS", value: MAX_PDF_SOURCE_IMAGE_PIXELS, unit: "pixels", reason: "Pixel budget of a decoded source image, checked by the pdfjs worker before PDFImage is constructed." },
|
|
@@ -47,6 +49,7 @@ export {
|
|
|
47
49
|
MAX_OCR_PDF_PAGES,
|
|
48
50
|
MAX_PDF_OCR_RENDER_PIXELS,
|
|
49
51
|
MAX_PDF_SOURCE_IMAGE_PIXELS,
|
|
52
|
+
MAX_SECTION_CHARS,
|
|
50
53
|
MAX_SOURCE_FILE_BYTES,
|
|
51
54
|
getActiveLimits
|
|
52
55
|
};
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
|
-
import * as path from "node:path";
|
|
3
2
|
import { DatabaseSync } from "node:sqlite";
|
|
4
|
-
import { getExtractorForPath } from "../adapters/registry.mjs";
|
|
5
3
|
import { clientDbPath } from "../store/db.mjs";
|
|
6
|
-
import { readObject } from "../store/objects.mjs";
|
|
7
4
|
function failure(namespace, code, message) {
|
|
8
5
|
return { namespace, error: { code, message } };
|
|
9
6
|
}
|
|
@@ -20,6 +17,30 @@ function limitUtf8(text, maxBytes) {
|
|
|
20
17
|
}
|
|
21
18
|
return { text: "", truncated: true };
|
|
22
19
|
}
|
|
20
|
+
function startLineOffset(text, startLine) {
|
|
21
|
+
if (startLine === 0) return 0;
|
|
22
|
+
let line = 0;
|
|
23
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
24
|
+
if (text[index] !== "\r" && text[index] !== "\n") continue;
|
|
25
|
+
if (text[index] === "\r" && text[index + 1] === "\n") index += 1;
|
|
26
|
+
line += 1;
|
|
27
|
+
if (line === startLine) return index + 1;
|
|
28
|
+
}
|
|
29
|
+
return text.length;
|
|
30
|
+
}
|
|
31
|
+
function countLines(text) {
|
|
32
|
+
if (text.length === 0) return 0;
|
|
33
|
+
let lines = 1;
|
|
34
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
35
|
+
if (text[index] === "\r" && text[index + 1] === "\n") {
|
|
36
|
+
lines += 1;
|
|
37
|
+
index += 1;
|
|
38
|
+
} else if (text[index] === "\r" || text[index] === "\n") {
|
|
39
|
+
lines += 1;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return lines;
|
|
43
|
+
}
|
|
23
44
|
function limitLines(text, maxLines) {
|
|
24
45
|
if (text.length === 0) return { text, truncated: false };
|
|
25
46
|
if (maxLines === 0) return { text: "", truncated: true };
|
|
@@ -34,9 +55,9 @@ function limitLines(text, maxLines) {
|
|
|
34
55
|
return { text, truncated: false };
|
|
35
56
|
}
|
|
36
57
|
async function readFromCitation(input) {
|
|
37
|
-
const { root, namespace, docVersionId, chunkOrdinal, maxBytes, maxLines } = input;
|
|
38
|
-
if (!Number.isInteger(maxBytes) || maxBytes < 0 || !Number.isInteger(maxLines) || maxLines < 0) {
|
|
39
|
-
return failure(namespace, "invalid_read_limit", "maxBytes and
|
|
58
|
+
const { root, namespace, docVersionId, chunkOrdinal, maxBytes, maxLines, startLine = 0 } = input;
|
|
59
|
+
if (!Number.isInteger(maxBytes) || maxBytes < 0 || !Number.isInteger(maxLines) || maxLines < 0 || !Number.isInteger(startLine) || startLine < 0) {
|
|
60
|
+
return failure(namespace, "invalid_read_limit", "maxBytes, maxLines, and startLine must be non-negative integers.");
|
|
40
61
|
}
|
|
41
62
|
let db;
|
|
42
63
|
try {
|
|
@@ -45,45 +66,31 @@ async function readFromCitation(input) {
|
|
|
45
66
|
return failure(namespace, "database_not_found", `No database exists for namespace ${namespace.client}/${namespace.project}.`);
|
|
46
67
|
}
|
|
47
68
|
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
48
|
-
const citation = db.prepare(`SELECT
|
|
69
|
+
const citation = db.prepare(`SELECT c.text AS text,
|
|
70
|
+
(SELECT COUNT(*) FROM chunk AS total_chunks WHERE total_chunks.doc_version_id = c.doc_version_id) AS totalChunks
|
|
49
71
|
FROM chunk AS c
|
|
50
72
|
JOIN doc_version AS dv ON dv.doc_version_id = c.doc_version_id
|
|
51
73
|
JOIN document AS d ON d.document_id = dv.document_id
|
|
52
74
|
JOIN project AS p ON p.project_id = d.project_id
|
|
53
75
|
JOIN client AS cl ON cl.client_id = p.client_id
|
|
54
|
-
LEFT JOIN object AS o ON o.sha256 = dv.sha256
|
|
55
76
|
WHERE c.doc_version_id = ? AND c.ordinal = ? AND cl.name = ? AND p.slug = ?`).get(docVersionId, chunkOrdinal, namespace.client, namespace.project);
|
|
56
77
|
if (!citation) return failure(namespace, "citation_not_found", "The citation does not exist in the requested document version and namespace.");
|
|
57
|
-
|
|
58
|
-
let bytes;
|
|
59
|
-
try {
|
|
60
|
-
const objectPath = path.resolve(root, citation.relPath);
|
|
61
|
-
const rootPath = path.resolve(root);
|
|
62
|
-
if (objectPath !== rootPath && !objectPath.startsWith(`${rootPath}${path.sep}`)) {
|
|
63
|
-
return failure(namespace, "object_path_invalid", "The recorded object path escapes the configured root.");
|
|
64
|
-
}
|
|
65
|
-
bytes = await readObject(root, citation.sha256);
|
|
66
|
-
} catch (error) {
|
|
67
|
-
if (error.code === "ENOENT") {
|
|
68
|
-
return failure(namespace, "object_missing", "The citation's recorded object file is missing.");
|
|
69
|
-
}
|
|
70
|
-
throw error;
|
|
71
|
-
}
|
|
72
|
-
const extracted = await getExtractorForPath(citation.label).extract(bytes);
|
|
73
|
-
const chunkText = extracted.sections[chunkOrdinal]?.text;
|
|
74
|
-
if (chunkText === void 0) {
|
|
75
|
-
return failure(namespace, "citation_not_found", "The stored object does not contain the requested extracted section.");
|
|
76
|
-
}
|
|
78
|
+
const chunkText = citation.text.slice(startLineOffset(citation.text, startLine));
|
|
77
79
|
const byteLimited = limitUtf8(chunkText, maxBytes);
|
|
78
80
|
const lineLimited = limitLines(byteLimited.text, maxLines);
|
|
79
|
-
const
|
|
81
|
+
const truncatedBy = byteLimited.truncated ? "bytes" : lineLimited.truncated ? "lines" : null;
|
|
82
|
+
const totalChunks = citation.totalChunks;
|
|
80
83
|
return {
|
|
81
84
|
namespace,
|
|
82
85
|
result: {
|
|
83
86
|
text: lineLimited.text,
|
|
84
|
-
truncated:
|
|
87
|
+
truncated: truncatedBy !== null,
|
|
85
88
|
totalChunks,
|
|
86
|
-
nextChunkOrdinal: chunkOrdinal + 1 < totalChunks ? chunkOrdinal + 1 : null
|
|
89
|
+
nextChunkOrdinal: chunkOrdinal + 1 < totalChunks ? chunkOrdinal + 1 : null,
|
|
90
|
+
chunkBytes: Buffer.byteLength(citation.text, "utf8"),
|
|
91
|
+
returnedBytes: Buffer.byteLength(lineLimited.text, "utf8"),
|
|
92
|
+
returnedLines: countLines(lineLimited.text),
|
|
93
|
+
truncatedBy
|
|
87
94
|
}
|
|
88
95
|
};
|
|
89
96
|
} catch (error) {
|
package/dist/src/server.mjs
CHANGED
|
@@ -49,8 +49,8 @@ async function main() {
|
|
|
49
49
|
}
|
|
50
50
|
const check = validateRagReadInput(args);
|
|
51
51
|
if (!check.ok) return toolError(check.namespace, check.error);
|
|
52
|
-
const { namespace, docVersionId, chunkOrdinal, maxBytes, maxLines } = check.input;
|
|
53
|
-
const read = await readFromCitation({ root, namespace, docVersionId, chunkOrdinal, maxBytes, maxLines });
|
|
52
|
+
const { namespace, docVersionId, chunkOrdinal, maxBytes, maxLines, startLine } = check.input;
|
|
53
|
+
const read = await readFromCitation({ root, namespace, docVersionId, chunkOrdinal, maxBytes, maxLines, startLine });
|
|
54
54
|
if ("error" in read) return toolError(namespace, read.error);
|
|
55
55
|
return toolResult({ namespace, result: read.result });
|
|
56
56
|
} catch (error) {
|
|
@@ -36,7 +36,8 @@ const ragReadInputSchema = z.object({
|
|
|
36
36
|
docVersionId: nonBlankString,
|
|
37
37
|
chunkOrdinal: z.number().multipleOf(1).min(0).refine(Number.isSafeInteger),
|
|
38
38
|
maxBytes: defaultedInteger(0, MAX_BYTES, DEFAULT_MAX_BYTES),
|
|
39
|
-
maxLines: defaultedInteger(0, MAX_LINES, DEFAULT_MAX_LINES)
|
|
39
|
+
maxLines: defaultedInteger(0, MAX_LINES, DEFAULT_MAX_LINES),
|
|
40
|
+
startLine: defaultedInteger(0, MAX_LINES, 0)
|
|
40
41
|
}).strict();
|
|
41
42
|
function unknownField(issues) {
|
|
42
43
|
const issue = issues.find((item) => item.code === "unrecognized_keys" && item.path.length === 0) ?? issues.find((item) => item.code === "unrecognized_keys");
|
|
@@ -77,6 +78,7 @@ function inputFailure(issues, kind) {
|
|
|
77
78
|
}
|
|
78
79
|
if (hasIssue(issues, "maxBytes")) return limitFailure("maxBytes", 0, MAX_BYTES);
|
|
79
80
|
if (hasIssue(issues, "maxLines")) return limitFailure("maxLines", 0, MAX_LINES);
|
|
81
|
+
if (hasIssue(issues, "startLine")) return limitFailure("startLine", 0, MAX_LINES);
|
|
80
82
|
}
|
|
81
83
|
if (issues.some((issue) => issue.path.length === 0)) {
|
|
82
84
|
return { code: "invalid_input", message: "Tool arguments must be an object." };
|
package/package.json
CHANGED
|
@@ -7,6 +7,7 @@ description: "Trigger: search or read the indexed knowledge base with explicit c
|
|
|
7
7
|
|
|
8
8
|
- Use `rag_find` to search a client/project namespace and get a bounded catalog of matching citations.
|
|
9
9
|
- Use `rag_read` only when a citation needs more surrounding text; pass its `docVersionId` and `chunkOrdinal` from `rag_find`.
|
|
10
|
+
- When a chunk exceeds the line limit, ensure each page fits `maxBytes` and repeat `rag_read` with `startLine` advanced by `returnedLines` to cover every line without overlap or gaps; `truncatedBy: null` marks the last page.
|
|
10
11
|
- The orchestrator only receives `rag_find`; delegate exploration when a citation needs reading.
|
|
11
12
|
|
|
12
13
|
## Evidence and scope
|