agent-rag-mcp 0.4.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/extensions/agent-rag.mjs +9 -6
- package/dist/src/adapters/docx.mjs +99 -8
- package/dist/src/cli/index.mjs +4 -1
- package/dist/src/limits.mjs +3 -0
- package/dist/src/retrieve/find.mjs +16 -4
- package/dist/src/retrieve/read.mjs +7 -1
- package/dist/src/tools/schema.mjs +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -28,7 +28,7 @@ The `rag` commands available today are:
|
|
|
28
28
|
- `rag review list` — Lists version pairs awaiting a human decision.
|
|
29
29
|
- `rag review link <from> <to> --type <conflicts_with|same_entity>` — Records a human link between versions with a note.
|
|
30
30
|
- `rag status` — Shows counts for the namespace.
|
|
31
|
-
- `rag doctor` — Verifies registered-object integrity and reports missing objects, orphan blobs,
|
|
31
|
+
- `rag doctor` — Verifies registered-object integrity and reports missing objects, orphan blobs, temporary files, and stored chunks over the DOCX section cap.
|
|
32
32
|
- `rag backup --to <dir>` — Backs up the manifest and object store.
|
|
33
33
|
- `rag restore --from <dir>` — Restores a backup under this root.
|
|
34
34
|
- `rag reindex` — Rebuilds the database from the manifest and objects.
|
|
@@ -85,10 +85,12 @@ Register two separate server processes in the adapter configuration (this exampl
|
|
|
85
85
|
|
|
86
86
|
The orchestrator receives only `rag_find`, which returns citations and bounded excerpts. The explorer also receives `rag_read`, which reads bounded text from a citation. Keeping them separate applies least privilege. The server reads `RAG_ROOT` to locate the knowledge base and `RAG_CAPABILITIES` to decide which tools to announce.
|
|
87
87
|
|
|
88
|
-
The extension adds a `rag` section to the agent system prompt describing the namespace and delegation rules;
|
|
88
|
+
The extension adds a `rag` section to the agent system prompt describing the namespace and delegation rules; it contains no client, project, or path value. For truncated excerpts, coverage questions, or longer verbatim quotes, it requires deeper text to be read by a subagent that exposes `rag_read` and forbids asserting absence from a catalog. Set `RAG_PROMPT=off` (exactly that value) to suppress it; any other value, including unset, keeps it.
|
|
89
89
|
|
|
90
90
|
The namespace travels in every `rag_find` or `rag_read` call; it does not come from the environment. For example, include `"namespace": {"client":"acme","project":"research"}` in each tool call's arguments. Do not configure `RAG_CLIENT` or `RAG_PROJECT` as a substitute for the per-call namespace.
|
|
91
91
|
|
|
92
|
+
`rag_find` returns `total`, the number of matching chunks before the `limit` slice; `citations.length` is how many citations were actually returned, so comparing them shows how many matches were cut by the limit. Its `excerptChars` input defaults to 800. Every successful `rag_read` result includes `totalChunks` and `nextChunkOrdinal`; the latter is the next zero-based chunk ordinal, or `null` when there is no following chunk. `truncated` means this chunk was not delivered whole, so re-read the same ordinal with higher limits; it does not indicate whether another chunk exists.
|
|
93
|
+
|
|
92
94
|
## Requirements
|
|
93
95
|
|
|
94
96
|
- Node.js >= 22.13.
|
|
@@ -101,6 +103,8 @@ The namespace travels in every `rag_find` or `rag_read` call; it does not come f
|
|
|
101
103
|
|
|
102
104
|
Version 0.1.0 has no embeddings or Tier 2; scanned PDFs go through OCR. Document parsing runs in a child process with a 512 MB V8 heap cap.
|
|
103
105
|
|
|
106
|
+
At ingest, DOCX files split by heading; heading groups over 8,192 characters are subdivided at whitespace, keeping the heading at the start of the first section. Files without headings use greedy paragraph chunks capped at 8,192 characters, and oversized paragraphs split at whitespace. Already-ingested DOCX documents keep their existing single chunk until re-ingested; `rag reindex` does not re-extract source documents.
|
|
107
|
+
|
|
104
108
|
## Design documents
|
|
105
109
|
|
|
106
110
|
- [Architecture](https://github.com/Lsarmientov/agent-rag/blob/main/docs/design/0001-arquitectura.md)
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
var w=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(r){throw n=[r],r}};var
|
|
1
|
+
var w=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(r){throw n=[r],r}};var He=w(()=>{"use strict"});import*as be from"node:path";import{DatabaseSync as Ir}from"node:sqlite";function Wt(e){if(e==="."||e===".."||!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(e))throw new ye(e)}function Q(e,t){return Wt(t),be.join(e,"clients",t,"rag.db")}var ye,Se=w(()=>{"use strict";He();ye=class extends Error{code="invalid_client_name";constructor(t){super(`Invalid client name "${t}". Client names must match [A-Za-z0-9][A-Za-z0-9._-]* and cannot be "." or "..".`),this.name="InvalidClientNameError"}}});var Re=w(()=>{"use strict"});import{convertToHtml as rn}from"mammoth";function on(e){if(typeof e=="string")return/\.docx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===Ne;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.docx$/i.test(r)}function Ye(e){let t={amp:"&",apos:"'",gt:">",lt:"<",nbsp:"\xA0",quot:'"'};return e.replace(/<br\b[^>]*>/gi,`
|
|
2
|
+
`).replace(/<[^>]*>/g,"").replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi,(n,r)=>{if(r[0]!=="#")return t[r.toLowerCase()]??n;let o=r[1]?.toLowerCase()==="x"?Number.parseInt(r.slice(2),16):Number.parseInt(r.slice(1),10);return Number.isInteger(o)&&o>=0&&o<=1114111?String.fromCodePoint(o):n})}function sn(e){let t=[],n=/<(h([1-6])|p|li)\b[^>]*>([\s\S]*?)<\/\1\s*>/gi;for(let r of e.matchAll(n)){let o=r[2]===void 0?null:Number(r[2]);t.push({text:Ye(r[3]??""),heading:o===null?null:Ye(r[3]??""),level:o})}return t.filter(r=>r.text.length>0)}function Je(e){return e!==void 0&&/\s/u.test(e)}function an(e){let t=[],n=0;for(;e.length-n>8192;){let r=Math.min(n+8192,e.length-1);for(;r>n&&!Je(e[r]);)r-=1;if(r===n){for(r=n+8192+1;r<e.length&&!Je(e[r]);)r+=1;if(r===e.length)break}t.push(e.slice(n,r)),n=r+1}return(n<e.length||t.length===0)&&t.push(e.slice(n)),t}function We(e){let t=[],n=null,r=()=>{n!==null&&t.push(n),n=null};for(let o of e){if(o.length>8192){r();let s=an(o);t.push(...s.slice(0,-1)),n=s.at(-1)??null,n!==null&&n.length>8192&&r();continue}let i=n===null?o:`${n}
|
|
3
|
+
${o}`;i.length<=8192?n=i:(r(),n=o)}return r(),t}function cn(e){let t=[],n=null,r=null,o=[],i=()=>{if(o.length>0){let s=n!==null&&o.length>1?[`${o[0]}
|
|
4
|
+
${o[1]}`,...o.slice(2)]:o;for(let[a,c]of We(s).entries())t.push({ordinal:t.length,heading:a===0?n:null,level:a===0?r:null,text:c})}o=[]};for(let s of e)s.heading!==null?(i(),n=s.heading,r=s.level):o.length===0&&(n=null,r=null),o.push(s.text);return i(),t}async function ln(e){let t=await rn({buffer:Buffer.from(e)}),n=sn(t.value);return{sections:n.some(o=>o.heading!==null)?cn(n):We(n.map(o=>o.text)).map((o,i)=>({ordinal:i,heading:null,level:null,text:o})),versionLabel:null,mediaType:Ne}}var Ne,Ke,Ze=w(()=>{"use strict";Re();Ne="application/vnd.openxmlformats-officedocument.wordprocessingml.document";Ke={name:"docx",version:"1.13.0",mediaTypes:[Ne],supports:on,extract:ln}});var b,D=w(()=>{"use strict";b=class extends Error{code;constructor(t,n){super(n),this.name="ResourceLimitError",this.code=t}}});import{createRequire as dn}from"node:module";import Ae from"tesseract.js";function fn(e){if(typeof e=="string"){let o=e.split(";")[0]?.trim().toLowerCase();return qe.has(o??"")||Qe.test(e)}if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return qe.has(n.split(";")[0]?.trim().toLowerCase()??"");let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&Qe.test(r)}function mn(e){if(e.byteLength>=8&&e[0]===137&&e[1]===80&&e[2]===78&&e[3]===71&&e[4]===13&&e[5]===10&&e[6]===26&&e[7]===10)return ne;if(e.byteLength>=3&&e[0]===255&&e[1]===216&&e[2]===255)return Le;throw new Error("Unsupported image data; expected PNG or JPEG bytes.")}function pn(e,t){let n=new DataView(e.buffer,e.byteOffset,e.byteLength);if(t===ne){if(e.byteLength<24||String.fromCharCode(...e.subarray(12,16))!=="IHDR")throw new Error("Invalid PNG image header.");return{width:n.getUint32(16),height:n.getUint32(20)}}let r=2;for(;r+3<e.byteLength;){if(e[r]!==255){r+=1;continue}for(;e[r]===255;)r+=1;let o=e[r];if(r+=1,o===217||o===218)break;if(o===216||o===1||o!==void 0&&o>=208&&o<=215)continue;if(r+2>e.byteLength)break;let i=n.getUint16(r);if(i<2||r+i>e.byteLength)break;if(o!==void 0&&(o>=192&&o<=195||o>=197&&o<=199||o>=201&&o<=203||o>=205&&o<=207)){if(i<7)break;return{height:n.getUint16(r+3),width:n.getUint16(r+5)}}r+=i}throw new Error("JPEG image dimensions could not be read.")}async function gn(e){let t=mn(e),{width:n,height:r}=pn(e,t);if(n===0||r===0||n>et/r)throw new b("image_too_large",`Image exceeds MAX_IMAGE_PIXELS (${et} pixels).`);let o=await Ae.createWorker("eng",Ae.OEM.LSTM_ONLY,Oe);try{await o.setParameters({tessedit_pageseg_mode:Ae.PSM.SINGLE_BLOCK});let{data:i}=await o.recognize(Buffer.from(e)),s=i.text.replace(/\s+/g," ").trim();return{sections:s.length>0?[{ordinal:0,heading:null,level:null,text:s}]:[],versionLabel:null,mediaType:t}}finally{await o.terminate()}}var ne,Le,qe,Qe,et,tt,un,Oe,re,Ie=w(()=>{"use strict";D();ne="image/png",Le="image/jpeg",qe=new Set([ne,Le]),Qe=/\.(?:png|jpe?g)$/i,et=5e7,tt=dn(import.meta.url)("@tesseract.js-data/eng"),un=tt.langPath,Oe={langPath:un,cacheMethod:"none",gzip:tt.gzip};re={name:"image-ocr",version:"7.0.0",mediaTypes:[ne,Le],supports:fn,extract:gn}});import{createCanvas as En}from"@napi-rs/canvas";import Pe from"tesseract.js";import{getDocument as hn}from"pdfjs-dist/legacy/build/pdf.mjs";function xn(e){if(typeof e=="string")return/\.pdf$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===oe;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.pdf$/i.test(r)}async function Tn(e){let t=hn({data:Uint8Array.from(e),maxImageSize:1e7,disableFontFace:!0,useWorkerFetch:!1,verbosity:0}),n=await t.promise;try{if(n.numPages>nt)throw new b("too_many_pages",`PDF exceeds MAX_PDF_PAGES (${nt} pages).`);let r=[];for(let i=1;i<=n.numPages;i+=1){let c=(await(await n.getPage(i)).getTextContent()).items.filter(d=>"str"in d).map(d=>"str"in d?d.str:"").join(" ").replace(/\s+/g," ").trim();c.length>0&&r.push({ordinal:r.length,heading:null,level:null,text:c})}if(r.length>0)return{sections:r,versionLabel:null,mediaType:oe};if(n.numPages>100)throw new b("too_many_pages",`PDF exceeds MAX_OCR_PDF_PAGES (${100} pages) for OCR.`);let o=await Pe.createWorker("eng",Pe.OEM.LSTM_ONLY,Oe);try{await o.setParameters({tessedit_pageseg_mode:Pe.PSM.SINGLE_BLOCK});for(let i=1;i<=n.numPages;i+=1){let s=await n.getPage(i),a=s.getViewport({scale:_n}),c=Math.ceil(a.width),d=Math.ceil(a.height);if(c===0||d===0||c>5e6/d)throw new b("image_too_large",`PDF page exceeds MAX_PDF_OCR_RENDER_PIXELS (${5e6} pixels).`);let l=En(c,d),u=l.getContext("2d");await s.render({canvas:l,canvasContext:u,viewport:a}).promise;let{data:g}=await o.recognize(l.toBuffer("image/png")),m=g.text.replace(/\s+/g," ").trim();m.length>0&&r.push({ordinal:i-1,heading:`Page ${i}`,level:null,text:m})}}finally{await o.terminate()}return{sections:r,versionLabel:null,mediaType:oe}}finally{await t.destroy()}}var oe,nt,_n,it,st=w(()=>{"use strict";Ie();Re();D();oe="application/pdf",nt=2e3,_n=2;it={name:"pdf",version:"6.4.299",mediaTypes:[oe],supports:xn,extract:Tn}});function at(e){let t=e.match(/^(.*?)[-_ ]v(\d+(?:\.\d+)*)(\.[^./\\]+)?$/);return t?{prefix:t[1].replace(/[-_ ]+$/,""),version:t[2],suffix:t[3]??null}:null}var ct=w(()=>{"use strict"});function lt(e,t){if(typeof e=="string")return e;if(typeof e!="object"||e===null)return null;let n=e;for(let r of t)if(typeof n[r]=="string")return n[r];return null}function Sn(e){let t=lt(e,["mediaType","mimeType","contentType"]);if(t!==null)return bn.has(t.split(";")[0].trim().toLowerCase());let n=lt(e,["filename","name","path","extension"]);return n===null?!1:/(?:^|[\\/])[^\\/]+\.(?:txt|md)$/i.test(n)||/^\.(?:txt|md)$/i.test(n)}function wn(e){let t=e.split(`
|
|
2
5
|
`),n=[],r=null,o=/^ {0,3}(#{1,6})(?:[ \t]+|$)(.*)$/;if(!t.some(s=>o.test(s.replace(/\r$/,""))))return e.length===0?[]:[{ordinal:0,heading:null,level:null,text:e}];let i=()=>{r&&n.push({ordinal:n.length,heading:r.heading,level:r.level,text:r.lines.join(`
|
|
3
|
-
`).trim()})};for(let s of t){let a=o.exec(s.replace(/\r$/,""));if(!a){r?r.lines.push(s):s.length>0&&(r={heading:null,level:null,lines:[s]});continue}i(),r={heading:a[2].replace(/[ \t]+#+[ \t]*$/,"").trim(),level:a[1].length,lines:[]}}return i(),n}async function _n(e){let t=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(e),n=t.startsWith("\uFEFF")?t.slice(1):t,r=hn(n),o=r.some(s=>s.heading!==null),i=r.find(s=>s.heading!==null&&rt(s.heading)!==null)?.heading??null;return{sections:r,versionLabel:i,mediaType:o?"text/markdown":n.length>0?"text/plain":null}}var gn,Oe,st=w(()=>{"use strict";ot();gn=new Set(["text/plain","text/markdown"]);Oe={name:"plain-text",version:"1.0.0",supports:En,extract:_n}});function xn(e,t){let n=Array.from(e,i=>i.codePointAt(0)),r=Array.from(t,i=>i.codePointAt(0)),o=Math.min(n.length,r.length);for(let i=0;i<o;i+=1){let s=n[i],a=r[i];if(s!==a)return s-a}return n.length-r.length}function Ie(e,t){if(e===null)return"null";if(typeof e=="string")return JSON.stringify(e);if(typeof e=="boolean")return e?"true":"false";if(typeof e=="number"){if(!Number.isFinite(e))throw new D;return JSON.stringify(e)}if(typeof e=="bigint")return e<=BigInt(Number.MAX_SAFE_INTEGER)&&e>=BigInt(Number.MIN_SAFE_INTEGER)?JSON.stringify(Number(e)):JSON.stringify(e.toString());if(Array.isArray(e)){if(t.has(e))throw new TypeError("Structured content must be acyclic.");t.add(e);try{return`[${e.map(n=>n===void 0?"null":Ie(n,t)).join(",")}]`}finally{t.delete(e)}}if(typeof e=="object"){let n=e;if(t.has(n))throw new TypeError("Structured content must be acyclic.");t.add(n);try{return`{${Object.keys(n).filter(o=>n[o]!==void 0).sort(xn).map(o=>`${JSON.stringify(o)}:${Ie(n[o],t)}`).join(",")}}`}finally{t.delete(n)}}return"null"}function oe(e){return Ie(e,new Set)}var D,at=w(()=>{"use strict";D=class extends Error{code="non_finite_number";constructor(){super("Canonical structured content contains a non-finite number."),this.name="NonFiniteNumberCanonicalizationError"}}});import{parseDocument as Tn}from"yaml";function ct(e,t){if(typeof e=="string")return e;if(typeof e!="object"||e===null)return null;let n=e;for(let r of t)if(typeof n[r]=="string")return n[r];return null}function yn(e,t){if(typeof e!="string"){let o=ct(e,["mediaType","mimeType","contentType"]);if(o!==null)return t.mediaTypes.includes(o.split(";")[0].trim().toLowerCase())}let n=typeof e=="string"?e:ct(e,["filename","name","path","extension"]);if(n===null)return!1;let r=n.toLowerCase();return t.extensions.some(o=>r.endsWith(o))}function dt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function ut(e){return JSON.parse(e,(t,n,r)=>{let o=r?.source;return typeof n=="number"&&!Number.isSafeInteger(n)&&o!==void 0&&/^-?(?:0|[1-9][0-9]*)$/.test(o)?BigInt(o):n})}function Pe(e){try{return{canonicalContent:oe(e)}}catch(t){if(t instanceof D)return{canonicalizationError:t.code};throw t}}function bn(e){let t;try{t=ut(e)}catch{throw new U("extraction_invalid_json")}return{sections:typeof t=="object"&&t!==null&&!Array.isArray(t)?Object.entries(t).map(([r,o],i)=>({ordinal:i,heading:r,level:1,text:dt(o)})):[{ordinal:0,heading:null,level:null,text:dt(t)}],...Pe(t)}}function Sn(e){let t=[],n=[];for(let r of e.split(/\r?\n/)){let o=r.trim();if(o.length!==0){try{n.push(ut(o))}catch{throw new U("extraction_invalid_json")}t.push({ordinal:t.length,heading:null,level:null,text:o})}}return{sections:t,...Pe(n)}}function j(e,t=new Set){if(typeof e!="object"||e===null)return e;if(t.has(e))throw new U("extraction_invalid_yaml");t.add(e);try{if(e instanceof Map){let r=Object.create(null);for(let[o,i]of e){let s=typeof o=="string"?o:oe(j(o,t));r[s]=j(i,t)}return r}if(Array.isArray(e))return e.map(r=>j(r,t));let n=Object.create(null);for(let[r,o]of Object.entries(e))n[r]=j(o,t);return n}finally{t.delete(e)}}function lt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function wn(e){let t=e.contents,n=new Map;for(let r of t?.items??[]){let o=r.key?.range?.[0];if(o===void 0)continue;let i=r.key&&Object.hasOwn(r.key,"value")?r.key.value:r.key?.toJSON?.(),s=j(i),a=typeof s=="string"?s:oe(s);n.set(a,o)}return n}function Rn(e){let t,n,r;try{let i=Tn(e,{schema:"core",uniqueKeys:!1,intAsBigInt:!0});if(i.errors.length>0)throw new Error("invalid YAML document");t=j(i.toJS({mapAsMap:!0})),n=wn(i)}catch(i){if(!(i instanceof D))throw new U("extraction_invalid_yaml");r=i.code,t=null,n=new Map}if(r!==void 0)return{sections:[{ordinal:0,heading:null,level:null,text:e.trim()}],canonicalizationError:r};let o;if(typeof t=="object"&&t!==null&&!Array.isArray(t)){let i=Object.entries(t).map(([s,a])=>({heading:s,value:a,start:n.get(s)})).filter(s=>s.start!==void 0).sort((s,a)=>s.start-a.start);o=i.map(({heading:s,start:a},c)=>({ordinal:c,heading:s,level:1,text:e.slice(a,i[c+1]?.start??e.length).trim()})),o.length===0&&(o=[{ordinal:0,heading:null,level:null,text:lt(t)}])}else o=[{ordinal:0,heading:null,level:null,text:lt(t)}];return{sections:o,...Pe(t)}}function Xe(e){return{name:e.name,version:"1.0.0",mediaTypes:e.mediaTypes,supports(t){return yn(t,e)},async extract(t){let n=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(t),r=n.startsWith("\uFEFF")?n.slice(1):n;if(r.length===0)return{sections:[],versionLabel:null,mediaType:null};let o;switch(e.kind){case"json":o=bn(r);break;case"jsonl":o=Sn(r);break;case"yaml":o=Rn(r);break}return{sections:o.sections,versionLabel:null,mediaType:e.mediaTypes[0]??null,...o.canonicalContent===void 0?{}:{canonicalContent:o.canonicalContent},...o.canonicalizationError===void 0?{}:{canonicalizationError:o.canonicalizationError}}}}}var U,ft,mt,Ce,pt=w(()=>{"use strict";at();U=class extends Error{code;constructor(t){super(t==="extraction_invalid_json"?"Structured JSON is invalid.":"Structured YAML is invalid."),this.name="StructuredTextExtractionError",this.code=t}};ft=Xe({kind:"json",name:"json",extensions:[".json"],mediaTypes:["application/json"]}),mt=Xe({kind:"jsonl",name:"jsonl",extensions:[".jsonl"],mediaTypes:["application/x-ndjson"]}),Ce=Xe({kind:"yaml",name:"yaml",extensions:[".yaml",".yml"],mediaTypes:["application/yaml"]})});import{PassThrough as Nn}from"node:stream";import{StringDecoder as An}from"node:string_decoder";import Ln from"jszip";function On(e){if(typeof e=="string")return/\.pptx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===ve;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.pptx$/i.test(r)}function In(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10);return String.fromCodePoint(r)}}})}function gt(e){return Number(e.match(/\/slide(\d+)\.xml$/)?.[1]??Number.MAX_SAFE_INTEGER)}function x(e){throw new b("archive_too_large",e)}function Pn(e,t,n,r){t>n&&x(`PPTX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&x(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}function F(e,t,n){t+8>n&&x("PPTX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&x("PPTX ZIP64 metadata exceeds safe bounds."),Number(r)}function H(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Xn(e,t){let n=F(e,t+8,t+20),r=o=>{if(!H(e,o,101075792)||o+12>t)return!1;let i=F(e,o+4,t);return i>=44&&o+12+i===t};if(r(n))return n;for(let o=t-56;o>=0;o-=1)if(r(o))return o;x("PPTX ZIP64 end record cannot be located safely.")}function Cn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),l=e.getUint16(s+2,!0),d=s+4,u=d+l;if((u>a||u>n)&&x("PPTX central-directory extra field is truncated."),c===1)return F(e,d,u);s=u}x("PPTX ZIP64 entry size is missing from its central-directory extra field.")}function vn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let p=e.byteLength-22;p>=n;p-=1)if(H(t,p,101010256)&&p+22+t.getUint16(p+20,!0)===e.byteLength){r=p;break}if(r<0)return new Map;let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=H(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&x("PPTX ZIP64 end record is missing.");let d=o,u=i,g=s,E=r;if(c){let p=Xn(t,a),R=a;d=F(t,p+32,R),u=F(t,p+40,R),g=F(t,p+48,R),E=p}d>L&&x(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>E||g>E||u+g>E)&&x("PPTX central-directory bounds cannot be checked safely.");let h=E-u-g,f=g+h,m=f+u,y=0,A=0,z=new Map;for(;f+4<=m&&H(t,f,33639248);){f+46>m&&x("PPTX central-directory bounds cannot be checked safely.");let p=t.getUint16(f+28,!0),R=t.getUint16(f+30,!0),he=t.getUint16(f+32,!0),G=f+46+p+R+he;G>m&&x("PPTX central-directory entry is truncated."),y+=1,y>L&&x(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let X=Cn(t,f,m,p,R),_e=new TextDecoder().decode(e.subarray(f+46,f+46+p));z.set(_e,X),X>O-A&&x(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=X,f=G}if(f<m&&H(t,f,84233040)&&f+6<=m){let p=t.getUint16(f+4,!0);f+6+p===m&&(f=m)}return(f!==m||y!==d)&&x("PPTX central-directory count or bounds cannot be checked safely."),z}async function kn(e,t,n){let r=[],o=new An("utf8"),i=new Nn,s=e.nodeStream("nodebuffer");s.on("error",c=>i.destroy(c)),s.pipe(i);let a=0;try{for await(let c of i){let l=Buffer.isBuffer(c)?c:Buffer.from(c);a+=l.byteLength,Pn(e.name,a,n,t+a),r.push(o.write(l))}}catch(c){throw c instanceof Error&&c.message.includes("uncompressed data size mismatch")&&x(`PPTX entry ${e.name} decompressed size differs from its declared size (${n} bytes).`),c}return r.push(o.end()),{xml:r.join(""),bytesRead:a}}async function Mn(e){let t=vn(e),r=(await Ln.loadAsync(e)).file(/^ppt\/slides\/slide\d+\.xml$/).sort((s,a)=>gt(s.name)-gt(a.name)),o=[],i=0;for(let s of r){let a=t.get(s.name);a===void 0&&x(`PPTX central-directory size for ${s.name} cannot be checked safely.`);let c=await kn(s,i,a);i+=c.bytesRead;let l=[...c.xml.matchAll(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/g)].map(d=>In(d[1]??"").trim()).filter(d=>d.length>0).join(" ");l.length>0&&o.push({ordinal:o.length,heading:null,level:null,text:l})}return{sections:o,versionLabel:null,mediaType:ve}}var ve,L,O,Et,ke=w(()=>{"use strict";M();ve="application/vnd.openxmlformats-officedocument.presentationml.presentation",L=5e3,O=128*1024*1024;Et={name:"pptx",version:"3.10.2",mediaTypes:[ve],supports:On,extract:Mn}});import{posix as ht}from"node:path";import{PassThrough as Dn}from"node:stream";import{StringDecoder as jn}from"node:string_decoder";import Un from"jszip";function $n(e){if(typeof e=="string")return/\.xlsx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===Me;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.xlsx$/i.test(r)}function T(e){throw new b("sheet_too_large",e)}function _(e){throw new b("archive_too_large",e)}function bt(e){let t=/^\$?([A-Z]{1,3})\$?(\d+)$/i.exec(e);if(!t?.[1]||!t[2])return null;let n=0;for(let o of t[1].toUpperCase())n=n*26+o.charCodeAt(0)-64;let r=Number(t[2]);return Number.isSafeInteger(r)&&r>0&&n<=Fn?{row:r,column:n}:null}function Tt(e,t){(!Number.isSafeInteger(e)||e<1||!Number.isSafeInteger(t)||t<1)&&T("Worksheet contains an invalid row or column reference."),(e>Y||e*t>ae)&&T(`Worksheet exceeds MAX_SHEET_ROWS (${Y}) or MAX_SHEET_CELLS (${ae}).`)}function S(e,t){return e.match(new RegExp(`\\b${t}\\s*=\\s*[\\"']([^\\"']*)[\\"']`,"i"))?.[1]}function $(e,t,n){t+8>n&&_("XLSX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&_("XLSX ZIP64 metadata exceeds safe bounds."),Number(r)}function V(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Bn(e,t){let n=$(e,t+8,t+20),r=i=>{if(!V(e,i,101075792)||i+12>t)return!1;let s=$(e,i+4,t);return s>=44&&i+12+s===t};if(r(n))return n;let o=0;for(let i=t-56;i>=0;i-=1)if(o>=xt&&_(`XLSX ZIP64 recovery scan exceeds MAX_ZIP64_RECOVERY_SCAN_ITERATIONS (${xt} iterations).`),o+=1,r(i))return i;_("XLSX ZIP64 end record cannot be located safely.")}function zn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),l=e.getUint16(s+2,!0),d=s+4,u=d+l;if((u>a||u>n)&&_("XLSX central-directory extra field is truncated."),c===1)return $(e,d,u);s=u}_("XLSX ZIP64 entry size is missing from its central-directory extra field.")}function Gn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let p=e.byteLength-22;p>=n;p-=1)if(V(t,p,101010256)&&p+22+t.getUint16(p+20,!0)===e.byteLength){r=p;break}r<0&&_("XLSX end-of-central-directory record is missing.");let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=V(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&_("XLSX ZIP64 end record is missing.");let d=o,u=i,g=s,E=r;if(c){let p=Bn(t,a),R=a;d=$(t,p+32,R),u=$(t,p+40,R),g=$(t,p+48,R),E=p}d>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>E||g>E||u+g>E)&&_("XLSX central-directory bounds cannot be checked safely.");let h=E-u-g,f=g+h,m=f+u,y=0,A=0,z=new Map;for(;f+4<=m&&V(t,f,33639248);){f+46>m&&_("XLSX central-directory bounds cannot be checked safely.");let p=t.getUint16(f+28,!0),R=t.getUint16(f+30,!0),he=t.getUint16(f+32,!0),G=f+46+p+R+he;G>m&&_("XLSX central-directory entry is truncated."),y+=1,y>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let X=zn(t,f,m,p,R),_e=new TextDecoder().decode(e.subarray(f+46,f+46+p));z.set(_e,X),X>O-A&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=X,f=G}if(f<m&&V(t,f,84233040)&&f+6<=m){let p=t.getUint16(f+4,!0);f+6+p===m&&(f=m)}return(f!==m||y!==d)&&_("XLSX central-directory count or bounds cannot be checked safely."),z}function Hn(e,t,n,r){t>n&&_(`XLSX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}async function St(e,t,n,r){let o=[],i=new jn("utf8"),s=new Dn,a=e.nodeStream("nodebuffer");a.on("error",d=>s.destroy(d)),a.pipe(s);let c=0;try{for await(let d of s){let u=Buffer.isBuffer(d)?d:Buffer.from(d);c+=u.byteLength,Hn(e.name,c,t,n.bytes+c);let g=i.write(u);o.push(g),r?.(g,u.byteLength)}}catch(d){throw d instanceof Error&&d.message.includes("uncompressed data size mismatch")&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),d}let l=i.end();return o.push(l),r?.(l,0),c!==t&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),n.bytes+=c,o.join("")}function Vn(e,t,n,r){n.bytes+=t,n.bytes>_t&&T(`Worksheet XML scan exceeds MAX_SHEET_SCAN_BYTES (${_t} bytes).`);let o=r.pending+e,i=0;for(r.pending="";;){let s=o.indexOf("<",i);if(s<0)break;let a=o.indexOf(">",s+1);if(a<0){r.pending=o.slice(s),Buffer.byteLength(r.pending)>ie&&T(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${ie} bytes).`);break}let c=o.slice(s,a+1);if(i=a+1,Buffer.byteLength(c)>ie&&T(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${ie} bytes).`),/^<(?:[\w.-]+:)?row\b/i.test(c)){r.rowCount+=1;let l=S(c,"r");l!==void 0&&!/^\d+$/.test(l)&&T("Worksheet contains an invalid row reference.");let d=Number(l??r.rowCount);(!Number.isSafeInteger(d)||d<1)&&T("Worksheet contains an invalid row reference."),(d>Y||r.rowCount>Y)&&T(`Worksheet exceeds MAX_SHEET_ROWS (${Y}).`),r.maxRow=Math.max(r.maxRow,d),Tt(r.maxRow,r.maxColumn)}else if(/^<(?:[\w.-]+:)?c\b/i.test(c)){r.cellCount+=1,r.cellCount>ae&&T(`Worksheet exceeds MAX_SHEET_CELLS (${ae}).`);let l=S(c,"r"),d=l===void 0?null:bt(l);l!==void 0&&!d&&T("Worksheet contains an invalid cell reference."),d&&(r.maxRow=Math.max(r.maxRow,d.row),r.maxColumn=Math.max(r.maxColumn,d.column),Tt(r.maxRow,r.maxColumn))}}}function W(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10),o=r===9||r===10||r===13||r>=32&&r<=55295||r>=57344&&r<=65533||r>=65536&&r<=1114111;return(!Number.isInteger(r)||!o)&&T("XLSX contains an invalid XML character reference."),String.fromCodePoint(r)}}})}function wt(e){return[...e.replace(/<(?:[\w.-]+:)?rPh\b[\s\S]*?<\/(?:[\w.-]+:)?rPh\s*>/gi,"").matchAll(/<(?:[\w.-]+:)?t\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?t\s*>/gi)].map(n=>W(n[1]??"")).join("")}function Yn(e){return[...e.matchAll(/<(?:[\w.-]+:)?si\b[^>]*\/>|<(?:[\w.-]+:)?si\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?si\s*>/gi)].map(t=>t[0].endsWith("/>")?"":wt(t[1]??""))}function Jn(e){let t;try{t=decodeURIComponent(W(e))}catch(o){if(o instanceof b)throw o;return null}let n=t.split(/[?#]/,1)[0]??"",r=ht.normalize(n.startsWith("/")?n.slice(1):ht.join("xl",n));return r.startsWith("xl/")?r:null}function Wn(e){let t=new Map;for(let n of e.matchAll(/<(?:[\w.-]+:)?Relationship\b[^>]*\/?>/gi)){let r=n[0];if(!/\/worksheet$/i.test(S(r,"Type")??"")||/^external$/i.test(S(r,"TargetMode")??""))continue;let o=S(r,"Id"),i=S(r,"Target"),s=i?Jn(i):null;o&&s&&t.set(o,s)}return t}function Kn(e,t){let n=[];for(let r of e.matchAll(/<(?:[\w.-]+:)?sheet\b[^>]*\/?>/gi)){let o=r[0],i=S(o,"name"),s=S(o,"r:id")??S(o,"id"),a=s?t.get(s):void 0;(i===void 0||a===void 0)&&_("XLSX workbook contains an unresolved worksheet relationship."),n.push({name:W(i),part:a})}return n}function Zn(e){let t=e.replace(/"(?:""|[^"])*"/g,"").replace(/\\./g,"").replace(/_.|\*./g,"").replace(/\[[^\]]*\]/g,""),n=/y/i.test(t)&&/m/i.test(t)&&/d/i.test(t);return{date:n,dateTime:n&&(/h|s/i.test(t)||/am\/pm|\[(?:h+|s+)\]/i.test(e))}}function qn(e){let t=new Map;for(let r of e.matchAll(/<(?:[\w.-]+:)?numFmt\b[^>]*\/?\s*>/gi)){let o=Number(S(r[0],"numFmtId")),i=S(r[0],"formatCode");Number.isSafeInteger(o)&&i!==void 0&&t.set(o,W(i))}return[...(e.match(/<(?:[\w.-]+:)?cellXfs\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?cellXfs\s*>/i)?.[1]??"").matchAll(/<(?:[\w.-]+:)?xf\b[^>]*\/?\s*>/gi)].map(r=>{let o=Number(S(r[0],"numFmtId")??"0");return[14,15,16,17].includes(o)?{date:!0,dateTime:!1}:o===22?{date:!0,dateTime:!0}:Zn(t.get(o)??"")})}function Qn(e,t){let n=Number(e);(e.trim()===""||!Number.isFinite(n))&&T("Worksheet contains an invalid date serial.");let r=Math.floor(n),o=n-r,i=r>=60?r-1:r,s=Date.UTC(1899,11,31)+i*864e5+Math.round(o*864e5),a=new Date(s);return Number.isFinite(a.getTime())||T("Worksheet date serial is outside the supported range."),t?a.toISOString():a.toISOString().slice(0,10)}function er(e,t,n){let r=e.indexOf(">"),o=r<0?e:e.slice(0,r+1),i=S(o,"t");if(i==="s"){let u=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1]?.trim();(u===void 0||!/^\d+$/.test(u))&&T("Worksheet contains an invalid shared-string index.");let g=Number(u);return(!Number.isSafeInteger(g)||t[g]===void 0)&&T("Worksheet shared-string index is outside the shared-strings table."),t[g]??""}if(i==="inlineStr"){let u=e.match(/<(?:[\w.-]+:)?is\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?is\s*>/i)?.[1]??"";return wt(u).trim()}let s=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1];if(s===void 0)return"";let a=W(s).trim(),c=S(o,"s"),l=c===void 0?0:Number(c);(!Number.isSafeInteger(l)||l<0||c!==void 0&&n[l]===void 0)&&T("Worksheet contains an invalid cell style reference.");let d=n[l];return d?.date?Qn(a,d.dateTime):a}function tr(e,t,n){let r=new Map,o=0;for(let s of e.matchAll(/<(?:[\w.-]+:)?row\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?row\s*>/gi)){o+=1;let a=s[0].slice(0,s[0].indexOf(">")+1),c=Number(S(a,"r")??o),l=1;for(let d of(s[1]??"").matchAll(/<(?:[\w.-]+:)?c\b[^>]*?(?:\/>|>([\s\S]*?)<\/(?:[\w.-]+:)?c\s*>)/gi)){let u=d[0],g=u.slice(0,u.indexOf(">")+1),E=bt(S(g,"r")??""),h=E?.column??l,f=E?.row??c;if(l=h+1,!Number.isSafeInteger(f)||f<1||!Number.isSafeInteger(h)||h<1)continue;let m=r.get(f)??new Map;m.set(h,er(u,t,n)),r.set(f,m)}}let i=[];for(let s of[...r].sort(([a],[c])=>a-c)){let a=s[1],c=0;for(let d of a.keys())c=Math.max(c,d);let l=Array.from({length:c},(d,u)=>a.get(u+1)??"");for(;l.length>0&&l[l.length-1]==="";)l.pop();l.some(d=>d.length>0)&&i.push(l.join(" "))}return i.join(`
|
|
4
|
-
`)}function
|
|
6
|
+
`).trim()})};for(let s of t){let a=o.exec(s.replace(/\r$/,""));if(!a){r?r.lines.push(s):s.length>0&&(r={heading:null,level:null,lines:[s]});continue}i(),r={heading:a[2].replace(/[ \t]+#+[ \t]*$/,"").trim(),level:a[1].length,lines:[]}}return i(),n}async function Rn(e){let t=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(e),n=t.startsWith("\uFEFF")?t.slice(1):t,r=wn(n),o=r.some(s=>s.heading!==null),i=r.find(s=>s.heading!==null&&at(s.heading)!==null)?.heading??null;return{sections:r,versionLabel:i,mediaType:o?"text/markdown":n.length>0?"text/plain":null}}var bn,Ce,dt=w(()=>{"use strict";ct();bn=new Set(["text/plain","text/markdown"]);Ce={name:"plain-text",version:"1.0.0",supports:Sn,extract:Rn}});function Nn(e,t){let n=Array.from(e,i=>i.codePointAt(0)),r=Array.from(t,i=>i.codePointAt(0)),o=Math.min(n.length,r.length);for(let i=0;i<o;i+=1){let s=n[i],a=r[i];if(s!==a)return s-a}return n.length-r.length}function Xe(e,t){if(e===null)return"null";if(typeof e=="string")return JSON.stringify(e);if(typeof e=="boolean")return e?"true":"false";if(typeof e=="number"){if(!Number.isFinite(e))throw new j;return JSON.stringify(e)}if(typeof e=="bigint")return e<=BigInt(Number.MAX_SAFE_INTEGER)&&e>=BigInt(Number.MIN_SAFE_INTEGER)?JSON.stringify(Number(e)):JSON.stringify(e.toString());if(Array.isArray(e)){if(t.has(e))throw new TypeError("Structured content must be acyclic.");t.add(e);try{return`[${e.map(n=>n===void 0?"null":Xe(n,t)).join(",")}]`}finally{t.delete(e)}}if(typeof e=="object"){let n=e;if(t.has(n))throw new TypeError("Structured content must be acyclic.");t.add(n);try{return`{${Object.keys(n).filter(o=>n[o]!==void 0).sort(Nn).map(o=>`${JSON.stringify(o)}:${Xe(n[o],t)}`).join(",")}}`}finally{t.delete(n)}}return"null"}function ie(e){return Xe(e,new Set)}var j,ut=w(()=>{"use strict";j=class extends Error{code="non_finite_number";constructor(){super("Canonical structured content contains a non-finite number."),this.name="NonFiniteNumberCanonicalizationError"}}});import{parseDocument as An}from"yaml";function ft(e,t){if(typeof e=="string")return e;if(typeof e!="object"||e===null)return null;let n=e;for(let r of t)if(typeof n[r]=="string")return n[r];return null}function Ln(e,t){if(typeof e!="string"){let o=ft(e,["mediaType","mimeType","contentType"]);if(o!==null)return t.mediaTypes.includes(o.split(";")[0].trim().toLowerCase())}let n=typeof e=="string"?e:ft(e,["filename","name","path","extension"]);if(n===null)return!1;let r=n.toLowerCase();return t.extensions.some(o=>r.endsWith(o))}function mt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function gt(e){return JSON.parse(e,(t,n,r)=>{let o=r?.source;return typeof n=="number"&&!Number.isSafeInteger(n)&&o!==void 0&&/^-?(?:0|[1-9][0-9]*)$/.test(o)?BigInt(o):n})}function ke(e){try{return{canonicalContent:ie(e)}}catch(t){if(t instanceof j)return{canonicalizationError:t.code};throw t}}function On(e){let t;try{t=gt(e)}catch{throw new F("extraction_invalid_json")}return{sections:typeof t=="object"&&t!==null&&!Array.isArray(t)?Object.entries(t).map(([r,o],i)=>({ordinal:i,heading:r,level:1,text:mt(o)})):[{ordinal:0,heading:null,level:null,text:mt(t)}],...ke(t)}}function In(e){let t=[],n=[];for(let r of e.split(/\r?\n/)){let o=r.trim();if(o.length!==0){try{n.push(gt(o))}catch{throw new F("extraction_invalid_json")}t.push({ordinal:t.length,heading:null,level:null,text:o})}}return{sections:t,...ke(n)}}function U(e,t=new Set){if(typeof e!="object"||e===null)return e;if(t.has(e))throw new F("extraction_invalid_yaml");t.add(e);try{if(e instanceof Map){let r=Object.create(null);for(let[o,i]of e){let s=typeof o=="string"?o:ie(U(o,t));r[s]=U(i,t)}return r}if(Array.isArray(e))return e.map(r=>U(r,t));let n=Object.create(null);for(let[r,o]of Object.entries(e))n[r]=U(o,t);return n}finally{t.delete(e)}}function pt(e){return typeof e=="string"?e:JSON.stringify(e,(t,n)=>typeof n=="bigint"?n.toString():n,2)??"null"}function Pn(e){let t=e.contents,n=new Map;for(let r of t?.items??[]){let o=r.key?.range?.[0];if(o===void 0)continue;let i=r.key&&Object.hasOwn(r.key,"value")?r.key.value:r.key?.toJSON?.(),s=U(i),a=typeof s=="string"?s:ie(s);n.set(a,o)}return n}function Cn(e){let t,n,r;try{let i=An(e,{schema:"core",uniqueKeys:!1,intAsBigInt:!0});if(i.errors.length>0)throw new Error("invalid YAML document");t=U(i.toJS({mapAsMap:!0})),n=Pn(i)}catch(i){if(!(i instanceof j))throw new F("extraction_invalid_yaml");r=i.code,t=null,n=new Map}if(r!==void 0)return{sections:[{ordinal:0,heading:null,level:null,text:e.trim()}],canonicalizationError:r};let o;if(typeof t=="object"&&t!==null&&!Array.isArray(t)){let i=Object.entries(t).map(([s,a])=>({heading:s,value:a,start:n.get(s)})).filter(s=>s.start!==void 0).sort((s,a)=>s.start-a.start);o=i.map(({heading:s,start:a},c)=>({ordinal:c,heading:s,level:1,text:e.slice(a,i[c+1]?.start??e.length).trim()})),o.length===0&&(o=[{ordinal:0,heading:null,level:null,text:pt(t)}])}else o=[{ordinal:0,heading:null,level:null,text:pt(t)}];return{sections:o,...ke(t)}}function ve(e){return{name:e.name,version:"1.0.0",mediaTypes:e.mediaTypes,supports(t){return Ln(t,e)},async extract(t){let n=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}).decode(t),r=n.startsWith("\uFEFF")?n.slice(1):n;if(r.length===0)return{sections:[],versionLabel:null,mediaType:null};let o;switch(e.kind){case"json":o=On(r);break;case"jsonl":o=In(r);break;case"yaml":o=Cn(r);break}return{sections:o.sections,versionLabel:null,mediaType:e.mediaTypes[0]??null,...o.canonicalContent===void 0?{}:{canonicalContent:o.canonicalContent},...o.canonicalizationError===void 0?{}:{canonicalizationError:o.canonicalizationError}}}}}var F,Et,ht,Me,_t=w(()=>{"use strict";ut();F=class extends Error{code;constructor(t){super(t==="extraction_invalid_json"?"Structured JSON is invalid.":"Structured YAML is invalid."),this.name="StructuredTextExtractionError",this.code=t}};Et=ve({kind:"json",name:"json",extensions:[".json"],mediaTypes:["application/json"]}),ht=ve({kind:"jsonl",name:"jsonl",extensions:[".jsonl"],mediaTypes:["application/x-ndjson"]}),Me=ve({kind:"yaml",name:"yaml",extensions:[".yaml",".yml"],mediaTypes:["application/yaml"]})});import{PassThrough as Xn}from"node:stream";import{StringDecoder as kn}from"node:string_decoder";import vn from"jszip";function Mn(e){if(typeof e=="string")return/\.pptx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===De;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.pptx$/i.test(r)}function Dn(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10);return String.fromCodePoint(r)}}})}function xt(e){return Number(e.match(/\/slide(\d+)\.xml$/)?.[1]??Number.MAX_SAFE_INTEGER)}function T(e){throw new b("archive_too_large",e)}function jn(e,t,n,r){t>n&&T(`PPTX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&T(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}function B(e,t,n){t+8>n&&T("PPTX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&T("PPTX ZIP64 metadata exceeds safe bounds."),Number(r)}function V(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Un(e,t){let n=B(e,t+8,t+20),r=o=>{if(!V(e,o,101075792)||o+12>t)return!1;let i=B(e,o+4,t);return i>=44&&o+12+i===t};if(r(n))return n;for(let o=t-56;o>=0;o-=1)if(r(o))return o;T("PPTX ZIP64 end record cannot be located safely.")}function Fn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),d=e.getUint16(s+2,!0),l=s+4,u=l+d;if((u>a||u>n)&&T("PPTX central-directory extra field is truncated."),c===1)return B(e,l,u);s=u}T("PPTX ZIP64 entry size is missing from its central-directory extra field.")}function Bn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let E=e.byteLength-22;E>=n;E-=1)if(V(t,E,101010256)&&E+22+t.getUint16(E+20,!0)===e.byteLength){r=E;break}if(r<0)return new Map;let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=V(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&T("PPTX ZIP64 end record is missing.");let l=o,u=i,g=s,m=r;if(c){let E=Un(t,a),R=a;l=B(t,E+32,R),u=B(t,E+40,R),g=B(t,E+48,R),m=E}l>L&&T(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>m||g>m||u+g>m)&&T("PPTX central-directory bounds cannot be checked safely.");let h=m-u-g,f=g+h,p=f+u,x=0,A=0,H=new Map;for(;f+4<=p&&V(t,f,33639248);){f+46>p&&T("PPTX central-directory bounds cannot be checked safely.");let E=t.getUint16(f+28,!0),R=t.getUint16(f+30,!0),_e=t.getUint16(f+32,!0),G=f+46+E+R+_e;G>p&&T("PPTX central-directory entry is truncated."),x+=1,x>L&&T(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let C=Fn(t,f,p,E,R),xe=new TextDecoder().decode(e.subarray(f+46,f+46+E));H.set(xe,C),C>O-A&&T(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=C,f=G}if(f<p&&V(t,f,84233040)&&f+6<=p){let E=t.getUint16(f+4,!0);f+6+E===p&&(f=p)}return(f!==p||x!==l)&&T("PPTX central-directory count or bounds cannot be checked safely."),H}async function $n(e,t,n){let r=[],o=new kn("utf8"),i=new Xn,s=e.nodeStream("nodebuffer");s.on("error",c=>i.destroy(c)),s.pipe(i);let a=0;try{for await(let c of i){let d=Buffer.isBuffer(c)?c:Buffer.from(c);a+=d.byteLength,jn(e.name,a,n,t+a),r.push(o.write(d))}}catch(c){throw c instanceof Error&&c.message.includes("uncompressed data size mismatch")&&T(`PPTX entry ${e.name} decompressed size differs from its declared size (${n} bytes).`),c}return r.push(o.end()),{xml:r.join(""),bytesRead:a}}async function zn(e){let t=Bn(e),r=(await vn.loadAsync(e)).file(/^ppt\/slides\/slide\d+\.xml$/).sort((s,a)=>xt(s.name)-xt(a.name)),o=[],i=0;for(let s of r){let a=t.get(s.name);a===void 0&&T(`PPTX central-directory size for ${s.name} cannot be checked safely.`);let c=await $n(s,i,a);i+=c.bytesRead;let d=[...c.xml.matchAll(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/g)].map(l=>Dn(l[1]??"").trim()).filter(l=>l.length>0).join(" ");d.length>0&&o.push({ordinal:o.length,heading:null,level:null,text:d})}return{sections:o,versionLabel:null,mediaType:De}}var De,L,O,Tt,je=w(()=>{"use strict";D();De="application/vnd.openxmlformats-officedocument.presentationml.presentation",L=5e3,O=128*1024*1024;Tt={name:"pptx",version:"3.10.2",mediaTypes:[De],supports:Mn,extract:zn}});import{posix as yt}from"node:path";import{PassThrough as Hn}from"node:stream";import{StringDecoder as Gn}from"node:string_decoder";import Vn from"jszip";function Jn(e){if(typeof e=="string")return/\.xlsx$/i.test(e);if(typeof e!="object"||e===null)return!1;let t=e,n=t.mediaType??t.mimeType??t.contentType;if(typeof n=="string")return n.split(";")[0].trim().toLowerCase()===Ue;let r=t.filename??t.name??t.path??t.extension;return typeof r=="string"&&/(?:^|[\\/])[^\\/]+\.xlsx$/i.test(r)}function y(e){throw new b("sheet_too_large",e)}function _(e){throw new b("archive_too_large",e)}function Nt(e){let t=/^\$?([A-Z]{1,3})\$?(\d+)$/i.exec(e);if(!t?.[1]||!t[2])return null;let n=0;for(let o of t[1].toUpperCase())n=n*26+o.charCodeAt(0)-64;let r=Number(t[2]);return Number.isSafeInteger(r)&&r>0&&n<=Yn?{row:r,column:n}:null}function wt(e,t){(!Number.isSafeInteger(e)||e<1||!Number.isSafeInteger(t)||t<1)&&y("Worksheet contains an invalid row or column reference."),(e>J||e*t>ce)&&y(`Worksheet exceeds MAX_SHEET_ROWS (${J}) or MAX_SHEET_CELLS (${ce}).`)}function S(e,t){return e.match(new RegExp(`\\b${t}\\s*=\\s*[\\"']([^\\"']*)[\\"']`,"i"))?.[1]}function $(e,t,n){t+8>n&&_("XLSX ZIP64 metadata is truncated.");let r=e.getBigUint64(t,!0);return r>BigInt(Number.MAX_SAFE_INTEGER)&&_("XLSX ZIP64 metadata exceeds safe bounds."),Number(r)}function Y(e,t,n){return t>=0&&t+4<=e.byteLength&&e.getUint32(t,!0)===n}function Wn(e,t){let n=$(e,t+8,t+20),r=i=>{if(!Y(e,i,101075792)||i+12>t)return!1;let s=$(e,i+4,t);return s>=44&&i+12+s===t};if(r(n))return n;let o=0;for(let i=t-56;i>=0;i-=1)if(o>=St&&_(`XLSX ZIP64 recovery scan exceeds MAX_ZIP64_RECOVERY_SCAN_ITERATIONS (${St} iterations).`),o+=1,r(i))return i;_("XLSX ZIP64 end record cannot be located safely.")}function Kn(e,t,n,r,o){let i=e.getUint32(t+24,!0);if(i!==4294967295)return i;let s=t+46+r,a=s+o;for(;s+4<=a;){let c=e.getUint16(s,!0),d=e.getUint16(s+2,!0),l=s+4,u=l+d;if((u>a||u>n)&&_("XLSX central-directory extra field is truncated."),c===1)return $(e,l,u);s=u}_("XLSX ZIP64 entry size is missing from its central-directory extra field.")}function Zn(e){let t=new DataView(e.buffer,e.byteOffset,e.byteLength),n=Math.max(0,e.byteLength-22-65535),r=-1;for(let E=e.byteLength-22;E>=n;E-=1)if(Y(t,E,101010256)&&E+22+t.getUint16(E+20,!0)===e.byteLength){r=E;break}r<0&&_("XLSX end-of-central-directory record is missing.");let o=t.getUint16(r+10,!0),i=t.getUint32(r+12,!0),s=t.getUint32(r+16,!0),a=r-20,c=Y(t,a,117853008);(o===65535||i===4294967295||s===4294967295)&&!c&&_("XLSX ZIP64 end record is missing.");let l=o,u=i,g=s,m=r;if(c){let E=Wn(t,a),R=a;l=$(t,E+32,R),u=$(t,E+40,R),g=$(t,E+48,R),m=E}l>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`),(u>m||g>m||u+g>m)&&_("XLSX central-directory bounds cannot be checked safely.");let h=m-u-g,f=g+h,p=f+u,x=0,A=0,H=new Map;for(;f+4<=p&&Y(t,f,33639248);){f+46>p&&_("XLSX central-directory bounds cannot be checked safely.");let E=t.getUint16(f+28,!0),R=t.getUint16(f+30,!0),_e=t.getUint16(f+32,!0),G=f+46+E+R+_e;G>p&&_("XLSX central-directory entry is truncated."),x+=1,x>L&&_(`XLSX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${L}).`);let C=Kn(t,f,p,E,R),xe=new TextDecoder().decode(e.subarray(f+46,f+46+E));H.set(xe,C),C>O-A&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`),A+=C,f=G}if(f<p&&Y(t,f,84233040)&&f+6<=p){let E=t.getUint16(f+4,!0);f+6+E===p&&(f=p)}return(f!==p||x!==l)&&_("XLSX central-directory count or bounds cannot be checked safely."),H}function qn(e,t,n,r){t>n&&_(`XLSX entry ${e} decompressed size differs from its declared size (${n} bytes).`),r>O&&_(`XLSX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${O} bytes).`)}async function At(e,t,n,r){let o=[],i=new Gn("utf8"),s=new Hn,a=e.nodeStream("nodebuffer");a.on("error",l=>s.destroy(l)),a.pipe(s);let c=0;try{for await(let l of s){let u=Buffer.isBuffer(l)?l:Buffer.from(l);c+=u.byteLength,qn(e.name,c,t,n.bytes+c);let g=i.write(u);o.push(g),r?.(g,u.byteLength)}}catch(l){throw l instanceof Error&&l.message.includes("uncompressed data size mismatch")&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),l}let d=i.end();return o.push(d),r?.(d,0),c!==t&&_(`XLSX entry ${e.name} decompressed size differs from its declared size (${t} bytes).`),n.bytes+=c,o.join("")}function Qn(e,t,n,r){n.bytes+=t,n.bytes>bt&&y(`Worksheet XML scan exceeds MAX_SHEET_SCAN_BYTES (${bt} bytes).`);let o=r.pending+e,i=0;for(r.pending="";;){let s=o.indexOf("<",i);if(s<0)break;let a=o.indexOf(">",s+1);if(a<0){r.pending=o.slice(s),Buffer.byteLength(r.pending)>se&&y(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${se} bytes).`);break}let c=o.slice(s,a+1);if(i=a+1,Buffer.byteLength(c)>se&&y(`Worksheet metadata exceeds MAX_SHEET_TAG_BYTES (${se} bytes).`),/^<(?:[\w.-]+:)?row\b/i.test(c)){r.rowCount+=1;let d=S(c,"r");d!==void 0&&!/^\d+$/.test(d)&&y("Worksheet contains an invalid row reference.");let l=Number(d??r.rowCount);(!Number.isSafeInteger(l)||l<1)&&y("Worksheet contains an invalid row reference."),(l>J||r.rowCount>J)&&y(`Worksheet exceeds MAX_SHEET_ROWS (${J}).`),r.maxRow=Math.max(r.maxRow,l),wt(r.maxRow,r.maxColumn)}else if(/^<(?:[\w.-]+:)?c\b/i.test(c)){r.cellCount+=1,r.cellCount>ce&&y(`Worksheet exceeds MAX_SHEET_CELLS (${ce}).`);let d=S(c,"r"),l=d===void 0?null:Nt(d);d!==void 0&&!l&&y("Worksheet contains an invalid cell reference."),l&&(r.maxRow=Math.max(r.maxRow,l.row),r.maxColumn=Math.max(r.maxColumn,l.column),wt(r.maxRow,r.maxColumn))}}}function K(e){return e.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi,(t,n)=>{switch(n.toLowerCase()){case"amp":return"&";case"lt":return"<";case"gt":return">";case"quot":return'"';case"apos":return"'";default:{let r=n.startsWith("#x")||n.startsWith("#X")?Number.parseInt(n.slice(2),16):Number.parseInt(n.slice(1),10),o=r===9||r===10||r===13||r>=32&&r<=55295||r>=57344&&r<=65533||r>=65536&&r<=1114111;return(!Number.isInteger(r)||!o)&&y("XLSX contains an invalid XML character reference."),String.fromCodePoint(r)}}})}function Lt(e){return[...e.replace(/<(?:[\w.-]+:)?rPh\b[\s\S]*?<\/(?:[\w.-]+:)?rPh\s*>/gi,"").matchAll(/<(?:[\w.-]+:)?t\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?t\s*>/gi)].map(n=>K(n[1]??"")).join("")}function er(e){return[...e.matchAll(/<(?:[\w.-]+:)?si\b[^>]*\/>|<(?:[\w.-]+:)?si\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?si\s*>/gi)].map(t=>t[0].endsWith("/>")?"":Lt(t[1]??""))}function tr(e){let t;try{t=decodeURIComponent(K(e))}catch(o){if(o instanceof b)throw o;return null}let n=t.split(/[?#]/,1)[0]??"",r=yt.normalize(n.startsWith("/")?n.slice(1):yt.join("xl",n));return r.startsWith("xl/")?r:null}function nr(e){let t=new Map;for(let n of e.matchAll(/<(?:[\w.-]+:)?Relationship\b[^>]*\/?>/gi)){let r=n[0];if(!/\/worksheet$/i.test(S(r,"Type")??"")||/^external$/i.test(S(r,"TargetMode")??""))continue;let o=S(r,"Id"),i=S(r,"Target"),s=i?tr(i):null;o&&s&&t.set(o,s)}return t}function rr(e,t){let n=[];for(let r of e.matchAll(/<(?:[\w.-]+:)?sheet\b[^>]*\/?>/gi)){let o=r[0],i=S(o,"name"),s=S(o,"r:id")??S(o,"id"),a=s?t.get(s):void 0;(i===void 0||a===void 0)&&_("XLSX workbook contains an unresolved worksheet relationship."),n.push({name:K(i),part:a})}return n}function or(e){let t=e.replace(/"(?:""|[^"])*"/g,"").replace(/\\./g,"").replace(/_.|\*./g,"").replace(/\[[^\]]*\]/g,""),n=/y/i.test(t)&&/m/i.test(t)&&/d/i.test(t);return{date:n,dateTime:n&&(/h|s/i.test(t)||/am\/pm|\[(?:h+|s+)\]/i.test(e))}}function ir(e){let t=new Map;for(let r of e.matchAll(/<(?:[\w.-]+:)?numFmt\b[^>]*\/?\s*>/gi)){let o=Number(S(r[0],"numFmtId")),i=S(r[0],"formatCode");Number.isSafeInteger(o)&&i!==void 0&&t.set(o,K(i))}return[...(e.match(/<(?:[\w.-]+:)?cellXfs\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?cellXfs\s*>/i)?.[1]??"").matchAll(/<(?:[\w.-]+:)?xf\b[^>]*\/?\s*>/gi)].map(r=>{let o=Number(S(r[0],"numFmtId")??"0");return[14,15,16,17].includes(o)?{date:!0,dateTime:!1}:o===22?{date:!0,dateTime:!0}:or(t.get(o)??"")})}function sr(e,t){let n=Number(e);(e.trim()===""||!Number.isFinite(n))&&y("Worksheet contains an invalid date serial.");let r=Math.floor(n),o=n-r,i=r>=60?r-1:r,s=Date.UTC(1899,11,31)+i*864e5+Math.round(o*864e5),a=new Date(s);return Number.isFinite(a.getTime())||y("Worksheet date serial is outside the supported range."),t?a.toISOString():a.toISOString().slice(0,10)}function ar(e,t,n){let r=e.indexOf(">"),o=r<0?e:e.slice(0,r+1),i=S(o,"t");if(i==="s"){let u=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1]?.trim();(u===void 0||!/^\d+$/.test(u))&&y("Worksheet contains an invalid shared-string index.");let g=Number(u);return(!Number.isSafeInteger(g)||t[g]===void 0)&&y("Worksheet shared-string index is outside the shared-strings table."),t[g]??""}if(i==="inlineStr"){let u=e.match(/<(?:[\w.-]+:)?is\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?is\s*>/i)?.[1]??"";return Lt(u).trim()}let s=e.match(/<(?:[\w.-]+:)?v\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?v\s*>/i)?.[1];if(s===void 0)return"";let a=K(s).trim(),c=S(o,"s"),d=c===void 0?0:Number(c);(!Number.isSafeInteger(d)||d<0||c!==void 0&&n[d]===void 0)&&y("Worksheet contains an invalid cell style reference.");let l=n[d];return l?.date?sr(a,l.dateTime):a}function cr(e,t,n){let r=new Map,o=0;for(let s of e.matchAll(/<(?:[\w.-]+:)?row\b[^>]*>([\s\S]*?)<\/(?:[\w.-]+:)?row\s*>/gi)){o+=1;let a=s[0].slice(0,s[0].indexOf(">")+1),c=Number(S(a,"r")??o),d=1;for(let l of(s[1]??"").matchAll(/<(?:[\w.-]+:)?c\b[^>]*?(?:\/>|>([\s\S]*?)<\/(?:[\w.-]+:)?c\s*>)/gi)){let u=l[0],g=u.slice(0,u.indexOf(">")+1),m=Nt(S(g,"r")??""),h=m?.column??d,f=m?.row??c;if(d=h+1,!Number.isSafeInteger(f)||f<1||!Number.isSafeInteger(h)||h<1)continue;let p=r.get(f)??new Map;p.set(h,ar(u,t,n)),r.set(f,p)}}let i=[];for(let s of[...r].sort(([a],[c])=>a-c)){let a=s[1],c=0;for(let l of a.keys())c=Math.max(c,l);let d=Array.from({length:c},(l,u)=>a.get(u+1)??"");for(;d.length>0&&d[d.length-1]==="";)d.pop();d.some(l=>l.length>0)&&i.push(d.join(" "))}return i.join(`
|
|
7
|
+
`)}function W(e,t){let n=Object.values(e.files).filter(r=>!r.dir&&r.name.toLowerCase()===t.toLowerCase());return n.length>1&&_(`XLSX package contains ambiguous case variants of ${t}.`),n[0]}function lr(e){return/<(?:[\w.-]+:)?c\b[^>]*\bt\s*=\s*["']s["']/i.test(e)}async function ae(e,t,n,r){let o=W(e,t);if(!o)throw new Error(`XLSX package is missing ${t}.`);let i=n.get(o.name);return i===void 0&&_(`XLSX central-directory size for ${o.name} cannot be checked safely.`),At(o,i,r)}async function Rt(e,t,n,r){let o=W(e,t);if(!o)return null;let i=n.get(o.name);i===void 0&&_(`XLSX central-directory size for ${o.name} cannot be checked safely.`);let s={bytes:0},a={pending:"",rowCount:0,cellCount:0,maxRow:0,maxColumn:1};return At(o,i,r,(c,d)=>{Qn(c,d,s,a)})}async function dr(e){try{let t=Zn(e),n=await Vn.loadAsync(e),r={bytes:0};if(!W(n,"xl/workbook.xml")){for(let[h,f]of Object.entries(n.files))!f.dir&&/^xl\/worksheets\/[^/]+\.xml$/i.test(h)&&await Rt(n,h,t,r);y("XLSX package is missing xl/workbook.xml.")}let o=await ae(n,"xl/workbook.xml",t,r),i=await ae(n,"xl/_rels/workbook.xml.rels",t,r),s=rr(o,nr(i));s.length===0&&_("XLSX workbook has no resolvable worksheets.");let a=new Map;for(let h of s){let f=h.part.toLowerCase();if(a.has(f))continue;let p=await Rt(n,h.part,t,r);p!==null&&a.set(f,p)}let c=W(n,"xl/sharedStrings.xml"),d=[...a.values()].some(lr);!c&&d&&_("XLSX worksheet references shared strings, but the shared-strings part is missing.");let l=c?er(await ae(n,c.name,t,r)):[],u=W(n,"xl/styles.xml"),g=u?ir(await ae(n,u.name,t,r)):[],m=[];for(let h of s){let f=a.get(h.part.toLowerCase());f===void 0&&_(`XLSX package is missing worksheet ${h.part}.`);let p=cr(f,l,g);p.trim().length!==0&&m.push({ordinal:m.length,heading:h.name,level:1,text:p})}return{sections:m,versionLabel:null,mediaType:Ue}}catch(t){if(t instanceof b)throw t;_("XLSX package could not be parsed safely.")}}var Ue,J,ce,se,bt,Yn,St,Ot,It=w(()=>{"use strict";je();D();Ue="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",J=1e5,ce=1e6,se=64*1024,bt=16*1024*1024,Yn=16384,St=1e6;Ot={name:"xlsx",version:"4.4.0",mediaTypes:[Ue],supports:Jn,extract:dr}});import*as Be from"node:path";function Pt(e){let t=Be.extname(e).toLowerCase(),n=ur.get(t);if(!n||!n.supports({filename:e}))throw new Fe(e);return n}var Fe,ur,Ct=w(()=>{"use strict";Ze();Ie();st();dt();_t();je();It();Fe=class extends Error{code="unsupported_media_type";extension;constructor(t){let n=Be.extname(t).toLowerCase()||"(none)";super(`Unsupported media type for extension "${n}".`),this.name="UnsupportedMediaTypeError",this.extension=n}},ur=new Map([[".pdf",it],[".docx",Ke],[".png",re],[".jpg",re],[".jpeg",re],[".xlsx",Ot],[".pptx",Tt],[".txt",Ce],[".md",Ce],[".yaml",Me],[".yml",Me],[".json",Et],[".jsonl",ht]])});import*as N from"node:fs/promises";import*as le from"node:path";function mr(e){if(!/^[0-9a-f]{64}$/.test(e)||e.includes("..")||e.includes("/")||e.includes("\\"))throw new Error("Invalid SHA-256 hash: expected 64 lowercase hexadecimal characters.");return le.join("objects",e.slice(0,2),e.slice(2,4),e)}async function Xt(e,t){return N.readFile(le.join(e,mr(t)))}var Ao,Lo,fr,Oo,kt=w(()=>{"use strict";D();Ao=64*1024,Lo=1440*60*1e3,fr=["EPERM","ENOTSUP","EXDEV","EACCES"],Oo=new Set(fr)});import*as v from"node:fs";import*as X from"node:path";var q=class extends Error{code;path;constructor(t,n,r){let o=r instanceof Error?r.message:String(r);super(n==="config_invalid"?`Invalid JSON in config file ${t}: ${o}`:`Unable to read config file ${t}: ${o}`,{cause:r}),this.name="ConfigDiscoveryError",this.code=n,this.path=t}};function Jt(e){let t;try{t=v.readFileSync(e,"utf8")}catch(n){throw new q(e,"config_unreadable",n)}try{return JSON.parse(t)}catch(n){throw new q(e,"config_invalid",n)}}function Te({cwd:e,homeDir:t,argv:n,env:r}){let o={};for(let x=0;x<n.length;x+=1){let A=/^--(client|project|root)=(.*)$/.exec(n[x]);A?o[A[1]]=A[2]:/^--(client|project|root)$/.test(n[x])&&n[x+1]!==void 0&&(o[n[x].slice(2)]=n[x+1],x+=1)}let i={client:r.RAG_CLIENT,project:r.RAG_PROJECT,root:r.RAG_ROOT},s=X.join(e,"rag.config.json"),a=X.join(X.dirname(e),"rag.config.json"),c=X.join(t,".rag","config.json"),d=v.existsSync(s)?s:v.existsSync(a)?a:v.existsSync(c)?c:null,l=d===null?{}:Jt(d),u=typeof l.client=="string"?l.client:void 0,g=typeof l.project=="string"?l.project:void 0,m=o.client??i.client??u??null,h=o.project??i.project??g??null,f=m===null&&h===null?null:{client:m,project:h},p=o.client!==void 0||o.project!==void 0?"args":i.client!==void 0||i.project!==void 0?"env":u!==void 0||g!==void 0?d===s?"cwd":d===a?"parent":"home":"unscoped";return{namespace:f,source:p,configPath:p==="cwd"||p==="parent"||p==="home"?d:null,failClosed:typeof l.failClosed=="boolean"?l.failClosed:!0,root:o.root??i.root??(typeof l.root=="string"?l.root:X.join(t,".rag"))}}Se();import*as te from"node:fs";import{DatabaseSync as Kt}from"node:sqlite";var Zt=800,we=/[\p{L}\p{M}\p{N}]+/gu;function qt(e){return e.match(we)??[]}function Qt(e){return e.map(t=>`"${t.replaceAll('"','""')}"`).join(" AND ")}function Ge(e){return e.normalize("NFD").replace(new RegExp("\\p{M}","gu"),"").toLowerCase()}function en(e,t){let n=new Set(t.map(Ge));for(let r of e.matchAll(we)){let o=r[0];if(!n.has(Ge(o)))continue;let i=r.index;return{start:i,end:i+o.length}}return null}function tn(e,t,n,r){let o=Math.min(r,e.length);if(o===0)return"";let i=Math.min(n-t,o),s=Math.max(0,Math.min(t-Math.floor((o-i)/2),e.length-o)),a=s+o,c=s,d=a,l=Array.from(e.matchAll(we),m=>({start:m.index,end:m.index+m[0].length})),u=l.find(m=>m.start<s&&s<m.end);u&&(s=l.find(m=>m.start>=u.end)?.start??e.length);let g=l.find(m=>m.start<a&&a<m.end);if(g){let m=l.filter(h=>h.end<=g.start);a=m[m.length-1]?.end??0}return a<=s?e.slice(c,d):e.slice(s,a)}function nn(e,t){return e<t?-1:e>t?1:0}function ee(e,t,n){return{namespace:e,error:{code:t,message:n}}}function Ve(e){let{root:t,namespace:n,query:r}=e,o=e.excerptLength??Zt;if(!Number.isInteger(o)||o<1)return ee(n,"invalid_excerpt_length","excerptLength must be a positive integer.");let i=qt(r);if(i.length===0)return{namespace:n,result:{citations:[],total:0}};let s;try{let a=Q(t,n.client);if(!te.existsSync(a)||!te.statSync(a).isFile())return ee(n,"database_not_found",`No database exists for namespace ${n.client}/${n.project}.`);s=new Kt(a,{readOnly:!0});let c=s.prepare(`SELECT dv.doc_version_id AS docVersionId, d.document_id AS documentId,
|
|
5
8
|
e.entity_key AS entityKey, d.label AS label, dv.version_label AS versionLabel,
|
|
6
9
|
s.ordinal AS sectionOrdinal, c.ordinal AS chunkOrdinal, c.text AS text,
|
|
7
10
|
c.char_start AS charStart, c.char_end AS charEnd, bm25(chunk_fts) AS score
|
|
@@ -15,13 +18,13 @@ var w=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(r){throw n=
|
|
|
15
18
|
LEFT JOIN section AS s ON s.section_id = c.section_id
|
|
16
19
|
WHERE chunk_fts MATCH ? AND cl.name = ? AND p.slug = ?
|
|
17
20
|
AND d.deleted_at IS NULL AND e.deleted_at IS NULL
|
|
18
|
-
ORDER BY score ASC, dv.doc_version_id COLLATE BINARY ASC, c.ordinal ASC`).all(
|
|
21
|
+
ORDER BY score ASC, dv.doc_version_id COLLATE BINARY ASC, c.ordinal ASC`).all(Qt(i),n.client,n.project),d=[];for(let l of c){let u=en(l.text,i);if(!u)return ee(n,"match_offsets_unavailable","A matched FTS token could not be located in its stored chunk text.");d.push({docVersionId:l.docVersionId,documentId:l.documentId,entityKey:l.entityKey,label:l.label,versionLabel:l.versionLabel,sectionOrdinal:l.sectionOrdinal,chunkOrdinal:l.chunkOrdinal,charStart:l.charStart,charEnd:l.charEnd,matchStart:u.start,matchEnd:u.end,score:l.score,excerpt:tn(l.text,u.start,u.end,o)})}return d.sort((l,u)=>l.score-u.score||nn(l.docVersionId,u.docVersionId)||l.chunkOrdinal-u.chunkOrdinal),{namespace:n,result:{citations:d,total:d.length}}}catch(a){let c=a instanceof Error?a.message:String(a);return ee(n,"database_unavailable",c)}finally{s?.close()}}Ct();Se();kt();import*as de from"node:fs";import*as Z from"node:path";import{DatabaseSync as pr}from"node:sqlite";function I(e,t,n){return{namespace:e,error:{code:t,message:n}}}function gr(e,t){let n=Buffer.from(e,"utf8");if(n.byteLength<=t)return{text:e,truncated:!1};let r=t;for(;r>0;)try{return{text:new TextDecoder("utf-8",{fatal:!0}).decode(n.subarray(0,r)),truncated:!0}}catch{r-=1}return{text:"",truncated:!0}}function Er(e,t){if(e.length===0)return{text:e,truncated:!1};if(t===0)return{text:"",truncated:!0};let n=1;for(let r=0;r<e.length;r+=1){if(e[r]!=="\r"&&e[r]!==`
|
|
19
22
|
`)continue;let o=r;if(e[r]==="\r"&&e[r+1]===`
|
|
20
|
-
`&&(r+=1),n===t)return{text:e.slice(0,o),truncated:!0};n+=1}return{text:e,truncated:!1}}async function
|
|
23
|
+
`&&(r+=1),n===t)return{text:e.slice(0,o),truncated:!0};n+=1}return{text:e,truncated:!1}}async function vt(e){let{root:t,namespace:n,docVersionId:r,chunkOrdinal:o,maxBytes:i,maxLines:s}=e;if(!Number.isInteger(i)||i<0||!Number.isInteger(s)||s<0)return I(n,"invalid_read_limit","maxBytes and maxLines must be non-negative integers.");let a;try{let c=Q(t,n.client);if(!de.existsSync(c)||!de.statSync(c).isFile())return I(n,"database_not_found",`No database exists for namespace ${n.client}/${n.project}.`);a=new pr(c,{readOnly:!0});let d=a.prepare(`SELECT o.sha256 AS sha256, o.rel_path AS relPath, d.label AS label
|
|
21
24
|
FROM chunk AS c
|
|
22
25
|
JOIN doc_version AS dv ON dv.doc_version_id = c.doc_version_id
|
|
23
26
|
JOIN document AS d ON d.document_id = dv.document_id
|
|
24
27
|
JOIN project AS p ON p.project_id = d.project_id
|
|
25
28
|
JOIN client AS cl ON cl.client_id = p.client_id
|
|
26
29
|
LEFT JOIN object AS o ON o.sha256 = dv.sha256
|
|
27
|
-
WHERE c.doc_version_id = ? AND c.ordinal = ? AND cl.name = ? AND p.slug = ?`).get(r,o,n.client,n.project);if(!
|
|
30
|
+
WHERE c.doc_version_id = ? AND c.ordinal = ? AND cl.name = ? AND p.slug = ?`).get(r,o,n.client,n.project);if(!d)return I(n,"citation_not_found","The citation does not exist in the requested document version and namespace.");if(!d.sha256||!d.relPath)return I(n,"object_missing","The citation's recorded object is missing.");let l;try{let p=Z.resolve(t,d.relPath),x=Z.resolve(t);if(p!==x&&!p.startsWith(`${x}${Z.sep}`))return I(n,"object_path_invalid","The recorded object path escapes the configured root.");l=await Xt(t,d.sha256)}catch(p){if(p.code==="ENOENT")return I(n,"object_missing","The citation's recorded object file is missing.");throw p}let u=await Pt(d.label).extract(l),g=u.sections[o]?.text;if(g===void 0)return I(n,"citation_not_found","The stored object does not contain the requested extracted section.");let m=gr(g,i),h=Er(m.text,s),f=u.sections.length;return{namespace:n,result:{text:h.text,truncated:m.truncated||h.truncated,totalChunks:f,nextChunkOrdinal:o+1<f?o+1:null}}}catch(c){let d=c instanceof Error?c.message:String(c);return I(n,"object_read_failed",d)}finally{a?.close()}}import{z as k}from"zod";var Mt=100,Dt=2e3,jt=1048576,Ut=1e4,hr=10,_r=800,xr=65536,Tr=200;function Ft(e){let t=new Set;for(let n of(e??"find,read").split(",").map(r=>r.trim())){if(n!=="find"&&n!=="read")throw new Error(`invalid_capability: Unknown RAG_CAPABILITIES value "${n}".`);t.add(n)}return t}var fe=k.string().min(1).refine(e=>e.trim().length>0),$e=k.object({client:fe,project:fe}).strict();function me(e,t,n){return k.number().int().min(e).max(t).optional().transform(r=>r??n)}var Bt=k.object({namespace:$e,query:fe,limit:me(1,Mt,hr),excerptChars:me(1,Dt,_r)}).strict(),$t=k.object({namespace:$e,docVersionId:fe,chunkOrdinal:k.number().multipleOf(1).min(0).refine(Number.isSafeInteger),maxBytes:me(0,jt,xr),maxLines:me(0,Ut,Tr)}).strict();function yr(e){let t=e.find(i=>i.code==="unrecognized_keys"&&i.path.length===0)??e.find(i=>i.code==="unrecognized_keys");if(!t||t.code!=="unrecognized_keys")return null;let n=t.keys[0];if(n===void 0)return null;let r=t.path.map(String).join(".");return{code:"unknown_field",message:`Unknown field "${r.length>0?`${r}.${n}`:n}".`}}function P(e,t){return e.some(n=>n.path[0]===t)}function ue(e,t,n){return{code:"invalid_limit",message:`${e} must be an integer from ${t} to ${n}.`}}function zt(e,t){let n=yr(e);if(n)return n;if(P(e,"namespace"))return{code:"namespace_required",message:"namespace must contain non-empty client and project strings."};if(t==="find"){if(P(e,"query"))return{code:"invalid_query",message:"query must be a non-empty string."};if(P(e,"limit"))return ue("limit",1,Mt);if(P(e,"excerptChars"))return ue("excerptChars",1,Dt)}else{if(P(e,"docVersionId"))return{code:"invalid_input",message:"docVersionId must be a non-empty string."};if(P(e,"chunkOrdinal"))return{code:"invalid_input",message:"chunkOrdinal must be a non-negative integer."};if(P(e,"maxBytes"))return ue("maxBytes",0,jt);if(P(e,"maxLines"))return ue("maxLines",0,Ut)}return e.some(r=>r.path.length===0)?{code:"invalid_input",message:"Tool arguments must be an object."}:{code:"invalid_input",message:"Tool arguments are invalid."}}function br(e){let t=$e.safeParse(e);return t.success?t.data:null}function Ht(e){let t=Bt.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:pe(e),error:zt(t.error.issues,"find")}}function Gt(e){let t=$t.safeParse(e);return t.success?{ok:!0,input:t.data,namespace:t.data.namespace}:{ok:!1,namespace:pe(e),error:zt(t.error.issues,"read")}}function Sr(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}function pe(e){return Sr(e)?br(e.namespace):null}function Vt(e){return k.toJSONSchema(e,{io:"input"})}var ge={name:"rag_find",description:"Search a client/project knowledge base and return bounded citations.",inputSchema:Vt(Bt)},Ee={name:"rag_read",description:"Read bounded text from a cited document chunk in the explicit namespace.",inputSchema:Vt($t)};function ze(e,t=!1){return{content:[{type:"text",text:JSON.stringify(e)}],details:e,...t?{isError:!0}:{}}}function z(e,t){return ze({namespace:e,error:t},!0)}function he(e){return z(e,{code:"cancelled",message:"Tool call was cancelled."})}function Yt(e,t){let n=t instanceof Error?t.message:String(t);return z(pe(e),{code:"retrieval_failed",message:n})}var wr="The namespace travels on every call: never infer `client` or `project` from the cwd, repo name, config, env, or a placeholder like `default`. If the user has not stated it, ask before the first call.\nUse the RAG tool the role actually exposes; when only `rag_find` is available, delegate for deeper text instead of paraphrasing a catalog entry.\nWhen an excerpt is truncated, when the user asks whether the document covers something, or when a longer verbatim quote is needed, hand off to the subagent that exposes `rag_read` with the namespace and the citation's `docVersionId`; never assert absence from a catalog.";function $o(e){e.on("before_agent_start",n=>{if(process.env.RAG_PROMPT==="off")return;let r=n.systemPromptOptions.sections??(n.systemPromptOptions.sections={});r.rag=wr});let t=Ft(process.env.RAG_CAPABILITIES);t.has("find")&&e.registerTool({name:ge.name,label:"RAG Find",description:ge.description,parameters:ge.inputSchema,async execute(n,r,o,i,s){try{let a=Ht(r);if(!a.ok)return z(a.namespace,a.error);let{namespace:c,query:d,limit:l,excerptChars:u}=a.input;if(o?.aborted)return he(c);let g=Te({cwd:s.cwd,homeDir:s.cwd,argv:[],env:process.env}).root,m=Ve({root:g,namespace:c,query:d,excerptLength:u});return o?.aborted?he(c):"error"in m?z(c,m.error):ze({namespace:c,result:{...m.result,citations:m.result.citations.slice(0,l)}})}catch(a){return Yt(r,a)}}}),t.has("read")&&e.registerTool({name:Ee.name,label:"RAG Read",description:Ee.description,parameters:Ee.inputSchema,async execute(n,r,o,i,s){try{let a=Gt(r);if(!a.ok)return z(a.namespace,a.error);let{namespace:c,docVersionId:d,chunkOrdinal:l,maxBytes:u,maxLines:g}=a.input;if(o?.aborted)return he(c);let m=Te({cwd:s.cwd,homeDir:s.cwd,argv:[],env:process.env}).root,h=await vt({root:m,namespace:c,docVersionId:d,chunkOrdinal:l,maxBytes:u,maxLines:g});return o?.aborted?he(c):"error"in h?z(c,h.error):ze({namespace:c,result:h.result})}catch(a){return Yt(r,a)}}})}export{$o as default};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { convertToHtml } from "mammoth";
|
|
2
|
+
import { MAX_SECTION_CHARS } from "../limits.mjs";
|
|
2
3
|
const DOCX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
3
4
|
function supports(ref) {
|
|
4
5
|
if (typeof ref === "string") return /\.docx$/i.test(ref);
|
|
@@ -9,14 +10,104 @@ function supports(ref) {
|
|
|
9
10
|
const fileName = fields.filename ?? fields.name ?? fields.path ?? fields.extension;
|
|
10
11
|
return typeof fileName === "string" && /(?:^|[\\/])[^\\/]+\.docx$/i.test(fileName);
|
|
11
12
|
}
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
function htmlText(value) {
|
|
14
|
+
const entities = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: "\xA0", quot: '"' };
|
|
15
|
+
return value.replace(/<br\b[^>]*>/gi, "\n").replace(/<[^>]*>/g, "").replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (entity, code) => {
|
|
16
|
+
if (code[0] !== "#") return entities[code.toLowerCase()] ?? entity;
|
|
17
|
+
const numeric = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
|
|
18
|
+
return Number.isInteger(numeric) && numeric >= 0 && numeric <= 1114111 ? String.fromCodePoint(numeric) : entity;
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
function readBlocks(html) {
|
|
22
|
+
const blocks = [];
|
|
23
|
+
const tags = /<(h([1-6])|p|li)\b[^>]*>([\s\S]*?)<\/\1\s*>/gi;
|
|
24
|
+
for (const match of html.matchAll(tags)) {
|
|
25
|
+
const level = match[2] === void 0 ? null : Number(match[2]);
|
|
26
|
+
blocks.push({ text: htmlText(match[3] ?? ""), heading: level === null ? null : htmlText(match[3] ?? ""), level });
|
|
27
|
+
}
|
|
28
|
+
return blocks.filter((block) => block.text.length > 0);
|
|
29
|
+
}
|
|
30
|
+
function isWhitespace(value) {
|
|
31
|
+
return value !== void 0 && /\s/u.test(value);
|
|
32
|
+
}
|
|
33
|
+
function splitLongParagraph(text) {
|
|
34
|
+
const sections = [];
|
|
35
|
+
let start = 0;
|
|
36
|
+
while (text.length - start > MAX_SECTION_CHARS) {
|
|
37
|
+
let boundary = Math.min(start + MAX_SECTION_CHARS, text.length - 1);
|
|
38
|
+
while (boundary > start && !isWhitespace(text[boundary])) boundary -= 1;
|
|
39
|
+
if (boundary === start) {
|
|
40
|
+
boundary = start + MAX_SECTION_CHARS + 1;
|
|
41
|
+
while (boundary < text.length && !isWhitespace(text[boundary])) boundary += 1;
|
|
42
|
+
if (boundary === text.length) break;
|
|
43
|
+
}
|
|
44
|
+
sections.push(text.slice(start, boundary));
|
|
45
|
+
start = boundary + 1;
|
|
46
|
+
}
|
|
47
|
+
if (start < text.length || sections.length === 0) sections.push(text.slice(start));
|
|
48
|
+
return sections;
|
|
49
|
+
}
|
|
50
|
+
function packParagraphs(paragraphs) {
|
|
51
|
+
const sections = [];
|
|
52
|
+
let current = null;
|
|
53
|
+
const flush = () => {
|
|
54
|
+
if (current !== null) sections.push(current);
|
|
55
|
+
current = null;
|
|
19
56
|
};
|
|
57
|
+
for (const paragraph of paragraphs) {
|
|
58
|
+
if (paragraph.length > MAX_SECTION_CHARS) {
|
|
59
|
+
flush();
|
|
60
|
+
const pieces = splitLongParagraph(paragraph);
|
|
61
|
+
sections.push(...pieces.slice(0, -1));
|
|
62
|
+
current = pieces.at(-1) ?? null;
|
|
63
|
+
if (current !== null && current.length > MAX_SECTION_CHARS) flush();
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
const candidate = current === null ? paragraph : `${current}
|
|
67
|
+
${paragraph}`;
|
|
68
|
+
if (candidate.length <= MAX_SECTION_CHARS) current = candidate;
|
|
69
|
+
else {
|
|
70
|
+
flush();
|
|
71
|
+
current = paragraph;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
flush();
|
|
75
|
+
return sections;
|
|
76
|
+
}
|
|
77
|
+
function groupByHeadings(blocks) {
|
|
78
|
+
const sections = [];
|
|
79
|
+
let heading = null;
|
|
80
|
+
let level = null;
|
|
81
|
+
let paragraphs = [];
|
|
82
|
+
const flush = () => {
|
|
83
|
+
if (paragraphs.length > 0) {
|
|
84
|
+
const packable = heading !== null && paragraphs.length > 1 ? [`${paragraphs[0]}
|
|
85
|
+
${paragraphs[1]}`, ...paragraphs.slice(2)] : paragraphs;
|
|
86
|
+
for (const [index, text] of packParagraphs(packable).entries()) {
|
|
87
|
+
sections.push({ ordinal: sections.length, heading: index === 0 ? heading : null, level: index === 0 ? level : null, text });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
paragraphs = [];
|
|
91
|
+
};
|
|
92
|
+
for (const block of blocks) {
|
|
93
|
+
if (block.heading !== null) {
|
|
94
|
+
flush();
|
|
95
|
+
heading = block.heading;
|
|
96
|
+
level = block.level;
|
|
97
|
+
} else if (paragraphs.length === 0) {
|
|
98
|
+
heading = null;
|
|
99
|
+
level = null;
|
|
100
|
+
}
|
|
101
|
+
paragraphs.push(block.text);
|
|
102
|
+
}
|
|
103
|
+
flush();
|
|
104
|
+
return sections;
|
|
105
|
+
}
|
|
106
|
+
async function extract(bytes) {
|
|
107
|
+
const result = await convertToHtml({ buffer: Buffer.from(bytes) });
|
|
108
|
+
const blocks = readBlocks(result.value);
|
|
109
|
+
const sections = blocks.some((block) => block.heading !== null) ? groupByHeadings(blocks) : packParagraphs(blocks.map((block) => block.text)).map((text, ordinal) => ({ ordinal, heading: null, level: null, text }));
|
|
110
|
+
return { sections, versionLabel: null, mediaType: DOCX_MEDIA_TYPE };
|
|
20
111
|
}
|
|
21
112
|
const docxExtractor = {
|
|
22
113
|
name: "docx",
|
package/dist/src/cli/index.mjs
CHANGED
|
@@ -11,7 +11,7 @@ import { clientDbPath, openDatabase, validateClientName } from "../store/db.mjs"
|
|
|
11
11
|
import { backupRoot, reindexRoot, restoreRoot, writeRootManifest } from "../store/backup.mjs";
|
|
12
12
|
import { readObject } from "../store/objects.mjs";
|
|
13
13
|
import { rehashNamespace } from "../store/rehash.mjs";
|
|
14
|
-
import { getActiveLimits } from "../limits.mjs";
|
|
14
|
+
import { getActiveLimits, MAX_SECTION_CHARS } from "../limits.mjs";
|
|
15
15
|
import { renderOutput, resolveOutputFormat } from "./format.mjs";
|
|
16
16
|
import {
|
|
17
17
|
linkReviewVersions,
|
|
@@ -327,10 +327,12 @@ async function verifyNamespaceObjects(root, namespace) {
|
|
|
327
327
|
let objects;
|
|
328
328
|
let chunks;
|
|
329
329
|
let indexedChunks;
|
|
330
|
+
let oversizedChunks;
|
|
330
331
|
try {
|
|
331
332
|
objects = db.prepare("SELECT sha256, byte_size FROM object ORDER BY sha256").all();
|
|
332
333
|
chunks = Number(db.prepare("SELECT COUNT(*) AS count FROM chunk").get()?.count ?? 0);
|
|
333
334
|
indexedChunks = Number(db.prepare("SELECT COUNT(*) AS count FROM chunk_fts_docsize").get()?.count ?? 0);
|
|
335
|
+
oversizedChunks = Number(db.prepare("SELECT COUNT(*) AS count FROM chunk WHERE length(text) > ?").get(MAX_SECTION_CHARS)?.count ?? 0);
|
|
334
336
|
} finally {
|
|
335
337
|
db.close();
|
|
336
338
|
}
|
|
@@ -364,6 +366,7 @@ async function verifyNamespaceObjects(root, namespace) {
|
|
|
364
366
|
corrupt,
|
|
365
367
|
limits: await getActiveLimits(),
|
|
366
368
|
...indexHealthy ? {} : { chunks, indexedChunks },
|
|
369
|
+
...oversizedChunks === 0 ? {} : { oversizedChunks },
|
|
367
370
|
// Keep healthy doctor envelopes stable; surface counts whenever artifacts need attention.
|
|
368
371
|
...artifacts.unreferencedObjects === 0 && artifacts.leftoverTemporaries === 0 ? {} : artifacts
|
|
369
372
|
}
|
package/dist/src/limits.mjs
CHANGED
|
@@ -4,6 +4,7 @@ const MAX_PDF_OCR_RENDER_PIXELS = 5e6;
|
|
|
4
4
|
const DEFAULT_PARSE_MAX_OLD_SPACE_SIZE_MB = 512;
|
|
5
5
|
const DEFAULT_PARSE_TIMEOUT_MS = 12e4;
|
|
6
6
|
const MAX_SOURCE_FILE_BYTES = 64 * 1024 * 1024;
|
|
7
|
+
const MAX_SECTION_CHARS = 8192;
|
|
7
8
|
const LIMIT_NAMES_BY_REJECTION = {
|
|
8
9
|
source_too_large: ["MAX_SOURCE_FILE_BYTES"],
|
|
9
10
|
too_many_pages: ["MAX_PDF_PAGES", "MAX_OCR_PDF_PAGES"],
|
|
@@ -27,6 +28,7 @@ async function getActiveLimits() {
|
|
|
27
28
|
]);
|
|
28
29
|
return [
|
|
29
30
|
{ name: "MAX_SOURCE_FILE_BYTES", value: MAX_SOURCE_FILE_BYTES2, unit: "bytes", reason: "Allows large manuals while bounding bytes held for hashing and extraction." },
|
|
31
|
+
{ name: "MAX_SECTION_CHARS", value: MAX_SECTION_CHARS, unit: "characters", reason: "Keeps DOCX sections within a predictable retrieval text window." },
|
|
30
32
|
{ name: "MAX_PDF_PAGES", value: MAX_PDF_PAGES, unit: "pages", reason: "Covers exceptionally long references while limiting page extraction from hostile PDFs." },
|
|
31
33
|
{ name: "MAX_OCR_PDF_PAGES", value: MAX_OCR_PDF_PAGES, unit: "pages", reason: "OCR is far slower than text extraction, so scanned PDFs use a separate page budget." },
|
|
32
34
|
{ name: "MAX_PDF_SOURCE_IMAGE_PIXELS", value: MAX_PDF_SOURCE_IMAGE_PIXELS, unit: "pixels", reason: "Pixel budget of a decoded source image, checked by the pdfjs worker before PDFImage is constructed." },
|
|
@@ -47,6 +49,7 @@ export {
|
|
|
47
49
|
MAX_OCR_PDF_PAGES,
|
|
48
50
|
MAX_PDF_OCR_RENDER_PIXELS,
|
|
49
51
|
MAX_PDF_SOURCE_IMAGE_PIXELS,
|
|
52
|
+
MAX_SECTION_CHARS,
|
|
50
53
|
MAX_SOURCE_FILE_BYTES,
|
|
51
54
|
getActiveLimits
|
|
52
55
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import { DatabaseSync } from "node:sqlite";
|
|
3
3
|
import { clientDbPath } from "../store/db.mjs";
|
|
4
|
-
const DEFAULT_EXCERPT_LENGTH =
|
|
4
|
+
const DEFAULT_EXCERPT_LENGTH = 800;
|
|
5
5
|
const WORDS = /[\p{L}\p{M}\p{N}]+/gu;
|
|
6
6
|
function tokenizeQuery(query) {
|
|
7
7
|
return query.match(WORDS) ?? [];
|
|
@@ -23,11 +23,23 @@ function matchRange(text, queryTokens) {
|
|
|
23
23
|
return null;
|
|
24
24
|
}
|
|
25
25
|
function makeExcerpt(text, matchStart, matchEnd, limit) {
|
|
26
|
-
const excerptLength = Math.min(limit,
|
|
26
|
+
const excerptLength = Math.min(limit, text.length);
|
|
27
27
|
if (excerptLength === 0) return "";
|
|
28
28
|
const matchLength = Math.min(matchEnd - matchStart, excerptLength);
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
let start = Math.max(0, Math.min(matchStart - Math.floor((excerptLength - matchLength) / 2), text.length - excerptLength));
|
|
30
|
+
let end = start + excerptLength;
|
|
31
|
+
const unsnappedStart = start;
|
|
32
|
+
const unsnappedEnd = end;
|
|
33
|
+
const words = Array.from(text.matchAll(WORDS), (word) => ({ start: word.index, end: word.index + word[0].length }));
|
|
34
|
+
const startWord = words.find((word) => word.start < start && start < word.end);
|
|
35
|
+
if (startWord) start = words.find((word) => word.start >= startWord.end)?.start ?? text.length;
|
|
36
|
+
const endWord = words.find((word) => word.start < end && end < word.end);
|
|
37
|
+
if (endWord) {
|
|
38
|
+
const previousWords = words.filter((word) => word.end <= endWord.start);
|
|
39
|
+
end = previousWords[previousWords.length - 1]?.end ?? 0;
|
|
40
|
+
}
|
|
41
|
+
if (end <= start) return text.slice(unsnappedStart, unsnappedEnd);
|
|
42
|
+
return text.slice(start, end);
|
|
31
43
|
}
|
|
32
44
|
function compareText(left, right) {
|
|
33
45
|
return left < right ? -1 : left > right ? 1 : 0;
|
|
@@ -76,9 +76,15 @@ async function readFromCitation(input) {
|
|
|
76
76
|
}
|
|
77
77
|
const byteLimited = limitUtf8(chunkText, maxBytes);
|
|
78
78
|
const lineLimited = limitLines(byteLimited.text, maxLines);
|
|
79
|
+
const totalChunks = extracted.sections.length;
|
|
79
80
|
return {
|
|
80
81
|
namespace,
|
|
81
|
-
result: {
|
|
82
|
+
result: {
|
|
83
|
+
text: lineLimited.text,
|
|
84
|
+
truncated: byteLimited.truncated || lineLimited.truncated,
|
|
85
|
+
totalChunks,
|
|
86
|
+
nextChunkOrdinal: chunkOrdinal + 1 < totalChunks ? chunkOrdinal + 1 : null
|
|
87
|
+
}
|
|
82
88
|
};
|
|
83
89
|
} catch (error) {
|
|
84
90
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -4,7 +4,7 @@ const MAX_EXCERPT_CHARS = 2e3;
|
|
|
4
4
|
const MAX_BYTES = 1048576;
|
|
5
5
|
const MAX_LINES = 1e4;
|
|
6
6
|
const DEFAULT_LIMIT = 10;
|
|
7
|
-
const DEFAULT_EXCERPT_CHARS =
|
|
7
|
+
const DEFAULT_EXCERPT_CHARS = 800;
|
|
8
8
|
const DEFAULT_MAX_BYTES = 65536;
|
|
9
9
|
const DEFAULT_MAX_LINES = 200;
|
|
10
10
|
function parseCapabilities(value) {
|