@alete-ai/gate-ingest 0.1.0

package/dist/index.cjs

@@ -0,0 +1,12 @@
+'use strict';var js=require('@mdream/js'),plugins=require('@mdream/js/plugins'),openredaction=require('openredaction');var p=plugins.createPlugin({onNodeEnter(a){let e=a.name,t=a.attributes||{};switch(e){case "form":return `
+[FORM_START]
+`;case "input":{let r=t.type||"text",n=t.name||t.id||"",c=t.placeholder||"";return `[INPUT:${r}:${n}:${c}]`}case "select":return "[SELECT_START]";case "option":return " (OPTION:";case "button":return "[BUTTON:";case "label":return "LABEL[";case "nav":return `
+[NAV_START]
+`;case "a":return `[LINK:${t.href||"#"}]`;default:return}},onNodeExit(a){switch(a.name){case "form":return `
+[FORM_END]
+`;case "select":return "[SELECT_END]";case "option":return ") ";case "button":return "]";case "label":return "]";case "nav":return `
+[NAV_END]
+`;default:return}}});function d(a){let e=a.replace(/\\?\[FORM_START\\?\]/g,"structFormStart").replace(/\\?\[FORM_END\\?\]/g,"structFormEnd").replace(/\\?\[SELECT_START\\?\]/g,"structSelectStart").replace(/\\?\[SELECT_END\\?\]/g,"structSelectEnd").replace(/\\?\[NAV_START\\?\]/g,"structNavStart").replace(/\\?\[NAV_END\\?\]/g,"structNavEnd");return e=e.replace(/LABEL\\?\[/g,"structLabel "),e=e.replace(/\\?\[INPUT:([^:]+):([^:]*):([^\\\]]*)\\?\]/g,(t,r,n)=>{let c=n.replace(/[^a-zA-Z0-9]/g,"").slice(0,20);return `structInput${r.charAt(0).toUpperCase()+r.slice(1)}${c}`}).replace(/\\?\[LINK:[^\\\]]+\\?\]/g,()=>"structLinkElement").replace(/\\?\[BUTTON:([^\\\]]+)\\?\]/g,(t,r)=>`structButton${r.replace(/[^a-zA-Z0-9]/g,"").slice(0,20)}`),e=e.replace(/\[([^\]]*)\]\(([^)]+)\)/g,(t,r)=>`structLinkElement${r.replace(/[^a-zA-Z0-9]/g,"").slice(0,20)}`).replace(/!\[([^\]]*)\]\(([^)]+)\)/g,(t,r)=>`structImage${r.replace(/[^a-zA-Z0-9]/g,"").slice(0,20)}`),e=e.replace(/^# (.*$)/gm,(t,r)=>`sysHeader1 ${r.replace(/\[([^\]]+)\]\([^)]+\)/g,"$1").replace(/[^a-zA-Z0-9]/g,"").slice(0,30)}`).replace(/^## (.*$)/gm,(t,r)=>`sysHeader2 ${r.replace(/\[([^\]]+)\]\([^)]+\)/g,"$1").replace(/[^a-zA-Z0-9]/g,"").slice(0,30)}`).replace(/^### (.*$)/gm,(t,r)=>`sysHeader3 ${r.replace(/\[([^\]]+)\]\([^)]+\)/g,"$1").replace(/[^a-zA-Z0-9]/g,"").slice(0,30)}`),e=e.replace(/https?:\/\/[^\s]+/g,"").replace(/[#*`_\[\]():]/g," "),e.split(/\s+/).filter(t=>t.startsWith("struct")||t.startsWith("sys")||/^[A-Z]/.test(t)&&t.length>2).join(" ").trim()}var o=class{engine;constructor(e={}){let t=[];e.redactPii!==false&&(t.push("EMAIL"),t.push("PHONE_US","PHONE_UK","PHONE_INTERNATIONAL"),t.push(...openredaction.getPatternsByCategory("government").map(n=>n.type))),e.redactFinancials!==false&&t.push(...openredaction.getPatternsByCategory("financial").map(n=>n.type)),e.redactCredentials!==false&&t.push(...openredaction.getPatternsByCategory("credentials").map(n=>n.type)),e.redactMedical!==false&&t.push(...openredaction.getPatternsByCategory("healthcare").map(n=>n.type)),e.redactInfrastructure!==false&&t.push(...openredaction.getPatternsByCategory("network").map(n=>n.type));let r={patterns:t,includeNames:false,includeAddresses:false,redactionMode:"placeholder",enableContextAnalysis:true,enableFalsePositiveFilter:true,falsePositiveThreshold:.7,deterministic:true};this.engine=new openredaction.OpenRedaction(r);}async redact(e){return e&&(await this.engine.detect(e)).redacted}async hasSensitiveInfo(e){return e?(await this.engine.detect(e)).detections.length>0:false}async process(e){if(!e)return {redacted:e,hasSensitiveInfo:false};let t=await this.engine.detect(e);return {redacted:t.redacted,hasSensitiveInfo:t.detections.length>0}}};var _=(n=>(n.SENSITIVE_PORTAL="sensitive_portal",n.DIGESTIBLE_ARTICLE="digestible_article",n.NOISE="noise",n.UNKNOWN="unknown",n))(_||{}),T={SEMANTIC_TOKEN_CAP:15e3},l={defaultSemanticTokenCap:T.SEMANTIC_TOKEN_CAP},h=false;function I(a={}){l={...l,...a},console.log(`\u{1F6E1}\uFE0F Alete Gate: Ingestion substrate initialized. Semantic cap: ${l.defaultSemanticTokenCap} tokens. Explore our ecosystem at https://alete.ai/`),h=true;}function b(a){let e=a.trim().split(/\s+/).length;return Math.ceil(e*1.33)}function C(a,e){if(b(a)<=e)return {truncated:a,isTruncated:false};let r=Math.floor(e/1.33);return {truncated:a.trim().split(/\s+/).slice(0,r).join(" ")+`
+
+... [Content truncated due to token cap]`,isTruncated:true}}async function w(a,e={}){h||I();let t=js.htmlToMarkdown(a,{hooks:[p]}),r={},n=js.htmlToMarkdown(a,js.withMinimalPreset({isolateMain:false,plugins:{tagOverrides:{a:{enter:"",exit:""},img:{enter:"",exit:""},svg:{enter:"",exit:""},canvas:{enter:"",exit:""}},frontmatter:{onExtract:s=>{r=s;}}}})).trim();if(!r.title){let s=n.match(/^# (.*)$/m);s&&(r.title=s[1].replace(/[#*`_]/g,"").trim());}if(!r.description){let s=n.replace(/^# .*$/m,"").trim();if(s){let u=s.replace(/[#*`_]/g,"").trim();r.description=u.slice(0,150).replace(/\n/g," ")+(u.length>150?"...":"");}}let c=e.semanticTokenCap??l.defaultSemanticTokenCap??T.SEMANTIC_TOKEN_CAP,{truncated:E,isTruncated:N}=C(n,c);n=E;let g=false;if(e.redact){let s=typeof e.redact=="object"?e.redact:{},f=await new o(s).process(n);n=f.redacted,g=f.hasSensitiveInfo;}return {structural:d(t),semantic:n,hasSensitiveInfo:g,metadata:r,isTruncated:N}}exports.GateLabel=_;exports.Redactor=o;exports.initialize=I;exports.mapToTokens=d;exports.plugin=p;exports.processHtml=w;//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/config.ts","../src/token-mapper.ts","../src/sanitization/Redactor.ts","../src/index.ts"],"names":["structuralPlugin","createPlugin","element","tag","attrs","type","name","placeholder","mapToTokens","text","processed","_","cleanName","content","alt","word","Redactor","options","patterns","getPatternsByCategory","p","engineOptions","OpenRedaction","result","GateLabel","DEFAULTS","globalConfig","isInitialized","initialize","config","estimateTokens","wordCount","truncateToCap","cap","wordsToKeep","processHtml","html","structuralMd","htmlToMarkdown","metadata","semantic","withMinimalPreset","fm","h1Match","bodyOnly","cleanBody","truncated","isTruncated","hasSensitiveInfo","redactorOptions"],"mappings":"uHAOO,IAAMA,CAAAA,CAAmBC,oBAAAA,CAAa,CAC3C,WAAA,CAAYC,CAAAA,CAAS,CACnB,IAAMC,CAAAA,CAAMD,CAAAA,CAAQ,IAAA,CACdE,CAAAA,CAAQF,CAAAA,CAAQ,UAAA,EAAc,EAAC,CAErC,OAAQC,CAAAA,EACN,KAAK,OACH,OAAO;AAAA;AAAA,CAAA,CACT,KAAK,QAAS,CACZ,IAAME,EAAOD,CAAAA,CAAM,IAAA,EAAQ,OACrBE,CAAAA,CAAOF,CAAAA,CAAM,MAAQA,CAAAA,CAAM,EAAA,EAAM,GACjCG,CAAAA,CAAcH,CAAAA,CAAM,aAAe,EAAA,CACzC,OAAO,UAAUC,CAAI,CAAA,CAAA,EAAIC,CAAI,CAAA,CAAA,EAAIC,CAAW,GAC9C,CACA,KAAK,SACH,OAAO,gBAAA,CACT,KAAK,QAAA,CACH,OAAO,YACT,KAAK,QAAA,CACH,OAAO,UAAA,CACT,KAAK,QACH,OAAO,QAAA,CACT,KAAK,KAAA,CACH,OAAO;AAAA;AAAA,CAAA,CACT,KAAK,IAEH,OAAO,CAAA,MAAA,EADMH,EAAM,IAAA,EAAQ,GACP,IAEtB,QACE,MACJ,CACF,CAAA,CAEA,UAAA,CAAWF,EAAS,CAElB,OADYA,EAAQ,IAAA,EAElB,KAAK,MAAA,CACH,OAAO;AAAA;AAAA,CAAA,CACT,KAAK,QAAA,CACH,OAAO,eACT,KAAK,QAAA,CACH,OAAO,IAAA,CACT,KAAK,QAAA,CACH,OAAO,IACT,KAAK,OAAA,CACH,OAAO,GAAA,CACT,KAAK,MACH,OAAO;AAAA;AAAA,CAAA,CACT,QACE,MACJ,CACF,CACF,CAAC,ECtDM,SAASM,CAAAA,CAAYC,CAAAA,CAAsB,CAGhD,IAAIC,CAAAA,CAAYD,EACb,OAAA,CAAQ,uBAAA,CAAyB,iBAAiB,CAAA,CAClD,OAAA,CAAQ,qBAAA,CAAuB,eAAe,CAAA,CAC9C,OAAA,CAAQ,yBAAA,CAA2B,mBAAmB,CAAA,CACtD,OAAA,CAAQ,uBAAA,CAAyB,iBAAiB,CAAA,CAClD,QAAQ,sBAAA,CAAwB,gBAAgB,CAAA,CAChD,OAAA,CAAQ,oBAAA,CAAsB,cAAc,CAAA,CAG/C,OAAAC,CAAAA,CAAYA,CAAAA,CAAU,OAAA,CAAQ,aAAA,CAAe,cAAc,CAAA,CAG3DA,CAAAA,CAAYA,CAAAA,CAET,QAAQ,6CAAA,CAA+C,CAACC,CAAAA,CAAGN,CAAAA,CAAMC,CAAAA,GAAS,CACzE,IAAMM,CAAAA,CAAYN,EAAK,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAAA,CAC/D,OAAO,CAAA,WAAA,EAAcD,CAAAA,CAAK,MAAA,CAAO,CAAC,CAAA,CAAE,WAAA,EAAY,CAAIA,CAAAA,CAAK,KAAA,CAAM,CAAC,CAAC,CAAA,EAAGO,CAAS,CAAA,CAC/E,CAAC,CAAA,CAEA,QAAQ,0BAAA,CAA4B,IAAM,mBAAmB,CAAA,CAE7D,OAAA,CAAQ,8BAAA,CAAgC,CAACD,CAAAA,CAAGF,CAAAA,GAEpC,CAAA,YAAA,EADOA,CAAAA,CAAK,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,EAAG,EAAE,CAChC,CAAA,CAC5B,CAAA,CAGHC,CAAAA,CAAYA,CAAAA,CAET,OAAA,CAAQ,0BAAA,CAA4B,CAACC,CAAAA,CAAGE,CAAAA,GAEhC,CAAA,iBAAA,EADOA,CAAAA,CAAQ,OAAA,CAAQ,eAAA,CAAiB,EAAE,EAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC9B,CAAA,CACjC,CAAA,CAEA,OAAA,CAAQ,2BAAA,CAA6B,CAACF,CAAAA,CAAGG,CAAAA,GAEjC,CAAA,WAAA,EADOA,CAAAA,CAAI,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,MAAM,CAAA,CAAG,EAAE,CAChC,CAAA,CAC3B,CAAA,CAGHJ,CAAAA,CAAYA,CAAAA,CACT,OAAA,CAAQ,YAAA,CAAc,CAACC,CAAAA,CAAGE,CAAAA,GAElB,CAAA,WAAA,EADOA,CAAAA,CAAQ,OAAA,CAAQ,wBAAA,CAA0B,IAAI,CAAA,CAAE,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC5E,CAAA,CAC3B,CAAA,CACA,OAAA,CAAQ,aAAA,CAAe,CAACF,CAAAA,CAAGE,CAAAA,GAEnB,cADOA,CAAAA,CAAQ,OAAA,CAAQ,wBAAA,CAA0B,IAAI,CAAA,CAAE,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC5E,CAAA,CAC3B,CAAA,CACA,OAAA,CAAQ,eAAgB,CAACF,CAAAA,CAAGE,CAAAA,GAEpB,CAAA,WAAA,EADOA,CAAAA,CAAQ,OAAA,CAAQ,wBAAA,CAA0B,IAAI,EAAE,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC5E,EAC3B,CAAA,CAGHH,CAAAA,CAAYA,CAAAA,CACT,OAAA,CAAQ,oBAAA,CAAsB,EAAE,CAAA,CAChC,OAAA,CAAQ,gBAAA,CAAkB,GAAG,CAAA,CAGzBA,CAAAA,CACJ,KAAA,CAAM,KAAK,CAAA,CACX,MAAA,CAAOK,GACCA,CAAAA,CAAK,UAAA,CAAW,QAAQ,CAAA,EACxBA,CAAAA,CAAK,UAAA,CAAW,KAAK,CAAA,EACpB,SAAS,IAAA,CAAKA,CAAI,CAAA,EAAKA,CAAAA,CAAK,MAAA,CAAS,CAC9C,CAAA,CACA,IAAA,CAAK,GAAG,CAAA,CACR,IAAA,EACL,CC7DO,IAAMC,CAAAA,CAAN,KAAe,CACZ,MAAA,CAER,WAAA,CAAYC,EAA2B,EAAC,CAAG,CACzC,IAAMC,CAAAA,CAAqB,EAAC,CAGxBD,CAAAA,CAAQ,YAAc,KAAA,GACxBC,CAAAA,CAAS,IAAA,CAAK,OAAO,CAAA,CACrBA,CAAAA,CAAS,IAAA,CAAK,UAAA,CAAY,WAAY,qBAAqB,CAAA,CAC3DA,CAAAA,CAAS,IAAA,CAAK,GAAGC,mCAAAA,CAAsB,YAAY,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAAA,CAInEH,CAAAA,CAAQ,gBAAA,GAAqB,OAC/BC,CAAAA,CAAS,IAAA,CAAK,GAAGC,mCAAAA,CAAsB,WAAW,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAIlEH,CAAAA,CAAQ,iBAAA,GAAsB,KAAA,EAChCC,CAAAA,CAAS,KAAK,GAAGC,mCAAAA,CAAsB,aAAa,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAIpEH,CAAAA,CAAQ,aAAA,GAAkB,KAAA,EAC5BC,CAAAA,CAAS,IAAA,CAAK,GAAGC,oCAAsB,YAAY,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAInEH,EAAQ,oBAAA,GAAyB,KAAA,EACnCC,CAAAA,CAAS,IAAA,CAAK,GAAGC,mCAAAA,CAAsB,SAAS,CAAA,CAAE,IAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAGpE,IAAMC,CAAAA,CAAsC,CAC1C,QAAA,CAAAH,CAAAA,CAEA,YAAA,CAAc,KAAA,CACd,gBAAA,CAAkB,KAAA,CAElB,aAAA,CAAe,aAAA,CACf,sBAAuB,IAAA,CACvB,yBAAA,CAA2B,IAAA,CAC3B,sBAAA,CAAwB,EAAA,CACxB,aAAA,CAAe,IACjB,CAAA,CAEA,KAAK,MAAA,CAAS,IAAII,2BAAAA,CAAcD,CAAa,EAC/C,CAKA,MAAa,MAAA,CAAOZ,EAA+B,CACjD,OAAKA,CAAAA,EAAAA,CACU,MAAM,IAAA,CAAK,MAAA,CAAO,MAAA,CAAOA,CAAI,CAAA,EAC9B,QAChB,CAKA,MAAa,gBAAA,CAAiBA,CAAAA,CAAgC,CAC5D,OAAKA,GACU,MAAM,IAAA,CAAK,MAAA,CAAO,MAAA,CAAOA,CAAI,CAAA,EAC9B,UAAA,CAAW,MAAA,CAAS,EAFhB,KAGpB,CAKA,MAAa,OAAA,CAAQA,CAAAA,CAAwE,CAC3F,GAAI,CAACA,EAAM,OAAO,CAAE,QAAA,CAAUA,CAAAA,CAAM,gBAAA,CAAkB,KAAM,CAAA,CAC5D,IAAMc,CAAAA,CAAS,MAAM,IAAA,CAAK,MAAA,CAAO,MAAA,CAAOd,CAAI,CAAA,CAC5C,OAAO,CACL,QAAA,CAAUc,CAAAA,CAAO,QAAA,CACjB,gBAAA,CAAkBA,CAAAA,CAAO,UAAA,CAAW,MAAA,CAAS,CAC/C,CACF,CACF,ECzFO,IAAKC,CAAAA,CAAAA,CAAAA,CAAAA,GACVA,CAAAA,CAAA,gBAAA,CAAmB,kBAAA,CACnBA,EAAA,kBAAA,CAAqB,oBAAA,CACrBA,CAAAA,CAAA,KAAA,CAAQ,OAAA,CACRA,CAAAA,CAAA,OAAA,CAAU,SAAA,CAJAA,CAAAA,CAAAA,EAAAA,CAAAA,EAAA,EAAA,CAAA,CAuDNC,CAAAA,CAAW,CACf,kBAAA,CAAoB,IACtB,CAAA,CAEIC,CAAAA,CAA6B,CAC/B,uBAAA,CAAyBD,CAAAA,CAAS,kBACpC,CAAA,CAEIE,CAAAA,CAAgB,MAKb,SAASC,CAAAA,CAAWC,EAAuB,EAAC,CAAS,CAC1DH,CAAAA,CAAe,CAAE,GAAGA,CAAAA,CAAc,GAAGG,CAAO,CAAA,CAC5C,OAAA,CAAQ,GAAA,CAAI,CAAA,2EAAA,EAAkEH,CAAAA,CAAa,uBAAuB,CAAA,mDAAA,CAAqD,CAAA,CACvKC,CAAAA,CAAgB,KAClB,CAMA,SAASG,CAAAA,CAAerB,CAAAA,CAAsB,CAC5C,IAAMsB,EAAYtB,CAAAA,CAAK,IAAA,EAAK,CAAE,KAAA,CAAM,KAAK,CAAA,CAAE,MAAA,CAC3C,OAAO,KAAK,IAAA,CAAKsB,CAAAA,CAAY,IAAI,CACnC,CAKA,SAASC,CAAAA,CAAcvB,CAAAA,CAAcwB,EAA0D,CAE7F,GADeH,CAAAA,CAAerB,CAAI,CAAA,EACpBwB,CAAAA,CACZ,OAAO,CAAE,SAAA,CAAWxB,CAAAA,CAAM,WAAA,CAAa,KAAM,CAAA,CAI/C,IAAMyB,CAAAA,CAAc,IAAA,CAAK,MAAMD,CAAAA,CAAM,IAAI,CAAA,CAIzC,OAAO,CAAE,SAAA,CAHKxB,CAAAA,CAAK,IAAA,GAAO,KAAA,CAAM,KAAK,CAAA,CACb,KAAA,CAAM,CAAA,CAAGyB,CAAW,CAAA,CAAE,IAAA,CAAK,GAAG,CAAA,CAAI;;AAAA,wCAAA,CAAA,CAEtC,WAAA,CAAa,IAAK,CACxC,CAMA,eAAsBC,CAAAA,CAAYC,CAAAA,CAAcnB,CAAAA,CAA4B,EAAC,CAA6B,CACnGU,GACHC,CAAAA,EAAW,CAIb,IAAMS,CAAAA,CAAeC,iBAAAA,CAAeF,CAAAA,CAAM,CACxC,KAAA,CAAO,CAACpC,CAAgB,CAC1B,CAAC,CAAA,CAEGuC,EAAmC,EAAC,CACpCC,CAAAA,CAAWF,iBAAAA,CAAeF,CAAAA,CAAMK,oBAAAA,CAAkB,CACpD,WAAA,CAAa,KAAA,CACb,OAAA,CAAS,CACP,YAAA,CAAc,CACZ,EAAG,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAAA,CACzB,IAAK,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAAA,CAC3B,IAAK,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAAA,CAC3B,OAAQ,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAChC,EACA,WAAA,CAAa,CACX,SAAA,CAAYC,CAAAA,EAAO,CACjBH,CAAAA,CAAWG,EACb,CACF,CACF,CACF,CAAC,CAAC,CAAA,CAAE,MAAK,CAGT,GAAI,CAACH,CAAAA,CAAS,KAAA,CAAO,CACnB,IAAMI,CAAAA,CAAUH,CAAAA,CAAS,KAAA,CAAM,WAAW,CAAA,CACtCG,CAAAA,GAEFJ,EAAS,KAAA,CAAQI,CAAAA,CAAQ,CAAC,CAAA,CAAE,OAAA,CAAQ,SAAA,CAAW,EAAE,CAAA,CAAE,IAAA,EAAK,EAE5D,CACA,GAAI,CAACJ,EAAS,WAAA,CAAa,CAEzB,IAAMK,CAAAA,CAAWJ,CAAAA,CAAS,QAAQ,SAAA,CAAW,EAAE,CAAA,CAAE,IAAA,EAAK,CACtD,GAAII,EAAU,CAEZ,IAAMC,CAAAA,CAAYD,CAAAA,CAAS,OAAA,CAAQ,SAAA,CAAW,EAAE,CAAA,CAAE,IAAA,EAAK,CACvDL,CAAAA,CAAS,WAAA,CAAcM,CAAAA,CAAU,MAAM,CAAA,CAAG,GAAG,CAAA,CAAE,OAAA,CAAQ,KAAA,CAAO,GAAG,GAAKA,CAAAA,CAAU,MAAA,CAAS,GAAA,CAAM,KAAA,CAAQ,EAAA,EACzG,CACF,CAGA,IAAMZ,CAAAA,CAAMhB,CAAAA,CAAQ,gBAAA,EAAoBS,CAAAA,CAAa,uBAAA,EAA2BD,EAAS,kBAAA,CACnF,CAAE,SAAA,CAAAqB,CAAAA,CAAW,WAAA,CAAAC,CAAY,EAAIf,CAAAA,CAAcQ,CAAAA,CAAUP,CAAG,CAAA,CAC9DO,CAAAA,CAAWM,CAAAA,CAEX,IAAIE,CAAAA,CAAmB,KAAA,CAGvB,GAAI/B,CAAAA,CAAQ,MAAA,CAAQ,CAClB,IAAMgC,CAAAA,CAAkB,OAAOhC,CAAAA,CAAQ,MAAA,EAAW,QAAA,CAAWA,CAAAA,CAAQ,OAAS,EAAC,CAEzEM,CAAAA,CAAS,MADE,IAAIP,CAAAA,CAASiC,CAAe,CAAA,CACf,OAAA,CAAQT,CAAQ,CAAA,CAC9CA,CAAAA,CAAWjB,CAAAA,CAAO,SAClByB,CAAAA,CAAmBzB,CAAAA,CAAO,iBAC5B,CAEA,OAAO,CACL,WAAYf,CAAAA,CAAY6B,CAAY,CAAA,CACpC,QAAA,CAAAG,CAAAA,CACA,gBAAA,CAAAQ,EACA,QAAA,CAAAT,CAAAA,CACA,WAAA,CAAAQ,CACF,CACF","file":"index.cjs","sourcesContent":["import { createPlugin } from '@mdream/js/plugins';\n\n/**\n * Custom mdream plugin to retain structural artifacts.\n * This preserves the \"structural footprint\" of sensitive portals\n * by injecting intermediate markers with attribute context.\n */\nexport const structuralPlugin = createPlugin({\n  onNodeEnter(element) {\n    const tag = element.name;\n    const attrs = element.attributes || {};\n\n    switch (tag) {\n      case 'form':\n        return '\\n[FORM_START]\\n';\n      case 'input': {\n        const type = attrs.type || 'text';\n        const name = attrs.name || attrs.id || '';\n        const placeholder = attrs.placeholder || '';\n        return `[INPUT:${type}:${name}:${placeholder}]`;\n      }\n      case 'select':\n        return '[SELECT_START]';\n      case 'option':\n        return ' (OPTION:';\n      case 'button':\n        return '[BUTTON:';\n      case 'label':\n        return 'LABEL[';\n      case 'nav':\n        return '\\n[NAV_START]\\n';\n      case 'a': {\n        const href = attrs.href || '#';\n        return `[LINK:${href}]`;\n      }\n      default:\n        return undefined;\n    }\n  },\n\n  onNodeExit(element) {\n    const tag = element.name;\n    switch (tag) {\n      case 'form':\n        return '\\n[FORM_END]\\n';\n      case 'select':\n        return '[SELECT_END]';\n      case 'option':\n        return ') ';\n      case 'button':\n        return ']';\n      case 'label':\n        return ']';\n      case 'nav':\n        return '\\n[NAV_END]\\n';\n      default:\n        return undefined;\n    }\n  }\n});\n","/**\n * Maps structural Markdown/HTML artifacts to explicit alphanumeric tokens.\n * This prevents Apple's NLTokenizer from stripping critical punctuation.\n * We use camelCase because NLTokenizer splits snake_case (STRUCT_FORM_START -> STRUCT, FORM, START).\n */\nexport function mapToTokens(text: string): string {\n  // 1. Process explicit markers from structuralPlugin\n  // We handle potential escaping from mdream\n  let processed = text\n    .replace(/\\\\?\\[FORM_START\\\\?\\]/g, 'structFormStart')\n    .replace(/\\\\?\\[FORM_END\\\\?\\]/g, 'structFormEnd')\n    .replace(/\\\\?\\[SELECT_START\\\\?\\]/g, 'structSelectStart')\n    .replace(/\\\\?\\[SELECT_END\\\\?\\]/g, 'structSelectEnd')\n    .replace(/\\\\?\\[NAV_START\\\\?\\]/g, 'structNavStart')\n    .replace(/\\\\?\\[NAV_END\\\\?\\]/g, 'structNavEnd');\n\n  // 1.1 Process Label marker\n  processed = processed.replace(/LABEL\\\\?\\[/g, 'structLabel ');\n\n  // 2. Process attribute-based markers\n  processed = processed\n    // Inputs: [INPUT:type:name:placeholder] -> structInputType {type} {name}\n    .replace(/\\\\?\\[INPUT:([^:]+):([^:]*):([^\\\\\\]]*)\\\\?\\]/g, (_, type, name) => {\n      const cleanName = name.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structInput${type.charAt(0).toUpperCase() + type.slice(1)}${cleanName}`;\n    })\n    // Links: [LINK:url] -> structLink\n    .replace(/\\\\?\\[LINK:[^\\\\\\]]+\\\\?\\]/g, () => 'structLinkElement')\n    // Buttons: [BUTTON:text] -> structButton {text}\n    .replace(/\\\\?\\[BUTTON:([^\\\\\\]]+)\\\\?\\]/g, (_, text) => {\n      const clean = text.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structButton${clean}`;\n    });\n\n  // 3. Process Standard Markdown artifacts (if any remain) into clean tokens\n  processed = processed\n    // Links: [text](url) -> structLinkElement {text}\n    .replace(/\\[([^\\]]*)\\]\\(([^)]+)\\)/g, (_, content) => {\n      const clean = content.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structLinkElement${clean}`;\n    })\n    // Images: ![alt](url) -> structImage {alt}\n    .replace(/!\\[([^\\]]*)\\]\\(([^)]+)\\)/g, (_, alt) => {\n      const clean = alt.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structImage${clean}`;\n    });\n\n  // 4. Process Headers into clean tokens\n  processed = processed\n    .replace(/^# (.*$)/gm, (_, content) => {\n      const clean = content.replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, '$1').replace(/[^a-zA-Z0-9]/g, '').slice(0, 30);\n      return `sysHeader1 ${clean}`;\n    })\n    .replace(/^## (.*$)/gm, (_, content) => {\n      const clean = content.replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, '$1').replace(/[^a-zA-Z0-9]/g, '').slice(0, 30);\n      return `sysHeader2 ${clean}`;\n    })\n    .replace(/^### (.*$)/gm, (_, content) => {\n      const clean = content.replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, '$1').replace(/[^a-zA-Z0-9]/g, '').slice(0, 30);\n      return `sysHeader3 ${clean}`;\n    });\n\n  // 5. Final cleaning: remove URLs and punctuation (including colons now)\n  processed = processed\n    .replace(/https?:\\/\\/[^\\s]+/g, '') // Remove URLs\n    .replace(/[#*`_\\[\\]():]/g, ' '); // Remove remaining markdown chars + colons\n\n  // 6. Aggressively strip remaining natural language noise\n  return processed\n    .split(/\\s+/)\n    .filter(word => {\n      return word.startsWith('struct') || \n             word.startsWith('sys') || \n             (/^[A-Z]/.test(word) && word.length > 2); // Keep capitalized words (titles, labels) > 2 chars\n    })\n    .join(' ')\n    .trim();\n}\n","import { OpenRedaction, type OpenRedactionOptions, getPatternsByCategory } from 'openredaction';\n\nexport interface RedactorOptions {\n  redactPii?: boolean;\n  redactFinancials?: boolean;\n  redactCredentials?: boolean;\n  redactInfrastructure?: boolean;\n  redactMedical?: boolean;\n  customPlaceholders?: Record<string, string>;\n}\n\n/**\n * Sovereign wrapper for the PII Shield engine.\n * Adopts a \"Narrative-First\" strategy: Preserves names, dates, and locations\n * to maintain story coherence while redacting \"Toxic Identifiers\" (SSNs, Credit Cards, Secrets).\n */\nexport class Redactor {\n  private engine: OpenRedaction;\n\n  constructor(options: RedactorOptions = {}) {\n    const patterns: string[] = [];\n    \n    // 1. Personal & Contact (excluding Names, Addresses, Dates)\n    if (options.redactPii !== false) {\n      patterns.push('EMAIL');\n      patterns.push('PHONE_US', 'PHONE_UK', 'PHONE_INTERNATIONAL');\n      patterns.push(...getPatternsByCategory('government').map(p => p.type));\n    }\n\n    // 2. Financial\n    if (options.redactFinancials !== false) {\n      patterns.push(...getPatternsByCategory('financial').map(p => p.type));\n    }\n\n    // 3. Credentials\n    if (options.redactCredentials !== false) {\n      patterns.push(...getPatternsByCategory('credentials').map(p => p.type));\n    }\n\n    // 4. Medical\n    if (options.redactMedical !== false) {\n      patterns.push(...getPatternsByCategory('healthcare').map(p => p.type));\n    }\n\n    // 5. Infrastructure\n    if (options.redactInfrastructure !== false) {\n      patterns.push(...getPatternsByCategory('network').map(p => p.type));\n    }\n\n    const engineOptions: OpenRedactionOptions = {\n      patterns,\n      // Narrative-First safety: Double-down on disabling entities\n      includeNames: false,\n      includeAddresses: false,\n      \n      redactionMode: 'placeholder',\n      enableContextAnalysis: true,\n      enableFalsePositiveFilter: true,\n      falsePositiveThreshold: 0.7,\n      deterministic: true,\n    };\n\n    this.engine = new OpenRedaction(engineOptions);\n  }\n\n  /**\n   * Redacts sensitive information from the given text.\n   */\n  public async redact(text: string): Promise<string> {\n    if (!text) return text;\n    const result = await this.engine.detect(text);\n    return result.redacted;\n  }\n\n  /**\n   * Checks if the text contains any sensitive information without modifying it.\n   */\n  public async hasSensitiveInfo(text: string): Promise<boolean> {\n    if (!text) return false;\n    const result = await this.engine.detect(text);\n    return result.detections.length > 0;\n  }\n\n  /**\n   * Performs both detection and redaction in a single pass.\n   */\n  public async process(text: string): Promise<{ redacted: string, hasSensitiveInfo: boolean }> {\n    if (!text) return { redacted: text, hasSensitiveInfo: false };\n    const result = await this.engine.detect(text);\n    return {\n      redacted: result.redacted,\n      hasSensitiveInfo: result.detections.length > 0\n    };\n  }\n}\n","import { htmlToMarkdown, withMinimalPreset } from '@mdream/js';\nimport { structuralPlugin } from './config.js';\nimport { mapToTokens } from './token-mapper.js';\nimport { Redactor, type RedactorOptions } from './sanitization/Redactor.js';\n\nexport enum GateLabel {\n  SENSITIVE_PORTAL = 'sensitive_portal',\n  DIGESTIBLE_ARTICLE = 'digestible_article',\n  NOISE = 'noise',\n  UNKNOWN = 'unknown',\n}\n\nexport interface IngestionResult {\n  /**\n   * Alphanumeric tokenized text for Apple MaxEnt classifier.\n   * High structural fidelity, low natural language noise.\n   */\n  structural: string;\n  \n  /**\n   * Clean, readable Markdown for LLM analysis.\n   * Low structural noise, high semantic fidelity.\n   */\n  semantic: string;\n\n  /**\n   * Whether the semantic content contains sensitive PII that was redacted.\n   */\n  hasSensitiveInfo?: boolean;\n\n  /**\n   * Extracted metadata from the HTML (title, description, author, etc.)\n   */\n  metadata?: Record<string, string>;\n\n  /**\n   * Whether the semantic content was truncated due to the token cap.\n   */\n  isTruncated?: boolean;\n}\n\nexport interface IngestionOptions {\n  /**\n   * Redaction configuration. If true, uses default settings.\n   */\n  redact?: RedactorOptions | boolean;\n\n  /**\n   * Override the semantic token cap for this specific call.\n   */\n  semanticTokenCap?: number;\n}\n\nexport interface GlobalConfig {\n  /**\n   * The default token cap for semantic returns. Defaults to 15,000.\n   */\n  defaultSemanticTokenCap?: number;\n}\n\nconst DEFAULTS = {\n  SEMANTIC_TOKEN_CAP: 15000,\n};\n\nlet globalConfig: GlobalConfig = {\n  defaultSemanticTokenCap: DEFAULTS.SEMANTIC_TOKEN_CAP,\n};\n\nlet isInitialized = false;\n\n/**\n * Initializes the Alete Gate ingestion substrate with global configuration.\n */\nexport function initialize(config: GlobalConfig = {}): void {\n  globalConfig = { ...globalConfig, ...config };\n  console.log(`🛡️ Alete Gate: Ingestion substrate initialized. Semantic cap: ${globalConfig.defaultSemanticTokenCap} tokens. Explore our ecosystem at https://alete.ai/`);\n  isInitialized = true;\n}\n\n/**\n * Estimates the number of tokens in a text string.\n * Uses the 1.33x multiplier (tokens per word) for a safe estimation.\n */\nfunction estimateTokens(text: string): number {\n  const wordCount = text.trim().split(/\\s+/).length;\n  return Math.ceil(wordCount * 1.33);\n}\n\n/**\n * Truncates text to fit within a token cap.\n */\nfunction truncateToCap(text: string, cap: number): { truncated: string; isTruncated: boolean } {\n  const tokens = estimateTokens(text);\n  if (tokens <= cap) {\n    return { truncated: text, isTruncated: false };\n  }\n\n  // Calculate approximate words to keep\n  const wordsToKeep = Math.floor(cap / 1.33);\n  const words = text.trim().split(/\\s+/);\n  const truncated = words.slice(0, wordsToKeep).join(' ') + '\\n\\n... [Content truncated due to token cap]';\n  \n  return { truncated, isTruncated: true };\n}\n\n/**\n * The unified ingestion pipeline for Alete Gate.\n * Converts raw HTML into both structural tokens and semantic Markdown.\n */\nexport async function processHtml(html: string, options: IngestionOptions = {}): Promise<IngestionResult> {\n  if (!isInitialized) {\n    initialize();\n  }\n\n  // 1. Generate Structural Markdown using the custom plugin\n  const structuralMd = htmlToMarkdown(html, {\n    hooks: [structuralPlugin]\n  });\n\n  let metadata: Record<string, string> = {};\n  let semantic = htmlToMarkdown(html, withMinimalPreset({\n    isolateMain: false,\n    plugins: {\n      tagOverrides: {\n        a: { enter: '', exit: '' },\n        img: { enter: '', exit: '' },\n        svg: { enter: '', exit: '' },\n        canvas: { enter: '', exit: '' },\n      },\n      frontmatter: {\n        onExtract: (fm) => {\n          metadata = fm;\n        }\n      }\n    }\n  })).trim();\n\n  // 2.1 Fallback metadata if not found in head\n  if (!metadata.title) {\n    const h1Match = semantic.match(/^# (.*)$/m);\n    if (h1Match) {\n      // Clean markdown from title\n      metadata.title = h1Match[1].replace(/[#*`_]/g, '').trim();\n    }\n  }\n  if (!metadata.description) {\n    // Take first 150 chars of semantic (excluding title)\n    const bodyOnly = semantic.replace(/^# .*$/m, '').trim();\n    if (bodyOnly) {\n      // Clean markdown from description\n      const cleanBody = bodyOnly.replace(/[#*`_]/g, '').trim();\n      metadata.description = cleanBody.slice(0, 150).replace(/\\n/g, ' ') + (cleanBody.length > 150 ? '...' : '');\n    }\n  }\n\n  // 2.2 Apply token cap\n  const cap = options.semanticTokenCap ?? globalConfig.defaultSemanticTokenCap ?? DEFAULTS.SEMANTIC_TOKEN_CAP;\n  const { truncated, isTruncated } = truncateToCap(semantic, cap);\n  semantic = truncated;\n\n  let hasSensitiveInfo = false;\n\n  // 3. Redaction Pipeline\n  if (options.redact) {\n    const redactorOptions = typeof options.redact === 'object' ? options.redact : {};\n    const redactor = new Redactor(redactorOptions);\n    const result = await redactor.process(semantic);\n    semantic = result.redacted;\n    hasSensitiveInfo = result.hasSensitiveInfo;\n  }\n\n  return {\n    structural: mapToTokens(structuralMd),\n    semantic,\n    hasSensitiveInfo,\n    metadata,\n    isTruncated\n  };\n}\n\nexport { structuralPlugin as plugin } from './config.js';\nexport { mapToTokens } from './token-mapper.js';\nexport { Redactor, type RedactorOptions };\n"]}

package/dist/index.d.cts

@@ -0,0 +1,106 @@
+import * as _mdream_js from '@mdream/js';
+
+interface RedactorOptions {
+    redactPii?: boolean;
+    redactFinancials?: boolean;
+    redactCredentials?: boolean;
+    redactInfrastructure?: boolean;
+    redactMedical?: boolean;
+    customPlaceholders?: Record<string, string>;
+}
+/**
+ * Sovereign wrapper for the PII Shield engine.
+ * Adopts a "Narrative-First" strategy: Preserves names, dates, and locations
+ * to maintain story coherence while redacting "Toxic Identifiers" (SSNs, Credit Cards, Secrets).
+ */
+declare class Redactor {
+    private engine;
+    constructor(options?: RedactorOptions);
+    /**
+     * Redacts sensitive information from the given text.
+     */
+    redact(text: string): Promise<string>;
+    /**
+     * Checks if the text contains any sensitive information without modifying it.
+     */
+    hasSensitiveInfo(text: string): Promise<boolean>;
+    /**
+     * Performs both detection and redaction in a single pass.
+     */
+    process(text: string): Promise<{
+        redacted: string;
+        hasSensitiveInfo: boolean;
+    }>;
+}
+
+/**
+ * Custom mdream plugin to retain structural artifacts.
+ * This preserves the "structural footprint" of sensitive portals
+ * by injecting intermediate markers with attribute context.
+ */
+declare const structuralPlugin: _mdream_js.TransformPlugin;
+
+/**
+ * Maps structural Markdown/HTML artifacts to explicit alphanumeric tokens.
+ * This prevents Apple's NLTokenizer from stripping critical punctuation.
+ * We use camelCase because NLTokenizer splits snake_case (STRUCT_FORM_START -> STRUCT, FORM, START).
+ */
+declare function mapToTokens(text: string): string;
+
+declare enum GateLabel {
+    SENSITIVE_PORTAL = "sensitive_portal",
+    DIGESTIBLE_ARTICLE = "digestible_article",
+    NOISE = "noise",
+    UNKNOWN = "unknown"
+}
+interface IngestionResult {
+    /**
+     * Alphanumeric tokenized text for Apple MaxEnt classifier.
+     * High structural fidelity, low natural language noise.
+     */
+    structural: string;
+    /**
+     * Clean, readable Markdown for LLM analysis.
+     * Low structural noise, high semantic fidelity.
+     */
+    semantic: string;
+    /**
+     * Whether the semantic content contains sensitive PII that was redacted.
+     */
+    hasSensitiveInfo?: boolean;
+    /**
+     * Extracted metadata from the HTML (title, description, author, etc.)
+     */
+    metadata?: Record<string, string>;
+    /**
+     * Whether the semantic content was truncated due to the token cap.
+     */
+    isTruncated?: boolean;
+}
+interface IngestionOptions {
+    /**
+     * Redaction configuration. If true, uses default settings.
+     */
+    redact?: RedactorOptions | boolean;
+    /**
+     * Override the semantic token cap for this specific call.
+     */
+    semanticTokenCap?: number;
+}
+interface GlobalConfig {
+    /**
+     * The default token cap for semantic returns. Defaults to 15,000.
+     */
+    defaultSemanticTokenCap?: number;
+}
+/**
+ * Initializes the Alete Gate ingestion substrate with global configuration.
+ */
+declare function initialize(config?: GlobalConfig): void;
+/**
+ * The unified ingestion pipeline for Alete Gate.
+ * Converts raw HTML into both structural tokens and semantic Markdown.
+ */
+declare function processHtml(html: string, options?: IngestionOptions): Promise<IngestionResult>;
+
+export { GateLabel, type GlobalConfig, type IngestionOptions, type IngestionResult, Redactor, type RedactorOptions, initialize, mapToTokens, structuralPlugin as plugin, processHtml };

package/dist/index.d.ts

@@ -0,0 +1,106 @@
+import * as _mdream_js from '@mdream/js';
+
+interface RedactorOptions {
+    redactPii?: boolean;
+    redactFinancials?: boolean;
+    redactCredentials?: boolean;
+    redactInfrastructure?: boolean;
+    redactMedical?: boolean;
+    customPlaceholders?: Record<string, string>;
+}
+/**
+ * Sovereign wrapper for the PII Shield engine.
+ * Adopts a "Narrative-First" strategy: Preserves names, dates, and locations
+ * to maintain story coherence while redacting "Toxic Identifiers" (SSNs, Credit Cards, Secrets).
+ */
+declare class Redactor {
+    private engine;
+    constructor(options?: RedactorOptions);
+    /**
+     * Redacts sensitive information from the given text.
+     */
+    redact(text: string): Promise<string>;
+    /**
+     * Checks if the text contains any sensitive information without modifying it.
+     */
+    hasSensitiveInfo(text: string): Promise<boolean>;
+    /**
+     * Performs both detection and redaction in a single pass.
+     */
+    process(text: string): Promise<{
+        redacted: string;
+        hasSensitiveInfo: boolean;
+    }>;
+}
+
+/**
+ * Custom mdream plugin to retain structural artifacts.
+ * This preserves the "structural footprint" of sensitive portals
+ * by injecting intermediate markers with attribute context.
+ */
+declare const structuralPlugin: _mdream_js.TransformPlugin;
+
+/**
+ * Maps structural Markdown/HTML artifacts to explicit alphanumeric tokens.
+ * This prevents Apple's NLTokenizer from stripping critical punctuation.
+ * We use camelCase because NLTokenizer splits snake_case (STRUCT_FORM_START -> STRUCT, FORM, START).
+ */
+declare function mapToTokens(text: string): string;
+
+declare enum GateLabel {
+    SENSITIVE_PORTAL = "sensitive_portal",
+    DIGESTIBLE_ARTICLE = "digestible_article",
+    NOISE = "noise",
+    UNKNOWN = "unknown"
+}
+interface IngestionResult {
+    /**
+     * Alphanumeric tokenized text for Apple MaxEnt classifier.
+     * High structural fidelity, low natural language noise.
+     */
+    structural: string;
+    /**
+     * Clean, readable Markdown for LLM analysis.
+     * Low structural noise, high semantic fidelity.
+     */
+    semantic: string;
+    /**
+     * Whether the semantic content contains sensitive PII that was redacted.
+     */
+    hasSensitiveInfo?: boolean;
+    /**
+     * Extracted metadata from the HTML (title, description, author, etc.)
+     */
+    metadata?: Record<string, string>;
+    /**
+     * Whether the semantic content was truncated due to the token cap.
+     */
+    isTruncated?: boolean;
+}
+interface IngestionOptions {
+    /**
+     * Redaction configuration. If true, uses default settings.
+     */
+    redact?: RedactorOptions | boolean;
+    /**
+     * Override the semantic token cap for this specific call.
+     */
+    semanticTokenCap?: number;
+}
+interface GlobalConfig {
+    /**
+     * The default token cap for semantic returns. Defaults to 15,000.
+     */
+    defaultSemanticTokenCap?: number;
+}
+/**
+ * Initializes the Alete Gate ingestion substrate with global configuration.
+ */
+declare function initialize(config?: GlobalConfig): void;
+/**
+ * The unified ingestion pipeline for Alete Gate.
+ * Converts raw HTML into both structural tokens and semantic Markdown.
+ */
+declare function processHtml(html: string, options?: IngestionOptions): Promise<IngestionResult>;
+
+export { GateLabel, type GlobalConfig, type IngestionOptions, type IngestionResult, Redactor, type RedactorOptions, initialize, mapToTokens, structuralPlugin as plugin, processHtml };

package/dist/index.js

@@ -0,0 +1,12 @@
+import {htmlToMarkdown,withMinimalPreset}from'@mdream/js';import {createPlugin}from'@mdream/js/plugins';import {getPatternsByCategory,OpenRedaction}from'openredaction';var p=createPlugin({onNodeEnter(a){let e=a.name,t=a.attributes||{};switch(e){case "form":return `
+[FORM_START]
+`;case "input":{let r=t.type||"text",n=t.name||t.id||"",c=t.placeholder||"";return `[INPUT:${r}:${n}:${c}]`}case "select":return "[SELECT_START]";case "option":return " (OPTION:";case "button":return "[BUTTON:";case "label":return "LABEL[";case "nav":return `
+[NAV_START]
+`;case "a":return `[LINK:${t.href||"#"}]`;default:return}},onNodeExit(a){switch(a.name){case "form":return `
+[FORM_END]
+`;case "select":return "[SELECT_END]";case "option":return ") ";case "button":return "]";case "label":return "]";case "nav":return `
+[NAV_END]
+`;default:return}}});function d(a){let e=a.replace(/\\?\[FORM_START\\?\]/g,"structFormStart").replace(/\\?\[FORM_END\\?\]/g,"structFormEnd").replace(/\\?\[SELECT_START\\?\]/g,"structSelectStart").replace(/\\?\[SELECT_END\\?\]/g,"structSelectEnd").replace(/\\?\[NAV_START\\?\]/g,"structNavStart").replace(/\\?\[NAV_END\\?\]/g,"structNavEnd");return e=e.replace(/LABEL\\?\[/g,"structLabel "),e=e.replace(/\\?\[INPUT:([^:]+):([^:]*):([^\\\]]*)\\?\]/g,(t,r,n)=>{let c=n.replace(/[^a-zA-Z0-9]/g,"").slice(0,20);return `structInput${r.charAt(0).toUpperCase()+r.slice(1)}${c}`}).replace(/\\?\[LINK:[^\\\]]+\\?\]/g,()=>"structLinkElement").replace(/\\?\[BUTTON:([^\\\]]+)\\?\]/g,(t,r)=>`structButton${r.replace(/[^a-zA-Z0-9]/g,"").slice(0,20)}`),e=e.replace(/\[([^\]]*)\]\(([^)]+)\)/g,(t,r)=>`structLinkElement${r.replace(/[^a-zA-Z0-9]/g,"").slice(0,20)}`).replace(/!\[([^\]]*)\]\(([^)]+)\)/g,(t,r)=>`structImage${r.replace(/[^a-zA-Z0-9]/g,"").slice(0,20)}`),e=e.replace(/^# (.*$)/gm,(t,r)=>`sysHeader1 ${r.replace(/\[([^\]]+)\]\([^)]+\)/g,"$1").replace(/[^a-zA-Z0-9]/g,"").slice(0,30)}`).replace(/^## (.*$)/gm,(t,r)=>`sysHeader2 ${r.replace(/\[([^\]]+)\]\([^)]+\)/g,"$1").replace(/[^a-zA-Z0-9]/g,"").slice(0,30)}`).replace(/^### (.*$)/gm,(t,r)=>`sysHeader3 ${r.replace(/\[([^\]]+)\]\([^)]+\)/g,"$1").replace(/[^a-zA-Z0-9]/g,"").slice(0,30)}`),e=e.replace(/https?:\/\/[^\s]+/g,"").replace(/[#*`_\[\]():]/g," "),e.split(/\s+/).filter(t=>t.startsWith("struct")||t.startsWith("sys")||/^[A-Z]/.test(t)&&t.length>2).join(" ").trim()}var o=class{engine;constructor(e={}){let t=[];e.redactPii!==false&&(t.push("EMAIL"),t.push("PHONE_US","PHONE_UK","PHONE_INTERNATIONAL"),t.push(...getPatternsByCategory("government").map(n=>n.type))),e.redactFinancials!==false&&t.push(...getPatternsByCategory("financial").map(n=>n.type)),e.redactCredentials!==false&&t.push(...getPatternsByCategory("credentials").map(n=>n.type)),e.redactMedical!==false&&t.push(...getPatternsByCategory("healthcare").map(n=>n.type)),e.redactInfrastructure!==false&&t.push(...getPatternsByCategory("network").map(n=>n.type));let r={patterns:t,includeNames:false,includeAddresses:false,redactionMode:"placeholder",enableContextAnalysis:true,enableFalsePositiveFilter:true,falsePositiveThreshold:.7,deterministic:true};this.engine=new OpenRedaction(r);}async redact(e){return e&&(await this.engine.detect(e)).redacted}async hasSensitiveInfo(e){return e?(await this.engine.detect(e)).detections.length>0:false}async process(e){if(!e)return {redacted:e,hasSensitiveInfo:false};let t=await this.engine.detect(e);return {redacted:t.redacted,hasSensitiveInfo:t.detections.length>0}}};var _=(n=>(n.SENSITIVE_PORTAL="sensitive_portal",n.DIGESTIBLE_ARTICLE="digestible_article",n.NOISE="noise",n.UNKNOWN="unknown",n))(_||{}),T={SEMANTIC_TOKEN_CAP:15e3},l={defaultSemanticTokenCap:T.SEMANTIC_TOKEN_CAP},h=false;function I(a={}){l={...l,...a},console.log(`\u{1F6E1}\uFE0F Alete Gate: Ingestion substrate initialized. Semantic cap: ${l.defaultSemanticTokenCap} tokens. Explore our ecosystem at https://alete.ai/`),h=true;}function b(a){let e=a.trim().split(/\s+/).length;return Math.ceil(e*1.33)}function C(a,e){if(b(a)<=e)return {truncated:a,isTruncated:false};let r=Math.floor(e/1.33);return {truncated:a.trim().split(/\s+/).slice(0,r).join(" ")+`
+
+... [Content truncated due to token cap]`,isTruncated:true}}async function w(a,e={}){h||I();let t=htmlToMarkdown(a,{hooks:[p]}),r={},n=htmlToMarkdown(a,withMinimalPreset({isolateMain:false,plugins:{tagOverrides:{a:{enter:"",exit:""},img:{enter:"",exit:""},svg:{enter:"",exit:""},canvas:{enter:"",exit:""}},frontmatter:{onExtract:s=>{r=s;}}}})).trim();if(!r.title){let s=n.match(/^# (.*)$/m);s&&(r.title=s[1].replace(/[#*`_]/g,"").trim());}if(!r.description){let s=n.replace(/^# .*$/m,"").trim();if(s){let u=s.replace(/[#*`_]/g,"").trim();r.description=u.slice(0,150).replace(/\n/g," ")+(u.length>150?"...":"");}}let c=e.semanticTokenCap??l.defaultSemanticTokenCap??T.SEMANTIC_TOKEN_CAP,{truncated:E,isTruncated:N}=C(n,c);n=E;let g=false;if(e.redact){let s=typeof e.redact=="object"?e.redact:{},f=await new o(s).process(n);n=f.redacted,g=f.hasSensitiveInfo;}return {structural:d(t),semantic:n,hasSensitiveInfo:g,metadata:r,isTruncated:N}}export{_ as GateLabel,o as Redactor,I as initialize,d as mapToTokens,p as plugin,w as processHtml};//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/config.ts","../src/token-mapper.ts","../src/sanitization/Redactor.ts","../src/index.ts"],"names":["structuralPlugin","createPlugin","element","tag","attrs","type","name","placeholder","mapToTokens","text","processed","_","cleanName","content","alt","word","Redactor","options","patterns","getPatternsByCategory","p","engineOptions","OpenRedaction","result","GateLabel","DEFAULTS","globalConfig","isInitialized","initialize","config","estimateTokens","wordCount","truncateToCap","cap","wordsToKeep","processHtml","html","structuralMd","htmlToMarkdown","metadata","semantic","withMinimalPreset","fm","h1Match","bodyOnly","cleanBody","truncated","isTruncated","hasSensitiveInfo","redactorOptions"],"mappings":"wKAOO,IAAMA,CAAAA,CAAmBC,YAAAA,CAAa,CAC3C,WAAA,CAAYC,CAAAA,CAAS,CACnB,IAAMC,CAAAA,CAAMD,CAAAA,CAAQ,IAAA,CACdE,CAAAA,CAAQF,CAAAA,CAAQ,UAAA,EAAc,EAAC,CAErC,OAAQC,CAAAA,EACN,KAAK,OACH,OAAO;AAAA;AAAA,CAAA,CACT,KAAK,QAAS,CACZ,IAAME,EAAOD,CAAAA,CAAM,IAAA,EAAQ,OACrBE,CAAAA,CAAOF,CAAAA,CAAM,MAAQA,CAAAA,CAAM,EAAA,EAAM,GACjCG,CAAAA,CAAcH,CAAAA,CAAM,aAAe,EAAA,CACzC,OAAO,UAAUC,CAAI,CAAA,CAAA,EAAIC,CAAI,CAAA,CAAA,EAAIC,CAAW,GAC9C,CACA,KAAK,SACH,OAAO,gBAAA,CACT,KAAK,QAAA,CACH,OAAO,YACT,KAAK,QAAA,CACH,OAAO,UAAA,CACT,KAAK,QACH,OAAO,QAAA,CACT,KAAK,KAAA,CACH,OAAO;AAAA;AAAA,CAAA,CACT,KAAK,IAEH,OAAO,CAAA,MAAA,EADMH,EAAM,IAAA,EAAQ,GACP,IAEtB,QACE,MACJ,CACF,CAAA,CAEA,UAAA,CAAWF,EAAS,CAElB,OADYA,EAAQ,IAAA,EAElB,KAAK,MAAA,CACH,OAAO;AAAA;AAAA,CAAA,CACT,KAAK,QAAA,CACH,OAAO,eACT,KAAK,QAAA,CACH,OAAO,IAAA,CACT,KAAK,QAAA,CACH,OAAO,IACT,KAAK,OAAA,CACH,OAAO,GAAA,CACT,KAAK,MACH,OAAO;AAAA;AAAA,CAAA,CACT,QACE,MACJ,CACF,CACF,CAAC,ECtDM,SAASM,CAAAA,CAAYC,CAAAA,CAAsB,CAGhD,IAAIC,CAAAA,CAAYD,EACb,OAAA,CAAQ,uBAAA,CAAyB,iBAAiB,CAAA,CAClD,OAAA,CAAQ,qBAAA,CAAuB,eAAe,CAAA,CAC9C,OAAA,CAAQ,yBAAA,CAA2B,mBAAmB,CAAA,CACtD,OAAA,CAAQ,uBAAA,CAAyB,iBAAiB,CAAA,CAClD,QAAQ,sBAAA,CAAwB,gBAAgB,CAAA,CAChD,OAAA,CAAQ,oBAAA,CAAsB,cAAc,CAAA,CAG/C,OAAAC,CAAAA,CAAYA,CAAAA,CAAU,OAAA,CAAQ,aAAA,CAAe,cAAc,CAAA,CAG3DA,CAAAA,CAAYA,CAAAA,CAET,QAAQ,6CAAA,CAA+C,CAACC,CAAAA,CAAGN,CAAAA,CAAMC,CAAAA,GAAS,CACzE,IAAMM,CAAAA,CAAYN,EAAK,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAAA,CAC/D,OAAO,CAAA,WAAA,EAAcD,CAAAA,CAAK,MAAA,CAAO,CAAC,CAAA,CAAE,WAAA,EAAY,CAAIA,CAAAA,CAAK,KAAA,CAAM,CAAC,CAAC,CAAA,EAAGO,CAAS,CAAA,CAC/E,CAAC,CAAA,CAEA,QAAQ,0BAAA,CAA4B,IAAM,mBAAmB,CAAA,CAE7D,OAAA,CAAQ,8BAAA,CAAgC,CAACD,CAAAA,CAAGF,CAAAA,GAEpC,CAAA,YAAA,EADOA,CAAAA,CAAK,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,EAAG,EAAE,CAChC,CAAA,CAC5B,CAAA,CAGHC,CAAAA,CAAYA,CAAAA,CAET,OAAA,CAAQ,0BAAA,CAA4B,CAACC,CAAAA,CAAGE,CAAAA,GAEhC,CAAA,iBAAA,EADOA,CAAAA,CAAQ,OAAA,CAAQ,eAAA,CAAiB,EAAE,EAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC9B,CAAA,CACjC,CAAA,CAEA,OAAA,CAAQ,2BAAA,CAA6B,CAACF,CAAAA,CAAGG,CAAAA,GAEjC,CAAA,WAAA,EADOA,CAAAA,CAAI,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,MAAM,CAAA,CAAG,EAAE,CAChC,CAAA,CAC3B,CAAA,CAGHJ,CAAAA,CAAYA,CAAAA,CACT,OAAA,CAAQ,YAAA,CAAc,CAACC,CAAAA,CAAGE,CAAAA,GAElB,CAAA,WAAA,EADOA,CAAAA,CAAQ,OAAA,CAAQ,wBAAA,CAA0B,IAAI,CAAA,CAAE,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC5E,CAAA,CAC3B,CAAA,CACA,OAAA,CAAQ,aAAA,CAAe,CAACF,CAAAA,CAAGE,CAAAA,GAEnB,cADOA,CAAAA,CAAQ,OAAA,CAAQ,wBAAA,CAA0B,IAAI,CAAA,CAAE,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC5E,CAAA,CAC3B,CAAA,CACA,OAAA,CAAQ,eAAgB,CAACF,CAAAA,CAAGE,CAAAA,GAEpB,CAAA,WAAA,EADOA,CAAAA,CAAQ,OAAA,CAAQ,wBAAA,CAA0B,IAAI,EAAE,OAAA,CAAQ,eAAA,CAAiB,EAAE,CAAA,CAAE,KAAA,CAAM,CAAA,CAAG,EAAE,CAC5E,EAC3B,CAAA,CAGHH,CAAAA,CAAYA,CAAAA,CACT,OAAA,CAAQ,oBAAA,CAAsB,EAAE,CAAA,CAChC,OAAA,CAAQ,gBAAA,CAAkB,GAAG,CAAA,CAGzBA,CAAAA,CACJ,KAAA,CAAM,KAAK,CAAA,CACX,MAAA,CAAOK,GACCA,CAAAA,CAAK,UAAA,CAAW,QAAQ,CAAA,EACxBA,CAAAA,CAAK,UAAA,CAAW,KAAK,CAAA,EACpB,SAAS,IAAA,CAAKA,CAAI,CAAA,EAAKA,CAAAA,CAAK,MAAA,CAAS,CAC9C,CAAA,CACA,IAAA,CAAK,GAAG,CAAA,CACR,IAAA,EACL,CC7DO,IAAMC,CAAAA,CAAN,KAAe,CACZ,MAAA,CAER,WAAA,CAAYC,EAA2B,EAAC,CAAG,CACzC,IAAMC,CAAAA,CAAqB,EAAC,CAGxBD,CAAAA,CAAQ,YAAc,KAAA,GACxBC,CAAAA,CAAS,IAAA,CAAK,OAAO,CAAA,CACrBA,CAAAA,CAAS,IAAA,CAAK,UAAA,CAAY,WAAY,qBAAqB,CAAA,CAC3DA,CAAAA,CAAS,IAAA,CAAK,GAAGC,qBAAAA,CAAsB,YAAY,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAAA,CAInEH,CAAAA,CAAQ,gBAAA,GAAqB,OAC/BC,CAAAA,CAAS,IAAA,CAAK,GAAGC,qBAAAA,CAAsB,WAAW,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAIlEH,CAAAA,CAAQ,iBAAA,GAAsB,KAAA,EAChCC,CAAAA,CAAS,KAAK,GAAGC,qBAAAA,CAAsB,aAAa,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAIpEH,CAAAA,CAAQ,aAAA,GAAkB,KAAA,EAC5BC,CAAAA,CAAS,IAAA,CAAK,GAAGC,sBAAsB,YAAY,CAAA,CAAE,GAAA,CAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAInEH,EAAQ,oBAAA,GAAyB,KAAA,EACnCC,CAAAA,CAAS,IAAA,CAAK,GAAGC,qBAAAA,CAAsB,SAAS,CAAA,CAAE,IAAIC,CAAAA,EAAKA,CAAAA,CAAE,IAAI,CAAC,CAAA,CAGpE,IAAMC,CAAAA,CAAsC,CAC1C,QAAA,CAAAH,CAAAA,CAEA,YAAA,CAAc,KAAA,CACd,gBAAA,CAAkB,KAAA,CAElB,aAAA,CAAe,aAAA,CACf,sBAAuB,IAAA,CACvB,yBAAA,CAA2B,IAAA,CAC3B,sBAAA,CAAwB,EAAA,CACxB,aAAA,CAAe,IACjB,CAAA,CAEA,KAAK,MAAA,CAAS,IAAII,aAAAA,CAAcD,CAAa,EAC/C,CAKA,MAAa,MAAA,CAAOZ,EAA+B,CACjD,OAAKA,CAAAA,EAAAA,CACU,MAAM,IAAA,CAAK,MAAA,CAAO,MAAA,CAAOA,CAAI,CAAA,EAC9B,QAChB,CAKA,MAAa,gBAAA,CAAiBA,CAAAA,CAAgC,CAC5D,OAAKA,GACU,MAAM,IAAA,CAAK,MAAA,CAAO,MAAA,CAAOA,CAAI,CAAA,EAC9B,UAAA,CAAW,MAAA,CAAS,EAFhB,KAGpB,CAKA,MAAa,OAAA,CAAQA,CAAAA,CAAwE,CAC3F,GAAI,CAACA,EAAM,OAAO,CAAE,QAAA,CAAUA,CAAAA,CAAM,gBAAA,CAAkB,KAAM,CAAA,CAC5D,IAAMc,CAAAA,CAAS,MAAM,IAAA,CAAK,MAAA,CAAO,MAAA,CAAOd,CAAI,CAAA,CAC5C,OAAO,CACL,QAAA,CAAUc,CAAAA,CAAO,QAAA,CACjB,gBAAA,CAAkBA,CAAAA,CAAO,UAAA,CAAW,MAAA,CAAS,CAC/C,CACF,CACF,ECzFO,IAAKC,CAAAA,CAAAA,CAAAA,CAAAA,GACVA,CAAAA,CAAA,gBAAA,CAAmB,kBAAA,CACnBA,EAAA,kBAAA,CAAqB,oBAAA,CACrBA,CAAAA,CAAA,KAAA,CAAQ,OAAA,CACRA,CAAAA,CAAA,OAAA,CAAU,SAAA,CAJAA,CAAAA,CAAAA,EAAAA,CAAAA,EAAA,EAAA,CAAA,CAuDNC,CAAAA,CAAW,CACf,kBAAA,CAAoB,IACtB,CAAA,CAEIC,CAAAA,CAA6B,CAC/B,uBAAA,CAAyBD,CAAAA,CAAS,kBACpC,CAAA,CAEIE,CAAAA,CAAgB,MAKb,SAASC,CAAAA,CAAWC,EAAuB,EAAC,CAAS,CAC1DH,CAAAA,CAAe,CAAE,GAAGA,CAAAA,CAAc,GAAGG,CAAO,CAAA,CAC5C,OAAA,CAAQ,GAAA,CAAI,CAAA,2EAAA,EAAkEH,CAAAA,CAAa,uBAAuB,CAAA,mDAAA,CAAqD,CAAA,CACvKC,CAAAA,CAAgB,KAClB,CAMA,SAASG,CAAAA,CAAerB,CAAAA,CAAsB,CAC5C,IAAMsB,EAAYtB,CAAAA,CAAK,IAAA,EAAK,CAAE,KAAA,CAAM,KAAK,CAAA,CAAE,MAAA,CAC3C,OAAO,KAAK,IAAA,CAAKsB,CAAAA,CAAY,IAAI,CACnC,CAKA,SAASC,CAAAA,CAAcvB,CAAAA,CAAcwB,EAA0D,CAE7F,GADeH,CAAAA,CAAerB,CAAI,CAAA,EACpBwB,CAAAA,CACZ,OAAO,CAAE,SAAA,CAAWxB,CAAAA,CAAM,WAAA,CAAa,KAAM,CAAA,CAI/C,IAAMyB,CAAAA,CAAc,IAAA,CAAK,MAAMD,CAAAA,CAAM,IAAI,CAAA,CAIzC,OAAO,CAAE,SAAA,CAHKxB,CAAAA,CAAK,IAAA,GAAO,KAAA,CAAM,KAAK,CAAA,CACb,KAAA,CAAM,CAAA,CAAGyB,CAAW,CAAA,CAAE,IAAA,CAAK,GAAG,CAAA,CAAI;;AAAA,wCAAA,CAAA,CAEtC,WAAA,CAAa,IAAK,CACxC,CAMA,eAAsBC,CAAAA,CAAYC,CAAAA,CAAcnB,CAAAA,CAA4B,EAAC,CAA6B,CACnGU,GACHC,CAAAA,EAAW,CAIb,IAAMS,CAAAA,CAAeC,cAAAA,CAAeF,CAAAA,CAAM,CACxC,KAAA,CAAO,CAACpC,CAAgB,CAC1B,CAAC,CAAA,CAEGuC,EAAmC,EAAC,CACpCC,CAAAA,CAAWF,cAAAA,CAAeF,CAAAA,CAAMK,iBAAAA,CAAkB,CACpD,WAAA,CAAa,KAAA,CACb,OAAA,CAAS,CACP,YAAA,CAAc,CACZ,EAAG,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAAA,CACzB,IAAK,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAAA,CAC3B,IAAK,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAAA,CAC3B,OAAQ,CAAE,KAAA,CAAO,EAAA,CAAI,IAAA,CAAM,EAAG,CAChC,EACA,WAAA,CAAa,CACX,SAAA,CAAYC,CAAAA,EAAO,CACjBH,CAAAA,CAAWG,EACb,CACF,CACF,CACF,CAAC,CAAC,CAAA,CAAE,MAAK,CAGT,GAAI,CAACH,CAAAA,CAAS,KAAA,CAAO,CACnB,IAAMI,CAAAA,CAAUH,CAAAA,CAAS,KAAA,CAAM,WAAW,CAAA,CACtCG,CAAAA,GAEFJ,EAAS,KAAA,CAAQI,CAAAA,CAAQ,CAAC,CAAA,CAAE,OAAA,CAAQ,SAAA,CAAW,EAAE,CAAA,CAAE,IAAA,EAAK,EAE5D,CACA,GAAI,CAACJ,EAAS,WAAA,CAAa,CAEzB,IAAMK,CAAAA,CAAWJ,CAAAA,CAAS,QAAQ,SAAA,CAAW,EAAE,CAAA,CAAE,IAAA,EAAK,CACtD,GAAII,EAAU,CAEZ,IAAMC,CAAAA,CAAYD,CAAAA,CAAS,OAAA,CAAQ,SAAA,CAAW,EAAE,CAAA,CAAE,IAAA,EAAK,CACvDL,CAAAA,CAAS,WAAA,CAAcM,CAAAA,CAAU,MAAM,CAAA,CAAG,GAAG,CAAA,CAAE,OAAA,CAAQ,KAAA,CAAO,GAAG,GAAKA,CAAAA,CAAU,MAAA,CAAS,GAAA,CAAM,KAAA,CAAQ,EAAA,EACzG,CACF,CAGA,IAAMZ,CAAAA,CAAMhB,CAAAA,CAAQ,gBAAA,EAAoBS,CAAAA,CAAa,uBAAA,EAA2BD,EAAS,kBAAA,CACnF,CAAE,SAAA,CAAAqB,CAAAA,CAAW,WAAA,CAAAC,CAAY,EAAIf,CAAAA,CAAcQ,CAAAA,CAAUP,CAAG,CAAA,CAC9DO,CAAAA,CAAWM,CAAAA,CAEX,IAAIE,CAAAA,CAAmB,KAAA,CAGvB,GAAI/B,CAAAA,CAAQ,MAAA,CAAQ,CAClB,IAAMgC,CAAAA,CAAkB,OAAOhC,CAAAA,CAAQ,MAAA,EAAW,QAAA,CAAWA,CAAAA,CAAQ,OAAS,EAAC,CAEzEM,CAAAA,CAAS,MADE,IAAIP,CAAAA,CAASiC,CAAe,CAAA,CACf,OAAA,CAAQT,CAAQ,CAAA,CAC9CA,CAAAA,CAAWjB,CAAAA,CAAO,SAClByB,CAAAA,CAAmBzB,CAAAA,CAAO,iBAC5B,CAEA,OAAO,CACL,WAAYf,CAAAA,CAAY6B,CAAY,CAAA,CACpC,QAAA,CAAAG,CAAAA,CACA,gBAAA,CAAAQ,EACA,QAAA,CAAAT,CAAAA,CACA,WAAA,CAAAQ,CACF,CACF","file":"index.js","sourcesContent":["import { createPlugin } from '@mdream/js/plugins';\n\n/**\n * Custom mdream plugin to retain structural artifacts.\n * This preserves the \"structural footprint\" of sensitive portals\n * by injecting intermediate markers with attribute context.\n */\nexport const structuralPlugin = createPlugin({\n  onNodeEnter(element) {\n    const tag = element.name;\n    const attrs = element.attributes || {};\n\n    switch (tag) {\n      case 'form':\n        return '\\n[FORM_START]\\n';\n      case 'input': {\n        const type = attrs.type || 'text';\n        const name = attrs.name || attrs.id || '';\n        const placeholder = attrs.placeholder || '';\n        return `[INPUT:${type}:${name}:${placeholder}]`;\n      }\n      case 'select':\n        return '[SELECT_START]';\n      case 'option':\n        return ' (OPTION:';\n      case 'button':\n        return '[BUTTON:';\n      case 'label':\n        return 'LABEL[';\n      case 'nav':\n        return '\\n[NAV_START]\\n';\n      case 'a': {\n        const href = attrs.href || '#';\n        return `[LINK:${href}]`;\n      }\n      default:\n        return undefined;\n    }\n  },\n\n  onNodeExit(element) {\n    const tag = element.name;\n    switch (tag) {\n      case 'form':\n        return '\\n[FORM_END]\\n';\n      case 'select':\n        return '[SELECT_END]';\n      case 'option':\n        return ') ';\n      case 'button':\n        return ']';\n      case 'label':\n        return ']';\n      case 'nav':\n        return '\\n[NAV_END]\\n';\n      default:\n        return undefined;\n    }\n  }\n});\n","/**\n * Maps structural Markdown/HTML artifacts to explicit alphanumeric tokens.\n * This prevents Apple's NLTokenizer from stripping critical punctuation.\n * We use camelCase because NLTokenizer splits snake_case (STRUCT_FORM_START -> STRUCT, FORM, START).\n */\nexport function mapToTokens(text: string): string {\n  // 1. Process explicit markers from structuralPlugin\n  // We handle potential escaping from mdream\n  let processed = text\n    .replace(/\\\\?\\[FORM_START\\\\?\\]/g, 'structFormStart')\n    .replace(/\\\\?\\[FORM_END\\\\?\\]/g, 'structFormEnd')\n    .replace(/\\\\?\\[SELECT_START\\\\?\\]/g, 'structSelectStart')\n    .replace(/\\\\?\\[SELECT_END\\\\?\\]/g, 'structSelectEnd')\n    .replace(/\\\\?\\[NAV_START\\\\?\\]/g, 'structNavStart')\n    .replace(/\\\\?\\[NAV_END\\\\?\\]/g, 'structNavEnd');\n\n  // 1.1 Process Label marker\n  processed = processed.replace(/LABEL\\\\?\\[/g, 'structLabel ');\n\n  // 2. Process attribute-based markers\n  processed = processed\n    // Inputs: [INPUT:type:name:placeholder] -> structInputType {type} {name}\n    .replace(/\\\\?\\[INPUT:([^:]+):([^:]*):([^\\\\\\]]*)\\\\?\\]/g, (_, type, name) => {\n      const cleanName = name.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structInput${type.charAt(0).toUpperCase() + type.slice(1)}${cleanName}`;\n    })\n    // Links: [LINK:url] -> structLink\n    .replace(/\\\\?\\[LINK:[^\\\\\\]]+\\\\?\\]/g, () => 'structLinkElement')\n    // Buttons: [BUTTON:text] -> structButton {text}\n    .replace(/\\\\?\\[BUTTON:([^\\\\\\]]+)\\\\?\\]/g, (_, text) => {\n      const clean = text.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structButton${clean}`;\n    });\n\n  // 3. Process Standard Markdown artifacts (if any remain) into clean tokens\n  processed = processed\n    // Links: [text](url) -> structLinkElement {text}\n    .replace(/\\[([^\\]]*)\\]\\(([^)]+)\\)/g, (_, content) => {\n      const clean = content.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structLinkElement${clean}`;\n    })\n    // Images: ![alt](url) -> structImage {alt}\n    .replace(/!\\[([^\\]]*)\\]\\(([^)]+)\\)/g, (_, alt) => {\n      const clean = alt.replace(/[^a-zA-Z0-9]/g, '').slice(0, 20);\n      return `structImage${clean}`;\n    });\n\n  // 4. Process Headers into clean tokens\n  processed = processed\n    .replace(/^# (.*$)/gm, (_, content) => {\n      const clean = content.replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, '$1').replace(/[^a-zA-Z0-9]/g, '').slice(0, 30);\n      return `sysHeader1 ${clean}`;\n    })\n    .replace(/^## (.*$)/gm, (_, content) => {\n      const clean = content.replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, '$1').replace(/[^a-zA-Z0-9]/g, '').slice(0, 30);\n      return `sysHeader2 ${clean}`;\n    })\n    .replace(/^### (.*$)/gm, (_, content) => {\n      const clean = content.replace(/\\[([^\\]]+)\\]\\([^)]+\\)/g, '$1').replace(/[^a-zA-Z0-9]/g, '').slice(0, 30);\n      return `sysHeader3 ${clean}`;\n    });\n\n  // 5. Final cleaning: remove URLs and punctuation (including colons now)\n  processed = processed\n    .replace(/https?:\\/\\/[^\\s]+/g, '') // Remove URLs\n    .replace(/[#*`_\\[\\]():]/g, ' '); // Remove remaining markdown chars + colons\n\n  // 6. Aggressively strip remaining natural language noise\n  return processed\n    .split(/\\s+/)\n    .filter(word => {\n      return word.startsWith('struct') || \n             word.startsWith('sys') || \n             (/^[A-Z]/.test(word) && word.length > 2); // Keep capitalized words (titles, labels) > 2 chars\n    })\n    .join(' ')\n    .trim();\n}\n","import { OpenRedaction, type OpenRedactionOptions, getPatternsByCategory } from 'openredaction';\n\nexport interface RedactorOptions {\n  redactPii?: boolean;\n  redactFinancials?: boolean;\n  redactCredentials?: boolean;\n  redactInfrastructure?: boolean;\n  redactMedical?: boolean;\n  customPlaceholders?: Record<string, string>;\n}\n\n/**\n * Sovereign wrapper for the PII Shield engine.\n * Adopts a \"Narrative-First\" strategy: Preserves names, dates, and locations\n * to maintain story coherence while redacting \"Toxic Identifiers\" (SSNs, Credit Cards, Secrets).\n */\nexport class Redactor {\n  private engine: OpenRedaction;\n\n  constructor(options: RedactorOptions = {}) {\n    const patterns: string[] = [];\n    \n    // 1. Personal & Contact (excluding Names, Addresses, Dates)\n    if (options.redactPii !== false) {\n      patterns.push('EMAIL');\n      patterns.push('PHONE_US', 'PHONE_UK', 'PHONE_INTERNATIONAL');\n      patterns.push(...getPatternsByCategory('government').map(p => p.type));\n    }\n\n    // 2. Financial\n    if (options.redactFinancials !== false) {\n      patterns.push(...getPatternsByCategory('financial').map(p => p.type));\n    }\n\n    // 3. Credentials\n    if (options.redactCredentials !== false) {\n      patterns.push(...getPatternsByCategory('credentials').map(p => p.type));\n    }\n\n    // 4. Medical\n    if (options.redactMedical !== false) {\n      patterns.push(...getPatternsByCategory('healthcare').map(p => p.type));\n    }\n\n    // 5. Infrastructure\n    if (options.redactInfrastructure !== false) {\n      patterns.push(...getPatternsByCategory('network').map(p => p.type));\n    }\n\n    const engineOptions: OpenRedactionOptions = {\n      patterns,\n      // Narrative-First safety: Double-down on disabling entities\n      includeNames: false,\n      includeAddresses: false,\n      \n      redactionMode: 'placeholder',\n      enableContextAnalysis: true,\n      enableFalsePositiveFilter: true,\n      falsePositiveThreshold: 0.7,\n      deterministic: true,\n    };\n\n    this.engine = new OpenRedaction(engineOptions);\n  }\n\n  /**\n   * Redacts sensitive information from the given text.\n   */\n  public async redact(text: string): Promise<string> {\n    if (!text) return text;\n    const result = await this.engine.detect(text);\n    return result.redacted;\n  }\n\n  /**\n   * Checks if the text contains any sensitive information without modifying it.\n   */\n  public async hasSensitiveInfo(text: string): Promise<boolean> {\n    if (!text) return false;\n    const result = await this.engine.detect(text);\n    return result.detections.length > 0;\n  }\n\n  /**\n   * Performs both detection and redaction in a single pass.\n   */\n  public async process(text: string): Promise<{ redacted: string, hasSensitiveInfo: boolean }> {\n    if (!text) return { redacted: text, hasSensitiveInfo: false };\n    const result = await this.engine.detect(text);\n    return {\n      redacted: result.redacted,\n      hasSensitiveInfo: result.detections.length > 0\n    };\n  }\n}\n","import { htmlToMarkdown, withMinimalPreset } from '@mdream/js';\nimport { structuralPlugin } from './config.js';\nimport { mapToTokens } from './token-mapper.js';\nimport { Redactor, type RedactorOptions } from './sanitization/Redactor.js';\n\nexport enum GateLabel {\n  SENSITIVE_PORTAL = 'sensitive_portal',\n  DIGESTIBLE_ARTICLE = 'digestible_article',\n  NOISE = 'noise',\n  UNKNOWN = 'unknown',\n}\n\nexport interface IngestionResult {\n  /**\n   * Alphanumeric tokenized text for Apple MaxEnt classifier.\n   * High structural fidelity, low natural language noise.\n   */\n  structural: string;\n  \n  /**\n   * Clean, readable Markdown for LLM analysis.\n   * Low structural noise, high semantic fidelity.\n   */\n  semantic: string;\n\n  /**\n   * Whether the semantic content contains sensitive PII that was redacted.\n   */\n  hasSensitiveInfo?: boolean;\n\n  /**\n   * Extracted metadata from the HTML (title, description, author, etc.)\n   */\n  metadata?: Record<string, string>;\n\n  /**\n   * Whether the semantic content was truncated due to the token cap.\n   */\n  isTruncated?: boolean;\n}\n\nexport interface IngestionOptions {\n  /**\n   * Redaction configuration. If true, uses default settings.\n   */\n  redact?: RedactorOptions | boolean;\n\n  /**\n   * Override the semantic token cap for this specific call.\n   */\n  semanticTokenCap?: number;\n}\n\nexport interface GlobalConfig {\n  /**\n   * The default token cap for semantic returns. Defaults to 15,000.\n   */\n  defaultSemanticTokenCap?: number;\n}\n\nconst DEFAULTS = {\n  SEMANTIC_TOKEN_CAP: 15000,\n};\n\nlet globalConfig: GlobalConfig = {\n  defaultSemanticTokenCap: DEFAULTS.SEMANTIC_TOKEN_CAP,\n};\n\nlet isInitialized = false;\n\n/**\n * Initializes the Alete Gate ingestion substrate with global configuration.\n */\nexport function initialize(config: GlobalConfig = {}): void {\n  globalConfig = { ...globalConfig, ...config };\n  console.log(`🛡️ Alete Gate: Ingestion substrate initialized. Semantic cap: ${globalConfig.defaultSemanticTokenCap} tokens. Explore our ecosystem at https://alete.ai/`);\n  isInitialized = true;\n}\n\n/**\n * Estimates the number of tokens in a text string.\n * Uses the 1.33x multiplier (tokens per word) for a safe estimation.\n */\nfunction estimateTokens(text: string): number {\n  const wordCount = text.trim().split(/\\s+/).length;\n  return Math.ceil(wordCount * 1.33);\n}\n\n/**\n * Truncates text to fit within a token cap.\n */\nfunction truncateToCap(text: string, cap: number): { truncated: string; isTruncated: boolean } {\n  const tokens = estimateTokens(text);\n  if (tokens <= cap) {\n    return { truncated: text, isTruncated: false };\n  }\n\n  // Calculate approximate words to keep\n  const wordsToKeep = Math.floor(cap / 1.33);\n  const words = text.trim().split(/\\s+/);\n  const truncated = words.slice(0, wordsToKeep).join(' ') + '\\n\\n... [Content truncated due to token cap]';\n  \n  return { truncated, isTruncated: true };\n}\n\n/**\n * The unified ingestion pipeline for Alete Gate.\n * Converts raw HTML into both structural tokens and semantic Markdown.\n */\nexport async function processHtml(html: string, options: IngestionOptions = {}): Promise<IngestionResult> {\n  if (!isInitialized) {\n    initialize();\n  }\n\n  // 1. Generate Structural Markdown using the custom plugin\n  const structuralMd = htmlToMarkdown(html, {\n    hooks: [structuralPlugin]\n  });\n\n  let metadata: Record<string, string> = {};\n  let semantic = htmlToMarkdown(html, withMinimalPreset({\n    isolateMain: false,\n    plugins: {\n      tagOverrides: {\n        a: { enter: '', exit: '' },\n        img: { enter: '', exit: '' },\n        svg: { enter: '', exit: '' },\n        canvas: { enter: '', exit: '' },\n      },\n      frontmatter: {\n        onExtract: (fm) => {\n          metadata = fm;\n        }\n      }\n    }\n  })).trim();\n\n  // 2.1 Fallback metadata if not found in head\n  if (!metadata.title) {\n    const h1Match = semantic.match(/^# (.*)$/m);\n    if (h1Match) {\n      // Clean markdown from title\n      metadata.title = h1Match[1].replace(/[#*`_]/g, '').trim();\n    }\n  }\n  if (!metadata.description) {\n    // Take first 150 chars of semantic (excluding title)\n    const bodyOnly = semantic.replace(/^# .*$/m, '').trim();\n    if (bodyOnly) {\n      // Clean markdown from description\n      const cleanBody = bodyOnly.replace(/[#*`_]/g, '').trim();\n      metadata.description = cleanBody.slice(0, 150).replace(/\\n/g, ' ') + (cleanBody.length > 150 ? '...' : '');\n    }\n  }\n\n  // 2.2 Apply token cap\n  const cap = options.semanticTokenCap ?? globalConfig.defaultSemanticTokenCap ?? DEFAULTS.SEMANTIC_TOKEN_CAP;\n  const { truncated, isTruncated } = truncateToCap(semantic, cap);\n  semantic = truncated;\n\n  let hasSensitiveInfo = false;\n\n  // 3. Redaction Pipeline\n  if (options.redact) {\n    const redactorOptions = typeof options.redact === 'object' ? options.redact : {};\n    const redactor = new Redactor(redactorOptions);\n    const result = await redactor.process(semantic);\n    semantic = result.redacted;\n    hasSensitiveInfo = result.hasSensitiveInfo;\n  }\n\n  return {\n    structural: mapToTokens(structuralMd),\n    semantic,\n    hasSensitiveInfo,\n    metadata,\n    isTruncated\n  };\n}\n\nexport { structuralPlugin as plugin } from './config.js';\nexport { mapToTokens } from './token-mapper.js';\nexport { Redactor, type RedactorOptions };\n"]}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@alete-ai/gate-ingest",
+  "version": "0.1.0",
+  "description": "Unified ingestion and token-mapping pipeline for the Alete PrivacyGatekeeper.",
+  "keywords": [
+    "edge",
+    "crawler",
+    "scraper",
+    "firecrawl",
+    "crawl4ai",
+    "analysis"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "browser": "./dist/index.browser.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "browser": "./dist/index.browser.js",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "dependencies": {
+    "@mdream/js": "^1.1.1",
+    "openredaction": "^1.1.2"
+  },
+  "devDependencies": {
+    "tsup": "^8.0.0",
+    "typescript": "^5.0.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test": "vitest run"
+  }
+}

package/src/config.ts

@@ -0,0 +1,60 @@
+import { createPlugin } from '@mdream/js/plugins';
+
+/**
+ * Custom mdream plugin to retain structural artifacts.
+ * This preserves the "structural footprint" of sensitive portals
+ * by injecting intermediate markers with attribute context.
+ */
+export const structuralPlugin = createPlugin({
+  onNodeEnter(element) {
+    const tag = element.name;
+    const attrs = element.attributes || {};
+
+    switch (tag) {
+      case 'form':
+        return '\n[FORM_START]\n';
+      case 'input': {
+        const type = attrs.type || 'text';
+        const name = attrs.name || attrs.id || '';
+        const placeholder = attrs.placeholder || '';
+        return `[INPUT:${type}:${name}:${placeholder}]`;
+      }
+      case 'select':
+        return '[SELECT_START]';
+      case 'option':
+        return ' (OPTION:';
+      case 'button':
+        return '[BUTTON:';
+      case 'label':
+        return 'LABEL[';
+      case 'nav':
+        return '\n[NAV_START]\n';
+      case 'a': {
+        const href = attrs.href || '#';
+        return `[LINK:${href}]`;
+      }
+      default:
+        return undefined;
+    }
+  },
+
+  onNodeExit(element) {
+    const tag = element.name;
+    switch (tag) {
+      case 'form':
+        return '\n[FORM_END]\n';
+      case 'select':
+        return '[SELECT_END]';
+      case 'option':
+        return ') ';
+      case 'button':
+        return ']';
+      case 'label':
+        return ']';
+      case 'nav':
+        return '\n[NAV_END]\n';
+      default:
+        return undefined;
+    }
+  }
+});

package/src/index.test.ts

@@ -0,0 +1,100 @@
+import { describe, it, expect } from 'vitest';
+import { processHtml } from './index.js';
+
+describe('gate-ingest pipeline', () => {
+  it('processes a login form into structural tokens', async () => {
+    const html = `
+      <html>
+        <body>
+          <nav>
+            <a href="/home">Home</a>
+          </nav>
+          <form action="/login" method="POST">
+            <label for="username">Username:</label>
+            <input type="text" name="username" id="username" placeholder="Enter username">
+            <label for="password">Password:</label>
+            <input type="password" name="password" id="password">
+            <button type="submit">Login</button>
+          </form>
+        </body>
+      </html>
+    `;
+
+    const { structural, semantic } = await processHtml(html);
+
+    // Check structural tokens
+    expect(structural).toContain('structNavStart');
+    expect(structural).toContain('structLinkElement');
+    expect(structural).toContain('structNavEnd');
+    expect(structural).toContain('structFormStart');
+    expect(structural).toContain('structLabel Username');
+    expect(structural).toContain('structInputTextusername');
+    expect(structural).toContain('structLabel Password');
+    expect(structural).toContain('structButtonLogin');
+    expect(structural).toContain('structFormEnd');
+
+    // Check semantic markdown - Form elements are filtered by withMinimalPreset
+    // so we don't expect them in the semantic output, only in structural.
+    expect(semantic).not.toContain('STRUCT_FORM_START');
+  });
+
+  it('processes a news article into clean markdown', async () => {
+    const html = `
+      <html>
+        <body>
+          <h1>Breaking News</h1>
+          <p>This is a <strong>major</strong> story about privacy.</p>
+          <div class="sidebar">
+            <a href="/related">Related article</a>
+          </div>
+        </body>
+      </html>
+    `;
+
+    const { structural, semantic } = await processHtml(html);
+
+    // Check structural tokens
+    expect(structural).toContain('sysHeader1 BreakingNews');
+    
+    // Check semantic markdown
+    expect(semantic).toContain('# Breaking News');
+    expect(semantic).toContain('This is a **major** story about privacy.');
+  });
+
+  it('extracts page metadata', async () => {
+    const html = `
+      <html>
+        <head>
+          <title>Privacy Policy</title>
+          <meta name="description" content="Our commitment to your data.">
+        </head>
+        <body>
+          <p>We respect your privacy.</p>
+        </body>
+      </html>
+    `;
+
+    const { metadata } = await processHtml(html);
+
+    expect(metadata).toBeDefined();
+    expect(metadata?.title).toBe('Privacy Policy');
+    expect(metadata?.description).toBe('Our commitment to your data.');
+  });
+
+  it('respects the semantic token cap', async () => {
+    // Generate a long text with varied content
+    const words = ['apple', 'banana', 'cherry', 'date', 'elderberry', 'fig', 'grape', 'honeydew'];
+    const longText = Array(100).fill(0).map((_, i) => words[i % words.length]).join(' ');
+    const html = `<html><body><main><p>${longText}</p></main></body></html>`;
+
+    // 1.33 * 100 = 133 tokens. Cap at 50 tokens.
+    const { semantic, isTruncated } = await processHtml(html, { semanticTokenCap: 50 });
+
+    expect(isTruncated).toBe(true);
+    expect(semantic).toContain('... [Content truncated due to token cap]');
+    // 50 / 1.33 = 37.5 words
+    const wordCount = semantic.split('...')[0].trim().split(/\s+/).length;
+    expect(wordCount).toBeLessThanOrEqual(40);
+  });
+});
+