@griddo/cx 11.14.1 → 11.14.2-rc.1

package/build/commands/start-render.js

@@ -142,7 +142,7 @@ ${K.red("stack")}
 ${JSON.stringify(e,null,2)}`),new Sr(e)}async function AQ(A){try{await A()}catch(e){e instanceof Sr?d.error("Internal Griddo RenderError"):e instanceof Error?d.error(e.message):d.error(`An unexpected error occurred ${e}`);try{let t=await ee(),r=t.currentRenderingDomain;r?(t.needsRollbackOnError?(d.info("Cleaning exports dir..."),await eQ(r)):d.info("No rollback needed, skipping..."),t.domains[r].isRendering=!1,t.domains[r].renderMode=he.ERROR,await eA(t)):d.warn("No currentRenderingDomain set, skipping cleanup")}catch{d.info("Early render stage, no db.json created yet...")}throw e}}var tQ=S(require("node:fs/promises")),rQ=S(require("node:path"));async function sQ(){let{version:A}=JSON.parse(await tQ.default.readFile(rQ.default.resolve(__dirname,"../..","package.json"),"utf-8")),e=`
 ${K.yellow(`Griddo Exporter ${A}`)}
 ${K.dim(`Node ${process.version.slice(1)}`)}
-`;d.log(e)}var SD=S(RD());function ND(A={}){let{timeoutMs:e}=A;return{async request(t){let{statusCode:r,headers:s,body:n}=await(0,SD.request)(t.url,{method:t.method,headers:t.headers,body:t.body,signal:e?AbortSignal.timeout(e):void 0}),o={};for(let[c,g]of Object.entries(s))g!==void 0&&(o[c]=Array.isArray(g)?g.join(", "):String(g));let i,a=async()=>(i===void 0&&(i=await n.text()),i);return{status:r,statusText:"",ok:r>=200&&r<300,headers:o,json:async()=>JSON.parse(await a()),text:a}}}}var Ji=class extends Error{constructor(e){super(`Circuit breaker is open. Cooldown: ${e}ms`),this.name="CircuitOpenError"}};function XE(A,e){let{failureThreshold:t,cooldownMs:r,onOpen:s,onClose:n}=e,o="closed",i=0,a=0;function c(){o==="half-open"?(o="closed",i=0,n?.()):i=0}function g(){i++,i>=t&&(o="open",a=Date.now(),s?.())}function Q(){return o==="closed"?!0:o==="open"&&Date.now()-a>=r?(o="half-open",!0):o==="half-open"}return{async request(u){if(!Q())throw new Ji(r);try{let l=await A.request(u);return c(),l}catch(l){throw g(),l}}}}function FD(A){return new Promise(e=>setTimeout(e,A))}function bD(A,e,t,r){let s=t==="exponential"?A*2**(e-1):A;return r&&(s=Math.round(s*(.5+Math.random()))),s}function KE(A,e){let{attempts:t,delayMs:r,backoff:s="fixed",jitter:n=!1,retryOn:o,onRetry:i}=e;return{async request(a){let c;for(let g=1;g<=t;g++)try{let Q=await A.request(a);if(o?.(Q)&&g<t){let u=bD(r,g,s,n);i?.({request:a,attempt:g,delayMs:u,response:Q}),await FD(u);continue}return Q}catch(Q){if(c=Q,g<t){let u=bD(r,g,s,n);i?.({request:a,attempt:g,delayMs:u,error:c}),await FD(u)}}throw c}}}var zE;function Mn(){return zE||(zE=XE(KE(ND({timeoutMs:Yu}),{attempts:da,delayMs:Hu*1e3,backoff:"exponential",jitter:!0,retryOn:A=>A.status>=500,onRetry:({request:A,attempt:e,delayMs:t,error:r,response:s})=>{let n=s?`HTTP ${s.status}`:r?.message;d.warn(`Retry ${e}/${da}: ${A.method} ${A.url} \u2014 ${n} (next in ${t}ms)`)}}),{failureThreshold:Gu,cooldownMs:Ou,onOpen:()=>d.warn("Circuit breaker OPEN \u2014 requests will fail fast"),onClose:()=>d.info("Circuit breaker CLOSED \u2014 recovered")}),d.verbose("HTTP adapter: undici (timeout + retry + circuit breaker)")),zE}var fA=process.env.GRIDDO_API_URL,rO=process.env.GRIDDO_PUBLIC_API_URL,ZJ=`${fA}/ai/embeddings`,jJ=`${rO}/alert`,XJ=`${fA}/domains`,UD=`${fA}/sites/all`,MD=`${fA}/page`,TD=`${fA}/login_check`,KJ=`${fA}/debug/reset-render`,kD=`${fA}/domains/robots`,zJ=`${fA}/search`,LD=`${fA}/settings`,sO=`${fA}/domains/`,xD=[sO,"/llms"],Br=`${fA}/site/`,HD=[Br,"/build/end"],YD=[Br,"/build/start"],GD=[Br,"/distributor"],OD=[Br,"/sitemap"],vD=[Br,"/all"],VD=[Br,"/languages"],JD=[Br,"/socials"];var PD={name:"@griddo/cx",description:"Griddo SSG based on Gatsby",version:"11.14.1",authors:["Hisco <francis.vega@griddo.io>"],license:"UNLICENSED",homepage:"https://griddo.io",repository:{type:"git",url:"https://github.com/griddo/griddo"},bin:{"griddo-render":"cli.mjs"},exports:{".":{types:"./build/index.d.ts",import:"./build/index.js",require:"./build/index.js"},"./react":{types:"./build/react/index.d.ts",import:"./build/react/index.js",require:"./build/react/index.js"}},scripts:{"// NPM":"",prepare:"yarn run build","// BUILD":"",build:"rm -rf build && sh ./exporter/build.sh","build:debug":"rm -rf build && sh ./exporter/build.sh --debug","// TESTS":"",test:"vitest run","test:watch":"vitest","// INFRA SCRIPTS":"","prepare-domains-render":"node ./build/commands/prepare-domains-render","start-render":"node ./build/commands/start-render","end-render":"node ./build/commands/end-render","upload-search-content":"node ./build/commands/upload-search-content","notify-indexnow":"node ./build/commands/notify-indexnow","reset-render":"node ./build/commands/reset-render","// ONLY LOCAL SCRIPTS":"","prepare-assets-directory":"node ./build/commands/prepare-assets-directory","create-rollback-copy":"rm -rf ../../exports-backup && cp -r ../../exports ../../exports-backup",render:"npm run build && node --env-file=.env cli.mjs render","// LINTER & FORMATTER":"",lint:"biome check --write",format:"biome format --write",flint:"npm run lint && npm run format","ts-lint":"tsgo --noEmit","watch:ts-lint":"tsc --noEmit --watch"},dependencies:{"@kreuzberg/html-to-markdown":"2.24.5",gatsby:"5.15.0","html-react-parser":"5.2.11",jiti:"^2.6.1",undici:"7.24.6"},devDependencies:{"@biomejs/biome":"2.3.4","@griddo/core":"11.14.1","@types/node":"20.19.4","@typescript/native-preview":"7.0.0-dev.20260401.1",cheerio:"1.1.2",esbuild:"0.25.12","p-limit":"7.2.0",typescript:"6.0.2",vitest:"^4.0.18"},peerDependencies:{react:">=18 <19","react-dom":">=18 <19"},engines:{node:">=20.19"},files:["build","exporter","src","gatsby-browser.tsx","gatsby-config.ts","gatsby-node.ts","gatsby-ssr.tsx","global.d.ts","tsconfig.commands.json","tsconfig.exporter.json","tsconfig.json","plugins","cli.mjs"],publishConfig:{access:"public"},gitHead:"e5aea08976bf7de8fa5c0732992813d09401efd2"};var Pi={"x-application-id":"griddo-cx","x-client-version":PD.version,"x-client-name":"CX"};var oO=Mn(),$E=class{headers;async login(){try{let e=await oO.request({url:TD,method:"POST",headers:Object.assign({},Pi,{"Content-Type":"application/json",Connection:"close"}),body:JSON.stringify({username:Fu,password:bu})});if(!e.ok)throw new Error("Error while login in the API");let{token:t}=await e.json();if(!t||typeof t!="string")throw new Error("Login response missing valid token");return this.headers={Authorization:`bearer ${t}`,"Cache-Control":"no-store"},this.headers}catch(e){Ie(Pu,e)}}},At=new $E;var Dw=S(require("node:path"));async function iO(A){let e=process.hrtime.bigint();await Promise.resolve(A());let t=process.hrtime.bigint();return Number((Number(t-e)/1e9).toFixed(3))}async function we(A,e,t){let{maxRetrys:r=vu,skip:s}=t||{};if(s){d.info(`Skiping life-cycle ${A}. Reason: ${s}
+`;d.log(e)}var SD=S(RD());function ND(A={}){let{timeoutMs:e}=A;return{async request(t){let{statusCode:r,headers:s,body:n}=await(0,SD.request)(t.url,{method:t.method,headers:t.headers,body:t.body,signal:e?AbortSignal.timeout(e):void 0}),o={};for(let[c,g]of Object.entries(s))g!==void 0&&(o[c]=Array.isArray(g)?g.join(", "):String(g));let i,a=async()=>(i===void 0&&(i=await n.text()),i);return{status:r,statusText:"",ok:r>=200&&r<300,headers:o,json:async()=>JSON.parse(await a()),text:a}}}}var Ji=class extends Error{constructor(e){super(`Circuit breaker is open. Cooldown: ${e}ms`),this.name="CircuitOpenError"}};function XE(A,e){let{failureThreshold:t,cooldownMs:r,onOpen:s,onClose:n}=e,o="closed",i=0,a=0;function c(){o==="half-open"?(o="closed",i=0,n?.()):i=0}function g(){i++,i>=t&&(o="open",a=Date.now(),s?.())}function Q(){return o==="closed"?!0:o==="open"&&Date.now()-a>=r?(o="half-open",!0):o==="half-open"}return{async request(u){if(!Q())throw new Ji(r);try{let l=await A.request(u);return c(),l}catch(l){throw g(),l}}}}function FD(A){return new Promise(e=>setTimeout(e,A))}function bD(A,e,t,r){let s=t==="exponential"?A*2**(e-1):A;return r&&(s=Math.round(s*(.5+Math.random()))),s}function KE(A,e){let{attempts:t,delayMs:r,backoff:s="fixed",jitter:n=!1,retryOn:o,onRetry:i}=e;return{async request(a){let c;for(let g=1;g<=t;g++)try{let Q=await A.request(a);if(o?.(Q)&&g<t){let u=bD(r,g,s,n);i?.({request:a,attempt:g,delayMs:u,response:Q}),await FD(u);continue}return Q}catch(Q){if(c=Q,g<t){let u=bD(r,g,s,n);i?.({request:a,attempt:g,delayMs:u,error:c}),await FD(u)}}throw c}}}var zE;function Mn(){return zE||(zE=XE(KE(ND({timeoutMs:Yu}),{attempts:da,delayMs:Hu*1e3,backoff:"exponential",jitter:!0,retryOn:A=>A.status>=500,onRetry:({request:A,attempt:e,delayMs:t,error:r,response:s})=>{let n=s?`HTTP ${s.status}`:r?.message;d.warn(`Retry ${e}/${da}: ${A.method} ${A.url} \u2014 ${n} (next in ${t}ms)`)}}),{failureThreshold:Gu,cooldownMs:Ou,onOpen:()=>d.warn("Circuit breaker OPEN \u2014 requests will fail fast"),onClose:()=>d.info("Circuit breaker CLOSED \u2014 recovered")}),d.verbose("HTTP adapter: undici (timeout + retry + circuit breaker)")),zE}var fA=process.env.GRIDDO_API_URL,rO=process.env.GRIDDO_PUBLIC_API_URL,ZJ=`${fA}/ai/embeddings`,jJ=`${rO}/alert`,XJ=`${fA}/domains`,UD=`${fA}/sites/all`,MD=`${fA}/page`,TD=`${fA}/login_check`,KJ=`${fA}/debug/reset-render`,kD=`${fA}/domains/robots`,zJ=`${fA}/search`,LD=`${fA}/settings`,sO=`${fA}/domains/`,xD=[sO,"/llms"],Br=`${fA}/site/`,HD=[Br,"/build/end"],YD=[Br,"/build/start"],GD=[Br,"/distributor"],OD=[Br,"/sitemap"],vD=[Br,"/all"],VD=[Br,"/languages"],JD=[Br,"/socials"];var PD={name:"@griddo/cx",description:"Griddo SSG based on Gatsby",version:"11.14.2-rc.1",authors:["Hisco <francis.vega@griddo.io>"],license:"UNLICENSED",homepage:"https://griddo.io",repository:{type:"git",url:"https://github.com/griddo/griddo"},bin:{"griddo-render":"cli.mjs"},exports:{".":{types:"./build/index.d.ts",import:"./build/index.js",require:"./build/index.js"},"./react":{types:"./build/react/index.d.ts",import:"./build/react/index.js",require:"./build/react/index.js"}},scripts:{"// NPM":"",prepare:"yarn run build","// BUILD":"",build:"rm -rf build && sh ./exporter/build.sh","build:debug":"rm -rf build && sh ./exporter/build.sh --debug","// TESTS":"",test:"vitest run","test:watch":"vitest","// INFRA SCRIPTS":"","prepare-domains-render":"node ./build/commands/prepare-domains-render","start-render":"node ./build/commands/start-render","end-render":"node ./build/commands/end-render","upload-search-content":"node ./build/commands/upload-search-content","notify-indexnow":"node ./build/commands/notify-indexnow","reset-render":"node ./build/commands/reset-render","// ONLY LOCAL SCRIPTS":"","prepare-assets-directory":"node ./build/commands/prepare-assets-directory","create-rollback-copy":"rm -rf ../../exports-backup && cp -r ../../exports ../../exports-backup",render:"npm run build && node --env-file=.env cli.mjs render","// LINTER & FORMATTER":"",lint:"biome check --write",format:"biome format --write",flint:"npm run lint && npm run format","ts-lint":"tsgo --noEmit","watch:ts-lint":"tsc --noEmit --watch"},dependencies:{"@kreuzberg/html-to-markdown":"2.24.5",gatsby:"5.15.0","html-react-parser":"5.2.11",jiti:"^2.6.1",undici:"7.24.6"},devDependencies:{"@biomejs/biome":"2.3.4","@griddo/core":"11.14.2-rc.1","@types/node":"20.19.4","@typescript/native-preview":"7.0.0-dev.20260401.1",cheerio:"1.1.2",esbuild:"0.25.12","p-limit":"7.2.0",typescript:"6.0.2",vitest:"^4.0.18"},peerDependencies:{react:">=18 <19","react-dom":">=18 <19"},engines:{node:">=20.19"},files:["build","exporter","src","gatsby-browser.tsx","gatsby-config.ts","gatsby-node.ts","gatsby-ssr.tsx","global.d.ts","tsconfig.commands.json","tsconfig.exporter.json","tsconfig.json","plugins","cli.mjs"],publishConfig:{access:"public"},gitHead:"a08bea26f0f939c8f93c9d59b5d54e80fd776700"};var Pi={"x-application-id":"griddo-cx","x-client-version":PD.version,"x-client-name":"CX"};var oO=Mn(),$E=class{headers;async login(){try{let e=await oO.request({url:TD,method:"POST",headers:Object.assign({},Pi,{"Content-Type":"application/json",Connection:"close"}),body:JSON.stringify({username:Fu,password:bu})});if(!e.ok)throw new Error("Error while login in the API");let{token:t}=await e.json();if(!t||typeof t!="string")throw new Error("Login response missing valid token");return this.headers={Authorization:`bearer ${t}`,"Cache-Control":"no-store"},this.headers}catch(e){Ie(Pu,e)}}},At=new $E;var Dw=S(require("node:path"));async function iO(A){let e=process.hrtime.bigint();await Promise.resolve(A());let t=process.hrtime.bigint();return Number((Number(t-e)/1e9).toFixed(3))}async function we(A,e,t){let{maxRetrys:r=vu,skip:s}=t||{};if(s){d.info(`Skiping life-cycle ${A}. Reason: ${s}
 `);return}let n=0;for(;n<r;)try{d.info(`start ${A} life-cycle`);let o=await iO(e);d.success(`${A} - ${o}s
 `);break}catch(o){let i=K.red(` Error in ${A} LifeCycle `),a=K.yellow(`Attempt (${n+1})`);d.log(`
 ${i} ${a}
@@ -170,7 +170,7 @@ ${A} ${s}`)}async function Lm(A){console.info(`API calls with ${ps} threads`),co
 
 `).trim(),s}async function*Jm(A){let e=await oa.default.opendir(A);for await(let t of e)t.isDirectory()?yield*Jm(dr.default.join(A,t.name)):t.isFile()&&dr.default.extname(t.name)===".html"&&(yield dr.default.join(A,t.name))}async function*xO(A,e){let t=Jm(A);for await(let r of t)try{let s=await oa.default.readFile(r,"utf8"),n=LO(s,e);yield{path:r,content:n}}catch(s){Ie(Wu,s)}}var ga=S(require("node:fs/promises")),ca=S(require("node:path"));var aa=A=>A.replace(/\/+$/,"");async function HO(A){try{return await ga.default.readFile(A,"utf-8")}catch(e){return e.code==="ENOENT"?d.verbose(`client template for llms.txt file not found at ${A}, using defaults.`):d.error(`Unexpected error reading ${A}:`,e),`{{ LLMS_HEADER }}
 
-{{ LLMS_PAGE_LINKS }}`}}async function Pm(A){if(Tu){d.verbose(`${A} skipped llms.txt generation: disabled by environment variable.`);return}let e=ys?".md":"";try{let r=(await ee()).domains[A];if(!r?.id)throw new Error(`Domain configuration or ID not found for domain: ${A}`);let s=r.id,n=[];try{let[B,y]=xD,C=`${B}${s}${y}`;n=await Ye({endpoint:C})}catch(B){d.error(`Failed to fetch LLM data for domain ${A}. Aborting llms.txt generation.`,B);return}let o=n.map(({title:B,url:y,socialDescription:C})=>{let f=C?`: ${C}`:"",D=aa(y),M=(()=>{try{return new URL(y).pathname==="/"}catch{return y==="/"||y===""}})(),F=ys&&M?`${D}/index.md`:`${D}${e}`;return`- [${B}](${F})${f}`}).join(`
+{{ LLMS_PAGE_LINKS }}`}}async function Pm(A){if(Tu){d.verbose(`${A} skipped llms.txt generation: disabled by environment variable.`);return}let e=ys?".md":"";try{let r=(await ee()).domains[A];if(!r?.id)throw new Error(`Domain configuration or ID not found for domain: ${A}`);let s=r.id,n=[];try{let[B,y]=xD,C=`${B}${s}${y}`;n=await Ye({endpoint:C})}catch(B){d.error(`Failed to fetch LLM data for domain ${A}. Aborting llms.txt generation.`,B);return}let o=n.map(({title:B,url:y,description:C})=>{let f=C?`: ${C}`:"",D=aa(y),M=(()=>{try{return new URL(y).pathname==="/"}catch{return y==="/"||y===""}})(),F=ys&&M?`${D}/index.md`:`${D}${e}`;return`- [${B}](${F})${f}`}).join(`
 `),{__root:i,__components:a}=await q({domain:A}),c=ca.default.join(a,"static/llms.md"),g=await HO(c),Q=`llms.txt for the domain ${A}
 Generated: ${new Date().toISOString().replace("T"," ").slice(0,19)}`,u=g.replace(/\{\{\s*LLMS_HEADER\s*\}\}/g,`${Q}`).replace(/\{\{\s*LLMS_PAGE_LINKS\s*\}\}/g,`${o}`),l=ca.default.join(i,"current-dist"),E=ca.default.join(l,"llms.txt");await ga.default.mkdir(l,{recursive:!0}),await ga.default.writeFile(E,u,"utf-8"),d.verbose(`Successfully generated llms.txt at ${E}`)}catch(t){d.error(`Critical error generating llms.txt for ${A}:`,t)}}async function qm(A){let{domain:e}=A;await Pm(e),await Vm(e)}async function Wm(A){let{domain:e}=A;await jD(e)}var _m=require("node:crypto"),uu=S(require("node:fs/promises")),Qu=S(require("node:path"));var YO=32;function Zm(A){return(0,_m.createHash)("sha256").update(A,"utf-8").digest("hex").slice(0,YO)}function hu(A){return Pn?`${Zm(A)}.txt`:null}async function jm(A){let e=hu(A);if(!e)return;let t=Zm(A),{__root:r}=await q({domain:A}),s=Qu.default.join(r,"current-dist"),n=Qu.default.join(s,e);try{await uu.default.mkdir(s,{recursive:!0}),await uu.default.writeFile(n,t,"utf-8"),d.verbose(`indexnow: wrote host-key file ${n}`)}catch(o){d.warn(`indexnow: failed to write host-key file ${n}: ${o?.message??String(o)}`)}}var Xm=S(require("node:fs/promises")),Iu=S(require("node:path"));async function GO(){return(await Ye({endpoint:kD}))?.filter(e=>!!e.path).map(({path:e,content:t})=>({path:e,content:t||`User-agent: *
 \r