docs-cache 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,8 @@
1
- import{rm as C,mkdtemp as A,writeFile as G,mkdir as _,access as Y,rename as B,open as N,lstat as fe,cp as me,symlink as de,readFile as X}from"node:fs/promises";import h from"node:path";import E from"picocolors";import{t as R,r as j,D as he,g as we,u as D,s as P,a as pe}from"../shared/docs-cache.D9_kM5zq.mjs";import{a as H,l as ye,D as ge,b as Se}from"../shared/docs-cache.goBsJvLg.mjs";import{execFile as W}from"node:child_process";import De,{tmpdir as J}from"node:os";import{promisify as K}from"node:util";import{writeLock as ve,resolveLockPath as xe,readLock as Ce}from"../lock.mjs";import{M as U,v as q}from"./verify.mjs";import{createHash as V,randomBytes as Pe}from"node:crypto";import{createWriteStream as Z,createReadStream as $e,constants as Q}from"node:fs";import{pipeline as Ee}from"node:stream/promises";import ee from"fast-glob";const Me=/^(https?:\/\/)([^@]+)@/i,b=e=>e.replace(Me,"$1***@"),Oe=K(W),Te=3e4,Fe=new Set(["file:","ftp:","data:","javascript:"]),Ie=e=>{try{const t=new URL(e);if(Fe.has(t.protocol))throw new Error(`Blocked protocol '${t.protocol}' in repo URL '${b(e)}'.`)}catch(t){if(t instanceof TypeError)return;throw t}},be=e=>{if(Ie(e),e.startsWith("git@")){const t=e.indexOf("@"),r=e.indexOf(":",t+1);return r===-1?null:e.slice(t+1,r)||null}try{const t=new URL(e);return t.protocol!=="https:"&&t.protocol!=="ssh:"?null:t.hostname||null}catch{return null}},te=(e,t)=>{const r=be(e);if(!r)throw new Error(`Unsupported repo URL '${b(e)}'. Use HTTPS or SSH.`);const s=r.toLowerCase();if(!t.map(o=>o.toLowerCase()).includes(s))throw new Error(`Host '${r}' is not in allowHosts for '${b(e)}'.`)},re=e=>{const t=e.trim().split(`
2
- `).filter(Boolean);return t.length===0?null:t[0].split(/\s+/)[0]||null},ke=async e=>{te(e.repo,e.allowHosts);const{stdout:t}=await Oe("git",["ls-remote",e.repo,e.ref],{timeout:e.timeoutMs??Te,maxBuffer:1024*1024}),r=re(t);if(!r)throw new Error(`Unable to resolve ref '${e.ref}' for ${b(e.repo)}.`);return{repo:e.repo,ref:e.ref,resolvedCommit:r}},oe=K(W),se=3e4,L=async(e,t)=>{await oe("git",["-c","core.hooksPath=/dev/null","-c","submodule.recurse=false","-c","protocol.file.allow=never","-c","protocol.ext.allow=never",...e],{cwd:t?.cwd,timeout:t?.timeoutMs??se,maxBuffer:1024*1024,env:{PATH:process.env.PATH,HOME:process.env.HOME,USER:process.env.USER,USERPROFILE:process.env.USERPROFILE,TMPDIR:process.env.TMPDIR,TMP:process.env.TMP,TEMP:process.env.TEMP,SYSTEMROOT:process.env.SYSTEMROOT,WINDIR:process.env.WINDIR,SSH_AUTH_SOCK:process.env.SSH_AUTH_SOCK,SSH_AGENT_PID:process.env.SSH_AGENT_PID,HTTP_PROXY:process.env.HTTP_PROXY,HTTPS_PROXY:process.env.HTTPS_PROXY,NO_PROXY:process.env.NO_PROXY,GIT_TERMINAL_PROMPT:"0",GIT_CONFIG_NOSYSTEM:"1",GIT_CONFIG_NOGLOBAL:"1",...process.platform==="win32"?{}:{GIT_ASKPASS:"/bin/false"}}})},_e=async(e,t,r,s)=>{const o=h.join(r,"archive.tar");await L(["archive","--remote",e,"--format=tar","--output",o,t],{timeoutMs:s}),await oe("tar",["-xf",o,"-C",r],{timeout:s??se,maxBuffer:1024*1024}),await C(o,{force:!0})},Re=e=>{if(!e||e.length===0)return!1;for(const t of e)if(!t||t.includes("**"))return!1;return!0},Le=e=>{if(!e)return[];const t=e.map(r=>{const s=r.replace(/\\/g,"/"),o=s.indexOf("*");return(o===-1?s:s.slice(0,o)).replace(/\/+$|\/$/,"")});return Array.from(new Set(t.filter(r=>r.length>0)))},Ae=async(e,t)=>{const r=/^[0-9a-f]{7,40}$/i.test(e.ref),s=Re(e.include),o=["clone","--no-checkout","--filter=blob:none","--depth",String(e.depth),"--recurse-submodules=no","--no-tags"];if(s&&o.push("--sparse"),r||(o.push("--single-branch"),e.ref!=="HEAD"&&o.push("--branch",e.ref)),o.push(e.repo,t),await L(o,{timeoutMs:e.timeoutMs}),s){const l=Le(e.include);l.length>0&&await L(["-C",t,"sparse-checkout","set",...l],{timeoutMs:e.timeoutMs})}await L(["-C",t,"checkout","--detach",e.resolvedCommit],{timeoutMs:e.timeoutMs})},Be=async e=>{const t=await A(h.join(J(),`docs-cache-${e.sourceId}-`));try{return await _e(e.repo,e.resolvedCommit,t,e.timeoutMs),t}catch(r){throw await C(t,{recursive:!0,force:!0}),r}},Ne=async e=>{H(e.sourceId,"sourceId");try{const t=await Be(e);return{repoDir:t,cleanup:async()=>{await C(t,{recursive:!0,force:!0})}}}catch{const t=await A(h.join(J(),`docs-cache-${e.sourceId}-`));try{return await Ae(e,t),{repoDir:t,cleanup:async()=>{await C(t,{recursive:!0,force:!0})}}}catch(r){throw await C(t,{recursive:!0,force:!0}),r}}},je=async e=>{const t=new Map(e.sources.map(m=>[m.id,m])),r={};for(const[m,w]of Object.entries(e.lock.sources)){const u=t.get(m),f=u?.targetDir?R(j(e.configPath,u.targetDir)):void 0;r[m]={repo:w.repo,ref:w.ref,resolvedCommit:w.resolvedCommit,bytes:w.bytes,fileCount:w.fileCount,manifestSha256:w.manifestSha256,updatedAt:w.updatedAt,cachePath:R(h.join(e.cacheDir,m)),...f?{targetDir:f}:{}}}const s={generatedAt:new Date().toISOString(),cacheDir:R(e.cacheDir),sources:r},o=h.join(e.cacheDir,he),l=`${JSON.stringify(s,null,2)}
3
- `;await G(o,l,"utf8")},F=e=>R(e),z=Number(process.env.DOCS_CACHE_STREAM_THRESHOLD_MB??"2"),He=Number.isFinite(z)&&z>0?Math.floor(z*1024*1024):1024*1024,Ue=(e,t)=>{const r=h.resolve(e);if(!h.resolve(t).startsWith(r+h.sep))throw new Error(`Path traversal detected: ${t}`)},ie=async e=>{try{return await N(e,Q.O_RDONLY|Q.O_NOFOLLOW)}catch(t){const r=t.code;if(r==="ELOOP")return null;if(r==="EINVAL"||r==="ENOSYS"||r==="ENOTSUP")return(await fe(e)).isSymbolicLink()?null:await N(e,"r");throw t}},ze=async(e,t=5e3)=>{const r=Date.now();for(;Date.now()-r<t;)try{const s=await N(e,"wx");return{release:async()=>{await s.close(),await C(e,{force:!0})}}}catch(s){if(s.code!=="EEXIST")throw s;await new Promise(o=>setTimeout(o,100))}throw new Error(`Failed to acquire lock for ${e}.`)},Ge=async e=>{H(e.sourceId,"sourceId");const t=we(e.cacheDir,e.sourceId);await _(e.cacheDir,{recursive:!0});const r=await A(h.join(e.cacheDir,`.tmp-${e.sourceId}-`));try{const s=await ee(e.include,{cwd:e.repoDir,ignore:[".git/**",...e.exclude??[]],dot:!0,onlyFiles:!0,followSymbolicLinks:!1});s.sort((i,c)=>F(i).localeCompare(F(c)));const o=new Set;for(const i of s)o.add(h.dirname(i));await Promise.all(Array.from(o,i=>_(h.join(r,i),{recursive:!0})));let l=0,m=0;const w=Math.max(1,Math.min(s.length,Math.max(8,Math.min(128,De.cpus().length*8)))),u=h.join(r,U),f=Z(u,{encoding:"utf8"}),S=V("sha256"),v=async i=>new Promise((c,y)=>{const n=p=>{f.off("drain",d),y(p)},d=()=>{f.off("error",n),c()};f.once("error",n),f.write(i)?(f.off("error",n),c()):f.once("drain",d)});for(let i=0;i<s.length;i+=w){const c=s.slice(i,i+w),y=await Promise.all(c.map(async n=>{const d=F(n),p=h.join(e.repoDir,n),a=await ie(p);if(!a)return null;try{const $=await a.stat();if(!$.isFile())return null;const M=h.join(r,n);if(Ue(r,M),$.size>=He){const I=$e(p,{fd:a.fd,autoClose:!1}),O=Z(M);await Ee(I,O)}else{const I=await a.readFile();await G(M,I)}return{path:d,size:$.size}}finally{await a.close()}}));for(const n of y){if(!n)continue;if(e.maxFiles!==void 0&&m+1>e.maxFiles)throw new Error(`Materialized content exceeds maxFiles (${e.maxFiles}).`);if(l+=n.size,l>e.maxBytes)throw new Error(`Materialized content exceeds maxBytes (${e.maxBytes}).`);const d=`${JSON.stringify(n)}
4
- `;S.update(d),await v(d),m+=1}}await new Promise((i,c)=>{f.end(()=>i()),f.once("error",c)});const x=S.digest("hex"),g=async i=>{try{return await Y(i),!0}catch{return!1}};return await(async(i,c)=>{const y=await ze(`${c}.lock`);try{const n=await g(c),d=`${c}.bak-${Pe(8).toString("hex")}`;n&&await B(c,d);try{await B(i,c)}catch(p){if(n)try{await B(d,c)}catch(a){const $=a instanceof Error?a.message:String(a);process.stderr.write(`Warning: Failed to restore backup: ${$}
5
- `)}throw p}n&&await C(d,{recursive:!0,force:!0})}finally{await y.release()}})(r,t.sourceDir),{bytes:l,fileCount:m,manifestSha256:x}}catch(s){throw await C(r,{recursive:!0,force:!0}),s}},Ye=async e=>{H(e.sourceId,"sourceId");const t=await ee(e.include,{cwd:e.repoDir,ignore:[".git/**",...e.exclude??[]],dot:!0,onlyFiles:!0,followSymbolicLinks:!1});t.sort((l,m)=>F(l).localeCompare(F(m)));let r=0,s=0;const o=V("sha256");for(const l of t){const m=F(l),w=h.join(e.repoDir,l),u=await ie(w);if(u)try{const f=await u.stat();if(!f.isFile())continue;if(e.maxFiles!==void 0&&s+1>e.maxFiles)throw new Error(`Materialized content exceeds maxFiles (${e.maxFiles}).`);if(r+=f.size,r>e.maxBytes)throw new Error(`Materialized content exceeds maxBytes (${e.maxBytes}).`);const S=`${JSON.stringify({path:m,size:f.size})}
6
- `;o.update(S),s+=1}finally{await u.close()}}return{bytes:r,fileCount:s,manifestSha256:o.digest("hex")}},Xe=async e=>{await C(e,{recursive:!0,force:!0})},ae=async e=>{const t=h.dirname(e.targetDir);await _(t,{recursive:!0}),await Xe(e.targetDir);const r=process.platform==="win32"?"copy":"symlink";if((e.mode??r)==="copy"){await me(e.sourceDir,e.targetDir,{recursive:!0});return}const s=process.platform==="win32"?"junction":"dir";await de(e.sourceDir,e.targetDir,s)},We=e=>{if(e<1024)return`${e} B`;const t=["KB","MB","GB","TB"];let r=e,s=-1;for(;r>=1024&&s<t.length-1;)r/=1024,s+=1;return`${r.toFixed(1)} ${t[s]}`},k=async e=>{try{return await Y(e),!0}catch{return!1}},ne=async(e,t)=>{const r=h.join(e,t);return await k(r)?await k(h.join(r,U)):!1},ce=async(e,t={})=>{const{config:r,resolvedPath:s,sources:o}=await ye(e.configPath),l=r.defaults??ge.defaults,m=pe(s,r.cacheDir??Se,e.cacheDirOverride),w=xe(s),u=await k(w);let f=null;u&&(f=await Ce(w));const S=t.resolveRemoteCommit??ke,v=e.sourceFilter?.length?o.filter(g=>e.sourceFilter?.includes(g.id)):o,x=await Promise.all(v.map(async g=>{const i=f?.sources?.[g.id];if(e.offline){const d=await ne(m,g.id);return{id:g.id,repo:i?.repo??g.repo,ref:i?.ref??g.ref??l.ref,resolvedCommit:i?.resolvedCommit??"offline",lockCommit:i?.resolvedCommit??null,status:i&&d?"up-to-date":"missing",bytes:i?.bytes,fileCount:i?.fileCount,manifestSha256:i?.manifestSha256}}const c=await S({repo:g.repo,ref:g.ref,allowHosts:l.allowHosts,timeoutMs:e.timeoutMs}),y=i?.resolvedCommit===c.resolvedCommit,n=i?y?"up-to-date":"changed":"missing";return{id:g.id,repo:c.repo,ref:c.ref,resolvedCommit:c.resolvedCommit,lockCommit:i?.resolvedCommit??null,status:n,bytes:i?.bytes,fileCount:i?.fileCount,manifestSha256:i?.manifestSha256}}));return{config:r,configPath:s,cacheDir:m,lockPath:w,lockExists:u,lockData:f,results:x,sources:v,defaults:l}},Je=async()=>{const e=h.resolve(process.cwd(),"package.json");try{const t=await X(e,"utf8"),r=JSON.parse(t.toString());return typeof r.version=="string"?r.version:"0.0.0"}catch{}try{const t=await X(new URL("../package.json",import.meta.url),"utf8"),r=JSON.parse(t.toString());return typeof r.version=="string"?r.version:"0.0.0"}catch{return"0.0.0"}},Ke=async(e,t)=>{const r=await Je(),s=new Date().toISOString(),o={...t?.sources??{}};for(const l of e.results){const m=o[l.id];o[l.id]={repo:l.repo,ref:l.ref,resolvedCommit:l.resolvedCommit,bytes:l.bytes??m?.bytes??0,fileCount:l.fileCount??m?.fileCount??0,manifestSha256:l.manifestSha256??m?.manifestSha256??l.resolvedCommit,updatedAt:s}}return{version:1,generatedAt:s,toolVersion:r,sources:o}},le=async(e,t={})=>{const r=process.hrtime.bigint();let s=0;const o=await ce(e,t);await _(o.cacheDir,{recursive:!0});const l=o.lockData,m=o.results.filter(u=>{const f=o.sources.find(S=>S.id===u.id);return u.status==="missing"&&(f?.required??!0)});if(e.failOnMiss&&m.length>0)throw new Error(`Missing required source(s): ${m.map(u=>u.id).join(", ")}.`);if(!e.lockOnly){const u=o.defaults,f=t.fetchSource??Ne,S=t.materializeSource??Ge,v=async(i,c)=>{const y=i?.length?o.results.filter(n=>i.includes(n.id)):o.results;return(await Promise.all(y.map(async n=>{const d=o.sources.find(a=>a.id===n.id);if(!d)return null;const p=await ne(o.cacheDir,n.id);return c||n.status!=="up-to-date"||!p?{result:n,source:d}:null}))).filter(Boolean)},x=async()=>{await Promise.all(o.sources.map(async i=>{if(!i.targetDir)return;const c=j(o.configPath,i.targetDir);await k(c)||await ae({sourceDir:h.join(o.cacheDir,i.id),targetDir:c,mode:i.targetMode??u.targetMode})}))},g=async i=>{const c=e.concurrency??4;let y=0;const n=async()=>{const d=i[y];if(!d||!d.source)return;y+=1;const{result:p,source:a}=d,$=o.lockData?.sources?.[a.id];e.json||D.step("Fetching",a.id);const M=await f({sourceId:a.id,repo:a.repo,ref:a.ref,resolvedCommit:p.resolvedCommit,cacheDir:o.cacheDir,depth:a.depth??u.depth,include:a.include??u.include,timeoutMs:e.timeoutMs});try{const I=h.join(o.cacheDir,a.id,U);if(p.status!=="up-to-date"&&$?.manifestSha256&&await k(I)){const T=await Ye({sourceId:a.id,repoDir:M.repoDir,cacheDir:o.cacheDir,include:a.include??u.include,exclude:a.exclude,maxBytes:a.maxBytes??u.maxBytes,maxFiles:a.maxFiles??u.maxFiles});if(T.manifestSha256===$.manifestSha256){p.bytes=T.bytes,p.fileCount=T.fileCount,p.manifestSha256=T.manifestSha256,p.status="up-to-date",e.json||D.item(P.success,a.id,"no content changes"),await n();return}}const O=await S({sourceId:a.id,repoDir:M.repoDir,cacheDir:o.cacheDir,include:a.include??u.include,exclude:a.exclude,maxBytes:a.maxBytes??u.maxBytes,maxFiles:a.maxFiles??u.maxFiles});if(a.targetDir){const T=j(o.configPath,a.targetDir);await ae({sourceDir:h.join(o.cacheDir,a.id),targetDir:T,mode:a.targetMode??u.targetMode})}p.bytes=O.bytes,p.fileCount=O.fileCount,p.manifestSha256=O.manifestSha256,e.json||D.item(P.success,a.id,`synced ${O.fileCount} files`)}finally{await M.cleanup()}await n()};await Promise.all(Array.from({length:Math.min(c,i.length)},n))};if(e.offline)await x();else{const i=await v();await g(i),await x()}if(!e.offline){const i=(await q({configPath:o.configPath,cacheDirOverride:o.cacheDir})).results.filter(c=>!c.ok);if(i.length>0){const c=await v(i.map(n=>n.id),!0);c.length>0&&(await g(c),await x());const y=(await q({configPath:o.configPath,cacheDirOverride:o.cacheDir})).results.filter(n=>!n.ok);if(y.length>0&&(s+=1,!e.json)){const n=y.map(d=>`${d.id} (${d.issues.join("; ")})`).join(", ");D.line(`${P.warn} Verify failed for ${y.length} source(s): ${n}`)}}}}const w=await Ke(o,l);if(await ve(o.lockPath,w),!e.json){const u=Number(process.hrtime.bigint()-r)/1e6,f=o.results.reduce((v,x)=>v+(x.bytes??0),0),S=o.results.reduce((v,x)=>v+(x.fileCount??0),0);D.line(`${P.info} Completed in ${u.toFixed(0)}ms \xB7 ${We(f)} \xB7 ${S} files${s?` \xB7 ${s} warning${s===1?"":"s"}`:""}`)}return o.config.index&&await je({cacheDir:o.cacheDir,configPath:o.configPath,lock:w,sources:o.sources}),o.lockExists=!0,o},ue=e=>{const t={upToDate:e.results.filter(r=>r.status==="up-to-date").length,changed:e.results.filter(r=>r.status==="changed").length,missing:e.results.filter(r=>r.status==="missing").length};if(e.results.length===0){D.line(`${P.info} No sources to sync.`);return}D.line(`${P.info} ${e.results.length} sources (${t.upToDate} up-to-date, ${t.changed} changed, ${t.missing} missing)`);for(const r of e.results){const s=D.hash(r.resolvedCommit),o=D.hash(r.lockCommit);if(r.status==="up-to-date"){D.item(P.success,r.id,`${E.dim("up-to-date")} ${E.gray(s)}`);continue}if(r.status==="changed"){D.item(P.warn,r.id,`${E.dim("changed")} ${E.gray(o)} ${E.dim("->")} ${E.gray(s)}`);continue}D.item(P.warn,r.id,`${E.dim("missing")} ${E.gray(s)}`)}},qe={__proto__:null,getSyncPlan:ce,printSyncPlan:ue,runSync:le};export{ue as a,le as b,te as e,re as p,b as r,qe as s};
1
+ import{createHash as L,randomBytes as me}from"node:crypto";import{rm as $,mkdtemp as N,writeFile as W,mkdir as R,access as X,rename as B,open as j,lstat as he,symlink as we,cp as pe,readFile as J}from"node:fs/promises";import w from"node:path";import M from"picocolors";import{t as _,r as H,D as ge,g as ye,u as v,s as O,a as Se}from"../shared/docs-cache.D9_kM5zq.mjs";import{a as U,l as De,D as ve,b as xe}from"../shared/docs-cache.goBsJvLg.mjs";import{execFile as K}from"node:child_process";import Ce,{tmpdir as q}from"node:os";import{promisify as V}from"node:util";import{writeLock as Pe,resolveLockPath as Ee,readLock as Me}from"../lock.mjs";import{M as z,v as Z}from"./verify.mjs";import{createWriteStream as Q,createReadStream as Oe,constants as ee}from"node:fs";import{pipeline as $e}from"node:stream/promises";import te from"fast-glob";const Te=/^(https?:\/\/)([^@]+)@/i,I=e=>e.replace(Te,"$1***@"),ke=V(K),Fe=3e4,Ie=new Set(["file:","ftp:","data:","javascript:"]),be=e=>{try{const r=new URL(e);if(Ie.has(r.protocol))throw new Error(`Blocked protocol '${r.protocol}' in repo URL '${I(e)}'.`)}catch(r){if(r instanceof TypeError)return;throw r}},Re=e=>{if(be(e),e.startsWith("git@")){const r=e.indexOf("@"),t=e.indexOf(":",r+1);return t===-1?null:e.slice(r+1,t)||null}try{const r=new URL(e);return r.protocol!=="https:"&&r.protocol!=="ssh:"?null:r.hostname||null}catch{return null}},re=(e,r)=>{const t=Re(e);if(!t)throw new Error(`Unsupported repo URL '${I(e)}'. Use HTTPS or SSH.`);const i=t.toLowerCase();if(!r.map(o=>o.toLowerCase()).includes(i))throw new Error(`Host '${t}' is not in allowHosts for '${I(e)}'.`)},oe=e=>{const r=e.trim().split(`
2
+ `).filter(Boolean);return r.length===0?null:r[0].split(/\s+/)[0]||null},_e=async e=>{re(e.repo,e.allowHosts);const{stdout:r}=await ke("git",["ls-remote",e.repo,e.ref],{timeout:e.timeoutMs??Fe,maxBuffer:1024*1024}),t=oe(r);if(!t)throw new Error(`Unable to resolve ref '${e.ref}' for ${I(e.repo)}.`);return{repo:e.repo,ref:e.ref,resolvedCommit:t}},se=V(K),ie=3e4,A=async(e,r)=>{await se("git",["-c","core.hooksPath=/dev/null","-c","submodule.recurse=false","-c","protocol.file.allow=never","-c","protocol.ext.allow=never",...e],{cwd:r?.cwd,timeout:r?.timeoutMs??ie,maxBuffer:1024*1024,env:{PATH:process.env.PATH,HOME:process.env.HOME,USER:process.env.USER,USERPROFILE:process.env.USERPROFILE,TMPDIR:process.env.TMPDIR,TMP:process.env.TMP,TEMP:process.env.TEMP,SYSTEMROOT:process.env.SYSTEMROOT,WINDIR:process.env.WINDIR,SSH_AUTH_SOCK:process.env.SSH_AUTH_SOCK,SSH_AGENT_PID:process.env.SSH_AGENT_PID,HTTP_PROXY:process.env.HTTP_PROXY,HTTPS_PROXY:process.env.HTTPS_PROXY,NO_PROXY:process.env.NO_PROXY,GIT_TERMINAL_PROMPT:"0",GIT_CONFIG_NOSYSTEM:"1",GIT_CONFIG_NOGLOBAL:"1",...process.platform==="win32"?{}:{GIT_ASKPASS:"/bin/false"}}})},Ae=async(e,r,t,i)=>{const o=w.join(t,"archive.tar");await A(["archive","--remote",e,"--format=tar","--output",o,r],{timeoutMs:i}),await se("tar",["-xf",o,"-C",t],{timeout:i??ie,maxBuffer:1024*1024}),await $(o,{force:!0})},Le=e=>{if(!e||e.length===0)return!1;for(const r of e)if(!r||r.includes("**"))return!1;return!0},Ne=e=>{if(!e)return[];const r=e.map(t=>{const i=t.replace(/\\/g,"/"),o=i.indexOf("*");return(o===-1?i:i.slice(0,o)).replace(/\/+$|\/$/,"")});return Array.from(new Set(r.filter(t=>t.length>0)))},Be=async(e,r)=>{const t=/^[0-9a-f]{7,40}$/i.test(e.ref),i=Le(e.include),o=["clone","--no-checkout","--filter=blob:none","--depth",String(e.depth),"--recurse-submodules=no","--no-tags"];if(i&&o.push("--sparse"),t||(o.push("--single-branch"),e.ref!=="HEAD"&&o.push("--branch",e.ref)),o.push(e.repo,r),await A(o,{timeoutMs:e.timeoutMs}),i){const s=Ne(e.include);s.length>0&&await A(["-C",r,"sparse-checkout","set",...s],{timeoutMs:e.timeoutMs})}await A(["-C",r,"checkout","--detach",e.resolvedCommit],{timeoutMs:e.timeoutMs})},je=async e=>{const r=await N(w.join(q(),`docs-cache-${e.sourceId}-`));try{return await Ae(e.repo,e.resolvedCommit,r,e.timeoutMs),r}catch(t){throw await $(r,{recursive:!0,force:!0}),t}},He=async e=>{U(e.sourceId,"sourceId");try{const r=await je(e);return{repoDir:r,cleanup:async()=>{await $(r,{recursive:!0,force:!0})}}}catch{const r=await N(w.join(q(),`docs-cache-${e.sourceId}-`));try{return await Be(e,r),{repoDir:r,cleanup:async()=>{await $(r,{recursive:!0,force:!0})}}}catch(t){throw await $(r,{recursive:!0,force:!0}),t}}},Ue=async e=>{const r=new Map(e.sources.map(u=>[u.id,u])),t={};for(const[u,m]of Object.entries(e.lock.sources)){const l=r.get(u),h=l?.targetDir?_(H(e.configPath,l.targetDir)):void 0;t[u]={repo:m.repo,ref:m.ref,resolvedCommit:m.resolvedCommit,bytes:m.bytes,fileCount:m.fileCount,manifestSha256:m.manifestSha256,updatedAt:m.updatedAt,cachePath:_(w.join(e.cacheDir,u)),...h?{targetDir:h}:{}}}const i={generatedAt:new Date().toISOString(),cacheDir:_(e.cacheDir),sources:t},o=w.join(e.cacheDir,ge),s=`${JSON.stringify(i,null,2)}
3
+ `;await W(o,s,"utf8")},F=e=>_(e),G=Number(process.env.DOCS_CACHE_STREAM_THRESHOLD_MB??"2"),ze=Number.isFinite(G)&&G>0?Math.floor(G*1024*1024):1024*1024,Ge=(e,r)=>{const t=w.resolve(e);if(!w.resolve(r).startsWith(t+w.sep))throw new Error(`Path traversal detected: ${r}`)},ae=async e=>{try{return await j(e,ee.O_RDONLY|ee.O_NOFOLLOW)}catch(r){const t=r.code;if(t==="ELOOP")return null;if(t==="EINVAL"||t==="ENOSYS"||t==="ENOTSUP")return(await he(e)).isSymbolicLink()?null:await j(e,"r");throw r}},Ye=async(e,r=5e3)=>{const t=Date.now();for(;Date.now()-t<r;)try{const i=await j(e,"wx");return{release:async()=>{await i.close(),await $(e,{force:!0})}}}catch(i){if(i.code!=="EEXIST")throw i;await new Promise(o=>setTimeout(o,100))}throw new Error(`Failed to acquire lock for ${e}.`)},We=async e=>{U(e.sourceId,"sourceId");const r=ye(e.cacheDir,e.sourceId);await R(e.cacheDir,{recursive:!0});const t=await N(w.join(e.cacheDir,`.tmp-${e.sourceId}-`));let i=null;const o=async()=>{const s=i;!s||s.closed||s.destroyed||await new Promise(u=>{const m=()=>{s.off("close",l),s.off("error",h),u()},l=()=>m(),h=()=>m();s.once("close",l),s.once("error",h);try{s.end()}catch{m()}})};try{const s=await te(e.include,{cwd:e.repoDir,ignore:[".git/**",...e.exclude??[]],dot:!0,onlyFiles:!0,followSymbolicLinks:!1});s.sort((f,c)=>F(f).localeCompare(F(c)));const u=new Set;for(const f of s)u.add(w.dirname(f));await Promise.all(Array.from(u,f=>R(w.join(t,f),{recursive:!0})));let m=0,l=0;const h=Math.max(1,Math.min(s.length,Math.max(8,Math.min(128,Ce.cpus().length*8)))),C=w.join(t,z),g=Q(C,{encoding:"utf8"});i=g;const x=L("sha256"),y=async f=>new Promise((c,p)=>{const d=P=>{g.off("drain",a),p(P)},a=()=>{g.off("error",d),c()};g.once("error",d),g.write(f)?(g.off("error",d),c()):g.once("drain",a)});for(let f=0;f<s.length;f+=h){const c=s.slice(f,f+h),p=await Promise.all(c.map(async d=>{const a=F(d),P=w.join(e.repoDir,d),D=await ae(P);if(!D)return null;try{const k=await D.stat();if(!k.isFile())return null;const T=w.join(t,d);if(Ge(t,T),k.size>=ze){const E=Oe(P,{fd:D.fd,autoClose:!1}),de=Q(T);await $e(E,de)}else{const E=await D.readFile();await W(T,E)}return{path:a,size:k.size}}finally{await D.close()}}));for(const d of p){if(!d)continue;if(e.maxFiles!==void 0&&l+1>e.maxFiles)throw new Error(`Materialized content exceeds maxFiles (${e.maxFiles}).`);if(m+=d.size,m>e.maxBytes)throw new Error(`Materialized content exceeds maxBytes (${e.maxBytes}).`);const a=`${JSON.stringify(d)}
4
+ `;x.update(a),await y(a),l+=1}}await new Promise((f,c)=>{g.end(()=>f()),g.once("error",c)});const n=x.digest("hex"),S=async f=>{try{return await X(f),!0}catch{return!1}};return await(async(f,c)=>{const p=await Ye(`${c}.lock`);try{const d=await S(c),a=`${c}.bak-${me(8).toString("hex")}`;d&&await B(c,a);try{await B(f,c)}catch(P){if(d)try{await B(a,c)}catch(D){const k=D instanceof Error?D.message:String(D);process.stderr.write(`Warning: Failed to restore backup: ${k}
5
+ `)}throw P}d&&await $(a,{recursive:!0,force:!0})}finally{await p.release()}})(t,r.sourceDir),{bytes:m,fileCount:l,manifestSha256:n}}catch(s){try{await o()}catch{}throw await $(t,{recursive:!0,force:!0}),s}},Xe=async e=>{U(e.sourceId,"sourceId");const r=await te(e.include,{cwd:e.repoDir,ignore:[".git/**",...e.exclude??[]],dot:!0,onlyFiles:!0,followSymbolicLinks:!1});r.sort((s,u)=>F(s).localeCompare(F(u)));let t=0,i=0;const o=L("sha256");for(const s of r){const u=F(s),m=w.join(e.repoDir,s),l=await ae(m);if(l)try{const h=await l.stat();if(!h.isFile())continue;if(e.maxFiles!==void 0&&i+1>e.maxFiles)throw new Error(`Materialized content exceeds maxFiles (${e.maxFiles}).`);if(t+=h.size,t>e.maxBytes)throw new Error(`Materialized content exceeds maxBytes (${e.maxBytes}).`);const C=`${JSON.stringify({path:u,size:h.size})}
6
+ `;o.update(C),i+=1}finally{await l.close()}}return{bytes:t,fileCount:i,manifestSha256:o.digest("hex")}},Je=async(e,r)=>{await r.rm(e,{recursive:!0,force:!0})},Y=async e=>{const r=e.deps??{cp:pe,mkdir:R,rm:$,symlink:we,stderr:process.stderr},t=w.dirname(e.targetDir);await r.mkdir(t,{recursive:!0}),await Je(e.targetDir,r);const i=process.platform==="win32"?"copy":"symlink";if((e.mode??i)==="copy"){await r.cp(e.sourceDir,e.targetDir,{recursive:!0});return}const o=process.platform==="win32"?"junction":"dir";try{await r.symlink(e.sourceDir,e.targetDir,o)}catch(s){const u=s.code;if(u&&new Set(["EPERM","EACCES","ENOTSUP","EINVAL"]).has(u)){if(e.explicitTargetMode){const m=s instanceof Error?s.message:String(s);r.stderr.write(`Warning: Failed to create symlink at ${e.targetDir}. Falling back to copy. ${m}
7
+ `)}await r.cp(e.sourceDir,e.targetDir,{recursive:!0});return}throw s}},Ke=e=>{if(e<1024)return`${e} B`;const r=["KB","MB","GB","TB"];let t=e,i=-1;for(;t>=1024&&i<r.length-1;)t/=1024,i+=1;return`${t.toFixed(1)} ${r[i]}`},b=async e=>{try{return await X(e),!0}catch{return!1}},ne=async(e,r)=>{const t=w.join(e,r);return await b(t)?await b(w.join(t,z)):!1},ce=e=>{if(!e||e.length===0)return[];const r=e.map(t=>t.trim()).filter(t=>t.length>0);return Array.from(new Set(r)).sort()},qe=e=>{const r={include:ce(e.include),exclude:ce(e.exclude)},t=L("sha256");return t.update(JSON.stringify(r)),t.digest("hex")},le=async(e,r={})=>{const{config:t,resolvedPath:i,sources:o}=await De(e.configPath),s=t.defaults??ve.defaults,u=Se(i,t.cacheDir??xe,e.cacheDirOverride),m=Ee(i),l=await b(m);let h=null;l&&(h=await Me(m));const C=r.resolveRemoteCommit??_e,g=e.sourceFilter?.length?o.filter(y=>e.sourceFilter?.includes(y.id)):o,x=await Promise.all(g.map(async y=>{const n=h?.sources?.[y.id],S=y.include??s.include,f=y.exclude,c=qe({include:S,exclude:f});if(e.offline){const P=await ne(u,y.id);return{id:y.id,repo:n?.repo??y.repo,ref:n?.ref??y.ref??s.ref,resolvedCommit:n?.resolvedCommit??"offline",lockCommit:n?.resolvedCommit??null,lockRulesSha256:n?.rulesSha256,status:n&&P?"up-to-date":"missing",bytes:n?.bytes,fileCount:n?.fileCount,manifestSha256:n?.manifestSha256,rulesSha256:c}}const p=await C({repo:y.repo,ref:y.ref,allowHosts:s.allowHosts,timeoutMs:e.timeoutMs}),d=n?.resolvedCommit===p.resolvedCommit&&n?.rulesSha256===c,a=n?d?"up-to-date":"changed":"missing";return{id:y.id,repo:p.repo,ref:p.ref,resolvedCommit:p.resolvedCommit,lockCommit:n?.resolvedCommit??null,lockRulesSha256:n?.rulesSha256,status:a,bytes:n?.bytes,fileCount:n?.fileCount,manifestSha256:n?.manifestSha256,rulesSha256:c}}));return{config:t,configPath:i,cacheDir:u,lockPath:m,lockExists:l,lockData:h,results:x,sources:g,defaults:s}},Ve=async()=>{const e=w.resolve(process.cwd(),"package.json");try{const r=await J(e,"utf8"),t=JSON.parse(r.toString());return typeof t.version=="string"?t.version:"0.0.0"}catch{}try{const r=await J(new URL("../package.json",import.meta.url),"utf8"),t=JSON.parse(r.toString());return typeof t.version=="string"?t.version:"0.0.0"}catch{return"0.0.0"}},Ze=async(e,r)=>{const t=await Ve(),i=new Date().toISOString(),o={...r?.sources??{}};for(const s of e.results){const u=o[s.id];o[s.id]={repo:s.repo,ref:s.ref,resolvedCommit:s.resolvedCommit,bytes:s.bytes??u?.bytes??0,fileCount:s.fileCount??u?.fileCount??0,manifestSha256:s.manifestSha256??u?.manifestSha256??s.resolvedCommit,rulesSha256:s.rulesSha256??u?.rulesSha256,updatedAt:i}}return{version:1,generatedAt:i,toolVersion:t,sources:o}},ue=async(e,r={})=>{const t=process.hrtime.bigint();let i=0;const o=await le(e,r);await R(o.cacheDir,{recursive:!0});const s=o.lockData,u=o.results.filter(l=>{const h=o.sources.find(C=>C.id===l.id);return l.status==="missing"&&(h?.required??!0)});if(e.failOnMiss&&u.length>0)throw new Error(`Missing required source(s): ${u.map(l=>l.id).join(", ")}.`);if(!e.lockOnly){const l=o.defaults,h=r.fetchSource??He,C=r.materializeSource??We,g=async(n,S)=>{const f=n?.length?o.results.filter(c=>n.includes(c.id)):o.results;return(await Promise.all(f.map(async c=>{const p=o.sources.find(a=>a.id===c.id);if(!p)return null;const d=await ne(o.cacheDir,c.id);return S||c.status!=="up-to-date"||!d?{result:c,source:p}:null}))).filter(Boolean)},x=async()=>{await Promise.all(o.sources.map(async n=>{if(!n.targetDir)return;const S=H(o.configPath,n.targetDir);await b(S)||await Y({sourceDir:w.join(o.cacheDir,n.id),targetDir:S,mode:n.targetMode??l.targetMode,explicitTargetMode:n.targetMode!==void 0})}))},y=async n=>{const S=e.concurrency??4;let f=0;const c=async()=>{const p=n[f];if(!p||!p.source)return;f+=1;const{result:d,source:a}=p,P=o.lockData?.sources?.[a.id];e.json||v.step("Fetching",a.id);const D=await h({sourceId:a.id,repo:a.repo,ref:a.ref,resolvedCommit:d.resolvedCommit,cacheDir:o.cacheDir,depth:a.depth??l.depth,include:a.include??l.include,timeoutMs:e.timeoutMs});try{const k=w.join(o.cacheDir,a.id,z);if(d.status!=="up-to-date"&&P?.manifestSha256&&await b(k)){const E=await Xe({sourceId:a.id,repoDir:D.repoDir,cacheDir:o.cacheDir,include:a.include??l.include,exclude:a.exclude,maxBytes:a.maxBytes??l.maxBytes,maxFiles:a.maxFiles??l.maxFiles});if(E.manifestSha256===P.manifestSha256){d.bytes=E.bytes,d.fileCount=E.fileCount,d.manifestSha256=E.manifestSha256,d.status="up-to-date",e.json||v.item(O.success,a.id,"no content changes"),await c();return}}const T=await C({sourceId:a.id,repoDir:D.repoDir,cacheDir:o.cacheDir,include:a.include??l.include,exclude:a.exclude,maxBytes:a.maxBytes??l.maxBytes,maxFiles:a.maxFiles??l.maxFiles});if(a.targetDir){const E=H(o.configPath,a.targetDir);await Y({sourceDir:w.join(o.cacheDir,a.id),targetDir:E,mode:a.targetMode??l.targetMode,explicitTargetMode:a.targetMode!==void 0})}d.bytes=T.bytes,d.fileCount=T.fileCount,d.manifestSha256=T.manifestSha256,e.json||v.item(O.success,a.id,`synced ${T.fileCount} files`)}finally{await D.cleanup()}await c()};await Promise.all(Array.from({length:Math.min(S,n.length)},c))};if(e.offline)await x();else{const n=await g();await y(n),await x()}if(!e.offline){const n=(await Z({configPath:o.configPath,cacheDirOverride:o.cacheDir})).results.filter(S=>!S.ok);if(n.length>0){const S=await g(n.map(c=>c.id),!0);S.length>0&&(await y(S),await x());const f=(await Z({configPath:o.configPath,cacheDirOverride:o.cacheDir})).results.filter(c=>!c.ok);if(f.length>0&&(i+=1,!e.json)){const c=f.map(p=>`${p.id} (${p.issues.join("; ")})`).join(", ");v.line(`${O.warn} Verify failed for ${f.length} source(s): ${c}`)}}}}const m=await Ze(o,s);if(await Pe(o.lockPath,m),!e.json){const l=Number(process.hrtime.bigint()-t)/1e6,h=o.results.reduce((g,x)=>g+(x.bytes??0),0),C=o.results.reduce((g,x)=>g+(x.fileCount??0),0);v.line(`${O.info} Completed in ${l.toFixed(0)}ms \xB7 ${Ke(h)} \xB7 ${C} files${i?` \xB7 ${i} warning${i===1?"":"s"}`:""}`)}return o.config.index&&await Ue({cacheDir:o.cacheDir,configPath:o.configPath,lock:m,sources:o.sources}),o.lockExists=!0,o},fe=e=>{const r={upToDate:e.results.filter(t=>t.status==="up-to-date").length,changed:e.results.filter(t=>t.status==="changed").length,missing:e.results.filter(t=>t.status==="missing").length};if(e.results.length===0){v.line(`${O.info} No sources to sync.`);return}v.line(`${O.info} ${e.results.length} sources (${r.upToDate} up-to-date, ${r.changed} changed, ${r.missing} missing)`);for(const t of e.results){const i=v.hash(t.resolvedCommit),o=v.hash(t.lockCommit),s=!!t.lockRulesSha256&&!!t.rulesSha256&&t.lockRulesSha256!==t.rulesSha256;if(t.status==="up-to-date"){v.item(O.success,t.id,`${M.dim("up-to-date")} ${M.gray(i)}`);continue}if(t.status==="changed"){if(t.lockCommit===t.resolvedCommit&&s){v.item(O.warn,t.id,`${M.dim("rules changed")} ${M.gray(i)}`);continue}v.item(O.warn,t.id,`${M.dim("changed")} ${M.gray(o)} ${M.dim("->")} ${M.gray(i)}`);continue}v.item(O.warn,t.id,`${M.dim("missing")} ${M.gray(i)}`)}},Qe={__proto__:null,getSyncPlan:le,printSyncPlan:fe,runSync:ue};export{Y as a,fe as b,ue as c,re as e,oe as p,I as r,Qe as s};
7
8
  //# sourceMappingURL=sync.mjs.map
package/dist/lock.mjs ADDED
@@ -0,0 +1,3 @@
1
+ import{readFile as m,writeFile as d}from"node:fs/promises";import c from"node:path";const u="docs.lock",a=e=>typeof e=="object"&&e!==null&&!Array.isArray(e),i=(e,o)=>{if(typeof e!="string"||e.length===0)throw new Error(`${o} must be a non-empty string.`);return e},w=(e,o)=>{if(typeof e!="number"||Number.isNaN(e))throw new Error(`${o} must be a number.`);return e},l=(e,o)=>{const n=w(e,o);if(n<0)throw new Error(`${o} must be zero or greater.`);return n},f=e=>{if(!a(e))throw new Error("Lock file must be a JSON object.");if(e.version!==1)throw new Error("Lock file version must be 1.");const o=i(e.generatedAt,"generatedAt"),n=i(e.toolVersion,"toolVersion");if(!a(e.sources))throw new Error("sources must be an object.");const t={};for(const[r,s]of Object.entries(e.sources)){if(!a(s))throw new Error(`sources.${r} must be an object.`);t[r]={repo:i(s.repo,`sources.${r}.repo`),ref:i(s.ref,`sources.${r}.ref`),resolvedCommit:i(s.resolvedCommit,`sources.${r}.resolvedCommit`),bytes:l(s.bytes,`sources.${r}.bytes`),fileCount:l(s.fileCount,`sources.${r}.fileCount`),manifestSha256:i(s.manifestSha256,`sources.${r}.manifestSha256`),rulesSha256:s.rulesSha256===void 0?void 0:i(s.rulesSha256,`sources.${r}.rulesSha256`),updatedAt:i(s.updatedAt,`sources.${r}.updatedAt`)}}return{version:1,generatedAt:o,toolVersion:n,sources:t}},h=e=>c.resolve(c.dirname(e),u),b=async e=>{let o;try{o=await m(e,"utf8")}catch(t){const r=t instanceof Error?t.message:String(t);throw new Error(`Failed to read lock file at ${e}: ${r}`)}let n;try{n=JSON.parse(o)}catch(t){const r=t instanceof Error?t.message:String(t);throw new Error(`Invalid JSON in ${e}: ${r}`)}return f(n)},$=async(e,o)=>{const n=`${JSON.stringify(o,null,2)}
2
+ `;await d(e,n,"utf8")};export{u as DEFAULT_LOCK_FILENAME,b as readLock,h as resolveLockPath,f as validateLock,$ as writeLock};
3
+ //# sourceMappingURL=lock.mjs.map
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "docs-cache",
3
3
  "private": false,
4
4
  "type": "module",
5
- "version": "0.1.0",
5
+ "version": "0.1.2",
6
6
  "description": "CLI for deterministic local caching of external documentation for agents and tools",
7
7
  "author": "Frederik Bosch",
8
8
  "license": "MIT",
@@ -34,6 +34,7 @@
34
34
  "bin",
35
35
  "dist/cli.mjs",
36
36
  "dist/chunks/*.mjs",
37
+ "dist/lock.mjs",
37
38
  "dist/shared/*.mjs",
38
39
  "README.md",
39
40
  "LICENSE"
@@ -78,7 +79,7 @@
78
79
  "build": "unbuild",
79
80
  "dev": "unbuild --stub",
80
81
  "lint": "biome check .",
81
- "release": "pnpm run lint && pnpm run typecheck && bumpp && pnpm publih --access public",
82
+ "release": "pnpm run lint && pnpm run typecheck && bumpp && pnpm publish --access public",
82
83
  "test": "pnpm build && node --test",
83
84
  "test:coverage": "pnpm build && c8 --include dist --exclude bin --reporter=text node --test",
84
85
  "bench": "pnpm build && node scripts/benchmarks/run.mjs",