@redue/cli 0.1.0-alpha.3 → 0.1.0-alpha.31

package/src/history-worker.mjs

@@ -1,18 +1,26 @@
 // Native history is an optional acceleration. Keep its operations off the
 // control thread so a wedged native call cannot block status or shutdown.
+// Each helper serves one query; the observer starts it ahead of that query.
 import fs from 'node:fs';
+import {mark} from './decision-profile.mjs';
+mark('history.worker_loaded');
 import watcher from '@parcel/watcher';
 
-try {
-  const [root,previous,next,backend,faultFile]=process.argv.slice(2);
-  let fault;
-  try {fault=faultFile&&fs.readFileSync(faultFile,'utf8').trim();}
-  catch(e){if(e.code!=='ENOENT')throw e;}
-  if(fault==='hang')Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0);
-  if(fault==='fail')throw Error('injected history failure');
-  await watcher.writeSnapshot(root,next,{backend});
-  const changes=previous?await watcher.getEventsSince(root,previous,{backend}):[];
-  process.send({ok:true,changes});
-} catch(error) {
-  process.send({ok:false,error:error.message,code:error.code||error.name});
-}
+process.on('disconnect',()=>process.exit(0));
+process.once('message',async({root,previous,next,backend,faultFile})=>{
+  try {
+    let fault;
+    try {fault=faultFile&&fs.readFileSync(faultFile,'utf8').trim();}
+    catch(e){if(e.code!=='ENOENT')throw e;}
+    if(fault==='hang')Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0);
+    if(fault==='fail')throw Error('injected history failure');
+    mark('history.snapshot_start');
+    await watcher.writeSnapshot(root,next,{backend});
+    mark('history.snapshot_complete');
+    const changes=previous?await watcher.getEventsSince(root,previous,{backend}):[];
+    mark('history.events_complete',{events:changes.length});
+    process.send({ok:true,changes});
+  } catch(error) {
+    process.send({ok:false,error:error.message,code:error.code||error.name});
+  }
+});

package/src/index.mjs

@@ -5,12 +5,18 @@ import {createHash} from 'node:crypto';
 import micromatch from './glob.mjs';
 import {sha} from './plan.mjs';
 import {resolveLinks} from './installed-inputs.mjs';
-import {validInputRelative} from './path-identity.mjs';
+import {validInputRelative,identityFields,racyUntil} from './path-identity.mjs';
+import {isSandboxed} from './sandbox.mjs';
 
 const tracked = root => [...new Set(execFileSync('git',
   ['ls-files','--cached','--others','--exclude-standard','-z'],
   {cwd:root,env:{...process.env,GIT_OPTIONAL_LOCKS:'0'},maxBuffer:64*1024*1024})
   .toString().split('\0').filter(Boolean))];
+// Git-ignored paths at the top of each ignored tree: a directory ends in '/'.
+const ignored = root => [...new Set(execFileSync('git',
+  ['ls-files','--others','--ignored','--exclude-standard','--directory','-z'],
+  {cwd:root,env:{...process.env,GIT_OPTIONAL_LOCKS:'0'},maxBuffer:64*1024*1024})
+  .toString().split('\0').filter(Boolean))];
 const xor = (left,right) => { for(let i=0;i<32;i++) left[i]^=right[i]; };
 const token = (file,hash) => createHash('sha256').update(file+'\0'+hash).digest();
 const stablePrefix = glob => {
@@ -35,37 +41,52 @@ const pathPrefixes = rel => {
   return prefixes;
 };
 
+// An installed file is observed by identity, not content: every write changes
+// ctime, so its entry changes without reading the whole installation. The
+// compiler's own reads are content-checked by its probe and certificate. A file
+// changed within the racy window could still be rewritten in the same timestamp
+// tick at the same size; such files are tracked in `racy`, and the daemon waits
+// the window out and records them again before it writes a run checkpoint.
+const identity=st=>identityFields.map(key=>String(st[key])).join(':');
+// An installed link to a directory (pnpm links every package this way) is
+// observed by its link chain alone, when `observedDir` says the directory it
+// resolves to is itself observed installed: its own entries carry its contents.
 function fileHash(root, rel,installed=false,before=fs.lstatSync(path.join(root,rel),
-  {bigint:true})) {
+  {bigint:true}),observedDir=null) {
   const file=path.join(root, rel);
-  let value,references=[],bytes=Number(before.size);
+  let value,references=[],bytes=installed?0:Number(before.size),racy=0;
   if(before.isSymbolicLink()){
     const resolution=resolveLinks(file,root,[],true);
     if(!resolution.physical)throw Error(`symlink input target observation unavailable: ${rel}: ${resolution.reason}`);
     const target=fs.statSync(resolution.physical,{bigint:true});
-    if(!target.isFile())throw Error(`directory input symlink unsupported: ${rel}`);
-    value='link:'+JSON.stringify(resolution.links)+':'+target.mode+':'+
-      (installed&&target.nlink>1?target.nlink+':':'')+
-      hashRegular(resolution.physical).hash;
+    const physical=path.relative(root,resolution.physical).split(path.sep).join('/');
+    const directory=target.isDirectory()&&installed&&observedDir?.(physical);
+    if(!target.isFile()&&!directory)throw Error(`directory input symlink unsupported: ${rel} -> ${physical}`);
+    if(directory)value='dirlink:'+JSON.stringify(resolution.links)+':'+physical;
+    else{
+      if(installed)racy=racyUntil(target);else bytes=Number(target.size);
+      value='link:'+JSON.stringify(resolution.links)+':'+(installed?'stat:'+identity(target):
+        target.mode+':'+hashRegular(resolution.physical).hash);
+    }
     references=[...new Set([...resolution.links.map(link=>path.relative(root,link.path)
       .split(path.sep).join('/')),path.relative(root,resolution.physical)
       .split(path.sep).join('/')])];
-    bytes=Number(target.size);
-    const targetAfter=fs.statSync(resolution.physical,{bigint:true});
-    for(const key of ['dev','ino','mode','nlink','size','mtimeNs','ctimeNs'])
-      if(target[key]!==targetAfter[key])throw Error(`input changed during inspection: ${rel}`);
+    // A directory's own timestamps follow its entries, which are observed separately.
+    if(!directory){const targetAfter=fs.statSync(resolution.physical,{bigint:true});
+      for(const key of identityFields)
+        if(target[key]!==targetAfter[key])throw Error(`input changed during inspection: ${rel}`);}
     const resolutionAfter=resolveLinks(file,root,[],true);
     if(resolutionAfter.physical!==resolution.physical||
       JSON.stringify(resolutionAfter.links)!==JSON.stringify(resolution.links))
       throw Error(`input link changed during inspection: ${rel}`);
   }
-  else if(before.isFile()) value='file:'+before.mode+':'+
-    (installed&&before.nlink>1?before.nlink+':':'')+hashRegular(file).hash;
+  else if(before.isFile()&&installed){value='stat:'+identity(before);racy=racyUntil(before);}
+  else if(before.isFile()) value='file:'+before.mode+':'+hashRegular(file).hash;
   else throw Error(`non-file input: ${rel}`);
   const after=fs.lstatSync(file,{bigint:true});
-  for(const key of ['dev','ino','mode','nlink','size','mtimeNs','ctimeNs'])
+  for(const key of identityFields)
     if(before[key]!==after[key]) throw Error(`input changed during inspection: ${rel}`);
-  return {hash:sha(value),references,bytes};
+  return {hash:sha(value),references,bytes,racy};
 }
 const readBuffer=Buffer.allocUnsafe(65536);
 function hashRegular(file) {
@@ -81,6 +102,7 @@ export class InputIndex {
     const matcherStarted=performance.now();
     this.bundle=bundle; this.root=bundle.root; this.files=new Map(); this.members=new Map();
     this.symlinkRefs=new Map();this.aliasDependencies=new Map();
+    this.racy=new Map();
     this.trackGit=options.trackGit!==false;
     this.installedRoots=[...new Set(Object.values(bundle.plans).flatMap(plan=>
       plan.installedPhysicalRoots||[]))];
@@ -149,21 +171,46 @@ export class InputIndex {
     }
     return {files,bytes,durationMs:performance.now()-started};
   }
+  // When the racy window of the named checks' installed members ends (0: none).
+  racyUntil(names=null) {
+    let until=0;
+    for(const [rel,time] of this.racy)
+      if(!names||[...this.members.get(rel)||[]].some(name=>names.includes(name)))until=Math.max(until,time);
+    return until;
+  }
+  // Records again every installed file whose racy window has passed.
+  settleRacy() {
+    for(const [rel,time] of [...this.racy])if(time<=Date.now())this.record(rel,rel+' changed');
+  }
   summary(name) { const row=this.checks.get(name); return {fingerprint:this.fingerprint(name),
     revision:row.revision,planId:row.plan.id,files:row.files.size,probeAt:row.probeAt}; }
-  record(rel, changeReason=rel+' changed', observedStat=null) {
+  // `only`, when given, limits which checks may take the file.
+  record(rel, changeReason=rel+' changed', observedStat=null, only=null) {
     if(!validInputRelative(rel)) {
       this.unavailable='invalid event path'; return false;
     }
     const names=this.candidates(rel), old=this.files.get(rel), matched=[];
-    for(const name of names) if(this.matches(this.checks.get(name),rel)) matched.push(name);
+    for(const name of names) if((!only||only(name))&&this.matches(this.checks.get(name),rel)) matched.push(name);
     if(!matched.length && old===undefined) return false;
     let fresh,stat,inspection;
     try{stat=observedStat||fs.lstatSync(path.join(this.root,rel),{bigint:true});}
     catch(e){if(e.code!=='ENOENT'){this.unavailable=String(e);return false;}}
     if(stat&&(stat.isFile()||stat.isSymbolicLink())) {
       const installed=this.installedPath(rel);
-      try{const started=performance.now();inspection=fileHash(this.root,rel,installed,stat);
+      try{const started=performance.now();
+        // A linked directory counts as observed only strictly inside an
+        // installed root, or as a checkout directory outside node_modules (a
+        // workspace package the package manager links), and only if every
+        // check matching the link also observes the directory's own files
+        // (its package.json stands for them).
+        // The second form is for recorded checks only: their recorder
+        // classifies a read through the link where it leads.
+        const recorded=name=>['node-script-v1','npm-tests-direct-v1'].includes(this.checks.get(name).plan.qualification)||isSandboxed(this.checks.get(name).plan.qualification);
+        const observedDir=installed?physical=>!this.includeInstalled&&
+          ([...this.installedRootSet].some(root=>physical.startsWith(root+'/'))||
+            !physical.split('/').includes('node_modules')&&!this.installedPath(physical+'/package.json')&&matched.every(recorded))&&
+          matched.every(name=>this.matches(this.checks.get(name),physical+'/package.json')):null;
+        inspection=fileHash(this.root,rel,installed,stat,observedDir);
         fresh=inspection.hash;
         this.profile.hashingMs+=performance.now()-started;
         this.rehashedFiles++;this.rehashedBytes+=inspection.bytes;}
@@ -182,6 +229,7 @@ export class InputIndex {
       }
     }
     const oldNames=this.members.get(rel)||new Set(), newNames=new Set(fresh===undefined?[]:matched);
+    if(inspection?.racy>Date.now()&&newNames.size)this.racy.set(rel,inspection.racy);else this.racy.delete(rel);
     if(old===fresh && oldNames.size===newNames.size && [...oldNames].every(n=>newNames.has(n))) return false;
     for(const name of new Set([...oldNames,...newNames])) {
       const row=this.checks.get(name);
@@ -226,6 +274,10 @@ export class InputIndex {
     this.unavailable=null;
     const trackedStarted=performance.now();
     const names=new Set(this.trackGit?tracked(this.root):[]);
+    // A sandboxed check's reads are not recorded, so git-ignored files are its
+    // inputs too: every ignored file its patterns take, outside the scan roots
+    // (installed trees) and the directories no pattern can take a file from.
+    const ignoredTrees=this.trackGit&&[...this.checks.values()].some(row=>row.plan.ignoredInputs)?ignored(this.root):[];
     this.profile.trackedMs+=performance.now()-trackedStarted;
     const enumerationStarted=performance.now();
     // Traverse declared input roots to include ignored/generated inputs. Avoid unrelated
@@ -269,6 +321,29 @@ export class InputIndex {
       }
       visit(base,prefix);
     }
+    // Only the checks that take ignored files get them here, as before for the others.
+    const ignoring=name=>this.checks.get(name).plan.ignoredInputs===true;
+    const takes=rel=>[...this.checks.values()].some(row=>row.plan.ignoredInputs&&this.matches(row,rel));
+    // A node_modules that is not installed input is skipped, as above. An
+    // ignored link is taken when it leads to a file in the checkout; a link
+    // to a directory or out of the checkout is not a file input.
+    const linkToFile=rel=>{try{const target=fs.realpathSync(path.join(this.root,rel));
+      return fs.statSync(target).isFile()&&target.startsWith(fs.realpathSync(this.root)+path.sep);}catch{return false;}};
+    const visitIgnored=rel=>{
+      const base=path.posix.basename(rel);
+      if(this.prefixes.has(rel)||this.installedPath(rel)&&!this.installedAncestors.has(rel)||base==='.git'||
+        base==='node_modules'&&!this.installedPath(rel))return;
+      let st;try{st=fs.lstatSync(path.join(this.root,rel),{bigint:true});}
+      catch(e){if(e.code==='ENOENT')return;throw e;}
+      this.profile.enumeratedEntries++;
+      if(st.isSymbolicLink()&&!linkToFile(rel))return;
+      if(st.isDirectory()){
+        if(!takes(rel+'/.redue-probe'))return;
+        for(const ent of fs.readdirSync(path.join(this.root,rel),{withFileTypes:true}))visitIgnored(rel+'/'+ent.name);
+      }else if(this.candidates(rel).size){
+        const started=performance.now();this.record(rel,'cold reconciliation',st,ignoring);processingMs+=performance.now()-started;scanned++;}
+    };
+    for(const entry of ignoredTrees)visitIgnored(entry.replace(/\/$/,''));
     this.profile.enumerationMs+=performance.now()-enumerationStarted-processingMs;
     const recordStarted=performance.now();
     for(const rel of names) {
@@ -283,9 +358,10 @@ export class InputIndex {
     const full=path.join(this.root,rel);
     let subtree=false;
     try {
-      if(fs.lstatSync(full).isSymbolicLink()&&fs.statSync(full).isDirectory()){
-        this.unavailable=`directory symlink input unsupported: ${rel}`;return;}
-      if(fs.statSync(full).isDirectory()) {this.scanSubtree(rel);subtree=true;}
+      // A link is an entry of its own: record() accepts an observed installed
+      // directory link and refuses any other.
+      if(fs.lstatSync(full).isSymbolicLink())this.record(rel);
+      else if(fs.statSync(full).isDirectory()) {this.scanSubtree(rel);subtree=true;}
       else this.record(rel);
     } catch(e) {
       if(e.code==='ENOENT') {
@@ -304,3 +380,14 @@ export class InputIndex {
     return out;
   }
 }
+
+// The index state a worker sends and a restart retains; hydrate() in daemon.mjs
+// reads it back.
+export function dehydrateIndex(value) {
+  return {files:[...value.files],
+    aliasDependencies:[...value.aliasDependencies].map(([name,refs])=>[name,[...refs]]),
+    rows:[...value.checks].map(([name,row])=>({name,files:[...row.files],
+      sum:row.sum.toString('hex'),revision:row.revision,lastReason:row.lastReason})),
+    eventCount:value.eventCount,rehashedFiles:value.rehashedFiles,
+    rehashedBytes:value.rehashedBytes,profile:value.profile,racy:[...value.racy]};
+}

package/src/installed-inputs.mjs

@@ -84,6 +84,10 @@ export function resolveDeclaredInstalled(root,patterns,allowed=[]) {
     if(!resolved.physical){
       if(resolved.reason==='resolved target missing'&&!resolved.links.length){
         internal.add(pattern);continue;}
+      // An exclusion of a path that does not exist yet, through links (a runner
+      // cache inside a linked package): exclude it where the links lead.
+      if(negative&&resolved.reason==='resolved target missing'&&resolved.prospective&&within(resolved.prospective,root)){
+        internal.add('!'+posix(path.relative(root,resolved.prospective))+(suffix?'/'+suffix:''));continue;}
       result.unresolved.push(`${prefix}: ${resolved.reason}`);continue;
     }
     const boundary=within(resolved.physical,root)?root:
@@ -99,7 +103,9 @@ export function resolveDeclaredInstalled(root,patterns,allowed=[]) {
     else {if(!external.has(boundary))external.set(boundary,new Set());
       external.get(boundary).add(mapped);}
     if(!negative)roots.add(resolved.physical);
-    result.mappings.push({logical:pattern,physical:resolved.physical,root:boundary,
+    // An exclusion maps to the same pattern whether or not its path exists, so
+    // the plan stays the same when a run creates it (a runner cache).
+    if(!negative)result.mappings.push({logical:pattern,physical:resolved.physical,root:boundary,
       links:resolved.links.map(link=>link.path)});
   }
   result.internalPatterns=[...internal].sort();

package/src/installed-observation.mjs

@@ -0,0 +1,23 @@
+// Whether a check observes an installed file: the file is in the checkout
+// and matched by the check's patterns, and a link it goes through that leads
+// elsewhere in the checkout leads to a matched path too. A recorded run's
+// hard-linked installed files are accepted only when every link is one the
+// check observes, so a pattern that leaves a folder out (a runner cache in
+// node_modules) cannot hide one.
+import fs from 'node:fs';
+import path from 'node:path';
+import {observedRelative} from './path-identity.mjs';
+
+// `matches(rel)`: the check's pattern matcher over checkout-relative paths.
+// A link that leads outside the checkout is the plan's to resolve (a
+// permitted root or a refusal).
+export function observedBy(root,matches){
+  let realRoot=root;try{realRoot=fs.realpathSync(root);}catch{}
+  return file=>{
+    const logical=observedRelative(root,path.resolve(file));
+    if(logical===null||!matches(logical))return false;
+    let real;try{real=fs.realpathSync(file);}catch{return false;}
+    const physical=observedRelative(realRoot,real);
+    return physical===null||physical===logical||matches(physical);
+  };
+}