@redue/cli 0.1.0-alpha.18 → 0.1.0-alpha.19

package/src/cli.mjs

@@ -3,8 +3,9 @@ import {builtInProbe,probeFloorMs,publishedDeadline} from './probe-deadline.mjs'
 import fs from 'node:fs';
 import {directQualification,directInterpretation,observedNpmContext} from './direct-typescript.mjs';
 import {directRunCertificateAllowed,directRunCertificate,directRunRecordable,directRunRecording,probeQuestions,validateRunRecord} from './direct-run-certificate.mjs';
-import {runFactChange} from './typescript-list.mjs';
+import {queryAnswer,runFactChange} from './typescript-list.mjs';
 import {changedFiles,coverageReadable,gitIgnored,recordedEnvironment,testEnvironmentIssue,testGuardConfig,testRunGuard,testRunFacts,sharedInstalledFile,testsInterpretation,testsQualification} from './test-direct.mjs';
+import {childProgram,lookup,pnpmConfigFiles,scriptCaches,scriptInstalledPaths,scriptInterpretation,scriptQualification,scriptRunFacts,toolOutput,toolchainDirs} from './node-script.mjs';
 import path from 'node:path';
 import net from 'node:net';
 import os from 'node:os';
@@ -69,7 +70,8 @@ const readsFile=path.join(state,'environment-reads-v1.json');
 function environmentReads() {
   try{const value=JSON.parse(fs.readFileSync(readsFile,'utf8'));return value?.schema===1&&value.checks||{};}catch{return {};}
 }
-function contextHashes() {const reads=config.checks.some(check=>check.qualification===testsQualification)?environmentReads():{};
+const recordedRecipes=[testsQualification,scriptQualification];
+function contextHashes() {const reads=config.checks.some(check=>recordedRecipes.includes(check.qualification))?environmentReads():{};
   return Object.fromEntries(config.checks.map(check=>[
   check.name,Object.fromEntries([...(check.environment?.variables||[])
     .map(key=>[key,digest(process.env[key]??'<unset>')]),
@@ -271,20 +273,28 @@ async function execute() {
     const recorded=directRunRecordable(selected)&&!unobservedDirectRun;
     const launched=recorded?[command[0],'--require',path.join(here,'run-recorder.cjs'),
       ...command.slice(1)]:command;
-    // A test run's processes record what the guard checks (test-recorder.cjs).
-    const testRun=selected.qualification===testsQualification&&['darwin','linux'].includes(process.platform);
-    let testRecord=null,launchEnv=process.env,testFacts={};
+    // A test run's processes record what the guard checks (test-recorder.cjs),
+    // and so do a general recipe run's, through the package manager.
+    const general=selected.qualification===scriptQualification;
+    const testRun=recordedRecipes.includes(selected.qualification)&&['darwin','linux'].includes(process.platform);
+    let testRecord=null,launchEnv=process.env,testFacts={},toolchain=[];
     if(testRun){
       // The package directory and installed node_modules, as the plan has them.
-      try{testFacts=testRunFacts(root,selected);}
-      catch(e){captureIssues.push(`test run facts unavailable (${e.code||e.message})`);}
+      try{testFacts=general?scriptRunFacts(root,selected):testRunFacts(root,selected);}
+      catch(e){captureIssues.push(`${general?'script':'test'} run facts unavailable (${e.code||e.message})`);}
       const environmentIssue=testEnvironmentIssue();
       if(environmentIssue)captureIssues.push(environmentIssue);
       testRecord=path.join(state,`test-record-${runId}`);
       fs.mkdirSync(testRecord,{mode:0o700});
-      const guardFile=path.join(testRecord,'guard.json');
-      fs.writeFileSync(guardFile,JSON.stringify(testGuardConfig(testRecord,root,process.env,testFacts)),{mode:0o600});
+      const guardFile=path.join(testRecord,'guard.json'),guardConfig=testGuardConfig(testRecord,root,process.env,testFacts);
+      if(general){toolchain=toolchainDirs();
+        // The Node on PATH, which the environment identifies by its file.
+        Object.assign(guardConfig,{installed:scriptInstalledPaths(root,testFacts.dirs||[]),toolchain,
+          node:lookup('node',process.env.PATH,process.cwd())});}
+      fs.writeFileSync(guardFile,JSON.stringify(guardConfig),{mode:0o600});
       launchEnv=recordedEnvironment(guardFile);
+      // npm's and pnpm's update checks read state outside the checkout and the network.
+      if(general)launchEnv.npm_config_update_notifier='false';
     }
     let runnerPid=null;
     const record=[];
@@ -357,10 +367,16 @@ async function execute() {
     }else if(!unobservedDirectRun&&!recording)probeAfter=probeHash(probePlan);
     let guard=null;
     if(testRecord){
-      let coverage=null;try{coverage=coverageReadable(root,testFacts);}catch{}
+      let coverage=null;if(!general)try{coverage=coverageReadable(root,testFacts);}catch{}
+      const programs=general?{runner:{file:command[0],args:command.slice(1),cwd:plan.cwd,searchPath:process.env.PATH,
+        env:{preload:true,bashEnv:Boolean(process.env.BASH_ENV)}},
+        judge:(child,main)=>childProgram(child,{toolchain,parentMain:main}),trusted:scriptCaches}:null;
       guard=testRunGuard(testRecord,{root,sourcePatterns:plan.sourcePatterns||plan.patterns,files:snap?.files,
-        runner:runnerPid,environment:selected.environment||{},coverage});
+        runner:runnerPid,environment:selected.environment||{},coverage,programs});
       runFacts=guard.runFacts;
+      // The pnpm executable reads its configuration natively: kept by content.
+      if(guard.opaque)for(const file of pnpmConfigFiles())try{runFacts.push(['readFile',file,queryAnswer('readFile',file)]);}
+        catch{guard.issues.push(`pnpm configuration ${file} cannot be read`);}
       const shared=sharedInstalledFile(root,testFacts,path.join(state,'pnpm-store-links-v1.json'));
       if(shared)guard.issues.push(`installed file ${shared.file} ${shared.reason}; a change through it is not observed`);
       fs.rmSync(testRecord,{recursive:true,force:true});
@@ -392,7 +408,8 @@ async function execute() {
       const directory=rel=>{try{return fs.lstatSync(path.join(root,rel)).isDirectory();}catch{return false;}};
       const changed=changedFiles(snap.files,end.files),events=(end.inputEventPaths||[]).filter(rel=>!directory(rel));
       const touched=[...new Set([...changed,...events])].sort();
-      const ignored=touched.length&&touched.every(guard.isOutput)?gitIgnored(root,touched):new Set();
+      const output=rel=>guard.isOutput(rel)||general&&toolOutput(rel,guard);
+      const ignored=touched.length&&touched.every(output)?gitIgnored(root,touched):new Set();
       if(touched.every(file=>ignored.has(file))){eventsExplained=true;if(changed.length)outputs=changed;}
     }
     // The end certificate, recorded or probed, must cover what this compile received.
@@ -447,9 +464,10 @@ async function execute() {
     }
     // A recorded run's receipt carries its run facts, even none; observers that
     // predate run facts do not reuse version 2.
-    const receipt={result,invocation,target,stable,fingerprintVersion:recorded||testRun?2:1,
+    // The recipe that made the receipt, which status and the hook name.
+    const receipt={result,invocation,target,stable,fingerprintVersion:recorded||testRun?2:1,recipe:selected.qualification||null,
       ...(recorded||testRun?{runFacts}:{}),
-      ...(testRun?{verificationRecipe:testsInterpretation,guardIssues:guard?.issues||['the run record is unavailable'],
+      ...(testRun?{verificationRecipe:general?scriptInterpretation:testsInterpretation,guardIssues:guard?.issues||['the run record is unavailable'],
         environmentReads:guard?.environmentReads||{},...(nondeterministic?{nondeterministic}:{})}:{}),
       resolutionLinks:plan.resolutionLinks||[],
       coverage:{qualified:plan.unresolved.length===0,unresolved:plan.unresolved},

package/src/daemon.mjs

@@ -352,6 +352,7 @@ function rows(contextHashes,synchronizedInstalled=false,synchronizedFilesystem=f
       plan_id:row.plan.id,observed_at:lastObservation,
       invocation:r?.invocation||null,target_provenance:r?.target||null,
       ...(r?.verificationRecipe?{verification_recipe:r.verificationRecipe}:{}),
+      ...(r?.recipe?{recipe:r.recipe}:{}),
       coverage_at_run:r?.coverage||null,declared_inputs_match:null,reuse_eligible:false,
       ...(r?.runFacts?.length?{run_facts:r.runFacts.length}:{})};
     if(receiptStateError){item.reason=receiptStateError;return item;}
@@ -794,8 +795,8 @@ function markInputEvent(name,rel,type,replay) {
   if(!hasHistory||historyDisabled||replay){
     const serial=(inputEventSerial.get(name)||0)+1;
     inputEventSerial.set(name,serial);
-    // Kept only for test checks; trimmed in batches.
-    if(index?.checks.get(name)?.plan.qualification==='npm-tests-direct-v1'){
+    // Kept only for recorded test and script checks; trimmed in batches.
+    if(['npm-tests-direct-v1','node-script-v1'].includes(index?.checks.get(name)?.plan.qualification)){
       const list=inputEventPaths.get(name)||[];list.push([serial,rel]);
       if(list.length>2*eventPathLimit)list.splice(0,list.length-eventPathLimit);inputEventPaths.set(name,list);
     }

package/src/declared-plan.mjs

@@ -7,6 +7,7 @@ import {permittedRoots,resolveDeclaredInstalled} from './installed-inputs.mjs';
 import {qualifyTypeScript} from './typescript-qualification.mjs';
 import {directBuildInfo} from './direct-typescript.mjs';
 import {qualifyTests,testWorkspacePatterns,testsQualification} from './test-direct.mjs';
+import {qualifyNodeScript,scriptQualification} from './node-script.mjs';
 import {yarnWorkspaceTypecheckInputs} from './yarn-workspace-inputs.mjs';
 import {pnpmTypecheckInputs} from './pnpm-inputs.mjs';
 import {checkRun} from './typescript-invocation.mjs';
@@ -82,6 +83,7 @@ export function discoverDeclared(config) {
       [...(selected.installedInputs||[]),...(workspace?.installed||[]),...excluded],allowedRoots);
     timing('plan.installed_resolution',installedStarted,{mappings:installed.mappings.length});
     const automatic=selected.qualification===testsQualification?qualifyTests(root,selected):
+      selected.qualification===scriptQualification?qualifyNodeScript(root,selected):
       qualifyTypeScript(root,selected,{workspace,issue:workspaceIssue});
     const sourcePatterns=[...new Set([...selected.inputs,...(workspace?.source||[]),
       ...(workspace?.generated||[]),...excluded])].sort();
@@ -92,9 +94,10 @@ export function discoverDeclared(config) {
       ...(workspaceIssue?[workspaceIssue]:[])];
     for(const item of workspace?.configurationInputs||[])configurationInputs.add(item);
     for(const item of workspace?.workspacePatterns||[])workspacePatterns.add(item);
-    // A workspace test check is qualified against the current package list: a
-    // package manifest appearing or pnpm's package list changing re-plans.
-    if(selected.qualification===testsQualification&&selected.workspace)try{
+    // A workspace test check, and a general check in a workspace, is qualified
+    // against the current package list: a package manifest appearing or pnpm's
+    // package list changing re-plans.
+    if(selected.qualification===testsQualification&&selected.workspace||selected.qualification===scriptQualification)try{
       for(const item of testWorkspacePatterns(root,JSON.parse(fs.readFileSync(path.join(root,'package.json'),'utf8'))))
         workspacePatterns.add(item);
       if(fs.existsSync(path.join(root,'pnpm-workspace.yaml')))configurationInputs.add('pnpm-workspace.yaml');

package/src/index.mjs

@@ -55,7 +55,7 @@ function fileHash(root, rel,installed=false,before=fs.lstatSync(path.join(root,r
     const target=fs.statSync(resolution.physical,{bigint:true});
     const physical=path.relative(root,resolution.physical).split(path.sep).join('/');
     const directory=target.isDirectory()&&installed&&observedDir?.(physical);
-    if(!target.isFile()&&!directory)throw Error(`directory input symlink unsupported: ${rel}`);
+    if(!target.isFile()&&!directory)throw Error(`directory input symlink unsupported: ${rel} -> ${physical}`);
     if(directory)value='dirlink:'+JSON.stringify(resolution.links)+':'+physical;
     else{
       if(installed)racy=racyUntil(target);else bytes=Number(target.size);
@@ -192,10 +192,16 @@ export class InputIndex {
       const installed=this.installedPath(rel);
       try{const started=performance.now();
         // A linked directory counts as observed only strictly inside an
-        // installed root, and only if every check matching the link also
-        // observes the directory's own files (its package.json stands for them).
+        // installed root, or as a checkout directory outside node_modules (a
+        // workspace package the package manager links), and only if every
+        // check matching the link also observes the directory's own files
+        // (its package.json stands for them).
+        // The second form is for recorded checks only: their recorder
+        // classifies a read through the link where it leads.
+        const recorded=name=>['node-script-v1','npm-tests-direct-v1'].includes(this.checks.get(name).plan.qualification);
         const observedDir=installed?physical=>!this.includeInstalled&&
-          [...this.installedRootSet].some(root=>physical.startsWith(root+'/'))&&
+          ([...this.installedRootSet].some(root=>physical.startsWith(root+'/'))||
+            !physical.split('/').includes('node_modules')&&!this.installedPath(physical+'/package.json')&&matched.every(recorded))&&
           matched.every(name=>this.matches(this.checks.get(name),physical+'/package.json')):null;
         inspection=fileHash(this.root,rel,installed,stat,observedDir);
         fresh=inspection.hash;

package/src/node-onboarding.mjs

@@ -1,5 +1,9 @@
 import {directProject,directQualification} from './direct-typescript.mjs';
-import {testProject,testRecipeScript,testCheckInputs,testsQualification} from './test-direct.mjs';
+import {testProject,testRecipeScript,testCheckInputs,testEnvironmentIssue,testsQualification} from './test-direct.mjs';
+import {commandLineIssue,scriptCheckInputs,scriptPath,scriptProject,scriptQualification} from './node-script.mjs';
+// Init proposes the general recipe for typecheck and lint scripts; test
+// scripts use the direct test recipe (Vitest, Jest).
+const generalKinds=['typecheck','lint'];
 import fs from 'node:fs';
 import path from 'node:path';
 import {spawnSync} from 'node:child_process';
@@ -130,7 +134,8 @@ function sourceInputs(root,kind,lockfiles) {
     '!node_modules/**','!.git/**'])];
   const dirs=likelyDirs.filter(name=>exists(path.join(root,name)))
     .map(name=>`${name}/**`);
-  return [...new Set([...files,...(dirs.length?dirs:['package.json'])])];
+  // A workspace package's node_modules under them is installed, not source.
+  return [...new Set([...files,...(dirs.length?[...dirs,'!**/node_modules/**']:['package.json'])])];
 }
 function classify(kind,scriptName,script,pkg,managerInfo,root) {
   const value=script.trim();
@@ -191,8 +196,27 @@ export function workspacePackages(root){
   return workspaceManifests(root,patterns).map(({file,pkg:member})=>
     ({name:member.name,dir:path.posix.dirname(file),scripts:member.scripts&&typeof member.scripts==='object'?member.scripts:{}}));
 }
+// The general recipe for a root script, or why it does not apply: a
+// supported package manager and platform, and a script (with its pre and post
+// scripts) that, read as text, starts only Node programs.
+function generalCheck(root,kind,script){
+  if(!['darwin','linux'].includes(process.platform))throw Error('the general recipe records runs on macOS and Linux only');
+  const environmentIssue=testEnvironmentIssue();
+  if(environmentIssue)throw Error(environmentIssue);
+  let project;
+  try{project=scriptProject(root,script);}catch(e){throw Error(e.code?.replace(/^test-/,'')||e.message);}
+  for(const name of ['pre'+script,script,'post'+script]){
+    if(typeof project.scripts[name]!=='string')continue;
+    const issue=commandLineIssue(project.scripts[name],{cwd:root,searchPath:scriptPath(root,process.env.PATH)});
+    if(issue)throw Error(name===script?issue:`${name}: ${issue}`);
+  }
+  return {config:{name:kind,script,kind,qualification:scriptQualification,...scriptCheckInputs(project.dirs)},manager:project.manager};
+}
 export function discoverNodeProject(root,{recipe=null}={}) {
-  if(recipe&&!['typescript-direct','tests-direct'].includes(recipe))throw Error('unknown recipe; use typescript-direct or tests-direct');
+  if(recipe&&!['typescript-direct','tests-direct','node-script'].includes(recipe))
+    throw Error('unknown recipe; use typescript-direct, tests-direct or node-script');
+  // Without --recipe, init tries the strict recipes first, then the general one.
+  const auto=recipe===null;
   root=realObservedPath(root);
   const packageFile=path.join(root,'package.json');
   if(!exists(packageFile))throw Error('run init from a repository containing package.json');
@@ -225,7 +249,7 @@ export function discoverNodeProject(root,{recipe=null}={}) {
     if(workspaces&&classification.qualification==='typescript-noemit-v1')classification={level:'recording',
       reason:'workspace task and installed-input closure need explicit qualification'};
     const check={name:kind,script,kind,inputs:sourceInputs(root,kind,pm.lockfiles||[])};
-    if(recipe==='typescript-direct'&&kind==='typecheck'){
+    if((recipe==='typescript-direct'||auto&&pm.name==='npm'&&!workspaces)&&kind==='typecheck'){
       try{
         if(pm.name!=='npm'||workspaces||observationIssue||pm.issue)throw Error('direct recipe requires a supported single-package npm checkout');
         let version;
@@ -236,7 +260,7 @@ export function discoverNodeProject(root,{recipe=null}={}) {
           reason:`explicit local TypeScript ${version} compilation; npm launcher/lifecycle scripts are not executed; package.json stays unchanged`};
       }catch(e){classification={level:'recording',reason:`Direct recipe unavailable (${e.code||e.message}); original npm script retained, without bypassing required behavior`};}
     }
-    if(recipe==='tests-direct'&&kind==='test'){
+    if((recipe==='tests-direct'||auto)&&kind==='test'){
       try{
         if(!['npm','pnpm','yarn'].includes(pm.name)||workspaces||observationIssue||pm.issue)
           throw Error('test recipe requires a supported single-package npm, pnpm or Yarn checkout');
@@ -250,7 +274,7 @@ export function discoverNodeProject(root,{recipe=null}={}) {
           reason:`explicit local ${project.runner} ${project.version} run of script ${JSON.stringify(chosen.script)}; the whole checkout is its input; npm and lifecycle scripts are not executed`};
       }catch(e){classification={level:'recording',reason:`Test recipe unavailable (${e.code||e.message}); original npm script retained`};}
     }
-    if(pm.name==='pnpm'&&kind==='typecheck'&&!workspaces&&!pm.issue&&(script===undefined||tscRuns(scripts[script]))){
+    if(recipe!=='node-script'&&pm.name==='pnpm'&&kind==='typecheck'&&!workspaces&&!pm.issue&&(script===undefined||tscRuns(scripts[script]))){
       try{
         parts=tscChecks('typecheck',script??null,scripts,run=>{
           const contract=pnpmTypecheckInputs(root,'.',run);
@@ -263,6 +287,14 @@ export function discoverNodeProject(root,{recipe=null}={}) {
           qualification:'pnpm-tsc-v1'};}
       catch(e){classification={level:'recording',reason:`pnpm TypeScript contract needs review: ${e.message}`};}
     }
+    if((auto||recipe==='node-script')&&script&&generalKinds.includes(kind)&&classification.level!=='ready'&&!observationIssue){
+      try{
+        const general=generalCheck(root,kind,script);
+        checks.push({config:general.config,level:'ready',qualification:scriptQualification,
+          reason:`general recipe: ${general.manager} runs script ${JSON.stringify(script)} with REDUE's recorder in every Node process; the whole checkout is its input`});
+        continue;
+      }catch(e){classification={...classification,reason:`General recipe unavailable (${e.message}); ${classification.reason}`};}
+    }
     if(!script&&classification.level!=='ready')continue;
     if(parts&&classification.level==='ready'){
       for(const part of parts)checks.push({config:{...check,...part,qualification:classification.qualification},...classification});
@@ -302,7 +334,7 @@ export function discoverNodeProject(root,{recipe=null}={}) {
           ['@which:pnpm','--filter',name,'run',selected.script];
         const check={name:`${name}:${kind}`,kind,command,
           inputs:['package.json',...(pm.lockfiles||[]),`${dir}/**`,
-            `!${dir}/node_modules/**`]};
+            `!${dir}/node_modules/**`,`!${dir}/**/node_modules/**`]};
         let level=pm.issue&&pm.name!=='pnpm'?'unsupported':'recording';
         let reason=pm.issue||'workspace script found; shared configuration and installed inputs need review';
         if(!pm.issue&&pm.name==='yarn'&&pm.layout==='node-modules/classic'&&