@redue/cli 0.1.0-alpha.3 → 0.1.0-alpha.30

package/src/cli.mjs

@@ -1,5 +1,15 @@
 #!/usr/bin/env node
+import {builtInProbe,probeFloorMs,publishedDeadline} from './probe-deadline.mjs';
 import fs from 'node:fs';
+import {directQualification,directInterpretation,observedNpmContext} from './direct-typescript.mjs';
+import {directRunCertificateAllowed,directRunCertificate,directRunRecordable,directRunRecording,probeQuestions,validateRunRecord} from './direct-run-certificate.mjs';
+import {queryAnswer,runFactChange} from './typescript-list.mjs';
+import {patternMatcher} from './glob.mjs';
+import {observedBy} from './installed-observation.mjs';
+import {changedFiles,coverageReadable,gitIgnored,recordedEnvironment,testEnvironmentIssue,testGuardConfig,testRunGuard,testRunFacts,sharedInstalledFile,testsInterpretation,testsQualification} from './test-direct.mjs';
+import {childProgram,generalIssue,lookup,pnpmConfigFiles,scriptCaches,scriptInstalledPaths,scriptInterpretation,scriptQualification,scriptRunFacts,pnpmNameless,toolOutput,toolchainDirs} from './node-script.mjs';
+import {denialLine,docsQualification,endWithChild,hiddenDocs,hiddenEntries,hiddenQualification,hiddenRefusal,hidesIgnored,isSandboxed,runToEnd,supervisorIssue,visibleChange,testRun as sandboxTestRun,sandboxArgv,sandboxContext,sandboxDirs,sandboxEnv,
+  sandboxInterpretation,sandboxInterpretationOf,sandboxPlaces,sandboxQualification,sandboxSupport,systemPlaces} from './sandbox.mjs';
 import path from 'node:path';
 import net from 'node:net';
 import os from 'node:os';
@@ -15,14 +25,17 @@ import {createHash,randomUUID} from 'node:crypto';
 import {assertOwnedStatePlacement} from './owned-state.mjs';
 import {permittedRoots} from './installed-inputs.mjs';
 import {acquireRunLock,inspectRunLock,recoverRunLock} from './run-lock.mjs';
-import {atomicJson,commitReceipt,receiptRevision} from './state-store.mjs';
-import {timing} from './decision-profile.mjs';
+import {atomicJson,commitReceipt,readReceipts,receiptRevision} from './state-store.mjs';
+import {timing,profiling,mark,traceContext} from './decision-profile.mjs';
 import {readCachedStatus,unavailableCached} from './cached-state.mjs';
 import {renderRun} from './presentation.mjs';
+import {note} from './stats.mjs';
 
+mark('cli.loaded');
 const here=path.dirname(fileURLToPath(import.meta.url));
 const [configArg,action,name]=process.argv.slice(2);
 if(!configArg||!action)throw Error('internal usage: cli.mjs CONFIG ACTION [check]');
+const configStarted=performance.now();
 const configFile=path.resolve(configArg), config=JSON.parse(fs.readFileSync(configFile));
 const maintenance=['stop','uninstall'].includes(action);
 const root=maintenance?path.resolve(config.root):realObservedPath(config.root), state=path.resolve(config.state),
@@ -34,10 +47,28 @@ if([path.parse(state).root,os.homedir(),path.dirname(os.homedir())]
   !/^(?:vstate|redue)-/.test(path.basename(state)))
   throw Error('state directory name must start with redue- (or legacy vstate-); use a dedicated directory outside the checkout');
 const ownerFile=path.join(state,'vstate-owner-v1.json');
+timing('cli.config_resolution',configStarted);
 // A native Windows decision reconciles declared contents instead of trusting
 // watcher history. The measured 6k-file pnpm workspace can legitimately need
 // more than the former 15 s default; uncertainty still wins on expiry.
-const decisionDeadlineMs=process.platform==='win32'?30000:15000;
+// A decision can include a full probe, whose deadline the observer publishes.
+// Read when a wait starts, so a deadline the previous decision raised counts.
+const decisionBaseMs=process.platform==='win32'?30000:15000;
+function publishedProbeDeadline() {
+  try{return publishedDeadline(JSON.parse(fs.readFileSync(path.join(state,'status.json'),'utf8')).probe_deadline_ms);}
+  catch{return 0;}
+}
+// The wait for a decision, read when it starts: the base at once, and the
+// published deadline when status.json has been read. That file is rewritten
+// at every decision, and a first read of a fresh file is slow where a scanner
+// checks files as they are opened, so it is read off the main thread while
+// the call runs. On Windows it is read before the call, as before: the
+// observer replaces it during the call, and a replacement fails there while a
+// reader holds the file open. An override is exact.
+const decisionWait=override=>override?{timeout:Number(override),deadline:null}:
+  process.platform==='win32'?{timeout:Math.max(decisionBaseMs,publishedProbeDeadline()+10000),deadline:null}:{timeout:decisionBaseMs,
+  deadline:fs.promises.readFile(path.join(state,'status.json'),'utf8')
+    .then(text=>Math.max(decisionBaseMs,publishedDeadline(JSON.parse(text).probe_deadline_ms)+10000),()=>decisionBaseMs)};
 const digest=value=>createHash('sha256').update(value).digest('hex');
 function nodeIdentity() {const real=fs.realpathSync(process.execPath),st=fs.statSync(real,{bigint:true});
   return digest(JSON.stringify([real,process.version,...['dev','ino','mode','size','mtimeNs','ctimeNs']
@@ -49,7 +80,15 @@ function pathExecutableIdentity(name) {
   catch{}
   return digest('<unavailable>');
 }
-function contextHashes() {return Object.fromEntries(config.checks.map(check=>[
+// The variables each test check's last run read directly (see testRunGuard).
+const readsFile=path.join(state,'environment-reads-v1.json');
+function environmentReads() {
+  try{const value=JSON.parse(fs.readFileSync(readsFile,'utf8'));return value?.schema===1&&value.checks||{};}catch{return {};}
+}
+const recordedRecipes=[testsQualification,scriptQualification];
+// A sandboxed check's run is recorded where its sandbox cannot start (execute).
+function contextHashes() {const reads=config.checks.some(check=>recordedRecipes.includes(check.qualification)||isSandboxed(check.qualification))?environmentReads():{};
+  return Object.fromEntries(config.checks.map(check=>[
   check.name,Object.fromEntries([...(check.environment?.variables||[])
     .map(key=>[key,digest(process.env[key]??'<unset>')]),
     ...(check.environment?.prefixes||[]).map(prefix=>[
@@ -59,15 +98,22 @@ function contextHashes() {return Object.fromEntries(config.checks.map(check=>[
           .sort(([a],[b])=>a.localeCompare(b))))]),
     ...(check.environment?.pathExecutables||[]).map(name=>[
       `__vstate_path_executable_${name}__`,pathExecutableIdentity(name)]),
-    ...(check.environment?.executableIdentity?[['__vstate_node_identity__',nodeIdentity()]]:[])])]));}
+    ...(check.environment?.executableIdentity?[['__vstate_node_identity__',nodeIdentity()]]:[]),
+    // A sandboxed run's environment, toolchain and places (sandbox.mjs).
+    ...(isSandboxed(check.qualification)?Object.entries(sandboxContext(check,process.env,root)):[]),
+    // The variables the last test run read directly, compared here: which
+    // list was compared, and the first variable whose value now differs.
+    // Control requests are small, so the values stay out of them.
+    ...(reads[check.name]?[['__redue_env_reads__',digest(JSON.stringify(reads[check.name]))],
+      ['__redue_env_changed__',Object.keys(reads[check.name]).find(key=>
+        (process.env[key]===undefined?'<unset>':digest(process.env[key]))!==reads[check.name][key])||'']]:[])])]));}
 function probeHash(plan) {
   const started=performance.now();
   const values=[];
+  // The observer's current deadline, so a large project's probe is not cut short.
+  const timeout=Math.max(probeFloorMs(plan.qualification),publishedProbeDeadline());
   for(const argv of plan.probes||[]) {
-    const timeout=plan.qualification==='pnpm-tsc-v1'||
-      (process.platform==='win32'&&['typescript-noemit-v1','yarn-workspace-tsc-v1']
-        .includes(plan.qualification))?10000:5000;
-    const out=spawnSync(argv[0],argv.slice(1),{cwd:plan.cwd,
+    const out=spawnSync(argv[0],[...argv.slice(1),...(builtInProbe(argv)?[`--redue-deadline=${timeout}`]:[])],{cwd:plan.cwd,
       timeout,maxBuffer:1024*1024,stdio:['ignore','pipe','pipe'],env:process.env});
     if(out.error||out.signal||out.status!==0){
       timing('cli.probe',started,{outcome:out.error?.code||out.signal||out.status||'error',
@@ -90,20 +136,26 @@ function owner(){
     atomicJson(ownerFile,expected);
   }
 }
-function call(message,timeout=5000) {
+// deadline, when given, resolves to the whole wait in ms, which replaces
+// timeout when it is longer.
+function call(message,timeout=5000,deadline=null) {
   return new Promise((resolve,reject)=>{
-    let data='',done=false;const client=net.createConnection(socket);
-    const timer=setTimeout(()=>{client.destroy();reject(Error('observer synchronization timeout'));},timeout);
-    client.on('connect',()=>client.write(JSON.stringify(message)+'\n'));
+    let data='',done=false;mark('client.control_connect',{deadlineMs:timeout});const client=net.createConnection(socket);
+    const started=Date.now(),expire=()=>{mark('client.control_timeout');client.destroy();reject(Error('observer synchronization timeout'));};
+    let timer=setTimeout(expire,timeout);
+    deadline?.then(total=>{if(done||total<=timeout)return;mark('client.control_deadline',{deadlineMs:total});
+      clearTimeout(timer);timer=setTimeout(expire,Math.max(0,total-(Date.now()-started)));});
+    client.on('connect',()=>{mark('client.control_submit');client.write(JSON.stringify({...message,...(profiling?{diagnostic:traceContext()}: {})})+'\n');});
     client.on('data',chunk=>data+=chunk);
-    client.on('end',()=>{clearTimeout(timer);if(done)return;done=true;
+    client.on('end',()=>{mark('client.control_complete',{bytes:data.length});clearTimeout(timer);if(done)return;done=true;
       try{resolve(JSON.parse(data));}catch(e){reject(e);}});
-    client.on('error',e=>{clearTimeout(timer);reject(e);});
+    client.on('error',e=>{mark('client.control_error',{code:e.code});clearTimeout(timer);done=true;reject(e);});
   });
 }
-async function decisionCall(message,timeout) {
-  const started=Date.now(),remaining=()=>Math.max(1,timeout-(Date.now()-started));
-  try{return await call(message,remaining());}
+async function decisionCall(message,{timeout,deadline}) {
+  let total=timeout;deadline?.then(value=>{total=Math.max(total,value);});
+  const started=Date.now(),remaining=()=>Math.max(1,total-(Date.now()-started));
+  try{return await call(message,timeout,deadline);}
   catch(error){
     if(process.platform!=='win32'||!['ENOENT','ECONNREFUSED','ECONNRESET'].includes(error.code))throw error;
     const pidFile=path.join(state,'observer.lock','pid');
@@ -144,16 +196,32 @@ async function decisionCall(message,timeout) {
   }
 }
 const cached=()=>readCachedStatus(state,config.checks);
+async function testRunBoundary(phase) {
+  if(process.env.VSTATE_TEST_FAULTS!=='1'||process.env.VSTATE_TEST_RUN_BOUNDARY!==phase)return;
+  const release=path.join(state,'test-run-release');
+  fs.writeFileSync(path.join(state,'test-run-paused'),phase);
+  const until=Date.now()+5000;
+  while(!fs.existsSync(release)&&Date.now()<until)await new Promise(resolve=>setTimeout(resolve,10));
+  if(!fs.existsSync(release))throw Error('test run boundary was not released');
+}
 async function execute() {
   const totalStarted=performance.now();
   const selected=config.checks.find(c=>c.name===name);
   if(!selected)throw Error(`unknown check ${name}`);
-  const runLock=acquireRunLock(state);
+  // Recording uses existing state only. Never initialize or repair ownership
+  // as a fallback for unavailable observation.
+  const expectedOwner={schema:1,kind:'vstate-product-alpha',config:configFile,root,state};
+  const existingOwner=JSON.parse(fs.readFileSync(ownerFile,'utf8'));
+  if(JSON.stringify(existingOwner)!==JSON.stringify(expectedOwner))
+    throw Error('state ownership marker mismatch');
+  const lockStarted=performance.now(),runLock=acquireRunLock(state);
+  timing('cli.run_lock',lockStarted);
   const captureIssues=[];
-  let activeChild=null,requestedSignal=null,treeStop=null,preserveRunLock=false;
+  let activeChild=null,requestedSignal=null,treeStop=null,preserveRunLock=false,ownGroup=false;
   const forward=signal=>{requestedSignal=signal;
     if(activeChild&&activeChild.exitCode===null){
-      if(process.platform==='linux'&&activeChild.pid){
+      // A Linux run, and a sandboxed one on macOS, has its own process group.
+      if((process.platform==='linux'||ownGroup)&&activeChild.pid){
         try{process.kill(-activeChild.pid,signal);}
         catch(e){if(e.code!=='ESRCH')throw e;}
       }else if(process.platform==='win32'&&activeChild.pid){
@@ -172,6 +240,7 @@ async function execute() {
   try {
     if(!fs.existsSync(path.join(state,'plans-v1.json')))
       throw Error('observer input plan is not ready; wait for redue status to report ready');
+    const resolutionStarted=performance.now();
     const plan=JSON.parse(fs.readFileSync(path.join(state,'plans-v1.json'))).plans[name];
     if(!plan)throw Error('selected input plan unavailable');
     if(plan.selectedConfigHash!==digest(JSON.stringify(selected)))
@@ -181,22 +250,120 @@ async function execute() {
       throw Error('invalid execution argv');
     if(digest(JSON.stringify(command))!==plan.commandIdentity.argvHash)
       throw Error('execution argv differs from discovered plan');
+    timing('cli.command_resolution',resolutionStarted);
     const runId=randomUUID();
     const contextStarted=performance.now();
     const environmentBefore=contextHashes()[name];
+    const observedContext=selected.qualification===directQualification?{interpretation:directInterpretation,npmProxyDigest:observedNpmContext()}:null;
     timing('cli.context_before',contextStarted);
-    const
-      captureTimeout=Number(process.env.VSTATE_CAPTURE_TIMEOUT_MS||decisionDeadlineMs);
+    const captureWait=decisionWait(process.env.VSTATE_CAPTURE_TIMEOUT_MS);
     let before=null,after=null;
     // Expensive state probes can take seconds. Keep the filesystem checkpoint
     // adjacent to process launch so a late notification from earlier work
     // does not enter the execution interval before the check even starts.
-    const probeBefore=probeHash({cwd:plan.cwd,probes:selected.probes||[],
-      qualification:selected.qualification});
+    const callerCertificate=directRunCertificateAllowed(selected);
+    const probePlan={cwd:plan.cwd,probes:selected.probes||[],qualification:selected.qualification};
+    let probeBefore=callerCertificate?null:probeHash(probePlan);
     const startSnapshotStarted=performance.now();
-    try{before=await call({action:'snapshot',name,contextHashes:contextHashes()},captureTimeout);}
-    catch{captureIssues.push('start observation unavailable');}
+    try{before=await call({action:'snapshot',name,contextHashes:contextHashes(),
+      ...(callerCertificate?{runCertificate:1}:{})},captureWait.timeout,captureWait.deadline);}
+    catch(error){captureIssues.push('start observation unavailable'+
+      (typeof error.code==='string'&&/^[A-Z0-9_]+$/.test(error.code)?` (${error.code})`:''));}
     timing('cli.start_checkpoint',startSnapshotStarted,{available:before?.snapshots?.[name]?1:0});
+    // Missing discovery metadata can recover through a full probe. A missing
+    // healthy start boundary cannot: freeze this decision before any recovery.
+    // Only the reviewed direct recipe on macOS and Linux enters this shortcut.
+    const unobservedDirectRun=callerCertificate&&
+      before?.snapshots?.[name]?.observationHealthy!==true;
+    if(unobservedDirectRun){
+      captureIssues.push('qualification probes and end checkpoint not performed: no healthy start checkpoint');
+      mark('cli.run_validation_skipped',{reason:'start_observation_unavailable',probes:2,endCheckpoint:1});
+    }
+    await testRunBoundary('after-start-checkpoint');
+    // Start the observed interval before caller validation, not after it. Any
+    // intervening edit (even restored before launch) remains disqualifying.
+    // With a certificate, the compile records its own answers instead of two probes.
+    // Without one after an edit, the recording can become the certificate at the
+    // end checkpoint; that checkpoint probes in full if it cannot.
+    let certificate=null,recording=null;
+    if(callerCertificate&&!unobservedDirectRun){
+      certificate=directRunCertificate(plan,selected,before?.snapshots?.[name],root);
+      if(!certificate)recording=directRunRecording(plan,selected,before?.snapshots?.[name],root);
+      probeBefore=certificate?.value??(recording?null:probeHash(probePlan));
+    }
+    // The record arrives on fd 3, so the compiler's environment is unchanged.
+    // Every direct run is recorded: even one checked by full probes needs its
+    // lookups outside the certificate, which no probe makes.
+    const recorded=directRunRecordable(selected)&&!unobservedDirectRun;
+    const launched=recorded?[command[0],'--require',path.join(here,'run-recorder.cjs'),
+      ...command.slice(1)]:command;
+    // A sandboxed check runs inside the OS sandbox. Where the sandbox cannot
+    // start, a script the general recipe takes is recorded as that recipe
+    // records it (`fallback`); any other runs as it is, and is not reused.
+    const sandboxed=isSandboxed(selected.qualification),support=sandboxed?sandboxSupport():null;
+    const fallback=Boolean(support?.issue&&typeof selected.script==='string'&&!generalIssue(root,selected.script));
+    // A test run's processes record what the guard checks (test-recorder.cjs),
+    // and so do a general recipe run's, through the package manager.
+    const general=selected.qualification===scriptQualification||fallback;
+    const testRun=(recordedRecipes.includes(selected.qualification)||fallback)&&['darwin','linux'].includes(process.platform);
+    let testRecord=null,launchEnv=process.env,testFacts={},toolchain=[];
+    if(testRun){
+      // The package directory and installed node_modules, as the plan has them.
+      try{testFacts=general?scriptRunFacts(root,selected):testRunFacts(root,selected);}
+      catch(e){captureIssues.push(`${general?'script':'test'} run facts unavailable (${e.code||e.message})`);}
+      const environmentIssue=testEnvironmentIssue();
+      if(environmentIssue)captureIssues.push(environmentIssue);
+      testRecord=path.join(state,`test-record-${runId}`);
+      fs.mkdirSync(testRecord,{mode:0o700});
+      const guardFile=path.join(testRecord,'guard.json'),guardConfig=testGuardConfig(testRecord,root,process.env,testFacts);
+      if(general){toolchain=toolchainDirs();
+        // The Node on PATH, which the environment identifies by its file.
+        Object.assign(guardConfig,{installed:scriptInstalledPaths(root,testFacts.dirs||[]),toolchain,
+          node:lookup('node',process.env.PATH,process.cwd())});}
+      fs.writeFileSync(guardFile,JSON.stringify(guardConfig),{mode:0o600});
+      launchEnv=recordedEnvironment(guardFile);
+      // npm's and pnpm's update checks read state outside the checkout and the network.
+      if(general)launchEnv.npm_config_update_notifier='false';
+    }
+    // The sandboxed run, with the environment REDUE gives it.
+    let sandbox=null;
+    if(sandboxed&&!fallback){
+      if(support.issue)captureIssues.push(`the run was not sandboxed: ${support.issue}`);
+      else{
+        const dirs=sandboxDirs(state,name);
+        // A test run: loopback networking on Linux, supervised (the launcher
+        // reports in REDUE's state, outside what the test may touch), a
+        // private /tmp, and a fresh home each run: what a test leaves in its
+        // home is not observed, so it must not reach a later run.
+        const tests=sandboxTestRun(selected),stats=tests&&support.mode==='landlock'?path.join(state,`launcher-${runId}.json`):null;
+        if(tests){dirs.home=path.join(dirs.tmp,'home');fs.mkdirSync(dirs.home,{mode:0o700});}
+        const reads=[root,dirs.home,dirs.tmp,...sandboxPlaces(selected,process.env,root),...systemPlaces];
+        // sandbox-v2 hides the checkout's git-ignored entries, and sandbox-v3
+        // its documentation too. Where git cannot list them, nothing is
+        // hidden, and the receipt compares every file.
+        const hidden=hidesIgnored(selected.qualification)?hiddenEntries(root):null;
+        const docs=hidden&&selected.qualification===docsQualification?hiddenDocs(root):null;
+        const options={reads,tmp:dirs.tmp,python:support.python,stats,
+          ...tests?{...support.mode==='landlock'?{network:'loopback'}:{},writes:[root,dirs.home,dirs.tmp],
+            fault:process.env.VSTATE_TEST_FAULTS==='1'?process.env.VSTATE_TEST_SUPERVISOR||null:null}:{}};
+        sandbox={mode:support.mode,dirs,reads,hidden,docs,options,stats,
+          argv:sandboxArgv(support.mode,command,{...options,hidden:(hidden||[]).map(rel=>path.join(root,rel.replace(/\/$/,''))),
+            ...docs?{docs:{root,entries:docs}}:{}}),
+          env:sandboxEnv(selected,process.env,{root,...dirs})};
+      }
+    }
+    const plainEnv={...process.env,...selected.script?{npm_config_update_notifier:'false'}:{}};
+    // Output kept, and its own process group, for a run as this recipe makes it.
+    const own=sandboxed&&!fallback;
+    ownGroup=own&&process.platform!=='win32';
+    // A sandboxed run's output is passed through and its start and end kept,
+    // to tell a refusal by the sandbox from a real failure.
+    const output={head:'',tail:''};
+    const keep=chunk=>{const text=chunk.toString('utf8');
+      if(output.head.length<65536)output.head+=text.slice(0,65536-output.head.length);
+      output.tail=(output.tail+text).slice(-65536);};
+    let runnerPid=null;
+    const record=[];
     // A Windows full reconciliation may finish while queued watcher callbacks
     // are still arriving. Fold callbacks delivered before process launch into
     // the starting checkpoint; callbacks after launch still disqualify reuse.
@@ -204,29 +371,40 @@ async function execute() {
       await new Promise(resolve=>setTimeout(resolve,100));
       const prelaunchStarted=performance.now();
       try{before=await call({action:'prelaunch',name,
-        contextHashes:contextHashes()},Math.min(captureTimeout,5000));}
+        contextHashes:contextHashes()},Math.min(captureWait.timeout,5000));}
       catch{before=null;captureIssues.push('prelaunch observation unavailable');}
       timing('cli.prelaunch_checkpoint',prelaunchStarted,
         {available:before?.snapshots?.[name]?1:0});
     }
     const snap=before?.snapshots?.[name];
     if(!snap||!snap.observationHealthy)captureIssues.push('start checkpoint unavailable');
-    const started=Date.now();
+    await testRunBoundary('before-launch');
+    const started=Date.now(),executionStarted=performance.now();
     const invocation=await new Promise(resolve=>{
       if(requestedSignal){resolve({status:'interrupted',exitCode:null,signal:requestedSignal});return;}
       let child,spawnError=null;
       try{runLock.phase('launching');
-        const launch=windowsLaunch(command);
+        const launch=windowsLaunch(sandbox?.argv??launched);
         child=spawn(launch.file,launch.args,{cwd:plan.cwd,
-        stdio:'inherit',env:process.env,detached:process.platform==='linux',
-        ...launch.options});}
+        // A sandboxed run's own group gets no terminal input, which would stop it.
+        stdio:recorded?['inherit','inherit','inherit','pipe']:own?['ignore','pipe','pipe']:'inherit',
+        env:sandbox?.env??(own?plainEnv:launchEnv),
+        detached:process.platform==='linux'||ownGroup,...launch.options});
+        child.stdio[3]?.on('data',chunk=>record.push(chunk));
+        if(own){child.stdout.on('data',chunk=>{process.stdout.write(chunk);keep(chunk);});
+          child.stderr.on('data',chunk=>{process.stderr.write(chunk);keep(chunk);});
+          // What the check left running is stopped when it exits, as a CI
+          // job's end does (on Linux the launcher also stops daemons that
+          // left the group), and the run ends a second after its exit.
+          endWithChild(child);}}
       catch(e){resolve({status:'start_failed',exitCode:null,signal:null,errorCode:e.code||e.name});return;}
-      activeChild=child;runLock.phase('running',child.pid??null);
+      activeChild=child;runnerPid=child.pid??null;runLock.phase('running',child.pid??null);
+      mark('cli.child_started',{childPid:child.pid});
       if(requestedSignal)forward(requestedSignal);
       child.on('error',e=>{spawnError=e.code||e.name;});
       child.on('close',async(exit,signal)=>{
         if(treeStop)await treeStop;
-        if(process.platform==='linux'&&requestedSignal&&child.pid){
+        if((process.platform==='linux'||ownGroup)&&requestedSignal&&child.pid){
           // A verification command may have forked children in its process
           // group. Normal cancellation must reach them even if the leader has
           // already exited. Commands that daemonize into a new session remain
@@ -241,35 +419,226 @@ async function execute() {
       });
     });
     const ended=Date.now();
+    timing('cli.compiler_execution',executionStarted,{status:invocation.status,exit:invocation.exitCode});
+    await testRunBoundary('after-exit');
     invocation.startedAt=started;invocation.durationMs=ended-started;
     invocation.argvHash=plan.commandIdentity.argvHash;
     invocation.argc=command.length;
     invocation.runId=runId;
     invocation.reporting='normal';
-    const result=invocation.status==='exited'?(invocation.exitCode===0?'PASS':'FAIL'):null;
+    let result=invocation.status==='exited'?(invocation.exitCode===0?'PASS':'FAIL'):null;
     const target={source:'direct-command',status:invocation.status==='exited'?'direct_executed':'unknown'};
-    const environmentAfter=contextHashes()[name],probeAfter=probeHash({cwd:plan.cwd,
-      probes:selected.probes||[],qualification:selected.qualification});
+    const afterContextStarted=performance.now(),environmentAfter=contextHashes()[name];
+    timing('cli.context_after',afterContextStarted);
+    let probeAfter=null;
+    const recordText=Buffer.concat(record).toString('utf8');
+    // Lookups the certificate does not cover, kept with the receipt.
+    let runFacts=[];
+    if(certificate){
+      const issue=validateRunRecord(recordText,certificate,runFacts);
+      if(issue)captureIssues.push(`compiler inputs differ from discovery certificate (${issue})`);
+      probeAfter=certificate.value;
+    }else if(!unobservedDirectRun&&!recording)probeAfter=probeHash(probePlan);
+    let guard=null;
+    if(testRecord){
+      let coverage=null;if(!general)try{coverage=coverageReadable(root,testFacts);}catch{}
+      const programs=general?{runner:{file:command[0],args:command.slice(1),cwd:plan.cwd,searchPath:process.env.PATH,
+        env:{preload:true,bashEnv:Boolean(process.env.BASH_ENV)}},
+        judge:(child,main)=>childProgram(child,{toolchain,parentMain:main}),trusted:scriptCaches}:null;
+      guard=testRunGuard(testRecord,{root,sourcePatterns:plan.sourcePatterns||plan.patterns,files:snap?.files,
+        runner:runnerPid,environment:selected.environment||{},coverage,programs});
+      runFacts=guard.runFacts;
+      // The pnpm executable reads its configuration natively: kept by content.
+      if(guard.opaque)for(const file of pnpmConfigFiles())try{runFacts.push(['readFile',file,queryAnswer('readFile',file)]);}
+        catch{guard.issues.push(`pnpm configuration ${file} cannot be read`);}
+      // Hard links count only where the check observes them.
+      // As the index matches: source patterns, or additive (installed) ones.
+      const source=patternMatcher(plan.sourcePatterns||plan.patterns||[]),additive=patternMatcher(plan.additivePatterns||[]);
+      const linkFacts=[],observed=observedBy(root,rel=>source(rel)||additive(rel));
+      const shared=sharedInstalledFile(root,testFacts,path.join(state,'pnpm-store-links-v1.json'),{observed,facts:linkFacts});
+      if(shared)guard.issues.push(`installed file ${shared.file} ${shared.reason}; a change through it is not observed`);
+      else runFacts.push(...linkFacts);
+      fs.rmSync(testRecord,{recursive:true,force:true});
+    }
+    // The daemon reads the recording from owned state; control requests are small.
+    const recordFile=recording&&path.join(state,`run-record-${recording}.json`);
+    if(recordFile)try{fs.writeFileSync(recordFile,recordText,{mode:0o600});}catch{}
     const endSnapshotStarted=performance.now();
-    try{after=await call({action:'snapshot',name,contextHashes:contextHashes()},captureTimeout);}
-    catch{captureIssues.push('end observation unavailable');}
-    timing('cli.end_checkpoint',endSnapshotStarted,{available:after?.snapshots?.[name]?1:0});
+    if(!unobservedDirectRun){
+      // The start checkpoint's probe may have raised the deadline.
+      const endWait=decisionWait(process.env.VSTATE_CAPTURE_TIMEOUT_MS);
+      try{after=await call({action:'snapshot',name,contextHashes:contextHashes(),
+        ...((testRun||sandboxed)&&Number.isSafeInteger(snap?.inputEventSerial)?{sinceSerial:snap.inputEventSerial}:{}),
+        ...(recording?{runCertificate:1,adopt:recording}:recorded&&!certificate?{runCertificate:1,runFacts:1}:{})},
+        endWait.timeout,endWait.deadline);}
+      catch{captureIssues.push('end observation unavailable');}
+      finally{if(recordFile)fs.rmSync(recordFile,{force:true});}
+      timing('cli.end_checkpoint',endSnapshotStarted,{available:after?.snapshots?.[name]?1:0});
+    }
     const end=after?.snapshots?.[name];
-    if(!end||!end.observationHealthy)captureIssues.push('end checkpoint unavailable');
+    // A test run's own outputs: git-ignored files it created without reading
+    // what was there (test/tmp). When they are the only observed change, the
+    // run is kept with the end checkpoint, which includes them.
+    let outputs=null;
+    let eventsExplained=false,unexplained=end?.inputEventPaths||null;
+    if(guard&&snap&&end&&snap.planId===end.planId&&snap.probeHash===end.probeHash&&snap.observationHealthy&&end.observationHealthy&&
+      (snap.inputEventSerial===end.inputEventSerial||end.inputEventPathsComplete===true)){
+      // A directory event is not an input change; a change to its files shows
+      // in the end checkpoint's file map, which is compared too. A link is kept.
+      const directory=rel=>{try{return fs.lstatSync(path.join(root,rel)).isDirectory();}catch{return false;}};
+      const changed=changedFiles(snap.files,end.files),events=(end.inputEventPaths||[]).filter(rel=>!directory(rel));
+      const touched=[...new Set([...changed,...events])].sort();
+      const output=rel=>guard.isOutput(rel)||general&&(toolOutput(rel,guard)||pnpmNameless(rel,guard,root,[snap.files,end.files]));
+      const ignored=touched.length&&touched.every(output)?gitIgnored(root,touched):new Set();
+      unexplained=touched.filter(file=>!ignored.has(file));
+      if(!unexplained.length){eventsExplained=true;if(changed.length)outputs=changed;}
+    }
+    // A run that hid git-ignored files: what it wrote beneath them (test
+    // output, coverage data) is not a change to its inputs (visibleChange);
+    // anything else is named.
+    let hiddenOnly=false;
+    if(sandbox?.hidden&&snap&&end&&snap.planId===end.planId&&snap.observationHealthy&&end.observationHealthy&&
+      (snap.inputEventSerial===end.inputEventSerial||end.inputEventPathsComplete===true)){
+      const before=snap.files||{},after=end.files||{},events=end.inputEventPaths||[];
+      // During the run a new file outside the hidden entries was readable: it counts.
+      const found=visibleChange(root,sandbox.hidden,before,after,[...Object.keys(before),...Object.keys(after),...events],
+        {during:true,docs:sandbox.docs});
+      if(found===null){eventsExplained=true;hiddenOnly=true;}else unexplained=[found];
+    }
+    // The end certificate, recorded or probed, must cover what this compile received.
+    // With no input event, gap or content change in between, it also holds at the start.
+    // A run checked by full probes is compared with it too, for its run facts.
+    if(recorded&&!certificate){
+      const endCertificate=directRunCertificate(plan,selected,end,root),
+        questions=endCertificate||(!recording&&probeQuestions(plan,selected,end,root));
+      if(!questions)captureIssues.push('no discovery certificate at the end checkpoint');
+      else{
+        const issue=validateRunRecord(recordText,questions,runFacts=[]);
+        if(issue)captureIssues.push(`compiler inputs differ from discovery certificate (${issue})`);
+        if(recording)probeBefore=probeAfter=endCertificate.value;
+      }
+    }
+    // The receipt holds only run facts that still answer as they did during the run.
+    runFacts.sort();
+    if(runFacts.length>10000)captureIssues.push('too many lookups outside the discovery certificate');
+    else{const changed=runFactChange(runFacts);
+      if(changed)captureIssues.push(`a lookup outside the discovery certificate changed during execution: ${changed}`);}
+    if(recording&&snap?.observationGaps!==end?.observationGaps)
+      captureIssues.push('observation gap during execution');
+    if(!unobservedDirectRun&&(!end||!end.observationHealthy))captureIssues.push('end checkpoint unavailable');
     if(JSON.stringify(environmentBefore)!==JSON.stringify(environmentAfter))
       captureIssues.push('declared environment changed during execution');
-    if(!probeBefore||!probeAfter||probeBefore!==snap?.probeHash||probeAfter!==end?.probeHash)
+    if(!unobservedDirectRun&&(!probeBefore||!probeAfter||probeBefore!==(recording?end:snap)?.probeHash||
+      probeAfter!==end?.probeHash))
       captureIssues.push('declared state probe unavailable or changed');
     if(before?.observation?.generation!==after?.observation?.generation)
       captureIssues.push('observer generation changed during execution');
     if(snap?.planGeneration!==end?.planGeneration)
       captureIssues.push('input plan rebuilt during execution');
-    if(snap?.inputEventSerial!==end?.inputEventSerial)
-      captureIssues.push('declared input event occurred during execution');
-    if(!snap||!end||snap.planId!==end.planId||snap.fingerprint!==end.fingerprint||
-      snap.revision!==end.revision)captureIssues.push('declared inputs changed or were not observed');
+    // The reason names up to five files the run did not explain, and a count of the rest.
+    if(snap?.inputEventSerial!==end?.inputEventSerial&&!eventsExplained){
+      const named=[...new Set(unexplained||[])],more=named.length-5,
+        unrecorded=end?.inputEventPathsComplete===false?' (not every event path was kept)':'';
+      captureIssues.push('declared input event occurred during execution'+(named.length?
+        `: ${named.slice(0,5).join(', ')}${more>0?` and ${more} more`:''}`:'')+unrecorded);
+    }
+    // A recorded certificate changes only the probe part of the fingerprint.
+    if(!outputs&&!hiddenOnly&&(!snap||!end||snap.planId!==end.planId||(recording?snap.inputKey!==end.inputKey:snap.fingerprint!==end.fingerprint)||
+      snap.revision!==end.revision))captureIssues.push('declared inputs changed or were not observed');
+    // A run that hid git-ignored files and failed with a refusal that names
+    // one runs again in the sandbox without hiding them: the check needs
+    // them. That run's result is kept, compared on every file, when the
+    // inputs held through it too. Its own refusal is handled as below.
+    let hiddenRerun=null;
+    if((sandbox?.hidden?.length||sandbox?.docs?.length)&&result==='FAIL'&&!requestedSignal){
+      const line=hiddenRefusal(output.head+'\n'+output.tail,[...sandbox.hidden||[],...sandbox.docs||[]]);
+      if(line){
+        console.error(`redue: the sandboxed run failed with what looks like a refusal of a hidden file (${line}); running it again in the sandbox without hiding files`);
+        const again=await runToEnd(sandboxArgv(sandbox.mode,command,sandbox.options),
+          {cwd:plan.cwd,env:sandbox.env,echo:true,started:child=>{activeChild=child;}});
+        activeChild=null;
+        hiddenRerun={line,exitCode:again.status,signal:again.signal||null,...again.error?{errorCode:again.error.code||again.error.name}:{}};
+        if(requestedSignal){
+          captureIssues.push('interrupted during the run without hidden files');
+          Object.assign(invocation,{status:'interrupted',exitCode:null,signal:requestedSignal});
+        }
+        else if(again.error||again.status===null)
+          captureIssues.push(`the run without hidden files did not finish (${again.error?.code||again.signal}), so a refusal of a file git ignores is not ruled out (${line})`);
+        else{
+          const wait=decisionWait(process.env.VSTATE_CAPTURE_TIMEOUT_MS);
+          // Inputs, plan, observer and environment held through it as through the first run.
+          let confirmed=null;const environmentNow=contextHashes()[name];
+          try{confirmed=await call({action:'snapshot',name,contextHashes:contextHashes(),
+            ...Number.isSafeInteger(snap?.inputEventSerial)?{sinceSerial:snap.inputEventSerial}:{}},wait.timeout,wait.deadline);}catch{}
+          const last=confirmed?.snapshots?.[name];
+          if(!last?.observationHealthy||!snap||last.planId!==snap.planId||last.fingerprint!==snap.fingerprint||last.revision!==snap.revision||
+            last.inputEventSerial!==end?.inputEventSerial||last.planGeneration!==snap.planGeneration||
+            confirmed.observation?.generation!==before?.observation?.generation)
+            captureIssues.push('declared inputs changed or were not observed during the run without hidden files');
+          if(JSON.stringify(environmentNow)!==JSON.stringify(environmentBefore))
+            captureIssues.push('declared environment changed during the run without hidden files');
+          invocation.hiddenExitCode=invocation.exitCode;invocation.exitCode=again.status;
+          result=again.status===0?'PASS':'FAIL';
+          output.head=`${again.stdout}\n${again.stderr}`.slice(0,65536);output.tail='';
+          sandbox.hidden=null;sandbox.docs=null;
+        }
+      }
+    }
+    // A sandboxed run that failed with what looks like a refusal by the
+    // sandbox runs again outside it. When that run exits differently, the
+    // sandbox changed the result: the plain run's result is kept, not reused.
+    // An interrupt before or during it stops there: the run is interrupted.
+    let sandboxRerun=null;
+    if(sandbox&&result==='FAIL'&&!requestedSignal){
+      const line=denialLine(output.head+'\n'+output.tail);
+      if(line){
+        console.error(`redue: the sandboxed run failed with what looks like a refusal by the sandbox (${line}); running it again outside the sandbox`);
+        // Interrupting REDUE interrupts the rerun's group too (forward).
+        const plain=await runToEnd(command,{cwd:plan.cwd,env:plainEnv,echo:true,started:child=>{activeChild=child;}});
+        activeChild=null;
+        sandboxRerun={line,exitCode:plain.status,signal:plain.signal||null,...plain.error?{errorCode:plain.error.code||plain.error.name}:{}};
+        if(requestedSignal){
+          captureIssues.push('interrupted during the run outside the sandbox');
+          Object.assign(invocation,{status:'interrupted',exitCode:null,signal:requestedSignal});
+        }
+        // A rerun that did not start or finish tells nothing: the sandboxed result stands, not reused.
+        else if(plain.error||plain.status===null)
+          captureIssues.push(`the run outside the sandbox did not finish (${plain.error?.code||plain.signal}), so a refusal by the sandbox is not ruled out (${line})`);
+        else if(plain.status!==invocation.exitCode){
+          captureIssues.push(`the sandbox changed the result: exit ${invocation.exitCode} inside it, ${plain.status??plain.signal} outside it (${line}); `+
+            'if this repeats, run redue init again to choose a recipe for this check');
+          invocation.sandboxedExitCode=invocation.exitCode;invocation.exitCode=plain.status;
+        }
+      }
+    }
+    // A test run whose network was not supervised throughout is not reused.
+    if(sandbox?.stats){const issue=supervisorIssue(sandbox.stats);if(issue)captureIssues.push(issue);fs.rmSync(sandbox.stats,{force:true});}
+    if(sandbox)fs.rmSync(sandbox.dirs.tmp,{recursive:true,force:true});
     const stable=captureIssues.length===0;
-    const receipt={result,invocation,target,stable,fingerprintVersion:1,
+    // Two runs on the same inputs, environment and plan that disagree make the
+    // check nondeterministic until its inputs change.
+    let nondeterministic=false;
+    const declared=hashes=>Object.entries(hashes||{}).filter(([key])=>!key.startsWith('__redue_env_'));
+    if(sandboxRerun)result=invocation.status==='exited'?(invocation.exitCode===0?'PASS':'FAIL'):null;
+    if((testRun||sandboxed)&&stable&&result){
+      const prior=readReceipts(state).checks[name];
+      // A difference explained by inputs REDUE did not observe proves nothing.
+      nondeterministic=Boolean(!guard?.issues.length&&prior?.stable&&!prior.guardIssues?.length&&prior.result&&prior.planId===(snap?.planId??plan.id)&&
+        prior.fingerprint===((recording||outputs?end:snap)?.fingerprint??null)&&
+        JSON.stringify(declared(prior.environmentHashes))===JSON.stringify(declared(environmentBefore))&&
+        JSON.stringify(prior.runFacts||[])===JSON.stringify([...runFacts].sort())&&
+        JSON.stringify(prior.environmentReads||{})===JSON.stringify(guard?.environmentReads||{})&&
+        (prior.result!==result||prior.nondeterministic===true));
+    }
+    // A recorded run's receipt carries its run facts, even none; observers that
+    // predate run facts do not reuse version 2.
+    // The recipe that made the receipt, which status and the hook name.
+    // A sandboxed receipt names what the run did: hid the git-ignored files or not.
+    const receipt={result,invocation,target,stable,fingerprintVersion:recorded||testRun?2:1,
+      recipe:(fallback?scriptQualification:own?(sandbox?.docs?docsQualification:sandbox?.hidden?hiddenQualification:sandboxQualification):selected.qualification)||null,
+      ...(recorded||testRun?{runFacts}:{}),
+      ...(testRun?{verificationRecipe:general?scriptInterpretation:testsInterpretation,guardIssues:guard?.issues||['the run record is unavailable'],
+        environmentReads:guard?.environmentReads||{},...(nondeterministic?{nondeterministic}:{})}:{}),
       resolutionLinks:plan.resolutionLinks||[],
       coverage:{qualified:plan.unresolved.length===0,unresolved:plan.unresolved},
       checkpoint:{status:stable?'stable':'unavailable_or_changed',issues:captureIssues,
@@ -277,14 +646,23 @@ async function execute() {
         ...(process.env.REDUE_PUBLIC_INPUT_TRACE==='1'?{
           startInputEventSerial:snap?.inputEventSerial??null,
           endInputEventSerial:end?.inputEventSerial??null}: {})},
-      planId:snap?.planId??plan.id,fingerprint:snap?.fingerprint??null,
+      planId:snap?.planId??plan.id,fingerprint:(recording||outputs?end:snap)?.fingerprint??null,
       provider:plan.provider,environmentHashes:environmentBefore,
-      files:snap?.files??null,observerGeneration:before?.observation?.generation??null,
+      ...(observedContext?{observedContext,verificationRecipe:directInterpretation}:{}),
+      // `hidden`: the git-ignored entries the run did not see, which the observer does not compare.
+      ...(own?{verificationRecipe:sandboxInterpretationOf(sandbox?.docs?docsQualification:sandbox?.hidden?hiddenQualification:sandboxQualification,selected.kind),
+        sandbox:{mode:sandbox?.mode??null,hidden:sandbox?.hidden??null,...sandbox?.docs?{docs:sandbox.docs}:{},...hiddenRerun?{hiddenRerun}:{},...sandboxRerun?{rerun:sandboxRerun}:{}}}:{}),
+      ...(fallback?{sandbox:{mode:null,fallback:'node-script',issue:support.issue}}:{}),
+      files:(outputs?end:snap)?.files??null,observerGeneration:before?.observation?.generation??null,
       startRevision:snap?.revision??null,endRevision:end?.revision??null};
     const persistenceStarted=performance.now();
     commitReceipt(state,name,receipt);
+    note(state,{kind:'run',check:name,ms:invocation.durationMs??null,result,stable});
+    if(testRun){const reads=environmentReads();reads[name]=receipt.environmentReads;atomicJson(readsFile,{schema:1,checks:reads});}
     timing('cli.receipt_persistence',persistenceStarted);
+    const reloadStarted=performance.now();
     try{await call({action:'reload'});}catch{/* receipt revisions invalidate older status even when reload IPC is unavailable */}
+    timing('cli.receipt_reload',reloadStarted);
     const summary={check:name,result,target:target.status,
       invocation:invocation.status,...(invocation.errorCode?{error_code:invocation.errorCode}:{}),
       stable,coverage_qualified:receipt.coverage.qualified};
@@ -367,16 +745,30 @@ async function main(){
       if(processAlive(pid)){await call({action:'stop'});
         for(let i=0;i<100&&fs.existsSync(path.join(state,'observer.lock'));i++)
           await new Promise(r=>setTimeout(r,100));
-        if(fs.existsSync(path.join(state,'observer.lock')))throw Error('observer did not stop');}
+        if(fs.existsSync(path.join(state,'observer.lock')))throw Error('observer did not stop');
+        // It releases the lock before it exits. On Windows a file it still
+        // holds open stays listed after deletion, and removing the
+        // directory fails with ENOTEMPTY until the process is gone.
+        for(let i=0;i<100&&processAlive(pid);i++)await new Promise(r=>setTimeout(r,100));}
     }
-    fs.rmSync(state,{recursive:true,force:true});
+    // A stopping observer can still be closing its files (Windows: ENOTEMPTY, EBUSY).
+    fs.rmSync(state,{recursive:true,force:true,maxRetries:10,retryDelay:200});
     console.log(JSON.stringify({removed:state}));return;
   }
   if(action==='status'){console.log(JSON.stringify(cached()));return;}
   if(action==='sync'||action==='detail'){
     const started=performance.now();
+    if(profiling&&config.checks.some(check=>check.qualification===directQualification)){
+      const proxyShapes=Object.entries(process.env).filter(([key])=>/^npm_config_.*proxy$/i.test(key)).map(([key,value])=>{
+        try{const url=new URL(value),portDigest=digest(url.port);url.port='';
+          return {key,kind:'url',portDigest,withoutPortDigest:digest(url.href)};}
+        catch{return {key,kind:value===''?'empty':'non-url',valueDigest:digest(value)};}
+      });
+      timing('cli.direct_context',started,{interpretation:directInterpretation,
+        observedNpmDigest:observedNpmContext(),proxyShapes});
+    }
     let value;try{value=await decisionCall({action:'sync',contextHashes:contextHashes()},
-      Number(process.env.VSTATE_SYNC_TIMEOUT_MS||decisionDeadlineMs));
+      decisionWait(process.env.VSTATE_SYNC_TIMEOUT_MS));
       // A client upgrade can encounter an older running observer. Its successful
       // control response does not establish that it evaluated the latest receipt.
       if(value.receipt_revision===null){
@@ -390,7 +782,9 @@ async function main(){
       }}
     catch(e){value=unavailableCached(state,e.message,config.checks);process.exitCode=2;}
     timing('cli.synchronized_status',started,{available:value.observation?.healthy?1:0});
-    if(action==='sync')console.log(JSON.stringify(value));
+    // In the front's process (bin/redue.mjs), the value is handed back.
+    if(action==='sync'&&process.argv.includes('--value'))syncValue=value;
+    else if(action==='sync')console.log(JSON.stringify(value));
     else for(const row of value.checks)console.log(`${row.name}: ${row.freshness}/${row.result||'NO RESULT'} — ${row.reason}`);
     return;
   }
@@ -398,4 +792,5 @@ async function main(){
   if(action==='run'){await execute();return;}
   throw Error(`unknown action ${action}`);
 }
-main().catch(e=>{console.error(`redue: ${e.message}`);process.exitCode=2;});
+export let syncValue=null;
+export const finished=main().catch(e=>{console.error(`redue: ${e.message}`);process.exitCode=2;});