@redue/cli 0.0.0-stage → 0.1.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +128 -0
- package/LICENSE +180 -0
- package/README.md +162 -2
- package/bin/redue.mjs +355 -0
- package/bin/vstate.mjs +3 -0
- package/docs/README.md +21 -0
- package/docs/user/agents.md +207 -0
- package/docs/user/configuration.md +60 -0
- package/docs/user/direct-typescript.md +96 -0
- package/docs/user/getting-started.md +148 -0
- package/docs/user/json.md +59 -0
- package/docs/user/npm-launcher-notice.md +42 -0
- package/docs/user/pilot.md +142 -0
- package/docs/user/receipt-selection-notice.md +60 -0
- package/docs/user/shell.md +25 -0
- package/docs/user/supported-projects.md +90 -0
- package/examples/redue.config.json +8 -0
- package/package.json +47 -4
- package/src/agent-integration.mjs +248 -0
- package/src/agent-policy.md +70 -0
- package/src/cached-state.mjs +42 -0
- package/src/cli.mjs +459 -0
- package/src/control-endpoint.mjs +11 -0
- package/src/daemon.mjs +1082 -0
- package/src/decision-profile.mjs +32 -0
- package/src/decision-validation.mjs +41 -0
- package/src/declared-plan.mjs +125 -0
- package/src/direct-run-certificate.mjs +40 -0
- package/src/direct-typescript-probe.mjs +56 -0
- package/src/direct-typescript.mjs +76 -0
- package/src/executable-lookup.mjs +18 -0
- package/src/glob.mjs +9 -0
- package/src/history-worker.mjs +23 -0
- package/src/index.mjs +306 -0
- package/src/installed-inputs.mjs +110 -0
- package/src/linux-inotify.py +228 -0
- package/src/linux-observer.mjs +79 -0
- package/src/node-onboarding.mjs +274 -0
- package/src/npm-typecheck-environment.mjs +36 -0
- package/src/npm-typecheck-probe.mjs +120 -0
- package/src/observation.mjs +39 -0
- package/src/owned-state.mjs +32 -0
- package/src/path-identity.mjs +43 -0
- package/src/plan-worker.mjs +140 -0
- package/src/plan.mjs +10 -0
- package/src/platform-observation.mjs +17 -0
- package/src/platform-state.mjs +14 -0
- package/src/pnpm-inputs.mjs +355 -0
- package/src/pnpm-typecheck-probe.mjs +47 -0
- package/src/presentation.mjs +53 -0
- package/src/probe-result.mjs +21 -0
- package/src/process-liveness.mjs +11 -0
- package/src/project-location.mjs +34 -0
- package/src/run-lock.mjs +77 -0
- package/src/state-store.mjs +73 -0
- package/src/toolchain-probe.mjs +21 -0
- package/src/typescript-contract-probe.mjs +5 -0
- package/src/typescript-input-facts.mjs +65 -0
- package/src/typescript-list.mjs +72 -0
- package/src/typescript-qualification.mjs +77 -0
- package/src/windows-command.mjs +16 -0
- package/src/windows-process.mjs +15 -0
- package/src/yarn-workspace-inputs.mjs +148 -0
- package/src/yarn-workspace-typecheck-probe.mjs +107 -0
package/src/index.mjs
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import {execFileSync} from 'node:child_process';
|
|
4
|
+
import {createHash} from 'node:crypto';
|
|
5
|
+
import micromatch from './glob.mjs';
|
|
6
|
+
import {sha} from './plan.mjs';
|
|
7
|
+
import {resolveLinks} from './installed-inputs.mjs';
|
|
8
|
+
import {validInputRelative} from './path-identity.mjs';
|
|
9
|
+
|
|
10
|
+
const tracked = root => [...new Set(execFileSync('git',
|
|
11
|
+
['ls-files','--cached','--others','--exclude-standard','-z'],
|
|
12
|
+
{cwd:root,env:{...process.env,GIT_OPTIONAL_LOCKS:'0'},maxBuffer:64*1024*1024})
|
|
13
|
+
.toString().split('\0').filter(Boolean))];
|
|
14
|
+
const xor = (left,right) => { for(let i=0;i<32;i++) left[i]^=right[i]; };
|
|
15
|
+
const token = (file,hash) => createHash('sha256').update(file+'\0'+hash).digest();
|
|
16
|
+
const stablePrefix = glob => {
|
|
17
|
+
const parts=glob.replace(/^!/, '').split('/'), fixed=[];
|
|
18
|
+
for(const p of parts) { if(/[?*[{(]/.test(p)) break; fixed.push(p); }
|
|
19
|
+
// An exact file is its own scan root. Scanning its parent could enumerate an
|
|
20
|
+
// entire ignored installation tree for one declared metadata file.
|
|
21
|
+
return fixed.join('/');
|
|
22
|
+
};
|
|
23
|
+
const groupMatchers = patterns => {
|
|
24
|
+
const groups=new Map();
|
|
25
|
+
for(const pattern of patterns) {
|
|
26
|
+
const prefix=stablePrefix(pattern);
|
|
27
|
+
if(!groups.has(prefix))groups.set(prefix,[]);
|
|
28
|
+
groups.get(prefix).push(micromatch.matcher(pattern,{dot:true}));
|
|
29
|
+
}
|
|
30
|
+
return groups;
|
|
31
|
+
};
|
|
32
|
+
const pathPrefixes = rel => {
|
|
33
|
+
const parts=rel.split('/'), prefixes=[''];
|
|
34
|
+
for(let i=1;i<=parts.length;i++)prefixes.push(parts.slice(0,i).join('/'));
|
|
35
|
+
return prefixes;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
function fileHash(root, rel,installed=false,before=fs.lstatSync(path.join(root,rel),
|
|
39
|
+
{bigint:true})) {
|
|
40
|
+
const file=path.join(root, rel);
|
|
41
|
+
let value,references=[],bytes=Number(before.size);
|
|
42
|
+
if(before.isSymbolicLink()){
|
|
43
|
+
const resolution=resolveLinks(file,root,[],true);
|
|
44
|
+
if(!resolution.physical)throw Error(`symlink input target observation unavailable: ${rel}: ${resolution.reason}`);
|
|
45
|
+
const target=fs.statSync(resolution.physical,{bigint:true});
|
|
46
|
+
if(!target.isFile())throw Error(`directory input symlink unsupported: ${rel}`);
|
|
47
|
+
value='link:'+JSON.stringify(resolution.links)+':'+target.mode+':'+
|
|
48
|
+
(installed&&target.nlink>1?target.nlink+':':'')+
|
|
49
|
+
hashRegular(resolution.physical).hash;
|
|
50
|
+
references=[...new Set([...resolution.links.map(link=>path.relative(root,link.path)
|
|
51
|
+
.split(path.sep).join('/')),path.relative(root,resolution.physical)
|
|
52
|
+
.split(path.sep).join('/')])];
|
|
53
|
+
bytes=Number(target.size);
|
|
54
|
+
const targetAfter=fs.statSync(resolution.physical,{bigint:true});
|
|
55
|
+
for(const key of ['dev','ino','mode','nlink','size','mtimeNs','ctimeNs'])
|
|
56
|
+
if(target[key]!==targetAfter[key])throw Error(`input changed during inspection: ${rel}`);
|
|
57
|
+
const resolutionAfter=resolveLinks(file,root,[],true);
|
|
58
|
+
if(resolutionAfter.physical!==resolution.physical||
|
|
59
|
+
JSON.stringify(resolutionAfter.links)!==JSON.stringify(resolution.links))
|
|
60
|
+
throw Error(`input link changed during inspection: ${rel}`);
|
|
61
|
+
}
|
|
62
|
+
else if(before.isFile()) value='file:'+before.mode+':'+
|
|
63
|
+
(installed&&before.nlink>1?before.nlink+':':'')+hashRegular(file).hash;
|
|
64
|
+
else throw Error(`non-file input: ${rel}`);
|
|
65
|
+
const after=fs.lstatSync(file,{bigint:true});
|
|
66
|
+
for(const key of ['dev','ino','mode','nlink','size','mtimeNs','ctimeNs'])
|
|
67
|
+
if(before[key]!==after[key]) throw Error(`input changed during inspection: ${rel}`);
|
|
68
|
+
return {hash:sha(value),references,bytes};
|
|
69
|
+
}
|
|
70
|
+
const readBuffer=Buffer.allocUnsafe(65536);
|
|
71
|
+
function hashRegular(file) {
|
|
72
|
+
const h=createHash('sha256'), fd=fs.openSync(file,'r');
|
|
73
|
+
try { let n; while((n=fs.readSync(fd,readBuffer,0,readBuffer.length,null))>0)
|
|
74
|
+
h.update(readBuffer.subarray(0,n)); }
|
|
75
|
+
finally { fs.closeSync(fd); }
|
|
76
|
+
return {hash:h.digest('hex')};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export class InputIndex {
|
|
80
|
+
constructor(bundle, options={}) {
|
|
81
|
+
const matcherStarted=performance.now();
|
|
82
|
+
this.bundle=bundle; this.root=bundle.root; this.files=new Map(); this.members=new Map();
|
|
83
|
+
this.symlinkRefs=new Map();this.aliasDependencies=new Map();
|
|
84
|
+
this.trackGit=options.trackGit!==false;
|
|
85
|
+
this.installedRoots=[...new Set(Object.values(bundle.plans).flatMap(plan=>
|
|
86
|
+
plan.installedPhysicalRoots||[]))];
|
|
87
|
+
this.installedRootSet=new Set(this.installedRoots);
|
|
88
|
+
this.installedAncestors=new Set(this.installedRoots.flatMap(pathPrefixes));
|
|
89
|
+
this.includeInstalled=options.includeInstalled||false;
|
|
90
|
+
this.checks=new Map(); this.prefixes=new Map(); this.eventCount=0; this.rehashedFiles=0;
|
|
91
|
+
this.rehashedBytes=0; this.lastChange=0; this.unavailable=null;
|
|
92
|
+
this.profile={matcherMs:0,trackedMs:0,enumerationMs:0,matchingMs:0,
|
|
93
|
+
hashingMs:0,recordMs:0,enumeratedEntries:0};
|
|
94
|
+
for(const [name,plan] of Object.entries(bundle.plans)) {
|
|
95
|
+
const source=plan.sourcePatterns||plan.patterns;
|
|
96
|
+
const positive=source.filter(p=>!p.startsWith('!'));
|
|
97
|
+
const negative=source.filter(p=>p.startsWith('!')).map(p=>p.slice(1));
|
|
98
|
+
const additivePositive=(plan.additivePatterns||[]).filter(p=>!p.startsWith('!'));
|
|
99
|
+
const additiveNegative=(plan.additivePatterns||[]).filter(p=>p.startsWith('!'))
|
|
100
|
+
.map(p=>p.slice(1));
|
|
101
|
+
const row={plan,positive:groupMatchers(positive),negative:groupMatchers(negative),
|
|
102
|
+
additivePositive:groupMatchers(additivePositive),
|
|
103
|
+
additiveNegative:groupMatchers(additiveNegative),
|
|
104
|
+
files:new Set(),sum:Buffer.alloc(32),revision:0,lastReason:null,probeHash:null,probeAt:0};
|
|
105
|
+
this.checks.set(name,row);
|
|
106
|
+
for(const p of [...positive,...additivePositive]) {
|
|
107
|
+
const prefix=stablePrefix(p);
|
|
108
|
+
if(!this.prefixes.has(prefix)) this.prefixes.set(prefix,new Set());
|
|
109
|
+
this.prefixes.get(prefix).add(name);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
this.profile.matcherMs=performance.now()-matcherStarted;
|
|
113
|
+
}
|
|
114
|
+
candidates(rel) {
|
|
115
|
+
const names=new Set(this.prefixes.get('')||[]);
|
|
116
|
+
const parts=rel.split('/');
|
|
117
|
+
for(let i=1;i<=parts.length;i++)
|
|
118
|
+
for(const name of this.prefixes.get(parts.slice(0,i).join('/'))||[]) names.add(name);
|
|
119
|
+
return names;
|
|
120
|
+
}
|
|
121
|
+
matches(row,rel) { const started=performance.now(), prefixes=pathPrefixes(rel);
|
|
122
|
+
const any=(groups)=>prefixes.some(prefix=>(groups.get(prefix)||[]).some(fn=>fn(rel)));
|
|
123
|
+
const result=(any(row.positive) && !any(row.negative))||
|
|
124
|
+
(any(row.additivePositive)&&!any(row.additiveNegative));
|
|
125
|
+
this.profile.matchingMs+=performance.now()-started;return result; }
|
|
126
|
+
installedPath(rel) {
|
|
127
|
+
if(this.includeInstalled||this.installedAncestors.has(rel))return true;
|
|
128
|
+
for(let slash=rel.length;slash>0;slash=rel.lastIndexOf('/',slash-1)){
|
|
129
|
+
if(this.installedRootSet.has(rel.slice(0,slash)))return true;
|
|
130
|
+
if(slash===0)break;
|
|
131
|
+
}
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
fingerprint(name) {
|
|
135
|
+
const row=this.checks.get(name);
|
|
136
|
+
return sha(row.plan.id+'\0'+row.files.size+'\0'+row.sum.toString('hex')+'\0'+(row.probeHash||''));
|
|
137
|
+
}
|
|
138
|
+
async recheckInstalled(names) {
|
|
139
|
+
const selected=new Set(names),started=performance.now();
|
|
140
|
+
let files=0,bytes=0;
|
|
141
|
+
for(const rel of [...this.files.keys()]){
|
|
142
|
+
if(!this.installedPath(rel)||![...this.members.get(rel)||[]]
|
|
143
|
+
.some(name=>selected.has(name)))continue;
|
|
144
|
+
const before=this.rehashedBytes;
|
|
145
|
+
this.record(rel,rel+' changed');
|
|
146
|
+
if(this.unavailable)throw Error(this.unavailable);
|
|
147
|
+
files++;bytes+=this.rehashedBytes-before;
|
|
148
|
+
if(files%32===0)await new Promise(resolve=>setImmediate(resolve));
|
|
149
|
+
}
|
|
150
|
+
return {files,bytes,durationMs:performance.now()-started};
|
|
151
|
+
}
|
|
152
|
+
summary(name) { const row=this.checks.get(name); return {fingerprint:this.fingerprint(name),
|
|
153
|
+
revision:row.revision,planId:row.plan.id,files:row.files.size,probeAt:row.probeAt}; }
|
|
154
|
+
record(rel, changeReason=rel+' changed', observedStat=null) {
|
|
155
|
+
if(!validInputRelative(rel)) {
|
|
156
|
+
this.unavailable='invalid event path'; return false;
|
|
157
|
+
}
|
|
158
|
+
const names=this.candidates(rel), old=this.files.get(rel), matched=[];
|
|
159
|
+
for(const name of names) if(this.matches(this.checks.get(name),rel)) matched.push(name);
|
|
160
|
+
if(!matched.length && old===undefined) return false;
|
|
161
|
+
let fresh,stat,inspection;
|
|
162
|
+
try{stat=observedStat||fs.lstatSync(path.join(this.root,rel),{bigint:true});}
|
|
163
|
+
catch(e){if(e.code!=='ENOENT'){this.unavailable=String(e);return false;}}
|
|
164
|
+
if(stat&&(stat.isFile()||stat.isSymbolicLink())) {
|
|
165
|
+
const installed=this.installedPath(rel);
|
|
166
|
+
try{const started=performance.now();inspection=fileHash(this.root,rel,installed,stat);
|
|
167
|
+
fresh=inspection.hash;
|
|
168
|
+
this.profile.hashingMs+=performance.now()-started;
|
|
169
|
+
this.rehashedFiles++;this.rehashedBytes+=inspection.bytes;}
|
|
170
|
+
catch(e){this.unavailable=String(e);return false;}
|
|
171
|
+
} else if(stat){this.unavailable=`non-file input: ${rel}`;return false;}
|
|
172
|
+
for(const reference of this.aliasDependencies.get(rel)||[]){
|
|
173
|
+
const aliases=this.symlinkRefs.get(reference);
|
|
174
|
+
aliases?.delete(rel);if(aliases?.size===0)this.symlinkRefs.delete(reference);
|
|
175
|
+
}
|
|
176
|
+
this.aliasDependencies.delete(rel);
|
|
177
|
+
if(inspection?.references.length){
|
|
178
|
+
this.aliasDependencies.set(rel,new Set(inspection.references));
|
|
179
|
+
for(const reference of inspection.references){
|
|
180
|
+
if(!this.symlinkRefs.has(reference))this.symlinkRefs.set(reference,new Set());
|
|
181
|
+
this.symlinkRefs.get(reference).add(rel);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
const oldNames=this.members.get(rel)||new Set(), newNames=new Set(fresh===undefined?[]:matched);
|
|
185
|
+
if(old===fresh && oldNames.size===newNames.size && [...oldNames].every(n=>newNames.has(n))) return false;
|
|
186
|
+
for(const name of new Set([...oldNames,...newNames])) {
|
|
187
|
+
const row=this.checks.get(name);
|
|
188
|
+
if(oldNames.has(name) && old!==undefined) {xor(row.sum,token(rel,old));row.files.delete(rel);}
|
|
189
|
+
if(newNames.has(name) && fresh!==undefined) {xor(row.sum,token(rel,fresh));row.files.add(rel);}
|
|
190
|
+
row.revision++;row.lastReason=changeReason;
|
|
191
|
+
}
|
|
192
|
+
if(fresh!==undefined && newNames.size) {this.files.set(rel,fresh);this.members.set(rel,newNames);}
|
|
193
|
+
else {this.files.delete(rel);this.members.delete(rel);}
|
|
194
|
+
if(old!==fresh || oldNames.size!==newNames.size) {
|
|
195
|
+
this.eventCount++;this.lastChange=Date.now();return true;
|
|
196
|
+
}
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
scanSubtree(rel) {
|
|
200
|
+
const found=new Set(), roots=new Set();
|
|
201
|
+
// Directory notifications can name only an ancestor of an exact input.
|
|
202
|
+
// Revisit matching declared roots without traversing unrelated siblings.
|
|
203
|
+
if([...this.prefixes.keys()].some(prefix=>!prefix||prefix===rel||rel.startsWith(prefix+'/')))
|
|
204
|
+
roots.add(rel);
|
|
205
|
+
for(const prefix of this.prefixes.keys())
|
|
206
|
+
if(!rel||prefix.startsWith(rel+'/'))roots.add(prefix);
|
|
207
|
+
const visit=(file,r)=>{
|
|
208
|
+
let st;try{st=fs.lstatSync(file);}catch(e){if(e.code==='ENOENT')return;throw e;}
|
|
209
|
+
if(st.isDirectory()) {
|
|
210
|
+
if(r && (path.posix.basename(r)==='.git'||
|
|
211
|
+
(path.posix.basename(r)==='node_modules'&&!this.installedPath(r)))&&
|
|
212
|
+
!this.prefixes.has(r)) return;
|
|
213
|
+
for(const entry of fs.readdirSync(file,{withFileTypes:true}))
|
|
214
|
+
visit(path.join(file,entry.name),r?r+'/'+entry.name:entry.name);
|
|
215
|
+
} else if(!found.has(r)) {found.add(r);this.record(r);}
|
|
216
|
+
};
|
|
217
|
+
for(const root of roots) {
|
|
218
|
+
const base=path.join(this.root,root);
|
|
219
|
+
let present=true;
|
|
220
|
+
try{fs.lstatSync(base);}catch(e){if(e.code==='ENOENT')present=false;else throw e;}
|
|
221
|
+
if(present)visit(base,root);
|
|
222
|
+
}
|
|
223
|
+
for(const old of [...this.files.keys()]) if((old===rel||old.startsWith(rel+'/'))&&!found.has(old)) this.record(old);
|
|
224
|
+
}
|
|
225
|
+
coldScan(progress) {
|
|
226
|
+
this.unavailable=null;
|
|
227
|
+
const trackedStarted=performance.now();
|
|
228
|
+
const names=new Set(this.trackGit?tracked(this.root):[]);
|
|
229
|
+
this.profile.trackedMs+=performance.now()-trackedStarted;
|
|
230
|
+
const enumerationStarted=performance.now();
|
|
231
|
+
// Traverse declared input roots to include ignored/generated inputs. Avoid unrelated
|
|
232
|
+
// node_modules/.git trees unless a pattern explicitly names them.
|
|
233
|
+
const roots=[...this.prefixes.keys()].filter(prefix=>prefix||!this.trackGit)
|
|
234
|
+
.sort((a,b)=>a.length-b.length);
|
|
235
|
+
const selected=[], selectedSet=new Set();
|
|
236
|
+
for(const prefix of roots) {
|
|
237
|
+
const parts=prefix.split('/');
|
|
238
|
+
if(parts.some((_,i)=>selectedSet.has(parts.slice(0,i+1).join('/'))))continue;
|
|
239
|
+
selected.push(prefix);selectedSet.add(prefix);
|
|
240
|
+
}
|
|
241
|
+
let scanned=0,processingMs=0;
|
|
242
|
+
for(const prefix of selected) {
|
|
243
|
+
const base=path.join(this.root,prefix);
|
|
244
|
+
const visit=(file,rel)=>{
|
|
245
|
+
this.profile.enumeratedEntries++;
|
|
246
|
+
let st;try{st=fs.lstatSync(file,{bigint:true});}
|
|
247
|
+
catch(e){if(e.code==='ENOENT')return;throw e;}
|
|
248
|
+
if(st.isDirectory()) {
|
|
249
|
+
if((path.posix.basename(rel)==='.git'||
|
|
250
|
+
(path.posix.basename(rel)==='node_modules'&&!this.installedPath(rel))) &&
|
|
251
|
+
!this.prefixes.has(rel))return;
|
|
252
|
+
for(const ent of fs.readdirSync(file,{withFileTypes:true}))
|
|
253
|
+
visit(path.join(file,ent.name),rel?rel+'/'+ent.name:ent.name);
|
|
254
|
+
} else {
|
|
255
|
+
names.delete(rel);
|
|
256
|
+
const started=performance.now();
|
|
257
|
+
if(this.candidates(rel).size)this.record(rel,'cold reconciliation',st);
|
|
258
|
+
processingMs+=performance.now()-started;
|
|
259
|
+
scanned++;
|
|
260
|
+
if(progress&&scanned%256===0)progress({scanned,total:null});
|
|
261
|
+
}
|
|
262
|
+
};
|
|
263
|
+
let st;try{st=fs.lstatSync(base,{bigint:true});}
|
|
264
|
+
catch(e){if(e.code==='ENOENT')continue;throw e;}
|
|
265
|
+
if(st.isSymbolicLink()) {
|
|
266
|
+
try{if(fs.statSync(base).isDirectory()){
|
|
267
|
+
this.unavailable=`glob rooted at directory symlink unsupported: ${prefix}`;continue;}}
|
|
268
|
+
catch(e){if(e.code!=='ENOENT')throw e;}
|
|
269
|
+
}
|
|
270
|
+
visit(base,prefix);
|
|
271
|
+
}
|
|
272
|
+
this.profile.enumerationMs+=performance.now()-enumerationStarted-processingMs;
|
|
273
|
+
const recordStarted=performance.now();
|
|
274
|
+
for(const rel of names) {
|
|
275
|
+
if(this.candidates(rel).size)this.record(rel,'cold reconciliation');
|
|
276
|
+
scanned++;
|
|
277
|
+
if(progress&&scanned%256===0)progress({scanned,total:null});
|
|
278
|
+
}
|
|
279
|
+
if(progress)progress({scanned,total:scanned});
|
|
280
|
+
this.profile.recordMs+=processingMs+performance.now()-recordStarted;
|
|
281
|
+
}
|
|
282
|
+
updatePath(rel) {
|
|
283
|
+
const full=path.join(this.root,rel);
|
|
284
|
+
let subtree=false;
|
|
285
|
+
try {
|
|
286
|
+
if(fs.lstatSync(full).isSymbolicLink()&&fs.statSync(full).isDirectory()){
|
|
287
|
+
this.unavailable=`directory symlink input unsupported: ${rel}`;return;}
|
|
288
|
+
if(fs.statSync(full).isDirectory()) {this.scanSubtree(rel);subtree=true;}
|
|
289
|
+
else this.record(rel);
|
|
290
|
+
} catch(e) {
|
|
291
|
+
if(e.code==='ENOENT') {
|
|
292
|
+
if([...this.files.keys()].some(p=>p.startsWith(rel+'/'))) {
|
|
293
|
+
this.scanSubtree(rel);subtree=true;}
|
|
294
|
+
else this.record(rel);
|
|
295
|
+
} else {this.unavailable=String(e);}
|
|
296
|
+
}
|
|
297
|
+
const aliases=new Set(this.symlinkRefs.get(rel)||[]);
|
|
298
|
+
if(subtree)for(const [reference,names] of this.symlinkRefs)
|
|
299
|
+
if(reference.startsWith(rel+'/'))for(const name of names)aliases.add(name);
|
|
300
|
+
for(const alias of aliases)if(alias!==rel)this.record(alias,`${rel} changed`);
|
|
301
|
+
}
|
|
302
|
+
exportFiles(name) { const row=this.checks.get(name), out={};
|
|
303
|
+
for(const file of row.files) out[file]=this.files.get(file);
|
|
304
|
+
return out;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// Declared installed-input paths, link resolution, and permitted observation roots.
|
|
2
|
+
// Physical contents and membership are observed separately by InputIndex.
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import os from 'node:os';
|
|
6
|
+
import {createHash} from 'node:crypto';
|
|
7
|
+
import {normalizedPath,samePath,withinPath as within,realObservedPath} from './path-identity.mjs';
|
|
8
|
+
|
|
9
|
+
const hash=value=>createHash('sha256').update(value).digest('hex');
|
|
10
|
+
const posix=file=>file.split(path.sep).join('/');
|
|
11
|
+
export function resolveLinks(file,checkout,allowedRoots=[],allowFile=false) {
|
|
12
|
+
const roots=[checkout,...allowedRoots],links=[],seen=new Set();
|
|
13
|
+
let candidate=normalizedPath(file);
|
|
14
|
+
for(let depth=0;depth<40;depth++){
|
|
15
|
+
let current=path.parse(candidate).root,redirected=false;
|
|
16
|
+
const parts=candidate.slice(current.length).split(path.sep).filter(Boolean);
|
|
17
|
+
for(let i=0;i<parts.length;i++){
|
|
18
|
+
current=path.join(current,parts[i]);
|
|
19
|
+
let st;try{st=fs.lstatSync(current);}catch(e){
|
|
20
|
+
return {physical:null,links,prospective:candidate,
|
|
21
|
+
reason:e.code==='ENOENT'?'resolved target missing':
|
|
22
|
+
`resolution unavailable: ${e.code||e.name}`};}
|
|
23
|
+
if(!st.isSymbolicLink()){
|
|
24
|
+
// NTFS may spell an existing component as RUNNER~1 or its long name.
|
|
25
|
+
// Normalize that component without skipping a later reparse point.
|
|
26
|
+
if(process.platform==='win32')current=fs.realpathSync.native(current);
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
if(!roots.some(root=>within(current,root)))
|
|
30
|
+
return {physical:null,links,reason:'link redirect outside observed roots'};
|
|
31
|
+
if([...seen].some(previous=>samePath(previous,current)))
|
|
32
|
+
return {physical:null,links,reason:'symlink cycle'};
|
|
33
|
+
seen.add(current);
|
|
34
|
+
const target=fs.readlinkSync(current),signature=hash(target);
|
|
35
|
+
links.push({path:current,targetHash:signature});
|
|
36
|
+
candidate=normalizedPath(path.resolve(path.dirname(current),target,...parts.slice(i+1)));
|
|
37
|
+
redirected=true;break;
|
|
38
|
+
}
|
|
39
|
+
if(redirected)continue;
|
|
40
|
+
candidate=current;
|
|
41
|
+
if(!roots.some(root=>within(candidate,root)))
|
|
42
|
+
return {physical:null,links,reason:'resolved target outside permitted roots'};
|
|
43
|
+
let st;try{st=fs.statSync(candidate);}catch(e){
|
|
44
|
+
return {physical:null,links,reason:e.code==='ENOENT'?'resolved target missing':
|
|
45
|
+
`resolution unavailable: ${e.code||e.name}`};}
|
|
46
|
+
if(!st.isDirectory()&&!(allowFile&&st.isFile()))
|
|
47
|
+
return {physical:null,links,reason:'resolved input has unsupported type'};
|
|
48
|
+
return {physical:candidate,links,reason:null};
|
|
49
|
+
}
|
|
50
|
+
return {physical:null,links,reason:'symlink resolution depth exceeded'};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function permittedRoots(root,values=[]) {
|
|
54
|
+
const allowed=[];
|
|
55
|
+
for(const value of values) {
|
|
56
|
+
if(typeof value!=='string'||!path.isAbsolute(value))
|
|
57
|
+
throw Error('external observation root must be absolute');
|
|
58
|
+
const real=realObservedPath(value);
|
|
59
|
+
if([path.parse(real).root,os.homedir()].includes(real)||within(root,real))
|
|
60
|
+
throw Error('external observation root is too broad');
|
|
61
|
+
if(!within(real,root))allowed.push(real);
|
|
62
|
+
}
|
|
63
|
+
return [...new Set(allowed)];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function resolveDeclaredInstalled(root,patterns,allowed=[]) {
|
|
67
|
+
const result={internalPatterns:[],externalPatterns:{},links:[],triggers:[],
|
|
68
|
+
unresolved:[],mappings:[],physicalRoots:[]};
|
|
69
|
+
const internal=new Set(),external=new Map(),links=new Map(),triggers=new Set(),roots=new Set();
|
|
70
|
+
for(const pattern of patterns||[]) {
|
|
71
|
+
const negative=pattern.startsWith('!'),raw=negative?pattern.slice(1):pattern;
|
|
72
|
+
// Exclusions must also apply at the logical link location. The mapped
|
|
73
|
+
// physical exclusion alone cannot stop a broad installed-tree scan from
|
|
74
|
+
// trying to hash a workspace directory symlink as a file.
|
|
75
|
+
if(negative)internal.add(pattern);
|
|
76
|
+
const parts=raw.split('/'),fixed=[];
|
|
77
|
+
for(const part of parts){if(/[?*[{(]/.test(part))break;fixed.push(part);}
|
|
78
|
+
if(!fixed.length){internal.add(pattern);continue;}
|
|
79
|
+
const prefix=fixed.join('/'),suffix=parts.slice(fixed.length).join('/'),
|
|
80
|
+
logical=path.join(root,prefix);
|
|
81
|
+
const resolved=resolveLinks(logical,root,allowed,!suffix);
|
|
82
|
+
for(const link of resolved.links){links.set(link.path,link);
|
|
83
|
+
if(within(link.path,root))triggers.add(posix(path.relative(root,link.path)));}
|
|
84
|
+
if(!resolved.physical){
|
|
85
|
+
if(resolved.reason==='resolved target missing'&&!resolved.links.length){
|
|
86
|
+
internal.add(pattern);continue;}
|
|
87
|
+
result.unresolved.push(`${prefix}: ${resolved.reason}`);continue;
|
|
88
|
+
}
|
|
89
|
+
const boundary=within(resolved.physical,root)?root:
|
|
90
|
+
allowed.find(allowedRoot=>within(resolved.physical,allowedRoot));
|
|
91
|
+
if(!boundary){result.unresolved.push(`${prefix}: resolved target outside permitted roots`);continue;}
|
|
92
|
+
if(boundary!==root&&resolved.physical===boundary){
|
|
93
|
+
result.unresolved.push(`${prefix}: external target needs a stable permitted parent root`);continue;}
|
|
94
|
+
const rel=posix(path.relative(boundary,resolved.physical));
|
|
95
|
+
const directory=fs.statSync(resolved.physical).isDirectory();
|
|
96
|
+
const mapped=(negative?'!':'')+(suffix?(rel?rel+'/':'')+suffix:
|
|
97
|
+
directory?(rel?rel+'/**':'**/*'):rel);
|
|
98
|
+
if(boundary===root)internal.add(mapped);
|
|
99
|
+
else {if(!external.has(boundary))external.set(boundary,new Set());
|
|
100
|
+
external.get(boundary).add(mapped);}
|
|
101
|
+
if(!negative)roots.add(resolved.physical);
|
|
102
|
+
result.mappings.push({logical:pattern,physical:resolved.physical,root:boundary,
|
|
103
|
+
links:resolved.links.map(link=>link.path)});
|
|
104
|
+
}
|
|
105
|
+
result.internalPatterns=[...internal].sort();
|
|
106
|
+
result.externalPatterns=Object.fromEntries([...external].map(([key,values])=>[key,[...values].sort()]));
|
|
107
|
+
result.links=[...links.values()].sort((a,b)=>a.path.localeCompare(b.path));
|
|
108
|
+
result.triggers=[...triggers].sort();result.physicalRoots=[...roots].sort();
|
|
109
|
+
return result;
|
|
110
|
+
}
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Linux-only inotify transport. Emits paths and continuity faults, never file data.
|
|
3
|
+
|
|
4
|
+
Unlike the pinned Parcel inotify backend, IN_Q_OVERFLOW is reported. The Node
|
|
5
|
+
engine owns all content inspection and deterministic reconciliation.
|
|
6
|
+
"""
|
|
7
|
+
import ctypes
|
|
8
|
+
import errno
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import select
|
|
13
|
+
import struct
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
IN_MODIFY = 0x00000002
|
|
18
|
+
IN_ATTRIB = 0x00000004
|
|
19
|
+
IN_CLOSE_WRITE = 0x00000008
|
|
20
|
+
IN_MOVED_FROM = 0x00000040
|
|
21
|
+
IN_MOVED_TO = 0x00000080
|
|
22
|
+
IN_CREATE = 0x00000100
|
|
23
|
+
IN_DELETE = 0x00000200
|
|
24
|
+
IN_DELETE_SELF = 0x00000400
|
|
25
|
+
IN_MOVE_SELF = 0x00000800
|
|
26
|
+
IN_UNMOUNT = 0x00002000
|
|
27
|
+
IN_Q_OVERFLOW = 0x00004000
|
|
28
|
+
IN_IGNORED = 0x00008000
|
|
29
|
+
IN_ISDIR = 0x40000000
|
|
30
|
+
FILE_MASK = IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | IN_DELETE_SELF | IN_MOVE_SELF
|
|
31
|
+
MASK = (IN_MODIFY | IN_ATTRIB | IN_CLOSE_WRITE | IN_MOVED_FROM |
|
|
32
|
+
IN_MOVED_TO | IN_CREATE | IN_DELETE | IN_DELETE_SELF |
|
|
33
|
+
IN_MOVE_SELF | IN_UNMOUNT)
|
|
34
|
+
HEADER = struct.Struct('iIII')
|
|
35
|
+
LOCAL_FILESYSTEMS = {'ext2', 'ext3', 'ext4', 'xfs', 'btrfs', 'f2fs',
|
|
36
|
+
'tmpfs', 'zfs'}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def emit(kind, **fields):
|
|
40
|
+
print(json.dumps({'kind': kind, **fields}, ensure_ascii=True), flush=True)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def filesystem_type(folder):
|
|
44
|
+
best = ('', None)
|
|
45
|
+
with open('/proc/self/mountinfo', encoding='utf-8') as mounts:
|
|
46
|
+
for line in mounts:
|
|
47
|
+
before, after = line.rstrip('\n').split(' - ', 1)
|
|
48
|
+
point = re.sub(r'\\([0-7]{3})', lambda m: chr(int(m[1], 8)),
|
|
49
|
+
before.split()[4])
|
|
50
|
+
if (folder == point or folder.startswith(point.rstrip('/') + '/')) \
|
|
51
|
+
and len(point) > len(best[0]):
|
|
52
|
+
best = (point, after.split()[0])
|
|
53
|
+
return best[1]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def main(root):
|
|
57
|
+
root = os.path.realpath(root)
|
|
58
|
+
if filesystem_type(root) not in LOCAL_FILESYSTEMS:
|
|
59
|
+
emit('gap', reason='inotify_filesystem_untrusted')
|
|
60
|
+
return 2
|
|
61
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
62
|
+
libc.inotify_init1.argtypes = [ctypes.c_int]
|
|
63
|
+
libc.inotify_init1.restype = ctypes.c_int
|
|
64
|
+
libc.inotify_add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32]
|
|
65
|
+
libc.inotify_add_watch.restype = ctypes.c_int
|
|
66
|
+
libc.inotify_rm_watch.argtypes = [ctypes.c_int, ctypes.c_int]
|
|
67
|
+
libc.inotify_rm_watch.restype = ctypes.c_int
|
|
68
|
+
fd = libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC)
|
|
69
|
+
if fd < 0:
|
|
70
|
+
raise OSError(ctypes.get_errno(), 'inotify_init1 failed')
|
|
71
|
+
watches = {}
|
|
72
|
+
file_watches = {}
|
|
73
|
+
file_paths = {}
|
|
74
|
+
intentional_removals = set()
|
|
75
|
+
|
|
76
|
+
def add_tree(folder):
|
|
77
|
+
stack = [folder]
|
|
78
|
+
while stack:
|
|
79
|
+
current = stack.pop()
|
|
80
|
+
wd = libc.inotify_add_watch(fd, os.fsencode(current), MASK)
|
|
81
|
+
if wd < 0:
|
|
82
|
+
raise OSError(ctypes.get_errno(), 'inotify_add_watch failed')
|
|
83
|
+
watches[wd] = current
|
|
84
|
+
with os.scandir(current) as entries:
|
|
85
|
+
for entry in entries:
|
|
86
|
+
if entry.is_dir(follow_symlinks=False):
|
|
87
|
+
stack.append(entry.path)
|
|
88
|
+
|
|
89
|
+
def drain():
|
|
90
|
+
while True:
|
|
91
|
+
try:
|
|
92
|
+
block = os.read(fd, 65536)
|
|
93
|
+
except BlockingIOError:
|
|
94
|
+
return True
|
|
95
|
+
if not block:
|
|
96
|
+
emit('gap', reason='inotify_closed')
|
|
97
|
+
return False
|
|
98
|
+
offset = 0
|
|
99
|
+
while offset < len(block):
|
|
100
|
+
wd, mask, _, length = HEADER.unpack_from(block, offset)
|
|
101
|
+
offset += HEADER.size
|
|
102
|
+
raw = block[offset:offset + length].split(b'\0', 1)[0]
|
|
103
|
+
offset += length
|
|
104
|
+
if mask & IN_Q_OVERFLOW:
|
|
105
|
+
emit('gap', reason='inotify_overflow')
|
|
106
|
+
return False
|
|
107
|
+
if wd in intentional_removals and mask & IN_IGNORED:
|
|
108
|
+
intentional_removals.remove(wd)
|
|
109
|
+
continue
|
|
110
|
+
if wd in file_watches:
|
|
111
|
+
if mask & (IN_UNMOUNT | IN_DELETE_SELF | IN_MOVE_SELF | IN_IGNORED):
|
|
112
|
+
emit('gap', reason='inotify_file_watch_lost')
|
|
113
|
+
return False
|
|
114
|
+
for watched in file_watches[wd]:
|
|
115
|
+
emit('event', type='update', path=watched)
|
|
116
|
+
continue
|
|
117
|
+
parent = watches.get(wd)
|
|
118
|
+
if parent is None:
|
|
119
|
+
emit('gap', reason='inotify_watch_unknown')
|
|
120
|
+
return False
|
|
121
|
+
target = os.path.join(parent, os.fsdecode(raw)) if raw else parent
|
|
122
|
+
if mask & (IN_UNMOUNT | IN_DELETE_SELF | IN_MOVE_SELF | IN_IGNORED):
|
|
123
|
+
emit('gap', reason='inotify_watch_lost')
|
|
124
|
+
return False
|
|
125
|
+
if (mask & IN_ISDIR) and (mask & (IN_CREATE | IN_MOVED_TO)):
|
|
126
|
+
try:
|
|
127
|
+
add_tree(target)
|
|
128
|
+
except OSError as error:
|
|
129
|
+
emit('gap', reason='inotify_watch_add_failed', errno=error.errno)
|
|
130
|
+
return False
|
|
131
|
+
# Contents can appear before their new directory receives
|
|
132
|
+
# a watch. The engine reconciles instead of guessing.
|
|
133
|
+
emit('gap', reason='inotify_new_directory')
|
|
134
|
+
return False
|
|
135
|
+
if (mask & IN_ISDIR) and (mask & (IN_DELETE | IN_MOVED_FROM)):
|
|
136
|
+
emit('gap', reason='inotify_directory_removed')
|
|
137
|
+
return False
|
|
138
|
+
event = ('delete' if mask & (IN_DELETE | IN_MOVED_FROM) else
|
|
139
|
+
'create' if mask & (IN_CREATE | IN_MOVED_TO) else 'update')
|
|
140
|
+
emit('event', type=event, path=target)
|
|
141
|
+
|
|
142
|
+
def replace_file_watches(paths):
|
|
143
|
+
requested = set(paths)
|
|
144
|
+
removed = False
|
|
145
|
+
for old in set(file_paths) - requested:
|
|
146
|
+
wd = file_paths.pop(old)
|
|
147
|
+
aliases = file_watches[wd]
|
|
148
|
+
aliases.remove(old)
|
|
149
|
+
if not aliases:
|
|
150
|
+
del file_watches[wd]
|
|
151
|
+
intentional_removals.add(wd)
|
|
152
|
+
libc.inotify_rm_watch(fd, wd)
|
|
153
|
+
removed = True
|
|
154
|
+
# Consume IN_IGNORED before adding new watches so a recycled watch
|
|
155
|
+
# descriptor cannot be mistaken for an intentionally removed one.
|
|
156
|
+
if removed and not drain():
|
|
157
|
+
return False
|
|
158
|
+
for added in requested - set(file_paths):
|
|
159
|
+
absolute = os.path.abspath(added)
|
|
160
|
+
if os.path.commonpath((root, absolute)) != root or \
|
|
161
|
+
os.path.commonpath((root, os.path.realpath(absolute))) != root:
|
|
162
|
+
raise ValueError('installed watch escaped checkout')
|
|
163
|
+
wd = libc.inotify_add_watch(fd, os.fsencode(absolute), FILE_MASK)
|
|
164
|
+
if wd < 0:
|
|
165
|
+
raise OSError(ctypes.get_errno(), 'installed inotify_add_watch failed')
|
|
166
|
+
file_paths[added] = wd
|
|
167
|
+
file_watches.setdefault(wd, set()).add(added)
|
|
168
|
+
return True
|
|
169
|
+
|
|
170
|
+
try:
|
|
171
|
+
add_tree(root)
|
|
172
|
+
emit('ready', watches=len(watches))
|
|
173
|
+
stdin = b''
|
|
174
|
+
heartbeat = time.monotonic()
|
|
175
|
+
while True:
|
|
176
|
+
injected = os.environ.get('VSTATE_TEST_LINUX_GAP_FILE') if \
|
|
177
|
+
os.environ.get('VSTATE_TEST_FAULTS') == '1' else None
|
|
178
|
+
if injected and os.path.exists(injected):
|
|
179
|
+
with open(injected, encoding='utf-8') as source:
|
|
180
|
+
reason = source.read(64).strip()
|
|
181
|
+
os.unlink(injected)
|
|
182
|
+
emit('gap', reason=reason if reason in
|
|
183
|
+
('inotify_overflow', 'inotify_watch_lost',
|
|
184
|
+
'inotify_watch_add_failed') else 'inotify_test_gap')
|
|
185
|
+
return 2
|
|
186
|
+
readable, _, _ = select.select([fd, 0], [], [], 0.25)
|
|
187
|
+
if fd in readable and not drain():
|
|
188
|
+
return 2
|
|
189
|
+
if 0 in readable:
|
|
190
|
+
chunk = os.read(0, 4096)
|
|
191
|
+
if not chunk:
|
|
192
|
+
return 0
|
|
193
|
+
stdin += chunk
|
|
194
|
+
while b'\n' in stdin:
|
|
195
|
+
line, stdin = stdin.split(b'\n', 1)
|
|
196
|
+
message = json.loads(line)
|
|
197
|
+
if message.get('kind') == 'barrier':
|
|
198
|
+
hang = os.environ.get('VSTATE_TEST_LINUX_HANG_FILE') if \
|
|
199
|
+
os.environ.get('VSTATE_TEST_FAULTS') == '1' else None
|
|
200
|
+
if hang and os.path.exists(hang):
|
|
201
|
+
continue
|
|
202
|
+
if not drain():
|
|
203
|
+
return 2
|
|
204
|
+
emit('barrier', id=message['id'])
|
|
205
|
+
elif message.get('kind') == 'watch-files':
|
|
206
|
+
try:
|
|
207
|
+
if not replace_file_watches(message['paths']):
|
|
208
|
+
return 2
|
|
209
|
+
except (OSError, ValueError) as error:
|
|
210
|
+
emit('gap', reason='inotify_file_watch_add_failed',
|
|
211
|
+
errno=getattr(error, 'errno', None))
|
|
212
|
+
return 2
|
|
213
|
+
if not drain():
|
|
214
|
+
return 2
|
|
215
|
+
emit('watched', id=message['id'], files=len(file_paths))
|
|
216
|
+
if time.monotonic() - heartbeat >= 1:
|
|
217
|
+
emit('heartbeat', watches=len(watches))
|
|
218
|
+
heartbeat = time.monotonic()
|
|
219
|
+
finally:
|
|
220
|
+
os.close(fd)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
if __name__ == '__main__':
|
|
224
|
+
try:
|
|
225
|
+
sys.exit(main(sys.argv[1]))
|
|
226
|
+
except (OSError, ValueError, IndexError) as error:
|
|
227
|
+
emit('gap', reason='inotify_backend_error', errno=getattr(error, 'errno', None))
|
|
228
|
+
sys.exit(2)
|