cyc-stack 0.0.0-stage → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +12 -0
- package/.claude-plugin/marketplace.json +14 -0
- package/.claude-plugin/plugin.json +17 -0
- package/.codex-plugin/plugin.json +33 -0
- package/.cursor-plugin/marketplace.json +13 -0
- package/.cursor-plugin/plugin.json +19 -0
- package/INSTALL.md +108 -0
- package/LICENSE +21 -0
- package/README.md +359 -2
- package/bin/cyc.mjs +285 -0
- package/catalog/playbooks.json +20 -0
- package/docs/README.md +30 -0
- package/docs/assets/cyc-method-library.svg +39 -0
- package/docs/assets/cyc-overview.svg +44 -0
- package/docs/assets/cyc-readme-banner-mobile.svg +31 -0
- package/docs/assets/cyc-readme-banner.svg +39 -0
- package/docs/assets/cyc-workflow-diagram.svg +415 -0
- package/docs/code-quality.md +42 -0
- package/docs/context.md +40 -0
- package/docs/cyc-method-map.md +58 -0
- package/docs/cyc-workflow.md +188 -0
- package/docs/engineering/code-standards.md +103 -0
- package/docs/engineering/domain-design.md +81 -0
- package/docs/engineering/failure-handling.md +99 -0
- package/docs/impact-analysis.md +70 -0
- package/docs/installation.md +196 -0
- package/docs/onboarding-recovery.md +75 -0
- package/docs/principles.md +49 -0
- package/docs/project-tools.md +108 -0
- package/docs/response-standard.md +202 -0
- package/docs/runtime.md +247 -0
- package/docs/skill-contract.md +70 -0
- package/docs/skill-craft.md +23 -0
- package/docs/validation.md +39 -0
- package/docs/work-assessments.md +102 -0
- package/hooks/claude.json +10 -0
- package/hooks/codex.json +28 -0
- package/hooks/cursor.json +8 -0
- package/methods/cyc-build/SKILL.md +65 -0
- package/methods/cyc-build/references/file-writes.md +11 -0
- package/methods/cyc-challenge/SKILL.md +48 -0
- package/methods/cyc-context/SKILL.md +76 -0
- package/methods/cyc-context/references/evidence.md +27 -0
- package/methods/cyc-context/references/templates.md +54 -0
- package/methods/cyc-context/scripts/evidence.mjs +61 -0
- package/methods/cyc-correct/SKILL.md +43 -0
- package/methods/cyc-deliver/SKILL.md +52 -0
- package/methods/cyc-design/SKILL.md +50 -0
- package/methods/cyc-design/frontend-design/SKILL.md +43 -0
- package/methods/cyc-design/frontend-design/references/color.md +77 -0
- package/methods/cyc-design/frontend-design/references/craft-rules.md +61 -0
- package/methods/cyc-design/frontend-design/references/decision-sheet.md +110 -0
- package/methods/cyc-design/frontend-design/references/experience-review.md +58 -0
- package/methods/cyc-design/frontend-design/references/generic-tells.md +76 -0
- package/methods/cyc-design/frontend-design/references/layout-archetypes.md +188 -0
- package/methods/cyc-design/frontend-design/references/reference-standards.md +38 -0
- package/methods/cyc-design/frontend-design/references/typography.md +51 -0
- package/methods/cyc-design/frontend-design/references/ux-foundations.md +45 -0
- package/methods/cyc-design/frontend-design/references/visual-material.md +42 -0
- package/methods/cyc-error-ledger/SKILL.md +53 -0
- package/methods/cyc-error-ledger/references/entry-format.md +38 -0
- package/methods/cyc-explore/SKILL.md +41 -0
- package/methods/cyc-frame/SKILL.md +49 -0
- package/methods/cyc-impact/SKILL.md +64 -0
- package/methods/cyc-reflect/SKILL.md +48 -0
- package/methods/cyc-remember/SKILL.md +53 -0
- package/methods/cyc-scout/SKILL.md +67 -0
- package/methods/cyc-setup/SKILL.md +60 -0
- package/methods/cyc-teach/SKILL.md +40 -0
- package/methods/cyc-understand/SKILL.md +40 -0
- package/methods/cyc-understand/references/diagnose.md +53 -0
- package/methods/cyc-verify/SKILL.md +84 -0
- package/methods/cyc-verify/references/recipe.md +53 -0
- package/package.json +93 -4
- package/plugin.json +40 -0
- package/rules/cyc-delivery.mdc +33 -0
- package/scripts/baseline-worker.mjs +10 -0
- package/scripts/catalog.mjs +29 -0
- package/scripts/check-response.mjs +13 -0
- package/scripts/claude-project-hook.mjs +12 -0
- package/scripts/context-review.mjs +600 -0
- package/scripts/host-activation.mjs +66 -0
- package/scripts/install.mjs +387 -0
- package/scripts/onboarding.mjs +110 -0
- package/scripts/packet.mjs +107 -0
- package/scripts/project-tools.mjs +196 -0
- package/scripts/recovery.mjs +66 -0
- package/scripts/response-gate.mjs +105 -0
- package/scripts/response-hook.mjs +237 -0
- package/scripts/response-mode.mjs +12 -0
- package/scripts/run-gate.mjs +264 -0
- package/scripts/runtime.mjs +1723 -0
- package/scripts/source-hash-worker.mjs +5 -0
- package/scripts/source-snapshot.mjs +405 -0
- package/scripts/test-output.mjs +258 -0
- package/scripts/work-review.mjs +257 -0
- package/skills/cyc/SKILL.md +130 -0
- package/skills/cyc/references/playbooks/bug-fix.md +13 -0
- package/skills/cyc/references/playbooks/evaluate.md +13 -0
- package/skills/cyc/references/playbooks/experience.md +7 -0
- package/skills/cyc/references/playbooks/explore.md +9 -0
- package/skills/cyc/references/playbooks/feature.md +14 -0
- package/skills/cyc/references/playbooks/handoff.md +11 -0
- package/skills/cyc/references/playbooks/investigation.md +7 -0
- package/skills/cyc/references/playbooks/performance.md +7 -0
- package/skills/cyc/references/playbooks/plan.md +11 -0
- package/skills/cyc/references/playbooks/refactor.md +12 -0
- package/skills/cyc/references/playbooks/reflect.md +11 -0
- package/skills/cyc/references/playbooks/resume.md +11 -0
- package/skills/cyc/references/playbooks/review.md +9 -0
- package/skills/cyc/references/playbooks/teach.md +11 -0
- package/skills/cyc/references/playbooks/verify.md +7 -0
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
import { closeSync, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
3
|
+
import { spawnSync } from 'node:child_process';
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
5
|
+
import { availableParallelism } from 'node:os';
|
|
6
|
+
import { Worker } from 'node:worker_threads';
|
|
7
|
+
|
|
8
|
+
const hash = value => createHash('sha256').update(value).digest('hex');
|
|
9
|
+
// Defaults for the selected source; a project can set its own in source.maxFiles and source.maxBytes.
|
|
10
|
+
export const sourceLimits = Object.freeze({ maxFiles: 200000, maxBytes: 2 * 1024 * 1024 * 1024 });
|
|
11
|
+
const ignored = new Set(['.git', 'node_modules', '.next', 'dist', 'build', 'coverage']);
|
|
12
|
+
const internal = ['.cyc/runtime', '.cyc/runs', '.cyc/engine'];
|
|
13
|
+
const within = (path, parent) => path === parent || path.startsWith(`${parent}/`);
|
|
14
|
+
// Controller bookkeeping inside the project, never project source.
|
|
15
|
+
export const controllerPath = path => internal.some(p => within(path, p));
|
|
16
|
+
const secret = path => /(^|\/)\.env(?:$|\.)/.test(path) && !/\.(example|sample|template)$/.test(path);
|
|
17
|
+
function paths(root, value, label) {
|
|
18
|
+
if (!Array.isArray(value) || value.some(p => typeof p !== 'string' || !p || isAbsolute(p) || p.includes('\\') || p.split('/').some(x => !x || x === '.' || x === '..'))) throw new Error(`source.${label} must contain literal project-relative paths, not globs.`);
|
|
19
|
+
for (const path of value) {
|
|
20
|
+
if (!/[*?]/.test(path)) continue;
|
|
21
|
+
const parts = path.split('/');
|
|
22
|
+
let prefix = root;
|
|
23
|
+
for (let i = 0; i < parts.length; i += 1) {
|
|
24
|
+
prefix = resolve(prefix, parts[i]);
|
|
25
|
+
let stat;
|
|
26
|
+
try { stat = lstatSync(prefix); } catch (error) {
|
|
27
|
+
if (!['ENOENT', 'ENOTDIR'].includes(error.code)) throw error;
|
|
28
|
+
}
|
|
29
|
+
if (stat && !stat.isSymbolicLink()) continue;
|
|
30
|
+
// Do not follow links to establish wildcard-looking names in their targets.
|
|
31
|
+
if (parts.slice(stat ? i + 1 : i).some(part => /[*?]/.test(part))) throw new Error('source.' + label + ' entries must be literal file or directory paths. Unsupported wildcard path: ' + path + '. Name the inputs or directory prefixes explicitly.');
|
|
32
|
+
break;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return value;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function limit(policy, key) {
|
|
39
|
+
const value = policy[key] ?? sourceLimits[key];
|
|
40
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`source.${key} must be a positive integer.`);
|
|
41
|
+
return value;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function sourcePolicy(root, policy = {}) {
|
|
45
|
+
return { include: paths(root, policy.include ?? [], 'include'), exclude: paths(root, policy.exclude ?? [], 'exclude'), maxFiles: limit(policy, 'maxFiles'), maxBytes: limit(policy, 'maxBytes') };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// A snapshot taken with useCache reuses a content hash only for a file whose size, modification time,
|
|
49
|
+
// change time and file ID all match an entry recorded after the file had been still for settleMs; the
|
|
50
|
+
// settle rule covers a write within the file system's timestamp resolution, as Git does for its index.
|
|
51
|
+
// Ordinary writes move the change time, but not every write does: a write through a memory-mapped view
|
|
52
|
+
// on Windows leaves all of them unchanged, Windows tools can set the change time back, and FAT has
|
|
53
|
+
// none. So the cache serves only views between checks; snapshots that record evidence hash every file,
|
|
54
|
+
// and in doing so refresh the cache.
|
|
55
|
+
const cacheFile = '.cyc/runs/source-cache.json';
|
|
56
|
+
const settleMs = 2000;
|
|
57
|
+
const identity = stat => [stat.size, stat.mtimeMs, stat.ctimeMs, stat.ino];
|
|
58
|
+
const sameIdentity = (entry, id) => entry[0] === id[0] && entry[1] === id[1] && entry[2] === id[2] && entry[5] === id[3];
|
|
59
|
+
function readCache(root) {
|
|
60
|
+
try {
|
|
61
|
+
const cache = JSON.parse(readFileSync(resolve(root, cacheFile), 'utf8'));
|
|
62
|
+
if (cache?.version === 1 && cache.entries && typeof cache.entries === 'object' && !Array.isArray(cache.entries)) return cache.entries;
|
|
63
|
+
} catch { /* a missing or unreadable cache only means hashing every file */ }
|
|
64
|
+
return {};
|
|
65
|
+
}
|
|
66
|
+
function storeCache(root, entries) {
|
|
67
|
+
const path = resolve(root, cacheFile), temporary = `${path}.${process.pid}.${Date.now()}.tmp`;
|
|
68
|
+
try { mkdirSync(dirname(path), { recursive: true }); writeFileSync(temporary, JSON.stringify({ version: 1, entries }), { mode: 0o600 }); renameSync(temporary, path); }
|
|
69
|
+
catch { rmSync(temporary, { force: true }); /* the cache is an optimization; a failed write costs only time */ }
|
|
70
|
+
}
|
|
71
|
+
// Removes the entries a stage's full reads proved wrong (see keep in inspectSource), matched by their
|
|
72
|
+
// wrong hash so a corrected entry stays; returns whether any was removed.
|
|
73
|
+
function dropDisproved(entries, stage) {
|
|
74
|
+
let removed = false;
|
|
75
|
+
for (const [path, wrong] of stage.disproved ?? []) if (Object.hasOwn(entries, path) && entries[path]?.[3] === wrong) { delete entries[path]; removed = true; }
|
|
76
|
+
return removed;
|
|
77
|
+
}
|
|
78
|
+
// Stores the cache updates snapshots collected in a stage (see inspectSource), without any entry they
|
|
79
|
+
// proved wrong: a snapshot that threw after disproving one never staged its correction.
|
|
80
|
+
export function saveCache(project, stage) {
|
|
81
|
+
if (!stage?.changed && !stage?.disproved?.size) return;
|
|
82
|
+
try {
|
|
83
|
+
const root = realpathSync(project), entries = stage.changed ? stage.entries : readCache(root);
|
|
84
|
+
if (dropDisproved(entries, stage) || stage.changed) storeCache(root, entries);
|
|
85
|
+
} catch { /* as in storeCache, a failed write costs only time */ }
|
|
86
|
+
}
|
|
87
|
+
// For an operation that refuses: stores none of the stage's new hashes, but removes the stored entries
|
|
88
|
+
// its full reads proved wrong, so the cache does not go on hiding a change from the next operation.
|
|
89
|
+
export function forgetDisproved(project, stage) {
|
|
90
|
+
if (!stage?.disproved?.size) return;
|
|
91
|
+
try {
|
|
92
|
+
const root = realpathSync(project), entries = readCache(root);
|
|
93
|
+
if (dropDisproved(entries, stage)) storeCache(root, entries);
|
|
94
|
+
} catch { /* as in storeCache, a failed write costs only time */ }
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Files are read in chunks, so a large file costs a bounded buffer rather than its size in memory.
|
|
98
|
+
// Like readFileSync, a read stops at the size the open file reports, or at its end when it reports
|
|
99
|
+
// none, so a file that grows while it is read hashes as it did; unlike it, a file over 2 GiB is hashed
|
|
100
|
+
// rather than refused with ERR_FS_FILE_TOO_LARGE. `beat` marks progress through a large file.
|
|
101
|
+
const chunkBytes = 1024 * 1024;
|
|
102
|
+
function digestFile(absolute, buffer, beat) {
|
|
103
|
+
const digest = createHash('sha256'), fd = openSync(absolute, 'r');
|
|
104
|
+
try {
|
|
105
|
+
const stat = fstatSync(fd), size = stat.isFile() ? stat.size : 0;
|
|
106
|
+
for (let total = 0; ;) {
|
|
107
|
+
const length = size ? Math.min(buffer.length, size - total) : buffer.length;
|
|
108
|
+
const read = length ? readSync(fd, buffer, 0, length, null) : 0;
|
|
109
|
+
if (!read) break;
|
|
110
|
+
digest.update(buffer.subarray(0, read));
|
|
111
|
+
total += read;
|
|
112
|
+
beat?.();
|
|
113
|
+
}
|
|
114
|
+
} finally { closeSync(fd); }
|
|
115
|
+
return digest.digest();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// A full-content snapshot reads its files on worker threads. Opening a file dominates the time, and
|
|
119
|
+
// on Windows file system filters make each open cost about 0.4 ms: one thread took 29 s for 60,000
|
|
120
|
+
// files, eight took 7 s, and more did not help. Small snapshots stay on this thread, as workers take
|
|
121
|
+
// about 25 ms to start. Workers only read and hash regular files no larger than the byte limit; this
|
|
122
|
+
// thread applies the results in path order, so limits, links, cache updates and the order of `files`
|
|
123
|
+
// are exactly as a serial read. Anything else (a link, a directory, a vanished file, a file the limit
|
|
124
|
+
// stops before it is read, any error) is handed back and inspected here in order, so an error
|
|
125
|
+
// surfaces exactly as a serial snapshot raises it.
|
|
126
|
+
const parallelFrom = 1000;
|
|
127
|
+
const maxWorkers = 8;
|
|
128
|
+
// No worker started after startupMs, or no read moved for stallMs: this thread reads the files it
|
|
129
|
+
// is waiting for itself. A worker reports each chunk it reads, so a long read is not a stall.
|
|
130
|
+
const startupMs = 2000;
|
|
131
|
+
const stallMs = 5000;
|
|
132
|
+
const workerFile = new URL('./source-hash-worker.mjs', import.meta.url);
|
|
133
|
+
const [PENDING, FILE, DEFER] = [0, 1, 2];
|
|
134
|
+
const [NEXT, DONE, STOP, STARTED, BEAT] = [0, 1, 2, 3, 4];
|
|
135
|
+
const jobPath = (shared, i) => Buffer.from(shared.names.buffer, shared.offsets[i], shared.offsets[i + 1] - shared.offsets[i]).toString();
|
|
136
|
+
|
|
137
|
+
// The jobs as memory every thread shares: paths, a claim counter and progress, and per job a state,
|
|
138
|
+
// the identity before and after reading (8 numbers) and the SHA-256 (32 bytes).
|
|
139
|
+
export function shareJobs(root, paths, maxBytes) {
|
|
140
|
+
const n = paths.length;
|
|
141
|
+
const encoded = paths.map(path => Buffer.from(path));
|
|
142
|
+
const offsets = new Float64Array(new SharedArrayBuffer(8 * (n + 1)));
|
|
143
|
+
for (let i = 0; i < n; i++) offsets[i + 1] = offsets[i] + encoded[i].length;
|
|
144
|
+
const names = new Uint8Array(new SharedArrayBuffer(Math.max(1, offsets[n])));
|
|
145
|
+
encoded.forEach((bytes, i) => names.set(bytes, offsets[i]));
|
|
146
|
+
return { root, maxBytes, names, offsets, control: new Int32Array(new SharedArrayBuffer(20)), slots: new Int32Array(new SharedArrayBuffer(4 * n)), numbers: new Float64Array(new SharedArrayBuffer(64 * n)), digests: new Uint8Array(new SharedArrayBuffer(32 * n)) };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function readJob(shared, i, buffer) {
|
|
150
|
+
let state = DEFER;
|
|
151
|
+
try {
|
|
152
|
+
const absolute = resolve(shared.root, jobPath(shared, i));
|
|
153
|
+
const stat = lstatSync(absolute);
|
|
154
|
+
// A file over the byte limit on its own fails the snapshot; the main thread raises that unread.
|
|
155
|
+
if (stat.isFile() && stat.size <= shared.maxBytes) {
|
|
156
|
+
const digest = digestFile(absolute, buffer, () => Atomics.add(shared.control, BEAT, 1));
|
|
157
|
+
shared.numbers.set([...identity(stat), ...identity(lstatSync(absolute))], i * 8);
|
|
158
|
+
shared.digests.set(digest, i * 32);
|
|
159
|
+
state = FILE;
|
|
160
|
+
}
|
|
161
|
+
} catch { /* read again on the main thread, in order, where the error is raised */ }
|
|
162
|
+
Atomics.store(shared.slots, i, state);
|
|
163
|
+
Atomics.add(shared.control, DONE, 1);
|
|
164
|
+
Atomics.notify(shared.control, DONE);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Runs on each worker (source-hash-worker.mjs): claim the next job until none are left or the
|
|
168
|
+
// snapshot has stopped. A job claimed after the stop is marked unread, so nothing waits on it.
|
|
169
|
+
export function readJobs(shared) {
|
|
170
|
+
const { control, slots } = shared, buffer = Buffer.allocUnsafe(chunkBytes);
|
|
171
|
+
Atomics.add(control, STARTED, 1);
|
|
172
|
+
for (let i; (i = Atomics.add(control, NEXT, 1)) < slots.length;) {
|
|
173
|
+
if (Atomics.load(control, STOP)) {
|
|
174
|
+
Atomics.store(slots, i, DEFER); Atomics.add(control, DONE, 1); Atomics.notify(control, DONE);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
readJob(shared, i, buffer);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function parallelReads(root, paths, workers, maxBytes) {
|
|
182
|
+
const shared = shareJobs(root, paths, maxBytes), n = paths.length;
|
|
183
|
+
const { control, slots, numbers, digests } = shared;
|
|
184
|
+
// Like the check runner, the workers get no NODE_OPTIONS preloads, and their output never reaches
|
|
185
|
+
// a command's printed result. A worker that fails to start only leaves more for the others.
|
|
186
|
+
const { NODE_OPTIONS, ...env } = process.env;
|
|
187
|
+
let started = 0;
|
|
188
|
+
for (; started < workers; started++) {
|
|
189
|
+
try { new Worker(workerFile, { workerData: shared, execArgv: [], env, stdout: true, stderr: true }).on('error', () => {}).unref(); }
|
|
190
|
+
catch { break; }
|
|
191
|
+
}
|
|
192
|
+
if (!started) return null;
|
|
193
|
+
const createdAt = Date.now();
|
|
194
|
+
const progress = () => Atomics.load(control, DONE) + Atomics.load(control, BEAT);
|
|
195
|
+
let helping = false, readByWorkers = 0;
|
|
196
|
+
// Waits up to 100 ms for a job to finish (`done` is the count read before the caller checked), and
|
|
197
|
+
// returns the progress seen and when it last moved.
|
|
198
|
+
function wait(done, seen, since) {
|
|
199
|
+
Atomics.wait(control, DONE, done, 100);
|
|
200
|
+
const now = progress();
|
|
201
|
+
return now !== seen ? [now, Date.now()] : [seen, since];
|
|
202
|
+
}
|
|
203
|
+
return {
|
|
204
|
+
workers: started,
|
|
205
|
+
get readByWorkers() { return readByWorkers; },
|
|
206
|
+
// The read for job i, or undefined when this thread must read it in order.
|
|
207
|
+
take(i) {
|
|
208
|
+
if (i >= n) return undefined;
|
|
209
|
+
for (let seen = progress(), since = Date.now(); ;) {
|
|
210
|
+
const done = Atomics.load(control, DONE), state = Atomics.load(slots, i);
|
|
211
|
+
if (state === FILE) {
|
|
212
|
+
readByWorkers++;
|
|
213
|
+
const values = Array.from(numbers.subarray(i * 8, i * 8 + 8));
|
|
214
|
+
return { id: values.slice(0, 4), after: values.slice(4), value: Buffer.from(digests.buffer, i * 32, 32).toString('hex') };
|
|
215
|
+
}
|
|
216
|
+
if (state === DEFER || helping) return undefined;
|
|
217
|
+
if (!Atomics.load(control, STARTED) && Date.now() - createdAt >= startupMs) { helping = true; continue; }
|
|
218
|
+
[seen, since] = wait(done, seen, since);
|
|
219
|
+
if (Date.now() - since >= stallMs) helping = true;
|
|
220
|
+
}
|
|
221
|
+
},
|
|
222
|
+
// Stop handing out jobs and wait for reads in progress, so no worker holds a file open afterwards
|
|
223
|
+
// (on Windows an open file keeps its folder from being renamed or removed until it is closed).
|
|
224
|
+
// A read that makes no progress for stallMs is given up on, so a worker that died cannot hang it.
|
|
225
|
+
finish() {
|
|
226
|
+
Atomics.store(control, STOP, 1);
|
|
227
|
+
const claimed = Math.min(Atomics.load(control, NEXT), n);
|
|
228
|
+
for (let i = 0, seen = progress(), since = Date.now(); i < claimed;) {
|
|
229
|
+
const done = Atomics.load(control, DONE);
|
|
230
|
+
if (Atomics.load(slots, i) !== PENDING) { i++; continue; }
|
|
231
|
+
[seen, since] = wait(done, seen, since);
|
|
232
|
+
if (Date.now() - since >= stallMs) return;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// The controller can reuse classification without persisting a duplicate path list.
|
|
239
|
+
// `signatures: 'stat'` records size and modification time instead of content, for callers
|
|
240
|
+
// that only need to notice a change cheaply (the delivery gate's prompt-to-Stop comparison).
|
|
241
|
+
// `useCache` reuses settled hashes from the cache instead of reading every file; only `writeCache`
|
|
242
|
+
// callers, controller operations that already write run state, update it, so read-only commands
|
|
243
|
+
// leave the project untouched. `writeCache: true` stores the update at once. A stage object instead
|
|
244
|
+
// collects it: later snapshots given the same stage read the cache from it, and saveCache stores it,
|
|
245
|
+
// so the controller stores new hashes only when an operation returns. `workers` sets how many
|
|
246
|
+
// threads read a full-content snapshot; by default it follows the number of files. `hashing` reports
|
|
247
|
+
// how many files worker threads read.
|
|
248
|
+
export function inspectSource(project, { signatures = 'content', useCache = false, writeCache = false, workers } = {}) {
|
|
249
|
+
const root = realpathSync(project);
|
|
250
|
+
const config = resolve(root, '.cyc/config.json');
|
|
251
|
+
const policy = existsSync(config) ? JSON.parse(readFileSync(config, 'utf8').replace(/^\uFEFF/, '')).source ?? {} : {};
|
|
252
|
+
const { include, exclude, maxFiles, maxBytes } = sourcePolicy(root, policy);
|
|
253
|
+
const startedAt = Date.now();
|
|
254
|
+
const content = signatures !== 'stat';
|
|
255
|
+
const stage = typeof writeCache === 'object' && writeCache !== null ? writeCache : null;
|
|
256
|
+
const cached = content && (useCache || writeCache) ? stage?.entries ?? readCache(root) : {};
|
|
257
|
+
const fresh = Object.create(null);
|
|
258
|
+
let cacheChanged = false;
|
|
259
|
+
// Directories are shared by many files; look each one up once per snapshot.
|
|
260
|
+
const ancestors = new Map();
|
|
261
|
+
function ancestorStat(name) {
|
|
262
|
+
if (!ancestors.has(name)) {
|
|
263
|
+
try { ancestors.set(name, { stat: lstatSync(resolve(root, name)) }); } catch (error) { ancestors.set(name, { error }); }
|
|
264
|
+
}
|
|
265
|
+
const found = ancestors.get(name);
|
|
266
|
+
if (found.error) throw found.error;
|
|
267
|
+
return found.stat;
|
|
268
|
+
}
|
|
269
|
+
let buffer;
|
|
270
|
+
function contentHash(path, stat) {
|
|
271
|
+
const id = identity(stat), entry = Object.hasOwn(cached, path) ? cached[path] : undefined;
|
|
272
|
+
if (useCache && Array.isArray(entry) && sameIdentity(entry, id) && typeof entry[3] === 'string') { fresh[path] = entry; return entry[3]; }
|
|
273
|
+
const absolute = resolve(root, path);
|
|
274
|
+
const value = digestFile(absolute, buffer ??= Buffer.allocUnsafe(chunkBytes)).toString('hex');
|
|
275
|
+
return keep(path, id, value, identity(lstatSync(absolute)));
|
|
276
|
+
}
|
|
277
|
+
// Records a hash read from disk, by this thread or a worker, in the cache when the rules allow.
|
|
278
|
+
function keep(path, id, value, after) {
|
|
279
|
+
const entry = Object.hasOwn(cached, path) ? cached[path] : undefined;
|
|
280
|
+
const steady = after.every((part, i) => part === id[i]);
|
|
281
|
+
// An entry with this file's identity but other bytes is one the cache got wrong (see above). A stage
|
|
282
|
+
// remembers it, so an operation that then refuses still removes it (forgetDisproved).
|
|
283
|
+
if (stage && steady && Array.isArray(entry) && sameIdentity(entry, id) && entry[3] !== value) (stage.disproved ??= new Map()).set(path, entry[3]);
|
|
284
|
+
// Cache only bytes read from a file that had already settled and did not change while being read.
|
|
285
|
+
if (startedAt - Math.max(id[1], id[2]) >= settleMs && steady) {
|
|
286
|
+
// An entry that already says the same is kept as it is, so a full snapshot of unchanged files writes nothing.
|
|
287
|
+
fresh[path] = Array.isArray(entry) && sameIdentity(entry, id) && entry[3] === value ? entry : [id[0], id[1], id[2], value, startedAt, id[3]];
|
|
288
|
+
if (fresh[path] !== entry) cacheChanged = true;
|
|
289
|
+
} else if (entry !== undefined) cacheChanged = true;
|
|
290
|
+
return value;
|
|
291
|
+
}
|
|
292
|
+
const files = Object.create(null);
|
|
293
|
+
const skipped = [];
|
|
294
|
+
let bytes = 0;
|
|
295
|
+
let count = 0;
|
|
296
|
+
const candidates = new Set();
|
|
297
|
+
const tracked = new Set();
|
|
298
|
+
const recoverableTracked = new Set();
|
|
299
|
+
// Large repositories list hundreds of thousands of paths; allow for that and for a slow first index read.
|
|
300
|
+
const git = args => spawnSync('git', ['-C', root, ...args], { encoding: 'utf8', windowsHide: true, timeout: 120000, maxBuffer: 512 * 1024 * 1024 });
|
|
301
|
+
const probe = git(['rev-parse', '--is-inside-work-tree']);
|
|
302
|
+
const gitAware = probe.status === 0 && probe.stdout.trim() === 'true';
|
|
303
|
+
if (!gitAware && existsSync(resolve(root, '.git'))) throw new Error('Cannot inspect Git file selection; repair Git access before starting CYC.');
|
|
304
|
+
function excluded(path, forced = false) {
|
|
305
|
+
if (controllerPath(path) || path.split('/').includes('.git')) return true;
|
|
306
|
+
if (tracked.has(path)) return false;
|
|
307
|
+
if (secret(path)) return true;
|
|
308
|
+
if (forced) return false;
|
|
309
|
+
return exclude.some(p => within(path, p)) || path.split('/').some(p => ignored.has(p));
|
|
310
|
+
}
|
|
311
|
+
function walk(path, forced = false) {
|
|
312
|
+
if (path && excluded(path, forced)) return;
|
|
313
|
+
const absolute = resolve(root, path);
|
|
314
|
+
let stat;
|
|
315
|
+
try { stat = lstatSync(absolute); } catch (e) { if (e.code === 'ENOENT') return; throw e; }
|
|
316
|
+
if (stat.isDirectory() && !stat.isSymbolicLink()) {
|
|
317
|
+
for (const name of readdirSync(absolute)) walk(path ? `${path}/${name}` : name, forced);
|
|
318
|
+
} else candidates.add(path);
|
|
319
|
+
}
|
|
320
|
+
if (gitAware) {
|
|
321
|
+
for (const [args, isTracked] of [[['ls-files', '-z', '--cached', '-v', '--', '.'], true], [['ls-files', '-z', '--others', '--exclude-standard', '--', '.'], false]]) {
|
|
322
|
+
const result = git(args);
|
|
323
|
+
if (result.status !== 0) throw new Error(`Git file selection failed: ${result.stderr || result.error?.message}`);
|
|
324
|
+
for (const entry of result.stdout.split('\0').filter(Boolean)) {
|
|
325
|
+
const path = isTracked ? entry.slice(2) : entry;
|
|
326
|
+
candidates.add(path);
|
|
327
|
+
if (isTracked) {
|
|
328
|
+
tracked.add(path);
|
|
329
|
+
// A successful stash can still omit assume-unchanged or skip-worktree bytes.
|
|
330
|
+
if (entry.startsWith('H ')) recoverableTracked.add(path);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
} else walk('');
|
|
335
|
+
// Explicit inclusion brings ignored generated inputs into coverage, without following links.
|
|
336
|
+
for (const path of include) {
|
|
337
|
+
let ancestor = '';
|
|
338
|
+
for (const part of path.split('/')) {
|
|
339
|
+
ancestor = ancestor ? `${ancestor}/${part}` : part;
|
|
340
|
+
try { if (lstatSync(resolve(root, ancestor)).isSymbolicLink()) { candidates.add(ancestor); ancestor = null; break; } }
|
|
341
|
+
catch (e) { if (e.code !== 'ENOENT') throw e; }
|
|
342
|
+
}
|
|
343
|
+
if (ancestor) walk(path, true);
|
|
344
|
+
}
|
|
345
|
+
// Folders first: a path below a link or a missing folder is never read. Every outcome is then
|
|
346
|
+
// applied in path order, which fixes the order of `files` and so the snapshot hash.
|
|
347
|
+
const steps = [];
|
|
348
|
+
for (const path of [...candidates].sort()) {
|
|
349
|
+
if (excluded(path, include.some(p => within(path, p) || within(p, path)))) continue;
|
|
350
|
+
let name = '', step = { leaf: path };
|
|
351
|
+
for (const part of path.split('/').slice(0, -1)) {
|
|
352
|
+
name = name ? `${name}/${part}` : part;
|
|
353
|
+
let stat;
|
|
354
|
+
try { stat = ancestorStat(name); } catch (error) { step = { error }; break; }
|
|
355
|
+
if (stat.isSymbolicLink()) { step = { link: name }; break; }
|
|
356
|
+
}
|
|
357
|
+
steps.push(step);
|
|
358
|
+
}
|
|
359
|
+
function link(name) {
|
|
360
|
+
if (Object.hasOwn(files, name)) return;
|
|
361
|
+
const target = readlinkSync(resolve(root, name));
|
|
362
|
+
files[name] = hash(`symlink:${target}`);
|
|
363
|
+
skipped.push({ path: name, target });
|
|
364
|
+
}
|
|
365
|
+
// `read` is a worker's read of a regular file; without one, this thread inspects the path itself.
|
|
366
|
+
function addLeaf(path, read) {
|
|
367
|
+
let stat;
|
|
368
|
+
if (!read) {
|
|
369
|
+
try { stat = lstatSync(resolve(root, path)); } catch (e) { if (e.code === 'ENOENT') return; throw e; }
|
|
370
|
+
if (stat.isSymbolicLink()) return link(path);
|
|
371
|
+
if (stat.isDirectory()) {
|
|
372
|
+
files[path] = hash('directory-entry');
|
|
373
|
+
skipped.push({ path, target: 'Git submodule or tracked file replaced by a directory; verify separately.' });
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
if (!stat.isFile()) return;
|
|
377
|
+
}
|
|
378
|
+
bytes += read ? read.id[0] : stat.size;
|
|
379
|
+
if (++count > maxFiles) throw new Error(`Selected source has more than ${maxFiles} files. Exclude generated or vendored folders with source.exclude, or raise source.maxFiles in .cyc/config.json.`);
|
|
380
|
+
if (bytes > maxBytes) throw new Error(`Selected source is larger than ${maxBytes} bytes. Exclude generated or vendored folders with source.exclude, or raise source.maxBytes in .cyc/config.json.`);
|
|
381
|
+
files[path] = !content ? `stat:${stat.size}:${stat.mtimeMs}` : read ? keep(path, read.id, read.value, read.after) : contentHash(path, stat);
|
|
382
|
+
}
|
|
383
|
+
const leaves = steps.flatMap(step => step.leaf ? [step.leaf] : []);
|
|
384
|
+
const threads = !content || useCache ? 0 : workers ?? (leaves.length < parallelFrom ? 0 : Math.min(maxWorkers, Math.max(2, availableParallelism())));
|
|
385
|
+
// A copy of this module without its worker file reads on this thread. Past maxFiles + 1 paths the
|
|
386
|
+
// file limit has usually been reached, so shared memory stops there; any later paths read here.
|
|
387
|
+
const reads = threads > 0 && leaves.length > 0 && existsSync(workerFile) ? parallelReads(root, leaves.slice(0, maxFiles + 1), threads, maxBytes) : null;
|
|
388
|
+
try {
|
|
389
|
+
let job = 0;
|
|
390
|
+
for (const step of steps) {
|
|
391
|
+
if (step.error) { if (step.error.code === 'ENOENT') continue; throw step.error; }
|
|
392
|
+
if (step.link) link(step.link);
|
|
393
|
+
else addLeaf(step.leaf, reads?.take(job++));
|
|
394
|
+
}
|
|
395
|
+
} finally { reads?.finish(); }
|
|
396
|
+
// Entries for files no longer selected drop out, so the cache tracks the current selection.
|
|
397
|
+
if (content && writeCache && (cacheChanged || Object.keys(fresh).length !== Object.keys(cached).length)) {
|
|
398
|
+
if (stage) Object.assign(stage, { entries: fresh, changed: true });
|
|
399
|
+
else storeCache(root, fresh);
|
|
400
|
+
}
|
|
401
|
+
// Collected without a prototype so a file named __proto__ is kept; returned as an ordinary object.
|
|
402
|
+
return { snapshot: { hash: hash(JSON.stringify(files)), files: { ...files }, skipped, selection: gitAware ? 'git' : 'filesystem' }, recoverableTracked, hashing: { workers: reads?.workers ?? 0, readByWorkers: reads?.readByWorkers ?? 0 } };
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
export const sourceSnapshot = (project, options) => inspectSource(project, options).snapshot;
|