@incremental/watcher 0.0.0-stage → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/session.ts ADDED
@@ -0,0 +1,336 @@
1
+ // The watcher loop (spec/adapters/watcher.md). A session holds the event history
2
+ // since a store baseline B and turns it into a ChangeSet for `affected`. It mirrors
3
+ // the typescript run loop (PLAN 10.4) for observe and advance, with one difference:
4
+ // the history is only complete if the watch was running and the tree equalled B
5
+ // when it started, so the session advances only from a complete history.
6
+ import { spawnSync } from "node:child_process";
7
+ import { createHash } from "node:crypto";
8
+ import { existsSync, readFileSync, statSync } from "node:fs";
9
+ import { createRequire } from "node:module";
10
+ import path from "node:path";
11
+ import {
12
+ connect,
13
+ defaultEngine,
14
+ repoRoot,
15
+ type incremental,
16
+ type Change,
17
+ type Context,
18
+ type Input,
19
+ type Observation,
20
+ type Outcome,
21
+ type Plan,
22
+ } from "@incremental/client";
23
+ import { relOf, SKIP_DIRS, Watcher, walk } from "./watch.js";
24
+
25
+ export class ConfigError extends Error {
26
+ constructor(message: string) {
27
+ super(message);
28
+ this.name = "ConfigError";
29
+ }
30
+ }
31
+
32
+ // A task: a command and the repo-relative files or directories it reads. Inputs
33
+ // are declared, not traced: the watcher cannot see what the command reads.
34
+ export interface TaskDef {
35
+ id: string;
36
+ command: string[];
37
+ inputs: string[];
38
+ }
39
+
40
+ export interface SessionOptions {
41
+ root: string;
42
+ store: string;
43
+ tasks: TaskDef[];
44
+ timeoutMs: number;
45
+ }
46
+
47
+ export interface TaskRun {
48
+ task: string;
49
+ outcome: Outcome;
50
+ durationMs: number;
51
+ }
52
+
53
+ export interface CycleReport {
54
+ // "full" runs every task; "incremental" runs the tasks `affected` selected.
55
+ mode: "full" | "incremental";
56
+ // Why the cycle is full, or null when it is incremental.
57
+ reason: string | null;
58
+ stored: string | null;
59
+ changes: Change[];
60
+ plan: Plan | null;
61
+ selected: string[];
62
+ results: TaskRun[];
63
+ advanced: boolean;
64
+ headEnd: string;
65
+ }
66
+
67
+ const require_ = createRequire(import.meta.url);
68
+ const WATCHER_RUNNER = `incremental-watcher@${(require_("../../package.json") as { version: string }).version}`;
69
+
70
+ function gitSourcePath(): string {
71
+ return (
72
+ process.env.INCREMENTAL_GIT ||
73
+ path.join(repoRoot(), "adapters", "git", "dist", "incremental-git")
74
+ );
75
+ }
76
+
77
+ // The git change source (PLAN 11.7), used only for the baseline check and for
78
+ // the advance. The watcher itself runs no git.
79
+ interface GitChangeSet {
80
+ baseline: string;
81
+ head: string;
82
+ dirty: boolean;
83
+ changes: Change[];
84
+ }
85
+
86
+ function gitChangeSet(root: string, ref: string): GitChangeSet {
87
+ const r = spawnSync(gitSourcePath(), ["changes", "--base", ref, "--root", root], {
88
+ encoding: "utf8",
89
+ });
90
+ if (r.error !== undefined)
91
+ throw new Error(`cannot run the git change source: ${r.error.message}`);
92
+ if (r.status !== 0)
93
+ throw new Error(
94
+ `git change source failed (exit ${r.status}): ${(r.stdout || r.stderr || "").trim()}`,
95
+ );
96
+ return JSON.parse(r.stdout) as GitChangeSet;
97
+ }
98
+
99
+ // The context (PLAN 8.4). The watcher observes no tool, lockfile or probe, so the
100
+ // reserved keys are constant and caps are all false. In balanced mode that makes
101
+ // any change to a path that is not a task input a fallback (P6, P7), which is sound.
102
+ export function contextFor(): Context {
103
+ return {
104
+ lockfile: "absent",
105
+ tsconfig: "absent",
106
+ pkgjson: "absent",
107
+ runner: WATCHER_RUNNER,
108
+ node: process.version,
109
+ fmt: 1,
110
+ caps: { probes: false, dirs: false, env: false },
111
+ };
112
+ }
113
+
114
+ type FileInput = Extract<Input, { kind: "file" }>;
115
+
116
+ function sha256(data: Buffer): string {
117
+ return createHash("sha256").update(data).digest("hex");
118
+ }
119
+
120
+ // The file inputs of a task: each declared path, directories expanded. Missing
121
+ // paths contribute nothing (a later creation is an added path, which falls back).
122
+ export function inputsOf(root: string, task: TaskDef): FileInput[] {
123
+ const files = new Map<string, FileInput>();
124
+ for (const entry of task.inputs) {
125
+ const abs = path.resolve(root, entry);
126
+ const rel = relOf(root, abs);
127
+ if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) {
128
+ throw new ConfigError(`task ${task.id}: input ${entry} is outside the repository`);
129
+ }
130
+ if (rel.split("/").some((s) => SKIP_DIRS.has(s))) {
131
+ throw new ConfigError(`task ${task.id}: input ${entry} is under an ignored directory`);
132
+ }
133
+ if (existsSync(abs)) {
134
+ const st = statSync(abs);
135
+ const kinds: Map<string, string> = st.isDirectory()
136
+ ? walk(root, abs)
137
+ : new Map([[rel, "file"]]);
138
+ for (const [child, kind] of kinds) {
139
+ if (kind === "file") {
140
+ files.set(child, {
141
+ kind: "file",
142
+ path: child,
143
+ hash: sha256(readFileSync(path.join(root, child))),
144
+ });
145
+ }
146
+ }
147
+ }
148
+ }
149
+ return [...files.values()].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
150
+ }
151
+
152
+ export class Session {
153
+ private client: incremental | null = null;
154
+ private readonly watcher: Watcher;
155
+ // The baseline the event history is complete since, or null.
156
+ private base: string | null = null;
157
+ // The first cycle of a session runs everything: a restart has no history.
158
+ private first = true;
159
+
160
+ constructor(private readonly opts: SessionOptions) {
161
+ this.watcher = new Watcher(opts.root);
162
+ }
163
+
164
+ // Connects to the store and opens a watch window against the stored baseline.
165
+ async open(): Promise<void> {
166
+ this.client = await connect({
167
+ store: this.opts.store,
168
+ root: this.opts.root,
169
+ command: [defaultEngine(), "serve"],
170
+ timeoutMs: 60_000,
171
+ });
172
+ await this.arm();
173
+ }
174
+
175
+ async close(): Promise<void> {
176
+ this.watcher.stop();
177
+ await this.client?.close();
178
+ this.client = null;
179
+ }
180
+
181
+ // Duration of the last full check of the tree, in milliseconds.
182
+ lastCheckMs(): number {
183
+ return this.watcher.lastWalkMs;
184
+ }
185
+
186
+ // Events the watcher has recorded since the window started, for callers that
187
+ // want to know whether a cycle has work to do.
188
+ pending(): number {
189
+ return this.watcher.state().changes.length;
190
+ }
191
+
192
+ // Reports an event loss from outside the watcher (for example a dropped
193
+ // notification). The next cycle runs everything.
194
+ reportLoss(reason: string): void {
195
+ this.watcher.loss(reason);
196
+ }
197
+
198
+ // Starts a new window. The history is complete only if the tree equalled the
199
+ // stored baseline when the window started, and the window is then the baseline.
200
+ private async arm(): Promise<void> {
201
+ this.watcher.start();
202
+ const stored = (await this.need().baseline()).baseline;
203
+ this.base = null;
204
+ if (stored === null) return;
205
+ let cs: GitChangeSet;
206
+ try {
207
+ cs = gitChangeSet(this.opts.root, stored);
208
+ } catch {
209
+ return;
210
+ }
211
+ if (cs.baseline === stored && cs.changes.length === 0 && !cs.dirty) this.base = stored;
212
+ }
213
+
214
+ private need(): incremental {
215
+ if (this.client === null) throw new Error("the session is not open");
216
+ return this.client;
217
+ }
218
+
219
+ // One cycle: the ChangeSet since the stored baseline, `affected`, the selected
220
+ // tasks run and observed, then advance if the history was complete.
221
+ async cycle(): Promise<CycleReport> {
222
+ const client = this.need();
223
+ const { root } = this.opts;
224
+ const stored = (await client.baseline()).baseline;
225
+ const startHead = gitChangeSet(root, "HEAD").head;
226
+ const gen = this.watcher.generation;
227
+ const state = this.watcher.state();
228
+ const complete = this.base !== null && this.base === stored && state.lost === null;
229
+
230
+ let reason: string | null = null;
231
+ if (this.first) reason = "no_history";
232
+ else if (this.base === null) reason = "not_watching_since_baseline";
233
+ else if (this.base !== stored) reason = "baseline_moved";
234
+ else if (state.lost !== null) reason = `watcher_lost: ${state.lost}`;
235
+
236
+ let plan: Plan | null = null;
237
+ if (reason === null && stored !== null) {
238
+ plan = await client.affected(
239
+ { baseline: stored, changes: state.changes },
240
+ {
241
+ baseline: stored,
242
+ context: contextFor(),
243
+ declared_tasks: this.opts.tasks.map((t) => t.id),
244
+ verify: false,
245
+ policy: "balanced",
246
+ },
247
+ );
248
+ if (plan.fallback !== null) reason = `${plan.fallback.reason}`;
249
+ const declared = new Set(this.opts.tasks.map((t) => t.id));
250
+ if (reason === null && plan.tasks.some((t) => !declared.has(t.id)))
251
+ reason = "plan_unknown_task";
252
+ }
253
+
254
+ const full = reason !== null || stored === null;
255
+ const selectedIds = full
256
+ ? this.opts.tasks.map((t) => t.id)
257
+ : (plan?.tasks ?? []).map((t) => t.id);
258
+ const selected = this.opts.tasks.filter((t) => selectedIds.includes(t.id));
259
+
260
+ const results: TaskRun[] = [];
261
+ const observations: Observation[] = [];
262
+ const obsBaseline = stored ?? startHead;
263
+ for (const task of selected) {
264
+ const inputs = inputsOf(root, task);
265
+ const run = runTask(root, task, this.opts.timeoutMs);
266
+ results.push({ task: task.id, ...run });
267
+ observations.push({
268
+ task: task.id,
269
+ baseline: obsBaseline,
270
+ dirty: true,
271
+ context: contextFor(),
272
+ inputs,
273
+ outputs: [],
274
+ outcome: run.outcome,
275
+ duration_ms: run.durationMs,
276
+ });
277
+ }
278
+
279
+ const end = gitChangeSet(root, "HEAD");
280
+ // The tree is clean at HEAD, the window covered the baseline, and no event
281
+ // arrived during the run: the runs saw exactly the tree that the baseline names.
282
+ const treeClean =
283
+ complete && !end.dirty && end.head === startHead && this.watcher.generation === gen;
284
+ // Observations are reusable only from a clean tree (dirty ones are never trusted).
285
+ for (const o of observations) o.dirty = !treeClean;
286
+ if (observations.length > 0) await client.observeMany(observations);
287
+
288
+ const allRan = results.every((r) => r.outcome !== "error");
289
+ const canAdvance = treeClean && stored !== null && allRan;
290
+ let advanced = false;
291
+ if (canAdvance && end.head !== stored) {
292
+ await client.advance(end.head, stored);
293
+ advanced = true;
294
+ this.base = end.head;
295
+ this.watcher.rebase();
296
+ }
297
+ // A full cycle that did not advance starts a new window, so the next cycle can
298
+ // be incremental again if the tree equals the stored baseline.
299
+ if (full && !advanced) await this.arm();
300
+
301
+ const report: CycleReport = {
302
+ mode: full ? "full" : "incremental",
303
+ reason,
304
+ stored,
305
+ changes: state.changes,
306
+ plan,
307
+ selected: selected.map((t) => t.id),
308
+ results,
309
+ advanced,
310
+ headEnd: end.head,
311
+ };
312
+ this.first = false;
313
+ return report;
314
+ }
315
+ }
316
+
317
+ // Runs one task command in the repository. A command that cannot start, or is
318
+ // killed, is an error, never a pass.
319
+ export function runTask(
320
+ root: string,
321
+ task: TaskDef,
322
+ timeoutMs: number,
323
+ ): { outcome: Outcome; durationMs: number } {
324
+ const started = Date.now();
325
+ const [cmd, ...args] = task.command;
326
+ if (cmd === undefined) throw new ConfigError(`task ${task.id}: empty command`);
327
+ const r = spawnSync(cmd, args, {
328
+ cwd: root,
329
+ encoding: "utf8",
330
+ timeout: timeoutMs,
331
+ maxBuffer: 64 * 1024 * 1024,
332
+ });
333
+ const durationMs = Date.now() - started;
334
+ if (r.error !== undefined || r.signal !== null) return { outcome: "error", durationMs };
335
+ return { outcome: r.status === 0 ? "pass" : "fail", durationMs };
336
+ }
package/src/watch.ts ADDED
@@ -0,0 +1,215 @@
1
+ // The file-system watch (spec/adapters/watcher.md). File events only wake the
2
+ // caller up. The changes are never read from the events: state() compares a
3
+ // fresh stat walk of the tree with the snapshot taken at the start of the window.
4
+ // So an event the OS drops (macOS FSEvents can coalesce or lose them silently)
5
+ // delays a run but cannot hide a change. A file whose mtime is too close to the
6
+ // snapshot to tell a later edit apart (the "racy" case of git) is hashed at
7
+ // snapshot time and compared by content.
8
+ import { createHash } from "node:crypto";
9
+ import { lstatSync, readdirSync, readFileSync, realpathSync, watch, type FSWatcher } from "node:fs";
10
+ import path from "node:path";
11
+ import type { Change } from "@incremental/client";
12
+
13
+ function sha256(data: Buffer): string {
14
+ return createHash("sha256").update(data).digest("hex");
15
+ }
16
+
17
+ // Directories the watcher never enters or reports. Task inputs must not lie under them.
18
+ export const SKIP_DIRS = new Set(["node_modules", ".git", "dist", ".incremental", ".toolchain"]);
19
+
20
+ type Kind = "file" | "dir" | "other";
21
+
22
+ export interface WatchState {
23
+ changes: Change[];
24
+ // Why the changes cannot be trusted as a complete set, or null.
25
+ lost: string | null;
26
+ }
27
+
28
+ export function relOf(root: string, abs: string): string {
29
+ return path.relative(root, abs).split(path.sep).join("/");
30
+ }
31
+
32
+ // Every entry under dir as repo-relative path to kind, skipping SKIP_DIRS and not
33
+ // following symbolic links. Used for the start snapshot and for new directories.
34
+ // Stat signature of a file at snapshot time.
35
+ export interface Entry {
36
+ kind: Kind;
37
+ size: number;
38
+ mtimeMs: number;
39
+ ino: number;
40
+ // Content hash, taken at snapshot time for a racy file only.
41
+ hash?: string;
42
+ }
43
+
44
+ // A file modified within this many milliseconds before the snapshot is racy:
45
+ // a later edit could carry the same timestamp, so its stat cannot be trusted.
46
+ const RACY_MS = 2_000;
47
+
48
+ // Every entry under dir with its stat signature, skipping SKIP_DIRS and not
49
+ // following symbolic links.
50
+ export function statWalk(
51
+ root: string,
52
+ dir: string,
53
+ out: Map<string, Entry> = new Map(),
54
+ ): Map<string, Entry> {
55
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
56
+ if (!SKIP_DIRS.has(entry.name)) {
57
+ const abs = path.join(dir, entry.name);
58
+ const rel = relOf(root, abs);
59
+ const st = lstatSync(abs);
60
+ const kind: Kind = st.isFile() ? "file" : st.isDirectory() ? "dir" : "other";
61
+ out.set(rel, { kind, size: st.size, mtimeMs: st.mtimeMs, ino: st.ino });
62
+ if (kind === "dir") statWalk(root, abs, out);
63
+ }
64
+ }
65
+ return out;
66
+ }
67
+
68
+ export function walk(
69
+ root: string,
70
+ dir: string,
71
+ out: Map<string, Kind> = new Map(),
72
+ ): Map<string, Kind> {
73
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
74
+ if (!SKIP_DIRS.has(entry.name)) {
75
+ const abs = path.join(dir, entry.name);
76
+ const rel = relOf(root, abs);
77
+ if (entry.isDirectory()) {
78
+ out.set(rel, "dir");
79
+ walk(root, abs, out);
80
+ } else if (entry.isFile()) {
81
+ out.set(rel, "file");
82
+ } else {
83
+ out.set(rel, "other");
84
+ }
85
+ }
86
+ }
87
+ return out;
88
+ }
89
+
90
+ function _kindOf(abs: string): Kind | null {
91
+ try {
92
+ const st = lstatSync(abs);
93
+ return st.isFile() ? "file" : st.isDirectory() ? "dir" : "other";
94
+ } catch {
95
+ return null;
96
+ }
97
+ }
98
+
99
+ export class Watcher {
100
+ readonly root: string;
101
+ private handle: FSWatcher | null = null;
102
+ // The tree at the start of the current window (the baseline view).
103
+ private base = new Map<string, Entry>();
104
+ private baseTime = 0;
105
+ // Duration of the last state() walk, in milliseconds.
106
+ lastWalkMs = 0;
107
+ private lostReason: string | null = null;
108
+ // Incremented on every event or loss. A caller compares it to detect events
109
+ // that happened during a run. It is a wake-up signal only, never a change list.
110
+ generation = 0;
111
+
112
+ constructor(root: string) {
113
+ this.root = realpathSync(root);
114
+ }
115
+
116
+ // Starts watching and snapshots the tree. The watch is opened first so an
117
+ // edit during the snapshot is recorded, not missed.
118
+ start(): void {
119
+ this.stop();
120
+ this.handle = watch(this.root, { recursive: true, encoding: "utf8" }, (_event, name) =>
121
+ this.onEvent(name),
122
+ );
123
+ this.handle.on("error", (e: Error) => this.loss(`watch error: ${e.message}`));
124
+ this.rebase();
125
+ }
126
+
127
+ // A new window: the current tree is the baseline view and nothing is changed.
128
+ rebase(): void {
129
+ this.baseTime = Date.now();
130
+ this.base = statWalk(this.root, this.root);
131
+ for (const [rel, entry] of this.base) {
132
+ if (entry.kind === "file" && this.racyAt(entry, this.baseTime)) {
133
+ entry.hash = sha256(readFileSync(path.join(this.root, rel)));
134
+ }
135
+ }
136
+ this.lostReason = null;
137
+ this.generation++;
138
+ }
139
+
140
+ stop(): void {
141
+ this.handle?.close();
142
+ this.handle = null;
143
+ }
144
+
145
+ // Reports a loss from outside the event stream (for example an overflow the
146
+ // caller learned about). The window cannot be trusted until rebase().
147
+ loss(reason: string): void {
148
+ this.lostReason ??= reason;
149
+ this.generation++;
150
+ }
151
+
152
+ private onEvent(name: string | null): void {
153
+ if (typeof name !== "string") {
154
+ this.loss("event without a file name");
155
+ return;
156
+ }
157
+ const segments = name.split(path.sep).join("/").split("/");
158
+ if (segments.some((s) => SKIP_DIRS.has(s))) return;
159
+ this.generation++;
160
+ }
161
+
162
+ // The changes since the window started, from a fresh walk against the
163
+ // snapshot. A file is added or deleted by its presence, modified by its stat
164
+ // signature (or by the racy rule). A removed or replaced directory, or a
165
+ // symbolic link, is lost: it cannot be compared entry by entry.
166
+ state(): WatchState {
167
+ const started = performance.now();
168
+ const fresh = statWalk(this.root, this.root);
169
+ this.lastWalkMs = performance.now() - started;
170
+ const changes: Change[] = [];
171
+ let lost = this.lostReason;
172
+ for (const [rel, now] of fresh) {
173
+ if (now.kind === "other") {
174
+ lost ??= `symbolic link or special file: ${rel}`;
175
+ } else if (now.kind !== "dir") {
176
+ const before = this.base.get(rel);
177
+ if (before === undefined) {
178
+ changes.push({ path: rel, status: "added" });
179
+ } else if (before.kind !== "file") {
180
+ lost ??= `directory or link replaced by a file: ${rel}`;
181
+ } else if (before.hash !== undefined) {
182
+ if (sha256(readFileSync(path.join(this.root, rel))) !== before.hash)
183
+ changes.push({ path: rel, status: "modified" });
184
+ } else if (
185
+ before.size !== now.size ||
186
+ before.mtimeMs !== now.mtimeMs ||
187
+ before.ino !== now.ino
188
+ ) {
189
+ changes.push({ path: rel, status: "modified" });
190
+ }
191
+ }
192
+ }
193
+ for (const [rel, before] of this.base) {
194
+ if (before.kind !== "other") {
195
+ const now = fresh.get(rel);
196
+ if (now === undefined) {
197
+ if (before.kind === "dir") lost ??= `directory removed: ${rel}`;
198
+ else changes.push({ path: rel, status: "deleted" });
199
+ } else if (before.kind === "dir" && now.kind !== "dir") {
200
+ lost ??= `directory replaced: ${rel}`;
201
+ } else if (before.kind === "file" && now.kind === "dir") {
202
+ lost ??= `file replaced by a directory: ${rel}`;
203
+ }
204
+ }
205
+ }
206
+ return {
207
+ changes: changes.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)),
208
+ lost,
209
+ };
210
+ }
211
+
212
+ private racyAt(entry: Entry, time: number): boolean {
213
+ return entry.mtimeMs >= time - RACY_MS;
214
+ }
215
+ }