@incremental/watcher 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/watch.ts ADDED
@@ -0,0 +1,215 @@
1
+ // The file-system watch (spec/adapters/watcher.md). File events only wake the
2
+ // caller up. The changes are never read from the events: state() compares a
3
+ // fresh stat walk of the tree with the snapshot taken at the start of the window.
4
+ // So an event the OS drops (macOS FSEvents can coalesce or lose them silently)
5
+ // delays a run but cannot hide a change. A file whose mtime is too close to the
6
+ // snapshot to tell a later edit apart (the "racy" case of git) is hashed at
7
+ // snapshot time and compared by content.
8
+ import { createHash } from "node:crypto";
9
+ import { lstatSync, readdirSync, readFileSync, realpathSync, watch, type FSWatcher } from "node:fs";
10
+ import path from "node:path";
11
+ import type { Change } from "@incremental/client";
12
+
13
+ function sha256(data: Buffer): string {
14
+ return createHash("sha256").update(data).digest("hex");
15
+ }
16
+
17
+ // Directories the watcher never enters or reports. Task inputs must not lie under them.
18
+ export const SKIP_DIRS = new Set(["node_modules", ".git", "dist", ".incremental", ".toolchain"]);
19
+
20
+ type Kind = "file" | "dir" | "other";
21
+
22
+ export interface WatchState {
23
+ changes: Change[];
24
+ // Why the changes cannot be trusted as a complete set, or null.
25
+ lost: string | null;
26
+ }
27
+
28
+ export function relOf(root: string, abs: string): string {
29
+ return path.relative(root, abs).split(path.sep).join("/");
30
+ }
31
+
32
+ // Every entry under dir as repo-relative path to kind, skipping SKIP_DIRS and not
33
+ // following symbolic links. Used for the start snapshot and for new directories.
34
+ // Stat signature of a file at snapshot time.
35
+ export interface Entry {
36
+ kind: Kind;
37
+ size: number;
38
+ mtimeMs: number;
39
+ ino: number;
40
+ // Content hash, taken at snapshot time for a racy file only.
41
+ hash?: string;
42
+ }
43
+
44
+ // A file modified within this many milliseconds before the snapshot is racy:
45
+ // a later edit could carry the same timestamp, so its stat cannot be trusted.
46
+ const RACY_MS = 2_000;
47
+
48
+ // Every entry under dir with its stat signature, skipping SKIP_DIRS and not
49
+ // following symbolic links.
50
+ export function statWalk(
51
+ root: string,
52
+ dir: string,
53
+ out: Map<string, Entry> = new Map(),
54
+ ): Map<string, Entry> {
55
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
56
+ if (!SKIP_DIRS.has(entry.name)) {
57
+ const abs = path.join(dir, entry.name);
58
+ const rel = relOf(root, abs);
59
+ const st = lstatSync(abs);
60
+ const kind: Kind = st.isFile() ? "file" : st.isDirectory() ? "dir" : "other";
61
+ out.set(rel, { kind, size: st.size, mtimeMs: st.mtimeMs, ino: st.ino });
62
+ if (kind === "dir") statWalk(root, abs, out);
63
+ }
64
+ }
65
+ return out;
66
+ }
67
+
68
+ export function walk(
69
+ root: string,
70
+ dir: string,
71
+ out: Map<string, Kind> = new Map(),
72
+ ): Map<string, Kind> {
73
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
74
+ if (!SKIP_DIRS.has(entry.name)) {
75
+ const abs = path.join(dir, entry.name);
76
+ const rel = relOf(root, abs);
77
+ if (entry.isDirectory()) {
78
+ out.set(rel, "dir");
79
+ walk(root, abs, out);
80
+ } else if (entry.isFile()) {
81
+ out.set(rel, "file");
82
+ } else {
83
+ out.set(rel, "other");
84
+ }
85
+ }
86
+ }
87
+ return out;
88
+ }
89
+
90
+ function _kindOf(abs: string): Kind | null {
91
+ try {
92
+ const st = lstatSync(abs);
93
+ return st.isFile() ? "file" : st.isDirectory() ? "dir" : "other";
94
+ } catch {
95
+ return null;
96
+ }
97
+ }
98
+
99
+ export class Watcher {
100
+ readonly root: string;
101
+ private handle: FSWatcher | null = null;
102
+ // The tree at the start of the current window (the baseline view).
103
+ private base = new Map<string, Entry>();
104
+ private baseTime = 0;
105
+ // Duration of the last state() walk, in milliseconds.
106
+ lastWalkMs = 0;
107
+ private lostReason: string | null = null;
108
+ // Incremented on every event or loss. A caller compares it to detect events
109
+ // that happened during a run. It is a wake-up signal only, never a change list.
110
+ generation = 0;
111
+
112
+ constructor(root: string) {
113
+ this.root = realpathSync(root);
114
+ }
115
+
116
+ // Starts watching and snapshots the tree. The watch is opened first so an
117
+ // edit during the snapshot is recorded, not missed.
118
+ start(): void {
119
+ this.stop();
120
+ this.handle = watch(this.root, { recursive: true, encoding: "utf8" }, (_event, name) =>
121
+ this.onEvent(name),
122
+ );
123
+ this.handle.on("error", (e: Error) => this.loss(`watch error: ${e.message}`));
124
+ this.rebase();
125
+ }
126
+
127
+ // A new window: the current tree is the baseline view and nothing is changed.
128
+ rebase(): void {
129
+ this.baseTime = Date.now();
130
+ this.base = statWalk(this.root, this.root);
131
+ for (const [rel, entry] of this.base) {
132
+ if (entry.kind === "file" && this.racyAt(entry, this.baseTime)) {
133
+ entry.hash = sha256(readFileSync(path.join(this.root, rel)));
134
+ }
135
+ }
136
+ this.lostReason = null;
137
+ this.generation++;
138
+ }
139
+
140
+ stop(): void {
141
+ this.handle?.close();
142
+ this.handle = null;
143
+ }
144
+
145
+ // Reports a loss from outside the event stream (for example an overflow the
146
+ // caller learned about). The window cannot be trusted until rebase().
147
+ loss(reason: string): void {
148
+ this.lostReason ??= reason;
149
+ this.generation++;
150
+ }
151
+
152
+ private onEvent(name: string | null): void {
153
+ if (typeof name !== "string") {
154
+ this.loss("event without a file name");
155
+ return;
156
+ }
157
+ const segments = name.split(path.sep).join("/").split("/");
158
+ if (segments.some((s) => SKIP_DIRS.has(s))) return;
159
+ this.generation++;
160
+ }
161
+
162
+ // The changes since the window started, from a fresh walk against the
163
+ // snapshot. A file is added or deleted by its presence, modified by its stat
164
+ // signature (or by the racy rule). A removed or replaced directory, or a
165
+ // symbolic link, is lost: it cannot be compared entry by entry.
166
+ state(): WatchState {
167
+ const started = performance.now();
168
+ const fresh = statWalk(this.root, this.root);
169
+ this.lastWalkMs = performance.now() - started;
170
+ const changes: Change[] = [];
171
+ let lost = this.lostReason;
172
+ for (const [rel, now] of fresh) {
173
+ if (now.kind === "other") {
174
+ lost ??= `symbolic link or special file: ${rel}`;
175
+ } else if (now.kind !== "dir") {
176
+ const before = this.base.get(rel);
177
+ if (before === undefined) {
178
+ changes.push({ path: rel, status: "added" });
179
+ } else if (before.kind !== "file") {
180
+ lost ??= `directory or link replaced by a file: ${rel}`;
181
+ } else if (before.hash !== undefined) {
182
+ if (sha256(readFileSync(path.join(this.root, rel))) !== before.hash)
183
+ changes.push({ path: rel, status: "modified" });
184
+ } else if (
185
+ before.size !== now.size ||
186
+ before.mtimeMs !== now.mtimeMs ||
187
+ before.ino !== now.ino
188
+ ) {
189
+ changes.push({ path: rel, status: "modified" });
190
+ }
191
+ }
192
+ }
193
+ for (const [rel, before] of this.base) {
194
+ if (before.kind !== "other") {
195
+ const now = fresh.get(rel);
196
+ if (now === undefined) {
197
+ if (before.kind === "dir") lost ??= `directory removed: ${rel}`;
198
+ else changes.push({ path: rel, status: "deleted" });
199
+ } else if (before.kind === "dir" && now.kind !== "dir") {
200
+ lost ??= `directory replaced: ${rel}`;
201
+ } else if (before.kind === "file" && now.kind === "dir") {
202
+ lost ??= `file replaced by a directory: ${rel}`;
203
+ }
204
+ }
205
+ }
206
+ return {
207
+ changes: changes.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)),
208
+ lost,
209
+ };
210
+ }
211
+
212
+ private racyAt(entry: Entry, time: number): boolean {
213
+ return entry.mtimeMs >= time - RACY_MS;
214
+ }
215
+ }
@@ -0,0 +1,269 @@
1
+ // Integration tests: a scratch git repository with two tasks, run through the real
2
+ // engine and incremental-git (spec/adapters/watcher.md).
3
+ import assert from "node:assert/strict";
4
+ import { spawnSync } from "node:child_process";
5
+ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
6
+ import os from "node:os";
7
+ import path from "node:path";
8
+ import { test } from "node:test";
9
+ import { connect, defaultEngine } from "@incremental/client";
10
+ import { ConfigError, inputsOf, Session, type TaskDef } from "../src/index.js";
11
+ import { Watcher } from "../src/watch.js";
12
+
13
+ const GIT_ENV = {
14
+ ...process.env,
15
+ GIT_AUTHOR_NAME: "t",
16
+ GIT_AUTHOR_EMAIL: "t@example.com",
17
+ GIT_COMMITTER_NAME: "t",
18
+ GIT_COMMITTER_EMAIL: "t@example.com",
19
+ };
20
+
21
+ function git(cwd: string, ...args: string[]): string {
22
+ const r = spawnSync("git", args, { cwd, encoding: "utf8", env: GIT_ENV });
23
+ if (r.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${r.stderr}`);
24
+ return r.stdout.trim();
25
+ }
26
+
27
+ function write(root: string, rel: string, content: string): void {
28
+ const abs = path.join(root, rel);
29
+ mkdirSync(path.dirname(abs), { recursive: true });
30
+ writeFileSync(abs, content);
31
+ }
32
+
33
+ function commit(root: string, message: string): void {
34
+ git(root, "add", "-A");
35
+ git(root, "commit", "-q", "-m", message);
36
+ }
37
+
38
+ const TASKS: TaskDef[] = ["a", "b"].map((p) => ({
39
+ id: `pkg/${p}`,
40
+ command: [process.execPath, "-e", "process.exit(0)"],
41
+ inputs: [`pkg/${p}`],
42
+ }));
43
+
44
+ // A repository, an initialised store, and the stored baseline set to HEAD.
45
+ async function fixture(): Promise<{ root: string; store: string; base: string }> {
46
+ const base = mkdtempSync(path.join(os.tmpdir(), "incremental-watcher-"));
47
+ const root = path.join(base, "repo");
48
+ mkdirSync(root);
49
+ git(root, "init", "-q", "-b", "main");
50
+ for (const p of ["a", "b"]) write(root, `pkg/${p}/src/index.ts`, `export const ${p} = 1;\n`);
51
+ write(root, "README.md", "fixture\n");
52
+ commit(root, "init");
53
+ const store = path.join(base, "store");
54
+ const init = spawnSync(defaultEngine(), ["init", "--store", store], { encoding: "utf8" });
55
+ assert.equal(init.status, 0, init.stderr);
56
+ const client = await connect({ store, root, command: [defaultEngine(), "serve"] });
57
+ try {
58
+ await client.advance(git(root, "rev-parse", "HEAD"), null);
59
+ } finally {
60
+ await client.close();
61
+ }
62
+ return { root, store, base };
63
+ }
64
+
65
+ function sessionFor(root: string, store: string): Session {
66
+ return new Session({ root, store, tasks: TASKS, timeoutMs: 60_000 });
67
+ }
68
+
69
+ async function waitForEvent(session: Session): Promise<void> {
70
+ const until = Date.now() + 5_000;
71
+ while (session.pending() === 0) {
72
+ if (Date.now() > until) throw new Error("no file-system event arrived");
73
+ await new Promise((r) => setTimeout(r, 50));
74
+ }
75
+ }
76
+
77
+ async function storedBaseline(store: string, root: string): Promise<string | null> {
78
+ const client = await connect({ store, root, command: [defaultEngine(), "serve"] });
79
+ try {
80
+ return (await client.baseline()).baseline;
81
+ } finally {
82
+ await client.close();
83
+ }
84
+ }
85
+
86
+ test("an edit event adds a ChangeSet entry and invalidates only the task that reads it", async () => {
87
+ const { root, store, base } = await fixture();
88
+ const session = sessionFor(root, store);
89
+ try {
90
+ await session.open();
91
+ const first = await session.cycle();
92
+ assert.equal(first.mode, "full");
93
+ assert.equal(first.reason, "no_history");
94
+ assert.deepEqual(first.selected, ["pkg/a", "pkg/b"]);
95
+
96
+ const quiet = await session.cycle();
97
+ assert.equal(quiet.mode, "incremental");
98
+ assert.deepEqual(quiet.selected, []);
99
+
100
+ write(root, "pkg/a/src/index.ts", "export const a = 2;\n");
101
+ await waitForEvent(session);
102
+ const edit = await session.cycle();
103
+ assert.equal(edit.mode, "incremental", `reason: ${edit.reason}`);
104
+ assert.deepEqual(edit.changes, [{ path: "pkg/a/src/index.ts", status: "modified" }]);
105
+ assert.equal(edit.plan?.fallback, null);
106
+ assert.deepEqual(edit.selected, ["pkg/a"]);
107
+ assert.deepEqual(
108
+ edit.results.map((r) => r.outcome),
109
+ ["pass"],
110
+ );
111
+ assert.equal(edit.advanced, false, "an uncommitted edit never advances");
112
+ } finally {
113
+ await session.close();
114
+ rmSync(base, { recursive: true, force: true });
115
+ }
116
+ });
117
+
118
+ test("a committed edit advances the baseline once the run is complete and the tree is clean", async () => {
119
+ const { root, store, base } = await fixture();
120
+ const session = sessionFor(root, store);
121
+ try {
122
+ await session.open();
123
+ await session.cycle();
124
+ write(root, "pkg/b/src/index.ts", "export const b = 2;\n");
125
+ await waitForEvent(session);
126
+ const edit = await session.cycle();
127
+ assert.deepEqual(edit.selected, ["pkg/b"]);
128
+ assert.equal(edit.advanced, false);
129
+
130
+ commit(root, "edit b");
131
+ const committed = await session.cycle();
132
+ assert.equal(committed.mode, "incremental", `reason: ${committed.reason}`);
133
+ assert.deepEqual(committed.selected, ["pkg/b"]);
134
+ assert.equal(committed.advanced, true);
135
+ assert.equal(await storedBaseline(store, root), git(root, "rev-parse", "HEAD"));
136
+
137
+ const quiet = await session.cycle();
138
+ assert.equal(quiet.mode, "incremental");
139
+ assert.deepEqual(quiet.selected, []);
140
+ } finally {
141
+ await session.close();
142
+ rmSync(base, { recursive: true, force: true });
143
+ }
144
+ });
145
+
146
+ test("an event loss produces a full run and no partial plan, then the watch re-arms", async () => {
147
+ const { root, store, base } = await fixture();
148
+ const session = sessionFor(root, store);
149
+ try {
150
+ await session.open();
151
+ await session.cycle();
152
+ session.reportLoss("overflow");
153
+ const lost = await session.cycle();
154
+ assert.equal(lost.mode, "full");
155
+ assert.match(lost.reason ?? "", /^watcher_lost: overflow/);
156
+ assert.equal(lost.plan, null, "no plan is computed from a lost window");
157
+ assert.deepEqual(lost.selected, ["pkg/a", "pkg/b"]);
158
+ assert.equal(lost.advanced, false);
159
+
160
+ const after = await session.cycle();
161
+ assert.equal(after.mode, "incremental", `reason: ${after.reason}`);
162
+ assert.equal(after.reason, null);
163
+ } finally {
164
+ await session.close();
165
+ rmSync(base, { recursive: true, force: true });
166
+ }
167
+ });
168
+
169
+ test("a restart with no in-memory history runs everything", async () => {
170
+ const { root, store, base } = await fixture();
171
+ try {
172
+ const before = await storedBaseline(store, root);
173
+ const first = sessionFor(root, store);
174
+ await first.open();
175
+ await first.cycle();
176
+ await first.close();
177
+
178
+ const restarted = sessionFor(root, store);
179
+ try {
180
+ await restarted.open();
181
+ const r = await restarted.cycle();
182
+ assert.equal(r.mode, "full");
183
+ assert.equal(r.reason, "no_history");
184
+ assert.deepEqual(r.selected, ["pkg/a", "pkg/b"]);
185
+ } finally {
186
+ await restarted.close();
187
+ }
188
+ assert.equal(
189
+ await storedBaseline(store, root),
190
+ before,
191
+ "a watcher run without a complete window does not move the baseline",
192
+ );
193
+ } finally {
194
+ rmSync(base, { recursive: true, force: true });
195
+ }
196
+ });
197
+
198
+ test("no advance when the watcher was not running since the baseline", async () => {
199
+ const { root, store, base } = await fixture();
200
+ const before = await storedBaseline(store, root);
201
+ try {
202
+ // A commit made while no watcher ran: the tree no longer equals the baseline.
203
+ write(root, "pkg/a/src/index.ts", "export const a = 3;\n");
204
+ commit(root, "edit while no watcher ran");
205
+
206
+ const session = sessionFor(root, store);
207
+ try {
208
+ await session.open();
209
+ const r1 = await session.cycle();
210
+ assert.equal(r1.advanced, false);
211
+ assert.deepEqual(r1.selected, ["pkg/a", "pkg/b"]);
212
+ const r2 = await session.cycle();
213
+ assert.equal(r2.mode, "full");
214
+ assert.equal(r2.reason, "not_watching_since_baseline");
215
+ assert.equal(r2.advanced, false);
216
+ } finally {
217
+ await session.close();
218
+ }
219
+ assert.equal(await storedBaseline(store, root), before, "the store baseline did not move");
220
+ } finally {
221
+ rmSync(base, { recursive: true, force: true });
222
+ }
223
+ });
224
+
225
+ test("a task input under an ignored directory is refused, not silently skipped", () => {
226
+ const { root } = { root: mkdtempSync(path.join(os.tmpdir(), "incremental-watcher-cfg-")) };
227
+ try {
228
+ write(root, "node_modules/dep/index.js", "x\n");
229
+ assert.throws(
230
+ () => inputsOf(root, { id: "t", command: ["true"], inputs: ["node_modules/dep"] }),
231
+ ConfigError,
232
+ );
233
+ } finally {
234
+ rmSync(root, { recursive: true, force: true });
235
+ }
236
+ });
237
+
238
+ // Silent drops (spec/adapters/watcher.md): the OS may deliver no event for an
239
+ // edit. The state must still report it, because it is read from a stat walk
240
+ // against the snapshot, never from the event stream.
241
+ test("an edit with no event delivered is still reported, and a rebase clears it", async () => {
242
+ const dir = mkdtempSync(path.join(os.tmpdir(), "incremental-watch-silent-"));
243
+ try {
244
+ writeFileSync(path.join(dir, "a.ts"), "export const a = 1;\n");
245
+ writeFileSync(path.join(dir, "b.ts"), "export const b = 1;\n");
246
+ const w = new Watcher(dir);
247
+ w.rebase();
248
+ // Let the snapshot age past the racy window before a file can be compared by stat.
249
+ await new Promise((r) => setTimeout(r, 2_100));
250
+ const before = w.state();
251
+ assert.deepEqual(before.changes, [], "nothing changed, nothing reported");
252
+
253
+ writeFileSync(path.join(dir, "a.ts"), "export const a = 2;\n");
254
+ rmSync(path.join(dir, "b.ts"));
255
+ // No start(): no watch is open, so no event exists for either change.
256
+ const after = w.state();
257
+ assert.deepEqual(
258
+ after.changes.map((c) => `${c.status} ${c.path}`),
259
+ ["modified a.ts", "deleted b.ts"],
260
+ );
261
+ assert.equal(after.lost, null);
262
+
263
+ w.rebase();
264
+ await new Promise((r) => setTimeout(r, 2_100));
265
+ assert.deepEqual(w.state().changes, []);
266
+ } finally {
267
+ rmSync(dir, { recursive: true, force: true });
268
+ }
269
+ });
package/tsconfig.json ADDED
@@ -0,0 +1,8 @@
1
+ {
2
+ "extends": "../../tsconfig.base.json",
3
+ "compilerOptions": {
4
+ "rootDir": ".",
5
+ "outDir": "dist"
6
+ },
7
+ "include": ["src/**/*.ts", "tests/**/*.ts"]
8
+ }