@openclaw/fs-safe 0.20.0 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +9 -1
  3. package/dist/advanced.d.ts +2 -0
  4. package/dist/advanced.js +1 -0
  5. package/dist/archive-zip-directory.js +4 -0
  6. package/dist/archive-zip-loader.js +3 -1
  7. package/dist/archive-zip-manifest.js +3 -1
  8. package/dist/atomic.d.ts +1 -1
  9. package/dist/copy-publication.d.ts +1 -3
  10. package/dist/copy-publication.js +2 -6
  11. package/dist/deny-mutation-match.d.ts +2 -0
  12. package/dist/deny-mutation-match.js +71 -0
  13. package/dist/deny-mutations.js +4 -3
  14. package/dist/file-identity.js +10 -2
  15. package/dist/file-lock-sync-admission.js +5 -1
  16. package/dist/file-lock-sync-root-io.d.ts +2 -2
  17. package/dist/file-lock-sync-root-io.js +1 -1
  18. package/dist/file-lock-sync.js +2 -2
  19. package/dist/file-store-prune.js +4 -4
  20. package/dist/mutation-authority.js +5 -0
  21. package/dist/native-binding.d.ts +33 -0
  22. package/dist/native-pinned-write.js +8 -2
  23. package/dist/pinned-mutation-admission.js +4 -2
  24. package/dist/replace-file-buffer.d.ts +4 -0
  25. package/dist/replace-file-buffer.js +36 -0
  26. package/dist/replace-file-copy-fallback.d.ts +2 -0
  27. package/dist/replace-file-copy-fallback.js +66 -38
  28. package/dist/replace-file-descriptor.d.ts +4 -0
  29. package/dist/replace-file-descriptor.js +9 -1
  30. package/dist/replace-file-destination.d.ts +21 -0
  31. package/dist/replace-file-destination.js +123 -0
  32. package/dist/replace-file-mutation.d.ts +26 -0
  33. package/dist/replace-file-mutation.js +47 -0
  34. package/dist/replace-file-temp-owner.d.ts +2 -2
  35. package/dist/replace-file-temp-owner.js +16 -4
  36. package/dist/replace-file-types.d.ts +55 -0
  37. package/dist/replace-file-types.js +1 -0
  38. package/dist/replace-file.d.ts +3 -55
  39. package/dist/replace-file.js +31 -11
  40. package/dist/retained-file-types.d.ts +50 -0
  41. package/dist/retained-file-types.js +1 -0
  42. package/dist/retained-file.d.ts +3 -0
  43. package/dist/retained-file.js +121 -0
  44. package/dist/root-directory-entry.d.ts +9 -0
  45. package/dist/root-directory-entry.js +28 -0
  46. package/dist/root-directory-list.d.ts +9 -1
  47. package/dist/root-directory-list.js +88 -37
  48. package/dist/root-handle-context.d.ts +4 -0
  49. package/dist/root-handle-context.js +12 -0
  50. package/dist/root-impl.d.ts +3 -3
  51. package/dist/root-impl.js +8 -2
  52. package/dist/root-move-noreplace.js +3 -3
  53. package/dist/root-walk.d.ts +19 -12
  54. package/dist/root-walk.js +49 -18
  55. package/dist/sidecar-lock-acquire.js +3 -3
  56. package/dist/sidecar-lock-reclaim.d.ts +2 -2
  57. package/dist/sidecar-lock-reclaim.js +6 -6
  58. package/dist/sidecar-lock.js +4 -4
  59. package/dist/staged-symlink-types.d.ts +4 -14
  60. package/dist/temp-target.js +3 -2
  61. package/dist/temp-workspace-admission.js +22 -21
  62. package/dist/temp-workspace-child-admission.d.ts +1 -1
  63. package/dist/temp-workspace-child-admission.js +14 -9
  64. package/dist/temp-workspace-ownership.d.ts +8 -0
  65. package/dist/temp-workspace-ownership.js +52 -0
  66. package/dist/test-hooks.d.ts +4 -0
  67. package/dist/watch-alias.d.ts +6 -0
  68. package/dist/watch-alias.js +88 -0
  69. package/dist/watch-hints.d.ts +9 -0
  70. package/dist/watch-hints.js +93 -0
  71. package/dist/watch-native.d.ts +36 -0
  72. package/dist/watch-native.js +73 -0
  73. package/dist/watch-scan.d.ts +28 -0
  74. package/dist/watch-scan.js +300 -0
  75. package/dist/watch-stream.d.ts +8 -0
  76. package/dist/watch-stream.js +32 -0
  77. package/dist/watch-types.d.ts +60 -0
  78. package/dist/watch-types.js +1 -0
  79. package/dist/watch.d.ts +5 -0
  80. package/dist/watch.js +530 -0
  81. package/docs/advanced.md +1 -0
  82. package/docs/archive.md +6 -0
  83. package/docs/atomic.md +82 -0
  84. package/docs/contributing.md +74 -6
  85. package/docs/durability.md +7 -0
  86. package/docs/index.md +1 -0
  87. package/docs/install.md +2 -0
  88. package/docs/native-helper.md +9 -0
  89. package/docs/native.md +24 -6
  90. package/docs/public-api.md +23 -2
  91. package/docs/retained-file.md +115 -0
  92. package/docs/root.md +25 -8
  93. package/docs/sidecar-lock.md +1 -1
  94. package/docs/staged-symlink.md +2 -1
  95. package/docs/temp.md +24 -4
  96. package/docs/testing.md +186 -4
  97. package/docs/types.md +6 -0
  98. package/docs/walk.md +22 -1
  99. package/docs/watch.md +251 -0
  100. package/package.json +12 -8
package/docs/testing.md CHANGED
@@ -1,5 +1,118 @@
1
1
  # Testing
2
2
 
3
+ ## Coverage gates
4
+
5
+ The coverage workflow measures `src/**/*.ts` with V8 on Linux, macOS, and
6
+ Windows, then merges their counters before enforcing the thresholds in
7
+ `vitest.config.ts`: lines 94%, statements 92%, functions 95%, and branches 89%.
8
+ `pnpm test:coverage:collect` disables per-OS thresholds so platform-only paths
9
+ are credited by their own OS; `pnpm test:coverage:merge` requires all three
10
+ reports and enforces the combined gate. The DACL batch child entrypoint also
11
+ runs in-process in a unit test so its protocol and budget handling are measured.
12
+
13
+ The Linux Rust job uses pinned `cargo-llvm-cov` and a pinned nightly compiler
14
+ for line and branch instrumentation. It combines native crate unit tests with
15
+ TS native suites running the instrumented addon, and exports LCOV plus separate
16
+ unit-only and combined summaries. The Rust line threshold is 71%; addon execution
17
+ must also increase covered Rust lines beyond the unit tests. Only native crate sources compiled on Linux are measured;
18
+ macOS/Windows Rust implementations and the archive WASM crate are outside this
19
+ report. Run `pnpm build` followed by `bash scripts/native-coverage.sh` on Linux
20
+ after installing the toolchain versions specified in `coverage.yml`.
21
+
22
+ Percentages complement behavioral gates: mutation-policy proof, nightly watch
23
+ stress, and platform lanes are equally important. High coverage cannot establish
24
+ root confinement, race safety, event delivery, or bounded resource retirement.
25
+
26
+ ## Watch stress campaign
27
+
28
+ Build from the exact revision being qualified with `pnpm install --frozen-lockfile`,
29
+ `pnpm native:build`, and `pnpm build`, then run on a disposable machine:
30
+
31
+ ```sh
32
+ node scripts/watch-stress.mjs --scenario all
33
+ ```
34
+
35
+ Individual scenario names are `scale`, `fanout`, `churn`, `lifecycle`,
36
+ `adversarial`, `limits`, `idle`, and `soak`. The runner uses plain Node and no
37
+ additional dependencies. Each scenario prints one JSON result line; progress
38
+ goes to stderr. `all` isolates scenarios in child processes and stops at the
39
+ first failure. The full qualification remains manual. A smaller nightly campaign runs outside
40
+ per-PR CI; it can also be dispatched with `watch-stress.yml`.
41
+
42
+ Run `node scripts/watch-stress.mjs --scenario oracle-selftest` first to check
43
+ that a poisoned cache fails comparison and can recover only after invalidation.
44
+ All fixture Roots live in `os.tmpdir()`. The consumer cache refreshes only from
45
+ `onInvalidate`, using guarded Root reads of invalidated paths/scopes. After
46
+ quiescence and a fresh `reconcile()`, checkpoints compare it with an independent
47
+ filesystem walk, including file-content hashes. A mismatch is a failure, with
48
+ no comparison retry or checkpoint-triggered cache refresh. Transient guarded
49
+ read failures retain already-invalidated consumer work and settle at 25 ms
50
+ intervals, with a 120-second flush deadline; these errors are counted in results.
51
+
52
+ Scale uses 50,000 files in 2,000 child directories. Fan-out checks 64 and 256
53
+ distinct Roots, one shared hub thread, and return to the warmed handle baseline.
54
+ Churn first creates 1,024 entries while JavaScript is blocked to exceed the
55
+ default 256-path detail budget, then runs at least five minutes with persistent
56
+ differences across 10,000-mutation batches and isolated edit latency measurements.
57
+ Lifecycle performs 10,000 ready/close cycles plus admission
58
+ cancellation, close-during-ready, 1,000 scope replacements, and callback-close.
59
+ Adversarial cases exercise Root swaps, outside symlinks, recursive deletion and
60
+ 10,000 same-name create/delete pairs. Soak runs 60 minutes, checking the oracle
61
+ each minute. Linux needs passwordless `sudo` for the limits scenario; it lowers
62
+ `fs.inotify.max_user_watches` to 1 in a child shell with a restoration trap and
63
+ verifies restoration. Never run that scenario on a shared production host.
64
+
65
+ RSS limits are fixed before execution: 512 MiB peak for churn/soak, at most
66
+ 64 MiB churn growth after warmup, and at most 32 MiB lifecycle growth after 3,000
67
+ cycles. Soak collects garbage twice at every checkpoint, limits collected-heap
68
+ and external-memory growth to 8 MiB after minute five, and requires the fitted
69
+ RSS slope over minutes 31–60 to stay at or below 1 MiB/minute. The runner launches
70
+ soak with `--expose-gc` automatically, including through `--scenario all`.
71
+ Reports include samples and fitted slopes. Idle runs for ten minutes
72
+ with 16 subscriptions and a one-hour reconciliation interval to isolate native
73
+ hub wakeups, requiring less than 1% of one CPU and, on Linux, at most 30 hub
74
+ context switches. macOS captures `ps -M`; Windows captures PowerShell thread
75
+ CPU time and handle counts. macOS descriptor counts use `lsof`.
76
+
77
+ The macOS limits case also exercises injected UserDropped/KernelDropped flags
78
+ through the native decoder and labels these as synthetic. Natural FSEvents drop
79
+ flags are not independently observable through the current public batch. Windows
80
+ records native overflow and recovery, but the shared batch does not distinguish
81
+ RDCW kernel-buffer loss from bounded native queue loss; a Windows qualification
82
+ must retain that limitation rather than call it proved kernel overflow.
83
+
84
+ ### Nightly workload
85
+
86
+ The nightly workflow runs on `ubuntu-latest` (x64), `ubuntu-24.04-arm`,
87
+ `macos-15` (arm64), `macos-15-intel`, and the Windows latest 16-core runner,
88
+ using Node 24 and freshly built native bindings. Each job has a 25-minute
89
+ budget, uploads one JSON result per scenario (including failures), and fails
90
+ if any scenario fails. Remaining scenarios still run after a failure. Only
91
+ Linux runs the kernel-limits scenario automatically.
92
+
93
+ The same harness accepts these environment overrides; defaults remain the
94
+ full qualification workload. Child processes inherit the settings, which are
95
+ recorded in every JSON result.
96
+
97
+ | Environment variable | Default | Nightly |
98
+ | --- | ---: | ---: |
99
+ | `FS_SAFE_STRESS_SCALE_DIRECTORIES` (25 files each) | 2,000 | 400 (10,000 files) |
100
+ | `FS_SAFE_STRESS_FANOUT` (maximum subscriptions) | 256 | 64 |
101
+ | `FS_SAFE_STRESS_CHURN_SECONDS` | 300 | 60 |
102
+ | `FS_SAFE_STRESS_LIFECYCLE_CYCLES` | 10,000 | 1,000 |
103
+ | `FS_SAFE_STRESS_IDLE_SECONDS` | 600 | 120 |
104
+ | `FS_SAFE_STRESS_SOAK_MINUTES` (minimum 10) | 60 | 10 |
105
+
106
+ Soaks shorter than 30 minutes keep the hard peak-RSS, collected-heap, and external-memory
107
+ growth gates, but report the second-half RSS slope and its limit without using it
108
+ to pass or fail (`memory.rssSlopeGated: false`): [#701](https://github.com/openclaw/fs-safe/pull/701)
109
+ found that V8 capacity expansion and allocator retention can raise RSS while live memory stays flat.
110
+ Runs of 30 minutes or longer enforce the same second-half slope limit; only runs of
111
+ at least 60 minutes report `memory.qualification: true`.
112
+ Lifecycle memory is sampled across ten intervals, with
113
+ the first two excluded as warm-up; idle's Linux wakeup bound scales with the
114
+ requested duration (three per minute). Adversarial cases remain unchanged.
115
+
3
116
  ## Linux openat2 fallback
4
117
 
5
118
  Build the host addon and package first. The test hook is cached with the native
@@ -9,7 +122,7 @@ between tests in one process:
9
122
  ```bash
10
123
  pnpm native:build
11
124
  pnpm build
12
- FS_SAFE_TEST_NO_OPENAT2=1 FS_SAFE_NATIVE_MODE=require pnpm test test/linux-openat2-fallback.test.ts test/root-move-noreplace.test.ts test/root-move-native-integration.test.ts test/native-write-containment.test.ts
125
+ FS_SAFE_TEST_NO_OPENAT2=1 FS_SAFE_NATIVE_MODE=require pnpm test test/linux-openat2-parity.test.ts test/linux-openat2-fallback.test.ts test/root-move-noreplace.test.ts test/root-move-native-integration.test.ts test/native-write-containment.test.ts
13
126
  ```
14
127
 
15
128
  On Linux, the seccomp harness also exercises the real syscall failure without
@@ -20,13 +133,21 @@ unprivileged seccomp filter; it affects only its child process:
20
133
  cc test/fixtures/deny-openat2.c -o /tmp/fs-safe-deny-openat2
21
134
  /tmp/fs-safe-deny-openat2 ENOSYS node test/fixtures/linux-openat2-fallback.mjs "$PWD/native/fs-safe-native.linux-x64-gnu.node"
22
135
  /tmp/fs-safe-deny-openat2 EPERM node test/fixtures/linux-openat2-fallback.mjs "$PWD/native/fs-safe-native.linux-x64-gnu.node"
136
+ FS_SAFE_TEST_OPENAT2_FILTER=/tmp/fs-safe-deny-openat2 FS_SAFE_NATIVE_MODE=require pnpm test test/linux-openat2-parity.test.ts
23
137
  ```
24
138
 
25
139
  Use the matching native artifact filename on other Linux architectures/libcs.
26
- The fixture proves nested moves, collisions, read/write, traversal, symlink and
27
- hardlink rejection, cached selection, and `helper-unavailable` for strict
140
+ The fixtures prove in-root alias operations and policy parity, nested moves,
141
+ collisions, read/write, traversal and escaping-link rejection, hardlink rejection,
142
+ cached selection, and `helper-unavailable` for strict
28
143
  bounded cleanup. Bounded-cleanup success tests require real `openat2`; run the
29
- full suite with the environment hook unset.
144
+ full suite with the environment hook unset. PR CI's `Native check
145
+ (linux-x64-no-openat2)` runs the native Node suites and watch proofs with the
146
+ hook set. It replaces the four bounded-cleanup success suites and quarantine
147
+ success proof with the explicit refusal fixture above. XFS tree-clone proof
148
+ requires the same openat2/NO_XDEV primitive and runs in the ordinary Linux
149
+ lanes. Bun's full compatibility
150
+ suite remains in the normal native lanes, because it includes bounded cleanup.
30
151
 
31
152
  `@openclaw/fs-safe/test-hooks` exposes test-only injection points. Registration
32
153
  is allowed only when `process.env.NODE_ENV === "test"` or
@@ -186,6 +307,14 @@ For tests that need a private temp workspace, [`withTempWorkspace`](temp.md) mak
186
307
 
187
308
  ## Repo test shards
188
309
 
310
+ On macOS, after building the native addon, run the native watch cleanup allocation
311
+ regression with `MallocStackLogging=1 node --expose-gc scripts/watch-cleanup-leak-proof.mjs`.
312
+ It compares `leaks` results before and after 100 and 1,000 subscription cycles,
313
+ requiring zero growth in leaked allocations. Allocation stacks are saved under
314
+ `.artifacts/watch-cleanup-leaks`. The native macOS CI lane runs this short proof;
315
+ the full stress qualification remains manual, with the smaller nightly
316
+ campaign above.
317
+
189
318
  Run the full local gate before handoff:
190
319
 
191
320
  ```sh
@@ -210,6 +339,59 @@ workspace reads that bypass pinned file descriptors.
210
339
 
211
340
  `pnpm check` also runs `pnpm lint:file-size`. New source and test files should stay under 500 lines. Existing larger files have explicit budgets in `scripts/check-file-size.mjs`; do not increase those budgets as part of unrelated work.
212
341
 
342
+ ## Watch memory diagnosis
343
+
344
+ The original soak rule rejected more than 64 MiB RSS growth after minute five.
345
+ That outcome remains in `memory.legacyRss`, with its original limit and pass/fail
346
+ value; it is no longer the soak pass criterion. Static guarded poll scans alone
347
+ reproduced growth from 72 to 181 MiB RSS while collected heap stayed near 7–8 MiB:
348
+ V8 expanded its almost-empty young generation to 128 MiB, with 103 MiB physically
349
+ committed. A diagnostic run limiting that space ended at 85 MiB RSS with the
350
+ same live heap. The normal harness keeps Node's default nursery sizing.
351
+
352
+ The 60-minute qualification separates this capacity warm-up from the later RSS
353
+ trend. The measured Linux event run had under 1 MiB collected-heap drift and a
354
+ 0.55 MiB/minute second-half RSS slope; its final ten-minute RSS range was 1.60 MiB.
355
+ The 8 MiB live-memory allowance and 1 MiB/minute RSS slope leave measurement
356
+ margin while rejecting retained growth and continued rapid RSS growth. The
357
+ 512 MiB peak ceiling is unchanged. Native allocation leaks need independent
358
+ accounting/profiling too: the investigation found a much smaller cleanup-hook
359
+ context leak even when registrations, pending sets, and TSFN counters retired.
360
+
361
+ Build the native addon and package from the same revision, then run each control
362
+ in a fresh Node process on a disposable machine:
363
+
364
+ ```sh
365
+ node --expose-gc scripts/watch-memory.mjs --arm events --minutes 60 --output .artifacts/events
366
+ node --expose-gc scripts/watch-memory.mjs --arm none --minutes 30 --output .artifacts/none
367
+ node --expose-gc scripts/watch-memory.mjs --arm poll --minutes 30 --output .artifacts/poll
368
+ node --expose-gc scripts/watch-memory.mjs --arm lifecycle --minutes 30 --output .artifacts/lifecycle
369
+ node --expose-gc scripts/watch-memory.mjs --arm steady --minutes 30 --output .artifacts/steady
370
+ ```
371
+
372
+ `events` and `poll` combine low-rate edits, a 10,000-operation burst every fifth
373
+ minute, and subscription cycling. `steady` omits cycling; `lifecycle` omits
374
+ writes. `none` drives the same writer and consumer cache using explicit synthetic
375
+ invalidations, without constructing subscriptions. That arm is an allocation
376
+ control, not evidence of watcher correctness. Every arm checks its consumer
377
+ cache against an independent filesystem walk at each checkpoint.
378
+
379
+ The diagnostic runner emits JSONL to stdout and `memory.jsonl` in its output
380
+ directory. Each minute includes all five `process.memoryUsage()` fields before
381
+ and after collection, V8 heap-space capacity, Linux `smaps_rollup`, operation and
382
+ guarded-read counts, and live/created/destroyed native watch allocations.
383
+ `--gc none` measures the same workload without forced collection. `--snapshots`
384
+ writes V8 snapshots at minutes 5 and 30; use a separate run because snapshots
385
+ perturb memory usage. On macOS, `--tools` saves `vmmap --summary` and `leaks`
386
+ reports at minutes 5, 30, and 60. `--smoke` is a short calibration, explicitly
387
+ marked in output; it is not a duration-qualified soak.
388
+
389
+ The native allocation getter is internal and requires `NODE_ENV=test` or
390
+ `VITEST=true`; the runner sets the former. After close, it requires no live
391
+ registrations, pending sets, callback payloads, or thread-safe functions.
392
+ Diagnostic success establishes correctness and retirement; it does not classify
393
+ an RSS curve as a leak or automatically approve a memory budget.
394
+
213
395
  ## See also
214
396
 
215
397
  - [Security model](security-model.md) — what the boundary is supposed to defend; design tests around the same threats.
package/docs/types.md CHANGED
@@ -4,6 +4,12 @@ The types most callers reach for. Shared data shapes are exported from `@opencla
4
4
 
5
5
  For atomic replacement, `ReplaceFileAtomicFileSystem` and `ReplaceFileAtomicSyncFileSystem` are exported from `@openclaw/fs-safe/atomic`. Async adapters use `chmod()` on the `FileHandle` returned by their required `open()` operation. The synchronous type adds optional `fchmodSync(fd, mode)`; custom sync adapters that explicitly request `mode` or `preserveExistingMode` must implement it. See [Atomic writes](atomic.md#test-injection).
6
6
 
7
+ `ReplaceFileAtomicDestinationState` is also exported from that subpath. It is a
8
+ readonly union of `{ state: "removed", path }` and
9
+ `{ state: "writing" | "published", path, dev: bigint, ino: bigint }`. Both atomic
10
+ replacement option types accept synchronous `assertBeforeMutation` and
11
+ `onDestinationState` callbacks. See [authority and destination state](atomic.md#atomic-write-authority-and-destination-state).
12
+
7
13
  ```ts
8
14
  import type {
9
15
  BasePathOptions,
package/docs/walk.md CHANGED
@@ -107,11 +107,32 @@ Unreadable directories are skipped rather than throwing, but every skipped direc
107
107
  `Root.walk(rel, options)` is the root-bounded counterpart to these standalone
108
108
  inventory helpers. It yields `{ relativePath, kind, size }` incrementally and
109
109
  accepts `maxDepth`, `maxEntries`, `symlinkPolicy: "skip" |
110
- "follow-within-root"`, `order: "sorted" | "filesystem"`, and an `AbortSignal`. The default budget behavior yields
110
+ "follow-within-root" | "include"`, `order: "sorted" | "filesystem"`, and an `AbortSignal`. The default budget behavior yields
111
111
  one `kind: "truncated"` marker and ends; pass `limitBehavior: "throw"` for a
112
112
  typed `FsSafeError("too-large")` instead.
113
113
 
114
114
  For followed symlinks, both `kind` and `size` describe the resolved target.
115
+ With `symlinkPolicy: "include"`, links retain `kind: "symlink"` and their own
116
+ size. Targets are neither resolved nor visited; dangling and outside-root links
117
+ are included. Filters receive these entries, and links consume the same entry
118
+ budget as other names. A directory replaced by a symlink after observation
119
+ fails with `path-mismatch` before descent, or produces a `directory-error`
120
+ entry when `onDirectoryError` is `"skip-and-report"`.
121
+ The starting directory retains existing Root path resolution; include mode
122
+ controls the entries beneath that directory.
123
+
124
+ ```ts
125
+ for await (const entry of capability.walk("", { symlinkPolicy: "include" })) {
126
+ if (entry.kind === "symlink") reportLink(entry.relativePath);
127
+ }
128
+ ```
129
+
130
+ Existing skip/follow calls keep their result types without a symlink variant.
131
+ For explicitly annotated include-mode values, use `RootWalkOptions<"include">`
132
+ and `RootWalkEntry<"include">`. `RootWalkSymlinkPolicy` describes all
133
+ three policies when the policy is selected dynamically; the unparameterized
134
+ entry and options types retain their previous shapes. Use
135
+ `RootWalkOptions<RootWalkSymlinkPolicy>` for a dynamically selected policy.
115
136
 
116
137
  The caller's starting path retains Root home shorthand: `~` and `~/dir` expand
117
138
  the home directory when iteration starts and must resolve inside the Root.
package/docs/watch.md ADDED
@@ -0,0 +1,251 @@
1
+ # Guarded filesystem observation
2
+
3
+ `@openclaw/fs-safe/watch` observes literal paths under an admitted `Root`.
4
+ Notifications are advisory invalidations, not a transaction log, stable content,
5
+ or authority to read a reported pathname. Guarded metadata scans through the
6
+ original Root determine what may be published. Use the Root again to read data.
7
+
8
+ ```ts
9
+ import { root } from "@openclaw/fs-safe/root";
10
+ import { watch } from "@openclaw/fs-safe/watch";
11
+
12
+ const workspace = await root("/trusted/workspace");
13
+ const subscription = watch(workspace, {
14
+ mode: "auto",
15
+ scopes: [
16
+ { path: "config.json", kind: "entry" },
17
+ { path: "skills", kind: "tree", depth: 8 },
18
+ ],
19
+ exclude: entry => entry.kind === "directory" && entry.path.endsWith("node_modules"),
20
+ onInvalidate(invalidation) {
21
+ // Schedule application-owned settling/reload work; this callback is synchronous.
22
+ console.log(invalidation.reason, invalidation.changes);
23
+ },
24
+ });
25
+ await subscription.ready;
26
+ await subscription.setScopes([{ path: "skills", kind: "tree" }]);
27
+ await subscription.reconcile();
28
+ await subscription.close();
29
+ ```
30
+
31
+ ## Contract
32
+
33
+ The subpath exports `watch` and the types `WatchScope`, `WatchEntry`,
34
+ `WatchChange`, `WatchInvalidation`, `WatchFailure`, `WatchHealth`, `WatchOptions`,
35
+ and `WatchSubscription`.
36
+
37
+ An `entry` scope observes only that entry, including its identity and metadata.
38
+ A `tree` also observes descendants to its depth (default 32, maximum 128).
39
+ Depth zero observes only the entry. The empty string and `.` select the Root;
40
+ trailing separators are accepted and normalized after validation. Scopes are
41
+ literal names, never globs or home expansion. Absolute paths, traversal, NULs,
42
+ and platform namespace aliases reject. Windows scopes also reject reserved
43
+ device components and trailing dots/spaces that Win32 would silently alias. Filesystem identity determines ordinary
44
+ case/Unicode aliases; names are not compared by lowercasing.
45
+
46
+ Symlinks are observed as entries and never followed. A symbolic parent fails
47
+ admission: separately admit a caller-trusted link target if needed. Missing
48
+ entries and ordinary blocking files can become directories in later scans.
49
+ Replacing, renaming, or losing the admitted Root fails observation; it never
50
+ silently adopts a new Root. Directory entry scopes ignore child-only mtime/size
51
+ changes, but include permission-mode changes to the directory itself.
52
+
53
+ Each invalidation has `reason: "event" | "reconcile" | "overflow"` and optional
54
+ bounded `changes: { path, type: "content" | "structural" }[]`. Missing detail
55
+ means invalidate **every configured scope**. Initial admission and every
56
+ successful `setScopes` publish one undetailed `reconcile` invalidation. A rename
57
+ or identity replacement is structural; metadata changes to the same ordinary
58
+ file may be content changes. Neither means the file is settled or readable.
59
+ An admitted native hint can invalidate an entry even when its before/after scan
60
+ metadata is identical: the path may have changed and been restored between scans
61
+ (ABA), or content may have changed without a distinguishable metadata change.
62
+ Structural hints remain conservative in that case.
63
+ This does not promise delivery for a differently spelled alias that appears and
64
+ disappears entirely between scans: without an observed identity, it cannot be
65
+ admitted as the selected path. Observation is not a complete transient history.
66
+ Raw event names remain private: detail comes from guarded scans, prior guarded
67
+ snapshots, or explicitly configured targets. Unclassifiable hints inside selected,
68
+ non-excluded territory lose detail. Hints for excluded or unselected paths are
69
+ ignored. Exclusion callbacks are synchronous; excluded directories are recorded
70
+ without descent. Bounded exclusion records recognize late deletion hints.
71
+ Genuine backend event loss and detail-budget exhaustion still invalidate every
72
+ scope, including when an excluded subtree caused the underlying event pressure.
73
+
74
+ ## Transport and mode
75
+
76
+ | Platform/runtime | `auto` | Event transport / limitation |
77
+ | --- | --- | --- |
78
+ | Node.js on Linux with addon | `events` | One shared Rust thread and inotify instance; a nonrecursive watch per distinct directory inode. |
79
+ | Node.js on macOS with addon | `events` | One FSEvents stream per subscription on a shared serial dispatch queue. Pathname activity after a swap remains advisory. |
80
+ | Node.js on Windows with addon | `events` | One recursive ReadDirectoryChangesW Root handle per subscription on the shared IOCP hub; the open handle prevents ordinary renames of the Root's ancestors. |
81
+ | Bun / other unsupported runtimes | `poll` | TSFN lifetime and shutdown have not been qualified; `events` rejects. |
82
+ | Missing/disabled addon | `poll` | `events` rejects with `FsSafeError("helper-unavailable")`. |
83
+
84
+ The shared hub sleeps until a filesystem event, command, or callback acknowledgement:
85
+ Linux blocks on inotify plus eventfd, Windows on IOCP, and macOS on its command
86
+ channel (FSEvents wakes it from the serial dispatch queue). There is no native
87
+ polling timer; the independent JS reconciliation interval remains authoritative.
88
+
89
+ `mode` is required. `poll` never starts or loads the watch hub; guarded scans
90
+ may still use the existing addon. Existing `FS_SAFE_NATIVE_MODE=off` and
91
+ `require` policies apply: `require` plus an unavailable event backend rejects
92
+ `auto` too. Health reports the selected `events` or `poll` mode and failures
93
+ from selection have `operation: "watch"`.
94
+
95
+ Linux installs each watch **before** enumerating children. It opens directories
96
+ beneath the Root using the shared guarded native open (openat2, or its checked
97
+ openat fallback), checks exact identities, verifies the procfs namespace, and
98
+ registers through `/proc/self/fd/N/.`. The final `/.` makes `IN_DONT_FOLLOW` apply
99
+ to the directory instead of rejecting the procfs magic symlink. The descriptor
100
+ closes immediately: inotify retains the inode reference. Reconciliation replaces
101
+ registrations when the directory inventory changes. Queue overflow invalidates
102
+ all owners; exhausted watch capacity reports `failure.code: "watch-limit"`.
103
+ Nonblocking TSFN batches cannot block the hub on JavaScript, and per-owner
104
+ pending detail and queued batches are bounded. The last removal stops and joins
105
+ the native thread. No Worker threads, eval programs, or JS `fs.watch` are used.
106
+
107
+ macOS uses FileEvents, NoDefer and WatchRoot with a 30 ms FSEvents latency.
108
+ After guarded admission, streams use selected tree anchors and entry parents,
109
+ falling back to the nearest admitted ancestor for missing paths. Nested anchors
110
+ are deduplicated, with at most 128 paths. The eight shallowest non-overlapping
111
+ excluded directories are also passed to `FSEventStreamSetExclusionPaths`.
112
+ When the stream paths change, the old stream is stopped, invalidated and released
113
+ on its dispatch queue, then its replacement starts before another guarded pass
114
+ covers the handover. Native exclusions reduce traffic but cannot eliminate real
115
+ FSEvents drops, including during recursive deletion.
116
+ Absolute hints are reduced lexically against the admitted canonical Root;
117
+ outside paths never become detail. Dropped/wrapped streams, RootChanged and
118
+ Unmount trigger guarded reconciliation. Pathname hints can reflect activity
119
+ after a swap, but the Root is never replaced and names require guarded admission.
120
+ Removal synchronously stops, invalidates and releases the stream on its queue.
121
+
122
+ Windows opens one identity-checked Root handle per subscription with
123
+ READ/WRITE/DELETE sharing, backup semantics and overlapped I/O. Each handle
124
+ observes the entire subtree; guarded scans filter hints to configured scopes.
125
+ No descendant watch handles are retained, so directories inside the Root can be
126
+ renamed while watching, including directories containing selected scopes.
127
+ Completed buffers are copied and the
128
+ read re-armed before names are examined. Zero-byte / enumeration-loss completions
129
+ invalidate every scope. Buffers are 1 MiB on confirmed local volumes, and 64 KiB
130
+ on network/UNC or unclassified volumes. Cancellation waits for IOCP completion before closing
131
+ the handle or freeing its buffer; there are no detached retirement waits.
132
+
133
+ On Windows, an open file handle—including fs-safe's pinned reads, editors, and
134
+ antivirus—blocks renaming that file's ancestor directories, even with delete
135
+ sharing. The watch backend retains only the Root's `ReadDirectoryChangesW` handle,
136
+ which permits renames beneath the Root. Callers renaming directories concurrently
137
+ with reads should use bounded retries, as Windows tools do.
138
+
139
+ Windows prevents ordinary renames of the Root's own ancestors while its directory
140
+ handle is open, even with DELETE sharing. Use `mode: "poll"` when callers must not
141
+ retain that handle. Renaming or replacing the Root still fails guarded observation;
142
+ the subscription never adopts another location. Subscriptions have independent
143
+ handles and delivery queues, and closing one does not retire another's observation.
144
+
145
+ ## Budgets and lifecycle
146
+
147
+ | Option | Default / bound |
148
+ | --- | --- |
149
+ | `scopes` | At most 128 literal scopes |
150
+ | `persistent` | `true`; `false` lets Node exit with the subscription still open |
151
+ | `intervalMs` | 30000 with events; 1000 with poll; minimum 20 ms |
152
+ | `pollIntervalMs` | Optional polling override; minimum 20 ms, maximum 2147483647 ms (same as `intervalMs`) |
153
+ | `maxDirectories` | 4096 observed directories, including scope ancestors |
154
+ | `maxEntries` | 100000 examined entries per pass, including excluded entries |
155
+ | `maxPendingPaths` | 256; maximum 4096 |
156
+ | Reconciliation | One active pass and one coalesced pending pass; no convergence/pass budget |
157
+
158
+ Periodic guarded reconciliation runs without needing an event. It catches
159
+ missed events and works on filesystems where native hints are incomplete.
160
+ When polling is selected, the interval is `pollIntervalMs`, then `intervalMs`,
161
+ then 1000 ms, in that order. This applies to explicit `mode: "poll"`, `auto`
162
+ selecting polling, and `auto` falling back after an unsupported event backend.
163
+ `pollIntervalMs` does not change the events reconciliation interval, which
164
+ remains `intervalMs` or 30000 ms. For example, `mode: "auto", pollIntervalMs: 25`
165
+ uses 25 ms polling when needed and retains the 30-second events reconciliation.
166
+ Scans are metadata comparisons: content changes preserving all compared
167
+ metadata may be missed in polling mode. No mode promises transactional
168
+ snapshots, complete history, or hard real-time delivery.
169
+
170
+ `ready` resolves after the first complete guarded scan establishes the baseline,
171
+ even while writes continue. Events mode installs each directory registration
172
+ before listing it (FSEvents and recursive RDCW anchors cover the crawl). A changed
173
+ registration/listing identity is retried up to three times per directory; further
174
+ churn invalidates that subtree. Poll mode starts with its first scan and detects
175
+ changes during the crawl on the next comparison. Neither mode waits for two
176
+ agreeing scans.
177
+ `ready` and `reconcile()` do not drain the operating system's event queue.
178
+ For example, FSEvents can deliver coalesced setup creation activity after `ready`,
179
+ even with its stream starting at the current event ID. Consumers must tolerate
180
+ these advisory invalidations; tests measuring a quiet interval should first
181
+ observe a selected sentinel edit and drain its trailing events.
182
+
183
+ Each later pass compares with the previous snapshot, publishes bounded differences,
184
+ and adopts its result as the next snapshot. Vanishing entries, kind changes, and
185
+ transient descendant scan errors produce structural invalidations, preserving the
186
+ Root identity checks. Events during a pass coalesce into one pending pass and
187
+ retain bounded detail regardless of how long the scan takes. The 25 ms hint
188
+ coalescing window does not impose a scan deadline. A full native callback queue
189
+ retains its bounded pending batch for retry. Genuine backend loss, exhausted
190
+ detail capacity, or an unclassifiable selected hint emits `overflow` without
191
+ detail. Sustained writes cannot exhaust a pass budget or disable observation.
192
+
193
+ `reconcile()` resolves after a complete pass that **started after the call**. Calls
194
+ waiting for the same future pass coalesce; an earlier in-flight pass cannot satisfy
195
+ a new call. It rejects only when observation becomes unavailable or is closed.
196
+ `setScopes` fences the old generation immediately and resolves after the new
197
+ baseline scan; superseded scope calls reject `AbortError`.
198
+ By default, an open subscription keeps the Node event loop alive, matching
199
+ `fs.watch`. Set `persistent: false` for caches used by one-shot commands:
200
+ the subscription's timers and native delivery handle do not keep Node alive,
201
+ including during startup or reconciliation. Invalidations still arrive while
202
+ other work keeps the process alive. Persistent and non-persistent subscriptions
203
+ have independent lifetimes; closing the last persistent one lets Node exit.
204
+ Native environment cleanup retires any remaining event registrations and joins
205
+ the hub at exit. `signal` triggers close;
206
+ await `close()` or `[Symbol.asyncDispose]()` to join owned work.
207
+
208
+ `health()` returns `starting`, `ready`, `reconciling`, `unavailable`, or `closed`,
209
+ the actual mode, observed `directories`, and optional `{ operation, code, error }`
210
+ failure. A running reconciliation reports `reconciling`, then returns to `ready`.
211
+ Observation becomes `unavailable` on loss of Root authority (removed, replaced,
212
+ or inaccessible), fatal native backend/registration failures such as `watch-limit`,
213
+ deterministic size limits (`too-large`, operation `scan`), or callback contract
214
+ violations. Invalid scope admission, including symbolic parents, still rejects;
215
+ it never grants authority through a link. Transient descendant churn does not
216
+ make an admitted subscription unavailable. Callbacks may synchronously retire
217
+ the owner. Returning a thenable or a synchronous or asynchronous generator object
218
+ from `onInvalidate`, `onHealth`, or `exclude` rejects observation without advancing
219
+ generators. Application async
220
+ work remains application-owned and is not joined by the subscription.
221
+
222
+ `close()` is terminal, idempotent, and joined. Observation failures remain in
223
+ health but do not make successful retirement reject. Retirement failures do
224
+ reject, with a `SuppressedError` retaining an earlier observation failure when
225
+ both exist. No new generation or callback can be admitted after close.
226
+
227
+ The stress harness also provides `node scripts/watch-stress.mjs --scenario soak-short`,
228
+ a two-minute lifecycle and collected-memory smoke. It is separate from the
229
+ full one-hour `soak` scenario and its stronger memory-growth qualification.
230
+
231
+ ## Seeded consumer-cache stress test
232
+
233
+ After building the package and host binding, run the model against real temporary
234
+ Roots in both modes:
235
+
236
+ ```sh
237
+ node scripts/watch-stress/model-runner.mjs --seeds 2000 --mode both --output watch-model-results.json
238
+ ```
239
+
240
+ Each seed generates file and directory edits, renames, replacements, deep trees,
241
+ symlink retargets, bursts, scope changes, and subscription retirement. A consumer
242
+ queues refreshes only from `onInvalidate`; checkpoints drain those requests after
243
+ quiescence and `reconcile()`, then compare its cache with a guarded Root walk and
244
+ an independent mutation model. Native hints never authorize consumer reads.
245
+
246
+ `--seed`, `--steps`, `--settle` (milliseconds), and `--concurrency` control a run.
247
+ Failures are shrunk by fast-check and saved beside the report as `.<mode>-<seed>.failure.json`;
248
+ replay one with `--replay <failure.json>`. Event mode requires a working native
249
+ binding and never silently falls back to polling. The small
250
+ `test/watch-model.test.ts` corpus runs in ordinary CI; native-event cases also run
251
+ when `FS_SAFE_TEST_WATCH_EVENTS=1`. Keep fixtures on normal `os.tmpdir()` storage.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@openclaw/fs-safe",
3
- "version": "0.20.0",
3
+ "version": "0.21.1",
4
4
  "description": "Capability-style filesystem roots for Node.js apps that handle untrusted relative paths.",
5
5
  "keywords": [
6
6
  "filesystem",
@@ -95,6 +95,10 @@
95
95
  "types": "./dist/file-lock.d.ts",
96
96
  "default": "./dist/file-lock.js"
97
97
  },
98
+ "./watch": {
99
+ "types": "./dist/watch.d.ts",
100
+ "default": "./dist/watch.js"
101
+ },
98
102
  "./walk": {
99
103
  "types": "./dist/walk.d.ts",
100
104
  "default": "./dist/walk.js"
@@ -169,13 +173,13 @@
169
173
  "archive:producer-smoke": "node scripts/archive-producer-smoke.mjs"
170
174
  },
171
175
  "optionalDependencies": {
172
- "@openclaw/fs-safe-darwin-arm64": "0.20.0",
173
- "@openclaw/fs-safe-darwin-x64": "0.20.0",
174
- "@openclaw/fs-safe-linux-arm64-gnu": "0.20.0",
175
- "@openclaw/fs-safe-linux-arm64-musl": "0.20.0",
176
- "@openclaw/fs-safe-linux-x64-gnu": "0.20.0",
177
- "@openclaw/fs-safe-linux-x64-musl": "0.20.0",
178
- "@openclaw/fs-safe-win32-x64-msvc": "0.20.0",
176
+ "@openclaw/fs-safe-darwin-arm64": "0.21.1",
177
+ "@openclaw/fs-safe-darwin-x64": "0.21.1",
178
+ "@openclaw/fs-safe-linux-arm64-gnu": "0.21.1",
179
+ "@openclaw/fs-safe-linux-arm64-musl": "0.21.1",
180
+ "@openclaw/fs-safe-linux-x64-gnu": "0.21.1",
181
+ "@openclaw/fs-safe-linux-x64-musl": "0.21.1",
182
+ "@openclaw/fs-safe-win32-x64-msvc": "0.21.1",
179
183
  "jszip": "^3.10.2"
180
184
  },
181
185
  "devDependencies": {