@tiphys/kernel 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +663 -0
- package/LICENSE +202 -0
- package/assurance-modes.yaml +299 -0
- package/checklists/clean-room.yaml +325 -0
- package/checklists/env-failure-diagnosis.yaml +68 -0
- package/checklists/flake-playbook.yaml +68 -0
- package/checklists/hazard-review.yaml +144 -0
- package/checklists/plan-review.yaml +103 -0
- package/dist/bin/tiphys.d.ts +2 -0
- package/dist/bin/tiphys.js +92 -0
- package/dist/src/adapters/load.d.ts +202 -0
- package/dist/src/adapters/load.js +440 -0
- package/dist/src/brief.d.ts +32 -0
- package/dist/src/brief.js +66 -0
- package/dist/src/checklists.d.ts +169 -0
- package/dist/src/checklists.js +310 -0
- package/dist/src/checks.d.ts +1539 -0
- package/dist/src/checks.js +5025 -0
- package/dist/src/cli.d.ts +9 -0
- package/dist/src/cli.js +68 -0
- package/dist/src/commands/brief.d.ts +92 -0
- package/dist/src/commands/brief.js +343 -0
- package/dist/src/commands/checklist.d.ts +42 -0
- package/dist/src/commands/checklist.js +168 -0
- package/dist/src/commands/cutover.d.ts +35 -0
- package/dist/src/commands/cutover.js +448 -0
- package/dist/src/commands/doctor.d.ts +264 -0
- package/dist/src/commands/doctor.js +1546 -0
- package/dist/src/commands/gates.d.ts +9 -0
- package/dist/src/commands/gates.js +360 -0
- package/dist/src/commands/init.d.ts +30 -0
- package/dist/src/commands/init.js +177 -0
- package/dist/src/commands/lock.d.ts +34 -0
- package/dist/src/commands/lock.js +340 -0
- package/dist/src/commands/mode.d.ts +40 -0
- package/dist/src/commands/mode.js +134 -0
- package/dist/src/commands/next.d.ts +130 -0
- package/dist/src/commands/next.js +597 -0
- package/dist/src/commands/plan.d.ts +20 -0
- package/dist/src/commands/plan.js +105 -0
- package/dist/src/commands/pool.d.ts +1 -0
- package/dist/src/commands/pool.js +139 -0
- package/dist/src/commands/resume.d.ts +1 -0
- package/dist/src/commands/resume.js +88 -0
- package/dist/src/commands/spawn.d.ts +1 -0
- package/dist/src/commands/spawn.js +195 -0
- package/dist/src/commands/status.d.ts +21 -0
- package/dist/src/commands/status.js +122 -0
- package/dist/src/commands/sync.d.ts +47 -0
- package/dist/src/commands/sync.js +341 -0
- package/dist/src/commands/teardown.d.ts +1 -0
- package/dist/src/commands/teardown.js +87 -0
- package/dist/src/commands/tuition.d.ts +21 -0
- package/dist/src/commands/tuition.js +218 -0
- package/dist/src/commands/validate.d.ts +78 -0
- package/dist/src/commands/validate.js +430 -0
- package/dist/src/commands/watch.d.ts +1 -0
- package/dist/src/commands/watch.js +172 -0
- package/dist/src/cutover.d.ts +584 -0
- package/dist/src/cutover.js +1444 -0
- package/dist/src/exclusion.d.ts +389 -0
- package/dist/src/exclusion.js +843 -0
- package/dist/src/exec/env.d.ts +278 -0
- package/dist/src/exec/env.js +334 -0
- package/dist/src/fleet.d.ts +223 -0
- package/dist/src/fleet.js +298 -0
- package/dist/src/gates/adapters/http-json.d.ts +5 -0
- package/dist/src/gates/adapters/http-json.js +283 -0
- package/dist/src/gates/adapters/migrations-command.d.ts +1 -0
- package/dist/src/gates/adapters/migrations-command.js +373 -0
- package/dist/src/gates/citations.d.ts +408 -0
- package/dist/src/gates/citations.js +1169 -0
- package/dist/src/gates/coverage.d.ts +375 -0
- package/dist/src/gates/coverage.js +836 -0
- package/dist/src/gates/credentials.d.ts +233 -0
- package/dist/src/gates/credentials.js +752 -0
- package/dist/src/gates/deploy.d.ts +1 -0
- package/dist/src/gates/deploy.js +33 -0
- package/dist/src/gates/gate-classes.d.ts +56 -0
- package/dist/src/gates/gate-classes.js +633 -0
- package/dist/src/gates/manifest.d.ts +99 -0
- package/dist/src/gates/manifest.js +208 -0
- package/dist/src/gates/merge-preconditions.d.ts +319 -0
- package/dist/src/gates/merge-preconditions.js +932 -0
- package/dist/src/gates/migrations.d.ts +1 -0
- package/dist/src/gates/migrations.js +36 -0
- package/dist/src/gates/pin.d.ts +114 -0
- package/dist/src/gates/pin.js +154 -0
- package/dist/src/gates/red-witness.d.ts +22 -0
- package/dist/src/gates/red-witness.js +482 -0
- package/dist/src/gates/release.d.ts +283 -0
- package/dist/src/gates/release.js +820 -0
- package/dist/src/gates/result.d.ts +116 -0
- package/dist/src/gates/result.js +91 -0
- package/dist/src/gates/run.d.ts +614 -0
- package/dist/src/gates/run.js +1614 -0
- package/dist/src/gates/schemas/citation-config.schema.json +59 -0
- package/dist/src/gates/schemas/coverage-config.schema.json +77 -0
- package/dist/src/gates/schemas/gate-manifest.schema.json +125 -0
- package/dist/src/gates/schemas/gate-result.schema.json +160 -0
- package/dist/src/gates/schemas/phase-declaration.schema.json +87 -0
- package/dist/src/gates/schemas/release-record.schema.json +119 -0
- package/dist/src/gates/schemas/verifier-config.schema.json +101 -0
- package/dist/src/gates/schemas/witness-spec.schema.json +110 -0
- package/dist/src/gates/scope.d.ts +131 -0
- package/dist/src/gates/scope.js +1018 -0
- package/dist/src/gates/suite.d.ts +217 -0
- package/dist/src/gates/suite.js +968 -0
- package/dist/src/gates/validate.d.ts +121 -0
- package/dist/src/gates/validate.js +414 -0
- package/dist/src/hooks.d.ts +84 -0
- package/dist/src/hooks.js +125 -0
- package/dist/src/index.d.ts +31 -0
- package/dist/src/index.js +30 -0
- package/dist/src/liveness.d.ts +321 -0
- package/dist/src/liveness.js +396 -0
- package/dist/src/lock.d.ts +256 -0
- package/dist/src/lock.js +792 -0
- package/dist/src/model-resolution.d.ts +159 -0
- package/dist/src/model-resolution.js +307 -0
- package/dist/src/modes.d.ts +149 -0
- package/dist/src/modes.js +258 -0
- package/dist/src/path-identity.d.ts +34 -0
- package/dist/src/path-identity.js +48 -0
- package/dist/src/plan.d.ts +73 -0
- package/dist/src/plan.js +153 -0
- package/dist/src/pool.d.ts +326 -0
- package/dist/src/pool.js +988 -0
- package/dist/src/roles.d.ts +461 -0
- package/dist/src/roles.js +776 -0
- package/dist/src/spawn.d.ts +482 -0
- package/dist/src/spawn.js +1003 -0
- package/dist/src/status.d.ts +116 -0
- package/dist/src/status.js +148 -0
- package/dist/src/task.d.ts +504 -0
- package/dist/src/task.js +307 -0
- package/dist/src/teardown.d.ts +39 -0
- package/dist/src/teardown.js +422 -0
- package/dist/src/tuition.d.ts +159 -0
- package/dist/src/tuition.js +311 -0
- package/dist/src/validate.d.ts +263 -0
- package/dist/src/validate.js +742 -0
- package/dist/src/version.d.ts +3 -0
- package/dist/src/version.js +38 -0
- package/dist/src/watcher.d.ts +275 -0
- package/dist/src/watcher.js +849 -0
- package/dist/src/witness/run.d.ts +299 -0
- package/dist/src/witness/run.js +1373 -0
- package/dist/src/witness/spec.d.ts +270 -0
- package/dist/src/witness/spec.js +475 -0
- package/dist/tsconfig.src.tsbuildinfo +1 -0
- package/gate-registry.yaml +526 -0
- package/gates.manifest.json +257 -0
- package/package.json +72 -3
- package/role-model-config.yaml +88 -0
- package/roles/README.md +128 -0
- package/roles/_shared-dispatch-contract.md +87 -0
- package/roles/adversarial-plan-reviewer.md +80 -0
- package/roles/clean-room-reviewer.md +140 -0
- package/roles/implementer.md +463 -0
- package/roles/investigator.md +138 -0
- package/roles/plan-writer.md +95 -0
- package/schemas/README.md +82 -0
- package/schemas/assurance-modes.schema.json +264 -0
- package/schemas/charter.schema.json +185 -0
- package/schemas/checklist.schema.json +114 -0
- package/schemas/cutover-state.schema.json +64 -0
- package/schemas/decision-record.schema.json +88 -0
- package/schemas/executor-record.schema.json +36 -0
- package/schemas/final-report.schema.json +90 -0
- package/schemas/finding.schema.json +106 -0
- package/schemas/gate-registry.schema.json +260 -0
- package/schemas/mechanism-index.schema.json +94 -0
- package/schemas/model-resolution.schema.json +362 -0
- package/schemas/plan.schema.json +300 -0
- package/schemas/report.schema.json +579 -0
- package/schemas/role-brief.schema.json +105 -0
- package/schemas/role-model-config.schema.json +90 -0
- package/schemas/status-line.schema.json +40 -0
- package/schemas/tuition.schema.json +191 -0
- package/schemas/verdict.schema.json +295 -0
- package/schemas/work-history.schema.json +183 -0
- package/schemas/write-bypass.schema.json +69 -0
- package/templates/charter.example.yaml +54 -0
- package/templates/decision-record.example.yaml +27 -0
- package/templates/final-report.example.yaml +80 -0
- package/templates/plan.example.yaml +87 -0
- package/templates/report.example.yaml +236 -0
- package/templates/warnings.md +74 -0
- package/templates/work-history.example.yaml +185 -0
- package/tuition/README.md +76 -0
- package/tuition/T-001.yaml +48 -0
- package/tuition/T-002.yaml +51 -0
- package/tuition/T-003.yaml +100 -0
- package/tuition/T-004.yaml +52 -0
- package/tuition/T-005.yaml +72 -0
- package/tuition/T-006.yaml +81 -0
- package/tuition/T-007.yaml +56 -0
- package/tuition/T-008.yaml +111 -0
- package/tuition/T-009.yaml +50 -0
- package/tuition/T-015.yaml +36 -0
- package/tuition/T-016.yaml +36 -0
- package/tuition/T-017.yaml +46 -0
- package/tuition/T-018.yaml +84 -0
- package/tuition/T-021.yaml +40 -0
- package/tuition/T-022.yaml +36 -0
- package/tuition/mechanism-index.yaml +256 -0
|
@@ -0,0 +1,849 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { closeSync, fsyncSync, openSync, renameSync, statSync, unlinkSync, watch, writeFileSync, writeSync, } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
5
|
+
import { CADENCE, readBeacon, renderBeacon, surveyTaskRecords, } from "./liveness.js";
|
|
6
|
+
import { classifyEntry, executorRecordPath, readRegularFileIfPresent, refuseOpenForWrite, runStep, runStepAsync, turnEndPath, } from "./task.js";
|
|
7
|
+
/**
|
|
8
|
+
* The watcher (kernel plan v1, M1-P5 step 1; R-078, R-079; DR-0007;
|
|
9
|
+
* plan constraints C-1, C-2 and C-3).
|
|
10
|
+
*
|
|
11
|
+
* ONE CORE, TWO ENTRY MODES (DR-0007). Resident mode (tiphys watch) is a
|
|
12
|
+
* plain foreground process the caller owns. Single-pass mode (tiphys
|
|
13
|
+
* watch --once) performs exactly one evaluation of the same wake sources
|
|
14
|
+
* and exits, for external triggers on reclaimable substrates. Both modes
|
|
15
|
+
* run the same evaluation function over the same on-disk state; there is
|
|
16
|
+
* no adapter and no second implementation.
|
|
17
|
+
*
|
|
18
|
+
* C-3, and why there is nothing here to guard: the kernel never runs a
|
|
19
|
+
* watcher outside the caller's own foreground process. No flag exists
|
|
20
|
+
* that would make it outlive its caller, this module launches no child
|
|
21
|
+
* process of any kind, and arming is explicit: the operator starts it
|
|
22
|
+
* and the beacon is what proves it ran (FM-054, where about 30 minutes
|
|
23
|
+
* of supervision were lost to a shell that helpfully moved one out of
|
|
24
|
+
* sight).
|
|
25
|
+
*
|
|
26
|
+
* C-2: nothing here has any notion of a running program. Wake sources
|
|
27
|
+
* are FILES, currency comes from files, and the only clock question ever
|
|
28
|
+
* asked is "is this timestamp older than that one".
|
|
29
|
+
*
|
|
30
|
+
* C-1: task currency is read from tasks/<id>/meta.json status and the
|
|
31
|
+
* turn-end file, never from a log tail. state/last-wake.json is written
|
|
32
|
+
* and never read back by any code path in this kernel; deleting it or
|
|
33
|
+
* filling it with garbage changes no decision this module makes.
|
|
34
|
+
*
|
|
35
|
+
* REASON LINES (grammar harvested from firstmate, FM-002):
|
|
36
|
+
*
|
|
37
|
+
* signal <task-id> turn-end a turn-end file appeared for an open task
|
|
38
|
+
* stale <task-id> deadline an open task passed its executor.json
|
|
39
|
+
* deadline with no turn-end (PR-207)
|
|
40
|
+
* check <name> a one-shot check was requested through
|
|
41
|
+
* state/check-request
|
|
42
|
+
* heartbeat <n> the nth heartbeat since the last reset
|
|
43
|
+
*
|
|
44
|
+
* A wake is ONE line on stdout and then exit 0. The watcher surfaces
|
|
45
|
+
* wakes and never classifies, triages or absorbs them (FM-057:
|
|
46
|
+
* firstmate's absorb-triage grew to most of its 1126 lines because its
|
|
47
|
+
* completion signals were weak; a proposal to add classification here is
|
|
48
|
+
* a signal-design red flag, not a feature request).
|
|
49
|
+
*
|
|
50
|
+
* THE STALE ENUMERATION, and why it has exactly two entries in M1. The
|
|
51
|
+
* plan's grammar is "stale <what>" for "an open task whose worktree or
|
|
52
|
+
* meta is in a contradictory state, enumerated in the module docs", and
|
|
53
|
+
* PR-207 names the first entry:
|
|
54
|
+
*
|
|
55
|
+
* stale <task-id> deadline the executor record's deadline passed and
|
|
56
|
+
* no turn-end arrived (PR-207)
|
|
57
|
+
* stale <task-id> meta tasks/<id>/meta.json EXISTS and does not
|
|
58
|
+
* parse, so this module cannot establish
|
|
59
|
+
* whether the task is open (fix round,
|
|
60
|
+
* second reviewer finding 2)
|
|
61
|
+
*
|
|
62
|
+
* The second entry is the plan's own "meta is in a contradictory state"
|
|
63
|
+
* clause, not an invention: a record that exists and cannot be read is
|
|
64
|
+
* the one case where staying quiet would mean asserting "nothing to
|
|
65
|
+
* supervise here" from evidence this module does not have. A task
|
|
66
|
+
* directory with NO meta.json at all is a different thing (a spawn in
|
|
67
|
+
* progress or a rollback residue) and is not surfaced.
|
|
68
|
+
*
|
|
69
|
+
* Tuition T-002 asked for "task open, no turn-end, worktree dirty" as a
|
|
70
|
+
* wake condition; the declared-deadline half of that is what this module
|
|
71
|
+
* detects, from file evidence alone, and it is witnessed against a spawn
|
|
72
|
+
* that was genuinely stopped mid-payload rather than against a
|
|
73
|
+
* hand-built file state. The deadline-less half is not invented here,
|
|
74
|
+
* because the plan's own not-proven list states that abandonment of a
|
|
75
|
+
* task launched WITHOUT a deadline is not auto-detected in M1.
|
|
76
|
+
*
|
|
77
|
+
* WHAT AN EVALUATION IS, and why it matters for the beacon (PR-206,
|
|
78
|
+
* PR-009). An EVALUATION is a pass over the wake sources that also
|
|
79
|
+
* rewrites state/watcher.beacon: it happens when a resident watcher
|
|
80
|
+
* starts, on every heartbeat tick, whenever a wake is surfaced, and on
|
|
81
|
+
* every watch --once pass including a no-wake one. Between heartbeats a
|
|
82
|
+
* resident watcher SCANS its wake sources every poll interval (and
|
|
83
|
+
* immediately on a filesystem change); a scan that surfaces nothing
|
|
84
|
+
* writes nothing. That is what makes the beacon mean "supervision
|
|
85
|
+
* executed on schedule" rather than "a process is alive", and it is what
|
|
86
|
+
* puts the oldest healthy beacon age at one backoff cap plus one poll
|
|
87
|
+
* interval, which is the bound the guard's threshold has to clear
|
|
88
|
+
* (PR-009, enforced in src/liveness.ts).
|
|
89
|
+
*
|
|
90
|
+
* CADENCE LIVES ON DISK (FM-006, FM-045), never in process memory, so a
|
|
91
|
+
* resident restart and a single pass share one schedule:
|
|
92
|
+
* state/watcher.cadence.json holds {lastHeartbeatAt, backoffStreak}. On
|
|
93
|
+
* a virgin fleet the first evaluation initializes it to the current time
|
|
94
|
+
* and surfaces no heartbeat; the first heartbeat falls due one base
|
|
95
|
+
* interval later (PR-205).
|
|
96
|
+
*
|
|
97
|
+
* Turn-end currency (PR-204). A turn-end wake is surfaced at most once
|
|
98
|
+
* across both modes. state/watcher.seen.json records, per task, the
|
|
99
|
+
* identity of the last surfaced turn-end (size, modification time and a
|
|
100
|
+
* content signature, FM-005). The record is advanced under a claim file,
|
|
101
|
+
* written stage-then-rename and confirmed by reading it back, which is
|
|
102
|
+
* the discipline src/lock.ts established (it is mirrored rather than
|
|
103
|
+
* reused: the lock's primitive is lease-shaped and src/lock.ts is not in
|
|
104
|
+
* this phase's edit scope). The wake is appended to state/last-wake.json
|
|
105
|
+
* BEFORE the seen record advances (enqueue-before-suppress, FM-046: a
|
|
106
|
+
* stop between the two duplicates a wake rather than dropping it). Two
|
|
107
|
+
* passes racing on the same turn-end therefore resolve to exactly one
|
|
108
|
+
* surfacing; the loser reports no-wake.
|
|
109
|
+
*
|
|
110
|
+
* NO HEALTH FROM AN ABSENCE OF EVIDENCE (fix round, the property both
|
|
111
|
+
* reviewers' blocking findings share). Silence from this module means
|
|
112
|
+
* "there is nothing to surface", and it may only be said about states
|
|
113
|
+
* this module has POSITIVELY ESTABLISHED. Wherever it cannot establish
|
|
114
|
+
* one, the output is the loud one, never the quiet one:
|
|
115
|
+
*
|
|
116
|
+
* - a seen-state claim it could not take within the bounded wait is a
|
|
117
|
+
* STUCK CLAIM, not ordinary contention: nonzero exit, one reason
|
|
118
|
+
* line naming the file, and NO beacon write, so the guard sees
|
|
119
|
+
* supervision stop. This is src/lock.ts's rule, adopted verbatim
|
|
120
|
+
* rather than reinvented (see the note below).
|
|
121
|
+
* - a meta.json that exists and does not parse is surfaced as
|
|
122
|
+
* "stale <task-id> meta" rather than skipped.
|
|
123
|
+
* - any other raise ends the pass loudly (see the classification
|
|
124
|
+
* below).
|
|
125
|
+
*
|
|
126
|
+
* WHERE THE STUCK-CLAIM RULE COMES FROM, recorded because it should have
|
|
127
|
+
* reached this module the first time and did not: src/lock.ts's
|
|
128
|
+
* applyLeaseMutation has used the identical O_EXCL claim-file pattern
|
|
129
|
+
* since M1-P3, and its module doc (src/lock.ts, "No steal protocol
|
|
130
|
+
* exists on purpose (FM-058)") states the rule this module now follows:
|
|
131
|
+
* a claim file left behind by a mutation that stopped makes later
|
|
132
|
+
* mutations FAIL LOUDLY after a bounded wait, naming the file for manual
|
|
133
|
+
* removal, and no code ever breaks someone else's claim. That behavior
|
|
134
|
+
* was established by M1-P3's CR-204 after the U-2 race investigation
|
|
135
|
+
* (delivery/verification/u2-race-flake-investigation.md,
|
|
136
|
+
* delivery/review/clean-room-m1-p3.md). The first draft of this module
|
|
137
|
+
* copied the mechanism and dropped the loudness, which the second
|
|
138
|
+
* M1-P5 reviewer reproduced as a permanent, silent loss of every future
|
|
139
|
+
* turn-end wake. Anyone adding a third claim-file user should read
|
|
140
|
+
* src/lock.ts first.
|
|
141
|
+
*
|
|
142
|
+
* HOW A RAISED ERROR IS CLASSIFIED, decided once and applied
|
|
143
|
+
* structurally, because this module does filesystem work in a loop and
|
|
144
|
+
* the previous phase's two worst defects were raises walking past
|
|
145
|
+
* handlers that only understood returned failures (M1-P4 F-1 and F-2):
|
|
146
|
+
*
|
|
147
|
+
* - A file that is ABSENT is not an error. Task directories appear and
|
|
148
|
+
* vanish under a concurrent teardown, so every read of task state
|
|
149
|
+
* goes through a helper that turns ENOENT into "not there" and
|
|
150
|
+
* rethrows everything else.
|
|
151
|
+
* - Any OTHER raise ends the pass with a reason line and a nonzero
|
|
152
|
+
* exit. The watcher stops loudly instead of looping blind: a stopped
|
|
153
|
+
* watcher stops advancing the beacon, and the liveness guard then
|
|
154
|
+
* tells the operator on the next spawn, teardown or doctor.
|
|
155
|
+
* - The whole pass is wrapped by runStep (src/task.ts), so this holds
|
|
156
|
+
* for every site in it without depending on remembering it at each.
|
|
157
|
+
*/
|
|
158
|
+
/** Exit code of a --once pass that found nothing actionable. */
|
|
159
|
+
export const NO_WAKE_EXIT = 3;
|
|
160
|
+
/** Wake-source and cadence state files, all under the fleet's state/. */
|
|
161
|
+
export const CADENCE_STATE_FILE = join("state", "watcher.cadence.json");
|
|
162
|
+
export const SEEN_STATE_FILE = join("state", "watcher.seen.json");
|
|
163
|
+
export const LAST_WAKE_FILE = join("state", "last-wake.json");
|
|
164
|
+
export const CHECK_REQUEST_FILE = join("state", "check-request");
|
|
165
|
+
export function cadencePath(fleet) {
|
|
166
|
+
return join(fleet.root, CADENCE_STATE_FILE);
|
|
167
|
+
}
|
|
168
|
+
export function seenPath(fleet) {
|
|
169
|
+
return join(fleet.root, SEEN_STATE_FILE);
|
|
170
|
+
}
|
|
171
|
+
export function lastWakePath(fleet) {
|
|
172
|
+
return join(fleet.root, LAST_WAKE_FILE);
|
|
173
|
+
}
|
|
174
|
+
export function checkRequestPath(fleet) {
|
|
175
|
+
return join(fleet.root, CHECK_REQUEST_FILE);
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Absent is not an error; anything else is (see the module docs).
|
|
179
|
+
*
|
|
180
|
+
* The read goes through readRegularFileIfPresent (src/task.ts), so the
|
|
181
|
+
* path's TYPE is established before anything is opened. Every wake source
|
|
182
|
+
* and every cadence file this module reads arrives here, and each of them
|
|
183
|
+
* used to block forever on a named pipe: turn-end, executor.json,
|
|
184
|
+
* check-request, watcher.seen.json and watcher.cadence.json, five of the
|
|
185
|
+
* paths CR-520 measured. The loud behavior is unchanged in kind, a
|
|
186
|
+
* nonzero exit with one reason line, and the line now NAMES THE PATH,
|
|
187
|
+
* which the raw EISDIR it replaces did not.
|
|
188
|
+
*/
|
|
189
|
+
function readIfPresent(path) {
|
|
190
|
+
const read = readRegularFileIfPresent(path);
|
|
191
|
+
if (read.kind === "absent") {
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
if (read.kind === "refused") {
|
|
195
|
+
throw new Error(read.reason);
|
|
196
|
+
}
|
|
197
|
+
return read.body;
|
|
198
|
+
}
|
|
199
|
+
function statIfPresent(path) {
|
|
200
|
+
try {
|
|
201
|
+
return statSync(path);
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
if (error.code === "ENOENT") {
|
|
205
|
+
return undefined;
|
|
206
|
+
}
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
export function sameIdentity(a, b) {
|
|
211
|
+
return a.size === b.size && a.mtimeMs === b.mtimeMs && a.signature === b.signature;
|
|
212
|
+
}
|
|
213
|
+
function identityOf(path) {
|
|
214
|
+
const stats = statIfPresent(path);
|
|
215
|
+
if (stats === undefined) {
|
|
216
|
+
return undefined;
|
|
217
|
+
}
|
|
218
|
+
const body = readIfPresent(path);
|
|
219
|
+
if (body === undefined) {
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
222
|
+
return {
|
|
223
|
+
size: stats.size,
|
|
224
|
+
mtimeMs: stats.mtimeMs,
|
|
225
|
+
signature: createHash("sha256").update(body).digest("hex"),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
export function readSeenState(fleet) {
|
|
229
|
+
const raw = readIfPresent(seenPath(fleet));
|
|
230
|
+
if (raw === undefined) {
|
|
231
|
+
return {};
|
|
232
|
+
}
|
|
233
|
+
let parsed;
|
|
234
|
+
try {
|
|
235
|
+
parsed = JSON.parse(raw);
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
// A seen file that does not parse is treated as empty: the cost is a
|
|
239
|
+
// duplicated wake, and the alternative (refusing to surface) would
|
|
240
|
+
// lose one. Duplicate rather than drop is this phase's standing rule
|
|
241
|
+
// (FM-046).
|
|
242
|
+
return {};
|
|
243
|
+
}
|
|
244
|
+
if (parsed === null || typeof parsed !== "object") {
|
|
245
|
+
return {};
|
|
246
|
+
}
|
|
247
|
+
const out = {};
|
|
248
|
+
for (const [taskId, value] of Object.entries(parsed)) {
|
|
249
|
+
const candidate = value;
|
|
250
|
+
if (typeof candidate.size === "number" &&
|
|
251
|
+
typeof candidate.mtimeMs === "number" &&
|
|
252
|
+
typeof candidate.signature === "string") {
|
|
253
|
+
out[taskId] = {
|
|
254
|
+
size: candidate.size,
|
|
255
|
+
mtimeMs: candidate.mtimeMs,
|
|
256
|
+
signature: candidate.signature,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
return out;
|
|
261
|
+
}
|
|
262
|
+
export function readCadenceState(fleet) {
|
|
263
|
+
const raw = readIfPresent(cadencePath(fleet));
|
|
264
|
+
if (raw === undefined) {
|
|
265
|
+
return undefined;
|
|
266
|
+
}
|
|
267
|
+
let parsed;
|
|
268
|
+
try {
|
|
269
|
+
parsed = JSON.parse(raw);
|
|
270
|
+
}
|
|
271
|
+
catch {
|
|
272
|
+
return undefined;
|
|
273
|
+
}
|
|
274
|
+
const candidate = parsed;
|
|
275
|
+
if (typeof candidate.lastHeartbeatAt !== "string" ||
|
|
276
|
+
Number.isNaN(Date.parse(candidate.lastHeartbeatAt)) ||
|
|
277
|
+
typeof candidate.backoffStreak !== "number" ||
|
|
278
|
+
!Number.isFinite(candidate.backoffStreak) ||
|
|
279
|
+
candidate.backoffStreak < 0) {
|
|
280
|
+
return undefined;
|
|
281
|
+
}
|
|
282
|
+
return { lastHeartbeatAt: candidate.lastHeartbeatAt, backoffStreak: candidate.backoffStreak };
|
|
283
|
+
}
|
|
284
|
+
export function writeCadenceState(fleet, state) {
|
|
285
|
+
atomicWrite(cadencePath(fleet), `${JSON.stringify(state, null, 2)}\n`);
|
|
286
|
+
}
|
|
287
|
+
/** The doubling heartbeat interval, capped (FM-044). */
|
|
288
|
+
export function intervalMsFor(streak, cadence) {
|
|
289
|
+
const exponent = Math.min(streak, 52);
|
|
290
|
+
const raw = cadence.baseIntervalMs * 2 ** exponent;
|
|
291
|
+
return Math.min(raw, cadence.backoffCapMs);
|
|
292
|
+
}
|
|
293
|
+
/** When the next heartbeat falls due, from on-disk state alone. */
|
|
294
|
+
export function nextHeartbeatDueMs(state, cadence) {
|
|
295
|
+
return Date.parse(state.lastHeartbeatAt) + intervalMsFor(state.backoffStreak, cadence);
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Write a file through a staged rename, so a reader (doctor, the guard,
|
|
299
|
+
* another pass) never sees a half-written state file.
|
|
300
|
+
*
|
|
301
|
+
* THE DESTINATION IS PROBED (fix round 4). rename(2) would silently
|
|
302
|
+
* replace a non-regular entry at the destination. That is not a block, but
|
|
303
|
+
* it is a SILENT SUCCESS where a directory at the same path is loud
|
|
304
|
+
* (rename refuses that one), and this module's rule is that an unexpected
|
|
305
|
+
* entry under state/ is reported rather than absorbed. Absent is fine:
|
|
306
|
+
* creating the file is the point.
|
|
307
|
+
*
|
|
308
|
+
* THE STAGE PATH IS UNIQUE PER WRITE, and that is a defect fix rather than
|
|
309
|
+
* a tidy-up (fix round 4). It used to be `${path}.stage`, a fixed name, so
|
|
310
|
+
* two passes writing the same state file raced on one temporary: both
|
|
311
|
+
* wrote it, the first renamed it away, and the second's rename failed with
|
|
312
|
+
* a raw ENOENT. Measured on this phase's head before this round, with two
|
|
313
|
+
* single passes released together onto one pending turn-end: the pass that
|
|
314
|
+
* had ALREADY advanced the seen state died on its beacon write before
|
|
315
|
+
* cmdWatch printed anything, and the other reported no-wake, so the wake
|
|
316
|
+
* was suppressed forever and NEITHER pass surfaced it. That is a drop
|
|
317
|
+
* where the plan's whole surfacing protocol is built to duplicate rather
|
|
318
|
+
* than drop (PR-204, FM-046), and it is criterion 7's own failure state.
|
|
319
|
+
*
|
|
320
|
+
* A unique name removes the contention instead of guarding it: two passes
|
|
321
|
+
* now stage into different files and rename(2) settles the order, which is
|
|
322
|
+
* what makes the write atomic in the first place. The name comes from
|
|
323
|
+
* randomUUID and deliberately NOT from any process identifier: plan
|
|
324
|
+
* constraint C-2 bars those, criterion 14 greps these sources for one, and
|
|
325
|
+
* a temporary filename must not be the one place it turns up. It also
|
|
326
|
+
* means nothing can pre-plant an entry at a stage path, so the write-side
|
|
327
|
+
* blocking hazard is eliminated at that path rather than classified there.
|
|
328
|
+
*/
|
|
329
|
+
function atomicWrite(path, body) {
|
|
330
|
+
const refusedTarget = refuseOpenForWrite(path);
|
|
331
|
+
if (refusedTarget !== undefined) {
|
|
332
|
+
throw new Error(`the state file could not be rewritten: ${refusedTarget}`);
|
|
333
|
+
}
|
|
334
|
+
const stage = `${path}.${randomUUID()}.stage`;
|
|
335
|
+
writeFileSync(stage, body);
|
|
336
|
+
try {
|
|
337
|
+
renameSync(stage, path);
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
// A unique stage name is invisible to any cleanup, so this pass owns
|
|
341
|
+
// removing its own leavings before it reports the failure.
|
|
342
|
+
try {
|
|
343
|
+
unlinkSync(stage);
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
// Already gone; nothing to clean.
|
|
347
|
+
}
|
|
348
|
+
throw error;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Rewrite the beacon. The timestamp STRICTLY increases even when two
|
|
353
|
+
* evaluations land in the same millisecond, because criterion 8 turns on
|
|
354
|
+
* a no-wake pass advancing it and "advanced" has to mean something a
|
|
355
|
+
* reader can compare.
|
|
356
|
+
*/
|
|
357
|
+
export function writeBeacon(fleet, nowMs, streak, cadence) {
|
|
358
|
+
const previous = readBeacon(fleet.beaconPath);
|
|
359
|
+
const previousMs = previous === undefined ? undefined : Date.parse(previous.writtenAt);
|
|
360
|
+
const stampMs = previousMs === undefined || nowMs > previousMs ? nowMs : previousMs + 1;
|
|
361
|
+
atomicWrite(fleet.beaconPath, renderBeacon({
|
|
362
|
+
writtenAt: new Date(stampMs).toISOString(),
|
|
363
|
+
backoffStreak: streak,
|
|
364
|
+
intervalMs: intervalMsFor(streak, cadence),
|
|
365
|
+
}));
|
|
366
|
+
return stampMs;
|
|
367
|
+
}
|
|
368
|
+
export function wakeLine(wake) {
|
|
369
|
+
if (wake.kind === "signal") {
|
|
370
|
+
return `signal ${wake.taskId} ${wake.event}`;
|
|
371
|
+
}
|
|
372
|
+
if (wake.kind === "stale") {
|
|
373
|
+
return `stale ${wake.taskId} ${wake.what}`;
|
|
374
|
+
}
|
|
375
|
+
return `check ${wake.name}`;
|
|
376
|
+
}
|
|
377
|
+
export function heartbeatLine(n) {
|
|
378
|
+
return `heartbeat ${String(n)}`;
|
|
379
|
+
}
|
|
380
|
+
/** The executor launch record, as far as this module reads it (PR-207). */
|
|
381
|
+
function deadlineOf(fleet, taskId) {
|
|
382
|
+
const raw = readIfPresent(executorRecordPath(fleet, taskId));
|
|
383
|
+
if (raw === undefined) {
|
|
384
|
+
return undefined;
|
|
385
|
+
}
|
|
386
|
+
let parsed;
|
|
387
|
+
try {
|
|
388
|
+
parsed = JSON.parse(raw);
|
|
389
|
+
}
|
|
390
|
+
catch {
|
|
391
|
+
return undefined;
|
|
392
|
+
}
|
|
393
|
+
const deadline = parsed.deadline;
|
|
394
|
+
if (typeof deadline !== "string") {
|
|
395
|
+
return undefined;
|
|
396
|
+
}
|
|
397
|
+
const ms = Date.parse(deadline);
|
|
398
|
+
return Number.isNaN(ms) ? undefined : ms;
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* One scan of the wake sources, in the grammar's own priority order:
|
|
402
|
+
* turn-end signals first, then stale conditions, then a requested check.
|
|
403
|
+
* Task ids are visited in sorted order so two passes over the same fleet
|
|
404
|
+
* choose the same wake.
|
|
405
|
+
*
|
|
406
|
+
* Stale conditions are NOT suppressed by the seen-state, and that is
|
|
407
|
+
* deliberate: PR-204's at-most-once rule is about turn-end EDGES, while
|
|
408
|
+
* a passed deadline with no turn-end is a standing condition that stays
|
|
409
|
+
* true until an operator acts on it. Reporting it again on the next pass
|
|
410
|
+
* is the correct behavior for something nobody has dealt with yet.
|
|
411
|
+
*/
|
|
412
|
+
function scanUnsafe(fleet, nowMs) {
|
|
413
|
+
// ONE classifier of task records, shared with the liveness guard
|
|
414
|
+
// (src/liveness.ts surveyTaskRecords). This module does not carry a
|
|
415
|
+
// second opinion about what a task record is or about what an
|
|
416
|
+
// unreadable one means: two implementations of one property disagreed
|
|
417
|
+
// once already (delta review NEW-1, tuition T-005).
|
|
418
|
+
const survey = surveyTaskRecords(fleet);
|
|
419
|
+
const problem = survey.problems[0];
|
|
420
|
+
if (problem !== undefined) {
|
|
421
|
+
// A survey that did not complete is not an idle fleet. Raising here
|
|
422
|
+
// reaches runStep, which turns it into the module's loud outcome: a
|
|
423
|
+
// nonzero exit with one reason line, and no beacon write.
|
|
424
|
+
throw new Error(problem);
|
|
425
|
+
}
|
|
426
|
+
const open = survey.open;
|
|
427
|
+
const seen = readSeenState(fleet);
|
|
428
|
+
for (const id of open) {
|
|
429
|
+
const identity = identityOf(turnEndPath(fleet, id));
|
|
430
|
+
if (identity === undefined) {
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
const previous = seen[id];
|
|
434
|
+
if (previous !== undefined && sameIdentity(previous, identity)) {
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
return { kind: "signal", taskId: id, event: "turn-end", identity };
|
|
438
|
+
}
|
|
439
|
+
for (const id of open) {
|
|
440
|
+
if (statIfPresent(turnEndPath(fleet, id)) !== undefined) {
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
const deadlineMs = deadlineOf(fleet, id);
|
|
444
|
+
if (deadlineMs !== undefined && deadlineMs <= nowMs) {
|
|
445
|
+
return { kind: "stale", taskId: id, what: "deadline" };
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
const firstUnreadable = survey.unreadable[0];
|
|
449
|
+
if (firstUnreadable !== undefined) {
|
|
450
|
+
return { kind: "stale", taskId: firstUnreadable, what: "meta" };
|
|
451
|
+
}
|
|
452
|
+
const request = statIfPresent(checkRequestPath(fleet));
|
|
453
|
+
if (request !== undefined) {
|
|
454
|
+
return { kind: "check", name: "requested" };
|
|
455
|
+
}
|
|
456
|
+
return undefined;
|
|
457
|
+
}
|
|
458
|
+
/** Scan, with the module's raise classification applied structurally. */
|
|
459
|
+
export function scanWakeSources(fleet, nowMs) {
|
|
460
|
+
const scanned = runStep("scanning the watcher wake sources", () => scanUnsafe(fleet, nowMs));
|
|
461
|
+
if (!scanned.ok) {
|
|
462
|
+
return { ok: false, reason: scanned.reason };
|
|
463
|
+
}
|
|
464
|
+
return { ok: true, wake: scanned.value };
|
|
465
|
+
}
|
|
466
|
+
/**
|
|
467
|
+
* Durably append one wake record, one JSON object per line (JSON per
|
|
468
|
+
* DR-0006; the line framing is what makes an append safe without a
|
|
469
|
+
* read-modify-write of the whole file). The plan fixes the file's name
|
|
470
|
+
* and its append discipline, not its internal framing; this is the
|
|
471
|
+
* choice, recorded rather than assumed.
|
|
472
|
+
*
|
|
473
|
+
* NOTHING IN THIS KERNEL READS THIS FILE (plan constraint C-1). It is a
|
|
474
|
+
* durability record for a human and for later milestones, never a source
|
|
475
|
+
* of currency: currency comes from meta.json, the turn-end file, the
|
|
476
|
+
* executor record and the seen state.
|
|
477
|
+
*
|
|
478
|
+
* WHAT ENQUEUE-BEFORE-SUPPRESS THEREFORE DOES AND DOES NOT BUY (CR-522,
|
|
479
|
+
* recorded rather than fixed). FM-046 and the plan put this append BEFORE
|
|
480
|
+
* the seen-state advance so a stop between the two duplicates a wake
|
|
481
|
+
* rather than dropping it. The at-most-once guarantee is real and is
|
|
482
|
+
* delivered by the seen state. The DURABILITY half is weaker than it
|
|
483
|
+
* reads: because nothing reads this file, no code path can restore a wake
|
|
484
|
+
* from it, and the channel that actually delivers a wake to a consumer is
|
|
485
|
+
* stdout, written by cmdWatch AFTER claimSignal has already advanced the
|
|
486
|
+
* seen state and released the claim. A stop inside that window suppresses
|
|
487
|
+
* the wake permanently, for every later pass and both modes, and the guard
|
|
488
|
+
* goes on reporting the fleet fresh. The window is small and closing it
|
|
489
|
+
* means printing inside the claim, which changes the surfacing protocol
|
|
490
|
+
* the plan fixes; it is carried as an M2 item, not improvised here.
|
|
491
|
+
*/
|
|
492
|
+
function appendWakeRecord(fleet, nowMs, line) {
|
|
493
|
+
const record = `${JSON.stringify({ at: new Date(nowMs).toISOString(), line })}\n`;
|
|
494
|
+
// Probed before it is opened, for the same reason every read here is
|
|
495
|
+
// (fix round 4). openSync with "a" is O_WRONLY|O_APPEND|O_CREAT, and on
|
|
496
|
+
// a named pipe with no reader that BLOCKS. This path is reached only
|
|
497
|
+
// when a wake is being surfaced, which is why the earlier round's probe
|
|
498
|
+
// of state/ files missed it, and blocking here hangs the watcher with a
|
|
499
|
+
// real pending signal, in both modes.
|
|
500
|
+
const refused = refuseOpenForWrite(lastWakePath(fleet));
|
|
501
|
+
if (refused !== undefined) {
|
|
502
|
+
throw new Error(`the wake record could not be appended: ${refused}`);
|
|
503
|
+
}
|
|
504
|
+
const handle = openSync(lastWakePath(fleet), "a");
|
|
505
|
+
try {
|
|
506
|
+
writeSync(handle, record);
|
|
507
|
+
fsyncSync(handle);
|
|
508
|
+
}
|
|
509
|
+
finally {
|
|
510
|
+
closeSync(handle);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
const CLAIM_WAIT_TOTAL_MS = 5000;
|
|
514
|
+
const CLAIM_WAIT_POLL_MS = 5;
|
|
515
|
+
/**
|
|
516
|
+
* Deterministic interleave seam for the concurrency witness, modelled on
|
|
517
|
+
* the one src/commands/lock.ts already carries. When
|
|
518
|
+
* TIPHYS_WATCH_TEST_HOLD names a barrier path, a pass that has DECIDED to
|
|
519
|
+
* surface a turn-end writes <barrier>.observed and then waits for
|
|
520
|
+
* <barrier> to appear before it touches the seen state, so a test can
|
|
521
|
+
* place a second pass exactly inside the window. It is LOUD when it does
|
|
522
|
+
* not actually hold (the M1-P3 lesson): the wait is monotonic and a
|
|
523
|
+
* timeout raises rather than continuing with a stale observation, which
|
|
524
|
+
* is how a witness scores an interleave it never staged.
|
|
525
|
+
*/
|
|
526
|
+
const HOLD_WAIT_LIMIT_MS = 30_000;
|
|
527
|
+
async function maybeHoldForTest() {
|
|
528
|
+
const barrier = process.env.TIPHYS_WATCH_TEST_HOLD;
|
|
529
|
+
if (barrier === undefined || barrier === "") {
|
|
530
|
+
return;
|
|
531
|
+
}
|
|
532
|
+
writeFileSync(`${barrier}.observed`, "");
|
|
533
|
+
const startNs = process.hrtime.bigint();
|
|
534
|
+
for (;;) {
|
|
535
|
+
if (statIfPresent(barrier) !== undefined) {
|
|
536
|
+
writeFileSync(`${barrier}.released`, "barrier appeared\n");
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
const elapsedMs = Number(process.hrtime.bigint() - startNs) / 1e6;
|
|
540
|
+
if (elapsedMs >= HOLD_WAIT_LIMIT_MS) {
|
|
541
|
+
writeFileSync(`${barrier}.released`, "timed out without holding\n");
|
|
542
|
+
throw new Error(`watcher test hold at ${barrier} timed out after ` +
|
|
543
|
+
`${String(Math.round(elapsedMs))}ms: the interleave was never staged`);
|
|
544
|
+
}
|
|
545
|
+
await sleep(CLAIM_WAIT_POLL_MS);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* Advance the seen state for one surfaced turn-end, under a claim file,
|
|
550
|
+
* stage-then-rename, confirmed by reading it back. "lost" means another
|
|
551
|
+
* pass got there first, which is the loser's cue to report no-wake
|
|
552
|
+
* (PR-204). "stuck" means the claim could not be taken at all.
|
|
553
|
+
*/
|
|
554
|
+
async function claimSignal(fleet, taskId, identity, nowMs, line) {
|
|
555
|
+
const claimPath = `${seenPath(fleet)}.mutex`;
|
|
556
|
+
const deadline = Date.now() + CLAIM_WAIT_TOTAL_MS;
|
|
557
|
+
for (;;) {
|
|
558
|
+
try {
|
|
559
|
+
writeFileSync(claimPath, "", { flag: "wx" });
|
|
560
|
+
break;
|
|
561
|
+
}
|
|
562
|
+
catch (error) {
|
|
563
|
+
if (error.code !== "EEXIST") {
|
|
564
|
+
throw error;
|
|
565
|
+
}
|
|
566
|
+
if (Date.now() >= deadline) {
|
|
567
|
+
// Whether another pass is inside the window right now or one
|
|
568
|
+
// stopped inside it, this pass cannot establish anything about
|
|
569
|
+
// this wake, so it says so instead of reporting quiet. No steal
|
|
570
|
+
// protocol: the file is named for a human to remove, exactly as
|
|
571
|
+
// src/lock.ts does, and nothing here breaks someone else's claim.
|
|
572
|
+
return {
|
|
573
|
+
kind: "stuck",
|
|
574
|
+
reason: `supervision is stuck: the seen-state claim ${claimPath} was still ` +
|
|
575
|
+
`present after ${String(CLAIM_WAIT_TOTAL_MS)}ms, so the turn-end of ` +
|
|
576
|
+
`task ${taskId} could not be surfaced and no wake was reported; if no ` +
|
|
577
|
+
`other watcher is running, remove that file and re-run`,
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
await sleep(CLAIM_WAIT_POLL_MS);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
try {
|
|
584
|
+
const current = readSeenState(fleet);
|
|
585
|
+
const previous = current[taskId];
|
|
586
|
+
if (previous !== undefined && sameIdentity(previous, identity)) {
|
|
587
|
+
return { kind: "lost" };
|
|
588
|
+
}
|
|
589
|
+
// Enqueue before suppress (FM-046): the wake record is durable
|
|
590
|
+
// before anything can suppress the wake.
|
|
591
|
+
appendWakeRecord(fleet, nowMs, line);
|
|
592
|
+
const next = { ...current, [taskId]: identity };
|
|
593
|
+
atomicWrite(seenPath(fleet), `${JSON.stringify(next, null, 2)}\n`);
|
|
594
|
+
const confirmed = readSeenState(fleet)[taskId];
|
|
595
|
+
if (confirmed === undefined || !sameIdentity(confirmed, identity)) {
|
|
596
|
+
return { kind: "lost" };
|
|
597
|
+
}
|
|
598
|
+
return { kind: "won" };
|
|
599
|
+
}
|
|
600
|
+
finally {
|
|
601
|
+
try {
|
|
602
|
+
unlinkSync(claimPath);
|
|
603
|
+
}
|
|
604
|
+
catch {
|
|
605
|
+
// Already gone; nothing to release.
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Take the one-shot check request, atomically: the rename is the
|
|
611
|
+
* exclusion, so two racing passes cannot both surface one request.
|
|
612
|
+
*
|
|
613
|
+
* PROBE BEFORE THE RENAME, not just before the read (CR-520). The rename
|
|
614
|
+
* is DESTRUCTIVE of the wake source: it is the exclusion mechanism, so by
|
|
615
|
+
* design nothing else can find the request afterwards. Reading second and
|
|
616
|
+
* probing third meant that a named pipe at state/check-request was first
|
|
617
|
+
* consumed and then blocked forever, so the fleet lost its wake source AND
|
|
618
|
+
* reported nothing. Ordering the probe first means a request this pass
|
|
619
|
+
* cannot consume is left exactly where an operator can see it, and the
|
|
620
|
+
* pass says so out loud instead.
|
|
621
|
+
*/
|
|
622
|
+
function claimCheckRequest(fleet) {
|
|
623
|
+
const path = checkRequestPath(fleet);
|
|
624
|
+
const taken = `${path}.taken`;
|
|
625
|
+
const entry = classifyEntry(path);
|
|
626
|
+
if (entry.kind === "irregular" || entry.kind === "unexaminable") {
|
|
627
|
+
throw new Error(`the check request could not be taken and was left in place: ${entry.reason}`);
|
|
628
|
+
}
|
|
629
|
+
try {
|
|
630
|
+
renameSync(path, taken);
|
|
631
|
+
}
|
|
632
|
+
catch (error) {
|
|
633
|
+
if (error.code === "ENOENT") {
|
|
634
|
+
return undefined;
|
|
635
|
+
}
|
|
636
|
+
throw error;
|
|
637
|
+
}
|
|
638
|
+
const body = readIfPresent(taken) ?? "";
|
|
639
|
+
try {
|
|
640
|
+
unlinkSync(taken);
|
|
641
|
+
}
|
|
642
|
+
catch {
|
|
643
|
+
// Best effort: the request has already been taken by this pass.
|
|
644
|
+
}
|
|
645
|
+
const name = body.trim().split("\n")[0] ?? "";
|
|
646
|
+
return name === "" ? "unnamed" : name;
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* Surface a wake: make it durable, suppress repeats where the plan says
|
|
650
|
+
* repeats are wrong, and return the line to print. "lost" means another
|
|
651
|
+
* pass owns this wake and this one must report no-wake; "stuck" means
|
|
652
|
+
* this pass could not establish anything and must say so out loud.
|
|
653
|
+
*/
|
|
654
|
+
export async function surfaceWake(fleet, wake, nowMs) {
|
|
655
|
+
const line = wakeLine(wake);
|
|
656
|
+
if (wake.kind === "signal") {
|
|
657
|
+
await maybeHoldForTest();
|
|
658
|
+
const claimed = await claimSignal(fleet, wake.taskId, wake.identity, nowMs, line);
|
|
659
|
+
if (claimed.kind === "won") {
|
|
660
|
+
return { kind: "surfaced", line };
|
|
661
|
+
}
|
|
662
|
+
if (claimed.kind === "stuck") {
|
|
663
|
+
return { kind: "stuck", reason: claimed.reason };
|
|
664
|
+
}
|
|
665
|
+
return { kind: "lost" };
|
|
666
|
+
}
|
|
667
|
+
if (wake.kind === "check") {
|
|
668
|
+
const name = claimCheckRequest(fleet);
|
|
669
|
+
if (name === undefined) {
|
|
670
|
+
return { kind: "lost" };
|
|
671
|
+
}
|
|
672
|
+
const named = wakeLine({ kind: "check", name });
|
|
673
|
+
appendWakeRecord(fleet, nowMs, named);
|
|
674
|
+
return { kind: "surfaced", line: named };
|
|
675
|
+
}
|
|
676
|
+
appendWakeRecord(fleet, nowMs, line);
|
|
677
|
+
return { kind: "surfaced", line };
|
|
678
|
+
}
|
|
679
|
+
/** Read cadence state, initializing a virgin fleet in place (PR-205). */
|
|
680
|
+
function loadOrInitCadence(fleet, nowMs) {
|
|
681
|
+
const existing = readCadenceState(fleet);
|
|
682
|
+
if (existing !== undefined) {
|
|
683
|
+
return existing;
|
|
684
|
+
}
|
|
685
|
+
const fresh = {
|
|
686
|
+
lastHeartbeatAt: new Date(nowMs).toISOString(),
|
|
687
|
+
backoffStreak: 0,
|
|
688
|
+
};
|
|
689
|
+
writeCadenceState(fleet, fresh);
|
|
690
|
+
return fresh;
|
|
691
|
+
}
|
|
692
|
+
/**
|
|
693
|
+
* Scan and, if there is something to surface, surface it. Returns
|
|
694
|
+
* undefined ONLY when nothing was surfaced and that is a fact this pass
|
|
695
|
+
* established: no wake pending, or a wake another live pass owns. A
|
|
696
|
+
* stuck claim returns a loud outcome instead, and the beacon is written
|
|
697
|
+
* ONLY when a wake is surfaced (see the module docs on what an
|
|
698
|
+
* evaluation is, and on health from an absence of evidence).
|
|
699
|
+
*/
|
|
700
|
+
async function scanAndSurface(fleet, options, nowMs) {
|
|
701
|
+
const scanned = scanWakeSources(fleet, nowMs);
|
|
702
|
+
if (!scanned.ok) {
|
|
703
|
+
return { code: 1, line: "", reason: scanned.reason };
|
|
704
|
+
}
|
|
705
|
+
const wake = scanned.wake;
|
|
706
|
+
if (wake === undefined) {
|
|
707
|
+
return undefined;
|
|
708
|
+
}
|
|
709
|
+
const surfaced = await runStepAsync("surfacing the wake", () => surfaceWake(fleet, wake, nowMs));
|
|
710
|
+
if (!surfaced.ok) {
|
|
711
|
+
return { code: 1, line: "", reason: surfaced.reason };
|
|
712
|
+
}
|
|
713
|
+
const result = surfaced.value;
|
|
714
|
+
if (result.kind === "stuck") {
|
|
715
|
+
// Loud, and deliberately WITHOUT a beacon write: supervision did not
|
|
716
|
+
// execute, so the beacon must be allowed to go stale and the guard
|
|
717
|
+
// must be allowed to notice.
|
|
718
|
+
return { code: 1, line: "", reason: result.reason };
|
|
719
|
+
}
|
|
720
|
+
if (result.kind === "lost") {
|
|
721
|
+
return undefined;
|
|
722
|
+
}
|
|
723
|
+
// Any surfaced non-heartbeat wake resets the backoff (plan step 1).
|
|
724
|
+
writeCadenceState(fleet, {
|
|
725
|
+
lastHeartbeatAt: new Date(nowMs).toISOString(),
|
|
726
|
+
backoffStreak: 0,
|
|
727
|
+
});
|
|
728
|
+
writeBeacon(fleet, nowMs, 0, options.cadence);
|
|
729
|
+
return { code: 0, line: result.line };
|
|
730
|
+
}
|
|
731
|
+
/**
|
|
732
|
+
* A heartbeat evaluation: advance the streak, record it, rewrite the
|
|
733
|
+
* beacon. Returns the heartbeat ordinal since the last cadence reset,
|
|
734
|
+
* which is the n in "heartbeat <n>".
|
|
735
|
+
*/
|
|
736
|
+
export function heartbeatTick(fleet, state, cadence, nowMs) {
|
|
737
|
+
// CR-505: the ordinal is recomputed from the cadence file, never from
|
|
738
|
+
// the caller's snapshot. A resident watcher reads the cadence before
|
|
739
|
+
// its wait and ticks after it, so a concurrent single pass that reset
|
|
740
|
+
// the backoff inside that window would otherwise have its reset
|
|
741
|
+
// overwritten by snapshot + 1.
|
|
742
|
+
const current = readCadenceState(fleet) ?? state;
|
|
743
|
+
const n = current.backoffStreak + 1;
|
|
744
|
+
writeCadenceState(fleet, {
|
|
745
|
+
lastHeartbeatAt: new Date(nowMs).toISOString(),
|
|
746
|
+
backoffStreak: n,
|
|
747
|
+
});
|
|
748
|
+
writeBeacon(fleet, nowMs, n, cadence);
|
|
749
|
+
return n;
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* Single-pass mode: exactly one evaluation, then exit. A heartbeat that
|
|
753
|
+
* is due is surfaced (this pass may be the only supervision this fleet
|
|
754
|
+
* gets), and a pass with nothing actionable prints nothing and exits
|
|
755
|
+
* with the documented no-wake code. The beacon is rewritten either way
|
|
756
|
+
* (PR-206).
|
|
757
|
+
*/
|
|
758
|
+
export async function runOnce(fleet, options) {
|
|
759
|
+
const nowMs = (options.now ?? Date.now)();
|
|
760
|
+
const state = loadOrInitCadence(fleet, nowMs);
|
|
761
|
+
const surfaced = await scanAndSurface(fleet, options, nowMs);
|
|
762
|
+
if (surfaced !== undefined) {
|
|
763
|
+
return surfaced;
|
|
764
|
+
}
|
|
765
|
+
if (nowMs >= nextHeartbeatDueMs(state, options.cadence)) {
|
|
766
|
+
const n = heartbeatTick(fleet, state, options.cadence, nowMs);
|
|
767
|
+
appendWakeRecord(fleet, nowMs, heartbeatLine(n));
|
|
768
|
+
return { code: 0, line: heartbeatLine(n) };
|
|
769
|
+
}
|
|
770
|
+
writeBeacon(fleet, nowMs, state.backoffStreak, options.cadence);
|
|
771
|
+
return { code: NO_WAKE_EXIT, line: "" };
|
|
772
|
+
}
|
|
773
|
+
/**
|
|
774
|
+
* Wait for a filesystem change under the fleet's state/ and tasks/, or
|
|
775
|
+
* for the timeout, whichever comes first. Watching is an optimization
|
|
776
|
+
* over the poll interval, so a platform that cannot watch simply waits.
|
|
777
|
+
*/
|
|
778
|
+
async function waitForChange(fleet, timeoutMs) {
|
|
779
|
+
const watchers = [];
|
|
780
|
+
let resolveChange;
|
|
781
|
+
const changed = new Promise((resolve) => {
|
|
782
|
+
resolveChange = resolve;
|
|
783
|
+
});
|
|
784
|
+
const onEvent = () => {
|
|
785
|
+
resolveChange?.();
|
|
786
|
+
};
|
|
787
|
+
for (const [dir, recursive] of [
|
|
788
|
+
[fleet.stateDir, false],
|
|
789
|
+
[fleet.tasksDir, true],
|
|
790
|
+
]) {
|
|
791
|
+
try {
|
|
792
|
+
const handle = watch(dir, { recursive }, onEvent);
|
|
793
|
+
handle.on("error", onEvent);
|
|
794
|
+
watchers.push(handle);
|
|
795
|
+
}
|
|
796
|
+
catch {
|
|
797
|
+
// No watch on this directory: the poll interval is the fallback,
|
|
798
|
+
// which is exactly what it is for.
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
try {
|
|
802
|
+
await Promise.race([changed, sleep(Math.max(0, timeoutMs))]);
|
|
803
|
+
}
|
|
804
|
+
finally {
|
|
805
|
+
for (const handle of watchers) {
|
|
806
|
+
handle.close();
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* Resident mode: a plain foreground loop the caller owns. It evaluates
|
|
812
|
+
* once at startup (so an already-pending wake is surfaced immediately
|
|
813
|
+
* and the beacon proves arming), then alternates cheap scans at the poll
|
|
814
|
+
* interval with heartbeat evaluations on the backoff schedule.
|
|
815
|
+
*/
|
|
816
|
+
export async function runResident(fleet, options) {
|
|
817
|
+
const now = options.now ?? Date.now;
|
|
818
|
+
// Startup evaluation: an already-pending wake is surfaced at once, and
|
|
819
|
+
// the beacon is written whatever happens, so "the watcher is armed" is
|
|
820
|
+
// an observable fact and not a claim (plan step 1, exit test A5).
|
|
821
|
+
const startupState = loadOrInitCadence(fleet, now());
|
|
822
|
+
const startupWake = await scanAndSurface(fleet, options, now());
|
|
823
|
+
if (startupWake !== undefined) {
|
|
824
|
+
return startupWake;
|
|
825
|
+
}
|
|
826
|
+
writeBeacon(fleet, now(), startupState.backoffStreak, options.cadence);
|
|
827
|
+
let ticksThisRun = 0;
|
|
828
|
+
for (;;) {
|
|
829
|
+
const state = readCadenceState(fleet) ?? loadOrInitCadence(fleet, now());
|
|
830
|
+
const dueMs = nextHeartbeatDueMs(state, options.cadence);
|
|
831
|
+
const waitMs = Math.min(options.cadence.pollIntervalMs, Math.max(0, dueMs - now()));
|
|
832
|
+
await waitForChange(fleet, waitMs);
|
|
833
|
+
const surfaced = await scanAndSurface(fleet, options, now());
|
|
834
|
+
if (surfaced !== undefined) {
|
|
835
|
+
return surfaced;
|
|
836
|
+
}
|
|
837
|
+
if (now() >= dueMs) {
|
|
838
|
+
const n = heartbeatTick(fleet, state, options.cadence, now());
|
|
839
|
+
ticksThisRun += 1;
|
|
840
|
+
// Resident mode is SILENT on heartbeats unless the caller asked for
|
|
841
|
+
// a bounded run: a supervisor that exited every heartbeat would be
|
|
842
|
+
// no supervisor at all (criterion 3).
|
|
843
|
+
if (options.maxHeartbeats !== undefined && ticksThisRun >= options.maxHeartbeats) {
|
|
844
|
+
appendWakeRecord(fleet, now(), heartbeatLine(n));
|
|
845
|
+
return { code: 0, line: heartbeatLine(n) };
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
}
|