@abelxiaoxing/cadence 2.0.25 → 2.0.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -89,12 +89,15 @@ cadence server status
89
89
  cadence server restart # after an upgrade or a .env / models.json change
90
90
  cadence server stop # stays registered and starts again at the next boot or login
91
91
  cadence server uninstall # run before npm uninstall -g
92
+ cadence doctor # what is left over or recently recovered; --fix for the rest
92
93
  ```
93
94
 
95
+ A power loss, shutdown or crash needs no cleanup: the next start recovers automatically, marks interrupted work as interrupted and says what it did.
96
+
94
97
  Upgrade with `npm install -g @abelxiaoxing/cadence@latest`, then `cadence server restart`.
95
98
  Restart, stop and uninstall stop running work in order rather than leaving it detached.
96
99
  Uninstall removes only the service definition; projects, history and Pi settings stay.
97
- Linux root and named services, remote HTTPS access and recovery are covered in [service management](docs/service-management.md) and [remote access](docs/remote-access.md).
100
+ Linux root and named services, remote HTTPS access and automatic recovery are covered in [service management](docs/service-management.md) and [remote access](docs/remote-access.md).
98
101
 
99
102
  ## Development
100
103
 
@@ -90,7 +90,7 @@ Automatic review is never recorded as human review.
90
90
 
91
91
  To request orderly closure, ask Cadence to cancel in Pi, or use **Cancel workflow — request stop** in the browser (the browser coordinator cannot cancel).
92
92
  A pending or unknown cancellation is not reported as stopped.
93
- Failures, timeouts and unknown process/auth/storage outcomes remain distinct; unknown resources retain fences for explicit informed recovery.
93
+ Failures, timeouts and unknown process/auth/storage outcomes remain distinct; an owner that cannot settle unknown work restarts itself and recovers automatically.
94
94
 
95
95
  ## Service and compatibility
96
96
 
@@ -57,7 +57,7 @@ Never put provider credentials in a browser or URL.
57
57
  `token-rotate` invalidates old links/cookies, pending grants, authenticated sockets and incomplete downloads without canceling admitted tasks.
58
58
  Environment-backed tokens require explicit Stop/restart with changed environment instead; an `.env` token's rotation rewrites only its `CADENCE_TOKEN` line.
59
59
  Links may already exist in clipboard/history or external logs; redirecting cannot retract disclosure.
60
- Use [management and recovery](service-management.md) for Stop, uncertainty and exact local resolution.
60
+ Use [management and automatic recovery](service-management.md) for Stop, recovery and `cadence doctor`.
61
61
  Loopback HTTPS/viewport tests are not physical-device or second-machine deployment qualification.
62
62
 
63
63
  ## Unified workbench entry
@@ -23,8 +23,7 @@ Defaults are loopback `127.0.0.1:17423`, normal Pi agent configuration, and deta
23
23
  An explicit port0 selects and persists a positive port once; restart uses the selected configuration, not a fallback endpoint.
24
24
  Missing/corrupt existing setup is preserved, not filled in or reset.
25
25
  Commands without `--workspace` address the unified workbench, which has no execution cwd and never creates `~/cadence`.
26
- Use `cadence workbench start|status|diagnose|stop|token-link|token-rotate|recover inspect|recover resolve` to name that target explicitly.
27
- `recover` without `--workspace` also addresses account/workbench state, never the current directory; project-owner recovery requires an explicit `--workspace`.
26
+ Use `cadence workbench start|status|diagnose|stop|token-link|token-rotate` to name that target explicitly; `cadence doctor` covers the account, the workbench and every registered project (see [Automatic recovery](#automatic-recovery)).
28
27
  Its start accepts `--foreground`, `--state-home`, `--agent-dir`, `--server-config` and a private `--config` containing an `HttpPolicy` object (the schema used below for project HTTP policy).
29
28
  The configuration's port/token come from the `.env`; a selected HTTP policy is retained across orderly restarts.
30
29
  A project root containing the private state home is refused, with no alternate directory creation; native `/cadence` also requires a real project selection.
@@ -60,7 +59,7 @@ The private account state stores bounded project references and opening receipts
60
59
  Private coordination/Pi credential directories and foreign-owned project selections are refused.
61
60
  Direct project entry links remain available as a compatibility route; that route alone still uses separate loopback project entries.
62
61
  Git subdirectories and aliases resolve to the same checkout, while overlapping parent/child owners still refuse.
63
- An unknown, quarantined or inaccessible selection is never treated as a new project or recovered by choosing another state home.
62
+ An unknown or inaccessible selection is never treated as a new project or recovered by choosing another state home.
64
63
 
65
64
  When run as root, Git observation requires a repository Git recognizes as root-owned: Cadence clears `safe.directory` exceptions per command, removes inherited `SUDO_UID` and disables fsmonitor.
66
65
  A foreign-owned repository refuses even if a global `safe.directory=*` would allow it; selecting a path is not consent to execute its configured clean/process filters.
@@ -94,11 +93,12 @@ A reinstall without `--workspace` explicitly selects the unified entry after ord
94
93
  A plain `restart` retains the definition's existing target, including an older explicit project service, rather than silently migrating it.
95
94
  The `.env` [port and token](#port-and-token) select the endpoint (17423 by default) and the token; an occupied endpoint fails instead of falling back.
96
95
  Install and restart print the private entry link once the owner answers; the JSON result on stdout never carries the token.
97
- A successful manager start request is not owner readiness: an unready owner reports `quarantined` or `unconfirmed`, retains the separate `managerStatus`, exits nonzero and names the recovery inspection command.
98
- Status also retains the manager's `active`/`activating`/`failed` observation separately when the workbench itself is quarantined.
96
+ A successful manager start request is not owner readiness: an unready owner reports `quarantined` or `unconfirmed`, retains the separate `managerStatus`, exits nonzero and names `cadence doctor`.
97
+ Status also retains the manager's `active`/`activating`/`failed` observation separately.
98
+ A definition whose project directory was removed gets a readable refusal (exit78, so the unit does not restart in a loop); `cadence server install` switches it to the workbench.
99
99
  `restart` rewrites an existing definition with the current Node and package paths, keeping its workspace and the user's disable choices, and starts it; run it after an npm upgrade, a Node change or a `models.json` edit.
100
100
  `stop` joins the workbench's accepted openings and associated exact-epoch project owners, then closes the entry; unrelated project owners stay independent.
101
- If one associated owner refuses or has a changed epoch, Stop still reaches the other known owners serially, while the workbench remains fenced rather than reporting a successful Stop.
101
+ An associated owner that is not running (or whose directory is gone) has nothing to stop; one whose Stop is unconfirmed keeps its own state, and the workbench still stops.
102
102
  Project-specific `stop --workspace <path>` remains scoped to that owner.
103
103
  The generated registration stays for the next boot or login.
104
104
  `uninstall` stops it orderly and deletes only what Cadence generated; workspace, state home, history, linger and Pi sources stay.
@@ -107,7 +107,7 @@ A definition Cadence did not generate is left unchanged (`foreign`), and `status
107
107
 
108
108
  Pi's installer omits peers, so the standalone owner and workers load the Pi SDK (`pi-coding-agent`, `pi-ai`) from exact 0.84.3 optional dependencies.
109
109
  Package files may be hard links into the package manager's store (Bun's default install); Cadence binds their exact content, not their link count.
110
- Missing or unusable Pi sources refuse before an owner registers; a new owner without the SDK quarantines.
110
+ Missing or unusable Pi sources refuse before an owner registers; an owner that cannot load the SDK refuses at stage `pi-runtime` and exits.
111
111
  The no-project workbench starts without either an SDK runtime or Pi authentication; missing sources are reported only when a project is explicitly opened.
112
112
  For an explicit single-project definition, the older guards remain: no SDK means no definition, and missing `auth.json` means registered but waiting for Pi login.
113
113
 
@@ -152,7 +152,7 @@ It refuses without any effect when that token was edited since the start, so a p
152
152
  A stopped workspace selected earlier with a generated, environment or file token, or another port, is reselected to the file at its next `start --server-config`.
153
153
  Under the new owner reservation Cadence retires the previous token with a new access generation and replaces the selected port and token source; the history and every other selected setting stay.
154
154
  A different agent directory, host or proxy still conflicts, and a running owner refuses until it has stopped.
155
- An interrupted replacement keeps the reservation as a fence and is never repaired automatically, like any interrupted start.
155
+ An interrupted replacement is recovered by the next start like any interrupted start.
156
156
 
157
157
  ### Linux systemd system/root unit (no linger)
158
158
 
@@ -180,7 +180,7 @@ An old but inactive unit does not own a subsequently detached owner: install, re
180
180
  An unconfirmed or quarantined Stop, changed generated file, changed HOME/XDG state selection, a mask or a foreign unit prevents replacement/adoption.
181
181
  `restart`, `stop` and `uninstall` await the authenticated original owner Stop before invoking systemctl, so manager timeouts are never used as proof of settlement.
182
182
  Restart keeps disablement and does not unmask anything; uninstall removes only the generated definition, never data or linger.
183
- Direct systemctl shutdown still has the unit's ExecStop and timeout behavior described below; it is not an alternative recovery mechanism.
183
+ Direct systemctl shutdown still has the unit's ExecStop and timeout behavior described below.
184
184
 
185
185
  ### Independent named Linux services
186
186
 
@@ -196,7 +196,7 @@ A named service uses `~/.pi/agent/cadence/servers/<name>.env` under its effectiv
196
196
  A new named configuration requires `--port`; existing port/token lines are preserved, and a conflicting `--port` refuses instead of rewriting them.
197
197
  Edit the selected named file explicitly and restart that same `--name` to change its port.
198
198
  An occupied port still refuses rather than silently falling back; use `project open` when an automatically selected detached port is preferable.
199
- All projects still share one coordination state home for their account, retaining canonical overlap checks and uncertain fences.
199
+ All projects still share one coordination state home for their account, retaining canonical overlap checks.
200
200
 
201
201
  ### Linux systemd user unit
202
202
 
@@ -212,10 +212,11 @@ It carries `ConditionPathExists=` for the CLI; explicit single-project definitio
212
212
  `restart` stops an active unit with its loaded definition, rewrites the file and starts it, without enabling a disabled unit; a masked unit is never enabled or started.
213
213
  `ExecStop` is the authenticated orderly Stop, so admitted work is stopped, not detached; `stop` and `uninstall` (`disable --now`) use it too.
214
214
  `KillMode=mixed` sends systemd's SIGTERM only to the management client.
215
- An owner still settling after `TimeoutStopSec=90` (for example a quarantined one) is SIGKILLed, which is not orderly Stop and can leave recovery fences.
215
+ An owner still settling after `TimeoutStopSec=90` is SIGKILLed, which is not orderly Stop; the next start recovers what it leaves.
216
216
  `Restart=on-failure` is bounded to three starts in five minutes.
217
217
  Deterministic foreground startup refusals exit78 (`EX_CONFIG`), and newly generated units use `RestartPreventExitStatus=78` so these refusals do not loop.
218
- Ordinary unexpected process failures retain the bounded restart policy; a restart never removes fences.
218
+ Ordinary unexpected process failures retain the bounded restart policy, and every restart first recovers what the failed owner left.
219
+ An owner that hits a runtime fault stops its own workers and exits 75; its unit restarts it after `RestartSec=10`.
219
220
  Existing generated units get this setting on the next successful explicit install/restart, not by package installation or recovery.
220
221
  Without linger, user units start at login and stop at logout through the same Stop.
221
222
  Linger starts the user manager, with every enabled user unit, at boot and keeps it running after logout.
@@ -229,9 +230,9 @@ It bakes the same PATH, LANG/LC_ALL, XDG_STATE_HOME and PI_CODING_AGENT_DIR as t
229
230
  For an explicit single-project service, missing `auth.json` leaves the agent written but not loaded; the project-less workbench can start without Pi login.
230
231
  Because launchd stops a job only by signaling it, `restart`, `stop`, `uninstall` and a reinstall first stop the owner through the authenticated orderly Stop, and change nothing when that Stop is `unconfirmed`, `quarantined` or `unavailable`.
231
232
  Only then does Cadence unload (`bootout`) or replace the agent; `stop` leaves it loaded, and the finished job starts again at the next login.
232
- There is no `KeepAlive`: launchd has no start limit, and a restart never clears fences, so a failed owner stays down until `cadence server restart`.
233
+ There is no `KeepAlive`: a failed owner stays down until the next login or `cadence server restart`, which recovers it; an owner that faults at runtime starts its own replacement.
233
234
  `restart` keeps a `launchctl disable` and reports `disabled` without loading.
234
- At logout or shutdown launchd signals the job, which is not orderly Stop and can leave recovery fences; run `cadence server stop` first while work is running.
235
+ At logout or shutdown launchd signals the job, which is not orderly Stop: running work is interrupted, and the next start recovers the owner automatically.
235
236
 
236
237
  ### Windows login Run entry
237
238
 
@@ -242,7 +243,7 @@ It bakes only a custom PI_CODING_AGENT_DIR, because the login environment alread
242
243
  For an explicit single-project service, missing `auth.json` leaves the entry registered but starts nothing; the project-less workbench can start without Pi login.
243
244
  Nothing supervises the detached owner, so a failed owner is not restarted, and every `restart`, `stop`, `uninstall` and reinstall first runs the authenticated orderly Stop and changes nothing unless it is confirmed; Cadence never kills a process.
244
245
  A `Cadence` Run value without Cadence's launcher is `foreign` and left unchanged; registry values are only tested for presence, because `reg.exe` prints them in the console code page.
245
- At sign-out or shutdown Windows ends the owner without orderly Stop, which can leave recovery fences; run `cadence server stop` first while work is running.
246
+ At sign-out or shutdown Windows ends the owner without orderly Stop: running work is interrupted, and the next start recovers the owner automatically.
246
247
 
247
248
  ## Status, departure and Stop
248
249
 
@@ -255,10 +256,10 @@ At sign-out or shutdown Windows ends the owner without orderly Stop, which can l
255
256
  Checks, native process outcomes and human review remain separate facts.
256
257
  - `unconfirmed` means startup/Stop acknowledgment or settlement was not established within its reporting bound.
257
258
  It is not permission to restart, kill a guessed PID or remove a lock.
258
- `quarantined` preserves uncertain process/auth/storage work and prevents unsafe writers.
259
+ `quarantined` means the owner could not settle some process/auth/storage work: it refuses new work, lets the original work settle, stops its own workers and restarts itself (recovered automatically).
259
260
  - A host that refuses during `start` reports `{"status":"refused","error":<code>,"stage":<stage>}` and exits without registering an owner.
260
261
  Stages are `package` (installed web assets or Agent profiles), `pi-agent-directory`, `pi-auth`, `pi-models` and `service`; stderr names the selected Pi source to fix.
261
- `quarantined` with stage `pi-runtime` or `service` means the owner registered first; it keeps its fences for informed recovery.
262
+ `quarantined` with stage `pi-runtime` or `service` means the owner registered first; it exits, and the next start recovers its registration.
262
263
  Reports carry fixed codes and stages only, never paths, configuration bytes or OS error text.
263
264
  - The selected Pi agent directory must be a real directory (not a symlink) owned by you; `auth.json` and `models.json` must be your own single-link regular files.
264
265
  Their mode bits are not checked: Pi creates its agent directory with your umask (0755, or 0775 under umask 002), and Cadence's own state stays 0700 regardless.
@@ -271,17 +272,17 @@ At sign-out or shutdown Windows ends the owner without orderly Stop, which can l
271
272
  The owner loads `models.json` once at start: after it changes, the model catalog, the chosen model and the workspace view refuse as `stale` (credential changes in `auth.json` are read fresh).
272
273
  Restart the service to reload it (`cadence server restart`, or `stop` then `start`), then choose the owner model again.
273
274
  - A foreground owner can be managed from another terminal.
274
- Ctrl-C, SIGTERM, machine crash or terminal destruction is **not** the authenticated orderly Stop route and can leave recovery fences.
275
- A recovered `stopped` registration means acknowledged fence release, not retroactive native closure certification.
275
+ Ctrl-C, SIGTERM, machine crash or terminal destruction is **not** the authenticated orderly Stop route: running work is interrupted, and the next start recovers the owner automatically.
276
+ A recovered `stopped` registration records that the old owner was proven gone, not that its interrupted work completed.
276
277
 
277
- `diagnose` reports fixed pending/recording facts and `proxyAuth: direct|validated-proxy` when the owner is reachable; a blocked global registry instead reports its fixed lock reason and local recovery hint.
278
+ `diagnose` reports fixed pending/recording facts and `proxyAuth: direct|validated-proxy` when the owner is reachable; a busy global registry instead reports its fixed lock reason and names `cadence doctor`.
278
279
  No normal status/diagnosis prints access secrets, provider credentials or outbound proxy URLs.
279
280
  Keep the explicitly revealed token link private.
280
281
  Browser entry redirects to a token-free URL and uses an HttpOnly cookie; that does not retract a link already copied into history, a clipboard or external logs.
281
282
  File/generated token rotation invalidates old access and pending confirmations, not admitted work.
282
283
  Environment-token replacement requires explicit Stop/restart with changed environment.
283
284
  Inbound remote access still requires the selected same-host HTTPS reverse-proxy policy; outbound proxy routing below is independent.
284
- Browser access, even with a valid token, cannot perform local management/recovery or arbitrary manual shell checks.
285
+ Browser access, even with a valid token, cannot perform local management, `cadence doctor --fix` or arbitrary manual shell checks.
285
286
  The native client's local capability uses an owned direct loopback dispatcher, never the Pi process's provider proxy.
286
287
  Detach joins that client's original dispatcher closure without changing the parent's global dispatcher or canceling service work.
287
288
 
@@ -309,121 +310,54 @@ Checks inherit the owner's frozen ordinary launch environment (including the can
309
310
  Resolved model auth is never merged into checks.
310
311
  Local fabricated-provider plain/gzip routing tests are not paid-provider, all-adapter, Node22 or other-platform qualification.
311
312
 
312
- ## Informed local recovery
313
+ ## Automatic recovery
313
314
 
314
- ```sh
315
- # Account/workbench recovery (no cwd or project required):
316
- cadence workbench recover inspect
317
- cadence workbench recover resolve
318
- # A specific project owner:
319
- cadence recover inspect --workspace /absolute/project
320
- cadence recover resolve --workspace /absolute/project
315
+ Cadence is built for unclean endings: power loss, shutdown or logout while it runs, a crash, `kill -9`, or a supervisor's forced stop.
316
+ The next start (`cadence start`, `cadence project open`, the workbench, the background service, or native `/cadence`) recovers by itself and prints one line, for example:
317
+
318
+ ```text
319
+ Cadence recovered project /home/me/app automatically: the machine restarted while Cadence was running; released its locks, continued its history after the last complete entry.
321
320
  ```
322
321
 
323
- Inspection is readonly and handles the global registry lock before the selected workbench or project-owner recovery.
324
- Use the same account and state home as the service; workbench recovery also accepts `--server-config <path>` for a named or custom configuration.
325
- A recovery refusal reports a fixed code/stage or `blocked-preserve-state`, never raw state/configuration bytes.
326
- Do not delete state files or move to a different state home to bypass a refusal.
322
+ The WebUI shows the same recovery once at the top of the page.
323
+ Nothing is asked of you unless Cadence cannot prove the old process is gone.
327
324
 
328
- ### Residual global registry lock
325
+ What a start does, in order:
329
326
 
330
- `status` and `diagnose` report `reason: "registry-locked"` and the lock holder observation (`live`, `absent`, `ambiguous` or `unreadable`).
331
- Project Open reports `registry-locked` instead of an opaque `unavailable`; the folder picker explains the local recovery command.
332
- This lock is shared by every project in the selected state home, so trying a different project does not bypass it.
333
- Use the same account and `--state-home` as the owner; omit that flag only when it uses the default state home:
327
+ - It proves the previous owner gone: new locks record the machine's boot identity and the process's start identity, so a restart of the machine, an absent process or a reused process ID is certain.
328
+ - It stops worker processes the dead owner left running on the same boot, by their exact recorded identity; after a machine restart none can remain.
329
+ - It releases the dead owner's registration and locks, and every other dead reservation in the account (including one whose project directory was removed).
330
+ - It continues the history after the last complete entry (a batch cut short by the crash stays where it is, untouched).
331
+ - Private files that a crash left half-written are moved to `<state-home>/cadence-assistant-v1/recovery-archive-v1/` and the affected part starts afresh; archived files are deleted after 30 days (or sooner beyond 2 GiB).
334
332
 
335
- ```sh
336
- cadence recover inspect --workspace /absolute/project --state-home /absolute/state-home
337
- cadence recover resolve --workspace /absolute/project --state-home /absolute/state-home
338
- ```
333
+ Work that was running is marked interrupted (its outcome stays unknown and is never guessed); nothing is replayed, and worktrees and partial edits are kept.
334
+ Start an interrupted run or message again when you want it.
339
335
 
340
- With a registry lock present, inspection reports `scope: "global-registry"`, the exact lock-incarnation digest, holder PID/host, reservation count and any previous resolver's liveness.
341
- A project with no selected owner can be used to address this global operation; explicit `--workspace` must still name an existing suitable directory.
342
- Alternatively, `cadence workbench recover inspect|resolve --state-home <path>` addresses the same global lock without any project directory or workbench registration, including from outside a project or when every old project directory is gone.
343
- Resolve requires the same four fresh local-terminal acknowledgments described below, with the displayed lock-incarnation digest instead of a service epoch.
344
- The holder must be on this host and its PID probe must specifically report absence; age, a remote host, permission denial or a reused live PID never authorizes release.
345
-
346
- Recovery validates the registry and checks that unreserved project selections have no remaining owner/checkout fences.
347
- A normally stopped project whose working directory was later removed is checked through its retained private metadata; recovery neither requires nor reconstructs that directory.
348
- A still-reserved project whose working directory was removed also remains a valid retained reservation: canonical existing ancestors and overlap/endpoint constraints are checked without requiring the missing tail.
349
- Its reservation, owner/checkout locks and history remain fenced and unchanged; releasing the global lock only restores access to independent projects.
350
- Dangling/rebound aliases, permission errors and corrupt entries are not treated as absent directories.
351
- An interrupted reservation remains reserved even when its project initialization is incomplete: unrelated projects can reopen, but that project's uncertain resources are not repaired.
352
- An older release interrupted after removing a reservation but before releasing its project locks is unsafe and remains blocked, as do corrupt/partial registry bytes, linked files and changed resources.
353
- Inspection reports `blocked-preserve-state` when it cannot establish those conditions; preserve that state for manual diagnosis rather than deleting locks.
354
-
355
- Before releasing only `registry.lock`, the resolver exclusively creates, flushes and closes a private receipt under `service-registry-v1/recovery-v1/<lock-incarnation>/<attempt>.json`, retaining the original lock bytes and acknowledged digest.
356
- It then rechecks the lock incarnation, registry, unreserved project selections, receipt and terminal authority.
357
- Concurrent resolvers cannot release a successor lock.
358
- If a resolver dies after the receipt but before release, a new inspection and fresh terminal confirmation may append the next attempt after that resolver's same-host PID is absent; neither receipt nor lock is silently reclaimed.
359
- Torn or unsafe receipts remain preserved and refused.
360
- Inspection is bounded to 1024 namespace entries and 128 attempts for one lock incarnation; it never evicts evidence to make space.
361
-
362
- All project reservations, owner locks, checkout locks, capabilities, history and unknown outcomes remain unchanged.
363
- No Open/Start is replayed, no process is signaled, and no service is installed or started by recovery.
364
- After success, explicitly open the desired project again.
365
- If its own owner is still fenced, run `recover inspect` again and follow project-owner recovery separately; restoring global registry access does not certify or release that owner's work.
366
-
367
- ### Workbench recovery
368
-
369
- After a crash or failed Stop, an `alive`, `stopping` or `quarantined` registration may outlive its gateway process.
370
- Do not keep restarting it or delete `workbenches-v1`; inspect the exact selection instead:
336
+ An owner that hits a fault while running (for example a worker that will not exit, or a history write that failed) stops accepting work, lets its work settle, stops its own workers and restarts itself; the restart recovers as above.
337
+ It restarts itself at most three times in a row.
338
+
339
+ ### cadence doctor
371
340
 
372
341
  ```sh
373
- cadence workbench recover inspect --server-config /absolute/cadence.env --state-home /absolute/state-home
374
- cadence workbench recover resolve --server-config /absolute/cadence.env --state-home /absolute/state-home
342
+ cadence doctor # readonly: what is left over and what recovered recently
343
+ cadence doctor --fix # also handle the cases below, after one confirmation
344
+ cadence doctor --fix --yes # the same without the prompt, for scripts
345
+ cadence doctor --workspace /absolute/project --state-home /absolute/state-home
375
346
  ```
376
347
 
377
- If the first inspection reports `scope: "global-registry"`, resolve that lock first, then inspect again for `scope: "workbench"`.
378
- Prevent the supervisor from repeatedly launching the old process while recovering; use its original account/scope and inspect processes externally.
379
- A live owner, a known remote host, permission-denied PID probe, changed bytes/incarnations, linked or partial state, or a live/ambiguous resolver refuses release.
380
- PID absence alone never grants recovery; the same four fresh terminal acknowledgments bind the exact workbench epoch and selected resources.
381
-
382
- New registrations record their hostname.
383
- Older registrations did not: inspection explicitly reports `legacyHost: true`, `host: null` and `local-host-confirmation-required`, not proven same-host absence.
384
- Resolve then additionally requires typing `local-host` after independently establishing that this state originated on this host and was not copied/shared from another host.
385
- If that provenance cannot be established, do not confirm; preserve the state for manual diagnosis.
386
- This attestation cannot override a recorded foreign hostname or a live/ambiguous local PID.
387
- No `--yes`, piped or pasted answers, browser recovery or implicit takeover is supported.
388
-
389
- Before release, an exclusive immutable receipt under `workbenches-v1/<configuration-hash>/recovery-v1/<lock-incarnation>/<attempt>.json` is flushed and closed.
390
- The resolver then rechecks the resources, receipt, owner absence and fresh terminal authority, marks only the gateway registration stopped and releases only its exact gateway lock.
391
- Concurrent resolvers cannot release a successor lock; a resolver that crashed after its receipt can be superseded only after its same-host PID is absent and another fresh confirmation.
392
- A stopped registration is eligible only when its pinned recovery receipt belongs to that exact lock incarnation; an interrupted new startup cannot borrow the predecessor's dead PID to release its new fence.
393
- Partial/corrupt receipts remain preserved; there are at most128attempts per lock incarnation, with no evidence eviction.
394
-
395
- Recovery also acknowledges and retires the gateway's **exact old project associations**, recorded by their reference-file incarnations in that receipt.
396
- It does not delete those recent-project references or settle/stop their owners: all project registrations, reservations, locks, opening receipts and history remain unchanged, including unknown outcomes and unfinished Open operations.
397
- This prevents a deleted old project from immediately quarantining the recovered workbench again on its next Stop.
398
- Old references require a new explicit **Open** before proxy access or whole-workbench Stop associates them again; merely restarting, viewing the recent list or recovering never replays an Open.
399
- After recovery, explicitly start/restart the workbench.
400
- Separately inspect/recover any still-fenced project owner before opening it.
401
- The receipt is operator acknowledgment, not proof of native process closure or credential/storage correctness.
402
-
403
- ### Project-owner recovery
404
-
405
- With no registry lock present, inspection displays the epoch, exact coordination/history digests, owner PID/host observation and independent process/credential/storage uncertainty.
406
- Those digests identify preserved resources, not successful work or credential validity.
407
- A journal with no unmatched intent does not prove every escaped descendant or external credential writer stopped.
408
-
409
- Before resolve, independently inspect and stop old processes externally, inspect the selected Pi credential file and SDK locks, and inspect retained storage/history.
410
- Cadence does not signal arbitrary processes, reclaim SDK locks, repair credentials, truncate damaged history, migrate old formats or delete old resources.
411
- Resolve requires a native local TTY, four fresh typed responses within120seconds and the same unchanged resources: separate process/escaped-descendant, credential/writeback and storage acknowledgments, then the exact displayed epoch.
412
- There is no `--yes`, piped-input confirmation, browser body boolean, HTTP recovery endpoint or pasted multiline authorization.
413
- An SSH terminal executes locally on the owner host; this is an operator interaction boundary, not proof of a physical human's identity.
414
-
415
- Only a same-host owner whose PID probe specifically reports absence is eligible; live, remote, ambiguous, refreshed or partial resources refuse.
416
- Absence alone never clears anything.
417
- The original durable append/flush/close of a new fenced recovery fact completes first.
418
- Then exact registration and registry state are updated, and the exact capability, owner lock and associated v1 lock are released in order, rechecking remaining resources, history and local authority before every effect.
419
- Other workspace reservations remain untouched.
420
- The v1 lock is last.
421
- An unrelated legacy lock is never eligible.
422
-
423
- Recovery preserves old unknown outcomes and damaged bytes.
424
- Its result is **operator acknowledgment, not certification**; restart chooses a new epoch and never replays queues/auth work.
425
- New work requires fresh plan/action consent.
426
- Failure/crash between append and release can leave a durable bridge and partial fences.
427
- A refused command is not proof of no effects.
428
- Reinspect and preserve that state; reading/restart does not repair it, and automatic retry/reset is not a safe resolution.
429
- Recognized old history formats stay readonly and are not migrated to enable recovery.
348
+ `cadence recover inspect|resolve` remain as aliases of `doctor` and `doctor --fix`.
349
+ Use the same account and state home as the service.
350
+
351
+ Only these cases wait for you, each named with one sentence and the next step:
352
+
353
+ - **Running but not responding**: the owner process exists and is Cadence's own (its start identity matches) but does not answer.
354
+ `--fix` stops it (TERM, then KILL) and recovers.
355
+ An interactive `cadence start` asks the same question.
356
+ - **Another machine**: the state names a different host (a shared or copied state home).
357
+ Cadence cannot observe that process; `--fix` takes the state over without touching any process.
358
+ Confirm only if that machine no longer runs Cadence with this state.
359
+ - **Unidentifiable process**: a process with the recorded ID exists and Cadence cannot tell whether it is the old owner (older locks on some platforms).
360
+ If no Cadence is running, `--fix` takes the state over.
361
+ - **Workers that did not stop**: leftover workers ignored TERM and KILL; stop them yourself (or restart the machine) and start again.
362
+
363
+ Doctor never deletes a project, its history or anything outside Cadence's private state; damaged private files go to the archive, and the doctor report names it.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@abelxiaoxing/cadence",
3
- "version": "2.0.25",
3
+ "version": "2.0.26",
4
4
  "description": "Conversation-driven Pi workspace: persistent native and WebUI workflows, explicit plan approval, checks and local delivery.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -78,7 +78,8 @@
78
78
  "src/coordinator-tools.mjs",
79
79
  "src/workspace.ts",
80
80
  "src/data-shapes.ts",
81
- "src/service/checkout-lock.ts"
81
+ "src/service/checkout-lock.ts",
82
+ "src/service/liveness.ts"
82
83
  ],
83
84
  "pi": {
84
85
  "extensions": [
@@ -408,7 +408,7 @@ export function registerCadenceMode(pi: ExtensionAPI): void {
408
408
  // A signal (including the timeout) reports code 0 from Pi's exec.
409
409
  if (launched.killed || launched.code !== 0)
410
410
  throw new Error(
411
- `Cadence project owner could not start${launched.killed ? " before its startup deadline" : ""}.${bun ? " Pi runs on Bun here; on-demand startup needs Node.js 22.19+ on PATH." : ""} No install, fallback or uncertain-lock cleanup was attempted. Use cadence-service diagnose --workspace <path> for details.`,
411
+ `Cadence project owner could not start${launched.killed ? " before its startup deadline" : ""}.${bun ? " Pi runs on Bun here; on-demand startup needs Node.js 22.19+ on PATH." : ""}${launched.stderr.trim() ? `\n${launched.stderr.trim().split("\n").slice(-3).join("\n")}` : ""}\nDetails: cadence doctor --workspace ${JSON.stringify(ctx.cwd)}`,
412
412
  );
413
413
  let workspace = effectiveWorkspace(ctx.cwd).workspace;
414
414
  try {
@@ -446,18 +446,29 @@ export function registerCadenceMode(pi: ExtensionAPI): void {
446
446
  const observed = new ServiceCoordination(
447
447
  effectiveWorkspace(ctx.cwd).workspace,
448
448
  ).inspect();
449
- if (
450
- (observed.status === "absent" || observed.status === "stopped") &&
451
- allowStart
452
- ) {
449
+ const notRunning =
450
+ observed.status === "absent" || observed.status === "stopped";
451
+ if (notRunning && allowStart) {
453
452
  await launchOwner(ctx);
454
453
  check();
455
454
  }
456
455
  try {
457
456
  await connection.attach(ctx.cwd);
458
457
  } catch (error) {
459
- reattach = true;
460
- throw error;
458
+ if (notRunning || !allowStart) {
459
+ reattach = true;
460
+ throw error;
461
+ }
462
+ // A registration left by an owner that died (machine restart, crash)
463
+ // does not answer; opening the project recovers it automatically.
464
+ await launchOwner(ctx);
465
+ check();
466
+ try {
467
+ await connection.attach(ctx.cwd);
468
+ } catch (again) {
469
+ reattach = true;
470
+ throw again;
471
+ }
461
472
  }
462
473
  check();
463
474
  })();
@@ -2,5 +2,5 @@
2
2
  export const nativeBuild = {
3
3
  kind: "native",
4
4
  contractHash:
5
- "b855efce6a9694452dd77fcdad0b5dbecce617643cc81ac7c55dafb0f2ca7848",
5
+ "663f71a44cd33f95377644cfc10c2cb657298f191b36d6ca05b810e849c54e68",
6
6
  } as const;
@@ -17,7 +17,7 @@ import { HttpServiceTransport } from "./service-http-client.ts";
17
17
  import { effectiveWorkspace } from "./workspace.ts";
18
18
 
19
19
  export const MANAGEMENT_GUIDANCE =
20
- "Cadence owner absent, unavailable or incompatible. Explicitly run cadence-service status --workspace <path>, cadence-service diagnose --workspace <path>, or cadence-service start --workspace <path>. Stop older Cadence sessions first. For retained fences use cadence-service recover inspect; resolve requires a local terminal and separate process, credential and storage acknowledgments. Never treat unavailable/quarantined as stopped. No service was started and no embedded executor is available.";
20
+ "Cadence owner absent, unavailable or incompatible. Start it with cadence start --workspace <path> (it recovers automatically from an unclean shutdown), or see what is wrong with cadence doctor --workspace <path>. Stop older Cadence sessions first. No service was started and no embedded executor is available.";
21
21
 
22
22
  const UNIT = "cadence.service";
23
23
  const LABEL = "com.abelxiaoxing.cadence";
@@ -13,7 +13,7 @@ import {
13
13
  unlinkSync as unlinkSync2,
14
14
  writeFileSync as writeFileSync2
15
15
  } from "node:fs";
16
- import { hostname as hostname2 } from "node:os";
16
+ import { hostname as hostname3 } from "node:os";
17
17
  import { join as join3 } from "node:path";
18
18
 
19
19
  // src/data-shapes.ts
@@ -305,8 +305,77 @@ import {
305
305
  unlinkSync,
306
306
  writeFileSync
307
307
  } from "node:fs";
308
- import { hostname } from "node:os";
308
+ import { hostname as hostname2 } from "node:os";
309
309
  import { dirname as dirname2, join as join2 } from "node:path";
310
+
311
+ // src/service/liveness.ts
312
+ import { spawnSync as spawnSync2 } from "node:child_process";
313
+ import { readFileSync } from "node:fs";
314
+ import { hostname, uptime } from "node:os";
315
+ var LEGACY_BOOT_MARGIN_MS = 10 * 60 * 1000;
316
+ var cachedBoot;
317
+ function bootId() {
318
+ if (cachedBoot !== undefined)
319
+ return cachedBoot;
320
+ cachedBoot = null;
321
+ try {
322
+ if (process.platform === "linux") {
323
+ const id = readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
324
+ if (/^[0-9a-f-]{36}$/.test(id))
325
+ cachedBoot = id;
326
+ } else if (process.platform === "darwin") {
327
+ const r = spawnSync2("sysctl", ["-n", "kern.bootsessionuuid"], {
328
+ encoding: "utf8",
329
+ timeout: 2000
330
+ });
331
+ const id = r.status === 0 ? r.stdout.trim() : "";
332
+ if (/^[0-9A-Fa-f-]{36}$/.test(id))
333
+ cachedBoot = id;
334
+ }
335
+ } catch {
336
+ cachedBoot = null;
337
+ }
338
+ return cachedBoot;
339
+ }
340
+ function linuxStat(pid) {
341
+ try {
342
+ const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
343
+ return stat.slice(stat.lastIndexOf(")") + 2).trim().split(" ");
344
+ } catch {
345
+ return null;
346
+ }
347
+ }
348
+ function processStart(pid) {
349
+ if (!Number.isSafeInteger(pid) || pid < 1)
350
+ return null;
351
+ if (process.platform === "linux") {
352
+ const fields = linuxStat(pid);
353
+ const ticks = fields?.[19];
354
+ return ticks && /^\d+$/.test(ticks) ? `ticks:${ticks}` : null;
355
+ }
356
+ if (process.platform === "darwin") {
357
+ const r = spawnSync2("ps", ["-o", "lstart=", "-p", String(pid)], {
358
+ encoding: "utf8",
359
+ timeout: 2000,
360
+ env: { ...process.env, LC_ALL: "C" }
361
+ });
362
+ const value = r.status === 0 ? r.stdout.trim() : "";
363
+ return value ? `lstart:${value}` : null;
364
+ }
365
+ return null;
366
+ }
367
+ var cachedSelf;
368
+ function selfMark() {
369
+ cachedSelf ??= processStart(process.pid);
370
+ return {
371
+ pid: process.pid,
372
+ host: hostname(),
373
+ boot: bootId(),
374
+ started: cachedSelf
375
+ };
376
+ }
377
+
378
+ // src/service/checkout-lock.ts
310
379
  class CheckoutStore {
311
380
  root;
312
381
  privateRoot;
@@ -357,13 +426,16 @@ class CheckoutStore {
357
426
  acquire() {
358
427
  this.inspectPrivate();
359
428
  safeDirectory(this.privateRoot, true);
429
+ const mark = selfMark();
360
430
  const lock = {
361
431
  schema: "cadence-assistant-lock/1",
362
432
  token: randomUUID(),
363
433
  runId: randomUUID(),
364
434
  pid: process.pid,
365
- host: hostname(),
366
- root: this.root
435
+ host: hostname2(),
436
+ root: this.root,
437
+ boot: mark.boot,
438
+ started: mark.started
367
439
  };
368
440
  let fd;
369
441
  try {
@@ -384,7 +456,7 @@ class CheckoutStore {
384
456
  throw new Error("Coordination lock changed");
385
457
  }
386
458
  release(lock) {
387
- if (lock.pid !== process.pid || lock.host !== hostname())
459
+ if (lock.pid !== process.pid || lock.host !== hostname2())
388
460
  throw new Error("Only the owning process may release a lock");
389
461
  this.assertOwner(lock);
390
462
  retryWindowsFileOperation(() => unlinkSync(this.lockPath), () => this.assertOwner(lock));
@@ -406,7 +478,7 @@ class TaskStore extends CheckoutStore {
406
478
  this.tasks = join3(this.root, ".cadence", "assistant", "tasks");
407
479
  }
408
480
  recover(expected, acknowledgeEscapedProcesses) {
409
- if (!acknowledgeEscapedProcesses || expected.host !== hostname2())
481
+ if (!acknowledgeEscapedProcesses || expected.host !== hostname3())
410
482
  throw new Error("Recovery requires informed local-owner confirmation");
411
483
  this.assertOwner(expected);
412
484
  try {
@@ -458,7 +530,7 @@ class TaskStore extends CheckoutStore {
458
530
  if (Buffer.byteLength(text) > MAX_RECORD_BYTES)
459
531
  throw new Error("Task record limit");
460
532
  const lock = held ?? this.acquire();
461
- if (lock.pid !== process.pid || lock.host !== hostname2())
533
+ if (lock.pid !== process.pid || lock.host !== hostname3())
462
534
  throw new Error("Not the lock owner");
463
535
  try {
464
536
  this.assertOwner(lock);
@@ -515,7 +587,7 @@ class TaskStore extends CheckoutStore {
515
587
  return data ? diagnosticText(data.toString("utf8"), 65536) : "Private diagnostic unavailable";
516
588
  }
517
589
  writeLog(lock, output) {
518
- if (lock.pid !== process.pid || lock.host !== hostname2())
590
+ if (lock.pid !== process.pid || lock.host !== hostname3())
519
591
  throw new Error("Not the lock owner");
520
592
  this.assertOwner(lock);
521
593
  const name = `${randomUUID2()}.log`;
@@ -23,6 +23,7 @@ import {
23
23
  retryWindowsFileOperation,
24
24
  safeDirectory,
25
25
  } from "../workspace.ts";
26
+ import { selfMark } from "./liveness.ts";
26
27
  export interface CheckoutLock {
27
28
  schema: "cadence-assistant-lock/1";
28
29
  token: string;
@@ -30,6 +31,9 @@ export interface CheckoutLock {
30
31
  host: string;
31
32
  runId: string;
32
33
  root: string;
34
+ /** Liveness marks (absent in locks written by older releases). */
35
+ boot?: string | null;
36
+ started?: string | null;
33
37
  }
34
38
  export class CheckoutStore {
35
39
  readonly root: string;
@@ -100,6 +104,7 @@ export class CheckoutStore {
100
104
  acquire(): CheckoutLock {
101
105
  this.inspectPrivate();
102
106
  safeDirectory(this.privateRoot, true);
107
+ const mark = selfMark();
103
108
  const lock: CheckoutLock = {
104
109
  schema: "cadence-assistant-lock/1",
105
110
  token: randomUUID(),
@@ -107,6 +112,8 @@ export class CheckoutStore {
107
112
  pid: process.pid,
108
113
  host: hostname(),
109
114
  root: this.root,
115
+ boot: mark.boot,
116
+ started: mark.started,
110
117
  };
111
118
  let fd: number;
112
119
  try {