@1agh/maude 0.60.7 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/studio/acp/index.ts +1 -0
- package/apps/studio/ai-banner.tsx +1 -0
- package/apps/studio/annotations-context-toolbar.tsx +3 -1
- package/apps/studio/annotations-layer.tsx +33 -16
- package/apps/studio/api.ts +108 -18
- package/apps/studio/artboard-guides-overlay.tsx +5 -1
- package/apps/studio/assets-s3.ts +6 -1
- package/apps/studio/bin/_import-figma.mjs +8 -3
- package/apps/studio/build.ts +1 -1
- package/apps/studio/canvas-artifacts.ts +21 -0
- package/apps/studio/canvas-build.ts +14 -9
- package/apps/studio/canvas-comment-mount.tsx +27 -30
- package/apps/studio/canvas-icons.tsx +1 -1
- package/apps/studio/canvas-lib.tsx +94 -5
- package/apps/studio/canvas-list-watch.ts +14 -1
- package/apps/studio/canvas-shell.tsx +8 -0
- package/apps/studio/client/app.jsx +224 -45
- package/apps/studio/client/panels/GitPanel.jsx +121 -13
- package/apps/studio/client/panels/SettingsPanel.jsx +1 -1
- package/apps/studio/client/panels/SyncConsentDialog.jsx +182 -0
- package/apps/studio/client/panels/SyncPanel.jsx +595 -1
- package/apps/studio/client/styles/3-shell-maude.css +38 -0
- package/apps/studio/clip-ops.ts +8 -1
- package/apps/studio/cloud/endpoints.ts +265 -3
- package/apps/studio/collab/origins.ts +3 -1
- package/apps/studio/comments-overlay.tsx +5 -0
- package/apps/studio/config.schema.json +24 -0
- package/apps/studio/context-menu.tsx +40 -27
- package/apps/studio/context.ts +57 -8
- package/apps/studio/cursors-overlay.tsx +25 -13
- package/apps/studio/dist/client.bundle.js +686 -686
- package/apps/studio/dist/comment-mount.js +2 -2
- package/apps/studio/dist/styles.css +1 -1
- package/apps/studio/exporters/jobs.ts +77 -15
- package/apps/studio/exporters/remote.ts +190 -0
- package/apps/studio/exporters/video-encode-lib.ts +10 -4
- package/apps/studio/figma/to-strokes.ts +11 -9
- package/apps/studio/gifenc.d.ts +51 -0
- package/apps/studio/git/log-format.ts +88 -0
- package/apps/studio/git/safe-rel.ts +96 -0
- package/apps/studio/git/service.ts +46 -27
- package/apps/studio/hmr-broadcast.ts +10 -0
- package/apps/studio/http.ts +384 -6
- package/apps/studio/participants-chrome.tsx +1 -0
- package/apps/studio/photo-store.ts +7 -0
- package/apps/studio/react-augment.d.ts +17 -0
- package/apps/studio/runtime-bundle.ts +6 -1
- package/apps/studio/server.ts +43 -8
- package/apps/studio/sync/agent.ts +70 -82
- package/apps/studio/sync/asset-push.ts +28 -71
- package/apps/studio/sync/autocommit.ts +106 -5
- package/apps/studio/sync/cell-file-events.ts +117 -0
- package/apps/studio/sync/cell-pairing.ts +20 -5
- package/apps/studio/sync/cell-write-nudge.ts +244 -0
- package/apps/studio/sync/codec.ts +155 -3
- package/apps/studio/sync/cold-start-apply.ts +211 -0
- package/apps/studio/sync/ctl-heal.ts +253 -0
- package/apps/studio/sync/ctl-provider.ts +217 -0
- package/apps/studio/sync/decide-file.ts +335 -0
- package/apps/studio/sync/file-ledger.ts +581 -0
- package/apps/studio/sync/file-membership.ts +32 -0
- package/apps/studio/sync/file-plane.ts +1400 -0
- package/apps/studio/sync/file-pull.ts +41 -4
- package/apps/studio/sync/hub-link.ts +16 -1
- package/apps/studio/sync/hub-listing.ts +46 -0
- package/apps/studio/sync/hubs-config.ts +16 -0
- package/apps/studio/sync/index.ts +915 -308
- package/apps/studio/sync/journal-client.ts +200 -0
- package/apps/studio/sync/migrate-seed.ts +99 -67
- package/apps/studio/sync/poke.ts +50 -0
- package/apps/studio/sync/projection.ts +13 -0
- package/apps/studio/sync/pull-budget.ts +86 -0
- package/apps/studio/sync/settings.ts +110 -0
- package/apps/studio/sync/status.ts +68 -0
- package/apps/studio/sync/trash.ts +243 -0
- package/apps/studio/sync/untrusted.ts +30 -10
- package/apps/studio/test/_helpers.ts +8 -0
- package/apps/studio/test/canvas-build.test.ts +63 -0
- package/apps/studio/test/canvas-list-watch.test.ts +17 -0
- package/apps/studio/test/canvas-move-api.test.ts +31 -0
- package/apps/studio/test/canvas-origin-gate.test.ts +12 -0
- package/apps/studio/test/canvas-shell-build-error.test.ts +49 -0
- package/apps/studio/test/cloud-history-hardening.test.ts +165 -0
- package/apps/studio/test/cloud-history-posture.test.ts +230 -0
- package/apps/studio/test/cloud-session-role.test.ts +30 -0
- package/apps/studio/test/cloud-shell-surfaces.test.ts +39 -0
- package/apps/studio/test/cold-start-apply.test.ts +303 -0
- package/apps/studio/test/collab-stress.test.ts +9 -1
- package/apps/studio/test/export-lane.test.ts +245 -0
- package/apps/studio/test/fixtures/video-comp-fixture.tsx +1 -1
- package/apps/studio/test/git-log-format.test.ts +95 -0
- package/apps/studio/test/git-safe-rel.test.ts +132 -0
- package/apps/studio/test/hmr-broadcast.test.ts +26 -0
- package/apps/studio/test/peer-selection-follows-camera.test.tsx +131 -0
- package/apps/studio/test/shared-doc-cell-pairing.test.ts +5 -2
- package/apps/studio/test/sync-agent.test.ts +78 -0
- package/apps/studio/test/sync-asset-push.test.ts +71 -108
- package/apps/studio/test/sync-autocommit.test.ts +80 -0
- package/apps/studio/test/sync-cell-write-nudge.test.ts +346 -0
- package/apps/studio/test/sync-ctl-channel.test.ts +508 -0
- package/apps/studio/test/sync-decide-file.test.ts +420 -0
- package/apps/studio/test/sync-file-ledger.test.ts +334 -0
- package/apps/studio/test/sync-file-membership.test.ts +17 -1
- package/apps/studio/test/sync-file-plane.test.ts +976 -0
- package/apps/studio/test/sync-hub-listing.test.ts +46 -0
- package/apps/studio/test/sync-meta-codec.test.ts +76 -0
- package/apps/studio/test/sync-move-retirement.test.ts +231 -0
- package/apps/studio/test/sync-panel-surface.test.ts +20 -0
- package/apps/studio/test/sync-path-pull.test.ts +67 -1
- package/apps/studio/test/sync-pull-budget.test.ts +169 -0
- package/apps/studio/test/sync-seed-defers-to-hub.test.ts +83 -0
- package/apps/studio/test/sync-settings-routes.test.ts +195 -0
- package/apps/studio/test/sync-settings.test.ts +151 -0
- package/apps/studio/test/sync-status.test.ts +69 -0
- package/apps/studio/test/sync-trash.test.ts +132 -0
- package/apps/studio/test/workspace-containment.test.ts +45 -9
- package/apps/studio/tsconfig.json +9 -10
- package/apps/studio/use-annotation-resize.tsx +14 -3
- package/apps/studio/use-collab.tsx +3 -1
- package/apps/studio/whats-new.json +90 -0
- package/apps/studio/workspace-mode.ts +110 -62
- package/apps/studio/ws.ts +22 -1
- package/cli/bin/claude-design-server.mjs +19 -0
- package/cli/commands/design.mjs +25 -6
- package/cli/commands/hub-workspace.mjs +243 -22
- package/cli/commands/hub-workspace.test.mjs +171 -0
- package/cli/commands/hub.mjs +71 -1
- package/cli/lib/design-link.mjs +186 -1
- package/cli/lib/design-ownership.mjs +330 -0
- package/cli/lib/design-ownership.test.mjs +329 -0
- package/cli/lib/hubs-config.mjs +21 -0
- package/cli/lib/hubs-config.test.mjs +47 -1
- package/cli/lib/workspace-plan.mjs +298 -5
- package/cli/lib/workspace-plan.test.mjs +215 -1
- package/package.json +10 -10
- package/plugins/design/templates/_shell.html +43 -2
- package/plugins/design/templates/design-system-inspiration/SUB-AGENT-PROMPTS.md +1 -1
- package/plugins/design/templates/design-system-inspiration/core/preview/_motion-readme.md.tpl +1 -1
- package/apps/studio/server.mjs +0 -1312
- package/apps/studio/sync/asset-pull.ts +0 -210
- package/apps/studio/sync/asset-push-worker.ts +0 -84
- package/apps/studio/sync/asset-sweep.ts +0 -262
- package/apps/studio/test/sync-asset-pull.test.ts +0 -161
- package/apps/studio/test/sync-asset-push-worker.test.ts +0 -183
- package/apps/studio/test/sync-asset-sweep.test.ts +0 -243
|
@@ -0,0 +1,1400 @@
|
|
|
1
|
+
// The file plane — Sync v2 Increment 3 (DDR-226 §§3–7).
|
|
2
|
+
//
|
|
3
|
+
// ONE lane, both directions, one decision function. This module is what
|
|
4
|
+
// replaces the sprawl: the asset sweep, the fast-lane push, the asset pull and
|
|
5
|
+
// the manifest file-pull all answered a slice of "what should happen to this
|
|
6
|
+
// file", each with its own trigger, its own idea of the answer, and its own
|
|
7
|
+
// echo suppression. Here there is a single question asked per path —
|
|
8
|
+
// `decideFile(local, remote, ancestor)` — and a single place that carries out
|
|
9
|
+
// whatever it says.
|
|
10
|
+
//
|
|
11
|
+
// ── The pass ────────────────────────────────────────────────────────────────
|
|
12
|
+
//
|
|
13
|
+
// 1. Read the hub's journal from our cursor (or from 0 when re-anchoring).
|
|
14
|
+
// Its compaction IS the remote manifest; a delta read is the steady state.
|
|
15
|
+
// 2. Scan local disk through the classifier, hashing only files whose
|
|
16
|
+
// (size, mtime) no longer match the ledger's stat cache.
|
|
17
|
+
// 3. For every path in the union, ask `decideFile`.
|
|
18
|
+
// 4. Do what it said, in an order that survives being killed: bytes land
|
|
19
|
+
// before ancestors move, conflicts park before they adopt.
|
|
20
|
+
//
|
|
21
|
+
// ── Why the cursor is not just an optimization ──────────────────────────────
|
|
22
|
+
//
|
|
23
|
+
// A cursor means the hub can answer "what changed since 412" instead of
|
|
24
|
+
// "here is everything", which is what lets a poke be cheap enough to act on
|
|
25
|
+
// immediately. But it also carries the safety property: a cursor the hub
|
|
26
|
+
// cannot honour comes back as `reanchor`, and we then re-read from 0 rather
|
|
27
|
+
// than assuming nothing changed. "No cursor" must never render as "no news".
|
|
28
|
+
//
|
|
29
|
+
// ── Deletion (Increment 6) ──────────────────────────────────────────────────
|
|
30
|
+
//
|
|
31
|
+
// A deletion is a JOURNAL ROW, never an absence. Absence is not authority
|
|
32
|
+
// (DDR-076): a path missing from a page means "no news", and a file missing
|
|
33
|
+
// from a tree means "gone from THIS disk" — neither is a statement about what
|
|
34
|
+
// the project holds. So a delete travels as a tombstone with a CAS
|
|
35
|
+
// precondition, exactly like a write, and an edit that raced it wins.
|
|
36
|
+
//
|
|
37
|
+
// Nothing is ever unlinked. Losers go to `_trash/` on both ends, which is
|
|
38
|
+
// runtime state (DDR-115) and therefore never replicates — one person's delete
|
|
39
|
+
// must not become everyone's copy of the deleted file. On the hub the
|
|
40
|
+
// object-storage blob is content-addressed, so a delete leaves it
|
|
41
|
+
// unreferenced rather than destroyed.
|
|
42
|
+
//
|
|
43
|
+
// And the breakers are the load-bearing part, because the dangerous shapes are
|
|
44
|
+
// ordinary: a branch switch, a `git clean`, a half-finished restore. See
|
|
45
|
+
// `DELETE_BREAKER_MAX`.
|
|
46
|
+
|
|
47
|
+
import { createHash } from 'node:crypto';
|
|
48
|
+
import {
|
|
49
|
+
type Dirent,
|
|
50
|
+
existsSync,
|
|
51
|
+
mkdirSync,
|
|
52
|
+
readdirSync,
|
|
53
|
+
readFileSync,
|
|
54
|
+
realpathSync,
|
|
55
|
+
renameSync,
|
|
56
|
+
statSync,
|
|
57
|
+
writeFileSync,
|
|
58
|
+
} from 'node:fs';
|
|
59
|
+
import path from 'node:path';
|
|
60
|
+
|
|
61
|
+
import { conflictCopyName, decideFile, type FileState } from './decide-file.ts';
|
|
62
|
+
import type { DeliveryState, FileLedger } from './file-ledger.ts';
|
|
63
|
+
import {
|
|
64
|
+
type CanvasGroupLike,
|
|
65
|
+
classifyProjectFile,
|
|
66
|
+
type FileClass,
|
|
67
|
+
isFilePlaneClass,
|
|
68
|
+
} from './file-membership.ts';
|
|
69
|
+
import { fetchJournal, type JournalEntry } from './journal-client.ts';
|
|
70
|
+
import { createPullBudget } from './pull-budget.ts';
|
|
71
|
+
|
|
72
|
+
/** How long to wait for one file's bytes. Generous — these run to videos. */
|
|
73
|
+
const GET_TIMEOUT_MS = 120_000;
|
|
74
|
+
const PUT_TIMEOUT_MS = 120_000;
|
|
75
|
+
|
|
76
|
+
/** Refuse an implausible body rather than streaming it to disk. */
|
|
77
|
+
const MAX_FILE_BYTES = 512 * 1024 * 1024;
|
|
78
|
+
|
|
79
|
+
/** How many files one pass will move. The remainder is the next pass's work. */
|
|
80
|
+
const MAX_FILES_PER_PASS = 200;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* How many consecutive `reanchor` answers before we stop obeying them.
|
|
84
|
+
*
|
|
85
|
+
* A re-anchor is a full compaction read plus `pruneRemotes`, and it sets
|
|
86
|
+
* `degraded`, under which every differing path parks a copy of the hub's
|
|
87
|
+
* bytes. A hub that answers `reanchor` to everything is therefore a
|
|
88
|
+
* disk-filling primitive rather than a peer with a rotated epoch.
|
|
89
|
+
*/
|
|
90
|
+
export const REANCHOR_STORM_LIMIT = 5;
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* F-11 (post-1.0 burn-down) — how long a re-anchor hold lasts before ONE more
|
|
94
|
+
* attempt is allowed. The counter used to be reset only by a pass the hub did
|
|
95
|
+
* NOT answer `reanchor` to — but the held branch returns before that line, so
|
|
96
|
+
* once over the limit every later pass was held too and the plane was bricked
|
|
97
|
+
* until a desktop restart (its comment claimed otherwise). A hub that
|
|
98
|
+
* legitimately rotates its epoch six times (a cell restarted repeatedly, a
|
|
99
|
+
* restore drill) must converge eventually; a hostile hub that answers
|
|
100
|
+
* `reanchor` forever must still be capped. Time gives both: held for the
|
|
101
|
+
* window, then exactly one retry — a storm costs one full read per window,
|
|
102
|
+
* a real rotation recovers on the first quiet retry.
|
|
103
|
+
*/
|
|
104
|
+
export const REANCHOR_HOLD_RECOVERY_MS = 15 * 20_000; // 15 poll ticks (index.ts REMOTE_POLL_MS)
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* How many FIRST-ANCHOR conflicts one pass will resolve before it stops and asks.
|
|
108
|
+
*
|
|
109
|
+
* The flip-day shape, and it is structural rather than hypothetical: a project
|
|
110
|
+
* that has been linked with the file plane off has a hub `system/**` that is
|
|
111
|
+
* stale by construction, so the first pass with it on finds dozens of paths
|
|
112
|
+
* where both sides have content and this machine has never reconciled either —
|
|
113
|
+
* every one a `diverged` with no ancestor. Resolving them silently means a
|
|
114
|
+
* person opens their editor to a tree of `*.maude-conflict-*` files they did
|
|
115
|
+
* not ask for and cannot easily undo in bulk.
|
|
116
|
+
*
|
|
117
|
+
* Under the limit it is ordinary conflict handling. Over it, the pass holds
|
|
118
|
+
* everything, reports the count, and waits for a bulk keep-local / keep-cloud
|
|
119
|
+
* answer — with the Open-decision-2 fallback of proceeding conservatively
|
|
120
|
+
* (park, propagate nothing) if nobody answers, so a headless desktop cannot
|
|
121
|
+
* stall forever.
|
|
122
|
+
*/
|
|
123
|
+
export const FIRST_ANCHOR_STORM_LIMIT = 10;
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Deletion breakers — DDR-226 §8, and the only protection now that
|
|
127
|
+
* `propagateDeletes` ships ON rather than after a soak release.
|
|
128
|
+
*
|
|
129
|
+
* The shapes these exist for are ordinary, not exotic. A branch switch removes
|
|
130
|
+
* half the design folder; a `git clean` removes all of it; a botched restore on
|
|
131
|
+
* one machine looks exactly like a deliberate purge to every other. In each
|
|
132
|
+
* case the mechanism is working perfectly and the outcome is a disaster, so
|
|
133
|
+
* the rule is a rate, not a permission: past `MAX` files or `MAX_FRACTION` of
|
|
134
|
+
* what this machine tracks, in one pass, nothing is removed and the pass says
|
|
135
|
+
* what it was about to do.
|
|
136
|
+
*
|
|
137
|
+
* Both directions, because both are lossy. Outbound turns a local accident
|
|
138
|
+
* into everyone's; inbound turns a hostile or broken hub into a local wipe.
|
|
139
|
+
*/
|
|
140
|
+
export const DELETE_BREAKER_MAX = 10;
|
|
141
|
+
export const DELETE_BREAKER_MAX_FRACTION = 0.25;
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The window the budget is measured over, and the ceiling inside it.
|
|
145
|
+
*
|
|
146
|
+
* The first version of this breaker counted ONE PASS and reset completely on
|
|
147
|
+
* the next, which makes it a rate limit rather than a budget — and a rate
|
|
148
|
+
* limit is the wrong control here, because the thing it guards against is
|
|
149
|
+
* cumulative. Ten per pass, six passes a minute once a hub can poke, and a
|
|
150
|
+
* two-hundred-file project is gone in three minutes with the limit never once
|
|
151
|
+
* tripping. Two per pass was under every arm of it unconditionally, at every
|
|
152
|
+
* project size, forever.
|
|
153
|
+
*
|
|
154
|
+
* So the count is cumulative, windowed, and PERSISTED on the ledger —
|
|
155
|
+
* otherwise "restart the app" is the bypass. A window rather than a lifetime
|
|
156
|
+
* total because a real project does delete a lot of files eventually, just
|
|
157
|
+
* not forty of them in an hour without somebody meaning it.
|
|
158
|
+
*/
|
|
159
|
+
export const DELETE_BUDGET_WINDOW_MS = 60 * 60 * 1000;
|
|
160
|
+
export const DELETE_BUDGET_PER_WINDOW = 25;
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The proportion arm needs a small floor — in a project tracking one file,
|
|
164
|
+
* deleting that file is 100% — but the floor used to be the hole: at 3, two
|
|
165
|
+
* deletions per pass were under every arm forever. It is safe at 2 now only
|
|
166
|
+
* because the BUDGET arm catches the patient drain independently. Neither
|
|
167
|
+
* number is load-bearing alone; the three together are.
|
|
168
|
+
*/
|
|
169
|
+
export const DELETE_BREAKER_MIN_FOR_FRACTION = 2;
|
|
170
|
+
|
|
171
|
+
/** Walk depth ceiling — matches the classifier's own shape cap. */
|
|
172
|
+
const MAX_WALK_DEPTH = 8;
|
|
173
|
+
|
|
174
|
+
/** Directories no plane path can live under. */
|
|
175
|
+
const SKIPPED_DIRS = new Set(['_trash', '_history', '_untrusted', '_smoke', 'node_modules']);
|
|
176
|
+
|
|
177
|
+
export interface FilePlaneResult {
|
|
178
|
+
pulled: string[];
|
|
179
|
+
pushed: string[];
|
|
180
|
+
conflicts: { rel: string; copy: string | null }[];
|
|
181
|
+
/** Refused by re-classification or per-class admission. */
|
|
182
|
+
dropped: { rel: string; reason: string }[];
|
|
183
|
+
failed: { rel: string; reason: string }[];
|
|
184
|
+
/** Present and equal — the converged steady state. */
|
|
185
|
+
synced: number;
|
|
186
|
+
/** The hub asked us to start over. */
|
|
187
|
+
reanchored: boolean;
|
|
188
|
+
/** The pass stopped on the aggregate byte budget. */
|
|
189
|
+
budgetExhausted?: true;
|
|
190
|
+
/** Files removed this pass — `parked` names the `_trash/` copy when there is one. */
|
|
191
|
+
deleted: { rel: string; parked: string | null }[];
|
|
192
|
+
/** A delete burst tripped a breaker; nothing was removed. */
|
|
193
|
+
deleteHeld?: { direction: 'out' | 'in'; count: number; paths: string[] };
|
|
194
|
+
/** Consecutive `reanchor` answers tripped the storm limit; the pass held. */
|
|
195
|
+
reanchorHeld?: true;
|
|
196
|
+
/**
|
|
197
|
+
* More first-anchor conflicts than one pass will decide alone. The paths are
|
|
198
|
+
* listed so the panel can offer one keep-local / keep-cloud for all of them.
|
|
199
|
+
*/
|
|
200
|
+
firstAnchorHeld?: { count: number; paths: string[] };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export interface FilePlaneOptions {
|
|
204
|
+
designRoot: string;
|
|
205
|
+
hubUrl: string;
|
|
206
|
+
/** Read at call time — silent renewal swaps the credential in place. */
|
|
207
|
+
token: () => string;
|
|
208
|
+
ledger: FileLedger;
|
|
209
|
+
canvasGroups?: readonly CanvasGroupLike[];
|
|
210
|
+
/**
|
|
211
|
+
* Whether `code-module` entries may LAND here. Computed by the caller from
|
|
212
|
+
* LOCAL state (hubs.json role / loopback pairing) — never from anything the
|
|
213
|
+
* hub said. The hub gates the write side too now, but a gate on one side is
|
|
214
|
+
* half a gate, so both ask.
|
|
215
|
+
*/
|
|
216
|
+
allowCodeModules: boolean;
|
|
217
|
+
/** Names conflict copies. Same exposure class as `syncMeta.by` (hostname). */
|
|
218
|
+
label: string;
|
|
219
|
+
/** Increment 6. Off means a local absence is HELD, never propagated. */
|
|
220
|
+
propagateDeletes?: boolean;
|
|
221
|
+
/**
|
|
222
|
+
* The user's bulk answer to a first-anchor storm — `'keep-local'` pushes
|
|
223
|
+
* ours over theirs, `'keep-cloud'` takes theirs and parks ours. Absent means
|
|
224
|
+
* a storm holds and asks (see `FIRST_ANCHOR_STORM_LIMIT`).
|
|
225
|
+
*/
|
|
226
|
+
resolveFirstAnchor?: 'keep-local' | 'keep-cloud';
|
|
227
|
+
fetchImpl?: typeof fetch;
|
|
228
|
+
log?: Pick<Console, 'log' | 'warn'>;
|
|
229
|
+
now?: () => number;
|
|
230
|
+
maxPassBytes?: number;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
interface LocalFile {
|
|
234
|
+
rel: string;
|
|
235
|
+
abs: string;
|
|
236
|
+
hash: string;
|
|
237
|
+
size: number;
|
|
238
|
+
mtimeMs: number;
|
|
239
|
+
cls: FileClass;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const sha256 = (bytes: Uint8Array | string): string =>
|
|
243
|
+
createHash('sha256').update(bytes).digest('hex');
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Walk the design root and hash what the classifier admits.
|
|
247
|
+
*
|
|
248
|
+
* The stat cache is what makes this cheap: a converged project re-reads
|
|
249
|
+
* nothing, because every file's `(size, mtimeMs)` still matches the ledger.
|
|
250
|
+
*/
|
|
251
|
+
export function scanLocalFiles(
|
|
252
|
+
designRoot: string,
|
|
253
|
+
ledger: FileLedger,
|
|
254
|
+
canvasGroups?: readonly CanvasGroupLike[]
|
|
255
|
+
): Map<string, LocalFile> {
|
|
256
|
+
const found: { rel: string; abs: string; size: number; mtimeMs: number }[] = [];
|
|
257
|
+
|
|
258
|
+
const walk = (dir: string, rel: string, depth: number): void => {
|
|
259
|
+
if (depth > MAX_WALK_DEPTH) return;
|
|
260
|
+
let entries: Dirent[];
|
|
261
|
+
try {
|
|
262
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
263
|
+
} catch {
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
for (const entry of entries) {
|
|
267
|
+
const name = entry.name;
|
|
268
|
+
if (name.startsWith('.') || SKIPPED_DIRS.has(name)) continue;
|
|
269
|
+
if (name.startsWith('_') && entry.isDirectory()) continue;
|
|
270
|
+
const childRel = rel ? `${rel}/${name}` : name;
|
|
271
|
+
const childAbs = path.join(dir, name);
|
|
272
|
+
if (entry.isDirectory()) {
|
|
273
|
+
walk(childAbs, childRel, depth + 1);
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
// A symlink never enters the plane — on either side.
|
|
277
|
+
if (!entry.isFile()) continue;
|
|
278
|
+
let st: ReturnType<typeof statSync>;
|
|
279
|
+
try {
|
|
280
|
+
st = statSync(childAbs);
|
|
281
|
+
} catch {
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
if (st.size > MAX_FILE_BYTES) continue;
|
|
285
|
+
found.push({ rel: childRel, abs: childAbs, size: st.size, mtimeMs: st.mtimeMs });
|
|
286
|
+
}
|
|
287
|
+
};
|
|
288
|
+
walk(designRoot, '', 1);
|
|
289
|
+
|
|
290
|
+
// Classify against the walked SNAPSHOT, so the sibling-css split cannot race
|
|
291
|
+
// a concurrent write (the same discipline the hub manifest uses).
|
|
292
|
+
const names = new Set(found.map((f) => f.rel));
|
|
293
|
+
const opts = { canvasGroups, hasFile: (r: string) => names.has(r) };
|
|
294
|
+
|
|
295
|
+
const out = new Map<string, LocalFile>();
|
|
296
|
+
for (const f of found) {
|
|
297
|
+
const cls = classifyProjectFile(f.rel, opts);
|
|
298
|
+
if (!isFilePlaneClass(cls)) continue;
|
|
299
|
+
let hash = ledger.cachedHash(f.rel, f.size, f.mtimeMs);
|
|
300
|
+
if (hash === null) {
|
|
301
|
+
try {
|
|
302
|
+
hash = sha256(readFileSync(f.abs));
|
|
303
|
+
} catch {
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
ledger.noteLocal(f.rel, hash, f.size, f.mtimeMs);
|
|
307
|
+
}
|
|
308
|
+
out.set(f.rel, { rel: f.rel, abs: f.abs, hash, size: f.size, mtimeMs: f.mtimeMs, cls });
|
|
309
|
+
}
|
|
310
|
+
return out;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Fold journal entries into "what the hub currently holds", newest row wins.
|
|
315
|
+
*/
|
|
316
|
+
export function foldRemote(entries: JournalEntry[]): Map<string, JournalEntry> {
|
|
317
|
+
const out = new Map<string, JournalEntry>();
|
|
318
|
+
for (const e of entries) {
|
|
319
|
+
const prev = out.get(e.path);
|
|
320
|
+
if (!prev || e.seq > prev.seq) out.set(e.path, e);
|
|
321
|
+
}
|
|
322
|
+
return out;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
export interface FilePlane {
|
|
326
|
+
/** One full pass. Never throws. */
|
|
327
|
+
reconcile(): Promise<FilePlaneResult>;
|
|
328
|
+
/** Paths whose delivery state the panel should show. */
|
|
329
|
+
doruceka(): Record<string, DeliveryState>;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
export function createFilePlane(opts: FilePlaneOptions): FilePlane {
|
|
333
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
334
|
+
const log = opts.log ?? console;
|
|
335
|
+
const now = opts.now ?? Date.now;
|
|
336
|
+
const base = opts.hubUrl.replace(/\/+$/, '');
|
|
337
|
+
const { ledger, designRoot } = opts;
|
|
338
|
+
/**
|
|
339
|
+
* The design root with its own symlinks resolved. Containment is judged
|
|
340
|
+
* against THIS, not the configured string — otherwise a project reached
|
|
341
|
+
* through a symlinked path (a Syncthing tree, a `/tmp` fixture on macOS)
|
|
342
|
+
* fails its own containment check for every legitimate write.
|
|
343
|
+
*/
|
|
344
|
+
const realRoot = (() => {
|
|
345
|
+
try {
|
|
346
|
+
return realpathSync(designRoot);
|
|
347
|
+
} catch {
|
|
348
|
+
return designRoot;
|
|
349
|
+
}
|
|
350
|
+
})();
|
|
351
|
+
|
|
352
|
+
const auth = () => ({ authorization: `Bearer ${opts.token()}` });
|
|
353
|
+
|
|
354
|
+
/** Consecutive passes the hub answered `reanchor` to. Reset by any that did
|
|
355
|
+
* not — and decayed by time once held (F-11), so a hold is a window, never
|
|
356
|
+
* a brick. */
|
|
357
|
+
let reanchorsInARow = 0;
|
|
358
|
+
let reanchorHeldSince = 0;
|
|
359
|
+
|
|
360
|
+
/** Fetch one file's bytes and verify them against the hash we were promised. */
|
|
361
|
+
async function fetchVerified(
|
|
362
|
+
rel: string,
|
|
363
|
+
expectHash: string,
|
|
364
|
+
ceiling: number
|
|
365
|
+
): Promise<{ ok: true; bytes: Uint8Array } | { ok: false; reason: string; overCap?: true }> {
|
|
366
|
+
let res: Response;
|
|
367
|
+
try {
|
|
368
|
+
res = await fetchImpl(
|
|
369
|
+
`${base}/_project-file/${rel.split('/').map(encodeURIComponent).join('/')}`,
|
|
370
|
+
{ headers: auth(), signal: AbortSignal.timeout(GET_TIMEOUT_MS) }
|
|
371
|
+
);
|
|
372
|
+
} catch (err) {
|
|
373
|
+
return { ok: false, reason: (err as Error).message };
|
|
374
|
+
}
|
|
375
|
+
if (!res.ok) return { ok: false, reason: `HTTP ${res.status}` };
|
|
376
|
+
|
|
377
|
+
// The cap is consulted BEFORE the body exists, and again as it arrives.
|
|
378
|
+
// Buffering first and measuring after is not a cap at all: a hub answering
|
|
379
|
+
// with a body that never ends fills this process's heap for the whole
|
|
380
|
+
// timeout window and the check never gets to run. The hub's own
|
|
381
|
+
// `streamAndHash` has always had this shape; the asymmetry was the bug.
|
|
382
|
+
const cap = Math.max(0, Math.min(MAX_FILE_BYTES, ceiling));
|
|
383
|
+
const declared = Number(res.headers?.get?.('content-length') ?? Number.NaN);
|
|
384
|
+
if (Number.isFinite(declared) && declared > cap) {
|
|
385
|
+
return {
|
|
386
|
+
ok: false,
|
|
387
|
+
reason: `over the cap before a byte was read (${declared} B > ${cap} B)`,
|
|
388
|
+
overCap: true,
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const bytes = await readCapped(res, cap);
|
|
393
|
+
if (!bytes.ok) return bytes;
|
|
394
|
+
// The hub may REFUSE to serve; it must never be able to SUBSTITUTE.
|
|
395
|
+
const got = sha256(bytes.bytes);
|
|
396
|
+
if (got !== expectHash) {
|
|
397
|
+
return { ok: false, reason: 'content hash mismatch (racing a write?)' };
|
|
398
|
+
}
|
|
399
|
+
return { ok: true, bytes: bytes.bytes };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* Read a response body, abandoning it the moment it exceeds `cap`.
|
|
404
|
+
*
|
|
405
|
+
* Falls back to `arrayBuffer()` only when the response has no readable
|
|
406
|
+
* stream (a test double, or a runtime without one) — there the length check
|
|
407
|
+
* is still after the fact, but a double is not the threat.
|
|
408
|
+
*/
|
|
409
|
+
async function readCapped(
|
|
410
|
+
res: Response,
|
|
411
|
+
cap: number
|
|
412
|
+
): Promise<{ ok: true; bytes: Uint8Array } | { ok: false; reason: string; overCap?: true }> {
|
|
413
|
+
const body = res.body;
|
|
414
|
+
if (!body?.getReader) {
|
|
415
|
+
const all = new Uint8Array(await res.arrayBuffer());
|
|
416
|
+
if (all.byteLength > cap) {
|
|
417
|
+
return {
|
|
418
|
+
ok: false,
|
|
419
|
+
reason: `implausible size (${all.byteLength} B > ${cap} B)`,
|
|
420
|
+
overCap: true,
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
return { ok: true, bytes: all };
|
|
424
|
+
}
|
|
425
|
+
const reader = body.getReader();
|
|
426
|
+
const chunks: Uint8Array[] = [];
|
|
427
|
+
let total = 0;
|
|
428
|
+
try {
|
|
429
|
+
for (;;) {
|
|
430
|
+
const { done, value } = await reader.read();
|
|
431
|
+
if (done) break;
|
|
432
|
+
if (!value) continue;
|
|
433
|
+
total += value.byteLength;
|
|
434
|
+
if (total > cap) {
|
|
435
|
+
await reader.cancel().catch(() => {});
|
|
436
|
+
return {
|
|
437
|
+
ok: false,
|
|
438
|
+
reason: `over the cap mid-stream (${total} B > ${cap} B)`,
|
|
439
|
+
overCap: true,
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
chunks.push(value);
|
|
443
|
+
}
|
|
444
|
+
} catch (err) {
|
|
445
|
+
return { ok: false, reason: (err as Error).message };
|
|
446
|
+
}
|
|
447
|
+
const bytes = new Uint8Array(total);
|
|
448
|
+
let at = 0;
|
|
449
|
+
for (const c of chunks) {
|
|
450
|
+
bytes.set(c, at);
|
|
451
|
+
at += c.byteLength;
|
|
452
|
+
}
|
|
453
|
+
return { ok: true, bytes };
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* What the hub CLAIMS a row costs — a hint, never the fact.
|
|
458
|
+
*
|
|
459
|
+
* A missing or absurd `size` charges nothing, which is exactly the hole
|
|
460
|
+
* `chargeOverrun` closes: a hub declaring `size: 0` for two hundred 512 MB
|
|
461
|
+
* rows would otherwise land ~100 GB per pass against a budget that never
|
|
462
|
+
* moved — literally the arithmetic `pull-budget.ts` exists to prevent.
|
|
463
|
+
*/
|
|
464
|
+
function claimedSize(row?: { size?: number | null }): number {
|
|
465
|
+
const n = row?.size;
|
|
466
|
+
return typeof n === 'number' && Number.isFinite(n) && n > 0 ? n : 0;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Charge what actually arrived, over and above what was claimed.
|
|
471
|
+
*
|
|
472
|
+
* `file-pull.ts` — the lane this replaces — has always done this, with a
|
|
473
|
+
* comment saying an understated size must not buy a bigger transfer. The
|
|
474
|
+
* rule survives the rewrite.
|
|
475
|
+
*/
|
|
476
|
+
function chargeOverrun(
|
|
477
|
+
budget: ReturnType<typeof createPullBudget>,
|
|
478
|
+
rel: string,
|
|
479
|
+
actual: number,
|
|
480
|
+
claimed: number,
|
|
481
|
+
out: FilePlaneResult
|
|
482
|
+
): boolean {
|
|
483
|
+
if (actual <= claimed) return true;
|
|
484
|
+
if (budget.take(actual - claimed)) return true;
|
|
485
|
+
out.failed.push({
|
|
486
|
+
rel,
|
|
487
|
+
reason: `pass byte budget reached (declared ${claimed} B, sent ${actual} B)`,
|
|
488
|
+
});
|
|
489
|
+
out.budgetExhausted = true;
|
|
490
|
+
return false;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* `realpathSync` for a path that does not exist yet — resolve the deepest
|
|
495
|
+
* ancestor that DOES exist and re-attach the tail. `path.join` is lexical
|
|
496
|
+
* and never touches disk, which is exactly why it is not a containment check.
|
|
497
|
+
*/
|
|
498
|
+
function realpathOfDeepestExisting(p: string): string {
|
|
499
|
+
let cur = p;
|
|
500
|
+
for (;;) {
|
|
501
|
+
try {
|
|
502
|
+
// RE-ATTACH THE TAIL. Resolving the deepest existing ancestor and
|
|
503
|
+
// returning just that loses every directory below it that does not
|
|
504
|
+
// exist yet — which flattens `system/deep/a.css` to `a.css` on exactly
|
|
505
|
+
// the fresh-link path where none of those directories exist. The hub's
|
|
506
|
+
// own helper has always done this; this one was written without it.
|
|
507
|
+
return path.join(realpathSync(cur), path.relative(cur, p));
|
|
508
|
+
} catch {
|
|
509
|
+
const up = path.dirname(cur);
|
|
510
|
+
if (up === cur) return p;
|
|
511
|
+
cur = up;
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Where `rel` may actually be written, or null.
|
|
518
|
+
*
|
|
519
|
+
* The lane this replaced carried a lexical containment assertion; this one
|
|
520
|
+
* carried neither that nor the refusal to overwrite a non-file. Lexical
|
|
521
|
+
* traversal is still blocked upstream (the classifier's shape gate refuses
|
|
522
|
+
* `..`, absolutes, backslashes and control chars, and `journal-client.ts`
|
|
523
|
+
* re-validates independently) — but a SYMLINKED INTERMEDIATE DIRECTORY is a
|
|
524
|
+
* different question, and `writeFileSync` + `renameSync` follow those
|
|
525
|
+
* happily. `<designRoot>/system/ds/assets -> ~/.ssh` would land
|
|
526
|
+
* `system/ds/assets/config.css` outside the design root entirely.
|
|
527
|
+
*
|
|
528
|
+
* The hub defends this case explicitly on its own write surfaces; the
|
|
529
|
+
* receiver did not, which is an asymmetry against DDR-226 §9's "receivers
|
|
530
|
+
* re-shape-validate paths and re-classify locally". Same two-guard shape as
|
|
531
|
+
* the hub's: resolve the real parent, assert it is under the real root, and
|
|
532
|
+
* then do every filesystem op on `join(realParent, basename)`.
|
|
533
|
+
*/
|
|
534
|
+
function safeTarget(rel: string): { abs: string; parent: string } | null {
|
|
535
|
+
const lexical = path.join(designRoot, rel);
|
|
536
|
+
const abs = realpathOfDeepestExisting(lexical);
|
|
537
|
+
const parent = path.dirname(abs);
|
|
538
|
+
if (parent !== realRoot && !parent.startsWith(realRoot + path.sep)) {
|
|
539
|
+
log.warn?.(
|
|
540
|
+
`[sync/files] refusing ${rel}: it resolves outside the design root (a symlinked directory on the path)`
|
|
541
|
+
);
|
|
542
|
+
return null;
|
|
543
|
+
}
|
|
544
|
+
// A directory, a symlink, a socket — anything that is not a regular file
|
|
545
|
+
// sitting where a file belongs is refused rather than replaced.
|
|
546
|
+
try {
|
|
547
|
+
if (!statSync(abs).isFile()) {
|
|
548
|
+
log.warn?.(`[sync/files] refusing ${rel}: the target exists and is not a regular file`);
|
|
549
|
+
return null;
|
|
550
|
+
}
|
|
551
|
+
} catch {
|
|
552
|
+
/* absent is the normal case for a create */
|
|
553
|
+
}
|
|
554
|
+
return { abs, parent };
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Would this peer ever accept `rel` at all? The cheap gate, run before a
|
|
559
|
+
* hub-named path is allowed to become persistent state.
|
|
560
|
+
*
|
|
561
|
+
* Deliberately weaker than the admission check at decision time: it has no
|
|
562
|
+
* scanned local map, so it asks the disk directly. Anything it lets through
|
|
563
|
+
* is still fully re-classified there.
|
|
564
|
+
*/
|
|
565
|
+
function admissible(rel: string): boolean {
|
|
566
|
+
const cls = classifyProjectFile(rel, {
|
|
567
|
+
canvasGroups: opts.canvasGroups,
|
|
568
|
+
hasFile: (r) => existsSync(path.join(designRoot, r)),
|
|
569
|
+
});
|
|
570
|
+
return isFilePlaneClass(cls);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/** Land bytes at `rel`, atomically. Throws on failure — `adoptAfter` catches. */
|
|
574
|
+
function materialize(rel: string, bytes: Uint8Array): void {
|
|
575
|
+
const target = safeTarget(rel);
|
|
576
|
+
if (!target) throw new Error(`refusing to write ${rel} — it does not resolve inside the root`);
|
|
577
|
+
mkdirSync(target.parent, { recursive: true });
|
|
578
|
+
const tmp = `${target.abs}.part`;
|
|
579
|
+
writeFileSync(tmp, bytes);
|
|
580
|
+
renameSync(tmp, target.abs);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/** Copy the local file aside under a name both ends can see. Returns the rel. */
|
|
584
|
+
function parkLocal(rel: string): string | null {
|
|
585
|
+
const abs = path.join(designRoot, rel);
|
|
586
|
+
const copyRel = conflictCopyName(rel, now(), opts.label);
|
|
587
|
+
const target = safeTarget(copyRel);
|
|
588
|
+
if (!target) return null;
|
|
589
|
+
try {
|
|
590
|
+
const bytes = readFileSync(abs);
|
|
591
|
+
mkdirSync(target.parent, { recursive: true });
|
|
592
|
+
const tmp = `${target.abs}.part`;
|
|
593
|
+
writeFileSync(tmp, bytes);
|
|
594
|
+
renameSync(tmp, target.abs);
|
|
595
|
+
return copyRel;
|
|
596
|
+
} catch (err) {
|
|
597
|
+
log.warn?.(`[sync/files] could not park ${rel}: ${(err as Error).message}`);
|
|
598
|
+
return null;
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Upload one file with a compare-and-swap against the state we decided from.
|
|
604
|
+
*/
|
|
605
|
+
async function push(
|
|
606
|
+
local: LocalFile,
|
|
607
|
+
expect: string | null
|
|
608
|
+
): Promise<
|
|
609
|
+
| { ok: true; seq: number | null }
|
|
610
|
+
| { ok: false; conflict: true; current: string | null }
|
|
611
|
+
| { ok: false; conflict?: false; reason: string }
|
|
612
|
+
> {
|
|
613
|
+
let bytes: Uint8Array;
|
|
614
|
+
try {
|
|
615
|
+
bytes = readFileSync(local.abs);
|
|
616
|
+
} catch (err) {
|
|
617
|
+
return { ok: false, reason: (err as Error).message };
|
|
618
|
+
}
|
|
619
|
+
// In flight, so a journal row carrying this hash reads as our own echo
|
|
620
|
+
// rather than as a remote change we must react to.
|
|
621
|
+
ledger.outboxAdd(local.hash);
|
|
622
|
+
try {
|
|
623
|
+
const res = await fetchImpl(
|
|
624
|
+
`${base}/api/file/${local.rel.split('/').map(encodeURIComponent).join('/')}`,
|
|
625
|
+
{
|
|
626
|
+
method: 'PUT',
|
|
627
|
+
headers: {
|
|
628
|
+
...auth(),
|
|
629
|
+
'content-type': 'application/octet-stream',
|
|
630
|
+
'x-maude-content-sha256': local.hash,
|
|
631
|
+
'x-maude-expect-hash': expect ?? 'none',
|
|
632
|
+
},
|
|
633
|
+
body: bytes as unknown as BodyInit,
|
|
634
|
+
signal: AbortSignal.timeout(PUT_TIMEOUT_MS),
|
|
635
|
+
}
|
|
636
|
+
);
|
|
637
|
+
if (res.status === 409) {
|
|
638
|
+
const body = (await res.json().catch(() => ({}))) as { current?: unknown };
|
|
639
|
+
return {
|
|
640
|
+
ok: false,
|
|
641
|
+
conflict: true,
|
|
642
|
+
current: typeof body?.current === 'string' ? body.current : null,
|
|
643
|
+
};
|
|
644
|
+
}
|
|
645
|
+
if (!res.ok) return { ok: false, reason: `HTTP ${res.status}` };
|
|
646
|
+
const body = (await res.json().catch(() => ({}))) as { seq?: unknown };
|
|
647
|
+
return { ok: true, seq: typeof body?.seq === 'number' ? body.seq : null };
|
|
648
|
+
} catch (err) {
|
|
649
|
+
return { ok: false, reason: (err as Error).message };
|
|
650
|
+
} finally {
|
|
651
|
+
ledger.outboxDone(local.hash);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Tell the hub a file is gone here — Increment 6.
|
|
657
|
+
*
|
|
658
|
+
* Carries the same `x-maude-expect-hash` precondition a write does, and for
|
|
659
|
+
* the same reason: a delete that raced somebody's edit must LOSE. The hub
|
|
660
|
+
* answers 409 with what it now holds, the next pass re-decides against that,
|
|
661
|
+
* and `local-deleted-but-remote-moved` brings their work back instead of
|
|
662
|
+
* removing it. An edit beats a delete, enforced at the door rather than hoped
|
|
663
|
+
* for by ordering.
|
|
664
|
+
*/
|
|
665
|
+
async function pushDelete(
|
|
666
|
+
rel: string,
|
|
667
|
+
expect: string | null
|
|
668
|
+
): Promise<{ ok: true } | { ok: false; conflict: true } | { ok: false; reason: string }> {
|
|
669
|
+
try {
|
|
670
|
+
const res = await fetchImpl(
|
|
671
|
+
`${base}/api/file/${rel.split('/').map(encodeURIComponent).join('/')}`,
|
|
672
|
+
{
|
|
673
|
+
method: 'DELETE',
|
|
674
|
+
headers: { ...auth(), 'x-maude-expect-hash': expect ?? 'none' },
|
|
675
|
+
signal: AbortSignal.timeout(PUT_TIMEOUT_MS),
|
|
676
|
+
}
|
|
677
|
+
);
|
|
678
|
+
if (res.status === 409) return { ok: false, conflict: true };
|
|
679
|
+
if (!res.ok) return { ok: false, reason: `HTTP ${res.status}` };
|
|
680
|
+
return { ok: true };
|
|
681
|
+
} catch (err) {
|
|
682
|
+
return { ok: false, reason: (err as Error).message };
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
/** Move a file into `_trash/<stamp>/<rel>`; returns the trash rel or null. */
|
|
687
|
+
function quarantineLocal(rel: string): string | null {
|
|
688
|
+
const abs = path.join(designRoot, rel);
|
|
689
|
+
if (!existsSync(abs)) return null;
|
|
690
|
+
const stamp = new Date(now()).toISOString().replace(/[:.]/g, '-');
|
|
691
|
+
const destRel = `_trash/${stamp}/${rel}`;
|
|
692
|
+
// THROUGH `safeTarget`, like every other write on this side. This was the
|
|
693
|
+
// one path that skipped it, and it is the worst one to skip: the
|
|
694
|
+
// DESTINATION is where a file goes to be recoverable, so a symlinked
|
|
695
|
+
// `_trash/` that lands it outside the design root turns "quarantined,
|
|
696
|
+
// never unlinked" into a deletion with extra steps.
|
|
697
|
+
const target = safeTarget(destRel);
|
|
698
|
+
if (!target) {
|
|
699
|
+
log.warn?.(
|
|
700
|
+
`[sync/files] refusing to quarantine ${rel} — _trash/ does not resolve inside the root`
|
|
701
|
+
);
|
|
702
|
+
return null;
|
|
703
|
+
}
|
|
704
|
+
try {
|
|
705
|
+
mkdirSync(target.parent, { recursive: true });
|
|
706
|
+
renameSync(abs, target.abs);
|
|
707
|
+
return destRel;
|
|
708
|
+
} catch (err) {
|
|
709
|
+
log.warn?.(`[sync/files] could not quarantine ${rel}: ${(err as Error).message}`);
|
|
710
|
+
return null;
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
async function reconcile(): Promise<FilePlaneResult> {
|
|
715
|
+
const out: FilePlaneResult = {
|
|
716
|
+
pulled: [],
|
|
717
|
+
pushed: [],
|
|
718
|
+
conflicts: [],
|
|
719
|
+
dropped: [],
|
|
720
|
+
failed: [],
|
|
721
|
+
deleted: [],
|
|
722
|
+
synced: 0,
|
|
723
|
+
reanchored: false,
|
|
724
|
+
};
|
|
725
|
+
|
|
726
|
+
// ── 1. The hub's side ────────────────────────────────────────────────
|
|
727
|
+
const startedFrom = ledger.cursor();
|
|
728
|
+
let fullRead = startedFrom === 0;
|
|
729
|
+
let page = await fetchJournal({
|
|
730
|
+
hubUrl: opts.hubUrl,
|
|
731
|
+
token: opts.token(),
|
|
732
|
+
since: startedFrom,
|
|
733
|
+
epoch: ledger.epoch(),
|
|
734
|
+
fetchImpl,
|
|
735
|
+
});
|
|
736
|
+
// Unreachable / refused / journal-less: this pass does nothing, and the
|
|
737
|
+
// next one asks again. NEVER "nothing changed".
|
|
738
|
+
if (page === null) return out;
|
|
739
|
+
|
|
740
|
+
// Whether our ANCESTORS still describe the hub's log.
|
|
741
|
+
//
|
|
742
|
+
// Computed BEFORE re-anchoring, because re-anchoring adopts the hub's
|
|
743
|
+
// epoch — ask afterwards and the answer is always "fine", and the degraded
|
|
744
|
+
// rows in the decision table could never fire at all. The whole point of
|
|
745
|
+
// those rows is the pass that discovers the log moved out from under us.
|
|
746
|
+
let degraded = ledger.isDegraded(page.epoch);
|
|
747
|
+
|
|
748
|
+
if (page.reanchor) {
|
|
749
|
+
// A RE-ANCHOR STORM IS A DISK-FILLING PRIMITIVE, so obedience is capped.
|
|
750
|
+
//
|
|
751
|
+
// Re-anchoring is a full compaction read plus `pruneRemotes`, and it sets
|
|
752
|
+
// `degraded`, under which every differing path parks a copy of the hub's
|
|
753
|
+
// bytes. A hub that answers `reanchor` to every request therefore drives
|
|
754
|
+
// unbounded work and (before the park memo) unbounded files. Past the
|
|
755
|
+
// limit the pass holds and says so; after REANCHOR_HOLD_RECOVERY_MS one
|
|
756
|
+
// fresh attempt is allowed, so a legitimate epoch rotation still
|
|
757
|
+
// converges while a storm stays capped at one full read per window.
|
|
758
|
+
reanchorsInARow += 1;
|
|
759
|
+
if (reanchorsInARow > REANCHOR_STORM_LIMIT) {
|
|
760
|
+
// F-11 — the hold is a WINDOW, not a brick. Once the recovery window
|
|
761
|
+
// has passed, allow exactly one fresh attempt (counter back to 1): a
|
|
762
|
+
// legitimate epoch-rotation burst converges on its first quiet retry,
|
|
763
|
+
// while a hub that answers `reanchor` forever is capped at one full
|
|
764
|
+
// read per window instead of bricking the plane until a restart.
|
|
765
|
+
if (reanchorHeldSince === 0) reanchorHeldSince = now();
|
|
766
|
+
if (now() - reanchorHeldSince < REANCHOR_HOLD_RECOVERY_MS) {
|
|
767
|
+
log.warn?.(
|
|
768
|
+
`[sync/files] the hub has asked to re-anchor ${reanchorsInARow} times in a row — holding this pass (retry allowed in ${Math.ceil((REANCHOR_HOLD_RECOVERY_MS - (now() - reanchorHeldSince)) / 60000)} min). Ancestors are untouched and nothing was overwritten.`
|
|
769
|
+
);
|
|
770
|
+
out.reanchorHeld = true;
|
|
771
|
+
return out;
|
|
772
|
+
}
|
|
773
|
+
reanchorsInARow = 1;
|
|
774
|
+
reanchorHeldSince = 0;
|
|
775
|
+
}
|
|
776
|
+
out.reanchored = true;
|
|
777
|
+
log.warn?.(
|
|
778
|
+
`[sync/files] re-anchoring against the hub (${page.reason ?? 'cursor not in this log'}).`
|
|
779
|
+
);
|
|
780
|
+
// A cursor we held that the hub cannot honour means our anchor is not in
|
|
781
|
+
// their log — whether the epoch rotated or the log rewound underneath
|
|
782
|
+
// it. Either way the ancestors stop being overwrite authority for this
|
|
783
|
+
// pass; they are demoted, not discarded.
|
|
784
|
+
if (startedFrom > 0) degraded = true;
|
|
785
|
+
ledger.reanchor(page.epoch);
|
|
786
|
+
fullRead = true;
|
|
787
|
+
page = await fetchJournal({
|
|
788
|
+
hubUrl: opts.hubUrl,
|
|
789
|
+
token: opts.token(),
|
|
790
|
+
since: 0,
|
|
791
|
+
fetchImpl,
|
|
792
|
+
});
|
|
793
|
+
if (page === null || page.reanchor) return out;
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
if (!out.reanchored) {
|
|
797
|
+
reanchorsInARow = 0;
|
|
798
|
+
reanchorHeldSince = 0;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
// THE HUB'S SIDE IS THE LEDGER'S REPLICA, UPDATED BY THIS PAGE — not the
|
|
802
|
+
// page itself. A delta says "these changed"; it says nothing at all about
|
|
803
|
+
// the paths it omits. Reading that silence as "the hub does not have them"
|
|
804
|
+
// would push every converged file back up on every pass, and would be
|
|
805
|
+
// absence-as-authority (DDR-076) rebuilt one layer above the table that
|
|
806
|
+
// forbids it. Only a FULL read may retract a remembered remote.
|
|
807
|
+
const delta = foldRemote(page.entries);
|
|
808
|
+
for (const [rel, row] of delta) {
|
|
809
|
+
// CLASSIFY BEFORE REMEMBERING. Admission used to run at decision time
|
|
810
|
+
// only, which meant every path the hub named — including a page of pure
|
|
811
|
+
// junk — became a ledger row on disk and a key in `_sync.json` first,
|
|
812
|
+
// and a second `stuck` row immediately after. One response could grow
|
|
813
|
+
// this machine's persistent state without bound, and the rows were never
|
|
814
|
+
// pruned by count, so the poisoning survived restarts. The full
|
|
815
|
+
// admission below is unchanged; this is the cheaper gate in front of it.
|
|
816
|
+
if (!admissible(rel)) {
|
|
817
|
+
// Reported, so a refusal is never silent — but not REMEMBERED. `drop`
|
|
818
|
+
// declines to mint a row for a path this machine does not already
|
|
819
|
+
// track, so the pass says what it refused without the hub being able
|
|
820
|
+
// to grow our ledger by naming things.
|
|
821
|
+
drop(out, rel, 'the hub offered a path this peer does not admit', ledger);
|
|
822
|
+
delta.delete(rel);
|
|
823
|
+
continue;
|
|
824
|
+
}
|
|
825
|
+
ledger.noteRemote(rel, row.deleted ? null : row.sha256, row.seq);
|
|
826
|
+
}
|
|
827
|
+
if (fullRead) ledger.pruneRemotes(new Set(delta.keys()));
|
|
828
|
+
|
|
829
|
+
// ── 2. Ours ──────────────────────────────────────────────────────────
|
|
830
|
+
const local = scanLocalFiles(designRoot, ledger, opts.canvasGroups);
|
|
831
|
+
|
|
832
|
+
// ── 3. Decide ────────────────────────────────────────────────────────
|
|
833
|
+
let work: {
|
|
834
|
+
rel: string;
|
|
835
|
+
decision: ReturnType<typeof decideFile>;
|
|
836
|
+
local: LocalFile | undefined;
|
|
837
|
+
row: JournalEntry | undefined;
|
|
838
|
+
remoteHash: string | null;
|
|
839
|
+
}[] = [];
|
|
840
|
+
|
|
841
|
+
// Every path either side knows about — including ones only the LEDGER
|
|
842
|
+
// remembers, which is how a file that stopped changing still gets checked.
|
|
843
|
+
const paths = new Set<string>([
|
|
844
|
+
...local.keys(),
|
|
845
|
+
...delta.keys(),
|
|
846
|
+
...Object.keys(ledger.rows()),
|
|
847
|
+
]);
|
|
848
|
+
for (const rel of paths) {
|
|
849
|
+
const row = delta.get(rel);
|
|
850
|
+
const here = local.get(rel);
|
|
851
|
+
// What the hub holds: this page when it spoke about the path, otherwise
|
|
852
|
+
// what we last learned. `undefined` (never learned) reads as null only
|
|
853
|
+
// after a full read has had the chance to say so.
|
|
854
|
+
const remoteHash = row ? (row.deleted ? null : row.sha256) : (ledger.remoteOf(rel) ?? null);
|
|
855
|
+
|
|
856
|
+
// ADMISSION RUNS FOR EVERY PATH THE HUB OFFERS — not only for the ones
|
|
857
|
+
// this page happened to mention.
|
|
858
|
+
//
|
|
859
|
+
// Gating on "the delta carried a row" was a hole with teeth: a cursor
|
|
860
|
+
// read is silent about unchanged paths, so a code module refused on the
|
|
861
|
+
// pass that introduced it sailed straight through on the next tick,
|
|
862
|
+
// sourced from the remembered remote with no gate in front of it. The
|
|
863
|
+
// admission belongs to the OFFER, not to the notification.
|
|
864
|
+
if (remoteHash !== null) {
|
|
865
|
+
// THIS peer's verdict on the path, not the hub's. A disagreement is a
|
|
866
|
+
// drop, not a negotiation (DDR-054: the hub's class is a hint).
|
|
867
|
+
const cls = classifyProjectFile(rel, {
|
|
868
|
+
canvasGroups: opts.canvasGroups,
|
|
869
|
+
hasFile: (r) => local.has(r) || existsSync(path.join(designRoot, r)),
|
|
870
|
+
});
|
|
871
|
+
if (!isFilePlaneClass(cls)) {
|
|
872
|
+
drop(out, rel, `classifies '${cls}' here`, ledger);
|
|
873
|
+
continue;
|
|
874
|
+
}
|
|
875
|
+
if (cls === 'code-module' && !opts.allowCodeModules) {
|
|
876
|
+
drop(
|
|
877
|
+
out,
|
|
878
|
+
rel,
|
|
879
|
+
'code modules replicate only from an owner-vouched or loopback hub',
|
|
880
|
+
ledger
|
|
881
|
+
);
|
|
882
|
+
continue;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
const state: FileState = {
|
|
887
|
+
path: rel,
|
|
888
|
+
local: here?.hash ?? null,
|
|
889
|
+
remote: remoteHash,
|
|
890
|
+
ancestor: ledger.ancestorOf(rel),
|
|
891
|
+
...(row?.deleted ? { remoteTombstone: true } : {}),
|
|
892
|
+
...(degraded ? { epochChanged: true } : {}),
|
|
893
|
+
...(remoteHash && ledger.outboxHas(remoteHash) ? { selfInFlight: true } : {}),
|
|
894
|
+
...(opts.propagateDeletes ? { propagateDeletes: true } : {}),
|
|
895
|
+
};
|
|
896
|
+
const decision = decideFile(state);
|
|
897
|
+
if (decision.action === 'noop' && !decision.parkRemote) {
|
|
898
|
+
if (decision.adoptAncestor && here) {
|
|
899
|
+
// Record agreement so the next pass is a stat and nothing more.
|
|
900
|
+
//
|
|
901
|
+
// The state comes from the REMEMBERED remote, not from this page. A
|
|
902
|
+
// converged file is precisely the one a delta never mentions, so
|
|
903
|
+
// reading `row` here made every settled file report `local-only` —
|
|
904
|
+
// a panel saying "not delivered" about files that had been on the
|
|
905
|
+
// hub for hours. That is the status-lies failure DDR-214 exists to
|
|
906
|
+
// end, and it is worse than no panel: it teaches people to distrust
|
|
907
|
+
// the one surface meant to answer the question.
|
|
908
|
+
void ledger.adoptAfter(rel, here.hash, () => {}, {
|
|
909
|
+
...(row ? { remoteSeq: row.seq } : {}),
|
|
910
|
+
size: here.size,
|
|
911
|
+
mtimeMs: here.mtimeMs,
|
|
912
|
+
state: remoteHash !== null ? 'on-hub' : 'local-only',
|
|
913
|
+
});
|
|
914
|
+
out.synced += 1;
|
|
915
|
+
}
|
|
916
|
+
continue;
|
|
917
|
+
}
|
|
918
|
+
work.push({ rel, decision, local: here, row, remoteHash });
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// DELETION BREAKERS. Counted before anything is applied, in both
|
|
922
|
+
// directions, because a delete you have already done is not one a prompt
|
|
923
|
+
// can take back.
|
|
924
|
+
const tracked = Object.keys(ledger.rows()).length;
|
|
925
|
+
|
|
926
|
+
/**
|
|
927
|
+
* Three arms, and the cumulative one is the load-bearing addition.
|
|
928
|
+
*
|
|
929
|
+
* burst — more than `MAX` in one pass. The accident shape.
|
|
930
|
+
* proportion — more than a quarter of what this machine tracks. Catches
|
|
931
|
+
* a small project where ten is most of it.
|
|
932
|
+
* budget — more than `PER_WINDOW` across the whole window, counting
|
|
933
|
+
* what previous passes already applied. Catches the patient
|
|
934
|
+
* drain the first two are blind to, and survives a restart.
|
|
935
|
+
*/
|
|
936
|
+
const overBreaker = (direction: 'out' | 'in', n: number): boolean => {
|
|
937
|
+
const already = ledger.deletesInWindow(direction, DELETE_BUDGET_WINDOW_MS);
|
|
938
|
+
if (n > DELETE_BREAKER_MAX) return true;
|
|
939
|
+
if (
|
|
940
|
+
n >= DELETE_BREAKER_MIN_FOR_FRACTION &&
|
|
941
|
+
tracked > 0 &&
|
|
942
|
+
n / tracked > DELETE_BREAKER_MAX_FRACTION
|
|
943
|
+
) {
|
|
944
|
+
return true;
|
|
945
|
+
}
|
|
946
|
+
return already + n > DELETE_BUDGET_PER_WINDOW;
|
|
947
|
+
};
|
|
948
|
+
|
|
949
|
+
for (const direction of ['out', 'in'] as const) {
|
|
950
|
+
const action = direction === 'out' ? 'propagate-delete' : 'quarantine';
|
|
951
|
+
const hits = work.filter((w) => w.decision.action === action);
|
|
952
|
+
if (hits.length === 0 || !overBreaker(direction, hits.length)) continue;
|
|
953
|
+
const paths = hits.map((w) => w.rel).sort();
|
|
954
|
+
log.warn?.(
|
|
955
|
+
direction === 'out'
|
|
956
|
+
? `[sync/files] ${paths.length} of ${tracked} tracked files are gone from this machine — NOT telling the project. If that was a branch switch or a bad restore, nothing is lost; if you meant it, confirm the deletion.`
|
|
957
|
+
: `[sync/files] the project wants to remove ${paths.length} of ${tracked} tracked files here — holding. Nothing was deleted.`
|
|
958
|
+
);
|
|
959
|
+
out.deleteHeld = { direction, count: paths.length, paths: paths.slice(0, 200) };
|
|
960
|
+
for (const w of hits) {
|
|
961
|
+
ledger.setState(w.rel, 'conflict', {
|
|
962
|
+
reason:
|
|
963
|
+
direction === 'out'
|
|
964
|
+
? 'gone from this machine, as part of a batch too large to propagate unasked'
|
|
965
|
+
: 'the project wants this removed, as part of a batch too large to apply unasked',
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
work = work.filter((w) => !hits.includes(w));
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
// FLIP-DAY BREAKER. Counted before anything is applied, because the point
|
|
972
|
+
// is to not have parked forty files by the time anyone notices.
|
|
973
|
+
const firstAnchor = work.filter(
|
|
974
|
+
(w) =>
|
|
975
|
+
w.decision.action === 'conflict-aside' &&
|
|
976
|
+
ledger.ancestorOf(w.rel) === null &&
|
|
977
|
+
w.local !== undefined
|
|
978
|
+
);
|
|
979
|
+
if (firstAnchor.length > 0 && opts.resolveFirstAnchor === 'keep-local') {
|
|
980
|
+
// Ours wins the set: drop the pull half of each conflict and let the
|
|
981
|
+
// push half send local up. Nothing is parked, because nothing is lost —
|
|
982
|
+
// the hub's copy is still in its own journal and its own object storage.
|
|
983
|
+
for (const w of firstAnchor) {
|
|
984
|
+
w.decision = {
|
|
985
|
+
action: 'push',
|
|
986
|
+
reason: 'you chose to keep this machine’s copies for the whole set',
|
|
987
|
+
};
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
if (firstAnchor.length > FIRST_ANCHOR_STORM_LIMIT && !opts.resolveFirstAnchor) {
|
|
991
|
+
const paths = firstAnchor.map((w) => w.rel).sort();
|
|
992
|
+
log.warn?.(
|
|
993
|
+
`[sync/files] ${paths.length} files differ on both sides and this machine has never reconciled any of them — holding, rather than parking ${paths.length} conflict copies. Choose keep-local or keep-cloud for the set.`
|
|
994
|
+
);
|
|
995
|
+
out.firstAnchorHeld = { count: paths.length, paths: paths.slice(0, 200) };
|
|
996
|
+
for (const w of firstAnchor) {
|
|
997
|
+
ledger.setState(w.rel, 'conflict', {
|
|
998
|
+
reason: 'both sides have content and neither has been reconciled here yet',
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
// Everything that is NOT a first-anchor conflict still flows: holding a
|
|
1002
|
+
// whole pass over one class would stall ordinary sync too.
|
|
1003
|
+
work = work.filter((w) => !firstAnchor.includes(w));
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
// ── 4. Apply ─────────────────────────────────────────────────────────
|
|
1007
|
+
//
|
|
1008
|
+
// Referenced assets first (DDR-223's strokes→bytes coupling): a picture a
|
|
1009
|
+
// just-arrived annotation points at is the one a person is staring at.
|
|
1010
|
+
const referenced = referencedAssetNames(designRoot);
|
|
1011
|
+
work.sort((a, b) => rank(a.rel, referenced) - rank(b.rel, referenced));
|
|
1012
|
+
|
|
1013
|
+
const budget = createPullBudget({
|
|
1014
|
+
label: 'sync/files',
|
|
1015
|
+
log,
|
|
1016
|
+
...(opts.maxPassBytes !== undefined ? { maxBytes: opts.maxPassBytes } : {}),
|
|
1017
|
+
});
|
|
1018
|
+
|
|
1019
|
+
let moved = 0;
|
|
1020
|
+
for (const item of work) {
|
|
1021
|
+
if (moved >= MAX_FILES_PER_PASS) {
|
|
1022
|
+
log.warn?.(
|
|
1023
|
+
`[sync/files] ${work.length - moved} more path(s) to reconcile; taking them next pass.`
|
|
1024
|
+
);
|
|
1025
|
+
break;
|
|
1026
|
+
}
|
|
1027
|
+
if (budget.exhausted()) {
|
|
1028
|
+
out.budgetExhausted = true;
|
|
1029
|
+
break;
|
|
1030
|
+
}
|
|
1031
|
+
const handled = await applyOne(item, out, budget);
|
|
1032
|
+
if (handled) moved += 1;
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
// ── 5. Position ──────────────────────────────────────────────────────
|
|
1036
|
+
//
|
|
1037
|
+
// Only advance the cursor when the pass actually consumed the page. A
|
|
1038
|
+
// partial pass re-reads the same range next time, which is free (the
|
|
1039
|
+
// decisions for already-converged paths are `noop`).
|
|
1040
|
+
if (!page.truncated && out.failed.length === 0) {
|
|
1041
|
+
ledger.setPosition(page.epoch, page.head);
|
|
1042
|
+
} else {
|
|
1043
|
+
ledger.setPosition(page.epoch, ledger.cursor());
|
|
1044
|
+
}
|
|
1045
|
+
ledger.flush();
|
|
1046
|
+
|
|
1047
|
+
if (out.pulled.length || out.pushed.length || out.conflicts.length) {
|
|
1048
|
+
log.log?.(
|
|
1049
|
+
`[sync/files] ${out.pulled.length} down, ${out.pushed.length} up, ${out.conflicts.length} conflict(s), ${out.synced} already in step${
|
|
1050
|
+
out.failed.length ? `, ${out.failed.length} failed` : ''
|
|
1051
|
+
}.`
|
|
1052
|
+
);
|
|
1053
|
+
}
|
|
1054
|
+
// CHARGE WHAT ACTUALLY HAPPENED against the window, per direction. Applied
|
|
1055
|
+
// deletions only — a decision the breaker held, or one that failed, costs
|
|
1056
|
+
// nothing, or a hub could exhaust the budget with attempts.
|
|
1057
|
+
ledger.noteDeletes(
|
|
1058
|
+
'out',
|
|
1059
|
+
out.deleted.filter((d) => d.parked === null).map((d) => d.rel),
|
|
1060
|
+
DELETE_BUDGET_WINDOW_MS
|
|
1061
|
+
);
|
|
1062
|
+
ledger.noteDeletes(
|
|
1063
|
+
'in',
|
|
1064
|
+
out.deleted.filter((d) => d.parked !== null).map((d) => d.rel),
|
|
1065
|
+
DELETE_BUDGET_WINDOW_MS
|
|
1066
|
+
);
|
|
1067
|
+
|
|
1068
|
+
return out;
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
async function applyOne(
|
|
1072
|
+
item: {
|
|
1073
|
+
rel: string;
|
|
1074
|
+
decision: ReturnType<typeof decideFile>;
|
|
1075
|
+
local?: LocalFile;
|
|
1076
|
+
row?: JournalEntry;
|
|
1077
|
+
remoteHash: string | null;
|
|
1078
|
+
},
|
|
1079
|
+
out: FilePlaneResult,
|
|
1080
|
+
budget: ReturnType<typeof createPullBudget>
|
|
1081
|
+
): Promise<boolean> {
|
|
1082
|
+
const { rel, decision, local: here, row, remoteHash } = item;
|
|
1083
|
+
|
|
1084
|
+
switch (decision.action) {
|
|
1085
|
+
case 'noop': {
|
|
1086
|
+
// Only the epoch-degraded rows reach here: keep local, park THEIR
|
|
1087
|
+
// copy so it is recoverable, and let the push half send ours up.
|
|
1088
|
+
if (decision.parkRemote && remoteHash) {
|
|
1089
|
+
// ONCE per remote hash. The decision is `noop`, so the ancestor
|
|
1090
|
+
// deliberately does not move and the next pass sees the identical
|
|
1091
|
+
// state — without this memo a hub that re-anchors every request
|
|
1092
|
+
// (or merely rotates its epoch, which is a legitimate event) writes
|
|
1093
|
+
// a fresh timestamped copy of every diverged path on every pass,
|
|
1094
|
+
// and each copy is then scanned as `create-up` and pushed back up.
|
|
1095
|
+
//
|
|
1096
|
+
// B13 — honoured only while the copy it names STILL EXISTS. The
|
|
1097
|
+
// memo's claim is "a recoverable copy was made"; after the user
|
|
1098
|
+
// deleted it or a `_trash/` prune swept it, skipping the park on the
|
|
1099
|
+
// memo's word would be a noop with no recoverable copy anywhere.
|
|
1100
|
+
{
|
|
1101
|
+
const memoRow = ledger.row(rel);
|
|
1102
|
+
if (
|
|
1103
|
+
memoRow?.parkedRemote === remoteHash &&
|
|
1104
|
+
memoRow.conflictCopy &&
|
|
1105
|
+
existsSync(path.join(designRoot, memoRow.conflictCopy))
|
|
1106
|
+
) {
|
|
1107
|
+
return true;
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
const claim = claimedSize(row);
|
|
1111
|
+
if (!budget.take(claim)) {
|
|
1112
|
+
out.budgetExhausted = true;
|
|
1113
|
+
return false;
|
|
1114
|
+
}
|
|
1115
|
+
const got = await fetchVerified(rel, remoteHash, budget.remaining() + claim);
|
|
1116
|
+
if (!got.ok) {
|
|
1117
|
+
if (got.overCap && budget.remaining() + claim < MAX_FILE_BYTES) {
|
|
1118
|
+
out.budgetExhausted = true;
|
|
1119
|
+
}
|
|
1120
|
+
return false;
|
|
1121
|
+
}
|
|
1122
|
+
if (!chargeOverrun(budget, rel, got.bytes.byteLength, claim, out)) return false;
|
|
1123
|
+
const copyRel = conflictCopyName(rel, now(), 'hub');
|
|
1124
|
+
try {
|
|
1125
|
+
materialize(copyRel, got.bytes);
|
|
1126
|
+
ledger.setState(rel, 'conflict', {
|
|
1127
|
+
reason: 'the hub’s log restarted; their copy is parked beside yours',
|
|
1128
|
+
conflictCopy: copyRel,
|
|
1129
|
+
parkedRemote: remoteHash,
|
|
1130
|
+
});
|
|
1131
|
+
out.conflicts.push({ rel, copy: copyRel });
|
|
1132
|
+
} catch (err) {
|
|
1133
|
+
out.failed.push({ rel, reason: (err as Error).message });
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
return true;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
case 'pull': {
|
|
1140
|
+
// From the REMEMBERED remote, not only from this page. A pull that
|
|
1141
|
+
// failed last pass leaves the hub's hash known and this page silent
|
|
1142
|
+
// about it; sourcing the fetch from the page alone would mean a
|
|
1143
|
+
// transient 500 permanently stranded the file.
|
|
1144
|
+
if (!remoteHash) return false;
|
|
1145
|
+
const claim = claimedSize(row);
|
|
1146
|
+
if (!budget.take(claim)) {
|
|
1147
|
+
out.budgetExhausted = true;
|
|
1148
|
+
return false;
|
|
1149
|
+
}
|
|
1150
|
+
const got = await fetchVerified(rel, remoteHash, budget.remaining() + claim);
|
|
1151
|
+
if (!got.ok) {
|
|
1152
|
+
out.failed.push({ rel, reason: got.reason });
|
|
1153
|
+
ledger.setState(rel, 'stuck', { reason: got.reason });
|
|
1154
|
+
// Refused because it would not fit in what is LEFT, not because it is
|
|
1155
|
+
// implausible on its own: that is the budget speaking, so the pass
|
|
1156
|
+
// says so and the next one picks the file up with a full budget.
|
|
1157
|
+
if (got.overCap && budget.remaining() + claim < MAX_FILE_BYTES) {
|
|
1158
|
+
out.budgetExhausted = true;
|
|
1159
|
+
}
|
|
1160
|
+
return false;
|
|
1161
|
+
}
|
|
1162
|
+
if (!chargeOverrun(budget, rel, got.bytes.byteLength, claim, out)) return false;
|
|
1163
|
+
const landed = await ledger.adoptAfter(rel, remoteHash, () => materialize(rel, got.bytes), {
|
|
1164
|
+
...(row ? { remoteSeq: row.seq } : {}),
|
|
1165
|
+
state: 'on-hub',
|
|
1166
|
+
});
|
|
1167
|
+
if (landed) {
|
|
1168
|
+
// Re-stat so the cache matches what is now on disk, or the very next
|
|
1169
|
+
// pass re-hashes everything it just wrote.
|
|
1170
|
+
restat(rel);
|
|
1171
|
+
out.pulled.push(rel);
|
|
1172
|
+
} else {
|
|
1173
|
+
out.failed.push({ rel, reason: 'could not materialize' });
|
|
1174
|
+
}
|
|
1175
|
+
return landed;
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
case 'push':
|
|
1179
|
+
case 'revive': {
|
|
1180
|
+
if (!here) return false;
|
|
1181
|
+
ledger.setState(rel, 'pushing');
|
|
1182
|
+
const expect = decision.action === 'revive' ? null : remoteHash;
|
|
1183
|
+
const res = await push(here, expect);
|
|
1184
|
+
if (res.ok) {
|
|
1185
|
+
await ledger.adoptAfter(rel, here.hash, () => {}, {
|
|
1186
|
+
...(res.seq !== null ? { remoteSeq: res.seq } : {}),
|
|
1187
|
+
size: here.size,
|
|
1188
|
+
mtimeMs: here.mtimeMs,
|
|
1189
|
+
state: 'on-hub',
|
|
1190
|
+
});
|
|
1191
|
+
out.pushed.push(rel);
|
|
1192
|
+
return true;
|
|
1193
|
+
}
|
|
1194
|
+
if (res.conflict) {
|
|
1195
|
+
// The hub moved under us. Do NOT retry blindly — re-decide next
|
|
1196
|
+
// pass against what it actually holds now, which is exactly what a
|
|
1197
|
+
// cursor read will hand us.
|
|
1198
|
+
ledger.setState(rel, 'conflict', {
|
|
1199
|
+
reason: 'the hub changed this file while the upload was in flight',
|
|
1200
|
+
});
|
|
1201
|
+
out.conflicts.push({ rel, copy: null });
|
|
1202
|
+
return true;
|
|
1203
|
+
}
|
|
1204
|
+
out.failed.push({ rel, reason: res.reason });
|
|
1205
|
+
ledger.setState(rel, 'stuck', { reason: res.reason });
|
|
1206
|
+
return false;
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
case 'conflict-aside': {
|
|
1210
|
+
if (!remoteHash || !here) return false;
|
|
1211
|
+
const claim = claimedSize(row);
|
|
1212
|
+
if (!budget.take(claim)) {
|
|
1213
|
+
out.budgetExhausted = true;
|
|
1214
|
+
return false;
|
|
1215
|
+
}
|
|
1216
|
+
// PARK FIRST. If the copy does not land, the overwrite is refused —
|
|
1217
|
+
// DDR-102's fail-closed rule, applied to files: a loser we cannot
|
|
1218
|
+
// recover is a loser we must not create.
|
|
1219
|
+
const copyRel = parkLocal(rel);
|
|
1220
|
+
if (copyRel === null) {
|
|
1221
|
+
out.failed.push({ rel, reason: 'could not park the local copy — refusing to overwrite' });
|
|
1222
|
+
ledger.setState(rel, 'stuck', {
|
|
1223
|
+
reason: 'both sides changed this file and the local copy could not be parked',
|
|
1224
|
+
});
|
|
1225
|
+
return false;
|
|
1226
|
+
}
|
|
1227
|
+
const got = await fetchVerified(rel, remoteHash, budget.remaining() + claim);
|
|
1228
|
+
if (!got.ok) {
|
|
1229
|
+
out.failed.push({ rel, reason: got.reason });
|
|
1230
|
+
if (got.overCap && budget.remaining() + claim < MAX_FILE_BYTES) {
|
|
1231
|
+
out.budgetExhausted = true;
|
|
1232
|
+
}
|
|
1233
|
+
return false;
|
|
1234
|
+
}
|
|
1235
|
+
if (!chargeOverrun(budget, rel, got.bytes.byteLength, claim, out)) return false;
|
|
1236
|
+
const landed = await ledger.adoptAfter(rel, remoteHash, () => materialize(rel, got.bytes), {
|
|
1237
|
+
...(row ? { remoteSeq: row.seq } : {}),
|
|
1238
|
+
state: 'conflict',
|
|
1239
|
+
});
|
|
1240
|
+
if (!landed) {
|
|
1241
|
+
out.failed.push({ rel, reason: 'could not materialize the hub copy' });
|
|
1242
|
+
return false;
|
|
1243
|
+
}
|
|
1244
|
+
restat(rel);
|
|
1245
|
+
ledger.setState(rel, 'conflict', {
|
|
1246
|
+
reason: 'both sides changed this file; your version is beside it',
|
|
1247
|
+
conflictCopy: copyRel,
|
|
1248
|
+
});
|
|
1249
|
+
out.conflicts.push({ rel, copy: copyRel });
|
|
1250
|
+
// The copy travels too, so BOTH ends see the conflict rather than only
|
|
1251
|
+
// the machine that happened to lose.
|
|
1252
|
+
const copyLocal = statLocal(copyRel);
|
|
1253
|
+
if (copyLocal) {
|
|
1254
|
+
const res = await push(copyLocal, null);
|
|
1255
|
+
if (res.ok) out.pushed.push(copyRel);
|
|
1256
|
+
}
|
|
1257
|
+
return true;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
case 'quarantine': {
|
|
1261
|
+
// The hub deleted a file this machine still holds UNCHANGED. Quarantine
|
|
1262
|
+
// rather than unlink: `_trash/` is the recoverability spine, and it is
|
|
1263
|
+
// runtime state (DDR-115), so one person's delete never replicates as
|
|
1264
|
+
// everyone's copy of the deleted file.
|
|
1265
|
+
const parked = quarantineLocal(rel);
|
|
1266
|
+
if (parked === null) {
|
|
1267
|
+
// PARK FIRST, the same fail-closed rule the conflict path follows: a
|
|
1268
|
+
// loser we cannot recover is a loser we must not create. Reporting a
|
|
1269
|
+
// delete here and forgetting the row would also resurrect the file
|
|
1270
|
+
// on the next pass, because with no ancestor it reads as brand new.
|
|
1271
|
+
out.failed.push({ rel, reason: 'could not quarantine — refusing to delete' });
|
|
1272
|
+
ledger.setState(rel, 'stuck', {
|
|
1273
|
+
reason:
|
|
1274
|
+
'the project deleted this file and it could not be moved to _trash/, so it was kept',
|
|
1275
|
+
});
|
|
1276
|
+
return false;
|
|
1277
|
+
}
|
|
1278
|
+
ledger.forget(rel);
|
|
1279
|
+
out.deleted.push({ rel, parked });
|
|
1280
|
+
return true;
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
case 'propagate-delete': {
|
|
1284
|
+
// Gone here, and the hub still holds exactly what we last reconciled —
|
|
1285
|
+
// the Syncthing rule. The CAS carries our ancestor, so an edit that
|
|
1286
|
+
// landed in between wins and this comes back as a conflict instead.
|
|
1287
|
+
const res = await pushDelete(rel, ledger.ancestorOf(rel));
|
|
1288
|
+
if (res.ok) {
|
|
1289
|
+
ledger.forget(rel);
|
|
1290
|
+
out.deleted.push({ rel, parked: null });
|
|
1291
|
+
return true;
|
|
1292
|
+
}
|
|
1293
|
+
if ('conflict' in res && res.conflict) {
|
|
1294
|
+
// Somebody edited it after we last saw it. Say nothing further; the
|
|
1295
|
+
// next pass reads their row and `local-deleted-but-remote-moved`
|
|
1296
|
+
// brings the file back.
|
|
1297
|
+
ledger.setState(rel, 'stuck', {
|
|
1298
|
+
reason: 'deleted here, but somebody changed it on the hub — their edit wins',
|
|
1299
|
+
});
|
|
1300
|
+
return false;
|
|
1301
|
+
}
|
|
1302
|
+
out.failed.push({ rel, reason: 'reason' in res ? res.reason : 'delete refused' });
|
|
1303
|
+
return false;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
default: {
|
|
1307
|
+
const never: never = decision.action;
|
|
1308
|
+
throw new Error(`file-plane: unhandled action ${String(never)}`);
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
function restat(rel: string): void {
|
|
1314
|
+
try {
|
|
1315
|
+
const st = statSync(path.join(designRoot, rel));
|
|
1316
|
+
const row = ledger.row(rel);
|
|
1317
|
+
if (row?.syncedHash) ledger.noteLocal(rel, row.syncedHash, st.size, st.mtimeMs);
|
|
1318
|
+
} catch {
|
|
1319
|
+
/* the next pass re-hashes it */
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
function statLocal(rel: string): LocalFile | null {
|
|
1324
|
+
const abs = path.join(designRoot, rel);
|
|
1325
|
+
try {
|
|
1326
|
+
const st = statSync(abs);
|
|
1327
|
+
const hash = sha256(readFileSync(abs));
|
|
1328
|
+
return { rel, abs, hash, size: st.size, mtimeMs: st.mtimeMs, cls: 'inert-media' };
|
|
1329
|
+
} catch {
|
|
1330
|
+
return null;
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
return {
|
|
1335
|
+
reconcile,
|
|
1336
|
+
doruceka() {
|
|
1337
|
+
const out: Record<string, DeliveryState> = {};
|
|
1338
|
+
for (const [rel, row] of Object.entries(ledger.rows())) {
|
|
1339
|
+
out[rel] = row.state ?? 'local-only';
|
|
1340
|
+
}
|
|
1341
|
+
return out;
|
|
1342
|
+
},
|
|
1343
|
+
};
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
function drop(out: FilePlaneResult, rel: string, reason: string, ledger?: FileLedger): void {
|
|
1347
|
+
out.dropped.push({ rel, reason });
|
|
1348
|
+
// Only for a path this machine actually tracks. A path we have never held
|
|
1349
|
+
// and will never accept is not "stuck" — it is not ours, and minting a row
|
|
1350
|
+
// to say so is how a hostile page turned one response into permanent state.
|
|
1351
|
+
if (ledger && !ledger.row(rel)) return;
|
|
1352
|
+
// A REFUSAL OUTRANKS EVERYTHING (DDR-214, applied to files). A path this
|
|
1353
|
+
// peer declines is not "local-only" — it is not here at all, and never will
|
|
1354
|
+
// be until something changes. Letting it fall through to the default state
|
|
1355
|
+
// would put a file in the panel's ordinary column that is in fact being
|
|
1356
|
+
// actively refused, which is the shape of "we didn't know it was stuck".
|
|
1357
|
+
ledger?.setState(rel, 'stuck', { reason });
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/** `assets/<name>` referenced anywhere in the tree — the priority front-queue. */
|
|
1361
|
+
function referencedAssetNames(designRoot: string): Set<string> {
|
|
1362
|
+
const out = new Set<string>();
|
|
1363
|
+
const RE = /assets\/([A-Za-z0-9._-]+\.[A-Za-z0-9]+)/g;
|
|
1364
|
+
const SCANNED = /\.(?:annotations\.svg|tsx|jsx|css|meta\.json)$/i;
|
|
1365
|
+
const walk = (dir: string, depth: number): void => {
|
|
1366
|
+
if (depth > MAX_WALK_DEPTH) return;
|
|
1367
|
+
let entries: Dirent[];
|
|
1368
|
+
try {
|
|
1369
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
1370
|
+
} catch {
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
for (const entry of entries) {
|
|
1374
|
+
if (entry.name.startsWith('.') || SKIPPED_DIRS.has(entry.name)) continue;
|
|
1375
|
+
const abs = path.join(dir, entry.name);
|
|
1376
|
+
if (entry.isDirectory()) {
|
|
1377
|
+
if (entry.name === 'assets') continue;
|
|
1378
|
+
walk(abs, depth + 1);
|
|
1379
|
+
continue;
|
|
1380
|
+
}
|
|
1381
|
+
if (!SCANNED.test(entry.name)) continue;
|
|
1382
|
+
try {
|
|
1383
|
+
for (const m of readFileSync(abs, 'utf8').matchAll(RE)) {
|
|
1384
|
+
if (m[1]) out.add(m[1]);
|
|
1385
|
+
}
|
|
1386
|
+
} catch {
|
|
1387
|
+
/* unreadable is not referenced */
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
};
|
|
1391
|
+
walk(designRoot, 1);
|
|
1392
|
+
return out;
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
/** 0 = a referenced asset (front of the queue), 1 = everything else. */
|
|
1396
|
+
function rank(rel: string, referenced: Set<string>): number {
|
|
1397
|
+
const slash = rel.lastIndexOf('/');
|
|
1398
|
+
const base = slash === -1 ? rel : rel.slice(slash + 1);
|
|
1399
|
+
return referenced.has(base) ? 0 : 1;
|
|
1400
|
+
}
|