@volter/world-core 3.0.155 → 3.0.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generated/pack-facts.json +143 -12
- package/dist/inject.cjs +150 -86
- package/dist/src/actions.d.ts +15 -0
- package/dist/src/actions.js +82 -21
- package/dist/src/ancestry.d.ts +5 -4
- package/dist/src/ancestry.js +53 -28
- package/dist/src/changeset.js +1 -1
- package/dist/src/clickhouse/compression.d.ts +28 -0
- package/dist/src/clickhouse/compression.js +281 -0
- package/dist/src/clickhouse/index.d.ts +1 -0
- package/dist/src/clickhouse/index.js +1 -0
- package/dist/src/derived-core.d.ts +12 -1
- package/dist/src/derived-core.js +145 -11
- package/dist/src/derived.js +3 -0
- package/dist/src/fork.js +1 -1
- package/dist/src/grpc-wire.js +2 -1
- package/dist/src/h2.js +63 -10
- package/dist/src/head.d.ts +3 -0
- package/dist/src/head.js +9 -0
- package/dist/src/history.d.ts +1 -1
- package/dist/src/history.js +7 -7
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.js +5 -1
- package/dist/src/log.d.ts +23 -3
- package/dist/src/log.js +160 -37
- package/dist/src/managed-original-inputs.d.ts +101 -0
- package/dist/src/managed-original-inputs.js +125 -0
- package/dist/src/mongodb/client.d.ts +8 -0
- package/dist/src/mongodb/client.js +27 -0
- package/dist/src/mongodb/index.d.ts +2 -0
- package/dist/src/mongodb/index.js +5 -0
- package/dist/src/mongodb/stream.d.ts +74 -0
- package/dist/src/mongodb/stream.js +206 -0
- package/dist/src/observe.js +1 -1
- package/dist/src/pack-fetch.js +20 -8
- package/dist/src/packRegistry.d.ts +10 -14
- package/dist/src/packRegistry.js +2 -13
- package/dist/src/protocol.d.ts +13 -0
- package/dist/src/protocol.js +13 -0
- package/dist/src/redis/engine.d.ts +10 -4
- package/dist/src/redis/engine.js +103 -32
- package/dist/src/redis/member-storage.js +20 -8
- package/dist/src/redis/service-times.d.ts +25 -0
- package/dist/src/redis/service-times.js +87 -0
- package/dist/src/redis/stream.d.ts +2 -1
- package/dist/src/redis/stream.js +109 -20
- package/dist/src/runtime-state.d.ts +11 -0
- package/dist/src/runtime-state.js +13 -0
- package/dist/src/runtime.d.ts +4 -2
- package/dist/src/runtime.js +5 -2
- package/dist/src/serve-http.d.ts +2 -0
- package/dist/src/serve-http.js +21 -4
- package/dist/src/serve.d.ts +10 -0
- package/dist/src/serve.js +43 -3
- package/dist/src/storage.d.ts +2 -1
- package/dist/src/storage.js +6 -3
- package/dist/src/world-boot.d.ts +1 -0
- package/dist/src/world-boot.js +6 -0
- package/dist/src/world-clock.js +6 -2
- package/dist/src/world-store.d.ts +108 -7
- package/dist/src/world-store.js +883 -103
- package/dist/src/xml.d.ts +30 -0
- package/dist/src/xml.js +74 -0
- package/generated/pack-facts.json +143 -12
- package/inject.cjs +150 -86
- package/package.json +20 -1
- package/src/actions.ts +78 -21
- package/src/ancestry.ts +46 -24
- package/src/changeset.ts +1 -1
- package/src/clickhouse/compression.ts +172 -0
- package/src/clickhouse/index.ts +2 -0
- package/src/derived-core.ts +142 -11
- package/src/derived.ts +3 -0
- package/src/fork.ts +1 -1
- package/src/grpc-wire.ts +2 -1
- package/src/h2.ts +46 -5
- package/src/head.ts +9 -0
- package/src/history.ts +7 -7
- package/src/index.ts +6 -1
- package/src/log.ts +147 -32
- package/src/managed-original-inputs.ts +163 -0
- package/src/mongodb/client.ts +27 -0
- package/src/mongodb/index.ts +5 -0
- package/src/mongodb/stream.ts +213 -0
- package/src/observe.ts +1 -1
- package/src/pack-fetch.ts +20 -8
- package/src/packRegistry.ts +3 -15
- package/src/protocol.ts +13 -0
- package/src/redis/engine.ts +100 -57
- package/src/redis/member-storage.ts +17 -9
- package/src/redis/service-times.ts +81 -0
- package/src/redis/stream.ts +66 -14
- package/src/runtime-state.ts +14 -0
- package/src/runtime.ts +6 -1
- package/src/serve-http.ts +23 -4
- package/src/serve.ts +45 -3
- package/src/storage.ts +5 -3
- package/src/world-boot.ts +6 -0
- package/src/world-clock.ts +6 -2
- package/src/world-store.ts +704 -91
- package/src/xml.ts +71 -0
package/dist/src/world-store.js
CHANGED
|
@@ -23,23 +23,480 @@
|
|
|
23
23
|
// runs on the serve path. Its bespoke inode-identity lock and reservation semantics
|
|
24
24
|
// cannot be expressed through this generic seam without weakening them, so it stays on
|
|
25
25
|
// `node:fs`. See its module header.
|
|
26
|
-
import { appendFileSync, chmodSync, closeSync, existsSync, fsyncSync, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, realpathSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
26
|
+
import { appendFileSync, chmodSync, closeSync, existsSync, fsyncSync, fstatSync, linkSync, ftruncateSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, realpathSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync, } from 'node:fs';
|
|
27
27
|
import { hostname } from 'node:os';
|
|
28
|
+
import { randomUUID } from 'node:crypto';
|
|
28
29
|
import { refuseReadOnlyWrite } from "./request-scope.js";
|
|
29
30
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
31
|
+
/** Capture at the outermost lock helper, then carry the same caller through nested helpers. */
|
|
32
|
+
export function storageLockScope(options = {}) {
|
|
33
|
+
const frame = options.caller ? undefined : new Error().stack?.split('\n')[3];
|
|
34
|
+
const location = frame?.slice(frame.lastIndexOf('(') + 1).trim().replace(/^at\s+/, '');
|
|
35
|
+
const site = location?.match(/^(.*):(\d+):\d+\)?$/);
|
|
36
|
+
const caller = options.caller ?? (site ? `${site[1].replace(/^file:\/\//, '')}:${site[2]}` : 'unknown:0');
|
|
37
|
+
return { ...options, caller, operation: options.operation ?? caller };
|
|
38
|
+
}
|
|
39
|
+
/** The files a service's tree is folded from, by name in its state directory (log.ts: the parent log, the branch log,
|
|
40
|
+
* the branch's pointer), and the generation a store keeps beside them. */
|
|
41
|
+
export const TREE_FILES = { parent: 'events.jsonl', branch: 'actions.jsonl', pointer: 'branch.json' };
|
|
42
|
+
export const TREE_GENERATION = 'tree.generation';
|
|
43
|
+
const TREE_FILE_NAMES = new Set(Object.values(TREE_FILES));
|
|
30
44
|
// ── FsWorldStore — the DEFAULT, byte-identical to the historical node:fs behavior ────
|
|
31
45
|
let fsAtomicSequence = 0;
|
|
32
46
|
/** How long a lock may live before an acquirer treats it as abandoned. Above the 10s
|
|
33
47
|
* wait timeout so a live-but-slow holder is never reclaimed out from under itself. */
|
|
34
48
|
const LOCK_STALE_MS = 60_000;
|
|
35
|
-
|
|
49
|
+
/**
|
|
50
|
+
* How long an EMPTY lock file is a taker's, between its exclusive create and its holder's record (S2 R13). A taker
|
|
51
|
+
* writes the record in the call after the create (in a tab's file step, in the same request), and one whose record is
|
|
52
|
+
* not written gives the lock up and takes it again (withLock), so no live holder holds a lock with an empty record past
|
|
53
|
+
* that moment. An empty file older than this is a taker's that ended between the two calls (a process killed there,
|
|
54
|
+
* or a state directory copied while one was there), and it is no one's: before, it was waited on until its file was
|
|
55
|
+
* 60 s old, and a lock taken to keep its live owner (the ancestry coordinator) waited on it for ever. Twenty e946aa4c
|
|
56
|
+
* dry-run1: a 0-byte branch-ancestry.lock, the Redis twin's every write waiting 10 s on it inside its history lock,
|
|
57
|
+
* and Twenty's Redis client's 5 s connect timing out (FindAllViews 500, the walk's step 7).
|
|
58
|
+
*/
|
|
59
|
+
// Five seconds: in a tab without the kernel's file step the create and the record are two requests of the kernel,
|
|
60
|
+
// and Twenty's Redis-twin handoffs have exceeded 1 s; and under the 10 s a
|
|
61
|
+
// waiter waits before it gives up, so a waiter reclaims such a file rather than timing out on it.
|
|
62
|
+
const EMPTY_RECORD_GRACE_MS = 5_000;
|
|
63
|
+
/** How many takes whose record could not be written are made before the failure is the caller's. */
|
|
64
|
+
const RECORD_ATTEMPTS = 3;
|
|
65
|
+
/** The locks this process holds now, by path, and the token each was recorded with (withLock). */
|
|
66
|
+
const ownLocks = new Map();
|
|
67
|
+
const lockReadings = [];
|
|
68
|
+
function fileSteps() {
|
|
69
|
+
const tree = process[Symbol.for('tabnode.run.vfs')];
|
|
70
|
+
const offered = tree?.[Symbol.for('browser-substrate.kernel-file-steps')];
|
|
71
|
+
return offered && typeof offered === 'object' ? offered : undefined;
|
|
72
|
+
}
|
|
73
|
+
/** The bytes a holder's record holds for its token, as JSON.stringify writes them: a key and a string, no space. */
|
|
74
|
+
const tokenNeedle = (token) => `"token":${JSON.stringify(token)}`;
|
|
75
|
+
/** A two-call take whose path did not hold its own record after it: another reclaimed it between the calls. */
|
|
76
|
+
const RECLAIMED_BEFORE_RECORD = 'VOLTER_LOCK_RECLAIMED';
|
|
77
|
+
/** A take whose record could not be written whole: nothing of it is held. */
|
|
78
|
+
const RECORD_NOT_WRITTEN = 'VOLTER_LOCK_UNRECORDED';
|
|
79
|
+
/** Where link(2) is not an exclusive create the store can use: the take falls back to two calls. */
|
|
80
|
+
const LINK_UNSUPPORTED = new Set(['EPERM', 'ENOTSUP', 'EOPNOTSUPP', 'ENOSYS', 'EXDEV', 'EMLINK', 'EINVAL']);
|
|
81
|
+
/**
|
|
82
|
+
* A FILE MADE EXCLUSIVELY WITH ITS RECORD ALREADY IN IT (S2 R13): a lock, or a lock's reclaim record, is never seen
|
|
83
|
+
* empty while its taker lives, so an empty one is always a dead taker's and is reclaimed (lockIsStale, recoveryIsStale).
|
|
84
|
+
* - In a tab: the kernel's file step makes it and writes the record in one request (fileSteps, S2 R17d).
|
|
85
|
+
* - On a host: the record is written whole to a private file of the same directory, which link(2) puts at the path,
|
|
86
|
+
* an exclusive create that fails EEXIST; the private name is then removed. The lock is opened for its descriptor.
|
|
87
|
+
* Never in a tab: tabnode's link copies over the path (no EEXIST), so it is no exclusive create there.
|
|
88
|
+
* - Neither (a tab whose kernel offers no step for the path, a host filesystem without link): two calls, the
|
|
89
|
+
* exclusive create and the record, and the path read back: held only if it holds this take's token. A record that
|
|
90
|
+
* is not written leaves the file removed only where it is still empty or this take's (read before it is removed).
|
|
91
|
+
* EEXIST is thrown as the create's; RECLAIMED_BEFORE_RECORD and RECORD_NOT_WRITTEN say a take not held.
|
|
92
|
+
*/
|
|
93
|
+
function createRecorded(path, text, token, operation) {
|
|
94
|
+
const bytes = Buffer.byteLength(text);
|
|
95
|
+
const steps = fileSteps();
|
|
96
|
+
const stepped = steps?.createExclusive(path, text, 0o666);
|
|
97
|
+
if (stepped === true)
|
|
98
|
+
return { fd: null, bytes };
|
|
99
|
+
if (stepped === false) {
|
|
100
|
+
// made in that one request and its record short. Removed only while the path still holds that short record (a
|
|
101
|
+
// prefix of this take's own text) or nothing: a stall past the grace may have let another reclaim it and take it
|
|
102
|
+
lockRecordLine('publish-failed', path, { operation });
|
|
103
|
+
let held;
|
|
104
|
+
try {
|
|
105
|
+
held = readFileSync(path, 'utf8');
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
held = undefined;
|
|
109
|
+
}
|
|
110
|
+
if (held !== undefined && (held === '' || text.startsWith(held))) {
|
|
111
|
+
try {
|
|
112
|
+
unlinkSync(path);
|
|
113
|
+
}
|
|
114
|
+
catch { /* gone */ }
|
|
115
|
+
}
|
|
116
|
+
throw Object.assign(new Error(`${path}: its record was not written whole`), { code: RECORD_NOT_WRITTEN });
|
|
117
|
+
}
|
|
118
|
+
const inTab = process[Symbol.for('tabnode.run.vfs')] !== undefined;
|
|
119
|
+
if (!inTab) {
|
|
120
|
+
const temporary = `${path}.take.${process.pid}.${randomUUID()}`;
|
|
121
|
+
let linked = false;
|
|
122
|
+
try {
|
|
123
|
+
writeFileSync(temporary, text, { flag: 'wx' });
|
|
124
|
+
try {
|
|
125
|
+
linkSync(temporary, path);
|
|
126
|
+
linked = true;
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
if (error.code === 'EEXIST')
|
|
130
|
+
sweepTakeFiles(path);
|
|
131
|
+
if (!LINK_UNSUPPORTED.has(error.code ?? ''))
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
try {
|
|
137
|
+
unlinkSync(temporary);
|
|
138
|
+
}
|
|
139
|
+
catch { /* not made, or gone */ }
|
|
140
|
+
}
|
|
141
|
+
if (linked) {
|
|
142
|
+
let fd = null;
|
|
143
|
+
try {
|
|
144
|
+
fd = openSync(path, 'r+');
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
fd = null; /* held; its record is then not rewritten */
|
|
148
|
+
}
|
|
149
|
+
return { fd, bytes };
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const fd = openSync(path, 'wx');
|
|
153
|
+
try {
|
|
154
|
+
writeFileSync(fd, text);
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
lockRecordLine('publish-failed', path, { fd, operation }, error);
|
|
158
|
+
try {
|
|
159
|
+
closeSync(fd);
|
|
160
|
+
}
|
|
161
|
+
catch { /* not held */ }
|
|
162
|
+
let held = '';
|
|
163
|
+
try {
|
|
164
|
+
held = readFileSync(path, 'utf8');
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
held = 'gone';
|
|
168
|
+
}
|
|
169
|
+
if (held === '' || held.includes(tokenNeedle(token))) {
|
|
170
|
+
try {
|
|
171
|
+
unlinkSync(path);
|
|
172
|
+
}
|
|
173
|
+
catch { /* gone */ }
|
|
174
|
+
}
|
|
175
|
+
throw Object.assign(new Error(`${path}: its record could not be written`), { code: RECORD_NOT_WRITTEN, cause: error });
|
|
176
|
+
}
|
|
177
|
+
if (readLockHolder(path)?.token !== token) {
|
|
178
|
+
lockRecordLine('take-reclaimed-before-record', path, { fd, operation });
|
|
179
|
+
try {
|
|
180
|
+
closeSync(fd);
|
|
181
|
+
}
|
|
182
|
+
catch { /* not held */ }
|
|
183
|
+
throw Object.assign(new Error(`${path}: reclaimed before its record`), { code: RECLAIMED_BEFORE_RECORD });
|
|
184
|
+
}
|
|
185
|
+
return { fd, bytes };
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* A take's private file a process left at a crash between its link and its removal (`<lock>.take.<pid>.<uuid>`):
|
|
189
|
+
* removed where older than the grace, once a process per lock, when its take meets the lock held. It holds nothing.
|
|
190
|
+
*/
|
|
191
|
+
const sweptTakes = new Set();
|
|
192
|
+
function sweepTakeFiles(path) {
|
|
193
|
+
if (sweptTakes.has(path))
|
|
194
|
+
return;
|
|
195
|
+
sweptTakes.add(path);
|
|
196
|
+
const prefix = `${basename(path)}.take.`;
|
|
36
197
|
try {
|
|
37
|
-
|
|
198
|
+
for (const name of readdirSync(dirname(path))) {
|
|
199
|
+
if (!name.startsWith(prefix))
|
|
200
|
+
continue;
|
|
201
|
+
const at = join(dirname(path), name);
|
|
202
|
+
try {
|
|
203
|
+
if (Date.now() - statSync(at).mtimeMs > EMPTY_RECORD_GRACE_MS)
|
|
204
|
+
unlinkSync(at);
|
|
205
|
+
}
|
|
206
|
+
catch { /* gone, or another's to remove */ }
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
catch { /* an instrument of tidiness: nothing waits on it */ }
|
|
210
|
+
}
|
|
211
|
+
/** A file createRecorded made, let go: its descriptor closed, and the path removed only while it holds the token. */
|
|
212
|
+
function releaseRecorded(path, token, fd) {
|
|
213
|
+
if (fd !== null) {
|
|
214
|
+
try {
|
|
215
|
+
closeSync(fd);
|
|
216
|
+
}
|
|
217
|
+
catch { /* released below */ }
|
|
218
|
+
}
|
|
219
|
+
try {
|
|
220
|
+
const released = fd === null ? fileSteps()?.changeHolding(path, tokenNeedle(token), undefined) : undefined;
|
|
221
|
+
if (released?.changed)
|
|
222
|
+
return;
|
|
223
|
+
if (released === undefined && readLockHolder(path)?.token === token)
|
|
224
|
+
unlinkSync(path);
|
|
225
|
+
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
lockRecordLine('release-failed', path, {}, error);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* A leftover reclaim record (`.recovery`) no reclaimer is behind (S2 R13 review): empty, or recorded by a process of
|
|
232
|
+
* this host that is not alive, and older than EMPTY_RECORD_GRACE_MS. It held every waiter of its lock until each
|
|
233
|
+
* timed out, as an empty lock file did.
|
|
234
|
+
*/
|
|
235
|
+
function recoveryIsStale(recoveryPath) {
|
|
236
|
+
let file;
|
|
237
|
+
try {
|
|
238
|
+
file = statSync(recoveryPath);
|
|
38
239
|
}
|
|
39
240
|
catch {
|
|
241
|
+
return false;
|
|
242
|
+
}
|
|
243
|
+
if (Date.now() - file.mtimeMs <= EMPTY_RECORD_GRACE_MS)
|
|
244
|
+
return false;
|
|
245
|
+
if (file.size === 0)
|
|
246
|
+
return true;
|
|
247
|
+
const holder = readLockHolder(recoveryPath);
|
|
248
|
+
return holder !== null && holder.hostname === ownHostname() && Number.isInteger(holder.pid) && !pidAlive(holder.pid);
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* A WORLD'S CARRIED LOCKS, REMOVED AT ITS OPEN (S2 K4). A World booted over its kept state (`up --keep-state`, a
|
|
252
|
+
* tab's restored archive) carries whatever lock files its last life left in its services' state directories. One whose
|
|
253
|
+
* holder record cannot be read is no holder's once its taker has gone, and a restore gives it the restore's file time,
|
|
254
|
+
* so the empty-record grace and the 60 s age ran from the restore: every waiter waited them, or gave up at 10 s (Twenty
|
|
255
|
+
* e946aa4c run 2: anthropic's projection.lock, "holder record unreadable", 10,086 ms, then its credentials 500 and the
|
|
256
|
+
* server 10 s late). A carried `.recovery` that is not empty and does not parse was never ended by R13's rule at all.
|
|
257
|
+
*
|
|
258
|
+
* A file is carried only when all of these hold: it is a world-core lock or its reclaim record (`*.lock`,
|
|
259
|
+
* `*.lock.recovery`) in a service's state directory (`<root>/<service>/<stateDir>/world`, never a yarn.lock or any
|
|
260
|
+
* project file); its text is empty or a holder record's beginning (`{"pid"`) that does not parse; and its file time is
|
|
261
|
+
* before `before`, the start of this boot's claim. A restore sets the time before the claim; a taker that is live now
|
|
262
|
+
* (a CLI's changeset rebase, a branch World keeping this one as its parent's storage, a two-call taker between its
|
|
263
|
+
* create and its record) made its file after it. Each removal is still a reclaimer's: a lock's under its `.recovery`,
|
|
264
|
+
* taken with its record (createRecorded), a held `.recovery` skipped; the file read again, renamed aside, the aside copy
|
|
265
|
+
* read again and linked back where it now holds a record (EEXIST there is the path taken since, left to its holder).
|
|
266
|
+
* A record that parses is kept, under its rules (a dead holder's, our own pid's, the 60 s age, which a restore resets:
|
|
267
|
+
* not this rule's). Answers the paths removed.
|
|
268
|
+
*/
|
|
269
|
+
export function removeCarriedLocks(roots, options) {
|
|
270
|
+
const removed = [];
|
|
271
|
+
const isHolder = (text) => Number.isInteger(lockHolderOf('', text)?.pid);
|
|
272
|
+
const carriedText = (text) => {
|
|
273
|
+
const trimmed = text.trim();
|
|
274
|
+
return trimmed === '' || (trimmed.startsWith('{"pid"') && !isHolder(trimmed));
|
|
275
|
+
};
|
|
276
|
+
/** The file's text where it is carried now (small, older than the claim, unreadable as a record); else undefined. */
|
|
277
|
+
const carriedNow = (path) => {
|
|
278
|
+
try {
|
|
279
|
+
const file = statSync(path);
|
|
280
|
+
if (file.size > 1024 || !(file.mtimeMs < options.before))
|
|
281
|
+
return undefined;
|
|
282
|
+
const text = readFileSync(path, 'utf8');
|
|
283
|
+
return carriedText(text) ? text : undefined;
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
return undefined;
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
/** Renamed aside, read again, put back where a taker recorded itself meanwhile; true when it is gone. */
|
|
290
|
+
const setAside = (path, text) => {
|
|
291
|
+
const aside = `${path}.carried.${process.pid}.${randomUUID()}`;
|
|
292
|
+
try {
|
|
293
|
+
renameSync(path, aside);
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
if (error.code !== 'ENOENT')
|
|
297
|
+
lockRecordLine('carried-rename-failed', path, {}, error);
|
|
298
|
+
return false;
|
|
299
|
+
}
|
|
300
|
+
let after;
|
|
301
|
+
try {
|
|
302
|
+
after = readFileSync(aside, 'utf8');
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
after = undefined;
|
|
306
|
+
}
|
|
307
|
+
if (after !== undefined && isHolder(after)) {
|
|
308
|
+
try {
|
|
309
|
+
linkSync(aside, path);
|
|
310
|
+
lockRecordLine('carried-restored', path, {});
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
lockRecordLine(error.code === 'EEXIST' ? 'carried-retaken' : 'carried-restore-failed', path, {}, error);
|
|
314
|
+
}
|
|
315
|
+
try {
|
|
316
|
+
unlinkSync(aside);
|
|
317
|
+
}
|
|
318
|
+
catch (error) {
|
|
319
|
+
lockRecordLine('carried-unlink-failed', aside, {}, error);
|
|
320
|
+
}
|
|
321
|
+
return false;
|
|
322
|
+
}
|
|
323
|
+
storageLockLine(`[world] carried storage lock renamed aside at open path=${path} bytes=${Buffer.byteLength(text)}`);
|
|
324
|
+
try {
|
|
325
|
+
unlinkSync(aside);
|
|
326
|
+
}
|
|
327
|
+
catch (error) {
|
|
328
|
+
lockRecordLine('carried-unlink-failed', aside, {}, error);
|
|
329
|
+
}
|
|
330
|
+
return true;
|
|
331
|
+
};
|
|
332
|
+
const removeLock = (path) => {
|
|
333
|
+
const recoveryPath = `${path}.recovery`, recoveryToken = randomUUID();
|
|
334
|
+
let recovery;
|
|
335
|
+
try {
|
|
336
|
+
recovery = createRecorded(recoveryPath, `${JSON.stringify({ pid: process.pid, hostname: ownHostname(), at: new Date().toISOString(), token: recoveryToken })}\n`, recoveryToken, 'carried lock removal');
|
|
337
|
+
}
|
|
338
|
+
catch {
|
|
339
|
+
return;
|
|
340
|
+
} // a reclaimer is at work, or the recovery is carried too and goes first (visit)
|
|
341
|
+
try {
|
|
342
|
+
const text = carriedNow(path);
|
|
343
|
+
if (text !== undefined && setAside(path, text))
|
|
344
|
+
removed.push(path);
|
|
345
|
+
}
|
|
346
|
+
finally {
|
|
347
|
+
releaseRecorded(recoveryPath, recoveryToken, recovery.fd);
|
|
348
|
+
}
|
|
349
|
+
};
|
|
350
|
+
const visit = (directory, depth) => {
|
|
351
|
+
let entries;
|
|
352
|
+
try {
|
|
353
|
+
entries = readdirSync(directory, { withFileTypes: true });
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
// A directory's carried reclaim records go first: a lock's removal is made under its `.recovery`.
|
|
359
|
+
const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
|
|
360
|
+
for (const name of files.filter((name) => name.endsWith('.lock.recovery'))) {
|
|
361
|
+
const path = join(directory, name), text = carriedNow(path);
|
|
362
|
+
if (text !== undefined && setAside(path, text))
|
|
363
|
+
removed.push(path);
|
|
364
|
+
}
|
|
365
|
+
for (const name of files.filter((name) => name.endsWith('.lock'))) {
|
|
366
|
+
const path = join(directory, name);
|
|
367
|
+
if (carriedNow(path) !== undefined)
|
|
368
|
+
removeLock(path);
|
|
369
|
+
}
|
|
370
|
+
for (const entry of entries) {
|
|
371
|
+
if (entry.isDirectory() && depth < 8 && entry.name !== 'node_modules' && entry.name !== '.git')
|
|
372
|
+
visit(join(directory, entry.name), depth + 1);
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
// Only the services' state directories (`<root>/<service>/<stateDir>/world`, and `<root>/<stateDir>/world`): never a
|
|
376
|
+
// service's own data beside them (a PostgreSQL cluster, a project tree).
|
|
377
|
+
for (const root of roots) {
|
|
378
|
+
const states = [join(root, options.stateDir, 'world')];
|
|
379
|
+
try {
|
|
380
|
+
for (const entry of readdirSync(root, { withFileTypes: true }))
|
|
381
|
+
if (entry.isDirectory())
|
|
382
|
+
states.push(join(root, entry.name, options.stateDir, 'world'));
|
|
383
|
+
}
|
|
384
|
+
catch {
|
|
385
|
+
continue;
|
|
386
|
+
}
|
|
387
|
+
for (const state of states)
|
|
388
|
+
if (existsSync(state))
|
|
389
|
+
visit(state, 0);
|
|
390
|
+
}
|
|
391
|
+
return removed;
|
|
392
|
+
}
|
|
393
|
+
/** This host's name. In a tab (where the kernel's file steps are offered) it is read once per process: every take
|
|
394
|
+
* records it, and each read was a request of the kernel (its UTS); a tab's process renamed while it runs keeps
|
|
395
|
+
* recording, and comparing, the name it started with. On a host it is read live, as before: there a rename
|
|
396
|
+
* (name.local -> name-2.local on macOS) must reach lockIsStale's comparison, or a dead holder recorded under the new
|
|
397
|
+
* name is never reclaimed, and the read costs no request. */
|
|
398
|
+
let hostnameRead;
|
|
399
|
+
const ownHostname = () => (fileSteps() ? (hostnameRead ??= hostname()) : hostname());
|
|
400
|
+
function recordLockReading(reading) {
|
|
401
|
+
try {
|
|
402
|
+
const text = `${JSON.stringify(reading.holder)}\n`;
|
|
403
|
+
// a take with no descriptor rewrites its record by path, and only while the file still holds its token: a file a
|
|
404
|
+
// stale sweep reclaimed and another made is never written over (the descriptor wrote the reclaimed file's own)
|
|
405
|
+
if (reading.fd === null) {
|
|
406
|
+
fileSteps()?.changeHolding(reading.path, tokenNeedle(reading.holder.token), text);
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
// Padded with spaces to the record it replaces, which JSON allows after a value: a write that ends early leaves the
|
|
410
|
+
// new record's text over the old one's, never a shorter file with the old one's tail (malformed, and a malformed
|
|
411
|
+
// record keeps the 60 s rule, which a coordinator's live-owner lock never reaches).
|
|
412
|
+
const bytes = Buffer.byteLength(text), padded = reading.recordBytes !== undefined && bytes < reading.recordBytes
|
|
413
|
+
? `${text.slice(0, -1)}${' '.repeat(reading.recordBytes - bytes)}\n` : text;
|
|
414
|
+
writeSync(reading.fd, padded, 0, 'utf8');
|
|
415
|
+
ftruncateSync(reading.fd, Buffer.byteLength(padded));
|
|
416
|
+
reading.recordBytes = Buffer.byteLength(padded);
|
|
417
|
+
}
|
|
418
|
+
catch (error) {
|
|
419
|
+
lockRecordLine('rewrite-failed', reading.path, { fd: reading.fd, operation: reading.holder.operation }, error);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
function storageLockLine(line) {
|
|
423
|
+
try {
|
|
424
|
+
console.warn(`${line.replace(/[\r\n]/g, ' ')} at=${Date.now()}`);
|
|
425
|
+
}
|
|
426
|
+
catch { /* an instrument */ }
|
|
427
|
+
}
|
|
428
|
+
/** Failure-only ownership evidence, on the same channel as the existing hold and reclaim warnings. */
|
|
429
|
+
function lockRecordLine(stage, path, fields, error) {
|
|
430
|
+
try {
|
|
431
|
+
const cause = error;
|
|
432
|
+
console.warn('[world-storage-lock]', JSON.stringify({
|
|
433
|
+
at: Date.now(), stage, path, pid: process.pid, hostname: ownHostname(), ...fields,
|
|
434
|
+
...(typeof cause?.name === 'string' ? { error: cause.name } : {}),
|
|
435
|
+
...(typeof cause?.code === 'string' ? { code: cause.code } : {}),
|
|
436
|
+
}));
|
|
437
|
+
}
|
|
438
|
+
catch { /* an instrument must not change the lock's result */ }
|
|
439
|
+
}
|
|
440
|
+
/** Name a bookkeeping phase within an existing acquisition; never acquire another lock. */
|
|
441
|
+
export function withStorageLockOperation(path, operation, fn) {
|
|
442
|
+
let reading;
|
|
443
|
+
for (let i = lockReadings.length - 1; i >= 0; i -= 1) {
|
|
444
|
+
if (lockReadings[i].path === path) {
|
|
445
|
+
reading = lockReadings[i];
|
|
446
|
+
break;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
if (!reading)
|
|
450
|
+
return fn();
|
|
451
|
+
const previous = reading.holder.operation;
|
|
452
|
+
reading.holder.operation = operation;
|
|
453
|
+
recordLockReading(reading);
|
|
454
|
+
try {
|
|
455
|
+
return fn();
|
|
456
|
+
}
|
|
457
|
+
finally {
|
|
458
|
+
reading.holder.operation = previous;
|
|
459
|
+
recordLockReading(reading);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
/** A path that is not there: no such entry, or a part of it that is not a directory. */
|
|
463
|
+
const absent = (error) => { const code = error?.code; return code === 'ENOENT' || code === 'ENOTDIR'; };
|
|
464
|
+
function readLockHolder(lockPath, diagnostic) {
|
|
465
|
+
let text;
|
|
466
|
+
try {
|
|
467
|
+
text = readFileSync(lockPath, 'utf8');
|
|
468
|
+
}
|
|
469
|
+
catch (error) {
|
|
470
|
+
if (diagnostic)
|
|
471
|
+
lockRecordLine('read-failed', lockPath, { during: diagnostic }, error);
|
|
472
|
+
return null;
|
|
473
|
+
}
|
|
474
|
+
return lockHolderOf(lockPath, text, diagnostic);
|
|
475
|
+
}
|
|
476
|
+
/** A record's text as its holder (readLockHolder's parse); `null` text is no file, read as readLockHolder reads one. */
|
|
477
|
+
function lockHolderOf(lockPath, text, diagnostic) {
|
|
478
|
+
if (text === null) {
|
|
479
|
+
if (diagnostic)
|
|
480
|
+
lockRecordLine('read-failed', lockPath, { during: diagnostic, code: 'ENOENT' });
|
|
481
|
+
return null;
|
|
482
|
+
}
|
|
483
|
+
try {
|
|
484
|
+
return JSON.parse(text);
|
|
485
|
+
}
|
|
486
|
+
catch (error) {
|
|
487
|
+
if (diagnostic)
|
|
488
|
+
lockRecordLine('parse-failed', lockPath, { during: diagnostic, bytes: Buffer.byteLength(text) }, error);
|
|
40
489
|
return null; // missing, empty (mid-write), or malformed
|
|
41
490
|
}
|
|
42
491
|
}
|
|
492
|
+
/** The same ownership account reaches a timeout's client as the storage trace. */
|
|
493
|
+
function lockHolderMessage(holder) {
|
|
494
|
+
if (!holder)
|
|
495
|
+
return ' (holder record unreadable)';
|
|
496
|
+
const since = typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
|
|
497
|
+
const wait = holder.waitingFor;
|
|
498
|
+
return ` (held by pid ${holder.pid} on ${holder.hostname}${Number.isFinite(since) ? ` for ${Date.now() - since}ms` : ''}; token=${holder.token ?? '?'} operation=${holder.operation ?? 'unknown'} caller=${holder.caller ?? 'unknown'}${wait ? ` waitingFor=${wait.path} waitingOperation=${wait.operation} waitingSince=${wait.since}` : ''})`;
|
|
499
|
+
}
|
|
43
500
|
function pidAlive(pid) {
|
|
44
501
|
try {
|
|
45
502
|
process.kill(pid, 0);
|
|
@@ -50,16 +507,34 @@ function pidAlive(pid) {
|
|
|
50
507
|
return error.code === 'EPERM';
|
|
51
508
|
}
|
|
52
509
|
}
|
|
53
|
-
/** A lock is stale when its holder is provably gone (same host + dead pid)
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
* mtime) empty lock is left alone, avoiding a race with the live writer. */
|
|
510
|
+
/** A lock is stale when its holder is provably gone (same host + dead pid), when its file is empty past
|
|
511
|
+
* EMPTY_RECORD_GRACE_MS (a taker that ended before recording itself; a live holder never holds an empty record), or
|
|
512
|
+
* when it has outlived LOCK_STALE_MS. The age falls back to the lockfile's own mtime when the `at` record is
|
|
513
|
+
* unreadable, while a freshly-created (recent mtime) empty lock is left alone, avoiding a race with the live writer. */
|
|
58
514
|
function lockIsStale(lockPath, retainLiveOwner = false) {
|
|
59
515
|
const holder = readLockHolder(lockPath);
|
|
60
|
-
if (holder && holder.hostname ===
|
|
516
|
+
if (holder && holder.hostname === ownHostname() && Number.isInteger(holder.pid) && !pidAlive(holder.pid)) {
|
|
517
|
+
return true;
|
|
518
|
+
}
|
|
519
|
+
// A lock recorded under THIS process that this process does not hold now is no holder's: a release that could not
|
|
520
|
+
// remove its file, or an earlier process numbered as this one (a tab numbers its processes the same at each boot,
|
|
521
|
+
// and a World's state outlives them). Waiting on it was waiting on ourselves: Twenty 46's Redis twin, pid 10013,
|
|
522
|
+
// timed out on `actions.jsonl.lock` "held by pid 10013" and exited.
|
|
523
|
+
if (holder && holder.hostname === ownHostname() && holder.pid === process.pid && ownLocks.get(lockPath) !== holder.token) {
|
|
61
524
|
return true;
|
|
62
525
|
}
|
|
526
|
+
// An empty record past its grace names no holder, alive or not (EMPTY_RECORD_GRACE_MS): asked before the live
|
|
527
|
+
// owner's rule, which keeps a lock whose record says who holds it, and an empty one says no one.
|
|
528
|
+
if (!holder) {
|
|
529
|
+
try {
|
|
530
|
+
const file = statSync(lockPath);
|
|
531
|
+
if (file.size === 0 && Date.now() - file.mtimeMs > EMPTY_RECORD_GRACE_MS)
|
|
532
|
+
return true;
|
|
533
|
+
}
|
|
534
|
+
catch {
|
|
535
|
+
return false;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
63
538
|
if (retainLiveOwner)
|
|
64
539
|
return false;
|
|
65
540
|
const recordedAt = holder && typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
|
|
@@ -84,15 +559,138 @@ function sleepSync(ms) {
|
|
|
84
559
|
* file lock — so the existing twins and the conformance suite are unaffected.
|
|
85
560
|
*/
|
|
86
561
|
export class FsWorldStore {
|
|
562
|
+
// ONE CALL OF THE FILE SYSTEM FOR ONE QUESTION. Each of these asked whether the path exists and then asked again
|
|
563
|
+
// for what it wanted; `stat` asked three times. A process on a host pays microseconds for that. A twin in a tab
|
|
564
|
+
// pays a kernel exchange for each, and a tree's stamp is a handful of these on every operation: Twenty's Redis twin
|
|
565
|
+
// spent 54 s of 136 in `existsSync` and `statSync` under `treeStamp`, and its server's requests waited on it.
|
|
87
566
|
read(path) {
|
|
88
|
-
|
|
567
|
+
try {
|
|
568
|
+
return readFileSync(path, 'utf8');
|
|
569
|
+
}
|
|
570
|
+
catch (error) {
|
|
571
|
+
if (absent(error))
|
|
572
|
+
return null;
|
|
573
|
+
throw error;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* A small file's text, kept while its own facts stand. THE GUARANTEE: the key is the path's own lstat (device,
|
|
578
|
+
* inode, size, modification and change times); this process's own writes through the store drop what is kept at
|
|
579
|
+
* once; another process's write is seen when any of those facts moves. That is the guarantee a tree's stamp
|
|
580
|
+
* already rests on for its logs (their size and modification time), with the inode and change time added: a
|
|
581
|
+
* replaced file (writeAtomic) is another inode, a file written in place has another time. What it cannot see is a
|
|
582
|
+
* rewrite to the same size on the same or a reused inode within one tick of the file system's clock, with no check
|
|
583
|
+
* between; the logs' stamp has the same limit. A link is read through, and not kept.
|
|
584
|
+
* A text read after a write that came between the lstat and the read is kept under the OLDER key, so the next
|
|
585
|
+
* check finds the facts moved and reads again: what is kept is never newer than its key, and never served past it.
|
|
586
|
+
*/
|
|
587
|
+
kept = new Map();
|
|
588
|
+
readKept(path) {
|
|
589
|
+
// A missing file is asked for without an error being made of it: a World with no parent has no branch.json, and
|
|
590
|
+
// every stamp asks.
|
|
591
|
+
let own;
|
|
592
|
+
try {
|
|
593
|
+
own = lstatSync(path, { throwIfNoEntry: false });
|
|
594
|
+
}
|
|
595
|
+
catch (error) {
|
|
596
|
+
if (!absent(error))
|
|
597
|
+
throw error;
|
|
598
|
+
}
|
|
599
|
+
if (!own) {
|
|
600
|
+
this.kept.delete(path);
|
|
601
|
+
return null;
|
|
602
|
+
}
|
|
603
|
+
if (own.isSymbolicLink())
|
|
604
|
+
return this.read(path);
|
|
605
|
+
const key = `${own.dev}:${own.ino}:${own.size}:${own.mtimeMs}:${own.ctimeMs}`;
|
|
606
|
+
const held = this.kept.get(path);
|
|
607
|
+
if (held && held.key === key)
|
|
608
|
+
return held.text;
|
|
609
|
+
const text = this.read(path);
|
|
610
|
+
if (text === null) {
|
|
611
|
+
this.kept.delete(path);
|
|
89
612
|
return null;
|
|
90
|
-
|
|
613
|
+
}
|
|
614
|
+
this.kept.set(path, { key, text });
|
|
615
|
+
return text;
|
|
616
|
+
}
|
|
617
|
+
/**
|
|
618
|
+
* A TREE'S GENERATION: one file beside a tree's three files, a byte longer after every write of any of them, so a
|
|
619
|
+
* reader that has the tree's stamp asks ONE question to know it still stands (log.ts treeFacts), where it asked
|
|
620
|
+
* three, two of them of files a World with no parent does not have.
|
|
621
|
+
*
|
|
622
|
+
* WHAT MAKES IT SOUND. A writer appends a byte BEFORE its write and another AFTER it; a reader takes the
|
|
623
|
+
* generation's facts BEFORE it reads the files' own. The file only grows, so its size never repeats on one inode,
|
|
624
|
+
* and a replaced or removed file is another inode or none. So for any write W and any reader that kept a stamp S
|
|
625
|
+
* under generation G: if W's first byte is after G was read, the kept G is behind and S is not served; if it is
|
|
626
|
+
* before, then either W itself was before S's own reads (S holds it), or W's second byte is after G was read, and
|
|
627
|
+
* again G is behind. An append is one call the system orders among all writers: each byte is one append of its
|
|
628
|
+
* own, whole or not at all, on a POSIX filesystem and in a tab's one kernel (browser-substrate
|
|
629
|
+
* packages/node/src/kernel-virtual-filesystem.ts appendFileSync, an O_APPEND open and writeAll, which sends a write
|
|
630
|
+
* of at most DATA_BYTES as one SYS_write request the kernel serves whole; or, by a file step, its table's own
|
|
631
|
+
* O_APPEND open and write of the one byte, filesystem.ts appendFileBytes). The proof rests on that; log.ts
|
|
632
|
+
* appendToBranchLog's (`generationAfterOwnWrite`: one write is exactly two bytes) as well.
|
|
633
|
+
*
|
|
634
|
+
* WHERE THAT STOPS. (1) A writer that dies between its write and its second byte, with a reader's whole check
|
|
635
|
+
* between its first byte and its write: that reader keeps the older stamp until the next write of the tree. (2) A
|
|
636
|
+
* writer of these files that does not come through a store of this version (an older package, a copy made over a
|
|
637
|
+
* live directory by another program): the generation does not move. A directory made or removed whole has no
|
|
638
|
+
* generation or another's, which serves nothing kept. (3) A stamp that names files outside its own directory (a
|
|
639
|
+
* World with a parent: its view and its ancestors' logs) is never kept under a generation (log.ts).
|
|
640
|
+
*/
|
|
641
|
+
generation(dirPath) {
|
|
642
|
+
let own;
|
|
643
|
+
try {
|
|
644
|
+
own = lstatSync(join(dirPath, TREE_GENERATION), { throwIfNoEntry: false });
|
|
645
|
+
}
|
|
646
|
+
catch (error) {
|
|
647
|
+
if (!absent(error))
|
|
648
|
+
throw error;
|
|
649
|
+
}
|
|
650
|
+
if (!own || !own.isFile())
|
|
651
|
+
return null;
|
|
652
|
+
return `${own.dev}:${own.ino}:${own.size}:${own.mtimeMs}:${own.ctimeMs}`;
|
|
653
|
+
}
|
|
654
|
+
/** One write is `moved` twice, a byte each: the same file, those two bytes longer. */
|
|
655
|
+
generationAfterOwnWrite(dirPath, before) {
|
|
656
|
+
const now = this.generation(dirPath);
|
|
657
|
+
if (now === null)
|
|
658
|
+
return null;
|
|
659
|
+
const [dev, ino, size] = before.split(':'), [nowDev, nowIno, nowSize] = now.split(':');
|
|
660
|
+
return nowDev === dev && nowIno === ino && Number(nowSize) === Number(size) + 2 ? now : null;
|
|
661
|
+
}
|
|
662
|
+
/** One more byte on the generation beside `path`, where `path` is one of a tree's files. A directory that is not
|
|
663
|
+
* there has no reader to tell: the write that makes it is followed by the byte that makes its generation. */
|
|
664
|
+
moved(path) {
|
|
665
|
+
if (!TREE_FILE_NAMES.has(basename(path)))
|
|
666
|
+
return;
|
|
667
|
+
try {
|
|
668
|
+
appendFileSync(join(dirname(path), TREE_GENERATION), '.');
|
|
669
|
+
}
|
|
670
|
+
catch (error) {
|
|
671
|
+
if (!absent(error))
|
|
672
|
+
throw error;
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
/** A write of `path`, with the generation beside it moved before and after where it is one of a tree's files. */
|
|
676
|
+
writing(path, write) {
|
|
677
|
+
this.moved(path);
|
|
678
|
+
try {
|
|
679
|
+
write();
|
|
680
|
+
}
|
|
681
|
+
finally {
|
|
682
|
+
this.moved(path);
|
|
683
|
+
}
|
|
91
684
|
}
|
|
92
685
|
readLines(path) {
|
|
93
|
-
|
|
94
|
-
return
|
|
95
|
-
|
|
686
|
+
try {
|
|
687
|
+
return readFileSync(path, 'utf8').split('\n');
|
|
688
|
+
}
|
|
689
|
+
catch (error) {
|
|
690
|
+
if (absent(error))
|
|
691
|
+
return [];
|
|
692
|
+
throw error;
|
|
693
|
+
}
|
|
96
694
|
}
|
|
97
695
|
readRange(path, start) {
|
|
98
696
|
let fd;
|
|
@@ -124,62 +722,94 @@ export class FsWorldStore {
|
|
|
124
722
|
}
|
|
125
723
|
append(path, data) {
|
|
126
724
|
refuseReadOnlyWrite('the World'); // a read-only request writes nothing, whatever path it takes to the store
|
|
725
|
+
this.kept.delete(path);
|
|
726
|
+
// A tree's file in a tab: the generation's byte, the append and the byte again as one request of the kernel
|
|
727
|
+
// (fileSteps), by its own opens, writes and closes in this order. Not where every append is to be fsynced.
|
|
728
|
+
if (TREE_FILE_NAMES.has(basename(path)) && process.env.VOLTER_DURABLE !== '1'
|
|
729
|
+
&& fileSteps()?.appendBetween(join(dirname(path), TREE_GENERATION), '.', path, data))
|
|
730
|
+
return;
|
|
127
731
|
// Historical appendDurable: a completed append syscall survives a process crash;
|
|
128
732
|
// fsync additionally survives kernel-panic/power-loss when VOLTER_DURABLE=1.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
733
|
+
this.writing(path, () => {
|
|
734
|
+
const fd = openSync(path, 'a');
|
|
735
|
+
try {
|
|
736
|
+
appendFileSync(fd, data);
|
|
737
|
+
if (process.env.VOLTER_DURABLE === '1')
|
|
738
|
+
fsyncSync(fd);
|
|
739
|
+
}
|
|
740
|
+
finally {
|
|
741
|
+
closeSync(fd);
|
|
742
|
+
}
|
|
743
|
+
});
|
|
138
744
|
}
|
|
139
745
|
write(path, data, options = {}) {
|
|
140
746
|
refuseReadOnlyWrite('the World'); // a read-only request writes nothing, whatever path it takes to the store
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
747
|
+
this.kept.delete(path);
|
|
748
|
+
this.writing(path, () => {
|
|
749
|
+
if (!options.secret) {
|
|
750
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
751
|
+
writeFileSync(path, data);
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
755
|
+
writeFileSync(path, data, { mode: 0o600 });
|
|
756
|
+
chmodSync(path, 0o600);
|
|
757
|
+
});
|
|
149
758
|
}
|
|
150
759
|
writeAtomic(path, data, options = {}) {
|
|
151
760
|
refuseReadOnlyWrite('the World'); // a read-only request writes nothing, whatever path it takes to the store
|
|
761
|
+
this.kept.delete(path);
|
|
152
762
|
mkdirSync(dirname(path), { recursive: true });
|
|
153
763
|
const tmp = `${path}.${process.pid}.${Date.now()}.${fsAtomicSequence++}.tmp`;
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
764
|
+
this.writing(path, () => {
|
|
765
|
+
try {
|
|
766
|
+
writeFileSync(tmp, data, options.secret ? { mode: 0o600 } : undefined);
|
|
767
|
+
renameSync(tmp, path);
|
|
768
|
+
if (options.secret)
|
|
769
|
+
chmodSync(path, 0o600);
|
|
770
|
+
}
|
|
771
|
+
finally {
|
|
772
|
+
rmSync(tmp, { force: true });
|
|
773
|
+
}
|
|
774
|
+
});
|
|
163
775
|
}
|
|
164
776
|
exists(path) {
|
|
165
777
|
return existsSync(path);
|
|
166
778
|
}
|
|
167
779
|
remove(path) {
|
|
168
780
|
refuseReadOnlyWrite('the World'); // a read-only request writes nothing, whatever path it takes to the store
|
|
169
|
-
|
|
781
|
+
for (const held of this.kept.keys())
|
|
782
|
+
if (held === path || held.startsWith(`${path}/`))
|
|
783
|
+
this.kept.delete(held);
|
|
784
|
+
this.writing(path, () => rmSync(path, { recursive: true, force: true }));
|
|
170
785
|
}
|
|
171
786
|
mkdir(dirPath) {
|
|
172
787
|
mkdirSync(dirPath, { recursive: true });
|
|
173
788
|
}
|
|
174
789
|
list(dirPath) {
|
|
175
|
-
|
|
176
|
-
return
|
|
177
|
-
|
|
790
|
+
try {
|
|
791
|
+
return readdirSync(dirPath);
|
|
792
|
+
}
|
|
793
|
+
catch (error) {
|
|
794
|
+
// No such directory is an empty one, as before; a path that IS there and is not a directory still fails.
|
|
795
|
+
const code = error.code;
|
|
796
|
+
if (code === 'ENOENT' || (code === 'ENOTDIR' && !existsSync(dirPath)))
|
|
797
|
+
return [];
|
|
798
|
+
throw error;
|
|
799
|
+
}
|
|
178
800
|
}
|
|
801
|
+
/** The real path of each path already resolved: a state directory's does not move under a running process, and
|
|
802
|
+
* every history lock asked for it again. A path that does not exist yet is not kept. */
|
|
803
|
+
canonical = new Map();
|
|
179
804
|
canonicalPath(path) {
|
|
180
805
|
const absolute = resolve(path);
|
|
806
|
+
const known = this.canonical.get(absolute);
|
|
807
|
+
if (known !== undefined)
|
|
808
|
+
return known;
|
|
181
809
|
try {
|
|
182
|
-
|
|
810
|
+
const real = realpathSync(absolute);
|
|
811
|
+
this.canonical.set(absolute, real);
|
|
812
|
+
return real;
|
|
183
813
|
}
|
|
184
814
|
catch (error) {
|
|
185
815
|
if (error.code !== 'ENOENT' || dirname(absolute) === absolute)
|
|
@@ -188,98 +818,248 @@ export class FsWorldStore {
|
|
|
188
818
|
}
|
|
189
819
|
}
|
|
190
820
|
stat(path) {
|
|
191
|
-
|
|
821
|
+
// The path itself first: where it is no link, that is the whole answer. A link is followed by a second call,
|
|
822
|
+
// and one that leads nowhere is absent, as it was.
|
|
823
|
+
// Asked without an error being made of a missing path (a tree's parent log is absent in a World with no parent).
|
|
824
|
+
let own;
|
|
825
|
+
try {
|
|
826
|
+
own = lstatSync(path, { throwIfNoEntry: false });
|
|
827
|
+
}
|
|
828
|
+
catch (error) {
|
|
829
|
+
if (!absent(error))
|
|
830
|
+
throw error;
|
|
831
|
+
}
|
|
832
|
+
if (!own)
|
|
192
833
|
return null;
|
|
193
|
-
|
|
194
|
-
|
|
834
|
+
if (!own.isSymbolicLink())
|
|
835
|
+
return { size: own.size, mtimeMs: own.mtimeMs, isDirectory: own.isDirectory() };
|
|
836
|
+
try {
|
|
837
|
+
const s = statSync(path);
|
|
838
|
+
return { size: s.size, mtimeMs: s.mtimeMs, isDirectory: s.isDirectory(), isSymbolicLink: true };
|
|
839
|
+
}
|
|
840
|
+
catch (error) {
|
|
841
|
+
if (absent(error) || error.code === 'ELOOP')
|
|
842
|
+
return null;
|
|
843
|
+
throw error;
|
|
844
|
+
}
|
|
195
845
|
}
|
|
196
846
|
/** Run `fn` holding an exclusive cross-process file lock. The lockfile records
|
|
197
|
-
*
|
|
847
|
+
* identity, operation, caller and any nested wait; a contender that finds a stale lock (dead pid or age >
|
|
198
848
|
* LOCK_STALE_MS) reclaims it by atomically renaming it aside — so a crashed holder
|
|
199
849
|
* can't wedge the world forever. Reclaim is race-safe: only the process that wins
|
|
200
850
|
* the rename clears the stale inode, and the exclusive `wx` create still decides the
|
|
201
851
|
* winner. (Moved verbatim from storage.ts `withFileLock`.) */
|
|
202
852
|
withLock(lockPath, fn, options = {}) {
|
|
203
|
-
mkdirSync(dirname(lockPath), { recursive: true });
|
|
204
853
|
const started = Date.now();
|
|
854
|
+
const scope = storageLockScope(options);
|
|
205
855
|
let fd = null;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
856
|
+
// A kernel's file step takes the lock with its record in one request and keeps no descriptor (fileSteps).
|
|
857
|
+
const steps = fileSteps();
|
|
858
|
+
// The token is the take's own: its release removes the file only while it still holds this token, so a file a
|
|
859
|
+
// stale-lock sweep reclaimed and another process re-created is never removed by the first holder.
|
|
860
|
+
const token = randomUUID();
|
|
861
|
+
const holderNow = () => ({ pid: process.pid, hostname: ownHostname(), at: new Date().toISOString(), token, operation: scope.operation, caller: scope.caller });
|
|
862
|
+
let record;
|
|
863
|
+
let taken = false;
|
|
864
|
+
let unrecordedTakes = 0;
|
|
865
|
+
let recordBytes;
|
|
866
|
+
let made = false;
|
|
867
|
+
let reported = false;
|
|
868
|
+
let attempts = 0;
|
|
869
|
+
let sleeps = 0;
|
|
870
|
+
let sleepMs = 0;
|
|
871
|
+
let waitedOn = null;
|
|
872
|
+
let nestedWait;
|
|
873
|
+
const finishNestedWait = () => {
|
|
874
|
+
if (!nestedWait)
|
|
875
|
+
return;
|
|
876
|
+
const waitedMs = Date.now() - nestedWait.since;
|
|
877
|
+
for (const reading of nestedWait.holders) {
|
|
878
|
+
const key = `${nestedWait.path}\0${scope.operation}`;
|
|
879
|
+
const previous = reading.nestedWaits.get(key);
|
|
880
|
+
reading.nestedWaits.set(key, { path: nestedWait.path, operation: scope.operation, waitedMs: (previous?.waitedMs ?? 0) + waitedMs });
|
|
881
|
+
delete reading.holder.waitingFor;
|
|
882
|
+
recordLockReading(reading);
|
|
209
883
|
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
884
|
+
nestedWait = undefined;
|
|
885
|
+
};
|
|
886
|
+
const reportWait = () => {
|
|
887
|
+
if (reported || !nestedWait || Date.now() - started < 2_000)
|
|
888
|
+
return;
|
|
889
|
+
reported = true;
|
|
890
|
+
const holder = readLockHolder(nestedWait.path);
|
|
891
|
+
const since = holder && typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
|
|
892
|
+
storageLockLine(`[world] storage lock still held path=${nestedWait.path} holder=${holder?.pid ?? '?'}@${holder?.hostname ?? '?'} token=${holder?.token ?? '?'} operation=${holder?.operation ?? holder?.caller ?? 'unknown:0'} holderMs=${Number.isFinite(since) ? Date.now() - since : '?'} waiter=${process.pid} waitingOperation=${scope.operation} caller=${scope.caller}`);
|
|
893
|
+
};
|
|
894
|
+
const slept = (path = lockPath) => {
|
|
895
|
+
if (sleeps === 0)
|
|
896
|
+
waitedOn = readLockHolder(path);
|
|
897
|
+
if (nestedWait?.path !== path) {
|
|
898
|
+
finishNestedWait();
|
|
899
|
+
nestedWait = { path, since: Date.now(), holders: [...lockReadings] };
|
|
900
|
+
for (const reading of nestedWait.holders) {
|
|
901
|
+
reading.holder.waitingFor = { path, operation: scope.operation, since: nestedWait.since };
|
|
902
|
+
recordLockReading(reading);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
reportWait();
|
|
906
|
+
const sleeping = Date.now();
|
|
907
|
+
sleeps += 1;
|
|
908
|
+
sleepSync(25);
|
|
909
|
+
sleepMs += Date.now() - sleeping;
|
|
910
|
+
};
|
|
911
|
+
try {
|
|
912
|
+
while (!taken) {
|
|
913
|
+
// This runs in the synchronous loop: a blocked holder cannot service a timer.
|
|
914
|
+
reportWait();
|
|
915
|
+
try {
|
|
916
|
+
attempts += 1;
|
|
917
|
+
// The holder is recorded WITH the take (createRecorded): no other process sees this lock without its record.
|
|
918
|
+
record = holderNow();
|
|
919
|
+
const taking = createRecorded(lockPath, `${JSON.stringify(record)}\n`, token, scope.operation);
|
|
920
|
+
fd = taking.fd;
|
|
921
|
+
recordBytes = taking.bytes;
|
|
922
|
+
taken = true;
|
|
923
|
+
}
|
|
924
|
+
catch (error) {
|
|
925
|
+
const code = error.code;
|
|
926
|
+
// The lock's directory is made when it is found missing, once, not before every take: it is there for
|
|
927
|
+
// every take after a state directory's first.
|
|
928
|
+
if (code === 'ENOENT' && !made) {
|
|
929
|
+
made = true;
|
|
930
|
+
mkdirSync(dirname(lockPath), { recursive: true });
|
|
931
|
+
continue;
|
|
221
932
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
933
|
+
// A two-call take whose path did not hold its record after it (another reclaimed it between): taken again.
|
|
934
|
+
if (code === RECLAIMED_BEFORE_RECORD)
|
|
935
|
+
continue;
|
|
936
|
+
// A take whose record could not be written: made again, a few times, before the failure is the caller's.
|
|
937
|
+
if (code === RECORD_NOT_WRITTEN) {
|
|
938
|
+
unrecordedTakes += 1;
|
|
939
|
+
if (unrecordedTakes >= RECORD_ATTEMPTS)
|
|
940
|
+
throw Object.assign(new Error(`World storage lock ${lockPath}: its holder's record could not be written`), { code: 'EIO', cause: error });
|
|
941
|
+
sleepSync(25 * unrecordedTakes);
|
|
228
942
|
continue;
|
|
229
943
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
944
|
+
if (code !== 'EEXIST')
|
|
945
|
+
throw error;
|
|
946
|
+
if (lockIsStale(lockPath, options.retainLiveOwner)) {
|
|
947
|
+
// Serialize reclaimers, then re-read: a contender's earlier stale observation
|
|
948
|
+
// must never rename another contender's newly acquired, live lock. The reclaim record is made as a lock
|
|
949
|
+
// is, with its record and a token of its own, and released by that token.
|
|
950
|
+
const recoveryPath = `${lockPath}.recovery`;
|
|
951
|
+
const recoveryToken = randomUUID();
|
|
952
|
+
let recovery;
|
|
953
|
+
try {
|
|
954
|
+
recovery = createRecorded(recoveryPath, `${JSON.stringify({ pid: process.pid, hostname: ownHostname(), at: new Date().toISOString(), token: recoveryToken })}\n`, recoveryToken, 'lock recovery');
|
|
955
|
+
}
|
|
956
|
+
catch (error) {
|
|
957
|
+
const recoveryCode = error.code;
|
|
958
|
+
if (recoveryCode === RECLAIMED_BEFORE_RECORD || recoveryCode === RECORD_NOT_WRITTEN)
|
|
959
|
+
continue;
|
|
960
|
+
if (recoveryCode !== 'EEXIST')
|
|
961
|
+
throw error;
|
|
962
|
+
// A reclaim record no reclaimer is behind is reclaimed as an empty lock file is (recoveryIsStale).
|
|
963
|
+
if (recoveryIsStale(recoveryPath)) {
|
|
964
|
+
const aside = `${recoveryPath}.stale.${process.pid}.${randomUUID()}`;
|
|
965
|
+
try {
|
|
966
|
+
renameSync(recoveryPath, aside);
|
|
967
|
+
unlinkSync(aside);
|
|
968
|
+
lockRecordLine('recovery-reclaimed', recoveryPath, {});
|
|
969
|
+
}
|
|
970
|
+
catch (cause) {
|
|
971
|
+
if (cause.code !== 'ENOENT')
|
|
972
|
+
throw cause;
|
|
973
|
+
}
|
|
974
|
+
continue;
|
|
239
975
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
976
|
+
if (Date.now() - started > 10_000)
|
|
977
|
+
throw new Error(`Lock recovery is active or interrupted: ${recoveryPath}${lockHolderMessage(readLockHolder(recoveryPath, 'timeout'))}. Inspect its owner before removing the recovery record.`);
|
|
978
|
+
slept(recoveryPath);
|
|
979
|
+
continue;
|
|
980
|
+
}
|
|
981
|
+
try {
|
|
982
|
+
if (lockIsStale(lockPath, options.retainLiveOwner)) {
|
|
983
|
+
const holder = readLockHolder(lockPath);
|
|
984
|
+
const salvage = `${lockPath}.stale.${process.pid}.${Date.now()}`;
|
|
985
|
+
try {
|
|
986
|
+
renameSync(lockPath, salvage);
|
|
987
|
+
console.warn(`[world] reclaiming stale storage lock ${lockPath} (held by pid ${holder?.pid ?? '?'} on ${holder?.hostname ?? '?'}) (reclaimer pid ${process.pid} on ${ownHostname()})`);
|
|
988
|
+
unlinkSync(salvage);
|
|
989
|
+
}
|
|
990
|
+
catch (error) {
|
|
991
|
+
if (error.code !== 'ENOENT')
|
|
992
|
+
throw error;
|
|
993
|
+
}
|
|
243
994
|
}
|
|
244
995
|
}
|
|
996
|
+
finally {
|
|
997
|
+
releaseRecorded(recoveryPath, recoveryToken, recovery.fd);
|
|
998
|
+
}
|
|
999
|
+
continue;
|
|
245
1000
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
1001
|
+
if (Date.now() - started > 10_000) {
|
|
1002
|
+
const holder = readLockHolder(lockPath, 'timeout');
|
|
1003
|
+
throw new Error(`Timed out waiting for world storage lock: ${lockPath}${lockHolderMessage(holder)}`);
|
|
249
1004
|
}
|
|
250
|
-
|
|
251
|
-
}
|
|
252
|
-
if (Date.now() - started > 10_000) {
|
|
253
|
-
const holder = readLockHolder(lockPath);
|
|
254
|
-
const since = holder && typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
|
|
255
|
-
throw new Error(`Timed out waiting for world storage lock: ${lockPath}${holder ? ` (held by pid ${holder.pid} on ${holder.hostname}${Number.isFinite(since) ? ` for ${Date.now() - since}ms` : ''})` : ''}`);
|
|
1005
|
+
slept();
|
|
256
1006
|
}
|
|
257
|
-
sleepSync(25);
|
|
258
1007
|
}
|
|
259
1008
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
1009
|
+
finally {
|
|
1010
|
+
finishNestedWait();
|
|
1011
|
+
if (sleeps > 0) {
|
|
1012
|
+
const waitedMs = Date.now() - started;
|
|
1013
|
+
const first = waitedOn;
|
|
1014
|
+
storageLockLine(`[world] storage lock wait path=${lockPath} outcome=${taken ? 'acquired' : 'failed'} waiter=${process.pid}@${ownHostname()} operation=${scope.operation} attempts=${attempts} sleeps=${sleeps} requestedSleepMs=${sleeps * 25} sleepMs=${sleepMs} outsideSleepMs=${waitedMs - sleepMs} waitedMs=${waitedMs} firstHolder=${first?.pid ?? '?'}@${first?.hostname ?? '?'} token=${first?.token ?? '?'}`);
|
|
1015
|
+
}
|
|
264
1016
|
}
|
|
265
|
-
|
|
1017
|
+
// The holder's record was written with the take, in the loop above: a take whose record could not be written is
|
|
1018
|
+
// never held (EMPTY_RECORD_GRACE_MS).
|
|
1019
|
+
ownLocks.set(lockPath, token);
|
|
266
1020
|
const acquired = Date.now();
|
|
1021
|
+
const reading = { path: lockPath, fd, holder: record, nestedWaits: new Map(), ...(recordBytes !== undefined ? { recordBytes } : {}) };
|
|
1022
|
+
lockReadings.push(reading);
|
|
267
1023
|
try {
|
|
268
1024
|
return fn();
|
|
269
1025
|
}
|
|
270
1026
|
finally {
|
|
271
1027
|
// Every World on the machine waits on some of these locks; a long hold stalls them all.
|
|
272
1028
|
const heldMs = Date.now() - acquired;
|
|
1029
|
+
if (heldMs > 2_000)
|
|
1030
|
+
storageLockLine(`[world] releasing storage lock path=${lockPath} operation=${scope.operation} heldMs=${heldMs} nestedWaits=${JSON.stringify([...reading.nestedWaits.values()])}`);
|
|
273
1031
|
if (heldMs > 2_000)
|
|
274
1032
|
console.warn(`[world] held storage lock ${lockPath} for ${heldMs}ms (pid ${process.pid})\n${new Error().stack?.split('\n').slice(2, 8).join('\n') ?? ''}`);
|
|
275
|
-
|
|
276
|
-
|
|
1033
|
+
lockReadings.pop();
|
|
1034
|
+
ownLocks.delete(lockPath);
|
|
1035
|
+
if (fd !== null) {
|
|
1036
|
+
try {
|
|
1037
|
+
closeSync(fd);
|
|
1038
|
+
}
|
|
1039
|
+
catch (error) {
|
|
1040
|
+
lockRecordLine('close-failed', lockPath, { fd }, error);
|
|
1041
|
+
throw error;
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
// By the token the take wrote, not by inode: a filesystem may answer a descriptor's stat and the path's with
|
|
1045
|
+
// different identities (a tab's, where a committed path is the store's entry), and a lock released by inode
|
|
1046
|
+
// was then never removed, left recorded as this live process's (above). A kernel's file step reads the record,
|
|
1047
|
+
// compares its token and unlinks in one request; its answer is what the read below would have found.
|
|
277
1048
|
try {
|
|
278
|
-
const
|
|
279
|
-
if (
|
|
280
|
-
|
|
1049
|
+
const released = fd === null ? steps?.changeHolding(lockPath, tokenNeedle(token), undefined) : undefined;
|
|
1050
|
+
if (!released?.changed) {
|
|
1051
|
+
const observed = released === undefined ? readLockHolder(lockPath, 'release') : lockHolderOf(lockPath, released.observed, 'release');
|
|
1052
|
+
if (observed?.token === token)
|
|
1053
|
+
unlinkSync(lockPath);
|
|
1054
|
+
else if (observed)
|
|
1055
|
+
lockRecordLine('release-token-mismatch', lockPath, {
|
|
1056
|
+
token, recordedToken: observed.token, recordedPid: observed.pid, recordedHostname: observed.hostname,
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
catch (error) {
|
|
1061
|
+
lockRecordLine('release-failed', lockPath, { fd }, error);
|
|
281
1062
|
}
|
|
282
|
-
catch { /* already reclaimed by a stale-lock sweep */ }
|
|
283
1063
|
}
|
|
284
1064
|
}
|
|
285
1065
|
}
|