@bjornpagen/bumbledb-log 0.19.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/braids.ts +8 -6
- package/src/bytes.ts +5 -194
- package/src/chain.ts +67 -111
- package/src/codec.ts +150 -233
- package/src/descriptor.ts +67 -612
- package/src/errors.ts +91 -62
- package/src/index.ts +25 -21
- package/src/keys.ts +49 -62
- package/src/manifest.ts +81 -129
- package/src/replica.ts +82 -128
- package/src/store-s3.ts +1 -1
- package/src/store.ts +245 -229
- package/src/tenants.ts +19 -41
- package/src/vector.ts +5 -75
- package/src/writer.ts +275 -168
- package/src/value.ts +0 -336
package/src/store-s3.ts
CHANGED
package/src/store.ts
CHANGED
|
@@ -3,17 +3,19 @@
|
|
|
3
3
|
* infrastructure failures on the ErrStore channel. `fsStore` is tier-1,
|
|
4
4
|
* not a dev double — deployment case 5's production backend.
|
|
5
5
|
*
|
|
6
|
-
* The one on-disk protocol, shared with the Rust driver
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* a
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
6
|
+
* The one on-disk protocol, shared with the Rust driver and pinned by
|
|
7
|
+
* the `lease/` corpus goldens: create-only publishes an exclusive
|
|
8
|
+
* synced temp (under the reserved `~tmp` namespace) with link(2), where
|
|
9
|
+
* EEXIST is the honest exists; the etag is the blake3 of the content,
|
|
10
|
+
* lowercase hex, computed on every read and never stored. The mutation
|
|
11
|
+
* lock is a fenced CAS lease: a versioned `LEASE/1` body whose identity
|
|
12
|
+
* is a monotonic token file `{root}/~lease/{key}/{n}`, with `~head`
|
|
13
|
+
* naming the current token. A contender mints the next token iff the
|
|
14
|
+
* current lease's own bytes are expired — expiry is the only break.
|
|
15
|
+
* Release rewrites the held token with an already-expired body so the
|
|
16
|
+
* next acquirer does not wait us out. `created` and `swapped` resolve
|
|
17
|
+
* only after fsync of the object file and its parent directory,
|
|
18
|
+
* including newly created ancestors. Stale temps are swept at open.
|
|
17
19
|
*/
|
|
18
20
|
|
|
19
21
|
import * as fs from "node:fs/promises"
|
|
@@ -21,21 +23,10 @@ import * as path from "node:path"
|
|
|
21
23
|
import { internalBlake3 } from "@bjornpagen/bumbledb"
|
|
22
24
|
import * as errors from "@superbuilders/errors"
|
|
23
25
|
import { regex } from "arkregex"
|
|
24
|
-
import { bytesEqual, toHex } from "#bytes.ts"
|
|
26
|
+
import { bytesEqual, toHex, U64_MAX } from "#bytes.ts"
|
|
25
27
|
import { wrapStore } from "#errors.ts"
|
|
26
28
|
import type { StoreKey } from "#keys.ts"
|
|
27
|
-
|
|
28
|
-
function reservedTemp(basename: string, pid: number, seq: number): string {
|
|
29
|
-
return `.${basename}.tmp.${pid}.${seq}`
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
function reservedLease(basename: string, token: bigint): string {
|
|
33
|
-
return `.${basename}.lease.${token}`
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function isReservedName(name: string): boolean {
|
|
37
|
-
return name.startsWith(".")
|
|
38
|
-
}
|
|
29
|
+
import { LEASE_NAMESPACE, reservedLease, reservedTemp, TEMP_NAMESPACE } from "#keys.ts"
|
|
39
30
|
|
|
40
31
|
declare const etagBrand: unique symbol
|
|
41
32
|
type Etag = string & { readonly [etagBrand]: typeof etagBrand }
|
|
@@ -58,8 +49,6 @@ type Create =
|
|
|
58
49
|
|
|
59
50
|
type Swap = { readonly tag: "swapped"; readonly etag: Etag } | { readonly tag: "moved" } | { readonly tag: "ambiguous" }
|
|
60
51
|
|
|
61
|
-
type Liveness = { readonly tag: "alive" } | { readonly tag: "dead" } | { readonly tag: "unknown" }
|
|
62
|
-
|
|
63
52
|
interface Lease {
|
|
64
53
|
readonly holder: bigint
|
|
65
54
|
readonly token: bigint
|
|
@@ -89,26 +78,28 @@ interface ObjectStore {
|
|
|
89
78
|
delete(key: StoreKey): Promise<void>
|
|
90
79
|
}
|
|
91
80
|
|
|
81
|
+
/** A held fenced lease: identity is the token file `{dir}/{token}`. */
|
|
92
82
|
interface FsLease {
|
|
83
|
+
readonly root: string
|
|
93
84
|
readonly dir: string
|
|
94
|
-
readonly name: string
|
|
95
85
|
readonly holder: bigint
|
|
96
86
|
readonly token: bigint
|
|
97
87
|
readonly path: string
|
|
98
88
|
}
|
|
99
89
|
|
|
100
|
-
interface LeaseFile {
|
|
101
|
-
readonly path: string
|
|
102
|
-
readonly token: bigint
|
|
103
|
-
readonly lease: Lease | null
|
|
104
|
-
readonly readable: boolean
|
|
105
|
-
}
|
|
106
|
-
|
|
107
90
|
/** Ceiling of the jittered wait between probes of an unexpired lease. */
|
|
108
91
|
const LOCK_RETRY_MS = 10
|
|
109
92
|
|
|
110
|
-
/**
|
|
111
|
-
const
|
|
93
|
+
/** How long a mutation lease stays current, in milliseconds. */
|
|
94
|
+
const MUTATION_TTL_MS = 5_000
|
|
95
|
+
|
|
96
|
+
/** A live temp under `~tmp` exists only for write-then-link. Anything
|
|
97
|
+
* older than this is crash litter the open sweep deletes. */
|
|
98
|
+
const TEMP_STALE_MS = 30_000
|
|
99
|
+
|
|
100
|
+
/** `~head` is not a StoreKey. It names the current token so a
|
|
101
|
+
* successor reads `{dir}/{n}` without listing. */
|
|
102
|
+
const HEAD = "~head"
|
|
112
103
|
|
|
113
104
|
function ourHolder(): bigint {
|
|
114
105
|
return BigInt(process.pid)
|
|
@@ -122,53 +113,60 @@ function codeOf(error: Error): string | undefined {
|
|
|
122
113
|
return (error as NodeJS.ErrnoException).code
|
|
123
114
|
}
|
|
124
115
|
|
|
116
|
+
/** The lease body's magic first line. Version 1 of the one lock protocol. */
|
|
117
|
+
const LEASE_MAGIC = "LEASE/1"
|
|
118
|
+
|
|
125
119
|
function encodeLease(lease: Lease): Uint8Array {
|
|
126
|
-
return new TextEncoder().encode(`${lease.holder}\n${lease.token}\n${lease.expires}\n`)
|
|
120
|
+
return new TextEncoder().encode(`${LEASE_MAGIC}\n${lease.holder}\n${lease.token}\n${lease.expires}\n`)
|
|
127
121
|
}
|
|
128
122
|
|
|
129
|
-
const
|
|
130
|
-
const UNSIGNED_DECIMAL = regex("^\\d+$")
|
|
123
|
+
const U64_DECIMAL = regex("^\\d+$")
|
|
131
124
|
|
|
132
|
-
function
|
|
133
|
-
|
|
134
|
-
if (lines.length !== 3) {
|
|
125
|
+
function u64Line(line: string): bigint | null {
|
|
126
|
+
if (!U64_DECIMAL.test(line)) {
|
|
135
127
|
return null
|
|
136
128
|
}
|
|
137
|
-
const
|
|
138
|
-
if (
|
|
129
|
+
const value = BigInt(line)
|
|
130
|
+
if (value > U64_MAX) {
|
|
139
131
|
return null
|
|
140
132
|
}
|
|
141
|
-
|
|
142
|
-
return null
|
|
143
|
-
}
|
|
144
|
-
return { holder: BigInt(holderLine), token: BigInt(tokenLine), expires: BigInt(expiresLine) }
|
|
133
|
+
return value
|
|
145
134
|
}
|
|
146
135
|
|
|
147
|
-
|
|
148
|
-
|
|
136
|
+
/** Lines as Rust `str::lines`: split on `\n`, one trailing terminator
|
|
137
|
+
* unyielded, a trailing `\r` stripped per line. */
|
|
138
|
+
function leaseLines(raw: string): string[] {
|
|
139
|
+
const parts = raw.split("\n")
|
|
140
|
+
if (parts.length > 0 && parts[parts.length - 1] === "") {
|
|
141
|
+
parts.pop()
|
|
142
|
+
}
|
|
143
|
+
return parts.map((line) => (line.endsWith("\r") ? line.slice(0, -1) : line))
|
|
149
144
|
}
|
|
150
145
|
|
|
151
|
-
/**
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
146
|
+
/** The `LEASE/1` body: magic line, then holder, token, expires as
|
|
147
|
+
* decimal u64 lines, and nothing after. Anything else is not a lease
|
|
148
|
+
* and never breakable. */
|
|
149
|
+
function parseLease(raw: string): Lease | null {
|
|
150
|
+
const lines = leaseLines(raw)
|
|
151
|
+
if (lines.length !== 4) {
|
|
152
|
+
return null
|
|
155
153
|
}
|
|
156
|
-
|
|
157
|
-
|
|
154
|
+
const [magic, holderLine, tokenLine, expiresLine] = lines
|
|
155
|
+
if (magic !== LEASE_MAGIC || holderLine === undefined || tokenLine === undefined || expiresLine === undefined) {
|
|
156
|
+
return null
|
|
157
|
+
}
|
|
158
|
+
const holder = u64Line(holderLine)
|
|
159
|
+
const token = u64Line(tokenLine)
|
|
160
|
+
const expires = u64Line(expiresLine)
|
|
161
|
+
if (holder === null || token === null || expires === null) {
|
|
162
|
+
return null
|
|
158
163
|
}
|
|
159
|
-
return {
|
|
164
|
+
return { holder, token, expires }
|
|
160
165
|
}
|
|
161
166
|
|
|
162
|
-
/**
|
|
163
|
-
function
|
|
164
|
-
|
|
165
|
-
case "dead":
|
|
166
|
-
return true
|
|
167
|
-
case "alive":
|
|
168
|
-
return false
|
|
169
|
-
case "unknown":
|
|
170
|
-
return false
|
|
171
|
-
}
|
|
167
|
+
/** Expiry of the lease's own bytes: the only break. */
|
|
168
|
+
function leaseExpired(lease: Lease, nowMs: number): boolean {
|
|
169
|
+
return lease.expires <= BigInt(nowMs)
|
|
172
170
|
}
|
|
173
171
|
|
|
174
172
|
function isUnproved(error: Error): boolean {
|
|
@@ -218,12 +216,12 @@ async function ensureParent(target: string, root: string): Promise<void> {
|
|
|
218
216
|
|
|
219
217
|
let tempSeq = 0
|
|
220
218
|
|
|
221
|
-
/** Write `bytes` to a fresh `wx` temp
|
|
219
|
+
/** Write `bytes` to a fresh `wx` temp under `{root}/~tmp` and fsync it.
|
|
222
220
|
* The caller publishes the synced temp atomically. */
|
|
223
|
-
async function syncedTemp(
|
|
224
|
-
await ensureParent(target, root)
|
|
221
|
+
async function syncedTemp(root: string, bytes: Uint8Array): Promise<string> {
|
|
225
222
|
tempSeq += 1
|
|
226
|
-
const temp = path.join(
|
|
223
|
+
const temp = path.join(root, ...reservedTemp(process.pid, tempSeq).split("/"))
|
|
224
|
+
await fs.mkdir(path.dirname(temp), { recursive: true })
|
|
227
225
|
const handle = await fs.open(temp, "wx")
|
|
228
226
|
const written = await errors.try(
|
|
229
227
|
(async function writeAll() {
|
|
@@ -234,7 +232,7 @@ async function syncedTemp(target: string, bytes: Uint8Array, root: string): Prom
|
|
|
234
232
|
await handle.close()
|
|
235
233
|
if (written.error) {
|
|
236
234
|
await fs.rm(temp, { force: true })
|
|
237
|
-
throw errors.wrap(written.error, `write ${
|
|
235
|
+
throw errors.wrap(written.error, `write temp for ${root}`)
|
|
238
236
|
}
|
|
239
237
|
return temp
|
|
240
238
|
}
|
|
@@ -254,107 +252,134 @@ async function publishLink(temp: string, dest: string): Promise<"linked" | "occu
|
|
|
254
252
|
throw linked.error
|
|
255
253
|
}
|
|
256
254
|
|
|
257
|
-
|
|
258
|
-
|
|
255
|
+
/** The lease directory for `key`: `{root}/~lease/{key}`. */
|
|
256
|
+
function leaseDir(root: string, key: string): string {
|
|
257
|
+
return path.join(root, LEASE_NAMESPACE, ...key.split("/"))
|
|
258
|
+
}
|
|
259
259
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
260
|
+
function tokenPath(dir: string, token: bigint): string {
|
|
261
|
+
return path.join(dir, String(token))
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function headPath(dir: string): string {
|
|
265
|
+
return path.join(dir, HEAD)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async function readHead(dir: string): Promise<bigint | null> {
|
|
269
|
+
const read = await errors.try(fs.readFile(headPath(dir), "utf8"))
|
|
270
|
+
if (read.error) {
|
|
271
|
+
if (codeOf(read.error) === "ENOENT") {
|
|
272
|
+
return null
|
|
265
273
|
}
|
|
266
|
-
throw
|
|
274
|
+
throw read.error
|
|
267
275
|
}
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
if (match === null || match[1] !== name || match[2] === undefined) {
|
|
272
|
-
continue
|
|
273
|
-
}
|
|
274
|
-
const filePath = path.join(dir, entry)
|
|
275
|
-
const read = await errors.try(fs.readFile(filePath, "utf8"))
|
|
276
|
-
if (read.error) {
|
|
277
|
-
if (codeOf(read.error) === "ENOENT") {
|
|
278
|
-
continue
|
|
279
|
-
}
|
|
280
|
-
found.push({ path: filePath, token: BigInt(match[2]), lease: null, readable: false })
|
|
281
|
-
continue
|
|
282
|
-
}
|
|
283
|
-
found.push({
|
|
284
|
-
path: filePath,
|
|
285
|
-
token: BigInt(match[2]),
|
|
286
|
-
lease: parseLease(read.data),
|
|
287
|
-
readable: true
|
|
288
|
-
})
|
|
276
|
+
const token = u64Line(read.data.trim())
|
|
277
|
+
if (token === null || token < 1n) {
|
|
278
|
+
return null
|
|
289
279
|
}
|
|
290
|
-
return
|
|
280
|
+
return token
|
|
291
281
|
}
|
|
292
282
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const
|
|
296
|
-
|
|
297
|
-
|
|
283
|
+
async function writeHead(root: string, dir: string, token: bigint): Promise<void> {
|
|
284
|
+
const dest = headPath(dir)
|
|
285
|
+
const temp = await syncedTemp(root, new TextEncoder().encode(String(token)))
|
|
286
|
+
const replaced = await errors.try(fs.rename(temp, dest))
|
|
287
|
+
if (replaced.error) {
|
|
288
|
+
await fs.rm(temp, { force: true })
|
|
289
|
+
throw replaced.error
|
|
298
290
|
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
291
|
+
await fsyncDir(dir)
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Removes `{dir}/{1..=current-1}` after `~head` names `current`. */
|
|
295
|
+
async function forgetPredecessors(dir: string, current: bigint): Promise<void> {
|
|
296
|
+
for (let token = current - 1n; token >= 1n; token -= 1n) {
|
|
297
|
+
await fs.rm(tokenPath(dir, token), { force: true })
|
|
302
298
|
}
|
|
303
|
-
|
|
304
|
-
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
interface CurrentLease {
|
|
302
|
+
readonly token: bigint
|
|
303
|
+
readonly lease: Lease
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function probeFrom(dir: string, start: bigint): Promise<CurrentLease | null> {
|
|
307
|
+
return (async function probe(): Promise<CurrentLease | null> {
|
|
308
|
+
let best: CurrentLease | null = null
|
|
309
|
+
let token = start
|
|
310
|
+
while (token <= U64_MAX) {
|
|
311
|
+
const read = await errors.try(fs.readFile(tokenPath(dir, token), "utf8"))
|
|
312
|
+
if (read.error) {
|
|
313
|
+
if (codeOf(read.error) === "ENOENT") {
|
|
314
|
+
break
|
|
315
|
+
}
|
|
316
|
+
token += 1n
|
|
317
|
+
continue
|
|
318
|
+
}
|
|
319
|
+
const lease = parseLease(read.data)
|
|
320
|
+
if (lease !== null) {
|
|
321
|
+
best = { token, lease }
|
|
322
|
+
}
|
|
323
|
+
token += 1n
|
|
324
|
+
}
|
|
325
|
+
return best
|
|
326
|
+
})()
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/** The current lease is `{dir}/{n}` for the token `~head` names, or
|
|
330
|
+
* the highest `{dir}/{n}` at or after that hint. A mint past a stale
|
|
331
|
+
* head is still visible: the probe opens `n`, `n+1`, … until a gap. */
|
|
332
|
+
async function currentLease(dir: string): Promise<CurrentLease | null> {
|
|
333
|
+
const start = (await readHead(dir)) ?? 1n
|
|
334
|
+
const found = await probeFrom(dir, start)
|
|
335
|
+
if (found === null && start > 1n) {
|
|
336
|
+
return await probeFrom(dir, 1n)
|
|
305
337
|
}
|
|
306
|
-
|
|
338
|
+
return found
|
|
307
339
|
}
|
|
308
340
|
|
|
309
|
-
async function
|
|
310
|
-
|
|
311
|
-
|
|
341
|
+
async function tryMint(
|
|
342
|
+
root: string,
|
|
343
|
+
dir: string,
|
|
344
|
+
key: string,
|
|
345
|
+
token: bigint,
|
|
346
|
+
holder: bigint,
|
|
347
|
+
ttlMs: number
|
|
348
|
+
): Promise<boolean> {
|
|
349
|
+
await fs.mkdir(dir, { recursive: true })
|
|
350
|
+
const body = encodeLease({ holder, token, expires: BigInt(Date.now() + ttlMs) })
|
|
351
|
+
const dest = path.join(root, ...reservedLease(key, token).split("/"))
|
|
352
|
+
const temp = await syncedTemp(root, body)
|
|
353
|
+
const published = await errors.try(publishLink(temp, dest))
|
|
354
|
+
await fs.rm(temp, { force: true })
|
|
355
|
+
if (published.error) {
|
|
356
|
+
throw published.error
|
|
357
|
+
}
|
|
358
|
+
if (published.data === "occupied") {
|
|
312
359
|
return false
|
|
313
360
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
361
|
+
await fsyncDir(dir)
|
|
362
|
+
const headed = await errors.try(writeHead(root, dir, token))
|
|
363
|
+
if (headed.error === undefined) {
|
|
364
|
+
await forgetPredecessors(dir, token)
|
|
317
365
|
}
|
|
318
|
-
return
|
|
366
|
+
return true
|
|
319
367
|
}
|
|
320
368
|
|
|
369
|
+
/** Acquire the fenced lease on `{root}/~lease/{key}`: mint the next
|
|
370
|
+
* monotonic token iff the current lease's own bytes are expired.
|
|
371
|
+
* `wait` sleeps out a live holder; `refuse` throws. */
|
|
321
372
|
async function acquireFsLease(
|
|
322
|
-
|
|
323
|
-
|
|
373
|
+
root: string,
|
|
374
|
+
key: string,
|
|
324
375
|
ttlMs: number,
|
|
325
376
|
contend: "wait" | "refuse" = "wait"
|
|
326
377
|
): Promise<FsLease> {
|
|
327
|
-
|
|
378
|
+
const dir = leaseDir(root, key)
|
|
379
|
+
const holder = ourHolder()
|
|
328
380
|
for (;;) {
|
|
329
|
-
const
|
|
330
|
-
|
|
331
|
-
let highest = 0n
|
|
332
|
-
let blocked = false
|
|
333
|
-
const dead: LeaseFile[] = []
|
|
334
|
-
for (const incumbent of incumbents) {
|
|
335
|
-
if (incumbent.token > highest) {
|
|
336
|
-
highest = incumbent.token
|
|
337
|
-
}
|
|
338
|
-
const liveness = livenessOf(incumbent.lease, incumbent.readable, now)
|
|
339
|
-
switch (liveness.tag) {
|
|
340
|
-
case "dead":
|
|
341
|
-
dead.push(incumbent)
|
|
342
|
-
break
|
|
343
|
-
case "alive": {
|
|
344
|
-
const holder = incumbent.lease === null ? null : incumbent.lease.holder
|
|
345
|
-
if (holder === ourHolder()) {
|
|
346
|
-
blocked = true
|
|
347
|
-
} else {
|
|
348
|
-
blocked = true
|
|
349
|
-
}
|
|
350
|
-
break
|
|
351
|
-
}
|
|
352
|
-
case "unknown":
|
|
353
|
-
blocked = true
|
|
354
|
-
break
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
if (blocked) {
|
|
381
|
+
const current = await currentLease(dir)
|
|
382
|
+
if (current !== null && !leaseExpired(current.lease, Date.now())) {
|
|
358
383
|
if (contend === "refuse") {
|
|
359
384
|
throw errors.new("replica directory has an owner")
|
|
360
385
|
}
|
|
@@ -363,64 +388,43 @@ async function acquireFsLease(
|
|
|
363
388
|
})
|
|
364
389
|
continue
|
|
365
390
|
}
|
|
366
|
-
const
|
|
367
|
-
const
|
|
368
|
-
|
|
369
|
-
const body = encodeLease({
|
|
370
|
-
holder,
|
|
371
|
-
token,
|
|
372
|
-
expires: BigInt(now + ttlMs)
|
|
373
|
-
})
|
|
374
|
-
const wrote = await errors.try(syncedTemp(dest, body, dir))
|
|
375
|
-
if (wrote.error) {
|
|
376
|
-
if (isVanished(wrote.error) || isUnproved(wrote.error)) {
|
|
377
|
-
continue
|
|
378
|
-
}
|
|
379
|
-
throw wrote.error
|
|
380
|
-
}
|
|
381
|
-
const published = await errors.try(publishLink(wrote.data, dest))
|
|
382
|
-
await fs.rm(wrote.data, { force: true })
|
|
383
|
-
if (published.error) {
|
|
384
|
-
if (isVanished(published.error) || isUnproved(published.error)) {
|
|
385
|
-
continue
|
|
386
|
-
}
|
|
387
|
-
throw published.error
|
|
388
|
-
}
|
|
389
|
-
if (published.data === "occupied") {
|
|
391
|
+
const token = current === null ? 1n : current.token + 1n
|
|
392
|
+
const minted = await tryMint(root, dir, key, token, holder, ttlMs)
|
|
393
|
+
if (!minted) {
|
|
390
394
|
continue
|
|
391
395
|
}
|
|
392
|
-
|
|
393
|
-
const held: FsLease = { dir, name, holder, token, path: dest }
|
|
394
|
-
for (const stale of dead) {
|
|
395
|
-
if (stale.lease !== null) {
|
|
396
|
-
await casDrop(stale.path, stale.lease)
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
return held
|
|
396
|
+
return { root, dir, holder, token, path: tokenPath(dir, token) }
|
|
400
397
|
}
|
|
401
398
|
}
|
|
402
399
|
|
|
400
|
+
/** True iff this token is still the max — a stale holder lost the CAS
|
|
401
|
+
* and must not publish. */
|
|
402
|
+
async function stillCurrent(held: FsLease): Promise<boolean> {
|
|
403
|
+
const current = await currentLease(held.dir)
|
|
404
|
+
return current !== null && current.token === held.token
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** Release by rewriting the held token with an already-expired body so
|
|
408
|
+
* the next acquirer does not wait us out. */
|
|
403
409
|
async function releaseFsLease(held: FsLease): Promise<void> {
|
|
404
|
-
const
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
}
|
|
408
|
-
const lease = parseLease(read.data)
|
|
409
|
-
if (lease === null) {
|
|
410
|
+
const body = encodeLease({ holder: held.holder, token: held.token, expires: 0n })
|
|
411
|
+
const wrote = await errors.try(syncedTemp(held.root, body))
|
|
412
|
+
if (wrote.error) {
|
|
410
413
|
return
|
|
411
414
|
}
|
|
412
|
-
|
|
415
|
+
const replaced = await errors.try(fs.rename(wrote.data, held.path))
|
|
416
|
+
if (replaced.error) {
|
|
417
|
+
await fs.rm(wrote.data, { force: true })
|
|
413
418
|
return
|
|
414
419
|
}
|
|
415
|
-
await casDrop(held.path, lease)
|
|
416
420
|
const synced = await errors.try(fsyncDir(held.dir))
|
|
417
421
|
if (synced.error) {
|
|
418
422
|
return
|
|
419
423
|
}
|
|
420
424
|
}
|
|
421
425
|
|
|
422
|
-
async function
|
|
423
|
-
const listed = await errors.try(fs.readdir(
|
|
426
|
+
async function sweepStaleTemps(dir: string): Promise<void> {
|
|
427
|
+
const listed = await errors.try(fs.readdir(dir, { withFileTypes: true }))
|
|
424
428
|
if (listed.error) {
|
|
425
429
|
if (codeOf(listed.error) === "ENOENT") {
|
|
426
430
|
return
|
|
@@ -429,37 +433,31 @@ async function sweepReserved(root: string): Promise<void> {
|
|
|
429
433
|
}
|
|
430
434
|
const now = Date.now()
|
|
431
435
|
for (const entry of listed.data) {
|
|
432
|
-
|
|
433
|
-
if (entry.isDirectory()) {
|
|
434
|
-
if (isReservedName(entry.name)) {
|
|
435
|
-
continue
|
|
436
|
-
}
|
|
437
|
-
await sweepReserved(full)
|
|
436
|
+
if (!entry.isFile()) {
|
|
438
437
|
continue
|
|
439
438
|
}
|
|
440
|
-
|
|
439
|
+
const full = path.join(dir, entry.name)
|
|
440
|
+
const st = await errors.try(fs.stat(full))
|
|
441
|
+
if (st.error) {
|
|
441
442
|
continue
|
|
442
443
|
}
|
|
443
|
-
if (
|
|
444
|
+
if (now - st.data.mtimeMs > TEMP_STALE_MS) {
|
|
444
445
|
await fs.rm(full, { force: true })
|
|
445
|
-
continue
|
|
446
|
-
}
|
|
447
|
-
const match = LEASE_NAME_RE.exec(entry.name)
|
|
448
|
-
if (match === null) {
|
|
449
|
-
continue
|
|
450
|
-
}
|
|
451
|
-
const read = await errors.try(fs.readFile(full, "utf8"))
|
|
452
|
-
if (read.error) {
|
|
453
|
-
continue
|
|
454
|
-
}
|
|
455
|
-
const lease = parseLease(read.data)
|
|
456
|
-
const liveness = livenessOf(lease, true, now)
|
|
457
|
-
if (breakable(liveness) && lease !== null) {
|
|
458
|
-
await casDrop(full, lease)
|
|
459
446
|
}
|
|
460
447
|
}
|
|
461
448
|
}
|
|
462
449
|
|
|
450
|
+
/** Sweep crash litter: stale temps under `~tmp`, and superseded tokens
|
|
451
|
+
* directly under `~lease` once `~head` names the current one. */
|
|
452
|
+
async function sweepReserved(root: string): Promise<void> {
|
|
453
|
+
await sweepStaleTemps(path.join(root, TEMP_NAMESPACE))
|
|
454
|
+
const dir = path.join(root, LEASE_NAMESPACE)
|
|
455
|
+
const current = await currentLease(dir)
|
|
456
|
+
if (current !== null) {
|
|
457
|
+
await forgetPredecessors(dir, current.token)
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
463
461
|
async function resolveAmbiguousCreate(store: ObjectStore, key: StoreKey, attempted: Uint8Array): Promise<CreateProbe> {
|
|
464
462
|
const fetched = await store.get(key)
|
|
465
463
|
if (fetched === null) {
|
|
@@ -489,7 +487,11 @@ function cloneFetched(fetched: Fetched): Fetched {
|
|
|
489
487
|
/** The five verbs over one local directory. One machine is load-bearing. */
|
|
490
488
|
function fsStore(root: string): ObjectStore {
|
|
491
489
|
const rootPath = path.resolve(root)
|
|
492
|
-
const swept =
|
|
490
|
+
const swept = (async function open() {
|
|
491
|
+
await sweepReserved(rootPath)
|
|
492
|
+
await fs.mkdir(path.join(rootPath, TEMP_NAMESPACE), { recursive: true })
|
|
493
|
+
await fs.mkdir(path.join(rootPath, LEASE_NAMESPACE), { recursive: true })
|
|
494
|
+
})()
|
|
493
495
|
|
|
494
496
|
function objectPath(key: StoreKey): string {
|
|
495
497
|
return path.join(rootPath, ...key.split("/"))
|
|
@@ -507,8 +509,8 @@ function fsStore(root: string): ObjectStore {
|
|
|
507
509
|
return { bytes, etag: contentEtag(bytes) }
|
|
508
510
|
}
|
|
509
511
|
|
|
510
|
-
async function withKeyLease<T>(
|
|
511
|
-
const held = await acquireFsLease(
|
|
512
|
+
async function withKeyLease<T>(key: StoreKey, body: (held: FsLease) => Promise<T>): Promise<T> {
|
|
513
|
+
const held = await acquireFsLease(rootPath, key, MUTATION_TTL_MS)
|
|
512
514
|
const ran = await errors.try(body(held))
|
|
513
515
|
await releaseFsLease(held)
|
|
514
516
|
if (ran.error) {
|
|
@@ -549,11 +551,12 @@ function fsStore(root: string): ObjectStore {
|
|
|
549
551
|
const target = objectPath(key)
|
|
550
552
|
const ran = await errors.try(
|
|
551
553
|
(async function createBody(): Promise<Create> {
|
|
552
|
-
return await withKeyLease(
|
|
553
|
-
if (!(await
|
|
554
|
+
return await withKeyLease(key, async function underLease(held): Promise<Create> {
|
|
555
|
+
if (!(await stillCurrent(held))) {
|
|
554
556
|
return { tag: "ambiguous" }
|
|
555
557
|
}
|
|
556
|
-
|
|
558
|
+
await ensureParent(target, rootPath)
|
|
559
|
+
const temp = await syncedTemp(rootPath, bytes)
|
|
557
560
|
const published = await errors.try(publishLink(temp, target))
|
|
558
561
|
await fs.rm(temp, { force: true })
|
|
559
562
|
if (published.error) {
|
|
@@ -597,15 +600,15 @@ function fsStore(root: string): ObjectStore {
|
|
|
597
600
|
const target = objectPath(key)
|
|
598
601
|
const ran = await errors.try(
|
|
599
602
|
(async function swapBody(): Promise<Swap> {
|
|
600
|
-
return await withKeyLease(
|
|
601
|
-
if (!(await
|
|
603
|
+
return await withKeyLease(key, async function underLease(held): Promise<Swap> {
|
|
604
|
+
if (!(await stillCurrent(held))) {
|
|
602
605
|
return { tag: "ambiguous" }
|
|
603
606
|
}
|
|
604
607
|
const current = await readFetched(target)
|
|
605
608
|
if (current === null || current.etag !== etag) {
|
|
606
609
|
return { tag: "moved" }
|
|
607
610
|
}
|
|
608
|
-
const temp = await syncedTemp(
|
|
611
|
+
const temp = await syncedTemp(rootPath, bytes)
|
|
609
612
|
const renamed = await errors.try(fs.rename(temp, target))
|
|
610
613
|
if (renamed.error) {
|
|
611
614
|
await fs.rm(temp, { force: true })
|
|
@@ -636,8 +639,8 @@ function fsStore(root: string): ObjectStore {
|
|
|
636
639
|
const target = objectPath(key)
|
|
637
640
|
const ran = await errors.try(
|
|
638
641
|
(async function deleteBody() {
|
|
639
|
-
await withKeyLease(
|
|
640
|
-
if (!(await
|
|
642
|
+
await withKeyLease(key, async function underLease(held) {
|
|
643
|
+
if (!(await stillCurrent(held))) {
|
|
641
644
|
return
|
|
642
645
|
}
|
|
643
646
|
await fs.rm(target, { force: true })
|
|
@@ -707,5 +710,18 @@ function memStore(): ObjectStore {
|
|
|
707
710
|
|
|
708
711
|
export type { S3Config, S3Credentials } from "#store-s3.ts"
|
|
709
712
|
export { s3Store } from "#store-s3.ts"
|
|
710
|
-
export type { Create,
|
|
711
|
-
export {
|
|
713
|
+
export type { Create, Etag, Fetched, FsLease, ObjectStore, Poll, Swap }
|
|
714
|
+
export {
|
|
715
|
+
acquireFsLease,
|
|
716
|
+
encodeLease,
|
|
717
|
+
etag,
|
|
718
|
+
fsStore,
|
|
719
|
+
MUTATION_TTL_MS,
|
|
720
|
+
memStore,
|
|
721
|
+
parseLease,
|
|
722
|
+
releaseFsLease,
|
|
723
|
+
resolveAmbiguousCreate,
|
|
724
|
+
resolveAmbiguousSwap,
|
|
725
|
+
sweepStaleTemps,
|
|
726
|
+
syncedTemp
|
|
727
|
+
}
|