@bjornpagen/bumbledb-log 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -18
- package/package.json +5 -3
- package/src/braids.ts +19 -7
- package/src/bytes.ts +79 -11
- package/src/chain.ts +179 -63
- package/src/codec.ts +141 -62
- package/src/descriptor.ts +392 -348
- package/src/errors.ts +140 -10
- package/src/index.ts +23 -8
- package/src/keys.ts +212 -15
- package/src/manifest.ts +171 -75
- package/src/replica.ts +606 -295
- package/src/store-s3.ts +377 -0
- package/src/store.ts +524 -105
- package/src/tenants.ts +322 -26
- package/src/value.ts +80 -29
- package/src/vector.ts +173 -0
- package/src/writer.ts +484 -120
package/src/store.ts
CHANGED
|
@@ -6,74 +6,181 @@
|
|
|
6
6
|
* The one on-disk protocol, shared with the Rust driver: create-only
|
|
7
7
|
* publishes an exclusive synced temp with link(2), where EEXIST is the
|
|
8
8
|
* honest exists; the etag is the blake3 of the content, lowercase hex,
|
|
9
|
-
* computed on every read and never stored
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* `
|
|
15
|
-
*
|
|
9
|
+
* computed on every read and never stored. The mutation lock is a
|
|
10
|
+
* fenced CAS lease `{holder, token, expires}` in the reserved
|
|
11
|
+
* namespace, acquired and broken only through the store's own CAS;
|
|
12
|
+
* a contender breaks a lease iff it is expired. Liveness is
|
|
13
|
+
* Alive | Dead | Unknown; Unknown never breaks a lease. `created` and
|
|
14
|
+
* `swapped` resolve only after fsync of the object file and its parent
|
|
15
|
+
* directory, including newly created ancestors. Temps and leftover
|
|
16
|
+
* expired leases are swept at open.
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
import * as fs from "node:fs/promises"
|
|
19
20
|
import * as path from "node:path"
|
|
20
21
|
import { internalBlake3 } from "@bjornpagen/bumbledb"
|
|
21
22
|
import * as errors from "@superbuilders/errors"
|
|
22
|
-
import {
|
|
23
|
+
import { regex } from "arkregex"
|
|
24
|
+
import { bytesEqual, toHex } from "#bytes.ts"
|
|
23
25
|
import { wrapStore } from "#errors.ts"
|
|
26
|
+
import type { StoreKey } from "#keys.ts"
|
|
27
|
+
|
|
28
|
+
function reservedTemp(basename: string, pid: number, seq: number): string {
|
|
29
|
+
return `.${basename}.tmp.${pid}.${seq}`
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function reservedLease(basename: string, token: bigint): string {
|
|
33
|
+
return `.${basename}.lease.${token}`
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function isReservedName(name: string): boolean {
|
|
37
|
+
return name.startsWith(".")
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
declare const etagBrand: unique symbol
|
|
41
|
+
type Etag = string & { readonly [etagBrand]: typeof etagBrand }
|
|
42
|
+
|
|
43
|
+
function etag(raw: string): Etag {
|
|
44
|
+
return raw as Etag
|
|
45
|
+
}
|
|
24
46
|
|
|
25
47
|
interface Fetched {
|
|
26
48
|
readonly bytes: Uint8Array
|
|
27
|
-
readonly etag:
|
|
49
|
+
readonly etag: Etag
|
|
28
50
|
}
|
|
29
51
|
|
|
30
52
|
type Poll = { readonly tag: "unchanged" } | { readonly tag: "changed"; readonly fetched: Fetched }
|
|
31
53
|
|
|
32
|
-
type Create =
|
|
54
|
+
type Create =
|
|
55
|
+
| { readonly tag: "created"; readonly etag: Etag }
|
|
56
|
+
| { readonly tag: "exists" }
|
|
57
|
+
| { readonly tag: "ambiguous" }
|
|
58
|
+
|
|
59
|
+
type Swap = { readonly tag: "swapped"; readonly etag: Etag } | { readonly tag: "moved" } | { readonly tag: "ambiguous" }
|
|
33
60
|
|
|
34
|
-
type
|
|
61
|
+
type Liveness = { readonly tag: "alive" } | { readonly tag: "dead" } | { readonly tag: "unknown" }
|
|
62
|
+
|
|
63
|
+
interface Lease {
|
|
64
|
+
readonly holder: bigint
|
|
65
|
+
readonly token: bigint
|
|
66
|
+
readonly expires: bigint
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
type CreateProbe =
|
|
70
|
+
| { readonly tag: "landed"; readonly etag: Etag }
|
|
71
|
+
| { readonly tag: "lost"; readonly fetched: Fetched }
|
|
72
|
+
| { readonly tag: "absent" }
|
|
73
|
+
|
|
74
|
+
type SwapProbe =
|
|
75
|
+
| { readonly tag: "landed"; readonly etag: Etag }
|
|
76
|
+
| { readonly tag: "lost"; readonly fetched: Fetched }
|
|
77
|
+
| { readonly tag: "absent" }
|
|
35
78
|
|
|
36
79
|
interface ObjectStore {
|
|
37
80
|
/** GET; null on 404. */
|
|
38
|
-
get(key:
|
|
81
|
+
get(key: StoreKey): Promise<Fetched | null>
|
|
39
82
|
/** GET with If-None-Match — the cheap manifest poll. */
|
|
40
|
-
getIfChanged(key:
|
|
83
|
+
getIfChanged(key: StoreKey, etag: Etag): Promise<Poll>
|
|
41
84
|
/** PUT with If-None-Match: * — the log-slot arbitration primitive. */
|
|
42
|
-
putCreate(key:
|
|
85
|
+
putCreate(key: StoreKey, bytes: Uint8Array): Promise<Create>
|
|
43
86
|
/** PUT with If-Match — the manifest CAS primitive. */
|
|
44
|
-
putSwap(key:
|
|
87
|
+
putSwap(key: StoreKey, bytes: Uint8Array, etag: Etag): Promise<Swap>
|
|
45
88
|
/** DELETE, unconditional — the gc verb's tool. */
|
|
46
|
-
delete(key:
|
|
89
|
+
delete(key: StoreKey): Promise<void>
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
interface FsLease {
|
|
93
|
+
readonly dir: string
|
|
94
|
+
readonly name: string
|
|
95
|
+
readonly holder: bigint
|
|
96
|
+
readonly token: bigint
|
|
97
|
+
readonly path: string
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
interface LeaseFile {
|
|
101
|
+
readonly path: string
|
|
102
|
+
readonly token: bigint
|
|
103
|
+
readonly lease: Lease | null
|
|
104
|
+
readonly readable: boolean
|
|
47
105
|
}
|
|
48
106
|
|
|
49
|
-
/**
|
|
50
|
-
const
|
|
107
|
+
/** Ceiling of the jittered wait between probes of an unexpired lease. */
|
|
108
|
+
const LEASE_RETRY_MS = 10
|
|
51
109
|
|
|
52
|
-
/**
|
|
53
|
-
const
|
|
110
|
+
/** Mutation-lease lifetime: long enough for one verb, short enough to expire after a crash. */
|
|
111
|
+
const MUTATION_LEASE_MS = 30_000
|
|
54
112
|
|
|
55
|
-
function
|
|
56
|
-
return
|
|
113
|
+
function ourHolder(): bigint {
|
|
114
|
+
return BigInt(process.pid)
|
|
57
115
|
}
|
|
58
116
|
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
throw errors.new(`store key is not a slash path: ${key}`)
|
|
62
|
-
}
|
|
63
|
-
for (const segment of key.split("/")) {
|
|
64
|
-
if (segment.length === 0 || segment === "." || segment === "..") {
|
|
65
|
-
throw errors.new(`store key segment is illegal: ${key}`)
|
|
66
|
-
}
|
|
67
|
-
if (segment.endsWith(LOCK_SUFFIX)) {
|
|
68
|
-
throw errors.new(`store key collides with the lockfile suffix: ${key}`)
|
|
69
|
-
}
|
|
70
|
-
}
|
|
117
|
+
function contentEtag(bytes: Uint8Array): Etag {
|
|
118
|
+
return etag(toHex(new Uint8Array(internalBlake3(bytes))))
|
|
71
119
|
}
|
|
72
120
|
|
|
73
121
|
function codeOf(error: Error): string | undefined {
|
|
74
122
|
return (error as NodeJS.ErrnoException).code
|
|
75
123
|
}
|
|
76
124
|
|
|
125
|
+
function encodeLease(lease: Lease): Uint8Array {
|
|
126
|
+
return new TextEncoder().encode(`${lease.holder}\n${lease.token}\n${lease.expires}\n`)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const SIGNED_DECIMAL = regex("^-?\\d+$")
|
|
130
|
+
const UNSIGNED_DECIMAL = regex("^\\d+$")
|
|
131
|
+
|
|
132
|
+
function parseLease(raw: string): Lease | null {
|
|
133
|
+
const lines = raw.trim().split("\n")
|
|
134
|
+
if (lines.length !== 3) {
|
|
135
|
+
return null
|
|
136
|
+
}
|
|
137
|
+
const [holderLine, tokenLine, expiresLine] = lines
|
|
138
|
+
if (holderLine === undefined || tokenLine === undefined || expiresLine === undefined) {
|
|
139
|
+
return null
|
|
140
|
+
}
|
|
141
|
+
if (!SIGNED_DECIMAL.test(holderLine) || !UNSIGNED_DECIMAL.test(tokenLine) || !SIGNED_DECIMAL.test(expiresLine)) {
|
|
142
|
+
return null
|
|
143
|
+
}
|
|
144
|
+
return { holder: BigInt(holderLine), token: BigInt(tokenLine), expires: BigInt(expiresLine) }
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function leaseExpired(lease: Lease, nowMs: number): boolean {
|
|
148
|
+
return lease.expires <= BigInt(nowMs)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** EPERM/unreadable/unparseable ⇒ Unknown. Expiry is a fact of the lease bytes. */
|
|
152
|
+
function livenessOf(lease: Lease | null, readable: boolean, nowMs: number): Liveness {
|
|
153
|
+
if (!readable || lease === null) {
|
|
154
|
+
return { tag: "unknown" }
|
|
155
|
+
}
|
|
156
|
+
if (leaseExpired(lease, nowMs)) {
|
|
157
|
+
return { tag: "dead" }
|
|
158
|
+
}
|
|
159
|
+
return { tag: "alive" }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** A lease is breakable only when liveness is Dead. Unknown never breaks. */
|
|
163
|
+
function breakable(liveness: Liveness): boolean {
|
|
164
|
+
switch (liveness.tag) {
|
|
165
|
+
case "dead":
|
|
166
|
+
return true
|
|
167
|
+
case "alive":
|
|
168
|
+
return false
|
|
169
|
+
case "unknown":
|
|
170
|
+
return false
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function isUnproved(error: Error): boolean {
|
|
175
|
+
const code = codeOf(error)
|
|
176
|
+
return code === "EIO" || code === "EINTR" || code === "ETIMEDOUT" || code === "EAGAIN" || code === "EBUSY"
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** A temp or parent that vanished mid-link is not Exists and not Created. */
|
|
180
|
+
function isVanished(error: Error): boolean {
|
|
181
|
+
return codeOf(error) === "ENOENT"
|
|
182
|
+
}
|
|
183
|
+
|
|
77
184
|
async function fsyncDir(dir: string): Promise<void> {
|
|
78
185
|
const handle = await fs.open(dir, "r")
|
|
79
186
|
const synced = await errors.try(handle.sync())
|
|
@@ -83,15 +190,40 @@ async function fsyncDir(dir: string): Promise<void> {
|
|
|
83
190
|
}
|
|
84
191
|
}
|
|
85
192
|
|
|
193
|
+
async function ensureParent(target: string, root: string): Promise<void> {
|
|
194
|
+
const dir = path.dirname(target)
|
|
195
|
+
const resolvedRoot = path.resolve(root)
|
|
196
|
+
const ancestors: string[] = []
|
|
197
|
+
let cursor = dir
|
|
198
|
+
while (cursor.startsWith(resolvedRoot) && cursor !== resolvedRoot) {
|
|
199
|
+
ancestors.push(cursor)
|
|
200
|
+
cursor = path.dirname(cursor)
|
|
201
|
+
}
|
|
202
|
+
const missing: string[] = []
|
|
203
|
+
for (const ancestor of ancestors.reverse()) {
|
|
204
|
+
const st = await errors.try(fs.stat(ancestor))
|
|
205
|
+
if (st.error) {
|
|
206
|
+
if (codeOf(st.error) === "ENOENT") {
|
|
207
|
+
missing.push(ancestor)
|
|
208
|
+
continue
|
|
209
|
+
}
|
|
210
|
+
throw st.error
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
await fs.mkdir(dir, { recursive: true })
|
|
214
|
+
for (const created of missing) {
|
|
215
|
+
await fsyncDir(path.dirname(created))
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
86
219
|
let tempSeq = 0
|
|
87
220
|
|
|
88
221
|
/** Write `bytes` to a fresh `wx` temp file beside `target` and fsync it.
|
|
89
222
|
* The caller publishes the synced temp atomically. */
|
|
90
|
-
async function syncedTemp(target: string, bytes: Uint8Array): Promise<string> {
|
|
91
|
-
|
|
92
|
-
await fs.mkdir(dir, { recursive: true })
|
|
223
|
+
async function syncedTemp(target: string, bytes: Uint8Array, root: string): Promise<string> {
|
|
224
|
+
await ensureParent(target, root)
|
|
93
225
|
tempSeq += 1
|
|
94
|
-
const temp = path.join(
|
|
226
|
+
const temp = path.join(path.dirname(target), reservedTemp(path.basename(target), process.pid, tempSeq))
|
|
95
227
|
const handle = await fs.open(temp, "wx")
|
|
96
228
|
const written = await errors.try(
|
|
97
229
|
(async function writeAll() {
|
|
@@ -122,59 +254,244 @@ async function publishLink(temp: string, dest: string): Promise<"linked" | "occu
|
|
|
122
254
|
throw linked.error
|
|
123
255
|
}
|
|
124
256
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
257
|
+
const LEASE_NAME_RE = regex("^\\.(.+)\\.lease\\.(\\d+)$")
|
|
258
|
+
const TEMP_NAME_RE = regex("^\\.(.+)\\.tmp\\.\\d+\\.\\d+$")
|
|
259
|
+
|
|
260
|
+
async function listLeaseFiles(dir: string, name: string): Promise<LeaseFile[]> {
|
|
261
|
+
const listed = await errors.try(fs.readdir(dir))
|
|
262
|
+
if (listed.error) {
|
|
263
|
+
if (codeOf(listed.error) === "ENOENT") {
|
|
264
|
+
return []
|
|
265
|
+
}
|
|
266
|
+
throw listed.error
|
|
131
267
|
}
|
|
132
|
-
|
|
268
|
+
const found: LeaseFile[] = []
|
|
269
|
+
for (const entry of listed.data) {
|
|
270
|
+
const match = LEASE_NAME_RE.exec(entry)
|
|
271
|
+
if (match === null || match[1] !== name || match[2] === undefined) {
|
|
272
|
+
continue
|
|
273
|
+
}
|
|
274
|
+
const filePath = path.join(dir, entry)
|
|
275
|
+
const read = await errors.try(fs.readFile(filePath, "utf8"))
|
|
276
|
+
if (read.error) {
|
|
277
|
+
if (codeOf(read.error) === "ENOENT") {
|
|
278
|
+
continue
|
|
279
|
+
}
|
|
280
|
+
found.push({ path: filePath, token: BigInt(match[2]), lease: null, readable: false })
|
|
281
|
+
continue
|
|
282
|
+
}
|
|
283
|
+
found.push({
|
|
284
|
+
path: filePath,
|
|
285
|
+
token: BigInt(match[2]),
|
|
286
|
+
lease: parseLease(read.data),
|
|
287
|
+
readable: true
|
|
288
|
+
})
|
|
289
|
+
}
|
|
290
|
+
return found
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** Drop a lease file only when its bytes still name this holder and token. */
|
|
294
|
+
async function casDrop(leasePath: string, expected: Lease): Promise<void> {
|
|
295
|
+
const read = await errors.try(fs.readFile(leasePath, "utf8"))
|
|
296
|
+
if (read.error) {
|
|
297
|
+
return
|
|
298
|
+
}
|
|
299
|
+
const lease = parseLease(read.data)
|
|
300
|
+
if (lease === null) {
|
|
301
|
+
return
|
|
302
|
+
}
|
|
303
|
+
if (lease.holder !== expected.holder || lease.token !== expected.token) {
|
|
304
|
+
return
|
|
305
|
+
}
|
|
306
|
+
await fs.rm(leasePath, { force: true })
|
|
133
307
|
}
|
|
134
308
|
|
|
135
|
-
async function
|
|
136
|
-
const
|
|
309
|
+
async function heldByUs(held: FsLease): Promise<boolean> {
|
|
310
|
+
const read = await errors.try(fs.readFile(held.path, "utf8"))
|
|
311
|
+
if (read.error) {
|
|
312
|
+
return false
|
|
313
|
+
}
|
|
314
|
+
const lease = parseLease(read.data)
|
|
315
|
+
if (lease === null) {
|
|
316
|
+
return false
|
|
317
|
+
}
|
|
318
|
+
return lease.holder === held.holder && lease.token === held.token
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async function acquireFsLease(
|
|
322
|
+
dir: string,
|
|
323
|
+
name: string,
|
|
324
|
+
ttlMs: number,
|
|
325
|
+
contend: "wait" | "refuse" = "wait"
|
|
326
|
+
): Promise<FsLease> {
|
|
327
|
+
await fs.mkdir(dir, { recursive: true })
|
|
137
328
|
for (;;) {
|
|
138
|
-
const
|
|
139
|
-
const
|
|
140
|
-
|
|
329
|
+
const now = Date.now()
|
|
330
|
+
const incumbents = await listLeaseFiles(dir, name)
|
|
331
|
+
let highest = 0n
|
|
332
|
+
let blocked = false
|
|
333
|
+
const dead: LeaseFile[] = []
|
|
334
|
+
for (const incumbent of incumbents) {
|
|
335
|
+
if (incumbent.token > highest) {
|
|
336
|
+
highest = incumbent.token
|
|
337
|
+
}
|
|
338
|
+
const liveness = livenessOf(incumbent.lease, incumbent.readable, now)
|
|
339
|
+
switch (liveness.tag) {
|
|
340
|
+
case "dead":
|
|
341
|
+
dead.push(incumbent)
|
|
342
|
+
break
|
|
343
|
+
case "alive": {
|
|
344
|
+
const holder = incumbent.lease === null ? null : incumbent.lease.holder
|
|
345
|
+
if (holder === ourHolder()) {
|
|
346
|
+
blocked = true
|
|
347
|
+
} else {
|
|
348
|
+
blocked = true
|
|
349
|
+
}
|
|
350
|
+
break
|
|
351
|
+
}
|
|
352
|
+
case "unknown":
|
|
353
|
+
blocked = true
|
|
354
|
+
break
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
if (blocked) {
|
|
358
|
+
if (contend === "refuse") {
|
|
359
|
+
throw errors.new("replica directory has an owner")
|
|
360
|
+
}
|
|
361
|
+
await new Promise(function later(resolve) {
|
|
362
|
+
setTimeout(resolve, Math.random() * LEASE_RETRY_MS)
|
|
363
|
+
})
|
|
364
|
+
continue
|
|
365
|
+
}
|
|
366
|
+
const holder = ourHolder()
|
|
367
|
+
const token = highest + 1n
|
|
368
|
+
const dest = path.join(dir, reservedLease(name, token))
|
|
369
|
+
const body = encodeLease({
|
|
370
|
+
holder,
|
|
371
|
+
token,
|
|
372
|
+
expires: BigInt(now + ttlMs)
|
|
373
|
+
})
|
|
374
|
+
const wrote = await errors.try(syncedTemp(dest, body, dir))
|
|
375
|
+
if (wrote.error) {
|
|
376
|
+
if (isVanished(wrote.error) || isUnproved(wrote.error)) {
|
|
377
|
+
continue
|
|
378
|
+
}
|
|
379
|
+
throw wrote.error
|
|
380
|
+
}
|
|
381
|
+
const published = await errors.try(publishLink(wrote.data, dest))
|
|
382
|
+
await fs.rm(wrote.data, { force: true })
|
|
141
383
|
if (published.error) {
|
|
384
|
+
if (isVanished(published.error) || isUnproved(published.error)) {
|
|
385
|
+
continue
|
|
386
|
+
}
|
|
142
387
|
throw published.error
|
|
143
388
|
}
|
|
144
|
-
if (published.data === "
|
|
389
|
+
if (published.data === "occupied") {
|
|
390
|
+
continue
|
|
391
|
+
}
|
|
392
|
+
await fsyncDir(dir)
|
|
393
|
+
const held: FsLease = { dir, name, holder, token, path: dest }
|
|
394
|
+
for (const stale of dead) {
|
|
395
|
+
if (stale.lease !== null) {
|
|
396
|
+
await casDrop(stale.path, stale.lease)
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
return held
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
async function releaseFsLease(held: FsLease): Promise<void> {
|
|
404
|
+
const read = await errors.try(fs.readFile(held.path, "utf8"))
|
|
405
|
+
if (read.error) {
|
|
406
|
+
return
|
|
407
|
+
}
|
|
408
|
+
const lease = parseLease(read.data)
|
|
409
|
+
if (lease === null) {
|
|
410
|
+
return
|
|
411
|
+
}
|
|
412
|
+
if (lease.holder !== held.holder || lease.token !== held.token) {
|
|
413
|
+
return
|
|
414
|
+
}
|
|
415
|
+
await casDrop(held.path, lease)
|
|
416
|
+
const synced = await errors.try(fsyncDir(held.dir))
|
|
417
|
+
if (synced.error) {
|
|
418
|
+
return
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
async function sweepReserved(root: string): Promise<void> {
|
|
423
|
+
const listed = await errors.try(fs.readdir(root, { withFileTypes: true }))
|
|
424
|
+
if (listed.error) {
|
|
425
|
+
if (codeOf(listed.error) === "ENOENT") {
|
|
145
426
|
return
|
|
146
427
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
428
|
+
throw listed.error
|
|
429
|
+
}
|
|
430
|
+
const now = Date.now()
|
|
431
|
+
for (const entry of listed.data) {
|
|
432
|
+
const full = path.join(root, entry.name)
|
|
433
|
+
if (entry.isDirectory()) {
|
|
434
|
+
if (isReservedName(entry.name)) {
|
|
150
435
|
continue
|
|
151
436
|
}
|
|
152
|
-
|
|
437
|
+
await sweepReserved(full)
|
|
438
|
+
continue
|
|
153
439
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
throw errors.new(`lockfile body is not a pid: ${lockPath}`)
|
|
440
|
+
if (!entry.isFile() || !isReservedName(entry.name)) {
|
|
441
|
+
continue
|
|
157
442
|
}
|
|
158
|
-
if (
|
|
159
|
-
await
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
443
|
+
if (TEMP_NAME_RE.test(entry.name)) {
|
|
444
|
+
await fs.rm(full, { force: true })
|
|
445
|
+
continue
|
|
446
|
+
}
|
|
447
|
+
const match = LEASE_NAME_RE.exec(entry.name)
|
|
448
|
+
if (match === null) {
|
|
449
|
+
continue
|
|
450
|
+
}
|
|
451
|
+
const read = await errors.try(fs.readFile(full, "utf8"))
|
|
452
|
+
if (read.error) {
|
|
453
|
+
continue
|
|
454
|
+
}
|
|
455
|
+
const lease = parseLease(read.data)
|
|
456
|
+
const liveness = livenessOf(lease, true, now)
|
|
457
|
+
if (breakable(liveness) && lease !== null) {
|
|
458
|
+
await casDrop(full, lease)
|
|
164
459
|
}
|
|
165
460
|
}
|
|
166
461
|
}
|
|
167
462
|
|
|
168
|
-
async function
|
|
169
|
-
await
|
|
463
|
+
async function resolveAmbiguousCreate(store: ObjectStore, key: StoreKey, attempted: Uint8Array): Promise<CreateProbe> {
|
|
464
|
+
const fetched = await store.get(key)
|
|
465
|
+
if (fetched === null) {
|
|
466
|
+
return { tag: "absent" }
|
|
467
|
+
}
|
|
468
|
+
if (bytesEqual(fetched.bytes, attempted)) {
|
|
469
|
+
return { tag: "landed", etag: fetched.etag }
|
|
470
|
+
}
|
|
471
|
+
return { tag: "lost", fetched }
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
async function resolveAmbiguousSwap(store: ObjectStore, key: StoreKey, attempted: Uint8Array): Promise<SwapProbe> {
|
|
475
|
+
const fetched = await store.get(key)
|
|
476
|
+
if (fetched === null) {
|
|
477
|
+
return { tag: "absent" }
|
|
478
|
+
}
|
|
479
|
+
if (bytesEqual(fetched.bytes, attempted)) {
|
|
480
|
+
return { tag: "landed", etag: fetched.etag }
|
|
481
|
+
}
|
|
482
|
+
return { tag: "lost", fetched }
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function cloneFetched(fetched: Fetched): Fetched {
|
|
486
|
+
return { bytes: new Uint8Array(fetched.bytes), etag: fetched.etag }
|
|
170
487
|
}
|
|
171
488
|
|
|
172
489
|
/** The five verbs over one local directory. One machine is load-bearing. */
|
|
173
490
|
function fsStore(root: string): ObjectStore {
|
|
174
491
|
const rootPath = path.resolve(root)
|
|
492
|
+
const swept = sweepReserved(rootPath)
|
|
175
493
|
|
|
176
|
-
function objectPath(key:
|
|
177
|
-
checkKey(key)
|
|
494
|
+
function objectPath(key: StoreKey): string {
|
|
178
495
|
return path.join(rootPath, ...key.split("/"))
|
|
179
496
|
}
|
|
180
497
|
|
|
@@ -190,8 +507,19 @@ function fsStore(root: string): ObjectStore {
|
|
|
190
507
|
return { bytes, etag: contentEtag(bytes) }
|
|
191
508
|
}
|
|
192
509
|
|
|
510
|
+
async function withKeyLease<T>(target: string, body: (held: FsLease) => Promise<T>): Promise<T> {
|
|
511
|
+
const held = await acquireFsLease(path.dirname(target), path.basename(target), MUTATION_LEASE_MS)
|
|
512
|
+
const ran = await errors.try(body(held))
|
|
513
|
+
await releaseFsLease(held)
|
|
514
|
+
if (ran.error) {
|
|
515
|
+
throw ran.error
|
|
516
|
+
}
|
|
517
|
+
return ran.data
|
|
518
|
+
}
|
|
519
|
+
|
|
193
520
|
return {
|
|
194
521
|
async get(key) {
|
|
522
|
+
await swept
|
|
195
523
|
const target = objectPath(key)
|
|
196
524
|
const read = await errors.try(readFetched(target))
|
|
197
525
|
if (read.error) {
|
|
@@ -201,6 +529,7 @@ function fsStore(root: string): ObjectStore {
|
|
|
201
529
|
},
|
|
202
530
|
|
|
203
531
|
async getIfChanged(key, etag) {
|
|
532
|
+
await swept
|
|
204
533
|
const target = objectPath(key)
|
|
205
534
|
const read = await errors.try(readFetched(target))
|
|
206
535
|
if (read.error) {
|
|
@@ -216,69 +545,104 @@ function fsStore(root: string): ObjectStore {
|
|
|
216
545
|
},
|
|
217
546
|
|
|
218
547
|
async putCreate(key, bytes) {
|
|
548
|
+
await swept
|
|
219
549
|
const target = objectPath(key)
|
|
220
550
|
const ran = await errors.try(
|
|
221
551
|
(async function createBody(): Promise<Create> {
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
552
|
+
return await withKeyLease(target, async function underLease(held): Promise<Create> {
|
|
553
|
+
if (!(await heldByUs(held))) {
|
|
554
|
+
return { tag: "ambiguous" }
|
|
555
|
+
}
|
|
556
|
+
const temp = await syncedTemp(target, bytes, rootPath)
|
|
557
|
+
const published = await errors.try(publishLink(temp, target))
|
|
558
|
+
await fs.rm(temp, { force: true })
|
|
559
|
+
if (published.error) {
|
|
560
|
+
if (isUnproved(published.error) || isVanished(published.error)) {
|
|
561
|
+
return { tag: "ambiguous" }
|
|
562
|
+
}
|
|
563
|
+
throw published.error
|
|
564
|
+
}
|
|
565
|
+
if (published.data === "occupied") {
|
|
566
|
+
const st = await errors.try(fs.stat(target))
|
|
567
|
+
if (st.error) {
|
|
568
|
+
if (codeOf(st.error) === "ENOENT" || isUnproved(st.error)) {
|
|
569
|
+
return { tag: "ambiguous" }
|
|
570
|
+
}
|
|
571
|
+
throw st.error
|
|
572
|
+
}
|
|
573
|
+
if (st.data.isDirectory()) {
|
|
574
|
+
throw errors.new("key path is a directory")
|
|
575
|
+
}
|
|
576
|
+
return { tag: "exists" }
|
|
577
|
+
}
|
|
578
|
+
const synced = await errors.try(fsyncDir(path.dirname(target)))
|
|
579
|
+
if (synced.error) {
|
|
580
|
+
return { tag: "ambiguous" }
|
|
581
|
+
}
|
|
582
|
+
return { tag: "created", etag: contentEtag(bytes) }
|
|
583
|
+
})
|
|
233
584
|
})()
|
|
234
585
|
)
|
|
235
586
|
if (ran.error) {
|
|
587
|
+
if (isUnproved(ran.error) || isVanished(ran.error)) {
|
|
588
|
+
return { tag: "ambiguous" }
|
|
589
|
+
}
|
|
236
590
|
throw wrapStore(ran.error, `putCreate ${key}`)
|
|
237
591
|
}
|
|
238
592
|
return ran.data
|
|
239
593
|
},
|
|
240
594
|
|
|
241
595
|
async putSwap(key, bytes, etag) {
|
|
596
|
+
await swept
|
|
242
597
|
const target = objectPath(key)
|
|
243
|
-
const lock = `${target}${LOCK_SUFFIX}`
|
|
244
598
|
const ran = await errors.try(
|
|
245
599
|
(async function swapBody(): Promise<Swap> {
|
|
246
|
-
await
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
600
|
+
return await withKeyLease(target, async function underLease(held): Promise<Swap> {
|
|
601
|
+
if (!(await heldByUs(held))) {
|
|
602
|
+
return { tag: "ambiguous" }
|
|
603
|
+
}
|
|
604
|
+
const current = await readFetched(target)
|
|
605
|
+
if (current === null || current.etag !== etag) {
|
|
606
|
+
return { tag: "moved" }
|
|
607
|
+
}
|
|
608
|
+
const temp = await syncedTemp(target, bytes, rootPath)
|
|
609
|
+
const renamed = await errors.try(fs.rename(temp, target))
|
|
610
|
+
if (renamed.error) {
|
|
611
|
+
await fs.rm(temp, { force: true })
|
|
612
|
+
if (isUnproved(renamed.error)) {
|
|
613
|
+
return { tag: "ambiguous" }
|
|
252
614
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
})()
|
|
262
|
-
)
|
|
263
|
-
await releaseLock(lock)
|
|
264
|
-
if (swapped.error) {
|
|
265
|
-
throw swapped.error
|
|
266
|
-
}
|
|
267
|
-
return swapped.data
|
|
615
|
+
throw renamed.error
|
|
616
|
+
}
|
|
617
|
+
const synced = await errors.try(fsyncDir(path.dirname(target)))
|
|
618
|
+
if (synced.error) {
|
|
619
|
+
return { tag: "ambiguous" }
|
|
620
|
+
}
|
|
621
|
+
return { tag: "swapped", etag: contentEtag(bytes) }
|
|
622
|
+
})
|
|
268
623
|
})()
|
|
269
624
|
)
|
|
270
625
|
if (ran.error) {
|
|
626
|
+
if (isUnproved(ran.error)) {
|
|
627
|
+
return { tag: "ambiguous" }
|
|
628
|
+
}
|
|
271
629
|
throw wrapStore(ran.error, `putSwap ${key}`)
|
|
272
630
|
}
|
|
273
631
|
return ran.data
|
|
274
632
|
},
|
|
275
633
|
|
|
276
634
|
async delete(key) {
|
|
635
|
+
await swept
|
|
277
636
|
const target = objectPath(key)
|
|
278
637
|
const ran = await errors.try(
|
|
279
638
|
(async function deleteBody() {
|
|
280
|
-
await
|
|
281
|
-
|
|
639
|
+
await withKeyLease(target, async function underLease(held) {
|
|
640
|
+
if (!(await heldByUs(held))) {
|
|
641
|
+
return
|
|
642
|
+
}
|
|
643
|
+
await fs.rm(target, { force: true })
|
|
644
|
+
await fsyncDir(path.dirname(target))
|
|
645
|
+
})
|
|
282
646
|
})()
|
|
283
647
|
)
|
|
284
648
|
if (ran.error) {
|
|
@@ -288,5 +652,60 @@ function fsStore(root: string): ObjectStore {
|
|
|
288
652
|
}
|
|
289
653
|
}
|
|
290
654
|
|
|
291
|
-
|
|
292
|
-
|
|
655
|
+
/**
|
|
656
|
+
* The five verbs over one in-process Map. Single-process only: tests
|
|
657
|
+
* and ephemeral dev inside this process, no persistence, no
|
|
658
|
+
* cross-process claim, no configuration. Third `Etag` producer beside
|
|
659
|
+
* `fsStore` and `s3Store`: blake3 of the content, `fsStore`'s mint,
|
|
660
|
+
* carried as the same opaque brand. Every read returns a fresh buffer.
|
|
661
|
+
*/
|
|
662
|
+
function memStore(): ObjectStore {
|
|
663
|
+
const objects = new Map<StoreKey, Fetched>()
|
|
664
|
+
return {
|
|
665
|
+
async get(key) {
|
|
666
|
+
const current = objects.get(key)
|
|
667
|
+
return current === undefined ? null : cloneFetched(current)
|
|
668
|
+
},
|
|
669
|
+
|
|
670
|
+
async getIfChanged(key, etag) {
|
|
671
|
+
const current = objects.get(key)
|
|
672
|
+
if (current === undefined) {
|
|
673
|
+
throw wrapStore(errors.new("poll target absent"), `getIfChanged ${key}`)
|
|
674
|
+
}
|
|
675
|
+
if (current.etag === etag) {
|
|
676
|
+
return { tag: "unchanged" }
|
|
677
|
+
}
|
|
678
|
+
return { tag: "changed", fetched: cloneFetched(current) }
|
|
679
|
+
},
|
|
680
|
+
|
|
681
|
+
async putCreate(key, bytes) {
|
|
682
|
+
if (objects.has(key)) {
|
|
683
|
+
return { tag: "exists" }
|
|
684
|
+
}
|
|
685
|
+
const copy = new Uint8Array(bytes)
|
|
686
|
+
const tag = contentEtag(copy)
|
|
687
|
+
objects.set(key, { bytes: copy, etag: tag })
|
|
688
|
+
return { tag: "created", etag: tag }
|
|
689
|
+
},
|
|
690
|
+
|
|
691
|
+
async putSwap(key, bytes, etag) {
|
|
692
|
+
const current = objects.get(key)
|
|
693
|
+
if (current === undefined || current.etag !== etag) {
|
|
694
|
+
return { tag: "moved" }
|
|
695
|
+
}
|
|
696
|
+
const copy = new Uint8Array(bytes)
|
|
697
|
+
const tag = contentEtag(copy)
|
|
698
|
+
objects.set(key, { bytes: copy, etag: tag })
|
|
699
|
+
return { tag: "swapped", etag: tag }
|
|
700
|
+
},
|
|
701
|
+
|
|
702
|
+
async delete(key) {
|
|
703
|
+
objects.delete(key)
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
export type { S3Config, S3Credentials } from "#store-s3.ts"
|
|
709
|
+
export { s3Store } from "#store-s3.ts"
|
|
710
|
+
export type { Create, CreateProbe, Etag, Fetched, FsLease, Lease, Liveness, ObjectStore, Poll, Swap, SwapProbe }
|
|
711
|
+
export { acquireFsLease, etag, fsStore, memStore, releaseFsLease, resolveAmbiguousCreate, resolveAmbiguousSwap }
|