@takosjp/yurucommu-core 3.4.4 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,15 +8,18 @@
8
8
  import type {
9
9
  FirstResult,
10
10
  IDatabase,
11
- IObjectStorage,
11
+ ObjectStore,
12
+ ObjectStoreBody,
13
+ ObjectStoreObject,
14
+ ObjectStorePutOptions,
12
15
  IStaticAssets,
13
- ListObjectsResult,
14
- ObjectMetadata,
15
16
  PreparedStatement,
16
17
  QueryResult,
17
18
  RunResult,
18
- StorageObject,
19
19
  } from "./types.ts";
20
+ import { constants as fsConstants } from "node:fs";
21
+ import type { FileHandle } from "node:fs/promises";
22
+ import { createHash } from "node:crypto";
20
23
  import {
21
24
  assertPathChainWithinBasePath,
22
25
  isPathWithinBasePath,
@@ -34,40 +37,556 @@ declare const require: (specifier: string) => unknown;
34
37
  // Re-export MemoryKV as it works in Bun too.
35
38
  export { MemoryKV };
36
39
 
37
- const { mkdir, unlink, readdir, stat, realpath } = await import("fs/promises");
40
+ const {
41
+ mkdir,
42
+ unlink,
43
+ readdir,
44
+ stat,
45
+ lstat,
46
+ realpath,
47
+ open,
48
+ rename,
49
+ readFile,
50
+ readlink,
51
+ } = await import("fs/promises");
52
+
53
+ // Generation state is deliberately kept outside the public object namespace.
54
+ // The sibling directory name is derived from the storage directory, so a
55
+ // legacy object literally named `.yurucommu-objects` (or any descendant) is
56
+ // still a valid user key under the public root.
57
+ const INTERNAL_STORE_ROOT = ".yurucommu-objects";
58
+ const COMMIT_FILE = "commit.json";
59
+ const GENERATION_MARKER = "generation-";
60
+ const TEMP_MARKER = "tmp-";
61
+ const LEASE_SUFFIX = "lease";
62
+ const READER_LEASE_MARKER = "reader-";
63
+ const MARKER_READ_ATTEMPTS = 32;
64
+ const OBJECT_RESOLVE_ATTEMPTS = 64;
65
+ const OPEN_READ_FLAGS = fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW;
66
+ const GENERATION_ID_PATTERN = /^[0-9a-f-]{16,}$/u;
67
+ const DIGEST_PATTERN = /^[0-9a-f]{64}$/u;
68
+ const LEASE_BOOT_ID_PATTERN =
69
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu;
70
+ const LEASE_PID_NAMESPACE_ID_PATTERN = /^pid:\[[1-9][0-9]*\]$/u;
71
+ const LEASE_PROCESS_START_TOKEN_PATTERN = /^[0-9]{1,32}$/u;
72
+ const LEASE_TOKEN_MAX_LENGTH = 128;
73
+
74
+ type CommitState =
75
+ | { version: 1; state: "committed"; generation: string }
76
+ | { version: 1; state: "deleted"; generation: null };
77
+
78
+ type CommitRecord = CommitState & { key: string; keyHash: string };
79
+
80
+ type ResolvedObject = {
81
+ filePath: string;
82
+ bodyHandle?: FileHandle;
83
+ metadata: {
84
+ contentType?: string;
85
+ };
86
+ releaseBody?: () => Promise<void>;
87
+ };
38
88
 
39
- /**
40
- * Drain a ReadableStream into a single Uint8Array.
41
- */
42
- async function drainStream(
89
+ type TempLease = (() => Promise<void>) & {
90
+ assertOwned(): Promise<void>;
91
+ };
92
+
93
+ type LeaseProcessIdentity = {
94
+ bootId: string;
95
+ pidNamespaceId: string;
96
+ processStartToken: string;
97
+ };
98
+
99
+ type LeaseOwner = {
100
+ version: 2;
101
+ pid: number;
102
+ token: string;
103
+ processIdentity: LeaseProcessIdentity;
104
+ };
105
+
106
+ type LeaseOwnerState = "live" | "dead" | "unknown";
107
+
108
+ type LeaseProcessProbe = {
109
+ readPidNamespace(pid: number): Promise<string>;
110
+ readPidStat(pid: number): Promise<string>;
111
+ signal0(pid: number): void;
112
+ };
113
+
114
+ const defaultLeaseProcessProbe: LeaseProcessProbe = {
115
+ readPidNamespace(pid) {
116
+ return readlink(`/proc/${pid}/ns/pid`);
117
+ },
118
+ readPidStat(pid) {
119
+ return readFile(`/proc/${pid}/stat`, "utf8");
120
+ },
121
+ signal0(pid) {
122
+ process.kill(pid, 0);
123
+ },
124
+ };
125
+
126
+ type LiveLeaseState = {
127
+ generations: Set<string>;
128
+ reader: boolean;
129
+ };
130
+
131
+ type FileIdentity = {
132
+ dev: number;
133
+ ino: number;
134
+ nlink: number;
135
+ mode: number;
136
+ };
137
+
138
+ type BunStorageOptions = {
139
+ syncDirectory?: typeof syncDirectory;
140
+ /** Deterministic test seam at the lease-assertion/publication boundary. */
141
+ beforeCommitRename?: (commitPath: string) => Promise<void>;
142
+ /** Deterministic test seam for Linux /proc visibility and PID incarnation. */
143
+ leaseProcessProbe?: LeaseProcessProbe;
144
+ };
145
+
146
+ function fileIdentity(stats: FileIdentity): FileIdentity {
147
+ return {
148
+ dev: stats.dev,
149
+ ino: stats.ino,
150
+ nlink: stats.nlink,
151
+ mode: stats.mode,
152
+ };
153
+ }
154
+
155
+ function isSameFileIdentity(
156
+ expected: FileIdentity,
157
+ actual: FileIdentity,
158
+ ): boolean {
159
+ return (
160
+ expected.dev === actual.dev &&
161
+ expected.ino === actual.ino &&
162
+ // A concurrent atomic replacement may unlink the just-opened inode
163
+ // between lstat() and fstat(); POSIX then reports nlink=0 on the still
164
+ // valid descriptor. It is safe to accept that transition because the
165
+ // pre-open lstat required nlink===1 and dev/ino/type still match. Any
166
+ // other link count indicates a hardlink or inode substitution.
167
+ (expected.nlink === actual.nlink ||
168
+ actual.nlink === 0 ||
169
+ (expected.nlink === 0 && actual.nlink === 1)) &&
170
+ expected.mode === actual.mode
171
+ );
172
+ }
173
+
174
+ function isInternalStorageName(name: string): boolean {
175
+ return name.endsWith(".meta.json");
176
+ }
177
+
178
+ function keyHash(key: string): string {
179
+ return createHash("sha256").update(key, "utf8").digest("hex");
180
+ }
181
+
182
+ function isNotFoundError(error: unknown): boolean {
183
+ return (
184
+ typeof error === "object" &&
185
+ error !== null &&
186
+ "code" in error &&
187
+ (error as { code?: unknown }).code === "ENOENT"
188
+ );
189
+ }
190
+
191
+ function isFileIdentityRace(error: unknown): boolean {
192
+ return (
193
+ error instanceof Error &&
194
+ error.message === "BunStorage file changed while opening"
195
+ );
196
+ }
197
+
198
+ function hasExactObjectKeys(
199
+ value: Record<string, unknown>,
200
+ expected: readonly string[],
201
+ ): boolean {
202
+ const keys = Object.keys(value).sort();
203
+ return (
204
+ keys.length === expected.length &&
205
+ expected.every((key, index) => keys[index] === key)
206
+ );
207
+ }
208
+
209
+ function parseLeaseProcessIdentity(
210
+ value: unknown,
211
+ ): LeaseProcessIdentity | undefined {
212
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
213
+ return undefined;
214
+ }
215
+ const record = value as Record<string, unknown>;
216
+ if (
217
+ !hasExactObjectKeys(record, [
218
+ "bootId",
219
+ "pidNamespaceId",
220
+ "processStartToken",
221
+ ]) ||
222
+ typeof record.bootId !== "string" ||
223
+ !LEASE_BOOT_ID_PATTERN.test(record.bootId) ||
224
+ typeof record.pidNamespaceId !== "string" ||
225
+ !LEASE_PID_NAMESPACE_ID_PATTERN.test(record.pidNamespaceId) ||
226
+ typeof record.processStartToken !== "string" ||
227
+ !LEASE_PROCESS_START_TOKEN_PATTERN.test(record.processStartToken)
228
+ ) {
229
+ return undefined;
230
+ }
231
+ return {
232
+ bootId: record.bootId,
233
+ pidNamespaceId: record.pidNamespaceId,
234
+ processStartToken: record.processStartToken,
235
+ };
236
+ }
237
+
238
+ function parseLeaseOwner(ownerText: string): LeaseOwner | undefined {
239
+ try {
240
+ const value = JSON.parse(ownerText) as unknown;
241
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
242
+ return undefined;
243
+ }
244
+ const record = value as Record<string, unknown>;
245
+ const processIdentity = parseLeaseProcessIdentity(record.processIdentity);
246
+ if (
247
+ !hasExactObjectKeys(record, [
248
+ "pid",
249
+ "processIdentity",
250
+ "token",
251
+ "version",
252
+ ]) ||
253
+ record.version !== 2 ||
254
+ typeof record.pid !== "number" ||
255
+ !Number.isSafeInteger(record.pid) ||
256
+ record.pid <= 0 ||
257
+ typeof record.token !== "string" ||
258
+ record.token.length === 0 ||
259
+ record.token.length > LEASE_TOKEN_MAX_LENGTH ||
260
+ !processIdentity
261
+ ) {
262
+ return undefined;
263
+ }
264
+ return {
265
+ version: 2,
266
+ pid: record.pid,
267
+ token: record.token,
268
+ processIdentity,
269
+ };
270
+ } catch {
271
+ return undefined;
272
+ }
273
+ }
274
+
275
+ function parseProcessStartToken(statText: string): string | undefined {
276
+ // /proc/<pid>/stat field 2 is parenthesized and may itself contain spaces or
277
+ // parentheses. The final ')' closes it; field 22 is then tail index 19.
278
+ const commandEnd = statText.lastIndexOf(")");
279
+ if (commandEnd < 0) return undefined;
280
+ const token = statText
281
+ .slice(commandEnd + 1)
282
+ .trim()
283
+ .split(/\s+/u)[19];
284
+ return token && LEASE_PROCESS_START_TOKEN_PATTERN.test(token)
285
+ ? token
286
+ : undefined;
287
+ }
288
+
289
+ async function loadLocalLeaseProcessIdentity(): Promise<
290
+ LeaseProcessIdentity | undefined
291
+ > {
292
+ try {
293
+ const [bootId, pidNamespaceId, statText] = await Promise.all([
294
+ readFile("/proc/sys/kernel/random/boot_id", "utf8"),
295
+ readlink("/proc/self/ns/pid"),
296
+ readFile("/proc/self/stat", "utf8"),
297
+ ]);
298
+ return parseLeaseProcessIdentity({
299
+ bootId: bootId.trim(),
300
+ pidNamespaceId,
301
+ processStartToken: parseProcessStartToken(statText),
302
+ });
303
+ } catch {
304
+ return undefined;
305
+ }
306
+ }
307
+
308
+ let localLeaseProcessIdentity:
309
+ Promise<LeaseProcessIdentity | undefined> | undefined;
310
+
311
+ function getLocalLeaseProcessIdentity(): Promise<
312
+ LeaseProcessIdentity | undefined
313
+ > {
314
+ localLeaseProcessIdentity ??= loadLocalLeaseProcessIdentity();
315
+ return localLeaseProcessIdentity;
316
+ }
317
+
318
+ function pidIsDefinitelyAbsent(pid: number, probe: LeaseProcessProbe): boolean {
319
+ try {
320
+ probe.signal0(pid);
321
+ return false;
322
+ } catch (error) {
323
+ return (
324
+ typeof error === "object" &&
325
+ error !== null &&
326
+ "code" in error &&
327
+ (error as { code?: unknown }).code === "ESRCH"
328
+ );
329
+ }
330
+ }
331
+
332
+ async function classifyLeaseOwner(
333
+ owner: LeaseOwner,
334
+ probe: LeaseProcessProbe,
335
+ ): Promise<LeaseOwnerState> {
336
+ const localIdentity = await getLocalLeaseProcessIdentity();
337
+ if (
338
+ !localIdentity ||
339
+ owner.processIdentity.bootId !== localIdentity.bootId ||
340
+ owner.processIdentity.pidNamespaceId !== localIdentity.pidNamespaceId
341
+ ) {
342
+ return "unknown";
343
+ }
344
+
345
+ let targetNamespaceId: string;
346
+ try {
347
+ targetNamespaceId = await probe.readPidNamespace(owner.pid);
348
+ } catch (error) {
349
+ if (!isNotFoundError(error)) return "unknown";
350
+ // hidepid=2 deliberately presents a live different-UID process as ENOENT.
351
+ // Only ESRCH from signal 0 corroborates that this same-boot/same-namespace
352
+ // PID is absent; EPERM and every other result remain unknown/fail-safe.
353
+ return pidIsDefinitelyAbsent(owner.pid, probe) ? "dead" : "unknown";
354
+ }
355
+ if (targetNamespaceId !== localIdentity.pidNamespaceId) return "unknown";
356
+
357
+ let targetStat: string;
358
+ try {
359
+ targetStat = await probe.readPidStat(owner.pid);
360
+ } catch (error) {
361
+ if (!isNotFoundError(error)) return "unknown";
362
+ return pidIsDefinitelyAbsent(owner.pid, probe) ? "dead" : "unknown";
363
+ }
364
+ const targetStartToken = parseProcessStartToken(targetStat);
365
+ if (!targetStartToken) return "unknown";
366
+ return targetStartToken === owner.processIdentity.processStartToken
367
+ ? "live"
368
+ : "dead";
369
+ }
370
+
371
+ const emittedLeaseDiagnostics = new Set<string>();
372
+
373
+ function reportUnknownLeaseRetention(): void {
374
+ const code = "lease-owner-unknown";
375
+ if (emittedLeaseDiagnostics.has(code)) return;
376
+ emittedLeaseDiagnostics.add(code);
377
+ console.warn(
378
+ "BunStorage retained an unverifiable lease; automatic reclaim is disabled. Confirm that no writer is active before operator cleanup (lease-owner-unknown).",
379
+ );
380
+ }
381
+
382
+ function yieldForFilesystem(): Promise<void> {
383
+ return new Promise((resolve) => setTimeout(resolve, 0));
384
+ }
385
+
386
+ export async function writeBufferFully(
387
+ handle: FileHandle,
388
+ buffer: Uint8Array,
389
+ ): Promise<void> {
390
+ let offset = 0;
391
+ while (offset < buffer.byteLength) {
392
+ const result = await handle.write(buffer.subarray(offset));
393
+ const bytesWritten =
394
+ typeof result === "number" ? result : result?.bytesWritten;
395
+ if (
396
+ !Number.isInteger(bytesWritten) ||
397
+ bytesWritten <= 0 ||
398
+ bytesWritten > buffer.byteLength - offset
399
+ ) {
400
+ throw new Error(
401
+ `filesystem short write: expected ${buffer.byteLength - offset} bytes, received ${String(bytesWritten)}`,
402
+ );
403
+ }
404
+ offset += bytesWritten;
405
+ }
406
+ }
407
+
408
+ async function writeStreamToFile(
409
+ handle: FileHandle,
43
410
  stream: ReadableStream<Uint8Array>,
44
- ): Promise<Uint8Array> {
45
- const chunks: Uint8Array[] = [];
411
+ ): Promise<void> {
46
412
  const reader = stream.getReader();
47
- while (true) {
48
- const { done, value } = await reader.read();
49
- if (done) break;
50
- chunks.push(value);
413
+ try {
414
+ while (true) {
415
+ const { done, value } = await reader.read();
416
+ if (done) return;
417
+ if (!(value instanceof Uint8Array)) {
418
+ throw new Error("filesystem stream yielded a non-byte chunk");
419
+ }
420
+ await writeBufferFully(handle, value);
421
+ }
422
+ } catch (error) {
423
+ await reader.cancel(error).catch(() => undefined);
424
+ throw error;
425
+ } finally {
426
+ reader.releaseLock();
427
+ }
428
+ }
429
+
430
+ async function writeValueToFile(
431
+ handle: FileHandle,
432
+ value: Blob | ReadableStream | ArrayBuffer | string,
433
+ ): Promise<void> {
434
+ if (typeof value === "string") {
435
+ await writeBufferFully(handle, new TextEncoder().encode(value));
436
+ return;
51
437
  }
52
- const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
53
- const result = new Uint8Array(totalLength);
438
+ if (value instanceof ArrayBuffer) {
439
+ await writeBufferFully(handle, new Uint8Array(value));
440
+ return;
441
+ }
442
+ const stream = value instanceof Blob ? value.stream() : value;
443
+ await writeStreamToFile(handle, stream as ReadableStream<Uint8Array>);
444
+ }
445
+
446
+ async function syncAndClose(handle: FileHandle): Promise<void> {
447
+ try {
448
+ await handle.sync();
449
+ } finally {
450
+ await handle.close();
451
+ }
452
+ }
453
+
454
+ async function syncDirectory(directoryPath: string): Promise<void> {
455
+ let handle: FileHandle | undefined;
456
+ try {
457
+ // On Bun's POSIX filesystem, syncing the containing directory makes the
458
+ // preceding atomic rename durable across power loss, not just the file
459
+ // contents themselves.
460
+ handle = await open(directoryPath, "r");
461
+ await handle.sync();
462
+ } finally {
463
+ if (handle) await handle.close().catch(() => undefined);
464
+ }
465
+ }
466
+
467
+ async function readFileHandleFully(handle: FileHandle): Promise<Uint8Array> {
468
+ const size = (await handle.stat()).size;
469
+ if (!Number.isSafeInteger(size) || size < 0) {
470
+ throw new Error("invalid filesystem object size");
471
+ }
472
+ const content = new Uint8Array(size);
54
473
  let offset = 0;
55
- for (const chunk of chunks) {
56
- result.set(chunk, offset);
57
- offset += chunk.length;
474
+ while (offset < content.byteLength) {
475
+ const result = await handle.read(
476
+ content,
477
+ offset,
478
+ content.byteLength - offset,
479
+ null,
480
+ );
481
+ if (
482
+ !Number.isInteger(result.bytesRead) ||
483
+ result.bytesRead <= 0 ||
484
+ result.bytesRead > content.byteLength - offset
485
+ ) {
486
+ throw new Error(
487
+ `filesystem short read: expected ${content.byteLength - offset} bytes, received ${String(result.bytesRead)}`,
488
+ );
489
+ }
490
+ offset += result.bytesRead;
58
491
  }
59
- return result;
492
+ return content;
60
493
  }
61
494
 
62
495
  /**
63
- * Convert a put() value to Uint8Array.
496
+ * Expose an already-open object file as a lazy body stream.
497
+ *
498
+ * Pathname leases are released before this stream is returned. A writer may
499
+ * publish a replacement or unlink the selected generation while a response is
500
+ * still being sent, but the open descriptor keeps those exact bytes stable on
501
+ * POSIX until the stream is consumed or cancelled.
64
502
  */
65
- async function toUint8Array(
66
- value: ReadableStream | ArrayBuffer | string,
67
- ): Promise<Uint8Array> {
68
- if (typeof value === "string") return new TextEncoder().encode(value);
69
- if (value instanceof ArrayBuffer) return new Uint8Array(value);
70
- return drainStream(value);
503
+ function createFileBodyStream(
504
+ handle: FileHandle,
505
+ byteLength: number,
506
+ release: () => Promise<void>,
507
+ ): ReadableStream<Uint8Array> {
508
+ const chunkSize = 64 * 1024;
509
+ let offset = 0;
510
+ let released = false;
511
+
512
+ const finish = async (): Promise<void> => {
513
+ if (released) return;
514
+ released = true;
515
+ await release().catch(() => undefined);
516
+ };
517
+
518
+ return new ReadableStream<Uint8Array>(
519
+ {
520
+ async pull(controller) {
521
+ if (offset >= byteLength) {
522
+ controller.close();
523
+ await finish();
524
+ return;
525
+ }
526
+ const buffer = new Uint8Array(Math.min(chunkSize, byteLength - offset));
527
+ try {
528
+ const result = await handle.read(
529
+ buffer,
530
+ 0,
531
+ buffer.byteLength,
532
+ offset,
533
+ );
534
+ if (
535
+ !Number.isInteger(result.bytesRead) ||
536
+ result.bytesRead <= 0 ||
537
+ result.bytesRead > buffer.byteLength
538
+ ) {
539
+ throw new Error(
540
+ `filesystem short read: expected ${buffer.byteLength} bytes, received ${String(result.bytesRead)}`,
541
+ );
542
+ }
543
+ offset += result.bytesRead;
544
+ controller.enqueue(buffer.subarray(0, result.bytesRead));
545
+ if (offset >= byteLength) {
546
+ controller.close();
547
+ await finish();
548
+ }
549
+ } catch (error) {
550
+ controller.error(error);
551
+ await finish();
552
+ }
553
+ },
554
+ async cancel() {
555
+ await finish();
556
+ },
557
+ },
558
+ { highWaterMark: 0 },
559
+ );
560
+ }
561
+
562
+ async function readFileHandleText(handle: FileHandle): Promise<string> {
563
+ return new TextDecoder().decode(await readFileHandleFully(handle));
564
+ }
565
+
566
+ function parseObjectMetadata(valueText: string): {
567
+ contentType?: string;
568
+ } {
569
+ const value = JSON.parse(valueText) as {
570
+ contentType?: unknown;
571
+ httpMetadata?: unknown;
572
+ };
573
+ if (value === null || typeof value !== "object") {
574
+ throw new Error("Invalid BunStorage generation metadata");
575
+ }
576
+ const legacyHttpMetadata = value.httpMetadata;
577
+ const legacyContentType =
578
+ legacyHttpMetadata !== null &&
579
+ typeof legacyHttpMetadata === "object" &&
580
+ "contentType" in legacyHttpMetadata
581
+ ? (legacyHttpMetadata as { contentType?: unknown }).contentType
582
+ : undefined;
583
+ const contentType =
584
+ typeof value.contentType === "string"
585
+ ? value.contentType
586
+ : typeof legacyContentType === "string"
587
+ ? legacyContentType
588
+ : undefined;
589
+ return contentType === undefined ? {} : { contentType };
71
590
  }
72
591
 
73
592
  /**
@@ -75,13 +594,12 @@ async function toUint8Array(
75
594
  * Returns an empty object if the sidecar doesn't exist or can't be parsed.
76
595
  */
77
596
  async function readMetadata(metaPath: string): Promise<{
78
- httpMetadata?: ObjectMetadata["httpMetadata"];
79
- customMetadata?: Record<string, string>;
597
+ contentType?: string;
80
598
  }> {
81
599
  try {
82
600
  const metaFile = Bun.file(metaPath);
83
601
  if (await metaFile.exists()) {
84
- return JSON.parse(await metaFile.text());
602
+ return parseObjectMetadata(await metaFile.text());
85
603
  }
86
604
  } catch {
87
605
  // No metadata file or unreadable
@@ -206,19 +724,77 @@ class BunPreparedStatement implements PreparedStatement {
206
724
  }
207
725
 
208
726
  /**
209
- * Bun Filesystem Storage Adapter
727
+ * Bun Filesystem Storage Adapter.
728
+ *
729
+ * The generation/lease protocol requires a local Linux POSIX filesystem with
730
+ * atomic same-directory rename and stable open-descriptor semantics. It does
731
+ * not claim multi-host fencing. A lease from another boot, PID namespace, or
732
+ * unreadable /proc owner is retained for explicit operator inspection rather
733
+ * than automatically reclaimed.
210
734
  */
211
- export class BunStorage implements IObjectStorage {
735
+ export class BunStorage implements ObjectStore {
212
736
  private basePath: string;
737
+ /**
738
+ * Keep the directory-sync dependency injectable for deterministic storage
739
+ * durability tests. Production callers use the real fsync implementation.
740
+ */
741
+ private readonly syncDirectory: typeof syncDirectory;
742
+ private readonly beforeCommitRename?: (commitPath: string) => Promise<void>;
743
+ private readonly leaseProcessProbe: LeaseProcessProbe;
213
744
  private realBasePath: string | null = null;
745
+ private realInternalStorePath: string | null = null;
746
+ /**
747
+ * Generations currently being assembled by this adapter instance. A
748
+ * concurrent writer can have renamed its body before publishing the commit
749
+ * marker; keep that generation out of eager GC until its marker is durable.
750
+ */
751
+ private readonly activeGenerations = new Set<string>();
752
+ /**
753
+ * In-process read leases keep a resolved generation alive until its bytes
754
+ * have been opened/read. GC may run concurrently with a reader, so path
755
+ * resolution alone is not a sufficient lifetime guarantee.
756
+ */
757
+ private readonly generationReaders = new Map<string, number>();
758
+ /**
759
+ * A short reservation held while a reader resolves the current marker and
760
+ * opens its generation. It closes the race between reading the marker and
761
+ * acquiring the generation-specific lease.
762
+ */
763
+ private readonly keyReadReservations = new Map<string, number>();
764
+ /**
765
+ * Serialize the conservative post-reader reclamation pass per key. A hot
766
+ * object can have many concurrent streaming readers; running a full
767
+ * directory scan for every EOF makes writers wait behind an unbounded queue
768
+ * of equivalent cleanup work.
769
+ */
770
+ private readonly reclaimTasks = new Map<
771
+ string,
772
+ { requested: boolean; promise: Promise<void> }
773
+ >();
774
+ /**
775
+ * Last marker successfully read by this adapter. Atomic marker replacement
776
+ * can make a path disappear or resolve to the next inode for a few syscalls;
777
+ * retaining the last validated record lets readers use the old generation
778
+ * while the marker is in flight instead of reporting a false null.
779
+ */
780
+ private readonly lastCommitRecords = new Map<string, CommitRecord>();
214
781
 
215
- constructor(basePath: string) {
782
+ constructor(basePath: string, options: BunStorageOptions = {}) {
216
783
  this.basePath = basePath;
784
+ this.syncDirectory = options.syncDirectory ?? syncDirectory;
785
+ this.beforeCommitRename = options.beforeCommitRename;
786
+ this.leaseProcessProbe =
787
+ options.leaseProcessProbe ?? defaultLeaseProcessProbe;
217
788
  }
218
789
 
219
- static async create(basePath: string): Promise<BunStorage> {
790
+ static async create(
791
+ basePath: string,
792
+ options: BunStorageOptions = {},
793
+ ): Promise<BunStorage> {
220
794
  await mkdir(basePath, { recursive: true });
221
- return new BunStorage(basePath);
795
+ const storage = new BunStorage(basePath, options);
796
+ await storage.recoverInternalStore();
797
+ return storage;
222
798
  }
223
799
 
224
800
  private getFilePath(key: string): string {
@@ -232,6 +808,299 @@ export class BunStorage implements IObjectStorage {
232
808
  );
233
809
  }
234
810
 
811
+ private getInternalStorePath(): string {
812
+ const publicRoot = this.getResolvedBasePath();
813
+ return path.join(
814
+ path.dirname(publicRoot),
815
+ `.${path.basename(publicRoot) || "root"}${INTERNAL_STORE_ROOT}`,
816
+ );
817
+ }
818
+
819
+ private getInternalObjectPath(key: string): string {
820
+ return path.join(this.getInternalStorePath(), keyHash(key));
821
+ }
822
+
823
+ private isInternalStorePath(filePath: string): boolean {
824
+ return isPathWithinBasePath(
825
+ this.getInternalStorePath(),
826
+ path.resolve(filePath),
827
+ );
828
+ }
829
+
830
+ private getCommitPath(key: string): string {
831
+ return path.join(this.getInternalObjectPath(key), COMMIT_FILE);
832
+ }
833
+
834
+ private getGenerationPath(
835
+ key: string,
836
+ generation: string,
837
+ suffix: "body" | "meta.json",
838
+ ): string {
839
+ if (!GENERATION_ID_PATTERN.test(generation)) {
840
+ throw new Error("Invalid BunStorage generation identifier");
841
+ }
842
+ return path.join(
843
+ this.getInternalObjectPath(key),
844
+ `${GENERATION_MARKER}${generation}.${suffix}`,
845
+ );
846
+ }
847
+
848
+ private generationLeaseKey(key: string, generation: string): string {
849
+ return `${keyHash(key)}:${generation}`;
850
+ }
851
+
852
+ private acquireGenerationLease(
853
+ key: string,
854
+ generation: string,
855
+ ): () => Promise<void> {
856
+ const leaseKey = this.generationLeaseKey(key, generation);
857
+ this.generationReaders.set(
858
+ leaseKey,
859
+ (this.generationReaders.get(leaseKey) ?? 0) + 1,
860
+ );
861
+ let released = false;
862
+ return async () => {
863
+ if (released) return;
864
+ released = true;
865
+ const count = this.generationReaders.get(leaseKey) ?? 0;
866
+ if (count <= 1) this.generationReaders.delete(leaseKey);
867
+ else this.generationReaders.set(leaseKey, count - 1);
868
+ // A writer may have deferred this generation while the reader held its
869
+ // lease. Re-run conservative GC after release; failure is non-fatal and
870
+ // startup recovery remains the final orphan cleanup authority. Queue the
871
+ // pass so EOF never blocks behind another reader's identical scan.
872
+ this.scheduleReclaim(key);
873
+ };
874
+ }
875
+
876
+ private hasGenerationLease(key: string, generation: string): boolean {
877
+ return (
878
+ (this.generationReaders.get(this.generationLeaseKey(key, generation)) ??
879
+ 0) > 0
880
+ );
881
+ }
882
+
883
+ private acquireKeyReadReservation(key: string): () => void {
884
+ this.keyReadReservations.set(
885
+ key,
886
+ (this.keyReadReservations.get(key) ?? 0) + 1,
887
+ );
888
+ let released = false;
889
+ return () => {
890
+ if (released) return;
891
+ released = true;
892
+ const count = this.keyReadReservations.get(key) ?? 0;
893
+ if (count <= 1) this.keyReadReservations.delete(key);
894
+ else this.keyReadReservations.set(key, count - 1);
895
+ };
896
+ }
897
+
898
+ private hasKeyReadReservation(key: string): boolean {
899
+ return (this.keyReadReservations.get(key) ?? 0) > 0;
900
+ }
901
+
902
+ private scheduleReclaim(key: string): void {
903
+ const active = this.reclaimTasks.get(key);
904
+ if (active) {
905
+ // One trailing pass is enough to observe a lease released while the
906
+ // current scan was in flight; never build one promise per reader.
907
+ active.requested = true;
908
+ return;
909
+ }
910
+ const state: { requested: boolean; promise: Promise<void> } = {
911
+ requested: false,
912
+ promise: Promise.resolve(),
913
+ };
914
+ state.promise = (async () => {
915
+ state.requested = false;
916
+ await this.reclaimUnreferencedGenerations(key).catch(() => undefined);
917
+ })();
918
+ this.reclaimTasks.set(key, state);
919
+ void state.promise
920
+ .finally(() => {
921
+ if (this.reclaimTasks.get(key) === state) {
922
+ this.reclaimTasks.delete(key);
923
+ }
924
+ if (state.requested) this.scheduleReclaim(key);
925
+ })
926
+ .catch(() => undefined);
927
+ }
928
+
929
+ private async createTempLease(
930
+ objectPath: string,
931
+ leaseId: string,
932
+ kind: "generation" | "reader" = "generation",
933
+ ): Promise<TempLease> {
934
+ const marker = kind === "reader" ? READER_LEASE_MARKER : "";
935
+ const leasePath = path.join(
936
+ objectPath,
937
+ `${TEMP_MARKER}${marker}${leaseId}.${LEASE_SUFFIX}`,
938
+ );
939
+ const leaseToken = crypto.randomUUID();
940
+ const processIdentity = await getLocalLeaseProcessIdentity();
941
+ const leaseOwner = JSON.stringify({
942
+ version: 2,
943
+ pid: process.pid,
944
+ token: leaseToken,
945
+ // A missing /proc identity is deliberately persisted as unknown. This
946
+ // writer can still release its own inode/token, but a later process must
947
+ // retain it for operator inspection rather than guess that it is dead.
948
+ processIdentity: processIdentity ?? null,
949
+ });
950
+ let leaseHandle: FileHandle | undefined;
951
+ let leaseIdentity: FileIdentity | undefined;
952
+ try {
953
+ leaseHandle = await open(leasePath, "wx");
954
+ await writeBufferFully(leaseHandle, new TextEncoder().encode(leaseOwner));
955
+ await leaseHandle.sync();
956
+ leaseIdentity = fileIdentity(await leaseHandle.stat());
957
+ await this.assertOwnedInternalFile(leasePath);
958
+ } catch (error) {
959
+ if (leaseHandle) await leaseHandle.close().catch(() => undefined);
960
+ await this.unlinkOwnedInternalFile(leasePath);
961
+ throw error;
962
+ }
963
+
964
+ const ownsLease = async (): Promise<boolean> => {
965
+ if (!leaseHandle || !leaseIdentity) return false;
966
+ let currentPathHandle: FileHandle | undefined;
967
+ try {
968
+ const heldIdentity = fileIdentity(await leaseHandle.stat());
969
+ if (
970
+ heldIdentity.nlink !== 1 ||
971
+ heldIdentity.dev !== leaseIdentity.dev ||
972
+ heldIdentity.ino !== leaseIdentity.ino ||
973
+ heldIdentity.mode !== leaseIdentity.mode
974
+ ) {
975
+ return false;
976
+ }
977
+ const opened = await this.openOwnedInternalFile(leasePath);
978
+ currentPathHandle = opened.handle;
979
+ if (
980
+ opened.identity.dev !== leaseIdentity.dev ||
981
+ opened.identity.ino !== leaseIdentity.ino ||
982
+ opened.identity.mode !== leaseIdentity.mode
983
+ ) {
984
+ return false;
985
+ }
986
+ return (await readFileHandleText(currentPathHandle)) === leaseOwner;
987
+ } catch {
988
+ return false;
989
+ } finally {
990
+ if (currentPathHandle) {
991
+ await currentPathHandle.close().catch(() => undefined);
992
+ }
993
+ }
994
+ };
995
+
996
+ let released = false;
997
+ const release = async () => {
998
+ if (released) return;
999
+ released = true;
1000
+ try {
1001
+ if (await ownsLease()) await this.unlinkOwnedInternalFile(leasePath);
1002
+ } finally {
1003
+ await leaseHandle?.close().catch(() => undefined);
1004
+ leaseHandle = undefined;
1005
+ }
1006
+ };
1007
+ return Object.assign(release, {
1008
+ async assertOwned(): Promise<void> {
1009
+ if (released || !(await ownsLease())) {
1010
+ throw new Error("BunStorage writer lease ownership lost");
1011
+ }
1012
+ },
1013
+ });
1014
+ }
1015
+
1016
+ private async createReaderLease(key: string): Promise<TempLease | undefined> {
1017
+ const objectPath = this.getInternalObjectPath(key);
1018
+ try {
1019
+ await assertPathChainWithinBasePath(
1020
+ await this.getRealInternalStorePath(),
1021
+ objectPath,
1022
+ realpath,
1023
+ );
1024
+ await this.assertRealInternalDirectory(objectPath);
1025
+ return await this.createTempLease(
1026
+ objectPath,
1027
+ crypto.randomUUID(),
1028
+ "reader",
1029
+ );
1030
+ } catch (error) {
1031
+ if (isNotFoundError(error)) return undefined;
1032
+ throw error;
1033
+ }
1034
+ }
1035
+
1036
+ private async isLiveTempLease(leasePath: string): Promise<boolean> {
1037
+ let leaseHandle: FileHandle | undefined;
1038
+ try {
1039
+ try {
1040
+ const opened = await this.openOwnedInternalFile(leasePath);
1041
+ leaseHandle = opened.handle;
1042
+ } catch (error) {
1043
+ if (isNotFoundError(error)) return false;
1044
+ reportUnknownLeaseRetention();
1045
+ return true;
1046
+ }
1047
+
1048
+ const leaseOwner = parseLeaseOwner(await readFileHandleText(leaseHandle));
1049
+ if (!leaseOwner) {
1050
+ reportUnknownLeaseRetention();
1051
+ return true;
1052
+ }
1053
+ const state = await classifyLeaseOwner(
1054
+ leaseOwner,
1055
+ this.leaseProcessProbe,
1056
+ );
1057
+ if (state === "unknown") reportUnknownLeaseRetention();
1058
+ // Absence or a different start token is proof that this exact local
1059
+ // process incarnation is dead. Every other state is fail-safe retained.
1060
+ return state !== "dead";
1061
+ } catch {
1062
+ reportUnknownLeaseRetention();
1063
+ return true;
1064
+ } finally {
1065
+ if (leaseHandle) await leaseHandle.close().catch(() => undefined);
1066
+ }
1067
+ }
1068
+
1069
+ private async liveTempLeases(
1070
+ objectPath: string,
1071
+ entries: Array<{ name: string; isDirectory(): boolean }>,
1072
+ ): Promise<LiveLeaseState> {
1073
+ const generations = new Set<string>();
1074
+ let reader = false;
1075
+ for (const entry of entries) {
1076
+ if (entry.isDirectory()) continue;
1077
+ const generationMatch = entry.name.match(
1078
+ /^tmp-([0-9a-f-]{16,})\.lease$/u,
1079
+ );
1080
+ const readerMatch = entry.name.match(
1081
+ /^tmp-reader-([0-9a-f-]{16,})\.lease$/u,
1082
+ );
1083
+ if (!generationMatch && !readerMatch) continue;
1084
+ const leasePath = path.join(objectPath, entry.name);
1085
+ if (await this.isLiveTempLease(leasePath)) {
1086
+ if (readerMatch) reader = true;
1087
+ else generations.add(generationMatch![1]!);
1088
+ } else {
1089
+ await this.unlinkOwnedInternalFile(leasePath);
1090
+ }
1091
+ }
1092
+ return { generations, reader };
1093
+ }
1094
+
1095
+ private async hasLiveReaderLease(objectPath: string): Promise<boolean> {
1096
+ try {
1097
+ const entries = await readdir(objectPath, { withFileTypes: true });
1098
+ return (await this.liveTempLeases(objectPath, entries)).reader;
1099
+ } catch {
1100
+ return false;
1101
+ }
1102
+ }
1103
+
235
1104
  private getResolvedBasePath(): string {
236
1105
  return path.resolve(this.basePath);
237
1106
  }
@@ -247,6 +1116,47 @@ export class BunStorage implements IObjectStorage {
247
1116
  return this.realBasePath;
248
1117
  }
249
1118
 
1119
+ private async getRealInternalStorePath(): Promise<string> {
1120
+ if (this.realInternalStorePath) {
1121
+ // Re-check the lexical root on every access. A rolling process or an
1122
+ // operator-side repair can replace the directory with a symlink after
1123
+ // startup; the cached realpath must not make that substitution trusted.
1124
+ await this.assertRealInternalDirectory(this.getInternalStorePath());
1125
+ return this.realInternalStorePath;
1126
+ }
1127
+ const internalRoot = this.getInternalStorePath();
1128
+ const internalParent = path.dirname(internalRoot);
1129
+ const realInternalParent = await realpath(internalParent);
1130
+ // The metadata namespace is allowed outside the public root, but it must
1131
+ // remain within the same trusted parent and may not be redirected through
1132
+ // a symlink to an unrelated filesystem location.
1133
+ await assertPathChainWithinBasePath(
1134
+ realInternalParent,
1135
+ internalRoot,
1136
+ realpath,
1137
+ );
1138
+ try {
1139
+ const rootStats = await lstat(internalRoot);
1140
+ if (rootStats.isSymbolicLink() || !rootStats.isDirectory()) {
1141
+ throw new Error("BunStorage metadata root must be a real directory");
1142
+ }
1143
+ } catch (error) {
1144
+ if (!isNotFoundError(error)) throw error;
1145
+ }
1146
+ await mkdir(internalRoot, { recursive: true });
1147
+ const createdRootStats = await lstat(internalRoot);
1148
+ if (createdRootStats.isSymbolicLink() || !createdRootStats.isDirectory()) {
1149
+ throw new Error("BunStorage metadata root must be a real directory");
1150
+ }
1151
+ await this.syncDirectory(internalParent);
1152
+ const resolvedRoot = await realpath(internalRoot);
1153
+ if (!isPathWithinBasePath(realInternalParent, resolvedRoot)) {
1154
+ throw new Error("BunStorage metadata path escapes its parent");
1155
+ }
1156
+ this.realInternalStorePath = resolvedRoot;
1157
+ return resolvedRoot;
1158
+ }
1159
+
250
1160
  private async resolveExistingPath(filePath: string): Promise<string | null> {
251
1161
  try {
252
1162
  const realPath = await realpath(filePath);
@@ -260,193 +1170,1104 @@ export class BunStorage implements IObjectStorage {
260
1170
  }
261
1171
  }
262
1172
 
263
- async put(
264
- key: string,
265
- value: ReadableStream | ArrayBuffer | string,
266
- options?: {
267
- httpMetadata?: ObjectMetadata["httpMetadata"];
268
- customMetadata?: Record<string, string>;
269
- },
1173
+ private async resolveExistingInternalPath(
1174
+ filePath: string,
1175
+ ): Promise<string | null> {
1176
+ try {
1177
+ const realPath = await realpath(filePath);
1178
+ const realInternalRoot = await this.getRealInternalStorePath();
1179
+ if (!isPathWithinBasePath(realInternalRoot, realPath)) {
1180
+ throw new Error("BunStorage metadata path escapes its root");
1181
+ }
1182
+ return realPath;
1183
+ } catch {
1184
+ return null;
1185
+ }
1186
+ }
1187
+
1188
+ private async assertRealInternalDirectory(
1189
+ directoryPath: string,
270
1190
  ): Promise<void> {
271
- const filePath = this.getFilePath(key);
272
- const dir = path.dirname(filePath);
1191
+ const directoryStats = await lstat(directoryPath);
1192
+ if (directoryStats.isSymbolicLink() || !directoryStats.isDirectory()) {
1193
+ throw new Error("BunStorage internal path must be a real directory");
1194
+ }
1195
+ }
273
1196
 
274
- await assertPathChainWithinBasePath(
275
- await this.getRealBasePath(),
1197
+ /**
1198
+ * Open an adapter-owned regular file without following a final symlink or
1199
+ * accepting a hardlink to an external inode. The lstat/fstat identity check
1200
+ * closes the realpath-to-open replacement window; callers read through the
1201
+ * returned descriptor rather than reopening the path.
1202
+ */
1203
+ private async openOwnedRegularFile(
1204
+ filePath: string,
1205
+ rootPath: string,
1206
+ ): Promise<{ handle: FileHandle; identity: FileIdentity }> {
1207
+ await this.assertOwnedPathChain(rootPath, filePath);
1208
+ const before = await lstat(filePath);
1209
+ if (
1210
+ before.isSymbolicLink() ||
1211
+ !before.isFile() ||
1212
+ // During an atomic rename/unlink, Bun can expose the just-unlinked
1213
+ // inode as nlink=0 for one stat result. It is still safe to open only
1214
+ // when no additional link (nlink>1) is present.
1215
+ before.nlink < 0 ||
1216
+ before.nlink > 1
1217
+ ) {
1218
+ throw new Error("BunStorage file is not an owned regular file");
1219
+ }
1220
+ const expected = fileIdentity(before);
1221
+ let handle: FileHandle | undefined;
1222
+ try {
1223
+ handle = await open(filePath, OPEN_READ_FLAGS);
1224
+ const after = fileIdentity(await handle.stat());
1225
+ if (!isSameFileIdentity(expected, after)) {
1226
+ throw new Error("BunStorage file changed while opening");
1227
+ }
1228
+ // Re-check the path after opening as well. O_NOFOLLOW protects the
1229
+ // final component, while this catches a parent-directory replacement
1230
+ // that happened between the initial containment check and open().
1231
+ await this.assertOwnedPathChain(rootPath, filePath, false);
1232
+ return { handle, identity: after };
1233
+ } catch (error) {
1234
+ if (handle) await handle.close().catch(() => undefined);
1235
+ throw error;
1236
+ }
1237
+ }
1238
+
1239
+ /**
1240
+ * Internal files are adapter-owned, so every path component must be a real
1241
+ * directory/file rather than a symlink. realpath containment alone is not
1242
+ * sufficient: a symlink can point to another in-root inode and still win a
1243
+ * later path lookup, and the final lstat/open pair has a TOCTOU window.
1244
+ */
1245
+ private async assertOwnedPathChain(
1246
+ rootPath: string,
1247
+ targetPath: string,
1248
+ includeTarget = true,
1249
+ ): Promise<void> {
1250
+ const root = path.resolve(rootPath);
1251
+ const target = path.resolve(targetPath);
1252
+ if (!isPathWithinBasePath(root, target)) {
1253
+ throw new Error("BunStorage path escapes its owned root");
1254
+ }
1255
+ const relativePath = path.relative(root, target);
1256
+ let current = root;
1257
+ const components = relativePath ? relativePath.split(path.sep) : [];
1258
+ if (!includeTarget) components.pop();
1259
+ for (const component of components) {
1260
+ current = path.join(current, component);
1261
+ const stats = await lstat(current);
1262
+ if (stats.isSymbolicLink()) {
1263
+ throw new Error("BunStorage internal path contains a symlink");
1264
+ }
1265
+ }
1266
+ }
1267
+
1268
+ private async readOwnedInternalFileText(filePath: string): Promise<string> {
1269
+ const opened = await this.openOwnedInternalFile(filePath);
1270
+ try {
1271
+ return await readFileHandleText(opened.handle);
1272
+ } finally {
1273
+ await opened.handle.close().catch(() => undefined);
1274
+ }
1275
+ }
1276
+
1277
+ private async readOwnedInternalFileMetadata(filePath: string): Promise<{
1278
+ contentType?: string;
1279
+ }> {
1280
+ return this.parseGenerationMetadata(
1281
+ await this.readOwnedInternalFileText(filePath),
1282
+ );
1283
+ }
1284
+
1285
+ private async openOwnedInternalFile(
1286
+ filePath: string,
1287
+ ): Promise<{ handle: FileHandle; identity: FileIdentity }> {
1288
+ return this.openOwnedRegularFile(
276
1289
  filePath,
277
- realpath,
1290
+ await this.getRealInternalStorePath(),
278
1291
  );
1292
+ }
279
1293
 
280
- await mkdir(dir, { recursive: true });
1294
+ private async assertOwnedRegularFile(
1295
+ filePath: string,
1296
+ rootPath: string,
1297
+ ): Promise<FileIdentity> {
1298
+ const opened = await this.openOwnedRegularFile(filePath, rootPath);
1299
+ await opened.handle.close();
1300
+ return opened.identity;
1301
+ }
281
1302
 
282
- const realBasePath = await this.getRealBasePath();
283
- let realFilePath: string | null = null;
1303
+ private async assertOwnedInternalFile(
1304
+ filePath: string,
1305
+ ): Promise<FileIdentity> {
1306
+ return this.assertOwnedRegularFile(
1307
+ filePath,
1308
+ await this.getRealInternalStorePath(),
1309
+ );
1310
+ }
1311
+
1312
+ /**
1313
+ * Remove only an owned regular file. A symlink or hardlink is left in place
1314
+ * for recovery/security inspection rather than unlinking an unowned inode.
1315
+ */
1316
+ private async unlinkOwnedRegularFile(
1317
+ filePath: string,
1318
+ rootPath: string,
1319
+ ): Promise<void> {
284
1320
  try {
285
- realFilePath = await realpath(filePath);
286
- } catch {
287
- realFilePath = null;
1321
+ await this.assertOwnedRegularFile(filePath, rootPath);
1322
+ } catch (error) {
1323
+ if (isNotFoundError(error)) return;
1324
+ return;
288
1325
  }
289
- if (realFilePath) {
290
- if (!isPathWithinBasePath(realBasePath, realFilePath)) {
291
- throw new Error("Path escapes base directory");
1326
+ await unlink(filePath).catch(() => undefined);
1327
+ }
1328
+
1329
+ private async unlinkOwnedInternalFile(filePath: string): Promise<void> {
1330
+ await this.unlinkOwnedRegularFile(
1331
+ filePath,
1332
+ await this.getRealInternalStorePath(),
1333
+ );
1334
+ }
1335
+
1336
+ private async recoverInternalStore(): Promise<void> {
1337
+ try {
1338
+ // Validate the sibling namespace before creating or traversing it. A
1339
+ // symlink at this exact path must never redirect recovery elsewhere.
1340
+ const resolvedRoot = await this.getRealInternalStorePath();
1341
+ const objectEntries = await readdir(resolvedRoot, {
1342
+ withFileTypes: true,
1343
+ });
1344
+ for (const objectEntry of objectEntries) {
1345
+ if (!DIGEST_PATTERN.test(objectEntry.name)) {
1346
+ continue;
1347
+ }
1348
+ const objectPath = path.join(resolvedRoot, objectEntry.name);
1349
+ await this.assertRealInternalDirectory(objectPath);
1350
+ const resolvedObjectPath =
1351
+ await this.resolveExistingInternalPath(objectPath);
1352
+ if (!resolvedObjectPath) continue;
1353
+ const files = await readdir(resolvedObjectPath, {
1354
+ withFileTypes: true,
1355
+ });
1356
+ const liveLeaseState = await this.liveTempLeases(
1357
+ resolvedObjectPath,
1358
+ files,
1359
+ );
1360
+ let objectKey: string | undefined;
1361
+ let retainedGeneration: string | null = null;
1362
+ try {
1363
+ const markerPath = path.join(resolvedObjectPath, COMMIT_FILE);
1364
+ const markerStats = await lstat(markerPath).catch(() => null);
1365
+ if (markerStats && markerStats.isFile() && markerStats.nlink === 1) {
1366
+ const record = JSON.parse(
1367
+ await this.readOwnedInternalFileText(markerPath),
1368
+ ) as Partial<CommitRecord>;
1369
+ if (
1370
+ record.version === 1 &&
1371
+ typeof record.key === "string" &&
1372
+ record.keyHash === objectEntry.name &&
1373
+ keyHash(record.key) === objectEntry.name
1374
+ ) {
1375
+ objectKey = record.key;
1376
+ }
1377
+ if (
1378
+ objectKey !== undefined &&
1379
+ record.state === "committed" &&
1380
+ typeof record.generation === "string" &&
1381
+ GENERATION_ID_PATTERN.test(record.generation)
1382
+ ) {
1383
+ const bodyPath = path.join(
1384
+ resolvedObjectPath,
1385
+ `${GENERATION_MARKER}${record.generation}.body`,
1386
+ );
1387
+ const metaPath = path.join(
1388
+ resolvedObjectPath,
1389
+ `${GENERATION_MARKER}${record.generation}.meta.json`,
1390
+ );
1391
+ try {
1392
+ const body = await this.openOwnedInternalFile(bodyPath);
1393
+ await body.handle.close();
1394
+ // Parse metadata during recovery so a partially-written or
1395
+ // corrupt sidecar cannot be retained as a live generation.
1396
+ await this.readGenerationMetadata(metaPath);
1397
+ retainedGeneration = record.generation;
1398
+ } catch {
1399
+ // Missing, symlinked, hardlinked, or corrupt generations are
1400
+ // not eligible to remain referenced by the marker.
1401
+ }
1402
+ }
1403
+ }
1404
+ } catch {
1405
+ retainedGeneration = null;
1406
+ }
1407
+
1408
+ for (const entry of files) {
1409
+ if (entry.isDirectory()) continue;
1410
+ const leaseMatch = entry.name.match(/^tmp-([0-9a-f-]{16,})\.lease$/u);
1411
+ const readerLeaseMatch = entry.name.match(
1412
+ /^tmp-reader-([0-9a-f-]{16,})\.lease$/u,
1413
+ );
1414
+ if (leaseMatch || readerLeaseMatch) {
1415
+ if (
1416
+ (leaseMatch && liveLeaseState.generations.has(leaseMatch[1]!)) ||
1417
+ (readerLeaseMatch && liveLeaseState.reader)
1418
+ ) {
1419
+ continue;
1420
+ }
1421
+ await this.unlinkOwnedInternalFile(
1422
+ path.join(resolvedObjectPath, entry.name),
1423
+ );
1424
+ continue;
1425
+ }
1426
+ if (entry.name.startsWith(TEMP_MARKER)) {
1427
+ const tempGeneration = entry.name.match(
1428
+ /^tmp-([0-9a-f-]{16,})\./u,
1429
+ )?.[1];
1430
+ if (
1431
+ tempGeneration &&
1432
+ liveLeaseState.generations.has(tempGeneration)
1433
+ ) {
1434
+ continue;
1435
+ }
1436
+ await this.unlinkOwnedInternalFile(
1437
+ path.join(resolvedObjectPath, entry.name),
1438
+ );
1439
+ continue;
1440
+ }
1441
+ const generationMatch = entry.name.match(
1442
+ /^generation-([0-9a-f-]{16,})\.(?:body|meta\.json)$/u,
1443
+ );
1444
+ if (
1445
+ generationMatch &&
1446
+ generationMatch[1] !== retainedGeneration &&
1447
+ !liveLeaseState.generations.has(generationMatch[1]!) &&
1448
+ !liveLeaseState.reader
1449
+ ) {
1450
+ // Recovery can overlap a rolling writer. The marker and lease
1451
+ // values above are only scan snapshots: if a valid marker gave us
1452
+ // the object key, re-establish the same fresh deletion authority
1453
+ // used by steady-state GC immediately before unlinking.
1454
+ if (
1455
+ objectKey !== undefined &&
1456
+ !(await this.canReclaimGeneration(objectKey, generationMatch[1]!))
1457
+ ) {
1458
+ continue;
1459
+ }
1460
+ await this.unlinkOwnedInternalFile(
1461
+ path.join(resolvedObjectPath, entry.name),
1462
+ );
1463
+ }
1464
+ }
292
1465
  }
293
- } else {
294
- const realDirPath = await realpath(dir);
295
- if (!isPathWithinBasePath(realBasePath, realDirPath)) {
296
- throw new Error("Path escapes base directory");
1466
+ } catch (error) {
1467
+ // A storage directory can disappear between mkdir and recovery; the
1468
+ // first normal put will recreate its digest directory. Security and
1469
+ // namespace validation failures must remain observable instead of being
1470
+ // mistaken for an empty store.
1471
+ if (!isNotFoundError(error)) throw error;
1472
+ }
1473
+ }
1474
+
1475
+ private async readCommitRecord(
1476
+ key: string,
1477
+ allowCachedFallback = true,
1478
+ ): Promise<CommitRecord | undefined> {
1479
+ const markerPath = this.getCommitPath(key);
1480
+ let markerObserved = false;
1481
+ let markerText: string | undefined;
1482
+ let lastNotFound: unknown;
1483
+ for (let attempt = 0; attempt < MARKER_READ_ATTEMPTS; attempt += 1) {
1484
+ let markerHandle: FileHandle | undefined;
1485
+ try {
1486
+ // Validate every existing path component before opening the marker.
1487
+ // This distinguishes a missing marker (normal legacy fallback) from a
1488
+ // symlinked internal namespace that escapes its sibling root.
1489
+ const opened = await this.openOwnedInternalFile(markerPath);
1490
+ markerHandle = opened.handle;
1491
+ markerObserved = true;
1492
+ // Read through an open descriptor. Resolving the path and then asking
1493
+ // Bun.file() to open it leaves a rename/unlink window where Bun can
1494
+ // retain a stale `(... deleted)` path; an FD is either the old
1495
+ // complete marker or the new complete marker after atomic rename.
1496
+ markerText = await readFileHandleText(markerHandle);
1497
+ break;
1498
+ } catch (error) {
1499
+ if (!isNotFoundError(error) && !isFileIdentityRace(error)) throw error;
1500
+ lastNotFound = error;
1501
+ try {
1502
+ const markerStats = await lstat(markerPath);
1503
+ if (
1504
+ markerStats.isSymbolicLink() ||
1505
+ !markerStats.isFile() ||
1506
+ markerStats.nlink < 0 ||
1507
+ markerStats.nlink > 1
1508
+ ) {
1509
+ throw new Error("BunStorage commit marker is not owned");
1510
+ }
1511
+ markerObserved = true;
1512
+ } catch (probeError) {
1513
+ if (!isNotFoundError(probeError)) throw probeError;
1514
+ }
1515
+ if (attempt + 1 < MARKER_READ_ATTEMPTS) {
1516
+ await yieldForFilesystem();
1517
+ continue;
1518
+ }
1519
+ } finally {
1520
+ if (markerHandle) await markerHandle.close().catch(() => undefined);
1521
+ }
1522
+ }
1523
+ if (markerText === undefined) {
1524
+ // A marker that was ever visible must not silently fall through to the
1525
+ // legacy path after a rename race. The caller's resolve loop will reread
1526
+ // the marker; a truly absent marker still enables legacy compatibility.
1527
+ const cached = this.lastCommitRecords.get(key);
1528
+ if (allowCachedFallback && cached && lastNotFound) return cached;
1529
+ // Reclamation must never act on a cached marker. An absent/racing marker
1530
+ // is insufficient authority to delete a generation; a later pass or
1531
+ // startup recovery can retry after the marker is readable.
1532
+ if (!allowCachedFallback && lastNotFound) throw lastNotFound;
1533
+ if (markerObserved && lastNotFound) throw lastNotFound;
1534
+ return undefined;
1535
+ }
1536
+
1537
+ const record = JSON.parse(markerText) as Partial<CommitRecord>;
1538
+ if (
1539
+ record.version !== 1 ||
1540
+ typeof record.key !== "string" ||
1541
+ typeof record.keyHash !== "string" ||
1542
+ record.key !== key ||
1543
+ record.keyHash !== keyHash(key)
1544
+ ) {
1545
+ throw new Error("Invalid BunStorage commit marker identity");
1546
+ }
1547
+ if (record.state === "deleted" && record.generation === null) {
1548
+ this.lastCommitRecords.set(key, record as CommitRecord);
1549
+ return record as CommitRecord;
1550
+ }
1551
+ if (
1552
+ record.state !== "committed" ||
1553
+ typeof record.generation !== "string" ||
1554
+ !GENERATION_ID_PATTERN.test(record.generation)
1555
+ ) {
1556
+ throw new Error("Invalid BunStorage commit marker");
1557
+ }
1558
+ this.lastCommitRecords.set(key, record as CommitRecord);
1559
+ return record as CommitRecord;
1560
+ }
1561
+
1562
+ private async readGenerationMetadata(metaPath: string): Promise<{
1563
+ contentType?: string;
1564
+ }> {
1565
+ return this.readOwnedInternalFileMetadata(metaPath);
1566
+ }
1567
+
1568
+ private parseGenerationMetadata(valueText: string): {
1569
+ contentType?: string;
1570
+ } {
1571
+ return parseObjectMetadata(valueText);
1572
+ }
1573
+
1574
+ private async readGenerationMetadataHandle(handle: FileHandle): Promise<{
1575
+ contentType?: string;
1576
+ }> {
1577
+ return this.parseGenerationMetadata(await readFileHandleText(handle));
1578
+ }
1579
+
1580
+ private async resolveObject(
1581
+ key: string,
1582
+ withLease = false,
1583
+ ): Promise<ResolvedObject | null> {
1584
+ const releaseKeyRead = withLease
1585
+ ? this.acquireKeyReadReservation(key)
1586
+ : undefined;
1587
+ let readerLease: TempLease | undefined;
1588
+ const releaseReaderLease = async () => {
1589
+ const release = readerLease;
1590
+ readerLease = undefined;
1591
+ await release?.();
1592
+ if (release) this.scheduleReclaim(key);
1593
+ };
1594
+ try {
1595
+ if (withLease) readerLease = await this.createReaderLease(key);
1596
+ for (let attempt = 0; attempt < OBJECT_RESOLVE_ATTEMPTS; attempt += 1) {
1597
+ let commit: CommitRecord | undefined;
1598
+ try {
1599
+ commit = await this.readCommitRecord(key);
1600
+ } catch (error) {
1601
+ if (
1602
+ (isNotFoundError(error) || isFileIdentityRace(error)) &&
1603
+ attempt + 1 < OBJECT_RESOLVE_ATTEMPTS
1604
+ ) {
1605
+ await yieldForFilesystem();
1606
+ continue;
1607
+ }
1608
+ throw error;
1609
+ }
1610
+ if (commit) {
1611
+ if (commit.state === "deleted") return null;
1612
+ const generation = commit.generation;
1613
+ const generationLease = withLease
1614
+ ? this.acquireGenerationLease(key, generation)
1615
+ : undefined;
1616
+ let bodyHandle: FileHandle | undefined;
1617
+ let metadataHandle: FileHandle | undefined;
1618
+ let generationProtectionReleased = false;
1619
+ const releaseGenerationProtection = async () => {
1620
+ if (generationProtectionReleased) return;
1621
+ generationProtectionReleased = true;
1622
+ await generationLease?.();
1623
+ };
1624
+ let bodyReleased = false;
1625
+ const releaseBody = async () => {
1626
+ if (bodyReleased) return;
1627
+ bodyReleased = true;
1628
+ if (metadataHandle) {
1629
+ await metadataHandle.close().catch(() => undefined);
1630
+ metadataHandle = undefined;
1631
+ }
1632
+ if (bodyHandle) {
1633
+ await bodyHandle.close().catch(() => undefined);
1634
+ bodyHandle = undefined;
1635
+ }
1636
+ };
1637
+ const releaseAttempt = async () => {
1638
+ await releaseBody();
1639
+ await releaseGenerationProtection();
1640
+ };
1641
+ try {
1642
+ const filePath = this.getGenerationPath(key, generation, "body");
1643
+ const metaPath = this.getGenerationPath(
1644
+ key,
1645
+ generation,
1646
+ "meta.json",
1647
+ );
1648
+ const resolvedFilePath =
1649
+ await this.resolveExistingInternalPath(filePath);
1650
+ const resolvedMetaPath =
1651
+ await this.resolveExistingInternalPath(metaPath);
1652
+ if (!resolvedFilePath || !resolvedMetaPath) {
1653
+ await releaseAttempt();
1654
+ await yieldForFilesystem();
1655
+ continue;
1656
+ }
1657
+
1658
+ if (withLease) {
1659
+ // Open both files before releasing the marker read. Once the body
1660
+ // descriptor is open, a concurrent unlink cannot invalidate this
1661
+ // read on POSIX; a missing path simply causes a bounded marker
1662
+ // reread/retry against the winning generation. The owned-open
1663
+ // helper rejects hardlinks/symlinks and reads through the FD,
1664
+ // closing both the identity and path-swap windows.
1665
+ bodyHandle = (await this.openOwnedInternalFile(filePath)).handle;
1666
+ metadataHandle = (await this.openOwnedInternalFile(metaPath))
1667
+ .handle;
1668
+ const metadata =
1669
+ await this.readGenerationMetadataHandle(metadataHandle);
1670
+ await metadataHandle.close();
1671
+ metadataHandle = undefined;
1672
+ // Once the body FD is open and metadata is materialized, POSIX
1673
+ // unlink semantics keep those exact bytes readable without any
1674
+ // pathname lease. Release both local and cross-instance path
1675
+ // protection now so an undrained lazy stream cannot retain every
1676
+ // superseded generation or one lease FD per object lifetime.
1677
+ await releaseGenerationProtection();
1678
+ await releaseReaderLease();
1679
+ return {
1680
+ filePath: resolvedFilePath,
1681
+ bodyHandle,
1682
+ metadata,
1683
+ releaseBody,
1684
+ };
1685
+ }
1686
+
1687
+ const openedBody = await this.openOwnedInternalFile(filePath);
1688
+ const openedMeta = await this.openOwnedInternalFile(metaPath);
1689
+ await openedBody.handle.close();
1690
+ await openedMeta.handle.close();
1691
+ const metadata =
1692
+ await this.readGenerationMetadata(resolvedMetaPath);
1693
+ await releaseGenerationProtection();
1694
+ return { filePath: resolvedFilePath, metadata };
1695
+ } catch (error) {
1696
+ await releaseAttempt();
1697
+ if (
1698
+ isNotFoundError(error) &&
1699
+ attempt + 1 < OBJECT_RESOLVE_ATTEMPTS
1700
+ ) {
1701
+ await yieldForFilesystem();
1702
+ continue;
1703
+ }
1704
+ throw error;
1705
+ }
1706
+ }
1707
+
1708
+ // If the generation namespace exists but its marker was observed as
1709
+ // missing, an atomic rename may be between unlink and replacement.
1710
+ // Do not fall through to the legacy path (and return null) during that
1711
+ // churn; retry the marker read while the internal object directory is
1712
+ // present. A genuinely marker-less legacy object still falls through
1713
+ // after the bounded resolve attempts.
1714
+ if (withLease) {
1715
+ const internalObjectPath = this.getInternalObjectPath(key);
1716
+ if (
1717
+ (await this.resolveExistingInternalPath(internalObjectPath)) &&
1718
+ attempt + 1 < OBJECT_RESOLVE_ATTEMPTS
1719
+ ) {
1720
+ await yieldForFilesystem();
1721
+ continue;
1722
+ }
1723
+ }
1724
+
1725
+ // Objects written by older BunStorage versions use the body + metadata
1726
+ // sidecar layout. Keep those reads working until the object is rewritten.
1727
+ const filePath = this.getFilePath(key);
1728
+ const resolvedFilePath = await this.resolveExistingPath(filePath);
1729
+ if (!resolvedFilePath) return null;
1730
+ const file = Bun.file(resolvedFilePath);
1731
+ if (!(await file.exists())) return null;
1732
+ if (withLease) {
1733
+ let bodyHandle: FileHandle | undefined;
1734
+ let metadataHandle: FileHandle | undefined;
1735
+ try {
1736
+ const openedBody = await this.openOwnedRegularFile(
1737
+ resolvedFilePath,
1738
+ await this.getRealBasePath(),
1739
+ );
1740
+ bodyHandle = openedBody.handle;
1741
+ const resolvedMetaPath = await this.resolveExistingPath(
1742
+ this.getMetaPath(key),
1743
+ );
1744
+ let metadata: ResolvedObject["metadata"] = {};
1745
+ if (resolvedMetaPath) {
1746
+ const openedMeta = await this.openOwnedRegularFile(
1747
+ resolvedMetaPath,
1748
+ await this.getRealBasePath(),
1749
+ );
1750
+ metadataHandle = openedMeta.handle;
1751
+ metadata = this.parseGenerationMetadata(
1752
+ await readFileHandleText(metadataHandle),
1753
+ );
1754
+ await metadataHandle.close();
1755
+ metadataHandle = undefined;
1756
+ }
1757
+ let released = false;
1758
+ const releaseBody = async () => {
1759
+ if (released) return;
1760
+ released = true;
1761
+ if (bodyHandle) {
1762
+ await bodyHandle.close().catch(() => undefined);
1763
+ bodyHandle = undefined;
1764
+ }
1765
+ };
1766
+ await releaseReaderLease();
1767
+ return {
1768
+ filePath: resolvedFilePath,
1769
+ bodyHandle,
1770
+ metadata,
1771
+ releaseBody,
1772
+ };
1773
+ } catch (error) {
1774
+ if (metadataHandle)
1775
+ await metadataHandle.close().catch(() => undefined);
1776
+ if (bodyHandle) await bodyHandle.close().catch(() => undefined);
1777
+ if (isNotFoundError(error) && attempt < 2) continue;
1778
+ throw error;
1779
+ }
1780
+ }
1781
+ const resolvedMetaPath = await this.resolveExistingPath(
1782
+ this.getMetaPath(key),
1783
+ );
1784
+ const metadata = resolvedMetaPath
1785
+ ? await readMetadata(resolvedMetaPath)
1786
+ : {};
1787
+ return { filePath: resolvedFilePath, metadata };
1788
+ }
1789
+ return null;
1790
+ } finally {
1791
+ releaseKeyRead?.();
1792
+ await releaseReaderLease();
1793
+ }
1794
+ }
1795
+
1796
+ private async writeCommitRecord(
1797
+ key: string,
1798
+ record: CommitState,
1799
+ ): Promise<void> {
1800
+ const objectPath = this.getInternalObjectPath(key);
1801
+ await assertPathChainWithinBasePath(
1802
+ await this.getRealInternalStorePath(),
1803
+ objectPath,
1804
+ realpath,
1805
+ );
1806
+ await mkdir(objectPath, { recursive: true });
1807
+ await this.assertRealInternalDirectory(objectPath);
1808
+ await this.syncDirectory(path.dirname(objectPath));
1809
+ const markerPath = this.getCommitPath(key);
1810
+ const markerGeneration = crypto.randomUUID();
1811
+ const markerTempPath = path.join(
1812
+ objectPath,
1813
+ `${TEMP_MARKER}${markerGeneration}.commit.json`,
1814
+ );
1815
+ const releaseLease = await this.createTempLease(
1816
+ objectPath,
1817
+ markerGeneration,
1818
+ );
1819
+ const commitPayload: CommitRecord = {
1820
+ ...record,
1821
+ key,
1822
+ keyHash: keyHash(key),
1823
+ };
1824
+ let markerHandle: FileHandle | undefined;
1825
+ let markerTempCreated = false;
1826
+ try {
1827
+ markerHandle = await open(markerTempPath, "wx");
1828
+ markerTempCreated = true;
1829
+ await writeBufferFully(
1830
+ markerHandle,
1831
+ new TextEncoder().encode(JSON.stringify(commitPayload)),
1832
+ );
1833
+ await syncAndClose(markerHandle);
1834
+ markerHandle = undefined;
1835
+ await releaseLease.assertOwned();
1836
+ await this.beforeCommitRename?.(markerPath);
1837
+ await rename(markerTempPath, markerPath);
1838
+ // The marker rename is the publication point. Mark the temporary path
1839
+ // gone immediately so a subsequent directory-fsync error cannot make
1840
+ // finally misclassify the committed marker as unpublished.
1841
+ markerTempCreated = false;
1842
+ await this.syncDirectory(objectPath);
1843
+ } finally {
1844
+ if (markerHandle) await markerHandle.close().catch(() => undefined);
1845
+ if (markerTempCreated) await this.unlinkOwnedInternalFile(markerTempPath);
1846
+ await releaseLease();
1847
+ }
1848
+ }
1849
+
1850
+ private async removeGenerationIfUnreferenced(
1851
+ key: string,
1852
+ generation: string | null | undefined,
1853
+ ): Promise<void> {
1854
+ if (generation && (await this.canReclaimGeneration(key, generation))) {
1855
+ for (const suffix of ["body", "meta.json"] as const) {
1856
+ // Re-establish authority for each unlink. A reader or delayed writer
1857
+ // can publish/acquire a lease after removal of the paired file starts;
1858
+ // leaving one orphan is safer than deleting a newly-current pair.
1859
+ if (!(await this.canReclaimGeneration(key, generation))) break;
1860
+ const filePath = this.getGenerationPath(key, generation, suffix);
1861
+ await this.unlinkOwnedInternalFile(filePath);
297
1862
  }
298
1863
  }
1864
+ await this.reclaimUnreferencedGenerations(key);
1865
+ }
299
1866
 
300
- const content = await toUint8Array(value);
301
- await Bun.write(filePath, content);
1867
+ /**
1868
+ * Establish deletion authority for one generation without trusting a stale
1869
+ * marker snapshot.
1870
+ *
1871
+ * Writer ordering is lease -> generation files -> commit marker -> lease
1872
+ * release. Reader ordering is reader lease -> marker/open body -> lease
1873
+ * release. Checking both leases around an uncached marker read therefore
1874
+ * prevents GC from unlinking either the currently published generation or a
1875
+ * generation that an already-started reader can still reference.
1876
+ */
1877
+ private async canReclaimGeneration(
1878
+ key: string,
1879
+ generation: string,
1880
+ ): Promise<boolean> {
1881
+ if (
1882
+ this.activeGenerations.has(`${keyHash(key)}:${generation}`) ||
1883
+ this.hasGenerationLease(key, generation) ||
1884
+ this.hasKeyReadReservation(key)
1885
+ ) {
1886
+ return false;
1887
+ }
302
1888
 
303
- if (options?.httpMetadata || options?.customMetadata) {
304
- await Bun.write(
305
- this.getMetaPath(key),
306
- JSON.stringify({
307
- httpMetadata: options.httpMetadata,
308
- customMetadata: options.customMetadata,
309
- }),
1889
+ const objectPath = this.getInternalObjectPath(key);
1890
+ const generationLeasePath = path.join(
1891
+ objectPath,
1892
+ `${TEMP_MARKER}${generation}.${LEASE_SUFFIX}`,
1893
+ );
1894
+ if (
1895
+ (await this.isLiveTempLease(generationLeasePath)) ||
1896
+ (await this.hasLiveReaderLease(objectPath))
1897
+ ) {
1898
+ return false;
1899
+ }
1900
+
1901
+ let current: CommitRecord | undefined;
1902
+ try {
1903
+ current = await this.readCommitRecord(key, false);
1904
+ } catch {
1905
+ return false;
1906
+ }
1907
+ if (current?.state === "committed" && current.generation === generation) {
1908
+ return false;
1909
+ }
1910
+
1911
+ // A reader can acquire its cross-instance lease immediately before the
1912
+ // authoritative marker read and still hold the previous generation after
1913
+ // a concurrent publication. Recheck after the marker decision closes that
1914
+ // window. Rechecking the writer lease keeps deletion fail-closed if a
1915
+ // generation lease was transiently unreadable during the first probe.
1916
+ return !(
1917
+ (await this.hasLiveReaderLease(objectPath)) ||
1918
+ (await this.isLiveTempLease(generationLeasePath))
1919
+ );
1920
+ }
1921
+
1922
+ private async reclaimUnreferencedGenerations(key: string): Promise<void> {
1923
+ let current: CommitRecord | undefined;
1924
+ try {
1925
+ current = await this.readCommitRecord(key);
1926
+ } catch {
1927
+ // A malformed marker is fail-closed; startup recovery will handle it.
1928
+ return;
1929
+ }
1930
+ const retainedGeneration =
1931
+ current?.state === "committed" ? current.generation : null;
1932
+ const objectPath = this.getInternalObjectPath(key);
1933
+ const resolvedObjectPath =
1934
+ await this.resolveExistingInternalPath(objectPath);
1935
+ if (!resolvedObjectPath) return;
1936
+ let entries;
1937
+ try {
1938
+ entries = await readdir(resolvedObjectPath, { withFileTypes: true });
1939
+ } catch {
1940
+ return;
1941
+ }
1942
+ const liveLeaseState = await this.liveTempLeases(
1943
+ resolvedObjectPath,
1944
+ entries,
1945
+ );
1946
+ if (liveLeaseState.reader) return;
1947
+ for (const entry of entries) {
1948
+ if (entry.isDirectory()) continue;
1949
+ const generationMatch = entry.name.match(
1950
+ /^generation-([0-9a-f-]{16,})\.(?:body|meta\.json)$/u,
1951
+ );
1952
+ if (!generationMatch) continue;
1953
+ const generation = generationMatch[1]!;
1954
+ if (generation === retainedGeneration) continue;
1955
+ const activeKey = `${keyHash(key)}:${generation}`;
1956
+ // A writer may have renamed one or both generation files while its
1957
+ // commit marker is still pending. Preserve it until that writer has
1958
+ // either committed or cleaned up its temporary files.
1959
+ if (
1960
+ this.activeGenerations.has(activeKey) ||
1961
+ this.hasGenerationLease(key, generation) ||
1962
+ this.hasKeyReadReservation(key) ||
1963
+ liveLeaseState.generations.has(generation)
1964
+ ) {
1965
+ continue;
1966
+ }
1967
+ // `retainedGeneration` and `liveLeaseState` are scan snapshots. A writer
1968
+ // may publish and release its lease while this reclamation pass is in
1969
+ // flight, so re-establish current marker + lease authority immediately
1970
+ // before every destructive unlink.
1971
+ if (!(await this.canReclaimGeneration(key, generation))) continue;
1972
+ await this.unlinkOwnedInternalFile(
1973
+ path.join(resolvedObjectPath, entry.name),
1974
+ );
1975
+ }
1976
+ // A crashed local writer can leave body/metadata/marker temp files after
1977
+ // its exact owner incarnation is proved dead. Keep only temps belonging to
1978
+ // an active/retained generation; proved-unowned artifacts are reclaimed
1979
+ // during steady-state GC as well as startup recovery.
1980
+ for (const entry of entries) {
1981
+ if (entry.isDirectory() || !entry.name.startsWith(TEMP_MARKER)) {
1982
+ continue;
1983
+ }
1984
+ if (entry.name.endsWith(`.${LEASE_SUFFIX}`)) continue;
1985
+ const tempGeneration = entry.name.match(/^tmp-([0-9a-f-]{16,})\./u)?.[1];
1986
+ if (
1987
+ tempGeneration &&
1988
+ (this.activeGenerations.has(`${keyHash(key)}:${tempGeneration}`) ||
1989
+ this.hasGenerationLease(key, tempGeneration) ||
1990
+ this.hasKeyReadReservation(key) ||
1991
+ liveLeaseState.generations.has(tempGeneration))
1992
+ ) {
1993
+ continue;
1994
+ }
1995
+ await this.unlinkOwnedInternalFile(
1996
+ path.join(resolvedObjectPath, entry.name),
310
1997
  );
311
1998
  }
312
1999
  }
313
2000
 
314
- async get(key: string): Promise<StorageObject | null> {
2001
+ async put(
2002
+ key: string,
2003
+ value: ObjectStoreBody,
2004
+ options?: ObjectStorePutOptions,
2005
+ ): Promise<void> {
2006
+ // Serialize before touching the filesystem. A metadata-shape failure must
2007
+ // leave the currently committed generation completely untouched.
2008
+ const metadataPayload = JSON.stringify({
2009
+ contentType: options?.contentType,
2010
+ httpMetadata:
2011
+ options?.contentType === undefined
2012
+ ? undefined
2013
+ : { contentType: options.contentType },
2014
+ });
315
2015
  const filePath = this.getFilePath(key);
316
2016
 
2017
+ await assertPathChainWithinBasePath(
2018
+ await this.getRealBasePath(),
2019
+ filePath,
2020
+ realpath,
2021
+ );
2022
+
2023
+ const internalObjectPath = this.getInternalObjectPath(key);
2024
+ await assertPathChainWithinBasePath(
2025
+ await this.getRealInternalStorePath(),
2026
+ internalObjectPath,
2027
+ realpath,
2028
+ );
2029
+ await mkdir(internalObjectPath, { recursive: true });
2030
+ await this.assertRealInternalDirectory(internalObjectPath);
2031
+ await this.syncDirectory(path.dirname(internalObjectPath));
2032
+
2033
+ const previousCommit = await this.readCommitRecord(key);
2034
+ const previousGeneration =
2035
+ previousCommit?.state === "committed"
2036
+ ? previousCommit.generation
2037
+ : undefined;
2038
+
2039
+ // Body and metadata are prepared as one generation. Readers only follow
2040
+ // the commit marker, which is atomically replaced after both files are
2041
+ // complete and synced. This prevents bytes from one writer pairing with
2042
+ // metadata from another writer.
2043
+ const generation = crypto.randomUUID();
2044
+ const generationPath = this.getGenerationPath(key, generation, "body");
2045
+ const generationMetaPath = this.getGenerationPath(
2046
+ key,
2047
+ generation,
2048
+ "meta.json",
2049
+ );
2050
+ const commitPath = this.getCommitPath(key);
2051
+ const bodyTempPath = path.join(
2052
+ internalObjectPath,
2053
+ `${TEMP_MARKER}${generation}.body`,
2054
+ );
2055
+ const metadataTempPath = path.join(
2056
+ internalObjectPath,
2057
+ `${TEMP_MARKER}${generation}.meta.json`,
2058
+ );
2059
+ const commitTempPath = path.join(
2060
+ internalObjectPath,
2061
+ `${TEMP_MARKER}${generation}.commit.json`,
2062
+ );
2063
+ const activeGenerationKey = `${keyHash(key)}:${generation}`;
2064
+ this.activeGenerations.add(activeGenerationKey);
2065
+ let releaseLease: TempLease | undefined;
2066
+ let bodyHandle: FileHandle | undefined;
2067
+ let metadataHandle: FileHandle | undefined;
2068
+ let commitHandle: FileHandle | undefined;
2069
+ let bodyTempCreated = false;
2070
+ let metadataTempCreated = false;
2071
+ let commitTempCreated = false;
2072
+ let bodyGenerationCreated = false;
2073
+ let metadataGenerationCreated = false;
2074
+ let commitPublished = false;
317
2075
  try {
318
- const resolvedFilePath = await this.resolveExistingPath(filePath);
319
- if (!resolvedFilePath) return null;
320
- const file = Bun.file(resolvedFilePath);
321
- if (!(await file.exists())) return null;
322
-
323
- const content = new Uint8Array(await file.arrayBuffer());
324
- const resolvedMetaPath = await this.resolveExistingPath(
325
- this.getMetaPath(key),
2076
+ releaseLease = await this.createTempLease(internalObjectPath, generation);
2077
+ bodyHandle = await open(bodyTempPath, "wx");
2078
+ bodyTempCreated = true;
2079
+ await writeValueToFile(bodyHandle, value);
2080
+ await syncAndClose(bodyHandle);
2081
+ bodyHandle = undefined;
2082
+ await rename(bodyTempPath, generationPath);
2083
+ await this.syncDirectory(internalObjectPath);
2084
+ bodyTempCreated = false;
2085
+ bodyGenerationCreated = true;
2086
+
2087
+ metadataHandle = await open(metadataTempPath, "wx");
2088
+ metadataTempCreated = true;
2089
+ await writeBufferFully(
2090
+ metadataHandle,
2091
+ new TextEncoder().encode(metadataPayload),
326
2092
  );
327
- const metadata = resolvedMetaPath
328
- ? await readMetadata(resolvedMetaPath)
329
- : {};
2093
+ await syncAndClose(metadataHandle);
2094
+ metadataHandle = undefined;
2095
+ await rename(metadataTempPath, generationMetaPath);
2096
+ await this.syncDirectory(internalObjectPath);
2097
+ metadataTempCreated = false;
2098
+ metadataGenerationCreated = true;
2099
+
2100
+ const commitPayload: CommitRecord = {
2101
+ version: 1,
2102
+ key,
2103
+ keyHash: keyHash(key),
2104
+ state: "committed",
2105
+ generation,
2106
+ };
2107
+ commitHandle = await open(commitTempPath, "wx");
2108
+ commitTempCreated = true;
2109
+ await writeBufferFully(
2110
+ commitHandle,
2111
+ new TextEncoder().encode(JSON.stringify(commitPayload)),
2112
+ );
2113
+ await syncAndClose(commitHandle);
2114
+ commitHandle = undefined;
2115
+ // Publication is permitted only while this exact lease inode/token is
2116
+ // still owned. Recovery never expires a possibly-live owner solely by
2117
+ // age, so this assertion and the marker rename form the fencing edge:
2118
+ // a dead owner cannot resume, and a live owner cannot be reclaimed.
2119
+ await releaseLease.assertOwned();
2120
+ await this.beforeCommitRename?.(commitPath);
2121
+ await rename(commitTempPath, commitPath);
2122
+ // Rename makes the complete body+metadata generation authoritative.
2123
+ // Set both flags before fsync: if syncing the containing directory
2124
+ // fails, the marker still names a valid committed generation and must
2125
+ // never be deleted by the cleanup path below.
2126
+ commitTempCreated = false;
2127
+ commitPublished = true;
2128
+ await this.syncDirectory(internalObjectPath);
2129
+ } finally {
2130
+ if (bodyHandle) await bodyHandle.close().catch(() => undefined);
2131
+ if (metadataHandle) await metadataHandle.close().catch(() => undefined);
2132
+ if (commitHandle) await commitHandle.close().catch(() => undefined);
2133
+ if (!commitPublished) {
2134
+ if (bodyTempCreated) await this.unlinkOwnedInternalFile(bodyTempPath);
2135
+ if (metadataTempCreated)
2136
+ await this.unlinkOwnedInternalFile(metadataTempPath);
2137
+ if (commitTempCreated)
2138
+ await this.unlinkOwnedInternalFile(commitTempPath);
2139
+ if (bodyGenerationCreated)
2140
+ await this.unlinkOwnedInternalFile(generationPath);
2141
+ if (metadataGenerationCreated)
2142
+ await this.unlinkOwnedInternalFile(generationMetaPath);
2143
+ }
2144
+ this.activeGenerations.delete(activeGenerationKey);
2145
+ await releaseLease?.();
2146
+ }
330
2147
 
331
- let bodyUsed = false;
2148
+ // Legacy body/sidecar files are no longer needed once the generation is
2149
+ // committed. Removing them is best-effort; the marker remains the source
2150
+ // of truth even if an operator-owned filesystem refuses cleanup.
2151
+ if (!this.isInternalStorePath(filePath)) {
2152
+ await unlink(filePath).catch(() => undefined);
2153
+ }
2154
+ const legacyMetaPath = this.getMetaPath(key);
2155
+ if (!this.isInternalStorePath(legacyMetaPath)) {
2156
+ await unlink(legacyMetaPath).catch(() => undefined);
2157
+ }
2158
+ await this.removeGenerationIfUnreferenced(key, previousGeneration);
2159
+ }
2160
+
2161
+ async get(key: string): Promise<ObjectStoreObject | null> {
2162
+ let releaseBody: (() => Promise<void>) | undefined;
2163
+ let bodyHandedOff = false;
2164
+ try {
2165
+ const resolvedObject = await this.resolveObject(key, true);
2166
+ if (!resolvedObject) return null;
2167
+ releaseBody = resolvedObject.releaseBody;
2168
+ const metadata = resolvedObject.metadata;
2169
+
2170
+ if (!resolvedObject.bodyHandle) {
2171
+ throw new Error("BunStorage object body is not open");
2172
+ }
2173
+ const byteLength = (await resolvedObject.bodyHandle.stat()).size;
2174
+ if (!Number.isSafeInteger(byteLength) || byteLength < 0) {
2175
+ throw new Error("invalid filesystem object size");
2176
+ }
2177
+ const body = createFileBodyStream(
2178
+ resolvedObject.bodyHandle,
2179
+ byteLength,
2180
+ async () => {
2181
+ await releaseBody?.();
2182
+ releaseBody = undefined;
2183
+ },
2184
+ );
2185
+ bodyHandedOff = true;
332
2186
 
333
2187
  return {
334
2188
  key,
335
- body: new ReadableStream({
336
- start(controller) {
337
- controller.enqueue(content);
338
- controller.close();
339
- },
340
- }),
341
- bodyUsed,
342
- arrayBuffer: async () => {
343
- bodyUsed = true;
344
- return content.buffer as ArrayBuffer;
345
- },
346
- text: async () => {
347
- bodyUsed = true;
348
- return new TextDecoder().decode(content);
349
- },
350
- json: async <T>() => {
351
- bodyUsed = true;
352
- return JSON.parse(new TextDecoder().decode(content)) as T;
353
- },
354
- httpMetadata: metadata.httpMetadata,
355
- customMetadata: metadata.customMetadata,
2189
+ body,
2190
+ contentType: metadata.contentType,
2191
+ byteLength,
356
2192
  };
357
2193
  } catch {
358
2194
  return null;
2195
+ } finally {
2196
+ if (!bodyHandedOff) await releaseBody?.();
359
2197
  }
360
2198
  }
361
2199
 
362
- async delete(key: string | string[]): Promise<void> {
2200
+ async delete(key: string | readonly string[]): Promise<void> {
363
2201
  const keys = Array.isArray(key) ? key : [key];
364
2202
  for (const k of keys) {
2203
+ let markerPath: string;
2204
+ let legacyPath: string;
2205
+ let legacyMetaPath: string;
365
2206
  try {
366
- const filePath = await this.resolveExistingPath(this.getFilePath(k));
367
- if (filePath) await unlink(filePath);
2207
+ markerPath = this.getCommitPath(k);
2208
+ legacyPath = this.getFilePath(k);
2209
+ legacyMetaPath = this.getMetaPath(k);
368
2210
  } catch {
369
- /* ignore */
2211
+ // Preserve the historical delete contract: an invalid/traversal key
2212
+ // is ignored rather than becoming an observable filesystem error.
2213
+ continue;
370
2214
  }
371
2215
  try {
372
- const metaPath = await this.resolveExistingPath(this.getMetaPath(k));
373
- if (metaPath) await unlink(metaPath);
2216
+ await assertPathChainWithinBasePath(
2217
+ await this.getRealInternalStorePath(),
2218
+ markerPath,
2219
+ realpath,
2220
+ );
374
2221
  } catch {
375
- /* ignore */
2222
+ // Do not create a tombstone through a symlinked path outside storage.
2223
+ continue;
2224
+ }
2225
+ const markerExists = await Bun.file(markerPath).exists();
2226
+ const legacyExists =
2227
+ !this.isInternalStorePath(legacyPath) &&
2228
+ (await Bun.file(legacyPath).exists());
2229
+ const legacyMetaExists =
2230
+ !this.isInternalStorePath(legacyMetaPath) &&
2231
+ (await Bun.file(legacyMetaPath).exists());
2232
+ if (!markerExists && !legacyExists && !legacyMetaExists) continue;
2233
+
2234
+ let previousGeneration: string | undefined;
2235
+ try {
2236
+ const previousCommit = await this.readCommitRecord(k);
2237
+ previousGeneration =
2238
+ previousCommit?.state === "committed"
2239
+ ? previousCommit.generation
2240
+ : undefined;
2241
+ } catch {
2242
+ // A malformed marker is still replaced with a tombstone below; no
2243
+ // generation is trusted for eager cleanup in that case.
376
2244
  }
377
- }
378
- }
379
-
380
- async list(options?: {
381
- prefix?: string;
382
- limit?: number;
383
- cursor?: string;
384
- delimiter?: string;
385
- }): Promise<ListObjectsResult> {
386
- const objects: ListObjectsResult["objects"] = [];
387
- const realBasePath = await this.getRealBasePath();
388
2245
 
389
- const readDirRecursive = async (dir: string, prefix: string = "") => {
2246
+ // Publish a tombstone before removing legacy files. If cleanup is
2247
+ // interrupted, the tombstone still prevents a stale legacy body from
2248
+ // resurfacing through the compatibility read path.
2249
+ await this.writeCommitRecord(k, {
2250
+ version: 1,
2251
+ state: "deleted",
2252
+ generation: null,
2253
+ });
390
2254
  try {
391
- const entries = await readdir(dir, { withFileTypes: true });
392
- for (const entry of entries) {
393
- const fullPath = `${dir}/${entry.name}`;
394
- const realFullPath = await realpath(fullPath);
395
- if (!isPathWithinBasePath(realBasePath, realFullPath)) continue;
396
- const key = prefix ? `${prefix}/${entry.name}` : entry.name;
397
-
398
- if (entry.isDirectory()) {
399
- await readDirRecursive(fullPath, key);
400
- } else if (!entry.name.endsWith(".meta.json")) {
401
- if (!options?.prefix || key.startsWith(options.prefix)) {
402
- const stats = await stat(fullPath);
403
- objects.push({
404
- key,
405
- size: stats.size,
406
- uploaded: stats.mtime,
407
- });
408
- }
409
- }
2255
+ if (!this.isInternalStorePath(legacyPath)) {
2256
+ const filePath = await this.resolveExistingPath(legacyPath);
2257
+ if (filePath) await unlink(filePath);
410
2258
  }
411
2259
  } catch {
412
- // Directory doesn't exist
2260
+ /* ignore */
413
2261
  }
414
- };
415
-
416
- await readDirRecursive(realBasePath);
417
-
418
- const limit = options?.limit ?? 1000;
419
- const truncated = objects.length > limit;
420
-
421
- return {
422
- objects: objects.slice(0, limit),
423
- truncated,
424
- cursor: truncated ? String(limit) : undefined,
425
- };
426
- }
427
-
428
- async head(key: string): Promise<ObjectMetadata | null> {
429
- const filePath = this.getFilePath(key);
430
-
431
- try {
432
- const resolvedFilePath = await this.resolveExistingPath(filePath);
433
- if (!resolvedFilePath) return null;
434
- const file = Bun.file(resolvedFilePath);
435
- if (!(await file.exists())) return null;
436
-
437
- const resolvedMetaPath = await this.resolveExistingPath(
438
- this.getMetaPath(key),
439
- );
440
- const metadata = resolvedMetaPath
441
- ? await readMetadata(resolvedMetaPath)
442
- : {};
443
- return {
444
- contentLength: file.size,
445
- httpMetadata: metadata.httpMetadata,
446
- customMetadata: metadata.customMetadata,
447
- };
448
- } catch {
449
- return null;
2262
+ try {
2263
+ if (!this.isInternalStorePath(legacyMetaPath)) {
2264
+ const metaPath = await this.resolveExistingPath(legacyMetaPath);
2265
+ if (metaPath) await unlink(metaPath);
2266
+ }
2267
+ } catch {
2268
+ /* ignore */
2269
+ }
2270
+ await this.removeGenerationIfUnreferenced(k, previousGeneration);
450
2271
  }
451
2272
  }
452
2273
  }