@pithy-sh/storage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/package.json +52 -0
- package/pithy.manifest.json +70 -0
- package/src/capability.ts +104 -0
- package/src/cloudflare-test.d.ts +14 -0
- package/src/config/config.ts +105 -0
- package/src/data/share.ts +36 -0
- package/src/data/storageObject.ts +81 -0
- package/src/data/tables.ts +37 -0
- package/src/error/errors.ts +140 -0
- package/src/http/guard.ts +32 -0
- package/src/http/handlers.ts +684 -0
- package/src/http/routes.ts +313 -0
- package/src/http/schemas.ts +214 -0
- package/src/http/serve.ts +288 -0
- package/src/index.ts +42 -0
- package/src/migrations/0001_objects.ts +86 -0
- package/src/object/cloudflare.ts +55 -0
- package/src/object/key.ts +40 -0
- package/src/object/multipart.ts +166 -0
- package/src/object/store.ts +371 -0
- package/src/provision/provisionStorage.ts +192 -0
- package/src/provision/resolveStorageConfig.ts +80 -0
- package/src/quota/quota.ts +241 -0
- package/src/secret/registry.ts +118 -0
- package/src/seeds/example.ts +120 -0
- package/src/test-utils/liveStorage.ts +261 -0
- package/src/version.generated.ts +16 -0
- package/src/workflows/retryPolicy.ts +43 -0
- package/src/workflows/specs.ts +85 -0
- package/src/workflows/sweep.ts +226 -0
- package/src/workflows/worker.ts +103 -0
- package/src/workflows/wrangler.jsonc +54 -0
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { StorageMultipartFailedError } from "../error/errors";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Part orchestration for a multipart upload: how many parts, how big, and how the ETags the client
|
|
9
|
+
* reports back are turned into a completion list R2 will accept.
|
|
10
|
+
*
|
|
11
|
+
* **Why multipart at all.** A single presigned PUT is capped at 5 GiB, and it is not resumable — a
|
|
12
|
+
* dropped connection at 4 GiB starts over. Multipart raises the ceiling and makes a lost client
|
|
13
|
+
* cheap: `GET /storage/:id/parts` re-lists what R2 holds and re-presigns the rest, so the client
|
|
14
|
+
* sends only what is missing. The server holds no transfer state beyond the pending row.
|
|
15
|
+
*
|
|
16
|
+
* **The defaults and the ceiling they imply.** Threshold 100 MiB, part size 64 MiB. R2 allows at
|
|
17
|
+
* most 10,000 parts, so 10,000 × 64 MiB is a documented practical ceiling of ~625 GiB per object.
|
|
18
|
+
* That is well short of R2's own 4.995 TiB object cap (and its 4.995 GiB single-part cap); an
|
|
19
|
+
* adopter storing objects larger than ~625 GiB raises `partSizeBytes` in `pithy.config.ts` and gets
|
|
20
|
+
* a proportionally higher ceiling. The trade is deliberate: a bigger default part size would make
|
|
21
|
+
* every ordinary upload retry more expensive to serve a size almost nobody stores.
|
|
22
|
+
*
|
|
23
|
+
* **The 5 MiB floor** applies to every part but the last, and is S3's rule, not ours — R2 rejects a
|
|
24
|
+
* completion whose non-final part is smaller. Enforcing it here means the failure lands at plan time
|
|
25
|
+
* with a config-shaped message, rather than after the client has spent bandwidth on every part.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
/** Above this many bytes an upload goes multipart. Below it, one presigned PUT is simpler and cheaper. */
|
|
29
|
+
export const DEFAULT_MULTIPART_THRESHOLD_BYTES = 100 * 1024 * 1024;
|
|
30
|
+
|
|
31
|
+
/** Default bytes per part. With R2's 10,000-part cap this sets the ~625 GiB practical object ceiling. */
|
|
32
|
+
export const DEFAULT_PART_SIZE_BYTES = 64 * 1024 * 1024;
|
|
33
|
+
|
|
34
|
+
/** S3's floor for every part but the last. R2 rejects a completion that breaks it. */
|
|
35
|
+
export const MIN_PART_SIZE_BYTES = 5 * 1024 * 1024;
|
|
36
|
+
|
|
37
|
+
/** R2's per-part ceiling — the same 5 GiB that caps a single-part upload. */
|
|
38
|
+
export const MAX_PART_SIZE_BYTES = 5 * 1024 * 1024 * 1024;
|
|
39
|
+
|
|
40
|
+
/** R2's hard cap on parts in one multipart upload. */
|
|
41
|
+
export const MAX_UPLOAD_PARTS = 10_000;
|
|
42
|
+
|
|
43
|
+
/** One planned part: which number it is, where it starts, and how many bytes it carries. */
|
|
44
|
+
export const PartPlan = z
|
|
45
|
+
.object({
|
|
46
|
+
partNumber: z.number().int().min(1).describe("The part's 1-based index. R2 requires them contiguous from 1."),
|
|
47
|
+
offset: z.number().int().nonnegative().describe("The part's first byte offset within the whole object."),
|
|
48
|
+
length: z.number().int().positive().describe("The part's byte count. Every part but the last is `partSize`."),
|
|
49
|
+
})
|
|
50
|
+
.describe("One part of a planned multipart upload — the unit a client uploads and can re-send alone.");
|
|
51
|
+
export type PartPlan = z.output<typeof PartPlan>;
|
|
52
|
+
|
|
53
|
+
/** The whole plan for one object: the part size in force and every part it decomposes into. */
|
|
54
|
+
export const MultipartPlan = z
|
|
55
|
+
.object({
|
|
56
|
+
partSize: z.number().int().min(MIN_PART_SIZE_BYTES).describe("Bytes per part, for every part but the last."),
|
|
57
|
+
partCount: z.number().int().min(1).describe("How many parts the object decomposes into."),
|
|
58
|
+
parts: z.array(PartPlan).min(1).describe("Every part, ascending by `partNumber`."),
|
|
59
|
+
})
|
|
60
|
+
.describe("How one object is split for a multipart upload — the plan a client uploads against.");
|
|
61
|
+
export type MultipartPlan = z.output<typeof MultipartPlan>;
|
|
62
|
+
|
|
63
|
+
/** One part as the client reports it back after its PUT: the number it uploaded and the ETag R2 answered with. */
|
|
64
|
+
export const ReportedPart = z
|
|
65
|
+
.object({
|
|
66
|
+
partNumber: z.number().int().min(1).max(MAX_UPLOAD_PARTS).describe("The part number this ETag belongs to."),
|
|
67
|
+
etag: z.string().min(1).describe("The `ETag` response header R2 returned for that part, verbatim."),
|
|
68
|
+
})
|
|
69
|
+
.describe("One completed part, as the client reports it back — the input to completing an upload.");
|
|
70
|
+
export type ReportedPart = z.output<typeof ReportedPart>;
|
|
71
|
+
|
|
72
|
+
/** Whether an upload of `size` bytes should go multipart under `threshold`. */
|
|
73
|
+
export function needsMultipart(size: number, threshold: number = DEFAULT_MULTIPART_THRESHOLD_BYTES): boolean {
|
|
74
|
+
return size > threshold;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Split `size` bytes into parts of `partSize`.
|
|
79
|
+
*
|
|
80
|
+
* Throws `storage/multipart_failed` when the plan cannot exist: a part size outside R2's 5 MiB–5 GiB
|
|
81
|
+
* window, or a size that needs more than 10,000 parts. Both are configuration faults surfaced at
|
|
82
|
+
* plan time — before a client has uploaded anything against a plan R2 would refuse to complete.
|
|
83
|
+
*/
|
|
84
|
+
export function planMultipart(size: number, partSize: number = DEFAULT_PART_SIZE_BYTES): MultipartPlan {
|
|
85
|
+
if (!Number.isInteger(size) || size <= 0) {
|
|
86
|
+
throw new StorageMultipartFailedError({
|
|
87
|
+
message: "An upload needs a positive size.",
|
|
88
|
+
action: "Declare the file's byte count when you start the upload.",
|
|
89
|
+
detail: `multipart plan requested for size ${size}`,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
if (!Number.isInteger(partSize) || partSize < MIN_PART_SIZE_BYTES || partSize > MAX_PART_SIZE_BYTES) {
|
|
93
|
+
throw new StorageMultipartFailedError({
|
|
94
|
+
message: "The configured part size is outside what R2 accepts.",
|
|
95
|
+
action: "Set `partSizeBytes` between 5 MiB and 5 GiB in pithy.config.ts.",
|
|
96
|
+
detail: `partSize ${partSize} is outside [${MIN_PART_SIZE_BYTES}, ${MAX_PART_SIZE_BYTES}]`,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const partCount = Math.ceil(size / partSize);
|
|
101
|
+
if (partCount > MAX_UPLOAD_PARTS) {
|
|
102
|
+
throw new StorageMultipartFailedError({
|
|
103
|
+
message: "That file is too large for the configured part size.",
|
|
104
|
+
action: `Raise \`partSizeBytes\` — R2 allows at most ${MAX_UPLOAD_PARTS} parts per upload.`,
|
|
105
|
+
detail: `size ${size} at partSize ${partSize} needs ${partCount} parts`,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const parts: PartPlan[] = [];
|
|
110
|
+
for (let index = 0; index < partCount; index += 1) {
|
|
111
|
+
const offset = index * partSize;
|
|
112
|
+
// Only the final part may be short — that is the whole content of S3's 5 MiB floor.
|
|
113
|
+
parts.push({ partNumber: index + 1, offset, length: Math.min(partSize, size - offset) });
|
|
114
|
+
}
|
|
115
|
+
return { partSize, partCount, parts };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Turn the parts a client reports into the list R2's completion call takes: validated, deduplicated,
|
|
120
|
+
* and ascending.
|
|
121
|
+
*
|
|
122
|
+
* Sorting is not cosmetic — S3 rejects an out-of-order part list, and the order concurrent PUTs
|
|
123
|
+
* finish in is not the order the parts belong in. A gap, a duplicate, or a count that disagrees with
|
|
124
|
+
* the plan is refused here rather than sent to R2, because R2's own rejection arrives as an opaque
|
|
125
|
+
* SDK fault with nothing in it a client could act on.
|
|
126
|
+
*/
|
|
127
|
+
export function collectParts(reported: readonly ReportedPart[], expectedCount: number): ReportedPart[] {
|
|
128
|
+
const parsed = z.array(ReportedPart).safeParse(reported);
|
|
129
|
+
if (!parsed.success) {
|
|
130
|
+
throw new StorageMultipartFailedError({
|
|
131
|
+
detail: `reported parts failed validation: ${parsed.error.issues.map((i) => i.code).join(", ")}`,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const byNumber = new Map<number, string>();
|
|
136
|
+
for (const part of parsed.data) {
|
|
137
|
+
const existing = byNumber.get(part.partNumber);
|
|
138
|
+
if (existing !== undefined && existing !== part.etag) {
|
|
139
|
+
throw new StorageMultipartFailedError({
|
|
140
|
+
message: "Part list is inconsistent.",
|
|
141
|
+
detail: `part ${part.partNumber} reported twice with different etags`,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
byNumber.set(part.partNumber, part.etag);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
if (byNumber.size !== expectedCount) {
|
|
148
|
+
throw new StorageMultipartFailedError({
|
|
149
|
+
message: "The upload is missing parts. GET /storage/<id>/parts, then re-send the parts it reports missing.",
|
|
150
|
+
detail: `expected ${expectedCount} distinct parts, got ${byNumber.size}`,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
for (let partNumber = 1; partNumber <= expectedCount; partNumber += 1) {
|
|
155
|
+
if (!byNumber.has(partNumber)) {
|
|
156
|
+
throw new StorageMultipartFailedError({
|
|
157
|
+
message: "The upload is missing parts. GET /storage/<id>/parts, then re-send the parts it reports missing.",
|
|
158
|
+
detail: `part ${partNumber} was never reported`,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
return [...byNumber.entries()]
|
|
164
|
+
.map(([partNumber, etag]) => ({ partNumber, etag }))
|
|
165
|
+
.sort((a, b) => a.partNumber - b.partNumber);
|
|
166
|
+
}
|
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { R2Bucket, R2Conditional, R2Object, R2Range, ReadableStream } from "@cloudflare/workers-types";
|
|
5
|
+
import type { SecretsStoreEnv } from "@pithy-sh/secrets/src/env/bindings";
|
|
6
|
+
import { sharedSecretsStore } from "@pithy-sh/secrets/src/sharedSecretsStore";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
import { type R2StorageCredentials, r2CredentialsRegistry, STORAGE_R2_SECRET } from "../secret/registry";
|
|
9
|
+
import { r2Presigned } from "./cloudflare";
|
|
10
|
+
import type { ReportedPart } from "./multipart";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* `ObjectStore` — the object-plane seam, and the one piece of this package that is reusable outside it.
|
|
14
|
+
*
|
|
15
|
+
* **The load-bearing constraint: this module imports no D1, no routes, and no storage config.** It is
|
|
16
|
+
* constructed over an injected bucket binding and a *named* credential secret, which is exactly what
|
|
17
|
+
* lets `@pithy-sh/media` import it, point it at `MEDIA_BUCKET` under `media-r2-credentials`, and
|
|
18
|
+
* inherit none of storage's tables, routes, or opaque-key policy. Adding `@pithy-sh/storage` to media
|
|
19
|
+
* must not mount storage's routes or create `pithy_storage_objects`; keeping this file free of those
|
|
20
|
+
* imports is what guarantees it.
|
|
21
|
+
*
|
|
22
|
+
* It is **mechanism only**. It takes an explicit key and moves bytes. Key policy lives in `key.ts` and
|
|
23
|
+
* in the handlers: storage derives `obj/<uuid>`, media passes `media/<type>/<id>`, and neither knows
|
|
24
|
+
* the other's scheme.
|
|
25
|
+
*
|
|
26
|
+
* **Bindings inside the Worker, S3 outside it** (CLAUDE.md §Cloudflare access). Reads, head, list and
|
|
27
|
+
* delete go through the `R2Bucket` binding — no credentials, no round trip, and a body that streams.
|
|
28
|
+
* Presigned URLs, the multipart lifecycle, and server-side copy have no binding equivalent that keeps
|
|
29
|
+
* bytes out of the Worker, so they go over the S3 protocol through `@pithy-sh/cloudflare`. The seam
|
|
30
|
+
* hides which is which; only `object/cloudflare.ts` touches the manager.
|
|
31
|
+
*
|
|
32
|
+
* Every response crossing the seam is Zod-validated, so an unexpected shape fails loudly at the
|
|
33
|
+
* boundary rather than surfacing as `undefined` three layers up.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/** An object's metadata, as either a HEAD or a GET reports it. */
|
|
37
|
+
export const ObjectMetadata = z
|
|
38
|
+
.object({
|
|
39
|
+
key: z.string().min(1).describe("The R2 object key. Opaque to the caller — policy lives in `key.ts`."),
|
|
40
|
+
size: z.number().int().nonnegative().describe("The whole object's size in bytes, not the served range's."),
|
|
41
|
+
etag: z
|
|
42
|
+
.string()
|
|
43
|
+
.min(1)
|
|
44
|
+
.describe("R2's entity tag for this object version, unquoted. Quote it before it becomes an `ETag` header."),
|
|
45
|
+
contentType: z
|
|
46
|
+
.string()
|
|
47
|
+
.min(1)
|
|
48
|
+
.optional()
|
|
49
|
+
.describe("The stored `Content-Type`. Absent when the object was written without one."),
|
|
50
|
+
contentDisposition: z
|
|
51
|
+
.string()
|
|
52
|
+
.min(1)
|
|
53
|
+
.optional()
|
|
54
|
+
.describe(
|
|
55
|
+
"The stored `Content-Disposition`, when one was written. Recorded, not honored — the serve path derives its own, because this string is whatever the uploader sent.",
|
|
56
|
+
),
|
|
57
|
+
uploaded: z.date().describe("When R2 wrote this object version."),
|
|
58
|
+
checksumSha256: z
|
|
59
|
+
.string()
|
|
60
|
+
.regex(/^[0-9a-f]{64}$/)
|
|
61
|
+
.optional()
|
|
62
|
+
.describe("Lowercase hex SHA-256, when R2 holds one. Absent unless the upload supplied or requested it."),
|
|
63
|
+
})
|
|
64
|
+
.describe("One object's metadata — everything a serve path needs without transferring the body.");
|
|
65
|
+
export type ObjectMetadata = z.output<typeof ObjectMetadata>;
|
|
66
|
+
|
|
67
|
+
/** Which bytes of an object to read. Mirrors R2's own range shapes; exactly one form is meaningful. */
|
|
68
|
+
export const ObjectRange = z
|
|
69
|
+
.object({
|
|
70
|
+
offset: z.number().int().nonnegative().optional().describe("First byte to read, from the start of the object."),
|
|
71
|
+
length: z.number().int().positive().optional().describe("How many bytes to read from `offset`."),
|
|
72
|
+
suffix: z.number().int().positive().optional().describe("Read this many bytes from the *end* — `bytes=-N`."),
|
|
73
|
+
})
|
|
74
|
+
.describe("A byte range to read, as an HTTP `Range` header maps onto R2.");
|
|
75
|
+
export type ObjectRange = z.output<typeof ObjectRange>;
|
|
76
|
+
|
|
77
|
+
/** Preconditions a read must satisfy, so a conditional request can answer 304 without moving bytes. */
|
|
78
|
+
export const ObjectConditions = z
|
|
79
|
+
.object({
|
|
80
|
+
etagMatches: z.string().min(1).optional().describe("Read only if the object's etag equals this (`If-Match`)."),
|
|
81
|
+
etagDoesNotMatch: z
|
|
82
|
+
.string()
|
|
83
|
+
.min(1)
|
|
84
|
+
.optional()
|
|
85
|
+
.describe("Read only if the object's etag differs (`If-None-Match`) — the 304 path."),
|
|
86
|
+
uploadedBefore: z
|
|
87
|
+
.date()
|
|
88
|
+
.optional()
|
|
89
|
+
.describe("Read only if the object was written before this (`If-Unmodified-Since`)."),
|
|
90
|
+
uploadedAfter: z
|
|
91
|
+
.date()
|
|
92
|
+
.optional()
|
|
93
|
+
.describe("Read only if the object was written after this (`If-Modified-Since`)."),
|
|
94
|
+
})
|
|
95
|
+
.describe("Conditional-read preconditions. R2 returns metadata with no body when one fails.");
|
|
96
|
+
export type ObjectConditions = z.output<typeof ObjectConditions>;
|
|
97
|
+
|
|
98
|
+
/** One page of a bucket listing — the sweep's view, distinct from the D1 listing an owner sees. */
|
|
99
|
+
export const ObjectListing = z
|
|
100
|
+
.object({
|
|
101
|
+
objects: z.array(ObjectMetadata).describe("This page's objects, in R2's lexicographic key order."),
|
|
102
|
+
cursor: z
|
|
103
|
+
.string()
|
|
104
|
+
.min(1)
|
|
105
|
+
.optional()
|
|
106
|
+
.describe("Continuation cursor for the next page. Absent means this page was the last."),
|
|
107
|
+
})
|
|
108
|
+
.describe("One page of a bucket listing. Pagination is caller-driven — pass `cursor` back to advance.");
|
|
109
|
+
export type ObjectListing = z.output<typeof ObjectListing>;
|
|
110
|
+
|
|
111
|
+
/** One part already stored against an in-flight multipart upload — what makes an upload resumable. */
|
|
112
|
+
export const UploadedPart = z
|
|
113
|
+
.object({
|
|
114
|
+
partNumber: z.number().int().min(1).describe("The part's 1-based index within the upload."),
|
|
115
|
+
etag: z.string().min(1).describe("The part's entity tag, verbatim. Pass it back unchanged to complete."),
|
|
116
|
+
size: z.number().int().nonnegative().describe("The part's size in bytes, as R2 stored it."),
|
|
117
|
+
})
|
|
118
|
+
.describe("One uploaded part — the unit a resumed upload skips re-sending.");
|
|
119
|
+
export type UploadedPart = z.output<typeof UploadedPart>;
|
|
120
|
+
|
|
121
|
+
/** How long a presigned URL stays valid. */
|
|
122
|
+
export interface PresignOptions {
|
|
123
|
+
/** Lifetime in seconds. Defaults to one hour — long enough for a large part, short enough to leak little. */
|
|
124
|
+
expiresIn?: number;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Presign options for one part, which may additionally pin its exact byte count. */
|
|
128
|
+
export interface PresignPartOptions extends PresignOptions {
|
|
129
|
+
/** Exact byte count the client must send. Omitted by default; see `presignPart`. */
|
|
130
|
+
contentLength?: number;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** What to read, and under what preconditions. */
|
|
134
|
+
export interface GetOptions {
|
|
135
|
+
/** Serve only this byte range (206). */
|
|
136
|
+
range?: ObjectRange;
|
|
137
|
+
/** Serve only if these preconditions hold; otherwise the result carries metadata and no body (304). */
|
|
138
|
+
onlyIf?: ObjectConditions;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Which slice of the bucket to list. */
|
|
142
|
+
export interface ListOptions {
|
|
143
|
+
/** Only keys starting with this prefix. Omitted lists the whole bucket. */
|
|
144
|
+
prefix?: string;
|
|
145
|
+
/** Continuation cursor from a previous page. */
|
|
146
|
+
cursor?: string;
|
|
147
|
+
/** Page size. R2 caps this at 1,000. */
|
|
148
|
+
limit?: number;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** An object read: its metadata, its bytes, and the range R2 actually served. */
|
|
152
|
+
export interface ObjectBody {
|
|
153
|
+
/** The whole object's metadata — `size` is the object's, not the range's. */
|
|
154
|
+
metadata: ObjectMetadata;
|
|
155
|
+
/** The bytes, streamed. `null` when a precondition failed, which is the 304 signal. */
|
|
156
|
+
body: ReadableStream | null;
|
|
157
|
+
/** The byte range R2 served, when a range was asked for. `null` means the whole object. */
|
|
158
|
+
range: { offset: number; length: number } | null;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The S3-protocol half of the seam: everything the R2 binding cannot do without pulling bytes through
|
|
163
|
+
* the Worker. Declared here as a structural port so `store.ts` never imports `@pithy-sh/cloudflare`,
|
|
164
|
+
* and so a test can drive the store with no SDK and no network.
|
|
165
|
+
*/
|
|
166
|
+
export interface PresignedObjects {
|
|
167
|
+
/**
|
|
168
|
+
* Presign a PUT for one whole object. The length is signed, so the client must send exactly that many
|
|
169
|
+
* bytes. The type is **not** — S3 presigning marks `content-type` unsignable — so it is a hint the
|
|
170
|
+
* completion reconciles against what R2 actually stored.
|
|
171
|
+
*/
|
|
172
|
+
presignPut(key: string, contentType: string, contentLength: number, options?: PresignOptions): Promise<string>;
|
|
173
|
+
/** Presign a GET for one object. */
|
|
174
|
+
presignGet(key: string, options?: PresignOptions): Promise<string>;
|
|
175
|
+
/** Open a multipart upload; returns the `uploadId` every later step is addressed by. */
|
|
176
|
+
createMultipartUpload(key: string, contentType: string): Promise<string>;
|
|
177
|
+
/** Presign a PUT for one part — the only multipart step a client ever touches. */
|
|
178
|
+
presignUploadPart(key: string, uploadId: string, partNumber: number, options?: PresignPartOptions): Promise<string>;
|
|
179
|
+
/** Assemble the uploaded parts into one object. */
|
|
180
|
+
completeMultipartUpload(key: string, uploadId: string, parts: readonly ReportedPart[]): Promise<void>;
|
|
181
|
+
/** Discard an in-flight upload and its stored parts. Idempotent. */
|
|
182
|
+
abortMultipartUpload(key: string, uploadId: string): Promise<void>;
|
|
183
|
+
/** List the parts already stored against an in-flight upload, ascending. */
|
|
184
|
+
listParts(key: string, uploadId: string): Promise<UploadedPart[]>;
|
|
185
|
+
/** Copy an object within the bucket, server-side — the bytes never leave R2. */
|
|
186
|
+
copyObject(sourceKey: string, destinationKey: string): Promise<void>;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** The object plane: presigned transfer, the multipart lifecycle, and binding-backed reads. */
|
|
190
|
+
export interface ObjectStore {
|
|
191
|
+
/** Presign a PUT the client uploads a whole object to. Bytes never proxy through the Worker. */
|
|
192
|
+
presignPut(key: string, contentType: string, contentLength: number, options?: PresignOptions): Promise<string>;
|
|
193
|
+
/** Presign a GET — the escape hatch for serving without a Worker in the byte path. */
|
|
194
|
+
presignGet(key: string, options?: PresignOptions): Promise<string>;
|
|
195
|
+
/** Open a multipart upload. Persist the returned `uploadId`: it is the handle a resume needs. */
|
|
196
|
+
initMultipart(key: string, contentType: string): Promise<string>;
|
|
197
|
+
/**
|
|
198
|
+
* Presign a PUT for one part. `contentLength` is **not** signed by default: the final part's length
|
|
199
|
+
* differs from every other, so signing it would mean knowing the total size at mint time. R2 enforces
|
|
200
|
+
* the 5 MiB floor and 5 GiB ceiling itself, and the object's real size is confirmed at completion.
|
|
201
|
+
*/
|
|
202
|
+
presignPart(key: string, uploadId: string, partNumber: number, options?: PresignPartOptions): Promise<string>;
|
|
203
|
+
/** Assemble the reported parts into one object. Order and completeness are checked before R2 sees them. */
|
|
204
|
+
completeMultipart(key: string, uploadId: string, parts: readonly ReportedPart[]): Promise<void>;
|
|
205
|
+
/** Abort an in-flight upload. Idempotent, so a sweep or a retried teardown can re-run. */
|
|
206
|
+
abortMultipart(key: string, uploadId: string): Promise<void>;
|
|
207
|
+
/** The parts already stored against an in-flight upload — what a resuming client asks for. */
|
|
208
|
+
listParts(key: string, uploadId: string): Promise<UploadedPart[]>;
|
|
209
|
+
/** Read an object through the binding, honoring a range and conditional preconditions. `null` when absent. */
|
|
210
|
+
get(key: string, options?: GetOptions): Promise<ObjectBody | null>;
|
|
211
|
+
/** Read an object's metadata without its body. `null` when absent — a missing object is an answer. */
|
|
212
|
+
head(key: string): Promise<ObjectMetadata | null>;
|
|
213
|
+
/** One page of bucket keys. The orphan sweep's view; an owner's file list comes from D1. */
|
|
214
|
+
list(options?: ListOptions): Promise<ObjectListing>;
|
|
215
|
+
/** Server-side copy within the bucket. */
|
|
216
|
+
copy(sourceKey: string, destinationKey: string): Promise<void>;
|
|
217
|
+
/** Delete one object. Idempotent by protocol. */
|
|
218
|
+
delete(key: string): Promise<void>;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** How to build an object store. */
|
|
222
|
+
export interface ObjectStoreOptions {
|
|
223
|
+
/** The R2 bucket binding — `STORAGE_BUCKET` for storage, `MEDIA_BUCKET` for media. */
|
|
224
|
+
bucket: R2Bucket;
|
|
225
|
+
/**
|
|
226
|
+
* The env the credential secret resolves against. Only ever handed to `sharedSecretsStore`; no
|
|
227
|
+
* binding on it is read directly (CLAUDE.md §Secrets).
|
|
228
|
+
*/
|
|
229
|
+
env: SecretsStoreEnv;
|
|
230
|
+
/**
|
|
231
|
+
* The registry name the R2 credential bundle is stored under. Whatever name is passed, the *same*
|
|
232
|
+
* capability must have declared it with `r2CredentialsRegistry(name)` — a name no capability
|
|
233
|
+
* declared is absent from the aggregated registry and throws on first use.
|
|
234
|
+
*/
|
|
235
|
+
secretName?: string;
|
|
236
|
+
/**
|
|
237
|
+
* Test seam: build the S3 port from resolved credentials. Defaults to the real R2 adapter. Overriding
|
|
238
|
+
* it is how a test drives the store with no SDK, no credentials, and no network.
|
|
239
|
+
*/
|
|
240
|
+
presigned?: (credentials: R2StorageCredentials) => PresignedObjects;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Map our range shape onto R2's — the binding's union admits exactly one of these three forms. */
|
|
244
|
+
function toR2Range(range: ObjectRange | undefined): R2Range | undefined {
|
|
245
|
+
if (!range) return undefined;
|
|
246
|
+
if (range.suffix !== undefined) return { suffix: range.suffix };
|
|
247
|
+
if (range.offset !== undefined && range.length !== undefined) return { offset: range.offset, length: range.length };
|
|
248
|
+
if (range.offset !== undefined) return { offset: range.offset };
|
|
249
|
+
if (range.length !== undefined) return { length: range.length };
|
|
250
|
+
return undefined;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Lowercase hex for a checksum R2 hands back as raw bytes. */
|
|
254
|
+
function toHex(buffer: ArrayBuffer): string {
|
|
255
|
+
return [...new Uint8Array(buffer)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Validate an `R2Object` into {@link ObjectMetadata} — the one place binding shapes cross the seam. */
|
|
259
|
+
function toMetadata(object: R2Object): ObjectMetadata {
|
|
260
|
+
return ObjectMetadata.parse({
|
|
261
|
+
key: object.key,
|
|
262
|
+
size: object.size,
|
|
263
|
+
etag: object.etag,
|
|
264
|
+
contentType: object.httpMetadata?.contentType,
|
|
265
|
+
contentDisposition: object.httpMetadata?.contentDisposition,
|
|
266
|
+
uploaded: object.uploaded,
|
|
267
|
+
checksumSha256: object.checksums?.sha256 ? toHex(object.checksums.sha256) : undefined,
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Build an object store over a bucket binding and a named credential secret.
|
|
273
|
+
*
|
|
274
|
+
* Credentials resolve **lazily, once**: a read-only worker that never presigns never touches the
|
|
275
|
+
* secrets store, and a worker that presigns twice resolves once. The promise itself is memoized, so
|
|
276
|
+
* two concurrent presigns share one resolution rather than racing two.
|
|
277
|
+
*/
|
|
278
|
+
export function objectStore(options: ObjectStoreOptions): ObjectStore {
|
|
279
|
+
const secretName = options.secretName ?? STORAGE_R2_SECRET;
|
|
280
|
+
const build = options.presigned ?? r2Presigned;
|
|
281
|
+
const bucket = options.bucket;
|
|
282
|
+
let presignedPromise: Promise<PresignedObjects> | null = null;
|
|
283
|
+
|
|
284
|
+
function presigned(): Promise<PresignedObjects> {
|
|
285
|
+
if (!presignedPromise) {
|
|
286
|
+
presignedPromise = (async () => {
|
|
287
|
+
const registry = r2CredentialsRegistry(secretName);
|
|
288
|
+
const store = await sharedSecretsStore(options.env, registry);
|
|
289
|
+
return build(store.get(secretName));
|
|
290
|
+
})();
|
|
291
|
+
}
|
|
292
|
+
return presignedPromise;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
return {
|
|
296
|
+
async presignPut(key, contentType, contentLength, presignOptions) {
|
|
297
|
+
return (await presigned()).presignPut(key, contentType, contentLength, presignOptions);
|
|
298
|
+
},
|
|
299
|
+
|
|
300
|
+
async presignGet(key, presignOptions) {
|
|
301
|
+
return (await presigned()).presignGet(key, presignOptions);
|
|
302
|
+
},
|
|
303
|
+
|
|
304
|
+
async initMultipart(key, contentType) {
|
|
305
|
+
return (await presigned()).createMultipartUpload(key, contentType);
|
|
306
|
+
},
|
|
307
|
+
|
|
308
|
+
async presignPart(key, uploadId, partNumber, presignOptions) {
|
|
309
|
+
return (await presigned()).presignUploadPart(key, uploadId, partNumber, presignOptions);
|
|
310
|
+
},
|
|
311
|
+
|
|
312
|
+
async completeMultipart(key, uploadId, parts) {
|
|
313
|
+
await (await presigned()).completeMultipartUpload(key, uploadId, parts);
|
|
314
|
+
},
|
|
315
|
+
|
|
316
|
+
async abortMultipart(key, uploadId) {
|
|
317
|
+
await (await presigned()).abortMultipartUpload(key, uploadId);
|
|
318
|
+
},
|
|
319
|
+
|
|
320
|
+
async listParts(key, uploadId) {
|
|
321
|
+
return (await presigned()).listParts(key, uploadId);
|
|
322
|
+
},
|
|
323
|
+
|
|
324
|
+
async get(key, getOptions) {
|
|
325
|
+
const conditions = getOptions?.onlyIf;
|
|
326
|
+
const object = await bucket.get(key, {
|
|
327
|
+
range: toR2Range(getOptions?.range),
|
|
328
|
+
onlyIf: conditions ? (conditions as R2Conditional) : undefined,
|
|
329
|
+
});
|
|
330
|
+
if (!object) return null;
|
|
331
|
+
// A failed precondition yields an `R2Object` with no `body` — that absence *is* the 304 signal,
|
|
332
|
+
// so it is surfaced as a null body rather than swallowed as a miss.
|
|
333
|
+
const body = "body" in object ? object.body : null;
|
|
334
|
+
// R2 reports a range on *every* read, including a whole-object one. Only echo it when a range was
|
|
335
|
+
// asked for, so `range: null` keeps meaning "the whole object" and a serve path can key `206` off it.
|
|
336
|
+
const served = getOptions?.range && object.range ? object.range : undefined;
|
|
337
|
+
const offset = served && "offset" in served ? (served.offset ?? 0) : 0;
|
|
338
|
+
const length = served && "length" in served ? (served.length ?? 0) : 0;
|
|
339
|
+
return {
|
|
340
|
+
metadata: toMetadata(object),
|
|
341
|
+
body,
|
|
342
|
+
range: served ? { offset, length } : null,
|
|
343
|
+
};
|
|
344
|
+
},
|
|
345
|
+
|
|
346
|
+
async head(key) {
|
|
347
|
+
const object = await bucket.head(key);
|
|
348
|
+
return object ? toMetadata(object) : null;
|
|
349
|
+
},
|
|
350
|
+
|
|
351
|
+
async list(listOptions) {
|
|
352
|
+
const page = await bucket.list({
|
|
353
|
+
prefix: listOptions?.prefix,
|
|
354
|
+
cursor: listOptions?.cursor,
|
|
355
|
+
limit: listOptions?.limit,
|
|
356
|
+
});
|
|
357
|
+
return ObjectListing.parse({
|
|
358
|
+
objects: page.objects.map(toMetadata),
|
|
359
|
+
cursor: page.truncated ? page.cursor : undefined,
|
|
360
|
+
});
|
|
361
|
+
},
|
|
362
|
+
|
|
363
|
+
async copy(sourceKey, destinationKey) {
|
|
364
|
+
await (await presigned()).copyObject(sourceKey, destinationKey);
|
|
365
|
+
},
|
|
366
|
+
|
|
367
|
+
async delete(key) {
|
|
368
|
+
await bucket.delete(key);
|
|
369
|
+
},
|
|
370
|
+
};
|
|
371
|
+
}
|