@ultimat3/storage 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +27 -47
- package/README.md +130 -12
- package/package.json +12 -2
- package/src/accept-part.ts +30 -0
- package/src/accept.ts +7 -0
- package/src/attachment.ts +99 -6
- package/src/byte-pump.ts +46 -0
- package/src/driver-local-list.ts +75 -0
- package/src/driver-local-multipart.ts +133 -0
- package/src/driver-local-read.ts +13 -0
- package/src/driver-local-remove.ts +39 -0
- package/src/driver-local-stream.ts +104 -0
- package/src/driver-local-write.ts +28 -5
- package/src/driver-local.ts +257 -283
- package/src/driver-memory-multipart.ts +41 -0
- package/src/driver-memory.ts +205 -135
- package/src/driver-s3-client.ts +9 -0
- package/src/driver-s3-copy.ts +35 -0
- package/src/driver-s3-lock.ts +9 -2
- package/src/driver-s3-multipart-session.ts +87 -0
- package/src/driver-s3-multipart.ts +177 -0
- package/src/driver-s3-presign.ts +52 -0
- package/src/driver-s3-put-stream.ts +97 -0
- package/src/driver-s3-range.ts +99 -0
- package/src/driver-s3-settle.ts +150 -0
- package/src/driver-s3-signed.ts +69 -4
- package/src/driver-s3-sse.ts +29 -0
- package/src/driver-s3.ts +113 -65
- package/src/driver.ts +98 -38
- package/src/errors-multipart.ts +35 -0
- package/src/errors-promote.ts +14 -0
- package/src/errors-range.ts +47 -0
- package/src/errors-sniff.ts +35 -0
- package/src/errors-stream.ts +62 -0
- package/src/errors.ts +16 -1
- package/src/expire.ts +169 -0
- package/src/grant-multipart.ts +196 -0
- package/src/grant.ts +42 -25
- package/src/image-warm.ts +84 -0
- package/src/image.ts +57 -0
- package/src/index.ts +81 -2
- package/src/multipart-staged.ts +127 -0
- package/src/multipart.ts +119 -0
- package/src/put-stream.ts +52 -0
- package/src/range.ts +102 -0
- package/src/signed-url.ts +59 -2
- package/src/subject-target.ts +82 -0
- package/src/upload-client-multipart.ts +156 -0
- package/src/upload-client.ts +21 -0
- package/src/upload.ts +107 -21
package/CLAUDE.md
CHANGED
|
@@ -38,8 +38,15 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads.
|
|
|
38
38
|
|
|
39
39
|
| File | Owns |
|
|
40
40
|
|---|---|
|
|
41
|
-
| `driver.ts` | `StorageDriver` contract (
|
|
42
|
-
| `
|
|
41
|
+
| `driver.ts` | `StorageDriver` contract (11 methods; `registerAs`/`verifySigned`/`retentionOf` optional), `assertPutOptions`, + bounded `toBytes`/`sha256Base64`/`etagOf` |
|
|
42
|
+
| `byte-pump.ts` / `put-stream.ts` | the one counting read, `put()`'s and `putStream`'s; the `putStream` screen and limit |
|
|
43
|
+
| `multipart.ts` / `driver-s3-multipart*.ts` / `driver-s3-put-stream.ts` | part arithmetic + `MultipartCapability`; the six signed calls + s3's capability; the pump |
|
|
44
|
+
| `grant-multipart.ts` / `upload-client-multipart.ts` / `accept-part.ts` / `multipart-staged.ts` | uploads in parts: server calls, browser, local part PUT, local/memory sessions |
|
|
45
|
+
| `expire.ts` | `expireObjects`, `abortStaleUploads`, `objectPurgeTarget` |
|
|
46
|
+
| `driver-local-stream.ts` / `driver-s3-sse.ts` / `errors-stream.ts` | the staged local stream; the s3 SSE refusal; the stream codes |
|
|
47
|
+
| `driver-s3-settle.ts` | a lost Complete/PUT answer, settled by a stat (size + S3's etag of the bytes sent: MD5, or MD5-of-part-MD5s `-N`), then for multipart by an abort (204 = was open, nothing stored; 404 = completed, stat again). Neither settles: `X_STORAGE_WRITE_OUTCOME_UNKNOWN`, never `X_STORAGE_PUT_FAILED`. Plain `put()` settles the same way (`settleLostPut`); `isAnswered` (coded, or Bun's `S3Error`) is what counts as the provider answering |
|
|
48
|
+
| `range.ts` | `get(key, { range })`: screen + `servedRange` (`errors-range.ts`); s3's one-GET read is `driver-s3-range.ts` |
|
|
49
|
+
| `driver-local.ts` | dev default over `Bun.file`, `.meta/` sidecars; listing is `driver-local-list.ts` (`Bun.Glob`, cursor, `listFailed`), removal `driver-local-remove.ts` (a refused unlink vs one already gone) |
|
|
43
50
|
| `driver-local-write.ts` | the local WRITE: staged under `.meta/.tmp/`, renamed marker → sidecar → bytes, `X_STORAGE_KEY_CONFLICT` / `X_STORAGE_PUT_FAILED`, the per-key queue |
|
|
44
51
|
| `driver-local-read.ts` | `headObject`: sidecar + pending marker → trusted, re-checked, or untyped |
|
|
45
52
|
| `driver-local-sidecar.ts` | the sidecar shape and its one parser |
|
|
@@ -50,6 +57,7 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads.
|
|
|
50
57
|
| `driver-s3-client.ts` | `S3*Like` + `buildClient`, lazy (import never opens a socket) |
|
|
51
58
|
| `driver-s3-signed.ts` | signed requests: Bun's addressing restated, `requireEnv`, `fetch`, refusal as `S3Error` fields |
|
|
52
59
|
| `driver-s3-lock.ts` | the signed PUT's headers; `?retention` / `?legal-hold` |
|
|
60
|
+
| `driver-s3-copy.ts` | `copy(…, { ifMatch })`: a signed CopyObject |
|
|
53
61
|
| `object-lock.ts` | lock types, the `put()` screen, `isLocked`, the local/memory refusals |
|
|
54
62
|
| `driver-s3-region.ts` | a provider's wrong-region refusal → `X_CONFIG_INVALID` whose fix names `S3_REGION` |
|
|
55
63
|
| `path.ts` | key validation + `META_DIR` + `scopedKey`/`isWithinOrg`/`isTenantScoped` tenant boundary |
|
|
@@ -59,11 +67,14 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads.
|
|
|
59
67
|
| `image.ts` | deterministic variant keys; byte path over core's pipeline (png/jpeg encode only) |
|
|
60
68
|
| | `VARIANT_FORMATS` — what a variant KEY can carry — and NOT `IMAGE_FORMATS`, which is core's and means what core can PROBE. See below |
|
|
61
69
|
| | `variantKey` is the cache identity `@ultimat3/cli`'s `/media/*` route looks a variant up by — derived, never stored, so a request that misses transforms once and every later one is a disk read |
|
|
70
|
+
| `image-warm.ts` | `warmVariants`: a stored image's variants written ahead of `/media`, under its keys, the revision off the bytes read; core's spec, so a warmed variant is the one a miss would store |
|
|
62
71
|
| `storage.ts` | `defineStorage` + module-level `storage()` / `disk()` |
|
|
63
72
|
| `grant.ts` | `grantUpload` — the ONE way a presigned PUT is minted; the client never names a key |
|
|
64
73
|
| `accept.ts` | `acceptSignedUpload` / `readSignedObject` — the two decisions a `/_storage` route is made of |
|
|
65
|
-
| `attachment.ts` | `pending/` → row promotion, the `quarantine/` segment, and `
|
|
74
|
+
| `attachment.ts` | `pending/` → row promotion, the `quarantine/` segment, `sweepOrphans` and `deleteVariants` |
|
|
75
|
+
| `errors-sniff.ts` | the two `X_STORAGE_TYPE_REJECTED` refusals only a bounded sniff reaches: too much leading whitespace, XML past the window |
|
|
66
76
|
| `upload-client.ts` | the browser half: grant → PUT with progress → key. No `Bun.*`, no `node:` |
|
|
77
|
+
| `subject-target.ts` | `storageSubjectTarget(disk, keysOf)` — structural `SubjectTarget`: the app names the keys, export lists, erase deletes |
|
|
67
78
|
|
|
68
79
|
```bash
|
|
69
80
|
bun test # from packages/storage
|
|
@@ -108,11 +119,7 @@ Gotchas:
|
|
|
108
119
|
Without `dot`, dot-prefixed keys were missing from the listing while `put`/`get` handled them —
|
|
109
120
|
and `sweepOrphans` pages through `list()`. Pinned in `driver-parity.test.ts`.
|
|
110
121
|
- **A `list({ limit })` is a positive integer or a refusal** — `resolveListLimit` at the
|
|
111
|
-
`ListOptions` seam, so both disks answer one way.
|
|
112
|
-
disk and then drop its own `truncated` flag (`truncated && last !== undefined`, with an empty
|
|
113
|
-
page), so a paging caller read "complete, and there is nothing here" over a full disk, while the
|
|
114
|
-
s3 disk handed `maxKeys: 0` to the provider. Core's `assert` (`X_INVARIANT`), for the reason
|
|
115
|
-
`@ultimat3/seo`'s `chunk()` uses it: a bound with no code of its own is still a coded refusal.
|
|
122
|
+
`ListOptions` seam, so both disks answer one way. Core's `assert` (`X_INVARIANT`).
|
|
116
123
|
- **`list()` answers core's `Page<StorageListEntry>`, and no storage-local page type exists**
|
|
117
124
|
(`As of 25.0.0`; `ListPage` is deleted). Every driver builds it with core's `pageOf`, so
|
|
118
125
|
`hasMore` is derived from the cursor and cannot disagree with it. The cursor is minted only when
|
|
@@ -135,13 +142,11 @@ Gotchas:
|
|
|
135
142
|
S3 accepts both (delete marker, new version; the locked version stays). Pinned divergence: one
|
|
136
143
|
version per key. A pair in doubt never unlocks. `NoSuchObjectLockConfiguration` is a 404.
|
|
137
144
|
- **A live suite signs on a wall clock** (`performance`): `Date` is frozen → `RequestTimeTooSkewed`.
|
|
138
|
-
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
rule or a post-upload `object.size` check, neither of which is a driver's; `SignedUrlOptions`
|
|
144
|
-
says so and `driver-parity.test.ts` pins both halves.
|
|
145
|
+
- **A grant binds its size on every disk** (`As of 2026-11-02`). `grantUpload` requires `request.size` and passes `contentLength`; `s3Driver` signs `content-length` through core's `presignAwsUrl`, because Bun's presign signs `host` alone, so any other length is a 403 (`driver-s3.live.test.ts`). Local and memory hold it as the `x-max` ceiling, so a shorter body passes there: `driver-parity.test.ts` pins it. `maxBytes` with no `contentLength` on s3 is `X_STORAGE_UPLOAD_SIZE_UNDECLARED`. The content type stays unsigned.
|
|
146
|
+
- **`putStream` on s3 is signed multipart, never `S3File.writer()`.** Measured on Bun 1.4.4: `end(error)` COMPLETES the upload, so an overrun would replace the object, and the writer cannot send `x-amz-meta-*`. One part at a time; every exit but the Complete aborts; a Complete can fail with 200 and an `<Error>` body.
|
|
147
|
+
- **`promoteAttachment` sniffs a head window** (`validateStoredHead`, 64 KiB), then ties it to the
|
|
148
|
+
copy: `ifMatch`, then size and etag. Never refuse on etag alone: SSE-KMS mints one per copy.
|
|
149
|
+
- **`/media`'s variant key names its source's revision and format.** No `revision`: the old key.
|
|
145
150
|
- **`delete()` is idempotent for an ABSENT key and for nothing else.** Both drivers used to end in
|
|
146
151
|
`.catch(() => undefined)`, so a denied `s3:DeleteObject`, a `SlowDown`, an expired credential and
|
|
147
152
|
a read-only mount all resolved as success — and `sweepOrphans` returned them as deleted, which
|
|
@@ -159,7 +164,7 @@ Gotchas:
|
|
|
159
164
|
- **`serverSideEncryption` is declared and refused by BOTH drivers.** Bun exposes `acl`,
|
|
160
165
|
`storageClass` and `type` and nothing for `x-amz-server-side-encryption*`, and a POSIX file is
|
|
161
166
|
not encrypted at all. Refusing on the dev disk too is deliberate: a `put()` that works in `x dev`
|
|
162
|
-
and throws in production is two rules.
|
|
167
|
+
and throws in production is two rules. Bucket lifecycle (the app sweeps with `expire.ts`), storage classes and replication stay out
|
|
163
168
|
entirely — bucket-side, terraform's, axiom 7.
|
|
164
169
|
- **Quarantine is `pending/quarantine/`, INSIDE the pending prefix.** So `sweepOrphans` collects a
|
|
165
170
|
never-scanned upload with no second prefix to walk, and `isPendingKey` stays true for it.
|
|
@@ -167,9 +172,7 @@ Gotchas:
|
|
|
167
172
|
the app's scan verdict, and returns the ordinary pending key promotion accepts. Ultimate ships
|
|
168
173
|
no scanner — axiom 8, and `application/zip` is an accepted OOXML container by design.
|
|
169
174
|
- **`StorageListEntry` vs `StorageObject`.** A listing's `contentType` is optional; a `get()`'s is
|
|
170
|
-
not. `ListObjectsV2` returns no Content-Type, and
|
|
171
|
-
`application/octet-stream` while the local driver read the truth from its sidecar — two drivers
|
|
172
|
-
disagreeing about one object. The local `list()` now omits it too when there is no sidecar.
|
|
175
|
+
not. `ListObjectsV2` returns no Content-Type, and no driver invents one.
|
|
173
176
|
- **`META_DIR` lives in `path.ts`, not in `driver-local.ts`.** The sidecar namespace overlapped the
|
|
174
177
|
object namespace: `put('a/b', png)` writes `<root>/.meta/a/b.json`, and `.meta/a/b.json` was
|
|
175
178
|
itself a legal key, so an uploader could rewrite another object's recorded `contentType` to
|
|
@@ -177,17 +180,7 @@ Gotchas:
|
|
|
177
180
|
`assertSafeKey`, so it holds for S3 too — a key valid on one driver and refused on another is two
|
|
178
181
|
key rules, and the reservation folds case — see above. The `list()` skip is not a second line of
|
|
179
182
|
defence but the only one: the glob yields the sidecar tree.
|
|
180
|
-
- **`localDriver` refuses to construct outside development without a usable signing secret**
|
|
181
|
-
(`X_ENV_MISSING`, borrowed from core). Usable means neither the `signingSecret` option nor
|
|
182
|
-
`STORAGE_SIGNING_SECRET` is missing, empty **or** the published `DEV_SIGNING_SECRET` — pasting
|
|
183
|
-
the literal into `app.config.ts` configures nothing, so it is refused exactly as its absence is.
|
|
184
|
-
`DEV_SIGNING_SECRET` is published in this repo, and `acceptSignedUpload` trusts a signed URL's
|
|
185
|
-
constraints over the app's `uploadPolicy` — so the fallback is a universal grant to mint any
|
|
186
|
-
`PUT`. Refused at construction, not at the first `signedUrl()`: a process that cannot sign safely
|
|
187
|
-
must not finish booting. The cause names the environment `resolveEnvironment()` resolved, which
|
|
188
|
-
may be `NODE_ENV`'s, never a variable the process did not set. `usesDevStorageSecret()` is the
|
|
189
|
-
`x doctor` predicate, mirroring core's `usesDevCursorSecret()`; it reads the env var, so a disk
|
|
190
|
-
handed an explicit `signingSecret` is outside its question.
|
|
183
|
+
- **`localDriver` refuses to construct outside development without a usable signing secret** (`X_ENV_MISSING`, borrowed from core). Full rule: `docs/history/storage.md`.
|
|
191
184
|
- **`localDriver({ env })` and `usesDevStorageSecret({ env })` are ONE question about ONE table**:
|
|
192
185
|
the secret, `isLocal()` and the refusal's environment all read `options.env ?? process.env`, so
|
|
193
186
|
the boot's guard and the constructor it guards cannot answer from two tables.
|
|
@@ -196,8 +189,8 @@ Gotchas:
|
|
|
196
189
|
/_storage/:disk/*key` gates on `@ultimat3/policy`'s `evaluate()` (`storage:read`), which is tier
|
|
197
190
|
2 and unreachable from here — so a "serve this object" helper in this package could only ever be
|
|
198
191
|
a second authz path. This package's contribution to that route is `assertSafeKey`,
|
|
199
|
-
`isTenantScoped`/`isWithinOrg` and the driver's own `contentType`/`etag`; the `Response` is
|
|
200
|
-
|
|
192
|
+
`isTenantScoped`/`isWithinOrg` and the driver's own `contentType`/`etag`; the `Response` is
|
|
193
|
+
`@ultimat3/http`'s `objectResponse`, structural over those fields, as `accept.ts`'s header says.
|
|
201
194
|
- `acceptSignedUpload` still has no mounted route: the signing secret is closed over inside
|
|
202
195
|
`localDriver` and no `StorageDriver` method exposes it, so a host cannot verify a signed PUT
|
|
203
196
|
through the `Storage` seam it is handed. Serving reads needed none of it (policy decides, not a
|
|
@@ -236,19 +229,7 @@ Gotchas:
|
|
|
236
229
|
- `acceptSignedUpload` refuses a URL signed with **no** content type (`unconstrained`). `grantUpload`
|
|
237
230
|
always sets one, so such a URL is hand-rolled, and trusting the uploader's header instead is the
|
|
238
231
|
only other option.
|
|
239
|
-
- **The signed base is the disk's REGISTERED name, and it is stated once — on the driver.**
|
|
240
|
-
`defineStorage` tells each driver its registration key (`StorageDriver.registerAs`), the driver
|
|
241
|
-
rebases its URLs onto `signedUrlBaseFor(diskName)`, and `accept.ts` reads `disk.signedUrlBase`
|
|
242
|
-
rather than deriving a second one. Both halves used to derive it from `disk.name` — the DRIVER
|
|
243
|
-
kind — so they agreed with each other and disagreed with the mounted `/_storage/:disk/*key`
|
|
244
|
-
route, which resolves the segment through the registry: a disk registered as `uploads` minted
|
|
245
|
-
`/_storage/local/...` and 404'd its own signatures. Latent only because every disk in this repo
|
|
246
|
-
happens to be named `local`. An explicit `baseUrl` on `localDriver` outranks the registration
|
|
247
|
-
(the operator saying where the route is mounted); an unregistered driver still mints under
|
|
248
|
-
`local`. One driver instance under two disk names is refused at `defineStorage`
|
|
249
|
-
(`X_CONFIG_INVALID`) — it could only mint under one of them. `@ultimat3/cli`'s
|
|
250
|
-
`STORAGE_BASE_PATH` is a third statement of the mount prefix and should import
|
|
251
|
-
`DEFAULT_SIGNED_URL_BASE` instead.
|
|
232
|
+
- **The signed base is the disk's REGISTERED name, and it is stated once — on the driver.** Full rule: `docs/history/storage.md`.
|
|
252
233
|
- **`accept.ts` asks the `isTenantScoped`/`isWithinOrg` PAIR, exactly as `runtime-storage.ts` does.**
|
|
253
234
|
`isWithinOrg` alone refused every un-scoped key, so an app's own `brand/logo.png` was unreachable
|
|
254
235
|
through a URL it had just signed. `isTenantScoped` is case-INSENSITIVE and `isWithinOrg` is not:
|
|
@@ -274,8 +255,7 @@ Gotchas:
|
|
|
274
255
|
default header, and `decodeError` keeps a refusal `X_STORAGE_UPLOAD_FAILED` with the disk's
|
|
275
256
|
status. A request with no response at all is the transport's `X_CLIENT_TRANSPORT_FAILED` — it was a
|
|
276
257
|
bare `TypeError` before. This file calls no `fetch`; the XHR above is the one exemption.
|
|
277
|
-
- **Bundle** (`
|
|
278
|
-
**17,767 B** on `clientTransport`; the +5.9 kB over 20.2.1 is core's transport graph.
|
|
258
|
+
- **Bundle** (`uploadFile` only, browser, minified, `As of 2026-09-22`): **17,767 B**.
|
|
279
259
|
- `exactOptionalPropertyTypes` is on — declare optional fields as `x?: T | undefined`.
|
|
280
260
|
|
|
281
261
|
Why each rule above is shaped the way it is: [`docs/history/storage.md`](../../docs/history/storage.md).
|
package/README.md
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Named disks. **Call sites name a disk, never a driver.**
|
|
4
4
|
|
|
5
|
+
`bun add @ultimat3/storage` · guide: [Storage And Uploads](https://github.com/developerz-ai/ultimate/wiki/Storage-And-Uploads)
|
|
6
|
+
|
|
5
7
|
```ts
|
|
6
8
|
import { defineStorage, disk, localDriver, s3Driver, scopedKey } from '@ultimat3/storage';
|
|
7
9
|
|
|
@@ -191,8 +193,49 @@ for a disk that really does write large objects server-side, and remember S3 cap
|
|
|
191
193
|
**`get()` buffers too, so it has the same ceiling**: `maxGetBytes`, on all three drivers,
|
|
192
194
|
defaulting to that disk's `maxPutBytes`. An object past it is `X_STORAGE_TOO_LARGE` — decided on
|
|
193
195
|
the disk's own size (a HEAD on s3) before a byte is read — and is read with `stream()`. It exists
|
|
194
|
-
because an object's size is not the server's to choose: a
|
|
195
|
-
unmeasured, so a later `get()` of it was heap growth the uploader picked.
|
|
196
|
+
because an object's size is not the server's to choose: a URL signed with no `contentLength` lands
|
|
197
|
+
in a bucket unmeasured, so a later `get()` of it was heap growth the uploader picked.
|
|
198
|
+
|
|
199
|
+
## Large objects — `streamRange(key, [first, last])` and `putStream(key, stream, { maxBytes })`
|
|
200
|
+
|
|
201
|
+
`streamRange` reads no byte before `first`: s3 and local through `slice(...).stream()`, memory from
|
|
202
|
+
its copy. Inclusive and clamped like `get`'s range, with no ceiling because nothing is buffered;
|
|
203
|
+
past the end is `X_STORAGE_RANGE_UNSATISFIABLE`. `putStream` takes `contentType`, `cacheControl`,
|
|
204
|
+
`metadata` and a REQUIRED `maxBytes` (`PutStreamOptions`). Past it the source is cancelled, nothing
|
|
205
|
+
is stored and the key stays as it was (`X_STORAGE_TOO_LARGE`). `checksum`, `retention`, `legalHold`
|
|
206
|
+
and `serverSideEncryption` need the bytes up front: the type omits them and an untyped caller gets
|
|
207
|
+
`X_INVARIANT`. On s3 a body under one part is one signed PUT and past it a signed multipart upload,
|
|
208
|
+
one part at a time (5 MiB or `maxBytes / 10000`), aborted on every exit but the Complete: Bun's
|
|
209
|
+
`S3File.writer()` completes on `end(error)` and sends no `x-amz-meta-*`. Local stages under
|
|
210
|
+
`.meta/.tmp/`, then runs `put()`'s commit steps. A driver outside this package reuses
|
|
211
|
+
`assertPutStream`, `pumpBounded`, `streamLimit`, `assertStreamRange`, `streamedRange` and `signedByteBound`.
|
|
212
|
+
|
|
213
|
+
## Ranged reads — `get(key, { range: [first, last] })`
|
|
214
|
+
|
|
215
|
+
```ts
|
|
216
|
+
import { disk } from '@ultimat3/storage';
|
|
217
|
+
|
|
218
|
+
const page = await disk('debug').get('runs/r1/http.ndjson', { range: [0, 65_535] });
|
|
219
|
+
page.bytes; // at most 65 536 bytes
|
|
220
|
+
page.range; // [0, 65_535] — or [0, size - 1] when the object is shorter
|
|
221
|
+
page.object.size; // the WHOLE object's size, for `Content-Range: bytes 0-65535/<size>`
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
| Rule | |
|
|
225
|
+
|---|---|
|
|
226
|
+
| Offsets | `first` and `last` are BOTH inclusive — HTTP's `bytes=first-last`, S3's `Range:` |
|
|
227
|
+
| On the wire | s3: ONE signed GET with `Range:` and no HEAD — the bytes, the window and total size (`Content-Range`), the `ETag`, type and date all come from that one response, so a write racing the read (a shrink, a grow, an A → B → A of one length) can never pair one version's bytes with another's description. Asks at most `maxGetBytes`; the provider's 416 is `X_STORAGE_RANGE_UNSATISFIABLE` (its size from a HEAD). local: `Bun.file().slice()` in the key's queue; memory: a slice of its copy |
|
|
228
|
+
| Clamp | a `last` past the end is clamped to `size - 1`, as HTTP clamps it; `read.range` says what was served |
|
|
229
|
+
| Refusal | a `first` at or past the end — an empty object has none — is `X_STORAGE_RANGE_UNSATISFIABLE` (416); its `fix:` is the whole-object range, or the plain `get(key)` for an empty one |
|
|
230
|
+
| Screen | a backwards, negative, fractional or malformed tuple is the caller's defect: core's `X_INVARIANT` |
|
|
231
|
+
| Ceiling | `maxGetBytes` bounds the RANGE, not the object: a 40MB log is refused whole and readable 1MB at a time |
|
|
232
|
+
| Absence | `read.range` is present exactly when a range was asked — a third-party driver that ignored the option answers none |
|
|
233
|
+
|
|
234
|
+
**A SEALED object cannot be range-read meaningfully.** `seal()` (`@ultimat3/core`) writes one
|
|
235
|
+
`x1` envelope — one AES-256-GCM ciphertext, base64url, whose tag authenticates every byte — so a
|
|
236
|
+
slice of it neither opens nor means anything, and there is no framed or chunked form of the
|
|
237
|
+
envelope to slice along (`As of 2026-10`). Range-read plain objects. To page a sealed log, seal
|
|
238
|
+
each page as its OWN object (`runs/<id>/http/0000`, `…/0001`) and read the page you need whole.
|
|
196
239
|
|
|
197
240
|
## Write-once objects (S3 Object Lock)
|
|
198
241
|
|
|
@@ -232,10 +275,19 @@ shipped driver refuses it (`X_NOT_IMPLEMENTED`) with the out-of-band command in
|
|
|
232
275
|
`x-amz-server-side-encryption*`; a local disk writes plain files. A typed refusal an engineer
|
|
233
276
|
meets at the call site beats a silent absence discovered during a security review.
|
|
234
277
|
|
|
235
|
-
Encryption at rest is therefore a **bucket default**, and so are lifecycle rules, storage
|
|
236
|
-
and cross-region replication: all four are bucket-side configuration and belong to
|
|
237
|
-
to the framework (axiom 7 — zero platform primitives). Ultimate half-building any of
|
|
238
|
-
be a second place to look for the same setting.
|
|
278
|
+
Encryption at rest is therefore a **bucket default**, and so are bucket lifecycle rules, storage
|
|
279
|
+
classes and cross-region replication: all four are bucket-side configuration and belong to
|
|
280
|
+
terraform, not to the framework (axiom 7 — zero platform primitives). Ultimate half-building any of
|
|
281
|
+
them would be a second place to look for the same setting.
|
|
282
|
+
|
|
283
|
+
What the app sweeps itself, on every disk, is `expire.ts` (`As of 2026-11-06`): the window is the
|
|
284
|
+
app's and required at the call.
|
|
285
|
+
|
|
286
|
+
| Call | Answers |
|
|
287
|
+
|---|---|
|
|
288
|
+
| `expireObjects({ disk, prefix, olderThanMs, keep?, clock? })` | `{ deleted, failed, spared }`. Undated is spared, a locked object is `failed`, never forced; an empty `prefix` is refused |
|
|
289
|
+
| `abortStaleUploads({ disk, olderThanMs, prefix?, match? })` | `{ aborted, failed, spared }` over the disk's open upload sessions |
|
|
290
|
+
| `objectPurgeTarget({ name, disk, prefix, match?, olderThanMs })` | `@ultimat3/jobs`' `PurgeTarget`, structurally: both sweeps as one target of the app's `purge()`. `match(key)` narrows both to one kind of file under every tenant (`org/<id>/imports/`) |
|
|
239
291
|
|
|
240
292
|
## Keys
|
|
241
293
|
|
|
@@ -286,10 +338,12 @@ a test fixture), or the guard answers about one process and the disk signs accor
|
|
|
286
338
|
Verification is constant-time, checks the signature *before* the expiry (a forged URL never
|
|
287
339
|
learns it was merely late), takes a `Clock` so tests freeze time, and returns
|
|
288
340
|
`{ ok: false, reason }` rather than throwing — `malformed | unsafe-key | signature-mismatch |
|
|
289
|
-
expired`. S3 presign
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
341
|
+
expired`. An S3 presign binds method and expiry, and `contentLength` where it is given: `s3Driver`
|
|
342
|
+
then signs `content-length` with core's `presignAwsUrl`, so the provider refuses any other length
|
|
343
|
+
(403, `driver-s3.live.test.ts`). Bun's presign, the path without one, signs `host` alone. Either way
|
|
344
|
+
`contentType` is unsigned: it becomes `response-content-type`, which governs a GET, so a PUT's
|
|
345
|
+
`Content-Type` is the uploader's, and the type is `promoteAttachment`'s sniff. `s3Driver` refuses
|
|
346
|
+
`maxBytes` with no `contentLength` (`X_STORAGE_UPLOAD_SIZE_UNDECLARED`): it has no ceiling to sign.
|
|
293
347
|
|
|
294
348
|
## Uploads sniff the content type
|
|
295
349
|
|
|
@@ -311,6 +365,14 @@ allowlist names the type it means.
|
|
|
311
365
|
|
|
312
366
|
`validateUpload({ key, declaredContentType, bytes }, uploadPolicy({ maxBytes: 5e6 }))`
|
|
313
367
|
|
|
368
|
+
**The same rule over an object already on a disk:**
|
|
369
|
+
`validateStoredHead({ key, declaredContentType, head, size }, policy)`, where `head` is
|
|
370
|
+
`disk.get(key, { range: [0, SNIFF_WINDOW_BYTES - 1] })`'s bytes (64 KiB) and `size` the object's. No
|
|
371
|
+
checksum, and no buffering of the object. It answers `{ key, contentType, size }`. A character cut at
|
|
372
|
+
the window's end is not a bad sequence. An XML or `+xml` type over the window is refused, because XML
|
|
373
|
+
runs a script anywhere in the document and only `validateUpload` reads all of it. On both paths, over
|
|
374
|
+
`MAX_LEADING_WHITESPACE` (4,096 B) of leading whitespace is `X_STORAGE_TYPE_REJECTED`.
|
|
375
|
+
|
|
314
376
|
## The direct-upload round trip
|
|
315
377
|
|
|
316
378
|
Three calls, one per hop. The **client never names the key** — it asks for a grant and is told
|
|
@@ -360,6 +422,26 @@ no bytes, and the signed `maxBytes` caps the body read. Then it calls `acceptSig
|
|
|
360
422
|
policy built from the grant's own signed type and ceiling. The routes serve `definedStorage()`,
|
|
361
423
|
the process's one registry, which is the app's `defineStorage` when it declared one (#524).
|
|
362
424
|
|
|
425
|
+
## Uploads in parts — `grantMultipart`, `signPart`, `listParts`, `completeUpload`, `abortUpload`
|
|
426
|
+
|
|
427
|
+
`As of 2026-11-06`. Resumable, and past S3's 5 GiB single PUT. Same policy gate as `grantUpload`
|
|
428
|
+
(`decideGrant`, one function), every call held to the actor's org. `StorageDriver.multipart` is
|
|
429
|
+
optional (`MultipartCapability`, `multipart.ts`); every shipped disk has one: s3 over its signed
|
|
430
|
+
wire (`driver-s3-multipart*.ts`), local and memory over a part store (`multipart-staged.ts`), whose
|
|
431
|
+
part URLs carry `x-upload`/`x-part` inside the HMAC and land through `acceptSignedPart`. In the
|
|
432
|
+
browser, `uploadFile({ file, grant, multipart })` switches at `partThresholdBytes` (64 MiB): four
|
|
433
|
+
parts in flight, each retried with core's `retry`, `resume: grant` sends only what `listParts`
|
|
434
|
+
lacks. `completeUpload` reads the etags on the server, so no bucket CORS exposes `ETag`.
|
|
435
|
+
`signPart` and `completeUpload` REQUIRE the grant's `policy`: a part no object under `maxBytes`
|
|
436
|
+
holds, or an assembled total past it, is `X_STORAGE_TOO_LARGE`. The declared `size` is the
|
|
437
|
+
client's claim and the server keeps no grant, so the policy is the bound.
|
|
438
|
+
|
|
439
|
+
| Code | When |
|
|
440
|
+
|---|---|
|
|
441
|
+
| `X_STORAGE_MULTIPART_UNSUPPORTED` | the driver has no `multipart` |
|
|
442
|
+
| `X_STORAGE_UPLOAD_UNKNOWN` | the session was never opened, or is completed or aborted |
|
|
443
|
+
| `X_STORAGE_PARTS_INVALID` | a named part is missing, out of order, has another etag, or is under 5 MiB and not the last |
|
|
444
|
+
|
|
363
445
|
## Attachments and orphans
|
|
364
446
|
|
|
365
447
|
An upload happens **before** the row it belongs to exists, so it lands at
|
|
@@ -370,9 +452,10 @@ An upload happens **before** the row it belongs to exists, so it lands at
|
|
|
370
452
|
| `pendingKey(orgId, uploadName(id, filename))` | `org/o1/pending/u-1.png` |
|
|
371
453
|
| `quarantineKey(orgId, name)` | `org/o1/pending/quarantine/u-1.png` |
|
|
372
454
|
| `attachmentKey(orgId, { entity, id, field }, name)` | `org/o1/post/p-1/cover/u-1.png` |
|
|
373
|
-
| `promoteAttachment({ disk, key, orgId, target, policy })` | `stat`, `copy`, then `delete` — never the reverse. **`policy` is required**: the object is measured here (`stat().size` over `policy.maxBytes` is `X_STORAGE_TOO_LARGE`, and it stays under `pending/` for the sweep), because an s3 presign bounds no size. A call after the source is gone, with the attached key present, is `X_STORAGE_ALREADY_PROMOTED` (409, `meta.attachedKey`): an earlier call moved the bytes, so this one answers the row that holds them and never deletes that key. The key must be a **pending** one (`isPendingKey`): an attached key a client sent back is another row's file, and moving it would delete the victim's copy (`X_STORAGE_NOT_PENDING`) |
|
|
455
|
+
| `promoteAttachment({ disk, key, orgId, target, policy })` | `stat`, `copy`, then `delete` — never the reverse. **`policy` is required**: the object is measured here (`stat().size` over `policy.maxBytes` is `X_STORAGE_TOO_LARGE`, and it stays under `pending/` for the sweep), because an s3 presign bounds no size. Then one ranged read of the first 64 KiB through `validateStoredHead`, because it binds no type or bytes either: a contradiction is `X_STORAGE_TYPE_REJECTED`, and it stays pending too. The URL stays valid after that read, so `copy(from, to, { ifMatch })` names the revision judged (`X_STORAGE_OBJECT_CHANGED`, on s3 a signed CopyObject), and the copy is checked: another size is refused and deleted, and an etag that differs or is unknown (SSE-KMS mints one per copy) means the copy's own head is judged. A call after the source is gone, with the attached key present, is `X_STORAGE_ALREADY_PROMOTED` (409, `meta.attachedKey`): an earlier call moved the bytes, so this one answers the row that holds them and never deletes that key. The key must be a **pending** one (`isPendingKey`): an attached key a client sent back is another row's file, and moving it would delete the victim's copy (`X_STORAGE_NOT_PENDING`) |
|
|
374
456
|
| `releaseQuarantine({ disk, key, orgId })` | quarantine → pending; returns the released key |
|
|
375
457
|
| `sweepOrphans({ disk, orgId, olderThanMs })` | `{ deleted, failed }` for stale `pending/` keys. `olderThanMs` is a whole number of 0 or more, refused otherwise (`X_INVARIANT`) — `NaN` read as "everything is old enough" and deleted an upload made a moment ago |
|
|
458
|
+
| `deleteVariants({ disk, key })` | `{ deleted, failed }` for every stored variant of one source, every revision (`list({ prefix: `${key}@` })`). **An erasure of one object must call it**: `disk.delete` leaves the variants' bytes behind. The source is the caller's to delete; a variant keeps its prefix, so an org-wide erasure by prefix already collects them |
|
|
376
459
|
|
|
377
460
|
The filename contributes nothing but its extension, and only if it matches `.[a-z0-9]{1,12}`.
|
|
378
461
|
`sweepOrphans` can only reach the `pending/` prefix of one org — a sweep that could touch an
|
|
@@ -397,12 +480,20 @@ const grant = await grantUpload({ disk, orgId, request, quarantine: true });
|
|
|
397
480
|
|
|
398
481
|
// promoteAttachment on that key throws X_STORAGE_QUARANTINED. Your scan job decides:
|
|
399
482
|
const released = await releaseQuarantine({ disk, key, orgId }); // clean
|
|
400
|
-
await promoteAttachment({ disk, key: released, orgId, target });
|
|
483
|
+
await promoteAttachment({ disk, key: released, orgId, target, policy });
|
|
401
484
|
```
|
|
402
485
|
|
|
403
486
|
Inside `pending/` deliberately: an upload nobody ever scanned is still an orphan, so
|
|
404
487
|
`sweepOrphans` collects it with no second prefix to walk.
|
|
405
488
|
|
|
489
|
+
## A person's objects, for a subject export or erase
|
|
490
|
+
|
|
491
|
+
`storageSubjectTarget(() => disk(), keysOf, { name? })` is a `defineSubject({ targets })` entry
|
|
492
|
+
(`@ultimat3/jobs`): `keysOf(subject)` answers the person's keys (a list, or `disk().list({ prefix })`
|
|
493
|
+
over a prefix the app owns). The export lists each object that exists (`STORAGE_SUBJECT_COLUMNS`);
|
|
494
|
+
the erase deletes them in bounded batches. There is no listing by owner: a key the app does not
|
|
495
|
+
name is not found.
|
|
496
|
+
|
|
406
497
|
## Errors
|
|
407
498
|
|
|
408
499
|
| Code | Fires when |
|
|
@@ -416,6 +507,7 @@ Inside `pending/` deliberately: an upload nobody ever scanned is still an orphan
|
|
|
416
507
|
| `X_STORAGE_URL_INVALID` | a signed request that does not match what was signed — edited constraint, wrong base, wrong method, contradicting `Content-Type` |
|
|
417
508
|
| `X_STORAGE_URL_EXPIRED` | the grant's window closed; the signature was fine |
|
|
418
509
|
| `X_STORAGE_ORG_MISMATCH` | the key is well-formed and unforged, and belongs to another org |
|
|
510
|
+
| `X_STORAGE_UPLOAD_SIZE_UNDECLARED` | `grantUpload` with no whole `request.size` of at least 1, or `signedUrl({ maxBytes })` on s3 with no `contentLength` |
|
|
419
511
|
| `X_STORAGE_UPLOAD_FAILED` | client half: the presigned `PUT` answered non-2xx or never landed |
|
|
420
512
|
| `X_STORAGE_DELETE_FAILED` | the disk REFUSED a delete — denied `s3:DeleteObject`, a throttle, an expired credential, a read-only mount. An **absent** key is still not an error |
|
|
421
513
|
| `X_STORAGE_LIST_FAILED` | the disk REFUSED a listing — denied `s3:ListBucket`, a throttle, an unreadable root. An **empty** disk is still not an error |
|
|
@@ -425,6 +517,7 @@ Inside `pending/` deliberately: an upload nobody ever scanned is still an orphan
|
|
|
425
517
|
| `X_STORAGE_PUT_FAILED` | the disk REFUSED a `put()`/`copy()` — `EACCES`, `ENOSPC`, `EROFS`, a provider's refused PUT |
|
|
426
518
|
| `X_STORAGE_OBJECT_LOCKED` | local, memory: a delete, overwrite or copy onto a key under retention or a legal hold; the cause names the key, the mode and `retainUntil` or the hold, and the bytes are unchanged. Never raised by s3, where the provider keeps the locked version |
|
|
427
519
|
| `X_STORAGE_READ_FAILED` | the disk REFUSED a `get`/`stat`/`exists`/`stream` or the read half of a `copy` — a denied `s3:GetObject`, a throttle, an unreadable file. An **absent** object is still `X_STORAGE_NOT_FOUND`, including one deleted between the existence check and the read |
|
|
520
|
+
| `X_STORAGE_RANGE_UNSATISFIABLE` | `get(key, { range })` whose first byte is at or past the object's size — the 416 |
|
|
428
521
|
| `X_STORAGE_KEY_CONFLICT` | local and memory disks (never s3): the key's path is another key's directory, or the reverse (`a` and `a/b`) |
|
|
429
522
|
| `X_NOT_IMPLEMENTED` | `serverSideEncryption` on any driver |
|
|
430
523
|
| `X_ENV_MISSING` | core's: S3 credential env vars, or a `localDriver` built outside development where neither `signingSecret` nor `STORAGE_SIGNING_SECRET` holds a secret other than the published `DEV_SIGNING_SECRET` |
|
|
@@ -445,6 +538,12 @@ a job boundary the class is gone and the `code` is what survives — match on th
|
|
|
445
538
|
`variantKey()`, `srcsetDescriptors()`, `fitDimensions()` are pure — `@ultimat3/seo` builds
|
|
446
539
|
`srcset` from them without decoding a byte.
|
|
447
540
|
|
|
541
|
+
`warmVariants({ disk, key, widths, formats })` (`image-warm.ts`) writes every width x format of a
|
|
542
|
+
stored image under the key `/media` reads — `variantKey(key, { width, format, revision })`, the
|
|
543
|
+
revision off the etag of the bytes it read — so the job an upload enqueues pays the decode and no
|
|
544
|
+
visitor does. Answers `{ written, failed, skipped }`: a width above the source's is skipped, a coded
|
|
545
|
+
failure (an `avif`) is one entry and the rest are still written. `As of 2026-11`.
|
|
546
|
+
|
|
448
547
|
A variant key is the WHOLE source key plus the transform: `photos/hero.png` at width 640 is
|
|
449
548
|
`photos/hero.png@w640.webp`. The source extension stays in, so `hero.png` and `hero.jpg` are two
|
|
450
549
|
cache identities — it was cut, and both minted `photos/hero@w640.webp`. `width` and `height` are
|
|
@@ -452,6 +551,14 @@ whole numbers of at least 1 on `variantKey` and `fitDimensions` (`X_INVARIANT` o
|
|
|
452
551
|
`wNaN`, no `w1.5`). `fitDimensions` never answers a zero edge (1000×1 at width 100 is 100×1) and
|
|
453
552
|
never upscales except under `fit: 'cover'`, which is the exact box asked for.
|
|
454
553
|
|
|
554
|
+
`variantKey(key, { revision })` adds `r<8 hex>` (`hero.png@w640-r0123abcd.webp`), so a variant names
|
|
555
|
+
the bytes it was made from; absent, the key is byte-identical to one minted without it.
|
|
556
|
+
`variantRevision(etag)` is that part of an etag, however a provider quoted it, or `undefined` when it
|
|
557
|
+
cannot tell two revisions apart. `variantFormatOf(contentType)` maps a stored `png`, `jpeg` or `webp`
|
|
558
|
+
to its format and anything else to `undefined`. A key the route cannot state from those two is a
|
|
559
|
+
variant served and never stored. `variantSourceOf(key)` reads a revisioned key back into
|
|
560
|
+
`{ source, revision }`, which is how a route refuses a variant whose source is gone or overwritten.
|
|
561
|
+
|
|
455
562
|
**`VARIANT_FORMATS` / `VariantFormat` are what a variant KEY can carry, `As of 2026-08`** —
|
|
456
563
|
`avif`, `webp`, `jpeg`, `png` — and this package exports no `IMAGE_FORMATS` and no `ImageFormat`. Those two names are
|
|
457
564
|
`@ultimat3/core`'s, over the six formats it can PROBE (`png`, `jpeg`, `webp`, `avif`, `gif`,
|
|
@@ -472,3 +579,14 @@ the outputs that keep alpha. The encoded size is always exactly `fitDimensions()
|
|
|
472
579
|
the box, it does not letterbox to it. `blurPlaceholder()` returns a ThumbHash PNG `data:` URI, at
|
|
473
580
|
most 32px on its long edge.
|
|
474
581
|
`bun test` from `packages/storage`.
|
|
582
|
+
|
|
583
|
+
## Supporting exports
|
|
584
|
+
|
|
585
|
+
`As of 2026-11-02`.
|
|
586
|
+
|
|
587
|
+
| Name | What |
|
|
588
|
+
|---|---|
|
|
589
|
+
| `CopyOptions` | `copy(from, to, { ifMatch })`: the source's `etag` as this disk reported it |
|
|
590
|
+
| `DeleteVariantsInput` | what `deleteVariants` takes: `{ disk, key }` |
|
|
591
|
+
| `StoredHeadCandidate` | what `validateStoredHead` reads: the key, the type, the first bytes and the WHOLE object's size |
|
|
592
|
+
| `ValidatedHead` | what `validateStoredHead` answers |
|
package/package.json
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/storage",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "30.0.0",
|
|
4
4
|
"description": "Named disks over Bun.file and Bun.s3: safe keys, signed URLs, sniffed uploads",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"ultimate",
|
|
7
|
+
"bun",
|
|
8
|
+
"framework",
|
|
9
|
+
"storage",
|
|
10
|
+
"uploads",
|
|
11
|
+
"s3",
|
|
12
|
+
"files"
|
|
13
|
+
],
|
|
14
|
+
"homepage": "https://github.com/developerz-ai/ultimate/wiki/Storage-And-Uploads",
|
|
5
15
|
"license": "MIT",
|
|
6
16
|
"type": "module",
|
|
7
17
|
"repository": {
|
|
@@ -32,6 +42,6 @@
|
|
|
32
42
|
"test": "bun test"
|
|
33
43
|
},
|
|
34
44
|
"dependencies": {
|
|
35
|
-
"@ultimat3/core": "
|
|
45
|
+
"@ultimat3/core": "30.0.0"
|
|
36
46
|
}
|
|
37
47
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Single responsibility: the one decision a part PUT on `/_storage` is made of, on a disk that signs
|
|
2
|
+
// its own part URLs — verified, held to the actor's tenant, bounded by the signed length, then
|
|
3
|
+
// handed to the disk's open session. Transport-free, as `accept.ts` is: the route owns the request.
|
|
4
|
+
|
|
5
|
+
import { type SignedRequestInput, signedUploadConstraints } from './accept';
|
|
6
|
+
import { signedUrlRejected, tooLarge } from './errors';
|
|
7
|
+
import { multipartUnsupported } from './errors-multipart';
|
|
8
|
+
import type { UploadedPart } from './multipart';
|
|
9
|
+
|
|
10
|
+
export interface AcceptSignedPartInput extends SignedRequestInput {
|
|
11
|
+
readonly bytes: Uint8Array;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Store one part, or refuse. The URL must be a part URL — a single PUT's URL is
|
|
16
|
+
* `acceptSignedUpload`'s — and the disk must keep parts itself (`multipart.receivePart`); an s3
|
|
17
|
+
* part never reaches this process. An upload that is not open is `X_STORAGE_UPLOAD_UNKNOWN`.
|
|
18
|
+
*/
|
|
19
|
+
export async function acceptSignedPart(input: AcceptSignedPartInput): Promise<UploadedPart> {
|
|
20
|
+
const constraints = await signedUploadConstraints(input);
|
|
21
|
+
const { key, part, maxBytes } = constraints;
|
|
22
|
+
if (part === undefined) {
|
|
23
|
+
throw signedUrlRejected('not-a-part', `"${key}" was signed for the object, not one part`);
|
|
24
|
+
}
|
|
25
|
+
const size = input.bytes.byteLength;
|
|
26
|
+
if (maxBytes !== undefined && size > maxBytes) throw tooLarge(key, size, maxBytes);
|
|
27
|
+
const receive = input.disk.multipart?.receivePart;
|
|
28
|
+
if (receive === undefined) throw multipartUnsupported(input.disk.name, key);
|
|
29
|
+
return receive.call(input.disk.multipart, key, part.uploadId, part.partNumber, input.bytes);
|
|
30
|
+
}
|
package/src/accept.ts
CHANGED
|
@@ -142,6 +142,13 @@ export async function signedUploadConstraints(
|
|
|
142
142
|
export async function acceptSignedUpload(input: AcceptSignedUploadInput): Promise<StorageObject> {
|
|
143
143
|
const constraints = await constraintsFor(input, 'PUT');
|
|
144
144
|
const key = constraints.key;
|
|
145
|
+
// A part URL writes one part of an open upload, never the object: `acceptSignedPart` takes it.
|
|
146
|
+
if (constraints.part !== undefined) {
|
|
147
|
+
throw signedUrlRejected(
|
|
148
|
+
'part',
|
|
149
|
+
`"${key}" was signed for one part of an upload, not the object`,
|
|
150
|
+
);
|
|
151
|
+
}
|
|
145
152
|
|
|
146
153
|
// A PUT signed with no content type bounds nothing — any bytes, any type, at that key. The
|
|
147
154
|
// grant always sets one, so this is a hand-rolled URL, and refusing is the only fail-closed
|
package/src/attachment.ts
CHANGED
|
@@ -8,9 +8,10 @@ import type { Clock } from '@ultimat3/core';
|
|
|
8
8
|
import { assert, finiteCount, renderThrowable, systemClock } from '@ultimat3/core';
|
|
9
9
|
import type { StorageDriver, StorageListEntry, StorageObject } from './driver';
|
|
10
10
|
import { objectNotFound, orgMismatch, tooLarge } from './errors';
|
|
11
|
-
import { alreadyPromoted, notPending, quarantined } from './errors-promote';
|
|
12
|
-
import { isWithinOrg, orgPrefix, scopedKey } from './path';
|
|
11
|
+
import { alreadyPromoted, notPending, objectChanged, quarantined } from './errors-promote';
|
|
12
|
+
import { assertSafeKey, isWithinOrg, orgPrefix, scopedKey } from './path';
|
|
13
13
|
import type { UploadPolicy } from './upload';
|
|
14
|
+
import { SNIFF_WINDOW_BYTES, validateStoredHead } from './upload';
|
|
14
15
|
|
|
15
16
|
/** The one segment an unattached upload lives under. `sweepOrphans` reads only this prefix. */
|
|
16
17
|
export const PENDING_SEGMENT = 'pending';
|
|
@@ -126,13 +127,61 @@ export interface PromoteAttachmentInput {
|
|
|
126
127
|
readonly orgId: string;
|
|
127
128
|
readonly target: AttachmentTarget;
|
|
128
129
|
/**
|
|
129
|
-
* The policy the upload was GRANTED under — required, never defaulted.
|
|
130
|
-
*
|
|
131
|
-
*
|
|
130
|
+
* The policy the upload was GRANTED under — required, never defaulted. A URL an app signed itself
|
|
131
|
+
* may carry no size, so `policy.maxBytes` binds nothing until this call measures the object; a
|
|
132
|
+
* default here would silently replace an app's own ceiling with the framework's.
|
|
132
133
|
*/
|
|
133
134
|
readonly policy: UploadPolicy;
|
|
134
135
|
}
|
|
135
136
|
|
|
137
|
+
/** The head window of one stored object, under the type it is stored as. */
|
|
138
|
+
async function judge(
|
|
139
|
+
disk: StorageDriver,
|
|
140
|
+
object: StorageObject,
|
|
141
|
+
key: string,
|
|
142
|
+
policy: UploadPolicy,
|
|
143
|
+
): Promise<void> {
|
|
144
|
+
const head =
|
|
145
|
+
object.size === 0
|
|
146
|
+
? new Uint8Array()
|
|
147
|
+
: (await disk.get(key, { range: [0, SNIFF_WINDOW_BYTES - 1] })).bytes;
|
|
148
|
+
validateStoredHead(
|
|
149
|
+
{ key, declaredContentType: object.contentType, head, size: object.size },
|
|
150
|
+
policy,
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* `ifMatch` is the first line and this the second, for a disk whose `copy` honours no condition.
|
|
156
|
+
* The upload URL stays valid after the sniff, so a copy is only trusted once it is known to be the
|
|
157
|
+
* revision judged. Another size is another object, whatever else agrees.
|
|
158
|
+
*
|
|
159
|
+
* An etag alone does not settle it either way. An empty one says nothing, and an SSE-KMS or SSE-C
|
|
160
|
+
* bucket mints a new etag per copy, so refusing on a difference would refuse every promotion there.
|
|
161
|
+
* So a copy that cannot be proved the same is judged itself, by the rule the source just passed:
|
|
162
|
+
* what lands on the row's key has been looked at either way. A refusal leaves nothing attached; a
|
|
163
|
+
* delete that fails raises its own code, because the bytes are still there.
|
|
164
|
+
*/
|
|
165
|
+
async function assertCopyIsWhatWasJudged(
|
|
166
|
+
disk: StorageDriver,
|
|
167
|
+
judged: StorageObject,
|
|
168
|
+
copy: StorageObject,
|
|
169
|
+
policy: UploadPolicy,
|
|
170
|
+
): Promise<void> {
|
|
171
|
+
const proved =
|
|
172
|
+
judged.etag !== '' && copy.etag === judged.etag && copy.contentType === judged.contentType;
|
|
173
|
+
if (proved && copy.size === judged.size) return;
|
|
174
|
+
try {
|
|
175
|
+
if (copy.size !== judged.size) {
|
|
176
|
+
throw objectChanged(copy.key, `${judged.size}B`, `${copy.size}B`);
|
|
177
|
+
}
|
|
178
|
+
await judge(disk, copy, copy.key, policy);
|
|
179
|
+
} catch (error) {
|
|
180
|
+
await disk.delete(copy.key);
|
|
181
|
+
throw error;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
136
185
|
/**
|
|
137
186
|
* Move a pending upload onto the row that now exists. Copy first, delete second: a delete that
|
|
138
187
|
* ran first would lose the bytes on a failed write, and the sweep would have collected them
|
|
@@ -142,6 +191,12 @@ export interface PromoteAttachmentInput {
|
|
|
142
191
|
* object stays under `pending/` for `sweepOrphans` — never at a key a row points to. `stat` and
|
|
143
192
|
* not `get`: this is the call that learns the size, so it cannot be one that buffers it.
|
|
144
193
|
*
|
|
194
|
+
* SNIFFED before it moves, off a head window. A provider's presigned PUT binds method, expiry and at
|
|
195
|
+
* most a size, so a bucket holds whatever bytes and whatever type the uploader sent, and this is the first
|
|
196
|
+
* code that looks: bytes that contradict the stored type are `X_STORAGE_TYPE_REJECTED`, and stay
|
|
197
|
+
* pending too. A ranged read because `maxGetBytes` bounds the range, never the object. An overwrite
|
|
198
|
+
* after that read is `X_STORAGE_OBJECT_CHANGED`, or the copy itself is judged, and never lands.
|
|
199
|
+
*
|
|
145
200
|
* A source that is gone with the destination already there is `X_STORAGE_ALREADY_PROMOTED`
|
|
146
201
|
* (409, `meta.attachedKey`), never the destination answered as if this call had moved it. 27.1.0
|
|
147
202
|
* answered it: a confirm sent twice then inserted a second row for one key, that insert failed,
|
|
@@ -179,7 +234,13 @@ export async function promoteAttachment(input: PromoteAttachmentInput): Promise<
|
|
|
179
234
|
if (pending.size > input.policy.maxBytes) {
|
|
180
235
|
throw tooLarge(input.key, pending.size, input.policy.maxBytes);
|
|
181
236
|
}
|
|
182
|
-
|
|
237
|
+
await judge(disk, pending, input.key, input.policy);
|
|
238
|
+
const object = await disk.copy(
|
|
239
|
+
input.key,
|
|
240
|
+
attached,
|
|
241
|
+
pending.etag === '' ? undefined : { ifMatch: pending.etag },
|
|
242
|
+
);
|
|
243
|
+
await assertCopyIsWhatWasJudged(disk, pending, object, input.policy);
|
|
183
244
|
await disk.delete(input.key);
|
|
184
245
|
return object;
|
|
185
246
|
}
|
|
@@ -251,3 +312,35 @@ export async function sweepOrphans(input: SweepOrphansInput): Promise<SweepResul
|
|
|
251
312
|
} while (cursor !== null);
|
|
252
313
|
return { deleted, failed };
|
|
253
314
|
}
|
|
315
|
+
|
|
316
|
+
export interface DeleteVariantsInput {
|
|
317
|
+
readonly disk: StorageDriver;
|
|
318
|
+
/** The SOURCE key. Its variants are every key that opens with it and `@`. */
|
|
319
|
+
readonly key: string;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Delete every stored variant of one source, of every revision, and answer as `sweepOrphans` does:
|
|
324
|
+
* a refusal is recorded and the walk continues, so an erasure never reports what is still there.
|
|
325
|
+
* The source itself is the caller's to delete. `@` is in the prefix, so `x.png2@w1.webp` is
|
|
326
|
+
* another source's variant and stays.
|
|
327
|
+
*/
|
|
328
|
+
export async function deleteVariants(input: DeleteVariantsInput): Promise<SweepResult> {
|
|
329
|
+
const prefix = `${assertSafeKey(input.key)}@`;
|
|
330
|
+
const deleted: string[] = [];
|
|
331
|
+
const failed: SweepFailure[] = [];
|
|
332
|
+
let cursor: string | null = null;
|
|
333
|
+
do {
|
|
334
|
+
const page = await input.disk.list({ prefix, cursor });
|
|
335
|
+
for (const object of page.rows) {
|
|
336
|
+
try {
|
|
337
|
+
await input.disk.delete(object.key);
|
|
338
|
+
deleted.push(object.key);
|
|
339
|
+
} catch (error) {
|
|
340
|
+
failed.push({ key: object.key, reason: renderThrowable(error) });
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
cursor = page.nextCursor;
|
|
344
|
+
} while (cursor !== null);
|
|
345
|
+
return { deleted, failed };
|
|
346
|
+
}
|