@ultimat3/storage 23.0.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +39 -8
- package/README.md +87 -6
- package/package.json +2 -2
- package/src/accept.ts +3 -6
- package/src/attachment.ts +40 -7
- package/src/driver-local-read.ts +104 -0
- package/src/driver-local-sidecar.ts +38 -0
- package/src/driver-local-write.ts +185 -0
- package/src/driver-local.ts +85 -172
- package/src/driver-memory.ts +36 -11
- package/src/driver-s3.ts +103 -30
- package/src/driver.ts +63 -1
- package/src/errors.ts +94 -12
- package/src/grant.ts +2 -2
- package/src/image.ts +34 -8
- package/src/index.ts +14 -9
- package/src/iso-bmff.ts +50 -0
- package/src/path.ts +17 -2
- package/src/signed-url.ts +41 -8
- package/src/signing-secret.ts +71 -0
- package/src/upload.ts +31 -5
package/CLAUDE.md
CHANGED
|
@@ -15,8 +15,9 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads.
|
|
|
15
15
|
wrap it in its own `action()` with its own policy. See `docs/architecture/17-uploads.md`.
|
|
16
16
|
|
|
17
17
|
- **Every byte ceiling and every TTL is screened where it is DECLARED, through core's
|
|
18
|
-
`finiteCount` and nothing else** — `uploadPolicy({ maxBytes })`,
|
|
19
|
-
`
|
|
18
|
+
`finiteCount` and nothing else** — `uploadPolicy({ maxBytes })`, every driver's `maxPutBytes`
|
|
19
|
+
and `maxGetBytes`, variant `width`/`height`, `grantUpload({ expiresInMs })`, `buildSignedUrl`
|
|
20
|
+
and the s3 presign TTL. No private copy:
|
|
20
21
|
`size > NaN` is false, so an unscreened `NaN` ceiling stops deciding anything. Variant `quality`
|
|
21
22
|
is core's `assertFiniteImageQuality`, not a second screen.
|
|
22
23
|
- **The default is taken on `undefined` and on nothing else** — `x === undefined ? D : x`, never
|
|
@@ -37,14 +38,19 @@ Tier 1. Object storage: named disks, safe keys, signed URLs, sniffed uploads.
|
|
|
37
38
|
|
|
38
39
|
| File | Owns |
|
|
39
40
|
|---|---|
|
|
40
|
-
| `driver.ts` | `StorageDriver` contract (
|
|
41
|
+
| `driver.ts` | `StorageDriver` contract (9 methods; `registerAs`/`verifySigned` optional), `assertPutOptions`, + bounded `toBytes`/`sha256Base64`/`etagOf` |
|
|
41
42
|
| `driver-local.ts` | dev default over `Bun.file`, `.meta/` sidecars, `Bun.Glob` listing |
|
|
42
|
-
| `driver-
|
|
43
|
+
| `driver-local-write.ts` | the local WRITE: staged under `.meta/.tmp/`, renamed marker → sidecar → bytes, `X_STORAGE_KEY_CONFLICT` / `X_STORAGE_PUT_FAILED`, the per-key queue |
|
|
44
|
+
| `driver-local-read.ts` | `headObject`: sidecar + pending marker → trusted, re-checked, or untyped |
|
|
45
|
+
| `driver-local-sidecar.ts` | the sidecar shape and its one parser |
|
|
46
|
+
| `signing-secret.ts` | `DEV_SIGNING_SECRET`, `usesDevStorageSecret`, `resolveSigningSecret` |
|
|
47
|
+
| `driver-memory.ts` | `memoryDriver()` — a test's disk over a `Map`: `localDriver`'s contract and refusals, `objects()` for an assertion about the bucket. `driver-memory.test.ts` runs each claim on BOTH disks. Signs through `resolveSigningSecret` (`signing-secret.ts`), the one rule both share |
|
|
43
48
|
| `driver-s3.ts` | `Bun.S3Client`, built lazily (import must never open a socket) |
|
|
44
49
|
| `driver-s3-region.ts` | a provider's wrong-region refusal → `X_CONFIG_INVALID` whose fix names `S3_REGION` |
|
|
45
50
|
| `path.ts` | key validation + `META_DIR` + `scopedKey`/`isWithinOrg`/`isTenantScoped` tenant boundary |
|
|
46
51
|
| `signed-url.ts` | HMAC over the constraint tuple, constant-time verify |
|
|
47
52
|
| `upload.ts` | magic-byte sniff + size/allowlist/checksum policy |
|
|
53
|
+
| `iso-bmff.ts` | the `ftyp` MAJOR BRAND → media type (AVIF, HEIC, MOV, M4A, MP4, 3GP) |
|
|
48
54
|
| `image.ts` | deterministic variant keys; byte path over core's pipeline (png/jpeg encode only) |
|
|
49
55
|
| | `VARIANT_FORMATS` — what a variant KEY can carry — and NOT `IMAGE_FORMATS`, which is core's and means what core can PROBE. See below |
|
|
50
56
|
| | `variantKey` is the cache identity `@ultimat3/cli`'s `/media/*` route looks a variant up by — derived, never stored, so a request that misses transforms once and every later one is a disk read |
|
|
@@ -60,6 +66,33 @@ bun run typecheck
|
|
|
60
66
|
```
|
|
61
67
|
|
|
62
68
|
Gotchas:
|
|
69
|
+
- **The local disk refuses a key that is another key's path, and that is pinned, not fixed**
|
|
70
|
+
(`As of 2026-10`). `put('a')` then `put('a/b')` — or the reverse, or `a` beside `a.json/b` in the
|
|
71
|
+
sidecar tree — is `X_STORAGE_KEY_CONFLICT`, decided by `claim()` for BOTH paths before either is
|
|
72
|
+
touched. Suffixed on-disk names would lift it and orphan every existing root's objects without a
|
|
73
|
+
migration; do not change the layout without one. `driver-contract.test.ts` runs the claim on all
|
|
74
|
+
three disks.
|
|
75
|
+
- **The local commit order is marker → sidecar → bytes → clear marker. Do not reorder**
|
|
76
|
+
(`As of 2026-10`). Bytes-first serves new bytes under the old type; sidecar-first alone serves
|
|
77
|
+
old bytes under the new one. The marker holds the new etag, so `headObject` knows the one state
|
|
78
|
+
in which the sidecar may not describe the bytes: `get`/`stat`/`copy` re-check against the bytes,
|
|
79
|
+
`list` reports no type and `etag: ''`. A fresh key is absent until its bytes land.
|
|
80
|
+
`driver-local-crash.test.ts` dies at every step through `WriteSteps`. Writers and `get()` of one
|
|
81
|
+
key also queue in-process (`keyedQueue`) — unqueued, two puts interleaved 3/3.
|
|
82
|
+
- **A refused write is `X_STORAGE_PUT_FAILED` and a refused read `X_STORAGE_READ_FAILED`**, on
|
|
83
|
+
local and s3 (both were bare throws); a 404 mid-read stays `X_STORAGE_NOT_FOUND`; `assertPutOptions` holds `metadata`/`cacheControl`/`contentType` to their types on
|
|
84
|
+
all three drivers before a byte moves.
|
|
85
|
+
- **`get()` has a ceiling on every disk** (`maxGetBytes`, default `maxPutBytes`), decided on the
|
|
86
|
+
disk's own size before a byte is read. `promoteAttachment` REQUIRES the policy and measures with
|
|
87
|
+
`stat()`, which is a REQUIRED driver method — on s3 it is the first measurement of a direct upload.
|
|
88
|
+
- **`lastModified` is optional — absent when the provider sent none**, never epoch 0 and never an
|
|
89
|
+
invalid `Date`. `sweepOrphans` spares an object with no proven age.
|
|
90
|
+
- **The base PATH is inside the signature** (`SIGNED_URL_VERSION` `v2`): every local disk shares
|
|
91
|
+
one secret, so the canonical string must name the disk. Path, not origin — the verifying route
|
|
92
|
+
sees a path — and `signedUrlBasePath` is how an absolute `baseUrl` is compared.
|
|
93
|
+
- **`isWithinOrg` never throws**: an org id that cannot be one contains nothing.
|
|
94
|
+
- **A variant key keeps the whole source key** (`hero.png@w640.webp`), so two sources differing
|
|
95
|
+
only by extension are two cache identities.
|
|
63
96
|
- **A driver's semantics are pinned in ONE test with the other driver beside them.**
|
|
64
97
|
`driver-parity.test.ts` drives `localDriver` over a temp dir and `s3Driver` over `FakeS3Client`
|
|
65
98
|
in a single `test()` per claim, so neither disk can move alone. Where the two genuinely cannot
|
|
@@ -200,7 +233,7 @@ Gotchas:
|
|
|
200
233
|
implementation `@ultimat3/auth` uses, not a second copy. Add new secret comparisons through it.
|
|
201
234
|
- `verifySignedUrl` never throws, and `parseConstraints` is where that is kept: the key is decoded
|
|
202
235
|
through the guarded `decodeSegment`, so a `%ZZ` in the path is `'malformed'` rather than the bare
|
|
203
|
-
`URIError` `decodeURIComponent` raises. Same shape as `@ultimat3/
|
|
236
|
+
`URIError` `decodeURIComponent` raises. Same shape as `@ultimat3/core`'s `readCookie`.
|
|
204
237
|
- `acceptSignedUpload` refuses a URL signed with **no** content type (`unconstrained`). `grantUpload`
|
|
205
238
|
always sets one, so such a URL is hand-rolled, and trusting the uploader's header instead is the
|
|
206
239
|
only other option.
|
|
@@ -214,9 +247,7 @@ Gotchas:
|
|
|
214
247
|
happens to be named `local`. An explicit `baseUrl` on `localDriver` outranks the registration
|
|
215
248
|
(the operator saying where the route is mounted); an unregistered driver still mints under
|
|
216
249
|
`local`. One driver instance under two disk names is refused at `defineStorage`
|
|
217
|
-
(`X_CONFIG_INVALID`) — it could only mint under one of them.
|
|
218
|
-
twice (`/_storage/local` in the driver, `/_storage` in `verifySignedUrl`'s default) and NO
|
|
219
|
-
genuine URL verified at all: the key parsed as `local/<key>`. `@ultimat3/cli`'s
|
|
250
|
+
(`X_CONFIG_INVALID`) — it could only mint under one of them. `@ultimat3/cli`'s
|
|
220
251
|
`STORAGE_BASE_PATH` is a third statement of the mount prefix and should import
|
|
221
252
|
`DEFAULT_SIGNED_URL_BASE` instead.
|
|
222
253
|
- **`accept.ts` asks the `isTenantScoped`/`isWithinOrg` PAIR, exactly as `dev-storage.ts` does.**
|
package/README.md
CHANGED
|
@@ -69,6 +69,51 @@ sidecar so `get()`/`list()` round-trip everything `put()` was handed; sidecars n
|
|
|
69
69
|
`list()`. `s3Driver` cannot: it refuses `cacheControl`/`metadata` on `put()` (`X_NOT_IMPLEMENTED`,
|
|
70
70
|
Bun exposes no header hook yet).
|
|
71
71
|
|
|
72
|
+
**The local disk writes an object by rename, in an order no crash can make lie.** An object there
|
|
73
|
+
is two files, so a `put()` or `copy()` stages bytes, sidecar and a *pending marker* (the new etag)
|
|
74
|
+
under `<root>/.meta/.tmp/` and then: renames the marker to `.meta/<key>.json.pending`, renames the
|
|
75
|
+
sidecar, renames the bytes, removes the marker. What a reader sees if the process dies:
|
|
76
|
+
|
|
77
|
+
| Died before | Fresh key | Overwrite |
|
|
78
|
+
|---|---|---|
|
|
79
|
+
| the marker or the sidecar rename | absent | the previous object, whole |
|
|
80
|
+
| the bytes rename | absent (a sidecar alone is no object) | the previous BYTES; `get`/`stat` answer `application/octet-stream` and the bytes' own etag, `list` no type and `etag: ''` — never the new type |
|
|
81
|
+
| clearing the marker | the new object to `get`/`stat` (re-checked against the bytes); `list` reports it untyped until the next put | same |
|
|
82
|
+
|
|
83
|
+
No row serves a content type or an etag the bytes were not checked against. A marker that
|
|
84
|
+
cannot be cleared after the renames landed is not a failed put — the object is committed and the
|
|
85
|
+
last row is what it leaves. `stat()`, `list()` and a copy's source read queue behind the key's
|
|
86
|
+
writer exactly as `get()` does. Writers and `get()` of
|
|
87
|
+
ONE key are also serialised inside a process, so two concurrent puts cannot interleave; across
|
|
88
|
+
processes the marker is what covers the window. `stream()` hands back bytes with no metadata and
|
|
89
|
+
checks nothing. The layout is otherwise unchanged — an existing root needs no migration.
|
|
90
|
+
|
|
91
|
+
**A refused write is coded.** `EACCES`, `ENOSPC`, `EROFS` or a root that is a file on the local
|
|
92
|
+
disk, and a provider's refused PUT on s3, are `X_STORAGE_PUT_FAILED` from `put()` and `copy()`,
|
|
93
|
+
with the errno or provider code in the cause. `put()` options are held to their types first
|
|
94
|
+
(`assertPutOptions`: string `contentType`/`cacheControl`, a plain object of strings for
|
|
95
|
+
`metadata`, `X_INVARIANT` otherwise) on all three drivers. A refused READ is `X_STORAGE_READ_FAILED` the same
|
|
96
|
+
way. No driver method leaves with a bare throw for a wrong-typed argument either: a key that is not
|
|
97
|
+
a string is `X_STORAGE_PATH_UNSAFE`, a body that is not bytes, a `Blob` or a stream and a
|
|
98
|
+
non-string `list()` prefix or cursor are `X_INVARIANT` (`driver-read-failed.test.ts` sweeps every
|
|
99
|
+
method on every disk). The one thing no code can cover is a `stream()` that fails after it was
|
|
100
|
+
handed back: that is the stream's own error.
|
|
101
|
+
|
|
102
|
+
**A key cannot be another key's path on the local disk.** A POSIX path is a file or a directory:
|
|
103
|
+
`put('a')` then `put('a/b')` — or the reverse, or `a` beside `a.json/b`, which collide in the
|
|
104
|
+
sidecar tree — is `X_STORAGE_KEY_CONFLICT`, refused before a byte moves, where it used to be a
|
|
105
|
+
bare `ENOTDIR` / `EISDIR`. Its `fix` names the disk as `defineStorage` registered it, not the
|
|
106
|
+
driver kind. `s3Driver` and `memoryDriver` hold both keys; `driver-contract.test.ts`
|
|
107
|
+
pins the divergence.
|
|
108
|
+
|
|
109
|
+
`stat(key)` answers what a read would — size, type, etag, age — **without the bytes**, and
|
|
110
|
+
`undefined` for nothing there. It is a required `StorageDriver` method: `promoteAttachment` measures
|
|
111
|
+
with it, and a driver that cannot measure does not compile.
|
|
112
|
+
|
|
113
|
+
`lastModified` is **optional** on a listing entry and on a `StorageObject`: absent when the
|
|
114
|
+
provider reported none, never epoch 0 — which `sweepOrphans` read as "older than any window" and
|
|
115
|
+
deleted. A reader that needs an age handles `undefined`; the sweep spares it.
|
|
116
|
+
|
|
72
117
|
**`StorageListEntry.contentType` is optional; `StorageObject.contentType` is not.** S3's
|
|
73
118
|
`ListObjectsV2` returns no Content-Type, so a listed s3 object simply has none — reading the real
|
|
74
119
|
value would cost one `HeadObject` per row, which is what `list()` exists to avoid. It used to
|
|
@@ -100,6 +145,12 @@ is the architecture and not an optimisation — see the round trip below. Raise
|
|
|
100
145
|
for a disk that really does write large objects server-side, and remember S3 caps a single PUT at
|
|
101
146
|
5GB whatever you set.
|
|
102
147
|
|
|
148
|
+
**`get()` buffers too, so it has the same ceiling**: `maxGetBytes`, on all three drivers,
|
|
149
|
+
defaulting to that disk's `maxPutBytes`. An object past it is `X_STORAGE_TOO_LARGE` — decided on
|
|
150
|
+
the disk's own size (a HEAD on s3) before a byte is read — and is read with `stream()`. It exists
|
|
151
|
+
because an object's size is not the server's to choose: a presigned PUT lands in a bucket
|
|
152
|
+
unmeasured, so a later `get()` of it was heap growth the uploader picked.
|
|
153
|
+
|
|
103
154
|
## Server-side encryption, storage classes, lifecycle
|
|
104
155
|
|
|
105
156
|
`PutOptions.serverSideEncryption` exists so the gap is visible **at the type level**, and every
|
|
@@ -123,7 +174,9 @@ it, `put('.meta/a/b.json', …)` overwrote the recorded content type of `a/b` an
|
|
|
123
174
|
that object answered attacker HTML from the app's own origin. No sanitising — a key that needed
|
|
124
175
|
fixing was built wrong.
|
|
125
176
|
`scopedKey('org-1', 'avatars', 'a.png')` is `org/org-1/avatars/a.png`; guard every
|
|
126
|
-
client-supplied key with `isWithinOrg(key, ctx.actor.orgId)
|
|
177
|
+
client-supplied key with `isWithinOrg(key, ctx.actor.orgId)` — a predicate that never throws: an
|
|
178
|
+
org id that cannot be one (empty, or carrying a separator) contains nothing, so an actor with no
|
|
179
|
+
org claim is `false` and not an `X_STORAGE_PATH_UNSAFE` blaming the key. A surface that serves objects pairs
|
|
127
180
|
it with `isTenantScoped(key)`: only a key already inside `org/` is another tenant's to refuse, so
|
|
128
181
|
`disk().put('brand/logo.png', …)` stays reachable while `org/org-2/…` never is. `accept.ts` asks
|
|
129
182
|
the pair too. `isTenantScoped` folds case (`Org/`, `ORG/`) and `isWithinOrg` does not, so a
|
|
@@ -132,9 +185,17 @@ case-variant prefix — one directory, not two, on APFS or NTFS — is refused r
|
|
|
132
185
|
## Signed URLs
|
|
133
186
|
|
|
134
187
|
The HMAC covers the **constraints**, not just the key —
|
|
135
|
-
`
|
|
188
|
+
`v2 \n basePath \n METHOD \n key \n expiresAt \n maxBytes \n contentType`.
|
|
136
189
|
A client that edits `?x-max=` invalidates the signature — it cannot widen what it was granted.
|
|
137
190
|
|
|
191
|
+
`basePath` is the disk: the PATH of the base the URL was minted under (`/_storage/<registered
|
|
192
|
+
name>`). Every local disk signs with the one `STORAGE_SIGNING_SECRET`, so without it a URL for
|
|
193
|
+
`uploads/<key>` verified on `private/<key>` with only the path segment edited. The origin is not
|
|
194
|
+
signed — the route that verifies is handed a path — and an absolute `baseUrl`
|
|
195
|
+
(`https://cdn.example.com/_storage/local`) is compared by its pathname (`signedUrlBasePath`), where
|
|
196
|
+
it used to make every URL minted under it `malformed`. `canonicalRequest` and `signConstraints`
|
|
197
|
+
take the base path as their last argument.
|
|
198
|
+
|
|
138
199
|
The HMAC key is `signingSecret`, else `STORAGE_SIGNING_SECRET`, else the shipped
|
|
139
200
|
`DEV_SIGNING_SECRET` — and **only in `development` or `test`**, read with a fallback of `production`: a process that names no environment at all (`ULTIMATE_ENV` and `NODE_ENV` both unset) is refused like a production one, `As of 2026-09-23`. Anywhere else `localDriver` refuses
|
|
140
201
|
to construct (`X_ENV_MISSING`) unless one of those two is set to something that is not the shipped
|
|
@@ -161,10 +222,20 @@ it on every grant, so refusing would break every s3 upload an app mints.
|
|
|
161
222
|
|
|
162
223
|
`Content-Type` is attacker-controlled. A `.png` that is really an HTML document is stored XSS
|
|
163
224
|
the moment a surface serves it back with the declared type. `validateUpload()` reads the magic
|
|
164
|
-
bytes (PNG, JPEG, GIF, WebP, PDF, ZIP/OOXML, SVG,
|
|
165
|
-
payload whose bytes contradict the declaration. Checks run cheapest-first: key → size →
|
|
225
|
+
bytes (PNG, JPEG, GIF, WebP, PDF, ZIP/OOXML, SVG, the ISO-BMFF family, HTML, plain text) and
|
|
226
|
+
rejects any payload whose bytes contradict the declaration. Checks run cheapest-first: key → size →
|
|
166
227
|
allowlist → sniff → checksum.
|
|
167
228
|
|
|
229
|
+
Every ISO base media file opens with the same `ftyp` box, so the box alone says "container" and
|
|
230
|
+
the **major brand** after it decides (`iso-bmff.ts`): `avif`/`avis` → `image/avif`, `heic`/`heix`…
|
|
231
|
+
→ `image/heic`, `qt ` → `video/quicktime`, `M4A `/`M4B ` → `audio/mp4`, `3gp*` → `video/3gpp`,
|
|
232
|
+
anything else → `video/mp4`. Two brands name only a container and stand in as `application/zip`
|
|
233
|
+
does for OOXML: `video/mp4` for a declared `audio/mp4` (an audio-only MP4 carries `isom`), and
|
|
234
|
+
`image/heif` (`mif1`) for `image/heic` and `image/avif`. `audio/x-m4a` and `video/x-m4v` — what a
|
|
235
|
+
browser reports for a picked file — normalise to `audio/mp4` and `video/mp4`, and
|
|
236
|
+
`uploadPolicy({ allowedContentTypes })` normalises its list the same way, so an alias in an
|
|
237
|
+
allowlist names the type it means.
|
|
238
|
+
|
|
168
239
|
`validateUpload({ key, declaredContentType, bytes }, uploadPolicy({ maxBytes: 5e6 }))`
|
|
169
240
|
|
|
170
241
|
## The direct-upload round trip
|
|
@@ -226,7 +297,7 @@ An upload happens **before** the row it belongs to exists, so it lands at
|
|
|
226
297
|
| `pendingKey(orgId, uploadName(id, filename))` | `org/o1/pending/u-1.png` |
|
|
227
298
|
| `quarantineKey(orgId, name)` | `org/o1/pending/quarantine/u-1.png` |
|
|
228
299
|
| `attachmentKey(orgId, { entity, id, field }, name)` | `org/o1/post/p-1/cover/u-1.png` |
|
|
229
|
-
| `promoteAttachment({ disk, key, orgId, target })` | `copy
|
|
300
|
+
| `promoteAttachment({ disk, key, orgId, target, policy })` | `stat`, `copy`, then `delete` — never the reverse. **`policy` is required**: the object is measured here (`stat().size` over `policy.maxBytes` is `X_STORAGE_TOO_LARGE`, and it stays under `pending/` for the sweep), because an s3 presign bounds no size. A retry after the source is gone answers the already-attached object instead of `X_STORAGE_NOT_FOUND`. The key must be a **pending** one (`isPendingKey`): an attached key a client sent back is another row's file, and moving it would delete the victim's copy (`X_STORAGE_NOT_PENDING`) |
|
|
230
301
|
| `releaseQuarantine({ disk, key, orgId })` | quarantine → pending; returns the released key |
|
|
231
302
|
| `sweepOrphans({ disk, orgId, olderThanMs })` | `{ deleted, failed }` for stale `pending/` keys. `olderThanMs` is a whole number of 0 or more, refused otherwise (`X_INVARIANT`) — `NaN` read as "everything is old enough" and deleted an upload made a moment ago |
|
|
232
303
|
|
|
@@ -266,7 +337,7 @@ Inside `pending/` deliberately: an upload nobody ever scanned is still an orphan
|
|
|
266
337
|
| `X_STORAGE_DISK_UNKNOWN` | `disk(name)` is not in `storage.disks`; cause lists the real ones |
|
|
267
338
|
| `X_STORAGE_NOT_FOUND` | `get`/`stream` on a key that does not exist |
|
|
268
339
|
| `X_STORAGE_PATH_UNSAFE` | traversal, absolute key, backslash, NUL, `%2e`, empty segment |
|
|
269
|
-
| `X_STORAGE_TOO_LARGE` | payload over the policy `maxBytes
|
|
340
|
+
| `X_STORAGE_TOO_LARGE` | payload over the policy `maxBytes` (at validation, or measured by `promoteAttachment`), or over a disk's `maxPutBytes` / `maxGetBytes` |
|
|
270
341
|
| `X_STORAGE_TYPE_REJECTED` | declared type off the allowlist, or contradicted by magic bytes |
|
|
271
342
|
| `X_STORAGE_CHECKSUM_MISMATCH` | supplied base64 SHA-256 does not describe the bytes |
|
|
272
343
|
| `X_STORAGE_URL_INVALID` | a signed request that does not match what was signed — edited constraint, wrong base, wrong method, contradicting `Content-Type` |
|
|
@@ -277,6 +348,9 @@ Inside `pending/` deliberately: an upload nobody ever scanned is still an orphan
|
|
|
277
348
|
| `X_STORAGE_LIST_FAILED` | the disk REFUSED a listing — denied `s3:ListBucket`, a throttle, an unreadable root. An **empty** disk is still not an error |
|
|
278
349
|
| `X_STORAGE_QUARANTINED` | `promoteAttachment` on a key nothing has released from `pending/quarantine/` |
|
|
279
350
|
| `X_STORAGE_NOT_PENDING` | `promoteAttachment` on a key outside the org's `pending/` prefix — most often another row's attached key |
|
|
351
|
+
| `X_STORAGE_PUT_FAILED` | the disk REFUSED a `put()`/`copy()` — `EACCES`, `ENOSPC`, `EROFS`, a provider's refused PUT |
|
|
352
|
+
| `X_STORAGE_READ_FAILED` | the disk REFUSED a `get`/`stat`/`exists`/`stream` or the read half of a `copy` — a denied `s3:GetObject`, a throttle, an unreadable file. An **absent** object is still `X_STORAGE_NOT_FOUND`, including one deleted between the existence check and the read |
|
|
353
|
+
| `X_STORAGE_KEY_CONFLICT` | local disk only: the key's path is another key's directory, or the reverse (`a` and `a/b`) |
|
|
280
354
|
| `X_NOT_IMPLEMENTED` | S3 user metadata / cache-control; `serverSideEncryption` on either driver |
|
|
281
355
|
| `X_ENV_MISSING` | core's: S3 credential env vars, or a `localDriver` built outside development where neither `signingSecret` nor `STORAGE_SIGNING_SECRET` holds a secret other than the published `DEV_SIGNING_SECRET` |
|
|
282
356
|
| `X_IMAGE_UNSUPPORTED` | core's: an `avif` encode, a source no built-in decoder reads, or a `variantKey` format no variant can carry |
|
|
@@ -296,6 +370,13 @@ a job boundary the class is gone and the `code` is what survives — match on th
|
|
|
296
370
|
`variantKey()`, `srcsetDescriptors()`, `fitDimensions()` are pure — `@ultimat3/seo` builds
|
|
297
371
|
`srcset` from them without decoding a byte.
|
|
298
372
|
|
|
373
|
+
A variant key is the WHOLE source key plus the transform: `photos/hero.png` at width 640 is
|
|
374
|
+
`photos/hero.png@w640.webp`. The source extension stays in, so `hero.png` and `hero.jpg` are two
|
|
375
|
+
cache identities — it was cut, and both minted `photos/hero@w640.webp`. `width` and `height` are
|
|
376
|
+
whole numbers of at least 1 on `variantKey` and `fitDimensions` (`X_INVARIANT` otherwise — no
|
|
377
|
+
`wNaN`, no `w1.5`). `fitDimensions` never answers a zero edge (1000×1 at width 100 is 100×1) and
|
|
378
|
+
never upscales except under `fit: 'cover'`, which is the exact box asked for.
|
|
379
|
+
|
|
299
380
|
**`VARIANT_FORMATS` / `VariantFormat` are what a variant KEY can carry, `As of 2026-08`** —
|
|
300
381
|
`avif`, `webp`, `jpeg`, `png` — and this package exports no `IMAGE_FORMATS` and no `ImageFormat`. Those two names are
|
|
301
382
|
`@ultimat3/core`'s, over the six formats it can PROBE (`png`, `jpeg`, `webp`, `avif`, `gif`,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/storage",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "24.0.0",
|
|
4
4
|
"description": "Named disks over Bun.file and Bun.s3: safe keys, signed URLs, sniffed uploads",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -32,6 +32,6 @@
|
|
|
32
32
|
"test": "bun test"
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@ultimat3/core": "
|
|
35
|
+
"@ultimat3/core": "24.0.0"
|
|
36
36
|
}
|
|
37
37
|
}
|
package/src/accept.ts
CHANGED
|
@@ -113,12 +113,9 @@ async function constraintsFor(
|
|
|
113
113
|
// The PAIR is the question "does this key belong to somebody else?". `isWithinOrg` alone
|
|
114
114
|
// answered `false` for every un-scoped key, so an app's own `brand/logo.png` was unreachable
|
|
115
115
|
// through a URL it had just signed — `path.ts` says so and `dev-storage.ts` already asks it this
|
|
116
|
-
// way. An actor with no org is inside no org,
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
const orgId = input.orgId;
|
|
120
|
-
if (isTenantScoped(constraints.key) && (orgId === '' || !isWithinOrg(constraints.key, orgId))) {
|
|
121
|
-
throw orgMismatch(constraints.key, orgId);
|
|
116
|
+
// way. An actor with no org is inside no org, and `isWithinOrg` answers exactly that.
|
|
117
|
+
if (isTenantScoped(constraints.key) && !isWithinOrg(constraints.key, input.orgId)) {
|
|
118
|
+
throw orgMismatch(constraints.key, input.orgId);
|
|
122
119
|
}
|
|
123
120
|
return constraints;
|
|
124
121
|
}
|
package/src/attachment.ts
CHANGED
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
// so the tenant prefix is a construction, not a check somebody remembered to write.
|
|
6
6
|
|
|
7
7
|
import type { Clock } from '@ultimat3/core';
|
|
8
|
-
import { finiteCount, renderThrowable, systemClock } from '@ultimat3/core';
|
|
8
|
+
import { assert, finiteCount, renderThrowable, systemClock } from '@ultimat3/core';
|
|
9
9
|
import type { ListPage, StorageDriver, StorageListEntry, StorageObject } from './driver';
|
|
10
|
-
import { notPending, orgMismatch, quarantined } from './errors';
|
|
10
|
+
import { notPending, objectNotFound, orgMismatch, quarantined, tooLarge } from './errors';
|
|
11
11
|
import { isWithinOrg, orgPrefix, scopedKey } from './path';
|
|
12
|
+
import type { UploadPolicy } from './upload';
|
|
12
13
|
|
|
13
14
|
/** The one segment an unattached upload lives under. `sweepOrphans` reads only this prefix. */
|
|
14
15
|
export const PENDING_SEGMENT = 'pending';
|
|
@@ -123,13 +124,27 @@ export interface PromoteAttachmentInput {
|
|
|
123
124
|
readonly key: string;
|
|
124
125
|
readonly orgId: string;
|
|
125
126
|
readonly target: AttachmentTarget;
|
|
127
|
+
/**
|
|
128
|
+
* The policy the upload was GRANTED under — required, never defaulted. On a bucket-backed disk
|
|
129
|
+
* the presigned PUT carries no size, so `policy.maxBytes` bound nothing until this call measured
|
|
130
|
+
* the object; a default here would silently replace an app's own ceiling with the framework's.
|
|
131
|
+
*/
|
|
132
|
+
readonly policy: UploadPolicy;
|
|
126
133
|
}
|
|
127
134
|
|
|
128
135
|
/**
|
|
129
136
|
* Move a pending upload onto the row that now exists. Copy first, delete second: a delete that
|
|
130
137
|
* ran first would lose the bytes on a failed write, and the sweep would have collected them
|
|
131
|
-
* anyway.
|
|
132
|
-
*
|
|
138
|
+
* anyway.
|
|
139
|
+
*
|
|
140
|
+
* MEASURED before it moves: `stat().size` over `policy.maxBytes` is `X_STORAGE_TOO_LARGE`, and the
|
|
141
|
+
* object stays under `pending/` for `sweepOrphans` — never at a key a row points to. `stat` and
|
|
142
|
+
* not `get`: this is the call that learns the size, so it cannot be one that buffers it.
|
|
143
|
+
*
|
|
144
|
+
* Idempotent across a retry: a source that is gone with the destination already there answers the
|
|
145
|
+
* destination. Promotion is copy-then-delete inside a request whose row write can still roll back,
|
|
146
|
+
* and the retry used to raise `X_STORAGE_NOT_FOUND` for a file sitting exactly where it belonged.
|
|
147
|
+
* A source that is gone with NO destination is still `X_STORAGE_NOT_FOUND`.
|
|
133
148
|
*
|
|
134
149
|
* The copy is `disk.copy`, not `get` + `put`: promotion used to download the whole object into
|
|
135
150
|
* this process and upload it again, so attaching a 500MB file moved a gigabyte through the pod
|
|
@@ -141,10 +156,27 @@ export async function promoteAttachment(input: PromoteAttachmentInput): Promise<
|
|
|
141
156
|
// copy-then-delete below would move that row's file here and delete theirs.
|
|
142
157
|
if (!isPendingKey(input.key, input.orgId)) throw notPending(input.key, input.orgId);
|
|
143
158
|
if (isQuarantinedKey(input.key, input.orgId)) throw quarantined(input.key, input.orgId);
|
|
159
|
+
// The type says required; an untyped caller upgrading from the four-field call does not read
|
|
160
|
+
// types, and `input.policy.maxBytes` on `undefined` is a bare `TypeError` two lines down.
|
|
161
|
+
assert(
|
|
162
|
+
typeof input.policy?.maxBytes === 'number',
|
|
163
|
+
'promoteAttachment was called with no policy, so nothing bounds the size of the upload it attaches',
|
|
164
|
+
'promoteAttachment({ disk, key, orgId, target, policy: uploadPolicy({ maxBytes }) }) — the policy the upload was granted under',
|
|
165
|
+
);
|
|
166
|
+
const disk = input.disk;
|
|
144
167
|
const name = input.key.slice(input.key.lastIndexOf('/') + 1);
|
|
145
168
|
const attached = attachmentKey(input.orgId, input.target, name);
|
|
146
|
-
const
|
|
147
|
-
|
|
169
|
+
const pending = await disk.stat(input.key);
|
|
170
|
+
if (pending === undefined) {
|
|
171
|
+
const already = await disk.stat(attached);
|
|
172
|
+
if (already !== undefined) return already;
|
|
173
|
+
throw objectNotFound(disk.name, input.key);
|
|
174
|
+
}
|
|
175
|
+
if (pending.size > input.policy.maxBytes) {
|
|
176
|
+
throw tooLarge(input.key, pending.size, input.policy.maxBytes);
|
|
177
|
+
}
|
|
178
|
+
const object = await disk.copy(input.key, attached);
|
|
179
|
+
await disk.delete(input.key);
|
|
148
180
|
return object;
|
|
149
181
|
}
|
|
150
182
|
|
|
@@ -204,7 +236,8 @@ export async function sweepOrphans(input: SweepOrphansInput): Promise<SweepResul
|
|
|
204
236
|
...(cursor === undefined ? {} : { cursor }),
|
|
205
237
|
});
|
|
206
238
|
for (const object of page.objects) {
|
|
207
|
-
|
|
239
|
+
// An object whose age the provider did not report is SPARED: only a proven age is swept.
|
|
240
|
+
if (object.lastModified === undefined || object.lastModified.getTime() > cutoff) continue;
|
|
208
241
|
if ((await input.keep?.(object)) === true) continue;
|
|
209
242
|
try {
|
|
210
243
|
await input.disk.delete(object.key);
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// Single responsibility: what the local disk KNOWS about one object — its bytes file, its sidecar,
|
|
2
|
+
// and the pending marker that says whether the two can be trusted as a pair. A write that died
|
|
3
|
+
// between its renames leaves a sidecar describing bytes that never landed; this is where that is
|
|
4
|
+
// read as "type unknown" instead of served as the truth.
|
|
5
|
+
|
|
6
|
+
import { renderFixShellArg, stringField } from '@ultimat3/core';
|
|
7
|
+
import type { StorageListEntry } from './driver';
|
|
8
|
+
import { parseSidecar, type Sidecar } from './driver-local-sidecar';
|
|
9
|
+
import { pendingPathOf, sidecarPathOf } from './driver-local-write';
|
|
10
|
+
import { isStorageError, objectNotFound, readFailed } from './errors';
|
|
11
|
+
|
|
12
|
+
async function readSidecar(root: string, key: string): Promise<Sidecar | undefined> {
|
|
13
|
+
const file = Bun.file(`${root}/${sidecarPathOf(key)}`);
|
|
14
|
+
if (!(await file.exists())) return undefined;
|
|
15
|
+
try {
|
|
16
|
+
const raw: unknown = await file.json();
|
|
17
|
+
return parseSidecar(raw);
|
|
18
|
+
} catch {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** The etag a write in flight — or one that died — announced before it moved anything. */
|
|
24
|
+
async function pendingEtag(root: string, key: string): Promise<string | undefined> {
|
|
25
|
+
const file = Bun.file(`${root}/${pendingPathOf(key)}`);
|
|
26
|
+
if (!(await file.exists())) return undefined;
|
|
27
|
+
return (await file.text().catch(() => '')).trim();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* A read the OS refused, coded: `EACCES`, `EIO`, a directory where the bytes should be. A file
|
|
32
|
+
* that is simply gone — deleted between the `exists()` and the read — is "not found", as it
|
|
33
|
+
* would have been one call earlier.
|
|
34
|
+
*/
|
|
35
|
+
const refusedRead =
|
|
36
|
+
(root: string, key: string) =>
|
|
37
|
+
(error: unknown): never => {
|
|
38
|
+
if (isStorageError(error)) throw error;
|
|
39
|
+
throw stringField(error, 'code') === 'ENOENT'
|
|
40
|
+
? objectNotFound('local', key)
|
|
41
|
+
: readFailed(
|
|
42
|
+
'local',
|
|
43
|
+
key,
|
|
44
|
+
error,
|
|
45
|
+
`ls -l ${renderFixShellArg(`${root}/${key}`, '<the object file>')}`,
|
|
46
|
+
);
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/** The whole object, buffered — `get()`'s one read. */
|
|
50
|
+
export async function readObjectBytes(root: string, key: string): Promise<Uint8Array> {
|
|
51
|
+
return Bun.file(`${root}/${key}`)
|
|
52
|
+
.arrayBuffer()
|
|
53
|
+
.then((buffer) => new Uint8Array(buffer), refusedRead(root, key));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** `etagOf`, streamed: a copy or a `stat()` must settle a doubt without buffering the object. */
|
|
57
|
+
export async function etagOfFile(root: string, key: string): Promise<string> {
|
|
58
|
+
const hasher = new Bun.CryptoHasher('sha256');
|
|
59
|
+
try {
|
|
60
|
+
for await (const chunk of Bun.file(`${root}/${key}`).stream()) hasher.update(chunk);
|
|
61
|
+
} catch (error) {
|
|
62
|
+
return refusedRead(root, key)(error);
|
|
63
|
+
}
|
|
64
|
+
return hasher.digest('hex').slice(0, 32);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* One object's facts, or `undefined` when there are no bytes at the key — the bytes file is what
|
|
69
|
+
* makes an object exist, so a sidecar alone (a fresh write that died early) is nothing at all.
|
|
70
|
+
*
|
|
71
|
+
* The sidecar is TRUSTED unless the pending marker names its etag: that is the one state in which
|
|
72
|
+
* the sidecar is the new write's and the bytes may still be the old object's. Then, and for an
|
|
73
|
+
* object with no sidecar, the answer depends on `trueEtag`:
|
|
74
|
+
* - absent (a listing, which never reads bytes): no content type, `etag: ''` — "cannot know";
|
|
75
|
+
* - given: the bytes decide. A match keeps the sidecar; a mismatch drops every field it claimed
|
|
76
|
+
* and reports the bytes' own etag.
|
|
77
|
+
*
|
|
78
|
+
* No path here answers a type or an etag the bytes were not checked against while in doubt.
|
|
79
|
+
*/
|
|
80
|
+
export async function headObject(
|
|
81
|
+
root: string,
|
|
82
|
+
key: string,
|
|
83
|
+
trueEtag?: () => string | Promise<string>,
|
|
84
|
+
): Promise<StorageListEntry | undefined> {
|
|
85
|
+
const file = Bun.file(`${root}/${key}`);
|
|
86
|
+
if (!(await file.exists())) return undefined;
|
|
87
|
+
const recorded = await readSidecar(root, key);
|
|
88
|
+
const inDoubt = recorded !== undefined && (await pendingEtag(root, key)) === recorded.etag;
|
|
89
|
+
let sidecar = recorded;
|
|
90
|
+
let etag = recorded?.etag ?? '';
|
|
91
|
+
if (recorded === undefined || inDoubt) {
|
|
92
|
+
etag = trueEtag === undefined ? '' : await trueEtag();
|
|
93
|
+
if (recorded?.etag !== etag) sidecar = undefined;
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
key,
|
|
97
|
+
size: file.size,
|
|
98
|
+
etag,
|
|
99
|
+
lastModified: new Date(file.lastModified),
|
|
100
|
+
...(sidecar?.contentType === undefined ? {} : { contentType: sidecar.contentType }),
|
|
101
|
+
...(sidecar?.cacheControl === undefined ? {} : { cacheControl: sidecar.cacheControl }),
|
|
102
|
+
...(sidecar?.metadata === undefined ? {} : { metadata: sidecar.metadata }),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Single responsibility: the local disk's sidecar — the JSON file under `.meta/` holding what a
|
|
2
|
+
// POSIX file has nowhere to keep (content type, etag, cache-control, user metadata), and the one
|
|
3
|
+
// parser that reads it back as untrusted bytes off a disk.
|
|
4
|
+
|
|
5
|
+
export interface Sidecar {
|
|
6
|
+
readonly contentType: string;
|
|
7
|
+
readonly etag: string;
|
|
8
|
+
readonly cacheControl?: string | undefined;
|
|
9
|
+
readonly metadata?: Readonly<Record<string, string>> | undefined;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// `!Array.isArray` is the load-bearing clause, matching `isPlainObject` in
|
|
13
|
+
// `@ultimat3/schema`'s `builder.ts`: `typeof [] === 'object'` and every value of `['a','b']` is a
|
|
14
|
+
// string, so an array in the `metadata` slot was handed back through `head()`/`get()` as object
|
|
15
|
+
// metadata — against a `Record<string, string>` every reader downstream is typed on.
|
|
16
|
+
const isStringRecord = (value: unknown): value is Readonly<Record<string, string>> =>
|
|
17
|
+
typeof value === 'object' &&
|
|
18
|
+
value !== null &&
|
|
19
|
+
!Array.isArray(value) &&
|
|
20
|
+
Object.values(value as Record<string, unknown>).every((entry) => typeof entry === 'string');
|
|
21
|
+
|
|
22
|
+
export function parseSidecar(raw: unknown): Sidecar | undefined {
|
|
23
|
+
if (typeof raw !== 'object' || raw === null) return undefined;
|
|
24
|
+
const record = raw as Record<string, unknown>;
|
|
25
|
+
const contentType = record['contentType'];
|
|
26
|
+
const etag = record['etag'];
|
|
27
|
+
if (typeof contentType !== 'string' || typeof etag !== 'string') return undefined;
|
|
28
|
+
// `put()` writes cacheControl/metadata into the same sidecar — dropping them here silently
|
|
29
|
+
// truncated what was just written, even though `Sidecar` itself declares both.
|
|
30
|
+
const cacheControl = record['cacheControl'];
|
|
31
|
+
const metadata = record['metadata'];
|
|
32
|
+
return {
|
|
33
|
+
contentType,
|
|
34
|
+
etag,
|
|
35
|
+
...(typeof cacheControl === 'string' ? { cacheControl } : {}),
|
|
36
|
+
...(isStringRecord(metadata) ? { metadata } : {}),
|
|
37
|
+
};
|
|
38
|
+
}
|