@ultimat3/storage 30.9.0 → 31.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +56 -113
- package/README.md +117 -24
- package/package.json +4 -3
- package/src/accept-stream.ts +125 -0
- package/src/accept.ts +24 -6
- package/src/adapter.ts +46 -0
- package/src/attachment.ts +43 -61
- package/src/driver-local-list.ts +1 -1
- package/src/driver-local-open.ts +112 -0
- package/src/driver-local-read.ts +1 -1
- package/src/driver-local-stream.ts +14 -3
- package/src/driver-local.ts +48 -16
- package/src/driver-memory.ts +39 -9
- package/src/driver-s3-copy-parts.ts +177 -0
- package/src/driver-s3-copy.ts +14 -7
- package/src/driver-s3-lock.ts +1 -3
- package/src/driver-s3-put-stream.ts +81 -17
- package/src/driver-s3-range.ts +14 -7
- package/src/driver-s3-signed.ts +1 -1
- package/src/driver-s3-stream.ts +36 -0
- package/src/driver-s3.ts +44 -27
- package/src/driver.ts +30 -5
- package/src/errors-promote.ts +3 -15
- package/src/errors-scan.ts +61 -0
- package/src/errors-stream.ts +13 -0
- package/src/errors-variant.ts +40 -0
- package/src/errors.ts +6 -0
- package/src/grant.ts +15 -10
- package/src/image-variant.ts +106 -0
- package/src/index.ts +21 -40
- package/src/path.ts +27 -1
- package/src/request-ceiling.ts +45 -0
- package/src/storage-instrument.ts +108 -0
- package/src/storage.ts +8 -2
- package/src/upload-scan.ts +91 -0
- package/src/upload-scanner-memory.ts +62 -0
- package/src/upload-scanner.ts +71 -0
- package/src/upload.ts +35 -2
package/src/grant.ts
CHANGED
|
@@ -7,13 +7,15 @@
|
|
|
7
7
|
import type { Clock } from '@ultimat3/core';
|
|
8
8
|
import { finiteCount, systemClock } from '@ultimat3/core';
|
|
9
9
|
import type { AttachmentTarget } from './attachment';
|
|
10
|
-
import { attachmentKey, pendingKey,
|
|
10
|
+
import { attachmentKey, pendingKey, uploadName } from './attachment';
|
|
11
11
|
import type { StorageDriver } from './driver';
|
|
12
12
|
import { contentTypeNotAllowed, tooLarge } from './errors';
|
|
13
|
+
import { targetUnscanned } from './errors-scan';
|
|
13
14
|
import { uploadSizeUndeclared } from './errors-stream';
|
|
14
15
|
import { DEFAULT_SIGNED_URL_TTL_MS } from './signed-url';
|
|
15
16
|
import type { UploadPolicy } from './upload';
|
|
16
17
|
import { normalizeContentType, uploadPolicy } from './upload';
|
|
18
|
+
import { uploadScanner } from './upload-scanner';
|
|
17
19
|
|
|
18
20
|
export interface UploadRequest {
|
|
19
21
|
/** The client's own name for the file. Used for its extension and nothing else. */
|
|
@@ -46,14 +48,12 @@ export interface GrantUploadInput {
|
|
|
46
48
|
readonly orgId: string;
|
|
47
49
|
readonly request: UploadRequest;
|
|
48
50
|
readonly policy?: UploadPolicy | undefined;
|
|
49
|
-
/** Absent means the row does not exist yet, so the key lands under `pending/`. */
|
|
50
|
-
readonly target?: AttachmentTarget | undefined;
|
|
51
51
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
52
|
+
* Absent means the row does not exist yet, so the key lands under `pending/`. Refused while an
|
|
53
|
+
* `uploadScanner` is installed (`X_STORAGE_TARGET_UNSCANNED`): a targeted key skips
|
|
54
|
+
* `promoteAttachment`, the one place a scan runs.
|
|
55
55
|
*/
|
|
56
|
-
readonly
|
|
56
|
+
readonly target?: AttachmentTarget | undefined;
|
|
57
57
|
readonly expiresInMs?: number | undefined;
|
|
58
58
|
readonly clock?: Clock | undefined;
|
|
59
59
|
/** Determinism seam for tests. Defaults to `crypto.randomUUID()`. */
|
|
@@ -82,9 +82,14 @@ export function decideGrant(caller: string, input: GrantUploadInput): GrantedUpl
|
|
|
82
82
|
(input.uploadId ?? (() => crypto.randomUUID()))(),
|
|
83
83
|
input.request.filename,
|
|
84
84
|
);
|
|
85
|
-
const
|
|
86
|
-
input.
|
|
87
|
-
|
|
85
|
+
const key =
|
|
86
|
+
input.target === undefined
|
|
87
|
+
? pendingKey(input.orgId, name)
|
|
88
|
+
: attachmentKey(input.orgId, input.target, name);
|
|
89
|
+
const scanner = uploadScanner();
|
|
90
|
+
if (input.target !== undefined && scanner !== undefined) {
|
|
91
|
+
throw targetUnscanned(caller, key, scanner.name);
|
|
92
|
+
}
|
|
88
93
|
|
|
89
94
|
if (!policy.allowedContentTypes.includes(declared)) {
|
|
90
95
|
throw contentTypeNotAllowed(key, declared, policy.allowedContentTypes);
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Single responsibility: `variant()` — one resized image, made by the app's image driver and kept on
|
|
2
|
+
// the disk under `variantKey(…, { revision })`, so the second ask is a `stat` and never a decode.
|
|
3
|
+
// The driver is an ARGUMENT, structurally `@ultimat3/seo`'s `ImageTransformDriver`: storage is tier
|
|
4
|
+
// 1 beside seo and imports none of it; the `images` port value is what an app passes.
|
|
5
|
+
|
|
6
|
+
import { assert, finiteCount, IMAGE_MIME_TYPES } from '@ultimat3/core';
|
|
7
|
+
import type { StorageDriver, StorageObject } from './driver';
|
|
8
|
+
import { objectNotFound } from './errors';
|
|
9
|
+
import {
|
|
10
|
+
driverEncodingMismatch,
|
|
11
|
+
notARaster,
|
|
12
|
+
variantFormatUnknown,
|
|
13
|
+
variantSourceChanged,
|
|
14
|
+
} from './errors-variant';
|
|
15
|
+
import { type VariantFormat, variantFormatOf, variantKey, variantRevision } from './image';
|
|
16
|
+
import { normalizeContentType } from './upload';
|
|
17
|
+
|
|
18
|
+
/** `@ultimat3/seo`'s `TransformRequest`, field for field: `src` is the SOURCE storage key. */
|
|
19
|
+
export interface VariantRequest {
|
|
20
|
+
src: string;
|
|
21
|
+
width: number;
|
|
22
|
+
format?: string;
|
|
23
|
+
quality?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** `@ultimat3/seo`'s `TransformedImage`, field for field. */
|
|
27
|
+
export interface VariantBytes {
|
|
28
|
+
bytes: Uint8Array;
|
|
29
|
+
contentType: string;
|
|
30
|
+
width: number;
|
|
31
|
+
height: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The half of `ImageTransformDriver` this needs — so `runtime.ts`'s `images` driver, a
|
|
36
|
+
* `builtinImageDriver({ read: (src) => disk.get(src).then((r) => r.bytes) })` or a CDN's, is passed
|
|
37
|
+
* as it is. Its `read` must resolve `src` against the SAME disk handed to `variant()`.
|
|
38
|
+
*/
|
|
39
|
+
export interface VariantTransformer {
|
|
40
|
+
readonly name: string;
|
|
41
|
+
transform(request: VariantRequest): Promise<VariantBytes>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface VariantOptions {
|
|
45
|
+
readonly disk: StorageDriver;
|
|
46
|
+
readonly images: VariantTransformer;
|
|
47
|
+
/** Whole pixels, at least one. A width above the source's is the driver's to clamp. */
|
|
48
|
+
readonly width: number;
|
|
49
|
+
/** Omitted: the source's own encoding (png, jpeg, webp); a gif or avif source must name one. */
|
|
50
|
+
readonly format?: VariantFormat | undefined;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface StoredVariant {
|
|
54
|
+
readonly key: string;
|
|
55
|
+
readonly object: StorageObject;
|
|
56
|
+
/** True when THIS call transformed and wrote it; false when it was already on the disk. */
|
|
57
|
+
readonly created: boolean;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** What a driver can resample: every raster core can probe. An svg has no pixels to resize. */
|
|
61
|
+
const RASTER_TYPES: ReadonlySet<string> = new Set(
|
|
62
|
+
Object.entries(IMAGE_MIME_TYPES)
|
|
63
|
+
.filter(([format]) => format !== 'svg')
|
|
64
|
+
.map(([, type]) => type),
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* The variant of `key` at `width` (and `format`), from the disk if it is there, else transformed by
|
|
69
|
+
* `images` and written. Idempotent: the key is derived from the source's revision, so a repeat is a
|
|
70
|
+
* hit and an overwritten source is a different key. Concurrent first calls both write the same key
|
|
71
|
+
* with the same bytes — harmless, and cheaper than a lock.
|
|
72
|
+
*/
|
|
73
|
+
export async function variant(key: string, options: VariantOptions): Promise<StoredVariant> {
|
|
74
|
+
const width = finiteCount('variant', 'width', options.width, 1);
|
|
75
|
+
const { disk, images } = options;
|
|
76
|
+
const source = await disk.stat(key);
|
|
77
|
+
if (source === undefined) throw objectNotFound(disk.name, key);
|
|
78
|
+
const sourceType = normalizeContentType(source.contentType);
|
|
79
|
+
if (!RASTER_TYPES.has(sourceType)) throw notARaster(key, sourceType);
|
|
80
|
+
const format = options.format === undefined ? variantFormatOf(sourceType) : options.format;
|
|
81
|
+
if (format === undefined) throw variantFormatUnknown(key, sourceType);
|
|
82
|
+
const revision = variantRevision(source.etag);
|
|
83
|
+
assert(
|
|
84
|
+
revision !== undefined,
|
|
85
|
+
`variant('${key}'): the source etag "${source.etag}" carries no 8-hex revision, so its variant has no key that an overwrite would miss`,
|
|
86
|
+
`x doctor --json # the disk driver answers no content etag; serve ${key} through /media/* instead, which transforms without storing`,
|
|
87
|
+
);
|
|
88
|
+
const target = variantKey(key, { width, format, revision });
|
|
89
|
+
|
|
90
|
+
const stored = await disk.stat(target);
|
|
91
|
+
if (stored !== undefined) return { key: target, object: stored, created: false };
|
|
92
|
+
|
|
93
|
+
const made = await images.transform({ src: key, width, format });
|
|
94
|
+
const expected = IMAGE_MIME_TYPES[format];
|
|
95
|
+
if (normalizeContentType(made.contentType) !== expected) {
|
|
96
|
+
throw driverEncodingMismatch(key, images.name, expected, made.contentType);
|
|
97
|
+
}
|
|
98
|
+
// The driver read the source itself: bracketing it with two stats is how these bytes are known to
|
|
99
|
+
// be the revision the key names, and not an overwrite that landed while it read.
|
|
100
|
+
const after = await disk.stat(key);
|
|
101
|
+
if (after?.etag !== source.etag) {
|
|
102
|
+
throw variantSourceChanged(key, source.etag, after?.etag ?? 'deleted');
|
|
103
|
+
}
|
|
104
|
+
const object = await disk.put(target, made.bytes, { contentType: expected });
|
|
105
|
+
return { key: target, object, created: true };
|
|
106
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -5,11 +5,12 @@ export type { AcceptSignedUploadInput, SignedRequestInput } from './accept';
|
|
|
5
5
|
export { acceptSignedUpload, readSignedObject, signedUploadConstraints } from './accept';
|
|
6
6
|
export type { AcceptSignedPartInput } from './accept-part';
|
|
7
7
|
export { acceptSignedPart } from './accept-part';
|
|
8
|
+
export type { AcceptSignedUploadStreamInput } from './accept-stream';
|
|
9
|
+
export { acceptSignedUploadStream } from './accept-stream';
|
|
8
10
|
export type {
|
|
9
11
|
AttachmentTarget,
|
|
10
12
|
DeleteVariantsInput,
|
|
11
13
|
PromoteAttachmentInput,
|
|
12
|
-
ReleaseQuarantineInput,
|
|
13
14
|
SweepFailure,
|
|
14
15
|
SweepOrphansInput,
|
|
15
16
|
SweepResult,
|
|
@@ -19,17 +20,10 @@ export {
|
|
|
19
20
|
attachmentPrefix,
|
|
20
21
|
deleteVariants,
|
|
21
22
|
isPendingKey,
|
|
22
|
-
isQuarantinedKey,
|
|
23
23
|
PENDING_SEGMENT,
|
|
24
24
|
pendingKey,
|
|
25
|
-
pendingPrefix,
|
|
26
25
|
promoteAttachment,
|
|
27
|
-
QUARANTINE_SEGMENT,
|
|
28
|
-
quarantineKey,
|
|
29
|
-
quarantinePrefix,
|
|
30
|
-
releaseQuarantine,
|
|
31
26
|
sweepOrphans,
|
|
32
|
-
uploadExtension,
|
|
33
27
|
uploadName,
|
|
34
28
|
} from './attachment';
|
|
35
29
|
export type { PumpLimit } from './byte-pump';
|
|
@@ -47,13 +41,13 @@ export type {
|
|
|
47
41
|
StorageListEntry,
|
|
48
42
|
StorageObject,
|
|
49
43
|
StorageRead,
|
|
44
|
+
StreamOptions,
|
|
50
45
|
} from './driver';
|
|
51
46
|
export {
|
|
52
47
|
// Exported for the same reason `toBytes` is: a driver written outside this package has to
|
|
53
48
|
// refuse a `limit` the same way both shipped ones do, or it is a third answer to one question.
|
|
54
49
|
assertPutOptions,
|
|
55
50
|
DEFAULT_CONTENT_TYPE,
|
|
56
|
-
DEFAULT_LIST_LIMIT,
|
|
57
51
|
etagOf,
|
|
58
52
|
resolveListLimit,
|
|
59
53
|
sha256Base64,
|
|
@@ -77,36 +71,21 @@ export type { S3FetchLike } from './driver-s3-signed';
|
|
|
77
71
|
export type { StorageErrorCode, StorageErrorInit } from './errors';
|
|
78
72
|
export {
|
|
79
73
|
checksumMismatch,
|
|
80
|
-
contentTypeMismatch,
|
|
81
|
-
contentTypeNotAllowed,
|
|
82
|
-
contentTypeUnrecognised,
|
|
83
74
|
deleteFailed,
|
|
84
|
-
diskUnknown,
|
|
85
75
|
getTooLarge,
|
|
86
76
|
isStorageError,
|
|
87
77
|
keyConflict,
|
|
88
78
|
keyUnshared,
|
|
89
|
-
listFailed,
|
|
90
79
|
objectNotFound,
|
|
91
80
|
orgMismatch,
|
|
92
|
-
pathUnsafe,
|
|
93
81
|
putFailed,
|
|
94
|
-
putTooLarge,
|
|
95
82
|
readFailed,
|
|
96
|
-
STORAGE_ERROR_CODES,
|
|
97
|
-
STORAGE_ERROR_TITLES,
|
|
98
83
|
StorageError,
|
|
99
|
-
signedUrlExpired,
|
|
100
|
-
signedUrlRejected,
|
|
101
|
-
signedUrlUnverifiable,
|
|
102
|
-
signingSecretMissing,
|
|
103
84
|
tooLarge,
|
|
104
|
-
uploadFailed,
|
|
105
85
|
xmlBodyUnreadable,
|
|
106
86
|
} from './errors';
|
|
107
|
-
export {
|
|
108
|
-
export {
|
|
109
|
-
export { rangeTooLarge, rangeUnsatisfiable } from './errors-range';
|
|
87
|
+
export { alreadyPromoted } from './errors-promote';
|
|
88
|
+
export { rangeTooLarge } from './errors-range';
|
|
110
89
|
export type {
|
|
111
90
|
AbortResult,
|
|
112
91
|
AbortStaleUploadsInput,
|
|
@@ -156,6 +135,14 @@ export {
|
|
|
156
135
|
variantRevision,
|
|
157
136
|
variantSourceOf,
|
|
158
137
|
} from './image';
|
|
138
|
+
export type {
|
|
139
|
+
StoredVariant,
|
|
140
|
+
VariantBytes,
|
|
141
|
+
VariantOptions,
|
|
142
|
+
VariantRequest,
|
|
143
|
+
VariantTransformer,
|
|
144
|
+
} from './image-variant';
|
|
145
|
+
export { variant } from './image-variant';
|
|
159
146
|
export type { WarmFailure, WarmVariantsInput, WarmVariantsResult } from './image-warm';
|
|
160
147
|
export { warmVariants } from './image-warm';
|
|
161
148
|
export type {
|
|
@@ -165,7 +152,7 @@ export type {
|
|
|
165
152
|
SignPartOptions,
|
|
166
153
|
UploadedPart,
|
|
167
154
|
} from './multipart';
|
|
168
|
-
export {
|
|
155
|
+
export { MAX_PARTS, MIN_PART_BYTES, partBytesFor } from './multipart';
|
|
169
156
|
export type {
|
|
170
157
|
LockedAction,
|
|
171
158
|
ObjectLock,
|
|
@@ -173,19 +160,17 @@ export type {
|
|
|
173
160
|
ObjectRetention,
|
|
174
161
|
RetentionMode,
|
|
175
162
|
} from './object-lock';
|
|
176
|
-
export { isLocked, ObjectLockedError
|
|
163
|
+
export { isLocked, ObjectLockedError } from './object-lock';
|
|
177
164
|
export {
|
|
178
165
|
assertSafeKey,
|
|
179
166
|
isSafeKey,
|
|
180
167
|
isTenantScoped,
|
|
181
168
|
isWithinOrg,
|
|
182
169
|
joinKey,
|
|
183
|
-
keyDirname,
|
|
184
170
|
keyExtname,
|
|
185
171
|
MAX_KEY_LENGTH,
|
|
186
172
|
META_DIR,
|
|
187
173
|
ORG_PREFIX,
|
|
188
|
-
orgPrefix,
|
|
189
174
|
scopedKey,
|
|
190
175
|
} from './path';
|
|
191
176
|
export type { PutStreamOptions } from './put-stream';
|
|
@@ -195,7 +180,8 @@ export { assertPutStream, streamLimit } from './put-stream';
|
|
|
195
180
|
* fourth answer to what a range means.
|
|
196
181
|
*/
|
|
197
182
|
export type { ByteRange, GetOptions } from './range';
|
|
198
|
-
export {
|
|
183
|
+
export { assertStreamRange, streamedRange } from './range';
|
|
184
|
+
export type { RequestCeiling } from './request-ceiling';
|
|
199
185
|
export type {
|
|
200
186
|
SignedPart,
|
|
201
187
|
SignedUrlConstraints,
|
|
@@ -208,10 +194,7 @@ export {
|
|
|
208
194
|
buildSignedUrl,
|
|
209
195
|
canonicalRequest,
|
|
210
196
|
DEFAULT_SIGNED_URL_BASE,
|
|
211
|
-
DEFAULT_SIGNED_URL_TTL_MS,
|
|
212
197
|
SIGNED_URL_FAILURES,
|
|
213
|
-
SIGNED_URL_PARAMS,
|
|
214
|
-
SIGNED_URL_VERSION,
|
|
215
198
|
signConstraints,
|
|
216
199
|
signedUrlBaseFor,
|
|
217
200
|
signedUrlBasePath,
|
|
@@ -229,8 +212,9 @@ export type {
|
|
|
229
212
|
StorageSubjectTarget,
|
|
230
213
|
StorageSubjectTargetOptions,
|
|
231
214
|
} from './subject-target';
|
|
232
|
-
export {
|
|
215
|
+
export { storageSubjectTarget } from './subject-target';
|
|
233
216
|
export type {
|
|
217
|
+
ScanUnavailablePolicy,
|
|
234
218
|
StoredHeadCandidate,
|
|
235
219
|
UploadCandidate,
|
|
236
220
|
UploadPolicy,
|
|
@@ -239,14 +223,12 @@ export type {
|
|
|
239
223
|
ValidatedUpload,
|
|
240
224
|
} from './upload';
|
|
241
225
|
export {
|
|
242
|
-
contentTypeMatches,
|
|
243
226
|
DEFAULT_MAX_UPLOAD_BYTES,
|
|
244
227
|
DOCUMENT_CONTENT_TYPES,
|
|
245
228
|
IMAGE_CONTENT_TYPES,
|
|
246
229
|
MAX_LEADING_WHITESPACE,
|
|
247
230
|
normalizeContentType,
|
|
248
231
|
SNIFF_WINDOW_BYTES,
|
|
249
|
-
sniffContentType,
|
|
250
232
|
uploadPolicy,
|
|
251
233
|
validateStoredHead,
|
|
252
234
|
validateUpload,
|
|
@@ -260,18 +242,17 @@ export type {
|
|
|
260
242
|
UploadSource,
|
|
261
243
|
} from './upload-client';
|
|
262
244
|
export {
|
|
263
|
-
defaultSignedPut,
|
|
264
245
|
fetchSignedPut,
|
|
265
246
|
uploadFile,
|
|
266
247
|
xhrSignedPut,
|
|
267
248
|
} from './upload-client';
|
|
268
249
|
export type {
|
|
269
250
|
MultipartUploadClient,
|
|
270
|
-
MultipartUploadInput,
|
|
271
251
|
UploadSessionRef,
|
|
272
252
|
} from './upload-client-multipart';
|
|
273
253
|
export {
|
|
274
254
|
DEFAULT_PART_THRESHOLD_BYTES,
|
|
275
255
|
PART_CONCURRENCY,
|
|
276
|
-
uploadInParts,
|
|
277
256
|
} from './upload-client-multipart';
|
|
257
|
+
// The `uploadScanner` port's reference: the interface and its installer are on `./adapter`.
|
|
258
|
+
export { EICAR_TEST_STRING, memoryUploadScanner } from './upload-scanner-memory';
|
package/src/path.ts
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
// and an S3 disk into a cross-tenant read. Rejection is total: no sanitising, no rewriting —
|
|
4
4
|
// a key that needed fixing was built wrong, and silently fixing it hides the bug.
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import { renderFixLiteral } from '@ultimat3/core';
|
|
7
|
+
import { pathUnsafe, StorageError } from './errors';
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* S3's own limit; keeping local and remote disks interchangeable requires the same ceiling.
|
|
@@ -32,6 +33,12 @@ export const META_DIR = '.meta';
|
|
|
32
33
|
// `%2e%2e%2f` decodes to `../` in any layer that decodes twice (proxy, then framework).
|
|
33
34
|
const ENCODED_SEPARATOR = /%(?:2e|2f|5c|00)/i;
|
|
34
35
|
|
|
36
|
+
/**
|
|
37
|
+
* A Win32 device name, with or without an extension and in any case: `nul.txt` opens the device,
|
|
38
|
+
* never a file, on a local disk under Windows. Refused on every driver, so a key is one rule.
|
|
39
|
+
*/
|
|
40
|
+
const DEVICE_NAME = /^(?:con|prn|aux|nul|conin\$|conout\$|com[0-9¹²³]|lpt[0-9¹²³])\s*(?:\..*)?$/i;
|
|
41
|
+
|
|
35
42
|
/** NUL and friends: a C-string API downstream truncates at the NUL and opens a different file. */
|
|
36
43
|
function hasControlByte(key: string): boolean {
|
|
37
44
|
for (let index = 0; index < key.length; index += 1) {
|
|
@@ -67,6 +74,10 @@ function unsafeReason(key: string): string | undefined {
|
|
|
67
74
|
if (segment.length === 0) return 'contains an empty segment ("//" or a trailing "/")';
|
|
68
75
|
if (segment === '.' || segment === '..') return `contains a "${segment}" segment`;
|
|
69
76
|
if (segment !== segment.trim()) return `has a padded segment ${JSON.stringify(segment)}`;
|
|
77
|
+
// Win32 drops a trailing dot: `.meta.` IS the reserved sidecar dir, `a.` IS `a`.
|
|
78
|
+
if (segment.endsWith('.')) return `has a segment ending in "." ${JSON.stringify(segment)}`;
|
|
79
|
+
if (DEVICE_NAME.test(segment))
|
|
80
|
+
return `has a Windows device-name segment ${JSON.stringify(segment)}`;
|
|
70
81
|
}
|
|
71
82
|
return undefined;
|
|
72
83
|
}
|
|
@@ -75,6 +86,21 @@ export function isSafeKey(key: string): boolean {
|
|
|
75
86
|
return unsafeReason(key) === undefined;
|
|
76
87
|
}
|
|
77
88
|
|
|
89
|
+
/**
|
|
90
|
+
* The local disk's one extra rule, on Windows only: `a:b` is the NTFS stream `b` of the file `a`, so
|
|
91
|
+
* two keys would share one file's bytes. Everywhere else — s3, Linux, macOS — a `:` is an ordinary
|
|
92
|
+
* key byte, and ISO timestamps carry three, so the shared rule above admits it.
|
|
93
|
+
*/
|
|
94
|
+
export function assertLocalKey(key: string, platform: string = process.platform): string {
|
|
95
|
+
if (platform !== 'win32' || !key.includes(':')) return key;
|
|
96
|
+
throw new StorageError({
|
|
97
|
+
code: 'X_STORAGE_PATH_UNSAFE',
|
|
98
|
+
cause: `key ${JSON.stringify(key)} contains a ":", which a local disk on Windows stores as an NTFS alternate data stream of another file`,
|
|
99
|
+
fix: `await disk.put(${renderFixLiteral(key.replaceAll(':', '-'), "'<key>'")}, body) # the same key with each ":" as "-" (2026-11-09T12-00-00Z for a timestamp)`,
|
|
100
|
+
meta: { key, reason: 'contains a ":" on win32' },
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
78
104
|
/** Returns the key unchanged, so it composes: `Bun.file(join(root, assertSafeKey(key)))`. */
|
|
79
105
|
export function assertSafeKey(key: string): string {
|
|
80
106
|
const reason = unsafeReason(key);
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Single responsibility: the request ceiling of a disk whose signed PUT lands on THIS process (local,
|
|
2
|
+
// memory). One request cannot carry more than the server reads in one, so a single-request grant
|
|
3
|
+
// past it is refused when it is signed — the fix names the multipart grant, whose parts each fit.
|
|
4
|
+
|
|
5
|
+
import { finiteCount } from '@ultimat3/core';
|
|
6
|
+
import { StorageError } from './errors';
|
|
7
|
+
|
|
8
|
+
/** A number, or a reader of one: a host whose ceiling is declared after the disk is built. */
|
|
9
|
+
export type RequestCeiling = number | (() => number);
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Screened where it is declared, as every byte ceiling is; a reader is screened each time it is
|
|
13
|
+
* read, at signing. The answer is `undefined` for "no ceiling declared".
|
|
14
|
+
*/
|
|
15
|
+
export const screenRequestCeiling = (
|
|
16
|
+
driver: string,
|
|
17
|
+
value: RequestCeiling | undefined,
|
|
18
|
+
): (() => number | undefined) => {
|
|
19
|
+
const screen = (bytes: number): number =>
|
|
20
|
+
finiteCount(`the ${driver} disk driver`, 'maxRequestBytes', bytes, 1);
|
|
21
|
+
if (value === undefined) return () => undefined;
|
|
22
|
+
if (typeof value === 'function') return () => screen(value());
|
|
23
|
+
const fixed = screen(value);
|
|
24
|
+
return () => fixed;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** A single-request PUT URL bound past what one request may carry. A part URL is never refused. */
|
|
28
|
+
export function assertWithinRequestCeiling(
|
|
29
|
+
driver: string,
|
|
30
|
+
ceiling: number | undefined,
|
|
31
|
+
url: {
|
|
32
|
+
readonly method?: string | undefined;
|
|
33
|
+
readonly part?: unknown;
|
|
34
|
+
readonly bound: number | undefined;
|
|
35
|
+
},
|
|
36
|
+
): void {
|
|
37
|
+
if (ceiling === undefined || url.part !== undefined || url.method !== 'PUT') return;
|
|
38
|
+
if (url.bound === undefined || url.bound <= ceiling) return;
|
|
39
|
+
throw new StorageError({
|
|
40
|
+
code: 'X_STORAGE_TOO_LARGE',
|
|
41
|
+
cause: `a single signed PUT of up to ${url.bound}B on the ${driver} disk is one request to this server, whose ceiling is ${ceiling}B (http.maxRequestBodyBytes)`,
|
|
42
|
+
fix: 'await grantMultipart({ disk, orgId, request, policy }) # parts, each one request under the ceiling — then signPart and completeUpload',
|
|
43
|
+
meta: { driver, bytes: url.bound, maxBytes: ceiling },
|
|
44
|
+
});
|
|
45
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// Single responsibility: the disk an app resolves through `defineStorage`, reporting each call on
|
|
2
|
+
// core's instrumentation bus as `storage.<verb>` inside a client span. One wrapper over the
|
|
3
|
+
// `StorageDriver` contract, so every driver — shipped or third-party — is instrumented the same.
|
|
4
|
+
|
|
5
|
+
import type { StorageEvent } from '@ultimat3/core';
|
|
6
|
+
import { instrument, subscribed, withSpan } from '@ultimat3/core';
|
|
7
|
+
import type { StorageDriver } from './driver';
|
|
8
|
+
|
|
9
|
+
type StorageVerb = 'put' | 'get' | 'stat' | 'copy' | 'delete' | 'list';
|
|
10
|
+
|
|
11
|
+
/** Which event each instrumented method reports as: a streamed write is a put, a stream a get. */
|
|
12
|
+
const VERB_OF: Readonly<Record<string, StorageVerb>> = Object.freeze({
|
|
13
|
+
put: 'put',
|
|
14
|
+
putStream: 'put',
|
|
15
|
+
get: 'get',
|
|
16
|
+
stream: 'get',
|
|
17
|
+
streamRange: 'get',
|
|
18
|
+
stat: 'stat',
|
|
19
|
+
copy: 'copy',
|
|
20
|
+
delete: 'delete',
|
|
21
|
+
list: 'list',
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
/** The key a call is ABOUT: the destination of a copy, the prefix of a listing. */
|
|
25
|
+
function keyOf(method: string, args: readonly unknown[]): string {
|
|
26
|
+
if (method === 'copy') return typeof args[1] === 'string' ? args[1] : '';
|
|
27
|
+
if (method === 'list') {
|
|
28
|
+
const options = args[0];
|
|
29
|
+
const prefix =
|
|
30
|
+
typeof options === 'object' && options !== null && 'prefix' in options
|
|
31
|
+
? options.prefix
|
|
32
|
+
: undefined;
|
|
33
|
+
return typeof prefix === 'string' ? prefix : '';
|
|
34
|
+
}
|
|
35
|
+
return typeof args[0] === 'string' ? args[0] : '';
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** A size the answer states — `StorageObject.size`, or a `get()`'s object — never a guess. */
|
|
39
|
+
function bytesOf(answer: unknown): number | undefined {
|
|
40
|
+
if (typeof answer !== 'object' || answer === null) return undefined;
|
|
41
|
+
const holder = 'object' in answer ? answer.object : answer;
|
|
42
|
+
if (typeof holder !== 'object' || holder === null || !('size' in holder)) return undefined;
|
|
43
|
+
return typeof holder.size === 'number' ? holder.size : undefined;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function report(name: `storage.${StorageVerb}`, event: StorageEvent): void {
|
|
47
|
+
if (subscribed(name)) instrument(name, event);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The driver, every contract method wrapped and every other member (a test disk's `objects()`,
|
|
52
|
+
* `multipart`, `registerAs`) passed through bound to the driver. The driver's calls to its OWN
|
|
53
|
+
* methods stay unwrapped: one caller's call is one event.
|
|
54
|
+
*/
|
|
55
|
+
export function instrumentedDisk(disk: string, driver: StorageDriver): StorageDriver {
|
|
56
|
+
const wrapped = new Map<PropertyKey, unknown>();
|
|
57
|
+
return new Proxy(driver, {
|
|
58
|
+
get(target, property) {
|
|
59
|
+
const known = wrapped.get(property);
|
|
60
|
+
if (known !== undefined) return known;
|
|
61
|
+
const value: unknown = Reflect.get(target, property, target);
|
|
62
|
+
if (typeof value !== 'function') return value;
|
|
63
|
+
// A Proxy `get` is handed every name, `constructor` included: an own key or no verb.
|
|
64
|
+
const verb =
|
|
65
|
+
typeof property === 'string' && Object.hasOwn(VERB_OF, property)
|
|
66
|
+
? VERB_OF[property]
|
|
67
|
+
: undefined;
|
|
68
|
+
const method = value as (...args: unknown[]) => unknown;
|
|
69
|
+
const bound =
|
|
70
|
+
verb === undefined
|
|
71
|
+
? method.bind(target)
|
|
72
|
+
: (...args: unknown[]): Promise<unknown> =>
|
|
73
|
+
call(disk, verb, keyOf(property as string, args), driver.name, () =>
|
|
74
|
+
Promise.resolve(method.apply(target, args)),
|
|
75
|
+
);
|
|
76
|
+
wrapped.set(property, bound);
|
|
77
|
+
return bound;
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function call(
|
|
83
|
+
disk: string,
|
|
84
|
+
verb: StorageVerb,
|
|
85
|
+
key: string,
|
|
86
|
+
driverName: string,
|
|
87
|
+
work: () => Promise<unknown>,
|
|
88
|
+
): Promise<unknown> {
|
|
89
|
+
const name = `storage.${verb}` as const;
|
|
90
|
+
const started = performance.now();
|
|
91
|
+
try {
|
|
92
|
+
const answer = await withSpan(name, work, {
|
|
93
|
+
kind: 'client',
|
|
94
|
+
attributes: { 'ultimate.storage.disk': disk, 'ultimate.storage.driver': driverName },
|
|
95
|
+
});
|
|
96
|
+
const bytes = bytesOf(answer);
|
|
97
|
+
report(name, {
|
|
98
|
+
disk,
|
|
99
|
+
key,
|
|
100
|
+
...(bytes === undefined ? {} : { bytes }),
|
|
101
|
+
durationMs: performance.now() - started,
|
|
102
|
+
});
|
|
103
|
+
return answer;
|
|
104
|
+
} catch (error) {
|
|
105
|
+
report(name, { disk, key, durationMs: performance.now() - started, error });
|
|
106
|
+
throw error;
|
|
107
|
+
}
|
|
108
|
+
}
|
package/src/storage.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { ConfigInvalidError } from '@ultimat3/core';
|
|
|
8
8
|
import type { StorageDriver } from './driver';
|
|
9
9
|
import { diskUnknown } from './errors';
|
|
10
10
|
import { isSafeKey, isTenantScoped } from './path';
|
|
11
|
+
import { instrumentedDisk } from './storage-instrument';
|
|
11
12
|
|
|
12
13
|
export interface StorageConfig {
|
|
13
14
|
readonly disks: Readonly<Record<string, StorageDriver>>;
|
|
@@ -96,12 +97,12 @@ export function defineStorage(config: StorageConfig): Storage {
|
|
|
96
97
|
// `.put()` was a bare `TypeError` inside app code — `diskUnknown` unreachable for `constructor`,
|
|
97
98
|
// `toString`, `valueOf`, `hasOwnProperty` and `__proto__`. One function was already answering
|
|
98
99
|
// one question two ways: `default: 'constructor'` is refused above, off `Object.keys`.
|
|
99
|
-
const
|
|
100
|
+
const declared = new Map(Object.entries(config.disks));
|
|
100
101
|
// One driver instance, one disk name. A driver told two names keeps the last, and every URL it
|
|
101
102
|
// minted under the first then resolves to a disk it is not — refused here rather than 404ing
|
|
102
103
|
// one alias at a time. Two disks over one root are two `localDriver()` calls.
|
|
103
104
|
const registered = new Map<StorageDriver, string>();
|
|
104
|
-
for (const [diskName, driver] of
|
|
105
|
+
for (const [diskName, driver] of declared) {
|
|
105
106
|
const already = registered.get(driver);
|
|
106
107
|
if (already !== undefined) {
|
|
107
108
|
throw new ConfigInvalidError({
|
|
@@ -112,6 +113,11 @@ export function defineStorage(config: StorageConfig): Storage {
|
|
|
112
113
|
registered.set(driver, diskName);
|
|
113
114
|
driver.registerAs?.(diskName);
|
|
114
115
|
}
|
|
116
|
+
// What `disk()` hands out: the declared driver, reporting each call on the bus (`storage.<verb>`).
|
|
117
|
+
// Built once per disk, so a disk is the same object on every resolution.
|
|
118
|
+
const disks = new Map(
|
|
119
|
+
[...declared].map(([diskName, driver]) => [diskName, instrumentedDisk(diskName, driver)]),
|
|
120
|
+
);
|
|
115
121
|
const shared = sharedPrefixes(config.shared, names);
|
|
116
122
|
const storageInstance: Storage = {
|
|
117
123
|
defaultDisk,
|