@ultimat3/storage 30.8.0 → 31.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +56 -113
- package/README.md +117 -24
- package/package.json +4 -3
- package/src/accept-stream.ts +125 -0
- package/src/accept.ts +24 -6
- package/src/adapter.ts +46 -0
- package/src/attachment.ts +43 -61
- package/src/driver-local-list.ts +1 -1
- package/src/driver-local-open.ts +112 -0
- package/src/driver-local-read.ts +1 -1
- package/src/driver-local-stream.ts +14 -3
- package/src/driver-local.ts +48 -16
- package/src/driver-memory.ts +39 -9
- package/src/driver-s3-copy-parts.ts +177 -0
- package/src/driver-s3-copy.ts +14 -7
- package/src/driver-s3-lock.ts +1 -3
- package/src/driver-s3-put-stream.ts +81 -17
- package/src/driver-s3-range.ts +14 -7
- package/src/driver-s3-signed.ts +1 -1
- package/src/driver-s3-stream.ts +36 -0
- package/src/driver-s3.ts +44 -27
- package/src/driver.ts +30 -5
- package/src/errors-promote.ts +3 -15
- package/src/errors-scan.ts +61 -0
- package/src/errors-stream.ts +13 -0
- package/src/errors-variant.ts +40 -0
- package/src/errors.ts +6 -0
- package/src/grant.ts +15 -10
- package/src/image-variant.ts +106 -0
- package/src/index.ts +21 -40
- package/src/path.ts +27 -1
- package/src/request-ceiling.ts +45 -0
- package/src/storage-instrument.ts +108 -0
- package/src/storage.ts +8 -2
- package/src/upload-scan.ts +91 -0
- package/src/upload-scanner-memory.ts +62 -0
- package/src/upload-scanner.ts +71 -0
- package/src/upload.ts +35 -2
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
settleLostPut,
|
|
23
23
|
} from './driver-s3-settle';
|
|
24
24
|
import type { S3WireOptions } from './driver-s3-signed';
|
|
25
|
-
import {
|
|
25
|
+
import { MAX_PART_BYTES } from './multipart';
|
|
26
26
|
import type { PutStreamOptions } from './put-stream';
|
|
27
27
|
import { streamLimit } from './put-stream';
|
|
28
28
|
|
|
@@ -35,6 +35,29 @@ export interface S3PutStreamInput extends Pick<S3UploadTarget, 'key' | 'refused'
|
|
|
35
35
|
readonly stat: S3Stat;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
/** A part's size before growth: big enough to amortise a request, small enough to hold four. */
|
|
39
|
+
export const STREAM_PART_BYTES = 8 * 1024 * 1024;
|
|
40
|
+
/** Parts in flight AND being filled, together: what one `putStream` holds is at most this many parts. */
|
|
41
|
+
export const STREAM_PART_BUFFERS = 4;
|
|
42
|
+
/** Parts per size step: the size doubles every 1,000, so 10,000 parts reach 8 TiB, past S3's 5 TiB. */
|
|
43
|
+
const PARTS_PER_STEP = 1000;
|
|
44
|
+
/** The first buffer for a body that may yet fit one PUT: grown by doubling, never preallocated. */
|
|
45
|
+
const FIRST_BUFFER_BYTES = 64 * 1024;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Part `n`'s size: `STREAM_PART_BYTES`, doubled every 1,000 parts, capped at S3's 5 GiB part. Never
|
|
49
|
+
* derived from the declared ceiling, which made a 1 TB `maxBytes` cut 100 MB parts for every body.
|
|
50
|
+
*/
|
|
51
|
+
export function streamPartBytes(partNumber: number): number {
|
|
52
|
+
// A size schedule, not a retry curve: doubled once per 1,000 parts while it stays a legal part.
|
|
53
|
+
let size = STREAM_PART_BYTES;
|
|
54
|
+
for (let past = PARTS_PER_STEP; past < partNumber; past += PARTS_PER_STEP) {
|
|
55
|
+
if (size + size > MAX_PART_BYTES) break;
|
|
56
|
+
size += size;
|
|
57
|
+
}
|
|
58
|
+
return size;
|
|
59
|
+
}
|
|
60
|
+
|
|
38
61
|
/** The whole body, stored — or nothing stored and the key as it was. */
|
|
39
62
|
export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput): Promise<void> {
|
|
40
63
|
const { key, body, options, maxBytes, driver, refused, stat } = input;
|
|
@@ -42,30 +65,66 @@ export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput):
|
|
|
42
65
|
const putOptions: PutOptions = options;
|
|
43
66
|
// Screened before a byte is read: a header that cannot travel fails here, not on the Create.
|
|
44
67
|
const headers = objectHeaders(putOptions);
|
|
45
|
-
|
|
46
|
-
let
|
|
47
|
-
let
|
|
68
|
+
// ONE buffer per part, chunks copied in: a chunk is never held beside the part it went into.
|
|
69
|
+
let buffer = new Uint8Array(FIRST_BUFFER_BYTES);
|
|
70
|
+
let filled = 0;
|
|
48
71
|
let total = 0;
|
|
49
72
|
let uploadId: string | undefined;
|
|
50
73
|
const etags: string[] = [];
|
|
74
|
+
const inFlight = new Set<Promise<void>>();
|
|
75
|
+
/** The first refused part, if any; later ones add nothing the caller can act on. */
|
|
76
|
+
const failures: unknown[] = [];
|
|
51
77
|
|
|
52
|
-
const
|
|
78
|
+
const dispatch = async (part: Uint8Array): Promise<void> => {
|
|
53
79
|
uploadId ??= await createS3Upload(target, headers);
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
80
|
+
const open = uploadId;
|
|
81
|
+
const partNumber = etags.length + 1;
|
|
82
|
+
etags.push('');
|
|
83
|
+
const sending = uploadS3Part(target, open, partNumber, part).then(
|
|
84
|
+
(etag) => {
|
|
85
|
+
etags[partNumber - 1] = etag;
|
|
86
|
+
},
|
|
87
|
+
(error: unknown) => {
|
|
88
|
+
failures.push(error);
|
|
89
|
+
},
|
|
90
|
+
);
|
|
91
|
+
const tracked = sending.finally(() => inFlight.delete(tracked));
|
|
92
|
+
inFlight.add(tracked);
|
|
93
|
+
// A new buffer only once one is free: in flight plus filling stays at `STREAM_PART_BUFFERS`.
|
|
94
|
+
while (inFlight.size >= STREAM_PART_BUFFERS) await Promise.race(inFlight);
|
|
95
|
+
if (failures.length > 0) throw failures[0];
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const append = async (chunk: Uint8Array): Promise<void> => {
|
|
99
|
+
let at = 0;
|
|
100
|
+
while (at < chunk.byteLength) {
|
|
101
|
+
const partBytes = streamPartBytes(etags.length + 1);
|
|
102
|
+
if (filled === buffer.byteLength && buffer.byteLength < partBytes) {
|
|
103
|
+
// Still the first part: grown by doubling, so a small body never costs a whole part.
|
|
104
|
+
const grown = new Uint8Array(Math.min(partBytes, buffer.byteLength * 2));
|
|
105
|
+
grown.set(buffer.subarray(0, filled));
|
|
106
|
+
buffer = grown;
|
|
107
|
+
}
|
|
108
|
+
const taken = Math.min(chunk.byteLength - at, buffer.byteLength - filled);
|
|
109
|
+
buffer.set(chunk.subarray(at, at + taken), filled);
|
|
110
|
+
filled += taken;
|
|
111
|
+
at += taken;
|
|
112
|
+
if (filled === partBytes) {
|
|
113
|
+
filled = 0;
|
|
114
|
+
await dispatch(buffer);
|
|
115
|
+
// Allocated once a slot is free, never before: the bound is on buffers, not requests.
|
|
116
|
+
buffer = new Uint8Array(streamPartBytes(etags.length + 1));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
58
119
|
};
|
|
59
120
|
|
|
60
121
|
try {
|
|
61
122
|
await pumpBounded(body, streamLimit(driver, key, maxBytes), async (chunk) => {
|
|
62
|
-
chunks.push(chunk);
|
|
63
|
-
held += chunk.byteLength;
|
|
64
123
|
total += chunk.byteLength;
|
|
65
|
-
|
|
124
|
+
await append(chunk);
|
|
66
125
|
});
|
|
67
|
-
if (uploadId === undefined) {
|
|
68
|
-
const whole =
|
|
126
|
+
if (uploadId === undefined && inFlight.size === 0) {
|
|
127
|
+
const whole = buffer.subarray(0, filled);
|
|
69
128
|
const rethrow = (error: unknown): never => {
|
|
70
129
|
throw error;
|
|
71
130
|
};
|
|
@@ -75,8 +134,11 @@ export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput):
|
|
|
75
134
|
if (refusal !== undefined) refused(refusal);
|
|
76
135
|
return;
|
|
77
136
|
}
|
|
78
|
-
if (
|
|
137
|
+
if (filled > 0) await dispatch(buffer.subarray(0, filled));
|
|
138
|
+
await Promise.all(inFlight);
|
|
139
|
+
if (failures.length > 0) throw failures[0];
|
|
79
140
|
const open = uploadId;
|
|
141
|
+
if (open === undefined) return;
|
|
80
142
|
await completeS3Upload(
|
|
81
143
|
target,
|
|
82
144
|
open,
|
|
@@ -89,8 +151,10 @@ export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput):
|
|
|
89
151
|
await settleComplete(stat, { ...target, driver }, open, sent, lost);
|
|
90
152
|
});
|
|
91
153
|
} catch (error) {
|
|
92
|
-
//
|
|
93
|
-
//
|
|
154
|
+
// A part still sending would land after the abort and bill: every one settles first. The
|
|
155
|
+
// refusal the caller reads is the first one, so an abort that fails too is left to the
|
|
156
|
+
// bucket's lifecycle rule rather than replacing it.
|
|
157
|
+
await Promise.allSettled(inFlight);
|
|
94
158
|
if (uploadId !== undefined) await abortS3Upload(target, uploadId).catch(() => undefined);
|
|
95
159
|
throw error;
|
|
96
160
|
}
|
package/src/driver-s3-range.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// slice and its description come from one response about one version. A HEAD beside it could not
|
|
4
4
|
// promise that: a write between the two, or an A → B → A overwrite of one length, slips past both.
|
|
5
5
|
|
|
6
|
+
import { renderThrowable } from '@ultimat3/core';
|
|
6
7
|
import type { StorageObject, StorageRead } from './driver';
|
|
7
8
|
import { DEFAULT_CONTENT_TYPE } from './driver';
|
|
8
9
|
import { type S3Refusal, type S3WireOptions, sendSigned } from './driver-s3-signed';
|
|
@@ -23,14 +24,20 @@ export interface S3RangeDeps {
|
|
|
23
24
|
|
|
24
25
|
const CONTENT_RANGE = /^bytes (\d+)-(\d+)\/(\d+)$/;
|
|
25
26
|
|
|
27
|
+
/** An answer refused before its body is read: the body is dropped, never left on the socket. */
|
|
28
|
+
async function discardBody(response: Response): Promise<void> {
|
|
29
|
+
try {
|
|
30
|
+
await response.body?.cancel();
|
|
31
|
+
} catch {
|
|
32
|
+
// A body that cannot be cancelled is already done: the refusal is the answer either way.
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
26
36
|
/**
|
|
27
37
|
* The window asked, CAPPED at `maxGetBytes` before it is sent: the ceiling bounds the range, and the
|
|
28
38
|
* object's size is not known until the answer — so at most the ceiling crosses the wire, and an
|
|
29
39
|
* answer that shows the object went on past the cap is refused before its body is read.
|
|
30
40
|
*/
|
|
31
|
-
const errorMessage = (thrown: unknown): string =>
|
|
32
|
-
thrown instanceof Error ? thrown.message : String(thrown);
|
|
33
|
-
|
|
34
41
|
export async function readS3Range(
|
|
35
42
|
wire: S3WireOptions,
|
|
36
43
|
key: string,
|
|
@@ -52,13 +59,13 @@ export async function readS3Range(
|
|
|
52
59
|
const { response } = result;
|
|
53
60
|
const window = CONTENT_RANGE.exec(response.headers.get('content-range') ?? '');
|
|
54
61
|
if (response.status !== 206 || window === null) {
|
|
55
|
-
await response
|
|
62
|
+
await discardBody(response);
|
|
56
63
|
return deps.unreadable(key, `the ranged GET answered ${response.status} with no byte range`);
|
|
57
64
|
}
|
|
58
65
|
const served: ByteRange = [Number(window[1]), Number(window[2])];
|
|
59
66
|
const size = Number(window[3]);
|
|
60
67
|
if (served[0] !== first || served[1] > capped) {
|
|
61
|
-
await response
|
|
68
|
+
await discardBody(response);
|
|
62
69
|
return deps.unreadable(
|
|
63
70
|
key,
|
|
64
71
|
`asked bytes ${first}-${capped}, answered ${served[0]}-${served[1]}`,
|
|
@@ -66,7 +73,7 @@ export async function readS3Range(
|
|
|
66
73
|
}
|
|
67
74
|
// Cut by the CAP, not by the end of the object: the range asked is over the ceiling.
|
|
68
75
|
if (last > capped && size - 1 > capped) {
|
|
69
|
-
await response
|
|
76
|
+
await discardBody(response);
|
|
70
77
|
throw rangeTooLarge(deps.driver, key, [first, Math.min(last, size - 1)], deps.maxGetBytes);
|
|
71
78
|
}
|
|
72
79
|
let bytes: Uint8Array;
|
|
@@ -74,7 +81,7 @@ export async function readS3Range(
|
|
|
74
81
|
bytes = new Uint8Array(await response.arrayBuffer());
|
|
75
82
|
} catch (thrown) {
|
|
76
83
|
// The headers came, the body broke (a reset connection): the disk's coded read failure.
|
|
77
|
-
return deps.unreadable(key, `the ranged body broke mid-read: ${
|
|
84
|
+
return deps.unreadable(key, `the ranged body broke mid-read: ${renderThrowable(thrown)}`);
|
|
78
85
|
}
|
|
79
86
|
if (bytes.byteLength !== served[1] - served[0] + 1) {
|
|
80
87
|
return deps.unreadable(
|
package/src/driver-s3-signed.ts
CHANGED
|
@@ -53,7 +53,7 @@ export interface S3Refusal {
|
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
export interface S3SignedRequest {
|
|
56
|
-
readonly method: 'GET' | 'PUT' | 'POST' | 'DELETE';
|
|
56
|
+
readonly method: 'GET' | 'HEAD' | 'PUT' | 'POST' | 'DELETE';
|
|
57
57
|
readonly key: string;
|
|
58
58
|
/** A sub-resource such as `retention` — sent and signed as `?retention=`. */
|
|
59
59
|
readonly subresource?: string | undefined;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Single responsibility: the s3 disk's PINNED streamed read — one signed GET carrying `If-Match`, so
|
|
2
|
+
// the provider serves the revision a `stat()` described or answers 412. Bun's `S3File.stream()`
|
|
3
|
+
// sends no condition, and an overwrite between the stat and the read served new bytes under the
|
|
4
|
+
// old ETag and length.
|
|
5
|
+
|
|
6
|
+
import { type S3Refusal, type S3WireOptions, sendSigned } from './driver-s3-signed';
|
|
7
|
+
import type { ByteRange } from './range';
|
|
8
|
+
|
|
9
|
+
export interface S3PinnedDeps {
|
|
10
|
+
/** The provider's refusal as the disk's coded read error (404 is not-found there). */
|
|
11
|
+
readonly refused: (refusal: S3Refusal) => never;
|
|
12
|
+
/** Another revision is at the key: the disk's `X_STORAGE_OBJECT_CHANGED`. */
|
|
13
|
+
readonly changed: () => never;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** The bytes of revision `etag`, whole or `[first, last]` already clamped to the object. */
|
|
17
|
+
export async function streamS3Pinned(
|
|
18
|
+
wire: S3WireOptions,
|
|
19
|
+
key: string,
|
|
20
|
+
etag: string,
|
|
21
|
+
range: ByteRange | undefined,
|
|
22
|
+
deps: S3PinnedDeps,
|
|
23
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
24
|
+
const result = await sendSigned(wire, {
|
|
25
|
+
method: 'GET',
|
|
26
|
+
key,
|
|
27
|
+
headers: {
|
|
28
|
+
'if-match': etag,
|
|
29
|
+
...(range === undefined ? {} : { range: `bytes=${range[0]}-${range[1]}` }),
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
if (!result.ok) {
|
|
33
|
+
return result.refusal.statusCode === 412 ? deps.changed() : deps.refused(result.refusal);
|
|
34
|
+
}
|
|
35
|
+
return result.response.body ?? new Blob([]).stream();
|
|
36
|
+
}
|
package/src/driver-s3.ts
CHANGED
|
@@ -27,12 +27,13 @@ import {
|
|
|
27
27
|
type StorageListEntry,
|
|
28
28
|
type StorageObject,
|
|
29
29
|
type StorageRead,
|
|
30
|
+
type StreamOptions,
|
|
30
31
|
sha256Base64,
|
|
31
32
|
toBytes,
|
|
32
33
|
} from './driver';
|
|
33
34
|
import { isAbsentObject } from './driver-s3-absent';
|
|
34
35
|
import { buildClient, dated, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
|
|
35
|
-
import {
|
|
36
|
+
import { copyS3 } from './driver-s3-copy-parts';
|
|
36
37
|
import { needsSignedPut, readObjectLock, signedPut } from './driver-s3-lock';
|
|
37
38
|
import { s3Multipart } from './driver-s3-multipart-session';
|
|
38
39
|
import { s3SignedUrl } from './driver-s3-presign';
|
|
@@ -40,8 +41,9 @@ import { putS3Stream } from './driver-s3-put-stream';
|
|
|
40
41
|
import { readS3Range } from './driver-s3-range';
|
|
41
42
|
import { regionMismatch } from './driver-s3-region';
|
|
42
43
|
import { settleLostPut } from './driver-s3-settle';
|
|
43
|
-
import type { S3FetchLike } from './driver-s3-signed';
|
|
44
|
+
import type { S3FetchLike, S3Refusal } from './driver-s3-signed';
|
|
44
45
|
import { refuseUnsupportedPut } from './driver-s3-sse';
|
|
46
|
+
import { streamS3Pinned } from './driver-s3-stream';
|
|
45
47
|
import {
|
|
46
48
|
checksumMismatch,
|
|
47
49
|
deleteFailed,
|
|
@@ -53,6 +55,7 @@ import {
|
|
|
53
55
|
readFailed,
|
|
54
56
|
} from './errors';
|
|
55
57
|
import { objectChanged } from './errors-promote';
|
|
58
|
+
import { revisionChanged } from './errors-stream';
|
|
56
59
|
import { assertObjectLockOptions, type ObjectLock } from './object-lock';
|
|
57
60
|
import { assertSafeKey } from './path';
|
|
58
61
|
import { assertPutStream, type PutStreamOptions } from './put-stream';
|
|
@@ -197,6 +200,14 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
197
200
|
};
|
|
198
201
|
};
|
|
199
202
|
|
|
203
|
+
/** A pinned read's two refusals: the provider's, coded as a read, and another revision's. */
|
|
204
|
+
const pinned = (key: string, etag: string) => ({
|
|
205
|
+
refused: (refusal: S3Refusal): never => refusedRead(key)(refusal),
|
|
206
|
+
changed: (): never => {
|
|
207
|
+
throw revisionChanged(diskName, key, etag);
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
|
|
200
211
|
/** The HEAD a lost write's answer is settled against, and the `.catch` of a single PUT. */
|
|
201
212
|
const statOf = async (key: string) => ((await present(key)) ? statObject(key) : undefined);
|
|
202
213
|
const lostPut = (key: string, bytes: Uint8Array, attributes: PutOptions | undefined) =>
|
|
@@ -313,14 +324,21 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
313
324
|
return { object, bytes };
|
|
314
325
|
},
|
|
315
326
|
|
|
316
|
-
async stream(key: string): Promise<ReadableStream<Uint8Array>> {
|
|
327
|
+
async stream(key: string, streamOptions?: StreamOptions): Promise<ReadableStream<Uint8Array>> {
|
|
317
328
|
const safe = assertSafeKey(key);
|
|
329
|
+
const etag = streamOptions?.etag;
|
|
330
|
+
if (etag !== undefined)
|
|
331
|
+
return streamS3Pinned(options, safe, etag, undefined, pinned(safe, etag));
|
|
318
332
|
const file = conn().file(safe);
|
|
319
333
|
if (!(await present(safe))) throw objectNotFound(diskName, safe);
|
|
320
334
|
return file.stream();
|
|
321
335
|
},
|
|
322
336
|
|
|
323
|
-
async streamRange(
|
|
337
|
+
async streamRange(
|
|
338
|
+
key: string,
|
|
339
|
+
range: ByteRange,
|
|
340
|
+
streamOptions?: StreamOptions,
|
|
341
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
324
342
|
const safe = assertSafeKey(key);
|
|
325
343
|
assertStreamRange(DRIVER_NAME, range);
|
|
326
344
|
if (!(await present(safe))) throw objectNotFound(diskName, safe);
|
|
@@ -328,6 +346,10 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
328
346
|
// already answered 206. Refused here it is still a status.
|
|
329
347
|
const { size } = await statObject(safe);
|
|
330
348
|
const [first, last] = streamedRange(DRIVER_NAME, safe, range, size);
|
|
349
|
+
const etag = streamOptions?.etag;
|
|
350
|
+
if (etag !== undefined) {
|
|
351
|
+
return streamS3Pinned(options, safe, etag, [first, last], pinned(safe, etag));
|
|
352
|
+
}
|
|
331
353
|
return conn()
|
|
332
354
|
.file(safe)
|
|
333
355
|
.slice(first, last + 1)
|
|
@@ -335,33 +357,28 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
|
|
|
335
357
|
},
|
|
336
358
|
|
|
337
359
|
/**
|
|
338
|
-
* Bytes from one key to another
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
* win available today: promoting a 500MB attachment used to be `get()` + `put()`, a full GB
|
|
342
|
-
* resident in the pod. A true server-side copy needs a Bun API that does not exist in 1.3.
|
|
360
|
+
* Bytes from one key to another, server-side: ONE signed CopyObject (directive COPY), so the
|
|
361
|
+
* source's type, `Cache-Control` and `x-amz-meta-*` travel as on the local and memory disks.
|
|
362
|
+
* Past CopyObject's 5 GiB ceiling, a multipart UploadPartCopy carrying the same headers.
|
|
343
363
|
*/
|
|
344
364
|
async copy(from: string, to: string, copyOptions?: CopyOptions): Promise<StorageObject> {
|
|
345
365
|
const source = assertSafeKey(from);
|
|
346
366
|
const destination = assertSafeKey(to);
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
.file(destination)
|
|
363
|
-
.write(file, { type: stat.type ?? DEFAULT_CONTENT_TYPE })
|
|
364
|
-
.catch(refusedWrite(destination));
|
|
367
|
+
const ifMatch = copyOptions?.ifMatch;
|
|
368
|
+
const found = await statOf(source);
|
|
369
|
+
if (found === undefined) throw objectNotFound(diskName, source);
|
|
370
|
+
const expected = ifMatch ?? found.etag;
|
|
371
|
+
await copyS3(
|
|
372
|
+
options,
|
|
373
|
+
{ source, destination, ifMatch, size: found.size, stat: statOf },
|
|
374
|
+
(refusal) => {
|
|
375
|
+
if (refusal.statusCode === 404 && refusal.code !== 'NoSuchUpload') {
|
|
376
|
+
throw objectNotFound(diskName, source);
|
|
377
|
+
}
|
|
378
|
+
if (refusal.statusCode === 412) throw objectChanged(source, expected, 'another');
|
|
379
|
+
return refusedWrite(destination)(refusal);
|
|
380
|
+
},
|
|
381
|
+
).catch(refusedWrite(destination));
|
|
365
382
|
return statObject(destination);
|
|
366
383
|
},
|
|
367
384
|
|
package/src/driver.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// and never unbounded, which is the same failure with the sender choosing the size.
|
|
5
5
|
|
|
6
6
|
import type { Clock, Page } from '@ultimat3/core';
|
|
7
|
-
import { assert } from '@ultimat3/core';
|
|
7
|
+
import { assert, describeValue } from '@ultimat3/core';
|
|
8
8
|
import { pumpBounded } from './byte-pump';
|
|
9
9
|
import { putTooLarge } from './errors';
|
|
10
10
|
import type { MultipartCapability } from './multipart';
|
|
@@ -172,14 +172,22 @@ export interface StorageDriver {
|
|
|
172
172
|
* the range's length: s3 sends `Range:`, the local disk slices the file, memory slices its copy.
|
|
173
173
|
*/
|
|
174
174
|
get(key: string, options?: GetOptions): Promise<StorageRead>;
|
|
175
|
-
/**
|
|
176
|
-
|
|
175
|
+
/**
|
|
176
|
+
* Bytes without buffering — the only safe path for anything over a few MB. `etag` pins the read
|
|
177
|
+
* to the revision a `stat()` described: another is `X_STORAGE_OBJECT_CHANGED`, never its bytes
|
|
178
|
+
* under the old validator.
|
|
179
|
+
*/
|
|
180
|
+
stream(key: string, options?: StreamOptions): Promise<ReadableStream<Uint8Array>>;
|
|
177
181
|
/**
|
|
178
182
|
* The bytes `[first, last]` (inclusive, clamped, as `get`'s range) with none before them read, so
|
|
179
183
|
* a seek late in a film costs its window. `X_STORAGE_RANGE_UNSATISFIABLE` past the end. A method and
|
|
180
184
|
* not an option on `stream`, which a third-party driver would ignore and serve the wrong bytes.
|
|
181
185
|
*/
|
|
182
|
-
streamRange(
|
|
186
|
+
streamRange(
|
|
187
|
+
key: string,
|
|
188
|
+
range: ByteRange,
|
|
189
|
+
options?: StreamOptions,
|
|
190
|
+
): Promise<ReadableStream<Uint8Array>>;
|
|
183
191
|
/**
|
|
184
192
|
* Bytes from one key to another without them passing through this process. Idempotent in the
|
|
185
193
|
* destination and non-destructive in the source: the caller deletes the source afterwards if
|
|
@@ -273,14 +281,31 @@ const isStringRecord = (value: unknown): boolean =>
|
|
|
273
281
|
* `metadata: { n: 1n }` reached `JSON.stringify` on the local disk as a bare `TypeError`, and the
|
|
274
282
|
* memory disk stored it — against the `Record<string, string>` every reader is typed on.
|
|
275
283
|
*/
|
|
284
|
+
/** A streamed read's options: the revision it may serve. */
|
|
285
|
+
export interface StreamOptions {
|
|
286
|
+
/** The etag a `stat()` answered; any other revision is refused rather than served. */
|
|
287
|
+
readonly etag?: string | undefined;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** Printable ASCII: what one HTTP header value may carry, on the way in and on the way out. */
|
|
291
|
+
export const HEADER_VALUE = /^[\x20-\x7e]*$/;
|
|
292
|
+
|
|
276
293
|
export function assertPutOptions(driver: string, options: PutOptions | undefined): void {
|
|
277
294
|
if (options === undefined) return;
|
|
278
295
|
for (const name of ['contentType', 'cacheControl'] as const) {
|
|
296
|
+
const value = options[name];
|
|
279
297
|
assert(
|
|
280
|
-
|
|
298
|
+
value === undefined || typeof value === 'string',
|
|
281
299
|
`put() on the ${driver} disk was given a ${name} that is not a string`,
|
|
282
300
|
`put(key, body, { ${name}: 'a string' }) — or omit ${name}`,
|
|
283
301
|
);
|
|
302
|
+
// Served back as a header on every disk and SENT as one on s3: a line break is a second
|
|
303
|
+
// header, and on the signed wire a refused one read as a lost answer.
|
|
304
|
+
assert(
|
|
305
|
+
value === undefined || HEADER_VALUE.test(value),
|
|
306
|
+
`put() on the ${driver} disk was given a ${name} that cannot travel as a header (${describeValue(value)})`,
|
|
307
|
+
`put(key, body, { ${name}: 'text/plain' }) — printable ASCII, no line breaks`,
|
|
308
|
+
);
|
|
284
309
|
}
|
|
285
310
|
assert(
|
|
286
311
|
options.metadata === undefined || isStringRecord(options.metadata),
|
package/src/errors-promote.ts
CHANGED
|
@@ -1,23 +1,11 @@
|
|
|
1
|
-
// The refusals of the pending → attached move (`promoteAttachment
|
|
2
|
-
//
|
|
1
|
+
// The refusals of the pending → attached move (`promoteAttachment`): a key that is not a pending
|
|
2
|
+
// upload, one an earlier call already moved, and one that changed while it was moved. A scanner's
|
|
3
|
+
// refusals are `errors-scan.ts`'s.
|
|
3
4
|
// Their codes are registered with the rest of the package's in `errors.ts`.
|
|
4
5
|
|
|
5
6
|
import { renderFixLiteral } from '@ultimat3/core';
|
|
6
7
|
import { StorageError } from './errors';
|
|
7
8
|
|
|
8
|
-
/**
|
|
9
|
-
* A key still under the quarantine prefix. The framework never scans bytes — that is the app's
|
|
10
|
-
* job — so the only thing it can enforce is that nothing leaves quarantine without the app
|
|
11
|
-
* saying so, which is what `promoteAttachment` refusing this key means.
|
|
12
|
-
*/
|
|
13
|
-
export const quarantined = (key: string, orgId: string): StorageError =>
|
|
14
|
-
new StorageError({
|
|
15
|
-
code: 'X_STORAGE_QUARANTINED',
|
|
16
|
-
cause: `"${key}" is still under the quarantine prefix, so nothing has cleared it for use`,
|
|
17
|
-
fix: `scan the bytes, then releaseQuarantine({ disk, key: '${key}', orgId: '${orgId}' }) — promote the key it returns`,
|
|
18
|
-
meta: { key, orgId },
|
|
19
|
-
});
|
|
20
|
-
|
|
21
9
|
/**
|
|
22
10
|
* A key inside the org and outside its `pending/` prefix — most often another row's attached key
|
|
23
11
|
* a client sent back. Promotion copies then deletes, so accepting it moved the victim's file onto
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// The three refusals of a scanned promotion (`upload-scan.ts`): the scanner said infected, said
|
|
2
|
+
// hold, or could not say. Their codes are registered with the rest of the package's in `errors.ts`.
|
|
3
|
+
|
|
4
|
+
import { renderFixLiteral } from '@ultimat3/core';
|
|
5
|
+
import { StorageError } from './errors';
|
|
6
|
+
|
|
7
|
+
/** Never retryable with the same bytes, and never overridable by a policy: they are deleted. */
|
|
8
|
+
export const uploadInfected = (
|
|
9
|
+
key: string,
|
|
10
|
+
scanner: string,
|
|
11
|
+
signature: string,
|
|
12
|
+
removed: boolean,
|
|
13
|
+
): StorageError =>
|
|
14
|
+
new StorageError({
|
|
15
|
+
code: 'X_STORAGE_UPLOAD_INFECTED',
|
|
16
|
+
cause: `the ${scanner} upload scanner found "${signature}" in "${key}"; ${
|
|
17
|
+
removed
|
|
18
|
+
? 'the pending upload and its scan copy were deleted'
|
|
19
|
+
: 'deleting it failed, so sweepOrphans will collect what is left under pending/'
|
|
20
|
+
}`,
|
|
21
|
+
fix: 'grantUpload({ disk, orgId, request }) for a new, clean file — this one is refused for good',
|
|
22
|
+
meta: { key, scanner, signature, removed },
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The scanner rejected rather than answered. `reason` is already rendered (`renderThrowable`): a
|
|
27
|
+
* daemon's own words are the most useful thing in the cause, and an `unknown` is never interpolated.
|
|
28
|
+
*/
|
|
29
|
+
export const scanUnavailable = (key: string, scanner: string, reason: string): StorageError =>
|
|
30
|
+
new StorageError({
|
|
31
|
+
code: 'X_STORAGE_SCAN_UNAVAILABLE',
|
|
32
|
+
cause: `the ${scanner} upload scanner gave no verdict for "${key}": ${reason}`,
|
|
33
|
+
fix: `promoteAttachment({ disk, key: ${renderFixLiteral(key, "'<pending key>'")}, orgId, target, policy }) # again once the ${scanner} scanner answers — the upload is still pending; or uploadPolicy({ onScanUnavailable: 'accept' }) to promote unscanned files`,
|
|
34
|
+
meta: { key, scanner },
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The scanner answered `hold`: a decision made outside this call (a review, an async vendor scan)
|
|
39
|
+
* has not cleared the upload yet. It keeps the code the retired `quarantine: true` flow raised —
|
|
40
|
+
* the same state, a 409 — now reached through the one scan path. `reason` is the scanner's string.
|
|
41
|
+
*/
|
|
42
|
+
export const uploadHeld = (key: string, scanner: string, reason: string): StorageError =>
|
|
43
|
+
new StorageError({
|
|
44
|
+
code: 'X_STORAGE_QUARANTINED',
|
|
45
|
+
cause: `the ${scanner} upload scanner is holding "${key}" (${reason}); the upload stays pending and nothing was attached`,
|
|
46
|
+
fix: `promoteAttachment({ disk, key: ${renderFixLiteral(key, "'<pending key>'")}, orgId, target, policy }) # again once your ${scanner} scanner answers clean for this pendingKey — sweepOrphans collects it if it never does`,
|
|
47
|
+
meta: { key, scanner, reason },
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* A grant aimed straight at a row's key while a scanner is installed. That key never passes through
|
|
52
|
+
* `promoteAttachment`, the one place a scan runs, so signing it would land bytes nothing scanned
|
|
53
|
+
* exactly where a row reads them.
|
|
54
|
+
*/
|
|
55
|
+
export const targetUnscanned = (caller: string, key: string, scanner: string): StorageError =>
|
|
56
|
+
new StorageError({
|
|
57
|
+
code: 'X_STORAGE_TARGET_UNSCANNED',
|
|
58
|
+
cause: `${caller} was given a target ("${key}") while the ${scanner} upload scanner is installed; a targeted key never passes through promoteAttachment, so its bytes would reach the row unscanned`,
|
|
59
|
+
fix: `${caller}({ disk, orgId, request, policy }) # no target — then promoteAttachment({ disk, key, orgId, target, policy }) once the upload lands`,
|
|
60
|
+
meta: { key, scanner },
|
|
61
|
+
});
|
package/src/errors-stream.ts
CHANGED
|
@@ -21,6 +21,19 @@ export const streamTooLarge = (
|
|
|
21
21
|
meta: { disk, key, bytes, maxBytes },
|
|
22
22
|
});
|
|
23
23
|
|
|
24
|
+
/**
|
|
25
|
+
* A streamed read pinned to a revision the key no longer holds: written between the `stat()` whose
|
|
26
|
+
* etag, size and type a response was built on and the read of its bytes. Same code as a promotion
|
|
27
|
+
* that met another revision, for the same reason.
|
|
28
|
+
*/
|
|
29
|
+
export const revisionChanged = (disk: string, key: string, expected: string): StorageError =>
|
|
30
|
+
new StorageError({
|
|
31
|
+
code: 'X_STORAGE_OBJECT_CHANGED',
|
|
32
|
+
cause: `"${key}" on the ${disk} disk was overwritten after it was described as ${expected}, so its bytes are another revision's and were not served`,
|
|
33
|
+
fix: `const object = await disk.stat(${keyArg(key)}); await disk.stream(${keyArg(key)}, { etag: object.etag }) # describe it again, then read that revision`,
|
|
34
|
+
meta: { disk, key, expected },
|
|
35
|
+
});
|
|
36
|
+
|
|
24
37
|
/** `rangeUnsatisfiable` for a call that holds no size: the next step is learning it. */
|
|
25
38
|
export const streamRangeUnsatisfiable = (
|
|
26
39
|
disk: string,
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// The refusals of `variant()` (`image-variant.ts`). A source that is no raster and a driver that
|
|
2
|
+
// answered another encoding are core's `X_IMAGE_UNSUPPORTED`, as every other image refusal here is;
|
|
3
|
+
// a source overwritten mid-transform is storage's `X_STORAGE_OBJECT_CHANGED`.
|
|
4
|
+
|
|
5
|
+
import { type ImageUnsupportedError, imageUnsupported, renderFixLiteral } from '@ultimat3/core';
|
|
6
|
+
import { StorageError } from './errors';
|
|
7
|
+
|
|
8
|
+
export const notARaster = (key: string, contentType: string): ImageUnsupportedError =>
|
|
9
|
+
imageUnsupported(
|
|
10
|
+
`"${key}" is stored as ${contentType}, which is not a raster image, so it has no width to resize to`,
|
|
11
|
+
`await disk.stat(${renderFixLiteral(key, "'<key>'")}) # variant() takes a png, jpeg, webp, gif or avif source: check its contentType, or serve it as-is`,
|
|
12
|
+
{ key, contentType },
|
|
13
|
+
);
|
|
14
|
+
|
|
15
|
+
export const variantFormatUnknown = (key: string, contentType: string): ImageUnsupportedError =>
|
|
16
|
+
imageUnsupported(
|
|
17
|
+
`"${key}" is a ${contentType} source and no variant format was named, so no key can say what the transform will answer`,
|
|
18
|
+
`variant(${renderFixLiteral(key, "'<key>'")}, { disk, images, width, format: 'webp' })`,
|
|
19
|
+
{ key, contentType },
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
export const driverEncodingMismatch = (
|
|
23
|
+
key: string,
|
|
24
|
+
driver: string,
|
|
25
|
+
expected: string,
|
|
26
|
+
actual: string,
|
|
27
|
+
): ImageUnsupportedError =>
|
|
28
|
+
imageUnsupported(
|
|
29
|
+
`the ${driver} image driver answered ${actual} for "${key}", whose key names ${expected}, so the bytes would be served under the wrong type`,
|
|
30
|
+
`variant(${renderFixLiteral(key, "'<key>'")}, { disk, images, width, format: '<the format the ${driver} driver produces>' }) # or make its transform() honour request.format`,
|
|
31
|
+
{ key, driver, expected, actual },
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
export const variantSourceChanged = (key: string, expected: string, actual: string): StorageError =>
|
|
35
|
+
new StorageError({
|
|
36
|
+
code: 'X_STORAGE_OBJECT_CHANGED',
|
|
37
|
+
cause: `"${key}" was overwritten while its variant was being made (etag ${expected} → ${actual}), so the bytes cannot be filed under either revision`,
|
|
38
|
+
fix: `variant(${renderFixLiteral(key, "'<key>'")}, { disk, images, width }) again — it keys the variant by the new revision`,
|
|
39
|
+
meta: { key, expected, actual },
|
|
40
|
+
});
|
package/src/errors.ts
CHANGED
|
@@ -39,6 +39,9 @@ export const STORAGE_OWNED_ERROR_CODES = [
|
|
|
39
39
|
'X_STORAGE_MULTIPART_UNSUPPORTED',
|
|
40
40
|
'X_STORAGE_UPLOAD_UNKNOWN',
|
|
41
41
|
'X_STORAGE_PARTS_INVALID',
|
|
42
|
+
'X_STORAGE_UPLOAD_INFECTED',
|
|
43
|
+
'X_STORAGE_SCAN_UNAVAILABLE',
|
|
44
|
+
'X_STORAGE_TARGET_UNSCANNED',
|
|
42
45
|
] as const;
|
|
43
46
|
|
|
44
47
|
/**
|
|
@@ -89,6 +92,9 @@ export const STORAGE_ERROR_TITLES: Readonly<Record<StorageOwnedErrorCode, string
|
|
|
89
92
|
X_STORAGE_MULTIPART_UNSUPPORTED: 'this disk cannot take an upload in parts',
|
|
90
93
|
X_STORAGE_UPLOAD_UNKNOWN: 'no open multipart upload has that id',
|
|
91
94
|
X_STORAGE_PARTS_INVALID: 'the parts named do not match the parts stored',
|
|
95
|
+
X_STORAGE_UPLOAD_INFECTED: 'the upload scanner found malware in the file',
|
|
96
|
+
X_STORAGE_SCAN_UNAVAILABLE: 'the upload scanner could not give a verdict',
|
|
97
|
+
X_STORAGE_TARGET_UNSCANNED: 'a targeted upload grant would bypass the upload scanner',
|
|
92
98
|
};
|
|
93
99
|
|
|
94
100
|
// One unconditional call, so a second package claiming one of storage's codes throws
|