@ultimat3/storage 30.9.0 → 31.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,7 +22,7 @@ import {
22
22
  settleLostPut,
23
23
  } from './driver-s3-settle';
24
24
  import type { S3WireOptions } from './driver-s3-signed';
25
- import { partBytesFor } from './multipart';
25
+ import { MAX_PART_BYTES } from './multipart';
26
26
  import type { PutStreamOptions } from './put-stream';
27
27
  import { streamLimit } from './put-stream';
28
28
 
@@ -35,6 +35,29 @@ export interface S3PutStreamInput extends Pick<S3UploadTarget, 'key' | 'refused'
35
35
  readonly stat: S3Stat;
36
36
  }
37
37
 
38
+ /** A part's size before growth: big enough to amortise a request, small enough to hold four. */
39
+ export const STREAM_PART_BYTES = 8 * 1024 * 1024;
40
+ /** Parts in flight AND being filled, together: what one `putStream` holds is at most this many parts. */
41
+ export const STREAM_PART_BUFFERS = 4;
42
+ /** Parts per size step: the size doubles every 1,000, so 10,000 parts reach 8 TiB, past S3's 5 TiB. */
43
+ const PARTS_PER_STEP = 1000;
44
+ /** The first buffer for a body that may yet fit one PUT: grown by doubling, never preallocated. */
45
+ const FIRST_BUFFER_BYTES = 64 * 1024;
46
+
47
+ /**
48
+ * Part `n`'s size: `STREAM_PART_BYTES`, doubled every 1,000 parts, capped at S3's 5 GiB part. Never
49
+ * derived from the declared ceiling, which made a 1 TB `maxBytes` cut 100 MB parts for every body.
50
+ */
51
+ export function streamPartBytes(partNumber: number): number {
52
+ // A size schedule, not a retry curve: doubled once per 1,000 parts while it stays a legal part.
53
+ let size = STREAM_PART_BYTES;
54
+ for (let past = PARTS_PER_STEP; past < partNumber; past += PARTS_PER_STEP) {
55
+ if (size + size > MAX_PART_BYTES) break;
56
+ size += size;
57
+ }
58
+ return size;
59
+ }
60
+
38
61
  /** The whole body, stored — or nothing stored and the key as it was. */
39
62
  export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput): Promise<void> {
40
63
  const { key, body, options, maxBytes, driver, refused, stat } = input;
@@ -42,30 +65,66 @@ export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput):
42
65
  const putOptions: PutOptions = options;
43
66
  // Screened before a byte is read: a header that cannot travel fails here, not on the Create.
44
67
  const headers = objectHeaders(putOptions);
45
- const partBytes = partBytesFor(maxBytes);
46
- let chunks: Uint8Array[] = [];
47
- let held = 0;
68
+ // ONE buffer per part, chunks copied in: a chunk is never held beside the part it went into.
69
+ let buffer = new Uint8Array(FIRST_BUFFER_BYTES);
70
+ let filled = 0;
48
71
  let total = 0;
49
72
  let uploadId: string | undefined;
50
73
  const etags: string[] = [];
74
+ const inFlight = new Set<Promise<void>>();
75
+ /** The first refused part, if any; later ones add nothing the caller can act on. */
76
+ const failures: unknown[] = [];
51
77
 
52
- const sendPart = async (): Promise<void> => {
78
+ const dispatch = async (part: Uint8Array): Promise<void> => {
53
79
  uploadId ??= await createS3Upload(target, headers);
54
- const part = new Uint8Array(Bun.concatArrayBuffers(chunks));
55
- chunks = [];
56
- held = 0;
57
- etags.push(await uploadS3Part(target, uploadId, etags.length + 1, part));
80
+ const open = uploadId;
81
+ const partNumber = etags.length + 1;
82
+ etags.push('');
83
+ const sending = uploadS3Part(target, open, partNumber, part).then(
84
+ (etag) => {
85
+ etags[partNumber - 1] = etag;
86
+ },
87
+ (error: unknown) => {
88
+ failures.push(error);
89
+ },
90
+ );
91
+ const tracked = sending.finally(() => inFlight.delete(tracked));
92
+ inFlight.add(tracked);
93
+ // A new buffer only once one is free: in flight plus filling stays at `STREAM_PART_BUFFERS`.
94
+ while (inFlight.size >= STREAM_PART_BUFFERS) await Promise.race(inFlight);
95
+ if (failures.length > 0) throw failures[0];
96
+ };
97
+
98
+ const append = async (chunk: Uint8Array): Promise<void> => {
99
+ let at = 0;
100
+ while (at < chunk.byteLength) {
101
+ const partBytes = streamPartBytes(etags.length + 1);
102
+ if (filled === buffer.byteLength && buffer.byteLength < partBytes) {
103
+ // Still the first part: grown by doubling, so a small body never costs a whole part.
104
+ const grown = new Uint8Array(Math.min(partBytes, buffer.byteLength * 2));
105
+ grown.set(buffer.subarray(0, filled));
106
+ buffer = grown;
107
+ }
108
+ const taken = Math.min(chunk.byteLength - at, buffer.byteLength - filled);
109
+ buffer.set(chunk.subarray(at, at + taken), filled);
110
+ filled += taken;
111
+ at += taken;
112
+ if (filled === partBytes) {
113
+ filled = 0;
114
+ await dispatch(buffer);
115
+ // Allocated once a slot is free, never before: the bound is on buffers, not requests.
116
+ buffer = new Uint8Array(streamPartBytes(etags.length + 1));
117
+ }
118
+ }
58
119
  };
59
120
 
60
121
  try {
61
122
  await pumpBounded(body, streamLimit(driver, key, maxBytes), async (chunk) => {
62
- chunks.push(chunk);
63
- held += chunk.byteLength;
64
123
  total += chunk.byteLength;
65
- if (held >= partBytes) await sendPart();
124
+ await append(chunk);
66
125
  });
67
- if (uploadId === undefined) {
68
- const whole = new Uint8Array(Bun.concatArrayBuffers(chunks));
126
+ if (uploadId === undefined && inFlight.size === 0) {
127
+ const whole = buffer.subarray(0, filled);
69
128
  const rethrow = (error: unknown): never => {
70
129
  throw error;
71
130
  };
@@ -75,8 +134,11 @@ export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput):
75
134
  if (refusal !== undefined) refused(refusal);
76
135
  return;
77
136
  }
78
- if (held > 0) await sendPart();
137
+ if (filled > 0) await dispatch(buffer.subarray(0, filled));
138
+ await Promise.all(inFlight);
139
+ if (failures.length > 0) throw failures[0];
79
140
  const open = uploadId;
141
+ if (open === undefined) return;
80
142
  await completeS3Upload(
81
143
  target,
82
144
  open,
@@ -89,8 +151,10 @@ export async function putS3Stream(wire: S3WireOptions, input: S3PutStreamInput):
89
151
  await settleComplete(stat, { ...target, driver }, open, sent, lost);
90
152
  });
91
153
  } catch (error) {
92
- // Parts bill until they are aborted. The refusal the caller reads is the first one, so an abort
93
- // that fails too is left to the bucket's lifecycle rule rather than replacing it.
154
+ // A part still sending would land after the abort and bill: every one settles first. The
155
+ // refusal the caller reads is the first one, so an abort that fails too is left to the
156
+ // bucket's lifecycle rule rather than replacing it.
157
+ await Promise.allSettled(inFlight);
94
158
  if (uploadId !== undefined) await abortS3Upload(target, uploadId).catch(() => undefined);
95
159
  throw error;
96
160
  }
@@ -3,6 +3,7 @@
3
3
  // slice and its description come from one response about one version. A HEAD beside it could not
4
4
  // promise that: a write between the two, or an A → B → A overwrite of one length, slips past both.
5
5
 
6
+ import { renderThrowable } from '@ultimat3/core';
6
7
  import type { StorageObject, StorageRead } from './driver';
7
8
  import { DEFAULT_CONTENT_TYPE } from './driver';
8
9
  import { type S3Refusal, type S3WireOptions, sendSigned } from './driver-s3-signed';
@@ -23,14 +24,20 @@ export interface S3RangeDeps {
23
24
 
24
25
  const CONTENT_RANGE = /^bytes (\d+)-(\d+)\/(\d+)$/;
25
26
 
27
+ /** An answer refused before its body is read: the body is dropped, never left on the socket. */
28
+ async function discardBody(response: Response): Promise<void> {
29
+ try {
30
+ await response.body?.cancel();
31
+ } catch {
32
+ // A body that cannot be cancelled is already done: the refusal is the answer either way.
33
+ }
34
+ }
35
+
26
36
  /**
27
37
  * The window asked, CAPPED at `maxGetBytes` before it is sent: the ceiling bounds the range, and the
28
38
  * object's size is not known until the answer — so at most the ceiling crosses the wire, and an
29
39
  * answer that shows the object went on past the cap is refused before its body is read.
30
40
  */
31
- const errorMessage = (thrown: unknown): string =>
32
- thrown instanceof Error ? thrown.message : String(thrown);
33
-
34
41
  export async function readS3Range(
35
42
  wire: S3WireOptions,
36
43
  key: string,
@@ -52,13 +59,13 @@ export async function readS3Range(
52
59
  const { response } = result;
53
60
  const window = CONTENT_RANGE.exec(response.headers.get('content-range') ?? '');
54
61
  if (response.status !== 206 || window === null) {
55
- await response.body?.cancel().catch(() => undefined);
62
+ await discardBody(response);
56
63
  return deps.unreadable(key, `the ranged GET answered ${response.status} with no byte range`);
57
64
  }
58
65
  const served: ByteRange = [Number(window[1]), Number(window[2])];
59
66
  const size = Number(window[3]);
60
67
  if (served[0] !== first || served[1] > capped) {
61
- await response.body?.cancel().catch(() => undefined);
68
+ await discardBody(response);
62
69
  return deps.unreadable(
63
70
  key,
64
71
  `asked bytes ${first}-${capped}, answered ${served[0]}-${served[1]}`,
@@ -66,7 +73,7 @@ export async function readS3Range(
66
73
  }
67
74
  // Cut by the CAP, not by the end of the object: the range asked is over the ceiling.
68
75
  if (last > capped && size - 1 > capped) {
69
- await response.body?.cancel().catch(() => undefined);
76
+ await discardBody(response);
70
77
  throw rangeTooLarge(deps.driver, key, [first, Math.min(last, size - 1)], deps.maxGetBytes);
71
78
  }
72
79
  let bytes: Uint8Array;
@@ -74,7 +81,7 @@ export async function readS3Range(
74
81
  bytes = new Uint8Array(await response.arrayBuffer());
75
82
  } catch (thrown) {
76
83
  // The headers came, the body broke (a reset connection): the disk's coded read failure.
77
- return deps.unreadable(key, `the ranged body broke mid-read: ${errorMessage(thrown)}`);
84
+ return deps.unreadable(key, `the ranged body broke mid-read: ${renderThrowable(thrown)}`);
78
85
  }
79
86
  if (bytes.byteLength !== served[1] - served[0] + 1) {
80
87
  return deps.unreadable(
@@ -53,7 +53,7 @@ export interface S3Refusal {
53
53
  }
54
54
 
55
55
  export interface S3SignedRequest {
56
- readonly method: 'GET' | 'PUT' | 'POST' | 'DELETE';
56
+ readonly method: 'GET' | 'HEAD' | 'PUT' | 'POST' | 'DELETE';
57
57
  readonly key: string;
58
58
  /** A sub-resource such as `retention` — sent and signed as `?retention=`. */
59
59
  readonly subresource?: string | undefined;
@@ -0,0 +1,36 @@
1
+ // Single responsibility: the s3 disk's PINNED streamed read — one signed GET carrying `If-Match`, so
2
+ // the provider serves the revision a `stat()` described or answers 412. Bun's `S3File.stream()`
3
+ // sends no condition, and an overwrite between the stat and the read served new bytes under the
4
+ // old ETag and length.
5
+
6
+ import { type S3Refusal, type S3WireOptions, sendSigned } from './driver-s3-signed';
7
+ import type { ByteRange } from './range';
8
+
9
+ export interface S3PinnedDeps {
10
+ /** The provider's refusal as the disk's coded read error (404 is not-found there). */
11
+ readonly refused: (refusal: S3Refusal) => never;
12
+ /** Another revision is at the key: the disk's `X_STORAGE_OBJECT_CHANGED`. */
13
+ readonly changed: () => never;
14
+ }
15
+
16
+ /** The bytes of revision `etag`, whole or `[first, last]` already clamped to the object. */
17
+ export async function streamS3Pinned(
18
+ wire: S3WireOptions,
19
+ key: string,
20
+ etag: string,
21
+ range: ByteRange | undefined,
22
+ deps: S3PinnedDeps,
23
+ ): Promise<ReadableStream<Uint8Array>> {
24
+ const result = await sendSigned(wire, {
25
+ method: 'GET',
26
+ key,
27
+ headers: {
28
+ 'if-match': etag,
29
+ ...(range === undefined ? {} : { range: `bytes=${range[0]}-${range[1]}` }),
30
+ },
31
+ });
32
+ if (!result.ok) {
33
+ return result.refusal.statusCode === 412 ? deps.changed() : deps.refused(result.refusal);
34
+ }
35
+ return result.response.body ?? new Blob([]).stream();
36
+ }
package/src/driver-s3.ts CHANGED
@@ -27,12 +27,13 @@ import {
27
27
  type StorageListEntry,
28
28
  type StorageObject,
29
29
  type StorageRead,
30
+ type StreamOptions,
30
31
  sha256Base64,
31
32
  toBytes,
32
33
  } from './driver';
33
34
  import { isAbsentObject } from './driver-s3-absent';
34
35
  import { buildClient, dated, type S3ClientLike, type S3ListResultLike } from './driver-s3-client';
35
- import { signedCopy } from './driver-s3-copy';
36
+ import { copyS3 } from './driver-s3-copy-parts';
36
37
  import { needsSignedPut, readObjectLock, signedPut } from './driver-s3-lock';
37
38
  import { s3Multipart } from './driver-s3-multipart-session';
38
39
  import { s3SignedUrl } from './driver-s3-presign';
@@ -40,8 +41,9 @@ import { putS3Stream } from './driver-s3-put-stream';
40
41
  import { readS3Range } from './driver-s3-range';
41
42
  import { regionMismatch } from './driver-s3-region';
42
43
  import { settleLostPut } from './driver-s3-settle';
43
- import type { S3FetchLike } from './driver-s3-signed';
44
+ import type { S3FetchLike, S3Refusal } from './driver-s3-signed';
44
45
  import { refuseUnsupportedPut } from './driver-s3-sse';
46
+ import { streamS3Pinned } from './driver-s3-stream';
45
47
  import {
46
48
  checksumMismatch,
47
49
  deleteFailed,
@@ -53,6 +55,7 @@ import {
53
55
  readFailed,
54
56
  } from './errors';
55
57
  import { objectChanged } from './errors-promote';
58
+ import { revisionChanged } from './errors-stream';
56
59
  import { assertObjectLockOptions, type ObjectLock } from './object-lock';
57
60
  import { assertSafeKey } from './path';
58
61
  import { assertPutStream, type PutStreamOptions } from './put-stream';
@@ -197,6 +200,14 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
197
200
  };
198
201
  };
199
202
 
203
+ /** A pinned read's two refusals: the provider's, coded as a read, and another revision's. */
204
+ const pinned = (key: string, etag: string) => ({
205
+ refused: (refusal: S3Refusal): never => refusedRead(key)(refusal),
206
+ changed: (): never => {
207
+ throw revisionChanged(diskName, key, etag);
208
+ },
209
+ });
210
+
200
211
  /** The HEAD a lost write's answer is settled against, and the `.catch` of a single PUT. */
201
212
  const statOf = async (key: string) => ((await present(key)) ? statObject(key) : undefined);
202
213
  const lostPut = (key: string, bytes: Uint8Array, attributes: PutOptions | undefined) =>
@@ -313,14 +324,21 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
313
324
  return { object, bytes };
314
325
  },
315
326
 
316
- async stream(key: string): Promise<ReadableStream<Uint8Array>> {
327
+ async stream(key: string, streamOptions?: StreamOptions): Promise<ReadableStream<Uint8Array>> {
317
328
  const safe = assertSafeKey(key);
329
+ const etag = streamOptions?.etag;
330
+ if (etag !== undefined)
331
+ return streamS3Pinned(options, safe, etag, undefined, pinned(safe, etag));
318
332
  const file = conn().file(safe);
319
333
  if (!(await present(safe))) throw objectNotFound(diskName, safe);
320
334
  return file.stream();
321
335
  },
322
336
 
323
- async streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>> {
337
+ async streamRange(
338
+ key: string,
339
+ range: ByteRange,
340
+ streamOptions?: StreamOptions,
341
+ ): Promise<ReadableStream<Uint8Array>> {
324
342
  const safe = assertSafeKey(key);
325
343
  assertStreamRange(DRIVER_NAME, range);
326
344
  if (!(await present(safe))) throw objectNotFound(diskName, safe);
@@ -328,6 +346,10 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
328
346
  // already answered 206. Refused here it is still a status.
329
347
  const { size } = await statObject(safe);
330
348
  const [first, last] = streamedRange(DRIVER_NAME, safe, range, size);
349
+ const etag = streamOptions?.etag;
350
+ if (etag !== undefined) {
351
+ return streamS3Pinned(options, safe, etag, [first, last], pinned(safe, etag));
352
+ }
331
353
  return conn()
332
354
  .file(safe)
333
355
  .slice(first, last + 1)
@@ -335,33 +357,28 @@ export function s3Driver(options: S3DriverOptions): StorageDriver {
335
357
  },
336
358
 
337
359
  /**
338
- * Bytes from one key to another without a round trip through this process. Bun exposes no
339
- * `CopyObject`, so the source `S3File` is handed to `write()` — Bun's own union accepts one —
340
- * and the bytes move inside Bun rather than through the app's heap. That is the half of the
341
- * win available today: promoting a 500MB attachment used to be `get()` + `put()`, a full GB
342
- * resident in the pod. A true server-side copy needs a Bun API that does not exist in 1.3.
360
+ * Bytes from one key to another, server-side: ONE signed CopyObject (directive COPY), so the
361
+ * source's type, `Cache-Control` and `x-amz-meta-*` travel as on the local and memory disks.
362
+ * Past CopyObject's 5 GiB ceiling, a multipart UploadPartCopy carrying the same headers.
343
363
  */
344
364
  async copy(from: string, to: string, copyOptions?: CopyOptions): Promise<StorageObject> {
345
365
  const source = assertSafeKey(from);
346
366
  const destination = assertSafeKey(to);
347
- // Bun's `write` takes no condition, so a revision-bound copy is the signed CopyObject.
348
- if (copyOptions?.ifMatch !== undefined) {
349
- const refusal = await signedCopy(options, source, destination, copyOptions.ifMatch).catch(
350
- refusedWrite(destination),
351
- );
352
- if (refusal?.statusCode === 404) throw objectNotFound(diskName, source);
353
- if (refusal?.statusCode === 412)
354
- throw objectChanged(source, copyOptions.ifMatch, 'another');
355
- if (refusal !== undefined) refusedWrite(destination)(refusal);
356
- return statObject(destination);
357
- }
358
- const file = conn().file(source);
359
- if (!(await present(source))) throw objectNotFound(diskName, source);
360
- const stat = await file.stat().catch(refusedRead(source));
361
- await conn()
362
- .file(destination)
363
- .write(file, { type: stat.type ?? DEFAULT_CONTENT_TYPE })
364
- .catch(refusedWrite(destination));
367
+ const ifMatch = copyOptions?.ifMatch;
368
+ const found = await statOf(source);
369
+ if (found === undefined) throw objectNotFound(diskName, source);
370
+ const expected = ifMatch ?? found.etag;
371
+ await copyS3(
372
+ options,
373
+ { source, destination, ifMatch, size: found.size, stat: statOf },
374
+ (refusal) => {
375
+ if (refusal.statusCode === 404 && refusal.code !== 'NoSuchUpload') {
376
+ throw objectNotFound(diskName, source);
377
+ }
378
+ if (refusal.statusCode === 412) throw objectChanged(source, expected, 'another');
379
+ return refusedWrite(destination)(refusal);
380
+ },
381
+ ).catch(refusedWrite(destination));
365
382
  return statObject(destination);
366
383
  },
367
384
 
package/src/driver.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  // and never unbounded, which is the same failure with the sender choosing the size.
5
5
 
6
6
  import type { Clock, Page } from '@ultimat3/core';
7
- import { assert } from '@ultimat3/core';
7
+ import { assert, describeValue } from '@ultimat3/core';
8
8
  import { pumpBounded } from './byte-pump';
9
9
  import { putTooLarge } from './errors';
10
10
  import type { MultipartCapability } from './multipart';
@@ -172,14 +172,22 @@ export interface StorageDriver {
172
172
  * the range's length: s3 sends `Range:`, the local disk slices the file, memory slices its copy.
173
173
  */
174
174
  get(key: string, options?: GetOptions): Promise<StorageRead>;
175
- /** Bytes without buffering — the only safe path for anything over a few MB. */
176
- stream(key: string): Promise<ReadableStream<Uint8Array>>;
175
+ /**
176
+ * Bytes without buffering — the only safe path for anything over a few MB. `etag` pins the read
177
+ * to the revision a `stat()` described: another is `X_STORAGE_OBJECT_CHANGED`, never its bytes
178
+ * under the old validator.
179
+ */
180
+ stream(key: string, options?: StreamOptions): Promise<ReadableStream<Uint8Array>>;
177
181
  /**
178
182
  * The bytes `[first, last]` (inclusive, clamped, as `get`'s range) with none before them read, so
179
183
  * a seek late in a film costs its window. `X_STORAGE_RANGE_UNSATISFIABLE` past the end. A method and
180
184
  * not an option on `stream`, which a third-party driver would ignore and serve the wrong bytes.
181
185
  */
182
- streamRange(key: string, range: ByteRange): Promise<ReadableStream<Uint8Array>>;
186
+ streamRange(
187
+ key: string,
188
+ range: ByteRange,
189
+ options?: StreamOptions,
190
+ ): Promise<ReadableStream<Uint8Array>>;
183
191
  /**
184
192
  * Bytes from one key to another without them passing through this process. Idempotent in the
185
193
  * destination and non-destructive in the source: the caller deletes the source afterwards if
@@ -273,14 +281,31 @@ const isStringRecord = (value: unknown): boolean =>
273
281
  * `metadata: { n: 1n }` reached `JSON.stringify` on the local disk as a bare `TypeError`, and the
274
282
  * memory disk stored it — against the `Record<string, string>` every reader is typed on.
275
283
  */
284
+ /** A streamed read's options: the revision it may serve. */
285
+ export interface StreamOptions {
286
+ /** The etag a `stat()` answered; any other revision is refused rather than served. */
287
+ readonly etag?: string | undefined;
288
+ }
289
+
290
+ /** Printable ASCII: what one HTTP header value may carry, on the way in and on the way out. */
291
+ export const HEADER_VALUE = /^[\x20-\x7e]*$/;
292
+
276
293
  export function assertPutOptions(driver: string, options: PutOptions | undefined): void {
277
294
  if (options === undefined) return;
278
295
  for (const name of ['contentType', 'cacheControl'] as const) {
296
+ const value = options[name];
279
297
  assert(
280
- options[name] === undefined || typeof options[name] === 'string',
298
+ value === undefined || typeof value === 'string',
281
299
  `put() on the ${driver} disk was given a ${name} that is not a string`,
282
300
  `put(key, body, { ${name}: 'a string' }) — or omit ${name}`,
283
301
  );
302
+ // Served back as a header on every disk and SENT as one on s3: a line break is a second
303
+ // header, and on the signed wire a refused one read as a lost answer.
304
+ assert(
305
+ value === undefined || HEADER_VALUE.test(value),
306
+ `put() on the ${driver} disk was given a ${name} that cannot travel as a header (${describeValue(value)})`,
307
+ `put(key, body, { ${name}: 'text/plain' }) — printable ASCII, no line breaks`,
308
+ );
284
309
  }
285
310
  assert(
286
311
  options.metadata === undefined || isStringRecord(options.metadata),
@@ -1,23 +1,11 @@
1
- // The refusals of the pending → attached move (`promoteAttachment`, `releaseQuarantine`): a key
2
- // still in quarantine, a key that is not a pending upload, and one an earlier call already moved.
1
+ // The refusals of the pending → attached move (`promoteAttachment`): a key that is not a pending
2
+ // upload, one an earlier call already moved, and one that changed while it was moved. A scanner's
3
+ // refusals are `errors-scan.ts`'s.
3
4
  // Their codes are registered with the rest of the package's in `errors.ts`.
4
5
 
5
6
  import { renderFixLiteral } from '@ultimat3/core';
6
7
  import { StorageError } from './errors';
7
8
 
8
- /**
9
- * A key still under the quarantine prefix. The framework never scans bytes — that is the app's
10
- * job — so the only thing it can enforce is that nothing leaves quarantine without the app
11
- * saying so, which is what `promoteAttachment` refusing this key means.
12
- */
13
- export const quarantined = (key: string, orgId: string): StorageError =>
14
- new StorageError({
15
- code: 'X_STORAGE_QUARANTINED',
16
- cause: `"${key}" is still under the quarantine prefix, so nothing has cleared it for use`,
17
- fix: `scan the bytes, then releaseQuarantine({ disk, key: '${key}', orgId: '${orgId}' }) — promote the key it returns`,
18
- meta: { key, orgId },
19
- });
20
-
21
9
  /**
22
10
  * A key inside the org and outside its `pending/` prefix — most often another row's attached key
23
11
  * a client sent back. Promotion copies then deletes, so accepting it moved the victim's file onto
@@ -0,0 +1,61 @@
1
+ // The three refusals of a scanned promotion (`upload-scan.ts`): the scanner said infected, said
2
+ // hold, or could not say. Their codes are registered with the rest of the package's in `errors.ts`.
3
+
4
+ import { renderFixLiteral } from '@ultimat3/core';
5
+ import { StorageError } from './errors';
6
+
7
+ /** Never retryable with the same bytes, and never overridable by a policy: they are deleted. */
8
+ export const uploadInfected = (
9
+ key: string,
10
+ scanner: string,
11
+ signature: string,
12
+ removed: boolean,
13
+ ): StorageError =>
14
+ new StorageError({
15
+ code: 'X_STORAGE_UPLOAD_INFECTED',
16
+ cause: `the ${scanner} upload scanner found "${signature}" in "${key}"; ${
17
+ removed
18
+ ? 'the pending upload and its scan copy were deleted'
19
+ : 'deleting it failed, so sweepOrphans will collect what is left under pending/'
20
+ }`,
21
+ fix: 'grantUpload({ disk, orgId, request }) for a new, clean file — this one is refused for good',
22
+ meta: { key, scanner, signature, removed },
23
+ });
24
+
25
+ /**
26
+ * The scanner rejected rather than answered. `reason` is already rendered (`renderThrowable`): a
27
+ * daemon's own words are the most useful thing in the cause, and an `unknown` is never interpolated.
28
+ */
29
+ export const scanUnavailable = (key: string, scanner: string, reason: string): StorageError =>
30
+ new StorageError({
31
+ code: 'X_STORAGE_SCAN_UNAVAILABLE',
32
+ cause: `the ${scanner} upload scanner gave no verdict for "${key}": ${reason}`,
33
+ fix: `promoteAttachment({ disk, key: ${renderFixLiteral(key, "'<pending key>'")}, orgId, target, policy }) # again once the ${scanner} scanner answers — the upload is still pending; or uploadPolicy({ onScanUnavailable: 'accept' }) to promote unscanned files`,
34
+ meta: { key, scanner },
35
+ });
36
+
37
+ /**
38
+ * The scanner answered `hold`: a decision made outside this call (a review, an async vendor scan)
39
+ * has not cleared the upload yet. It keeps the code the retired `quarantine: true` flow raised —
40
+ * the same state, a 409 — now reached through the one scan path. `reason` is the scanner's string.
41
+ */
42
+ export const uploadHeld = (key: string, scanner: string, reason: string): StorageError =>
43
+ new StorageError({
44
+ code: 'X_STORAGE_QUARANTINED',
45
+ cause: `the ${scanner} upload scanner is holding "${key}" (${reason}); the upload stays pending and nothing was attached`,
46
+ fix: `promoteAttachment({ disk, key: ${renderFixLiteral(key, "'<pending key>'")}, orgId, target, policy }) # again once your ${scanner} scanner answers clean for this pendingKey — sweepOrphans collects it if it never does`,
47
+ meta: { key, scanner, reason },
48
+ });
49
+
50
+ /**
51
+ * A grant aimed straight at a row's key while a scanner is installed. That key never passes through
52
+ * `promoteAttachment`, the one place a scan runs, so signing it would land bytes nothing scanned
53
+ * exactly where a row reads them.
54
+ */
55
+ export const targetUnscanned = (caller: string, key: string, scanner: string): StorageError =>
56
+ new StorageError({
57
+ code: 'X_STORAGE_TARGET_UNSCANNED',
58
+ cause: `${caller} was given a target ("${key}") while the ${scanner} upload scanner is installed; a targeted key never passes through promoteAttachment, so its bytes would reach the row unscanned`,
59
+ fix: `${caller}({ disk, orgId, request, policy }) # no target — then promoteAttachment({ disk, key, orgId, target, policy }) once the upload lands`,
60
+ meta: { key, scanner },
61
+ });
@@ -21,6 +21,19 @@ export const streamTooLarge = (
21
21
  meta: { disk, key, bytes, maxBytes },
22
22
  });
23
23
 
24
+ /**
25
+ * A streamed read pinned to a revision the key no longer holds: written between the `stat()` whose
26
+ * etag, size and type a response was built on and the read of its bytes. Same code as a promotion
27
+ * that met another revision, for the same reason.
28
+ */
29
+ export const revisionChanged = (disk: string, key: string, expected: string): StorageError =>
30
+ new StorageError({
31
+ code: 'X_STORAGE_OBJECT_CHANGED',
32
+ cause: `"${key}" on the ${disk} disk was overwritten after it was described as ${expected}, so its bytes are another revision's and were not served`,
33
+ fix: `const object = await disk.stat(${keyArg(key)}); await disk.stream(${keyArg(key)}, { etag: object.etag }) # describe it again, then read that revision`,
34
+ meta: { disk, key, expected },
35
+ });
36
+
24
37
  /** `rangeUnsatisfiable` for a call that holds no size: the next step is learning it. */
25
38
  export const streamRangeUnsatisfiable = (
26
39
  disk: string,
@@ -0,0 +1,40 @@
1
+ // The refusals of `variant()` (`image-variant.ts`). A source that is no raster and a driver that
2
+ // answered another encoding are core's `X_IMAGE_UNSUPPORTED`, as every other image refusal here is;
3
+ // a source overwritten mid-transform is storage's `X_STORAGE_OBJECT_CHANGED`.
4
+
5
+ import { type ImageUnsupportedError, imageUnsupported, renderFixLiteral } from '@ultimat3/core';
6
+ import { StorageError } from './errors';
7
+
8
+ export const notARaster = (key: string, contentType: string): ImageUnsupportedError =>
9
+ imageUnsupported(
10
+ `"${key}" is stored as ${contentType}, which is not a raster image, so it has no width to resize to`,
11
+ `await disk.stat(${renderFixLiteral(key, "'<key>'")}) # variant() takes a png, jpeg, webp, gif or avif source: check its contentType, or serve it as-is`,
12
+ { key, contentType },
13
+ );
14
+
15
+ export const variantFormatUnknown = (key: string, contentType: string): ImageUnsupportedError =>
16
+ imageUnsupported(
17
+ `"${key}" is a ${contentType} source and no variant format was named, so no key can say what the transform will answer`,
18
+ `variant(${renderFixLiteral(key, "'<key>'")}, { disk, images, width, format: 'webp' })`,
19
+ { key, contentType },
20
+ );
21
+
22
+ export const driverEncodingMismatch = (
23
+ key: string,
24
+ driver: string,
25
+ expected: string,
26
+ actual: string,
27
+ ): ImageUnsupportedError =>
28
+ imageUnsupported(
29
+ `the ${driver} image driver answered ${actual} for "${key}", whose key names ${expected}, so the bytes would be served under the wrong type`,
30
+ `variant(${renderFixLiteral(key, "'<key>'")}, { disk, images, width, format: '<the format the ${driver} driver produces>' }) # or make its transform() honour request.format`,
31
+ { key, driver, expected, actual },
32
+ );
33
+
34
+ export const variantSourceChanged = (key: string, expected: string, actual: string): StorageError =>
35
+ new StorageError({
36
+ code: 'X_STORAGE_OBJECT_CHANGED',
37
+ cause: `"${key}" was overwritten while its variant was being made (etag ${expected} → ${actual}), so the bytes cannot be filed under either revision`,
38
+ fix: `variant(${renderFixLiteral(key, "'<key>'")}, { disk, images, width }) again — it keys the variant by the new revision`,
39
+ meta: { key, expected, actual },
40
+ });
package/src/errors.ts CHANGED
@@ -39,6 +39,9 @@ export const STORAGE_OWNED_ERROR_CODES = [
39
39
  'X_STORAGE_MULTIPART_UNSUPPORTED',
40
40
  'X_STORAGE_UPLOAD_UNKNOWN',
41
41
  'X_STORAGE_PARTS_INVALID',
42
+ 'X_STORAGE_UPLOAD_INFECTED',
43
+ 'X_STORAGE_SCAN_UNAVAILABLE',
44
+ 'X_STORAGE_TARGET_UNSCANNED',
42
45
  ] as const;
43
46
 
44
47
  /**
@@ -89,6 +92,9 @@ export const STORAGE_ERROR_TITLES: Readonly<Record<StorageOwnedErrorCode, string
89
92
  X_STORAGE_MULTIPART_UNSUPPORTED: 'this disk cannot take an upload in parts',
90
93
  X_STORAGE_UPLOAD_UNKNOWN: 'no open multipart upload has that id',
91
94
  X_STORAGE_PARTS_INVALID: 'the parts named do not match the parts stored',
95
+ X_STORAGE_UPLOAD_INFECTED: 'the upload scanner found malware in the file',
96
+ X_STORAGE_SCAN_UNAVAILABLE: 'the upload scanner could not give a verdict',
97
+ X_STORAGE_TARGET_UNSCANNED: 'a targeted upload grant would bypass the upload scanner',
92
98
  };
93
99
 
94
100
  // One unconditional call, so a second package claiming one of storage's codes throws