@ultimat3/storage 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +27 -47
- package/README.md +130 -12
- package/package.json +12 -2
- package/src/accept-part.ts +30 -0
- package/src/accept.ts +7 -0
- package/src/attachment.ts +99 -6
- package/src/byte-pump.ts +46 -0
- package/src/driver-local-list.ts +75 -0
- package/src/driver-local-multipart.ts +133 -0
- package/src/driver-local-read.ts +13 -0
- package/src/driver-local-remove.ts +39 -0
- package/src/driver-local-stream.ts +104 -0
- package/src/driver-local-write.ts +28 -5
- package/src/driver-local.ts +257 -283
- package/src/driver-memory-multipart.ts +41 -0
- package/src/driver-memory.ts +205 -135
- package/src/driver-s3-client.ts +9 -0
- package/src/driver-s3-copy.ts +35 -0
- package/src/driver-s3-lock.ts +9 -2
- package/src/driver-s3-multipart-session.ts +87 -0
- package/src/driver-s3-multipart.ts +177 -0
- package/src/driver-s3-presign.ts +52 -0
- package/src/driver-s3-put-stream.ts +97 -0
- package/src/driver-s3-range.ts +99 -0
- package/src/driver-s3-settle.ts +150 -0
- package/src/driver-s3-signed.ts +69 -4
- package/src/driver-s3-sse.ts +29 -0
- package/src/driver-s3.ts +113 -65
- package/src/driver.ts +98 -38
- package/src/errors-multipart.ts +35 -0
- package/src/errors-promote.ts +14 -0
- package/src/errors-range.ts +47 -0
- package/src/errors-sniff.ts +35 -0
- package/src/errors-stream.ts +62 -0
- package/src/errors.ts +16 -1
- package/src/expire.ts +169 -0
- package/src/grant-multipart.ts +196 -0
- package/src/grant.ts +42 -25
- package/src/image-warm.ts +84 -0
- package/src/image.ts +57 -0
- package/src/index.ts +81 -2
- package/src/multipart-staged.ts +127 -0
- package/src/multipart.ts +119 -0
- package/src/put-stream.ts +52 -0
- package/src/range.ts +102 -0
- package/src/signed-url.ts +59 -2
- package/src/subject-target.ts +82 -0
- package/src/upload-client-multipart.ts +156 -0
- package/src/upload-client.ts +21 -0
- package/src/upload.ts +107 -21
package/src/errors-promote.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// still in quarantine, a key that is not a pending upload, and one an earlier call already moved.
|
|
3
3
|
// Their codes are registered with the rest of the package's in `errors.ts`.
|
|
4
4
|
|
|
5
|
+
import { renderFixLiteral } from '@ultimat3/core';
|
|
5
6
|
import { StorageError } from './errors';
|
|
6
7
|
|
|
7
8
|
/**
|
|
@@ -42,3 +43,16 @@ export const alreadyPromoted = (key: string, attachedKey: string, orgId: string)
|
|
|
42
43
|
fix: `await disk.stat('${attachedKey}') answers the moved object: answer the row that references that key if one exists, write it with that key if none does, and never delete the key on a failure of your own`,
|
|
43
44
|
meta: { key, attachedKey, orgId },
|
|
44
45
|
});
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The object is no longer the revision that was judged. A presigned PUT stays valid until it
|
|
49
|
+
* expires, so an uploader can overwrite between the sniff and the copy, and what would land on the
|
|
50
|
+
* row's key is bytes nothing looked at.
|
|
51
|
+
*/
|
|
52
|
+
export const objectChanged = (key: string, expected: string, actual: string): StorageError =>
|
|
53
|
+
new StorageError({
|
|
54
|
+
code: 'X_STORAGE_OBJECT_CHANGED',
|
|
55
|
+
cause: `"${key}" changed while it was being promoted: it was judged at ${expected} and is now ${actual}`,
|
|
56
|
+
fix: `promoteAttachment({ disk, key: ${renderFixLiteral(key, "'<pending key>'")}, orgId, target, policy }) once the upload has finished: it reads the object again`,
|
|
57
|
+
meta: { key, expected, actual },
|
|
58
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// The two refusals of a ranged `get(key, { range })`: a first byte past the end (HTTP's 416) and a
|
|
2
|
+
// window over the disk's get ceiling. Their codes are registered with the rest of the package's in
|
|
3
|
+
// `errors.ts`; each `fix:` is the call to make next, with the numbers this object allows.
|
|
4
|
+
|
|
5
|
+
import { renderFixLiteral } from '@ultimat3/core';
|
|
6
|
+
import { StorageError } from './errors';
|
|
7
|
+
|
|
8
|
+
const keyArg = (key: string): string => renderFixLiteral(key, "'<key>'");
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* A first byte at or past the end. The last byte is CLAMPED (`range.ts`), so only the first can
|
|
12
|
+
* miss; an empty object has no first byte at all, so its fix is the plain read.
|
|
13
|
+
*/
|
|
14
|
+
export const rangeUnsatisfiable = (
|
|
15
|
+
disk: string,
|
|
16
|
+
key: string,
|
|
17
|
+
range: readonly [number, number],
|
|
18
|
+
size: number,
|
|
19
|
+
): StorageError =>
|
|
20
|
+
new StorageError({
|
|
21
|
+
code: 'X_STORAGE_RANGE_UNSATISFIABLE',
|
|
22
|
+
cause: `get() asked "${key}" for bytes ${range[0]}-${range[1]}, and the object on the ${disk} disk is ${size}B — its first byte is past the end`,
|
|
23
|
+
fix:
|
|
24
|
+
size === 0
|
|
25
|
+
? `await disk.get(${keyArg(key)}) # the object is empty: no range has a first byte`
|
|
26
|
+
: `await disk.get(${keyArg(key)}, { range: [0, ${size - 1}] }) # the whole ${size}B object; any first byte below ${size}`,
|
|
27
|
+
meta: { disk, key, first: range[0], last: range[1], size },
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* A window over `maxGetBytes` — the ceiling bounds the RANGE, so the repair is a smaller window
|
|
32
|
+
* from the same first byte, or a higher ceiling on the disk.
|
|
33
|
+
*/
|
|
34
|
+
export const rangeTooLarge = (
|
|
35
|
+
disk: string,
|
|
36
|
+
key: string,
|
|
37
|
+
range: readonly [number, number],
|
|
38
|
+
maxBytes: number,
|
|
39
|
+
): StorageError => {
|
|
40
|
+
const bytes = range[1] - range[0] + 1;
|
|
41
|
+
return new StorageError({
|
|
42
|
+
code: 'X_STORAGE_TOO_LARGE',
|
|
43
|
+
cause: `the range asked of "${key}" is ${bytes}B against the ${disk} disk's get ceiling of ${maxBytes}B, and get() buffers the range`,
|
|
44
|
+
fix: `await disk.get(${keyArg(key)}, { range: [${range[0]}, ${range[0] + maxBytes - 1}] }) # one window within the ceiling; or raise maxGetBytes to ${bytes} on the disk`,
|
|
45
|
+
meta: { disk, key, bytes, maxBytes, first: range[0], last: range[1] },
|
|
46
|
+
});
|
|
47
|
+
};
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// The two refusals only a bounded sniff can reach: markup that could sit past what was read. Their
|
|
2
|
+
// code is `X_STORAGE_TYPE_REJECTED`, registered with the rest of the package's in `errors.ts`; they
|
|
3
|
+
// are their own constructors because the sniff saw nothing to name, so `contentTypeMismatch`'s
|
|
4
|
+
// "re-upload with the sniffed type" would be advice about a type nobody found.
|
|
5
|
+
|
|
6
|
+
import { renderFixLiteral } from '@ultimat3/core';
|
|
7
|
+
import { StorageError } from './errors';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A body opening with more whitespace than `limit` bytes. Past a window nothing is seen, so padding
|
|
11
|
+
* is how markup would hide from a head read, and no real file opens this way.
|
|
12
|
+
*/
|
|
13
|
+
export const leadingWhitespacePastLimit = (
|
|
14
|
+
key: string,
|
|
15
|
+
declared: string,
|
|
16
|
+
limit: number,
|
|
17
|
+
): StorageError =>
|
|
18
|
+
new StorageError({
|
|
19
|
+
code: 'X_STORAGE_TYPE_REJECTED',
|
|
20
|
+
cause: `"${key}" declares ${declared} and opens with more than ${limit} bytes of whitespace, so its type cannot be read`,
|
|
21
|
+
fix: `validateUpload({ key: ${renderFixLiteral(key, "'a/file.txt'")}, declaredContentType: ${renderFixLiteral(declared, "'text/plain'")}, bytes: new TextEncoder().encode(text.trimStart()) }, policy) — where text is the file with its leading whitespace removed`,
|
|
22
|
+
meta: { key, declared, limit },
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* An XML document too long for a head window to screen: XML runs a script anywhere in the tree, so
|
|
27
|
+
* no prefix can clear it. The server-side path reads all of it, which is the way through.
|
|
28
|
+
*/
|
|
29
|
+
export const xmlBodyPastWindow = (key: string, declared: string, window: number): StorageError =>
|
|
30
|
+
new StorageError({
|
|
31
|
+
code: 'X_STORAGE_TYPE_REJECTED',
|
|
32
|
+
cause: `"${key}" declares ${declared} and is over the ${window} bytes promotion reads, and an XML document can run a script anywhere in it`,
|
|
33
|
+
fix: `validateUpload({ key: ${renderFixLiteral(key, "'a/data.xml'")}, declaredContentType: ${renderFixLiteral(declared, "'application/xml'")}, bytes }, policy) — upload it through the server, which reads the whole body, then disk.put(key, bytes, { contentType })`,
|
|
34
|
+
meta: { key, declared, window },
|
|
35
|
+
});
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// The refusals of the streaming calls and of a size a grant was not told. Registered with the rest
|
|
2
|
+
// of the package's codes in `errors.ts`; each `fix:` names the call that goes next, because these
|
|
3
|
+
// calls have their own option where `put()`'s refusal names the disk's ceiling.
|
|
4
|
+
|
|
5
|
+
import { describeValue, renderFixLiteral, renderThrowable } from '@ultimat3/core';
|
|
6
|
+
import { StorageError } from './errors';
|
|
7
|
+
|
|
8
|
+
const keyArg = (key: string): string => renderFixLiteral(key, "'<key>'");
|
|
9
|
+
|
|
10
|
+
/** A `putStream` body past the call's own `maxBytes`. Counted, so `bytes` is how far it had got. */
|
|
11
|
+
export const streamTooLarge = (
|
|
12
|
+
disk: string,
|
|
13
|
+
key: string,
|
|
14
|
+
bytes: number,
|
|
15
|
+
maxBytes: number,
|
|
16
|
+
): StorageError =>
|
|
17
|
+
new StorageError({
|
|
18
|
+
code: 'X_STORAGE_TOO_LARGE',
|
|
19
|
+
cause: `the stream for "${key}" passed its maxBytes of ${maxBytes}B at ${bytes}B on the ${disk} disk, so the write was abandoned and nothing was stored`,
|
|
20
|
+
fix: `await disk.putStream(${keyArg(key)}, stream, { maxBytes: ${bytes * 2} }) # a higher maxBytes on the call, if a body this large is meant`,
|
|
21
|
+
meta: { disk, key, bytes, maxBytes },
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
/** `rangeUnsatisfiable` for a call that holds no size: the next step is learning it. */
|
|
25
|
+
export const streamRangeUnsatisfiable = (
|
|
26
|
+
disk: string,
|
|
27
|
+
key: string,
|
|
28
|
+
range: readonly [number, number],
|
|
29
|
+
size: number,
|
|
30
|
+
): StorageError =>
|
|
31
|
+
new StorageError({
|
|
32
|
+
code: 'X_STORAGE_RANGE_UNSATISFIABLE',
|
|
33
|
+
cause: `streamRange() asked "${key}" for bytes ${range[0]}-${range[1]}, and the object on the ${disk} disk is ${size}B, so its first byte is past the end`,
|
|
34
|
+
fix: `const { size } = await disk.stat(${keyArg(key)}) # the object's size now; any first byte below it is a window`,
|
|
35
|
+
meta: { disk, key, first: range[0], last: range[1], size },
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* A grant with nothing to bind. A presign has one way to hold a size, the exact `Content-Length`
|
|
40
|
+
* inside the signature, so "any size up to the policy" is a URL that bounds nothing.
|
|
41
|
+
*/
|
|
42
|
+
export const uploadSizeUndeclared = (key: string, size: unknown): StorageError =>
|
|
43
|
+
new StorageError({
|
|
44
|
+
code: 'X_STORAGE_UPLOAD_SIZE_UNDECLARED',
|
|
45
|
+
cause: `the upload for "${key}" declared its size as ${describeValue(size)}, not a whole byte count of at least 1, so the URL could not bind one`,
|
|
46
|
+
fix: 'grantUpload({ disk, orgId, request: { filename, contentType, size: file.size } })',
|
|
47
|
+
meta: { key },
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* A write sent whose answer was lost, and nothing could settle what it did. Never `putFailed`: that
|
|
52
|
+
* says the key is as it was, and the object may be the one sent. `X_DB_COMMIT_UNKNOWN`'s idea.
|
|
53
|
+
*/
|
|
54
|
+
export const writeOutcomeUnknown = (disk: string, key: string, lost: unknown): StorageError => {
|
|
55
|
+
const reason = renderThrowable(lost);
|
|
56
|
+
return new StorageError({
|
|
57
|
+
code: 'X_STORAGE_WRITE_OUTCOME_UNKNOWN',
|
|
58
|
+
cause: `the write of "${key}" on the ${disk} disk was sent and its answer lost (${reason}), and neither a stat nor an abort could say whether it landed`,
|
|
59
|
+
fix: `await disk.stat(${keyArg(key)}) # the size and etag now at the key; the same as what was sent means it landed`,
|
|
60
|
+
meta: { disk, key, reason },
|
|
61
|
+
});
|
|
62
|
+
};
|
package/src/errors.ts
CHANGED
|
@@ -32,6 +32,13 @@ export const STORAGE_OWNED_ERROR_CODES = [
|
|
|
32
32
|
'X_STORAGE_OBJECT_LOCKED',
|
|
33
33
|
'X_STORAGE_KEY_UNSHARED',
|
|
34
34
|
'X_STORAGE_ALREADY_PROMOTED',
|
|
35
|
+
'X_STORAGE_RANGE_UNSATISFIABLE',
|
|
36
|
+
'X_STORAGE_OBJECT_CHANGED',
|
|
37
|
+
'X_STORAGE_UPLOAD_SIZE_UNDECLARED',
|
|
38
|
+
'X_STORAGE_WRITE_OUTCOME_UNKNOWN',
|
|
39
|
+
'X_STORAGE_MULTIPART_UNSUPPORTED',
|
|
40
|
+
'X_STORAGE_UPLOAD_UNKNOWN',
|
|
41
|
+
'X_STORAGE_PARTS_INVALID',
|
|
35
42
|
] as const;
|
|
36
43
|
|
|
37
44
|
/**
|
|
@@ -74,6 +81,14 @@ export const STORAGE_ERROR_TITLES: Readonly<Record<StorageOwnedErrorCode, string
|
|
|
74
81
|
X_STORAGE_OBJECT_LOCKED: 'the object is under retention or a legal hold',
|
|
75
82
|
X_STORAGE_KEY_UNSHARED: 'object key is outside every tenant and every shared prefix',
|
|
76
83
|
X_STORAGE_ALREADY_PROMOTED: 'the upload was already promoted onto its row',
|
|
84
|
+
X_STORAGE_RANGE_UNSATISFIABLE: 'the requested byte range starts past the end of the object',
|
|
85
|
+
X_STORAGE_OBJECT_CHANGED: 'the object changed while it was being promoted',
|
|
86
|
+
X_STORAGE_UPLOAD_SIZE_UNDECLARED: 'an upload grant was asked for without the size of the file',
|
|
87
|
+
X_STORAGE_WRITE_OUTCOME_UNKNOWN:
|
|
88
|
+
'a write was sent and its answer lost, and the disk could not say whether it landed',
|
|
89
|
+
X_STORAGE_MULTIPART_UNSUPPORTED: 'this disk cannot take an upload in parts',
|
|
90
|
+
X_STORAGE_UPLOAD_UNKNOWN: 'no open multipart upload has that id',
|
|
91
|
+
X_STORAGE_PARTS_INVALID: 'the parts named do not match the parts stored',
|
|
77
92
|
};
|
|
78
93
|
|
|
79
94
|
// One unconditional call, so a second package claiming one of storage's codes throws
|
|
@@ -193,7 +208,7 @@ export const getTooLarge = (
|
|
|
193
208
|
code: 'X_STORAGE_TOO_LARGE',
|
|
194
209
|
cause: `"${key}" is ${bytes}B against the ${disk} disk's get ceiling of ${maxBytes}B, and get() buffers the whole object`,
|
|
195
210
|
fix:
|
|
196
|
-
`
|
|
211
|
+
`await disk.stream(${renderFixLiteral(key, "'<key>'")}) # unbuffered; or a window with get(key, { range }), or raise the ceiling: ` +
|
|
197
212
|
(disk === 's3'
|
|
198
213
|
? `s3Driver({ bucket, maxGetBytes: ${bytes} })`
|
|
199
214
|
: disk === 'memory'
|
package/src/expire.ts
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// Single responsibility: objects and upload sessions past a window the APP names, swept on any
|
|
2
|
+
// disk. Bucket lifecycle rules stay the operator's (axiom 7); a local disk has none, and abandoned
|
|
3
|
+
// parts bill silently forever without one, so a sweep that works the same on every disk is
|
|
4
|
+
// mechanism. No timer here: `objectPurgeTarget` hands the sweep to the app's one `purge()` job.
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
assert,
|
|
8
|
+
type Clock,
|
|
9
|
+
finiteCount,
|
|
10
|
+
logger,
|
|
11
|
+
renderThrowable,
|
|
12
|
+
systemClock,
|
|
13
|
+
} from '@ultimat3/core';
|
|
14
|
+
import type { SweepFailure, SweepResult } from './attachment';
|
|
15
|
+
import type { StorageDriver, StorageListEntry } from './driver';
|
|
16
|
+
import type { MultipartSession } from './multipart';
|
|
17
|
+
|
|
18
|
+
export interface ExpireObjectsInput {
|
|
19
|
+
readonly disk: StorageDriver;
|
|
20
|
+
/** Required and non-empty: a sweep of the whole disk is never a default. */
|
|
21
|
+
readonly prefix: string;
|
|
22
|
+
/** Age past which an object goes. Required: the window is the app's. */
|
|
23
|
+
readonly olderThanMs: number;
|
|
24
|
+
/** `true` spares an object the app still accounts for. */
|
|
25
|
+
readonly keep?: ((object: StorageListEntry) => boolean | Promise<boolean>) | undefined;
|
|
26
|
+
readonly clock?: Clock | undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** `SweepResult`'s two lists, plus what was spared: undated, inside the window, or kept. */
|
|
30
|
+
export interface ExpireResult extends SweepResult {
|
|
31
|
+
readonly spared: readonly string[];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const screenPrefix = (caller: string, prefix: string): string => {
|
|
35
|
+
assert(
|
|
36
|
+
typeof prefix === 'string' && prefix !== '',
|
|
37
|
+
`${caller} was handed an empty prefix, which is the whole disk`,
|
|
38
|
+
`${caller}({ disk, prefix: 'exports/', olderThanMs }) # name the prefix the window applies to`,
|
|
39
|
+
);
|
|
40
|
+
return prefix;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const cutoffOf = (caller: string, olderThanMs: number, clock: Clock | undefined): number =>
|
|
44
|
+
// Screened first: `now - NaN` is NaN, and every age comparison against it reads "old enough".
|
|
45
|
+
(clock ?? systemClock).now().getTime() - finiteCount(caller, 'olderThanMs', olderThanMs, 0);
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Delete every object under `prefix` older than the window. An object with no reported age is
|
|
49
|
+
* spared, as `sweepOrphans` spares it; a refusal — a locked object included — is recorded in
|
|
50
|
+
* `failed` and the sweep goes on, never forced and never reported as deleted.
|
|
51
|
+
*/
|
|
52
|
+
export async function expireObjects(input: ExpireObjectsInput): Promise<ExpireResult> {
|
|
53
|
+
const prefix = screenPrefix('expireObjects', input.prefix);
|
|
54
|
+
const cutoff = cutoffOf('expireObjects', input.olderThanMs, input.clock);
|
|
55
|
+
const deleted: string[] = [];
|
|
56
|
+
const failed: SweepFailure[] = [];
|
|
57
|
+
const spared: string[] = [];
|
|
58
|
+
let cursor: string | null = null;
|
|
59
|
+
do {
|
|
60
|
+
const page = await input.disk.list({ prefix, cursor });
|
|
61
|
+
for (const object of page.rows) {
|
|
62
|
+
const modified = object.lastModified?.getTime();
|
|
63
|
+
if (modified === undefined || modified > cutoff || (await input.keep?.(object)) === true) {
|
|
64
|
+
spared.push(object.key);
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
await input.disk.delete(object.key);
|
|
69
|
+
deleted.push(object.key);
|
|
70
|
+
} catch (error) {
|
|
71
|
+
failed.push({ key: object.key, reason: renderThrowable(error) });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
cursor = page.nextCursor;
|
|
75
|
+
} while (cursor !== null);
|
|
76
|
+
return { deleted, failed, spared };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface AbortStaleUploadsInput {
|
|
80
|
+
readonly disk: StorageDriver;
|
|
81
|
+
/** Age past which an open session is abandoned. */
|
|
82
|
+
readonly olderThanMs: number;
|
|
83
|
+
/** Absent is every session on the disk: an open session under no prefix bills all the same. */
|
|
84
|
+
readonly prefix?: string | undefined;
|
|
85
|
+
/** Only sessions whose key this answers `true` for: a per-tenant kind no prefix spells. */
|
|
86
|
+
readonly match?: ((key: string) => boolean) | undefined;
|
|
87
|
+
readonly clock?: Clock | undefined;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface AbortResult {
|
|
91
|
+
readonly aborted: readonly MultipartSession[];
|
|
92
|
+
readonly failed: readonly SweepFailure[];
|
|
93
|
+
/** Undated or inside the window: possibly a phone still on the train. */
|
|
94
|
+
readonly spared: readonly MultipartSession[];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Abort every upload session opened before the window and never completed. */
|
|
98
|
+
export async function abortStaleUploads(input: AbortStaleUploadsInput): Promise<AbortResult> {
|
|
99
|
+
const cutoff = cutoffOf('abortStaleUploads', input.olderThanMs, input.clock);
|
|
100
|
+
const multipart = input.disk.multipart;
|
|
101
|
+
if (multipart === undefined) return { aborted: [], failed: [], spared: [] };
|
|
102
|
+
const aborted: MultipartSession[] = [];
|
|
103
|
+
const failed: SweepFailure[] = [];
|
|
104
|
+
const spared: MultipartSession[] = [];
|
|
105
|
+
for (const session of await multipart.listUploads(input.prefix)) {
|
|
106
|
+
if (input.match !== undefined && !input.match(session.key)) continue;
|
|
107
|
+
const initiated = session.initiated?.getTime();
|
|
108
|
+
if (initiated === undefined || initiated > cutoff) {
|
|
109
|
+
spared.push(session);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
await multipart.abort(session.key, session.uploadId);
|
|
114
|
+
aborted.push(session);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
failed.push({ key: session.key, reason: renderThrowable(error) });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return { aborted, failed, spared };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export interface ObjectPurgeTargetInput {
|
|
123
|
+
/** The step's name in the purge job, unique within it. */
|
|
124
|
+
readonly name: string;
|
|
125
|
+
readonly disk: StorageDriver;
|
|
126
|
+
readonly prefix: string;
|
|
127
|
+
/**
|
|
128
|
+
* Only keys this answers `true` for, under `prefix`: `org/<id>/imports/` is one kind of file
|
|
129
|
+
* under every tenant, and no prefix names it without also naming every avatar beside it.
|
|
130
|
+
*/
|
|
131
|
+
readonly match?: ((key: string) => boolean) | undefined;
|
|
132
|
+
readonly olderThanMs: number;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Structurally `@ultimat3/jobs`' `PurgeTarget`, so this package imports nothing upward and the
|
|
137
|
+
* app's one `purge({ targets })` carries it. One pass expires the objects and aborts the sessions
|
|
138
|
+
* under the prefix, on the JOB's clock. A refusal is logged and counted out, never retried into a
|
|
139
|
+
* dead letter: a locked object stays locked on every attempt.
|
|
140
|
+
*/
|
|
141
|
+
export function objectPurgeTarget(input: ObjectPurgeTargetInput): {
|
|
142
|
+
readonly name: string;
|
|
143
|
+
purgeExpired(nowMs: number): Promise<number>;
|
|
144
|
+
} {
|
|
145
|
+
const prefix = screenPrefix('objectPurgeTarget', input.prefix);
|
|
146
|
+
finiteCount('objectPurgeTarget', 'olderThanMs', input.olderThanMs, 0);
|
|
147
|
+
return {
|
|
148
|
+
name: input.name,
|
|
149
|
+
async purgeExpired(nowMs: number): Promise<number> {
|
|
150
|
+
const clock: Clock = { now: () => new Date(nowMs), monotonic: () => nowMs };
|
|
151
|
+
const window = { disk: input.disk, prefix, olderThanMs: input.olderThanMs, clock };
|
|
152
|
+
const match = input.match;
|
|
153
|
+
// Outside `match` is spared, never deleted: `keep` is `expireObjects`' own way of saying so.
|
|
154
|
+
const objects = await expireObjects(
|
|
155
|
+
match === undefined ? window : { ...window, keep: (object) => !match(object.key) },
|
|
156
|
+
);
|
|
157
|
+
const uploads = await abortStaleUploads({ ...window, match });
|
|
158
|
+
const failed = [...objects.failed, ...uploads.failed];
|
|
159
|
+
if (failed.length > 0) {
|
|
160
|
+
logger.warn('storage.expire.refused', {
|
|
161
|
+
target: input.name,
|
|
162
|
+
refused: failed.length,
|
|
163
|
+
first: failed[0]?.reason,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
return objects.deleted.length + uploads.aborted.length;
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
// Single responsibility: the server half of an upload in parts — open a session under
|
|
2
|
+
// `grantUpload`'s policy gate, sign a part, list what landed, complete, abort. Each is a function the
|
|
3
|
+
// app wraps in its own `action()`, and each is held to the ACTOR's tenant, so an upload id read off a
|
|
4
|
+
// request reaches nothing outside the caller's org.
|
|
5
|
+
|
|
6
|
+
import { assert } from '@ultimat3/core';
|
|
7
|
+
import type { StorageDriver, StorageObject } from './driver';
|
|
8
|
+
import { orgMismatch, tooLarge } from './errors';
|
|
9
|
+
import { multipartUnsupported, partsInvalid } from './errors-multipart';
|
|
10
|
+
import { decideGrant, type GrantUploadInput } from './grant';
|
|
11
|
+
import {
|
|
12
|
+
type CompletedPart,
|
|
13
|
+
MAX_PART_BYTES,
|
|
14
|
+
MAX_PARTS,
|
|
15
|
+
MIN_PART_BYTES,
|
|
16
|
+
type MultipartCapability,
|
|
17
|
+
matchParts,
|
|
18
|
+
partBytesFor,
|
|
19
|
+
type UploadedPart,
|
|
20
|
+
} from './multipart';
|
|
21
|
+
import { isWithinOrg } from './path';
|
|
22
|
+
import { DEFAULT_SIGNED_URL_TTL_MS } from './signed-url';
|
|
23
|
+
import type { UploadPolicy } from './upload';
|
|
24
|
+
|
|
25
|
+
export interface GrantMultipartInput extends GrantUploadInput {
|
|
26
|
+
/** A part size the app prefers; raised to what 10,000 parts need, never below S3's 5 MiB. */
|
|
27
|
+
readonly partBytes?: number | undefined;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface MultipartGrant {
|
|
31
|
+
readonly key: string;
|
|
32
|
+
readonly uploadId: string;
|
|
33
|
+
/** Every part but the last is exactly this long. */
|
|
34
|
+
readonly partBytes: number;
|
|
35
|
+
/** How many parts the declared size takes at `partBytes`. */
|
|
36
|
+
readonly maxParts: number;
|
|
37
|
+
/** Normalised: the type the object is created under. */
|
|
38
|
+
readonly contentType: string;
|
|
39
|
+
/** The declared size, already held to the policy. The client's claim: `policy` is the bound. */
|
|
40
|
+
readonly size: number;
|
|
41
|
+
/** Epoch ms: the part URLs' window from now. Each `signPart` opens a fresh one. */
|
|
42
|
+
readonly expiresAt: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** What names one open session. `orgId` is the actor's, resolved server-side. */
|
|
46
|
+
export interface UploadSessionInput {
|
|
47
|
+
readonly disk: StorageDriver;
|
|
48
|
+
readonly orgId: string;
|
|
49
|
+
readonly key: string;
|
|
50
|
+
readonly uploadId: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface SignPartInput extends UploadSessionInput {
|
|
54
|
+
readonly partNumber: number;
|
|
55
|
+
/** This part's exact size: `partBytes`, or less for the last part. Signed. */
|
|
56
|
+
readonly size: number;
|
|
57
|
+
/** The policy the grant was made under: a part no object under its `maxBytes` holds is refused. */
|
|
58
|
+
readonly policy: UploadPolicy;
|
|
59
|
+
readonly expiresInMs?: number | undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface CompleteUploadInput extends UploadSessionInput {
|
|
63
|
+
/** Ascending. Etags are optional: the disk's own, read by `listParts` here, are what is sent. */
|
|
64
|
+
readonly parts: readonly CompletedPart[];
|
|
65
|
+
/** The policy the grant was made under: a completed object past its `maxBytes` is refused. */
|
|
66
|
+
readonly policy: UploadPolicy;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const capabilityOf = (disk: StorageDriver, key: string): MultipartCapability => {
|
|
70
|
+
if (disk.multipart === undefined) throw multipartUnsupported(disk.name, key);
|
|
71
|
+
return disk.multipart;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/** The key a session was opened for, held to the actor's org exactly as `accept.ts` holds a URL. */
|
|
75
|
+
const sessionOf = (input: UploadSessionInput): MultipartCapability => {
|
|
76
|
+
// Every session a grant opens is under `org/<id>/`, so a key outside the actor's org is refused
|
|
77
|
+
// whether or not another tenant owns it: no shared prefix takes parts.
|
|
78
|
+
if (!isWithinOrg(input.key, input.orgId)) {
|
|
79
|
+
throw orgMismatch(input.key, input.orgId);
|
|
80
|
+
}
|
|
81
|
+
return capabilityOf(input.disk, input.key);
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The type says required; an untyped caller upgrading from the call without one does not read
|
|
86
|
+
* types, and with no policy nothing bounds what the session stores — `promoteAttachment`'s guard.
|
|
87
|
+
*/
|
|
88
|
+
const assertPolicy = (caller: string, policy: UploadPolicy | undefined, fix: string): number => {
|
|
89
|
+
assert(
|
|
90
|
+
typeof policy?.maxBytes === 'number',
|
|
91
|
+
`${caller} was called with no policy, so nothing bounds the size of the upload in parts`,
|
|
92
|
+
fix,
|
|
93
|
+
);
|
|
94
|
+
return policy.maxBytes;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The smallest object part `partNumber` at `size` bytes can belong to. Every part before it is at
|
|
99
|
+
* least S3's floor and at least as long as it (each but the last is exactly `partBytes`), so an
|
|
100
|
+
* honest grant's part never implies more than its declared size, and so never more than the policy.
|
|
101
|
+
*/
|
|
102
|
+
const impliedBytes = (partNumber: number, size: number): number =>
|
|
103
|
+
Math.max((partNumber - 1) * MIN_PART_BYTES + size, partNumber * size);
|
|
104
|
+
|
|
105
|
+
/** The part size: the app's preference, raised to what the size and the protocol require. */
|
|
106
|
+
const partSizeFor = (size: number, preferred: number | undefined): number => {
|
|
107
|
+
const floor = partBytesFor(size);
|
|
108
|
+
if (preferred === undefined) return floor;
|
|
109
|
+
assert(
|
|
110
|
+
Number.isSafeInteger(preferred) && preferred >= MIN_PART_BYTES && preferred <= MAX_PART_BYTES,
|
|
111
|
+
`grantMultipart was handed partBytes ${String(preferred)}, outside S3's ${MIN_PART_BYTES}..${MAX_PART_BYTES} bytes`,
|
|
112
|
+
`grantMultipart({ disk, orgId, request, partBytes: ${MIN_PART_BYTES} }) # or omit it`,
|
|
113
|
+
);
|
|
114
|
+
return Math.max(floor, preferred);
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/** Refuses as `grantUpload` refuses, then opens the session the parts go into. */
|
|
118
|
+
export async function grantMultipart(input: GrantMultipartInput): Promise<MultipartGrant> {
|
|
119
|
+
const granted = decideGrant('grantMultipart', input);
|
|
120
|
+
const multipart = capabilityOf(input.disk, granted.key);
|
|
121
|
+
const partBytes = partSizeFor(granted.size, input.partBytes);
|
|
122
|
+
const uploadId = await multipart.create(granted.key, { contentType: granted.contentType });
|
|
123
|
+
return {
|
|
124
|
+
key: granted.key,
|
|
125
|
+
uploadId,
|
|
126
|
+
partBytes,
|
|
127
|
+
maxParts: Math.ceil(granted.size / partBytes),
|
|
128
|
+
contentType: granted.contentType,
|
|
129
|
+
size: granted.size,
|
|
130
|
+
expiresAt: granted.now + granted.expiresInMs,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* One part's URL, its length inside the signature. A part past what an object under the policy
|
|
136
|
+
* holds is `X_STORAGE_TOO_LARGE`, so a session's parts cannot outgrow the policy before the
|
|
137
|
+
* `completeUpload` that would refuse them: unassembled parts bill too.
|
|
138
|
+
*/
|
|
139
|
+
export async function signPart(input: SignPartInput): Promise<string> {
|
|
140
|
+
const maxBytes = assertPolicy(
|
|
141
|
+
'signPart',
|
|
142
|
+
input.policy,
|
|
143
|
+
'signPart({ disk, orgId, key, uploadId, partNumber, size, policy: uploadPolicy({ maxBytes }) }) — the policy the grant was made under',
|
|
144
|
+
);
|
|
145
|
+
const multipart = sessionOf(input);
|
|
146
|
+
const { partNumber, size } = input;
|
|
147
|
+
assert(
|
|
148
|
+
Number.isSafeInteger(partNumber) && partNumber >= 1 && partNumber <= MAX_PARTS,
|
|
149
|
+
`signPart was handed partNumber ${String(partNumber)}, outside 1..${MAX_PARTS}`,
|
|
150
|
+
'signPart({ disk, orgId, key, uploadId, partNumber: 1, size })',
|
|
151
|
+
);
|
|
152
|
+
assert(
|
|
153
|
+
Number.isSafeInteger(size) && size >= 1 && size <= MAX_PART_BYTES,
|
|
154
|
+
`signPart was handed size ${String(size)}, not a part of 1..${MAX_PART_BYTES} bytes`,
|
|
155
|
+
'signPart({ disk, orgId, key, uploadId, partNumber, size: grant.partBytes })',
|
|
156
|
+
);
|
|
157
|
+
const implied = impliedBytes(partNumber, size);
|
|
158
|
+
if (implied > maxBytes) throw tooLarge(input.key, implied, maxBytes);
|
|
159
|
+
return multipart.signPart(input.key, input.uploadId, partNumber, {
|
|
160
|
+
contentLength: size,
|
|
161
|
+
expiresInMs: input.expiresInMs === undefined ? DEFAULT_SIGNED_URL_TTL_MS : input.expiresInMs,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** What landed, ascending — the resume: a client sends only the parts not here. */
|
|
166
|
+
export function listParts(input: UploadSessionInput): Promise<readonly UploadedPart[]> {
|
|
167
|
+
return sessionOf(input).listParts(input.key, input.uploadId);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The parts named, checked against what the disk holds, then assembled. The etags sent are the
|
|
172
|
+
* disk's own, so the browser never needs a bucket's CORS to expose `ETag`. A total past the policy
|
|
173
|
+
* aborts the session: those bytes were never going to be promoted.
|
|
174
|
+
*/
|
|
175
|
+
export async function completeUpload(input: CompleteUploadInput): Promise<StorageObject> {
|
|
176
|
+
const maxBytes = assertPolicy(
|
|
177
|
+
'completeUpload',
|
|
178
|
+
input.policy,
|
|
179
|
+
'completeUpload({ disk, orgId, key, uploadId, parts, policy: uploadPolicy({ maxBytes }) }) — the policy the grant was made under',
|
|
180
|
+
);
|
|
181
|
+
const multipart = sessionOf(input);
|
|
182
|
+
const stored = await multipart.listParts(input.key, input.uploadId);
|
|
183
|
+
const matched = matchParts(input.parts, stored);
|
|
184
|
+
if ('problem' in matched) throw partsInvalid(input.key, input.uploadId, matched.problem);
|
|
185
|
+
const total = matched.parts.reduce((sum, part) => sum + part.size, 0);
|
|
186
|
+
if (total > maxBytes) {
|
|
187
|
+
await multipart.abort(input.key, input.uploadId);
|
|
188
|
+
throw tooLarge(input.key, total, maxBytes);
|
|
189
|
+
}
|
|
190
|
+
return multipart.complete(input.key, input.uploadId, matched.parts);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Drops the session and every part. An id already gone is the state asked for. */
|
|
194
|
+
export function abortUpload(input: UploadSessionInput): Promise<void> {
|
|
195
|
+
return sessionOf(input).abort(input.key, input.uploadId);
|
|
196
|
+
}
|