@remit/storage-service 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backends/filesystem.d.ts +12 -0
- package/dist/backends/filesystem.js +316 -0
- package/dist/backends/s3.d.ts +3 -0
- package/dist/backends/s3.js +413 -0
- package/dist/filesystem.js +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/outbox-sweep.d.ts +33 -0
- package/dist/outbox-sweep.js +37 -0
- package/dist/outbox-upload-url.d.ts +44 -0
- package/{src/outbox-upload-url.ts → dist/outbox-upload-url.js} +20 -50
- package/dist/s3.d.ts +3 -0
- package/dist/s3.js +28 -0
- package/dist/signed-path.d.ts +14 -0
- package/dist/signed-path.js +46 -0
- package/dist/storage.d.ts +255 -0
- package/dist/storage.js +228 -0
- package/dist/uri.d.ts +9 -0
- package/dist/uri.js +28 -0
- package/package.json +51 -46
- package/src/backends/filesystem.ts +0 -517
- package/src/backends/s3.test.ts +0 -447
- package/src/backends/s3.ts +0 -624
- package/src/index.ts +0 -53
- package/src/outbox-attachment-retrieve.test.ts +0 -56
- package/src/outbox-sweep.test.ts +0 -196
- package/src/outbox-sweep.ts +0 -103
- package/src/outbox-upload-url.test.ts +0 -290
- package/src/s3.ts +0 -36
- package/src/signed-path.ts +0 -86
- package/src/storage.test.ts +0 -646
- package/src/storage.ts +0 -699
- package/src/uri.ts +0 -47
- package/tsconfig.json +0 -8
- /package/{src/filesystem.ts → dist/filesystem.d.ts} +0 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare const signStoragePath: (masterSecret: string, label: string, relativePath: string, claims: readonly number[]) => string;
|
|
2
|
+
export type SignedPathFailure = "missing" | "malformed" | "expired" | "bad-signature";
|
|
3
|
+
export type SignedPathResult = {
|
|
4
|
+
valid: true;
|
|
5
|
+
} | {
|
|
6
|
+
valid: false;
|
|
7
|
+
reason: SignedPathFailure;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Verify a presented signature against the path and claims it must cover. The
|
|
11
|
+
* first claim is always the expiry, in unix seconds. Pure, so the decision can
|
|
12
|
+
* be unit-tested without a live server.
|
|
13
|
+
*/
|
|
14
|
+
export declare const verifyStoragePath: (masterSecret: string, label: string, relativePath: string, claims: readonly number[], sig: string | undefined, nowSeconds: number) => SignedPathResult;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
2
|
+
/**
|
|
3
|
+
* The HMAC scheme behind every signed storage URL this stack mints.
|
|
4
|
+
*
|
|
5
|
+
* It started as the read side — `/content/*` is served against a signature
|
|
6
|
+
* because a bearer token cannot ride on an `<img src>` inside email HTML — and
|
|
7
|
+
* the write side needs exactly the same capability: a URL that grants access to
|
|
8
|
+
* one path, for a while, and is unforgeable without the key. Rather than a
|
|
9
|
+
* second scheme, both use this one with a different derivation label, so a
|
|
10
|
+
* signature minted to read content can never be replayed as permission to
|
|
11
|
+
* write, and vice versa.
|
|
12
|
+
*
|
|
13
|
+
* The signed message is the storage path followed by every claim in a fixed
|
|
14
|
+
* order. Because the path is part of the message, a signature minted for one
|
|
15
|
+
* account's object cannot be presented for another's.
|
|
16
|
+
*/
|
|
17
|
+
const deriveSigningKey = (masterSecret, label) => createHmac("sha256", masterSecret).update(label).digest();
|
|
18
|
+
const canonicalMessage = (relativePath, claims) => [relativePath, ...claims].join("\n");
|
|
19
|
+
const computeSignature = (key, relativePath, claims) => createHmac("sha256", key)
|
|
20
|
+
.update(canonicalMessage(relativePath, claims))
|
|
21
|
+
.digest("base64url");
|
|
22
|
+
export const signStoragePath = (masterSecret, label, relativePath, claims) => computeSignature(deriveSigningKey(masterSecret, label), relativePath, claims);
|
|
23
|
+
/**
|
|
24
|
+
* Verify a presented signature against the path and claims it must cover. The
|
|
25
|
+
* first claim is always the expiry, in unix seconds. Pure, so the decision can
|
|
26
|
+
* be unit-tested without a live server.
|
|
27
|
+
*/
|
|
28
|
+
export const verifyStoragePath = (masterSecret, label, relativePath, claims, sig, nowSeconds) => {
|
|
29
|
+
if (!sig || claims.length === 0)
|
|
30
|
+
return { valid: false, reason: "missing" };
|
|
31
|
+
const [exp] = claims;
|
|
32
|
+
if (!claims.every((claim) => Number.isInteger(claim) && claim > 0)) {
|
|
33
|
+
return { valid: false, reason: "malformed" };
|
|
34
|
+
}
|
|
35
|
+
if (exp < nowSeconds)
|
|
36
|
+
return { valid: false, reason: "expired" };
|
|
37
|
+
const expected = Buffer.from(signStoragePath(masterSecret, label, relativePath, claims));
|
|
38
|
+
const presented = Buffer.from(sig);
|
|
39
|
+
if (expected.length !== presented.length) {
|
|
40
|
+
return { valid: false, reason: "bad-signature" };
|
|
41
|
+
}
|
|
42
|
+
if (!timingSafeEqual(expected, presented)) {
|
|
43
|
+
return { valid: false, reason: "bad-signature" };
|
|
44
|
+
}
|
|
45
|
+
return { valid: true };
|
|
46
|
+
};
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { Readable } from "node:stream";
|
|
2
|
+
import { ContentEncoding, StorageType } from "@remit/domain-enums";
|
|
3
|
+
export type StorageTypeValue = (typeof StorageType)[keyof typeof StorageType];
|
|
4
|
+
export type ContentEncodingValue = (typeof ContentEncoding)[keyof typeof ContentEncoding];
|
|
5
|
+
export interface StorageReference {
|
|
6
|
+
uri: string;
|
|
7
|
+
storageType: StorageTypeValue;
|
|
8
|
+
storageLocation: string;
|
|
9
|
+
storageKey: string;
|
|
10
|
+
sizeBytes: number;
|
|
11
|
+
/**
|
|
12
|
+
* SHA-256 (hex) of the logical, pre-compression content.
|
|
13
|
+
* Identifies the content regardless of how it is stored on the backend,
|
|
14
|
+
* and is used by `buildDeduplicatedKey` for content-addressable storage.
|
|
15
|
+
* Note: this is NOT the checksum S3 validates against the received bytes —
|
|
16
|
+
* S3's own transport-level checksum is handled by the SDK on PUT.
|
|
17
|
+
*/
|
|
18
|
+
checksumSha256: string;
|
|
19
|
+
contentEncoding: ContentEncodingValue;
|
|
20
|
+
}
|
|
21
|
+
/** Parameters for storing a message body (raw RFC822 content) */
|
|
22
|
+
export interface StoreMessageBodyParams {
|
|
23
|
+
accountConfigId: string;
|
|
24
|
+
accountId: string;
|
|
25
|
+
messageId: string;
|
|
26
|
+
content: Buffer;
|
|
27
|
+
}
|
|
28
|
+
/** Parameters for streaming a message body (raw RFC822 content) to storage */
|
|
29
|
+
export interface StoreMessageBodyStreamParams {
|
|
30
|
+
accountConfigId: string;
|
|
31
|
+
accountId: string;
|
|
32
|
+
messageId: string;
|
|
33
|
+
/** Readable stream of the raw message bytes; streamed to storage, never buffered whole. */
|
|
34
|
+
content: Readable;
|
|
35
|
+
}
|
|
36
|
+
/** Parameters for storing a body part (attachment, inline content) */
|
|
37
|
+
export interface StoreBodyPartParams {
|
|
38
|
+
accountConfigId: string;
|
|
39
|
+
accountId: string;
|
|
40
|
+
messageId: string;
|
|
41
|
+
partPath: string;
|
|
42
|
+
content: Buffer;
|
|
43
|
+
contentType?: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Parameters for storing a file attached to a draft outbox message.
|
|
47
|
+
*
|
|
48
|
+
* Deliberately not the deduplicated path: a content-addressed object belongs to
|
|
49
|
+
* no single draft, so nothing may delete it when the draft goes. A per-draft key
|
|
50
|
+
* dies with the draft.
|
|
51
|
+
*/
|
|
52
|
+
export interface StoreOutboxAttachmentParams {
|
|
53
|
+
accountConfigId: string;
|
|
54
|
+
accountId: string;
|
|
55
|
+
outboxMessageId: string;
|
|
56
|
+
outboxAttachmentId: string;
|
|
57
|
+
content: Buffer;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* One uploaded object under a draft's attachment prefix. Reservations are not
|
|
61
|
+
* here — they live in the ledger, which is the only thing that can say whether
|
|
62
|
+
* an object present in storage is an attachment or garbage.
|
|
63
|
+
*/
|
|
64
|
+
export interface OutboxAttachmentListItem {
|
|
65
|
+
outboxAttachmentId: string;
|
|
66
|
+
key: string;
|
|
67
|
+
sizeBytes: number;
|
|
68
|
+
}
|
|
69
|
+
/** Somewhere for a client to PUT one file, for a while */
|
|
70
|
+
export interface OutboxAttachmentUploadTarget {
|
|
71
|
+
/**
|
|
72
|
+
* Absolute URL the client PUTs raw bytes to. Opaque by design: block storage
|
|
73
|
+
* on one deployment, this deployment's own upload route on another, and a
|
|
74
|
+
* client cannot tell which it was handed.
|
|
75
|
+
*/
|
|
76
|
+
uploadUrl: string;
|
|
77
|
+
}
|
|
78
|
+
export interface CreateOutboxAttachmentUploadUrlParams {
|
|
79
|
+
accountConfigId: string;
|
|
80
|
+
accountId: string;
|
|
81
|
+
outboxMessageId: string;
|
|
82
|
+
outboxAttachmentId: string;
|
|
83
|
+
/** Exact byte count the upload may carry. */
|
|
84
|
+
sizeBytes: number;
|
|
85
|
+
expiresAt: number;
|
|
86
|
+
}
|
|
87
|
+
/** Parameters for storing deduplicated content (attachments shared across messages) */
|
|
88
|
+
export interface StoreDeduplicatedParams {
|
|
89
|
+
accountConfigId: string;
|
|
90
|
+
accountId: string;
|
|
91
|
+
content: Buffer;
|
|
92
|
+
contentType?: string;
|
|
93
|
+
}
|
|
94
|
+
/** Metadata for a single attachment in a parsed message body. No binary content. */
|
|
95
|
+
export interface ParsedAttachmentMeta {
|
|
96
|
+
filename: string | null;
|
|
97
|
+
contentType: string;
|
|
98
|
+
contentDisposition: string | null;
|
|
99
|
+
contentId: string | null;
|
|
100
|
+
size: number;
|
|
101
|
+
}
|
|
102
|
+
/** Pre-parsed message body cached as parsed.json.gz to skip mailparser on warm reads */
|
|
103
|
+
export interface ParsedBody {
|
|
104
|
+
text: string | null;
|
|
105
|
+
html: string | null;
|
|
106
|
+
attachments: ParsedAttachmentMeta[];
|
|
107
|
+
}
|
|
108
|
+
/** Parameters for storing a pre-parsed message body */
|
|
109
|
+
export interface StoreParsedBodyParams {
|
|
110
|
+
accountConfigId: string;
|
|
111
|
+
accountId: string;
|
|
112
|
+
messageId: string;
|
|
113
|
+
parsed: ParsedBody;
|
|
114
|
+
}
|
|
115
|
+
export interface StorageService {
|
|
116
|
+
/** Store a message body (raw RFC822 content) */
|
|
117
|
+
storeMessageBody(params: StoreMessageBodyParams): Promise<StorageReference>;
|
|
118
|
+
/**
|
|
119
|
+
* Store a message body from a readable stream, never buffering the whole
|
|
120
|
+
* body. Use this on the sync hot path so a ranged FETCH streams straight to
|
|
121
|
+
* storage.
|
|
122
|
+
*/
|
|
123
|
+
storeMessageBodyStream(params: StoreMessageBodyStreamParams): Promise<StorageReference>;
|
|
124
|
+
/** Store a body part (attachment, inline content) */
|
|
125
|
+
storeBodyPart(params: StoreBodyPartParams): Promise<StorageReference>;
|
|
126
|
+
/**
|
|
127
|
+
* Check whether a body-part object already exists, keyed exactly as
|
|
128
|
+
* `storeBodyPart` writes it. Used by the lazy per-part generation path to
|
|
129
|
+
* skip leaves already materialized on a prior read, so regeneration is
|
|
130
|
+
* idempotent and re-reads stay cheap.
|
|
131
|
+
*/
|
|
132
|
+
bodyPartExists(accountConfigId: string, accountId: string, messageId: string, partPath: string): Promise<boolean>;
|
|
133
|
+
/**
|
|
134
|
+
* Retrieve a body-part's content (attachment or inline leaf) by
|
|
135
|
+
* account/message id and IMAP section path; returns null on NoSuchKey.
|
|
136
|
+
*/
|
|
137
|
+
retrieveBodyPart(accountConfigId: string, accountId: string, messageId: string, partPath: string): Promise<Buffer | null>;
|
|
138
|
+
/**
|
|
139
|
+
* Store a file attached to a draft outbox message, uncompressed: the size on
|
|
140
|
+
* the backend is then the decoded size the per-message cap is expressed in,
|
|
141
|
+
* and `listOutboxAttachments` can total a draft without opening anything.
|
|
142
|
+
*/
|
|
143
|
+
storeOutboxAttachment(params: StoreOutboxAttachmentParams): Promise<StorageReference>;
|
|
144
|
+
/**
|
|
145
|
+
* Every object stored against a draft, paged to the end. Only the sweep reads
|
|
146
|
+
* this, and a sweep that stops early leaves exactly the orphan it was called
|
|
147
|
+
* to collect — so there is no limit to get wrong.
|
|
148
|
+
*/
|
|
149
|
+
listOutboxAttachments(accountConfigId: string, accountId: string, outboxMessageId: string): Promise<OutboxAttachmentListItem[]>;
|
|
150
|
+
/**
|
|
151
|
+
* Remove every file and reservation stored against a draft. Unlike the read
|
|
152
|
+
* above this takes no limit and pages to the end: a sweep that stops early
|
|
153
|
+
* leaves exactly the orphan it was called to prevent.
|
|
154
|
+
*/
|
|
155
|
+
deleteOutboxAttachments(accountConfigId: string, accountId: string, outboxMessageId: string): Promise<void>;
|
|
156
|
+
/**
|
|
157
|
+
* Every draft under an account that has attachment objects in storage,
|
|
158
|
+
* including drafts whose row is long gone. The sweep needs the prefixes that
|
|
159
|
+
* exist, not the rows that exist — a discarded draft is invisible to the
|
|
160
|
+
* database and is exactly the case worth collecting.
|
|
161
|
+
*/
|
|
162
|
+
listOutboxDraftsWithAttachments(accountConfigId: string, accountId: string): Promise<string[]>;
|
|
163
|
+
/** Remove one attachment object. */
|
|
164
|
+
deleteOutboxAttachment(accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string): Promise<void>;
|
|
165
|
+
/**
|
|
166
|
+
* The size storage actually holds for an attachment, or null when nothing
|
|
167
|
+
* was uploaded. This is the only number a completion may believe — the
|
|
168
|
+
* client's word for it is not evidence, and on a deployment where the bytes
|
|
169
|
+
* went straight to block storage it is the only reading available.
|
|
170
|
+
*/
|
|
171
|
+
statOutboxAttachment(accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string): Promise<{
|
|
172
|
+
sizeBytes: number;
|
|
173
|
+
} | null>;
|
|
174
|
+
/**
|
|
175
|
+
* The bytes of one attachment, for building the MIME part it goes out as.
|
|
176
|
+
* Null when nothing is stored under the key.
|
|
177
|
+
*/
|
|
178
|
+
retrieveOutboxAttachment(accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string): Promise<Buffer | null>;
|
|
179
|
+
/**
|
|
180
|
+
* Mint somewhere to PUT one file. The backend decides what that is — block
|
|
181
|
+
* storage direct, or this deployment's upload route — and the caller never
|
|
182
|
+
* branches on which it got. Whatever it returns must bind the size: an
|
|
183
|
+
* upload that carries more than was reserved has to fail at the thing
|
|
184
|
+
* receiving it, not afterwards.
|
|
185
|
+
*/
|
|
186
|
+
createOutboxAttachmentUploadUrl(params: CreateOutboxAttachmentUploadUrlParams): Promise<OutboxAttachmentUploadTarget>;
|
|
187
|
+
/** Store deduplicated content (content-addressable, for attachments) */
|
|
188
|
+
storeDeduplicated(params: StoreDeduplicatedParams): Promise<StorageReference>;
|
|
189
|
+
/** Store a pre-parsed message body as gzipped JSON (parsed-body cache) */
|
|
190
|
+
storeParsedBody(params: StoreParsedBodyParams): Promise<StorageReference>;
|
|
191
|
+
/** Retrieve a pre-parsed message body by account/message id; returns null on NoSuchKey */
|
|
192
|
+
retrieveParsedBody(accountConfigId: string, accountId: string, messageId: string): Promise<ParsedBody | null>;
|
|
193
|
+
/** Retrieve a raw message body (RFC822 .eml) by account/message id; returns null on NoSuchKey */
|
|
194
|
+
retrieveMessageBody(accountConfigId: string, accountId: string, messageId: string): Promise<Buffer | null>;
|
|
195
|
+
/**
|
|
196
|
+
* Retrieve a raw message body as a readable stream of decompressed .eml
|
|
197
|
+
* bytes; returns null on NoSuchKey. Use on the export path so a whole
|
|
198
|
+
* mailbox is never buffered in memory.
|
|
199
|
+
*/
|
|
200
|
+
retrieveMessageBodyStream(accountConfigId: string, accountId: string, messageId: string): Promise<Readable | null>;
|
|
201
|
+
/** Retrieve content by URI */
|
|
202
|
+
retrieve(uri: string): Promise<Buffer>;
|
|
203
|
+
/** Check if content exists */
|
|
204
|
+
exists(uri: string): Promise<boolean>;
|
|
205
|
+
/** Delete content by URI */
|
|
206
|
+
delete(uri: string): Promise<void>;
|
|
207
|
+
/**
|
|
208
|
+
* Stream a finished export archive (already-compressed ZIP) to the export
|
|
209
|
+
* key for the given config + request, uncompressed at rest. The body is
|
|
210
|
+
* streamed via a multipart upload and never buffered whole. Returns the raw
|
|
211
|
+
* storage key, which `getPresignedDownloadUrl` can then sign.
|
|
212
|
+
*/
|
|
213
|
+
storeExportArchiveStream(accountConfigId: string, exportRequestId: string, body: Readable): Promise<string>;
|
|
214
|
+
/**
|
|
215
|
+
* Generate a presigned URL that grants temporary GET access to a raw S3 key.
|
|
216
|
+
* Use for export archives — the key must already exist in the bucket.
|
|
217
|
+
* Not supported on filesystem backends; throws if called there.
|
|
218
|
+
*/
|
|
219
|
+
getPresignedDownloadUrl(key: string, expiresInSeconds: number): Promise<string>;
|
|
220
|
+
}
|
|
221
|
+
export declare const buildMessageBodyKey: (accountConfigId: string, accountId: string, messageId: string) => string;
|
|
222
|
+
export declare const buildParsedBodyKey: (accountConfigId: string, accountId: string, messageId: string) => string;
|
|
223
|
+
export declare const buildBodyPartKey: (accountConfigId: string, accountId: string, messageId: string, partPath: string) => string;
|
|
224
|
+
export declare const buildOutboxAttachmentPrefix: (accountConfigId: string, accountId: string, outboxMessageId: string) => string;
|
|
225
|
+
export declare const buildOutboxAttachmentKey: (accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string) => string;
|
|
226
|
+
export interface ParsedOutboxAttachmentKey {
|
|
227
|
+
accountConfigId: string;
|
|
228
|
+
accountId: string;
|
|
229
|
+
outboxMessageId: string;
|
|
230
|
+
outboxAttachmentId: string;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Read the ids back out of an attachment key — the inverse of
|
|
234
|
+
* `buildOutboxAttachmentKey`, for the upload route, which is handed a path and
|
|
235
|
+
* has to work out what it addresses. Returns null on anything else, including a
|
|
236
|
+
* reservation key: bytes are never written to one.
|
|
237
|
+
*/
|
|
238
|
+
export declare const parseOutboxAttachmentKey: (storageKey: string) => ParsedOutboxAttachmentKey | null;
|
|
239
|
+
export declare const buildDeduplicatedKey: (accountConfigId: string, accountId: string, checksumSha256: string) => string;
|
|
240
|
+
export interface ParsedContentStorageKey {
|
|
241
|
+
accountConfigId: string;
|
|
242
|
+
accountId: string;
|
|
243
|
+
messageId: string;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Parse the account/message ids out of a `/content/*` storage key such as
|
|
247
|
+
* `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/1.2`
|
|
248
|
+
* (the inverse of `buildMessageBodyKey`/`buildParsedBodyKey`/`buildBodyPartKey`
|
|
249
|
+
* above). Returns null when the shape doesn't match.
|
|
250
|
+
*/
|
|
251
|
+
export declare const parseContentStorageKey: (storageKey: string) => ParsedContentStorageKey | null;
|
|
252
|
+
export declare const computeChecksum: (content: Buffer) => string;
|
|
253
|
+
export declare const buildExportArchiveKey: (accountConfigId: string, exportRequestId: string) => string;
|
|
254
|
+
export declare const isStorageNotFoundError: (error: unknown) => boolean;
|
|
255
|
+
export declare const createMockStorageService: () => StorageService;
|
package/dist/storage.js
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { Readable } from "node:stream";
|
|
3
|
+
import { ContentEncoding, StorageType } from "@remit/domain-enums";
|
|
4
|
+
// Path builders - centralized path formatting per RFC 011 + #224.
|
|
5
|
+
// The outer segment is `accountConfigId` (Cognito-derived tenant scope) so
|
|
6
|
+
// the edge can verify cross-tenant isolation at the URL prefix without
|
|
7
|
+
// loading any database record. The inner `accountId` distinguishes between
|
|
8
|
+
// multiple IMAP-account records owned by the same tenant.
|
|
9
|
+
export const buildMessageBodyKey = (accountConfigId, accountId, messageId) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/body.eml`;
|
|
10
|
+
export const buildParsedBodyKey = (accountConfigId, accountId, messageId) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/parsed.json.gz`;
|
|
11
|
+
export const buildBodyPartKey = (accountConfigId, accountId, messageId, partPath) => `accounts/${accountConfigId}/${accountId}/messages/${messageId}/parts/${partPath}`;
|
|
12
|
+
export const buildOutboxAttachmentPrefix = (accountConfigId, accountId, outboxMessageId) => `accounts/${accountConfigId}/${accountId}/outbox/${outboxMessageId}/attachments/`;
|
|
13
|
+
export const buildOutboxAttachmentKey = (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => `${buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId)}${outboxAttachmentId}`;
|
|
14
|
+
// Every segment is a generated id, so the shape is closed rather than
|
|
15
|
+
// "anything without a slash": `..` matches `[^/]+` and the write side joins
|
|
16
|
+
// this key onto a filesystem root. The read side has `resolveContentPath` for
|
|
17
|
+
// the same reason (#310 review P1) — a signature should not be the only thing
|
|
18
|
+
// standing between a URL and the storage root.
|
|
19
|
+
const OUTBOX_ATTACHMENT_KEY = /^accounts\/([A-Za-z0-9_-]+)\/([A-Za-z0-9_-]+)\/outbox\/([A-Za-z0-9_-]+)\/attachments\/([A-Za-z0-9_-]+)$/;
|
|
20
|
+
/**
|
|
21
|
+
* Read the ids back out of an attachment key — the inverse of
|
|
22
|
+
* `buildOutboxAttachmentKey`, for the upload route, which is handed a path and
|
|
23
|
+
* has to work out what it addresses. Returns null on anything else, including a
|
|
24
|
+
* reservation key: bytes are never written to one.
|
|
25
|
+
*/
|
|
26
|
+
export const parseOutboxAttachmentKey = (storageKey) => {
|
|
27
|
+
const match = storageKey.match(OUTBOX_ATTACHMENT_KEY);
|
|
28
|
+
if (!match)
|
|
29
|
+
return null;
|
|
30
|
+
return {
|
|
31
|
+
accountConfigId: match[1],
|
|
32
|
+
accountId: match[2],
|
|
33
|
+
outboxMessageId: match[3],
|
|
34
|
+
outboxAttachmentId: match[4],
|
|
35
|
+
};
|
|
36
|
+
};
|
|
37
|
+
export const buildDeduplicatedKey = (accountConfigId, accountId, checksumSha256) => `accounts/${accountConfigId}/${accountId}/dedup/${checksumSha256.slice(0, 2)}/${checksumSha256}`;
|
|
38
|
+
/**
|
|
39
|
+
* Parse the account/message ids out of a `/content/*` storage key such as
|
|
40
|
+
* `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/1.2`
|
|
41
|
+
* (the inverse of `buildMessageBodyKey`/`buildParsedBodyKey`/`buildBodyPartKey`
|
|
42
|
+
* above). Returns null when the shape doesn't match.
|
|
43
|
+
*/
|
|
44
|
+
export const parseContentStorageKey = (storageKey) => {
|
|
45
|
+
const match = storageKey.match(/^accounts\/([^/]+)\/([^/]+)\/messages\/([^/]+)\//);
|
|
46
|
+
if (!match)
|
|
47
|
+
return null;
|
|
48
|
+
return {
|
|
49
|
+
accountConfigId: match[1],
|
|
50
|
+
accountId: match[2],
|
|
51
|
+
messageId: match[3],
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
export const computeChecksum = (content) => createHash("sha256").update(content).digest("hex");
|
|
55
|
+
export const buildExportArchiveKey = (accountConfigId, exportRequestId) => `exports/${accountConfigId}/${exportRequestId}/export.zip`;
|
|
56
|
+
export const isStorageNotFoundError = (error) => {
|
|
57
|
+
if (typeof error !== "object" || error === null)
|
|
58
|
+
return false;
|
|
59
|
+
const obj = error;
|
|
60
|
+
if (obj.name === "NoSuchKey")
|
|
61
|
+
return true;
|
|
62
|
+
if (obj.Code === "NoSuchKey")
|
|
63
|
+
return true;
|
|
64
|
+
if (obj.code === "ENOENT")
|
|
65
|
+
return true;
|
|
66
|
+
return false;
|
|
67
|
+
};
|
|
68
|
+
export const createMockStorageService = () => {
|
|
69
|
+
const storage = new Map();
|
|
70
|
+
const storeInternal = (key, content) => {
|
|
71
|
+
const checksumSha256 = computeChecksum(content);
|
|
72
|
+
const uri = `mock://${key}`;
|
|
73
|
+
storage.set(uri, content);
|
|
74
|
+
return {
|
|
75
|
+
uri,
|
|
76
|
+
storageType: StorageType.Filesystem,
|
|
77
|
+
storageLocation: "mock",
|
|
78
|
+
storageKey: key,
|
|
79
|
+
sizeBytes: content.length,
|
|
80
|
+
checksumSha256,
|
|
81
|
+
contentEncoding: ContentEncoding.None,
|
|
82
|
+
};
|
|
83
|
+
};
|
|
84
|
+
const storeMessageBody = async (params) => {
|
|
85
|
+
const { accountConfigId, accountId, messageId, content } = params;
|
|
86
|
+
return storeInternal(buildMessageBodyKey(accountConfigId, accountId, messageId), content);
|
|
87
|
+
};
|
|
88
|
+
const storeMessageBodyStream = async (params) => {
|
|
89
|
+
const { accountConfigId, accountId, messageId, content } = params;
|
|
90
|
+
const chunks = [];
|
|
91
|
+
for await (const chunk of content) {
|
|
92
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
93
|
+
}
|
|
94
|
+
return storeInternal(buildMessageBodyKey(accountConfigId, accountId, messageId), Buffer.concat(chunks));
|
|
95
|
+
};
|
|
96
|
+
const storeBodyPart = async (params) => {
|
|
97
|
+
const { accountConfigId, accountId, messageId, partPath, content } = params;
|
|
98
|
+
return storeInternal(buildBodyPartKey(accountConfigId, accountId, messageId, partPath), content);
|
|
99
|
+
};
|
|
100
|
+
const storeOutboxAttachment = async (params) => {
|
|
101
|
+
const { accountConfigId, accountId, outboxMessageId, outboxAttachmentId, content, } = params;
|
|
102
|
+
return storeInternal(buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId), content);
|
|
103
|
+
};
|
|
104
|
+
const listOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
105
|
+
const prefix = `mock://${buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId)}`;
|
|
106
|
+
const items = [];
|
|
107
|
+
for (const [uri, content] of storage.entries()) {
|
|
108
|
+
if (!uri.startsWith(prefix))
|
|
109
|
+
continue;
|
|
110
|
+
items.push({
|
|
111
|
+
outboxAttachmentId: uri.slice(prefix.length),
|
|
112
|
+
key: uri.slice("mock://".length),
|
|
113
|
+
sizeBytes: content.length,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return items;
|
|
117
|
+
};
|
|
118
|
+
const listOutboxDraftsWithAttachments = async (accountConfigId, accountId) => {
|
|
119
|
+
const root = `mock://accounts/${accountConfigId}/${accountId}/outbox/`;
|
|
120
|
+
const drafts = new Set();
|
|
121
|
+
for (const uri of storage.keys()) {
|
|
122
|
+
if (!uri.startsWith(root))
|
|
123
|
+
continue;
|
|
124
|
+
const rest = uri.slice(root.length);
|
|
125
|
+
const slash = rest.indexOf("/");
|
|
126
|
+
if (slash > 0)
|
|
127
|
+
drafts.add(rest.slice(0, slash));
|
|
128
|
+
}
|
|
129
|
+
return [...drafts];
|
|
130
|
+
};
|
|
131
|
+
const deleteOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
132
|
+
storage.delete(`mock://${buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId)}`);
|
|
133
|
+
};
|
|
134
|
+
const statOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
135
|
+
const content = storage.get(`mock://${buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId)}`);
|
|
136
|
+
return content ? { sizeBytes: content.length } : null;
|
|
137
|
+
};
|
|
138
|
+
const retrieveOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => storage.get(`mock://${buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId)}`) ?? null;
|
|
139
|
+
const createOutboxAttachmentUploadUrl = async (params) => ({
|
|
140
|
+
uploadUrl: `mock://upload/${buildOutboxAttachmentKey(params.accountConfigId, params.accountId, params.outboxMessageId, params.outboxAttachmentId)}?max=${params.sizeBytes}&exp=${params.expiresAt}`,
|
|
141
|
+
});
|
|
142
|
+
const deleteOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
143
|
+
const prefix = `mock://${buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId)}`;
|
|
144
|
+
for (const uri of [...storage.keys()]) {
|
|
145
|
+
if (uri.startsWith(prefix))
|
|
146
|
+
storage.delete(uri);
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
const storeDeduplicated = async (params) => {
|
|
150
|
+
const { accountConfigId, accountId, content } = params;
|
|
151
|
+
const checksumSha256 = computeChecksum(content);
|
|
152
|
+
return storeInternal(buildDeduplicatedKey(accountConfigId, accountId, checksumSha256), content);
|
|
153
|
+
};
|
|
154
|
+
const storeParsedBody = async (params) => {
|
|
155
|
+
const { accountConfigId, accountId, messageId, parsed } = params;
|
|
156
|
+
const content = Buffer.from(JSON.stringify(parsed), "utf8");
|
|
157
|
+
return storeInternal(buildParsedBodyKey(accountConfigId, accountId, messageId), content);
|
|
158
|
+
};
|
|
159
|
+
const retrieveParsedBody = async (accountConfigId, accountId, messageId) => {
|
|
160
|
+
const uri = `mock://${buildParsedBodyKey(accountConfigId, accountId, messageId)}`;
|
|
161
|
+
const content = storage.get(uri);
|
|
162
|
+
if (!content)
|
|
163
|
+
return null;
|
|
164
|
+
return JSON.parse(content.toString("utf8"));
|
|
165
|
+
};
|
|
166
|
+
const retrieveBodyPart = async (accountConfigId, accountId, messageId, partPath) => {
|
|
167
|
+
const uri = `mock://${buildBodyPartKey(accountConfigId, accountId, messageId, partPath)}`;
|
|
168
|
+
return storage.get(uri) ?? null;
|
|
169
|
+
};
|
|
170
|
+
const retrieveMessageBody = async (accountConfigId, accountId, messageId) => {
|
|
171
|
+
const uri = `mock://${buildMessageBodyKey(accountConfigId, accountId, messageId)}`;
|
|
172
|
+
return storage.get(uri) ?? null;
|
|
173
|
+
};
|
|
174
|
+
const retrieveMessageBodyStream = async (accountConfigId, accountId, messageId) => {
|
|
175
|
+
const uri = `mock://${buildMessageBodyKey(accountConfigId, accountId, messageId)}`;
|
|
176
|
+
const content = storage.get(uri);
|
|
177
|
+
if (!content)
|
|
178
|
+
return null;
|
|
179
|
+
return Readable.from(content);
|
|
180
|
+
};
|
|
181
|
+
const bodyPartExists = async (accountConfigId, accountId, messageId, partPath) => {
|
|
182
|
+
const uri = `mock://${buildBodyPartKey(accountConfigId, accountId, messageId, partPath)}`;
|
|
183
|
+
return storage.has(uri);
|
|
184
|
+
};
|
|
185
|
+
return {
|
|
186
|
+
storeMessageBody,
|
|
187
|
+
storeMessageBodyStream,
|
|
188
|
+
storeBodyPart,
|
|
189
|
+
bodyPartExists,
|
|
190
|
+
retrieveBodyPart,
|
|
191
|
+
storeOutboxAttachment,
|
|
192
|
+
listOutboxAttachments,
|
|
193
|
+
listOutboxDraftsWithAttachments,
|
|
194
|
+
deleteOutboxAttachments,
|
|
195
|
+
deleteOutboxAttachment,
|
|
196
|
+
statOutboxAttachment,
|
|
197
|
+
retrieveOutboxAttachment,
|
|
198
|
+
createOutboxAttachmentUploadUrl,
|
|
199
|
+
storeDeduplicated,
|
|
200
|
+
storeParsedBody,
|
|
201
|
+
retrieveParsedBody,
|
|
202
|
+
retrieveMessageBody,
|
|
203
|
+
retrieveMessageBodyStream,
|
|
204
|
+
retrieve: async (uri) => {
|
|
205
|
+
const content = storage.get(uri);
|
|
206
|
+
if (!content) {
|
|
207
|
+
throw Object.assign(new Error(`Not found: ${uri}`), {
|
|
208
|
+
name: "NoSuchKey",
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
return content;
|
|
212
|
+
},
|
|
213
|
+
exists: async (uri) => storage.has(uri),
|
|
214
|
+
delete: async (uri) => {
|
|
215
|
+
storage.delete(uri);
|
|
216
|
+
},
|
|
217
|
+
storeExportArchiveStream: async (accountConfigId, exportRequestId, body) => {
|
|
218
|
+
const key = buildExportArchiveKey(accountConfigId, exportRequestId);
|
|
219
|
+
const chunks = [];
|
|
220
|
+
for await (const chunk of body) {
|
|
221
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
222
|
+
}
|
|
223
|
+
storage.set(`mock://${key}`, Buffer.concat(chunks));
|
|
224
|
+
return key;
|
|
225
|
+
},
|
|
226
|
+
getPresignedDownloadUrl: async (key, _expiresInSeconds) => `mock://presigned/${key}`,
|
|
227
|
+
};
|
|
228
|
+
};
|
package/dist/uri.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { StorageType } from "@remit/domain-enums";
|
|
2
|
+
export type StorageTypeValue = (typeof StorageType)[keyof typeof StorageType];
|
|
3
|
+
export interface ParsedStorageUri {
|
|
4
|
+
storageType: StorageTypeValue;
|
|
5
|
+
storageLocation: string;
|
|
6
|
+
storageKey: string;
|
|
7
|
+
}
|
|
8
|
+
export declare const parseStorageUri: (uri: string) => ParsedStorageUri;
|
|
9
|
+
export declare const buildStorageUri: (storageType: StorageTypeValue, storageLocation: string, storageKey: string) => string;
|
package/dist/uri.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { StorageType } from "@remit/domain-enums";
|
|
2
|
+
export const parseStorageUri = (uri) => {
|
|
3
|
+
const url = new URL(uri);
|
|
4
|
+
if (url.protocol === "s3:") {
|
|
5
|
+
return {
|
|
6
|
+
storageType: StorageType.S3,
|
|
7
|
+
storageLocation: url.hostname,
|
|
8
|
+
storageKey: url.pathname.slice(1),
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
if (url.protocol === "file:") {
|
|
12
|
+
return {
|
|
13
|
+
storageType: StorageType.Filesystem,
|
|
14
|
+
storageLocation: "",
|
|
15
|
+
storageKey: url.pathname,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
throw new Error(`Unsupported storage URI scheme: ${url.protocol}`);
|
|
19
|
+
};
|
|
20
|
+
export const buildStorageUri = (storageType, storageLocation, storageKey) => {
|
|
21
|
+
if (storageType === StorageType.S3) {
|
|
22
|
+
return `s3://${storageLocation}/${storageKey}`;
|
|
23
|
+
}
|
|
24
|
+
if (storageType === StorageType.Filesystem) {
|
|
25
|
+
return `file://${storageLocation}/${storageKey}`;
|
|
26
|
+
}
|
|
27
|
+
throw new Error(`Unsupported storage type: ${storageType}`);
|
|
28
|
+
};
|