@remit/storage-service 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backends/filesystem.d.ts +12 -0
- package/dist/backends/filesystem.js +316 -0
- package/dist/backends/s3.d.ts +3 -0
- package/dist/backends/s3.js +413 -0
- package/dist/filesystem.js +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/outbox-sweep.d.ts +33 -0
- package/dist/outbox-sweep.js +37 -0
- package/dist/outbox-upload-url.d.ts +44 -0
- package/{src/outbox-upload-url.ts → dist/outbox-upload-url.js} +20 -50
- package/dist/s3.d.ts +3 -0
- package/dist/s3.js +28 -0
- package/dist/signed-path.d.ts +14 -0
- package/dist/signed-path.js +46 -0
- package/dist/storage.d.ts +255 -0
- package/dist/storage.js +228 -0
- package/dist/uri.d.ts +9 -0
- package/dist/uri.js +28 -0
- package/package.json +51 -46
- package/src/backends/filesystem.ts +0 -624
- package/src/backends/s3.test.ts +0 -601
- package/src/backends/s3.ts +0 -761
- package/src/index.ts +0 -60
- package/src/outbox-attachment-retrieve.test.ts +0 -56
- package/src/outbox-sweep.test.ts +0 -196
- package/src/outbox-sweep.ts +0 -103
- package/src/outbox-upload-url.test.ts +0 -290
- package/src/s3.ts +0 -36
- package/src/signed-path.ts +0 -86
- package/src/storage.test.ts +0 -886
- package/src/storage.ts +0 -866
- package/src/uri.ts +0 -47
- package/tsconfig.json +0 -8
- /package/{src/filesystem.ts → dist/filesystem.d.ts} +0 -0
package/src/storage.ts
DELETED
|
@@ -1,866 +0,0 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import { Readable } from "node:stream";
|
|
3
|
-
import { ContentEncoding, StorageType } from "@remit/domain-enums";
|
|
4
|
-
|
|
5
|
-
export type StorageTypeValue = (typeof StorageType)[keyof typeof StorageType];
|
|
6
|
-
export type ContentEncodingValue =
|
|
7
|
-
(typeof ContentEncoding)[keyof typeof ContentEncoding];
|
|
8
|
-
|
|
9
|
-
export interface StorageReference {
|
|
10
|
-
uri: string;
|
|
11
|
-
storageType: StorageTypeValue;
|
|
12
|
-
storageLocation: string;
|
|
13
|
-
storageKey: string;
|
|
14
|
-
sizeBytes: number;
|
|
15
|
-
/**
|
|
16
|
-
* SHA-256 (hex) of the logical, pre-compression content.
|
|
17
|
-
* Identifies the content regardless of how it is stored on the backend,
|
|
18
|
-
* and is used by `buildDeduplicatedKey` for content-addressable storage.
|
|
19
|
-
* Note: this is NOT the checksum S3 validates against the received bytes —
|
|
20
|
-
* S3's own transport-level checksum is handled by the SDK on PUT.
|
|
21
|
-
*/
|
|
22
|
-
checksumSha256: string;
|
|
23
|
-
contentEncoding: ContentEncodingValue;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/** Parameters for storing a message body (raw RFC822 content) */
|
|
27
|
-
export interface StoreMessageBodyParams {
|
|
28
|
-
accountConfigId: string;
|
|
29
|
-
accountId: string;
|
|
30
|
-
messageId: string;
|
|
31
|
-
content: Buffer;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/** Parameters for streaming a message body (raw RFC822 content) to storage */
|
|
35
|
-
export interface StoreMessageBodyStreamParams {
|
|
36
|
-
accountConfigId: string;
|
|
37
|
-
accountId: string;
|
|
38
|
-
messageId: string;
|
|
39
|
-
/** Readable stream of the raw message bytes; streamed to storage, never buffered whole. */
|
|
40
|
-
content: Readable;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/** Parameters for storing a body part (attachment, inline content) */
|
|
44
|
-
export interface StoreBodyPartParams {
|
|
45
|
-
accountConfigId: string;
|
|
46
|
-
accountId: string;
|
|
47
|
-
messageId: string;
|
|
48
|
-
partPath: string;
|
|
49
|
-
content: Buffer;
|
|
50
|
-
contentType?: string;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* Parameters for storing a file attached to a draft outbox message.
|
|
55
|
-
*
|
|
56
|
-
* Deliberately not the deduplicated path: a content-addressed object belongs to
|
|
57
|
-
* no single draft, so nothing may delete it when the draft goes. A per-draft key
|
|
58
|
-
* dies with the draft.
|
|
59
|
-
*/
|
|
60
|
-
export interface StoreOutboxAttachmentParams {
|
|
61
|
-
accountConfigId: string;
|
|
62
|
-
accountId: string;
|
|
63
|
-
outboxMessageId: string;
|
|
64
|
-
outboxAttachmentId: string;
|
|
65
|
-
content: Buffer;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* One uploaded object under a draft's attachment prefix. Reservations are not
|
|
70
|
-
* here — they live in the ledger, which is the only thing that can say whether
|
|
71
|
-
* an object present in storage is an attachment or garbage.
|
|
72
|
-
*/
|
|
73
|
-
export interface OutboxAttachmentListItem {
|
|
74
|
-
outboxAttachmentId: string;
|
|
75
|
-
key: string;
|
|
76
|
-
sizeBytes: number;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
/** Somewhere for a client to PUT one file, for a while */
|
|
80
|
-
export interface OutboxAttachmentUploadTarget {
|
|
81
|
-
/**
|
|
82
|
-
* Absolute URL the client PUTs raw bytes to. Opaque by design: block storage
|
|
83
|
-
* on one deployment, this deployment's own upload route on another, and a
|
|
84
|
-
* client cannot tell which it was handed.
|
|
85
|
-
*/
|
|
86
|
-
uploadUrl: string;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
export interface CreateOutboxAttachmentUploadUrlParams {
|
|
90
|
-
accountConfigId: string;
|
|
91
|
-
accountId: string;
|
|
92
|
-
outboxMessageId: string;
|
|
93
|
-
outboxAttachmentId: string;
|
|
94
|
-
/** Exact byte count the upload may carry. */
|
|
95
|
-
sizeBytes: number;
|
|
96
|
-
expiresAt: number;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/** Parameters for storing deduplicated content (attachments shared across messages) */
|
|
100
|
-
export interface StoreDeduplicatedParams {
|
|
101
|
-
accountConfigId: string;
|
|
102
|
-
accountId: string;
|
|
103
|
-
content: Buffer;
|
|
104
|
-
contentType?: string;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/** Metadata for a single attachment in a parsed message body. No binary content. */
|
|
108
|
-
export interface ParsedAttachmentMeta {
|
|
109
|
-
filename: string | null;
|
|
110
|
-
contentType: string;
|
|
111
|
-
contentDisposition: string | null;
|
|
112
|
-
contentId: string | null;
|
|
113
|
-
size: number;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/** Pre-parsed message body cached as parsed.json.gz to skip mailparser on warm reads */
|
|
117
|
-
export interface ParsedBody {
|
|
118
|
-
text: string | null;
|
|
119
|
-
html: string | null;
|
|
120
|
-
attachments: ParsedAttachmentMeta[];
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/** Parameters for storing a pre-parsed message body */
|
|
124
|
-
export interface StoreParsedBodyParams {
|
|
125
|
-
accountConfigId: string;
|
|
126
|
-
accountId: string;
|
|
127
|
-
messageId: string;
|
|
128
|
-
parsed: ParsedBody;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
/** Parameters for storing extracted attachment text (attachment-scan worker, #450) */
|
|
132
|
-
export interface StoreExtractedTextParams {
|
|
133
|
-
accountConfigId: string;
|
|
134
|
-
accountId: string;
|
|
135
|
-
messageId: string;
|
|
136
|
-
partPath: string;
|
|
137
|
-
text: string;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Marker written in place of extracted text when extraction is skipped or
|
|
142
|
-
* fails, so the idempotency check terminates retries on a poison or
|
|
143
|
-
* unsupported attachment instead of re-running extraction on every
|
|
144
|
-
* redelivery.
|
|
145
|
-
*/
|
|
146
|
-
export interface ExtractedTextMarker {
|
|
147
|
-
status: "skipped" | "failed";
|
|
148
|
-
reason: string;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/** Parameters for storing an extraction skip/failure marker */
|
|
152
|
-
export interface StoreExtractedSkippedParams {
|
|
153
|
-
accountConfigId: string;
|
|
154
|
-
accountId: string;
|
|
155
|
-
messageId: string;
|
|
156
|
-
partPath: string;
|
|
157
|
-
marker: ExtractedTextMarker;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/** One extracted-text artifact found under a message's `extracted/` prefix */
|
|
161
|
-
export interface ExtractedTextListItem {
|
|
162
|
-
partPath: string;
|
|
163
|
-
key: string;
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
export interface StorageService {
|
|
167
|
-
/** Store a message body (raw RFC822 content) */
|
|
168
|
-
storeMessageBody(params: StoreMessageBodyParams): Promise<StorageReference>;
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
* Store a message body from a readable stream, never buffering the whole
|
|
172
|
-
* body. Use this on the sync hot path so a ranged FETCH streams straight to
|
|
173
|
-
* storage.
|
|
174
|
-
*/
|
|
175
|
-
storeMessageBodyStream(
|
|
176
|
-
params: StoreMessageBodyStreamParams,
|
|
177
|
-
): Promise<StorageReference>;
|
|
178
|
-
|
|
179
|
-
/** Store a body part (attachment, inline content) */
|
|
180
|
-
storeBodyPart(params: StoreBodyPartParams): Promise<StorageReference>;
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Check whether a body-part object already exists, keyed exactly as
|
|
184
|
-
* `storeBodyPart` writes it. Used by the lazy per-part generation path to
|
|
185
|
-
* skip leaves already materialized on a prior read, so regeneration is
|
|
186
|
-
* idempotent and re-reads stay cheap.
|
|
187
|
-
*/
|
|
188
|
-
bodyPartExists(
|
|
189
|
-
accountConfigId: string,
|
|
190
|
-
accountId: string,
|
|
191
|
-
messageId: string,
|
|
192
|
-
partPath: string,
|
|
193
|
-
): Promise<boolean>;
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* Retrieve a body-part's content (attachment or inline leaf) by
|
|
197
|
-
* account/message id and IMAP section path; returns null on NoSuchKey.
|
|
198
|
-
*/
|
|
199
|
-
retrieveBodyPart(
|
|
200
|
-
accountConfigId: string,
|
|
201
|
-
accountId: string,
|
|
202
|
-
messageId: string,
|
|
203
|
-
partPath: string,
|
|
204
|
-
): Promise<Buffer | null>;
|
|
205
|
-
|
|
206
|
-
/**
|
|
207
|
-
* Store a file attached to a draft outbox message, uncompressed: the size on
|
|
208
|
-
* the backend is then the decoded size the per-message cap is expressed in,
|
|
209
|
-
* and `listOutboxAttachments` can total a draft without opening anything.
|
|
210
|
-
*/
|
|
211
|
-
storeOutboxAttachment(
|
|
212
|
-
params: StoreOutboxAttachmentParams,
|
|
213
|
-
): Promise<StorageReference>;
|
|
214
|
-
|
|
215
|
-
/**
|
|
216
|
-
* Every object stored against a draft, paged to the end. Only the sweep reads
|
|
217
|
-
* this, and a sweep that stops early leaves exactly the orphan it was called
|
|
218
|
-
* to collect — so there is no limit to get wrong.
|
|
219
|
-
*/
|
|
220
|
-
listOutboxAttachments(
|
|
221
|
-
accountConfigId: string,
|
|
222
|
-
accountId: string,
|
|
223
|
-
outboxMessageId: string,
|
|
224
|
-
): Promise<OutboxAttachmentListItem[]>;
|
|
225
|
-
|
|
226
|
-
/**
|
|
227
|
-
* Remove every file and reservation stored against a draft. Unlike the read
|
|
228
|
-
* above this takes no limit and pages to the end: a sweep that stops early
|
|
229
|
-
* leaves exactly the orphan it was called to prevent.
|
|
230
|
-
*/
|
|
231
|
-
deleteOutboxAttachments(
|
|
232
|
-
accountConfigId: string,
|
|
233
|
-
accountId: string,
|
|
234
|
-
outboxMessageId: string,
|
|
235
|
-
): Promise<void>;
|
|
236
|
-
|
|
237
|
-
/**
|
|
238
|
-
* Every draft under an account that has attachment objects in storage,
|
|
239
|
-
* including drafts whose row is long gone. The sweep needs the prefixes that
|
|
240
|
-
* exist, not the rows that exist — a discarded draft is invisible to the
|
|
241
|
-
* database and is exactly the case worth collecting.
|
|
242
|
-
*/
|
|
243
|
-
listOutboxDraftsWithAttachments(
|
|
244
|
-
accountConfigId: string,
|
|
245
|
-
accountId: string,
|
|
246
|
-
): Promise<string[]>;
|
|
247
|
-
|
|
248
|
-
/** Remove one attachment object. */
|
|
249
|
-
deleteOutboxAttachment(
|
|
250
|
-
accountConfigId: string,
|
|
251
|
-
accountId: string,
|
|
252
|
-
outboxMessageId: string,
|
|
253
|
-
outboxAttachmentId: string,
|
|
254
|
-
): Promise<void>;
|
|
255
|
-
|
|
256
|
-
/**
|
|
257
|
-
* The size storage actually holds for an attachment, or null when nothing
|
|
258
|
-
* was uploaded. This is the only number a completion may believe — the
|
|
259
|
-
* client's word for it is not evidence, and on a deployment where the bytes
|
|
260
|
-
* went straight to block storage it is the only reading available.
|
|
261
|
-
*/
|
|
262
|
-
statOutboxAttachment(
|
|
263
|
-
accountConfigId: string,
|
|
264
|
-
accountId: string,
|
|
265
|
-
outboxMessageId: string,
|
|
266
|
-
outboxAttachmentId: string,
|
|
267
|
-
): Promise<{ sizeBytes: number } | null>;
|
|
268
|
-
|
|
269
|
-
/**
|
|
270
|
-
* The bytes of one attachment, for building the MIME part it goes out as.
|
|
271
|
-
* Null when nothing is stored under the key.
|
|
272
|
-
*/
|
|
273
|
-
retrieveOutboxAttachment(
|
|
274
|
-
accountConfigId: string,
|
|
275
|
-
accountId: string,
|
|
276
|
-
outboxMessageId: string,
|
|
277
|
-
outboxAttachmentId: string,
|
|
278
|
-
): Promise<Buffer | null>;
|
|
279
|
-
|
|
280
|
-
/**
|
|
281
|
-
* Mint somewhere to PUT one file. The backend decides what that is — block
|
|
282
|
-
* storage direct, or this deployment's upload route — and the caller never
|
|
283
|
-
* branches on which it got. Whatever it returns must bind the size: an
|
|
284
|
-
* upload that carries more than was reserved has to fail at the thing
|
|
285
|
-
* receiving it, not afterwards.
|
|
286
|
-
*/
|
|
287
|
-
createOutboxAttachmentUploadUrl(
|
|
288
|
-
params: CreateOutboxAttachmentUploadUrlParams,
|
|
289
|
-
): Promise<OutboxAttachmentUploadTarget>;
|
|
290
|
-
|
|
291
|
-
/** Store deduplicated content (content-addressable, for attachments) */
|
|
292
|
-
storeDeduplicated(params: StoreDeduplicatedParams): Promise<StorageReference>;
|
|
293
|
-
|
|
294
|
-
/** Store a pre-parsed message body as gzipped JSON (parsed-body cache) */
|
|
295
|
-
storeParsedBody(params: StoreParsedBodyParams): Promise<StorageReference>;
|
|
296
|
-
|
|
297
|
-
/** Retrieve a pre-parsed message body by account/message id; returns null on NoSuchKey */
|
|
298
|
-
retrieveParsedBody(
|
|
299
|
-
accountConfigId: string,
|
|
300
|
-
accountId: string,
|
|
301
|
-
messageId: string,
|
|
302
|
-
): Promise<ParsedBody | null>;
|
|
303
|
-
|
|
304
|
-
/** Store extracted attachment text (attachment-scan worker, #450) */
|
|
305
|
-
storeExtractedText(
|
|
306
|
-
params: StoreExtractedTextParams,
|
|
307
|
-
): Promise<StorageReference>;
|
|
308
|
-
|
|
309
|
-
/** Store a skip/failure marker in place of extracted text */
|
|
310
|
-
storeExtractedSkipped(
|
|
311
|
-
params: StoreExtractedSkippedParams,
|
|
312
|
-
): Promise<StorageReference>;
|
|
313
|
-
|
|
314
|
-
/**
|
|
315
|
-
* Whether an extraction result — either the text artifact or the
|
|
316
|
-
* skip/failure marker — already exists for this part. Makes extraction
|
|
317
|
-
* idempotent: a redelivered SQS message short-circuits instead of
|
|
318
|
-
* re-running extraction (#450).
|
|
319
|
-
*/
|
|
320
|
-
extractedResultExists(
|
|
321
|
-
accountConfigId: string,
|
|
322
|
-
accountId: string,
|
|
323
|
-
messageId: string,
|
|
324
|
-
partPath: string,
|
|
325
|
-
): Promise<boolean>;
|
|
326
|
-
|
|
327
|
-
/** Retrieve previously extracted text; returns null on NoSuchKey */
|
|
328
|
-
retrieveExtractedText(
|
|
329
|
-
accountConfigId: string,
|
|
330
|
-
accountId: string,
|
|
331
|
-
messageId: string,
|
|
332
|
-
partPath: string,
|
|
333
|
-
): Promise<string | null>;
|
|
334
|
-
|
|
335
|
-
/**
|
|
336
|
-
* List every extracted-text artifact stored for a message — the
|
|
337
|
-
* `.skipped.json` markers are excluded. Used by the search-index worker
|
|
338
|
-
* (#452) to fold attachment text into a message's vectors.
|
|
339
|
-
*/
|
|
340
|
-
listExtractedTexts(
|
|
341
|
-
accountConfigId: string,
|
|
342
|
-
accountId: string,
|
|
343
|
-
messageId: string,
|
|
344
|
-
): Promise<ExtractedTextListItem[]>;
|
|
345
|
-
|
|
346
|
-
/** Retrieve a raw message body (RFC822 .eml) by account/message id; returns null on NoSuchKey */
|
|
347
|
-
retrieveMessageBody(
|
|
348
|
-
accountConfigId: string,
|
|
349
|
-
accountId: string,
|
|
350
|
-
messageId: string,
|
|
351
|
-
): Promise<Buffer | null>;
|
|
352
|
-
|
|
353
|
-
/**
|
|
354
|
-
* Retrieve a raw message body as a readable stream of decompressed .eml
|
|
355
|
-
* bytes; returns null on NoSuchKey. Use on the export path so a whole
|
|
356
|
-
* mailbox is never buffered in memory.
|
|
357
|
-
*/
|
|
358
|
-
retrieveMessageBodyStream(
|
|
359
|
-
accountConfigId: string,
|
|
360
|
-
accountId: string,
|
|
361
|
-
messageId: string,
|
|
362
|
-
): Promise<Readable | null>;
|
|
363
|
-
|
|
364
|
-
/** Retrieve content by URI */
|
|
365
|
-
retrieve(uri: string): Promise<Buffer>;
|
|
366
|
-
|
|
367
|
-
/** Check if content exists */
|
|
368
|
-
exists(uri: string): Promise<boolean>;
|
|
369
|
-
|
|
370
|
-
/** Delete content by URI */
|
|
371
|
-
delete(uri: string): Promise<void>;
|
|
372
|
-
|
|
373
|
-
/**
|
|
374
|
-
* Stream a finished export archive (already-compressed ZIP) to the export
|
|
375
|
-
* key for the given config + request, uncompressed at rest. The body is
|
|
376
|
-
* streamed via a multipart upload and never buffered whole. Returns the raw
|
|
377
|
-
* storage key, which `getPresignedDownloadUrl` can then sign.
|
|
378
|
-
*/
|
|
379
|
-
storeExportArchiveStream(
|
|
380
|
-
accountConfigId: string,
|
|
381
|
-
exportRequestId: string,
|
|
382
|
-
body: Readable,
|
|
383
|
-
): Promise<string>;
|
|
384
|
-
|
|
385
|
-
/**
|
|
386
|
-
* Generate a presigned URL that grants temporary GET access to a raw S3 key.
|
|
387
|
-
* Use for export archives — the key must already exist in the bucket.
|
|
388
|
-
* Not supported on filesystem backends; throws if called there.
|
|
389
|
-
*/
|
|
390
|
-
getPresignedDownloadUrl(
|
|
391
|
-
key: string,
|
|
392
|
-
expiresInSeconds: number,
|
|
393
|
-
): Promise<string>;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
// Path builders - centralized path formatting per RFC 011 + #224.
|
|
397
|
-
// The outer segment is `accountConfigId` (Cognito-derived tenant scope) so
|
|
398
|
-
// the edge can verify cross-tenant isolation at the URL prefix without
|
|
399
|
-
// loading any database record. The inner `accountId` distinguishes between
|
|
400
|
-
// multiple IMAP-account records owned by the same tenant.
|
|
401
|
-
export const buildMessageBodyKey = (
|
|
402
|
-
accountConfigId: string,
|
|
403
|
-
accountId: string,
|
|
404
|
-
messageId: string,
|
|
405
|
-
): string =>
|
|
406
|
-
`accounts/${accountConfigId}/${accountId}/messages/${messageId}/body.eml`;
|
|
407
|
-
|
|
408
|
-
export const buildParsedBodyKey = (
|
|
409
|
-
accountConfigId: string,
|
|
410
|
-
accountId: string,
|
|
411
|
-
messageId: string,
|
|
412
|
-
): string =>
|
|
413
|
-
`accounts/${accountConfigId}/${accountId}/messages/${messageId}/parsed.json.gz`;
|
|
414
|
-
|
|
415
|
-
export const buildBodyPartKey = (
|
|
416
|
-
accountConfigId: string,
|
|
417
|
-
accountId: string,
|
|
418
|
-
messageId: string,
|
|
419
|
-
partPath: string,
|
|
420
|
-
): string =>
|
|
421
|
-
`accounts/${accountConfigId}/${accountId}/messages/${messageId}/parts/${partPath}`;
|
|
422
|
-
|
|
423
|
-
export const buildExtractedTextKey = (
|
|
424
|
-
accountConfigId: string,
|
|
425
|
-
accountId: string,
|
|
426
|
-
messageId: string,
|
|
427
|
-
partPath: string,
|
|
428
|
-
): string =>
|
|
429
|
-
`${buildExtractedPrefix(accountConfigId, accountId, messageId)}${partPath}.txt.gz`;
|
|
430
|
-
|
|
431
|
-
export const buildExtractedSkippedKey = (
|
|
432
|
-
accountConfigId: string,
|
|
433
|
-
accountId: string,
|
|
434
|
-
messageId: string,
|
|
435
|
-
partPath: string,
|
|
436
|
-
): string =>
|
|
437
|
-
`${buildExtractedPrefix(accountConfigId, accountId, messageId)}${partPath}.skipped.json`;
|
|
438
|
-
|
|
439
|
-
export const buildExtractedPrefix = (
|
|
440
|
-
accountConfigId: string,
|
|
441
|
-
accountId: string,
|
|
442
|
-
messageId: string,
|
|
443
|
-
): string =>
|
|
444
|
-
`accounts/${accountConfigId}/${accountId}/messages/${messageId}/extracted/`;
|
|
445
|
-
|
|
446
|
-
export const buildOutboxAttachmentPrefix = (
|
|
447
|
-
accountConfigId: string,
|
|
448
|
-
accountId: string,
|
|
449
|
-
outboxMessageId: string,
|
|
450
|
-
): string =>
|
|
451
|
-
`accounts/${accountConfigId}/${accountId}/outbox/${outboxMessageId}/attachments/`;
|
|
452
|
-
|
|
453
|
-
export const buildOutboxAttachmentKey = (
|
|
454
|
-
accountConfigId: string,
|
|
455
|
-
accountId: string,
|
|
456
|
-
outboxMessageId: string,
|
|
457
|
-
outboxAttachmentId: string,
|
|
458
|
-
): string =>
|
|
459
|
-
`${buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId)}${outboxAttachmentId}`;
|
|
460
|
-
|
|
461
|
-
// Every segment is a generated id, so the shape is closed rather than
|
|
462
|
-
// "anything without a slash": `..` matches `[^/]+` and the write side joins
|
|
463
|
-
// this key onto a filesystem root. The read side has `resolveContentPath` for
|
|
464
|
-
// the same reason (#310 review P1) — a signature should not be the only thing
|
|
465
|
-
// standing between a URL and the storage root.
|
|
466
|
-
const OUTBOX_ATTACHMENT_KEY =
|
|
467
|
-
/^accounts\/([A-Za-z0-9_-]+)\/([A-Za-z0-9_-]+)\/outbox\/([A-Za-z0-9_-]+)\/attachments\/([A-Za-z0-9_-]+)$/;
|
|
468
|
-
export interface ParsedOutboxAttachmentKey {
|
|
469
|
-
accountConfigId: string;
|
|
470
|
-
accountId: string;
|
|
471
|
-
outboxMessageId: string;
|
|
472
|
-
outboxAttachmentId: string;
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
/**
|
|
476
|
-
* Read the ids back out of an attachment key — the inverse of
|
|
477
|
-
* `buildOutboxAttachmentKey`, for the upload route, which is handed a path and
|
|
478
|
-
* has to work out what it addresses. Returns null on anything else, including a
|
|
479
|
-
* reservation key: bytes are never written to one.
|
|
480
|
-
*/
|
|
481
|
-
export const parseOutboxAttachmentKey = (
|
|
482
|
-
storageKey: string,
|
|
483
|
-
): ParsedOutboxAttachmentKey | null => {
|
|
484
|
-
const match = storageKey.match(OUTBOX_ATTACHMENT_KEY);
|
|
485
|
-
if (!match) return null;
|
|
486
|
-
return {
|
|
487
|
-
accountConfigId: match[1],
|
|
488
|
-
accountId: match[2],
|
|
489
|
-
outboxMessageId: match[3],
|
|
490
|
-
outboxAttachmentId: match[4],
|
|
491
|
-
};
|
|
492
|
-
};
|
|
493
|
-
|
|
494
|
-
export const buildDeduplicatedKey = (
|
|
495
|
-
accountConfigId: string,
|
|
496
|
-
accountId: string,
|
|
497
|
-
checksumSha256: string,
|
|
498
|
-
): string =>
|
|
499
|
-
`accounts/${accountConfigId}/${accountId}/dedup/${checksumSha256.slice(0, 2)}/${checksumSha256}`;
|
|
500
|
-
|
|
501
|
-
export interface ParsedContentStorageKey {
|
|
502
|
-
accountConfigId: string;
|
|
503
|
-
accountId: string;
|
|
504
|
-
messageId: string;
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
/**
|
|
508
|
-
* Parse the account/message ids out of a `/content/*` storage key such as
|
|
509
|
-
* `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/1.2`
|
|
510
|
-
* (the inverse of `buildMessageBodyKey`/`buildParsedBodyKey`/`buildBodyPartKey`
|
|
511
|
-
* above). Returns null when the shape doesn't match.
|
|
512
|
-
*/
|
|
513
|
-
export const parseContentStorageKey = (
|
|
514
|
-
storageKey: string,
|
|
515
|
-
): ParsedContentStorageKey | null => {
|
|
516
|
-
const match = storageKey.match(
|
|
517
|
-
/^accounts\/([^/]+)\/([^/]+)\/messages\/([^/]+)\//,
|
|
518
|
-
);
|
|
519
|
-
if (!match) return null;
|
|
520
|
-
return {
|
|
521
|
-
accountConfigId: match[1],
|
|
522
|
-
accountId: match[2],
|
|
523
|
-
messageId: match[3],
|
|
524
|
-
};
|
|
525
|
-
};
|
|
526
|
-
|
|
527
|
-
export const computeChecksum = (content: Buffer): string =>
|
|
528
|
-
createHash("sha256").update(content).digest("hex");
|
|
529
|
-
|
|
530
|
-
export const buildExportArchiveKey = (
|
|
531
|
-
accountConfigId: string,
|
|
532
|
-
exportRequestId: string,
|
|
533
|
-
): string => `exports/${accountConfigId}/${exportRequestId}/export.zip`;
|
|
534
|
-
|
|
535
|
-
export const isStorageNotFoundError = (error: unknown): boolean => {
|
|
536
|
-
if (typeof error !== "object" || error === null) return false;
|
|
537
|
-
const obj = error as Record<string, unknown>;
|
|
538
|
-
if (obj.name === "NoSuchKey") return true;
|
|
539
|
-
if (obj.Code === "NoSuchKey") return true;
|
|
540
|
-
if (obj.code === "ENOENT") return true;
|
|
541
|
-
return false;
|
|
542
|
-
};
|
|
543
|
-
|
|
544
|
-
export const createMockStorageService = (): StorageService => {
|
|
545
|
-
const storage = new Map<string, Buffer>();
|
|
546
|
-
|
|
547
|
-
const storeInternal = (key: string, content: Buffer): StorageReference => {
|
|
548
|
-
const checksumSha256 = computeChecksum(content);
|
|
549
|
-
const uri = `mock://${key}`;
|
|
550
|
-
storage.set(uri, content);
|
|
551
|
-
return {
|
|
552
|
-
uri,
|
|
553
|
-
storageType: StorageType.Filesystem,
|
|
554
|
-
storageLocation: "mock",
|
|
555
|
-
storageKey: key,
|
|
556
|
-
sizeBytes: content.length,
|
|
557
|
-
checksumSha256,
|
|
558
|
-
contentEncoding: ContentEncoding.None,
|
|
559
|
-
};
|
|
560
|
-
};
|
|
561
|
-
|
|
562
|
-
const storeMessageBody: StorageService["storeMessageBody"] = async (
|
|
563
|
-
params,
|
|
564
|
-
) => {
|
|
565
|
-
const { accountConfigId, accountId, messageId, content } = params;
|
|
566
|
-
return storeInternal(
|
|
567
|
-
buildMessageBodyKey(accountConfigId, accountId, messageId),
|
|
568
|
-
content,
|
|
569
|
-
);
|
|
570
|
-
};
|
|
571
|
-
|
|
572
|
-
const storeMessageBodyStream: StorageService["storeMessageBodyStream"] =
|
|
573
|
-
async (params) => {
|
|
574
|
-
const { accountConfigId, accountId, messageId, content } = params;
|
|
575
|
-
const chunks: Buffer[] = [];
|
|
576
|
-
for await (const chunk of content) {
|
|
577
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
578
|
-
}
|
|
579
|
-
return storeInternal(
|
|
580
|
-
buildMessageBodyKey(accountConfigId, accountId, messageId),
|
|
581
|
-
Buffer.concat(chunks),
|
|
582
|
-
);
|
|
583
|
-
};
|
|
584
|
-
|
|
585
|
-
const storeBodyPart: StorageService["storeBodyPart"] = async (params) => {
|
|
586
|
-
const { accountConfigId, accountId, messageId, partPath, content } = params;
|
|
587
|
-
return storeInternal(
|
|
588
|
-
buildBodyPartKey(accountConfigId, accountId, messageId, partPath),
|
|
589
|
-
content,
|
|
590
|
-
);
|
|
591
|
-
};
|
|
592
|
-
|
|
593
|
-
const storeOutboxAttachment: StorageService["storeOutboxAttachment"] = async (
|
|
594
|
-
params,
|
|
595
|
-
) => {
|
|
596
|
-
const {
|
|
597
|
-
accountConfigId,
|
|
598
|
-
accountId,
|
|
599
|
-
outboxMessageId,
|
|
600
|
-
outboxAttachmentId,
|
|
601
|
-
content,
|
|
602
|
-
} = params;
|
|
603
|
-
return storeInternal(
|
|
604
|
-
buildOutboxAttachmentKey(
|
|
605
|
-
accountConfigId,
|
|
606
|
-
accountId,
|
|
607
|
-
outboxMessageId,
|
|
608
|
-
outboxAttachmentId,
|
|
609
|
-
),
|
|
610
|
-
content,
|
|
611
|
-
);
|
|
612
|
-
};
|
|
613
|
-
|
|
614
|
-
const listOutboxAttachments: StorageService["listOutboxAttachments"] = async (
|
|
615
|
-
accountConfigId,
|
|
616
|
-
accountId,
|
|
617
|
-
outboxMessageId,
|
|
618
|
-
) => {
|
|
619
|
-
const prefix = `mock://${buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId)}`;
|
|
620
|
-
const items: OutboxAttachmentListItem[] = [];
|
|
621
|
-
for (const [uri, content] of storage.entries()) {
|
|
622
|
-
if (!uri.startsWith(prefix)) continue;
|
|
623
|
-
items.push({
|
|
624
|
-
outboxAttachmentId: uri.slice(prefix.length),
|
|
625
|
-
key: uri.slice("mock://".length),
|
|
626
|
-
sizeBytes: content.length,
|
|
627
|
-
});
|
|
628
|
-
}
|
|
629
|
-
return items;
|
|
630
|
-
};
|
|
631
|
-
|
|
632
|
-
const listOutboxDraftsWithAttachments: StorageService["listOutboxDraftsWithAttachments"] =
|
|
633
|
-
async (accountConfigId, accountId) => {
|
|
634
|
-
const root = `mock://accounts/${accountConfigId}/${accountId}/outbox/`;
|
|
635
|
-
const drafts = new Set<string>();
|
|
636
|
-
for (const uri of storage.keys()) {
|
|
637
|
-
if (!uri.startsWith(root)) continue;
|
|
638
|
-
const rest = uri.slice(root.length);
|
|
639
|
-
const slash = rest.indexOf("/");
|
|
640
|
-
if (slash > 0) drafts.add(rest.slice(0, slash));
|
|
641
|
-
}
|
|
642
|
-
return [...drafts];
|
|
643
|
-
};
|
|
644
|
-
|
|
645
|
-
const deleteOutboxAttachment: StorageService["deleteOutboxAttachment"] =
|
|
646
|
-
async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
647
|
-
storage.delete(
|
|
648
|
-
`mock://${buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId)}`,
|
|
649
|
-
);
|
|
650
|
-
};
|
|
651
|
-
|
|
652
|
-
const statOutboxAttachment: StorageService["statOutboxAttachment"] = async (
|
|
653
|
-
accountConfigId,
|
|
654
|
-
accountId,
|
|
655
|
-
outboxMessageId,
|
|
656
|
-
outboxAttachmentId,
|
|
657
|
-
) => {
|
|
658
|
-
const content = storage.get(
|
|
659
|
-
`mock://${buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId)}`,
|
|
660
|
-
);
|
|
661
|
-
return content ? { sizeBytes: content.length } : null;
|
|
662
|
-
};
|
|
663
|
-
|
|
664
|
-
const retrieveOutboxAttachment: StorageService["retrieveOutboxAttachment"] =
|
|
665
|
-
async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) =>
|
|
666
|
-
storage.get(
|
|
667
|
-
`mock://${buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId)}`,
|
|
668
|
-
) ?? null;
|
|
669
|
-
|
|
670
|
-
const createOutboxAttachmentUploadUrl: StorageService["createOutboxAttachmentUploadUrl"] =
|
|
671
|
-
async (params) => ({
|
|
672
|
-
uploadUrl: `mock://upload/${buildOutboxAttachmentKey(params.accountConfigId, params.accountId, params.outboxMessageId, params.outboxAttachmentId)}?max=${params.sizeBytes}&exp=${params.expiresAt}`,
|
|
673
|
-
});
|
|
674
|
-
|
|
675
|
-
const deleteOutboxAttachments: StorageService["deleteOutboxAttachments"] =
|
|
676
|
-
async (accountConfigId, accountId, outboxMessageId) => {
|
|
677
|
-
const prefix = `mock://${buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId)}`;
|
|
678
|
-
for (const uri of [...storage.keys()]) {
|
|
679
|
-
if (uri.startsWith(prefix)) storage.delete(uri);
|
|
680
|
-
}
|
|
681
|
-
};
|
|
682
|
-
|
|
683
|
-
const storeDeduplicated: StorageService["storeDeduplicated"] = async (
|
|
684
|
-
params,
|
|
685
|
-
) => {
|
|
686
|
-
const { accountConfigId, accountId, content } = params;
|
|
687
|
-
const checksumSha256 = computeChecksum(content);
|
|
688
|
-
return storeInternal(
|
|
689
|
-
buildDeduplicatedKey(accountConfigId, accountId, checksumSha256),
|
|
690
|
-
content,
|
|
691
|
-
);
|
|
692
|
-
};
|
|
693
|
-
|
|
694
|
-
const storeParsedBody: StorageService["storeParsedBody"] = async (params) => {
|
|
695
|
-
const { accountConfigId, accountId, messageId, parsed } = params;
|
|
696
|
-
const content = Buffer.from(JSON.stringify(parsed), "utf8");
|
|
697
|
-
return storeInternal(
|
|
698
|
-
buildParsedBodyKey(accountConfigId, accountId, messageId),
|
|
699
|
-
content,
|
|
700
|
-
);
|
|
701
|
-
};
|
|
702
|
-
|
|
703
|
-
const retrieveParsedBody: StorageService["retrieveParsedBody"] = async (
|
|
704
|
-
accountConfigId,
|
|
705
|
-
accountId,
|
|
706
|
-
messageId,
|
|
707
|
-
) => {
|
|
708
|
-
const uri = `mock://${buildParsedBodyKey(accountConfigId, accountId, messageId)}`;
|
|
709
|
-
const content = storage.get(uri);
|
|
710
|
-
if (!content) return null;
|
|
711
|
-
return JSON.parse(content.toString("utf8")) as ParsedBody;
|
|
712
|
-
};
|
|
713
|
-
|
|
714
|
-
const retrieveBodyPart: StorageService["retrieveBodyPart"] = async (
|
|
715
|
-
accountConfigId,
|
|
716
|
-
accountId,
|
|
717
|
-
messageId,
|
|
718
|
-
partPath,
|
|
719
|
-
) => {
|
|
720
|
-
const uri = `mock://${buildBodyPartKey(accountConfigId, accountId, messageId, partPath)}`;
|
|
721
|
-
return storage.get(uri) ?? null;
|
|
722
|
-
};
|
|
723
|
-
|
|
724
|
-
const storeExtractedText: StorageService["storeExtractedText"] = async (
|
|
725
|
-
params,
|
|
726
|
-
) => {
|
|
727
|
-
const { accountConfigId, accountId, messageId, partPath, text } = params;
|
|
728
|
-
return storeInternal(
|
|
729
|
-
buildExtractedTextKey(accountConfigId, accountId, messageId, partPath),
|
|
730
|
-
Buffer.from(text, "utf8"),
|
|
731
|
-
);
|
|
732
|
-
};
|
|
733
|
-
|
|
734
|
-
const storeExtractedSkipped: StorageService["storeExtractedSkipped"] = async (
|
|
735
|
-
params,
|
|
736
|
-
) => {
|
|
737
|
-
const { accountConfigId, accountId, messageId, partPath, marker } = params;
|
|
738
|
-
return storeInternal(
|
|
739
|
-
buildExtractedSkippedKey(accountConfigId, accountId, messageId, partPath),
|
|
740
|
-
Buffer.from(JSON.stringify(marker), "utf8"),
|
|
741
|
-
);
|
|
742
|
-
};
|
|
743
|
-
|
|
744
|
-
const extractedResultExists: StorageService["extractedResultExists"] = async (
|
|
745
|
-
accountConfigId,
|
|
746
|
-
accountId,
|
|
747
|
-
messageId,
|
|
748
|
-
partPath,
|
|
749
|
-
) => {
|
|
750
|
-
const textUri = `mock://${buildExtractedTextKey(accountConfigId, accountId, messageId, partPath)}`;
|
|
751
|
-
const skippedUri = `mock://${buildExtractedSkippedKey(accountConfigId, accountId, messageId, partPath)}`;
|
|
752
|
-
return storage.has(textUri) || storage.has(skippedUri);
|
|
753
|
-
};
|
|
754
|
-
|
|
755
|
-
const retrieveExtractedText: StorageService["retrieveExtractedText"] = async (
|
|
756
|
-
accountConfigId,
|
|
757
|
-
accountId,
|
|
758
|
-
messageId,
|
|
759
|
-
partPath,
|
|
760
|
-
) => {
|
|
761
|
-
const uri = `mock://${buildExtractedTextKey(accountConfigId, accountId, messageId, partPath)}`;
|
|
762
|
-
const content = storage.get(uri);
|
|
763
|
-
if (!content) return null;
|
|
764
|
-
return content.toString("utf8");
|
|
765
|
-
};
|
|
766
|
-
|
|
767
|
-
const listExtractedTexts: StorageService["listExtractedTexts"] = async (
|
|
768
|
-
accountConfigId,
|
|
769
|
-
accountId,
|
|
770
|
-
messageId,
|
|
771
|
-
) => {
|
|
772
|
-
const prefix = `mock://${buildExtractedPrefix(accountConfigId, accountId, messageId)}`;
|
|
773
|
-
const suffix = ".txt.gz";
|
|
774
|
-
const items: ExtractedTextListItem[] = [];
|
|
775
|
-
for (const uri of storage.keys()) {
|
|
776
|
-
if (!uri.startsWith(prefix) || !uri.endsWith(suffix)) continue;
|
|
777
|
-
const key = uri.slice("mock://".length);
|
|
778
|
-
items.push({
|
|
779
|
-
partPath: uri.slice(prefix.length, -suffix.length),
|
|
780
|
-
key,
|
|
781
|
-
});
|
|
782
|
-
}
|
|
783
|
-
return items;
|
|
784
|
-
};
|
|
785
|
-
|
|
786
|
-
const retrieveMessageBody: StorageService["retrieveMessageBody"] = async (
|
|
787
|
-
accountConfigId,
|
|
788
|
-
accountId,
|
|
789
|
-
messageId,
|
|
790
|
-
) => {
|
|
791
|
-
const uri = `mock://${buildMessageBodyKey(accountConfigId, accountId, messageId)}`;
|
|
792
|
-
return storage.get(uri) ?? null;
|
|
793
|
-
};
|
|
794
|
-
|
|
795
|
-
const retrieveMessageBodyStream: StorageService["retrieveMessageBodyStream"] =
|
|
796
|
-
async (accountConfigId, accountId, messageId) => {
|
|
797
|
-
const uri = `mock://${buildMessageBodyKey(accountConfigId, accountId, messageId)}`;
|
|
798
|
-
const content = storage.get(uri);
|
|
799
|
-
if (!content) return null;
|
|
800
|
-
return Readable.from(content);
|
|
801
|
-
};
|
|
802
|
-
|
|
803
|
-
const bodyPartExists: StorageService["bodyPartExists"] = async (
|
|
804
|
-
accountConfigId,
|
|
805
|
-
accountId,
|
|
806
|
-
messageId,
|
|
807
|
-
partPath,
|
|
808
|
-
) => {
|
|
809
|
-
const uri = `mock://${buildBodyPartKey(accountConfigId, accountId, messageId, partPath)}`;
|
|
810
|
-
return storage.has(uri);
|
|
811
|
-
};
|
|
812
|
-
|
|
813
|
-
return {
|
|
814
|
-
storeMessageBody,
|
|
815
|
-
storeMessageBodyStream,
|
|
816
|
-
storeBodyPart,
|
|
817
|
-
bodyPartExists,
|
|
818
|
-
retrieveBodyPart,
|
|
819
|
-
storeOutboxAttachment,
|
|
820
|
-
listOutboxAttachments,
|
|
821
|
-
listOutboxDraftsWithAttachments,
|
|
822
|
-
deleteOutboxAttachments,
|
|
823
|
-
deleteOutboxAttachment,
|
|
824
|
-
statOutboxAttachment,
|
|
825
|
-
retrieveOutboxAttachment,
|
|
826
|
-
createOutboxAttachmentUploadUrl,
|
|
827
|
-
storeDeduplicated,
|
|
828
|
-
storeParsedBody,
|
|
829
|
-
retrieveParsedBody,
|
|
830
|
-
storeExtractedText,
|
|
831
|
-
storeExtractedSkipped,
|
|
832
|
-
extractedResultExists,
|
|
833
|
-
retrieveExtractedText,
|
|
834
|
-
listExtractedTexts,
|
|
835
|
-
retrieveMessageBody,
|
|
836
|
-
retrieveMessageBodyStream,
|
|
837
|
-
retrieve: async (uri) => {
|
|
838
|
-
const content = storage.get(uri);
|
|
839
|
-
if (!content) {
|
|
840
|
-
throw Object.assign(new Error(`Not found: ${uri}`), {
|
|
841
|
-
name: "NoSuchKey",
|
|
842
|
-
});
|
|
843
|
-
}
|
|
844
|
-
return content;
|
|
845
|
-
},
|
|
846
|
-
exists: async (uri) => storage.has(uri),
|
|
847
|
-
delete: async (uri) => {
|
|
848
|
-
storage.delete(uri);
|
|
849
|
-
},
|
|
850
|
-
storeExportArchiveStream: async (
|
|
851
|
-
accountConfigId,
|
|
852
|
-
exportRequestId,
|
|
853
|
-
body,
|
|
854
|
-
) => {
|
|
855
|
-
const key = buildExportArchiveKey(accountConfigId, exportRequestId);
|
|
856
|
-
const chunks: Buffer[] = [];
|
|
857
|
-
for await (const chunk of body) {
|
|
858
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
859
|
-
}
|
|
860
|
-
storage.set(`mock://${key}`, Buffer.concat(chunks));
|
|
861
|
-
return key;
|
|
862
|
-
},
|
|
863
|
-
getPresignedDownloadUrl: async (key: string, _expiresInSeconds: number) =>
|
|
864
|
-
`mock://presigned/${key}`,
|
|
865
|
-
};
|
|
866
|
-
};
|