@remit/storage-service 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backends/filesystem.d.ts +12 -0
- package/dist/backends/filesystem.js +316 -0
- package/dist/backends/s3.d.ts +3 -0
- package/dist/backends/s3.js +413 -0
- package/dist/filesystem.js +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/outbox-sweep.d.ts +33 -0
- package/dist/outbox-sweep.js +37 -0
- package/dist/outbox-upload-url.d.ts +44 -0
- package/{src/outbox-upload-url.ts → dist/outbox-upload-url.js} +20 -50
- package/dist/s3.d.ts +3 -0
- package/dist/s3.js +28 -0
- package/dist/signed-path.d.ts +14 -0
- package/dist/signed-path.js +46 -0
- package/dist/storage.d.ts +255 -0
- package/dist/storage.js +228 -0
- package/dist/uri.d.ts +9 -0
- package/dist/uri.js +28 -0
- package/package.json +51 -46
- package/src/backends/filesystem.ts +0 -624
- package/src/backends/s3.test.ts +0 -601
- package/src/backends/s3.ts +0 -761
- package/src/index.ts +0 -60
- package/src/outbox-attachment-retrieve.test.ts +0 -56
- package/src/outbox-sweep.test.ts +0 -196
- package/src/outbox-sweep.ts +0 -103
- package/src/outbox-upload-url.test.ts +0 -290
- package/src/s3.ts +0 -36
- package/src/signed-path.ts +0 -86
- package/src/storage.test.ts +0 -886
- package/src/storage.ts +0 -866
- package/src/uri.ts +0 -47
- package/tsconfig.json +0 -8
- /package/{src/filesystem.ts → dist/filesystem.d.ts} +0 -0
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { PassThrough } from "node:stream";
|
|
3
|
+
import { pipeline } from "node:stream/promises";
|
|
4
|
+
import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
|
|
5
|
+
import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, } from "@aws-sdk/client-s3";
|
|
6
|
+
import { Upload } from "@aws-sdk/lib-storage";
|
|
7
|
+
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
8
|
+
import { ContentEncoding, StorageType } from "@remit/domain-enums";
|
|
9
|
+
import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
|
|
10
|
+
import { parseStorageUri } from "../uri.js";
|
|
11
|
+
export const createS3StorageService = (client, bucketName) => {
|
|
12
|
+
const storeInternal = async (params) => {
|
|
13
|
+
const { key, content, contentType, compress = true } = params;
|
|
14
|
+
const contentChecksumSha256 = computeChecksum(content);
|
|
15
|
+
const contentEncoding = compress
|
|
16
|
+
? ContentEncoding.Gzip
|
|
17
|
+
: ContentEncoding.None;
|
|
18
|
+
const body = compress ? gzipSync(content) : content;
|
|
19
|
+
await client.send(new PutObjectCommand({
|
|
20
|
+
Bucket: bucketName,
|
|
21
|
+
Key: key,
|
|
22
|
+
Body: body,
|
|
23
|
+
ContentType: contentType,
|
|
24
|
+
ContentEncoding: compress ? "gzip" : undefined,
|
|
25
|
+
}));
|
|
26
|
+
return {
|
|
27
|
+
uri: `s3://${bucketName}/${key}`,
|
|
28
|
+
storageType: StorageType.S3,
|
|
29
|
+
storageLocation: bucketName,
|
|
30
|
+
storageKey: key,
|
|
31
|
+
sizeBytes: body.length,
|
|
32
|
+
checksumSha256: contentChecksumSha256,
|
|
33
|
+
contentEncoding,
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
const storeMessageBody = (params) => {
|
|
37
|
+
const { accountConfigId, accountId, messageId, content } = params;
|
|
38
|
+
return storeInternal({
|
|
39
|
+
key: buildMessageBodyKey(accountConfigId, accountId, messageId),
|
|
40
|
+
content,
|
|
41
|
+
contentType: "message/rfc822",
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
const storeMessageBodyStream = async (params) => {
|
|
45
|
+
const { accountConfigId, accountId, messageId, content } = params;
|
|
46
|
+
const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
|
|
47
|
+
// Hash the logical (pre-gzip) bytes as they flow, gzip them, and feed
|
|
48
|
+
// the gzipped stream to a multipart Upload — the whole body is never
|
|
49
|
+
// held in memory. `Upload` buffers only one part at a time.
|
|
50
|
+
const hash = createHash("sha256");
|
|
51
|
+
const hashTap = new PassThrough();
|
|
52
|
+
hashTap.on("data", (chunk) => hash.update(chunk));
|
|
53
|
+
const gzip = createGzip();
|
|
54
|
+
const uploadBody = new PassThrough();
|
|
55
|
+
let storedBytes = 0;
|
|
56
|
+
uploadBody.on("data", (chunk) => {
|
|
57
|
+
storedBytes += chunk.length;
|
|
58
|
+
});
|
|
59
|
+
const upload = new Upload({
|
|
60
|
+
client,
|
|
61
|
+
params: {
|
|
62
|
+
Bucket: bucketName,
|
|
63
|
+
Key: key,
|
|
64
|
+
Body: uploadBody,
|
|
65
|
+
ContentType: "message/rfc822",
|
|
66
|
+
ContentEncoding: "gzip",
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
const pumped = pipeline(content, hashTap, gzip, uploadBody);
|
|
70
|
+
await Promise.all([upload.done(), pumped]).catch(async (error) => {
|
|
71
|
+
// Abort the multipart upload so a failed pump doesn't leak parts.
|
|
72
|
+
await upload.abort().catch(() => { });
|
|
73
|
+
throw error;
|
|
74
|
+
});
|
|
75
|
+
return {
|
|
76
|
+
uri: `s3://${bucketName}/${key}`,
|
|
77
|
+
storageType: StorageType.S3,
|
|
78
|
+
storageLocation: bucketName,
|
|
79
|
+
storageKey: key,
|
|
80
|
+
sizeBytes: storedBytes,
|
|
81
|
+
checksumSha256: hash.digest("hex"),
|
|
82
|
+
contentEncoding: ContentEncoding.Gzip,
|
|
83
|
+
};
|
|
84
|
+
};
|
|
85
|
+
const storeBodyPart = (params) => {
|
|
86
|
+
const { accountConfigId, accountId, messageId, partPath, content, contentType, } = params;
|
|
87
|
+
return storeInternal({
|
|
88
|
+
key: buildBodyPartKey(accountConfigId, accountId, messageId, partPath),
|
|
89
|
+
content,
|
|
90
|
+
contentType,
|
|
91
|
+
});
|
|
92
|
+
};
|
|
93
|
+
const bodyPartExists = (accountConfigId, accountId, messageId, partPath) => {
|
|
94
|
+
const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
|
|
95
|
+
return client
|
|
96
|
+
.send(new HeadObjectCommand({ Bucket: bucketName, Key: key }))
|
|
97
|
+
.then(() => true)
|
|
98
|
+
.catch(() => false);
|
|
99
|
+
};
|
|
100
|
+
const retrieveBodyPart = async (accountConfigId, accountId, messageId, partPath) => {
|
|
101
|
+
const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
|
|
102
|
+
const response = await client
|
|
103
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
104
|
+
.catch((error) => {
|
|
105
|
+
if (isStorageNotFoundError(error))
|
|
106
|
+
return null;
|
|
107
|
+
throw error;
|
|
108
|
+
});
|
|
109
|
+
if (!response)
|
|
110
|
+
return null;
|
|
111
|
+
if (!response.Body) {
|
|
112
|
+
throw new Error(`Empty response body for body part: ${key}`);
|
|
113
|
+
}
|
|
114
|
+
const body = await response.Body.transformToByteArray();
|
|
115
|
+
const buffer = Buffer.from(body);
|
|
116
|
+
return response.ContentEncoding === "gzip" ? gunzipSync(buffer) : buffer;
|
|
117
|
+
};
|
|
118
|
+
const storeOutboxAttachment = (params) => {
|
|
119
|
+
const { accountConfigId, accountId, outboxMessageId, outboxAttachmentId, content, } = params;
|
|
120
|
+
// `application/octet-stream` rather than the uploader's declared type:
|
|
121
|
+
// the object is only ever read back to build a MIME part, and a stored
|
|
122
|
+
// `text/html` would be a type any future signed-URL read path could be
|
|
123
|
+
// tricked into serving from the content origin.
|
|
124
|
+
return storeInternal({
|
|
125
|
+
key: buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId),
|
|
126
|
+
content,
|
|
127
|
+
contentType: "application/octet-stream",
|
|
128
|
+
compress: false,
|
|
129
|
+
});
|
|
130
|
+
};
|
|
131
|
+
const listOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
132
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
133
|
+
const items = [];
|
|
134
|
+
let continuationToken;
|
|
135
|
+
do {
|
|
136
|
+
const response = await client.send(new ListObjectsV2Command({
|
|
137
|
+
Bucket: bucketName,
|
|
138
|
+
Prefix: prefix,
|
|
139
|
+
ContinuationToken: continuationToken,
|
|
140
|
+
}));
|
|
141
|
+
for (const object of response.Contents ?? []) {
|
|
142
|
+
if (!object.Key)
|
|
143
|
+
continue;
|
|
144
|
+
items.push({
|
|
145
|
+
outboxAttachmentId: object.Key.slice(prefix.length),
|
|
146
|
+
key: object.Key,
|
|
147
|
+
sizeBytes: object.Size ?? 0,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
continuationToken = response.IsTruncated
|
|
151
|
+
? response.NextContinuationToken
|
|
152
|
+
: undefined;
|
|
153
|
+
} while (continuationToken);
|
|
154
|
+
return items;
|
|
155
|
+
};
|
|
156
|
+
const listOutboxDraftsWithAttachments = async (accountConfigId, accountId) => {
|
|
157
|
+
const root = `accounts/${accountConfigId}/${accountId}/outbox/`;
|
|
158
|
+
const drafts = new Set();
|
|
159
|
+
let continuationToken;
|
|
160
|
+
// Delimited listing: S3 answers with the directory-like prefixes rather
|
|
161
|
+
// than every object under them, so this stays one call per page of
|
|
162
|
+
// drafts instead of one per object.
|
|
163
|
+
do {
|
|
164
|
+
const response = await client.send(new ListObjectsV2Command({
|
|
165
|
+
Bucket: bucketName,
|
|
166
|
+
Prefix: root,
|
|
167
|
+
Delimiter: "/",
|
|
168
|
+
ContinuationToken: continuationToken,
|
|
169
|
+
}));
|
|
170
|
+
for (const entry of response.CommonPrefixes ?? []) {
|
|
171
|
+
if (!entry.Prefix)
|
|
172
|
+
continue;
|
|
173
|
+
drafts.add(entry.Prefix.slice(root.length).replace(/\/$/, ""));
|
|
174
|
+
}
|
|
175
|
+
continuationToken = response.IsTruncated
|
|
176
|
+
? response.NextContinuationToken
|
|
177
|
+
: undefined;
|
|
178
|
+
} while (continuationToken);
|
|
179
|
+
return [...drafts];
|
|
180
|
+
};
|
|
181
|
+
const deleteOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
182
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
183
|
+
await client.send(new DeleteObjectCommand({
|
|
184
|
+
Bucket: bucketName,
|
|
185
|
+
Key: `${prefix}${outboxAttachmentId}`,
|
|
186
|
+
}));
|
|
187
|
+
};
|
|
188
|
+
const statOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
189
|
+
const head = await client
|
|
190
|
+
.send(new HeadObjectCommand({
|
|
191
|
+
Bucket: bucketName,
|
|
192
|
+
Key: buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId),
|
|
193
|
+
}))
|
|
194
|
+
.catch((error) => {
|
|
195
|
+
if (isStorageNotFoundError(error))
|
|
196
|
+
return null;
|
|
197
|
+
// A HEAD on a missing key answers 404 with no NoSuchKey code, so the
|
|
198
|
+
// SDK's NotFound is the same absence.
|
|
199
|
+
if (error?.name === "NotFound")
|
|
200
|
+
return null;
|
|
201
|
+
throw error;
|
|
202
|
+
});
|
|
203
|
+
if (!head)
|
|
204
|
+
return null;
|
|
205
|
+
return { sizeBytes: head.ContentLength ?? 0 };
|
|
206
|
+
};
|
|
207
|
+
const retrieveOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
208
|
+
const key = buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId);
|
|
209
|
+
const response = await client
|
|
210
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
211
|
+
.catch((error) => {
|
|
212
|
+
if (isStorageNotFoundError(error))
|
|
213
|
+
return null;
|
|
214
|
+
throw error;
|
|
215
|
+
});
|
|
216
|
+
if (!response)
|
|
217
|
+
return null;
|
|
218
|
+
if (!response.Body) {
|
|
219
|
+
throw new Error(`Empty response body for outbox attachment: ${key}`);
|
|
220
|
+
}
|
|
221
|
+
const bytes = await response.Body.transformToByteArray();
|
|
222
|
+
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
223
|
+
};
|
|
224
|
+
// The browser PUTs straight to the bucket and these bytes never touch the
|
|
225
|
+
// API. `ContentLength` on the command hoists `content-length` into
|
|
226
|
+
// X-Amz-SignedHeaders, so S3 recomputes the signature over the length it was
|
|
227
|
+
// sent: a request declaring any other size fails the signature check, and a
|
|
228
|
+
// PUT is read for exactly that many bytes. The size is bound by S3 itself,
|
|
229
|
+
// not by anything downstream trusting the client.
|
|
230
|
+
const createOutboxAttachmentUploadUrl = async (params) => {
|
|
231
|
+
const expiresIn = Math.max(1, params.expiresAt - Math.floor(Date.now() / 1000));
|
|
232
|
+
const uploadUrl = await getSignedUrl(client, new PutObjectCommand({
|
|
233
|
+
Bucket: bucketName,
|
|
234
|
+
Key: buildOutboxAttachmentKey(params.accountConfigId, params.accountId, params.outboxMessageId, params.outboxAttachmentId),
|
|
235
|
+
ContentLength: params.sizeBytes,
|
|
236
|
+
}), { expiresIn });
|
|
237
|
+
return { uploadUrl };
|
|
238
|
+
};
|
|
239
|
+
const deleteOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
240
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
241
|
+
let continuationToken;
|
|
242
|
+
do {
|
|
243
|
+
const response = await client.send(new ListObjectsV2Command({
|
|
244
|
+
Bucket: bucketName,
|
|
245
|
+
Prefix: prefix,
|
|
246
|
+
ContinuationToken: continuationToken,
|
|
247
|
+
}));
|
|
248
|
+
const keys = (response.Contents ?? [])
|
|
249
|
+
.map((object) => object.Key)
|
|
250
|
+
.filter((key) => typeof key === "string");
|
|
251
|
+
if (keys.length > 0) {
|
|
252
|
+
await client.send(new DeleteObjectsCommand({
|
|
253
|
+
Bucket: bucketName,
|
|
254
|
+
Delete: { Objects: keys.map((Key) => ({ Key })) },
|
|
255
|
+
}));
|
|
256
|
+
}
|
|
257
|
+
continuationToken = response.IsTruncated
|
|
258
|
+
? response.NextContinuationToken
|
|
259
|
+
: undefined;
|
|
260
|
+
} while (continuationToken);
|
|
261
|
+
};
|
|
262
|
+
const storeDeduplicated = (params) => {
|
|
263
|
+
const { accountConfigId, accountId, content, contentType } = params;
|
|
264
|
+
const checksumSha256 = computeChecksum(content);
|
|
265
|
+
return storeInternal({
|
|
266
|
+
key: buildDeduplicatedKey(accountConfigId, accountId, checksumSha256),
|
|
267
|
+
content,
|
|
268
|
+
contentType,
|
|
269
|
+
});
|
|
270
|
+
};
|
|
271
|
+
const storeParsedBody = (params) => {
|
|
272
|
+
const { accountConfigId, accountId, messageId, parsed } = params;
|
|
273
|
+
const content = Buffer.from(JSON.stringify(parsed), "utf8");
|
|
274
|
+
return storeInternal({
|
|
275
|
+
key: buildParsedBodyKey(accountConfigId, accountId, messageId),
|
|
276
|
+
content,
|
|
277
|
+
contentType: "application/json",
|
|
278
|
+
});
|
|
279
|
+
};
|
|
280
|
+
const retrieveParsedBody = async (accountConfigId, accountId, messageId) => {
|
|
281
|
+
const key = buildParsedBodyKey(accountConfigId, accountId, messageId);
|
|
282
|
+
const response = await client
|
|
283
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
284
|
+
.catch((error) => {
|
|
285
|
+
if (isStorageNotFoundError(error))
|
|
286
|
+
return null;
|
|
287
|
+
throw error;
|
|
288
|
+
});
|
|
289
|
+
if (!response)
|
|
290
|
+
return null;
|
|
291
|
+
if (!response.Body) {
|
|
292
|
+
throw new Error(`Empty response body for parsed body: ${key}`);
|
|
293
|
+
}
|
|
294
|
+
const body = await response.Body.transformToByteArray();
|
|
295
|
+
const buffer = Buffer.from(body);
|
|
296
|
+
const decoded = response.ContentEncoding === "gzip" ? gunzipSync(buffer) : buffer;
|
|
297
|
+
return JSON.parse(decoded.toString("utf8"));
|
|
298
|
+
};
|
|
299
|
+
const retrieve = async (uri) => {
|
|
300
|
+
const { storageKey } = parseStorageUri(uri);
|
|
301
|
+
const response = await client.send(new GetObjectCommand({
|
|
302
|
+
Bucket: bucketName,
|
|
303
|
+
Key: storageKey,
|
|
304
|
+
}));
|
|
305
|
+
if (!response.Body) {
|
|
306
|
+
throw new Error(`Empty response body for: ${uri}`);
|
|
307
|
+
}
|
|
308
|
+
const body = await response.Body.transformToByteArray();
|
|
309
|
+
const buffer = Buffer.from(body);
|
|
310
|
+
if (response.ContentEncoding === "gzip") {
|
|
311
|
+
return gunzipSync(buffer);
|
|
312
|
+
}
|
|
313
|
+
return buffer;
|
|
314
|
+
};
|
|
315
|
+
const retrieveMessageBody = async (accountConfigId, accountId, messageId) => {
|
|
316
|
+
const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
|
|
317
|
+
const response = await client
|
|
318
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
319
|
+
.catch((error) => {
|
|
320
|
+
if (isStorageNotFoundError(error))
|
|
321
|
+
return null;
|
|
322
|
+
throw error;
|
|
323
|
+
});
|
|
324
|
+
if (!response)
|
|
325
|
+
return null;
|
|
326
|
+
if (!response.Body) {
|
|
327
|
+
throw new Error(`Empty response body for message body: ${key}`);
|
|
328
|
+
}
|
|
329
|
+
const body = await response.Body.transformToByteArray();
|
|
330
|
+
const buffer = Buffer.from(body);
|
|
331
|
+
return response.ContentEncoding === "gzip" ? gunzipSync(buffer) : buffer;
|
|
332
|
+
};
|
|
333
|
+
const retrieveMessageBodyStream = async (accountConfigId, accountId, messageId) => {
|
|
334
|
+
const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
|
|
335
|
+
const response = await client
|
|
336
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
337
|
+
.catch((error) => {
|
|
338
|
+
if (isStorageNotFoundError(error))
|
|
339
|
+
return null;
|
|
340
|
+
throw error;
|
|
341
|
+
});
|
|
342
|
+
if (!response)
|
|
343
|
+
return null;
|
|
344
|
+
if (!response.Body) {
|
|
345
|
+
throw new Error(`Empty response body for message body: ${key}`);
|
|
346
|
+
}
|
|
347
|
+
const source = response.Body;
|
|
348
|
+
if (response.ContentEncoding === "gzip") {
|
|
349
|
+
return source.pipe(createGunzip());
|
|
350
|
+
}
|
|
351
|
+
return source;
|
|
352
|
+
};
|
|
353
|
+
const exists = async (uri) => {
|
|
354
|
+
const { storageKey } = parseStorageUri(uri);
|
|
355
|
+
return client
|
|
356
|
+
.send(new HeadObjectCommand({
|
|
357
|
+
Bucket: bucketName,
|
|
358
|
+
Key: storageKey,
|
|
359
|
+
}))
|
|
360
|
+
.then(() => true)
|
|
361
|
+
.catch(() => false);
|
|
362
|
+
};
|
|
363
|
+
const del = async (uri) => {
|
|
364
|
+
const { storageKey } = parseStorageUri(uri);
|
|
365
|
+
await client.send(new DeleteObjectCommand({
|
|
366
|
+
Bucket: bucketName,
|
|
367
|
+
Key: storageKey,
|
|
368
|
+
}));
|
|
369
|
+
};
|
|
370
|
+
const storeExportArchiveStream = async (accountConfigId, exportRequestId, body) => {
|
|
371
|
+
const key = buildExportArchiveKey(accountConfigId, exportRequestId);
|
|
372
|
+
const upload = new Upload({
|
|
373
|
+
client,
|
|
374
|
+
params: {
|
|
375
|
+
Bucket: bucketName,
|
|
376
|
+
Key: key,
|
|
377
|
+
Body: body,
|
|
378
|
+
ContentType: "application/zip",
|
|
379
|
+
},
|
|
380
|
+
});
|
|
381
|
+
await upload.done().catch(async (error) => {
|
|
382
|
+
await upload.abort().catch(() => { });
|
|
383
|
+
throw error;
|
|
384
|
+
});
|
|
385
|
+
return key;
|
|
386
|
+
};
|
|
387
|
+
const getPresignedDownloadUrl = (key, expiresInSeconds) => getSignedUrl(client, new GetObjectCommand({ Bucket: bucketName, Key: key }), { expiresIn: expiresInSeconds });
|
|
388
|
+
return {
|
|
389
|
+
storeMessageBody,
|
|
390
|
+
storeMessageBodyStream,
|
|
391
|
+
storeBodyPart,
|
|
392
|
+
bodyPartExists,
|
|
393
|
+
retrieveBodyPart,
|
|
394
|
+
storeOutboxAttachment,
|
|
395
|
+
listOutboxAttachments,
|
|
396
|
+
listOutboxDraftsWithAttachments,
|
|
397
|
+
deleteOutboxAttachments,
|
|
398
|
+
deleteOutboxAttachment,
|
|
399
|
+
statOutboxAttachment,
|
|
400
|
+
retrieveOutboxAttachment,
|
|
401
|
+
createOutboxAttachmentUploadUrl,
|
|
402
|
+
storeDeduplicated,
|
|
403
|
+
storeParsedBody,
|
|
404
|
+
retrieveParsedBody,
|
|
405
|
+
retrieveMessageBody,
|
|
406
|
+
retrieveMessageBodyStream,
|
|
407
|
+
retrieve,
|
|
408
|
+
exists,
|
|
409
|
+
delete: del,
|
|
410
|
+
storeExportArchiveStream,
|
|
411
|
+
getPresignedDownloadUrl,
|
|
412
|
+
};
|
|
413
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { createFilesystemStorageService } from "./backends/filesystem.js";
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { type OutboxSweepResult, sweepAbandonedOutboxAttachments, } from "./outbox-sweep.js";
|
|
2
|
+
export { type AuthorizeUploadInput, type AuthorizeUploadResult, authorizeUploadRequest, UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, UPLOAD_URL_TTL_SECONDS, } from "./outbox-upload-url.js";
|
|
3
|
+
export { type SignedPathFailure, type SignedPathResult, signStoragePath, verifyStoragePath, } from "./signed-path.js";
|
|
4
|
+
export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, type ContentEncodingValue, type CreateOutboxAttachmentUploadUrlParams, computeChecksum, createMockStorageService, isStorageNotFoundError, type OutboxAttachmentListItem, type OutboxAttachmentUploadTarget, type ParsedAttachmentMeta, type ParsedBody, type ParsedContentStorageKey, type ParsedOutboxAttachmentKey, parseContentStorageKey, parseOutboxAttachmentKey, type StorageReference, type StorageService, type StorageTypeValue, type StoreBodyPartParams, type StoreDeduplicatedParams, type StoreMessageBodyParams, type StoreOutboxAttachmentParams, type StoreParsedBodyParams, } from "./storage.js";
|
|
5
|
+
export { buildStorageUri, type ParsedStorageUri, parseStorageUri, } from "./uri.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { sweepAbandonedOutboxAttachments, } from "./outbox-sweep.js";
|
|
2
|
+
export { authorizeUploadRequest, UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, UPLOAD_URL_TTL_SECONDS, } from "./outbox-upload-url.js";
|
|
3
|
+
export { signStoragePath, verifyStoragePath, } from "./signed-path.js";
|
|
4
|
+
export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, createMockStorageService, isStorageNotFoundError, parseContentStorageKey, parseOutboxAttachmentKey, } from "./storage.js";
|
|
5
|
+
export { buildStorageUri, parseStorageUri, } from "./uri.js";
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { StorageService } from "./storage.js";
|
|
2
|
+
/**
|
|
3
|
+
* Collect attachment bytes the database does not know about.
|
|
4
|
+
*
|
|
5
|
+
* A presigned upload URL stays valid for its whole window, including after the
|
|
6
|
+
* draft it was minted for is discarded, and on a deployment where the browser
|
|
7
|
+
* PUTs straight to block storage there is nothing in that path to consult. So
|
|
8
|
+
* an object can exist that no row names. That is the only case this exists for,
|
|
9
|
+
* and the rule is the whole of it: an object survives if the database still has
|
|
10
|
+
* an attachment row for it.
|
|
11
|
+
*
|
|
12
|
+
* Two orderings matter and both are deliberate. Objects are listed *before* the
|
|
13
|
+
* rows are read, so an attachment reserved between the two reads is seen as a
|
|
14
|
+
* row and not as an object, and is kept. And a draft whose row lookup fails is
|
|
15
|
+
* skipped entirely — "I could not find out what is live" must never be acted on
|
|
16
|
+
* as "nothing is live".
|
|
17
|
+
*/
|
|
18
|
+
export interface OutboxSweepResult {
|
|
19
|
+
inspected: number;
|
|
20
|
+
deleted: number;
|
|
21
|
+
/** Drafts skipped because their attachment rows could not be read. */
|
|
22
|
+
skipped: number;
|
|
23
|
+
}
|
|
24
|
+
export interface OutboxSweepDeps {
|
|
25
|
+
storage: StorageService;
|
|
26
|
+
/**
|
|
27
|
+
* The attachment ids the database holds for a draft. Throwing is the right
|
|
28
|
+
* way to say "unknown" — the sweep then leaves that draft alone.
|
|
29
|
+
*/
|
|
30
|
+
listKnownAttachmentIds: (accountConfigId: string, outboxMessageId: string) => Promise<string[]>;
|
|
31
|
+
onSkipped: (outboxMessageId: string, error: unknown) => void;
|
|
32
|
+
}
|
|
33
|
+
export declare const sweepAbandonedOutboxAttachments: (deps: OutboxSweepDeps, accountConfigId: string, accountId: string) => Promise<OutboxSweepResult>;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export const sweepAbandonedOutboxAttachments = async (deps, accountConfigId, accountId) => {
|
|
2
|
+
let inspected = 0;
|
|
3
|
+
let deleted = 0;
|
|
4
|
+
let skipped = 0;
|
|
5
|
+
const outboxMessageIds = await deps.storage.listOutboxDraftsWithAttachments(accountConfigId, accountId);
|
|
6
|
+
for (const outboxMessageId of outboxMessageIds) {
|
|
7
|
+
const objects = await deps.storage.listOutboxAttachments(accountConfigId, accountId, outboxMessageId);
|
|
8
|
+
if (objects.length === 0)
|
|
9
|
+
continue;
|
|
10
|
+
const known = await deps
|
|
11
|
+
.listKnownAttachmentIds(accountConfigId, outboxMessageId)
|
|
12
|
+
.then((ids) => new Set(ids))
|
|
13
|
+
.catch((error) => {
|
|
14
|
+
deps.onSkipped(outboxMessageId, error);
|
|
15
|
+
return null;
|
|
16
|
+
});
|
|
17
|
+
if (known === null) {
|
|
18
|
+
skipped += 1;
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
inspected += objects.length;
|
|
22
|
+
const abandoned = objects.filter((object) => !known.has(object.outboxAttachmentId));
|
|
23
|
+
// A draft whose every object is abandoned is the common case — it was
|
|
24
|
+
// discarded — and taking the prefix in one call beats one round trip per
|
|
25
|
+
// object.
|
|
26
|
+
if (abandoned.length === objects.length) {
|
|
27
|
+
await deps.storage.deleteOutboxAttachments(accountConfigId, accountId, outboxMessageId);
|
|
28
|
+
deleted += abandoned.length;
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
for (const object of abandoned) {
|
|
32
|
+
await deps.storage.deleteOutboxAttachment(accountConfigId, accountId, outboxMessageId, object.outboxAttachmentId);
|
|
33
|
+
deleted += 1;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return { inspected, deleted, skipped };
|
|
37
|
+
};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The self-hosted half of the upload seam: what a minted URL looks like when
|
|
3
|
+
* there is no block storage to presign against, and how the route that receives
|
|
4
|
+
* the bytes decides to accept them.
|
|
5
|
+
*
|
|
6
|
+
* A hosted deployment mints a presigned PUT instead and none of this runs — the
|
|
7
|
+
* bytes never reach the deployment at all. Both halves bind the same two
|
|
8
|
+
* things, an expiry and an exact byte count, so a client sees one contract.
|
|
9
|
+
*/
|
|
10
|
+
/** Domain separation: a read grant must never be presentable as a write grant. */
|
|
11
|
+
export declare const UPLOAD_URL_SIGNING_LABEL = "remit-outbox-upload-url-signing-v1";
|
|
12
|
+
export declare const UPLOAD_ROUTE_PREFIX = "/outbox-upload/";
|
|
13
|
+
/**
|
|
14
|
+
* How long a mint's URL, and the reservation behind it, stay good. Long enough
|
|
15
|
+
* for a slow phone to push 25 MB, short enough that a draft abandoned mid-upload
|
|
16
|
+
* stops holding room within the same session.
|
|
17
|
+
*/
|
|
18
|
+
export declare const UPLOAD_URL_TTL_SECONDS = 900;
|
|
19
|
+
export interface AuthorizeUploadInput {
|
|
20
|
+
secret: string | undefined;
|
|
21
|
+
/** Decoded storage key the signature was minted over. */
|
|
22
|
+
relativePath: string;
|
|
23
|
+
exp: string | undefined;
|
|
24
|
+
max: string | undefined;
|
|
25
|
+
sig: string | undefined;
|
|
26
|
+
nowSeconds: number;
|
|
27
|
+
}
|
|
28
|
+
export type AuthorizeUploadResult = {
|
|
29
|
+
authorized: true;
|
|
30
|
+
maxBytes: number;
|
|
31
|
+
} | {
|
|
32
|
+
authorized: false;
|
|
33
|
+
status: number;
|
|
34
|
+
reason: string;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Decide whether an upload request may write to the key it names.
|
|
38
|
+
*
|
|
39
|
+
* Pure, so the decision is testable without a live server. Mirrors
|
|
40
|
+
* `authorizeContentRequest` on the read side, with one addition: the byte count
|
|
41
|
+
* is part of the signed message, so the caller learns from the signature how
|
|
42
|
+
* much this URL is allowed to carry and can cut the stream off there.
|
|
43
|
+
*/
|
|
44
|
+
export declare const authorizeUploadRequest: (input: AuthorizeUploadInput) => AuthorizeUploadResult;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { verifyStoragePath } from "./signed-path.js";
|
|
2
|
-
|
|
3
2
|
/**
|
|
4
3
|
* The self-hosted half of the upload seam: what a minted URL looks like when
|
|
5
4
|
* there is no block storage to presign against, and how the route that receives
|
|
@@ -9,33 +8,15 @@ import { verifyStoragePath } from "./signed-path.js";
|
|
|
9
8
|
* bytes never reach the deployment at all. Both halves bind the same two
|
|
10
9
|
* things, an expiry and an exact byte count, so a client sees one contract.
|
|
11
10
|
*/
|
|
12
|
-
|
|
13
11
|
/** Domain separation: a read grant must never be presentable as a write grant. */
|
|
14
12
|
export const UPLOAD_URL_SIGNING_LABEL = "remit-outbox-upload-url-signing-v1";
|
|
15
|
-
|
|
16
13
|
export const UPLOAD_ROUTE_PREFIX = "/outbox-upload/";
|
|
17
|
-
|
|
18
14
|
/**
|
|
19
15
|
* How long a mint's URL, and the reservation behind it, stay good. Long enough
|
|
20
16
|
* for a slow phone to push 25 MB, short enough that a draft abandoned mid-upload
|
|
21
17
|
* stops holding room within the same session.
|
|
22
18
|
*/
|
|
23
19
|
export const UPLOAD_URL_TTL_SECONDS = 900;
|
|
24
|
-
|
|
25
|
-
export interface AuthorizeUploadInput {
|
|
26
|
-
secret: string | undefined;
|
|
27
|
-
/** Decoded storage key the signature was minted over. */
|
|
28
|
-
relativePath: string;
|
|
29
|
-
exp: string | undefined;
|
|
30
|
-
max: string | undefined;
|
|
31
|
-
sig: string | undefined;
|
|
32
|
-
nowSeconds: number;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export type AuthorizeUploadResult =
|
|
36
|
-
| { authorized: true; maxBytes: number }
|
|
37
|
-
| { authorized: false; status: number; reason: string };
|
|
38
|
-
|
|
39
20
|
/**
|
|
40
21
|
* Decide whether an upload request may write to the key it names.
|
|
41
22
|
*
|
|
@@ -44,35 +25,24 @@ export type AuthorizeUploadResult =
|
|
|
44
25
|
* is part of the signed message, so the caller learns from the signature how
|
|
45
26
|
* much this URL is allowed to carry and can cut the stream off there.
|
|
46
27
|
*/
|
|
47
|
-
export const authorizeUploadRequest = (
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
input.secret,
|
|
68
|
-
UPLOAD_URL_SIGNING_LABEL,
|
|
69
|
-
input.relativePath,
|
|
70
|
-
[exp, maxBytes],
|
|
71
|
-
input.sig,
|
|
72
|
-
input.nowSeconds,
|
|
73
|
-
);
|
|
74
|
-
if (result.valid) return { authorized: true, maxBytes };
|
|
75
|
-
|
|
76
|
-
const status = result.reason === "missing" ? 401 : 403;
|
|
77
|
-
return { authorized: false, status, reason: result.reason };
|
|
28
|
+
export const authorizeUploadRequest = (input) => {
|
|
29
|
+
if (!input.secret || input.secret.length === 0) {
|
|
30
|
+
// No secret means no upload URL could have been signed, so nothing
|
|
31
|
+
// presented here can be trusted. Fail closed.
|
|
32
|
+
return {
|
|
33
|
+
authorized: false,
|
|
34
|
+
status: 500,
|
|
35
|
+
reason: "upload-signing-not-configured",
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
const exp = Number(input.exp);
|
|
39
|
+
const maxBytes = Number(input.max);
|
|
40
|
+
if (!Number.isInteger(exp) || !Number.isInteger(maxBytes)) {
|
|
41
|
+
return { authorized: false, status: 403, reason: "malformed" };
|
|
42
|
+
}
|
|
43
|
+
const result = verifyStoragePath(input.secret, UPLOAD_URL_SIGNING_LABEL, input.relativePath, [exp, maxBytes], input.sig, input.nowSeconds);
|
|
44
|
+
if (result.valid)
|
|
45
|
+
return { authorized: true, maxBytes };
|
|
46
|
+
const status = result.reason === "missing" ? 401 : 403;
|
|
47
|
+
return { authorized: false, status, reason: result.reason };
|
|
78
48
|
};
|
package/dist/s3.d.ts
ADDED
package/dist/s3.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { S3Client } from "@aws-sdk/client-s3";
|
|
2
|
+
import { createFilesystemStorageService } from "./backends/filesystem.js";
|
|
3
|
+
import { createS3StorageService } from "./backends/s3.js";
|
|
4
|
+
export { createS3StorageService } from "./backends/s3.js";
|
|
5
|
+
export const createStorageService = () => {
|
|
6
|
+
const bucketName = process.env.S3_BUCKET_NAME;
|
|
7
|
+
if (bucketName) {
|
|
8
|
+
const client = new S3Client({
|
|
9
|
+
endpoint: process.env.S3_ENDPOINT,
|
|
10
|
+
// The SDK default (WHEN_SUPPORTED) hangs a CRC32 of an empty body off
|
|
11
|
+
// every presigned PUT as signed query params. Real S3 tolerates it;
|
|
12
|
+
// MinIO and LocalStack reject the request, and S3_ENDPOINT exists to
|
|
13
|
+
// point at exactly those.
|
|
14
|
+
requestChecksumCalculation: "WHEN_REQUIRED",
|
|
15
|
+
});
|
|
16
|
+
return createS3StorageService(client, bucketName);
|
|
17
|
+
}
|
|
18
|
+
const basePath = process.env.STORAGE_LOCAL_PATH ?? ".remit/storage";
|
|
19
|
+
// BETTER_AUTH_URL is the deployment's own public base — the value the token
|
|
20
|
+
// issuer and the JWKS discovery document already agree on — so a minted
|
|
21
|
+
// upload URL resolves from wherever the browser reached the API. Both it and
|
|
22
|
+
// the signing secret are required on the self-host stack; without them a
|
|
23
|
+
// mint fails loud inside the backend rather than handing out a URL that
|
|
24
|
+
// cannot be verified.
|
|
25
|
+
const origin = process.env.BETTER_AUTH_URL;
|
|
26
|
+
const signingSecret = process.env.BETTER_AUTH_SECRET;
|
|
27
|
+
return createFilesystemStorageService(basePath, origin && signingSecret ? { origin, signingSecret } : undefined);
|
|
28
|
+
};
|