@remit/storage-service 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backends/filesystem.d.ts +12 -0
- package/dist/backends/filesystem.js +296 -0
- package/dist/backends/s3.d.ts +3 -0
- package/dist/backends/s3.js +370 -0
- package/dist/filesystem.js +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/outbox-sweep.d.ts +33 -0
- package/dist/outbox-sweep.js +37 -0
- package/dist/outbox-upload-url.d.ts +44 -0
- package/{src/outbox-upload-url.ts → dist/outbox-upload-url.js} +20 -50
- package/dist/s3.d.ts +3 -0
- package/dist/s3.js +28 -0
- package/dist/signed-path.d.ts +14 -0
- package/dist/signed-path.js +46 -0
- package/dist/storage.d.ts +241 -0
- package/dist/storage.js +219 -0
- package/dist/uri.d.ts +9 -0
- package/dist/uri.js +28 -0
- package/package.json +51 -46
- package/src/backends/filesystem.ts +0 -517
- package/src/backends/s3.test.ts +0 -447
- package/src/backends/s3.ts +0 -624
- package/src/index.ts +0 -53
- package/src/outbox-attachment-retrieve.test.ts +0 -56
- package/src/outbox-sweep.test.ts +0 -196
- package/src/outbox-sweep.ts +0 -103
- package/src/outbox-upload-url.test.ts +0 -290
- package/src/s3.ts +0 -36
- package/src/signed-path.ts +0 -86
- package/src/storage.test.ts +0 -646
- package/src/storage.ts +0 -699
- package/src/uri.ts +0 -47
- package/tsconfig.json +0 -8
- /package/{src/filesystem.ts → dist/filesystem.d.ts} +0 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { StorageService } from "../storage.js";
|
|
2
|
+
/**
|
|
3
|
+
* Where a minted upload URL points, and what signs it. Absent on a process that
|
|
4
|
+
* only reads storage; a mint then fails loud rather than handing out a URL
|
|
5
|
+
* nothing can verify.
|
|
6
|
+
*/
|
|
7
|
+
export interface FilesystemUploadUrlConfig {
|
|
8
|
+
/** Public origin of the deployment serving the upload route. */
|
|
9
|
+
origin: string;
|
|
10
|
+
signingSecret: string;
|
|
11
|
+
}
|
|
12
|
+
export declare const createFilesystemStorageService: (basePath: string, uploadUrls?: FilesystemUploadUrlConfig) => StorageService;
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import { createWriteStream } from "node:fs";
|
|
2
|
+
import { mkdir, readdir, readFile, rm, stat, unlink, writeFile, } from "node:fs/promises";
|
|
3
|
+
import { dirname, join, resolve } from "node:path";
|
|
4
|
+
import { Readable } from "node:stream";
|
|
5
|
+
import { pipeline } from "node:stream/promises";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
import { createGunzip, gunzipSync, gzip } from "node:zlib";
|
|
8
|
+
import { ContentEncoding, StorageType } from "@remit/domain-enums";
|
|
9
|
+
import { UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, } from "../outbox-upload-url.js";
|
|
10
|
+
import { signStoragePath } from "../signed-path.js";
|
|
11
|
+
import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
|
|
12
|
+
const gzipAsync = promisify(gzip);
|
|
13
|
+
export const createFilesystemStorageService = (basePath, uploadUrls) => {
|
|
14
|
+
const storeInternal = async (params) => {
|
|
15
|
+
const { key, content, compress = true } = params;
|
|
16
|
+
const checksumSha256 = computeChecksum(content);
|
|
17
|
+
const contentEncoding = compress
|
|
18
|
+
? ContentEncoding.Gzip
|
|
19
|
+
: ContentEncoding.None;
|
|
20
|
+
const body = compress ? await gzipAsync(content) : content;
|
|
21
|
+
const fullPath = join(basePath, key);
|
|
22
|
+
await mkdir(dirname(fullPath), { recursive: true });
|
|
23
|
+
await writeFile(fullPath, body);
|
|
24
|
+
return {
|
|
25
|
+
uri: `file://${fullPath}`,
|
|
26
|
+
storageType: StorageType.Filesystem,
|
|
27
|
+
storageLocation: basePath,
|
|
28
|
+
storageKey: key,
|
|
29
|
+
sizeBytes: body.length,
|
|
30
|
+
checksumSha256,
|
|
31
|
+
contentEncoding,
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
const storeMessageBody = (params) => {
|
|
35
|
+
const { accountConfigId, accountId, messageId, content } = params;
|
|
36
|
+
return storeInternal({
|
|
37
|
+
key: buildMessageBodyKey(accountConfigId, accountId, messageId),
|
|
38
|
+
content,
|
|
39
|
+
});
|
|
40
|
+
};
|
|
41
|
+
const storeBodyPart = (params) => {
|
|
42
|
+
const { accountConfigId, accountId, messageId, partPath, content } = params;
|
|
43
|
+
return storeInternal({
|
|
44
|
+
key: buildBodyPartKey(accountConfigId, accountId, messageId, partPath),
|
|
45
|
+
content,
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
const bodyPartExists = (accountConfigId, accountId, messageId, partPath) => {
|
|
49
|
+
const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
|
|
50
|
+
const fullPath = join(basePath, key);
|
|
51
|
+
return stat(fullPath)
|
|
52
|
+
.then(() => true)
|
|
53
|
+
.catch(() => false);
|
|
54
|
+
};
|
|
55
|
+
const retrieveBodyPart = async (accountConfigId, accountId, messageId, partPath) => {
|
|
56
|
+
const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
|
|
57
|
+
const fullPath = join(basePath, key);
|
|
58
|
+
const buffer = await readFile(fullPath).catch((error) => {
|
|
59
|
+
if (isStorageNotFoundError(error))
|
|
60
|
+
return null;
|
|
61
|
+
throw error;
|
|
62
|
+
});
|
|
63
|
+
if (!buffer)
|
|
64
|
+
return null;
|
|
65
|
+
return buffer[0] === 0x1f && buffer[1] === 0x8b
|
|
66
|
+
? gunzipSync(buffer)
|
|
67
|
+
: buffer;
|
|
68
|
+
};
|
|
69
|
+
const storeOutboxAttachment = (params) => {
|
|
70
|
+
const { accountConfigId, accountId, outboxMessageId, outboxAttachmentId, content, } = params;
|
|
71
|
+
return storeInternal({
|
|
72
|
+
key: buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId),
|
|
73
|
+
content,
|
|
74
|
+
compress: false,
|
|
75
|
+
});
|
|
76
|
+
};
|
|
77
|
+
const listOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
78
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
79
|
+
const entries = await readdir(join(basePath, prefix)).catch((error) => {
|
|
80
|
+
if (isStorageNotFoundError(error))
|
|
81
|
+
return [];
|
|
82
|
+
throw error;
|
|
83
|
+
});
|
|
84
|
+
const items = [];
|
|
85
|
+
for (const name of entries) {
|
|
86
|
+
const { size } = await stat(join(basePath, prefix, name));
|
|
87
|
+
items.push({
|
|
88
|
+
outboxAttachmentId: name,
|
|
89
|
+
key: `${prefix}${name}`,
|
|
90
|
+
sizeBytes: size,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
return items;
|
|
94
|
+
};
|
|
95
|
+
const listOutboxDraftsWithAttachments = async (accountConfigId, accountId) => {
|
|
96
|
+
// An account that has never held an attachment has no directory, and
|
|
97
|
+
// that is genuinely nothing to do. A storage root that is not there at
|
|
98
|
+
// all is a different thing entirely — a process that cannot see the
|
|
99
|
+
// volume it is meant to be collecting from — and it must not read as an
|
|
100
|
+
// empty answer. Unmounted, the sweep would report every account swept
|
|
101
|
+
// and collect nothing, forever.
|
|
102
|
+
const rootExists = await stat(basePath)
|
|
103
|
+
.then(() => true)
|
|
104
|
+
.catch((error) => {
|
|
105
|
+
if (isStorageNotFoundError(error))
|
|
106
|
+
return false;
|
|
107
|
+
throw error;
|
|
108
|
+
});
|
|
109
|
+
if (!rootExists) {
|
|
110
|
+
throw new Error(`storage root ${basePath} does not exist; this process cannot see the storage it was asked to read`);
|
|
111
|
+
}
|
|
112
|
+
const root = join(basePath, `accounts/${accountConfigId}/${accountId}/outbox`);
|
|
113
|
+
const entries = await readdir(root, { withFileTypes: true }).catch((error) => {
|
|
114
|
+
if (isStorageNotFoundError(error))
|
|
115
|
+
return [];
|
|
116
|
+
throw error;
|
|
117
|
+
});
|
|
118
|
+
return entries
|
|
119
|
+
.filter((entry) => entry.isDirectory())
|
|
120
|
+
.map((entry) => entry.name);
|
|
121
|
+
};
|
|
122
|
+
const deleteOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
123
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
124
|
+
const entries = await readdir(join(basePath, prefix)).catch((error) => {
|
|
125
|
+
if (isStorageNotFoundError(error))
|
|
126
|
+
return [];
|
|
127
|
+
throw error;
|
|
128
|
+
});
|
|
129
|
+
for (const name of entries) {
|
|
130
|
+
if (name !== outboxAttachmentId)
|
|
131
|
+
continue;
|
|
132
|
+
await rm(join(basePath, prefix, name), { force: true });
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
const statOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
136
|
+
const fullPath = join(basePath, buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId));
|
|
137
|
+
const stats = await stat(fullPath).catch((error) => {
|
|
138
|
+
if (isStorageNotFoundError(error))
|
|
139
|
+
return null;
|
|
140
|
+
throw error;
|
|
141
|
+
});
|
|
142
|
+
return stats ? { sizeBytes: stats.size } : null;
|
|
143
|
+
};
|
|
144
|
+
const retrieveOutboxAttachment = (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => readFile(join(basePath, buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId))).catch((error) => {
|
|
145
|
+
if (isStorageNotFoundError(error))
|
|
146
|
+
return null;
|
|
147
|
+
throw error;
|
|
148
|
+
});
|
|
149
|
+
// This backend has no presigned anything, so the URL addresses the
|
|
150
|
+
// deployment's own upload route. Its authority is the same HMAC the read side
|
|
151
|
+
// uses on /content, under the write label, covering the storage key, the
|
|
152
|
+
// expiry and the exact byte count — a URL for one attachment, for a while,
|
|
153
|
+
// for one size.
|
|
154
|
+
const createOutboxAttachmentUploadUrl = async (params) => {
|
|
155
|
+
if (!uploadUrls) {
|
|
156
|
+
throw new Error("the filesystem storage backend cannot mint an upload URL without an origin and a signing secret");
|
|
157
|
+
}
|
|
158
|
+
const key = buildOutboxAttachmentKey(params.accountConfigId, params.accountId, params.outboxMessageId, params.outboxAttachmentId);
|
|
159
|
+
const sig = signStoragePath(uploadUrls.signingSecret, UPLOAD_URL_SIGNING_LABEL, key, [params.expiresAt, params.sizeBytes]);
|
|
160
|
+
const origin = uploadUrls.origin.replace(/\/+$/, "");
|
|
161
|
+
const query = new URLSearchParams({
|
|
162
|
+
exp: String(params.expiresAt),
|
|
163
|
+
max: String(params.sizeBytes),
|
|
164
|
+
sig,
|
|
165
|
+
});
|
|
166
|
+
return {
|
|
167
|
+
uploadUrl: `${origin}${UPLOAD_ROUTE_PREFIX}${key}?${query.toString()}`,
|
|
168
|
+
};
|
|
169
|
+
};
|
|
170
|
+
const deleteOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
171
|
+
const dirPath = join(basePath, buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId));
|
|
172
|
+
await rm(dirPath, { recursive: true, force: true });
|
|
173
|
+
};
|
|
174
|
+
const storeDeduplicated = (params) => {
|
|
175
|
+
const { accountConfigId, accountId, content } = params;
|
|
176
|
+
const checksumSha256 = computeChecksum(content);
|
|
177
|
+
return storeInternal({
|
|
178
|
+
key: buildDeduplicatedKey(accountConfigId, accountId, checksumSha256),
|
|
179
|
+
content,
|
|
180
|
+
});
|
|
181
|
+
};
|
|
182
|
+
const storeParsedBody = (params) => {
|
|
183
|
+
const { accountConfigId, accountId, messageId, parsed } = params;
|
|
184
|
+
const content = Buffer.from(JSON.stringify(parsed), "utf8");
|
|
185
|
+
return storeInternal({
|
|
186
|
+
key: buildParsedBodyKey(accountConfigId, accountId, messageId),
|
|
187
|
+
content,
|
|
188
|
+
});
|
|
189
|
+
};
|
|
190
|
+
const retrieveParsedBody = async (accountConfigId, accountId, messageId) => {
|
|
191
|
+
const key = buildParsedBodyKey(accountConfigId, accountId, messageId);
|
|
192
|
+
const fullPath = join(basePath, key);
|
|
193
|
+
const buffer = await readFile(fullPath).catch((error) => {
|
|
194
|
+
if (isStorageNotFoundError(error))
|
|
195
|
+
return null;
|
|
196
|
+
throw error;
|
|
197
|
+
});
|
|
198
|
+
if (!buffer)
|
|
199
|
+
return null;
|
|
200
|
+
const decoded = buffer[0] === 0x1f && buffer[1] === 0x8b ? gunzipSync(buffer) : buffer;
|
|
201
|
+
return JSON.parse(decoded.toString("utf8"));
|
|
202
|
+
};
|
|
203
|
+
// Resolve path from URI, handling both absolute and relative paths
|
|
204
|
+
// Note: URIs like file://.remit/storage/... parse incorrectly - '.remit' becomes hostname
|
|
205
|
+
// We reconstruct the original path by combining hostname + pathname
|
|
206
|
+
const resolvePathFromUri = (uri) => {
|
|
207
|
+
const url = new URL(uri);
|
|
208
|
+
const hostname = url.hostname;
|
|
209
|
+
const pathname = url.pathname;
|
|
210
|
+
// If hostname exists, it was part of a relative path (e.g., file://.remit/... -> hostname='.remit')
|
|
211
|
+
if (hostname) {
|
|
212
|
+
const relativePath = hostname + pathname;
|
|
213
|
+
return resolve(relativePath);
|
|
214
|
+
}
|
|
215
|
+
// Absolute path (file:///absolute/path)
|
|
216
|
+
return pathname;
|
|
217
|
+
};
|
|
218
|
+
const retrieve = async (uri) => {
|
|
219
|
+
const fullPath = resolvePathFromUri(uri);
|
|
220
|
+
const buffer = await readFile(fullPath);
|
|
221
|
+
if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
|
|
222
|
+
return gunzipSync(buffer);
|
|
223
|
+
}
|
|
224
|
+
return buffer;
|
|
225
|
+
};
|
|
226
|
+
const exists = async (uri) => {
|
|
227
|
+
const fullPath = resolvePathFromUri(uri);
|
|
228
|
+
return stat(fullPath)
|
|
229
|
+
.then(() => true)
|
|
230
|
+
.catch(() => false);
|
|
231
|
+
};
|
|
232
|
+
const del = async (uri) => {
|
|
233
|
+
const fullPath = resolvePathFromUri(uri);
|
|
234
|
+
await unlink(fullPath);
|
|
235
|
+
};
|
|
236
|
+
return {
|
|
237
|
+
storeMessageBody,
|
|
238
|
+
storeBodyPart,
|
|
239
|
+
bodyPartExists,
|
|
240
|
+
retrieveBodyPart,
|
|
241
|
+
storeOutboxAttachment,
|
|
242
|
+
listOutboxAttachments,
|
|
243
|
+
listOutboxDraftsWithAttachments,
|
|
244
|
+
deleteOutboxAttachments,
|
|
245
|
+
deleteOutboxAttachment,
|
|
246
|
+
statOutboxAttachment,
|
|
247
|
+
retrieveOutboxAttachment,
|
|
248
|
+
createOutboxAttachmentUploadUrl,
|
|
249
|
+
storeDeduplicated,
|
|
250
|
+
storeParsedBody,
|
|
251
|
+
retrieveParsedBody,
|
|
252
|
+
retrieveMessageBody: async (accountConfigId, accountId, messageId) => {
|
|
253
|
+
const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
|
|
254
|
+
const fullPath = join(basePath, key);
|
|
255
|
+
const buffer = await readFile(fullPath).catch((error) => {
|
|
256
|
+
if (isStorageNotFoundError(error))
|
|
257
|
+
return null;
|
|
258
|
+
throw error;
|
|
259
|
+
});
|
|
260
|
+
if (!buffer)
|
|
261
|
+
return null;
|
|
262
|
+
if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
|
|
263
|
+
return gunzipSync(buffer);
|
|
264
|
+
}
|
|
265
|
+
return buffer;
|
|
266
|
+
},
|
|
267
|
+
retrieveMessageBodyStream: async (accountConfigId, accountId, messageId) => {
|
|
268
|
+
const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
|
|
269
|
+
const fullPath = join(basePath, key);
|
|
270
|
+
const buffer = await readFile(fullPath).catch((error) => {
|
|
271
|
+
if (isStorageNotFoundError(error))
|
|
272
|
+
return null;
|
|
273
|
+
throw error;
|
|
274
|
+
});
|
|
275
|
+
if (!buffer)
|
|
276
|
+
return null;
|
|
277
|
+
if (buffer[0] === 0x1f && buffer[1] === 0x8b) {
|
|
278
|
+
return Readable.from(buffer).pipe(createGunzip());
|
|
279
|
+
}
|
|
280
|
+
return Readable.from(buffer);
|
|
281
|
+
},
|
|
282
|
+
retrieve,
|
|
283
|
+
exists,
|
|
284
|
+
delete: del,
|
|
285
|
+
storeExportArchiveStream: async (accountConfigId, exportRequestId, body) => {
|
|
286
|
+
const key = buildExportArchiveKey(accountConfigId, exportRequestId);
|
|
287
|
+
const fullPath = join(basePath, key);
|
|
288
|
+
await mkdir(dirname(fullPath), { recursive: true });
|
|
289
|
+
await pipeline(body, createWriteStream(fullPath));
|
|
290
|
+
return key;
|
|
291
|
+
},
|
|
292
|
+
getPresignedDownloadUrl: (_key, _expiresInSeconds) => {
|
|
293
|
+
throw new Error("getPresignedDownloadUrl is not supported on the filesystem backend");
|
|
294
|
+
},
|
|
295
|
+
};
|
|
296
|
+
};
|
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
import { promisify } from "node:util";
|
|
2
|
+
import { createGunzip, gunzipSync, gzip } from "node:zlib";
|
|
3
|
+
import { DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, } from "@aws-sdk/client-s3";
|
|
4
|
+
import { Upload } from "@aws-sdk/lib-storage";
|
|
5
|
+
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
6
|
+
import { ContentEncoding, StorageType } from "@remit/domain-enums";
|
|
7
|
+
import { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, isStorageNotFoundError, } from "../storage.js";
|
|
8
|
+
import { parseStorageUri } from "../uri.js";
|
|
9
|
+
const gzipAsync = promisify(gzip);
|
|
10
|
+
export const createS3StorageService = (client, bucketName) => {
|
|
11
|
+
const storeInternal = async (params) => {
|
|
12
|
+
const { key, content, contentType, compress = true } = params;
|
|
13
|
+
const contentChecksumSha256 = computeChecksum(content);
|
|
14
|
+
const contentEncoding = compress
|
|
15
|
+
? ContentEncoding.Gzip
|
|
16
|
+
: ContentEncoding.None;
|
|
17
|
+
const body = compress ? await gzipAsync(content) : content;
|
|
18
|
+
await client.send(new PutObjectCommand({
|
|
19
|
+
Bucket: bucketName,
|
|
20
|
+
Key: key,
|
|
21
|
+
Body: body,
|
|
22
|
+
ContentType: contentType,
|
|
23
|
+
ContentEncoding: compress ? "gzip" : undefined,
|
|
24
|
+
}));
|
|
25
|
+
return {
|
|
26
|
+
uri: `s3://${bucketName}/${key}`,
|
|
27
|
+
storageType: StorageType.S3,
|
|
28
|
+
storageLocation: bucketName,
|
|
29
|
+
storageKey: key,
|
|
30
|
+
sizeBytes: body.length,
|
|
31
|
+
checksumSha256: contentChecksumSha256,
|
|
32
|
+
contentEncoding,
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
const storeMessageBody = (params) => {
|
|
36
|
+
const { accountConfigId, accountId, messageId, content } = params;
|
|
37
|
+
return storeInternal({
|
|
38
|
+
key: buildMessageBodyKey(accountConfigId, accountId, messageId),
|
|
39
|
+
content,
|
|
40
|
+
contentType: "message/rfc822",
|
|
41
|
+
});
|
|
42
|
+
};
|
|
43
|
+
const storeBodyPart = (params) => {
|
|
44
|
+
const { accountConfigId, accountId, messageId, partPath, content, contentType, } = params;
|
|
45
|
+
return storeInternal({
|
|
46
|
+
key: buildBodyPartKey(accountConfigId, accountId, messageId, partPath),
|
|
47
|
+
content,
|
|
48
|
+
contentType,
|
|
49
|
+
});
|
|
50
|
+
};
|
|
51
|
+
const bodyPartExists = (accountConfigId, accountId, messageId, partPath) => {
|
|
52
|
+
const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
|
|
53
|
+
return client
|
|
54
|
+
.send(new HeadObjectCommand({ Bucket: bucketName, Key: key }))
|
|
55
|
+
.then(() => true)
|
|
56
|
+
.catch(() => false);
|
|
57
|
+
};
|
|
58
|
+
const retrieveBodyPart = async (accountConfigId, accountId, messageId, partPath) => {
|
|
59
|
+
const key = buildBodyPartKey(accountConfigId, accountId, messageId, partPath);
|
|
60
|
+
const response = await client
|
|
61
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
62
|
+
.catch((error) => {
|
|
63
|
+
if (isStorageNotFoundError(error))
|
|
64
|
+
return null;
|
|
65
|
+
throw error;
|
|
66
|
+
});
|
|
67
|
+
if (!response)
|
|
68
|
+
return null;
|
|
69
|
+
if (!response.Body) {
|
|
70
|
+
throw new Error(`Empty response body for body part: ${key}`);
|
|
71
|
+
}
|
|
72
|
+
const body = await response.Body.transformToByteArray();
|
|
73
|
+
const buffer = Buffer.from(body);
|
|
74
|
+
return response.ContentEncoding === "gzip" ? gunzipSync(buffer) : buffer;
|
|
75
|
+
};
|
|
76
|
+
const storeOutboxAttachment = (params) => {
|
|
77
|
+
const { accountConfigId, accountId, outboxMessageId, outboxAttachmentId, content, } = params;
|
|
78
|
+
// `application/octet-stream` rather than the uploader's declared type:
|
|
79
|
+
// the object is only ever read back to build a MIME part, and a stored
|
|
80
|
+
// `text/html` would be a type any future signed-URL read path could be
|
|
81
|
+
// tricked into serving from the content origin.
|
|
82
|
+
return storeInternal({
|
|
83
|
+
key: buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId),
|
|
84
|
+
content,
|
|
85
|
+
contentType: "application/octet-stream",
|
|
86
|
+
compress: false,
|
|
87
|
+
});
|
|
88
|
+
};
|
|
89
|
+
const listOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
90
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
91
|
+
const items = [];
|
|
92
|
+
let continuationToken;
|
|
93
|
+
do {
|
|
94
|
+
const response = await client.send(new ListObjectsV2Command({
|
|
95
|
+
Bucket: bucketName,
|
|
96
|
+
Prefix: prefix,
|
|
97
|
+
ContinuationToken: continuationToken,
|
|
98
|
+
}));
|
|
99
|
+
for (const object of response.Contents ?? []) {
|
|
100
|
+
if (!object.Key)
|
|
101
|
+
continue;
|
|
102
|
+
items.push({
|
|
103
|
+
outboxAttachmentId: object.Key.slice(prefix.length),
|
|
104
|
+
key: object.Key,
|
|
105
|
+
sizeBytes: object.Size ?? 0,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
continuationToken = response.IsTruncated
|
|
109
|
+
? response.NextContinuationToken
|
|
110
|
+
: undefined;
|
|
111
|
+
} while (continuationToken);
|
|
112
|
+
return items;
|
|
113
|
+
};
|
|
114
|
+
const listOutboxDraftsWithAttachments = async (accountConfigId, accountId) => {
|
|
115
|
+
const root = `accounts/${accountConfigId}/${accountId}/outbox/`;
|
|
116
|
+
const drafts = new Set();
|
|
117
|
+
let continuationToken;
|
|
118
|
+
// Delimited listing: S3 answers with the directory-like prefixes rather
|
|
119
|
+
// than every object under them, so this stays one call per page of
|
|
120
|
+
// drafts instead of one per object.
|
|
121
|
+
do {
|
|
122
|
+
const response = await client.send(new ListObjectsV2Command({
|
|
123
|
+
Bucket: bucketName,
|
|
124
|
+
Prefix: root,
|
|
125
|
+
Delimiter: "/",
|
|
126
|
+
ContinuationToken: continuationToken,
|
|
127
|
+
}));
|
|
128
|
+
for (const entry of response.CommonPrefixes ?? []) {
|
|
129
|
+
if (!entry.Prefix)
|
|
130
|
+
continue;
|
|
131
|
+
drafts.add(entry.Prefix.slice(root.length).replace(/\/$/, ""));
|
|
132
|
+
}
|
|
133
|
+
continuationToken = response.IsTruncated
|
|
134
|
+
? response.NextContinuationToken
|
|
135
|
+
: undefined;
|
|
136
|
+
} while (continuationToken);
|
|
137
|
+
return [...drafts];
|
|
138
|
+
};
|
|
139
|
+
const deleteOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
140
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
141
|
+
await client.send(new DeleteObjectCommand({
|
|
142
|
+
Bucket: bucketName,
|
|
143
|
+
Key: `${prefix}${outboxAttachmentId}`,
|
|
144
|
+
}));
|
|
145
|
+
};
|
|
146
|
+
const statOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
147
|
+
const head = await client
|
|
148
|
+
.send(new HeadObjectCommand({
|
|
149
|
+
Bucket: bucketName,
|
|
150
|
+
Key: buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId),
|
|
151
|
+
}))
|
|
152
|
+
.catch((error) => {
|
|
153
|
+
if (isStorageNotFoundError(error))
|
|
154
|
+
return null;
|
|
155
|
+
// A HEAD on a missing key answers 404 with no NoSuchKey code, so the
|
|
156
|
+
// SDK's NotFound is the same absence.
|
|
157
|
+
if (error?.name === "NotFound")
|
|
158
|
+
return null;
|
|
159
|
+
throw error;
|
|
160
|
+
});
|
|
161
|
+
if (!head)
|
|
162
|
+
return null;
|
|
163
|
+
return { sizeBytes: head.ContentLength ?? 0 };
|
|
164
|
+
};
|
|
165
|
+
const retrieveOutboxAttachment = async (accountConfigId, accountId, outboxMessageId, outboxAttachmentId) => {
|
|
166
|
+
const key = buildOutboxAttachmentKey(accountConfigId, accountId, outboxMessageId, outboxAttachmentId);
|
|
167
|
+
const response = await client
|
|
168
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
169
|
+
.catch((error) => {
|
|
170
|
+
if (isStorageNotFoundError(error))
|
|
171
|
+
return null;
|
|
172
|
+
throw error;
|
|
173
|
+
});
|
|
174
|
+
if (!response)
|
|
175
|
+
return null;
|
|
176
|
+
if (!response.Body) {
|
|
177
|
+
throw new Error(`Empty response body for outbox attachment: ${key}`);
|
|
178
|
+
}
|
|
179
|
+
const bytes = await response.Body.transformToByteArray();
|
|
180
|
+
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
181
|
+
};
|
|
182
|
+
// The browser PUTs straight to the bucket and these bytes never touch the
|
|
183
|
+
// API. `ContentLength` on the command hoists `content-length` into
|
|
184
|
+
// X-Amz-SignedHeaders, so S3 recomputes the signature over the length it was
|
|
185
|
+
// sent: a request declaring any other size fails the signature check, and a
|
|
186
|
+
// PUT is read for exactly that many bytes. The size is bound by S3 itself,
|
|
187
|
+
// not by anything downstream trusting the client.
|
|
188
|
+
const createOutboxAttachmentUploadUrl = async (params) => {
|
|
189
|
+
const expiresIn = Math.max(1, params.expiresAt - Math.floor(Date.now() / 1000));
|
|
190
|
+
const uploadUrl = await getSignedUrl(client, new PutObjectCommand({
|
|
191
|
+
Bucket: bucketName,
|
|
192
|
+
Key: buildOutboxAttachmentKey(params.accountConfigId, params.accountId, params.outboxMessageId, params.outboxAttachmentId),
|
|
193
|
+
ContentLength: params.sizeBytes,
|
|
194
|
+
}), { expiresIn });
|
|
195
|
+
return { uploadUrl };
|
|
196
|
+
};
|
|
197
|
+
const deleteOutboxAttachments = async (accountConfigId, accountId, outboxMessageId) => {
|
|
198
|
+
const prefix = buildOutboxAttachmentPrefix(accountConfigId, accountId, outboxMessageId);
|
|
199
|
+
let continuationToken;
|
|
200
|
+
do {
|
|
201
|
+
const response = await client.send(new ListObjectsV2Command({
|
|
202
|
+
Bucket: bucketName,
|
|
203
|
+
Prefix: prefix,
|
|
204
|
+
ContinuationToken: continuationToken,
|
|
205
|
+
}));
|
|
206
|
+
const keys = (response.Contents ?? [])
|
|
207
|
+
.map((object) => object.Key)
|
|
208
|
+
.filter((key) => typeof key === "string");
|
|
209
|
+
if (keys.length > 0) {
|
|
210
|
+
await client.send(new DeleteObjectsCommand({
|
|
211
|
+
Bucket: bucketName,
|
|
212
|
+
Delete: { Objects: keys.map((Key) => ({ Key })) },
|
|
213
|
+
}));
|
|
214
|
+
}
|
|
215
|
+
continuationToken = response.IsTruncated
|
|
216
|
+
? response.NextContinuationToken
|
|
217
|
+
: undefined;
|
|
218
|
+
} while (continuationToken);
|
|
219
|
+
};
|
|
220
|
+
const storeDeduplicated = (params) => {
|
|
221
|
+
const { accountConfigId, accountId, content, contentType } = params;
|
|
222
|
+
const checksumSha256 = computeChecksum(content);
|
|
223
|
+
return storeInternal({
|
|
224
|
+
key: buildDeduplicatedKey(accountConfigId, accountId, checksumSha256),
|
|
225
|
+
content,
|
|
226
|
+
contentType,
|
|
227
|
+
});
|
|
228
|
+
};
|
|
229
|
+
const storeParsedBody = (params) => {
|
|
230
|
+
const { accountConfigId, accountId, messageId, parsed } = params;
|
|
231
|
+
const content = Buffer.from(JSON.stringify(parsed), "utf8");
|
|
232
|
+
return storeInternal({
|
|
233
|
+
key: buildParsedBodyKey(accountConfigId, accountId, messageId),
|
|
234
|
+
content,
|
|
235
|
+
contentType: "application/json",
|
|
236
|
+
});
|
|
237
|
+
};
|
|
238
|
+
const retrieveParsedBody = async (accountConfigId, accountId, messageId) => {
|
|
239
|
+
const key = buildParsedBodyKey(accountConfigId, accountId, messageId);
|
|
240
|
+
const response = await client
|
|
241
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
242
|
+
.catch((error) => {
|
|
243
|
+
if (isStorageNotFoundError(error))
|
|
244
|
+
return null;
|
|
245
|
+
throw error;
|
|
246
|
+
});
|
|
247
|
+
if (!response)
|
|
248
|
+
return null;
|
|
249
|
+
if (!response.Body) {
|
|
250
|
+
throw new Error(`Empty response body for parsed body: ${key}`);
|
|
251
|
+
}
|
|
252
|
+
const body = await response.Body.transformToByteArray();
|
|
253
|
+
const buffer = Buffer.from(body);
|
|
254
|
+
const decoded = response.ContentEncoding === "gzip" ? gunzipSync(buffer) : buffer;
|
|
255
|
+
return JSON.parse(decoded.toString("utf8"));
|
|
256
|
+
};
|
|
257
|
+
const retrieve = async (uri) => {
|
|
258
|
+
const { storageKey } = parseStorageUri(uri);
|
|
259
|
+
const response = await client.send(new GetObjectCommand({
|
|
260
|
+
Bucket: bucketName,
|
|
261
|
+
Key: storageKey,
|
|
262
|
+
}));
|
|
263
|
+
if (!response.Body) {
|
|
264
|
+
throw new Error(`Empty response body for: ${uri}`);
|
|
265
|
+
}
|
|
266
|
+
const body = await response.Body.transformToByteArray();
|
|
267
|
+
const buffer = Buffer.from(body);
|
|
268
|
+
if (response.ContentEncoding === "gzip") {
|
|
269
|
+
return gunzipSync(buffer);
|
|
270
|
+
}
|
|
271
|
+
return buffer;
|
|
272
|
+
};
|
|
273
|
+
const retrieveMessageBody = async (accountConfigId, accountId, messageId) => {
|
|
274
|
+
const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
|
|
275
|
+
const response = await client
|
|
276
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
277
|
+
.catch((error) => {
|
|
278
|
+
if (isStorageNotFoundError(error))
|
|
279
|
+
return null;
|
|
280
|
+
throw error;
|
|
281
|
+
});
|
|
282
|
+
if (!response)
|
|
283
|
+
return null;
|
|
284
|
+
if (!response.Body) {
|
|
285
|
+
throw new Error(`Empty response body for message body: ${key}`);
|
|
286
|
+
}
|
|
287
|
+
const body = await response.Body.transformToByteArray();
|
|
288
|
+
const buffer = Buffer.from(body);
|
|
289
|
+
return response.ContentEncoding === "gzip" ? gunzipSync(buffer) : buffer;
|
|
290
|
+
};
|
|
291
|
+
const retrieveMessageBodyStream = async (accountConfigId, accountId, messageId) => {
|
|
292
|
+
const key = buildMessageBodyKey(accountConfigId, accountId, messageId);
|
|
293
|
+
const response = await client
|
|
294
|
+
.send(new GetObjectCommand({ Bucket: bucketName, Key: key }))
|
|
295
|
+
.catch((error) => {
|
|
296
|
+
if (isStorageNotFoundError(error))
|
|
297
|
+
return null;
|
|
298
|
+
throw error;
|
|
299
|
+
});
|
|
300
|
+
if (!response)
|
|
301
|
+
return null;
|
|
302
|
+
if (!response.Body) {
|
|
303
|
+
throw new Error(`Empty response body for message body: ${key}`);
|
|
304
|
+
}
|
|
305
|
+
const source = response.Body;
|
|
306
|
+
if (response.ContentEncoding === "gzip") {
|
|
307
|
+
return source.pipe(createGunzip());
|
|
308
|
+
}
|
|
309
|
+
return source;
|
|
310
|
+
};
|
|
311
|
+
const exists = async (uri) => {
|
|
312
|
+
const { storageKey } = parseStorageUri(uri);
|
|
313
|
+
return client
|
|
314
|
+
.send(new HeadObjectCommand({
|
|
315
|
+
Bucket: bucketName,
|
|
316
|
+
Key: storageKey,
|
|
317
|
+
}))
|
|
318
|
+
.then(() => true)
|
|
319
|
+
.catch(() => false);
|
|
320
|
+
};
|
|
321
|
+
const del = async (uri) => {
|
|
322
|
+
const { storageKey } = parseStorageUri(uri);
|
|
323
|
+
await client.send(new DeleteObjectCommand({
|
|
324
|
+
Bucket: bucketName,
|
|
325
|
+
Key: storageKey,
|
|
326
|
+
}));
|
|
327
|
+
};
|
|
328
|
+
const storeExportArchiveStream = async (accountConfigId, exportRequestId, body) => {
|
|
329
|
+
const key = buildExportArchiveKey(accountConfigId, exportRequestId);
|
|
330
|
+
const upload = new Upload({
|
|
331
|
+
client,
|
|
332
|
+
params: {
|
|
333
|
+
Bucket: bucketName,
|
|
334
|
+
Key: key,
|
|
335
|
+
Body: body,
|
|
336
|
+
ContentType: "application/zip",
|
|
337
|
+
},
|
|
338
|
+
});
|
|
339
|
+
await upload.done().catch(async (error) => {
|
|
340
|
+
await upload.abort().catch(() => { });
|
|
341
|
+
throw error;
|
|
342
|
+
});
|
|
343
|
+
return key;
|
|
344
|
+
};
|
|
345
|
+
const getPresignedDownloadUrl = (key, expiresInSeconds) => getSignedUrl(client, new GetObjectCommand({ Bucket: bucketName, Key: key }), { expiresIn: expiresInSeconds });
|
|
346
|
+
return {
|
|
347
|
+
storeMessageBody,
|
|
348
|
+
storeBodyPart,
|
|
349
|
+
bodyPartExists,
|
|
350
|
+
retrieveBodyPart,
|
|
351
|
+
storeOutboxAttachment,
|
|
352
|
+
listOutboxAttachments,
|
|
353
|
+
listOutboxDraftsWithAttachments,
|
|
354
|
+
deleteOutboxAttachments,
|
|
355
|
+
deleteOutboxAttachment,
|
|
356
|
+
statOutboxAttachment,
|
|
357
|
+
retrieveOutboxAttachment,
|
|
358
|
+
createOutboxAttachmentUploadUrl,
|
|
359
|
+
storeDeduplicated,
|
|
360
|
+
storeParsedBody,
|
|
361
|
+
retrieveParsedBody,
|
|
362
|
+
retrieveMessageBody,
|
|
363
|
+
retrieveMessageBodyStream,
|
|
364
|
+
retrieve,
|
|
365
|
+
exists,
|
|
366
|
+
delete: del,
|
|
367
|
+
storeExportArchiveStream,
|
|
368
|
+
getPresignedDownloadUrl,
|
|
369
|
+
};
|
|
370
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { createFilesystemStorageService } from "./backends/filesystem.js";
|