@remit/storage-service 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backends/filesystem.d.ts +12 -0
- package/dist/backends/filesystem.js +296 -0
- package/dist/backends/s3.d.ts +3 -0
- package/dist/backends/s3.js +370 -0
- package/dist/filesystem.js +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/outbox-sweep.d.ts +33 -0
- package/dist/outbox-sweep.js +37 -0
- package/dist/outbox-upload-url.d.ts +44 -0
- package/{src/outbox-upload-url.ts → dist/outbox-upload-url.js} +20 -50
- package/dist/s3.d.ts +3 -0
- package/dist/s3.js +28 -0
- package/dist/signed-path.d.ts +14 -0
- package/dist/signed-path.js +46 -0
- package/dist/storage.d.ts +241 -0
- package/dist/storage.js +219 -0
- package/dist/uri.d.ts +9 -0
- package/dist/uri.js +28 -0
- package/package.json +51 -46
- package/src/backends/filesystem.ts +0 -517
- package/src/backends/s3.test.ts +0 -447
- package/src/backends/s3.ts +0 -624
- package/src/index.ts +0 -53
- package/src/outbox-attachment-retrieve.test.ts +0 -56
- package/src/outbox-sweep.test.ts +0 -196
- package/src/outbox-sweep.ts +0 -103
- package/src/outbox-upload-url.test.ts +0 -290
- package/src/s3.ts +0 -36
- package/src/signed-path.ts +0 -86
- package/src/storage.test.ts +0 -646
- package/src/storage.ts +0 -699
- package/src/uri.ts +0 -47
- package/tsconfig.json +0 -8
- /package/{src/filesystem.ts → dist/filesystem.d.ts} +0 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { type OutboxSweepResult, sweepAbandonedOutboxAttachments, } from "./outbox-sweep.js";
|
|
2
|
+
export { type AuthorizeUploadInput, type AuthorizeUploadResult, authorizeUploadRequest, UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, UPLOAD_URL_TTL_SECONDS, } from "./outbox-upload-url.js";
|
|
3
|
+
export { type SignedPathFailure, type SignedPathResult, signStoragePath, verifyStoragePath, } from "./signed-path.js";
|
|
4
|
+
export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, type ContentEncodingValue, type CreateOutboxAttachmentUploadUrlParams, computeChecksum, createMockStorageService, isStorageNotFoundError, type OutboxAttachmentListItem, type OutboxAttachmentUploadTarget, type ParsedAttachmentMeta, type ParsedBody, type ParsedContentStorageKey, type ParsedOutboxAttachmentKey, parseContentStorageKey, parseOutboxAttachmentKey, type StorageReference, type StorageService, type StorageTypeValue, type StoreBodyPartParams, type StoreDeduplicatedParams, type StoreMessageBodyParams, type StoreOutboxAttachmentParams, type StoreParsedBodyParams, } from "./storage.js";
|
|
5
|
+
export { buildStorageUri, type ParsedStorageUri, parseStorageUri, } from "./uri.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { sweepAbandonedOutboxAttachments, } from "./outbox-sweep.js";
|
|
2
|
+
export { authorizeUploadRequest, UPLOAD_ROUTE_PREFIX, UPLOAD_URL_SIGNING_LABEL, UPLOAD_URL_TTL_SECONDS, } from "./outbox-upload-url.js";
|
|
3
|
+
export { signStoragePath, verifyStoragePath, } from "./signed-path.js";
|
|
4
|
+
export { buildBodyPartKey, buildDeduplicatedKey, buildExportArchiveKey, buildMessageBodyKey, buildOutboxAttachmentKey, buildOutboxAttachmentPrefix, buildParsedBodyKey, computeChecksum, createMockStorageService, isStorageNotFoundError, parseContentStorageKey, parseOutboxAttachmentKey, } from "./storage.js";
|
|
5
|
+
export { buildStorageUri, parseStorageUri, } from "./uri.js";
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { StorageService } from "./storage.js";
|
|
2
|
+
/**
|
|
3
|
+
* Collect attachment bytes the database does not know about.
|
|
4
|
+
*
|
|
5
|
+
* A presigned upload URL stays valid for its whole window, including after the
|
|
6
|
+
* draft it was minted for is discarded, and on a deployment where the browser
|
|
7
|
+
* PUTs straight to block storage there is nothing in that path to consult. So
|
|
8
|
+
* an object can exist that no row names. That is the only case this exists for,
|
|
9
|
+
* and the rule is the whole of it: an object survives if the database still has
|
|
10
|
+
* an attachment row for it.
|
|
11
|
+
*
|
|
12
|
+
* Two orderings matter and both are deliberate. Objects are listed *before* the
|
|
13
|
+
* rows are read, so an attachment reserved between the two reads is seen as a
|
|
14
|
+
* row and not as an object, and is kept. And a draft whose row lookup fails is
|
|
15
|
+
* skipped entirely — "I could not find out what is live" must never be acted on
|
|
16
|
+
* as "nothing is live".
|
|
17
|
+
*/
|
|
18
|
+
export interface OutboxSweepResult {
|
|
19
|
+
inspected: number;
|
|
20
|
+
deleted: number;
|
|
21
|
+
/** Drafts skipped because their attachment rows could not be read. */
|
|
22
|
+
skipped: number;
|
|
23
|
+
}
|
|
24
|
+
export interface OutboxSweepDeps {
|
|
25
|
+
storage: StorageService;
|
|
26
|
+
/**
|
|
27
|
+
* The attachment ids the database holds for a draft. Throwing is the right
|
|
28
|
+
* way to say "unknown" — the sweep then leaves that draft alone.
|
|
29
|
+
*/
|
|
30
|
+
listKnownAttachmentIds: (accountConfigId: string, outboxMessageId: string) => Promise<string[]>;
|
|
31
|
+
onSkipped: (outboxMessageId: string, error: unknown) => void;
|
|
32
|
+
}
|
|
33
|
+
export declare const sweepAbandonedOutboxAttachments: (deps: OutboxSweepDeps, accountConfigId: string, accountId: string) => Promise<OutboxSweepResult>;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export const sweepAbandonedOutboxAttachments = async (deps, accountConfigId, accountId) => {
|
|
2
|
+
let inspected = 0;
|
|
3
|
+
let deleted = 0;
|
|
4
|
+
let skipped = 0;
|
|
5
|
+
const outboxMessageIds = await deps.storage.listOutboxDraftsWithAttachments(accountConfigId, accountId);
|
|
6
|
+
for (const outboxMessageId of outboxMessageIds) {
|
|
7
|
+
const objects = await deps.storage.listOutboxAttachments(accountConfigId, accountId, outboxMessageId);
|
|
8
|
+
if (objects.length === 0)
|
|
9
|
+
continue;
|
|
10
|
+
const known = await deps
|
|
11
|
+
.listKnownAttachmentIds(accountConfigId, outboxMessageId)
|
|
12
|
+
.then((ids) => new Set(ids))
|
|
13
|
+
.catch((error) => {
|
|
14
|
+
deps.onSkipped(outboxMessageId, error);
|
|
15
|
+
return null;
|
|
16
|
+
});
|
|
17
|
+
if (known === null) {
|
|
18
|
+
skipped += 1;
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
inspected += objects.length;
|
|
22
|
+
const abandoned = objects.filter((object) => !known.has(object.outboxAttachmentId));
|
|
23
|
+
// A draft whose every object is abandoned is the common case — it was
|
|
24
|
+
// discarded — and taking the prefix in one call beats one round trip per
|
|
25
|
+
// object.
|
|
26
|
+
if (abandoned.length === objects.length) {
|
|
27
|
+
await deps.storage.deleteOutboxAttachments(accountConfigId, accountId, outboxMessageId);
|
|
28
|
+
deleted += abandoned.length;
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
for (const object of abandoned) {
|
|
32
|
+
await deps.storage.deleteOutboxAttachment(accountConfigId, accountId, outboxMessageId, object.outboxAttachmentId);
|
|
33
|
+
deleted += 1;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return { inspected, deleted, skipped };
|
|
37
|
+
};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The self-hosted half of the upload seam: what a minted URL looks like when
|
|
3
|
+
* there is no block storage to presign against, and how the route that receives
|
|
4
|
+
* the bytes decides to accept them.
|
|
5
|
+
*
|
|
6
|
+
* A hosted deployment mints a presigned PUT instead and none of this runs — the
|
|
7
|
+
* bytes never reach the deployment at all. Both halves bind the same two
|
|
8
|
+
* things, an expiry and an exact byte count, so a client sees one contract.
|
|
9
|
+
*/
|
|
10
|
+
/** Domain separation: a read grant must never be presentable as a write grant. */
|
|
11
|
+
export declare const UPLOAD_URL_SIGNING_LABEL = "remit-outbox-upload-url-signing-v1";
|
|
12
|
+
export declare const UPLOAD_ROUTE_PREFIX = "/outbox-upload/";
|
|
13
|
+
/**
|
|
14
|
+
* How long a mint's URL, and the reservation behind it, stay good. Long enough
|
|
15
|
+
* for a slow phone to push 25 MB, short enough that a draft abandoned mid-upload
|
|
16
|
+
* stops holding room within the same session.
|
|
17
|
+
*/
|
|
18
|
+
export declare const UPLOAD_URL_TTL_SECONDS = 900;
|
|
19
|
+
export interface AuthorizeUploadInput {
|
|
20
|
+
secret: string | undefined;
|
|
21
|
+
/** Decoded storage key the signature was minted over. */
|
|
22
|
+
relativePath: string;
|
|
23
|
+
exp: string | undefined;
|
|
24
|
+
max: string | undefined;
|
|
25
|
+
sig: string | undefined;
|
|
26
|
+
nowSeconds: number;
|
|
27
|
+
}
|
|
28
|
+
export type AuthorizeUploadResult = {
|
|
29
|
+
authorized: true;
|
|
30
|
+
maxBytes: number;
|
|
31
|
+
} | {
|
|
32
|
+
authorized: false;
|
|
33
|
+
status: number;
|
|
34
|
+
reason: string;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Decide whether an upload request may write to the key it names.
|
|
38
|
+
*
|
|
39
|
+
* Pure, so the decision is testable without a live server. Mirrors
|
|
40
|
+
* `authorizeContentRequest` on the read side, with one addition: the byte count
|
|
41
|
+
* is part of the signed message, so the caller learns from the signature how
|
|
42
|
+
* much this URL is allowed to carry and can cut the stream off there.
|
|
43
|
+
*/
|
|
44
|
+
export declare const authorizeUploadRequest: (input: AuthorizeUploadInput) => AuthorizeUploadResult;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { verifyStoragePath } from "./signed-path.js";
|
|
2
|
-
|
|
3
2
|
/**
|
|
4
3
|
* The self-hosted half of the upload seam: what a minted URL looks like when
|
|
5
4
|
* there is no block storage to presign against, and how the route that receives
|
|
@@ -9,33 +8,15 @@ import { verifyStoragePath } from "./signed-path.js";
|
|
|
9
8
|
* bytes never reach the deployment at all. Both halves bind the same two
|
|
10
9
|
* things, an expiry and an exact byte count, so a client sees one contract.
|
|
11
10
|
*/
|
|
12
|
-
|
|
13
11
|
/** Domain separation: a read grant must never be presentable as a write grant. */
|
|
14
12
|
export const UPLOAD_URL_SIGNING_LABEL = "remit-outbox-upload-url-signing-v1";
|
|
15
|
-
|
|
16
13
|
export const UPLOAD_ROUTE_PREFIX = "/outbox-upload/";
|
|
17
|
-
|
|
18
14
|
/**
|
|
19
15
|
* How long a mint's URL, and the reservation behind it, stay good. Long enough
|
|
20
16
|
* for a slow phone to push 25 MB, short enough that a draft abandoned mid-upload
|
|
21
17
|
* stops holding room within the same session.
|
|
22
18
|
*/
|
|
23
19
|
export const UPLOAD_URL_TTL_SECONDS = 900;
|
|
24
|
-
|
|
25
|
-
export interface AuthorizeUploadInput {
|
|
26
|
-
secret: string | undefined;
|
|
27
|
-
/** Decoded storage key the signature was minted over. */
|
|
28
|
-
relativePath: string;
|
|
29
|
-
exp: string | undefined;
|
|
30
|
-
max: string | undefined;
|
|
31
|
-
sig: string | undefined;
|
|
32
|
-
nowSeconds: number;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export type AuthorizeUploadResult =
|
|
36
|
-
| { authorized: true; maxBytes: number }
|
|
37
|
-
| { authorized: false; status: number; reason: string };
|
|
38
|
-
|
|
39
20
|
/**
|
|
40
21
|
* Decide whether an upload request may write to the key it names.
|
|
41
22
|
*
|
|
@@ -44,35 +25,24 @@ export type AuthorizeUploadResult =
|
|
|
44
25
|
* is part of the signed message, so the caller learns from the signature how
|
|
45
26
|
* much this URL is allowed to carry and can cut the stream off there.
|
|
46
27
|
*/
|
|
47
|
-
export const authorizeUploadRequest = (
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
input.secret,
|
|
68
|
-
UPLOAD_URL_SIGNING_LABEL,
|
|
69
|
-
input.relativePath,
|
|
70
|
-
[exp, maxBytes],
|
|
71
|
-
input.sig,
|
|
72
|
-
input.nowSeconds,
|
|
73
|
-
);
|
|
74
|
-
if (result.valid) return { authorized: true, maxBytes };
|
|
75
|
-
|
|
76
|
-
const status = result.reason === "missing" ? 401 : 403;
|
|
77
|
-
return { authorized: false, status, reason: result.reason };
|
|
28
|
+
export const authorizeUploadRequest = (input) => {
|
|
29
|
+
if (!input.secret || input.secret.length === 0) {
|
|
30
|
+
// No secret means no upload URL could have been signed, so nothing
|
|
31
|
+
// presented here can be trusted. Fail closed.
|
|
32
|
+
return {
|
|
33
|
+
authorized: false,
|
|
34
|
+
status: 500,
|
|
35
|
+
reason: "upload-signing-not-configured",
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
const exp = Number(input.exp);
|
|
39
|
+
const maxBytes = Number(input.max);
|
|
40
|
+
if (!Number.isInteger(exp) || !Number.isInteger(maxBytes)) {
|
|
41
|
+
return { authorized: false, status: 403, reason: "malformed" };
|
|
42
|
+
}
|
|
43
|
+
const result = verifyStoragePath(input.secret, UPLOAD_URL_SIGNING_LABEL, input.relativePath, [exp, maxBytes], input.sig, input.nowSeconds);
|
|
44
|
+
if (result.valid)
|
|
45
|
+
return { authorized: true, maxBytes };
|
|
46
|
+
const status = result.reason === "missing" ? 401 : 403;
|
|
47
|
+
return { authorized: false, status, reason: result.reason };
|
|
78
48
|
};
|
package/dist/s3.d.ts
ADDED
package/dist/s3.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { S3Client } from "@aws-sdk/client-s3";
|
|
2
|
+
import { createFilesystemStorageService } from "./backends/filesystem.js";
|
|
3
|
+
import { createS3StorageService } from "./backends/s3.js";
|
|
4
|
+
export { createS3StorageService } from "./backends/s3.js";
|
|
5
|
+
export const createStorageService = () => {
|
|
6
|
+
const bucketName = process.env.S3_BUCKET_NAME;
|
|
7
|
+
if (bucketName) {
|
|
8
|
+
const client = new S3Client({
|
|
9
|
+
endpoint: process.env.S3_ENDPOINT,
|
|
10
|
+
// The SDK default (WHEN_SUPPORTED) hangs a CRC32 of an empty body off
|
|
11
|
+
// every presigned PUT as signed query params. Real S3 tolerates it;
|
|
12
|
+
// MinIO and LocalStack reject the request, and S3_ENDPOINT exists to
|
|
13
|
+
// point at exactly those.
|
|
14
|
+
requestChecksumCalculation: "WHEN_REQUIRED",
|
|
15
|
+
});
|
|
16
|
+
return createS3StorageService(client, bucketName);
|
|
17
|
+
}
|
|
18
|
+
const basePath = process.env.STORAGE_LOCAL_PATH ?? ".remit/storage";
|
|
19
|
+
// BETTER_AUTH_URL is the deployment's own public base — the value the token
|
|
20
|
+
// issuer and the JWKS discovery document already agree on — so a minted
|
|
21
|
+
// upload URL resolves from wherever the browser reached the API. Both it and
|
|
22
|
+
// the signing secret are required on the self-host stack; without them a
|
|
23
|
+
// mint fails loud inside the backend rather than handing out a URL that
|
|
24
|
+
// cannot be verified.
|
|
25
|
+
const origin = process.env.BETTER_AUTH_URL;
|
|
26
|
+
const signingSecret = process.env.BETTER_AUTH_SECRET;
|
|
27
|
+
return createFilesystemStorageService(basePath, origin && signingSecret ? { origin, signingSecret } : undefined);
|
|
28
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare const signStoragePath: (masterSecret: string, label: string, relativePath: string, claims: readonly number[]) => string;
|
|
2
|
+
export type SignedPathFailure = "missing" | "malformed" | "expired" | "bad-signature";
|
|
3
|
+
export type SignedPathResult = {
|
|
4
|
+
valid: true;
|
|
5
|
+
} | {
|
|
6
|
+
valid: false;
|
|
7
|
+
reason: SignedPathFailure;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Verify a presented signature against the path and claims it must cover. The
|
|
11
|
+
* first claim is always the expiry, in unix seconds. Pure, so the decision can
|
|
12
|
+
* be unit-tested without a live server.
|
|
13
|
+
*/
|
|
14
|
+
export declare const verifyStoragePath: (masterSecret: string, label: string, relativePath: string, claims: readonly number[], sig: string | undefined, nowSeconds: number) => SignedPathResult;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
2
|
+
/**
|
|
3
|
+
* The HMAC scheme behind every signed storage URL this stack mints.
|
|
4
|
+
*
|
|
5
|
+
* It started as the read side — `/content/*` is served against a signature
|
|
6
|
+
* because a bearer token cannot ride on an `<img src>` inside email HTML — and
|
|
7
|
+
* the write side needs exactly the same capability: a URL that grants access to
|
|
8
|
+
* one path, for a while, and is unforgeable without the key. Rather than a
|
|
9
|
+
* second scheme, both use this one with a different derivation label, so a
|
|
10
|
+
* signature minted to read content can never be replayed as permission to
|
|
11
|
+
* write, and vice versa.
|
|
12
|
+
*
|
|
13
|
+
* The signed message is the storage path followed by every claim in a fixed
|
|
14
|
+
* order. Because the path is part of the message, a signature minted for one
|
|
15
|
+
* account's object cannot be presented for another's.
|
|
16
|
+
*/
|
|
17
|
+
const deriveSigningKey = (masterSecret, label) => createHmac("sha256", masterSecret).update(label).digest();
|
|
18
|
+
const canonicalMessage = (relativePath, claims) => [relativePath, ...claims].join("\n");
|
|
19
|
+
const computeSignature = (key, relativePath, claims) => createHmac("sha256", key)
|
|
20
|
+
.update(canonicalMessage(relativePath, claims))
|
|
21
|
+
.digest("base64url");
|
|
22
|
+
export const signStoragePath = (masterSecret, label, relativePath, claims) => computeSignature(deriveSigningKey(masterSecret, label), relativePath, claims);
|
|
23
|
+
/**
|
|
24
|
+
* Verify a presented signature against the path and claims it must cover. The
|
|
25
|
+
* first claim is always the expiry, in unix seconds. Pure, so the decision can
|
|
26
|
+
* be unit-tested without a live server.
|
|
27
|
+
*/
|
|
28
|
+
export const verifyStoragePath = (masterSecret, label, relativePath, claims, sig, nowSeconds) => {
|
|
29
|
+
if (!sig || claims.length === 0)
|
|
30
|
+
return { valid: false, reason: "missing" };
|
|
31
|
+
const [exp] = claims;
|
|
32
|
+
if (!claims.every((claim) => Number.isInteger(claim) && claim > 0)) {
|
|
33
|
+
return { valid: false, reason: "malformed" };
|
|
34
|
+
}
|
|
35
|
+
if (exp < nowSeconds)
|
|
36
|
+
return { valid: false, reason: "expired" };
|
|
37
|
+
const expected = Buffer.from(signStoragePath(masterSecret, label, relativePath, claims));
|
|
38
|
+
const presented = Buffer.from(sig);
|
|
39
|
+
if (expected.length !== presented.length) {
|
|
40
|
+
return { valid: false, reason: "bad-signature" };
|
|
41
|
+
}
|
|
42
|
+
if (!timingSafeEqual(expected, presented)) {
|
|
43
|
+
return { valid: false, reason: "bad-signature" };
|
|
44
|
+
}
|
|
45
|
+
return { valid: true };
|
|
46
|
+
};
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { Readable } from "node:stream";
|
|
2
|
+
import { ContentEncoding, StorageType } from "@remit/domain-enums";
|
|
3
|
+
export type StorageTypeValue = (typeof StorageType)[keyof typeof StorageType];
|
|
4
|
+
export type ContentEncodingValue = (typeof ContentEncoding)[keyof typeof ContentEncoding];
|
|
5
|
+
export interface StorageReference {
|
|
6
|
+
uri: string;
|
|
7
|
+
storageType: StorageTypeValue;
|
|
8
|
+
storageLocation: string;
|
|
9
|
+
storageKey: string;
|
|
10
|
+
sizeBytes: number;
|
|
11
|
+
/**
|
|
12
|
+
* SHA-256 (hex) of the logical, pre-compression content.
|
|
13
|
+
* Identifies the content regardless of how it is stored on the backend,
|
|
14
|
+
* and is used by `buildDeduplicatedKey` for content-addressable storage.
|
|
15
|
+
* Note: this is NOT the checksum S3 validates against the received bytes —
|
|
16
|
+
* S3's own transport-level checksum is handled by the SDK on PUT.
|
|
17
|
+
*/
|
|
18
|
+
checksumSha256: string;
|
|
19
|
+
contentEncoding: ContentEncodingValue;
|
|
20
|
+
}
|
|
21
|
+
/** Parameters for storing a message body (raw RFC822 content) */
|
|
22
|
+
export interface StoreMessageBodyParams {
|
|
23
|
+
accountConfigId: string;
|
|
24
|
+
accountId: string;
|
|
25
|
+
messageId: string;
|
|
26
|
+
content: Buffer;
|
|
27
|
+
}
|
|
28
|
+
/** Parameters for storing a body part (attachment, inline content) */
|
|
29
|
+
export interface StoreBodyPartParams {
|
|
30
|
+
accountConfigId: string;
|
|
31
|
+
accountId: string;
|
|
32
|
+
messageId: string;
|
|
33
|
+
partPath: string;
|
|
34
|
+
content: Buffer;
|
|
35
|
+
contentType?: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Parameters for storing a file attached to a draft outbox message.
|
|
39
|
+
*
|
|
40
|
+
* Deliberately not the deduplicated path: a content-addressed object belongs to
|
|
41
|
+
* no single draft, so nothing may delete it when the draft goes. A per-draft key
|
|
42
|
+
* dies with the draft.
|
|
43
|
+
*/
|
|
44
|
+
export interface StoreOutboxAttachmentParams {
|
|
45
|
+
accountConfigId: string;
|
|
46
|
+
accountId: string;
|
|
47
|
+
outboxMessageId: string;
|
|
48
|
+
outboxAttachmentId: string;
|
|
49
|
+
content: Buffer;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* One uploaded object under a draft's attachment prefix. Reservations are not
|
|
53
|
+
* here — they live in the ledger, which is the only thing that can say whether
|
|
54
|
+
* an object present in storage is an attachment or garbage.
|
|
55
|
+
*/
|
|
56
|
+
export interface OutboxAttachmentListItem {
|
|
57
|
+
outboxAttachmentId: string;
|
|
58
|
+
key: string;
|
|
59
|
+
sizeBytes: number;
|
|
60
|
+
}
|
|
61
|
+
/** Somewhere for a client to PUT one file, for a while */
|
|
62
|
+
export interface OutboxAttachmentUploadTarget {
|
|
63
|
+
/**
|
|
64
|
+
* Absolute URL the client PUTs raw bytes to. Opaque by design: block storage
|
|
65
|
+
* on one deployment, this deployment's own upload route on another, and a
|
|
66
|
+
* client cannot tell which it was handed.
|
|
67
|
+
*/
|
|
68
|
+
uploadUrl: string;
|
|
69
|
+
}
|
|
70
|
+
export interface CreateOutboxAttachmentUploadUrlParams {
|
|
71
|
+
accountConfigId: string;
|
|
72
|
+
accountId: string;
|
|
73
|
+
outboxMessageId: string;
|
|
74
|
+
outboxAttachmentId: string;
|
|
75
|
+
/** Exact byte count the upload may carry. */
|
|
76
|
+
sizeBytes: number;
|
|
77
|
+
expiresAt: number;
|
|
78
|
+
}
|
|
79
|
+
/** Parameters for storing deduplicated content (attachments shared across messages) */
|
|
80
|
+
export interface StoreDeduplicatedParams {
|
|
81
|
+
accountConfigId: string;
|
|
82
|
+
accountId: string;
|
|
83
|
+
content: Buffer;
|
|
84
|
+
contentType?: string;
|
|
85
|
+
}
|
|
86
|
+
/** Metadata for a single attachment in a parsed message body. No binary content. */
|
|
87
|
+
export interface ParsedAttachmentMeta {
|
|
88
|
+
filename: string | null;
|
|
89
|
+
contentType: string;
|
|
90
|
+
contentDisposition: string | null;
|
|
91
|
+
contentId: string | null;
|
|
92
|
+
size: number;
|
|
93
|
+
}
|
|
94
|
+
/** Pre-parsed message body cached as parsed.json.gz to skip mailparser on warm reads */
|
|
95
|
+
export interface ParsedBody {
|
|
96
|
+
text: string | null;
|
|
97
|
+
html: string | null;
|
|
98
|
+
attachments: ParsedAttachmentMeta[];
|
|
99
|
+
}
|
|
100
|
+
/** Parameters for storing a pre-parsed message body */
|
|
101
|
+
export interface StoreParsedBodyParams {
|
|
102
|
+
accountConfigId: string;
|
|
103
|
+
accountId: string;
|
|
104
|
+
messageId: string;
|
|
105
|
+
parsed: ParsedBody;
|
|
106
|
+
}
|
|
107
|
+
export interface StorageService {
|
|
108
|
+
/** Store a message body (raw RFC822 content) */
|
|
109
|
+
storeMessageBody(params: StoreMessageBodyParams): Promise<StorageReference>;
|
|
110
|
+
/** Store a body part (attachment, inline content) */
|
|
111
|
+
storeBodyPart(params: StoreBodyPartParams): Promise<StorageReference>;
|
|
112
|
+
/**
|
|
113
|
+
* Check whether a body-part object already exists, keyed exactly as
|
|
114
|
+
* `storeBodyPart` writes it. Used by the lazy per-part generation path to
|
|
115
|
+
* skip leaves already materialized on a prior read, so regeneration is
|
|
116
|
+
* idempotent and re-reads stay cheap.
|
|
117
|
+
*/
|
|
118
|
+
bodyPartExists(accountConfigId: string, accountId: string, messageId: string, partPath: string): Promise<boolean>;
|
|
119
|
+
/**
|
|
120
|
+
* Retrieve a body-part's content (attachment or inline leaf) by
|
|
121
|
+
* account/message id and IMAP section path; returns null on NoSuchKey.
|
|
122
|
+
*/
|
|
123
|
+
retrieveBodyPart(accountConfigId: string, accountId: string, messageId: string, partPath: string): Promise<Buffer | null>;
|
|
124
|
+
/**
|
|
125
|
+
* Store a file attached to a draft outbox message, uncompressed: the size on
|
|
126
|
+
* the backend is then the decoded size the per-message cap is expressed in,
|
|
127
|
+
* and `listOutboxAttachments` can total a draft without opening anything.
|
|
128
|
+
*/
|
|
129
|
+
storeOutboxAttachment(params: StoreOutboxAttachmentParams): Promise<StorageReference>;
|
|
130
|
+
/**
|
|
131
|
+
* Every object stored against a draft, paged to the end. Only the sweep reads
|
|
132
|
+
* this, and a sweep that stops early leaves exactly the orphan it was called
|
|
133
|
+
* to collect — so there is no limit to get wrong.
|
|
134
|
+
*/
|
|
135
|
+
listOutboxAttachments(accountConfigId: string, accountId: string, outboxMessageId: string): Promise<OutboxAttachmentListItem[]>;
|
|
136
|
+
/**
|
|
137
|
+
* Remove every file and reservation stored against a draft. Unlike the read
|
|
138
|
+
* above this takes no limit and pages to the end: a sweep that stops early
|
|
139
|
+
* leaves exactly the orphan it was called to prevent.
|
|
140
|
+
*/
|
|
141
|
+
deleteOutboxAttachments(accountConfigId: string, accountId: string, outboxMessageId: string): Promise<void>;
|
|
142
|
+
/**
|
|
143
|
+
* Every draft under an account that has attachment objects in storage,
|
|
144
|
+
* including drafts whose row is long gone. The sweep needs the prefixes that
|
|
145
|
+
* exist, not the rows that exist — a discarded draft is invisible to the
|
|
146
|
+
* database and is exactly the case worth collecting.
|
|
147
|
+
*/
|
|
148
|
+
listOutboxDraftsWithAttachments(accountConfigId: string, accountId: string): Promise<string[]>;
|
|
149
|
+
/** Remove one attachment object. */
|
|
150
|
+
deleteOutboxAttachment(accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string): Promise<void>;
|
|
151
|
+
/**
|
|
152
|
+
* The size storage actually holds for an attachment, or null when nothing
|
|
153
|
+
* was uploaded. This is the only number a completion may believe — the
|
|
154
|
+
* client's word for it is not evidence, and on a deployment where the bytes
|
|
155
|
+
* went straight to block storage it is the only reading available.
|
|
156
|
+
*/
|
|
157
|
+
statOutboxAttachment(accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string): Promise<{
|
|
158
|
+
sizeBytes: number;
|
|
159
|
+
} | null>;
|
|
160
|
+
/**
|
|
161
|
+
* The bytes of one attachment, for building the MIME part it goes out as.
|
|
162
|
+
* Null when nothing is stored under the key.
|
|
163
|
+
*/
|
|
164
|
+
retrieveOutboxAttachment(accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string): Promise<Buffer | null>;
|
|
165
|
+
/**
|
|
166
|
+
* Mint somewhere to PUT one file. The backend decides what that is — block
|
|
167
|
+
* storage direct, or this deployment's upload route — and the caller never
|
|
168
|
+
* branches on which it got. Whatever it returns must bind the size: an
|
|
169
|
+
* upload that carries more than was reserved has to fail at the thing
|
|
170
|
+
* receiving it, not afterwards.
|
|
171
|
+
*/
|
|
172
|
+
createOutboxAttachmentUploadUrl(params: CreateOutboxAttachmentUploadUrlParams): Promise<OutboxAttachmentUploadTarget>;
|
|
173
|
+
/** Store deduplicated content (content-addressable, for attachments) */
|
|
174
|
+
storeDeduplicated(params: StoreDeduplicatedParams): Promise<StorageReference>;
|
|
175
|
+
/** Store a pre-parsed message body as gzipped JSON (parsed-body cache) */
|
|
176
|
+
storeParsedBody(params: StoreParsedBodyParams): Promise<StorageReference>;
|
|
177
|
+
/** Retrieve a pre-parsed message body by account/message id; returns null on NoSuchKey */
|
|
178
|
+
retrieveParsedBody(accountConfigId: string, accountId: string, messageId: string): Promise<ParsedBody | null>;
|
|
179
|
+
/** Retrieve a raw message body (RFC822 .eml) by account/message id; returns null on NoSuchKey */
|
|
180
|
+
retrieveMessageBody(accountConfigId: string, accountId: string, messageId: string): Promise<Buffer | null>;
|
|
181
|
+
/**
|
|
182
|
+
* Retrieve a raw message body as a readable stream of decompressed .eml
|
|
183
|
+
* bytes; returns null on NoSuchKey. Use on the export path so a whole
|
|
184
|
+
* mailbox is never buffered in memory.
|
|
185
|
+
*/
|
|
186
|
+
retrieveMessageBodyStream(accountConfigId: string, accountId: string, messageId: string): Promise<Readable | null>;
|
|
187
|
+
/** Retrieve content by URI */
|
|
188
|
+
retrieve(uri: string): Promise<Buffer>;
|
|
189
|
+
/** Check if content exists */
|
|
190
|
+
exists(uri: string): Promise<boolean>;
|
|
191
|
+
/** Delete content by URI */
|
|
192
|
+
delete(uri: string): Promise<void>;
|
|
193
|
+
/**
|
|
194
|
+
* Stream a finished export archive (already-compressed ZIP) to the export
|
|
195
|
+
* key for the given config + request, uncompressed at rest. The body is
|
|
196
|
+
* streamed via a multipart upload and never buffered whole. Returns the raw
|
|
197
|
+
* storage key, which `getPresignedDownloadUrl` can then sign.
|
|
198
|
+
*/
|
|
199
|
+
storeExportArchiveStream(accountConfigId: string, exportRequestId: string, body: Readable): Promise<string>;
|
|
200
|
+
/**
|
|
201
|
+
* Generate a presigned URL that grants temporary GET access to a raw S3 key.
|
|
202
|
+
* Use for export archives — the key must already exist in the bucket.
|
|
203
|
+
* Not supported on filesystem backends; throws if called there.
|
|
204
|
+
*/
|
|
205
|
+
getPresignedDownloadUrl(key: string, expiresInSeconds: number): Promise<string>;
|
|
206
|
+
}
|
|
207
|
+
export declare const buildMessageBodyKey: (accountConfigId: string, accountId: string, messageId: string) => string;
|
|
208
|
+
export declare const buildParsedBodyKey: (accountConfigId: string, accountId: string, messageId: string) => string;
|
|
209
|
+
export declare const buildBodyPartKey: (accountConfigId: string, accountId: string, messageId: string, partPath: string) => string;
|
|
210
|
+
export declare const buildOutboxAttachmentPrefix: (accountConfigId: string, accountId: string, outboxMessageId: string) => string;
|
|
211
|
+
export declare const buildOutboxAttachmentKey: (accountConfigId: string, accountId: string, outboxMessageId: string, outboxAttachmentId: string) => string;
|
|
212
|
+
export interface ParsedOutboxAttachmentKey {
|
|
213
|
+
accountConfigId: string;
|
|
214
|
+
accountId: string;
|
|
215
|
+
outboxMessageId: string;
|
|
216
|
+
outboxAttachmentId: string;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Read the ids back out of an attachment key — the inverse of
|
|
220
|
+
* `buildOutboxAttachmentKey`, for the upload route, which is handed a path and
|
|
221
|
+
* has to work out what it addresses. Returns null on anything else, including a
|
|
222
|
+
* reservation key: bytes are never written to one.
|
|
223
|
+
*/
|
|
224
|
+
export declare const parseOutboxAttachmentKey: (storageKey: string) => ParsedOutboxAttachmentKey | null;
|
|
225
|
+
export declare const buildDeduplicatedKey: (accountConfigId: string, accountId: string, checksumSha256: string) => string;
|
|
226
|
+
export interface ParsedContentStorageKey {
|
|
227
|
+
accountConfigId: string;
|
|
228
|
+
accountId: string;
|
|
229
|
+
messageId: string;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Parse the account/message ids out of a `/content/*` storage key such as
|
|
233
|
+
* `accounts/{accountConfigId}/{accountId}/messages/{messageId}/parts/1.2`
|
|
234
|
+
* (the inverse of `buildMessageBodyKey`/`buildParsedBodyKey`/`buildBodyPartKey`
|
|
235
|
+
* above). Returns null when the shape doesn't match.
|
|
236
|
+
*/
|
|
237
|
+
export declare const parseContentStorageKey: (storageKey: string) => ParsedContentStorageKey | null;
|
|
238
|
+
export declare const computeChecksum: (content: Buffer) => string;
|
|
239
|
+
export declare const buildExportArchiveKey: (accountConfigId: string, exportRequestId: string) => string;
|
|
240
|
+
export declare const isStorageNotFoundError: (error: unknown) => boolean;
|
|
241
|
+
export declare const createMockStorageService: () => StorageService;
|