@open-mercato/storage-s3 0.6.7-develop.6860.1.f796b22e80 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/storage_s3/api/delete/storage-providers/s3/delete.js +5 -20
- package/dist/modules/storage_s3/api/delete/storage-providers/s3/delete.js.map +2 -2
- package/dist/modules/storage_s3/api/get/storage-providers/s3/download.js +5 -7
- package/dist/modules/storage_s3/api/get/storage-providers/s3/download.js.map +2 -2
- package/dist/modules/storage_s3/api/get/storage-providers/s3/list.js +4 -6
- package/dist/modules/storage_s3/api/get/storage-providers/s3/list.js.map +2 -2
- package/dist/modules/storage_s3/api/post/storage-providers/s3/signed-url.js +8 -70
- package/dist/modules/storage_s3/api/post/storage-providers/s3/signed-url.js.map +3 -3
- package/dist/modules/storage_s3/api/post/storage-providers/s3/upload.js +15 -92
- package/dist/modules/storage_s3/api/post/storage-providers/s3/upload.js.map +2 -2
- package/dist/modules/storage_s3/di.js +2 -6
- package/dist/modules/storage_s3/di.js.map +2 -2
- package/dist/modules/storage_s3/integration.js +1 -1
- package/dist/modules/storage_s3/integration.js.map +1 -1
- package/dist/modules/storage_s3/lib/health.js +1 -18
- package/dist/modules/storage_s3/lib/health.js.map +2 -2
- package/dist/modules/storage_s3/lib/key-scope.js +4 -4
- package/dist/modules/storage_s3/lib/key-scope.js.map +2 -2
- package/dist/modules/storage_s3/lib/s3-driver.js +15 -44
- package/dist/modules/storage_s3/lib/s3-driver.js.map +2 -2
- package/dist/modules/storage_s3/lib/storage-service.js +22 -126
- package/dist/modules/storage_s3/lib/storage-service.js.map +3 -3
- package/jest.config.cjs +0 -1
- package/package.json +6 -13
- package/src/modules/storage_s3/__tests__/s3Driver.test.ts +8 -23
- package/src/modules/storage_s3/__tests__/s3ListRoute.test.ts +1 -7
- package/src/modules/storage_s3/__tests__/s3Routes.test.ts +0 -26
- package/src/modules/storage_s3/__tests__/storageService.test.ts +5 -20
- package/src/modules/storage_s3/api/delete/storage-providers/s3/delete.ts +5 -22
- package/src/modules/storage_s3/api/get/storage-providers/s3/download.ts +5 -7
- package/src/modules/storage_s3/api/get/storage-providers/s3/list.ts +4 -6
- package/src/modules/storage_s3/api/post/storage-providers/s3/signed-url.ts +6 -77
- package/src/modules/storage_s3/api/post/storage-providers/s3/upload.ts +16 -101
- package/src/modules/storage_s3/di.ts +2 -14
- package/src/modules/storage_s3/integration.ts +1 -1
- package/src/modules/storage_s3/lib/health.ts +0 -18
- package/src/modules/storage_s3/lib/key-scope.ts +4 -4
- package/src/modules/storage_s3/lib/s3-driver.ts +10 -65
- package/src/modules/storage_s3/lib/storage-service.ts +22 -131
- package/dist/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].js +0 -145
- package/dist/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].js.map +0 -7
- package/dist/modules/storage_s3/lib/endpoint-safety.js +0 -133
- package/dist/modules/storage_s3/lib/endpoint-safety.js.map +0 -7
- package/dist/modules/storage_s3/lib/quota-accounting.js +0 -24
- package/dist/modules/storage_s3/lib/quota-accounting.js.map +0 -7
- package/dist/modules/storage_s3/lib/quota-recovery-queue.js +0 -9
- package/dist/modules/storage_s3/lib/quota-recovery-queue.js.map +0 -7
- package/dist/modules/storage_s3/workers/quota-recovery.js +0 -70
- package/dist/modules/storage_s3/workers/quota-recovery.js.map +0 -7
- package/src/modules/storage_s3/__tests__/endpoint-safety.test.ts +0 -71
- package/src/modules/storage_s3/__tests__/health.test.ts +0 -38
- package/src/modules/storage_s3/__tests__/i18nPackageExports.test.ts +0 -13
- package/src/modules/storage_s3/__tests__/quota-lifecycle.test.ts +0 -216
- package/src/modules/storage_s3/__tests__/quota-recovery.test.ts +0 -83
- package/src/modules/storage_s3/__tests__/upload.test.ts +0 -93
- package/src/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].ts +0 -150
- package/src/modules/storage_s3/i18n/de.json +0 -17
- package/src/modules/storage_s3/i18n/en.json +0 -17
- package/src/modules/storage_s3/i18n/es.json +0 -17
- package/src/modules/storage_s3/i18n/ko.json +0 -17
- package/src/modules/storage_s3/i18n/pl.json +0 -17
- package/src/modules/storage_s3/lib/endpoint-safety.ts +0 -179
- package/src/modules/storage_s3/lib/quota-accounting.ts +0 -29
- package/src/modules/storage_s3/lib/quota-recovery-queue.ts +0 -17
- package/src/modules/storage_s3/workers/quota-recovery.ts +0 -82
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
import { lookup as dnsLookup } from "node:dns/promises";
|
|
2
|
-
import { Agent as HttpAgent } from "node:http";
|
|
3
|
-
import { Agent as HttpsAgent } from "node:https";
|
|
4
|
-
import { isIP } from "node:net";
|
|
5
|
-
import { parseBooleanWithDefault } from "@open-mercato/shared/lib/boolean";
|
|
6
|
-
import {
|
|
7
|
-
assertSafeOutboundUrl,
|
|
8
|
-
assertStaticallySafeOutboundUrl,
|
|
9
|
-
parseOutboundUrl,
|
|
10
|
-
resolveSafeOutboundUrl
|
|
11
|
-
} from "@open-mercato/shared/lib/url-safety";
|
|
12
|
-
const SUBJECT = "S3 endpoint";
|
|
13
|
-
class UnsafeS3EndpointError extends Error {
|
|
14
|
-
constructor(reason, message) {
|
|
15
|
-
super(message ?? `S3 endpoint rejected: ${reason}`);
|
|
16
|
-
this.name = "UnsafeS3EndpointError";
|
|
17
|
-
this.reason = reason;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
const s3EndpointErrorFactory = (reason, message) => new UnsafeS3EndpointError(reason, message);
|
|
21
|
-
function assertStaticallySafeS3Endpoint(rawEndpoint, deps = {}) {
|
|
22
|
-
if (!rawEndpoint) return;
|
|
23
|
-
const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled();
|
|
24
|
-
assertStaticallySafeOutboundUrl(rawEndpoint, {
|
|
25
|
-
errorFactory: s3EndpointErrorFactory,
|
|
26
|
-
subject: SUBJECT,
|
|
27
|
-
allowPrivate
|
|
28
|
-
});
|
|
29
|
-
}
|
|
30
|
-
async function assertSafeS3Endpoint(rawEndpoint, deps = {}) {
|
|
31
|
-
if (!rawEndpoint) return;
|
|
32
|
-
const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled();
|
|
33
|
-
await assertSafeOutboundUrl(rawEndpoint, {
|
|
34
|
-
errorFactory: s3EndpointErrorFactory,
|
|
35
|
-
subject: SUBJECT,
|
|
36
|
-
allowPrivate,
|
|
37
|
-
lookupHost: deps.lookupHost
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
function createSafeS3EndpointLookup(rawEndpoint, deps = {}) {
|
|
41
|
-
if (!rawEndpoint) return void 0;
|
|
42
|
-
const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled();
|
|
43
|
-
if (allowPrivate) return void 0;
|
|
44
|
-
const { hostname } = parseOutboundUrl(rawEndpoint, {
|
|
45
|
-
errorFactory: s3EndpointErrorFactory,
|
|
46
|
-
subject: SUBJECT
|
|
47
|
-
});
|
|
48
|
-
const endpointHostname = normalizeLookupHostname(hostname);
|
|
49
|
-
return (host, options, callback) => {
|
|
50
|
-
const lookupHostname = normalizeLookupHostname(host);
|
|
51
|
-
if (!isAllowedS3LookupHost(lookupHostname, endpointHostname)) {
|
|
52
|
-
callback(new UnsafeS3EndpointError(
|
|
53
|
-
"blocked_hostname",
|
|
54
|
-
`S3 endpoint lookup host "${host}" does not match configured host "${endpointHostname}"`
|
|
55
|
-
), "", 0);
|
|
56
|
-
return;
|
|
57
|
-
}
|
|
58
|
-
resolveSafeOutboundUrl(buildLookupValidationUrl(rawEndpoint, lookupHostname), {
|
|
59
|
-
errorFactory: s3EndpointErrorFactory,
|
|
60
|
-
subject: SUBJECT,
|
|
61
|
-
lookupHost: deps.lookupHost ?? defaultLookupHost
|
|
62
|
-
}).then(({ addresses }) => {
|
|
63
|
-
if (!addresses || addresses.length === 0) {
|
|
64
|
-
const family = isIP(lookupHostname);
|
|
65
|
-
if (options.all) callback(null, [{ address: lookupHostname, family }]);
|
|
66
|
-
else callback(null, lookupHostname, family);
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
if (options.all) {
|
|
70
|
-
callback(null, addresses.map((record) => ({ address: record.address, family: record.family })));
|
|
71
|
-
return;
|
|
72
|
-
}
|
|
73
|
-
callback(null, addresses[0].address, addresses[0].family);
|
|
74
|
-
}).catch((error) => {
|
|
75
|
-
callback(error instanceof Error ? error : new UnsafeS3EndpointError("dns_resolution_failed"), "", 0);
|
|
76
|
-
});
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
function normalizeLookupHostname(hostname) {
|
|
80
|
-
let normalized = hostname.trim().toLowerCase();
|
|
81
|
-
if (normalized.startsWith("[") && normalized.endsWith("]")) {
|
|
82
|
-
normalized = normalized.slice(1, -1);
|
|
83
|
-
}
|
|
84
|
-
while (normalized.endsWith(".")) {
|
|
85
|
-
normalized = normalized.slice(0, -1);
|
|
86
|
-
}
|
|
87
|
-
return normalized;
|
|
88
|
-
}
|
|
89
|
-
function isAllowedS3LookupHost(lookupHostname, endpointHostname) {
|
|
90
|
-
if (lookupHostname === endpointHostname) return true;
|
|
91
|
-
if (isIP(endpointHostname)) return false;
|
|
92
|
-
return lookupHostname.endsWith(`.${endpointHostname}`);
|
|
93
|
-
}
|
|
94
|
-
function buildLookupValidationUrl(rawEndpoint, lookupHostname) {
|
|
95
|
-
const { url } = parseOutboundUrl(rawEndpoint, {
|
|
96
|
-
errorFactory: s3EndpointErrorFactory,
|
|
97
|
-
subject: SUBJECT
|
|
98
|
-
});
|
|
99
|
-
const host = isIP(lookupHostname) === 6 ? `[${lookupHostname}]` : lookupHostname;
|
|
100
|
-
const port = url.port ? `:${url.port}` : "";
|
|
101
|
-
return `${url.protocol}//${host}${port}`;
|
|
102
|
-
}
|
|
103
|
-
const S3_CONNECTION_TIMEOUT_MS = 1e4;
|
|
104
|
-
const S3_REQUEST_TIMEOUT_MS = 12e4;
|
|
105
|
-
function createSafeS3RequestHandler(rawEndpoint, deps = {}) {
|
|
106
|
-
const lookup = createSafeS3EndpointLookup(rawEndpoint, deps);
|
|
107
|
-
const timeouts = {
|
|
108
|
-
connectionTimeout: S3_CONNECTION_TIMEOUT_MS,
|
|
109
|
-
requestTimeout: S3_REQUEST_TIMEOUT_MS
|
|
110
|
-
};
|
|
111
|
-
if (!lookup) return timeouts;
|
|
112
|
-
return {
|
|
113
|
-
...timeouts,
|
|
114
|
-
httpAgent: new HttpAgent({ lookup }),
|
|
115
|
-
httpsAgent: new HttpsAgent({ lookup })
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
function isAllowInternalS3EndpointsEnabled() {
|
|
119
|
-
return parseBooleanWithDefault(process.env.OM_STORAGE_S3_ALLOW_INTERNAL_ENDPOINTS, false);
|
|
120
|
-
}
|
|
121
|
-
const defaultLookupHost = async (hostname) => {
|
|
122
|
-
const records = await dnsLookup(hostname, { all: true, verbatim: true });
|
|
123
|
-
return records;
|
|
124
|
-
};
|
|
125
|
-
export {
|
|
126
|
-
UnsafeS3EndpointError,
|
|
127
|
-
assertSafeS3Endpoint,
|
|
128
|
-
assertStaticallySafeS3Endpoint,
|
|
129
|
-
createSafeS3EndpointLookup,
|
|
130
|
-
createSafeS3RequestHandler,
|
|
131
|
-
isAllowInternalS3EndpointsEnabled
|
|
132
|
-
};
|
|
133
|
-
//# sourceMappingURL=endpoint-safety.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../../src/modules/storage_s3/lib/endpoint-safety.ts"],
|
|
4
|
-
"sourcesContent": ["import { lookup as dnsLookup } from 'node:dns/promises'\nimport { Agent as HttpAgent } from 'node:http'\nimport { Agent as HttpsAgent } from 'node:https'\nimport { isIP, type LookupFunction } from 'node:net'\nimport { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'\nimport {\n assertSafeOutboundUrl,\n assertStaticallySafeOutboundUrl,\n parseOutboundUrl,\n resolveSafeOutboundUrl,\n type HostLookup,\n type ResolvedHostAddress,\n type UrlSafetyReason,\n} from '@open-mercato/shared/lib/url-safety'\n\nconst SUBJECT = 'S3 endpoint'\n\nexport class UnsafeS3EndpointError extends Error {\n public readonly reason: string\n\n constructor(reason: string, message?: string) {\n super(message ?? `S3 endpoint rejected: ${reason}`)\n this.name = 'UnsafeS3EndpointError'\n this.reason = reason\n }\n}\n\nconst s3EndpointErrorFactory = (reason: UrlSafetyReason, message: string) =>\n new UnsafeS3EndpointError(reason, message)\n\nexport type AssertStaticS3EndpointDeps = {\n allowInternal?: boolean\n}\n\nexport function assertStaticallySafeS3Endpoint(\n rawEndpoint: string | null | undefined,\n deps: AssertStaticS3EndpointDeps = {},\n): void {\n if (!rawEndpoint) return\n const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()\n assertStaticallySafeOutboundUrl(rawEndpoint, {\n errorFactory: s3EndpointErrorFactory,\n subject: SUBJECT,\n allowPrivate,\n })\n}\n\nexport type AssertSafeS3EndpointDeps = {\n lookupHost?: HostLookup\n allowInternal?: boolean\n}\n\nexport async function assertSafeS3Endpoint(\n rawEndpoint: string | null | undefined,\n deps: AssertSafeS3EndpointDeps = {},\n): Promise<void> {\n if (!rawEndpoint) return\n const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()\n await assertSafeOutboundUrl(rawEndpoint, {\n errorFactory: s3EndpointErrorFactory,\n subject: SUBJECT,\n allowPrivate,\n lookupHost: deps.lookupHost,\n })\n}\n\nexport type SafeS3EndpointLookupDeps = {\n lookupHost?: HostLookup\n allowInternal?: boolean\n}\n\nexport function createSafeS3EndpointLookup(\n rawEndpoint: string | null | undefined,\n deps: SafeS3EndpointLookupDeps = {},\n): LookupFunction | undefined {\n if (!rawEndpoint) return undefined\n const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()\n if (allowPrivate) return undefined\n\n const { hostname } = parseOutboundUrl(rawEndpoint, {\n errorFactory: s3EndpointErrorFactory,\n subject: SUBJECT,\n })\n const endpointHostname = normalizeLookupHostname(hostname)\n\n return (host, options, callback) => {\n const lookupHostname = normalizeLookupHostname(host)\n if (!isAllowedS3LookupHost(lookupHostname, endpointHostname)) {\n callback(new UnsafeS3EndpointError(\n 'blocked_hostname',\n `S3 endpoint lookup host \"${host}\" does not match configured host \"${endpointHostname}\"`,\n ), '', 0)\n return\n }\n\n resolveSafeOutboundUrl(buildLookupValidationUrl(rawEndpoint, lookupHostname), {\n errorFactory: s3EndpointErrorFactory,\n subject: SUBJECT,\n lookupHost: deps.lookupHost ?? defaultLookupHost,\n }).then(({ addresses }) => {\n if (!addresses || addresses.length === 0) {\n const family = isIP(lookupHostname)\n if (options.all) callback(null, [{ address: lookupHostname, family }] as ResolvedHostAddress[])\n else callback(null, lookupHostname, family)\n return\n }\n if (options.all) {\n callback(null, addresses.map((record) => ({ address: record.address, family: record.family })))\n return\n }\n callback(null, addresses[0].address, addresses[0].family)\n }).catch((error: unknown) => {\n callback(error instanceof Error ? error : new UnsafeS3EndpointError('dns_resolution_failed'), '', 0)\n })\n }\n}\n\nfunction normalizeLookupHostname(hostname: string): string {\n let normalized = hostname.trim().toLowerCase()\n if (normalized.startsWith('[') && normalized.endsWith(']')) {\n normalized = normalized.slice(1, -1)\n }\n while (normalized.endsWith('.')) {\n normalized = normalized.slice(0, -1)\n }\n return normalized\n}\n\nfunction isAllowedS3LookupHost(lookupHostname: string, endpointHostname: string): boolean {\n if (lookupHostname === endpointHostname) return true\n if (isIP(endpointHostname)) return false\n return lookupHostname.endsWith(`.${endpointHostname}`)\n}\n\nfunction buildLookupValidationUrl(rawEndpoint: string, lookupHostname: string): string {\n const { url } = parseOutboundUrl(rawEndpoint, {\n errorFactory: s3EndpointErrorFactory,\n subject: SUBJECT,\n })\n const host = isIP(lookupHostname) === 6 ? `[${lookupHostname}]` : lookupHostname\n const port = url.port ? `:${url.port}` : ''\n return `${url.protocol}//${host}${port}`\n}\n\nexport type SafeS3RequestHandlerOptions = {\n connectionTimeout: number\n requestTimeout: number\n httpAgent?: HttpAgent\n httpsAgent?: HttpsAgent\n}\n\nconst S3_CONNECTION_TIMEOUT_MS = 10_000\nconst S3_REQUEST_TIMEOUT_MS = 120_000\n\nexport function createSafeS3RequestHandler(\n rawEndpoint: string | null | undefined,\n deps: SafeS3EndpointLookupDeps = {},\n): SafeS3RequestHandlerOptions | undefined {\n const lookup = createSafeS3EndpointLookup(rawEndpoint, deps)\n const timeouts = {\n connectionTimeout: S3_CONNECTION_TIMEOUT_MS,\n requestTimeout: S3_REQUEST_TIMEOUT_MS,\n }\n if (!lookup) return timeouts\n return {\n ...timeouts,\n httpAgent: new HttpAgent({ lookup }),\n httpsAgent: new HttpsAgent({ lookup }),\n }\n}\n\nexport function isAllowInternalS3EndpointsEnabled(): boolean {\n return parseBooleanWithDefault(process.env.OM_STORAGE_S3_ALLOW_INTERNAL_ENDPOINTS, false)\n}\n\nconst defaultLookupHost: HostLookup = async (hostname) => {\n const records = await dnsLookup(hostname, { all: true, verbatim: true })\n return records\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,UAAU,iBAAiB;AACpC,SAAS,SAAS,iBAAiB;AACnC,SAAS,SAAS,kBAAkB;AACpC,SAAS,YAAiC;AAC1C,SAAS,+BAA+B;AACxC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAIK;AAEP,MAAM,UAAU;AAET,MAAM,8BAA8B,MAAM;AAAA,EAG/C,YAAY,QAAgB,SAAkB;AAC5C,UAAM,WAAW,yBAAyB,MAAM,EAAE;AAClD,SAAK,OAAO;AACZ,SAAK,SAAS;AAAA,EAChB;AACF;AAEA,MAAM,yBAAyB,CAAC,QAAyB,YACvD,IAAI,sBAAsB,QAAQ,OAAO;AAMpC,SAAS,+BACd,aACA,OAAmC,CAAC,GAC9B;AACN,MAAI,CAAC,YAAa;AAClB,QAAM,eAAe,KAAK,iBAAiB,kCAAkC;AAC7E,kCAAgC,aAAa;AAAA,IAC3C,cAAc;AAAA,IACd,SAAS;AAAA,IACT;AAAA,EACF,CAAC;AACH;AAOA,eAAsB,qBACpB,aACA,OAAiC,CAAC,GACnB;AACf,MAAI,CAAC,YAAa;AAClB,QAAM,eAAe,KAAK,iBAAiB,kCAAkC;AAC7E,QAAM,sBAAsB,aAAa;AAAA,IACvC,cAAc;AAAA,IACd,SAAS;AAAA,IACT;AAAA,IACA,YAAY,KAAK;AAAA,EACnB,CAAC;AACH;AAOO,SAAS,2BACd,aACA,OAAiC,CAAC,GACN;AAC5B,MAAI,CAAC,YAAa,QAAO;AACzB,QAAM,eAAe,KAAK,iBAAiB,kCAAkC;AAC7E,MAAI,aAAc,QAAO;AAEzB,QAAM,EAAE,SAAS,IAAI,iBAAiB,aAAa;AAAA,IACjD,cAAc;AAAA,IACd,SAAS;AAAA,EACX,CAAC;AACD,QAAM,mBAAmB,wBAAwB,QAAQ;AAEzD,SAAO,CAAC,MAAM,SAAS,aAAa;AAClC,UAAM,iBAAiB,wBAAwB,IAAI;AACnD,QAAI,CAAC,sBAAsB,gBAAgB,gBAAgB,GAAG;AAC5D,eAAS,IAAI;AAAA,QACX;AAAA,QACA,4BAA4B,IAAI,qCAAqC,gBAAgB;AAAA,MACvF,GAAG,IAAI,CAAC;AACR;AAAA,IACF;AAEA,2BAAuB,yBAAyB,aAAa,cAAc,GAAG;AAAA,MAC5E,cAAc;AAAA,MACd,SAAS;AAAA,MACT,YAAY,KAAK,cAAc;AAAA,IACjC,CAAC,EAAE,KAAK,CAAC,EAAE,UAAU,MAAM;AACzB,UAAI,CAAC,aAAa,UAAU,WAAW,GAAG;AACxC,cAAM,SAAS,KAAK,cAAc;AAClC,YAAI,QAAQ,IAAK,UAAS,MAAM,CAAC,EAAE,SAAS,gBAAgB,OAAO,CAAC,CAA0B;AAAA,YACzF,UAAS,MAAM,gBAAgB,MAAM;AAC1C;AAAA,MACF;AACA,UAAI,QAAQ,KAAK;AACf,iBAAS,MAAM,UAAU,IAAI,CAAC,YAAY,EAAE,SAAS,OAAO,SAAS,QAAQ,OAAO,OAAO,EAAE,CAAC;AAC9F;AAAA,MACF;AACA,eAAS,MAAM,UAAU,CAAC,EAAE,SAAS,UAAU,CAAC,EAAE,MAAM;AAAA,IAC1D,CAAC,EAAE,MAAM,CAAC,UAAmB;AAC3B,eAAS,iBAAiB,QAAQ,QAAQ,IAAI,sBAAsB,uBAAuB,GAAG,IAAI,CAAC;AAAA,IACrG,CAAC;AAAA,EACH;AACF;AAEA,SAAS,wBAAwB,UAA0B;AACzD,MAAI,aAAa,SAAS,KAAK,EAAE,YAAY;AAC7C,MAAI,WAAW,WAAW,GAAG,KAAK,WAAW,SAAS,GAAG,GAAG;AAC1D,iBAAa,WAAW,MAAM,GAAG,EAAE;AAAA,EACrC;AACA,SAAO,WAAW,SAAS,GAAG,GAAG;AAC/B,iBAAa,WAAW,MAAM,GAAG,EAAE;AAAA,EACrC;AACA,SAAO;AACT;AAEA,SAAS,sBAAsB,gBAAwB,kBAAmC;AACxF,MAAI,mBAAmB,iBAAkB,QAAO;AAChD,MAAI,KAAK,gBAAgB,EAAG,QAAO;AACnC,SAAO,eAAe,SAAS,IAAI,gBAAgB,EAAE;AACvD;AAEA,SAAS,yBAAyB,aAAqB,gBAAgC;AACrF,QAAM,EAAE,IAAI,IAAI,iBAAiB,aAAa;AAAA,IAC5C,cAAc;AAAA,IACd,SAAS;AAAA,EACX,CAAC;AACD,QAAM,OAAO,KAAK,cAAc,MAAM,IAAI,IAAI,cAAc,MAAM;AAClE,QAAM,OAAO,IAAI,OAAO,IAAI,IAAI,IAAI,KAAK;AACzC,SAAO,GAAG,IAAI,QAAQ,KAAK,IAAI,GAAG,IAAI;AACxC;AASA,MAAM,2BAA2B;AACjC,MAAM,wBAAwB;AAEvB,SAAS,2BACd,aACA,OAAiC,CAAC,GACO;AACzC,QAAM,SAAS,2BAA2B,aAAa,IAAI;AAC3D,QAAM,WAAW;AAAA,IACf,mBAAmB;AAAA,IACnB,gBAAgB;AAAA,EAClB;AACA,MAAI,CAAC,OAAQ,QAAO;AACpB,SAAO;AAAA,IACL,GAAG;AAAA,IACH,WAAW,IAAI,UAAU,EAAE,OAAO,CAAC;AAAA,IACnC,YAAY,IAAI,WAAW,EAAE,OAAO,CAAC;AAAA,EACvC;AACF;AAEO,SAAS,oCAA6C;AAC3D,SAAO,wBAAwB,QAAQ,IAAI,wCAAwC,KAAK;AAC1F;AAEA,MAAM,oBAAgC,OAAO,aAAa;AACxD,QAAM,UAAU,MAAM,UAAU,UAAU,EAAE,KAAK,MAAM,UAAU,KAAK,CAAC;AACvE,SAAO;AACT;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { isS3KeyScopedToTenant } from "./key-scope.js";
|
|
2
|
-
async function reconcileTenantS3Objects(input) {
|
|
3
|
-
const objects = [];
|
|
4
|
-
let continuationToken;
|
|
5
|
-
do {
|
|
6
|
-
const page = await input.driver.listObjects("", 1e3, continuationToken);
|
|
7
|
-
for (const file of page.files) {
|
|
8
|
-
if (isS3KeyScopedToTenant(file.key, input.organizationId, input.tenantId)) {
|
|
9
|
-
objects.push({ path: file.key, bytes: file.size });
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
|
-
continuationToken = page.truncated ? page.nextContinuationToken : void 0;
|
|
13
|
-
} while (continuationToken);
|
|
14
|
-
await input.quotaService.reconcileStandaloneObjects({
|
|
15
|
-
tenantId: input.tenantId,
|
|
16
|
-
organizationId: input.organizationId,
|
|
17
|
-
storageDriver: "s3",
|
|
18
|
-
objects
|
|
19
|
-
});
|
|
20
|
-
}
|
|
21
|
-
export {
|
|
22
|
-
reconcileTenantS3Objects
|
|
23
|
-
};
|
|
24
|
-
//# sourceMappingURL=quota-accounting.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../../src/modules/storage_s3/lib/quota-accounting.ts"],
|
|
4
|
-
"sourcesContent": ["import type { AttachmentQuotaService } from '@open-mercato/core/modules/attachments/lib/quota-service'\nimport { isS3KeyScopedToTenant } from './key-scope'\nimport type { S3StorageDriver } from './s3-driver'\n\nexport async function reconcileTenantS3Objects(input: {\n driver: S3StorageDriver\n quotaService: AttachmentQuotaService\n tenantId: string\n organizationId: string\n}): Promise<void> {\n const objects: Array<{ path: string; bytes: number }> = []\n let continuationToken: string | undefined\n do {\n const page = await input.driver.listObjects('', 1000, continuationToken)\n for (const file of page.files) {\n if (isS3KeyScopedToTenant(file.key, input.organizationId, input.tenantId)) {\n objects.push({ path: file.key, bytes: file.size })\n }\n }\n continuationToken = page.truncated ? page.nextContinuationToken : undefined\n } while (continuationToken)\n\n await input.quotaService.reconcileStandaloneObjects({\n tenantId: input.tenantId,\n organizationId: input.organizationId,\n storageDriver: 's3',\n objects,\n })\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,6BAA6B;AAGtC,eAAsB,yBAAyB,OAK7B;AAChB,QAAM,UAAkD,CAAC;AACzD,MAAI;AACJ,KAAG;AACD,UAAM,OAAO,MAAM,MAAM,OAAO,YAAY,IAAI,KAAM,iBAAiB;AACvE,eAAW,QAAQ,KAAK,OAAO;AAC7B,UAAI,sBAAsB,KAAK,KAAK,MAAM,gBAAgB,MAAM,QAAQ,GAAG;AACzE,gBAAQ,KAAK,EAAE,MAAM,KAAK,KAAK,OAAO,KAAK,KAAK,CAAC;AAAA,MACnD;AAAA,IACF;AACA,wBAAoB,KAAK,YAAY,KAAK,wBAAwB;AAAA,EACpE,SAAS;AAET,QAAM,MAAM,aAAa,2BAA2B;AAAA,IAClD,UAAU,MAAM;AAAA,IAChB,gBAAgB,MAAM;AAAA,IACtB,eAAe;AAAA,IACf;AAAA,EACF,CAAC;AACH;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { createModuleQueue } from "@open-mercato/queue";
|
|
2
|
-
const queue = createModuleQueue("storage-s3-quota-recovery", { concurrency: 2 });
|
|
3
|
-
async function scheduleStorageS3QuotaRecovery(payload, delayMs) {
|
|
4
|
-
await queue.enqueue(payload, { delayMs: Math.max(1e3, delayMs) });
|
|
5
|
-
}
|
|
6
|
-
export {
|
|
7
|
-
scheduleStorageS3QuotaRecovery
|
|
8
|
-
};
|
|
9
|
-
//# sourceMappingURL=quota-recovery-queue.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../../src/modules/storage_s3/lib/quota-recovery-queue.ts"],
|
|
4
|
-
"sourcesContent": ["import { createModuleQueue } from '@open-mercato/queue'\n\nexport type StorageS3QuotaRecoveryJob = {\n reservationId: string\n tenantId: string\n organizationId: string\n absenceCheck?: number\n}\n\nconst queue = createModuleQueue<StorageS3QuotaRecoveryJob>('storage-s3-quota-recovery', { concurrency: 2 })\n\nexport async function scheduleStorageS3QuotaRecovery(\n payload: StorageS3QuotaRecoveryJob,\n delayMs: number,\n): Promise<void> {\n await queue.enqueue(payload, { delayMs: Math.max(1_000, delayMs) })\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,yBAAyB;AASlC,MAAM,QAAQ,kBAA6C,6BAA6B,EAAE,aAAa,EAAE,CAAC;AAE1G,eAAsB,+BACpB,SACA,SACe;AACf,QAAM,MAAM,QAAQ,SAAS,EAAE,SAAS,KAAK,IAAI,KAAO,OAAO,EAAE,CAAC;AACpE;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
import { S3StorageDriver } from "../lib/s3-driver.js";
|
|
2
|
-
import { scheduleStorageS3QuotaRecovery } from "../lib/quota-recovery-queue.js";
|
|
3
|
-
const metadata = {
|
|
4
|
-
queue: "storage-s3-quota-recovery",
|
|
5
|
-
id: "storage_s3:quota-recovery",
|
|
6
|
-
concurrency: 2
|
|
7
|
-
};
|
|
8
|
-
async function handle(job, ctx) {
|
|
9
|
-
const quotaService = ctx.resolve("attachmentQuotaService");
|
|
10
|
-
const scope = {
|
|
11
|
-
tenantId: job.payload.tenantId,
|
|
12
|
-
organizationId: job.payload.organizationId
|
|
13
|
-
};
|
|
14
|
-
const record = await quotaService.getReservation(job.payload.reservationId, scope);
|
|
15
|
-
if (!record || record.status === "committed") return;
|
|
16
|
-
if (record.expiresAt && record.expiresAt.getTime() > Date.now()) {
|
|
17
|
-
await scheduleStorageS3QuotaRecovery(job.payload, record.expiresAt.getTime() - Date.now());
|
|
18
|
-
return;
|
|
19
|
-
}
|
|
20
|
-
const claimed = await quotaService.claimExpired(record.id, scope);
|
|
21
|
-
if (!claimed) return;
|
|
22
|
-
try {
|
|
23
|
-
const credentialsService = ctx.resolve("integrationCredentialsService");
|
|
24
|
-
const credentials = await credentialsService.resolve("storage_s3", {
|
|
25
|
-
tenantId: claimed.tenantId,
|
|
26
|
-
organizationId: claimed.organizationId
|
|
27
|
-
});
|
|
28
|
-
if (!credentials) throw new Error("S3 integration is not configured for quota recovery.");
|
|
29
|
-
const driver = new S3StorageDriver({
|
|
30
|
-
...credentials,
|
|
31
|
-
organizationId: claimed.organizationId,
|
|
32
|
-
tenantId: claimed.tenantId
|
|
33
|
-
});
|
|
34
|
-
if (claimed.source === "storage_s3_signed") {
|
|
35
|
-
const listed = await driver.listObjects(claimed.storagePath, 1);
|
|
36
|
-
const object = listed.files.find((file) => file.key === claimed.storagePath);
|
|
37
|
-
if (!object) {
|
|
38
|
-
if (record.status === "storing" && (job.payload.absenceCheck ?? 0) === 0 && claimed.expiresAt) {
|
|
39
|
-
await scheduleStorageS3QuotaRecovery(
|
|
40
|
-
{
|
|
41
|
-
...job.payload,
|
|
42
|
-
absenceCheck: 1
|
|
43
|
-
},
|
|
44
|
-
claimed.expiresAt.getTime() - Date.now()
|
|
45
|
-
);
|
|
46
|
-
return;
|
|
47
|
-
}
|
|
48
|
-
await quotaService.release(claimed.id, claimed.leaseToken);
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
const validSize = claimed.uploadTokenHash ? object.size <= claimed.reservedBytes : object.size === claimed.reservedBytes;
|
|
52
|
-
if (validSize) {
|
|
53
|
-
await quotaService.commitRecoveredStandalone(claimed.id, claimed.leaseToken, object.size);
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
await driver.deleteStrict("", claimed.storagePath);
|
|
58
|
-
await quotaService.release(claimed.id, claimed.leaseToken);
|
|
59
|
-
} catch (error) {
|
|
60
|
-
if (claimed.expiresAt) {
|
|
61
|
-
await scheduleStorageS3QuotaRecovery(job.payload, claimed.expiresAt.getTime() - Date.now());
|
|
62
|
-
}
|
|
63
|
-
throw error;
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
export {
|
|
67
|
-
handle as default,
|
|
68
|
-
metadata
|
|
69
|
-
};
|
|
70
|
-
//# sourceMappingURL=quota-recovery.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../../src/modules/storage_s3/workers/quota-recovery.ts"],
|
|
4
|
-
"sourcesContent": ["import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'\nimport type { AttachmentQuotaService } from '@open-mercato/core/modules/attachments/lib/quota-service'\nimport { S3StorageDriver } from '../lib/s3-driver'\nimport { scheduleStorageS3QuotaRecovery, type StorageS3QuotaRecoveryJob } from '../lib/quota-recovery-queue'\n\nexport const metadata: WorkerMeta = {\n queue: 'storage-s3-quota-recovery',\n id: 'storage_s3:quota-recovery',\n concurrency: 2,\n}\n\ntype HandlerContext = JobContext & { resolve: <T = unknown>(name: string) => T }\n\ntype IntegrationCredentialsService = {\n resolve(integrationId: string, scope: { tenantId: string; organizationId: string }): Promise<Record<string, unknown> | null>\n}\n\nexport default async function handle(\n job: QueuedJob<StorageS3QuotaRecoveryJob>,\n ctx: HandlerContext,\n): Promise<void> {\n const quotaService = ctx.resolve<AttachmentQuotaService>('attachmentQuotaService')\n const scope = {\n tenantId: job.payload.tenantId,\n organizationId: job.payload.organizationId,\n }\n const record = await quotaService.getReservation(job.payload.reservationId, scope)\n if (!record || record.status === 'committed') return\n if (record.expiresAt && record.expiresAt.getTime() > Date.now()) {\n await scheduleStorageS3QuotaRecovery(job.payload, record.expiresAt.getTime() - Date.now())\n return\n }\n\n const claimed = await quotaService.claimExpired(record.id, scope)\n if (!claimed) return\n try {\n const credentialsService = ctx.resolve<IntegrationCredentialsService>('integrationCredentialsService')\n const credentials = await credentialsService.resolve('storage_s3', {\n tenantId: claimed.tenantId,\n organizationId: claimed.organizationId,\n })\n if (!credentials) throw new Error('S3 integration is not configured for quota recovery.')\n const driver = new S3StorageDriver({\n ...credentials,\n organizationId: claimed.organizationId,\n tenantId: claimed.tenantId,\n })\n\n if (claimed.source === 'storage_s3_signed') {\n const listed = await driver.listObjects(claimed.storagePath, 1)\n const object = listed.files.find((file) => file.key === claimed.storagePath)\n if (!object) {\n if (record.status === 'storing' && (job.payload.absenceCheck ?? 0) === 0 && claimed.expiresAt) {\n await scheduleStorageS3QuotaRecovery({\n ...job.payload,\n absenceCheck: 1,\n },\n claimed.expiresAt.getTime() - Date.now(),\n )\n return\n }\n await quotaService.release(claimed.id, claimed.leaseToken)\n return\n }\n const validSize = claimed.uploadTokenHash\n ? object.size <= claimed.reservedBytes\n : object.size === claimed.reservedBytes\n if (validSize) {\n await quotaService.commitRecoveredStandalone(claimed.id, claimed.leaseToken, object.size)\n return\n }\n }\n\n await driver.deleteStrict('', claimed.storagePath)\n await quotaService.release(claimed.id, claimed.leaseToken)\n } catch (error) {\n if (claimed.expiresAt) {\n await scheduleStorageS3QuotaRecovery(job.payload, claimed.expiresAt.getTime() - Date.now())\n }\n throw error\n }\n}\n"],
|
|
5
|
-
"mappings": "AAEA,SAAS,uBAAuB;AAChC,SAAS,sCAAsE;AAExE,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,IAAI;AAAA,EACJ,aAAa;AACf;AAQA,eAAO,OACL,KACA,KACe;AACf,QAAM,eAAe,IAAI,QAAgC,wBAAwB;AACjF,QAAM,QAAQ;AAAA,IACZ,UAAU,IAAI,QAAQ;AAAA,IACtB,gBAAgB,IAAI,QAAQ;AAAA,EAC9B;AACA,QAAM,SAAS,MAAM,aAAa,eAAe,IAAI,QAAQ,eAAe,KAAK;AACjF,MAAI,CAAC,UAAU,OAAO,WAAW,YAAa;AAC9C,MAAI,OAAO,aAAa,OAAO,UAAU,QAAQ,IAAI,KAAK,IAAI,GAAG;AAC/D,UAAM,+BAA+B,IAAI,SAAS,OAAO,UAAU,QAAQ,IAAI,KAAK,IAAI,CAAC;AACzF;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,aAAa,aAAa,OAAO,IAAI,KAAK;AAChE,MAAI,CAAC,QAAS;AACd,MAAI;AACF,UAAM,qBAAqB,IAAI,QAAuC,+BAA+B;AACrG,UAAM,cAAc,MAAM,mBAAmB,QAAQ,cAAc;AAAA,MACjE,UAAU,QAAQ;AAAA,MAClB,gBAAgB,QAAQ;AAAA,IAC1B,CAAC;AACD,QAAI,CAAC,YAAa,OAAM,IAAI,MAAM,sDAAsD;AACxF,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC,GAAG;AAAA,MACH,gBAAgB,QAAQ;AAAA,MACxB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAED,QAAI,QAAQ,WAAW,qBAAqB;AAC1C,YAAM,SAAS,MAAM,OAAO,YAAY,QAAQ,aAAa,CAAC;AAC9D,YAAM,SAAS,OAAO,MAAM,KAAK,CAAC,SAAS,KAAK,QAAQ,QAAQ,WAAW;AAC3E,UAAI,CAAC,QAAQ;AACX,YAAI,OAAO,WAAW,cAAc,IAAI,QAAQ,gBAAgB,OAAO,KAAK,QAAQ,WAAW;AAC7F,gBAAM;AAAA,YAA+B;AAAA,cACnC,GAAG,IAAI;AAAA,cACP,cAAc;AAAA,YAChB;AAAA,YACE,QAAQ,UAAU,QAAQ,IAAI,KAAK,IAAI;AAAA,UACzC;AACA;AAAA,QACF;AACA,cAAM,aAAa,QAAQ,QAAQ,IAAI,QAAQ,UAAU;AACzD;AAAA,MACF;AACA,YAAM,YAAY,QAAQ,kBACtB,OAAO,QAAQ,QAAQ,gBACvB,OAAO,SAAS,QAAQ;AAC5B,UAAI,WAAW;AACb,cAAM,aAAa,0BAA0B,QAAQ,IAAI,QAAQ,YAAY,OAAO,IAAI;AACxF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,OAAO,aAAa,IAAI,QAAQ,WAAW;AACjD,UAAM,aAAa,QAAQ,QAAQ,IAAI,QAAQ,UAAU;AAAA,EAC3D,SAAS,OAAO;AACd,QAAI,QAAQ,WAAW;AACrB,YAAM,+BAA+B,IAAI,SAAS,QAAQ,UAAU,QAAQ,IAAI,KAAK,IAAI,CAAC;AAAA,IAC5F;AACA,UAAM;AAAA,EACR;AACF;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
/** @jest-environment node */
|
|
2
|
-
|
|
3
|
-
import { createSafeS3EndpointLookup, UnsafeS3EndpointError } from '../lib/endpoint-safety'
|
|
4
|
-
|
|
5
|
-
function runLookup(
|
|
6
|
-
lookup: NonNullable<ReturnType<typeof createSafeS3EndpointLookup>>,
|
|
7
|
-
hostname: string,
|
|
8
|
-
): Promise<{ address: string; family: number | undefined }> {
|
|
9
|
-
return new Promise((resolve, reject) => {
|
|
10
|
-
lookup(hostname, {}, (err, address, family) => {
|
|
11
|
-
if (err) {
|
|
12
|
-
reject(err)
|
|
13
|
-
return
|
|
14
|
-
}
|
|
15
|
-
if (typeof address !== 'string') {
|
|
16
|
-
reject(new Error('Expected a single lookup result'))
|
|
17
|
-
return
|
|
18
|
-
}
|
|
19
|
-
resolve({ address, family })
|
|
20
|
-
})
|
|
21
|
-
})
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
describe('S3 endpoint safety', () => {
|
|
25
|
-
it('validates the DNS result used by the AWS SDK lookup callback', async () => {
|
|
26
|
-
const lookupHost = jest
|
|
27
|
-
.fn()
|
|
28
|
-
.mockResolvedValueOnce([{ address: '93.184.216.34', family: 4 }])
|
|
29
|
-
.mockResolvedValueOnce([{ address: '169.254.169.254', family: 4 }])
|
|
30
|
-
|
|
31
|
-
const lookup = createSafeS3EndpointLookup('https://objects.example.com', { lookupHost })
|
|
32
|
-
expect(lookup).toBeDefined()
|
|
33
|
-
|
|
34
|
-
await expect(runLookup(lookup!, 'objects.example.com')).resolves.toEqual({
|
|
35
|
-
address: '93.184.216.34',
|
|
36
|
-
family: 4,
|
|
37
|
-
})
|
|
38
|
-
await expect(runLookup(lookup!, 'objects.example.com')).rejects.toMatchObject({
|
|
39
|
-
reason: 'private_ip_resolved',
|
|
40
|
-
})
|
|
41
|
-
})
|
|
42
|
-
|
|
43
|
-
it('validates virtual-hosted bucket endpoint hosts requested by the AWS SDK', async () => {
|
|
44
|
-
const lookupHost = jest.fn().mockResolvedValueOnce([{ address: '93.184.216.34', family: 4 }])
|
|
45
|
-
const lookup = createSafeS3EndpointLookup('https://objects.example.com', { lookupHost })
|
|
46
|
-
|
|
47
|
-
await expect(runLookup(lookup!, 'test-bucket.objects.example.com')).resolves.toEqual({
|
|
48
|
-
address: '93.184.216.34',
|
|
49
|
-
family: 4,
|
|
50
|
-
})
|
|
51
|
-
|
|
52
|
-
expect(lookupHost).toHaveBeenCalledWith('test-bucket.objects.example.com')
|
|
53
|
-
})
|
|
54
|
-
|
|
55
|
-
it('rejects virtual-hosted bucket endpoint hosts that resolve to private addresses', async () => {
|
|
56
|
-
const lookupHost = jest.fn().mockResolvedValueOnce([{ address: '169.254.169.254', family: 4 }])
|
|
57
|
-
const lookup = createSafeS3EndpointLookup('https://objects.example.com', { lookupHost })
|
|
58
|
-
|
|
59
|
-
await expect(runLookup(lookup!, 'test-bucket.objects.example.com')).rejects.toMatchObject({
|
|
60
|
-
reason: 'private_ip_resolved',
|
|
61
|
-
})
|
|
62
|
-
})
|
|
63
|
-
|
|
64
|
-
it('refuses lookup attempts for hosts other than the configured endpoint host', async () => {
|
|
65
|
-
const lookup = createSafeS3EndpointLookup('https://objects.example.com')
|
|
66
|
-
|
|
67
|
-
await expect(runLookup(lookup!, 'metadata.google.internal')).rejects.toBeInstanceOf(
|
|
68
|
-
UnsafeS3EndpointError,
|
|
69
|
-
)
|
|
70
|
-
})
|
|
71
|
-
})
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
/** @jest-environment node */
|
|
2
|
-
|
|
3
|
-
const mockSend = jest.fn()
|
|
4
|
-
|
|
5
|
-
jest.mock('@aws-sdk/client-s3', () => ({
|
|
6
|
-
S3Client: jest.fn().mockImplementation(() => ({ send: mockSend })),
|
|
7
|
-
HeadBucketCommand: jest.fn().mockImplementation((params) => ({ _type: 'HeadBucket', ...params })),
|
|
8
|
-
}))
|
|
9
|
-
|
|
10
|
-
import { S3Client } from '@aws-sdk/client-s3'
|
|
11
|
-
import { s3HealthCheck } from '../lib/health'
|
|
12
|
-
|
|
13
|
-
const BASE_CREDENTIALS = {
|
|
14
|
-
bucket: 'test-bucket',
|
|
15
|
-
region: 'eu-central-1',
|
|
16
|
-
authMode: 'access_keys',
|
|
17
|
-
accessKeyId: 'AKID',
|
|
18
|
-
secretAccessKey: 'SECRET',
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
beforeEach(() => {
|
|
22
|
-
mockSend.mockReset()
|
|
23
|
-
;(S3Client as jest.Mock).mockClear()
|
|
24
|
-
})
|
|
25
|
-
|
|
26
|
-
describe('s3HealthCheck', () => {
|
|
27
|
-
it('rejects link-local custom endpoints before probing the bucket', async () => {
|
|
28
|
-
const result = await s3HealthCheck.check({
|
|
29
|
-
...BASE_CREDENTIALS,
|
|
30
|
-
endpoint: 'http://169.254.169.254/latest/meta-data/',
|
|
31
|
-
})
|
|
32
|
-
|
|
33
|
-
expect(result.status).toBe('unhealthy')
|
|
34
|
-
expect(result.message).toMatch(/private or reserved IP range/i)
|
|
35
|
-
expect(S3Client).not.toHaveBeenCalled()
|
|
36
|
-
expect(mockSend).not.toHaveBeenCalled()
|
|
37
|
-
})
|
|
38
|
-
})
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { createRequire } from 'node:module'
|
|
2
|
-
import { readFileSync } from 'node:fs'
|
|
3
|
-
|
|
4
|
-
const packageRequire = createRequire(__filename)
|
|
5
|
-
|
|
6
|
-
describe('storage_s3 locale package exports', () => {
|
|
7
|
-
it.each(['en', 'de', 'es', 'pl'] as const)('resolves the %s locale through the package export map', (locale) => {
|
|
8
|
-
const localePath = packageRequire.resolve(`@open-mercato/storage-s3/modules/storage_s3/i18n/${locale}.json`)
|
|
9
|
-
const translations = JSON.parse(readFileSync(localePath, 'utf8')) as Record<string, unknown>
|
|
10
|
-
|
|
11
|
-
expect(translations['storage_s3.errors.unauthorized']).toEqual(expect.any(String))
|
|
12
|
-
})
|
|
13
|
-
})
|
|
@@ -1,216 +0,0 @@
|
|
|
1
|
-
/** @jest-environment node */
|
|
2
|
-
|
|
3
|
-
const mockPutObject = jest.fn(async () => {})
|
|
4
|
-
const mockDelete = jest.fn(async () => {})
|
|
5
|
-
const mockGetSignedUrl = jest.fn(async () => 'https://s3.test/signed')
|
|
6
|
-
const mockListObjects = jest.fn(async () => ({ files: [], truncated: false }))
|
|
7
|
-
const mockScheduleRecovery = jest.fn(async () => {})
|
|
8
|
-
|
|
9
|
-
jest.mock('../lib/s3-driver', () => ({
|
|
10
|
-
S3StorageDriver: jest.fn().mockImplementation(() => ({
|
|
11
|
-
putObject: mockPutObject,
|
|
12
|
-
delete: mockDelete,
|
|
13
|
-
deleteStrict: mockDelete,
|
|
14
|
-
getSignedUrl: mockGetSignedUrl,
|
|
15
|
-
listObjects: mockListObjects,
|
|
16
|
-
getBucket: () => 'bucket',
|
|
17
|
-
})),
|
|
18
|
-
}))
|
|
19
|
-
|
|
20
|
-
const mockQuotaService = {
|
|
21
|
-
reconcileStandaloneObjects: jest.fn(async () => {}),
|
|
22
|
-
reserve: jest.fn(async () => ({ id: 'reservation-1', leaseToken: 'lease-1', expiresAt: new Date(Date.now() + 60_000) })),
|
|
23
|
-
claimPendingByUploadTokenHash: jest.fn(async () => null as Record<string, unknown> | null),
|
|
24
|
-
beginStorage: jest.fn(async () => {}),
|
|
25
|
-
markStored: jest.fn(async () => {}),
|
|
26
|
-
completeStandalone: jest.fn(async () => {}),
|
|
27
|
-
release: jest.fn(async () => {}),
|
|
28
|
-
releaseCommittedByPath: jest.fn(async () => {}),
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
jest.mock('@open-mercato/shared/lib/di/container', () => ({
|
|
32
|
-
createRequestContainer: jest.fn(async () => ({
|
|
33
|
-
resolve: (key: string) => {
|
|
34
|
-
if (key === 'integrationCredentialsService') {
|
|
35
|
-
return { resolve: jest.fn(async () => ({ bucket: 'bucket', region: 'us-east-1' })) }
|
|
36
|
-
}
|
|
37
|
-
if (key === 'attachmentQuotaService') return mockQuotaService
|
|
38
|
-
if (key === 'storageS3QuotaRecoveryScheduler') return mockScheduleRecovery
|
|
39
|
-
return null
|
|
40
|
-
},
|
|
41
|
-
})),
|
|
42
|
-
}))
|
|
43
|
-
|
|
44
|
-
jest.mock('@open-mercato/shared/lib/auth/server', () => ({
|
|
45
|
-
getAuthFromRequest: jest.fn(async () => ({ tenantId: 'tenant-1', orgId: 'org-1' })),
|
|
46
|
-
}))
|
|
47
|
-
|
|
48
|
-
function multipartRequest() {
|
|
49
|
-
const form = new FormData()
|
|
50
|
-
form.set('file', new File([new Uint8Array([1, 2, 3])], 'safe.pdf', { type: 'application/pdf' }))
|
|
51
|
-
return new Request('http://localhost/api/storage-providers/s3/upload', { method: 'POST', body: form })
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
describe('storage_s3 quota lifecycle', () => {
|
|
55
|
-
beforeEach(() => {
|
|
56
|
-
jest.clearAllMocks()
|
|
57
|
-
process.env.OM_ATTACHMENT_MAX_UPLOAD_MB = '5'
|
|
58
|
-
process.env.OM_ATTACHMENT_TENANT_QUOTA_MB = '10'
|
|
59
|
-
})
|
|
60
|
-
|
|
61
|
-
afterEach(() => {
|
|
62
|
-
delete process.env.OM_ATTACHMENT_MAX_UPLOAD_MB
|
|
63
|
-
delete process.env.OM_ATTACHMENT_TENANT_QUOTA_MB
|
|
64
|
-
})
|
|
65
|
-
|
|
66
|
-
it('routes multipart uploads through the shared reservation service', async () => {
|
|
67
|
-
const { POST } = await import('../api/post/storage-providers/s3/upload')
|
|
68
|
-
const response = await POST(multipartRequest())
|
|
69
|
-
|
|
70
|
-
expect(response.status).toBe(200)
|
|
71
|
-
expect(mockQuotaService.reserve).toHaveBeenCalledWith(expect.objectContaining({
|
|
72
|
-
tenantId: 'tenant-1',
|
|
73
|
-
organizationId: 'org-1',
|
|
74
|
-
bytes: 3,
|
|
75
|
-
source: 'storage_s3_upload',
|
|
76
|
-
}))
|
|
77
|
-
expect(mockQuotaService.completeStandalone).toHaveBeenCalledWith(
|
|
78
|
-
'reservation-1',
|
|
79
|
-
'lease-1',
|
|
80
|
-
3,
|
|
81
|
-
)
|
|
82
|
-
expect(mockScheduleRecovery).toHaveBeenCalledWith({
|
|
83
|
-
reservationId: 'reservation-1',
|
|
84
|
-
tenantId: 'tenant-1',
|
|
85
|
-
organizationId: 'org-1',
|
|
86
|
-
}, expect.any(Number))
|
|
87
|
-
})
|
|
88
|
-
|
|
89
|
-
it('releases multipart capacity when durable recovery cannot be scheduled', async () => {
|
|
90
|
-
mockScheduleRecovery.mockRejectedValueOnce(new Error('queue unavailable'))
|
|
91
|
-
const { POST } = await import('../api/post/storage-providers/s3/upload')
|
|
92
|
-
|
|
93
|
-
const response = await POST(multipartRequest())
|
|
94
|
-
|
|
95
|
-
expect(response.status).toBe(500)
|
|
96
|
-
expect(mockQuotaService.release).toHaveBeenCalledWith('reservation-1', 'lease-1')
|
|
97
|
-
expect(mockPutObject).not.toHaveBeenCalled()
|
|
98
|
-
})
|
|
99
|
-
|
|
100
|
-
it('reserves exact bytes before generating a signed upload URL', async () => {
|
|
101
|
-
const { POST } = await import('../api/post/storage-providers/s3/signed-url')
|
|
102
|
-
const key = 'uploads/org_org-1/tenant_tenant-1/safe.pdf'
|
|
103
|
-
const response = await POST(new Request('http://localhost/api/storage-providers/s3/signed-url', {
|
|
104
|
-
method: 'POST',
|
|
105
|
-
headers: { 'content-type': 'application/json' },
|
|
106
|
-
body: JSON.stringify({ key, operation: 'upload', contentType: 'application/pdf', size: 3 }),
|
|
107
|
-
}))
|
|
108
|
-
|
|
109
|
-
expect(response.status).toBe(200)
|
|
110
|
-
expect(mockQuotaService.reserve).toHaveBeenCalledWith(expect.objectContaining({
|
|
111
|
-
storagePath: key,
|
|
112
|
-
bytes: 3,
|
|
113
|
-
source: 'storage_s3_signed',
|
|
114
|
-
}))
|
|
115
|
-
expect(mockQuotaService.reconcileStandaloneObjects).toHaveBeenCalled()
|
|
116
|
-
expect(await response.json()).toEqual(expect.objectContaining({
|
|
117
|
-
reservationId: 'reservation-1',
|
|
118
|
-
url: expect.stringContaining('/api/storage-providers/s3/signed-upload/'),
|
|
119
|
-
}))
|
|
120
|
-
})
|
|
121
|
-
|
|
122
|
-
it('atomically consumes a compatibility token so replay cannot delete the winning object', async () => {
|
|
123
|
-
mockQuotaService.claimPendingByUploadTokenHash
|
|
124
|
-
.mockResolvedValueOnce({
|
|
125
|
-
id: 'reservation-1',
|
|
126
|
-
leaseToken: 'lease-claimed',
|
|
127
|
-
source: 'storage_s3_signed',
|
|
128
|
-
reservedBytes: 3,
|
|
129
|
-
storagePath: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
130
|
-
tenantId: 'tenant-1',
|
|
131
|
-
organizationId: 'org-1',
|
|
132
|
-
})
|
|
133
|
-
.mockResolvedValueOnce(null)
|
|
134
|
-
const { PUT } = await import('../api/put/storage-providers/s3/signed-upload/[token]')
|
|
135
|
-
const request = () => new Request('http://localhost/api/storage-providers/s3/signed-upload/token', {
|
|
136
|
-
method: 'PUT',
|
|
137
|
-
headers: { 'content-type': 'application/pdf' },
|
|
138
|
-
body: new Uint8Array([1, 2, 3]),
|
|
139
|
-
})
|
|
140
|
-
|
|
141
|
-
const first = await PUT(request(), { params: { token: 'token' } })
|
|
142
|
-
const replay = await PUT(request(), { params: { token: 'token' } })
|
|
143
|
-
|
|
144
|
-
expect(first.status).toBe(200)
|
|
145
|
-
expect(replay.status).toBe(410)
|
|
146
|
-
expect(mockPutObject).toHaveBeenCalledTimes(1)
|
|
147
|
-
expect(mockDelete).not.toHaveBeenCalled()
|
|
148
|
-
})
|
|
149
|
-
|
|
150
|
-
it('stops a chunked compatibility upload as soon as it crosses the reserved bound', async () => {
|
|
151
|
-
mockQuotaService.claimPendingByUploadTokenHash.mockResolvedValueOnce({
|
|
152
|
-
id: 'reservation-1',
|
|
153
|
-
leaseToken: 'lease-claimed',
|
|
154
|
-
source: 'storage_s3_signed',
|
|
155
|
-
reservedBytes: 3,
|
|
156
|
-
storagePath: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
157
|
-
tenantId: 'tenant-1',
|
|
158
|
-
organizationId: 'org-1',
|
|
159
|
-
})
|
|
160
|
-
const stream = new ReadableStream<Uint8Array>({
|
|
161
|
-
start(controller) {
|
|
162
|
-
controller.enqueue(new Uint8Array([1, 2]))
|
|
163
|
-
controller.enqueue(new Uint8Array([3, 4]))
|
|
164
|
-
controller.close()
|
|
165
|
-
},
|
|
166
|
-
})
|
|
167
|
-
const { PUT } = await import('../api/put/storage-providers/s3/signed-upload/[token]')
|
|
168
|
-
const response = await PUT(new Request('http://localhost/api/storage-providers/s3/signed-upload/token', {
|
|
169
|
-
method: 'PUT',
|
|
170
|
-
body: stream,
|
|
171
|
-
duplex: 'half',
|
|
172
|
-
} as RequestInit & { duplex: 'half' }), { params: { token: 'token' } })
|
|
173
|
-
|
|
174
|
-
expect(response.status).toBe(413)
|
|
175
|
-
expect(mockPutObject).not.toHaveBeenCalled()
|
|
176
|
-
expect(mockQuotaService.release).toHaveBeenCalledWith('reservation-1', 'lease-claimed')
|
|
177
|
-
})
|
|
178
|
-
|
|
179
|
-
it('removes committed quota only after standalone object deletion succeeds', async () => {
|
|
180
|
-
const { DELETE } = await import('../api/delete/storage-providers/s3/delete')
|
|
181
|
-
const key = 'uploads/org_org-1/tenant_tenant-1/safe.pdf'
|
|
182
|
-
const response = await DELETE(new Request('http://localhost/api/storage-providers/s3/delete', {
|
|
183
|
-
method: 'DELETE',
|
|
184
|
-
headers: { 'content-type': 'application/json' },
|
|
185
|
-
body: JSON.stringify({ key }),
|
|
186
|
-
}))
|
|
187
|
-
|
|
188
|
-
expect(response.status).toBe(204)
|
|
189
|
-
expect(mockDelete).toHaveBeenCalled()
|
|
190
|
-
expect(mockQuotaService.releaseCommittedByPath).toHaveBeenCalledWith(expect.objectContaining({
|
|
191
|
-
tenantId: 'tenant-1',
|
|
192
|
-
storageDriver: 's3',
|
|
193
|
-
storagePath: key,
|
|
194
|
-
}))
|
|
195
|
-
})
|
|
196
|
-
|
|
197
|
-
it('releases a programmatic signed-url reservation when recovery scheduling fails', async () => {
|
|
198
|
-
const { createStorageService } = await import('../lib/storage-service')
|
|
199
|
-
const service = createStorageService({
|
|
200
|
-
bucket: 'bucket',
|
|
201
|
-
region: 'us-east-1',
|
|
202
|
-
quotaService: mockQuotaService as never,
|
|
203
|
-
quotaRecoveryScheduler: jest.fn(async () => {
|
|
204
|
-
throw new Error('queue unavailable')
|
|
205
|
-
}),
|
|
206
|
-
})
|
|
207
|
-
|
|
208
|
-
await expect(service.getSignedUrl({
|
|
209
|
-
key: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
210
|
-
operation: 'upload',
|
|
211
|
-
size: 3,
|
|
212
|
-
scope: { tenantId: 'tenant-1', organizationId: 'org-1' },
|
|
213
|
-
})).rejects.toThrow('queue unavailable')
|
|
214
|
-
expect(mockQuotaService.release).toHaveBeenCalledWith('reservation-1', 'lease-1')
|
|
215
|
-
})
|
|
216
|
-
})
|