@open-mercato/storage-s3 0.6.7 → 0.6.8-develop.6874.1.982d6097d8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/storage_s3/api/delete/storage-providers/s3/delete.js +20 -5
- package/dist/modules/storage_s3/api/delete/storage-providers/s3/delete.js.map +2 -2
- package/dist/modules/storage_s3/api/get/storage-providers/s3/download.js +7 -5
- package/dist/modules/storage_s3/api/get/storage-providers/s3/download.js.map +2 -2
- package/dist/modules/storage_s3/api/get/storage-providers/s3/list.js +6 -4
- package/dist/modules/storage_s3/api/get/storage-providers/s3/list.js.map +2 -2
- package/dist/modules/storage_s3/api/post/storage-providers/s3/signed-url.js +70 -8
- package/dist/modules/storage_s3/api/post/storage-providers/s3/signed-url.js.map +3 -3
- package/dist/modules/storage_s3/api/post/storage-providers/s3/upload.js +92 -15
- package/dist/modules/storage_s3/api/post/storage-providers/s3/upload.js.map +2 -2
- package/dist/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].js +145 -0
- package/dist/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].js.map +7 -0
- package/dist/modules/storage_s3/di.js +6 -2
- package/dist/modules/storage_s3/di.js.map +2 -2
- package/dist/modules/storage_s3/integration.js +1 -1
- package/dist/modules/storage_s3/integration.js.map +1 -1
- package/dist/modules/storage_s3/lib/endpoint-safety.js +133 -0
- package/dist/modules/storage_s3/lib/endpoint-safety.js.map +7 -0
- package/dist/modules/storage_s3/lib/health.js +18 -1
- package/dist/modules/storage_s3/lib/health.js.map +2 -2
- package/dist/modules/storage_s3/lib/key-scope.js +4 -4
- package/dist/modules/storage_s3/lib/key-scope.js.map +2 -2
- package/dist/modules/storage_s3/lib/quota-accounting.js +24 -0
- package/dist/modules/storage_s3/lib/quota-accounting.js.map +7 -0
- package/dist/modules/storage_s3/lib/quota-recovery-queue.js +9 -0
- package/dist/modules/storage_s3/lib/quota-recovery-queue.js.map +7 -0
- package/dist/modules/storage_s3/lib/s3-driver.js +44 -15
- package/dist/modules/storage_s3/lib/s3-driver.js.map +2 -2
- package/dist/modules/storage_s3/lib/storage-service.js +126 -22
- package/dist/modules/storage_s3/lib/storage-service.js.map +3 -3
- package/dist/modules/storage_s3/workers/quota-recovery.js +70 -0
- package/dist/modules/storage_s3/workers/quota-recovery.js.map +7 -0
- package/jest.config.cjs +1 -0
- package/package.json +13 -6
- package/src/modules/storage_s3/__tests__/endpoint-safety.test.ts +71 -0
- package/src/modules/storage_s3/__tests__/health.test.ts +38 -0
- package/src/modules/storage_s3/__tests__/i18nPackageExports.test.ts +13 -0
- package/src/modules/storage_s3/__tests__/quota-lifecycle.test.ts +216 -0
- package/src/modules/storage_s3/__tests__/quota-recovery.test.ts +83 -0
- package/src/modules/storage_s3/__tests__/s3Driver.test.ts +23 -8
- package/src/modules/storage_s3/__tests__/s3ListRoute.test.ts +7 -1
- package/src/modules/storage_s3/__tests__/s3Routes.test.ts +26 -0
- package/src/modules/storage_s3/__tests__/storageService.test.ts +20 -5
- package/src/modules/storage_s3/__tests__/upload.test.ts +93 -0
- package/src/modules/storage_s3/api/delete/storage-providers/s3/delete.ts +22 -5
- package/src/modules/storage_s3/api/get/storage-providers/s3/download.ts +7 -5
- package/src/modules/storage_s3/api/get/storage-providers/s3/list.ts +6 -4
- package/src/modules/storage_s3/api/post/storage-providers/s3/signed-url.ts +77 -6
- package/src/modules/storage_s3/api/post/storage-providers/s3/upload.ts +101 -16
- package/src/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].ts +150 -0
- package/src/modules/storage_s3/di.ts +14 -2
- package/src/modules/storage_s3/i18n/de.json +19 -0
- package/src/modules/storage_s3/i18n/en.json +19 -0
- package/src/modules/storage_s3/i18n/es.json +19 -0
- package/src/modules/storage_s3/i18n/ko.json +19 -0
- package/src/modules/storage_s3/i18n/pl.json +19 -0
- package/src/modules/storage_s3/integration.ts +1 -1
- package/src/modules/storage_s3/lib/endpoint-safety.ts +179 -0
- package/src/modules/storage_s3/lib/health.ts +18 -0
- package/src/modules/storage_s3/lib/key-scope.ts +4 -4
- package/src/modules/storage_s3/lib/quota-accounting.ts +29 -0
- package/src/modules/storage_s3/lib/quota-recovery-queue.ts +17 -0
- package/src/modules/storage_s3/lib/s3-driver.ts +65 -10
- package/src/modules/storage_s3/lib/storage-service.ts +131 -22
- package/src/modules/storage_s3/workers/quota-recovery.ts +82 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { S3StorageDriver } from "../lib/s3-driver.js";
|
|
2
|
+
import { scheduleStorageS3QuotaRecovery } from "../lib/quota-recovery-queue.js";
|
|
3
|
+
const metadata = {
|
|
4
|
+
queue: "storage-s3-quota-recovery",
|
|
5
|
+
id: "storage_s3:quota-recovery",
|
|
6
|
+
concurrency: 2
|
|
7
|
+
};
|
|
8
|
+
async function handle(job, ctx) {
|
|
9
|
+
const quotaService = ctx.resolve("attachmentQuotaService");
|
|
10
|
+
const scope = {
|
|
11
|
+
tenantId: job.payload.tenantId,
|
|
12
|
+
organizationId: job.payload.organizationId
|
|
13
|
+
};
|
|
14
|
+
const record = await quotaService.getReservation(job.payload.reservationId, scope);
|
|
15
|
+
if (!record || record.status === "committed") return;
|
|
16
|
+
if (record.expiresAt && record.expiresAt.getTime() > Date.now()) {
|
|
17
|
+
await scheduleStorageS3QuotaRecovery(job.payload, record.expiresAt.getTime() - Date.now());
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
const claimed = await quotaService.claimExpired(record.id, scope);
|
|
21
|
+
if (!claimed) return;
|
|
22
|
+
try {
|
|
23
|
+
const credentialsService = ctx.resolve("integrationCredentialsService");
|
|
24
|
+
const credentials = await credentialsService.resolve("storage_s3", {
|
|
25
|
+
tenantId: claimed.tenantId,
|
|
26
|
+
organizationId: claimed.organizationId
|
|
27
|
+
});
|
|
28
|
+
if (!credentials) throw new Error("S3 integration is not configured for quota recovery.");
|
|
29
|
+
const driver = new S3StorageDriver({
|
|
30
|
+
...credentials,
|
|
31
|
+
organizationId: claimed.organizationId,
|
|
32
|
+
tenantId: claimed.tenantId
|
|
33
|
+
});
|
|
34
|
+
if (claimed.source === "storage_s3_signed") {
|
|
35
|
+
const listed = await driver.listObjects(claimed.storagePath, 1);
|
|
36
|
+
const object = listed.files.find((file) => file.key === claimed.storagePath);
|
|
37
|
+
if (!object) {
|
|
38
|
+
if (record.status === "storing" && (job.payload.absenceCheck ?? 0) === 0 && claimed.expiresAt) {
|
|
39
|
+
await scheduleStorageS3QuotaRecovery(
|
|
40
|
+
{
|
|
41
|
+
...job.payload,
|
|
42
|
+
absenceCheck: 1
|
|
43
|
+
},
|
|
44
|
+
claimed.expiresAt.getTime() - Date.now()
|
|
45
|
+
);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
await quotaService.release(claimed.id, claimed.leaseToken);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const validSize = claimed.uploadTokenHash ? object.size <= claimed.reservedBytes : object.size === claimed.reservedBytes;
|
|
52
|
+
if (validSize) {
|
|
53
|
+
await quotaService.commitRecoveredStandalone(claimed.id, claimed.leaseToken, object.size);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
await driver.deleteStrict("", claimed.storagePath);
|
|
58
|
+
await quotaService.release(claimed.id, claimed.leaseToken);
|
|
59
|
+
} catch (error) {
|
|
60
|
+
if (claimed.expiresAt) {
|
|
61
|
+
await scheduleStorageS3QuotaRecovery(job.payload, claimed.expiresAt.getTime() - Date.now());
|
|
62
|
+
}
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
export {
|
|
67
|
+
handle as default,
|
|
68
|
+
metadata
|
|
69
|
+
};
|
|
70
|
+
//# sourceMappingURL=quota-recovery.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/storage_s3/workers/quota-recovery.ts"],
|
|
4
|
+
"sourcesContent": ["import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'\nimport type { AttachmentQuotaService } from '@open-mercato/core/modules/attachments/lib/quota-service'\nimport { S3StorageDriver } from '../lib/s3-driver'\nimport { scheduleStorageS3QuotaRecovery, type StorageS3QuotaRecoveryJob } from '../lib/quota-recovery-queue'\n\nexport const metadata: WorkerMeta = {\n queue: 'storage-s3-quota-recovery',\n id: 'storage_s3:quota-recovery',\n concurrency: 2,\n}\n\ntype HandlerContext = JobContext & { resolve: <T = unknown>(name: string) => T }\n\ntype IntegrationCredentialsService = {\n resolve(integrationId: string, scope: { tenantId: string; organizationId: string }): Promise<Record<string, unknown> | null>\n}\n\nexport default async function handle(\n job: QueuedJob<StorageS3QuotaRecoveryJob>,\n ctx: HandlerContext,\n): Promise<void> {\n const quotaService = ctx.resolve<AttachmentQuotaService>('attachmentQuotaService')\n const scope = {\n tenantId: job.payload.tenantId,\n organizationId: job.payload.organizationId,\n }\n const record = await quotaService.getReservation(job.payload.reservationId, scope)\n if (!record || record.status === 'committed') return\n if (record.expiresAt && record.expiresAt.getTime() > Date.now()) {\n await scheduleStorageS3QuotaRecovery(job.payload, record.expiresAt.getTime() - Date.now())\n return\n }\n\n const claimed = await quotaService.claimExpired(record.id, scope)\n if (!claimed) return\n try {\n const credentialsService = ctx.resolve<IntegrationCredentialsService>('integrationCredentialsService')\n const credentials = await credentialsService.resolve('storage_s3', {\n tenantId: claimed.tenantId,\n organizationId: claimed.organizationId,\n })\n if (!credentials) throw new Error('S3 integration is not configured for quota recovery.')\n const driver = new S3StorageDriver({\n ...credentials,\n organizationId: claimed.organizationId,\n tenantId: claimed.tenantId,\n })\n\n if (claimed.source === 'storage_s3_signed') {\n const listed = await driver.listObjects(claimed.storagePath, 1)\n const object = listed.files.find((file) => file.key === claimed.storagePath)\n if (!object) {\n if (record.status === 'storing' && (job.payload.absenceCheck ?? 0) === 0 && claimed.expiresAt) {\n await scheduleStorageS3QuotaRecovery({\n ...job.payload,\n absenceCheck: 1,\n },\n claimed.expiresAt.getTime() - Date.now(),\n )\n return\n }\n await quotaService.release(claimed.id, claimed.leaseToken)\n return\n }\n const validSize = claimed.uploadTokenHash\n ? object.size <= claimed.reservedBytes\n : object.size === claimed.reservedBytes\n if (validSize) {\n await quotaService.commitRecoveredStandalone(claimed.id, claimed.leaseToken, object.size)\n return\n }\n }\n\n await driver.deleteStrict('', claimed.storagePath)\n await quotaService.release(claimed.id, claimed.leaseToken)\n } catch (error) {\n if (claimed.expiresAt) {\n await scheduleStorageS3QuotaRecovery(job.payload, claimed.expiresAt.getTime() - Date.now())\n }\n throw error\n }\n}\n"],
|
|
5
|
+
"mappings": "AAEA,SAAS,uBAAuB;AAChC,SAAS,sCAAsE;AAExE,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,IAAI;AAAA,EACJ,aAAa;AACf;AAQA,eAAO,OACL,KACA,KACe;AACf,QAAM,eAAe,IAAI,QAAgC,wBAAwB;AACjF,QAAM,QAAQ;AAAA,IACZ,UAAU,IAAI,QAAQ;AAAA,IACtB,gBAAgB,IAAI,QAAQ;AAAA,EAC9B;AACA,QAAM,SAAS,MAAM,aAAa,eAAe,IAAI,QAAQ,eAAe,KAAK;AACjF,MAAI,CAAC,UAAU,OAAO,WAAW,YAAa;AAC9C,MAAI,OAAO,aAAa,OAAO,UAAU,QAAQ,IAAI,KAAK,IAAI,GAAG;AAC/D,UAAM,+BAA+B,IAAI,SAAS,OAAO,UAAU,QAAQ,IAAI,KAAK,IAAI,CAAC;AACzF;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,aAAa,aAAa,OAAO,IAAI,KAAK;AAChE,MAAI,CAAC,QAAS;AACd,MAAI;AACF,UAAM,qBAAqB,IAAI,QAAuC,+BAA+B;AACrG,UAAM,cAAc,MAAM,mBAAmB,QAAQ,cAAc;AAAA,MACjE,UAAU,QAAQ;AAAA,MAClB,gBAAgB,QAAQ;AAAA,IAC1B,CAAC;AACD,QAAI,CAAC,YAAa,OAAM,IAAI,MAAM,sDAAsD;AACxF,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC,GAAG;AAAA,MACH,gBAAgB,QAAQ;AAAA,MACxB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAED,QAAI,QAAQ,WAAW,qBAAqB;AAC1C,YAAM,SAAS,MAAM,OAAO,YAAY,QAAQ,aAAa,CAAC;AAC9D,YAAM,SAAS,OAAO,MAAM,KAAK,CAAC,SAAS,KAAK,QAAQ,QAAQ,WAAW;AAC3E,UAAI,CAAC,QAAQ;AACX,YAAI,OAAO,WAAW,cAAc,IAAI,QAAQ,gBAAgB,OAAO,KAAK,QAAQ,WAAW;AAC7F,gBAAM;AAAA,YAA+B;AAAA,cACnC,GAAG,IAAI;AAAA,cACP,cAAc;AAAA,YAChB;AAAA,YACE,QAAQ,UAAU,QAAQ,IAAI,KAAK,IAAI;AAAA,UACzC;AACA;AAAA,QACF;AACA,cAAM,aAAa,QAAQ,QAAQ,IAAI,QAAQ,UAAU;AACzD;AAAA,MACF;AACA,YAAM,YAAY,QAAQ,kBACtB,OAAO,QAAQ,QAAQ,gBACvB,OAAO,SAAS,QAAQ;AAC5B,UAAI,WAAW;AACb,cAAM,aAAa,0BAA0B,QAAQ,IAAI,QAAQ,YAAY,OAAO,IAAI;AACxF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,OAAO,aAAa,IAAI,QAAQ,WAAW;AACjD,UAAM,aAAa,QAAQ,QAAQ,IAAI,QAAQ,UAAU;AAAA,EAC3D,SAAS,OAAO;AACd,QAAI,QAAQ,WAAW;AACrB,YAAM,+BAA+B,IAAI,SAAS,QAAQ,UAAU,QAAQ,IAAI,KAAK,IAAI,CAAC;AAAA,IAC5F;AACA,UAAM;AAAA,EACR;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/jest.config.cjs
CHANGED
|
@@ -9,6 +9,7 @@ module.exports = {
|
|
|
9
9
|
moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
|
|
10
10
|
moduleNameMapper: {
|
|
11
11
|
'^@open-mercato/core/(.*)$': '<rootDir>/../core/src/$1',
|
|
12
|
+
'^@open-mercato/queue$': '<rootDir>/../queue/src/index.ts',
|
|
12
13
|
'^@open-mercato/shared/(.*)$': '<rootDir>/../shared/src/$1',
|
|
13
14
|
},
|
|
14
15
|
transform: {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/storage-s3",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.8-develop.6874.1.982d6097d8",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
"types": "./src/*.ts",
|
|
17
17
|
"default": "./dist/*.js"
|
|
18
18
|
},
|
|
19
|
+
"./*.json": "./src/*.json",
|
|
19
20
|
"./*": {
|
|
20
21
|
"types": [
|
|
21
22
|
"./src/*.ts",
|
|
@@ -23,6 +24,7 @@
|
|
|
23
24
|
],
|
|
24
25
|
"default": "./dist/*.js"
|
|
25
26
|
},
|
|
27
|
+
"./*/*.json": "./src/*/*.json",
|
|
26
28
|
"./*/*": {
|
|
27
29
|
"types": [
|
|
28
30
|
"./src/*/*.ts",
|
|
@@ -30,6 +32,7 @@
|
|
|
30
32
|
],
|
|
31
33
|
"default": "./dist/*/*.js"
|
|
32
34
|
},
|
|
35
|
+
"./*/*/*.json": "./src/*/*/*.json",
|
|
33
36
|
"./*/*/*": {
|
|
34
37
|
"types": [
|
|
35
38
|
"./src/*/*/*.ts",
|
|
@@ -37,6 +40,7 @@
|
|
|
37
40
|
],
|
|
38
41
|
"default": "./dist/*/*/*.js"
|
|
39
42
|
},
|
|
43
|
+
"./*/*/*/*.json": "./src/*/*/*/*.json",
|
|
40
44
|
"./*/*/*/*": {
|
|
41
45
|
"types": [
|
|
42
46
|
"./src/*/*/*/*.ts",
|
|
@@ -44,6 +48,7 @@
|
|
|
44
48
|
],
|
|
45
49
|
"default": "./dist/*/*/*/*.js"
|
|
46
50
|
},
|
|
51
|
+
"./*/*/*/*/*.json": "./src/*/*/*/*/*.json",
|
|
47
52
|
"./*/*/*/*/*": {
|
|
48
53
|
"types": [
|
|
49
54
|
"./src/*/*/*/*/*.ts",
|
|
@@ -69,19 +74,20 @@
|
|
|
69
74
|
"dependencies": {
|
|
70
75
|
"@aws-sdk/client-s3": "^3.1085.0",
|
|
71
76
|
"@aws-sdk/s3-request-presigner": "^3.1085.0",
|
|
72
|
-
"@open-mercato/core": "0.6.
|
|
77
|
+
"@open-mercato/core": "0.6.8-develop.6874.1.982d6097d8",
|
|
78
|
+
"@open-mercato/queue": "0.6.8-develop.6874.1.982d6097d8"
|
|
73
79
|
},
|
|
74
80
|
"peerDependencies": {
|
|
75
81
|
"@mikro-orm/postgresql": "^6.6.10",
|
|
76
|
-
"@open-mercato/shared": "0.6.
|
|
82
|
+
"@open-mercato/shared": "0.6.8-develop.6874.1.982d6097d8"
|
|
77
83
|
},
|
|
78
84
|
"devDependencies": {
|
|
79
|
-
"@open-mercato/shared": "0.6.
|
|
85
|
+
"@open-mercato/shared": "0.6.8-develop.6874.1.982d6097d8",
|
|
80
86
|
"@types/jest": "^30.0.0",
|
|
81
87
|
"esbuild": "^0.28.1",
|
|
82
88
|
"glob": "^13.0.6",
|
|
83
89
|
"jest": "^30.4.2",
|
|
84
|
-
"ts-jest": "^29.4.
|
|
90
|
+
"ts-jest": "^29.4.12",
|
|
85
91
|
"typescript": "7.0.2"
|
|
86
92
|
},
|
|
87
93
|
"publishConfig": {
|
|
@@ -91,5 +97,6 @@
|
|
|
91
97
|
"type": "git",
|
|
92
98
|
"url": "https://github.com/open-mercato/open-mercato",
|
|
93
99
|
"directory": "packages/storage-s3"
|
|
94
|
-
}
|
|
100
|
+
},
|
|
101
|
+
"stableVersion": "0.6.7"
|
|
95
102
|
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/** @jest-environment node */
|
|
2
|
+
|
|
3
|
+
import { createSafeS3EndpointLookup, UnsafeS3EndpointError } from '../lib/endpoint-safety'
|
|
4
|
+
|
|
5
|
+
function runLookup(
|
|
6
|
+
lookup: NonNullable<ReturnType<typeof createSafeS3EndpointLookup>>,
|
|
7
|
+
hostname: string,
|
|
8
|
+
): Promise<{ address: string; family: number | undefined }> {
|
|
9
|
+
return new Promise((resolve, reject) => {
|
|
10
|
+
lookup(hostname, {}, (err, address, family) => {
|
|
11
|
+
if (err) {
|
|
12
|
+
reject(err)
|
|
13
|
+
return
|
|
14
|
+
}
|
|
15
|
+
if (typeof address !== 'string') {
|
|
16
|
+
reject(new Error('Expected a single lookup result'))
|
|
17
|
+
return
|
|
18
|
+
}
|
|
19
|
+
resolve({ address, family })
|
|
20
|
+
})
|
|
21
|
+
})
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe('S3 endpoint safety', () => {
|
|
25
|
+
it('validates the DNS result used by the AWS SDK lookup callback', async () => {
|
|
26
|
+
const lookupHost = jest
|
|
27
|
+
.fn()
|
|
28
|
+
.mockResolvedValueOnce([{ address: '93.184.216.34', family: 4 }])
|
|
29
|
+
.mockResolvedValueOnce([{ address: '169.254.169.254', family: 4 }])
|
|
30
|
+
|
|
31
|
+
const lookup = createSafeS3EndpointLookup('https://objects.example.com', { lookupHost })
|
|
32
|
+
expect(lookup).toBeDefined()
|
|
33
|
+
|
|
34
|
+
await expect(runLookup(lookup!, 'objects.example.com')).resolves.toEqual({
|
|
35
|
+
address: '93.184.216.34',
|
|
36
|
+
family: 4,
|
|
37
|
+
})
|
|
38
|
+
await expect(runLookup(lookup!, 'objects.example.com')).rejects.toMatchObject({
|
|
39
|
+
reason: 'private_ip_resolved',
|
|
40
|
+
})
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
it('validates virtual-hosted bucket endpoint hosts requested by the AWS SDK', async () => {
|
|
44
|
+
const lookupHost = jest.fn().mockResolvedValueOnce([{ address: '93.184.216.34', family: 4 }])
|
|
45
|
+
const lookup = createSafeS3EndpointLookup('https://objects.example.com', { lookupHost })
|
|
46
|
+
|
|
47
|
+
await expect(runLookup(lookup!, 'test-bucket.objects.example.com')).resolves.toEqual({
|
|
48
|
+
address: '93.184.216.34',
|
|
49
|
+
family: 4,
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
expect(lookupHost).toHaveBeenCalledWith('test-bucket.objects.example.com')
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
it('rejects virtual-hosted bucket endpoint hosts that resolve to private addresses', async () => {
|
|
56
|
+
const lookupHost = jest.fn().mockResolvedValueOnce([{ address: '169.254.169.254', family: 4 }])
|
|
57
|
+
const lookup = createSafeS3EndpointLookup('https://objects.example.com', { lookupHost })
|
|
58
|
+
|
|
59
|
+
await expect(runLookup(lookup!, 'test-bucket.objects.example.com')).rejects.toMatchObject({
|
|
60
|
+
reason: 'private_ip_resolved',
|
|
61
|
+
})
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('refuses lookup attempts for hosts other than the configured endpoint host', async () => {
|
|
65
|
+
const lookup = createSafeS3EndpointLookup('https://objects.example.com')
|
|
66
|
+
|
|
67
|
+
await expect(runLookup(lookup!, 'metadata.google.internal')).rejects.toBeInstanceOf(
|
|
68
|
+
UnsafeS3EndpointError,
|
|
69
|
+
)
|
|
70
|
+
})
|
|
71
|
+
})
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/** @jest-environment node */
|
|
2
|
+
|
|
3
|
+
const mockSend = jest.fn()
|
|
4
|
+
|
|
5
|
+
jest.mock('@aws-sdk/client-s3', () => ({
|
|
6
|
+
S3Client: jest.fn().mockImplementation(() => ({ send: mockSend })),
|
|
7
|
+
HeadBucketCommand: jest.fn().mockImplementation((params) => ({ _type: 'HeadBucket', ...params })),
|
|
8
|
+
}))
|
|
9
|
+
|
|
10
|
+
import { S3Client } from '@aws-sdk/client-s3'
|
|
11
|
+
import { s3HealthCheck } from '../lib/health'
|
|
12
|
+
|
|
13
|
+
const BASE_CREDENTIALS = {
|
|
14
|
+
bucket: 'test-bucket',
|
|
15
|
+
region: 'eu-central-1',
|
|
16
|
+
authMode: 'access_keys',
|
|
17
|
+
accessKeyId: 'AKID',
|
|
18
|
+
secretAccessKey: 'SECRET',
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
mockSend.mockReset()
|
|
23
|
+
;(S3Client as jest.Mock).mockClear()
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
describe('s3HealthCheck', () => {
|
|
27
|
+
it('rejects link-local custom endpoints before probing the bucket', async () => {
|
|
28
|
+
const result = await s3HealthCheck.check({
|
|
29
|
+
...BASE_CREDENTIALS,
|
|
30
|
+
endpoint: 'http://169.254.169.254/latest/meta-data/',
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
expect(result.status).toBe('unhealthy')
|
|
34
|
+
expect(result.message).toMatch(/private or reserved IP range/i)
|
|
35
|
+
expect(S3Client).not.toHaveBeenCalled()
|
|
36
|
+
expect(mockSend).not.toHaveBeenCalled()
|
|
37
|
+
})
|
|
38
|
+
})
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { createRequire } from 'node:module'
|
|
2
|
+
import { readFileSync } from 'node:fs'
|
|
3
|
+
|
|
4
|
+
const packageRequire = createRequire(__filename)
|
|
5
|
+
|
|
6
|
+
describe('storage_s3 locale package exports', () => {
|
|
7
|
+
it.each(['en', 'de', 'es', 'pl'] as const)('resolves the %s locale through the package export map', (locale) => {
|
|
8
|
+
const localePath = packageRequire.resolve(`@open-mercato/storage-s3/modules/storage_s3/i18n/${locale}.json`)
|
|
9
|
+
const translations = JSON.parse(readFileSync(localePath, 'utf8')) as Record<string, unknown>
|
|
10
|
+
|
|
11
|
+
expect(translations['storage_s3.errors.unauthorized']).toEqual(expect.any(String))
|
|
12
|
+
})
|
|
13
|
+
})
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/** @jest-environment node */
|
|
2
|
+
|
|
3
|
+
const mockPutObject = jest.fn(async () => {})
|
|
4
|
+
const mockDelete = jest.fn(async () => {})
|
|
5
|
+
const mockGetSignedUrl = jest.fn(async () => 'https://s3.test/signed')
|
|
6
|
+
const mockListObjects = jest.fn(async () => ({ files: [], truncated: false }))
|
|
7
|
+
const mockScheduleRecovery = jest.fn(async () => {})
|
|
8
|
+
|
|
9
|
+
jest.mock('../lib/s3-driver', () => ({
|
|
10
|
+
S3StorageDriver: jest.fn().mockImplementation(() => ({
|
|
11
|
+
putObject: mockPutObject,
|
|
12
|
+
delete: mockDelete,
|
|
13
|
+
deleteStrict: mockDelete,
|
|
14
|
+
getSignedUrl: mockGetSignedUrl,
|
|
15
|
+
listObjects: mockListObjects,
|
|
16
|
+
getBucket: () => 'bucket',
|
|
17
|
+
})),
|
|
18
|
+
}))
|
|
19
|
+
|
|
20
|
+
const mockQuotaService = {
|
|
21
|
+
reconcileStandaloneObjects: jest.fn(async () => {}),
|
|
22
|
+
reserve: jest.fn(async () => ({ id: 'reservation-1', leaseToken: 'lease-1', expiresAt: new Date(Date.now() + 60_000) })),
|
|
23
|
+
claimPendingByUploadTokenHash: jest.fn(async () => null as Record<string, unknown> | null),
|
|
24
|
+
beginStorage: jest.fn(async () => {}),
|
|
25
|
+
markStored: jest.fn(async () => {}),
|
|
26
|
+
completeStandalone: jest.fn(async () => {}),
|
|
27
|
+
release: jest.fn(async () => {}),
|
|
28
|
+
releaseCommittedByPath: jest.fn(async () => {}),
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
jest.mock('@open-mercato/shared/lib/di/container', () => ({
|
|
32
|
+
createRequestContainer: jest.fn(async () => ({
|
|
33
|
+
resolve: (key: string) => {
|
|
34
|
+
if (key === 'integrationCredentialsService') {
|
|
35
|
+
return { resolve: jest.fn(async () => ({ bucket: 'bucket', region: 'us-east-1' })) }
|
|
36
|
+
}
|
|
37
|
+
if (key === 'attachmentQuotaService') return mockQuotaService
|
|
38
|
+
if (key === 'storageS3QuotaRecoveryScheduler') return mockScheduleRecovery
|
|
39
|
+
return null
|
|
40
|
+
},
|
|
41
|
+
})),
|
|
42
|
+
}))
|
|
43
|
+
|
|
44
|
+
jest.mock('@open-mercato/shared/lib/auth/server', () => ({
|
|
45
|
+
getAuthFromRequest: jest.fn(async () => ({ tenantId: 'tenant-1', orgId: 'org-1' })),
|
|
46
|
+
}))
|
|
47
|
+
|
|
48
|
+
function multipartRequest() {
|
|
49
|
+
const form = new FormData()
|
|
50
|
+
form.set('file', new File([new Uint8Array([1, 2, 3])], 'safe.pdf', { type: 'application/pdf' }))
|
|
51
|
+
return new Request('http://localhost/api/storage-providers/s3/upload', { method: 'POST', body: form })
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
describe('storage_s3 quota lifecycle', () => {
|
|
55
|
+
beforeEach(() => {
|
|
56
|
+
jest.clearAllMocks()
|
|
57
|
+
process.env.OM_ATTACHMENT_MAX_UPLOAD_MB = '5'
|
|
58
|
+
process.env.OM_ATTACHMENT_TENANT_QUOTA_MB = '10'
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
afterEach(() => {
|
|
62
|
+
delete process.env.OM_ATTACHMENT_MAX_UPLOAD_MB
|
|
63
|
+
delete process.env.OM_ATTACHMENT_TENANT_QUOTA_MB
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
it('routes multipart uploads through the shared reservation service', async () => {
|
|
67
|
+
const { POST } = await import('../api/post/storage-providers/s3/upload')
|
|
68
|
+
const response = await POST(multipartRequest())
|
|
69
|
+
|
|
70
|
+
expect(response.status).toBe(200)
|
|
71
|
+
expect(mockQuotaService.reserve).toHaveBeenCalledWith(expect.objectContaining({
|
|
72
|
+
tenantId: 'tenant-1',
|
|
73
|
+
organizationId: 'org-1',
|
|
74
|
+
bytes: 3,
|
|
75
|
+
source: 'storage_s3_upload',
|
|
76
|
+
}))
|
|
77
|
+
expect(mockQuotaService.completeStandalone).toHaveBeenCalledWith(
|
|
78
|
+
'reservation-1',
|
|
79
|
+
'lease-1',
|
|
80
|
+
3,
|
|
81
|
+
)
|
|
82
|
+
expect(mockScheduleRecovery).toHaveBeenCalledWith({
|
|
83
|
+
reservationId: 'reservation-1',
|
|
84
|
+
tenantId: 'tenant-1',
|
|
85
|
+
organizationId: 'org-1',
|
|
86
|
+
}, expect.any(Number))
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
it('releases multipart capacity when durable recovery cannot be scheduled', async () => {
|
|
90
|
+
mockScheduleRecovery.mockRejectedValueOnce(new Error('queue unavailable'))
|
|
91
|
+
const { POST } = await import('../api/post/storage-providers/s3/upload')
|
|
92
|
+
|
|
93
|
+
const response = await POST(multipartRequest())
|
|
94
|
+
|
|
95
|
+
expect(response.status).toBe(500)
|
|
96
|
+
expect(mockQuotaService.release).toHaveBeenCalledWith('reservation-1', 'lease-1')
|
|
97
|
+
expect(mockPutObject).not.toHaveBeenCalled()
|
|
98
|
+
})
|
|
99
|
+
|
|
100
|
+
it('reserves exact bytes before generating a signed upload URL', async () => {
|
|
101
|
+
const { POST } = await import('../api/post/storage-providers/s3/signed-url')
|
|
102
|
+
const key = 'uploads/org_org-1/tenant_tenant-1/safe.pdf'
|
|
103
|
+
const response = await POST(new Request('http://localhost/api/storage-providers/s3/signed-url', {
|
|
104
|
+
method: 'POST',
|
|
105
|
+
headers: { 'content-type': 'application/json' },
|
|
106
|
+
body: JSON.stringify({ key, operation: 'upload', contentType: 'application/pdf', size: 3 }),
|
|
107
|
+
}))
|
|
108
|
+
|
|
109
|
+
expect(response.status).toBe(200)
|
|
110
|
+
expect(mockQuotaService.reserve).toHaveBeenCalledWith(expect.objectContaining({
|
|
111
|
+
storagePath: key,
|
|
112
|
+
bytes: 3,
|
|
113
|
+
source: 'storage_s3_signed',
|
|
114
|
+
}))
|
|
115
|
+
expect(mockQuotaService.reconcileStandaloneObjects).toHaveBeenCalled()
|
|
116
|
+
expect(await response.json()).toEqual(expect.objectContaining({
|
|
117
|
+
reservationId: 'reservation-1',
|
|
118
|
+
url: expect.stringContaining('/api/storage-providers/s3/signed-upload/'),
|
|
119
|
+
}))
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
it('atomically consumes a compatibility token so replay cannot delete the winning object', async () => {
|
|
123
|
+
mockQuotaService.claimPendingByUploadTokenHash
|
|
124
|
+
.mockResolvedValueOnce({
|
|
125
|
+
id: 'reservation-1',
|
|
126
|
+
leaseToken: 'lease-claimed',
|
|
127
|
+
source: 'storage_s3_signed',
|
|
128
|
+
reservedBytes: 3,
|
|
129
|
+
storagePath: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
130
|
+
tenantId: 'tenant-1',
|
|
131
|
+
organizationId: 'org-1',
|
|
132
|
+
})
|
|
133
|
+
.mockResolvedValueOnce(null)
|
|
134
|
+
const { PUT } = await import('../api/put/storage-providers/s3/signed-upload/[token]')
|
|
135
|
+
const request = () => new Request('http://localhost/api/storage-providers/s3/signed-upload/token', {
|
|
136
|
+
method: 'PUT',
|
|
137
|
+
headers: { 'content-type': 'application/pdf' },
|
|
138
|
+
body: new Uint8Array([1, 2, 3]),
|
|
139
|
+
})
|
|
140
|
+
|
|
141
|
+
const first = await PUT(request(), { params: { token: 'token' } })
|
|
142
|
+
const replay = await PUT(request(), { params: { token: 'token' } })
|
|
143
|
+
|
|
144
|
+
expect(first.status).toBe(200)
|
|
145
|
+
expect(replay.status).toBe(410)
|
|
146
|
+
expect(mockPutObject).toHaveBeenCalledTimes(1)
|
|
147
|
+
expect(mockDelete).not.toHaveBeenCalled()
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
it('stops a chunked compatibility upload as soon as it crosses the reserved bound', async () => {
|
|
151
|
+
mockQuotaService.claimPendingByUploadTokenHash.mockResolvedValueOnce({
|
|
152
|
+
id: 'reservation-1',
|
|
153
|
+
leaseToken: 'lease-claimed',
|
|
154
|
+
source: 'storage_s3_signed',
|
|
155
|
+
reservedBytes: 3,
|
|
156
|
+
storagePath: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
157
|
+
tenantId: 'tenant-1',
|
|
158
|
+
organizationId: 'org-1',
|
|
159
|
+
})
|
|
160
|
+
const stream = new ReadableStream<Uint8Array>({
|
|
161
|
+
start(controller) {
|
|
162
|
+
controller.enqueue(new Uint8Array([1, 2]))
|
|
163
|
+
controller.enqueue(new Uint8Array([3, 4]))
|
|
164
|
+
controller.close()
|
|
165
|
+
},
|
|
166
|
+
})
|
|
167
|
+
const { PUT } = await import('../api/put/storage-providers/s3/signed-upload/[token]')
|
|
168
|
+
const response = await PUT(new Request('http://localhost/api/storage-providers/s3/signed-upload/token', {
|
|
169
|
+
method: 'PUT',
|
|
170
|
+
body: stream,
|
|
171
|
+
duplex: 'half',
|
|
172
|
+
} as RequestInit & { duplex: 'half' }), { params: { token: 'token' } })
|
|
173
|
+
|
|
174
|
+
expect(response.status).toBe(413)
|
|
175
|
+
expect(mockPutObject).not.toHaveBeenCalled()
|
|
176
|
+
expect(mockQuotaService.release).toHaveBeenCalledWith('reservation-1', 'lease-claimed')
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
it('removes committed quota only after standalone object deletion succeeds', async () => {
|
|
180
|
+
const { DELETE } = await import('../api/delete/storage-providers/s3/delete')
|
|
181
|
+
const key = 'uploads/org_org-1/tenant_tenant-1/safe.pdf'
|
|
182
|
+
const response = await DELETE(new Request('http://localhost/api/storage-providers/s3/delete', {
|
|
183
|
+
method: 'DELETE',
|
|
184
|
+
headers: { 'content-type': 'application/json' },
|
|
185
|
+
body: JSON.stringify({ key }),
|
|
186
|
+
}))
|
|
187
|
+
|
|
188
|
+
expect(response.status).toBe(204)
|
|
189
|
+
expect(mockDelete).toHaveBeenCalled()
|
|
190
|
+
expect(mockQuotaService.releaseCommittedByPath).toHaveBeenCalledWith(expect.objectContaining({
|
|
191
|
+
tenantId: 'tenant-1',
|
|
192
|
+
storageDriver: 's3',
|
|
193
|
+
storagePath: key,
|
|
194
|
+
}))
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
it('releases a programmatic signed-url reservation when recovery scheduling fails', async () => {
|
|
198
|
+
const { createStorageService } = await import('../lib/storage-service')
|
|
199
|
+
const service = createStorageService({
|
|
200
|
+
bucket: 'bucket',
|
|
201
|
+
region: 'us-east-1',
|
|
202
|
+
quotaService: mockQuotaService as never,
|
|
203
|
+
quotaRecoveryScheduler: jest.fn(async () => {
|
|
204
|
+
throw new Error('queue unavailable')
|
|
205
|
+
}),
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
await expect(service.getSignedUrl({
|
|
209
|
+
key: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
210
|
+
operation: 'upload',
|
|
211
|
+
size: 3,
|
|
212
|
+
scope: { tenantId: 'tenant-1', organizationId: 'org-1' },
|
|
213
|
+
})).rejects.toThrow('queue unavailable')
|
|
214
|
+
expect(mockQuotaService.release).toHaveBeenCalledWith('reservation-1', 'lease-1')
|
|
215
|
+
})
|
|
216
|
+
})
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/** @jest-environment node */
|
|
2
|
+
|
|
3
|
+
const mockListObjects = jest.fn(async () => ({ files: [], truncated: false }))
|
|
4
|
+
const mockScheduleRecovery = jest.fn(async () => {})
|
|
5
|
+
|
|
6
|
+
jest.mock('../lib/s3-driver', () => ({
|
|
7
|
+
S3StorageDriver: jest.fn().mockImplementation(() => ({
|
|
8
|
+
listObjects: mockListObjects,
|
|
9
|
+
deleteStrict: jest.fn(async () => {}),
|
|
10
|
+
})),
|
|
11
|
+
}))
|
|
12
|
+
|
|
13
|
+
jest.mock('../lib/quota-recovery-queue', () => ({
|
|
14
|
+
scheduleStorageS3QuotaRecovery: mockScheduleRecovery,
|
|
15
|
+
}))
|
|
16
|
+
|
|
17
|
+
import handle from '../workers/quota-recovery'
|
|
18
|
+
import { S3StorageDriver } from '../lib/s3-driver'
|
|
19
|
+
|
|
20
|
+
function reservation(status: 'storing' | 'recovering') {
|
|
21
|
+
return {
|
|
22
|
+
id: 'reservation-1',
|
|
23
|
+
tenantId: 'tenant-1',
|
|
24
|
+
organizationId: 'org-1',
|
|
25
|
+
reservedBytes: 3,
|
|
26
|
+
actualBytes: null,
|
|
27
|
+
status,
|
|
28
|
+
source: 'storage_s3_signed',
|
|
29
|
+
storageDriver: 's3',
|
|
30
|
+
partitionCode: null,
|
|
31
|
+
storagePath: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
32
|
+
leaseToken: 'lease-1',
|
|
33
|
+
uploadTokenHash: 'token-hash',
|
|
34
|
+
expiresAt: new Date(Date.now() - 1_000),
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
describe('storage_s3 quota recovery', () => {
|
|
39
|
+
beforeEach(() => {
|
|
40
|
+
jest.clearAllMocks()
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
it('retains capacity and rechecks once when an active proxy upload has no visible object yet', async () => {
|
|
44
|
+
const current = reservation('storing')
|
|
45
|
+
const claimed = { ...reservation('recovering'), expiresAt: new Date(Date.now() + 60_000) }
|
|
46
|
+
const quotaService = {
|
|
47
|
+
getReservation: jest.fn(async () => current),
|
|
48
|
+
claimExpired: jest.fn(async () => claimed),
|
|
49
|
+
release: jest.fn(async () => {}),
|
|
50
|
+
commitRecoveredStandalone: jest.fn(async () => {}),
|
|
51
|
+
}
|
|
52
|
+
const ctx = {
|
|
53
|
+
resolve: (name: string) => {
|
|
54
|
+
if (name === 'attachmentQuotaService') return quotaService
|
|
55
|
+
if (name === 'integrationCredentialsService') {
|
|
56
|
+
return { resolve: jest.fn(async () => ({ bucket: 'bucket', region: 'us-east-1' })) }
|
|
57
|
+
}
|
|
58
|
+
throw new Error(`Unexpected dependency: ${name}`)
|
|
59
|
+
},
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const payload = {
|
|
63
|
+
reservationId: current.id,
|
|
64
|
+
tenantId: current.tenantId,
|
|
65
|
+
organizationId: current.organizationId,
|
|
66
|
+
}
|
|
67
|
+
await handle({ payload, id: 'job-1' } as never, ctx as never)
|
|
68
|
+
|
|
69
|
+
expect(quotaService.getReservation).toHaveBeenCalledWith(current.id, {
|
|
70
|
+
tenantId: 'tenant-1',
|
|
71
|
+
organizationId: 'org-1',
|
|
72
|
+
})
|
|
73
|
+
expect(S3StorageDriver).toHaveBeenCalledWith(expect.objectContaining({
|
|
74
|
+
organizationId: 'org-1',
|
|
75
|
+
tenantId: 'tenant-1',
|
|
76
|
+
}))
|
|
77
|
+
expect(mockScheduleRecovery).toHaveBeenCalledWith(
|
|
78
|
+
{ ...payload, absenceCheck: 1 },
|
|
79
|
+
expect.any(Number),
|
|
80
|
+
)
|
|
81
|
+
expect(quotaService.release).not.toHaveBeenCalled()
|
|
82
|
+
})
|
|
83
|
+
})
|