@open-mercato/storage-s3 0.6.7-develop.6860.1.f796b22e80 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/storage_s3/api/delete/storage-providers/s3/delete.js +5 -20
- package/dist/modules/storage_s3/api/delete/storage-providers/s3/delete.js.map +2 -2
- package/dist/modules/storage_s3/api/get/storage-providers/s3/download.js +5 -7
- package/dist/modules/storage_s3/api/get/storage-providers/s3/download.js.map +2 -2
- package/dist/modules/storage_s3/api/get/storage-providers/s3/list.js +4 -6
- package/dist/modules/storage_s3/api/get/storage-providers/s3/list.js.map +2 -2
- package/dist/modules/storage_s3/api/post/storage-providers/s3/signed-url.js +8 -70
- package/dist/modules/storage_s3/api/post/storage-providers/s3/signed-url.js.map +3 -3
- package/dist/modules/storage_s3/api/post/storage-providers/s3/upload.js +15 -92
- package/dist/modules/storage_s3/api/post/storage-providers/s3/upload.js.map +2 -2
- package/dist/modules/storage_s3/di.js +2 -6
- package/dist/modules/storage_s3/di.js.map +2 -2
- package/dist/modules/storage_s3/integration.js +1 -1
- package/dist/modules/storage_s3/integration.js.map +1 -1
- package/dist/modules/storage_s3/lib/health.js +1 -18
- package/dist/modules/storage_s3/lib/health.js.map +2 -2
- package/dist/modules/storage_s3/lib/key-scope.js +4 -4
- package/dist/modules/storage_s3/lib/key-scope.js.map +2 -2
- package/dist/modules/storage_s3/lib/s3-driver.js +15 -44
- package/dist/modules/storage_s3/lib/s3-driver.js.map +2 -2
- package/dist/modules/storage_s3/lib/storage-service.js +22 -126
- package/dist/modules/storage_s3/lib/storage-service.js.map +3 -3
- package/jest.config.cjs +0 -1
- package/package.json +6 -13
- package/src/modules/storage_s3/__tests__/s3Driver.test.ts +8 -23
- package/src/modules/storage_s3/__tests__/s3ListRoute.test.ts +1 -7
- package/src/modules/storage_s3/__tests__/s3Routes.test.ts +0 -26
- package/src/modules/storage_s3/__tests__/storageService.test.ts +5 -20
- package/src/modules/storage_s3/api/delete/storage-providers/s3/delete.ts +5 -22
- package/src/modules/storage_s3/api/get/storage-providers/s3/download.ts +5 -7
- package/src/modules/storage_s3/api/get/storage-providers/s3/list.ts +4 -6
- package/src/modules/storage_s3/api/post/storage-providers/s3/signed-url.ts +6 -77
- package/src/modules/storage_s3/api/post/storage-providers/s3/upload.ts +16 -101
- package/src/modules/storage_s3/di.ts +2 -14
- package/src/modules/storage_s3/integration.ts +1 -1
- package/src/modules/storage_s3/lib/health.ts +0 -18
- package/src/modules/storage_s3/lib/key-scope.ts +4 -4
- package/src/modules/storage_s3/lib/s3-driver.ts +10 -65
- package/src/modules/storage_s3/lib/storage-service.ts +22 -131
- package/dist/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].js +0 -145
- package/dist/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].js.map +0 -7
- package/dist/modules/storage_s3/lib/endpoint-safety.js +0 -133
- package/dist/modules/storage_s3/lib/endpoint-safety.js.map +0 -7
- package/dist/modules/storage_s3/lib/quota-accounting.js +0 -24
- package/dist/modules/storage_s3/lib/quota-accounting.js.map +0 -7
- package/dist/modules/storage_s3/lib/quota-recovery-queue.js +0 -9
- package/dist/modules/storage_s3/lib/quota-recovery-queue.js.map +0 -7
- package/dist/modules/storage_s3/workers/quota-recovery.js +0 -70
- package/dist/modules/storage_s3/workers/quota-recovery.js.map +0 -7
- package/src/modules/storage_s3/__tests__/endpoint-safety.test.ts +0 -71
- package/src/modules/storage_s3/__tests__/health.test.ts +0 -38
- package/src/modules/storage_s3/__tests__/i18nPackageExports.test.ts +0 -13
- package/src/modules/storage_s3/__tests__/quota-lifecycle.test.ts +0 -216
- package/src/modules/storage_s3/__tests__/quota-recovery.test.ts +0 -83
- package/src/modules/storage_s3/__tests__/upload.test.ts +0 -93
- package/src/modules/storage_s3/api/put/storage-providers/s3/signed-upload/[token].ts +0 -150
- package/src/modules/storage_s3/i18n/de.json +0 -17
- package/src/modules/storage_s3/i18n/en.json +0 -17
- package/src/modules/storage_s3/i18n/es.json +0 -17
- package/src/modules/storage_s3/i18n/ko.json +0 -17
- package/src/modules/storage_s3/i18n/pl.json +0 -17
- package/src/modules/storage_s3/lib/endpoint-safety.ts +0 -179
- package/src/modules/storage_s3/lib/quota-accounting.ts +0 -29
- package/src/modules/storage_s3/lib/quota-recovery-queue.ts +0 -17
- package/src/modules/storage_s3/workers/quota-recovery.ts +0 -82
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
/** @jest-environment node */
|
|
2
|
-
|
|
3
|
-
const mockListObjects = jest.fn(async () => ({ files: [], truncated: false }))
|
|
4
|
-
const mockScheduleRecovery = jest.fn(async () => {})
|
|
5
|
-
|
|
6
|
-
jest.mock('../lib/s3-driver', () => ({
|
|
7
|
-
S3StorageDriver: jest.fn().mockImplementation(() => ({
|
|
8
|
-
listObjects: mockListObjects,
|
|
9
|
-
deleteStrict: jest.fn(async () => {}),
|
|
10
|
-
})),
|
|
11
|
-
}))
|
|
12
|
-
|
|
13
|
-
jest.mock('../lib/quota-recovery-queue', () => ({
|
|
14
|
-
scheduleStorageS3QuotaRecovery: mockScheduleRecovery,
|
|
15
|
-
}))
|
|
16
|
-
|
|
17
|
-
import handle from '../workers/quota-recovery'
|
|
18
|
-
import { S3StorageDriver } from '../lib/s3-driver'
|
|
19
|
-
|
|
20
|
-
function reservation(status: 'storing' | 'recovering') {
|
|
21
|
-
return {
|
|
22
|
-
id: 'reservation-1',
|
|
23
|
-
tenantId: 'tenant-1',
|
|
24
|
-
organizationId: 'org-1',
|
|
25
|
-
reservedBytes: 3,
|
|
26
|
-
actualBytes: null,
|
|
27
|
-
status,
|
|
28
|
-
source: 'storage_s3_signed',
|
|
29
|
-
storageDriver: 's3',
|
|
30
|
-
partitionCode: null,
|
|
31
|
-
storagePath: 'uploads/org_org-1/tenant_tenant-1/safe.pdf',
|
|
32
|
-
leaseToken: 'lease-1',
|
|
33
|
-
uploadTokenHash: 'token-hash',
|
|
34
|
-
expiresAt: new Date(Date.now() - 1_000),
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
describe('storage_s3 quota recovery', () => {
|
|
39
|
-
beforeEach(() => {
|
|
40
|
-
jest.clearAllMocks()
|
|
41
|
-
})
|
|
42
|
-
|
|
43
|
-
it('retains capacity and rechecks once when an active proxy upload has no visible object yet', async () => {
|
|
44
|
-
const current = reservation('storing')
|
|
45
|
-
const claimed = { ...reservation('recovering'), expiresAt: new Date(Date.now() + 60_000) }
|
|
46
|
-
const quotaService = {
|
|
47
|
-
getReservation: jest.fn(async () => current),
|
|
48
|
-
claimExpired: jest.fn(async () => claimed),
|
|
49
|
-
release: jest.fn(async () => {}),
|
|
50
|
-
commitRecoveredStandalone: jest.fn(async () => {}),
|
|
51
|
-
}
|
|
52
|
-
const ctx = {
|
|
53
|
-
resolve: (name: string) => {
|
|
54
|
-
if (name === 'attachmentQuotaService') return quotaService
|
|
55
|
-
if (name === 'integrationCredentialsService') {
|
|
56
|
-
return { resolve: jest.fn(async () => ({ bucket: 'bucket', region: 'us-east-1' })) }
|
|
57
|
-
}
|
|
58
|
-
throw new Error(`Unexpected dependency: ${name}`)
|
|
59
|
-
},
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
const payload = {
|
|
63
|
-
reservationId: current.id,
|
|
64
|
-
tenantId: current.tenantId,
|
|
65
|
-
organizationId: current.organizationId,
|
|
66
|
-
}
|
|
67
|
-
await handle({ payload, id: 'job-1' } as never, ctx as never)
|
|
68
|
-
|
|
69
|
-
expect(quotaService.getReservation).toHaveBeenCalledWith(current.id, {
|
|
70
|
-
tenantId: 'tenant-1',
|
|
71
|
-
organizationId: 'org-1',
|
|
72
|
-
})
|
|
73
|
-
expect(S3StorageDriver).toHaveBeenCalledWith(expect.objectContaining({
|
|
74
|
-
organizationId: 'org-1',
|
|
75
|
-
tenantId: 'tenant-1',
|
|
76
|
-
}))
|
|
77
|
-
expect(mockScheduleRecovery).toHaveBeenCalledWith(
|
|
78
|
-
{ ...payload, absenceCheck: 1 },
|
|
79
|
-
expect.any(Number),
|
|
80
|
-
)
|
|
81
|
-
expect(quotaService.release).not.toHaveBeenCalled()
|
|
82
|
-
})
|
|
83
|
-
})
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
/** @jest-environment node */
|
|
2
|
-
|
|
3
|
-
const mockGetAuthFromRequest = jest.fn()
|
|
4
|
-
jest.mock('@open-mercato/shared/lib/auth/server', () => ({
|
|
5
|
-
getAuthFromRequest: (...args: unknown[]) => mockGetAuthFromRequest(...args),
|
|
6
|
-
}))
|
|
7
|
-
|
|
8
|
-
jest.mock('@open-mercato/shared/lib/i18n/server', () => ({
|
|
9
|
-
resolveTranslations: async () => ({
|
|
10
|
-
t: (key: string) => `translated:${key}`,
|
|
11
|
-
}),
|
|
12
|
-
}))
|
|
13
|
-
|
|
14
|
-
const mockResolveCredentials = jest.fn()
|
|
15
|
-
const mockCreateRequestContainer = jest.fn(async () => ({
|
|
16
|
-
resolve: (key: string) => {
|
|
17
|
-
if (key === 'integrationCredentialsService') return { resolve: mockResolveCredentials }
|
|
18
|
-
throw new Error(`Unexpected dependency: ${key}`)
|
|
19
|
-
},
|
|
20
|
-
}))
|
|
21
|
-
jest.mock('@open-mercato/shared/lib/di/container', () => ({
|
|
22
|
-
createRequestContainer: () => mockCreateRequestContainer(),
|
|
23
|
-
}))
|
|
24
|
-
|
|
25
|
-
const mockListObjects = jest.fn()
|
|
26
|
-
const mockPutObject = jest.fn()
|
|
27
|
-
jest.mock('../lib/s3-driver', () => ({
|
|
28
|
-
S3StorageDriver: jest.fn().mockImplementation(() => ({
|
|
29
|
-
getBucket: () => 'test-bucket',
|
|
30
|
-
listObjects: (...args: unknown[]) => mockListObjects(...args),
|
|
31
|
-
putObject: (...args: unknown[]) => mockPutObject(...args),
|
|
32
|
-
})),
|
|
33
|
-
}))
|
|
34
|
-
|
|
35
|
-
import { POST } from '../api/post/storage-providers/s3/upload'
|
|
36
|
-
|
|
37
|
-
describe('storage_s3 direct upload route', () => {
|
|
38
|
-
const originalMaxUploadMb = process.env.OM_ATTACHMENT_MAX_UPLOAD_MB
|
|
39
|
-
|
|
40
|
-
beforeEach(() => {
|
|
41
|
-
jest.clearAllMocks()
|
|
42
|
-
mockGetAuthFromRequest.mockResolvedValue({ tenantId: 'tenant-1', orgId: 'org-1' })
|
|
43
|
-
mockResolveCredentials.mockResolvedValue({ bucket: 'test-bucket', region: 'eu-central-1' })
|
|
44
|
-
mockListObjects.mockResolvedValue({ files: [], truncated: false })
|
|
45
|
-
mockPutObject.mockResolvedValue(undefined)
|
|
46
|
-
delete process.env.OM_ATTACHMENT_MAX_UPLOAD_MB
|
|
47
|
-
})
|
|
48
|
-
|
|
49
|
-
afterAll(() => {
|
|
50
|
-
if (originalMaxUploadMb === undefined) delete process.env.OM_ATTACHMENT_MAX_UPLOAD_MB
|
|
51
|
-
else process.env.OM_ATTACHMENT_MAX_UPLOAD_MB = originalMaxUploadMb
|
|
52
|
-
})
|
|
53
|
-
|
|
54
|
-
it('rejects oversized streamed metadata without trusting content length', async () => {
|
|
55
|
-
process.env.OM_ATTACHMENT_MAX_UPLOAD_MB = '0.000001'
|
|
56
|
-
const boundary = 'oversized-key'
|
|
57
|
-
const body = new TextEncoder().encode([
|
|
58
|
-
`--${boundary}\r\nContent-Disposition: form-data; name="key"\r\n\r\n`,
|
|
59
|
-
'x'.repeat(1024 * 1024),
|
|
60
|
-
`\r\n--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="small.txt"\r\n`,
|
|
61
|
-
'Content-Type: text/plain\r\n\r\ns\r\n',
|
|
62
|
-
`--${boundary}--\r\n`,
|
|
63
|
-
].join(''))
|
|
64
|
-
const request = new Request('http://example.test/api/storage-providers/s3/upload', {
|
|
65
|
-
method: 'POST',
|
|
66
|
-
headers: { 'content-type': `multipart/form-data; boundary=${boundary}` },
|
|
67
|
-
body,
|
|
68
|
-
})
|
|
69
|
-
|
|
70
|
-
expect(request.headers.get('content-length')).toBeNull()
|
|
71
|
-
const response = await POST(request)
|
|
72
|
-
|
|
73
|
-
expect(response.status).toBe(413)
|
|
74
|
-
await expect(response.json()).resolves.toEqual({
|
|
75
|
-
error: 'translated:storage_s3.errors.maxUploadSize',
|
|
76
|
-
})
|
|
77
|
-
expect(mockCreateRequestContainer).not.toHaveBeenCalled()
|
|
78
|
-
expect(mockPutObject).not.toHaveBeenCalled()
|
|
79
|
-
})
|
|
80
|
-
|
|
81
|
-
it('preserves valid multipart parsing for a normal upload', async () => {
|
|
82
|
-
const form = new FormData()
|
|
83
|
-
form.set('file', new File([new TextEncoder().encode('safe')], 'safe.txt', { type: 'text/plain' }))
|
|
84
|
-
const response = await POST(new Request('http://example.test/api/storage-providers/s3/upload', {
|
|
85
|
-
method: 'POST',
|
|
86
|
-
body: form,
|
|
87
|
-
}))
|
|
88
|
-
|
|
89
|
-
expect(response.status).toBe(200)
|
|
90
|
-
await expect(response.json()).resolves.toMatchObject({ bucket: 'test-bucket', size: 4 })
|
|
91
|
-
expect(mockPutObject).toHaveBeenCalledTimes(1)
|
|
92
|
-
})
|
|
93
|
-
})
|
|
@@ -1,150 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto'
|
|
2
|
-
import path from 'node:path'
|
|
3
|
-
import { NextResponse } from 'next/server'
|
|
4
|
-
import { z } from 'zod'
|
|
5
|
-
import type { OpenApiRouteDoc } from '@open-mercato/shared/lib/openapi'
|
|
6
|
-
import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
|
|
7
|
-
import {
|
|
8
|
-
detectAttachmentMimeType,
|
|
9
|
-
hasDangerousExecutableExtension,
|
|
10
|
-
isActiveContentAttachment,
|
|
11
|
-
} from '@open-mercato/core/modules/attachments/lib/security'
|
|
12
|
-
import type { AttachmentQuotaService } from '@open-mercato/core/modules/attachments/lib/quota-service'
|
|
13
|
-
import { S3StorageDriver } from '../../../../../lib/s3-driver'
|
|
14
|
-
|
|
15
|
-
export const metadata = {
|
|
16
|
-
path: '/storage-providers/s3/signed-upload/:token',
|
|
17
|
-
PUT: { requireAuth: false },
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
type RouteContext = { params?: Promise<{ token?: string }> | { token?: string } }
|
|
21
|
-
|
|
22
|
-
class UploadBodyTooLargeError extends Error {}
|
|
23
|
-
|
|
24
|
-
async function readBoundedBody(req: Request, maxBytes: number): Promise<Buffer> {
|
|
25
|
-
if (!req.body) return Buffer.alloc(0)
|
|
26
|
-
const reader = req.body.getReader()
|
|
27
|
-
const chunks: Buffer[] = []
|
|
28
|
-
let totalBytes = 0
|
|
29
|
-
try {
|
|
30
|
-
while (true) {
|
|
31
|
-
const { done, value } = await reader.read()
|
|
32
|
-
if (done) break
|
|
33
|
-
totalBytes += value.byteLength
|
|
34
|
-
if (totalBytes > maxBytes) {
|
|
35
|
-
await reader.cancel()
|
|
36
|
-
throw new UploadBodyTooLargeError('Attachment exceeds the reserved upload size.')
|
|
37
|
-
}
|
|
38
|
-
chunks.push(Buffer.from(value))
|
|
39
|
-
}
|
|
40
|
-
} finally {
|
|
41
|
-
reader.releaseLock()
|
|
42
|
-
}
|
|
43
|
-
return Buffer.concat(chunks, totalBytes)
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export async function PUT(req: Request, ctx: RouteContext) {
|
|
47
|
-
const params = ctx.params instanceof Promise ? await ctx.params : ctx.params
|
|
48
|
-
const token = params?.token?.trim()
|
|
49
|
-
if (!token) return NextResponse.json({ error: 'Upload token is required.' }, { status: 400 })
|
|
50
|
-
|
|
51
|
-
const { resolve } = await createRequestContainer()
|
|
52
|
-
let quotaService: AttachmentQuotaService | null = null
|
|
53
|
-
try {
|
|
54
|
-
quotaService = resolve('attachmentQuotaService') as AttachmentQuotaService
|
|
55
|
-
} catch {
|
|
56
|
-
// Fail closed below when the attachments quota service is not registered.
|
|
57
|
-
}
|
|
58
|
-
if (!quotaService) return NextResponse.json({ error: 'Storage quota accounting is unavailable.' }, { status: 500 })
|
|
59
|
-
|
|
60
|
-
const tokenHash = createHash('sha256').update(token).digest('hex')
|
|
61
|
-
const reservation = await quotaService.claimPendingByUploadTokenHash(tokenHash)
|
|
62
|
-
if (!reservation || reservation.source !== 'storage_s3_signed') {
|
|
63
|
-
return NextResponse.json({ error: 'Upload token is invalid or expired.' }, { status: 410 })
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
const declaredLengthHeader = req.headers.get('content-length')
|
|
67
|
-
const declaredLength = declaredLengthHeader == null ? null : Number(declaredLengthHeader)
|
|
68
|
-
if (declaredLength !== null && Number.isFinite(declaredLength) && declaredLength > reservation.reservedBytes) {
|
|
69
|
-
await quotaService.release(reservation.id, reservation.leaseToken).catch(() => {})
|
|
70
|
-
return NextResponse.json({ error: 'Attachment exceeds the reserved upload size.' }, { status: 413 })
|
|
71
|
-
}
|
|
72
|
-
let buffer: Buffer
|
|
73
|
-
try {
|
|
74
|
-
buffer = await readBoundedBody(req, reservation.reservedBytes)
|
|
75
|
-
} catch (error) {
|
|
76
|
-
await quotaService.release(reservation.id, reservation.leaseToken).catch(() => {})
|
|
77
|
-
if (error instanceof UploadBodyTooLargeError) {
|
|
78
|
-
return NextResponse.json({ error: error.message }, { status: 413 })
|
|
79
|
-
}
|
|
80
|
-
return NextResponse.json({ error: 'Failed to read attachment.' }, { status: 400 })
|
|
81
|
-
}
|
|
82
|
-
if (buffer.length > reservation.reservedBytes) {
|
|
83
|
-
await quotaService.release(reservation.id, reservation.leaseToken).catch(() => {})
|
|
84
|
-
return NextResponse.json({ error: 'Attachment exceeds the reserved upload size.' }, { status: 413 })
|
|
85
|
-
}
|
|
86
|
-
const fileName = path.posix.basename(reservation.storagePath)
|
|
87
|
-
if (hasDangerousExecutableExtension(fileName)) {
|
|
88
|
-
await quotaService.release(reservation.id, reservation.leaseToken).catch(() => {})
|
|
89
|
-
return NextResponse.json({ error: 'Executable file types are not allowed as attachments.' }, { status: 400 })
|
|
90
|
-
}
|
|
91
|
-
const contentType = detectAttachmentMimeType(buffer, fileName, req.headers.get('content-type'))
|
|
92
|
-
if (isActiveContentAttachment(buffer, fileName, contentType)) {
|
|
93
|
-
await quotaService.release(reservation.id, reservation.leaseToken).catch(() => {})
|
|
94
|
-
return NextResponse.json({ error: 'Active content uploads are not allowed.' }, { status: 400 })
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const credentialsService = resolve('integrationCredentialsService') as {
|
|
98
|
-
resolve(integrationId: string, scope: { tenantId: string; organizationId: string }): Promise<Record<string, unknown> | null>
|
|
99
|
-
}
|
|
100
|
-
const credentials = await credentialsService.resolve('storage_s3', {
|
|
101
|
-
tenantId: reservation.tenantId,
|
|
102
|
-
organizationId: reservation.organizationId,
|
|
103
|
-
})
|
|
104
|
-
if (!credentials) {
|
|
105
|
-
await quotaService.release(reservation.id, reservation.leaseToken).catch(() => {})
|
|
106
|
-
return NextResponse.json({ error: 'S3 integration is not configured.' }, { status: 400 })
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const driver = new S3StorageDriver({
|
|
110
|
-
...credentials,
|
|
111
|
-
organizationId: reservation.organizationId,
|
|
112
|
-
tenantId: reservation.tenantId,
|
|
113
|
-
})
|
|
114
|
-
let objectStored = false
|
|
115
|
-
try {
|
|
116
|
-
await driver.putObject(reservation.storagePath, buffer, contentType)
|
|
117
|
-
objectStored = true
|
|
118
|
-
await quotaService.markStored(reservation.id, reservation.leaseToken)
|
|
119
|
-
await quotaService.completeStandalone(reservation.id, reservation.leaseToken, buffer.length)
|
|
120
|
-
return new NextResponse(null, { status: 200 })
|
|
121
|
-
} catch (error) {
|
|
122
|
-
if (objectStored) {
|
|
123
|
-
try {
|
|
124
|
-
await driver.deleteStrict('', reservation.storagePath)
|
|
125
|
-
await quotaService.release(reservation.id, reservation.leaseToken)
|
|
126
|
-
} catch {
|
|
127
|
-
// Retain the reservation when object absence cannot be proven.
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
return NextResponse.json({ error: 'Failed to persist attachment.' }, { status: 500 })
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
export default PUT
|
|
135
|
-
|
|
136
|
-
export const openApi: OpenApiRouteDoc = {
|
|
137
|
-
tag: 'Storage',
|
|
138
|
-
summary: 'Upload through a bounded one-time compatibility URL',
|
|
139
|
-
methods: {
|
|
140
|
-
PUT: {
|
|
141
|
-
summary: 'Upload an object using a bounded one-time token',
|
|
142
|
-
responses: [{ status: 200, description: 'Upload completed', schema: z.any() }],
|
|
143
|
-
errors: [
|
|
144
|
-
{ status: 400, description: 'Invalid upload', schema: z.object({ error: z.string() }) },
|
|
145
|
-
{ status: 410, description: 'Token invalid or expired', schema: z.object({ error: z.string() }) },
|
|
146
|
-
{ status: 413, description: 'Upload exceeds reserved size', schema: z.object({ error: z.string() }) },
|
|
147
|
-
],
|
|
148
|
-
},
|
|
149
|
-
},
|
|
150
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"storage_s3.errors.activeContentBlocked": "Das Hochladen aktiver Inhalte ist nicht erlaubt.",
|
|
3
|
-
"storage_s3.errors.dangerousExecutable": "Ausführbare Dateitypen sind als Anhänge nicht erlaubt.",
|
|
4
|
-
"storage_s3.errors.fileNotFound": "Datei nicht gefunden",
|
|
5
|
-
"storage_s3.errors.fileRequired": "Das Feld „file“ ist erforderlich",
|
|
6
|
-
"storage_s3.errors.integrationNotConfigured": "Die S3-Integration ist nicht konfiguriert.",
|
|
7
|
-
"storage_s3.errors.invalidPayload": "Ungültige Anfragedaten",
|
|
8
|
-
"storage_s3.errors.invalidQuery": "Ungültige Abfrageparameter",
|
|
9
|
-
"storage_s3.errors.keyAccessDenied": "Zugriff verweigert: Der Schlüssel ist nicht auf diesen Tenant beschränkt.",
|
|
10
|
-
"storage_s3.errors.keyOverrideAccessDenied": "Zugriff verweigert: Die Schlüsselüberschreibung ist nicht auf diesen Tenant beschränkt.",
|
|
11
|
-
"storage_s3.errors.keyRequired": "Der Abfrageparameter „key“ ist erforderlich",
|
|
12
|
-
"storage_s3.errors.maxUploadSize": "Der Anhang überschreitet die maximal zulässige Upload-Größe.",
|
|
13
|
-
"storage_s3.errors.multipartRequired": "multipart/form-data wurde erwartet",
|
|
14
|
-
"storage_s3.errors.prefixAccessDenied": "Zugriff verweigert: Das Präfix ist nicht auf diesen Tenant beschränkt.",
|
|
15
|
-
"storage_s3.errors.quotaExceeded": "Das Speicherlimit für Anhänge dieses Tenants wurde überschritten.",
|
|
16
|
-
"storage_s3.errors.unauthorized": "Nicht autorisiert"
|
|
17
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"storage_s3.errors.activeContentBlocked": "Active content uploads are not allowed.",
|
|
3
|
-
"storage_s3.errors.dangerousExecutable": "Executable file types are not allowed as attachments.",
|
|
4
|
-
"storage_s3.errors.fileNotFound": "File not found",
|
|
5
|
-
"storage_s3.errors.fileRequired": "file field is required",
|
|
6
|
-
"storage_s3.errors.integrationNotConfigured": "S3 integration is not configured.",
|
|
7
|
-
"storage_s3.errors.invalidPayload": "Invalid payload",
|
|
8
|
-
"storage_s3.errors.invalidQuery": "Invalid query parameters",
|
|
9
|
-
"storage_s3.errors.keyAccessDenied": "Access denied: key is not scoped to this tenant.",
|
|
10
|
-
"storage_s3.errors.keyOverrideAccessDenied": "Access denied: key override is not scoped to this tenant.",
|
|
11
|
-
"storage_s3.errors.keyRequired": "key query param is required",
|
|
12
|
-
"storage_s3.errors.maxUploadSize": "Attachment exceeds the maximum upload size.",
|
|
13
|
-
"storage_s3.errors.multipartRequired": "Expected multipart/form-data",
|
|
14
|
-
"storage_s3.errors.prefixAccessDenied": "Access denied: prefix is not scoped to this tenant.",
|
|
15
|
-
"storage_s3.errors.quotaExceeded": "Attachment storage quota exceeded for this tenant.",
|
|
16
|
-
"storage_s3.errors.unauthorized": "Unauthorized"
|
|
17
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"storage_s3.errors.activeContentBlocked": "No se permite la carga de contenido activo.",
|
|
3
|
-
"storage_s3.errors.dangerousExecutable": "No se permiten archivos ejecutables como adjuntos.",
|
|
4
|
-
"storage_s3.errors.fileNotFound": "Archivo no encontrado",
|
|
5
|
-
"storage_s3.errors.fileRequired": "El campo file es obligatorio",
|
|
6
|
-
"storage_s3.errors.integrationNotConfigured": "La integración de S3 no está configurada.",
|
|
7
|
-
"storage_s3.errors.invalidPayload": "Contenido de solicitud no válido",
|
|
8
|
-
"storage_s3.errors.invalidQuery": "Parámetros de consulta no válidos",
|
|
9
|
-
"storage_s3.errors.keyAccessDenied": "Acceso denegado: la clave no está limitada a este tenant.",
|
|
10
|
-
"storage_s3.errors.keyOverrideAccessDenied": "Acceso denegado: la sobrescritura de la clave no está limitada a este tenant.",
|
|
11
|
-
"storage_s3.errors.keyRequired": "Se requiere el parámetro de consulta key",
|
|
12
|
-
"storage_s3.errors.maxUploadSize": "El archivo adjunto supera el tamaño máximo de carga permitido.",
|
|
13
|
-
"storage_s3.errors.multipartRequired": "Se esperaba multipart/form-data",
|
|
14
|
-
"storage_s3.errors.prefixAccessDenied": "Acceso denegado: el prefijo no está limitado a este tenant.",
|
|
15
|
-
"storage_s3.errors.quotaExceeded": "Se superó la cuota de almacenamiento de adjuntos para este tenant.",
|
|
16
|
-
"storage_s3.errors.unauthorized": "No autorizado"
|
|
17
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"storage_s3.errors.activeContentBlocked": "활성 콘텐츠 업로드는 허용되지 않습니다.",
|
|
3
|
-
"storage_s3.errors.dangerousExecutable": "실행 파일 형식은 첨부가 허용되지 않습니다.",
|
|
4
|
-
"storage_s3.errors.fileNotFound": "파일을 찾을 수 없습니다",
|
|
5
|
-
"storage_s3.errors.fileRequired": "파일 필드는 필수입니다",
|
|
6
|
-
"storage_s3.errors.integrationNotConfigured": "S3 통합이 구성되지 않았습니다.",
|
|
7
|
-
"storage_s3.errors.invalidPayload": "잘못된 페이로드",
|
|
8
|
-
"storage_s3.errors.invalidQuery": "잘못된 쿼리 매개변수",
|
|
9
|
-
"storage_s3.errors.keyAccessDenied": "액세스 거부됨: 키 범위가 이 테넌트로 지정되지 않았습니다.",
|
|
10
|
-
"storage_s3.errors.keyOverrideAccessDenied": "액세스 거부됨: 키 재정의 범위가 이 테넌트로 지정되지 않았습니다.",
|
|
11
|
-
"storage_s3.errors.keyRequired": "주요 쿼리 매개변수가 필요합니다.",
|
|
12
|
-
"storage_s3.errors.maxUploadSize": "첨부파일이 최대 업로드 크기를 초과합니다.",
|
|
13
|
-
"storage_s3.errors.multipartRequired": "예상되는 멀티파트/양식 데이터",
|
|
14
|
-
"storage_s3.errors.prefixAccessDenied": "액세스 거부됨: 접두사의 범위가 이 테넌트로 지정되지 않았습니다.",
|
|
15
|
-
"storage_s3.errors.quotaExceeded": "이 테넌트에 대한 첨부 파일 저장소 할당량이 초과되었습니다.",
|
|
16
|
-
"storage_s3.errors.unauthorized": "승인되지 않은"
|
|
17
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"storage_s3.errors.activeContentBlocked": "Przesyłanie treści aktywnych nie jest dozwolone.",
|
|
3
|
-
"storage_s3.errors.dangerousExecutable": "Pliki wykonywalne nie są dozwolone jako załączniki.",
|
|
4
|
-
"storage_s3.errors.fileNotFound": "Nie znaleziono pliku",
|
|
5
|
-
"storage_s3.errors.fileRequired": "Pole file jest wymagane",
|
|
6
|
-
"storage_s3.errors.integrationNotConfigured": "Integracja S3 nie jest skonfigurowana.",
|
|
7
|
-
"storage_s3.errors.invalidPayload": "Nieprawidłowe dane żądania",
|
|
8
|
-
"storage_s3.errors.invalidQuery": "Nieprawidłowe parametry zapytania",
|
|
9
|
-
"storage_s3.errors.keyAccessDenied": "Odmowa dostępu: klucz nie jest ograniczony do tego tenantu.",
|
|
10
|
-
"storage_s3.errors.keyOverrideAccessDenied": "Odmowa dostępu: nadpisany klucz nie jest ograniczony do tego tenantu.",
|
|
11
|
-
"storage_s3.errors.keyRequired": "Wymagany jest parametr zapytania key",
|
|
12
|
-
"storage_s3.errors.maxUploadSize": "Załącznik przekracza maksymalny dopuszczalny rozmiar przesyłania.",
|
|
13
|
-
"storage_s3.errors.multipartRequired": "Oczekiwano multipart/form-data",
|
|
14
|
-
"storage_s3.errors.prefixAccessDenied": "Odmowa dostępu: prefiks nie jest ograniczony do tego tenantu.",
|
|
15
|
-
"storage_s3.errors.quotaExceeded": "Przekroczono limit miejsca na załączniki dla tego tenantu.",
|
|
16
|
-
"storage_s3.errors.unauthorized": "Brak autoryzacji"
|
|
17
|
-
}
|
|
@@ -1,179 +0,0 @@
|
|
|
1
|
-
import { lookup as dnsLookup } from 'node:dns/promises'
|
|
2
|
-
import { Agent as HttpAgent } from 'node:http'
|
|
3
|
-
import { Agent as HttpsAgent } from 'node:https'
|
|
4
|
-
import { isIP, type LookupFunction } from 'node:net'
|
|
5
|
-
import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'
|
|
6
|
-
import {
|
|
7
|
-
assertSafeOutboundUrl,
|
|
8
|
-
assertStaticallySafeOutboundUrl,
|
|
9
|
-
parseOutboundUrl,
|
|
10
|
-
resolveSafeOutboundUrl,
|
|
11
|
-
type HostLookup,
|
|
12
|
-
type ResolvedHostAddress,
|
|
13
|
-
type UrlSafetyReason,
|
|
14
|
-
} from '@open-mercato/shared/lib/url-safety'
|
|
15
|
-
|
|
16
|
-
const SUBJECT = 'S3 endpoint'
|
|
17
|
-
|
|
18
|
-
export class UnsafeS3EndpointError extends Error {
|
|
19
|
-
public readonly reason: string
|
|
20
|
-
|
|
21
|
-
constructor(reason: string, message?: string) {
|
|
22
|
-
super(message ?? `S3 endpoint rejected: ${reason}`)
|
|
23
|
-
this.name = 'UnsafeS3EndpointError'
|
|
24
|
-
this.reason = reason
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
const s3EndpointErrorFactory = (reason: UrlSafetyReason, message: string) =>
|
|
29
|
-
new UnsafeS3EndpointError(reason, message)
|
|
30
|
-
|
|
31
|
-
export type AssertStaticS3EndpointDeps = {
|
|
32
|
-
allowInternal?: boolean
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export function assertStaticallySafeS3Endpoint(
|
|
36
|
-
rawEndpoint: string | null | undefined,
|
|
37
|
-
deps: AssertStaticS3EndpointDeps = {},
|
|
38
|
-
): void {
|
|
39
|
-
if (!rawEndpoint) return
|
|
40
|
-
const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()
|
|
41
|
-
assertStaticallySafeOutboundUrl(rawEndpoint, {
|
|
42
|
-
errorFactory: s3EndpointErrorFactory,
|
|
43
|
-
subject: SUBJECT,
|
|
44
|
-
allowPrivate,
|
|
45
|
-
})
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export type AssertSafeS3EndpointDeps = {
|
|
49
|
-
lookupHost?: HostLookup
|
|
50
|
-
allowInternal?: boolean
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export async function assertSafeS3Endpoint(
|
|
54
|
-
rawEndpoint: string | null | undefined,
|
|
55
|
-
deps: AssertSafeS3EndpointDeps = {},
|
|
56
|
-
): Promise<void> {
|
|
57
|
-
if (!rawEndpoint) return
|
|
58
|
-
const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()
|
|
59
|
-
await assertSafeOutboundUrl(rawEndpoint, {
|
|
60
|
-
errorFactory: s3EndpointErrorFactory,
|
|
61
|
-
subject: SUBJECT,
|
|
62
|
-
allowPrivate,
|
|
63
|
-
lookupHost: deps.lookupHost,
|
|
64
|
-
})
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export type SafeS3EndpointLookupDeps = {
|
|
68
|
-
lookupHost?: HostLookup
|
|
69
|
-
allowInternal?: boolean
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
export function createSafeS3EndpointLookup(
|
|
73
|
-
rawEndpoint: string | null | undefined,
|
|
74
|
-
deps: SafeS3EndpointLookupDeps = {},
|
|
75
|
-
): LookupFunction | undefined {
|
|
76
|
-
if (!rawEndpoint) return undefined
|
|
77
|
-
const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()
|
|
78
|
-
if (allowPrivate) return undefined
|
|
79
|
-
|
|
80
|
-
const { hostname } = parseOutboundUrl(rawEndpoint, {
|
|
81
|
-
errorFactory: s3EndpointErrorFactory,
|
|
82
|
-
subject: SUBJECT,
|
|
83
|
-
})
|
|
84
|
-
const endpointHostname = normalizeLookupHostname(hostname)
|
|
85
|
-
|
|
86
|
-
return (host, options, callback) => {
|
|
87
|
-
const lookupHostname = normalizeLookupHostname(host)
|
|
88
|
-
if (!isAllowedS3LookupHost(lookupHostname, endpointHostname)) {
|
|
89
|
-
callback(new UnsafeS3EndpointError(
|
|
90
|
-
'blocked_hostname',
|
|
91
|
-
`S3 endpoint lookup host "${host}" does not match configured host "${endpointHostname}"`,
|
|
92
|
-
), '', 0)
|
|
93
|
-
return
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
resolveSafeOutboundUrl(buildLookupValidationUrl(rawEndpoint, lookupHostname), {
|
|
97
|
-
errorFactory: s3EndpointErrorFactory,
|
|
98
|
-
subject: SUBJECT,
|
|
99
|
-
lookupHost: deps.lookupHost ?? defaultLookupHost,
|
|
100
|
-
}).then(({ addresses }) => {
|
|
101
|
-
if (!addresses || addresses.length === 0) {
|
|
102
|
-
const family = isIP(lookupHostname)
|
|
103
|
-
if (options.all) callback(null, [{ address: lookupHostname, family }] as ResolvedHostAddress[])
|
|
104
|
-
else callback(null, lookupHostname, family)
|
|
105
|
-
return
|
|
106
|
-
}
|
|
107
|
-
if (options.all) {
|
|
108
|
-
callback(null, addresses.map((record) => ({ address: record.address, family: record.family })))
|
|
109
|
-
return
|
|
110
|
-
}
|
|
111
|
-
callback(null, addresses[0].address, addresses[0].family)
|
|
112
|
-
}).catch((error: unknown) => {
|
|
113
|
-
callback(error instanceof Error ? error : new UnsafeS3EndpointError('dns_resolution_failed'), '', 0)
|
|
114
|
-
})
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
function normalizeLookupHostname(hostname: string): string {
|
|
119
|
-
let normalized = hostname.trim().toLowerCase()
|
|
120
|
-
if (normalized.startsWith('[') && normalized.endsWith(']')) {
|
|
121
|
-
normalized = normalized.slice(1, -1)
|
|
122
|
-
}
|
|
123
|
-
while (normalized.endsWith('.')) {
|
|
124
|
-
normalized = normalized.slice(0, -1)
|
|
125
|
-
}
|
|
126
|
-
return normalized
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
function isAllowedS3LookupHost(lookupHostname: string, endpointHostname: string): boolean {
|
|
130
|
-
if (lookupHostname === endpointHostname) return true
|
|
131
|
-
if (isIP(endpointHostname)) return false
|
|
132
|
-
return lookupHostname.endsWith(`.${endpointHostname}`)
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
function buildLookupValidationUrl(rawEndpoint: string, lookupHostname: string): string {
|
|
136
|
-
const { url } = parseOutboundUrl(rawEndpoint, {
|
|
137
|
-
errorFactory: s3EndpointErrorFactory,
|
|
138
|
-
subject: SUBJECT,
|
|
139
|
-
})
|
|
140
|
-
const host = isIP(lookupHostname) === 6 ? `[${lookupHostname}]` : lookupHostname
|
|
141
|
-
const port = url.port ? `:${url.port}` : ''
|
|
142
|
-
return `${url.protocol}//${host}${port}`
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
export type SafeS3RequestHandlerOptions = {
|
|
146
|
-
connectionTimeout: number
|
|
147
|
-
requestTimeout: number
|
|
148
|
-
httpAgent?: HttpAgent
|
|
149
|
-
httpsAgent?: HttpsAgent
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
const S3_CONNECTION_TIMEOUT_MS = 10_000
|
|
153
|
-
const S3_REQUEST_TIMEOUT_MS = 120_000
|
|
154
|
-
|
|
155
|
-
export function createSafeS3RequestHandler(
|
|
156
|
-
rawEndpoint: string | null | undefined,
|
|
157
|
-
deps: SafeS3EndpointLookupDeps = {},
|
|
158
|
-
): SafeS3RequestHandlerOptions | undefined {
|
|
159
|
-
const lookup = createSafeS3EndpointLookup(rawEndpoint, deps)
|
|
160
|
-
const timeouts = {
|
|
161
|
-
connectionTimeout: S3_CONNECTION_TIMEOUT_MS,
|
|
162
|
-
requestTimeout: S3_REQUEST_TIMEOUT_MS,
|
|
163
|
-
}
|
|
164
|
-
if (!lookup) return timeouts
|
|
165
|
-
return {
|
|
166
|
-
...timeouts,
|
|
167
|
-
httpAgent: new HttpAgent({ lookup }),
|
|
168
|
-
httpsAgent: new HttpsAgent({ lookup }),
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
export function isAllowInternalS3EndpointsEnabled(): boolean {
|
|
173
|
-
return parseBooleanWithDefault(process.env.OM_STORAGE_S3_ALLOW_INTERNAL_ENDPOINTS, false)
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
const defaultLookupHost: HostLookup = async (hostname) => {
|
|
177
|
-
const records = await dnsLookup(hostname, { all: true, verbatim: true })
|
|
178
|
-
return records
|
|
179
|
-
}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import type { AttachmentQuotaService } from '@open-mercato/core/modules/attachments/lib/quota-service'
|
|
2
|
-
import { isS3KeyScopedToTenant } from './key-scope'
|
|
3
|
-
import type { S3StorageDriver } from './s3-driver'
|
|
4
|
-
|
|
5
|
-
export async function reconcileTenantS3Objects(input: {
|
|
6
|
-
driver: S3StorageDriver
|
|
7
|
-
quotaService: AttachmentQuotaService
|
|
8
|
-
tenantId: string
|
|
9
|
-
organizationId: string
|
|
10
|
-
}): Promise<void> {
|
|
11
|
-
const objects: Array<{ path: string; bytes: number }> = []
|
|
12
|
-
let continuationToken: string | undefined
|
|
13
|
-
do {
|
|
14
|
-
const page = await input.driver.listObjects('', 1000, continuationToken)
|
|
15
|
-
for (const file of page.files) {
|
|
16
|
-
if (isS3KeyScopedToTenant(file.key, input.organizationId, input.tenantId)) {
|
|
17
|
-
objects.push({ path: file.key, bytes: file.size })
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
continuationToken = page.truncated ? page.nextContinuationToken : undefined
|
|
21
|
-
} while (continuationToken)
|
|
22
|
-
|
|
23
|
-
await input.quotaService.reconcileStandaloneObjects({
|
|
24
|
-
tenantId: input.tenantId,
|
|
25
|
-
organizationId: input.organizationId,
|
|
26
|
-
storageDriver: 's3',
|
|
27
|
-
objects,
|
|
28
|
-
})
|
|
29
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { createModuleQueue } from '@open-mercato/queue'
|
|
2
|
-
|
|
3
|
-
export type StorageS3QuotaRecoveryJob = {
|
|
4
|
-
reservationId: string
|
|
5
|
-
tenantId: string
|
|
6
|
-
organizationId: string
|
|
7
|
-
absenceCheck?: number
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
const queue = createModuleQueue<StorageS3QuotaRecoveryJob>('storage-s3-quota-recovery', { concurrency: 2 })
|
|
11
|
-
|
|
12
|
-
export async function scheduleStorageS3QuotaRecovery(
|
|
13
|
-
payload: StorageS3QuotaRecoveryJob,
|
|
14
|
-
delayMs: number,
|
|
15
|
-
): Promise<void> {
|
|
16
|
-
await queue.enqueue(payload, { delayMs: Math.max(1_000, delayMs) })
|
|
17
|
-
}
|