@open-mercato/storage-s3 0.6.7 → 0.6.8-develop.6874.1.982d6097d8

package/src/modules/storage_s3/lib/endpoint-safety.ts

@@ -0,0 +1,179 @@
+import { lookup as dnsLookup } from 'node:dns/promises'
+import { Agent as HttpAgent } from 'node:http'
+import { Agent as HttpsAgent } from 'node:https'
+import { isIP, type LookupFunction } from 'node:net'
+import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'
+import {
+  assertSafeOutboundUrl,
+  assertStaticallySafeOutboundUrl,
+  parseOutboundUrl,
+  resolveSafeOutboundUrl,
+  type HostLookup,
+  type ResolvedHostAddress,
+  type UrlSafetyReason,
+} from '@open-mercato/shared/lib/url-safety'
+
+const SUBJECT = 'S3 endpoint'
+
+export class UnsafeS3EndpointError extends Error {
+  public readonly reason: string
+
+  constructor(reason: string, message?: string) {
+    super(message ?? `S3 endpoint rejected: ${reason}`)
+    this.name = 'UnsafeS3EndpointError'
+    this.reason = reason
+  }
+}
+
+const s3EndpointErrorFactory = (reason: UrlSafetyReason, message: string) =>
+  new UnsafeS3EndpointError(reason, message)
+
+export type AssertStaticS3EndpointDeps = {
+  allowInternal?: boolean
+}
+
+export function assertStaticallySafeS3Endpoint(
+  rawEndpoint: string | null | undefined,
+  deps: AssertStaticS3EndpointDeps = {},
+): void {
+  if (!rawEndpoint) return
+  const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()
+  assertStaticallySafeOutboundUrl(rawEndpoint, {
+    errorFactory: s3EndpointErrorFactory,
+    subject: SUBJECT,
+    allowPrivate,
+  })
+}
+
+export type AssertSafeS3EndpointDeps = {
+  lookupHost?: HostLookup
+  allowInternal?: boolean
+}
+
+export async function assertSafeS3Endpoint(
+  rawEndpoint: string | null | undefined,
+  deps: AssertSafeS3EndpointDeps = {},
+): Promise<void> {
+  if (!rawEndpoint) return
+  const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()
+  await assertSafeOutboundUrl(rawEndpoint, {
+    errorFactory: s3EndpointErrorFactory,
+    subject: SUBJECT,
+    allowPrivate,
+    lookupHost: deps.lookupHost,
+  })
+}
+
+export type SafeS3EndpointLookupDeps = {
+  lookupHost?: HostLookup
+  allowInternal?: boolean
+}
+
+export function createSafeS3EndpointLookup(
+  rawEndpoint: string | null | undefined,
+  deps: SafeS3EndpointLookupDeps = {},
+): LookupFunction | undefined {
+  if (!rawEndpoint) return undefined
+  const allowPrivate = deps.allowInternal ?? isAllowInternalS3EndpointsEnabled()
+  if (allowPrivate) return undefined
+
+  const { hostname } = parseOutboundUrl(rawEndpoint, {
+    errorFactory: s3EndpointErrorFactory,
+    subject: SUBJECT,
+  })
+  const endpointHostname = normalizeLookupHostname(hostname)
+
+  return (host, options, callback) => {
+    const lookupHostname = normalizeLookupHostname(host)
+    if (!isAllowedS3LookupHost(lookupHostname, endpointHostname)) {
+      callback(new UnsafeS3EndpointError(
+        'blocked_hostname',
+        `S3 endpoint lookup host "${host}" does not match configured host "${endpointHostname}"`,
+      ), '', 0)
+      return
+    }
+
+    resolveSafeOutboundUrl(buildLookupValidationUrl(rawEndpoint, lookupHostname), {
+      errorFactory: s3EndpointErrorFactory,
+      subject: SUBJECT,
+      lookupHost: deps.lookupHost ?? defaultLookupHost,
+    }).then(({ addresses }) => {
+      if (!addresses || addresses.length === 0) {
+        const family = isIP(lookupHostname)
+        if (options.all) callback(null, [{ address: lookupHostname, family }] as ResolvedHostAddress[])
+        else callback(null, lookupHostname, family)
+        return
+      }
+      if (options.all) {
+        callback(null, addresses.map((record) => ({ address: record.address, family: record.family })))
+        return
+      }
+      callback(null, addresses[0].address, addresses[0].family)
+    }).catch((error: unknown) => {
+      callback(error instanceof Error ? error : new UnsafeS3EndpointError('dns_resolution_failed'), '', 0)
+    })
+  }
+}
+
+function normalizeLookupHostname(hostname: string): string {
+  let normalized = hostname.trim().toLowerCase()
+  if (normalized.startsWith('[') && normalized.endsWith(']')) {
+    normalized = normalized.slice(1, -1)
+  }
+  while (normalized.endsWith('.')) {
+    normalized = normalized.slice(0, -1)
+  }
+  return normalized
+}
+
+function isAllowedS3LookupHost(lookupHostname: string, endpointHostname: string): boolean {
+  if (lookupHostname === endpointHostname) return true
+  if (isIP(endpointHostname)) return false
+  return lookupHostname.endsWith(`.${endpointHostname}`)
+}
+
+function buildLookupValidationUrl(rawEndpoint: string, lookupHostname: string): string {
+  const { url } = parseOutboundUrl(rawEndpoint, {
+    errorFactory: s3EndpointErrorFactory,
+    subject: SUBJECT,
+  })
+  const host = isIP(lookupHostname) === 6 ? `[${lookupHostname}]` : lookupHostname
+  const port = url.port ? `:${url.port}` : ''
+  return `${url.protocol}//${host}${port}`
+}
+
+export type SafeS3RequestHandlerOptions = {
+  connectionTimeout: number
+  requestTimeout: number
+  httpAgent?: HttpAgent
+  httpsAgent?: HttpsAgent
+}
+
+const S3_CONNECTION_TIMEOUT_MS = 10_000
+const S3_REQUEST_TIMEOUT_MS = 120_000
+
+export function createSafeS3RequestHandler(
+  rawEndpoint: string | null | undefined,
+  deps: SafeS3EndpointLookupDeps = {},
+): SafeS3RequestHandlerOptions | undefined {
+  const lookup = createSafeS3EndpointLookup(rawEndpoint, deps)
+  const timeouts = {
+    connectionTimeout: S3_CONNECTION_TIMEOUT_MS,
+    requestTimeout: S3_REQUEST_TIMEOUT_MS,
+  }
+  if (!lookup) return timeouts
+  return {
+    ...timeouts,
+    httpAgent: new HttpAgent({ lookup }),
+    httpsAgent: new HttpsAgent({ lookup }),
+  }
+}
+
+export function isAllowInternalS3EndpointsEnabled(): boolean {
+  return parseBooleanWithDefault(process.env.OM_STORAGE_S3_ALLOW_INTERNAL_ENDPOINTS, false)
+}
+
+const defaultLookupHost: HostLookup = async (hostname) => {
+  const records = await dnsLookup(hostname, { all: true, verbatim: true })
+  return records
+}

package/src/modules/storage_s3/lib/health.ts

@@ -1,4 +1,5 @@
 import { S3Client, HeadBucketCommand } from '@aws-sdk/client-s3'
+import { assertSafeS3Endpoint, createSafeS3RequestHandler } from './endpoint-safety'
 import type { S3DriverConfig } from './s3-driver'
 
 export const s3HealthCheck = {
@@ -67,11 +68,28 @@ export const s3HealthCheck = {
       }
     }
 
+    try {
+      await assertSafeS3Endpoint(cfg.endpoint)
+    } catch (err: unknown) {
+      const message = err instanceof Error ? err.message : 'Unsafe S3 endpoint'
+      const reason = err && typeof err === 'object' && 'reason' in err ? (err as { reason?: unknown }).reason : undefined
+      return {
+        status: 'unhealthy',
+        message,
+        details: {
+          ...(typeof reason === 'string' ? { reason } : {}),
+        },
+        checkedAt,
+      }
+    }
+
+    const requestHandler = createSafeS3RequestHandler(cfg.endpoint)
     const client = new S3Client({
       region: cfg.region ?? 'us-east-1',
       endpoint: cfg.endpoint,
       forcePathStyle: cfg.forcePathStyle ?? false,
       credentials: credentialConfig,
+      ...(requestHandler ? { requestHandler } : {}),
     })
 
     try {

package/src/modules/storage_s3/lib/key-scope.ts

@@ -113,7 +113,7 @@ export function assertS3KeyScopedToTenant(
   pathPrefix?: string,
 ): void {
   if (!isS3KeyScopedToTenant(storagePath, scope, pathPrefix)) {
-    throw new Error('S3 key is not scoped to the active tenant')
+    throw new Error('[internal] S3 key is not scoped to the active tenant')
   }
 }
 
@@ -123,7 +123,7 @@ export function assertS3KeyAddressableByTenantScope(
   pathPrefix?: string,
 ): void {
   if (!isS3KeyAddressableByTenantScope(storagePath, scope, pathPrefix)) {
-    throw new Error('S3 key is not scoped to the active tenant')
+    throw new Error('[internal] S3 key is not scoped to the active tenant')
   }
 }
 
@@ -140,7 +140,7 @@ export function assertS3ListPrefixScopedToTenant(
 
   if (parts[0]?.startsWith('org_') || parts[1]?.startsWith('tenant_')) {
     if (parts[0] !== segments.orgSegment || parts[1] !== segments.tenantSegment) {
-      throw new Error('S3 prefix is not scoped to the active tenant')
+      throw new Error('[internal] S3 prefix is not scoped to the active tenant')
     }
     return
   }
@@ -149,7 +149,7 @@ export function assertS3ListPrefixScopedToTenant(
     (parts[1]?.startsWith('org_') && parts[1] !== segments.orgSegment)
     || (parts[2]?.startsWith('tenant_') && parts[2] !== segments.tenantSegment)
   ) {
-    throw new Error('S3 prefix is not scoped to the active tenant')
+    throw new Error('[internal] S3 prefix is not scoped to the active tenant')
   }
 }
 

package/src/modules/storage_s3/lib/quota-accounting.ts

@@ -0,0 +1,29 @@
+import type { AttachmentQuotaService } from '@open-mercato/core/modules/attachments/lib/quota-service'
+import { isS3KeyScopedToTenant } from './key-scope'
+import type { S3StorageDriver } from './s3-driver'
+
+export async function reconcileTenantS3Objects(input: {
+  driver: S3StorageDriver
+  quotaService: AttachmentQuotaService
+  tenantId: string
+  organizationId: string
+}): Promise<void> {
+  const objects: Array<{ path: string; bytes: number }> = []
+  let continuationToken: string | undefined
+  do {
+    const page = await input.driver.listObjects('', 1000, continuationToken)
+    for (const file of page.files) {
+      if (isS3KeyScopedToTenant(file.key, input.organizationId, input.tenantId)) {
+        objects.push({ path: file.key, bytes: file.size })
+      }
+    }
+    continuationToken = page.truncated ? page.nextContinuationToken : undefined
+  } while (continuationToken)
+
+  await input.quotaService.reconcileStandaloneObjects({
+    tenantId: input.tenantId,
+    organizationId: input.organizationId,
+    storageDriver: 's3',
+    objects,
+  })
+}

package/src/modules/storage_s3/lib/quota-recovery-queue.ts

@@ -0,0 +1,17 @@
+import { createModuleQueue } from '@open-mercato/queue'
+
+export type StorageS3QuotaRecoveryJob = {
+  reservationId: string
+  tenantId: string
+  organizationId: string
+  absenceCheck?: number
+}
+
+const queue = createModuleQueue<StorageS3QuotaRecoveryJob>('storage-s3-quota-recovery', { concurrency: 2 })
+
+export async function scheduleStorageS3QuotaRecovery(
+  payload: StorageS3QuotaRecoveryJob,
+  delayMs: number,
+): Promise<void> {
+  await queue.enqueue(payload, { delayMs: Math.max(1_000, delayMs) })
+}

package/src/modules/storage_s3/lib/s3-driver.ts

@@ -10,7 +10,12 @@ import {
   ListObjectsV2Command,
 } from '@aws-sdk/client-s3'
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
-import type { StorageDriver, StoreFilePayload, StoredFile, ReadFileResult } from '@open-mercato/core/modules/attachments/lib/drivers'
+import type { PrepareFilePayload, StorageDriver, StoreFilePayload, StoredFile, ReadFileResult } from '@open-mercato/core/modules/attachments/lib/drivers'
+import {
+  assertSafeS3Endpoint,
+  assertStaticallySafeS3Endpoint,
+  createSafeS3RequestHandler,
+} from './endpoint-safety'
 import {
   assertS3KeyAddressableByTenantScope,
   assertS3KeyScopedToTenant,
@@ -71,12 +76,14 @@ export class S3StorageDriver implements StorageDriver {
   readonly key = 's3'
   private readonly client: S3Client
   private readonly bucket: string
+  private readonly endpoint?: string
   private readonly pathPrefix: string
   private readonly scope: S3TenantScope | null
 
   constructor(config: Record<string, unknown>) {
     const cfg = config as S3DriverConfig
     this.bucket = cfg.bucket
+    this.endpoint = cfg.endpoint
     this.pathPrefix = cfg.pathPrefix ?? ''
     this.scope = cfg.organizationId && cfg.tenantId
       ? { organizationId: cfg.organizationId, tenantId: cfg.tenantId }
@@ -109,11 +116,15 @@ export class S3StorageDriver implements StorageDriver {
       }
     }
 
+    assertStaticallySafeS3Endpoint(cfg.endpoint)
+    const requestHandler = createSafeS3RequestHandler(cfg.endpoint)
+
     this.client = new S3Client({
       region: cfg.region ?? 'us-east-1',
       endpoint: cfg.endpoint,
       forcePathStyle: cfg.forcePathStyle ?? false,
       credentials,
+      ...(requestHandler ? { requestHandler } : {}),
     })
   }
 
@@ -132,17 +143,24 @@ export class S3StorageDriver implements StorageDriver {
       : storagePath.replace(/^\/+/, '')
     const firstSegment = prefix.split('/').filter(Boolean)[0]
     if (firstSegment !== partitionCode) {
-      throw new Error('S3 key is not scoped to the requested partition')
+      throw new Error('[internal] S3 key is not scoped to the requested partition')
     }
   }
 
-  async store(payload: StoreFilePayload): Promise<StoredFile> {
+  prepareStoragePath(payload: PrepareFilePayload): string {
     const orgSegment = resolveOrgSegment(payload.orgId ?? null)
     const tenantSegment = resolveTenantSegment(payload.tenantId ?? null)
     const safeName = sanitizeFileName(payload.fileName || 'file')
     const uniqueSuffix = randomUUID().replace(/-/g, '').slice(0, 12)
     const storedName = `${Date.now()}_${uniqueSuffix}_${safeName}`
-    const key = `${this.pathPrefix}${payload.partitionCode}/${orgSegment}/${tenantSegment}/${storedName}`
+    return `${this.pathPrefix}${payload.partitionCode}/${orgSegment}/${tenantSegment}/${storedName}`
+  }
+
+  async store(payload: StoreFilePayload): Promise<StoredFile> {
+    const key = payload.storagePath ?? this.prepareStoragePath(payload)
+    this.assertPartitionScoped(payload.partitionCode, key)
+    this.assertKeyScoped(key)
+    await this.assertEndpointSafe()
 
     await this.client.send(
       new PutObjectCommand({
@@ -150,6 +168,7 @@ export class S3StorageDriver implements StorageDriver {
         Key: key,
         Body: payload.buffer,
         ContentLength: payload.buffer.length,
+        IfNoneMatch: '*',
       }),
     )
 
@@ -159,11 +178,12 @@ export class S3StorageDriver implements StorageDriver {
   async read(partitionCode: string, storagePath: string): Promise<ReadFileResult> {
     this.assertPartitionScoped(partitionCode, storagePath)
     this.assertKeyAddressable(storagePath)
+    await this.assertEndpointSafe()
     const response = await this.client.send(
       new GetObjectCommand({ Bucket: this.bucket, Key: storagePath }),
     )
     if (!response.Body) {
-      throw new Error(`S3 object body is empty for key: ${storagePath}`)
+      throw new Error(`[internal] S3 object body is empty for key: ${storagePath}`)
     }
     const buffer = await streamToBuffer(response.Body as NodeJS.ReadableStream)
     return { buffer, contentType: response.ContentType }
@@ -173,14 +193,19 @@ export class S3StorageDriver implements StorageDriver {
     this.assertPartitionScoped(partitionCode, storagePath)
     this.assertKeyAddressable(storagePath)
     try {
-      await this.client.send(
-        new DeleteObjectCommand({ Bucket: this.bucket, Key: storagePath }),
-      )
+      await this.deleteStrict(partitionCode, storagePath)
     } catch {
-      // best-effort removal
+      // Backward-compatible best-effort deletion.
     }
   }
 
+  async deleteStrict(partitionCode: string, storagePath: string): Promise<void> {
+    this.assertPartitionScoped(partitionCode, storagePath)
+    this.assertKeyAddressable(storagePath)
+    await this.assertEndpointSafe()
+    await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: storagePath }))
+  }
+
   async toLocalPath(
     partitionCode: string,
     storagePath: string,
@@ -207,6 +232,7 @@ export class S3StorageDriver implements StorageDriver {
    */
   async putObject(key: string, buffer: Buffer, contentType?: string): Promise<void> {
     this.assertKeyScoped(key)
+    await this.assertEndpointSafe()
     await this.client.send(
       new PutObjectCommand({
         Bucket: this.bucket,
@@ -214,6 +240,7 @@ export class S3StorageDriver implements StorageDriver {
         Body: buffer,
         ContentType: contentType,
         ContentLength: buffer.length,
+        IfNoneMatch: '*',
       }),
     )
   }
@@ -234,6 +261,7 @@ export class S3StorageDriver implements StorageDriver {
   }> {
     const activeScope = scope ?? this.scope
     assertS3ListPrefixScopedToTenant(prefix, activeScope, this.pathPrefix)
+    await this.assertEndpointSafe()
     const response = await this.client.send(
       new ListObjectsV2Command({
         Bucket: this.bucket,
@@ -260,24 +288,51 @@ export class S3StorageDriver implements StorageDriver {
   async getSignedUrl(
     storagePath: string,
     operation: 'upload' | 'download',
-    expiresIn = 3600,
+    expiresIn?: number,
     contentType?: string,
+    contentLength?: number,
+    createOnly?: boolean,
+    scope?: S3TenantScope | null,
+  ): Promise<string>
+  async getSignedUrl(
+    storagePath: string,
+    operation: 'upload' | 'download',
+    expiresIn: number,
+    contentType: string | undefined,
     scope?: S3TenantScope | null,
+  ): Promise<string>
+  async getSignedUrl(
+    storagePath: string,
+    operation: 'upload' | 'download',
+    expiresIn = 3600,
+    contentType?: string,
+    contentLengthOrScope?: number | S3TenantScope | null,
+    createOnly = false,
+    explicitScope?: S3TenantScope | null,
   ): Promise<string> {
+    const contentLength = typeof contentLengthOrScope === 'number' ? contentLengthOrScope : undefined
+    const scope = typeof contentLengthOrScope === 'number' ? explicitScope : contentLengthOrScope
     if (operation === 'upload') {
       this.assertKeyScoped(storagePath, scope)
     } else {
       this.assertKeyAddressable(storagePath, scope)
     }
+    await this.assertEndpointSafe()
     if (operation === 'upload') {
       const command = new PutObjectCommand({
         Bucket: this.bucket,
         Key: storagePath,
         ContentType: contentType,
+        ContentLength: contentLength,
+        ...(createOnly ? { IfNoneMatch: '*' } : {}),
       })
       return getSignedUrl(this.client, command, { expiresIn })
     }
     const command = new GetObjectCommand({ Bucket: this.bucket, Key: storagePath })
     return getSignedUrl(this.client, command, { expiresIn })
   }
+
+  private async assertEndpointSafe(): Promise<void> {
+    await assertSafeS3Endpoint(this.endpoint)
+  }
 }