lopata 0.14.2 → 0.15.0

package/src/cli/dev.ts

@@ -19,6 +19,7 @@ import {
 } from '../api'
 import { QueuePullConsumer } from '../bindings/queue'
 import type { AckRequest, PullRequest } from '../bindings/queue'
+import type { FileR2Bucket } from '../bindings/r2'
 import { CFWebSocket } from '../bindings/websocket-pair'
 import { autoLoadConfig, loadConfig } from '../config'
 import { handleDashboardRequest } from '../dashboard-serve'
@@ -28,6 +29,7 @@ import { GenerationManager } from '../generation-manager'
 import { loadLopataConfig } from '../lopata-config'
 import { addCfProperty } from '../request-cf'
 import { extractHostname, RouteDispatcher } from '../route-matcher'
+import { handleS3Request, matchS3Path } from '../s3/proxy'
 import { getTraceStore } from '../tracing/store'
 import type { TraceEvent } from '../tracing/types'
 import { WorkerRegistry } from '../worker-registry'
@@ -260,6 +262,41 @@ export async function run(ctx: CliContext, args: string[]) {
 				return handleDashboardRequest(request)
 			}
 
+			// S3-compatible proxy: /__s3/{bucket}/{key...} → R2 binding on active worker
+			const s3Match = matchS3Path(url.pathname)
+			if (s3Match) {
+				const targetManager = resolveWorkerParam(url, registry, manager)
+				const gen = targetManager.active
+				const binding = gen?.env[s3Match.bucket] as FileR2Bucket | undefined
+				const resolveBucket = (name: string) => gen?.env[name] as FileR2Bucket | undefined
+				const listAllBuckets = () => {
+					if (!gen) return []
+					const out: Array<{ name: string; creationDate: Date }> = []
+					for (const [name, value] of Object.entries(gen.env)) {
+						// Duck-typed R2Bucket check — instrumentBinding wraps in a Proxy, so instanceof
+						// isn't reliable. R2 bindings are distinguished by having these methods.
+						if (
+							value
+							&& typeof (value as { put?: unknown }).put === 'function'
+							&& typeof (value as { head?: unknown }).head === 'function'
+							&& typeof (value as { createMultipartUpload?: unknown }).createMultipartUpload === 'function'
+						) {
+							out.push({ name, creationDate: new Date(0) })
+						}
+					}
+					return out
+				}
+				const rewritten = new URL(request.url)
+				rewritten.pathname = '/' + s3Match.keyPath
+				const virtualReq = new Request(rewritten.toString(), {
+					method: request.method,
+					headers: request.headers,
+					body: request.body,
+					duplex: 'half',
+				})
+				return handleS3Request(virtualReq, s3Match.bucket, binding, resolveBucket, listAllBuckets)
+			}
+
 			// Queue pull consumer endpoints: POST /cdn-cgi/handler/queues/<name>/messages/pull and /ack
 			const queuePullMatch = url.pathname.match(/^\/cdn-cgi\/handler\/queues\/([^/]+)\/messages\/(pull|ack)$/)
 			if (queuePullMatch && request.method === 'POST') {
@@ -471,6 +508,13 @@ export async function run(ctx: CliContext, args: string[]) {
 	console.log(`[lopata] Server running at http://${hostname}:${port}`)
 	console.log(`[lopata] Dashboard: http://${hostname}:${port}/__dashboard`)
 
+	if (hostname === '0.0.0.0') {
+		console.warn(
+			`[lopata] WARNING: S3 endpoint at /__s3/<bucket> is UNAUTHENTICATED and reachable from the network.`,
+		)
+		console.warn('[lopata]          Anyone on your LAN can read and write local R2 buckets.')
+	}
+
 	// Graceful shutdown
 	const shutdown = () => {
 		console.log('\n[lopata] Shutting down…')

package/src/s3/chunked.ts

@@ -0,0 +1,129 @@
+/**
+ * AWS sigv4 streaming payload decoder.
+ *
+ * When an S3 client signs with STREAMING-AWS4-HMAC-SHA256-PAYLOAD (the default
+ * in aws-sdk-js v3), the request body is wrapped in chunk framing:
+ *
+ *     <hex-size>;chunk-signature=<hex-sig>\r\n
+ *     <data bytes, length = hex-size>\r\n
+ *     ...
+ *     0;chunk-signature=<hex-sig>\r\n
+ *     \r\n
+ *
+ * This decoder strips the framing and emits only the data bytes. It does NOT
+ * verify signatures (lopata's S3 proxy is unauthenticated dev-only).
+ */
+
+export function isAwsChunked(headers: Headers): boolean {
+	const h = headers.get('x-amz-content-sha256')
+	if (!h) return false
+	return h === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
+		|| h === 'STREAMING-UNSIGNED-PAYLOAD-TRAILER'
+		|| h === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER'
+}
+
+/**
+ * Decode an aws-chunked ReadableStream into a stream of just the data bytes.
+ * Buffers incoming bytes, consumes framing lines, and emits data chunks.
+ */
+export function decodeAwsChunked(stream: ReadableStream<Uint8Array>): ReadableStream<Uint8Array> {
+	const reader = stream.getReader()
+	let buffer = new Uint8Array(0)
+	// expecting 'header' (chunk-size;chunk-signature=...) or 'data'
+	type State =
+		| { kind: 'header' }
+		| { kind: 'data'; remaining: number }
+		| { kind: 'crlf-after-data' }
+		| { kind: 'done' }
+	let state: State = { kind: 'header' }
+
+	function append(chunk: Uint8Array) {
+		const merged = new Uint8Array(buffer.length + chunk.length)
+		merged.set(buffer, 0)
+		merged.set(chunk, buffer.length)
+		buffer = merged
+	}
+
+	function indexOfCRLF(buf: Uint8Array): number {
+		for (let i = 0; i < buf.length - 1; i++) {
+			if (buf[i] === 0x0d && buf[i + 1] === 0x0a) return i
+		}
+		return -1
+	}
+
+	return new ReadableStream({
+		async pull(controller) {
+			while (true) {
+				if (state.kind === 'done') {
+					controller.close()
+					return
+				}
+
+				if (state.kind === 'header') {
+					const crlf = indexOfCRLF(buffer)
+					if (crlf === -1) {
+						const { done, value } = await reader.read()
+						if (done) {
+							// Stream ended without proper termination — that's ok if we're empty.
+							controller.close()
+							return
+						}
+						append(value)
+						continue
+					}
+					const line = new TextDecoder().decode(buffer.subarray(0, crlf))
+					buffer = buffer.subarray(crlf + 2)
+					const sizeHex = line.split(';')[0]!.trim()
+					const size = parseInt(sizeHex, 16)
+					if (Number.isNaN(size)) {
+						controller.error(new Error(`Invalid chunk size: ${line}`))
+						return
+					}
+					if (size === 0) {
+						state = { kind: 'done' }
+						controller.close()
+						return
+					}
+					state = { kind: 'data', remaining: size }
+					continue
+				}
+
+				if (state.kind === 'data') {
+					if (buffer.length === 0) {
+						const { done, value } = await reader.read()
+						if (done) {
+							controller.error(new Error('Unexpected end of aws-chunked stream inside data'))
+							return
+						}
+						append(value)
+						continue
+					}
+					const take = Math.min(state.remaining, buffer.length)
+					controller.enqueue(buffer.subarray(0, take))
+					buffer = buffer.subarray(take)
+					state = { kind: 'data', remaining: state.remaining - take }
+					if (state.remaining === 0) {
+						state = { kind: 'crlf-after-data' }
+					}
+					return
+				}
+
+				if (state.kind === 'crlf-after-data') {
+					while (buffer.length < 2) {
+						const { done, value } = await reader.read()
+						if (done) {
+							controller.error(new Error('Unexpected end of aws-chunked stream before CRLF'))
+							return
+						}
+						append(value)
+					}
+					buffer = buffer.subarray(2)
+					state = { kind: 'header' }
+				}
+			}
+		},
+		cancel(reason) {
+			return reader.cancel(reason)
+		},
+	})
+}

package/src/s3/headers.ts

@@ -0,0 +1,131 @@
+import { HTTP_METADATA_FIELDS, type R2HTTPMetadata, type R2Object, type R2Range } from '../bindings/r2'
+
+export function extractPutOptions(req: Request): {
+	httpMetadata: R2HTTPMetadata
+	customMetadata: Record<string, string>
+} {
+	const h = req.headers
+	const httpMetadata: R2HTTPMetadata = {}
+	for (const [header, field] of HTTP_METADATA_FIELDS) {
+		const v = h.get(header)
+		if (!v) continue
+		if (field === 'cacheExpiry') {
+			const d = new Date(v)
+			if (!Number.isNaN(d.getTime())) httpMetadata.cacheExpiry = d
+		} else {
+			;(httpMetadata[field] as string) = v
+		}
+	}
+
+	const customMetadata: Record<string, string> = {}
+	for (const [k, v] of h) {
+		if (k.toLowerCase().startsWith('x-amz-meta-')) {
+			customMetadata[k.slice('x-amz-meta-'.length)] = v
+		}
+	}
+	return { httpMetadata, customMetadata }
+}
+
+export function applyObjectHeaders(obj: R2Object, headers = new Headers()): Headers {
+	headers.set('ETag', `"${obj.etag}"`)
+	headers.set('Last-Modified', obj.uploaded.toUTCString())
+	headers.set('Content-Length', String(obj.size))
+	obj.writeHttpMetadata(headers)
+	for (const [k, v] of Object.entries(obj.customMetadata)) {
+		headers.set(`x-amz-meta-${k}`, v)
+	}
+	return headers
+}
+
+/**
+ * Parse a Range header: bytes=<start>-<end> | bytes=<start>- | bytes=-<suffix>.
+ * Returns null if the header is absent or malformed.
+ */
+export function parseRange(header: string | null): R2Range | null {
+	if (!header) return null
+	const m = header.match(/^bytes=(\d*)-(\d*)$/)
+	if (!m) return null
+	const startStr = m[1]!
+	const endStr = m[2]!
+	if (startStr === '' && endStr === '') return null
+	if (startStr === '') {
+		return { suffix: Number(endStr) }
+	}
+	const offset = Number(startStr)
+	if (endStr === '') return { offset }
+	return { offset, length: Number(endStr) - offset + 1 }
+}
+
+export interface Conditional {
+	ifMatch?: string[]
+	ifNoneMatch?: string[]
+	ifModifiedSince?: Date
+	ifUnmodifiedSince?: Date
+}
+
+export function parseConditional(headers: Headers): Conditional {
+	const c: Conditional = {}
+	const ifMatch = headers.get('if-match')
+	if (ifMatch) c.ifMatch = ifMatch.split(',').map((s) => s.trim().replace(/^"|"$/g, ''))
+	const ifNoneMatch = headers.get('if-none-match')
+	if (ifNoneMatch) c.ifNoneMatch = ifNoneMatch.split(',').map((s) => s.trim().replace(/^"|"$/g, ''))
+	const ims = headers.get('if-modified-since')
+	if (ims) {
+		const d = new Date(ims)
+		if (!Number.isNaN(d.getTime())) c.ifModifiedSince = d
+	}
+	const iums = headers.get('if-unmodified-since')
+	if (iums) {
+		const d = new Date(iums)
+		if (!Number.isNaN(d.getTime())) c.ifUnmodifiedSince = d
+	}
+	return c
+}
+
+/**
+ * Evaluate a conditional against an existing object.
+ * Returns 'match' (preconditions met, proceed), 'not-modified' (GET/HEAD 304),
+ * or 'precondition-failed' (412).
+ *
+ * Per RFC 7232 & S3 semantics:
+ *   If-Match fails → 412
+ *   If-Unmodified-Since fails → 412
+ *   If-None-Match matches → 304 for GET/HEAD, 412 for others
+ *   If-Modified-Since not modified → 304 for GET/HEAD, ignored otherwise
+ */
+export type ConditionalResult = 'match' | 'not-modified' | 'precondition-failed'
+
+export function evaluateConditional(
+	cond: Conditional,
+	obj: { etag: string; uploaded: Date },
+	method: 'read' | 'write',
+): ConditionalResult {
+	const etag = obj.etag
+	// Normalise "uploaded" to second precision — HTTP dates have 1-second resolution.
+	const uploadedSec = Math.floor(obj.uploaded.getTime() / 1000) * 1000
+
+	if (cond.ifMatch) {
+		const matches = cond.ifMatch.some((t) => t === '*' || t === etag)
+		if (!matches) return 'precondition-failed'
+	}
+	if (cond.ifUnmodifiedSince) {
+		if (uploadedSec > cond.ifUnmodifiedSince.getTime()) return 'precondition-failed'
+	}
+	if (cond.ifNoneMatch) {
+		const matches = cond.ifNoneMatch.some((t) => t === '*' || t === etag)
+		if (matches) return method === 'read' ? 'not-modified' : 'precondition-failed'
+	}
+	if (cond.ifModifiedSince && method === 'read') {
+		if (uploadedSec <= cond.ifModifiedSince.getTime()) return 'not-modified'
+	}
+	return 'match'
+}
+
+export function corsHeaders(origin: string | null): Headers {
+	const h = new Headers()
+	h.set('Access-Control-Allow-Origin', origin ?? '*')
+	h.set('Access-Control-Allow-Methods', 'GET, PUT, POST, HEAD, DELETE, OPTIONS')
+	h.set('Access-Control-Allow-Headers', '*')
+	h.set('Access-Control-Expose-Headers', '*')
+	return h
+}