lopata 0.0.1

package/runtime/cli/dev.ts

@@ -0,0 +1,392 @@
+// Capture deeper stacks in dev mode (default is 10, async boundaries still truncate)
+Error.stackTraceLimit = 50;
+
+import "../plugin";
+import type { CliContext } from "./context";
+import { parseFlag } from "./context";
+import { autoLoadConfig, loadConfig } from "../config";
+import { GenerationManager } from "../generation-manager";
+import { FileWatcher } from "../file-watcher";
+import { WorkerRegistry } from "../worker-registry";
+import { loadBunflareConfig } from "../bunflare-config";
+import { QueuePullConsumer } from "../bindings/queue";
+import type { PullRequest, AckRequest } from "../bindings/queue";
+import { getDatabase } from "../db";
+import { addCfProperty } from "../request-cf";
+import { handleDashboardRequest, setDashboardConfig, setGenerationManager, setWorkerRegistry } from "../dashboard/api";
+import { CFWebSocket } from "../bindings/websocket-pair";
+import { getTraceStore } from "../tracing/store";
+import type { TraceEvent } from "../tracing/types";
+import path from "node:path";
+
+export async function run(ctx: CliContext) {
+	const envFlag = parseFlag(ctx.args, "--env") ?? parseFlag(ctx.args, "-e");
+	const listenFlag = parseFlag(ctx.args, "--listen");
+	const portFlag = parseFlag(ctx.args, "--port");
+
+	const baseDir = process.cwd();
+	const watchers: FileWatcher[] = [];
+
+	// Try to load bunflare.config.ts for multi-worker mode
+	const bunflareConfig = await loadBunflareConfig(baseDir);
+
+	let manager: GenerationManager;
+
+	if (bunflareConfig) {
+		// ─── Multi-worker mode ─────────────────────────────────────────
+		console.log("[bunflare] Multi-worker mode (bunflare.config.ts found)");
+
+		// Create executor factory based on isolation mode
+		let executorFactory: import("../bindings/do-executor").DOExecutorFactory | undefined;
+		if (bunflareConfig.isolation === "isolated") {
+			const { WorkerExecutorFactory } = await import("../bindings/do-executor-worker");
+			executorFactory = new WorkerExecutorFactory();
+			console.log("[bunflare] DO isolation: isolated (Worker threads)");
+		} else if (bunflareConfig.isolation && bunflareConfig.isolation !== "dev") {
+			console.warn(`[bunflare] Unknown isolation mode "${bunflareConfig.isolation}", using "dev"`);
+		}
+
+		const registry = new WorkerRegistry();
+
+		// Load main worker config
+		const mainConfig = await loadConfig(bunflareConfig.main, envFlag);
+		const mainBaseDir = path.dirname(bunflareConfig.main);
+		console.log(`[bunflare] Main worker: ${mainConfig.name}${envFlag ? ` (env: ${envFlag})` : ""}`);
+		setDashboardConfig(mainConfig);
+
+		const mainManager = new GenerationManager(mainConfig, mainBaseDir, {
+			workerName: mainConfig.name,
+			workerRegistry: registry,
+			isMain: true,
+			cron: bunflareConfig.cron,
+			executorFactory,
+			configPath: bunflareConfig.main,
+			browserConfig: bunflareConfig.browser,
+		});
+		registry.register(mainConfig.name, mainManager, true);
+
+		// Load auxiliary workers
+		for (const workerDef of bunflareConfig.workers ?? []) {
+			const auxConfig = await loadConfig(workerDef.config, envFlag);
+			const auxBaseDir = path.dirname(workerDef.config);
+			console.log(`[bunflare] Auxiliary worker: ${workerDef.name} (${auxConfig.name})`);
+
+			const auxManager = new GenerationManager(auxConfig, auxBaseDir, {
+				workerName: workerDef.name,
+				workerRegistry: registry,
+				isMain: false,
+				cron: bunflareConfig.cron,
+				executorFactory,
+				configPath: workerDef.config,
+			});
+			registry.register(workerDef.name, auxManager);
+
+			// Load aux worker first so main's service bindings can resolve
+			try {
+				const gen = await auxManager.reload();
+				console.log(`[bunflare] Auxiliary worker "${workerDef.name}" → generation ${gen.id}`);
+			} catch (err) {
+				console.error(`[bunflare] Failed to load auxiliary worker "${workerDef.name}":`, err);
+			}
+
+			// File watcher for aux worker
+			const auxSrcDir = path.dirname(path.resolve(auxBaseDir, auxConfig.main));
+			const auxWatcher = new FileWatcher(auxSrcDir, () => {
+				auxManager.reload().then(gen => {
+					console.log(`[bunflare] Auxiliary worker "${workerDef.name}" reloaded → generation ${gen.id}`);
+				}).catch(err => {
+					console.error(`[bunflare] Reload failed for "${workerDef.name}":`, err);
+				});
+			});
+			auxWatcher.start();
+			watchers.push(auxWatcher);
+			console.log(`[bunflare] Watching ${auxSrcDir} for changes (${workerDef.name})`);
+		}
+
+		// Load main worker after aux workers
+		const firstGen = await mainManager.reload();
+		console.log(`[bunflare] Main worker → generation ${firstGen.id}`);
+
+		manager = mainManager;
+		setGenerationManager(manager);
+		setWorkerRegistry(registry);
+
+		// File watcher for main worker
+		const mainSrcDir = path.dirname(path.resolve(mainBaseDir, mainConfig.main));
+		const mainWatcher = new FileWatcher(mainSrcDir, () => {
+			mainManager.reload().then(gen => {
+				console.log(`[bunflare] Main worker reloaded → generation ${gen.id}`);
+			}).catch(err => {
+				console.error("[bunflare] Reload failed:", err);
+			});
+		});
+		mainWatcher.start();
+		watchers.push(mainWatcher);
+		console.log(`[bunflare] Watching ${mainSrcDir} for changes (main)`);
+	} else {
+		// ─── Single-worker mode (current behavior) ────────────────────
+		const config = await autoLoadConfig(baseDir, envFlag);
+		console.log(`[bunflare] Loaded config: ${config.name}${envFlag ? ` (env: ${envFlag})` : ""}`);
+		setDashboardConfig(config);
+
+		manager = new GenerationManager(config, baseDir);
+		const firstGen = await manager.reload();
+		console.log(`[bunflare] Generation ${firstGen.id} loaded`);
+		setGenerationManager(manager);
+
+		// File watcher — watch the source directory
+		const srcDir = path.dirname(path.resolve(baseDir, config.main));
+		const watcher = new FileWatcher(srcDir, () => {
+			manager.reload().then(gen => {
+				console.log(`[bunflare] Reloaded → generation ${gen.id}`);
+			}).catch(err => {
+				console.error("[bunflare] Reload failed:", err);
+			});
+		});
+		watcher.start();
+		watchers.push(watcher);
+		console.log(`[bunflare] Watching ${srcDir} for changes`);
+	}
+
+	// Start server — one Bun.serve(), delegates to active generation
+	const port = parseInt(portFlag ?? process.env.PORT ?? "8787", 10);
+	const hostname = listenFlag ?? process.env.HOST ?? "localhost";
+
+	const server = Bun.serve({
+		port,
+		hostname,
+		async fetch(request, server) {
+			addCfProperty(request);
+
+			const url = new URL(request.url);
+
+			// Dashboard trace WebSocket stream (must be before dashboard catch-all for server.upgrade)
+			if (url.pathname === "/__dashboard/api/traces/ws") {
+				const upgraded = server.upgrade(request, { data: { type: "trace-stream", _url: request.url } as any });
+				if (!upgraded) return new Response("WebSocket upgrade failed", { status: 500 });
+				return undefined as unknown as Response;
+			}
+
+			// Dashboard routes (HTML, assets, API)
+			if (url.pathname.startsWith("/__dashboard")) {
+				return handleDashboardRequest(request);
+			}
+
+			// Queue pull consumer endpoints: POST /cdn-cgi/handler/queues/<name>/messages/pull and /ack
+			const queuePullMatch = url.pathname.match(/^\/cdn-cgi\/handler\/queues\/([^/]+)\/messages\/(pull|ack)$/);
+			if (queuePullMatch && request.method === "POST") {
+				const queueName = decodeURIComponent(queuePullMatch[1]!);
+				const action = queuePullMatch[2]!;
+				const queueDb = getDatabase();
+				const pullConsumer = new QueuePullConsumer(queueDb, queueName);
+
+				try {
+					const body = await request.json() as PullRequest | AckRequest;
+					if (action === "pull") {
+						const result = pullConsumer.pull(body as PullRequest);
+						return Response.json(result);
+					} else {
+						const result = pullConsumer.ack(body as AckRequest);
+						return Response.json(result);
+					}
+				} catch (err) {
+					return Response.json({ error: String(err) }, { status: 400 });
+				}
+			}
+
+			// Email handler: POST /cdn-cgi/handler/email?from=...&to=...
+			if (url.pathname === "/cdn-cgi/handler/email" && request.method === "POST") {
+				const gen = manager.active;
+				if (!gen) return new Response("No active generation", { status: 503 });
+				const from = url.searchParams.get("from") ?? "";
+				const to = url.searchParams.get("to") ?? "";
+				const raw = await request.arrayBuffer();
+				return gen.callEmail(new Uint8Array(raw), from, to);
+			}
+
+			// Manual trigger: GET /cdn-cgi/handler/scheduled?cron=<expression>
+			if (url.pathname === "/cdn-cgi/handler/scheduled") {
+				const gen = manager.active;
+				if (!gen) return new Response("No active generation", { status: 503 });
+				const cronExpr = url.searchParams.get("cron") ?? "* * * * *";
+				return gen.callScheduled(cronExpr);
+			}
+
+			// Delegate to active generation
+			const gen = manager.active;
+			if (!gen) {
+				return new Response("No active generation", { status: 503 });
+			}
+
+			return (await gen.callFetch(request, server)) as Response;
+		},
+		websocket: {
+			open(ws) {
+				const data = ws.data as unknown as Record<string, unknown>;
+				if (data.type === "trace-stream") {
+					// Trace streaming WebSocket
+					const store = getTraceStore();
+
+					let filter: { path?: string; status?: string; attributeFilters?: Array<{ key: string; value: string; type: "include" | "exclude" }> } = {};
+					let buffer: TraceEvent[] = [];
+					const MAX_BUFFER = 1000;
+
+					// Track which traceIds pass/fail the filter so child spans don't leak
+					const allowedTraces = new Set<string>();
+					const excludedTraces = new Set<string>();
+
+					function isRootSpanFiltered(span: { name: string; status: string; parentSpanId: string | null; attributes: Record<string, unknown> }): boolean {
+						if (filter.status && filter.status !== "all") {
+							if (span.status !== "unset" && span.status !== filter.status) return true;
+						}
+						if (filter.path) {
+							if (!matchGlob(span.name, filter.path)) return true;
+						}
+						if (filter.attributeFilters && filter.attributeFilters.length > 0) {
+							const attrs = span.attributes;
+							for (const af of filter.attributeFilters) {
+								const val = attrs[af.key];
+								const matches = val !== undefined && String(val).toLowerCase().includes(af.value.toLowerCase());
+								if (af.type === "include" && !matches) return true;
+								if (af.type === "exclude" && matches) return true;
+							}
+						}
+						return false;
+					}
+
+					const unsubscribe = store.subscribe((event) => {
+						// Determine traceId for this event
+						const traceId = event.type === "span.event" ? event.event.traceId
+							: event.span.traceId;
+
+						// For root spans, evaluate filter and track decision
+						if ((event.type === "span.start" || event.type === "span.end") && event.span.parentSpanId === null) {
+							if (isRootSpanFiltered(event.span)) {
+								excludedTraces.add(traceId);
+								allowedTraces.delete(traceId);
+								return;
+							}
+							excludedTraces.delete(traceId);
+							allowedTraces.add(traceId);
+						} else {
+							// Child span or event: check if its trace was already filtered
+							if (excludedTraces.has(traceId)) return;
+							// If we haven't seen the root span yet, allow it through
+						}
+
+						if (buffer.length < MAX_BUFFER) {
+							buffer.push(event);
+						}
+					});
+
+					const interval = setInterval(() => {
+						if (buffer.length > 0) {
+							ws.send(JSON.stringify({ type: "batch", events: buffer }));
+							buffer = [];
+						}
+					}, 500);
+
+					// Send initial traces (after filter is available from query params)
+					try {
+						// Parse filter from initial connection URL if provided
+						const reqUrl = new URL((data as any)._url ?? "ws://localhost");
+						const statusParam = reqUrl.searchParams.get("status");
+						const pathParam = reqUrl.searchParams.get("path");
+						if (statusParam) filter.status = statusParam;
+						if (pathParam) filter.path = pathParam;
+					} catch {}
+
+					let sinceMs = 15 * 60 * 1000; // default 15 minutes
+					const since = Date.now() - sinceMs;
+					const recent = store.getRecentTraces(since, 200);
+					ws.send(JSON.stringify({ type: "initial", traces: recent }));
+
+					// Store cleanup handles on ws.data
+					(data as any)._traceCleanup = { unsubscribe, interval };
+					(data as any)._setFilter = (f: typeof filter & { sinceMs?: number }) => {
+						filter = f;
+						if (f.sinceMs !== undefined) sinceMs = f.sinceMs;
+						// Reset trace tracking when filter changes
+						allowedTraces.clear();
+						excludedTraces.clear();
+						// Re-send filtered initial traces so the client replaces stale data
+						const freshSince = sinceMs > 0 ? Date.now() - sinceMs : 0;
+						const freshTraces = store.getRecentTraces(freshSince, 200);
+						ws.send(JSON.stringify({ type: "initial", traces: freshTraces }));
+					};
+					return;
+				}
+
+				// CF WebSocket bridge
+				const cfSocket = (data as { cfSocket: CFWebSocket }).cfSocket;
+				cfSocket.addEventListener("message", (ev: Event) => {
+					const msgData = (ev as MessageEvent).data;
+					ws.send(msgData);
+				});
+				cfSocket.addEventListener("close", (ev: Event) => {
+					const ce = ev as CloseEvent;
+					ws.close(ce.code, ce.reason);
+				});
+			},
+			message(ws, message) {
+				const data = ws.data as unknown as Record<string, unknown>;
+				if (data.type === "trace-stream") {
+					try {
+						const msg = JSON.parse(typeof message === "string" ? message : new TextDecoder().decode(message));
+						if (msg.type === "filter") {
+							const setFilter = (data as any)._setFilter;
+							if (setFilter) setFilter({ path: msg.path, status: msg.status, attributeFilters: msg.attributeFilters, sinceMs: msg.sinceMs });
+						}
+					} catch {}
+					return;
+				}
+
+				const cfSocket = (data as { cfSocket: CFWebSocket }).cfSocket;
+				if (cfSocket._peer && cfSocket._peer._accepted) {
+					cfSocket._peer._dispatchWSEvent({ type: "message", data: typeof message === "string" ? message : message.buffer as ArrayBuffer });
+				} else if (cfSocket._peer) {
+					cfSocket._peer._eventQueue.push({ type: "message", data: typeof message === "string" ? message : message.buffer as ArrayBuffer });
+				}
+			},
+			close(ws, code, reason) {
+				const data = ws.data as unknown as Record<string, unknown>;
+				if (data.type === "trace-stream") {
+					const cleanup = (data as any)._traceCleanup;
+					if (cleanup) {
+						cleanup.unsubscribe();
+						clearInterval(cleanup.interval);
+					}
+					return;
+				}
+
+				const cfSocket = (data as { cfSocket: CFWebSocket }).cfSocket;
+				if (cfSocket._peer && cfSocket._peer.readyState !== 3 /* CLOSED */) {
+					const evt = { type: "close" as const, code: code ?? 1000, reason: reason ?? "", wasClean: true };
+					if (cfSocket._peer._accepted) {
+						cfSocket._peer._dispatchWSEvent(evt);
+					} else {
+						cfSocket._peer._eventQueue.push(evt);
+					}
+					cfSocket._peer.readyState = 3;
+				}
+				cfSocket.readyState = 3;
+			},
+		},
+	});
+
+	console.log(`[bunflare] Server running at http://${hostname}:${port}`);
+	console.log(`[bunflare] Dashboard: http://${hostname}:${port}/__dashboard`);
+
+	// Keep the process alive until terminated
+	await new Promise(() => {});
+}
+
+function matchGlob(text: string, pattern: string): boolean {
+	// Placeholder approach: protect ** before escaping special chars
+	const regex = pattern
+		.replace(/\*\*/g, "\0")
+		.replace(/[.+^${}()|[\]\\]/g, "\\$&")
+		.replace(/\0/g, ".*")
+		.replace(/\*/g, "[^/]*");
+	return new RegExp(`^${regex}`).test(text);
+}

package/runtime/cli/kv.ts

@@ -0,0 +1,84 @@
+import type { CliContext } from "./context";
+import { parseFlag, resolveBinding } from "./context";
+import { SqliteKVNamespace } from "../bindings/kv";
+
+export async function run(ctx: CliContext, args: string[]) {
+	const sub = args[0];
+	if (sub !== "key") {
+		console.error(`Usage: bunflare kv key <list|get|put|delete> [options]`);
+		process.exit(1);
+	}
+
+	const action = args[1];
+	const config = await ctx.config();
+	const bindingFlag = parseFlag(ctx.args, "--binding");
+	const binding = resolveBinding(config.kv_namespaces, bindingFlag, "KV namespace");
+	const kv = new SqliteKVNamespace(ctx.db(), binding.binding);
+
+	switch (action) {
+		case "list": {
+			const prefix = parseFlag(ctx.args, "--prefix") ?? "";
+			let cursor = "";
+			let total = 0;
+			do {
+				const result = await kv.list({ prefix, cursor: cursor || undefined });
+				for (const key of result.keys) {
+					let line = key.name;
+					if (key.expiration) {
+						const exp = new Date(key.expiration * 1000).toISOString().slice(0, 19).replace("T", " ");
+						line += `  (expires: ${exp})`;
+					}
+					console.log(line);
+				}
+				total += result.keys.length;
+				cursor = result.cursor;
+			} while (cursor);
+			if (total === 0) console.log("(no keys)");
+			break;
+		}
+		case "get": {
+			const key = args[2];
+			if (!key) {
+				console.error("Usage: bunflare kv key get <key> [--binding NAME]");
+				process.exit(1);
+			}
+			const value = await kv.get(key);
+			if (value === null) {
+				console.error(`Key not found: ${key}`);
+				process.exit(1);
+			}
+			if (typeof value === "string") {
+				process.stdout.write(value);
+				// Add newline if stdout is a terminal
+				if (process.stdout.isTTY) process.stdout.write("\n");
+			} else {
+				process.stdout.write(String(value));
+			}
+			break;
+		}
+		case "put": {
+			const key = args[2];
+			const value = args[3];
+			if (!key || value === undefined) {
+				console.error("Usage: bunflare kv key put <key> <value> [--binding NAME]");
+				process.exit(1);
+			}
+			await kv.put(key, value);
+			console.log(`Put ${key}`);
+			break;
+		}
+		case "delete": {
+			const key = args[2];
+			if (!key) {
+				console.error("Usage: bunflare kv key delete <key> [--binding NAME]");
+				process.exit(1);
+			}
+			await kv.delete(key);
+			console.log(`Deleted ${key}`);
+			break;
+		}
+		default:
+			console.error(`Usage: bunflare kv key <list|get|put|delete> [options]`);
+			process.exit(1);
+	}
+}

package/runtime/cli/queues.ts

@@ -0,0 +1,109 @@
+import type { CliContext } from "./context";
+import { parseFlag } from "./context";
+
+export async function run(ctx: CliContext, args: string[]) {
+	const action = args[0];
+
+	switch (action) {
+		case "list": {
+			const config = await ctx.config();
+			const producers = config.queues?.producers ?? [];
+			const consumers = config.queues?.consumers ?? [];
+			if (producers.length === 0 && consumers.length === 0) {
+				console.log("No queues configured.");
+				return;
+			}
+			const queues = new Map<string, { producer?: string; consumer: boolean }>();
+			for (const p of producers) {
+				queues.set(p.queue, { producer: p.binding, consumer: false });
+			}
+			for (const c of consumers) {
+				const existing = queues.get(c.queue);
+				if (existing) {
+					existing.consumer = true;
+				} else {
+					queues.set(c.queue, { consumer: true });
+				}
+			}
+			for (const [name, info] of queues) {
+				const parts = [name];
+				if (info.producer) parts.push(`binding=${info.producer}`);
+				if (info.consumer) parts.push("(consumer)");
+				console.log(parts.join("  "));
+			}
+			break;
+		}
+		case "message": {
+			const sub = args[1];
+			const queueName = args[2];
+
+			switch (sub) {
+				case "list": {
+					if (!queueName) {
+						console.error("Usage: bunflare queues message list <queue>");
+						process.exit(1);
+					}
+					const db = ctx.db();
+					const rows = db.query<
+						{ id: string; status: string; attempts: number; created_at: number; content_type: string; body: Buffer },
+						[string]
+					>(
+						"SELECT id, status, attempts, created_at, content_type, body FROM queue_messages WHERE queue = ? ORDER BY created_at DESC LIMIT 100",
+					).all(queueName);
+					if (rows.length === 0) {
+						console.log("(no messages)");
+						return;
+					}
+					for (const row of rows) {
+						const date = new Date(row.created_at).toISOString().slice(0, 19).replace("T", " ");
+						let preview = "";
+						if (row.content_type === "json" || row.content_type === "text") {
+							preview = Buffer.from(row.body).toString().slice(0, 80);
+						} else {
+							preview = `(${row.content_type}, ${row.body.length} bytes)`;
+						}
+						console.log(`${row.id}  ${row.status.padEnd(8)}  attempts=${row.attempts}  ${date}  ${preview}`);
+					}
+					break;
+				}
+				case "send": {
+					const body = args[3];
+					if (!queueName || body === undefined) {
+						console.error("Usage: bunflare queues message send <queue> <body>");
+						process.exit(1);
+					}
+					const { SqliteQueueProducer } = await import("../bindings/queue");
+					const producer = new SqliteQueueProducer(ctx.db(), queueName);
+					// Try parsing as JSON, fall back to text
+					let parsed: unknown;
+					try {
+						parsed = JSON.parse(body);
+					} catch {
+						parsed = body;
+					}
+					await producer.send(parsed, { contentType: typeof parsed === "string" ? "text" : "json" });
+					console.log(`Sent message to queue "${queueName}"`);
+					break;
+				}
+				case "purge": {
+					if (!queueName) {
+						console.error("Usage: bunflare queues message purge <queue>");
+						process.exit(1);
+					}
+					const db = ctx.db();
+					const result = db.run("DELETE FROM queue_messages WHERE queue = ?", [queueName]);
+					db.run("DELETE FROM queue_leases WHERE queue = ?", [queueName]);
+					console.log(`Purged ${result.changes} message(s) from queue "${queueName}"`);
+					break;
+				}
+				default:
+					console.error("Usage: bunflare queues message <list|send|purge> <queue>");
+					process.exit(1);
+			}
+			break;
+		}
+		default:
+			console.error("Usage: bunflare queues <list|message> [options]");
+			process.exit(1);
+	}
+}