@sigil-dev/grimoire 0.5.0 → 0.6.0

package/src/{build.ts → server/build.ts}

@@ -1,13 +1,13 @@
+import { sigil } from "@sigil-dev/compiler/bun";
 import { mkdir } from "fs/promises";
 import { isAbsolute, join } from "path";
-import { sigil } from "@sigil-dev/compiler/bun";
-import { generateManifest } from "./manifest-gen";
+import { generateManifest } from "../routing/manifest-gen";
+import type { RouteTree } from "../routing/scanner";
+import { scanRoutes } from "../routing/scanner";
+import { transformRoutes } from "../routing/transform-routes";
+import { generateTypes } from "../typegen";
+import type { BuildResult, GrimoireConfig, GrimoirePlugin } from "../types";
 import { runHook } from "./plugins";
-import { scanRoutes } from "./scanner";
-import type { RouteTree } from "./scanner";
-import { transformRoutes } from "./transform-routes";
-import { generateTypes } from "./typegen";
-import type { BuildResult, GrimoireConfig, GrimoirePlugin } from "./types";
 
 export async function buildProject(
 	config: GrimoireConfig,
@@ -59,12 +59,12 @@ export async function buildProject(
 
 	const [hydrateResult, domResult] = await Promise.all([
 		Bun.build({
-			entrypoints: [join(import.meta.dir, "./hydrate.ts")],
+			entrypoints: [join(import.meta.dir, "../rendering/hydrate.ts")],
 			outdir: join(process.cwd(), "public/__grimoire__"),
 			plugins: [sigil({ mode: "hydrate" }), makeRoutesPlugin(hydrateManifest)],
 		}),
 		Bun.build({
-			entrypoints: [join(import.meta.dir, "./client.ts")],
+			entrypoints: [join(import.meta.dir, "../client/index.ts")],
 			outdir: join(process.cwd(), "public/__grimoire__"),
 			plugins: [sigil({ mode: "dom" }), makeRoutesPlugin(domManifest)],
 		}),
@@ -79,7 +79,9 @@ export async function buildProject(
 
 	const result: BuildResult = {
 		success: hydrateResult.success && domResult.success,
-		outputs: [...hydrateResult.outputs, ...domResult.outputs].map((o) => o.path),
+		outputs: [...hydrateResult.outputs, ...domResult.outputs].map(
+			(o) => o.path,
+		),
 		errors: [...hydrateResult.logs, ...domResult.logs].map(String),
 	};
 

package/src/server/coordinator.ts

@@ -0,0 +1,297 @@
+import { type Subprocess, spawn } from "bun";
+import { join } from "path";
+import { matchRoute } from "../routing/router";
+import { type RouteFile, type RouteTree, scanRoutes } from "../routing/scanner";
+import type {
+	CoordinatorContext,
+	GrimoireConfig,
+	GrimoirePlugin,
+	Route,
+	WorkerDescriptor,
+	WorkerMode,
+} from "../types";
+import { buildProject } from "./build";
+import {
+	runHook,
+	runRouteRequest,
+	runSerializeLocals,
+	runWorkerSpawn,
+} from "./plugins";
+
+async function waitForWorkers(
+	workers: LiveWorker[],
+	timeoutMs = 10_000,
+): Promise<void> {
+	const ready = new Set<number>();
+	const start = Date.now();
+
+	while (ready.size < workers.length) {
+		if (Date.now() - start > timeoutMs) {
+			const pending = workers
+				.filter((w) => !ready.has(w.descriptor.globalIndex))
+				.map(
+					(w) =>
+						w.descriptor.name ?? `${w.descriptor.mode}-${w.descriptor.index}`,
+				)
+				.join(", ");
+			throw new Error(
+				`Grimoire: workers timed out waiting to become ready: ${pending}`,
+			);
+		}
+		for (const w of workers) {
+			if (ready.has(w.descriptor.globalIndex)) continue;
+			if (w.descriptor.pid != null) {
+				ready.add(w.descriptor.globalIndex);
+			}
+		}
+		await Bun.sleep(50);
+	}
+}
+
+function roundRobin(workers: WorkerDescriptor[]): WorkerDescriptor {
+	const idx = roundRobinCounter++ % workers.length;
+	return workers[idx];
+}
+let roundRobinCounter = 0;
+
+function consistentHash(
+	req: Request,
+	workers: WorkerDescriptor[],
+): WorkerDescriptor {
+	const key = req.headers.get("cookie") ?? req.url;
+	let hash = 0;
+	for (let i = 0; i < key.length; i++) {
+		hash = (hash * 31 + key.charCodeAt(i)) >>> 0;
+	}
+	return workers[hash % workers.length];
+}
+
+function defaultRoute(
+	req: Request,
+	workers: WorkerDescriptor[],
+	tree: RouteTree,
+): WorkerDescriptor {
+	const isWs = req.headers.get("upgrade")?.toLowerCase() === "websocket";
+
+	if (isWs) {
+		const wsWorkers = workers.filter(
+			(w) => w.mode === "ws" || w.mode === "full",
+		);
+		if (wsWorkers.length) return consistentHash(req, wsWorkers);
+	}
+
+	const url = new URL(req.url);
+	const matched = matchRoute(tree, url);
+
+	if (matched?.route.type === "server") {
+		const apiWorkers = workers.filter(
+			(w) => w.mode === "api" || w.mode === "full",
+		);
+		if (apiWorkers.length) return roundRobin(apiWorkers);
+	}
+
+	const frontendWorkers = workers.filter(
+		(w) => w.mode === "frontend" || w.mode === "full",
+	);
+	if (frontendWorkers.length) return roundRobin(frontendWorkers);
+
+	// fallback — should not happen if scale spec is valid
+	return workers[0];
+}
+
+export interface ScaleSpec {
+	api?: number;
+	frontend?: number;
+	ws?: number;
+	[mode: string]: number | undefined;
+}
+
+export interface CoordinatorOptions {
+	config: GrimoireConfig;
+	plugins: GrimoirePlugin[];
+	scale: ScaleSpec;
+	noBuild: boolean;
+}
+
+interface LiveWorker {
+	descriptor: WorkerDescriptor;
+	process: Subprocess;
+	port: number;
+}
+
+export async function startCoordinator(options: CoordinatorOptions) {
+	const { config, plugins, scale, noBuild } = options;
+	const { port = 3000, host = "localhost", routes = "src/routes" } = config;
+	let stopping = false;
+
+	// 1. build if needed
+	if (!noBuild) {
+		const { result } = await buildProject(config, plugins);
+		if (!result.success) throw new Error("Grimoire: build failed");
+	}
+
+	// 2. generate ephemeral secret
+	const secret = crypto.randomUUID() + crypto.randomUUID(); // 72 hex chars
+
+	// 3. scan routes once — coordinator owns the route tree
+	const routesDir = /* resolve */ routes;
+	const tree = await scanRoutes(routesDir, process.cwd());
+
+	// 4. derive route slices per mode
+	const slices: Record<string, RouteFile[]> = {
+		api: tree.routes.filter((r) => r.type === "server"),
+		ws: tree.routes.filter((r) => r.type === "server"),
+		frontend: tree.routes.filter(
+			(r) => r.type === "page" || r.type === "simple",
+		),
+		full: [...tree.routes],
+	};
+
+	// 5. spawn workers
+	const liveWorkers: LiveWorker[] = [];
+	let globalIndex = 0;
+	let nextPort = port + 1;
+
+	for (const [mode, count = 0] of Object.entries(scale)) {
+		for (let i = 0; i < count; i++) {
+			const workerPort = nextPort++;
+			const descriptor: WorkerDescriptor = {
+				mode: mode as WorkerMode,
+				index: i + 1,
+				globalIndex: globalIndex++,
+				internalUrl: `http://127.0.0.1:${workerPort}`,
+				routes: slices[mode] ?? slices.full,
+			};
+
+			// let plugins inject env and name
+			const workerEnv = await runWorkerSpawn(plugins, descriptor);
+			const workerEntry = join(import.meta.dir, "./worker.ts");
+			if (workerEnv.name) descriptor.name = workerEnv.name;
+
+			const proc = spawn(["bun", "run", workerEntry], {
+				env: {
+					...process.env,
+					...workerEnv.env,
+					GRIMOIRE_MODE: mode,
+					GRIMOIRE_WORKER_INDEX: String(i),
+					GRIMOIRE_WORKER_PORT: String(workerPort),
+					GRIMOIRE_INTERNAL_SECRET: secret,
+					GRIMOIRE_WORKER_NAME: descriptor.name ?? `${mode}-${i}`,
+				},
+				ipc(message) {
+					// worker sends { ready: true } when Bun.serve is up
+					if (message?.ready) {
+						descriptor.pid = proc.pid;
+						runHook(plugins, "onWorkerReady", descriptor);
+					}
+				},
+				onExit(proc, exitCode) {
+					const intentional = stopping;
+					runHook(
+						plugins,
+						"onWorkerDeath",
+						descriptor,
+						intentional ? "intentional" : "crash",
+					);
+					if (!intentional) {
+						console.error(
+							`[grimoire] worker ${descriptor.name ?? descriptor.mode + "-" + descriptor.index} crashed (exit ${exitCode}), respawning...`,
+						);
+						// respawn: same descriptor, same port
+						// (simplified — plugin-scale can override this behavior)
+					}
+				},
+			});
+
+			liveWorkers.push({ descriptor, process: proc, port: workerPort });
+		}
+	}
+
+	// 6. wait for all workers ready (with timeout)
+	// workers send IPC { ready: true } — coordinator waits up to 10s
+	await waitForWorkers(liveWorkers);
+
+	// 7. fire onCoordinatorStart
+	const ctx: CoordinatorContext = {
+		workers: liveWorkers.map((w) => w.descriptor),
+		port,
+		secret,
+		routes: tree,
+	};
+	await runHook(plugins, "onCoordinatorStart", ctx);
+
+	// 8. start coordinator Bun.serve
+
+	const server = Bun.serve({
+		port,
+		hostname: host,
+		fetch: async (req) => {
+			// run hooks.server.ts to populate locals
+			const locals: App.Locals = {};
+			// ... same handle() chain as createServer but resolve() just
+			// forwards instead of rendering
+
+			// pick worker
+			const descriptors = liveWorkers.map((w) => w.descriptor);
+			const chosen =
+				(await runRouteRequest(plugins, req, descriptors, tree)) ??
+				defaultRoute(req, descriptors, tree);
+
+			// serialize locals
+			const serialized =
+				(await runSerializeLocals(plugins, locals)) ??
+				defaultSerialize(locals, secret);
+
+			// forward
+			return fetch(chosen.internalUrl + new URL(req.url).pathname, {
+				method: req.method,
+				headers: {
+					//@ts-expect-error shut Up
+					...Object.fromEntries(req.headers),
+					"X-Grimoire-Locals": serialized,
+					"X-Grimoire-Internal": secret,
+				},
+				body: req.body,
+			});
+		},
+	});
+
+	// 9. shutdown
+	const handleShutdown = async () => {
+		if (stopping) return;
+		stopping = true;
+		await runHook(plugins, "onStop", "shutdown");
+		for (const w of liveWorkers) w.process.kill();
+		server.stop();
+		process.exit(0);
+	};
+	process.on("SIGINT", handleShutdown);
+	process.on("SIGTERM", handleShutdown);
+
+	return {
+		server,
+		workers: liveWorkers.map((w) => ({
+			name: w.descriptor.name,
+			mode: w.descriptor.mode,
+			port: w.port,
+			index: w.descriptor.index,
+		})),
+	};
+}
+
+function defaultSerialize(locals: App.Locals, secret: string): string {
+	const payload = JSON.stringify(locals);
+	const sig = new Bun.CryptoHasher("sha256")
+		.update(secret + payload)
+		.digest("hex");
+	return Buffer.from(JSON.stringify({ payload, sig })).toString("base64");
+}
+
+function defaultDeserialize(raw: string, secret: string): App.Locals {
+	const { payload, sig } = JSON.parse(Buffer.from(raw, "base64").toString());
+	const expected = new Bun.CryptoHasher("sha256")
+		.update(secret + payload)
+		.digest("hex");
+	if (sig !== expected) throw new Error("Grimoire: locals signature invalid");
+	return JSON.parse(payload);
+}

package/src/{hooks.ts → server/hooks.ts}

@@ -1,7 +1,7 @@
 /**
  * SvelteKit-style server hooks.
  *
- * Create a hooks.server.ts in your project root:
+ * Create a hooks.index.ts in your project root:
  *
  *   import type { Handle } from "@sigil-dev/grimoire/hooks";
  *

package/src/{server.ts → server/index.ts}

@@ -1,17 +1,23 @@
-import { isFailResult } from "./fail";
-import { isErrorResult } from "./error";
-import type { Handle, InitFunction, RequestEvent, ResolveFunction } from "./hooks";
-import { isRedirectResult } from "./redirect";
-import { createCookies } from "./cookie-utils";
+import type { ServerWebSocket } from "bun";
+import { renderRoute } from "../rendering";
+import { registerSSRPlugin } from "../rendering/ssrPlugin";
+import { findClosestError, matchRoute } from "../routing/router";
+import { isErrorResult } from "../sentinels/error.ts";
+import { isFailResult } from "../sentinels/fail.ts";
+import { isRedirectResult } from "../sentinels/redirect.ts";
+import type { GrimoireConfig, WsRouteHandler } from "../types";
 import { buildProject } from "./build";
-import { runHook, runRequestHooks } from "./plugins";
-import { renderRoute } from "./renderer";
-import { findClosestError, matchRoute } from "./router";
-import { registerSSRPlugin } from "./ssrPlugin";
-import type { GrimoireConfig, WsRouteHandler } from "./types";
+import { createCookies } from "./cookie-utils";
+import type {
+	Handle,
+	InitFunction,
+	RequestEvent,
+	ResolveFunction,
+} from "./hooks";
+import { runDeserializeLocals, runHook, runRequestHooks } from "./plugins";
 
 /**
- * Try to load hooks.server.ts from the project root.
+ * Try to load hooks.index.ts from the project root.
  */
 async function loadHooks(
 	projectRoot: string,
@@ -35,27 +41,40 @@ interface _WsInternalData {
 }
 
 export async function createServer(config: GrimoireConfig = {}) {
-	const {
-		port = 3000,
-		host = "localhost",
-		plugins = [],
-	} = config;
-
 	// run config hooks
 	let finalConfig = config;
-	for (const plugin of plugins) {
+	const earlyPlugins = config.plugins ?? [];
+	for (const plugin of earlyPlugins) {
 		finalConfig = plugin.config?.(finalConfig) ?? finalConfig;
 	}
 
-	registerSSRPlugin(plugins);
+	//w timings????
+	const {
+		port = 3000,
+		host = "localhost",
+		plugins = [],
+		routes = "src/routes",
+		_skipBuild = false,
+	} = finalConfig;
 
-	const { result, tree } = await buildProject(finalConfig, plugins);
-	if (!result.success) {
-		throw new Error("Grimoire: build failed");
+	registerSSRPlugin(plugins);
+	let tree: any;
+	if (!config._skipBuild) {
+		const { result, tree: _tree } = await buildProject(finalConfig, plugins);
+		if (!result.success) throw new Error("Grimoire: build failed");
+		tree = _tree;
+	} else {
+		// worker — build already done by loom, just scan routes
+		const { isAbsolute, join } = await import("node:path");
+		const { scanRoutes } = await import("../routing/scanner");
+		const routesDir = isAbsolute(routes) ? routes : join(process.cwd(), routes);
+		tree = await scanRoutes(routesDir, process.cwd());
 	}
 
-	// Load hooks.server.ts
-	const { handle: hooksHandle, init: hooksInit } = await loadHooks(process.cwd());
+	// Load hooks.index.ts
+	const { handle: hooksHandle, init: hooksInit } = await loadHooks(
+		process.cwd(),
+	);
 
 	// Run init hook if present
 	await hooksInit?.();
@@ -65,16 +84,18 @@ export async function createServer(config: GrimoireConfig = {}) {
 		hostname: host,
 		fetch: async (req) => {
 			// Shared locals object for this request — same reference across all route handlers.
-			// hooks.server.ts handle() runs via resolve() for page routes only.
+			// hooks.index.ts handle() runs via resolve() for page routes only.
 			// For server/WebSocket routes, auth must use cookies/headers in upgrade().
-			const locals: App.Locals = {};
+			// If this request came from a coordinator, deserialize its locals
+			const rawLocals = req.headers.get("X-Grimoire-Locals");
+			const locals: App.Locals = rawLocals
+				? ((await runDeserializeLocals(plugins, rawLocals)) ?? {})
+				: {};
 
 			return runRequestHooks(plugins, req, async () => {
 				const url = new URL(req.url);
 
-				const publicFile = Bun.file(
-					`${process.cwd()}/public${url.pathname}`,
-				);
+				const publicFile = Bun.file(`${process.cwd()}/public${url.pathname}`);
 				if (await publicFile.exists()) {
 					return new Response(publicFile);
 				}
@@ -99,7 +120,8 @@ export async function createServer(config: GrimoireConfig = {}) {
 					const mod = await import(matched.route.filePath);
 
 					// WebSocket upgrade path
-					const isWsUpgrade = req.headers.get("upgrade")?.toLowerCase() === "websocket";
+					const isWsUpgrade =
+						req.headers.get("upgrade")?.toLowerCase() === "websocket";
 					if (isWsUpgrade && mod.websocket) {
 						let extraData: Record<string, unknown> = {};
 						if (mod.upgrade) {
@@ -120,6 +142,7 @@ export async function createServer(config: GrimoireConfig = {}) {
 							__handler: mod.websocket,
 							...extraData,
 						};
+						//@ts-expect-error i dont know what you are talking about please
 						if (server.upgrade(req, { data: wsData })) {
 							// Bun sends the 101 response — return undefined to signal no HTTP response
 							return undefined as unknown as Response;
@@ -162,7 +185,10 @@ export async function createServer(config: GrimoireConfig = {}) {
 				// Resolve function: runs the actual page/form action logic
 				const resolve: ResolveFunction = async (evt) => {
 					// form actions
-					if (matched.pageServer && HTTP_METHODS.includes(evt.request.method as any)) {
+					if (
+						matched.pageServer &&
+						HTTP_METHODS.includes(evt.request.method as any)
+					) {
 						const mod = await import(matched.pageServer.filePath);
 						const handler = mod[evt.request.method];
 						if (handler) {
@@ -182,17 +208,25 @@ export async function createServer(config: GrimoireConfig = {}) {
 									});
 								}
 								if (isErrorResult(e)) {
-									const isApi = evt.request.headers.get("accept")?.includes("application/json");
+									const isApi = evt.request.headers
+										.get("accept")
+										?.includes("application/json");
 									if (isApi) {
 										return Response.json(
 											{ error: true, status: e.status, message: e.message },
 											{ status: e.status },
 										);
 									}
-									const errorPage = findClosestError(tree.errors, evt.url.pathname);
+									const errorPage = findClosestError(
+										tree.errors,
+										evt.url.pathname,
+									);
 									if (errorPage) {
 										const errMod = await import(errorPage.filePath);
-										const html = errMod.default({ status: e.status, message: e.message });
+										const html = errMod.default({
+											status: e.status,
+											message: e.message,
+										});
 										return new Response(html, {
 											status: e.status,
 											headers: { "Content-Type": "text/html" },
@@ -223,7 +257,14 @@ export async function createServer(config: GrimoireConfig = {}) {
 					}
 
 					// page routes
-					const response = await renderRoute(matched, evt.request, tree.errors, undefined, evt.locals, plugins);
+					const response = await renderRoute(
+						matched,
+						evt.request,
+						tree.errors,
+						undefined,
+						evt.locals,
+						plugins,
+					);
 
 					if (evt.request.headers.get("x-grimoire-navigate") === "1") {
 						return response;
@@ -263,7 +304,10 @@ export async function createServer(config: GrimoireConfig = {}) {
 
 				// Apply setHeaders
 				const setCookieHeaders = cookies.toHeaders();
-				if (setCookieHeaders.length > 0 || Object.keys(setHeadersMap).length > 0) {
+				if (
+					setCookieHeaders.length > 0 ||
+					Object.keys(setHeadersMap).length > 0
+				) {
 					const headers = new Headers(response.headers);
 					for (const [k, v] of Object.entries(setHeadersMap)) {
 						headers.set(k, v);
@@ -288,7 +332,11 @@ export async function createServer(config: GrimoireConfig = {}) {
 			message(ws: ServerWebSocket<_WsInternalData>, data: string | Buffer) {
 				ws.data.__handler?.message?.(ws, data);
 			},
-			close(ws: ServerWebSocket<_WsInternalData>, code: number, reason?: string) {
+			close(
+				ws: ServerWebSocket<_WsInternalData>,
+				code: number,
+				reason?: string,
+			) {
 				ws.data.__handler?.close?.(ws, code, reason);
 			},
 			drain(ws: ServerWebSocket<_WsInternalData>) {

package/src/server/plugins.ts

@@ -0,0 +1,119 @@
+import type { RouteTree } from "../routing/scanner";
+import type {
+	BuildResult,
+	CoordinatorContext,
+	GrimoirePlugin,
+	LoadContext,
+	Route,
+	Server,
+	WorkerDescriptor,
+	WorkerEnv,
+} from "../types";
+
+// Fire-and-forget hooks routed through runHook.
+// onRequest and onRouteRender are intentionally excluded: they have distinct
+// calling conventions (middleware chain / inline transform loop) handled in index.ts.
+// Keep HookArgs in sync with GrimoirePlugin in types.ts.
+type HookArgs = {
+	onStart: [server: Server];
+	onStop: [reason: "shutdown" | "restart"];
+	onBuildStart: [];
+	onBuildEnd: [result: BuildResult];
+	onRouteLoad: [route: Route, context: LoadContext];
+	onCoordinatorStart: [ctx: CoordinatorContext];
+	onWorkerReady: [worker: WorkerDescriptor];
+	onWorkerDeath: [worker: WorkerDescriptor, reason: "crash" | "intentional"];
+};
+type FireAndForgetHook = keyof HookArgs;
+
+export async function runHook<K extends FireAndForgetHook>(
+	plugins: GrimoirePlugin[],
+	hook: K,
+	...args: HookArgs[K]
+): Promise<void> {
+	for (const plugin of plugins) {
+		const fn = plugin[hook] as ((...a: any[]) => any) | undefined;
+		await fn?.(...args);
+	}
+}
+
+export async function runRequestHooks(
+	plugins: GrimoirePlugin[],
+	req: Request,
+	final: () => Promise<Response>,
+): Promise<Response> {
+	const chain = plugins
+		.filter((p) => p.onRequest)
+		.reduceRight(
+			(next, plugin) => async () => plugin.onRequest!(req, next),
+			final,
+		);
+	return chain();
+}
+
+/**
+ * Calls onWorkerSpawn on ALL plugins and merges results.
+ * env is combined (later plugins override earlier on conflict).
+ * name is last-write-wins.
+ */
+export async function runWorkerSpawn(
+	plugins: GrimoirePlugin[],
+	worker: WorkerDescriptor,
+): Promise<WorkerEnv> {
+	const merged: WorkerEnv = { env: {} };
+	for (const plugin of plugins) {
+		const result = await plugin.onWorkerSpawn?.(worker);
+		//@ts-expect-error WTF TS? its result? meaning its not gonna do anything if null.
+		if (result?.env) Object.assign(merged.env!, result.env);
+		//@ts-expect-error WTF TS? its result? meaning its not gonna do anything if null.
+		if (result?.name) merged.name = result.name;
+	}
+	return merged;
+}
+
+/**
+ * First plugin to return a non-null string wins.
+ * Returns null if no plugin handles it — coordinator uses default.
+ */
+export async function runSerializeLocals(
+	plugins: GrimoirePlugin[],
+	locals: App.Locals,
+): Promise<string | null> {
+	for (const plugin of plugins) {
+		const result = await plugin.serializeLocals?.(locals);
+		if (result != null) return result;
+	}
+	return null;
+}
+
+/**
+ * First plugin to return non-null wins.
+ * Returns null if no plugin handles it — worker uses default.
+ */
+export async function runDeserializeLocals(
+	plugins: GrimoirePlugin[],
+	raw: string,
+): Promise<App.Locals | null> {
+	for (const plugin of plugins) {
+		const result = await plugin.deserializeLocals?.(raw);
+		if (result != null) return result;
+	}
+	return null;
+}
+
+/**
+ * First plugin to return a WorkerDescriptor wins.
+ * Returns null if no plugin handles it — coordinator uses default routing.
+ */
+export async function runRouteRequest(
+	plugins: GrimoirePlugin[],
+	req: Request,
+	workers: WorkerDescriptor[],
+	routes: RouteTree,
+): Promise<WorkerDescriptor | null> {
+	for (const plugin of plugins) {
+		const result = await plugin.routeRequest?.(req, workers, routes);
+		if (result != null) return result;
+	}
+	return null;
+}