arcway 0.1.13 → 0.1.15

package/README.md

@@ -120,7 +120,7 @@ export default {
   server: {
     port: 3000,
     shutdownTimeoutMs: 10_000,
-    maxBodySize: 1_048_576, // 1 MB
+    maxBodySize: 26_214_400, // 25 MB
   },
   api: {
     pathPrefix: '',        // Prefix all API routes (e.g., '/api')

package/client/head.js

@@ -137,4 +137,4 @@ function renderHeadToString(headData) {
   return parts.join('\n');
 }
 
-export { Head, clearSSRHeadData, extractHeadChildren, renderHeadToString, setSSRHeadData };
+export { Head, clearSSRHeadData, renderHeadToString, setSSRHeadData };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arcway",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "description": "A convention-based framework for building modular monoliths with strict domain boundaries.",
   "license": "MIT",
   "type": "module",

package/server/boot/index.js

@@ -1,16 +1,10 @@
 import { makeConfig } from '../config/loader.js';
-import { createDB } from '../db/index.js';
-import Redis from '../redis/index.js';
-import Events from '../events/index.js';
+import { createInfrastructure, destroyInfrastructure } from './infrastructure.js';
 import { EventHandler } from '../events/handler.js';
 import { JobRunner } from '../jobs/runner.js';
-import Logger from '../logger/index.js';
+import { WorkerPool } from '../jobs/worker-pool.js';
 import { loadEnvFiles } from '../env.js';
 import { McpRouter } from '../mcp/index.js';
-import Queue from '../queue/index.js';
-import Cache from '../cache/index.js';
-import Files from '../files/index.js';
-import { Mail } from '../mail/index.js';
 import WebServer from '../web-server.js';
 import { FileWatcher } from '../filewatcher/index.js';
 import { ApiRouter } from '../router/api-router.js';
@@ -27,30 +21,13 @@ async function boot(options) {
   const envFiles = loadEnvFiles(rootDir, mode);
   const config = await makeConfig(rootDir, { overrides: options.configOverrides, mode });
 
-  const log = new Logger(config.logger);
+  const infrastructure = await createInfrastructure(config, { runMigrations: true });
+  const { db, redis, queue, cache, files, mail, events, log } = infrastructure;
+
   const mcpRouter = new McpRouter(config.mcp, { log });
 
   if (envFiles.length > 0) log.info('Env files loaded', { envFiles });
 
-  const db = await createDB(config.database, { log });
-  await db.runMigrations();
-
-  const redis = new Redis(config.redis, { log });
-  await redis.connect();
-
-  const queue = new Queue(config.queue, { db, redis, log });
-  await queue.init();
-
-  const cache = new Cache(config.cache, { redis, log });
-  await cache.init();
-
-  const files = new Files(config.files, { log });
-  await files.init();
-
-  const mail = new Mail(config.mail, { db, log, queue });
-
-  const events = new Events(config.events, { redis, log });
-
   const eventHandler = new EventHandler(config.events, {
     events,
     log,
@@ -58,11 +35,32 @@ async function boot(options) {
   });
   await eventHandler.init();
 
-  const jobRunner = new JobRunner(config.jobs, { db, queue, cache, files, mail, events, log });
+  // Workers rebuild their own infrastructure from a JSON-safe config snapshot,
+  // so anything non-serializable on the main-thread config (streams, functions)
+  // must not ride through workerData. makeConfig already yields plain objects.
+  const workerPool =
+    config.jobs?.workerPoolSize > 0
+      ? new WorkerPool({
+          size: config.jobs.workerPoolSize,
+          workerData: { config },
+          defaultTimeoutMs: config.jobs?.staleTimeoutMs ?? null,
+        })
+      : null;
+
+  const jobRunner = new JobRunner(config.jobs, {
+    db,
+    queue,
+    cache,
+    files,
+    mail,
+    events,
+    log,
+    workerPool,
+  });
   await jobRunner.init();
 
-  const fileWatcher = new FileWatcher(rootDir, { log });
-  await fileWatcher.start();
+  const fileWatcher = mode === 'development' ? new FileWatcher(rootDir, { log }) : null;
+  if (fileWatcher) await fileWatcher.start();
 
   const appContext = { db, redis, events, queue, cache, files, mail, log, fileWatcher };
 
@@ -70,13 +68,14 @@ async function boot(options) {
 
   const apiRouter = new ApiRouter(config.api, {
     log,
+    mode,
     fileWatcher,
     appContext,
     sessionConfig: config.session,
   });
   await apiRouter.init();
 
-  const pagesRouter = new PagesRouter(config, { rootDir, log, fileWatcher, appContext });
+  const pagesRouter = new PagesRouter(config, { rootDir, log, mode, fileWatcher, appContext });
   await pagesRouter.init();
 
   const healthDeps = {
@@ -115,13 +114,12 @@ async function boot(options) {
 
   const shutdown = async () => {
     await jobRunner.shutdown();
+    if (workerPool) await workerPool.shutdown();
     await pagesRouter.close();
     await apiRouter.close();
     await webServer.close();
-    await fileWatcher.close();
-    await events.disconnect();
-    await redis.disconnect();
-    await db.destroy();
+    if (fileWatcher) await fileWatcher.close();
+    await destroyInfrastructure(infrastructure);
     await mcpRouter.cleanup(rootDir);
   };
 

package/server/boot/infrastructure.js

@@ -0,0 +1,47 @@
+import { createDB } from '../db/index.js';
+import Redis from '../redis/index.js';
+import Queue from '../queue/index.js';
+import Cache from '../cache/index.js';
+import Files from '../files/index.js';
+import { Mail } from '../mail/index.js';
+import Events from '../events/index.js';
+import Logger from '../logger/index.js';
+
+// Shared service construction for both the main-thread boot path and per-job
+// worker threads (see server/jobs/worker-entry.js). Workers call this with
+// `runMigrations: false` — only the main process should touch schema.
+async function createInfrastructure(config, { runMigrations = false } = {}) {
+  const log = new Logger(config.logger);
+
+  const db = await createDB(config.database, { log });
+  if (runMigrations) await db.runMigrations();
+
+  const redis = new Redis(config.redis, { log });
+  await redis.connect();
+
+  const queue = new Queue(config.queue, { db, redis, log });
+  await queue.init();
+
+  const cache = new Cache(config.cache, { redis, log });
+  await cache.init();
+
+  const files = new Files(config.files, { log });
+  await files.init();
+
+  const mail = new Mail(config.mail, { db, log, queue });
+
+  const events = new Events(config.events, { redis, log });
+
+  return { db, redis, queue, cache, files, mail, events, log };
+}
+
+async function destroyInfrastructure(services) {
+  if (!services) return;
+  // Order matches boot/index.js teardown: events → redis → db. Mail/queue/
+  // cache/files share redis/db and do not own independent connections.
+  await services.events?.disconnect?.();
+  await services.redis?.disconnect?.();
+  await services.db?.destroy?.();
+}
+
+export { createInfrastructure, destroyInfrastructure };

package/server/config/modules/jobs.js

@@ -1,5 +1,14 @@
+import os from 'node:os';
 import path from 'node:path';
 
+// Leave one core free for the main thread (dispatch/polling/HTTP). 0 disables
+// worker threads entirely and runs handlers inline on the main thread.
+function defaultWorkerPoolSize() {
+  const cores =
+    typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length;
+  return Math.max(1, cores - 1);
+}
+
 const DEFAULTS = {
   enabled: true,
   backoffMs: 1000,
@@ -14,6 +23,7 @@ function resolve(config, { rootDir } = {}) {
   if (jobs.dir && !path.isAbsolute(jobs.dir)) {
     jobs.dir = path.resolve(rootDir, jobs.dir);
   }
+  if (jobs.workerPoolSize === undefined) jobs.workerPoolSize = defaultWorkerPoolSize();
   return { ...config, jobs };
 }
 

package/server/config/modules/server.js

@@ -1,7 +1,7 @@
 const DEFAULTS = {
   host: '0.0.0.0',
   port: 3000,
-  maxBodySize: 1024 * 1024,
+  maxBodySize: 25 * 1024 * 1024,
   shutdownTimeoutMs: 10000,
   trustProxy: false,
 };

package/server/context.js

@@ -64,4 +64,4 @@ function buildContext(appContext, extras) {
   return ctx;
 }
 
-export { buildContext, trackDb };
+export { buildContext };

package/server/graphql/index.js

@@ -2,9 +2,7 @@ import { discoverGraphQL } from './discovery.js';
 import { mergeGraphQLSchemas, mergeGraphQLResolvers } from './merge.js';
 import { createLoaderFactory } from './loaders.js';
 import { createGraphQLHandler } from './handler.js';
-import { attachGraphQLSubscriptions } from './subscriptions.js';
 export {
-  attachGraphQLSubscriptions,
   createGraphQLHandler,
   createLoaderFactory,
   discoverGraphQL,

package/server/jobs/drivers/knex-queue.js

@@ -1,7 +1,7 @@
-import { buildContext } from '../../context.js';
 import { validateEnqueue, toError, calculateBackoff } from '../queue.js';
 import { checkDbThroughput } from '../throughput.js';
 import LeaseManager from './lease.js';
+import { runHandler } from './run-handler.js';
 const DEFAULT_STALE_TIMEOUT_MS = 5 * 60 * 1e3;
 class KnexJobQueue {
   db;
@@ -11,12 +11,14 @@ class KnexJobQueue {
   registered = new Map();
   _size = 0;
   leaseManager;
+  workerPool;
   constructor(db, options) {
     this.db = db;
     this.tableName = options?.tableName ?? 'arcway_jobs';
     this.backoffMs = options?.backoffMs ?? 1000;
     this.staleTimeoutMs = options?.staleTimeoutMs ?? DEFAULT_STALE_TIMEOUT_MS;
     this.leaseManager = new LeaseManager(db, { tableName: options?.leaseTableName });
+    this.workerPool = options?.workerPool ?? null;
   }
   /** Create the jobs table if it doesn't exist. Must be called before use. */
   async init() {
@@ -39,9 +41,9 @@ class KnexJobQueue {
     await this.leaseManager.init();
     await this.syncSize();
   }
-  register(domain, definition, store) {
+  register(domain, definition, store, filePath) {
     const qualifiedName = `${domain}/${definition.name}`;
-    this.registered.set(qualifiedName, { domain, definition, store });
+    this.registered.set(qualifiedName, { domain, definition, store, filePath });
   }
   async enqueue(qualifiedName, payload, options) {
     const { reg, validatedPayload, maxRetries, delay } = validateEnqueue(
@@ -163,8 +165,7 @@ class KnexJobQueue {
 
     try {
       console.log(`[job] ${qualifiedName} attempt ${attempt}/${maxAttempts}`);
-      const ctx = buildContext(reg.store, { payload });
-      await reg.definition.handler(ctx);
+      await runHandler(reg, payload, this.workerPool);
       console.log(`[job] ${qualifiedName} completed`);
       await this._updateJob(jobId, { status: 'completed', attempt });
       this._size--;

package/server/jobs/drivers/memory-queue.js

@@ -1,7 +1,7 @@
-import { buildContext } from '../../context.js';
 import { validateEnqueue, toError, calculateBackoff } from '../queue.js';
 import { MemoryThroughputTracker } from '../throughput.js';
 import LeaseManager from './lease.js';
+import { runHandler } from './run-handler.js';
 
 class JobDispatcher {
   queue = [];
@@ -9,15 +9,17 @@ class JobDispatcher {
   backoffMs;
   throughputTracker = new MemoryThroughputTracker();
   leaseManager;
+  workerPool;
 
   constructor(options) {
     this.backoffMs = options?.backoffMs ?? 1000;
     this.leaseManager = new LeaseManager(null);
+    this.workerPool = options?.workerPool ?? null;
   }
 
-  register(domain, definition, store) {
+  register(domain, definition, store, filePath) {
     const qualifiedName = `${domain}/${definition.name}`;
-    this.registered.set(qualifiedName, { domain, definition, store });
+    this.registered.set(qualifiedName, { domain, definition, store, filePath });
   }
 
   async enqueue(qualifiedName, payload, options) {
@@ -89,8 +91,7 @@ class JobDispatcher {
 
     try {
       console.log(`[job] ${qualifiedName} attempt ${job.attempt}/${maxAttempts}`);
-      const ctx = buildContext(reg.store, { payload: job.payload });
-      await reg.definition.handler(ctx);
+      await runHandler(reg, job.payload, this.workerPool);
       job.status = 'completed';
       console.log(`[job] ${qualifiedName} completed`);
       this.throughputTracker.record(qualifiedName);

package/server/jobs/drivers/run-handler.js

@@ -0,0 +1,22 @@
+import { buildContext } from '../../context.js';
+
+// Dispatch a job handler either inline (main thread, legacy / tests / continuous
+// jobs) or through the WorkerPool. The worker builds its own ctx from the
+// workerData config snapshot and calls handler(ctx) — see worker-entry.js.
+// We only route through the pool when both a pool *and* a filePath are
+// available; system jobs register without a filePath and stay inline on
+// purpose (they use namespaced services that don't round-trip through
+// workerData).
+async function runHandler(reg, payload, workerPool) {
+  if (workerPool && reg.filePath) {
+    await workerPool.run(
+      { handlerPath: reg.filePath, payload, withContext: true },
+      { timeoutMs: reg.definition.staleTimeout },
+    );
+    return;
+  }
+  const ctx = buildContext(reg.store, { payload });
+  await reg.definition.handler(ctx);
+}
+
+export { runHandler };

package/server/jobs/runner.js

@@ -21,7 +21,13 @@ async function discoverJobs(jobsDir) {
     if (!job.name) {
       job.name = relativePath.replace(/\\/g, '/').replace(/\.js$/, '');
     }
-    jobs.push({ definition: job, fileName: name, cooldownMs: job.cooldownMs, staleTimeout: job.staleTimeout });
+    jobs.push({
+      definition: job,
+      fileName: name,
+      filePath,
+      cooldownMs: job.cooldownMs,
+      staleTimeout: job.staleTimeout,
+    });
   }
   return jobs;
 }
@@ -39,10 +45,14 @@ class JobRunner {
   _stopped = false;
   _lastEnqueuedMinute = new Map();
 
-  constructor(config, { db, queue, cache, files, mail, events, log } = {}) {
+  constructor(config, { db, queue, cache, files, mail, events, log, workerPool } = {}) {
     this._config = config;
     this._log = log;
-    this._dispatcher = new JobDispatcher({ backoffMs: config?.backoffMs, staleTimeoutMs: config?.staleTimeoutMs });
+    this._dispatcher = new JobDispatcher({
+      backoffMs: config?.backoffMs,
+      staleTimeoutMs: config?.staleTimeoutMs,
+      workerPool,
+    });
     this._appContext = { db, queue, cache, files, mail, events, log };
   }
 
@@ -62,10 +72,11 @@ class JobRunner {
     const jobsDir = this._config?.dir;
     if (!jobsDir) return;
 
-    // Discover and register user jobs
+    // Discover and register user jobs. `filePath` is captured so the
+    // dispatcher can dynamic-import the handler inside a worker thread.
     const discovered = await discoverJobs(jobsDir);
-    for (const { definition, fileName } of discovered) {
-      this._dispatcher.register('app', definition, this._appContext);
+    for (const { definition, fileName, filePath } of discovered) {
+      this._dispatcher.register('app', definition, this._appContext, filePath);
       this._jobs.push({
         jobName: definition.name,
         fileName,

package/server/jobs/worker-entry.js

@@ -0,0 +1,122 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { pathToFileURL } from 'node:url';
+import { serializeError } from './worker-pool.js';
+import { buildContext } from '../context.js';
+import { createInfrastructure, destroyInfrastructure } from '../boot/infrastructure.js';
+
+// Worker-side entry for the jobs WorkerPool. Each worker owns its own DB /
+// redis / queue / cache / files / mail / events / log — built lazily on the
+// first task and reused for every subsequent task. The pool tells the worker
+// to tear those down with a `{ __shutdown: true }` control message (see
+// WorkerPool.shutdown) before terminate().
+if (!parentPort) {
+  throw new Error('worker-entry.js must be run as a worker thread');
+}
+
+const moduleCache = new Map();
+let infrastructure = null;
+let infrastructurePromise = null;
+let shuttingDown = false;
+
+async function ensureInfrastructure() {
+  if (infrastructure) return infrastructure;
+  if (!infrastructurePromise) {
+    const config = workerData?.config;
+    if (!config) {
+      throw new Error('Worker started without workerData.config — cannot build job context');
+    }
+    infrastructurePromise = createInfrastructure(config, { runMigrations: false }).then(
+      (services) => {
+        infrastructure = services;
+        return services;
+      },
+    );
+  }
+  return infrastructurePromise;
+}
+
+async function loadHandler(handlerPath, handlerExport) {
+  let mod = moduleCache.get(handlerPath);
+  if (!mod) {
+    const specifier = handlerPath.startsWith('file:')
+      ? handlerPath
+      : pathToFileURL(handlerPath).href;
+    mod = await import(specifier);
+    moduleCache.set(handlerPath, mod);
+  }
+  const exportName = handlerExport ?? 'default';
+  const exported = mod[exportName];
+  // Job files export `{ handler, ... }` as default; generic callers (and the
+  // Phase 1 unit tests) can also export a plain function directly.
+  const fn =
+    typeof exported === 'function'
+      ? exported
+      : typeof exported?.handler === 'function'
+        ? exported.handler
+        : null;
+  if (!fn) {
+    throw new Error(
+      `Worker handler export "${exportName}" in ${handlerPath} is not a function`,
+    );
+  }
+  return fn;
+}
+
+async function runTask(task) {
+  const { handlerPath, handlerExport, payload, withContext } = task ?? {};
+  if (!handlerPath) throw new Error('Worker task is missing handlerPath');
+  const fn = await loadHandler(handlerPath, handlerExport);
+  // `withContext: true` (the jobs path) gives the handler a real ctx built
+  // from the worker's own services. `withContext: false` (the Phase 1 tests
+  // and any future cpu-only caller) gets the raw payload — no DB spin-up.
+  if (withContext) {
+    const services = await ensureInfrastructure();
+    const ctx = buildContext(services, { payload });
+    return fn(ctx);
+  }
+  return fn(payload);
+}
+
+async function handleShutdown(id) {
+  shuttingDown = true;
+  try {
+    if (infrastructure) {
+      await destroyInfrastructure(infrastructure);
+      infrastructure = null;
+    }
+    parentPort.postMessage({ id, status: 'ok', result: null, __shutdown: true });
+  } catch (err) {
+    parentPort.postMessage({
+      id,
+      status: 'error',
+      error: serializeError(err),
+      __shutdown: true,
+    });
+  } finally {
+    // Exit cleanly so the pool's graceful-shutdown wait resolves on the
+    // 'exit' event rather than having to fall back to terminate().
+    process.exit(0);
+  }
+}
+
+parentPort.on('message', async (msg) => {
+  const { id, task } = msg ?? {};
+  if (task?.__shutdown) {
+    await handleShutdown(id);
+    return;
+  }
+  if (shuttingDown) {
+    parentPort.postMessage({
+      id,
+      status: 'error',
+      error: serializeError(new Error('Worker is shutting down')),
+    });
+    return;
+  }
+  try {
+    const result = await runTask(task);
+    parentPort.postMessage({ id, status: 'ok', result });
+  } catch (err) {
+    parentPort.postMessage({ id, status: 'error', error: serializeError(err) });
+  }
+});