arcway 0.1.0

package/server/graphql/discovery.js

@@ -0,0 +1,70 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { loadModule } from '../module-loader.js';
+async function discoverGraphQL(rootDir) {
+  const graphqlDir = path.join(rootDir, 'graphql');
+  if (!fs.existsSync(graphqlDir)) return [];
+  const entry = {};
+  let hasContent = false;
+  const schemaPath = findFile(graphqlDir, 'schema');
+  if (schemaPath) {
+    let module;
+    try {
+      module = await loadModule(schemaPath, import.meta.url);
+    } catch (err) {
+      throw new Error(`Failed to load GraphQL schema at ${schemaPath}: ${err}`);
+    }
+    const schema = module.default;
+    if (typeof schema !== 'string') {
+      throw new Error(
+        `GraphQL schema must default-export an SDL string. Got ${typeof schema} (${schemaPath})`,
+      );
+    }
+    entry.schema = schema;
+    hasContent = true;
+  }
+  const resolversPath = findFile(graphqlDir, 'resolvers');
+  if (resolversPath) {
+    let module;
+    try {
+      module = await loadModule(resolversPath, import.meta.url);
+    } catch (err) {
+      throw new Error(`Failed to load GraphQL resolvers at ${resolversPath}: ${err}`);
+    }
+    const resolvers = module.default;
+    if (!resolvers || typeof resolvers !== 'object' || Array.isArray(resolvers)) {
+      throw new Error(
+        `GraphQL resolvers must default-export an object. Got ${typeof resolvers} (${resolversPath})`,
+      );
+    }
+    entry.resolvers = resolvers;
+    hasContent = true;
+  }
+  const loadersPath = findFile(graphqlDir, 'loaders');
+  if (loadersPath) {
+    let module;
+    try {
+      module = await loadModule(loadersPath, import.meta.url);
+    } catch (err) {
+      throw new Error(`Failed to load GraphQL loaders at ${loadersPath}: ${err}`);
+    }
+    const loaderFactory = module.default;
+    if (typeof loaderFactory !== 'function') {
+      throw new Error(
+        `GraphQL loaders must default-export a function. Got ${typeof loaderFactory} (${loadersPath})`,
+      );
+    }
+    entry.loaderFactory = loaderFactory;
+    hasContent = true;
+  }
+  if (hasContent) {
+    return [entry];
+  }
+  return [];
+}
+function findFile(dir, name) {
+  const jsPath = path.join(dir, `${name}.js`);
+  if (fs.existsSync(jsPath)) return jsPath;
+  return null;
+}
+export { discoverGraphQL };

package/server/graphql/handler.js

@@ -0,0 +1,41 @@
+import { createSchema, createYoga } from 'graphql-yoga';
+import { randomUUID } from 'node:crypto';
+import { mergeGraphQLSchemas, mergeGraphQLResolvers } from './merge.js';
+import { createLoaderFactory } from './loaders.js';
+import { parseCookiesFromHeader, resolveSession } from '../session/helpers.js';
+function createGraphQLHandler(options) {
+  const { domains, graphiql, session: sessionConfig, log } = options;
+  const typeDefs = mergeGraphQLSchemas(domains);
+  const resolvers = mergeGraphQLResolvers(domains);
+  const makeLoaders = createLoaderFactory(domains);
+  const schema = createSchema({ typeDefs, resolvers });
+  const yoga = createYoga({
+    schema,
+    graphqlEndpoint: '/graphql',
+    graphiql: graphiql ?? false,
+    // Disable yoga's own CORS — Arcway handles CORS at the framework level
+    cors: false,
+    context: async ({ request }) => {
+      const loaders = makeLoaders();
+      const requestId = request.headers.get('x-request-id') ?? randomUUID();
+      const headers = {};
+      request.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+      const cookies = parseCookiesFromHeader(request.headers.get('cookie') ?? '');
+      const session = await resolveSession(cookies, sessionConfig);
+      return { loaders, session, requestId, headers, cookies };
+    },
+    logging: log
+      ? {
+          debug: (...args) => log.info(String(args[0])),
+          info: (...args) => log.info(String(args[0])),
+          warn: (...args) => log.warn(String(args[0])),
+          error: (...args) => log.error(String(args[0])),
+        }
+      : false,
+  });
+  const handler = yoga;
+  return { handler, schema, session: sessionConfig };
+}
+export { createGraphQLHandler };

package/server/graphql/index.js

@@ -0,0 +1,13 @@
+import { discoverGraphQL } from './discovery.js';
+import { mergeGraphQLSchemas, mergeGraphQLResolvers } from './merge.js';
+import { createLoaderFactory } from './loaders.js';
+import { createGraphQLHandler } from './handler.js';
+import { attachGraphQLSubscriptions } from './subscriptions.js';
+export {
+  attachGraphQLSubscriptions,
+  createGraphQLHandler,
+  createLoaderFactory,
+  discoverGraphQL,
+  mergeGraphQLResolvers,
+  mergeGraphQLSchemas,
+};

package/server/graphql/loaders.js

@@ -0,0 +1,19 @@
+import DataLoader from 'dataloader';
+function createLoaderFactory(entries) {
+  const factories = [];
+  for (const entry of entries) {
+    if (!entry.loaderFactory) continue;
+    factories.push(entry.loaderFactory);
+  }
+  return () => {
+    const loaders = {};
+    for (const factory of factories) {
+      const batchFns = factory();
+      for (const [name, batchFn] of Object.entries(batchFns)) {
+        loaders[name] = new DataLoader(batchFn);
+      }
+    }
+    return loaders;
+  };
+}
+export { createLoaderFactory };

package/server/graphql/merge.js

@@ -0,0 +1,48 @@
+const BASE_SCHEMA = `
+  type Query {
+    _system_health: Boolean
+  }
+  type Mutation {
+    _system_noop: Boolean
+  }
+`;
+const BASE_RESOLVERS = {
+  Query: {
+    _system_health: () => true,
+  },
+  Mutation: {
+    _system_noop: () => true,
+  },
+};
+function mergeGraphQLSchemas(entries) {
+  const parts = [BASE_SCHEMA];
+  for (const entry of entries) {
+    if (entry.schema) {
+      parts.push(entry.schema);
+    }
+  }
+  return parts.join('\n\n');
+}
+function mergeGraphQLResolvers(entries) {
+  const merged = {};
+  for (const [typeName, fields] of Object.entries(BASE_RESOLVERS)) {
+    merged[typeName] = { ...fields };
+  }
+  for (const entry of entries) {
+    if (!entry.resolvers) continue;
+    for (const [typeName, fields] of Object.entries(entry.resolvers)) {
+      if (!merged[typeName]) {
+        merged[typeName] = {};
+      }
+      if (typeof fields === 'object' && fields !== null && !Array.isArray(fields)) {
+        for (const [fieldName, resolver] of Object.entries(fields)) {
+          merged[typeName][fieldName] = resolver;
+        }
+      } else {
+        merged[typeName] = fields;
+      }
+    }
+  }
+  return merged;
+}
+export { mergeGraphQLResolvers, mergeGraphQLSchemas };

package/server/graphql/subscriptions.js

@@ -0,0 +1,43 @@
+import { WebSocketServer } from 'ws';
+import { useServer } from 'graphql-ws/use/ws';
+import { parseCookiesFromHeader, resolveSession, flattenHeaders } from '../session/helpers.js';
+function attachGraphQLSubscriptions(options) {
+  const { server, schema, session: sessionConfig, log } = options;
+  const wss = new WebSocketServer({ noServer: true });
+  server.on('upgrade', (req, socket, head) => {
+    const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
+    if (url.pathname !== '/graphql') return;
+    const protocol = req.headers['sec-websocket-protocol'];
+    if (!protocol?.includes('graphql-transport-ws')) return;
+    wss.handleUpgrade(req, socket, head, (ws) => {
+      wss.emit('connection', ws, req);
+    });
+  });
+  const serverCleanup = useServer(
+    {
+      schema,
+      context: async (ctx) => {
+        const req = ctx.extra.request;
+        const headers = flattenHeaders(req.headers);
+        const cookies = parseCookiesFromHeader(req.headers.cookie ?? '');
+        const session = await resolveSession(cookies, sessionConfig);
+        return { session, headers, cookies };
+      },
+      onConnect: () => {
+        log?.info('GraphQL subscription client connected');
+      },
+      onDisconnect: () => {
+        log?.info('GraphQL subscription client disconnected');
+      },
+    },
+    wss,
+  );
+  return {
+    wss,
+    close: async () => {
+      await serverCleanup.dispose();
+      wss.close();
+    },
+  };
+}
+export { attachGraphQLSubscriptions };

package/server/health.js

@@ -0,0 +1,34 @@
+import { toErrorMessage } from './helpers.js';
+
+async function checkHealth(deps) {
+  const components = {};
+  const dbStart = Date.now();
+  try {
+    await deps.db.raw('SELECT 1');
+    components.database = { status: 'ok', responseMs: Date.now() - dbStart };
+  } catch (err) {
+    components.database = {
+      status: 'error',
+      responseMs: Date.now() - dbStart,
+      error: toErrorMessage(err),
+    };
+  }
+  if (deps.redisClients) {
+    for (const { name, client } of deps.redisClients) {
+      const start = Date.now();
+      try {
+        await client.ping();
+        components[name] = { status: 'ok', responseMs: Date.now() - start };
+      } catch (err) {
+        components[name] = {
+          status: 'error',
+          responseMs: Date.now() - start,
+          error: toErrorMessage(err),
+        };
+      }
+    }
+  }
+  const allOk = Object.values(components).every((c) => c.status === 'ok');
+  return { status: allOk ? 'ok' : 'degraded', components };
+}
+export { checkHealth };

package/server/helpers.js

@@ -0,0 +1,9 @@
+/**
+ * Extract a human-readable message from an unknown caught value.
+ * Replaces the repeated `err instanceof Error ? err.message : String(err)` pattern.
+ */
+function toErrorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+
+export { toErrorMessage };

package/server/index.js

@@ -0,0 +1,55 @@
+import { type } from 'arktype';
+import { buildContext } from './context.js';
+import * as vault from './lib/vault/index.js';
+import { defineRoute } from './route.js';
+import boot from './boot.js';
+import {
+  Arcway,
+  Solo,
+  createTestContext,
+  createEventStub,
+  createQueueStub,
+  createCacheStub,
+  createFilesStub,
+  createMailStub,
+  createLoggerStub,
+} from './testing/index.js';
+import {
+  createRateLimitMiddleware,
+  MemoryRateLimitStore,
+  RedisRateLimitStore,
+} from './rate-limit/index.js';
+import { wsSendToSocket, wsBroadcastToPath } from './ws/registry.js';
+import {
+  SYSTEM_JOB_DOMAIN,
+  addSystemJobEntry,
+  registerSystemJobs,
+  validateNoSystemJobCollision,
+  getApplicableSystemJobs,
+} from './system-jobs/index.js';
+export {
+  MemoryRateLimitStore,
+  RedisRateLimitStore,
+  Arcway,
+  SYSTEM_JOB_DOMAIN,
+  Solo,
+  addSystemJobEntry,
+  boot,
+  buildContext,
+  createCacheStub,
+  createEventStub,
+  createFilesStub,
+  createLoggerStub,
+  createMailStub,
+  createQueueStub,
+  createRateLimitMiddleware,
+  createTestContext,
+  defineRoute,
+  getApplicableSystemJobs,
+  registerSystemJobs,
+  type,
+  validateNoSystemJobCollision,
+  vault,
+  wsBroadcastToPath,
+  wsSendToSocket,
+};

package/server/internals.js

@@ -0,0 +1,139 @@
+export * from './constants.js';
+import { loadEnvFiles } from './env.js';
+import { makeConfig } from './config/loader.js';
+import { createDB, MigrationSource } from './db/index.js';
+import { runSeeds } from './db/seeds.js';
+import MemoryTransport from './events/drivers/memory.js';
+import RedisTransport from './events/drivers/redis.js';
+import { EventHandler, listenerPathToEvent } from './events/handler.js';
+import JobDispatcher from './jobs/drivers/memory-queue.js';
+import { discoverJobs, JobRunner } from './jobs/runner.js';
+import { parseCron, matchesCron } from './jobs/cron.js';
+import Queue from './queue/index.js';
+import KnexQueueDriver from './queue/drivers/knex.js';
+import RedisQueueDriver from './queue/drivers/redis.js';
+import Cache from './cache/index.js';
+import MemoryCacheDriver from './cache/drivers/memory.js';
+import RedisCacheDriver from './cache/drivers/redis.js';
+import Files from './files/index.js';
+import LocalFileDriver from './files/drivers/local.js';
+import S3FileDriver from './files/drivers/s3.js';
+import { generateOpenAPISpec, typeToOpenAPISchema, toOpenAPIPath } from './docs/openapi.js';
+import { checkServerClientBoundaries, extractImports } from './lint/boundaries.js';
+import { filePathToPattern, compilePattern, discoverRoutes, matchRoute } from './router/routes.js';
+import { ApiRouter } from './router/api-router.js';
+import WebServer from './web-server.js';
+import {
+  discoverMiddleware,
+  getMiddlewareForRoute,
+  buildMiddlewareChain,
+} from './router/middleware.js';
+import { buildCorsHeaders, buildPreflightHeaders } from './router/cors.js';
+import {
+  resolveSessionConfig,
+  unsealSession,
+  sealSession,
+  buildSessionSetCookie,
+  buildSessionClearCookie,
+} from './session/index.js';
+import { checkHealth } from './health.js';
+import MemoryRateLimitStore from './rate-limit/drivers/memory.js';
+import { createRateLimitMiddleware } from './rate-limit/index.js';
+import { createHttpServer, listen, closeServer } from './server.js';
+import { startWatcher } from './watcher.js';
+import { build } from './build.js';
+import {
+  discoverPages,
+  discoverLayouts,
+  discoverErrorPages,
+  resolveLayoutChain,
+  matchPage,
+} from './pages/discovery.js';
+import { buildPages, patternToFileName } from './pages/build.js';
+import { createPagesHandler } from './pages/handler.js';
+import { createPagesWatcher } from './pages/watcher.js';
+import {
+  discoverGraphQL,
+  mergeGraphQLSchemas,
+  mergeGraphQLResolvers,
+  createLoaderFactory,
+  createGraphQLHandler,
+} from './graphql/index.js';
+import {
+  discoverWebSocketHandlers,
+  mergeWebSocketHandlers,
+  createWebSocketServer,
+} from './ws/index.js';
+import { attachGraphQLSubscriptions } from './graphql/subscriptions.js';
+export {
+  ApiRouter,
+  Cache,
+  makeConfig,
+  Files,
+  KnexQueueDriver,
+  LocalFileDriver,
+  MemoryCacheDriver,
+  JobDispatcher,
+  MemoryRateLimitStore,
+  MemoryTransport,
+  MigrationSource,
+  RedisCacheDriver,
+  RedisQueueDriver,
+  RedisTransport,
+  Queue,
+  S3FileDriver,
+  attachGraphQLSubscriptions,
+  build,
+  buildCorsHeaders,
+  buildMiddlewareChain,
+  buildPages,
+  buildPreflightHeaders,
+  buildSessionClearCookie,
+  buildSessionSetCookie,
+  checkHealth,
+  checkServerClientBoundaries,
+  closeServer,
+  compilePattern,
+  createGraphQLHandler,
+  createHttpServer,
+  JobRunner,
+  createDB,
+  createLoaderFactory,
+  createPagesHandler,
+  createPagesWatcher,
+  createRateLimitMiddleware,
+  createWebSocketServer,
+  discoverJobs,
+  EventHandler,
+  discoverErrorPages,
+  discoverGraphQL,
+  discoverLayouts,
+  discoverMiddleware,
+  discoverPages,
+  discoverRoutes,
+  discoverWebSocketHandlers,
+  extractImports,
+  filePathToPattern,
+  generateOpenAPISpec,
+  getMiddlewareForRoute,
+  listen,
+  listenerPathToEvent,
+  loadEnvFiles,
+  matchPage,
+  matchRoute,
+  matchesCron,
+  mergeGraphQLResolvers,
+  mergeGraphQLSchemas,
+  mergeWebSocketHandlers,
+  parseCron,
+  patternToFileName,
+  resolveLayoutChain,
+  resolveSessionConfig,
+  runSeeds,
+  sealSession,
+  startWatcher,
+  toOpenAPIPath,
+  typeToOpenAPISchema,
+  unsealSession,
+  WebServer,
+};

package/server/jobs/cron.js

@@ -0,0 +1,10 @@
+import { CronExpressionParser } from 'cron-parser';
+function parseCron(expression) {
+  return CronExpressionParser.parse(expression);
+}
+function matchesCron(parsed, date) {
+  const normalized = new Date(date);
+  normalized.setSeconds(0, 0);
+  return parsed.includesDate(normalized);
+}
+export { matchesCron, parseCron };

package/server/jobs/drivers/knex-queue.js

@@ -0,0 +1,207 @@
+import { buildContext } from '../../context.js';
+import { validateEnqueue, toError, calculateBackoff } from '../queue.js';
+import { checkDbThroughput } from '../throughput.js';
+import LeaseManager from './lease.js';
+const STALE_JOB_TIMEOUT_MS = 5 * 60 * 1e3;
+class KnexJobQueue {
+  db;
+  tableName;
+  backoffMs;
+  registered = new Map();
+  _size = 0;
+  leaseManager;
+  constructor(db, options) {
+    this.db = db;
+    this.tableName = options?.tableName ?? 'arcway_jobs';
+    this.backoffMs = options?.backoffMs ?? 1000;
+    this.leaseManager = new LeaseManager(db, { tableName: options?.leaseTableName });
+  }
+  /** Create the jobs table if it doesn't exist. Must be called before use. */
+  async init() {
+    const exists = await this.db.schema.hasTable(this.tableName);
+    if (!exists) {
+      await this.db.schema.createTable(this.tableName, (table) => {
+        table.increments('id').primary();
+        table.string('qualified_name').notNullable().index();
+        table.string('domain').notNullable();
+        table.string('job_name').notNullable();
+        table.text('payload').notNullable();
+        table.integer('max_retries').notNullable().defaultTo(0);
+        table.integer('attempt').notNullable().defaultTo(0);
+        table.bigInteger('run_at').notNullable();
+        table.string('status').notNullable().defaultTo('pending').index();
+        table.timestamp('created_at').defaultTo(this.db.fn.now());
+        table.timestamp('updated_at').defaultTo(this.db.fn.now());
+      });
+    }
+    await this.leaseManager.init();
+    await this.syncSize();
+  }
+  register(domain, definition, store) {
+    const qualifiedName = `${domain}/${definition.name}`;
+    this.registered.set(qualifiedName, { domain, definition, store });
+  }
+  async enqueue(qualifiedName, payload, options) {
+    const { reg, validatedPayload, maxRetries, delay } = validateEnqueue(
+      qualifiedName,
+      payload,
+      this.registered,
+      options,
+    );
+    await this.db(this.tableName).insert({
+      qualified_name: qualifiedName,
+      domain: reg.domain,
+      job_name: reg.definition.name,
+      payload: JSON.stringify(validatedPayload),
+      max_retries: maxRetries,
+      attempt: 0,
+      run_at: Date.now() + delay,
+      status: 'pending',
+    });
+    this._size++;
+  }
+  async process() {
+    const now = Date.now();
+    await this._recoverStaleJobs(now);
+    const jobs = await this._claimReadyJobs(now);
+
+    const results = [];
+    for (const row of jobs) {
+      const result = await this._executeJob(row);
+      if (result) results.push(result);
+    }
+    return results;
+  }
+
+  async _recoverStaleJobs(now) {
+    const staleCutoff = new Date(now - STALE_JOB_TIMEOUT_MS).toISOString();
+    await this.db(this.tableName)
+      .where('status', 'running')
+      .andWhere('updated_at', '<', staleCutoff)
+      .update({ status: 'pending', updated_at: new Date().toISOString() });
+  }
+
+  async _claimReadyJobs(now) {
+    return this.db.transaction(async (trx) => {
+      const readyRows = await trx(this.tableName)
+        .where('status', 'pending')
+        .andWhere('run_at', '<=', now)
+        .orderBy('id', 'asc')
+        .select('id', 'qualified_name');
+      if (readyRows.length === 0) return [];
+
+      const throttledTypes = new Set();
+      const allowedIds = [];
+      for (const row of readyRows) {
+        const qualifiedName = row.qualified_name;
+        if (throttledTypes.has(qualifiedName)) continue;
+        const reg = this.registered.get(qualifiedName);
+        if (reg?.definition.throughput) {
+          const allowed = await checkDbThroughput(
+            trx,
+            this.tableName,
+            qualifiedName,
+            reg.definition.throughput,
+          );
+          if (!allowed) {
+            throttledTypes.add(qualifiedName);
+            continue;
+          }
+        }
+        allowedIds.push(row.id);
+      }
+      if (allowedIds.length === 0) return [];
+
+      await trx(this.tableName)
+        .whereIn('id', allowedIds)
+        .update({ status: 'running', updated_at: new Date().toISOString() });
+      return trx(this.tableName).whereIn('id', allowedIds).select('*');
+    });
+  }
+
+  async _executeJob(row) {
+    const qualifiedName = row.qualified_name;
+    const reg = this.registered.get(qualifiedName);
+    if (!reg) {
+      await this.db(this.tableName).where('id', row.id).update({ status: 'pending' });
+      return null;
+    }
+
+    const jobId = row.id;
+    const maxConcurrency = reg.definition.maxConcurrency;
+
+    if (maxConcurrency !== undefined) {
+      const acquired = await this.leaseManager.acquire(qualifiedName, jobId, maxConcurrency);
+      if (!acquired) {
+        await this.db(this.tableName)
+          .where('id', jobId)
+          .update({ status: 'pending', updated_at: new Date().toISOString() });
+        return null;
+      }
+      this.leaseManager.startHeartbeat(jobId);
+    }
+
+    const attempt = row.attempt + 1;
+    const maxAttempts = row.max_retries + 1;
+    const payload = JSON.parse(row.payload);
+
+    try {
+      console.log(`[job] ${qualifiedName} attempt ${attempt}/${maxAttempts}`);
+      const ctx = buildContext(reg.store, { payload });
+      await reg.definition.handler(ctx);
+      console.log(`[job] ${qualifiedName} completed`);
+      await this._updateJob(jobId, { status: 'completed', attempt });
+      this._size--;
+      return { id: String(jobId), status: 'completed', attempts: attempt };
+    } catch (err) {
+      const error = toError(err);
+      console.error(
+        `[job] ${qualifiedName} attempt ${attempt}/${maxAttempts} failed:`,
+        error.message,
+      );
+
+      if (attempt < maxAttempts) {
+        const backoffMs = calculateBackoff(this.backoffMs, attempt);
+        console.log(
+          `[job] ${qualifiedName} retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxAttempts})`,
+        );
+        await this._updateJob(jobId, {
+          status: 'pending',
+          attempt,
+          run_at: Date.now() + backoffMs,
+        });
+        return null;
+      }
+
+      await this._updateJob(jobId, { status: 'failed', attempt });
+      this._size--;
+      return { id: String(jobId), status: 'failed', error, attempts: attempt };
+    } finally {
+      if (maxConcurrency !== undefined) {
+        await this.leaseManager.release(jobId, qualifiedName);
+      }
+    }
+  }
+
+  async _updateJob(jobId, fields) {
+    await this.db(this.tableName)
+      .where('id', jobId)
+      .update({ ...fields, updated_at: new Date().toISOString() });
+  }
+  get size() {
+    return this._size;
+  }
+  /** Release all leases held by this runner (call during graceful shutdown). */
+  async shutdown() {
+    await this.leaseManager.releaseAll();
+  }
+  /** Sync the in-memory size counter with the database. */
+  async syncSize() {
+    const result = await this.db(this.tableName)
+      .where('status', 'pending')
+      .count('id as count')
+      .first();
+    this._size = Number(result?.count ?? 0);
+  }
+}
+export default KnexJobQueue;