keryx 0.0.1

package/initializers/pubsub.ts

@@ -0,0 +1,86 @@
+import { api } from "../api";
+import { Initializer } from "../classes/Initializer";
+import { config } from "../config";
+import pkg from "../package.json";
+
+const namespace = "pubsub";
+const redisPubSubChannel = `keryx:pubsub:${pkg.name}`;
+
+export type PubSubMessage = {
+  channel: string;
+  message: string;
+  sender: string;
+};
+
+export type ClientSubscribeMessage = {
+  messageType: "subscribe";
+  messageId: string | number;
+  channel: string;
+};
+
+export type ClientUnsubscribeMessage = {
+  messageType: "unsubscribe";
+  messageId: string | number;
+  channel: string;
+};
+
+declare module "../classes/API" {
+  export interface API {
+    [namespace]: Awaited<ReturnType<PubSub["initialize"]>>;
+  }
+}
+
+export class PubSub extends Initializer {
+  constructor() {
+    super(namespace);
+    this.startPriority = 150;
+    this.stopPriority = 950;
+  }
+
+  async initialize() {
+    async function broadcast(
+      channel: string,
+      message: any,
+      sender = "unknown-sender",
+    ) {
+      const payload: PubSubMessage = { channel, message, sender };
+      return api.redis.redis.publish(
+        redisPubSubChannel,
+        JSON.stringify(payload),
+      );
+    }
+
+    return { broadcast };
+  }
+
+  async start() {
+    if (api.redis.subscription) {
+      await api.redis.subscription.subscribe(redisPubSubChannel);
+      api.redis.subscription.on("message", this.handleMessage.bind(this));
+    }
+  }
+
+  async stop() {
+    if (api.redis.subscription) {
+      api.redis.subscription.removeAllListeners("message");
+      await api.redis.subscription.unsubscribe(redisPubSubChannel);
+    }
+  }
+
+  async handleMessage(
+    _pubSubChannel: string,
+    incomingMessage: string | Buffer,
+  ) {
+    const payload = JSON.parse(incomingMessage.toString()) as PubSubMessage;
+    for (const connection of api.connections.connections) {
+      if (connection.subscriptions.has(payload.channel)) {
+        connection.onBroadcastMessageReceived(payload);
+      }
+    }
+
+    // Forward to MCP as notifications
+    if (config.server.mcp.enabled && api.mcp?.sendNotification) {
+      api.mcp.sendNotification(payload);
+    }
+  }
+}

package/initializers/redis.ts

@@ -0,0 +1,77 @@
+import { Redis as RedisClient } from "ioredis";
+import { api, logger } from "../api";
+import { Initializer } from "../classes/Initializer";
+import { ErrorType, TypedError } from "../classes/TypedError";
+import { config } from "../config";
+import { formatConnectionStringForLogging } from "../util/connectionString";
+
+const namespace = "redis";
+const testKey = `__keryx_test_key:${config.process.name}`;
+
+declare module "../classes/API" {
+  export interface API {
+    [namespace]: Awaited<ReturnType<Redis["initialize"]>>;
+  }
+}
+
+export class Redis extends Initializer {
+  constructor() {
+    super(namespace);
+    this.loadPriority = 200;
+    this.startPriority = 110;
+    this.stopPriority = 990;
+  }
+
+  async initialize() {
+    const redisContainer = {} as {
+      redis: RedisClient;
+      subscription: RedisClient;
+    };
+    return redisContainer;
+  }
+
+  async start() {
+    api.redis.redis = new RedisClient(config.redis.connectionString);
+    api.redis.subscription = new RedisClient(config.redis.connectionString);
+
+    try {
+      await api.redis.redis.set(testKey, Date.now());
+      await api.redis.redis.del(testKey);
+      await api.redis.subscription.set(testKey, Date.now());
+      await api.redis.subscription.del(testKey);
+    } catch (e) {
+      throw new TypedError({
+        type: ErrorType.SERVER_INITIALIZATION,
+        message: `Cannot connect to redis (${formatConnectionStringForLogging(config.redis.connectionString)}): ${e}`,
+      });
+    }
+
+    logger.info(
+      `redis connections established (${formatConnectionStringForLogging(config.redis.connectionString)})`,
+    );
+  }
+
+  async stop() {
+    let acted = false;
+
+    if (api.redis.redis) {
+      try {
+        await api.redis.redis.quit();
+        acted = true;
+      } catch (e) {
+        logger.error(`error closing redis connection: ${e}`);
+      }
+    }
+
+    if (api.redis.subscription) {
+      try {
+        await api.redis.subscription.quit();
+        acted = true;
+      } catch (e) {
+        logger.error(`error closing redis subscription connection: ${e}`);
+      }
+    }
+
+    if (acted) logger.info("redis connections closed");
+  }
+}

package/initializers/resque.ts

@@ -0,0 +1,354 @@
+import {
+  Queue,
+  Scheduler,
+  Worker,
+  type Job,
+  type ParsedJob,
+} from "node-resque";
+import {
+  Action,
+  api,
+  config,
+  Connection,
+  logger,
+  RUN_MODE,
+  type ActionParams,
+} from "../api";
+import { Initializer } from "../classes/Initializer";
+import { TypedError } from "../classes/TypedError";
+
+const namespace = "resque";
+
+declare module "../classes/API" {
+  export interface API {
+    [namespace]: Awaited<ReturnType<Resque["initialize"]>>;
+  }
+}
+
+let SERVER_JOB_COUNTER = 1;
+
+export class Resque extends Initializer {
+  constructor() {
+    super(namespace);
+
+    this.loadPriority = 250;
+    this.startPriority = 10000;
+    this.stopPriority = 900;
+  }
+
+  startQueue = async () => {
+    api.resque.queue = new Queue(
+      { connection: { redis: api.redis.redis } },
+      api.resque.jobs,
+    );
+
+    api.resque.queue.on("error", (error) => {
+      logger.error(`[resque:queue] ${error}`);
+    });
+
+    await api.resque.queue.connect();
+  };
+
+  stopQueue = async () => {
+    if (api.resque.queue) {
+      return api.resque.queue.end();
+    }
+  };
+
+  startScheduler = async () => {
+    if (config.tasks.enabled === true) {
+      api.resque.scheduler = new Scheduler({
+        connection: { redis: api.redis.redis },
+        timeout: config.tasks.timeout,
+        stuckWorkerTimeout: config.tasks.stuckWorkerTimeout,
+        retryStuckJobs: config.tasks.retryStuckJobs,
+      });
+
+      api.resque.scheduler.on("error", (error) => {
+        logger.error(`[resque:scheduler] ${error}`);
+      });
+
+      await api.resque.scheduler.connect();
+
+      api.resque.scheduler.on("start", () => {
+        logger.info(`[resque:scheduler] started`);
+      });
+      api.resque.scheduler.on("end", () => {
+        logger.info(`[resque:scheduler] ended`);
+      });
+      api.resque.scheduler.on("poll", () => {
+        logger.debug(`[resque:scheduler] polling`);
+      });
+      api.resque.scheduler.on("leader", () => {
+        logger.info(`[resque:scheduler] leader elected`);
+      });
+      api.resque.scheduler.on(
+        "cleanStuckWorker",
+        (workerName, errorPayload, delta) => {
+          logger.warn(
+            `[resque:scheduler] cleaning stuck worker: ${workerName}, ${errorPayload}, ${delta}`,
+          );
+        },
+      );
+
+      api.resque.scheduler.start();
+      await api.actions.enqueueAllRecurrent();
+    }
+  };
+
+  stopScheduler = async () => {
+    if (api.resque.scheduler && api.resque.scheduler.connection.connected) {
+      await api.resque.scheduler.end();
+    }
+  };
+
+  startWorkers = async () => {
+    let id = 0;
+
+    while (id < config.tasks.taskProcessors) {
+      const worker = new Worker(
+        {
+          connection: { redis: api.redis.redis },
+          queues: Array.isArray(config.tasks.queues)
+            ? config.tasks.queues
+            : await config.tasks.queues(),
+          timeout: config.tasks.timeout,
+          name: `worker:${id}`,
+        },
+        api.resque.jobs,
+      );
+
+      // normal worker emitters
+      worker.on("start", () => {
+        logger.info(`[resque:${worker.name}] started`);
+      });
+      worker.on("end", () => {
+        logger.info(`[resque:${worker.name}] ended`);
+      });
+      worker.on("cleaning_worker", () => {
+        logger.debug(`[resque:${worker.name}] cleaning worker`);
+      });
+      worker.on("poll", (queue) => {
+        logger.debug(`[resque:${worker.name}] polling, ${queue}`);
+      });
+      worker.on("job", (queue, job: ParsedJob) => {
+        logger.debug(
+          `[resque:${worker.name}] job acquired, ${queue}, ${job.class}, ${JSON.stringify(job.args[0])}`,
+        );
+      });
+      worker.on("reEnqueue", (queue, job: ParsedJob, _plugin) => {
+        logger.debug(
+          `[resque:${worker.name}] job reEnqueue, ${queue}, ${job.class}, ${JSON.stringify(job.args[0])}`,
+        );
+      });
+      worker.on("pause", () => {
+        logger.debug(`[resque:${worker.name}] paused`);
+      });
+
+      worker.on("failure", (queue, job, failure, duration) => {
+        logger.warn(
+          `[resque:${worker.name}] job failed, ${queue}, ${job.class}, ${JSON.stringify(job?.args[0] ?? {})}: ${failure} (${duration}ms)`,
+        );
+      });
+      worker.on("error", (error, queue, job) => {
+        logger.warn(
+          `[resque:${worker.name}] job error, ${queue}, ${job?.class}, ${JSON.stringify(job?.args[0] ?? {})}: ${error}`,
+        );
+      });
+
+      worker.on("success", (queue, job: ParsedJob, result, duration) => {
+        logger.info(
+          `[resque:${worker.name}] job success ${queue}, ${job.class}, ${JSON.stringify(job.args[0])} | ${JSON.stringify(result)} (${duration}ms)`,
+        );
+      });
+
+      api.resque.workers.push(worker);
+      id++;
+    }
+
+    for (const worker of api.resque.workers) {
+      try {
+        await worker.connect();
+        await worker.start();
+      } catch (error) {
+        logger.fatal(`[resque:${worker.name}] ${error}`);
+        throw error;
+      }
+    }
+  };
+
+  stopWorkers = async () => {
+    // Signal all workers to stop polling/pinging before closing connections.
+    // worker.end() clears timers and closes the Redis connection, but if a
+    // poll() or ping() callback already fired and has an in-flight Redis
+    // command, it will reject with "Connection is closed." Setting running=false
+    // first ensures no NEW operations start, then we drain any in-flight ones.
+    for (const worker of api.resque.workers) {
+      worker.running = false;
+    }
+    await Bun.sleep(250);
+
+    while (true) {
+      const worker = api.resque.workers.pop();
+      if (!worker) break;
+      await worker.end();
+    }
+    api.resque.workers = [];
+  };
+
+  /** Load all actions as tasks and wrap them for node-resque jobs */
+  loadJobs = async () => {
+    const jobs: Record<string, Job<any>> = {};
+
+    for (const action of api.actions.actions) {
+      const job = this.wrapActionAsJob(action);
+      jobs[action.name] = job;
+    }
+
+    return jobs;
+  };
+
+  wrapActionAsJob = (
+    action: Action,
+  ): Job<Awaited<ReturnType<(typeof action)["run"]>>> => {
+    const job: Job<ReturnType<Action["run"]>> = {
+      plugins: [],
+      pluginOptions: {},
+
+      perform: async function (params: ActionParams<typeof action>) {
+        const connection = new Connection(
+          "resque",
+          `job:${api.process.name}:${SERVER_JOB_COUNTER++}}`,
+        );
+        const paramsAsFormData = new FormData();
+
+        if (typeof params.entries === "function") {
+          for (const [key, value] of params.entries()) {
+            paramsAsFormData.append(key, value);
+          }
+        } else if (typeof params === "object" && params !== null) {
+          for (const [key, value] of Object.entries(params)) {
+            if (value !== undefined && value !== null) {
+              paramsAsFormData.append(key, String(value));
+            }
+          }
+        }
+
+        const fanOutId = params._fanOutId as string | undefined;
+
+        let response: Awaited<ReturnType<(typeof action)["run"]>>;
+        let error: TypedError | undefined;
+        try {
+          const payload = await connection.act(action.name, paramsAsFormData);
+          response = payload.response;
+          error = payload.error;
+
+          if (error) throw error;
+        } catch (e) {
+          // Collect fan-out error before re-throwing
+          if (fanOutId) {
+            const metaKey = `fanout:${fanOutId}`;
+            const errorsKey = `fanout:${fanOutId}:errors`;
+            const errorMessage = e instanceof Error ? e.message : String(e);
+            await api.redis.redis.rpush(
+              errorsKey,
+              JSON.stringify({ params, error: errorMessage }),
+            );
+            await api.redis.redis.hincrby(metaKey, "failed", 1);
+            // Refresh TTL on all fan-out keys
+            const ttl = await api.redis.redis.ttl(metaKey);
+            if (ttl > 0) {
+              await api.redis.redis.expire(metaKey, ttl);
+              await api.redis.redis.expire(`fanout:${fanOutId}:results`, ttl);
+              await api.redis.redis.expire(errorsKey, ttl);
+            }
+          }
+          throw e;
+        } finally {
+          if (
+            action.task &&
+            action.task.frequency &&
+            action.task.frequency > 0
+          ) {
+            await api.actions.enqueueRecurrent(action);
+          }
+        }
+
+        // Collect fan-out result on success
+        if (fanOutId) {
+          const metaKey = `fanout:${fanOutId}`;
+          const resultsKey = `fanout:${fanOutId}:results`;
+          await api.redis.redis.rpush(
+            resultsKey,
+            JSON.stringify({ params, result: response }),
+          );
+          await api.redis.redis.hincrby(metaKey, "completed", 1);
+          // Refresh TTL on all fan-out keys
+          const ttl = await api.redis.redis.ttl(metaKey);
+          if (ttl > 0) {
+            await api.redis.redis.expire(metaKey, ttl);
+            await api.redis.redis.expire(resultsKey, ttl);
+            await api.redis.redis.expire(`fanout:${fanOutId}:errors`, ttl);
+          }
+        }
+
+        return response;
+      },
+    };
+
+    if (action.task && action.task.frequency && action.task.frequency > 0) {
+      job.plugins!.push("JobLock");
+      job.pluginOptions!.JobLock = { reEnqueue: false };
+      job.plugins!.push("QueueLock");
+      job.plugins!.push("DelayQueueLock");
+    }
+
+    return job;
+  };
+
+  async initialize() {
+    const resqueContainer = {
+      jobs: await this.loadJobs(),
+      workers: [] as Worker[],
+      startQueue: this.startQueue,
+      stopQueue: this.stopQueue,
+      startScheduler: this.startScheduler,
+      stopScheduler: this.stopScheduler,
+      startWorkers: this.startWorkers,
+      stopWorkers: this.stopWorkers,
+      wrapActionAsJob: this.wrapActionAsJob,
+    } as {
+      queue: Queue;
+      scheduler: Scheduler;
+      workers: Worker[];
+      jobs: Awaited<ReturnType<Resque["loadJobs"]>>;
+      startQueue: () => Promise<void>;
+      stopQueue: () => Promise<void>;
+      startScheduler: () => Promise<void>;
+      stopScheduler: () => Promise<void>;
+      startWorkers: () => Promise<void>;
+      stopWorkers: () => Promise<void>;
+      wrapActionAsJob: (action: Action) => Job<any>;
+    };
+
+    return resqueContainer;
+  }
+
+  async start() {
+    await this.startQueue();
+
+    if (api.runMode === RUN_MODE.SERVER) {
+      await this.startScheduler();
+      await this.startWorkers();
+    }
+  }
+
+  async stop() {
+    if (api.runMode === RUN_MODE.SERVER) {
+      await this.stopWorkers();
+      await this.stopScheduler();
+    }
+
+    await this.stopQueue();
+  }
+}

package/initializers/servers.ts

@@ -0,0 +1,66 @@
+import path from "path";
+import { api, RUN_MODE } from "../api";
+import { Initializer } from "../classes/Initializer";
+import type { Server } from "../classes/Server";
+import { globLoader } from "../util/glob";
+
+const namespace = "servers";
+
+declare module "../classes/API" {
+  export interface API {
+    [namespace]: Awaited<ReturnType<Servers["initialize"]>>;
+  }
+}
+
+export class Servers extends Initializer {
+  constructor() {
+    super(namespace);
+    this.loadPriority = 800;
+    this.startPriority = 550;
+    this.stopPriority = 100;
+    this.runModes = [RUN_MODE.SERVER];
+  }
+
+  async initialize() {
+    // Load framework servers from the package directory
+    const frameworkServers = await globLoader<Server<any>>(
+      path.join(api.packageDir, "servers"),
+    );
+
+    // Load user project servers (if rootDir differs from packageDir)
+    let userServers: Server<any>[] = [];
+    if (api.rootDir !== api.packageDir) {
+      try {
+        userServers = await globLoader<Server<any>>(
+          path.join(api.rootDir, "servers"),
+        );
+      } catch {
+        // user project may not have servers, that's fine
+      }
+    }
+
+    const servers = [...frameworkServers, ...userServers];
+
+    for (const server of servers) {
+      await server.initialize();
+    }
+
+    return { servers };
+  }
+
+  async start() {
+    const { servers } = api[namespace];
+
+    for (const server of servers) {
+      await server.start();
+    }
+  }
+
+  async stop() {
+    const { servers } = api[namespace];
+
+    for (const server of servers) {
+      await server.stop();
+    }
+  }
+}

package/initializers/session.ts

@@ -0,0 +1,84 @@
+import { api, Connection } from "../api";
+import { Initializer } from "../classes/Initializer";
+import { config } from "../config";
+
+const namespace = "session";
+const prefix = `${namespace}` as const;
+
+export interface SessionData<
+  T extends Record<string, any> = Record<string, any>,
+> {
+  id: string;
+  cookieName: typeof config.session.cookieName;
+  createdAt: number;
+  data: T;
+}
+
+function getKey(connectionId: Connection["id"]) {
+  return `${prefix}:${connectionId}`;
+}
+
+async function load<T extends Record<string, any>>(connection: Connection) {
+  const key = getKey(connection.id);
+  const data = await api.redis.redis.get(key);
+  if (!data) return null;
+  await api.redis.redis.expire(key, config.session.ttl);
+  return JSON.parse(data) as SessionData<T>;
+}
+
+async function create<T extends Record<string, any>>(
+  connection: Connection,
+  data = {} as T,
+) {
+  const key = getKey(connection.id);
+
+  const sessionData: SessionData<T> = {
+    id: connection.id,
+    cookieName: config.session.cookieName,
+    createdAt: new Date().getTime(),
+    data,
+  };
+
+  await api.redis.redis.set(key, JSON.stringify(sessionData));
+  await api.redis.redis.expire(key, config.session.ttl);
+  return sessionData;
+}
+
+async function update<T extends Record<string, any>>(
+  session: SessionData<T>,
+  data: Record<string, any>,
+) {
+  const key = getKey(session.id);
+  session.data = { ...session.data, ...data };
+  await api.redis.redis.set(key, JSON.stringify(session));
+  await api.redis.redis.expire(key, config.session.ttl);
+  return session.data;
+}
+
+async function destroy(connection: Connection) {
+  const key = getKey(connection.id);
+  const response = await api.redis.redis.del(key);
+  return response > 0;
+}
+
+declare module "../classes/API" {
+  export interface API {
+    [namespace]: Awaited<ReturnType<Session["initialize"]>>;
+  }
+}
+
+export class Session extends Initializer {
+  constructor() {
+    super(namespace);
+    this.startPriority = 600;
+  }
+
+  async initialize() {
+    return {
+      load,
+      create,
+      update,
+      destroy,
+    };
+  }
+}