express-file-cluster 0.1.0

package/src/tasks/bullmq-backend.ts

@@ -0,0 +1,76 @@
+import type { Job } from 'bullmq';
+import { taskRegistry, setEnqueueImpl } from './index.js';
+import { runInThread } from './thread-runner.js';
+
+interface BullMQOpts {
+  redisUrl: string;
+  concurrency: number;
+}
+
+export async function initBullMQ(opts: BullMQOpts): Promise<void> {
+  let bullmq: typeof import('bullmq');
+  try {
+    bullmq = await import('bullmq');
+  } catch {
+    throw new Error('[EFC] bullmq is not installed. Run: npm install bullmq');
+  }
+
+  const connection = parseRedisUrl(opts.redisUrl);
+  const queue = new bullmq.Queue('efc', { connection });
+
+  const worker = new bullmq.Worker(
+    'efc',
+    async (job: Job) => {
+      const def = taskRegistry.get(job.name);
+      if (!def) {
+        throw new Error(`[EFC] No task registered for name: ${job.name}`);
+      }
+
+      if (def.options.thread && def.filePath) {
+        await runInThread(def.filePath, job.data);
+      } else {
+        await def.handler(job.data);
+      }
+    },
+    { connection, concurrency: opts.concurrency },
+  );
+
+  worker.on('failed', (job: Job | undefined, err: Error) => {
+    console.error(`[EFC] Task ${job?.name ?? 'unknown'} failed:`, err.message);
+  });
+
+  worker.on('completed', (job: Job) => {
+    console.log(`[EFC] Task ${job.name} completed (id=${job.id})`);
+  });
+
+  setEnqueueImpl(async (name, payload) => {
+    const def = taskRegistry.get(name);
+    if (!def) {
+      throw new Error(`[EFC] Cannot enqueue unknown task: "${name}". Is the task file in tasksDir?`);
+    }
+
+    await queue.add(name, payload, {
+      attempts: def.options.retries ?? 3,
+      backoff: {
+        type: def.options.backoff ?? 'exponential',
+        delay: 1000,
+      },
+    });
+  });
+
+  console.log('[EFC] BullMQ backend initialised');
+}
+
+function parseRedisUrl(url: string): { host: string; port: number; password?: string } {
+  try {
+    const u = new URL(url);
+    const result: { host: string; port: number; password?: string } = {
+      host: u.hostname || 'localhost',
+      port: parseInt(u.port || '6379', 10),
+    };
+    if (u.password) result.password = u.password;
+    return result;
+  } catch {
+    return { host: 'localhost', port: 6379 };
+  }
+}

package/src/tasks/index.ts

@@ -0,0 +1,51 @@
+import type { TaskDefinition, TaskOptions } from '../types.js';
+
+export const taskRegistry = new Map<string, TaskDefinition>();
+
+type HandlerFn<T> = (payload: T) => Promise<void>;
+type DefineTaskOverload = {
+  <T>(handler: HandlerFn<T>): TaskDefinition<T>;
+  <T>(options: TaskOptions, handler: HandlerFn<T>): TaskDefinition<T>;
+};
+
+export const defineTask: DefineTaskOverload = <T>(
+  handlerOrOptions: HandlerFn<T> | TaskOptions,
+  maybeHandler?: HandlerFn<T>,
+): TaskDefinition<T> => {
+  let options: TaskOptions = {};
+  let handler: HandlerFn<T>;
+
+  if (typeof handlerOrOptions === 'function') {
+    handler = handlerOrOptions;
+  } else {
+    options = handlerOrOptions;
+    if (!maybeHandler) throw new Error('[EFC] defineTask: handler function is required');
+    handler = maybeHandler;
+  }
+
+  return {
+    handler: handler as (payload: unknown) => Promise<void>,
+    options,
+    name: '',
+  };
+};
+
+type EnqueueImpl = (name: string, payload: unknown) => Promise<void>;
+let _impl: EnqueueImpl | null = null;
+
+export function setEnqueueImpl(impl: EnqueueImpl): void {
+  _impl = impl;
+}
+
+export async function enqueue<T>(name: string, payload: T): Promise<void> {
+  if (!_impl) {
+    throw new Error(
+      `[EFC] Task queue not initialised. Set tasks.backend in ignite() config.`,
+    );
+  }
+  return _impl(name, payload as unknown);
+}
+
+export function registerTask(name: string, def: TaskDefinition): void {
+  taskRegistry.set(name, { ...def, name });
+}

package/src/tasks/scanner.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { scanTasks } from './scanner.js';
+import { taskRegistry } from './index.js';
+
+describe('scanTasks', () => {
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'efc-test-'));
+    taskRegistry.clear();
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('returns silently when directory does not exist', async () => {
+    await expect(scanTasks('/nonexistent/path/xyz')).resolves.toBeUndefined();
+    expect(taskRegistry.size).toBe(0);
+  });
+
+  it('returns silently for an empty directory', async () => {
+    await scanTasks(tmpDir);
+    expect(taskRegistry.size).toBe(0);
+  });
+
+  it('registers a valid task from a .js file', async () => {
+    const taskFile = path.join(tmpDir, 'MyTask.js');
+    fs.writeFileSync(
+      taskFile,
+      `export default { handler: async (p) => {}, options: {}, name: '' };`,
+    );
+
+    await scanTasks(tmpDir);
+    expect(taskRegistry.has('MyTask')).toBe(true);
+    const def = taskRegistry.get('MyTask');
+    expect(typeof def?.handler).toBe('function');
+  });
+
+  it('skips files with no valid default export and logs a warning', async () => {
+    const taskFile = path.join(tmpDir, 'BadTask.js');
+    fs.writeFileSync(taskFile, `export const foo = 42;`);
+
+    await scanTasks(tmpDir);
+    expect(taskRegistry.has('BadTask')).toBe(false);
+  });
+
+  it('skips unimportable files gracefully', async () => {
+    const taskFile = path.join(tmpDir, 'BrokenTask.js');
+    fs.writeFileSync(taskFile, `this is not valid javascript !!!`);
+
+    await expect(scanTasks(tmpDir)).resolves.toBeUndefined();
+    expect(taskRegistry.has('BrokenTask')).toBe(false);
+  });
+});

package/src/tasks/scanner.ts

@@ -0,0 +1,30 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { registerTask } from './index.js';
+import type { TaskDefinition } from '../types.js';
+
+export async function scanTasks(tasksDir: string): Promise<void> {
+  if (!fs.existsSync(tasksDir)) return;
+
+  const files = fs.readdirSync(tasksDir).filter((f) => /\.(ts|js|mts|mjs)$/.test(f));
+
+  for (const file of files) {
+    const filePath = path.join(tasksDir, file);
+    const taskName = path.basename(file, path.extname(file));
+
+    try {
+      const mod = await import(filePath) as Record<string, unknown>;
+      const def = mod['default'] as TaskDefinition | undefined;
+
+      if (!def || typeof def.handler !== 'function') {
+        console.warn(`[EFC] Task file ${file} has no valid default export — skipping`);
+        continue;
+      }
+
+      registerTask(taskName, { ...def, filePath });
+      console.log(`[EFC] Registered task: ${taskName}`);
+    } catch (err) {
+      console.warn(`[EFC] Failed to load task ${file}:`, err);
+    }
+  }
+}

package/src/tasks/thread-runner.ts

@@ -0,0 +1,40 @@
+import { Worker, workerData, parentPort, isMainThread } from 'node:worker_threads';
+import type { TaskDefinition } from '../types.js';
+
+// ── Worker-thread side ────────────────────────────────────────────────────────
+if (!isMainThread) {
+  const { handlerPath, payload } = workerData as { handlerPath: string; payload: unknown };
+
+  (async () => {
+    try {
+      const mod = await import(handlerPath) as Record<string, unknown>;
+      const def = mod['default'] as TaskDefinition | undefined;
+      if (!def || typeof def.handler !== 'function') {
+        throw new Error(`[EFC] Thread runner: no valid task export in ${handlerPath}`);
+      }
+      await def.handler(payload);
+      parentPort?.postMessage({ ok: true });
+    } catch (err) {
+      parentPort?.postMessage({ ok: false, error: String(err) });
+    }
+  })();
+}
+
+// ── Main-thread side ──────────────────────────────────────────────────────────
+export function runInThread(handlerPath: string, payload: unknown): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(new URL('./thread-runner.js', import.meta.url), {
+      workerData: { handlerPath, payload },
+    });
+
+    worker.on('message', (msg: { ok: boolean; error?: string }) => {
+      if (msg.ok) resolve();
+      else reject(new Error(msg.error ?? 'Thread task failed'));
+    });
+
+    worker.on('error', reject);
+    worker.on('exit', (code) => {
+      if (code !== 0) reject(new Error(`Thread worker exited with code ${code}`));
+    });
+  });
+}

package/src/types.ts

@@ -0,0 +1,68 @@
+import type { RequestHandler, ErrorRequestHandler } from 'express';
+
+export type DatabaseEngine = 'mongodb' | 'postgresql';
+export type AuthStrategy = 'http-only' | 'localStorage';
+export type TaskBackend = 'bullmq' | 'pg-boss';
+
+export interface TaskConfig {
+  backend: TaskBackend;
+  redisUrl?: string;
+  concurrency?: number;
+}
+
+export interface EFCConfig {
+  port?: number;
+  apiDir: string;
+  tasksDir?: string;
+  database?: DatabaseEngine;
+  databaseUrl?: string;
+  authStrategy?: AuthStrategy;
+  jwtSecret?: string;
+  cluster?: boolean;
+  workers?: number;
+  tasks?: TaskConfig | false;
+  globalMiddlewares?: RequestHandler[];
+  onWorkerReady?: (id: number) => void;
+  onWorkerCrash?: (id: number, code: number) => void;
+  onError?: ErrorRequestHandler;
+}
+
+export interface RouteEntry {
+  urlPath: string;
+  filePath: string;
+  params: string[];
+}
+
+export interface TaskOptions {
+  thread?: boolean;
+  retries?: number;
+  backoff?: 'fixed' | 'exponential';
+  concurrency?: number;
+  schedule?: string;
+}
+
+export interface TaskDefinition<TPayload = unknown> {
+  handler: (payload: TPayload) => Promise<void>;
+  options: TaskOptions;
+  name: string;
+  filePath?: string;
+}
+
+export interface FieldDefinition {
+  type: 'string' | 'number' | 'boolean' | 'date' | 'object' | 'array';
+  required?: boolean;
+  unique?: boolean;
+  default?: unknown;
+}
+
+export type ModelSchema = Record<string, FieldDefinition>;
+
+export interface ModelCRUD<T extends Record<string, unknown>> {
+  find(filter?: Partial<T>): Promise<T[]>;
+  findById(id: string): Promise<(T & { id: string }) | null>;
+  findOne(filter: Partial<T>): Promise<(T & { id: string }) | null>;
+  create(data: Partial<T>): Promise<T & { id: string }>;
+  update(id: string, data: Partial<T>): Promise<(T & { id: string }) | null>;
+  delete(id: string): Promise<void>;
+  count(filter?: Partial<T>): Promise<number>;
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsup.config.ts

@@ -0,0 +1,26 @@
+import { defineConfig } from 'tsup';
+
+export default defineConfig({
+  entry: {
+    index: 'src/index.ts',
+    'auth/index': 'src/auth/index.ts',
+    'tasks/index': 'src/tasks/index.ts',
+    'cli/index': 'src/cli/index.ts',
+  },
+  format: ['cjs', 'esm'],
+  dts: true,
+  clean: true,
+  splitting: false,
+  sourcemap: true,
+  target: 'node18',
+  banner: {
+    js: '',
+  },
+  esbuildOptions(options, context) {
+    if (context.format === 'esm') {
+      options.banner = {
+        js: context.entry === 'src/cli/index.ts' ? '#!/usr/bin/env node' : '',
+      };
+    }
+  },
+});