@nicnocquee/dataqueue 1.33.0 → 1.35.0-beta.20260224075710

package/src/index.ts

@@ -1,9 +1,13 @@
+import { EventEmitter } from 'node:events';
 import { createProcessor } from './processor.js';
+import { createSupervisor } from './supervisor.js';
 import {
   JobQueueConfig,
   JobQueue,
   JobOptions,
+  AddJobOptions,
   ProcessorOptions,
+  SupervisorOptions,
   JobHandlers,
   JobType,
   PostgresJobQueueConfig,
@@ -11,6 +15,9 @@ import {
   CronScheduleOptions,
   CronScheduleStatus,
   EditCronScheduleOptions,
+  QueueEventMap,
+  QueueEventName,
+  QueueEmitFn,
 } from './types.js';
 import { QueueBackend, CronScheduleInput } from './backend.js';
 import { setLogContext } from './log-context.js';
@@ -23,6 +30,8 @@ import { getNextCronOccurrence, validateCronExpression } from './cron.js';
  * Initialize the job queue system.
  *
  * Defaults to PostgreSQL when `backend` is omitted.
+ * For PostgreSQL, provide either `databaseConfig` or `pool` (bring your own).
+ * For Redis, provide either `redisConfig` or `client` (bring your own).
  */
 export const initJobQueue = <PayloadMap = any>(
   config: JobQueueConfig,
@@ -34,15 +43,39 @@ export const initJobQueue = <PayloadMap = any>(
 
   if (backendType === 'postgres') {
     const pgConfig = config as PostgresJobQueueConfig;
-    const pool = createPool(pgConfig.databaseConfig);
-    backend = new PostgresBackend(pool);
+    if (pgConfig.pool) {
+      backend = new PostgresBackend(pgConfig.pool);
+    } else if (pgConfig.databaseConfig) {
+      const pool = createPool(pgConfig.databaseConfig);
+      backend = new PostgresBackend(pool);
+    } else {
+      throw new Error(
+        'PostgreSQL backend requires either "databaseConfig" or "pool" to be provided.',
+      );
+    }
   } else if (backendType === 'redis') {
-    const redisConfig = (config as RedisJobQueueConfig).redisConfig;
-    backend = new RedisBackend(redisConfig);
+    const redisConfig = config as RedisJobQueueConfig;
+    if (redisConfig.client) {
+      backend = new RedisBackend(
+        redisConfig.client as any,
+        redisConfig.keyPrefix,
+      );
+    } else if (redisConfig.redisConfig) {
+      backend = new RedisBackend(redisConfig.redisConfig);
+    } else {
+      throw new Error(
+        'Redis backend requires either "redisConfig" or "client" to be provided.',
+      );
+    }
   } else {
     throw new Error(`Unknown backend: ${backendType}`);
   }
 
+  const emitter = new EventEmitter();
+  const emit: QueueEmitFn = (event, data) => {
+    emitter.emit(event, data);
+  };
+
   /**
    * Enqueue due cron jobs. Shared by the public API and the processor hook.
    */
@@ -84,6 +117,9 @@ export const initJobQueue = <PayloadMap = any>(
         timeoutMs: schedule.timeoutMs ?? undefined,
         forceKillOnTimeout: schedule.forceKillOnTimeout,
         tags: schedule.tags,
+        retryDelay: schedule.retryDelay ?? undefined,
+        retryBackoff: schedule.retryBackoff ?? undefined,
+        retryDelayMax: schedule.retryDelayMax ?? undefined,
       });
 
       // Advance to next occurrence
@@ -107,8 +143,21 @@ export const initJobQueue = <PayloadMap = any>(
   return {
     // Job queue operations
     addJob: withLogContext(
-      (job: JobOptions<PayloadMap, any>) =>
-        backend.addJob<PayloadMap, any>(job),
+      async (job: JobOptions<PayloadMap, any>, options?: AddJobOptions) => {
+        const jobId = await backend.addJob<PayloadMap, any>(job, options);
+        emit('job:added', { jobId, jobType: job.jobType });
+        return jobId;
+      },
+      config.verbose ?? false,
+    ),
+    addJobs: withLogContext(
+      async (jobs: JobOptions<PayloadMap, any>[], options?: AddJobOptions) => {
+        const jobIds = await backend.addJobs<PayloadMap, any>(jobs, options);
+        for (let i = 0; i < jobIds.length; i++) {
+          emit('job:added', { jobId: jobIds[i], jobType: jobs[i].jobType });
+        }
+        return jobIds;
+      },
       config.verbose ?? false,
     ),
     getJob: withLogContext(
@@ -140,15 +189,18 @@ export const initJobQueue = <PayloadMap = any>(
       ) => backend.getJobs<PayloadMap, any>(filters, limit, offset),
       config.verbose ?? false,
     ),
-    retryJob: (jobId: number) => backend.retryJob(jobId),
+    retryJob: async (jobId: number) => {
+      await backend.retryJob(jobId);
+      emit('job:retried', { jobId });
+    },
     cleanupOldJobs: (daysToKeep?: number, batchSize?: number) =>
       backend.cleanupOldJobs(daysToKeep, batchSize),
     cleanupOldJobEvents: (daysToKeep?: number, batchSize?: number) =>
       backend.cleanupOldJobEvents(daysToKeep, batchSize),
-    cancelJob: withLogContext(
-      (jobId: number) => backend.cancelJob(jobId),
-      config.verbose ?? false,
-    ),
+    cancelJob: withLogContext(async (jobId: number) => {
+      await backend.cancelJob(jobId);
+      emit('job:cancelled', { jobId });
+    }, config.verbose ?? false),
     editJob: withLogContext(
       <T extends JobType<PayloadMap>>(
         jobId: number,
@@ -206,9 +258,19 @@ export const initJobQueue = <PayloadMap = any>(
       handlers: JobHandlers<PayloadMap>,
       options?: ProcessorOptions,
     ) =>
-      createProcessor<PayloadMap>(backend, handlers, options, async () => {
-        await enqueueDueCronJobsImpl();
-      }),
+      createProcessor<PayloadMap>(
+        backend,
+        handlers,
+        options,
+        async () => {
+          await enqueueDueCronJobsImpl();
+        },
+        emit,
+      ),
+
+    // Background supervisor — automated maintenance
+    createSupervisor: (options?: SupervisorOptions) =>
+      createSupervisor(backend, options, emit),
 
     // Job events
     getJobEvents: withLogContext(
@@ -262,6 +324,9 @@ export const initJobQueue = <PayloadMap = any>(
           timezone: options.timezone ?? 'UTC',
           allowOverlap: options.allowOverlap ?? false,
           nextRunAt,
+          retryDelay: options.retryDelay ?? null,
+          retryBackoff: options.retryBackoff ?? null,
+          retryDelayMax: options.retryDelayMax ?? null,
         };
         return backend.addCronSchedule(input);
       },
@@ -320,6 +385,33 @@ export const initJobQueue = <PayloadMap = any>(
       config.verbose ?? false,
     ),
 
+    // Event hooks
+    on: <K extends QueueEventName>(
+      event: K,
+      listener: (data: QueueEventMap[K]) => void,
+    ) => {
+      emitter.on(event, listener as (...args: any[]) => void);
+    },
+    once: <K extends QueueEventName>(
+      event: K,
+      listener: (data: QueueEventMap[K]) => void,
+    ) => {
+      emitter.once(event, listener as (...args: any[]) => void);
+    },
+    off: <K extends QueueEventName>(
+      event: K,
+      listener: (data: QueueEventMap[K]) => void,
+    ) => {
+      emitter.off(event, listener as (...args: any[]) => void);
+    },
+    removeAllListeners: (event?: QueueEventName) => {
+      if (event) {
+        emitter.removeAllListeners(event);
+      } else {
+        emitter.removeAllListeners();
+      }
+    },
+
     // Advanced access
     getPool: () => {
       if (!(backend instanceof PostgresBackend)) {

package/src/install-mcp-command.test.ts

@@ -0,0 +1,216 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import {
+  runInstallMcp,
+  upsertMcpConfig,
+  InstallMcpDeps,
+} from './install-mcp-command.js';
+
+describe('upsertMcpConfig', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('creates new config file when it does not exist', () => {
+    // Setup
+    const deps = {
+      existsSync: vi.fn(() => false),
+      readFileSync: vi.fn(),
+      writeFileSync: vi.fn(),
+    };
+
+    // Act
+    upsertMcpConfig(
+      '/path/mcp.json',
+      'dataqueue',
+      { command: 'npx', args: ['dataqueue-cli', 'mcp'] },
+      deps,
+    );
+
+    // Assert
+    const written = JSON.parse(deps.writeFileSync.mock.calls[0][1] as string);
+    expect(written.mcpServers.dataqueue).toEqual({
+      command: 'npx',
+      args: ['dataqueue-cli', 'mcp'],
+    });
+  });
+
+  it('adds to existing config without overwriting other servers', () => {
+    // Setup
+    const existing = JSON.stringify({
+      mcpServers: { other: { command: 'other' } },
+    });
+    const deps = {
+      existsSync: vi.fn(() => true),
+      readFileSync: vi.fn(() => existing),
+      writeFileSync: vi.fn(),
+    };
+
+    // Act
+    upsertMcpConfig(
+      '/path/mcp.json',
+      'dataqueue',
+      { command: 'npx', args: ['dataqueue-cli', 'mcp'] },
+      deps,
+    );
+
+    // Assert
+    const written = JSON.parse(deps.writeFileSync.mock.calls[0][1] as string);
+    expect(written.mcpServers.other).toEqual({ command: 'other' });
+    expect(written.mcpServers.dataqueue).toEqual({
+      command: 'npx',
+      args: ['dataqueue-cli', 'mcp'],
+    });
+  });
+
+  it('overwrites existing dataqueue entry', () => {
+    // Setup
+    const existing = JSON.stringify({
+      mcpServers: { dataqueue: { command: 'old' } },
+    });
+    const deps = {
+      existsSync: vi.fn(() => true),
+      readFileSync: vi.fn(() => existing),
+      writeFileSync: vi.fn(),
+    };
+
+    // Act
+    upsertMcpConfig('/path/mcp.json', 'dataqueue', { command: 'new' }, deps);
+
+    // Assert
+    const written = JSON.parse(deps.writeFileSync.mock.calls[0][1] as string);
+    expect(written.mcpServers.dataqueue).toEqual({ command: 'new' });
+  });
+
+  it('handles malformed JSON in existing file', () => {
+    // Setup
+    const deps = {
+      existsSync: vi.fn(() => true),
+      readFileSync: vi.fn(() => 'not json'),
+      writeFileSync: vi.fn(),
+    };
+
+    // Act
+    upsertMcpConfig('/path/mcp.json', 'dataqueue', { command: 'npx' }, deps);
+
+    // Assert
+    const written = JSON.parse(deps.writeFileSync.mock.calls[0][1] as string);
+    expect(written.mcpServers.dataqueue).toEqual({ command: 'npx' });
+  });
+});
+
+describe('runInstallMcp', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  function makeDeps(overrides: Partial<InstallMcpDeps> = {}): InstallMcpDeps {
+    return {
+      log: vi.fn(),
+      error: vi.fn(),
+      exit: vi.fn(),
+      cwd: '/project',
+      readFileSync: vi.fn(() => '{}'),
+      writeFileSync: vi.fn(),
+      mkdirSync: vi.fn(),
+      existsSync: vi.fn(() => false),
+      ...overrides,
+    };
+  }
+
+  it('installs MCP config for Cursor (option 1)', async () => {
+    // Setup
+    const deps = makeDeps({ selectedClient: '1' });
+
+    // Act
+    await runInstallMcp(deps);
+
+    // Assert
+    expect(deps.mkdirSync).toHaveBeenCalledWith('/project/.cursor', {
+      recursive: true,
+    });
+    const written = JSON.parse(
+      (deps.writeFileSync as ReturnType<typeof vi.fn>).mock
+        .calls[0][1] as string,
+    );
+    expect(written.mcpServers.dataqueue.command).toBe('npx');
+    expect(written.mcpServers.dataqueue.args).toEqual(['dataqueue-cli', 'mcp']);
+  });
+
+  it('installs MCP config for Claude Code (option 2)', async () => {
+    // Setup
+    const deps = makeDeps({ selectedClient: '2' });
+
+    // Act
+    await runInstallMcp(deps);
+
+    // Assert
+    expect(deps.writeFileSync).toHaveBeenCalledWith(
+      '/project/.mcp.json',
+      expect.any(String),
+    );
+  });
+
+  it('installs MCP config for VS Code (option 3)', async () => {
+    // Setup
+    const deps = makeDeps({ selectedClient: '3' });
+
+    // Act
+    await runInstallMcp(deps);
+
+    // Assert
+    expect(deps.mkdirSync).toHaveBeenCalledWith('/project/.vscode', {
+      recursive: true,
+    });
+    expect(deps.writeFileSync).toHaveBeenCalledWith(
+      expect.stringContaining('.vscode/mcp.json'),
+      expect.any(String),
+    );
+  });
+
+  it('exits with error for invalid choice', async () => {
+    // Setup
+    const deps = makeDeps({ selectedClient: '99' });
+
+    // Act
+    await runInstallMcp(deps);
+
+    // Assert
+    expect(deps.error).toHaveBeenCalledWith(
+      expect.stringContaining('Invalid choice'),
+    );
+    expect(deps.exit).toHaveBeenCalledWith(1);
+  });
+
+  it('handles install errors', async () => {
+    // Setup
+    const deps = makeDeps({
+      selectedClient: '1',
+      writeFileSync: vi.fn(() => {
+        throw new Error('permission denied');
+      }),
+    });
+
+    // Act
+    await runInstallMcp(deps);
+
+    // Assert
+    expect(deps.error).toHaveBeenCalledWith(
+      'Failed to install MCP config:',
+      expect.any(Error),
+    );
+    expect(deps.exit).toHaveBeenCalledWith(1);
+  });
+
+  it('logs done message on success', async () => {
+    // Setup
+    const deps = makeDeps({ selectedClient: '1' });
+
+    // Act
+    await runInstallMcp(deps);
+
+    // Assert
+    expect(deps.log).toHaveBeenCalledWith(
+      expect.stringContaining('npx dataqueue-cli mcp'),
+    );
+  });
+});

package/src/install-mcp-command.ts

@@ -0,0 +1,185 @@
+import fs from 'fs';
+import path from 'path';
+import readline from 'readline';
+
+export interface InstallMcpDeps {
+  log?: (...args: unknown[]) => void;
+  error?: (...args: unknown[]) => void;
+  exit?: (code: number) => void;
+  cwd?: string;
+  readFileSync?: (p: string, enc: BufferEncoding) => string;
+  writeFileSync?: (p: string, data: string) => void;
+  mkdirSync?: (p: string, opts?: fs.MakeDirectoryOptions) => void;
+  existsSync?: (p: string) => boolean;
+  /** Override for selecting the client (skips interactive prompt). */
+  selectedClient?: string;
+}
+
+interface McpClientConfig {
+  label: string;
+  install: (
+    deps: Required<
+      Pick<
+        InstallMcpDeps,
+        | 'cwd'
+        | 'readFileSync'
+        | 'writeFileSync'
+        | 'mkdirSync'
+        | 'existsSync'
+        | 'log'
+      >
+    >,
+  ) => void;
+}
+
+/**
+ * Merges the dataqueue MCP server config into an existing JSON config file.
+ *
+ * @param filePath - Path to the MCP config file.
+ * @param serverKey - Key name for the server entry.
+ * @param serverConfig - Server configuration object.
+ * @param deps - Injectable file system functions.
+ */
+export function upsertMcpConfig(
+  filePath: string,
+  serverKey: string,
+  serverConfig: Record<string, unknown>,
+  deps: {
+    readFileSync: (p: string, enc: BufferEncoding) => string;
+    writeFileSync: (p: string, data: string) => void;
+    existsSync: (p: string) => boolean;
+  },
+): void {
+  let config: Record<string, unknown> = {};
+
+  if (deps.existsSync(filePath)) {
+    try {
+      config = JSON.parse(deps.readFileSync(filePath, 'utf-8'));
+    } catch {
+      config = {};
+    }
+  }
+
+  if (!config.mcpServers || typeof config.mcpServers !== 'object') {
+    config.mcpServers = {};
+  }
+
+  (config.mcpServers as Record<string, unknown>)[serverKey] = serverConfig;
+  deps.writeFileSync(filePath, JSON.stringify(config, null, 2) + '\n');
+}
+
+const MCP_SERVER_CONFIG = {
+  command: 'npx',
+  args: ['dataqueue-cli', 'mcp'],
+};
+
+const MCP_CLIENTS: Record<string, McpClientConfig> = {
+  '1': {
+    label: 'Cursor',
+    install: (deps) => {
+      const configDir = path.join(deps.cwd, '.cursor');
+      deps.mkdirSync(configDir, { recursive: true });
+      const configFile = path.join(configDir, 'mcp.json');
+      upsertMcpConfig(configFile, 'dataqueue', MCP_SERVER_CONFIG, deps);
+      deps.log(`  ✓ .cursor/mcp.json`);
+    },
+  },
+  '2': {
+    label: 'Claude Code',
+    install: (deps) => {
+      const configFile = path.join(deps.cwd, '.mcp.json');
+      upsertMcpConfig(configFile, 'dataqueue', MCP_SERVER_CONFIG, deps);
+      deps.log(`  ✓ .mcp.json`);
+    },
+  },
+  '3': {
+    label: 'VS Code (Copilot)',
+    install: (deps) => {
+      const configDir = path.join(deps.cwd, '.vscode');
+      deps.mkdirSync(configDir, { recursive: true });
+      const configFile = path.join(configDir, 'mcp.json');
+      upsertMcpConfig(configFile, 'dataqueue', MCP_SERVER_CONFIG, deps);
+      deps.log(`  ✓ .vscode/mcp.json`);
+    },
+  },
+  '4': {
+    label: 'Windsurf',
+    install: (deps) => {
+      const homeDir = process.env.HOME || process.env.USERPROFILE || '';
+      const configFile = path.join(
+        homeDir,
+        '.codeium',
+        'windsurf',
+        'mcp_config.json',
+      );
+      deps.mkdirSync(path.dirname(configFile), { recursive: true });
+      upsertMcpConfig(configFile, 'dataqueue', MCP_SERVER_CONFIG, deps);
+      deps.log(`  ✓ ~/.codeium/windsurf/mcp_config.json`);
+    },
+  },
+};
+
+/**
+ * Installs the DataQueue MCP server config for the selected AI client.
+ *
+ * @param deps - Injectable dependencies for testing.
+ */
+export async function runInstallMcp({
+  log = console.log,
+  error = console.error,
+  exit = (code: number) => process.exit(code),
+  cwd = process.cwd(),
+  readFileSync = fs.readFileSync,
+  writeFileSync = fs.writeFileSync,
+  mkdirSync = fs.mkdirSync,
+  existsSync = fs.existsSync,
+  selectedClient,
+}: InstallMcpDeps = {}): Promise<void> {
+  log('DataQueue MCP Server Installer\n');
+  log('Select your AI client:\n');
+
+  for (const [key, client] of Object.entries(MCP_CLIENTS)) {
+    log(`  ${key}) ${client.label}`);
+  }
+  log('');
+
+  let choice = selectedClient;
+
+  if (!choice) {
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+
+    choice = await new Promise<string>((resolve) => {
+      rl.question('Enter choice (1-4): ', (answer) => {
+        rl.close();
+        resolve(answer.trim());
+      });
+    });
+  }
+
+  const client = MCP_CLIENTS[choice];
+  if (!client) {
+    error(`Invalid choice: "${choice}". Expected 1-4.`);
+    exit(1);
+    return;
+  }
+
+  log(`\nInstalling MCP config for ${client.label}...`);
+
+  try {
+    client.install({
+      cwd,
+      readFileSync,
+      writeFileSync,
+      mkdirSync,
+      existsSync,
+      log,
+    });
+    log('\nDone! The MCP server will run via: npx dataqueue-cli mcp');
+  } catch (err) {
+    error('Failed to install MCP config:', err);
+    exit(1);
+  }
+}