@nicnocquee/dataqueue 1.25.0 → 1.26.0-beta.20260223202259

package/src/mcp-server.ts

@@ -0,0 +1,231 @@
+#!/usr/bin/env node
+
+/**
+ * DataQueue MCP Server — exposes documentation search over stdio.
+ * Run via: dataqueue-cli mcp
+ */
+
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { z } from 'zod';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+interface DocPage {
+  slug: string;
+  title: string;
+  description: string;
+  content: string;
+}
+
+/** @internal Loads docs-content.json from the ai/ directory bundled with the package. */
+export function loadDocsContent(
+  docsPath: string = path.join(__dirname, '../ai/docs-content.json'),
+): DocPage[] {
+  const raw = fs.readFileSync(docsPath, 'utf-8');
+  return JSON.parse(raw) as DocPage[];
+}
+
+/** @internal Scores a doc page against a search query using simple term matching. */
+export function scorePageForQuery(page: DocPage, queryTerms: string[]): number {
+  const titleLower = page.title.toLowerCase();
+  const descLower = page.description.toLowerCase();
+  const contentLower = page.content.toLowerCase();
+
+  let score = 0;
+  for (const term of queryTerms) {
+    if (titleLower.includes(term)) score += 10;
+    if (descLower.includes(term)) score += 5;
+
+    const contentMatches = contentLower.split(term).length - 1;
+    score += Math.min(contentMatches, 10);
+  }
+  return score;
+}
+
+/** @internal Extracts a relevant excerpt around the first match of any query term. */
+export function extractExcerpt(
+  content: string,
+  queryTerms: string[],
+  maxLength = 500,
+): string {
+  const lower = content.toLowerCase();
+  let earliestIndex = -1;
+
+  for (const term of queryTerms) {
+    const idx = lower.indexOf(term);
+    if (idx !== -1 && (earliestIndex === -1 || idx < earliestIndex)) {
+      earliestIndex = idx;
+    }
+  }
+
+  if (earliestIndex === -1) {
+    return content.slice(0, maxLength);
+  }
+
+  const start = Math.max(0, earliestIndex - 100);
+  const end = Math.min(content.length, start + maxLength);
+  let excerpt = content.slice(start, end);
+
+  if (start > 0) excerpt = '...' + excerpt;
+  if (end < content.length) excerpt = excerpt + '...';
+
+  return excerpt;
+}
+
+/**
+ * Creates and starts the DataQueue MCP server over stdio.
+ *
+ * @param deps - Injectable dependencies for testing.
+ */
+export async function startMcpServer(
+  deps: {
+    docsPath?: string;
+    transport?: InstanceType<typeof StdioServerTransport>;
+  } = {},
+): Promise<McpServer> {
+  const pages = loadDocsContent(deps.docsPath);
+
+  const server = new McpServer({
+    name: 'dataqueue-docs',
+    version: '1.0.0',
+  });
+
+  server.resource('llms-txt', 'dataqueue://llms.txt', async () => {
+    const llmsPath = path.join(
+      __dirname,
+      '../ai/skills/dataqueue-core/SKILL.md',
+    );
+    let content: string;
+    try {
+      content = fs.readFileSync(llmsPath, 'utf-8');
+    } catch {
+      content = pages
+        .map((p) => `## ${p.title}\n\nSlug: ${p.slug}\n\n${p.description}`)
+        .join('\n\n');
+    }
+    return { contents: [{ uri: 'dataqueue://llms.txt', text: content }] };
+  });
+
+  server.tool(
+    'list-doc-pages',
+    'List all available DataQueue documentation pages with titles and descriptions.',
+    {},
+    async () => {
+      const listing = pages.map((p) => ({
+        slug: p.slug,
+        title: p.title,
+        description: p.description,
+      }));
+      return {
+        content: [
+          { type: 'text' as const, text: JSON.stringify(listing, null, 2) },
+        ],
+      };
+    },
+  );
+
+  server.tool(
+    'get-doc-page',
+    'Fetch a specific DataQueue doc page by slug. Returns full page content as markdown.',
+    {
+      slug: z
+        .string()
+        .describe('The doc page slug, e.g. "usage/add-job" or "api/job-queue"'),
+    },
+    async ({ slug }) => {
+      const page = pages.find((p) => p.slug === slug);
+      if (!page) {
+        return {
+          content: [
+            {
+              type: 'text' as const,
+              text: `Page not found: "${slug}". Use list-doc-pages to see available slugs.`,
+            },
+          ],
+          isError: true,
+        };
+      }
+      const header = page.description
+        ? `# ${page.title}\n\n> ${page.description}\n\n`
+        : `# ${page.title}\n\n`;
+      return {
+        content: [{ type: 'text' as const, text: header + page.content }],
+      };
+    },
+  );
+
+  server.tool(
+    'search-docs',
+    'Full-text search across all DataQueue documentation pages. Returns matching sections with page titles and content excerpts.',
+    {
+      query: z
+        .string()
+        .describe('Search query, e.g. "cron scheduling" or "waitForToken"'),
+    },
+    async ({ query }) => {
+      const queryTerms = query
+        .toLowerCase()
+        .split(/\s+/)
+        .filter((t) => t.length > 1);
+
+      if (queryTerms.length === 0) {
+        return {
+          content: [
+            { type: 'text' as const, text: 'Please provide a search query.' },
+          ],
+          isError: true,
+        };
+      }
+
+      const scored = pages
+        .map((page) => ({
+          page,
+          score: scorePageForQuery(page, queryTerms),
+        }))
+        .filter((r) => r.score > 0)
+        .sort((a, b) => b.score - a.score)
+        .slice(0, 5);
+
+      if (scored.length === 0) {
+        return {
+          content: [
+            {
+              type: 'text' as const,
+              text: `No results for "${query}". Try different keywords or use list-doc-pages to browse.`,
+            },
+          ],
+        };
+      }
+
+      const results = scored.map((r) => {
+        const excerpt = extractExcerpt(r.page.content, queryTerms);
+        return `## ${r.page.title} (${r.page.slug})\n\n${r.page.description}\n\n${excerpt}`;
+      });
+
+      return {
+        content: [{ type: 'text' as const, text: results.join('\n\n---\n\n') }],
+      };
+    },
+  );
+
+  const transport = deps.transport ?? new StdioServerTransport();
+  await server.connect(transport);
+  return server;
+}
+
+const isDirectRun =
+  process.argv[1] &&
+  (process.argv[1].endsWith('/mcp-server.js') ||
+    process.argv[1].endsWith('/mcp-server.cjs'));
+
+if (isDirectRun) {
+  startMcpServer().catch((err) => {
+    console.error('Failed to start MCP server:', err);
+    process.exit(1);
+  });
+}

package/src/processor.ts

@@ -1,5 +1,4 @@
 import { Worker } from 'worker_threads';
-import { Pool } from 'pg';
 import {
   JobRecord,
   ProcessorOptions,
@@ -13,71 +12,11 @@ import {
   WaitSignal,
   WaitDuration,
   WaitTokenResult,
+  QueueEmitFn,
 } from './types.js';
 import { QueueBackend } from './backend.js';
-import { PostgresBackend } from './backends/postgres.js';
-import {
-  waitJob,
-  updateStepData,
-  createWaitpoint,
-  getWaitpoint,
-} from './queue.js';
 import { log, setLogContext } from './log-context.js';
 
-/**
- * Try to extract the underlying pg Pool from a QueueBackend.
- * Returns null for non-PostgreSQL backends.
- */
-function tryExtractPool(backend: QueueBackend): Pool | null {
-  if (backend instanceof PostgresBackend) {
-    return backend.getPool();
-  }
-  return null;
-}
-
-/**
- * Build a JobContext without wait support (for non-PostgreSQL backends).
- * prolong/onTimeout work normally; wait-related methods throw helpful errors.
- */
-function buildBasicContext(
-  backend: QueueBackend,
-  jobId: number,
-  baseCtx: {
-    prolong: JobContext['prolong'];
-    onTimeout: JobContext['onTimeout'];
-  },
-): JobContext {
-  const waitError = () =>
-    new Error(
-      'Wait features (waitFor, waitUntil, createToken, waitForToken, ctx.run) are currently only supported with the PostgreSQL backend.',
-    );
-  return {
-    prolong: baseCtx.prolong,
-    onTimeout: baseCtx.onTimeout,
-    run: async <T>(_stepName: string, fn: () => Promise<T>): Promise<T> => {
-      // Without PostgreSQL, just execute the function directly (no persistence)
-      return fn();
-    },
-    waitFor: async () => {
-      throw waitError();
-    },
-    waitUntil: async () => {
-      throw waitError();
-    },
-    createToken: async () => {
-      throw waitError();
-    },
-    waitForToken: async () => {
-      throw waitError();
-    },
-    setProgress: async (percent: number) => {
-      if (percent < 0 || percent > 100)
-        throw new Error('Progress must be between 0 and 100');
-      await backend.updateProgress(jobId, Math.round(percent));
-    },
-  };
-}
-
 /**
  * Validates that a handler can be serialized for worker thread execution.
  * Throws an error with helpful message if serialization fails.
@@ -388,7 +327,7 @@ function createNoOpContext(
  * Marks pending waits as completed and fetches token outputs.
  */
 async function resolveCompletedWaits(
-  pool: Pool,
+  backend: QueueBackend,
   stepData: Record<string, any>,
 ): Promise<void> {
   for (const key of Object.keys(stepData)) {
@@ -401,7 +340,7 @@ async function resolveCompletedWaits(
       stepData[key] = { ...entry, completed: true };
     } else if (entry.type === 'token' && entry.tokenId) {
       // Token-based wait -- fetch the waitpoint result
-      const wp = await getWaitpoint(pool, entry.tokenId);
+      const wp = await backend.getWaitpoint(entry.tokenId);
       if (wp && wp.status === 'completed') {
         stepData[key] = {
           ...entry,
@@ -422,10 +361,10 @@ async function resolveCompletedWaits(
 
 /**
  * Build the extended JobContext with step tracking and wait support.
+ * Works with any QueueBackend (Postgres or Redis).
  */
 function buildWaitContext(
   backend: QueueBackend,
-  pool: Pool,
   jobId: number,
   stepData: Record<string, any>,
   baseCtx: {
@@ -455,7 +394,7 @@ function buildWaitContext(
 
       // Persist step result
       stepData[stepName] = { __completed: true, result };
-      await updateStepData(pool, jobId, stepData);
+      await backend.updateStepData(jobId, stepData);
 
       return result;
     },
@@ -498,7 +437,7 @@ function buildWaitContext(
     },
 
     createToken: async (options?) => {
-      const token = await createWaitpoint(pool, jobId, options);
+      const token = await backend.createWaitpoint(jobId, options);
       return token;
     },
 
@@ -517,7 +456,7 @@ function buildWaitContext(
       }
 
       // Check if the token is already completed (e.g., completed while job was still processing)
-      const wp = await getWaitpoint(pool, tokenId);
+      const wp = await backend.getWaitpoint(tokenId);
       if (wp && wp.status === 'completed') {
         const result: WaitTokenResult<T> = {
           ok: true,
@@ -529,7 +468,7 @@ function buildWaitContext(
           completed: true,
           result,
         };
-        await updateStepData(pool, jobId, stepData);
+        await backend.updateStepData(jobId, stepData);
         return result;
       }
       if (wp && wp.status === 'timed_out') {
@@ -543,7 +482,7 @@ function buildWaitContext(
           completed: true,
           result,
         };
-        await updateStepData(pool, jobId, stepData);
+        await backend.updateStepData(jobId, stepData);
         return result;
       }
 
@@ -563,7 +502,12 @@ function buildWaitContext(
 }
 
 /**
- * Process a single job using the provided handler map
+ * Process a single job using the provided handler map.
+ *
+ * @param backend - The queue backend.
+ * @param job - The job record to process.
+ * @param jobHandlers - Map of job type to handler function.
+ * @param emit - Optional callback to emit lifecycle events to the queue's EventEmitter.
  */
 export async function processJobWithHandlers<
   PayloadMap,
@@ -572,6 +516,7 @@ export async function processJobWithHandlers<
   backend: QueueBackend,
   job: JobRecord<PayloadMap, T>,
   jobHandlers: JobHandlers<PayloadMap>,
+  emit?: QueueEmitFn,
 ): Promise<void> {
   const handler = jobHandlers[job.jobType];
 
@@ -580,28 +525,30 @@ export async function processJobWithHandlers<
       `No handler registered for job type: ${job.jobType}`,
       job.jobType,
     );
-    await backend.failJob(
-      job.id,
-      new Error(`No handler registered for job type: ${job.jobType}`),
-      FailureReason.NoHandler,
+    const noHandlerError = new Error(
+      `No handler registered for job type: ${job.jobType}`,
     );
+    await backend.failJob(job.id, noHandlerError, FailureReason.NoHandler);
+    emit?.('job:failed', {
+      jobId: job.id,
+      jobType: job.jobType,
+      error: noHandlerError,
+      willRetry: false,
+    });
     return;
   }
 
   // Load step data (may contain completed steps from previous invocations)
   const stepData: Record<string, any> = { ...(job.stepData || {}) };
 
-  // Try to get pool for wait features (PostgreSQL-only)
-  const pool = tryExtractPool(backend);
-
   // If resuming from a wait, resolve any pending wait entries
   const hasStepHistory = Object.keys(stepData).some((k) =>
     k.startsWith('__wait_'),
   );
-  if (hasStepHistory && pool) {
-    await resolveCompletedWaits(pool, stepData);
+  if (hasStepHistory) {
+    await resolveCompletedWaits(backend, stepData);
     // Persist the resolved step data
-    await updateStepData(pool, job.id, stepData);
+    await backend.updateStepData(job.id, stepData);
   }
 
   // Per-job timeout logic
@@ -685,10 +632,20 @@ export async function processJobWithHandlers<
             },
           };
 
-      // Build context: full wait support for PostgreSQL, basic for others
-      const ctx = pool
-        ? buildWaitContext(backend, pool, job.id, stepData, baseCtx)
-        : buildBasicContext(backend, job.id, baseCtx);
+      // Build context: full wait support for all backends
+      const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
+
+      // Wrap setProgress to also emit the event
+      if (emit) {
+        const originalSetProgress = ctx.setProgress;
+        ctx.setProgress = async (percent: number) => {
+          await originalSetProgress(percent);
+          emit('job:progress', {
+            jobId: job.id,
+            progress: Math.round(percent),
+          });
+        };
+      }
 
       // If forceKillOnTimeout was set but timeoutMs was missing, warn
       if (forceKillOnTimeout && !hasTimeout) {
@@ -715,31 +672,21 @@ export async function processJobWithHandlers<
 
     // Job completed successfully -- complete via backend
     await backend.completeJob(job.id);
+    emit?.('job:completed', { jobId: job.id, jobType: job.jobType });
   } catch (error) {
     if (timeoutId) clearTimeout(timeoutId);
 
     // Check if this is a WaitSignal (not a real error)
     if (error instanceof WaitSignal) {
-      if (!pool) {
-        // Wait signals should never happen with non-PostgreSQL backends
-        // since the context methods throw, but guard just in case
-        await backend.failJob(
-          job.id,
-          new Error(
-            'WaitSignal received but wait features require the PostgreSQL backend.',
-          ),
-          FailureReason.HandlerError,
-        );
-        return;
-      }
       log(
         `Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? 'none'}, tokenId=${error.tokenId ?? 'none'}`,
       );
-      await waitJob(pool, job.id, {
+      await backend.waitJob(job.id, {
         waitUntil: error.waitUntil,
         waitTokenId: error.tokenId,
         stepData: error.stepData,
       });
+      emit?.('job:waiting', { jobId: job.id, jobType: job.jobType });
       return;
     }
 
@@ -755,16 +702,28 @@ export async function processJobWithHandlers<
     ) {
       failureReason = FailureReason.Timeout;
     }
-    await backend.failJob(
-      job.id,
-      error instanceof Error ? error : new Error(String(error)),
-      failureReason,
-    );
+    const failError = error instanceof Error ? error : new Error(String(error));
+    await backend.failJob(job.id, failError, failureReason);
+    emit?.('job:failed', {
+      jobId: job.id,
+      jobType: job.jobType,
+      error: failError,
+      willRetry: job.attempts + 1 < job.maxAttempts,
+    });
   }
 }
 
 /**
- * Process a batch of jobs using the provided handler map and concurrency limit
+ * Process a batch of jobs using the provided handler map and concurrency limit.
+ *
+ * @param backend - The queue backend.
+ * @param workerId - Identifier for the worker claiming jobs.
+ * @param batchSize - Maximum jobs to claim per batch.
+ * @param jobType - Optional job type filter.
+ * @param jobHandlers - Map of job type to handler function.
+ * @param concurrency - Max parallel jobs within the batch.
+ * @param onError - Legacy error callback.
+ * @param emit - Optional callback to emit lifecycle events.
  */
 export async function processBatchWithHandlers<PayloadMap>(
   backend: QueueBackend,
@@ -774,16 +733,26 @@ export async function processBatchWithHandlers<PayloadMap>(
   jobHandlers: JobHandlers<PayloadMap>,
   concurrency?: number,
   onError?: (error: Error) => void,
+  emit?: QueueEmitFn,
 ): Promise<number> {
   const jobs = await backend.getNextBatch<PayloadMap, JobType<PayloadMap>>(
     workerId,
     batchSize,
     jobType,
   );
+
+  // Emit job:processing for each claimed job
+  if (emit) {
+    for (const job of jobs) {
+      emit('job:processing', { jobId: job.id, jobType: job.jobType });
+    }
+  }
+
   if (!concurrency || concurrency >= jobs.length) {
-    // Default: all in parallel
     await Promise.all(
-      jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers)),
+      jobs.map((job) =>
+        processJobWithHandlers(backend, job, jobHandlers, emit),
+      ),
     );
     return jobs.length;
   }
@@ -797,7 +766,7 @@ export async function processBatchWithHandlers<PayloadMap>(
       while (running < concurrency && idx < jobs.length) {
         const job = jobs[idx++];
         running++;
-        processJobWithHandlers(backend, job, jobHandlers)
+        processJobWithHandlers(backend, job, jobHandlers, emit)
           .then(() => {
             running--;
             finished++;
@@ -818,16 +787,21 @@ export async function processBatchWithHandlers<PayloadMap>(
 }
 
 /**
- * Start a job processor that continuously processes jobs
- * @param backend - The queue backend
- * @param handlers - The job handlers for this processor instance
+ * Start a job processor that continuously processes jobs.
+ *
+ * @param backend - The queue backend.
+ * @param handlers - The job handlers for this processor instance.
  * @param options - The processor options. Leave pollInterval empty to run only once. Use jobType to filter jobs by type.
- * @returns {Processor} The processor instance
+ * @param onBeforeBatch - Optional callback invoked before each batch. Used internally to enqueue due cron jobs.
+ * @param emit - Optional callback to emit lifecycle events to the queue's EventEmitter.
+ * @returns The processor instance.
  */
 export const createProcessor = <PayloadMap = any>(
   backend: QueueBackend,
   handlers: JobHandlers<PayloadMap>,
   options: ProcessorOptions = {},
+  onBeforeBatch?: () => Promise<void>,
+  emit?: QueueEmitFn,
 ): Processor => {
   const {
     workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
@@ -847,6 +821,21 @@ export const createProcessor = <PayloadMap = any>(
   const processJobs = async (): Promise<number> => {
     if (!running) return 0;
 
+    // Run pre-batch hook (e.g. enqueue due cron jobs) before processing
+    if (onBeforeBatch) {
+      try {
+        await onBeforeBatch();
+      } catch (hookError) {
+        log(`onBeforeBatch hook error: ${hookError}`);
+        const err =
+          hookError instanceof Error ? hookError : new Error(String(hookError));
+        if (onError) {
+          onError(err);
+        }
+        emit?.('error', err);
+      }
+    }
+
     log(
       `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(',') : jobType}` : ''}`,
     );
@@ -860,11 +849,13 @@ export const createProcessor = <PayloadMap = any>(
         handlers,
         concurrency,
         onError,
+        emit,
       );
-      // Only process one batch in start; do not schedule next batch here
       return processed;
     } catch (error) {
-      onError(error instanceof Error ? error : new Error(String(error)));
+      const err = error instanceof Error ? error : new Error(String(error));
+      onError(err);
+      emit?.('error', err);
     }
     return 0;
   };