bunqueue-dashboard 0.0.35 → 0.0.36

package/agent/queue/runtime.ts

@@ -0,0 +1,119 @@
+import { Queue, type QueueMetricType, type QueueMetrics } from 'bunqueue/client';
+import { managedAuthToken } from '../managedTarget';
+import type { ServerConfig } from '../manager';
+import type { QueueLimitSnapshot, QueueOperationsPort } from './types';
+import { QueueOperationsUnavailableError } from './types';
+
+export type QueueOperationsClient = Pick<
+  Queue,
+  | 'waitUntilReady'
+  | 'getGlobalRateLimit'
+  | 'getGlobalConcurrency'
+  | 'getRateLimitTtl'
+  | 'isMaxed'
+  | 'getDeduplicationJobId'
+  | 'removeDeduplicationKey'
+  | 'getMetrics'
+  | 'trimEvents'
+  | 'close'
+>;
+export type QueueOperationsClientFactory = (
+  config: ServerConfig,
+  queue: string
+) => QueueOperationsClient;
+
+/** Serializes operator reads and mutations and owns each dedicated TCP session. */
+export class QueueOperationsRuntime implements QueueOperationsPort {
+  private gate: Promise<void> = Promise.resolve();
+  private closed = false;
+
+  constructor(private readonly createClient: QueueOperationsClientFactory = defaultClient) {}
+
+  limits(config: ServerConfig, queue: string, maxJobs?: number): Promise<QueueLimitSnapshot> {
+    return this.withQueue(config, queue, async (client) => {
+      const [rateLimit, concurrency, rateLimitTtl, maxed] = await Promise.all([
+        client.getGlobalRateLimit(),
+        client.getGlobalConcurrency(),
+        client.getRateLimitTtl(maxJobs),
+        client.isMaxed(),
+      ]);
+      return { rateLimit, concurrency, rateLimitTtl, maxed };
+    });
+  }
+
+  deduplicationJobId(
+    config: ServerConfig,
+    queue: string,
+    deduplicationId: string
+  ): Promise<string | null> {
+    return this.withQueue(config, queue, (client) =>
+      client.getDeduplicationJobId(deduplicationId)
+    );
+  }
+
+  removeDeduplicationKey(
+    config: ServerConfig,
+    queue: string,
+    deduplicationId: string
+  ): Promise<number> {
+    return this.withQueue(config, queue, (client) => client.removeDeduplicationKey(deduplicationId));
+  }
+
+  metrics(
+    config: ServerConfig,
+    queue: string,
+    type: QueueMetricType,
+    start: number,
+    end: number
+  ): Promise<QueueMetrics> {
+    return this.withQueue(config, queue, (client) => client.getMetrics(type, start, end));
+  }
+
+  trimEvents(config: ServerConfig, queue: string, maxLength: number): Promise<number> {
+    return this.withQueue(config, queue, (client) => client.trimEvents(maxLength));
+  }
+
+  async close(): Promise<void> {
+    this.closed = true;
+    await this.gate;
+  }
+
+  private withQueue<T>(
+    config: ServerConfig,
+    queue: string,
+    operation: (client: QueueOperationsClient) => Promise<T>
+  ): Promise<T> {
+    return this.serial(async () => {
+      const client = this.createClient(config, queue);
+      try {
+        await client.waitUntilReady();
+        return await operation(client);
+      } finally {
+        client.close();
+      }
+    });
+  }
+
+  private serial<T>(operation: () => Promise<T>): Promise<T> {
+    if (this.closed) {
+      return Promise.reject(new QueueOperationsUnavailableError('Queue operations runtime is closed'));
+    }
+    const result = this.gate.then(operation, operation);
+    this.gate = result.then(
+      () => undefined,
+      () => undefined
+    );
+    return result;
+  }
+}
+
+function connectionFor(config: ServerConfig) {
+  return { host: '127.0.0.1', port: config.tcpPort, token: managedAuthToken(config) };
+}
+
+function defaultClient(config: ServerConfig, queue: string): QueueOperationsClient {
+  return new Queue(queue, {
+    autoBatch: { enabled: false },
+    connection: { ...connectionFor(config), poolSize: 1 },
+  });
+}

package/agent/queue/types.ts

@@ -0,0 +1,34 @@
+import type { QueueMetricType, QueueMetrics } from 'bunqueue/client';
+import type { ServerConfig } from '../manager';
+
+export interface QueueLimitSnapshot {
+  rateLimit: { max: number; duration: number } | null;
+  concurrency: number | null;
+  rateLimitTtl: number;
+  maxed: boolean;
+}
+
+export interface QueueOperationsPort {
+  limits(config: ServerConfig, queue: string, maxJobs?: number): Promise<QueueLimitSnapshot>;
+  deduplicationJobId(
+    config: ServerConfig,
+    queue: string,
+    deduplicationId: string
+  ): Promise<string | null>;
+  removeDeduplicationKey(
+    config: ServerConfig,
+    queue: string,
+    deduplicationId: string
+  ): Promise<number>;
+  metrics(
+    config: ServerConfig,
+    queue: string,
+    type: QueueMetricType,
+    start: number,
+    end: number
+  ): Promise<QueueMetrics>;
+  trimEvents(config: ServerConfig, queue: string, maxLength: number): Promise<number>;
+  close(): Promise<void>;
+}
+
+export class QueueOperationsUnavailableError extends Error {}

package/agent/queue/validation.ts

@@ -0,0 +1,100 @@
+import { readLimitedJsonBody } from '../server/jsonBody';
+
+const MAX_BODY_BYTES = 8 * 1024;
+const MAX_PAGE_INDEX = 10_000;
+const MAX_EVENT_RETENTION = 1_000_000;
+
+export function exactQuery(url: string, allowed: readonly string[]): URLSearchParams {
+  const query = new URL(url).searchParams;
+  for (const key of query.keys()) {
+    if (!allowed.includes(key)) throw new Error(`Unknown Queue operation option: ${key}`);
+    if (query.getAll(key).length !== 1) {
+      throw new Error(`Duplicate Queue operation option: ${key}`);
+    }
+  }
+  return query;
+}
+
+export function validateQueueName(queue: string): string {
+  if (
+    !queue ||
+    queue === '.' ||
+    queue === '..' ||
+    queue.length > 256 ||
+    !/^[a-zA-Z0-9_\-.:]+$/.test(queue)
+  ) {
+    throw new Error('Queue name must contain 1-256 letters, numbers, underscores, dashes, dots or colons');
+  }
+  return queue;
+}
+
+export function requiredDeduplicationId(value: string | null): string {
+  if (!value || value.length > 1024) {
+    throw new Error('Deduplication ID must contain 1-1024 characters');
+  }
+  return value;
+}
+
+export function optionalMaxJobs(query: URLSearchParams): number | undefined {
+  const raw = query.get('maxJobs');
+  return raw === null ? undefined : boundedInteger(raw, 'maxJobs', 0, 1_000_000);
+}
+
+export function metricsRange(query: URLSearchParams): { start: number; end: number } {
+  const start = optionalInteger(query.get('start'), 'start', 0, MAX_PAGE_INDEX, 0);
+  const rawEnd = query.get('end');
+  const end = rawEnd === null ? -1 : boundedInteger(rawEnd, 'end', -1, MAX_PAGE_INDEX);
+  if (end !== -1 && end < start) throw new Error('Metrics end must be -1 or at least start');
+  return { start, end };
+}
+
+export function eventRetention(value: unknown): number {
+  return bodyInteger(value, 'maxLength', 0, MAX_EVENT_RETENTION);
+}
+
+export async function exactJsonBody(
+  request: Request,
+  allowed: readonly string[]
+): Promise<Record<string, unknown>> {
+  const value = await readLimitedJsonBody(request, {
+    scope: 'Queue operation',
+    maxBytes: MAX_BODY_BYTES,
+    limitLabel: '8 KiB',
+    missingMessage: 'Queue operation request must be valid JSON',
+    invalidJsonMessage: 'Queue operation request must be valid JSON',
+  });
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error('Queue operation request must be an object');
+  }
+  const body = value as Record<string, unknown>;
+  const unknown = Object.keys(body).find((key) => !allowed.includes(key));
+  if (unknown) throw new Error(`Unknown Queue operation body field: ${unknown}`);
+  return body;
+}
+
+function optionalInteger(
+  value: string | null,
+  label: string,
+  minimum: number,
+  maximum: number,
+  fallback: number
+): number {
+  return value === null ? fallback : boundedInteger(value, label, minimum, maximum);
+}
+
+function boundedInteger(
+  raw: string,
+  label: string,
+  minimum: number,
+  maximum: number
+): number {
+  if (!/^-?\d+$/.test(raw)) throw new Error(`${label} must be an integer`);
+  return bodyInteger(Number(raw), label, minimum, maximum);
+}
+
+function bodyInteger(value: unknown, label: string, minimum: number, maximum: number): number {
+  if (!Number.isSafeInteger(value) || (value as number) < minimum || (value as number) > maximum) {
+    throw new Error(`${label} must be an integer from ${minimum} to ${maximum}`);
+  }
+  return value as number;
+}

package/agent/server/controlRoutes.ts

@@ -0,0 +1,153 @@
+import { type ProcessManager, validateConfigPatch } from '../manager';
+import { safeErrorMessage } from '../errorMessage';
+import {
+  WorkflowRuntimeUnavailableError,
+  type WorkflowRuntimePort,
+} from '../workflow/runtime';
+import { readLimitedJsonBody } from './jsonBody';
+import type { AgentLifecyclePort } from './lifecycle';
+import type { RouteResponse } from './types';
+
+const MAX_CONFIG_BODY_BYTES = 64 * 1024;
+
+async function statusWithHealth(manager: ProcessManager) {
+  const snapshot = manager.getStatus();
+  let healthy = false;
+  let version: string | undefined;
+  if (snapshot.status === 'running') {
+    const port = snapshot.runningConfig?.httpPort ?? snapshot.config.httpPort;
+    try {
+      const response = await fetch(`http://127.0.0.1:${port}/health`, {
+        signal: AbortSignal.timeout(1500),
+      });
+      if (response.ok) {
+        const body: unknown = await response.json();
+        if (body !== null && typeof body === 'object' && !Array.isArray(body)) {
+          const health = body as Record<string, unknown>;
+          healthy = health.ok === true;
+          if (typeof health.version === 'string') version = health.version;
+        }
+      }
+    } catch {
+      // The managed process is not healthy yet.
+    }
+  }
+  const database = await manager.dbStats().catch(() => null);
+  return { ...snapshot, healthy, version, db: database };
+}
+
+export async function routeControlRequest(
+  request: Request,
+  pathname: string,
+  method: string,
+  manager: ProcessManager,
+  runtime: WorkflowRuntimePort,
+  lifecycle?: AgentLifecyclePort
+): Promise<RouteResponse | null> {
+  if (pathname === '/control/status') {
+    return { status: 200, body: await statusWithHealth(manager) };
+  }
+  if (pathname === '/control/logs') {
+    return { status: 200, body: { lines: manager.getLogs() } };
+  }
+  if (pathname === '/control/start' && method === 'POST') {
+    return coordinated(lifecycle, async () => {
+      await manager.start();
+      return { status: 200, body: await statusWithHealth(manager) };
+    });
+  }
+  if (pathname === '/control/stop' && method === 'POST') {
+    return coordinated(lifecycle, async () => {
+      await stopWithRuntimeCleanup(manager, runtime);
+      return { status: 200, body: await statusWithHealth(manager) };
+    });
+  }
+  if (pathname === '/control/restart' && method === 'POST') {
+    return coordinated(lifecycle, async () => {
+      const closeResult = await settle(() => runtime.close());
+      if (!closeResult.ok) {
+        await stopAfterCloseFailure(manager, workflowCloseError(closeResult.error));
+      }
+      await manager.restart();
+      return { status: 200, body: await statusWithHealth(manager) };
+    });
+  }
+  if (pathname === '/control/config' && method === 'GET') {
+    return { status: 200, body: manager.getConfig() };
+  }
+  if (pathname === '/control/config' && method === 'PUT') {
+    const patch = validateConfigPatch(
+      await readLimitedJsonBody(request, {
+        scope: 'Agent configuration',
+        maxBytes: MAX_CONFIG_BODY_BYTES,
+        limitLabel: '64 KiB',
+      })
+    );
+    return { status: 200, body: manager.setConfig(patch) };
+  }
+  return null;
+}
+
+async function stopWithRuntimeCleanup(
+  manager: ProcessManager,
+  runtime: WorkflowRuntimePort
+): Promise<void> {
+  const closeResult = await settle(() => runtime.close());
+  try {
+    await manager.stop();
+  } catch (stopFailure) {
+    if (!closeResult.ok) throw combinedShutdownError(closeResult.error, stopFailure);
+    throw stopFailure;
+  }
+  if (!closeResult.ok) throw workflowCloseError(closeResult.error);
+}
+
+async function stopAfterCloseFailure(
+  manager: ProcessManager,
+  closeFailure: unknown
+): Promise<never> {
+  try {
+    await manager.stop();
+  } catch (stopFailure) {
+    throw combinedShutdownError(closeFailure, stopFailure);
+  }
+  throw closeFailure;
+}
+
+function combinedShutdownError(closeFailure: unknown, stopFailure: unknown): Error {
+  return new WorkflowRuntimeUnavailableError(
+    `Workflow Engine shutdown failed (${message(closeFailure)}); ` +
+      `managed server stop also failed: ${message(stopFailure)}`
+  );
+}
+
+function workflowCloseError(error: unknown): WorkflowRuntimeUnavailableError {
+  try {
+    if (error instanceof WorkflowRuntimeUnavailableError) return error;
+  } catch {
+    // A hostile thrown Proxy must still become an unavailable-runtime error.
+  }
+  return new WorkflowRuntimeUnavailableError(`Workflow Engine shutdown failed: ${message(error)}`);
+}
+
+function message(error: unknown): string {
+  return safeErrorMessage(error);
+}
+
+function coordinated<T>(
+  lifecycle: AgentLifecyclePort | undefined,
+  operation: () => Promise<T>
+): Promise<T> {
+  return lifecycle ? lifecycle.run(operation) : operation();
+}
+
+type SettledOperation = { ok: true } | { ok: false; error: unknown };
+
+async function settle(operation: () => Promise<unknown>): Promise<SettledOperation> {
+  try {
+    await operation();
+    return { ok: true };
+  } catch (error) {
+    return { ok: false, error };
+  }
+}

package/agent/server/databaseRoutes.ts

@@ -0,0 +1,165 @@
+import {
+  type DbFilter,
+  dbCell,
+  dbInfo,
+  dbRows,
+  dbSchema,
+  dbTables,
+  exportWithTimeout,
+  queryWithTimeout,
+} from '../db';
+import { corsHeaders } from './policy';
+import { readLimitedJsonBody } from './jsonBody';
+import type { RouteResponse } from './types';
+
+const MAX_QUERY_BODY_BYTES = 64 * 1024;
+
+function exportFilter(query: URLSearchParams): DbFilter | undefined {
+  const column = query.get('fcol');
+  const operation = query.get('fop');
+  const value = query.get('fval');
+  const hasPart = column !== null || operation !== null || value !== null;
+  if (!hasPart) return undefined;
+  if (
+    !column ||
+    !value ||
+    (operation !== 'contains' && operation !== 'eq' && operation !== 'ne')
+  ) {
+    throw new Error('Database export filter requires valid fcol, fop and fval values');
+  }
+  return { column, op: operation, value };
+}
+
+function validateExportQuery(query: URLSearchParams): void {
+  const allowed = new Set(['orderBy', 'dir', 'fcol', 'fop', 'fval']);
+  for (const key of query.keys()) {
+    if (!allowed.has(key)) throw new Error(`Unknown database export option: ${key}`);
+    if (query.getAll(key).length !== 1) {
+      throw new Error(`Duplicate database export option: ${key}`);
+    }
+  }
+}
+
+async function exportTable(
+  request: Request,
+  table: string,
+  dataPath: string,
+  origin: string | null,
+  allowedOrigins: string[]
+): Promise<Response> {
+  const query = new URL(request.url).searchParams;
+  validateExportQuery(query);
+  const orderBy = query.get('orderBy');
+  if (orderBy === '') throw new Error('Database export orderBy must not be empty');
+  const direction = query.get('dir');
+  if (direction !== null && direction !== 'asc' && direction !== 'desc') {
+    throw new Error('Database export dir must be "asc" or "desc"');
+  }
+  const exported = await exportWithTimeout(
+    dataPath,
+    table,
+    orderBy ?? undefined,
+    direction === 'desc' ? 'desc' : 'asc',
+    exportFilter(query),
+    request.signal
+  );
+  return new Response(exported.content, {
+    status: 200,
+    headers: {
+      'Content-Type': 'text/csv; charset=utf-8',
+      'Content-Length': String(exported.bytes),
+      'Cache-Control': 'no-store, no-transform',
+      'X-Content-Type-Options': 'nosniff',
+      'X-Bunqueue-Db-Export-Version': '1',
+      'X-Bunqueue-Db-Export-Table': encodeURIComponent(exported.table),
+      'X-Bunqueue-Db-Export-Rows': String(exported.rowCount),
+      'X-Bunqueue-Db-Export-Bytes': String(exported.bytes),
+      'X-Bunqueue-Db-Export-Cap': exported.cap ?? 'none',
+      'Access-Control-Expose-Headers':
+        'Content-Length, X-Bunqueue-Db-Export-Version, X-Bunqueue-Db-Export-Table, X-Bunqueue-Db-Export-Rows, X-Bunqueue-Db-Export-Bytes, X-Bunqueue-Db-Export-Cap',
+      ...corsHeaders(origin, allowedOrigins),
+    },
+  });
+}
+
+function browseFilter(query: URLSearchParams): DbFilter | undefined {
+  const column = query.get('fcol');
+  const operation = query.get('fop');
+  const value = query.get('fval');
+  if (!column || !value) return undefined;
+  return {
+    column,
+    op: operation === 'eq' ? 'eq' : operation === 'ne' ? 'ne' : 'contains',
+    value,
+  };
+}
+
+export async function routeDatabaseRequest(
+  request: Request,
+  pathname: string,
+  method: string,
+  dataPath: string,
+  origin: string | null,
+  allowedOrigins: string[]
+): Promise<RouteResponse | Response | null> {
+  if (pathname === '/db/info' && method === 'GET') {
+    return { status: 200, body: { ok: true, ...dbInfo(dataPath) } };
+  }
+  if (pathname === '/db/tables' && method === 'GET') {
+    return { status: 200, body: { ok: true, tables: dbTables(dataPath) } };
+  }
+  if (pathname.startsWith('/db/tables/') && method === 'GET') {
+    const segments = pathname.slice('/db/tables/'.length).split('/');
+    const table = decodeURIComponent(segments[0] ?? '');
+    const subresource = segments[1];
+    if (segments.length === 2 && subresource === 'export') {
+      return exportTable(request, table, dataPath, origin, allowedOrigins);
+    }
+    if (segments.length === 2 && subresource === 'schema') {
+      return { status: 200, body: { ok: true, ...dbSchema(dataPath, table) } };
+    }
+    if (segments.length === 2 && subresource === 'cell') {
+      const query = new URL(request.url).searchParams;
+      const rowid = query.get('rowid');
+      if (rowid === null) throw new Error('rowid is required');
+      return {
+        status: 200,
+        body: {
+          ok: true,
+          ...dbCell(dataPath, table, rowid, query.get('column') ?? ''),
+        },
+      };
+    }
+    if (segments.length !== 1) {
+      return { status: 404, body: { ok: false, error: 'Not found' } };
+    }
+    const query = new URL(request.url).searchParams;
+    return {
+      status: 200,
+      body: {
+        ok: true,
+        ...dbRows(
+          dataPath,
+          table,
+          Number(query.get('limit')) || 50,
+          Number(query.get('offset')) || 0,
+          query.get('orderBy') || undefined,
+          query.get('dir') === 'desc' ? 'desc' : 'asc',
+          browseFilter(query)
+        ),
+      },
+    };
+  }
+  if (pathname === '/db/query' && method === 'POST') {
+    const { sql } = (await readLimitedJsonBody(request, {
+      scope: 'Database query',
+      maxBytes: MAX_QUERY_BODY_BYTES,
+      limitLabel: '64 KiB',
+    })) as { sql?: string };
+    return {
+      status: 200,
+      body: { ok: true, ...(await queryWithTimeout(dataPath, sql ?? '')) },
+    };
+  }
+  return null;
+}

package/agent/server/errors.ts

@@ -0,0 +1,48 @@
+import {
+  DbExportBusyError,
+  DbExportUnavailableError,
+  MissingDbError,
+} from '../db';
+import { safeErrorMessage } from '../errorMessage';
+import { QueueOperationsUnavailableError } from '../queue/types';
+import { WorkflowRuntimeUnavailableError } from '../workflow/runtime';
+import { AgentLifecycleClosedError } from './lifecycle';
+
+const CONFLICT_MESSAGES = [
+  'Start the managed Bunqueue server',
+  'Stop the managed Bunqueue server',
+  'Another backup operation is already running',
+  'Another maintenance operation is already running',
+  'Cannot start the managed Bunqueue server',
+  'Cannot restart the managed Bunqueue server',
+  'BUNQUEUE_WORKFLOW_QUEUE_NAME must',
+  'BUNQUEUE_WORKFLOW_CONCURRENCY must',
+  'Managed Bunqueue server restarted',
+];
+
+export function errorStatus(error: unknown): number {
+  try {
+    if (error instanceof AgentLifecycleClosedError) return 503;
+    if (error instanceof MissingDbError) return 404;
+    if (error instanceof DbExportBusyError) return 429;
+    if (error instanceof DbExportUnavailableError) return 503;
+    if (error instanceof QueueOperationsUnavailableError) {
+      return error.message.includes('runtime is closed') ? 503 : 409;
+    }
+    if (error instanceof WorkflowRuntimeUnavailableError) {
+      // Configuration shape is actionable by the caller; a missing/unreadable
+      // configured module is an unavailable server-side dependency.
+      return /^(Set BUNQUEUE_WORKFLOW_MODULE|BUNQUEUE_WORKFLOW_MODULE must|BUNQUEUE_WORKFLOW_QUEUE_NAME must|BUNQUEUE_WORKFLOW_CONCURRENCY must|Workflow module must|workflowNames must)/.test(error.message)
+        ? 409
+        : 503;
+    }
+  } catch {
+    return 500;
+  }
+  const message = safeErrorMessage(error);
+  return CONFLICT_MESSAGES.some((prefix) => message.startsWith(prefix)) ? 409 : 400;
+}
+
+export function errorMessage(error: unknown): string {
+  return safeErrorMessage(error);
+}