bunqueue-client 0.1.9 → 0.2.0

package/src/flow-plan.ts

@@ -0,0 +1,209 @@
+import { randomUUID } from 'node:crypto';
+
+import type { FlowJob, FlowOptions } from './flow-types.js';
+import { compact } from './frame.js';
+import { type JobOptions, wireJobOptions } from './types.js';
+
+const MAX_FLOW_DEPTH = 100;
+const MAX_FLOW_JOBS = 10_000;
+const QUEUE_RE = /^[a-zA-Z0-9_\-.:]+$/;
+
+export type FlowIdFactory = () => string;
+
+export interface AtomicFlowJobInput {
+  id: string;
+  queue: string;
+  input: Record<string, unknown>;
+}
+
+export interface PlannedFlowNode<T = unknown> {
+  id: string;
+  name: string;
+  queueName: string;
+  data: T | undefined;
+  children?: PlannedFlowNode<T>[];
+}
+
+export interface FlowPlan<T = unknown> {
+  jobs: AtomicFlowJobInput[];
+  roots: PlannedFlowNode<T>[];
+}
+
+interface PlanState {
+  readonly ids: Set<string>;
+  readonly seen: WeakSet<object>;
+  readonly jobs: AtomicFlowJobInput[];
+  readonly idFactory: FlowIdFactory;
+  count: number;
+}
+
+interface FlowLinks {
+  parentId?: string;
+  dependsOn?: string[];
+  childrenIds?: string[];
+}
+
+function assertNameAndQueue(name: unknown, queueName: unknown, depth: number): void {
+  if (typeof name !== 'string' || name.length === 0 || name.length > 256) {
+    throw new Error('flow job name must be a non-empty string of at most 256 characters');
+  }
+  if (
+    typeof queueName !== 'string' ||
+    queueName.length === 0 ||
+    queueName.length > 256 ||
+    !QUEUE_RE.test(queueName)
+  ) {
+    throw new Error('flow queueName is invalid');
+  }
+  if (depth > MAX_FLOW_DEPTH) {
+    throw new Error(`flow exceeds the ${MAX_FLOW_DEPTH} level depth limit`);
+  }
+}
+
+function assertAtomicOptions(opts: JobOptions): void {
+  if (opts.repeat !== undefined) throw new Error('repeat is not supported inside an atomic flow');
+  if (opts.deduplication !== undefined) {
+    throw new Error('deduplication is not supported inside an atomic flow');
+  }
+  if (opts.debounce !== undefined) {
+    throw new Error('debounce is not supported inside an atomic flow');
+  }
+  if (
+    opts.parentId !== undefined ||
+    opts.dependsOn !== undefined ||
+    opts.childrenIds !== undefined
+  ) {
+    throw new Error('flow topology options are owned by FlowProducer');
+  }
+}
+
+function assertUserData(data: unknown): void {
+  if (!data || typeof data !== 'object' || Array.isArray(data)) return;
+  for (const key of Object.keys(data)) {
+    if (key === 'name' || key.startsWith('__')) {
+      throw new Error(`flow job data key is reserved: ${key}`);
+    }
+  }
+}
+
+export function flowData(
+  name: string,
+  data: unknown,
+  internal: Record<string, unknown> = {}
+): Record<string, unknown> {
+  assertUserData(data);
+  if (data && typeof data === 'object' && !Array.isArray(data)) {
+    return { ...(data as Record<string, unknown>), name, ...internal };
+  }
+  return data === undefined || data === null
+    ? { name, ...internal }
+    : { name, payload: data, ...internal };
+}
+
+export function flowInput(
+  data: Record<string, unknown>,
+  opts: JobOptions,
+  links: FlowLinks = {}
+): Record<string, unknown> {
+  assertAtomicOptions(opts);
+  const mapped = { ...wireJobOptions(opts) };
+  delete mapped.jobId;
+  delete mapped.parentId;
+  delete mapped.dependsOn;
+  delete mapped.childrenIds;
+  return compact({
+    data,
+    ...mapped,
+    customId: opts.jobId,
+    parentId: links.parentId,
+    dependsOn: links.dependsOn,
+    childrenIds: links.childrenIds,
+  });
+}
+
+export function allocateFlowId(
+  opts: JobOptions,
+  ids: Set<string>,
+  idFactory: FlowIdFactory
+): string {
+  const id = opts.jobId ?? idFactory();
+  if (typeof id !== 'string' || id.length === 0 || id.length > 1_024 || id.includes(':')) {
+    throw new Error('flow jobId must be non-empty and cannot contain a colon');
+  }
+  if (ids.has(id)) throw new Error(`duplicate flow job id: ${id}`);
+  ids.add(id);
+  return id;
+}
+
+function visit<T>(
+  node: FlowJob<T>,
+  parent: { id: string; queue: string } | undefined,
+  depth: number,
+  options: FlowOptions | undefined,
+  state: PlanState
+): PlannedFlowNode<T> {
+  if (!node || typeof node !== 'object') throw new Error('flow node must be an object');
+  assertNameAndQueue(node.name, node.queueName, depth);
+  if (node.children !== undefined && !Array.isArray(node.children)) {
+    throw new Error('flow children must be an array');
+  }
+  if (state.seen.has(node as object)) throw new Error('flow contains a cycle or shared node');
+  state.seen.add(node as object);
+  state.count += 1;
+  if (state.count > MAX_FLOW_JOBS) {
+    throw new Error(`flow exceeds the ${MAX_FLOW_JOBS} job limit`);
+  }
+
+  const defaults = options?.queuesOptions?.[node.queueName];
+  const opts = defaults ? { ...defaults, ...node.opts } : (node.opts ?? {});
+  assertAtomicOptions(opts);
+  const id = allocateFlowId(opts, state.ids, state.idFactory);
+  const children = (node.children ?? []).map((child) =>
+    visit(child, { id, queue: node.queueName }, depth + 1, options, state)
+  );
+  const childIds = children.map((child) => child.id);
+  const internal: Record<string, unknown> = {};
+  if (parent) {
+    internal.__parentId = parent.id;
+    internal.__parentQueue = parent.queue;
+  }
+  if (childIds.length > 0) internal.__childrenIds = childIds;
+  const data = flowData(node.name, node.data, internal);
+  state.jobs.push({
+    id,
+    queue: node.queueName,
+    input: flowInput(data, opts, {
+      parentId: parent?.id,
+      dependsOn: childIds.length > 0 ? childIds : undefined,
+      childrenIds: childIds.length > 0 ? childIds : undefined,
+    }),
+  });
+  return {
+    id,
+    name: node.name,
+    queueName: node.queueName,
+    data: node.data,
+    children: children.length > 0 ? children : undefined,
+  };
+}
+
+export function planFlows<T>(
+  flows: FlowJob<T>[],
+  options?: FlowOptions,
+  idFactory: FlowIdFactory = randomUUID
+): FlowPlan<T> {
+  for (const defaults of Object.values(options?.queuesOptions ?? {})) {
+    if ((defaults as Partial<JobOptions>).jobId !== undefined) {
+      throw new Error('jobId cannot be a queue default');
+    }
+  }
+  const state: PlanState = {
+    ids: new Set(),
+    seen: new WeakSet(),
+    jobs: [],
+    idFactory,
+    count: 0,
+  };
+  const roots = flows.map((flow) => visit(flow, undefined, 0, options, state));
+  return { jobs: state.jobs, roots };
+}

package/src/flow-types.ts

@@ -13,6 +13,10 @@ export interface FlowJob<T = unknown> {
   children?: FlowJob<T>[];
 }
 
+export interface FlowOptions {
+  queuesOptions?: Record<string, Omit<Partial<JobOptions>, 'jobId'>>;
+}
+
 export interface JobNode<T = unknown> {
   job: Job<T>;
   children?: JobNode<T>[];

package/src/flow.ts

@@ -1,22 +1,22 @@
 /**
- * FlowProducer: parent/children job trees, chains and fan-in flows.
- * Mirrors the official client's TCP flow logic (bottom-up creation,
- * UpdateParent fix-up, rollback via Cancel on failure).
+ * FlowProducer: atomic parent/children trees, chains and fan-in flows.
  */
 
 import { Connection } from './connection.js';
 import { CommandError } from './errors.js';
+import { commitFlow } from './flow-commit.js';
+import { type PlannedFlowNode, planFlows } from './flow-plan.js';
+import { planBulkThen, planChain } from './flow-plan-legacy.js';
 import type {
   FlowJob,
+  FlowOptions,
   FlowProducerOptions,
   FlowStep,
   GetFlowOptions,
   JobNode,
 } from './flow-types.js';
-import { compact } from './frame.js';
 import { Job } from './job.js';
-import type { JobResponse, OkResponse } from './responses.js';
-import { type JobOptions, jobPayload, wireJobOptions } from './types.js';
+import type { JobResponse } from './responses.js';
 
 export class FlowProducer {
   readonly connection: Connection;
@@ -36,29 +36,17 @@ export class FlowProducer {
     this.ownsConnection = opts.connection === undefined;
   }
 
-  /** Add a flow tree. Children are created (and processed) BEFORE their parent. */
-  async add<T = unknown>(flow: FlowJob<T>): Promise<JobNode<T>> {
-    const created: string[] = [];
-    try {
-      return await this.addNode(flow, null, created);
-    } catch (err) {
-      await this.rollback(created);
-      throw err;
-    }
+  /** Add a flow tree in one broker-side atomic commit. */
+  async add<T = unknown>(flow: FlowJob<T>, options?: FlowOptions): Promise<JobNode<T>> {
+    const plan = planFlows([flow], options);
+    const snapshots = await commitFlow(this.connection, plan.jobs);
+    return this.buildNode(plan.roots[0], snapshots);
   }
 
   async addBulk<T = unknown>(flows: FlowJob<T>[]): Promise<JobNode<T>[]> {
-    const created: string[] = [];
-    const results: JobNode<T>[] = [];
-    try {
-      for (const flow of flows) {
-        results.push(await this.addNode(flow, null, created));
-      }
-      return results;
-    } catch (err) {
-      await this.rollback(created);
-      throw err;
-    }
+    const plan = planFlows(flows);
+    const snapshots = await commitFlow(this.connection, plan.jobs);
+    return plan.roots.map((root) => this.buildNode(root, snapshots));
   }
 
   /** Fetch a flow tree starting from a job id (recursive over childrenIds). */
@@ -74,32 +62,9 @@ export class FlowProducer {
   /** Add a sequential chain: step[0] → step[1] → ... via dependsOn. */
   async addChain<T = unknown>(steps: FlowStep<T>[]): Promise<{ jobIds: string[] }> {
     if (steps.length === 0) return { jobIds: [] };
-    const jobIds: string[] = [];
-    let prevId: string | null = null;
-    try {
-      for (const step of steps) {
-        const data: Record<string, unknown> = compact({
-          ...jobPayload(step.name, step.data),
-          __flowParentId: prevId ?? undefined,
-        });
-        // Connection.call compacts internally, so no outer compact() is needed
-        // (nesting it confused generic inference of `data`/`response`).
-        const response: OkResponse = await this.connection.call<OkResponse>({
-          cmd: 'PUSH',
-          queue: step.queueName,
-          data,
-          ...wireJobOptions(step.opts),
-          dependsOn: prevId ? [prevId] : undefined,
-        });
-        const id = String(response.id);
-        jobIds.push(id);
-        prevId = id;
-      }
-      return { jobIds };
-    } catch (err) {
-      await this.rollback(jobIds);
-      throw err;
-    }
+    const plan = planChain(steps);
+    await commitFlow(this.connection, plan.jobs);
+    return { jobIds: plan.ids };
   }
 
   /** Parallel jobs converging into a final fan-in job. */
@@ -107,47 +72,9 @@ export class FlowProducer {
     parallel: FlowStep<T>[],
     final: FlowStep<T>
   ): Promise<{ parallelIds: string[]; finalId: string }> {
-    const parallelIds: string[] = [];
-    try {
-      const results = await Promise.allSettled(
-        parallel.map(async (step) => {
-          const response = await this.connection.call<OkResponse>(
-            compact({
-              cmd: 'PUSH',
-              queue: step.queueName,
-              data: jobPayload(step.name, step.data),
-              ...wireJobOptions(step.opts),
-            }) as { cmd: string }
-          );
-          return String(response.id);
-        })
-      );
-      const errors: unknown[] = [];
-      for (const r of results) {
-        if (r.status === 'fulfilled') parallelIds.push(r.value);
-        else errors.push(r.reason);
-      }
-      if (errors.length > 0) {
-        await this.rollback(parallelIds);
-        throw errors[0] instanceof Error ? errors[0] : new Error(String(errors[0]));
-      }
-
-      const finalData = {
-        ...jobPayload(final.name, final.data),
-        __flowParentIds: parallelIds,
-      };
-      const finalId = await this.pushWithParent(
-        final.queueName,
-        finalData,
-        final.opts,
-        null,
-        parallelIds
-      );
-      return { parallelIds, finalId };
-    } catch (err) {
-      await this.rollback(parallelIds);
-      throw err;
-    }
+    const plan = planBulkThen(parallel, final);
+    await commitFlow(this.connection, plan.jobs);
+    return { parallelIds: plan.parallelIds, finalId: plan.finalId };
   }
 
   close(): void {
@@ -156,64 +83,17 @@ export class FlowProducer {
 
   // ----------------------------------------------------------------- internals
 
-  private async addNode<T>(
-    node: FlowJob<T>,
-    parentRef: { id: string; queue: string } | null,
-    created: string[]
-  ): Promise<JobNode<T>> {
-    const childNodes: JobNode<T>[] = [];
-    const childIds: string[] = [];
-
-    if (node.children && node.children.length > 0) {
-      const tempParent = { id: 'pending', queue: node.queueName };
-      const results = await Promise.all(
-        node.children.map((child) => this.addNode(child, tempParent, created))
-      );
-      for (const childNode of results) {
-        childNodes.push(childNode);
-        childIds.push(childNode.job.id);
-      }
-    }
-
-    const jobData: Record<string, unknown> = jobPayload(node.name, node.data);
-    if (parentRef) {
-      jobData.__parentId = parentRef.id;
-      jobData.__parentQueue = parentRef.queue;
-    }
-    if (childIds.length > 0) jobData.__childrenIds = childIds;
-
-    const id = await this.pushWithParent(node.queueName, jobData, node.opts, parentRef, childIds);
-    created.push(id);
-
-    const job = new Job<T>({ id, queue: node.queueName, data: jobData }, this.connection);
-    return { job, children: childNodes.length > 0 ? childNodes : undefined };
-  }
-
-  private async pushWithParent(
-    queue: string,
-    data: Record<string, unknown>,
-    opts: JobOptions | undefined,
-    parentRef: { id: string; queue: string } | null,
-    childIds: string[]
-  ): Promise<string> {
-    const response = await this.connection.call<OkResponse>(
-      compact({
-        cmd: 'PUSH',
-        queue,
-        data,
-        ...wireJobOptions(opts),
-        parentId: parentRef?.id,
-        childrenIds: childIds.length > 0 ? childIds : undefined,
-        dependsOn: childIds.length > 0 ? childIds : undefined,
-      }) as { cmd: string }
-    );
-    const parentJobId = String(response.id);
-
-    // fix up children with the real parent id (like the official client)
-    for (const childId of childIds) {
-      await this.connection.call({ cmd: 'UpdateParent', childId, parentId: parentJobId });
-    }
-    return parentJobId;
+  private buildNode<T>(
+    node: PlannedFlowNode<T>,
+    snapshots: ReadonlyMap<string, Record<string, unknown>>
+  ): JobNode<T> {
+    const snapshot = snapshots.get(node.id);
+    if (!snapshot) throw new Error(`Committed flow snapshot missing for ${node.id}`);
+    const children = node.children?.map((child) => this.buildNode(child, snapshots));
+    return {
+      job: new Job<T>(snapshot, this.connection),
+      children: children && children.length > 0 ? children : undefined,
+    };
   }
 
   private async fetchNode<T>(
@@ -249,16 +129,4 @@ export class FlowProducer {
     }
     return { job, children: children.length > 0 ? children : undefined };
   }
-
-  private async rollback(jobIds: string[]): Promise<void> {
-    await Promise.all(
-      jobIds.map(async (id) => {
-        try {
-          await this.connection.call({ cmd: 'Cancel', id });
-        } catch {
-          // ignore cleanup errors
-        }
-      })
-    );
-  }
 }

package/src/frame.ts

@@ -3,9 +3,9 @@
  * prefix + msgpack payload. Runtime-neutral (plain Buffer operations).
  */
 
-import { ConnectionClosedError } from './errors.js';
+import { ConnectionClosedError, SerializationError } from './errors.js';
 
-export const PROTOCOL_VERSION = 2;
+export const PROTOCOL_VERSION = 3;
 export const MAX_FRAME_SIZE = 64 * 1024 * 1024; // mirror server-side limit
 
 /** Drop undefined-valued keys so the msgpack frame stays minimal. */
@@ -19,6 +19,9 @@ export function compact<T extends Record<string, unknown>>(obj: T): T {
 
 /** Frame a msgpack payload with the 4-byte big-endian length prefix. */
 export function frame(payload: Uint8Array): Buffer {
+  if (payload.length > MAX_FRAME_SIZE) {
+    throw new SerializationError(`frame size ${payload.length} exceeds maximum ${MAX_FRAME_SIZE}`);
+  }
   const framed = Buffer.allocUnsafe(4 + payload.length);
   framed.writeUInt32BE(payload.length, 0);
   framed.set(payload, 4);

package/src/index.ts

@@ -1,30 +1,7 @@
-/**
- * bunqueue-client — cross-runtime TypeScript SDK for bunqueue (TCP mode).
- * Works on Node.js ≥20, Bun and Deno ≥2 (uses only node: builtins + msgpackr).
- */
+/** Public API is sourced from the canonical client; the build supplies portable I/O. */
+export * from '../../../src/client/index.js';
 
-// Simple Mode (Bunqueue): all-in-one Queue + Worker, 1:1 with the official client
-export { Bunqueue } from './bunqueue/bunqueue.js';
-export type { DlqFilter, DlqStats } from './bunqueue/dlq-rate-limit.js';
-export type {
-  BatchConfig,
-  BatchProcessor,
-  BunqueueConnection,
-  BunqueueDebounceConfig,
-  BunqueueDeduplicationConfig,
-  BunqueueDlqConfig,
-  BunqueueMiddleware,
-  BunqueueOptions,
-  CircuitBreakerConfig,
-  CircuitState,
-  JobTtlConfig,
-  PriorityAgingConfig,
-  RateLimiterOptions,
-  RetryConfig,
-  RetryStrategy,
-  TriggerRule,
-} from './bunqueue/types.js';
-export type { Command, ConnectionOptions, Response, TlsOption } from './connection.js';
+// Transport-specific helpers are additive; core client types remain canonical.
 export { Connection } from './connection.js';
 export { ConnectionPool } from './connection-pool.js';
 export type { ConnectionLike } from './connection-types.js';
@@ -34,19 +11,9 @@ export {
   CommandError,
   CommandTimeoutError,
   ConnectionClosedError,
-  UnrecoverableError,
+  SerializationError,
 } from './errors.js';
-export { FlowProducer } from './flow.js';
-export type {
-  FlowJob,
-  FlowProducerOptions,
-  FlowStep,
-  GetFlowOptions,
-  JobNode,
-} from './flow-types.js';
-export { MAX_FRAME_SIZE, PROTOCOL_VERSION } from './frame.js';
-export type { JobRaw } from './job.js';
-export { Job } from './job.js';
+export { consoleLogger, noopLogger } from './observability.js';
 export type {
   Logger,
   LogLevel,
@@ -54,40 +21,5 @@ export type {
   TelemetryEvent,
   TelemetryHandler,
 } from './observability.js';
-export { consoleLogger, noopLogger } from './observability.js';
-export type { BulkJobEntry, QueueOptions } from './queue.js';
-export { Queue } from './queue.js';
-export type { SchedulerOptions } from './queue-admin.js';
-export type {
-  BatchResponse,
-  CountResponse,
-  DataResponse,
-  JobCountsResponse,
-  JobResponse,
-  JobsResponse,
-  OkResponse,
-  PausedResponse,
-  ProgressResponse,
-  PulledJobResponse,
-  PulledJobsResponse,
-  ResultResponse,
-  StateResponse,
-  WaitJobResponse,
-} from './responses.js';
-export type {
-  BackoffOptions,
-  DeduplicationOptions,
-  JobCounts,
-  JobOptions,
-  JobStateName,
-  RepeatOptions,
-} from './types.js';
-export { Worker } from './worker.js';
-export type {
-  AckBatchOptions,
-  Processor,
-  WorkerEventMap,
-  WorkerOptions,
-} from './worker-types.js';
-
-export const __version__ = '0.1.7';
+export { MAX_FRAME_SIZE, PROTOCOL_VERSION } from './frame.js';
+export const __version__ = '0.2.0';

package/src/job.ts

@@ -33,19 +33,35 @@ export class Job<T = unknown> {
   }
 
   get data(): T {
-    return this.raw.data as T;
+    const data = this.raw.data;
+    if (this.topLevelName() !== undefined) return data as T;
+    if (data && typeof data === 'object' && !Array.isArray(data)) {
+      const record = data as Record<string, unknown>;
+      if (typeof record.name === 'string') {
+        const { name: _legacyName, ...userData } = record;
+        return userData as T;
+      }
+    }
+    return data as T;
   }
 
-  /** Job name (stored inside `data.name`, mirroring the official client). */
+  /** Job name, with compatibility for pre-name-field data envelopes. */
   get name(): string | undefined {
+    const modern = this.topLevelName();
+    if (modern !== undefined) return modern;
     const data = this.raw.data;
-    if (data && typeof data === 'object' && 'name' in data) {
+    if (data && typeof data === 'object' && !Array.isArray(data) && 'name' in data) {
       const name = (data as Record<string, unknown>).name;
-      return name === undefined || name === null ? undefined : String(name);
+      return typeof name === 'string' ? name : undefined;
     }
     return undefined;
   }
 
+  private topLevelName(): string | undefined {
+    const name = this.raw.name;
+    return name === undefined || name === null ? undefined : String(name);
+  }
+
   get priority(): number {
     return Number(this.raw.priority ?? 0);
   }
@@ -164,16 +180,24 @@ export class Job<T = unknown> {
   }
 
   async discard(): Promise<void> {
-    await this.call({ cmd: 'Discard', id: this.id });
+    await this.call({
+      cmd: 'Discard',
+      id: this.id,
+      ...(this.token === undefined ? {} : { token: this.token }),
+    });
   }
 
   async promote(): Promise<void> {
     await this.call({ cmd: 'Promote', id: this.id });
   }
 
-  /** failed → waiting (BullMQ contract; MoveToWait over TCP). */
+  /** Move the job back to waiting, forwarding a Worker-owned lease when present. */
   async retry(): Promise<void> {
-    await this.call({ cmd: 'MoveToWait', id: this.id });
+    await this.call({
+      cmd: 'MoveToWait',
+      id: this.id,
+      ...(this.token === undefined ? {} : { token: this.token }),
+    });
   }
 
   async updateData(data: unknown): Promise<void> {
@@ -190,10 +214,20 @@ export class Job<T = unknown> {
   }
 
   async changeDelay(delayMs: number): Promise<void> {
-    await this.call({ cmd: 'ChangeDelay', id: this.id, delay: delayMs });
+    await this.call({
+      cmd: 'ChangeDelay',
+      id: this.id,
+      delay: delayMs,
+      ...(this.token === undefined ? {} : { token: this.token }),
+    });
   }
 
   async moveToDelayed(delayMs: number): Promise<void> {
-    await this.call({ cmd: 'MoveToDelayed', id: this.id, delay: delayMs });
+    await this.call({
+      cmd: 'MoveToDelayed',
+      id: this.id,
+      delay: delayMs,
+      ...(this.token === undefined ? {} : { token: this.token }),
+    });
   }
 }