duron 0.3.0-beta.12 → 0.3.0-beta.14

package/src/adapters/postgres/base.ts

@@ -141,7 +141,15 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
    *
    * @returns Promise resolving to the job ID, or `null` if creation failed
    */
-  protected async _createJob({ queue, groupKey, input, timeoutMs, checksum, concurrencyLimit }: CreateJobOptions) {
+  protected async _createJob({
+    queue,
+    groupKey,
+    input,
+    timeoutMs,
+    checksum,
+    concurrencyLimit,
+    concurrencyStepLimit,
+  }: CreateJobOptions) {
     const [result] = await this.db
       .insert(this.tables.jobsTable)
       .values({
@@ -152,6 +160,7 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
         status: JOB_STATUS_CREATED,
         timeout_ms: timeoutMs,
         concurrency_limit: concurrencyLimit,
+        concurrency_step_limit: concurrencyStepLimit,
       })
       .returning({ id: this.tables.jobsTable.id })
 
@@ -667,7 +676,8 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
         j.finished_at as "finishedAt",
         j.created_at as "createdAt",
         j.updated_at as "updatedAt",
-        j.concurrency_limit as "concurrencyLimit"
+        j.concurrency_limit as "concurrencyLimit",
+        j.concurrency_step_limit as "concurrencyStepLimit"
     `),
     )
 
@@ -1038,6 +1048,7 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
         createdAt: jobsTable.created_at,
         updatedAt: jobsTable.updated_at,
         concurrencyLimit: jobsTable.concurrency_limit,
+        concurrencyStepLimit: jobsTable.concurrency_step_limit,
         clientId: jobsTable.client_id,
         durationMs,
       })
@@ -1241,6 +1252,7 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
         createdAt: jobsTable.created_at,
         updatedAt: jobsTable.updated_at,
         concurrencyLimit: jobsTable.concurrency_limit,
+        concurrencyStepLimit: jobsTable.concurrency_step_limit,
         clientId: jobsTable.client_id,
         durationMs,
       })
@@ -1418,21 +1430,29 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
 
   /**
    * Internal method to get spans for a job or step.
+   * For step queries, uses a recursive CTE to find all descendant spans.
    */
   protected async _getSpans(options: GetSpansOptions): Promise<GetSpansResult> {
     const spansTable = this.tables.spansTable
     const filters = options.filters ?? {}
 
-    // Build WHERE clause
-    const where = this._buildSpansWhereClause(options.jobId, options.stepId, filters)
-
     // Build sort
     const sortInput = options.sort ?? { field: 'startTimeUnixNano', order: 'asc' }
-    const sortFieldMap: Record<SpanSort['field'], any> = {
-      name: spansTable.name,
-      startTimeUnixNano: spansTable.start_time_unix_nano,
-      endTimeUnixNano: spansTable.end_time_unix_nano,
+    const sortFieldMap: Record<SpanSort['field'], string> = {
+      name: 'name',
+      startTimeUnixNano: 'start_time_unix_nano',
+      endTimeUnixNano: 'end_time_unix_nano',
     }
+    const sortField = sortFieldMap[sortInput.field]
+    const sortOrder = sortInput.order === 'asc' ? 'ASC' : 'DESC'
+
+    // For step queries, use a recursive CTE to get descendant spans
+    if (options.stepId) {
+      return this._getStepSpansRecursive(options.stepId, sortField, sortOrder, filters)
+    }
+
+    // Build WHERE clause for job queries
+    const where = this._buildSpansWhereClause(options.jobId, undefined, filters)
 
     // Get total count
     const total = await this.db.$count(spansTable, where)
@@ -1443,8 +1463,11 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
       }
     }
 
-    const sortField = sortFieldMap[sortInput.field]
-    const orderByClause = sortInput.order === 'asc' ? asc(sortField) : desc(sortField)
+    const sortFieldColumn = sortFieldMap[sortInput.field]
+    const orderByClause =
+      sortInput.order === 'asc'
+        ? asc(spansTable[sortFieldColumn as keyof typeof spansTable] as any)
+        : desc(spansTable[sortFieldColumn as keyof typeof spansTable] as any)
 
     const rows = await this.db
       .select({
@@ -1483,6 +1506,89 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
     }
   }
 
+  /**
+   * Get spans for a step using a recursive CTE to traverse the span hierarchy.
+   * This returns the step's span and all its descendant spans (children, grandchildren, etc.)
+   */
+  protected async _getStepSpansRecursive(
+    stepId: string,
+    sortField: string,
+    sortOrder: string,
+    _filters?: GetSpansOptions['filters'],
+  ): Promise<GetSpansResult> {
+    const schemaName = this.schema
+
+    // Use a recursive CTE to find all descendant spans
+    // 1. Base case: find the span with step_id = stepId
+    // 2. Recursive case: find all spans where parent_span_id = span_id of a span we've already found
+    const query = sql`
+      WITH RECURSIVE span_tree AS (
+        -- Base case: the span(s) for the step
+        SELECT * FROM ${sql.identifier(schemaName)}.spans WHERE step_id = ${stepId}::uuid
+        UNION ALL
+        -- Recursive case: children of spans we've found
+        SELECT s.* FROM ${sql.identifier(schemaName)}.spans s
+        INNER JOIN span_tree st ON s.parent_span_id = st.span_id
+      )
+      SELECT
+        id,
+        trace_id as "traceId",
+        span_id as "spanId",
+        parent_span_id as "parentSpanId",
+        job_id as "jobId",
+        step_id as "stepId",
+        name,
+        kind,
+        start_time_unix_nano as "startTimeUnixNano",
+        end_time_unix_nano as "endTimeUnixNano",
+        status_code as "statusCode",
+        status_message as "statusMessage",
+        attributes,
+        events
+      FROM span_tree
+      ORDER BY ${sql.identifier(sortField)} ${sql.raw(sortOrder)}
+    `
+
+    // Raw SQL returns numeric types as strings, so we type them as such
+    const rows = (await this.db.execute(query)) as unknown as Array<{
+      id: string | number
+      traceId: string
+      spanId: string
+      parentSpanId: string | null
+      jobId: string | null
+      stepId: string | null
+      name: string
+      kind: string | number
+      startTimeUnixNano: string | bigint | null
+      endTimeUnixNano: string | bigint | null
+      statusCode: string | number
+      statusMessage: string | null
+      attributes: Record<string, any>
+      events: Array<{ name: string; timeUnixNano: string; attributes?: Record<string, any> }>
+    }>
+
+    // Convert types: raw SQL returns numeric types as strings
+    const spans = rows.map((row) => ({
+      ...row,
+      // Convert id to number (bigserial comes as string from raw SQL)
+      id: typeof row.id === 'string' ? Number.parseInt(row.id, 10) : row.id,
+      // Convert kind and statusCode to proper types
+      kind: (typeof row.kind === 'string' ? Number.parseInt(row.kind, 10) : row.kind) as 0 | 1 | 2 | 3 | 4,
+      statusCode: (typeof row.statusCode === 'string' ? Number.parseInt(row.statusCode, 10) : row.statusCode) as
+        | 0
+        | 1
+        | 2,
+      // Convert BigInt to string for JSON serialization
+      startTimeUnixNano: row.startTimeUnixNano?.toString() ?? null,
+      endTimeUnixNano: row.endTimeUnixNano?.toString() ?? null,
+    }))
+
+    return {
+      spans,
+      total: spans.length,
+    }
+  }
+
   /**
    * Internal method to delete all spans for a job.
    */
@@ -1496,12 +1602,15 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
   }
 
   /**
-   * Build WHERE clause for spans queries.
-   * When querying by jobId or stepId, we find all spans that share the same trace_id
-   * as spans with that job/step. This includes spans from external libraries that
+   * Build WHERE clause for spans queries (used for job queries only).
+   * When querying by jobId, we find all spans that share the same trace_id
+   * as spans with that job. This includes spans from external libraries that
    * don't have the duron.job.id attribute but are part of the same trace.
+   *
+   * Note: Step queries are handled separately by _getStepSpansRecursive using
+   * a recursive CTE to traverse the span hierarchy.
    */
-  protected _buildSpansWhereClause(jobId?: string, stepId?: string, filters?: GetSpansOptions['filters']) {
+  protected _buildSpansWhereClause(jobId?: string, _stepId?: string, filters?: GetSpansOptions['filters']) {
     const spansTable = this.tables.spansTable
 
     // Build condition for finding spans by trace_id (includes external spans)
@@ -1514,12 +1623,6 @@ export class PostgresBaseAdapter<Database extends DrizzleDatabase, Connection> e
         spansTable.trace_id,
         this.db.select({ traceId: spansTable.trace_id }).from(spansTable).where(eq(spansTable.job_id, jobId)),
       )
-    } else if (stepId) {
-      // Find all spans that share a trace_id with any span that has this step_id
-      traceCondition = inArray(
-        spansTable.trace_id,
-        this.db.select({ traceId: spansTable.trace_id }).from(spansTable).where(eq(spansTable.step_id, stepId)),
-      )
     }
 
     return and(

package/src/adapters/postgres/schema.ts

@@ -37,6 +37,7 @@ export default function createSchema(schemaName: string) {
       finished_at: timestamp('finished_at', { withTimezone: true }),
       client_id: text('client_id'),
       concurrency_limit: integer('concurrency_limit').notNull().default(10),
+      concurrency_step_limit: integer('concurrency_step_limit').notNull().default(100),
       created_at: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
       updated_at: timestamp('updated_at', { withTimezone: true })
         .notNull()
@@ -59,6 +60,7 @@ export default function createSchema(schemaName: string) {
       index('idx_jobs_client_id').on(table.client_id),
       index('idx_jobs_checksum').on(table.checksum),
       index('idx_jobs_concurrency_limit').on(table.concurrency_limit),
+      index('idx_jobs_concurrency_step_limit').on(table.concurrency_step_limit),
       // Composite indexes
       index('idx_jobs_action_status').on(table.action_name, table.status),
       index('idx_jobs_action_group').on(table.action_name, table.group_key),

package/src/adapters/schemas.ts

@@ -45,6 +45,7 @@ export const JobSchema = z.object({
   createdAt: DateSchema,
   updatedAt: DateSchema,
   concurrencyLimit: z.coerce.number(),
+  concurrencyStepLimit: z.coerce.number(),
   clientId: z.string().nullable().optional(),
   /** Duration in milliseconds (finishedAt - startedAt). Null if job hasn't finished. */
   durationMs: z.coerce.number().nullable().default(null),
@@ -146,6 +147,8 @@ export const CreateJobOptionsSchema = z.object({
   timeoutMs: z.number(),
   /** The concurrency limit for this job's group */
   concurrencyLimit: z.number(),
+  /** The concurrency limit for steps within this job */
+  concurrencyStepLimit: z.number(),
 })
 
 export const RecoverJobsOptionsSchema = z.object({

package/src/client.ts

@@ -130,93 +130,145 @@ export interface TelemetryOptions {
   serviceName?: string
 }
 
-const BaseOptionsSchema = z.object({
+/**
+ * Base configuration options for a Duron client instance.
+ * These options control job fetching, concurrency, and recovery behavior.
+ */
+export interface BaseOptionsInput {
   /**
    * Unique identifier for this Duron instance.
    * Used for multi-process coordination and job ownership.
-   * Defaults to a random UUID if not provided.
+   * If not provided, a random UUID will be generated.
+   *
+   * @example 'worker-1', 'api-server', 'background-processor'
    */
-  id: z.string().optional(),
+  id?: string
 
   /**
-   * Synchronization pattern for fetching jobs.
-   * - `'pull'`: Periodically poll the database for new jobs
-   * - `'push'`: Listen for database notifications when jobs are available
-   * - `'hybrid'`: Use both pull and push patterns (recommended)
-   * - `false`: Disable automatic job fetching (manual fetching only)
+   * Synchronization pattern for fetching jobs from the database.
+   *
+   * - `'pull'`: Periodically poll the database for new jobs at `pullInterval`
+   * - `'push'`: Listen for database notifications when jobs are available (real-time)
+   * - `'hybrid'`: Use both pull and push patterns (recommended for reliability)
+   * - `false`: Disable automatic job fetching (use `fetch()` manually)
    *
    * @default 'hybrid'
+   *
+   * @example
+   * ```typescript
+   * // Real-time job processing with fallback polling
+   * syncPattern: 'hybrid'
+   *
+   * // Disable auto-fetching for API-only servers
+   * syncPattern: false
+   * ```
    */
-  syncPattern: z.union([z.literal('pull'), z.literal('push'), z.literal('hybrid'), z.literal(false)]).default('hybrid'),
+  syncPattern?: 'pull' | 'push' | 'hybrid' | false
 
   /**
-   * Interval in milliseconds between pull operations when using pull or hybrid sync pattern.
+   * Interval in milliseconds between pull operations when using `'pull'` or `'hybrid'` sync pattern.
+   * Lower values mean faster job pickup but more database queries.
    *
    * @default 5000
    */
-  pullInterval: z.number().default(5_000),
+  pullInterval?: number
 
   /**
-   * Maximum number of jobs to fetch in a single batch.
+   * Maximum number of jobs to fetch in a single batch from the database.
+   * Higher values reduce database round-trips but may increase memory usage.
    *
    * @default 10
    */
-  batchSize: z.number().default(10),
+  batchSize?: number
 
   /**
    * Maximum number of jobs that can run concurrently per action.
-   * This controls the concurrency limit for the action's fastq queue.
+   * This controls the concurrency limit for each action's internal queue.
+   * Use this to prevent any single action from consuming all resources.
    *
    * @default 100
    */
-  actionConcurrencyLimit: z.number().default(100),
+  actionConcurrencyLimit?: number
 
   /**
    * Maximum number of jobs that can run concurrently per group key.
    * Jobs with the same group key will respect this limit.
-   * This can be overridden using action -> groups -> concurrency.
+   * This is the default value; it can be overridden per-job using `action.groups.concurrency`.
    *
    * @default 10
+   *
+   * @example
+   * ```typescript
+   * // Limit concurrent jobs per user to 2
+   * groupConcurrencyLimit: 2
+   * ```
    */
-  groupConcurrencyLimit: z.number().default(10),
+  groupConcurrencyLimit?: number
 
   /**
    * Whether to run database migrations on startup.
    * When enabled, Duron will automatically apply pending migrations when the adapter starts.
+   * Disable this if you manage migrations separately or use a read-only database connection.
    *
    * @default true
    */
-  migrateOnStart: z.boolean().default(true),
+  migrateOnStart?: boolean
 
   /**
    * Whether to recover stuck jobs on startup.
    * Stuck jobs are jobs that were marked as active but the process that owned them
-   * is no longer running.
+   * is no longer running (e.g., after a crash or restart).
+   * These jobs will be reset to 'created' status so they can be picked up again.
    *
    * @default true
    */
-  recoverJobsOnStart: z.boolean().default(true),
+  recoverJobsOnStart?: boolean
 
   /**
    * Enable multi-process mode for job recovery.
    * When enabled, Duron will ping other processes to check if they're alive
-   * before recovering their jobs.
+   * before recovering their jobs. This prevents recovering jobs from processes
+   * that are still running but slow to respond.
+   *
+   * Only enable this if you're running multiple Duron instances sharing the same database.
    *
    * @default false
    */
-  multiProcessMode: z.boolean().default(false),
+  multiProcessMode?: boolean
 
   /**
    * Timeout in milliseconds to wait for process ping responses in multi-process mode.
    * Processes that don't respond within this timeout will have their jobs recovered.
+   * Increase this value if your processes may be temporarily unresponsive under load.
    *
-   * @default 5000 (5 seconds)
+   * @default 5000
    */
-  processTimeout: z.number().default(5 * 1000), // 5 seconds
+  processTimeout?: number
+}
+
+const BaseOptionsSchema = z.object({
+  id: z.string().optional(),
+  syncPattern: z.union([z.literal('pull'), z.literal('push'), z.literal('hybrid'), z.literal(false)]).default('hybrid'),
+  pullInterval: z.number().default(5_000),
+  batchSize: z.number().default(10),
+  actionConcurrencyLimit: z.number().default(100),
+  groupConcurrencyLimit: z.number().default(10),
+  migrateOnStart: z.boolean().default(true),
+  recoverJobsOnStart: z.boolean().default(true),
+  multiProcessMode: z.boolean().default(false),
+  processTimeout: z.number().default(5 * 1000),
 })
 
+// Compile-time check: ensure BaseOptionsInput is assignable to the Zod schema's input type
+type _EnsureBaseOptionsCompatible = BaseOptionsInput extends z.input<typeof BaseOptionsSchema>
+  ? true
+  : 'ERROR: BaseOptionsInput does not match Zod schema input type'
+
+declare const _baseOptionsCheck: _EnsureBaseOptionsCompatible
+const _checkOptions: _EnsureBaseOptionsCompatible = true
+
 /**
- * Options for configuring a Duron instance.
+ * Options for configuring a Duron client instance.
  *
  * @template TActions - Record of action definitions keyed by action name
  * @template TVariables - Type of variables available to actions
@@ -224,7 +276,7 @@ const BaseOptionsSchema = z.object({
 export interface ClientOptions<
   TActions extends Record<string, Action<any, any, TVariables>>,
   TVariables = Record<string, unknown>,
-> extends z.input<typeof BaseOptionsSchema> {
+> extends BaseOptionsInput {
   /**
    * The database adapter to use for storing jobs and steps.
    * Required.
@@ -510,6 +562,7 @@ export class Client<
       timeoutMs: action.expire,
       checksum: action.checksum,
       concurrencyLimit,
+      concurrencyStepLimit: action.steps.concurrency,
     })
 
     if (!jobId) {