@trigger.dev/redis-worker 4.0.0-v4-beta.21 → 4.0.0-v4-beta.22

package/dist/index.d.cts

@@ -62,6 +62,7 @@ declare class SimpleQueue<TMessageCatalog extends MessageCatalogSchema> {
     size({ includeFuture }?: {
         includeFuture?: boolean;
     }): Promise<number>;
+    getJob(id: string): Promise<QueueItem<TMessageCatalog> | null>;
     moveToDeadLetterQueue(id: string, errorMessage: string): Promise<void>;
     sizeOfDeadLetterQueue(): Promise<number>;
     redriveFromDeadLetterQueue(id: string): Promise<void>;
@@ -75,26 +76,44 @@ declare module "@internal/redis" {
         redriveFromDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, callback?: Callback<number>): Result<number, Context>;
         moveToDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, errorMessage: string, callback?: Callback<number>): Result<number, Context>;
         enqueueItemOnce(queue: string, items: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
+        getJob(queue: string, items: string, id: string, callback?: Callback<[string, string, string] | null>): Result<[string, string, string] | null, Context>;
     }
 }
 
+declare const CronSchema: z.ZodObject<{
+    cron: z.ZodString;
+    lastTimestamp: z.ZodOptional<z.ZodNumber>;
+    timestamp: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    timestamp: number;
+    cron: string;
+    lastTimestamp?: number | undefined;
+}, {
+    timestamp: number;
+    cron: string;
+    lastTimestamp?: number | undefined;
+}>;
+type CronSchema = z.infer<typeof CronSchema>;
 type WorkerCatalog = {
     [key: string]: {
         schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
         visibilityTimeoutMs: number;
         retry?: RetryOptions;
+        cron?: string;
+        jitterInMs?: number;
     };
 };
 type QueueCatalogFromWorkerCatalog<Catalog extends WorkerCatalog> = {
     [K in keyof Catalog]: Catalog[K]["schema"];
 };
-type JobHandler<Catalog extends WorkerCatalog, K extends keyof Catalog> = (params: {
+type JobHandlerParams<Catalog extends WorkerCatalog, K extends keyof Catalog> = {
     id: string;
     payload: z.infer<Catalog[K]["schema"]>;
     visibilityTimeoutMs: number;
     attempt: number;
     deduplicationKey?: string;
-}) => Promise<void>;
+};
+type JobHandler<Catalog extends WorkerCatalog, K extends keyof Catalog> = (params: JobHandlerParams<Catalog, K>) => Promise<void>;
 type WorkerConcurrencyOptions = {
     workers?: number;
     tasksPerWorker?: number;
@@ -129,7 +148,7 @@ declare class Worker<TCatalog extends WorkerCatalog> {
     private isShuttingDown;
     private concurrency;
     private shutdownTimeoutMs;
-    private limiter;
+    private limiters;
     constructor(options: WorkerOptions<TCatalog>);
     start(): this;
     /**
@@ -172,6 +191,7 @@ declare class Worker<TCatalog extends WorkerCatalog> {
      */
     reschedule(id: string, availableAt: Date): Promise<void>;
     ack(id: string): Promise<void>;
+    getJob(id: string): Promise<QueueItem<QueueCatalogFromWorkerCatalog<TCatalog>> | null>;
     /**
      * The main loop that each worker runs. It repeatedly polls for items,
      * processes them, and then waits before the next iteration.
@@ -183,6 +203,11 @@ declare class Worker<TCatalog extends WorkerCatalog> {
     private processItem;
     private withHistogram;
     private static delay;
+    private setupCron;
+    private enqueueCronJob;
+    private rescheduleCronJob;
+    private calculateNextScheduledAt;
+    private timestampIdentifier;
     private setupSubscriber;
     private handleRedriveMessage;
     private setupShutdownHandlers;
@@ -190,4 +215,4 @@ declare class Worker<TCatalog extends WorkerCatalog> {
     stop(): Promise<void>;
 }
 
-export { type AnyMessageCatalog, type AnyQueueItem, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type QueueItem, SimpleQueue, Worker, type WorkerCatalog, type WorkerConcurrencyOptions };
+export { type AnyMessageCatalog, type AnyQueueItem, CronSchema, type JobHandler, type JobHandlerParams, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type QueueItem, SimpleQueue, Worker, type WorkerCatalog, type WorkerConcurrencyOptions };

package/dist/index.d.ts

@@ -62,6 +62,7 @@ declare class SimpleQueue<TMessageCatalog extends MessageCatalogSchema> {
     size({ includeFuture }?: {
         includeFuture?: boolean;
     }): Promise<number>;
+    getJob(id: string): Promise<QueueItem<TMessageCatalog> | null>;
     moveToDeadLetterQueue(id: string, errorMessage: string): Promise<void>;
     sizeOfDeadLetterQueue(): Promise<number>;
     redriveFromDeadLetterQueue(id: string): Promise<void>;
@@ -75,26 +76,44 @@ declare module "@internal/redis" {
         redriveFromDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, callback?: Callback<number>): Result<number, Context>;
         moveToDeadLetterQueue(queue: string, items: string, dlq: string, dlqItems: string, id: string, errorMessage: string, callback?: Callback<number>): Result<number, Context>;
         enqueueItemOnce(queue: string, items: string, id: string, score: number, serializedItem: string, callback?: Callback<number>): Result<number, Context>;
+        getJob(queue: string, items: string, id: string, callback?: Callback<[string, string, string] | null>): Result<[string, string, string] | null, Context>;
     }
 }
 
+declare const CronSchema: z.ZodObject<{
+    cron: z.ZodString;
+    lastTimestamp: z.ZodOptional<z.ZodNumber>;
+    timestamp: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    timestamp: number;
+    cron: string;
+    lastTimestamp?: number | undefined;
+}, {
+    timestamp: number;
+    cron: string;
+    lastTimestamp?: number | undefined;
+}>;
+type CronSchema = z.infer<typeof CronSchema>;
 type WorkerCatalog = {
     [key: string]: {
         schema: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
         visibilityTimeoutMs: number;
         retry?: RetryOptions;
+        cron?: string;
+        jitterInMs?: number;
     };
 };
 type QueueCatalogFromWorkerCatalog<Catalog extends WorkerCatalog> = {
     [K in keyof Catalog]: Catalog[K]["schema"];
 };
-type JobHandler<Catalog extends WorkerCatalog, K extends keyof Catalog> = (params: {
+type JobHandlerParams<Catalog extends WorkerCatalog, K extends keyof Catalog> = {
     id: string;
     payload: z.infer<Catalog[K]["schema"]>;
     visibilityTimeoutMs: number;
     attempt: number;
     deduplicationKey?: string;
-}) => Promise<void>;
+};
+type JobHandler<Catalog extends WorkerCatalog, K extends keyof Catalog> = (params: JobHandlerParams<Catalog, K>) => Promise<void>;
 type WorkerConcurrencyOptions = {
     workers?: number;
     tasksPerWorker?: number;
@@ -129,7 +148,7 @@ declare class Worker<TCatalog extends WorkerCatalog> {
     private isShuttingDown;
     private concurrency;
     private shutdownTimeoutMs;
-    private limiter;
+    private limiters;
     constructor(options: WorkerOptions<TCatalog>);
     start(): this;
     /**
@@ -172,6 +191,7 @@ declare class Worker<TCatalog extends WorkerCatalog> {
      */
     reschedule(id: string, availableAt: Date): Promise<void>;
     ack(id: string): Promise<void>;
+    getJob(id: string): Promise<QueueItem<QueueCatalogFromWorkerCatalog<TCatalog>> | null>;
     /**
      * The main loop that each worker runs. It repeatedly polls for items,
      * processes them, and then waits before the next iteration.
@@ -183,6 +203,11 @@ declare class Worker<TCatalog extends WorkerCatalog> {
     private processItem;
     private withHistogram;
     private static delay;
+    private setupCron;
+    private enqueueCronJob;
+    private rescheduleCronJob;
+    private calculateNextScheduledAt;
+    private timestampIdentifier;
     private setupSubscriber;
     private handleRedriveMessage;
     private setupShutdownHandlers;
@@ -190,4 +215,4 @@ declare class Worker<TCatalog extends WorkerCatalog> {
     stop(): Promise<void>;
 }
 
-export { type AnyMessageCatalog, type AnyQueueItem, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type QueueItem, SimpleQueue, Worker, type WorkerCatalog, type WorkerConcurrencyOptions };
+export { type AnyMessageCatalog, type AnyQueueItem, CronSchema, type JobHandler, type JobHandlerParams, type MessageCatalogKey, type MessageCatalogSchema, type MessageCatalogValue, type QueueItem, SimpleQueue, Worker, type WorkerCatalog, type WorkerConcurrencyOptions };

package/dist/index.js

@@ -7,6 +7,8 @@ import { webcrypto } from 'node:crypto';
 import '@trigger.dev/core/v3/utils/flattenAttributes';
 import { calculateNextRetryDelay } from '@trigger.dev/core/v3';
 import { shutdownManager } from '@trigger.dev/core/v3/serverOnly';
+import { z } from 'zod';
+import { parseExpression } from 'cron-parser';
 
 const require = createRequire(import.meta.url || process.cwd() + '/index.js');
 var __create = Object.create;
@@ -8736,7 +8738,7 @@ var require_Redis = __commonJS({
     var lodash_1 = require_lodash3();
     var Deque = require_denque();
     var debug = (0, utils_1.Debug)("redis");
-    var Redis3 = class _Redis extends Commander_1.default {
+    var Redis4 = class _Redis extends Commander_1.default {
       constructor(arg1, arg2, arg3) {
         super();
         this.status = "wait";
@@ -9299,12 +9301,12 @@ var require_Redis = __commonJS({
         }).catch(lodash_1.noop);
       }
     };
-    Redis3.Cluster = cluster_1.default;
-    Redis3.Command = Command_1.default;
-    Redis3.defaultOptions = RedisOptions_1.DEFAULT_REDIS_OPTIONS;
-    (0, applyMixin_1.default)(Redis3, events_1.EventEmitter);
-    (0, transaction_1.addTransactionSupport)(Redis3.prototype);
-    exports.default = Redis3;
+    Redis4.Cluster = cluster_1.default;
+    Redis4.Command = Command_1.default;
+    Redis4.defaultOptions = RedisOptions_1.DEFAULT_REDIS_OPTIONS;
+    (0, applyMixin_1.default)(Redis4, events_1.EventEmitter);
+    (0, transaction_1.addTransactionSupport)(Redis4.prototype);
+    exports.default = Redis4;
   }
 });
 
@@ -9557,7 +9559,8 @@ var SimpleQueue = class {
             id,
             item: parsedItem,
             job: parsedItem.job,
-            timestamp
+            timestamp,
+            availableJobs: Object.keys(this.schema)
           });
           continue;
         }
@@ -9638,8 +9641,30 @@ var SimpleQueue = class {
       throw e;
     }
   }
+  async getJob(id) {
+    const result = await this.redis.getJob(`queue`, `items`, id);
+    if (!result) {
+      return null;
+    }
+    const [_, score, serializedItem] = result;
+    const item = JSON.parse(serializedItem);
+    return {
+      id,
+      job: item.job,
+      item: item.item,
+      visibilityTimeoutMs: item.visibilityTimeoutMs,
+      attempt: item.attempt ?? 0,
+      timestamp: new Date(Number(score)),
+      deduplicationKey: item.deduplicationKey ?? void 0
+    };
+  }
   async moveToDeadLetterQueue(id, errorMessage) {
     try {
+      this.logger.debug(`SimpleQueue ${this.name}.moveToDeadLetterQueue(): moving item to DLQ`, {
+        queue: this.name,
+        id,
+        errorMessage
+      });
       const result = await this.redis.moveToDeadLetterQueue(
         `queue`,
         `items`,
@@ -9760,6 +9785,25 @@ var SimpleQueue = class {
         return dequeued
       `
     });
+    this.redis.defineCommand("getJob", {
+      numberOfKeys: 2,
+      lua: `
+        local queue = KEYS[1]
+        local items = KEYS[2]
+        local jobId = ARGV[1]
+
+        local serializedItem = redis.call('HGET', items, jobId)
+
+        if serializedItem == false then
+          return nil
+        end
+
+        -- get the score from the queue sorted set
+        local score = redis.call('ZSCORE', queue, jobId)
+
+        return { jobId, score, serializedItem }
+      `
+    });
     this.redis.defineCommand("ackItem", {
       numberOfKeys: 2,
       lua: `
@@ -11001,6 +11045,11 @@ function validateConcurrency(concurrency) {
     throw new TypeError("Expected `concurrency` to be a number from 1 and up");
   }
 }
+var CronSchema = z.object({
+  cron: z.string(),
+  lastTimestamp: z.number().optional(),
+  timestamp: z.number()
+});
 var defaultRetrySettings = {
   maxAttempts: 12,
   factor: 2,
@@ -11029,7 +11078,6 @@ var Worker = class _Worker {
     this.jobs = options.jobs;
     const { workers = 1, tasksPerWorker = 1, limit = 10 } = options.concurrency ?? {};
     this.concurrency = { workers, tasksPerWorker, limit };
-    this.limiter = pLimit(this.concurrency.limit);
     const masterQueueObservableGauge = this.meter.createObservableGauge("redis_worker.queue.size", {
       description: "The number of items in the queue",
       unit: "items",
@@ -11080,7 +11128,7 @@ var Worker = class _Worker {
   concurrency;
   shutdownTimeoutMs;
   // The p-limit limiter to control overall concurrency.
-  limiter;
+  limiters = {};
   async #updateQueueSizeMetric(observableResult) {
     const queueSize = await this.queue.size();
     observableResult.observe(queueSize, {
@@ -11094,19 +11142,30 @@ var Worker = class _Worker {
     });
   }
   async #updateConcurrencyLimitActiveMetric(observableResult) {
-    observableResult.observe(this.limiter.activeCount, {
-      worker_name: this.options.name
-    });
+    for (const [workerId, limiter] of Object.entries(this.limiters)) {
+      observableResult.observe(limiter.activeCount, {
+        worker_name: this.options.name,
+        worker_id: workerId
+      });
+    }
   }
   async #updateConcurrencyLimitPendingMetric(observableResult) {
-    observableResult.observe(this.limiter.pendingCount, {
-      worker_name: this.options.name
-    });
+    for (const [workerId, limiter] of Object.entries(this.limiters)) {
+      observableResult.observe(limiter.pendingCount, {
+        worker_name: this.options.name,
+        worker_id: workerId
+      });
+    }
   }
   start() {
     const { workers, tasksPerWorker } = this.concurrency;
+    this.logger.info("Starting worker", {
+      workers,
+      tasksPerWorker,
+      concurrency: this.concurrency
+    });
     for (let i = 0; i < workers; i++) {
-      this.workerLoops.push(this.runWorkerLoop(`worker-${nanoid(12)}`, tasksPerWorker));
+      this.workerLoops.push(this.runWorkerLoop(`worker-${nanoid(12)}`, tasksPerWorker, i, workers));
     }
     this.setupShutdownHandlers();
     this.subscriber = createRedisClient(this.options.redisOptions, {
@@ -11118,6 +11177,7 @@ var Worker = class _Worker {
       }
     });
     this.setupSubscriber();
+    this.setupCron();
     return this;
   }
   /**
@@ -11256,37 +11316,78 @@ var Worker = class _Worker {
       }
     );
   }
+  async getJob(id) {
+    return this.queue.getJob(id);
+  }
   /**
    * The main loop that each worker runs. It repeatedly polls for items,
    * processes them, and then waits before the next iteration.
    */
-  async runWorkerLoop(workerId, taskCount) {
+  async runWorkerLoop(workerId, taskCount, workerIndex, totalWorkers) {
+    const limiter = pLimit(this.concurrency.limit);
+    this.limiters[workerId] = limiter;
     const pollIntervalMs = this.options.pollIntervalMs ?? 1e3;
     const immediatePollIntervalMs = this.options.immediatePollIntervalMs ?? 100;
+    const delayBetweenWorkers = this.options.pollIntervalMs ?? 1e3;
+    const delay = delayBetweenWorkers * (totalWorkers - workerIndex);
+    await _Worker.delay(delay);
+    this.logger.info("Starting worker loop", {
+      workerIndex,
+      totalWorkers,
+      delay,
+      workerId,
+      taskCount,
+      pollIntervalMs,
+      immediatePollIntervalMs,
+      concurrencyOptions: this.concurrency
+    });
     while (!this.isShuttingDown) {
-      if (this.limiter.activeCount + this.limiter.pendingCount >= this.concurrency.limit) {
+      if (limiter.activeCount + limiter.pendingCount >= this.concurrency.limit) {
+        this.logger.debug("Worker at capacity, waiting", {
+          workerId,
+          concurrencyOptions: this.concurrency,
+          activeCount: limiter.activeCount,
+          pendingCount: limiter.pendingCount
+        });
         await _Worker.delay(pollIntervalMs);
         continue;
       }
+      const $taskCount = Math.min(
+        taskCount,
+        this.concurrency.limit - limiter.activeCount - limiter.pendingCount
+      );
       try {
         const items = await this.withHistogram(
           this.metrics.dequeueDuration,
-          this.queue.dequeue(taskCount),
+          this.queue.dequeue($taskCount),
           {
             worker_id: workerId,
-            task_count: taskCount
+            task_count: $taskCount
           }
         );
         if (items.length === 0) {
+          this.logger.debug("No items to dequeue", {
+            workerId,
+            concurrencyOptions: this.concurrency,
+            activeCount: limiter.activeCount,
+            pendingCount: limiter.pendingCount
+          });
           await _Worker.delay(pollIntervalMs);
           continue;
         }
+        this.logger.debug("Dequeued items", {
+          workerId,
+          itemCount: items.length,
+          concurrencyOptions: this.concurrency,
+          activeCount: limiter.activeCount,
+          pendingCount: limiter.pendingCount
+        });
         for (const item of items) {
-          this.limiter(() => this.processItem(item, items.length, workerId)).catch(
-            (err) => {
-              this.logger.error("Unhandled error in processItem:", { error: err, workerId, item });
-            }
-          );
+          limiter(
+            () => this.processItem(item, items.length, workerId, limiter)
+          ).catch((err) => {
+            this.logger.error("Unhandled error in processItem:", { error: err, workerId, item });
+          });
         }
       } catch (error) {
         this.logger.error("Error dequeuing items:", { name: this.options.name, error });
@@ -11295,17 +11396,22 @@ var Worker = class _Worker {
       }
       await _Worker.delay(immediatePollIntervalMs);
     }
+    this.logger.info("Worker loop finished", { workerId });
   }
   /**
    * Processes a single item.
    */
-  async processItem({ id, job, item, visibilityTimeoutMs, attempt, timestamp, deduplicationKey }, batchSize, workerId) {
+  async processItem({ id, job, item, visibilityTimeoutMs, attempt, timestamp, deduplicationKey }, batchSize, workerId, limiter) {
     const catalogItem = this.options.catalog[job];
     const handler = this.jobs[job];
     if (!handler) {
       this.logger.error(`No handler found for job type: ${job}`);
       return;
     }
+    if (!catalogItem) {
+      this.logger.error(`No catalog item found for job type: ${job}`);
+      return;
+    }
     await startSpan(
       this.tracer,
       "processItem",
@@ -11321,6 +11427,9 @@ var Worker = class _Worker {
           }
         );
         await this.queue.ack(id, deduplicationKey);
+        if (catalogItem.cron) {
+          await this.rescheduleCronJob(job, catalogItem, item);
+        }
       },
       {
         kind: SpanKind.CONSUMER,
@@ -11331,9 +11440,9 @@ var Worker = class _Worker {
           job_timestamp: timestamp.getTime(),
           job_age_in_ms: Date.now() - timestamp.getTime(),
           worker_id: workerId,
-          worker_limit_concurrency: this.limiter.concurrency,
-          worker_limit_active: this.limiter.activeCount,
-          worker_limit_pending: this.limiter.pendingCount,
+          worker_limit_concurrency: limiter.concurrency,
+          worker_limit_active: limiter.activeCount,
+          worker_limit_pending: limiter.pendingCount,
           worker_name: this.options.name,
           batch_size: batchSize
         }
@@ -11367,6 +11476,9 @@ var Worker = class _Worker {
             errorMessage
           });
           await this.queue.moveToDeadLetterQueue(id, errorMessage);
+          if (catalogItem.cron) {
+            await this.rescheduleCronJob(job, catalogItem, item);
+          }
           return;
         }
         const retryDate = new Date(Date.now() + retryDelay);
@@ -11419,6 +11531,93 @@ var Worker = class _Worker {
   static delay(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
+  setupCron() {
+    const cronJobs = Object.entries(this.options.catalog).filter(([_, value]) => value.cron);
+    if (cronJobs.length === 0) {
+      return;
+    }
+    this.logger.info("Setting up cron jobs", {
+      cronJobs: cronJobs.map(([job, value]) => ({
+        job,
+        cron: value.cron,
+        jitterInMs: value.jitterInMs
+      }))
+    });
+    const enqueuePromises = cronJobs.map(
+      ([job, value]) => this.enqueueCronJob(value.cron, job, value.jitterInMs)
+    );
+    Promise.allSettled(enqueuePromises).then((results) => {
+      results.forEach((result) => {
+        if (result.status === "fulfilled") {
+          this.logger.info("Enqueued cron job", { result: result.value });
+        } else {
+          this.logger.error("Failed to enqueue cron job", { reason: result.reason });
+        }
+      });
+    });
+  }
+  async enqueueCronJob(cron, job, jitter, lastTimestamp) {
+    const scheduledAt = this.calculateNextScheduledAt(cron, lastTimestamp);
+    const identifier = [job, this.timestampIdentifier(scheduledAt)].join(":");
+    const appliedJitter = typeof jitter === "number" ? Math.random() * jitter - jitter / 2 : 0;
+    const availableAt = new Date(scheduledAt.getTime() + appliedJitter);
+    const enqueued = await this.enqueueOnce({
+      id: identifier,
+      job,
+      payload: {
+        timestamp: scheduledAt.getTime(),
+        lastTimestamp: lastTimestamp?.getTime(),
+        cron
+      },
+      availableAt
+    });
+    this.logger.info("Enqueued cron job", {
+      identifier,
+      cron,
+      job,
+      scheduledAt,
+      enqueued,
+      availableAt,
+      appliedJitter,
+      jitter
+    });
+    return {
+      identifier,
+      cron,
+      job,
+      scheduledAt,
+      enqueued
+    };
+  }
+  async rescheduleCronJob(job, catalogItem, item) {
+    if (!catalogItem.cron) {
+      return;
+    }
+    return this.enqueueCronJob(
+      catalogItem.cron,
+      job,
+      catalogItem.jitterInMs,
+      new Date(item.timestamp)
+    );
+  }
+  calculateNextScheduledAt(cron, lastTimestamp) {
+    const scheduledAt = parseExpression(cron, {
+      currentDate: lastTimestamp
+    }).next().toDate();
+    if (scheduledAt < /* @__PURE__ */ new Date()) {
+      return this.calculateNextScheduledAt(cron);
+    }
+    return scheduledAt;
+  }
+  timestampIdentifier(timestamp) {
+    const year = timestamp.getUTCFullYear();
+    const month = timestamp.getUTCMonth();
+    const day = timestamp.getUTCDate();
+    const hour = timestamp.getUTCHours();
+    const minute = timestamp.getUTCMinutes();
+    const second = timestamp.getUTCSeconds();
+    return `${year}-${month}-${day}-${hour}-${minute}-${second}`;
+  }
   setupSubscriber() {
     const channel = `${this.options.name}:redrive`;
     this.subscriber?.subscribe(channel, (err) => {
@@ -11475,6 +11674,6 @@ var Worker = class _Worker {
   }
 };
 
-export { SimpleQueue, Worker };
+export { CronSchema, SimpleQueue, Worker };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map