@trigger.dev/redis-worker 0.0.0-prerelease-20250503193704 → 0.0.0-process-keep-alive-20250618160907

package/dist/index.cjs

@@ -8,7 +8,6 @@ var crypto = require('crypto');
 require('@trigger.dev/core/v3/utils/flattenAttributes');
 var v3 = require('@trigger.dev/core/v3');
 var serverOnly = require('@trigger.dev/core/v3/serverOnly');
-var promClient = require('prom-client');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 
@@ -8743,7 +8742,7 @@ var require_Redis = __commonJS({
     var lodash_1 = require_lodash3();
     var Deque = require_denque();
     var debug = (0, utils_1.Debug)("redis");
-    var Redis3 = class _Redis extends Commander_1.default {
+    var Redis4 = class _Redis extends Commander_1.default {
       constructor(arg1, arg2, arg3) {
         super();
         this.status = "wait";
@@ -9306,12 +9305,12 @@ var require_Redis = __commonJS({
         }).catch(lodash_1.noop);
       }
     };
-    Redis3.Cluster = cluster_1.default;
-    Redis3.Command = Command_1.default;
-    Redis3.defaultOptions = RedisOptions_1.DEFAULT_REDIS_OPTIONS;
-    (0, applyMixin_1.default)(Redis3, events_1.EventEmitter);
-    (0, transaction_1.addTransactionSupport)(Redis3.prototype);
-    exports.default = Redis3;
+    Redis4.Cluster = cluster_1.default;
+    Redis4.Command = Command_1.default;
+    Redis4.defaultOptions = RedisOptions_1.DEFAULT_REDIS_OPTIONS;
+    (0, applyMixin_1.default)(Redis4, events_1.EventEmitter);
+    (0, transaction_1.addTransactionSupport)(Redis4.prototype);
+    exports.default = Redis4;
   }
 });
 
@@ -9509,6 +9508,39 @@ var SimpleQueue = class {
       throw e;
     }
   }
+  async enqueueOnce({
+    id,
+    job,
+    item,
+    attempt,
+    availableAt,
+    visibilityTimeoutMs
+  }) {
+    if (!id) {
+      throw new Error("enqueueOnce requires an id");
+    }
+    try {
+      const score = availableAt ? availableAt.getTime() : Date.now();
+      const deduplicationKey = nanoid();
+      const serializedItem = JSON.stringify({
+        job,
+        item,
+        visibilityTimeoutMs,
+        attempt,
+        deduplicationKey
+      });
+      const result = await this.redis.enqueueItemOnce(`queue`, `items`, id, score, serializedItem);
+      return result === 1;
+    } catch (e) {
+      this.logger.error(`SimpleQueue ${this.name}.enqueueOnce(): error enqueuing`, {
+        queue: this.name,
+        error: e,
+        id,
+        item
+      });
+      throw e;
+    }
+  }
   async dequeue(count = 1) {
     const now = Date.now();
     try {
@@ -9531,7 +9563,8 @@ var SimpleQueue = class {
             id,
             item: parsedItem,
             job: parsedItem.job,
-            timestamp
+            timestamp,
+            availableJobs: Object.keys(this.schema)
           });
           continue;
         }
@@ -9823,6 +9856,25 @@ var SimpleQueue = class {
         return 1
       `
     });
+    this.redis.defineCommand("enqueueItemOnce", {
+      numberOfKeys: 2,
+      lua: `
+        local queue = KEYS[1]
+        local items = KEYS[2]
+        local id = ARGV[1]
+        local score = ARGV[2]
+        local serializedItem = ARGV[3]
+
+        -- Only add if not exists
+        local added = redis.call('HSETNX', items, id, serializedItem)
+        if added == 1 then
+          redis.call('ZADD', queue, 'NX', score, id)
+          return 1
+        else
+          return 0
+        end
+      `
+    });
   }
 };
 
@@ -10179,6 +10231,173 @@ var BaseContext = (
 );
 var ROOT_CONTEXT = new BaseContext();
 
+// ../../node_modules/.pnpm/@opentelemetry+api@1.9.0/node_modules/@opentelemetry/api/build/esm/metrics/NoopMeter.js
+var __extends = /* @__PURE__ */ function() {
+  var extendStatics = function(d, b) {
+    extendStatics = Object.setPrototypeOf || { __proto__: [] } instanceof Array && function(d2, b2) {
+      d2.__proto__ = b2;
+    } || function(d2, b2) {
+      for (var p in b2) if (Object.prototype.hasOwnProperty.call(b2, p)) d2[p] = b2[p];
+    };
+    return extendStatics(d, b);
+  };
+  return function(d, b) {
+    if (typeof b !== "function" && b !== null)
+      throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
+    extendStatics(d, b);
+    function __() {
+      this.constructor = d;
+    }
+    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
+  };
+}();
+var NoopMeter = (
+  /** @class */
+  function() {
+    function NoopMeter2() {
+    }
+    NoopMeter2.prototype.createGauge = function(_name, _options) {
+      return NOOP_GAUGE_METRIC;
+    };
+    NoopMeter2.prototype.createHistogram = function(_name, _options) {
+      return NOOP_HISTOGRAM_METRIC;
+    };
+    NoopMeter2.prototype.createCounter = function(_name, _options) {
+      return NOOP_COUNTER_METRIC;
+    };
+    NoopMeter2.prototype.createUpDownCounter = function(_name, _options) {
+      return NOOP_UP_DOWN_COUNTER_METRIC;
+    };
+    NoopMeter2.prototype.createObservableGauge = function(_name, _options) {
+      return NOOP_OBSERVABLE_GAUGE_METRIC;
+    };
+    NoopMeter2.prototype.createObservableCounter = function(_name, _options) {
+      return NOOP_OBSERVABLE_COUNTER_METRIC;
+    };
+    NoopMeter2.prototype.createObservableUpDownCounter = function(_name, _options) {
+      return NOOP_OBSERVABLE_UP_DOWN_COUNTER_METRIC;
+    };
+    NoopMeter2.prototype.addBatchObservableCallback = function(_callback, _observables) {
+    };
+    NoopMeter2.prototype.removeBatchObservableCallback = function(_callback) {
+    };
+    return NoopMeter2;
+  }()
+);
+var NoopMetric = (
+  /** @class */
+  /* @__PURE__ */ function() {
+    function NoopMetric2() {
+    }
+    return NoopMetric2;
+  }()
+);
+var NoopCounterMetric = (
+  /** @class */
+  function(_super) {
+    __extends(NoopCounterMetric2, _super);
+    function NoopCounterMetric2() {
+      return _super !== null && _super.apply(this, arguments) || this;
+    }
+    NoopCounterMetric2.prototype.add = function(_value, _attributes) {
+    };
+    return NoopCounterMetric2;
+  }(NoopMetric)
+);
+var NoopUpDownCounterMetric = (
+  /** @class */
+  function(_super) {
+    __extends(NoopUpDownCounterMetric2, _super);
+    function NoopUpDownCounterMetric2() {
+      return _super !== null && _super.apply(this, arguments) || this;
+    }
+    NoopUpDownCounterMetric2.prototype.add = function(_value, _attributes) {
+    };
+    return NoopUpDownCounterMetric2;
+  }(NoopMetric)
+);
+var NoopGaugeMetric = (
+  /** @class */
+  function(_super) {
+    __extends(NoopGaugeMetric2, _super);
+    function NoopGaugeMetric2() {
+      return _super !== null && _super.apply(this, arguments) || this;
+    }
+    NoopGaugeMetric2.prototype.record = function(_value, _attributes) {
+    };
+    return NoopGaugeMetric2;
+  }(NoopMetric)
+);
+var NoopHistogramMetric = (
+  /** @class */
+  function(_super) {
+    __extends(NoopHistogramMetric2, _super);
+    function NoopHistogramMetric2() {
+      return _super !== null && _super.apply(this, arguments) || this;
+    }
+    NoopHistogramMetric2.prototype.record = function(_value, _attributes) {
+    };
+    return NoopHistogramMetric2;
+  }(NoopMetric)
+);
+var NoopObservableMetric = (
+  /** @class */
+  function() {
+    function NoopObservableMetric2() {
+    }
+    NoopObservableMetric2.prototype.addCallback = function(_callback) {
+    };
+    NoopObservableMetric2.prototype.removeCallback = function(_callback) {
+    };
+    return NoopObservableMetric2;
+  }()
+);
+var NoopObservableCounterMetric = (
+  /** @class */
+  function(_super) {
+    __extends(NoopObservableCounterMetric2, _super);
+    function NoopObservableCounterMetric2() {
+      return _super !== null && _super.apply(this, arguments) || this;
+    }
+    return NoopObservableCounterMetric2;
+  }(NoopObservableMetric)
+);
+var NoopObservableGaugeMetric = (
+  /** @class */
+  function(_super) {
+    __extends(NoopObservableGaugeMetric2, _super);
+    function NoopObservableGaugeMetric2() {
+      return _super !== null && _super.apply(this, arguments) || this;
+    }
+    return NoopObservableGaugeMetric2;
+  }(NoopObservableMetric)
+);
+var NoopObservableUpDownCounterMetric = (
+  /** @class */
+  function(_super) {
+    __extends(NoopObservableUpDownCounterMetric2, _super);
+    function NoopObservableUpDownCounterMetric2() {
+      return _super !== null && _super.apply(this, arguments) || this;
+    }
+    return NoopObservableUpDownCounterMetric2;
+  }(NoopObservableMetric)
+);
+var NOOP_METER = new NoopMeter();
+var NOOP_COUNTER_METRIC = new NoopCounterMetric();
+var NOOP_GAUGE_METRIC = new NoopGaugeMetric();
+var NOOP_HISTOGRAM_METRIC = new NoopHistogramMetric();
+var NOOP_UP_DOWN_COUNTER_METRIC = new NoopUpDownCounterMetric();
+var NOOP_OBSERVABLE_COUNTER_METRIC = new NoopObservableCounterMetric();
+var NOOP_OBSERVABLE_GAUGE_METRIC = new NoopObservableGaugeMetric();
+var NOOP_OBSERVABLE_UP_DOWN_COUNTER_METRIC = new NoopObservableUpDownCounterMetric();
+
+// ../../node_modules/.pnpm/@opentelemetry+api@1.9.0/node_modules/@opentelemetry/api/build/esm/metrics/Metric.js
+var ValueType;
+(function(ValueType2) {
+  ValueType2[ValueType2["INT"] = 0] = "INT";
+  ValueType2[ValueType2["DOUBLE"] = 1] = "DOUBLE";
+})(ValueType || (ValueType = {}));
+
 // ../../node_modules/.pnpm/@opentelemetry+api@1.9.0/node_modules/@opentelemetry/api/build/esm/context/NoopContextManager.js
 var __read3 = function(o, n) {
   var m = typeof Symbol === "function" && o[Symbol.iterator];
@@ -10541,8 +10760,54 @@ var SpanStatusCode;
   SpanStatusCode2[SpanStatusCode2["ERROR"] = 2] = "ERROR";
 })(SpanStatusCode || (SpanStatusCode = {}));
 
+// ../../node_modules/.pnpm/@opentelemetry+api@1.9.0/node_modules/@opentelemetry/api/build/esm/metrics/NoopMeterProvider.js
+var NoopMeterProvider = (
+  /** @class */
+  function() {
+    function NoopMeterProvider2() {
+    }
+    NoopMeterProvider2.prototype.getMeter = function(_name, _version, _options) {
+      return NOOP_METER;
+    };
+    return NoopMeterProvider2;
+  }()
+);
+var NOOP_METER_PROVIDER = new NoopMeterProvider();
+
+// ../../node_modules/.pnpm/@opentelemetry+api@1.9.0/node_modules/@opentelemetry/api/build/esm/api/metrics.js
+var API_NAME3 = "metrics";
+var MetricsAPI = (
+  /** @class */
+  function() {
+    function MetricsAPI2() {
+    }
+    MetricsAPI2.getInstance = function() {
+      if (!this._instance) {
+        this._instance = new MetricsAPI2();
+      }
+      return this._instance;
+    };
+    MetricsAPI2.prototype.setGlobalMeterProvider = function(provider) {
+      return registerGlobal(API_NAME3, provider, DiagAPI.instance());
+    };
+    MetricsAPI2.prototype.getMeterProvider = function() {
+      return getGlobal(API_NAME3) || NOOP_METER_PROVIDER;
+    };
+    MetricsAPI2.prototype.getMeter = function(name, version, options) {
+      return this.getMeterProvider().getMeter(name, version, options);
+    };
+    MetricsAPI2.prototype.disable = function() {
+      unregisterGlobal(API_NAME3, DiagAPI.instance());
+    };
+    return MetricsAPI2;
+  }()
+);
+
+// ../../node_modules/.pnpm/@opentelemetry+api@1.9.0/node_modules/@opentelemetry/api/build/esm/metrics-api.js
+var metrics = MetricsAPI.getInstance();
+
 // ../../node_modules/.pnpm/@opentelemetry+api@1.9.0/node_modules/@opentelemetry/api/build/esm/api/trace.js
-var API_NAME3 = "trace";
+var API_NAME4 = "trace";
 var TraceAPI = (
   /** @class */
   function() {
@@ -10564,20 +10829,20 @@ var TraceAPI = (
       return this._instance;
     };
     TraceAPI2.prototype.setGlobalTracerProvider = function(provider) {
-      var success = registerGlobal(API_NAME3, this._proxyTracerProvider, DiagAPI.instance());
+      var success = registerGlobal(API_NAME4, this._proxyTracerProvider, DiagAPI.instance());
       if (success) {
         this._proxyTracerProvider.setDelegate(provider);
       }
       return success;
     };
     TraceAPI2.prototype.getTracerProvider = function() {
-      return getGlobal(API_NAME3) || this._proxyTracerProvider;
+      return getGlobal(API_NAME4) || this._proxyTracerProvider;
     };
     TraceAPI2.prototype.getTracer = function(name, version) {
       return this.getTracerProvider().getTracer(name, version);
     };
     TraceAPI2.prototype.disable = function() {
-      unregisterGlobal(API_NAME3, DiagAPI.instance());
+      unregisterGlobal(API_NAME4, DiagAPI.instance());
       this._proxyTracerProvider = new ProxyTracerProvider();
     };
     return TraceAPI2;
@@ -10743,6 +11008,8 @@ function validateConcurrency(concurrency) {
     throw new TypeError("Expected `concurrency` to be a number from 1 and up");
   }
 }
+
+// src/worker.ts
 var defaultRetrySettings = {
   maxAttempts: 12,
   factor: 2,
@@ -10757,6 +11024,7 @@ var Worker = class _Worker {
     this.options = options;
     this.logger = options.logger ?? new logger$1.Logger("Worker", "debug");
     this.tracer = options.tracer ?? trace.getTracer(options.name);
+    this.meter = options.meter ?? metrics.getMeter(options.name);
     this.shutdownTimeoutMs = options.shutdownTimeoutMs ?? 6e4;
     const schema = Object.fromEntries(
       Object.entries(this.options.catalog).map(([key, value]) => [key, value.schema])
@@ -10771,56 +11039,47 @@ var Worker = class _Worker {
     const { workers = 1, tasksPerWorker = 1, limit = 10 } = options.concurrency ?? {};
     this.concurrency = { workers, tasksPerWorker, limit };
     this.limiter = pLimit(this.concurrency.limit);
-    this.metrics.register = options.metrics?.register;
-    if (!this.metrics.register) {
-      return;
-    }
-    this.metrics.enqueueDuration = new promClient.Histogram({
-      name: "redis_worker_enqueue_duration_seconds",
-      help: "The duration of enqueue operations",
-      labelNames: ["worker_name", "job_type", "has_available_at"],
-      buckets: [1e-3, 5e-3, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
-      registers: [this.metrics.register]
-    });
-    this.metrics.dequeueDuration = new promClient.Histogram({
-      name: "redis_worker_dequeue_duration_seconds",
-      help: "The duration of dequeue operations",
-      labelNames: ["worker_name", "worker_id", "task_count"],
-      buckets: [1e-3, 5e-3, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
-      registers: [this.metrics.register]
-    });
-    this.metrics.jobDuration = new promClient.Histogram({
-      name: "redis_worker_job_duration_seconds",
-      help: "The duration of job operations",
-      labelNames: ["worker_name", "worker_id", "batch_size", "job_type", "attempt"],
-      // use different buckets here as jobs can take a while to run
-      buckets: [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 45, 60],
-      registers: [this.metrics.register]
-    });
-    this.metrics.ackDuration = new promClient.Histogram({
-      name: "redis_worker_ack_duration_seconds",
-      help: "The duration of ack operations",
-      labelNames: ["worker_name"],
-      buckets: [1e-3, 5e-3, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
-      registers: [this.metrics.register]
-    });
-    this.metrics.redriveDuration = new promClient.Histogram({
-      name: "redis_worker_redrive_duration_seconds",
-      help: "The duration of redrive operations",
-      labelNames: ["worker_name"],
-      buckets: [1e-3, 5e-3, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
-      registers: [this.metrics.register]
-    });
-    this.metrics.rescheduleDuration = new promClient.Histogram({
-      name: "redis_worker_reschedule_duration_seconds",
-      help: "The duration of reschedule operations",
-      labelNames: ["worker_name"],
-      buckets: [1e-3, 5e-3, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
-      registers: [this.metrics.register]
+    const masterQueueObservableGauge = this.meter.createObservableGauge("redis_worker.queue.size", {
+      description: "The number of items in the queue",
+      unit: "items",
+      valueType: ValueType.INT
     });
+    masterQueueObservableGauge.addCallback(this.#updateQueueSizeMetric.bind(this));
+    const deadLetterQueueObservableGauge = this.meter.createObservableGauge(
+      "redis_worker.queue.dead_letter_size",
+      {
+        description: "The number of items in the dead letter queue",
+        unit: "items",
+        valueType: ValueType.INT
+      }
+    );
+    deadLetterQueueObservableGauge.addCallback(this.#updateDeadLetterQueueSizeMetric.bind(this));
+    const concurrencyLimitActiveObservableGauge = this.meter.createObservableGauge(
+      "redis_worker.concurrency.active",
+      {
+        description: "The number of active workers",
+        unit: "workers",
+        valueType: ValueType.INT
+      }
+    );
+    concurrencyLimitActiveObservableGauge.addCallback(
+      this.#updateConcurrencyLimitActiveMetric.bind(this)
+    );
+    const concurrencyLimitPendingObservableGauge = this.meter.createObservableGauge(
+      "redis_worker.concurrency.pending",
+      {
+        description: "The number of pending workers",
+        unit: "workers",
+        valueType: ValueType.INT
+      }
+    );
+    concurrencyLimitPendingObservableGauge.addCallback(
+      this.#updateConcurrencyLimitPendingMetric.bind(this)
+    );
   }
   subscriber;
   tracer;
+  meter;
   metrics = {};
   queue;
   jobs;
@@ -10831,6 +11090,28 @@ var Worker = class _Worker {
   shutdownTimeoutMs;
   // The p-limit limiter to control overall concurrency.
   limiter;
+  async #updateQueueSizeMetric(observableResult) {
+    const queueSize = await this.queue.size();
+    observableResult.observe(queueSize, {
+      worker_name: this.options.name
+    });
+  }
+  async #updateDeadLetterQueueSizeMetric(observableResult) {
+    const deadLetterQueueSize = await this.queue.sizeOfDeadLetterQueue();
+    observableResult.observe(deadLetterQueueSize, {
+      worker_name: this.options.name
+    });
+  }
+  async #updateConcurrencyLimitActiveMetric(observableResult) {
+    observableResult.observe(this.limiter.activeCount, {
+      worker_name: this.options.name
+    });
+  }
+  async #updateConcurrencyLimitPendingMetric(observableResult) {
+    observableResult.observe(this.limiter.pendingCount, {
+      worker_name: this.options.name
+    });
+  }
   start() {
     const { workers, tasksPerWorker } = this.concurrency;
     for (let i = 0; i < workers; i++) {
@@ -10898,6 +11179,56 @@ var Worker = class _Worker {
       }
     );
   }
+  /**
+   * Enqueues a job for processing once. If the job is already in the queue, it will be ignored.
+   * @param options - The enqueue options.
+   * @param options.id - Required unique identifier for the job.
+   * @param options.job - The job type from the worker catalog.
+   * @param options.payload - The job payload that matches the schema defined in the catalog.
+   * @param options.visibilityTimeoutMs - Optional visibility timeout in milliseconds. Defaults to value from catalog.
+   * @param options.availableAt - Optional date when the job should become available for processing. Defaults to now.
+   * @returns A promise that resolves when the job is enqueued.
+   */
+  enqueueOnce({
+    id,
+    job,
+    payload,
+    visibilityTimeoutMs,
+    availableAt
+  }) {
+    return startSpan(
+      this.tracer,
+      "enqueueOnce",
+      async (span) => {
+        const timeout = visibilityTimeoutMs ?? this.options.catalog[job]?.visibilityTimeoutMs;
+        if (!timeout) {
+          throw new Error(`No visibility timeout found for job ${String(job)} with id ${id}`);
+        }
+        span.setAttribute("job_visibility_timeout_ms", timeout);
+        return this.withHistogram(
+          this.metrics.enqueueDuration,
+          this.queue.enqueueOnce({
+            id,
+            job,
+            item: payload,
+            visibilityTimeoutMs: timeout,
+            availableAt
+          }),
+          {
+            job_type: String(job),
+            has_available_at: availableAt ? "true" : "false"
+          }
+        );
+      },
+      {
+        kind: SpanKind.PRODUCER,
+        attributes: {
+          job_type: String(job),
+          job_id: id
+        }
+      }
+    );
+  }
   /**
    * Reschedules an existing job to a new available date.
    * If the job isn't in the queue, it will be ignored.
@@ -11082,14 +11413,15 @@ var Worker = class _Worker {
     });
   }
   async withHistogram(histogram, promise, labels) {
-    if (!histogram || !this.metrics.register) {
+    if (!histogram) {
       return promise;
     }
-    const end = histogram.startTimer({ worker_name: this.options.name, ...labels });
+    const start = Date.now();
     try {
       return await promise;
     } finally {
-      end();
+      const duration = (Date.now() - start) / 1e3;
+      histogram.record(duration, { worker_name: this.options.name, ...labels });
     }
   }
   // A simple helper to delay for a given number of milliseconds.