@trigger.dev/redis-worker 4.4.5 → 4.5.0-rc.0

package/dist/index.cjs

@@ -9384,12 +9384,20 @@ var require_built3 = __commonJS({
 // ../../internal-packages/redis/src/index.ts
 var import_ioredis = __toESM(require_built3());
 __toESM(require_built3());
+function defaultReconnectOnError(err) {
+  const msg = err.message ?? "";
+  if (msg.startsWith("READONLY") || msg.startsWith("LOADING") || msg.startsWith("UNBLOCKED")) {
+    return 2;
+  }
+  return false;
+}
 var defaultOptions = {
   retryStrategy: (times) => {
     const delay = Math.min(times * 50, 1e3);
     return delay;
   },
-  maxRetriesPerRequest: process.env.GITHUB_ACTIONS ? 50 : process.env.VITEST ? 5 : 20
+  maxRetriesPerRequest: process.env.GITHUB_ACTIONS ? 50 : process.env.VITEST ? 5 : 20,
+  reconnectOnError: defaultReconnectOnError
 };
 var logger = new logger$1.Logger("Redis", "debug");
 function createRedisClient(options, handlers) {
@@ -16353,9 +16361,581 @@ end
     }
   }
 };
+var BufferEntryStatus = zod.z.enum(["QUEUED", "DRAINING", "FAILED"]);
+var BufferEntryError = zod.z.object({
+  code: zod.z.string(),
+  message: zod.z.string()
+});
+var stringToInt = zod.z.string().transform((v, ctx) => {
+  const n = Number(v);
+  if (!Number.isInteger(n) || n < 0) {
+    ctx.addIssue({ code: zod.z.ZodIssueCode.custom, message: "expected non-negative integer string" });
+    return zod.z.NEVER;
+  }
+  return n;
+});
+var stringToDate = zod.z.string().transform((v, ctx) => {
+  const d = new Date(v);
+  if (Number.isNaN(d.getTime())) {
+    ctx.addIssue({ code: zod.z.ZodIssueCode.custom, message: "expected ISO date string" });
+    return zod.z.NEVER;
+  }
+  return d;
+});
+var stringToError = zod.z.string().transform((v, ctx) => {
+  try {
+    return BufferEntryError.parse(JSON.parse(v));
+  } catch {
+    ctx.addIssue({ code: zod.z.ZodIssueCode.custom, message: "expected JSON-encoded BufferEntryError" });
+    return zod.z.NEVER;
+  }
+});
+var BufferEntrySchema = zod.z.object({
+  runId: zod.z.string().min(1),
+  envId: zod.z.string().min(1),
+  orgId: zod.z.string().min(1),
+  payload: zod.z.string(),
+  status: BufferEntryStatus,
+  attempts: stringToInt,
+  createdAt: stringToDate,
+  lastError: stringToError.optional()
+});
+function serialiseSnapshot(snapshot) {
+  return JSON.stringify(snapshot);
+}
+function deserialiseSnapshot(serialised) {
+  return JSON.parse(serialised);
+}
+
+// src/mollifier/buffer.ts
+var MollifierBuffer = class {
+  redis;
+  entryTtlSeconds;
+  logger;
+  constructor(options) {
+    this.entryTtlSeconds = options.entryTtlSeconds;
+    this.logger = options.logger ?? new logger$1.Logger("MollifierBuffer", "debug");
+    this.redis = createRedisClient(
+      {
+        ...options.redisOptions,
+        retryStrategy(times) {
+          const delay = Math.min(times * 50, 1e3);
+          return delay;
+        },
+        maxRetriesPerRequest: 20
+      },
+      {
+        onError: (error) => {
+          this.logger.error("MollifierBuffer redis client error:", { error });
+        }
+      }
+    );
+    this.#registerCommands();
+  }
+  // Returns true if the entry was newly written; false if a duplicate runId
+  // was already buffered (idempotent no-op). Callers can use the boolean to
+  // record a duplicate-accept metric without affecting buffer state.
+  async accept(input) {
+    const entryKey = `mollifier:entries:${input.runId}`;
+    const queueKey = `mollifier:queue:${input.envId}`;
+    const orgsKey = "mollifier:orgs";
+    const createdAt = (/* @__PURE__ */ new Date()).toISOString();
+    const result = await this.redis.acceptMollifierEntry(
+      entryKey,
+      queueKey,
+      orgsKey,
+      input.runId,
+      input.envId,
+      input.orgId,
+      input.payload,
+      createdAt,
+      String(this.entryTtlSeconds),
+      "mollifier:org-envs:"
+    );
+    return result === 1;
+  }
+  async pop(envId) {
+    const queueKey = `mollifier:queue:${envId}`;
+    const orgsKey = "mollifier:orgs";
+    const entryPrefix = "mollifier:entries:";
+    const encoded = await this.redis.popAndMarkDraining(
+      queueKey,
+      orgsKey,
+      entryPrefix,
+      envId,
+      "mollifier:org-envs:"
+    );
+    if (!encoded) return null;
+    let raw;
+    try {
+      raw = JSON.parse(encoded);
+    } catch {
+      this.logger.error("MollifierBuffer.pop: failed to parse script result", { envId });
+      return null;
+    }
+    const parsed = BufferEntrySchema.safeParse(raw);
+    if (!parsed.success) {
+      this.logger.error("MollifierBuffer.pop: invalid entry shape", {
+        envId,
+        errors: parsed.error.flatten()
+      });
+      return null;
+    }
+    return parsed.data;
+  }
+  async getEntry(runId) {
+    const raw = await this.redis.hgetall(`mollifier:entries:${runId}`);
+    if (!raw || Object.keys(raw).length === 0) return null;
+    const parsed = BufferEntrySchema.safeParse(raw);
+    if (!parsed.success) {
+      this.logger.error("MollifierBuffer.getEntry: invalid entry shape", {
+        runId,
+        errors: parsed.error.flatten()
+      });
+      return null;
+    }
+    return parsed.data;
+  }
+  // Drainer walks these two methods to schedule pops with org-level
+  // fairness: one env per org per tick. The Lua scripts maintain both
+  // sets atomically with the per-env queues, so an org/env appears here
+  // exactly when at least one of its envs has a queued entry.
+  async listOrgs() {
+    return this.redis.smembers("mollifier:orgs");
+  }
+  async listEnvsForOrg(orgId) {
+    return this.redis.smembers(`mollifier:org-envs:${orgId}`);
+  }
+  async ack(runId) {
+    await this.redis.del(`mollifier:entries:${runId}`);
+  }
+  async requeue(runId) {
+    await this.redis.requeueMollifierEntry(
+      `mollifier:entries:${runId}`,
+      "mollifier:orgs",
+      "mollifier:queue:",
+      runId,
+      "mollifier:org-envs:"
+    );
+  }
+  // Returns true if the entry transitioned to FAILED; false if the entry no
+  // longer exists (TTL expired between pop and fail). Caller can use the
+  // boolean to skip downstream FAILED handling for ghost entries.
+  async fail(runId, error) {
+    const result = await this.redis.failMollifierEntry(
+      `mollifier:entries:${runId}`,
+      JSON.stringify(error)
+    );
+    return result === 1;
+  }
+  async getEntryTtlSeconds(runId) {
+    return this.redis.ttl(`mollifier:entries:${runId}`);
+  }
+  async evaluateTrip(envId, options) {
+    const rateKey = `mollifier:rate:${envId}`;
+    const trippedKey = `mollifier:tripped:${envId}`;
+    const result = await this.redis.mollifierEvaluateTrip(
+      rateKey,
+      trippedKey,
+      String(options.windowMs),
+      String(options.threshold),
+      String(options.holdMs)
+    );
+    return { count: result[0], tripped: result[1] === 1 };
+  }
+  async close() {
+    await this.redis.quit();
+  }
+  #registerCommands() {
+    this.redis.defineCommand("acceptMollifierEntry", {
+      numberOfKeys: 3,
+      lua: `
+        local entryKey = KEYS[1]
+        local queueKey = KEYS[2]
+        local orgsKey = KEYS[3]
+        local runId = ARGV[1]
+        local envId = ARGV[2]
+        local orgId = ARGV[3]
+        local payload = ARGV[4]
+        local createdAt = ARGV[5]
+        local ttlSeconds = tonumber(ARGV[6])
+        local orgEnvsPrefix = ARGV[7]
+
+        -- Idempotent: refuse if an entry for this runId already exists in any
+        -- state. Caller-side dedup is also enforced via API idempotency keys,
+        -- but the buffer must not double-enqueue if a caller retries.
+        if redis.call('EXISTS', entryKey) == 1 then
+          return 0
+        end
+
+        redis.call('HSET', entryKey,
+          'runId', runId,
+          'envId', envId,
+          'orgId', orgId,
+          'payload', payload,
+          'status', 'QUEUED',
+          'attempts', '0',
+          'createdAt', createdAt)
+        redis.call('EXPIRE', entryKey, ttlSeconds)
+        redis.call('LPUSH', queueKey, runId)
+        -- Org-level membership: maintained atomically with the per-env
+        -- queue so the drainer can walk orgs \u2192 envs-for-org and
+        -- schedule one env per org per tick. SADDs are idempotent if the
+        -- org/env are already tracked.
+        redis.call('SADD', orgsKey, orgId)
+        redis.call('SADD', orgEnvsPrefix .. orgId, envId)
+        return 1
+      `
+    });
+    this.redis.defineCommand("requeueMollifierEntry", {
+      numberOfKeys: 2,
+      lua: `
+        local entryKey = KEYS[1]
+        local orgsKey = KEYS[2]
+        local queuePrefix = ARGV[1]
+        local runId = ARGV[2]
+        local orgEnvsPrefix = ARGV[3]
+
+        local envId = redis.call('HGET', entryKey, 'envId')
+        local orgId = redis.call('HGET', entryKey, 'orgId')
+        if not envId then
+          return 0
+        end
+
+        local currentAttempts = redis.call('HGET', entryKey, 'attempts')
+        local nextAttempts = tonumber(currentAttempts or '0') + 1
+
+        redis.call('HSET', entryKey, 'status', 'QUEUED', 'attempts', tostring(nextAttempts))
+        redis.call('LPUSH', queuePrefix .. envId, runId)
+        -- Re-track the org/env: pop may have SREM'd them when the queue
+        -- last emptied. SADDs are idempotent if the values are still
+        -- present.
+        if orgId then
+          redis.call('SADD', orgsKey, orgId)
+          redis.call('SADD', orgEnvsPrefix .. orgId, envId)
+        end
+        return 1
+      `
+    });
+    this.redis.defineCommand("popAndMarkDraining", {
+      numberOfKeys: 2,
+      lua: `
+        local queueKey = KEYS[1]
+        local orgsKey = KEYS[2]
+        local entryPrefix = ARGV[1]
+        local envId = ARGV[2]
+        local orgEnvsPrefix = ARGV[3]
+
+        -- Helper: prune org-level membership when an env's queue empties.
+        -- Called only from the success branch where we know orgId from the
+        -- popped entry. The no-runId branch below can't reach this because
+        -- it has no entry to read orgId from \u2014 accept any stale org-envs
+        -- entries that result (bounded by env count, recovered next accept).
+        local function pruneOrgMembership(orgId)
+          if not orgId then return end
+          local orgEnvsKey = orgEnvsPrefix .. orgId
+          redis.call('SREM', orgEnvsKey, envId)
+          if redis.call('SCARD', orgEnvsKey) == 0 then
+            redis.call('SREM', orgsKey, orgId)
+          end
+        end
+
+        -- Loop to skip orphan queue references \u2014 runIds whose entry hash has
+        -- expired (TTL hit). HSET on a missing key would CREATE a partial
+        -- hash without a TTL, leaking memory. The loop is bounded by queue
+        -- length; entire Lua script remains atomic.
+        while true do
+          local runId = redis.call('RPOP', queueKey)
+          if not runId then
+            -- Queue is empty AND we have no entry to read orgId from, so
+            -- skip org-level cleanup. Stale org-envs entries are bounded
+            -- by env count and recovered on the next accept.
+            return nil
+          end
+
+          local entryKey = entryPrefix .. runId
+          if redis.call('EXISTS', entryKey) == 1 then
+            redis.call('HSET', entryKey, 'status', 'DRAINING')
+            local raw = redis.call('HGETALL', entryKey)
+            local result = {}
+            for i = 1, #raw, 2 do
+              result[raw[i]] = raw[i + 1]
+            end
+            -- Prune org-level membership if this pop drained the queue.
+            -- Atomic with the RPOP above \u2014 a concurrent accept AFTER this
+            -- script will SADD both back along with its LPUSH.
+            if redis.call('LLEN', queueKey) == 0 then
+              pruneOrgMembership(result['orgId'])
+            end
+            return cjson.encode(result)
+          end
+          -- Orphan queue reference: entry TTL expired while runId was queued.
+          -- Discard the reference and loop to the next.
+        end
+      `
+    });
+    this.redis.defineCommand("failMollifierEntry", {
+      numberOfKeys: 1,
+      lua: `
+        local entryKey = KEYS[1]
+        local errorPayload = ARGV[1]
+
+        -- Guard: never create a partial entry. If the hash expired between
+        -- pop and fail, the run is gone \u2014 nothing to mark FAILED.
+        if redis.call('EXISTS', entryKey) == 0 then
+          return 0
+        end
+
+        redis.call('HSET', entryKey, 'status', 'FAILED', 'lastError', errorPayload)
+        return 1
+      `
+    });
+    this.redis.defineCommand("mollifierEvaluateTrip", {
+      numberOfKeys: 2,
+      lua: `
+        local rateKey = KEYS[1]
+        local trippedKey = KEYS[2]
+        local windowMs = tonumber(ARGV[1])
+        local threshold = tonumber(ARGV[2])
+        local holdMs = tonumber(ARGV[3])
+
+        local count = redis.call('INCR', rateKey)
+        if count == 1 then
+          redis.call('PEXPIRE', rateKey, windowMs)
+        end
+
+        if count > threshold then
+          redis.call('PSETEX', trippedKey, holdMs, '1')
+        end
+
+        local tripped = redis.call('EXISTS', trippedKey)
+        return {count, tripped}
+      `
+    });
+  }
+};
+var MollifierDrainer = class {
+  buffer;
+  handler;
+  maxAttempts;
+  isRetryable;
+  pollIntervalMs;
+  maxOrgsPerTick;
+  logger;
+  limit;
+  // Rotation state. `orgCursor` advances through the active-orgs list.
+  // Each org has its own internal cursor in `perOrgEnvCursors` for
+  // cycling through that org's envs. Both reset on `start()`.
+  orgCursor = 0;
+  perOrgEnvCursors = /* @__PURE__ */ new Map();
+  isRunning = false;
+  stopping = false;
+  loopPromise = null;
+  constructor(options) {
+    this.buffer = options.buffer;
+    this.handler = options.handler;
+    this.maxAttempts = options.maxAttempts;
+    this.isRetryable = options.isRetryable;
+    this.pollIntervalMs = options.pollIntervalMs ?? 100;
+    this.maxOrgsPerTick = options.maxOrgsPerTick ?? 500;
+    this.logger = options.logger ?? new logger$1.Logger("MollifierDrainer", "debug");
+    this.limit = pLimit(options.concurrency);
+  }
+  async runOnce() {
+    const orgs = await this.buffer.listOrgs();
+    if (orgs.length === 0) return { drained: 0, failed: 0 };
+    const orgSlice = this.takeOrgSlice(orgs);
+    const envsByOrg = await Promise.all(
+      orgSlice.map((orgId) => this.buffer.listEnvsForOrg(orgId))
+    );
+    const targets = [];
+    for (let i = 0; i < orgSlice.length; i++) {
+      const orgId = orgSlice[i];
+      const envsForOrg = envsByOrg[i];
+      if (envsForOrg.length === 0) continue;
+      const envId = this.pickEnvForOrg(orgId, envsForOrg);
+      targets.push(envId);
+    }
+    const inflight = [];
+    for (const envId of targets) {
+      inflight.push(this.limit(() => this.processOneFromEnv(envId)));
+    }
+    const results = await Promise.all(inflight);
+    return {
+      drained: results.filter((r) => r === "drained").length,
+      failed: results.filter((r) => r === "failed").length
+    };
+  }
+  start() {
+    if (this.isRunning) return;
+    this.isRunning = true;
+    this.stopping = false;
+    this.orgCursor = 0;
+    this.perOrgEnvCursors = /* @__PURE__ */ new Map();
+    this.loopPromise = this.loop();
+  }
+  // Signal the loop to exit (`stopping = true`) and wait for it. With no
+  // timeout, wait indefinitely for the in-flight `runOnce` and its handlers
+  // to settle — same semantic as FairQueue / BatchQueue's `stop()`. With a
+  // timeout, race the loop promise against a deadline so a hung handler
+  // can't wedge the process past its termination grace period.
+  async stop(options = {}) {
+    if (!this.isRunning || !this.loopPromise) return;
+    this.stopping = true;
+    if (options.timeoutMs == null) {
+      await this.loopPromise;
+      return;
+    }
+    const timeoutSentinel = Symbol("mollifier.stop.timeout");
+    let timeoutHandle;
+    const timeoutPromise = new Promise((resolve) => {
+      timeoutHandle = setTimeout(() => resolve(timeoutSentinel), options.timeoutMs);
+    });
+    try {
+      const winner = await Promise.race([
+        this.loopPromise.then(() => "done"),
+        timeoutPromise
+      ]);
+      if (winner === timeoutSentinel) {
+        this.logger.warn(
+          "MollifierDrainer.stop: deadline exceeded; returning while loop iteration is in flight",
+          { timeoutMs: options.timeoutMs }
+        );
+      }
+    } finally {
+      if (timeoutHandle) clearTimeout(timeoutHandle);
+    }
+  }
+  // Transient Redis errors (e.g. a connection blip in `listOrgs` /
+  // `listEnvsForOrg` / `pop`) must not kill the polling loop permanently.
+  // We log each `runOnce` failure, back off so we don't spin tight on a
+  // sustained outage, and resume. The loop only exits when `stop()` flips
+  // `stopping`.
+  async loop() {
+    try {
+      let consecutiveErrors = 0;
+      while (!this.stopping) {
+        try {
+          const result = await this.runOnce();
+          consecutiveErrors = 0;
+          if (result.drained === 0 && result.failed === 0) {
+            await this.delay(this.pollIntervalMs);
+          }
+        } catch (err) {
+          consecutiveErrors += 1;
+          this.logger.error("MollifierDrainer.runOnce failed; backing off", {
+            err,
+            consecutiveErrors
+          });
+          await this.delay(this.backoffMs(consecutiveErrors));
+        }
+      }
+    } finally {
+      this.isRunning = false;
+    }
+  }
+  // Exponential backoff capped at 5s. Keeps the loop responsive after a
+  // brief blip while preventing a tight retry loop during a long Redis
+  // outage. 1 → 200ms, 2 → 400ms, 3 → 800ms, 4 → 1.6s, 5 → 3.2s, 6+ → 5s.
+  backoffMs(consecutiveErrors) {
+    const base = Math.max(this.pollIntervalMs, 100);
+    const capped = Math.min(base * 2 ** (consecutiveErrors - 1), 5e3);
+    return capped;
+  }
+  delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+  // Take up to `maxOrgsPerTick` orgs starting at the current cursor, with
+  // wrap-around. Cursor advances by 1 each tick so every org reaches
+  // every slot position (0..sliceSize-1) over a full cycle — no
+  // head-of-line bias within the slice. Orgs are sorted before slicing
+  // so rotation is deterministic regardless of Redis SET iteration order.
+  takeOrgSlice(orgs) {
+    const sorted = [...orgs].sort();
+    const n = sorted.length;
+    const sliceSize = Math.min(this.maxOrgsPerTick, n);
+    const start = this.orgCursor % n;
+    this.orgCursor = (this.orgCursor + 1) % Math.max(n, 1);
+    const end = start + sliceSize;
+    if (end <= n) return sorted.slice(start, end);
+    return [...sorted.slice(start), ...sorted.slice(0, end - n)];
+  }
+  // Pick one env from the org's active-envs list, rotating per org via
+  // the per-org cursor. Each org's cursor advances by 1 each visit, so
+  // an org with N envs cycles through them across N visits.
+  pickEnvForOrg(orgId, envsForOrg) {
+    const sorted = [...envsForOrg].sort();
+    const cursor = this.perOrgEnvCursors.get(orgId) ?? 0;
+    const idx = cursor % sorted.length;
+    this.perOrgEnvCursors.set(orgId, (cursor + 1) % sorted.length);
+    return sorted[idx];
+  }
+  // A failure for one env (e.g. a Redis hiccup mid-batch in `pop`, or in
+  // `requeue`/`fail` during error recovery inside `processEntry`) must not
+  // poison the rest of the batch — `Promise.all` would otherwise reject and
+  // bubble all the way to `loop()`. Catch both stages here so the failed env
+  // is just counted as "failed" for this tick and we move on.
+  async processOneFromEnv(envId) {
+    let entry;
+    try {
+      entry = await this.buffer.pop(envId);
+    } catch (err) {
+      this.logger.error("MollifierDrainer.pop failed", { envId, err });
+      return "failed";
+    }
+    if (!entry) return "empty";
+    try {
+      return await this.processEntry(entry);
+    } catch (err) {
+      this.logger.error("MollifierDrainer.processEntry failed", {
+        envId,
+        runId: entry.runId,
+        err
+      });
+      return "failed";
+    }
+  }
+  async processEntry(entry) {
+    try {
+      const payload = deserialiseSnapshot(entry.payload);
+      await this.handler({
+        runId: entry.runId,
+        envId: entry.envId,
+        orgId: entry.orgId,
+        payload,
+        attempts: entry.attempts,
+        createdAt: entry.createdAt
+      });
+      await this.buffer.ack(entry.runId);
+      return "drained";
+    } catch (err) {
+      const nextAttempts = entry.attempts + 1;
+      if (this.isRetryable(err) && nextAttempts < this.maxAttempts) {
+        await this.buffer.requeue(entry.runId);
+        this.logger.warn("MollifierDrainer: retryable error, requeued", {
+          runId: entry.runId,
+          attempts: nextAttempts
+        });
+        return "failed";
+      }
+      const code = err instanceof Error ? err.name : "Unknown";
+      const message = err instanceof Error ? err.message : String(err);
+      await this.buffer.fail(entry.runId, { code, message });
+      this.logger.error("MollifierDrainer: terminal failure", {
+        runId: entry.runId,
+        code,
+        message
+      });
+      return "failed";
+    }
+  }
+};
 
 exports.BaseScheduler = BaseScheduler;
 exports.BatchedSpanManager = BatchedSpanManager;
+exports.BufferEntryError = BufferEntryError;
+exports.BufferEntrySchema = BufferEntrySchema;
+exports.BufferEntryStatus = BufferEntryStatus;
 exports.CallbackFairQueueKeyProducer = CallbackFairQueueKeyProducer;
 exports.ConcurrencyManager = ConcurrencyManager;
 exports.CronSchema = CronSchema;
@@ -16371,6 +16951,8 @@ exports.ImmediateRetry = ImmediateRetry;
 exports.LinearBackoffRetry = LinearBackoffRetry;
 exports.MasterQueue = MasterQueue;
 exports.MessagingAttributes = MessagingAttributes;
+exports.MollifierBuffer = MollifierBuffer;
+exports.MollifierDrainer = MollifierDrainer;
 exports.NoRetry = NoRetry;
 exports.NoopScheduler = NoopScheduler;
 exports.RoundRobinScheduler = RoundRobinScheduler;
@@ -16382,7 +16964,9 @@ exports.Worker = Worker;
 exports.WorkerQueueManager = WorkerQueueManager;
 exports.createDefaultRetryStrategy = createDefaultRetryStrategy;
 exports.defaultRetryOptions = defaultRetryOptions;
+exports.deserialiseSnapshot = deserialiseSnapshot;
 exports.isAbortError = isAbortError;
 exports.noopTelemetry = noopTelemetry;
+exports.serialiseSnapshot = serialiseSnapshot;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map