@blokjs/runner 0.2.2 → 0.6.0

package/dist/concurrency/NatsKvConcurrencyBackend.js

@@ -0,0 +1,310 @@
+/**
+ * Tier 2 #6 follow-up · NATS KV-backed concurrency backend.
+ *
+ * Coordinates per-(workflow, concurrencyKey) lease state across processes
+ * via a single NATS JetStream KV value per bucket using revision-based
+ * compare-and-swap (OCC).
+ *
+ * Storage model: one KV key per `(workflowName, concurrencyKey)` pair.
+ * Value is a JSON `{leases: [{runId, expiresAt}]}` document. Bounded
+ * cardinality assumption — typical concurrency keys hold 1-50 active
+ * leases (per-tenant rate limits). For higher cardinality, a per-lease
+ * key model would scale better; revisit when needed.
+ *
+ * Atomicity: NATS KV's only guarantee is `kv.create(key, value)` (fails
+ * on conflict) and `kv.update(key, value, expectedRevision)` (fails on
+ * concurrent modification). The acquire loop reads → filters → checks
+ * limit → CAS update. On CAS failure, retry up to 10 times then
+ * fail-closed (deny the slot).
+ *
+ * Lease leak: each lease carries an `expiresAt`. Expired leases are
+ * lazy-purged inside the same `acquireSlot` call that observes them;
+ * an explicit `purgeExpired` sweep is also exposed for janitor use.
+ */
+import { ConcurrencyMetrics } from "../monitoring/ConcurrencyMetrics";
+const DEFAULT_BUCKET_NAME = "blok-concurrency";
+const MAX_CAS_RETRIES = 10;
+/**
+ * Read configuration from environment variables. Used by
+ * {@link createConcurrencyBackend} when the user opts into NATS KV.
+ */
+export function readNatsKvConfigFromEnv() {
+    const serversRaw = process.env.BLOK_CONCURRENCY_NATS_SERVERS ?? "nats://localhost:4222";
+    const servers = serversRaw
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+    return {
+        servers,
+        token: process.env.BLOK_CONCURRENCY_NATS_TOKEN,
+        user: process.env.BLOK_CONCURRENCY_NATS_USER,
+        pass: process.env.BLOK_CONCURRENCY_NATS_PASS,
+        bucketName: process.env.BLOK_CONCURRENCY_NATS_KV_BUCKET ?? DEFAULT_BUCKET_NAME,
+    };
+}
+export class NatsKvConcurrencyBackend {
+    name = "nats-kv";
+    nc = null;
+    kv = null;
+    config;
+    connected = false;
+    constructor(config) {
+        const env = readNatsKvConfigFromEnv();
+        this.config = {
+            servers: config?.servers ?? env.servers,
+            token: config?.token ?? env.token,
+            user: config?.user ?? env.user,
+            pass: config?.pass ?? env.pass,
+            bucketName: config?.bucketName ?? env.bucketName,
+        };
+    }
+    async connect() {
+        if (this.connected)
+            return;
+        // Security review FW-5 — refuse to start in production with the
+        // default bucket name. Two deployments sharing a NATS server with
+        // the default would contend on the same `(workflow, key)` buckets,
+        // silently corrupting concurrency state across tenants. The fix
+        // is operator-mandatory: set BLOK_CONCURRENCY_NATS_KV_BUCKET
+        // per-deployment.
+        const blokEnv = process.env.BLOK_ENV;
+        const nodeEnv = process.env.NODE_ENV;
+        const isProd = blokEnv === "production" || nodeEnv === "production";
+        if (isProd && this.config.bucketName === DEFAULT_BUCKET_NAME) {
+            throw new Error(`[blok] NATS KV concurrency backend refuses to start in production with the default bucket name ('${DEFAULT_BUCKET_NAME}'). Set BLOK_CONCURRENCY_NATS_KV_BUCKET to a deployment-unique value (e.g. 'blok-concurrency-acme-prod') to prevent cross-deployment collision on a shared NATS server.`);
+        }
+        let natsModule;
+        try {
+            natsModule = (await import("nats"));
+        }
+        catch (err) {
+            throw new Error(`NatsKvConcurrencyBackend requires the 'nats' package. Install it: \`bun add nats\` or \`npm install nats\`. Underlying error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        const connectOpts = { servers: this.config.servers };
+        if (this.config.token)
+            connectOpts.token = this.config.token;
+        if (this.config.user)
+            connectOpts.user = this.config.user;
+        if (this.config.pass)
+            connectOpts.pass = this.config.pass;
+        this.nc = await natsModule.connect(connectOpts);
+        // nats.js v2.x — KV lives at `nc.jetstream().views.kv(name)`. The
+        // returned `KV` auto-creates the bucket on first use given the
+        // connection has KV bucket-create permission. (Earlier versions
+        // exposed `nc.kv(name)` directly; that API was removed.)
+        const js = this.nc.jetstream();
+        this.kv = await js.views.kv(this.config.bucketName);
+        this.connected = true;
+    }
+    async disconnect() {
+        if (!this.connected)
+            return;
+        try {
+            await this.nc?.drain();
+        }
+        finally {
+            this.nc = null;
+            this.kv = null;
+            this.connected = false;
+        }
+    }
+    bucketKey(workflowName, concurrencyKey) {
+        // Use `__` (double underscore) — KV keys cannot contain `.` or
+        // `>` per NATS subject grammar; `__` is unambiguous and allows
+        // arbitrary workflow / key strings.
+        return `${this.encodeSegment(workflowName)}__${this.encodeSegment(concurrencyKey)}`;
+    }
+    encodeSegment(s) {
+        // NATS KV keys must match `[-/_=\.a-zA-Z0-9]+`. Replace anything
+        // outside the safe set with hex escape `_HHHH_` to keep the
+        // roundtrip lossless.
+        return s.replace(/[^-_=.a-zA-Z0-9]/g, (ch) => `_${ch.codePointAt(0)?.toString(16)}_`);
+    }
+    requireKv() {
+        if (!this.kv) {
+            throw new Error("NatsKvConcurrencyBackend not connected — call connect() first.");
+        }
+        return this.kv;
+    }
+    async acquireSlot(workflowName, concurrencyKey, concurrencyLimit, runId, leaseExpiresAt) {
+        const kv = this.requireKv();
+        const bucketKey = this.bucketKey(workflowName, concurrencyKey);
+        // PR 3 D2 — record OCC retry depth + outcome on every exit path.
+        const metricAttrs = { workflow_name: workflowName, concurrency_key: concurrencyKey };
+        for (let attempt = 0; attempt < MAX_CAS_RETRIES; attempt++) {
+            const entry = await this.safeGet(kv, bucketKey);
+            // PR 2 A6 — fetch failure (broker unreachable / non-NotFound
+            // error). Spinning 10× CAS retries on a connection problem just
+            // burns latency. Fail-fast so the trigger sees the issue and
+            // can fall back / alert. Existing run continues with no slot;
+            // the gate is conservative.
+            if (entry === "fetch-failed") {
+                console.warn(`[blok][concurrency][nats-kv] acquireSlot fetch-failed for ${workflowName}:${concurrencyKey} (attempt ${attempt + 1}); failing closed`);
+                ConcurrencyMetrics.getInstance().recordOccRetries({ ...metricAttrs, outcome: "fail-closed" }, attempt);
+                return { acquired: false, currentInFlight: -1 };
+            }
+            if (!entry) {
+                // Bucket doesn't exist — create with first lease.
+                const initial = { leases: [{ runId, expiresAt: leaseExpiresAt }] };
+                try {
+                    await kv.create(bucketKey, JSON.stringify(initial));
+                    ConcurrencyMetrics.getInstance().recordOccRetries({ ...metricAttrs, outcome: "success" }, attempt);
+                    return { acquired: true, currentInFlight: 1 };
+                }
+                catch {
+                    // Race — another process created. Retry.
+                    continue;
+                }
+            }
+            // Read current state, lazy-purge expired.
+            const current = this.parseBucket(entry);
+            const now = Date.now();
+            const active = current.leases.filter((l) => l.expiresAt > now);
+            // Idempotent re-acquire: refresh lease, don't grow count.
+            const existingIdx = active.findIndex((l) => l.runId === runId);
+            if (existingIdx >= 0) {
+                active[existingIdx] = { runId, expiresAt: leaseExpiresAt };
+                try {
+                    await kv.update(bucketKey, JSON.stringify({ leases: active }), entry.revision);
+                    ConcurrencyMetrics.getInstance().recordOccRetries({ ...metricAttrs, outcome: "success" }, attempt);
+                    return { acquired: true, currentInFlight: active.length };
+                }
+                catch {
+                    continue;
+                }
+            }
+            // Limit check.
+            if (active.length >= concurrencyLimit) {
+                ConcurrencyMetrics.getInstance().recordOccRetries({ ...metricAttrs, outcome: "denied" }, attempt);
+                return { acquired: false, currentInFlight: active.length };
+            }
+            // Insert + CAS.
+            const updated = { leases: [...active, { runId, expiresAt: leaseExpiresAt }] };
+            try {
+                await kv.update(bucketKey, JSON.stringify(updated), entry.revision);
+                ConcurrencyMetrics.getInstance().recordOccRetries({ ...metricAttrs, outcome: "success" }, attempt);
+                return { acquired: true, currentInFlight: updated.leases.length };
+            }
+            catch { }
+        }
+        // Retry exhausted — fail-closed.
+        console.warn(`[blok][concurrency][nats-kv] acquireSlot exhausted ${MAX_CAS_RETRIES} CAS retries for ${workflowName}:${concurrencyKey}; denying slot to runId=${runId}`);
+        ConcurrencyMetrics.getInstance().recordOccRetries({ ...metricAttrs, outcome: "fail-closed" }, MAX_CAS_RETRIES);
+        return { acquired: false, currentInFlight: -1 };
+    }
+    async releaseSlot(workflowName, concurrencyKey, runId) {
+        const kv = this.requireKv();
+        const bucketKey = this.bucketKey(workflowName, concurrencyKey);
+        for (let attempt = 0; attempt < MAX_CAS_RETRIES; attempt++) {
+            const entry = await this.safeGet(kv, bucketKey);
+            // PR 2 A6 — fetch failure on release. Lease will expire via
+            // TTL; safe to fail-fast.
+            if (entry === "fetch-failed") {
+                console.warn(`[blok][concurrency][nats-kv] releaseSlot fetch-failed for ${workflowName}:${concurrencyKey} (attempt ${attempt + 1}); lease for runId=${runId} will expire via TTL`);
+                return;
+            }
+            if (!entry)
+                return; // Idempotent — bucket already gone.
+            const current = this.parseBucket(entry);
+            const next = current.leases.filter((l) => l.runId !== runId);
+            // No-op when the runId wasn't holding a slot.
+            if (next.length === current.leases.length)
+                return;
+            if (next.length === 0) {
+                try {
+                    await kv.delete(bucketKey);
+                    return;
+                }
+                catch {
+                    // Another process beat us to delete — fine.
+                    return;
+                }
+            }
+            try {
+                await kv.update(bucketKey, JSON.stringify({ leases: next }), entry.revision);
+                return;
+            }
+            catch { }
+        }
+        console.warn(`[blok][concurrency][nats-kv] releaseSlot exhausted ${MAX_CAS_RETRIES} CAS retries for ${workflowName}:${concurrencyKey}; lease for runId=${runId} will expire via TTL`);
+    }
+    async purgeExpired(now) {
+        const kv = this.requireKv();
+        let purged = 0;
+        // **DRAIN THE ITERATOR FIRST.** nats.js v2.x `kv.keys()` returns a
+        // `QueuedIterator` backed by a JetStream watch consumer. Calling
+        // `kv.get()` mid-iteration interferes with the iterator's internal
+        // state — observed in practice that subsequent yields silently
+        // drop. Collect every key into an array before doing per-key
+        // reads, then operate on the array.
+        const allKeys = [];
+        for await (const key of await kv.keys()) {
+            allKeys.push(key);
+        }
+        for (const key of allKeys) {
+            const entry = await this.safeGet(kv, key);
+            // Treat both legitimate misses and fetch failures as "skip
+            // this bucket" — purge is a best-effort sweep.
+            if (!entry || entry === "fetch-failed")
+                continue;
+            const current = this.parseBucket(entry);
+            const active = current.leases.filter((l) => l.expiresAt > now);
+            const expired = current.leases.length - active.length;
+            if (expired === 0)
+                continue;
+            if (active.length === 0) {
+                try {
+                    await kv.delete(key);
+                    purged += expired;
+                }
+                catch {
+                    // best-effort
+                }
+                continue;
+            }
+            try {
+                await kv.update(key, JSON.stringify({ leases: active }), entry.revision);
+                purged += expired;
+            }
+            catch {
+                // CAS conflict — leave for next sweep.
+            }
+        }
+        return purged;
+    }
+    /**
+     * PR 2 A6 — distinguishes legitimate "key not found" from "broker
+     * unreachable / non-NotFound error". Returns:
+     *   - `NatsKvEntry` on a successful fetch.
+     *   - `null` when the key doesn't exist (NotFound code or null entry).
+     *   - `"fetch-failed"` for any other error (transient broker outage,
+     *     auth failure, network blip, etc.) so the OCC loop can fail-fast
+     *     instead of spinning 10× before fail-closing.
+     */
+    async safeGet(kv, key) {
+        try {
+            const e = await kv.get(key);
+            return e ?? null;
+        }
+        catch (err) {
+            // NATS surfaces "not found" via a code. Different `nats`
+            // package versions use different shapes; cover the common ones.
+            const code = err.code;
+            if (code === "NotFound" || code === "404")
+                return null;
+            return "fetch-failed";
+        }
+    }
+    parseBucket(entry) {
+        try {
+            const parsed = JSON.parse(entry.string());
+            if (!parsed || !Array.isArray(parsed.leases))
+                return { leases: [] };
+            return parsed;
+        }
+        catch {
+            return { leases: [] };
+        }
+    }
+}
+//# sourceMappingURL=NatsKvConcurrencyBackend.js.map

package/dist/concurrency/NatsKvConcurrencyBackend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"NatsKvConcurrencyBackend.js","sourceRoot":"","sources":["../../src/concurrency/NatsKvConcurrencyBackend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AAqBtE,MAAM,mBAAmB,GAAG,kBAAkB,CAAC;AAC/C,MAAM,eAAe,GAAG,EAAE,CAAC;AA2C3B;;;GAGG;AACH,MAAM,UAAU,uBAAuB;IACtC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,uBAAuB,CAAC;IACxF,MAAM,OAAO,GAAG,UAAU;SACxB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAC;IAClB,OAAO;QACN,OAAO;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,2BAA2B;QAC9C,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B;QAC5C,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B;QAC5C,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,mBAAmB;KAC9E,CAAC;AACH,CAAC;AAED,MAAM,OAAO,wBAAwB;IAC3B,IAAI,GAAG,SAAS,CAAC;IAElB,EAAE,GAA0B,IAAI,CAAC;IACjC,EAAE,GAAkB,IAAI,CAAC;IAChB,MAAM,CAA0B;IACzC,SAAS,GAAG,KAAK,CAAC;IAE1B,YAAY,MAAyC;QACpD,MAAM,GAAG,GAAG,uBAAuB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,GAAG;YACb,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,GAAG,CAAC,OAAO;YACvC,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,GAAG,CAAC,KAAK;YACjC,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,GAAG,CAAC,IAAI;YAC9B,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,GAAG,CAAC,IAAI;YAC9B,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,GAAG,CAAC,UAAU;SAChD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,gEAAgE;QAChE,kEAAkE;QAClE,mEAAmE;QACnE,gEAAgE;QAChE,6DAA6D;QAC7D,kBAAkB;QAClB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QACrC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QACrC,MAAM,MAAM,GAAG,OAAO,KAAK,YAAY,IAAI,OAAO,KAAK,YAAY,CAAC;QACpE,IAAI,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CACd,oGAAoG,mBAAmB,yKAAyK,CAChS,CAAC;QACH,CAAC;QAED,IAAI,UAAsB,CAAC;QAC3B,IAAI,CAAC;YACJ,UAAU,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,CAA0B,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CACd,iIAAiI,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACnL,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAA4B,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9E,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK;YAAE,WAAW,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7D,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI;YAAE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1D,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI;YAAE,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAE1D,IAAI,CAAC,EAAE,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAChD,kEAAkE;QAClE,+DAA+D;QAC/D,gEAAgE;QAChE,yDAAyD;QACzD,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,EAAE,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,UAAU;QACf,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;QACxB,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACxB,CAAC;IACF,CAAC;IAEO,SAAS,CAAC,YAAoB,EAAE,cAAsB;QAC7D,+DAA+D;QAC/D,+DAA+D;QAC/D,oCAAoC;QACpC,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,KAAK,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC;IACrF,CAAC;IAEO,aAAa,CAAC,CAAS;QAC9B,iEAAiE;QACjE,4DAA4D;QAC5D,sBAAsB;QACtB,OAAO,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACvF,CAAC;IAEO,SAAS;QAChB,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,IAAI,CAAC,EAAE,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,WAAW,CAChB,YAAoB,EACpB,cAAsB,EACtB,gBAAwB,EACxB,KAAa,EACb,cAAsB;QAEtB,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;QAE/D,iEAAiE;QACjE,MAAM,WAAW,GAAG,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC;QAErF,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,eAAe,EAAE,OAAO,EAAE,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAEhD,6DAA6D;YAC7D,gEAAgE;YAChE,6DAA6D;YAC7D,8DAA8D;YAC9D,4BAA4B;YAC5B,IAAI,KAAK,KAAK,cAAc,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CACX,6DAA6D,YAAY,IAAI,cAAc,aAAa,OAAO,GAAG,CAAC,mBAAmB,CACtI,CAAC;gBACF,kBAAkB,CAAC,WAAW,EAAE,CAAC,gBAAgB,CAAC,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,OAAO,CAAC,CAAC;gBACvG,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,kDAAkD;gBAClD,MAAM,OAAO,GAAgB,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;gBAChF,IAAI,CAAC;oBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;oBACpD,kBAAkB,CAAC,WAAW,EAAE,CAAC,gBAAgB,CAAC,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;oBACnG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;gBAC/C,CAAC;gBAAC,MAAM,CAAC;oBACR,yCAAyC;oBACzC,SAAS;gBACV,CAAC;YACF,CAAC;YAED,0CAA0C;YAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;YAE/D,0DAA0D;YAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;YAC/D,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;gBAC3D,IAAI,CAAC;oBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;oBAC/E,kBAAkB,CAAC,WAAW,EAAE,CAAC,gBAAgB,CAAC,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;oBACnG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC3D,CAAC;gBAAC,MAAM,CAAC;oBACR,SAAS;gBACV,CAAC;YACF,CAAC;YAED,eAAe;YACf,IAAI,MAAM,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;gBACvC,kBAAkB,CAAC,WAAW,EAAE,CAAC,gBAAgB,CAAC,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;gBAClG,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAC5D,CAAC;YAED,gBAAgB;YAChB,MAAM,OAAO,GAAgB,EAAE,MAAM,EAAE,CAAC,GAAG,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;YAC3F,IAAI,CAAC;gBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACpE,kBAAkB,CAAC,WAAW,EAAE,CAAC,gBAAgB,CAAC,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;gBACnG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnE,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACX,CAAC;QAED,iCAAiC;QACjC,OAAO,CAAC,IAAI,CACX,sDAAsD,eAAe,oBAAoB,YAAY,IAAI,cAAc,2BAA2B,KAAK,EAAE,CACzJ,CAAC;QACF,kBAAkB,CAAC,WAAW,EAAE,CAAC,gBAAgB,CAAC,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,eAAe,CAAC,CAAC;QAC/G,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC,EAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,YAAoB,EAAE,cAAsB,EAAE,KAAa;QAC5E,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;QAE/D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,eAAe,EAAE,OAAO,EAAE,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAChD,4DAA4D;YAC5D,0BAA0B;YAC1B,IAAI,KAAK,KAAK,cAAc,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CACX,6DAA6D,YAAY,IAAI,cAAc,aAAa,OAAO,GAAG,CAAC,sBAAsB,KAAK,sBAAsB,CACpK,CAAC;gBACF,OAAO;YACR,CAAC;YACD,IAAI,CAAC,KAAK;gBAAE,OAAO,CAAC,oCAAoC;YAExD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;YAE7D,8CAA8C;YAC9C,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM;gBAAE,OAAO;YAElD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;oBAC3B,OAAO;gBACR,CAAC;gBAAC,MAAM,CAAC;oBACR,4CAA4C;oBAC5C,OAAO;gBACR,CAAC;YACF,CAAC;YAED,IAAI,CAAC;gBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC7E,OAAO;YACR,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACX,CAAC;QAED,OAAO,CAAC,IAAI,CACX,sDAAsD,eAAe,oBAAoB,YAAY,IAAI,cAAc,qBAAqB,KAAK,sBAAsB,CACvK,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,mEAAmE;QACnE,iEAAiE;QACjE,mEAAmE;QACnE,+DAA+D;QAC/D,6DAA6D;QAC7D,oCAAoC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAC1C,2DAA2D;YAC3D,+CAA+C;YAC/C,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,cAAc;gBAAE,SAAS;YACjD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;YAC/D,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YACtD,IAAI,OAAO,KAAK,CAAC;gBAAE,SAAS;YAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACrB,MAAM,IAAI,OAAO,CAAC;gBACnB,CAAC;gBAAC,MAAM,CAAC;oBACR,cAAc;gBACf,CAAC;gBACD,SAAS;YACV,CAAC;YAED,IAAI,CAAC;gBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACzE,MAAM,IAAI,OAAO,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACR,uCAAuC;YACxC,CAAC;QACF,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,GAAW;QAC5C,IAAI,CAAC;YACJ,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,CAAC,IAAI,IAAI,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,yDAAyD;YACzD,gEAAgE;YAChE,MAAM,IAAI,GAAI,GAAyB,CAAC,IAAI,CAAC;YAC7C,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,KAAK;gBAAE,OAAO,IAAI,CAAC;YACvD,OAAO,cAAc,CAAC;QACvB,CAAC;IACF,CAAC;IAEO,WAAW,CAAC,KAAkB;QACrC,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAgB,CAAC;YACzD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;gBAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YACpE,OAAO,MAAM,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QACvB,CAAC;IACF,CAAC;CACD"}

package/dist/concurrency/QueueExpiredError.d.ts

@@ -0,0 +1,40 @@
+/**
+ * PR 1-5 polish · thrown by `TriggerBase.run()` when a queued run's
+ * `concurrencyQueueTimeoutMs` (PR 5 B2) has elapsed before the gate granted
+ * a slot.
+ *
+ * Distinct from `ConcurrencyLimitError`: a TTL-expired queued run will NEVER
+ * succeed (the timer won't re-fire). Conflating it with a transient denial
+ * misleads HTTP clients into retrying — the queued run is permanently dead,
+ * so the only correct response is `410 Gone`.
+ *
+ * Triggers catch this and translate:
+ * - HTTP trigger → `410 Gone` with structured body (no `Retry-After` — it
+ *   would contradict the 410 contract).
+ * - Worker trigger → ACK without retry (the in-process scheduler owns the
+ *   eventual dispatch and won't reschedule an expired run).
+ *
+ * The run record itself is already flipped to `expired` by
+ * `tracker.markRunExpired` before this error is thrown, so observability
+ * surfaces (Studio status badge, `RUN_EXPIRED` event) are independent of
+ * the transport-level response.
+ */
+export interface QueueExpiredInfo {
+    /** Workflow name whose queue TTL elapsed. */
+    workflowName: string;
+    /** Resolved key value (after evaluating the `concurrencyKey` expression). */
+    concurrencyKey: string;
+    /**
+     * The deadline that was breached (ms since epoch). The run was queued
+     * with `expiresAt = scheduledAt + concurrencyQueueTimeoutMs` and the
+     * dispatcher observed `now > expiresAt`.
+     */
+    queueExpiredAt: number;
+    /** Run id allocated by the tracer; flipped to `expired` before throw. */
+    runId: string;
+}
+export declare class QueueExpiredError extends Error {
+    readonly info: QueueExpiredInfo;
+    constructor(info: QueueExpiredInfo);
+}
+export declare function isQueueExpiredError(err: unknown): err is QueueExpiredError;

package/dist/concurrency/QueueExpiredError.js

@@ -0,0 +1,15 @@
+export class QueueExpiredError extends Error {
+    info;
+    constructor(info) {
+        super(`Queued run expired for workflow '${info.workflowName}' (key='${info.concurrencyKey}', ` +
+            `queueExpiredAt=${info.queueExpiredAt}). Run will not be retried.`);
+        this.name = "QueueExpiredError";
+        this.info = info;
+        // Restore prototype chain when extending Error in transpiled code.
+        Object.setPrototypeOf(this, QueueExpiredError.prototype);
+    }
+}
+export function isQueueExpiredError(err) {
+    return err instanceof QueueExpiredError;
+}
+//# sourceMappingURL=QueueExpiredError.js.map

package/dist/concurrency/QueueExpiredError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"QueueExpiredError.js","sourceRoot":"","sources":["../../src/concurrency/QueueExpiredError.ts"],"names":[],"mappings":"AAoCA,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC3B,IAAI,CAAmB;IAEvC,YAAY,IAAsB;QACjC,KAAK,CACJ,oCAAoC,IAAI,CAAC,YAAY,WAAW,IAAI,CAAC,cAAc,KAAK;YACvF,kBAAkB,IAAI,CAAC,cAAc,6BAA6B,CACnE,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,mEAAmE;QACnE,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC1D,CAAC;CACD;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC/C,OAAO,GAAG,YAAY,iBAAiB,CAAC;AACzC,CAAC"}

package/dist/concurrency/RedisConcurrencyBackend.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Tier C #4 follow-up · Redis-backed concurrency backend.
+ *
+ * Coordinates per-(workflow, concurrencyKey) lease state across processes
+ * via a single Redis key per bucket. Atomicity comes from server-side Lua
+ * scripts — `EVAL` runs single-threaded against the keyspace, so the
+ * read → filter → check-limit → write sequence is a single round-trip
+ * with no OCC retry loop (the headline win over the NATS KV backend's
+ * `WATCH`/`MULTI`/`EXEC`-style optimistic concurrency).
+ *
+ * Storage model: one Redis string key per `(workflowName, concurrencyKey)`
+ * bucket. Value is a JSON-encoded `{leases: [{runId, expiresAt}]}`
+ * document. Bounded-cardinality assumption identical to NATS KV — typical
+ * concurrency keys hold 1-50 active leases.
+ *
+ * Lease leak: each lease carries an `expiresAt`. Expired leases are
+ * lazy-purged inside the Lua script that observes them; an explicit
+ * `purgeExpired` SCAN sweep is also exposed for janitor use.
+ *
+ * Connection: ioredis is loaded via dynamic `import("ioredis")` so the
+ * dependency stays optional. Matches the existing pattern used by
+ * `triggers/worker`'s `RedisStreamsAdapter` and
+ * `triggers/pubsub`'s `RedisStreamsPubSubAdapter`.
+ */
+import type { ConcurrencySlotResult } from "../tracing/types";
+import type { ConcurrencyBackend } from "./ConcurrencyBackend";
+export interface RedisConcurrencyConfig {
+    /** Full Redis connection URL (e.g. `redis://[user:pass@]host:port[/db]`). Takes precedence over host/port. */
+    url?: string;
+    host?: string;
+    port?: number;
+    password?: string;
+    username?: string;
+    db?: number;
+    tls?: boolean;
+    /** Namespace prefix for every Redis key the backend touches. */
+    keyPrefix: string;
+}
+/**
+ * Read configuration from environment variables. Used by
+ * {@link createConcurrencyBackend} when the operator opts into Redis.
+ */
+export declare function readRedisConfigFromEnv(): RedisConcurrencyConfig;
+export declare class RedisConcurrencyBackend implements ConcurrencyBackend {
+    readonly name = "redis";
+    private client;
+    private readonly config;
+    private connected;
+    constructor(config?: Partial<RedisConcurrencyConfig>);
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    private bucketKey;
+    private encodeSegment;
+    private requireClient;
+    acquireSlot(workflowName: string, concurrencyKey: string, concurrencyLimit: number, runId: string, leaseExpiresAt: number): Promise<ConcurrencySlotResult>;
+    releaseSlot(workflowName: string, concurrencyKey: string, runId: string): Promise<void>;
+    purgeExpired(now: number): Promise<number>;
+    /**
+     * Decode the `{acquired, currentInFlight}` pair from a Lua eval result.
+     * ioredis returns Redis arrays as plain JS arrays of (string | number)
+     * — the script returns integers, so both elements should be numbers.
+     */
+    private parsePair;
+}