omnigateway 0.12.3 → 0.12.4

package/README.md

@@ -332,7 +332,10 @@ changed without a restart: the six routing weights, `maxAttempts`,
 `ponytailMode` (`off` | `lite` | `full` | `ultra`),
 and the two body-capture switches. Edit them with `omni settings set` or in the
 console. `quotaPollIntervalMs` is the one exception: the poller reads it once at
-boot, so a change to it takes a restart. Snapshot retention —
+boot, so a change to it takes a restart. `requestDeadlineMs` defaults to `0`,
+which imposes no deadline at all; what that means for whatever sits in front of
+the gateway is
+[Behind a reverse proxy](docs/deploying.md#behind-a-reverse-proxy). Snapshot retention —
 `snapshotKeepLatest` and `snapshotMaxAgeDays` — is stored alongside them but
 deliberately edited on the Database screen instead; see
 [Snapshots and restore](docs/operations.md#snapshots-and-restore) for why.

package/bin/omni.js

@@ -19590,9 +19590,16 @@ function memoryCoord(options = {}) {
   const now = options.now ?? (() => Date.now());
   const windows = new Map;
   const gauges = new Map;
+  const named = (holder) => {
+    if (holder === undefined)
+      return;
+    if (holder === "")
+      throw new TypeError("gauge holder must be a non-empty string");
+    return holder;
+  };
   const liveSlots = (key) => {
     const at = now();
-    const held = (gauges.get(key) ?? []).filter((expiresAt) => expiresAt > at);
+    const held = (gauges.get(key) ?? []).filter((slot) => slot.expiresAt > at);
     if (held.length === 0)
       gauges.delete(key);
     else
@@ -19647,18 +19654,45 @@ function memoryCoord(options = {}) {
       }
     },
     gauge: {
-      acquire(key, ttlMs) {
+      acquire(key, ttlMs, rawHolder) {
+        const holder = named(rawHolder);
         const held = liveSlots(key);
+        const expiresAt = now() + ttlMs;
+        const mine = holder === undefined ? -1 : held.findIndex((s) => s.holder === holder);
+        if (mine !== -1 && holder !== undefined) {
+          held[mine] = { expiresAt, holder };
+          gauges.set(key, held);
+          return Promise.resolve(held.length - 1);
+        }
         const before = held.length;
-        held.push(now() + ttlMs);
+        held.push(holder === undefined ? { expiresAt } : { expiresAt, holder });
         gauges.set(key, held);
         return Promise.resolve(before);
       },
-      release(key) {
+      renew(key, holder, ttlMs) {
+        named(holder);
+        const held = liveSlots(key);
+        const mine = held.findIndex((slot) => slot.holder === holder);
+        if (mine !== -1) {
+          held[mine] = { expiresAt: now() + ttlMs, holder };
+          gauges.set(key, held);
+        }
+        return Promise.resolve();
+      },
+      release(key, holder) {
+        named(holder);
         const held = liveSlots(key);
-        held.sort((x, y) => x - y).shift();
+        if (holder === undefined) {
+          held.sort((x, y) => x.expiresAt - y.expiresAt).shift();
+        } else {
+          const mine = held.findIndex((slot) => slot.holder === holder);
+          if (mine !== -1)
+            held.splice(mine, 1);
+        }
         if (held.length === 0)
           gauges.delete(key);
+        else
+          gauges.set(key, held);
         return Promise.resolve();
       },
       read(key) {
@@ -21194,7 +21228,7 @@ var NODE_GRACE_MS = 60000;
 var DEFAULT_SETTINGS = {
   weights: { tier: 10, health: 3, quota: 2, load: 2, cost: 1, latency: 1 },
   maxAttempts: 3,
-  requestDeadlineMs: 120000,
+  requestDeadlineMs: 0,
   breakerThreshold: 3,
   breakerCooldownMs: 30000,
   logRetentionDays: 30,
@@ -37474,7 +37508,7 @@ function createServiceDeps(input2) {
 }
 
 // apps/cli/src/version.ts
-var VERSION = "0.12.3";
+var VERSION = "0.12.4";
 
 // apps/cli/src/commands/service.ts
 function gatewayArgv(root) {

package/gateway.js

@@ -24789,9 +24789,16 @@ function memoryCoord(options = {}) {
   const now = options.now ?? (() => Date.now());
   const windows = new Map;
   const gauges = new Map;
+  const named = (holder) => {
+    if (holder === undefined)
+      return;
+    if (holder === "")
+      throw new TypeError("gauge holder must be a non-empty string");
+    return holder;
+  };
   const liveSlots = (key) => {
     const at = now();
-    const held = (gauges.get(key) ?? []).filter((expiresAt) => expiresAt > at);
+    const held = (gauges.get(key) ?? []).filter((slot) => slot.expiresAt > at);
     if (held.length === 0)
       gauges.delete(key);
     else
@@ -24846,18 +24853,45 @@ function memoryCoord(options = {}) {
       }
     },
     gauge: {
-      acquire(key, ttlMs) {
+      acquire(key, ttlMs, rawHolder) {
+        const holder = named(rawHolder);
         const held = liveSlots(key);
+        const expiresAt = now() + ttlMs;
+        const mine = holder === undefined ? -1 : held.findIndex((s) => s.holder === holder);
+        if (mine !== -1 && holder !== undefined) {
+          held[mine] = { expiresAt, holder };
+          gauges.set(key, held);
+          return Promise.resolve(held.length - 1);
+        }
         const before = held.length;
-        held.push(now() + ttlMs);
+        held.push(holder === undefined ? { expiresAt } : { expiresAt, holder });
         gauges.set(key, held);
         return Promise.resolve(before);
       },
-      release(key) {
+      renew(key, holder, ttlMs) {
+        named(holder);
+        const held = liveSlots(key);
+        const mine = held.findIndex((slot) => slot.holder === holder);
+        if (mine !== -1) {
+          held[mine] = { expiresAt: now() + ttlMs, holder };
+          gauges.set(key, held);
+        }
+        return Promise.resolve();
+      },
+      release(key, holder) {
+        named(holder);
         const held = liveSlots(key);
-        held.sort((x, y) => x - y).shift();
+        if (holder === undefined) {
+          held.sort((x, y) => x.expiresAt - y.expiresAt).shift();
+        } else {
+          const mine = held.findIndex((slot) => slot.holder === holder);
+          if (mine !== -1)
+            held.splice(mine, 1);
+        }
         if (held.length === 0)
           gauges.delete(key);
+        else
+          gauges.set(key, held);
         return Promise.resolve();
       },
       read(key) {
@@ -26626,7 +26660,7 @@ var NODE_GRACE_MS = 60000;
 var DEFAULT_SETTINGS = {
   weights: { tier: 10, health: 3, quota: 2, load: 2, cost: 1, latency: 1 },
   maxAttempts: 3,
-  requestDeadlineMs: 120000,
+  requestDeadlineMs: 0,
   breakerThreshold: 3,
   breakerCooldownMs: 30000,
   logRetentionDays: 30,
@@ -54861,6 +54895,8 @@ var LONG_WINDOWS = ["5h", "1w"];
 var GRAIN_MS = { "5h": 60000, "1w": 3600000 };
 var SEED_LOCK_MS = 5000;
 var GAUGE_TTL_MS = 300000;
+var RENEW_INTERVAL_MS = 1e5;
+var MAX_RENEWED_MS = 86400000;
 function bucketKey(keyId, window2) {
   return `lim:${keyId}:${window2}`;
 }
@@ -54919,10 +54955,11 @@ class ApiKeyRateLimiter {
   }
   async admit(keyId, limits, requestId) {
     if (!anyLimit(limits))
-      return { release: () => {}, headroom: {} };
+      return { release: () => {}, renew: async () => {}, headroom: {} };
     const now = this.now();
     this.limits.set(keyId, limits);
-    const claim2 = await this.claim(keyId, now, true);
+    const holder = configured(limits.concurrency) ? crypto.randomUUID() : undefined;
+    const claim2 = await this.claim(keyId, now, true, holder);
     let admitted = false;
     try {
       const counters = await this.counters(keyId, limits, claim2, now, requestId);
@@ -54930,13 +54967,29 @@ class ApiKeyRateLimiter {
       await this.refuse(keyId, decision, now, requestId);
       admitted = true;
       let released = false;
+      const renewUntil = now + MAX_RENEWED_MS;
+      const renew = async () => {
+        if (released || holder === undefined)
+          return;
+        await this.coord.gauge.renew(keyId, holder, GAUGE_TTL_MS);
+      };
+      const renewal = holder === undefined ? null : setInterval(() => {
+        if (this.now() >= renewUntil) {
+          clearInterval(renewal ?? undefined);
+          return;
+        }
+        renew();
+      }, RENEW_INTERVAL_MS);
+      renewal?.unref?.();
       const release = () => {
         if (released)
           return;
         released = true;
-        this.coord.gauge.release(keyId);
+        if (renewal !== null)
+          clearInterval(renewal);
+        this.coord.gauge.release(keyId, holder);
       };
-      return { release, headroom: decision.headroom };
+      return { release, renew, headroom: decision.headroom };
     } finally {
       if (!admitted)
         await this.rollback(keyId, claim2);
@@ -54959,15 +55012,15 @@ class ApiKeyRateLimiter {
         await this.rollback(keyId, claim2);
     }
   }
-  async claim(keyId, now, gauge) {
+  async claim(keyId, now, gauge, holder) {
     const ring = await this.coord.window.claim(keyId, WINDOW_MS["1m"], now);
-    const concurrency = gauge ? await this.coord.gauge.acquire(keyId, GAUGE_TTL_MS) : await this.coord.gauge.read(keyId);
-    return { stamp: ring.stamp, gauge, before: { requests: ring.before, concurrency } };
+    const concurrency = gauge ? await this.coord.gauge.acquire(keyId, GAUGE_TTL_MS, holder) : await this.coord.gauge.read(keyId);
+    return { stamp: ring.stamp, gauge, holder, before: { requests: ring.before, concurrency } };
   }
   async rollback(keyId, claim2) {
     await this.coord.window.rollback(keyId, claim2.stamp);
     if (claim2.gauge)
-      await this.coord.gauge.release(keyId);
+      await this.coord.gauge.release(keyId, claim2.holder);
   }
   debit(keyId, usage) {
     const limits = this.limits.get(keyId);
@@ -55094,6 +55147,8 @@ function withReset(headroom, window2, resetAt) {
 // apps/gateway/src/dispatch/loadRegistry.ts
 var PREFIX6 = "load:";
 var SLOT_TTL_MS = 300000;
+var RENEW_INTERVAL_MS2 = 1e5;
+var MAX_RENEWED_MS2 = 86400000;
 var EWMA_ALPHA = 0.3;
 function createLoadRegistry(coord = memoryCoord(), now = Date.now) {
   const local = new Map;
@@ -55106,12 +55161,23 @@ function createLoadRegistry(coord = memoryCoord(), now = Date.now) {
       local.set(key, (local.get(key) ?? 0) + 1);
       if (provider !== undefined)
         providers.set(provider, (providers.get(provider) ?? 0) + 1);
-      coord.gauge.acquire(PREFIX6 + key, SLOT_TTL_MS);
+      const holder = crypto.randomUUID();
+      coord.gauge.acquire(PREFIX6 + key, SLOT_TTL_MS, holder);
+      const renewUntil = now() + MAX_RENEWED_MS2;
+      const renewal = setInterval(() => {
+        if (now() >= renewUntil) {
+          clearInterval(renewal);
+          return;
+        }
+        coord.gauge.renew(PREFIX6 + key, holder, SLOT_TTL_MS);
+      }, RENEW_INTERVAL_MS2);
+      renewal.unref?.();
       let released = false;
       return (ttftMs) => {
         if (released)
           return;
         released = true;
+        clearInterval(renewal);
         const prior = history.get(key)?.ewmaTtftMs ?? null;
         history.set(key, {
           lastReleasedAt: now(),
@@ -55129,7 +55195,7 @@ function createLoadRegistry(coord = memoryCoord(), now = Date.now) {
           else
             providers.delete(provider);
         }
-        coord.gauge.release(PREFIX6 + key);
+        coord.gauge.release(PREFIX6 + key, holder);
       };
     },
     counts() {
@@ -62302,7 +62368,7 @@ function createRing(limits = DEFAULT_RING_LIMITS) {
 }
 
 // apps/gateway/src/version.ts
-var VERSION = "0.12.3";
+var VERSION = "0.12.4";
 
 // apps/gateway/src/app.ts
 var ADMIN_SESSION_TTL_MS = 12 * 60 * 60 * 1000;
@@ -62580,19 +62646,35 @@ var WINDOW_ROLLBACK = `
 `;
 var GAUGE_ACQUIRE = `
   redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
+  local mine = redis.call('ZSCORE', KEYS[1], ARGV[3])
   local before = redis.call('ZCARD', KEYS[1])
+  if mine then before = before - 1 end
   redis.call('ZADD', KEYS[1], tonumber(ARGV[1]) + tonumber(ARGV[2]), ARGV[3])
-  redis.call('PEXPIRE', KEYS[1], ARGV[2])
+  if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then
+    redis.call('PEXPIRE', KEYS[1], ARGV[2])
+  end
   redis.call('SADD', KEYS[2], KEYS[1])
   return before
 `;
+var GAUGE_RENEW = `
+  redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
+  redis.call('ZADD', KEYS[1], 'XX', tonumber(ARGV[1]) + tonumber(ARGV[2]), ARGV[3])
+  if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then
+    redis.call('PEXPIRE', KEYS[1], ARGV[2])
+  end
+  return 0
+`;
 var GAUGE_READ = `
   redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
   return redis.call('ZCARD', KEYS[1])
 `;
 var GAUGE_RELEASE = `
   redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
-  redis.call('ZPOPMIN', KEYS[1])
+  if ARGV[2] == '' then
+    redis.call('ZPOPMIN', KEYS[1])
+  else
+    redis.call('ZREM', KEYS[1], ARGV[2])
+  end
   return 0
 `;
 var GAUGE_SNAPSHOT = `
@@ -62601,8 +62683,10 @@ var GAUGE_SNAPSHOT = `
     redis.call('ZREMRANGEBYSCORE', key, '-inf', ARGV[1])
     local count = redis.call('ZCARD', key)
     if count > 0 then
-      out[#out + 1] = key
-      out[#out + 1] = count
+      if string.sub(key, 1, string.len(ARGV[2])) == ARGV[2] then
+        out[#out + 1] = key
+        out[#out + 1] = count
+      end
     else
       redis.call('SREM', KEYS[1], key)
     end
@@ -62667,6 +62751,7 @@ var SCRIPTS = [
   WINDOW_CLAIM,
   WINDOW_ROLLBACK,
   GAUGE_ACQUIRE,
+  GAUGE_RENEW,
   GAUGE_READ,
   GAUGE_RELEASE,
   GAUGE_SNAPSHOT,
@@ -62701,6 +62786,10 @@ function redisCoord(deps) {
     const colon = name.indexOf(":");
     return `${NS}gidx:${colon === -1 ? name : name.slice(0, colon + 1)}`;
   };
+  const blankHolder = (holder) => {
+    if (holder === "")
+      throw new TypeError("gauge holder must be a non-empty string");
+  };
   let faults = 0;
   const fault = (error62) => {
     healthy = false;
@@ -62836,20 +62925,28 @@ function redisCoord(deps) {
       }
     },
     gauge: {
-      acquire(name, ttlMs) {
-        return attempt2(async () => await evalScript(GAUGE_ACQUIRE, [key("g", name), indexOf2(name)], [now(), ttlMs, crypto.randomUUID()]), () => fallback.gauge.acquire(name, ttlMs));
+      acquire(name, ttlMs, holder) {
+        blankHolder(holder);
+        return attempt2(async () => await evalScript(GAUGE_ACQUIRE, [key("g", name), indexOf2(name)], [now(), ttlMs, holder ?? crypto.randomUUID()]), () => fallback.gauge.acquire(name, ttlMs, holder));
+      },
+      renew(name, holder, ttlMs) {
+        blankHolder(holder);
+        return attempt2(async () => {
+          await evalScript(GAUGE_RENEW, [key("g", name)], [now(), ttlMs, holder]);
+        }, () => fallback.gauge.renew(name, holder, ttlMs));
       },
-      release(name) {
+      release(name, holder) {
+        blankHolder(holder);
         return attempt2(async () => {
-          await evalScript(GAUGE_RELEASE, [key("g", name)], [now()]);
-        }, () => fallback.gauge.release(name));
+          await evalScript(GAUGE_RELEASE, [key("g", name)], [now(), holder ?? ""]);
+        }, () => fallback.gauge.release(name, holder));
       },
       read(name) {
         return attempt2(async () => await evalScript(GAUGE_READ, [key("g", name)], [now()]), () => fallback.gauge.read(name));
       },
       snapshot(prefix2) {
         return attempt2(async () => {
-          const flat = await evalScript(GAUGE_SNAPSHOT, [indexOf2(prefix2)], [now()]);
+          const flat = await evalScript(GAUGE_SNAPSHOT, [indexOf2(prefix2)], [now(), key("g", prefix2)]);
           const out = new Map;
           for (let i = 0;i + 1 < flat.length; i += 2) {
             out.set(String(flat[i]).slice(key("g", "").length), Number(flat[i + 1]));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnigateway",
-  "version": "0.12.3",
+  "version": "0.12.4",
   "description": "Self-hosted AI gateway with Anthropic- and OpenAI-compatible APIs, an admin console, and a CLI",
   "license": "MIT",
   "author": "Harismawan <mail@harismawan.com>",