@getstrata/core 1.1.9 → 2.0.1

package/dist/entries/metrics/prometheus.js

@@ -1,6 +1,23 @@
 // @bun
 // ../../src/core/metrics/prometheus.ts
+var HTTP_DURATION_BUCKETS_MS = [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 1e4];
+function emptyHistogram() {
+  return {
+    cumulative: HTTP_DURATION_BUCKETS_MS.map(() => 0),
+    sum: 0,
+    count: 0
+  };
+}
+
 class PrometheusRegistry {
+  maxSeries;
+  seriesLabels = new Map;
+  constructor(maxSeries = 4096) {
+    this.maxSeries = maxSeries;
+    if (!Number.isSafeInteger(maxSeries) || maxSeries < 1) {
+      throw new Error("Metrics maxSeries must be a positive safe integer.");
+    }
+  }
   httpRequestsTotal = new Map;
   httpRequestDurationMs = new Map;
   incrementHttpRequest(labels) {
@@ -8,10 +25,20 @@ class PrometheusRegistry {
     this.httpRequestsTotal.set(key, (this.httpRequestsTotal.get(key) ?? 0) + 1);
   }
   observeHttpDuration(labels, durationMs) {
+    if (!Number.isFinite(durationMs) || durationMs < 0)
+      return;
     const key = this.metricKey(labels);
-    const samples = this.httpRequestDurationMs.get(key) ?? [];
-    samples.push(durationMs);
-    this.httpRequestDurationMs.set(key, samples);
+    const histogram = this.httpRequestDurationMs.get(key) ?? emptyHistogram();
+    histogram.count += 1;
+    histogram.sum += durationMs;
+    for (let index = 0;index < HTTP_DURATION_BUCKETS_MS.length; index += 1) {
+      const bound = HTTP_DURATION_BUCKETS_MS[index];
+      if (bound !== undefined && durationMs <= bound) {
+        const count = histogram.cumulative[index] ?? 0;
+        histogram.cumulative[index] = count + 1;
+      }
+    }
+    this.httpRequestDurationMs.set(key, histogram);
   }
   renderMetrics() {
     const lines = [
@@ -21,16 +48,22 @@ class PrometheusRegistry {
     for (const [key, value] of this.httpRequestsTotal) {
       lines.push(`http_requests_total{${key}} ${value}`);
     }
-    lines.push("# HELP http_request_duration_ms_sum Sum of HTTP request durations in milliseconds.", "# TYPE http_request_duration_ms_sum counter");
-    for (const [key, samples] of this.httpRequestDurationMs) {
-      const sum = samples.reduce((total, sample) => total + sample, 0);
-      lines.push(`http_request_duration_ms_sum{${key}} ${sum}`);
+    lines.push("# HELP http_request_duration_ms HTTP request duration in milliseconds.", "# TYPE http_request_duration_ms histogram");
+    for (const [key, histogram] of this.httpRequestDurationMs) {
+      for (let index = 0;index < HTTP_DURATION_BUCKETS_MS.length; index += 1) {
+        const bound = HTTP_DURATION_BUCKETS_MS[index];
+        lines.push(`http_request_duration_ms_bucket{${key},le="${bound}"} ${histogram.cumulative[index] ?? 0}`);
+      }
+      lines.push(`http_request_duration_ms_bucket{${key},le="+Inf"} ${histogram.count}`);
+      lines.push(`http_request_duration_ms_sum{${key}} ${histogram.sum}`);
+      lines.push(`http_request_duration_ms_count{${key}} ${histogram.count}`);
     }
     return `${lines.join(`
 `)}
 `;
   }
   resetForTests() {
+    this.seriesLabels.clear();
     this.httpRequestsTotal.clear();
     this.httpRequestDurationMs.clear();
   }
@@ -40,9 +73,10 @@ class PrometheusRegistry {
     let totalRequests = 0;
     for (const [key, count] of this.httpRequestsTotal) {
       totalRequests += count;
-      const method = key.match(/method="([^"]+)"/)?.[1] ?? "GET";
-      const path = key.match(/path="([^"]+)"/)?.[1] ?? "/";
-      const status = key.match(/status="([^"]+)"/)?.[1] ?? "200";
+      const labels = this.seriesLabels.get(key);
+      if (!labels)
+        continue;
+      const { method, path, status } = labels;
       byStatus[status] = (byStatus[status] ?? 0) + count;
       const pathKey = `${method} ${path}`;
       const existing = pathCounts.get(pathKey);
@@ -59,12 +93,38 @@ class PrometheusRegistry {
       topPaths
     };
   }
+  getStorageStats() {
+    return {
+      series: this.seriesLabels.size,
+      histograms: this.httpRequestDurationMs.size,
+      bucketCounters: this.httpRequestDurationMs.size * HTTP_DURATION_BUCKETS_MS.length
+    };
+  }
   metricKey(labels) {
-    return `method="${labels.method}",path="${labels.path}",status="${labels.status}"`;
+    const normalized = {
+      method: /^(GET|HEAD|POST|PUT|PATCH|DELETE|CONNECT|OPTIONS|TRACE)$/.test(labels.method) ? labels.method : "OTHER",
+      path: labels.path.length <= 512 ? labels.path : "__overflow__",
+      status: /^[1-5][0-9]{2}$/.test(labels.status) ? labels.status : "000"
+    };
+    const escapeLabel = (value) => value.replaceAll("\\", "\\\\").replaceAll(`
+`, "\\n").replaceAll('"', "\\\"");
+    const encode = (value) => `method="${value.method}",path="${escapeLabel(value.path)}",status="${value.status}"`;
+    let key = encode(normalized);
+    if (!this.seriesLabels.has(key)) {
+      if (this.seriesLabels.size >= this.maxSeries) {
+        const overflow = { method: "OTHER", path: "__overflow__", status: "000" };
+        key = encode(overflow);
+        this.seriesLabels.set(key, overflow);
+      } else {
+        this.seriesLabels.set(key, normalized);
+      }
+    }
+    return key;
   }
 }
 var prometheusRegistry = new PrometheusRegistry;
 export {
+  HTTP_DURATION_BUCKETS_MS,
   PrometheusRegistry,
   prometheusRegistry
 };

package/dist/entries/queue/createAppQueue.js

@@ -124,7 +124,7 @@ var queueConfig = {
 };
 
 // ../../src/core/queue/jobRunner.ts
-async function runQueueJob(envelope, failedJobs) {
+async function runQueueJob(envelope, failedJobs, options = {}) {
   const job = jobRegistry.create(envelope.name);
   if (!job) {
     throw new Error(`Unknown job "${envelope.name}".`);
@@ -141,7 +141,7 @@ async function runQueueJob(envelope, failedJobs) {
       await runQueueJob({
         ...envelope,
         attempts: nextAttempt
-      }, failedJobs);
+      }, failedJobs, options);
       return;
     }
     await failedJobs.recordFailure({
@@ -149,6 +149,7 @@ async function runQueueJob(envelope, failedJobs) {
       payload: envelope.payload,
       exception: error instanceof Error ? error.stack ?? error.message : String(error)
     });
+    options.onFailureRecorded?.();
     throw error;
   }
 }
@@ -256,6 +257,30 @@ var QUEUE_LIST_KEY = queueListKey();
 var QUEUE_HIGH_KEY = queueHighKey();
 var QUEUE_LOW_KEY = queueLowKey();
 var QUEUE_KEYS = [QUEUE_HIGH_KEY, QUEUE_LIST_KEY, QUEUE_LOW_KEY];
+var DEFAULT_VISIBILITY_MS = 60000;
+function queueProcessingKey(queueKey) {
+  return `${queueKey}:processing`;
+}
+function queueProcessingLeaseKey(queueKey) {
+  return `${queueKey}:processing:leases`;
+}
+function readVisibilityMs() {
+  const parsed = Number(process.env.QUEUE_VISIBILITY_MS ?? String(DEFAULT_VISIBILITY_MS));
+  if (!Number.isSafeInteger(parsed) || parsed < 30) {
+    throw new Error("QUEUE_VISIBILITY_MS must be an integer of at least 30 milliseconds.");
+  }
+  return parsed;
+}
+function reservationExpired(lease, now, visibilityMs) {
+  if (lease === null || lease.length === 0) {
+    return true;
+  }
+  const reservedAt = Number(lease);
+  if (!Number.isFinite(reservedAt)) {
+    return true;
+  }
+  return now - reservedAt >= visibilityMs;
+}
 function queueKeyForPriority(priority = "default") {
   switch (priority) {
     case "high":
@@ -297,6 +322,129 @@ function parseQueueJobEnvelope(rawPayload) {
     attempts: typeof envelope.attempts === "number" ? envelope.attempts : 0
   };
 }
+var SCRIPT_HEADER = `
+local function checkType(key, expected)
+  local actual = redis.call('TYPE', key).ok
+  if actual ~= 'none' and actual ~= expected then
+    return redis.error_reply('Unexpected queue key type: ' .. key)
+  end
+end
+local function nowMs()
+  local time = redis.call('TIME')
+  return tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
+end
+`;
+var RESERVE_JOB = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
+if err then return err end
+local payload = redis.call('LINDEX', KEYS[1], -1)
+if not payload then return nil end
+local record = cjson.encode({strataReservation=1, id=ARGV[1], owner=ARGV[2], payload=payload})
+local lease = cjson.encode({owner=ARGV[2], expiresAt=nowMs()+tonumber(ARGV[3])})
+redis.call('LPUSH', KEYS[2], record)
+redis.call('HSET', KEYS[3], ARGV[1], lease)
+redis.call('RPOP', KEYS[1])
+return record
+`;
+var RENEW_OR_ACK = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'hash') or checkType(KEYS[3], 'list')
+if err then return err end
+local raw = redis.call('HGET', KEYS[2], ARGV[1])
+if not raw then return 0 end
+local ok, lease = pcall(cjson.decode, raw)
+local now = nowMs()
+if not ok or type(lease) ~= 'table' or lease.owner ~= ARGV[2] or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now then
+  return 0
+end
+if ARGV[4] == 'renew' then
+  redis.call('HSET', KEYS[2], ARGV[1], cjson.encode({owner=ARGV[2], expiresAt=now+tonumber(ARGV[5])}))
+  return 1
+end
+if ARGV[4] == 'quarantine' then redis.call('LPUSH', KEYS[3], ARGV[6]) end
+local removed = redis.call('LREM', KEYS[1], 1, ARGV[3])
+redis.call('HDEL', KEYS[2], ARGV[1])
+return removed
+`;
+var RECLAIM_JOBS = `${SCRIPT_HEADER}
+local err = checkType(KEYS[1], 'list') or checkType(KEYS[2], 'list') or checkType(KEYS[3], 'hash')
+if err then return err end
+local now = nowMs()
+local reclaimed = 0
+for _, record in ipairs(redis.call('LRANGE', KEYS[2], 0, -1)) do
+  local payload, field = record, record
+  local modern = false
+  local ok, reservation = pcall(cjson.decode, record)
+  if ok and type(reservation) == 'table' and reservation.strataReservation == 1 then
+    payload, field, modern = reservation.payload, reservation.id, true
+  end
+  local raw = redis.call('HGET', KEYS[3], field)
+  local expired = true
+  if raw then
+    if modern then
+      local leaseOk, lease = pcall(cjson.decode, raw)
+      expired = not leaseOk or type(lease) ~= 'table' or not tonumber(lease.expiresAt) or tonumber(lease.expiresAt) <= now
+    else
+      local reservedAt = tonumber(raw)
+      expired = not reservedAt or now - reservedAt >= tonumber(ARGV[1])
+    end
+  end
+  if expired then
+    redis.call('LPUSH', KEYS[1], payload)
+    redis.call('LREM', KEYS[2], 1, record)
+    redis.call('HDEL', KEYS[3], field)
+    reclaimed = reclaimed + 1
+  end
+end
+return reclaimed
+`;
+function queueInvalidKey(queueKey) {
+  return `${queueKey}:invalid`;
+}
+async function reserveQueueJob(client, queueKey, owner) {
+  const raw = await client.send("EVAL", [
+    RESERVE_JOB,
+    "3",
+    queueKey,
+    queueProcessingKey(queueKey),
+    queueProcessingLeaseKey(queueKey),
+    crypto.randomUUID(),
+    owner,
+    String(readVisibilityMs())
+  ]);
+  if (typeof raw !== "string")
+    return null;
+  const reservation = JSON.parse(raw);
+  return { ...reservation, record: raw };
+}
+async function updateQueueReservation(client, queueKey, reservation, action) {
+  return Number(await client.send("EVAL", [
+    RENEW_OR_ACK,
+    "3",
+    queueProcessingKey(queueKey),
+    queueProcessingLeaseKey(queueKey),
+    queueInvalidKey(queueKey),
+    reservation.id,
+    reservation.owner,
+    reservation.record,
+    action,
+    String(readVisibilityMs()),
+    reservation.payload
+  ])) === 1;
+}
+async function reclaimExpiredQueueReservations(client, queueKeys = QUEUE_KEYS) {
+  let reclaimed = 0;
+  for (const key of queueKeys) {
+    reclaimed += Number(await client.send("EVAL", [
+      RECLAIM_JOBS,
+      "3",
+      key,
+      queueProcessingKey(key),
+      queueProcessingLeaseKey(key),
+      String(readVisibilityMs())
+    ]));
+  }
+  return reclaimed;
+}
 
 class RedisQueue {
   client;
@@ -321,12 +469,15 @@ class RedisQueue {
 class QueueWorker {
   failedJobs;
   timeoutSeconds;
+  queueKeys;
+  owner = crypto.randomUUID();
   running = false;
   stopping = false;
   client;
-  constructor(redisUrl, failedJobs, timeoutSeconds = 5) {
+  constructor(redisUrl, failedJobs, timeoutSeconds = 5, queueKeys = QUEUE_KEYS) {
     this.failedJobs = failedJobs;
     this.timeoutSeconds = timeoutSeconds;
+    this.queueKeys = queueKeys;
     this.client = new RedisClient(redisUrl);
   }
   requestStop() {
@@ -335,38 +486,81 @@ class QueueWorker {
   isRunning() {
     return this.running;
   }
-  async processNext() {
-    let result = null;
-    for (const queueKey of QUEUE_KEYS) {
-      result = await this.client.brpop(queueKey, 1);
-      if (result) {
-        break;
+  async reserve() {
+    const deadline = performance.now() + this.timeoutSeconds * 1000;
+    do {
+      for (const key of this.queueKeys) {
+        if (this.stopping)
+          return null;
+        const reservation = await reserveQueueJob(this.client, key, this.owner);
+        if (reservation)
+          return { queueKey: key, reservation };
       }
-    }
-    if (!result) {
-      result = await this.client.brpop(QUEUE_LIST_KEY, this.timeoutSeconds);
-    }
-    if (!result) {
+      if (this.stopping || performance.now() >= deadline)
+        return null;
+      await Bun.sleep(Math.min(100, Math.max(1, deadline - performance.now())));
+    } while (!this.stopping);
+    return null;
+  }
+  async processNext() {
+    if (this.stopping)
       return false;
-    }
-    const [, rawPayload] = result;
-    const envelope = parseQueueJobEnvelope(rawPayload);
+    await reclaimExpiredQueueReservations(this.client, this.queueKeys);
+    const reserved = await this.reserve();
+    if (!reserved)
+      return false;
+    const { queueKey, reservation } = reserved;
+    const envelope = parseQueueJobEnvelope(reservation.payload);
     if (!envelope) {
+      await updateQueueReservation(this.client, queueKey, reservation, "quarantine");
       return true;
     }
+    let acknowledge = false;
+    let ownershipLost = false;
+    let renewal;
+    const heartbeat = setInterval(() => {
+      if (renewal || ownershipLost)
+        return;
+      renewal = (async () => {
+        try {
+          if (!await updateQueueReservation(this.client, queueKey, reservation, "renew")) {
+            ownershipLost = true;
+          }
+        } catch (error) {
+          ownershipLost = true;
+          console.error("[QueueWorker] Lease renewal failed:", error);
+        }
+      })().finally(() => {
+        renewal = undefined;
+      });
+    }, Math.max(1, Math.floor(readVisibilityMs() / 3)));
     try {
-      await runQueueJob(envelope, this.failedJobs);
+      await runQueueJob(envelope, this.failedJobs, {
+        onFailureRecorded: () => {
+          acknowledge = true;
+        }
+      });
+      acknowledge = true;
     } catch (error) {
       console.error("[QueueWorker] Job failed:", error);
+    } finally {
+      clearInterval(heartbeat);
+      await renewal;
+    }
+    if (acknowledge && !ownershipLost) {
+      await updateQueueReservation(this.client, queueKey, reservation, "ack");
     }
     return true;
   }
   async run() {
     this.running = true;
-    while (!this.stopping) {
-      await this.processNext();
+    try {
+      while (!this.stopping) {
+        await this.processNext();
+      }
+    } finally {
+      this.running = false;
     }
-    this.running = false;
   }
   close() {
     this.client.close();

package/dist/entries/queue/jobRunner.js

@@ -60,7 +60,7 @@ var queueConfig = {
 };
 
 // ../../src/core/queue/jobRunner.ts
-async function runQueueJob(envelope, failedJobs) {
+async function runQueueJob(envelope, failedJobs, options = {}) {
   const job = jobRegistry.create(envelope.name);
   if (!job) {
     throw new Error(`Unknown job "${envelope.name}".`);
@@ -77,7 +77,7 @@ async function runQueueJob(envelope, failedJobs) {
       await runQueueJob({
         ...envelope,
         attempts: nextAttempt
-      }, failedJobs);
+      }, failedJobs, options);
       return;
     }
     await failedJobs.recordFailure({
@@ -85,6 +85,7 @@ async function runQueueJob(envelope, failedJobs) {
       payload: envelope.payload,
       exception: error instanceof Error ? error.stack ?? error.message : String(error)
     });
+    options.onFailureRecorded?.();
     throw error;
   }
 }