@jairodri/huckleberry-cli 1.0.1 → 1.1.0

package/skills/claw-compactor/proxy/event-log.mjs

@@ -0,0 +1,213 @@
+/**
+ * Event Log — Persistent Circular Buffer for Monitoring
+ *
+ * Captures proxy events (requests, retries, reaps, timeouts, kills)
+ * in a fixed-size circular buffer. Persists to Redis LIST.
+ *
+ * Redis keys (with ccp: prefix):
+ *   events        — LIST of JSON event objects (capped at maxEvents via LTRIM)
+ *   events:nextId — STRING atomic counter via INCR
+ *   events:counts — HASH { type: count } via HINCRBY
+ *
+ * All public methods return new objects (immutable pattern).
+ */
+
+const DEFAULTS = Object.freeze({
+  maxEvents: 500,
+});
+
+const REDIS_KEY_EVENTS = "events";
+const REDIS_KEY_NEXT_ID = "events:nextId";
+const REDIS_KEY_COUNTS = "events:counts";
+
+/**
+ * @param {object} [options]
+ * @param {number} [options.maxEvents]
+ * @param {object} [options.redis] - Redis client from redis-client.mjs
+ */
+export function createEventLog(options = {}) {
+  const { redis, ...rest } = options;
+  const config = Object.freeze({ ...DEFAULTS, ...rest });
+
+  // In-memory state (always maintained)
+  let events = [];
+  let nextId = 1;
+  let counts = {};
+
+  // --------------------------------------------------
+  // Redis persistence
+  // --------------------------------------------------
+
+  async function loadFromRedis() {
+    if (!redis?.isReady()) return false;
+
+    try {
+      // Load next ID
+      const rawId = await redis.client.get(REDIS_KEY_NEXT_ID);
+      if (rawId) {
+        nextId = parseInt(rawId, 10) + 1;
+      }
+
+      // Load event counts
+      const rawCounts = await redis.client.hgetall(REDIS_KEY_COUNTS);
+      if (rawCounts && Object.keys(rawCounts).length > 0) {
+        const loaded = {};
+        for (const [type, count] of Object.entries(rawCounts)) {
+          loaded[type] = parseInt(count, 10) || 0;
+        }
+        counts = loaded;
+      }
+
+      // Load recent events (most recent first in Redis, reverse for our array)
+      const rawEvents = await redis.client.lrange(
+        REDIS_KEY_EVENTS,
+        0,
+        config.maxEvents - 1
+      );
+      if (rawEvents && rawEvents.length > 0) {
+        const loaded = [];
+        for (const raw of rawEvents) {
+          try {
+            loaded.push(Object.freeze(JSON.parse(raw)));
+          } catch {
+            // skip malformed
+          }
+        }
+        // Redis LIST: LPUSH means newest first, we want oldest first
+        events = loaded.reverse();
+      }
+
+      return events.length > 0 || Object.keys(counts).length > 0;
+    } catch (err) {
+      console.error(`[EventLog] Redis load error: ${err.message}`);
+      return false;
+    }
+  }
+
+  /**
+   * Persist a single event to Redis (fire-and-forget).
+   */
+  function pushToRedis(event) {
+    if (!redis?.isReady()) return;
+
+    const pipeline = redis.client.pipeline();
+    // Push to front of list (newest first)
+    pipeline.lpush(REDIS_KEY_EVENTS, JSON.stringify(event));
+    // Trim to max size
+    pipeline.ltrim(REDIS_KEY_EVENTS, 0, config.maxEvents - 1);
+    // Increment type counter
+    pipeline.hincrby(REDIS_KEY_COUNTS, event.type, 1);
+    // Update next ID tracker
+    pipeline.set(REDIS_KEY_NEXT_ID, String(event.id));
+
+    pipeline.exec().catch((err) => {
+      console.error(`[EventLog] Redis push error: ${err.message}`);
+    });
+  }
+
+  // --------------------------------------------------
+  // Core
+  // --------------------------------------------------
+
+  /**
+   * Push a new event into the log.
+   * @param {string} type - Event type (request, retry, reap, timeout, kill, error, etc.)
+   * @param {object} data - Event-specific data
+   * @returns {object} The created event (frozen)
+   */
+  function push(type, data = {}) {
+    const event = Object.freeze({
+      id: nextId++,
+      type,
+      ts: Date.now(),
+      isoTs: new Date().toISOString(),
+      ...data,
+    });
+
+    // Immutable append with size cap
+    events =
+      events.length >= config.maxEvents
+        ? [...events.slice(events.length - config.maxEvents + 1), event]
+        : [...events, event];
+
+    // Update counts
+    counts = { ...counts, [type]: (counts[type] || 0) + 1 };
+
+    // Persist to Redis
+    pushToRedis(event);
+
+    return event;
+  }
+
+  /**
+   * Get recent events, optionally filtered.
+   * @param {object} [opts]
+   * @param {number} [opts.limit] - Max events to return (default: 50)
+   * @param {number} [opts.sinceId] - Only return events after this ID
+   * @param {string} [opts.type] - Filter by event type
+   * @returns {object[]} Array of events (newest last)
+   */
+  function getRecent(opts = {}) {
+    const { limit = 50, sinceId = 0, type = null } = opts;
+
+    let filtered = events;
+    if (sinceId > 0) {
+      filtered = filtered.filter((e) => e.id > sinceId);
+    }
+    if (type) {
+      filtered = filtered.filter((e) => e.type === type);
+    }
+
+    return filtered.slice(-limit);
+  }
+
+  /**
+   * Get summary counts by event type.
+   * @returns {object} { [type]: count }
+   */
+  function getCounts() {
+    return Object.freeze({ ...counts });
+  }
+
+  /**
+   * Clear all events (in-memory + Redis).
+   */
+  function clear() {
+    events = [];
+    counts = {};
+
+    if (redis?.isReady()) {
+      const pipeline = redis.client.pipeline();
+      pipeline.del(REDIS_KEY_EVENTS);
+      pipeline.del(REDIS_KEY_COUNTS);
+      // Don't reset nextId — IDs should be monotonic
+      pipeline.exec().catch((err) => {
+        console.error(`[EventLog] Redis clear error: ${err.message}`);
+      });
+    }
+  }
+
+  // --------------------------------------------------
+  // Init: load from Redis
+  // --------------------------------------------------
+
+  const ready = (async () => {
+    const loaded = await loadFromRedis();
+    if (loaded) {
+      console.log(
+        `[EventLog] Loaded from Redis: ${events.length} events, ${Object.keys(counts).length} types`
+      );
+    } else {
+      console.log("[EventLog] Starting fresh (no Redis data)");
+    }
+  })();
+
+  return Object.freeze({
+    push,
+    getRecent,
+    getCounts,
+    clear,
+    ready,
+    config,
+  });
+}

package/skills/claw-compactor/proxy/fair-queue.mjs

@@ -0,0 +1,288 @@
+/**
+ * Fair Queue with Round-Robin Scheduling
+ *
+ * Ensures fair access when multiple OpenClaw instances share
+ * one Claude Code proxy. Round-robin between sources,
+ * priority support within each source's queue.
+ *
+ * All public methods return new objects (immutable pattern).
+ * Internal state is encapsulated in closure.
+ */
+
+const PRIORITY_ORDER = { high: 0, normal: 1, low: 2 };
+
+export function createFairQueue(options = {}) {
+  const {
+    maxConcurrent = 10,
+    maxPerSource = 20,
+    maxTotal = 100,
+    queueTimeoutMs = 60000,
+    maxLeaseMs = 600_000,
+  } = options;
+
+  let activeCount = 0;
+  let totalQueued = 0;
+  const sourceQueues = new Map();
+  let sourceOrder = [];
+  let rrIndex = 0;
+
+  // Lease tracking for slot leak detection
+  let nextLeaseId = 0;
+  let activeLeases = new Map(); // leaseId -> { sourceId, acquiredAt }
+
+  // Metrics (cumulative)
+  let metrics = {
+    totalProcessed: 0,
+    totalTimedOut: 0,
+    totalRejected: 0,
+    totalLeaked: 0,
+    perSource: {},
+  };
+
+  // Periodic sweep for timed-out entries + leaked slots
+  const sweepInterval = setInterval(() => {
+    const now = Date.now();
+
+    // Phase 1: Clean timed-out queue entries
+    for (const [sourceId, entries] of sourceQueues) {
+      const expired = entries.filter((e) => now - e.enqueuedAt > queueTimeoutMs);
+      for (const entry of expired) {
+        clearTimeout(entry.timer);
+        entry.reject(new Error(`Queue timeout: waited ${queueTimeoutMs}ms`));
+        totalQueued--;
+        metrics = { ...metrics, totalTimedOut: metrics.totalTimedOut + 1 };
+      }
+      const remaining = entries.filter((e) => now - e.enqueuedAt <= queueTimeoutMs);
+      if (remaining.length === 0) {
+        sourceQueues.delete(sourceId);
+        sourceOrder = sourceOrder.filter((id) => id !== sourceId);
+      } else {
+        sourceQueues.set(sourceId, remaining);
+      }
+    }
+
+    // Phase 2: Detect and force-release leaked slots
+    let leakedCount = 0;
+    for (const [leaseId, lease] of activeLeases) {
+      const held = now - lease.acquiredAt;
+      if (held > maxLeaseMs) {
+        console.warn(
+          `[${new Date().toISOString()}] SLOT_LEAK leaseId=${leaseId} ` +
+          `src=${lease.sourceId} held=${Math.round(held / 1000)}s, force-releasing`
+        );
+        activeLeases = new Map(activeLeases);
+        activeLeases.delete(leaseId);
+        activeCount--;
+        leakedCount++;
+      }
+    }
+    if (leakedCount > 0) {
+      metrics = { ...metrics, totalLeaked: metrics.totalLeaked + leakedCount };
+      tryDispatch();
+    }
+  }, 5000);
+
+  function tryDispatch() {
+    while (activeCount < maxConcurrent && totalQueued > 0) {
+      const entry = dequeueNext();
+      if (!entry) break;
+      clearTimeout(entry.timer);
+      activeCount++;
+      totalQueued--;
+      metrics = { ...metrics, totalProcessed: metrics.totalProcessed + 1 };
+
+      // Track per-source
+      const srcStats = metrics.perSource[entry.sourceId] || { processed: 0 };
+      metrics = {
+        ...metrics,
+        perSource: {
+          ...metrics.perSource,
+          [entry.sourceId]: { ...srcStats, processed: srcStats.processed + 1 },
+        },
+      };
+
+      const leaseId = nextLeaseId++;
+      let released = false;
+      activeLeases = new Map(activeLeases);
+      activeLeases.set(leaseId, { sourceId: entry.sourceId, acquiredAt: Date.now() });
+
+      entry.resolve(() => {
+        if (released) return; // idempotent — double-release is a no-op
+        released = true;
+        // Guard: if SLOT_LEAK already force-released this lease, skip decrement
+        if (activeLeases.has(leaseId)) {
+          activeLeases = new Map(activeLeases);
+          activeLeases.delete(leaseId);
+          activeCount--;
+        }
+        tryDispatch();
+      });
+    }
+  }
+
+  function dequeueNext() {
+    const activeSources = sourceOrder.filter((id) => {
+      const q = sourceQueues.get(id);
+      return q && q.length > 0;
+    });
+
+    if (activeSources.length === 0) return null;
+
+    rrIndex = rrIndex % activeSources.length;
+    const sourceId = activeSources[rrIndex];
+    rrIndex = (rrIndex + 1) % Math.max(activeSources.length, 1);
+
+    const queue = sourceQueues.get(sourceId);
+    const entry = queue[0];
+    const rest = queue.slice(1);
+
+    if (rest.length === 0) {
+      sourceQueues.delete(sourceId);
+      sourceOrder = sourceOrder.filter((id) => id !== sourceId);
+    } else {
+      sourceQueues.set(sourceId, rest);
+    }
+
+    return { ...entry, sourceId };
+  }
+
+  /**
+   * Acquire a processing slot. Returns a Promise that resolves
+   * with a release() function when a slot is available.
+   *
+   * @param {string} sourceId - Identifier for the requesting source
+   * @param {string} priority - "high" | "normal" | "low"
+   * @returns {Promise<Function>} release function to call when done
+   */
+  function acquire(sourceId, priority = "normal") {
+    // Fast path: slot available and nothing queued
+    if (activeCount < maxConcurrent && totalQueued === 0) {
+      activeCount++;
+      metrics = { ...metrics, totalProcessed: metrics.totalProcessed + 1 };
+      const srcStats = metrics.perSource[sourceId] || { processed: 0 };
+      metrics = {
+        ...metrics,
+        perSource: {
+          ...metrics.perSource,
+          [sourceId]: { ...srcStats, processed: srcStats.processed + 1 },
+        },
+      };
+
+      const leaseId = nextLeaseId++;
+      let released = false;
+      activeLeases = new Map(activeLeases);
+      activeLeases.set(leaseId, { sourceId, acquiredAt: Date.now() });
+
+      return Promise.resolve(() => {
+        if (released) return; // idempotent — double-release is a no-op
+        released = true;
+        // Guard: if SLOT_LEAK already force-released this lease, skip decrement
+        if (activeLeases.has(leaseId)) {
+          activeLeases = new Map(activeLeases);
+          activeLeases.delete(leaseId);
+          activeCount--;
+        }
+        tryDispatch();
+      });
+    }
+
+    // Check total queue limit
+    if (totalQueued >= maxTotal) {
+      metrics = { ...metrics, totalRejected: metrics.totalRejected + 1 };
+      return Promise.reject(
+        new Error(`Queue full: ${totalQueued}/${maxTotal} total`)
+      );
+    }
+
+    // Check per-source limit
+    const sourceQueue = sourceQueues.get(sourceId) || [];
+    if (sourceQueue.length >= maxPerSource) {
+      metrics = { ...metrics, totalRejected: metrics.totalRejected + 1 };
+      return Promise.reject(
+        new Error(`Source queue full: ${sourceQueue.length}/${maxPerSource} for ${sourceId}`)
+      );
+    }
+
+    return new Promise((resolve, reject) => {
+      const entry = {
+        resolve,
+        reject,
+        priority,
+        sourceId,
+        enqueuedAt: Date.now(),
+        timer: null,
+      };
+
+      entry.timer = setTimeout(() => {
+        const q = sourceQueues.get(sourceId) || [];
+        const filtered = q.filter((e) => e !== entry);
+        if (filtered.length === 0) {
+          sourceQueues.delete(sourceId);
+          sourceOrder = sourceOrder.filter((id) => id !== sourceId);
+        } else {
+          sourceQueues.set(sourceId, filtered);
+        }
+        totalQueued--;
+        metrics = { ...metrics, totalTimedOut: metrics.totalTimedOut + 1 };
+        reject(new Error(`Queue timeout: waited ${queueTimeoutMs}ms`));
+      }, queueTimeoutMs);
+
+      // Insert sorted by priority within this source's queue
+      const newQueue = [...sourceQueue, entry].sort(
+        (a, b) => (PRIORITY_ORDER[a.priority] ?? 1) - (PRIORITY_ORDER[b.priority] ?? 1)
+      );
+
+      sourceQueues.set(sourceId, newQueue);
+
+      if (!sourceOrder.includes(sourceId)) {
+        sourceOrder = [...sourceOrder, sourceId];
+      }
+
+      totalQueued++;
+      tryDispatch();
+    });
+  }
+
+  function getStats() {
+    const perSource = {};
+    for (const [sourceId, entries] of sourceQueues) {
+      perSource[sourceId] = entries.length;
+    }
+
+    const now = Date.now();
+    const leaseList = Array.from(activeLeases.entries()).map(([id, lease]) => ({
+      leaseId: id,
+      sourceId: lease.sourceId,
+      heldMs: now - lease.acquiredAt,
+    }));
+
+    return {
+      active: activeCount,
+      maxConcurrent,
+      totalQueued,
+      maxTotal,
+      maxPerSource,
+      queueTimeoutMs,
+      maxLeaseMs,
+      queuedPerSource: perSource,
+      sourceCount: sourceQueues.size,
+      activeLeases: leaseList,
+      metrics: { ...metrics },
+    };
+  }
+
+  function destroy() {
+    clearInterval(sweepInterval);
+    for (const entries of sourceQueues.values()) {
+      for (const entry of entries) {
+        clearTimeout(entry.timer);
+        entry.reject(new Error("Queue destroyed"));
+      }
+    }
+    sourceQueues.clear();
+    sourceOrder = [];
+    totalQueued = 0;
+  }
+
+  return { acquire, getStats, destroy };
+}