@stackmemoryai/stackmemory 1.5.4 → 1.5.6

package/dist/src/cli/commands/orchestrator.js

@@ -8,10 +8,13 @@ import {
   mkdirSync,
   rmSync,
   writeFileSync,
+  readdirSync,
   createWriteStream
 } from "fs";
 import { join, dirname } from "path";
 import { tmpdir, homedir } from "os";
+import { createReadStream } from "fs";
+import { createInterface } from "readline";
 import { fileURLToPath } from "url";
 import { Transform } from "stream";
 import { logger } from "../../core/monitoring/logger.js";
@@ -128,7 +131,7 @@ const DEFAULT_CONFIG = {
   inProgressState: "In Progress",
   inReviewState: "In Review",
   pollIntervalMs: 3e4,
-  maxConcurrent: 3,
+  maxConcurrent: 5,
   workspaceRoot: join(tmpdir(), "conductor_workspaces"),
   repoRoot: process.cwd(),
   baseBranch: "main",
@@ -158,6 +161,23 @@ class Conductor {
   stateCache = /* @__PURE__ */ new Map();
   activeStatesLower;
   terminalStatesLower;
+  /** Global rate limit backoff state */
+  rateLimit = {
+    inBackoff: false,
+    backoffUntil: 0,
+    totalHits: 0,
+    consecutiveHits: 0,
+    lastHitAt: 0
+  };
+  /** Aggregated usage stats */
+  usage = {
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheCreationTokens: 0,
+    cacheReadTokens: 0,
+    estimatedMessages: 0,
+    perAgent: /* @__PURE__ */ new Map()
+  };
   constructor(config = {}) {
     this.config = { ...DEFAULT_CONFIG, ...config };
     this.activeStatesLower = this.config.activeStates.map(
@@ -294,7 +314,9 @@ class Conductor {
       failed: this.failCount,
       totalAttempts: this.totalAttempts,
       uptime: Date.now() - this.startedAt,
-      issues
+      issues,
+      rateLimit: { ...this.rateLimit },
+      usage: { ...this.usage, perAgent: new Map(this.usage.perAgent) }
     };
   }
   // ── Status File ──
@@ -324,7 +346,31 @@ class Conductor {
       failed: this.failCount,
       totalAttempts: this.totalAttempts,
       maxConcurrent: this.config.maxConcurrent,
-      stopping: this.stopping
+      stopping: this.stopping,
+      rateLimit: {
+        inBackoff: this.rateLimit.inBackoff,
+        backoffUntil: this.rateLimit.backoffUntil,
+        backoffRemainingSec: this.rateLimit.inBackoff ? Math.max(
+          0,
+          Math.ceil((this.rateLimit.backoffUntil - Date.now()) / 1e3)
+        ) : 0,
+        totalHits: this.rateLimit.totalHits
+      },
+      usage: (() => {
+        const summary = this.getUsageSummary();
+        return {
+          inputTokens: summary.inputTokens,
+          outputTokens: summary.outputTokens,
+          totalTokens: summary.totalTokens,
+          estimatedMessages: summary.estimatedMessages,
+          tokensPerMin: summary.tokensPerMin,
+          budgetPct5x: summary.budgetPct5x,
+          budgetPct20x: summary.budgetPct20x,
+          minutesRemaining5x: summary.minutesRemaining5x,
+          minutesRemaining20x: summary.minutesRemaining20x,
+          cacheHitRate: summary.cacheHitRate
+        };
+      })()
     };
     try {
       writeFileSync(
@@ -360,7 +406,8 @@ class Conductor {
         phase: run.phase,
         filesModified: run.filesModified,
         toolCalls: run.toolCalls,
-        tokensUsed: run.tokensUsed
+        tokensUsed: run.tokensUsed,
+        workspacePath: run.workspacePath || void 0
       };
       writeFileSync(join(dir, "status.json"), JSON.stringify(status, null, 2));
     } catch {
@@ -390,6 +437,21 @@ class Conductor {
   }
   async poll() {
     if (!this.client || this.stopping) return;
+    if (this.rateLimit.inBackoff) {
+      const remaining = this.rateLimit.backoffUntil - Date.now();
+      if (remaining > 0) {
+        logger.info("Rate limit backoff active, skipping poll", {
+          remainingMs: remaining,
+          remainingSec: Math.ceil(remaining / 1e3),
+          totalHits: this.rateLimit.totalHits
+        });
+        return;
+      }
+      this.rateLimit.inBackoff = false;
+      this.rateLimit.backoffUntil = 0;
+      logger.info("Rate limit backoff expired, resuming dispatch");
+      console.log("[rate-limit] Backoff expired, resuming dispatch");
+    }
     await this.reconcile();
     const available = this.config.maxConcurrent - this.running.size;
     if (available <= 0) {
@@ -407,6 +469,29 @@ class Conductor {
     if (eligible.length === 0) return;
     const sorted = eligible.sort((a, b) => (a.priority || 4) - (b.priority || 4)).slice(0, available);
     const toDispatch = this.preflightFilter(sorted);
+    if (this.usage.inputTokens > 0) {
+      const summary = this.getUsageSummary();
+      logger.info("Usage summary", {
+        totalTokens: summary.totalTokens,
+        estimatedMessages: summary.estimatedMessages,
+        tokensPerMin: summary.tokensPerMin,
+        budgetPct5x: `${summary.budgetPct5x}%`,
+        budgetPct20x: `${summary.budgetPct20x}%`,
+        minutesRemaining5x: summary.minutesRemaining5x,
+        minutesRemaining20x: summary.minutesRemaining20x,
+        rateLimitHits: this.rateLimit.totalHits
+      });
+      if (summary.budgetPct5x >= 75 && summary.budgetPct5x < 100) {
+        console.log(
+          `[usage] \u26A0 ${summary.budgetPct5x}% of Max 5x budget used \u2014 ~${summary.minutesRemaining5x}min remaining at ${summary.tokensPerMin} tok/min`
+        );
+      }
+      if (summary.budgetPct5x >= 100) {
+        console.log(
+          `[usage] \u{1F6D1} Max 5x budget likely exhausted (${summary.estimatedMessages} est. messages / 225 limit). Expect 429s.`
+        );
+      }
+    }
     logger.info("Dispatching issues", {
       count: toDispatch.length,
       identifiers: toDispatch.map((i) => i.identifier),
@@ -569,6 +654,9 @@ class Conductor {
           error: run.error,
           attempt: run.attempt
         });
+        if (this.handleRateLimitError(run.error, issue.identifier)) {
+          throw err;
+        }
         if (run.workspacePath) {
           await this.runHook(
             "after-run",
@@ -595,6 +683,179 @@ class Conductor {
       }
     }
   }
+  // ── Rate Limit Detection ──
+  /**
+   * Check if an error indicates a rate limit (429) or usage cap.
+   * Triggers global backoff so no new agents are dispatched.
+   *
+   * Claude Max limits (approximate, shared between claude.ai + Claude Code):
+   * - Max 5x: ~225 messages per 5h window
+   * - Max 20x: ~900 messages per 5h window
+   * Messages are token-weighted: heavy agent usage ≈ 5-10x a casual message.
+   */
+  handleRateLimitError(error, identifier) {
+    const rateLimitPatterns = [
+      "usage limits",
+      "rate_limit",
+      "rate limit",
+      "overloaded",
+      "429",
+      "too many requests",
+      "quota exceeded",
+      "capacity"
+    ];
+    const isRateLimit = rateLimitPatterns.some(
+      (p) => error.toLowerCase().includes(p)
+    );
+    if (!isRateLimit) return false;
+    this.rateLimit.totalHits++;
+    this.rateLimit.lastHitAt = Date.now();
+    const timeSinceLast = Date.now() - this.rateLimit.lastHitAt;
+    if (timeSinceLast < 6e5) {
+      this.rateLimit.consecutiveHits++;
+    } else {
+      this.rateLimit.consecutiveHits = 1;
+    }
+    const backoffSec = Math.min(
+      60 * Math.pow(2, this.rateLimit.consecutiveHits - 1),
+      900
+    );
+    this.rateLimit.inBackoff = true;
+    this.rateLimit.backoffUntil = Date.now() + backoffSec * 1e3;
+    logger.warn("Rate limit hit \u2014 global backoff", {
+      identifier,
+      backoffSec,
+      consecutiveHits: this.rateLimit.consecutiveHits,
+      totalHits: this.rateLimit.totalHits,
+      error: error.slice(0, 200)
+    });
+    console.log(
+      `[rate-limit] Hit rate limit on ${identifier} \u2014 backing off ${backoffSec}s (hit #${this.rateLimit.totalHits})`
+    );
+    return true;
+  }
+  // ── Usage Tracking ──
+  /**
+   * Track token usage from a stream-json event (CLI mode) or JSON-RPC message (adapter mode).
+   * Updates both global and per-agent counters.
+   */
+  trackUsage(identifier, usage) {
+    const input = usage.input_tokens || 0;
+    const output = usage.output_tokens || 0;
+    const cacheCreate = usage.cache_creation_input_tokens || 0;
+    const cacheRead = usage.cache_read_input_tokens || 0;
+    this.usage.inputTokens += input;
+    this.usage.outputTokens += output;
+    this.usage.cacheCreationTokens += cacheCreate;
+    this.usage.cacheReadTokens += cacheRead;
+    this.usage.estimatedMessages = Math.ceil(
+      (this.usage.inputTokens + this.usage.outputTokens) / 1e4
+    );
+    const agent = this.usage.perAgent.get(identifier) || {
+      inputTokens: 0,
+      outputTokens: 0
+    };
+    agent.inputTokens += input;
+    agent.outputTokens += output;
+    this.usage.perAgent.set(identifier, agent);
+  }
+  /**
+   * Scan Claude Code JSONL logs for token usage from conductor-spawned sessions.
+   * Reads logs from ~/.claude/projects/ matching conductor workspace paths.
+   */
+  async scanUsageLogs() {
+    const logDirs = [
+      join(homedir(), ".claude", "projects"),
+      join(homedir(), ".config", "claude", "projects")
+    ];
+    for (const logDir of logDirs) {
+      if (!existsSync(logDir)) continue;
+      try {
+        const entries = readdirSync(logDir);
+        const conductorDirs = entries.filter(
+          (e) => e.includes("conductor_workspaces") || e.includes("conductor-")
+        );
+        for (const dir of conductorDirs) {
+          const projectDir = join(logDir, dir);
+          if (!existsSync(projectDir)) continue;
+          const jsonlFiles = readdirSync(projectDir).filter(
+            (f) => f.endsWith(".jsonl")
+          );
+          for (const file of jsonlFiles) {
+            await this.parseUsageFromJsonl(join(projectDir, file));
+          }
+        }
+      } catch (err) {
+        logger.debug("Usage log scan failed", {
+          error: err.message
+        });
+      }
+    }
+    return this.usage;
+  }
+  async parseUsageFromJsonl(filePath) {
+    try {
+      const rl = createInterface({
+        input: createReadStream(filePath),
+        crlfDelay: Infinity
+      });
+      for await (const line of rl) {
+        if (!line.trim()) continue;
+        try {
+          const entry = JSON.parse(line);
+          if (entry.type === "assistant" && entry.message?.usage) {
+            const usage = entry.message.usage;
+            const identifier = entry.sessionId?.slice(0, 8) || "unknown";
+            this.trackUsage(identifier, usage);
+          }
+        } catch {
+        }
+      }
+    } catch {
+    }
+  }
+  /**
+   * Get current usage summary with time-to-exhaustion estimate.
+   *
+   * Claude Max limits (approximate, shared between claude.ai + Claude Code):
+   * - Max 5x:  ~225 messages per 5h window (~10k tokens per "message")
+   * - Max 20x: ~900 messages per 5h window
+   *
+   * Heavy agent usage burns 5-10x faster than casual chat.
+   */
+  getUsageSummary() {
+    const totalCache = this.usage.cacheCreationTokens + this.usage.cacheReadTokens;
+    const cacheHitRate = totalCache > 0 ? this.usage.cacheReadTokens / totalCache : 0;
+    const uptimeMin = Math.max(1, (Date.now() - this.startedAt) / 6e4);
+    const totalTokens = this.usage.inputTokens + this.usage.outputTokens;
+    const tokensPerMin = Math.round(totalTokens / uptimeMin);
+    const estMessages = this.usage.estimatedMessages;
+    const MAX_5X_MESSAGES = 225;
+    const MAX_20X_MESSAGES = 900;
+    const budgetPct5x = Math.round(estMessages / MAX_5X_MESSAGES * 100);
+    const budgetPct20x = Math.round(estMessages / MAX_20X_MESSAGES * 100);
+    const msgsPerMin = tokensPerMin > 0 ? tokensPerMin / 1e4 : 0;
+    const minutesRemaining5x = msgsPerMin > 0 ? Math.round((MAX_5X_MESSAGES - estMessages) / msgsPerMin) : -1;
+    const minutesRemaining20x = msgsPerMin > 0 ? Math.round((MAX_20X_MESSAGES - estMessages) / msgsPerMin) : -1;
+    return {
+      totalTokens,
+      inputTokens: this.usage.inputTokens,
+      outputTokens: this.usage.outputTokens,
+      estimatedMessages: estMessages,
+      cacheHitRate: Math.round(cacheHitRate * 100),
+      tokensPerMin,
+      budgetPct5x,
+      budgetPct20x,
+      minutesRemaining5x: minutesRemaining5x < 0 ? -1 : minutesRemaining5x,
+      minutesRemaining20x: minutesRemaining20x < 0 ? -1 : minutesRemaining20x,
+      perAgent: Array.from(this.usage.perAgent.entries()).map(
+        ([id, stats]) => ({
+          id,
+          ...stats
+        })
+      )
+    };
+  }
   // ── Workspace Management ──
   async createWorkspace(issue) {
     const wsKey = this.sanitizeIdentifier(issue.identifier);
@@ -672,6 +933,21 @@ class Conductor {
   sanitizeIdentifier(identifier) {
     return identifier.replace(/[^A-Za-z0-9._-]/g, "_");
   }
+  /**
+   * Find the real claude binary, skipping shell wrappers (cmux, claude-smd).
+   * Wrappers inject --settings with hooks that block headless -p mode.
+   */
+  findClaudeBinary() {
+    const candidates = [
+      join(homedir(), ".local", "bin", "claude"),
+      "/usr/local/bin/claude",
+      "/opt/homebrew/bin/claude"
+    ];
+    for (const c of candidates) {
+      if (existsSync(c)) return c;
+    }
+    return "claude";
+  }
   // ── Agent Execution ──
   async runAgent(issue, run) {
     if (this.config.agentMode === "cli") {
@@ -679,7 +955,10 @@ class Conductor {
         return await this.runAgentCLI(issue, run);
       } catch (err) {
         const msg = err.message || "";
-        if (msg.includes("usage limits") || msg.includes("rate_limit") || msg.includes("overloaded")) {
+        if (this.handleRateLimitError(msg, issue.identifier)) {
+          throw err;
+        }
+        if (msg.includes("usage limits") || msg.includes("overloaded")) {
           logger.warn("CLI mode hit limits, falling back to adapter", {
             identifier: issue.identifier,
             error: msg.slice(0, 200)
@@ -702,13 +981,16 @@ class Conductor {
   runAgentCLI(issue, run) {
     return new Promise((resolve, reject) => {
       const prompt = this.buildPrompt(issue, run.attempt);
+      const claudeBin = this.findClaudeBinary();
       const proc = spawn(
-        "claude",
+        claudeBin,
         [
           "-p",
           "--output-format",
           "stream-json",
           "--dangerously-skip-permissions",
+          "--settings",
+          '{"hooks":{}}',
           prompt
         ],
         {
@@ -729,6 +1011,7 @@ class Conductor {
         }
       );
       run.process = proc;
+      proc.stdin.end();
       const logStream = this.openAgentLogStream(issue.identifier);
       run.logStream = logStream;
       const tee = new TeeTransform(logStream);
@@ -759,6 +1042,10 @@ class Conductor {
             }
             if (event.type === "assistant") {
               const message = event.message;
+              const msgUsage = message?.usage;
+              if (msgUsage) {
+                this.trackUsage(issue.identifier, msgUsage);
+              }
               const content = message?.content || [];
               for (const block of content) {
                 if (block.type === "tool_use") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackmemoryai/stackmemory",
-  "version": "1.5.4",
+  "version": "1.5.6",
   "description": "Lossless, project-scoped memory for AI coding tools. Durable context across sessions with 56 MCP tools, FTS5 search, conductor orchestrator, loop/watch monitoring, snapshot capture, pre-flight overlap checks, Claude/Codex/OpenCode wrappers, Linear sync, and automatic hooks.",
   "engines": {
     "node": ">=20.0.0",
@@ -100,6 +100,7 @@
     "postinstall": "node scripts/install-claude-hooks-auto.js || true",
     "init": "node dist/scripts/initialize.js",
     "build": "rm -rf dist && node esbuild.config.js",
+    "typecheck": "NODE_OPTIONS='--max-old-space-size=8192' tsc --noEmit",
     "lint": "eslint 'src/**/*.ts' 'scripts/**/*.ts'",
     "lint:fix": "eslint 'src/**/*.ts' 'scripts/**/*.ts' --fix --max-warnings=-1",
     "lint:fast": "oxlint src scripts",