kiro-spec-engine 1.47.10 → 1.47.12

package/lib/orchestrator/orchestration-engine.js

@@ -16,6 +16,23 @@ const path = require('path');
 const fsUtils = require('../utils/fs-utils');
 
 const SPECS_DIR = '.kiro/specs';
+const DEFAULT_RATE_LIMIT_MAX_RETRIES = 6;
+const DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS = 1000;
+const DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS = 30000;
+const DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL = true;
+const DEFAULT_RATE_LIMIT_PARALLEL_FLOOR = 1;
+const DEFAULT_RATE_LIMIT_COOLDOWN_MS = 30000;
+const RATE_LIMIT_BACKOFF_JITTER_RATIO = 0.5;
+const RATE_LIMIT_ERROR_PATTERNS = [
+  /(^|[^0-9])429([^0-9]|$)/i,
+  /too many requests/i,
+  /rate[\s-]?limit/i,
+  /resource exhausted/i,
+  /quota exceeded/i,
+  /exceeded.*quota/i,
+  /requests per minute/i,
+  /tokens per minute/i,
+];
 
 class OrchestrationEngine extends EventEmitter {
   /**
@@ -54,6 +71,28 @@ class OrchestrationEngine extends EventEmitter {
     this._stopped = false;
     /** @type {object|null} execution plan */
     this._executionPlan = null;
+    /** @type {number} max retries for rate-limit failures */
+    this._rateLimitMaxRetries = DEFAULT_RATE_LIMIT_MAX_RETRIES;
+    /** @type {number} base delay for rate-limit retries */
+    this._rateLimitBackoffBaseMs = DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS;
+    /** @type {number} max delay for rate-limit retries */
+    this._rateLimitBackoffMaxMs = DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS;
+    /** @type {boolean} enable adaptive parallel throttling on rate-limit signals */
+    this._rateLimitAdaptiveParallel = DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL;
+    /** @type {number} minimum effective parallelism during rate-limit cooldown */
+    this._rateLimitParallelFloor = DEFAULT_RATE_LIMIT_PARALLEL_FLOOR;
+    /** @type {number} cooldown before each adaptive parallel recovery step */
+    this._rateLimitCooldownMs = DEFAULT_RATE_LIMIT_COOLDOWN_MS;
+    /** @type {number|null} configured max parallel for current run */
+    this._baseMaxParallel = null;
+    /** @type {number|null} dynamic effective parallel limit for current run */
+    this._effectiveMaxParallel = null;
+    /** @type {number} timestamp after which recovery can step up */
+    this._rateLimitCooldownUntil = 0;
+    /** @type {() => number} */
+    this._random = typeof options.random === 'function' ? options.random : Math.random;
+    /** @type {() => number} */
+    this._now = typeof options.now === 'function' ? options.now : Date.now;
   }
 
   // ---------------------------------------------------------------------------
@@ -135,8 +174,10 @@ class OrchestrationEngine extends EventEmitter {
 
       // Get config for maxParallel and maxRetries
       const config = await this._orchestratorConfig.getConfig();
+      this._applyRetryPolicyConfig(config);
       const maxParallel = options.maxParallel || config.maxParallel || 3;
       const maxRetries = config.maxRetries || 2;
+      this._initializeAdaptiveParallel(maxParallel);
 
       // Step 5: Execute batches (Req 3.4)
       await this._executeBatches(batches, maxParallel, maxRetries);
@@ -250,7 +291,11 @@ class OrchestrationEngine extends EventEmitter {
     const inFlight = new Map(); // specName → Promise
 
     const launchNext = async () => {
-      while (pending.length > 0 && inFlight.size < maxParallel && !this._stopped) {
+      while (
+        pending.length > 0
+        && inFlight.size < this._getEffectiveMaxParallel(maxParallel)
+        && !this._stopped
+      ) {
         const specName = pending.shift();
         if (this._skippedSpecs.has(specName)) continue;
 
@@ -400,25 +445,47 @@ class OrchestrationEngine extends EventEmitter {
    * @private
    */
   async _handleSpecFailed(specName, agentId, maxRetries, error) {
+    const resolvedError = `${error || 'Unknown error'}`;
     const retryCount = this._retryCounts.get(specName) || 0;
+    const isRateLimitError = this._isRateLimitError(resolvedError);
+    const retryLimit = isRateLimitError
+      ? Math.max(maxRetries, this._rateLimitMaxRetries || DEFAULT_RATE_LIMIT_MAX_RETRIES)
+      : maxRetries;
 
-    if (retryCount < maxRetries && !this._stopped) {
+    if (retryCount < retryLimit && !this._stopped) {
       // Retry (Req 5.2)
       this._retryCounts.set(specName, retryCount + 1);
       this._statusMonitor.incrementRetry(specName);
-      this._statusMonitor.updateSpecStatus(specName, 'pending', null, error);
+      this._statusMonitor.updateSpecStatus(specName, 'pending', null, resolvedError);
+
+      const retryDelayMs = isRateLimitError
+        ? this._calculateRateLimitBackoffMs(retryCount)
+        : 0;
+      if (retryDelayMs > 0) {
+        this._onRateLimitSignal();
+        this.emit('spec:rate-limited', {
+          specName,
+          retryCount,
+          retryDelayMs,
+          error: resolvedError,
+        });
+        await this._sleep(retryDelayMs);
+        if (this._stopped) {
+          return;
+        }
+      }
 
       // Re-execute
       await this._executeSpec(specName, maxRetries);
     } else {
       // Final failure (Req 5.3)
       this._failedSpecs.add(specName);
-      this._statusMonitor.updateSpecStatus(specName, 'failed', agentId, error);
+      this._statusMonitor.updateSpecStatus(specName, 'failed', agentId, resolvedError);
 
       // Sync external status
       await this._syncExternalSafe(specName, 'failed');
 
-      this.emit('spec:failed', { specName, agentId, error, retryCount });
+      this.emit('spec:failed', { specName, agentId, error: resolvedError, retryCount });
 
       // Propagate failure to dependents (Req 3.6)
       this._propagateFailure(specName);
@@ -537,6 +604,243 @@ class OrchestrationEngine extends EventEmitter {
   // Validation & Helpers
   // ---------------------------------------------------------------------------
 
+  /**
+   * Resolve retry-related runtime config with safe defaults.
+   *
+   * @param {object} config
+   * @private
+   */
+  _applyRetryPolicyConfig(config) {
+    this._rateLimitMaxRetries = this._toNonNegativeInteger(
+      config && config.rateLimitMaxRetries,
+      DEFAULT_RATE_LIMIT_MAX_RETRIES
+    );
+
+    const baseMs = this._toPositiveInteger(
+      config && config.rateLimitBackoffBaseMs,
+      DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS
+    );
+    const maxMs = this._toPositiveInteger(
+      config && config.rateLimitBackoffMaxMs,
+      DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS
+    );
+
+    this._rateLimitBackoffBaseMs = Math.min(baseMs, maxMs);
+    this._rateLimitBackoffMaxMs = Math.max(baseMs, maxMs);
+    this._rateLimitAdaptiveParallel = this._toBoolean(
+      config && config.rateLimitAdaptiveParallel,
+      DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL
+    );
+    this._rateLimitParallelFloor = this._toPositiveInteger(
+      config && config.rateLimitParallelFloor,
+      DEFAULT_RATE_LIMIT_PARALLEL_FLOOR
+    );
+    this._rateLimitCooldownMs = this._toPositiveInteger(
+      config && config.rateLimitCooldownMs,
+      DEFAULT_RATE_LIMIT_COOLDOWN_MS
+    );
+  }
+
+  /**
+   * @param {number} maxParallel
+   * @private
+   */
+  _initializeAdaptiveParallel(maxParallel) {
+    const boundedMax = this._toPositiveInteger(maxParallel, 1);
+    this._baseMaxParallel = boundedMax;
+    this._effectiveMaxParallel = boundedMax;
+    this._rateLimitCooldownUntil = 0;
+  }
+
+  /**
+   * @param {number} maxParallel
+   * @returns {number}
+   * @private
+   */
+  _getEffectiveMaxParallel(maxParallel) {
+    const boundedMax = this._toPositiveInteger(maxParallel, 1);
+    if (!this._isAdaptiveParallelEnabled()) {
+      this._baseMaxParallel = boundedMax;
+      return boundedMax;
+    }
+
+    this._baseMaxParallel = boundedMax;
+    this._maybeRecoverParallelLimit(boundedMax);
+
+    const floor = Math.min(
+      boundedMax,
+      this._toPositiveInteger(this._rateLimitParallelFloor, DEFAULT_RATE_LIMIT_PARALLEL_FLOOR)
+    );
+    const effective = this._toPositiveInteger(this._effectiveMaxParallel, boundedMax);
+    return Math.max(floor, Math.min(boundedMax, effective));
+  }
+
+  /**
+   * @private
+   */
+  _onRateLimitSignal() {
+    if (!this._isAdaptiveParallelEnabled()) {
+      return;
+    }
+
+    const base = this._toPositiveInteger(this._baseMaxParallel, 1);
+    const current = this._toPositiveInteger(this._effectiveMaxParallel, base);
+    const floor = Math.min(
+      base,
+      this._toPositiveInteger(this._rateLimitParallelFloor, DEFAULT_RATE_LIMIT_PARALLEL_FLOOR)
+    );
+    const next = Math.max(floor, Math.floor(current / 2));
+
+    if (next < current) {
+      this._effectiveMaxParallel = next;
+      this.emit('parallel:throttled', {
+        reason: 'rate-limit',
+        previousMaxParallel: current,
+        effectiveMaxParallel: next,
+        floor,
+      });
+    } else {
+      this._effectiveMaxParallel = current;
+    }
+
+    this._rateLimitCooldownUntil = this._getNow() + this._rateLimitCooldownMs;
+  }
+
+  /**
+   * @param {number} maxParallel
+   * @private
+   */
+  _maybeRecoverParallelLimit(maxParallel) {
+    if (!this._isAdaptiveParallelEnabled()) {
+      return;
+    }
+
+    const boundedMax = this._toPositiveInteger(maxParallel, 1);
+    const current = this._toPositiveInteger(this._effectiveMaxParallel, boundedMax);
+    if (current >= boundedMax) {
+      this._effectiveMaxParallel = boundedMax;
+      return;
+    }
+
+    if (this._getNow() < this._rateLimitCooldownUntil) {
+      return;
+    }
+
+    const next = Math.min(boundedMax, current + 1);
+    if (next > current) {
+      this._effectiveMaxParallel = next;
+      this._rateLimitCooldownUntil = this._getNow() + this._rateLimitCooldownMs;
+      this.emit('parallel:recovered', {
+        previousMaxParallel: current,
+        effectiveMaxParallel: next,
+        maxParallel: boundedMax,
+      });
+    }
+  }
+
+  /**
+   * @returns {boolean}
+   * @private
+   */
+  _isAdaptiveParallelEnabled() {
+    if (typeof this._rateLimitAdaptiveParallel === 'boolean') {
+      return this._rateLimitAdaptiveParallel;
+    }
+    return DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL;
+  }
+
+  /**
+   * @returns {number}
+   * @private
+   */
+  _getNow() {
+    return typeof this._now === 'function' ? this._now() : Date.now();
+  }
+
+  /**
+   * @param {any} value
+   * @param {boolean} fallback
+   * @returns {boolean}
+   * @private
+   */
+  _toBoolean(value, fallback) {
+    if (typeof value === 'boolean') {
+      return value;
+    }
+    return fallback;
+  }
+
+  /**
+   * @param {any} value
+   * @param {number} fallback
+   * @returns {number}
+   * @private
+   */
+  _toPositiveInteger(value, fallback) {
+    const numeric = Number(value);
+    if (!Number.isFinite(numeric) || numeric <= 0) {
+      return fallback;
+    }
+    return Math.floor(numeric);
+  }
+
+  /**
+   * @param {any} value
+   * @param {number} fallback
+   * @returns {number}
+   * @private
+   */
+  _toNonNegativeInteger(value, fallback) {
+    const numeric = Number(value);
+    if (!Number.isFinite(numeric) || numeric < 0) {
+      return fallback;
+    }
+    return Math.floor(numeric);
+  }
+
+  /**
+   * @param {string} error
+   * @returns {boolean}
+   * @private
+   */
+  _isRateLimitError(error) {
+    return RATE_LIMIT_ERROR_PATTERNS.some(pattern => pattern.test(`${error || ''}`));
+  }
+
+  /**
+   * @param {number} retryCount
+   * @returns {number}
+   * @private
+   */
+  _calculateRateLimitBackoffMs(retryCount) {
+    const exponent = Math.max(0, retryCount);
+    const cappedBaseDelay = Math.min(
+      this._rateLimitBackoffMaxMs || DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS,
+      (this._rateLimitBackoffBaseMs || DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS) * (2 ** exponent)
+    );
+
+    const randomValue = typeof this._random === 'function' ? this._random() : Math.random();
+    const normalizedRandom = Number.isFinite(randomValue)
+      ? Math.min(1, Math.max(0, randomValue))
+      : 0.5;
+    const jitterFactor = (1 - RATE_LIMIT_BACKOFF_JITTER_RATIO)
+      + (normalizedRandom * RATE_LIMIT_BACKOFF_JITTER_RATIO);
+
+    return Math.max(1, Math.round(cappedBaseDelay * jitterFactor));
+  }
+
+  /**
+   * @param {number} ms
+   * @returns {Promise<void>}
+   * @private
+   */
+  _sleep(ms) {
+    if (!ms || ms <= 0) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
   /**
    * Validate that all spec directories exist (Req 6.4).
    *
@@ -625,6 +929,9 @@ class OrchestrationEngine extends EventEmitter {
     this._completedSpecs.clear();
     this._executionPlan = null;
     this._stopped = false;
+    this._baseMaxParallel = null;
+    this._effectiveMaxParallel = null;
+    this._rateLimitCooldownUntil = 0;
   }
 }
 

package/lib/orchestrator/orchestrator-config.js

@@ -24,6 +24,12 @@ const KNOWN_KEYS = new Set([
   'maxParallel',
   'timeoutSeconds',
   'maxRetries',
+  'rateLimitMaxRetries',
+  'rateLimitBackoffBaseMs',
+  'rateLimitBackoffMaxMs',
+  'rateLimitAdaptiveParallel',
+  'rateLimitParallelFloor',
+  'rateLimitCooldownMs',
   'apiKeyEnvVar',
   'bootstrapTemplate',
   'codexArgs',
@@ -36,6 +42,12 @@ const DEFAULT_CONFIG = Object.freeze({
   maxParallel: 3,
   timeoutSeconds: 600,
   maxRetries: 2,
+  rateLimitMaxRetries: 6,
+  rateLimitBackoffBaseMs: 1000,
+  rateLimitBackoffMaxMs: 30000,
+  rateLimitAdaptiveParallel: true,
+  rateLimitParallelFloor: 1,
+  rateLimitCooldownMs: 30000,
   apiKeyEnvVar: 'CODEX_API_KEY',
   bootstrapTemplate: null,
   codexArgs: [],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kiro-spec-engine",
-  "version": "1.47.10",
+  "version": "1.47.12",
   "description": "kiro-spec-engine (kse) - A CLI tool and npm package for spec-driven development with AI coding assistants. NOT the Kiro IDE desktop application.",
   "main": "index.js",
   "bin": {