kiro-spec-engine 1.47.10 → 1.47.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/README.zh.md +7 -0
- package/docs/command-reference.md +8 -0
- package/lib/commands/orchestrate.js +14 -0
- package/lib/orchestrator/orchestration-engine.js +390 -5
- package/lib/orchestrator/orchestrator-config.js +12 -0
- package/lib/orchestrator/status-monitor.js +107 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -413,6 +413,12 @@ Tip: `kse spec bootstrap|pipeline run|gate run --specs ...` now defaults to this
|
|
|
413
413
|
"maxParallel": 3,
|
|
414
414
|
"timeoutSeconds": 900,
|
|
415
415
|
"maxRetries": 2,
|
|
416
|
+
"rateLimitMaxRetries": 6,
|
|
417
|
+
"rateLimitBackoffBaseMs": 1000,
|
|
418
|
+
"rateLimitBackoffMaxMs": 30000,
|
|
419
|
+
"rateLimitAdaptiveParallel": true,
|
|
420
|
+
"rateLimitParallelFloor": 1,
|
|
421
|
+
"rateLimitCooldownMs": 30000,
|
|
416
422
|
"apiKeyEnvVar": "CODEX_API_KEY",
|
|
417
423
|
"codexArgs": ["--skip-git-repo-check"],
|
|
418
424
|
"codexCommand": "npx @openai/codex"
|
|
@@ -420,6 +426,7 @@ Tip: `kse spec bootstrap|pipeline run|gate run --specs ...` now defaults to this
|
|
|
420
426
|
```
|
|
421
427
|
|
|
422
428
|
If Codex CLI is globally installed, you can set `"codexCommand": "codex"`.
|
|
429
|
+
Use the `rateLimit*` settings to absorb transient 429/too-many-requests failures without stalling orchestration.
|
|
423
430
|
|
|
424
431
|
### Spec-Level Steering & Context Sync 🚀 NEW in v1.44.0
|
|
425
432
|
- **Spec Steering (L4)**: Independent `steering.md` per Spec with constraints, notes, and decisions — zero cross-agent conflict
|
package/README.zh.md
CHANGED
|
@@ -367,6 +367,12 @@ kse orchestrate stop
|
|
|
367
367
|
"maxParallel": 3,
|
|
368
368
|
"timeoutSeconds": 900,
|
|
369
369
|
"maxRetries": 2,
|
|
370
|
+
"rateLimitMaxRetries": 6,
|
|
371
|
+
"rateLimitBackoffBaseMs": 1000,
|
|
372
|
+
"rateLimitBackoffMaxMs": 30000,
|
|
373
|
+
"rateLimitAdaptiveParallel": true,
|
|
374
|
+
"rateLimitParallelFloor": 1,
|
|
375
|
+
"rateLimitCooldownMs": 30000,
|
|
370
376
|
"apiKeyEnvVar": "CODEX_API_KEY",
|
|
371
377
|
"codexArgs": ["--skip-git-repo-check"],
|
|
372
378
|
"codexCommand": "npx @openai/codex"
|
|
@@ -374,6 +380,7 @@ kse orchestrate stop
|
|
|
374
380
|
```
|
|
375
381
|
|
|
376
382
|
如果你已全局安装 Codex CLI,可将 `"codexCommand"` 改为 `"codex"`。
|
|
383
|
+
可通过 `rateLimit*` 配置吸收 429/too-many-requests 等限流抖动,避免编排流程卡死。
|
|
377
384
|
|
|
378
385
|
### Spec 级 Steering 与上下文同步 🚀 v1.44.0 新增
|
|
379
386
|
- **Spec Steering (L4)**: 每个 Spec 独立的 `steering.md`,包含约束、注意事项、决策记录 — 跨 Agent 零冲突
|
|
@@ -787,12 +787,20 @@ Recommended `.kiro/config/orchestrator.json`:
|
|
|
787
787
|
"maxParallel": 3,
|
|
788
788
|
"timeoutSeconds": 900,
|
|
789
789
|
"maxRetries": 2,
|
|
790
|
+
"rateLimitMaxRetries": 6,
|
|
791
|
+
"rateLimitBackoffBaseMs": 1000,
|
|
792
|
+
"rateLimitBackoffMaxMs": 30000,
|
|
793
|
+
"rateLimitAdaptiveParallel": true,
|
|
794
|
+
"rateLimitParallelFloor": 1,
|
|
795
|
+
"rateLimitCooldownMs": 30000,
|
|
790
796
|
"apiKeyEnvVar": "CODEX_API_KEY",
|
|
791
797
|
"codexArgs": ["--skip-git-repo-check"],
|
|
792
798
|
"codexCommand": "npx @openai/codex"
|
|
793
799
|
}
|
|
794
800
|
```
|
|
795
801
|
|
|
802
|
+
`rateLimit*` settings provide dedicated retry/backoff and adaptive parallel throttling when providers return 429 / too-many-requests errors.
|
|
803
|
+
|
|
796
804
|
### Scene Template Engine
|
|
797
805
|
|
|
798
806
|
```bash
|
|
@@ -124,6 +124,9 @@ async function runOrchestration(options = {}, dependencies = {}) {
|
|
|
124
124
|
'spec:start',
|
|
125
125
|
'spec:complete',
|
|
126
126
|
'spec:failed',
|
|
127
|
+
'spec:rate-limited',
|
|
128
|
+
'parallel:throttled',
|
|
129
|
+
'parallel:recovered',
|
|
127
130
|
'orchestration:complete'
|
|
128
131
|
];
|
|
129
132
|
|
|
@@ -322,6 +325,17 @@ function _printStatus(status) {
|
|
|
322
325
|
if (status.currentBatch !== undefined && status.totalBatches !== undefined) {
|
|
323
326
|
console.log(`Batch: ${status.currentBatch} / ${status.totalBatches}`);
|
|
324
327
|
}
|
|
328
|
+
if (status.parallel) {
|
|
329
|
+
const effective = status.parallel.effectiveMaxParallel ?? '-';
|
|
330
|
+
const max = status.parallel.maxParallel ?? '-';
|
|
331
|
+
const adaptive = status.parallel.adaptive === false ? 'off' : 'on';
|
|
332
|
+
console.log(`Parallel: ${effective} / ${max} (adaptive: ${adaptive})`);
|
|
333
|
+
}
|
|
334
|
+
if (status.rateLimit) {
|
|
335
|
+
const signals = status.rateLimit.signalCount || 0;
|
|
336
|
+
const backoff = status.rateLimit.totalBackoffMs || 0;
|
|
337
|
+
console.log(`Rate-limit: ${signals} signal(s), total backoff ${backoff}ms`);
|
|
338
|
+
}
|
|
325
339
|
if (status.specs) {
|
|
326
340
|
console.log('');
|
|
327
341
|
for (const [name, info] of Object.entries(status.specs)) {
|
|
@@ -16,6 +16,23 @@ const path = require('path');
|
|
|
16
16
|
const fsUtils = require('../utils/fs-utils');
|
|
17
17
|
|
|
18
18
|
const SPECS_DIR = '.kiro/specs';
|
|
19
|
+
const DEFAULT_RATE_LIMIT_MAX_RETRIES = 6;
|
|
20
|
+
const DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS = 1000;
|
|
21
|
+
const DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS = 30000;
|
|
22
|
+
const DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL = true;
|
|
23
|
+
const DEFAULT_RATE_LIMIT_PARALLEL_FLOOR = 1;
|
|
24
|
+
const DEFAULT_RATE_LIMIT_COOLDOWN_MS = 30000;
|
|
25
|
+
const RATE_LIMIT_BACKOFF_JITTER_RATIO = 0.5;
|
|
26
|
+
const RATE_LIMIT_ERROR_PATTERNS = [
|
|
27
|
+
/(^|[^0-9])429([^0-9]|$)/i,
|
|
28
|
+
/too many requests/i,
|
|
29
|
+
/rate[\s-]?limit/i,
|
|
30
|
+
/resource exhausted/i,
|
|
31
|
+
/quota exceeded/i,
|
|
32
|
+
/exceeded.*quota/i,
|
|
33
|
+
/requests per minute/i,
|
|
34
|
+
/tokens per minute/i,
|
|
35
|
+
];
|
|
19
36
|
|
|
20
37
|
class OrchestrationEngine extends EventEmitter {
|
|
21
38
|
/**
|
|
@@ -54,6 +71,28 @@ class OrchestrationEngine extends EventEmitter {
|
|
|
54
71
|
this._stopped = false;
|
|
55
72
|
/** @type {object|null} execution plan */
|
|
56
73
|
this._executionPlan = null;
|
|
74
|
+
/** @type {number} max retries for rate-limit failures */
|
|
75
|
+
this._rateLimitMaxRetries = DEFAULT_RATE_LIMIT_MAX_RETRIES;
|
|
76
|
+
/** @type {number} base delay for rate-limit retries */
|
|
77
|
+
this._rateLimitBackoffBaseMs = DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS;
|
|
78
|
+
/** @type {number} max delay for rate-limit retries */
|
|
79
|
+
this._rateLimitBackoffMaxMs = DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS;
|
|
80
|
+
/** @type {boolean} enable adaptive parallel throttling on rate-limit signals */
|
|
81
|
+
this._rateLimitAdaptiveParallel = DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL;
|
|
82
|
+
/** @type {number} minimum effective parallelism during rate-limit cooldown */
|
|
83
|
+
this._rateLimitParallelFloor = DEFAULT_RATE_LIMIT_PARALLEL_FLOOR;
|
|
84
|
+
/** @type {number} cooldown before each adaptive parallel recovery step */
|
|
85
|
+
this._rateLimitCooldownMs = DEFAULT_RATE_LIMIT_COOLDOWN_MS;
|
|
86
|
+
/** @type {number|null} configured max parallel for current run */
|
|
87
|
+
this._baseMaxParallel = null;
|
|
88
|
+
/** @type {number|null} dynamic effective parallel limit for current run */
|
|
89
|
+
this._effectiveMaxParallel = null;
|
|
90
|
+
/** @type {number} timestamp after which recovery can step up */
|
|
91
|
+
this._rateLimitCooldownUntil = 0;
|
|
92
|
+
/** @type {() => number} */
|
|
93
|
+
this._random = typeof options.random === 'function' ? options.random : Math.random;
|
|
94
|
+
/** @type {() => number} */
|
|
95
|
+
this._now = typeof options.now === 'function' ? options.now : Date.now;
|
|
57
96
|
}
|
|
58
97
|
|
|
59
98
|
// ---------------------------------------------------------------------------
|
|
@@ -135,8 +174,10 @@ class OrchestrationEngine extends EventEmitter {
|
|
|
135
174
|
|
|
136
175
|
// Get config for maxParallel and maxRetries
|
|
137
176
|
const config = await this._orchestratorConfig.getConfig();
|
|
177
|
+
this._applyRetryPolicyConfig(config);
|
|
138
178
|
const maxParallel = options.maxParallel || config.maxParallel || 3;
|
|
139
179
|
const maxRetries = config.maxRetries || 2;
|
|
180
|
+
this._initializeAdaptiveParallel(maxParallel);
|
|
140
181
|
|
|
141
182
|
// Step 5: Execute batches (Req 3.4)
|
|
142
183
|
await this._executeBatches(batches, maxParallel, maxRetries);
|
|
@@ -250,7 +291,11 @@ class OrchestrationEngine extends EventEmitter {
|
|
|
250
291
|
const inFlight = new Map(); // specName → Promise
|
|
251
292
|
|
|
252
293
|
const launchNext = async () => {
|
|
253
|
-
while (
|
|
294
|
+
while (
|
|
295
|
+
pending.length > 0
|
|
296
|
+
&& inFlight.size < this._getEffectiveMaxParallel(maxParallel)
|
|
297
|
+
&& !this._stopped
|
|
298
|
+
) {
|
|
254
299
|
const specName = pending.shift();
|
|
255
300
|
if (this._skippedSpecs.has(specName)) continue;
|
|
256
301
|
|
|
@@ -400,25 +445,53 @@ class OrchestrationEngine extends EventEmitter {
|
|
|
400
445
|
* @private
|
|
401
446
|
*/
|
|
402
447
|
async _handleSpecFailed(specName, agentId, maxRetries, error) {
|
|
448
|
+
const resolvedError = `${error || 'Unknown error'}`;
|
|
403
449
|
const retryCount = this._retryCounts.get(specName) || 0;
|
|
450
|
+
const isRateLimitError = this._isRateLimitError(resolvedError);
|
|
451
|
+
const retryLimit = isRateLimitError
|
|
452
|
+
? Math.max(maxRetries, this._rateLimitMaxRetries || DEFAULT_RATE_LIMIT_MAX_RETRIES)
|
|
453
|
+
: maxRetries;
|
|
404
454
|
|
|
405
|
-
if (retryCount <
|
|
455
|
+
if (retryCount < retryLimit && !this._stopped) {
|
|
406
456
|
// Retry (Req 5.2)
|
|
407
457
|
this._retryCounts.set(specName, retryCount + 1);
|
|
408
458
|
this._statusMonitor.incrementRetry(specName);
|
|
409
|
-
this._statusMonitor.updateSpecStatus(specName, 'pending', null,
|
|
459
|
+
this._statusMonitor.updateSpecStatus(specName, 'pending', null, resolvedError);
|
|
460
|
+
|
|
461
|
+
const retryDelayMs = isRateLimitError
|
|
462
|
+
? this._calculateRateLimitBackoffMs(retryCount)
|
|
463
|
+
: 0;
|
|
464
|
+
if (retryDelayMs > 0) {
|
|
465
|
+
this._onRateLimitSignal();
|
|
466
|
+
this._updateStatusMonitorRateLimit({
|
|
467
|
+
specName,
|
|
468
|
+
retryCount,
|
|
469
|
+
retryDelayMs,
|
|
470
|
+
error: resolvedError,
|
|
471
|
+
});
|
|
472
|
+
this.emit('spec:rate-limited', {
|
|
473
|
+
specName,
|
|
474
|
+
retryCount,
|
|
475
|
+
retryDelayMs,
|
|
476
|
+
error: resolvedError,
|
|
477
|
+
});
|
|
478
|
+
await this._sleep(retryDelayMs);
|
|
479
|
+
if (this._stopped) {
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
410
483
|
|
|
411
484
|
// Re-execute
|
|
412
485
|
await this._executeSpec(specName, maxRetries);
|
|
413
486
|
} else {
|
|
414
487
|
// Final failure (Req 5.3)
|
|
415
488
|
this._failedSpecs.add(specName);
|
|
416
|
-
this._statusMonitor.updateSpecStatus(specName, 'failed', agentId,
|
|
489
|
+
this._statusMonitor.updateSpecStatus(specName, 'failed', agentId, resolvedError);
|
|
417
490
|
|
|
418
491
|
// Sync external status
|
|
419
492
|
await this._syncExternalSafe(specName, 'failed');
|
|
420
493
|
|
|
421
|
-
this.emit('spec:failed', { specName, agentId, error, retryCount });
|
|
494
|
+
this.emit('spec:failed', { specName, agentId, error: resolvedError, retryCount });
|
|
422
495
|
|
|
423
496
|
// Propagate failure to dependents (Req 3.6)
|
|
424
497
|
this._propagateFailure(specName);
|
|
@@ -537,6 +610,315 @@ class OrchestrationEngine extends EventEmitter {
|
|
|
537
610
|
// Validation & Helpers
|
|
538
611
|
// ---------------------------------------------------------------------------
|
|
539
612
|
|
|
613
|
+
/**
|
|
614
|
+
* Resolve retry-related runtime config with safe defaults.
|
|
615
|
+
*
|
|
616
|
+
* @param {object} config
|
|
617
|
+
* @private
|
|
618
|
+
*/
|
|
619
|
+
_applyRetryPolicyConfig(config) {
|
|
620
|
+
this._rateLimitMaxRetries = this._toNonNegativeInteger(
|
|
621
|
+
config && config.rateLimitMaxRetries,
|
|
622
|
+
DEFAULT_RATE_LIMIT_MAX_RETRIES
|
|
623
|
+
);
|
|
624
|
+
|
|
625
|
+
const baseMs = this._toPositiveInteger(
|
|
626
|
+
config && config.rateLimitBackoffBaseMs,
|
|
627
|
+
DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS
|
|
628
|
+
);
|
|
629
|
+
const maxMs = this._toPositiveInteger(
|
|
630
|
+
config && config.rateLimitBackoffMaxMs,
|
|
631
|
+
DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS
|
|
632
|
+
);
|
|
633
|
+
|
|
634
|
+
this._rateLimitBackoffBaseMs = Math.min(baseMs, maxMs);
|
|
635
|
+
this._rateLimitBackoffMaxMs = Math.max(baseMs, maxMs);
|
|
636
|
+
this._rateLimitAdaptiveParallel = this._toBoolean(
|
|
637
|
+
config && config.rateLimitAdaptiveParallel,
|
|
638
|
+
DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL
|
|
639
|
+
);
|
|
640
|
+
this._rateLimitParallelFloor = this._toPositiveInteger(
|
|
641
|
+
config && config.rateLimitParallelFloor,
|
|
642
|
+
DEFAULT_RATE_LIMIT_PARALLEL_FLOOR
|
|
643
|
+
);
|
|
644
|
+
this._rateLimitCooldownMs = this._toPositiveInteger(
|
|
645
|
+
config && config.rateLimitCooldownMs,
|
|
646
|
+
DEFAULT_RATE_LIMIT_COOLDOWN_MS
|
|
647
|
+
);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* @param {number} maxParallel
|
|
652
|
+
* @private
|
|
653
|
+
*/
|
|
654
|
+
_initializeAdaptiveParallel(maxParallel) {
|
|
655
|
+
const boundedMax = this._toPositiveInteger(maxParallel, 1);
|
|
656
|
+
this._baseMaxParallel = boundedMax;
|
|
657
|
+
this._effectiveMaxParallel = boundedMax;
|
|
658
|
+
this._rateLimitCooldownUntil = 0;
|
|
659
|
+
this._updateStatusMonitorParallelTelemetry({
|
|
660
|
+
adaptive: this._isAdaptiveParallelEnabled(),
|
|
661
|
+
maxParallel: boundedMax,
|
|
662
|
+
effectiveMaxParallel: boundedMax,
|
|
663
|
+
floor: Math.min(
|
|
664
|
+
boundedMax,
|
|
665
|
+
this._toPositiveInteger(this._rateLimitParallelFloor, DEFAULT_RATE_LIMIT_PARALLEL_FLOOR)
|
|
666
|
+
),
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* @param {number} maxParallel
|
|
672
|
+
* @returns {number}
|
|
673
|
+
* @private
|
|
674
|
+
*/
|
|
675
|
+
_getEffectiveMaxParallel(maxParallel) {
|
|
676
|
+
const boundedMax = this._toPositiveInteger(maxParallel, 1);
|
|
677
|
+
const floor = Math.min(
|
|
678
|
+
boundedMax,
|
|
679
|
+
this._toPositiveInteger(this._rateLimitParallelFloor, DEFAULT_RATE_LIMIT_PARALLEL_FLOOR)
|
|
680
|
+
);
|
|
681
|
+
|
|
682
|
+
if (!this._isAdaptiveParallelEnabled()) {
|
|
683
|
+
this._baseMaxParallel = boundedMax;
|
|
684
|
+
this._effectiveMaxParallel = boundedMax;
|
|
685
|
+
this._updateStatusMonitorParallelTelemetry({
|
|
686
|
+
adaptive: false,
|
|
687
|
+
maxParallel: boundedMax,
|
|
688
|
+
effectiveMaxParallel: boundedMax,
|
|
689
|
+
floor,
|
|
690
|
+
});
|
|
691
|
+
return boundedMax;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
this._baseMaxParallel = boundedMax;
|
|
695
|
+
this._maybeRecoverParallelLimit(boundedMax);
|
|
696
|
+
|
|
697
|
+
const effective = this._toPositiveInteger(this._effectiveMaxParallel, boundedMax);
|
|
698
|
+
const resolved = Math.max(floor, Math.min(boundedMax, effective));
|
|
699
|
+
this._updateStatusMonitorParallelTelemetry({
|
|
700
|
+
adaptive: true,
|
|
701
|
+
maxParallel: boundedMax,
|
|
702
|
+
effectiveMaxParallel: resolved,
|
|
703
|
+
floor,
|
|
704
|
+
});
|
|
705
|
+
return resolved;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* @private
|
|
710
|
+
*/
|
|
711
|
+
_onRateLimitSignal() {
|
|
712
|
+
if (!this._isAdaptiveParallelEnabled()) {
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
const base = this._toPositiveInteger(this._baseMaxParallel, 1);
|
|
717
|
+
const current = this._toPositiveInteger(this._effectiveMaxParallel, base);
|
|
718
|
+
const floor = Math.min(
|
|
719
|
+
base,
|
|
720
|
+
this._toPositiveInteger(this._rateLimitParallelFloor, DEFAULT_RATE_LIMIT_PARALLEL_FLOOR)
|
|
721
|
+
);
|
|
722
|
+
const next = Math.max(floor, Math.floor(current / 2));
|
|
723
|
+
|
|
724
|
+
if (next < current) {
|
|
725
|
+
this._effectiveMaxParallel = next;
|
|
726
|
+
this._updateStatusMonitorParallelTelemetry({
|
|
727
|
+
event: 'throttled',
|
|
728
|
+
reason: 'rate-limit',
|
|
729
|
+
adaptive: true,
|
|
730
|
+
maxParallel: base,
|
|
731
|
+
effectiveMaxParallel: next,
|
|
732
|
+
floor,
|
|
733
|
+
});
|
|
734
|
+
this.emit('parallel:throttled', {
|
|
735
|
+
reason: 'rate-limit',
|
|
736
|
+
previousMaxParallel: current,
|
|
737
|
+
effectiveMaxParallel: next,
|
|
738
|
+
floor,
|
|
739
|
+
});
|
|
740
|
+
} else {
|
|
741
|
+
this._effectiveMaxParallel = current;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
this._rateLimitCooldownUntil = this._getNow() + this._rateLimitCooldownMs;
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* @param {number} maxParallel
|
|
749
|
+
* @private
|
|
750
|
+
*/
|
|
751
|
+
_maybeRecoverParallelLimit(maxParallel) {
|
|
752
|
+
if (!this._isAdaptiveParallelEnabled()) {
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
const boundedMax = this._toPositiveInteger(maxParallel, 1);
|
|
757
|
+
const current = this._toPositiveInteger(this._effectiveMaxParallel, boundedMax);
|
|
758
|
+
if (current >= boundedMax) {
|
|
759
|
+
this._effectiveMaxParallel = boundedMax;
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
if (this._getNow() < this._rateLimitCooldownUntil) {
|
|
764
|
+
return;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
const next = Math.min(boundedMax, current + 1);
|
|
768
|
+
if (next > current) {
|
|
769
|
+
this._effectiveMaxParallel = next;
|
|
770
|
+
this._rateLimitCooldownUntil = this._getNow() + this._rateLimitCooldownMs;
|
|
771
|
+
this._updateStatusMonitorParallelTelemetry({
|
|
772
|
+
event: 'recovered',
|
|
773
|
+
adaptive: true,
|
|
774
|
+
maxParallel: boundedMax,
|
|
775
|
+
effectiveMaxParallel: next,
|
|
776
|
+
});
|
|
777
|
+
this.emit('parallel:recovered', {
|
|
778
|
+
previousMaxParallel: current,
|
|
779
|
+
effectiveMaxParallel: next,
|
|
780
|
+
maxParallel: boundedMax,
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
/**
|
|
786
|
+
* @returns {boolean}
|
|
787
|
+
* @private
|
|
788
|
+
*/
|
|
789
|
+
_isAdaptiveParallelEnabled() {
|
|
790
|
+
if (typeof this._rateLimitAdaptiveParallel === 'boolean') {
|
|
791
|
+
return this._rateLimitAdaptiveParallel;
|
|
792
|
+
}
|
|
793
|
+
return DEFAULT_RATE_LIMIT_ADAPTIVE_PARALLEL;
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* @returns {number}
|
|
798
|
+
* @private
|
|
799
|
+
*/
|
|
800
|
+
_getNow() {
|
|
801
|
+
return typeof this._now === 'function' ? this._now() : Date.now();
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
/**
|
|
805
|
+
* @param {any} value
|
|
806
|
+
* @param {boolean} fallback
|
|
807
|
+
* @returns {boolean}
|
|
808
|
+
* @private
|
|
809
|
+
*/
|
|
810
|
+
_toBoolean(value, fallback) {
|
|
811
|
+
if (typeof value === 'boolean') {
|
|
812
|
+
return value;
|
|
813
|
+
}
|
|
814
|
+
return fallback;
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/**
|
|
818
|
+
* @param {any} value
|
|
819
|
+
* @param {number} fallback
|
|
820
|
+
* @returns {number}
|
|
821
|
+
* @private
|
|
822
|
+
*/
|
|
823
|
+
_toPositiveInteger(value, fallback) {
|
|
824
|
+
const numeric = Number(value);
|
|
825
|
+
if (!Number.isFinite(numeric) || numeric <= 0) {
|
|
826
|
+
return fallback;
|
|
827
|
+
}
|
|
828
|
+
return Math.floor(numeric);
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
/**
|
|
832
|
+
* @param {any} value
|
|
833
|
+
* @param {number} fallback
|
|
834
|
+
* @returns {number}
|
|
835
|
+
* @private
|
|
836
|
+
*/
|
|
837
|
+
_toNonNegativeInteger(value, fallback) {
|
|
838
|
+
const numeric = Number(value);
|
|
839
|
+
if (!Number.isFinite(numeric) || numeric < 0) {
|
|
840
|
+
return fallback;
|
|
841
|
+
}
|
|
842
|
+
return Math.floor(numeric);
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/**
|
|
846
|
+
* @param {string} error
|
|
847
|
+
* @returns {boolean}
|
|
848
|
+
* @private
|
|
849
|
+
*/
|
|
850
|
+
_isRateLimitError(error) {
|
|
851
|
+
return RATE_LIMIT_ERROR_PATTERNS.some(pattern => pattern.test(`${error || ''}`));
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/**
|
|
855
|
+
* @param {number} retryCount
|
|
856
|
+
* @returns {number}
|
|
857
|
+
* @private
|
|
858
|
+
*/
|
|
859
|
+
_calculateRateLimitBackoffMs(retryCount) {
|
|
860
|
+
const exponent = Math.max(0, retryCount);
|
|
861
|
+
const cappedBaseDelay = Math.min(
|
|
862
|
+
this._rateLimitBackoffMaxMs || DEFAULT_RATE_LIMIT_BACKOFF_MAX_MS,
|
|
863
|
+
(this._rateLimitBackoffBaseMs || DEFAULT_RATE_LIMIT_BACKOFF_BASE_MS) * (2 ** exponent)
|
|
864
|
+
);
|
|
865
|
+
|
|
866
|
+
const randomValue = typeof this._random === 'function' ? this._random() : Math.random();
|
|
867
|
+
const normalizedRandom = Number.isFinite(randomValue)
|
|
868
|
+
? Math.min(1, Math.max(0, randomValue))
|
|
869
|
+
: 0.5;
|
|
870
|
+
const jitterFactor = (1 - RATE_LIMIT_BACKOFF_JITTER_RATIO)
|
|
871
|
+
+ (normalizedRandom * RATE_LIMIT_BACKOFF_JITTER_RATIO);
|
|
872
|
+
|
|
873
|
+
return Math.max(1, Math.round(cappedBaseDelay * jitterFactor));
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* @param {number} ms
|
|
878
|
+
* @returns {Promise<void>}
|
|
879
|
+
* @private
|
|
880
|
+
*/
|
|
881
|
+
_sleep(ms) {
|
|
882
|
+
if (!ms || ms <= 0) {
|
|
883
|
+
return Promise.resolve();
|
|
884
|
+
}
|
|
885
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
/**
|
|
889
|
+
* Safely update StatusMonitor rate-limit telemetry.
|
|
890
|
+
*
|
|
891
|
+
* @param {object} payload
|
|
892
|
+
* @private
|
|
893
|
+
*/
|
|
894
|
+
_updateStatusMonitorRateLimit(payload) {
|
|
895
|
+
const handler = this._statusMonitor && this._statusMonitor.recordRateLimitEvent;
|
|
896
|
+
if (typeof handler === 'function') {
|
|
897
|
+
try {
|
|
898
|
+
handler.call(this._statusMonitor, payload);
|
|
899
|
+
} catch (_err) {
|
|
900
|
+
// Non-fatal status telemetry update.
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* Safely update StatusMonitor adaptive parallel telemetry.
|
|
907
|
+
*
|
|
908
|
+
* @param {object} payload
|
|
909
|
+
* @private
|
|
910
|
+
*/
|
|
911
|
+
_updateStatusMonitorParallelTelemetry(payload) {
|
|
912
|
+
const handler = this._statusMonitor && this._statusMonitor.updateParallelTelemetry;
|
|
913
|
+
if (typeof handler === 'function') {
|
|
914
|
+
try {
|
|
915
|
+
handler.call(this._statusMonitor, payload);
|
|
916
|
+
} catch (_err) {
|
|
917
|
+
// Non-fatal status telemetry update.
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
|
|
540
922
|
/**
|
|
541
923
|
* Validate that all spec directories exist (Req 6.4).
|
|
542
924
|
*
|
|
@@ -625,6 +1007,9 @@ class OrchestrationEngine extends EventEmitter {
|
|
|
625
1007
|
this._completedSpecs.clear();
|
|
626
1008
|
this._executionPlan = null;
|
|
627
1009
|
this._stopped = false;
|
|
1010
|
+
this._baseMaxParallel = null;
|
|
1011
|
+
this._effectiveMaxParallel = null;
|
|
1012
|
+
this._rateLimitCooldownUntil = 0;
|
|
628
1013
|
}
|
|
629
1014
|
}
|
|
630
1015
|
|
|
@@ -24,6 +24,12 @@ const KNOWN_KEYS = new Set([
|
|
|
24
24
|
'maxParallel',
|
|
25
25
|
'timeoutSeconds',
|
|
26
26
|
'maxRetries',
|
|
27
|
+
'rateLimitMaxRetries',
|
|
28
|
+
'rateLimitBackoffBaseMs',
|
|
29
|
+
'rateLimitBackoffMaxMs',
|
|
30
|
+
'rateLimitAdaptiveParallel',
|
|
31
|
+
'rateLimitParallelFloor',
|
|
32
|
+
'rateLimitCooldownMs',
|
|
27
33
|
'apiKeyEnvVar',
|
|
28
34
|
'bootstrapTemplate',
|
|
29
35
|
'codexArgs',
|
|
@@ -36,6 +42,12 @@ const DEFAULT_CONFIG = Object.freeze({
|
|
|
36
42
|
maxParallel: 3,
|
|
37
43
|
timeoutSeconds: 600,
|
|
38
44
|
maxRetries: 2,
|
|
45
|
+
rateLimitMaxRetries: 6,
|
|
46
|
+
rateLimitBackoffBaseMs: 1000,
|
|
47
|
+
rateLimitBackoffMaxMs: 30000,
|
|
48
|
+
rateLimitAdaptiveParallel: true,
|
|
49
|
+
rateLimitParallelFloor: 1,
|
|
50
|
+
rateLimitCooldownMs: 30000,
|
|
39
51
|
apiKeyEnvVar: 'CODEX_API_KEY',
|
|
40
52
|
bootstrapTemplate: null,
|
|
41
53
|
codexArgs: [],
|
|
@@ -56,6 +56,37 @@ class StatusMonitor {
|
|
|
56
56
|
* @type {Map<string, {status: string, batch: number, agentId: string|null, retryCount: number, error: string|null, turnCount: number}>}
|
|
57
57
|
*/
|
|
58
58
|
this._specs = new Map();
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Rate-limit telemetry summary.
|
|
62
|
+
* @type {{signalCount: number, retryCount: number, totalBackoffMs: number, lastSignalAt: string|null, lastSpecName: string|null, lastRetryCount: number, lastDelayMs: number, lastError: string|null}}
|
|
63
|
+
*/
|
|
64
|
+
this._rateLimit = {
|
|
65
|
+
signalCount: 0,
|
|
66
|
+
retryCount: 0,
|
|
67
|
+
totalBackoffMs: 0,
|
|
68
|
+
lastSignalAt: null,
|
|
69
|
+
lastSpecName: null,
|
|
70
|
+
lastRetryCount: 0,
|
|
71
|
+
lastDelayMs: 0,
|
|
72
|
+
lastError: null,
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Parallelism telemetry summary.
|
|
77
|
+
* @type {{adaptive: boolean|null, maxParallel: number|null, effectiveMaxParallel: number|null, floor: number|null, throttledCount: number, recoveredCount: number, lastReason: string|null, lastThrottledAt: string|null, lastRecoveredAt: string|null}}
|
|
78
|
+
*/
|
|
79
|
+
this._parallel = {
|
|
80
|
+
adaptive: null,
|
|
81
|
+
maxParallel: null,
|
|
82
|
+
effectiveMaxParallel: null,
|
|
83
|
+
floor: null,
|
|
84
|
+
throttledCount: 0,
|
|
85
|
+
recoveredCount: 0,
|
|
86
|
+
lastReason: null,
|
|
87
|
+
lastThrottledAt: null,
|
|
88
|
+
lastRecoveredAt: null,
|
|
89
|
+
};
|
|
59
90
|
}
|
|
60
91
|
|
|
61
92
|
// ---------------------------------------------------------------------------
|
|
@@ -125,6 +156,80 @@ class StatusMonitor {
|
|
|
125
156
|
}
|
|
126
157
|
}
|
|
127
158
|
|
|
159
|
+
/**
|
|
160
|
+
* Record a rate-limit retry/backoff signal.
|
|
161
|
+
*
|
|
162
|
+
* @param {object} data
|
|
163
|
+
* @param {string} [data.specName]
|
|
164
|
+
* @param {number} [data.retryCount]
|
|
165
|
+
* @param {number} [data.retryDelayMs]
|
|
166
|
+
* @param {string} [data.error]
|
|
167
|
+
*/
|
|
168
|
+
recordRateLimitEvent(data = {}) {
|
|
169
|
+
const delay = Number(data.retryDelayMs);
|
|
170
|
+
const retryCount = Number(data.retryCount);
|
|
171
|
+
|
|
172
|
+
this._rateLimit.signalCount++;
|
|
173
|
+
this._rateLimit.retryCount++;
|
|
174
|
+
this._rateLimit.totalBackoffMs += Number.isFinite(delay) && delay > 0 ? Math.round(delay) : 0;
|
|
175
|
+
this._rateLimit.lastSignalAt = new Date().toISOString();
|
|
176
|
+
|
|
177
|
+
if (typeof data.specName === 'string' && data.specName.trim()) {
|
|
178
|
+
this._rateLimit.lastSpecName = data.specName.trim();
|
|
179
|
+
}
|
|
180
|
+
if (Number.isFinite(retryCount) && retryCount >= 0) {
|
|
181
|
+
this._rateLimit.lastRetryCount = Math.floor(retryCount);
|
|
182
|
+
}
|
|
183
|
+
this._rateLimit.lastDelayMs = Number.isFinite(delay) && delay > 0 ? Math.round(delay) : 0;
|
|
184
|
+
if (data.error !== undefined && data.error !== null) {
|
|
185
|
+
this._rateLimit.lastError = `${data.error}`;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Update adaptive parallel telemetry.
|
|
191
|
+
*
|
|
192
|
+
* @param {object} data
|
|
193
|
+
* @param {'throttled'|'recovered'} [data.event]
|
|
194
|
+
* @param {string} [data.reason]
|
|
195
|
+
* @param {number} [data.maxParallel]
|
|
196
|
+
* @param {number} [data.effectiveMaxParallel]
|
|
197
|
+
* @param {number} [data.floor]
|
|
198
|
+
* @param {boolean} [data.adaptive]
|
|
199
|
+
*/
|
|
200
|
+
updateParallelTelemetry(data = {}) {
|
|
201
|
+
if (typeof data.adaptive === 'boolean') {
|
|
202
|
+
this._parallel.adaptive = data.adaptive;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const maxParallel = Number(data.maxParallel);
|
|
206
|
+
if (Number.isFinite(maxParallel) && maxParallel > 0) {
|
|
207
|
+
this._parallel.maxParallel = Math.floor(maxParallel);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const effective = Number(data.effectiveMaxParallel);
|
|
211
|
+
if (Number.isFinite(effective) && effective > 0) {
|
|
212
|
+
this._parallel.effectiveMaxParallel = Math.floor(effective);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const floor = Number(data.floor);
|
|
216
|
+
if (Number.isFinite(floor) && floor > 0) {
|
|
217
|
+
this._parallel.floor = Math.floor(floor);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (typeof data.reason === 'string' && data.reason.trim()) {
|
|
221
|
+
this._parallel.lastReason = data.reason.trim();
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (data.event === 'throttled') {
|
|
225
|
+
this._parallel.throttledCount++;
|
|
226
|
+
this._parallel.lastThrottledAt = new Date().toISOString();
|
|
227
|
+
} else if (data.event === 'recovered') {
|
|
228
|
+
this._parallel.recoveredCount++;
|
|
229
|
+
this._parallel.lastRecoveredAt = new Date().toISOString();
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
128
233
|
/**
|
|
129
234
|
* Set the overall orchestration state.
|
|
130
235
|
*
|
|
@@ -219,6 +324,8 @@ class StatusMonitor {
|
|
|
219
324
|
currentBatch: this._currentBatch,
|
|
220
325
|
totalBatches: this._totalBatches,
|
|
221
326
|
specs,
|
|
327
|
+
rateLimit: { ...this._rateLimit },
|
|
328
|
+
parallel: { ...this._parallel },
|
|
222
329
|
};
|
|
223
330
|
}
|
|
224
331
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "kiro-spec-engine",
|
|
3
|
-
"version": "1.47.
|
|
3
|
+
"version": "1.47.14",
|
|
4
4
|
"description": "kiro-spec-engine (kse) - A CLI tool and npm package for spec-driven development with AI coding assistants. NOT the Kiro IDE desktop application.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"bin": {
|