bullswarm 0.25.2 → 0.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +48 -10
- package/connectors/_schema.json +2 -0
- package/connectors/claude-code.json +6 -0
- package/connectors/codex.json +5 -0
- package/connectors/command-code.json +2 -0
- package/connectors/echo-worker.mjs +6 -0
- package/connectors/echo.json +1 -0
- package/connectors/grok.json +2 -0
- package/connectors/opencode2.json +2 -0
- package/package.json +1 -1
- package/skill/SKILL.md +5 -1
- package/skill/references/operations.md +21 -6
- package/src/cli.js +15 -3
- package/src/help.js +14 -6
- package/src/lib/config.js +38 -0
- package/src/lib/quota.js +380 -0
- package/src/lib/route.js +74 -12
- package/src/lib/state.js +12 -5
- package/src/lib/watch.js +63 -8
- package/src/meters/framework.js +36 -6
- package/src/meters/registry.js +17 -2
- package/src/setup.js +12 -0
- package/src/workflow/cli.js +10 -1
- package/src/workflow/pool-refresh.js +73 -0
- package/src/workflow/runner.js +2 -0
- package/src/workflow/runtime.js +10 -3
- package/src/workflow/v2-dispatch.js +75 -8
- package/src/workflow/v2-runtime.js +23 -3
- package/src/workflow/watch-cli.js +87 -11
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,45 @@
|
|
|
1
1
|
# bullswarm changelog
|
|
2
2
|
|
|
3
|
+
## 0.25.3 — usage-limit recovery and headroom-aware routing
|
|
4
|
+
|
|
5
|
+
- A provider that reports a usage limit is now its own mechanical failure kind,
|
|
6
|
+
`quota` — never `process`, `semantic`, or `auth`. The attempt is killed at
|
|
7
|
+
once instead of waiting out a CLI that printed its limit and then hung, and
|
|
8
|
+
the pool is quarantined until the reset the message named, falling back to
|
|
9
|
+
that pool's cached 5-hour `resets_at` and then to 30 minutes rather than the
|
|
10
|
+
flat 10. The quarantine record carries `kind: 'quota'` and excludes the pool
|
|
11
|
+
from every later dispatch, in that run and in others, until it expires; the
|
|
12
|
+
action moves to another pool with quota and is never retried on the one that
|
|
13
|
+
hit the limit. `bullswarm run`, the V1 runtime, and V2 dispatch all apply the
|
|
14
|
+
same deadline. Detection is shape-gated: an agent report that discusses usage
|
|
15
|
+
limits, or tool output quoting them, is not a limit, and phrases that other
|
|
16
|
+
services also emit (`rate limited`, `too many requests`, `quota exceeded`)
|
|
17
|
+
count only as a bare notice, never as narration about someone else's quota
|
|
18
|
+
("rate limited by the GitHub API, retrying"). Connectors declare their
|
|
19
|
+
own phrases under `quotaSignatures`; installed connectors receive new ones on
|
|
20
|
+
upgrade.
|
|
21
|
+
|
|
22
|
+
- Routing avoids pools that are close to their 5-hour limit. A pool at or above
|
|
23
|
+
`FIVE_HOUR_NEAR_LIMIT_PCT` (75) is chosen only when no eligible pool below it
|
|
24
|
+
exists for the lane, ahead of pace, an approved assignment, and incumbency;
|
|
25
|
+
pools at or above 90 stay excluded outright, and a pool with no 5-hour
|
|
26
|
+
reading counts as having headroom. Routing reasons and candidate lists name
|
|
27
|
+
the utilization that decided the pick, and `bullswarm pools` shows it as
|
|
28
|
+
`5h=<n>%` with a `NEAR-5H-LIMIT` label. Meters and quarantines are re-read
|
|
29
|
+
from the meter cache and core state before every action dispatch and before
|
|
30
|
+
every retry inside one — forced live right after a usage limit — so a long
|
|
31
|
+
run no longer dispatches from the pool snapshot frozen at launch.
|
|
32
|
+
|
|
33
|
+
- `workflow watch` reports a usage-limit retry as a notable event in both
|
|
34
|
+
modes' vocabulary: `⚠ <action> usage limit on <pool> · paused until
|
|
35
|
+
<deadline> · retrying on another pool`, then `↺ <action> now on <pool> ·
|
|
36
|
+
<model>` once the retry lands. Both print without `--verbose`, wake `--next`,
|
|
37
|
+
and appear in `--jsonl` as `attempt.quota` and `attempt.moved`. The new
|
|
38
|
+
`--classic` flag forces the older heartbeat-based watcher (transition-on-change
|
|
39
|
+
snapshots plus a periodic heartbeat, 60 seconds unless `--heartbeat <seconds>`
|
|
40
|
+
is given) for a V2 run; it is a no-op for legacy runs and cannot combine with
|
|
41
|
+
`--next`.
|
|
42
|
+
|
|
3
43
|
## 0.25.2 — event-based watch
|
|
4
44
|
|
|
5
45
|
- `workflow watch <run> --next` is safe to relaunch after every wake-up: each
|
package/README.md
CHANGED
|
@@ -37,12 +37,17 @@ detaches safely.
|
|
|
37
37
|
passing verification.
|
|
38
38
|
2. **Pace by meter.** The scheduling resource is the subscription window:
|
|
39
39
|
elapsed% minus used%, most-behind pool wins. Pace may only promote a
|
|
40
|
-
*cheaper* pool. Lanes are work-nature, never hard-coded to pools.
|
|
40
|
+
*cheaper* pool. Lanes are work-nature, never hard-coded to pools. The
|
|
41
|
+
5-hour window never paces — it gates: a pool at or above 75% of it is
|
|
42
|
+
chosen only when no eligible pool below that line exists, and one at or
|
|
43
|
+
above 90% is not dispatched at all.
|
|
41
44
|
3. **Delegate output is evidence, never authority.** The Workflow Planner may
|
|
42
45
|
propose actions, but only the deterministic kernel validates the program,
|
|
43
46
|
accepts requirement-scoped evidence, and computes completion.
|
|
44
47
|
4. **Quarantine re-probes.** A benched pool must be able to return to service
|
|
45
|
-
automatically; a lane is never allowed to silently go down.
|
|
48
|
+
automatically; a lane is never allowed to silently go down. A pool benched
|
|
49
|
+
for a usage limit waits for the reset the provider named, not a flat
|
|
50
|
+
guess — and never longer.
|
|
46
51
|
|
|
47
52
|
## Install
|
|
48
53
|
|
|
@@ -94,7 +99,7 @@ bullswarm health # re-judge saved outputs; catch gate failures
|
|
|
94
99
|
| `delegate` | Explain and execute the smallest reliable shape: one content-verified agent, or the planning contract for an autonomous workflow you author (`--orchestrator` dispatches a planner agent instead). |
|
|
95
100
|
| `run` | route → dispatch → watch → verify → one JSON verdict |
|
|
96
101
|
| `health` | Re-judge saved outputs against their verdicts; surface verify-gate failures and quarantine clusters |
|
|
97
|
-
| `pools` | Show each pool's meter state, pace position, quarantine status |
|
|
102
|
+
| `pools` | Show each pool's meter state, pace position, 5-hour utilization (`5h=<n>%`, flagged `NEAR-5H-LIMIT` at or above 75%), quarantine status |
|
|
98
103
|
| `strategy` | Interactive provider/model control center with live high/medium/low route previews and an agent-facing JSON API |
|
|
99
104
|
| `doctor` | Machine-readable readiness report; self-heals on first call |
|
|
100
105
|
| `workflow` | Start an autonomous goal, or run / validate / draft / inspect explicit workflows and their live instances. |
|
|
@@ -209,8 +214,15 @@ best eligible models on its configured interval. Disable it with
|
|
|
209
214
|
`strategy auto off --yes`. Discovery commands, model argument syntax, pricing,
|
|
210
215
|
and benchmark declarations remain connector-owned. Unknown license value,
|
|
211
216
|
prices, and benchmarks stay `null` rather than being guessed. An assignment is
|
|
212
|
-
only a preference: quarantine, exhaustion, burst gates,
|
|
213
|
-
still win.
|
|
217
|
+
only a preference: quarantine, exhaustion, burst gates, 5-hour headroom, and
|
|
218
|
+
capability checks still win. Routing prefers pools below
|
|
219
|
+
`FIVE_HOUR_NEAR_LIMIT_PCT` (75) of their 5-hour window over pools at or above
|
|
220
|
+
it, ahead of pace, an approved assignment, and incumbency; a near-limit pool is
|
|
221
|
+
still picked when it is the only eligible one, and a pool with no 5-hour
|
|
222
|
+
reading counts as having headroom. The routing reason and every candidate row
|
|
223
|
+
name the utilization that decided the pick, and meters and quarantines are
|
|
224
|
+
re-read before each dispatch — and again, live, right after a usage limit —
|
|
225
|
+
so a long run never routes off the snapshot it launched with.
|
|
214
226
|
|
|
215
227
|
Model exclusions are hard routing policy. An excluded model is removed from
|
|
216
228
|
recommendations and assignments, and Bullswarm pins a same-tier allowed model
|
|
@@ -490,9 +502,18 @@ prints one attach line, then one line per notable event as it happens
|
|
|
490
502
|
planner turn, stall/recovery, cancellation, and the existing pause and
|
|
491
503
|
terminal `outcome:` / `next:` lines) and stays silent while work is merely
|
|
492
504
|
in progress. Agent starts, mechanical retries, and steering delivery print
|
|
493
|
-
only with `--verbose`.
|
|
494
|
-
|
|
495
|
-
|
|
505
|
+
only with `--verbose`. A usage-limit failure (`failureKind: 'quota'`) always
|
|
506
|
+
prints, verbose or not: `⚠ <actionId> usage limit on <pool> · paused until
|
|
507
|
+
<deadline> · retrying on another pool`, followed once the mechanical retry
|
|
508
|
+
lands on another pool by `↺ <actionId> now on <pool> · <model>`. The
|
|
509
|
+
periodic heartbeat is off unless you pass `--heartbeat <seconds>`;
|
|
510
|
+
`--stall-after <seconds>` (default 300) reports a running agent that has
|
|
511
|
+
gone silent. Pass `--classic` to force the older heartbeat-based watcher
|
|
512
|
+
instead (the transition-on-change snapshot stream plus a periodic
|
|
513
|
+
heartbeat, every 60 seconds unless `--heartbeat <seconds>` is given) —
|
|
514
|
+
legacy (non-V2) runs already behave this way and `--classic` is a no-op for
|
|
515
|
+
them; `--classic` cannot combine with `--next`, which exists only for event
|
|
516
|
+
mode. `--next` prints no attach line and
|
|
496
517
|
exits after the first notable event so a background terminal can wake the
|
|
497
518
|
caller; relaunch until the outcome line reports a pause or a terminal
|
|
498
519
|
status (exit 0 while the run continues or delivered, 1 when it ended
|
|
@@ -507,12 +528,14 @@ reported does not produce a duplicate stall line (its recovery still
|
|
|
507
528
|
prints). `--jsonl` emits one JSON object per notable event with a stable
|
|
508
529
|
`type` (`attach`, `action.finished`,
|
|
509
530
|
`evidence.recorded`, `stage.completed`, `planner.finished`, `agent.stalled`,
|
|
510
|
-
`agent.recovered`, `cancellation.requested`, `
|
|
531
|
+
`agent.recovered`, `cancellation.requested`, `attempt.quota`,
|
|
532
|
+
`attempt.moved`, `paused`, `finished`,
|
|
511
533
|
`interrupted`, and with `--verbose` `action.started`, `attempt.retrying`,
|
|
512
534
|
`steering.delivered`); in that mode the relaunch line is not printed and
|
|
513
535
|
every object instead carries the `sequence` it was emitted at, which is the
|
|
514
536
|
value to pass as `--after`. `--once` still prints one current snapshot. Legacy
|
|
515
|
-
(non-V2) runs keep the compact transition-plus-heartbeat stream unchanged
|
|
537
|
+
(non-V2) runs keep the compact transition-plus-heartbeat stream unchanged,
|
|
538
|
+
the same stream `--classic` opts a V2 run into.
|
|
516
539
|
|
|
517
540
|
```bash
|
|
518
541
|
bullswarm workflow watch <shortId>
|
|
@@ -523,6 +546,7 @@ bullswarm workflow watch <shortId> --jsonl # one JSON object per event
|
|
|
523
546
|
bullswarm workflow watch <shortId> --once # one current/terminal snapshot
|
|
524
547
|
bullswarm workflow watch <shortId> --verbose # started / retry / steering too
|
|
525
548
|
bullswarm workflow watch <shortId> --stall-after 120 --heartbeat 30
|
|
549
|
+
bullswarm workflow watch <shortId> --classic # older heartbeat-based watcher instead of event mode
|
|
526
550
|
```
|
|
527
551
|
|
|
528
552
|
`workflow tui` is the interactive, Claude-style `/workflows` view. For an
|
|
@@ -611,6 +635,20 @@ channel, so reading source text such as an auth-signature matcher cannot falsely
|
|
|
611
635
|
quarantine Grok or Command Code. Error-shaped semantic results and stderr
|
|
612
636
|
diagnostics still trigger the auth/quota guard.
|
|
613
637
|
|
|
638
|
+
A provider that reports a usage limit — `You've hit your session limit ·
|
|
639
|
+
resets 8:20pm (Asia/Hong_Kong)`, `usage_credits_required`, `rate limit
|
|
640
|
+
exceeded`, `quota exceeded` — is its own mechanical failure kind, `quota`,
|
|
641
|
+
never `process`, `semantic`, or `auth`. The attempt is killed immediately
|
|
642
|
+
even if the CLI would otherwise hang, and the pool is quarantined until the
|
|
643
|
+
reset time parsed from the message, falling back to that pool's cached 5-hour
|
|
644
|
+
`resets_at` and then to 30 minutes. The quarantine record carries
|
|
645
|
+
`kind: 'quota'` and excludes the pool from every later dispatch, in this run
|
|
646
|
+
and in others, until it expires; the action is immediately re-dispatched on
|
|
647
|
+
another pool with quota and never retried on the one that hit the limit. An
|
|
648
|
+
agent report that merely discusses usage limits, or tool output that quotes
|
|
649
|
+
them, is not a limit: detection is shape-gated to lines that look like a
|
|
650
|
+
provider notice.
|
|
651
|
+
|
|
614
652
|
After ten minutes without transport, parsed-event, or semantic-action evidence,
|
|
615
653
|
an active child is labeled `suspected_stalled`. This is an inspection signal,
|
|
616
654
|
not a death verdict and never an automatic kill: buffered CLIs can be silent
|
package/connectors/_schema.json
CHANGED
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
"$comment-cwd": "cwdMode documents how the CLI resolves its project: 'pwd' means the CLI resolves from $PWD so the watcher MUST set PWD and spawn inside --add-dir"
|
|
11
11
|
},
|
|
12
12
|
"authSignatures": ["strings in output that mean auth/throttle failure"],
|
|
13
|
+
"quotaSignatures": ["strings in output that mean this credential is out of quota RIGHT NOW (usage/rate limit hit)"],
|
|
14
|
+
"$comment-quotaSignatures": "Optional. Merged on top of DEFAULT_QUOTA_SIGNATURES (src/lib/quota.js), matched case-insensitively, and only honored when the matched line is quota-shaped (<=300 chars and the phrase starts in the first 40 chars or the line is error-shaped) so agent prose and tool output that merely discuss limits are never killed. A hit is classified as the mechanical failure kind `quota` (checked before authSignatures), kills the attempt, and quarantines the pool until the reset time parsed from the message, else the pool's cached five_hour.resets_at, else 30 minutes.",
|
|
13
15
|
"outputExtraction": {
|
|
14
16
|
"$comment": "how to get the real answer out of stdout+stderr+files",
|
|
15
17
|
"strategy": "stdout|stdout-tail|json-field|file|event-stream",
|
|
@@ -23,6 +23,12 @@
|
|
|
23
23
|
"not logged in",
|
|
24
24
|
"please run /login"
|
|
25
25
|
],
|
|
26
|
+
"$comment-quotaSignatures": "usage-limit phrases: classified `quota`, quarantined until the reset the message names. DEFAULT_QUOTA_SIGNATURES (src/lib/quota.js) apply on top of these.",
|
|
27
|
+
"quotaSignatures": [
|
|
28
|
+
"hit your session limit",
|
|
29
|
+
"hit your limit",
|
|
30
|
+
"hit your usage limit"
|
|
31
|
+
],
|
|
26
32
|
"outputExtraction": {
|
|
27
33
|
"strategy": "event-stream"
|
|
28
34
|
},
|
package/connectors/codex.json
CHANGED
|
@@ -21,6 +21,11 @@
|
|
|
21
21
|
"unauthorized",
|
|
22
22
|
"invalid api key"
|
|
23
23
|
],
|
|
24
|
+
"$comment-quotaSignatures": "usage_credits_required is a spent window, not a broken credential: it classifies `quota` (checked before authSignatures) so the pool is quarantined until its reset instead of re-probed in 10 minutes.",
|
|
25
|
+
"quotaSignatures": [
|
|
26
|
+
"usage_credits_required",
|
|
27
|
+
"usage limit"
|
|
28
|
+
],
|
|
24
29
|
"outputExtraction": {
|
|
25
30
|
"strategy": "event-stream"
|
|
26
31
|
},
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
"not authenticated",
|
|
22
22
|
"cmd login"
|
|
23
23
|
],
|
|
24
|
+
"$comment-quotaSignatures": "empty on purpose: no cmd-specific usage-limit wording has been observed yet; DEFAULT_QUOTA_SIGNATURES in src/lib/quota.js covers the generic provider phrasings.",
|
|
25
|
+
"quotaSignatures": [],
|
|
24
26
|
"outputExtraction": {
|
|
25
27
|
"strategy": "event-stream"
|
|
26
28
|
},
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// echo-worker.mjs — deterministic test delegate for bullswarm.
|
|
2
2
|
// Reads a task file; behavior is driven by directives in the task text.
|
|
3
3
|
// FAIL:auth -> prints an auth failure, exits 0 (the lying-exit trap)
|
|
4
|
+
// FAIL:quota -> prints a provider usage limit naming its reset, exits 0
|
|
4
5
|
// FAIL:exit -> prints a complete answer, exits 1 (exit-1-after-success)
|
|
5
6
|
// INTENT: -> prints only an announcement, exits 0
|
|
6
7
|
// otherwise -> echoes the task as a completed answer, exit 0
|
|
@@ -11,6 +12,11 @@ const task = readFileSync(process.argv[2], 'utf8');
|
|
|
11
12
|
const sleepMatch = task.match(/SLEEP_MS:(\d+)/);
|
|
12
13
|
if (sleepMatch) await new Promise((resolve) => setTimeout(resolve, Number(sleepMatch[1])));
|
|
13
14
|
|
|
15
|
+
if (task.includes('FAIL:quota')) {
|
|
16
|
+
// A usage limit is not a broken credential: it names when it resets.
|
|
17
|
+
console.log('Error: usage limit reached · resets in 45 minutes');
|
|
18
|
+
process.exit(0);
|
|
19
|
+
}
|
|
14
20
|
if (task.includes('FAIL:auth-hang')) {
|
|
15
21
|
console.log('Authentication failed: quota exhausted; waiting process should be terminated.');
|
|
16
22
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
package/connectors/echo.json
CHANGED
package/connectors/grok.json
CHANGED
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
"rate limit",
|
|
18
18
|
"invalid api key"
|
|
19
19
|
],
|
|
20
|
+
"$comment-quotaSignatures": "empty on purpose: no grok-specific usage-limit wording has been observed yet, and the generic phrasings (rate limit exceeded, too many requests, quota exceeded) already come from DEFAULT_QUOTA_SIGNATURES in src/lib/quota.js.",
|
|
21
|
+
"quotaSignatures": [],
|
|
20
22
|
"outputExtraction": {
|
|
21
23
|
"strategy": "event-stream"
|
|
22
24
|
},
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
"$comment-cwdMode": "QUIRK: resolves its project from $PWD, not the spawn cwd. The watcher MUST set env.PWD and spawn with cwd inside the target repo, or it will silently analyse the wrong repository and answer confidently about it."
|
|
9
9
|
},
|
|
10
10
|
"authSignatures": ["No cookie auth credentials found", "unauthorized"],
|
|
11
|
+
"$comment-quotaSignatures": "empty on purpose: provider quota errors reach opencode from the upstream API in generic form (insufficient_quota, rate limit exceeded, too many requests), which DEFAULT_QUOTA_SIGNATURES in src/lib/quota.js already matches.",
|
|
12
|
+
"quotaSignatures": [],
|
|
11
13
|
"outputExtraction": { "strategy": "event-stream" },
|
|
12
14
|
"eventStream": {
|
|
13
15
|
"format": "jsonl",
|
package/package.json
CHANGED
package/skill/SKILL.md
CHANGED
|
@@ -124,7 +124,11 @@ acting are printed instead of skipped and a stall you already saw does not
|
|
|
124
124
|
report twice. In `--jsonl` mode there is no such line — take `--after` from the
|
|
125
125
|
`sequence` field of the last object. V2 watch prints one line per notable event
|
|
126
126
|
and stays silent while work is merely in progress; `--heartbeat` is opt-in and
|
|
127
|
-
`--stall-after` (default 300s) reports a silent running agent.
|
|
127
|
+
`--stall-after` (default 300s) reports a silent running agent. A usage-limit
|
|
128
|
+
failure always prints — `⚠ ... usage limit on <pool> · paused until <deadline>
|
|
129
|
+
· retrying on another pool`, then `↺ ... now on <pool> · <model>` once the
|
|
130
|
+
mechanical retry lands — even without `--verbose`. Pass `--classic` for the
|
|
131
|
+
older heartbeat-based watcher instead (it cannot combine with `--next`).
|
|
128
132
|
`watch` also exits at a durable planning pause; that is not completion.
|
|
129
133
|
|
|
130
134
|
Read action outputs and actual artifacts, and probe important edge cases yourself.
|
|
@@ -51,7 +51,13 @@ already-reported stall does not fire again (its recovery still prints). With
|
|
|
51
51
|
`--jsonl` that line is absent: take `--after` from the `sequence` field carried
|
|
52
52
|
by every emitted object. `--heartbeat` is opt-in for V2 (legacy still
|
|
53
53
|
defaults to 60s). `--stall-after` (default 300s) reports a silent running
|
|
54
|
-
agent.
|
|
54
|
+
agent. A usage-limit failure always prints, verbose or not: `⚠ ... usage
|
|
55
|
+
limit on <pool> · paused until <deadline> · retrying on another pool`, then
|
|
56
|
+
`↺ ... now on <pool> · <model>` once the mechanical retry lands on another
|
|
57
|
+
pool. Use `--verbose` only for diagnosis. `--classic` forces the older
|
|
58
|
+
heartbeat-based watcher (transition-on-change snapshots plus a periodic
|
|
59
|
+
heartbeat) instead of event mode; it is a no-op for legacy runs and cannot
|
|
60
|
+
combine with `--next`.
|
|
55
61
|
The result command is the stable delivery/verification envelope; do not scrape
|
|
56
62
|
task files or assume the last provider response is the deliverable.
|
|
57
63
|
|
|
@@ -149,9 +155,13 @@ bullswarm strategy inventory --json
|
|
|
149
155
|
bullswarm strategy routes --json
|
|
150
156
|
```
|
|
151
157
|
|
|
152
|
-
Automatic routing chooses the most-behind capable eligible pool
|
|
153
|
-
gates and quarantine, and applies only
|
|
154
|
-
and exclusions.
|
|
158
|
+
Automatic routing chooses the most-behind capable eligible pool among those
|
|
159
|
+
with 5-hour headroom, honors burst gates and quarantine, and applies only
|
|
160
|
+
explicitly approved model assignments and exclusions. A pool at or above 75%
|
|
161
|
+
of its 5-hour window is picked only when no eligible pool below that line
|
|
162
|
+
exists; `bullswarm pools` shows the reading as `5h=<n>%` with a
|
|
163
|
+
`NEAR-5H-LIMIT` label, and meters and quarantines are re-read before every
|
|
164
|
+
dispatch rather than frozen at launch. Humans can use bare `bullswarm strategy` to toggle providers
|
|
155
165
|
and multi-select high/medium/low per model. Agents should consume the inventory
|
|
156
166
|
and apply validated changes with `strategy set-provider`, `strategy set-model`,
|
|
157
167
|
or one atomic `strategy configure --file <json> --yes`. Never weaken those
|
|
@@ -159,8 +169,13 @@ controls in a prompt.
|
|
|
159
169
|
|
|
160
170
|
## Recovery and stopping rules
|
|
161
171
|
|
|
162
|
-
- Auth
|
|
163
|
-
another eligible pool.
|
|
172
|
+
- Auth signatures quarantine the affected pool for a 10-minute re-probe
|
|
173
|
+
window; later dispatches use another eligible pool.
|
|
174
|
+
- A provider usage limit is the distinct failure kind `quota`: the attempt is
|
|
175
|
+
killed at once, the pool is quarantined until the reset the message named
|
|
176
|
+
(else its cached 5-hour `resets_at`, else 30 minutes), and the action moves
|
|
177
|
+
to a pool that still has quota. The quarantine holds across runs until it
|
|
178
|
+
expires. Discussing usage limits in a report is not a usage limit.
|
|
164
179
|
- A quota-gated preferred orchestrator falls back unless it was strictly pinned
|
|
165
180
|
for QA.
|
|
166
181
|
- Silence is evidence to inspect, not automatic proof of a hang. Check the TUI
|
package/src/cli.js
CHANGED
|
@@ -78,13 +78,17 @@ async function cmdPools(opts) {
|
|
|
78
78
|
? 'unmetered'
|
|
79
79
|
: `used ${p.usedPct ?? '?'}% elapsed ${p.elapsedPct ?? '?'}% [${src}]`;
|
|
80
80
|
const burst = p.burstGate ? ' BURST-GATED' : '';
|
|
81
|
+
// 5h is a gate, never a pace (doctrine M3): show the reading and whether
|
|
82
|
+
// routing now deprioritizes this pool for it.
|
|
83
|
+
const fiveHour = p.fiveHourUsedPct == null ? '' : ` 5h=${Math.round(p.fiveHourUsedPct * 10) / 10}%`;
|
|
84
|
+
const nearLimit = p.nearFiveHourLimit === true ? ' NEAR-5H-LIMIT' : '';
|
|
81
85
|
const status = !p.enabled
|
|
82
86
|
? 'disabled'
|
|
83
87
|
: p.quarantine
|
|
84
88
|
? `QUARANTINED until ${new Date(p.quarantine.until).toLocaleTimeString()} (${p.quarantine.reason})`
|
|
85
|
-
: `ready${burst}`;
|
|
89
|
+
: `ready${burst}${nearLimit}`;
|
|
86
90
|
console.log(
|
|
87
|
-
`${p.name.padEnd(14)} cost=${p.costRank} lanes=${p.lanes.join('/')} ${meter} surplus=${p.pace ?? '-'} ${status}`,
|
|
91
|
+
`${p.name.padEnd(14)} cost=${p.costRank} lanes=${p.lanes.join('/')} ${meter} surplus=${p.pace ?? '-'}${fiveHour} ${status}`,
|
|
88
92
|
);
|
|
89
93
|
}
|
|
90
94
|
return 0;
|
|
@@ -228,6 +232,9 @@ async function cmdRun(opts) {
|
|
|
228
232
|
timeoutSec: opts.timeout == null ? null : Number(opts.timeout),
|
|
229
233
|
env: childDepthEnv(process.env),
|
|
230
234
|
model: selectedModel,
|
|
235
|
+
// Lets a usage-limit verdict fall back to this pool's cached 5h meter
|
|
236
|
+
// reset when the provider's message named no reset time of its own.
|
|
237
|
+
bullswarmDir: getBullswarmDir(),
|
|
231
238
|
onActivity: (event) => heartbeat.activity(event),
|
|
232
239
|
onAgentEvent: () => heartbeat.event(),
|
|
233
240
|
});
|
|
@@ -240,7 +247,12 @@ async function cmdRun(opts) {
|
|
|
240
247
|
state.incumbents ??= {};
|
|
241
248
|
state.incumbents[lane] = connector.name;
|
|
242
249
|
} else if (verdict.quarantineHint) {
|
|
243
|
-
|
|
250
|
+
// A usage limit carries its own deadline (the reset the provider named);
|
|
251
|
+
// an auth failure keeps the flat re-probe window.
|
|
252
|
+
quarantinePool(state, connector.name, verdict.why, now, {
|
|
253
|
+
until: verdict.quarantineUntil ?? null,
|
|
254
|
+
kind: verdict.failureKind === 'quota' ? 'quota' : 'auth',
|
|
255
|
+
});
|
|
244
256
|
verdict.quarantinedUntil = state.pools[connector.name]?.quarantine?.until;
|
|
245
257
|
}
|
|
246
258
|
|
package/src/help.js
CHANGED
|
@@ -987,20 +987,26 @@ const workflowTuiText = rich({
|
|
|
987
987
|
});
|
|
988
988
|
|
|
989
989
|
const workflowWatchText = rich({
|
|
990
|
-
usage: 'bullswarm workflow watch <runId> [--interval <seconds>] [--heartbeat <seconds>] [--stall-after <seconds>] [--next [--after <sequence>] [--since <iso-timestamp>]] [--jsonl] [--once] [--verbose]',
|
|
990
|
+
usage: 'bullswarm workflow watch <runId> [--classic] [--interval <seconds>] [--heartbeat <seconds>] [--stall-after <seconds>] [--next [--after <sequence>] [--since <iso-timestamp>]] [--jsonl] [--once] [--verbose]',
|
|
991
991
|
purpose: "Follow one V2 run by printing one attach line, then one line per notable event "
|
|
992
992
|
+ '(action finished/failed/blocked/cancelled, evidence, stage completion, stall/recovery, planning, '
|
|
993
|
-
+ 'cancellation) and staying silent while work is merely in progress.
|
|
994
|
-
+ '
|
|
993
|
+
+ 'cancellation) and staying silent while work is merely in progress. A usage-limit failure always '
|
|
994
|
+
+ 'prints, verbose or not: an `⚠ ... usage limit on <pool> · paused until <deadline> · retrying on '
|
|
995
|
+
+ 'another pool` line, then an `↺ ... now on <pool> · <model>` line once the mechanical retry lands. '
|
|
996
|
+
+ '`--classic` forces the older heartbeat-based watcher instead (transition-on-change snapshots plus '
|
|
997
|
+
+ 'a periodic heartbeat); legacy runs already behave this way and are unaffected. `--next` prints no '
|
|
998
|
+
+ 'attach line and returns after the first notable event, or immediately at a pause or terminal status '
|
|
999
|
+
+ '(event mode only — it cannot combine with `--classic`). '
|
|
995
1000
|
+ 'Every `--next` exit that leaves the run going prints a `next:` relaunch line carrying `--after` and '
|
|
996
1001
|
+ '`--since`; pass those two values back on the relaunch so events committed while no watcher was '
|
|
997
1002
|
+ 'attached are printed instead of skipped and an already-reported stall does not fire again. '
|
|
998
|
-
+ '`--heartbeat` is opt-in for V2; legacy runs keep the historical heartbeat
|
|
999
|
-
+ 'full-screen tui and the machine-oriented events replay.',
|
|
1003
|
+
+ '`--heartbeat` is opt-in for V2; legacy runs (and `--classic`) keep the historical 60s heartbeat by '
|
|
1004
|
+
+ 'default. Distinct from the full-screen tui and the machine-oriented events replay.',
|
|
1000
1005
|
args: [{ name: '<runId>', desc: 'shortId or runId' }],
|
|
1001
1006
|
options: [
|
|
1007
|
+
{ flag: '--classic', desc: 'force the older heartbeat-based watcher (transition-on-change snapshots plus a periodic heartbeat) instead of event mode; no-op for legacy runs; cannot combine with --next', default: 'off (event mode for V2 runs)' },
|
|
1002
1008
|
{ flag: '--interval <seconds>', desc: 'poll interval while following', default: '2' },
|
|
1003
|
-
{ flag: '--heartbeat <seconds>', desc: 'print a periodic heartbeat line when nothing has changed; opt-in for V2, must be >= 1', default: 'off for V2, 60 for legacy' },
|
|
1009
|
+
{ flag: '--heartbeat <seconds>', desc: 'print a periodic heartbeat line when nothing has changed; opt-in for V2, must be >= 1', default: 'off for V2, 60 for legacy and --classic' },
|
|
1004
1010
|
{ flag: '--stall-after <seconds>', desc: 'report a running agent as silent after this many seconds without activity; must be >= 1', default: '300' },
|
|
1005
1011
|
{ flag: '--next', desc: 'print no attach line; exit after the first poll that printed a notable event, or immediately at a pause or terminal status', default: 'off (follows until terminal or pause)' },
|
|
1006
1012
|
{ flag: '--after <sequence>', desc: 'start from this durable event sequence instead of the current high-water mark, so events committed since the previous watcher exited are printed; use the value from the previous `next:` line (in --jsonl, the `sequence` field of the last object)', default: 'attach at the current high-water mark' },
|
|
@@ -1014,11 +1020,13 @@ const workflowWatchText = rich({
|
|
|
1014
1020
|
'exits 0 if the run reaches a delivered status (or on --once), 1 if it reaches a non-delivered terminal status',
|
|
1015
1021
|
'--next exits 0 while the run continues or when it delivered, 1 when it ended without delivering or the kernel is not running',
|
|
1016
1022
|
'for a V2 run, a --next exit that leaves the run going ends with `next: bullswarm workflow watch <shortId> --next --after <sequence> --since <iso>`; pause, terminal and interrupted exits keep their own outcome/next lines',
|
|
1023
|
+
'--classic --next is rejected with exit 2: --next only applies to event mode',
|
|
1017
1024
|
],
|
|
1018
1025
|
examples: [
|
|
1019
1026
|
{ cmd: 'bullswarm workflow watch ab12cd --next', note: 'print the next notable event and exit; relaunch until outcome reports a pause or a terminal status' },
|
|
1020
1027
|
{ cmd: 'bullswarm workflow watch ab12cd --next --after 42 --since 2026-09-08T10:15:00.000Z', note: 'the relaunch: copy both values from the `next:` line the previous exit printed' },
|
|
1021
1028
|
{ cmd: 'bullswarm workflow watch ab12cd --stall-after 120 --heartbeat 30' },
|
|
1029
|
+
{ cmd: 'bullswarm workflow watch ab12cd --classic', note: 'the older heartbeat-based watcher instead of event mode' },
|
|
1022
1030
|
],
|
|
1023
1031
|
next: 'bullswarm workflow runs result <runId> --json once it finishes, or bullswarm workflow tui <runId> for the interactive view.',
|
|
1024
1032
|
});
|
package/src/lib/config.js
CHANGED
|
@@ -13,6 +13,7 @@ import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { loadState } from './state.js';
|
|
15
15
|
import { paceScore, isQuarantined } from './route.js';
|
|
16
|
+
import { FIVE_HOUR_NEAR_LIMIT_PCT } from '../meters/framework.js';
|
|
16
17
|
import { expandClaudeAccountConnectors } from './claude-accounts.js';
|
|
17
18
|
import { expandOpenCodeKaihkConnectors } from './opencode-kaihk.js';
|
|
18
19
|
|
|
@@ -66,6 +67,10 @@ export function buildPools(bullswarmDir, now = Date.now(), readings = {}) {
|
|
|
66
67
|
elapsedPct: null,
|
|
67
68
|
pace: null,
|
|
68
69
|
burstGate: false,
|
|
70
|
+
// 5h window (doctrine M3): gates routing, never paces it.
|
|
71
|
+
fiveHourUsedPct: null,
|
|
72
|
+
fiveHourResetsAt: null,
|
|
73
|
+
nearFiveHourLimit: false,
|
|
69
74
|
meterSnapshot: null,
|
|
70
75
|
subscription: {
|
|
71
76
|
...(conn.subscription ?? {}),
|
|
@@ -85,6 +90,15 @@ export function buildPools(bullswarmDir, now = Date.now(), readings = {}) {
|
|
|
85
90
|
const ps = state.pools[p.name] ?? {};
|
|
86
91
|
|
|
87
92
|
const reading = readings[p.name];
|
|
93
|
+
// The 5h gate is independent of the pacing window: a reading may carry a
|
|
94
|
+
// 5h utilization with no weekly/monthly window to pace by, and routing
|
|
95
|
+
// still has to see that the pool is close to its 5h limit.
|
|
96
|
+
if (reading) {
|
|
97
|
+
const fiveHour = fiveHourFromReading(reading);
|
|
98
|
+
p.fiveHourUsedPct = fiveHour.usedPct;
|
|
99
|
+
p.fiveHourResetsAt = fiveHour.resetsAt;
|
|
100
|
+
p.nearFiveHourLimit = fiveHour.nearLimit;
|
|
101
|
+
}
|
|
88
102
|
if (reading?.pacing) {
|
|
89
103
|
// Provider-truth path (M1/M2)
|
|
90
104
|
p.meterSource = reading.source; // live | cache | stale
|
|
@@ -112,6 +126,30 @@ export function buildPools(bullswarmDir, now = Date.now(), readings = {}) {
|
|
|
112
126
|
return { state, connectors, pools };
|
|
113
127
|
}
|
|
114
128
|
|
|
129
|
+
/** null-safe finite coercion: null/undefined/NaN all mean "no reading". */
|
|
130
|
+
function finiteOrNull(value) {
|
|
131
|
+
if (value == null) return null;
|
|
132
|
+
const n = Number(value);
|
|
133
|
+
return Number.isFinite(n) ? n : null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* 5h window fields from a meter reading. Prefers the flat fields paceSnapshot
|
|
138
|
+
* produces and falls back to the raw snapshot, so a reading assembled by an
|
|
139
|
+
* older code path still reports a real 5h number instead of null.
|
|
140
|
+
*/
|
|
141
|
+
function fiveHourFromReading(reading) {
|
|
142
|
+
const usedPct = finiteOrNull(reading?.fiveHourUsedPct)
|
|
143
|
+
?? finiteOrNull(reading?.snapshot?.five_hour?.utilization);
|
|
144
|
+
const raw = reading?.fiveHourResetsAt ?? reading?.snapshot?.five_hour?.resets_at ?? null;
|
|
145
|
+
const resetsMs = typeof raw === 'string' ? Date.parse(raw) : NaN;
|
|
146
|
+
return {
|
|
147
|
+
usedPct,
|
|
148
|
+
resetsAt: Number.isFinite(resetsMs) ? new Date(resetsMs).toISOString() : null,
|
|
149
|
+
nearLimit: usedPct != null && usedPct >= FIVE_HOUR_NEAR_LIMIT_PCT,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
115
153
|
/**
|
|
116
154
|
* Async variant that fetches live readings for pools with readers.
|
|
117
155
|
*/
|