bullswarm 0.25.1 → 0.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/README.md +81 -16
- package/connectors/_schema.json +2 -0
- package/connectors/claude-code.json +6 -0
- package/connectors/codex.json +5 -0
- package/connectors/command-code.json +2 -0
- package/connectors/echo-worker.mjs +6 -0
- package/connectors/echo.json +1 -0
- package/connectors/grok.json +2 -0
- package/connectors/opencode2.json +2 -0
- package/package.json +1 -1
- package/skill/SKILL.md +16 -1
- package/skill/references/operations.md +35 -6
- package/src/cli.js +15 -3
- package/src/help.js +32 -10
- package/src/lib/config.js +38 -0
- package/src/lib/quota.js +380 -0
- package/src/lib/route.js +74 -12
- package/src/lib/state.js +12 -5
- package/src/lib/watch.js +63 -8
- package/src/meters/framework.js +36 -6
- package/src/meters/registry.js +17 -2
- package/src/setup.js +12 -0
- package/src/workflow/cli.js +47 -5
- package/src/workflow/pool-refresh.js +73 -0
- package/src/workflow/runner.js +2 -0
- package/src/workflow/runtime.js +10 -3
- package/src/workflow/v2-dispatch.js +75 -8
- package/src/workflow/v2-runtime.js +23 -3
- package/src/workflow/watch-cli.js +538 -37
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,67 @@
|
|
|
1
1
|
# bullswarm changelog
|
|
2
2
|
|
|
3
|
+
## 0.25.3 — usage-limit recovery and headroom-aware routing
|
|
4
|
+
|
|
5
|
+
- A provider that reports a usage limit is now its own mechanical failure kind,
|
|
6
|
+
`quota` — never `process`, `semantic`, or `auth`. The attempt is killed at
|
|
7
|
+
once instead of waiting out a CLI that printed its limit and then hung, and
|
|
8
|
+
the pool is quarantined until the reset the message named, falling back to
|
|
9
|
+
that pool's cached 5-hour `resets_at` and then to 30 minutes rather than the
|
|
10
|
+
flat 10. The quarantine record carries `kind: 'quota'` and excludes the pool
|
|
11
|
+
from every later dispatch, in that run and in others, until it expires; the
|
|
12
|
+
action moves to another pool with quota and is never retried on the one that
|
|
13
|
+
hit the limit. `bullswarm run`, the V1 runtime, and V2 dispatch all apply the
|
|
14
|
+
same deadline. Detection is shape-gated: an agent report that discusses usage
|
|
15
|
+
limits, or tool output quoting them, is not a limit, and phrases that other
|
|
16
|
+
services also emit (`rate limited`, `too many requests`, `quota exceeded`)
|
|
17
|
+
count only as a bare notice, never as narration about someone else's quota
|
|
18
|
+
("rate limited by the GitHub API, retrying"). Connectors declare their
|
|
19
|
+
own phrases under `quotaSignatures`; installed connectors receive new ones on
|
|
20
|
+
upgrade.
|
|
21
|
+
|
|
22
|
+
- Routing avoids pools that are close to their 5-hour limit. A pool at or above
|
|
23
|
+
`FIVE_HOUR_NEAR_LIMIT_PCT` (75) is chosen only when no eligible pool below it
|
|
24
|
+
exists for the lane, ahead of pace, an approved assignment, and incumbency;
|
|
25
|
+
pools at or above 90 stay excluded outright, and a pool with no 5-hour
|
|
26
|
+
reading counts as having headroom. Routing reasons and candidate lists name
|
|
27
|
+
the utilization that decided the pick, and `bullswarm pools` shows it as
|
|
28
|
+
`5h=<n>%` with a `NEAR-5H-LIMIT` label. Meters and quarantines are re-read
|
|
29
|
+
from the meter cache and core state before every action dispatch and before
|
|
30
|
+
every retry inside one — forced live right after a usage limit — so a long
|
|
31
|
+
run no longer dispatches from the pool snapshot frozen at launch.
|
|
32
|
+
|
|
33
|
+
- `workflow watch` reports a usage-limit retry as a notable event in both
|
|
34
|
+
modes' vocabulary: `⚠ <action> usage limit on <pool> · paused until
|
|
35
|
+
<deadline> · retrying on another pool`, then `↺ <action> now on <pool> ·
|
|
36
|
+
<model>` once the retry lands. Both print without `--verbose`, wake `--next`,
|
|
37
|
+
and appear in `--jsonl` as `attempt.quota` and `attempt.moved`. The new
|
|
38
|
+
`--classic` flag forces the older heartbeat-based watcher (transition-on-change
|
|
39
|
+
snapshots plus a periodic heartbeat, 60 seconds unless `--heartbeat <seconds>`
|
|
40
|
+
is given) for a V2 run; it is a no-op for legacy runs and cannot combine with
|
|
41
|
+
`--next`.
|
|
42
|
+
|
|
43
|
+
## 0.25.2 — event-based watch
|
|
44
|
+
|
|
45
|
+
- `workflow watch <run> --next` is safe to relaunch after every wake-up: each
|
|
46
|
+
such exit prints `next: bullswarm workflow watch <id> --next --after <seq>
|
|
47
|
+
--since <time>`; relaunching with those values replays notable events that
|
|
48
|
+
landed while no watcher was attached, reports a level at most once, and does
|
|
49
|
+
not repeat a stall already reported (its recovery line still prints). `--jsonl`
|
|
50
|
+
objects carry `sequence`.
|
|
51
|
+
|
|
52
|
+
- `workflow watch` for V2 runs is event-based by default: one attach line,
|
|
53
|
+
then one line per notable event (action finished/failed/blocked/cancelled,
|
|
54
|
+
evidence, stage completion, planner turn, stall/recovery, cancellation)
|
|
55
|
+
and silence while work is merely in progress. `--next` prints no attach
|
|
56
|
+
line and exits after the first notable event (0 while the run continues or
|
|
57
|
+
delivered, 1 when it ended without delivering or the kernel is not
|
|
58
|
+
running). `--stall-after <seconds>` (default 300) reports a silent running
|
|
59
|
+
agent; `--heartbeat <seconds>` is opt-in for V2 (legacy still defaults to
|
|
60
|
+
60s). `--jsonl` emits one object per event with a stable `type`. Agent
|
|
61
|
+
starts, mechanical retries, and steering delivery remain `--verbose` only.
|
|
62
|
+
`--once` and legacy (non-V2) transition-plus-heartbeat output are
|
|
63
|
+
unchanged.
|
|
64
|
+
|
|
3
65
|
## 0.25.0 — shared programs that finish with the graph
|
|
4
66
|
|
|
5
67
|
- New goal workflows share the target worktree by default. File territories
|
package/README.md
CHANGED
|
@@ -37,12 +37,17 @@ detaches safely.
|
|
|
37
37
|
passing verification.
|
|
38
38
|
2. **Pace by meter.** The scheduling resource is the subscription window:
|
|
39
39
|
elapsed% minus used%, most-behind pool wins. Pace may only promote a
|
|
40
|
-
*cheaper* pool. Lanes are work-nature, never hard-coded to pools.
|
|
40
|
+
*cheaper* pool. Lanes are work-nature, never hard-coded to pools. The
|
|
41
|
+
5-hour window never paces — it gates: a pool at or above 75% of it is
|
|
42
|
+
chosen only when no eligible pool below that line exists, and one at or
|
|
43
|
+
above 90% is not dispatched at all.
|
|
41
44
|
3. **Delegate output is evidence, never authority.** The Workflow Planner may
|
|
42
45
|
propose actions, but only the deterministic kernel validates the program,
|
|
43
46
|
accepts requirement-scoped evidence, and computes completion.
|
|
44
47
|
4. **Quarantine re-probes.** A benched pool must be able to return to service
|
|
45
|
-
automatically; a lane is never allowed to silently go down.
|
|
48
|
+
automatically; a lane is never allowed to silently go down. A pool benched
|
|
49
|
+
for a usage limit waits for the reset the provider named, not a flat
|
|
50
|
+
guess — and never longer.
|
|
46
51
|
|
|
47
52
|
## Install
|
|
48
53
|
|
|
@@ -94,7 +99,7 @@ bullswarm health # re-judge saved outputs; catch gate failures
|
|
|
94
99
|
| `delegate` | Explain and execute the smallest reliable shape: one content-verified agent, or the planning contract for an autonomous workflow you author (`--orchestrator` dispatches a planner agent instead). |
|
|
95
100
|
| `run` | route → dispatch → watch → verify → one JSON verdict |
|
|
96
101
|
| `health` | Re-judge saved outputs against their verdicts; surface verify-gate failures and quarantine clusters |
|
|
97
|
-
| `pools` | Show each pool's meter state, pace position, quarantine status |
|
|
102
|
+
| `pools` | Show each pool's meter state, pace position, 5-hour utilization (`5h=<n>%`, flagged `NEAR-5H-LIMIT` at or above 75%), quarantine status |
|
|
98
103
|
| `strategy` | Interactive provider/model control center with live high/medium/low route previews and an agent-facing JSON API |
|
|
99
104
|
| `doctor` | Machine-readable readiness report; self-heals on first call |
|
|
100
105
|
| `workflow` | Start an autonomous goal, or run / validate / draft / inspect explicit workflows and their live instances. |
|
|
@@ -209,8 +214,15 @@ best eligible models on its configured interval. Disable it with
|
|
|
209
214
|
`strategy auto off --yes`. Discovery commands, model argument syntax, pricing,
|
|
210
215
|
and benchmark declarations remain connector-owned. Unknown license value,
|
|
211
216
|
prices, and benchmarks stay `null` rather than being guessed. An assignment is
|
|
212
|
-
only a preference: quarantine, exhaustion, burst gates,
|
|
213
|
-
still win.
|
|
217
|
+
only a preference: quarantine, exhaustion, burst gates, 5-hour headroom, and
|
|
218
|
+
capability checks still win. Routing prefers pools below
|
|
219
|
+
`FIVE_HOUR_NEAR_LIMIT_PCT` (75) of their 5-hour window over pools at or above
|
|
220
|
+
it, ahead of pace, an approved assignment, and incumbency; a near-limit pool is
|
|
221
|
+
still picked when it is the only eligible one, and a pool with no 5-hour
|
|
222
|
+
reading counts as having headroom. The routing reason and every candidate row
|
|
223
|
+
name the utilization that decided the pick, and meters and quarantines are
|
|
224
|
+
re-read before each dispatch — and again, live, right after a usage limit —
|
|
225
|
+
so a long run never routes off the snapshot it launched with.
|
|
214
226
|
|
|
215
227
|
Model exclusions are hard routing policy. An excluded model is removed from
|
|
216
228
|
recommendations and assignments, and Bullswarm pins a same-tier allowed model
|
|
@@ -302,7 +314,9 @@ The detached response includes a short ID and exact observation commands:
|
|
|
302
314
|
|
|
303
315
|
```bash
|
|
304
316
|
bullswarm workflow runs show <shortId>
|
|
305
|
-
bullswarm workflow watch <shortId> #
|
|
317
|
+
bullswarm workflow watch <shortId> # V2: attach, then one line per notable event
|
|
318
|
+
bullswarm workflow watch <shortId> --next # print the next notable event and exit
|
|
319
|
+
# relaunch with the --after/--since it prints
|
|
306
320
|
bullswarm workflow # unified human workflow home
|
|
307
321
|
bullswarm workflow tui <shortId> # jump directly to one run timeline
|
|
308
322
|
bullswarm workflow tui --json <shortId>
|
|
@@ -482,20 +496,57 @@ auditing completed runs.
|
|
|
482
496
|
|
|
483
497
|
### Live workflow dashboard
|
|
484
498
|
|
|
485
|
-
For ordinary observation, use the non-interactive watcher.
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
499
|
+
For ordinary observation, use the non-interactive watcher. For V2 runs it
|
|
500
|
+
prints one attach line, then one line per notable event as it happens
|
|
501
|
+
(action finished/failed/blocked/cancelled, evidence, stage completion,
|
|
502
|
+
planner turn, stall/recovery, cancellation, and the existing pause and
|
|
503
|
+
terminal `outcome:` / `next:` lines) and stays silent while work is merely
|
|
504
|
+
in progress. Agent starts, mechanical retries, and steering delivery print
|
|
505
|
+
only with `--verbose`. A usage-limit failure (`failureKind: 'quota'`) always
|
|
506
|
+
prints, verbose or not: `⚠ <actionId> usage limit on <pool> · paused until
|
|
507
|
+
<deadline> · retrying on another pool`, followed once the mechanical retry
|
|
508
|
+
lands on another pool by `↺ <actionId> now on <pool> · <model>`. The
|
|
509
|
+
periodic heartbeat is off unless you pass `--heartbeat <seconds>`;
|
|
510
|
+
`--stall-after <seconds>` (default 300) reports a running agent that has
|
|
511
|
+
gone silent. Pass `--classic` to force the older heartbeat-based watcher
|
|
512
|
+
instead (the transition-on-change snapshot stream plus a periodic
|
|
513
|
+
heartbeat, every 60 seconds unless `--heartbeat <seconds>` is given) —
|
|
514
|
+
legacy (non-V2) runs already behave this way and `--classic` is a no-op for
|
|
515
|
+
them; `--classic` cannot combine with `--next`, which exists only for event
|
|
516
|
+
mode. `--next` prints no attach line and
|
|
517
|
+
exits after the first notable event so a background terminal can wake the
|
|
518
|
+
caller; relaunch until the outcome line reports a pause or a terminal
|
|
519
|
+
status (exit 0 while the run continues or delivered, 1 when it ended
|
|
520
|
+
without delivering or the kernel is not running). Every `--next` exit that
|
|
521
|
+
leaves the run going ends with a relaunch line —
|
|
522
|
+
`next: bullswarm workflow watch <shortId> --next --after <sequence> --since <iso>` —
|
|
523
|
+
and the relaunch should copy those two values verbatim: `--after` starts
|
|
524
|
+
from the durable event sequence the previous watcher consumed, so events
|
|
525
|
+
committed while nothing was attached are printed instead of skipped, and
|
|
526
|
+
`--since` is that watcher's exit time, so an agent whose silence it already
|
|
527
|
+
reported does not produce a duplicate stall line (its recovery still
|
|
528
|
+
prints). `--jsonl` emits one JSON object per notable event with a stable
|
|
529
|
+
`type` (`attach`, `action.finished`,
|
|
530
|
+
`evidence.recorded`, `stage.completed`, `planner.finished`, `agent.stalled`,
|
|
531
|
+
`agent.recovered`, `cancellation.requested`, `attempt.quota`,
|
|
532
|
+
`attempt.moved`, `paused`, `finished`,
|
|
533
|
+
`interrupted`, and with `--verbose` `action.started`, `attempt.retrying`,
|
|
534
|
+
`steering.delivered`); in that mode the relaunch line is not printed and
|
|
535
|
+
every object instead carries the `sequence` it was emitted at, which is the
|
|
536
|
+
value to pass as `--after`. `--once` still prints one current snapshot. Legacy
|
|
537
|
+
(non-V2) runs keep the compact transition-plus-heartbeat stream unchanged,
|
|
538
|
+
the same stream `--classic` opts a V2 run into.
|
|
493
539
|
|
|
494
540
|
```bash
|
|
495
541
|
bullswarm workflow watch <shortId>
|
|
496
|
-
bullswarm workflow watch <shortId> --
|
|
542
|
+
bullswarm workflow watch <shortId> --next # next notable event, then exit
|
|
543
|
+
bullswarm workflow watch <shortId> --next --after 42 --since 2026-09-08T10:15:00.000Z
|
|
544
|
+
# the relaunch: values copied from the previous next: line
|
|
545
|
+
bullswarm workflow watch <shortId> --jsonl # one JSON object per event
|
|
497
546
|
bullswarm workflow watch <shortId> --once # one current/terminal snapshot
|
|
498
|
-
bullswarm workflow watch <shortId> --verbose #
|
|
547
|
+
bullswarm workflow watch <shortId> --verbose # started / retry / steering too
|
|
548
|
+
bullswarm workflow watch <shortId> --stall-after 120 --heartbeat 30
|
|
549
|
+
bullswarm workflow watch <shortId> --classic # older heartbeat-based watcher instead of event mode
|
|
499
550
|
```
|
|
500
551
|
|
|
501
552
|
`workflow tui` is the interactive, Claude-style `/workflows` view. For an
|
|
@@ -584,6 +635,20 @@ channel, so reading source text such as an auth-signature matcher cannot falsely
|
|
|
584
635
|
quarantine Grok or Command Code. Error-shaped semantic results and stderr
|
|
585
636
|
diagnostics still trigger the auth/quota guard.
|
|
586
637
|
|
|
638
|
+
A provider that reports a usage limit — `You've hit your session limit ·
|
|
639
|
+
resets 8:20pm (Asia/Hong_Kong)`, `usage_credits_required`, `rate limit
|
|
640
|
+
exceeded`, `quota exceeded` — is its own mechanical failure kind, `quota`,
|
|
641
|
+
never `process`, `semantic`, or `auth`. The attempt is killed immediately
|
|
642
|
+
even if the CLI would otherwise hang, and the pool is quarantined until the
|
|
643
|
+
reset time parsed from the message, falling back to that pool's cached 5-hour
|
|
644
|
+
`resets_at` and then to 30 minutes. The quarantine record carries
|
|
645
|
+
`kind: 'quota'` and excludes the pool from every later dispatch, in this run
|
|
646
|
+
and in others, until it expires; the action is immediately re-dispatched on
|
|
647
|
+
another pool with quota and never retried on the one that hit the limit. An
|
|
648
|
+
agent report that merely discusses usage limits, or tool output that quotes
|
|
649
|
+
them, is not a limit: detection is shape-gated to lines that look like a
|
|
650
|
+
provider notice.
|
|
651
|
+
|
|
587
652
|
After ten minutes without transport, parsed-event, or semantic-action evidence,
|
|
588
653
|
an active child is labeled `suspected_stalled`. This is an inspection signal,
|
|
589
654
|
not a death verdict and never an automatic kill: buffered CLIs can be silent
|
package/connectors/_schema.json
CHANGED
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
"$comment-cwd": "cwdMode documents how the CLI resolves its project: 'pwd' means the CLI resolves from $PWD so the watcher MUST set PWD and spawn inside --add-dir"
|
|
11
11
|
},
|
|
12
12
|
"authSignatures": ["strings in output that mean auth/throttle failure"],
|
|
13
|
+
"quotaSignatures": ["strings in output that mean this credential is out of quota RIGHT NOW (usage/rate limit hit)"],
|
|
14
|
+
"$comment-quotaSignatures": "Optional. Merged on top of DEFAULT_QUOTA_SIGNATURES (src/lib/quota.js), matched case-insensitively, and only honored when the matched line is quota-shaped (<=300 chars and the phrase starts in the first 40 chars or the line is error-shaped) so agent prose and tool output that merely discuss limits are never killed. A hit is classified as the mechanical failure kind `quota` (checked before authSignatures), kills the attempt, and quarantines the pool until the reset time parsed from the message, else the pool's cached five_hour.resets_at, else 30 minutes.",
|
|
13
15
|
"outputExtraction": {
|
|
14
16
|
"$comment": "how to get the real answer out of stdout+stderr+files",
|
|
15
17
|
"strategy": "stdout|stdout-tail|json-field|file|event-stream",
|
|
@@ -23,6 +23,12 @@
|
|
|
23
23
|
"not logged in",
|
|
24
24
|
"please run /login"
|
|
25
25
|
],
|
|
26
|
+
"$comment-quotaSignatures": "usage-limit phrases: classified `quota`, quarantined until the reset the message names. DEFAULT_QUOTA_SIGNATURES (src/lib/quota.js) apply on top of these.",
|
|
27
|
+
"quotaSignatures": [
|
|
28
|
+
"hit your session limit",
|
|
29
|
+
"hit your limit",
|
|
30
|
+
"hit your usage limit"
|
|
31
|
+
],
|
|
26
32
|
"outputExtraction": {
|
|
27
33
|
"strategy": "event-stream"
|
|
28
34
|
},
|
package/connectors/codex.json
CHANGED
|
@@ -21,6 +21,11 @@
|
|
|
21
21
|
"unauthorized",
|
|
22
22
|
"invalid api key"
|
|
23
23
|
],
|
|
24
|
+
"$comment-quotaSignatures": "usage_credits_required is a spent window, not a broken credential: it classifies `quota` (checked before authSignatures) so the pool is quarantined until its reset instead of re-probed in 10 minutes.",
|
|
25
|
+
"quotaSignatures": [
|
|
26
|
+
"usage_credits_required",
|
|
27
|
+
"usage limit"
|
|
28
|
+
],
|
|
24
29
|
"outputExtraction": {
|
|
25
30
|
"strategy": "event-stream"
|
|
26
31
|
},
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
"not authenticated",
|
|
22
22
|
"cmd login"
|
|
23
23
|
],
|
|
24
|
+
"$comment-quotaSignatures": "empty on purpose: no cmd-specific usage-limit wording has been observed yet; DEFAULT_QUOTA_SIGNATURES in src/lib/quota.js covers the generic provider phrasings.",
|
|
25
|
+
"quotaSignatures": [],
|
|
24
26
|
"outputExtraction": {
|
|
25
27
|
"strategy": "event-stream"
|
|
26
28
|
},
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// echo-worker.mjs — deterministic test delegate for bullswarm.
|
|
2
2
|
// Reads a task file; behavior is driven by directives in the task text.
|
|
3
3
|
// FAIL:auth -> prints an auth failure, exits 0 (the lying-exit trap)
|
|
4
|
+
// FAIL:quota -> prints a provider usage limit naming its reset, exits 0
|
|
4
5
|
// FAIL:exit -> prints a complete answer, exits 1 (exit-1-after-success)
|
|
5
6
|
// INTENT: -> prints only an announcement, exits 0
|
|
6
7
|
// otherwise -> echoes the task as a completed answer, exit 0
|
|
@@ -11,6 +12,11 @@ const task = readFileSync(process.argv[2], 'utf8');
|
|
|
11
12
|
const sleepMatch = task.match(/SLEEP_MS:(\d+)/);
|
|
12
13
|
if (sleepMatch) await new Promise((resolve) => setTimeout(resolve, Number(sleepMatch[1])));
|
|
13
14
|
|
|
15
|
+
if (task.includes('FAIL:quota')) {
|
|
16
|
+
// A usage limit is not a broken credential: it names when it resets.
|
|
17
|
+
console.log('Error: usage limit reached · resets in 45 minutes');
|
|
18
|
+
process.exit(0);
|
|
19
|
+
}
|
|
14
20
|
if (task.includes('FAIL:auth-hang')) {
|
|
15
21
|
console.log('Authentication failed: quota exhausted; waiting process should be terminated.');
|
|
16
22
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
package/connectors/echo.json
CHANGED
package/connectors/grok.json
CHANGED
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
"rate limit",
|
|
18
18
|
"invalid api key"
|
|
19
19
|
],
|
|
20
|
+
"$comment-quotaSignatures": "empty on purpose: no grok-specific usage-limit wording has been observed yet, and the generic phrasings (rate limit exceeded, too many requests, quota exceeded) already come from DEFAULT_QUOTA_SIGNATURES in src/lib/quota.js.",
|
|
21
|
+
"quotaSignatures": [],
|
|
20
22
|
"outputExtraction": {
|
|
21
23
|
"strategy": "event-stream"
|
|
22
24
|
},
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
"$comment-cwdMode": "QUIRK: resolves its project from $PWD, not the spawn cwd. The watcher MUST set env.PWD and spawn with cwd inside the target repo, or it will silently analyse the wrong repository and answer confidently about it."
|
|
9
9
|
},
|
|
10
10
|
"authSignatures": ["No cookie auth credentials found", "unauthorized"],
|
|
11
|
+
"$comment-quotaSignatures": "empty on purpose: provider quota errors reach opencode from the upstream API in generic form (insufficient_quota, rate limit exceeded, too many requests), which DEFAULT_QUOTA_SIGNATURES in src/lib/quota.js already matches.",
|
|
12
|
+
"quotaSignatures": [],
|
|
11
13
|
"outputExtraction": { "strategy": "event-stream" },
|
|
12
14
|
"eventStream": {
|
|
13
15
|
"format": "jsonl",
|
package/package.json
CHANGED
package/skill/SKILL.md
CHANGED
|
@@ -109,11 +109,26 @@ validate again. A valid launch detaches and returns `shortId`; report it.
|
|
|
109
109
|
## 3. Observe and judge the result
|
|
110
110
|
|
|
111
111
|
```bash
|
|
112
|
-
bullswarm workflow watch <shortId>
|
|
112
|
+
bullswarm workflow watch <shortId> --next
|
|
113
|
+
bullswarm workflow watch <shortId> --next --after <sequence> --since <iso-timestamp>
|
|
113
114
|
bullswarm workflow runs result <shortId> --json
|
|
114
115
|
```
|
|
115
116
|
|
|
116
117
|
When the user asked you to complete the work, follow the run through its result.
|
|
118
|
+
Launch `bullswarm workflow watch <shortId> --next` in a background terminal, act
|
|
119
|
+
on the printed event when it exits, and relaunch until the outcome line reports
|
|
120
|
+
a pause or a terminal status. Each exit that leaves the run going ends with
|
|
121
|
+
`next: bullswarm workflow watch <shortId> --next --after <sequence> --since <iso>`:
|
|
122
|
+
relaunch with exactly those two values, so events committed while you were
|
|
123
|
+
acting are printed instead of skipped and a stall you already saw does not
|
|
124
|
+
report twice. In `--jsonl` mode there is no such line — take `--after` from the
|
|
125
|
+
`sequence` field of the last object. V2 watch prints one line per notable event
|
|
126
|
+
and stays silent while work is merely in progress; `--heartbeat` is opt-in and
|
|
127
|
+
`--stall-after` (default 300s) reports a silent running agent. A usage-limit
|
|
128
|
+
failure always prints — `⚠ ... usage limit on <pool> · paused until <deadline>
|
|
129
|
+
· retrying on another pool`, then `↺ ... now on <pool> · <model>` once the
|
|
130
|
+
mechanical retry lands — even without `--verbose`. Pass `--classic` for the
|
|
131
|
+
older heartbeat-based watcher instead (it cannot combine with `--next`).
|
|
117
132
|
`watch` also exits at a durable planning pause; that is not completion.
|
|
118
133
|
|
|
119
134
|
Read action outputs and actual artifacts, and probe important edge cases yourself.
|
|
@@ -30,6 +30,8 @@ deprecated alias for `--orchestrator <pool> --orchestrator-strict`.
|
|
|
30
30
|
Observe and consume:
|
|
31
31
|
|
|
32
32
|
```bash
|
|
33
|
+
bullswarm workflow watch <shortId> --next
|
|
34
|
+
bullswarm workflow watch <shortId> --next --after <sequence> --since <iso-timestamp>
|
|
33
35
|
bullswarm workflow watch <shortId>
|
|
34
36
|
bullswarm workflow tui <shortId>
|
|
35
37
|
bullswarm workflow tui --json <shortId>
|
|
@@ -37,7 +39,25 @@ bullswarm workflow events --json <shortId> --after 0
|
|
|
37
39
|
bullswarm workflow runs result <shortId> --json
|
|
38
40
|
```
|
|
39
41
|
|
|
40
|
-
|
|
42
|
+
V2 watch prints one attach line, then one line per notable event, and stays
|
|
43
|
+
silent while work is merely in progress. Launch
|
|
44
|
+
`bullswarm workflow watch <shortId> --next` in a background terminal, act on the
|
|
45
|
+
printed event when it exits, and relaunch until the outcome line reports a
|
|
46
|
+
pause or a terminal status. Every `--next` exit that leaves the run going ends
|
|
47
|
+
with `next: bullswarm workflow watch <shortId> --next --after <sequence> --since <iso>`;
|
|
48
|
+
relaunch with those exact `--after` and `--since` values so events committed
|
|
49
|
+
while no watcher was attached are printed rather than skipped and an
|
|
50
|
+
already-reported stall does not fire again (its recovery still prints). With
|
|
51
|
+
`--jsonl` that line is absent: take `--after` from the `sequence` field carried
|
|
52
|
+
by every emitted object. `--heartbeat` is opt-in for V2 (legacy still
|
|
53
|
+
defaults to 60s). `--stall-after` (default 300s) reports a silent running
|
|
54
|
+
agent. A usage-limit failure always prints, verbose or not: `⚠ ... usage
|
|
55
|
+
limit on <pool> · paused until <deadline> · retrying on another pool`, then
|
|
56
|
+
`↺ ... now on <pool> · <model>` once the mechanical retry lands on another
|
|
57
|
+
pool. Use `--verbose` only for diagnosis. `--classic` forces the older
|
|
58
|
+
heartbeat-based watcher (transition-on-change snapshots plus a periodic
|
|
59
|
+
heartbeat) instead of event mode; it is a no-op for legacy runs and cannot
|
|
60
|
+
combine with `--next`.
|
|
41
61
|
The result command is the stable delivery/verification envelope; do not scrape
|
|
42
62
|
task files or assume the last provider response is the deliverable.
|
|
43
63
|
|
|
@@ -135,9 +155,13 @@ bullswarm strategy inventory --json
|
|
|
135
155
|
bullswarm strategy routes --json
|
|
136
156
|
```
|
|
137
157
|
|
|
138
|
-
Automatic routing chooses the most-behind capable eligible pool
|
|
139
|
-
gates and quarantine, and applies only
|
|
140
|
-
and exclusions.
|
|
158
|
+
Automatic routing chooses the most-behind capable eligible pool among those
|
|
159
|
+
with 5-hour headroom, honors burst gates and quarantine, and applies only
|
|
160
|
+
explicitly approved model assignments and exclusions. A pool at or above 75%
|
|
161
|
+
of its 5-hour window is picked only when no eligible pool below that line
|
|
162
|
+
exists; `bullswarm pools` shows the reading as `5h=<n>%` with a
|
|
163
|
+
`NEAR-5H-LIMIT` label, and meters and quarantines are re-read before every
|
|
164
|
+
dispatch rather than frozen at launch. Humans can use bare `bullswarm strategy` to toggle providers
|
|
141
165
|
and multi-select high/medium/low per model. Agents should consume the inventory
|
|
142
166
|
and apply validated changes with `strategy set-provider`, `strategy set-model`,
|
|
143
167
|
or one atomic `strategy configure --file <json> --yes`. Never weaken those
|
|
@@ -145,8 +169,13 @@ controls in a prompt.
|
|
|
145
169
|
|
|
146
170
|
## Recovery and stopping rules
|
|
147
171
|
|
|
148
|
-
- Auth
|
|
149
|
-
another eligible pool.
|
|
172
|
+
- Auth signatures quarantine the affected pool for a 10-minute re-probe
|
|
173
|
+
window; later dispatches use another eligible pool.
|
|
174
|
+
- A provider usage limit is the distinct failure kind `quota`: the attempt is
|
|
175
|
+
killed at once, the pool is quarantined until the reset the message named
|
|
176
|
+
(else its cached 5-hour `resets_at`, else 30 minutes), and the action moves
|
|
177
|
+
to a pool that still has quota. The quarantine holds across runs until it
|
|
178
|
+
expires. Discussing usage limits in a report is not a usage limit.
|
|
150
179
|
- A quota-gated preferred orchestrator falls back unless it was strictly pinned
|
|
151
180
|
for QA.
|
|
152
181
|
- Silence is evidence to inspect, not automatic proof of a hang. Check the TUI
|
package/src/cli.js
CHANGED
|
@@ -78,13 +78,17 @@ async function cmdPools(opts) {
|
|
|
78
78
|
? 'unmetered'
|
|
79
79
|
: `used ${p.usedPct ?? '?'}% elapsed ${p.elapsedPct ?? '?'}% [${src}]`;
|
|
80
80
|
const burst = p.burstGate ? ' BURST-GATED' : '';
|
|
81
|
+
// 5h is a gate, never a pace (doctrine M3): show the reading and whether
|
|
82
|
+
// routing now deprioritizes this pool for it.
|
|
83
|
+
const fiveHour = p.fiveHourUsedPct == null ? '' : ` 5h=${Math.round(p.fiveHourUsedPct * 10) / 10}%`;
|
|
84
|
+
const nearLimit = p.nearFiveHourLimit === true ? ' NEAR-5H-LIMIT' : '';
|
|
81
85
|
const status = !p.enabled
|
|
82
86
|
? 'disabled'
|
|
83
87
|
: p.quarantine
|
|
84
88
|
? `QUARANTINED until ${new Date(p.quarantine.until).toLocaleTimeString()} (${p.quarantine.reason})`
|
|
85
|
-
: `ready${burst}`;
|
|
89
|
+
: `ready${burst}${nearLimit}`;
|
|
86
90
|
console.log(
|
|
87
|
-
`${p.name.padEnd(14)} cost=${p.costRank} lanes=${p.lanes.join('/')} ${meter} surplus=${p.pace ?? '-'} ${status}`,
|
|
91
|
+
`${p.name.padEnd(14)} cost=${p.costRank} lanes=${p.lanes.join('/')} ${meter} surplus=${p.pace ?? '-'}${fiveHour} ${status}`,
|
|
88
92
|
);
|
|
89
93
|
}
|
|
90
94
|
return 0;
|
|
@@ -228,6 +232,9 @@ async function cmdRun(opts) {
|
|
|
228
232
|
timeoutSec: opts.timeout == null ? null : Number(opts.timeout),
|
|
229
233
|
env: childDepthEnv(process.env),
|
|
230
234
|
model: selectedModel,
|
|
235
|
+
// Lets a usage-limit verdict fall back to this pool's cached 5h meter
|
|
236
|
+
// reset when the provider's message named no reset time of its own.
|
|
237
|
+
bullswarmDir: getBullswarmDir(),
|
|
231
238
|
onActivity: (event) => heartbeat.activity(event),
|
|
232
239
|
onAgentEvent: () => heartbeat.event(),
|
|
233
240
|
});
|
|
@@ -240,7 +247,12 @@ async function cmdRun(opts) {
|
|
|
240
247
|
state.incumbents ??= {};
|
|
241
248
|
state.incumbents[lane] = connector.name;
|
|
242
249
|
} else if (verdict.quarantineHint) {
|
|
243
|
-
|
|
250
|
+
// A usage limit carries its own deadline (the reset the provider named);
|
|
251
|
+
// an auth failure keeps the flat re-probe window.
|
|
252
|
+
quarantinePool(state, connector.name, verdict.why, now, {
|
|
253
|
+
until: verdict.quarantineUntil ?? null,
|
|
254
|
+
kind: verdict.failureKind === 'quota' ? 'quota' : 'auth',
|
|
255
|
+
});
|
|
244
256
|
verdict.quarantinedUntil = state.pools[connector.name]?.quarantine?.until;
|
|
245
257
|
}
|
|
246
258
|
|
package/src/help.js
CHANGED
|
@@ -987,25 +987,47 @@ const workflowTuiText = rich({
|
|
|
987
987
|
});
|
|
988
988
|
|
|
989
989
|
const workflowWatchText = rich({
|
|
990
|
-
usage: 'bullswarm workflow watch <runId> [--interval <seconds>] [--heartbeat <seconds>] [--jsonl] [--once] [--verbose]',
|
|
991
|
-
purpose: "Follow one run
|
|
992
|
-
+ '
|
|
993
|
-
+ '
|
|
994
|
-
+ '
|
|
995
|
-
+ '
|
|
990
|
+
usage: 'bullswarm workflow watch <runId> [--classic] [--interval <seconds>] [--heartbeat <seconds>] [--stall-after <seconds>] [--next [--after <sequence>] [--since <iso-timestamp>]] [--jsonl] [--once] [--verbose]',
|
|
991
|
+
purpose: "Follow one V2 run by printing one attach line, then one line per notable event "
|
|
992
|
+
+ '(action finished/failed/blocked/cancelled, evidence, stage completion, stall/recovery, planning, '
|
|
993
|
+
+ 'cancellation) and staying silent while work is merely in progress. A usage-limit failure always '
|
|
994
|
+
+ 'prints, verbose or not: an `⚠ ... usage limit on <pool> · paused until <deadline> · retrying on '
|
|
995
|
+
+ 'another pool` line, then an `↺ ... now on <pool> · <model>` line once the mechanical retry lands. '
|
|
996
|
+
+ '`--classic` forces the older heartbeat-based watcher instead (transition-on-change snapshots plus '
|
|
997
|
+
+ 'a periodic heartbeat); legacy runs already behave this way and are unaffected. `--next` prints no '
|
|
998
|
+
+ 'attach line and returns after the first notable event, or immediately at a pause or terminal status '
|
|
999
|
+
+ '(event mode only — it cannot combine with `--classic`). '
|
|
1000
|
+
+ 'Every `--next` exit that leaves the run going prints a `next:` relaunch line carrying `--after` and '
|
|
1001
|
+
+ '`--since`; pass those two values back on the relaunch so events committed while no watcher was '
|
|
1002
|
+
+ 'attached are printed instead of skipped and an already-reported stall does not fire again. '
|
|
1003
|
+
+ '`--heartbeat` is opt-in for V2; legacy runs (and `--classic`) keep the historical 60s heartbeat by '
|
|
1004
|
+
+ 'default. Distinct from the full-screen tui and the machine-oriented events replay.',
|
|
996
1005
|
args: [{ name: '<runId>', desc: 'shortId or runId' }],
|
|
997
1006
|
options: [
|
|
1007
|
+
{ flag: '--classic', desc: 'force the older heartbeat-based watcher (transition-on-change snapshots plus a periodic heartbeat) instead of event mode; no-op for legacy runs; cannot combine with --next', default: 'off (event mode for V2 runs)' },
|
|
998
1008
|
{ flag: '--interval <seconds>', desc: 'poll interval while following', default: '2' },
|
|
999
|
-
{ flag: '--heartbeat <seconds>', desc: '
|
|
1000
|
-
{ flag: '--
|
|
1009
|
+
{ flag: '--heartbeat <seconds>', desc: 'print a periodic heartbeat line when nothing has changed; opt-in for V2, must be >= 1', default: 'off for V2, 60 for legacy and --classic' },
|
|
1010
|
+
{ flag: '--stall-after <seconds>', desc: 'report a running agent as silent after this many seconds without activity; must be >= 1', default: '300' },
|
|
1011
|
+
{ flag: '--next', desc: 'print no attach line; exit after the first poll that printed a notable event, or immediately at a pause or terminal status', default: 'off (follows until terminal or pause)' },
|
|
1012
|
+
{ flag: '--after <sequence>', desc: 'start from this durable event sequence instead of the current high-water mark, so events committed since the previous watcher exited are printed; use the value from the previous `next:` line (in --jsonl, the `sequence` field of the last object)', default: 'attach at the current high-water mark' },
|
|
1013
|
+
{ flag: '--since <iso-timestamp>', desc: 'the previous watcher\'s exit time; a running agent already silent at attach is reported only if its silence crossed --stall-after at or after this time, so no duplicate stall line prints (its recovery still does); use the value from the previous `next:` line', default: 'report every agent silent past --stall-after at attach' },
|
|
1014
|
+
{ flag: '--jsonl', desc: 'emit one JSON object per line instead of human text; every object carries the `sequence` it was emitted at, and the `next:` relaunch line is not printed', default: 'off (human text)' },
|
|
1001
1015
|
{ flag: '--once', desc: 'print a single current snapshot and exit immediately instead of following', default: 'off (follows until terminal)' },
|
|
1002
|
-
{ flag: '--verbose', desc: 'include per-agent action detail
|
|
1016
|
+
{ flag: '--verbose', desc: 'include started, retry, and steering-delivered lines (V2) and per-agent action detail (legacy)', default: 'off (compact)' },
|
|
1003
1017
|
],
|
|
1004
1018
|
safety: [
|
|
1005
1019
|
'read-only — polls durable state/events on a timer; writes nothing',
|
|
1006
1020
|
'exits 0 if the run reaches a delivered status (or on --once), 1 if it reaches a non-delivered terminal status',
|
|
1021
|
+
'--next exits 0 while the run continues or when it delivered, 1 when it ended without delivering or the kernel is not running',
|
|
1022
|
+
'for a V2 run, a --next exit that leaves the run going ends with `next: bullswarm workflow watch <shortId> --next --after <sequence> --since <iso>`; pause, terminal and interrupted exits keep their own outcome/next lines',
|
|
1023
|
+
'--classic --next is rejected with exit 2: --next only applies to event mode',
|
|
1024
|
+
],
|
|
1025
|
+
examples: [
|
|
1026
|
+
{ cmd: 'bullswarm workflow watch ab12cd --next', note: 'print the next notable event and exit; relaunch until outcome reports a pause or a terminal status' },
|
|
1027
|
+
{ cmd: 'bullswarm workflow watch ab12cd --next --after 42 --since 2026-09-08T10:15:00.000Z', note: 'the relaunch: copy both values from the `next:` line the previous exit printed' },
|
|
1028
|
+
{ cmd: 'bullswarm workflow watch ab12cd --stall-after 120 --heartbeat 30' },
|
|
1029
|
+
{ cmd: 'bullswarm workflow watch ab12cd --classic', note: 'the older heartbeat-based watcher instead of event mode' },
|
|
1007
1030
|
],
|
|
1008
|
-
examples: [{ cmd: 'bullswarm workflow watch ab12cd --heartbeat 30' }],
|
|
1009
1031
|
next: 'bullswarm workflow runs result <runId> --json once it finishes, or bullswarm workflow tui <runId> for the interactive view.',
|
|
1010
1032
|
});
|
|
1011
1033
|
|
package/src/lib/config.js
CHANGED
|
@@ -13,6 +13,7 @@ import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { loadState } from './state.js';
|
|
15
15
|
import { paceScore, isQuarantined } from './route.js';
|
|
16
|
+
import { FIVE_HOUR_NEAR_LIMIT_PCT } from '../meters/framework.js';
|
|
16
17
|
import { expandClaudeAccountConnectors } from './claude-accounts.js';
|
|
17
18
|
import { expandOpenCodeKaihkConnectors } from './opencode-kaihk.js';
|
|
18
19
|
|
|
@@ -66,6 +67,10 @@ export function buildPools(bullswarmDir, now = Date.now(), readings = {}) {
|
|
|
66
67
|
elapsedPct: null,
|
|
67
68
|
pace: null,
|
|
68
69
|
burstGate: false,
|
|
70
|
+
// 5h window (doctrine M3): gates routing, never paces it.
|
|
71
|
+
fiveHourUsedPct: null,
|
|
72
|
+
fiveHourResetsAt: null,
|
|
73
|
+
nearFiveHourLimit: false,
|
|
69
74
|
meterSnapshot: null,
|
|
70
75
|
subscription: {
|
|
71
76
|
...(conn.subscription ?? {}),
|
|
@@ -85,6 +90,15 @@ export function buildPools(bullswarmDir, now = Date.now(), readings = {}) {
|
|
|
85
90
|
const ps = state.pools[p.name] ?? {};
|
|
86
91
|
|
|
87
92
|
const reading = readings[p.name];
|
|
93
|
+
// The 5h gate is independent of the pacing window: a reading may carry a
|
|
94
|
+
// 5h utilization with no weekly/monthly window to pace by, and routing
|
|
95
|
+
// still has to see that the pool is close to its 5h limit.
|
|
96
|
+
if (reading) {
|
|
97
|
+
const fiveHour = fiveHourFromReading(reading);
|
|
98
|
+
p.fiveHourUsedPct = fiveHour.usedPct;
|
|
99
|
+
p.fiveHourResetsAt = fiveHour.resetsAt;
|
|
100
|
+
p.nearFiveHourLimit = fiveHour.nearLimit;
|
|
101
|
+
}
|
|
88
102
|
if (reading?.pacing) {
|
|
89
103
|
// Provider-truth path (M1/M2)
|
|
90
104
|
p.meterSource = reading.source; // live | cache | stale
|
|
@@ -112,6 +126,30 @@ export function buildPools(bullswarmDir, now = Date.now(), readings = {}) {
|
|
|
112
126
|
return { state, connectors, pools };
|
|
113
127
|
}
|
|
114
128
|
|
|
129
|
+
/** null-safe finite coercion: null/undefined/NaN all mean "no reading". */
|
|
130
|
+
function finiteOrNull(value) {
|
|
131
|
+
if (value == null) return null;
|
|
132
|
+
const n = Number(value);
|
|
133
|
+
return Number.isFinite(n) ? n : null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* 5h window fields from a meter reading. Prefers the flat fields paceSnapshot
|
|
138
|
+
* produces and falls back to the raw snapshot, so a reading assembled by an
|
|
139
|
+
* older code path still reports a real 5h number instead of null.
|
|
140
|
+
*/
|
|
141
|
+
function fiveHourFromReading(reading) {
|
|
142
|
+
const usedPct = finiteOrNull(reading?.fiveHourUsedPct)
|
|
143
|
+
?? finiteOrNull(reading?.snapshot?.five_hour?.utilization);
|
|
144
|
+
const raw = reading?.fiveHourResetsAt ?? reading?.snapshot?.five_hour?.resets_at ?? null;
|
|
145
|
+
const resetsMs = typeof raw === 'string' ? Date.parse(raw) : NaN;
|
|
146
|
+
return {
|
|
147
|
+
usedPct,
|
|
148
|
+
resetsAt: Number.isFinite(resetsMs) ? new Date(resetsMs).toISOString() : null,
|
|
149
|
+
nearLimit: usedPct != null && usedPct >= FIVE_HOUR_NEAR_LIMIT_PCT,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
115
153
|
/**
|
|
116
154
|
* Async variant that fetches live readings for pools with readers.
|
|
117
155
|
*/
|