@yemi33/minions 0.1.2149 → 0.1.2150
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/js/render-work-items.js +21 -13
- package/dashboard.js +67 -10
- package/docs/README.md +1 -1
- package/docs/auto-discovery.md +2 -1
- package/docs/branch-derivation.md +68 -0
- package/docs/cooldown-merge-semantics.md +4 -4
- package/docs/design-state-storage.md +5 -5
- package/docs/kb-sweep.md +2 -2
- package/docs/managed-spawn.md +1 -1
- package/docs/timeouts-and-liveness.md +120 -0
- package/docs/watches.md +9 -9
- package/docs/worktree-lifecycle.md +164 -0
- package/engine/cli.js +3 -2
- package/package.json +1 -1
|
@@ -15,6 +15,10 @@ const _WI_ENRICHMENT_FIELDS = [
|
|
|
15
15
|
'_pr', '_prUrl', '_pendingReason', '_skipReason', '_blockedBy',
|
|
16
16
|
'_humanFeedback', '_reopened', '_managedSpawnPartial', '_securityFlag',
|
|
17
17
|
'_artifacts', 'referencesCount', 'acceptanceCriteriaCount',
|
|
18
|
+
// W-mq5xg5e9000nec0e — slim shape marker so the detail/edit modals know
|
|
19
|
+
// to hydrate the full description via GET /api/work-items/<id> rather than
|
|
20
|
+
// pre-filling textareas with the truncated snippet.
|
|
21
|
+
'_descriptionTruncated',
|
|
18
22
|
// W-mq08kuog001110a6 — _preDispatchEval IS persisted to disk by
|
|
19
23
|
// engine/dispatch.js#_persistInvalidWorkItem, but the slim /api/status
|
|
20
24
|
// overlay may drop it on later passes. Carry it across the overlay so
|
|
@@ -302,17 +306,19 @@ function renderWorkItems(items, opts) {
|
|
|
302
306
|
async function editWorkItem(id, source) {
|
|
303
307
|
const cached = allWorkItems.find(i => i.id === id);
|
|
304
308
|
if (!cached) return;
|
|
305
|
-
// Hydrate the full record before rendering the form. The
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
//
|
|
309
|
-
//
|
|
310
|
-
//
|
|
311
|
-
//
|
|
312
|
-
//
|
|
309
|
+
// Hydrate the full record before rendering the form. The bulk
|
|
310
|
+
// /api/work-items list endpoint truncates `description` and replaces
|
|
311
|
+
// `references` / `acceptanceCriteria` arrays with `referencesCount` /
|
|
312
|
+
// `acceptanceCriteriaCount` integers (W-mphejzmj000718bf / W-mq5xg5e9000nec0e
|
|
313
|
+
// — see dashboard.js slimWorkItemForList). Without hydration the textareas
|
|
314
|
+
// would pre-fill with truncated or empty values, and Save would POST those
|
|
315
|
+
// back to handleWorkItemsUpdate, which treats them as defined
|
|
316
|
+
// (dashboard.js:5157, :5164-5165) and silently wipes the stored values:
|
|
317
|
+
// data-loss on every pencil-edit of a pending/failed WI.
|
|
313
318
|
// See PR #2816 review feedback.
|
|
314
319
|
let item = cached;
|
|
315
320
|
const needsHydration = !cached.description ||
|
|
321
|
+
cached._descriptionTruncated === true ||
|
|
316
322
|
(cached.acceptanceCriteriaCount > 0 && !Array.isArray(cached.acceptanceCriteria)) ||
|
|
317
323
|
(cached.referencesCount > 0 && !Array.isArray(cached.references));
|
|
318
324
|
if (needsHydration) {
|
|
@@ -802,12 +808,14 @@ function openWorkItemDetail(id) {
|
|
|
802
808
|
if (!cached) return;
|
|
803
809
|
|
|
804
810
|
// Render the modal immediately from the cached (slim) record so the click
|
|
805
|
-
// feels instant. The /api/
|
|
806
|
-
// PR link, artifacts, etc. — everything except the heavy free-text
|
|
807
|
-
// (description, full acceptanceCriteria, full references)
|
|
808
|
-
// W-mphejzmj000718bf. We then hydrate the
|
|
809
|
-
// GET /api/work-items/<id> and re-render in
|
|
811
|
+
// feels instant. The bulk /api/work-items list is the source of badges,
|
|
812
|
+
// status, PR link, artifacts, etc. — everything except the heavy free-text
|
|
813
|
+
// fields (description hard-capped, full acceptanceCriteria, full references)
|
|
814
|
+
// — see W-mphejzmj000718bf / W-mq5xg5e9000nec0e. We then hydrate the
|
|
815
|
+
// missing/truncated fields from GET /api/work-items/<id> and re-render in
|
|
816
|
+
// place.
|
|
810
817
|
const needsHydration = !cached.description ||
|
|
818
|
+
cached._descriptionTruncated === true ||
|
|
811
819
|
(cached.acceptanceCriteriaCount > 0 && !Array.isArray(cached.acceptanceCriteria)) ||
|
|
812
820
|
(cached.referencesCount > 0 && !Array.isArray(cached.references));
|
|
813
821
|
|
package/dashboard.js
CHANGED
|
@@ -4948,6 +4948,51 @@ function restartEngine() {
|
|
|
4948
4948
|
// GET/HEAD/OPTIONS are treated as read-only/preflight and bypass these checks.
|
|
4949
4949
|
const MUTATING_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
|
|
4950
4950
|
|
|
4951
|
+
// W-mq5xg5e9000nec0e — Slim a work-item record for the polled bulk
|
|
4952
|
+
// /api/work-items list endpoint. With ~744 items including done history,
|
|
4953
|
+
// the full shape ballooned to 4.2 MB (three plan-to-prd "meeting follow-up"
|
|
4954
|
+
// items each embedded a ~169 KB transcript inline in `description`); the
|
|
4955
|
+
// dashboard refresh loop re-downloads and re-renders that on every cycle
|
|
4956
|
+
// and the browser tab OOM-crashed. Three transforms:
|
|
4957
|
+
//
|
|
4958
|
+
// * description — hard-cap to WORK_ITEMS_SLIM_DESCRIPTION_CAP chars (with
|
|
4959
|
+
// a "… [truncated; fetch full record at /api/work-items/<id>]" marker)
|
|
4960
|
+
// so a single oversized transcript can never reproduce a multi-hundred-
|
|
4961
|
+
// KB list payload again. When truncation occurs, `_descriptionTruncated:
|
|
4962
|
+
// true` is set so the detail/edit modal client paths
|
|
4963
|
+
// (dashboard/js/render-work-items.js) know to hydrate via
|
|
4964
|
+
// GET /api/work-items/<id> before rendering or pre-filling the edit form.
|
|
4965
|
+
// * acceptanceCriteria → acceptanceCriteriaCount integer (drop array).
|
|
4966
|
+
// * references → referencesCount integer (drop array).
|
|
4967
|
+
//
|
|
4968
|
+
// GET /api/work-items/<id> (handleWorkItemsById) still returns the FULL
|
|
4969
|
+
// record — modal hydration depends on it. Frontend consumers are already
|
|
4970
|
+
// wired to read referencesCount / acceptanceCriteriaCount integers and to
|
|
4971
|
+
// lazy-fetch the full record on click (W-mphejzmj000718bf).
|
|
4972
|
+
//
|
|
4973
|
+
// Exported (as `_slimWorkItemForList`) for direct unit testing — production
|
|
4974
|
+
// callers go through the GET /api/work-items handler's builder closure.
|
|
4975
|
+
const WORK_ITEMS_SLIM_DESCRIPTION_CAP = 2048;
|
|
4976
|
+
const WORK_ITEMS_SLIM_DESCRIPTION_MARKER = '\n\n… [truncated; fetch full record at /api/work-items/<id>]';
|
|
4977
|
+
function slimWorkItemForList(item) {
|
|
4978
|
+
if (!item || typeof item !== 'object') return item;
|
|
4979
|
+
// Shallow copy — never mutate the cached item from queries.getWorkItems().
|
|
4980
|
+
const slim = { ...item };
|
|
4981
|
+
if (typeof slim.description === 'string' && slim.description.length > WORK_ITEMS_SLIM_DESCRIPTION_CAP) {
|
|
4982
|
+
slim.description = slim.description.slice(0, WORK_ITEMS_SLIM_DESCRIPTION_CAP) + WORK_ITEMS_SLIM_DESCRIPTION_MARKER;
|
|
4983
|
+
slim._descriptionTruncated = true;
|
|
4984
|
+
}
|
|
4985
|
+
if (Array.isArray(slim.acceptanceCriteria)) {
|
|
4986
|
+
slim.acceptanceCriteriaCount = slim.acceptanceCriteria.length;
|
|
4987
|
+
delete slim.acceptanceCriteria;
|
|
4988
|
+
}
|
|
4989
|
+
if (Array.isArray(slim.references)) {
|
|
4990
|
+
slim.referencesCount = slim.references.length;
|
|
4991
|
+
delete slim.references;
|
|
4992
|
+
}
|
|
4993
|
+
return slim;
|
|
4994
|
+
}
|
|
4995
|
+
|
|
4951
4996
|
const server = http.createServer(async (req, res) => {
|
|
4952
4997
|
// ── Security headers (applied to every response) ──────────────────────────
|
|
4953
4998
|
// Baseline CSP + clickjacking/mime/referrer protections. The dashboard HTML
|
|
@@ -5435,10 +5480,11 @@ const server = http.createServer(async (req, res) => {
|
|
|
5435
5480
|
}
|
|
5436
5481
|
|
|
5437
5482
|
// GET /api/work-items/<id> — return a single FULL work-item record by id
|
|
5438
|
-
// (W-mphejzmj000718bf). The /api/
|
|
5439
|
-
// shape
|
|
5440
|
-
// to keep the SPA payload <
|
|
5441
|
-
// endpoint on
|
|
5483
|
+
// (W-mphejzmj000718bf). The polled GET /api/work-items list endpoint ships
|
|
5484
|
+
// the slim shape above (description truncated, acceptanceCriteria/references
|
|
5485
|
+
// replaced with count integers) to keep the SPA payload < ~300 KB; the
|
|
5486
|
+
// work-item detail/edit modals call THIS endpoint on demand for the full
|
|
5487
|
+
// record. Always returns the unslimmed record.
|
|
5442
5488
|
async function handleWorkItemsById(req, res, match) {
|
|
5443
5489
|
try {
|
|
5444
5490
|
const id = decodeURIComponent(match[1] || '').trim();
|
|
@@ -11070,7 +11116,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
11070
11116
|
builder: () => getAgents(),
|
|
11071
11117
|
});
|
|
11072
11118
|
}},
|
|
11073
|
-
{ method: 'GET', path: '/api/work-items', desc: 'Fully-enriched work items (per-project files joined + dispatch/PR cross-reference) — fresh on every request', handler: (req, res) => {
|
|
11119
|
+
{ method: 'GET', path: '/api/work-items', desc: 'Fully-enriched work items (per-project files joined + dispatch/PR cross-reference) — fresh on every request; description hard-capped + acceptanceCriteria/references replaced with *Count integers (W-mq5xg5e9000nec0e); detail modal lazy-loads the full record via GET /api/work-items/<id>', handler: (req, res) => {
|
|
11074
11120
|
const config = queries.getConfig();
|
|
11075
11121
|
const projects = config.projects || [];
|
|
11076
11122
|
const inputs = [
|
|
@@ -11086,7 +11132,11 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
11086
11132
|
return serveFreshJson(req, res, {
|
|
11087
11133
|
tag: 'work-items',
|
|
11088
11134
|
inputs,
|
|
11089
|
-
|
|
11135
|
+
// W-mq5xg5e9000nec0e — Slim every item before stringify so the polled
|
|
11136
|
+
// refresh-loop payload stays < ~300 KB even when description fields
|
|
11137
|
+
// include 100+ KB transcripts. slimWorkItemForList shallow-copies so
|
|
11138
|
+
// queries.getWorkItems()'s in-memory cache is never mutated.
|
|
11139
|
+
builder: () => getWorkItems().map(slimWorkItemForList),
|
|
11090
11140
|
});
|
|
11091
11141
|
}},
|
|
11092
11142
|
{ method: 'GET', path: '/api/pull-requests', desc: 'Fully-enriched pull requests (per-project files joined + url backfill + _project stamp)', handler: (req, res) => {
|
|
@@ -11350,10 +11400,12 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
11350
11400
|
// GET /api/work-items/<id> — fetch a single FULL work-item record by id.
|
|
11351
11401
|
// Registered AFTER all static /api/work-items/* routes so the regex never
|
|
11352
11402
|
// shadows them (route matching is sequential, first match wins).
|
|
11353
|
-
// W-mphejzmj000718bf
|
|
11354
|
-
//
|
|
11355
|
-
//
|
|
11356
|
-
|
|
11403
|
+
// W-mphejzmj000718bf / W-mq5xg5e9000nec0e: the bulk GET /api/work-items
|
|
11404
|
+
// list endpoint ships a slimmed shape (description hard-capped to
|
|
11405
|
+
// WORK_ITEMS_SLIM_DESCRIPTION_CAP chars, acceptanceCriteria/references
|
|
11406
|
+
// arrays replaced with *Count integers) to cut payload size; the detail
|
|
11407
|
+
// and edit modals call this endpoint to fetch the full record on demand.
|
|
11408
|
+
{ method: 'GET', path: /^\/api\/work-items\/([^/?]+)$/, template: '/api/work-items/<id>', desc: 'Fetch a single full work-item record by id (description, acceptanceCriteria, references). The bulk /api/work-items list ships a slimmed shape (description truncated + *Count integers); this endpoint backs the detail and edit modals.', handler: handleWorkItemsById },
|
|
11357
11409
|
|
|
11358
11410
|
// Pinned notes
|
|
11359
11411
|
{ method: 'GET', path: '/api/pinned', desc: 'Get pinned notes', handler: async (req, res) => {
|
|
@@ -12337,6 +12389,11 @@ module.exports = {
|
|
|
12337
12389
|
// staleness verdict it stamps on engine.heartbeatStale is the contract under
|
|
12338
12390
|
// test. No production caller imports this; it is a test seam.
|
|
12339
12391
|
_buildStatusFastState,
|
|
12392
|
+
// W-mq5xg5e9000nec0e — exported for direct unit testing of the slim shape
|
|
12393
|
+
// produced by GET /api/work-items. Production callers go through the
|
|
12394
|
+
// route's `builder` closure (getWorkItems().map(slimWorkItemForList)).
|
|
12395
|
+
_slimWorkItemForList: slimWorkItemForList,
|
|
12396
|
+
_WORK_ITEMS_SLIM_DESCRIPTION_CAP: WORK_ITEMS_SLIM_DESCRIPTION_CAP,
|
|
12340
12397
|
};
|
|
12341
12398
|
|
|
12342
12399
|
// Start the HTTP server only when run directly (node dashboard.js).
|
package/docs/README.md
CHANGED
|
@@ -17,7 +17,7 @@ Architecture, design proposals, and lifecycle references for people working on t
|
|
|
17
17
|
- [completion-reports.md](completion-reports.md) — Canonical schema for the per-spawn completion JSON: trust nonce, `failure_class` enum, `noop` semantics, `retryable` / `needs_rerun` shape, and the artifacts array.
|
|
18
18
|
- [constants.md](constants.md) — Cross-cutting status / type / condition constants (`WI_STATUS`, `WORK_TYPE`, `PR_STATUS`, `WATCH_CONDITION`, …) and the no-magic-strings invariant.
|
|
19
19
|
- [constellation-bridge.md](constellation-bridge.md) — Read-only cross-repo bridge: `engine.constellationBridge.enabled` flag, marker-file contract, and the `minions bridge` subcommand for local debugging.
|
|
20
|
-
- [constellation-style-telemetry.md](constellation-style-telemetry.md) — Feasibility study for a local-first usage/analytics layer
|
|
20
|
+
- [constellation-style-telemetry.md](constellation-style-telemetry.md) — Feasibility study (design proposal, not implemented) for a local-first usage/analytics layer modelled on Constellation's telemetry stack — typed append-only event log + retention/rollup discipline + dashboard Usage page. Explains why a 1:1 PostgreSQL/multi-tenant port is the wrong goal for Minions.
|
|
21
21
|
- [cooldown-merge-semantics.md](cooldown-merge-semantics.md) — Scoping deliverable defining merge semantics for `saveCooldowns` (longer-of TTL merge, key-level upserts, gitignored on-disk format).
|
|
22
22
|
- [copilot-cli-schema.md](copilot-cli-schema.md) — Behavior and schema reference for the GitHub Copilot CLI adapter (capability flags, stdin vs `-p`, model discovery, effort levels).
|
|
23
23
|
- [dead-code-audit-retractions.md](dead-code-audit-retractions.md) — Retracted dead-code-audit findings (false positives) that future audits MUST read before re-citing.
|
package/docs/auto-discovery.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Auto-Discovery & Execution Pipeline
|
|
2
2
|
|
|
3
|
-
> Last verified: 2026-06-
|
|
3
|
+
> Last verified: 2026-06-09 against `engine.js` `tickInner()` and `routing.md`.
|
|
4
4
|
|
|
5
5
|
How the minions engine finds work and dispatches agents automatically.
|
|
6
6
|
|
|
@@ -19,6 +19,7 @@ tick()
|
|
|
19
19
|
2.5 runCleanup() Periodic cleanup (every 60 ticks ≈ 10min)
|
|
20
20
|
2.52 sweepKeepProcesses() keep_processes TTL/dead-PID sweep (every 180 ticks)
|
|
21
21
|
2.53 sweepManagedSpawn() managed_spawn TTL/dead-PID/log-rotate sweep (every 180 ticks)
|
|
22
|
+
2.54 pruneWorktreesPeriodic() Periodic worktree GC: in-root + out-of-root git registry sweep (every worktreePruneIntervalTicks ≈ 30 ticks; catches Windows EPERM/EBUSY stragglers and `git worktree list` entries outside worktreeRoot)
|
|
22
23
|
2.55 checkWatches() Persistent watch jobs (every 18 tick-equivalents)
|
|
23
24
|
2.6 pollPrStatus() Poll ADO + GitHub for build, review, merge status (wall-clock cadence from prPollStatusEvery × tickInterval, default ≈ 12min)
|
|
24
25
|
processPendingRebases() Run any rebase work queued from the previous tick
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Branch Derivation
|
|
2
|
+
|
|
3
|
+
How the engine decides which branch a dispatch is going to push to.
|
|
4
|
+
CLAUDE.md → Branch Naming holds the one-line summary; the structured
|
|
5
|
+
vs. loose-extractor rules and the canonical PR-fix incident live here.
|
|
6
|
+
|
|
7
|
+
> Source of truth: `engine/shared.js`
|
|
8
|
+
> (`deriveWorkItemBranchName`, `extractStructuredWorkItemPrRef`,
|
|
9
|
+
> `extractWorkItemPrRef`, `copyWorkItemPrFields`),
|
|
10
|
+
> `engine.js#getWorkItemPrRef` + `getStructuredWorkItemPrRef`,
|
|
11
|
+
> `dashboard.js POST /api/work-items`. Last verified: 2026-06-09.
|
|
12
|
+
|
|
13
|
+
## Engine-side fallback
|
|
14
|
+
|
|
15
|
+
`shared.deriveWorkItemBranchName(item, config)` is the single helper used
|
|
16
|
+
by every engine.js fallback site and `dashboard.js POST /api/work-items`.
|
|
17
|
+
It returns `work/<wi-id>` (sanitized). Agents authoring branches by hand
|
|
18
|
+
follow the long-form `user/<loginname>/<wi-id>-<slug>` convention
|
|
19
|
+
taught in `playbooks/shared-rules.md`. The engine fallback is
|
|
20
|
+
intentionally short because the engine has no operator login context.
|
|
21
|
+
|
|
22
|
+
Three rule overrides that skip the fallback:
|
|
23
|
+
|
|
24
|
+
| Condition | Branch used |
|
|
25
|
+
|-----------|-------------|
|
|
26
|
+
| `item.branch` is set | Use as-is |
|
|
27
|
+
| Item targets an existing PR (see below) | Reuse the PR's source branch |
|
|
28
|
+
| Item is part of a shared-branch plan | Use `feature_branch` from the PRD |
|
|
29
|
+
|
|
30
|
+
## PR-fix exception (issue #2999 / W-mpx6i5kh000ac040)
|
|
31
|
+
|
|
32
|
+
`shared.extractWorkItemPrRef(item)` is the single source of truth for
|
|
33
|
+
"does this WI target an existing PR?" — used by `dashboard.js#getWorkItemPrRef`
|
|
34
|
+
and `engine.js#getWorkItemPrRef`. It walks, in order:
|
|
35
|
+
|
|
36
|
+
1. Structured fields: `targetPr`, `pr_id`, `prUrl`, `prNumber`,
|
|
37
|
+
`pullRequest`, `sourcePr`, `pr`, `prId`.
|
|
38
|
+
2. `references[*].url`.
|
|
39
|
+
3. `meta.pr_followup.parent_pr_url`.
|
|
40
|
+
4. **(loose only)** Regex-scans description and title for PR URLs or
|
|
41
|
+
canonical `github:owner/repo#N` / `ado:org/proj/repo#N` ids.
|
|
42
|
+
|
|
43
|
+
When a ref is detected, `copyWorkItemPrFields` stamps
|
|
44
|
+
`targetPr` / `pr_id` / `prNumber`, `item.branch` is unset, and
|
|
45
|
+
`discoverFromWorkItems` reuses the PR's source branch.
|
|
46
|
+
|
|
47
|
+
## Structured-vs-loose split (W-mq18ec6h000p7b87)
|
|
48
|
+
|
|
49
|
+
The PR-ref extractor has **two** variants — pick the right one for the
|
|
50
|
+
call site:
|
|
51
|
+
|
|
52
|
+
| Helper | What it walks | Used by | Why |
|
|
53
|
+
|--------|---------------|---------|-----|
|
|
54
|
+
| `shared.extractStructuredWorkItemPrRef(item)` | Structured fields + `references[*].url` + `meta.pr_followup.parent_pr_url`. **No** description / title scan. | `engine.js#getStructuredWorkItemPrRef` → `pr_not_found` dispatch gate. | Gating blocks dispatch and MUST require explicit operator intent. A description like "see PR #3015 for context" must NOT trip the gate. |
|
|
55
|
+
| `shared.extractWorkItemPrRef(item)` | Structured walk + last-resort description / title scan. | `engine.js#getWorkItemPrRef` (branch derivation, prompt PR context, `resolveWorkItemPrRecord`); `dashboard.js#getWorkItemPrRef` (POST `/api/work-items` create-time `targetPr` stamping). | Callers downgrade gracefully when no PR record matches; stamp path preserves the operator UX of pasting a PR URL into description prose and getting `targetPr` auto-stamped. |
|
|
56
|
+
|
|
57
|
+
**Rule of thumb: gate uses structured-only; stamp uses loose.**
|
|
58
|
+
Stamping is best-effort and reversible; gating blocks dispatch and should
|
|
59
|
+
require explicit operator intent.
|
|
60
|
+
|
|
61
|
+
## Canonical bad incident — P-c8a1d2e3 (2026-06-05)
|
|
62
|
+
|
|
63
|
+
A refactor WI with no structured PR fields whose description merely
|
|
64
|
+
mentioned PR #3015 / PR #3012 as cross-references got stuck in
|
|
65
|
+
`_pendingReason: 'pr_not_found'` forever because the gate called the
|
|
66
|
+
loose extractor. Fix: split the helpers so the gate calls
|
|
67
|
+
`extractStructuredWorkItemPrRef` and the stamp path calls
|
|
68
|
+
`extractWorkItemPrRef`.
|
|
@@ -16,7 +16,7 @@ the window.
|
|
|
16
16
|
|
|
17
17
|
`mutateCooldowns` already runs through `mutateJsonFileLocked`, which acquires
|
|
18
18
|
an exclusive `withFileLock` for the read-modify-write
|
|
19
|
-
(source: `engine/shared.js
|
|
19
|
+
(source: `engine/shared.js` `mutateCooldowns` ~L1625 and `mutateJsonFileLocked` ~L1542), so the callback
|
|
20
20
|
receives the freshly-read `diskCooldowns` snapshot — but the current code
|
|
21
21
|
throws that snapshot away.
|
|
22
22
|
|
|
@@ -113,8 +113,8 @@ acceptance from P-bfa3b verbatim.
|
|
|
113
113
|
|
|
114
114
|
- `engine/cooldown.js:63-101` — current lost-update site
|
|
115
115
|
- `engine/cooldown.js:38-60` — `loadCooldowns` + `_lastDiskCooldownKeys` baseline
|
|
116
|
-
- `engine/shared.js
|
|
117
|
-
- `engine/shared.js
|
|
118
|
-
lock; `skipWriteIfUnchanged` enabled for cooldowns
|
|
116
|
+
- `engine/shared.js` `mutateCooldowns` (~L1625) — already lock-protected
|
|
117
|
+
- `engine/shared.js` `mutateJsonFileLocked` (~L1542) — reads disk inside the
|
|
118
|
+
lock; `skipWriteIfUnchanged` enabled for cooldowns
|
|
119
119
|
- `prd/bug-fix-plan-from-weekly-audit-2026-05-27.json` — P-bfa3a (this scoping)
|
|
120
120
|
and P-bfa3b (implementation acceptance criteria)
|
|
@@ -30,11 +30,11 @@ Minions persists all runtime state as flat JSON files guarded by file-lock-based
|
|
|
30
30
|
|
|
31
31
|
**Total live state:** ~1.8 MB across 9+ JSON files.
|
|
32
32
|
|
|
33
|
-
(source: `engine/shared.js
|
|
33
|
+
(source: `engine/shared.js` `mutateJsonFileLocked` (~L1542) for locking, `engine/queries.js` for paths, live file sizes from `ls -la engine/*.json`)
|
|
34
34
|
|
|
35
35
|
### 1.2 Concurrency Model
|
|
36
36
|
|
|
37
|
-
All mutations go through `mutateJsonFileLocked()` (source: `engine/shared.js
|
|
37
|
+
All mutations go through `mutateJsonFileLocked()` (source: `engine/shared.js` ~L1542):
|
|
38
38
|
|
|
39
39
|
```
|
|
40
40
|
acquire .lock file (exclusive create via fs.openSync 'wx')
|
|
@@ -46,10 +46,10 @@ release .lock file
|
|
|
46
46
|
```
|
|
47
47
|
|
|
48
48
|
Key properties:
|
|
49
|
-
- **Synchronous blocking** — `withFileLock` spins with `sleepMs(25)` until lock acquired or 5s timeout (source: `engine/shared.js
|
|
49
|
+
- **Synchronous blocking** — `withFileLock` spins with `sleepMs(25)` until lock acquired or 5s timeout (source: `engine/shared.js` `withFileLock` ~L1329)
|
|
50
50
|
- **Whole-file granularity** — updating one field in one work item rewrites all 180 items (370 KB)
|
|
51
51
|
- **Stale lock recovery** — locks older than 5 min (`LOCK_STALE_MS = 300_000`) are force-removed; holders that recorded a `{pid, ts}` payload are kept alive past the threshold while `process.kill(pid, 0)` succeeds, with a hard last-resort cap at 5×LOCK_STALE_MS (source: `engine/shared.js`, P-b7d4e8f2)
|
|
52
|
-
- **Read caching** — only `dispatch.json` has a 2s TTL cache (source: `engine/queries.js
|
|
52
|
+
- **Read caching** — only `dispatch.json` has a 2s TTL cache (source: `engine/queries.js`)
|
|
53
53
|
|
|
54
54
|
### 1.3 Read vs Write Ratio
|
|
55
55
|
|
|
@@ -232,7 +232,7 @@ Stay with files. Fix the two highest-pain issues immediately:
|
|
|
232
232
|
|
|
233
233
|
3. **Add read caches to `work-items.json` and `pull-requests.json`** — Same 2s TTL pattern as dispatch.json (source: `engine/queries.js:82-91`). These are read 8+ times per tick but only written 1-2 times.
|
|
234
234
|
|
|
235
|
-
4. **Convert `log.json` to append-only JSONL** — Eliminates the parse-entire-file-to-append pattern in `_flushLogBuffer()` (source: `engine/shared.js
|
|
235
|
+
4. **Convert `log.json` to append-only JSONL** — Eliminates the parse-entire-file-to-append pattern in `_flushLogBuffer()` (source: `engine/shared.js` `_flushLogBuffer` ~L499). Log rotation becomes `readFile → keep last 2000 lines → writeFile` instead of `parse JSON array → splice → stringify → write`.
|
|
236
236
|
|
|
237
237
|
### Phase 2: `node:sqlite` Migration (When API stabilizes — estimated Node 26 LTS)
|
|
238
238
|
|
package/docs/kb-sweep.md
CHANGED
|
@@ -37,7 +37,7 @@ The remaining survivors are sent to Claude Haiku in batches of `LLM_BATCH_SIZE =
|
|
|
37
37
|
|
|
38
38
|
Each action archives the file via the same `_archiveKbFile()` helper used by Pass 1; reclassification rewrites the `category:` frontmatter line and moves the file into the new category directory (source: [`engine/kb-sweep.js:243-279`](../engine/kb-sweep.js#L243)).
|
|
39
39
|
|
|
40
|
-
Reclassification targets are validated against `shared.KB_CATEGORIES` (`architecture`, `conventions`, `project-notes`, `build-reports`, `reviews` — source: [`engine/shared.js
|
|
40
|
+
Reclassification targets are validated against `shared.KB_CATEGORIES` (`architecture`, `conventions`, `project-notes`, `build-reports`, `reviews` — source: [`engine/shared.js`](../engine/shared.js) `KB_CATEGORIES`); unknown categories are silently dropped.
|
|
41
41
|
|
|
42
42
|
If a batch returns invalid JSON or the runtime is unavailable, that batch is skipped with a warning and the rest of the sweep continues (source: [`engine/kb-sweep.js:139-151`](../engine/kb-sweep.js#L139)).
|
|
43
43
|
|
|
@@ -116,7 +116,7 @@ Memory still wins when present; the disk file is a fallback (source: [`engine/kb
|
|
|
116
116
|
|
|
117
117
|
## Automatic Periodic Sweep (opt-in)
|
|
118
118
|
|
|
119
|
-
The engine tick loop can also auto-spawn the KB sweep without dashboard interaction. Gated by `engine.autoConsolidateMemory` (default `false` — source: [`engine/shared.js
|
|
119
|
+
The engine tick loop can also auto-spawn the KB sweep without dashboard interaction. Gated by `engine.autoConsolidateMemory` (default `false` — source: [`engine/shared.js`](../engine/shared.js) `ENGINE_DEFAULTS.autoConsolidateMemory`):
|
|
120
120
|
|
|
121
121
|
- When `engine.autoConsolidateMemory: true`, every tick the engine consults `shouldAutoSweep()` from [`engine/kb-sweep.js`](../engine/kb-sweep.js) and, when the 4-hour cadence has elapsed since the last completion, calls `spawnSweepRunnerDetached()` to fire-and-forget a fresh `engine/kb-sweep-runner.js` process (source: [`engine.js`](../engine.js) tick step 2.1).
|
|
122
122
|
- The inbox→`notes.md` consolidation runs every tick *regardless* of this flag via `consolidateInbox()`; `autoConsolidateMemory` controls **only** the heavier `knowledge/` sweep.
|
package/docs/managed-spawn.md
CHANGED
|
@@ -203,7 +203,7 @@ Killing a spec from outside Minions (raw `Stop-Process`) leaves a stale row in `
|
|
|
203
203
|
|
|
204
204
|
## Configuration
|
|
205
205
|
|
|
206
|
-
All knobs live under `engine.managedSpawn` in `engine/shared.js
|
|
206
|
+
All knobs live under `engine.managedSpawn` in `engine/shared.js` (`ENGINE_DEFAULTS.managedSpawn`). Override per install via `config.json`:
|
|
207
207
|
|
|
208
208
|
| Key | Default | Notes |
|
|
209
209
|
|---|---|---|
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Timeouts & Liveness
|
|
2
|
+
|
|
3
|
+
What kills (or doesn't kill) a live agent. CLAUDE.md → Timeouts & Liveness
|
|
4
|
+
keeps the core invariant; the spawn-phase watchdog, steering safety nets,
|
|
5
|
+
and stale-orphan detection details live here.
|
|
6
|
+
|
|
7
|
+
> Source of truth: `engine/timeout.js`, `engine/spawn-phase-watchdog.js`,
|
|
8
|
+
> `engine/shared.js` (`ENGINE_DEFAULTS`, `getProcessCpuSeconds`,
|
|
9
|
+
> `killImmediate`), `engine/steering-store.js`. See also:
|
|
10
|
+
> [engine-restart.md](engine-restart.md). Last verified: 2026-06-09.
|
|
11
|
+
|
|
12
|
+
## Core invariant
|
|
13
|
+
|
|
14
|
+
**A live tracked agent is never killed for being silent.** Long builds,
|
|
15
|
+
installs, multi-file edits routinely produce no stdout for many minutes.
|
|
16
|
+
|
|
17
|
+
Only two things kill a live tracked process (`engine/timeout.js`):
|
|
18
|
+
|
|
19
|
+
1. **Hard wall-clock timeout** `engine.agentTimeout` (default 5h from
|
|
20
|
+
`startedAt`; per-fan-out `meta.deadline`).
|
|
21
|
+
2. **Steering kill** — explicit human steering → `killImmediate()` so the
|
|
22
|
+
agent re-spawns with `--resume <session>`.
|
|
23
|
+
|
|
24
|
+
**Don't add output-silence timers for live tracked processes.**
|
|
25
|
+
|
|
26
|
+
## Stale-orphan detection
|
|
27
|
+
|
|
28
|
+
`engine.heartbeatTimeout` (default 5 min) is the **grace window after the
|
|
29
|
+
engine loses the tracked process handle** — not a heartbeat timer.
|
|
30
|
+
|
|
31
|
+
Per-type overrides in `ENGINE_DEFAULTS.heartbeatTimeouts`:
|
|
32
|
+
`implement` / `implement:large` / `fix` / `test` / `verify` → 15 min;
|
|
33
|
+
`plan` → 10 min.
|
|
34
|
+
|
|
35
|
+
Orphan declaration requires four checks (all must pass):
|
|
36
|
+
|
|
37
|
+
1. `isTrackedProcessAlive` returns false.
|
|
38
|
+
2. 64 KB tail scan for `[process-exit] code=N`.
|
|
39
|
+
3. `isOsPidAliveForDispatch` returns false.
|
|
40
|
+
4. Full-log re-scan still shows no completion.
|
|
41
|
+
|
|
42
|
+
After engine restart, gated on `engineRestartGraceUntil` (default
|
|
43
|
+
20 min) — see [engine-restart.md](engine-restart.md).
|
|
44
|
+
|
|
45
|
+
## Steering safety nets (W-mq066js7000fff1f-c)
|
|
46
|
+
|
|
47
|
+
Three knobs backstop the steering pipeline. `engine/timeout.js`
|
|
48
|
+
defensively requires `./steering-store` and swallows `MODULE_NOT_FOUND`
|
|
49
|
+
only, so the gates work even before the store has shipped.
|
|
50
|
+
|
|
51
|
+
### Kill-retry escalation ladder
|
|
52
|
+
|
|
53
|
+
`engine.steeringMaxKillRetries` (default `3`, range `1–5`). After a
|
|
54
|
+
steering kill, if the process hasn't exited within 30 s:
|
|
55
|
+
|
|
56
|
+
1. Retry gracefully at 60 s, 120 s (last interval reused past the cap).
|
|
57
|
+
2. Fire a platform hard kill:
|
|
58
|
+
- **Windows:** `taskkill /F /T /PID <pid>`.
|
|
59
|
+
- **Unix:** descendant-tree SIGKILL — `pgrep -P` deepest-first +
|
|
60
|
+
`pkill -KILL -P <pid>`.
|
|
61
|
+
3. After the cap: `[steering-stuck]` on `live-output.log` +
|
|
62
|
+
`[engine-system]` inbox notice; `_steeringGaveUp = true`.
|
|
63
|
+
|
|
64
|
+
### Deferred-steering safety net
|
|
65
|
+
|
|
66
|
+
`engine.steeringDeferredMaxMs` (default `900000` = 15 min, range
|
|
67
|
+
`60_000–14_400_000`). Per-tick, any deferred message older than this
|
|
68
|
+
without a `sessionId` is **stranded** — `[steering-warn]` +
|
|
69
|
+
`_steeringStranded: true` via `mutateDispatch`; steering store →
|
|
70
|
+
`status='stranded'`. Re-warn is guarded by
|
|
71
|
+
`_deferredSteeringStrandedFiles`.
|
|
72
|
+
|
|
73
|
+
### Stale-session purge
|
|
74
|
+
|
|
75
|
+
`onAgentClose` clears `session.json` on `No conversation found`.
|
|
76
|
+
`dropSteeringForPurgedSession(agentId, sessionId, liveOutputPath)` runs
|
|
77
|
+
BEFORE the unlink, walks `steeringStore.listForAgent(agentId)` dropping
|
|
78
|
+
`{queued, live_kill, deferred, re_spawning}` whose `_steeringSessionId`
|
|
79
|
+
matches (status `dropped`, last_error `session-purged`) + writes:
|
|
80
|
+
|
|
81
|
+
```
|
|
82
|
+
[steering-failed] Session <id> was purged by runtime; message <id>
|
|
83
|
+
dropped, please re-send.
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Spawn-phase watchdog (W-mq0e2dae000a003d)
|
|
87
|
+
|
|
88
|
+
`engine/spawn-phase-watchdog.js#checkSpawnPhaseStalls` runs every tick
|
|
89
|
+
alongside `checkTimeouts` / `checkSteering` and kills children wedged
|
|
90
|
+
in MCP-init. Four gates, **ALL** required:
|
|
91
|
+
|
|
92
|
+
1. Fresh spawn (skips `procInfo.reattached`).
|
|
93
|
+
2. Elapsed since `procInfo.startedAt` ≥ `engine.spawnPhaseGraceMs`
|
|
94
|
+
(default 120000 ms).
|
|
95
|
+
3. The last 16 KB of `live-output.log` contains zero non-startup events:
|
|
96
|
+
- Copilot startup-only set:
|
|
97
|
+
`session.{mcp_server_status_changed, mcp_servers_loaded, skills_loaded, tools_updated, info}`.
|
|
98
|
+
- Claude startup-only set:
|
|
99
|
+
`{type:'system', subtype:'init'|'hook_started'|'hook_response'}`.
|
|
100
|
+
- Non-JSON lines and any other JSON `type` count as real activity.
|
|
101
|
+
4. Per-process CPU seconds (via `shared.getProcessCpuSeconds(pid)` —
|
|
102
|
+
PowerShell on Windows, `/proc/<pid>/stat` on Linux, `ps -o cputime=`
|
|
103
|
+
on macOS) ≤ `engine.spawnPhaseMaxCpuSeconds` (default 5).
|
|
104
|
+
|
|
105
|
+
On fire:
|
|
106
|
+
|
|
107
|
+
- Kill via `shared.killImmediate`.
|
|
108
|
+
- Write a structured `spawn-phase-stall-<id>` inbox note with the
|
|
109
|
+
live-output tail.
|
|
110
|
+
- Complete the dispatch with
|
|
111
|
+
`failureClass: SPAWN_PHASE_STALL` + `agentRetryable: true` so the next
|
|
112
|
+
tick re-spawns the agent with a fresh runtime invocation.
|
|
113
|
+
|
|
114
|
+
`SPAWN_PHASE_STALL` is also in `FORCE_DEMOTE_FAILURE_CLASSES` as
|
|
115
|
+
defense-in-depth — a stray completion report can't paper over a wedge.
|
|
116
|
+
|
|
117
|
+
**CPU sampling fails open:** a null result skips the kill so OS-level
|
|
118
|
+
glitches don't masquerade as wedges.
|
|
119
|
+
|
|
120
|
+
Toggle off via `engine.spawnPhaseWatchdogEnabled: false`.
|
package/docs/watches.md
CHANGED
|
@@ -24,7 +24,7 @@ A watch is a small JSON record persisted to `engine/watches.json`. It binds:
|
|
|
24
24
|
|
|
25
25
|
## Lifecycle (`WATCH_STATUS`)
|
|
26
26
|
|
|
27
|
-
Defined in `engine/shared.js
|
|
27
|
+
Defined in `engine/shared.js` (`WATCH_STATUS`):
|
|
28
28
|
|
|
29
29
|
| Status | Meaning |
|
|
30
30
|
|-------------|-------------------------------------------------------------------------|
|
|
@@ -37,10 +37,10 @@ Pause/resume flips the `status` field via `POST /api/watches/update` *(source: `
|
|
|
37
37
|
|
|
38
38
|
## Conditions (`WATCH_CONDITION`)
|
|
39
39
|
|
|
40
|
-
Defined in `engine/shared.js
|
|
40
|
+
Defined in `engine/shared.js` (`WATCH_CONDITION`). Conditions split into two families:
|
|
41
41
|
|
|
42
42
|
### Absolute conditions (`WATCH_ABSOLUTE_CONDITIONS`)
|
|
43
|
-
*(source: `engine/shared.js
|
|
43
|
+
*(source: `engine/shared.js` `WATCH_ABSOLUTE_CONDITIONS`)*
|
|
44
44
|
|
|
45
45
|
`merged`, `build-fail`, `build-pass`, `completed`, `failed`, `concluded`, `approved`, `rejected`, `ready-for-merge`, `retry-limit-reached`, `all-items-done`, `item-failed-n-times`.
|
|
46
46
|
|
|
@@ -49,12 +49,12 @@ When `stopAfter === 0`, these are **fire-once** — the engine flips the watch t
|
|
|
49
49
|
> **Per-target override (W-mp7hg58e000b5212):** the global `WATCH_ABSOLUTE_CONDITIONS` set is the legacy fallback. Each target type now declares its own `absoluteConditions: [...]` array in its spec; `registerTargetType` normalizes that into a `Set` that takes precedence at evaluation time. The plugin contract (see below) uses this to keep absolute-vs-change semantics local to each target type. Plugins that omit `absoluteConditions` get an empty set (all change-based).
|
|
50
50
|
|
|
51
51
|
### Change-based conditions
|
|
52
|
-
`status-change`, `any`, `new-comments`, `vote-change`, `stage-complete`, `ran`, `enabled`, `disabled`, `activity-change`, plus the predicate conditions added under P-w4e2f6a1 / P-w5b8d2c9 for the `pr`, `work-item`, `plan`, and `pipeline` target types (`head-commit-change`, `mergeable-flipped`, `behind-master`, `draft-flipped`, `stalled`, `dependency-met`, `stage-advanced`, `stuck-in-stage`). See `engine/shared.js
|
|
52
|
+
`status-change`, `any`, `new-comments`, `vote-change`, `stage-complete`, `ran`, `enabled`, `disabled`, `activity-change`, plus the predicate conditions added under P-w4e2f6a1 / P-w5b8d2c9 for the `pr`, `work-item`, `plan`, and `pipeline` target types (`head-commit-change`, `mergeable-flipped`, `behind-master`, `draft-flipped`, `stalled`, `dependency-met`, `stage-advanced`, `stuck-in-stage`). See `engine/shared.js` `WATCH_CONDITION` for the canonical enum.
|
|
53
53
|
|
|
54
54
|
These compare the live entity against the watch's `_lastState` snapshot and run forever when `stopAfter === 0`. Baseline `_lastState` is captured on the first check so the very next change triggers the watch *(source: `engine/watches.js:434, 520`)*.
|
|
55
55
|
|
|
56
56
|
### Tick-counted conditions
|
|
57
|
-
`stalled`, `stuck-in-stage` — require N consecutive unchanged captures (default `WATCH_STALLED_DEFAULT_TICKS = 12`, `WATCH_STUCK_STAGE_DEFAULT_TICKS = 12`, both in `engine/shared.js
|
|
57
|
+
`stalled`, `stuck-in-stage` — require N consecutive unchanged captures (default `WATCH_STALLED_DEFAULT_TICKS = 12`, `WATCH_STUCK_STAGE_DEFAULT_TICKS = 12`, both in `engine/shared.js`). Counters (`_unchangedTicks`, `_stuckStageTicks`) are recomputed inside `_captureState` by comparing the fresh snapshot against `prevState`.
|
|
58
58
|
|
|
59
59
|
### Predicate conditions
|
|
60
60
|
|
|
@@ -65,7 +65,7 @@ Several condition keys evaluate a derived predicate on the captured entity/state
|
|
|
65
65
|
- **plan** — `all-items-done` (`items_done === items_total > 0`), `item-failed-n-times` (any `missing_features[*]._retryCount >= ENGINE_DEFAULTS.maxRetries`).
|
|
66
66
|
- **pipeline** — `stage-advanced` (`current_stage_id` changed within the same `runId`), `stuck-in-stage` (current stage unchanged for `WATCH_STUCK_STAGE_DEFAULT_TICKS` checks, default 12).
|
|
67
67
|
|
|
68
|
-
Compound state-assertion predicates (`ready-for-merge`, `retry-limit-reached`, `all-items-done`, `item-failed-n-times`) live in `WATCH_ABSOLUTE_CONDITIONS` so they fire-once when `stopAfter === 0` — without that they would re-fire every tick while the assertion holds *(source: `engine/shared.js
|
|
68
|
+
Compound state-assertion predicates (`ready-for-merge`, `retry-limit-reached`, `all-items-done`, `item-failed-n-times`) live in `WATCH_ABSOLUTE_CONDITIONS` so they fire-once when `stopAfter === 0` — without that they would re-fire every tick while the assertion holds *(source: `engine/shared.js` `WATCH_ABSOLUTE_CONDITIONS`)*.
|
|
69
69
|
|
|
70
70
|
## Target Types — `TARGET_TYPES` Registry
|
|
71
71
|
|
|
@@ -89,7 +89,7 @@ Canonical example: `watches.d/http.js` (W-mp7i22mu00191b07) — a generic HTTP p
|
|
|
89
89
|
|
|
90
90
|
### Built-in target types
|
|
91
91
|
|
|
92
|
-
The eight built-ins are registered at module load *(source: `engine/watches.js
|
|
92
|
+
The eight built-ins are registered at module load *(source: `engine/watches.js` — the long `registerTargetType(...)` block)*. Constants live in `WATCH_TARGET_TYPE` in `engine/shared.js`.
|
|
93
93
|
|
|
94
94
|
| `targetType` | Target value | Conditions | Notes |
|
|
95
95
|
|---------------|--------------------------------------|----------------------------------------------------------------------------|-------|
|
|
@@ -175,7 +175,7 @@ I/O happens **outside the lock**: notifications via `writeToInbox`, follow-up ac
|
|
|
175
175
|
| `resume-plan` | Set PRD `status=PLAN_STATUS.ACTIVE` and clear `planStale` |
|
|
176
176
|
| `cc-triage` | Invoke Command Center headlessly via the loopback `POST /api/command-center/triage` endpoint with the trigger context (and optional completion-report / live-output artifacts). Wraps the prompt in `<UNTRUSTED-INPUT>`, uses a default 10-min timeout (capped at 1 h), and is isolated from the user CC session |
|
|
177
177
|
|
|
178
|
-
Constants live in `WATCH_ACTION_TYPE` (`engine/shared.js
|
|
178
|
+
Constants live in `WATCH_ACTION_TYPE` (`engine/shared.js`); handlers in `engine/watch-actions.js`.
|
|
179
179
|
|
|
180
180
|
### Templating
|
|
181
181
|
|
|
@@ -246,7 +246,7 @@ Absolute conditions firing under `stopAfter === 0` flip `status` to `expired`; `
|
|
|
246
246
|
|
|
247
247
|
## See Also
|
|
248
248
|
|
|
249
|
-
- `engine/shared.js
|
|
249
|
+
- `engine/shared.js` — `WATCH_STATUS`, `WATCH_TARGET_TYPE`, `WATCH_CONDITION`, `WATCH_ABSOLUTE_CONDITIONS`, `WATCH_ACTION_TYPE` constants
|
|
250
250
|
- `engine/watches.js` — registry, lifecycle, tick integration, `watches.d/` plugin loader
|
|
251
251
|
- `engine/watch-actions.js` — action registry and built-in handlers (including `minions-api`)
|
|
252
252
|
- `watches.d/http.js` — canonical user-extensible target type plugin
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# Worktree Lifecycle
|
|
2
|
+
|
|
3
|
+
Deep-dive for the four engine pieces that own git-worktree state for dispatch:
|
|
4
|
+
the **pool** (recycling), the **live guard** (don't wipe an agent's work),
|
|
5
|
+
the **quarantine path** (dirty/divergent → quarantine dir + retry), and the
|
|
6
|
+
**Windows file-lock retry** (EPERM/EBUSY footgun). CLAUDE.md → Worktree
|
|
7
|
+
Lifecycle keeps the cross-cutting invariants; the detail lives here.
|
|
8
|
+
|
|
9
|
+
> Source of truth: `engine/worktree-pool.js`, `engine/shared.js#removeWorktree`
|
|
10
|
+
> + `_retryFsOp`, `engine.js` (`_quarantineDirtyWorktree`, `_renameWithRetry`,
|
|
11
|
+
> `_killGitDescendantsForWorktree`, `pruneOrphanWorktrees*`, `gcDispatchWorktreeIfOrphan`),
|
|
12
|
+
> `engine/cleanup.js`. Last verified: 2026-06-09.
|
|
13
|
+
|
|
14
|
+
## Live-worktree guard (W-mq5rwwss000f30a7)
|
|
15
|
+
|
|
16
|
+
**Invariant:** every code path that wipes, renames, or recycles a worktree
|
|
17
|
+
MUST call `shared.isWorktreePathLive(path, { db?, excludeDispatchId? })`
|
|
18
|
+
first and skip on `true`. Originally added after W-mq5n1zx5000hcfb5 — an
|
|
19
|
+
agent's worktree was wiped 4× consecutively by the reaper while the
|
|
20
|
+
dispatch was still active.
|
|
21
|
+
|
|
22
|
+
- **Backed by SQL** against `dispatches` (status IN ('pending','active')),
|
|
23
|
+
reading `json_extract(data, '$.worktreePath')` with a `data.meta.worktreePath`
|
|
24
|
+
fallback. The pending → active transition persists `item.worktreePath`
|
|
25
|
+
on the dispatch row so the guard has a path to correlate.
|
|
26
|
+
- **Fails OPEN** (returns `true`) when SQLite is unreachable or the query
|
|
27
|
+
throws — better to leak a worktree than nuke an agent's unpushed work.
|
|
28
|
+
- **Wired sites:** `shared.removeWorktree` (accepts `{ excludeDispatchId }`
|
|
29
|
+
forwarded to the guard); pool-return chain in `engine.js`
|
|
30
|
+
(`excludeDispatchId: id` so the dispatch can clean up its own worktree);
|
|
31
|
+
dispatch-end orphan GC (`gcDispatchWorktreeIfOrphan`); `_quarantineDirtyWorktree`
|
|
32
|
+
(returns `{ skipped: true, quarantinedPath: null }` on skip — callers
|
|
33
|
+
MUST honor `skipped` and not set `quarantined: true`); `engine/cleanup.js`
|
|
34
|
+
orphan-dir sweep.
|
|
35
|
+
- **On skip:** drops a deduped operator note at
|
|
36
|
+
`notes/inbox/engine-worktree-skip-live-<basename>-<date>.md`.
|
|
37
|
+
|
|
38
|
+
## Worktree pool (opt-in)
|
|
39
|
+
|
|
40
|
+
`ENGINE_DEFAULTS.worktreePoolSize > 0` enables `engine/worktree-pool.js` to
|
|
41
|
+
recycle worktree dirs across branches.
|
|
42
|
+
|
|
43
|
+
- **Borrow** in `spawnAgent` only when (a) the new branch doesn't exist on
|
|
44
|
+
origin AND (b) the dispatch is not shared-branch / `useExistingBranch`.
|
|
45
|
+
- **Return** in `onAgentClose` BEFORE `completeDispatch`:
|
|
46
|
+
`git reset --hard HEAD` → `git clean -fd` → `git fetch origin <main>`
|
|
47
|
+
→ `git checkout --detach origin/<main>` → mark IDLE.
|
|
48
|
+
- **State** at `engine/worktree-pool.json`; git ops outside any lock.
|
|
49
|
+
|
|
50
|
+
## Quarantine path (dirty / divergent)
|
|
51
|
+
|
|
52
|
+
When `discoverFromWorkItems` finds a worktree in a `WORKTREE_DIRTY`,
|
|
53
|
+
`WORKTREE_DIVERGENT`, or post-stuck state, `_quarantineDirtyWorktree`
|
|
54
|
+
moves it to `<root>/.quarantined/<basename>-<utc>` and lets the next tick
|
|
55
|
+
recreate a clean worktree. Six layered defenses cover the Windows EBUSY
|
|
56
|
+
race when git status descendants still hold packfile handles
|
|
57
|
+
(W-mq5n1zx5000hcfb5; PR #3156).
|
|
58
|
+
|
|
59
|
+
### Layer 1b — `--no-optional-locks` on every status probe (load-bearing)
|
|
60
|
+
|
|
61
|
+
`_statusPorcelainCmd()` emits `git --no-optional-locks status …`. Skips
|
|
62
|
+
the `.git/index.lock` acquire around the untracked-cache refresh inside
|
|
63
|
+
`status`. Typical probe duration under AV scanning drops from 6–12s to
|
|
64
|
+
<500ms, which removes the timeout that leaks the child in the first
|
|
65
|
+
place. **Most incidents are closed by 1b alone.**
|
|
66
|
+
Toggle: `ENGINE_DEFAULTS.statusProbeUseNoOptionalLocks` (default `true`).
|
|
67
|
+
|
|
68
|
+
### Layer 1a — `_renameWithRetry` with jittered backoff
|
|
69
|
+
|
|
70
|
+
6 attempts × 250 ms base × 2^N exponential + 200 ms random jitter
|
|
71
|
+
(~16 s worst-case). Only retries on `EBUSY|EPERM|EACCES|ENOTEMPTY`;
|
|
72
|
+
rethrows other codes immediately. Toggles:
|
|
73
|
+
`ENGINE_DEFAULTS.quarantineRenameRetryAttempts` (6),
|
|
74
|
+
`quarantineRenameRetryBaseMs` (250).
|
|
75
|
+
|
|
76
|
+
### Layer 2a — `_killGitDescendantsForWorktree` (Windows-only)
|
|
77
|
+
|
|
78
|
+
PowerShell sweep, 2 s timeout. Shell-out built via single-quoted literal
|
|
79
|
+
(`'` escaped as `''`) so worktree-path interpolation is injection-safe.
|
|
80
|
+
POSIX no-op. Best-effort — failure logged, never throws.
|
|
81
|
+
Toggle: `ENGINE_DEFAULTS.statusProbeKillDescendantsWin32` (default `true`).
|
|
82
|
+
|
|
83
|
+
### Layer 2b — `git worktree remove --force` fallback
|
|
84
|
+
|
|
85
|
+
Triggers only when 1a exhausts retries; destroys worktree contents (no
|
|
86
|
+
quarantine dir preserved). Toggle:
|
|
87
|
+
`ENGINE_DEFAULTS.quarantineForceRemoveFallback` (default `true`).
|
|
88
|
+
|
|
89
|
+
### Layer 1c — `FAILURE_CLASS.WORKTREE_QUARANTINE_ENV_BLOCKED`
|
|
90
|
+
|
|
91
|
+
Routed when `cleanResult.quarantineError && !cleanResult.quarantined`.
|
|
92
|
+
Added to `dispatch.js#isRetryableFailureReason` never-retry set so the
|
|
93
|
+
per-agent retry counter doesn't bump (env failure, not the agent's fault).
|
|
94
|
+
The auto-recovery loop in `engine.js#discoverFromWorkItems` recognizes
|
|
95
|
+
the new class via both enum check and regex on legacy `failReason`
|
|
96
|
+
strings, then re-queues under the existing
|
|
97
|
+
`ENGINE_DEFAULTS.quarantineAutoRecoveryMax` (default 2) cap.
|
|
98
|
+
|
|
99
|
+
### Layer 3a/3b — metrics + inbox alert
|
|
100
|
+
|
|
101
|
+
Counters at
|
|
102
|
+
`metrics._engine.worktreeQuarantineOutcomes.{attempts, success, successAfterRetry, fallbackForceRemove, totalFailure}`.
|
|
103
|
+
Total-failure path writes a structured inbox alert with the exact
|
|
104
|
+
PowerShell/POSIX recovery commands and `git worktree prune` instructions.
|
|
105
|
+
|
|
106
|
+
### Auto-recovery cap
|
|
107
|
+
|
|
108
|
+
`discoverFromWorkItems` auto-recovers `WORKTREE_DIRTY`/`WORKTREE_DIVERGENT`
|
|
109
|
+
quarantine (#2996) up to `ENGINE_DEFAULTS.quarantineAutoRecoveryMax`
|
|
110
|
+
(default 2, tracked on `_quarantineRecoveryCount`); after the cap,
|
|
111
|
+
`_quarantineRecoveryGaveUp` triggers a warn-once.
|
|
112
|
+
|
|
113
|
+
## Windows file-lock on removal (footgun #6 detail)
|
|
114
|
+
|
|
115
|
+
`fs.rmSync` against an active git worktree can lose to lingering file
|
|
116
|
+
handles (CC log streams, virus scanners, Explorer previews).
|
|
117
|
+
`shared.removeWorktree` retries via `shared._retryFsOp`
|
|
118
|
+
(`worktreeRemoveRetryAttempts` × exponential `worktreeRemoveRetryBaseMs`,
|
|
119
|
+
codes `EPERM|EBUSY|EACCES|ENOTEMPTY`).
|
|
120
|
+
|
|
121
|
+
After `worktreeStuckThreshold` consecutive failures the path is
|
|
122
|
+
**escalated**:
|
|
123
|
+
|
|
124
|
+
- A `notes/inbox/engine-worktree-stuck-<basename>-<date>.md` note with
|
|
125
|
+
Windows holder-identification hints is written (deduped per UTC day).
|
|
126
|
+
- Per-tick warns are suppressed for `worktreeStuckSuppressMs`.
|
|
127
|
+
- The retry cadence drops to `worktreeStuckSlowRetryMs`.
|
|
128
|
+
- When the holder finally releases, a
|
|
129
|
+
`worktree-recovered-<basename>` note clears the alert.
|
|
130
|
+
|
|
131
|
+
**Don't fight this with `fs.rmSync({force:true})` outside
|
|
132
|
+
`removeWorktree`** — you'll bypass the retry, escalation, and metrics
|
|
133
|
+
layers.
|
|
134
|
+
|
|
135
|
+
## Periodic prune (W-mq5o6bvy000x7191)
|
|
136
|
+
|
|
137
|
+
`pruneWorktreesPeriodic` runs `pruneOrphanWorktrees` (in-root) +
|
|
138
|
+
`pruneOrphanWorktreesFromGitRegistry` (out-of-root via
|
|
139
|
+
`git worktree list --porcelain`) per project at
|
|
140
|
+
`ENGINE_DEFAULTS.worktreePruneIntervalTicks` cadence — catches Windows
|
|
141
|
+
EPERM/EBUSY stragglers that the dispatch-end GC couldn't reap and sweeps
|
|
142
|
+
the `git worktree list` registry for OUT-of-root entries the in-root
|
|
143
|
+
scanner is blind to.
|
|
144
|
+
|
|
145
|
+
## Holder identification + opt-in auto-reap (W-mq6f2fe0000557fa)
|
|
146
|
+
|
|
147
|
+
Orphan-sweep escalations also run `shared.findProcessesWithCwdInside(wt)`
|
|
148
|
+
(cross-platform: PowerShell `Get-CimInstance Win32_Process` on Windows;
|
|
149
|
+
`/proc/*/cwd` walk on Linux; `lsof -d cwd` + `ps` on macOS) and append a
|
|
150
|
+
`## Live holders` section listing pid / cmdline / age to the
|
|
151
|
+
`engine-worktree-stuck-<basename>` escalation note.
|
|
152
|
+
|
|
153
|
+
Setting `engine.autoReapOrphanWorktreeHolders: true`
|
|
154
|
+
(Settings → Worker Pool & Worktrees → "Auto-reap orphan worktree holders")
|
|
155
|
+
additionally kills any holder whose `cmdline` matches `spawn-agent.js`
|
|
156
|
+
AND references the worktree basename AND whose age exceeds
|
|
157
|
+
`engine.agentTimeout * 2`, then retries `removeWorktree` once. Default
|
|
158
|
+
**OFF** — killing a foreign process is destructive.
|
|
159
|
+
|
|
160
|
+
- Scan timeout: `engine.orphanHolderScanTimeoutMs` (default 5000ms,
|
|
161
|
+
clamped 1000–30000).
|
|
162
|
+
- Motivating incident: W-mq1k8z6o003acd89 / stuck `spawn-agent.js`
|
|
163
|
+
PID 16828 that the periodic prune couldn't shake loose without manual
|
|
164
|
+
intervention.
|
package/engine/cli.js
CHANGED
|
@@ -163,8 +163,9 @@ function handleCommand(cmd, args) {
|
|
|
163
163
|
//
|
|
164
164
|
// `minions work --help` used to create ghost work items with title='--help'
|
|
165
165
|
// because the bare-string `title` was truthy and bypassed the `!title`
|
|
166
|
-
// usage check.
|
|
167
|
-
//
|
|
166
|
+
// usage check. The fix lives in per-command guards (`_isHelpArg` /
|
|
167
|
+
// `looksLikeFlagOrHelp`) on `work`/`spawn`/`plan`/`complete`, which print
|
|
168
|
+
// command-specific `Usage:` output. `pr` and `bridge` handle help inline.
|
|
168
169
|
//
|
|
169
170
|
// Intercept here so a single guard covers the whole command set. `pr` and
|
|
170
171
|
// `bridge` already handle `help`/`--help`/`-h` inline (see their own
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yemi33/minions",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2150",
|
|
4
4
|
"description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
|
|
5
5
|
"bin": {
|
|
6
6
|
"minions": "bin/minions.js"
|