brl-subagent 2.3.9 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT.md +5 -0
- package/README.md +20 -8
- package/package.json +11 -9
- package/src/concurrency.ts +1 -0
- package/src/diff.ts +1 -0
- package/src/event-bus.ts +8 -2
- package/src/git.ts +68 -0
- package/src/history.ts +59 -1
- package/src/index.ts +244 -406
- package/src/kill-escalation.ts +94 -0
- package/src/logging.ts +58 -4
- package/src/messaging.ts +1 -0
- package/src/metrics.ts +1 -0
- package/src/model-availability.ts +1 -0
- package/src/notify-completion.ts +1 -0
- package/src/params.ts +1 -0
- package/src/paths.ts +1 -0
- package/src/preflight.ts +30 -14
- package/src/prelude.ts +1 -0
- package/src/presets.ts +1 -0
- package/src/proc.ts +140 -0
- package/src/prompt.ts +1 -0
- package/src/recovery-engine.ts +262 -0
- package/src/recovery.ts +325 -0
- package/src/reports.ts +64 -8
- package/src/router.ts +1 -0
- package/src/run-registry.ts +269 -0
- package/src/runner.ts +207 -16
- package/src/sanitize.ts +54 -3
- package/src/scheduler.ts +1 -0
- package/src/schema.ts +254 -0
- package/src/session-manager.ts +142 -111
- package/src/state.ts +52 -7
- package/src/templates.ts +1 -0
- package/src/transcript-path.ts +41 -0
- package/src/transcript-tail.ts +1 -0
- package/src/transcript.ts +14 -12
- package/src/tui-format.ts +1 -0
- package/src/tui.ts +231 -296
- package/src/types.ts +130 -2
- package/src/unit-run.ts +9 -0
- package/src/validate.ts +16 -2
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
// Purpose: Pure recovery decision engine — identity, owner liveness classification, and the boot-scan plan over injected process deps.
|
|
2
|
+
/**
|
|
3
|
+
* Option B U1 — recovery decision engine (pure over `RecoveryDeps`).
|
|
4
|
+
*
|
|
5
|
+
* The engine decides, for one persisted `running` record, whether its
|
|
6
|
+
* conductor is still live, whether to reject ownership, and whether to reap a
|
|
7
|
+
* marker-verified orphan child. It performs NO process work itself: every
|
|
8
|
+
* effect goes through the injected `RecoveryDeps`, so the full matrix is
|
|
9
|
+
* unit-testable without spawning processes. The boot orchestration lives in
|
|
10
|
+
* `src/recovery.ts`; the production `/proc` wiring in `src/proc.ts`.
|
|
11
|
+
*
|
|
12
|
+
* Decision order (`decideRecovery`):
|
|
13
|
+
* - status not "running" → skip
|
|
14
|
+
* - `interruptedAt` already set → never re-mark, but re-check for a leftover
|
|
15
|
+
* live marker child and reap it again (B2 revisit)
|
|
16
|
+
* - absent owner → skip (`no-owner`)
|
|
17
|
+
* - owner present but unparsable → mark-only (`owner-unverifiable`), no kill
|
|
18
|
+
* - owner alive (pid + start token match) → leave untouched
|
|
19
|
+
* - owner alive but token unreadable → skip (`ownership-unverifiable`)
|
|
20
|
+
* - owner dead → reap marker-verified orphan children, then mark
|
|
21
|
+
*
|
|
22
|
+
* Reaping is the ONE `escalateKill` sequence (src/kill-escalation.ts): SIGTERM
|
|
23
|
+
* every pid, wait one grace window, SIGKILL any survivor.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import type { ProcessOwner, SubagentRun } from "./types";
|
|
27
|
+
import { resolveTerminalRunEntry } from "./state";
|
|
28
|
+
import { escalateKill, type EscalationTarget } from "./kill-escalation";
|
|
29
|
+
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
// Dependency injection surface
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
|
|
34
|
+
/** The process primitives the decision engine needs (injected for testability). */
|
|
35
|
+
export interface RecoveryDeps {
|
|
36
|
+
/** True when the pid exists (SIGCONT/`kill -0` semantics; EPERM counts as alive). */
|
|
37
|
+
pidAlive(pid: number): boolean;
|
|
38
|
+
/** The process start-time token, or undefined when /proc is unavailable/unreadable. */
|
|
39
|
+
startTokenOf(pid: number): string | undefined;
|
|
40
|
+
/** Pids whose environment carries `CHILD_MARKER_ENV_KEY=<marker>`; [] when /proc is unavailable. */
|
|
41
|
+
findByMarker(marker: string): number[];
|
|
42
|
+
/** Send a signal to a pid (may throw when the pid is already gone). */
|
|
43
|
+
kill(pid: number, signal: NodeJS.Signals): void;
|
|
44
|
+
/** Sleep between SIGTERM and SIGKILL. */
|
|
45
|
+
sleep(ms: number): Promise<void>;
|
|
46
|
+
/** Grace period in ms between SIGTERM and SIGKILL. */
|
|
47
|
+
graceMs: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Which persisted store a candidate record came from. */
|
|
51
|
+
export type RecoveryKind = "run" | "agent";
|
|
52
|
+
|
|
53
|
+
/** A persisted `running` record projected into the shape the engine reasons over. */
|
|
54
|
+
export interface RecoveryRecord {
|
|
55
|
+
id: string;
|
|
56
|
+
kind: RecoveryKind;
|
|
57
|
+
status: string;
|
|
58
|
+
interruptedAt?: string;
|
|
59
|
+
owner?: ProcessOwner;
|
|
60
|
+
childMarker?: string;
|
|
61
|
+
/** Original record for the caller's write-back; opaque to this module. */
|
|
62
|
+
source?: unknown;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export type RecoverySkipReason =
|
|
66
|
+
| "not-running"
|
|
67
|
+
| "already-interrupted"
|
|
68
|
+
| "no-owner"
|
|
69
|
+
| "owner-alive"
|
|
70
|
+
| "ownership-unverifiable";
|
|
71
|
+
|
|
72
|
+
export type RecoveryMarkReason = "owner-dead" | "child-orphaned" | "owner-unverifiable";
|
|
73
|
+
|
|
74
|
+
/** What the engine decided to do with one record (before any process work). */
|
|
75
|
+
export type RecoveryPlan =
|
|
76
|
+
| { decision: "skip"; reason: RecoverySkipReason }
|
|
77
|
+
| { decision: "mark"; reason: RecoveryMarkReason; reap: number[] }
|
|
78
|
+
/** Already-marked record re-checked this boot: reap leftovers, never re-mark. */
|
|
79
|
+
| { decision: "reap"; reason: "revisit-interrupted"; reap: number[] };
|
|
80
|
+
|
|
81
|
+
/** The engine's verdict on an owner identity. */
|
|
82
|
+
export type OwnerState = "alive" | "dead" | "unknown";
|
|
83
|
+
|
|
84
|
+
/** The result of applying a plan (kills performed, survivors logged). */
|
|
85
|
+
export interface RecoveryOutcome {
|
|
86
|
+
id: string;
|
|
87
|
+
kind: RecoveryKind;
|
|
88
|
+
decision: "skip" | "mark" | "reap";
|
|
89
|
+
reason: string;
|
|
90
|
+
reaped: number[];
|
|
91
|
+
survived: number[];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// Candidate projection
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Durability identity for a newly dispatched run record. A retry inherits the
|
|
100
|
+
* original's dispatchId and records `resumeOf`/`attempt`; a fresh dispatch
|
|
101
|
+
* mints a new dispatchId with attempt 1. These are RECORD fields, not
|
|
102
|
+
* delegate_task params (D4).
|
|
103
|
+
*/
|
|
104
|
+
export function newDispatchIdentity(source: SubagentRun | undefined): {
|
|
105
|
+
dispatchId: string;
|
|
106
|
+
resumeOf?: string;
|
|
107
|
+
attempt: number;
|
|
108
|
+
} {
|
|
109
|
+
return {
|
|
110
|
+
dispatchId: source?.dispatchId ?? crypto.randomUUID(),
|
|
111
|
+
resumeOf: source?.id,
|
|
112
|
+
attempt: source ? (source.attempt ?? 1) + 1 : 1,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Collapse run entries to ONE per id before the scan, using the ONE shared
|
|
118
|
+
* preference rule (`resolveTerminalRunEntry`, src/state.ts): terminal-first;
|
|
119
|
+
* among non-terminal entries the D3 `interruptedAt` mark wins over the
|
|
120
|
+
* original spawn entry; otherwise the first entry seen. Ids keep their
|
|
121
|
+
* first-seen order. Delegating keeps the recovery scan and U3's display/retry
|
|
122
|
+
* lookup from diverging (the historical duplicate rule picked the marked clone
|
|
123
|
+
* while the resolver picked the unmarked spawn).
|
|
124
|
+
*/
|
|
125
|
+
export function dedupeRunEntriesById(runs: SubagentRun[]): SubagentRun[] {
|
|
126
|
+
const ids: string[] = [];
|
|
127
|
+
const seen = new Set<string>();
|
|
128
|
+
for (const run of runs) {
|
|
129
|
+
if (seen.has(run.id)) continue;
|
|
130
|
+
seen.add(run.id);
|
|
131
|
+
ids.push(run.id);
|
|
132
|
+
}
|
|
133
|
+
const collapsed: SubagentRun[] = [];
|
|
134
|
+
for (const id of ids) {
|
|
135
|
+
const resolved = resolveTerminalRunEntry(runs, id);
|
|
136
|
+
if (resolved) collapsed.push(resolved);
|
|
137
|
+
}
|
|
138
|
+
return collapsed;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
// The decision engine (pure over RecoveryDeps)
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Classify the recorded owner's liveness. `unknown` means the pid is alive but
|
|
147
|
+
* the start token could not be read, so pid reuse cannot be excluded — the
|
|
148
|
+
* caller must not act on it.
|
|
149
|
+
*/
|
|
150
|
+
export function classifyOwner(owner: ProcessOwner, deps: RecoveryDeps): OwnerState {
|
|
151
|
+
if (!deps.pidAlive(owner.pid)) return "dead";
|
|
152
|
+
// An empty recorded token means the WRITER could not read one (non-/proc
|
|
153
|
+
// platform). Even on a host that can read tokens now, the record's identity
|
|
154
|
+
// is unverifiable — never act on it.
|
|
155
|
+
if (!owner.start) return "unknown";
|
|
156
|
+
const token = deps.startTokenOf(owner.pid);
|
|
157
|
+
if (token === undefined) return "unknown";
|
|
158
|
+
return token === owner.start ? "alive" : "dead";
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* True when an owner carries a usable pid (positive integer). Anything else —
|
|
163
|
+
* a missing pid, a string, 0, a negative, NaN, a fraction — is UNPARSABLE
|
|
164
|
+
* ownership: the record is marked but the kill path is never touched.
|
|
165
|
+
*/
|
|
166
|
+
function hasParsableOwnerPid(owner: ProcessOwner): boolean {
|
|
167
|
+
return Number.isInteger(owner.pid) && owner.pid > 0;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Decide what to do with one persisted record. Pure — no process is touched.
|
|
172
|
+
* Reaping pids come ONLY from a marker scan: without a verified marker match
|
|
173
|
+
* the engine never proposes a kill. An unparsable owner is mark-only (see the
|
|
174
|
+
* module header's three-way owner policy): interrupt is certain, ownership is
|
|
175
|
+
* not, so `reap` is empty and the kill path stays untouched.
|
|
176
|
+
*/
|
|
177
|
+
export function decideRecovery(record: RecoveryRecord, deps: RecoveryDeps): RecoveryPlan {
|
|
178
|
+
if (record.status !== "running") return { decision: "skip", reason: "not-running" };
|
|
179
|
+
if (record.interruptedAt) {
|
|
180
|
+
// Already marked: NEVER re-mark (idempotent), but a SIGKILL survivor or a
|
|
181
|
+
// child whose marker was unreadable at mark time must stay revisitable.
|
|
182
|
+
// The registry entry is retained, so every later boot re-checks it.
|
|
183
|
+
const reap = record.childMarker ? deps.findByMarker(record.childMarker) : [];
|
|
184
|
+
return reap.length > 0
|
|
185
|
+
? { decision: "reap", reason: "revisit-interrupted", reap }
|
|
186
|
+
: { decision: "skip", reason: "already-interrupted" };
|
|
187
|
+
}
|
|
188
|
+
if (!record.owner) return { decision: "skip", reason: "no-owner" };
|
|
189
|
+
if (!hasParsableOwnerPid(record.owner)) {
|
|
190
|
+
return { decision: "mark", reason: "owner-unverifiable", reap: [] };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const ownerState = classifyOwner(record.owner, deps);
|
|
194
|
+
if (ownerState === "alive") return { decision: "skip", reason: "owner-alive" };
|
|
195
|
+
if (ownerState === "unknown") return { decision: "skip", reason: "ownership-unverifiable" };
|
|
196
|
+
|
|
197
|
+
const reap = record.childMarker ? deps.findByMarker(record.childMarker) : [];
|
|
198
|
+
return {
|
|
199
|
+
decision: "mark",
|
|
200
|
+
reason: reap.length > 0 ? "child-orphaned" : "owner-dead",
|
|
201
|
+
reap,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Escalate a set of pids together: SIGTERM every one, wait ONE grace window,
|
|
207
|
+
* then SIGKILL any survivor. Returns the pids still alive afterwards. Thin
|
|
208
|
+
* adapter over the shared `escalateKill` helper (src/kill-escalation.ts), which
|
|
209
|
+
* `recoverRecord` (one record) and `runBootScan` (the whole scan) both call, so
|
|
210
|
+
* a boot with N orphans waits one grace period total, never N.
|
|
211
|
+
*/
|
|
212
|
+
export async function reapPids(pids: number[], deps: RecoveryDeps): Promise<number[]> {
|
|
213
|
+
const unique = [...new Set(pids)];
|
|
214
|
+
const targets: EscalationTarget[] = unique.map((pid) => ({
|
|
215
|
+
terminate: () => {
|
|
216
|
+
deps.kill(pid, "SIGTERM");
|
|
217
|
+
},
|
|
218
|
+
forceKill: () => {
|
|
219
|
+
deps.kill(pid, "SIGKILL");
|
|
220
|
+
},
|
|
221
|
+
verifyDeath: () => !deps.pidAlive(pid),
|
|
222
|
+
}));
|
|
223
|
+
const survivors = await escalateKill(targets, {
|
|
224
|
+
graceMs: deps.graceMs,
|
|
225
|
+
sleep: deps.sleep,
|
|
226
|
+
});
|
|
227
|
+
const survivedSet = new Set(survivors);
|
|
228
|
+
return unique.filter((_, index) => survivedSet.has(targets[index]));
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Apply a record's plan: SIGTERM every marker-verified orphan, wait the grace
|
|
233
|
+
* period, SIGKILL any survivor, then report. Never throws on a vanished pid.
|
|
234
|
+
*/
|
|
235
|
+
export async function recoverRecord(
|
|
236
|
+
record: RecoveryRecord,
|
|
237
|
+
deps: RecoveryDeps,
|
|
238
|
+
): Promise<RecoveryOutcome> {
|
|
239
|
+
const plan = decideRecovery(record, deps);
|
|
240
|
+
if (plan.decision === "skip") {
|
|
241
|
+
return {
|
|
242
|
+
id: record.id,
|
|
243
|
+
kind: record.kind,
|
|
244
|
+
decision: "skip",
|
|
245
|
+
reason: plan.reason,
|
|
246
|
+
reaped: [],
|
|
247
|
+
survived: [],
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const reaped = [...new Set(plan.reap)];
|
|
252
|
+
const survived = await reapPids(reaped, deps);
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
id: record.id,
|
|
256
|
+
kind: record.kind,
|
|
257
|
+
decision: plan.decision,
|
|
258
|
+
reason: plan.reason,
|
|
259
|
+
reaped,
|
|
260
|
+
survived,
|
|
261
|
+
};
|
|
262
|
+
}
|
package/src/recovery.ts
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
// Purpose: Boot-time recovery scan — decides whether a persisted running record is live, reaps orphaned children, and marks interrupted records.
|
|
2
|
+
/**
|
|
3
|
+
* Option B U1 — recovery boot orchestration.
|
|
4
|
+
*
|
|
5
|
+
* A run record is written BEFORE its effect (the spawn), so a crashed
|
|
6
|
+
* conductor leaves `running` records and — on the foreground path — an
|
|
7
|
+
* orphaned child process. This module runs the boot-time scan that decides, in
|
|
8
|
+
* a FRESH process, which of those records are genuinely live:
|
|
9
|
+
*
|
|
10
|
+
* - `interruptedAt` already set → never re-mark, but re-check for a leftover
|
|
11
|
+
* live marker child and reap it again (B2 revisit)
|
|
12
|
+
* - owner alive (pid + start token match) → leave untouched
|
|
13
|
+
* - owner dead → reap a marker-verified orphan child (SIGTERM → grace →
|
|
14
|
+
* SIGKILL), then mark `interruptedAt` on the record
|
|
15
|
+
*
|
|
16
|
+
* The decision logic itself is the pure engine in `src/recovery-engine.ts`
|
|
17
|
+
* (type/external surface unchanged); this module owns candidate sourcing and
|
|
18
|
+
* the scan loop. `recoverInflightRuns` sources candidates from the durable
|
|
19
|
+
* registry (`src/run-registry.ts`) — NOT from the current session's run
|
|
20
|
+
* entries, which a fresh process cannot see after a crash — and
|
|
21
|
+
* `recoverProduction` is the entry point that also enforces the platform guard.
|
|
22
|
+
*
|
|
23
|
+
* Reaping waits ONE grace window for the whole scan (`runBootScan` collects
|
|
24
|
+
* every orphan pid and escalates them together), so N orphans cost one grace
|
|
25
|
+
* period, not N.
|
|
26
|
+
*
|
|
27
|
+
* PLATFORM BOUNDARY (required): the `/proc` reads live in `src/proc.ts`.
|
|
28
|
+
* Where `/proc` is unavailable, `recoverProduction` SKIPS the whole scan with
|
|
29
|
+
* one log line — `process.kill(pid, 0)` liveness alone cannot rule out pid
|
|
30
|
+
* reuse, so we NEVER kill an unverified process and NEVER mark a record when
|
|
31
|
+
* ownership cannot be verified.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { listInflightRuns, type InflightRun } from "./run-registry";
|
|
35
|
+
import { assertSafeAgentId } from "./sanitize";
|
|
36
|
+
import { isProcAvailable, defaultRecoveryDeps } from "./proc";
|
|
37
|
+
import { decideRecovery, reapPids, type RecoveryDeps, type RecoveryRecord } from "./recovery-engine";
|
|
38
|
+
import type { Logger } from "./logging";
|
|
39
|
+
|
|
40
|
+
// Re-export the pure engine and the production wiring so existing importers
|
|
41
|
+
// (`index.ts`, the recovery/registry suites) keep their single entry point.
|
|
42
|
+
export * from "./recovery-engine";
|
|
43
|
+
export * from "./proc";
|
|
44
|
+
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
// Boot scan orchestration
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
|
|
49
|
+
/** Aggregate counts from one boot scan (also the shape the log line carries). */
|
|
50
|
+
export interface BootScanSummary {
|
|
51
|
+
scanned: number;
|
|
52
|
+
skipped: number;
|
|
53
|
+
/** Records whose interruption mark was DURABLY written (issue #304). */
|
|
54
|
+
marked: number;
|
|
55
|
+
/** Records whose mark the engine decided but whose durable write failed (#304). */
|
|
56
|
+
markFailures: number;
|
|
57
|
+
reaped: number;
|
|
58
|
+
/** Already-marked records that had leftover children reaped again this boot. */
|
|
59
|
+
revisited: number;
|
|
60
|
+
skippedUnverifiable: number;
|
|
61
|
+
skippedNoOwner: number;
|
|
62
|
+
/** Registry entries skipped fail-closed because their id failed the write guard. */
|
|
63
|
+
skippedUnsafeIds: number;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The all-zero summary (a skipped scan / an empty registry). */
|
|
67
|
+
export function emptyBootScanSummary(): BootScanSummary {
|
|
68
|
+
return {
|
|
69
|
+
scanned: 0,
|
|
70
|
+
skipped: 0,
|
|
71
|
+
marked: 0,
|
|
72
|
+
markFailures: 0,
|
|
73
|
+
reaped: 0,
|
|
74
|
+
revisited: 0,
|
|
75
|
+
skippedUnverifiable: 0,
|
|
76
|
+
skippedNoOwner: 0,
|
|
77
|
+
skippedUnsafeIds: 0,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface BootScanOptions {
|
|
82
|
+
records: RecoveryRecord[];
|
|
83
|
+
deps: RecoveryDeps;
|
|
84
|
+
/**
|
|
85
|
+
* Persist the D3 mark for this record (registry entry; agent record for
|
|
86
|
+
* background). Returns `false` when the durable write did NOT land — the scan
|
|
87
|
+
* then counts a `markFailure` instead of claiming `marked` (issue #304). A
|
|
88
|
+
* `void` return keeps the historical assume-success contract.
|
|
89
|
+
*/
|
|
90
|
+
mark(record: RecoveryRecord, interruptedAt: string): boolean | void;
|
|
91
|
+
now?: () => Date;
|
|
92
|
+
log?: Pick<Logger, "debug" | "info" | "warn">;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The warning context shared by the two mark outcomes (issue #306 nit 3): the
|
|
97
|
+
* record identity, why it was marked, which orphan pids were reaped, and which
|
|
98
|
+
* of those survived the escalation. One construction so the "marked" and
|
|
99
|
+
* "mark failed" warnings cannot drift apart.
|
|
100
|
+
*/
|
|
101
|
+
function markWarningPayload(
|
|
102
|
+
record: RecoveryRecord,
|
|
103
|
+
reason: string,
|
|
104
|
+
reaped: number[],
|
|
105
|
+
survived: ReadonlySet<number>,
|
|
106
|
+
): { id: string; kind: RecoveryRecord["kind"]; reason: string; reaped: number[]; survived: number[] } {
|
|
107
|
+
return {
|
|
108
|
+
id: record.id,
|
|
109
|
+
kind: record.kind,
|
|
110
|
+
reason,
|
|
111
|
+
reaped,
|
|
112
|
+
survived: reaped.filter((pid) => survived.has(pid)),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Run the decision engine over every candidate, reap markers, and mark the
|
|
118
|
+
* records whose conductor is dead. Every decision is logged; the UI is
|
|
119
|
+
* deliberately silent (the notice/offer surface lands in U3).
|
|
120
|
+
*
|
|
121
|
+
* Idempotent: a marked record has `interruptedAt` set in memory before the
|
|
122
|
+
* next iteration, so a second scan over the same array is a no-op.
|
|
123
|
+
*
|
|
124
|
+
* Wait-once: every orphan pid across every record is reaped in ONE
|
|
125
|
+
* SIGTERM → grace → SIGKILL window (the `reapActiveChildren` shape), so a boot
|
|
126
|
+
* with N orphans is not an N × graceMs stall.
|
|
127
|
+
*/
|
|
128
|
+
export async function runBootScan(options: BootScanOptions): Promise<BootScanSummary> {
|
|
129
|
+
const { records, deps, mark } = options;
|
|
130
|
+
const nowIso = (options.now?.() ?? new Date()).toISOString();
|
|
131
|
+
const summary = emptyBootScanSummary();
|
|
132
|
+
summary.scanned = records.length;
|
|
133
|
+
|
|
134
|
+
// Plan every record first (pure), then reap ALL proposed pids together —
|
|
135
|
+
// including leftovers of already-marked records (never re-marked).
|
|
136
|
+
const planned = records.map((record) => ({ record, plan: decideRecovery(record, deps) }));
|
|
137
|
+
const allReaps = planned.flatMap(({ plan }) =>
|
|
138
|
+
plan.decision === "mark" || plan.decision === "reap" ? plan.reap : [],
|
|
139
|
+
);
|
|
140
|
+
const survived = new Set(await reapPids(allReaps, deps));
|
|
141
|
+
|
|
142
|
+
for (const { record, plan } of planned) {
|
|
143
|
+
if (plan.decision === "mark") {
|
|
144
|
+
const durable = mark(record, nowIso) !== false;
|
|
145
|
+
// In-memory mark first: guarantees the scan is idempotent even when
|
|
146
|
+
// the caller's write-back is append-only (run entries) or deferred.
|
|
147
|
+
record.interruptedAt = nowIso;
|
|
148
|
+
const reaped = [...new Set(plan.reap)];
|
|
149
|
+
summary.reaped += reaped.length;
|
|
150
|
+
const markWarning = markWarningPayload(record, plan.reason, reaped, survived);
|
|
151
|
+
if (durable) {
|
|
152
|
+
summary.marked++;
|
|
153
|
+
options.log?.warn("Recovery: marked interrupted record", markWarning);
|
|
154
|
+
} else {
|
|
155
|
+
// Issue #304: the mark did NOT land. Count the failure and warn with
|
|
156
|
+
// the id — this must be visible even when nothing was marked, so it
|
|
157
|
+
// does not hide behind the caller's `summary.marked > 0` gate.
|
|
158
|
+
summary.markFailures++;
|
|
159
|
+
options.log?.warn("Recovery: registry mark failed", markWarning);
|
|
160
|
+
}
|
|
161
|
+
} else if (plan.decision === "reap") {
|
|
162
|
+
// B2 revisit: the record was already marked (so it is never re-marked),
|
|
163
|
+
// but it still carried a live marker child — reap it again this boot.
|
|
164
|
+
const reaped = [...new Set(plan.reap)];
|
|
165
|
+
summary.revisited++;
|
|
166
|
+
summary.reaped += reaped.length;
|
|
167
|
+
options.log?.warn("Recovery: reaped a leftover child of an interrupted record", {
|
|
168
|
+
id: record.id,
|
|
169
|
+
kind: record.kind,
|
|
170
|
+
reaped,
|
|
171
|
+
survived: reaped.filter((pid) => survived.has(pid)),
|
|
172
|
+
});
|
|
173
|
+
} else {
|
|
174
|
+
summary.skipped++;
|
|
175
|
+
if (plan.reason === "ownership-unverifiable") summary.skippedUnverifiable++;
|
|
176
|
+
if (plan.reason === "no-owner") summary.skippedNoOwner++;
|
|
177
|
+
options.log?.debug("Recovery: left record untouched", {
|
|
178
|
+
id: record.id,
|
|
179
|
+
kind: record.kind,
|
|
180
|
+
reason: plan.reason,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return summary;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// ---------------------------------------------------------------------------
|
|
189
|
+
// Registry-sourced scan (the production candidate source)
|
|
190
|
+
// ---------------------------------------------------------------------------
|
|
191
|
+
|
|
192
|
+
/** The subset of a persisted background agent record the scan reasons over. */
|
|
193
|
+
export interface PersistedAgentLike {
|
|
194
|
+
status: string;
|
|
195
|
+
interruptedAt?: string;
|
|
196
|
+
owner?: RecoveryRecord["owner"];
|
|
197
|
+
childMarker?: string;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export interface RegistryScanOptions {
|
|
201
|
+
deps: RecoveryDeps;
|
|
202
|
+
/**
|
|
203
|
+
* Persist the D3 mark for this record (registry entry; agent record for
|
|
204
|
+
* background). Returns `false` when the durable write did not land.
|
|
205
|
+
*/
|
|
206
|
+
mark(record: RecoveryRecord, interruptedAt: string): boolean | void;
|
|
207
|
+
/**
|
|
208
|
+
* Read ONE background run's agent record by id (never the whole store).
|
|
209
|
+
* Returns null when the record is absent; absent defaults to an in-flight
|
|
210
|
+
* entry so a registry without an agent file is still marked, not skipped.
|
|
211
|
+
*/
|
|
212
|
+
readAgentRecord?(id: string): PersistedAgentLike | null;
|
|
213
|
+
now?: () => Date;
|
|
214
|
+
log?: Pick<Logger, "debug" | "info" | "warn">;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* The production boot scan: candidate runs come from the durable registry
|
|
219
|
+
* (`listInflightRuns`), NOT from the current session's entries and NOT from a
|
|
220
|
+
* full parse of `.pi/subagents/`. This is what makes detection work on pi's
|
|
221
|
+
* default fresh startup after a conductor crash.
|
|
222
|
+
*
|
|
223
|
+
* Foreground/unit entries become `kind: "run"` candidates straight from the
|
|
224
|
+
* registry; background entries become `kind: "agent"` candidates enriched by
|
|
225
|
+
* reading THAT ONE agent file (so a stale registry entry whose agent record is
|
|
226
|
+
* already terminal is dropped). Legacy records with no registry entry are
|
|
227
|
+
* invisible here — consistent with the maintained `no-owner` policy (don't act
|
|
228
|
+
* on what we cannot verify).
|
|
229
|
+
*/
|
|
230
|
+
export async function recoverInflightRuns(options: RegistryScanOptions): Promise<BootScanSummary> {
|
|
231
|
+
// Issue #304 read/write asymmetry, fail-closed: `listInflightRuns` accepts
|
|
232
|
+
// any string id, but every write (and `markInterrupted`) resolves through
|
|
233
|
+
// `assertSafeAgentId`. Acting on an entry the writer could never have
|
|
234
|
+
// produced is unsafe, so SKIP those here — counted and reported in ONE log
|
|
235
|
+
// line — instead of attempting a mark that would silently no-op.
|
|
236
|
+
const addressable: InflightRun[] = [];
|
|
237
|
+
const unsafeIds: string[] = [];
|
|
238
|
+
for (const entry of listInflightRuns()) {
|
|
239
|
+
try {
|
|
240
|
+
assertSafeAgentId(entry.id);
|
|
241
|
+
addressable.push(entry);
|
|
242
|
+
} catch {
|
|
243
|
+
unsafeIds.push(entry.id);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
if (unsafeIds.length > 0) {
|
|
247
|
+
options.log?.warn("Recovery: skipped registry entries whose id fails the UUID guard", {
|
|
248
|
+
count: unsafeIds.length,
|
|
249
|
+
ids: unsafeIds,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const records: RecoveryRecord[] = [];
|
|
254
|
+
for (const entry of addressable) {
|
|
255
|
+
if (entry.kind === "background") {
|
|
256
|
+
const agent = options.readAgentRecord?.(entry.id) ?? null;
|
|
257
|
+
if (agent && agent.status !== "running") continue;
|
|
258
|
+
records.push({
|
|
259
|
+
id: entry.id,
|
|
260
|
+
kind: "agent",
|
|
261
|
+
status: "running",
|
|
262
|
+
interruptedAt: agent?.interruptedAt ?? entry.interruptedAt,
|
|
263
|
+
owner: agent?.owner ?? entry.owner,
|
|
264
|
+
childMarker: agent?.childMarker ?? entry.childMarker,
|
|
265
|
+
source: entry,
|
|
266
|
+
});
|
|
267
|
+
} else {
|
|
268
|
+
records.push({
|
|
269
|
+
id: entry.id,
|
|
270
|
+
kind: "run",
|
|
271
|
+
status: "running",
|
|
272
|
+
interruptedAt: entry.interruptedAt,
|
|
273
|
+
owner: entry.owner,
|
|
274
|
+
childMarker: entry.childMarker,
|
|
275
|
+
source: entry,
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
const summary = await runBootScan({
|
|
280
|
+
records,
|
|
281
|
+
deps: options.deps,
|
|
282
|
+
mark: options.mark,
|
|
283
|
+
now: options.now,
|
|
284
|
+
log: options.log,
|
|
285
|
+
});
|
|
286
|
+
summary.skippedUnsafeIds = unsafeIds.length;
|
|
287
|
+
return summary;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// ---------------------------------------------------------------------------
|
|
291
|
+
// Production entry (owns the /proc platform guard)
|
|
292
|
+
// ---------------------------------------------------------------------------
|
|
293
|
+
|
|
294
|
+
export interface ProductionRecoveryOptions {
|
|
295
|
+
/** Process deps; defaults to the real /proc wiring (`defaultRecoveryDeps`). */
|
|
296
|
+
deps?: RecoveryDeps;
|
|
297
|
+
/** Persist the D3 mark for this record; `false` = the durable write failed. */
|
|
298
|
+
mark(record: RecoveryRecord, interruptedAt: string): boolean | void;
|
|
299
|
+
readAgentRecord?(id: string): PersistedAgentLike | null;
|
|
300
|
+
now?: () => Date;
|
|
301
|
+
log?: Pick<Logger, "debug" | "info" | "warn">;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* The production boot-recovery entry. It owns the `/proc` platform guard so the
|
|
306
|
+
* guarantee "nothing is marked or killed without /proc ownership verification"
|
|
307
|
+
* is enforced — and testable — BELOW the extension entry point: on a platform
|
|
308
|
+
* without `/proc` it returns an empty summary without scanning, instead of
|
|
309
|
+
* relying on the caller's early return.
|
|
310
|
+
*/
|
|
311
|
+
export async function recoverProduction(options: ProductionRecoveryOptions): Promise<BootScanSummary> {
|
|
312
|
+
if (!isProcAvailable()) {
|
|
313
|
+
options.log?.info(
|
|
314
|
+
"Boot recovery skipped: /proc start-token verification unavailable on this platform",
|
|
315
|
+
);
|
|
316
|
+
return emptyBootScanSummary();
|
|
317
|
+
}
|
|
318
|
+
return recoverInflightRuns({
|
|
319
|
+
deps: options.deps ?? defaultRecoveryDeps(),
|
|
320
|
+
mark: options.mark,
|
|
321
|
+
readAgentRecord: options.readAgentRecord,
|
|
322
|
+
now: options.now,
|
|
323
|
+
log: options.log,
|
|
324
|
+
});
|
|
325
|
+
}
|
package/src/reports.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// Purpose: Compliance reporting: file-access records and secrets-exposure detection.
|
|
1
2
|
/**
|
|
2
3
|
* brl-subagent — Compliance Reports (E5)
|
|
3
4
|
*
|
|
@@ -128,6 +129,11 @@ function extractFilesFromOutputSummary(outputSummary: string): string[] {
|
|
|
128
129
|
|
|
129
130
|
/**
|
|
130
131
|
* Check if a file path matches any sensitive file pattern.
|
|
132
|
+
*
|
|
133
|
+
* This is the PATH-based detector for the secrets exposure report: it answers
|
|
134
|
+
* "did this run touch a sensitive FILE?" rather than "did the output text
|
|
135
|
+
* happen to mention one?". It was previously unreachable dead code; keep it in
|
|
136
|
+
* sync with SENSITIVE_FILE_PATTERNS / FILE_SEVERITY_MAP above.
|
|
131
137
|
*/
|
|
132
138
|
function isSensitiveFile(filePath: string): { match: boolean; pattern?: string; severity?: "high" | "medium" | "low" } {
|
|
133
139
|
const normalizedPath = filePath.toLowerCase();
|
|
@@ -221,20 +227,40 @@ export function buildFileAccessReport(runs: SubagentRun[]): FileAccessReport {
|
|
|
221
227
|
*/
|
|
222
228
|
export function buildSecretsExposureReport(runs: SubagentRun[]): SecretsExposureReport {
|
|
223
229
|
const exposures: SecretsExposureEntry[] = [];
|
|
230
|
+
// Dedupe key set: one exposure per (file, runId) regardless of whether the
|
|
231
|
+
// finding came from the text scan or the path scan.
|
|
232
|
+
const seen = new Set<string>();
|
|
224
233
|
|
|
225
234
|
for (const run of runs) {
|
|
226
235
|
// Check task description for sensitive keywords
|
|
227
236
|
const taskCheck = hasSensitiveTaskKeywords(run.task);
|
|
228
237
|
if (taskCheck.match && taskCheck.keyword) {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
238
|
+
const file = `task:${taskCheck.keyword}`;
|
|
239
|
+
const key = `${file}\x00${run.id}`;
|
|
240
|
+
if (!seen.has(key)) {
|
|
241
|
+
seen.add(key);
|
|
242
|
+
exposures.push({
|
|
243
|
+
file,
|
|
244
|
+
runId: run.id,
|
|
245
|
+
runLabel: run.label,
|
|
246
|
+
severity: "medium",
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// Files this run touched (mirrors buildFileAccessReport). Computed BEFORE
|
|
252
|
+
// the text scan so a text hit that is only a FRAGMENT of a touched path
|
|
253
|
+
// (e.g. ".env" inside "config/prod/.env") is not double-reported — the
|
|
254
|
+
// path scan below reports the real, full path instead.
|
|
255
|
+
const touched: string[] = [];
|
|
256
|
+
if (run.fullOutput && run.fullOutput.includes("diff --git")) {
|
|
257
|
+
touched.push(...extractFilesFromGitDiff(run.fullOutput));
|
|
258
|
+
}
|
|
259
|
+
if (touched.length === 0 && run.outputSummary) {
|
|
260
|
+
touched.push(...extractFilesFromOutputSummary(run.outputSummary));
|
|
235
261
|
}
|
|
236
262
|
|
|
237
|
-
// Check output for sensitive file
|
|
263
|
+
// Check output text for sensitive file mentions.
|
|
238
264
|
const outputToCheck = [
|
|
239
265
|
run.outputSummary,
|
|
240
266
|
run.fullOutput,
|
|
@@ -254,8 +280,18 @@ export function buildSecretsExposureReport(runs: SubagentRun[]): SecretsExposure
|
|
|
254
280
|
}
|
|
255
281
|
}
|
|
256
282
|
|
|
283
|
+
// Report the text that actually matched, not the regex
|
|
284
|
+
// literal — the `file` field is what a reader acts on.
|
|
285
|
+
const matched = outputToCheck.match(pattern)?.[0] ?? patternStr;
|
|
286
|
+
// Fragment of a touched file? The path scan below reports the full
|
|
287
|
+
// path — skip this fragment so one file yields ONE entry.
|
|
288
|
+
if (touched.some((p) => p.toLowerCase().includes(matched.toLowerCase()))) continue;
|
|
289
|
+
const dedupeKey = `${matched}\x00${run.id}`;
|
|
290
|
+
if (seen.has(dedupeKey)) continue;
|
|
291
|
+
seen.add(dedupeKey);
|
|
292
|
+
|
|
257
293
|
exposures.push({
|
|
258
|
-
file:
|
|
294
|
+
file: matched,
|
|
259
295
|
runId: run.id,
|
|
260
296
|
runLabel: run.label,
|
|
261
297
|
severity,
|
|
@@ -263,6 +299,26 @@ export function buildSecretsExposureReport(runs: SubagentRun[]): SecretsExposure
|
|
|
263
299
|
}
|
|
264
300
|
}
|
|
265
301
|
}
|
|
302
|
+
|
|
303
|
+
// Path-based detection: flag the sensitive FILES this run actually
|
|
304
|
+
// touched — the full path, even when the output text never names it (and
|
|
305
|
+
// even when it names it in a different case: the path is normalized
|
|
306
|
+
// before matching).
|
|
307
|
+
for (const file of touched) {
|
|
308
|
+
const check = isSensitiveFile(file);
|
|
309
|
+
if (!check.match) continue;
|
|
310
|
+
|
|
311
|
+
const dedupeKey = `${file}\x00${run.id}`;
|
|
312
|
+
if (seen.has(dedupeKey)) continue;
|
|
313
|
+
seen.add(dedupeKey);
|
|
314
|
+
|
|
315
|
+
exposures.push({
|
|
316
|
+
file,
|
|
317
|
+
runId: run.id,
|
|
318
|
+
runLabel: run.label,
|
|
319
|
+
severity: check.severity ?? "high",
|
|
320
|
+
});
|
|
321
|
+
}
|
|
266
322
|
}
|
|
267
323
|
|
|
268
324
|
return { exposures };
|