@indigoai-us/hq-cli 5.277.0 → 5.279.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/commands/lanes.js +97 -7
- package/dist/lib/checkpoint-ontology/apply.js +53 -2
- package/dist/lib/checkpoint-ontology/plan.d.ts +10 -0
- package/dist/lib/checkpoint-ontology/plan.js +10 -0
- package/dist/lib/lanes/config.d.ts +2 -1
- package/dist/lib/lanes/config.js +16 -0
- package/dist/lib/lanes/reconcile.d.ts +14 -0
- package/dist/lib/lanes/reconcile.js +275 -5
- package/dist/lib/lanes/runaway-tool-guard.d.ts +38 -0
- package/dist/lib/lanes/runaway-tool-guard.js +284 -0
- package/dist/lib/lanes/types.d.ts +10 -4
- package/dist/lib/lanes/watcher.d.ts +2 -0
- package/dist/lib/lanes/watcher.js +10 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,38 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.279.0] — 2026-09-29
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Claude keep-alive resumes use the same account selection as manual resumes,
|
|
10
|
+
including profile pins and the optional recent-use tiebreak. If no eligible
|
|
11
|
+
account can be selected, the lane resumes on its current account and records
|
|
12
|
+
the failure.
|
|
13
|
+
|
|
14
|
+
## [5.278.0] — 2026-09-29
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
|
|
18
|
+
- The CI and no-hermes pull request checks use read-only repository access, and CI pins its third-party actions to fixed commits. The fast-route benchmark workflow is pinned in a follow-up.
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- Reconcile can stop an over-budget same-user search process descended from a
|
|
23
|
+
running lane when `rg`, `grep`, `ugrep`, or `ag` disables ignores on a broad
|
|
24
|
+
HQ root, or `find` searches a broad root without a shallow-depth or prune
|
|
25
|
+
exemption. The `lanes.runaway-tool-guard` hq-flags key is off by default. A
|
|
26
|
+
stopped search records its command and duration, then gets a notice to narrow
|
|
27
|
+
the search.
|
|
28
|
+
|
|
29
|
+
- Add the default-off `checkpoint.ontology.hub-mention-floor` hq-flags key.
|
|
30
|
+
When it is on, checkpoint ontology apply rejects a knowledge link to a
|
|
31
|
+
non-person entity that 50 or more other entity files link to when the input
|
|
32
|
+
names it fewer than 2 times in the text the model read (the chunk's line
|
|
33
|
+
range of the annotation-stripped copy when there is one). Concept links
|
|
34
|
+
redirected to a project are checked the same way. The rejection is reported
|
|
35
|
+
as `hub_under_mentioned`.
|
|
36
|
+
|
|
5
37
|
## [5.277.0] — 2026-09-29
|
|
6
38
|
|
|
7
39
|
### Fixed
|
package/dist/commands/lanes.js
CHANGED
|
@@ -5534,12 +5534,12 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5534
5534
|
const at = now.toISOString();
|
|
5535
5535
|
const from = lane.account ?? null;
|
|
5536
5536
|
// The capacity decision and the record write run under the target
|
|
5537
|
-
// account's admission lock,
|
|
5538
|
-
// slot is taken by a running lane or
|
|
5539
|
-
// pointed at
|
|
5540
|
-
// (`pending_resume` set by this write
|
|
5541
|
-
// into one free slot cannot both start.
|
|
5542
|
-
|
|
5537
|
+
// account's admission lock, or while reconcile holds all provider
|
|
5538
|
+
// admission locks. A slot is taken by a running lane or a lane another move
|
|
5539
|
+
// has pointed at its target account but whose spawn has not claimed it yet
|
|
5540
|
+
// (`pending_resume` is set by this write and cleared by the claim), so two
|
|
5541
|
+
// moves into one free slot cannot both start.
|
|
5542
|
+
const writeMove = async () => {
|
|
5543
5543
|
const admission = input.deferTurnStart
|
|
5544
5544
|
? undefined
|
|
5545
5545
|
: await assessLaneAdmission(ctx, lane, {
|
|
@@ -5602,7 +5602,10 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5602
5602
|
return next;
|
|
5603
5603
|
}), input.grantContext);
|
|
5604
5604
|
return { lane: moved, from, to: target, copy, queued };
|
|
5605
|
-
}
|
|
5605
|
+
};
|
|
5606
|
+
return input.admissionLocksHeld
|
|
5607
|
+
? writeMove()
|
|
5608
|
+
: withLaneAdmissionLock(hqRoot, lane.provider, writeMove, ctx.sleep);
|
|
5606
5609
|
}
|
|
5607
5610
|
async function selectAutomaticClaudeResumeAccount(ctx, lane, workerProfile, options = {}) {
|
|
5608
5611
|
if (lane.provider !== "claude")
|
|
@@ -6993,6 +6996,93 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
|
6993
6996
|
recoverQueuedMessages,
|
|
6994
6997
|
mesh: ctx.mesh,
|
|
6995
6998
|
env: ctx.env,
|
|
6999
|
+
prepareKeepAliveClaudeResume: async (lane) => {
|
|
7000
|
+
const recoveredLane = laneWithRecoveredProviderSession(hqRoot, lane);
|
|
7001
|
+
const pendingModel = recoveredLane.pending_settings?.model;
|
|
7002
|
+
const selectionLane = pendingModel === undefined
|
|
7003
|
+
? recoveredLane
|
|
7004
|
+
: { ...recoveredLane, model: pendingModel, model_source: undefined };
|
|
7005
|
+
let automatic;
|
|
7006
|
+
try {
|
|
7007
|
+
automatic = await selectAutomaticClaudeResumeAccount(ctx, selectionLane, undefined, {
|
|
7008
|
+
repairUnavailableAccount: true,
|
|
7009
|
+
});
|
|
7010
|
+
}
|
|
7011
|
+
catch (err) {
|
|
7012
|
+
return {
|
|
7013
|
+
lane: recoveredLane,
|
|
7014
|
+
selectionFailure: err instanceof LanesError
|
|
7015
|
+
? `${err.error}: ${err.message}`
|
|
7016
|
+
: err instanceof Error
|
|
7017
|
+
? err.message
|
|
7018
|
+
: String(err),
|
|
7019
|
+
};
|
|
7020
|
+
}
|
|
7021
|
+
if (automatic === undefined)
|
|
7022
|
+
return { lane: recoveredLane };
|
|
7023
|
+
const { selection } = automatic;
|
|
7024
|
+
if (opts.dryRun) {
|
|
7025
|
+
return {
|
|
7026
|
+
lane: {
|
|
7027
|
+
...selectionLane,
|
|
7028
|
+
account: selection.account,
|
|
7029
|
+
account_selection: selection,
|
|
7030
|
+
...(pendingModel !== undefined
|
|
7031
|
+
? { model: pendingModel, model_source: "cli" }
|
|
7032
|
+
: automatic.preservedModel !== undefined
|
|
7033
|
+
? { model: automatic.preservedModel, model_source: "preserved" }
|
|
7034
|
+
: {}),
|
|
7035
|
+
},
|
|
7036
|
+
};
|
|
7037
|
+
}
|
|
7038
|
+
let latest;
|
|
7039
|
+
try {
|
|
7040
|
+
latest = readLane(hqRoot, lane.lane_id);
|
|
7041
|
+
}
|
|
7042
|
+
catch (err) {
|
|
7043
|
+
throw new LanesError("keep_alive_account_selection_lane_unreadable", `Cannot re-read lane ${lane.lane_id} before applying Claude account selection: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id });
|
|
7044
|
+
}
|
|
7045
|
+
if (latest.version !== lane.version ||
|
|
7046
|
+
latest.state !== lane.state ||
|
|
7047
|
+
latest.keep_alive !== true ||
|
|
7048
|
+
latest.keep_alive_opted_in_at !== lane.keep_alive_opted_in_at ||
|
|
7049
|
+
latest.provider_session_id !== lane.provider_session_id ||
|
|
7050
|
+
latest.last_resume?.session_id !== lane.last_resume?.session_id)
|
|
7051
|
+
return { lane: latest };
|
|
7052
|
+
if (selection.account === lane.account) {
|
|
7053
|
+
return { lane: persistClaudeAccountSelection(hqRoot, recoveredLane, selection) };
|
|
7054
|
+
}
|
|
7055
|
+
try {
|
|
7056
|
+
const moved = await prepareAccountMove(ctx, recoveredLane, {
|
|
7057
|
+
account: selection.account,
|
|
7058
|
+
cliModel: pendingModel,
|
|
7059
|
+
accountHistoryReason: "keep_alive_resume",
|
|
7060
|
+
...(pendingModel === undefined && automatic.preservedModel !== undefined
|
|
7061
|
+
? { preservedModel: automatic.preservedModel }
|
|
7062
|
+
: {}),
|
|
7063
|
+
deferTurnStart: true,
|
|
7064
|
+
admissionLocksHeld: true,
|
|
7065
|
+
accountSelection: selection,
|
|
7066
|
+
});
|
|
7067
|
+
return { lane: moved.lane };
|
|
7068
|
+
}
|
|
7069
|
+
catch (err) {
|
|
7070
|
+
let latest;
|
|
7071
|
+
try {
|
|
7072
|
+
latest = readLane(hqRoot, lane.lane_id);
|
|
7073
|
+
}
|
|
7074
|
+
catch (readErr) {
|
|
7075
|
+
return {
|
|
7076
|
+
lane,
|
|
7077
|
+
selectionFailure: `account move failed: ${err instanceof Error ? err.message : String(err)}; lane re-read failed: ${readErr instanceof Error ? readErr.message : String(readErr)}`,
|
|
7078
|
+
};
|
|
7079
|
+
}
|
|
7080
|
+
return {
|
|
7081
|
+
lane: latest,
|
|
7082
|
+
selectionFailure: `account move failed: ${err instanceof LanesError ? `${err.error}: ${err.message}` : err instanceof Error ? err.message : String(err)}`,
|
|
7083
|
+
};
|
|
7084
|
+
}
|
|
7085
|
+
},
|
|
6996
7086
|
});
|
|
6997
7087
|
printJson(summary);
|
|
6998
7088
|
if (!summary.ok) {
|
|
@@ -30,6 +30,10 @@ export const MIN_CREATE_CONFIDENCE = 0.7;
|
|
|
30
30
|
const DOMINANT_SIBLING_RATIO = 3;
|
|
31
31
|
/** Inbound links the dominant sibling must have before a redirect. */
|
|
32
32
|
const DOMINANT_SIBLING_MIN_INBOUND = 5;
|
|
33
|
+
/** checkpoint.ontology.hub-mention-floor: other entity files that must link here before the floor applies. */
|
|
34
|
+
const HUB_MIN_INBOUND = 50;
|
|
35
|
+
/** checkpoint.ontology.hub-mention-floor: whole-word canonical-name mentions required to keep the link. */
|
|
36
|
+
const HUB_MIN_MENTIONS = 2;
|
|
33
37
|
export class ApplyError extends Error {
|
|
34
38
|
constructor(message) {
|
|
35
39
|
super(message);
|
|
@@ -67,6 +71,16 @@ export function parseCandidate(raw) {
|
|
|
67
71
|
evidence: typeof c.evidence === "string" ? c.evidence : "",
|
|
68
72
|
};
|
|
69
73
|
}
|
|
74
|
+
function escapeRegExp(name) {
|
|
75
|
+
return name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
76
|
+
}
|
|
77
|
+
/** Whole-word, any-case matches of `name` in `text`, with Unicode letters and digits as word characters. Empty or whitespace-only names match nothing. */
|
|
78
|
+
function countMentions(text, name) {
|
|
79
|
+
if (name.trim() === "")
|
|
80
|
+
return 0;
|
|
81
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}])${escapeRegExp(name)}(?![\\p{L}\\p{N}])`, "giu");
|
|
82
|
+
return text.match(pattern)?.length ?? 0;
|
|
83
|
+
}
|
|
70
84
|
/**
|
|
71
85
|
* checkpoint.ontology.named-evidence: true when `evidence` holds one of
|
|
72
86
|
* `names` as a word, matching case. A name inside a path, domain, handle, tag
|
|
@@ -305,8 +319,11 @@ export function applyOntologyRun(options) {
|
|
|
305
319
|
if (types.size > 1)
|
|
306
320
|
crossTypeNames.add(name);
|
|
307
321
|
}
|
|
308
|
-
// checkpoint.ontology.dominant-sibling-link
|
|
309
|
-
|
|
322
|
+
// checkpoint.ontology.dominant-sibling-link and checkpoint.ontology.hub-mention-floor:
|
|
323
|
+
// how many other entity files link to each entity.
|
|
324
|
+
const inbound = plan.dominant_sibling_link === true || plan.hub_mention_floor === true
|
|
325
|
+
? countInboundLinks(company.dir, index)
|
|
326
|
+
: new Map();
|
|
310
327
|
const siblings = new Map();
|
|
311
328
|
if (plan.dominant_sibling_link === true) {
|
|
312
329
|
const grouped = new Map();
|
|
@@ -347,6 +364,32 @@ export function applyOntologyRun(options) {
|
|
|
347
364
|
const ownCompanyRel = `ontology/entities/company/${company.slug}.md`;
|
|
348
365
|
const isOwnCompany = (entity, rel) => entity.type === "company" && (rel === ownCompanyRel || crossTypeKey(entity.canonical_name) === crossTypeKey(company.slug));
|
|
349
366
|
const inputs = liveInputs(options.hqRoot, company, plan);
|
|
367
|
+
const inputTexts = new Map();
|
|
368
|
+
// The text the model was given for this input: the annotation-stripped copy
|
|
369
|
+
// when there is one, cut to the chunk's line range.
|
|
370
|
+
const inputText = (input) => {
|
|
371
|
+
const cached = inputTexts.get(input.id);
|
|
372
|
+
if (cached !== undefined)
|
|
373
|
+
return cached;
|
|
374
|
+
if (input.read_path !== undefined && input.read_path.split("/").includes("..")) {
|
|
375
|
+
throw new ApplyError("ontology plan read_path leaves the run directory");
|
|
376
|
+
}
|
|
377
|
+
const file = input.read_path !== undefined
|
|
378
|
+
? path.join(options.runDir, input.read_path)
|
|
379
|
+
: path.join(company.dir, input.path);
|
|
380
|
+
let text = fs.readFileSync(file, "utf8");
|
|
381
|
+
if (input.lines !== null)
|
|
382
|
+
text = text.split("\n").slice(input.lines[0] - 1, input.lines[1]).join("\n");
|
|
383
|
+
inputTexts.set(input.id, text);
|
|
384
|
+
return text;
|
|
385
|
+
};
|
|
386
|
+
// checkpoint.ontology.hub-mention-floor: the final target of a knowledge link is a
|
|
387
|
+
// non-person entity many files link to, and the input names it fewer than twice.
|
|
388
|
+
const underMentionedHub = (input, rel, entity) => plan.hub_mention_floor === true &&
|
|
389
|
+
input.kind === "knowledge" &&
|
|
390
|
+
entity.type !== "person" &&
|
|
391
|
+
(inbound.get(rel) ?? 0) >= HUB_MIN_INBOUND &&
|
|
392
|
+
countMentions(inputText(input), entity.canonical_name) < HUB_MIN_MENTIONS;
|
|
350
393
|
const blocked = new Set();
|
|
351
394
|
const updates = new Map();
|
|
352
395
|
const update = (rel) => {
|
|
@@ -414,6 +457,10 @@ export function applyOntologyRun(options) {
|
|
|
414
457
|
note(report.rejected, "unnamed_in_evidence");
|
|
415
458
|
continue;
|
|
416
459
|
}
|
|
460
|
+
if (underMentionedHub(input, targetRel, targetEntity)) {
|
|
461
|
+
note(report.rejected, "hub_under_mentioned");
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
417
464
|
report.matched += 1;
|
|
418
465
|
report.links.push({
|
|
419
466
|
input: candidate.input,
|
|
@@ -470,6 +517,10 @@ export function applyOntologyRun(options) {
|
|
|
470
517
|
continue;
|
|
471
518
|
}
|
|
472
519
|
const linked = index.byRel.get(projects[0]);
|
|
520
|
+
if (underMentionedHub(input, projects[0], linked)) {
|
|
521
|
+
note(report.rejected, "hub_under_mentioned");
|
|
522
|
+
continue;
|
|
523
|
+
}
|
|
473
524
|
report.matched += 1;
|
|
474
525
|
report.links.push({ input: candidate.input, action: "link", type: linked.type, canonical_name: linked.canonical_name, path: projects[0] });
|
|
475
526
|
continue;
|
|
@@ -81,6 +81,14 @@ export declare const SKIP_AMBIGUOUS_CONCEPT_LINKS_FLAG = "checkpoint.ontology.sk
|
|
|
81
81
|
* and recall from 0.784 to 0.816, and never changed a create.
|
|
82
82
|
*/
|
|
83
83
|
export declare const DOMINANT_SIBLING_LINK_FLAG = "checkpoint.ontology.dominant-sibling-link";
|
|
84
|
+
/**
|
|
85
|
+
* A knowledge link to a non-person entity that 50 or more
|
|
86
|
+
* other entity files link to is rejected as `hub_under_mentioned` when the
|
|
87
|
+
* input names it fewer than 2 times (whole word, any case, canonical name
|
|
88
|
+
* only). The plan carries `hub_mention_floor: true`. CO-62: a replay of six
|
|
89
|
+
* runs raised knowledge precision from 0.753 to 0.781 at the same recall.
|
|
90
|
+
*/
|
|
91
|
+
export declare const HUB_MENTION_FLOOR_FLAG = "checkpoint.ontology.hub-mention-floor";
|
|
84
92
|
/**
|
|
85
93
|
* A knowledge concept candidate whose name is an alias of more than one
|
|
86
94
|
* entity, and which apply would defer as `ambiguous_alias`, links instead to
|
|
@@ -186,6 +194,8 @@ export interface OntologyPlan {
|
|
|
186
194
|
skip_ambiguous_concept_links?: true;
|
|
187
195
|
/** Set by checkpoint.ontology.dominant-sibling-link: apply redirects a knowledge link from a weak cross-type sibling to the one other files link to. */
|
|
188
196
|
dominant_sibling_link?: true;
|
|
197
|
+
/** Set by checkpoint.ontology.hub-mention-floor: apply rejects a knowledge link to an under-mentioned non-person hub. */
|
|
198
|
+
hub_mention_floor?: true;
|
|
189
199
|
/** Set by checkpoint.ontology.concept-project-link: apply links a knowledge concept with an ambiguous alias to the one project of that canonical name. */
|
|
190
200
|
concept_project_link?: true;
|
|
191
201
|
/** Set by checkpoint.ontology.named-evidence: apply rejects a knowledge link to the company's own entity, or a concept-project-link redirect, whose evidence does not name it as a word. */
|
|
@@ -86,6 +86,14 @@ export const SKIP_AMBIGUOUS_CONCEPT_LINKS_FLAG = "checkpoint.ontology.skip-ambig
|
|
|
86
86
|
* and recall from 0.784 to 0.816, and never changed a create.
|
|
87
87
|
*/
|
|
88
88
|
export const DOMINANT_SIBLING_LINK_FLAG = "checkpoint.ontology.dominant-sibling-link";
|
|
89
|
+
/**
|
|
90
|
+
* A knowledge link to a non-person entity that 50 or more
|
|
91
|
+
* other entity files link to is rejected as `hub_under_mentioned` when the
|
|
92
|
+
* input names it fewer than 2 times (whole word, any case, canonical name
|
|
93
|
+
* only). The plan carries `hub_mention_floor: true`. CO-62: a replay of six
|
|
94
|
+
* runs raised knowledge precision from 0.753 to 0.781 at the same recall.
|
|
95
|
+
*/
|
|
96
|
+
export const HUB_MENTION_FLOOR_FLAG = "checkpoint.ontology.hub-mention-floor";
|
|
89
97
|
/**
|
|
90
98
|
* A knowledge concept candidate whose name is an alias of more than one
|
|
91
99
|
* entity, and which apply would defer as `ambiguous_alias`, links instead to
|
|
@@ -317,6 +325,7 @@ export async function planOntologyRun(options) {
|
|
|
317
325
|
const firstNamePeople = await options.readFlag(FIRST_NAME_PEOPLE_FLAG, company);
|
|
318
326
|
const skipAmbiguousConceptLinks = await options.readFlag(SKIP_AMBIGUOUS_CONCEPT_LINKS_FLAG, company);
|
|
319
327
|
const dominantSiblingLink = await options.readFlag(DOMINANT_SIBLING_LINK_FLAG, company);
|
|
328
|
+
const hubMentionFloor = await options.readFlag(HUB_MENTION_FLOOR_FLAG, company);
|
|
320
329
|
const conceptProjectLink = await options.readFlag(CONCEPT_PROJECT_LINK_FLAG, company);
|
|
321
330
|
const namedEvidence = await options.readFlag(NAMED_EVIDENCE_FLAG, company);
|
|
322
331
|
const queuedTouchesOn = borrowed || (await options.readFlag(QUEUED_TOUCHES_FLAG, company));
|
|
@@ -441,6 +450,7 @@ export async function planOntologyRun(options) {
|
|
|
441
450
|
...(firstNamePeople ? { first_name_people: true } : {}),
|
|
442
451
|
...(skipAmbiguousConceptLinks ? { skip_ambiguous_concept_links: true } : {}),
|
|
443
452
|
...(dominantSiblingLink ? { dominant_sibling_link: true } : {}),
|
|
453
|
+
...(hubMentionFloor ? { hub_mention_floor: true } : {}),
|
|
444
454
|
...(conceptProjectLink ? { concept_project_link: true } : {}),
|
|
445
455
|
...(namedEvidence ? { named_evidence: true } : {}),
|
|
446
456
|
inputs,
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* that resolve to the same home are refused (identity-keyed-cap rule).
|
|
8
8
|
*/
|
|
9
9
|
import * as fs from "node:fs";
|
|
10
|
-
import { type AccountConfig, type LaneProvider, type LanesConfig, type LivenessWindows, type ProviderConfig, type TrackerChecksConfig } from "./types.js";
|
|
10
|
+
import { type AccountConfig, type LaneProvider, type LanesConfig, type RunawaySearchGuardConfig, type LivenessWindows, type ProviderConfig, type TrackerChecksConfig } from "./types.js";
|
|
11
11
|
export declare const LANES_CONFIG_REL: string;
|
|
12
12
|
export declare const DEFAULT_MAX_QUEUED = 3;
|
|
13
13
|
export declare const DEFAULT_CPU_BUSY_PAUSE = 0.85;
|
|
@@ -44,6 +44,7 @@ export declare const DEFAULT_BLOCKER_FRESHNESS_HOURS = 4;
|
|
|
44
44
|
/** Manager key results must be re-measured within six hours by default. */
|
|
45
45
|
export declare const DEFAULT_KR_FRESHNESS_HOURS = 6;
|
|
46
46
|
export declare const DEFAULT_TRACKER_CHECKS: TrackerChecksConfig;
|
|
47
|
+
export declare const DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG: RunawaySearchGuardConfig;
|
|
47
48
|
export declare const DEFAULT_LIVENESS_WINDOWS: LivenessWindows;
|
|
48
49
|
export declare const DEFAULT_PROVIDER_MAX_RUNNING: Record<LaneProvider, number>;
|
|
49
50
|
export declare const DEFAULT_ACCOUNT_HOMES: Record<LaneProvider, string>;
|
package/dist/lib/lanes/config.js
CHANGED
|
@@ -56,6 +56,10 @@ export const DEFAULT_TRACKER_CHECKS = {
|
|
|
56
56
|
kpi_value_max_age_ms: 24 * 60 * 60 * 1000,
|
|
57
57
|
directive_ack_timeout_ms: 30 * 60 * 1000,
|
|
58
58
|
};
|
|
59
|
+
export const DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG = {
|
|
60
|
+
max_age_seconds: 20,
|
|
61
|
+
find_maxdepth_exempt: 3,
|
|
62
|
+
};
|
|
59
63
|
export const DEFAULT_LIVENESS_WINDOWS = {
|
|
60
64
|
live_within_ms: DEFAULT_LIVE_WITHIN_MS,
|
|
61
65
|
turn_start_lost_ms: DEFAULT_TURN_START_LOST_MS,
|
|
@@ -417,6 +421,16 @@ function parseTrackerChecks(raw) {
|
|
|
417
421
|
directive_ack_timeout_ms: positiveInt(checks.directive_ack_timeout_ms, "tracker.checks.directive_ack_timeout_ms", DEFAULT_TRACKER_CHECKS.directive_ack_timeout_ms),
|
|
418
422
|
};
|
|
419
423
|
}
|
|
424
|
+
function parseRunawaySearchGuard(raw) {
|
|
425
|
+
const guard = raw === undefined ? {} : raw;
|
|
426
|
+
if (!isPlainObject(guard)) {
|
|
427
|
+
throw new LanesConfigError("invalid_config", "runaway_search_guard must be an object");
|
|
428
|
+
}
|
|
429
|
+
return {
|
|
430
|
+
max_age_seconds: positiveInt(guard.max_age_seconds, "runaway_search_guard.max_age_seconds", DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG.max_age_seconds),
|
|
431
|
+
find_maxdepth_exempt: positiveInt(guard.find_maxdepth_exempt, "runaway_search_guard.find_maxdepth_exempt", DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG.find_maxdepth_exempt),
|
|
432
|
+
};
|
|
433
|
+
}
|
|
420
434
|
export function documentedDefaultConfig(deps = defaultDeps()) {
|
|
421
435
|
const providers = {};
|
|
422
436
|
for (const provider of LANE_PROVIDERS) {
|
|
@@ -440,6 +454,7 @@ export function documentedDefaultConfig(deps = defaultDeps()) {
|
|
|
440
454
|
},
|
|
441
455
|
liveness: { ...DEFAULT_LIVENESS_WINDOWS },
|
|
442
456
|
tracker: { checks: { ...DEFAULT_TRACKER_CHECKS } },
|
|
457
|
+
runaway_search_guard: { ...DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG },
|
|
443
458
|
session_question_timeout_secs: DEFAULT_SESSION_QUESTION_TIMEOUT_SECS,
|
|
444
459
|
blocker_freshness_hours: DEFAULT_BLOCKER_FRESHNESS_HOURS,
|
|
445
460
|
kr_freshness_hours: DEFAULT_KR_FRESHNESS_HOURS,
|
|
@@ -484,6 +499,7 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
484
499
|
throttle: parseThrottle(raw.throttle),
|
|
485
500
|
liveness: parseLiveness(raw.liveness),
|
|
486
501
|
tracker: { checks: parseTrackerChecks(raw.tracker) },
|
|
502
|
+
runaway_search_guard: parseRunawaySearchGuard(raw.runaway_search_guard),
|
|
487
503
|
session_question_timeout_secs: positiveInt(raw.session_question_timeout_secs, "session_question_timeout_secs", DEFAULT_SESSION_QUESTION_TIMEOUT_SECS),
|
|
488
504
|
blocker_freshness_hours: positiveNumber(raw.blocker_freshness_hours, "blocker_freshness_hours", DEFAULT_BLOCKER_FRESHNESS_HOURS),
|
|
489
505
|
kr_freshness_hours: positiveNumber(raw.kr_freshness_hours, "kr_freshness_hours", DEFAULT_KR_FRESHNESS_HOURS),
|
|
@@ -9,6 +9,7 @@ import { type LivenessDeps } from "./liveness.js";
|
|
|
9
9
|
import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
|
|
10
10
|
import { type ThrottleDeps } from "./throttle.js";
|
|
11
11
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
12
|
+
import { type RunawayProcessInfo } from "./runaway-tool-guard.js";
|
|
12
13
|
import { type KeepAliveBlockedReason } from "./keep-alive.js";
|
|
13
14
|
import type { LaneCompanyRef, LaneRecord, LivenessState, QueuedReason, QuestionRecord, SeniorRef } from "./types.js";
|
|
14
15
|
import { type LaneMesh } from "./mesh-events.js";
|
|
@@ -91,6 +92,19 @@ export interface ReconcileDeps {
|
|
|
91
92
|
meshEmitTimeoutMs?: number;
|
|
92
93
|
/** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
|
|
93
94
|
readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
|
|
95
|
+
/** Reuse the command's Claude resume selection and session-move path for keep-alive. */
|
|
96
|
+
prepareKeepAliveClaudeResume?: (lane: LaneRecord) => Promise<{
|
|
97
|
+
lane: LaneRecord;
|
|
98
|
+
selectionFailure?: string;
|
|
99
|
+
}>;
|
|
100
|
+
/** Process-table snapshot seam for the opt-in runaway-search guard. */
|
|
101
|
+
runawayToolProcessLister?: () => readonly RunawayProcessInfo[];
|
|
102
|
+
/** Test seam: prevent real process signals while exercising the guard. */
|
|
103
|
+
runawayToolProcessSignal?: (pid: number, signal: NodeJS.Signals) => void;
|
|
104
|
+
/** Test seam for interleaving a watcher signal with an append-only guard event. */
|
|
105
|
+
runawayToolEventAppend?: (target: string, line: string) => void;
|
|
106
|
+
/** Test seam for validating a candidate PID against its process-table starttime. */
|
|
107
|
+
runawayToolProcessIsCurrent?: (pid: number, expectedStarttime: number) => boolean;
|
|
94
108
|
/**
|
|
95
109
|
* Awaited after lane/question records and liveness are loaded, before any
|
|
96
110
|
* CAS write. Test seam so two in-process passes can share a snapshot.
|
|
@@ -18,16 +18,18 @@ import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./err
|
|
|
18
18
|
import { loadWorkerProfile } from "./profile.js";
|
|
19
19
|
import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
|
|
20
20
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
|
|
21
|
-
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
21
|
+
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
22
22
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
23
23
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
24
24
|
import { readLanesRegistryFlag } from "./registry-flag.js";
|
|
25
25
|
import { resolveLaneCompany } from "./lane-company.js";
|
|
26
|
+
import { appendDropboxMessage } from "./dropbox.js";
|
|
27
|
+
import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
|
|
26
28
|
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
27
29
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
28
30
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
29
31
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
30
|
-
import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
32
|
+
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
31
33
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
32
34
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
33
35
|
import { resolveManagerStatusGateForLane } from "./manager-status-gate.js";
|
|
@@ -658,6 +660,128 @@ function classifyDeadLocalLane(hqRoot, laneId, now) {
|
|
|
658
660
|
lane,
|
|
659
661
|
};
|
|
660
662
|
}
|
|
663
|
+
function runawayGuardCompanyKey(company) {
|
|
664
|
+
return company
|
|
665
|
+
? `${company.uid ?? ""}\u0000${company.slug ?? ""}`
|
|
666
|
+
: "unresolved-company";
|
|
667
|
+
}
|
|
668
|
+
function reportRunawayGuardError(summary, stage, error) {
|
|
669
|
+
summary.ok = false;
|
|
670
|
+
summary.errors.push({
|
|
671
|
+
code: `runaway_tool_guard_${stage}_failed`,
|
|
672
|
+
message: `runaway search guard ${stage} failed; no additional search processes were stopped`,
|
|
673
|
+
});
|
|
674
|
+
process.stderr.write(`${JSON.stringify({
|
|
675
|
+
event: "lanes.runaway_tool_guard.failed",
|
|
676
|
+
stage,
|
|
677
|
+
error: error instanceof Error ? error.name : "UnknownError",
|
|
678
|
+
fallback: "continue_without_argv",
|
|
679
|
+
})}\n`);
|
|
680
|
+
}
|
|
681
|
+
async function runRunawayToolGuardPass(hqRoot, lanes, config, deps, now, summary) {
|
|
682
|
+
if (deps.dryRun)
|
|
683
|
+
return;
|
|
684
|
+
const localRunningLanes = lanes.filter((lane) => lane.state === "running" && laneRunsHere(lane, deps.host));
|
|
685
|
+
if (localRunningLanes.length === 0)
|
|
686
|
+
return;
|
|
687
|
+
const groups = new Map();
|
|
688
|
+
for (const lane of localRunningLanes) {
|
|
689
|
+
let worker;
|
|
690
|
+
try {
|
|
691
|
+
worker = readPidRecord(path.join(laneRunDir(hqRoot, lane.lane_id), "worker.pid"));
|
|
692
|
+
if (!worker ||
|
|
693
|
+
worker.starttime === undefined ||
|
|
694
|
+
!isPidRecordAlive(worker, deps.isAlive))
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
catch (error) {
|
|
698
|
+
reportRunawayGuardError(summary, "worker_pid_read", error);
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
let company;
|
|
702
|
+
try {
|
|
703
|
+
company = resolveLaneCompany(hqRoot, lane).company;
|
|
704
|
+
}
|
|
705
|
+
catch (error) {
|
|
706
|
+
reportRunawayGuardError(summary, "company_resolution", error);
|
|
707
|
+
continue;
|
|
708
|
+
}
|
|
709
|
+
const key = runawayGuardCompanyKey(company);
|
|
710
|
+
const group = groups.get(key) ?? { company, lanes: [] };
|
|
711
|
+
group.lanes.push({ laneId: lane.lane_id, agentPid: worker.pid, agentStarttime: worker.starttime });
|
|
712
|
+
groups.set(key, group);
|
|
713
|
+
}
|
|
714
|
+
if (groups.size === 0)
|
|
715
|
+
return;
|
|
716
|
+
const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
717
|
+
const enabledGroups = new Set();
|
|
718
|
+
const env = deps.env ?? process.env;
|
|
719
|
+
await Promise.all([...groups.entries()].map(async ([key, group]) => {
|
|
720
|
+
try {
|
|
721
|
+
const enabled = await readRegistryFlag(RUNAWAY_TOOL_GUARD_FLAG, {
|
|
722
|
+
company: group.company ?? undefined,
|
|
723
|
+
env,
|
|
724
|
+
});
|
|
725
|
+
if (enabled)
|
|
726
|
+
enabledGroups.add(key);
|
|
727
|
+
}
|
|
728
|
+
catch (error) {
|
|
729
|
+
reportRunawayGuardError(summary, "flag_read", error);
|
|
730
|
+
}
|
|
731
|
+
}));
|
|
732
|
+
if (enabledGroups.size === 0)
|
|
733
|
+
return;
|
|
734
|
+
const laneProcesses = [...groups.entries()]
|
|
735
|
+
.filter(([key]) => enabledGroups.has(key))
|
|
736
|
+
.flatMap(([, group]) => group.lanes);
|
|
737
|
+
if (laneProcesses.length === 0)
|
|
738
|
+
return;
|
|
739
|
+
let processes;
|
|
740
|
+
try {
|
|
741
|
+
processes = (deps.runawayToolProcessLister ?? listHostProcesses)();
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
reportRunawayGuardError(summary, "process_table_read", error);
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
const runawayConfig = config.runaway_search_guard ?? DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG;
|
|
748
|
+
try {
|
|
749
|
+
runRunawayToolGuard({
|
|
750
|
+
hqRoot,
|
|
751
|
+
lanes: laneProcesses,
|
|
752
|
+
processes,
|
|
753
|
+
config: runawayConfig,
|
|
754
|
+
now,
|
|
755
|
+
currentUid: process.getuid?.() ?? -1,
|
|
756
|
+
isCurrentProcess: deps.runawayToolProcessIsCurrent ??
|
|
757
|
+
((pid, expectedStarttime) => readProcStarttime(pid) === expectedStarttime),
|
|
758
|
+
signal: deps.runawayToolProcessSignal ?? ((pid, signal) => process.kill(pid, signal)),
|
|
759
|
+
recordEvent: (laneId, event) => appendWatcherEvent(watcherEventsPath(hqRoot, laneId), event, deps.runawayToolEventAppend),
|
|
760
|
+
notifyLane: (laneId, text) => appendDropboxMessage(hqRoot, laneId, {
|
|
761
|
+
at: now.toISOString(),
|
|
762
|
+
message_id: `runaway-search-guard:${laneId}:${now.getTime()}`,
|
|
763
|
+
kind: "system_notice",
|
|
764
|
+
text,
|
|
765
|
+
}),
|
|
766
|
+
reportError: (stage, errorName) => {
|
|
767
|
+
summary.ok = false;
|
|
768
|
+
summary.errors.push({
|
|
769
|
+
code: `runaway_tool_guard_${stage}_failed`,
|
|
770
|
+
message: `runaway search guard could not ${stage} a stopped process (${errorName})`,
|
|
771
|
+
});
|
|
772
|
+
process.stderr.write(`${JSON.stringify({
|
|
773
|
+
event: "lanes.runaway_tool_guard.failed",
|
|
774
|
+
stage,
|
|
775
|
+
error: errorName,
|
|
776
|
+
fallback: "continue_without_argv",
|
|
777
|
+
})}\n`);
|
|
778
|
+
},
|
|
779
|
+
});
|
|
780
|
+
}
|
|
781
|
+
catch (error) {
|
|
782
|
+
reportRunawayGuardError(summary, "execution", error);
|
|
783
|
+
}
|
|
784
|
+
}
|
|
661
785
|
/** Unlocked pass body. `runReconcilePass` takes the process lock around this. */
|
|
662
786
|
export async function runPassBody(hqRoot, deps) {
|
|
663
787
|
const now = deps.now();
|
|
@@ -728,6 +852,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
728
852
|
: QUESTION_AGE_MS;
|
|
729
853
|
if (deps.afterLoad)
|
|
730
854
|
await deps.afterLoad();
|
|
855
|
+
await runRunawayToolGuardPass(hqRoot, lanes, config, deps, now, summary);
|
|
731
856
|
const startDeps = await prepareManagerStatusGateStarts(hqRoot, lanes, deps, summary);
|
|
732
857
|
markOverdueLanes(hqRoot, lanes, summary, deps, now);
|
|
733
858
|
await heartbeatAndDeadHosts(hqRoot, lanes, summary, startDeps, now);
|
|
@@ -876,7 +1001,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
876
1001
|
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
877
1002
|
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
878
1003
|
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
879
|
-
resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
|
|
1004
|
+
await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
|
|
880
1005
|
const drainLanes = loadLaneRecords(hqRoot, summary);
|
|
881
1006
|
await admitQueued(hqRoot, drainLanes, summary, startDeps, now);
|
|
882
1007
|
await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, startDeps);
|
|
@@ -1598,7 +1723,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
|
1598
1723
|
throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
|
|
1599
1724
|
}
|
|
1600
1725
|
}
|
|
1601
|
-
function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany) {
|
|
1726
|
+
async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany) {
|
|
1602
1727
|
const nowMs = now.getTime();
|
|
1603
1728
|
const nowIso = now.toISOString();
|
|
1604
1729
|
const config = loadLanesConfig(hqRoot);
|
|
@@ -1983,6 +2108,44 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
1983
2108
|
});
|
|
1984
2109
|
continue;
|
|
1985
2110
|
}
|
|
2111
|
+
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
2112
|
+
if (seniorId !== undefined) {
|
|
2113
|
+
let grantPreflight;
|
|
2114
|
+
try {
|
|
2115
|
+
grantPreflight = withHireGrantAdmission(hqRoot, seniorId, (snapshot) => {
|
|
2116
|
+
const latestLane = readLane(hqRoot, current.lane_id);
|
|
2117
|
+
if (latestLane.version !== current.version ||
|
|
2118
|
+
latestLane.state !== current.state ||
|
|
2119
|
+
latestLane.keep_alive !== true ||
|
|
2120
|
+
latestLane.keep_alive_opted_in_at !== current.keep_alive_opted_in_at ||
|
|
2121
|
+
latestLane.senior.kind !== "lane" ||
|
|
2122
|
+
latestLane.senior.id !== seniorId)
|
|
2123
|
+
return { kind: "stale" };
|
|
2124
|
+
const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
|
|
2125
|
+
return refusal === null
|
|
2126
|
+
? { kind: "eligible" }
|
|
2127
|
+
: { kind: "refused", error: refusal.error, message: refusal.message };
|
|
2128
|
+
});
|
|
2129
|
+
}
|
|
2130
|
+
catch (err) {
|
|
2131
|
+
markDirty(summary, {
|
|
2132
|
+
kind: "lane",
|
|
2133
|
+
id: current.lane_id,
|
|
2134
|
+
why: `keep-alive hire admission preflight failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
2135
|
+
});
|
|
2136
|
+
continue;
|
|
2137
|
+
}
|
|
2138
|
+
if (grantPreflight.kind === "stale")
|
|
2139
|
+
continue;
|
|
2140
|
+
if (grantPreflight.kind === "refused") {
|
|
2141
|
+
summary.skipped.push({
|
|
2142
|
+
kind: "lane",
|
|
2143
|
+
id: current.lane_id,
|
|
2144
|
+
why: `keep_alive_hire_admission: ${grantPreflight.error}: ${grantPreflight.message}`,
|
|
2145
|
+
});
|
|
2146
|
+
continue;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
1986
2149
|
const resumedFrom = current.state === "done" ? "done" : "interrupted";
|
|
1987
2150
|
const resumeText = resumedFrom === "interrupted"
|
|
1988
2151
|
? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
|
|
@@ -1993,6 +2156,114 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
1993
2156
|
if (carriesAnsweredQuestions && !deps.dryRun) {
|
|
1994
2157
|
writeAnsweredQuestions(hqRoot, current, questions);
|
|
1995
2158
|
}
|
|
2159
|
+
let accountSelectionFailure;
|
|
2160
|
+
if (current.provider === "claude" && deps.prepareKeepAliveClaudeResume !== undefined) {
|
|
2161
|
+
const optInBeforePreparation = current.keep_alive_opted_in_at;
|
|
2162
|
+
let preparedLane;
|
|
2163
|
+
try {
|
|
2164
|
+
const prepared = await deps.prepareKeepAliveClaudeResume(current);
|
|
2165
|
+
preparedLane = prepared.lane;
|
|
2166
|
+
accountSelectionFailure = prepared.selectionFailure;
|
|
2167
|
+
}
|
|
2168
|
+
catch (err) {
|
|
2169
|
+
// Account balancing is best-effort for keep-alive. A failure to read
|
|
2170
|
+
// balance inputs must not prevent the existing session from resuming.
|
|
2171
|
+
try {
|
|
2172
|
+
preparedLane = readLane(hqRoot, current.lane_id);
|
|
2173
|
+
}
|
|
2174
|
+
catch (readErr) {
|
|
2175
|
+
markDirty(summary, {
|
|
2176
|
+
kind: "lane",
|
|
2177
|
+
id: current.lane_id,
|
|
2178
|
+
why: `keep-alive account selection failed and lane could not be re-read: ${readErr instanceof Error ? readErr.message : String(readErr)}`,
|
|
2179
|
+
});
|
|
2180
|
+
continue;
|
|
2181
|
+
}
|
|
2182
|
+
accountSelectionFailure = err instanceof Error ? err.message : String(err);
|
|
2183
|
+
}
|
|
2184
|
+
const preparedSessionId = preparedLane.provider_session_id ?? preparedLane.last_resume?.session_id;
|
|
2185
|
+
if (preparedLane.state !== initialState ||
|
|
2186
|
+
preparedLane.keep_alive !== true ||
|
|
2187
|
+
!laneRunsHere(preparedLane, deps.host) ||
|
|
2188
|
+
preparedSessionId !== currentSessionId)
|
|
2189
|
+
continue;
|
|
2190
|
+
let latest;
|
|
2191
|
+
try {
|
|
2192
|
+
latest = deps.dryRun
|
|
2193
|
+
? (deps.readKeepAliveRecheckLane?.(hqRoot, current.lane_id) ?? readLane(hqRoot, current.lane_id))
|
|
2194
|
+
: readLane(hqRoot, current.lane_id);
|
|
2195
|
+
}
|
|
2196
|
+
catch (err) {
|
|
2197
|
+
markDirty(summary, {
|
|
2198
|
+
kind: "lane",
|
|
2199
|
+
id: current.lane_id,
|
|
2200
|
+
why: `keep-alive eligibility recheck failed after account selection: ${err instanceof Error ? err.message : String(err)}`,
|
|
2201
|
+
});
|
|
2202
|
+
continue;
|
|
2203
|
+
}
|
|
2204
|
+
if (latest.state !== initialState || !laneRunsHere(latest, deps.host))
|
|
2205
|
+
continue;
|
|
2206
|
+
if (latest.keep_alive !== true) {
|
|
2207
|
+
recordKeepAliveSkip(hqRoot, latest, "not_opted_in", summary, deps);
|
|
2208
|
+
continue;
|
|
2209
|
+
}
|
|
2210
|
+
if (latest.keep_alive_opted_in_at !== optInBeforePreparation) {
|
|
2211
|
+
summary.skipped.push({
|
|
2212
|
+
kind: "lane",
|
|
2213
|
+
id: latest.lane_id,
|
|
2214
|
+
why: "keep_alive_eligibility_changed_during_account_selection",
|
|
2215
|
+
});
|
|
2216
|
+
continue;
|
|
2217
|
+
}
|
|
2218
|
+
const latestSessionId = latest.provider_session_id ?? latest.last_resume?.session_id;
|
|
2219
|
+
if (latestSessionId !== currentSessionId)
|
|
2220
|
+
continue;
|
|
2221
|
+
let latestEligibilityAtMs;
|
|
2222
|
+
if (initialState === "done") {
|
|
2223
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, latest.lane_id);
|
|
2224
|
+
if (!envelope || envelope.envelope.decision !== "done")
|
|
2225
|
+
continue;
|
|
2226
|
+
if (doneKeepAliveVerdict(latest, envelope.envelope, latest.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, latest, questions)) !== "eligible")
|
|
2227
|
+
continue;
|
|
2228
|
+
try {
|
|
2229
|
+
latestEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, latest, envelope.ref);
|
|
2230
|
+
}
|
|
2231
|
+
catch (err) {
|
|
2232
|
+
markDirty(summary, {
|
|
2233
|
+
kind: "lane",
|
|
2234
|
+
id: latest.lane_id,
|
|
2235
|
+
why: `keep-alive envelope timestamp unavailable after account selection: ${err instanceof Error ? err.message : String(err)}`,
|
|
2236
|
+
});
|
|
2237
|
+
continue;
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
else {
|
|
2241
|
+
latestEligibilityAtMs = Date.parse(latest.interrupted_at ?? "");
|
|
2242
|
+
}
|
|
2243
|
+
const latestOptedInAtMs = Date.parse(latest.keep_alive_opted_in_at ?? "");
|
|
2244
|
+
if (!Number.isFinite(latestEligibilityAtMs) || !Number.isFinite(latestOptedInAtMs))
|
|
2245
|
+
continue;
|
|
2246
|
+
if (latestEligibilityAtMs <= latestOptedInAtMs) {
|
|
2247
|
+
recordKeepAliveSkip(hqRoot, latest, initialState === "done" ? "envelope_before_opt_in" : "interruption_before_opt_in", summary, deps);
|
|
2248
|
+
continue;
|
|
2249
|
+
}
|
|
2250
|
+
current = deps.dryRun ? preparedLane : latest;
|
|
2251
|
+
}
|
|
2252
|
+
if (accountSelectionFailure !== undefined) {
|
|
2253
|
+
summary.skipped.push({
|
|
2254
|
+
kind: "lane",
|
|
2255
|
+
id: current.lane_id,
|
|
2256
|
+
why: `keep_alive_account_selection_failed: ${accountSelectionFailure}`,
|
|
2257
|
+
});
|
|
2258
|
+
if (!deps.dryRun) {
|
|
2259
|
+
recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
|
|
2260
|
+
event: "keep_alive_account_selection_failed",
|
|
2261
|
+
at: nowIso,
|
|
2262
|
+
account: current.account ?? null,
|
|
2263
|
+
reason: accountSelectionFailure,
|
|
2264
|
+
});
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
1996
2267
|
const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
1997
2268
|
latestLane.state === current.state &&
|
|
1998
2269
|
latestLane.keep_alive === true &&
|
|
@@ -2020,7 +2291,6 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
2020
2291
|
keep_alive_exhausted_at: undefined,
|
|
2021
2292
|
keep_alive_blocked_reason: undefined,
|
|
2022
2293
|
}), deps.dryRun);
|
|
2023
|
-
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
2024
2294
|
let queued;
|
|
2025
2295
|
if (seniorId) {
|
|
2026
2296
|
try {
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { RunawaySearchGuardConfig } from "./types.js";
|
|
2
|
+
export declare const RUNAWAY_TOOL_GUARD_FLAG = "lanes.runaway-tool-guard";
|
|
3
|
+
export interface RunawayProcessInfo {
|
|
4
|
+
pid: number;
|
|
5
|
+
ppid: number;
|
|
6
|
+
elapsedSeconds: number;
|
|
7
|
+
uid: number;
|
|
8
|
+
command: string;
|
|
9
|
+
argv: readonly string[];
|
|
10
|
+
cwd: string;
|
|
11
|
+
starttime: number | null;
|
|
12
|
+
}
|
|
13
|
+
export interface RunawayLaneProcess {
|
|
14
|
+
laneId: string;
|
|
15
|
+
agentPid: number;
|
|
16
|
+
agentStarttime: number;
|
|
17
|
+
}
|
|
18
|
+
export interface RunawayToolGuardInput {
|
|
19
|
+
hqRoot: string;
|
|
20
|
+
lanes: readonly RunawayLaneProcess[];
|
|
21
|
+
processes: readonly RunawayProcessInfo[];
|
|
22
|
+
config: RunawaySearchGuardConfig;
|
|
23
|
+
now: Date;
|
|
24
|
+
currentUid: number;
|
|
25
|
+
isCurrentProcess: (pid: number, expectedStarttime: number) => boolean;
|
|
26
|
+
signal: (pid: number, signal: NodeJS.Signals) => void;
|
|
27
|
+
recordEvent: (laneId: string, event: Record<string, unknown>) => void;
|
|
28
|
+
notifyLane: (laneId: string, text: string) => void;
|
|
29
|
+
reportError: (stage: "signal" | "record" | "notify", errorName: string) => void;
|
|
30
|
+
}
|
|
31
|
+
export interface RunawayToolGuardResult {
|
|
32
|
+
stopped: number;
|
|
33
|
+
}
|
|
34
|
+
export declare function runRunawayToolGuard(input: RunawayToolGuardInput): RunawayToolGuardResult;
|
|
35
|
+
/** Read the host process table once. argv and cwd are read only for search tools. */
|
|
36
|
+
export declare function listHostProcesses(): RunawayProcessInfo[];
|
|
37
|
+
export { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG } from "./config.js";
|
|
38
|
+
//# sourceMappingURL=runaway-tool-guard.d.ts.map
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { readProcStarttime } from "./spawn.js";
|
|
6
|
+
export const RUNAWAY_TOOL_GUARD_FLAG = "lanes.runaway-tool-guard";
|
|
7
|
+
const SEARCH_COMMANDS = new Set(["rg", "grep", "ugrep", "ag", "find"]);
|
|
8
|
+
const VALUE_OPTIONS = new Set([
|
|
9
|
+
"-A", "--after-context", "-B", "--before-context", "-C", "--context",
|
|
10
|
+
"-e", "--regexp", "-f", "--file", "-g", "--glob", "-t", "--type",
|
|
11
|
+
"-T", "--type-not", "-M", "--max-count", "-m", "--max-count",
|
|
12
|
+
"-j", "--threads", "--ignore-file", "--include", "--exclude",
|
|
13
|
+
"--exclude-dir", "--exclude-from", "--encoding", "--sort",
|
|
14
|
+
]);
|
|
15
|
+
function commandName(command) {
|
|
16
|
+
return path.basename(command).toLowerCase();
|
|
17
|
+
}
|
|
18
|
+
function hasNoIgnoreFlag(command, args) {
|
|
19
|
+
if (command === "rg" || command === "ag") {
|
|
20
|
+
return args.some((arg) => ["--no-ignore", "--no-ignore-vcs", "--unrestricted", "--skip-vcs-ignores"].includes(arg) ||
|
|
21
|
+
/^-[A-Za-z]*u[A-Za-z]*$/.test(arg));
|
|
22
|
+
}
|
|
23
|
+
if (command === "grep" || command === "ugrep") {
|
|
24
|
+
return args.some((arg) => ["--recursive", "--dereference-recursive"].includes(arg) ||
|
|
25
|
+
/^-[A-Za-z]*[rR][A-Za-z]*$/.test(arg));
|
|
26
|
+
}
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
function findDepth(args) {
|
|
30
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
31
|
+
if (args[index] !== "-maxdepth")
|
|
32
|
+
continue;
|
|
33
|
+
const value = Number(args[index + 1]);
|
|
34
|
+
return Number.isInteger(value) && value >= 0 ? value : null;
|
|
35
|
+
}
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
function positionalArgs(command, args) {
|
|
39
|
+
if (command === "find") {
|
|
40
|
+
let index = 0;
|
|
41
|
+
while (index < args.length) {
|
|
42
|
+
const arg = args[index];
|
|
43
|
+
if (["-H", "-L", "-P"].includes(arg) || /^-O\d+$/.test(arg)) {
|
|
44
|
+
index += 1;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
if (arg === "-D") {
|
|
48
|
+
index += 2;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (/^-D.+/.test(arg)) {
|
|
52
|
+
index += 1;
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
const paths = [];
|
|
58
|
+
for (; index < args.length; index += 1) {
|
|
59
|
+
const arg = args[index];
|
|
60
|
+
if (arg.startsWith("-"))
|
|
61
|
+
break;
|
|
62
|
+
paths.push(arg);
|
|
63
|
+
}
|
|
64
|
+
return paths;
|
|
65
|
+
}
|
|
66
|
+
const positionals = [];
|
|
67
|
+
let explicitPattern = false;
|
|
68
|
+
let afterTerminator = false;
|
|
69
|
+
let listFiles = false;
|
|
70
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
71
|
+
const arg = args[index];
|
|
72
|
+
if (afterTerminator) {
|
|
73
|
+
positionals.push(arg);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (arg === "--") {
|
|
77
|
+
afterTerminator = true;
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
if (arg === "--files") {
|
|
81
|
+
listFiles = true;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (VALUE_OPTIONS.has(arg)) {
|
|
85
|
+
if (["-e", "--regexp", "-f", "--file"].includes(arg))
|
|
86
|
+
explicitPattern = true;
|
|
87
|
+
index += 1;
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (arg.startsWith("--")) {
|
|
91
|
+
if (arg.startsWith("--regexp=") || arg.startsWith("--file="))
|
|
92
|
+
explicitPattern = true;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
if (arg.startsWith("-") && arg.length > 1) {
|
|
96
|
+
if (/^-e.+$/.test(arg) || /^-f.+$/.test(arg))
|
|
97
|
+
explicitPattern = true;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
positionals.push(arg);
|
|
101
|
+
}
|
|
102
|
+
if (listFiles || explicitPattern)
|
|
103
|
+
return positionals;
|
|
104
|
+
return positionals.slice(1);
|
|
105
|
+
}
|
|
106
|
+
function canonicalPath(candidate) {
|
|
107
|
+
try {
|
|
108
|
+
return fs.realpathSync(candidate);
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return path.resolve(candidate);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
function searchesBroadRoot(hqRoot, processInfo, command, args) {
|
|
115
|
+
const roots = new Set([
|
|
116
|
+
canonicalPath(hqRoot),
|
|
117
|
+
canonicalPath(path.join(hqRoot, "workspace")),
|
|
118
|
+
canonicalPath(os.homedir()),
|
|
119
|
+
path.parse(path.resolve("/")).root,
|
|
120
|
+
]);
|
|
121
|
+
const operands = positionalArgs(command, args);
|
|
122
|
+
const candidates = operands.length > 0
|
|
123
|
+
? operands.map((operand) => path.resolve(processInfo.cwd || ".", operand))
|
|
124
|
+
: [processInfo.cwd || "."];
|
|
125
|
+
return candidates.some((candidate) => roots.has(canonicalPath(candidate)));
|
|
126
|
+
}
|
|
127
|
+
function isUnboundedSearch(hqRoot, processInfo, findMaxdepthExempt) {
|
|
128
|
+
const command = commandName(processInfo.command);
|
|
129
|
+
if (!SEARCH_COMMANDS.has(command))
|
|
130
|
+
return false;
|
|
131
|
+
const args = processInfo.argv.slice(1);
|
|
132
|
+
if (command === "find") {
|
|
133
|
+
if (args.includes("-prune"))
|
|
134
|
+
return false;
|
|
135
|
+
const depth = findDepth(args);
|
|
136
|
+
if (depth !== null && depth <= findMaxdepthExempt)
|
|
137
|
+
return false;
|
|
138
|
+
return searchesBroadRoot(hqRoot, processInfo, command, args);
|
|
139
|
+
}
|
|
140
|
+
if (!hasNoIgnoreFlag(command, args))
|
|
141
|
+
return false;
|
|
142
|
+
return searchesBroadRoot(hqRoot, processInfo, command, args);
|
|
143
|
+
}
|
|
144
|
+
function laneAncestor(processInfo, processByPid, laneByPid, currentUid) {
|
|
145
|
+
let parentPid = processInfo.ppid;
|
|
146
|
+
const visited = new Set([processInfo.pid]);
|
|
147
|
+
for (let depth = 0; depth < 128; depth += 1) {
|
|
148
|
+
const laneId = laneByPid.get(parentPid);
|
|
149
|
+
if (laneId)
|
|
150
|
+
return laneId;
|
|
151
|
+
if (parentPid <= 1 || visited.has(parentPid))
|
|
152
|
+
return null;
|
|
153
|
+
visited.add(parentPid);
|
|
154
|
+
const parent = processByPid.get(parentPid);
|
|
155
|
+
if (!parent || parent.uid !== currentUid)
|
|
156
|
+
return null;
|
|
157
|
+
parentPid = parent.ppid;
|
|
158
|
+
}
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
function errorName(error) {
|
|
162
|
+
return error instanceof Error ? error.name : "UnknownError";
|
|
163
|
+
}
|
|
164
|
+
export function runRunawayToolGuard(input) {
|
|
165
|
+
const processByPid = new Map();
|
|
166
|
+
for (const processInfo of input.processes) {
|
|
167
|
+
if (Number.isSafeInteger(processInfo.pid) && processInfo.pid > 1) {
|
|
168
|
+
processByPid.set(processInfo.pid, processInfo);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
const laneByPid = new Map();
|
|
172
|
+
const ambiguousPids = new Set();
|
|
173
|
+
for (const lane of input.lanes) {
|
|
174
|
+
const agent = processByPid.get(lane.agentPid);
|
|
175
|
+
if (!agent ||
|
|
176
|
+
!Number.isSafeInteger(lane.agentStarttime) ||
|
|
177
|
+
agent.starttime !== lane.agentStarttime ||
|
|
178
|
+
agent.uid !== input.currentUid ||
|
|
179
|
+
agent.pid === agent.ppid)
|
|
180
|
+
continue;
|
|
181
|
+
if (laneByPid.has(lane.agentPid))
|
|
182
|
+
ambiguousPids.add(lane.agentPid);
|
|
183
|
+
else
|
|
184
|
+
laneByPid.set(lane.agentPid, lane.laneId);
|
|
185
|
+
}
|
|
186
|
+
for (const pid of ambiguousPids)
|
|
187
|
+
laneByPid.delete(pid);
|
|
188
|
+
let stopped = 0;
|
|
189
|
+
for (const processInfo of input.processes) {
|
|
190
|
+
if (processInfo.uid !== input.currentUid ||
|
|
191
|
+
!Number.isFinite(processInfo.elapsedSeconds) ||
|
|
192
|
+
processInfo.elapsedSeconds <= input.config.max_age_seconds ||
|
|
193
|
+
processInfo.starttime === null ||
|
|
194
|
+
!isUnboundedSearch(input.hqRoot, processInfo, input.config.find_maxdepth_exempt))
|
|
195
|
+
continue;
|
|
196
|
+
const laneId = laneAncestor(processInfo, processByPid, laneByPid, input.currentUid);
|
|
197
|
+
if (!laneId || !input.isCurrentProcess(processInfo.pid, processInfo.starttime))
|
|
198
|
+
continue;
|
|
199
|
+
try {
|
|
200
|
+
input.signal(processInfo.pid, "SIGTERM");
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
if (error.code !== "ESRCH") {
|
|
204
|
+
input.reportError("signal", errorName(error));
|
|
205
|
+
}
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
const command = commandName(processInfo.command);
|
|
209
|
+
const durationSeconds = Math.floor(processInfo.elapsedSeconds);
|
|
210
|
+
stopped += 1;
|
|
211
|
+
try {
|
|
212
|
+
input.recordEvent(laneId, {
|
|
213
|
+
event: "runaway_search_stopped",
|
|
214
|
+
at: input.now.toISOString(),
|
|
215
|
+
pid: processInfo.pid,
|
|
216
|
+
command,
|
|
217
|
+
duration_seconds: durationSeconds,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
input.reportError("record", errorName(error));
|
|
222
|
+
}
|
|
223
|
+
try {
|
|
224
|
+
input.notifyLane(laneId, `runaway-search-guard stopped ${command} after ${durationSeconds}s. Narrow the search to the smallest repo path that holds the answer and keep ignore rules on.`);
|
|
225
|
+
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
input.reportError("notify", errorName(error));
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return { stopped };
|
|
231
|
+
}
|
|
232
|
+
/** Read the host process table once. argv and cwd are read only for search tools. */
|
|
233
|
+
export function listHostProcesses() {
|
|
234
|
+
if (process.platform !== "linux")
|
|
235
|
+
return [];
|
|
236
|
+
let output;
|
|
237
|
+
try {
|
|
238
|
+
output = execFileSync("ps", ["-eo", "pid=,ppid=,etimes=,uid=,comm="], {
|
|
239
|
+
encoding: "utf8",
|
|
240
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
241
|
+
timeout: 2_000,
|
|
242
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
243
|
+
windowsHide: true,
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
throw new Error("ps process-table read failed");
|
|
248
|
+
}
|
|
249
|
+
const processes = [];
|
|
250
|
+
for (const line of output.split("\n")) {
|
|
251
|
+
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(.+)$/);
|
|
252
|
+
if (!match)
|
|
253
|
+
continue;
|
|
254
|
+
const pid = Number(match[1]);
|
|
255
|
+
const ppid = Number(match[2]);
|
|
256
|
+
const elapsedSeconds = Number(match[3]);
|
|
257
|
+
const uid = Number(match[4]);
|
|
258
|
+
const command = match[5].trim();
|
|
259
|
+
if (![pid, ppid, elapsedSeconds, uid].every(Number.isSafeInteger) || !command)
|
|
260
|
+
continue;
|
|
261
|
+
let argv = [];
|
|
262
|
+
let cwd = "";
|
|
263
|
+
let starttime = null;
|
|
264
|
+
if (SEARCH_COMMANDS.has(commandName(command))) {
|
|
265
|
+
try {
|
|
266
|
+
const commandLine = fs.readFileSync(`/proc/${pid}/cmdline`);
|
|
267
|
+
argv = commandLine.toString("utf8").split("\0");
|
|
268
|
+
if (argv.at(-1) === "")
|
|
269
|
+
argv.pop();
|
|
270
|
+
cwd = fs.readlinkSync(`/proc/${pid}/cwd`);
|
|
271
|
+
starttime = readProcStarttime(pid);
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
argv = [];
|
|
275
|
+
cwd = "";
|
|
276
|
+
starttime = null;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
processes.push({ pid, ppid, elapsedSeconds, uid, command, argv, cwd, starttime });
|
|
280
|
+
}
|
|
281
|
+
return processes;
|
|
282
|
+
}
|
|
283
|
+
export { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG } from "./config.js";
|
|
284
|
+
//# sourceMappingURL=runaway-tool-guard.js.map
|
|
@@ -500,9 +500,8 @@ export interface LaneRecord {
|
|
|
500
500
|
/** Eligible accounts for a queue entry created without an explicit account. */
|
|
501
501
|
admission_accounts?: string[];
|
|
502
502
|
/**
|
|
503
|
-
*
|
|
504
|
-
*
|
|
505
|
-
* like the other histories.
|
|
503
|
+
* Moves of the lane's session to another account of the same provider,
|
|
504
|
+
* newest last and capped like the other histories.
|
|
506
505
|
*/
|
|
507
506
|
account_history?: AccountHistoryEntry[];
|
|
508
507
|
/** Bounded audit history for explicitly authorized resumes from failed. */
|
|
@@ -665,7 +664,7 @@ export interface AccountHistoryEntry {
|
|
|
665
664
|
from: string | null;
|
|
666
665
|
to: string;
|
|
667
666
|
at: string;
|
|
668
|
-
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "operator_move";
|
|
667
|
+
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "keep_alive_resume" | "operator_move";
|
|
669
668
|
}
|
|
670
669
|
export interface FailedResumeHistoryEntry {
|
|
671
670
|
at: string;
|
|
@@ -891,6 +890,12 @@ export interface TrackerChecksConfig {
|
|
|
891
890
|
kpi_value_max_age_ms: number;
|
|
892
891
|
directive_ack_timeout_ms: number;
|
|
893
892
|
}
|
|
893
|
+
export interface RunawaySearchGuardConfig {
|
|
894
|
+
/** Wall-time threshold before an unbounded descendant search is stopped. */
|
|
895
|
+
max_age_seconds: number;
|
|
896
|
+
/** `find` searches at this depth or less are left alone. */
|
|
897
|
+
find_maxdepth_exempt: number;
|
|
898
|
+
}
|
|
894
899
|
export interface LanesConfig {
|
|
895
900
|
schema_version: typeof CONFIG_SCHEMA_VERSION | string;
|
|
896
901
|
providers: Record<LaneProvider, ProviderConfig>;
|
|
@@ -899,6 +904,7 @@ export interface LanesConfig {
|
|
|
899
904
|
tracker: {
|
|
900
905
|
checks: TrackerChecksConfig;
|
|
901
906
|
};
|
|
907
|
+
runaway_search_guard?: RunawaySearchGuardConfig;
|
|
902
908
|
/** Seconds an unanswered question waits at a session audience before advancing. */
|
|
903
909
|
session_question_timeout_secs: number;
|
|
904
910
|
/** Age in hours after which an open manager blocker must be re-verified. */
|
|
@@ -130,6 +130,8 @@ export declare function watcherEventsPath(hqRoot: string, laneId: string): strin
|
|
|
130
130
|
export declare function watcherOutLogPath(hqRoot: string, laneId: string): string;
|
|
131
131
|
export declare function appendCappedLog(filePath: string, line: string, maxBytes?: number): void;
|
|
132
132
|
export declare function writeWatcherDecision(filePath: string, record: Record<string, unknown>, maxBytes?: number): void;
|
|
133
|
+
/** Append a decision to the shared event stream without a read-modify-write. */
|
|
134
|
+
export declare function appendWatcherEvent(filePath: string, record: Record<string, unknown>, appendLine?: (target: string, line: string) => void): void;
|
|
133
135
|
export declare function watcherLastSignalPath(hqRoot: string, laneId: string): string;
|
|
134
136
|
export declare function appendWatcherSignal(hqRoot: string, laneId: string, signal: WatcherSignal): void;
|
|
135
137
|
/**
|
|
@@ -223,6 +223,16 @@ export function writeWatcherDecision(filePath, record, maxBytes = WATCHER_OUT_MA
|
|
|
223
223
|
}
|
|
224
224
|
appendCappedLog(filePath, JSON.stringify(compact), maxBytes);
|
|
225
225
|
}
|
|
226
|
+
/** Append a decision to the shared event stream without a read-modify-write. */
|
|
227
|
+
export function appendWatcherEvent(filePath, record, appendLine = (target, line) => fs.appendFileSync(target, line, "utf8")) {
|
|
228
|
+
const compact = {};
|
|
229
|
+
for (const [key, value] of Object.entries(record)) {
|
|
230
|
+
if (value !== undefined)
|
|
231
|
+
compact[key] = value;
|
|
232
|
+
}
|
|
233
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
234
|
+
appendLine(filePath, `${JSON.stringify(compact)}\n`);
|
|
235
|
+
}
|
|
226
236
|
export function watcherLastSignalPath(hqRoot, laneId) {
|
|
227
237
|
return path.join(laneRunDir(hqRoot, laneId), "watcher", "last-signal.json");
|
|
228
238
|
}
|