harnery 0.36.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commander.js +4 -0
- package/dist/commands/agents.d.ts.map +1 -1
- package/dist/commands/agents.js +87 -46
- package/dist/commands/artifacts.d.ts.map +1 -1
- package/dist/commands/artifacts.js +26 -1
- package/dist/commands/browse.d.ts.map +1 -1
- package/dist/commands/browse.js +11 -0
- package/dist/commands/diagnostics.d.ts +4 -0
- package/dist/commands/diagnostics.d.ts.map +1 -0
- package/dist/commands/diagnostics.js +229 -0
- package/dist/commands/files.d.ts +12 -0
- package/dist/commands/files.d.ts.map +1 -1
- package/dist/commands/files.js +54 -1
- package/dist/commands/governor.d.ts.map +1 -1
- package/dist/commands/governor.js +4 -0
- package/dist/commands/ledger-v3.d.ts.map +1 -1
- package/dist/commands/ledger-v3.js +71 -2
- package/dist/commands/qa-run.d.ts +4 -0
- package/dist/commands/qa-run.d.ts.map +1 -0
- package/dist/commands/qa-run.js +141 -0
- package/dist/commands/resources.d.ts +4 -0
- package/dist/commands/resources.d.ts.map +1 -0
- package/dist/commands/resources.js +29 -0
- package/dist/commands/supervisor.d.ts +4 -0
- package/dist/commands/supervisor.d.ts.map +1 -0
- package/dist/commands/supervisor.js +101 -0
- package/dist/commands/web.d.ts.map +1 -1
- package/dist/commands/web.js +17 -0
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +4 -0
- package/dist/core/agents/cli.js +7 -3
- package/dist/core/agents/health.d.ts +17 -0
- package/dist/core/agents/health.d.ts.map +1 -0
- package/dist/core/agents/health.js +109 -0
- package/dist/core/agents/mailbox.d.ts +92 -0
- package/dist/core/agents/mailbox.d.ts.map +1 -0
- package/dist/core/agents/mailbox.js +325 -0
- package/dist/core/agents/reconcile-coordination-v3.d.ts +44 -0
- package/dist/core/agents/reconcile-coordination-v3.d.ts.map +1 -0
- package/dist/core/agents/reconcile-coordination-v3.js +58 -0
- package/dist/core/agents/render/prompt-context.d.ts +5 -0
- package/dist/core/agents/render/prompt-context.d.ts.map +1 -1
- package/dist/core/agents/render/prompt-context.js +34 -13
- package/dist/core/agents/render/session-context.d.ts +8 -0
- package/dist/core/agents/render/session-context.d.ts.map +1 -1
- package/dist/core/agents/render/session-context.js +34 -3
- package/dist/core/agents/rules/commit-conflict.d.ts.map +1 -1
- package/dist/core/agents/rules/commit-conflict.js +50 -1
- package/dist/core/agents/rules/stop-hook.d.ts.map +1 -1
- package/dist/core/agents/rules/stop-hook.js +6 -10
- package/dist/core/agents/session-finalizer-v3.js +4 -1
- package/dist/core/artifacts/index.d.ts +26 -1
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +146 -6
- package/dist/core/config.d.ts +12 -0
- package/dist/core/config.d.ts.map +1 -1
- package/dist/core/config.js +43 -0
- package/dist/core/diagnostics/advice.d.ts +9 -0
- package/dist/core/diagnostics/advice.d.ts.map +1 -0
- package/dist/core/diagnostics/advice.js +112 -0
- package/dist/core/diagnostics/bundle.d.ts +35 -0
- package/dist/core/diagnostics/bundle.d.ts.map +1 -0
- package/dist/core/diagnostics/bundle.js +874 -0
- package/dist/core/diagnostics/comparison.d.ts +5 -0
- package/dist/core/diagnostics/comparison.d.ts.map +1 -0
- package/dist/core/diagnostics/comparison.js +444 -0
- package/dist/core/diagnostics/contract.d.ts +245 -0
- package/dist/core/diagnostics/contract.d.ts.map +1 -0
- package/dist/core/diagnostics/contract.js +18 -0
- package/dist/core/diagnostics/identity.d.ts +3 -0
- package/dist/core/diagnostics/identity.d.ts.map +1 -0
- package/dist/core/diagnostics/identity.js +10 -0
- package/dist/core/diagnostics/index.d.ts +8 -0
- package/dist/core/diagnostics/index.d.ts.map +1 -0
- package/dist/core/diagnostics/index.js +7 -0
- package/dist/core/diagnostics/replay.d.ts +8 -0
- package/dist/core/diagnostics/replay.d.ts.map +1 -0
- package/dist/core/diagnostics/replay.js +213 -0
- package/dist/core/diagnostics/sanitize.d.ts +9 -0
- package/dist/core/diagnostics/sanitize.d.ts.map +1 -0
- package/dist/core/diagnostics/sanitize.js +84 -0
- package/dist/core/events/legacy-storage/compression.d.ts +13 -0
- package/dist/core/events/legacy-storage/compression.d.ts.map +1 -0
- package/dist/core/events/legacy-storage/compression.js +107 -0
- package/dist/core/events/legacy-storage/index.d.ts +1 -0
- package/dist/core/events/legacy-storage/index.d.ts.map +1 -1
- package/dist/core/events/legacy-storage/index.js +1 -0
- package/dist/core/events/v3/archive-retention.d.ts +33 -0
- package/dist/core/events/v3/archive-retention.d.ts.map +1 -0
- package/dist/core/events/v3/archive-retention.js +195 -0
- package/dist/core/events/v3/capabilities.d.ts +1 -1
- package/dist/core/events/v3/capabilities.d.ts.map +1 -1
- package/dist/core/events/v3/capabilities.js +1 -0
- package/dist/core/events/v3/index.d.ts +1 -0
- package/dist/core/events/v3/index.d.ts.map +1 -1
- package/dist/core/events/v3/index.js +1 -0
- package/dist/core/events/v3/live-routing.d.ts.map +1 -1
- package/dist/core/events/v3/live-routing.js +1 -0
- package/dist/core/events/v3/producers/hook-base.d.ts +1 -1
- package/dist/core/events/v3/producers/hook-base.d.ts.map +1 -1
- package/dist/core/events/v3/producers/hook-base.js +13 -2
- package/dist/core/events/v3/producers/intake.d.ts.map +1 -1
- package/dist/core/events/v3/producers/intake.js +1 -0
- package/dist/core/events/v3/producers/recorder.d.ts +16 -0
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +88 -1
- package/dist/core/hooks/adapter/output.d.ts +8 -0
- package/dist/core/hooks/adapter/output.d.ts.map +1 -1
- package/dist/core/hooks/adapter/output.js +11 -1
- package/dist/core/hooks/cli.js +183 -192
- package/dist/core/hooks/health.d.ts +59 -0
- package/dist/core/hooks/health.d.ts.map +1 -0
- package/dist/core/hooks/health.js +84 -0
- package/dist/core/resources/contract.d.ts +90 -0
- package/dist/core/resources/contract.d.ts.map +1 -0
- package/dist/core/resources/contract.js +2 -0
- package/dist/core/resources/index.d.ts +6 -0
- package/dist/core/resources/index.d.ts.map +1 -0
- package/dist/core/resources/index.js +5 -0
- package/dist/core/resources/sampler.d.ts +27 -0
- package/dist/core/resources/sampler.d.ts.map +1 -0
- package/dist/core/resources/sampler.js +357 -0
- package/dist/core/resources/service-status.d.ts +3 -0
- package/dist/core/resources/service-status.d.ts.map +1 -0
- package/dist/core/resources/service-status.js +48 -0
- package/dist/core/resources/service.d.ts +26 -0
- package/dist/core/resources/service.d.ts.map +1 -0
- package/dist/core/resources/service.js +292 -0
- package/dist/core/resources/storage.d.ts +13 -0
- package/dist/core/resources/storage.d.ts.map +1 -0
- package/dist/core/resources/storage.js +30 -0
- package/dist/core/storage/atomic-json.d.ts +3 -0
- package/dist/core/storage/atomic-json.d.ts.map +1 -0
- package/dist/core/storage/atomic-json.js +15 -0
- package/dist/core/storage/builtins.d.ts.map +1 -1
- package/dist/core/storage/builtins.js +45 -6
- package/dist/core/storage/logger.d.ts +2 -0
- package/dist/core/storage/logger.d.ts.map +1 -1
- package/dist/core/storage/logger.js +2 -0
- package/dist/core/storage/query.d.ts +14 -0
- package/dist/core/storage/query.d.ts.map +1 -1
- package/dist/core/storage/query.js +51 -0
- package/dist/core/supervisor/activity.d.ts +4 -0
- package/dist/core/supervisor/activity.d.ts.map +1 -0
- package/dist/core/supervisor/activity.js +141 -0
- package/dist/core/supervisor/contract.d.ts +268 -0
- package/dist/core/supervisor/contract.d.ts.map +1 -0
- package/dist/core/supervisor/contract.js +35 -0
- package/dist/core/supervisor/explanations.d.ts +4 -0
- package/dist/core/supervisor/explanations.d.ts.map +1 -0
- package/dist/core/supervisor/explanations.js +71 -0
- package/dist/core/supervisor/findings.d.ts +20 -0
- package/dist/core/supervisor/findings.d.ts.map +1 -0
- package/dist/core/supervisor/findings.js +399 -0
- package/dist/core/supervisor/history.d.ts +9 -0
- package/dist/core/supervisor/history.d.ts.map +1 -0
- package/dist/core/supervisor/history.js +46 -0
- package/dist/core/supervisor/hook-health-alerts.d.ts +14 -0
- package/dist/core/supervisor/hook-health-alerts.d.ts.map +1 -0
- package/dist/core/supervisor/hook-health-alerts.js +70 -0
- package/dist/core/supervisor/hook-health-storage.d.ts +5 -0
- package/dist/core/supervisor/hook-health-storage.d.ts.map +1 -0
- package/dist/core/supervisor/hook-health-storage.js +24 -0
- package/dist/core/supervisor/hook-health.d.ts +72 -0
- package/dist/core/supervisor/hook-health.d.ts.map +1 -0
- package/dist/core/supervisor/hook-health.js +203 -0
- package/dist/core/supervisor/hooks.d.ts +10 -0
- package/dist/core/supervisor/hooks.d.ts.map +1 -0
- package/dist/core/supervisor/hooks.js +28 -0
- package/dist/core/supervisor/index.d.ts +15 -0
- package/dist/core/supervisor/index.d.ts.map +1 -0
- package/dist/core/supervisor/index.js +14 -0
- package/dist/core/supervisor/log-feed.d.ts +9 -0
- package/dist/core/supervisor/log-feed.d.ts.map +1 -0
- package/dist/core/supervisor/log-feed.js +103 -0
- package/dist/core/supervisor/service.d.ts +29 -0
- package/dist/core/supervisor/service.d.ts.map +1 -0
- package/dist/core/supervisor/service.js +450 -0
- package/dist/core/supervisor/services.d.ts +11 -0
- package/dist/core/supervisor/services.d.ts.map +1 -0
- package/dist/core/supervisor/services.js +155 -0
- package/dist/core/supervisor/status.d.ts +3 -0
- package/dist/core/supervisor/status.d.ts.map +1 -0
- package/dist/core/supervisor/status.js +48 -0
- package/dist/core/supervisor/storage.d.ts +30 -0
- package/dist/core/supervisor/storage.d.ts.map +1 -0
- package/dist/core/supervisor/storage.js +73 -0
- package/dist/core/supervisor/timeline.d.ts +3 -0
- package/dist/core/supervisor/timeline.d.ts.map +1 -0
- package/dist/core/supervisor/timeline.js +87 -0
- package/dist/core/workflow/admission.d.ts +23 -0
- package/dist/core/workflow/admission.d.ts.map +1 -0
- package/dist/core/workflow/admission.js +136 -0
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +77 -2
- package/dist/core/workflow/index.d.ts +3 -2
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -1
- package/dist/core/workflow/proof.d.ts +2 -1
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +39 -1
- package/dist/core/workflow/run-state.d.ts +3 -2
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +8 -1
- package/dist/core/workflow/types.d.ts +33 -1
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/types.js +2 -1
- package/dist/lib/browser/client.d.ts +5 -0
- package/dist/lib/browser/client.d.ts.map +1 -1
- package/dist/lib/browser/client.js +1 -0
- package/dist/lib/browser/index.d.ts +2 -0
- package/dist/lib/browser/index.d.ts.map +1 -1
- package/dist/lib/browser/index.js +2 -0
- package/dist/lib/browser/qa-run-contracts.d.ts +173 -0
- package/dist/lib/browser/qa-run-contracts.d.ts.map +1 -0
- package/dist/lib/browser/qa-run-contracts.js +226 -0
- package/dist/lib/browser/qa-run.d.ts +51 -0
- package/dist/lib/browser/qa-run.d.ts.map +1 -0
- package/dist/lib/browser/qa-run.js +684 -0
- package/dist/lib/coord-root-id.d.ts +9 -0
- package/dist/lib/coord-root-id.d.ts.map +1 -0
- package/dist/lib/coord-root-id.js +17 -0
- package/dist/lib/instructions/apply.d.ts +3 -1
- package/dist/lib/instructions/apply.d.ts.map +1 -1
- package/dist/lib/instructions/apply.js +74 -3
- package/dist/lib/instructions/templates.d.ts.map +1 -1
- package/dist/lib/instructions/templates.js +6 -0
- package/package.json +12 -1
- package/src/commander.ts +26 -0
- package/src/commands/agents.ts +102 -49
- package/src/commands/artifacts.ts +40 -2
- package/src/commands/browse.ts +15 -0
- package/src/commands/diagnostics.ts +271 -0
- package/src/commands/files.ts +102 -2
- package/src/commands/governor.ts +8 -0
- package/src/commands/ledger-v3.ts +82 -0
- package/src/commands/qa-run.ts +184 -0
- package/src/commands/resources.ts +38 -0
- package/src/commands/supervisor.ts +141 -0
- package/src/commands/web.ts +19 -0
- package/src/commands/workflow.ts +8 -0
- package/src/core/agents/cli.ts +7 -3
- package/src/core/agents/health.ts +142 -0
- package/src/core/agents/mailbox.ts +373 -0
- package/src/core/agents/reconcile-coordination-v3.ts +77 -0
- package/src/core/agents/render/prompt-context.ts +42 -12
- package/src/core/agents/render/session-context.ts +37 -3
- package/src/core/agents/rules/commit-conflict.ts +58 -1
- package/src/core/agents/rules/stop-hook.ts +6 -12
- package/src/core/agents/session-finalizer-v3.ts +4 -1
- package/src/core/artifacts/index.ts +196 -10
- package/src/core/config.ts +111 -2
- package/src/core/diagnostics/advice.ts +147 -0
- package/src/core/diagnostics/bundle.ts +1142 -0
- package/src/core/diagnostics/comparison.ts +565 -0
- package/src/core/diagnostics/contract.ts +311 -0
- package/src/core/diagnostics/identity.ts +11 -0
- package/src/core/diagnostics/index.ts +7 -0
- package/src/core/diagnostics/replay.ts +314 -0
- package/src/core/diagnostics/sanitize.ts +97 -0
- package/src/core/events/legacy-storage/compression.ts +130 -0
- package/src/core/events/legacy-storage/index.ts +1 -0
- package/src/core/events/v3/archive-retention.ts +251 -0
- package/src/core/events/v3/capabilities.ts +2 -0
- package/src/core/events/v3/index.ts +9 -0
- package/src/core/events/v3/live-routing.ts +1 -0
- package/src/core/events/v3/producers/hook-base.ts +14 -2
- package/src/core/events/v3/producers/intake.ts +1 -0
- package/src/core/events/v3/producers/recorder.ts +110 -1
- package/src/core/hooks/adapter/output.ts +12 -1
- package/src/core/hooks/cli.ts +192 -215
- package/src/core/hooks/health.ts +146 -0
- package/src/core/resources/contract.ts +98 -0
- package/src/core/resources/index.ts +5 -0
- package/src/core/resources/sampler.ts +458 -0
- package/src/core/resources/service-status.ts +62 -0
- package/src/core/resources/service.ts +337 -0
- package/src/core/resources/storage.ts +44 -0
- package/src/core/storage/atomic-json.ts +15 -0
- package/src/core/storage/builtins.ts +62 -6
- package/src/core/storage/logger.ts +2 -0
- package/src/core/storage/query.ts +58 -0
- package/src/core/supervisor/activity.ts +193 -0
- package/src/core/supervisor/contract.ts +312 -0
- package/src/core/supervisor/explanations.ts +104 -0
- package/src/core/supervisor/findings.ts +788 -0
- package/src/core/supervisor/history.ts +61 -0
- package/src/core/supervisor/hook-health-alerts.ts +108 -0
- package/src/core/supervisor/hook-health-storage.ts +25 -0
- package/src/core/supervisor/hook-health.ts +301 -0
- package/src/core/supervisor/hooks.ts +35 -0
- package/src/core/supervisor/index.ts +14 -0
- package/src/core/supervisor/log-feed.ts +131 -0
- package/src/core/supervisor/service.ts +511 -0
- package/src/core/supervisor/services.ts +203 -0
- package/src/core/supervisor/status.ts +59 -0
- package/src/core/supervisor/storage.ts +126 -0
- package/src/core/supervisor/timeline.ts +133 -0
- package/src/core/workflow/admission.ts +215 -0
- package/src/core/workflow/engine.ts +93 -1
- package/src/core/workflow/index.ts +10 -0
- package/src/core/workflow/proof.ts +54 -1
- package/src/core/workflow/run-state.ts +13 -1
- package/src/core/workflow/types.ts +39 -1
- package/src/lib/browser/client.ts +6 -0
- package/src/lib/browser/index.ts +29 -0
- package/src/lib/browser/qa-run-contracts.ts +380 -0
- package/src/lib/browser/qa-run.ts +784 -0
- package/src/lib/coord-root-id.ts +23 -0
- package/src/lib/instructions/apply.ts +81 -2
- package/src/lib/instructions/templates.ts +6 -0
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { SUPERVISOR_DIAGNOSTIC_LIMITS, SUPERVISOR_FINDING_POLICY, SUPERVISOR_FINDING_SCHEMA_VERSION, } from "./contract.js";
|
|
3
|
+
import { evaluateHookHealthAlerts } from "./hook-health-alerts.js";
|
|
4
|
+
const GIB = 1_024 * 1_024 * 1_024;
|
|
5
|
+
const MIB = 1_024 * 1_024;
|
|
6
|
+
const MACHINE_CPU_THRESHOLD = 90;
|
|
7
|
+
const MACHINE_MEMORY_THRESHOLD = 85;
|
|
8
|
+
const MACHINE_SWAP_THRESHOLD = 75;
|
|
9
|
+
const COLLECTOR_CPU_THRESHOLD_MS = 50;
|
|
10
|
+
const PROCESS_MEMORY_THRESHOLD = GIB;
|
|
11
|
+
const GROUP_MEMORY_THRESHOLD = 2 * GIB;
|
|
12
|
+
const OWNER_PROCESS_THRESHOLD = 50;
|
|
13
|
+
const MEMORY_GROWTH_MIN_BYTES = 256 * MIB;
|
|
14
|
+
const MEMORY_GROWTH_RATIO = 1.3;
|
|
15
|
+
const MEMORY_GROWTH_WINDOW_MS = 5 * 60_000;
|
|
16
|
+
/** Pure deterministic evaluator used by the live supervisor and replay. */
|
|
17
|
+
export function evaluateSupervisorFindings(input) {
|
|
18
|
+
return updateSupervisorFindings(input).active;
|
|
19
|
+
}
|
|
20
|
+
export function updateSupervisorFindings(input) {
|
|
21
|
+
const observedAt = (input.now ?? new Date(input.resource.sampled_at)).toISOString();
|
|
22
|
+
const previous = validFindings(input.previous)
|
|
23
|
+
? input.previous
|
|
24
|
+
: {
|
|
25
|
+
schema_version: SUPERVISOR_FINDING_SCHEMA_VERSION,
|
|
26
|
+
max_findings: SUPERVISOR_DIAGNOSTIC_LIMITS.max_findings,
|
|
27
|
+
active: [],
|
|
28
|
+
transitions: [],
|
|
29
|
+
};
|
|
30
|
+
const candidates = evaluateCandidates(input)
|
|
31
|
+
.sort((left, right) => severityRank(right.severity) - severityRank(left.severity))
|
|
32
|
+
.slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_active_findings);
|
|
33
|
+
const priorActive = new Map(previous.active.map((entry) => [entry.fingerprint, entry]));
|
|
34
|
+
const current = new Set();
|
|
35
|
+
const active = [];
|
|
36
|
+
const transitions = new Map(previous.transitions.map((entry) => [entry.id, entry]));
|
|
37
|
+
for (const candidate of candidates) {
|
|
38
|
+
const fingerprint = findingFingerprint(candidate);
|
|
39
|
+
current.add(fingerprint);
|
|
40
|
+
const prior = priorActive.get(fingerprint) ?? recentResolved(previous, fingerprint, observedAt);
|
|
41
|
+
const openedAt = prior?.opened_at ?? candidate.primary_source.observed_at;
|
|
42
|
+
const occurrenceCount = prior
|
|
43
|
+
? prior.state === "resolved"
|
|
44
|
+
? prior.occurrence_count + 1
|
|
45
|
+
: prior.occurrence_count
|
|
46
|
+
: 1;
|
|
47
|
+
const peak = peakEvidence(prior, candidate.evidence, observedAt);
|
|
48
|
+
const finding = {
|
|
49
|
+
schema_version: SUPERVISOR_FINDING_SCHEMA_VERSION,
|
|
50
|
+
id: deterministicId("find", `${fingerprint}\u0000${openedAt}`),
|
|
51
|
+
fingerprint,
|
|
52
|
+
source_kind: candidate.source_kind,
|
|
53
|
+
finding_kind: candidate.finding_kind,
|
|
54
|
+
severity: candidate.severity,
|
|
55
|
+
state: "opened",
|
|
56
|
+
scope_kind: candidate.scope_kind,
|
|
57
|
+
scope_id: candidate.scope_id,
|
|
58
|
+
summary: candidate.summary,
|
|
59
|
+
opened_at: openedAt,
|
|
60
|
+
observed_at: observedAt,
|
|
61
|
+
occurrence_count: occurrenceCount,
|
|
62
|
+
...peak,
|
|
63
|
+
...(candidate.attribution ? { attribution: candidate.attribution } : {}),
|
|
64
|
+
...(candidate.workload_context ? { workload_context: candidate.workload_context } : {}),
|
|
65
|
+
primary_source: candidate.primary_source,
|
|
66
|
+
evidence: candidate.evidence.slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_evidence_per_finding),
|
|
67
|
+
capabilities: boundCapabilities(candidate.capabilities),
|
|
68
|
+
};
|
|
69
|
+
active.push(finding);
|
|
70
|
+
transitions.set(finding.id, finding);
|
|
71
|
+
}
|
|
72
|
+
for (const prior of priorActive.values()) {
|
|
73
|
+
if (current.has(prior.fingerprint))
|
|
74
|
+
continue;
|
|
75
|
+
transitions.set(prior.id, {
|
|
76
|
+
...prior,
|
|
77
|
+
state: "resolved",
|
|
78
|
+
observed_at: observedAt,
|
|
79
|
+
resolved_at: observedAt,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
schema_version: SUPERVISOR_FINDING_SCHEMA_VERSION,
|
|
84
|
+
max_findings: SUPERVISOR_DIAGNOSTIC_LIMITS.max_findings,
|
|
85
|
+
active,
|
|
86
|
+
transitions: [...transitions.values()]
|
|
87
|
+
.sort((left, right) => left.observed_at.localeCompare(right.observed_at))
|
|
88
|
+
.slice(-SUPERVISOR_DIAGNOSTIC_LIMITS.max_findings),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function evaluateCandidates(input) {
|
|
92
|
+
const out = [];
|
|
93
|
+
const resourceCapability = {
|
|
94
|
+
source_kind: "resource.snapshot",
|
|
95
|
+
state: input.resource.support.state,
|
|
96
|
+
...(input.resource.support.reason
|
|
97
|
+
? { reason_code: "sampler-degraded", detail: input.resource.support.reason }
|
|
98
|
+
: {}),
|
|
99
|
+
};
|
|
100
|
+
const source = resourceSource(input.resource, resourceCapability.state);
|
|
101
|
+
const addMachine = (findingKind, severity, summary, value, unit) => out.push(candidate(findingKind, severity, "machine", "local", summary, source, value, unit, [
|
|
102
|
+
resourceCapability,
|
|
103
|
+
]));
|
|
104
|
+
const machine = input.resource.machine;
|
|
105
|
+
if (resourceCapability.state !== "supported") {
|
|
106
|
+
out.push(capabilityCandidate(resourceCapability, source));
|
|
107
|
+
}
|
|
108
|
+
if ((machine.cpu_percent ?? 0) >= MACHINE_CPU_THRESHOLD)
|
|
109
|
+
addMachine("machine.cpu-pressure", "warning", `Machine CPU is ${machine.cpu_percent}%.`, machine.cpu_percent, "percent");
|
|
110
|
+
if ((machine.memory_percent ?? 0) >= MACHINE_MEMORY_THRESHOLD)
|
|
111
|
+
addMachine("machine.memory-pressure", "critical", `Machine memory is ${machine.memory_percent}%.`, machine.memory_percent, "percent");
|
|
112
|
+
const swapPercent = machine.swap_total_bytes && machine.swap_used_bytes !== null
|
|
113
|
+
? round1((machine.swap_used_bytes / machine.swap_total_bytes) * 100)
|
|
114
|
+
: null;
|
|
115
|
+
if ((swapPercent ?? 0) >= MACHINE_SWAP_THRESHOLD)
|
|
116
|
+
addMachine("machine.swap-pressure", "warning", `Machine swap is ${swapPercent}% used.`, swapPercent, "percent");
|
|
117
|
+
if (input.resource.collector_cpu_ms >= COLLECTOR_CPU_THRESHOLD_MS)
|
|
118
|
+
out.push(candidate("supervisor.collector-overhead", "warning", "service", "supervisor", `Supervisor resource collection used ${input.resource.collector_cpu_ms} ms of CPU.`, source, input.resource.collector_cpu_ms, "milliseconds", [resourceCapability]));
|
|
119
|
+
for (const service of input.services.filter((entry) => entry.state === "stale").slice(0, 5)) {
|
|
120
|
+
const serviceSource = stableSource("service.health", service.id, service.heartbeat_at ?? input.resource.sampled_at, "supported");
|
|
121
|
+
out.push(candidate("service.stale", "warning", "service", service.id, serviceStaleSummary(service), serviceSource, undefined, undefined, [{ source_kind: "service.health", state: "supported" }]));
|
|
122
|
+
}
|
|
123
|
+
for (const hook of input.hooks.filter((entry) => entry.long_running).slice(0, 5)) {
|
|
124
|
+
const hookSource = stableSource("hook.process", `${hook.owner_id}:${hook.pid}`, input.resource.sampled_at, "supported");
|
|
125
|
+
out.push(candidate("hook.long-running", "warning", "hook", `${hook.owner_id}:${hook.pid}`, `Hook process ${hook.pid} has run for ${hook.age_seconds} seconds.`, hookSource, hook.age_seconds, "seconds", [{ source_kind: "hook.process", state: "supported" }]));
|
|
126
|
+
}
|
|
127
|
+
for (const process of input.resource.processes
|
|
128
|
+
.filter((entry) => entry.rss_bytes >= PROCESS_MEMORY_THRESHOLD)
|
|
129
|
+
.slice(0, 5))
|
|
130
|
+
out.push(candidate("process.memory-pressure", "critical", "process", process.start_id, `Process ${process.pid} uses ${process.rss_bytes} resident bytes.`, resourceSource(input.resource, resourceCapability.state, process.start_id), process.rss_bytes, "bytes", [resourceCapability], processAttribution(process)));
|
|
131
|
+
for (const group of input.resource.groups
|
|
132
|
+
.filter((entry) => entry.rss_bytes >= GROUP_MEMORY_THRESHOLD)
|
|
133
|
+
.slice(0, 5))
|
|
134
|
+
out.push(candidate("group.memory-pressure", "critical", group.kind, group.id, `${groupLabel(group)} uses ${group.rss_bytes} resident bytes.`, source, group.rss_bytes, "bytes", [resourceCapability], groupAttribution(group)));
|
|
135
|
+
for (const group of input.resource.groups
|
|
136
|
+
.filter((entry) => entry.process_count >= OWNER_PROCESS_THRESHOLD)
|
|
137
|
+
.slice(0, 5))
|
|
138
|
+
out.push(candidate("group.process-pressure", "warning", group.kind, group.id, `${groupLabel(group)} owns ${group.process_count} visible processes.`, source, group.process_count, "processes", [resourceCapability], groupAttribution(group)));
|
|
139
|
+
out.push(...memoryGrowthCandidates(input.resource, input.history, resourceCapability, input.activity));
|
|
140
|
+
if (input.logFeed.unavailable_families > 0) {
|
|
141
|
+
const logCapability = {
|
|
142
|
+
source_kind: "log.feed",
|
|
143
|
+
state: "partial",
|
|
144
|
+
reason_code: "families-unavailable",
|
|
145
|
+
detail: `${input.logFeed.unavailable_families} log families were unavailable.`,
|
|
146
|
+
};
|
|
147
|
+
out.push(capabilityCandidate(logCapability, stableSource("log.feed", "supervisor-log-feed", input.logFeed.captured_at, logCapability.state, undefined, input.logFeed.schema_version)));
|
|
148
|
+
}
|
|
149
|
+
if (input.hookHealth) {
|
|
150
|
+
const hookCapability = {
|
|
151
|
+
source_kind: input.hookHealth.capability.source_kind,
|
|
152
|
+
state: input.hookHealth.capability.state === "unavailable"
|
|
153
|
+
? "error"
|
|
154
|
+
: input.hookHealth.capability.state,
|
|
155
|
+
...(input.hookHealth.capability.reason
|
|
156
|
+
? {
|
|
157
|
+
reason_code: input.hookHealth.capability.reason,
|
|
158
|
+
detail: input.hookHealth.capability.reason,
|
|
159
|
+
}
|
|
160
|
+
: {}),
|
|
161
|
+
};
|
|
162
|
+
for (const alert of evaluateHookHealthAlerts(input.hookHealth)) {
|
|
163
|
+
const alertSource = stableSource("hook.terminal-log", alert.source.id, alert.source.observed_at, hookCapability.state, alert.source.id, input.hookHealth.schema_version);
|
|
164
|
+
out.push(candidate(alert.finding_kind, alert.severity, "hook", alert.scope_id, alert.summary, alertSource, alert.observed_value, alert.unit, [hookCapability], alert.owner_id
|
|
165
|
+
? {
|
|
166
|
+
state: "attributed",
|
|
167
|
+
owner_kind: "agent",
|
|
168
|
+
owner_id: alert.owner_id,
|
|
169
|
+
}
|
|
170
|
+
: { state: "unattributed", reason_code: "no-validated-process-anchor" }));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (input.coordination) {
|
|
174
|
+
const coordination = input.coordination;
|
|
175
|
+
if (coordination.capability.state !== "supported") {
|
|
176
|
+
out.push(capabilityCandidate(coordination.capability, stableSource(coordination.capability.source_kind, coordination.capability.reason_code ?? coordination.capability.state, coordination.observed_at, coordination.capability.state)));
|
|
177
|
+
}
|
|
178
|
+
for (const diagnostic of coordination.diagnostics.slice(0, 5)) {
|
|
179
|
+
const diagnosticSource = stableSource("coordination.v3", diagnostic.event_id ?? `${diagnostic.segment_ordinal}:${diagnostic.byte_offset}`, coordination.observed_at, coordination.capability.state, diagnostic.event_id);
|
|
180
|
+
out.push(candidate("coordination.ledger-diagnostic", "warning", "ledger", diagnostic.code, `Event ledger reported ${diagnostic.code}.`, diagnosticSource, 1, "count", [coordination.capability]));
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return out;
|
|
184
|
+
}
|
|
185
|
+
function capabilityCandidate(capability, source) {
|
|
186
|
+
return candidate("source.capability-degraded", "info", "source", capability.source_kind, `${capability.source_kind} capability is ${capability.state}${capability.reason_code ? ` (${capability.reason_code})` : ""}.`, source, undefined, undefined, [capability]);
|
|
187
|
+
}
|
|
188
|
+
function memoryGrowthCandidates(resource, history, capability, activity) {
|
|
189
|
+
const newestAt = Date.parse(resource.sampled_at);
|
|
190
|
+
const baseline = history.points.find((point) => newestAt - Date.parse(point.sampled_at) <= MEMORY_GROWTH_WINDOW_MS);
|
|
191
|
+
if (!baseline || newestAt - Date.parse(baseline.sampled_at) < 60_000)
|
|
192
|
+
return [];
|
|
193
|
+
const priorByKey = new Map(baseline.groups.map((group) => [groupKey(group), group]));
|
|
194
|
+
return resource.groups
|
|
195
|
+
.flatMap((group) => {
|
|
196
|
+
const prior = priorByKey.get(groupKey(group));
|
|
197
|
+
if (!prior || prior.rss_bytes <= 0)
|
|
198
|
+
return [];
|
|
199
|
+
const delta = group.rss_bytes - prior.rss_bytes;
|
|
200
|
+
if (delta < MEMORY_GROWTH_MIN_BYTES ||
|
|
201
|
+
group.rss_bytes / prior.rss_bytes < MEMORY_GROWTH_RATIO)
|
|
202
|
+
return [];
|
|
203
|
+
return [
|
|
204
|
+
memoryGrowthCandidate(group, activity, "resource.memory-growth", `${groupLabel(group)} grew by ${delta} resident bytes since ${baseline.sampled_at}.`, resourceSource(resource, capability.state), delta, capability),
|
|
205
|
+
];
|
|
206
|
+
})
|
|
207
|
+
.slice(0, 5);
|
|
208
|
+
}
|
|
209
|
+
function memoryGrowthCandidate(group, activity, findingKind, summary, source, delta, resourceCapability) {
|
|
210
|
+
const workload = workloadContext(group, activity, source.observed_at);
|
|
211
|
+
const severity = workload.context.relationship === "unexpected-idle-growth" ? "critical" : "warning";
|
|
212
|
+
const result = candidate(findingKind, severity, group.kind, group.id, summary, source, delta, "bytes", [resourceCapability, workload.capability], groupAttribution(group), workload.context);
|
|
213
|
+
if (workload.evidence)
|
|
214
|
+
result.evidence.push(workload.evidence);
|
|
215
|
+
return result;
|
|
216
|
+
}
|
|
217
|
+
function workloadContext(group, activity, observedAt) {
|
|
218
|
+
const capability = activity?.capability ?? {
|
|
219
|
+
source_kind: "coordination.activity-projection",
|
|
220
|
+
state: "unsupported",
|
|
221
|
+
reason_code: "activity-projection-not-captured",
|
|
222
|
+
};
|
|
223
|
+
const matched = group.kind === "agent"
|
|
224
|
+
? activity?.entries.find((entry) => entry.scope_id === group.id)
|
|
225
|
+
: undefined;
|
|
226
|
+
if (!matched) {
|
|
227
|
+
return {
|
|
228
|
+
context: {
|
|
229
|
+
relationship: "unknown",
|
|
230
|
+
declared_activity: "unknown",
|
|
231
|
+
task_state: "unknown",
|
|
232
|
+
observed_at: activity?.observed_at ?? observedAt,
|
|
233
|
+
source: stableSource("coordination.activity-projection", group.kind === "agent" ? group.id : "not-applicable", activity?.observed_at ?? observedAt, capability.state, undefined, activity?.schema_version),
|
|
234
|
+
},
|
|
235
|
+
capability,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
const relationship = matched.declared_activity === "working" && matched.task_state === "active"
|
|
239
|
+
? "active-work"
|
|
240
|
+
: matched.declared_activity === "unknown"
|
|
241
|
+
? "unknown"
|
|
242
|
+
: "unexpected-idle-growth";
|
|
243
|
+
const summary = `Agent declared ${matched.declared_activity} with task state ${matched.task_state}.`;
|
|
244
|
+
return {
|
|
245
|
+
context: {
|
|
246
|
+
relationship,
|
|
247
|
+
declared_activity: matched.declared_activity,
|
|
248
|
+
task_state: matched.task_state,
|
|
249
|
+
observed_at: matched.observed_at,
|
|
250
|
+
source: matched.source,
|
|
251
|
+
},
|
|
252
|
+
capability,
|
|
253
|
+
evidence: {
|
|
254
|
+
id: deterministicId("ev", `${findingKindKey(group)}\u0000${matched.source.id}`),
|
|
255
|
+
source: matched.source,
|
|
256
|
+
summary,
|
|
257
|
+
},
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
function findingKindKey(group) {
|
|
261
|
+
return `resource.memory-growth\u0000${group.kind}\u0000${group.id}`;
|
|
262
|
+
}
|
|
263
|
+
function processAttribution(process) {
|
|
264
|
+
if ((process.owner_kind === "agent" || process.owner_kind === "service") &&
|
|
265
|
+
process.owner_id &&
|
|
266
|
+
process.owner_root_pid !== null) {
|
|
267
|
+
return {
|
|
268
|
+
state: "attributed",
|
|
269
|
+
owner_kind: process.owner_kind,
|
|
270
|
+
owner_id: process.owner_id,
|
|
271
|
+
owner_root_pid: process.owner_root_pid,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
return { state: "unattributed", reason_code: "no-validated-process-anchor" };
|
|
275
|
+
}
|
|
276
|
+
function groupAttribution(group) {
|
|
277
|
+
if (group.kind === "agent" || group.kind === "service") {
|
|
278
|
+
return {
|
|
279
|
+
state: "attributed",
|
|
280
|
+
owner_kind: group.kind,
|
|
281
|
+
owner_id: group.id,
|
|
282
|
+
...(group.root_pids[0] !== undefined ? { owner_root_pid: group.root_pids[0] } : {}),
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
return { state: "unattributed", reason_code: "no-validated-process-anchor" };
|
|
286
|
+
}
|
|
287
|
+
function serviceStaleSummary(service) {
|
|
288
|
+
if (service.reason === "recorded-running-process-missing")
|
|
289
|
+
return `${service.id} was recorded running, but its process is no longer present.`;
|
|
290
|
+
if (service.reason === "remote-heartbeat-expired")
|
|
291
|
+
return `${service.id} has an expired remote heartbeat.`;
|
|
292
|
+
if (service.reason === "supervisor-reported-error")
|
|
293
|
+
return `${service.id} reported an error.`;
|
|
294
|
+
return `${service.id} has stale liveness evidence.`;
|
|
295
|
+
}
|
|
296
|
+
function candidate(findingKind, severity, scopeKind, scopeId, summary, source, observedValue, unit, capabilities, attribution, workloadContext) {
|
|
297
|
+
const evidence = {
|
|
298
|
+
id: deterministicId("ev", `${findingKind}\u0000${scopeKind}\u0000${scopeId}\u0000${source.id}`),
|
|
299
|
+
source,
|
|
300
|
+
summary: summary.slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_summary_chars),
|
|
301
|
+
...(observedValue !== undefined ? { observed_value: observedValue } : {}),
|
|
302
|
+
...(unit ? { unit } : {}),
|
|
303
|
+
};
|
|
304
|
+
return {
|
|
305
|
+
source_kind: source.source_kind,
|
|
306
|
+
finding_kind: findingKind,
|
|
307
|
+
severity,
|
|
308
|
+
scope_kind: scopeKind,
|
|
309
|
+
scope_id: scopeId,
|
|
310
|
+
summary: evidence.summary,
|
|
311
|
+
primary_source: source,
|
|
312
|
+
evidence: [evidence],
|
|
313
|
+
capabilities,
|
|
314
|
+
...(attribution ? { attribution } : {}),
|
|
315
|
+
...(workloadContext ? { workload_context: workloadContext } : {}),
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
function resourceSource(resource, capability, recordId) {
|
|
319
|
+
return stableSource("resource.snapshot", `${resource.platform}:${resource.namespace}`, resource.sampled_at, capability, recordId, resource.schema_version);
|
|
320
|
+
}
|
|
321
|
+
function stableSource(sourceKind, sourceId, observedAt, capability, recordId, schemaVersion) {
|
|
322
|
+
return {
|
|
323
|
+
id: deterministicId("src", `${sourceKind}\u0000${sourceId}\u0000${recordId ?? ""}\u0000${observedAt}`),
|
|
324
|
+
source_kind: sourceKind,
|
|
325
|
+
source_id: sourceId,
|
|
326
|
+
observed_at: observedAt,
|
|
327
|
+
...(recordId ? { record_id: recordId } : {}),
|
|
328
|
+
...(schemaVersion !== undefined ? { schema_version: schemaVersion } : {}),
|
|
329
|
+
capability,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
function recentResolved(findings, fingerprint, observedAt) {
|
|
333
|
+
const observedMs = Date.parse(observedAt);
|
|
334
|
+
return findings.transitions
|
|
335
|
+
.filter((entry) => entry.fingerprint === fingerprint &&
|
|
336
|
+
entry.state === "resolved" &&
|
|
337
|
+
entry.resolved_at !== undefined &&
|
|
338
|
+
observedMs - Date.parse(entry.resolved_at) >= 0 &&
|
|
339
|
+
observedMs - Date.parse(entry.resolved_at) <= SUPERVISOR_FINDING_POLICY.episode_gap_ms)
|
|
340
|
+
.sort((left, right) => (right.resolved_at ?? "").localeCompare(left.resolved_at ?? ""))[0];
|
|
341
|
+
}
|
|
342
|
+
function peakEvidence(prior, evidence, observedAt) {
|
|
343
|
+
const current = evidence
|
|
344
|
+
.filter((entry) => entry.observed_value !== undefined)
|
|
345
|
+
.sort((left, right) => right.observed_value - left.observed_value)[0];
|
|
346
|
+
if (prior?.peak_observed_value !== undefined &&
|
|
347
|
+
(!current || prior.peak_observed_value >= current.observed_value)) {
|
|
348
|
+
return {
|
|
349
|
+
peak_observed_value: prior.peak_observed_value,
|
|
350
|
+
...(prior.peak_observed_at ? { peak_observed_at: prior.peak_observed_at } : {}),
|
|
351
|
+
...(prior.peak_unit ? { peak_unit: prior.peak_unit } : {}),
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
return current
|
|
355
|
+
? {
|
|
356
|
+
peak_observed_value: current.observed_value,
|
|
357
|
+
peak_observed_at: current.source.observed_at || observedAt,
|
|
358
|
+
...(current.unit ? { peak_unit: current.unit } : {}),
|
|
359
|
+
}
|
|
360
|
+
: {};
|
|
361
|
+
}
|
|
362
|
+
function findingFingerprint(candidate) {
|
|
363
|
+
return digest([
|
|
364
|
+
candidate.source_kind,
|
|
365
|
+
candidate.finding_kind,
|
|
366
|
+
candidate.scope_kind,
|
|
367
|
+
candidate.scope_id,
|
|
368
|
+
candidate.primary_source.source_id,
|
|
369
|
+
].join("\u0000")).slice(0, 32);
|
|
370
|
+
}
|
|
371
|
+
function deterministicId(prefix, value) {
|
|
372
|
+
return `${prefix}_${digest(value).slice(0, 24)}`;
|
|
373
|
+
}
|
|
374
|
+
function digest(value) {
|
|
375
|
+
return createHash("sha256").update(value).digest("hex");
|
|
376
|
+
}
|
|
377
|
+
function validFindings(value) {
|
|
378
|
+
return (value?.schema_version === SUPERVISOR_FINDING_SCHEMA_VERSION &&
|
|
379
|
+
Array.isArray(value.active) &&
|
|
380
|
+
Array.isArray(value.transitions));
|
|
381
|
+
}
|
|
382
|
+
function boundCapabilities(capabilities) {
|
|
383
|
+
const unique = new Map(capabilities.map((entry) => [entry.source_kind, entry]));
|
|
384
|
+
return [...unique.values()].slice(0, SUPERVISOR_DIAGNOSTIC_LIMITS.max_capabilities);
|
|
385
|
+
}
|
|
386
|
+
function groupKey(group) {
|
|
387
|
+
return `${group.kind}:${group.id}`;
|
|
388
|
+
}
|
|
389
|
+
function groupLabel(group) {
|
|
390
|
+
return group.kind === "unattributed"
|
|
391
|
+
? "Unattributed processes"
|
|
392
|
+
: `${group.kind} group ${group.id}`;
|
|
393
|
+
}
|
|
394
|
+
function severityRank(value) {
|
|
395
|
+
return value === "critical" ? 3 : value === "warning" ? 2 : 1;
|
|
396
|
+
}
|
|
397
|
+
function round1(value) {
|
|
398
|
+
return Math.round(value * 10) / 10;
|
|
399
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ResourceSnapshot } from "../resources/contract.js";
|
|
2
|
+
import { type SupervisorHistory } from "./contract.js";
|
|
3
|
+
export declare const SUPERVISOR_HISTORY_INTERVAL_MS = 10000;
|
|
4
|
+
export declare const SUPERVISOR_HISTORY_MAX_POINTS = 90;
|
|
5
|
+
export declare function updateSupervisorHistory(previous: SupervisorHistory | undefined, resource: ResourceSnapshot): {
|
|
6
|
+
history: SupervisorHistory;
|
|
7
|
+
changed: boolean;
|
|
8
|
+
};
|
|
9
|
+
//# sourceMappingURL=history.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../../src/core/supervisor/history.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,eAAe,CAAC;AAEvB,eAAO,MAAM,8BAA8B,QAAS,CAAC;AACrD,eAAO,MAAM,6BAA6B,KAAK,CAAC;AAEhD,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,iBAAiB,GAAG,SAAS,EACvC,QAAQ,EAAE,gBAAgB,GACzB;IAAE,OAAO,EAAE,iBAAiB,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CAuClD"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { SUPERVISOR_HISTORY_SCHEMA_VERSION, } from "./contract.js";
|
|
2
|
+
export const SUPERVISOR_HISTORY_INTERVAL_MS = 10_000;
|
|
3
|
+
export const SUPERVISOR_HISTORY_MAX_POINTS = 90;
|
|
4
|
+
export function updateSupervisorHistory(previous, resource) {
|
|
5
|
+
const prior = validHistory(previous) ? previous.points : [];
|
|
6
|
+
const lastAt = prior.length > 0 ? Date.parse(prior[prior.length - 1].sampled_at) : Number.NaN;
|
|
7
|
+
const sampledAt = Date.parse(resource.sampled_at);
|
|
8
|
+
if (Number.isFinite(lastAt) &&
|
|
9
|
+
Number.isFinite(sampledAt) &&
|
|
10
|
+
sampledAt - lastAt < SUPERVISOR_HISTORY_INTERVAL_MS) {
|
|
11
|
+
return {
|
|
12
|
+
history: {
|
|
13
|
+
schema_version: SUPERVISOR_HISTORY_SCHEMA_VERSION,
|
|
14
|
+
interval_ms: SUPERVISOR_HISTORY_INTERVAL_MS,
|
|
15
|
+
max_points: SUPERVISOR_HISTORY_MAX_POINTS,
|
|
16
|
+
points: prior,
|
|
17
|
+
},
|
|
18
|
+
changed: false,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
const point = {
|
|
22
|
+
sampled_at: resource.sampled_at,
|
|
23
|
+
machine: {
|
|
24
|
+
cpu_percent: resource.machine.cpu_percent,
|
|
25
|
+
memory_percent: resource.machine.memory_percent,
|
|
26
|
+
memory_used_bytes: resource.machine.memory_used_bytes,
|
|
27
|
+
swap_used_bytes: resource.machine.swap_used_bytes,
|
|
28
|
+
process_count: resource.machine.process_count,
|
|
29
|
+
},
|
|
30
|
+
groups: resource.groups.map((group) => ({ ...group, root_pids: [...group.root_pids] })),
|
|
31
|
+
};
|
|
32
|
+
return {
|
|
33
|
+
history: {
|
|
34
|
+
schema_version: SUPERVISOR_HISTORY_SCHEMA_VERSION,
|
|
35
|
+
interval_ms: SUPERVISOR_HISTORY_INTERVAL_MS,
|
|
36
|
+
max_points: SUPERVISOR_HISTORY_MAX_POINTS,
|
|
37
|
+
points: [...prior, point].slice(-SUPERVISOR_HISTORY_MAX_POINTS),
|
|
38
|
+
},
|
|
39
|
+
changed: true,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function validHistory(value) {
|
|
43
|
+
return (value?.schema_version === SUPERVISOR_HISTORY_SCHEMA_VERSION &&
|
|
44
|
+
Array.isArray(value.points) &&
|
|
45
|
+
value.points.length <= SUPERVISOR_HISTORY_MAX_POINTS);
|
|
46
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SupervisorFindingEvidence, SupervisorFindingSeverity } from "./contract.js";
|
|
2
|
+
import { type CompletedHookHealth, type SupervisorHookHealth } from "./hook-health.js";
|
|
3
|
+
export interface HookHealthAlert {
|
|
4
|
+
finding_kind: "hook.execution-degraded" | "hook.execution-slow" | "hook.memory-heavy" | "hook.retry-cluster";
|
|
5
|
+
severity: SupervisorFindingSeverity;
|
|
6
|
+
scope_id: string;
|
|
7
|
+
summary: string;
|
|
8
|
+
observed_value: number;
|
|
9
|
+
unit: SupervisorFindingEvidence["unit"];
|
|
10
|
+
source: CompletedHookHealth;
|
|
11
|
+
owner_id?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare function evaluateHookHealthAlerts(health: SupervisorHookHealth): readonly HookHealthAlert[];
|
|
14
|
+
//# sourceMappingURL=hook-health-alerts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hook-health-alerts.d.ts","sourceRoot":"","sources":["../../../src/core/supervisor/hook-health-alerts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC1F,OAAO,EACL,KAAK,mBAAmB,EAKxB,KAAK,oBAAoB,EAC1B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,eAAe;IAC9B,YAAY,EACR,yBAAyB,GACzB,qBAAqB,GACrB,mBAAmB,GACnB,oBAAoB,CAAC;IACzB,QAAQ,EAAE,yBAAyB,CAAC;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,yBAAyB,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,oBAAoB,GAAG,SAAS,eAAe,EAAE,CAsDjG"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { SUPERVISOR_HIGH_COMPLETED_HOOK_RSS_BYTES, SUPERVISOR_HOOK_RETRY_CLUSTER_COUNT, SUPERVISOR_SLOW_COMPLETED_HOOK_MS, } from "./hook-health.js";
|
|
2
|
+
export function evaluateHookHealthAlerts(health) {
|
|
3
|
+
if (health.capability.state === "unavailable")
|
|
4
|
+
return [];
|
|
5
|
+
const alerts = [];
|
|
6
|
+
for (const aggregate of health.aggregates) {
|
|
7
|
+
const receipts = health.recent.filter((receipt) => `${receipt.adapter}:${receipt.hook_name}` === aggregate.key);
|
|
8
|
+
const problem = receipts.find((receipt) => receipt.outcome === "faulted" || receipt.outcome === "degraded");
|
|
9
|
+
if (problem)
|
|
10
|
+
alerts.push(problemAlert(aggregate, problem));
|
|
11
|
+
const slow = maxReceipt(receipts, (receipt) => receipt.duration_ms);
|
|
12
|
+
if (slow && slow.duration_ms >= SUPERVISOR_SLOW_COMPLETED_HOOK_MS) {
|
|
13
|
+
alerts.push({
|
|
14
|
+
finding_kind: "hook.execution-slow",
|
|
15
|
+
severity: "warning",
|
|
16
|
+
scope_id: aggregate.key,
|
|
17
|
+
summary: `${aggregate.hook_name} completed in ${slow.duration_ms} milliseconds.`,
|
|
18
|
+
observed_value: slow.duration_ms,
|
|
19
|
+
unit: "milliseconds",
|
|
20
|
+
source: slow,
|
|
21
|
+
...(slow.owner_id ? { owner_id: slow.owner_id } : {}),
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
const heavy = maxReceipt(receipts, (receipt) => receipt.rss_end_bytes);
|
|
25
|
+
if (heavy && heavy.rss_end_bytes >= SUPERVISOR_HIGH_COMPLETED_HOOK_RSS_BYTES) {
|
|
26
|
+
alerts.push({
|
|
27
|
+
finding_kind: "hook.memory-heavy",
|
|
28
|
+
severity: "warning",
|
|
29
|
+
scope_id: aggregate.key,
|
|
30
|
+
summary: `${aggregate.hook_name} completed with ${heavy.rss_end_bytes} resident bytes.`,
|
|
31
|
+
observed_value: heavy.rss_end_bytes,
|
|
32
|
+
unit: "bytes",
|
|
33
|
+
source: heavy,
|
|
34
|
+
...(heavy.owner_id ? { owner_id: heavy.owner_id } : {}),
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (health.summary.retry_count >= SUPERVISOR_HOOK_RETRY_CLUSTER_COUNT) {
|
|
39
|
+
const latestRetry = health.recent.find((receipt) => receipt.retry_worker);
|
|
40
|
+
if (latestRetry) {
|
|
41
|
+
alerts.push({
|
|
42
|
+
finding_kind: "hook.retry-cluster",
|
|
43
|
+
severity: "warning",
|
|
44
|
+
scope_id: `${latestRetry.adapter}:runtime-context-retry`,
|
|
45
|
+
summary: `${health.summary.retry_count} hook retry workers ran inside five minutes.`,
|
|
46
|
+
observed_value: health.summary.retry_count,
|
|
47
|
+
unit: "count",
|
|
48
|
+
source: latestRetry,
|
|
49
|
+
...(latestRetry.owner_id ? { owner_id: latestRetry.owner_id } : {}),
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return alerts.slice(0, 12);
|
|
54
|
+
}
|
|
55
|
+
function problemAlert(aggregate, receipt) {
|
|
56
|
+
const severity = receipt.outcome === "faulted" ? "critical" : "warning";
|
|
57
|
+
return {
|
|
58
|
+
finding_kind: "hook.execution-degraded",
|
|
59
|
+
severity,
|
|
60
|
+
scope_id: aggregate.key,
|
|
61
|
+
summary: `${aggregate.hook_name} completed with internal outcome ${receipt.outcome}.`,
|
|
62
|
+
observed_value: aggregate.degraded_count + aggregate.faulted_count,
|
|
63
|
+
unit: "count",
|
|
64
|
+
source: receipt,
|
|
65
|
+
...(receipt.owner_id ? { owner_id: receipt.owner_id } : {}),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
function maxReceipt(receipts, value) {
|
|
69
|
+
return receipts.reduce((current, receipt) => (!current || value(receipt) > value(current) ? receipt : current), undefined);
|
|
70
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type SupervisorHookHealth } from "./hook-health.js";
|
|
2
|
+
export declare function supervisorHookHealthPath(coordRoot: string): string;
|
|
3
|
+
export declare function writeSupervisorHookHealth(coordRoot: string, value: SupervisorHookHealth): void;
|
|
4
|
+
export declare function readSupervisorHookHealth(coordRoot: string): SupervisorHookHealth | undefined;
|
|
5
|
+
//# sourceMappingURL=hook-health-storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hook-health-storage.d.ts","sourceRoot":"","sources":["../../../src/core/supervisor/hook-health-storage.ts"],"names":[],"mappings":"AAGA,OAAO,EAAyC,KAAK,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAEpG,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAElE;AAED,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,GAAG,IAAI,CAE9F;AAED,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS,CAW5F"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { writePrivateJsonAtomic } from "../storage/atomic-json.js";
|
|
4
|
+
import { SUPERVISOR_HOOK_HEALTH_SCHEMA_VERSION } from "./hook-health.js";
|
|
5
|
+
export function supervisorHookHealthPath(coordRoot) {
|
|
6
|
+
return join(resolve(coordRoot), ".harnery", "supervisor", "hook-health.json");
|
|
7
|
+
}
|
|
8
|
+
export function writeSupervisorHookHealth(coordRoot, value) {
|
|
9
|
+
writePrivateJsonAtomic(supervisorHookHealthPath(coordRoot), value);
|
|
10
|
+
}
|
|
11
|
+
export function readSupervisorHookHealth(coordRoot) {
|
|
12
|
+
const path = supervisorHookHealthPath(coordRoot);
|
|
13
|
+
if (!existsSync(path))
|
|
14
|
+
return undefined;
|
|
15
|
+
try {
|
|
16
|
+
const value = JSON.parse(readFileSync(path, "utf8"));
|
|
17
|
+
return value?.schema_version === SUPERVISOR_HOOK_HEALTH_SCHEMA_VERSION
|
|
18
|
+
? value
|
|
19
|
+
: undefined;
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { type HookHealthOutcome } from "../hooks/health.js";
|
|
2
|
+
import type { HarneryLogRecordV1 } from "../storage/jsonl.js";
|
|
3
|
+
import type { SupervisorLogFeed } from "./contract.js";
|
|
4
|
+
export declare const SUPERVISOR_HOOK_HEALTH_SCHEMA_VERSION: 1;
|
|
5
|
+
export declare const SUPERVISOR_HOOK_HEALTH_RECENT_LIMIT = 50;
|
|
6
|
+
export declare const SUPERVISOR_HOOK_HEALTH_AGGREGATE_LIMIT = 40;
|
|
7
|
+
export declare const SUPERVISOR_SLOW_COMPLETED_HOOK_MS = 30000;
|
|
8
|
+
export declare const SUPERVISOR_HIGH_COMPLETED_HOOK_RSS_BYTES: number;
|
|
9
|
+
export declare const SUPERVISOR_HOOK_RETRY_CLUSTER_COUNT = 3;
|
|
10
|
+
export declare const SUPERVISOR_HOOK_RETRY_CLUSTER_WINDOW_MS: number;
|
|
11
|
+
export interface CompletedHookHealth {
|
|
12
|
+
id: string;
|
|
13
|
+
observed_at: string;
|
|
14
|
+
hook_name: string;
|
|
15
|
+
adapter: "claude-code" | "codex" | "cursor" | "unknown";
|
|
16
|
+
outcome: HookHealthOutcome;
|
|
17
|
+
duration_ms: number;
|
|
18
|
+
rss_start_bytes: number;
|
|
19
|
+
rss_end_bytes: number;
|
|
20
|
+
rss_delta_bytes: number;
|
|
21
|
+
retry_worker: boolean;
|
|
22
|
+
error_count: number;
|
|
23
|
+
error_phases: readonly string[];
|
|
24
|
+
payload_bytes: number;
|
|
25
|
+
pid: number;
|
|
26
|
+
owner_id?: string;
|
|
27
|
+
skipped_reason?: string;
|
|
28
|
+
v3_state?: string;
|
|
29
|
+
}
|
|
30
|
+
export interface HookHealthAggregate {
|
|
31
|
+
key: string;
|
|
32
|
+
hook_name: string;
|
|
33
|
+
adapter: CompletedHookHealth["adapter"];
|
|
34
|
+
invocation_count: number;
|
|
35
|
+
completed_count: number;
|
|
36
|
+
skipped_count: number;
|
|
37
|
+
degraded_count: number;
|
|
38
|
+
faulted_count: number;
|
|
39
|
+
retry_count: number;
|
|
40
|
+
duration_p50_ms: number;
|
|
41
|
+
duration_p95_ms: number;
|
|
42
|
+
duration_max_ms: number;
|
|
43
|
+
rss_end_max_bytes: number;
|
|
44
|
+
rss_delta_max_bytes: number;
|
|
45
|
+
latest_observed_at: string;
|
|
46
|
+
owner_ids: readonly string[];
|
|
47
|
+
}
|
|
48
|
+
export interface SupervisorHookHealth {
|
|
49
|
+
schema_version: typeof SUPERVISOR_HOOK_HEALTH_SCHEMA_VERSION;
|
|
50
|
+
captured_at: string;
|
|
51
|
+
capability: {
|
|
52
|
+
source_kind: "hook.terminal-log";
|
|
53
|
+
state: "supported" | "partial" | "unavailable";
|
|
54
|
+
reason?: string;
|
|
55
|
+
};
|
|
56
|
+
source_record_count: number;
|
|
57
|
+
malformed_record_count: number;
|
|
58
|
+
truncated: boolean;
|
|
59
|
+
summary: {
|
|
60
|
+
invocation_count: number;
|
|
61
|
+
degraded_count: number;
|
|
62
|
+
faulted_count: number;
|
|
63
|
+
slow_count: number;
|
|
64
|
+
high_memory_count: number;
|
|
65
|
+
retry_count: number;
|
|
66
|
+
};
|
|
67
|
+
aggregates: readonly HookHealthAggregate[];
|
|
68
|
+
recent: readonly CompletedHookHealth[];
|
|
69
|
+
}
|
|
70
|
+
export declare function projectHookHealth(feed: SupervisorLogFeed, now?: Date): SupervisorHookHealth;
|
|
71
|
+
export declare function parseCompletedHookHealth(record: HarneryLogRecordV1): CompletedHookHealth | undefined;
|
|
72
|
+
//# sourceMappingURL=hook-health.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hook-health.d.ts","sourceRoot":"","sources":["../../../src/core/supervisor/hook-health.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,iBAAiB,EACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEvD,eAAO,MAAM,qCAAqC,EAAG,CAAU,CAAC;AAChE,eAAO,MAAM,mCAAmC,KAAK,CAAC;AACtD,eAAO,MAAM,sCAAsC,KAAK,CAAC;AACzD,eAAO,MAAM,iCAAiC,QAAS,CAAC;AACxD,eAAO,MAAM,wCAAwC,QAAoB,CAAC;AAC1E,eAAO,MAAM,mCAAmC,IAAI,CAAC;AACrD,eAAO,MAAM,uCAAuC,QAAa,CAAC;AAElE,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;IACxD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,oBAAoB;IACnC,cAAc,EAAE,OAAO,qCAAqC,CAAC;IAC7D,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE;QACV,WAAW,EAAE,mBAAmB,CAAC;QACjC,KAAK,EAAE,WAAW,GAAG,SAAS,GAAG,aAAa,CAAC;QAC/C,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE;QACP,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,UAAU,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC3C,MAAM,EAAE,SAAS,mBAAmB,EAAE,CAAC;CACxC;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,OAAa,GAAG,oBAAoB,CAiDjG;AAED,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,kBAAkB,GACzB,mBAAmB,GAAG,SAAS,CA2DjC"}
|