@aiwg/cli 2026.10.3 → 2026.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/agentic/code/providers/model-capabilities.v1.json +96 -4
- package/bin/aiwg.mjs +50 -27
- package/dist/src/agents/grounding/knowledge-base.js +54 -6
- package/dist/src/artifacts/browser-export.js +6 -0
- package/dist/src/artifacts/checksum-manifest.js +1 -0
- package/dist/src/artifacts/citation-parser.js +173 -92
- package/dist/src/artifacts/citation-table-schema.js +59 -0
- package/dist/src/artifacts/cli.js +57 -18
- package/dist/src/artifacts/corpus-tools/snapshot.js +1 -1
- package/dist/src/artifacts/corpus-tools/vision-extract.js +2 -0
- package/dist/src/artifacts/corpus-views/ref-parser.js +3 -1
- package/dist/src/artifacts/corpus-views/renderers.js +3 -3
- package/dist/src/artifacts/discovery-eval.js +6 -4
- package/dist/src/artifacts/fortemi-core-query-adapter.js +36 -3
- package/dist/src/artifacts/fortemi-core-sync.js +166 -40
- package/dist/src/artifacts/framework-capabilities.js +171 -0
- package/dist/src/artifacts/framework-index-paths.js +73 -0
- package/dist/src/artifacts/framework-source-digest.js +75 -0
- package/dist/src/artifacts/index-builder.js +285 -131
- package/dist/src/artifacts/index-files.js +132 -18
- package/dist/src/artifacts/index-preparation.js +65 -0
- package/dist/src/artifacts/index-refresh.js +576 -0
- package/dist/src/artifacts/index-worker.mjs +9 -0
- package/dist/src/artifacts/prebuilt-build-lock.js +47 -4
- package/dist/src/artifacts/prebuilt-deployment.js +46 -0
- package/dist/src/artifacts/prebuilt-relations.js +80 -0
- package/dist/src/artifacts/query-engine.js +410 -88
- package/dist/src/artifacts/relations-builder.js +168 -0
- package/dist/src/artifacts/relations.js +148 -0
- package/dist/src/artifacts/stats.js +5 -2
- package/dist/src/artifacts/types.js +30 -10
- package/dist/src/artifacts/watcher.js +1 -1
- package/dist/src/auth/credential-store.js +2 -1
- package/dist/src/catalog/cli.d.mts +5 -0
- package/dist/src/channel/manager.d.mts +20 -0
- package/dist/src/channel/manager.mjs +19 -2
- package/dist/src/cli/command-log.js +2 -3
- package/dist/src/cli/handlers/artifacts.js +3 -0
- package/dist/src/cli/handlers/auth.js +3 -0
- package/dist/src/cli/handlers/decision.js +19 -12
- package/dist/src/cli/handlers/diagnose.js +4 -3
- package/dist/src/cli/handlers/effect.js +4 -2
- package/dist/src/cli/handlers/evidence.js +3 -0
- package/dist/src/cli/handlers/help.js +1 -0
- package/dist/src/cli/handlers/index.js +3 -1
- package/dist/src/cli/handlers/init.js +4 -1
- package/dist/src/cli/handlers/lint.js +5 -0
- package/dist/src/cli/handlers/models.js +104 -8
- package/dist/src/cli/handlers/ralph-launcher.js +2 -0
- package/dist/src/cli/handlers/refresh.js +12 -0
- package/dist/src/cli/handlers/resource-versions.js +3 -0
- package/dist/src/cli/handlers/rules.js +195 -0
- package/dist/src/cli/handlers/run.js +4 -0
- package/dist/src/cli/handlers/runtime-info.js +1 -2
- package/dist/src/cli/handlers/script-runner.js +3 -2
- package/dist/src/cli/handlers/session.js +19 -9
- package/dist/src/cli/handlers/sessions.js +98 -37
- package/dist/src/cli/handlers/setup-manifest.js +39 -44
- package/dist/src/cli/handlers/setup.js +41 -0
- package/dist/src/cli/handlers/steward.js +11 -0
- package/dist/src/cli/handlers/subcommands.js +57 -2
- package/dist/src/cli/handlers/uhp.js +3 -0
- package/dist/src/cli/handlers/use.js +294 -55
- package/dist/src/cli/handlers/utilities.js +26 -10
- package/dist/src/cli/handlers/version.js +8 -0
- package/dist/src/cli/router.js +15 -1
- package/dist/src/cli/services/deployment-verification.js +19 -9
- package/dist/src/cli/services/provider-prepare-coordinator.js +24 -0
- package/dist/src/cli/skill-usage.js +2 -3
- package/dist/src/config/aiwg-config.js +87 -37
- package/dist/src/config/cli.js +111 -5
- package/dist/src/config/decision-config.js +106 -0
- package/dist/src/config/dispatch-thresholds-runtime.d.mts +8 -0
- package/dist/src/config/gitignore.js +50 -0
- package/dist/src/config/index-graph-overrides.js +36 -0
- package/dist/src/config/project-artifacts-health.d.mts +37 -0
- package/dist/src/config/project-artifacts-runtime.d.mts +16 -0
- package/dist/src/config/user-config-dir.d.mts +9 -0
- package/dist/src/config/user-config.js +2 -1
- package/dist/src/config/user-registry.js +23 -15
- package/dist/src/config/user-state.js +7 -0
- package/dist/src/decision/ask.js +203 -43
- package/dist/src/decision/context-live-qualification.js +2 -2
- package/dist/src/decision/driver.js +13 -8
- package/dist/src/decision/ensemble/aggregate.js +23 -10
- package/dist/src/decision/ensemble-study/multifact-study.js +6 -2
- package/dist/src/decision/entry.js +2 -15
- package/dist/src/decision/graph-live-qualification.js +1 -1
- package/dist/src/decision/graph-run.js +9 -0
- package/dist/src/decision/heldout/collector.js +5 -13
- package/dist/src/decision/heldout/contract.js +30 -7
- package/dist/src/decision/jev-credentials.js +15 -1
- package/dist/src/decision/ledger.js +131 -28
- package/dist/src/decision/offload-activation.js +137 -0
- package/dist/src/decision/offload.js +550 -0
- package/dist/src/decision/qualification/load.js +4 -6
- package/dist/src/decision/savings.js +28 -0
- package/dist/src/effects/types.js +5 -1
- package/dist/src/effects/verifiers/git.js +2 -1
- package/dist/src/effects/verifiers/tracker.js +26 -14
- package/dist/src/extensions/commands/definitions.js +19 -0
- package/dist/src/extensions/project-local-remove.js +36 -1
- package/dist/src/gates/providers/loader.js +61 -17
- package/dist/src/installation/manager-command.d.mts +18 -0
- package/dist/src/installation/manager-command.mjs +14 -2
- package/dist/src/installation/manager.d.mts +49 -0
- package/dist/src/installation/manager.mjs +78 -3
- package/dist/src/issues/cli.js +4 -4
- package/dist/src/issues/local.js +24 -5
- package/dist/src/jobs/runner.js +1 -1
- package/dist/src/lint/cli.js +27 -1
- package/dist/src/lint/loader.js +2 -1
- package/dist/src/lint/reporters.js +25 -1
- package/dist/src/lint/rule-activation.js +43 -0
- package/dist/src/lint/runner.js +137 -62
- package/dist/src/mcp/adapters/codex-runtime.js +7 -1
- package/dist/src/mcp/claude-install.d.mts +4 -0
- package/dist/src/mcp/claude-install.mjs +41 -0
- package/dist/src/mcp/cli.d.mts +1 -0
- package/dist/src/mcp/cli.mjs +36 -11
- package/dist/src/mcp/config-file.d.mts +28 -0
- package/dist/src/mcp/config-layers.d.mts +36 -0
- package/dist/src/mcp/credentials.d.mts +51 -0
- package/dist/src/mcp/grok-build-config.d.mts +6 -0
- package/dist/src/mcp/grok-build-config.mjs +1 -1
- package/dist/src/mcp/helpers.mjs +3 -2
- package/dist/src/mcp/muse-native-config.d.mts +13 -0
- package/dist/src/mcp/muse-native-config.mjs +1 -0
- package/dist/src/mcp/omp-config.d.mts +4 -0
- package/dist/src/mcp/registry.js +3 -93
- package/dist/src/mcp/registry.mjs +3 -87
- package/dist/src/mcp/server-config.d.mts +3 -0
- package/dist/src/mcp/server-config.mjs +89 -0
- package/dist/src/mcp/server.mjs +37 -375
- package/dist/src/mcp/test-simple.mjs +2 -1
- package/dist/src/mcp/toml-editor.d.mts +4 -0
- package/dist/src/mcp/toml-string.d.mts +1 -0
- package/dist/src/mcp/toml-strip-mcp.d.mts +1 -0
- package/dist/src/mcp/tool-filters.d.mts +46 -0
- package/dist/src/mcp/tools/decision.mjs +12 -11
- package/dist/src/mcp/tools/discovery.mjs +62 -55
- package/dist/src/mcp/tools/orchestration.mjs +12 -2
- package/dist/src/mcp/tools/worker-invoke.mjs +1340 -23
- package/dist/src/mcp/tools/worker-shell-control.mjs +52 -21
- package/dist/src/mcp/tools/worker-stdin-runtime.mjs +3 -3
- package/dist/src/mission-protocol/codecs.js +8 -0
- package/dist/src/models/config-loader.js +7 -2
- package/dist/src/models/model-capabilities.v1.json +96 -4
- package/dist/src/models/model-discovery.js +76 -11
- package/dist/src/models/provider-policy.js +101 -7
- package/dist/src/models/qualification-digest.d.mts +2 -0
- package/dist/src/models/resolver.js +12 -10
- package/dist/src/models/worker-direct-readiness.js +65 -44
- package/dist/src/models/worker-dispatch-offer.js +3 -3
- package/dist/src/models/worker-health-readiness.js +16 -3
- package/dist/src/models/worker-installation.d.mts +39 -0
- package/dist/src/models/worker-installation.mjs +135 -28
- package/dist/src/process/bounded-child-process.d.mts +84 -0
- package/dist/src/process/bounded-child-process.mjs +43 -0
- package/dist/src/process/cli-command.d.mts +16 -0
- package/dist/src/process/signal-child.d.mts +6 -0
- package/dist/src/process/signal-child.mjs +7 -0
- package/dist/src/providers/deployed-skill-roots.js +20 -0
- package/dist/src/providers/grok-build-acp.js +3 -2
- package/dist/src/providers/grok-build-headless.js +12 -4
- package/dist/src/providers/grok-build-inspect.js +1 -0
- package/dist/src/providers/omp-agent.d.mts +8 -0
- package/dist/src/providers/omp-diagnostics.d.mts +1 -0
- package/dist/src/providers/omp-paths.d.mts +13 -0
- package/dist/src/providers/skill-ownership.d.mts +9 -0
- package/dist/src/providers/skill-ownership.mjs +88 -0
- package/dist/src/rules/activation.d.mts +21 -0
- package/dist/src/rules/activation.mjs +62 -0
- package/dist/src/rules/inline-budget.d.mts +26 -0
- package/dist/src/rules/inline-budget.mjs +66 -0
- package/dist/src/rules/inventory.js +129 -0
- package/dist/src/rules/pin-state.js +73 -0
- package/dist/src/rules/pins.d.mts +14 -0
- package/dist/src/rules/pins.mjs +112 -0
- package/dist/src/rules/rule-frontmatter.schema.json +197 -0
- package/dist/src/serve/mission-conductor.js +38 -12
- package/dist/src/serve/worker-invocation.js +30 -10
- package/dist/src/serve/worker-stdin-frame.d.mts +10 -0
- package/dist/src/sessions/adapter-registry.js +23 -19
- package/dist/src/sessions/adapters/claude.js +39 -54
- package/dist/src/sessions/adapters/codex.js +82 -67
- package/dist/src/sessions/adapters/copilot.js +4 -4
- package/dist/src/sessions/adapters/cursor.js +40 -57
- package/dist/src/sessions/adapters/deepseek-harness.js +15 -12
- package/dist/src/sessions/adapters/factory.js +31 -27
- package/dist/src/sessions/adapters/generic.js +10 -10
- package/dist/src/sessions/adapters/grok-build.js +11 -11
- package/dist/src/sessions/adapters/grokbot.js +24 -13
- package/dist/src/sessions/adapters/hermes.js +27 -29
- package/dist/src/sessions/adapters/muse-workspace-scan.js +31 -11
- package/dist/src/sessions/adapters/muse.js +150 -93
- package/dist/src/sessions/adapters/omp.js +28 -14
- package/dist/src/sessions/adapters/openclaw.js +89 -79
- package/dist/src/sessions/adapters/opencode.js +27 -24
- package/dist/src/sessions/adapters/openhuman.js +18 -60
- package/dist/src/sessions/adapters/pi.js +29 -14
- package/dist/src/sessions/adapters/warp.js +3 -3
- package/dist/src/sessions/adapters/windsurf.js +19 -48
- package/dist/src/sessions/batch-contracts.js +16 -8
- package/dist/src/sessions/batch-import.js +51 -11
- package/dist/src/sessions/contracts.js +40 -2
- package/dist/src/sessions/coverage-preflight.js +1 -1
- package/dist/src/sessions/export-plan.js +4 -3
- package/dist/src/sessions/import-exclusions.js +72 -0
- package/dist/src/sessions/import-lease.js +34 -18
- package/dist/src/sessions/import-limits.js +37 -0
- package/dist/src/sessions/importer.js +60 -38
- package/dist/src/sessions/lifecycle.js +45 -0
- package/dist/src/sessions/policy.js +133 -25
- package/dist/src/sessions/readers.js +106 -53
- package/dist/src/sessions/refresh.js +106 -0
- package/dist/src/sessions/repository.js +147 -52
- package/dist/src/sessions/workspace-discovery.js +221 -56
- package/dist/src/sessions/workspace-matching.js +107 -0
- package/dist/src/skills/importer.js +1 -0
- package/dist/src/smiths/context-pipeline/aiwg-md.js +11 -0
- package/dist/src/smiths/context-pipeline/claude-hook.js +3 -26
- package/dist/src/smiths/context-pipeline/finalization.js +6 -5
- package/dist/src/smiths/context-pipeline/generator.js +21 -4
- package/dist/src/smiths/context-pipeline/parallelism-section.js +7 -15
- package/dist/src/smiths/mcpsmith/generator.js +1 -1
- package/dist/src/smiths/toolsmith/runtime-discovery.d.mts +10 -0
- package/dist/src/smiths/toolsmith/runtime-discovery.mjs +76 -18
- package/dist/src/storage/backends/fs.js +18 -76
- package/dist/src/storage/backends/obsidian.js +1 -0
- package/dist/src/tracker/forge-http.js +2 -1
- package/dist/src/update/checker.d.mts +2 -0
- package/dist/src/update/service.d.mts +22 -0
- package/dist/src/utils/contained-filesystem.d.mts +37 -0
- package/dist/src/utils/contained-filesystem.mjs +683 -0
- package/dist/src/utils/temporary-directory.js +36 -0
- package/dist/src/writing/example-generator.js +1 -1
- package/package.json +15 -2
- package/schemas/models/model-catalog.v1.schema.json +1 -1
- package/schemas/models/provider-model-capabilities.v1.schema.json +6 -0
- package/tools/_resolve-impl.mjs +13 -15
- package/tools/agents/deploy-agents.mjs +49 -21
- package/tools/agents/providers/base.mjs +104 -134
- package/tools/agents/providers/claude.mjs +8 -24
- package/tools/agents/providers/codex.mjs +18 -11
- package/tools/agents/providers/cursor.mjs +19 -18
- package/tools/agents/providers/grok-build.mjs +3 -1
- package/tools/agents/providers/hermes.mjs +57 -47
- package/tools/agents/providers/openclaw.mjs +9 -4
- package/tools/agents/providers/pi.mjs +5 -0
- package/tools/agents/providers/provider-prepare-channel.mjs +5 -0
- package/tools/agents/providers/provider-prepare-coordinator.mjs +247 -0
- package/tools/agents/providers/provider-prepare-worker.mjs +84 -0
- package/tools/agents/providers/rule-delivery.mjs +1270 -0
- package/tools/agents/providers/warp.mjs +20 -26
- package/tools/cli/decision-offload-report.mjs +40 -0
- package/tools/cli/workspace-status.mjs +862 -0
- package/tools/plugin/package-plugins.mjs +186 -8
- package/tools/plugin/plugin-links.mjs +114 -0
- package/tools/providers/antigravity-transport.mjs +3 -1
- package/tools/providers/deepseek-harness-transport.mjs +6 -4
- package/tools/providers/omp-transport.mjs +4 -2
- package/tools/ralph-external/lib/consecutive-failure-circuit-breaker.mjs +1748 -0
- package/tools/ralph-external/lib/launcher-recovery-evidence.mjs +55 -0
- package/tools/ralph-external/lib/omp-adapter.mjs +1 -1
- package/tools/ralph-external/lib/pi-adapter.mjs +0 -4
- package/tools/ralph-external/lib/provider-adapter.mjs +59 -1
- package/tools/ralph-external/lib/recovery-probe.mjs +72 -0
- package/tools/ralph-external/lib/session-environment.mjs +28 -0
- package/tools/ralph-external/lib/systemd-client.mjs +49 -0
- package/tools/ralph-external/lib/worker-health.mjs +1 -1
- package/tools/ralph-external/lib/worker-isolation.mjs +717 -0
- package/tools/ralph-external/lib/worker-overflow.mjs +50 -0
- package/tools/ralph-external/process-monitor.mjs +1017 -0
- package/tools/ralph-external/session-launcher.mjs +5057 -0
- package/tools/scripts/resource-preflight.mjs +541 -0
- package/tools/skills/deploy-skills-codex.mjs +2 -1
- package/vscode-extension/schemas/aiwg.config.v1.json +1194 -231
|
@@ -0,0 +1,1017 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process Monitor for External Ralph Multi-Loop
|
|
3
|
+
*
|
|
4
|
+
* Monitors multiple loop processes for health, heartbeats, and crash detection.
|
|
5
|
+
* Uses heartbeat mechanism for stale detection and resource monitoring.
|
|
6
|
+
*
|
|
7
|
+
* @implements @.aiwg/requirements/use-cases/UC-273-multi-loop-supervisor.md
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { fileURLToPath } from 'node:url';
|
|
11
|
+
import { EventEmitter } from 'events';
|
|
12
|
+
import { execFileSync } from 'child_process';
|
|
13
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync, readlinkSync, readdirSync } from 'fs';
|
|
14
|
+
import { hostname } from 'node:os';
|
|
15
|
+
import { join, dirname } from 'path';
|
|
16
|
+
import { constants } from 'node:fs';
|
|
17
|
+
import { open, opendir, rename, unlink } from 'node:fs/promises';
|
|
18
|
+
import { randomUUID } from 'node:crypto';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import { createWorkerHealthObserver } from './lib/worker-health.mjs';
|
|
21
|
+
|
|
22
|
+
// This module is loaded standalone in some contexts (e.g. staged/partial
|
|
23
|
+
// installs, or copied in isolation for a test) without its sibling
|
|
24
|
+
// tools/_resolve-impl.mjs, or without src/dist/ nearby for it to resolve.
|
|
25
|
+
// boundedExecFileSync is only needed by getProcessHealth()'s `ps` sampling
|
|
26
|
+
// below; degrade that one feature instead of failing to load this module
|
|
27
|
+
// at all for everything else (#3062 finding 1 — regression covered by
|
|
28
|
+
// `WS19 staged installed monitor loads with no src/dist or MC state` in
|
|
29
|
+
// test/conformance/orchestration-v1/worker-status.test.ts). The
|
|
30
|
+
// `../_resolve-impl.mjs` import itself must be dynamic here, not a
|
|
31
|
+
// top-level static import: a missing *static* import specifier fails
|
|
32
|
+
// module linking before any of this module's own code (including a
|
|
33
|
+
// try/catch) ever runs.
|
|
34
|
+
//
|
|
35
|
+
// Only the specific "this dependency is genuinely missing" error is
|
|
36
|
+
// tolerated (#3062 B/C finding 12): a bare `catch {}` here would also
|
|
37
|
+
// swallow a real defect in bounded-child-process.mjs itself (a syntax
|
|
38
|
+
// error, a thrown assertion, ...) and silently report every live-process
|
|
39
|
+
// health query as unavailable instead of surfacing the actual failure.
|
|
40
|
+
let boundedExecFileSync = null;
|
|
41
|
+
let processHealthUnavailableReason = null;
|
|
42
|
+
let importImpl;
|
|
43
|
+
try {
|
|
44
|
+
({ importImpl } = await import('../_resolve-impl.mjs'));
|
|
45
|
+
} catch (error) {
|
|
46
|
+
// Only a missing resolver is tolerated. Loaders report its location as a
|
|
47
|
+
// file URL (Node) or a path in the message (Vitest), so check both forms.
|
|
48
|
+
const resolverUrl = new URL('../_resolve-impl.mjs', import.meta.url);
|
|
49
|
+
const missingResolver = error?.code === 'ERR_MODULE_NOT_FOUND'
|
|
50
|
+
&& (error.url === resolverUrl.href || String(error.message).includes(fileURLToPath(resolverUrl)));
|
|
51
|
+
if (!missingResolver) throw error;
|
|
52
|
+
processHealthUnavailableReason = 'bounded-child-process helper unavailable: resolver missing (tools/_resolve-impl.mjs)';
|
|
53
|
+
}
|
|
54
|
+
if (importImpl) {
|
|
55
|
+
try {
|
|
56
|
+
({ boundedExecFileSync } = await importImpl(import.meta.url, 'process/bounded-child-process.mjs'));
|
|
57
|
+
} catch (error) {
|
|
58
|
+
if (!['AIWG_IMPLEMENTATION_MISSING', 'AIWG_PACKAGE_ROOT_MISSING'].includes(error?.code)) throw error;
|
|
59
|
+
processHealthUnavailableReason = `bounded-child-process helper unavailable: implementation missing: ${error.message}`;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Side-effect-free Linux sample. Unknown identity never grants signal authority. */
|
|
64
|
+
export function sampleWorkerIdentity(pid) {
|
|
65
|
+
if (process.platform !== 'linux' || !Number.isSafeInteger(pid) || pid <= 0) return null;
|
|
66
|
+
try {
|
|
67
|
+
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
68
|
+
const fields = stat.slice(stat.lastIndexOf(')') + 2).trim().split(/\s+/);
|
|
69
|
+
const uid = Number(/^Uid:\s+(\d+)/m.exec(readFileSync(`/proc/${pid}/status`, 'utf8'))?.[1]);
|
|
70
|
+
const bootId = readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim();
|
|
71
|
+
const executable = readlinkSync(`/proc/${pid}/exe`);
|
|
72
|
+
if (!Number.isSafeInteger(uid) || !bootId || !fields[19]) return null;
|
|
73
|
+
return { pid, pgid: Number(fields[2]), uid, parentPid: Number(fields[1]), sessionId: Number(fields[3]),
|
|
74
|
+
host: hostname(), bootId, startTicks: fields[19], executable, state: fields[0],
|
|
75
|
+
// /proc stat fields 14/15 are utime/stime; 16/17 are cutime/cstime
|
|
76
|
+
// accumulated for waited-for children, including child-heavy CLIs.
|
|
77
|
+
cpuTicks: Number(fields[11]) + Number(fields[12]) + Number(fields[13]) + Number(fields[14]) };
|
|
78
|
+
} catch { return null; }
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function sameWorkerIdentity(expected, actual) {
|
|
82
|
+
return !!expected && !!actual && ['pid', 'pgid', 'uid', 'parentPid', 'sessionId', 'host', 'bootId', 'startTicks', 'executable']
|
|
83
|
+
.every(key => expected[key] === actual[key]);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** A negative PID must name a worker group, never broadcast or select our own. */
|
|
87
|
+
export function isSafeWorkerGroupTarget(pgid, supervisor = sampleWorkerIdentity(process.pid)) {
|
|
88
|
+
return Number.isSafeInteger(pgid) && pgid > 1 && pgid !== process.pid
|
|
89
|
+
&& (supervisor ? Number.isSafeInteger(supervisor.pgid) && supervisor.pgid >= 0
|
|
90
|
+
&& pgid !== supervisor.pgid : process.platform !== 'linux');
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Two matching kernel observations, bounded from the first visible candidate.
|
|
94
|
+
* Pin each candidate's first observed birth; PID reuse can never gain ownership.
|
|
95
|
+
*/
|
|
96
|
+
export function createWorkerIdentityCapture({ now = () => Number(process.hrtime.bigint()) / 1e6,
|
|
97
|
+
timeoutMs = 250 } = {}) {
|
|
98
|
+
let deadline = null;
|
|
99
|
+
const births = new Map();
|
|
100
|
+
const reused = new Set();
|
|
101
|
+
let previous = null;
|
|
102
|
+
let candidate = null;
|
|
103
|
+
let candidateVisible = false;
|
|
104
|
+
return {
|
|
105
|
+
started: () => deadline !== null,
|
|
106
|
+
expired: () => deadline !== null && now() >= deadline,
|
|
107
|
+
observe(current) {
|
|
108
|
+
if (!current || !Number.isSafeInteger(current.pid) || current.pid <= 0 || !current.startTicks) {
|
|
109
|
+
candidateVisible = false;
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
const birth = births.get(current.pid);
|
|
113
|
+
if (!birth) births.set(current.pid, { ...current });
|
|
114
|
+
else if (['uid', 'parentPid', 'host', 'bootId', 'startTicks'].some(key => birth[key] !== current[key])) reused.add(current.pid);
|
|
115
|
+
// A fresh identity can open another window, but never clear birth pins.
|
|
116
|
+
if (deadline === null || (now() >= deadline && (!candidateVisible || !sameWorkerIdentity(candidate, current)))) {
|
|
117
|
+
deadline = now() + timeoutMs;
|
|
118
|
+
previous = null;
|
|
119
|
+
}
|
|
120
|
+
candidate = { ...current };
|
|
121
|
+
candidateVisible = true;
|
|
122
|
+
if (reused.has(current.pid) || current.pid !== current.pgid || current.uid !== process.getuid?.()) {
|
|
123
|
+
previous = null;
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
if (now() >= deadline) return null;
|
|
127
|
+
const stable = sameWorkerIdentity(previous, current);
|
|
128
|
+
previous = { ...current };
|
|
129
|
+
return stable ? current : null;
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
let workerTickRate;
|
|
135
|
+
/** Host-qualified conversion, not an assumed Linux HZ constant. */
|
|
136
|
+
export function qualifyWorkerCpuTickRate(probe = execFileSync) {
|
|
137
|
+
if (process.platform !== 'linux') return null;
|
|
138
|
+
if (probe === execFileSync && workerTickRate !== undefined) return workerTickRate;
|
|
139
|
+
let rate = null;
|
|
140
|
+
try {
|
|
141
|
+
const getconf = ['/usr/bin/getconf', '/bin/getconf'].find(filename => existsSync(filename));
|
|
142
|
+
if (!getconf) return null;
|
|
143
|
+
const value = Number(String(probe(getconf, ['CLK_TCK'], { timeout: 1000, maxBuffer: 1024,
|
|
144
|
+
stdio: ['ignore', 'pipe', 'ignore'] })).trim());
|
|
145
|
+
if (Number.isSafeInteger(value) && value > 0 && value <= 1000000) rate = value;
|
|
146
|
+
} catch { /* unavailable conversion stays null */ }
|
|
147
|
+
if (probe === execFileSync) workerTickRate = rate;
|
|
148
|
+
return rate;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function sampleWorkerCpuTime(expected, actual, tickRate, previous = null) {
|
|
152
|
+
if (!sameWorkerIdentity(expected, actual) || !Number.isFinite(tickRate) || tickRate <= 0
|
|
153
|
+
|| !Number.isSafeInteger(actual.cpuTicks) || actual.cpuTicks < 0) return { cpuTimeMs: null, deltaMs: null };
|
|
154
|
+
const cpuTimeMs = actual.cpuTicks * 1000 / tickRate;
|
|
155
|
+
const deltaMs = previous && sameWorkerIdentity(expected, previous) && Number.isSafeInteger(previous.cpuTicks)
|
|
156
|
+
&& actual.cpuTicks >= previous.cpuTicks ? (actual.cpuTicks - previous.cpuTicks) * 1000 / tickRate : null;
|
|
157
|
+
return { cpuTimeMs, deltaMs };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** A failed/partial process listing is unavailable, never proof of absence. */
|
|
161
|
+
export function sampleWorkerGroup(pgid) {
|
|
162
|
+
if (!Number.isSafeInteger(pgid) || pgid <= 0 || process.platform !== 'linux') return null;
|
|
163
|
+
try {
|
|
164
|
+
const members = [];
|
|
165
|
+
for (const entry of readdirSync('/proc')) {
|
|
166
|
+
if (!/^\d+$/.test(entry)) continue;
|
|
167
|
+
const identity = sampleWorkerIdentity(Number(entry));
|
|
168
|
+
// A disappeared process is normal; an inaccessible live member is not.
|
|
169
|
+
if (!identity) {
|
|
170
|
+
try {
|
|
171
|
+
const stat = readFileSync(`/proc/${entry}/stat`, 'utf8');
|
|
172
|
+
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(/\s+/);
|
|
173
|
+
if (Number(fields[2]) === pgid && fields[0] !== 'Z') return null;
|
|
174
|
+
} catch (error) { if (error.code !== 'ENOENT' && error.code !== 'ESRCH') return null; }
|
|
175
|
+
} else if (identity.pgid === pgid && identity.state !== 'Z') members.push(identity);
|
|
176
|
+
}
|
|
177
|
+
return members;
|
|
178
|
+
} catch { return null; }
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Read /proc/<pid>/cgroup verbatim, for comparing kernel-reported cgroup
|
|
182
|
+
* containment against a systemd unit's own ControlGroup. Side-effect-free;
|
|
183
|
+
* unreadable/absent is null, never a forged match. */
|
|
184
|
+
export function sampleProcessCgroup(pid) {
|
|
185
|
+
if (process.platform !== 'linux' || !Number.isSafeInteger(pid) || pid <= 0) return null;
|
|
186
|
+
try {
|
|
187
|
+
const raw = readFileSync(`/proc/${pid}/cgroup`, 'utf8').trim();
|
|
188
|
+
if (!raw) return null;
|
|
189
|
+
// cgroup v2 (the unified hierarchy systemd --user units use): a single
|
|
190
|
+
// "0::<path>" line. Normalize to the bare path so this compares equal to
|
|
191
|
+
// systemd's own `ControlGroup` property, which omits the "0::" prefix.
|
|
192
|
+
// Falls back to the raw multi-line content on cgroup v1/hybrid hosts.
|
|
193
|
+
const unified = raw.split('\n').find(line => line.startsWith('0::'));
|
|
194
|
+
return unified ? unified.slice(3) : raw;
|
|
195
|
+
} catch { return null; }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Walk the real /proc ppid chain starting at `pid`, re-sampling each
|
|
199
|
+
* ancestor directly from the kernel. Never trusts a worker-reported
|
|
200
|
+
* parentPid value from a disk record -- each step re-reads /proc/<ppid>
|
|
201
|
+
* itself. Returns null when the starting pid cannot be sampled (unknown
|
|
202
|
+
* ancestry never satisfies a containment check); stops at pid 1 or
|
|
203
|
+
* `maxDepth` ancestors. */
|
|
204
|
+
export function sampleProcessAncestry(pid, maxDepth = 32) {
|
|
205
|
+
let current = sampleWorkerIdentity(pid);
|
|
206
|
+
if (!current) return null;
|
|
207
|
+
const chain = [current];
|
|
208
|
+
let steps = 0;
|
|
209
|
+
while (current.pid !== 1 && steps < maxDepth) {
|
|
210
|
+
const next = sampleWorkerIdentity(current.parentPid);
|
|
211
|
+
if (!next || next.pid === current.pid) break;
|
|
212
|
+
chain.push(next);
|
|
213
|
+
current = next;
|
|
214
|
+
steps += 1;
|
|
215
|
+
}
|
|
216
|
+
return chain;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Sample only the children of a known native transport supervisor. */
|
|
220
|
+
export function sampleWorkerChildren(pid) {
|
|
221
|
+
if (!Number.isSafeInteger(pid) || pid <= 0) return null;
|
|
222
|
+
try {
|
|
223
|
+
return readFileSync(`/proc/${pid}/task/${pid}/children`, 'utf8').trim().split(/\s+/)
|
|
224
|
+
.filter(Boolean).map(value => sampleWorkerIdentity(Number(value))).filter(Boolean);
|
|
225
|
+
} catch { return null; }
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* @typedef {Object} ProcessHealth
|
|
230
|
+
* @property {number} pid - Process ID
|
|
231
|
+
* @property {number} cpu - CPU usage percentage
|
|
232
|
+
* @property {number} memory - Resident memory usage in MiB
|
|
233
|
+
* @property {number} uptime - Process uptime in seconds
|
|
234
|
+
* @property {string} status - Process status (running, zombie, sleeping)
|
|
235
|
+
*/
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* @typedef {Object} HeartbeatRecord
|
|
239
|
+
* @property {string} loopId - Loop identifier
|
|
240
|
+
* @property {number} timestamp - Heartbeat timestamp
|
|
241
|
+
* @property {number} iteration - Current iteration number
|
|
242
|
+
* @property {string} status - Loop status
|
|
243
|
+
*/
|
|
244
|
+
|
|
245
|
+
export class ProcessMonitor extends EventEmitter {
|
|
246
|
+
/**
|
|
247
|
+
* @param {Object} config
|
|
248
|
+
* @param {string} config.projectRoot - Project root directory
|
|
249
|
+
* @param {number} [config.heartbeatIntervalMs=30000] - Heartbeat check interval
|
|
250
|
+
* @param {number} [config.staleThresholdMs=60000] - Stale detection threshold
|
|
251
|
+
*/
|
|
252
|
+
constructor(config) {
|
|
253
|
+
super();
|
|
254
|
+
this.projectRoot = config.projectRoot;
|
|
255
|
+
this.heartbeatIntervalMs = config.heartbeatIntervalMs || 30000; // 30 seconds
|
|
256
|
+
this.staleThresholdMs = config.staleThresholdMs || 60000; // 1 minute
|
|
257
|
+
|
|
258
|
+
this.heartbeatDir = join(this.projectRoot, '.aiwg', 'ralph', 'heartbeats');
|
|
259
|
+
this.monitoredLoops = new Map(); // loopId -> { pid, lastCheck }
|
|
260
|
+
this.heartbeatTimer = null;
|
|
261
|
+
|
|
262
|
+
// Ensure heartbeat directory exists
|
|
263
|
+
mkdirSync(this.heartbeatDir, { recursive: true });
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Start monitoring loop processes
|
|
268
|
+
* @param {string[]} loopIds - Loop IDs to monitor
|
|
269
|
+
*/
|
|
270
|
+
startMonitoring(loopIds) {
|
|
271
|
+
// Initialize monitoring for each loop
|
|
272
|
+
for (const loopId of loopIds) {
|
|
273
|
+
const stateFile = join(
|
|
274
|
+
this.projectRoot,
|
|
275
|
+
'.aiwg',
|
|
276
|
+
'ralph',
|
|
277
|
+
'loops',
|
|
278
|
+
loopId,
|
|
279
|
+
'state.json'
|
|
280
|
+
);
|
|
281
|
+
|
|
282
|
+
if (existsSync(stateFile)) {
|
|
283
|
+
const state = JSON.parse(readFileSync(stateFile, 'utf8'));
|
|
284
|
+
this.monitoredLoops.set(loopId, {
|
|
285
|
+
pid: state.currentPid,
|
|
286
|
+
lastCheck: Date.now(),
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Start heartbeat monitoring
|
|
292
|
+
if (!this.heartbeatTimer) {
|
|
293
|
+
this.heartbeatTimer = setInterval(
|
|
294
|
+
() => this.checkAllHeartbeats(),
|
|
295
|
+
this.heartbeatIntervalMs
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Stop monitoring a specific loop
|
|
302
|
+
* @param {string} loopId - Loop ID to stop monitoring
|
|
303
|
+
*/
|
|
304
|
+
stopMonitoring(loopId) {
|
|
305
|
+
this.monitoredLoops.delete(loopId);
|
|
306
|
+
|
|
307
|
+
// Stop timer if no loops are monitored
|
|
308
|
+
if (this.monitoredLoops.size === 0 && this.heartbeatTimer) {
|
|
309
|
+
clearInterval(this.heartbeatTimer);
|
|
310
|
+
this.heartbeatTimer = null;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* Check if a process is still alive
|
|
316
|
+
* @param {number} pid - Process ID
|
|
317
|
+
* @returns {boolean}
|
|
318
|
+
*/
|
|
319
|
+
isProcessAlive(pid) {
|
|
320
|
+
if (!Number.isSafeInteger(pid) || pid <= 0) {
|
|
321
|
+
return false;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
try {
|
|
325
|
+
// Signal 0 checks if process exists without killing it
|
|
326
|
+
process.kill(pid, 0);
|
|
327
|
+
return true;
|
|
328
|
+
} catch (error) {
|
|
329
|
+
// ESRCH = No such process
|
|
330
|
+
if (error.code === 'ESRCH') {
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
// EPERM = Process exists but no permission to signal
|
|
334
|
+
if (error.code === 'EPERM') {
|
|
335
|
+
return true;
|
|
336
|
+
}
|
|
337
|
+
// Other errors, assume dead
|
|
338
|
+
return false;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Get process health metrics
|
|
344
|
+
* @param {string} loopId - Loop ID
|
|
345
|
+
* @returns {ProcessHealth|null}
|
|
346
|
+
*/
|
|
347
|
+
getProcessHealth(loopId) {
|
|
348
|
+
const monitoring = this.monitoredLoops.get(loopId);
|
|
349
|
+
if (!monitoring || !monitoring.pid) {
|
|
350
|
+
return null;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const pid = monitoring.pid;
|
|
354
|
+
|
|
355
|
+
if (!this.isProcessAlive(pid)) {
|
|
356
|
+
return {
|
|
357
|
+
pid,
|
|
358
|
+
cpu: 0,
|
|
359
|
+
memory: 0,
|
|
360
|
+
uptime: 0,
|
|
361
|
+
status: 'dead',
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
if (!boundedExecFileSync) {
|
|
366
|
+
// #3062 B/C finding 12: surface *why* no metrics are available
|
|
367
|
+
// (module header) instead of returning a bare `null` indistinguishable
|
|
368
|
+
// from "process not monitored"/"ps produced nothing useful".
|
|
369
|
+
return { pid, cpu: null, memory: null, uptime: null, status: 'unavailable', unavailableReason: processHealthUnavailableReason };
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
try {
|
|
373
|
+
// Use ps command to get process stats
|
|
374
|
+
// RSS is reported in KiB; unlike %mem it does not lose small readings
|
|
375
|
+
// to percentage rounding or require an assumed host memory capacity.
|
|
376
|
+
const output = boundedExecFileSync(
|
|
377
|
+
'ps', ['-p', String(pid), '-o', '%cpu,rss,etime,stat', '--no-headers'],
|
|
378
|
+
'quick',
|
|
379
|
+
{ encoding: 'utf8' }
|
|
380
|
+
).trim();
|
|
381
|
+
|
|
382
|
+
if (!output) {
|
|
383
|
+
return null;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
const parts = output.split(/\s+/);
|
|
387
|
+
if (parts.length !== 4 || !/^\d+(?:\.\d+)?$/.test(parts[0]) || !/^\d+$/.test(parts[1])) {
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
const cpu = Number(parts[0]);
|
|
391
|
+
const rssKiB = Number(parts[1]);
|
|
392
|
+
if (!Number.isFinite(cpu) || !Number.isSafeInteger(rssKiB)) return null;
|
|
393
|
+
const etime = parts[2];
|
|
394
|
+
const stat = parts[3];
|
|
395
|
+
|
|
396
|
+
// Parse uptime (format: [[dd-]hh:]mm:ss or mm:ss)
|
|
397
|
+
const uptime = this.parseUptime(etime);
|
|
398
|
+
if (!Number.isFinite(uptime) || uptime < 0) return null;
|
|
399
|
+
|
|
400
|
+
const memory = rssKiB / 1024;
|
|
401
|
+
|
|
402
|
+
// Parse status
|
|
403
|
+
const status = this.parseStatus(stat);
|
|
404
|
+
|
|
405
|
+
return {
|
|
406
|
+
pid,
|
|
407
|
+
cpu,
|
|
408
|
+
memory,
|
|
409
|
+
uptime,
|
|
410
|
+
status,
|
|
411
|
+
};
|
|
412
|
+
} catch (error) {
|
|
413
|
+
// Process may have died between isAlive check and ps call
|
|
414
|
+
return null;
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Parse uptime from ps etime format
|
|
420
|
+
* @param {string} etime - Elapsed time string
|
|
421
|
+
* @returns {number} - Uptime in seconds
|
|
422
|
+
*/
|
|
423
|
+
parseUptime(etime) {
|
|
424
|
+
// Format: [[dd-]hh:]mm:ss
|
|
425
|
+
let seconds = 0;
|
|
426
|
+
|
|
427
|
+
// Handle days
|
|
428
|
+
if (etime.includes('-')) {
|
|
429
|
+
const [days, rest] = etime.split('-');
|
|
430
|
+
seconds += parseInt(days, 10) * 86400;
|
|
431
|
+
etime = rest;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
const parts = etime.split(':').map(p => parseInt(p, 10));
|
|
435
|
+
|
|
436
|
+
if (parts.length === 3) {
|
|
437
|
+
// hh:mm:ss
|
|
438
|
+
seconds += parts[0] * 3600 + parts[1] * 60 + parts[2];
|
|
439
|
+
} else if (parts.length === 2) {
|
|
440
|
+
// mm:ss
|
|
441
|
+
seconds += parts[0] * 60 + parts[1];
|
|
442
|
+
} else {
|
|
443
|
+
// Just seconds
|
|
444
|
+
seconds += parts[0];
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
return seconds;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Parse process status from ps stat field
|
|
452
|
+
* @param {string} stat - Status field from ps
|
|
453
|
+
* @returns {string} - Human-readable status
|
|
454
|
+
*/
|
|
455
|
+
parseStatus(stat) {
|
|
456
|
+
const statusChar = stat.charAt(0);
|
|
457
|
+
|
|
458
|
+
const statusMap = {
|
|
459
|
+
R: 'running',
|
|
460
|
+
S: 'sleeping',
|
|
461
|
+
D: 'disk-wait',
|
|
462
|
+
Z: 'zombie',
|
|
463
|
+
T: 'stopped',
|
|
464
|
+
t: 'tracing-stop',
|
|
465
|
+
W: 'paging',
|
|
466
|
+
X: 'dead',
|
|
467
|
+
x: 'dead',
|
|
468
|
+
K: 'wakekill',
|
|
469
|
+
P: 'parked',
|
|
470
|
+
};
|
|
471
|
+
|
|
472
|
+
return statusMap[statusChar] || 'unknown';
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Record heartbeat for a loop
|
|
477
|
+
* @param {string} loopId - Loop ID
|
|
478
|
+
* @param {Object} data - Heartbeat data
|
|
479
|
+
*/
|
|
480
|
+
recordHeartbeat(loopId, data = {}) {
|
|
481
|
+
const heartbeat = {
|
|
482
|
+
loopId,
|
|
483
|
+
timestamp: Date.now(),
|
|
484
|
+
iteration: data.iteration || 0,
|
|
485
|
+
status: data.status || 'running',
|
|
486
|
+
};
|
|
487
|
+
|
|
488
|
+
const heartbeatFile = join(this.heartbeatDir, `${loopId}.json`);
|
|
489
|
+
writeFileSync(heartbeatFile, JSON.stringify(heartbeat, null, 2));
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Get last heartbeat for a loop
|
|
494
|
+
* @param {string} loopId - Loop ID
|
|
495
|
+
* @returns {HeartbeatRecord|null}
|
|
496
|
+
*/
|
|
497
|
+
getLastHeartbeat(loopId) {
|
|
498
|
+
const heartbeatFile = join(this.heartbeatDir, `${loopId}.json`);
|
|
499
|
+
|
|
500
|
+
if (!existsSync(heartbeatFile)) {
|
|
501
|
+
return null;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
try {
|
|
505
|
+
return JSON.parse(readFileSync(heartbeatFile, 'utf8'));
|
|
506
|
+
} catch {
|
|
507
|
+
return null;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Check if a loop's heartbeat is stale
|
|
513
|
+
* @param {string} loopId - Loop ID
|
|
514
|
+
* @param {number} [thresholdMs] - Stale threshold in milliseconds
|
|
515
|
+
* @returns {boolean}
|
|
516
|
+
*/
|
|
517
|
+
isStale(loopId, thresholdMs = this.staleThresholdMs) {
|
|
518
|
+
const heartbeat = this.getLastHeartbeat(loopId);
|
|
519
|
+
|
|
520
|
+
if (!heartbeat || typeof heartbeat !== 'object' || Array.isArray(heartbeat)
|
|
521
|
+
|| !Number.isFinite(heartbeat.timestamp)) {
|
|
522
|
+
// Missing or malformed records cannot establish freshness. Numeric
|
|
523
|
+
// strings are invalid too: recordHeartbeat writes a numeric timestamp.
|
|
524
|
+
return true;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
const age = Date.now() - heartbeat.timestamp;
|
|
528
|
+
return age > thresholdMs;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Check all monitored loops for heartbeat staleness
|
|
533
|
+
*/
|
|
534
|
+
checkAllHeartbeats() {
|
|
535
|
+
for (const [loopId, monitoring] of this.monitoredLoops.entries()) {
|
|
536
|
+
// Check if process is alive
|
|
537
|
+
if (monitoring.pid && !this.isProcessAlive(monitoring.pid)) {
|
|
538
|
+
this.emit('crash', {
|
|
539
|
+
loopId,
|
|
540
|
+
pid: monitoring.pid,
|
|
541
|
+
reason: 'process_died',
|
|
542
|
+
});
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// Check if heartbeat is stale
|
|
547
|
+
if (this.isStale(loopId)) {
|
|
548
|
+
this.emit('stale', {
|
|
549
|
+
loopId,
|
|
550
|
+
pid: monitoring.pid,
|
|
551
|
+
lastHeartbeat: this.getLastHeartbeat(loopId),
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Get all monitored loop IDs
|
|
559
|
+
* @returns {string[]}
|
|
560
|
+
*/
|
|
561
|
+
getMonitoredLoops() {
|
|
562
|
+
return Array.from(this.monitoredLoops.keys());
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Stop all monitoring
|
|
567
|
+
*/
|
|
568
|
+
stopAll() {
|
|
569
|
+
if (this.heartbeatTimer) {
|
|
570
|
+
clearInterval(this.heartbeatTimer);
|
|
571
|
+
this.heartbeatTimer = null;
|
|
572
|
+
}
|
|
573
|
+
this.monitoredLoops.clear();
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
export default ProcessMonitor;
|
|
578
|
+
|
|
579
|
+
const STATUS_LIMITS = Object.freeze({ directories: 256, recordBytes: 65536, files: 32, sampleMs: 5000 });
|
|
580
|
+
const HEALTH_STATES = new Set(['launching', 'started', 'active', 'quiet', 'stalled', 'hung', 'failed-to-start', 'completed', 'failed', 'cancelled', 'crashed']);
|
|
581
|
+
const STATUS_SIGNALS = new Set(['semantic-start', 'semantic-progress', 'steps', 'observer-continuity', 'cpu', 'worktree-content',
|
|
582
|
+
'monotonic-clock', 'semantic-progress-history', 'process-identity', 'request-identity', 'enforced-deadline', 'confirmed-stop']);
|
|
583
|
+
const safeLabel = value => typeof value === 'string' && /^[\p{L}\p{N}_.:/@+~%\-]{1,256}$/u.test(value) ? value : null;
|
|
584
|
+
const statusError = code => Object.assign(new Error(code), { statusCode: code });
|
|
585
|
+
const object = value => value && typeof value === 'object' && !Array.isArray(value);
|
|
586
|
+
const monotonicMs = () => Number(process.hrtime.bigint()) / 1e6;
|
|
587
|
+
const healthSeverity = state => ['hung', 'failed-to-start', 'failed', 'cancelled', 'crashed'].includes(state) ? 4
|
|
588
|
+
: state === 'stalled' ? 3 : state === 'quiet' ? 2 : ['launching', 'started'].includes(state) ? 1 : 0;
|
|
589
|
+
const coverageSeverity = coverage => coverage === 'full' ? 0 : coverage === 'degraded' ? 1 : 2;
|
|
590
|
+
|
|
591
|
+
function statusObserverOwnership(ownerRecord, actual, bootId) {
|
|
592
|
+
const expected = ownerRecord?.observer;
|
|
593
|
+
return !ownerRecord ? 'unreserved' : safeLabel(ownerRecord.owner) && expected?.uid === process.getuid()
|
|
594
|
+
&& sameWorkerIdentity(expected, actual) && actual.bootId === bootId && !['Z', 'X', 'x'].includes(actual.state)
|
|
595
|
+
? 'active' : 'unavailable';
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
/** Pin every ancestor before use, including the selection root. Linux's fd
|
|
599
|
+
* paths keep a concurrent rename from redirecting reads or scratch writes. */
|
|
600
|
+
async function openStatusDirectory(filename) {
|
|
601
|
+
if (typeof filename !== 'string' || !path.isAbsolute(filename) || filename.includes('\0')
|
|
602
|
+
|| Buffer.byteLength(filename) > 4096 || filename.split('/').length > 256
|
|
603
|
+
|| filename.split('/').some(part => part === '..' || part === '.')) throw statusError('UNSAFE_PATH');
|
|
604
|
+
let directory = await open('/', constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
605
|
+
try {
|
|
606
|
+
for (const component of filename.split('/').filter(Boolean)) {
|
|
607
|
+
const next = await open(`/proc/self/fd/${directory.fd}/${component}`, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
608
|
+
await directory.close(); directory = next;
|
|
609
|
+
}
|
|
610
|
+
const stat = await directory.stat();
|
|
611
|
+
if (stat.uid !== process.getuid() || stat.mode & 0o077) throw statusError('NONPRIVATE_DIRECTORY');
|
|
612
|
+
return directory;
|
|
613
|
+
} catch (error) { await directory.close(); throw error; }
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
async function openStatusChild(root, name) {
|
|
617
|
+
const directory = await open(`/proc/self/fd/${root.fd}/${name}`, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
618
|
+
const stat = await directory.stat();
|
|
619
|
+
if (stat.uid !== process.getuid() || stat.mode & 0o077) { await directory.close(); throw statusError('NONPRIVATE_DIRECTORY'); }
|
|
620
|
+
return directory;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
async function statusRecord(directory, name, required = false) {
|
|
624
|
+
let handle;
|
|
625
|
+
try {
|
|
626
|
+
handle = await open(`/proc/self/fd/${directory.fd}/${name}`, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
627
|
+
const stat = await handle.stat();
|
|
628
|
+
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid() || stat.mode & 0o077) throw statusError('UNSAFE_RECORD');
|
|
629
|
+
if (stat.size > STATUS_LIMITS.recordBytes) throw statusError('RECORD_TOO_LARGE');
|
|
630
|
+
const bytes = Buffer.alloc(STATUS_LIMITS.recordBytes + 1);
|
|
631
|
+
let size = 0;
|
|
632
|
+
while (size < bytes.length) {
|
|
633
|
+
const chunk = await handle.read(bytes, size, bytes.length - size, null);
|
|
634
|
+
if (!chunk.bytesRead) break;
|
|
635
|
+
size += chunk.bytesRead;
|
|
636
|
+
}
|
|
637
|
+
if (size > STATUS_LIMITS.recordBytes) throw statusError('RECORD_TOO_LARGE');
|
|
638
|
+
let value;
|
|
639
|
+
try { value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, size))); }
|
|
640
|
+
catch { throw statusError('MALFORMED_RECORD'); }
|
|
641
|
+
if (!object(value)) throw statusError('MALFORMED_RECORD');
|
|
642
|
+
return value;
|
|
643
|
+
} catch (error) { if (!required && error.code === 'ENOENT') return null; throw error; }
|
|
644
|
+
finally { await handle?.close(); }
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
async function boundedEntries(directory, max) {
|
|
648
|
+
const listing = await opendir(`/proc/self/fd/${directory.fd}`);
|
|
649
|
+
const entries = [];
|
|
650
|
+
try {
|
|
651
|
+
while (entries.length <= max) {
|
|
652
|
+
const entry = await listing.read();
|
|
653
|
+
if (!entry) return { entries: entries.sort((a, b) => a.name.localeCompare(b.name)), complete: true };
|
|
654
|
+
entries.push(entry);
|
|
655
|
+
}
|
|
656
|
+
return { entries: entries.slice(0, max).sort((a, b) => a.name.localeCompare(b.name)), complete: false };
|
|
657
|
+
} finally { await listing.close(); }
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
function validHealthHistory(health, launchTime, now) {
|
|
661
|
+
if (!object(health) || !HEALTH_STATES.has(health.state) || !['semantic', 'process-only', 'unobserved'].includes(health.handshake)
|
|
662
|
+
|| !['full', 'degraded', 'unobserved'].includes(health.coverage)) return false;
|
|
663
|
+
if (health.coverage === 'full' && health.lastObservedAt == null
|
|
664
|
+
|| health.handshake !== 'unobserved' && health.handshakeAt == null) return false;
|
|
665
|
+
for (const key of ['lastObservedAt', 'lastActivityAt', 'lastProgressAt', 'handshakeAt', 'stalledAt']) {
|
|
666
|
+
if (health[key] != null && (!Number.isFinite(Date.parse(health[key])) || Date.parse(health[key]) < launchTime || Date.parse(health[key]) > now)) return false;
|
|
667
|
+
}
|
|
668
|
+
return ['observerGeneration', 'transitionCount', 'maxCheckGapSeconds'].every(key => health[key] == null
|
|
669
|
+
|| Number.isFinite(health[key]) && health[key] >= 0 && health[key] <= 1000000)
|
|
670
|
+
&& (health.unavailableSignals == null || Array.isArray(health.unavailableSignals) && health.unavailableSignals.length <= 32
|
|
671
|
+
&& health.unavailableSignals.every(item => STATUS_SIGNALS.has(item)));
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/** Pure projection shared with attached/bridge callers. It grants no process
|
|
675
|
+
* control or acceptance authority; all disk evidence remains recorded evidence.
|
|
676
|
+
* @returns {import('../../src/mission-protocol/worker-status.js').WorkerStatusRow} */
|
|
677
|
+
export function classifyWorkerStatus(records, facts = {}) {
|
|
678
|
+
const { invocation, process: identity, health: healthRecord, exit, scratch } = records;
|
|
679
|
+
const now = facts.now ?? Date.now();
|
|
680
|
+
const id = safeLabel(invocation?.invocationId ?? identity?.invocationId);
|
|
681
|
+
if (!id) throw statusError('INVOCATION_ID_MISSING');
|
|
682
|
+
for (const record of [identity, healthRecord, exit]) {
|
|
683
|
+
if (record && record.invocationId !== id) throw statusError('INVOCATION_MISMATCH');
|
|
684
|
+
}
|
|
685
|
+
const transport = invocation?.transport ?? (identity?.request ? 'mcp' : 'shell');
|
|
686
|
+
if (!['shell', 'mcp', 'http'].includes(transport)) throw statusError('INVALID_TRANSPORT');
|
|
687
|
+
const expected = identity?.worker;
|
|
688
|
+
// A 'transport-reported-unverified' worker is a diagnostic guess, never
|
|
689
|
+
// kernel-verified ownership (see session-launcher.mjs finish()). Treating
|
|
690
|
+
// it as `expected` for live-identity comparison would let a forged or
|
|
691
|
+
// merely unlucky worker record manufacture a mismatch against
|
|
692
|
+
// `facts.actual` and downgrade an otherwise kernel-confirmed exit to
|
|
693
|
+
// uncertain/IDENTITY_CHANGED. There is nothing reliable to compare
|
|
694
|
+
// against, so suppress the signal entirely rather than risk a false
|
|
695
|
+
// mismatch.
|
|
696
|
+
const actual = identity?.identityObservation === 'transport-reported-unverified' ? null : facts.actual ?? null;
|
|
697
|
+
const matched = transport !== 'mcp' && expected?.uid === process.getuid() && sameWorkerIdentity(expected, actual)
|
|
698
|
+
&& (facts.bootId == null || actual.bootId === facts.bootId);
|
|
699
|
+
const running = matched ? !['Z', 'X', 'x'].includes(actual.state) : null;
|
|
700
|
+
const requestBound = transport !== 'mcp' || !!safeLabel(identity?.request?.requestId);
|
|
701
|
+
const boot = facts.bootId ?? actual?.bootId;
|
|
702
|
+
const base = typeof identity?.monotonicStart === 'string' && /^\d{1,24}$/.test(identity.monotonicStart)
|
|
703
|
+
? Number(BigInt(identity.monotonicStart)) / 1e6 : NaN;
|
|
704
|
+
const current = facts.monotonicNow ?? monotonicMs();
|
|
705
|
+
const launchTime = Date.parse(identity?.launchTime);
|
|
706
|
+
const clocksKnown = boot && [expected?.bootId, identity?.supervisor?.bootId].includes(boot)
|
|
707
|
+
&& Number.isFinite(base) && base <= current && Number.isFinite(launchTime) && launchTime <= now;
|
|
708
|
+
const elapsedSeconds = clocksKnown ? (current - base) / 1000 : null;
|
|
709
|
+
const timestampBound = clocksKnown ? Math.min(now, launchTime + current - base) : now;
|
|
710
|
+
let effective = invocation?.health ?? null;
|
|
711
|
+
if (effective) {
|
|
712
|
+
const fields = ['startupSeconds', 'checkSeconds', 'quietSeconds', 'stallSeconds', 'diagnosticGraceSeconds', 'terminateGraceSeconds', 'hardTimeoutSeconds', 'maxSteps'];
|
|
713
|
+
if (fields.some(key => !Number.isFinite(effective[key]) || effective[key] <= 0)
|
|
714
|
+
|| effective.quietSeconds >= effective.stallSeconds || effective.checkSeconds > effective.startupSeconds
|
|
715
|
+
|| effective.hardTimeoutSeconds > 86400 || !['native', 'external', 'observed', 'unavailable'].includes(effective.stepEnforcement)) throw statusError('INVALID_THRESHOLDS');
|
|
716
|
+
const allowed = ['version', ...fields, 'stepEnforcement', 'scale', 'profileRef', 'runDir', 'origins', 'truncatedWindows'];
|
|
717
|
+
if (!object(effective) || Object.keys(effective).some(key => !allowed.includes(key))) throw statusError('INVALID_THRESHOLDS');
|
|
718
|
+
if (effective.version !== 1 || !Number.isFinite(effective.scale) || effective.scale < 1 || effective.scale > 6
|
|
719
|
+
|| !safeLabel(effective.profileRef) || !Number.isInteger(effective.maxSteps) || effective.maxSteps > 10000
|
|
720
|
+
|| effective.truncatedWindows && (!Array.isArray(effective.truncatedWindows)
|
|
721
|
+
|| effective.truncatedWindows.length > 3 || effective.truncatedWindows.some(window => !['startup', 'quiet', 'stall'].includes(window)))) throw statusError('INVALID_THRESHOLDS');
|
|
722
|
+
if (invocation.selected?.profileRef && effective.profileRef !== invocation.selected.profileRef) throw statusError('PROFILE_MISMATCH');
|
|
723
|
+
effective = { ...effective, profileRef: safeLabel(effective.profileRef), runDir: null };
|
|
724
|
+
if (effective.origins && (!object(effective.origins) || Object.keys(effective.origins).some(key => !allowed.includes(key)) || Object.values(effective.origins).some(value =>
|
|
725
|
+
!Array.isArray(value) || value.length > 8 || value.some(item => !safeLabel(item))))) throw statusError('INVALID_THRESHOLDS');
|
|
726
|
+
}
|
|
727
|
+
let history = healthRecord?.origin === 'observer' ? healthRecord.health : null;
|
|
728
|
+
const observerOwnership = statusObserverOwnership(facts.ownerRecord, facts.actualObserver, boot);
|
|
729
|
+
const scratchTrusted = scratch && observerOwnership === 'active' && scratch.owner === facts.ownerRecord.owner
|
|
730
|
+
&& scratch.invocationId === id && validHealthHistory(scratch.health, launchTime, timestampBound);
|
|
731
|
+
const findings = scratch && !scratchTrusted ? ['OBSERVER_HISTORY_UNTRUSTED: ignored scratch without matching live observer ownership'] : [];
|
|
732
|
+
// Scratch is never launch/exit authority and may not improve live health.
|
|
733
|
+
let usingScratch = false;
|
|
734
|
+
if (scratchTrusted && (!history || Date.parse(scratch.health.lastObservedAt) > Date.parse(history.lastObservedAt))) {
|
|
735
|
+
history = scratch.health; usingScratch = true;
|
|
736
|
+
}
|
|
737
|
+
if (validHealthHistory(facts.previous?.health, launchTime, timestampBound)
|
|
738
|
+
&& (!history || Date.parse(facts.previous.health.lastObservedAt) > Date.parse(history.lastObservedAt))) history = facts.previous.health;
|
|
739
|
+
const canUseExit = exit?.termination === 'confirmed' && running !== true && (!actual || matched) && requestBound;
|
|
740
|
+
if (canUseExit && validHealthHistory(exit.health, launchTime, timestampBound)) history = exit.health;
|
|
741
|
+
const historyValid = clocksKnown && validHealthHistory(history, launchTime, timestampBound);
|
|
742
|
+
const endTime = Date.parse(exit?.endedAt);
|
|
743
|
+
const endKnown = clocksKnown && Number.isFinite(endTime) && endTime >= launchTime && endTime <= timestampBound;
|
|
744
|
+
let observationTime = current;
|
|
745
|
+
const observer = createWorkerHealthObserver({ invocationId: id, transport,
|
|
746
|
+
health: { coverage: historyValid ? history.coverage : 'unobserved',
|
|
747
|
+
handshake: historyValid ? history.handshake : 'unobserved', progressEvidence: historyValid && history.handshake === 'semantic' ? ['recorded'] : [] },
|
|
748
|
+
effectiveHealth: effective ?? undefined, startedAt: clocksKnown ? base : current,
|
|
749
|
+
launchTime: clocksKnown ? identity.launchTime : new Date(now).toISOString(),
|
|
750
|
+
now: () => observationTime, wallNow: () => now, initialSnapshot: historyValid ? { ...history,
|
|
751
|
+
state: ['completed', 'failed', 'cancelled', 'crashed'].includes(history.state) ? 'started' : history.state } : undefined });
|
|
752
|
+
if (historyValid && matched) observer.processStarted(actual);
|
|
753
|
+
const prior = facts.previous;
|
|
754
|
+
if (prior && matched) observer.sampleActivity({ running, cpu: prior.cpuTicks });
|
|
755
|
+
observer.sampleActivity({ running, cpu: matched ? actual.cpuTicks : null });
|
|
756
|
+
if (prior?.counters?.stdoutBytes !== null && facts.counters?.stdoutBytes > prior?.counters?.stdoutBytes
|
|
757
|
+
|| prior?.counters?.stderrBytes !== null && facts.counters?.stderrBytes > prior?.counters?.stderrBytes) observer.observe({ kind: 'activity' });
|
|
758
|
+
let health = canUseExit && endKnown ? observer.snapshot() : observer.check();
|
|
759
|
+
let termination = running === true ? 'running' : 'uncertain', exitCode = null, terminal = false;
|
|
760
|
+
if (exit) {
|
|
761
|
+
if (!(exit.exitCode === null || Number.isInteger(exit.exitCode) && exit.exitCode >= 0 && exit.exitCode <= 255)
|
|
762
|
+
|| !['confirmed', 'uncertain'].includes(exit.termination)) throw statusError('INVALID_EXIT');
|
|
763
|
+
exitCode = exit.exitCode;
|
|
764
|
+
// A reused or still-live PID cannot be hidden by a forged stop record.
|
|
765
|
+
if (running === true || actual && !matched) {
|
|
766
|
+
termination = 'uncertain'; observer.gap('IDENTITY_CHANGED');
|
|
767
|
+
health = observer.snapshot(); health.reasonCode = 'IDENTITY_CHANGED';
|
|
768
|
+
}
|
|
769
|
+
if (running !== true && (!actual || matched) && requestBound && exit.termination === 'confirmed') {
|
|
770
|
+
termination = 'confirmed'; terminal = true;
|
|
771
|
+
if (endKnown) {
|
|
772
|
+
// Report the wait owner's end time, not the later query time. A query
|
|
773
|
+
// after a deadline cannot turn an earlier completion into a timeout.
|
|
774
|
+
observationTime = base + endTime - launchTime;
|
|
775
|
+
if (transport === 'mcp' && exit.health?.state === 'completed') observer.observe({ kind: 'terminal', success: true });
|
|
776
|
+
health = observer.finish({ termination, exitCode, cancelled: exit.cancelled === true, timedOut: exit.timedOut === true,
|
|
777
|
+
signal: exit.signal ?? null, finalValid: historyValid && exit.health?.state === 'completed', errorCode: exit.errorCode });
|
|
778
|
+
} else { health.coverage = 'unobserved'; health.reasonCode = 'OBSERVER_GAP'; }
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
health.termination = termination;
|
|
782
|
+
if (!clocksKnown) {
|
|
783
|
+
health.startupDeadlineAt = null; health.hardDeadlineAt = null;
|
|
784
|
+
health.coverage = 'unobserved'; health.unavailableSignals.push('monotonic-clock');
|
|
785
|
+
}
|
|
786
|
+
if (!historyValid) health.unavailableSignals.push('semantic-progress-history');
|
|
787
|
+
if (!requestBound) { health.coverage = 'degraded'; health.unavailableSignals.push('request-identity'); }
|
|
788
|
+
if (expected?.pid && !matched) health.unavailableSignals.push('process-identity');
|
|
789
|
+
if (facts.cpuTimeMs == null) health.unavailableSignals.push('cpu');
|
|
790
|
+
let baseline;
|
|
791
|
+
if (usingScratch) {
|
|
792
|
+
baseline = classifyWorkerStatus({ ...records, scratch: null }, facts);
|
|
793
|
+
if (baseline.terminal || healthSeverity(baseline.health.state) > healthSeverity(health.state)) health = { ...baseline.health };
|
|
794
|
+
if (coverageSeverity(baseline.health.coverage) > coverageSeverity(health.coverage)) health.coverage = baseline.health.coverage;
|
|
795
|
+
// Do not launder newer scratch progress/activity into the next watch tick.
|
|
796
|
+
for (const key of ['lastProgressAt', 'lastActivityAt', 'handshake', 'handshakeAt']) health[key] = baseline.health[key];
|
|
797
|
+
health.unavailableSignals.push(...baseline.health.unavailableSignals);
|
|
798
|
+
}
|
|
799
|
+
if (findings.length) {
|
|
800
|
+
if (health.coverage === 'full') health.coverage = 'degraded';
|
|
801
|
+
health.unavailableSignals.push('observer-continuity');
|
|
802
|
+
}
|
|
803
|
+
health.unavailableSignals = [...new Set(health.unavailableSignals)].slice(0, 32);
|
|
804
|
+
const age = value => clocksKnown && value != null && Number.isFinite(Date.parse(value))
|
|
805
|
+
? Math.max(0, (current - base - (Date.parse(value) - launchTime)) / 1000) : null;
|
|
806
|
+
const remainingSeconds = effective && elapsedSeconds !== null ? Math.max(0, effective.hardTimeoutSeconds - elapsedSeconds) : null;
|
|
807
|
+
const action = health.reasonCode === 'HARD_DEADLINE' ? 'CHECK_DEADLINE'
|
|
808
|
+
: ['failed', 'cancelled', 'crashed', 'failed-to-start', 'hung', 'stalled'].includes(health.state) ? 'DIAGNOSE_WORKER'
|
|
809
|
+
: health.coverage !== 'full' || termination !== 'confirmed' && !matched && transport !== 'mcp' ? 'RESTORE_OBSERVATION' : 'NONE';
|
|
810
|
+
const counters = facts.counters ?? { stdoutBytes: null, stderrBytes: null };
|
|
811
|
+
return { invocationId: id, task: safeLabel(invocation?.taskId ?? invocation?.taskDigest),
|
|
812
|
+
member: safeLabel(invocation?.selected?.member), transport,
|
|
813
|
+
identity: { pid: matched ? actual.pid : null, pgid: matched ? actual.pgid : null,
|
|
814
|
+
requestId: transport === 'mcp' ? safeLabel(identity?.request?.requestId) : null, source: matched ? 'kernel' : 'recorded' },
|
|
815
|
+
health, thresholds: effective, elapsedSeconds, remainingSeconds,
|
|
816
|
+
ages: { observationSeconds: baseline && health.lastObservedAt === baseline.health.lastObservedAt
|
|
817
|
+
? baseline.ages.observationSeconds : age(historyValid ? history.lastObservedAt : null),
|
|
818
|
+
activitySeconds: age(health.lastActivityAt), progressSeconds: age(health.lastProgressAt) },
|
|
819
|
+
counters: { ...counters, cpuTimeMs: facts.cpuTimeMs ?? null, exitCode,
|
|
820
|
+
available: { stdout: counters.stdoutBytes !== null, stderr: counters.stderrBytes !== null, cpu: facts.cpuTimeMs != null, exit: exitCode !== null } },
|
|
821
|
+
nextAction: action, terminal, termination, verification: null, evidence: 'recorded', observerOwnership, findings, error: null };
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
async function captureSizes(directory) {
|
|
825
|
+
const sizes = {};
|
|
826
|
+
for (const [key, names] of Object.entries({ stdoutBytes: ['stdout.events.jsonl', 'stdout.log'], stderrBytes: ['stderr.log'] })) {
|
|
827
|
+
sizes[key] = null;
|
|
828
|
+
for (const name of names) {
|
|
829
|
+
let handle;
|
|
830
|
+
try {
|
|
831
|
+
handle = await open(`/proc/self/fd/${directory.fd}/${name}`, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
832
|
+
const stat = await handle.stat();
|
|
833
|
+
if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.getuid() || stat.mode & 0o077) throw statusError('UNSAFE_CAPTURE');
|
|
834
|
+
if (stat.size > 10485760) throw statusError('CAPTURE_TOO_LARGE');
|
|
835
|
+
sizes[key] = stat.size; break;
|
|
836
|
+
} catch (error) { if (error.code !== 'ENOENT') throw error; }
|
|
837
|
+
finally { await handle?.close(); }
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
return sizes;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
/** Readers never create a monitor, walk a worktree or execute a subprocess. */
|
|
844
|
+
export async function sampleWorkerStatus(options, memory = new Map()) {
|
|
845
|
+
const now = options.now?.() ?? Date.now(), sampledAt = new Date(now).toISOString();
|
|
846
|
+
const began = monotonicMs();
|
|
847
|
+
const report = { version: 1, sampledAt, scope: 'direct-conductor', observerCoverage: 'unobserved',
|
|
848
|
+
workers: [], breakers: [], nextCheckAt: null, complete: true, findings: [], exitCode: 0 };
|
|
849
|
+
let root;
|
|
850
|
+
try {
|
|
851
|
+
root = await openStatusDirectory(options.runDir ?? options.runsRoot);
|
|
852
|
+
const listing = options.runDir ? { entries: [{ name: null }], complete: true }
|
|
853
|
+
: await boundedEntries(root, STATUS_LIMITS.directories);
|
|
854
|
+
report.complete = listing.complete;
|
|
855
|
+
if (!listing.complete) { report.findings.push('SELECTION_LIMIT: narrow --runs-root'); report.exitCode = 2; }
|
|
856
|
+
const sampleTime = options.monotonicNow?.() ?? monotonicMs();
|
|
857
|
+
const bootId = options.bootId ?? (() => { try { return readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim(); } catch { return null; } })();
|
|
858
|
+
for (const entry of listing.entries) {
|
|
859
|
+
if (monotonicMs() - began >= STATUS_LIMITS.sampleMs) {
|
|
860
|
+
report.complete = false; report.findings.push('SAMPLE_BUDGET: narrow selection'); if (report.exitCode !== 1) report.exitCode = 2; break;
|
|
861
|
+
}
|
|
862
|
+
let directory;
|
|
863
|
+
try {
|
|
864
|
+
directory = entry.name === null ? root : await openStatusChild(root, entry.name);
|
|
865
|
+
if (!(await boundedEntries(directory, STATUS_LIMITS.files)).complete) throw statusError('FILE_LIMIT');
|
|
866
|
+
const records = { invocation: await statusRecord(directory, 'invocation.json'),
|
|
867
|
+
process: await statusRecord(directory, 'process.json', true), health: await statusRecord(directory, 'health.json'),
|
|
868
|
+
exit: await statusRecord(directory, 'exit.json'), scratch: await statusRecord(directory, 'last-observed.json') };
|
|
869
|
+
const ownerRecord = await statusRecord(directory, '.status-observer');
|
|
870
|
+
const expected = records.process.worker;
|
|
871
|
+
// classifyWorkerStatus() also ignores an unverified worker for this
|
|
872
|
+
// comparison; skip the resample too so this never samples an
|
|
873
|
+
// unrelated process merely because a diagnostic record named its PID.
|
|
874
|
+
const actual = records.process.identityObservation !== 'transport-reported-unverified'
|
|
875
|
+
&& records.invocation?.transport !== 'mcp' && !records.process.request && expected?.uid === process.getuid()
|
|
876
|
+
? (options.sampleIdentity ?? sampleWorkerIdentity)(expected.pid) : null;
|
|
877
|
+
const key = records.process.invocationId;
|
|
878
|
+
const counters = await captureSizes(directory);
|
|
879
|
+
const ownerIdentity = ownerRecord?.observer;
|
|
880
|
+
const actualObserver = ownerIdentity?.uid === process.getuid()
|
|
881
|
+
? (options.sampleIdentity ?? sampleWorkerIdentity)(ownerIdentity.pid) : null;
|
|
882
|
+
const row = classifyWorkerStatus(records, { now, monotonicNow: sampleTime, bootId, actual, counters,
|
|
883
|
+
previous: memory.get(key), ownerRecord, actualObserver });
|
|
884
|
+
report.findings.push(...row.findings.map(finding => `${row.invocationId}: ${finding}`));
|
|
885
|
+
if (row.observerOwnership === 'unavailable') report.findings.push('OBSERVER_OWNER_UNAVAILABLE: reconcile reservation before observing');
|
|
886
|
+
memory.set(key, { cpuTicks: actual?.cpuTicks ?? null, counters, health: row.health });
|
|
887
|
+
if (memory.size > STATUS_LIMITS.directories) memory.delete(memory.keys().next().value);
|
|
888
|
+
report.workers.push(row);
|
|
889
|
+
if (options.observation) {
|
|
890
|
+
try { await options.observation.write(directory, row, sampledAt); }
|
|
891
|
+
catch (error) {
|
|
892
|
+
report.complete = false; report.exitCode = 1;
|
|
893
|
+
const code = safeLabel(error.statusCode ?? error.code) ?? 'OBSERVATION_WRITE_FAILURE';
|
|
894
|
+
row.findings.push(code); report.findings.push(`${row.invocationId}: ${code}`);
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
if (monotonicMs() - began >= STATUS_LIMITS.sampleMs) {
|
|
898
|
+
report.complete = false; report.findings.push('SAMPLE_OVERRUN: observation unavailable; narrow selection');
|
|
899
|
+
row.health.coverage = 'degraded'; row.health.unavailableSignals.push('observer-continuity'); row.nextAction = 'RESTORE_OBSERVATION';
|
|
900
|
+
}
|
|
901
|
+
if (row.nextAction !== 'NONE') report.exitCode = report.exitCode === 1 ? 1 : 2;
|
|
902
|
+
} catch (error) {
|
|
903
|
+
report.complete = false; report.exitCode = 1;
|
|
904
|
+
report.workers.push({ invocationId: null, task: null, member: null, transport: null,
|
|
905
|
+
error: { code: safeLabel(error.statusCode ?? error.code) ?? 'READ_FAILURE' }, nextAction: 'REPAIR_RUN_METADATA', terminal: false });
|
|
906
|
+
} finally { if (directory && directory !== root) await directory.close(); }
|
|
907
|
+
}
|
|
908
|
+
if (!listing.entries.length) report.findings.push('EMPTY_ROOT: no recorded workers; no-dir workers require conductor-context sampling');
|
|
909
|
+
report.observerCoverage = report.workers.length && report.workers.every(row => row.health?.coverage === 'full') ? 'full'
|
|
910
|
+
: report.workers.some(row => row.health?.coverage !== 'unobserved' && row.health) ? 'degraded' : 'unobserved';
|
|
911
|
+
const checks = report.workers.filter(row => row.thresholds && !row.terminal).map(row => {
|
|
912
|
+
const delays = [row.thresholds.checkSeconds, row.remainingSeconds,
|
|
913
|
+
row.thresholds.startupSeconds - row.elapsedSeconds,
|
|
914
|
+
row.thresholds.quietSeconds - row.ages.activitySeconds, row.thresholds.stallSeconds - row.ages.progressSeconds]
|
|
915
|
+
.filter(value => Number.isFinite(value) && value > 0);
|
|
916
|
+
return Math.min(...delays);
|
|
917
|
+
});
|
|
918
|
+
if (checks.length) report.nextCheckAt = new Date(now + Math.min(...checks) * 1000).toISOString();
|
|
919
|
+
// Tier 0 has no shared breaker authority. Empty is explicit; never infer
|
|
920
|
+
// provider-wide suspension or rewrite a managed lease from this query.
|
|
921
|
+
return report;
|
|
922
|
+
} catch (error) {
|
|
923
|
+
report.complete = false; report.exitCode = 1;
|
|
924
|
+
report.findings.push(safeLabel(error.statusCode ?? error.code) ?? 'READ_FAILURE'); return report;
|
|
925
|
+
} finally { await root?.close(); }
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/** Exclusive scratch owner; all temporary files are private owner scratch.
|
|
929
|
+
* Existing launcher evidence, leases and worker files are never overwritten. */
|
|
930
|
+
export function createStatusObservationOwner() {
|
|
931
|
+
const owner = randomUUID(), reservations = new Map();
|
|
932
|
+
return {
|
|
933
|
+
async write(directory, row, sampledAt) {
|
|
934
|
+
const stat = await directory.stat(), key = `${stat.dev}:${stat.ino}`;
|
|
935
|
+
if (!reservations.has(key)) {
|
|
936
|
+
const lockPath = `/proc/self/fd/${directory.fd}/.status-observer`;
|
|
937
|
+
let lock;
|
|
938
|
+
try { lock = await open(lockPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
939
|
+
catch (error) { if (error.code === 'EEXIST') throw statusError('OBSERVER_BUSY'); throw error; }
|
|
940
|
+
const pinned = await open(`/proc/self/fd/${directory.fd}`, constants.O_RDONLY | constants.O_DIRECTORY);
|
|
941
|
+
reservations.set(key, { directory: pinned, lock, sequence: 0 });
|
|
942
|
+
await lock.writeFile(JSON.stringify({ owner, observer: sampleWorkerIdentity(process.pid) }));
|
|
943
|
+
}
|
|
944
|
+
const reservation = reservations.get(key);
|
|
945
|
+
const base = `/proc/self/fd/${reservation.directory.fd}`;
|
|
946
|
+
const temporary = `${base}/.last-observed-${owner}`;
|
|
947
|
+
const file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
948
|
+
let published = false;
|
|
949
|
+
try {
|
|
950
|
+
await file.writeFile(JSON.stringify({ version: 1, owner, sequence: ++reservation.sequence,
|
|
951
|
+
invocationId: row.invocationId, sampledAt, health: row.health, counters: row.counters }));
|
|
952
|
+
await file.sync();
|
|
953
|
+
await file.close();
|
|
954
|
+
await rename(temporary, `${base}/last-observed.json`); published = true;
|
|
955
|
+
} finally {
|
|
956
|
+
await file.close();
|
|
957
|
+
if (!published) await unlink(temporary).catch(error => { if (error.code !== 'ENOENT') throw error; });
|
|
958
|
+
}
|
|
959
|
+
},
|
|
960
|
+
async close() {
|
|
961
|
+
for (const { directory, lock } of reservations.values()) {
|
|
962
|
+
try {
|
|
963
|
+
const ownerFile = await statusRecord(directory, '.status-observer');
|
|
964
|
+
if (ownerFile?.owner === owner) await unlink(`/proc/self/fd/${directory.fd}/.status-observer`);
|
|
965
|
+
} finally { await lock.close(); await directory.close(); }
|
|
966
|
+
}
|
|
967
|
+
reservations.clear();
|
|
968
|
+
},
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
export function formatWorkerStatus(report) {
|
|
973
|
+
const label = value => value == null ? '?' : String(value);
|
|
974
|
+
const seconds = value => value == null ? '?' : `${Math.round(value * 1000) / 1000}s`;
|
|
975
|
+
return [`Workers (${report.scope}) coverage=${report.observerCoverage} complete=${report.complete} breakers=${report.breakers.length} (no shared Tier 0 breaker authority)`,
|
|
976
|
+
'invocation | member | state | coverage | age | last progress | remaining | cause | next action',
|
|
977
|
+
...report.workers.map(row => row.error ? `? | ? | unreadable | ? | ? | ? | ? | ${row.error.code} | ${row.nextAction}`
|
|
978
|
+
: `${row.invocationId} | ${label(row.member)} | ${row.health.state} | ${row.health.coverage} | ${seconds(row.elapsedSeconds)} | ${seconds(row.ages.progressSeconds)} | ${seconds(row.remainingSeconds)} | ${row.health.reasonCode} | ${row.nextAction}\n`
|
|
979
|
+
+ ` thresholds=${JSON.stringify(row.thresholds)} termination=${row.termination} verification=${label(row.verification)} observer-gap=${row.health.maxCheckGapSeconds}s`),
|
|
980
|
+
...report.findings, `next check=${label(report.nextCheckAt)}`].join('\n');
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
/** One bounded foreground loop. Timeout only stops watching, never the worker. */
|
|
984
|
+
export async function watchWorkerStatus(options, dependencies = {}) {
|
|
985
|
+
const monotonic = dependencies.monotonicNow ?? monotonicMs;
|
|
986
|
+
const sleep = dependencies.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms)));
|
|
987
|
+
const emit = dependencies.emit ?? (report => console.log(options.json ? JSON.stringify(report) : formatWorkerStatus(report)));
|
|
988
|
+
const sample = dependencies.sample ?? sampleWorkerStatus;
|
|
989
|
+
const started = monotonic(), end = started + (options.deadlineSeconds ?? 0) * 1000;
|
|
990
|
+
if (options.watch && (!Number.isFinite(options.deadlineSeconds) || options.deadlineSeconds <= 0 || options.deadlineSeconds > 86400)) throw statusError('FINITE_WATCH_DEADLINE_REQUIRED');
|
|
991
|
+
if (options.observe && !options.watch) throw statusError('OBSERVE_REQUIRES_WATCH');
|
|
992
|
+
const maxSamples = options.watch ? Math.ceil(options.deadlineSeconds) + 1 : 1;
|
|
993
|
+
let samples = 0;
|
|
994
|
+
const memory = new Map(), observation = options.observe ? createStatusObservationOwner() : null;
|
|
995
|
+
let report;
|
|
996
|
+
try {
|
|
997
|
+
do {
|
|
998
|
+
report = await sample({ ...options, observation }, memory);
|
|
999
|
+
samples++;
|
|
1000
|
+
report.watch = options.watch ? { remainingSeconds: Math.max(0, (end - monotonic()) / 1000),
|
|
1001
|
+
expired: monotonic() >= end } : null;
|
|
1002
|
+
const sampleLimit = options.watch && samples >= maxSamples && monotonic() < end;
|
|
1003
|
+
if (sampleLimit) {
|
|
1004
|
+
report.complete = false; (report.findings ??= []).push('WATCH_SAMPLE_LIMIT: sampling stopped');
|
|
1005
|
+
if (report.exitCode !== 1) report.exitCode = 2;
|
|
1006
|
+
}
|
|
1007
|
+
await emit(report);
|
|
1008
|
+
if (!options.watch || sampleLimit || report.exitCode === 1 || report.complete && report.workers.every(row => row.terminal) || monotonic() >= end) return report.exitCode;
|
|
1009
|
+
const checks = report.workers.filter(row => !row.terminal).map(row => row.thresholds?.checkSeconds * 1000)
|
|
1010
|
+
.filter(value => Number.isFinite(value) && value > 0);
|
|
1011
|
+
const resolvedCadence = checks.length ? Math.min(...checks) : 15000;
|
|
1012
|
+
const nextDelay = report.nextCheckAt ? Date.parse(report.nextCheckAt) - Date.parse(report.sampledAt) : resolvedCadence;
|
|
1013
|
+
const cadence = Math.max(1000, resolvedCadence, Number.isFinite(nextDelay) ? nextDelay : resolvedCadence);
|
|
1014
|
+
await sleep(Math.min(cadence, end - monotonic()));
|
|
1015
|
+
} while (true);
|
|
1016
|
+
} finally { await observation?.close(); }
|
|
1017
|
+
}
|