@link-assistant/hive-mind 2.15.2 → 2.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.hi.md +12 -0
- package/README.md +15 -0
- package/README.ru.md +15 -0
- package/README.zh.md +24 -12
- package/package.json +24 -17
- package/src/agent-snapshot-store.lib.mjs +252 -0
- package/src/agent.lib.mjs +25 -25
- package/src/agent.version-gates.lib.mjs +73 -0
- package/src/bot-lifecycle.lib.mjs +63 -5
- package/src/child-exit.lib.mjs +53 -1
- package/src/claude.session-tokens.lib.mjs +10 -8
- package/src/claude.session-transcript-repair.lib.mjs +65 -34
- package/src/cleanup.mjs +57 -3
- package/src/codex.lib.mjs +11 -1
- package/src/development-log.lib.mjs +22 -7
- package/src/disk-guard.lib.mjs +21 -1
- package/src/formal-ai-version.lib.mjs +10 -6
- package/src/github-error-reporter.lib.mjs +84 -2
- package/src/github.lib.mjs +212 -152
- package/src/instrument.mjs +12 -14
- package/src/instrument.sanitize.lib.mjs +52 -0
- package/src/isolation-runner.lib.mjs +56 -33
- package/src/isolation-runner.parsers.lib.mjs +29 -3
- package/src/isolation-runner.resume.lib.mjs +263 -0
- package/src/log-bounded-read.lib.mjs +411 -0
- package/src/log-sanitize-stream.lib.mjs +267 -0
- package/src/log-sanitize-worker-entry.mjs +31 -0
- package/src/log-sanitize-worker.lib.mjs +186 -0
- package/src/log-upload.lib.mjs +16 -4
- package/src/pull-request-changes.lib.mjs +1 -1
- package/src/session-completion-state.lib.mjs +124 -0
- package/src/session-kill-diagnostics.lib.mjs +117 -12
- package/src/session-kill-policy.lib.mjs +18 -7
- package/src/session-kill-resume.in-place.lib.mjs +136 -0
- package/src/session-kill-resume.lib.mjs +48 -18
- package/src/session-monitor.kill-sections.lib.mjs +8 -0
- package/src/session-monitor.lib.mjs +132 -9
- package/src/session-store.lib.mjs +15 -1
- package/src/solve.clone-errors.lib.mjs +86 -0
- package/src/solve.config.lib.mjs +5 -2
- package/src/solve.repository.lib.mjs +36 -63
- package/src/solve.resource-diagnostics.lib.mjs +105 -5
- package/src/start-command-cli.lib.mjs +60 -0
- package/src/telegram-bot.mjs +31 -91
- package/src/telegram-log-command.lib.mjs +7 -3
- package/src/telegram-overrides-validation.lib.mjs +73 -0
- package/src/telegram-terminal-watch-command.lib.mjs +9 -1
- package/src/working-session-summary.lib.mjs +1 -1
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Clone failure classification and partial-clone cleanup for the solve command.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from solve.repository.lib.mjs, which had grown past the 1350-line
|
|
5
|
+
* warning threshold that scripts/check-file-line-limits.sh enforces (issue
|
|
6
|
+
* #1593, surfaced again by issue #2198). These two helpers are pure decision
|
|
7
|
+
* logic with no dependency on the repository-setup flow around them, so they
|
|
8
|
+
* are the natural seam.
|
|
9
|
+
*
|
|
10
|
+
* Both names stay re-exported from solve.repository.lib.mjs so existing
|
|
11
|
+
* importers - and tests/anonymous-clone-auth-2192.test.mjs,
|
|
12
|
+
* tests/test-issue-1957-incomplete-clone.mjs - are unaffected.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import fs from 'node:fs/promises';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
|
|
18
|
+
import { isENOSPC } from './lib.mjs';
|
|
19
|
+
import { reportError } from './sentry.lib.mjs';
|
|
20
|
+
// Issue #2192: GitHub throttles *anonymous* git downloads; the wording overlaps
|
|
21
|
+
// the permission, not-found and rate-limit cases, so it is checked before them.
|
|
22
|
+
import { isAnonymousDownloadLimit } from './git-auth-transport.lib.mjs';
|
|
23
|
+
|
|
24
|
+
// Classify git clone errors to determine if they are retryable
|
|
25
|
+
export const classifyCloneError = errorOutput => {
|
|
26
|
+
const output = errorOutput.toLowerCase();
|
|
27
|
+
// Issue #1211: ENOSPC (disk full) errors - NOT retryable, requires user action
|
|
28
|
+
if (isENOSPC(errorOutput) || output.includes('no space left on device') || (output.includes('unable to write file') && output.includes('error')) || output.includes('errno -28')) {
|
|
29
|
+
return { type: 'ENOSPC', retryable: false, description: 'No space left on device' };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Transient server errors (5xx) - typically retryable
|
|
33
|
+
if (output.includes('error: 500') || output.includes('internal server error') || output.includes('error: 502') || output.includes('error: 503') || output.includes('error: 504')) {
|
|
34
|
+
return { type: 'TRANSIENT', retryable: true, description: 'GitHub server error' };
|
|
35
|
+
}
|
|
36
|
+
// Network-related errors - typically retryable
|
|
37
|
+
// Issue #1957: git fetch-pack/sideband disconnects (e.g.
|
|
38
|
+
// "fetch-pack: unexpected disconnect while reading sideband packet",
|
|
39
|
+
// "early EOF", "the remote end hung up unexpectedly", "RPC failed",
|
|
40
|
+
// "index-pack failed") leave an incomplete or missing clone but are transient.
|
|
41
|
+
if (output.includes('connection refused') || output.includes('connection timed out') || output.includes('connection reset') || output.includes('unable to connect') || output.includes('network is unreachable') || output.includes('ssl error') || output.includes('unexpected disconnect') || output.includes('sideband') || output.includes('early eof') || output.includes('remote end hung up') || output.includes('rpc failed') || output.includes('fetch-pack') || output.includes('index-pack failed') || output.includes('transfer closed')) {
|
|
42
|
+
return { type: 'NETWORK', retryable: true, description: 'Network connectivity issue (interrupted transfer)' };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Issue #2192: GitHub refusing an *unauthenticated* download. Retryable, but
|
|
46
|
+
// waiting is not the remedy — the clone has to be authenticated. Checked
|
|
47
|
+
// before PERMISSION/NOT_FOUND/RATE_LIMIT because GitHub's wording ("limiting",
|
|
48
|
+
// "retry later or authenticate") overlaps all three.
|
|
49
|
+
if (isAnonymousDownloadLimit(errorOutput)) {
|
|
50
|
+
return { type: 'ANONYMOUS_RATE_LIMIT', retryable: true, description: 'GitHub is limiting unauthenticated downloads (this clone was not authenticated)' };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Authentication/permission errors - not retryable
|
|
54
|
+
if (output.includes('error: 401') || output.includes('error: 403') || output.includes('authentication failed') || output.includes('permission denied')) {
|
|
55
|
+
return { type: 'PERMISSION', retryable: false, description: 'Authentication or permission error' };
|
|
56
|
+
}
|
|
57
|
+
// Repository not found - not retryable
|
|
58
|
+
if (output.includes('error: 404') || output.includes('not found') || output.includes('repository not found')) {
|
|
59
|
+
return { type: 'NOT_FOUND', retryable: false, description: 'Repository not found' };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Rate limiting - retryable with backoff
|
|
63
|
+
if (output.includes('rate limit') || output.includes('too many requests') || output.includes('api rate limit exceeded')) {
|
|
64
|
+
return { type: 'RATE_LIMIT', retryable: true, description: 'Rate limit exceeded' };
|
|
65
|
+
}
|
|
66
|
+
// Default to retryable for unknown errors
|
|
67
|
+
return { type: 'UNKNOWN', retryable: true, description: 'Unknown error' };
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
// Issue #1957: remove leftovers from an interrupted clone so a retry can start clean.
|
|
71
|
+
// We empty the directory in place (rather than removing it) because the path was
|
|
72
|
+
// created up-front by setupTempDirectory and may be the configured working directory.
|
|
73
|
+
export const cleanPartialClone = async tempDir => {
|
|
74
|
+
try {
|
|
75
|
+
const entries = await fs.readdir(tempDir);
|
|
76
|
+
for (const entry of entries) {
|
|
77
|
+
await fs.rm(path.join(tempDir, entry), { recursive: true, force: true });
|
|
78
|
+
}
|
|
79
|
+
} catch (error) {
|
|
80
|
+
// Directory may not exist yet, or be unreadable — non-fatal; the retry/clone
|
|
81
|
+
// will surface any real problem with a clearer message.
|
|
82
|
+
if (error?.code !== 'ENOENT') {
|
|
83
|
+
reportError(error, { context: 'clean_partial_clone', tempDir, operation: 'empty_directory' });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
};
|
package/src/solve.config.lib.mjs
CHANGED
|
@@ -159,9 +159,12 @@ export const SOLVE_OPTION_DEFINITIONS = {
|
|
|
159
159
|
// solve honour the exact same value, so the report never differs by surface.
|
|
160
160
|
'on-session-kill': {
|
|
161
161
|
type: 'string',
|
|
162
|
-
description: 'What to do when a working session is killed (out of memory, disk full, forced kill): "
|
|
162
|
+
description: 'What to do when a working session is killed (out of memory, disk full, forced kill): "resume" (default) starts a new working session from the killed one\'s last tool session id and says so in the pull request and in Telegram, "report" only describes exactly what happened without restarting anything. Can also be set with HIVE_MIND_ON_SESSION_KILL.',
|
|
163
163
|
choices: ['report', 'resume'],
|
|
164
|
-
|
|
164
|
+
// Issue #2189: a kill that is only ever *offered* for resume is a kill
|
|
165
|
+
// nobody recovers from — the offer in the captured incident reached its
|
|
166
|
+
// operator six hours late. Bounded by --session-kill-resume-attempts.
|
|
167
|
+
default: 'resume',
|
|
165
168
|
},
|
|
166
169
|
'session-kill-resume-attempts': {
|
|
167
170
|
type: 'number',
|
|
@@ -28,12 +28,13 @@ const { log, formatAligned } = lib;
|
|
|
28
28
|
// Import exit handler
|
|
29
29
|
import { safeExit } from './exit-handler.lib.mjs';
|
|
30
30
|
import { ensureAiToolScratchIgnored } from './ai-tool-scratch.lib.mjs';
|
|
31
|
+
import { reclaimAgentSnapshotStores } from './agent-snapshot-store.lib.mjs';
|
|
31
32
|
import { parseForkFullNameFromGhOutput } from './github-repository-names.lib.mjs';
|
|
32
33
|
import { checkReplacementRepositoryBranchSafety } from './solve.repository-safety.lib.mjs';
|
|
33
34
|
import { buildForkReplacementBlockedReason, buildForkReplacementSafetyCheckDescription } from './solve.repository-recovery-message.lib.mjs';
|
|
34
35
|
// Issue #2192: GitHub throttles *anonymous* git downloads; a token must be sent
|
|
35
36
|
// preemptively (a credential helper is never consulted for a public repository).
|
|
36
|
-
import { GIT_AUTH_TRANSPORT_DISABLE, ensureAuthenticatedGitTransport
|
|
37
|
+
import { GIT_AUTH_TRANSPORT_DISABLE, ensureAuthenticatedGitTransport } from './git-auth-transport.lib.mjs';
|
|
37
38
|
|
|
38
39
|
// Import GitHub utilities for permission checks
|
|
39
40
|
const githubLib = await import('./github.lib.mjs');
|
|
@@ -931,69 +932,12 @@ Thank you!`;
|
|
|
931
932
|
|
|
932
933
|
return { repoToClone, forkedRepo, upstreamRemote, prForkOwner: forkOwner };
|
|
933
934
|
};
|
|
934
|
-
//
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
if (lib.isENOSPC(errorOutput) || output.includes('no space left on device') || (output.includes('unable to write file') && output.includes('error')) || output.includes('errno -28')) {
|
|
939
|
-
return { type: 'ENOSPC', retryable: false, description: 'No space left on device' };
|
|
940
|
-
}
|
|
941
|
-
|
|
942
|
-
// Transient server errors (5xx) - typically retryable
|
|
943
|
-
if (output.includes('error: 500') || output.includes('internal server error') || output.includes('error: 502') || output.includes('error: 503') || output.includes('error: 504')) {
|
|
944
|
-
return { type: 'TRANSIENT', retryable: true, description: 'GitHub server error' };
|
|
945
|
-
}
|
|
946
|
-
// Network-related errors - typically retryable
|
|
947
|
-
// Issue #1957: git fetch-pack/sideband disconnects (e.g.
|
|
948
|
-
// "fetch-pack: unexpected disconnect while reading sideband packet",
|
|
949
|
-
// "early EOF", "the remote end hung up unexpectedly", "RPC failed",
|
|
950
|
-
// "index-pack failed") leave an incomplete or missing clone but are transient.
|
|
951
|
-
if (output.includes('connection refused') || output.includes('connection timed out') || output.includes('connection reset') || output.includes('unable to connect') || output.includes('network is unreachable') || output.includes('ssl error') || output.includes('unexpected disconnect') || output.includes('sideband') || output.includes('early eof') || output.includes('remote end hung up') || output.includes('rpc failed') || output.includes('fetch-pack') || output.includes('index-pack failed') || output.includes('transfer closed')) {
|
|
952
|
-
return { type: 'NETWORK', retryable: true, description: 'Network connectivity issue (interrupted transfer)' };
|
|
953
|
-
}
|
|
954
|
-
|
|
955
|
-
// Issue #2192: GitHub refusing an *unauthenticated* download. Retryable, but
|
|
956
|
-
// waiting is not the remedy — the clone has to be authenticated. Checked
|
|
957
|
-
// before PERMISSION/NOT_FOUND/RATE_LIMIT because GitHub's wording ("limiting",
|
|
958
|
-
// "retry later or authenticate") overlaps all three.
|
|
959
|
-
if (isAnonymousDownloadLimit(errorOutput)) {
|
|
960
|
-
return { type: 'ANONYMOUS_RATE_LIMIT', retryable: true, description: 'GitHub is limiting unauthenticated downloads (this clone was not authenticated)' };
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
// Authentication/permission errors - not retryable
|
|
964
|
-
if (output.includes('error: 401') || output.includes('error: 403') || output.includes('authentication failed') || output.includes('permission denied')) {
|
|
965
|
-
return { type: 'PERMISSION', retryable: false, description: 'Authentication or permission error' };
|
|
966
|
-
}
|
|
967
|
-
// Repository not found - not retryable
|
|
968
|
-
if (output.includes('error: 404') || output.includes('not found') || output.includes('repository not found')) {
|
|
969
|
-
return { type: 'NOT_FOUND', retryable: false, description: 'Repository not found' };
|
|
970
|
-
}
|
|
935
|
+
// Issue #2198: clone-failure classification and partial-clone cleanup live in
|
|
936
|
+
// their own module to keep this file under the line-limit warning threshold.
|
|
937
|
+
// Re-exported so importers keep working.
|
|
938
|
+
import { classifyCloneError, cleanPartialClone } from './solve.clone-errors.lib.mjs';
|
|
971
939
|
|
|
972
|
-
|
|
973
|
-
if (output.includes('rate limit') || output.includes('too many requests') || output.includes('api rate limit exceeded')) {
|
|
974
|
-
return { type: 'RATE_LIMIT', retryable: true, description: 'Rate limit exceeded' };
|
|
975
|
-
}
|
|
976
|
-
// Default to retryable for unknown errors
|
|
977
|
-
return { type: 'UNKNOWN', retryable: true, description: 'Unknown error' };
|
|
978
|
-
};
|
|
979
|
-
|
|
980
|
-
// Issue #1957: remove leftovers from an interrupted clone so a retry can start clean.
|
|
981
|
-
// We empty the directory in place (rather than removing it) because the path was
|
|
982
|
-
// created up-front by setupTempDirectory and may be the configured working directory.
|
|
983
|
-
export const cleanPartialClone = async tempDir => {
|
|
984
|
-
try {
|
|
985
|
-
const entries = await fs.readdir(tempDir);
|
|
986
|
-
for (const entry of entries) {
|
|
987
|
-
await fs.rm(path.join(tempDir, entry), { recursive: true, force: true });
|
|
988
|
-
}
|
|
989
|
-
} catch (error) {
|
|
990
|
-
// Directory may not exist yet, or be unreadable — non-fatal; the retry/clone
|
|
991
|
-
// will surface any real problem with a clearer message.
|
|
992
|
-
if (error?.code !== 'ENOENT') {
|
|
993
|
-
reportError(error, { context: 'clean_partial_clone', tempDir, operation: 'empty_directory' });
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
};
|
|
940
|
+
export { classifyCloneError, cleanPartialClone };
|
|
997
941
|
// Clone repository and set up remotes with retry mechanism
|
|
998
942
|
export const cloneRepository = async (repoToClone, tempDir, argv, owner, repo) => {
|
|
999
943
|
const maxRetries = 3;
|
|
@@ -1340,6 +1284,30 @@ export const checkoutPrBranch = async (tempDir, branchName, prForkRemote, prFork
|
|
|
1340
1284
|
|
|
1341
1285
|
return checkoutResult;
|
|
1342
1286
|
};
|
|
1287
|
+
/**
|
|
1288
|
+
* Reclaim orphaned `@link-assistant/agent` snapshot stores (issue #2186).
|
|
1289
|
+
*
|
|
1290
|
+
* Deliberately *not* gated on `--auto-cleanup`: the stores this removes belong to
|
|
1291
|
+
* worktrees that no longer exist, so there is nothing left to restore them into
|
|
1292
|
+
* and keeping them has no debugging value. On a public repository auto-cleanup
|
|
1293
|
+
* defaults to off, and that must not also mean "leak ~5 GB/h of home-directory
|
|
1294
|
+
* state that no Hive Mind disk check can even see".
|
|
1295
|
+
*
|
|
1296
|
+
* Never fatal: this runs while solve is finalizing, after the work is done.
|
|
1297
|
+
*/
|
|
1298
|
+
export const cleanupAgentSnapshotStores = async () => {
|
|
1299
|
+
try {
|
|
1300
|
+
const { removed } = await reclaimAgentSnapshotStores({ log: async (message, options) => log(message, options) });
|
|
1301
|
+
if (removed.length > 0) await log(`🧹 Reclaimed ${removed.length} orphaned agent snapshot store(s)`);
|
|
1302
|
+
} catch (cleanupError) {
|
|
1303
|
+
reportError(cleanupError, {
|
|
1304
|
+
context: 'cleanup_agent_snapshot_stores',
|
|
1305
|
+
operation: 'reclaim_agent_snapshots',
|
|
1306
|
+
});
|
|
1307
|
+
await log(`⚠️ Could not reclaim orphaned agent snapshot stores: ${cleanupError.message}`, { level: 'warning' });
|
|
1308
|
+
}
|
|
1309
|
+
};
|
|
1310
|
+
|
|
1343
1311
|
// Cleanup temporary directory
|
|
1344
1312
|
export const cleanupTempDirectory = async (tempDir, argv, limitReached) => {
|
|
1345
1313
|
// Determine if we should skip cleanup
|
|
@@ -1370,4 +1338,9 @@ export const cleanupTempDirectory = async (tempDir, argv, limitReached) => {
|
|
|
1370
1338
|
const reason = argv.autoCleanupSource === 'repository-visibility-default' ? 'auto-cleanup is off by default for public repositories' : '--no-auto-cleanup';
|
|
1371
1339
|
await log(`\n📁 Keeping directory (${reason}): ${tempDir}`);
|
|
1372
1340
|
}
|
|
1341
|
+
|
|
1342
|
+
// Issue #2186: whatever was decided about the workspace above, agent state
|
|
1343
|
+
// whose worktree is already gone is reclaimed. The store belonging to
|
|
1344
|
+
// `tempDir` is untouched while `tempDir` still exists.
|
|
1345
|
+
await cleanupAgentSnapshotStores();
|
|
1373
1346
|
};
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import os from 'node:os';
|
|
3
|
+
import v8 from 'node:v8';
|
|
4
|
+
|
|
5
|
+
import { measureAgentSnapshotUsage } from './agent-snapshot-store.lib.mjs';
|
|
3
6
|
|
|
4
7
|
export const RESOURCE_MARKER_PREFIX = '📈 [RESOURCES]';
|
|
5
8
|
|
|
@@ -10,8 +13,19 @@ export const RESOURCE_PHASE_SOLVE_EXIT = 'solve_exit';
|
|
|
10
13
|
export const RESOURCE_PHASE_RESTART_BEFORE = 'restart_before';
|
|
11
14
|
export const RESOURCE_PHASE_RESTART_AFTER = 'restart_after';
|
|
12
15
|
export const RESOURCE_PHASE_BOT_HEARTBEAT = 'bot_heartbeat';
|
|
16
|
+
// Issue #2189: the run that died of a V8 heap OOM inside the log sanitizer had
|
|
17
|
+
// its last resource sample at `after_agent` (RSS 373 MB), ten minutes before the
|
|
18
|
+
// fatal error — the whole log-upload phase was untelemetered, so the post-mortem
|
|
19
|
+
// could not tell a heap blow-up from an external kill. These phases bracket it.
|
|
20
|
+
export const RESOURCE_PHASE_LOG_UPLOAD_START = 'log_upload_start';
|
|
21
|
+
export const RESOURCE_PHASE_LOG_UPLOAD_END = 'log_upload_end';
|
|
22
|
+
|
|
23
|
+
// A V8 heap this close to its own limit is the shape of an imminent
|
|
24
|
+
// "FATAL ERROR: Reached heap limit" abort; surface it while the process is
|
|
25
|
+
// still alive to print it.
|
|
26
|
+
export const HEAP_PRESSURE_WARN_PERCENT = 85;
|
|
13
27
|
|
|
14
|
-
const RESOURCE_PHASES_BY_PREFERENCE = [RESOURCE_PHASE_SOLVE_EXIT, RESOURCE_PHASE_AFTER_AGENT, RESOURCE_PHASE_RESTART_AFTER, RESOURCE_PHASE_AFTER_CLONE, RESOURCE_PHASE_SOLVE_START, RESOURCE_PHASE_RESTART_BEFORE];
|
|
28
|
+
const RESOURCE_PHASES_BY_PREFERENCE = [RESOURCE_PHASE_SOLVE_EXIT, RESOURCE_PHASE_LOG_UPLOAD_END, RESOURCE_PHASE_LOG_UPLOAD_START, RESOURCE_PHASE_AFTER_AGENT, RESOURCE_PHASE_RESTART_AFTER, RESOURCE_PHASE_AFTER_CLONE, RESOURCE_PHASE_SOLVE_START, RESOURCE_PHASE_RESTART_BEFORE];
|
|
15
29
|
|
|
16
30
|
function finiteNumber(value) {
|
|
17
31
|
return Number.isFinite(value) ? value : null;
|
|
@@ -139,7 +153,7 @@ export function formatExecutionContextForLog(context) {
|
|
|
139
153
|
}
|
|
140
154
|
|
|
141
155
|
export function captureResourceSnapshot(options = {}) {
|
|
142
|
-
const { phase = 'snapshot', diskPath = '/', now = () => new Date(), osImpl = os, fsImpl = fs, processImpl = process } = options;
|
|
156
|
+
const { phase = 'snapshot', diskPath = '/', now = () => new Date(), osImpl = os, fsImpl = fs, processImpl = process, v8Impl = v8 } = options;
|
|
143
157
|
|
|
144
158
|
const timestamp = (() => {
|
|
145
159
|
try {
|
|
@@ -196,12 +210,27 @@ export function captureResourceSnapshot(options = {}) {
|
|
|
196
210
|
return {
|
|
197
211
|
rssBytes: finiteNumber(usage.rss),
|
|
198
212
|
heapUsedBytes: finiteNumber(usage.heapUsed),
|
|
213
|
+
heapTotalBytes: finiteNumber(usage.heapTotal),
|
|
214
|
+
externalBytes: finiteNumber(usage.external),
|
|
199
215
|
};
|
|
200
216
|
} catch {
|
|
201
|
-
return { rssBytes: null, heapUsedBytes: null };
|
|
217
|
+
return { rssBytes: null, heapUsedBytes: null, heapTotalBytes: null, externalBytes: null };
|
|
202
218
|
}
|
|
203
219
|
})();
|
|
204
220
|
|
|
221
|
+
// Issue #2189: the heap *limit* is the number that was missing. A process can
|
|
222
|
+
// die of "JavaScript heap out of memory" with 10 GB of the machine still free,
|
|
223
|
+
// so RSS against total RAM says nothing; used heap against `heap_size_limit`
|
|
224
|
+
// says everything.
|
|
225
|
+
const heapLimitBytes = (() => {
|
|
226
|
+
try {
|
|
227
|
+
return finiteNumber(v8Impl.getHeapStatistics().heap_size_limit);
|
|
228
|
+
} catch {
|
|
229
|
+
return null;
|
|
230
|
+
}
|
|
231
|
+
})();
|
|
232
|
+
const heapUsedPercent = Number.isFinite(processMemory.heapUsedBytes) && Number.isFinite(heapLimitBytes) && heapLimitBytes > 0 ? clampPercent((processMemory.heapUsedBytes / heapLimitBytes) * 100) : null;
|
|
233
|
+
|
|
205
234
|
const disk = (() => {
|
|
206
235
|
const path = String(diskPath || '/');
|
|
207
236
|
try {
|
|
@@ -243,6 +272,10 @@ export function captureResourceSnapshot(options = {}) {
|
|
|
243
272
|
usedBytes: usedMemoryBytes,
|
|
244
273
|
processRssBytes: processMemory.rssBytes,
|
|
245
274
|
processHeapUsedBytes: processMemory.heapUsedBytes,
|
|
275
|
+
processHeapTotalBytes: processMemory.heapTotalBytes,
|
|
276
|
+
processExternalBytes: processMemory.externalBytes,
|
|
277
|
+
processHeapLimitBytes: heapLimitBytes,
|
|
278
|
+
processHeapUsedPercent: heapUsedPercent,
|
|
246
279
|
},
|
|
247
280
|
disk,
|
|
248
281
|
};
|
|
@@ -270,11 +303,33 @@ function numberField(name, value) {
|
|
|
270
303
|
return Number.isFinite(value) ? `${name}=${value}` : `${name}=null`;
|
|
271
304
|
}
|
|
272
305
|
|
|
306
|
+
/**
|
|
307
|
+
* Human-readable "used heap of the heap limit" summary. Issue #2189: this is the
|
|
308
|
+
* single line that would have made the incident self-diagnosing.
|
|
309
|
+
*/
|
|
310
|
+
export function formatHeapUsage(memory) {
|
|
311
|
+
const m = memory || {};
|
|
312
|
+
if (!Number.isFinite(m.processHeapUsedBytes)) return 'unknown';
|
|
313
|
+
const limit = Number.isFinite(m.processHeapLimitBytes) ? ` of ${formatBytes(m.processHeapLimitBytes)} limit` : '';
|
|
314
|
+
const percent = Number.isFinite(m.processHeapUsedPercent) ? ` (${m.processHeapUsedPercent.toFixed(1)}%)` : '';
|
|
315
|
+
return `${formatBytes(m.processHeapUsedBytes)} used${limit}${percent}`;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* True when the V8 heap is close enough to its own limit that the next big
|
|
320
|
+
* allocation can abort the process (issue #2189).
|
|
321
|
+
*/
|
|
322
|
+
export function isHeapUnderPressure(memory, warnPercent = HEAP_PRESSURE_WARN_PERCENT) {
|
|
323
|
+
const percent = memory?.processHeapUsedPercent;
|
|
324
|
+
return Number.isFinite(percent) && percent >= warnPercent;
|
|
325
|
+
}
|
|
326
|
+
|
|
273
327
|
export function buildResourceMarker(snapshot) {
|
|
274
328
|
const s = snapshot || {};
|
|
275
329
|
const cpu = s.cpu || {};
|
|
276
330
|
const memory = s.memory || {};
|
|
277
331
|
const disk = s.disk || {};
|
|
332
|
+
const agentState = s.agentState || null;
|
|
278
333
|
return [
|
|
279
334
|
RESOURCE_MARKER_PREFIX,
|
|
280
335
|
`phase=${encodeValue(s.phase || 'snapshot')}`,
|
|
@@ -287,6 +342,11 @@ export function buildResourceMarker(snapshot) {
|
|
|
287
342
|
numberField('memAvailableBytes', memory.availableBytes),
|
|
288
343
|
numberField('memUsedBytes', memory.usedBytes),
|
|
289
344
|
numberField('processRssBytes', memory.processRssBytes),
|
|
345
|
+
numberField('processHeapUsedBytes', memory.processHeapUsedBytes),
|
|
346
|
+
numberField('processHeapTotalBytes', memory.processHeapTotalBytes),
|
|
347
|
+
numberField('processExternalBytes', memory.processExternalBytes),
|
|
348
|
+
numberField('processHeapLimitBytes', memory.processHeapLimitBytes),
|
|
349
|
+
numberField('processHeapUsedPercent', memory.processHeapUsedPercent),
|
|
290
350
|
`diskPath=${encodeValue(disk.path || '/')}`,
|
|
291
351
|
numberField('diskTotalBytes', disk.totalBytes),
|
|
292
352
|
numberField('diskAvailableBytes', disk.availableBytes),
|
|
@@ -294,7 +354,13 @@ export function buildResourceMarker(snapshot) {
|
|
|
294
354
|
numberField('diskUsedPercent', disk.usedPercent),
|
|
295
355
|
disk.error ? `error=${encodeValue(disk.error)}` : null,
|
|
296
356
|
`mem=${encodeValue(`${formatBytes(memory.availableBytes)} available / ${formatBytes(memory.totalBytes)} total`)}`,
|
|
357
|
+
`heap=${encodeValue(formatHeapUsage(memory))}`,
|
|
297
358
|
`disk=${encodeValue(`${formatBytes(disk.availableBytes)} available / ${formatBytes(disk.totalBytes)} total`)}`,
|
|
359
|
+
// Issue #2186: only emitted when the agent state was actually measured, so
|
|
360
|
+
// markers written before this existed keep parsing byte-for-byte the same.
|
|
361
|
+
agentState ? `agentStatePath=${encodeValue(agentState.path)}` : null,
|
|
362
|
+
agentState ? numberField('agentStoreCount', finiteNumber(agentState.count)) : null,
|
|
363
|
+
agentState ? numberField('agentStoreBytes', finiteNumber(agentState.bytes)) : null,
|
|
298
364
|
]
|
|
299
365
|
.filter(Boolean)
|
|
300
366
|
.join(' ');
|
|
@@ -332,6 +398,11 @@ function parseMarkerLine(line) {
|
|
|
332
398
|
availableBytes: parseNumber(fields.memAvailableBytes),
|
|
333
399
|
usedBytes: parseNumber(fields.memUsedBytes),
|
|
334
400
|
processRssBytes: parseNumber(fields.processRssBytes),
|
|
401
|
+
processHeapUsedBytes: parseNumber(fields.processHeapUsedBytes),
|
|
402
|
+
processHeapTotalBytes: parseNumber(fields.processHeapTotalBytes),
|
|
403
|
+
processExternalBytes: parseNumber(fields.processExternalBytes),
|
|
404
|
+
processHeapLimitBytes: parseNumber(fields.processHeapLimitBytes),
|
|
405
|
+
processHeapUsedPercent: parseNumber(fields.processHeapUsedPercent),
|
|
335
406
|
},
|
|
336
407
|
disk: {
|
|
337
408
|
path: decodeURIComponent(fields.diskPath || '/'),
|
|
@@ -341,6 +412,15 @@ function parseMarkerLine(line) {
|
|
|
341
412
|
usedPercent: parseNumber(fields.diskUsedPercent),
|
|
342
413
|
error: fields.error ? decodeURIComponent(fields.error) : null,
|
|
343
414
|
},
|
|
415
|
+
// Issue #2186: absent in markers produced before agent state was measured,
|
|
416
|
+
// and absent on hosts where the agent data home does not exist.
|
|
417
|
+
agentState: fields.agentStatePath
|
|
418
|
+
? {
|
|
419
|
+
path: decodeURIComponent(fields.agentStatePath),
|
|
420
|
+
count: parseNumber(fields.agentStoreCount),
|
|
421
|
+
bytes: parseNumber(fields.agentStoreBytes),
|
|
422
|
+
}
|
|
423
|
+
: null,
|
|
344
424
|
};
|
|
345
425
|
}
|
|
346
426
|
|
|
@@ -376,13 +456,19 @@ export function formatResourceSnapshotForLog(snapshot, label = null) {
|
|
|
376
456
|
const cpu = s.cpu || {};
|
|
377
457
|
const memory = s.memory || {};
|
|
378
458
|
const disk = s.disk || {};
|
|
379
|
-
const lines = [`📈 Resource usage (${phaseLabel}):`, ` CPU load: ${formatNumber(cpu.load1)} ${formatNumber(cpu.load5)} ${formatNumber(cpu.load15)}${Number.isFinite(cpu.cpuCount) ? ` (${cpu.cpuCount} CPUs)` : ''}`, ` Memory: ${formatBytes(memory.availableBytes)} available / ${formatBytes(memory.totalBytes)} total (${formatBytes(memory.usedBytes)} used)`, ` Process RSS: ${formatBytes(memory.processRssBytes)}
|
|
459
|
+
const lines = [`📈 Resource usage (${phaseLabel}):`, ` CPU load: ${formatNumber(cpu.load1)} ${formatNumber(cpu.load5)} ${formatNumber(cpu.load15)}${Number.isFinite(cpu.cpuCount) ? ` (${cpu.cpuCount} CPUs)` : ''}`, ` Memory: ${formatBytes(memory.availableBytes)} available / ${formatBytes(memory.totalBytes)} total (${formatBytes(memory.usedBytes)} used)`, ` Process RSS: ${formatBytes(memory.processRssBytes)}, V8 heap: ${formatHeapUsage(memory)}`, ` Disk (${disk.path || '/'}): ${formatBytes(disk.availableBytes)} available / ${formatBytes(disk.totalBytes)} total${Number.isFinite(disk.usedPercent) ? ` (${disk.usedPercent.toFixed(1)}% used)` : ''}`];
|
|
460
|
+
// Issue #2186: `/` alone hid ~5 GB/h of agent snapshot growth under
|
|
461
|
+
// `~/.local/share`, so name the directory that is actually filling up.
|
|
462
|
+
if (s.agentState && Number(s.agentState.count) > 0) {
|
|
463
|
+
lines.push(` Agent snapshot stores (${s.agentState.path}): ${s.agentState.count} store(s), ${formatBytes(s.agentState.bytes)}${s.agentState.truncated ? '+ (measurement truncated)' : ''}`);
|
|
464
|
+
}
|
|
465
|
+
if (isHeapUnderPressure(memory)) lines.push(` ⚠️ V8 heap is at ${memory.processHeapUsedPercent.toFixed(1)}% of its limit — a further allocation can abort the process with "JavaScript heap out of memory"`);
|
|
380
466
|
if (disk.error) lines.push(` Disk probe error: ${disk.error}`);
|
|
381
467
|
lines.push(buildResourceMarker(snapshot));
|
|
382
468
|
return lines.join('\n');
|
|
383
469
|
}
|
|
384
470
|
|
|
385
|
-
export async function recordResourceSnapshot({ phase, log, diskPath = '/', label = null, capture = captureResourceSnapshot, logExecutionContext = false, detectContext = detectExecutionContext } = {}) {
|
|
471
|
+
export async function recordResourceSnapshot({ phase, log, diskPath = '/', label = null, capture = captureResourceSnapshot, logExecutionContext = false, detectContext = detectExecutionContext, measureAgentState = measureAgentSnapshotUsage } = {}) {
|
|
386
472
|
if (typeof log !== 'function') return null;
|
|
387
473
|
try {
|
|
388
474
|
// Issue #2001: optionally report the execution context (host vs container)
|
|
@@ -395,6 +481,17 @@ export async function recordResourceSnapshot({ phase, log, diskPath = '/', label
|
|
|
395
481
|
}
|
|
396
482
|
}
|
|
397
483
|
const snapshot = capture({ phase, diskPath });
|
|
484
|
+
// Issue #2186: agent state lives outside `diskPath` and needs the file
|
|
485
|
+
// system, so it is measured separately and stays best-effort — a missing or
|
|
486
|
+
// unreadable agent data home must never cost us the rest of the snapshot.
|
|
487
|
+
if (typeof measureAgentState === 'function') {
|
|
488
|
+
try {
|
|
489
|
+
const agentState = await measureAgentState();
|
|
490
|
+
if (agentState && Number(agentState.count) > 0) snapshot.agentState = agentState;
|
|
491
|
+
} catch {
|
|
492
|
+
/* agent state is a diagnostic extra, not a precondition */
|
|
493
|
+
}
|
|
494
|
+
}
|
|
398
495
|
await log(formatResourceSnapshotForLog(snapshot, label));
|
|
399
496
|
return snapshot;
|
|
400
497
|
} catch (error) {
|
|
@@ -422,6 +519,9 @@ export function summarizeResourceSnapshot(snapshot) {
|
|
|
422
519
|
availableBytes: memory.availableBytes,
|
|
423
520
|
usedBytes: memory.usedBytes,
|
|
424
521
|
processRssBytes: memory.processRssBytes,
|
|
522
|
+
processHeapUsedBytes: memory.processHeapUsedBytes,
|
|
523
|
+
processHeapLimitBytes: memory.processHeapLimitBytes,
|
|
524
|
+
processHeapUsedPercent: memory.processHeapUsedPercent,
|
|
425
525
|
},
|
|
426
526
|
disk: {
|
|
427
527
|
path: disk.path,
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared access to the `$` CLI (start-command, link-foundation/start).
|
|
3
|
+
*
|
|
4
|
+
* Extracted from src/isolation-runner.lib.mjs (issue #2189) so the resume/attach
|
|
5
|
+
* wrappers added for `start-command@0.33.0` can reach the same lazily-loaded
|
|
6
|
+
* `command-stream` `$` and the same PATH lookup without importing the runner —
|
|
7
|
+
* which would create a cycle — and without duplicating either.
|
|
8
|
+
*
|
|
9
|
+
* @see https://github.com/link-foundation/start
|
|
10
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2189
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { ensureUseM } from './use-m-bootstrap.lib.mjs';
|
|
14
|
+
|
|
15
|
+
/** The message every wrapper reports when `$` is not installed. */
|
|
16
|
+
export const START_COMMAND_MISSING_ERROR = '`$` (start-command) binary not found on PATH. Install link-foundation/start.';
|
|
17
|
+
|
|
18
|
+
let commandStreamDollarPromise = null;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Lazily load `command-stream`'s `$` template tag.
|
|
22
|
+
*
|
|
23
|
+
* Cached across calls; a failed load clears the cache so a transient failure
|
|
24
|
+
* (a cold `use-m` fetch, say) does not poison every later call.
|
|
25
|
+
*
|
|
26
|
+
* @returns {Promise<Function>} The `$` template tag
|
|
27
|
+
*/
|
|
28
|
+
export async function getCommandStreamDollar() {
|
|
29
|
+
if (!commandStreamDollarPromise) {
|
|
30
|
+
commandStreamDollarPromise = (async () => {
|
|
31
|
+
if (typeof globalThis.use === 'undefined') {
|
|
32
|
+
await ensureUseM();
|
|
33
|
+
}
|
|
34
|
+
const { $ } = await globalThis.use('command-stream');
|
|
35
|
+
return $;
|
|
36
|
+
})();
|
|
37
|
+
}
|
|
38
|
+
try {
|
|
39
|
+
return await commandStreamDollarPromise;
|
|
40
|
+
} catch (error) {
|
|
41
|
+
commandStreamDollarPromise = null;
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Find the `$` CLI binary path.
|
|
48
|
+
*
|
|
49
|
+
* @returns {Promise<string|null>} Path to the `$` binary, or null when absent
|
|
50
|
+
*/
|
|
51
|
+
export async function findStartCommandBinary() {
|
|
52
|
+
try {
|
|
53
|
+
const $ = await getCommandStreamDollar();
|
|
54
|
+
const result = await $({ mirror: false })`which $`;
|
|
55
|
+
const resolved = result.stdout?.toString().trim() || '';
|
|
56
|
+
return resolved || null;
|
|
57
|
+
} catch {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
}
|