@link-assistant/hive-mind 2.0.28 → 2.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +1 -1
- package/src/bot-lifecycle.lib.mjs +17 -1
- package/src/session-monitor.lib.mjs +11 -4
- package/src/solve.mjs +14 -3
- package/src/solve.resource-diagnostics.lib.mjs +322 -0
- package/src/solve.restart-shared.lib.mjs +14 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 2.0.29
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 0cafc64: Add solve resource diagnostics and Docker disk-usage fallback markers so Telegram completion messages can show full container filesystem usage even when the task container cannot be inspected after exit.
|
|
8
|
+
|
|
3
9
|
## 2.0.28
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* @see https://github.com/link-assistant/hive-mind/issues/1927
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
+
import { RESOURCE_PHASE_BOT_HEARTBEAT, captureResourceSnapshot, summarizeResourceSnapshot } from './solve.resource-diagnostics.lib.mjs';
|
|
16
|
+
|
|
15
17
|
const DEFAULT_HEARTBEAT_INTERVAL_MS = 60 * 1000;
|
|
16
18
|
|
|
17
19
|
/**
|
|
@@ -26,14 +28,28 @@ const DEFAULT_HEARTBEAT_INTERVAL_MS = 60 * 1000;
|
|
|
26
28
|
*
|
|
27
29
|
* @returns {{ start: () => void, stop: () => void, beat: () => void, get timer(): any }}
|
|
28
30
|
*/
|
|
29
|
-
export function createHeartbeat({ logger, getActiveSessionCount, intervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS, processImpl = process, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval } = {}) {
|
|
31
|
+
export function createHeartbeat({ logger, getActiveSessionCount, intervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS, processImpl = process, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, captureResources = captureResourceSnapshot, resourceDiskPath = '/' } = {}) {
|
|
30
32
|
let timer = null;
|
|
31
33
|
|
|
32
34
|
const beat = () => {
|
|
33
35
|
try {
|
|
36
|
+
let resources = null;
|
|
37
|
+
try {
|
|
38
|
+
if (typeof captureResources === 'function') {
|
|
39
|
+
resources = summarizeResourceSnapshot(
|
|
40
|
+
captureResources({
|
|
41
|
+
phase: RESOURCE_PHASE_BOT_HEARTBEAT,
|
|
42
|
+
diskPath: resourceDiskPath,
|
|
43
|
+
})
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
} catch {
|
|
47
|
+
resources = null;
|
|
48
|
+
}
|
|
34
49
|
logger.heartbeat({
|
|
35
50
|
activeSessions: typeof getActiveSessionCount === 'function' ? getActiveSessionCount(false) : undefined,
|
|
36
51
|
uptimeSec: Math.floor(processImpl.uptime()),
|
|
52
|
+
resources,
|
|
37
53
|
});
|
|
38
54
|
} catch {
|
|
39
55
|
/* heartbeat must never crash the bot */
|
|
@@ -343,10 +343,11 @@ async function resolvePullRequestUrlFromSessionLog(logPath, ctx, { verbose = fal
|
|
|
343
343
|
* build the Telegram extraSection. Returns an empty string if there is no
|
|
344
344
|
* repository or docker filesystem data to show.
|
|
345
345
|
*/
|
|
346
|
-
async function buildDiskDiagnosticsExtraSection(logPath, { verbose = false, readFile = fs.readFile, isolationBackend = null, containerFilesystemStartBytes = null, containerFilesystemAfterBytes = null } = {}) {
|
|
346
|
+
export async function buildDiskDiagnosticsExtraSection(logPath, { verbose = false, readFile = fs.readFile, isolationBackend = null, containerFilesystemStartBytes = null, containerFilesystemAfterBytes = null } = {}) {
|
|
347
347
|
if (!logPath && !Number.isFinite(containerFilesystemStartBytes) && !Number.isFinite(containerFilesystemAfterBytes)) return '';
|
|
348
348
|
try {
|
|
349
349
|
const diskLib = await import('./solve.disk-diagnostics.lib.mjs');
|
|
350
|
+
const resourceLib = await import('./solve.resource-diagnostics.lib.mjs');
|
|
350
351
|
let logText = '';
|
|
351
352
|
if (logPath) {
|
|
352
353
|
try {
|
|
@@ -358,11 +359,17 @@ async function buildDiskDiagnosticsExtraSection(logPath, { verbose = false, read
|
|
|
358
359
|
}
|
|
359
360
|
}
|
|
360
361
|
const parsed = diskLib.parseDiskMarkers(logText);
|
|
361
|
-
|
|
362
|
+
const parsedResources = resourceLib.parseResourceMarkers(logText);
|
|
363
|
+
const bestResourceMarker = resourceLib.selectBestDiskResourceMarker(parsedResources);
|
|
364
|
+
const solveStartResourceMarker = parsedResources.byPhase?.[resourceLib.RESOURCE_PHASE_SOLVE_START] || null;
|
|
365
|
+
const useResourceFallback = String(isolationBackend || '').toLowerCase() === 'docker' && !Number.isFinite(containerFilesystemAfterBytes) && Number.isFinite(bestResourceMarker?.disk?.usedBytes);
|
|
366
|
+
const effectiveContainerFilesystemAfterBytes = useResourceFallback ? bestResourceMarker.disk.usedBytes : containerFilesystemAfterBytes;
|
|
367
|
+
const effectiveContainerFilesystemStartBytes = useResourceFallback && Number.isFinite(solveStartResourceMarker?.disk?.usedBytes) ? solveStartResourceMarker.disk.usedBytes : containerFilesystemStartBytes;
|
|
368
|
+
if (!parsed.afterClone && !parsed.afterAgent && !Number.isFinite(effectiveContainerFilesystemStartBytes) && !Number.isFinite(effectiveContainerFilesystemAfterBytes)) return '';
|
|
362
369
|
return diskLib.formatDiskDiagnosticsBlock(parsed, {
|
|
363
370
|
isolationBackend,
|
|
364
|
-
containerFilesystemStartBytes,
|
|
365
|
-
containerFilesystemAfterBytes,
|
|
371
|
+
containerFilesystemStartBytes: effectiveContainerFilesystemStartBytes,
|
|
372
|
+
containerFilesystemAfterBytes: effectiveContainerFilesystemAfterBytes,
|
|
366
373
|
});
|
|
367
374
|
} catch (error) {
|
|
368
375
|
if (verbose) {
|
package/src/solve.mjs
CHANGED
|
@@ -49,7 +49,8 @@ const { runKeepWorkingUntilDone } = await import('./solve.keep-working.lib.mjs')
|
|
|
49
49
|
const { runEscalation } = await import('./solve.escalate.lib.mjs');
|
|
50
50
|
const { finalizeSolveProcess } = await import('./solve.finalize.lib.mjs');
|
|
51
51
|
const exitHandler = await import('./exit-handler.lib.mjs');
|
|
52
|
-
const { initializeExitHandler, installGlobalExitHandlers, safeExit, logActiveHandles } = exitHandler;
|
|
52
|
+
const { initializeExitHandler, installGlobalExitHandlers, safeExit: baseSafeExit, logActiveHandles } = exitHandler;
|
|
53
|
+
const { RESOURCE_PHASE_AFTER_AGENT, RESOURCE_PHASE_AFTER_CLONE, RESOURCE_PHASE_SOLVE_EXIT, RESOURCE_PHASE_SOLVE_START, recordResourceSnapshot } = await import('./solve.resource-diagnostics.lib.mjs');
|
|
53
54
|
const { createInterruptWrapper } = await import('./solve.interrupt.lib.mjs');
|
|
54
55
|
// Issue #1823: working-session guard for --do-not-shutdown-in-the-middle-of-working-session.
|
|
55
56
|
const { configureWorkingSession, beginWorkingSession, endWorkingSession } = await import('./working-session.lib.mjs');
|
|
@@ -69,9 +70,7 @@ const { prepareFeedbackAndTimestamps, checkUncommittedChanges, checkForkActions
|
|
|
69
70
|
const { validateAndExitOnInvalidClaudeSubAgentModel, validateAndExitOnInvalidModel } = await import('./models/index.mjs');
|
|
70
71
|
const { autoAcceptInviteForRepo } = await import('./solve.accept-invite.lib.mjs');
|
|
71
72
|
const { handleAutoForkOption, handleMaintainerForkAccess } = await import('./solve.fork-detection.lib.mjs');
|
|
72
|
-
// Initialize log file early (before argument parsing) to capture all output
|
|
73
73
|
const logFile = await initializeLogFile(null);
|
|
74
|
-
// Log version and raw command IMMEDIATELY after log file initialization
|
|
75
74
|
const versionInfo = await getVersionInfo();
|
|
76
75
|
await log('');
|
|
77
76
|
await log(`🚀 solve v${versionInfo}`);
|
|
@@ -80,6 +79,15 @@ await log('🔧 Raw command executed:');
|
|
|
80
79
|
await log(` ${rawCommand}`);
|
|
81
80
|
await log('');
|
|
82
81
|
|
|
82
|
+
let finalResourceSnapshotRecorded = false;
|
|
83
|
+
const safeExit = async (code = 0, reason = 'Process completed', options = {}) => {
|
|
84
|
+
if (!finalResourceSnapshotRecorded) {
|
|
85
|
+
finalResourceSnapshotRecorded = true;
|
|
86
|
+
await recordResourceSnapshot({ phase: RESOURCE_PHASE_SOLVE_EXIT, log, diskPath: '/', label: `solve exit ${code}` });
|
|
87
|
+
}
|
|
88
|
+
return await baseSafeExit(code, reason, options);
|
|
89
|
+
};
|
|
90
|
+
|
|
83
91
|
let argv;
|
|
84
92
|
try {
|
|
85
93
|
argv = await parseArguments(yargs, hideBin);
|
|
@@ -98,6 +106,7 @@ configureGitHubRateLimitLogging({
|
|
|
98
106
|
enabled: argv.githubRateLimitsLogging === true,
|
|
99
107
|
log,
|
|
100
108
|
});
|
|
109
|
+
await recordResourceSnapshot({ phase: RESOURCE_PHASE_SOLVE_START, log, diskPath: '/', label: 'solve start' });
|
|
101
110
|
|
|
102
111
|
// Early logs go to cwd; custom log dir takes effect after argv is parsed
|
|
103
112
|
// Conditionally import tool-specific functions after argv is parsed
|
|
@@ -507,6 +516,7 @@ try {
|
|
|
507
516
|
});
|
|
508
517
|
|
|
509
518
|
cleanupContext.diskDiagnostics = { beforeBytes: await recordAfterCloneSize({ tempDir, log }) };
|
|
519
|
+
await recordResourceSnapshot({ phase: RESOURCE_PHASE_AFTER_CLONE, log, diskPath: '/', label: 'after repository clone' });
|
|
510
520
|
|
|
511
521
|
// Verify default branch and status using the new module
|
|
512
522
|
// Pass argv, owner, repo, issueUrl for empty repository auto-initialization (--auto-init-repository)
|
|
@@ -830,6 +840,7 @@ try {
|
|
|
830
840
|
} catch (diskError) {
|
|
831
841
|
await log(`⚠️ Disk-size measurement failed: ${cleanErrorMessage(diskError)}`, { level: 'warning', verbose: true });
|
|
832
842
|
}
|
|
843
|
+
await recordResourceSnapshot({ phase: RESOURCE_PHASE_AFTER_AGENT, log, diskPath: '/', label: 'after AI execution' });
|
|
833
844
|
|
|
834
845
|
// Issue #1823: Mark the end of the AI working session. If a graceful-shutdown interrupt arrived
|
|
835
846
|
// during the session (deferred by the working-session guard), honor it now: auto-commit any
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
|
|
4
|
+
export const RESOURCE_MARKER_PREFIX = '📈 [RESOURCES]';
|
|
5
|
+
|
|
6
|
+
export const RESOURCE_PHASE_SOLVE_START = 'solve_start';
|
|
7
|
+
export const RESOURCE_PHASE_AFTER_CLONE = 'after_clone';
|
|
8
|
+
export const RESOURCE_PHASE_AFTER_AGENT = 'after_agent';
|
|
9
|
+
export const RESOURCE_PHASE_SOLVE_EXIT = 'solve_exit';
|
|
10
|
+
export const RESOURCE_PHASE_RESTART_BEFORE = 'restart_before';
|
|
11
|
+
export const RESOURCE_PHASE_RESTART_AFTER = 'restart_after';
|
|
12
|
+
export const RESOURCE_PHASE_BOT_HEARTBEAT = 'bot_heartbeat';
|
|
13
|
+
|
|
14
|
+
const RESOURCE_PHASES_BY_PREFERENCE = [RESOURCE_PHASE_SOLVE_EXIT, RESOURCE_PHASE_AFTER_AGENT, RESOURCE_PHASE_RESTART_AFTER, RESOURCE_PHASE_AFTER_CLONE, RESOURCE_PHASE_SOLVE_START, RESOURCE_PHASE_RESTART_BEFORE];
|
|
15
|
+
|
|
16
|
+
function finiteNumber(value) {
|
|
17
|
+
return Number.isFinite(value) ? value : null;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function clampPercent(value) {
|
|
21
|
+
if (!Number.isFinite(value)) return null;
|
|
22
|
+
return Math.max(0, Math.min(100, value));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function readLinuxMemAvailableBytes(readFileSync = fs.readFileSync, platform = process.platform) {
|
|
26
|
+
if (platform !== 'linux') return null;
|
|
27
|
+
try {
|
|
28
|
+
const text = readFileSync('/proc/meminfo', 'utf8');
|
|
29
|
+
const match = text.match(/^MemAvailable:\s+(\d+)\s+kB$/m);
|
|
30
|
+
if (!match) return null;
|
|
31
|
+
return Number.parseInt(match[1], 10) * 1024;
|
|
32
|
+
} catch {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function captureResourceSnapshot(options = {}) {
|
|
38
|
+
const { phase = 'snapshot', diskPath = '/', now = () => new Date(), osImpl = os, fsImpl = fs, processImpl = process } = options;
|
|
39
|
+
|
|
40
|
+
const timestamp = (() => {
|
|
41
|
+
try {
|
|
42
|
+
return now().toISOString();
|
|
43
|
+
} catch {
|
|
44
|
+
return new Date().toISOString();
|
|
45
|
+
}
|
|
46
|
+
})();
|
|
47
|
+
|
|
48
|
+
const load = (() => {
|
|
49
|
+
try {
|
|
50
|
+
const values = osImpl.loadavg();
|
|
51
|
+
return {
|
|
52
|
+
load1: finiteNumber(values[0]),
|
|
53
|
+
load5: finiteNumber(values[1]),
|
|
54
|
+
load15: finiteNumber(values[2]),
|
|
55
|
+
};
|
|
56
|
+
} catch {
|
|
57
|
+
return { load1: null, load5: null, load15: null };
|
|
58
|
+
}
|
|
59
|
+
})();
|
|
60
|
+
|
|
61
|
+
const cpuCount = (() => {
|
|
62
|
+
try {
|
|
63
|
+
const cpus = osImpl.cpus();
|
|
64
|
+
return Array.isArray(cpus) ? cpus.length : null;
|
|
65
|
+
} catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
})();
|
|
69
|
+
|
|
70
|
+
const totalMemoryBytes = (() => {
|
|
71
|
+
try {
|
|
72
|
+
return finiteNumber(osImpl.totalmem());
|
|
73
|
+
} catch {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
})();
|
|
77
|
+
|
|
78
|
+
const freeMemoryBytes = (() => {
|
|
79
|
+
try {
|
|
80
|
+
return finiteNumber(osImpl.freemem());
|
|
81
|
+
} catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
})();
|
|
85
|
+
|
|
86
|
+
const availableMemoryBytes = readLinuxMemAvailableBytes(fsImpl.readFileSync?.bind(fsImpl), processImpl.platform || process.platform) ?? freeMemoryBytes;
|
|
87
|
+
const usedMemoryBytes = totalMemoryBytes !== null && availableMemoryBytes !== null ? Math.max(0, totalMemoryBytes - availableMemoryBytes) : null;
|
|
88
|
+
|
|
89
|
+
const processMemory = (() => {
|
|
90
|
+
try {
|
|
91
|
+
const usage = processImpl.memoryUsage();
|
|
92
|
+
return {
|
|
93
|
+
rssBytes: finiteNumber(usage.rss),
|
|
94
|
+
heapUsedBytes: finiteNumber(usage.heapUsed),
|
|
95
|
+
};
|
|
96
|
+
} catch {
|
|
97
|
+
return { rssBytes: null, heapUsedBytes: null };
|
|
98
|
+
}
|
|
99
|
+
})();
|
|
100
|
+
|
|
101
|
+
const disk = (() => {
|
|
102
|
+
const path = String(diskPath || '/');
|
|
103
|
+
try {
|
|
104
|
+
if (typeof fsImpl.statfsSync !== 'function') {
|
|
105
|
+
return { path, totalBytes: null, freeBytes: null, availableBytes: null, usedBytes: null, usedPercent: null, error: 'statfs unavailable' };
|
|
106
|
+
}
|
|
107
|
+
const stat = fsImpl.statfsSync(path);
|
|
108
|
+
const blockSize = Number(stat.bsize || stat.frsize || 0);
|
|
109
|
+
const blocks = Number(stat.blocks);
|
|
110
|
+
const bfree = Number(stat.bfree);
|
|
111
|
+
const bavail = Number(stat.bavail);
|
|
112
|
+
const totalBytes = Number.isFinite(blockSize) && Number.isFinite(blocks) ? blockSize * blocks : null;
|
|
113
|
+
const freeBytes = Number.isFinite(blockSize) && Number.isFinite(bfree) ? blockSize * bfree : null;
|
|
114
|
+
const availableBytes = Number.isFinite(blockSize) && Number.isFinite(bavail) ? blockSize * bavail : freeBytes;
|
|
115
|
+
const usedBytes = totalBytes !== null && freeBytes !== null ? Math.max(0, totalBytes - freeBytes) : null;
|
|
116
|
+
const usedPercent = totalBytes && usedBytes !== null ? clampPercent((usedBytes / totalBytes) * 100) : null;
|
|
117
|
+
return { path, totalBytes, freeBytes, availableBytes, usedBytes, usedPercent, error: null };
|
|
118
|
+
} catch (error) {
|
|
119
|
+
return {
|
|
120
|
+
path,
|
|
121
|
+
totalBytes: null,
|
|
122
|
+
freeBytes: null,
|
|
123
|
+
availableBytes: null,
|
|
124
|
+
usedBytes: null,
|
|
125
|
+
usedPercent: null,
|
|
126
|
+
error: error?.message || String(error),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
})();
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
phase: String(phase || 'snapshot'),
|
|
133
|
+
timestamp,
|
|
134
|
+
cpu: { ...load, cpuCount },
|
|
135
|
+
memory: {
|
|
136
|
+
totalBytes: totalMemoryBytes,
|
|
137
|
+
freeBytes: freeMemoryBytes,
|
|
138
|
+
availableBytes: availableMemoryBytes,
|
|
139
|
+
usedBytes: usedMemoryBytes,
|
|
140
|
+
processRssBytes: processMemory.rssBytes,
|
|
141
|
+
processHeapUsedBytes: processMemory.heapUsedBytes,
|
|
142
|
+
},
|
|
143
|
+
disk,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export function formatBytes(bytes) {
|
|
148
|
+
if (!Number.isFinite(bytes)) return '? B';
|
|
149
|
+
const abs = Math.abs(bytes);
|
|
150
|
+
if (abs >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
|
|
151
|
+
if (abs >= 1024 ** 2) return `${Math.round(bytes / 1024 ** 2)} MB`;
|
|
152
|
+
if (abs >= 1024) return `${Math.round(bytes / 1024)} KB`;
|
|
153
|
+
return `${Math.round(bytes)} B`;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function formatNumber(value, decimals = 2) {
|
|
157
|
+
return Number.isFinite(value) ? value.toFixed(decimals) : '?';
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function encodeValue(value) {
|
|
161
|
+
if (value === null || value === undefined) return 'null';
|
|
162
|
+
return encodeURIComponent(String(value));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function numberField(name, value) {
|
|
166
|
+
return Number.isFinite(value) ? `${name}=${value}` : `${name}=null`;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function buildResourceMarker(snapshot) {
|
|
170
|
+
const s = snapshot || {};
|
|
171
|
+
const cpu = s.cpu || {};
|
|
172
|
+
const memory = s.memory || {};
|
|
173
|
+
const disk = s.disk || {};
|
|
174
|
+
return [
|
|
175
|
+
RESOURCE_MARKER_PREFIX,
|
|
176
|
+
`phase=${encodeValue(s.phase || 'snapshot')}`,
|
|
177
|
+
`ts=${encodeValue(s.timestamp || new Date().toISOString())}`,
|
|
178
|
+
numberField('load1', cpu.load1),
|
|
179
|
+
numberField('load5', cpu.load5),
|
|
180
|
+
numberField('load15', cpu.load15),
|
|
181
|
+
numberField('cpuCount', cpu.cpuCount),
|
|
182
|
+
numberField('memTotalBytes', memory.totalBytes),
|
|
183
|
+
numberField('memAvailableBytes', memory.availableBytes),
|
|
184
|
+
numberField('memUsedBytes', memory.usedBytes),
|
|
185
|
+
numberField('processRssBytes', memory.processRssBytes),
|
|
186
|
+
`diskPath=${encodeValue(disk.path || '/')}`,
|
|
187
|
+
numberField('diskTotalBytes', disk.totalBytes),
|
|
188
|
+
numberField('diskAvailableBytes', disk.availableBytes),
|
|
189
|
+
numberField('diskUsedBytes', disk.usedBytes),
|
|
190
|
+
numberField('diskUsedPercent', disk.usedPercent),
|
|
191
|
+
disk.error ? `error=${encodeValue(disk.error)}` : null,
|
|
192
|
+
`mem=${encodeValue(`${formatBytes(memory.availableBytes)} available / ${formatBytes(memory.totalBytes)} total`)}`,
|
|
193
|
+
`disk=${encodeValue(`${formatBytes(disk.availableBytes)} available / ${formatBytes(disk.totalBytes)} total`)}`,
|
|
194
|
+
]
|
|
195
|
+
.filter(Boolean)
|
|
196
|
+
.join(' ');
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function parseNumber(value) {
|
|
200
|
+
if (value === 'null' || value === undefined) return null;
|
|
201
|
+
const n = Number(value);
|
|
202
|
+
return Number.isFinite(n) ? n : null;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function parseMarkerLine(line) {
|
|
206
|
+
const idx = line.indexOf(RESOURCE_MARKER_PREFIX);
|
|
207
|
+
if (idx < 0) return null;
|
|
208
|
+
const payload = line.slice(idx + RESOURCE_MARKER_PREFIX.length).trim();
|
|
209
|
+
const parts = payload.split(/\s+/).filter(Boolean);
|
|
210
|
+
const fields = {};
|
|
211
|
+
for (const part of parts) {
|
|
212
|
+
const eq = part.indexOf('=');
|
|
213
|
+
if (eq <= 0) continue;
|
|
214
|
+
fields[part.slice(0, eq)] = part.slice(eq + 1);
|
|
215
|
+
}
|
|
216
|
+
const phase = decodeURIComponent(fields.phase || 'snapshot');
|
|
217
|
+
return {
|
|
218
|
+
phase,
|
|
219
|
+
timestamp: decodeURIComponent(fields.ts || ''),
|
|
220
|
+
cpu: {
|
|
221
|
+
load1: parseNumber(fields.load1),
|
|
222
|
+
load5: parseNumber(fields.load5),
|
|
223
|
+
load15: parseNumber(fields.load15),
|
|
224
|
+
cpuCount: parseNumber(fields.cpuCount),
|
|
225
|
+
},
|
|
226
|
+
memory: {
|
|
227
|
+
totalBytes: parseNumber(fields.memTotalBytes),
|
|
228
|
+
availableBytes: parseNumber(fields.memAvailableBytes),
|
|
229
|
+
usedBytes: parseNumber(fields.memUsedBytes),
|
|
230
|
+
processRssBytes: parseNumber(fields.processRssBytes),
|
|
231
|
+
},
|
|
232
|
+
disk: {
|
|
233
|
+
path: decodeURIComponent(fields.diskPath || '/'),
|
|
234
|
+
totalBytes: parseNumber(fields.diskTotalBytes),
|
|
235
|
+
availableBytes: parseNumber(fields.diskAvailableBytes),
|
|
236
|
+
usedBytes: parseNumber(fields.diskUsedBytes),
|
|
237
|
+
usedPercent: parseNumber(fields.diskUsedPercent),
|
|
238
|
+
error: fields.error ? decodeURIComponent(fields.error) : null,
|
|
239
|
+
},
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
export function parseResourceMarkers(logText) {
|
|
244
|
+
if (typeof logText !== 'string' || !logText) return { markers: [], byPhase: {} };
|
|
245
|
+
const markers = [];
|
|
246
|
+
const byPhase = {};
|
|
247
|
+
for (const line of logText.split(/\r?\n/)) {
|
|
248
|
+
const marker = parseMarkerLine(line);
|
|
249
|
+
if (!marker) continue;
|
|
250
|
+
markers.push(marker);
|
|
251
|
+
byPhase[marker.phase] = marker;
|
|
252
|
+
}
|
|
253
|
+
return { markers, byPhase };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
export function selectBestDiskResourceMarker(parsed) {
|
|
257
|
+
const byPhase = parsed?.byPhase || {};
|
|
258
|
+
for (const phase of RESOURCE_PHASES_BY_PREFERENCE) {
|
|
259
|
+
const marker = byPhase[phase];
|
|
260
|
+
if (Number.isFinite(marker?.disk?.usedBytes)) return marker;
|
|
261
|
+
}
|
|
262
|
+
const markers = Array.isArray(parsed?.markers) ? parsed.markers : [];
|
|
263
|
+
for (let i = markers.length - 1; i >= 0; i--) {
|
|
264
|
+
if (Number.isFinite(markers[i]?.disk?.usedBytes)) return markers[i];
|
|
265
|
+
}
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export function formatResourceSnapshotForLog(snapshot, label = null) {
|
|
270
|
+
const s = snapshot || {};
|
|
271
|
+
const phaseLabel = label || String(s.phase || 'snapshot').replace(/_/g, ' ');
|
|
272
|
+
const cpu = s.cpu || {};
|
|
273
|
+
const memory = s.memory || {};
|
|
274
|
+
const disk = s.disk || {};
|
|
275
|
+
const lines = [`📈 Resource usage (${phaseLabel}):`, ` CPU load: ${formatNumber(cpu.load1)} ${formatNumber(cpu.load5)} ${formatNumber(cpu.load15)}${Number.isFinite(cpu.cpuCount) ? ` (${cpu.cpuCount} CPUs)` : ''}`, ` Memory: ${formatBytes(memory.availableBytes)} available / ${formatBytes(memory.totalBytes)} total (${formatBytes(memory.usedBytes)} used)`, ` Process RSS: ${formatBytes(memory.processRssBytes)}${Number.isFinite(memory.processHeapUsedBytes) ? `, heap ${formatBytes(memory.processHeapUsedBytes)}` : ''}`, ` Disk (${disk.path || '/'}): ${formatBytes(disk.availableBytes)} available / ${formatBytes(disk.totalBytes)} total${Number.isFinite(disk.usedPercent) ? ` (${disk.usedPercent.toFixed(1)}% used)` : ''}`];
|
|
276
|
+
if (disk.error) lines.push(` Disk probe error: ${disk.error}`);
|
|
277
|
+
lines.push(buildResourceMarker(snapshot));
|
|
278
|
+
return lines.join('\n');
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
export async function recordResourceSnapshot({ phase, log, diskPath = '/', label = null, capture = captureResourceSnapshot } = {}) {
|
|
282
|
+
if (typeof log !== 'function') return null;
|
|
283
|
+
try {
|
|
284
|
+
const snapshot = capture({ phase, diskPath });
|
|
285
|
+
await log(formatResourceSnapshotForLog(snapshot, label));
|
|
286
|
+
return snapshot;
|
|
287
|
+
} catch (error) {
|
|
288
|
+
await log(`⚠️ Resource usage measurement failed (${phase || 'snapshot'}): ${error?.message || error}`, { level: 'warning', verbose: true });
|
|
289
|
+
return null;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
export function summarizeResourceSnapshot(snapshot) {
|
|
294
|
+
if (!snapshot) return null;
|
|
295
|
+
const cpu = snapshot.cpu || {};
|
|
296
|
+
const memory = snapshot.memory || {};
|
|
297
|
+
const disk = snapshot.disk || {};
|
|
298
|
+
return {
|
|
299
|
+
phase: snapshot.phase || null,
|
|
300
|
+
timestamp: snapshot.timestamp || null,
|
|
301
|
+
cpu: {
|
|
302
|
+
load1: cpu.load1,
|
|
303
|
+
load5: cpu.load5,
|
|
304
|
+
load15: cpu.load15,
|
|
305
|
+
cpuCount: cpu.cpuCount,
|
|
306
|
+
},
|
|
307
|
+
memory: {
|
|
308
|
+
totalBytes: memory.totalBytes,
|
|
309
|
+
availableBytes: memory.availableBytes,
|
|
310
|
+
usedBytes: memory.usedBytes,
|
|
311
|
+
processRssBytes: memory.processRssBytes,
|
|
312
|
+
},
|
|
313
|
+
disk: {
|
|
314
|
+
path: disk.path,
|
|
315
|
+
totalBytes: disk.totalBytes,
|
|
316
|
+
availableBytes: disk.availableBytes,
|
|
317
|
+
usedBytes: disk.usedBytes,
|
|
318
|
+
usedPercent: disk.usedPercent,
|
|
319
|
+
error: disk.error || null,
|
|
320
|
+
},
|
|
321
|
+
};
|
|
322
|
+
}
|
|
@@ -32,6 +32,7 @@ const fs = (await use('fs')).promises;
|
|
|
32
32
|
const lib = await import('./lib.mjs');
|
|
33
33
|
const { log, formatAligned, extractToolErrorCore } = lib;
|
|
34
34
|
const { ensurePullRequestBaseBranch } = await import('./solve.pr-base-guard.lib.mjs');
|
|
35
|
+
const { RESOURCE_PHASE_RESTART_AFTER, RESOURCE_PHASE_RESTART_BEFORE, recordResourceSnapshot } = await import('./solve.resource-diagnostics.lib.mjs');
|
|
35
36
|
|
|
36
37
|
// Import Sentry integration
|
|
37
38
|
const sentryLib = await import('./sentry.lib.mjs');
|
|
@@ -177,6 +178,13 @@ export const getUncommittedChangesDetails = async tempDir => {
|
|
|
177
178
|
export const executeToolIteration = async params => {
|
|
178
179
|
const { issueUrl, owner, repo, issueNumber, prNumber, branchName, tempDir, workspaceTmpDir, mergeStateStatus, feedbackLines, argv } = params;
|
|
179
180
|
|
|
181
|
+
await recordResourceSnapshot({
|
|
182
|
+
phase: RESOURCE_PHASE_RESTART_BEFORE,
|
|
183
|
+
log,
|
|
184
|
+
diskPath: '/',
|
|
185
|
+
label: 'before AI restart iteration',
|
|
186
|
+
});
|
|
187
|
+
|
|
180
188
|
// Import necessary modules for tool execution
|
|
181
189
|
const memoryCheck = await import('./memory-check.mjs');
|
|
182
190
|
const { getResourceSnapshot } = memoryCheck;
|
|
@@ -462,6 +470,12 @@ export const executeToolIteration = async params => {
|
|
|
462
470
|
}
|
|
463
471
|
|
|
464
472
|
await ensurePullRequestBaseBranch({ owner, repo, prNumber, argv, log, formatAligned, $ });
|
|
473
|
+
await recordResourceSnapshot({
|
|
474
|
+
phase: RESOURCE_PHASE_RESTART_AFTER,
|
|
475
|
+
log,
|
|
476
|
+
diskPath: '/',
|
|
477
|
+
label: 'after AI restart iteration',
|
|
478
|
+
});
|
|
465
479
|
return toolResult;
|
|
466
480
|
};
|
|
467
481
|
|