@link-assistant/hive-mind 2.1.5 → 2.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +1 -1
- package/src/config.lib.mjs +3 -2
- package/src/isolation-runner.lib.mjs +13 -1
- package/src/limits.lib.mjs +2 -2
- package/src/locales/en.lino +1 -0
- package/src/locales/hi.lino +1 -0
- package/src/locales/ru.lino +1 -0
- package/src/locales/zh.lino +1 -0
- package/src/queue-config.lib.mjs +6 -3
- package/src/queue-start-reservation.lib.mjs +41 -0
- package/src/session-monitor.lib.mjs +70 -1
- package/src/telegram-bot.mjs +7 -6
- package/src/telegram-solve-queue.lib.mjs +24 -19
- package/src/work-session-formatting.lib.mjs +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 2.1.7
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 0ce779b: Enforce the solve queue minimum start interval for immediate Telegram `/solve`
|
|
8
|
+
launches so direct starts consume the same global pacing slot as queued starts.
|
|
9
|
+
|
|
10
|
+
## 2.1.6
|
|
11
|
+
|
|
12
|
+
### Patch Changes
|
|
13
|
+
|
|
14
|
+
- efcbcce: Handle Docker `oomKilled` status markers as terminal Telegram work-session failures, delay Docker backend-gone killed notifications long enough for start-command to publish a real terminal status or log footer, pace queued task startups at a minimum 10-minute interval, and cap system resource cache freshness at 1 minute.
|
|
15
|
+
|
|
3
16
|
## 2.1.5
|
|
4
17
|
|
|
5
18
|
### Patch Changes
|
package/package.json
CHANGED
package/src/config.lib.mjs
CHANGED
|
@@ -687,8 +687,9 @@ export const cacheTtl = {
|
|
|
687
687
|
// because users still hit "Resets in 3m xs" rate-limit responses. The API
|
|
688
688
|
// returns null values or 429 when called too frequently.
|
|
689
689
|
usageApi: parseIntWithDefault('HIVE_MIND_USAGE_API_CACHE_TTL_MS', 13 * 60 * 1000), // 13 minutes
|
|
690
|
-
// System metrics cache TTL (RAM, CPU, disk)
|
|
691
|
-
|
|
690
|
+
// System metrics cache TTL (RAM, CPU, disk). Issue #2015 caps this at
|
|
691
|
+
// 1 minute so queue decisions do not use stale host pressure data.
|
|
692
|
+
system: Math.min(parseIntWithDefault('HIVE_MIND_SYSTEM_CACHE_TTL_MS', 60 * 1000), 60 * 1000), // max 1 minute
|
|
692
693
|
};
|
|
693
694
|
|
|
694
695
|
// File and path configurations
|
|
@@ -321,9 +321,18 @@ export function generateSessionId() {
|
|
|
321
321
|
export function parseSessionStatusOutput(output) {
|
|
322
322
|
const raw = (output || '').trim();
|
|
323
323
|
if (!raw) {
|
|
324
|
-
return { exists: false, uuid: null, status: null, exitCode: null, startTime: null, endTime: null, currentTime: null, logPath: null, command: null, isolation: null, workingDirectory: null, sessionName: null, processIds: {}, raw: '' };
|
|
324
|
+
return { exists: false, uuid: null, status: null, exitCode: null, startTime: null, endTime: null, currentTime: null, logPath: null, command: null, isolation: null, workingDirectory: null, sessionName: null, processIds: {}, oomKilled: null, raw: '' };
|
|
325
325
|
}
|
|
326
326
|
|
|
327
|
+
const normalizeBooleanField = value => {
|
|
328
|
+
if (typeof value === 'boolean') return value;
|
|
329
|
+
if (value === null || value === undefined) return null;
|
|
330
|
+
const normalized = String(value).trim().toLowerCase();
|
|
331
|
+
if (['true', '1', 'yes'].includes(normalized)) return true;
|
|
332
|
+
if (['false', '0', 'no'].includes(normalized)) return false;
|
|
333
|
+
return null;
|
|
334
|
+
};
|
|
335
|
+
|
|
327
336
|
try {
|
|
328
337
|
const parsed = JSON.parse(raw);
|
|
329
338
|
const data = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
@@ -350,6 +359,7 @@ export function parseSessionStatusOutput(output) {
|
|
|
350
359
|
workingDirectory: data?.workingDirectory || null,
|
|
351
360
|
sessionName: data?.sessionName || data?.options?.sessionName || null,
|
|
352
361
|
processIds,
|
|
362
|
+
oomKilled: normalizeBooleanField(data?.oomKilled ?? data?.OOMKilled ?? data?.options?.oomKilled ?? data?.state?.oomKilled ?? data?.State?.OOMKilled),
|
|
353
363
|
raw,
|
|
354
364
|
};
|
|
355
365
|
} catch {
|
|
@@ -365,6 +375,7 @@ export function parseSessionStatusOutput(output) {
|
|
|
365
375
|
const match = raw.match(new RegExp(`^\\s*${name}\\s+"?([^"\\n]+)"?\\s*$`, 'mi'));
|
|
366
376
|
return match ? match[1].trim() : null;
|
|
367
377
|
};
|
|
378
|
+
const readBooleanField = name => normalizeBooleanField(readField(name));
|
|
368
379
|
|
|
369
380
|
const status = readField('status')?.toLowerCase() || null;
|
|
370
381
|
const exitCodeText = readField('exitCode');
|
|
@@ -396,6 +407,7 @@ export function parseSessionStatusOutput(output) {
|
|
|
396
407
|
workingDirectory: readField('workingDirectory'),
|
|
397
408
|
sessionName: readField('sessionName'),
|
|
398
409
|
processIds,
|
|
410
|
+
oomKilled: readBooleanField('oomKilled'),
|
|
399
411
|
raw,
|
|
400
412
|
};
|
|
401
413
|
}
|
package/src/limits.lib.mjs
CHANGED
|
@@ -1239,12 +1239,12 @@ export function formatCodexLimitsSection(codexLimits, codexError = null, options
|
|
|
1239
1239
|
* Configurable via environment variables:
|
|
1240
1240
|
* - HIVE_MIND_API_CACHE_TTL_MS: General API cache TTL (default: 180000 = 3 minutes)
|
|
1241
1241
|
* - HIVE_MIND_USAGE_API_CACHE_TTL_MS: Claude Usage API cache TTL (default: 780000 = 13 minutes)
|
|
1242
|
-
* - HIVE_MIND_SYSTEM_CACHE_TTL_MS: System metrics cache TTL (default:
|
|
1242
|
+
* - HIVE_MIND_SYSTEM_CACHE_TTL_MS: System metrics cache TTL (default: 60000 = 1 minute, capped at 1 minute)
|
|
1243
1243
|
*/
|
|
1244
1244
|
export const CACHE_TTL = {
|
|
1245
1245
|
API: cacheTtl.api, // 3 minutes for regular API calls (GitHub)
|
|
1246
1246
|
USAGE_API: cacheTtl.usageApi, // 13 minutes for Claude Usage API (rate limited)
|
|
1247
|
-
SYSTEM: cacheTtl.system, //
|
|
1247
|
+
SYSTEM: cacheTtl.system, // max 1 minute for system metrics (RAM, CPU, disk)
|
|
1248
1248
|
};
|
|
1249
1249
|
|
|
1250
1250
|
/**
|
package/src/locales/en.lino
CHANGED
package/src/locales/hi.lino
CHANGED
package/src/locales/ru.lino
CHANGED
|
@@ -623,6 +623,7 @@ ru
|
|
|
623
623
|
executing "⏳ Выполняется..."
|
|
624
624
|
finished "Рабочий сеанс успешно завершен"
|
|
625
625
|
failed "Рабочий сеанс завершился с ошибкой (код выхода: {{exitCode}})"
|
|
626
|
+
killed "Рабочий сеанс остановлен: {{reason}}{{exitSuffix}}"
|
|
626
627
|
duration
|
|
627
628
|
label "Длительность"
|
|
628
629
|
session
|
package/src/locales/zh.lino
CHANGED
package/src/queue-config.lib.mjs
CHANGED
|
@@ -192,6 +192,8 @@ const parseIntWithDefault = (envVar, defaultValue) => {
|
|
|
192
192
|
return isNaN(parsed) ? defaultValue : parsed;
|
|
193
193
|
};
|
|
194
194
|
|
|
195
|
+
const MINIMUM_START_INTERVAL_MS = 10 * 60 * 1000;
|
|
196
|
+
|
|
195
197
|
// Parse links notation config from environment variable (if provided)
|
|
196
198
|
const linoConfig = parseQueueConfig(getenv('HIVE_MIND_QUEUE_CONFIG', ''));
|
|
197
199
|
|
|
@@ -271,9 +273,10 @@ export const QUEUE_CONFIG = {
|
|
|
271
273
|
GITHUB_API_THRESHOLD: getThresholdConfig('githubApi', 'HIVE_MIND_GITHUB_API_THRESHOLD', 'HIVE_MIND_GITHUB_API_STRATEGY', 0.5, 'enqueue').value,
|
|
272
274
|
|
|
273
275
|
// Timing
|
|
274
|
-
// MIN_START_INTERVAL_MS:
|
|
275
|
-
//
|
|
276
|
-
|
|
276
|
+
// MIN_START_INTERVAL_MS: Minimum global spacing between task startups.
|
|
277
|
+
// Issue #2015: after resource thresholds clear, starting a backlog in a burst
|
|
278
|
+
// can kill the next batch before host metrics have time to settle.
|
|
279
|
+
MIN_START_INTERVAL_MS: Math.max(parseIntWithDefault('HIVE_MIND_MIN_START_INTERVAL_MS', MINIMUM_START_INTERVAL_MS), MINIMUM_START_INTERVAL_MS), // at least 10 minutes between starts
|
|
277
280
|
CONSUMER_POLL_INTERVAL_MS: parseIntWithDefault('HIVE_MIND_CONSUMER_POLL_INTERVAL_MS', 60000), // 1 minute between queue checks
|
|
278
281
|
MESSAGE_UPDATE_INTERVAL_MS: parseIntWithDefault('HIVE_MIND_MESSAGE_UPDATE_INTERVAL_MS', 60000), // 1 minute between status message updates
|
|
279
282
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
const reservationLocks = new WeakMap();
|
|
2
|
+
|
|
3
|
+
function getReservationLock(queue) {
|
|
4
|
+
return reservationLocks.get(queue) || Promise.resolve();
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Atomically check and reserve the global startup slot for direct execution.
|
|
9
|
+
*
|
|
10
|
+
* Queue consumer starts are naturally serialized. Telegram direct starts need
|
|
11
|
+
* the same serialization because overlapping handlers can both pass resource
|
|
12
|
+
* checks before the first detached process is visible to process scanning.
|
|
13
|
+
*
|
|
14
|
+
* @param {Object} queue - SolveQueue-like object.
|
|
15
|
+
* @param {Object} options - Same options as queue.canStartCommand().
|
|
16
|
+
* @returns {Promise<Object>} canStartCommand() result plus reservation fields.
|
|
17
|
+
*/
|
|
18
|
+
export async function reserveStartSlotForQueue(queue, options = {}) {
|
|
19
|
+
const previousReservation = getReservationLock(queue);
|
|
20
|
+
let releaseReservation;
|
|
21
|
+
reservationLocks.set(
|
|
22
|
+
queue,
|
|
23
|
+
new Promise(resolve => {
|
|
24
|
+
releaseReservation = resolve;
|
|
25
|
+
})
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
await previousReservation.catch(() => {});
|
|
29
|
+
|
|
30
|
+
try {
|
|
31
|
+
const check = await queue.canStartCommand(options);
|
|
32
|
+
if (!check.canStart) {
|
|
33
|
+
return { ...check, startReserved: false };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const reservedStartTime = queue.recordStart(options.tool || 'claude');
|
|
37
|
+
return { ...check, startReserved: true, reservedStartTime };
|
|
38
|
+
} finally {
|
|
39
|
+
releaseReservation();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -24,7 +24,7 @@ import fs from 'fs/promises';
|
|
|
24
24
|
import { promisify } from 'util';
|
|
25
25
|
import { formatSessionCompletionMessage, getSessionCompletionExitCode, classifySessionOutcome } from './work-session-formatting.lib.mjs';
|
|
26
26
|
import { notifySubscribers, getSubscriberCount } from './telegram-subscribers.lib.mjs';
|
|
27
|
-
import { classifyExitStatus } from './session-status.lib.mjs';
|
|
27
|
+
import { classifyExitStatus, normalizeExitCode } from './session-status.lib.mjs';
|
|
28
28
|
import path from 'node:path';
|
|
29
29
|
import { readLastSessionIdFromLog, findLatestSessionLogId, buildResumeCommand, formatResumeSection } from './session-resume.lib.mjs';
|
|
30
30
|
|
|
@@ -504,6 +504,8 @@ function isNonIsolationSessionActive(sessionName, sessionInfo, verbose = false)
|
|
|
504
504
|
* this, because a written "Exit Code:" footer is proof the command terminated.
|
|
505
505
|
*/
|
|
506
506
|
export const STALE_EXECUTING_MIN_AGE_MS = 90 * 1000;
|
|
507
|
+
export const DOCKER_BACKEND_GONE_GRACE_MS = 2 * 60 * 1000;
|
|
508
|
+
const DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD = 'dockerBackendGoneFirstSeenAt';
|
|
507
509
|
|
|
508
510
|
function sessionStartMs(sessionInfo) {
|
|
509
511
|
const start = sessionInfo?.startTime;
|
|
@@ -513,6 +515,23 @@ function sessionStartMs(sessionInfo) {
|
|
|
513
515
|
return Number.isFinite(ms) ? ms : null;
|
|
514
516
|
}
|
|
515
517
|
|
|
518
|
+
function isDockerIsolation(sessionInfo, statusResult) {
|
|
519
|
+
return sessionInfo?.isolationBackend === 'docker' || statusResult?.isolation === 'docker';
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function getDockerBackendGoneFirstSeenMs(sessionInfo) {
|
|
523
|
+
const raw = sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
524
|
+
if (!raw) return null;
|
|
525
|
+
const ms = new Date(raw).getTime();
|
|
526
|
+
return Number.isFinite(ms) ? ms : null;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
function clearDockerBackendGoneMarker(sessionName, sessionInfo) {
|
|
530
|
+
if (!sessionInfo?.[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD]) return;
|
|
531
|
+
delete sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD];
|
|
532
|
+
persistSessionSnapshot(sessionName, sessionInfo);
|
|
533
|
+
}
|
|
534
|
+
|
|
516
535
|
/**
|
|
517
536
|
* Cross-check whether a session that `$ --status` still reports as `executing`
|
|
518
537
|
* has actually terminated. Issue #1927: start-command's status can get stuck on
|
|
@@ -552,13 +571,60 @@ async function resolveStaleExecutingState(sessionName, sessionInfo, statusResult
|
|
|
552
571
|
// Only `false` (definitively gone) counts as killed; `null` (unknown backend)
|
|
553
572
|
// is treated as "no signal" so we don't kill on an indeterminate probe.
|
|
554
573
|
if (alive === false) {
|
|
574
|
+
if (isDockerIsolation(sessionInfo, statusResult)) {
|
|
575
|
+
const nowMs = Date.now();
|
|
576
|
+
const firstSeenMs = getDockerBackendGoneFirstSeenMs(sessionInfo);
|
|
577
|
+
if (firstSeenMs === null) {
|
|
578
|
+
sessionInfo[DOCKER_BACKEND_GONE_FIRST_SEEN_FIELD] = new Date(nowMs).toISOString();
|
|
579
|
+
persistSessionSnapshot(sessionName, sessionInfo);
|
|
580
|
+
if (verbose) {
|
|
581
|
+
console.log(`[VERBOSE] Session ${sessionName} docker backend is gone but no terminal status/footer is available yet; deferring killed classification for ${DOCKER_BACKEND_GONE_GRACE_MS}ms`);
|
|
582
|
+
}
|
|
583
|
+
return null;
|
|
584
|
+
}
|
|
585
|
+
if (nowMs - firstSeenMs < DOCKER_BACKEND_GONE_GRACE_MS) {
|
|
586
|
+
if (verbose) {
|
|
587
|
+
console.log(`[VERBOSE] Session ${sessionName} docker backend is still gone; waiting for terminal status/footer before reporting killed`);
|
|
588
|
+
}
|
|
589
|
+
return null;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
555
592
|
return { exitCode: null, status: 'killed', reason: 'backend-gone' };
|
|
556
593
|
}
|
|
594
|
+
if (alive === true) {
|
|
595
|
+
clearDockerBackendGoneMarker(sessionName, sessionInfo);
|
|
596
|
+
}
|
|
557
597
|
}
|
|
558
598
|
|
|
559
599
|
return null;
|
|
560
600
|
}
|
|
561
601
|
|
|
602
|
+
function resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog }) {
|
|
603
|
+
const logPath = statusResult?.logPath || sessionInfo?.logPath || null;
|
|
604
|
+
let footer = null;
|
|
605
|
+
if (logPath) {
|
|
606
|
+
const readFooter = exitFromLog || runner.readSessionExitFromLog;
|
|
607
|
+
footer = readFooter ? readFooter(logPath, { verbose }) : null;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
const statusExitCode = normalizeExitCode(statusResult?.exitCode);
|
|
611
|
+
const footerExitCode = footer?.finished ? normalizeExitCode(footer.exitCode) : null;
|
|
612
|
+
let exitCode = 137;
|
|
613
|
+
if (statusExitCode !== null && statusExitCode > 0) {
|
|
614
|
+
exitCode = statusExitCode;
|
|
615
|
+
} else if (footerExitCode !== null && footerExitCode > 0) {
|
|
616
|
+
exitCode = footerExitCode;
|
|
617
|
+
}
|
|
618
|
+
const endTime = statusResult?.endTime || footer?.endTime || statusResult?.currentTime || null;
|
|
619
|
+
const corrected = { ...statusResult, status: 'oom-killed', exitCode, endTime };
|
|
620
|
+
|
|
621
|
+
if (verbose) {
|
|
622
|
+
console.log(`[VERBOSE] Session ${sessionName} status includes oomKilled=true; treating it as terminal oom-killed (exit ${exitCode})`);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
return { running: false, exitCode, status: 'oom-killed', statusResult: corrected, stale: true };
|
|
626
|
+
}
|
|
627
|
+
|
|
562
628
|
async function getIsolationSessionState(sessionName, sessionInfo, options = {}) {
|
|
563
629
|
const { verbose = false, statusProvider = null, exitFromLog = null, backendAlive = null, sessionRunning = null } = options;
|
|
564
630
|
const sessionId = sessionInfo.sessionId || sessionName;
|
|
@@ -568,6 +634,9 @@ async function getIsolationSessionState(sessionName, sessionInfo, options = {})
|
|
|
568
634
|
const statusResult = statusProvider ? await statusProvider(sessionId, sessionInfo) : await runner.querySessionStatus(sessionId, verbose);
|
|
569
635
|
|
|
570
636
|
if (statusResult?.exists && statusResult.status) {
|
|
637
|
+
if (statusResult.oomKilled === true) {
|
|
638
|
+
return resolveOomKilledState(sessionName, sessionInfo, statusResult, { verbose, runner, exitFromLog });
|
|
639
|
+
}
|
|
571
640
|
if (runner.isExecutingSessionStatus(statusResult.status)) {
|
|
572
641
|
// Issue #1927: an `executing` status is not trusted blindly — verify the
|
|
573
642
|
// process is really alive. start-command can keep reporting `executing`
|
package/src/telegram-bot.mjs
CHANGED
|
@@ -916,8 +916,9 @@ async function handleSolveCommand(ctx) {
|
|
|
916
916
|
await safeReply(ctx, t('telegram.url_session_running', { url: escapeMarkdown(normalizedUrl), session: activeSession.sessionName }, { locale: solveLocale }), { reply_to_message_id: ctx.message.message_id });
|
|
917
917
|
return;
|
|
918
918
|
}
|
|
919
|
-
const check = await solveQueue.canStartCommand({ tool: solveTool, locale: solveLocale }); // Skip Claude limits for agent (#1159)
|
|
920
919
|
const queueStats = solveQueue.getStats();
|
|
920
|
+
const hasPendingQueueItems = queueStats.queued > 0;
|
|
921
|
+
const check = hasPendingQueueItems ? await solveQueue.canStartCommand({ tool: solveTool, locale: solveLocale }) : await solveQueue.reserveStartSlot({ tool: solveTool, locale: solveLocale }); // Skip Claude limits for agent (#1159)
|
|
921
922
|
// Handle rejection: threshold strategy is 'reject' — fail immediately (issue #1267)
|
|
922
923
|
if (check.rejected) {
|
|
923
924
|
await safeReply(ctx, t('telegram.solve_rejected', { infoBlock, reason: escapeMarkdown(check.rejectReason || 'Unknown') }, { locale: solveLocale }), { reply_to_message_id: ctx.message.message_id });
|
|
@@ -935,18 +936,18 @@ async function handleSolveCommand(ctx) {
|
|
|
935
936
|
let solveLimitsAtStart = null;
|
|
936
937
|
if (solveShowLimits) ({ infoBlock, limitsAtStart: solveLimitsAtStart } = await captureStartSnapshotAndAppend({ infoBlock, tool: solveTool, verbose: VERBOSE, limitsLib, commandLabel: '/solve', locale: solveLocale }));
|
|
937
938
|
|
|
938
|
-
if (check.canStart &&
|
|
939
|
+
if (check.canStart && check.startReserved) {
|
|
939
940
|
const startingMessage = await safeReply(ctx, formatStartingWorkSessionMessage({ infoBlock, locale: solveLocale }), { reply_to_message_id: ctx.message.message_id });
|
|
940
941
|
await executeAndUpdateMessage(ctx, startingMessage, 'solve', argsWithLocale, infoBlock, effectiveSolveIsolation, solveTool, solveUrlContext, { showLimits: solveShowLimits, limitsAtStart: solveLimitsAtStart, locale: solveLocale });
|
|
941
942
|
} else {
|
|
942
|
-
const queueItem = solveQueue.enqueue({ url: normalizedUrl, args: argsWithLocale, ctx, requester, infoBlock, tool: solveTool, perCommandIsolation: effectiveSolveIsolation, urlContext: solveUrlContext, showLimits: solveShowLimits, limitsAtStart: solveLimitsAtStart, locale: solveLocale });
|
|
943
|
-
const queueMessage = buildSolveQueuedMessage({ locale: solveLocale, tool: solveTool, position: toolQueuedCount + 1, infoBlock, reason: check.reason ? escapeMarkdown(check.reason) : '' }); // tool-specific position (#1551)
|
|
944
|
-
const queuedMessage = await safeReply(ctx, queueMessage, { reply_to_message_id: ctx.message.message_id });
|
|
945
|
-
queueItem.messageInfo = { chatId: queuedMessage.chat.id, messageId: queuedMessage.message_id };
|
|
946
943
|
if (!solveQueue.executeCallback) {
|
|
947
944
|
const _t = (s, i) => trackSession(s, i, VERBOSE);
|
|
948
945
|
solveQueue.executeCallback = createIsolationAwareQueueCallback(ISOLATION_BACKEND, isolationRunner, _t, createQueueExecuteCallback(executeStartScreen, _t), VERBOSE);
|
|
949
946
|
}
|
|
947
|
+
const queueItem = solveQueue.enqueue({ url: normalizedUrl, args: argsWithLocale, ctx, requester, infoBlock, tool: solveTool, perCommandIsolation: effectiveSolveIsolation, urlContext: solveUrlContext, showLimits: solveShowLimits, limitsAtStart: solveLimitsAtStart, locale: solveLocale });
|
|
948
|
+
const queueMessage = buildSolveQueuedMessage({ locale: solveLocale, tool: solveTool, position: toolQueuedCount + 1, infoBlock, reason: check.reason ? escapeMarkdown(check.reason) : '' }); // tool-specific position (#1551)
|
|
949
|
+
const queuedMessage = await safeReply(ctx, queueMessage, { reply_to_message_id: ctx.message.message_id });
|
|
950
|
+
queueItem.messageInfo = { chatId: queuedMessage.chat.id, messageId: queuedMessage.message_id };
|
|
950
951
|
}
|
|
951
952
|
}
|
|
952
953
|
|
|
@@ -20,6 +20,7 @@ export { formatDuration, getRunningAgentProcesses, getRunningClaudeProcesses, ge
|
|
|
20
20
|
import { collectExecutingItems, formatDuration, formatQueueToolSection, formatWaitingReason, getRunningAgentProcesses, getRunningClaudeProcesses, getRunningCodexProcesses, getRunningGeminiProcesses, getRunningProcesses, getRunningQwenProcesses, getRunningSessionItems, groupQueueItemsByTool } from './telegram-solve-queue.helpers.lib.mjs';
|
|
21
21
|
export { QUEUE_CONFIG, THRESHOLD_STRATEGIES } from './queue-config.lib.mjs';
|
|
22
22
|
import { QUEUE_CONFIG } from './queue-config.lib.mjs';
|
|
23
|
+
import { reserveStartSlotForQueue } from './queue-start-reservation.lib.mjs';
|
|
23
24
|
import { formatExecutingWorkSessionMessage, formatStartingWorkSessionMessage } from './work-session-formatting.lib.mjs';
|
|
24
25
|
import { t } from './i18n.lib.mjs';
|
|
25
26
|
import { lt } from './limits-i18n.lib.mjs';
|
|
@@ -373,10 +374,22 @@ export class SolveQueue {
|
|
|
373
374
|
return count;
|
|
374
375
|
}
|
|
375
376
|
|
|
377
|
+
recordStart(tool = 'claude', startTime = Date.now()) {
|
|
378
|
+
this.lastStartTimeByTool[tool] = startTime;
|
|
379
|
+
this.lastStartTime = startTime;
|
|
380
|
+
return startTime;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
reserveStartSlot(options = {}) {
|
|
384
|
+
return reserveStartSlotForQueue(this, options);
|
|
385
|
+
}
|
|
386
|
+
|
|
376
387
|
/**
|
|
377
|
-
* Find startable
|
|
378
|
-
*
|
|
379
|
-
*
|
|
388
|
+
* Find the next startable item across all tool queues.
|
|
389
|
+
* With separate queues, each tool is checked independently so tool-specific
|
|
390
|
+
* limits do not block unrelated tools. Issue #2015 adds a global startup
|
|
391
|
+
* interval, so even when multiple tools are startable this returns only the
|
|
392
|
+
* oldest startable item to prevent burst launches.
|
|
380
393
|
*
|
|
381
394
|
* Also immediately rejects all queued items when a 'reject' strategy threshold
|
|
382
395
|
* is exceeded, instead of leaving them waiting indefinitely.
|
|
@@ -417,7 +430,8 @@ export class SolveQueue {
|
|
|
417
430
|
}
|
|
418
431
|
}
|
|
419
432
|
|
|
420
|
-
|
|
433
|
+
startableItems.sort((a, b) => a.item.createdAt - b.item.createdAt);
|
|
434
|
+
return startableItems.slice(0, 1);
|
|
421
435
|
}
|
|
422
436
|
|
|
423
437
|
/**
|
|
@@ -571,10 +585,10 @@ export class SolveQueue {
|
|
|
571
585
|
let rejected = false;
|
|
572
586
|
let rejectReason = null;
|
|
573
587
|
|
|
574
|
-
// Check minimum interval since last start
|
|
575
|
-
//
|
|
576
|
-
//
|
|
577
|
-
const lastStartTime = this.
|
|
588
|
+
// Check minimum interval since the last task start globally.
|
|
589
|
+
// Issue #2015: do not let another tool queue bypass startup pacing; host
|
|
590
|
+
// CPU/RAM/disk metrics need time to settle before any next task starts.
|
|
591
|
+
const lastStartTime = this.lastStartTime || null;
|
|
578
592
|
if (lastStartTime) {
|
|
579
593
|
const timeSinceLastStart = Date.now() - lastStartTime;
|
|
580
594
|
if (timeSinceLastStart < QUEUE_CONFIG.MIN_START_INTERVAL_MS) {
|
|
@@ -765,7 +779,6 @@ export class SolveQueue {
|
|
|
765
779
|
}
|
|
766
780
|
|
|
767
781
|
// Check CPU using 5-minute load average (more stable than 1-minute)
|
|
768
|
-
// Cache TTL is 2 minutes, which is appropriate for this metric
|
|
769
782
|
const cpuResult = await getCachedCpuInfo(this.verbose);
|
|
770
783
|
if (cpuResult.success) {
|
|
771
784
|
// Use loadAvg5 (5-minute average) instead of usagePercentage (1-minute based)
|
|
@@ -1072,7 +1085,7 @@ export class SolveQueue {
|
|
|
1072
1085
|
* - Each tool queue is checked independently
|
|
1073
1086
|
* - Claude limits only affect Claude queue
|
|
1074
1087
|
* - Agent queue can proceed even when Claude is blocked (and vice versa)
|
|
1075
|
-
* -
|
|
1088
|
+
* - The oldest startable item starts each cycle to preserve global pacing
|
|
1076
1089
|
*
|
|
1077
1090
|
* @see https://github.com/link-assistant/hive-mind/issues/1159
|
|
1078
1091
|
*/
|
|
@@ -1086,9 +1099,6 @@ export class SolveQueue {
|
|
|
1086
1099
|
continue;
|
|
1087
1100
|
}
|
|
1088
1101
|
|
|
1089
|
-
// Find startable items from each tool queue
|
|
1090
|
-
// Each tool is checked independently so they don't block each other
|
|
1091
|
-
// See: https://github.com/link-assistant/hive-mind/issues/1159
|
|
1092
1102
|
const startableItems = await this.findStartableItems();
|
|
1093
1103
|
|
|
1094
1104
|
if (startableItems.length === 0) {
|
|
@@ -1099,8 +1109,6 @@ export class SolveQueue {
|
|
|
1099
1109
|
continue;
|
|
1100
1110
|
}
|
|
1101
1111
|
|
|
1102
|
-
// Start items from each tool that can proceed
|
|
1103
|
-
// This allows parallel starts from different tool queues
|
|
1104
1112
|
for (const startable of startableItems) {
|
|
1105
1113
|
const { tool } = startable;
|
|
1106
1114
|
const toolQueue = this.getToolQueue(tool);
|
|
@@ -1113,12 +1121,9 @@ export class SolveQueue {
|
|
|
1113
1121
|
item.setStarting();
|
|
1114
1122
|
this.processing.set(item.id, item);
|
|
1115
1123
|
|
|
1116
|
-
|
|
1117
|
-
this.lastStartTimeByTool[tool] = Date.now();
|
|
1118
|
-
this.lastStartTime = Date.now(); // Legacy compatibility
|
|
1124
|
+
this.recordStart(tool);
|
|
1119
1125
|
this.stats.totalStarted++;
|
|
1120
1126
|
|
|
1121
|
-
// Update message to show Starting status
|
|
1122
1127
|
await this.updateItemMessage(item, formatStartingWorkSessionMessage({ infoBlock: item.infoBlock, locale: item.locale }));
|
|
1123
1128
|
|
|
1124
1129
|
this.log(`Starting: ${item.toString()} from ${tool} queue`);
|
|
@@ -145,7 +145,7 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
|
|
|
145
145
|
const showCode = finalExitCode !== null && !(!signal && finalExitCode === 1);
|
|
146
146
|
const exitSuffix = showCode ? ` (exit code: ${finalExitCode})` : '';
|
|
147
147
|
const reason = signal ? signal.reason : 'killed';
|
|
148
|
-
statusText = text(messageLocale, 'telegram.work_session_killed', `Work session ${reason}${exitSuffix}`, { reason, exitCode: finalExitCode ?? '', signal: signal?.signal ?? '' });
|
|
148
|
+
statusText = text(messageLocale, 'telegram.work_session_killed', `Work session ${reason}${exitSuffix}`, { reason, exitCode: finalExitCode ?? '', signal: signal?.signal ?? '', exitSuffix });
|
|
149
149
|
} else if (failed) {
|
|
150
150
|
statusText = text(messageLocale, 'telegram.work_session_failed', `Work session failed (exit code: ${finalExitCode})`, { exitCode: finalExitCode });
|
|
151
151
|
} else {
|