dsh-autorestart 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -30
- package/client.js +805 -133
- package/cordis.patch.yml +18 -8
- package/docs/README.zh.md +87 -19
- package/index.js +381 -59
- package/lib/activity.js +124 -0
- package/lib/lifecycle.js +17 -4
- package/lib/queued.js +146 -0
- package/locale/en.json +10 -6
- package/locale/zh.json +10 -6
- package/package.json +1 -1
package/index.js
CHANGED
|
@@ -76,6 +76,12 @@ import {
|
|
|
76
76
|
scheduleDetachedRestart,
|
|
77
77
|
} from './lib/restart.js'
|
|
78
78
|
import { compareSemver } from './lib/semver.js'
|
|
79
|
+
import { isWorkSessionEvent } from './lib/activity.js'
|
|
80
|
+
import {
|
|
81
|
+
createQueuedTracker,
|
|
82
|
+
DEFAULT_QUEUED_GRACE_MS,
|
|
83
|
+
normalizeQueuedGraceMs,
|
|
84
|
+
} from './lib/queued.js'
|
|
79
85
|
import {
|
|
80
86
|
createSleepController,
|
|
81
87
|
COUNTDOWN_CHOICES,
|
|
@@ -87,34 +93,52 @@ import {
|
|
|
87
93
|
} from './lib/sleep.js'
|
|
88
94
|
|
|
89
95
|
/**
|
|
90
|
-
* The
|
|
91
|
-
* finished before it is really restarted
|
|
96
|
+
* The RESTART reserve: how long the host waits after the LAST conversation
|
|
97
|
+
* finished before it is really restarted.
|
|
92
98
|
*
|
|
93
99
|
* This plugin can only see the DSH agents. Other plugins (a memory plugin
|
|
94
100
|
* that writes its notes a few seconds after the conversation ended, a summary,
|
|
95
101
|
* an MCP callback) queue their work with a timer of their own — work that is
|
|
96
|
-
* invisible here. A short, configurable
|
|
102
|
+
* invisible here. A short, configurable reserve window is left for it.
|
|
103
|
+
*
|
|
104
|
+
* The reserve belongs to the RESTART only: a restart is a daily operation, so
|
|
105
|
+
* it should happen as soon as the conversations are done. Default: none.
|
|
106
|
+
*
|
|
107
|
+
* The graceful shutdown has NO reserve of its own: it has a VISIBLE countdown
|
|
108
|
+
* (`sleepCountdownSeconds`), and that countdown IS the wait before the
|
|
109
|
+
* machine goes off. A second, invisible window next to it would only say the
|
|
110
|
+
* same thing twice — so there is none.
|
|
97
111
|
*
|
|
98
112
|
* The values are the ones the settings page offers (in milliseconds).
|
|
99
113
|
*/
|
|
100
|
-
const GRACE_CHOICES_MS = Object.freeze([0,
|
|
114
|
+
const GRACE_CHOICES_MS = Object.freeze([0, 5000, 10000, 20000, 30000, 60000])
|
|
101
115
|
|
|
102
|
-
/** The
|
|
103
|
-
const
|
|
116
|
+
/** The RESTART reserve that is used when nothing was configured. */
|
|
117
|
+
const DEFAULT_RESTART_GRACE_MS = 0
|
|
104
118
|
|
|
105
119
|
/**
|
|
106
|
-
*
|
|
120
|
+
* How often the "the restart is waiting for a long time" line may be logged.
|
|
121
|
+
*
|
|
122
|
+
* The restart gate polls every few seconds, so an unthrottled line would
|
|
123
|
+
* flood the log with the same sentence. Once a minute is enough to say that
|
|
124
|
+
* the wait is still going on.
|
|
125
|
+
*/
|
|
126
|
+
const LONG_WAIT_LOG_INTERVAL_MS = 60000
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Snap a reserve value onto the nearest offered choice.
|
|
107
130
|
*
|
|
108
131
|
* A profile that carries a value that is not offered (a hand-edited row, an
|
|
109
132
|
* older build) must not break the boot: the value is snapped onto the
|
|
110
133
|
* nearest choice, exactly like the countdown length is.
|
|
111
134
|
*
|
|
112
135
|
* @param {unknown} value
|
|
136
|
+
* @param {number} [fallback] the value used when the input is not a number
|
|
113
137
|
* @returns {number} one of {@link GRACE_CHOICES_MS}
|
|
114
138
|
*/
|
|
115
|
-
function snapGraceChoice(value) {
|
|
139
|
+
function snapGraceChoice(value, fallback = DEFAULT_RESTART_GRACE_MS) {
|
|
116
140
|
const raw = Number(value)
|
|
117
|
-
if (!Number.isFinite(raw) || raw < 0) return
|
|
141
|
+
if (!Number.isFinite(raw) || raw < 0) return fallback
|
|
118
142
|
let best = GRACE_CHOICES_MS[0]
|
|
119
143
|
let bestDistance = Math.abs(raw - best)
|
|
120
144
|
for (const choice of GRACE_CHOICES_MS) {
|
|
@@ -134,7 +158,7 @@ function snapGraceChoice(value) {
|
|
|
134
158
|
* `pluginVersion()`), but a bundled installation may not ship that file — so
|
|
135
159
|
* the constant below is the last resort. A test keeps the two in sync.
|
|
136
160
|
*/
|
|
137
|
-
const PLUGIN_VERSION_FALLBACK = '0.6.
|
|
161
|
+
const PLUGIN_VERSION_FALLBACK = '0.6.2'
|
|
138
162
|
|
|
139
163
|
/**
|
|
140
164
|
* The version of the graceful-shutdown FEATURE, not the version of the package.
|
|
@@ -380,8 +404,19 @@ function buildConfigSchema() {
|
|
|
380
404
|
// Other plugins queue work with timers of their own (a memory plugin
|
|
381
405
|
// writing its notes, a summary, an MCP callback) and that work is
|
|
382
406
|
// invisible here — so a short, configurable grace window is left for it.
|
|
383
|
-
|
|
384
|
-
|
|
407
|
+
// The RESTART reserve: the extra wait after the last conversation
|
|
408
|
+
// finished, before the host is really restarted. A restart is a daily
|
|
409
|
+
// operation, so the default is none (0 / 5 / 10 / 20 / 30 / 60 seconds).
|
|
410
|
+
restartGraceMs: schemastery.number().min(0).default(DEFAULT_RESTART_GRACE_MS)
|
|
411
|
+
.description('Milliseconds to wait after the last conversation finished before the host is really RESTARTED: the reserve for work other plugins queue with a timer (0 / 5 / 10 / 20 / 30 / 60 seconds, default 0 = no extra wait).'),
|
|
412
|
+
// The graceful shutdown has no reserve of its own: the visible countdown
|
|
413
|
+
// (sleepCountdownSeconds) IS the wait before the machine goes off.
|
|
414
|
+
// The upper bound of a QUEUED inbox message: a message that nobody
|
|
415
|
+
// claims within this window is stuck, and it stops blocking the restart.
|
|
416
|
+
// Without it a single unclaimed message kept the restart pending for
|
|
417
|
+
// ever (the "the restart never happens" defect).
|
|
418
|
+
queuedGraceMs: schemastery.number().min(0).default(DEFAULT_QUEUED_GRACE_MS)
|
|
419
|
+
.description('Milliseconds an unclaimed (queued) inbox message may block the restart: a message that is still unclaimed after this window is treated as stuck and no longer blocks the restart (0 = no upper bound, the old behaviour; default 60000).'),
|
|
385
420
|
autoReloadPage: schemastery.boolean().default(true)
|
|
386
421
|
.description('Reload the web page automatically when the host boot token changes.'),
|
|
387
422
|
blockManualRestart: schemastery.boolean().default(true)
|
|
@@ -448,7 +483,11 @@ const DEFAULTS = {
|
|
|
448
483
|
restartCommand: '',
|
|
449
484
|
relaunchDelaySeconds: 2,
|
|
450
485
|
settleMs: 2000,
|
|
451
|
-
restartGraceMs:
|
|
486
|
+
restartGraceMs: DEFAULT_RESTART_GRACE_MS,
|
|
487
|
+
// The upper bound of a queued (unclaimed) inbox message: after this time it
|
|
488
|
+
// is treated as stuck and stops blocking the restart. 0 switches the
|
|
489
|
+
// protection off (the old behaviour: a queued message blocks for ever).
|
|
490
|
+
queuedGraceMs: DEFAULT_QUEUED_GRACE_MS,
|
|
452
491
|
autoReloadPage: true,
|
|
453
492
|
blockManualRestart: true,
|
|
454
493
|
healthPath: '/__dsh_autorestart/health',
|
|
@@ -495,9 +534,12 @@ function normalizeConfig(raw) {
|
|
|
495
534
|
// profile that still carries an old value (15, 45, …) is snapped onto the
|
|
496
535
|
// nearest one instead of breaking, and 0 means "no countdown at all".
|
|
497
536
|
config.sleepCountdownSeconds = snapCountdownChoice(config.sleepCountdownSeconds)
|
|
498
|
-
// The
|
|
537
|
+
// The restart reserve is one of the offered choices as well: a hand-edited
|
|
499
538
|
// profile value is snapped onto the nearest one instead of breaking.
|
|
500
|
-
config.restartGraceMs = snapGraceChoice(config.restartGraceMs)
|
|
539
|
+
config.restartGraceMs = snapGraceChoice(config.restartGraceMs, DEFAULT_RESTART_GRACE_MS)
|
|
540
|
+
// The upper bound of a queued message is a plain number: a wrong or negative
|
|
541
|
+
// value falls back to the default, and 0 switches the protection off.
|
|
542
|
+
config.queuedGraceMs = normalizeQueuedGraceMs(config.queuedGraceMs)
|
|
501
543
|
if (typeof config.sleepCommand !== 'string') config.sleepCommand = ''
|
|
502
544
|
if (typeof config.showSleepButton !== 'boolean') config.showSleepButton = true
|
|
503
545
|
return config
|
|
@@ -1233,7 +1275,43 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1233
1275
|
/** When the last conversation activity was observed; the settle window starts here. */
|
|
1234
1276
|
let lastActivityAt = Date.now()
|
|
1235
1277
|
|
|
1236
|
-
/**
|
|
1278
|
+
/**
|
|
1279
|
+
* When a busy conversation was seen for the last time.
|
|
1280
|
+
*
|
|
1281
|
+
* The quiet window measures the time since the last REAL work: a running
|
|
1282
|
+
* turn or a queued message. The host polls the agent list every few
|
|
1283
|
+
* seconds, so this marker follows the work exactly — and it is NOT reset by
|
|
1284
|
+
* the routine session events the interface produces while it synchronises
|
|
1285
|
+
* (those used to reset the window every 2-3 seconds, so a window longer
|
|
1286
|
+
* than that interval could never fill and the restart never fired).
|
|
1287
|
+
*/
|
|
1288
|
+
let lastBusyAt = 0
|
|
1289
|
+
|
|
1290
|
+
/**
|
|
1291
|
+
* When the "the restart is waiting for a long time" line was logged last.
|
|
1292
|
+
*
|
|
1293
|
+
* The gate polls every few seconds, so the line is rate limited: at most one
|
|
1294
|
+
* per minute for the SAME restart request. A new request (`requestedAt`
|
|
1295
|
+
* changed) logs right away.
|
|
1296
|
+
*/
|
|
1297
|
+
let longWaitLog = { at: 0, requestedAt: 0 }
|
|
1298
|
+
|
|
1299
|
+
/**
|
|
1300
|
+
* When the diagnostic line of the restart gate was logged last.
|
|
1301
|
+
*
|
|
1302
|
+
* The gate is evaluated every few seconds, so the line is rate limited to
|
|
1303
|
+
* once a minute: it says what the gate decided and why, which is the first
|
|
1304
|
+
* thing to look at when "the restart never happens" is reported.
|
|
1305
|
+
*/
|
|
1306
|
+
let lastDecisionLogAt = 0
|
|
1307
|
+
|
|
1308
|
+
/**
|
|
1309
|
+
* Record REAL conversation activity (an agent turn that started or ended, a
|
|
1310
|
+
* message that was queued, a session event that says work happened).
|
|
1311
|
+
*
|
|
1312
|
+
* The routine session events of the interface do NOT go through here: they
|
|
1313
|
+
* would keep the quiet window from ever filling.
|
|
1314
|
+
*/
|
|
1237
1315
|
const noteActivity = () => {
|
|
1238
1316
|
lastActivityAt = Date.now()
|
|
1239
1317
|
}
|
|
@@ -1253,6 +1331,53 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1253
1331
|
return count
|
|
1254
1332
|
}
|
|
1255
1333
|
|
|
1334
|
+
/**
|
|
1335
|
+
* A short human-readable duration for the log ("45s", "2m 5s", "1h 3m").
|
|
1336
|
+
*
|
|
1337
|
+
* @param {number} ms
|
|
1338
|
+
* @returns {string}
|
|
1339
|
+
*/
|
|
1340
|
+
const describeWait = (ms) => {
|
|
1341
|
+
const total = Math.max(0, Math.round(Number(ms) || 0))
|
|
1342
|
+
const seconds = Math.floor(total / 1000)
|
|
1343
|
+
if (seconds < 60) return `${seconds}s`
|
|
1344
|
+
const minutes = Math.floor(seconds / 60)
|
|
1345
|
+
if (minutes < 60) {
|
|
1346
|
+
const rest = seconds % 60
|
|
1347
|
+
return rest === 0 ? `${minutes}m` : `${minutes}m ${rest}s`
|
|
1348
|
+
}
|
|
1349
|
+
const hours = Math.floor(minutes / 60)
|
|
1350
|
+
const restMinutes = minutes % 60
|
|
1351
|
+
return restMinutes === 0 ? `${hours}h` : `${hours}h ${restMinutes}m`
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
/**
|
|
1355
|
+
* The readable names of the sessions that hold the restart.
|
|
1356
|
+
*
|
|
1357
|
+
* A session without a title falls back to its id: the log has to say WHO
|
|
1358
|
+
* holds the restart, and an empty name would say nothing.
|
|
1359
|
+
*
|
|
1360
|
+
* @param {any[]} blockedBy
|
|
1361
|
+
* @returns {string}
|
|
1362
|
+
*/
|
|
1363
|
+
const describeBlockers = (blockedBy) => {
|
|
1364
|
+
const list = Array.isArray(blockedBy) ? blockedBy : []
|
|
1365
|
+
const names = []
|
|
1366
|
+
for (const entry of list) {
|
|
1367
|
+
if (!entry || typeof entry !== 'object') continue
|
|
1368
|
+
const name = typeof entry.name === 'string' && entry.name.trim() !== ''
|
|
1369
|
+
? entry.name.trim()
|
|
1370
|
+
: (typeof entry.label === 'string' && entry.label.trim() !== ''
|
|
1371
|
+
? entry.label.trim()
|
|
1372
|
+
: String(entry.sessionId || ''))
|
|
1373
|
+
if (name === '') continue
|
|
1374
|
+
names.push(name)
|
|
1375
|
+
}
|
|
1376
|
+
if (names.length === 0) return 'another conversation'
|
|
1377
|
+
if (names.length === 1) return names[0]
|
|
1378
|
+
return names.join(', ')
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1256
1381
|
/**
|
|
1257
1382
|
* Whether this agent is a subagent (a delegated child), and which session
|
|
1258
1383
|
* created it.
|
|
@@ -1329,10 +1454,28 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1329
1454
|
}
|
|
1330
1455
|
}
|
|
1331
1456
|
|
|
1457
|
+
/**
|
|
1458
|
+
* The tracker of the QUEUED messages.
|
|
1459
|
+
*
|
|
1460
|
+
* It remembers for how long a session has had an unclaimed inbox message, so
|
|
1461
|
+
* a message that nobody claims stops blocking the restart instead of
|
|
1462
|
+
* blocking it for ever. A session that is RUNNING is never judged here: a
|
|
1463
|
+
* live turn always blocks, under every grace setting.
|
|
1464
|
+
*/
|
|
1465
|
+
const queuedTracker = createQueuedTracker()
|
|
1466
|
+
|
|
1467
|
+
/** The sessions the tracker saw in the last round (for the cleanup). */
|
|
1468
|
+
const queuedSessions = new Set()
|
|
1469
|
+
|
|
1332
1470
|
/**
|
|
1333
1471
|
* Every conversation that is still working, subagents and teammates included.
|
|
1334
1472
|
* 'running' is a live turn; 'queued' is inbox work that has not started yet.
|
|
1335
1473
|
*
|
|
1474
|
+
* A queued message only blocks the restart until the grace window
|
|
1475
|
+
* (`queuedGraceMs`) is over: after that it is stuck — nobody will ever claim
|
|
1476
|
+
* it — so it stops blocking the restart and is named in the log. A running
|
|
1477
|
+
* conversation ALWAYS blocks.
|
|
1478
|
+
*
|
|
1336
1479
|
* Every entry carries the kind of the blocker and a readable name, so the
|
|
1337
1480
|
* interface can tell a session from a subagent and never has to show a raw
|
|
1338
1481
|
* session id.
|
|
@@ -1345,15 +1488,52 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1345
1488
|
} catch {
|
|
1346
1489
|
agents = []
|
|
1347
1490
|
}
|
|
1491
|
+
const seen = new Set()
|
|
1348
1492
|
for (const agent of agents) {
|
|
1349
1493
|
try {
|
|
1350
1494
|
if (!agent || !agent.id) continue
|
|
1351
1495
|
const sessionId = String(agent.id)
|
|
1496
|
+
seen.add(sessionId)
|
|
1352
1497
|
const described = describeBlocker(sessionId, agent)
|
|
1353
|
-
if (agent.status === 'running')
|
|
1354
|
-
|
|
1498
|
+
if (agent.status === 'running') {
|
|
1499
|
+
// A live turn ALWAYS blocks the restart — under every grace
|
|
1500
|
+
// setting. The queued timer of that session is dropped: the work is
|
|
1501
|
+
// being done, so it is not a stuck queue.
|
|
1502
|
+
queuedTracker.forget(sessionId)
|
|
1503
|
+
out.push({ sessionId, reason: 'running', ...described })
|
|
1504
|
+
} else if (pendingInboxCount(agent) > 0) {
|
|
1505
|
+
const observed = queuedTracker.observe(sessionId, true, config.queuedGraceMs)
|
|
1506
|
+
if (observed.becameStale) {
|
|
1507
|
+
// The message has been waiting longer than the grace window and
|
|
1508
|
+
// nobody claimed it: it is stuck. The session is named here, so
|
|
1509
|
+
// the user can see WHICH conversation holds the queue.
|
|
1510
|
+
log.warn(
|
|
1511
|
+
`the queued message of session ${sessionId} has been waiting ${describeWait(observed.ageMs)} and nobody claimed it; it no longer blocks the restart`,
|
|
1512
|
+
{
|
|
1513
|
+
sessionId,
|
|
1514
|
+
ageMs: observed.ageMs,
|
|
1515
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
1516
|
+
},
|
|
1517
|
+
)
|
|
1518
|
+
}
|
|
1519
|
+
if (observed.blocking) out.push({ sessionId, reason: 'queued', ...described })
|
|
1520
|
+
} else {
|
|
1521
|
+
// No queued work any more: the timer of that session is dropped, so
|
|
1522
|
+
// a NEW queue starts counting from zero.
|
|
1523
|
+
queuedTracker.observe(sessionId, false, config.queuedGraceMs)
|
|
1524
|
+
}
|
|
1355
1525
|
} catch {}
|
|
1356
1526
|
}
|
|
1527
|
+
// A session that is gone (closed) drops its record as well.
|
|
1528
|
+
for (const sessionId of queuedSessions) {
|
|
1529
|
+
if (!seen.has(sessionId)) queuedTracker.forget(sessionId)
|
|
1530
|
+
}
|
|
1531
|
+
queuedSessions.clear()
|
|
1532
|
+
for (const sessionId of seen) queuedSessions.add(sessionId)
|
|
1533
|
+
// Real work is still going on: the quiet window starts over. This is the
|
|
1534
|
+
// marker that makes the quiet window measure "since the last real work"
|
|
1535
|
+
// instead of "since the last session event of any kind".
|
|
1536
|
+
if (out.length > 0) lastBusyAt = Date.now()
|
|
1357
1537
|
return out
|
|
1358
1538
|
}
|
|
1359
1539
|
|
|
@@ -1695,13 +1875,33 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1695
1875
|
*/
|
|
1696
1876
|
const pendingRestart = () => state.restart !== null && state.restart !== undefined
|
|
1697
1877
|
|
|
1698
|
-
/**
|
|
1699
|
-
|
|
1878
|
+
/**
|
|
1879
|
+
* The moment the quiet window started.
|
|
1880
|
+
*
|
|
1881
|
+
* It is the LATER of the two markers:
|
|
1882
|
+
* - `lastActivityAt`: the last real activity signal (a turn that started
|
|
1883
|
+
* or ended, a message that was queued);
|
|
1884
|
+
* - `lastBusyAt`: the last moment a conversation was seen working (a
|
|
1885
|
+
* running turn or a queued message).
|
|
1886
|
+
*
|
|
1887
|
+
* The routine session events of the interface (a title, a projection, a
|
|
1888
|
+
* context request) are NOT activity: they arrive every two to three
|
|
1889
|
+
* seconds, so counting them would keep the quiet window from ever filling
|
|
1890
|
+
* — and the restart would never fire.
|
|
1891
|
+
*
|
|
1892
|
+
* @returns {number} a timestamp in milliseconds
|
|
1893
|
+
*/
|
|
1894
|
+
const quietSince = () => Math.max(lastActivityAt, lastBusyAt)
|
|
1895
|
+
|
|
1896
|
+
/** How long the host has been quiet (no real conversation activity at all). */
|
|
1897
|
+
const quietForMs = () => Math.max(0, Date.now() - quietSince())
|
|
1700
1898
|
|
|
1701
1899
|
/**
|
|
1702
|
-
* The
|
|
1703
|
-
* conversation finished
|
|
1704
|
-
*
|
|
1900
|
+
* The RESTART reserve: how much longer the host waits after the last
|
|
1901
|
+
* conversation finished before it is really restarted.
|
|
1902
|
+
*
|
|
1903
|
+
* A restart is a daily operation, so the default is none: the host restarts
|
|
1904
|
+
* as soon as the conversations are done and the settle window has passed.
|
|
1705
1905
|
*/
|
|
1706
1906
|
const graceMs = () => config.restartGraceMs
|
|
1707
1907
|
|
|
@@ -1716,11 +1916,45 @@ export function createRuntime(ctx, config, options = {}) {
|
|
|
1716
1916
|
/** The two-layer restart gate: no busy conversation AND a quiet window. */
|
|
1717
1917
|
const gate = () => restartGate(state, {
|
|
1718
1918
|
busyAgents: busyAgents(),
|
|
1719
|
-
lastActivityAt,
|
|
1919
|
+
lastActivityAt: quietSince(),
|
|
1720
1920
|
now: Date.now(),
|
|
1721
1921
|
settleMs: effectiveSettleMs(),
|
|
1722
1922
|
})
|
|
1723
1923
|
|
|
1924
|
+
/**
|
|
1925
|
+
* The description of a LONG wait, or null when the restart is not waiting
|
|
1926
|
+
* for the long time yet.
|
|
1927
|
+
*
|
|
1928
|
+
* A pending restart that passed the hard deadline while a conversation was
|
|
1929
|
+
* still running is not "just a little longer": it is a long wait, and the
|
|
1930
|
+
* user has to SEE it — how long it has been waiting, why, and who holds it.
|
|
1931
|
+
* The request itself is never dropped: the restart stays pending and fires
|
|
1932
|
+
* as soon as the gate is quiet again.
|
|
1933
|
+
*
|
|
1934
|
+
* @param {number} [now]
|
|
1935
|
+
* @returns {{ since: number, reason: string, waitedMs: number, blockedBy: any[] }|null}
|
|
1936
|
+
*/
|
|
1937
|
+
const longWaitInfo = (now = Date.now()) => {
|
|
1938
|
+
const restart = state.restart
|
|
1939
|
+
if (!restart || typeof restart !== 'object') return null
|
|
1940
|
+
const requestedAt = Number.isFinite(restart.requestedAt) ? restart.requestedAt : 0
|
|
1941
|
+
if (requestedAt <= 0) return null
|
|
1942
|
+
const hardMs = Number.isFinite(config.deferHardMs) ? config.deferHardMs : 0
|
|
1943
|
+
// The long wait begins when the hard deadline is reached.
|
|
1944
|
+
const since = requestedAt + hardMs
|
|
1945
|
+
if (now < since) return null
|
|
1946
|
+
const verdict = gate()
|
|
1947
|
+
if (verdict.allow) return null
|
|
1948
|
+
const blockedBy = goalAwareBlockingDetails(runtime, verdict.blockedBy)
|
|
1949
|
+
if (blockedBy.length === 0) return null
|
|
1950
|
+
return {
|
|
1951
|
+
since,
|
|
1952
|
+
reason: 'hard-deadline-waiting-for-conversations',
|
|
1953
|
+
waitedMs: Math.max(0, now - requestedAt),
|
|
1954
|
+
blockedBy,
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1724
1958
|
/**
|
|
1725
1959
|
* Cached session titles, used to label the blocking sessions in the
|
|
1726
1960
|
* client's "pending restart" details panel.
|
|
@@ -2073,7 +2307,7 @@ const titleOfSummary = (item) => {
|
|
|
2073
2307
|
const decision = restartDecision(state, {
|
|
2074
2308
|
anyAgentRunning: busy.length > 0,
|
|
2075
2309
|
busyAgents: busy,
|
|
2076
|
-
lastActivityAt,
|
|
2310
|
+
lastActivityAt: quietSince(),
|
|
2077
2311
|
settleMs: effectiveSettleMs(),
|
|
2078
2312
|
now,
|
|
2079
2313
|
softMs: config.deferSoftMs,
|
|
@@ -2082,6 +2316,23 @@ const titleOfSummary = (item) => {
|
|
|
2082
2316
|
forceRestartWhenBusy: config.forceRestartWhenBusy,
|
|
2083
2317
|
})
|
|
2084
2318
|
if (decision.log) log.info(decision.log)
|
|
2319
|
+
// The diagnostic line of the pending restart: what the gate decided and
|
|
2320
|
+
// why. It is rate limited (once a minute) so the 3s poll does not flood
|
|
2321
|
+
// the log, and it is the first thing to look at when "the restart never
|
|
2322
|
+
// happens" is reported.
|
|
2323
|
+
if (now - lastDecisionLogAt >= LONG_WAIT_LOG_INTERVAL_MS) {
|
|
2324
|
+
lastDecisionLogAt = now
|
|
2325
|
+
log.info(
|
|
2326
|
+
`restart gate: action=${decision.action} reason=${decision.reason} busyCount=${busy.length} quietForMs=${Math.max(0, now - quietSince())} effectiveSettleMs=${effectiveSettleMs()}`,
|
|
2327
|
+
{
|
|
2328
|
+
action: decision.action,
|
|
2329
|
+
reason: decision.reason,
|
|
2330
|
+
busyCount: busy.length,
|
|
2331
|
+
quietForMs: Math.max(0, now - quietSince()),
|
|
2332
|
+
effectiveSettleMs: effectiveSettleMs(),
|
|
2333
|
+
},
|
|
2334
|
+
)
|
|
2335
|
+
}
|
|
2085
2336
|
switch (decision.action) {
|
|
2086
2337
|
case 'wait':
|
|
2087
2338
|
return
|
|
@@ -2092,6 +2343,28 @@ const titleOfSummary = (item) => {
|
|
|
2092
2343
|
}
|
|
2093
2344
|
return
|
|
2094
2345
|
}
|
|
2346
|
+
case 'long-wait': {
|
|
2347
|
+
// The hard deadline is over and a conversation still holds the
|
|
2348
|
+
// restart. The request is KEPT — the user asked for a restart, it must
|
|
2349
|
+
// never be dropped silently — and the wait becomes VISIBLE: a clear
|
|
2350
|
+
// log line (rate limited, so the 3s poll does not flood the log) and
|
|
2351
|
+
// the `longWait` block of the state payload.
|
|
2352
|
+
const requestedAt = Number.isFinite(state.restart.requestedAt) ? state.restart.requestedAt : now
|
|
2353
|
+
const waitedMs = Math.max(0, now - requestedAt)
|
|
2354
|
+
const lastAt = longWaitLog.requestedAt === requestedAt ? longWaitLog.at : 0
|
|
2355
|
+
if (now - lastAt >= LONG_WAIT_LOG_INTERVAL_MS) {
|
|
2356
|
+
longWaitLog = { at: now, requestedAt }
|
|
2357
|
+
log.warn(
|
|
2358
|
+
`the restart has been waiting for ${describeWait(waitedMs)}; ${describeBlockers(decision.blockedBy)} still holds it — cancel the restart or wait for that work to finish`,
|
|
2359
|
+
{
|
|
2360
|
+
waitedMs,
|
|
2361
|
+
blockedBy: decision.blockedBy,
|
|
2362
|
+
reason: decision.reason,
|
|
2363
|
+
},
|
|
2364
|
+
)
|
|
2365
|
+
}
|
|
2366
|
+
return
|
|
2367
|
+
}
|
|
2095
2368
|
case 'restart':
|
|
2096
2369
|
case 'force-restart': {
|
|
2097
2370
|
void executeRestart()
|
|
@@ -2351,13 +2624,23 @@ const titleOfSummary = (item) => {
|
|
|
2351
2624
|
*/
|
|
2352
2625
|
pendingRestart,
|
|
2353
2626
|
/**
|
|
2354
|
-
* How long the host has been quiet (no conversation activity
|
|
2627
|
+
* How long the host has been quiet (no REAL conversation activity: a
|
|
2628
|
+
* running turn, a queued message, a turn that started or ended).
|
|
2629
|
+
*
|
|
2630
|
+
* The routine session events of the interface do not count.
|
|
2355
2631
|
*
|
|
2356
2632
|
* @returns {number}
|
|
2357
2633
|
*/
|
|
2358
2634
|
quietForMs,
|
|
2359
2635
|
/**
|
|
2360
|
-
* The
|
|
2636
|
+
* The moment the quiet window started.
|
|
2637
|
+
*
|
|
2638
|
+
* @returns {number} a timestamp in milliseconds
|
|
2639
|
+
*/
|
|
2640
|
+
quietSince,
|
|
2641
|
+
/**
|
|
2642
|
+
* The RESTART reserve: the extra wait after the last conversation
|
|
2643
|
+
* finished, before the host is really restarted.
|
|
2361
2644
|
*
|
|
2362
2645
|
* @returns {number} milliseconds
|
|
2363
2646
|
*/
|
|
@@ -2378,6 +2661,19 @@ const titleOfSummary = (item) => {
|
|
|
2378
2661
|
noteActivity,
|
|
2379
2662
|
busyAgents,
|
|
2380
2663
|
gate,
|
|
2664
|
+
/**
|
|
2665
|
+
* The description of a long wait (null when there is none).
|
|
2666
|
+
*
|
|
2667
|
+
* @param {number} [now]
|
|
2668
|
+
* @returns {{ since: number, reason: string, waitedMs: number, blockedBy: any[] }|null}
|
|
2669
|
+
*/
|
|
2670
|
+
longWaitInfo,
|
|
2671
|
+
/**
|
|
2672
|
+
* The upper bound of a queued (unclaimed) inbox message.
|
|
2673
|
+
*
|
|
2674
|
+
* @returns {number} milliseconds
|
|
2675
|
+
*/
|
|
2676
|
+
queuedGraceMs: () => config.queuedGraceMs,
|
|
2381
2677
|
sessionTitle,
|
|
2382
2678
|
disarmGoal,
|
|
2383
2679
|
disarmBlockingGoals,
|
|
@@ -3107,6 +3403,7 @@ function registerTools(ctx, runtime, config) {
|
|
|
3107
3403
|
quietForMs: { type: 'number' },
|
|
3108
3404
|
graceMs: { type: 'number' },
|
|
3109
3405
|
effectiveSettleMs: { type: 'number' },
|
|
3406
|
+
queuedGraceMs: { type: 'number' },
|
|
3110
3407
|
blockedBy: {
|
|
3111
3408
|
type: 'array',
|
|
3112
3409
|
items: {
|
|
@@ -3158,10 +3455,12 @@ function registerTools(ctx, runtime, config) {
|
|
|
3158
3455
|
pendingRestart: state.restart !== null,
|
|
3159
3456
|
gateAllow: verdict.allow,
|
|
3160
3457
|
quietForMs: Math.round(verdict.quietForMs),
|
|
3161
|
-
// The
|
|
3458
|
+
// The RESTART reserve (the extra wait after the last conversation
|
|
3162
3459
|
// finished) and the quiet window the gate really uses.
|
|
3163
3460
|
graceMs: runtime.graceMs(),
|
|
3164
3461
|
effectiveSettleMs: runtime.effectiveSettleMs(),
|
|
3462
|
+
// The upper bound of a queued (unclaimed) inbox message.
|
|
3463
|
+
queuedGraceMs: runtime.queuedGraceMs(),
|
|
3165
3464
|
blockedBy: blockingDetails(verdict.blockedBy),
|
|
3166
3465
|
parkedGoals: parkedGoalDetails(),
|
|
3167
3466
|
recovery: recoveryList(state),
|
|
@@ -3417,7 +3716,7 @@ function injectPageHeartbeat(html, bootId, routePath) {
|
|
|
3417
3716
|
const AUTORESTART_PROMPT = [
|
|
3418
3717
|
'## Restart policy (dsh-autorestart)',
|
|
3419
3718
|
'- When DeepSeek Harness itself must restart (for example to load an installed or updated plugin bundle), do NOT kill or relaunch the app yourself with pwsh/taskkill/Stop-Process/npm scripts. A hard guard blocks such commands anyway (denials mention `blockManualRestart`).',
|
|
3420
|
-
'- Call the `continuum_restart` tool instead (optional arguments: `reason`, `resume`, `defer_seconds`, `parkGoals`). It waits until every conversation is idle (no running turn, no
|
|
3719
|
+
'- Call the `continuum_restart` tool instead (optional arguments: `reason`, `resume`, `defer_seconds`, `parkGoals`). It waits until every conversation is idle (no running turn — a running turn ALWAYS blocks; an inbox message nobody claimed blocks only until `queuedGraceMs`, default 60000 ms, after which it is treated as stuck and no longer blocks) AND the host has been quiet for `settleMs` (default 2000 ms); the detached helper re-checks that gate immediately before killing, so other conversations are never killed mid-turn. It then relaunches the app, and after boot this plugin resumes the session and continues the work automatically.',
|
|
3421
3720
|
'- A goal-driven session is DISARMED automatically before the restart (its current round still finishes, nothing is lost), because such a session would otherwise keep the restart waiting forever: disarming only removes the process-local continuation authority, so no new round starts and the session goes idle. After the restart the goal is re-armed and continues. A goal whose round budget is spent cannot be continued and is reported instead. You can disarm your own goal on demand with `continuum_park_goal`, and retry a stuck recovery with `continuum_recover_now`.',
|
|
3422
3721
|
'- Persist anything the next turn needs before restarting (workspace files, `todo_write`, goal state); do not rely on in-memory state.',
|
|
3423
3722
|
'- Inspect the lifecycle with `continuum_status`; cancel a pending restart with `continuum_cancel_restart`.',
|
|
@@ -4019,7 +4318,7 @@ function buildChoiceRoute(ctx, runtime, config) {
|
|
|
4019
4318
|
res.end(JSON.stringify({
|
|
4020
4319
|
ok: false,
|
|
4021
4320
|
error: 'no-choice',
|
|
4022
|
-
instruction: 'Send { "includePlugins": true | false } and/or { "backupIncludePlugins": true | false } and/or { "backupOnRestart": true | false } and/or { "showSleepButton": true | false } and/or { "sleepCountdownSeconds": 0 | 10 | 20 | 30 | 60 } and/or { "restartGraceMs": 0 |
|
|
4321
|
+
instruction: 'Send { "includePlugins": true | false } and/or { "backupIncludePlugins": true | false } and/or { "backupOnRestart": true | false } and/or { "showSleepButton": true | false } and/or { "sleepCountdownSeconds": 0 | 10 | 20 | 30 | 60 } and/or { "restartGraceMs": 0 | 5000 | 10000 | 20000 | 30000 | 60000 }.',
|
|
4023
4322
|
}))
|
|
4024
4323
|
return
|
|
4025
4324
|
}
|
|
@@ -4158,16 +4457,16 @@ function sleepDryRun() {
|
|
|
4158
4457
|
* happened). The sleep feature counts down on screen itself, so it only needs
|
|
4159
4458
|
* the busy check — and it keeps `settleMs` out of the picture entirely.
|
|
4160
4459
|
*
|
|
4161
|
-
* The
|
|
4162
|
-
*
|
|
4163
|
-
*
|
|
4164
|
-
*
|
|
4460
|
+
* The shutdown has NO invisible reserve of its own: the visible countdown
|
|
4461
|
+
* (`sleepCountdownSeconds`) IS the wait before the machine goes off, and it is
|
|
4462
|
+
* on screen, so the user can see it and cancel it. A second, hidden window
|
|
4463
|
+
* next to it would only say the same thing twice.
|
|
4165
4464
|
*
|
|
4166
4465
|
* The restart gate itself is untouched: the restart feature keeps its quiet
|
|
4167
|
-
* window exactly as it was (plus
|
|
4466
|
+
* window exactly as it was (plus its own restart reserve).
|
|
4168
4467
|
*
|
|
4169
4468
|
* @param {object} runtime
|
|
4170
|
-
* @returns {{ allow: boolean, reason: 'busy'|'
|
|
4469
|
+
* @returns {{ allow: boolean, reason: 'busy'|'quiet', blockedBy: any[] }}
|
|
4171
4470
|
*/
|
|
4172
4471
|
function sleepGate(runtime) {
|
|
4173
4472
|
// A pending restart goes FIRST: the shutdown waits for it — and for the
|
|
@@ -4199,16 +4498,8 @@ function sleepGate(runtime) {
|
|
|
4199
4498
|
}
|
|
4200
4499
|
const list = Array.isArray(busy) ? busy : []
|
|
4201
4500
|
if (list.length > 0) return { allow: false, reason: 'busy', blockedBy: list }
|
|
4202
|
-
//
|
|
4203
|
-
//
|
|
4204
|
-
let quiet = 0
|
|
4205
|
-
let grace = 0
|
|
4206
|
-
try { quiet = Number(runtime.quietForMs()) } catch { quiet = 0 }
|
|
4207
|
-
try { grace = Number(runtime.graceMs()) } catch { grace = 0 }
|
|
4208
|
-
if (!Number.isFinite(quiet) || quiet < 0) quiet = 0
|
|
4209
|
-
if (Number.isFinite(grace) && grace > 0 && quiet < grace) {
|
|
4210
|
-
return { allow: false, reason: 'grace', blockedBy: [] }
|
|
4211
|
-
}
|
|
4501
|
+
// Nobody is busy and no restart is pending: the countdown may run. There is
|
|
4502
|
+
// no invisible reserve — the countdown on screen is the wait.
|
|
4212
4503
|
return { allow: true, reason: 'quiet', blockedBy: [] }
|
|
4213
4504
|
}
|
|
4214
4505
|
|
|
@@ -4290,11 +4581,11 @@ function createSleepService(ctx, runtime, config, options = {}) {
|
|
|
4290
4581
|
busyAgents: () => runtime.busyAgents(),
|
|
4291
4582
|
gate,
|
|
4292
4583
|
titleOf: (sessionId) => runtime.sessionTitle(sessionId),
|
|
4293
|
-
// The
|
|
4294
|
-
//
|
|
4295
|
-
// plugin
|
|
4296
|
-
|
|
4297
|
-
graceMs: () =>
|
|
4584
|
+
// The shutdown has NO invisible reserve: the countdown on screen IS the
|
|
4585
|
+
// wait before the machine goes off. The controller keeps the parameter for
|
|
4586
|
+
// compatibility, but the plugin never injects a window — and it never
|
|
4587
|
+
// injects the RESTART reserve either.
|
|
4588
|
+
graceMs: () => 0,
|
|
4298
4589
|
// A pending restart goes first: the shutdown waits for it.
|
|
4299
4590
|
restartPending: () => {
|
|
4300
4591
|
const pending = runtime.pendingRestart()
|
|
@@ -4429,9 +4720,10 @@ function sleepStatusPayload(sleep, config, runtime) {
|
|
|
4429
4720
|
armed: status.armed,
|
|
4430
4721
|
showSleepButton: config.showSleepButton !== false,
|
|
4431
4722
|
countdownSeconds: status.countdownSeconds,
|
|
4432
|
-
// The
|
|
4433
|
-
//
|
|
4434
|
-
|
|
4723
|
+
// The RESTART reserve is reported next to the countdown, so the settings
|
|
4724
|
+
// page can show it. The shutdown itself has NO invisible reserve: the
|
|
4725
|
+
// visible countdown is the wait before the machine goes off.
|
|
4726
|
+
restartGraceMs: config.restartGraceMs,
|
|
4435
4727
|
quietForMs: runtime && typeof runtime.quietForMs === 'function' ? runtime.quietForMs() : 0,
|
|
4436
4728
|
// When the shutdown was armed: the intent has to survive a restart, so the
|
|
4437
4729
|
// time is reported for the troubleshooting.
|
|
@@ -4573,10 +4865,14 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4573
4865
|
generation: state.generation,
|
|
4574
4866
|
pendingRestart: state.restart !== null,
|
|
4575
4867
|
settleMs: config.settleMs,
|
|
4576
|
-
// The
|
|
4577
|
-
// are needed to tell "waiting for the
|
|
4868
|
+
// The RESTART reserve and the quiet window the gate really uses:
|
|
4869
|
+
// both are needed to tell "waiting for the reserve" from "somebody
|
|
4870
|
+
// is busy".
|
|
4578
4871
|
graceMs: config.restartGraceMs,
|
|
4579
4872
|
effectiveSettleMs: runtime.effectiveSettleMs(),
|
|
4873
|
+
// The upper bound of a queued (unclaimed) inbox message: after this
|
|
4874
|
+
// window it stops blocking the restart.
|
|
4875
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
4580
4876
|
quietForMs: typeof verdict.quietForMs === 'number' ? verdict.quietForMs : 0,
|
|
4581
4877
|
busyCount: verdict.blockedBy.length,
|
|
4582
4878
|
agentCount,
|
|
@@ -4639,11 +4935,23 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4639
4935
|
waiters,
|
|
4640
4936
|
blockedBy,
|
|
4641
4937
|
allowRestart: verdict.allow === true,
|
|
4938
|
+
// WHY the gate says what it says. Without this, "allowRestart is
|
|
4939
|
+
// false and blockedBy is empty" looks like a contradiction: the
|
|
4940
|
+
// reason tells the two apart (an empty blocker list with the
|
|
4941
|
+
// reason "settling" means the quiet window is not full yet).
|
|
4942
|
+
reason: typeof verdict.reason === 'string' ? verdict.reason : 'unknown',
|
|
4642
4943
|
quietForMs: typeof verdict.quietForMs === 'number' ? verdict.quietForMs : 0,
|
|
4643
4944
|
settleMs: config.settleMs,
|
|
4644
|
-
// The
|
|
4945
|
+
// The RESTART reserve and the quiet window the gate really uses.
|
|
4645
4946
|
graceMs: config.restartGraceMs,
|
|
4646
4947
|
effectiveSettleMs: runtime.effectiveSettleMs(),
|
|
4948
|
+
// The upper bound of a queued (unclaimed) inbox message.
|
|
4949
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
4950
|
+
// The long wait: present only when the restart passed the hard
|
|
4951
|
+
// deadline while a conversation was still running. It says how
|
|
4952
|
+
// long the restart has been waiting, why, and who holds it — the
|
|
4953
|
+
// request itself is never dropped.
|
|
4954
|
+
longWait: runtime.longWaitInfo(),
|
|
4647
4955
|
updatedAt: live.updatedAt,
|
|
4648
4956
|
// The park route, so the client never has to guess the prefix, and
|
|
4649
4957
|
// the goals this plugin already disarmed (they are re-armed after
|
|
@@ -4669,7 +4977,10 @@ function registerHealthRoute(ctx, runtime, config, sleep) {
|
|
|
4669
4977
|
waiters: [],
|
|
4670
4978
|
blockedBy: [],
|
|
4671
4979
|
allowRestart: false,
|
|
4980
|
+
reason: 'unknown',
|
|
4672
4981
|
quietForMs: 0,
|
|
4982
|
+
queuedGraceMs: config.queuedGraceMs,
|
|
4983
|
+
longWait: null,
|
|
4673
4984
|
parkUrl: resolveParkPath(config),
|
|
4674
4985
|
recoverUrl: resolveRecoverPath(config),
|
|
4675
4986
|
choiceUrl: resolveChoicePath(config),
|
|
@@ -4838,9 +5149,20 @@ function trackAgents(ctx, runtime, config) {
|
|
|
4838
5149
|
}
|
|
4839
5150
|
})
|
|
4840
5151
|
|
|
4841
|
-
//
|
|
4842
|
-
|
|
5152
|
+
// A session event only restarts the settle window when it says that WORK
|
|
5153
|
+
// happened (a message, a tool, a turn). The host produces session events
|
|
5154
|
+
// every two to three seconds while the interface polls and synchronises —
|
|
5155
|
+
// counting those as activity kept the quiet window from ever filling, so a
|
|
5156
|
+
// restart with a grace window of ten seconds NEVER fired. When the type of
|
|
5157
|
+
// an event cannot be read, it is not activity: the plugin would rather
|
|
5158
|
+
// restart a little early than never restart at all.
|
|
5159
|
+
//
|
|
5160
|
+
// The hard guarantee is elsewhere and untouched: a conversation that is
|
|
5161
|
+
// really running blocks the restart through `busyAgents()`, which is never
|
|
5162
|
+
// relaxed.
|
|
5163
|
+
ctx.on('session/event', (payload) => {
|
|
4843
5164
|
try {
|
|
5165
|
+
if (!isWorkSessionEvent(payload)) return
|
|
4844
5166
|
runtime.noteActivity()
|
|
4845
5167
|
} catch {}
|
|
4846
5168
|
})
|