dsh-pause-resume 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +15 -7
- package/client.js +26 -11
- package/index.js +38 -11
- package/lib/engine.js +394 -157
- package/lib/epoch.js +27 -0
- package/lib/heartbeat.js +34 -4
- package/lib/probe.js +113 -36
- package/lib/snapshot.js +169 -29
- package/package.json +2 -1
package/lib/engine.js
CHANGED
|
@@ -6,26 +6,41 @@
|
|
|
6
6
|
* set, so two windows, a double press and a tool call can never overlap. The client is not allowed
|
|
7
7
|
* to infer a phase: it asks, and renders the answer.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* `action` is state-derived, never phase-derived. Live work asks for a pause; a plan with no live
|
|
10
|
+
* work asks for a resume; neither is `none`. That single rule is what makes a second press useful
|
|
11
|
+
* after a failed pause, a partial stop, or a restart: the button does what the Harness still needs,
|
|
12
|
+
* not what the last operation intended.
|
|
12
13
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
14
|
+
* A plan is kept until every item in it has a terminal outcome or a retryable failure. Attention
|
|
15
|
+
* outcomes (a continuation that was admitted but never opened a turn, a job that cannot be
|
|
16
|
+
* replayed) are terminal: they are carried into the last-restore record and shown as a dot, but
|
|
17
|
+
* they never cause the same message to be delivered twice. The crash plan is written to disk
|
|
18
|
+
* before the heartbeat overwrites the previous process's record, so a second or third restart
|
|
19
|
+
* still sees the same pending work.
|
|
15
20
|
*/
|
|
16
21
|
|
|
22
|
+
import { currentEpoch } from './epoch.js'
|
|
23
|
+
|
|
17
24
|
const TRANSIENT = new Set(['pausing', 'resuming'])
|
|
18
25
|
const TERMINAL = new Set(['restored', 'skipped', 'attention'])
|
|
19
26
|
|
|
20
|
-
|
|
27
|
+
function emptyLive() {
|
|
28
|
+
return { sessions: [], goals: [], jobs: [], readable: false, source: 'none' }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function createEngine({ store, probe, heartbeat = null, bootId = 'boot', epoch = null, now = () => Date.now(), graceMs = 2000, log = () => {} }) {
|
|
32
|
+
/**
|
|
33
|
+
* A reload creates a new instance and bumps the shared epoch. An old instance's press may still
|
|
34
|
+
* be awaiting a cancelled turn; it must not write its stale plan over the new instance's one.
|
|
35
|
+
*/
|
|
36
|
+
const isCurrent = () => epoch === null || currentEpoch() === epoch
|
|
21
37
|
let phase = 'idle'
|
|
22
38
|
let lastError = null
|
|
23
39
|
let result = null
|
|
24
40
|
/** Set while a press runs, so a second caller joins the answer instead of starting work again. */
|
|
25
41
|
let inflight = null
|
|
26
|
-
/**
|
|
27
|
-
let
|
|
28
|
-
let crashRead = false
|
|
42
|
+
/** Crash plan kept in memory only when its durable write failed; a reload loses it, the file does not. */
|
|
43
|
+
let crashFallback = null
|
|
29
44
|
let counter = 0
|
|
30
45
|
|
|
31
46
|
function newPlanId(kind) {
|
|
@@ -33,29 +48,22 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
33
48
|
return `${kind}-${now().toString(36)}-${counter.toString(36)}`
|
|
34
49
|
}
|
|
35
50
|
|
|
36
|
-
function
|
|
51
|
+
function pausedPlan() {
|
|
37
52
|
return store.read()
|
|
38
53
|
}
|
|
39
54
|
|
|
40
|
-
function
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
try {
|
|
44
|
-
const residue = heartbeat?.residue ?? null
|
|
45
|
-
// The id is derived from the residue itself: two polls of the same crash share one journal,
|
|
46
|
-
// and a different crash gets its own.
|
|
47
|
-
const at = Number.isFinite(residue?.at) ? residue.at : 0
|
|
48
|
-
const count = Array.isArray(residue?.sessions) ? residue.sessions.length : 0
|
|
49
|
-
crash = probe.crashPlan(residue, { planId: `crash-${at}-${count}` })
|
|
50
|
-
} catch (error) {
|
|
51
|
-
log(`cannot read crash residue: ${error?.message ?? error}`)
|
|
52
|
-
crash = null
|
|
53
|
-
}
|
|
54
|
-
return crash
|
|
55
|
+
function crashPlan() {
|
|
56
|
+
const stored = typeof store.readPending === 'function' ? store.readPending() : null
|
|
57
|
+
return stored ?? crashFallback
|
|
55
58
|
}
|
|
56
59
|
|
|
60
|
+
/** Exactly one plan is pending, and a finished pause outranks a leftover crash record. */
|
|
57
61
|
function currentPlan() {
|
|
58
|
-
return
|
|
62
|
+
return pausedPlan() ?? crashPlan()
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function isCrashPlan(plan) {
|
|
66
|
+
return plan !== null && plan !== undefined && plan.source === 'crash'
|
|
59
67
|
}
|
|
60
68
|
|
|
61
69
|
function itemKeys(plan) {
|
|
@@ -66,10 +74,17 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
66
74
|
return keys
|
|
67
75
|
}
|
|
68
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Rows written by an earlier process are evidence, not permission to skip: a new process must
|
|
79
|
+
* check the live Harness again. Only this boot's rows reduce what is still outstanding.
|
|
80
|
+
*/
|
|
69
81
|
function outstandingOf(plan) {
|
|
70
82
|
if (plan === null || plan === undefined) return 0
|
|
71
|
-
const
|
|
72
|
-
return itemKeys(plan).filter((key) =>
|
|
83
|
+
const rows = store.readJournal(plan.planId)
|
|
84
|
+
return itemKeys(plan).filter((key) => {
|
|
85
|
+
const row = rows.get(key)
|
|
86
|
+
return row === undefined || (row.boot !== null && row.boot !== bootId)
|
|
87
|
+
}).length
|
|
73
88
|
}
|
|
74
89
|
|
|
75
90
|
function countsOf(plan) {
|
|
@@ -83,14 +98,37 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
83
98
|
|
|
84
99
|
function liveNow() {
|
|
85
100
|
try {
|
|
86
|
-
|
|
87
|
-
return value ?? { sessions: [], goals: [], jobs: [], readable: false, source: 'none' }
|
|
101
|
+
return probe.live() ?? emptyLive()
|
|
88
102
|
} catch (error) {
|
|
89
103
|
log(`cannot read live work: ${error?.message ?? error}`)
|
|
90
|
-
return
|
|
104
|
+
return emptyLive()
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Only work that was actually stopped goes into the plan: a session whose turn was open, or a
|
|
110
|
+
* session that carries an armed goal. A session that ended normally is left alone, however it
|
|
111
|
+
* happens to look in the session list.
|
|
112
|
+
*/
|
|
113
|
+
function workOf(current) {
|
|
114
|
+
const sessions = new Map()
|
|
115
|
+
for (const session of current.sessions ?? []) {
|
|
116
|
+
if (session.running === true) sessions.set(session.sessionId, session)
|
|
117
|
+
}
|
|
118
|
+
for (const goal of current.goals ?? []) {
|
|
119
|
+
if (!sessions.has(goal.sessionId)) sessions.set(goal.sessionId, { sessionId: goal.sessionId, running: false, lastTurn: null })
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
sessions: [...sessions.values()],
|
|
123
|
+
goals: current.goals ?? [],
|
|
124
|
+
jobs: current.jobs ?? [], // ui-copy-ignore
|
|
91
125
|
}
|
|
92
126
|
}
|
|
93
127
|
|
|
128
|
+
function liveWorkCount(work) {
|
|
129
|
+
return work.sessions.length + work.goals.length + work.jobs.length
|
|
130
|
+
}
|
|
131
|
+
|
|
94
132
|
/**
|
|
95
133
|
* What the button shows. `action` is the one decision the client may not make itself:
|
|
96
134
|
* `none` (nothing to do), `pause`, `resume`, or `busy` (an operation is running).
|
|
@@ -98,167 +136,275 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
98
136
|
function describe() {
|
|
99
137
|
const plan = currentPlan()
|
|
100
138
|
const current = liveNow()
|
|
101
|
-
const busy = TRANSIENT.has(phase)
|
|
102
139
|
const work = workOf(current)
|
|
103
|
-
const
|
|
104
|
-
const
|
|
105
|
-
|
|
140
|
+
const busy = TRANSIENT.has(phase)
|
|
141
|
+
const workCount = liveWorkCount(work)
|
|
142
|
+
let action = 'none'
|
|
143
|
+
if (busy) action = 'busy'
|
|
144
|
+
else if (current.readable === true && workCount > 0) action = 'pause'
|
|
145
|
+
else if (plan !== null) action = 'resume'
|
|
146
|
+
const attentionRows = plan !== null ? plan.attention : result?.attention
|
|
147
|
+
const attention = Array.isArray(attentionRows) ? attentionRows.length : 0
|
|
148
|
+
const first = attention > 0 ? attentionRows[0] : null
|
|
149
|
+
const focus = first === null ? null : (typeof first === 'string' ? first : (first.id || first.detail || null))
|
|
106
150
|
return {
|
|
107
151
|
phase,
|
|
108
152
|
action,
|
|
109
153
|
source: plan === null ? null : plan.source ?? 'pause',
|
|
110
|
-
//
|
|
111
|
-
complete: plan === null ? null : plan.status
|
|
112
|
-
outstanding:
|
|
154
|
+
// `complete` means the stop finished and nothing is still turning. A partial stop is not.
|
|
155
|
+
complete: plan === null ? null : plan.status !== 'partial' && plan.status !== 'pausing' && lastError !== 'store',
|
|
156
|
+
outstanding: outstandingOf(plan), // ui-copy-ignore
|
|
113
157
|
counts: countsOf(plan), // ui-copy-ignore
|
|
114
158
|
live: { // ui-copy-ignore
|
|
115
159
|
sessions: work.sessions.filter((session) => session.running === true).length,
|
|
116
160
|
goals: current.goals.length,
|
|
117
161
|
jobs: current.jobs.length,
|
|
118
162
|
},
|
|
119
|
-
attention
|
|
120
|
-
|
|
163
|
+
attention,
|
|
164
|
+
focus, // ui-copy-ignore
|
|
165
|
+
error: lastError ?? (plan?.status === 'pausing' ? 'pause-interrupted' : plan?.status === 'partial' ? 'pause-partial' : null), // ui-copy-ignore
|
|
121
166
|
result,
|
|
122
167
|
}
|
|
123
168
|
}
|
|
124
169
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
170
|
+
function freshSession(session) {
|
|
171
|
+
// A session in the captured live set had its turn open, so the press owns a continuation for it.
|
|
172
|
+
// A crash row is different: it is evidence that a turn was open, not permission to send text.
|
|
173
|
+
const running = session.running === true
|
|
174
|
+
return {
|
|
175
|
+
sessionId: session.sessionId,
|
|
176
|
+
running,
|
|
177
|
+
intent: running && session.intent !== 'mount' ? 'continue' : 'mount',
|
|
178
|
+
epoch: running && Number.isSafeInteger(session.epoch) ? session.epoch : 0,
|
|
179
|
+
lastTurn: Number.isSafeInteger(session.lastTurn) ? session.lastTurn : null,
|
|
134
180
|
}
|
|
135
|
-
|
|
136
|
-
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function freshGoal(goal) {
|
|
184
|
+
return { sessionId: goal.sessionId, goalId: goal.goalId, revision: goal.revision }
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function freshJob(job) {
|
|
188
|
+
return { id: job.id, owner: job.owner ?? null, status: job.status ?? null }
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function unionSessions(base, additions) {
|
|
192
|
+
const out = []
|
|
193
|
+
const index = new Map()
|
|
194
|
+
for (const row of [...base, ...additions]) {
|
|
195
|
+
if (typeof row?.sessionId !== 'string') continue
|
|
196
|
+
const existing = index.get(row.sessionId)
|
|
197
|
+
if (existing !== undefined) {
|
|
198
|
+
// Stopping again refreshes the baseline; a plain union would keep a stale turn number.
|
|
199
|
+
if (Number.isSafeInteger(row.lastTurn)) existing.lastTurn = row.lastTurn
|
|
200
|
+
// A new capture of an actually-running session upgrades a crash-only row: the person then
|
|
201
|
+
// pressed pause while it was running, which is an explicit request to continue it later.
|
|
202
|
+
if (row.intent === 'continue' || row.running === true) {
|
|
203
|
+
existing.intent = 'continue'
|
|
204
|
+
existing.running = true
|
|
205
|
+
}
|
|
206
|
+
if (Number.isSafeInteger(row.epoch) && row.epoch > 0) existing.epoch = row.epoch
|
|
207
|
+
continue
|
|
208
|
+
}
|
|
209
|
+
const copy = { ...row }
|
|
210
|
+
index.set(copy.sessionId, copy)
|
|
211
|
+
out.push(copy)
|
|
137
212
|
}
|
|
138
|
-
return
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
213
|
+
return out
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function unionById(base, additions, idOf) {
|
|
217
|
+
const out = []
|
|
218
|
+
const seen = new Set()
|
|
219
|
+
for (const row of [...base, ...additions]) {
|
|
220
|
+
const id = idOf(row)
|
|
221
|
+
if (typeof id !== 'string' || id === '' || seen.has(id)) continue
|
|
222
|
+
seen.add(id)
|
|
223
|
+
out.push(row)
|
|
142
224
|
}
|
|
225
|
+
return out
|
|
143
226
|
}
|
|
144
227
|
|
|
145
|
-
function
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
status,
|
|
150
|
-
source: 'pause', // ui-copy-ignore
|
|
151
|
-
createdAt: new Date(now()).toISOString(),
|
|
152
|
-
sessions: workOf(current).sessions.map((session) => ({
|
|
153
|
-
sessionId: session.sessionId,
|
|
154
|
-
running: session.running === true,
|
|
155
|
-
lastTurn: Number.isSafeInteger(session.lastTurn) ? session.lastTurn : null,
|
|
156
|
-
})),
|
|
157
|
-
goals: workOf(current).goals.map((goal) => ({
|
|
158
|
-
sessionId: goal.sessionId,
|
|
159
|
-
goalId: goal.goalId,
|
|
160
|
-
revision: goal.revision,
|
|
161
|
-
})),
|
|
162
|
-
jobs: workOf(current).jobs.map((job) => ({ id: job.id, owner: job.owner ?? null, status: job.status ?? null })),
|
|
228
|
+
function persist(plan) {
|
|
229
|
+
if (isCurrent() === false) {
|
|
230
|
+
log('stale instance: refusing to write over a newer plan')
|
|
231
|
+
return null
|
|
163
232
|
}
|
|
233
|
+
if (isCrashPlan(plan) === true) {
|
|
234
|
+
const stored = store.writePending(plan)
|
|
235
|
+
if (stored !== null) crashFallback = null
|
|
236
|
+
return stored
|
|
237
|
+
}
|
|
238
|
+
return store.write(plan)
|
|
164
239
|
}
|
|
165
240
|
|
|
166
|
-
|
|
241
|
+
/**
|
|
242
|
+
* Pause, including the second attempt after a partial stop. Capture happens before anything is
|
|
243
|
+
* stopped, and the plan is durable before the first cancellation, so a crash in between is
|
|
244
|
+
* visible as an interrupted pause instead of a clean one.
|
|
245
|
+
*/
|
|
246
|
+
async function runPause(basePlan) {
|
|
167
247
|
const current = liveNow()
|
|
248
|
+
const work = workOf(current)
|
|
168
249
|
if (current.readable !== true) {
|
|
169
|
-
phase = 'idle'
|
|
250
|
+
phase = basePlan === null ? 'idle' : 'paused'
|
|
170
251
|
lastError = 'registry'
|
|
171
252
|
return { status: 'failed', reason: 'registry' }
|
|
172
253
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
return { status: 'idle', action: 'none' }
|
|
254
|
+
if (liveWorkCount(work) === 0) {
|
|
255
|
+
phase = basePlan === null ? 'idle' : 'paused'
|
|
256
|
+
return basePlan === null ? { status: 'idle', action: 'none' } : { status: 'already', phase }
|
|
177
257
|
}
|
|
178
258
|
lastError = null
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
259
|
+
const plan = basePlan === null
|
|
260
|
+
? {
|
|
261
|
+
version: 3, // ui-copy-ignore
|
|
262
|
+
planId: newPlanId('pause'),
|
|
263
|
+
status: 'pausing', // ui-copy-ignore
|
|
264
|
+
source: 'pause', // ui-copy-ignore
|
|
265
|
+
createdAt: new Date(now()).toISOString(),
|
|
266
|
+
sessions: [], // ui-copy-ignore
|
|
267
|
+
goals: [], // ui-copy-ignore
|
|
268
|
+
jobs: [], // ui-copy-ignore
|
|
269
|
+
attention: [], // ui-copy-ignore
|
|
270
|
+
}
|
|
271
|
+
: { ...basePlan, status: 'pausing', attention: Array.isArray(basePlan.attention) ? basePlan.attention : [] }
|
|
272
|
+
const before = new Map((basePlan?.sessions ?? []).filter((row) => typeof row?.sessionId === 'string').map((row) => [row.sessionId, row]))
|
|
273
|
+
const restoredBefore = Number.isSafeInteger(basePlan?.restore?.attempts) && basePlan.restore.attempts > 0
|
|
274
|
+
plan.sessions = unionSessions(plan.sessions ?? [], work.sessions.map(freshSession))
|
|
275
|
+
plan.sessions = plan.sessions.map((row) => {
|
|
276
|
+
if (row.intent !== 'continue') return row
|
|
277
|
+
const prior = before.get(row.sessionId)
|
|
278
|
+
const priorEpoch = Number.isSafeInteger(prior?.epoch) ? prior.epoch : 0
|
|
279
|
+
if (Number.isSafeInteger(row.epoch) && row.epoch > priorEpoch) return row
|
|
280
|
+
// A new pause cycle after a restore attempt needs a new request id: the previous one was
|
|
281
|
+
// already admitted, and the official prompt dedupe would silently swallow the new message.
|
|
282
|
+
// Retrying the same pause (no restore attempt yet) keeps the id, so admission stays idempotent.
|
|
283
|
+
return { ...row, epoch: restoredBefore && priorEpoch >= 1 ? priorEpoch + 1 : Math.max(priorEpoch, 1) }
|
|
284
|
+
})
|
|
285
|
+
plan.goals = unionById(plan.goals ?? [], work.goals.map(freshGoal), (goal) => `${goal?.sessionId}:${goal?.goalId}`)
|
|
286
|
+
plan.jobs = unionById(plan.jobs ?? [], work.jobs.map(freshJob), (job) => job?.id)
|
|
287
|
+
if (persist(plan) === null) {
|
|
288
|
+
// Nothing was stopped: the state is exactly what it was before the press.
|
|
289
|
+
phase = basePlan === null ? 'idle' : 'paused'
|
|
184
290
|
lastError = 'store'
|
|
185
291
|
return { status: 'failed', reason: 'store' }
|
|
186
292
|
}
|
|
293
|
+
|
|
187
294
|
let stopped = null
|
|
188
295
|
try {
|
|
189
296
|
stopped = await probe.stop(current, { graceMs })
|
|
190
297
|
} catch (error) {
|
|
191
298
|
log(`stopping work failed: ${error?.message ?? error}`)
|
|
192
|
-
stopped = { canceled: [], unknown: [], pausedGoals: [], goalErrors: [], killedJobs: [], jobErrors: [], stillRunning: [] }
|
|
299
|
+
stopped = { canceled: [], unknown: [], pausedGoals: [], goalErrors: [], killedJobs: [], jobErrors: [], stillRunning: [], sessions: [] }
|
|
300
|
+
}
|
|
301
|
+
if (isCurrent() === false) {
|
|
302
|
+
// A reload happened while this press was awaiting the stop. The new instance owns the plan
|
|
303
|
+
// now; writing here would erase whatever it captured in the meantime.
|
|
304
|
+
log('stale instance: pause result discarded')
|
|
305
|
+
return { status: 'failed', reason: 'reloaded' }
|
|
193
306
|
}
|
|
194
307
|
|
|
195
|
-
//
|
|
196
|
-
//
|
|
197
|
-
// pause is reported instead and left alone.
|
|
198
|
-
// A cancelled turn bumps the agent's turn counter, so the baseline has to be read after the
|
|
199
|
-
// stop, not before it. Without this a restore would mistake its own cancelled turn for work
|
|
200
|
-
// the person continued by hand and skip the continuation it was asked to send.
|
|
308
|
+
// A cancelled turn bumps the agent's turn counter, so the baseline is read after the stop.
|
|
309
|
+
// Without this a restore would mistake its own cancelled turn for work the person continued.
|
|
201
310
|
const baseline = new Map((stopped.sessions ?? []).map((session) => [session.sessionId, session.lastTurn]))
|
|
202
|
-
|
|
203
|
-
? { ...session, lastTurn: Number.isSafeInteger(baseline.get(session.sessionId)) ? baseline.get(session.sessionId) :
|
|
311
|
+
plan.sessions = plan.sessions.map((session) => baseline.has(session.sessionId)
|
|
312
|
+
? { ...session, lastTurn: Number.isSafeInteger(baseline.get(session.sessionId)) ? baseline.get(session.sessionId) : session.lastTurn }
|
|
204
313
|
: session)
|
|
205
|
-
|
|
314
|
+
// A goal that could not be paused is not in the plan: resuming it would bump a revision for no
|
|
315
|
+
// reason. It is reported instead, and the person can pause again after fixing whatever blocked it.
|
|
316
|
+
const pausedGoals = (stopped.pausedGoals ?? []).map((goal) => ({
|
|
206
317
|
sessionId: goal.sessionId,
|
|
207
318
|
goalId: goal.goalId,
|
|
208
319
|
revision: goal.pausedRevision,
|
|
209
320
|
}))
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
321
|
+
const keptGoals = (plan.goals ?? []).filter((goal) => !(stopped.pausedGoals ?? []).some((row) => row.sessionId === goal.sessionId && row.goalId === goal.goalId))
|
|
322
|
+
plan.goals = [...keptGoals, ...pausedGoals]
|
|
323
|
+
const stillRunning = stopped.stillRunning ?? []
|
|
324
|
+
// A session poll alone is not proof the stop was clean: a goal whose pause threw and a job whose
|
|
325
|
+
// kill threw are still live work, and reporting `paused` there hid exactly what needs a person.
|
|
326
|
+
const incomplete = stillRunning.length + (stopped.goalErrors ?? []).length + (stopped.jobErrors ?? []).length
|
|
327
|
+
plan.status = incomplete > 0 ? 'partial' : 'paused'
|
|
328
|
+
plan.stoppedAt = new Date(now()).toISOString()
|
|
329
|
+
plan.stopped = {
|
|
213
330
|
canceled: stopped.canceled ?? [],
|
|
214
331
|
unknown: stopped.unknown ?? [],
|
|
215
|
-
stillRunning
|
|
332
|
+
stillRunning,
|
|
216
333
|
goalErrors: stopped.goalErrors ?? [],
|
|
217
334
|
killedJobs: stopped.killedJobs ?? [],
|
|
218
335
|
jobErrors: stopped.jobErrors ?? [],
|
|
219
336
|
}
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
337
|
+
const errorId = (kind, detail) => {
|
|
338
|
+
const match = new RegExp(`^${kind} ([^:]+):`).exec(String(detail))
|
|
339
|
+
return match === null ? '' : match[1].trim()
|
|
340
|
+
}
|
|
341
|
+
plan.attention = [
|
|
342
|
+
...(Array.isArray(basePlan?.attention) ? basePlan.attention : []),
|
|
343
|
+
...stillRunning.map((id) => ({ kind: 'session', id, detail: 'still running' })),
|
|
344
|
+
...(stopped.goalErrors ?? []).map((detail) => {
|
|
345
|
+
const goalId = errorId('goal', detail)
|
|
346
|
+
const owner = [...(basePlan?.goals ?? []), ...(current.goals ?? [])].find((row) => row.goalId === goalId)
|
|
347
|
+
return { kind: 'goal', id: owner?.sessionId ?? goalId, detail }
|
|
348
|
+
}),
|
|
349
|
+
...(stopped.jobErrors ?? []).map((detail) => ({ kind: 'job', id: errorId('job', detail), detail })),
|
|
224
350
|
]
|
|
225
|
-
if (
|
|
226
|
-
// The
|
|
351
|
+
if (isCrashPlan(basePlan) === true) {
|
|
352
|
+
// The crash record described work that was not paused. This press did pause it, so the plan
|
|
353
|
+
// becomes a normal pause plan and the crash evidence is no longer a separate obligation.
|
|
354
|
+
plan.source = 'pause' // ui-copy-ignore
|
|
355
|
+
if (store.write(plan) === null) {
|
|
356
|
+
const kept = store.writePending(plan)
|
|
357
|
+
if (kept === null) {
|
|
358
|
+
crashFallback = plan
|
|
359
|
+
lastError = 'store'
|
|
360
|
+
}
|
|
361
|
+
} else if (typeof store.clearPending === 'function') {
|
|
362
|
+
if (store.clearPending() === false) lastError = 'store'
|
|
363
|
+
}
|
|
364
|
+
crashFallback = null
|
|
365
|
+
} else if (store.write(plan) === null) {
|
|
366
|
+
// The durable copy still says `pausing` or `paused`: the next boot reports it instead of
|
|
367
|
+
// claiming a stop that was never written.
|
|
227
368
|
lastError = 'store'
|
|
228
369
|
}
|
|
229
370
|
phase = 'paused'
|
|
230
|
-
return { status: 'paused', counts: countsOf(
|
|
371
|
+
return { status: plan.status === 'partial' ? 'partial' : 'paused', counts: countsOf(plan), outstanding: outstandingOf(plan) }
|
|
231
372
|
}
|
|
232
373
|
|
|
233
|
-
function summaryOf() {
|
|
234
|
-
return { restored: [], skipped: [], failed: [], attention: [], jobs: 0, at: new Date(now()).toISOString() }
|
|
374
|
+
function summaryOf(plan) {
|
|
375
|
+
return { planId: plan.planId, source: plan.source ?? 'pause', restored: [], skipped: [], failed: [], attention: [], jobs: 0, at: new Date(now()).toISOString() }
|
|
235
376
|
}
|
|
236
377
|
|
|
237
378
|
/**
|
|
238
|
-
* One restore pass.
|
|
239
|
-
*
|
|
240
|
-
*
|
|
379
|
+
* One restore pass. Items are not skipped from the journal: the live Harness decides whether a
|
|
380
|
+
* session is already back, which is the only answer that survives a second process, a manual
|
|
381
|
+
* resume, or a reload. A retryable failure keeps the plan; attention outcomes are terminal.
|
|
241
382
|
*/
|
|
242
383
|
async function runResume(plan) {
|
|
243
384
|
lastError = null
|
|
244
|
-
// A crash in the middle of a pause left a plan that never finished stopping.
|
|
245
|
-
//
|
|
246
|
-
// person asked for silence, not another round.
|
|
385
|
+
// A crash in the middle of a pause left a plan that never finished stopping. Mounting is safe;
|
|
386
|
+
// continuing is not, because the person asked for silence and the turn may have died mid-way.
|
|
247
387
|
if (plan.status === 'pausing') {
|
|
248
|
-
plan.source = 'interrupted'
|
|
388
|
+
plan.source = plan.source === 'crash' ? 'crash' : 'interrupted'
|
|
249
389
|
plan.attention = [...(plan.attention ?? []), { kind: 'pause', id: '', detail: 'pause interrupted' }]
|
|
250
390
|
plan.sessions = (plan.sessions ?? []).map((session) => ({ ...session, running: false, lastTurn: null }))
|
|
251
391
|
plan.status = 'paused'
|
|
392
|
+
if (persist(plan) === null) {
|
|
393
|
+
phase = 'paused'
|
|
394
|
+
lastError = 'store'
|
|
395
|
+
return { status: 'partial', summary: null, outstanding: outstandingOf(plan) }
|
|
396
|
+
}
|
|
252
397
|
}
|
|
253
|
-
const summary = summaryOf()
|
|
398
|
+
const summary = summaryOf(plan)
|
|
254
399
|
summary.jobs = countsOf(plan).jobs
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
400
|
+
const record = (key, status) => {
|
|
401
|
+
if (store.appendJournal(plan.planId, key, status, bootId) === false) {
|
|
402
|
+
// Reporting only: the restore already happened, so this never blocks the plan. It does mean
|
|
403
|
+
// a later boot counts the item as outstanding and checks it again, which is safe.
|
|
404
|
+
log(`cannot journal ${key}`)
|
|
405
|
+
}
|
|
259
406
|
}
|
|
260
407
|
const step = async (key, label, run) => {
|
|
261
|
-
if (done.has(key)) return
|
|
262
408
|
let outcome
|
|
263
409
|
try {
|
|
264
410
|
outcome = await run()
|
|
@@ -266,17 +412,15 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
266
412
|
outcome = { status: 'failed', detail: String(error?.message ?? error).slice(0, 120) }
|
|
267
413
|
}
|
|
268
414
|
const status = typeof outcome?.status === 'string' ? outcome.status : 'failed'
|
|
269
|
-
if (TERMINAL.has(status)) {
|
|
270
|
-
record(key)
|
|
271
|
-
if (status === 'restored') summary.restored.push(label)
|
|
272
|
-
else if (status === 'skipped') summary.skipped.push(label)
|
|
273
|
-
else {
|
|
274
|
-
summary.attention.push(`${label}: ${outcome?.detail ?? 'needs a look'}`)
|
|
275
|
-
plan.attention = [...(plan.attention ?? []), { kind: 'session', id: label, detail: String(outcome?.detail ?? '') }]
|
|
276
|
-
}
|
|
277
|
-
} else {
|
|
415
|
+
if (TERMINAL.has(status) === false) {
|
|
278
416
|
summary.failed.push(`${label}: ${outcome?.detail ?? 'unknown'}`)
|
|
417
|
+
return outcome
|
|
279
418
|
}
|
|
419
|
+
record(key, status)
|
|
420
|
+
if (status === 'restored') summary.restored.push(label)
|
|
421
|
+
else if (status === 'skipped') summary.skipped.push(label)
|
|
422
|
+
else summary.attention.push(`${label}: ${outcome?.detail ?? 'needs a look'}`)
|
|
423
|
+
return outcome
|
|
280
424
|
}
|
|
281
425
|
|
|
282
426
|
for (const session of plan.sessions ?? []) {
|
|
@@ -284,37 +428,58 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
284
428
|
}
|
|
285
429
|
for (const goal of plan.goals ?? []) {
|
|
286
430
|
const key = `g:${goal.sessionId}:${goal.goalId ?? ''}`
|
|
287
|
-
await step(key, `goal ${goal.goalId ?? goal.sessionId}`, () => probe.restoreGoal(goal, { plan }))
|
|
431
|
+
const outcome = await step(key, `goal ${goal.goalId ?? goal.sessionId}`, () => probe.restoreGoal(goal, { plan }))
|
|
432
|
+
if (outcome?.status === 'restored' && Number.isSafeInteger(outcome.revision)) {
|
|
433
|
+
// Remember what this plan armed. A restarted process sees `active + disarmed`, and that
|
|
434
|
+
// exact revision is the only proof the goal is ours to arm again rather than someone else's.
|
|
435
|
+
plan.goals = (plan.goals ?? []).map((row) => (row.sessionId === goal.sessionId && row.goalId === goal.goalId
|
|
436
|
+
? { ...row, armedRevision: outcome.revision }
|
|
437
|
+
: row))
|
|
438
|
+
if (persist(plan) === null) lastError = 'store'
|
|
439
|
+
}
|
|
288
440
|
}
|
|
289
441
|
// A killed job cannot be replayed: the same command may have had an unknown effect, so it is
|
|
290
|
-
// reported once and left to the person. It
|
|
442
|
+
// reported once and left to the person. It never blocks the plan and is never restarted.
|
|
291
443
|
for (const job of plan.jobs ?? []) summary.attention.push(`job ${job.id}: stopped, not restarted`)
|
|
292
444
|
|
|
293
445
|
result = summary
|
|
294
|
-
|
|
295
|
-
const pending = itemKeys(plan).filter((key) => !done.has(key))
|
|
296
|
-
if (failed || pending.length > 0) {
|
|
446
|
+
if (summary.failed.length > 0) {
|
|
297
447
|
plan.restore = { at: summary.at, attempts: (plan.restore?.attempts ?? 0) + 1, failed: summary.failed } // ui-copy-ignore
|
|
298
|
-
if (
|
|
448
|
+
if (persist(plan) === null) lastError = 'store'
|
|
299
449
|
else lastError = 'restore-partial'
|
|
300
450
|
phase = 'paused'
|
|
301
|
-
return { status: 'partial', summary, outstanding:
|
|
451
|
+
return { status: 'partial', summary, outstanding: outstandingOf(plan) }
|
|
302
452
|
}
|
|
303
453
|
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
454
|
+
if (isCrashPlan(plan) === true) {
|
|
455
|
+
if (typeof store.writeResult === 'function' && store.writeResult({ ...plan, status: 'restored', restoredAt: summary.at, summary }) === false) {
|
|
456
|
+
lastError = 'store'
|
|
457
|
+
phase = 'paused'
|
|
458
|
+
return { status: 'partial', summary, outstanding: 0 }
|
|
459
|
+
}
|
|
460
|
+
if (typeof store.clearPending !== 'function' || store.clearPending() === false) {
|
|
461
|
+
lastError = 'store'
|
|
462
|
+
phase = 'paused'
|
|
463
|
+
return { status: 'partial', summary, outstanding: 0 }
|
|
464
|
+
}
|
|
465
|
+
crashFallback = null
|
|
466
|
+
if (typeof store.clearJournal === 'function') store.clearJournal(plan.planId)
|
|
467
|
+
if (isCurrent() === false) {
|
|
468
|
+
log('stale instance: crash record left alone')
|
|
469
|
+
return { status: 'partial', summary, outstanding: 0 }
|
|
470
|
+
}
|
|
313
471
|
try {
|
|
314
472
|
heartbeat?.release?.()
|
|
315
473
|
} catch (error) {
|
|
316
474
|
log(`cannot release crash record: ${error?.message ?? error}`)
|
|
317
475
|
}
|
|
476
|
+
} else {
|
|
477
|
+
const consumed = store.consume(plan, summary)
|
|
478
|
+
if (!consumed?.ok) {
|
|
479
|
+
lastError = 'store'
|
|
480
|
+
phase = 'paused'
|
|
481
|
+
return { status: 'partial', summary, outstanding: 0 }
|
|
482
|
+
}
|
|
318
483
|
}
|
|
319
484
|
phase = 'idle'
|
|
320
485
|
return { status: 'resumed', summary }
|
|
@@ -329,7 +494,7 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
329
494
|
} catch (error) {
|
|
330
495
|
log(`${kind} failed: ${error?.message ?? error}`)
|
|
331
496
|
lastError = 'internal'
|
|
332
|
-
phase =
|
|
497
|
+
phase = currentPlan() === null ? 'idle' : 'paused'
|
|
333
498
|
return { status: 'failed', reason: 'internal' }
|
|
334
499
|
} finally {
|
|
335
500
|
inflight = null
|
|
@@ -341,11 +506,15 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
341
506
|
|
|
342
507
|
async function pause() {
|
|
343
508
|
if (TRANSIENT.has(phase) || inflight !== null) return { status: 'busy', phase }
|
|
344
|
-
|
|
509
|
+
const plan = currentPlan()
|
|
510
|
+
if (plan !== null) {
|
|
511
|
+
const work = workOf(liveNow())
|
|
512
|
+
if (liveWorkCount(work) === 0) return { status: 'already', phase }
|
|
513
|
+
}
|
|
345
514
|
// Set synchronously, before the first await: two windows pressing at once are then exclusive
|
|
346
515
|
// in the same tick they were dispatched, not one microtask later.
|
|
347
516
|
phase = 'pausing'
|
|
348
|
-
return once('pause', runPause)
|
|
517
|
+
return once('pause', () => runPause(plan))
|
|
349
518
|
}
|
|
350
519
|
|
|
351
520
|
async function resume() {
|
|
@@ -361,13 +530,81 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
361
530
|
})
|
|
362
531
|
}
|
|
363
532
|
|
|
364
|
-
/**
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
533
|
+
/**
|
|
534
|
+
* A reload or restart must not invent a phase: the disk and the running Harness decide it. The
|
|
535
|
+
* crash residue is handed in by the caller, because it has to be read before the heartbeat
|
|
536
|
+
* overwrites the previous process's file; it is persisted here so a second restart still sees it.
|
|
537
|
+
*/
|
|
538
|
+
async function hydrate(options = {}) {
|
|
539
|
+
crashFallback = null
|
|
540
|
+
lastError = null
|
|
541
|
+
const residue = options?.residue ?? heartbeat?.residue ?? null
|
|
542
|
+
const restored = typeof store.lastRestore === 'function' ? store.lastRestore() : null
|
|
543
|
+
result = restored !== null && typeof restored === 'object' ? (restored.summary ?? null) : null
|
|
544
|
+
let durable = true
|
|
545
|
+
|
|
546
|
+
if (residue !== null && residue !== undefined) {
|
|
547
|
+
let crash = null
|
|
548
|
+
try {
|
|
549
|
+
crash = probe.crashPlan(residue, {
|
|
550
|
+
planId: `crash-${Number.isFinite(residue.at) ? residue.at : 0}-${Array.isArray(residue.sessions) ? residue.sessions.length : 0}`,
|
|
551
|
+
})
|
|
552
|
+
} catch (error) {
|
|
553
|
+
log(`cannot derive crash plan: ${error?.message ?? error}`)
|
|
554
|
+
}
|
|
555
|
+
if (crash !== null) {
|
|
556
|
+
const existing = pausedPlan() ?? crashPlan()
|
|
557
|
+
if (existing === null) {
|
|
558
|
+
const stored = typeof store.writePending === 'function' ? store.writePending(crash) : null
|
|
559
|
+
// A failed write stays in memory for this process, but the caller must not let the
|
|
560
|
+
// heartbeat overwrite the file the next process would have read the same residue from.
|
|
561
|
+
if (stored === null) {
|
|
562
|
+
crashFallback = crash
|
|
563
|
+
durable = false
|
|
564
|
+
lastError = 'store'
|
|
565
|
+
heartbeat?.hold?.()
|
|
566
|
+
log('cannot record the crash plan; keeping the previous heartbeat')
|
|
567
|
+
} else {
|
|
568
|
+
log('crash plan recorded')
|
|
569
|
+
}
|
|
570
|
+
} else {
|
|
571
|
+
// A plan from an earlier pause outranks the crash record, but it must not hide it: every
|
|
572
|
+
// session the crash residue names is merged in as mount-only work. Otherwise a crash that
|
|
573
|
+
// happened after a partial restore would silently drop sessions the next press captured.
|
|
574
|
+
const known = new Set((existing.sessions ?? []).map((row) => row?.sessionId))
|
|
575
|
+
const added = crash.sessions
|
|
576
|
+
.filter((row) => known.has(row.sessionId) === false)
|
|
577
|
+
.map((row) => ({ sessionId: row.sessionId, intent: 'mount', epoch: 0, running: false, lastTurn: null, crash: true }))
|
|
578
|
+
if (added.length > 0) {
|
|
579
|
+
const plan = {
|
|
580
|
+
...existing,
|
|
581
|
+
sessions: [...(existing.sessions ?? []), ...added],
|
|
582
|
+
crashes: (Number.isSafeInteger(existing.crashes) ? existing.crashes : 0) + added.length,
|
|
583
|
+
attention: [
|
|
584
|
+
...(Array.isArray(existing.attention) ? existing.attention : []),
|
|
585
|
+
...added.map((row) => ({ kind: 'session', id: row.sessionId, detail: 'crashed: mounted only' })),
|
|
586
|
+
],
|
|
587
|
+
}
|
|
588
|
+
const isCrash = isCrashPlan(plan)
|
|
589
|
+
const stored = isCrash
|
|
590
|
+
? (typeof store.writePending === 'function' ? store.writePending(plan) : null)
|
|
591
|
+
: store.write(plan)
|
|
592
|
+
if (stored === null) {
|
|
593
|
+
durable = false
|
|
594
|
+
lastError = 'store'
|
|
595
|
+
if (isCrash) crashFallback = plan
|
|
596
|
+
heartbeat?.hold?.()
|
|
597
|
+
log('cannot merge the crash plan; keeping the previous heartbeat')
|
|
598
|
+
} else {
|
|
599
|
+
log(`crash sessions merged (${added.length})`)
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
phase = pausedPlan() !== null ? 'paused' : 'idle'
|
|
607
|
+
return { phase, durable }
|
|
371
608
|
}
|
|
372
609
|
|
|
373
610
|
return {
|
|
@@ -376,7 +613,7 @@ export function createEngine({ store, probe, heartbeat = null, now = () => Date.
|
|
|
376
613
|
resume,
|
|
377
614
|
hydrate,
|
|
378
615
|
get phase() { return phase },
|
|
379
|
-
get pendingPlan() { return
|
|
380
|
-
get crashPlan() { return
|
|
616
|
+
get pendingPlan() { return currentPlan },
|
|
617
|
+
get crashPlan() { return crashPlan },
|
|
381
618
|
}
|
|
382
619
|
}
|