dsh-pause-resume 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -35
- package/client.js +114 -29
- package/index.js +105 -32
- package/lib/engine.js +380 -94
- package/lib/heartbeat.js +200 -50
- package/lib/probe.js +284 -116
- package/lib/snapshot.js +270 -83
- package/package.json +2 -2
package/lib/engine.js
CHANGED
|
@@ -1,169 +1,419 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The only place the button's shape comes from.
|
|
3
3
|
*
|
|
4
|
-
* One phase value, written before any work starts and after it ends, plus a durable
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* One phase value, written before any work starts and after it ends, plus a durable plan. The two
|
|
5
|
+
* transient phases are exclusive by construction: `pause()` and `resume()` refuse while either is
|
|
6
|
+
* set, so two windows, a double press and a tool call can never overlap. The client is not allowed
|
|
7
|
+
* to infer a phase: it asks, and renders the answer.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
* `action` is state-derived, never phase-derived. Live work asks for a pause; a plan with no live
|
|
10
|
+
* work asks for a resume; neither is `none`. That single rule is what makes a second press useful
|
|
11
|
+
* after a failed pause, a partial stop, or a restart: the button does what the Harness still needs,
|
|
12
|
+
* not what the last operation intended.
|
|
13
|
+
*
|
|
14
|
+
* A plan is kept until every item in it has a terminal outcome or a retryable failure. Attention
|
|
15
|
+
* outcomes (a continuation that was admitted but never opened a turn, a job that cannot be
|
|
16
|
+
* replayed) are terminal: they are carried into the last-restore record and shown as a dot, but
|
|
17
|
+
* they never cause the same message to be delivered twice. The crash plan is written to disk
|
|
18
|
+
* before the heartbeat overwrites the previous process's record, so a second or third restart
|
|
19
|
+
* still sees the same pending work.
|
|
11
20
|
*/
|
|
12
21
|
|
|
13
22
|
const TRANSIENT = new Set(['pausing', 'resuming'])
|
|
23
|
+
const TERMINAL = new Set(['restored', 'skipped', 'attention'])
|
|
24
|
+
|
|
25
|
+
function emptyLive() {
|
|
26
|
+
return { sessions: [], goals: [], jobs: [], readable: false, source: 'none' }
|
|
27
|
+
}
|
|
14
28
|
|
|
15
|
-
export function createEngine({ store, probe, now = () => Date.now(), graceMs = 2000,
|
|
29
|
+
export function createEngine({ store, probe, heartbeat = null, bootId = 'boot', now = () => Date.now(), graceMs = 2000, log = () => {} }) {
|
|
16
30
|
let phase = 'idle'
|
|
17
31
|
let lastError = null
|
|
18
32
|
let result = null
|
|
19
33
|
/** Set while a press runs, so a second caller joins the answer instead of starting work again. */
|
|
20
34
|
let inflight = null
|
|
35
|
+
/** Crash plan kept in memory only when its durable write failed; a reload loses it, the file does not. */
|
|
36
|
+
let crashFallback = null
|
|
37
|
+
let counter = 0
|
|
21
38
|
|
|
22
|
-
|
|
39
|
+
function newPlanId(kind) {
|
|
40
|
+
counter += 1
|
|
41
|
+
return `${kind}-${now().toString(36)}-${counter.toString(36)}`
|
|
42
|
+
}
|
|
23
43
|
|
|
24
|
-
function
|
|
25
|
-
|
|
26
|
-
// `pausing` is included on purpose: it means the last press stopped halfway, so the work in it
|
|
27
|
-
// still has to be restored. Reporting it as nothing to restore would hide a half-done pause.
|
|
28
|
-
return snapshot
|
|
44
|
+
function pausedPlan() {
|
|
45
|
+
return store.read()
|
|
29
46
|
}
|
|
30
47
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
return { ...emptyLive, ...(value ?? {}) }
|
|
35
|
-
} catch (error) {
|
|
36
|
-
log(`cannot read live work: ${error?.message ?? error}`)
|
|
37
|
-
return emptyLive
|
|
38
|
-
}
|
|
48
|
+
function crashPlan() {
|
|
49
|
+
const stored = typeof store.readPending === 'function' ? store.readPending() : null
|
|
50
|
+
return stored ?? crashFallback
|
|
39
51
|
}
|
|
40
52
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
53
|
+
/** Exactly one plan is pending, and a finished pause outranks a leftover crash record. */
|
|
54
|
+
function currentPlan() {
|
|
55
|
+
return pausedPlan() ?? crashPlan()
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function isCrashPlan(plan) {
|
|
59
|
+
return plan !== null && plan !== undefined && plan.source === 'crash'
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function itemKeys(plan) {
|
|
63
|
+
if (plan === null || plan === undefined) return []
|
|
64
|
+
const keys = []
|
|
65
|
+
for (const session of plan.sessions ?? []) if (typeof session?.sessionId === 'string') keys.push(`s:${session.sessionId}`)
|
|
66
|
+
for (const goal of plan.goals ?? []) keys.push(`g:${goal.sessionId}:${goal.goalId ?? ''}`)
|
|
67
|
+
return keys
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Rows written by an earlier process are evidence, not permission to skip: a new process must
|
|
72
|
+
* check the live Harness again. Only this boot's rows reduce what is still outstanding.
|
|
73
|
+
*/
|
|
74
|
+
function outstandingOf(plan) {
|
|
75
|
+
if (plan === null || plan === undefined) return 0
|
|
76
|
+
const rows = store.readJournal(plan.planId)
|
|
77
|
+
return itemKeys(plan).filter((key) => {
|
|
78
|
+
const row = rows.get(key)
|
|
79
|
+
return row === undefined || (row.boot !== null && row.boot !== bootId)
|
|
80
|
+
}).length
|
|
49
81
|
}
|
|
50
82
|
|
|
51
83
|
function countsOf(plan) {
|
|
52
|
-
if (plan === null || plan === undefined) return { sessions: 0, goals: 0,
|
|
84
|
+
if (plan === null || plan === undefined) return { sessions: 0, goals: 0, jobs: 0 }
|
|
53
85
|
return {
|
|
54
86
|
sessions: Array.isArray(plan.sessions) ? plan.sessions.length : 0,
|
|
55
87
|
goals: Array.isArray(plan.goals) ? plan.goals.length : 0,
|
|
56
|
-
schedules: Array.isArray(plan.schedules) ? plan.schedules.length : 0,
|
|
57
88
|
jobs: Array.isArray(plan.jobs) ? plan.jobs.length : 0,
|
|
58
89
|
}
|
|
59
90
|
}
|
|
60
91
|
|
|
92
|
+
function liveNow() {
|
|
93
|
+
try {
|
|
94
|
+
return probe.live() ?? emptyLive()
|
|
95
|
+
} catch (error) {
|
|
96
|
+
log(`cannot read live work: ${error?.message ?? error}`)
|
|
97
|
+
return emptyLive()
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Only work that was actually stopped goes into the plan: a session whose turn was open, or a
|
|
103
|
+
* session that carries an armed goal. A session that ended normally is left alone, however it
|
|
104
|
+
* happens to look in the session list.
|
|
105
|
+
*/
|
|
106
|
+
function workOf(current) {
|
|
107
|
+
const sessions = new Map()
|
|
108
|
+
for (const session of current.sessions ?? []) {
|
|
109
|
+
if (session.running === true) sessions.set(session.sessionId, session)
|
|
110
|
+
}
|
|
111
|
+
for (const goal of current.goals ?? []) {
|
|
112
|
+
if (!sessions.has(goal.sessionId)) sessions.set(goal.sessionId, { sessionId: goal.sessionId, running: false, lastTurn: null })
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
sessions: [...sessions.values()],
|
|
116
|
+
goals: current.goals ?? [],
|
|
117
|
+
jobs: current.jobs ?? [], // ui-copy-ignore
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function liveWorkCount(work) {
|
|
122
|
+
return work.sessions.length + work.goals.length + work.jobs.length
|
|
123
|
+
}
|
|
124
|
+
|
|
61
125
|
/**
|
|
62
126
|
* What the button shows. `action` is the one decision the client may not make itself:
|
|
63
127
|
* `none` (nothing to do), `pause`, `resume`, or `busy` (an operation is running).
|
|
64
128
|
*/
|
|
65
|
-
|
|
66
|
-
const
|
|
67
|
-
const current =
|
|
68
|
-
const
|
|
129
|
+
function describe() {
|
|
130
|
+
const plan = currentPlan()
|
|
131
|
+
const current = liveNow()
|
|
132
|
+
const work = workOf(current)
|
|
69
133
|
const busy = TRANSIENT.has(phase)
|
|
70
|
-
const
|
|
71
|
-
|
|
134
|
+
const workCount = liveWorkCount(work)
|
|
135
|
+
let action = 'none'
|
|
136
|
+
if (busy) action = 'busy'
|
|
137
|
+
else if (current.readable === true && workCount > 0) action = 'pause'
|
|
138
|
+
else if (plan !== null) action = 'resume'
|
|
139
|
+
const attention = plan !== null
|
|
140
|
+
? (Array.isArray(plan.attention) ? plan.attention.length : 0)
|
|
141
|
+
: (Array.isArray(result?.attention) ? result.attention.length : 0)
|
|
72
142
|
return {
|
|
73
143
|
phase,
|
|
74
144
|
action,
|
|
75
|
-
source: plan === null ? null : plan.
|
|
76
|
-
complete
|
|
145
|
+
source: plan === null ? null : plan.source ?? 'pause',
|
|
146
|
+
// `complete` means the stop finished and nothing is still turning. A partial stop is not.
|
|
147
|
+
complete: plan === null ? null : plan.status !== 'partial' && plan.status !== 'pausing' && lastError !== 'store',
|
|
148
|
+
outstanding: outstandingOf(plan), // ui-copy-ignore
|
|
77
149
|
counts: countsOf(plan), // ui-copy-ignore
|
|
78
150
|
live: { // ui-copy-ignore
|
|
79
|
-
sessions:
|
|
151
|
+
sessions: work.sessions.filter((session) => session.running === true).length,
|
|
80
152
|
goals: current.goals.length,
|
|
81
153
|
jobs: current.jobs.length,
|
|
82
154
|
},
|
|
83
|
-
|
|
155
|
+
attention,
|
|
156
|
+
error: lastError ?? (plan?.status === 'pausing' ? 'pause-interrupted' : plan?.status === 'partial' ? 'pause-partial' : null), // ui-copy-ignore
|
|
84
157
|
result,
|
|
85
158
|
}
|
|
86
159
|
}
|
|
87
160
|
|
|
88
|
-
function
|
|
161
|
+
function freshSession(session) {
|
|
89
162
|
return {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
sessions: current.sessions, // ui-copy-ignore
|
|
94
|
-
goals: current.goals, // ui-copy-ignore
|
|
95
|
-
schedules: current.schedules,
|
|
96
|
-
jobs: current.jobs, // ui-copy-ignore
|
|
97
|
-
terminals: current.terminals,
|
|
163
|
+
sessionId: session.sessionId,
|
|
164
|
+
running: session.running === true,
|
|
165
|
+
lastTurn: Number.isSafeInteger(session.lastTurn) ? session.lastTurn : null,
|
|
98
166
|
}
|
|
99
167
|
}
|
|
100
168
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
169
|
+
function freshGoal(goal) {
|
|
170
|
+
return { sessionId: goal.sessionId, goalId: goal.goalId, revision: goal.revision }
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function freshJob(job) {
|
|
174
|
+
return { id: job.id, owner: job.owner ?? null, status: job.status ?? null }
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function unionSessions(base, additions) {
|
|
178
|
+
const out = []
|
|
179
|
+
const index = new Map()
|
|
180
|
+
for (const row of [...base, ...additions]) {
|
|
181
|
+
if (typeof row?.sessionId !== 'string') continue
|
|
182
|
+
const existing = index.get(row.sessionId)
|
|
183
|
+
if (existing !== undefined) {
|
|
184
|
+
// Stopping again refreshes the baseline; a plain union would keep a stale turn number.
|
|
185
|
+
if (Number.isSafeInteger(row.lastTurn)) existing.lastTurn = row.lastTurn
|
|
186
|
+
if (row.running === false) existing.running = false
|
|
187
|
+
continue
|
|
188
|
+
}
|
|
189
|
+
const copy = { ...row }
|
|
190
|
+
index.set(copy.sessionId, copy)
|
|
191
|
+
out.push(copy)
|
|
192
|
+
}
|
|
193
|
+
return out
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function unionById(base, additions, idOf) {
|
|
197
|
+
const out = []
|
|
198
|
+
const seen = new Set()
|
|
199
|
+
for (const row of [...base, ...additions]) {
|
|
200
|
+
const id = idOf(row)
|
|
201
|
+
if (typeof id !== 'string' || id === '' || seen.has(id)) continue
|
|
202
|
+
seen.add(id)
|
|
203
|
+
out.push(row)
|
|
204
|
+
}
|
|
205
|
+
return out
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function persist(plan) {
|
|
209
|
+
if (isCrashPlan(plan) === true) {
|
|
210
|
+
const stored = store.writePending(plan)
|
|
211
|
+
if (stored !== null) crashFallback = null
|
|
212
|
+
return stored
|
|
213
|
+
}
|
|
214
|
+
return store.write(plan)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Pause, including the second attempt after a partial stop. Capture happens before anything is
|
|
219
|
+
* stopped, and the plan is durable before the first cancellation, so a crash in between is
|
|
220
|
+
* visible as an interrupted pause instead of a clean one.
|
|
221
|
+
*/
|
|
222
|
+
async function runPause(basePlan) {
|
|
223
|
+
const current = liveNow()
|
|
224
|
+
const work = workOf(current)
|
|
225
|
+
if (current.readable !== true) {
|
|
226
|
+
phase = basePlan === null ? 'idle' : 'paused'
|
|
227
|
+
lastError = 'registry'
|
|
228
|
+
return { status: 'failed', reason: 'registry' }
|
|
229
|
+
}
|
|
230
|
+
if (liveWorkCount(work) === 0) {
|
|
231
|
+
phase = basePlan === null ? 'idle' : 'paused'
|
|
232
|
+
return basePlan === null ? { status: 'idle', action: 'none' } : { status: 'already', phase }
|
|
106
233
|
}
|
|
107
234
|
lastError = null
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
235
|
+
const plan = basePlan === null
|
|
236
|
+
? {
|
|
237
|
+
version: 3, // ui-copy-ignore
|
|
238
|
+
planId: newPlanId('pause'),
|
|
239
|
+
status: 'pausing', // ui-copy-ignore
|
|
240
|
+
source: 'pause', // ui-copy-ignore
|
|
241
|
+
createdAt: new Date(now()).toISOString(),
|
|
242
|
+
sessions: [], // ui-copy-ignore
|
|
243
|
+
goals: [], // ui-copy-ignore
|
|
244
|
+
jobs: [], // ui-copy-ignore
|
|
245
|
+
attention: [], // ui-copy-ignore
|
|
246
|
+
}
|
|
247
|
+
: { ...basePlan, status: 'pausing', attention: Array.isArray(basePlan.attention) ? basePlan.attention : [] }
|
|
248
|
+
plan.sessions = unionSessions(plan.sessions ?? [], work.sessions.map(freshSession))
|
|
249
|
+
plan.goals = unionById(plan.goals ?? [], work.goals.map(freshGoal), (goal) => `${goal?.sessionId}:${goal?.goalId}`)
|
|
250
|
+
plan.jobs = unionById(plan.jobs ?? [], work.jobs.map(freshJob), (job) => job?.id)
|
|
251
|
+
if (persist(plan) === null) {
|
|
252
|
+
// Nothing was stopped: the state is exactly what it was before the press.
|
|
253
|
+
phase = basePlan === null ? 'idle' : 'paused'
|
|
113
254
|
lastError = 'store'
|
|
114
255
|
return { status: 'failed', reason: 'store' }
|
|
115
256
|
}
|
|
257
|
+
|
|
116
258
|
let stopped = null
|
|
117
259
|
try {
|
|
118
260
|
stopped = await probe.stop(current, { graceMs })
|
|
119
261
|
} catch (error) {
|
|
120
262
|
log(`stopping work failed: ${error?.message ?? error}`)
|
|
121
|
-
stopped = { canceled: [], unknown: [] }
|
|
263
|
+
stopped = { canceled: [], unknown: [], pausedGoals: [], goalErrors: [], killedJobs: [], jobErrors: [], stillRunning: [], sessions: [] }
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// A cancelled turn bumps the agent's turn counter, so the baseline is read after the stop.
|
|
267
|
+
// Without this a restore would mistake its own cancelled turn for work the person continued.
|
|
268
|
+
const baseline = new Map((stopped.sessions ?? []).map((session) => [session.sessionId, session.lastTurn]))
|
|
269
|
+
plan.sessions = plan.sessions.map((session) => baseline.has(session.sessionId)
|
|
270
|
+
? { ...session, lastTurn: Number.isSafeInteger(baseline.get(session.sessionId)) ? baseline.get(session.sessionId) : session.lastTurn }
|
|
271
|
+
: session)
|
|
272
|
+
// A goal that could not be paused is not in the plan: resuming it would bump a revision for no
|
|
273
|
+
// reason. It is reported instead, and the person can pause again after fixing whatever blocked it.
|
|
274
|
+
const pausedGoals = (stopped.pausedGoals ?? []).map((goal) => ({
|
|
275
|
+
sessionId: goal.sessionId,
|
|
276
|
+
goalId: goal.goalId,
|
|
277
|
+
revision: goal.pausedRevision,
|
|
278
|
+
}))
|
|
279
|
+
const keptGoals = (plan.goals ?? []).filter((goal) => !(stopped.pausedGoals ?? []).some((row) => row.sessionId === goal.sessionId && row.goalId === goal.goalId))
|
|
280
|
+
plan.goals = [...keptGoals, ...pausedGoals]
|
|
281
|
+
const stillRunning = stopped.stillRunning ?? []
|
|
282
|
+
plan.status = stillRunning.length > 0 ? 'partial' : 'paused'
|
|
283
|
+
plan.stoppedAt = new Date(now()).toISOString()
|
|
284
|
+
plan.stopped = {
|
|
285
|
+
canceled: stopped.canceled ?? [],
|
|
286
|
+
unknown: stopped.unknown ?? [],
|
|
287
|
+
stillRunning,
|
|
288
|
+
goalErrors: stopped.goalErrors ?? [],
|
|
289
|
+
killedJobs: stopped.killedJobs ?? [],
|
|
290
|
+
jobErrors: stopped.jobErrors ?? [],
|
|
122
291
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
292
|
+
plan.attention = [
|
|
293
|
+
...(Array.isArray(basePlan?.attention) ? basePlan.attention : []),
|
|
294
|
+
...stillRunning.map((id) => ({ kind: 'session', id, detail: 'still running' })),
|
|
295
|
+
...(stopped.goalErrors ?? []).map((detail) => ({ kind: 'goal', id: '', detail })),
|
|
296
|
+
...(stopped.jobErrors ?? []).map((detail) => ({ kind: 'job', id: '', detail })),
|
|
297
|
+
]
|
|
298
|
+
if (isCrashPlan(basePlan) === true) {
|
|
299
|
+
// The crash record described work that was not paused. This press did pause it, so the plan
|
|
300
|
+
// becomes a normal pause plan and the crash evidence is no longer a separate obligation.
|
|
301
|
+
plan.source = 'pause' // ui-copy-ignore
|
|
302
|
+
if (store.write(plan) === null) {
|
|
303
|
+
const kept = store.writePending(plan)
|
|
304
|
+
if (kept === null) {
|
|
305
|
+
crashFallback = plan
|
|
306
|
+
lastError = 'store'
|
|
307
|
+
}
|
|
308
|
+
} else if (typeof store.clearPending === 'function') {
|
|
309
|
+
if (store.clearPending() === false) lastError = 'store'
|
|
310
|
+
}
|
|
311
|
+
crashFallback = null
|
|
312
|
+
} else if (store.write(plan) === null) {
|
|
313
|
+
// The durable copy still says `pausing` or `paused`: the next boot reports it instead of
|
|
314
|
+
// claiming a stop that was never written.
|
|
128
315
|
lastError = 'store'
|
|
129
316
|
}
|
|
130
317
|
phase = 'paused'
|
|
131
|
-
|
|
132
|
-
|
|
318
|
+
return { status: plan.status === 'partial' ? 'partial' : 'paused', counts: countsOf(plan), outstanding: outstandingOf(plan) }
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function summaryOf(plan) {
|
|
322
|
+
return { planId: plan.planId, source: plan.source ?? 'pause', restored: [], skipped: [], failed: [], attention: [], jobs: 0, at: new Date(now()).toISOString() }
|
|
133
323
|
}
|
|
134
324
|
|
|
325
|
+
/**
|
|
326
|
+
* One restore pass. Items are not skipped from the journal: the live Harness decides whether a
|
|
327
|
+
* session is already back, which is the only answer that survives a second process, a manual
|
|
328
|
+
* resume, or a reload. A retryable failure keeps the plan; attention outcomes are terminal.
|
|
329
|
+
*/
|
|
135
330
|
async function runResume(plan) {
|
|
136
331
|
lastError = null
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
332
|
+
// A crash in the middle of a pause left a plan that never finished stopping. Mounting is safe;
|
|
333
|
+
// continuing is not, because the person asked for silence and the turn may have died mid-way.
|
|
334
|
+
if (plan.status === 'pausing') {
|
|
335
|
+
plan.source = plan.source === 'crash' ? 'crash' : 'interrupted'
|
|
336
|
+
plan.attention = [...(plan.attention ?? []), { kind: 'pause', id: '', detail: 'pause interrupted' }]
|
|
337
|
+
plan.sessions = (plan.sessions ?? []).map((session) => ({ ...session, running: false, lastTurn: null }))
|
|
338
|
+
plan.status = 'paused'
|
|
339
|
+
if (persist(plan) === null) {
|
|
340
|
+
phase = 'paused'
|
|
341
|
+
lastError = 'store'
|
|
342
|
+
return { status: 'partial', summary: null, outstanding: outstandingOf(plan) }
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
const summary = summaryOf(plan)
|
|
346
|
+
summary.jobs = countsOf(plan).jobs
|
|
347
|
+
const record = (key, status) => {
|
|
348
|
+
if (store.appendJournal(plan.planId, key, status, bootId) === false) {
|
|
349
|
+
// Reporting only: the restore already happened, so this never blocks the plan. It does mean
|
|
350
|
+
// a later boot counts the item as outstanding and checks it again, which is safe.
|
|
351
|
+
log(`cannot journal ${key}`)
|
|
352
|
+
}
|
|
142
353
|
}
|
|
143
354
|
const step = async (key, label, run) => {
|
|
144
|
-
|
|
355
|
+
let outcome
|
|
145
356
|
try {
|
|
146
|
-
|
|
147
|
-
if (outcome === 'restored') summary.restored.push(label)
|
|
148
|
-
else summary.skipped.push(label)
|
|
357
|
+
outcome = await run()
|
|
149
358
|
} catch (error) {
|
|
150
|
-
|
|
359
|
+
outcome = { status: 'failed', detail: String(error?.message ?? error).slice(0, 120) }
|
|
151
360
|
}
|
|
152
|
-
|
|
361
|
+
const status = typeof outcome?.status === 'string' ? outcome.status : 'failed'
|
|
362
|
+
if (TERMINAL.has(status) === false) {
|
|
363
|
+
summary.failed.push(`${label}: ${outcome?.detail ?? 'unknown'}`)
|
|
364
|
+
return
|
|
365
|
+
}
|
|
366
|
+
record(key, status)
|
|
367
|
+
if (status === 'restored') summary.restored.push(label)
|
|
368
|
+
else if (status === 'skipped') summary.skipped.push(label)
|
|
369
|
+
else summary.attention.push(`${label}: ${outcome?.detail ?? 'needs a look'}`)
|
|
153
370
|
}
|
|
154
371
|
|
|
155
372
|
for (const session of plan.sessions ?? []) {
|
|
156
373
|
await step(`s:${session.sessionId}`, `session ${session.sessionId}`, () => probe.restoreSession(session, { plan }))
|
|
157
374
|
}
|
|
158
375
|
for (const goal of plan.goals ?? []) {
|
|
159
|
-
await step(`g:${goal.goalId ??
|
|
160
|
-
}
|
|
161
|
-
for (const schedule of plan.schedules ?? []) {
|
|
162
|
-
await step(`c:${schedule.id}`, `schedule ${schedule.id}`, () => probe.restoreSchedule(schedule, { plan }))
|
|
376
|
+
await step(`g:${goal.sessionId}:${goal.goalId ?? ''}`, `goal ${goal.goalId ?? goal.sessionId}`, () => probe.restoreGoal(goal, { plan }))
|
|
163
377
|
}
|
|
164
|
-
|
|
165
|
-
|
|
378
|
+
// A killed job cannot be replayed: the same command may have had an unknown effect, so it is
|
|
379
|
+
// reported once and left to the person. It never blocks the plan and is never restarted.
|
|
380
|
+
for (const job of plan.jobs ?? []) summary.attention.push(`job ${job.id}: stopped, not restarted`)
|
|
381
|
+
|
|
166
382
|
result = summary
|
|
383
|
+
if (summary.failed.length > 0) {
|
|
384
|
+
plan.restore = { at: summary.at, attempts: (plan.restore?.attempts ?? 0) + 1, failed: summary.failed } // ui-copy-ignore
|
|
385
|
+
if (persist(plan) === null) lastError = 'store'
|
|
386
|
+
else lastError = 'restore-partial'
|
|
387
|
+
phase = 'paused'
|
|
388
|
+
return { status: 'partial', summary, outstanding: outstandingOf(plan) }
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
if (isCrashPlan(plan) === true) {
|
|
392
|
+
if (typeof store.writeResult === 'function' && store.writeResult({ ...plan, status: 'restored', restoredAt: summary.at, summary }) === false) {
|
|
393
|
+
lastError = 'store'
|
|
394
|
+
phase = 'paused'
|
|
395
|
+
return { status: 'partial', summary, outstanding: 0 }
|
|
396
|
+
}
|
|
397
|
+
if (typeof store.clearPending !== 'function' || store.clearPending() === false) {
|
|
398
|
+
lastError = 'store'
|
|
399
|
+
phase = 'paused'
|
|
400
|
+
return { status: 'partial', summary, outstanding: 0 }
|
|
401
|
+
}
|
|
402
|
+
crashFallback = null
|
|
403
|
+
if (typeof store.clearJournal === 'function') store.clearJournal(plan.planId)
|
|
404
|
+
try {
|
|
405
|
+
heartbeat?.release?.()
|
|
406
|
+
} catch (error) {
|
|
407
|
+
log(`cannot release crash record: ${error?.message ?? error}`)
|
|
408
|
+
}
|
|
409
|
+
} else {
|
|
410
|
+
const consumed = store.consume(plan, summary)
|
|
411
|
+
if (!consumed?.ok) {
|
|
412
|
+
lastError = 'store'
|
|
413
|
+
phase = 'paused'
|
|
414
|
+
return { status: 'partial', summary, outstanding: 0 }
|
|
415
|
+
}
|
|
416
|
+
}
|
|
167
417
|
phase = 'idle'
|
|
168
418
|
return { status: 'resumed', summary }
|
|
169
419
|
}
|
|
@@ -177,7 +427,7 @@ export function createEngine({ store, probe, now = () => Date.now(), graceMs = 2
|
|
|
177
427
|
} catch (error) {
|
|
178
428
|
log(`${kind} failed: ${error?.message ?? error}`)
|
|
179
429
|
lastError = 'internal'
|
|
180
|
-
phase = 'idle'
|
|
430
|
+
phase = currentPlan() === null ? 'idle' : 'paused'
|
|
181
431
|
return { status: 'failed', reason: 'internal' }
|
|
182
432
|
} finally {
|
|
183
433
|
inflight = null
|
|
@@ -189,18 +439,22 @@ export function createEngine({ store, probe, now = () => Date.now(), graceMs = 2
|
|
|
189
439
|
|
|
190
440
|
async function pause() {
|
|
191
441
|
if (TRANSIENT.has(phase) || inflight !== null) return { status: 'busy', phase }
|
|
192
|
-
|
|
442
|
+
const plan = currentPlan()
|
|
443
|
+
if (plan !== null) {
|
|
444
|
+
const work = workOf(liveNow())
|
|
445
|
+
if (liveWorkCount(work) === 0) return { status: 'already', phase }
|
|
446
|
+
}
|
|
193
447
|
// Set synchronously, before the first await: two windows pressing at once are then exclusive
|
|
194
448
|
// in the same tick they were dispatched, not one microtask later.
|
|
195
449
|
phase = 'pausing'
|
|
196
|
-
return once('pause', runPause)
|
|
450
|
+
return once('pause', () => runPause(plan))
|
|
197
451
|
}
|
|
198
452
|
|
|
199
453
|
async function resume() {
|
|
200
454
|
if (TRANSIENT.has(phase) || inflight !== null) return { status: 'busy', phase }
|
|
201
455
|
phase = 'resuming'
|
|
202
456
|
return once('resume', async () => {
|
|
203
|
-
const plan =
|
|
457
|
+
const plan = currentPlan()
|
|
204
458
|
if (plan === null) {
|
|
205
459
|
phase = 'idle'
|
|
206
460
|
return { status: 'empty' }
|
|
@@ -209,11 +463,43 @@ export function createEngine({ store, probe, now = () => Date.now(), graceMs = 2
|
|
|
209
463
|
})
|
|
210
464
|
}
|
|
211
465
|
|
|
212
|
-
/**
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
466
|
+
/**
|
|
467
|
+
* A reload or restart must not invent a phase: the disk and the running Harness decide it. The
|
|
468
|
+
* crash residue is handed in by the caller, because it has to be read before the heartbeat
|
|
469
|
+
* overwrites the previous process's file; it is persisted here so a second restart still sees it.
|
|
470
|
+
*/
|
|
471
|
+
async function hydrate(options = {}) {
|
|
472
|
+
crashFallback = null
|
|
473
|
+
const residue = options?.residue ?? heartbeat?.residue ?? null
|
|
474
|
+
if (residue !== null && residue !== undefined && pausedPlan() === null && crashPlan() === null) {
|
|
475
|
+
let plan = null
|
|
476
|
+
try {
|
|
477
|
+
plan = probe.crashPlan(residue, { planId: `crash-${Number.isFinite(residue.at) ? residue.at : 0}-${Array.isArray(residue.sessions) ? residue.sessions.length : 0}` })
|
|
478
|
+
} catch (error) {
|
|
479
|
+
log(`cannot derive crash plan: ${error?.message ?? error}`)
|
|
480
|
+
}
|
|
481
|
+
if (plan !== null) {
|
|
482
|
+
const stored = typeof store.writePending === 'function' ? store.writePending(plan) : null
|
|
483
|
+
// A failed write keeps the plan in memory so this process can still restore it; only the
|
|
484
|
+
// next restart loses it.
|
|
485
|
+
crashFallback = stored ?? plan
|
|
486
|
+
log('crash plan recorded')
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
const restored = typeof store.lastRestore === 'function' ? store.lastRestore() : null
|
|
490
|
+
result = restored !== null && typeof restored === 'object' ? (restored.summary ?? null) : null
|
|
491
|
+
phase = pausedPlan() !== null ? 'paused' : 'idle'
|
|
492
|
+
lastError = null
|
|
493
|
+
return phase
|
|
216
494
|
}
|
|
217
495
|
|
|
218
|
-
return {
|
|
496
|
+
return {
|
|
497
|
+
describe,
|
|
498
|
+
pause,
|
|
499
|
+
resume,
|
|
500
|
+
hydrate,
|
|
501
|
+
get phase() { return phase },
|
|
502
|
+
get pendingPlan() { return currentPlan },
|
|
503
|
+
get crashPlan() { return crashPlan },
|
|
504
|
+
}
|
|
219
505
|
}
|