@sigrun/sigrun 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +19 -18
- package/dist/curation/runs.js +65 -50
- package/dist/curation/skill-tool.js +1 -1
- package/dist/extension.d.ts +2 -1
- package/dist/harness/blocks/tools.js +1 -1
- package/dist/harness/boot.js +94 -28
- package/dist/harness/decision-tool.js +1 -1
- package/dist/harness/decisions.js +1 -1
- package/dist/harness/event-bus.js +19 -9
- package/dist/harness/loader.js +14 -6
- package/dist/harness/migration-snapshot.js +104 -0
- package/dist/harness/packages.js +2 -1
- package/dist/harness/reports.js +83 -35
- package/dist/harness/tools.js +1 -1
- package/dist/jobs/docs.js +23 -3
- package/dist/jobs/index.js +74 -15
- package/dist/jobs/observe.js +26 -9
- package/dist/jobs/reconcile.js +12 -3
- package/dist/jobs/restart-limit.js +39 -0
- package/dist/sandbox/protocol.js +3 -2
- package/dist/sandbox/sandboxed-env.js +4 -0
- package/dist/sandbox/worker.js +21 -4
- package/dist/server/conversation-watch.js +49 -0
- package/dist/server/helpers.js +5 -5
- package/dist/server/host-bridge.js +6 -3
- package/dist/server/routes/admin.js +7 -36
- package/dist/server/routes/jobs.js +30 -1
- package/dist/server/routes/metrics.js +31 -7
- package/dist/server/routes/stream.js +6 -3
- package/dist/server/schema-fixture.js +59 -56
- package/dist/skills/index.js +1 -1
- package/node_modules/@sigrun/shared/CHANGELOG.md +4 -0
- package/node_modules/@sigrun/shared/dist/live.d.ts +25 -0
- package/node_modules/@sigrun/shared/dist/live.js +30 -0
- package/node_modules/@sigrun/shared/dist/views.d.ts +15 -0
- package/node_modules/@sigrun/shared/package.json +1 -1
- package/package.json +14 -14
- package/webui/dist/assets/{index-bUGUhbPh.js → index-B77MiI5E.js} +1 -1
- package/webui/dist/assets/{index-DALBZuYo.css → index-BvOKatk4.css} +1 -1
- package/webui/dist/index.html +2 -2
- package/dist/jobs/parked.js +0 -38
|
@@ -14,9 +14,25 @@
|
|
|
14
14
|
* migration and therefore does not pass through this helper.
|
|
15
15
|
*/
|
|
16
16
|
import { existsSync } from "node:fs";
|
|
17
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
18
|
+
import { createRequire } from "node:module";
|
|
17
19
|
import { DatabaseSync } from "node:sqlite";
|
|
18
20
|
import { CURRENT_SQLITE_SCHEMA_VERSION } from "@earendil-works/pi-durable/storage/sqlite";
|
|
19
21
|
export const SKIP_MIGRATION_SNAPSHOT_ENV = "SIGRUN_SKIP_MIGRATION_SNAPSHOT";
|
|
22
|
+
/**
|
|
23
|
+
* The sidecar recording which pi-durable version last opened a store
|
|
24
|
+
* (spec D4, 1.5.0 upgrade): `<storagePath>.durable-version`. A missing
|
|
25
|
+
* marker means the store predates the marker convention and counts as
|
|
26
|
+
* older than any running version.
|
|
27
|
+
*/
|
|
28
|
+
export const DURABLE_VERSION_MARKER_SUFFIX = ".durable-version";
|
|
29
|
+
/**
|
|
30
|
+
* The running pi-durable package version, read from the installed
|
|
31
|
+
* package's manifest through `createRequire` (the same bundler-stable
|
|
32
|
+
* shape `schema-fixture.ts` uses for HARNESS_VERSION), so a dependency
|
|
33
|
+
* bump moves the upgrade guard without a source edit.
|
|
34
|
+
*/
|
|
35
|
+
const DURABLE_PACKAGE_VERSION = createRequire(import.meta.url)("@earendil-works/pi-durable/package.json").version;
|
|
20
36
|
/** A required pre-migration snapshot could not be written. */
|
|
21
37
|
export class MigrationSnapshotError extends Error {
|
|
22
38
|
constructor(message, options) {
|
|
@@ -95,3 +111,91 @@ export async function snapshotBeforeMigration(storagePath, opts) {
|
|
|
95
111
|
db.close();
|
|
96
112
|
}
|
|
97
113
|
}
|
|
114
|
+
/** Compare two `major.minor[.patch]` versions by their minor. */
|
|
115
|
+
const MARKER_VERSION = /^(\d+)\.(\d+)(?:\.(\d+))?/;
|
|
116
|
+
function minorIsOlder(version, running) {
|
|
117
|
+
const parse = (v) => {
|
|
118
|
+
const match = /^(\d+)\.(\d+)(?:\.(\d+))?/.exec(v.trim());
|
|
119
|
+
if (match === null)
|
|
120
|
+
return undefined;
|
|
121
|
+
return [Number(match[1]), Number(match[2])];
|
|
122
|
+
};
|
|
123
|
+
const marked = parse(version);
|
|
124
|
+
const current = parse(running);
|
|
125
|
+
if (marked === undefined || current === undefined)
|
|
126
|
+
return false;
|
|
127
|
+
return (marked[0] < current[0] ||
|
|
128
|
+
(marked[0] === current[0] && marked[1] < current[1]));
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Pre-upgrade snapshot (spec D4, pi-durable 1.5.0): before pi-durable opens
|
|
132
|
+
* a store last written by an OLDER minor, preserve a copy the previous
|
|
133
|
+
* release can read. The version signal is the `<storagePath>.durable-version`
|
|
134
|
+
* sidecar — a missing marker (a pre-1.5 store, or one written before the
|
|
135
|
+
* convention) counts as older, so an unmarked existing store snapshots once
|
|
136
|
+
* and then never again once {@link writeDurableVersionMarker} has run.
|
|
137
|
+
*
|
|
138
|
+
* No-op when the skip env is set or the store file is absent (fresh
|
|
139
|
+
* installs have nothing to preserve). A failure to write a NEEDED snapshot
|
|
140
|
+
* throws `MigrationSnapshotError` with the same skip-env hint as the
|
|
141
|
+
* schema-migration guard. Returns the snapshot path, or `undefined`.
|
|
142
|
+
*/
|
|
143
|
+
export async function snapshotBeforeDurableUpgrade(storagePath, opts) {
|
|
144
|
+
if (opts.env[SKIP_MIGRATION_SNAPSHOT_ENV] === "1") {
|
|
145
|
+
opts.log(`sigrun: durable upgrade snapshot skipped (${SKIP_MIGRATION_SNAPSHOT_ENV}=1): ${storagePath}`);
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
if (!existsSync(storagePath))
|
|
149
|
+
return undefined;
|
|
150
|
+
const markerPath = `${storagePath}${DURABLE_VERSION_MARKER_SUFFIX}`;
|
|
151
|
+
let marked;
|
|
152
|
+
try {
|
|
153
|
+
marked = (await readFile(markerPath, "utf8")).trim();
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
marked = undefined;
|
|
157
|
+
}
|
|
158
|
+
if (marked !== undefined &&
|
|
159
|
+
MARKER_VERSION.test(marked) &&
|
|
160
|
+
!minorIsOlder(marked, DURABLE_PACKAGE_VERSION)) {
|
|
161
|
+
return undefined;
|
|
162
|
+
}
|
|
163
|
+
const snapshotPath = `${storagePath}.pre-pi-durable-${DURABLE_PACKAGE_VERSION}-${compactUtcStamp(opts.now())}.bak`;
|
|
164
|
+
let db;
|
|
165
|
+
try {
|
|
166
|
+
db = new DatabaseSync(storagePath);
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
opts.log(`sigrun: durable upgrade snapshot skipped; probe could not open ${storagePath}: ${messageOf(error)}`);
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
try {
|
|
174
|
+
db.exec(`VACUUM INTO ${sqlString(snapshotPath)}`);
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
// A non-database file defers to the real storage opener, whose
|
|
178
|
+
// error reports the underlying problem better than a snapshot
|
|
179
|
+
// guard's skip-env hint (the same deferral snapshotBeforeMigration
|
|
180
|
+
// makes for its schema probe).
|
|
181
|
+
if (messageOf(error).includes("file is not a database")) {
|
|
182
|
+
opts.log(`sigrun: durable upgrade snapshot skipped; ${storagePath} is not a database: ${messageOf(error)}`);
|
|
183
|
+
return undefined;
|
|
184
|
+
}
|
|
185
|
+
throw new MigrationSnapshotError(`could not snapshot ${storagePath} to ${snapshotPath} before the pi-durable ${marked ?? "unmarked"} → ${DURABLE_PACKAGE_VERSION} upgrade: ${messageOf(error)}; set ${SKIP_MIGRATION_SNAPSHOT_ENV}=1 only when you have an independent backup`, { cause: error });
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
finally {
|
|
189
|
+
db.close();
|
|
190
|
+
}
|
|
191
|
+
opts.log(`sigrun: durable upgrade snapshot ${snapshotPath} (${marked ?? "unmarked"} → ${DURABLE_PACKAGE_VERSION})`);
|
|
192
|
+
return snapshotPath;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Stamp the running pi-durable version beside the store. Boot calls this
|
|
196
|
+
* only after `openNodeSqliteStorage` succeeds, so the marker never claims a
|
|
197
|
+
* version the store has not actually been opened by.
|
|
198
|
+
*/
|
|
199
|
+
export async function writeDurableVersionMarker(storagePath) {
|
|
200
|
+
await writeFile(`${storagePath}${DURABLE_VERSION_MARKER_SUFFIX}`, DURABLE_PACKAGE_VERSION, "utf8");
|
|
201
|
+
}
|
package/dist/harness/packages.js
CHANGED
|
@@ -63,7 +63,8 @@ export const NPM_TIMEOUT_MS = 120_000;
|
|
|
63
63
|
* Runtime modules the HOST ships that extension packages may import bare.
|
|
64
64
|
* Extension packages declare these devDependencies-only (the
|
|
65
65
|
* provider-plugins dependency posture: pi-ai/pi-durable ship with sigrun,
|
|
66
|
-
* typebox ships as
|
|
66
|
+
* and typebox ships as sigrun's OWN direct dependency (pi-durable 1.5.0:
|
|
67
|
+
* pi-ai no longer carries or re-exports it) — structural typing at runtime,
|
|
67
68
|
* never identity), and npm never installs a dependency's devDependencies,
|
|
68
69
|
* so a package installed into the packages dir cannot resolve them on its
|
|
69
70
|
* own. {@link ensureHostModuleLinks} links sigrun's own copies into the
|
package/dist/harness/reports.js
CHANGED
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* pi-durable 1.
|
|
2
|
+
* pi-durable 1.5.0 `HarnessOptions.onReport` visibility (D5).
|
|
3
3
|
*
|
|
4
4
|
* Reports can arrive DURING `Harness.open`, before a Harness exists for
|
|
5
5
|
* follow-up work. This collector is therefore synchronous and small: count
|
|
6
|
-
* and log immediately
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* and log immediately. It never reads transcripts, and no failure in
|
|
7
|
+
* logging or an observer can escape into pi-durable.
|
|
8
|
+
*
|
|
9
|
+
* Its second responsibility (D2) is the restart-limit queue: it never
|
|
10
|
+
* calls the Harness from inside `onReport` (which runs inside Session
|
|
11
|
+
* machinery) — restart-limited tasks queue, and once boot registers the
|
|
12
|
+
* listener the queue drains via `setImmediate` (and each later report
|
|
13
|
+
* the same way).
|
|
9
14
|
*/
|
|
10
|
-
export const
|
|
11
|
-
"
|
|
15
|
+
export const REPORT_KINDS = [
|
|
16
|
+
"storage_failed",
|
|
12
17
|
"task_faulted",
|
|
13
|
-
"
|
|
18
|
+
"task_interrupted",
|
|
14
19
|
"migration_failed",
|
|
15
20
|
"task_outdated",
|
|
16
21
|
"task_error",
|
|
@@ -40,49 +45,89 @@ function field(name, value) {
|
|
|
40
45
|
return `${name}=${JSON.stringify(value)}`;
|
|
41
46
|
}
|
|
42
47
|
/** Format one report without exposing transcript content or ordinary stacks. */
|
|
43
|
-
function formatReport(
|
|
48
|
+
function formatReport(report) {
|
|
44
49
|
const fields = [];
|
|
45
|
-
if ("task" in
|
|
46
|
-
fields.push(["task",
|
|
50
|
+
if ("task" in report && report.task !== undefined) {
|
|
51
|
+
fields.push(["task", report.task.id], ["kind", report.task.kind], ["conversation", report.task.conversationId]);
|
|
47
52
|
}
|
|
48
|
-
if ("extension" in
|
|
49
|
-
fields.push(["extension",
|
|
53
|
+
if ("extension" in report && report.extension !== undefined) {
|
|
54
|
+
fields.push(["extension", report.extension]);
|
|
50
55
|
}
|
|
51
|
-
if ("conversationId" in
|
|
52
|
-
fields.push(["conversation",
|
|
56
|
+
if ("conversationId" in report && report.conversationId !== undefined) {
|
|
57
|
+
fields.push(["conversation", report.conversationId]);
|
|
53
58
|
}
|
|
54
|
-
if ("callback" in
|
|
55
|
-
fields.push(["callback",
|
|
56
|
-
if ("reason" in
|
|
57
|
-
fields.push(["reason",
|
|
58
|
-
fields.push(["message", errorMessage(error).slice(0, MESSAGE_LIMIT)]);
|
|
59
|
-
if (
|
|
60
|
-
fields.push(["stack", error.stack]);
|
|
59
|
+
if ("callback" in report)
|
|
60
|
+
fields.push(["callback", report.callback]);
|
|
61
|
+
if ("reason" in report)
|
|
62
|
+
fields.push(["reason", report.reason]);
|
|
63
|
+
fields.push(["message", errorMessage(report.error).slice(0, MESSAGE_LIMIT)]);
|
|
64
|
+
if (report.kind === "storage_failed" && report.error instanceof Error && report.error.stack !== undefined) {
|
|
65
|
+
fields.push(["stack", report.error.stack]);
|
|
61
66
|
}
|
|
62
67
|
return [
|
|
63
|
-
`sigrun: durable ${
|
|
68
|
+
`sigrun: durable ${report.kind}`,
|
|
64
69
|
...fields.map(([name, value]) => field(String(name), value)),
|
|
65
70
|
].join(" ");
|
|
66
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* The restart-limit predicate (D2). Schedules are an extension task
|
|
74
|
+
* kind, so a restart-limited schedule arrives as `task_faulted` (reason
|
|
75
|
+
* `restart_limit`); `task_interrupted` is the built-in tasks' spelling.
|
|
76
|
+
* Checked FIRST, before anything else.
|
|
77
|
+
*/
|
|
78
|
+
function isRestartLimited(report) {
|
|
79
|
+
return ((report.kind === "task_faulted" || report.kind === "task_interrupted") &&
|
|
80
|
+
report.reason === "restart_limit");
|
|
81
|
+
}
|
|
67
82
|
/** Create the one process-local report collector passed to Harness.open. */
|
|
68
83
|
export function createDurableReports(opts) {
|
|
69
|
-
const totals = Object.fromEntries(
|
|
70
|
-
let
|
|
84
|
+
const totals = Object.fromEntries(REPORT_KINDS.map((kind) => [kind, 0]));
|
|
85
|
+
let queued = [];
|
|
86
|
+
let listener;
|
|
87
|
+
let drainScheduled = false;
|
|
88
|
+
const scheduleDrain = () => {
|
|
89
|
+
if (drainScheduled)
|
|
90
|
+
return;
|
|
91
|
+
drainScheduled = true;
|
|
92
|
+
setImmediate(() => {
|
|
93
|
+
drainScheduled = false;
|
|
94
|
+
const sink = listener;
|
|
95
|
+
if (sink === undefined)
|
|
96
|
+
return;
|
|
97
|
+
const batch = queued;
|
|
98
|
+
queued = [];
|
|
99
|
+
for (const task of batch) {
|
|
100
|
+
try {
|
|
101
|
+
sink(task);
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
console.error("reports: restart-limited listener failed:", error);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
};
|
|
71
109
|
return {
|
|
72
|
-
onReport(
|
|
73
|
-
// Count and
|
|
110
|
+
onReport(report) {
|
|
111
|
+
// Count and log before observing: a throwing sink cannot lose the
|
|
74
112
|
// report, and pi-durable never receives a callback exception.
|
|
75
|
-
totals[
|
|
76
|
-
if (info.type === "task_parked")
|
|
77
|
-
parked.push(info.task);
|
|
113
|
+
totals[report.kind] += 1;
|
|
78
114
|
try {
|
|
79
|
-
opts.log(formatReport(
|
|
115
|
+
opts.log(formatReport(report));
|
|
80
116
|
}
|
|
81
117
|
catch {
|
|
82
118
|
// A host log sink is not allowed to fail durable scheduling.
|
|
83
119
|
}
|
|
120
|
+
// The restart-limit queue: the predicate is checked FIRST. Only
|
|
121
|
+
// the ReportTask is queued — the collector never touches the
|
|
122
|
+
// Harness; the listener (registered by boot after open) owns all
|
|
123
|
+
// follow-up reads/commits.
|
|
124
|
+
if (isRestartLimited(report)) {
|
|
125
|
+
queued.push(report.task);
|
|
126
|
+
if (listener !== undefined)
|
|
127
|
+
scheduleDrain();
|
|
128
|
+
}
|
|
84
129
|
try {
|
|
85
|
-
opts.onReport?.(
|
|
130
|
+
opts.onReport?.(report);
|
|
86
131
|
}
|
|
87
132
|
catch {
|
|
88
133
|
// Test/observer seams have the same containment rule.
|
|
@@ -91,10 +136,13 @@ export function createDurableReports(opts) {
|
|
|
91
136
|
counts() {
|
|
92
137
|
return { ...totals };
|
|
93
138
|
},
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
139
|
+
queuedRestartLimited() {
|
|
140
|
+
return [...queued];
|
|
141
|
+
},
|
|
142
|
+
onRestartLimited(sink) {
|
|
143
|
+
listener = sink;
|
|
144
|
+
if (queued.length > 0)
|
|
145
|
+
scheduleDrain();
|
|
98
146
|
},
|
|
99
147
|
};
|
|
100
148
|
}
|
package/dist/harness/tools.js
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* harness builds — so paths are conversation-relative exactly like the
|
|
27
27
|
* upstream coding tools.
|
|
28
28
|
*/
|
|
29
|
-
import { Type } from "
|
|
29
|
+
import { Type } from "typebox";
|
|
30
30
|
import { createBashTool, createEditTool, createReadTool, createWriteTool, } from "@earendil-works/pi-durable/tools";
|
|
31
31
|
import { defineExtension, defineTool } from "@earendil-works/pi-durable";
|
|
32
32
|
import { SessionMetaDoc } from "../server/session-meta.js";
|
package/dist/jobs/docs.js
CHANGED
|
@@ -206,8 +206,22 @@ export async function patchJob(harness, id, change, ctx) {
|
|
|
206
206
|
const prior = await harness.snapshot(JobDoc, id, ctx);
|
|
207
207
|
if (prior === undefined)
|
|
208
208
|
return;
|
|
209
|
+
// pi-durable 1.5.0 address semantics: a `tx.doc` acquire whose address
|
|
210
|
+
// was RETIRED between the pre-read and this commit CREATES a fresh
|
|
211
|
+
// incarnation from the seed — patchJob would resurrect a deleted job
|
|
212
|
+
// (the retire from deleteApi is hidden, reconcile's abort pass sees a
|
|
213
|
+
// live record). Discriminate with a sentinel the live record can never
|
|
214
|
+
// carry: the seed alone has it, so a created draft shows it; a live
|
|
215
|
+
// record never does. A detected creation is retired in the same commit,
|
|
216
|
+
// leaving the address absent — never resurrected.
|
|
217
|
+
const PROBE = "__patchJobSentinel";
|
|
209
218
|
await harness.commit(async (tx) => {
|
|
210
|
-
const
|
|
219
|
+
const seed = { ...prior, [PROBE]: true };
|
|
220
|
+
const draft = (await tx.doc(JobDoc, id, seed));
|
|
221
|
+
if (draft[PROBE] === true) {
|
|
222
|
+
await tx.retireDoc(JobDoc, id);
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
211
225
|
change(draft);
|
|
212
226
|
}, ctx);
|
|
213
227
|
}
|
|
@@ -250,11 +264,17 @@ export async function pruneHistory(storage, harness, jobId, limits, ctx) {
|
|
|
250
264
|
/**
|
|
251
265
|
* The fire key a stored run was written under, re-derived for retiring:
|
|
252
266
|
* scheduled runs key on `<jobId>:<fireAt epoch ms>`; manual runs on
|
|
253
|
-
* `<jobId>:manual:<
|
|
267
|
+
* `<jobId>:manual:<fireAt epoch ms>` — the SAME clock reading `runNow`
|
|
268
|
+
* passes as both the key tail and the fire's fireAt. (Deriving the
|
|
269
|
+
* manual key from `startedAt` instead wedges the settle whenever the
|
|
270
|
+
* fire's clock reading rolls over one ms before the run's `startedAt`
|
|
271
|
+
* stamp: the observer commits the settled record under a key that
|
|
272
|
+
* differs by 1ms from the key the fire stored, and the run stays
|
|
273
|
+
* `running` forever.)
|
|
254
274
|
*/
|
|
255
275
|
function runKeyOf(run) {
|
|
256
276
|
if (run.manual) {
|
|
257
|
-
return `${run.jobId}:manual:${Date.parse(run.
|
|
277
|
+
return `${run.jobId}:manual:${Date.parse(run.fireAt)}`;
|
|
258
278
|
}
|
|
259
279
|
return `${run.jobId}:${Date.parse(run.fireAt)}`;
|
|
260
280
|
}
|
package/dist/jobs/index.js
CHANGED
|
@@ -25,9 +25,9 @@
|
|
|
25
25
|
* per principal (§5.9 `accessOf`); the service serves raw rows.
|
|
26
26
|
*/
|
|
27
27
|
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
|
28
|
-
import { JobDoc, listRevisions, listRuns, pruneHistory, readJob, scanJobs, writeJobChange, } from "./docs.js";
|
|
28
|
+
import { JobDoc, listRevisions, listRuns, patchJob, pruneHistory, readJob, scanJobs, writeJobChange, } from "./docs.js";
|
|
29
29
|
import { fireJob } from "./fire.js";
|
|
30
|
-
import { reconcileJobs, } from "./reconcile.js";
|
|
30
|
+
import { reconcileJobs, createExecutor, liveScheduleTasksByJob, } from "./reconcile.js";
|
|
31
31
|
import { removeJobState, warnJobStateRemoval } from "./state-dir.js";
|
|
32
32
|
import { specHash } from "./spec.js";
|
|
33
33
|
/** Thrown when an API write targets a `file:`/`ext:` job id (409 `job_declared`). */
|
|
@@ -44,6 +44,13 @@ export class JobNotFoundError extends Error {
|
|
|
44
44
|
this.name = "JobNotFoundError";
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
|
+
/** Thrown when `resume` targets a job that is not restart-limited (409 `not_blocked`). */
|
|
48
|
+
export class JobNotBlockedError extends Error {
|
|
49
|
+
constructor(id) {
|
|
50
|
+
super(`job ${id} is not blocked by the restart limit`);
|
|
51
|
+
this.name = "JobNotBlockedError";
|
|
52
|
+
}
|
|
53
|
+
}
|
|
47
54
|
/** The live `jobHistory` limits (spec §8.1, ruling 14 — read per call). */
|
|
48
55
|
function historyLimits(settings) {
|
|
49
56
|
const jobHistory = settings().jobHistory;
|
|
@@ -108,19 +115,33 @@ export function createJobsService(deps) {
|
|
|
108
115
|
// the subscription is the only publish trigger.
|
|
109
116
|
if (deps.onChange !== undefined) {
|
|
110
117
|
const onChange = deps.onChange;
|
|
111
|
-
deps.harness.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
118
|
+
deps.harness.observeCommits({
|
|
119
|
+
advance: (publication) => {
|
|
120
|
+
try {
|
|
121
|
+
const seen = new Set();
|
|
122
|
+
for (const change of publication.changes) {
|
|
123
|
+
if (change.type !== "document")
|
|
124
|
+
continue;
|
|
125
|
+
if (change.record.kind !== JobDoc.definition.kind)
|
|
126
|
+
continue;
|
|
127
|
+
const jobId = change.record.key;
|
|
128
|
+
if (jobId === undefined || seen.has(jobId))
|
|
129
|
+
continue;
|
|
130
|
+
seen.add(jobId);
|
|
131
|
+
onChange(jobId);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
console.error("jobs/index: commit observer failed", error);
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
end: (end) => {
|
|
139
|
+
if (end.kind === "ended")
|
|
140
|
+
return;
|
|
141
|
+
if (end.kind === "failed" && end.error instanceof Error && end.error.message === "session closed")
|
|
142
|
+
return;
|
|
143
|
+
console.error("jobs/index: commit observation failed", end.kind === "failed" ? end.error : end);
|
|
144
|
+
},
|
|
124
145
|
});
|
|
125
146
|
}
|
|
126
147
|
/** Reconcile that one job's executor after a write. */
|
|
@@ -254,6 +275,44 @@ export function createJobsService(deps) {
|
|
|
254
275
|
const fireKey = `${id}:manual:${now}`;
|
|
255
276
|
return await fireJob(deps.runtime, id, fireKey, now, ctx);
|
|
256
277
|
},
|
|
278
|
+
async resume(id) {
|
|
279
|
+
const prior = await readJob(deps.harness, id, ctx);
|
|
280
|
+
if (prior === undefined)
|
|
281
|
+
throw new JobNotFoundError(id);
|
|
282
|
+
if (prior.state !== "blocked" || prior.reason !== "restart_limit") {
|
|
283
|
+
throw new JobNotBlockedError(id);
|
|
284
|
+
}
|
|
285
|
+
// Re-arm: waiting + no reason, then the executor through the
|
|
286
|
+
// SAME creation path reconcile uses.
|
|
287
|
+
await patchJob(deps.harness, id, (draft) => {
|
|
288
|
+
draft.state = "waiting";
|
|
289
|
+
delete draft.reason;
|
|
290
|
+
}, ctx);
|
|
291
|
+
const rearmed = await readJob(deps.harness, id, ctx);
|
|
292
|
+
if (rearmed === undefined)
|
|
293
|
+
throw new JobNotFoundError(id);
|
|
294
|
+
// Double-executor guard: if a live executor with the record's
|
|
295
|
+
// hash already exists (a raced reconcile created one between the
|
|
296
|
+
// block and the resume), reuse it instead of creating a second.
|
|
297
|
+
const live = await liveScheduleTasksByJob(deps.storage, ctx);
|
|
298
|
+
const existing = (live.get(id) ?? []).find((task) => task.input.hash === rearmed.hash);
|
|
299
|
+
if (existing === undefined) {
|
|
300
|
+
await createExecutor({
|
|
301
|
+
harness: deps.harness,
|
|
302
|
+
storage: deps.storage,
|
|
303
|
+
runtime: deps.runtime,
|
|
304
|
+
rootId: deps.rootId,
|
|
305
|
+
sources: deps.sources,
|
|
306
|
+
...(deps.jobsRoot === undefined
|
|
307
|
+
? {}
|
|
308
|
+
: { jobsRoot: deps.jobsRoot }),
|
|
309
|
+
}, id, rearmed, ctx);
|
|
310
|
+
}
|
|
311
|
+
const record = await readJob(deps.harness, id, ctx);
|
|
312
|
+
if (record === undefined)
|
|
313
|
+
throw new JobNotFoundError(id);
|
|
314
|
+
return wireOf(id, record);
|
|
315
|
+
},
|
|
257
316
|
reconcile: reconcileOne,
|
|
258
317
|
};
|
|
259
318
|
}
|
package/dist/jobs/observe.js
CHANGED
|
@@ -24,8 +24,10 @@ import { SessionMetaDoc } from "../server/session-meta.js";
|
|
|
24
24
|
import { JobDoc, JobRunDoc, listRuns, patchJob, pruneHistory, scanJobs } from "./docs.js";
|
|
25
25
|
/** The `sigrun.job.run` key a run record was written under. */
|
|
26
26
|
function runKeyOf(run) {
|
|
27
|
+
// Manual runs key on FIREAT (the same clock reading `runNow` used for
|
|
28
|
+
// the fire-key tail), never startedAt — see docs.ts's runKeyOf note.
|
|
27
29
|
if (run.manual) {
|
|
28
|
-
return `${run.jobId}:manual:${Date.parse(run.
|
|
30
|
+
return `${run.jobId}:manual:${Date.parse(run.fireAt)}`;
|
|
29
31
|
}
|
|
30
32
|
return `${run.jobId}:${Date.parse(run.fireAt)}`;
|
|
31
33
|
}
|
|
@@ -136,16 +138,31 @@ export function registerJobObservers(deps) {
|
|
|
136
138
|
// this process). A no-jobs harness pays ONE boolean check per terminal
|
|
137
139
|
// task, never a scan.
|
|
138
140
|
let jobsMayExist = false;
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
141
|
+
const offJobWritesObservation = deps.harness.observeCommits({
|
|
142
|
+
advance: (publication) => {
|
|
143
|
+
try {
|
|
144
|
+
for (const change of publication.changes) {
|
|
145
|
+
if (change.type !== "document")
|
|
146
|
+
continue;
|
|
147
|
+
if (change.record.kind === JobDoc.definition.kind) {
|
|
148
|
+
jobsMayExist = true;
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
146
152
|
}
|
|
147
|
-
|
|
153
|
+
catch (error) {
|
|
154
|
+
console.error("jobs/observe: commit observer failed", error);
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
end: (end) => {
|
|
158
|
+
if (end.kind === "ended")
|
|
159
|
+
return;
|
|
160
|
+
if (end.kind === "failed" && end.error instanceof Error && end.error.message === "session closed")
|
|
161
|
+
return;
|
|
162
|
+
console.error("jobs/observe: commit observation failed", end.kind === "failed" ? end.error : end);
|
|
163
|
+
},
|
|
148
164
|
});
|
|
165
|
+
const offJobWrites = () => offJobWritesObservation.detach();
|
|
149
166
|
const offTaskSettled = deps.eventBus.onHostEvent((event) => {
|
|
150
167
|
if (event.type !== "taskSettled")
|
|
151
168
|
return;
|
package/dist/jobs/reconcile.js
CHANGED
|
@@ -186,7 +186,14 @@ export async function reconcileJobs(deps, ctx = BACKGROUND_CONTEXT) {
|
|
|
186
186
|
for (const task of live) {
|
|
187
187
|
await deps.harness.abortTask(task.id, ctx);
|
|
188
188
|
}
|
|
189
|
-
|
|
189
|
+
// D2: a restart-limited record (state blocked, reason
|
|
190
|
+
// restart_limit) gets NO new executor — its executor ended
|
|
191
|
+
// faulted and only an explicit resume re-arms it. The guard
|
|
192
|
+
// wraps ONLY the creation: stale-task pruning above (and the
|
|
193
|
+
// hash-mismatch aborts below) still runs.
|
|
194
|
+
if (!(record.state === "blocked" && record.reason === "restart_limit")) {
|
|
195
|
+
await createExecutor(deps, id, record, ctx);
|
|
196
|
+
}
|
|
190
197
|
}
|
|
191
198
|
// A matching live task exists: leave it (the same task id
|
|
192
199
|
// survives an unchanged reboot). Extra stale live tasks for
|
|
@@ -236,8 +243,10 @@ async function retireRecord(harness, id, record, nowIso, ctx) {
|
|
|
236
243
|
async function patchRecord(harness, id, change, ctx) {
|
|
237
244
|
await patchJob(harness, id, change, ctx);
|
|
238
245
|
}
|
|
239
|
-
/** Create the schedule task for one enabled record (ONE commit).
|
|
240
|
-
|
|
246
|
+
/** Create the schedule task for one enabled record (ONE commit).
|
|
247
|
+
* Exported for `JobsService.resume` (D2): a restart-limited job's
|
|
248
|
+
* re-arm rides the same creation path. */
|
|
249
|
+
export async function createExecutor(deps, id, record, ctx) {
|
|
241
250
|
const firstAt = nextFireAt(record.spec.schedule, deps.runtime.now(), deps.runtime.defaultTz);
|
|
242
251
|
await deps.harness.commit(async (tx) => {
|
|
243
252
|
await tx.createTask(createScheduleTask(deps.runtime), {
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs restart-limit recovery (D2): the restart-limited listener.
|
|
3
|
+
*
|
|
4
|
+
* pi-durable 1.5.0 removed `task_parked`/`retryTask`: a schedule task
|
|
5
|
+
* cut off `settings.restartLimit` times in a row ends FAULTED (reason
|
|
6
|
+
* `restart_limit`), reported through `HarnessOptions.onReport`. The
|
|
7
|
+
* reports collector queues those tasks; boot registers this listener
|
|
8
|
+
* after open, and it marks the owning job record
|
|
9
|
+
* `state: "blocked"`, `reason: "restart_limit"` — the signal reconcile
|
|
10
|
+
* respects (a blocked job gets NO new executor) and `POST
|
|
11
|
+
* /v1/jobs/:id/resume` clears.
|
|
12
|
+
*
|
|
13
|
+
* The listener is async and owns all Harness reads/commits; a
|
|
14
|
+
* rejection is logged by the boot wrapper, never thrown into the
|
|
15
|
+
* collector's delivery lane.
|
|
16
|
+
*/
|
|
17
|
+
import { patchJob } from "./docs.js";
|
|
18
|
+
import { SCHEDULE_TASK_KIND } from "./task.js";
|
|
19
|
+
/**
|
|
20
|
+
* Mark the job owning a restart-limited schedule task blocked. Anything
|
|
21
|
+
* that is not a schedule task (built-in kinds, unknown ids, malformed
|
|
22
|
+
* inputs) is ignored — the restart-limit recovery surface is jobs-only.
|
|
23
|
+
* A failure REJECTS: boot's registration wrapper owns the logging (one
|
|
24
|
+
* log per failure, never a double-log).
|
|
25
|
+
*/
|
|
26
|
+
export async function markRestartLimitedJob(harness, task, context) {
|
|
27
|
+
if (task.kind !== SCHEDULE_TASK_KIND)
|
|
28
|
+
return;
|
|
29
|
+
const record = await harness.getTask(task.id, context);
|
|
30
|
+
if (record === undefined)
|
|
31
|
+
return;
|
|
32
|
+
const jobId = record.input.jobId;
|
|
33
|
+
if (typeof jobId !== "string")
|
|
34
|
+
return;
|
|
35
|
+
await patchJob(harness, jobId, (draft) => {
|
|
36
|
+
draft.state = "blocked";
|
|
37
|
+
draft.reason = "restart_limit";
|
|
38
|
+
}, context);
|
|
39
|
+
}
|
package/dist/sandbox/protocol.js
CHANGED
|
@@ -153,15 +153,16 @@ export function decodeWatchChange(value) {
|
|
|
153
153
|
throw new TypeError("invalid watcher error");
|
|
154
154
|
}
|
|
155
155
|
return {
|
|
156
|
+
kind: "failed",
|
|
156
157
|
error: new FileError(code, message, path),
|
|
157
158
|
};
|
|
158
159
|
}
|
|
159
160
|
if ("overflow" in value && value.overflow === true)
|
|
160
|
-
return {
|
|
161
|
+
return { kind: "overflow" };
|
|
161
162
|
if ("paths" in value &&
|
|
162
163
|
Array.isArray(value.paths) &&
|
|
163
164
|
value.paths.every((path) => typeof path === "string")) {
|
|
164
|
-
return { paths: value.paths };
|
|
165
|
+
return { kind: "paths", paths: value.paths };
|
|
165
166
|
}
|
|
166
167
|
throw new TypeError("invalid watcher change");
|
|
167
168
|
}
|
|
@@ -81,6 +81,10 @@ class BaseRemoteHandle {
|
|
|
81
81
|
this.client.releaseHandle(this.id);
|
|
82
82
|
}
|
|
83
83
|
}
|
|
84
|
+
/** pi-durable 1.5.0 renamed `FileWatcher.close` to `stop`; the wire verb stays "close". */
|
|
85
|
+
async stop(context) {
|
|
86
|
+
await this.close(context);
|
|
87
|
+
}
|
|
84
88
|
async call(method, args, context) {
|
|
85
89
|
if (this.closed)
|
|
86
90
|
throw new FileError("invalid", "sandbox handle is closed");
|
package/dist/sandbox/worker.js
CHANGED
|
@@ -68,7 +68,14 @@ async function handleCall(request, args) {
|
|
|
68
68
|
const method = request.op.slice("handle.".length);
|
|
69
69
|
if (method === "close") {
|
|
70
70
|
handles.delete(id);
|
|
71
|
-
|
|
71
|
+
// pi-durable 1.5.0: readers close, watchers stop — the wire verb is
|
|
72
|
+
// "close" for both (a rename only).
|
|
73
|
+
if (handle.kind === "watch") {
|
|
74
|
+
await handle.value.stop(BACKGROUND_CONTEXT);
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
await handle.value.close(BACKGROUND_CONTEXT);
|
|
78
|
+
}
|
|
72
79
|
return null;
|
|
73
80
|
}
|
|
74
81
|
if (handle.kind === "text" && method === "readLine") {
|
|
@@ -135,7 +142,11 @@ async function dispatch(request) {
|
|
|
135
142
|
return result.ok ? { ok: true, value: openHandle({ kind: "dir", value: result.value }) } : result;
|
|
136
143
|
}
|
|
137
144
|
case "watch": {
|
|
138
|
-
const result = await env.watch(args[0], (change) => write({
|
|
145
|
+
const result = await env.watch(args[0], (change) => write({
|
|
146
|
+
id: request.id,
|
|
147
|
+
event: "watch",
|
|
148
|
+
change: encodeValue(change.kind === "failed" ? { error: wireError(change.error) } : change.kind === "overflow" ? { overflow: true } : change),
|
|
149
|
+
}), BACKGROUND_CONTEXT);
|
|
139
150
|
return result.ok
|
|
140
151
|
? { ok: true, value: { handle: openHandle({ kind: "watch", value: result.value }), mode: result.value.mode } }
|
|
141
152
|
: result;
|
|
@@ -255,7 +266,13 @@ process.stdin.on("end", () => {
|
|
|
255
266
|
controller.abort();
|
|
256
267
|
for (const env of environments.values())
|
|
257
268
|
void env.cleanup(BACKGROUND_CONTEXT);
|
|
258
|
-
for (const handle of handles.values())
|
|
259
|
-
|
|
269
|
+
for (const handle of handles.values()) {
|
|
270
|
+
if (handle.kind === "watch") {
|
|
271
|
+
void handle.value.stop(BACKGROUND_CONTEXT);
|
|
272
|
+
}
|
|
273
|
+
else {
|
|
274
|
+
void handle.value.close(BACKGROUND_CONTEXT);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
260
277
|
process.exit(0);
|
|
261
278
|
});
|