@h-rig/run-worker 0.0.6-alpha.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/src/autohost.d.ts +34 -0
- package/dist/src/autohost.js +592 -0
- package/dist/src/constants.d.ts +4 -0
- package/dist/src/constants.js +12 -0
- package/dist/src/extension.d.ts +14 -0
- package/dist/src/extension.js +614 -0
- package/dist/src/index.d.ts +6 -0
- package/dist/src/index.js +629 -0
- package/dist/src/journal.d.ts +31 -0
- package/dist/src/journal.js +14 -0
- package/dist/src/notifications.d.ts +1 -0
- package/dist/src/notifications.js +19 -0
- package/dist/src/stall.d.ts +16 -0
- package/dist/src/stall.js +56 -0
- package/dist/src/utils.d.ts +23 -0
- package/dist/src/utils.js +38 -0
- package/package.json +30 -0
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
// packages/run-worker/src/autohost.ts
|
|
3
|
+
import { latestTimelineEntriesFromCustomEntries, parseInboxResolutionSentinel, parsePauseSentinel, parseResumeSentinel, parseStopSentinel, sessionIdFromSessionFile } from "@rig/contracts";
|
|
4
|
+
import { Duration, Effect, Fiber, Stream } from "effect";
|
|
5
|
+
import { createEnvCloseoutRunners } from "@rig/runtime/control-plane/native/closeout-runners";
|
|
6
|
+
import { CloseoutValidationError, runInProcessCloseout } from "@rig/runtime/control-plane/native/in-process-closeout";
|
|
7
|
+
import { projectRunFromSession } from "@rig/runtime/control-plane/run-session-projection";
|
|
8
|
+
import { localRunChanges } from "@rig/runtime/control-plane/run-discovery-stream";
|
|
9
|
+
import { updateRunTaskSourceLifecycle } from "@rig/runtime/control-plane/tasks/source-lifecycle";
|
|
10
|
+
import { resolveOwnerNamespaceKey } from "@rig/runtime/control-plane/remote-config";
|
|
11
|
+
import { resolveRigIdentity } from "@rig/runtime/control-plane/identity";
|
|
12
|
+
import { connectWorkerProjection, createRegistryClient } from "@rig/relay-registry";
|
|
13
|
+
import { coerceRegistryStatus } from "@rig/relay-registry/schema";
|
|
14
|
+
|
|
15
|
+
// packages/run-worker/src/journal.ts
|
|
16
|
+
import { RunSessionJournal } from "@rig/runtime/control-plane/run-session-writer";
|
|
17
|
+
async function createRunJournal(sessionManager, runId) {
|
|
18
|
+
try {
|
|
19
|
+
return new RunSessionJournal(sessionManager, runId);
|
|
20
|
+
} catch (error) {
|
|
21
|
+
console.warn(`[rig-run] RunSessionJournal unavailable; run-state arming deferred: ${error instanceof Error ? error.message : String(error)}`);
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// packages/run-worker/src/notifications.ts
|
|
27
|
+
import { resolve } from "path";
|
|
28
|
+
import { dispatchEventToTargets, loadNotificationConfig } from "@rig/runtime/control-plane/notifications";
|
|
29
|
+
async function dispatchRunNotifications(projectRoot, runId, taskId, outcome, detail) {
|
|
30
|
+
try {
|
|
31
|
+
const config = await loadNotificationConfig(resolve(projectRoot, ".rig", "notifications.json"));
|
|
32
|
+
if (config.targets.length === 0)
|
|
33
|
+
return;
|
|
34
|
+
const event = { runId, type: `run.${outcome}`, timestamp: new Date().toISOString(), payload: { taskId, detail } };
|
|
35
|
+
await Promise.race([
|
|
36
|
+
dispatchEventToTargets(event, config.targets),
|
|
37
|
+
new Promise((resolveCap) => setTimeout(resolveCap, 5000))
|
|
38
|
+
]);
|
|
39
|
+
} catch {}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// packages/run-worker/src/constants.ts
|
|
43
|
+
var RUN_PROCESS_STEER_TIMEOUT_MS = 10 * 60 * 1000;
|
|
44
|
+
var TRACKED_RUN_STALL_MS = 20 * 60 * 1000;
|
|
45
|
+
var RUN_PROCESS_STALL_SWEEP_MS = 60 * 1000;
|
|
46
|
+
var RUN_PROCESS_STALL_DETAIL = "Run process made no OMP session progress for 20+ minutes; recording a stall so recovery can requeue or resume the session.";
|
|
47
|
+
|
|
48
|
+
// packages/run-worker/src/stall.ts
|
|
49
|
+
function timestampMs(value) {
|
|
50
|
+
if (value === null || value === undefined)
|
|
51
|
+
return null;
|
|
52
|
+
const ms = value instanceof Date ? value.getTime() : typeof value === "number" ? value : Date.parse(value);
|
|
53
|
+
return Number.isFinite(ms) ? ms : null;
|
|
54
|
+
}
|
|
55
|
+
function computeRunStall(input) {
|
|
56
|
+
const lastActivityAt = timestampMs(input.lastActivityAt);
|
|
57
|
+
const now = timestampMs(input.now);
|
|
58
|
+
if (lastActivityAt === null || now === null || input.thresholdMs <= 0)
|
|
59
|
+
return false;
|
|
60
|
+
return now - lastActivityAt >= input.thresholdMs;
|
|
61
|
+
}
|
|
62
|
+
function appendRunStallDetected(journal, detail = RUN_PROCESS_STALL_DETAIL) {
|
|
63
|
+
if (!journal)
|
|
64
|
+
return false;
|
|
65
|
+
try {
|
|
66
|
+
journal.appendStall({ detail });
|
|
67
|
+
return true;
|
|
68
|
+
} catch (error) {
|
|
69
|
+
console.warn(`[rig-run] stall-detected append failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function startRunProcessStallMonitor(opts) {
|
|
74
|
+
if (!opts.journal)
|
|
75
|
+
return () => {};
|
|
76
|
+
let stallDetected = opts.alreadyStalled === true;
|
|
77
|
+
const thresholdMs = opts.thresholdMs ?? TRACKED_RUN_STALL_MS;
|
|
78
|
+
const now = opts.now ?? (() => Date.now());
|
|
79
|
+
const timer = setInterval(() => {
|
|
80
|
+
if (stallDetected)
|
|
81
|
+
return;
|
|
82
|
+
if (!computeRunStall({ lastActivityAt: opts.lastActivityAt(), now: now(), thresholdMs }))
|
|
83
|
+
return;
|
|
84
|
+
stallDetected = true;
|
|
85
|
+
appendRunStallDetected(opts.journal);
|
|
86
|
+
}, opts.intervalMs ?? RUN_PROCESS_STALL_SWEEP_MS);
|
|
87
|
+
if (typeof timer.unref === "function")
|
|
88
|
+
timer.unref();
|
|
89
|
+
return () => clearInterval(timer);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// packages/run-worker/src/utils.ts
|
|
93
|
+
import { createWorkflowStatusChanged, RIG_WORKFLOW_STATUS_CHANGED } from "@rig/contracts";
|
|
94
|
+
import { resolveRegistryBaseUrl, resolveRelayUrl } from "@rig/runtime/control-plane/remote-config";
|
|
95
|
+
function customEntries(entries) {
|
|
96
|
+
const customs = [];
|
|
97
|
+
for (const entry of entries) {
|
|
98
|
+
if (entry.type === "custom")
|
|
99
|
+
customs.push(entry);
|
|
100
|
+
}
|
|
101
|
+
return customs;
|
|
102
|
+
}
|
|
103
|
+
function rigProjectRoot() {
|
|
104
|
+
return process.env.PROJECT_RIG_ROOT?.trim() || process.env.RIG_HOST_PROJECT_ROOT?.trim() || process.cwd();
|
|
105
|
+
}
|
|
106
|
+
function registryBaseUrl() {
|
|
107
|
+
return resolveRegistryBaseUrl(rigProjectRoot());
|
|
108
|
+
}
|
|
109
|
+
function rigRelayUrl() {
|
|
110
|
+
return resolveRelayUrl();
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// packages/run-worker/src/autohost.ts
|
|
114
|
+
var REGISTRY_PROJECTION_TIMELINE_LIMIT = 100;
|
|
115
|
+
function registryRunProjection(input) {
|
|
116
|
+
const timeline = input.timeline ?? latestTimelineEntriesFromCustomEntries(input.entries, REGISTRY_PROJECTION_TIMELINE_LIMIT);
|
|
117
|
+
return {
|
|
118
|
+
runId: input.runId,
|
|
119
|
+
taskId: typeof input.folded.record.taskId === "string" ? input.folded.record.taskId : null,
|
|
120
|
+
title: typeof input.folded.record.title === "string" ? input.folded.record.title : input.title,
|
|
121
|
+
status: coerceRegistryStatus(input.folded.status, "running"),
|
|
122
|
+
source: "local",
|
|
123
|
+
startedAt: typeof input.folded.record.startedAt === "string" ? input.folded.record.startedAt : null,
|
|
124
|
+
updatedAt: input.folded.lastEventAt ?? (typeof input.folded.record.updatedAt === "string" ? input.folded.record.updatedAt : new Date().toISOString()),
|
|
125
|
+
completedAt: typeof input.folded.record.completedAt === "string" ? input.folded.record.completedAt : null,
|
|
126
|
+
joinLink: input.joinLink ?? null,
|
|
127
|
+
webLink: input.webLink ?? null,
|
|
128
|
+
relayUrl: input.relayUrl ?? null,
|
|
129
|
+
sessionPath: typeof input.folded.record.sessionPath === "string" ? input.folded.record.sessionPath : input.sessionPath ?? null,
|
|
130
|
+
prUrl: typeof input.folded.record.prUrl === "string" ? input.folded.record.prUrl : null,
|
|
131
|
+
worktreePath: typeof input.folded.record.worktreePath === "string" ? input.folded.record.worktreePath : process.cwd(),
|
|
132
|
+
collabCwd: process.cwd(),
|
|
133
|
+
cwd: process.cwd(),
|
|
134
|
+
dispatchHandle: process.env.RIG_RUN_ID?.trim() ?? null,
|
|
135
|
+
pendingApprovals: input.folded.pendingApprovals.length,
|
|
136
|
+
pendingInputs: input.folded.pendingUserInputs.length,
|
|
137
|
+
steeringCount: input.folded.steeringCount,
|
|
138
|
+
stallCount: input.folded.stallCount,
|
|
139
|
+
...typeof input.folded.record.errorText === "string" ? { errorSummary: input.folded.record.errorText } : { errorSummary: null },
|
|
140
|
+
timeline: [...timeline]
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
var OPERATOR_ACTOR = { kind: "operator" };
|
|
144
|
+
function textFromContent(content) {
|
|
145
|
+
if (typeof content === "string")
|
|
146
|
+
return content;
|
|
147
|
+
if (!Array.isArray(content))
|
|
148
|
+
return null;
|
|
149
|
+
const text = content.map((part) => part && typeof part === "object" && ("text" in part) && typeof part.text === "string" ? part.text : "").filter(Boolean).join(`
|
|
150
|
+
`).trim();
|
|
151
|
+
return text || null;
|
|
152
|
+
}
|
|
153
|
+
function textFromSessionEntry(entry) {
|
|
154
|
+
if (!entry || typeof entry !== "object")
|
|
155
|
+
return null;
|
|
156
|
+
const record = entry;
|
|
157
|
+
if (record.type === "custom_message")
|
|
158
|
+
return textFromContent(record.content);
|
|
159
|
+
if (record.type === "message")
|
|
160
|
+
return textFromAgentMessage(record.message);
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
function textFromAgentMessage(message) {
|
|
164
|
+
if (!message || typeof message !== "object")
|
|
165
|
+
return null;
|
|
166
|
+
return textFromContent(message.content);
|
|
167
|
+
}
|
|
168
|
+
function detectRunControlText(text, runId) {
|
|
169
|
+
const stop = parseStopSentinel(text, runId);
|
|
170
|
+
if (stop)
|
|
171
|
+
return { kind: "stop", reason: stop.reason };
|
|
172
|
+
if (parsePauseSentinel(text, runId))
|
|
173
|
+
return { kind: "pause" };
|
|
174
|
+
if (parseResumeSentinel(text, runId))
|
|
175
|
+
return { kind: "resume" };
|
|
176
|
+
const resolution = parseInboxResolutionSentinel(text, runId);
|
|
177
|
+
if (resolution)
|
|
178
|
+
return { kind: "inbox", resolution };
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
async function maybeStartRunProcessAutohost(api, ctx) {
|
|
182
|
+
if (process.env.RIG_RUN_PROCESS !== "1")
|
|
183
|
+
return null;
|
|
184
|
+
const envRunId = process.env.RIG_RUN_ID?.trim();
|
|
185
|
+
if (!envRunId)
|
|
186
|
+
throw new Error("RIG_RUN_ID is required when RIG_RUN_PROCESS=1");
|
|
187
|
+
const runId = sessionIdFromSessionFile(ctx.sessionManager.getSessionFile()) ?? envRunId;
|
|
188
|
+
console.log(`[rig-run] session_start hasUI=${ctx.hasUI ? "true" : "false"}`);
|
|
189
|
+
if (!ctx.hasUI)
|
|
190
|
+
return null;
|
|
191
|
+
const collab = ctx.collab;
|
|
192
|
+
const startHost = collab?.startCollabHost ?? collab?.startHost;
|
|
193
|
+
if (!startHost)
|
|
194
|
+
throw new Error("OMP collab host facade is unavailable for Rig run process.");
|
|
195
|
+
const identity = resolveRigIdentity(ctx);
|
|
196
|
+
const journal = await createRunJournal(ctx.sessionManager, runId);
|
|
197
|
+
const projectRoot = process.env.PROJECT_RIG_ROOT ?? process.cwd();
|
|
198
|
+
let runProjection = projectRunFromSession(customEntries(ctx.sessionManager.getBranch()), runId);
|
|
199
|
+
const taskIdAtStart = process.env.RIG_TASK_ID?.trim() || runProjection.record.taskId;
|
|
200
|
+
const runDisplayTitle = (() => {
|
|
201
|
+
const base = process.env.RIG_RUN_TITLE?.trim() || `Rig run ${runId}`;
|
|
202
|
+
return taskIdAtStart && !base.includes(taskIdAtStart) ? `[${taskIdAtStart}] ${base}` : base;
|
|
203
|
+
})();
|
|
204
|
+
collab?.stopHost;
|
|
205
|
+
let closeoutStarted = false;
|
|
206
|
+
let forcedSteerUsed = false;
|
|
207
|
+
let lastRunActivityAt = Date.now();
|
|
208
|
+
const markRunActivity = () => {
|
|
209
|
+
lastRunActivityAt = Date.now();
|
|
210
|
+
};
|
|
211
|
+
let runRegistryHeartbeat = null;
|
|
212
|
+
let runRegistryRoomId = null;
|
|
213
|
+
let workerProjectionConnection = null;
|
|
214
|
+
let workerProjectionFiber = null;
|
|
215
|
+
const runRegistryNamespace = identity?.owner?.namespaceKey ?? resolveOwnerNamespaceKey(rigProjectRoot()) ?? null;
|
|
216
|
+
const runRegistry = runRegistryNamespace ? createRegistryClient({ baseUrl: registryBaseUrl(), namespaceKey: runRegistryNamespace }) : null;
|
|
217
|
+
let runRegistryLinks = {};
|
|
218
|
+
const processedControlEntryIds = new Set((ctx.sessionManager.getEntries?.() ?? []).map((entry) => entry && typeof entry === "object" && typeof entry.id === "string" ? entry.id : null).filter((id) => id !== null));
|
|
219
|
+
const processedControlEntryObjects = new WeakSet;
|
|
220
|
+
let pendingPause = false;
|
|
221
|
+
let pendingStopReason;
|
|
222
|
+
let cachedProjectionParts = null;
|
|
223
|
+
const currentProjectionParts = () => {
|
|
224
|
+
const branch = ctx.sessionManager.getBranch();
|
|
225
|
+
const lastBranchEntry = branch.at(-1);
|
|
226
|
+
const lastBranchObject = lastBranchEntry && typeof lastBranchEntry === "object" ? lastBranchEntry : null;
|
|
227
|
+
if (!cachedProjectionParts || cachedProjectionParts.branchLength !== branch.length || cachedProjectionParts.lastBranchEntry !== lastBranchObject) {
|
|
228
|
+
const entries = customEntries(branch);
|
|
229
|
+
cachedProjectionParts = {
|
|
230
|
+
branchLength: branch.length,
|
|
231
|
+
lastBranchEntry: lastBranchObject,
|
|
232
|
+
entries,
|
|
233
|
+
folded: projectRunFromSession(entries, runId),
|
|
234
|
+
timeline: latestTimelineEntriesFromCustomEntries(entries, REGISTRY_PROJECTION_TIMELINE_LIMIT)
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
runProjection = cachedProjectionParts.folded;
|
|
238
|
+
return { entries: cachedProjectionParts.entries, folded: cachedProjectionParts.folded, timeline: cachedProjectionParts.timeline };
|
|
239
|
+
};
|
|
240
|
+
const buildCurrentRegistryProjection = (links = runRegistryLinks) => {
|
|
241
|
+
const { entries, folded, timeline } = currentProjectionParts();
|
|
242
|
+
return registryRunProjection({
|
|
243
|
+
runId,
|
|
244
|
+
folded,
|
|
245
|
+
entries,
|
|
246
|
+
title: runDisplayTitle,
|
|
247
|
+
joinLink: links.joinLink,
|
|
248
|
+
webLink: links.webLink,
|
|
249
|
+
relayUrl: links.relayUrl,
|
|
250
|
+
sessionPath: ctx.sessionManager.getSessionFile(),
|
|
251
|
+
timeline
|
|
252
|
+
});
|
|
253
|
+
};
|
|
254
|
+
const pushWorkerProjection = (links = runRegistryLinks) => {
|
|
255
|
+
if (!workerProjectionConnection)
|
|
256
|
+
return;
|
|
257
|
+
workerProjectionConnection.push(buildCurrentRegistryProjection(links));
|
|
258
|
+
};
|
|
259
|
+
const publishRunProjection = async (status) => {
|
|
260
|
+
const projection = buildCurrentRegistryProjection();
|
|
261
|
+
if (workerProjectionConnection)
|
|
262
|
+
workerProjectionConnection.push(projection);
|
|
263
|
+
if (runRegistry && runRegistryRoomId) {
|
|
264
|
+
await runRegistry.heartbeatRoom(runRegistryRoomId, status, projection).catch((err) => console.error(`[rig-run] registry-heartbeat-failed ${err instanceof Error ? err.message : String(err)}`));
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
const stopRunRegistry = (opts = {}) => {
|
|
268
|
+
if (runRegistryHeartbeat) {
|
|
269
|
+
clearInterval(runRegistryHeartbeat);
|
|
270
|
+
runRegistryHeartbeat = null;
|
|
271
|
+
}
|
|
272
|
+
if (workerProjectionFiber) {
|
|
273
|
+
Effect.runFork(Fiber.interrupt(workerProjectionFiber));
|
|
274
|
+
workerProjectionFiber = null;
|
|
275
|
+
}
|
|
276
|
+
if (workerProjectionConnection) {
|
|
277
|
+
workerProjectionConnection.close();
|
|
278
|
+
workerProjectionConnection = null;
|
|
279
|
+
}
|
|
280
|
+
if ((opts.removeRoom ?? true) && runRegistry && runRegistryRoomId) {
|
|
281
|
+
runRegistry.removeRoom(runRegistryRoomId).catch(() => {});
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
const stopRunStallMonitor = startRunProcessStallMonitor({
|
|
285
|
+
journal,
|
|
286
|
+
lastActivityAt: () => lastRunActivityAt,
|
|
287
|
+
alreadyStalled: runProjection.stallCount > 0
|
|
288
|
+
});
|
|
289
|
+
const appendRunStatus = (status, reason, force = false) => {
|
|
290
|
+
journal?.appendStatus(status, { actor: OPERATOR_ACTOR, reason, force });
|
|
291
|
+
};
|
|
292
|
+
const applyInboxResolution = (resolution) => {
|
|
293
|
+
if (resolution.kind === "approval") {
|
|
294
|
+
journal?.appendApprovalResolved({
|
|
295
|
+
requestId: resolution.requestId,
|
|
296
|
+
decision: resolution.decision,
|
|
297
|
+
...resolution.note !== undefined ? { note: resolution.note } : {},
|
|
298
|
+
actor: OPERATOR_ACTOR
|
|
299
|
+
});
|
|
300
|
+
journal?.appendTimeline({ type: "inbox-resolution", kind: "approval", requestId: resolution.requestId, decision: resolution.decision });
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
journal?.appendInputResolved({ requestId: resolution.requestId, answers: resolution.answers, actor: OPERATOR_ACTOR });
|
|
304
|
+
journal?.appendTimeline({ type: "inbox-resolution", kind: "input", requestId: resolution.requestId });
|
|
305
|
+
};
|
|
306
|
+
const applyRunControlText = (text) => {
|
|
307
|
+
const control = detectRunControlText(text, runId);
|
|
308
|
+
if (!control)
|
|
309
|
+
return;
|
|
310
|
+
markRunActivity();
|
|
311
|
+
if (control.kind === "pause") {
|
|
312
|
+
pendingPause = true;
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
if (control.kind === "resume") {
|
|
316
|
+
pendingPause = false;
|
|
317
|
+
pendingStopReason = undefined;
|
|
318
|
+
appendRunStatus("running", "operator resumed run", true);
|
|
319
|
+
publishRunProjection("running");
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
if (control.kind === "stop") {
|
|
323
|
+
pendingStopReason = control.reason ?? "operator requested stop";
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
applyInboxResolution(control.resolution);
|
|
327
|
+
};
|
|
328
|
+
const processRunControlMessages = () => {
|
|
329
|
+
for (const entry of ctx.sessionManager.getEntries?.() ?? []) {
|
|
330
|
+
const entryObject = entry && typeof entry === "object" ? entry : null;
|
|
331
|
+
const id = entryObject && typeof entryObject.id === "string" ? entryObject.id : null;
|
|
332
|
+
if (id && processedControlEntryIds.has(id))
|
|
333
|
+
continue;
|
|
334
|
+
if (!id && entryObject && processedControlEntryObjects.has(entryObject))
|
|
335
|
+
continue;
|
|
336
|
+
const text = textFromSessionEntry(entry);
|
|
337
|
+
if (id)
|
|
338
|
+
processedControlEntryIds.add(id);
|
|
339
|
+
else if (entryObject)
|
|
340
|
+
processedControlEntryObjects.add(entryObject);
|
|
341
|
+
if (text)
|
|
342
|
+
applyRunControlText(text);
|
|
343
|
+
}
|
|
344
|
+
};
|
|
345
|
+
let fatalTerminalPublishing = false;
|
|
346
|
+
const publishFatalTerminal = (error) => {
|
|
347
|
+
if (fatalTerminalPublishing)
|
|
348
|
+
return;
|
|
349
|
+
fatalTerminalPublishing = true;
|
|
350
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
351
|
+
journal?.appendStatus("failed", { actor: { kind: "agent" }, errorText: detail, force: true });
|
|
352
|
+
publishRunProjection("failed").catch((publishError) => {
|
|
353
|
+
console.error(`[rig-run] fatal-terminal-publish-failed ${publishError instanceof Error ? publishError.message : String(publishError)}`);
|
|
354
|
+
}).finally(() => {
|
|
355
|
+
stopRunStallMonitor();
|
|
356
|
+
stopRunRegistry({ removeRoom: false });
|
|
357
|
+
process.exitCode = 1;
|
|
358
|
+
setTimeout(() => process.exit(1), 0);
|
|
359
|
+
});
|
|
360
|
+
};
|
|
361
|
+
process.once("uncaughtException", publishFatalTerminal);
|
|
362
|
+
process.once("unhandledRejection", publishFatalTerminal);
|
|
363
|
+
const startRunCollabHost = async () => {
|
|
364
|
+
try {
|
|
365
|
+
const projection = await startHost.call(collab, {
|
|
366
|
+
title: runDisplayTitle,
|
|
367
|
+
...identity?.owner ? { owner: identity.owner } : {},
|
|
368
|
+
...identity?.selectedRepo ? { selectedRepo: identity.selectedRepo } : {},
|
|
369
|
+
relayUrl: rigRelayUrl()
|
|
370
|
+
});
|
|
371
|
+
runRegistryLinks = { joinLink: projection.joinLink, webLink: projection.webLink, relayUrl: projection.relayUrl ?? rigRelayUrl() };
|
|
372
|
+
const timeline = {
|
|
373
|
+
type: "collab-host-started",
|
|
374
|
+
roomId: projection.sessionId,
|
|
375
|
+
joinLink: projection.joinLink,
|
|
376
|
+
webLink: projection.webLink,
|
|
377
|
+
relayUrl: projection.relayUrl
|
|
378
|
+
};
|
|
379
|
+
journal?.appendTimeline(timeline);
|
|
380
|
+
console.log(`[rig-run] collab-host-started joinLink=${projection.joinLink || "(empty)"} relayUrl=${projection.relayUrl || "(empty)"}`);
|
|
381
|
+
ctx.ui.notify("Rig run collab host started.", "info");
|
|
382
|
+
if (runRegistry && identity?.owner) {
|
|
383
|
+
try {
|
|
384
|
+
runRegistryRoomId = projection.sessionId;
|
|
385
|
+
const initialProjection = buildCurrentRegistryProjection();
|
|
386
|
+
await runRegistry.registerRoom({
|
|
387
|
+
roomId: projection.sessionId,
|
|
388
|
+
owner: identity.owner,
|
|
389
|
+
repo: identity.selectedRepo ?? "",
|
|
390
|
+
title: runDisplayTitle,
|
|
391
|
+
status: "running",
|
|
392
|
+
joinLink: projection.joinLink ?? "",
|
|
393
|
+
webLink: projection.webLink ?? "",
|
|
394
|
+
relayUrl: projection.relayUrl ?? rigRelayUrl(),
|
|
395
|
+
startedAt: new Date().toISOString(),
|
|
396
|
+
cwd: process.cwd(),
|
|
397
|
+
sessionPath: ctx.sessionManager.getSessionFile() ?? undefined,
|
|
398
|
+
pid: process.pid,
|
|
399
|
+
projection: initialProjection
|
|
400
|
+
});
|
|
401
|
+
workerProjectionConnection = connectWorkerProjection({ baseUrl: registryBaseUrl(), namespaceKey: runRegistryNamespace, runId: projection.sessionId });
|
|
402
|
+
workerProjectionConnection.ready.catch((err) => console.error(`[rig-run] worker-projection-degraded (heartbeat remains authoritative): ${err instanceof Error ? err.message : String(err)}`));
|
|
403
|
+
pushWorkerProjection();
|
|
404
|
+
workerProjectionFiber = Effect.runFork(localRunChanges(projectRoot).pipe(Stream.debounce(Duration.millis(500)), Stream.runForEach(() => Effect.sync(() => pushWorkerProjection()))));
|
|
405
|
+
journal?.appendTimeline({ type: "registry-registered", roomId: projection.sessionId });
|
|
406
|
+
console.log(`[rig-run] registry-registered roomId=${projection.sessionId}`);
|
|
407
|
+
runRegistryHeartbeat = setInterval(() => {
|
|
408
|
+
const heartbeatProjection = buildCurrentRegistryProjection();
|
|
409
|
+
runRegistry.heartbeatRoom(projection.sessionId, coerceRegistryStatus(heartbeatProjection.status, "running"), heartbeatProjection).catch((err) => console.error(`[rig-run] registry-heartbeat-failed ${err instanceof Error ? err.message : String(err)}`));
|
|
410
|
+
}, 15000);
|
|
411
|
+
if (typeof runRegistryHeartbeat.unref === "function")
|
|
412
|
+
runRegistryHeartbeat.unref();
|
|
413
|
+
} catch (error) {
|
|
414
|
+
console.error(`[rig-run] registry-register-failed ${error instanceof Error ? error.message : String(error)}`);
|
|
415
|
+
ctx.ui.notify("Rig run could not register to the discovery registry; it may not appear in Runs.", "warning");
|
|
416
|
+
}
|
|
417
|
+
} else {
|
|
418
|
+
console.error(`[rig-run] registry-skip namespace=${runRegistryNamespace ? "set" : "MISSING"}`);
|
|
419
|
+
ctx.ui.notify("Rig run NOT registered (registry namespace missing) \u2014 it won't appear in Runs.", "warning");
|
|
420
|
+
}
|
|
421
|
+
} catch (error) {
|
|
422
|
+
console.error(`[rig-run] collab-host-start-failed ${error instanceof Error ? error.message : String(error)}`);
|
|
423
|
+
ctx.ui.notify("Rig run collab host could not reach the relay; session remains local.", "warning");
|
|
424
|
+
}
|
|
425
|
+
};
|
|
426
|
+
journal?.appendTimeline({ type: "stage", stage: "Connect", status: "running", detail: "run process session_start" });
|
|
427
|
+
journal?.appendTimeline({ type: "stage", stage: "Prepare", status: "completed", detail: "run process prepared" });
|
|
428
|
+
journal?.appendStatus("running", { actor: { kind: "agent" }, reason: "run process session started", force: true });
|
|
429
|
+
await startRunCollabHost();
|
|
430
|
+
if (taskIdAtStart) {
|
|
431
|
+
journal?.appendTimeline({ type: "stage", stage: "GitHub-sync", status: "running", detail: "reflecting rig:running to task source" });
|
|
432
|
+
try {
|
|
433
|
+
await updateRunTaskSourceLifecycle(projectRoot, {
|
|
434
|
+
runId,
|
|
435
|
+
taskId: taskIdAtStart,
|
|
436
|
+
sourceTask: runProjection.record.sourceTask,
|
|
437
|
+
worktreePath: process.cwd(),
|
|
438
|
+
logRoot: runProjection.record.logRoot ?? null,
|
|
439
|
+
sessionPath: runProjection.record.sessionPath ?? null
|
|
440
|
+
}, "running", "Rig started work on this task.");
|
|
441
|
+
journal?.appendTimeline({ type: "stage", stage: "GitHub-sync", status: "completed", detail: "reflected running" });
|
|
442
|
+
} catch (error) {
|
|
443
|
+
journal?.appendTimeline({ type: "stage", stage: "GitHub-sync", status: "failed", detail: error instanceof Error ? error.message : String(error) });
|
|
444
|
+
console.error(`[rig-run] running-reflection-failed ${error instanceof Error ? error.message : String(error)}`);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
const steerPi = async (message) => {
|
|
448
|
+
journal?.appendTimeline({ type: "closeout-steer", message });
|
|
449
|
+
api.sendUserMessage(message, { deliverAs: "steer" });
|
|
450
|
+
};
|
|
451
|
+
const agentEnd = async (event) => {
|
|
452
|
+
markRunActivity();
|
|
453
|
+
processRunControlMessages();
|
|
454
|
+
for (const message of event.messages) {
|
|
455
|
+
const text = textFromAgentMessage(message);
|
|
456
|
+
if (text)
|
|
457
|
+
applyRunControlText(text);
|
|
458
|
+
}
|
|
459
|
+
const aborted = event.messages.some((message) => message.role === "assistant" && message.stopReason === "aborted");
|
|
460
|
+
if (pendingStopReason !== undefined) {
|
|
461
|
+
const reason = pendingStopReason ?? "operator requested stop";
|
|
462
|
+
pendingStopReason = undefined;
|
|
463
|
+
closeoutStarted = true;
|
|
464
|
+
journal?.appendTimeline({ type: "interrupted", stage: "stopped", status: "stopped", detail: reason });
|
|
465
|
+
journal?.appendStatus("stopped", { actor: OPERATOR_ACTOR, reason, force: true });
|
|
466
|
+
await publishRunProjection("stopped");
|
|
467
|
+
stopRunStallMonitor();
|
|
468
|
+
stopRunRegistry({ removeRoom: false });
|
|
469
|
+
process.exitCode = 0;
|
|
470
|
+
setTimeout(() => process.exit(0), 0);
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
if (aborted) {
|
|
474
|
+
if (pendingPause) {
|
|
475
|
+
pendingPause = false;
|
|
476
|
+
appendRunStatus("paused", "operator paused run", true);
|
|
477
|
+
journal?.appendTimeline({ type: "interrupted", stage: "paused", status: "paused", detail: "operator paused run; parked for native collab resume" });
|
|
478
|
+
await publishRunProjection("paused");
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
journal?.appendTimeline({ type: "interrupted", stage: "aborted", status: "running", detail: "operator interrupted run without a Rig control sentinel" });
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
if (closeoutStarted)
|
|
485
|
+
return;
|
|
486
|
+
if (process.env.RIG_RUN_FORCE_STEER_ONCE === "1" && !forcedSteerUsed) {
|
|
487
|
+
forcedSteerUsed = true;
|
|
488
|
+
journal?.appendTimeline({ type: "force-steer-once-started" });
|
|
489
|
+
await steerPi("now reply with the word FIXED");
|
|
490
|
+
journal?.appendTimeline({ type: "force-steer-once-completed" });
|
|
491
|
+
}
|
|
492
|
+
closeoutStarted = true;
|
|
493
|
+
runProjection = projectRunFromSession(customEntries(ctx.sessionManager.getBranch()), runId);
|
|
494
|
+
const taskId = process.env.RIG_TASK_ID?.trim() || runProjection.record.taskId;
|
|
495
|
+
if (!taskId) {
|
|
496
|
+
const reason = "missing task id for closeout";
|
|
497
|
+
journal?.appendTimeline({ type: "closeout-skipped", reason: "missing-task-id" });
|
|
498
|
+
journal?.appendStatus("failed", { actor: { kind: "agent" }, reason, errorText: reason, force: true });
|
|
499
|
+
await publishRunProjection("failed");
|
|
500
|
+
stopRunStallMonitor();
|
|
501
|
+
stopRunRegistry({ removeRoom: false });
|
|
502
|
+
process.exitCode = 1;
|
|
503
|
+
setTimeout(() => process.exit(1), 0);
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
const { command, gitCommand } = createEnvCloseoutRunners(process.env);
|
|
507
|
+
let closeoutStatusAdvanced = false;
|
|
508
|
+
try {
|
|
509
|
+
await runInProcessCloseout({
|
|
510
|
+
projectRoot,
|
|
511
|
+
runId,
|
|
512
|
+
taskId,
|
|
513
|
+
branch: runProjection.record.branch ?? `rig/${taskId}-${runId}`,
|
|
514
|
+
workspace: process.cwd(),
|
|
515
|
+
artifactRoot: runProjection.record.artifactRoot ?? null,
|
|
516
|
+
sourceTask: runProjection.record.sourceTask && typeof runProjection.record.sourceTask === "object" && !Array.isArray(runProjection.record.sourceTask) ? runProjection.record.sourceTask : null,
|
|
517
|
+
command,
|
|
518
|
+
gitCommand,
|
|
519
|
+
steerPi,
|
|
520
|
+
onValidationStart: async () => {
|
|
521
|
+
journal?.appendStatus("validating", { actor: { kind: "agent" }, reason: "validating task before closeout", force: true });
|
|
522
|
+
await publishRunProjection("validating");
|
|
523
|
+
},
|
|
524
|
+
journalPhase: async (phase, outcome, detail) => {
|
|
525
|
+
journal?.appendCloseoutPhase({ phase, outcome, detail: detail ?? null });
|
|
526
|
+
if (!closeoutStatusAdvanced && phase !== "queued" && outcome === "started") {
|
|
527
|
+
closeoutStatusAdvanced = true;
|
|
528
|
+
journal?.appendStatus("closing-out", { actor: { kind: "agent" }, reason: "validation passed; running closeout automation", force: true });
|
|
529
|
+
await publishRunProjection("closing-out");
|
|
530
|
+
}
|
|
531
|
+
},
|
|
532
|
+
reflect: async (status, summary, opts) => {
|
|
533
|
+
await updateRunTaskSourceLifecycle(projectRoot, {
|
|
534
|
+
runId,
|
|
535
|
+
taskId,
|
|
536
|
+
sourceTask: runProjection.record.sourceTask,
|
|
537
|
+
worktreePath: process.cwd(),
|
|
538
|
+
logRoot: runProjection.record.logRoot ?? null,
|
|
539
|
+
sessionPath: runProjection.record.sessionPath ?? null
|
|
540
|
+
}, status, summary, opts);
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
journal?.appendStatus("completed", { actor: { kind: "agent" }, reason: "closeout completed" });
|
|
544
|
+
await publishRunProjection("completed");
|
|
545
|
+
await dispatchRunNotifications(projectRoot, runId, taskId, "completed", "closeout completed");
|
|
546
|
+
stopRunStallMonitor();
|
|
547
|
+
stopRunRegistry({ removeRoom: false });
|
|
548
|
+
process.exitCode = 0;
|
|
549
|
+
setTimeout(() => process.exit(0), 0);
|
|
550
|
+
} catch (error) {
|
|
551
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
552
|
+
const status = error instanceof CloseoutValidationError ? "needs-attention" : "failed";
|
|
553
|
+
journal?.appendStatus(status, { actor: { kind: "agent" }, errorText: detail, force: true });
|
|
554
|
+
await publishRunProjection(status);
|
|
555
|
+
await dispatchRunNotifications(projectRoot, runId, taskId, "failed", detail);
|
|
556
|
+
stopRunStallMonitor();
|
|
557
|
+
stopRunRegistry({ removeRoom: false });
|
|
558
|
+
process.exitCode = 1;
|
|
559
|
+
setTimeout(() => process.exit(1), 0);
|
|
560
|
+
}
|
|
561
|
+
};
|
|
562
|
+
const handleRunActivity = () => {
|
|
563
|
+
markRunActivity();
|
|
564
|
+
processRunControlMessages();
|
|
565
|
+
};
|
|
566
|
+
return { beforeAgentStart: handleRunActivity, messageStart: handleRunActivity, agentEnd };
|
|
567
|
+
}
|
|
568
|
+
async function maybeStartSpikeAutohost(ctx) {
|
|
569
|
+
if (process.env.RIG_SPIKE_AUTOHOST !== "1")
|
|
570
|
+
return;
|
|
571
|
+
const relayUrl = process.env.RIG_SPIKE_RELAY?.trim();
|
|
572
|
+
if (!relayUrl)
|
|
573
|
+
throw new Error("RIG_SPIKE_RELAY is required when RIG_SPIKE_AUTOHOST=1");
|
|
574
|
+
const collab = ctx.collab;
|
|
575
|
+
const startHost = collab?.startHost ?? collab?.startCollabHost;
|
|
576
|
+
if (!startHost)
|
|
577
|
+
throw new Error("OMP collab host facade is unavailable for detached PTY spike.");
|
|
578
|
+
const identity = resolveRigIdentity(ctx);
|
|
579
|
+
await startHost.call(collab, {
|
|
580
|
+
relayUrl,
|
|
581
|
+
title: "Rig detached PTY spike",
|
|
582
|
+
...identity?.owner ? { owner: identity.owner } : {},
|
|
583
|
+
...identity?.selectedRepo ? { selectedRepo: identity.selectedRepo } : {}
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// packages/run-worker/src/extension.ts
|
|
588
|
+
var __rigRunWorkerTest = {
|
|
589
|
+
computeRunStall,
|
|
590
|
+
appendRunStallDetected,
|
|
591
|
+
detectRunControlText
|
|
592
|
+
};
|
|
593
|
+
var __rigExtensionTest = __rigRunWorkerTest;
|
|
594
|
+
function rigWorkerExtension(api) {
|
|
595
|
+
let hooks = null;
|
|
596
|
+
api.on("session_start", async (_event, ctx) => {
|
|
597
|
+
hooks = await maybeStartRunProcessAutohost(api, ctx);
|
|
598
|
+
await maybeStartSpikeAutohost(ctx);
|
|
599
|
+
});
|
|
600
|
+
api.on("before_agent_start", () => {
|
|
601
|
+
hooks?.beforeAgentStart();
|
|
602
|
+
});
|
|
603
|
+
api.on("message_start", () => {
|
|
604
|
+
hooks?.messageStart();
|
|
605
|
+
});
|
|
606
|
+
api.on("agent_end", async (event) => {
|
|
607
|
+
await hooks?.agentEnd(event);
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
export {
|
|
611
|
+
timestampMs,
|
|
612
|
+
startRunProcessStallMonitor,
|
|
613
|
+
registryRunProjection,
|
|
614
|
+
maybeStartSpikeAutohost,
|
|
615
|
+
maybeStartRunProcessAutohost,
|
|
616
|
+
dispatchRunNotifications,
|
|
617
|
+
detectRunControlText,
|
|
618
|
+
rigWorkerExtension as default,
|
|
619
|
+
createRunJournal,
|
|
620
|
+
computeRunStall,
|
|
621
|
+
appendRunStallDetected,
|
|
622
|
+
__rigRunWorkerTest,
|
|
623
|
+
__rigExtensionTest,
|
|
624
|
+
TRACKED_RUN_STALL_MS,
|
|
625
|
+
RUN_PROCESS_STEER_TIMEOUT_MS,
|
|
626
|
+
RUN_PROCESS_STALL_SWEEP_MS,
|
|
627
|
+
RUN_PROCESS_STALL_DETAIL,
|
|
628
|
+
REGISTRY_PROJECTION_TIMELINE_LIMIT
|
|
629
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ExtensionContext } from "@oh-my-pi/pi-coding-agent";
|
|
2
|
+
import type { ApprovalDecision, RunActor, RunCloseoutPhase, RunCloseoutPhaseOutcome, RunStatus } from "@rig/contracts";
|
|
3
|
+
export type RunJournal = {
|
|
4
|
+
appendStatus(to: RunStatus, opts?: {
|
|
5
|
+
reason?: string | null;
|
|
6
|
+
actor?: RunActor;
|
|
7
|
+
errorText?: string | null;
|
|
8
|
+
force?: boolean;
|
|
9
|
+
}): void;
|
|
10
|
+
appendTimeline(payload: unknown): void;
|
|
11
|
+
appendCloseoutPhase(input: {
|
|
12
|
+
phase: RunCloseoutPhase;
|
|
13
|
+
outcome: RunCloseoutPhaseOutcome;
|
|
14
|
+
detail?: string | null;
|
|
15
|
+
}): void;
|
|
16
|
+
appendApprovalResolved(input: {
|
|
17
|
+
requestId: string;
|
|
18
|
+
decision: ApprovalDecision;
|
|
19
|
+
note?: string | null;
|
|
20
|
+
actor: RunActor;
|
|
21
|
+
}): void;
|
|
22
|
+
appendInputResolved(input: {
|
|
23
|
+
requestId: string;
|
|
24
|
+
answers: Record<string, string>;
|
|
25
|
+
actor: RunActor;
|
|
26
|
+
}): void;
|
|
27
|
+
appendStall(input: {
|
|
28
|
+
detail: string;
|
|
29
|
+
}): void;
|
|
30
|
+
};
|
|
31
|
+
export declare function createRunJournal(sessionManager: ExtensionContext["sessionManager"], runId: string): Promise<RunJournal | null>;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
// packages/run-worker/src/journal.ts
|
|
3
|
+
import { RunSessionJournal } from "@rig/runtime/control-plane/run-session-writer";
|
|
4
|
+
async function createRunJournal(sessionManager, runId) {
|
|
5
|
+
try {
|
|
6
|
+
return new RunSessionJournal(sessionManager, runId);
|
|
7
|
+
} catch (error) {
|
|
8
|
+
console.warn(`[rig-run] RunSessionJournal unavailable; run-state arming deferred: ${error instanceof Error ? error.message : String(error)}`);
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export {
|
|
13
|
+
createRunJournal
|
|
14
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function dispatchRunNotifications(projectRoot: string, runId: string, taskId: string, outcome: "completed" | "failed", detail: string | null): Promise<void>;
|