@volter/supercode-orchestrator 0.5.109 → 0.5.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents.mjs +25 -0
- package/board/ci.mjs +62 -9
- package/board/cli.mjs +1 -0
- package/board/dispatch.mjs +106 -24
- package/board/incidents.mjs +39 -2
- package/board/machine-fleet.mjs +4 -1
- package/board/session-work.mjs +30 -0
- package/board/workflow.mjs +8 -0
- package/package.json +1 -1
package/agents.mjs
CHANGED
|
@@ -174,6 +174,31 @@ export function localAgentRecord(root,org,id){
|
|
|
174
174
|
export function localAgentSessions(root,org,id){
|
|
175
175
|
const db=store(root);try{return db.prepare('SELECT session_ref AS session,machine,ended_at,end_reason FROM agent_runs WHERE org=? AND agent_id=? ORDER BY rowid DESC LIMIT 200').all(org,id);}finally{db.close();}
|
|
176
176
|
}
|
|
177
|
+
/** Retained starter receipts for this home's sessions, including explicit forks.
|
|
178
|
+
* Ended owners remain in the join so a live child keeps its original starter. */
|
|
179
|
+
export function localSessionDelegations(root, localMachine) {
|
|
180
|
+
const db = store(root);
|
|
181
|
+
try {
|
|
182
|
+
const rows = db.prepare(`SELECT r.*, json_extract(i.declaration_json,'$.starterSession') AS declared_starter
|
|
183
|
+
FROM agent_runs r LEFT JOIN agent_intents i ON i.org=r.org AND i.id=r.agent_id`).all();
|
|
184
|
+
const addresses = new Map();
|
|
185
|
+
const address = row => row.session_ref.startsWith('sc:') ? row.session_ref
|
|
186
|
+
: /^(claude-code|codex):[^:]+$/.test(row.session_ref) ? `sc:${row.machine ?? localMachine}:${row.session_ref}` : null;
|
|
187
|
+
for (const row of rows) {
|
|
188
|
+
const key = `${row.org}|${row.session_ref}`, value = address(row);
|
|
189
|
+
if (value) addresses.set(key, value);
|
|
190
|
+
}
|
|
191
|
+
for (const alias of db.prepare('SELECT * FROM agent_runtime_sessions').all()) {
|
|
192
|
+
const value = addresses.get(`${alias.org}|${alias.session_ref}`);
|
|
193
|
+
if (value) addresses.set(`${alias.org}|${alias.runtime_ref}`, value);
|
|
194
|
+
}
|
|
195
|
+
return rows.filter(row => row.ended_at == null).map(row => {
|
|
196
|
+
const ref = row.starter_session ?? row.declared_starter;
|
|
197
|
+
return { session: address(row), starter: typeof ref === 'string' && ref.startsWith('sc:')
|
|
198
|
+
? ref : addresses.get(`${row.org}|${ref}`) ?? null };
|
|
199
|
+
}).filter(edge => edge.session && edge.starter);
|
|
200
|
+
} finally { db.close(); }
|
|
201
|
+
}
|
|
177
202
|
/** Move identities minted before any Teams context into the Org that is now selected, keeping their ids. */
|
|
178
203
|
function adoptLocalOrg(root,scope){
|
|
179
204
|
const db=store(root);
|
package/board/ci.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { execFile } from 'node:child_process';
|
|
|
4
4
|
import { readFileSync, realpathSync, writeFileSync, renameSync, statSync } from 'node:fs';
|
|
5
5
|
import { dirname, join, relative } from 'node:path';
|
|
6
6
|
import { createHash } from 'node:crypto';
|
|
7
|
-
import { branchCheckouts, checkoutOf, isLocalWorkspace, resolveWorkspace } from './workspace.mjs';
|
|
7
|
+
import { branchCheckouts, checkoutOf, isLocalWorkspace, resolveWorkspace, plainWorkspace } from './workspace.mjs';
|
|
8
8
|
import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
|
|
9
9
|
|
|
10
10
|
// GitHub is read through its REST API with Node's own fetch, in this process and awaited (never a child the caller
|
|
@@ -134,16 +134,50 @@ async function repositoryWorkspace(board, id) {
|
|
|
134
134
|
const workspace = task?.workspace_kind === 'scratch' ? parent : task;
|
|
135
135
|
if (!workspace || workspace.workspace_kind === 'scratch') return null;
|
|
136
136
|
const here = isLocalWorkspace(workspace);
|
|
137
|
-
if (workspace.workspace_anchor)
|
|
137
|
+
if (workspace.workspace_anchor) {
|
|
138
|
+
// A portable remote declaration supplies the source identity. Reviewing
|
|
139
|
+
// its pushed head does not require that machine to keep the old checkout.
|
|
140
|
+
if (!here) {
|
|
141
|
+
const declared = plainWorkspace(workspace.workspace_kind, workspace.workspace_anchor, workspace.branch_name);
|
|
142
|
+
if (declared) return { ...declared, here: false };
|
|
143
|
+
}
|
|
144
|
+
const resolved = await resolveWorkspace(workspace, false);
|
|
145
|
+
// Resolution without create names a solo worktree even before it exists on
|
|
146
|
+
// this machine. Its portable repository and branch still identify the
|
|
147
|
+
// pushed PR. Only an actual directory can supply local source evidence.
|
|
148
|
+
return { ...resolved, here: here && checkoutPresent(resolved.path) };
|
|
149
|
+
}
|
|
138
150
|
return workspace.workspace_path ? { path: workspace.workspace_path, here } : null;
|
|
139
151
|
}
|
|
140
152
|
|
|
141
153
|
const pullRequestLists = new Map();
|
|
154
|
+
const mergedPullRequestLists = new Map();
|
|
155
|
+
|
|
156
|
+
/** Historical work can already be merged when an outage prevented the board's
|
|
157
|
+
* review request. Read only the arc's declared branch, never all closed PRs. */
|
|
158
|
+
async function mergedBranchPullRequests(repository, branch) {
|
|
159
|
+
const key = `${repository}:${branch}`;
|
|
160
|
+
let list = mergedPullRequestLists.get(key);
|
|
161
|
+
if (!list || Date.now() - list.at >= 60_000) {
|
|
162
|
+
const head = encodeURIComponent(`${repository.split('/')[0]}:${branch}`);
|
|
163
|
+
const items = (await github(`/repos/${repository}/pulls?state=closed&head=${head}&per_page=100`, true))
|
|
164
|
+
.filter(p => p.merged_at).map(p => pullRequest(repository, p));
|
|
165
|
+
if (mergedPullRequestLists.size >= 64) mergedPullRequestLists.delete(mergedPullRequestLists.keys().next().value);
|
|
166
|
+
list = { at: Date.now(), items }; mergedPullRequestLists.set(key, list);
|
|
167
|
+
}
|
|
168
|
+
return list.items;
|
|
169
|
+
}
|
|
142
170
|
|
|
143
171
|
async function branchCheckout(board, id, repository, branch) {
|
|
144
172
|
const task = board.db.prepare('SELECT c.machine FROM tasks t LEFT JOIN supercode_cards c ON c.task_id=t.id WHERE t.id=?').get(id);
|
|
145
173
|
if (!isLocalWorkspace(task ?? {})) return null;
|
|
146
|
-
try {
|
|
174
|
+
try { const path = (await checkoutOf(task?.machine, repository, branch))?.path; return checkoutPresent(path) ? path : null; } catch { return null; }
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function checkoutPresent(path) {
|
|
178
|
+
if (!path) return false;
|
|
179
|
+
try { return statSync(path).isDirectory(); }
|
|
180
|
+
catch (error) { if (['ENOENT', 'ENOTDIR'].includes(error.code)) return false; throw error; }
|
|
147
181
|
}
|
|
148
182
|
|
|
149
183
|
/** The sibling checkouts an arc registered: each subtask (and the card itself) that declares a repository
|
|
@@ -174,7 +208,7 @@ export function declaredWorkBranches(db, id) {
|
|
|
174
208
|
return [...new Set([...declared, ...reviewed].filter(Boolean))];
|
|
175
209
|
}
|
|
176
210
|
|
|
177
|
-
/** Every opened PR of the arc, each proposed to its repository's default branch: an open PR whose head is the card's
|
|
211
|
+
/** Every opened PR of the arc, plus an unjudged merged PR on its declared branch, each proposed to its repository's default branch: an open PR whose head is the card's
|
|
178
212
|
* work branch in the card's own repository, a `wt/<card id>` branch in any repository this machine keeps a checkout
|
|
179
213
|
* of, or a sibling checkout the arc registered (`registeredBranches`). The repository's default branch, declared or
|
|
180
214
|
* checked out (or no branch), is not a work branch: such a card's PRs are only its `wt/<card>` and registered ones, and with none it
|
|
@@ -195,20 +229,39 @@ export async function openReviewPullRequests(board, id) {
|
|
|
195
229
|
// Discovery is shared per repository for one minute; each selected PR/head is re-read before it authorizes a move.
|
|
196
230
|
let list = pullRequestLists.get(repository);
|
|
197
231
|
if (!list || Date.now() - list.at >= 60_000) {
|
|
198
|
-
let items = [];
|
|
199
|
-
try {
|
|
232
|
+
let items = [], defaultBranch = null;
|
|
233
|
+
try {
|
|
234
|
+
items = (await github(`/repos/${repository}/pulls?state=open&per_page=100`, true)).map((p) => pullRequest(repository, p));
|
|
235
|
+
defaultBranch = items.find(pr => pr.defaultBranch)?.defaultBranch ?? (await github(`/repos/${repository}`)).default_branch;
|
|
236
|
+
}
|
|
200
237
|
catch (error) { if (repository === ownRepository || registered.some((r) => r.repository === repository)) throw error; } // another repository unreachable here is not this arc's
|
|
201
|
-
list = { at: Date.now(), items };
|
|
238
|
+
list = { at: Date.now(), items, defaultBranch };
|
|
202
239
|
if (pullRequestLists.size >= 64) pullRequestLists.delete(pullRequestLists.keys().next().value);
|
|
203
240
|
pullRequestLists.set(repository, list);
|
|
204
241
|
}
|
|
205
242
|
// The repository's default branch is not a work branch, declared or checked out (a card working in a `dir` on main).
|
|
206
243
|
if (repository === ownRepository && branch) {
|
|
207
|
-
|
|
208
|
-
if (branch === defaultBranch) branch = null;
|
|
244
|
+
if (branch === list.defaultBranch) branch = null;
|
|
209
245
|
}
|
|
210
246
|
// a draft is not handed over yet: its review starts when it is marked ready; a PR to another base (main→prod) is a release, not the arc's work
|
|
211
247
|
for (const pr of list.items) if (!pr.isCrossRepository && !pr.isDraft && pr.baseRefName === pr.defaultBranch && mine(repository, pr.headRefName)) prs.push({ url: pr.url, head: pr.headRefOid });
|
|
248
|
+
// A manual review and merge during a dispatcher outage may predate any
|
|
249
|
+
// board request. The source-review door already accepts merged heads; make
|
|
250
|
+
// them discoverable so the ordinary independent verdict can be retained.
|
|
251
|
+
// An open PR (including a draft or wrong base) on that branch takes priority.
|
|
252
|
+
const historical = [...new Set([
|
|
253
|
+
...(repository === ownRepository ? branch ? [branch] : [`wt/${id}`, `wt/${id.replace(/^t_/, 't-')}`] : []),
|
|
254
|
+
...registered.filter(r => r.repository === repository).map(r => r.branch),
|
|
255
|
+
])];
|
|
256
|
+
for (const headBranch of historical) {
|
|
257
|
+
if (headBranch === list.defaultBranch) continue;
|
|
258
|
+
if (list.items.some(pr => pr.headRefName === headBranch)) continue;
|
|
259
|
+
for (const pr of await mergedBranchPullRequests(repository, headBranch)) {
|
|
260
|
+
if (!pr.isCrossRepository && pr.headRefName === headBranch && pr.headRefName !== pr.defaultBranch
|
|
261
|
+
&& pr.baseRefName === pr.defaultBranch && !judgedHead(board, id, pr.headRefOid)
|
|
262
|
+
&& !prs.some(known => known.url === pr.url)) prs.push({ url: pr.url, head: pr.headRefOid });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
212
265
|
}
|
|
213
266
|
if (!prs.length) return branch ? null : { waits: `the card has no work branch (${task?.branch_name ? `it declares ${task.branch_name}, its repository's default` : current ? `its checkout ${own.path} is on ${current}, its repository's default` : `its checkout ${own.path} is on no branch`}): give it one with --branch, or open its PR from a wt/${id} branch` };
|
|
214
267
|
return { pr: prs.map((p) => p.url), pr_head: prs.map((p) => p.head), branch };
|
package/board/cli.mjs
CHANGED
|
@@ -1701,6 +1701,7 @@ async function serve(root, args = {}) {
|
|
|
1701
1701
|
const refused = named?.refused ?? null;
|
|
1702
1702
|
log(refusal ? `board events: not published; the machine daemon did not take this home: ${refusal}`
|
|
1703
1703
|
: refused ? `board events: registered, but the team refused this machine's credential (${refused.status}); nothing is published until it is re-enrolled`
|
|
1704
|
+
: named?.published === false ? `board ownership: registered locally; this machine has no team connector to publish events`
|
|
1704
1705
|
: `board events: this home is published by the machine's connector`); }
|
|
1705
1706
|
log('dispatch wakes on board writes, document edits and board mail; the interval sweep reads each machine\'s own door');
|
|
1706
1707
|
const stuck = {};
|
package/board/dispatch.mjs
CHANGED
|
@@ -20,8 +20,11 @@ import { machineCall } from './doors.mjs';
|
|
|
20
20
|
import { processSession } from './caller-session.mjs';
|
|
21
21
|
import { callerOf, current, isCallersCall, ownCall } from './call.mjs';
|
|
22
22
|
import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
|
|
23
|
-
import { monitorsRuns, failedObservation, surfaceIncident, auditEndedRuns, originOf } from './incidents.mjs';
|
|
23
|
+
import { monitorsRuns, recoversRuns, retainRecovery, failedObservation, surfaceIncident, auditEndedRuns, originOf } from './incidents.mjs';
|
|
24
24
|
import { readMachineFleet } from './machine-fleet.mjs';
|
|
25
|
+
import { delegatedWork, idleInterval } from './session-work.mjs';
|
|
26
|
+
import { localSessionDelegations } from '../agents.mjs';
|
|
27
|
+
import { creatorOf, sessionProgress } from './workflow.mjs';
|
|
25
28
|
import { boardLabel, boardName } from '@volter/teams/board-name';
|
|
26
29
|
let fleetRoot = null;
|
|
27
30
|
import { inspectCompletionEvidence, noCodeChange, openReviewPullRequests, reviewDue, withCompletionEvidence, reviewCandidate, workflowRevision, retainWorkflowTick } from './ci.mjs';
|
|
@@ -376,6 +379,7 @@ async function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
376
379
|
if (bucket) report[bucket].push(id);
|
|
377
380
|
return;
|
|
378
381
|
}
|
|
382
|
+
if (recoversRuns(entry) && failedObservation(name)) retainRecovery(db, id, name, payload);
|
|
379
383
|
const r = fire(db, board, entry, id, name, { roles: ['platform'], payload });
|
|
380
384
|
staged.push(...r.effects);
|
|
381
385
|
if (bucket) report[bucket].push(id);
|
|
@@ -778,6 +782,7 @@ function launchFailed(board, entry, id, error, payload) {
|
|
|
778
782
|
return tx(board, (db) => {
|
|
779
783
|
if (!runCurrent(db, id, error.launchRun)) return null;
|
|
780
784
|
if (monitorsRuns(entry)) { surfaceIncident(db, board, entry, id, 'spawn_failed', payload, error.launchRun); return { effects: [] }; }
|
|
785
|
+
if (recoversRuns(entry)) retainRecovery(db, id, 'spawn_failed', payload, error.launchRun);
|
|
781
786
|
return fire(db, board, entry, id, 'spawn_failed', { roles: ['platform'], payload });
|
|
782
787
|
});
|
|
783
788
|
}
|
|
@@ -1103,7 +1108,7 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
|
|
|
1103
1108
|
}
|
|
1104
1109
|
}
|
|
1105
1110
|
|
|
1106
|
-
/**
|
|
1111
|
+
/** Native idle intervals are independent of dispatch cadence and transcript writes.
|
|
1107
1112
|
* Keep bounded readings on the run so an ordinary quiet stretch can be reviewed later. */
|
|
1108
1113
|
async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
1109
1114
|
if (!fleet.ok) return;
|
|
@@ -1116,8 +1121,30 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1116
1121
|
return Number(rows?.find((row) => row.locator?.session_id === session.session_id)?.updated_at_ms) || null;
|
|
1117
1122
|
};
|
|
1118
1123
|
const managers = [].concat(entry.params.managers ?? []).filter(Boolean);
|
|
1124
|
+
const delegations = localSessionDelegations(board.root, localMachine());
|
|
1125
|
+
const delegatedReadings = [...fleet.sessions];
|
|
1126
|
+
// Forks and concurrent sessions need not have a pane. Read only the exact
|
|
1127
|
+
// native identities this home's starter receipts name, before any write.
|
|
1128
|
+
for (const { session: address } of delegations) {
|
|
1129
|
+
if (delegatedReadings.some(row => row.address === address)) continue;
|
|
1130
|
+
const match = /^sc:([^:]+):(claude-code|codex):([^:]+)$/.exec(address);
|
|
1131
|
+
if (!match) continue;
|
|
1132
|
+
const [, machine, harness, session_id] = match;
|
|
1133
|
+
const child = { machine, harness, session_id, surface: 'native' };
|
|
1134
|
+
await readBefore(fleet, child);
|
|
1135
|
+
try {
|
|
1136
|
+
const { row } = nativeReading(nativeSessions(machine, harness, session_id), child);
|
|
1137
|
+
if (row?.activity) delegatedReadings.push({ address, state: row.activity.presence, turn: row.activity.turn });
|
|
1138
|
+
} catch { /* an unread child is never called working */ }
|
|
1139
|
+
}
|
|
1119
1140
|
const running = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
|
|
1120
|
-
|
|
1141
|
+
// every running card a session is on: a session waiting on its user is told once, naming them all
|
|
1142
|
+
const cardsOf = new Map();
|
|
1143
|
+
for (const { id } of running) {
|
|
1144
|
+
const session = currentRun(board.db, id)?.metadata?.supercode; await readBefore(fleet, session);
|
|
1145
|
+
await refreshCardInputs(board, id, { commits: true });
|
|
1146
|
+
if (addressOf(session)) cardsOf.set(addressOf(session), [...(cardsOf.get(addressOf(session)) ?? []), id]);
|
|
1147
|
+
}
|
|
1121
1148
|
for (const { id } of running) {
|
|
1122
1149
|
let staged = [];
|
|
1123
1150
|
txRead(board, (db) => {
|
|
@@ -1132,8 +1159,13 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1132
1159
|
// a session in no pane has its turn on its machine's discovery row, which sessionAlive just read
|
|
1133
1160
|
let nativeRow = null;
|
|
1134
1161
|
if (session.surface === 'native') try { nativeRow = nativeReading(nativeSessions(session.machine, session.harness ?? 'claude-code', session.session_id), session).row; } catch { /* unread: no turn */ }
|
|
1135
|
-
const live = fleet.sessions.find((x) => x.address === addressOf(session)) ?? (nativeRow ? { address: addressOf(session), turn: nativeRow.activity?.turn ?? null
|
|
1162
|
+
const live = fleet.sessions.find((x) => x.address === addressOf(session)) ?? (nativeRow ? { address: addressOf(session), turn: nativeRow.activity?.turn ?? null, waiting_for: nativeRow.activity?.evidence?.waiting_for ?? null,
|
|
1163
|
+
turn_source: nativeRow.activity?.evidence?.source ?? null, turn_since_ms: nativeRow.activity?.evidence?.native_state_since_ms ?? null } : undefined);
|
|
1136
1164
|
const observedAt = now();
|
|
1165
|
+
const progress = sessionProgress(db, task);
|
|
1166
|
+
const priorProgress = session.work_progress;
|
|
1167
|
+
const worked = Boolean(priorProgress) && (progress.event > priorProgress.event || progress.commit > priorProgress.commit);
|
|
1168
|
+
if (!priorProgress || worked) setRunSession(db, runRow.id, { work_progress: progress, ...(worked ? { goal_turns: 0 } : {}) });
|
|
1137
1169
|
const lastAtMs = Number(live?.last_message_at_ms) || updatedAt(session);
|
|
1138
1170
|
const lastAt = Math.floor(Number(lastAtMs) / 1000);
|
|
1139
1171
|
const reading = {
|
|
@@ -1141,12 +1173,13 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1141
1173
|
last_message_at: lastAt > 0 ? lastAt : null,
|
|
1142
1174
|
last_message_at_ms: lastAt > 0 ? Number(lastAtMs) : null,
|
|
1143
1175
|
status: live?.status ?? null, turn: live?.turn ?? null, quiet_seconds: lastAt > 0 ? observedAt - lastAt : null,
|
|
1144
|
-
threshold_seconds: settings.idleQuiet, turns_before: Number(session.goal_turns ?? 0),
|
|
1176
|
+
threshold_seconds: settings.idleQuiet, turns_before: worked ? 0 : Number(session.goal_turns ?? 0),
|
|
1177
|
+
progress,
|
|
1145
1178
|
};
|
|
1146
1179
|
const retain = (decision, extra = {}) => {
|
|
1147
1180
|
// A reading that decides nothing is kept once per state: rewriting it each pass would wake the next pass.
|
|
1148
1181
|
const prior = session.idle_observation;
|
|
1149
|
-
if (!['dispatched', 'dispatch_failed'].includes(decision) && prior?.decision === decision && prior.
|
|
1182
|
+
if (!['dispatched', 'dispatch_failed', 'waiting_on_user'].includes(decision) && prior?.decision === decision && prior.turn === reading.turn && prior.idle_since === reading.idle_since && JSON.stringify(prior.delegated_sessions) === JSON.stringify(extra.delegated_sessions) && prior.status === reading.status && !worked) return prior;
|
|
1150
1183
|
const observation = { ...reading, decision, turns_after: reading.turns_before, ...extra };
|
|
1151
1184
|
const patch = { idle_observation: observation };
|
|
1152
1185
|
if (decision === 'below_threshold') patch.idle_below_threshold = observation;
|
|
@@ -1154,28 +1187,76 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1154
1187
|
setRunSession(db, runRow.id, patch);
|
|
1155
1188
|
return observation;
|
|
1156
1189
|
};
|
|
1157
|
-
// A session its
|
|
1158
|
-
//
|
|
1159
|
-
|
|
1160
|
-
const
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
const
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1190
|
+
// A session waiting on its user is told to the manager, with only what was read of it. Whether a harness's own state is a question
|
|
1191
|
+
// put to its user is its harness's to say (`needs_input`); what it waits for is carried as it was read.
|
|
1192
|
+
const address = addressOf(session);
|
|
1193
|
+
const asked = live?.turn === 'needs_input';
|
|
1194
|
+
const edges = [...delegations, ...fleet.sessions.filter(row => row.starter_session?.startsWith('sc:')).map(row => ({ starter: row.starter_session, session: row.address }))];
|
|
1195
|
+
for (const child of db.prepare('SELECT id,current_run_id FROM tasks WHERE current_run_id IS NOT NULL').all()) {
|
|
1196
|
+
const starter = creatorOf(db, child.id)?.replace(/^session:/, '');
|
|
1197
|
+
const target = addressOf(currentRun(db, child.id)?.metadata?.supercode);
|
|
1198
|
+
if (starter?.startsWith('sc:') && target) edges.push({ starter, session: target });
|
|
1199
|
+
}
|
|
1200
|
+
const delegated = delegatedWork(address, edges, delegatedReadings);
|
|
1201
|
+
if (live?.turn === 'idle' || asked || live?.turn === 'working' || delegated.length) {
|
|
1202
|
+
if (session.unknown_since != null || session.unknown_notice) setRunSession(db, runRow.id, { unknown_since: null, unknown_notice: null });
|
|
1203
|
+
}
|
|
1204
|
+
// a reading that shows the session no longer waiting ends the prompt; a round that did not read it does not
|
|
1205
|
+
if (session.held_notice && live && !asked) setRunSession(db, runRow.id, { held_notice: null });
|
|
1206
|
+
if (asked) {
|
|
1207
|
+
// One prompt is one notice: the same wait (what it waits for, since when its harness says) on the same
|
|
1208
|
+
// session, whichever of its cards a round reads first and however long it stands.
|
|
1209
|
+
const key = [address, live.waiting_for ?? '', live.turn_since_ms ?? ''].join('|');
|
|
1210
|
+
if (session.held_notice?.key === key) return;
|
|
1211
|
+
// every run a card works under: an ended run can still be its card's current one
|
|
1212
|
+
const told = db.prepare('SELECT r.metadata FROM task_runs r WHERE r.id != ? AND (r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id))').all(runRow.id).some((r) => json(r.metadata, {})?.supercode?.held_notice?.key === key);
|
|
1213
|
+
setRunSession(db, runRow.id, { held_notice: { key, at: observedAt } });
|
|
1214
|
+
if (told) return;
|
|
1215
|
+
const observation = retain('waiting_on_user', { waiting_for: live.waiting_for ?? null, since_ms: live.turn_since_ms ?? null });
|
|
1216
|
+
const others = (cardsOf.get(address) ?? []).filter((card) => card !== id);
|
|
1217
|
+
const what = live.turn_source === 'pane_prompt' ? 'shows a prompt on its pane that waits for an answer'
|
|
1218
|
+
: `reports that it is waiting on its user${live.waiting_for ? ` (${live.waiting_for})` : ''}${live.turn_since_ms ? ` since ${new Date(Number(live.turn_since_ms)).toISOString().slice(0, 19)}Z` : ''}`;
|
|
1168
1219
|
event(db, id, 'session_held', observation, runRow.id);
|
|
1169
1220
|
const to = managers[0] ?? null;
|
|
1170
|
-
if (to) event(db, id, 'board_message', { to, text }, null);
|
|
1171
|
-
(report.held ??= []).push([id,
|
|
1221
|
+
if (to) event(db, id, 'board_message', { to, text: `${id}: its session ${address} ${what}.${others.length ? ` It is also on ${others.join(', ')}.` : ''}` }, null);
|
|
1222
|
+
(report.held ??= []).push([id, 'waiting_on_user', address]);
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
if (live?.turn === 'working' || delegated.length) {
|
|
1226
|
+
if (session.idle_since != null) setRunSession(db, runRow.id, { idle_since: null });
|
|
1227
|
+
retain(delegated.length ? 'delegated_work' : 'working', { delegated_sessions: delegated });
|
|
1228
|
+
return;
|
|
1229
|
+
}
|
|
1230
|
+
// An unread or unknown turn does not prove idle, but cannot remain silent
|
|
1231
|
+
// indefinitely. Bound the observed interval independently of transcript
|
|
1232
|
+
// writes, and tell the manager once with the native facts we actually read.
|
|
1233
|
+
if (live?.turn !== 'idle') {
|
|
1234
|
+
const since = Number(session.unknown_since ?? observedAt);
|
|
1235
|
+
reading.unknown_since = since;
|
|
1236
|
+
reading.quiet_seconds = Math.max(0, observedAt - since);
|
|
1237
|
+
if (session.unknown_since !== since) setRunSession(db, runRow.id, { unknown_since: since });
|
|
1238
|
+
if (reading.quiet_seconds < settings.idleQuiet) { retain('turn_unknown'); return; }
|
|
1239
|
+
if (session.unknown_notice) return;
|
|
1240
|
+
const key = [address, live?.turn_since_ms ?? 'unknown'].join('|');
|
|
1241
|
+
const told = db.prepare('SELECT r.metadata FROM task_runs r WHERE r.id != ? AND (r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id))').all(runRow.id)
|
|
1242
|
+
.some(r => json(r.metadata, {})?.supercode?.unknown_notice?.key === key);
|
|
1243
|
+
setRunSession(db, runRow.id, { unknown_notice: { key, at: observedAt, since } });
|
|
1244
|
+
const observation = retain('turn_unknown_reported', { waiting_for: live?.waiting_for ?? null, turn_source: live?.turn_source ?? null, since_ms: live?.turn_since_ms ?? null });
|
|
1245
|
+
event(db, id, 'session_turn_unknown', observation, runRow.id);
|
|
1246
|
+
const to = managers[0] ?? null;
|
|
1247
|
+
if (to && !told) {
|
|
1248
|
+
const others = (cardsOf.get(address) ?? []).filter(card => card !== id);
|
|
1249
|
+
event(db, id, 'board_message', { to, text: `${id}: its session ${address} has no known native turn after ${reading.quiet_seconds}s of observation (threshold ${settings.idleQuiet}s). Read status=${JSON.stringify(reading.status)}, turn=${JSON.stringify(reading.turn)}, source=${JSON.stringify(live?.turn_source ?? null)}, waiting_for=${JSON.stringify(live?.waiting_for ?? null)}, native_since_ms=${JSON.stringify(live?.turn_since_ms ?? null)}, last_message_at_ms=${JSON.stringify(reading.last_message_at_ms)}.${others.length ? ` It is also on ${others.join(', ')}.` : ''} Its liveness needs a decision; no idle state or permission answer was inferred.` }, null);
|
|
1250
|
+
}
|
|
1251
|
+
(report.held ??= []).push([id, 'turn_unknown', address]);
|
|
1172
1252
|
return;
|
|
1173
1253
|
}
|
|
1174
|
-
//
|
|
1175
|
-
|
|
1254
|
+
// A native idle receipt also works without a transcript timestamp.
|
|
1255
|
+
const interval = idleInterval(session, live, observedAt);
|
|
1256
|
+
reading.idle_since = interval.since; reading.quiet_seconds = interval.quiet;
|
|
1257
|
+
if (session.idle_since !== interval.since) setRunSession(db, runRow.id, { idle_since: interval.since });
|
|
1176
1258
|
if (reading.quiet_seconds < settings.idleQuiet) { retain('below_threshold'); return; }
|
|
1177
1259
|
if (session.goal_nudged_at && observedAt - session.goal_nudged_at < settings.idleQuiet) { retain('nudge_cooldown'); return; }
|
|
1178
|
-
const worked = Boolean(session.goal_nudged_at) && lastAt > session.goal_nudged_at;
|
|
1179
1260
|
try {
|
|
1180
1261
|
const where = whereOf(board, task);
|
|
1181
1262
|
// the ask names what the session may do, as its run's prompt did
|
|
@@ -1565,6 +1646,7 @@ async function reconcileLaunches(board, fleet, report, entry, effects, settings,
|
|
|
1565
1646
|
setRunSession(db, row.id, { launch_pending: false, ...(s.mode === 'fresh' ? { forgotten: true } : {}) });
|
|
1566
1647
|
const payload = { error: `launch ${s.launch_key} ended before session binding` };
|
|
1567
1648
|
if (monitorsRuns(entry)) { surfaceIncident(db, board, entry, row.task_id, 'spawn_failed', payload); return { effects: [] }; }
|
|
1649
|
+
if (recoversRuns(entry)) retainRecovery(db, row.task_id, 'spawn_failed', payload);
|
|
1568
1650
|
return fire(db, board, entry, row.task_id, 'spawn_failed', { roles: ['platform'], payload });
|
|
1569
1651
|
});
|
|
1570
1652
|
if (moved) effects.push(...moved.effects);
|
|
@@ -1687,7 +1769,7 @@ async function launchPane(board, task, runId, lane, fleet, { input = null, slot
|
|
|
1687
1769
|
if (paneLive && (prev.closed_at || prev.close_requested || prev.forgotten)) throw new Error(`session ${prev.session_id} still has live pane ${prev.pane}; cannot resume or replace it`);
|
|
1688
1770
|
if (!prev.closed_at && !prev.close_requested && (prev.pane ? paneLive : sessionAlive(fleet, prev))) {
|
|
1689
1771
|
// the session is kept; the heartbeat's count belongs to the run that ended, so the new run starts it again
|
|
1690
|
-
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, idle_observation: _io, idle_below_threshold: _ib, idle_eligible: _ie, close_requested: _q, closed_at: _c, close_confirmed_at: _cc, close_error: _ce, close_attempts: _ca, close_attempted_at: _ct, monitor_held: _mh, ...kept } = prev;
|
|
1772
|
+
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, idle_since: _is, unknown_since: _us, unknown_notice: _un, work_progress: _wp, idle_observation: _io, idle_below_threshold: _ib, idle_eligible: _ie, close_requested: _q, closed_at: _c, close_confirmed_at: _cc, close_error: _ce, close_attempts: _ca, close_attempted_at: _ct, monitor_held: _mh, ...kept } = prev;
|
|
1691
1773
|
session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
1692
1774
|
// a kept session is told what this run is for (its card came back to it, with what happened since), only while
|
|
1693
1775
|
// this run is still the card's current one (its pane and native reads were awaited)
|
|
@@ -2110,7 +2192,7 @@ async function reconnectContinuedSessions(board, fleet, entry, report) {
|
|
|
2110
2192
|
async function fileMail(board, to, body, key, { ask = false, wake = true, subject } = {}) {
|
|
2111
2193
|
if (boardCopyOf(board)) throw copyRefusal(board, 'files no mail for sessions or managers');
|
|
2112
2194
|
const request = {
|
|
2113
|
-
from: boardAddress(board.label), from_name: `${board.label} board`, to, id: key, wake, subject: String(subject ?? String(body).split('\n')[0]).slice(0, 160),
|
|
2195
|
+
from: boardAddress(board.label), from_name: `${board.label} board`, to, id: key, wake, notice: !ask, subject: String(subject ?? String(body).split('\n')[0]).slice(0, 160),
|
|
2114
2196
|
body: ask ? `[A question from the ${board.label} board] ${body}` : `[Board notice from the ${board.label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)`,
|
|
2115
2197
|
// the board's own Teams context (the call it files in): the sender is observed in that team, never the daemon's
|
|
2116
2198
|
context: current().facts.SUPERCODE_CONTEXT ?? null,
|
package/board/incidents.mjs
CHANGED
|
@@ -1,14 +1,33 @@
|
|
|
1
1
|
// Service supervision retains evidence; only a manager/creator act releases dispatch.
|
|
2
2
|
import { event, json, now } from './store.mjs';
|
|
3
|
-
import { statusOf, creatorOf } from './workflow.mjs';
|
|
3
|
+
import { statusOf, creatorOf, requestSessionClose } from './workflow.mjs';
|
|
4
4
|
|
|
5
|
-
export const
|
|
5
|
+
export const recoversRuns = entry => entry?.params?.run_failure_policy === 'recover';
|
|
6
|
+
export const monitorsRuns = entry => !recoversRuns(entry) && entry?.params?.run_failure_policy !== 'retry';
|
|
6
7
|
// Where a thrown failure came from: its first frames, with paths from the package root, kept as the incident's
|
|
7
8
|
// evidence so a bare message ("database is locked") still names the statement's caller.
|
|
8
9
|
export const originOf = (error) => String(error?.stack ?? '').split('\n').filter((line) => /^\s+at /.test(line)).slice(0, 5)
|
|
9
10
|
.map((line) => line.trim().replace(/\((?:file:\/\/)?[^()]*\/(orchestrator\/|teams\/|volter-teams\/)/, '($1'));
|
|
10
11
|
export const failedObservation = name => ['exited', 'session_closed', 'session_lost', 'spawn_failed', 'orphaned', 'timed_out', 'stale'].includes(name);
|
|
11
12
|
|
|
13
|
+
/** Recovery is an explicit workflow policy. Keep the failure before the
|
|
14
|
+
* workflow ends its run; neither discard the incident nor hold dispatch. */
|
|
15
|
+
export function retainRecovery(db, id, name, evidence = {}, runId = null) {
|
|
16
|
+
const task = db.prepare('SELECT current_run_id FROM tasks WHERE id=?').get(id);
|
|
17
|
+
runId ??= task?.current_run_id;
|
|
18
|
+
const row = db.prepare('SELECT * FROM task_runs WHERE id=? AND task_id=?').get(runId, id);
|
|
19
|
+
if (!row || row.ended_at != null) return;
|
|
20
|
+
if (db.prepare("SELECT 1 FROM task_events WHERE task_id=? AND run_id=? AND kind='run_incident'").get(id, runId)) return;
|
|
21
|
+
const session = json(row.metadata, {})?.supercode ?? {};
|
|
22
|
+
const reason = evidence.error ?? `run ${runId} needs continuation (${name})`;
|
|
23
|
+
const payload = { observation: name, run: runId, reason, policy: 'recover',
|
|
24
|
+
session: session.session_id ?? null, machine: session.machine ?? null, pane: session.pane ?? null,
|
|
25
|
+
launch_key: session.launch_key ?? null, pid: session.pid ?? null, evidence };
|
|
26
|
+
event(db, id, 'run_incident', payload, runId);
|
|
27
|
+
db.prepare('INSERT INTO task_comments(task_id,author,body,created_at) VALUES (?,?,?,?)')
|
|
28
|
+
.run(id, 'dispatcher', `Run #${runId}: ${reason}. The workflow will continue this card through its native session door. Evidence: ${JSON.stringify(payload)}`, now());
|
|
29
|
+
}
|
|
30
|
+
|
|
12
31
|
export function surfaceIncident(db, board, entry, id, name, evidence = {}, runId = null) {
|
|
13
32
|
const task = db.prepare('SELECT * FROM tasks WHERE id=?').get(id);
|
|
14
33
|
runId ??= task?.current_run_id;
|
|
@@ -42,6 +61,24 @@ export function surfaceIncident(db, board, entry, id, name, evidence = {}, runId
|
|
|
42
61
|
|
|
43
62
|
/** Older ended runs are audited before eligibility: a completed run is not permission to repeat. */
|
|
44
63
|
export function auditEndedRuns(db, board, entry) {
|
|
64
|
+
if (recoversRuns(entry)) {
|
|
65
|
+
// Changing the explicit policy releases only incident holds. A person's
|
|
66
|
+
// pause, dependency, dispatch switch or unrelated hold remains their act.
|
|
67
|
+
for (const task of db.prepare(`SELECT t.*,r.id run_id,r.metadata run_metadata FROM tasks t JOIN task_runs r
|
|
68
|
+
ON r.id=(SELECT MAX(id) FROM task_runs WHERE task_id=t.id)
|
|
69
|
+
WHERE t.current_run_id IS NULL AND r.ended_at IS NOT NULL AND t.status IN ('running','review')`).all()) {
|
|
70
|
+
const session = json(task.run_metadata, {})?.supercode;
|
|
71
|
+
if (!session?.monitor_held) continue;
|
|
72
|
+
const state = db.prepare('SELECT paused,blocked_by,archived FROM supercode_cards WHERE task_id=?').get(task.id);
|
|
73
|
+
if (json(state?.paused, null) || json(state?.blocked_by, [])?.length || state?.archived) continue;
|
|
74
|
+
const last = db.prepare("SELECT kind,payload FROM task_events WHERE task_id=? AND kind IN ('dispatch_held','dispatch_released','session_adopted') ORDER BY id DESC LIMIT 1").get(task.id);
|
|
75
|
+
const hold = json(last?.payload, {});
|
|
76
|
+
if (last?.kind !== 'dispatch_held' || hold.run !== task.run_id || hold.incident !== session.incident) continue;
|
|
77
|
+
requestSessionClose(db, task.id, { runId: task.run_id });
|
|
78
|
+
event(db, task.id, 'dispatch_released', { reason: 'workflow explicitly selected recovery', incident: session.incident, ended_run: task.run_id });
|
|
79
|
+
}
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
45
82
|
if (!monitorsRuns(entry)) return;
|
|
46
83
|
for (const task of db.prepare(`SELECT t.*,r.id run_id,r.outcome,r.error,r.metadata run_metadata FROM tasks t JOIN task_runs r ON r.id=(SELECT MAX(id) FROM task_runs WHERE task_id=t.id)
|
|
47
84
|
WHERE t.current_run_id IS NULL AND r.ended_at IS NOT NULL AND t.status NOT IN ('done','archived')`).all()) {
|
package/board/machine-fleet.mjs
CHANGED
|
@@ -66,7 +66,10 @@ export async function readMachineFleet(machines, local) {
|
|
|
66
66
|
native.set(address, { alive: running ? true : null, states: [running ? true : null] });
|
|
67
67
|
if (running) alive.add(address);
|
|
68
68
|
sessions.push({ address, machine: row.machine, harness: activity.harness, session_id: activity.session_id, pane: pane.pane, name: pane.title ?? pane.label ?? null,
|
|
69
|
-
|
|
69
|
+
starter_session: pane.starter_session ?? null,
|
|
70
|
+
state: running ? 'running' : 'unknown', turn: activity.turn ?? null, native_state: activity.evidence?.native_state ?? null,
|
|
71
|
+
// what a session that needs input says it waits for and since when, where its harness publishes them
|
|
72
|
+
waiting_for: activity.evidence?.waiting_for ?? null, turn_source: activity.evidence?.source ?? null, turn_since_ms: activity.evidence?.native_state_since_ms ?? null });
|
|
70
73
|
}
|
|
71
74
|
}
|
|
72
75
|
const ok = online.has(local);
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// The dispatcher observes native turns and explicit session delegation. A
|
|
2
|
+
// transcript write, a reminder reply, or sharing a profile proves no progress.
|
|
3
|
+
|
|
4
|
+
/** Sessions this session started, transitively. Cycles never imply work. */
|
|
5
|
+
export function delegatedWork(address, edges, sessions) {
|
|
6
|
+
const reached = new Set([address]), pending = [address], working = [];
|
|
7
|
+
const byStarter = new Map();
|
|
8
|
+
for (const { starter, session } of edges) {
|
|
9
|
+
if (!starter || !session || starter === session) continue;
|
|
10
|
+
const children = byStarter.get(starter) ?? [];
|
|
11
|
+
children.push(session); byStarter.set(starter, children);
|
|
12
|
+
}
|
|
13
|
+
while (pending.length) {
|
|
14
|
+
for (const child of byStarter.get(pending.pop()) ?? []) {
|
|
15
|
+
if (reached.has(child)) continue;
|
|
16
|
+
reached.add(child); pending.push(child);
|
|
17
|
+
const reading = sessions.find(row => row.address === child);
|
|
18
|
+
if (reading?.state === 'running' && reading.turn === 'working') working.push(child);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return working;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Idle time belongs to a native idle interval, never to transcript freshness. */
|
|
25
|
+
export function idleInterval(session, live, observedAt) {
|
|
26
|
+
const nativeSince = Number(live?.turn_since_ms);
|
|
27
|
+
const since = nativeSince > 0 && nativeSince <= observedAt * 1000
|
|
28
|
+
? Math.floor(nativeSince / 1000) : Number(session.idle_since ?? observedAt);
|
|
29
|
+
return { since, quiet: Math.max(0, observedAt - since) };
|
|
30
|
+
}
|
package/board/workflow.mjs
CHANGED
|
@@ -246,6 +246,14 @@ async function observeLastCommit(path, id, branches = []) {
|
|
|
246
246
|
}
|
|
247
247
|
|
|
248
248
|
function lastCommitAt(path, id) { return path ? commitTimes.get(`${path}:${id}`)?.value ?? null : null; }
|
|
249
|
+
/** Material card work. Comments, reminder replies and timestamp writes are
|
|
250
|
+
* deliberately absent; a heartbeat is a separately bounded waiting lease. */
|
|
251
|
+
export function sessionProgress(db, task) {
|
|
252
|
+
const event = one(db, `SELECT MAX(e.id) AS id FROM task_events e WHERE
|
|
253
|
+
(e.task_id=? AND e.kind IN ('specified','review_requested','review_pass','changes_requested','audited')) OR
|
|
254
|
+
(e.task_id IN (SELECT task_id FROM supercode_cards WHERE subtask_of=?) AND e.kind IN ('completed','cancelled'))`, task.id, task.id);
|
|
255
|
+
return { event: Number(event?.id ?? 0), commit: Number(lastCommitAt(task.workspace_path, task.id) ?? 0) };
|
|
256
|
+
}
|
|
249
257
|
/** Observe external facts before entering an optimistic board mutation. */
|
|
250
258
|
export async function refreshCardInputs(board, id, { commits = false } = {}) {
|
|
251
259
|
const task=board.db.prepare('SELECT t.*,c.subtask_of FROM tasks t LEFT JOIN supercode_cards c ON c.task_id=t.id WHERE t.id=?').get(id);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.111",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|