@volter/supercode-orchestrator 0.5.110 → 0.5.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents.mjs +25 -0
- package/board/ci.mjs +62 -9
- package/board/cli.mjs +1 -0
- package/board/dispatch.mjs +76 -18
- package/board/incidents.mjs +39 -2
- package/board/machine-fleet.mjs +1 -0
- package/board/session-work.mjs +30 -0
- package/board/workflow.mjs +8 -0
- package/package.json +2 -2
package/agents.mjs
CHANGED
|
@@ -174,6 +174,31 @@ export function localAgentRecord(root,org,id){
|
|
|
174
174
|
export function localAgentSessions(root,org,id){
|
|
175
175
|
const db=store(root);try{return db.prepare('SELECT session_ref AS session,machine,ended_at,end_reason FROM agent_runs WHERE org=? AND agent_id=? ORDER BY rowid DESC LIMIT 200').all(org,id);}finally{db.close();}
|
|
176
176
|
}
|
|
177
|
+
/** Retained starter receipts for this home's sessions, including explicit forks.
|
|
178
|
+
* Ended owners remain in the join so a live child keeps its original starter. */
|
|
179
|
+
export function localSessionDelegations(root, localMachine) {
|
|
180
|
+
const db = store(root);
|
|
181
|
+
try {
|
|
182
|
+
const rows = db.prepare(`SELECT r.*, json_extract(i.declaration_json,'$.starterSession') AS declared_starter
|
|
183
|
+
FROM agent_runs r LEFT JOIN agent_intents i ON i.org=r.org AND i.id=r.agent_id`).all();
|
|
184
|
+
const addresses = new Map();
|
|
185
|
+
const address = row => row.session_ref.startsWith('sc:') ? row.session_ref
|
|
186
|
+
: /^(claude-code|codex):[^:]+$/.test(row.session_ref) ? `sc:${row.machine ?? localMachine}:${row.session_ref}` : null;
|
|
187
|
+
for (const row of rows) {
|
|
188
|
+
const key = `${row.org}|${row.session_ref}`, value = address(row);
|
|
189
|
+
if (value) addresses.set(key, value);
|
|
190
|
+
}
|
|
191
|
+
for (const alias of db.prepare('SELECT * FROM agent_runtime_sessions').all()) {
|
|
192
|
+
const value = addresses.get(`${alias.org}|${alias.session_ref}`);
|
|
193
|
+
if (value) addresses.set(`${alias.org}|${alias.runtime_ref}`, value);
|
|
194
|
+
}
|
|
195
|
+
return rows.filter(row => row.ended_at == null).map(row => {
|
|
196
|
+
const ref = row.starter_session ?? row.declared_starter;
|
|
197
|
+
return { session: address(row), starter: typeof ref === 'string' && ref.startsWith('sc:')
|
|
198
|
+
? ref : addresses.get(`${row.org}|${ref}`) ?? null };
|
|
199
|
+
}).filter(edge => edge.session && edge.starter);
|
|
200
|
+
} finally { db.close(); }
|
|
201
|
+
}
|
|
177
202
|
/** Move identities minted before any Teams context into the Org that is now selected, keeping their ids. */
|
|
178
203
|
function adoptLocalOrg(root,scope){
|
|
179
204
|
const db=store(root);
|
package/board/ci.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { execFile } from 'node:child_process';
|
|
|
4
4
|
import { readFileSync, realpathSync, writeFileSync, renameSync, statSync } from 'node:fs';
|
|
5
5
|
import { dirname, join, relative } from 'node:path';
|
|
6
6
|
import { createHash } from 'node:crypto';
|
|
7
|
-
import { branchCheckouts, checkoutOf, isLocalWorkspace, resolveWorkspace } from './workspace.mjs';
|
|
7
|
+
import { branchCheckouts, checkoutOf, isLocalWorkspace, resolveWorkspace, plainWorkspace } from './workspace.mjs';
|
|
8
8
|
import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
|
|
9
9
|
|
|
10
10
|
// GitHub is read through its REST API with Node's own fetch, in this process and awaited (never a child the caller
|
|
@@ -134,16 +134,50 @@ async function repositoryWorkspace(board, id) {
|
|
|
134
134
|
const workspace = task?.workspace_kind === 'scratch' ? parent : task;
|
|
135
135
|
if (!workspace || workspace.workspace_kind === 'scratch') return null;
|
|
136
136
|
const here = isLocalWorkspace(workspace);
|
|
137
|
-
if (workspace.workspace_anchor)
|
|
137
|
+
if (workspace.workspace_anchor) {
|
|
138
|
+
// A portable remote declaration supplies the source identity. Reviewing
|
|
139
|
+
// its pushed head does not require that machine to keep the old checkout.
|
|
140
|
+
if (!here) {
|
|
141
|
+
const declared = plainWorkspace(workspace.workspace_kind, workspace.workspace_anchor, workspace.branch_name);
|
|
142
|
+
if (declared) return { ...declared, here: false };
|
|
143
|
+
}
|
|
144
|
+
const resolved = await resolveWorkspace(workspace, false);
|
|
145
|
+
// Resolution without create names a solo worktree even before it exists on
|
|
146
|
+
// this machine. Its portable repository and branch still identify the
|
|
147
|
+
// pushed PR. Only an actual directory can supply local source evidence.
|
|
148
|
+
return { ...resolved, here: here && checkoutPresent(resolved.path) };
|
|
149
|
+
}
|
|
138
150
|
return workspace.workspace_path ? { path: workspace.workspace_path, here } : null;
|
|
139
151
|
}
|
|
140
152
|
|
|
141
153
|
const pullRequestLists = new Map();
|
|
154
|
+
const mergedPullRequestLists = new Map();
|
|
155
|
+
|
|
156
|
+
/** Historical work can already be merged when an outage prevented the board's
|
|
157
|
+
* review request. Read only the arc's declared branch, never all closed PRs. */
|
|
158
|
+
async function mergedBranchPullRequests(repository, branch) {
|
|
159
|
+
const key = `${repository}:${branch}`;
|
|
160
|
+
let list = mergedPullRequestLists.get(key);
|
|
161
|
+
if (!list || Date.now() - list.at >= 60_000) {
|
|
162
|
+
const head = encodeURIComponent(`${repository.split('/')[0]}:${branch}`);
|
|
163
|
+
const items = (await github(`/repos/${repository}/pulls?state=closed&head=${head}&per_page=100`, true))
|
|
164
|
+
.filter(p => p.merged_at).map(p => pullRequest(repository, p));
|
|
165
|
+
if (mergedPullRequestLists.size >= 64) mergedPullRequestLists.delete(mergedPullRequestLists.keys().next().value);
|
|
166
|
+
list = { at: Date.now(), items }; mergedPullRequestLists.set(key, list);
|
|
167
|
+
}
|
|
168
|
+
return list.items;
|
|
169
|
+
}
|
|
142
170
|
|
|
143
171
|
async function branchCheckout(board, id, repository, branch) {
|
|
144
172
|
const task = board.db.prepare('SELECT c.machine FROM tasks t LEFT JOIN supercode_cards c ON c.task_id=t.id WHERE t.id=?').get(id);
|
|
145
173
|
if (!isLocalWorkspace(task ?? {})) return null;
|
|
146
|
-
try {
|
|
174
|
+
try { const path = (await checkoutOf(task?.machine, repository, branch))?.path; return checkoutPresent(path) ? path : null; } catch { return null; }
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function checkoutPresent(path) {
|
|
178
|
+
if (!path) return false;
|
|
179
|
+
try { return statSync(path).isDirectory(); }
|
|
180
|
+
catch (error) { if (['ENOENT', 'ENOTDIR'].includes(error.code)) return false; throw error; }
|
|
147
181
|
}
|
|
148
182
|
|
|
149
183
|
/** The sibling checkouts an arc registered: each subtask (and the card itself) that declares a repository
|
|
@@ -174,7 +208,7 @@ export function declaredWorkBranches(db, id) {
|
|
|
174
208
|
return [...new Set([...declared, ...reviewed].filter(Boolean))];
|
|
175
209
|
}
|
|
176
210
|
|
|
177
|
-
/** Every opened PR of the arc, each proposed to its repository's default branch: an open PR whose head is the card's
|
|
211
|
+
/** Every opened PR of the arc, plus an unjudged merged PR on its declared branch, each proposed to its repository's default branch: an open PR whose head is the card's
|
|
178
212
|
* work branch in the card's own repository, a `wt/<card id>` branch in any repository this machine keeps a checkout
|
|
179
213
|
* of, or a sibling checkout the arc registered (`registeredBranches`). The repository's default branch, declared or
|
|
180
214
|
* checked out (or no branch), is not a work branch: such a card's PRs are only its `wt/<card>` and registered ones, and with none it
|
|
@@ -195,20 +229,39 @@ export async function openReviewPullRequests(board, id) {
|
|
|
195
229
|
// Discovery is shared per repository for one minute; each selected PR/head is re-read before it authorizes a move.
|
|
196
230
|
let list = pullRequestLists.get(repository);
|
|
197
231
|
if (!list || Date.now() - list.at >= 60_000) {
|
|
198
|
-
let items = [];
|
|
199
|
-
try {
|
|
232
|
+
let items = [], defaultBranch = null;
|
|
233
|
+
try {
|
|
234
|
+
items = (await github(`/repos/${repository}/pulls?state=open&per_page=100`, true)).map((p) => pullRequest(repository, p));
|
|
235
|
+
defaultBranch = items.find(pr => pr.defaultBranch)?.defaultBranch ?? (await github(`/repos/${repository}`)).default_branch;
|
|
236
|
+
}
|
|
200
237
|
catch (error) { if (repository === ownRepository || registered.some((r) => r.repository === repository)) throw error; } // another repository unreachable here is not this arc's
|
|
201
|
-
list = { at: Date.now(), items };
|
|
238
|
+
list = { at: Date.now(), items, defaultBranch };
|
|
202
239
|
if (pullRequestLists.size >= 64) pullRequestLists.delete(pullRequestLists.keys().next().value);
|
|
203
240
|
pullRequestLists.set(repository, list);
|
|
204
241
|
}
|
|
205
242
|
// The repository's default branch is not a work branch, declared or checked out (a card working in a `dir` on main).
|
|
206
243
|
if (repository === ownRepository && branch) {
|
|
207
|
-
|
|
208
|
-
if (branch === defaultBranch) branch = null;
|
|
244
|
+
if (branch === list.defaultBranch) branch = null;
|
|
209
245
|
}
|
|
210
246
|
// a draft is not handed over yet: its review starts when it is marked ready; a PR to another base (main→prod) is a release, not the arc's work
|
|
211
247
|
for (const pr of list.items) if (!pr.isCrossRepository && !pr.isDraft && pr.baseRefName === pr.defaultBranch && mine(repository, pr.headRefName)) prs.push({ url: pr.url, head: pr.headRefOid });
|
|
248
|
+
// A manual review and merge during a dispatcher outage may predate any
|
|
249
|
+
// board request. The source-review door already accepts merged heads; make
|
|
250
|
+
// them discoverable so the ordinary independent verdict can be retained.
|
|
251
|
+
// An open PR (including a draft or wrong base) on that branch takes priority.
|
|
252
|
+
const historical = [...new Set([
|
|
253
|
+
...(repository === ownRepository ? branch ? [branch] : [`wt/${id}`, `wt/${id.replace(/^t_/, 't-')}`] : []),
|
|
254
|
+
...registered.filter(r => r.repository === repository).map(r => r.branch),
|
|
255
|
+
])];
|
|
256
|
+
for (const headBranch of historical) {
|
|
257
|
+
if (headBranch === list.defaultBranch) continue;
|
|
258
|
+
if (list.items.some(pr => pr.headRefName === headBranch)) continue;
|
|
259
|
+
for (const pr of await mergedBranchPullRequests(repository, headBranch)) {
|
|
260
|
+
if (!pr.isCrossRepository && pr.headRefName === headBranch && pr.headRefName !== pr.defaultBranch
|
|
261
|
+
&& pr.baseRefName === pr.defaultBranch && !judgedHead(board, id, pr.headRefOid)
|
|
262
|
+
&& !prs.some(known => known.url === pr.url)) prs.push({ url: pr.url, head: pr.headRefOid });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
212
265
|
}
|
|
213
266
|
if (!prs.length) return branch ? null : { waits: `the card has no work branch (${task?.branch_name ? `it declares ${task.branch_name}, its repository's default` : current ? `its checkout ${own.path} is on ${current}, its repository's default` : `its checkout ${own.path} is on no branch`}): give it one with --branch, or open its PR from a wt/${id} branch` };
|
|
214
267
|
return { pr: prs.map((p) => p.url), pr_head: prs.map((p) => p.head), branch };
|
package/board/cli.mjs
CHANGED
|
@@ -1701,6 +1701,7 @@ async function serve(root, args = {}) {
|
|
|
1701
1701
|
const refused = named?.refused ?? null;
|
|
1702
1702
|
log(refusal ? `board events: not published; the machine daemon did not take this home: ${refusal}`
|
|
1703
1703
|
: refused ? `board events: registered, but the team refused this machine's credential (${refused.status}); nothing is published until it is re-enrolled`
|
|
1704
|
+
: named?.published === false ? `board ownership: registered locally; this machine has no team connector to publish events`
|
|
1704
1705
|
: `board events: this home is published by the machine's connector`); }
|
|
1705
1706
|
log('dispatch wakes on board writes, document edits and board mail; the interval sweep reads each machine\'s own door');
|
|
1706
1707
|
const stuck = {};
|
package/board/dispatch.mjs
CHANGED
|
@@ -20,8 +20,11 @@ import { machineCall } from './doors.mjs';
|
|
|
20
20
|
import { processSession } from './caller-session.mjs';
|
|
21
21
|
import { callerOf, current, isCallersCall, ownCall } from './call.mjs';
|
|
22
22
|
import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
|
|
23
|
-
import { monitorsRuns, failedObservation, surfaceIncident, auditEndedRuns, originOf } from './incidents.mjs';
|
|
23
|
+
import { monitorsRuns, recoversRuns, retainRecovery, failedObservation, surfaceIncident, auditEndedRuns, originOf } from './incidents.mjs';
|
|
24
24
|
import { readMachineFleet } from './machine-fleet.mjs';
|
|
25
|
+
import { delegatedWork, idleInterval } from './session-work.mjs';
|
|
26
|
+
import { localSessionDelegations } from '../agents.mjs';
|
|
27
|
+
import { creatorOf, sessionProgress } from './workflow.mjs';
|
|
25
28
|
import { boardLabel, boardName } from '@volter/teams/board-name';
|
|
26
29
|
let fleetRoot = null;
|
|
27
30
|
import { inspectCompletionEvidence, noCodeChange, openReviewPullRequests, reviewDue, withCompletionEvidence, reviewCandidate, workflowRevision, retainWorkflowTick } from './ci.mjs';
|
|
@@ -376,6 +379,7 @@ async function supervise(board, fleet, settings, report, entry, effects) {
|
|
|
376
379
|
if (bucket) report[bucket].push(id);
|
|
377
380
|
return;
|
|
378
381
|
}
|
|
382
|
+
if (recoversRuns(entry) && failedObservation(name)) retainRecovery(db, id, name, payload);
|
|
379
383
|
const r = fire(db, board, entry, id, name, { roles: ['platform'], payload });
|
|
380
384
|
staged.push(...r.effects);
|
|
381
385
|
if (bucket) report[bucket].push(id);
|
|
@@ -778,6 +782,7 @@ function launchFailed(board, entry, id, error, payload) {
|
|
|
778
782
|
return tx(board, (db) => {
|
|
779
783
|
if (!runCurrent(db, id, error.launchRun)) return null;
|
|
780
784
|
if (monitorsRuns(entry)) { surfaceIncident(db, board, entry, id, 'spawn_failed', payload, error.launchRun); return { effects: [] }; }
|
|
785
|
+
if (recoversRuns(entry)) retainRecovery(db, id, 'spawn_failed', payload, error.launchRun);
|
|
781
786
|
return fire(db, board, entry, id, 'spawn_failed', { roles: ['platform'], payload });
|
|
782
787
|
});
|
|
783
788
|
}
|
|
@@ -1103,7 +1108,7 @@ export async function closeRequested(board, report, { taskId = null, caller = nu
|
|
|
1103
1108
|
}
|
|
1104
1109
|
}
|
|
1105
1110
|
|
|
1106
|
-
/**
|
|
1111
|
+
/** Native idle intervals are independent of dispatch cadence and transcript writes.
|
|
1107
1112
|
* Keep bounded readings on the run so an ordinary quiet stretch can be reviewed later. */
|
|
1108
1113
|
async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
1109
1114
|
if (!fleet.ok) return;
|
|
@@ -1116,11 +1121,28 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1116
1121
|
return Number(rows?.find((row) => row.locator?.session_id === session.session_id)?.updated_at_ms) || null;
|
|
1117
1122
|
};
|
|
1118
1123
|
const managers = [].concat(entry.params.managers ?? []).filter(Boolean);
|
|
1124
|
+
const delegations = localSessionDelegations(board.root, localMachine());
|
|
1125
|
+
const delegatedReadings = [...fleet.sessions];
|
|
1126
|
+
// Forks and concurrent sessions need not have a pane. Read only the exact
|
|
1127
|
+
// native identities this home's starter receipts name, before any write.
|
|
1128
|
+
for (const { session: address } of delegations) {
|
|
1129
|
+
if (delegatedReadings.some(row => row.address === address)) continue;
|
|
1130
|
+
const match = /^sc:([^:]+):(claude-code|codex):([^:]+)$/.exec(address);
|
|
1131
|
+
if (!match) continue;
|
|
1132
|
+
const [, machine, harness, session_id] = match;
|
|
1133
|
+
const child = { machine, harness, session_id, surface: 'native' };
|
|
1134
|
+
await readBefore(fleet, child);
|
|
1135
|
+
try {
|
|
1136
|
+
const { row } = nativeReading(nativeSessions(machine, harness, session_id), child);
|
|
1137
|
+
if (row?.activity) delegatedReadings.push({ address, state: row.activity.presence, turn: row.activity.turn });
|
|
1138
|
+
} catch { /* an unread child is never called working */ }
|
|
1139
|
+
}
|
|
1119
1140
|
const running = board.db.prepare(entry.workflow.run ? "SELECT id FROM tasks WHERE current_run_id IS NOT NULL" : "SELECT id FROM tasks WHERE status = 'running'").all();
|
|
1120
1141
|
// every running card a session is on: a session waiting on its user is told once, naming them all
|
|
1121
1142
|
const cardsOf = new Map();
|
|
1122
1143
|
for (const { id } of running) {
|
|
1123
|
-
const session = currentRun(board.db, id)?.metadata?.supercode; await readBefore(fleet, session);
|
|
1144
|
+
const session = currentRun(board.db, id)?.metadata?.supercode; await readBefore(fleet, session);
|
|
1145
|
+
await refreshCardInputs(board, id, { commits: true });
|
|
1124
1146
|
if (addressOf(session)) cardsOf.set(addressOf(session), [...(cardsOf.get(addressOf(session)) ?? []), id]);
|
|
1125
1147
|
}
|
|
1126
1148
|
for (const { id } of running) {
|
|
@@ -1140,6 +1162,10 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1140
1162
|
const live = fleet.sessions.find((x) => x.address === addressOf(session)) ?? (nativeRow ? { address: addressOf(session), turn: nativeRow.activity?.turn ?? null, waiting_for: nativeRow.activity?.evidence?.waiting_for ?? null,
|
|
1141
1163
|
turn_source: nativeRow.activity?.evidence?.source ?? null, turn_since_ms: nativeRow.activity?.evidence?.native_state_since_ms ?? null } : undefined);
|
|
1142
1164
|
const observedAt = now();
|
|
1165
|
+
const progress = sessionProgress(db, task);
|
|
1166
|
+
const priorProgress = session.work_progress;
|
|
1167
|
+
const worked = Boolean(priorProgress) && (progress.event > priorProgress.event || progress.commit > priorProgress.commit);
|
|
1168
|
+
if (!priorProgress || worked) setRunSession(db, runRow.id, { work_progress: progress, ...(worked ? { goal_turns: 0 } : {}) });
|
|
1143
1169
|
const lastAtMs = Number(live?.last_message_at_ms) || updatedAt(session);
|
|
1144
1170
|
const lastAt = Math.floor(Number(lastAtMs) / 1000);
|
|
1145
1171
|
const reading = {
|
|
@@ -1147,12 +1173,13 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1147
1173
|
last_message_at: lastAt > 0 ? lastAt : null,
|
|
1148
1174
|
last_message_at_ms: lastAt > 0 ? Number(lastAtMs) : null,
|
|
1149
1175
|
status: live?.status ?? null, turn: live?.turn ?? null, quiet_seconds: lastAt > 0 ? observedAt - lastAt : null,
|
|
1150
|
-
threshold_seconds: settings.idleQuiet, turns_before: Number(session.goal_turns ?? 0),
|
|
1176
|
+
threshold_seconds: settings.idleQuiet, turns_before: worked ? 0 : Number(session.goal_turns ?? 0),
|
|
1177
|
+
progress,
|
|
1151
1178
|
};
|
|
1152
1179
|
const retain = (decision, extra = {}) => {
|
|
1153
1180
|
// A reading that decides nothing is kept once per state: rewriting it each pass would wake the next pass.
|
|
1154
1181
|
const prior = session.idle_observation;
|
|
1155
|
-
if (!['dispatched', 'dispatch_failed', 'waiting_on_user'].includes(decision) && prior?.decision === decision && prior.
|
|
1182
|
+
if (!['dispatched', 'dispatch_failed', 'waiting_on_user'].includes(decision) && prior?.decision === decision && prior.turn === reading.turn && prior.idle_since === reading.idle_since && JSON.stringify(prior.delegated_sessions) === JSON.stringify(extra.delegated_sessions) && prior.status === reading.status && !worked) return prior;
|
|
1156
1183
|
const observation = { ...reading, decision, turns_after: reading.turns_before, ...extra };
|
|
1157
1184
|
const patch = { idle_observation: observation };
|
|
1158
1185
|
if (decision === 'below_threshold') patch.idle_below_threshold = observation;
|
|
@@ -1160,11 +1187,20 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1160
1187
|
setRunSession(db, runRow.id, patch);
|
|
1161
1188
|
return observation;
|
|
1162
1189
|
};
|
|
1163
|
-
// A session waiting on its user
|
|
1164
|
-
// threshold is told to the manager, with only what was read of it. Whether a harness's own state is a question
|
|
1190
|
+
// A session waiting on its user is told to the manager, with only what was read of it. Whether a harness's own state is a question
|
|
1165
1191
|
// put to its user is its harness's to say (`needs_input`); what it waits for is carried as it was read.
|
|
1166
1192
|
const address = addressOf(session);
|
|
1167
1193
|
const asked = live?.turn === 'needs_input';
|
|
1194
|
+
const edges = [...delegations, ...fleet.sessions.filter(row => row.starter_session?.startsWith('sc:')).map(row => ({ starter: row.starter_session, session: row.address }))];
|
|
1195
|
+
for (const child of db.prepare('SELECT id,current_run_id FROM tasks WHERE current_run_id IS NOT NULL').all()) {
|
|
1196
|
+
const starter = creatorOf(db, child.id)?.replace(/^session:/, '');
|
|
1197
|
+
const target = addressOf(currentRun(db, child.id)?.metadata?.supercode);
|
|
1198
|
+
if (starter?.startsWith('sc:') && target) edges.push({ starter, session: target });
|
|
1199
|
+
}
|
|
1200
|
+
const delegated = delegatedWork(address, edges, delegatedReadings);
|
|
1201
|
+
if (live?.turn === 'idle' || asked || live?.turn === 'working' || delegated.length) {
|
|
1202
|
+
if (session.unknown_since != null || session.unknown_notice) setRunSession(db, runRow.id, { unknown_since: null, unknown_notice: null });
|
|
1203
|
+
}
|
|
1168
1204
|
// a reading that shows the session no longer waiting ends the prompt; a round that did not read it does not
|
|
1169
1205
|
if (session.held_notice && live && !asked) setRunSession(db, runRow.id, { held_notice: null });
|
|
1170
1206
|
if (asked) {
|
|
@@ -1186,20 +1222,41 @@ async function observeIdle(board, fleet, settings, report, entry, effects) {
|
|
|
1186
1222
|
(report.held ??= []).push([id, 'waiting_on_user', address]);
|
|
1187
1223
|
return;
|
|
1188
1224
|
}
|
|
1189
|
-
if (live?.turn === 'working'
|
|
1190
|
-
if (session.
|
|
1191
|
-
|
|
1192
|
-
|
|
1225
|
+
if (live?.turn === 'working' || delegated.length) {
|
|
1226
|
+
if (session.idle_since != null) setRunSession(db, runRow.id, { idle_since: null });
|
|
1227
|
+
retain(delegated.length ? 'delegated_work' : 'working', { delegated_sessions: delegated });
|
|
1228
|
+
return;
|
|
1229
|
+
}
|
|
1230
|
+
// An unread or unknown turn does not prove idle, but cannot remain silent
|
|
1231
|
+
// indefinitely. Bound the observed interval independently of transcript
|
|
1232
|
+
// writes, and tell the manager once with the native facts we actually read.
|
|
1233
|
+
if (live?.turn !== 'idle') {
|
|
1234
|
+
const since = Number(session.unknown_since ?? observedAt);
|
|
1235
|
+
reading.unknown_since = since;
|
|
1236
|
+
reading.quiet_seconds = Math.max(0, observedAt - since);
|
|
1237
|
+
if (session.unknown_since !== since) setRunSession(db, runRow.id, { unknown_since: since });
|
|
1238
|
+
if (reading.quiet_seconds < settings.idleQuiet) { retain('turn_unknown'); return; }
|
|
1239
|
+
if (session.unknown_notice) return;
|
|
1240
|
+
const key = [address, live?.turn_since_ms ?? 'unknown'].join('|');
|
|
1241
|
+
const told = db.prepare('SELECT r.metadata FROM task_runs r WHERE r.id != ? AND (r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id))').all(runRow.id)
|
|
1242
|
+
.some(r => json(r.metadata, {})?.supercode?.unknown_notice?.key === key);
|
|
1243
|
+
setRunSession(db, runRow.id, { unknown_notice: { key, at: observedAt, since } });
|
|
1244
|
+
const observation = retain('turn_unknown_reported', { waiting_for: live?.waiting_for ?? null, turn_source: live?.turn_source ?? null, since_ms: live?.turn_since_ms ?? null });
|
|
1245
|
+
event(db, id, 'session_turn_unknown', observation, runRow.id);
|
|
1193
1246
|
const to = managers[0] ?? null;
|
|
1194
|
-
if (to
|
|
1195
|
-
|
|
1247
|
+
if (to && !told) {
|
|
1248
|
+
const others = (cardsOf.get(address) ?? []).filter(card => card !== id);
|
|
1249
|
+
event(db, id, 'board_message', { to, text: `${id}: its session ${address} has no known native turn after ${reading.quiet_seconds}s of observation (threshold ${settings.idleQuiet}s). Read status=${JSON.stringify(reading.status)}, turn=${JSON.stringify(reading.turn)}, source=${JSON.stringify(live?.turn_source ?? null)}, waiting_for=${JSON.stringify(live?.waiting_for ?? null)}, native_since_ms=${JSON.stringify(live?.turn_since_ms ?? null)}, last_message_at_ms=${JSON.stringify(reading.last_message_at_ms)}.${others.length ? ` It is also on ${others.join(', ')}.` : ''} Its liveness needs a decision; no idle state or permission answer was inferred.` }, null);
|
|
1250
|
+
}
|
|
1251
|
+
(report.held ??= []).push([id, 'turn_unknown', address]);
|
|
1196
1252
|
return;
|
|
1197
1253
|
}
|
|
1198
|
-
//
|
|
1199
|
-
|
|
1254
|
+
// A native idle receipt also works without a transcript timestamp.
|
|
1255
|
+
const interval = idleInterval(session, live, observedAt);
|
|
1256
|
+
reading.idle_since = interval.since; reading.quiet_seconds = interval.quiet;
|
|
1257
|
+
if (session.idle_since !== interval.since) setRunSession(db, runRow.id, { idle_since: interval.since });
|
|
1200
1258
|
if (reading.quiet_seconds < settings.idleQuiet) { retain('below_threshold'); return; }
|
|
1201
1259
|
if (session.goal_nudged_at && observedAt - session.goal_nudged_at < settings.idleQuiet) { retain('nudge_cooldown'); return; }
|
|
1202
|
-
const worked = Boolean(session.goal_nudged_at) && lastAt > session.goal_nudged_at;
|
|
1203
1260
|
try {
|
|
1204
1261
|
const where = whereOf(board, task);
|
|
1205
1262
|
// the ask names what the session may do, as its run's prompt did
|
|
@@ -1589,6 +1646,7 @@ async function reconcileLaunches(board, fleet, report, entry, effects, settings,
|
|
|
1589
1646
|
setRunSession(db, row.id, { launch_pending: false, ...(s.mode === 'fresh' ? { forgotten: true } : {}) });
|
|
1590
1647
|
const payload = { error: `launch ${s.launch_key} ended before session binding` };
|
|
1591
1648
|
if (monitorsRuns(entry)) { surfaceIncident(db, board, entry, row.task_id, 'spawn_failed', payload); return { effects: [] }; }
|
|
1649
|
+
if (recoversRuns(entry)) retainRecovery(db, row.task_id, 'spawn_failed', payload);
|
|
1592
1650
|
return fire(db, board, entry, row.task_id, 'spawn_failed', { roles: ['platform'], payload });
|
|
1593
1651
|
});
|
|
1594
1652
|
if (moved) effects.push(...moved.effects);
|
|
@@ -1711,7 +1769,7 @@ async function launchPane(board, task, runId, lane, fleet, { input = null, slot
|
|
|
1711
1769
|
if (paneLive && (prev.closed_at || prev.close_requested || prev.forgotten)) throw new Error(`session ${prev.session_id} still has live pane ${prev.pane}; cannot resume or replace it`);
|
|
1712
1770
|
if (!prev.closed_at && !prev.close_requested && (prev.pane ? paneLive : sessionAlive(fleet, prev))) {
|
|
1713
1771
|
// the session is kept; the heartbeat's count belongs to the run that ended, so the new run starts it again
|
|
1714
|
-
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, idle_observation: _io, idle_below_threshold: _ib, idle_eligible: _ie, close_requested: _q, closed_at: _c, close_confirmed_at: _cc, close_error: _ce, close_attempts: _ca, close_attempted_at: _ct, monitor_held: _mh, ...kept } = prev;
|
|
1772
|
+
const { missing_since: _m, goal_turns: _t, goal_nudged_at: _n, idle_since: _is, unknown_since: _us, unknown_notice: _un, work_progress: _wp, idle_observation: _io, idle_below_threshold: _ib, idle_eligible: _ie, close_requested: _q, closed_at: _c, close_confirmed_at: _cc, close_error: _ce, close_attempts: _ca, close_attempted_at: _ct, monitor_held: _mh, ...kept } = prev;
|
|
1715
1773
|
session = { ...kept, mode: 'adopt', launched_at: now(), seen_at: now() };
|
|
1716
1774
|
// a kept session is told what this run is for (its card came back to it, with what happened since), only while
|
|
1717
1775
|
// this run is still the card's current one (its pane and native reads were awaited)
|
|
@@ -2134,7 +2192,7 @@ async function reconnectContinuedSessions(board, fleet, entry, report) {
|
|
|
2134
2192
|
async function fileMail(board, to, body, key, { ask = false, wake = true, subject } = {}) {
|
|
2135
2193
|
if (boardCopyOf(board)) throw copyRefusal(board, 'files no mail for sessions or managers');
|
|
2136
2194
|
const request = {
|
|
2137
|
-
from: boardAddress(board.label), from_name: `${board.label} board`, to, id: key, wake, subject: String(subject ?? String(body).split('\n')[0]).slice(0, 160),
|
|
2195
|
+
from: boardAddress(board.label), from_name: `${board.label} board`, to, id: key, wake, notice: !ask, subject: String(subject ?? String(body).split('\n')[0]).slice(0, 160),
|
|
2138
2196
|
body: ask ? `[A question from the ${board.label} board] ${body}` : `[Board notice from the ${board.label} board] ${body}\n\n(An automated notice from the board's dispatcher; replies to this address go nowhere.)`,
|
|
2139
2197
|
// the board's own Teams context (the call it files in): the sender is observed in that team, never the daemon's
|
|
2140
2198
|
context: current().facts.SUPERCODE_CONTEXT ?? null,
|
package/board/incidents.mjs
CHANGED
|
@@ -1,14 +1,33 @@
|
|
|
1
1
|
// Service supervision retains evidence; only a manager/creator act releases dispatch.
|
|
2
2
|
import { event, json, now } from './store.mjs';
|
|
3
|
-
import { statusOf, creatorOf } from './workflow.mjs';
|
|
3
|
+
import { statusOf, creatorOf, requestSessionClose } from './workflow.mjs';
|
|
4
4
|
|
|
5
|
-
export const
|
|
5
|
+
export const recoversRuns = entry => entry?.params?.run_failure_policy === 'recover';
|
|
6
|
+
export const monitorsRuns = entry => !recoversRuns(entry) && entry?.params?.run_failure_policy !== 'retry';
|
|
6
7
|
// Where a thrown failure came from: its first frames, with paths from the package root, kept as the incident's
|
|
7
8
|
// evidence so a bare message ("database is locked") still names the statement's caller.
|
|
8
9
|
export const originOf = (error) => String(error?.stack ?? '').split('\n').filter((line) => /^\s+at /.test(line)).slice(0, 5)
|
|
9
10
|
.map((line) => line.trim().replace(/\((?:file:\/\/)?[^()]*\/(orchestrator\/|teams\/|volter-teams\/)/, '($1'));
|
|
10
11
|
export const failedObservation = name => ['exited', 'session_closed', 'session_lost', 'spawn_failed', 'orphaned', 'timed_out', 'stale'].includes(name);
|
|
11
12
|
|
|
13
|
+
/** Recovery is an explicit workflow policy. Keep the failure before the
|
|
14
|
+
* workflow ends its run; neither discard the incident nor hold dispatch. */
|
|
15
|
+
export function retainRecovery(db, id, name, evidence = {}, runId = null) {
|
|
16
|
+
const task = db.prepare('SELECT current_run_id FROM tasks WHERE id=?').get(id);
|
|
17
|
+
runId ??= task?.current_run_id;
|
|
18
|
+
const row = db.prepare('SELECT * FROM task_runs WHERE id=? AND task_id=?').get(runId, id);
|
|
19
|
+
if (!row || row.ended_at != null) return;
|
|
20
|
+
if (db.prepare("SELECT 1 FROM task_events WHERE task_id=? AND run_id=? AND kind='run_incident'").get(id, runId)) return;
|
|
21
|
+
const session = json(row.metadata, {})?.supercode ?? {};
|
|
22
|
+
const reason = evidence.error ?? `run ${runId} needs continuation (${name})`;
|
|
23
|
+
const payload = { observation: name, run: runId, reason, policy: 'recover',
|
|
24
|
+
session: session.session_id ?? null, machine: session.machine ?? null, pane: session.pane ?? null,
|
|
25
|
+
launch_key: session.launch_key ?? null, pid: session.pid ?? null, evidence };
|
|
26
|
+
event(db, id, 'run_incident', payload, runId);
|
|
27
|
+
db.prepare('INSERT INTO task_comments(task_id,author,body,created_at) VALUES (?,?,?,?)')
|
|
28
|
+
.run(id, 'dispatcher', `Run #${runId}: ${reason}. The workflow will continue this card through its native session door. Evidence: ${JSON.stringify(payload)}`, now());
|
|
29
|
+
}
|
|
30
|
+
|
|
12
31
|
export function surfaceIncident(db, board, entry, id, name, evidence = {}, runId = null) {
|
|
13
32
|
const task = db.prepare('SELECT * FROM tasks WHERE id=?').get(id);
|
|
14
33
|
runId ??= task?.current_run_id;
|
|
@@ -42,6 +61,24 @@ export function surfaceIncident(db, board, entry, id, name, evidence = {}, runId
|
|
|
42
61
|
|
|
43
62
|
/** Older ended runs are audited before eligibility: a completed run is not permission to repeat. */
|
|
44
63
|
export function auditEndedRuns(db, board, entry) {
|
|
64
|
+
if (recoversRuns(entry)) {
|
|
65
|
+
// Changing the explicit policy releases only incident holds. A person's
|
|
66
|
+
// pause, dependency, dispatch switch or unrelated hold remains their act.
|
|
67
|
+
for (const task of db.prepare(`SELECT t.*,r.id run_id,r.metadata run_metadata FROM tasks t JOIN task_runs r
|
|
68
|
+
ON r.id=(SELECT MAX(id) FROM task_runs WHERE task_id=t.id)
|
|
69
|
+
WHERE t.current_run_id IS NULL AND r.ended_at IS NOT NULL AND t.status IN ('running','review')`).all()) {
|
|
70
|
+
const session = json(task.run_metadata, {})?.supercode;
|
|
71
|
+
if (!session?.monitor_held) continue;
|
|
72
|
+
const state = db.prepare('SELECT paused,blocked_by,archived FROM supercode_cards WHERE task_id=?').get(task.id);
|
|
73
|
+
if (json(state?.paused, null) || json(state?.blocked_by, [])?.length || state?.archived) continue;
|
|
74
|
+
const last = db.prepare("SELECT kind,payload FROM task_events WHERE task_id=? AND kind IN ('dispatch_held','dispatch_released','session_adopted') ORDER BY id DESC LIMIT 1").get(task.id);
|
|
75
|
+
const hold = json(last?.payload, {});
|
|
76
|
+
if (last?.kind !== 'dispatch_held' || hold.run !== task.run_id || hold.incident !== session.incident) continue;
|
|
77
|
+
requestSessionClose(db, task.id, { runId: task.run_id });
|
|
78
|
+
event(db, task.id, 'dispatch_released', { reason: 'workflow explicitly selected recovery', incident: session.incident, ended_run: task.run_id });
|
|
79
|
+
}
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
45
82
|
if (!monitorsRuns(entry)) return;
|
|
46
83
|
for (const task of db.prepare(`SELECT t.*,r.id run_id,r.outcome,r.error,r.metadata run_metadata FROM tasks t JOIN task_runs r ON r.id=(SELECT MAX(id) FROM task_runs WHERE task_id=t.id)
|
|
47
84
|
WHERE t.current_run_id IS NULL AND r.ended_at IS NOT NULL AND t.status NOT IN ('done','archived')`).all()) {
|
package/board/machine-fleet.mjs
CHANGED
|
@@ -66,6 +66,7 @@ export async function readMachineFleet(machines, local) {
|
|
|
66
66
|
native.set(address, { alive: running ? true : null, states: [running ? true : null] });
|
|
67
67
|
if (running) alive.add(address);
|
|
68
68
|
sessions.push({ address, machine: row.machine, harness: activity.harness, session_id: activity.session_id, pane: pane.pane, name: pane.title ?? pane.label ?? null,
|
|
69
|
+
starter_session: pane.starter_session ?? null,
|
|
69
70
|
state: running ? 'running' : 'unknown', turn: activity.turn ?? null, native_state: activity.evidence?.native_state ?? null,
|
|
70
71
|
// what a session that needs input says it waits for and since when, where its harness publishes them
|
|
71
72
|
waiting_for: activity.evidence?.waiting_for ?? null, turn_source: activity.evidence?.source ?? null, turn_since_ms: activity.evidence?.native_state_since_ms ?? null });
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// The dispatcher observes native turns and explicit session delegation. A
|
|
2
|
+
// transcript write, a reminder reply, or sharing a profile proves no progress.
|
|
3
|
+
|
|
4
|
+
/** Sessions this session started, transitively. Cycles never imply work. */
|
|
5
|
+
export function delegatedWork(address, edges, sessions) {
|
|
6
|
+
const reached = new Set([address]), pending = [address], working = [];
|
|
7
|
+
const byStarter = new Map();
|
|
8
|
+
for (const { starter, session } of edges) {
|
|
9
|
+
if (!starter || !session || starter === session) continue;
|
|
10
|
+
const children = byStarter.get(starter) ?? [];
|
|
11
|
+
children.push(session); byStarter.set(starter, children);
|
|
12
|
+
}
|
|
13
|
+
while (pending.length) {
|
|
14
|
+
for (const child of byStarter.get(pending.pop()) ?? []) {
|
|
15
|
+
if (reached.has(child)) continue;
|
|
16
|
+
reached.add(child); pending.push(child);
|
|
17
|
+
const reading = sessions.find(row => row.address === child);
|
|
18
|
+
if (reading?.state === 'running' && reading.turn === 'working') working.push(child);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return working;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Idle time belongs to a native idle interval, never to transcript freshness. */
|
|
25
|
+
export function idleInterval(session, live, observedAt) {
|
|
26
|
+
const nativeSince = Number(live?.turn_since_ms);
|
|
27
|
+
const since = nativeSince > 0 && nativeSince <= observedAt * 1000
|
|
28
|
+
? Math.floor(nativeSince / 1000) : Number(session.idle_since ?? observedAt);
|
|
29
|
+
return { since, quiet: Math.max(0, observedAt - since) };
|
|
30
|
+
}
|
package/board/workflow.mjs
CHANGED
|
@@ -246,6 +246,14 @@ async function observeLastCommit(path, id, branches = []) {
|
|
|
246
246
|
}
|
|
247
247
|
|
|
248
248
|
function lastCommitAt(path, id) { return path ? commitTimes.get(`${path}:${id}`)?.value ?? null : null; }
|
|
249
|
+
/** Material card work. Comments, reminder replies and timestamp writes are
|
|
250
|
+
* deliberately absent; a heartbeat is a separately bounded waiting lease. */
|
|
251
|
+
export function sessionProgress(db, task) {
|
|
252
|
+
const event = one(db, `SELECT MAX(e.id) AS id FROM task_events e WHERE
|
|
253
|
+
(e.task_id=? AND e.kind IN ('specified','review_requested','review_pass','changes_requested','audited')) OR
|
|
254
|
+
(e.task_id IN (SELECT task_id FROM supercode_cards WHERE subtask_of=?) AND e.kind IN ('completed','cancelled'))`, task.id, task.id);
|
|
255
|
+
return { event: Number(event?.id ?? 0), commit: Number(lastCommitAt(task.workspace_path, task.id) ?? 0) };
|
|
256
|
+
}
|
|
249
257
|
/** Observe external facts before entering an optimistic board mutation. */
|
|
250
258
|
export async function refreshCardInputs(board, id, { commits = false } = {}) {
|
|
251
259
|
const task=board.db.prepare('SELECT t.*,c.subtask_of FROM tasks t LEFT JOIN supercode_cards c ON c.task_id=t.id WHERE t.id=?').get(id);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-orchestrator",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.112",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
|
|
6
6
|
"exports": {
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@volter/supercode-harness-sdk": "^0.3.60",
|
|
28
|
-
"@volter/teams": "0.2.
|
|
28
|
+
"@volter/teams": "0.2.46",
|
|
29
29
|
"@volter/ztrack": "^2.4.23",
|
|
30
30
|
"yaml": "^2.9.0"
|
|
31
31
|
},
|