@volter/supercode-orchestrator 0.5.214 → 0.5.216
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -9
- package/activation.mjs +47 -11
- package/adapters/chat-sdk.mjs +82 -15
- package/adapters/rh2.mjs +11 -7
- package/agents.mjs +1 -5
- package/bin/agent-channel.mjs +33 -177
- package/bin/connection-runner.mjs +3 -17
- package/bin/orchestrator.mjs +12 -7
- package/bin/runtime.mjs +74 -51
- package/board/call.mjs +2 -1
- package/board/ci.mjs +55 -28
- package/board/cli.mjs +117 -13
- package/board/client.mjs +9 -0
- package/board/dispatch.mjs +45 -15
- package/board/engine.mjs +2 -2
- package/board/fence.mjs +2 -0
- package/board/generation-owner.mjs +293 -0
- package/board/generation.mjs +49 -0
- package/board/listing.mjs +17 -5
- package/board/owner-door-worker.mjs +22 -7
- package/board/owner-door.mjs +2 -2
- package/board/owner-files.mjs +1 -1
- package/board/published.mjs +12 -7
- package/board/reply.mjs +2 -2
- package/board/session-work.mjs +85 -15
- package/connection.mjs +14 -20
- package/cron.mjs +57 -4
- package/daemon-control.d.ts +39 -0
- package/daemon-control.mjs +275 -0
- package/hermes-locks.mjs +13 -3
- package/index.d.ts +9 -4
- package/installed-apps.mjs +35 -14
- package/kanban-worker.mjs +4 -4
- package/lease.mjs +1 -1
- package/loop.mjs +198 -47
- package/mcp/mount.mjs +6 -5
- package/orchestration.mjs +11 -17
- package/package.json +10 -7
- package/reducer.mjs +31 -6
- package/source-lock.mjs +15 -24
- package/webhook.mjs +65 -12
package/board/owner-files.mjs
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import { realpathSync } from 'node:fs';
|
|
4
4
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
5
5
|
|
|
6
|
-
const RUNTIME = new Set(['board.lock', 'board.sock']);
|
|
6
|
+
const RUNTIME = new Set(['board.lock', 'board.sock', 'board-generation.json']);
|
|
7
7
|
let home = null;
|
|
8
8
|
|
|
9
9
|
/** The home this process owns (the owner's, set once when it binds). */
|
package/board/published.mjs
CHANGED
|
@@ -6,7 +6,7 @@ import { readFileSync } from 'node:fs';
|
|
|
6
6
|
// supercode_publication_dirty, whoever writes (this process, Hermes's native writer, a document import). A transaction
|
|
7
7
|
// publishes the marked cards and clears their marks; one that marked nothing reads one row and publishes nothing.
|
|
8
8
|
import { createHash, randomUUID } from 'node:crypto';
|
|
9
|
-
import { executionOf } from './session-work.mjs';
|
|
9
|
+
import { executionOf, readingContext } from './session-work.mjs';
|
|
10
10
|
import { cardState } from './card-state.mjs';
|
|
11
11
|
import { workflowFor } from './workflow.mjs';
|
|
12
12
|
import { cardAgentScopeTransition } from '../agents.mjs';
|
|
@@ -40,13 +40,13 @@ CREATE INDEX IF NOT EXISTS supercode_publication_heads_sequence ON supercode_pub
|
|
|
40
40
|
// is where a prune under way resumes, null when none is (prunePublications).
|
|
41
41
|
export const PUBLICATION_COLUMNS = { supercode_publications: { snapshot_bytes: 'INTEGER' }, supercode_publication_heads: { fingerprint: 'TEXT' }, supercode_publication_log: { prune_from: 'INTEGER' } };
|
|
42
42
|
|
|
43
|
-
// The snapshot's shape: 1 names the card as a person does (`ref`, VO-856).
|
|
44
|
-
const SNAPSHOT_SHAPE =
|
|
43
|
+
// The snapshot's shape: 1 names the card as a person does (`ref`, VO-856); 5 lists the files attached to it (`attachments`).
|
|
44
|
+
const SNAPSHOT_SHAPE = 5;
|
|
45
45
|
const SNAPSHOT_RELEASE = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
|
|
46
46
|
const REVIEW_KIND_LIST = ['review_requested', 'approved', 'review_approved', 'changes_requested', 'escalated'];
|
|
47
47
|
export const AGENT_KIND_LIST = ['agent_declared', 'agent_assignment_ended', 'agent_assignment_reassigned', 'agent_assignment_adopted'];
|
|
48
48
|
// The tables a card's snapshot reads (cardSnapshot), each with the column naming its card; a link names two.
|
|
49
|
-
const MARKED = [['tasks', ['id']], ['supercode_cards', ['task_id']], ['task_comments', ['task_id']], ['task_runs', ['task_id']], ['task_links', ['parent_id', 'child_id']], ['task_events', ['task_id']], ['supercode_card_agent_facts',['task_id']], ['supercode_card_agent_scopes',['task_id']], ['supercode_card_numbers', ['task_id']]];
|
|
49
|
+
const MARKED = [['tasks', ['id']], ['supercode_cards', ['task_id']], ['task_comments', ['task_id']], ['task_runs', ['task_id']], ['task_links', ['parent_id', 'child_id']], ['task_events', ['task_id']], ['supercode_card_agent_facts',['task_id']], ['supercode_card_agent_scopes',['task_id']], ['supercode_card_numbers', ['task_id']], ['task_attachments', ['task_id']]];
|
|
50
50
|
/**
|
|
51
51
|
* The triggers that mark a changed card, on a store that has these tables (CREATE … IF NOT EXISTS: laid out on every
|
|
52
52
|
* open). A card already marked is not marked again, so a writer's many writes while the owner is away leave one mark.
|
|
@@ -75,7 +75,7 @@ export function publicationTriggers(db) {
|
|
|
75
75
|
// Reviews and agent assignment events change a snapshot: an update marks when the
|
|
76
76
|
// row was one before or is one after
|
|
77
77
|
const when = (rows) => table === 'task_events' ? ` WHEN ${rows.map((row) => `${row}.kind IN ${snapshotEvents}`).join(' OR ')}` : '';
|
|
78
|
-
const invalidate = row => ['task_comments','task_events'].includes(table) ? `UPDATE supercode_publication_heads SET fingerprint=NULL WHERE resource_id=${row}.task_id;` : '';
|
|
78
|
+
const invalidate = row => ['task_comments','task_events','task_attachments'].includes(table) ? `UPDATE supercode_publication_heads SET fingerprint=NULL WHERE resource_id=${row}.task_id;` : '';
|
|
79
79
|
const mark = (row) => invalidate(row) + columns.map((column) => `INSERT INTO supercode_publication_dirty (task_id) SELECT ${row}.${column} WHERE ${row}.${column} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM supercode_publication_dirty WHERE task_id = ${row}.${column});`).join(' ');
|
|
80
80
|
db.exec(`CREATE TRIGGER IF NOT EXISTS supercode_publish_${table}_insert AFTER INSERT ON ${table}${when(['NEW'])} BEGIN ${mark('NEW')} END;
|
|
81
81
|
CREATE TRIGGER IF NOT EXISTS supercode_publish_${table}_update AFTER UPDATE ON ${table}${when(['OLD', 'NEW'])} BEGIN ${mark('NEW')} ${mark('OLD')} END;
|
|
@@ -193,7 +193,11 @@ const perCard=db=>{
|
|
|
193
193
|
// A card's short sequential number (store.mjs CARD_NUMBERS, named here as store.mjs imports this module), in a store
|
|
194
194
|
// that has numbered its cards.
|
|
195
195
|
const numbered=!!db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='supercode_card_numbers'").get();
|
|
196
|
+
const attached=!!db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='task_attachments'").get();
|
|
196
197
|
return {
|
|
198
|
+
// The files attached to it, by their board ids, as the board door lists and serves them (attachment-get, attachment-read):
|
|
199
|
+
// never where the board keeps them.
|
|
200
|
+
attachments:id=>attached?db.prepare('SELECT id,filename,content_type,size,uploaded_by,created_at FROM task_attachments WHERE task_id=? ORDER BY id').all(id):[],
|
|
197
201
|
number:id=>numbered?db.prepare('SELECT number FROM supercode_card_numbers WHERE task_id=?').get(id)?.number??null:null,
|
|
198
202
|
own:id=>db.prepare('SELECT * FROM supercode_cards WHERE task_id=?').get(id),
|
|
199
203
|
comments:id=>db.prepare('SELECT id,author,body,created_at FROM task_comments WHERE task_id=? ORDER BY id').all(id),
|
|
@@ -244,13 +248,14 @@ export function cardSnapshot(db,row,rows=perCard(db),key=null,options={}) {
|
|
|
244
248
|
const lane=own.stage??row.status;
|
|
245
249
|
const archived=Boolean(own.archived || lane==='archived');
|
|
246
250
|
return {id:row.id,readable_id:row.readable_id??null,number,ref:key&&number?`${key}-${number}`:null,title:row.title,body:row.body??'',assignee:row.assignee,status:taskStatusOf(lane,row.completed_at),board_status:cardState({...row,stage:lane,paused:own.paused,blocked_by:own.blocked_by},options).status,lane,
|
|
247
|
-
execution:executionOf(runs.find(run=>run.id===row.current_run_id)),
|
|
251
|
+
execution:executionOf(runs.find(run=>run.id===row.current_run_id),readingContext(db,row.id,row.current_run_id)),
|
|
248
252
|
priority:row.priority,created_by:row.created_by,created_at:at(row.created_at),started_at:at(row.started_at),completed_at:at(row.completed_at),result:row.result,skills:parse(row.skills,[]),model:row.model_override,provider:row.provider_override,
|
|
249
253
|
tags:parse(own.tags,null)??(row.tenant?[row.tenant]:[]),requested_by_unverified:parse(own.requested_by_unverified),machine:own.machine??null,subtask_of:own.subtask_of??null,archived,archive_reason:archived?rows.archiveReason(row.id):null,
|
|
250
254
|
blocked_by:parse(own.blocked_by,[]),paused:parse(own.paused),detached:parse(own.detached),every:own.every??null,
|
|
251
255
|
workspace:{kind:row.workspace_kind,path:row.workspace_path,branch:row.branch_name,root:own.workspace_root,anchor:own.workspace_anchor},
|
|
252
256
|
parents:rows.parents(row.id),
|
|
253
|
-
agents:rows.agents(row.id),reviews,children:rows.children(row.id),comments,attempts
|
|
257
|
+
agents:rows.agents(row.id),reviews,children:rows.children(row.id),comments,attempts,
|
|
258
|
+
attachments:rows.attachments(row.id).map(file=>({id:file.id,name:file.filename,content_type:file.content_type??null,size:file.size,by:file.uploaded_by??null,at:at(file.created_at)}))};
|
|
254
259
|
}
|
|
255
260
|
|
|
256
261
|
/**
|
package/board/reply.mjs
CHANGED
|
@@ -26,8 +26,8 @@ async function write(socket, text) {
|
|
|
26
26
|
return !socket.destroyed;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
export async function writeOwnerAnswer(socket, { code = 0, stdout = '', stderr = '' }) {
|
|
30
|
-
if (!await write(socket, `{"code":${JSON.stringify(code)},"stdout":"`)) return;
|
|
29
|
+
export async function writeOwnerAnswer(socket, { code = 0, stdout = '', stderr = '', unanswered = false }) {
|
|
30
|
+
if (!await write(socket, `{"code":${JSON.stringify(code)},${unanswered ? '"unanswered":true,' : ''}"stdout":"`)) return;
|
|
31
31
|
for (const [index, value] of [stdout, stderr].entries()) {
|
|
32
32
|
let carry = '';
|
|
33
33
|
for (const part of Array.isArray(value) ? value : [value]) {
|
package/board/session-work.mjs
CHANGED
|
@@ -1,23 +1,93 @@
|
|
|
1
1
|
// The dispatcher observes native turns and explicit session delegation. A
|
|
2
2
|
// transcript write, a reminder reply, or sharing a profile proves no progress.
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
4
|
+
const parsed = (value) => {
|
|
5
|
+
if (typeof value !== 'string') return value ?? null;
|
|
6
|
+
try { return JSON.parse(value); } catch { return null; }
|
|
7
|
+
};
|
|
8
|
+
const iso = (seconds) => Number(seconds) > 0 ? `${new Date(Number(seconds) * 1000).toISOString().slice(0, 16)}Z` : null;
|
|
9
|
+
const oneLine = (text, max = 200) => String(text ?? '').trim().split('\n')[0].slice(0, max);
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* A retained reading of the card's session, beside the card's workflow lane and apart from its launch receipt (ADR 0024).
|
|
13
|
+
* `state` is what the session is doing as the board last read it, and `reason` says it in words wherever the session is
|
|
14
|
+
* not plainly working: no session, ended, its machine unreachable, unread, waiting until a dated time (the card's last
|
|
15
|
+
* heartbeat `--until`), waiting on its user, or idle since a time. An ended attempt is never a live session; an unread
|
|
16
|
+
* live attempt is unknown.
|
|
17
|
+
* `heartbeat` is the current run's latest heartbeat event on the card ({ payload: { note, until }, at, run_id }) beside the
|
|
18
|
+
* run's own, `last` its latest run when it has no current one; both are optional (readingContext reads them).
|
|
19
|
+
*/
|
|
20
|
+
export function executionOf(run, { heartbeat = null, last = null } = {}) {
|
|
21
|
+
if (!run) {
|
|
22
|
+
const ended = last?.ended_at != null ? last : null;
|
|
23
|
+
return { state: 'awaiting_session', alive: false, observed_at: null, since: ended ? Number(ended.ended_at) : null,
|
|
24
|
+
reason: ended ? `no session works it; its last run ended${ended.outcome ?? ended.status ? ` (${ended.outcome ?? ended.status})` : ''} at ${iso(ended.ended_at)}${ended.error ? `: ${oneLine(ended.error)}` : ''}`
|
|
25
|
+
: 'no session works it' };
|
|
26
|
+
}
|
|
27
|
+
const session = parsed(run.metadata)?.supercode;
|
|
28
|
+
const machine = session?.machine ?? null;
|
|
29
|
+
if (run.ended_at != null || session?.close_confirmed_at || session?.released) {
|
|
30
|
+
const at = run.ended_at ?? session?.close_confirmed_at ?? null;
|
|
31
|
+
return { state: 'ended', alive: false, observed_at: session?.presence_observation?.observed_at ?? null, since: at == null ? null : Number(at),
|
|
32
|
+
reason: `its session ended${at ? ` at ${iso(at)}` : ''}` };
|
|
33
|
+
}
|
|
12
34
|
const reading = session?.presence_observation;
|
|
13
|
-
if (!reading) return { state: 'unknown', alive: null, observed_at: null };
|
|
35
|
+
if (!reading) return { state: 'unknown', alive: null, observed_at: null, reason: 'the board has not read its session yet' };
|
|
14
36
|
const freshUntil = Number(reading.observed_at) + 90;
|
|
15
|
-
// A stopped dispatcher cannot keep advertising its last
|
|
16
|
-
// Cached snapshot consumers use the same expiry
|
|
17
|
-
if (Date.now() / 1000 > freshUntil) return { state: 'unknown', alive: null, observed_at: reading.observed_at, fresh_until: freshUntil
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
37
|
+
// A stopped dispatcher cannot keep advertising its last reading, its machine wait included: every round renews the
|
|
38
|
+
// observation (supervise), so one older than its freshness is unknown. Cached snapshot consumers use the same expiry.
|
|
39
|
+
if (Date.now() / 1000 > freshUntil) return { state: 'unknown', alive: null, observed_at: reading.observed_at, fresh_until: freshUntil,
|
|
40
|
+
reason: `the board has not read its session since ${iso(reading.observed_at)}` };
|
|
41
|
+
const read = { ...reading, fresh_until: freshUntil };
|
|
42
|
+
// the run's machine wait: the board read the session's machine as unlinked from the team (supervise, machine_offline)
|
|
43
|
+
if (session.offline_since != null) {
|
|
44
|
+
return { ...read, state: 'unreachable', machine, since: Number(session.offline_since),
|
|
45
|
+
reason: `machine ${machine ?? '(unnamed)'} unreachable since ${iso(session.offline_since)}; the board cannot see its session` };
|
|
46
|
+
}
|
|
47
|
+
// the session's machine as the same round read it, where nothing else read the session: unlinked from the team is
|
|
48
|
+
// unreachable; a record that is out of date or not held says nothing of the link, and its session is unknown
|
|
49
|
+
if (reading.alive == null && reading.machine_reading && reading.machine_reading !== 'online') {
|
|
50
|
+
const offline = reading.machine_reading === 'offline';
|
|
51
|
+
return { ...read, state: offline ? 'unreachable' : 'unknown', machine,
|
|
52
|
+
reason: `machine ${machine ?? '(unnamed)'} ${offline ? 'is unlinked from the team' : 'cannot be read'}${reading.machine_error ? `: ${oneLine(reading.machine_error)}` : ''}; the board cannot see its session` };
|
|
53
|
+
}
|
|
54
|
+
if (reading.alive === false) return { ...read, state: 'ended', reason: `its session is gone from ${machine ?? 'its machine'}` };
|
|
55
|
+
if (reading.alive == null) return { ...read, state: 'unknown', reason: `its session on ${machine ?? 'its machine'} could not be read` };
|
|
56
|
+
if (session?.idle_observation?.decision === 'delegated_work' && session.idle_observation.turn === reading.turn) return { ...read, state: 'delegating', reason: null };
|
|
57
|
+
if (reading.turn === 'working') return { ...read, state: 'working', reason: null };
|
|
58
|
+
if (reading.turn === 'needs_input') {
|
|
59
|
+
const asked = typeof session?.question === 'string' ? session.question : session?.question?.question;
|
|
60
|
+
return { ...read, state: 'needs_input', reason: asked ? `its session asks: ${oneLine(asked)}` : 'its session waits on its user' };
|
|
61
|
+
}
|
|
62
|
+
// a bounded wait the session declared on its card (heartbeat --until), still ahead: the run's own heartbeat (a workflow
|
|
63
|
+
// with a run machine keeps it on the run, as the workflow's `card.last_events` reads it) or the card's, the later
|
|
64
|
+
// (only this run's: a wait an earlier run declared is not this session's)
|
|
65
|
+
const own = parsed(run.metadata)?.last_events?.heartbeat ?? null;
|
|
66
|
+
const card = heartbeat && run.id != null && heartbeat.run_id != null && Number(heartbeat.run_id) === Number(run.id) ? heartbeat : null;
|
|
67
|
+
const beat = own && !(Number(card?.at) > Number(own.at)) ? own : card;
|
|
68
|
+
const wait = parsed(beat?.payload);
|
|
69
|
+
if (Number(wait?.until) > Date.now() / 1000) {
|
|
70
|
+
return { ...read, state: 'waiting', until: Number(wait.until), since: Number(beat.at) || null,
|
|
71
|
+
reason: `waiting until ${iso(wait.until)}${wait.note ? `: ${oneLine(wait.note)}` : ''}` };
|
|
72
|
+
}
|
|
73
|
+
if (reading.turn === 'idle') {
|
|
74
|
+
const since = Number(session?.idle_since) || null;
|
|
75
|
+
return { ...read, state: 'idle', since, reason: since ? `idle, no turn since ${iso(since)}` : 'idle' };
|
|
76
|
+
}
|
|
77
|
+
const unknownSince = Number(session?.unknown_since) || null;
|
|
78
|
+
return { ...read, state: reading.turn ?? 'live', since: unknownSince,
|
|
79
|
+
reason: reading.turn ? null : `its session's turn is not known${unknownSince ? ` since ${iso(unknownSince)}` : ''}` };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** What executionOf reads beside a card's current run, for a card in a working lane (running or review) only, as the
|
|
83
|
+
* listing reads it (listing.mjs READING_CONTEXT): the current run's latest heartbeat event, and the latest run when
|
|
84
|
+
* none is current. Any other lane reads none. */
|
|
85
|
+
export function readingContext(db, taskId, currentRunId) {
|
|
86
|
+
const lane = db.prepare('SELECT COALESCE(c.stage,t.status) AS lane FROM tasks t LEFT JOIN supercode_cards c ON c.task_id=t.id WHERE t.id=?').get(taskId)?.lane;
|
|
87
|
+
if (!['running', 'review'].includes(lane)) return { heartbeat: null, last: null };
|
|
88
|
+
const heartbeat = currentRunId ? db.prepare("SELECT payload,created_at AS at,run_id FROM task_events WHERE task_id=? AND kind='heartbeat' AND run_id=? ORDER BY id DESC LIMIT 1").get(taskId, currentRunId) ?? null : null;
|
|
89
|
+
const last = currentRunId ? null : db.prepare('SELECT status,outcome,ended_at,error FROM task_runs WHERE task_id=? ORDER BY id DESC LIMIT 1').get(taskId) ?? null;
|
|
90
|
+
return { heartbeat, last };
|
|
21
91
|
}
|
|
22
92
|
|
|
23
93
|
/** Sessions this session started, transitively. Cycles never imply work. */
|
package/connection.mjs
CHANGED
|
@@ -8,13 +8,14 @@ import { fromContext } from '@volter/teams/context';
|
|
|
8
8
|
import { supercodeHome } from '@volter/teams/home';
|
|
9
9
|
|
|
10
10
|
// RH2's modules, each its own program under --modules-root.
|
|
11
|
-
export const MODULE_APPS = ['
|
|
11
|
+
export const MODULE_APPS = ['orchestration-rh2'];
|
|
12
12
|
// supercode's Workplace service's registrations (supercode ADR 0041 decision 3; runhuman-2 ADR 0055 decision 3): one program,
|
|
13
13
|
// sdk/teams/workplace/main.mjs, hosts them all, each with its own credentials on both sides. Their manifests ship here, so an
|
|
14
14
|
// organization's file selects them by these names and `rh2 apply-org` reads them with --modules-root SERVICE_REGISTRATIONS.
|
|
15
15
|
export const SERVICE_APPS = ['agents','cards','machines'];
|
|
16
|
-
|
|
17
|
-
export const
|
|
16
|
+
const SERVICE_ROOT = new URL('./workplace/', import.meta.resolve('@volter/supercode-teams'));
|
|
17
|
+
export const SERVICE_REGISTRATIONS = fileURLToPath(new URL('registrations/', SERVICE_ROOT));
|
|
18
|
+
export const SERVICE_PROGRAM = fileURLToPath(new URL('main.mjs', SERVICE_ROOT));
|
|
18
19
|
const SERVICE_TOKENS = { agents: ['AGENTS_WORKPLACE_TOKEN','AGENTS_TEAMS_TOKEN'], cards: ['CARDS_WORKPLACE_TOKEN','CARDS_TEAMS_TOKEN'], machines: ['MACHINES_WORKPLACE_TOKEN','MACHINES_TEAMS_TOKEN'] };
|
|
19
20
|
export function privateJson(path, value) {
|
|
20
21
|
mkdirSync(dirname(path),{recursive:true,mode:0o700});
|
|
@@ -48,14 +49,10 @@ export function generateModuleConfig(root, record, grant) {
|
|
|
48
49
|
const folder=dirname(connectionPath(root,record.id)), dir=join(folder,grant.name);
|
|
49
50
|
mkdirSync(dir,{recursive:true,mode:0o700});
|
|
50
51
|
const custody=join(dir,'credentials.env'), configPath=join(dir,'config.json');
|
|
51
|
-
writeFileSync(custody,`RH2_TOKEN=${grant.rh2.accessToken}\n
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
teams:{server:record.teams.server,team:record.teams.team,source:record.source,tokenEnv:'TEAMS_TOKEN'},
|
|
56
|
-
envFile:custody,stateFile:join(dir,'state.json'),defaultFloor:record.floor,
|
|
57
|
-
source:{key:`connection-${record.scope}`,namespace:`cx${record.scope}`,name:`${record.organization} / ${record.source}`,home:root,...(record.teamsContext?{teamsContext:record.teamsContext}:{}),...(process.env.SUPERCODE_BIN?{program:process.env.SUPERCODE_BIN}:{}),harness:record.harness??'hermes'},
|
|
58
|
-
alerts:{role:record.alertRole??'admin'},standingFile:join(folder,'standing.json')};
|
|
52
|
+
writeFileSync(custody,`RH2_TOKEN=${grant.rh2.accessToken}\n`,{mode:0o600});chmodSync(custody,0o600);
|
|
53
|
+
const config={rh2:{baseUrl:record.origin,issuer:record.issuer,organization:record.organization,tokenEnv:'RH2_TOKEN'},envFile:custody,
|
|
54
|
+
source:{key:`connection-${record.scope}`,name:`${record.organization} / ${record.source}`,home:root,...(process.env.SUPERCODE_BIN?{program:process.env.SUPERCODE_BIN}:{}),harness:record.harness??'hermes'},
|
|
55
|
+
alerts:{role:record.alertRole??'admin'}};
|
|
59
56
|
privateJson(configPath,config);
|
|
60
57
|
privateJson(join(root,'apps',`${grant.name}.${record.id}.json`),{app:grant.name,connection:record.id,configPath,moduleRoot:record.moduleRoot,
|
|
61
58
|
origin:record.origin,organizationId:record.organization,principalId:grant.rh2.principalId,installationId:grant.rh2.installationId,expiresAt:grant.rh2.expiresAt});
|
|
@@ -105,7 +102,7 @@ export async function connectWorkplace(flags) {
|
|
|
105
102
|
if(Boolean(flags['webhook-url'])!==Boolean(flags['webhook-listen']))throw new Error('--webhook-url and --webhook-listen go together: where the Workplace sends the agents registration\'s events, and the loopback address the service listens on');
|
|
106
103
|
const moduleRoot=flags['modules-root']?resolve(flags['modules-root']):null;
|
|
107
104
|
// An explicit local credential input is useful for an owned stack; a normal connect uses its existing Teams context.
|
|
108
|
-
let teams,owner
|
|
105
|
+
let teams,owner;
|
|
109
106
|
if(flags['teams-token-file']){
|
|
110
107
|
if(!flags['teams-url']||!flags.team)throw new Error('--teams-token-file needs --teams-url and --team');
|
|
111
108
|
teams={server:new URL(flags['teams-url']).origin,team:flags.team};owner=readFileSync(flags['teams-token-file'],'utf8').trim();
|
|
@@ -118,7 +115,7 @@ export async function connectWorkplace(flags) {
|
|
|
118
115
|
// Asked of the daemon whatever the environment says: a resumed session can lack its launch's SUPERCODE_AGENT_ID,
|
|
119
116
|
// and the daemon's retained binding is what names its agent (workspace.mjs does the same for every Teams verb).
|
|
120
117
|
const env=await agentSessionEnvironment(process.env,agentSessionHint(process.env));
|
|
121
|
-
teamsContext=agentContextName(contexts,env,flags.context??null);
|
|
118
|
+
const teamsContext=agentContextName(contexts,env,flags.context??null);
|
|
122
119
|
const {context}=fromContext({supercodeHome:supercodeHome(),name:teamsContext});
|
|
123
120
|
teams={server:context.origin,team:context.team_id};owner=contexts.credential(context.credential_ref).token;
|
|
124
121
|
}
|
|
@@ -128,8 +125,8 @@ export async function connectWorkplace(flags) {
|
|
|
128
125
|
const id=createHash('sha256').update(`${ticket.scope}\0${ticket.issuer}\0${ticket.organization}\0${teams.server}\0${teams.team}\0${flags.source}`).digest('hex').slice(0,24);
|
|
129
126
|
const path=connectionPath(root,id),prior=existsSync(path)?JSON.parse(readFileSync(path,'utf8')):null;
|
|
130
127
|
if(!prior&&!(Date.parse(ticket.expiresAt)>Date.now()))throw new Error('connection ticket expired; apply the organization file again');
|
|
131
|
-
const record=prior??{version:1,id,scope:ticket.scope,
|
|
132
|
-
selection:ticket.grants.map(g=>g.name),moduleRoot,...(flags['webhook-url']?{webhook:{url:flags['webhook-url'],listen:flags['webhook-listen']}}:{}),
|
|
128
|
+
const record=prior??{version:1,id,scope:ticket.scope,origin:ticket.origin,issuer:ticket.issuer,organization:ticket.organization,teams,source:flags.source,
|
|
129
|
+
selection:ticket.grants.map(g=>g.name),moduleRoot,...(flags['webhook-url']?{webhook:{url:flags['webhook-url'],listen:flags['webhook-listen']}}:{}),alertRole:flags['alert-role']??'admin',grants:[]};
|
|
133
130
|
record.root=root;
|
|
134
131
|
if(prior?.disconnectedAt)throw new Error('this connection was disconnected; apply the organization file again for a new ticket');
|
|
135
132
|
// The service's event address is part of the connection: a different one is a new connection, never silently dropped.
|
|
@@ -145,15 +142,13 @@ export async function connectWorkplace(flags) {
|
|
|
145
142
|
return {connection:id,root,modules:record.grants.map(g=>g.name),runner:running.pid,calls:[]};
|
|
146
143
|
}
|
|
147
144
|
}
|
|
148
|
-
record.standingSelection=flags['standing-file']?JSON.parse(readFileSync(flags['standing-file'],'utf8')):(ticket.grants.find(g=>g.name==='arcs-rooms')?.rooms??[]).map(seat=>({profile:seat.key,roomKey:seat.key,roleKeys:seat.roleKeys}));
|
|
149
|
-
if(record.standingSelection.some(row=>!/^[-a-z0-9]{1,64}$/.test(row.profile??'')||!/^[-a-z0-9]{1,64}$/.test(row.roomKey??'')||row.session||row.agent))throw new Error('standing selections name a profile and Room key; agent/session ids come from the home');
|
|
150
145
|
// Registry entries may become visible during setup. They are never permission to start a writer.
|
|
151
146
|
record.preparing=true;record.paused=true;
|
|
152
147
|
privateJson(path,record);
|
|
153
148
|
for(const selected of ticket.grants){
|
|
154
149
|
let grant=record.grants.find(g=>g.name===selected.name);
|
|
155
|
-
if(grant && JSON.stringify([grant.manifest,grant.teamsManifest
|
|
156
|
-
if(!grant){grant={name:selected.name,clientId:selected.clientId,
|
|
150
|
+
if(grant && JSON.stringify([grant.manifest,grant.teamsManifest])!==JSON.stringify([selected.manifest,selected.teamsManifest]))throw new Error('existing connection consent differs from the ticket');
|
|
151
|
+
if(!grant){grant={name:selected.name,clientId:selected.clientId,manifest:selected.manifest,teamsManifest:selected.teamsManifest};record.grants.push(grant);privateJson(path,record);}
|
|
157
152
|
if(SERVICE_APPS.includes(grant.name)&&!selected.teamsManifest)throw new Error(`${grant.name}: its Teams manifest is missing from the ticket (apply with --modules-root naming ${SERVICE_REGISTRATIONS})`);
|
|
158
153
|
if(!grant.rh2){
|
|
159
154
|
const recovering=typeof grant.redemption==='string';
|
|
@@ -191,7 +186,6 @@ export async function connectWorkplace(flags) {
|
|
|
191
186
|
if(!record.issuerRegistration){record.issuerStagedSecret??=`sca_${randomBytes(32).toString('base64url')}`;privateJson(path,record);record.issuerRegistration=(await connectionCall(record,'teams','POST',`/api/v1/teams/${encodeURIComponent(teams.team)}/apps`,{name:`Workplace issuer (${id.slice(0,8)})`,description:'The Workplace as a delegated issuer, verified against its JWKS',actions:['delegation.sessions','delegation.surfaces','delegation.notices','delegation.machines'],event_kinds:[],redirect_urls:['http://127.0.0.1/connected'],issuer:record.issuer,secret:record.issuerStagedSecret},owner)).data;privateJson(path,record);}
|
|
192
187
|
if(!record.issuerConsent&&!record.issuerCredential){record.issuerStagedCode??=`sci_${randomBytes(32).toString('base64url')}`;privateJson(path,record);record.issuerConsent=(await connectionCall(record,'teams','POST',`/api/v1/teams/${encodeURIComponent(teams.team)}/apps/${record.issuerRegistration.id}/installations`,{redirect_url:'http://127.0.0.1/connected',state:id,code:record.issuerStagedCode},owner)).data;privateJson(path,record);}
|
|
193
188
|
if(!record.issuerCredential){record.issuerStagedCredential??=newInstallationToken();privateJson(path,record);record.issuerCredential=await connectionCall(record,'teams','POST','/api/v1/apps/token',{app_id:record.issuerRegistration.id,app_secret:record.issuerStagedSecret,code:record.issuerStagedCode,redirect_url:'http://127.0.0.1/connected',credential_token:record.issuerStagedCredential});privateJson(path,record);delete record.issuerConsent;delete record.issuerRegistration.secret;privateJson(path,record);}
|
|
194
|
-
privateJson(join(dirname(path),'standing.json'),[]);
|
|
195
189
|
record.connectedAt??=new Date().toISOString();
|
|
196
190
|
record.preparing=false;record.paused=Boolean(flags['no-start']);privateJson(path,record);
|
|
197
191
|
const pid=record.paused?null:startConnectionRunner(root,id);
|
package/cron.mjs
CHANGED
|
@@ -716,18 +716,28 @@ function reapStaleFires(state, profile, now, effects, limitS) {
|
|
|
716
716
|
// an ended binding lives under a started_at-qualified slot, so search by the fire it ran
|
|
717
717
|
const binding = Object.values(profile.bindings).find((b) => b.residue?.fire_id === fire.id);
|
|
718
718
|
if (binding && !binding.ended_at) {
|
|
719
|
+
const ks = surfaceKeyString(binding.key);
|
|
720
|
+
const lid = leaseId(profile.name, ks);
|
|
721
|
+
// The lease, the start mark and the activity clock live in this daemon's memory only (§2.8). A run this daemon
|
|
722
|
+
// holds no worker for, is not starting and has never heard from had its worker on an earlier daemon: a stop
|
|
723
|
+
// closes every runtime (loop.mjs `stop`) and a crash takes them with it. That run is not silent, it is cut off,
|
|
724
|
+
// so its session is reopened and told to carry on, rather than timed out against a clock this daemon never kept.
|
|
725
|
+
if (rt.activity[lid] === undefined && !rt.leases[lid]) {
|
|
726
|
+
// a re-attach after a crash is in flight (handleRestart): it attaches, or interrupts and the next tick resumes
|
|
727
|
+
if (!rt.starting[lid]) resumeFire(state, profile, ks, binding, fire, now, effects);
|
|
728
|
+
continue;
|
|
729
|
+
}
|
|
719
730
|
// Hermes's inactivity watchdog, transcribed: a worker silent for the
|
|
720
731
|
// limit is a hung run. Without it the fire stays in flight and its job
|
|
721
732
|
// is never due again. Silence counts from the last worker event, the
|
|
722
|
-
// lease
|
|
733
|
+
// lease, the resume or the claim.
|
|
723
734
|
if (!(limitS > 0)) continue;
|
|
724
|
-
const ks = surfaceKeyString(binding.key);
|
|
725
|
-
const lid = leaseId(profile.name, ks);
|
|
726
735
|
const lastAt = [rt.activity[lid], rt.leases[lid]?.since, fire.started_at, fire.claimed_at].filter(Boolean).sort().pop();
|
|
727
736
|
const idle = Math.round((Date.parse(now) - Date.parse(lastAt)) / 1000);
|
|
728
737
|
if (idle < limitS) continue;
|
|
729
738
|
const job = profile.jobs[fire.job_id] ?? null;
|
|
730
|
-
const
|
|
739
|
+
const unstarted = rt.leases[lid] ? '' : fire.residue?.resumed_at ? `, its worker resumed at ${fire.residue.resumed_at} never started` : ', the worker never started';
|
|
740
|
+
const why = `no activity from the worker for ${idle}s (limit ${limitS}s, HERMES_CRON_TIMEOUT; last activity ${lastAt}${unstarted}); the run was stopped`;
|
|
731
741
|
finishFire(profile, fire, job, 'timeout', now, why, effects);
|
|
732
742
|
deliverResult(state, profile, { job, fire, sub: null }, why, true, now, effects);
|
|
733
743
|
endBinding(state, profile.name, ks, 'error', now, effects); // the fire records the timeout; EndReason is closed
|
|
@@ -742,6 +752,46 @@ function reapStaleFires(state, profile, now, effects, limitS) {
|
|
|
742
752
|
}
|
|
743
753
|
}
|
|
744
754
|
|
|
755
|
+
/** A run cut off by more restarts than this is failed with that reason rather than resumed again. */
|
|
756
|
+
const MAX_FIRE_RESUMES = 3;
|
|
757
|
+
|
|
758
|
+
/**
|
|
759
|
+
* A run whose worker went away with an earlier daemon, picked up by this one: the fire's own session is reopened
|
|
760
|
+
* (`resume_runtime_id`, which the stop or the failed re-attach recorded; a session that will not reopen starts fresh in
|
|
761
|
+
* the loop) on the job's worker, and its first line says the run was cut off and carries the job's prompt, so a
|
|
762
|
+
* reopened session continues and a fresh one does the job whole. Its silence counts from now.
|
|
763
|
+
*/
|
|
764
|
+
function resumeFire(state, profile, ks, binding, fire, now, effects) {
|
|
765
|
+
const rt = ensureRuntime(state);
|
|
766
|
+
const lid = leaseId(profile.name, ks);
|
|
767
|
+
const job = profile.jobs[fire.job_id] ?? null;
|
|
768
|
+
const cut = `the orchestrator restarted at ${now} while this run was in flight`;
|
|
769
|
+
const resumes = Number(fire.residue?.resumes ?? 0);
|
|
770
|
+
const budget = job && profile.worker ? budgetRefusal(state, profile, profile.name, now) : null;
|
|
771
|
+
const refused = !job ? 'its job is gone'
|
|
772
|
+
: !profile.worker ? `profile ${profile.name} has no worker configured`
|
|
773
|
+
: resumes >= MAX_FIRE_RESUMES ? `it was already resumed ${resumes} times (the last at ${fire.residue.resumed_at}), and each time the orchestrator restarted again before it finished`
|
|
774
|
+
: budget ? `not started: ${budget}` : null;
|
|
775
|
+
if (refused) {
|
|
776
|
+
const why = `${cut}, and it was not resumed: ${refused}`;
|
|
777
|
+
finishFire(profile, fire, job, 'failed', now, why, effects);
|
|
778
|
+
deliverResult(state, profile, { job, fire, sub: null }, why, true, now, effects);
|
|
779
|
+
endBinding(state, profile.name, ks, 'error', now, effects, why);
|
|
780
|
+
if (budget && resumes < MAX_FIRE_RESUMES) effects.push({ type: 'budget_refused', profile: profile.name, agent: profile.name, reason: budget });
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
if (binding.worker.session_id && binding.worker.session_id !== fire.id) binding.residue.resume_runtime_id ??= binding.worker.session_id;
|
|
784
|
+
fire.residue = { ...(fire.residue ?? {}), resumed_at: now, resumes: resumes + 1 };
|
|
785
|
+
rt.activity[lid] = now;
|
|
786
|
+
rt.starting[lid] = true;
|
|
787
|
+
const text = `[${cut}; this is the same scheduled run, picked up where it stopped] Carry the run on from where it stopped, without redoing what is already done, and end with its final report. The job, as it was given:\n\n${firePrompt(job)}`;
|
|
788
|
+
(rt.queues[lid] ??= []).push({ text, at: now, context_from: job.context_from ?? null, skills: job.skills ?? [], job_id: job.id });
|
|
789
|
+
const worker = { ...profile.worker, model: job.model ?? profile.worker.model, cwd: job.workdir ?? profile.worker.cwd };
|
|
790
|
+
effects.push({ type: 'runtime_start', profile: profile.name, keyString: ks, worker, prompt: text, resume_runtime_id: binding.residue.resume_runtime_id ?? null, agent: profile.name });
|
|
791
|
+
effects.push({ type: 'log', level: 'info', message: `cron ${profile.name}/${job.residue?.name ?? job.id}: run ${fire.id} lost its worker with an earlier daemon; ${binding.residue.resume_runtime_id ? `reopening ${binding.residue.resume_runtime_id}` : 'starting it afresh'}` });
|
|
792
|
+
effects.push({ type: 'save' });
|
|
793
|
+
}
|
|
794
|
+
|
|
745
795
|
/**
|
|
746
796
|
* The fire prompt is the job's prompt; `context_from` (job ids or "self") is
|
|
747
797
|
* carried beside it so the loop, which can read files, prepends those jobs'
|
|
@@ -832,6 +882,9 @@ function launchFire(state, profile, job, fire, prompt, now, effects) {
|
|
|
832
882
|
agent: profile.name, recurrence: { job_id: job.id }, residue: { fire_id: fire.id },
|
|
833
883
|
};
|
|
834
884
|
fire.started_at = now;
|
|
885
|
+
// this daemon holds the run from its launch: silence counts from here, and the reaper never takes it for a run an
|
|
886
|
+
// earlier daemon left behind (`resumeFire`)
|
|
887
|
+
rt.activity[leaseId(profile.name, ks)] = now;
|
|
835
888
|
const worker = { ...profile.worker, model: job.model ?? profile.worker.model, cwd: job.workdir ?? profile.worker.cwd };
|
|
836
889
|
// The prompt rides the queue: `handleWorker`'s `started` case flushes it once
|
|
837
890
|
// the lease exists, the same path an inbound message takes.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export interface DaemonIdentity {
|
|
2
|
+
pid: number;
|
|
3
|
+
started_at: string;
|
|
4
|
+
root: string;
|
|
5
|
+
host: string | null;
|
|
6
|
+
boot?: string | null;
|
|
7
|
+
incarnation: string;
|
|
8
|
+
launch_request?: string | null;
|
|
9
|
+
}
|
|
10
|
+
export interface DaemonStopReceipt {
|
|
11
|
+
version: 1;
|
|
12
|
+
daemon: DaemonIdentity;
|
|
13
|
+
status: "stopped" | "failed" | "refused";
|
|
14
|
+
errors: string[];
|
|
15
|
+
}
|
|
16
|
+
export function newDaemonIncarnation(): string;
|
|
17
|
+
export function sameDaemon(a: unknown, b: unknown): boolean;
|
|
18
|
+
export function clearDaemonLease(root: string, lease: DaemonIdentity): void;
|
|
19
|
+
export function requestDaemonStop(
|
|
20
|
+
root: string,
|
|
21
|
+
lease: DaemonIdentity,
|
|
22
|
+
options?: { signal?: AbortSignal; timeoutMs?: number },
|
|
23
|
+
): Promise<DaemonStopReceipt>;
|
|
24
|
+
export function describeDaemon(
|
|
25
|
+
root: string,
|
|
26
|
+
lease: DaemonIdentity,
|
|
27
|
+
options?: { signal?: AbortSignal; timeoutMs?: number },
|
|
28
|
+
): Promise<{
|
|
29
|
+
version: 1;
|
|
30
|
+
daemon: DaemonIdentity;
|
|
31
|
+
status: "running" | "refused";
|
|
32
|
+
errors: string[];
|
|
33
|
+
}>;
|
|
34
|
+
export function startDaemonControl(options: {
|
|
35
|
+
root: string;
|
|
36
|
+
lease: DaemonIdentity;
|
|
37
|
+
stop: (reason: string) => Promise<DaemonStopReceipt>;
|
|
38
|
+
complete: () => void;
|
|
39
|
+
}): Promise<{ close(): Promise<void> }>;
|