@ianwremmel/dispatch 0.32.1-bootstrap.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +59 -0
- package/.mcp.json +8 -0
- package/LICENSE +21 -0
- package/README.md +93 -0
- package/agents/.gitkeep +0 -0
- package/agents/build-graph.md +99 -0
- package/agents/milestone-reviewer.md +50 -0
- package/agents/pr-worker.md +172 -0
- package/agents/ticket-worker.md +97 -0
- package/bin/dispatch +101 -0
- package/bin/dispatch-mcp +19 -0
- package/bin/pr-status +931 -0
- package/commands/.gitkeep +0 -0
- package/commands/orchestrate.md +6 -0
- package/hooks/.gitkeep +0 -0
- package/hooks/claim-guard.mts +98 -0
- package/hooks/hooks.json +15 -0
- package/package.json +46 -0
- package/skills/.gitkeep +0 -0
- package/skills/land/SKILL.md +238 -0
- package/skills/land/credentials-dedicated.md +33 -0
- package/skills/land/credentials-shared.md +76 -0
- package/skills/land/mode-solo.md +76 -0
- package/skills/land/mode-team.md +103 -0
- package/skills/land/reference.md +152 -0
- package/skills/land/ticket.md +94 -0
- package/skills/orchestrate/SKILL.md +87 -0
- package/skills/tracker-adapter-linear/SKILL.md +142 -0
- package/src/commands/CLAUDE.md +12 -0
- package/src/commands/claim/check.mts +88 -0
- package/src/commands/claim/guard.mts +95 -0
- package/src/commands/claim/status.mts +49 -0
- package/src/commands/edge/add.mts +44 -0
- package/src/commands/edge/rm.mts +44 -0
- package/src/commands/edge/set.mts +56 -0
- package/src/commands/greet.mts +34 -0
- package/src/commands/mcp/ack.mts +43 -0
- package/src/commands/mcp/ping.mts +61 -0
- package/src/commands/mcp/status.mts +89 -0
- package/src/commands/mcp.mts +155 -0
- package/src/commands/milestone/rm.mts +35 -0
- package/src/commands/milestone/set.mts +49 -0
- package/src/commands/outcome/rm.mts +36 -0
- package/src/commands/outcome/set.mts +86 -0
- package/src/commands/pr/rm.mts +33 -0
- package/src/commands/pr/set.mts +110 -0
- package/src/commands/pr/yield.mts +114 -0
- package/src/commands/project/rm.mts +33 -0
- package/src/commands/project/set.mts +50 -0
- package/src/commands/queue.mts +41 -0
- package/src/commands/refresh/done.mts +42 -0
- package/src/commands/refresh/status.mts +40 -0
- package/src/commands/refresh.mts +56 -0
- package/src/commands/review/record.mts +46 -0
- package/src/commands/review/release.mts +49 -0
- package/src/commands/status.mts +85 -0
- package/src/commands/ticket/missing.mts +31 -0
- package/src/commands/ticket/rm.mts +33 -0
- package/src/commands/ticket/set.mts +134 -0
- package/src/commands/worker/rm.mts +46 -0
- package/src/commands/worker/set.mts +63 -0
- package/src/lib/cli/CLAUDE.md +13 -0
- package/src/lib/cli/cli.mts +226 -0
- package/src/lib/cli/index.mts +1 -0
- package/src/lib/command/CLAUDE.md +26 -0
- package/src/lib/command/__fixtures__/bad-export/oops.mts +1 -0
- package/src/lib/command/__fixtures__/bad-name/mismatch.mts +19 -0
- package/src/lib/command/__fixtures__/commands/cli-only.mts +20 -0
- package/src/lib/command/__fixtures__/commands/greet.mts +39 -0
- package/src/lib/command/__fixtures__/commands/math/add.mts +32 -0
- package/src/lib/command/__fixtures__/commands/mcp-only.mts +20 -0
- package/src/lib/command/__fixtures__/commands/needs-token.mts +19 -0
- package/src/lib/command/__fixtures__/commands/store/get.mts +26 -0
- package/src/lib/command/__fixtures__/commands/store.mts +26 -0
- package/src/lib/command/abstract-command.mts +104 -0
- package/src/lib/command/discovery.mts +100 -0
- package/src/lib/command/env.mts +19 -0
- package/src/lib/command/index.mts +6 -0
- package/src/lib/command/parse.mts +64 -0
- package/src/lib/command/test-support.mts +81 -0
- package/src/lib/command/transports.mts +17 -0
- package/src/lib/command/types.mts +53 -0
- package/src/lib/db/CLAUDE.md +13 -0
- package/src/lib/db/database.mts +160 -0
- package/src/lib/db/index.mts +4 -0
- package/src/lib/db/schema.mts +195 -0
- package/src/lib/db/time.mts +24 -0
- package/src/lib/db/with-database.mts +56 -0
- package/src/lib/errors/CLAUDE.md +18 -0
- package/src/lib/errors/command-error.mts +13 -0
- package/src/lib/errors/data-error.mts +12 -0
- package/src/lib/errors/definition-error.mts +6 -0
- package/src/lib/errors/dispatch-error.mts +27 -0
- package/src/lib/errors/ensure.mts +22 -0
- package/src/lib/errors/environment-error.mts +7 -0
- package/src/lib/errors/index.mts +8 -0
- package/src/lib/errors/json-rpc-error.mts +18 -0
- package/src/lib/errors/usage-error.mts +7 -0
- package/src/lib/graph/CLAUDE.md +17 -0
- package/src/lib/graph/anomalies.mts +110 -0
- package/src/lib/graph/derive.mts +96 -0
- package/src/lib/graph/index.mts +26 -0
- package/src/lib/graph/pipeline.mts +410 -0
- package/src/lib/graph/queries.mts +207 -0
- package/src/lib/graph/rows.mts +99 -0
- package/src/lib/graph/types.mts +137 -0
- package/src/lib/liveness/CLAUDE.md +14 -0
- package/src/lib/liveness/index.mts +10 -0
- package/src/lib/liveness/liveness.mts +147 -0
- package/src/lib/liveness/retire.mts +63 -0
- package/src/lib/logger/CLAUDE.md +12 -0
- package/src/lib/logger/index.mts +2 -0
- package/src/lib/logger/logger.mts +58 -0
- package/src/lib/logger/stream-sink.mts +23 -0
- package/src/lib/mcp/CLAUDE.md +21 -0
- package/src/lib/mcp/channel.mts +41 -0
- package/src/lib/mcp/dispatch.mts +60 -0
- package/src/lib/mcp/drain.mts +83 -0
- package/src/lib/mcp/index.mts +5 -0
- package/src/lib/mcp/mcp.mts +267 -0
- package/src/lib/mcp/tools.mts +77 -0
- package/src/lib/model/CLAUDE.md +8 -0
- package/src/lib/model/index.mts +3 -0
- package/src/lib/model/repo-caps.mts +95 -0
- package/src/lib/model/status.mts +91 -0
- package/src/lib/model/types.mts +83 -0
- package/src/lib/refresh/index.mts +2 -0
- package/src/lib/refresh/placeholders.mts +43 -0
- package/src/lib/refresh/refresh-service.mts +203 -0
- package/src/lib/schedule/CLAUDE.md +18 -0
- package/src/lib/schedule/caps.mts +113 -0
- package/src/lib/schedule/correlate.mts +69 -0
- package/src/lib/schedule/index.mts +7 -0
- package/src/lib/schedule/scheduler.mts +355 -0
- package/src/lib/schedule/tick.mts +266 -0
- package/src/lib/stores/CLAUDE.md +24 -0
- package/src/lib/stores/coordination.mts +359 -0
- package/src/lib/stores/cursor.mts +41 -0
- package/src/lib/stores/edge.mts +138 -0
- package/src/lib/stores/fetch-request.mts +346 -0
- package/src/lib/stores/index.mts +19 -0
- package/src/lib/stores/materialize.mts +69 -0
- package/src/lib/stores/milestone.mts +74 -0
- package/src/lib/stores/notice.mts +57 -0
- package/src/lib/stores/policy.mts +48 -0
- package/src/lib/stores/pr-event.mts +94 -0
- package/src/lib/stores/pr.mts +167 -0
- package/src/lib/stores/project.mts +79 -0
- package/src/lib/stores/refresh.mts +197 -0
- package/src/lib/stores/review.mts +113 -0
- package/src/lib/stores/session.mts +170 -0
- package/src/lib/stores/ticket.mts +246 -0
- package/src/lib/stores/watch.mts +360 -0
- package/src/lib/stores/worker.mts +121 -0
- package/src/lib/watch/adopt.mts +151 -0
- package/src/lib/watch/arm.mts +48 -0
- package/src/lib/watch/cadence.mts +45 -0
- package/src/lib/watch/diff.mts +274 -0
- package/src/lib/watch/index.mts +11 -0
- package/src/lib/watch/marker.mts +24 -0
- package/src/lib/watch/payload.mts +56 -0
- package/src/lib/watch/poll.mts +87 -0
- package/src/lib/watch/render.mts +61 -0
- package/src/lib/watch/snapshot.mts +312 -0
- package/src/main.mts +18 -0
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
import type {Database} from '../db/database.mts';
|
|
2
|
+
import {
|
|
3
|
+
DEFAULT_STALE_AFTER_SECONDS,
|
|
4
|
+
dispatchQueue,
|
|
5
|
+
milestoneStates,
|
|
6
|
+
} from '../graph/index.mts';
|
|
7
|
+
import {classifiedItems} from '../graph/index.mts';
|
|
8
|
+
import {derive, repoPrLoad} from '../graph/index.mts';
|
|
9
|
+
import {
|
|
10
|
+
CoordinationStore,
|
|
11
|
+
FetchRequestStore,
|
|
12
|
+
NoticeStore,
|
|
13
|
+
PolicyStore,
|
|
14
|
+
RefreshStore,
|
|
15
|
+
SessionStore,
|
|
16
|
+
} from '../stores/index.mts';
|
|
17
|
+
import type {NoticeKind} from '../stores/notice.mts';
|
|
18
|
+
import {RepoAdmission} from './caps.mts';
|
|
19
|
+
|
|
20
|
+
/** How many agents may be in flight at once when nothing configures it. */
|
|
21
|
+
export const DEFAULT_MAX_PARALLEL = 3;
|
|
22
|
+
|
|
23
|
+
export interface WorkOrder {
|
|
24
|
+
kind:
|
|
25
|
+
| 'dispatch_ticket'
|
|
26
|
+
| 'dispatch_pr'
|
|
27
|
+
| 'perform_milestone_review'
|
|
28
|
+
| 'park_human_blocked'
|
|
29
|
+
| 'alert_failure'
|
|
30
|
+
| 'project_complete';
|
|
31
|
+
meta: Record<string, string>;
|
|
32
|
+
body: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface TickResult {
|
|
36
|
+
orders: WorkOrder[];
|
|
37
|
+
/** The session's own row is gone; the server must exit, not re-register. */
|
|
38
|
+
retired: boolean;
|
|
39
|
+
/** Sources mid-ingest that held this tick's scheduling; empty when none did. */
|
|
40
|
+
ingesting: string[];
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The deterministic half of orchestration: everything the server decides on a
|
|
45
|
+
* tick. Reads the derived read-model, claims before it emits, and returns the
|
|
46
|
+
* work orders to push — it never touches the channel itself, so it tests
|
|
47
|
+
* without one.
|
|
48
|
+
*/
|
|
49
|
+
export class Scheduler {
|
|
50
|
+
readonly #db: Database;
|
|
51
|
+
readonly #session: string;
|
|
52
|
+
readonly #maxParallel: number;
|
|
53
|
+
|
|
54
|
+
constructor(
|
|
55
|
+
db: Database,
|
|
56
|
+
opts: {session: string; maxParallel?: number | undefined}
|
|
57
|
+
) {
|
|
58
|
+
this.#db = db;
|
|
59
|
+
this.#session = opts.session;
|
|
60
|
+
this.#maxParallel = opts.maxParallel ?? DEFAULT_MAX_PARALLEL;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async tick(now: string): Promise<TickResult> {
|
|
64
|
+
const sessions = new SessionStore(this.#db);
|
|
65
|
+
if (!(await sessions.heartbeat(this.#session, now))) {
|
|
66
|
+
return {orders: [], retired: true, ingesting: []};
|
|
67
|
+
}
|
|
68
|
+
await sessions.sweepStale(now, DEFAULT_STALE_AFTER_SECONDS);
|
|
69
|
+
|
|
70
|
+
await this.#refreshTickets(now);
|
|
71
|
+
|
|
72
|
+
const own = await sessions.getSession(this.#session);
|
|
73
|
+
const orders: WorkOrder[] = [];
|
|
74
|
+
|
|
75
|
+
// Nothing at all while a tracker scan is writing the graph. A scan fills
|
|
76
|
+
// the graph ticket by ticket and learns each ticket's dependencies in a
|
|
77
|
+
// second round trip, so under one a real blocker can simply be absent —
|
|
78
|
+
// and an item with no blocker edges is indistinguishable from an item
|
|
79
|
+
// with none, which admits work whose blocker is still open. Reviews and
|
|
80
|
+
// the completion notice read the same half-built graph, so the whole
|
|
81
|
+
// emission waits rather than just the queue. The scan itself is delivered
|
|
82
|
+
// off `fetch_request` rows, not from here, so holding this cannot stall
|
|
83
|
+
// the thing it is waiting for.
|
|
84
|
+
const ingesting = await new RefreshStore(this.#db).ingesting();
|
|
85
|
+
|
|
86
|
+
// No work order before the acknowledgement: a work order claims a node,
|
|
87
|
+
// which a session that never hears the channel would never release.
|
|
88
|
+
if (own?.ackedAt != null && ingesting.length === 0) {
|
|
89
|
+
// One admission budget per tick: the cap minus everything in flight. A
|
|
90
|
+
// claim is both the obligation to run an agent and that agent's compute
|
|
91
|
+
// grant, so it consumes capacity from the moment it exists. Reviews
|
|
92
|
+
// spend first: a review is the continuation of already-landed work and
|
|
93
|
+
// opens a gate other work waits behind.
|
|
94
|
+
const coordination = new CoordinationStore(this.#db);
|
|
95
|
+
const inFlight = await coordination.inFlightCount({
|
|
96
|
+
now,
|
|
97
|
+
staleAfterSeconds: DEFAULT_STALE_AFTER_SECONDS,
|
|
98
|
+
});
|
|
99
|
+
const budget = Math.max(0, this.#maxParallel - inFlight);
|
|
100
|
+
const reviews = await this.#reviews(now, budget);
|
|
101
|
+
orders.push(...reviews);
|
|
102
|
+
// The per-repo caps bound resources the claim ledger cannot see: a PR
|
|
103
|
+
// holds preview stacks and cloud quota for as long as it stays open,
|
|
104
|
+
// and a yielded worker holds no claim at all. Like the budget, they are
|
|
105
|
+
// read before any claim is taken, so two servers scheduling at the same
|
|
106
|
+
// instant can each spend the same free slot; the claims each one writes
|
|
107
|
+
// are what the other's next tick counts.
|
|
108
|
+
const admission = new RepoAdmission(
|
|
109
|
+
await new PolicyStore(this.#db).getRepoCaps(),
|
|
110
|
+
await repoPrLoad(this.#db, {
|
|
111
|
+
now,
|
|
112
|
+
staleAfterSeconds: DEFAULT_STALE_AFTER_SECONDS,
|
|
113
|
+
})
|
|
114
|
+
);
|
|
115
|
+
orders.push(
|
|
116
|
+
...(await this.#fill(now, budget - reviews.length, admission))
|
|
117
|
+
);
|
|
118
|
+
orders.push(...(await this.#conditions(now)));
|
|
119
|
+
}
|
|
120
|
+
return {orders, retired: false, ingesting};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Ask the session to re-read tickets whose tracker state can change under
|
|
125
|
+
* us, on a cadence derived from where each ticket is. The server cannot
|
|
126
|
+
* read the tracker itself, so it owns the *when*: the ask is a durable
|
|
127
|
+
* `refresh_ticket` instruction the drain delivers once the channel is
|
|
128
|
+
* acked, `ticket set` is the answer, and a status change the write reveals
|
|
129
|
+
* is pushed as an event. Backlog and terminal tickets are not asked about —
|
|
130
|
+
* the scan covers them — and one ask per ticket stays open at a time.
|
|
131
|
+
*
|
|
132
|
+
* In-flight tickets move on a person's timescale (a reply, a review), so
|
|
133
|
+
* five minutes; parked ones on an operator's (fifteen).
|
|
134
|
+
*/
|
|
135
|
+
async #refreshTickets(now: string): Promise<void> {
|
|
136
|
+
const requests = new FetchRequestStore(this.#db);
|
|
137
|
+
// An ask pushed ten minutes ago and never answered was lost in flight;
|
|
138
|
+
// clearing its mark lets the drain offer it again.
|
|
139
|
+
await requests.redeliverStaleTicketRefreshes(now, 600);
|
|
140
|
+
const rows = this.#db.all(
|
|
141
|
+
`SELECT n.external_id AS id, t.status, p.source
|
|
142
|
+
FROM ticket t
|
|
143
|
+
JOIN node n ON n.id = t.node_id
|
|
144
|
+
JOIN project p ON p.node_id = t.project_id
|
|
145
|
+
WHERE p.source IS NOT NULL
|
|
146
|
+
AND t.status IN ('in-progress','in-review','finished','delivered',
|
|
147
|
+
'paused','awaiting-external')`
|
|
148
|
+
);
|
|
149
|
+
const nowMs = Date.parse(now);
|
|
150
|
+
for (const row of rows) {
|
|
151
|
+
const status = String(row.status);
|
|
152
|
+
const cadenceMs =
|
|
153
|
+
status === 'paused' || status === 'awaiting-external'
|
|
154
|
+
? 900_000
|
|
155
|
+
: 300_000;
|
|
156
|
+
const ticket = String(row.id);
|
|
157
|
+
const last = await requests.lastTicketRefreshAt(ticket);
|
|
158
|
+
if (last !== null && nowMs - Date.parse(last) < cadenceMs) continue;
|
|
159
|
+
await requests.enqueueTicketRefresh({
|
|
160
|
+
source: String(row.source),
|
|
161
|
+
ticket,
|
|
162
|
+
at: now,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Claim-then-emit for the dispatch queue, up to the remaining budget. The
|
|
169
|
+
* budget bounds this tick's work; the cap passed to `claim` is what actually
|
|
170
|
+
* binds, because a second server scheduling concurrently computed its budget
|
|
171
|
+
* from the same pre-claim reading. A `full` outcome ends the pass — the
|
|
172
|
+
* later entries cannot fit either.
|
|
173
|
+
*
|
|
174
|
+
* A per-repo cap ends nothing: it refuses one entry, and a later entry for
|
|
175
|
+
* another repo may still fit. Only PR items are gated; a ticket-worker keeps
|
|
176
|
+
* planning and registering PR items, which then wait in the queue for a slot
|
|
177
|
+
* — that is the queue doing its job.
|
|
178
|
+
*/
|
|
179
|
+
async #fill(
|
|
180
|
+
now: string,
|
|
181
|
+
budget: number,
|
|
182
|
+
admission: RepoAdmission
|
|
183
|
+
): Promise<WorkOrder[]> {
|
|
184
|
+
const coordination = new CoordinationStore(this.#db);
|
|
185
|
+
const orders: WorkOrder[] = [];
|
|
186
|
+
|
|
187
|
+
for (const {entry, pass} of await dispatchQueue(this.#db, {now})) {
|
|
188
|
+
if (orders.length >= budget) break;
|
|
189
|
+
if (entry.item.kind === 'pr' && admission.admit(entry.item) !== null)
|
|
190
|
+
continue;
|
|
191
|
+
const claim = await coordination.claim({
|
|
192
|
+
node: entry.item.id,
|
|
193
|
+
session: this.#session,
|
|
194
|
+
claimedAt: now,
|
|
195
|
+
capacity: {
|
|
196
|
+
max: this.#maxParallel,
|
|
197
|
+
staleAfterSeconds: DEFAULT_STALE_AFTER_SECONDS,
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
if (claim.outcome === 'full') break;
|
|
201
|
+
if (claim.outcome !== 'claimed' && claim.outcome !== 'refreshed')
|
|
202
|
+
continue;
|
|
203
|
+
const passNote = pass === null ? '' : ` (pass: ${pass})`;
|
|
204
|
+
if (entry.item.kind === 'pr') {
|
|
205
|
+
admission.reserve(entry.item);
|
|
206
|
+
orders.push({
|
|
207
|
+
kind: 'dispatch_pr',
|
|
208
|
+
meta: {
|
|
209
|
+
pr: entry.item.id,
|
|
210
|
+
pass: pass ?? 'available',
|
|
211
|
+
...(entry.item.ticket === null ? {} : {ticket: entry.item.ticket}),
|
|
212
|
+
},
|
|
213
|
+
body: `Launch a pr-worker agent for PR item ${entry.item.id}${entry.item.ticket === null ? '' : ` (implements ticket ${entry.item.ticket})`}${passNote}. It is claimed for this session; the agent records the outcome when done.`,
|
|
214
|
+
});
|
|
215
|
+
} else {
|
|
216
|
+
orders.push({
|
|
217
|
+
kind: 'dispatch_ticket',
|
|
218
|
+
meta: {
|
|
219
|
+
project: entry.item.project ?? '',
|
|
220
|
+
ticket: entry.item.id,
|
|
221
|
+
pass: pass ?? 'available',
|
|
222
|
+
},
|
|
223
|
+
body: `Launch a ticket-worker agent for ticket ${entry.item.id}${passNote}. It is claimed for this session; the agent records the outcome when done.`,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
return orders;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* One review order per open gate, tracked by a milestone-keyed claim and
|
|
232
|
+
* bounded by the shared budget — a review order launches an agent too.
|
|
233
|
+
*/
|
|
234
|
+
async #reviews(now: string, budget: number): Promise<WorkOrder[]> {
|
|
235
|
+
const coordination = new CoordinationStore(this.#db);
|
|
236
|
+
const orders: WorkOrder[] = [];
|
|
237
|
+
for (const milestone of await milestoneStates(this.#db, {now})) {
|
|
238
|
+
if (orders.length >= budget) break;
|
|
239
|
+
if (!milestone.readyForReview || milestone.reviewRecorded) continue;
|
|
240
|
+
if (milestone.claim?.live === true) continue;
|
|
241
|
+
const claim = await coordination.claim({
|
|
242
|
+
node: milestone.id,
|
|
243
|
+
session: this.#session,
|
|
244
|
+
claimedAt: now,
|
|
245
|
+
capacity: {
|
|
246
|
+
max: this.#maxParallel,
|
|
247
|
+
staleAfterSeconds: DEFAULT_STALE_AFTER_SECONDS,
|
|
248
|
+
},
|
|
249
|
+
});
|
|
250
|
+
if (claim.outcome === 'full') break;
|
|
251
|
+
if (claim.outcome !== 'claimed' && claim.outcome !== 'refreshed')
|
|
252
|
+
continue;
|
|
253
|
+
orders.push({
|
|
254
|
+
kind: 'perform_milestone_review',
|
|
255
|
+
meta: {project: milestone.project, milestone: milestone.id},
|
|
256
|
+
body: `Launch a milestone-reviewer agent for milestone ${milestone.id} in project ${milestone.project}. Recording the review opens the gate.`,
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
return orders;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* The tick's non-scheduling duties, fired once per episode: park
|
|
264
|
+
* human-blocked work, surface unrecoverable failures, announce completion.
|
|
265
|
+
*/
|
|
266
|
+
async #conditions(now: string): Promise<WorkOrder[]> {
|
|
267
|
+
const notices = new NoticeStore(this.#db);
|
|
268
|
+
const orders: WorkOrder[] = [];
|
|
269
|
+
const items = await classifiedItems(this.#db, {now});
|
|
270
|
+
|
|
271
|
+
const emit = async (
|
|
272
|
+
kind: NoticeKind,
|
|
273
|
+
holding: {node: string; meta: Record<string, string>; body: string}[]
|
|
274
|
+
): Promise<void> => {
|
|
275
|
+
for (const {node, meta, body} of holding) {
|
|
276
|
+
if (await notices.has(kind, node)) continue;
|
|
277
|
+
await notices.record(kind, node, now);
|
|
278
|
+
orders.push({kind, meta, body});
|
|
279
|
+
}
|
|
280
|
+
await notices.prune(
|
|
281
|
+
kind,
|
|
282
|
+
holding.map((entry) => entry.node)
|
|
283
|
+
);
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
await emit(
|
|
287
|
+
'park_human_blocked',
|
|
288
|
+
items
|
|
289
|
+
.filter(
|
|
290
|
+
(entry) =>
|
|
291
|
+
entry.item.kind === 'ticket' &&
|
|
292
|
+
entry.classification === 'human-blocked' &&
|
|
293
|
+
entry.item.status !== 'paused' &&
|
|
294
|
+
entry.item.status !== 'awaiting-external'
|
|
295
|
+
)
|
|
296
|
+
.map((entry) => ({
|
|
297
|
+
node: entry.item.id,
|
|
298
|
+
meta: {
|
|
299
|
+
project: entry.item.project ?? '',
|
|
300
|
+
ticket: entry.item.id,
|
|
301
|
+
},
|
|
302
|
+
body: `Ticket ${entry.item.id} needs a human. Park it (awaiting-external, or paused if the tracker lacks it) and post the handoff through the tracker adapter.`,
|
|
303
|
+
}))
|
|
304
|
+
);
|
|
305
|
+
|
|
306
|
+
// A human-blocked PR item shares the alert channel with failures: the
|
|
307
|
+
// item has no status to park, so the alert is the only way the operator
|
|
308
|
+
// hears the question its worker left behind. The episode marker keys on
|
|
309
|
+
// the outcome kind too, so replacing a failed outcome with a fresh
|
|
310
|
+
// human-blocked question (or vice versa) starts a new episode.
|
|
311
|
+
await emit(
|
|
312
|
+
'alert_failure',
|
|
313
|
+
items
|
|
314
|
+
.filter(
|
|
315
|
+
(entry) =>
|
|
316
|
+
(entry.outcome?.outcome === 'failed' &&
|
|
317
|
+
entry.outcome.retryable !== true) ||
|
|
318
|
+
(entry.item.kind === 'pr' &&
|
|
319
|
+
entry.outcome?.outcome === 'human-blocked')
|
|
320
|
+
)
|
|
321
|
+
.map((entry) => ({
|
|
322
|
+
node: `${entry.item.id}#${entry.outcome?.outcome ?? ''}`,
|
|
323
|
+
meta:
|
|
324
|
+
entry.item.kind === 'pr'
|
|
325
|
+
? {
|
|
326
|
+
pr: entry.item.id,
|
|
327
|
+
...(entry.item.ticket === null
|
|
328
|
+
? {}
|
|
329
|
+
: {ticket: entry.item.ticket}),
|
|
330
|
+
}
|
|
331
|
+
: {project: entry.item.project ?? '', ticket: entry.item.id},
|
|
332
|
+
body:
|
|
333
|
+
entry.outcome?.outcome === 'human-blocked'
|
|
334
|
+
? `PR item ${entry.item.id} is waiting on an operator response${entry.outcome.detail == null ? '' : ` (${entry.outcome.detail})`}. Alert the operator on the PR if one exists, else on its ticket. An answer on the PR — a comment, a review, a push — requeues it on its own; if the answer arrives anywhere else, requeue it with \`dispatch outcome rm --id ${entry.item.id}\`.`
|
|
335
|
+
: entry.item.kind === 'pr'
|
|
336
|
+
? `PR item ${entry.item.id} failed unrecoverably${entry.outcome?.detail == null ? '' : ` (${entry.outcome.detail})`}. Alert the operator; requeue by removing the outcome once addressed.`
|
|
337
|
+
: `Ticket ${entry.item.id} failed unrecoverably${entry.outcome?.detail == null ? '' : ` (${entry.outcome.detail})`}. Alert the operator on the ticket; requeue by removing the outcome once addressed.`,
|
|
338
|
+
}))
|
|
339
|
+
);
|
|
340
|
+
|
|
341
|
+
const graph = await derive(this.#db, {now});
|
|
342
|
+
await emit(
|
|
343
|
+
'project_complete',
|
|
344
|
+
graph.projects
|
|
345
|
+
.filter((project) => project.terminal)
|
|
346
|
+
.map((project) => ({
|
|
347
|
+
node: project.id,
|
|
348
|
+
meta: {project: project.id},
|
|
349
|
+
body: `Project ${project.id} is complete: every ticket is verified or canceled. Announce it and stop when no other project remains.`,
|
|
350
|
+
}))
|
|
351
|
+
);
|
|
352
|
+
|
|
353
|
+
return orders;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import {nowIso} from '../db/time.mts';
|
|
2
|
+
import {withDatabase} from '../db/index.mts';
|
|
3
|
+
import type {Logger} from '../logger/index.mts';
|
|
4
|
+
import type {ChannelWriter} from '../mcp/channel.mts';
|
|
5
|
+
import {
|
|
6
|
+
CoordinationStore,
|
|
7
|
+
PrEventStore,
|
|
8
|
+
PrStore,
|
|
9
|
+
SessionStore,
|
|
10
|
+
WatchStore,
|
|
11
|
+
WorkerStore,
|
|
12
|
+
} from '../stores/index.mts';
|
|
13
|
+
import {
|
|
14
|
+
adoptOrphans,
|
|
15
|
+
githubSnapshot,
|
|
16
|
+
pollWatches,
|
|
17
|
+
renderSnapshot,
|
|
18
|
+
} from '../watch/index.mts';
|
|
19
|
+
import type {Snapshotter} from '../watch/index.mts';
|
|
20
|
+
import {Scheduler} from './scheduler.mts';
|
|
21
|
+
import type {WorkOrder} from './scheduler.mts';
|
|
22
|
+
|
|
23
|
+
/** First probe retry delay; doubles per unanswered probe up to the cap. */
|
|
24
|
+
const PROBE_DELAY_MS = 5_000;
|
|
25
|
+
const PROBE_DELAY_CAP_MS = 60_000;
|
|
26
|
+
|
|
27
|
+
export interface TickState {
|
|
28
|
+
readonly registryId: string;
|
|
29
|
+
readonly maxParallel?: number | undefined;
|
|
30
|
+
/** Last adoption sweep; a slow cadence, since orphans are rare. */
|
|
31
|
+
adoptDueAtMs: number;
|
|
32
|
+
probeDelayMs: number;
|
|
33
|
+
probeDueAtMs: number;
|
|
34
|
+
/** The session's row was retired; scheduling stops for good. */
|
|
35
|
+
retired: boolean;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function createTickState(
|
|
39
|
+
registryId: string,
|
|
40
|
+
maxParallel?: number
|
|
41
|
+
): TickState {
|
|
42
|
+
return {
|
|
43
|
+
registryId,
|
|
44
|
+
maxParallel,
|
|
45
|
+
adoptDueAtMs: 0,
|
|
46
|
+
probeDelayMs: PROBE_DELAY_MS,
|
|
47
|
+
probeDueAtMs: 0,
|
|
48
|
+
retired: false,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* One server tick: heartbeat and schedule, poll the PR watches, push what
|
|
54
|
+
* they observed, push the resulting work orders, and keep the acknowledgement
|
|
55
|
+
* handshake alive — an unanswered probe re-pushes on a capped backoff rather
|
|
56
|
+
* than latching a verdict. Runs on the timer and after every tool call.
|
|
57
|
+
*
|
|
58
|
+
* Observations are pushed only once the channel is acked, for the same reason
|
|
59
|
+
* work orders are: an event pushed into a channel the runner silently refused
|
|
60
|
+
* would be marked delivered while nobody heard it. Unlike a work order there
|
|
61
|
+
* is no later re-derivation — the next snapshot compares against a state that
|
|
62
|
+
* already contains the change — so until the ack lands the rows wait.
|
|
63
|
+
*/
|
|
64
|
+
export async function runServerTick(
|
|
65
|
+
channel: ChannelWriter,
|
|
66
|
+
env: NodeJS.ProcessEnv,
|
|
67
|
+
state: TickState,
|
|
68
|
+
opts: {nowMs?: number; log?: Logger; snapshot?: Snapshotter} = {}
|
|
69
|
+
): Promise<void> {
|
|
70
|
+
if (state.retired) return;
|
|
71
|
+
const now = nowIso();
|
|
72
|
+
const nowMs = opts.nowMs ?? Date.now();
|
|
73
|
+
|
|
74
|
+
let orders: WorkOrder[] = [];
|
|
75
|
+
let acked = false;
|
|
76
|
+
|
|
77
|
+
const schedule = async (): Promise<void> => {
|
|
78
|
+
await withDatabase(undefined, env, async (db) => {
|
|
79
|
+
const scheduler = new Scheduler(db, {
|
|
80
|
+
session: state.registryId,
|
|
81
|
+
maxParallel: state.maxParallel,
|
|
82
|
+
});
|
|
83
|
+
const result = await scheduler.tick(now);
|
|
84
|
+
if (result.retired) {
|
|
85
|
+
state.retired = true;
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
if (result.ingesting.length > 0) {
|
|
89
|
+
opts.log?.debug('scheduling held while the graph is being built', {
|
|
90
|
+
sources: result.ingesting.join(','),
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
orders = [...orders, ...result.orders];
|
|
94
|
+
|
|
95
|
+
const own = await new SessionStore(db).getSession(state.registryId);
|
|
96
|
+
if (own !== null && own.ackedAt === null && nowMs >= state.probeDueAtMs) {
|
|
97
|
+
channel.push(
|
|
98
|
+
'probe',
|
|
99
|
+
{server: state.registryId},
|
|
100
|
+
`Run \`dispatch mcp ack --server ${state.registryId}\` (the \`mcp_ack\` tool) to acknowledge this channel; work orders wait on it.`
|
|
101
|
+
);
|
|
102
|
+
state.probeDueAtMs = nowMs + state.probeDelayMs;
|
|
103
|
+
state.probeDelayMs = Math.min(
|
|
104
|
+
state.probeDelayMs * 2,
|
|
105
|
+
PROBE_DELAY_CAP_MS
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
acked = own !== null && own.ackedAt !== null;
|
|
109
|
+
});
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
// Heartbeat before the watch poll: snapshotting shells out to gh, and a
|
|
113
|
+
// slow pass must not let this session read as stale mid-tick.
|
|
114
|
+
await schedule();
|
|
115
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- the schedule() closure sets `state.retired`; the analyzer cannot see the write
|
|
116
|
+
if (state.retired) return;
|
|
117
|
+
|
|
118
|
+
try {
|
|
119
|
+
const {fired} = await pollWatches(env, {
|
|
120
|
+
snapshot: opts.snapshot ?? githubSnapshot,
|
|
121
|
+
log: opts.log,
|
|
122
|
+
});
|
|
123
|
+
// A fired watch re-queued its item unless a live worker still holds it,
|
|
124
|
+
// in which case the push below relays instead; a second pass dispatches
|
|
125
|
+
// the queued ones in this tick rather than the next. Claims make the
|
|
126
|
+
// extra pass idempotent.
|
|
127
|
+
if (fired.length > 0) await schedule();
|
|
128
|
+
} catch (error) {
|
|
129
|
+
opts.log?.error('watch pass failed', {
|
|
130
|
+
error: error instanceof Error ? error.message : String(error),
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- the schedule() closure sets `state.retired`; the analyzer cannot see the write
|
|
134
|
+
if (state.retired) return;
|
|
135
|
+
|
|
136
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- the schedule() closure assigns `acked`; the analyzer cannot see the write
|
|
137
|
+
if (acked) {
|
|
138
|
+
if (nowMs >= state.adoptDueAtMs) {
|
|
139
|
+
state.adoptDueAtMs = nowMs + 900_000;
|
|
140
|
+
try {
|
|
141
|
+
const adopted = await adoptOrphans(env, {log: opts.log});
|
|
142
|
+
if (adopted > 0) opts.log?.info('adopted orphaned PRs', {adopted});
|
|
143
|
+
} catch (error) {
|
|
144
|
+
opts.log?.error('adoption sweep failed', {
|
|
145
|
+
error: error instanceof Error ? error.message : String(error),
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
try {
|
|
150
|
+
await pushObservations(channel, env, state.registryId, now, opts.log);
|
|
151
|
+
} catch (error) {
|
|
152
|
+
// A push failure must not cost this tick's already-claimed orders; the
|
|
153
|
+
// rows stay undelivered and the next tick retries.
|
|
154
|
+
opts.log?.error('observation push failed', {
|
|
155
|
+
error: error instanceof Error ? error.message : String(error),
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
for (const order of orders) {
|
|
161
|
+
channel.push(order.kind, order.meta, order.body);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Hand the session every observation the poll owes it. Each names the PR item
|
|
167
|
+
* it belongs to, which is what lets the session route it to the worker
|
|
168
|
+
* holding that PR rather than acting on it itself.
|
|
169
|
+
*/
|
|
170
|
+
export async function pushObservations(
|
|
171
|
+
channel: ChannelWriter,
|
|
172
|
+
env: NodeJS.ProcessEnv,
|
|
173
|
+
session: string,
|
|
174
|
+
at: string,
|
|
175
|
+
log?: Logger
|
|
176
|
+
): Promise<void> {
|
|
177
|
+
await withDatabase(undefined, env, async (db) => {
|
|
178
|
+
const events = new PrEventStore(db);
|
|
179
|
+
const prs = new PrStore(db);
|
|
180
|
+
const watches = new WatchStore(db);
|
|
181
|
+
const workers = new WorkerStore(db);
|
|
182
|
+
const coordination = new CoordinationStore(db);
|
|
183
|
+
// An address only relays while its claim actually goes to this session;
|
|
184
|
+
// otherwise the event falls through to cold re-dispatch, which claims.
|
|
185
|
+
const relayTarget = async (
|
|
186
|
+
ref: string | null,
|
|
187
|
+
node: string
|
|
188
|
+
): Promise<string | null> => {
|
|
189
|
+
if (ref === null) return null;
|
|
190
|
+
const claimed = await coordination.claim({
|
|
191
|
+
node,
|
|
192
|
+
session,
|
|
193
|
+
claimedAt: at,
|
|
194
|
+
});
|
|
195
|
+
return claimed.outcome === 'claimed' || claimed.outcome === 'refreshed'
|
|
196
|
+
? ref
|
|
197
|
+
: null;
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
for (const event of await events.undelivered(session)) {
|
|
201
|
+
try {
|
|
202
|
+
// Every fallible step — the snapshot's JSON parse, the render, the
|
|
203
|
+
// lookups — runs before the event is claimed. A malformed snapshot
|
|
204
|
+
// then leaves the row undelivered to retry next tick instead of being
|
|
205
|
+
// marked delivered and lost, and the per-event catch keeps one bad
|
|
206
|
+
// event from aborting the whole drain.
|
|
207
|
+
const pr = await prs.getPr(event.node);
|
|
208
|
+
// When a live worker holds this node, its address rides the event and
|
|
209
|
+
// the session relays instead of letting the item cold-start.
|
|
210
|
+
//
|
|
211
|
+
// Re-take the claim as part of relaying. The worker gave its own back
|
|
212
|
+
// at `pr yield` so the watch could arm, and its terminal act — the
|
|
213
|
+
// outcome — requires one; a relayed event is the instruction to
|
|
214
|
+
// perform that act, so the address and the authority to use it have to
|
|
215
|
+
// travel together. Without this a terminal event always lands one
|
|
216
|
+
// dispatch short of being recorded, and the item wedges: the worker
|
|
217
|
+
// row suppresses re-dispatch, and the relay is already spent.
|
|
218
|
+
//
|
|
219
|
+
// Unbounded deliberately. The agent is running already, so this is
|
|
220
|
+
// work that was admitted once, not a second admission — and capacity
|
|
221
|
+
// is exactly what the yield handed back.
|
|
222
|
+
const agent = await relayTarget(
|
|
223
|
+
await workers.refFor(event.node, session),
|
|
224
|
+
event.node
|
|
225
|
+
);
|
|
226
|
+
// A ticket event has no PR payload; the session re-reads the ticket
|
|
227
|
+
// through the tracker adapter. A PR event renders from the snapshot
|
|
228
|
+
// the poll already stored — no subprocess, so no per-tick push cap.
|
|
229
|
+
const snapshot =
|
|
230
|
+
event.kind === 'ticket_changed' ||
|
|
231
|
+
pr?.repo == null ||
|
|
232
|
+
pr.prNumber == null
|
|
233
|
+
? null
|
|
234
|
+
: await watches.latestSnapshot(event.node);
|
|
235
|
+
const body =
|
|
236
|
+
snapshot !== null && pr?.repo != null && pr.prNumber != null
|
|
237
|
+
? renderSnapshot(pr.repo, pr.prNumber, snapshot)
|
|
238
|
+
: event.kind === 'ticket_changed'
|
|
239
|
+
? `${event.summary} Re-read the ticket through the tracker adapter before acting.`
|
|
240
|
+
: `${event.summary} No snapshot was stored; run \`pr-status --repo ${pr?.repo ?? '<repo>'} ${String(pr?.prNumber ?? 0)}\` yourself before acting.`;
|
|
241
|
+
const meta = {
|
|
242
|
+
// `agent` is the router's reserved key: stamped after the spread so
|
|
243
|
+
// no event producer can smuggle an address in.
|
|
244
|
+
...event.meta,
|
|
245
|
+
...(agent === null ? {} : {agent}),
|
|
246
|
+
item: event.node,
|
|
247
|
+
...(pr?.repo == null ? {} : {repo: pr.repo}),
|
|
248
|
+
...(pr?.prNumber == null ? {} : {pr: String(pr.prNumber)}),
|
|
249
|
+
};
|
|
250
|
+
|
|
251
|
+
// The claim is the last DB write before the push, and conditional:
|
|
252
|
+
// with session-NULL events drainable by any server, only the one that
|
|
253
|
+
// wins the claim pushes. It has to precede the push, so the single
|
|
254
|
+
// residual is a push that throws after the claim — a failed stdout
|
|
255
|
+
// write, i.e. a dying server, which a retry could not have helped.
|
|
256
|
+
if (!(await events.markDelivered(event.id, at))) continue;
|
|
257
|
+
channel.push(event.kind, meta, body);
|
|
258
|
+
} catch (error) {
|
|
259
|
+
log?.error('event delivery failed', {
|
|
260
|
+
node: event.node,
|
|
261
|
+
error: error instanceof Error ? error.message : String(error),
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
});
|
|
266
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# stores
|
|
2
|
+
|
|
3
|
+
One store per concept over the shared `Database`. A store is organized around a
|
|
4
|
+
concept, not a single table: an operation may write several tables atomically.
|
|
5
|
+
|
|
6
|
+
- `materialize.mts` — shared node create/promote (placeholder → concrete kind),
|
|
7
|
+
with the id-kind conflict check. Used by every write store.
|
|
8
|
+
- `project` / `milestone` / `ticket` / `pr` — per-kind upsert/remove/read.
|
|
9
|
+
- `edge` — the blocking DAG: add/remove/setEdges with cycle rejection.
|
|
10
|
+
- `session` — session lifecycle; claims cascade off it.
|
|
11
|
+
- `coordination` — claims and outcomes (transactionally linked). A claim is
|
|
12
|
+
also the compute grant, so it is the only ledger admissions budget against.
|
|
13
|
+
- `cursor` — delta-sync cursors.
|
|
14
|
+
- `refresh` — the per-tracker ingest state machine, its pending cursor, and the
|
|
15
|
+
completion-emitted marker.
|
|
16
|
+
- `fetch-request` — the durable instruction queue: enqueue, deliver, redeliver,
|
|
17
|
+
resolve, and the open count that decides when a refresh closes.
|
|
18
|
+
- `notice` — once-per-episode markers for the scheduler's condition orders.
|
|
19
|
+
- `policy` — the per-repo admission caps the running server was started with,
|
|
20
|
+
so a reader in another process reports against the caps in force.
|
|
21
|
+
- `review` — milestone reviews with the member snapshot that expires them.
|
|
22
|
+
|
|
23
|
+
The derived read-model (frontier, classification, milestone state, anomalies)
|
|
24
|
+
lives in `../graph`; the scheduler that acts on it lives in `../schedule`.
|