omp-conductor 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -30
- package/package.json +1 -1
- package/src/backups.ts +46 -0
- package/src/board.ts +254 -38
- package/src/brief-upgrade.ts +1 -30
- package/src/briefs/orchestrator.md +13 -2
- package/src/cli.ts +90 -29
- package/src/config.ts +24 -0
- package/src/daemon.ts +311 -131
- package/src/decisions.ts +67 -7
- package/src/diff-flags.ts +35 -6
- package/src/fleet.ts +65 -6
- package/src/label-projection.ts +93 -0
- package/src/lifecycle.ts +8 -1
- package/src/orchestrator-tick.ts +150 -0
- package/src/plugin.ts +1 -1
- package/src/routing.ts +20 -0
- package/src/setup.ts +1 -1
- package/src/store.ts +254 -2
- package/src/tracker/github.ts +489 -80
- package/src/types.ts +85 -3
- package/src/unblock.ts +156 -20
- package/src/upgrade.ts +57 -7
- package/src/worktree.ts +23 -4
package/src/orchestrator-tick.ts
CHANGED
|
@@ -70,6 +70,7 @@ import {
|
|
|
70
70
|
import {
|
|
71
71
|
DEFAULT_REPORT_SCOPE,
|
|
72
72
|
DENIED_RELEASE_GRANTS,
|
|
73
|
+
type DispatchSummary,
|
|
73
74
|
type FrictionSignal,
|
|
74
75
|
type ReportScope,
|
|
75
76
|
type ResolvedGrants,
|
|
@@ -132,6 +133,13 @@ const RETRY_OWNERSHIP_MS = 60_000;
|
|
|
132
133
|
export const STALL_MARKER_FILE = ".conductor-stalled";
|
|
133
134
|
export const STALL_TICKS = 2;
|
|
134
135
|
|
|
136
|
+
/** A turn older than this has its remaining tool calls refused (#189). */
|
|
137
|
+
const DEFAULT_TICK_BUDGET_SECONDS = 600;
|
|
138
|
+
/** Grace before a queued operator message preempts the turn's tool calls. */
|
|
139
|
+
const PENDING_MESSAGE_GRACE_MS = 60_000;
|
|
140
|
+
/** Routable candidates below which the queue digest tells the orchestrator to groom (#181). */
|
|
141
|
+
const DEFAULT_GROOM_BELOW = 4;
|
|
142
|
+
|
|
135
143
|
/**
|
|
136
144
|
* Written by herdr-conductor `recover.sh` *before* `agent start`, so a resumed
|
|
137
145
|
* fleet can reconcile orphans without waiting a full `intervalSeconds`. Cleared
|
|
@@ -244,6 +252,8 @@ interface TickApi {
|
|
|
244
252
|
*/
|
|
245
253
|
getActiveTools(): string[];
|
|
246
254
|
on(event: "session_start", handler: (event: { type: "session_start" }, ctx: TickContext) => void): void;
|
|
255
|
+
on(event: "turn_start", handler: (event: { type: "turn_start" }, ctx: TickContext) => void): void;
|
|
256
|
+
on(event: "turn_end", handler: (event: { type: "turn_end" }, ctx: TickContext) => void): void;
|
|
247
257
|
/**
|
|
248
258
|
* `deliverAs: "followUp"` + `triggerTurn: true`, verified against
|
|
249
259
|
* `AgentSession.sendCustomMessage` rather than assumed:
|
|
@@ -276,6 +286,13 @@ export interface TickConfig {
|
|
|
276
286
|
armedFile?: string;
|
|
277
287
|
accessFile?: string;
|
|
278
288
|
message?: string;
|
|
289
|
+
/**
|
|
290
|
+
* Seconds a turn may run before the tick guard refuses its remaining tool
|
|
291
|
+
* calls (#189). Optional; defaults to {@link DEFAULT_TICK_BUDGET_SECONDS} (10
|
|
292
|
+
* minutes, at or below a normal tick interval so a runaway turn is caught
|
|
293
|
+
* before the next tick).
|
|
294
|
+
*/
|
|
295
|
+
budgetSeconds?: number;
|
|
279
296
|
/**
|
|
280
297
|
* The herdr agent name this fleet's orchestrator pane is registered under, and
|
|
281
298
|
* the whole of {@link resolveTickOwnership}'s identity test under herdr.
|
|
@@ -353,6 +370,32 @@ export function recoveryDigestLine(recovered: readonly RunRecord[]): string | un
|
|
|
353
370
|
return `Auto-recovered since last tick: ${recovered.length} (${named}${rest}) — already handled, do not re-triage these.`;
|
|
354
371
|
}
|
|
355
372
|
|
|
373
|
+
/**
|
|
374
|
+
* One line telling the orchestrator the routable queue is running dry (#181),
|
|
375
|
+
* or `undefined` when healthy — no dispatch recorded yet, or the routable count
|
|
376
|
+
* is at/above the grooming trigger.
|
|
377
|
+
*/
|
|
378
|
+
export function queueDigestLine(
|
|
379
|
+
summary: DispatchSummary | undefined,
|
|
380
|
+
queueLabel: string,
|
|
381
|
+
labelPrefix: string,
|
|
382
|
+
groomBelow: number,
|
|
383
|
+
): string | undefined {
|
|
384
|
+
if (summary === undefined) return undefined;
|
|
385
|
+
if (summary.ready === 0) {
|
|
386
|
+
return `Queue: empty — nothing carries "${queueLabel}". Groom the backlog (Duty 2): promote or file the next issues, or say in this tick's report why there is nothing to do.`;
|
|
387
|
+
}
|
|
388
|
+
if (summary.routed === 0) {
|
|
389
|
+
return `Queue: ${summary.ready} ready but 0 routable — each needs exactly one "${labelPrefix}<repo>" label before dispatch can ever see it.`;
|
|
390
|
+
}
|
|
391
|
+
if (summary.routed >= groomBelow) return undefined;
|
|
392
|
+
let line = `Queue: running low — ${summary.routed} routable candidate(s), below the grooming trigger of ${groomBelow}.`;
|
|
393
|
+
if (summary.admitted === 0 && summary.holds.length > 0) {
|
|
394
|
+
line += ` All held: ${summary.holds.map((h) => `${h.reason} ${h.count}`).join(", ")}.`;
|
|
395
|
+
}
|
|
396
|
+
return line;
|
|
397
|
+
}
|
|
398
|
+
|
|
356
399
|
export const TICK_SCOPE_CONSTRAINTS: { readonly [K in ReportScope]: string } = {
|
|
357
400
|
material: "Report material events per your brief.",
|
|
358
401
|
escalations:
|
|
@@ -614,6 +657,15 @@ export function readTickConfig(cwd: string): TickConfigResult {
|
|
|
614
657
|
intervalSeconds = interval;
|
|
615
658
|
}
|
|
616
659
|
|
|
660
|
+
// Tolerant like the other optional fields: an integer >= the minimum is the
|
|
661
|
+
// turn budget in seconds, anything else degrades to absent (the shipped
|
|
662
|
+
// default) rather than invalidating the config.
|
|
663
|
+
const budgetRaw = raw["budgetSeconds"];
|
|
664
|
+
const budgetSeconds =
|
|
665
|
+
typeof budgetRaw === "number" && Number.isInteger(budgetRaw) && budgetRaw >= MIN_INTERVAL_SECONDS
|
|
666
|
+
? budgetRaw
|
|
667
|
+
: undefined;
|
|
668
|
+
|
|
617
669
|
// Relative paths resolve against the session cwd, so the files can sit beside
|
|
618
670
|
// the config that names them (`state/armed`) without hard-coding a deploy path.
|
|
619
671
|
const armedFile = optionalPath(raw["armedFile"], "armedFile", cwd, problems);
|
|
@@ -646,6 +698,7 @@ export function readTickConfig(cwd: string): TickConfigResult {
|
|
|
646
698
|
path,
|
|
647
699
|
config: {
|
|
648
700
|
intervalSeconds,
|
|
701
|
+
...(budgetSeconds === undefined ? {} : { budgetSeconds }),
|
|
649
702
|
...(armedFile === undefined ? {} : { armedFile }),
|
|
650
703
|
...(accessFile === undefined ? {} : { accessFile }),
|
|
651
704
|
...(message === undefined ? {} : { message }),
|
|
@@ -1012,6 +1065,41 @@ export function tickDecision(input: {
|
|
|
1012
1065
|
return { send: true, reason: "armed, nothing pending" };
|
|
1013
1066
|
}
|
|
1014
1067
|
|
|
1068
|
+
/**
|
|
1069
|
+
* The tick guard's decision, extracted pure from {@link armTickGuard} so it can
|
|
1070
|
+
* be pinned without a fake host: whether a turn's remaining tool calls should be
|
|
1071
|
+
* refused. Budget first, then a queued operator message; otherwise nothing is
|
|
1072
|
+
* blocked (#189).
|
|
1073
|
+
*/
|
|
1074
|
+
export function tickGuardDecision(input: {
|
|
1075
|
+
turnStartedAt: number | undefined;
|
|
1076
|
+
now: number;
|
|
1077
|
+
budgetMs: number;
|
|
1078
|
+
hasPending: boolean;
|
|
1079
|
+
}): { block: true; reason: string } | undefined {
|
|
1080
|
+
if (input.turnStartedAt === undefined) return undefined;
|
|
1081
|
+
const elapsed = input.now - input.turnStartedAt;
|
|
1082
|
+
if (elapsed > input.budgetMs) {
|
|
1083
|
+
return {
|
|
1084
|
+
block: true,
|
|
1085
|
+
reason:
|
|
1086
|
+
`Blocked: this turn has run ${Math.round(elapsed / 1000)}s, past the ${Math.round(input.budgetMs / 1000)}s tick budget (#189). ` +
|
|
1087
|
+
`Finish the turn NOW with a short report. Park anything you were waiting on as a watch — ` +
|
|
1088
|
+
`omp-conductor decision open --question "..." --resolves-when pr-checks-green:<url> | pr-mergeable:<url> | pr-merged:<url> | rate-limit-reset:github — ` +
|
|
1089
|
+
`or hand it to a subagent; the daemon flags met conditions in your next tick digest.`,
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
if (input.hasPending && elapsed > PENDING_MESSAGE_GRACE_MS) {
|
|
1093
|
+
return {
|
|
1094
|
+
block: true,
|
|
1095
|
+
reason:
|
|
1096
|
+
`Blocked: an operator message is queued behind this turn (#189). End the turn now and answer it — the person is waiting. ` +
|
|
1097
|
+
`Park any wait as a decision watch (--resolves-when) instead of polling.`,
|
|
1098
|
+
};
|
|
1099
|
+
}
|
|
1100
|
+
return undefined;
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1015
1103
|
/**
|
|
1016
1104
|
* Whether the Telegram bridge can still reach a person: a bot token, enabled,
|
|
1017
1105
|
* with exactly one paired owner.
|
|
@@ -1269,6 +1357,22 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
1269
1357
|
frictionStore.recoveredSince(scope.projectName, now - 2 * config.intervalSeconds * 1_000),
|
|
1270
1358
|
);
|
|
1271
1359
|
if (recovered !== undefined) content = `${content}\n${recovered}`;
|
|
1360
|
+
// The dispatch pass already persists the numbers that answer "is the
|
|
1361
|
+
// queue running dry", so this reads the store rather than asking the
|
|
1362
|
+
// tracker (#181). A failed config read skips the line rather than
|
|
1363
|
+
// wedging the tick.
|
|
1364
|
+
try {
|
|
1365
|
+
const project = findProject(loadConfig());
|
|
1366
|
+
const queue = queueDigestLine(
|
|
1367
|
+
frictionStore.latestDispatch(scope.projectName),
|
|
1368
|
+
project.queueLabel,
|
|
1369
|
+
project.routing.labelPrefix,
|
|
1370
|
+
project.groomBelow ?? DEFAULT_GROOM_BELOW,
|
|
1371
|
+
);
|
|
1372
|
+
if (queue !== undefined) content = `${content}\n${queue}`;
|
|
1373
|
+
} catch {
|
|
1374
|
+
// unreadable config: no queue digest this tick
|
|
1375
|
+
}
|
|
1272
1376
|
} catch (err) {
|
|
1273
1377
|
frictionStore?.close();
|
|
1274
1378
|
frictionStore = undefined;
|
|
@@ -1409,6 +1513,43 @@ function armTickHeartbeat(pi: TickApi, ctx: TickContext, config: TickConfig, ses
|
|
|
1409
1513
|
tick(pi, ctx, config, session);
|
|
1410
1514
|
}
|
|
1411
1515
|
|
|
1516
|
+
/**
|
|
1517
|
+
* The mechanical guard against a turn that never ends (#189).
|
|
1518
|
+
*
|
|
1519
|
+
* The harness delivers `turn_start`/`turn_end`, so the guard times a turn from
|
|
1520
|
+
* the first to the last tool-eligible phase and refuses further tool calls once
|
|
1521
|
+
* it exceeds `budgetMs` — or once an operator message has been queued behind it
|
|
1522
|
+
* for longer than {@link PENDING_MESSAGE_GRACE_MS}. Refusing every tool forces
|
|
1523
|
+
* the model to end the turn with text; nothing else unblocks the queue. `yield`
|
|
1524
|
+
* stays callable so a session that is also a task can still return.
|
|
1525
|
+
*/
|
|
1526
|
+
function armTickGuard(pi: TickApi, ctx: TickContext, budgetMs: number): void {
|
|
1527
|
+
let turnStartedAt: number | undefined;
|
|
1528
|
+
pi.on("turn_start", () => {
|
|
1529
|
+
turnStartedAt = Date.now();
|
|
1530
|
+
});
|
|
1531
|
+
pi.on("turn_end", () => {
|
|
1532
|
+
turnStartedAt = undefined;
|
|
1533
|
+
});
|
|
1534
|
+
(pi as TickApi & {
|
|
1535
|
+
on(
|
|
1536
|
+
event: "tool_call",
|
|
1537
|
+
handler: (
|
|
1538
|
+
event: { toolName: string; input: Record<string, unknown> },
|
|
1539
|
+
ctx: unknown,
|
|
1540
|
+
) => { block: true; reason: string } | undefined,
|
|
1541
|
+
): void;
|
|
1542
|
+
}).on("tool_call", (event) => {
|
|
1543
|
+
if (event.toolName === "yield") return undefined;
|
|
1544
|
+
return tickGuardDecision({
|
|
1545
|
+
turnStartedAt,
|
|
1546
|
+
now: Date.now(),
|
|
1547
|
+
budgetMs,
|
|
1548
|
+
hasPending: ctx.hasPendingMessages(),
|
|
1549
|
+
});
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1412
1553
|
export default function orchestratorTickExtension(pi: TickApi): void {
|
|
1413
1554
|
// Scoped to this registration rather than the module, so a second
|
|
1414
1555
|
// `session_start` can neither install a second heartbeat on the same session
|
|
@@ -1426,6 +1567,7 @@ export default function orchestratorTickExtension(pi: TickApi): void {
|
|
|
1426
1567
|
pendingSkips: 0,
|
|
1427
1568
|
};
|
|
1428
1569
|
let releaseGateArmed = false;
|
|
1570
|
+
let guardArmed = false;
|
|
1429
1571
|
// An activation file makes this a fleet directory before Herdr can prove
|
|
1430
1572
|
// which pane owns it. The gate therefore starts closed and only honours a
|
|
1431
1573
|
// configured grant after ownership is accepted.
|
|
@@ -1594,6 +1736,10 @@ export default function orchestratorTickExtension(pi: TickApi): void {
|
|
|
1594
1736
|
if (next.note !== undefined) pi.logger.error(`[omp-conductor] tick ownership: ${next.note}`);
|
|
1595
1737
|
releaseAuthorityAccepted = true;
|
|
1596
1738
|
armTickHeartbeat(pi, ctx, config, session);
|
|
1739
|
+
if (!guardArmed) {
|
|
1740
|
+
guardArmed = true;
|
|
1741
|
+
armTickGuard(pi, ctx, (config.budgetSeconds ?? DEFAULT_TICK_BUDGET_SECONDS) * 1000);
|
|
1742
|
+
}
|
|
1597
1743
|
pi.logger.info(`[omp-conductor] orchestrator tick active: ownership resolved on retry`, { agentName });
|
|
1598
1744
|
}, retryMs);
|
|
1599
1745
|
decided = true;
|
|
@@ -1608,6 +1754,10 @@ export default function orchestratorTickExtension(pi: TickApi): void {
|
|
|
1608
1754
|
if (ownership.note !== undefined) pi.logger.error(`[omp-conductor] tick ownership: ${ownership.note}`);
|
|
1609
1755
|
releaseAuthorityAccepted = true;
|
|
1610
1756
|
armTickHeartbeat(pi, ctx, config, session);
|
|
1757
|
+
if (!guardArmed) {
|
|
1758
|
+
guardArmed = true;
|
|
1759
|
+
armTickGuard(pi, ctx, (config.budgetSeconds ?? DEFAULT_TICK_BUDGET_SECONDS) * 1000);
|
|
1760
|
+
}
|
|
1611
1761
|
decided = true;
|
|
1612
1762
|
// Both gates are named at startup: "why is it not ticking?" is answered by
|
|
1613
1763
|
// looking at the files this line lists, and an unset channel gate on a fleet
|
package/src/plugin.ts
CHANGED
|
@@ -1358,7 +1358,7 @@ export default function conductorPlugin(pi: PluginApi): void {
|
|
|
1358
1358
|
}
|
|
1359
1359
|
|
|
1360
1360
|
case "pause":
|
|
1361
|
-
setPaused(true);
|
|
1361
|
+
setPaused(true, { source: "pause", reason: "via /conductor pause" });
|
|
1362
1362
|
ctx.ui.notify("Paused claiming only — ticks keep firing if armed. Prefer /conductor hold.", "info");
|
|
1363
1363
|
break;
|
|
1364
1364
|
|
package/src/routing.ts
CHANGED
|
@@ -55,6 +55,26 @@ export function isEligible(issue: ReadyIssue, p: ProjectConfig): boolean {
|
|
|
55
55
|
return !labels.has(inProgress) && !labels.has(blocked) && !labels.has(failed);
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
/**
|
|
59
|
+
* The label set an issue *effectively* carries once the projection outbox is
|
|
60
|
+
* accounted for (#201). Pending ops apply in enqueue order — the same order
|
|
61
|
+
* the projector will apply them — so an issue with a pending queue-label
|
|
62
|
+
* removal drops out of eligibility immediately, and a pending state-label
|
|
63
|
+
* removal stops a stale GitHub label from blocking redispatch while the
|
|
64
|
+
* tracker is still catching up. Pure: the physical label set is untouched.
|
|
65
|
+
*/
|
|
66
|
+
export function effectiveLabels(
|
|
67
|
+
labels: readonly string[],
|
|
68
|
+
pending: readonly { op: "add" | "remove"; label: string }[],
|
|
69
|
+
): string[] {
|
|
70
|
+
const set = new Set(labels);
|
|
71
|
+
for (const op of pending) {
|
|
72
|
+
if (op.op === "add") set.add(op.label);
|
|
73
|
+
else set.delete(op.label);
|
|
74
|
+
}
|
|
75
|
+
return [...set];
|
|
76
|
+
}
|
|
77
|
+
|
|
58
78
|
/**
|
|
59
79
|
* Partition eligible issues into dispatchable and needs-a-human.
|
|
60
80
|
*
|
package/src/setup.ts
CHANGED
|
@@ -1167,7 +1167,7 @@ export const AMEND_AREAS: {
|
|
|
1167
1167
|
const spend =
|
|
1168
1168
|
c.dailySpendUsd === null ? "no spend cap" : `$${c.dailySpendUsd}/day`;
|
|
1169
1169
|
return (
|
|
1170
|
-
`${c.maxConcurrentWorkers} workers, ${c.workerMaxTurns} turns, ` +
|
|
1170
|
+
`${c.maxConcurrentWorkers} workers (${c.maxConcurrentWorkersPerRepo}/repo), ${c.workerMaxTurns} turns, ` +
|
|
1171
1171
|
`${Math.round(c.workerWallClockMs / 60000)}m, ${spend}, ` +
|
|
1172
1172
|
`${c.maxAttemptsPerIssue} failed attempt${c.maxAttemptsPerIssue === 1 ? "" : "s"}, ` +
|
|
1173
1173
|
`${c.maxContinuationsPerIssue} continuation${c.maxContinuationsPerIssue === 1 ? "" : "s"}` +
|
package/src/store.ts
CHANGED
|
@@ -26,6 +26,7 @@ import type {
|
|
|
26
26
|
FrictionKind,
|
|
27
27
|
FrictionObservation,
|
|
28
28
|
FrictionSignal,
|
|
29
|
+
LabelOp,
|
|
29
30
|
MergeLock,
|
|
30
31
|
ReportDeliveryState,
|
|
31
32
|
ReportDraft,
|
|
@@ -111,6 +112,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
|
|
|
111
112
|
endedAt: true,
|
|
112
113
|
lastError: true,
|
|
113
114
|
settlementFlags: true,
|
|
115
|
+
report: true,
|
|
114
116
|
failureClass: true,
|
|
115
117
|
recoveryAction: true,
|
|
116
118
|
recoveredAt: true,
|
|
@@ -142,6 +144,7 @@ interface RunRow {
|
|
|
142
144
|
endedAt: number | null;
|
|
143
145
|
lastError: string | null;
|
|
144
146
|
settlementFlags: string | null;
|
|
147
|
+
report: string | null;
|
|
145
148
|
failureClass: string | null;
|
|
146
149
|
recoveryAction: string | null;
|
|
147
150
|
recoveredAt: number | null;
|
|
@@ -163,6 +166,35 @@ interface FrictionSurfaceRow {
|
|
|
163
166
|
at: number;
|
|
164
167
|
}
|
|
165
168
|
|
|
169
|
+
/** The `label_ops` table exactly as SQLite hands it back (#201). */
|
|
170
|
+
interface LabelOpRow {
|
|
171
|
+
id: number;
|
|
172
|
+
project: string;
|
|
173
|
+
issue: number;
|
|
174
|
+
op: "add" | "remove";
|
|
175
|
+
label: string;
|
|
176
|
+
createdAt: number;
|
|
177
|
+
attempts: number;
|
|
178
|
+
nextAttemptAt: number;
|
|
179
|
+
lastError: string | null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** NULL columns become absent properties, matching the other row converters. */
|
|
183
|
+
function toLabelOp(row: LabelOpRow): LabelOp {
|
|
184
|
+
const op: LabelOp = {
|
|
185
|
+
id: row.id,
|
|
186
|
+
project: row.project,
|
|
187
|
+
issue: row.issue,
|
|
188
|
+
op: row.op,
|
|
189
|
+
label: row.label,
|
|
190
|
+
createdAt: row.createdAt,
|
|
191
|
+
attempts: row.attempts,
|
|
192
|
+
nextAttemptAt: row.nextAttemptAt,
|
|
193
|
+
};
|
|
194
|
+
if (row.lastError !== null) op.lastError = row.lastError;
|
|
195
|
+
return op;
|
|
196
|
+
}
|
|
197
|
+
|
|
166
198
|
/** The `reports` table exactly as SQLite hands it back. */
|
|
167
199
|
interface ReportRow {
|
|
168
200
|
id: string;
|
|
@@ -229,6 +261,7 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
229
261
|
endedAt INTEGER,
|
|
230
262
|
lastError TEXT,
|
|
231
263
|
settlementFlags TEXT,
|
|
264
|
+
report TEXT,
|
|
232
265
|
failureClass TEXT,
|
|
233
266
|
recoveryAction TEXT,
|
|
234
267
|
recoveredAt INTEGER
|
|
@@ -360,6 +393,46 @@ CREATE TABLE IF NOT EXISTS decisions (
|
|
|
360
393
|
resolution TEXT
|
|
361
394
|
);
|
|
362
395
|
CREATE INDEX IF NOT EXISTS decisions_project_state ON decisions (project, state);
|
|
396
|
+
|
|
397
|
+
-- GitHub rate-limit refusals the tracker observed (#198). Written by the
|
|
398
|
+
-- daemon's tracker hook, not polled, so status can show what GitHub actually
|
|
399
|
+
-- refused next to the polled budget that might still look healthy. Only one
|
|
400
|
+
-- column: a refusal carries no id or dedupe key — rows are pruned by age
|
|
401
|
+
-- (24h) in the same write that inserts, kept long enough for status's 5m
|
|
402
|
+
-- window and for an operator who wants to see the last day of refusals.
|
|
403
|
+
CREATE TABLE IF NOT EXISTS gh_refusals (
|
|
404
|
+
at INTEGER NOT NULL
|
|
405
|
+
);
|
|
406
|
+
CREATE INDEX IF NOT EXISTS gh_refusals_at ON gh_refusals (at);
|
|
407
|
+
|
|
408
|
+
-- The daemon's tracked gh call count, per UTC day and call source (#198).
|
|
409
|
+
-- The single funnel in tracker/github.ts counts every spawn; the day/source
|
|
410
|
+
-- pair is the partition, so today's column and today's row survive a restart.
|
|
411
|
+
CREATE TABLE IF NOT EXISTS gh_calls (
|
|
412
|
+
day TEXT NOT NULL,
|
|
413
|
+
source TEXT NOT NULL,
|
|
414
|
+
calls INTEGER NOT NULL,
|
|
415
|
+
PRIMARY KEY (day, source)
|
|
416
|
+
);
|
|
417
|
+
-- The label projection outbox (#201). Every decided GitHub label change the
|
|
418
|
+
-- dispatcher made is a row here until the tracker has applied it; the daemon
|
|
419
|
+
-- retries with backoff, and eligibility reads pendingLabelOpsFor so a label
|
|
420
|
+
-- GitHub refuses to move cannot strand the issue the daemon already decided
|
|
421
|
+
-- to move on. The queue-label add and the state-label remove of one swap are
|
|
422
|
+
-- two rows, applied strictly in id order, which is what makes a swap atomic:
|
|
423
|
+
-- the remove cannot land before the add it was enqueued after.
|
|
424
|
+
CREATE TABLE IF NOT EXISTS label_ops (
|
|
425
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
426
|
+
project TEXT NOT NULL,
|
|
427
|
+
issue INTEGER NOT NULL,
|
|
428
|
+
op TEXT NOT NULL CHECK (op IN ('add', 'remove')),
|
|
429
|
+
label TEXT NOT NULL,
|
|
430
|
+
createdAt INTEGER NOT NULL,
|
|
431
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
432
|
+
nextAttemptAt INTEGER NOT NULL DEFAULT 0,
|
|
433
|
+
lastError TEXT
|
|
434
|
+
);
|
|
435
|
+
CREATE INDEX IF NOT EXISTS label_ops_project_next ON label_ops (project, nextAttemptAt);
|
|
363
436
|
`;
|
|
364
437
|
|
|
365
438
|
/**
|
|
@@ -435,6 +508,7 @@ function toRecord(row: RunRow): RunRecord {
|
|
|
435
508
|
const flags = toSettlementFlags(row.settlementFlags);
|
|
436
509
|
if (flags !== undefined) record.settlementFlags = flags;
|
|
437
510
|
}
|
|
511
|
+
if (row.report !== null) record.report = row.report;
|
|
438
512
|
if (row.failureClass !== null) record.failureClass = row.failureClass as FailureClass;
|
|
439
513
|
if (row.recoveryAction !== null) record.recoveryAction = row.recoveryAction as RecoveryAction;
|
|
440
514
|
if (row.recoveredAt !== null) record.recoveredAt = row.recoveredAt;
|
|
@@ -613,6 +687,12 @@ export function dbPath(): string {
|
|
|
613
687
|
return join(stateDir(), "conductor.db");
|
|
614
688
|
}
|
|
615
689
|
|
|
690
|
+
/** The UTC calendar day (`YYYY-MM-DD`) a moment falls on — the partition key
|
|
691
|
+
* for the daemon's observed github call counter (#198). */
|
|
692
|
+
export function utcDay(now: number = Date.now()): string {
|
|
693
|
+
return new Date(now).toISOString().slice(0, 10);
|
|
694
|
+
}
|
|
695
|
+
|
|
616
696
|
/**
|
|
617
697
|
* Open (creating if needed) the run store at `dbPath`; `:memory:` is honoured
|
|
618
698
|
* for tests. Safe to call on a fresh path — the schema is applied on open, so
|
|
@@ -665,6 +745,13 @@ export function openStore(dbPath: string): Store {
|
|
|
665
745
|
if (!columns.some((column) => column.name === "settlementFlags")) {
|
|
666
746
|
db.exec("ALTER TABLE runs ADD COLUMN settlementFlags TEXT");
|
|
667
747
|
}
|
|
748
|
+
// Rows written before the settlement report was persisted (#199) have no
|
|
749
|
+
// text to pool across attempts, so a continuation reads empty priors and
|
|
750
|
+
// audits exactly as it did before this release. No backfill: the bytes only
|
|
751
|
+
// ever existed in a worker's memory.
|
|
752
|
+
if (!columns.some((column) => column.name === "report")) {
|
|
753
|
+
db.exec("ALTER TABLE runs ADD COLUMN report TEXT");
|
|
754
|
+
}
|
|
668
755
|
// Every row written before #132 is unclassified, and NULL is the honest
|
|
669
756
|
// reading of that: the budget counters below deliberately still count an
|
|
670
757
|
// unclassified terminal row exactly as this release's predecessor did, so an
|
|
@@ -772,8 +859,8 @@ export function openStore(dbPath: string): Store {
|
|
|
772
859
|
`INSERT INTO runs (
|
|
773
860
|
id, project, issue, repo, branch, worktree, state, attempt, turns,
|
|
774
861
|
maxTurns, spendUsd, sessionFile, prUrl, headSha, salvageSha, salvageError,
|
|
775
|
-
salvageAckAt, startedAt, endedAt, lastError, settlementFlags
|
|
776
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
862
|
+
salvageAckAt, startedAt, endedAt, lastError, settlementFlags, report
|
|
863
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
777
864
|
);
|
|
778
865
|
const selectRun = db.query<RunRow, [string]>(`SELECT * FROM runs WHERE id = ?`);
|
|
779
866
|
const selectActive = db.query<RunRow, SqlValue[]>(
|
|
@@ -886,6 +973,14 @@ export function openStore(dbPath: string): Store {
|
|
|
886
973
|
ORDER BY startedAt DESC, rowid DESC
|
|
887
974
|
LIMIT 1`,
|
|
888
975
|
);
|
|
976
|
+
// Every attempt that settled with a report, in attempt order. NULLs are the
|
|
977
|
+
// rows written before #199, or one killed before the worker returned a
|
|
978
|
+
// report — skipping them makes a continuation read exactly today's priors.
|
|
979
|
+
const selectAttemptReports = db.query<{ attempt: number; report: string }, [string, number]>(
|
|
980
|
+
`SELECT attempt, report FROM runs
|
|
981
|
+
WHERE project = ? AND issue = ? AND report IS NOT NULL
|
|
982
|
+
ORDER BY attempt`,
|
|
983
|
+
);
|
|
889
984
|
const countStartedSince = db.query<{ n: number }, [string, number]>(
|
|
890
985
|
`SELECT COUNT(*) AS n FROM runs WHERE project = ? AND startedAt >= ?`,
|
|
891
986
|
);
|
|
@@ -938,6 +1033,31 @@ export function openStore(dbPath: string): Store {
|
|
|
938
1033
|
ON CONFLICT(project, kind) DO UPDATE SET at = excluded.at`,
|
|
939
1034
|
);
|
|
940
1035
|
|
|
1036
|
+
// Observed GitHub rate-limit refusals (#198). Insert + prune share one
|
|
1037
|
+
// transaction so a burst never grows the table without bound.
|
|
1038
|
+
const insertGhRefusal = db.query<unknown, [number]>(
|
|
1039
|
+
`INSERT INTO gh_refusals (at) VALUES (?)`,
|
|
1040
|
+
);
|
|
1041
|
+
const pruneGhRefusals = db.query<unknown, [number]>(
|
|
1042
|
+
`DELETE FROM gh_refusals WHERE at < ?`,
|
|
1043
|
+
);
|
|
1044
|
+
const selectGhRefusalsSince = db.query<{ n: number; latest: number | null }, [number]>(
|
|
1045
|
+
`SELECT COUNT(*) AS n, MAX(at) AS latest FROM gh_refusals WHERE at >= ?`,
|
|
1046
|
+
);
|
|
1047
|
+
const recordGhRefusalTx = db.transaction((at: number): void => {
|
|
1048
|
+
insertGhRefusal.run(at);
|
|
1049
|
+
pruneGhRefusals.run(at - 24 * 60 * 60 * 1000);
|
|
1050
|
+
});
|
|
1051
|
+
|
|
1052
|
+
// The daemon's tracked github call counts (#198): one UPSERT per spawn.
|
|
1053
|
+
const bumpGhCall = db.query<unknown, [string, string]>(
|
|
1054
|
+
`INSERT INTO gh_calls (day, source, calls) VALUES (?, ?, 1)
|
|
1055
|
+
ON CONFLICT(day, source) DO UPDATE SET calls = calls + 1`,
|
|
1056
|
+
);
|
|
1057
|
+
const selectGhCalls = db.query<{ source: string; calls: number }, [string]>(
|
|
1058
|
+
`SELECT source, calls FROM gh_calls WHERE day = ? ORDER BY source`,
|
|
1059
|
+
);
|
|
1060
|
+
|
|
941
1061
|
// The report outbox (#123). Every terminal transition names the attempt it is
|
|
942
1062
|
// settling, because a request can come back *after* the stale-`sending` sweep
|
|
943
1063
|
// has already reclaimed its row — and a late answer must not overwrite a
|
|
@@ -1071,6 +1191,72 @@ export function openStore(dbPath: string): Store {
|
|
|
1071
1191
|
},
|
|
1072
1192
|
);
|
|
1073
1193
|
|
|
1194
|
+
// The label projection outbox (#201). A decided label change is a row until
|
|
1195
|
+
// the tracker has applied it; the projector drains in id order, and one
|
|
1196
|
+
// issue's ops are atomic as a run — a failed op parks the rest of its issue
|
|
1197
|
+
// for the next pass rather than letting a later op land out of order.
|
|
1198
|
+
const insertLabelOp = db.query<unknown, SqlValue[]>(
|
|
1199
|
+
`INSERT INTO label_ops (project, issue, op, label, createdAt) VALUES (?, ?, ?, ?, ?)`,
|
|
1200
|
+
);
|
|
1201
|
+
const selectPendingLabelOps = db.query<LabelOpRow, [string, number]>(
|
|
1202
|
+
`SELECT o.* FROM label_ops AS o
|
|
1203
|
+
WHERE o.project = ? AND o.nextAttemptAt <= ?
|
|
1204
|
+
AND NOT EXISTS (
|
|
1205
|
+
SELECT 1 FROM label_ops AS e
|
|
1206
|
+
WHERE e.project = o.project AND e.issue = o.issue AND e.id < o.id
|
|
1207
|
+
)
|
|
1208
|
+
ORDER BY o.id`,
|
|
1209
|
+
);
|
|
1210
|
+
const selectPendingLabelOpsFor = db.query<LabelOpRow, [string, number]>(
|
|
1211
|
+
`SELECT * FROM label_ops WHERE project = ? AND issue = ? ORDER BY id`,
|
|
1212
|
+
);
|
|
1213
|
+
const selectOldestPendingLabelOp = db.query<LabelOpRow, [string]>(
|
|
1214
|
+
`SELECT * FROM label_ops WHERE project = ? ORDER BY createdAt ASC, id ASC LIMIT 1`,
|
|
1215
|
+
);
|
|
1216
|
+
const countLabelOps = db.query<{ n: number }, [string]>(
|
|
1217
|
+
`SELECT COUNT(*) AS n FROM label_ops WHERE project = ?`,
|
|
1218
|
+
);
|
|
1219
|
+
const deleteLabelOp = db.query<unknown, [number]>(`DELETE FROM label_ops WHERE id = ?`);
|
|
1220
|
+
const bumpLabelOpAttempt = db.query<unknown, [string, number, number]>(
|
|
1221
|
+
`UPDATE label_ops SET attempts = attempts + 1, lastError = ?, nextAttemptAt = ? WHERE id = ?`,
|
|
1222
|
+
);
|
|
1223
|
+
// Rate-limit refusals defer WITHOUT counting an attempt: a shared outage is
|
|
1224
|
+
// not the op's fault, so it must not burn the backoff escalation (#208).
|
|
1225
|
+
const parkLabelOp = db.query<unknown, [string, number, number]>(
|
|
1226
|
+
`UPDATE label_ops SET lastError = ?, nextAttemptAt = ? WHERE id = ?`,
|
|
1227
|
+
);
|
|
1228
|
+
// Coalesce: if the LATEST pending op for the same project+issue+label is
|
|
1229
|
+
// already the same operation, adding another is a duplicate that would sit
|
|
1230
|
+
// parked behind the oldest pending one, growing unboundedly (a reconcile
|
|
1231
|
+
// re-offering a removal the tracker keeps refusing). Only the latest matters:
|
|
1232
|
+
// a `remove → add → remove` sequence must keep the final remove even though
|
|
1233
|
+
// the first remove is still owed, because the add in between is a real
|
|
1234
|
+
// transition the final remove answers. Opposite transitions are never
|
|
1235
|
+
// coalesced. The check and insert share the transaction so they cannot race
|
|
1236
|
+
// (#201).
|
|
1237
|
+
const selectLatestLabelOp = db.query<{ op: "add" | "remove" }, [string, number, string]>(
|
|
1238
|
+
`SELECT op FROM label_ops
|
|
1239
|
+
WHERE project = ? AND issue = ? AND label = ?
|
|
1240
|
+
ORDER BY id DESC LIMIT 1`,
|
|
1241
|
+
);
|
|
1242
|
+
const enqueueLabelOpsTx = db.transaction(
|
|
1243
|
+
(
|
|
1244
|
+
rows: readonly {
|
|
1245
|
+
project: string;
|
|
1246
|
+
issue: number;
|
|
1247
|
+
op: "add" | "remove";
|
|
1248
|
+
label: string;
|
|
1249
|
+
createdAt: number;
|
|
1250
|
+
}[],
|
|
1251
|
+
): void => {
|
|
1252
|
+
for (const row of rows) {
|
|
1253
|
+
const latest = selectLatestLabelOp.get(row.project, row.issue, row.label);
|
|
1254
|
+
if (latest !== undefined && latest !== null && latest.op === row.op) continue;
|
|
1255
|
+
insertLabelOp.run(toSql(row.project), row.issue, row.op, row.label, row.createdAt);
|
|
1256
|
+
}
|
|
1257
|
+
},
|
|
1258
|
+
);
|
|
1259
|
+
|
|
1074
1260
|
const recordFriction = (project: string, observation: FrictionObservation): void => {
|
|
1075
1261
|
if (
|
|
1076
1262
|
!Number.isSafeInteger(observation.occurrences) ||
|
|
@@ -1108,6 +1294,12 @@ export function openStore(dbPath: string): Store {
|
|
|
1108
1294
|
);
|
|
1109
1295
|
};
|
|
1110
1296
|
|
|
1297
|
+
const recordGhRefusal = (at: number): void => {
|
|
1298
|
+
// Same guard as `recordFriction`: a bad clock must not corrupt the store.
|
|
1299
|
+
if (!Number.isSafeInteger(at) || at < 0) return;
|
|
1300
|
+
recordGhRefusalTx(at);
|
|
1301
|
+
};
|
|
1302
|
+
|
|
1111
1303
|
return {
|
|
1112
1304
|
createRun(r: Omit<RunRecord, "id">): RunRecord {
|
|
1113
1305
|
const record: RunRecord = { ...r, id: crypto.randomUUID() };
|
|
@@ -1133,6 +1325,7 @@ export function openStore(dbPath: string): Store {
|
|
|
1133
1325
|
toSql(record.endedAt),
|
|
1134
1326
|
toSql(record.lastError),
|
|
1135
1327
|
toSql(record.settlementFlags),
|
|
1328
|
+
toSql(record.report),
|
|
1136
1329
|
);
|
|
1137
1330
|
return record;
|
|
1138
1331
|
},
|
|
@@ -1200,6 +1393,10 @@ export function openStore(dbPath: string): Store {
|
|
|
1200
1393
|
return row ? toRecord(row) : undefined;
|
|
1201
1394
|
},
|
|
1202
1395
|
|
|
1396
|
+
attemptReports(project: string, issue: number): { attempt: number; report: string }[] {
|
|
1397
|
+
return selectAttemptReports.all(project, issue);
|
|
1398
|
+
},
|
|
1399
|
+
|
|
1203
1400
|
|
|
1204
1401
|
runsStartedSince(project: string, sinceEpochMs: number): number {
|
|
1205
1402
|
return countStartedSince.get(project, sinceEpochMs)?.n ?? 0;
|
|
@@ -1250,6 +1447,24 @@ export function openStore(dbPath: string): Store {
|
|
|
1250
1447
|
|
|
1251
1448
|
recordFriction,
|
|
1252
1449
|
|
|
1450
|
+
recordGhRefusal,
|
|
1451
|
+
|
|
1452
|
+
ghRefusalsSince(sinceMs: number): { count: number; latestAt?: number } {
|
|
1453
|
+
const row = selectGhRefusalsSince.get(sinceMs);
|
|
1454
|
+
if (row === null || row === undefined || row.n === 0 || row.latest === null) {
|
|
1455
|
+
return { count: 0 };
|
|
1456
|
+
}
|
|
1457
|
+
return { count: row.n, latestAt: row.latest };
|
|
1458
|
+
},
|
|
1459
|
+
|
|
1460
|
+
bumpGhCalls(day: string, source: string): void {
|
|
1461
|
+
bumpGhCall.run(day, source);
|
|
1462
|
+
},
|
|
1463
|
+
|
|
1464
|
+
ghCallsToday(day: string): { source: string; calls: number }[] {
|
|
1465
|
+
return selectGhCalls.all(day);
|
|
1466
|
+
},
|
|
1467
|
+
|
|
1253
1468
|
pendingFriction(
|
|
1254
1469
|
project: string,
|
|
1255
1470
|
sinceEpochMs: number,
|
|
@@ -1532,6 +1747,43 @@ export function openStore(dbPath: string): Store {
|
|
|
1532
1747
|
return row === null ? undefined : { ...row };
|
|
1533
1748
|
},
|
|
1534
1749
|
|
|
1750
|
+
enqueueLabelOps(
|
|
1751
|
+
project: string,
|
|
1752
|
+
ops: readonly { issue: number; op: "add" | "remove"; label: string }[],
|
|
1753
|
+
): void {
|
|
1754
|
+
if (ops.length === 0) return;
|
|
1755
|
+
const now = Date.now();
|
|
1756
|
+
enqueueLabelOpsTx(
|
|
1757
|
+
ops.map((op) => ({ ...op, project, createdAt: now })),
|
|
1758
|
+
);
|
|
1759
|
+
},
|
|
1760
|
+
|
|
1761
|
+
pendingLabelOps(project: string, now: number): LabelOp[] {
|
|
1762
|
+
return selectPendingLabelOps.all(project, now).map(toLabelOp);
|
|
1763
|
+
},
|
|
1764
|
+
|
|
1765
|
+
pendingLabelOpsFor(project: string, issue: number): LabelOp[] {
|
|
1766
|
+
return selectPendingLabelOpsFor.all(project, issue).map(toLabelOp);
|
|
1767
|
+
},
|
|
1768
|
+
|
|
1769
|
+
settleLabelOp(id: number): void {
|
|
1770
|
+
deleteLabelOp.run(id);
|
|
1771
|
+
},
|
|
1772
|
+
|
|
1773
|
+
deferLabelOp(id: number, error: string, nextAttemptAt: number, countAttempt = true): void {
|
|
1774
|
+
if (countAttempt) bumpLabelOpAttempt.run(error, nextAttemptAt, id);
|
|
1775
|
+
else parkLabelOp.run(error, nextAttemptAt, id);
|
|
1776
|
+
},
|
|
1777
|
+
|
|
1778
|
+
countPendingLabelOps(project: string): number {
|
|
1779
|
+
return countLabelOps.get(project)?.n ?? 0;
|
|
1780
|
+
},
|
|
1781
|
+
|
|
1782
|
+
oldestPendingLabelOpAt(project: string): number | undefined {
|
|
1783
|
+
const row = selectOldestPendingLabelOp.get(project);
|
|
1784
|
+
return row === null ? undefined : row.createdAt;
|
|
1785
|
+
},
|
|
1786
|
+
|
|
1535
1787
|
close(): void {
|
|
1536
1788
|
db.close(false);
|
|
1537
1789
|
},
|