@usine/orchestrator 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -50
- package/dist/board/items.d.ts +9 -1
- package/dist/board/items.d.ts.map +1 -1
- package/dist/board/items.js +16 -4
- package/dist/board/items.js.map +1 -1
- package/dist/board/projection.d.ts +7 -1
- package/dist/board/projection.d.ts.map +1 -1
- package/dist/board/projection.js +53 -14
- package/dist/board/projection.js.map +1 -1
- package/dist/board/tasks.d.ts +14 -2
- package/dist/board/tasks.d.ts.map +1 -1
- package/dist/board/tasks.js +23 -8
- package/dist/board/tasks.js.map +1 -1
- package/dist/credentials.d.ts +5 -6
- package/dist/credentials.d.ts.map +1 -1
- package/dist/credentials.js +22 -15
- package/dist/credentials.js.map +1 -1
- package/dist/current-instance/changes.d.ts +87 -0
- package/dist/current-instance/changes.d.ts.map +1 -0
- package/dist/current-instance/changes.js +138 -0
- package/dist/current-instance/changes.js.map +1 -0
- package/dist/current-instance/current-instance.d.ts +89 -0
- package/dist/current-instance/current-instance.d.ts.map +1 -0
- package/dist/current-instance/current-instance.js +163 -0
- package/dist/current-instance/current-instance.js.map +1 -0
- package/dist/exe-dev.d.ts +6 -2
- package/dist/exe-dev.d.ts.map +1 -1
- package/dist/exe-dev.js +7 -8
- package/dist/exe-dev.js.map +1 -1
- package/dist/idle-triggers.d.ts +1 -1
- package/dist/idle-triggers.d.ts.map +1 -1
- package/dist/idle-triggers.js +1 -1
- package/dist/idle-triggers.js.map +1 -1
- package/dist/label-reconciler.d.ts +5 -2
- package/dist/label-reconciler.d.ts.map +1 -1
- package/dist/label-reconciler.js.map +1 -1
- package/dist/local.d.ts +4 -18
- package/dist/local.d.ts.map +1 -1
- package/dist/local.js +7 -45
- package/dist/local.js.map +1 -1
- package/dist/loop-matches.d.ts +28 -0
- package/dist/loop-matches.d.ts.map +1 -0
- package/dist/loop-matches.js +74 -0
- package/dist/loop-matches.js.map +1 -0
- package/dist/operator/attempt.d.ts +60 -10
- package/dist/operator/attempt.d.ts.map +1 -1
- package/dist/operator/attempt.js +54 -11
- package/dist/operator/attempt.js.map +1 -1
- package/dist/operator/board.d.ts +43 -16
- package/dist/operator/board.d.ts.map +1 -1
- package/dist/operator/board.js +43 -16
- package/dist/operator/board.js.map +1 -1
- package/dist/operator/configuration.d.ts +258 -0
- package/dist/operator/configuration.d.ts.map +1 -0
- package/dist/operator/configuration.js +410 -0
- package/dist/operator/configuration.js.map +1 -0
- package/dist/operator/desk.d.ts +181 -0
- package/dist/operator/desk.d.ts.map +1 -0
- package/dist/operator/desk.js +463 -0
- package/dist/operator/desk.js.map +1 -0
- package/dist/operator/github-changes.d.ts +88 -0
- package/dist/operator/github-changes.d.ts.map +1 -0
- package/dist/operator/github-changes.js +266 -0
- package/dist/operator/github-changes.js.map +1 -0
- package/dist/operator/http.d.ts +191 -30
- package/dist/operator/http.d.ts.map +1 -1
- package/dist/operator/http.js +153 -23
- package/dist/operator/http.js.map +1 -1
- package/dist/operator/need.d.ts +44 -11
- package/dist/operator/need.d.ts.map +1 -1
- package/dist/operator/need.js +46 -9
- package/dist/operator/need.js.map +1 -1
- package/dist/operator/operator.d.ts +111 -27
- package/dist/operator/operator.d.ts.map +1 -1
- package/dist/operator/operator.js +286 -45
- package/dist/operator/operator.js.map +1 -1
- package/dist/operator/overview.d.ts +7 -7
- package/dist/operator/overview.d.ts.map +1 -1
- package/dist/operator/overview.js +5 -5
- package/dist/operator/overview.js.map +1 -1
- package/dist/operator/projection.d.ts +56 -4
- package/dist/operator/projection.d.ts.map +1 -1
- package/dist/operator/projection.js +125 -15
- package/dist/operator/projection.js.map +1 -1
- package/dist/operator/run.d.ts +92 -3
- package/dist/operator/run.d.ts.map +1 -1
- package/dist/operator/run.js +87 -4
- package/dist/operator/run.js.map +1 -1
- package/dist/operator/task.d.ts +93 -0
- package/dist/operator/task.d.ts.map +1 -0
- package/dist/operator/task.js +204 -0
- package/dist/operator/task.js.map +1 -0
- package/dist/operator/transcript.d.ts +14 -12
- package/dist/operator/transcript.d.ts.map +1 -1
- package/dist/operator/transcript.js +18 -9
- package/dist/operator/transcript.js.map +1 -1
- package/dist/operator/usage.d.ts +111 -0
- package/dist/operator/usage.d.ts.map +1 -0
- package/dist/operator/usage.js +219 -0
- package/dist/operator/usage.js.map +1 -0
- package/dist/orchestrator.d.ts +8 -5
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +4 -2
- package/dist/orchestrator.js.map +1 -1
- package/dist/polling/cycle.d.ts +28 -10
- package/dist/polling/cycle.d.ts.map +1 -1
- package/dist/polling/cycle.js +20 -6
- package/dist/polling/cycle.js.map +1 -1
- package/dist/polling/evaluation-failures.d.ts +42 -0
- package/dist/polling/evaluation-failures.d.ts.map +1 -0
- package/dist/polling/evaluation-failures.js +103 -0
- package/dist/polling/evaluation-failures.js.map +1 -0
- package/dist/polling/hints.d.ts +2 -2
- package/dist/polling/hints.d.ts.map +1 -1
- package/dist/polling/hints.js +2 -2
- package/dist/polling/hints.js.map +1 -1
- package/dist/polling/loop-evaluation.d.ts +21 -11
- package/dist/polling/loop-evaluation.d.ts.map +1 -1
- package/dist/polling/loop-evaluation.js +70 -27
- package/dist/polling/loop-evaluation.js.map +1 -1
- package/dist/polling/open-issues.d.ts +23 -0
- package/dist/polling/open-issues.d.ts.map +1 -0
- package/dist/polling/open-issues.js +77 -0
- package/dist/polling/open-issues.js.map +1 -0
- package/dist/polling/polling.d.ts +20 -11
- package/dist/polling/polling.d.ts.map +1 -1
- package/dist/polling/polling.js +201 -51
- package/dist/polling/polling.js.map +1 -1
- package/dist/polling/repository-sync.d.ts +20 -14
- package/dist/polling/repository-sync.d.ts.map +1 -1
- package/dist/polling/repository-sync.js +34 -27
- package/dist/polling/repository-sync.js.map +1 -1
- package/dist/polling-triggers.d.ts +1 -1
- package/dist/polling-triggers.d.ts.map +1 -1
- package/dist/polling-triggers.js +1 -1
- package/dist/polling-triggers.js.map +1 -1
- package/dist/reloads.d.ts +14 -0
- package/dist/reloads.d.ts.map +1 -0
- package/dist/reloads.js +7 -0
- package/dist/reloads.js.map +1 -0
- package/dist/runtime.d.ts +24 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +58 -0
- package/dist/runtime.js.map +1 -0
- package/dist/schedule-triggers.d.ts +14 -10
- package/dist/schedule-triggers.d.ts.map +1 -1
- package/dist/schedule-triggers.js +36 -31
- package/dist/schedule-triggers.js.map +1 -1
- package/dist/scheduler.d.ts +35 -4
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +327 -40
- package/dist/scheduler.js.map +1 -1
- package/dist/ssh.d.ts +39 -0
- package/dist/ssh.d.ts.map +1 -0
- package/dist/ssh.js +75 -0
- package/dist/ssh.js.map +1 -0
- package/dist/trigger-condition.d.ts +12 -0
- package/dist/trigger-condition.d.ts.map +1 -0
- package/dist/trigger-condition.js +88 -0
- package/dist/trigger-condition.js.map +1 -0
- package/dist/trigger-submission.d.ts +16 -13
- package/dist/trigger-submission.d.ts.map +1 -1
- package/dist/trigger-submission.js +21 -26
- package/dist/trigger-submission.js.map +1 -1
- package/dist/worker-engine.d.ts +10 -0
- package/dist/worker-engine.d.ts.map +1 -0
- package/dist/worker-engine.js +25 -0
- package/dist/worker-engine.js.map +1 -0
- package/dist/worker-status.d.ts +70 -0
- package/dist/worker-status.d.ts.map +1 -0
- package/dist/worker-status.js +55 -0
- package/dist/worker-status.js.map +1 -0
- package/package.json +31 -10
|
@@ -1,30 +1,64 @@
|
|
|
1
|
+
import { transcriptEvent } from "./projection.js";
|
|
2
|
+
import { TranscriptSessionsQuery } from "@usine/engine/transcript";
|
|
1
3
|
import { GitHubLabelName } from "@usine/config/label";
|
|
4
|
+
import { resolveParentModel } from "@usine/core/agent";
|
|
5
|
+
import { AttemptContext, AttemptId } from "@usine/core/attempt";
|
|
6
|
+
import { Intervention } from "@usine/core/attempt/channel";
|
|
2
7
|
import { Event, DedupKey, EventId } from "@usine/core/event";
|
|
3
8
|
import { CycleBudgetAskId } from "@usine/core/cycle-budget";
|
|
9
|
+
import { BoardMoveRecord, BoardMoves } from "@usine/engine/board-moves";
|
|
4
10
|
import { CycleBudgets } from "@usine/engine/cycle-budget";
|
|
5
11
|
import { LoopName } from "@usine/core/loop";
|
|
6
12
|
import { RunId } from "@usine/core/run";
|
|
7
|
-
import {
|
|
13
|
+
import { ApprovalSource } from "@usine/core/run/approval";
|
|
14
|
+
import { RetryRequest, RunAgainRequest, RunEngine, RunOperationRejectedError, } from "@usine/engine/engine";
|
|
8
15
|
import { ItemProjectionReader } from "@usine/engine/item-projection";
|
|
9
16
|
import { LoopControl } from "@usine/engine/loop-control";
|
|
10
17
|
import { RunReader } from "@usine/engine/run-reader";
|
|
11
18
|
import { RepositoryLabelProjectionReader } from "@usine/engine/repository-label-projection";
|
|
12
19
|
import { GitHubItemWriter } from "@usine/github/item-writer";
|
|
13
20
|
import { ChangedIssue, ChangedPullRequest } from "@usine/github/polling";
|
|
14
|
-
import { Context, DateTime, Effect, Exit, Layer, Option, Schema, SynchronizedRef } from "effect";
|
|
21
|
+
import { Context, DateTime, Effect, Exit, Layer, Match, Option, Schema, SynchronizedRef, } from "effect";
|
|
15
22
|
import { labelMutationForBoardMove, projectBoard } from "../board/projection.js";
|
|
23
|
+
import { LoopMatches } from "../loop-matches.js";
|
|
16
24
|
import { GitHubItemKey, PollingTriggers } from "../polling-triggers.js";
|
|
17
25
|
import { ScheduleTriggers } from "../schedule-triggers.js";
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
26
|
+
import { cycleLimitOf } from "../trigger-submission.js";
|
|
27
|
+
import { ApprovalNeed, ConfigFileNeed, CycleBudgetNeed, BoardProjectionNeed, PollFailureNeed, PreflightFindingNeed, RunFailureNeed, ScheduleFailureNeed, UnconfiguredWorkerNeed, } from "./need.js";
|
|
28
|
+
import { BoardItemNotFoundError as BoardItemNotFound, BoardLaneNotFoundError as BoardLaneNotFound, BoardLaneReadOnlyError as BoardLaneReadOnly, BoardProjectNotFoundError as BoardProjectNotFound, BoardTaskView, BoardView, LabelView, ReadOnlyBoardLaneView, WritableBoardLaneView, } from "./board.js";
|
|
20
29
|
import { ColdTranscriptRequest, } from "./transcript.js";
|
|
21
30
|
import { LoopView, ProjectView, ReadyOverview } from "./overview.js";
|
|
22
|
-
import { RunDetailView } from "./run.js";
|
|
23
|
-
import { readTranscript, runView } from "./projection.js";
|
|
24
|
-
|
|
31
|
+
import { RunDetailView, EditAndRetry, LoopNotFoundError, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
|
|
32
|
+
import { readRunningEvents, readTranscript, readUnrecordedEnds, runDuration, runView, workerAccesses, } from "./projection.js";
|
|
33
|
+
import { RunUsageReader, taskUsage } from "./usage.js";
|
|
34
|
+
import { RunGitHubChanges, } from "./github-changes.js";
|
|
35
|
+
import { TaskReader } from "./task.js";
|
|
36
|
+
import { OperatorDesk } from "./desk.js";
|
|
37
|
+
/** The number of closed Runs the overview keeps next to the open ones. */
|
|
25
38
|
const OVERVIEW_CLOSED_RUNS = 100;
|
|
26
|
-
/**
|
|
39
|
+
/** The number of newest Items a Board shows. */
|
|
27
40
|
const OVERVIEW_BOARD_ITEMS = 500;
|
|
41
|
+
/**
|
|
42
|
+
* Returns `overview` with the usage of every card that contains a `changed` Run recomputed from
|
|
43
|
+
* `usage`.
|
|
44
|
+
*/
|
|
45
|
+
const withRunUsage = (overview, changed, usage) => new ReadyOverview({
|
|
46
|
+
...overview,
|
|
47
|
+
projects: overview.projects.map((project) => new ProjectView({
|
|
48
|
+
...project,
|
|
49
|
+
board: new BoardView({
|
|
50
|
+
...project.board,
|
|
51
|
+
lanes: project.board.lanes.map((lane) => {
|
|
52
|
+
const items = lane.items.map((task) => task.runs.some(({ id }) => changed.has(id))
|
|
53
|
+
? new BoardTaskView({ ...task, usage: taskUsage(task.runs, usage) })
|
|
54
|
+
: task);
|
|
55
|
+
return lane._tag === "WritableBoardLane"
|
|
56
|
+
? new WritableBoardLaneView({ ...lane, items })
|
|
57
|
+
: new ReadOnlyBoardLaneView({ ...lane, items });
|
|
58
|
+
}),
|
|
59
|
+
}),
|
|
60
|
+
})),
|
|
61
|
+
});
|
|
28
62
|
const makeOperatorObservation = Effect.gen(function* () {
|
|
29
63
|
const budgets = yield* CycleBudgets;
|
|
30
64
|
const reader = yield* RunReader;
|
|
@@ -33,49 +67,74 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
33
67
|
const repositoryLabels = yield* RepositoryLabelProjectionReader;
|
|
34
68
|
const polling = yield* PollingTriggers;
|
|
35
69
|
const schedules = yield* ScheduleTriggers;
|
|
70
|
+
const loopMatches = yield* LoopMatches;
|
|
36
71
|
const cache = yield* SynchronizedRef.make(Option.none());
|
|
72
|
+
const usageReader = yield* RunUsageReader;
|
|
37
73
|
const projectOverview = Effect.fn("Operator.projectOverview")(function* (instance, pollFailures, preflightFindings, scheduleFailures, revision) {
|
|
38
|
-
const [snapshots, states, cycleAsks] = yield* Effect.all([
|
|
74
|
+
const [snapshots, states, cycleAsks, workerAttempts] = yield* Effect.all([
|
|
39
75
|
reader.list({ closedLimit: OVERVIEW_CLOSED_RUNS }),
|
|
40
76
|
controls.states,
|
|
41
77
|
budgets.asks(),
|
|
78
|
+
reader.attemptsByWorker(),
|
|
42
79
|
], { concurrency: "unbounded" });
|
|
43
80
|
const paused = new Set(states.filter((state) => state.paused).map((state) => state.loop));
|
|
44
|
-
const
|
|
81
|
+
const access = workerAccesses(instance.config.workers);
|
|
82
|
+
const views = snapshots.map((snapshot) => runView(snapshot, paused, access));
|
|
45
83
|
const runsByRepository = new Map();
|
|
46
84
|
for (const view of views) {
|
|
47
85
|
const projectRuns = runsByRepository.get(view.repository) ?? [];
|
|
48
86
|
projectRuns.push(view);
|
|
49
87
|
runsByRepository.set(view.repository, projectRuns);
|
|
50
88
|
}
|
|
51
|
-
const
|
|
89
|
+
const projected = yield* Effect.forEach(instance.config.repos, (repository) => Effect.gen(function* () {
|
|
52
90
|
const { name } = repository;
|
|
53
91
|
const projectFindings = preflightFindings.filter(({ repository: findingRepository }) => findingRepository === name);
|
|
54
92
|
const [boardItems, labelProjection] = yield* Effect.all([
|
|
55
93
|
itemReader.list(name, { limit: OVERVIEW_BOARD_ITEMS }),
|
|
56
94
|
repositoryLabels.get(name),
|
|
57
95
|
], { concurrency: "unbounded" });
|
|
58
|
-
const history = yield*
|
|
96
|
+
const [history, matches] = yield* Effect.all([
|
|
97
|
+
reader.forItems(boardItems.map(({ itemKey }) => itemKey)),
|
|
98
|
+
loopMatches.matching(instance, repository, boardItems),
|
|
99
|
+
], { concurrency: "unbounded" });
|
|
100
|
+
const itemLoops = new Map();
|
|
101
|
+
for (const { loop, item } of matches) {
|
|
102
|
+
itemLoops.set(item.itemKey, [...(itemLoops.get(item.itemKey) ?? []), loop.name]);
|
|
103
|
+
}
|
|
59
104
|
const merged = new Map((runsByRepository.get(name) ?? []).map((run) => [run.id, run]));
|
|
60
105
|
for (const saved of history) {
|
|
61
|
-
const view = runView(saved, paused);
|
|
106
|
+
const view = runView(saved, paused, access);
|
|
62
107
|
merged.set(view.id, view);
|
|
63
108
|
}
|
|
64
109
|
const projectRuns = [...merged.values()].sort((a, b) => b.openedAt.localeCompare(a.openedAt) || b.id.localeCompare(a.id));
|
|
110
|
+
// The reader keeps ended Attempts' reports, so only running Attempts are read again. A
|
|
111
|
+
// running Run's Event gives the lane its card stays in while its Agent changes labels.
|
|
112
|
+
const [usages, runEvents] = yield* Effect.all([
|
|
113
|
+
Effect.forEach(history, (summary) => usageReader
|
|
114
|
+
.read(summary)
|
|
115
|
+
.pipe(Effect.map((usage) => [summary.run.id, usage])), { concurrency: 4 }),
|
|
116
|
+
readRunningEvents(reader, history),
|
|
117
|
+
], { concurrency: "unbounded" });
|
|
118
|
+
const runUsage = new Map(usages);
|
|
119
|
+
yield* Effect.logDebug("Read the usage of the Board's Runs", {
|
|
120
|
+
repository: name,
|
|
121
|
+
runs: runUsage.size,
|
|
122
|
+
});
|
|
65
123
|
const catalog = Option.match(labelProjection, {
|
|
66
124
|
onNone: () => [],
|
|
67
125
|
onSome: ({ labels }) => labels,
|
|
68
126
|
});
|
|
69
|
-
const board = yield* projectBoard(repository, catalog.map(({ name }) => name), boardItems, projectRuns, cycleAsks.map(({ itemKey }) => itemKey));
|
|
127
|
+
const board = yield* projectBoard(repository, catalog.map(({ name }) => name), boardItems, projectRuns, cycleAsks.map(({ itemKey }) => itemKey), itemLoops, runUsage, runEvents);
|
|
128
|
+
const working = history.filter(({ attempts }) => attempts.some(({ outcome }) => Option.isNone(outcome)));
|
|
70
129
|
const status = cycleAsks.some((ask) => ask.repository === name) ||
|
|
71
130
|
board.failures.length > 0 ||
|
|
72
131
|
projectFindings.length > 0 ||
|
|
73
|
-
projectRuns.some((run) => run.
|
|
132
|
+
projectRuns.some((run) => run.canApprove || run.canResolveFailure)
|
|
74
133
|
? "needs-you"
|
|
75
134
|
: projectRuns.some((run) => run.canCancel)
|
|
76
135
|
? "active"
|
|
77
136
|
: "clear";
|
|
78
|
-
|
|
137
|
+
const project = new ProjectView({
|
|
79
138
|
repository: name,
|
|
80
139
|
status,
|
|
81
140
|
runs: projectRuns,
|
|
@@ -87,7 +146,13 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
87
146
|
})),
|
|
88
147
|
preflightFindings: projectFindings,
|
|
89
148
|
});
|
|
149
|
+
return { project, runUsage, working };
|
|
90
150
|
}), { concurrency: 4 });
|
|
151
|
+
const projects = projected.map(({ project }) => project);
|
|
152
|
+
const usage = {
|
|
153
|
+
runs: new Map(projected.flatMap(({ runUsage }) => [...runUsage])),
|
|
154
|
+
working: projected.flatMap(({ working }) => working),
|
|
155
|
+
};
|
|
91
156
|
const loops = instance.loops.map((loop) => new LoopView({
|
|
92
157
|
name: loop.name,
|
|
93
158
|
trigger: loop.trigger.on,
|
|
@@ -114,6 +179,7 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
114
179
|
for (const project of projects)
|
|
115
180
|
for (const run of project.runs)
|
|
116
181
|
allRuns.set(run.id, run);
|
|
182
|
+
const cardedItems = new Map(projects.flatMap(({ board }) => board.lanes.flatMap(({ items }) => items.flatMap((task) => task.items.map((item) => [item.itemKey, item])))));
|
|
117
183
|
for (const run of allRuns.values()) {
|
|
118
184
|
if (run.state === "Waiting") {
|
|
119
185
|
needsYou.push(new ApprovalNeed({
|
|
@@ -121,12 +187,13 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
121
187
|
repository: run.repository,
|
|
122
188
|
runId: run.id,
|
|
123
189
|
loop: run.loop,
|
|
190
|
+
item: cardedItems.get(run.itemKey) ?? null,
|
|
124
191
|
}));
|
|
125
192
|
}
|
|
126
|
-
else if (run.
|
|
193
|
+
else if (run.canResolveFailure) {
|
|
127
194
|
const attempt = run.attempts.at(-1);
|
|
128
195
|
if (attempt?.outcome?._tag !== "Failed") {
|
|
129
|
-
return yield* Effect.die(new Error(`Failed
|
|
196
|
+
return yield* Effect.die(new Error(`Failed run ${run.id} has no failed attempt`));
|
|
130
197
|
}
|
|
131
198
|
needsYou.push(new RunFailureNeed({
|
|
132
199
|
id: `run-failure:${run.id}`,
|
|
@@ -163,13 +230,56 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
163
230
|
failure,
|
|
164
231
|
}));
|
|
165
232
|
});
|
|
166
|
-
const
|
|
233
|
+
for (const { file, error } of instance.invalidAgents) {
|
|
234
|
+
needsYou.push(new ConfigFileNeed({ id: `config-file:${file}`, kind: "agent", error }));
|
|
235
|
+
}
|
|
236
|
+
for (const { file, error } of instance.quarantinedLoops) {
|
|
237
|
+
needsYou.push(new ConfigFileNeed({ id: `config-file:${file}`, kind: "loop", error }));
|
|
238
|
+
}
|
|
239
|
+
const configuredWorkers = new Set(instance.config.workers.map(({ name }) => name));
|
|
240
|
+
for (const { worker, attempts, running } of workerAttempts) {
|
|
241
|
+
if (configuredWorkers.has(worker))
|
|
242
|
+
continue;
|
|
243
|
+
needsYou.push(new UnconfiguredWorkerNeed({
|
|
244
|
+
id: `unconfigured-worker:${worker}`,
|
|
245
|
+
worker,
|
|
246
|
+
attempts,
|
|
247
|
+
running,
|
|
248
|
+
}));
|
|
249
|
+
}
|
|
250
|
+
const overview = new ReadyOverview({ projects, loops, needsYou });
|
|
167
251
|
yield* Effect.logDebug("Refreshed operator overview", {
|
|
168
252
|
revision,
|
|
169
253
|
runs: views.length,
|
|
170
254
|
needsYou: needsYou.length,
|
|
255
|
+
runsAtWork: usage.working.length,
|
|
171
256
|
});
|
|
172
|
-
return
|
|
257
|
+
return { overview, usage };
|
|
258
|
+
});
|
|
259
|
+
// Each session keeps its highest report, so a Run's usage changes only with more tokens or
|
|
260
|
+
// another session.
|
|
261
|
+
const refreshWorkingUsage = Effect.fn("Operator.refreshWorkingUsage")(function* (cached) {
|
|
262
|
+
if (cached.usage.working.length === 0)
|
|
263
|
+
return cached;
|
|
264
|
+
const read = yield* Effect.forEach(cached.usage.working, (summary) => usageReader.read(summary).pipe(Effect.map((usage) => [summary.run.id, usage])), { concurrency: 4 });
|
|
265
|
+
const changed = new Set(read.flatMap(([runId, usage]) => {
|
|
266
|
+
const before = cached.usage.runs.get(runId);
|
|
267
|
+
return before?.tokens === usage.tokens && before.sessions.length === usage.sessions.length
|
|
268
|
+
? []
|
|
269
|
+
: [runId];
|
|
270
|
+
}));
|
|
271
|
+
if (changed.size === 0)
|
|
272
|
+
return cached;
|
|
273
|
+
const runs = new Map([...cached.usage.runs, ...read]);
|
|
274
|
+
yield* Effect.logDebug("Refreshed the usage of Runs at work on the cached overview", {
|
|
275
|
+
runsAtWork: read.length,
|
|
276
|
+
changed: changed.size,
|
|
277
|
+
});
|
|
278
|
+
return {
|
|
279
|
+
...cached,
|
|
280
|
+
overview: withRunUsage(cached.overview, changed, runs),
|
|
281
|
+
usage: { ...cached.usage, runs },
|
|
282
|
+
};
|
|
173
283
|
});
|
|
174
284
|
const overview = Effect.fn("Operator.overview")((instance) => SynchronizedRef.modifyEffect(cache, (cached) => Effect.gen(function* () {
|
|
175
285
|
const [revision, latestPoll, latestSchedule] = yield* Effect.all([reader.revision(), polling.latest, schedules.latest], { concurrency: "unbounded" });
|
|
@@ -185,48 +295,71 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
185
295
|
onNone: () => [],
|
|
186
296
|
onSome: ({ failures }) => failures,
|
|
187
297
|
});
|
|
188
|
-
// Without the invalidation singleton staleness
|
|
189
|
-
//
|
|
298
|
+
// Without the invalidation singleton, staleness can't be detected, so the cache is
|
|
299
|
+
// bypassed.
|
|
190
300
|
const token = Option.map(revision, (value) => JSON.stringify([
|
|
191
301
|
value,
|
|
192
|
-
instance.config.repos.map(({ name, lanes }) => [name, lanes]),
|
|
193
|
-
instance.
|
|
302
|
+
instance.config.repos.map(({ name, tags, lanes }) => [name, tags, lanes]),
|
|
303
|
+
[...workerAccesses(instance.config.workers)],
|
|
304
|
+
instance.loops.map(({ name, agent, trigger, selector }) => [
|
|
305
|
+
name,
|
|
306
|
+
agent,
|
|
307
|
+
trigger,
|
|
308
|
+
selector,
|
|
309
|
+
]),
|
|
310
|
+
[...instance.invalidAgents, ...instance.quarantinedLoops].map(({ file, error }) => [
|
|
311
|
+
file,
|
|
312
|
+
error.stage,
|
|
313
|
+
error.message,
|
|
314
|
+
]),
|
|
194
315
|
pollFailures,
|
|
195
316
|
preflightFindings,
|
|
196
317
|
scheduleFailures,
|
|
197
318
|
]));
|
|
198
319
|
if (Option.isNone(token)) {
|
|
199
320
|
yield* Effect.logWarning("Projection revision singleton is missing; the overview cache is bypassed");
|
|
200
|
-
const projected = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
|
|
321
|
+
const { overview: projected } = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
|
|
201
322
|
return [projected, Option.none()];
|
|
202
323
|
}
|
|
203
324
|
if (Option.isSome(cached) && cached.value.token === token.value) {
|
|
204
325
|
yield* Effect.logDebug("Reusing unchanged operator overview", {
|
|
205
326
|
revision,
|
|
206
327
|
});
|
|
207
|
-
|
|
328
|
+
const current = yield* refreshWorkingUsage(cached.value);
|
|
329
|
+
return [current.overview, Option.some(current)];
|
|
208
330
|
}
|
|
209
331
|
const projected = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
|
|
210
|
-
return [projected, Option.some({ token: token.value,
|
|
332
|
+
return [projected.overview, Option.some({ token: token.value, ...projected })];
|
|
211
333
|
})));
|
|
212
|
-
const detail = Effect.fn("Operator.detail")(function* (runId, request = new ColdTranscriptRequest()) {
|
|
334
|
+
const detail = Effect.fn("Operator.detail")(function* (instance, runId, request = new ColdTranscriptRequest()) {
|
|
213
335
|
yield* Effect.annotateCurrentSpan({ runId });
|
|
214
336
|
const [snapshot, states] = yield* Effect.all([reader.inspect(runId), controls.states], { concurrency: "unbounded" });
|
|
215
337
|
const paused = new Set(states.filter((state) => state.paused).map((state) => state.loop));
|
|
216
|
-
const transcript = yield*
|
|
338
|
+
const [transcript, sessionState, usage, unrecordedEnds] = yield* Effect.all([
|
|
339
|
+
readTranscript(reader, snapshot, request),
|
|
340
|
+
Effect.forEach(snapshot.attempts.filter((attempt) => Option.isNone(attempt.outcome)), (attempt) => reader.transcript(new TranscriptSessionsQuery({ attemptId: attempt.id })), { concurrency: "unbounded" }),
|
|
341
|
+
usageReader.read(snapshot),
|
|
342
|
+
readUnrecordedEnds(reader, snapshot),
|
|
343
|
+
], { concurrency: "unbounded" });
|
|
217
344
|
return new RunDetailView({
|
|
218
|
-
run: runView(snapshot, paused),
|
|
345
|
+
run: runView(snapshot, paused, workerAccesses(instance.config.workers)),
|
|
219
346
|
eventDedupKey: snapshot.event.dedupKey,
|
|
220
347
|
eventPayload: snapshot.event.payload,
|
|
221
348
|
transcript: transcript.events,
|
|
349
|
+
sessionState: sessionState.flat().map(transcriptEvent),
|
|
222
350
|
earlierTranscript: transcript.earlier,
|
|
351
|
+
worker: snapshot.attempts.at(-1)?.worker ?? null,
|
|
352
|
+
duration: runDuration(snapshot.attempts, unrecordedEnds),
|
|
353
|
+
usage,
|
|
223
354
|
});
|
|
224
355
|
});
|
|
225
|
-
|
|
356
|
+
const tasks = yield* TaskReader;
|
|
357
|
+
const { desk } = yield* OperatorDesk;
|
|
358
|
+
return { detail, overview, task: tasks.read, desk };
|
|
226
359
|
});
|
|
227
|
-
/** The read-only operator surface
|
|
360
|
+
/** The read-only operator surface, which needs no trigger execution. */
|
|
228
361
|
export class OperatorObservation extends Context.Service()("@usine/orchestrator/OperatorObservation") {
|
|
229
|
-
static layer = Layer.effect(OperatorObservation, makeOperatorObservation);
|
|
362
|
+
static layer = Layer.effect(OperatorObservation, makeOperatorObservation).pipe(Layer.provide(Layer.mergeAll(TaskReader.layer, OperatorDesk.layer, LoopMatches.layer, RunUsageReader.layer)));
|
|
230
363
|
}
|
|
231
364
|
const makeOperator = Effect.gen(function* () {
|
|
232
365
|
const budgets = yield* CycleBudgets;
|
|
@@ -236,8 +369,11 @@ const makeOperator = Effect.gen(function* () {
|
|
|
236
369
|
const items = yield* ItemProjectionReader;
|
|
237
370
|
const repositoryLabels = yield* RepositoryLabelProjectionReader;
|
|
238
371
|
const github = yield* GitHubItemWriter;
|
|
372
|
+
const reader = yield* RunReader;
|
|
373
|
+
const boardMoves = yield* BoardMoves;
|
|
239
374
|
const polling = yield* PollingTriggers;
|
|
240
|
-
const
|
|
375
|
+
const githubChanges = yield* RunGitHubChanges;
|
|
376
|
+
const approve = Effect.fn("Operator.approve")(function* (instance, runId) {
|
|
241
377
|
yield* Effect.annotateCurrentSpan({ runId });
|
|
242
378
|
const current = yield* runs.inspect(runId);
|
|
243
379
|
if (current.run.state._tag !== "Waiting") {
|
|
@@ -258,14 +394,89 @@ const makeOperator = Effect.gen(function* () {
|
|
|
258
394
|
dedupKey,
|
|
259
395
|
payload: { runId, decision: "approved" },
|
|
260
396
|
receivedAt,
|
|
261
|
-
}));
|
|
397
|
+
}), new ApprovalSource({ channel: "ui", approver: instance.config.operator }));
|
|
262
398
|
yield* Effect.logInfo("Operator approved Run", { runId });
|
|
263
399
|
});
|
|
264
|
-
const retry = Effect.fn("Operator.retry")(function* (runId) {
|
|
400
|
+
const retry = Effect.fn("Operator.retry")(function* (instance, runId) {
|
|
265
401
|
yield* Effect.annotateCurrentSpan({ runId });
|
|
266
|
-
yield* runs.retry(runId);
|
|
402
|
+
yield* runs.retry(runId, new RetryRequest({ requestedBy: instance.config.operator, amendment: Option.none() }));
|
|
267
403
|
yield* Effect.logInfo("Operator retried Run", { runId });
|
|
268
404
|
});
|
|
405
|
+
const editAndRetry = Effect.fn("Operator.editAndRetry")(function* (instance, runId, edit) {
|
|
406
|
+
yield* Effect.annotateCurrentSpan({ runId, agent: edit.agent });
|
|
407
|
+
const file = instance.agents.find(({ name }) => name === edit.agent);
|
|
408
|
+
if (file === undefined)
|
|
409
|
+
return yield* new RosterAgentNotFoundError({
|
|
410
|
+
agent: edit.agent,
|
|
411
|
+
message: `Agent "${edit.agent}" is not in the roster`,
|
|
412
|
+
});
|
|
413
|
+
const agent = file.agent;
|
|
414
|
+
yield* runs.retry(runId, new RetryRequest({
|
|
415
|
+
requestedBy: instance.config.operator,
|
|
416
|
+
amendment: Option.some(new AttemptContext({
|
|
417
|
+
task: edit.task,
|
|
418
|
+
agent,
|
|
419
|
+
parentModel: resolveParentModel(agent, instance.config.settings.parentModel),
|
|
420
|
+
})),
|
|
421
|
+
}));
|
|
422
|
+
yield* Effect.logInfo("Operator amended and retried Run", { runId, agent: agent.name });
|
|
423
|
+
});
|
|
424
|
+
const runAgain = Effect.fn("Operator.runAgain")(function* (instance, runId, edit) {
|
|
425
|
+
yield* Effect.annotateCurrentSpan({ runId, agent: edit.agent });
|
|
426
|
+
const current = yield* runs.inspect(runId);
|
|
427
|
+
const loop = instance.loops.find(({ name }) => name === current.run.loop);
|
|
428
|
+
if (loop === undefined) {
|
|
429
|
+
yield* Effect.logInfo("Run again rejected; the Run's Loop is not loaded", {
|
|
430
|
+
runId,
|
|
431
|
+
loop: current.run.loop,
|
|
432
|
+
});
|
|
433
|
+
return yield* new LoopNotFoundError({
|
|
434
|
+
loop: current.run.loop,
|
|
435
|
+
message: `Run ${runId} cannot run again: loop "${current.run.loop}" is not loaded.`,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
const file = instance.agents.find(({ name }) => name === edit.agent);
|
|
439
|
+
if (file === undefined) {
|
|
440
|
+
yield* Effect.logInfo("Run again rejected; the Agent is not in the roster", {
|
|
441
|
+
runId,
|
|
442
|
+
agent: edit.agent,
|
|
443
|
+
});
|
|
444
|
+
return yield* new RosterAgentNotFoundError({
|
|
445
|
+
agent: edit.agent,
|
|
446
|
+
message: `Agent "${edit.agent}" is not in the roster`,
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
const agent = file.agent;
|
|
450
|
+
const result = yield* runs.runAgain(runId, new RunAgainRequest({
|
|
451
|
+
step: loop.step,
|
|
452
|
+
approval: loop.approval,
|
|
453
|
+
cycleLimit: cycleLimitOf(loop),
|
|
454
|
+
permissions: loop.permissions,
|
|
455
|
+
context: new AttemptContext({
|
|
456
|
+
task: edit.task,
|
|
457
|
+
agent,
|
|
458
|
+
parentModel: resolveParentModel(agent, instance.config.settings.parentModel),
|
|
459
|
+
}),
|
|
460
|
+
requestedBy: instance.config.operator,
|
|
461
|
+
}));
|
|
462
|
+
yield* Effect.logInfo("Operator ran a Run again", Match.valueTags(result, {
|
|
463
|
+
Claimed: ({ runId: openedRunId }) => ({ runId, agent: agent.name, openedRunId }),
|
|
464
|
+
CycleBlocked: ({ ask }) => ({ runId, agent: agent.name, askId: ask.id }),
|
|
465
|
+
}));
|
|
466
|
+
return result;
|
|
467
|
+
});
|
|
468
|
+
const intervene = Effect.fn("Operator.intervene")(function* (instance, runId, input) {
|
|
469
|
+
yield* Effect.annotateCurrentSpan({
|
|
470
|
+
runId,
|
|
471
|
+
attemptId: input.attemptId,
|
|
472
|
+
action: input.action._tag,
|
|
473
|
+
});
|
|
474
|
+
yield* runs.intervene(runId, input.attemptId, new Intervention({
|
|
475
|
+
operator: Option.getOrElse(instance.config.operator, () => "local operator"),
|
|
476
|
+
target: input.target,
|
|
477
|
+
action: input.action,
|
|
478
|
+
}));
|
|
479
|
+
});
|
|
269
480
|
const cancel = Effect.fn("Operator.cancel")(function* (runId) {
|
|
270
481
|
yield* Effect.annotateCurrentSpan({ runId });
|
|
271
482
|
yield* runs.cancel(runId);
|
|
@@ -293,10 +504,16 @@ const makeOperator = Effect.gen(function* () {
|
|
|
293
504
|
items.list(move.repository, { limit: OVERVIEW_BOARD_ITEMS }),
|
|
294
505
|
repositoryLabels.get(move.repository),
|
|
295
506
|
], { concurrency: "unbounded" });
|
|
507
|
+
// Place each card as the Board does, so the move records the lane that the card left.
|
|
508
|
+
const summaries = yield* reader.forItems(storedItems.map(({ itemKey }) => itemKey));
|
|
509
|
+
const runEvents = yield* readRunningEvents(reader, summaries);
|
|
510
|
+
const access = workerAccesses(instance.config.workers);
|
|
296
511
|
const board = yield* projectBoard(repository, Option.match(labelProjection, {
|
|
297
512
|
onNone: () => [],
|
|
298
513
|
onSome: ({ labels }) => labels.map(({ name }) => name),
|
|
299
|
-
}), storedItems,
|
|
514
|
+
}), storedItems,
|
|
515
|
+
// Placement reads no pause state, so these views leave every Loop unpaused.
|
|
516
|
+
summaries.map((summary) => runView(summary, new Set(), access)), [], new Map(), new Map(), runEvents);
|
|
300
517
|
const source = board.lanes.find(({ items: laneItems }) => laneItems.some(({ items }) => items.some(({ number }) => number === move.item)));
|
|
301
518
|
const movedItem = storedItems.find(({ number }) => number === move.item);
|
|
302
519
|
if (source === undefined || movedItem === undefined) {
|
|
@@ -310,7 +527,18 @@ const makeOperator = Effect.gen(function* () {
|
|
|
310
527
|
return yield* new BoardLaneReadOnly({ repository: move.repository, lane: move.toLane });
|
|
311
528
|
}
|
|
312
529
|
const mutation = labelMutationForBoardMove(board, target);
|
|
313
|
-
const
|
|
530
|
+
const movedAt = yield* DateTime.now;
|
|
531
|
+
// Recorded before the write, so the Run reader never attributes the swap to an Agent.
|
|
532
|
+
yield* boardMoves.record(new BoardMoveRecord({
|
|
533
|
+
repository: move.repository,
|
|
534
|
+
itemKey: movedItem.itemKey,
|
|
535
|
+
at: movedAt,
|
|
536
|
+
fromLane: source.name,
|
|
537
|
+
toLane: target.name,
|
|
538
|
+
added: mutation.add,
|
|
539
|
+
removed: mutation.remove,
|
|
540
|
+
}));
|
|
541
|
+
const changed = { number: move.item, updatedAt: movedAt };
|
|
314
542
|
const item = movedItem.source === "pr" ? new ChangedPullRequest(changed) : new ChangedIssue(changed);
|
|
315
543
|
const applied = yield* github.updateLabels(move.repository, move.item, mutation).pipe(
|
|
316
544
|
// A failed or interrupted request may have applied part of the label
|
|
@@ -343,21 +571,34 @@ const makeOperator = Effect.gen(function* () {
|
|
|
343
571
|
...observation,
|
|
344
572
|
approve,
|
|
345
573
|
cancel,
|
|
574
|
+
discardLeftovers: (worker) => runs.discardLeftovers(worker).pipe(Effect.asVoid),
|
|
346
575
|
moveBoardItem,
|
|
576
|
+
githubChanges: githubChanges.read,
|
|
347
577
|
pause,
|
|
348
578
|
retry,
|
|
579
|
+
resolveFailure: runs.resolveFailure,
|
|
580
|
+
editAndRetry,
|
|
581
|
+
runAgain,
|
|
582
|
+
intervene,
|
|
349
583
|
unpause,
|
|
350
584
|
resetCycleBudget,
|
|
351
585
|
};
|
|
352
586
|
});
|
|
353
|
-
/**
|
|
587
|
+
/**
|
|
588
|
+
* The operator's factory view and actions for approving, retrying, canceling, and directing work.
|
|
589
|
+
*/
|
|
354
590
|
export class Operator extends Context.Service()("@usine/orchestrator/Operator") {
|
|
355
|
-
static layer = Layer.effect(Operator, makeOperator).pipe(Layer.provide(OperatorObservation.layer));
|
|
591
|
+
static layer = Layer.effect(Operator, makeOperator).pipe(Layer.provide(Layer.merge(OperatorObservation.layer, RunGitHubChanges.layer)));
|
|
356
592
|
}
|
|
357
593
|
export { ProjectStatus, ProjectView, LoopView, ReadyOverview, NeedsSetupOverview, FactoryOverview, } from "./overview.js";
|
|
358
|
-
export {
|
|
359
|
-
export {
|
|
360
|
-
export {
|
|
361
|
-
export {
|
|
594
|
+
export { LabelAddedView, LabelRemovedView, CommentedView, ReviewedView, BoardMovedView, GitHubChangeView, AttemptGitHubChangesView, RunGitHubChangesView, } from "./github-changes.js";
|
|
595
|
+
export { ApprovalView, RunStateTag, RunView, RunDurationView, RunDetailView, EditAndRetry, LoopNotFoundError, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
|
|
596
|
+
export { UsageCostView, SessionUsageView, PoolUsageView, RunUsageView, TaskPoolUsageView, TaskUsageView, } from "./usage.js";
|
|
597
|
+
export { InfraFailureView, DefectFailureView, AgentFailureView, BlockedFailureView, QuestionFailureView, RunawayFailureView, ChildSessionBudgetFailureView, FailureView, SucceededAttemptView, FailedAttemptView, CanceledAttemptView, AttemptOutcomeView, LocalWorkerAccess, SshWorkerAccess, WorkerAccess, AttemptView, } from "./attempt.js";
|
|
598
|
+
export { BoardLaneId, BoardLaneOrigin, BoardItemLink, BoardPullRequestView, BoardTaskView, WritableBoardLaneView, ReadOnlyBoardLaneView, BoardLaneView, BoardProjectionError, BoardView, LabelView, BoardMove, BoardProjectNotFoundError, BoardItemNotFoundError, BoardLaneNotFoundError, BoardLaneReadOnlyError, } from "./board.js";
|
|
599
|
+
export { TaskItemView, TaskLaneView, TaskCommentView, TaskView, } from "./task.js";
|
|
600
|
+
export { ApprovalNeed, CycleBudgetNeed, RunFailureNeed, PollFailureNeed, PreflightFindingNeed, BoardProjectionNeed, ScheduleFailureNeed, ConfigFileNeed, UnconfiguredWorkerNeed, NeedView, } from "./need.js";
|
|
601
|
+
export { LoopSelectorView, LoopFileView, QuarantinedLoopView, AgentFileView, InvalidAgentView, GatewayView, WorkerSlotsView, PassedCheckView, FailedCheckView, PreflightCheckView, HarnessPreflightView, WorkerConfigurationView, RemovedWorkerView, LabelStateView, LabelLaneSourceView, StateLaneSourceView, LaneSourceView, LaneConfigurationView, RepositoryConfigurationView, SettingsView, ConfigurationView, OperatorConfiguration, } from "./configuration.js";
|
|
602
|
+
export { WorkerSilentView, ProviderRetryingView, ConnectivityWarningView, RunningAttemptView, SuppressedForOperatorView, ClosedUnchangedView, RunFailedView, LoopPausedView, CycleBudgetUsedUpView, AwaitingApprovalView, QueuedForWorkerView, AwaitingPollView, StallReasonView, StalledItemView, ClosedRunView, WorkerView, RepositoryPollView, DeskView, } from "./desk.js";
|
|
362
603
|
export { FailedTranscriptView, TranscriptPayload, TranscriptEventView, TranscriptPageCursor, ColdTranscriptRequest, TailTranscriptRequest, EarlierTranscriptRequest, TranscriptRequest, TranscriptAttemptNotFoundError, } from "./transcript.js";
|
|
363
604
|
//# sourceMappingURL=operator.js.map
|