@usine/orchestrator 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -52
- package/dist/board/items.d.ts +9 -1
- package/dist/board/items.d.ts.map +1 -1
- package/dist/board/items.js +16 -4
- package/dist/board/items.js.map +1 -1
- package/dist/board/projection.d.ts +7 -1
- package/dist/board/projection.d.ts.map +1 -1
- package/dist/board/projection.js +53 -14
- package/dist/board/projection.js.map +1 -1
- package/dist/board/tasks.d.ts +14 -2
- package/dist/board/tasks.d.ts.map +1 -1
- package/dist/board/tasks.js +23 -8
- package/dist/board/tasks.js.map +1 -1
- package/dist/credentials.d.ts +5 -6
- package/dist/credentials.d.ts.map +1 -1
- package/dist/credentials.js +22 -15
- package/dist/credentials.js.map +1 -1
- package/dist/current-instance/changes.d.ts +87 -0
- package/dist/current-instance/changes.d.ts.map +1 -0
- package/dist/current-instance/changes.js +138 -0
- package/dist/current-instance/changes.js.map +1 -0
- package/dist/current-instance/current-instance.d.ts +89 -0
- package/dist/current-instance/current-instance.d.ts.map +1 -0
- package/dist/current-instance/current-instance.js +163 -0
- package/dist/current-instance/current-instance.js.map +1 -0
- package/dist/exe-dev.d.ts +6 -2
- package/dist/exe-dev.d.ts.map +1 -1
- package/dist/exe-dev.js +7 -8
- package/dist/exe-dev.js.map +1 -1
- package/dist/idle-triggers.d.ts +1 -1
- package/dist/idle-triggers.d.ts.map +1 -1
- package/dist/idle-triggers.js +1 -1
- package/dist/idle-triggers.js.map +1 -1
- package/dist/label-reconciler.d.ts +5 -2
- package/dist/label-reconciler.d.ts.map +1 -1
- package/dist/label-reconciler.js.map +1 -1
- package/dist/local.d.ts +4 -18
- package/dist/local.d.ts.map +1 -1
- package/dist/local.js +7 -45
- package/dist/local.js.map +1 -1
- package/dist/loop-matches.d.ts +28 -0
- package/dist/loop-matches.d.ts.map +1 -0
- package/dist/loop-matches.js +74 -0
- package/dist/loop-matches.js.map +1 -0
- package/dist/operator/attempt.d.ts +57 -10
- package/dist/operator/attempt.d.ts.map +1 -1
- package/dist/operator/attempt.js +49 -10
- package/dist/operator/attempt.js.map +1 -1
- package/dist/operator/board.d.ts +43 -16
- package/dist/operator/board.d.ts.map +1 -1
- package/dist/operator/board.js +43 -16
- package/dist/operator/board.js.map +1 -1
- package/dist/operator/configuration.d.ts +258 -0
- package/dist/operator/configuration.d.ts.map +1 -0
- package/dist/operator/configuration.js +410 -0
- package/dist/operator/configuration.js.map +1 -0
- package/dist/operator/desk.d.ts +181 -0
- package/dist/operator/desk.d.ts.map +1 -0
- package/dist/operator/desk.js +463 -0
- package/dist/operator/desk.js.map +1 -0
- package/dist/operator/github-changes.d.ts +88 -0
- package/dist/operator/github-changes.d.ts.map +1 -0
- package/dist/operator/github-changes.js +266 -0
- package/dist/operator/github-changes.js.map +1 -0
- package/dist/operator/http.d.ts +152 -57
- package/dist/operator/http.d.ts.map +1 -1
- package/dist/operator/http.js +140 -21
- package/dist/operator/http.js.map +1 -1
- package/dist/operator/need.d.ts +44 -11
- package/dist/operator/need.d.ts.map +1 -1
- package/dist/operator/need.js +46 -9
- package/dist/operator/need.js.map +1 -1
- package/dist/operator/operator.d.ts +109 -29
- package/dist/operator/operator.d.ts.map +1 -1
- package/dist/operator/operator.js +254 -52
- package/dist/operator/operator.js.map +1 -1
- package/dist/operator/overview.d.ts +7 -7
- package/dist/operator/overview.d.ts.map +1 -1
- package/dist/operator/overview.js +5 -5
- package/dist/operator/overview.js.map +1 -1
- package/dist/operator/projection.d.ts +54 -5
- package/dist/operator/projection.d.ts.map +1 -1
- package/dist/operator/projection.js +100 -9
- package/dist/operator/projection.js.map +1 -1
- package/dist/operator/run.d.ts +71 -6
- package/dist/operator/run.d.ts.map +1 -1
- package/dist/operator/run.js +69 -6
- package/dist/operator/run.js.map +1 -1
- package/dist/operator/task.d.ts +93 -0
- package/dist/operator/task.d.ts.map +1 -0
- package/dist/operator/task.js +204 -0
- package/dist/operator/task.js.map +1 -0
- package/dist/operator/transcript.d.ts +12 -12
- package/dist/operator/transcript.d.ts.map +1 -1
- package/dist/operator/transcript.js +11 -9
- package/dist/operator/transcript.js.map +1 -1
- package/dist/operator/usage.d.ts +111 -0
- package/dist/operator/usage.d.ts.map +1 -0
- package/dist/operator/usage.js +219 -0
- package/dist/operator/usage.js.map +1 -0
- package/dist/orchestrator.d.ts +8 -5
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +4 -2
- package/dist/orchestrator.js.map +1 -1
- package/dist/polling/cycle.d.ts +28 -10
- package/dist/polling/cycle.d.ts.map +1 -1
- package/dist/polling/cycle.js +20 -6
- package/dist/polling/cycle.js.map +1 -1
- package/dist/polling/evaluation-failures.d.ts +42 -0
- package/dist/polling/evaluation-failures.d.ts.map +1 -0
- package/dist/polling/evaluation-failures.js +103 -0
- package/dist/polling/evaluation-failures.js.map +1 -0
- package/dist/polling/hints.d.ts +2 -2
- package/dist/polling/hints.d.ts.map +1 -1
- package/dist/polling/hints.js +2 -2
- package/dist/polling/hints.js.map +1 -1
- package/dist/polling/loop-evaluation.d.ts +21 -11
- package/dist/polling/loop-evaluation.d.ts.map +1 -1
- package/dist/polling/loop-evaluation.js +70 -27
- package/dist/polling/loop-evaluation.js.map +1 -1
- package/dist/polling/open-issues.d.ts +23 -0
- package/dist/polling/open-issues.d.ts.map +1 -0
- package/dist/polling/open-issues.js +77 -0
- package/dist/polling/open-issues.js.map +1 -0
- package/dist/polling/polling.d.ts +20 -11
- package/dist/polling/polling.d.ts.map +1 -1
- package/dist/polling/polling.js +201 -51
- package/dist/polling/polling.js.map +1 -1
- package/dist/polling/repository-sync.d.ts +20 -14
- package/dist/polling/repository-sync.d.ts.map +1 -1
- package/dist/polling/repository-sync.js +34 -27
- package/dist/polling/repository-sync.js.map +1 -1
- package/dist/polling-triggers.d.ts +1 -1
- package/dist/polling-triggers.d.ts.map +1 -1
- package/dist/polling-triggers.js +1 -1
- package/dist/polling-triggers.js.map +1 -1
- package/dist/reloads.d.ts +14 -0
- package/dist/reloads.d.ts.map +1 -0
- package/dist/reloads.js +7 -0
- package/dist/reloads.js.map +1 -0
- package/dist/runtime.d.ts +24 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +58 -0
- package/dist/runtime.js.map +1 -0
- package/dist/schedule-triggers.d.ts +14 -10
- package/dist/schedule-triggers.d.ts.map +1 -1
- package/dist/schedule-triggers.js +36 -31
- package/dist/schedule-triggers.js.map +1 -1
- package/dist/scheduler.d.ts +35 -4
- package/dist/scheduler.d.ts.map +1 -1
- package/dist/scheduler.js +327 -41
- package/dist/scheduler.js.map +1 -1
- package/dist/ssh.d.ts +39 -0
- package/dist/ssh.d.ts.map +1 -0
- package/dist/ssh.js +75 -0
- package/dist/ssh.js.map +1 -0
- package/dist/trigger-condition.d.ts +12 -0
- package/dist/trigger-condition.d.ts.map +1 -0
- package/dist/trigger-condition.js +88 -0
- package/dist/trigger-condition.js.map +1 -0
- package/dist/trigger-submission.d.ts +16 -13
- package/dist/trigger-submission.d.ts.map +1 -1
- package/dist/trigger-submission.js +19 -14
- package/dist/trigger-submission.js.map +1 -1
- package/dist/worker-engine.d.ts +10 -0
- package/dist/worker-engine.d.ts.map +1 -0
- package/dist/worker-engine.js +25 -0
- package/dist/worker-engine.js.map +1 -0
- package/dist/worker-status.d.ts +70 -0
- package/dist/worker-status.d.ts.map +1 -0
- package/dist/worker-status.js +55 -0
- package/dist/worker-status.js.map +1 -0
- package/package.json +31 -10
|
@@ -2,34 +2,63 @@ import { transcriptEvent } from "./projection.js";
|
|
|
2
2
|
import { TranscriptSessionsQuery } from "@usine/engine/transcript";
|
|
3
3
|
import { GitHubLabelName } from "@usine/config/label";
|
|
4
4
|
import { resolveParentModel } from "@usine/core/agent";
|
|
5
|
-
import { AttemptContext } from "@usine/core/attempt";
|
|
5
|
+
import { AttemptContext, AttemptId } from "@usine/core/attempt";
|
|
6
6
|
import { Intervention } from "@usine/core/attempt/channel";
|
|
7
7
|
import { Event, DedupKey, EventId } from "@usine/core/event";
|
|
8
8
|
import { CycleBudgetAskId } from "@usine/core/cycle-budget";
|
|
9
|
+
import { BoardMoveRecord, BoardMoves } from "@usine/engine/board-moves";
|
|
9
10
|
import { CycleBudgets } from "@usine/engine/cycle-budget";
|
|
10
11
|
import { LoopName } from "@usine/core/loop";
|
|
11
12
|
import { RunId } from "@usine/core/run";
|
|
12
|
-
import {
|
|
13
|
+
import { ApprovalSource } from "@usine/core/run/approval";
|
|
14
|
+
import { RetryRequest, RunAgainRequest, RunEngine, RunOperationRejectedError, } from "@usine/engine/engine";
|
|
13
15
|
import { ItemProjectionReader } from "@usine/engine/item-projection";
|
|
14
16
|
import { LoopControl } from "@usine/engine/loop-control";
|
|
15
17
|
import { RunReader } from "@usine/engine/run-reader";
|
|
16
18
|
import { RepositoryLabelProjectionReader } from "@usine/engine/repository-label-projection";
|
|
17
19
|
import { GitHubItemWriter } from "@usine/github/item-writer";
|
|
18
20
|
import { ChangedIssue, ChangedPullRequest } from "@usine/github/polling";
|
|
19
|
-
import { Context, DateTime, Effect, Exit, Layer, Option, Schema, SynchronizedRef } from "effect";
|
|
21
|
+
import { Context, DateTime, Effect, Exit, Layer, Match, Option, Schema, SynchronizedRef, } from "effect";
|
|
20
22
|
import { labelMutationForBoardMove, projectBoard } from "../board/projection.js";
|
|
23
|
+
import { LoopMatches } from "../loop-matches.js";
|
|
21
24
|
import { GitHubItemKey, PollingTriggers } from "../polling-triggers.js";
|
|
22
25
|
import { ScheduleTriggers } from "../schedule-triggers.js";
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
26
|
+
import { cycleLimitOf } from "../trigger-submission.js";
|
|
27
|
+
import { ApprovalNeed, ConfigFileNeed, CycleBudgetNeed, BoardProjectionNeed, PollFailureNeed, PreflightFindingNeed, RunFailureNeed, ScheduleFailureNeed, UnconfiguredWorkerNeed, } from "./need.js";
|
|
28
|
+
import { BoardItemNotFoundError as BoardItemNotFound, BoardLaneNotFoundError as BoardLaneNotFound, BoardLaneReadOnlyError as BoardLaneReadOnly, BoardProjectNotFoundError as BoardProjectNotFound, BoardTaskView, BoardView, LabelView, ReadOnlyBoardLaneView, WritableBoardLaneView, } from "./board.js";
|
|
25
29
|
import { ColdTranscriptRequest, } from "./transcript.js";
|
|
26
30
|
import { LoopView, ProjectView, ReadyOverview } from "./overview.js";
|
|
27
|
-
import { RunDetailView, EditAndRetry, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
|
|
28
|
-
import { readTranscript, runView } from "./projection.js";
|
|
29
|
-
|
|
31
|
+
import { RunDetailView, EditAndRetry, LoopNotFoundError, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
|
|
32
|
+
import { readRunningEvents, readTranscript, readUnrecordedEnds, runDuration, runView, workerAccesses, } from "./projection.js";
|
|
33
|
+
import { RunUsageReader, taskUsage } from "./usage.js";
|
|
34
|
+
import { RunGitHubChanges, } from "./github-changes.js";
|
|
35
|
+
import { TaskReader } from "./task.js";
|
|
36
|
+
import { OperatorDesk } from "./desk.js";
|
|
37
|
+
/** The number of closed Runs the overview keeps next to the open ones. */
|
|
30
38
|
const OVERVIEW_CLOSED_RUNS = 100;
|
|
31
|
-
/**
|
|
39
|
+
/** The number of newest Items a Board shows. */
|
|
32
40
|
const OVERVIEW_BOARD_ITEMS = 500;
|
|
41
|
+
/**
|
|
42
|
+
* Returns `overview` with the usage of every card that contains a `changed` Run recomputed from
|
|
43
|
+
* `usage`.
|
|
44
|
+
*/
|
|
45
|
+
const withRunUsage = (overview, changed, usage) => new ReadyOverview({
|
|
46
|
+
...overview,
|
|
47
|
+
projects: overview.projects.map((project) => new ProjectView({
|
|
48
|
+
...project,
|
|
49
|
+
board: new BoardView({
|
|
50
|
+
...project.board,
|
|
51
|
+
lanes: project.board.lanes.map((lane) => {
|
|
52
|
+
const items = lane.items.map((task) => task.runs.some(({ id }) => changed.has(id))
|
|
53
|
+
? new BoardTaskView({ ...task, usage: taskUsage(task.runs, usage) })
|
|
54
|
+
: task);
|
|
55
|
+
return lane._tag === "WritableBoardLane"
|
|
56
|
+
? new WritableBoardLaneView({ ...lane, items })
|
|
57
|
+
: new ReadOnlyBoardLaneView({ ...lane, items });
|
|
58
|
+
}),
|
|
59
|
+
}),
|
|
60
|
+
})),
|
|
61
|
+
});
|
|
33
62
|
const makeOperatorObservation = Effect.gen(function* () {
|
|
34
63
|
const budgets = yield* CycleBudgets;
|
|
35
64
|
const reader = yield* RunReader;
|
|
@@ -38,49 +67,74 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
38
67
|
const repositoryLabels = yield* RepositoryLabelProjectionReader;
|
|
39
68
|
const polling = yield* PollingTriggers;
|
|
40
69
|
const schedules = yield* ScheduleTriggers;
|
|
70
|
+
const loopMatches = yield* LoopMatches;
|
|
41
71
|
const cache = yield* SynchronizedRef.make(Option.none());
|
|
72
|
+
const usageReader = yield* RunUsageReader;
|
|
42
73
|
const projectOverview = Effect.fn("Operator.projectOverview")(function* (instance, pollFailures, preflightFindings, scheduleFailures, revision) {
|
|
43
|
-
const [snapshots, states, cycleAsks] = yield* Effect.all([
|
|
74
|
+
const [snapshots, states, cycleAsks, workerAttempts] = yield* Effect.all([
|
|
44
75
|
reader.list({ closedLimit: OVERVIEW_CLOSED_RUNS }),
|
|
45
76
|
controls.states,
|
|
46
77
|
budgets.asks(),
|
|
78
|
+
reader.attemptsByWorker(),
|
|
47
79
|
], { concurrency: "unbounded" });
|
|
48
80
|
const paused = new Set(states.filter((state) => state.paused).map((state) => state.loop));
|
|
49
|
-
const
|
|
81
|
+
const access = workerAccesses(instance.config.workers);
|
|
82
|
+
const views = snapshots.map((snapshot) => runView(snapshot, paused, access));
|
|
50
83
|
const runsByRepository = new Map();
|
|
51
84
|
for (const view of views) {
|
|
52
85
|
const projectRuns = runsByRepository.get(view.repository) ?? [];
|
|
53
86
|
projectRuns.push(view);
|
|
54
87
|
runsByRepository.set(view.repository, projectRuns);
|
|
55
88
|
}
|
|
56
|
-
const
|
|
89
|
+
const projected = yield* Effect.forEach(instance.config.repos, (repository) => Effect.gen(function* () {
|
|
57
90
|
const { name } = repository;
|
|
58
91
|
const projectFindings = preflightFindings.filter(({ repository: findingRepository }) => findingRepository === name);
|
|
59
92
|
const [boardItems, labelProjection] = yield* Effect.all([
|
|
60
93
|
itemReader.list(name, { limit: OVERVIEW_BOARD_ITEMS }),
|
|
61
94
|
repositoryLabels.get(name),
|
|
62
95
|
], { concurrency: "unbounded" });
|
|
63
|
-
const history = yield*
|
|
96
|
+
const [history, matches] = yield* Effect.all([
|
|
97
|
+
reader.forItems(boardItems.map(({ itemKey }) => itemKey)),
|
|
98
|
+
loopMatches.matching(instance, repository, boardItems),
|
|
99
|
+
], { concurrency: "unbounded" });
|
|
100
|
+
const itemLoops = new Map();
|
|
101
|
+
for (const { loop, item } of matches) {
|
|
102
|
+
itemLoops.set(item.itemKey, [...(itemLoops.get(item.itemKey) ?? []), loop.name]);
|
|
103
|
+
}
|
|
64
104
|
const merged = new Map((runsByRepository.get(name) ?? []).map((run) => [run.id, run]));
|
|
65
105
|
for (const saved of history) {
|
|
66
|
-
const view = runView(saved, paused);
|
|
106
|
+
const view = runView(saved, paused, access);
|
|
67
107
|
merged.set(view.id, view);
|
|
68
108
|
}
|
|
69
109
|
const projectRuns = [...merged.values()].sort((a, b) => b.openedAt.localeCompare(a.openedAt) || b.id.localeCompare(a.id));
|
|
110
|
+
// The reader keeps ended Attempts' reports, so only running Attempts are read again. A
|
|
111
|
+
// running Run's Event gives the lane its card stays in while its Agent changes labels.
|
|
112
|
+
const [usages, runEvents] = yield* Effect.all([
|
|
113
|
+
Effect.forEach(history, (summary) => usageReader
|
|
114
|
+
.read(summary)
|
|
115
|
+
.pipe(Effect.map((usage) => [summary.run.id, usage])), { concurrency: 4 }),
|
|
116
|
+
readRunningEvents(reader, history),
|
|
117
|
+
], { concurrency: "unbounded" });
|
|
118
|
+
const runUsage = new Map(usages);
|
|
119
|
+
yield* Effect.logDebug("Read the usage of the Board's Runs", {
|
|
120
|
+
repository: name,
|
|
121
|
+
runs: runUsage.size,
|
|
122
|
+
});
|
|
70
123
|
const catalog = Option.match(labelProjection, {
|
|
71
124
|
onNone: () => [],
|
|
72
125
|
onSome: ({ labels }) => labels,
|
|
73
126
|
});
|
|
74
|
-
const board = yield* projectBoard(repository, catalog.map(({ name }) => name), boardItems, projectRuns, cycleAsks.map(({ itemKey }) => itemKey));
|
|
127
|
+
const board = yield* projectBoard(repository, catalog.map(({ name }) => name), boardItems, projectRuns, cycleAsks.map(({ itemKey }) => itemKey), itemLoops, runUsage, runEvents);
|
|
128
|
+
const working = history.filter(({ attempts }) => attempts.some(({ outcome }) => Option.isNone(outcome)));
|
|
75
129
|
const status = cycleAsks.some((ask) => ask.repository === name) ||
|
|
76
130
|
board.failures.length > 0 ||
|
|
77
131
|
projectFindings.length > 0 ||
|
|
78
|
-
projectRuns.some((run) => run.
|
|
132
|
+
projectRuns.some((run) => run.canApprove || run.canResolveFailure)
|
|
79
133
|
? "needs-you"
|
|
80
134
|
: projectRuns.some((run) => run.canCancel)
|
|
81
135
|
? "active"
|
|
82
136
|
: "clear";
|
|
83
|
-
|
|
137
|
+
const project = new ProjectView({
|
|
84
138
|
repository: name,
|
|
85
139
|
status,
|
|
86
140
|
runs: projectRuns,
|
|
@@ -92,7 +146,13 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
92
146
|
})),
|
|
93
147
|
preflightFindings: projectFindings,
|
|
94
148
|
});
|
|
149
|
+
return { project, runUsage, working };
|
|
95
150
|
}), { concurrency: 4 });
|
|
151
|
+
const projects = projected.map(({ project }) => project);
|
|
152
|
+
const usage = {
|
|
153
|
+
runs: new Map(projected.flatMap(({ runUsage }) => [...runUsage])),
|
|
154
|
+
working: projected.flatMap(({ working }) => working),
|
|
155
|
+
};
|
|
96
156
|
const loops = instance.loops.map((loop) => new LoopView({
|
|
97
157
|
name: loop.name,
|
|
98
158
|
trigger: loop.trigger.on,
|
|
@@ -119,6 +179,7 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
119
179
|
for (const project of projects)
|
|
120
180
|
for (const run of project.runs)
|
|
121
181
|
allRuns.set(run.id, run);
|
|
182
|
+
const cardedItems = new Map(projects.flatMap(({ board }) => board.lanes.flatMap(({ items }) => items.flatMap((task) => task.items.map((item) => [item.itemKey, item])))));
|
|
122
183
|
for (const run of allRuns.values()) {
|
|
123
184
|
if (run.state === "Waiting") {
|
|
124
185
|
needsYou.push(new ApprovalNeed({
|
|
@@ -126,12 +187,13 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
126
187
|
repository: run.repository,
|
|
127
188
|
runId: run.id,
|
|
128
189
|
loop: run.loop,
|
|
190
|
+
item: cardedItems.get(run.itemKey) ?? null,
|
|
129
191
|
}));
|
|
130
192
|
}
|
|
131
|
-
else if (run.
|
|
193
|
+
else if (run.canResolveFailure) {
|
|
132
194
|
const attempt = run.attempts.at(-1);
|
|
133
195
|
if (attempt?.outcome?._tag !== "Failed") {
|
|
134
|
-
return yield* Effect.die(new Error(`Failed
|
|
196
|
+
return yield* Effect.die(new Error(`Failed run ${run.id} has no failed attempt`));
|
|
135
197
|
}
|
|
136
198
|
needsYou.push(new RunFailureNeed({
|
|
137
199
|
id: `run-failure:${run.id}`,
|
|
@@ -168,13 +230,56 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
168
230
|
failure,
|
|
169
231
|
}));
|
|
170
232
|
});
|
|
171
|
-
const
|
|
233
|
+
for (const { file, error } of instance.invalidAgents) {
|
|
234
|
+
needsYou.push(new ConfigFileNeed({ id: `config-file:${file}`, kind: "agent", error }));
|
|
235
|
+
}
|
|
236
|
+
for (const { file, error } of instance.quarantinedLoops) {
|
|
237
|
+
needsYou.push(new ConfigFileNeed({ id: `config-file:${file}`, kind: "loop", error }));
|
|
238
|
+
}
|
|
239
|
+
const configuredWorkers = new Set(instance.config.workers.map(({ name }) => name));
|
|
240
|
+
for (const { worker, attempts, running } of workerAttempts) {
|
|
241
|
+
if (configuredWorkers.has(worker))
|
|
242
|
+
continue;
|
|
243
|
+
needsYou.push(new UnconfiguredWorkerNeed({
|
|
244
|
+
id: `unconfigured-worker:${worker}`,
|
|
245
|
+
worker,
|
|
246
|
+
attempts,
|
|
247
|
+
running,
|
|
248
|
+
}));
|
|
249
|
+
}
|
|
250
|
+
const overview = new ReadyOverview({ projects, loops, needsYou });
|
|
172
251
|
yield* Effect.logDebug("Refreshed operator overview", {
|
|
173
252
|
revision,
|
|
174
253
|
runs: views.length,
|
|
175
254
|
needsYou: needsYou.length,
|
|
255
|
+
runsAtWork: usage.working.length,
|
|
176
256
|
});
|
|
177
|
-
return
|
|
257
|
+
return { overview, usage };
|
|
258
|
+
});
|
|
259
|
+
// Each session keeps its highest report, so a Run's usage changes only with more tokens or
|
|
260
|
+
// another session.
|
|
261
|
+
const refreshWorkingUsage = Effect.fn("Operator.refreshWorkingUsage")(function* (cached) {
|
|
262
|
+
if (cached.usage.working.length === 0)
|
|
263
|
+
return cached;
|
|
264
|
+
const read = yield* Effect.forEach(cached.usage.working, (summary) => usageReader.read(summary).pipe(Effect.map((usage) => [summary.run.id, usage])), { concurrency: 4 });
|
|
265
|
+
const changed = new Set(read.flatMap(([runId, usage]) => {
|
|
266
|
+
const before = cached.usage.runs.get(runId);
|
|
267
|
+
return before?.tokens === usage.tokens && before.sessions.length === usage.sessions.length
|
|
268
|
+
? []
|
|
269
|
+
: [runId];
|
|
270
|
+
}));
|
|
271
|
+
if (changed.size === 0)
|
|
272
|
+
return cached;
|
|
273
|
+
const runs = new Map([...cached.usage.runs, ...read]);
|
|
274
|
+
yield* Effect.logDebug("Refreshed the usage of Runs at work on the cached overview", {
|
|
275
|
+
runsAtWork: read.length,
|
|
276
|
+
changed: changed.size,
|
|
277
|
+
});
|
|
278
|
+
return {
|
|
279
|
+
...cached,
|
|
280
|
+
overview: withRunUsage(cached.overview, changed, runs),
|
|
281
|
+
usage: { ...cached.usage, runs },
|
|
282
|
+
};
|
|
178
283
|
});
|
|
179
284
|
const overview = Effect.fn("Operator.overview")((instance) => SynchronizedRef.modifyEffect(cache, (cached) => Effect.gen(function* () {
|
|
180
285
|
const [revision, latestPoll, latestSchedule] = yield* Effect.all([reader.revision(), polling.latest, schedules.latest], { concurrency: "unbounded" });
|
|
@@ -190,52 +295,71 @@ const makeOperatorObservation = Effect.gen(function* () {
|
|
|
190
295
|
onNone: () => [],
|
|
191
296
|
onSome: ({ failures }) => failures,
|
|
192
297
|
});
|
|
193
|
-
// Without the invalidation singleton staleness
|
|
194
|
-
//
|
|
298
|
+
// Without the invalidation singleton, staleness can't be detected, so the cache is
|
|
299
|
+
// bypassed.
|
|
195
300
|
const token = Option.map(revision, (value) => JSON.stringify([
|
|
196
301
|
value,
|
|
197
|
-
instance.config.repos.map(({ name, lanes }) => [name, lanes]),
|
|
198
|
-
instance.
|
|
302
|
+
instance.config.repos.map(({ name, tags, lanes }) => [name, tags, lanes]),
|
|
303
|
+
[...workerAccesses(instance.config.workers)],
|
|
304
|
+
instance.loops.map(({ name, agent, trigger, selector }) => [
|
|
305
|
+
name,
|
|
306
|
+
agent,
|
|
307
|
+
trigger,
|
|
308
|
+
selector,
|
|
309
|
+
]),
|
|
310
|
+
[...instance.invalidAgents, ...instance.quarantinedLoops].map(({ file, error }) => [
|
|
311
|
+
file,
|
|
312
|
+
error.stage,
|
|
313
|
+
error.message,
|
|
314
|
+
]),
|
|
199
315
|
pollFailures,
|
|
200
316
|
preflightFindings,
|
|
201
317
|
scheduleFailures,
|
|
202
318
|
]));
|
|
203
319
|
if (Option.isNone(token)) {
|
|
204
320
|
yield* Effect.logWarning("Projection revision singleton is missing; the overview cache is bypassed");
|
|
205
|
-
const projected = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
|
|
321
|
+
const { overview: projected } = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
|
|
206
322
|
return [projected, Option.none()];
|
|
207
323
|
}
|
|
208
324
|
if (Option.isSome(cached) && cached.value.token === token.value) {
|
|
209
325
|
yield* Effect.logDebug("Reusing unchanged operator overview", {
|
|
210
326
|
revision,
|
|
211
327
|
});
|
|
212
|
-
|
|
328
|
+
const current = yield* refreshWorkingUsage(cached.value);
|
|
329
|
+
return [current.overview, Option.some(current)];
|
|
213
330
|
}
|
|
214
331
|
const projected = yield* projectOverview(instance, pollFailures, preflightFindings, scheduleFailures, revision);
|
|
215
|
-
return [projected, Option.some({ token: token.value,
|
|
332
|
+
return [projected.overview, Option.some({ token: token.value, ...projected })];
|
|
216
333
|
})));
|
|
217
|
-
const detail = Effect.fn("Operator.detail")(function* (runId, request = new ColdTranscriptRequest()) {
|
|
334
|
+
const detail = Effect.fn("Operator.detail")(function* (instance, runId, request = new ColdTranscriptRequest()) {
|
|
218
335
|
yield* Effect.annotateCurrentSpan({ runId });
|
|
219
336
|
const [snapshot, states] = yield* Effect.all([reader.inspect(runId), controls.states], { concurrency: "unbounded" });
|
|
220
337
|
const paused = new Set(states.filter((state) => state.paused).map((state) => state.loop));
|
|
221
|
-
const [transcript, sessionState] = yield* Effect.all([
|
|
338
|
+
const [transcript, sessionState, usage, unrecordedEnds] = yield* Effect.all([
|
|
222
339
|
readTranscript(reader, snapshot, request),
|
|
223
340
|
Effect.forEach(snapshot.attempts.filter((attempt) => Option.isNone(attempt.outcome)), (attempt) => reader.transcript(new TranscriptSessionsQuery({ attemptId: attempt.id })), { concurrency: "unbounded" }),
|
|
341
|
+
usageReader.read(snapshot),
|
|
342
|
+
readUnrecordedEnds(reader, snapshot),
|
|
224
343
|
], { concurrency: "unbounded" });
|
|
225
344
|
return new RunDetailView({
|
|
226
|
-
run: runView(snapshot, paused),
|
|
345
|
+
run: runView(snapshot, paused, workerAccesses(instance.config.workers)),
|
|
227
346
|
eventDedupKey: snapshot.event.dedupKey,
|
|
228
347
|
eventPayload: snapshot.event.payload,
|
|
229
348
|
transcript: transcript.events,
|
|
230
349
|
sessionState: sessionState.flat().map(transcriptEvent),
|
|
231
350
|
earlierTranscript: transcript.earlier,
|
|
351
|
+
worker: snapshot.attempts.at(-1)?.worker ?? null,
|
|
352
|
+
duration: runDuration(snapshot.attempts, unrecordedEnds),
|
|
353
|
+
usage,
|
|
232
354
|
});
|
|
233
355
|
});
|
|
234
|
-
|
|
356
|
+
const tasks = yield* TaskReader;
|
|
357
|
+
const { desk } = yield* OperatorDesk;
|
|
358
|
+
return { detail, overview, task: tasks.read, desk };
|
|
235
359
|
});
|
|
236
|
-
/** The read-only operator surface
|
|
360
|
+
/** The read-only operator surface, which needs no trigger execution. */
|
|
237
361
|
export class OperatorObservation extends Context.Service()("@usine/orchestrator/OperatorObservation") {
|
|
238
|
-
static layer = Layer.effect(OperatorObservation, makeOperatorObservation);
|
|
362
|
+
static layer = Layer.effect(OperatorObservation, makeOperatorObservation).pipe(Layer.provide(Layer.mergeAll(TaskReader.layer, OperatorDesk.layer, LoopMatches.layer, RunUsageReader.layer)));
|
|
239
363
|
}
|
|
240
364
|
const makeOperator = Effect.gen(function* () {
|
|
241
365
|
const budgets = yield* CycleBudgets;
|
|
@@ -245,8 +369,11 @@ const makeOperator = Effect.gen(function* () {
|
|
|
245
369
|
const items = yield* ItemProjectionReader;
|
|
246
370
|
const repositoryLabels = yield* RepositoryLabelProjectionReader;
|
|
247
371
|
const github = yield* GitHubItemWriter;
|
|
372
|
+
const reader = yield* RunReader;
|
|
373
|
+
const boardMoves = yield* BoardMoves;
|
|
248
374
|
const polling = yield* PollingTriggers;
|
|
249
|
-
const
|
|
375
|
+
const githubChanges = yield* RunGitHubChanges;
|
|
376
|
+
const approve = Effect.fn("Operator.approve")(function* (instance, runId) {
|
|
250
377
|
yield* Effect.annotateCurrentSpan({ runId });
|
|
251
378
|
const current = yield* runs.inspect(runId);
|
|
252
379
|
if (current.run.state._tag !== "Waiting") {
|
|
@@ -267,12 +394,12 @@ const makeOperator = Effect.gen(function* () {
|
|
|
267
394
|
dedupKey,
|
|
268
395
|
payload: { runId, decision: "approved" },
|
|
269
396
|
receivedAt,
|
|
270
|
-
}));
|
|
397
|
+
}), new ApprovalSource({ channel: "ui", approver: instance.config.operator }));
|
|
271
398
|
yield* Effect.logInfo("Operator approved Run", { runId });
|
|
272
399
|
});
|
|
273
|
-
const retry = Effect.fn("Operator.retry")(function* (runId) {
|
|
400
|
+
const retry = Effect.fn("Operator.retry")(function* (instance, runId) {
|
|
274
401
|
yield* Effect.annotateCurrentSpan({ runId });
|
|
275
|
-
yield* runs.retry(runId);
|
|
402
|
+
yield* runs.retry(runId, new RetryRequest({ requestedBy: instance.config.operator, amendment: Option.none() }));
|
|
276
403
|
yield* Effect.logInfo("Operator retried Run", { runId });
|
|
277
404
|
});
|
|
278
405
|
const editAndRetry = Effect.fn("Operator.editAndRetry")(function* (instance, runId, edit) {
|
|
@@ -281,16 +408,63 @@ const makeOperator = Effect.gen(function* () {
|
|
|
281
408
|
if (file === undefined)
|
|
282
409
|
return yield* new RosterAgentNotFoundError({
|
|
283
410
|
agent: edit.agent,
|
|
284
|
-
message: `Agent ${edit.agent} is not in the roster`,
|
|
411
|
+
message: `Agent "${edit.agent}" is not in the roster`,
|
|
285
412
|
});
|
|
286
413
|
const agent = file.agent;
|
|
287
|
-
yield* runs.retry(runId,
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
414
|
+
yield* runs.retry(runId, new RetryRequest({
|
|
415
|
+
requestedBy: instance.config.operator,
|
|
416
|
+
amendment: Option.some(new AttemptContext({
|
|
417
|
+
task: edit.task,
|
|
418
|
+
agent,
|
|
419
|
+
parentModel: resolveParentModel(agent, instance.config.settings.parentModel),
|
|
420
|
+
})),
|
|
421
|
+
}));
|
|
292
422
|
yield* Effect.logInfo("Operator amended and retried Run", { runId, agent: agent.name });
|
|
293
423
|
});
|
|
424
|
+
const runAgain = Effect.fn("Operator.runAgain")(function* (instance, runId, edit) {
|
|
425
|
+
yield* Effect.annotateCurrentSpan({ runId, agent: edit.agent });
|
|
426
|
+
const current = yield* runs.inspect(runId);
|
|
427
|
+
const loop = instance.loops.find(({ name }) => name === current.run.loop);
|
|
428
|
+
if (loop === undefined) {
|
|
429
|
+
yield* Effect.logInfo("Run again rejected; the Run's Loop is not loaded", {
|
|
430
|
+
runId,
|
|
431
|
+
loop: current.run.loop,
|
|
432
|
+
});
|
|
433
|
+
return yield* new LoopNotFoundError({
|
|
434
|
+
loop: current.run.loop,
|
|
435
|
+
message: `Run ${runId} cannot run again: loop "${current.run.loop}" is not loaded.`,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
const file = instance.agents.find(({ name }) => name === edit.agent);
|
|
439
|
+
if (file === undefined) {
|
|
440
|
+
yield* Effect.logInfo("Run again rejected; the Agent is not in the roster", {
|
|
441
|
+
runId,
|
|
442
|
+
agent: edit.agent,
|
|
443
|
+
});
|
|
444
|
+
return yield* new RosterAgentNotFoundError({
|
|
445
|
+
agent: edit.agent,
|
|
446
|
+
message: `Agent "${edit.agent}" is not in the roster`,
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
const agent = file.agent;
|
|
450
|
+
const result = yield* runs.runAgain(runId, new RunAgainRequest({
|
|
451
|
+
step: loop.step,
|
|
452
|
+
approval: loop.approval,
|
|
453
|
+
cycleLimit: cycleLimitOf(loop),
|
|
454
|
+
permissions: loop.permissions,
|
|
455
|
+
context: new AttemptContext({
|
|
456
|
+
task: edit.task,
|
|
457
|
+
agent,
|
|
458
|
+
parentModel: resolveParentModel(agent, instance.config.settings.parentModel),
|
|
459
|
+
}),
|
|
460
|
+
requestedBy: instance.config.operator,
|
|
461
|
+
}));
|
|
462
|
+
yield* Effect.logInfo("Operator ran a Run again", Match.valueTags(result, {
|
|
463
|
+
Claimed: ({ runId: openedRunId }) => ({ runId, agent: agent.name, openedRunId }),
|
|
464
|
+
CycleBlocked: ({ ask }) => ({ runId, agent: agent.name, askId: ask.id }),
|
|
465
|
+
}));
|
|
466
|
+
return result;
|
|
467
|
+
});
|
|
294
468
|
const intervene = Effect.fn("Operator.intervene")(function* (instance, runId, input) {
|
|
295
469
|
yield* Effect.annotateCurrentSpan({
|
|
296
470
|
runId,
|
|
@@ -330,10 +504,16 @@ const makeOperator = Effect.gen(function* () {
|
|
|
330
504
|
items.list(move.repository, { limit: OVERVIEW_BOARD_ITEMS }),
|
|
331
505
|
repositoryLabels.get(move.repository),
|
|
332
506
|
], { concurrency: "unbounded" });
|
|
507
|
+
// Place each card as the Board does, so the move records the lane that the card left.
|
|
508
|
+
const summaries = yield* reader.forItems(storedItems.map(({ itemKey }) => itemKey));
|
|
509
|
+
const runEvents = yield* readRunningEvents(reader, summaries);
|
|
510
|
+
const access = workerAccesses(instance.config.workers);
|
|
333
511
|
const board = yield* projectBoard(repository, Option.match(labelProjection, {
|
|
334
512
|
onNone: () => [],
|
|
335
513
|
onSome: ({ labels }) => labels.map(({ name }) => name),
|
|
336
|
-
}), storedItems,
|
|
514
|
+
}), storedItems,
|
|
515
|
+
// Placement reads no pause state, so these views leave every Loop unpaused.
|
|
516
|
+
summaries.map((summary) => runView(summary, new Set(), access)), [], new Map(), new Map(), runEvents);
|
|
337
517
|
const source = board.lanes.find(({ items: laneItems }) => laneItems.some(({ items }) => items.some(({ number }) => number === move.item)));
|
|
338
518
|
const movedItem = storedItems.find(({ number }) => number === move.item);
|
|
339
519
|
if (source === undefined || movedItem === undefined) {
|
|
@@ -347,7 +527,18 @@ const makeOperator = Effect.gen(function* () {
|
|
|
347
527
|
return yield* new BoardLaneReadOnly({ repository: move.repository, lane: move.toLane });
|
|
348
528
|
}
|
|
349
529
|
const mutation = labelMutationForBoardMove(board, target);
|
|
350
|
-
const
|
|
530
|
+
const movedAt = yield* DateTime.now;
|
|
531
|
+
// Recorded before the write, so the Run reader never attributes the swap to an Agent.
|
|
532
|
+
yield* boardMoves.record(new BoardMoveRecord({
|
|
533
|
+
repository: move.repository,
|
|
534
|
+
itemKey: movedItem.itemKey,
|
|
535
|
+
at: movedAt,
|
|
536
|
+
fromLane: source.name,
|
|
537
|
+
toLane: target.name,
|
|
538
|
+
added: mutation.add,
|
|
539
|
+
removed: mutation.remove,
|
|
540
|
+
}));
|
|
541
|
+
const changed = { number: move.item, updatedAt: movedAt };
|
|
351
542
|
const item = movedItem.source === "pr" ? new ChangedPullRequest(changed) : new ChangedIssue(changed);
|
|
352
543
|
const applied = yield* github.updateLabels(move.repository, move.item, mutation).pipe(
|
|
353
544
|
// A failed or interrupted request may have applied part of the label
|
|
@@ -380,23 +571,34 @@ const makeOperator = Effect.gen(function* () {
|
|
|
380
571
|
...observation,
|
|
381
572
|
approve,
|
|
382
573
|
cancel,
|
|
574
|
+
discardLeftovers: (worker) => runs.discardLeftovers(worker).pipe(Effect.asVoid),
|
|
383
575
|
moveBoardItem,
|
|
576
|
+
githubChanges: githubChanges.read,
|
|
384
577
|
pause,
|
|
385
578
|
retry,
|
|
579
|
+
resolveFailure: runs.resolveFailure,
|
|
386
580
|
editAndRetry,
|
|
581
|
+
runAgain,
|
|
387
582
|
intervene,
|
|
388
583
|
unpause,
|
|
389
584
|
resetCycleBudget,
|
|
390
585
|
};
|
|
391
586
|
});
|
|
392
|
-
/**
|
|
587
|
+
/**
|
|
588
|
+
* The operator's factory view and actions for approving, retrying, canceling, and directing work.
|
|
589
|
+
*/
|
|
393
590
|
export class Operator extends Context.Service()("@usine/orchestrator/Operator") {
|
|
394
|
-
static layer = Layer.effect(Operator, makeOperator).pipe(Layer.provide(OperatorObservation.layer));
|
|
591
|
+
static layer = Layer.effect(Operator, makeOperator).pipe(Layer.provide(Layer.merge(OperatorObservation.layer, RunGitHubChanges.layer)));
|
|
395
592
|
}
|
|
396
593
|
export { ProjectStatus, ProjectView, LoopView, ReadyOverview, NeedsSetupOverview, FactoryOverview, } from "./overview.js";
|
|
397
|
-
export {
|
|
398
|
-
export {
|
|
399
|
-
export {
|
|
400
|
-
export {
|
|
594
|
+
export { LabelAddedView, LabelRemovedView, CommentedView, ReviewedView, BoardMovedView, GitHubChangeView, AttemptGitHubChangesView, RunGitHubChangesView, } from "./github-changes.js";
|
|
595
|
+
export { ApprovalView, RunStateTag, RunView, RunDurationView, RunDetailView, EditAndRetry, LoopNotFoundError, SessionIntervention, RosterAgentNotFoundError, } from "./run.js";
|
|
596
|
+
export { UsageCostView, SessionUsageView, PoolUsageView, RunUsageView, TaskPoolUsageView, TaskUsageView, } from "./usage.js";
|
|
597
|
+
export { InfraFailureView, DefectFailureView, AgentFailureView, BlockedFailureView, QuestionFailureView, RunawayFailureView, ChildSessionBudgetFailureView, FailureView, SucceededAttemptView, FailedAttemptView, CanceledAttemptView, AttemptOutcomeView, LocalWorkerAccess, SshWorkerAccess, WorkerAccess, AttemptView, } from "./attempt.js";
|
|
598
|
+
export { BoardLaneId, BoardLaneOrigin, BoardItemLink, BoardPullRequestView, BoardTaskView, WritableBoardLaneView, ReadOnlyBoardLaneView, BoardLaneView, BoardProjectionError, BoardView, LabelView, BoardMove, BoardProjectNotFoundError, BoardItemNotFoundError, BoardLaneNotFoundError, BoardLaneReadOnlyError, } from "./board.js";
|
|
599
|
+
export { TaskItemView, TaskLaneView, TaskCommentView, TaskView, } from "./task.js";
|
|
600
|
+
export { ApprovalNeed, CycleBudgetNeed, RunFailureNeed, PollFailureNeed, PreflightFindingNeed, BoardProjectionNeed, ScheduleFailureNeed, ConfigFileNeed, UnconfiguredWorkerNeed, NeedView, } from "./need.js";
|
|
601
|
+
export { LoopSelectorView, LoopFileView, QuarantinedLoopView, AgentFileView, InvalidAgentView, GatewayView, WorkerSlotsView, PassedCheckView, FailedCheckView, PreflightCheckView, HarnessPreflightView, WorkerConfigurationView, RemovedWorkerView, LabelStateView, LabelLaneSourceView, StateLaneSourceView, LaneSourceView, LaneConfigurationView, RepositoryConfigurationView, SettingsView, ConfigurationView, OperatorConfiguration, } from "./configuration.js";
|
|
602
|
+
export { WorkerSilentView, ProviderRetryingView, ConnectivityWarningView, RunningAttemptView, SuppressedForOperatorView, ClosedUnchangedView, RunFailedView, LoopPausedView, CycleBudgetUsedUpView, AwaitingApprovalView, QueuedForWorkerView, AwaitingPollView, StallReasonView, StalledItemView, ClosedRunView, WorkerView, RepositoryPollView, DeskView, } from "./desk.js";
|
|
401
603
|
export { FailedTranscriptView, TranscriptPayload, TranscriptEventView, TranscriptPageCursor, ColdTranscriptRequest, TailTranscriptRequest, EarlierTranscriptRequest, TranscriptRequest, TranscriptAttemptNotFoundError, } from "./transcript.js";
|
|
402
604
|
//# sourceMappingURL=operator.js.map
|