@knpkv/herdr-coordinator 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -3
- package/dist/chat.d.ts +136 -10
- package/dist/chat.d.ts.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/model.d.ts +2 -0
- package/dist/model.d.ts.map +1 -1
- package/dist/model.js +2 -0
- package/dist/model.js.map +1 -1
- package/dist/orchestrator-errors.d.ts +54 -0
- package/dist/orchestrator-errors.d.ts.map +1 -0
- package/dist/orchestrator-errors.js +36 -0
- package/dist/orchestrator-errors.js.map +1 -0
- package/dist/orchestrator-model.d.ts +1160 -0
- package/dist/orchestrator-model.d.ts.map +1 -0
- package/dist/orchestrator-model.js +240 -0
- package/dist/orchestrator-model.js.map +1 -0
- package/dist/orchestrator.d.ts +34 -0
- package/dist/orchestrator.d.ts.map +1 -0
- package/dist/orchestrator.js +853 -0
- package/dist/orchestrator.js.map +1 -0
- package/dist/pull-request-evidence-model.d.ts +153 -0
- package/dist/pull-request-evidence-model.d.ts.map +1 -0
- package/dist/pull-request-evidence-model.js +136 -0
- package/dist/pull-request-evidence-model.js.map +1 -0
- package/dist/pull-request-evidence.d.ts +19 -0
- package/dist/pull-request-evidence.d.ts.map +1 -0
- package/dist/pull-request-evidence.js +102 -0
- package/dist/pull-request-evidence.js.map +1 -0
- package/dist/store.d.ts +2 -2
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +19 -37
- package/dist/store.js.map +1 -1
- package/package.json +9 -7
|
@@ -0,0 +1,853 @@
|
|
|
1
|
+
import { NodeCrypto, NodeFileSystem, NodePath } from "@effect/platform-node";
|
|
2
|
+
import { SqliteClient } from "@effect/sql-sqlite-node";
|
|
3
|
+
import { securePrivateDatabaseFiles, verifyPrivatePathIdentity } from "@knpkv/herdr-fleet/sqlite";
|
|
4
|
+
import { makeSqliteWorkBridge } from "@knpkv/herdr-work/sql";
|
|
5
|
+
import { Clock, Context, Crypto, Effect, Equal, FileSystem, Layer, Option, Path, Schedule, Schema, Stream } from "effect";
|
|
6
|
+
import * as SqlClient from "effect/sql/SqlClient";
|
|
7
|
+
import { OrchestratorConflictError, OrchestratorNotFoundError, OrchestratorStorageError, OrchestratorTransitionError, OrchestratorValidationError, OrchestratorWorkerBindingConflictError, OrchestratorWorkerStartAuthorityError, OrchestratorWorkRevisionConflictError } from "./orchestrator-errors.js";
|
|
8
|
+
import { ActivityIdempotencyKey, DispatchRequestId, OrchestratorCommand, OrchestratorDispatchActivated, OrchestratorDispatchActivation, OrchestratorEvent, OrchestratorEventDetail, OrchestratorIdempotencyKey, OrchestratorPendingDispatch, OrchestratorPendingQuery, OrchestratorReceipt, OrchestratorRequest, OrchestratorResult, OrchestratorRoute, OrchestratorRoutedSubmission, OrchestratorSolEscalationSubmission, OrchestratorWorkLink } from "./orchestrator-model.js";
|
|
9
|
+
const Row = Schema.Struct({
|
|
10
|
+
activityIdempotencyKey: Schema.String,
|
|
11
|
+
dispatchRequestId: Schema.String,
|
|
12
|
+
idempotencyKey: Schema.String,
|
|
13
|
+
command: Schema.String,
|
|
14
|
+
acceptedAt: Schema.Number,
|
|
15
|
+
isRouted: Schema.Literals([0, 1]),
|
|
16
|
+
status: Schema.String,
|
|
17
|
+
route: Schema.NullOr(Schema.String),
|
|
18
|
+
workLink: Schema.NullOr(Schema.String)
|
|
19
|
+
});
|
|
20
|
+
const EventRow = Schema.Struct({
|
|
21
|
+
dispatchRequestId: Schema.String,
|
|
22
|
+
sequence: Schema.Number,
|
|
23
|
+
type: Schema.String,
|
|
24
|
+
activityIdempotencyKey: Schema.String,
|
|
25
|
+
occurredAt: Schema.Number,
|
|
26
|
+
detail: Schema.NullOr(Schema.String),
|
|
27
|
+
result: Schema.NullOr(Schema.String)
|
|
28
|
+
});
|
|
29
|
+
const LifecycleRow = Schema.Struct({
|
|
30
|
+
activityIdempotencyKey: Schema.String,
|
|
31
|
+
dispatchRequestId: Schema.String,
|
|
32
|
+
idempotencyKey: Schema.String,
|
|
33
|
+
command: Schema.String,
|
|
34
|
+
acceptedAt: Schema.Number,
|
|
35
|
+
isRouted: Schema.Literals([0, 1]),
|
|
36
|
+
status: Schema.String,
|
|
37
|
+
sequence: Schema.NullOr(Schema.Number),
|
|
38
|
+
type: Schema.NullOr(Schema.String),
|
|
39
|
+
eventActivityIdempotencyKey: Schema.NullOr(Schema.String),
|
|
40
|
+
occurredAt: Schema.NullOr(Schema.Number),
|
|
41
|
+
detail: Schema.NullOr(Schema.String),
|
|
42
|
+
result: Schema.NullOr(Schema.String),
|
|
43
|
+
route: Schema.NullOr(Schema.String),
|
|
44
|
+
workLink: Schema.NullOr(Schema.String)
|
|
45
|
+
});
|
|
46
|
+
const RecoveryRow = Schema.Struct({
|
|
47
|
+
dispatchRequestId: DispatchRequestId,
|
|
48
|
+
acceptedAt: Schema.Number
|
|
49
|
+
});
|
|
50
|
+
const SqliteColumnRow = Schema.Struct({ name: Schema.String });
|
|
51
|
+
const recoveryPageSize = 256;
|
|
52
|
+
const storageError = (operation) => (cause) => new OrchestratorStorageError({ cause, operation });
|
|
53
|
+
// The shared private-database checks report their own step names; prefixing keeps
|
|
54
|
+
// this module's established `sqlite.*` operation names.
|
|
55
|
+
const verifyPathIdentity = (path, fileSystem, paths, operation) => verifyPrivatePathIdentity(path, operation).pipe(Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, paths), Effect.mapError((error) => new OrchestratorStorageError({ cause: error.cause, operation: error.operation })));
|
|
56
|
+
const decodeCommand = (text) => Effect.try({
|
|
57
|
+
try: () => Schema.decodeUnknownSync(OrchestratorCommand)(JSON.parse(text)),
|
|
58
|
+
catch: storageError("decode.command")
|
|
59
|
+
});
|
|
60
|
+
const decodeRoute = (text) => Effect.try({
|
|
61
|
+
try: () => JSON.parse(text),
|
|
62
|
+
catch: storageError("decode.route.json")
|
|
63
|
+
}).pipe(Effect.flatMap((value) => Schema.decodeUnknownEffect(OrchestratorRoute)(value)), Effect.mapError(storageError("decode.route")));
|
|
64
|
+
const decodeWorkLink = (text) => Effect.try({
|
|
65
|
+
try: () => JSON.parse(text),
|
|
66
|
+
catch: storageError("decode.work-link.json")
|
|
67
|
+
}).pipe(Effect.flatMap((value) => Schema.decodeUnknownEffect(OrchestratorWorkLink)(value)), Effect.mapError(storageError("decode.work-link")));
|
|
68
|
+
const decodeMetadata = Effect.fn("Orchestrator.decodeMetadata")(function* (routeText, workLinkText) {
|
|
69
|
+
const route = routeText === null ? null : yield* decodeRoute(routeText);
|
|
70
|
+
const workLink = workLinkText === null ? null : yield* decodeWorkLink(workLinkText);
|
|
71
|
+
if (route === null && workLink !== null) {
|
|
72
|
+
return yield* new OrchestratorStorageError({
|
|
73
|
+
cause: { route, workLink },
|
|
74
|
+
operation: "decode.metadata-mismatch"
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
if (route?.model === "gpt-5.6-luna" && workLink !== null) {
|
|
78
|
+
return yield* new OrchestratorStorageError({
|
|
79
|
+
cause: { route, workLink },
|
|
80
|
+
operation: "decode.metadata-mismatch"
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
if (route?.model === "gpt-5.6-sol") {
|
|
84
|
+
if (workLink === null || (route.linkedRequestId !== null && !workLink.lineage.includes(route.linkedRequestId))) {
|
|
85
|
+
return yield* new OrchestratorStorageError({
|
|
86
|
+
cause: { route, workLink },
|
|
87
|
+
operation: "decode.metadata-mismatch"
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return { route, workLink };
|
|
92
|
+
});
|
|
93
|
+
const decodeEvent = (row) => Schema.decodeUnknownEffect(OrchestratorEvent)({
|
|
94
|
+
...row,
|
|
95
|
+
dispatchRequestId: row.dispatchRequestId,
|
|
96
|
+
type: row.type
|
|
97
|
+
}).pipe(Effect.mapError(storageError("decode.event")));
|
|
98
|
+
const currentStatus = Schema.Literals([
|
|
99
|
+
"accepted",
|
|
100
|
+
"queued",
|
|
101
|
+
"running",
|
|
102
|
+
"settled",
|
|
103
|
+
"delivery_failed",
|
|
104
|
+
"task_failed"
|
|
105
|
+
]);
|
|
106
|
+
const transitionTarget = Schema.Literals([
|
|
107
|
+
"queued",
|
|
108
|
+
"running",
|
|
109
|
+
"settled",
|
|
110
|
+
"delivery_failed",
|
|
111
|
+
"task_failed"
|
|
112
|
+
]);
|
|
113
|
+
const validTransition = (from, to) => (from === "accepted" && to === "queued") ||
|
|
114
|
+
(from === "queued" && (to === "running" || to === "delivery_failed")) ||
|
|
115
|
+
(from === "running" &&
|
|
116
|
+
(to === "settled" || to === "delivery_failed" || to === "task_failed"));
|
|
117
|
+
const validEventTransition = (from, to) => to !== "accepted" && validTransition(from, to);
|
|
118
|
+
const isTerminalEvent = (event) => event.type === "settled" || event.type === "delivery_failed" || event.type === "task_failed";
|
|
119
|
+
export class Orchestrator extends Context.Service()("herdr/Orchestrator") {
|
|
120
|
+
}
|
|
121
|
+
class SqliteFileSecurity extends Context.Service()("herdr/SqliteFileSecurity") {
|
|
122
|
+
}
|
|
123
|
+
const secureSqliteFiles = (filename, fileSystem, paths) => securePrivateDatabaseFiles(filename).pipe(Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, paths), Effect.mapError((error) => new OrchestratorStorageError({ cause: error.cause, operation: `sqlite.${error.operation}` })));
|
|
124
|
+
const makeOrchestrator = Effect.gen(function* () {
|
|
125
|
+
const sql = yield* SqlClient.SqlClient;
|
|
126
|
+
const cryptoService = yield* Crypto.Crypto;
|
|
127
|
+
const workSql = makeSqliteWorkBridge(sql);
|
|
128
|
+
const fileSecurity = yield* Effect.serviceOption(SqliteFileSecurity);
|
|
129
|
+
const secureFiles = Option.match(fileSecurity, {
|
|
130
|
+
onNone: () => Effect.void,
|
|
131
|
+
onSome: ({ secure }) => secure
|
|
132
|
+
});
|
|
133
|
+
const now = Clock.currentTimeMillis;
|
|
134
|
+
yield* workSql.initialize.pipe(Effect.mapError(storageError("initialize.work")));
|
|
135
|
+
yield* sql `
|
|
136
|
+
CREATE TABLE IF NOT EXISTS orchestrator_dispatches (
|
|
137
|
+
dispatch_request_id TEXT PRIMARY KEY,
|
|
138
|
+
idempotency_key TEXT NOT NULL UNIQUE,
|
|
139
|
+
activity_idempotency_key TEXT NOT NULL UNIQUE,
|
|
140
|
+
command TEXT NOT NULL,
|
|
141
|
+
accepted_at INTEGER NOT NULL,
|
|
142
|
+
is_routed INTEGER NOT NULL DEFAULT 0 CHECK (is_routed IN (0, 1)),
|
|
143
|
+
status TEXT NOT NULL
|
|
144
|
+
)
|
|
145
|
+
`.pipe(Effect.mapError(storageError("initialize.dispatches")));
|
|
146
|
+
yield* sql `
|
|
147
|
+
CREATE TABLE IF NOT EXISTS orchestrator_events (
|
|
148
|
+
dispatch_request_id TEXT NOT NULL,
|
|
149
|
+
sequence INTEGER NOT NULL,
|
|
150
|
+
type TEXT NOT NULL,
|
|
151
|
+
activity_idempotency_key TEXT NOT NULL,
|
|
152
|
+
occurred_at INTEGER NOT NULL,
|
|
153
|
+
detail TEXT,
|
|
154
|
+
result TEXT,
|
|
155
|
+
PRIMARY KEY (dispatch_request_id, sequence),
|
|
156
|
+
FOREIGN KEY (dispatch_request_id) REFERENCES orchestrator_dispatches(dispatch_request_id)
|
|
157
|
+
)
|
|
158
|
+
`.pipe(Effect.mapError(storageError("initialize.events")));
|
|
159
|
+
yield* sql `
|
|
160
|
+
CREATE TABLE IF NOT EXISTS orchestrator_dispatch_metadata (
|
|
161
|
+
dispatch_request_id TEXT PRIMARY KEY,
|
|
162
|
+
route TEXT NOT NULL,
|
|
163
|
+
work_link TEXT,
|
|
164
|
+
FOREIGN KEY (dispatch_request_id) REFERENCES orchestrator_dispatches(dispatch_request_id)
|
|
165
|
+
)
|
|
166
|
+
`.pipe(Effect.mapError(storageError("initialize.dispatch-metadata")));
|
|
167
|
+
yield* sql.withTransaction(Effect.gen(function* () {
|
|
168
|
+
const columns = yield* sql `PRAGMA table_info(orchestrator_dispatches)`.pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(SqliteColumnRow))), Effect.mapError(storageError("initialize.dispatch-columns")));
|
|
169
|
+
if (columns.some(({ name }) => name === "is_routed"))
|
|
170
|
+
return;
|
|
171
|
+
yield* sql `
|
|
172
|
+
ALTER TABLE orchestrator_dispatches
|
|
173
|
+
ADD COLUMN is_routed INTEGER NOT NULL DEFAULT 0 CHECK (is_routed IN (0, 1))
|
|
174
|
+
`.pipe(Effect.mapError(storageError("initialize.add-routed-discriminator")));
|
|
175
|
+
yield* sql `
|
|
176
|
+
UPDATE orchestrator_dispatches
|
|
177
|
+
SET is_routed = 1
|
|
178
|
+
WHERE EXISTS (
|
|
179
|
+
SELECT 1 FROM orchestrator_dispatch_metadata metadata
|
|
180
|
+
WHERE metadata.dispatch_request_id = orchestrator_dispatches.dispatch_request_id
|
|
181
|
+
)
|
|
182
|
+
`.pipe(Effect.mapError(storageError("initialize.backfill-routed-discriminator")));
|
|
183
|
+
})).pipe(Effect.catchTag("SqlError", (cause) => Effect.fail(storageError("initialize.route-migration")(cause))));
|
|
184
|
+
yield* sql `
|
|
185
|
+
CREATE INDEX IF NOT EXISTS orchestrator_pending_dispatches_order
|
|
186
|
+
ON orchestrator_dispatches (accepted_at ASC, dispatch_request_id ASC)
|
|
187
|
+
WHERE status IN ('accepted', 'queued')
|
|
188
|
+
`.pipe(Effect.mapError(storageError("initialize.pending-index")));
|
|
189
|
+
yield* sql `
|
|
190
|
+
CREATE INDEX IF NOT EXISTS orchestrator_running_dispatches_order
|
|
191
|
+
ON orchestrator_dispatches (accepted_at ASC, dispatch_request_id ASC)
|
|
192
|
+
WHERE status = 'running'
|
|
193
|
+
`.pipe(Effect.mapError(storageError("initialize.running-index")));
|
|
194
|
+
yield* secureFiles;
|
|
195
|
+
const validateLifecycleChain = Effect.fn("Orchestrator.validateLifecycleChain")(function* (dispatch, events) {
|
|
196
|
+
const dispatchActivityIdempotencyKey = yield* Schema.decodeUnknownEffect(ActivityIdempotencyKey)(dispatch.activityIdempotencyKey).pipe(Effect.mapError(() => new OrchestratorStorageError({
|
|
197
|
+
cause: dispatch.activityIdempotencyKey,
|
|
198
|
+
operation: "transition.lifecycle-chain-mismatch"
|
|
199
|
+
})));
|
|
200
|
+
const dispatchCommand = yield* decodeCommand(dispatch.command);
|
|
201
|
+
const first = events[0];
|
|
202
|
+
if (first === undefined ||
|
|
203
|
+
first.sequence !== 0 ||
|
|
204
|
+
first.type !== "accepted" ||
|
|
205
|
+
first.occurredAt !== dispatch.acceptedAt) {
|
|
206
|
+
return yield* new OrchestratorStorageError({
|
|
207
|
+
cause: { dispatch, event: first },
|
|
208
|
+
operation: "transition.lifecycle-chain-mismatch"
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
if (first.dispatchRequestId !== dispatch.dispatchRequestId ||
|
|
212
|
+
first.activityIdempotencyKey !== dispatchActivityIdempotencyKey ||
|
|
213
|
+
first.activityIdempotencyKey !== dispatchCommand.activityIdempotencyKey) {
|
|
214
|
+
return yield* new OrchestratorStorageError({
|
|
215
|
+
cause: { dispatch, event: first },
|
|
216
|
+
operation: "transition.activity-idempotency-mismatch"
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
let previousType = first.type;
|
|
220
|
+
let previousOccurredAt = first.occurredAt;
|
|
221
|
+
for (const [index, event] of events.entries()) {
|
|
222
|
+
if (event.dispatchRequestId !== dispatch.dispatchRequestId ||
|
|
223
|
+
event.activityIdempotencyKey !== dispatchActivityIdempotencyKey ||
|
|
224
|
+
event.activityIdempotencyKey !== dispatchCommand.activityIdempotencyKey) {
|
|
225
|
+
return yield* new OrchestratorStorageError({
|
|
226
|
+
cause: { dispatchRequestId: dispatch.dispatchRequestId, event, index, previousType },
|
|
227
|
+
operation: "transition.activity-idempotency-mismatch"
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
if (event.sequence !== index ||
|
|
231
|
+
event.occurredAt < previousOccurredAt ||
|
|
232
|
+
(index > 0 && !validEventTransition(previousType, event.type))) {
|
|
233
|
+
return yield* new OrchestratorStorageError({
|
|
234
|
+
cause: { dispatchRequestId: dispatch.dispatchRequestId, event, index, previousType },
|
|
235
|
+
operation: "transition.lifecycle-chain-mismatch"
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
previousType = event.type;
|
|
239
|
+
previousOccurredAt = event.occurredAt;
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
const loadValidatedEvents = Effect.fn("Orchestrator.loadValidatedEvents")(function* (dispatchRequestId) {
|
|
243
|
+
const decodedDispatchRequestId = yield* Schema.decodeUnknownEffect(DispatchRequestId)(dispatchRequestId).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "dispatch request ID is invalid" })));
|
|
244
|
+
const rows = yield* sql `
|
|
245
|
+
SELECT d.dispatch_request_id AS "dispatchRequestId", d.idempotency_key AS "idempotencyKey",
|
|
246
|
+
d.activity_idempotency_key AS "activityIdempotencyKey", d.command,
|
|
247
|
+
d.accepted_at AS "acceptedAt", d.is_routed AS "isRouted", d.status,
|
|
248
|
+
m.route, m.work_link AS "workLink",
|
|
249
|
+
e.sequence, e.type, e.activity_idempotency_key AS "eventActivityIdempotencyKey",
|
|
250
|
+
e.occurred_at AS "occurredAt", e.detail, e.result
|
|
251
|
+
FROM orchestrator_dispatches d
|
|
252
|
+
LEFT JOIN orchestrator_dispatch_metadata m
|
|
253
|
+
ON m.dispatch_request_id = d.dispatch_request_id
|
|
254
|
+
LEFT JOIN orchestrator_events e
|
|
255
|
+
ON e.dispatch_request_id = d.dispatch_request_id
|
|
256
|
+
WHERE d.dispatch_request_id = ${decodedDispatchRequestId}
|
|
257
|
+
ORDER BY e.sequence ASC
|
|
258
|
+
`.pipe(Effect.mapError(storageError("events.snapshot")));
|
|
259
|
+
const decodedRows = yield* Schema.decodeUnknownEffect(Schema.Array(LifecycleRow))(rows).pipe(Effect.mapError(storageError("events.decode-snapshot")));
|
|
260
|
+
const first = decodedRows[0];
|
|
261
|
+
if (first === undefined) {
|
|
262
|
+
return yield* new OrchestratorNotFoundError({ dispatchRequestId: decodedDispatchRequestId });
|
|
263
|
+
}
|
|
264
|
+
const status = yield* Schema.decodeUnknownEffect(currentStatus)(first.status).pipe(Effect.mapError(() => new OrchestratorStorageError({ cause: first.status, operation: "decode.dispatch.status" })));
|
|
265
|
+
const metadata = yield* decodeMetadata(first.route, first.workLink);
|
|
266
|
+
if ((first.isRouted === 1) !== (metadata.route !== null)) {
|
|
267
|
+
return yield* new OrchestratorStorageError({
|
|
268
|
+
cause: { isRouted: first.isRouted, route: metadata.route },
|
|
269
|
+
operation: "decode.metadata-presence-mismatch"
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
const dispatch = {
|
|
273
|
+
activityIdempotencyKey: first.activityIdempotencyKey,
|
|
274
|
+
acceptedAt: first.acceptedAt,
|
|
275
|
+
command: first.command,
|
|
276
|
+
dispatchRequestId: first.dispatchRequestId,
|
|
277
|
+
idempotencyKey: first.idempotencyKey,
|
|
278
|
+
isRouted: first.isRouted,
|
|
279
|
+
status,
|
|
280
|
+
route: metadata.route,
|
|
281
|
+
workLink: metadata.workLink
|
|
282
|
+
};
|
|
283
|
+
const events = yield* Effect.gen(function* () {
|
|
284
|
+
const eventRows = [];
|
|
285
|
+
for (const row of decodedRows) {
|
|
286
|
+
const hasEvent = row.sequence !== null || row.type !== null || row.eventActivityIdempotencyKey !== null ||
|
|
287
|
+
row.occurredAt !== null;
|
|
288
|
+
if (!hasEvent)
|
|
289
|
+
continue;
|
|
290
|
+
if (row.sequence === null || row.type === null || row.eventActivityIdempotencyKey === null ||
|
|
291
|
+
row.occurredAt === null) {
|
|
292
|
+
return yield* new OrchestratorStorageError({ cause: row, operation: "events.snapshot-row-mismatch" });
|
|
293
|
+
}
|
|
294
|
+
eventRows.push({
|
|
295
|
+
activityIdempotencyKey: row.eventActivityIdempotencyKey,
|
|
296
|
+
detail: row.detail,
|
|
297
|
+
dispatchRequestId: row.dispatchRequestId,
|
|
298
|
+
occurredAt: row.occurredAt,
|
|
299
|
+
result: row.result,
|
|
300
|
+
sequence: row.sequence,
|
|
301
|
+
type: row.type
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
return yield* Effect.forEach(eventRows, decodeEvent);
|
|
305
|
+
});
|
|
306
|
+
yield* validateLifecycleChain(dispatch, events);
|
|
307
|
+
const last = events.at(-1);
|
|
308
|
+
if (last === undefined || last.type !== dispatch.status) {
|
|
309
|
+
return yield* new OrchestratorStorageError({
|
|
310
|
+
cause: { dispatch, event: last },
|
|
311
|
+
operation: "events.status-event-mismatch"
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
const command = yield* decodeCommand(dispatch.command);
|
|
315
|
+
yield* Schema.decodeUnknownEffect(OrchestratorRequest)({
|
|
316
|
+
...dispatch,
|
|
317
|
+
command
|
|
318
|
+
}).pipe(Effect.mapError(storageError("events.decode-authority")));
|
|
319
|
+
return { dispatch, events };
|
|
320
|
+
});
|
|
321
|
+
const validateLinkedParent = Effect.fn("Orchestrator.validateLinkedParent")(function* (route) {
|
|
322
|
+
if (route?.model !== "gpt-5.6-sol" || route.linkedRequestId === null)
|
|
323
|
+
return;
|
|
324
|
+
const parent = yield* loadValidatedEvents(route.linkedRequestId);
|
|
325
|
+
if (parent.dispatch.route?.model !== "gpt-5.6-luna" ||
|
|
326
|
+
(parent.dispatch.status !== "delivery_failed" && parent.dispatch.status !== "task_failed")) {
|
|
327
|
+
return yield* new OrchestratorValidationError({
|
|
328
|
+
detail: "linked Sol escalation must reference a failed Luna request"
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
});
|
|
332
|
+
const validateDurableReadback = Effect.fn("Orchestrator.validateDurableReadback")(function* (dispatch, operation) {
|
|
333
|
+
yield* validateLinkedParent(dispatch.route);
|
|
334
|
+
if (dispatch.route?.model !== "gpt-5.6-sol" || dispatch.workLink === null)
|
|
335
|
+
return;
|
|
336
|
+
yield* workSql.requireDispatchHandoff({
|
|
337
|
+
dispatchRequestId: dispatch.dispatchRequestId,
|
|
338
|
+
handoff: dispatch.workLink.handoff,
|
|
339
|
+
lineage: dispatch.workLink.lineage
|
|
340
|
+
}).pipe(Effect.mapError(storageError(operation)));
|
|
341
|
+
});
|
|
342
|
+
const listPending = Effect.fn("Orchestrator.pending")(function* (query = {}) {
|
|
343
|
+
const decodedQuery = yield* Schema.decodeUnknownEffect(OrchestratorPendingQuery)(query).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "pending query is invalid" })));
|
|
344
|
+
return yield* sql.withTransaction(Effect.gen(function* () {
|
|
345
|
+
const limit = decodedQuery.limit ?? 256;
|
|
346
|
+
const rows = decodedQuery.after === undefined
|
|
347
|
+
? yield* sql `
|
|
348
|
+
SELECT dispatch_request_id AS "dispatchRequestId", idempotency_key AS "idempotencyKey",
|
|
349
|
+
activity_idempotency_key AS "activityIdempotencyKey", command,
|
|
350
|
+
accepted_at AS "acceptedAt", is_routed AS "isRouted", status,
|
|
351
|
+
m.route, m.work_link AS "workLink"
|
|
352
|
+
FROM orchestrator_dispatches
|
|
353
|
+
LEFT JOIN orchestrator_dispatch_metadata m USING (dispatch_request_id)
|
|
354
|
+
WHERE status IN ('accepted', 'queued')
|
|
355
|
+
ORDER BY accepted_at ASC, dispatch_request_id ASC
|
|
356
|
+
LIMIT ${limit}
|
|
357
|
+
`.pipe(Effect.mapError(storageError("pending.list")))
|
|
358
|
+
: yield* sql `
|
|
359
|
+
SELECT dispatch_request_id AS "dispatchRequestId", idempotency_key AS "idempotencyKey",
|
|
360
|
+
activity_idempotency_key AS "activityIdempotencyKey", command,
|
|
361
|
+
accepted_at AS "acceptedAt", is_routed AS "isRouted", status,
|
|
362
|
+
m.route, m.work_link AS "workLink"
|
|
363
|
+
FROM orchestrator_dispatches
|
|
364
|
+
LEFT JOIN orchestrator_dispatch_metadata m USING (dispatch_request_id)
|
|
365
|
+
WHERE status IN ('accepted', 'queued')
|
|
366
|
+
AND (accepted_at > ${decodedQuery.after.acceptedAt}
|
|
367
|
+
OR (accepted_at = ${decodedQuery.after.acceptedAt}
|
|
368
|
+
AND dispatch_request_id > ${decodedQuery.after.dispatchRequestId}))
|
|
369
|
+
ORDER BY accepted_at ASC, dispatch_request_id ASC
|
|
370
|
+
LIMIT ${limit}
|
|
371
|
+
`.pipe(Effect.mapError(storageError("pending.list")));
|
|
372
|
+
const decodedRows = yield* Schema.decodeUnknownEffect(Schema.Array(Row))(rows).pipe(Effect.mapError(storageError("pending.decode-row")));
|
|
373
|
+
const pending = yield* Effect.forEach(decodedRows, (row) => Effect.gen(function* () {
|
|
374
|
+
const snapshot = yield* loadValidatedEvents(row.dispatchRequestId);
|
|
375
|
+
if (snapshot.dispatch.status !== "accepted" && snapshot.dispatch.status !== "queued") {
|
|
376
|
+
return Option.none();
|
|
377
|
+
}
|
|
378
|
+
yield* validateDurableReadback(snapshot.dispatch, "pending.work-link");
|
|
379
|
+
const command = yield* decodeCommand(snapshot.dispatch.command);
|
|
380
|
+
const pendingInput = {
|
|
381
|
+
acceptedAt: snapshot.dispatch.acceptedAt,
|
|
382
|
+
activityIdempotencyKey: snapshot.dispatch.activityIdempotencyKey,
|
|
383
|
+
command,
|
|
384
|
+
dispatchRequestId: snapshot.dispatch.dispatchRequestId,
|
|
385
|
+
idempotencyKey: snapshot.dispatch.idempotencyKey,
|
|
386
|
+
status: snapshot.dispatch.status
|
|
387
|
+
};
|
|
388
|
+
const pendingDispatch = yield* Schema.decodeUnknownEffect(OrchestratorPendingDispatch)(snapshot.dispatch.route === null
|
|
389
|
+
? pendingInput
|
|
390
|
+
: { ...pendingInput, route: snapshot.dispatch.route, workLink: snapshot.dispatch.workLink }).pipe(Effect.mapError(storageError("pending.decode")));
|
|
391
|
+
return Option.some(pendingDispatch);
|
|
392
|
+
}));
|
|
393
|
+
return pending.flatMap((value) => Option.match(value, {
|
|
394
|
+
onNone: () => [],
|
|
395
|
+
onSome: (pendingDispatch) => [pendingDispatch]
|
|
396
|
+
}));
|
|
397
|
+
})).pipe(Effect.catchTag("SqlError", (cause) => Effect.fail(new OrchestratorStorageError({ cause, operation: "pending.transaction" }))));
|
|
398
|
+
});
|
|
399
|
+
const allocateTimestamp = Effect.fn("Orchestrator.allocateTimestamp")(function* (floor) {
|
|
400
|
+
return Math.max(yield* now, floor);
|
|
401
|
+
});
|
|
402
|
+
const appendTransition = Effect.fn("Orchestrator.transition")(function* (dispatchRequestId, target, detail, result) {
|
|
403
|
+
const decodedDispatchRequestId = yield* Schema.decodeUnknownEffect(DispatchRequestId)(dispatchRequestId).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "dispatch request ID is invalid" })));
|
|
404
|
+
yield* secureFiles;
|
|
405
|
+
const decodedTarget = yield* Schema.decodeUnknownEffect(transitionTarget)(target).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: `invalid transition target ${target}` })));
|
|
406
|
+
const decodedDetail = yield* Schema.decodeUnknownEffect(Schema.NullOr(OrchestratorEventDetail))(detail).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "invalid transition detail" })));
|
|
407
|
+
const decodedResult = yield* Schema.decodeUnknownEffect(Schema.NullOr(OrchestratorResult))(result).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "invalid transition result" })));
|
|
408
|
+
const event = yield* sql.withTransaction(Effect.gen(function* () {
|
|
409
|
+
const snapshot = yield* loadValidatedEvents(decodedDispatchRequestId).pipe(Effect.catchTag("OrchestratorStorageError", (error) => error.operation === "events.status-event-mismatch"
|
|
410
|
+
? Effect.fail(new OrchestratorStorageError({ cause: error.cause, operation: "transition.status-event-mismatch" }))
|
|
411
|
+
: Effect.fail(error)));
|
|
412
|
+
const dispatch = snapshot.dispatch;
|
|
413
|
+
yield* validateDurableReadback(dispatch, "transition.work-link");
|
|
414
|
+
if (decodedTarget === "running" && dispatch.route?.model === "gpt-5.6-sol") {
|
|
415
|
+
return yield* new OrchestratorValidationError({
|
|
416
|
+
detail: "routed Sol dispatches require workerStarted with Work lane authority"
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
if (!validTransition(dispatch.status, decodedTarget)) {
|
|
420
|
+
return yield* new OrchestratorTransitionError({
|
|
421
|
+
dispatchRequestId: decodedDispatchRequestId,
|
|
422
|
+
from: dispatch.status,
|
|
423
|
+
to: decodedTarget
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
const last = snapshot.events.at(-1);
|
|
427
|
+
if (last === undefined) {
|
|
428
|
+
return yield* new OrchestratorStorageError({
|
|
429
|
+
cause: decodedDispatchRequestId,
|
|
430
|
+
operation: "transition.missing-event"
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
const timestamp = yield* allocateTimestamp(last.occurredAt);
|
|
434
|
+
const event = yield* Schema.decodeUnknownEffect(OrchestratorEvent)({
|
|
435
|
+
activityIdempotencyKey: last.activityIdempotencyKey,
|
|
436
|
+
dispatchRequestId: decodedDispatchRequestId,
|
|
437
|
+
occurredAt: timestamp,
|
|
438
|
+
sequence: last.sequence + 1,
|
|
439
|
+
type: decodedTarget,
|
|
440
|
+
...(decodedTarget === "settled"
|
|
441
|
+
? { detail: null, result: decodedResult }
|
|
442
|
+
: decodedTarget === "delivery_failed" || decodedTarget === "task_failed"
|
|
443
|
+
? { detail: decodedDetail, result: null }
|
|
444
|
+
: { detail: null, result: null })
|
|
445
|
+
}).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "transition exceeds event bounds" })));
|
|
446
|
+
const insertedRows = yield* sql `
|
|
447
|
+
INSERT INTO orchestrator_events
|
|
448
|
+
(dispatch_request_id, sequence, type, activity_idempotency_key, occurred_at, detail, result)
|
|
449
|
+
VALUES (${event.dispatchRequestId}, ${event.sequence}, ${event.type},
|
|
450
|
+
${event.activityIdempotencyKey}, ${event.occurredAt}, ${event.detail}, ${event.result})
|
|
451
|
+
ON CONFLICT (dispatch_request_id, sequence) DO NOTHING
|
|
452
|
+
RETURNING dispatch_request_id AS "dispatchRequestId", sequence, type,
|
|
453
|
+
activity_idempotency_key AS "activityIdempotencyKey", occurred_at AS "occurredAt", detail, result
|
|
454
|
+
`.pipe(Effect.mapError(storageError("transition.insert-event")));
|
|
455
|
+
if (insertedRows.length === 0) {
|
|
456
|
+
const winnerRows = yield* sql `
|
|
457
|
+
SELECT dispatch_request_id AS "dispatchRequestId", sequence, type,
|
|
458
|
+
activity_idempotency_key AS "activityIdempotencyKey", occurred_at AS "occurredAt", detail, result
|
|
459
|
+
FROM orchestrator_events
|
|
460
|
+
WHERE dispatch_request_id = ${event.dispatchRequestId} AND sequence = ${event.sequence}
|
|
461
|
+
`.pipe(Effect.mapError(storageError("transition.reload-event")));
|
|
462
|
+
const winner = yield* Schema.decodeUnknownEffect(Schema.Array(EventRow))(winnerRows).pipe(Effect.mapError(storageError("transition.decode-winner")), Effect.flatMap((rows) => {
|
|
463
|
+
const first = rows[0];
|
|
464
|
+
return first === undefined
|
|
465
|
+
? Effect.fail(new OrchestratorStorageError({ cause: event, operation: "transition.missing-winner" }))
|
|
466
|
+
: decodeEvent(first);
|
|
467
|
+
}));
|
|
468
|
+
if (winner.dispatchRequestId === event.dispatchRequestId && winner.type !== "accepted") {
|
|
469
|
+
return yield* new OrchestratorTransitionError({
|
|
470
|
+
dispatchRequestId: decodedDispatchRequestId,
|
|
471
|
+
from: winner.type,
|
|
472
|
+
to: decodedTarget
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
return yield* new OrchestratorStorageError({
|
|
476
|
+
cause: { event, winner },
|
|
477
|
+
operation: "transition.concurrent-event-mismatch"
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
yield* sql `
|
|
481
|
+
UPDATE orchestrator_dispatches SET status = ${event.type}
|
|
482
|
+
WHERE dispatch_request_id = ${decodedDispatchRequestId}
|
|
483
|
+
`.pipe(Effect.mapError(storageError("transition.update-dispatch")));
|
|
484
|
+
return event;
|
|
485
|
+
})).pipe(Effect.catchTag("SqlError", (cause) => Effect.fail(storageError("transition.transaction")(cause))));
|
|
486
|
+
return event;
|
|
487
|
+
});
|
|
488
|
+
const workerStarted = Effect.fn("Orchestrator.workerStarted")(function* (activation) {
|
|
489
|
+
const decoded = yield* Schema.decodeUnknownEffect(OrchestratorDispatchActivation)(activation).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "worker-start activation is invalid" })));
|
|
490
|
+
yield* secureFiles;
|
|
491
|
+
return yield* sql.withTransaction(Effect.gen(function* () {
|
|
492
|
+
const snapshot = yield* loadValidatedEvents(decoded.dispatchRequestId);
|
|
493
|
+
const dispatch = snapshot.dispatch;
|
|
494
|
+
if (dispatch.route?.model !== "gpt-5.6-sol" ||
|
|
495
|
+
dispatch.workLink === null ||
|
|
496
|
+
dispatch.workLink.handoff.laneId !== decoded.laneId) {
|
|
497
|
+
return yield* new OrchestratorValidationError({
|
|
498
|
+
detail: "worker start must match the routed dispatch Work lane"
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
if (decoded.expectedRevision !== dispatch.workLink.handoff.expectedRevision) {
|
|
502
|
+
return yield* new OrchestratorWorkerStartAuthorityError({
|
|
503
|
+
actualRevision: decoded.expectedRevision,
|
|
504
|
+
expectedRevision: dispatch.workLink.handoff.expectedRevision,
|
|
505
|
+
laneId: decoded.laneId,
|
|
506
|
+
reason: "accepted_revision_mismatch"
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
yield* validateDurableReadback(dispatch, "worker-start.work-link");
|
|
510
|
+
if (dispatch.status !== "queued") {
|
|
511
|
+
if (dispatch.status !== "running" &&
|
|
512
|
+
dispatch.status !== "settled" &&
|
|
513
|
+
dispatch.status !== "delivery_failed" &&
|
|
514
|
+
dispatch.status !== "task_failed") {
|
|
515
|
+
return yield* new OrchestratorTransitionError({
|
|
516
|
+
dispatchRequestId: decoded.dispatchRequestId,
|
|
517
|
+
from: dispatch.status,
|
|
518
|
+
to: "running"
|
|
519
|
+
});
|
|
520
|
+
}
|
|
521
|
+
const event = snapshot.events.find(({ type }) => type === "running");
|
|
522
|
+
if (event === undefined) {
|
|
523
|
+
return yield* new OrchestratorTransitionError({
|
|
524
|
+
dispatchRequestId: decoded.dispatchRequestId,
|
|
525
|
+
from: dispatch.status,
|
|
526
|
+
to: "running"
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
const binding = yield* workSql.requireAgentBinding(decoded).pipe(Effect.mapError((error) => error._tag === "WorkAgentBindingConflictError"
|
|
530
|
+
? new OrchestratorWorkerBindingConflictError({ dispatchRequestId: decoded.dispatchRequestId })
|
|
531
|
+
: storageError("worker-start.work-binding")(error)));
|
|
532
|
+
if (event.occurredAt !== binding.checkpoint.occurredAt) {
|
|
533
|
+
return yield* new OrchestratorStorageError({
|
|
534
|
+
cause: { binding, events: snapshot.events },
|
|
535
|
+
operation: "worker-start.replay-boundary-mismatch"
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
return yield* Schema.decodeUnknownEffect(OrchestratorDispatchActivated)({ binding, event }).pipe(Effect.mapError(storageError("worker-start.decode-replay")));
|
|
539
|
+
}
|
|
540
|
+
const last = snapshot.events.at(-1);
|
|
541
|
+
if (last === undefined) {
|
|
542
|
+
return yield* new OrchestratorStorageError({
|
|
543
|
+
cause: decoded.dispatchRequestId,
|
|
544
|
+
operation: "worker-start.missing-event"
|
|
545
|
+
});
|
|
546
|
+
}
|
|
547
|
+
const timestamp = yield* allocateTimestamp(last.occurredAt);
|
|
548
|
+
const bindingDecision = yield* workSql.acceptAgentBinding(decoded, timestamp).pipe(Effect.mapError((error) => {
|
|
549
|
+
if (error._tag === "WorkAgentBindingConflictError") {
|
|
550
|
+
return new OrchestratorWorkerBindingConflictError({ dispatchRequestId: decoded.dispatchRequestId });
|
|
551
|
+
}
|
|
552
|
+
if (error._tag === "WorkAgentBindingAuthorityError") {
|
|
553
|
+
return new OrchestratorWorkerStartAuthorityError({
|
|
554
|
+
actualRevision: error.actualRevision,
|
|
555
|
+
expectedRevision: error.expectedRevision,
|
|
556
|
+
laneId: error.laneId,
|
|
557
|
+
reason: error.reason
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
return storageError("worker-start.work-binding")(error);
|
|
561
|
+
}));
|
|
562
|
+
if (bindingDecision._tag === "replayed") {
|
|
563
|
+
return yield* new OrchestratorStorageError({
|
|
564
|
+
cause: bindingDecision.binding,
|
|
565
|
+
operation: "worker-start.partial-binding"
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
const binding = bindingDecision.binding;
|
|
569
|
+
const event = yield* Schema.decodeUnknownEffect(OrchestratorEvent)({
|
|
570
|
+
activityIdempotencyKey: last.activityIdempotencyKey,
|
|
571
|
+
dispatchRequestId: decoded.dispatchRequestId,
|
|
572
|
+
occurredAt: binding.checkpoint.occurredAt,
|
|
573
|
+
sequence: last.sequence + 1,
|
|
574
|
+
type: "running",
|
|
575
|
+
detail: null,
|
|
576
|
+
result: null
|
|
577
|
+
}).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "worker-start event exceeds bounds" })));
|
|
578
|
+
yield* sql `
|
|
579
|
+
INSERT INTO orchestrator_events
|
|
580
|
+
(dispatch_request_id, sequence, type, activity_idempotency_key, occurred_at, detail, result)
|
|
581
|
+
VALUES (${event.dispatchRequestId}, ${event.sequence}, ${event.type},
|
|
582
|
+
${event.activityIdempotencyKey}, ${event.occurredAt}, NULL, NULL)
|
|
583
|
+
`.pipe(Effect.mapError(storageError("worker-start.insert-event")));
|
|
584
|
+
const updated = yield* sql `
|
|
585
|
+
UPDATE orchestrator_dispatches SET status = 'running'
|
|
586
|
+
WHERE dispatch_request_id = ${decoded.dispatchRequestId} AND status = 'queued'
|
|
587
|
+
RETURNING dispatch_request_id
|
|
588
|
+
`.pipe(Effect.mapError(storageError("worker-start.update-dispatch")));
|
|
589
|
+
if (updated.length !== 1) {
|
|
590
|
+
return yield* new OrchestratorTransitionError({
|
|
591
|
+
dispatchRequestId: decoded.dispatchRequestId,
|
|
592
|
+
from: dispatch.status,
|
|
593
|
+
to: "running"
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
return yield* Schema.decodeUnknownEffect(OrchestratorDispatchActivated)({ binding, event }).pipe(Effect.mapError(storageError("worker-start.decode")));
|
|
597
|
+
})).pipe(Effect.catchTag("SqlError", (cause) => Effect.fail(storageError("worker-start.transaction")(cause))));
|
|
598
|
+
});
|
|
599
|
+
const submitInternal = Effect.fn("Orchestrator.submitInternal")(function* (decodedCommand, decodedKey, route, workLink) {
|
|
600
|
+
const encodedCommand = JSON.stringify(decodedCommand);
|
|
601
|
+
const encodedRoute = route === null ? null : JSON.stringify(route);
|
|
602
|
+
const encodedWorkLink = workLink === null ? null : JSON.stringify(workLink);
|
|
603
|
+
yield* secureFiles;
|
|
604
|
+
return yield* sql.withTransaction(Effect.gen(function* () {
|
|
605
|
+
const replayIds = yield* sql `
|
|
606
|
+
SELECT dispatch_request_id AS "dispatchRequestId"
|
|
607
|
+
FROM orchestrator_dispatches
|
|
608
|
+
WHERE idempotency_key = ${decodedKey}
|
|
609
|
+
OR activity_idempotency_key = ${decodedCommand.activityIdempotencyKey}
|
|
610
|
+
ORDER BY dispatch_request_id ASC
|
|
611
|
+
LIMIT 1
|
|
612
|
+
`.pipe(Effect.mapError(storageError("submit.preload-replay")), Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({
|
|
613
|
+
dispatchRequestId: DispatchRequestId
|
|
614
|
+
})))), Effect.mapError(storageError("submit.decode-preloaded-replay")));
|
|
615
|
+
const replayId = replayIds[0]?.dispatchRequestId;
|
|
616
|
+
const dispatchRequestId = replayId ??
|
|
617
|
+
(yield* cryptoService.randomUUIDv4.pipe(Effect.mapError(storageError("submit.request-id")), Effect.flatMap((value) => Schema.decodeUnknownEffect(DispatchRequestId)(value).pipe(Effect.mapError(() => new OrchestratorStorageError({ cause: value, operation: "submit.request-id-schema" }))))));
|
|
618
|
+
yield* validateLinkedParent(route);
|
|
619
|
+
const acceptedAt = yield* now;
|
|
620
|
+
const insertedRows = yield* sql `
|
|
621
|
+
INSERT INTO orchestrator_dispatches
|
|
622
|
+
(dispatch_request_id, idempotency_key, activity_idempotency_key, command, accepted_at, is_routed, status)
|
|
623
|
+
VALUES (${dispatchRequestId}, ${decodedKey}, ${decodedCommand.activityIdempotencyKey},
|
|
624
|
+
${encodedCommand}, ${acceptedAt}, ${route === null ? 0 : 1}, 'accepted')
|
|
625
|
+
ON CONFLICT DO NOTHING
|
|
626
|
+
RETURNING dispatch_request_id
|
|
627
|
+
`.pipe(Effect.mapError(storageError("submit.insert-or-reload")));
|
|
628
|
+
const wasInserted = insertedRows.length !== 0;
|
|
629
|
+
if (!wasInserted) {
|
|
630
|
+
const persistedRows = yield* sql `
|
|
631
|
+
SELECT d.dispatch_request_id AS "dispatchRequestId", d.idempotency_key AS "idempotencyKey",
|
|
632
|
+
d.activity_idempotency_key AS "activityIdempotencyKey", d.command,
|
|
633
|
+
d.accepted_at AS "acceptedAt", d.is_routed AS "isRouted", d.status,
|
|
634
|
+
m.route, m.work_link AS "workLink"
|
|
635
|
+
FROM orchestrator_dispatches d
|
|
636
|
+
LEFT JOIN orchestrator_dispatch_metadata m USING (dispatch_request_id)
|
|
637
|
+
WHERE d.idempotency_key = ${decodedKey}
|
|
638
|
+
OR d.activity_idempotency_key = ${decodedCommand.activityIdempotencyKey}
|
|
639
|
+
`.pipe(Effect.mapError(storageError("submit.reload")));
|
|
640
|
+
const persisted = yield* Schema.decodeUnknownEffect(Schema.Array(Row))(persistedRows).pipe(Effect.mapError(storageError("submit.decode-persisted")));
|
|
641
|
+
const first = persisted[0];
|
|
642
|
+
if (first === undefined) {
|
|
643
|
+
return yield* new OrchestratorStorageError({
|
|
644
|
+
cause: { dispatchRequestId, idempotencyKey: decodedKey },
|
|
645
|
+
operation: "submit.insert-or-reload-empty"
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
if (first.idempotencyKey !== decodedKey) {
|
|
649
|
+
return yield* new OrchestratorConflictError({
|
|
650
|
+
detail: "activity idempotency key was already used",
|
|
651
|
+
idempotencyKey: decodedKey
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
const oldCommand = yield* decodeCommand(first.command);
|
|
655
|
+
if (!Equal.equals(oldCommand, decodedCommand)) {
|
|
656
|
+
return yield* new OrchestratorConflictError({
|
|
657
|
+
detail: "idempotency key was already used for a different command",
|
|
658
|
+
idempotencyKey: decodedKey
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
const acceptedAt = yield* Schema.decodeUnknownEffect(Schema.Number)(first.acceptedAt).pipe(Effect.mapError(storageError("submit.decode-accepted-at")));
|
|
662
|
+
const snapshot = yield* loadValidatedEvents(first.dispatchRequestId).pipe(Effect.catchTag("OrchestratorStorageError", (error) => Effect.fail(new OrchestratorStorageError({
|
|
663
|
+
cause: error.cause,
|
|
664
|
+
operation: "submit.accepted-event-mismatch"
|
|
665
|
+
}))));
|
|
666
|
+
if (!Equal.equals(snapshot.dispatch.route, route) || !Equal.equals(snapshot.dispatch.workLink, workLink)) {
|
|
667
|
+
return yield* new OrchestratorConflictError({
|
|
668
|
+
detail: "idempotency key was already used with different route or Work lineage",
|
|
669
|
+
idempotencyKey: decodedKey
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
const accepted = snapshot.events[0];
|
|
673
|
+
if (accepted === undefined ||
|
|
674
|
+
accepted.type !== "accepted" ||
|
|
675
|
+
accepted.dispatchRequestId !== first.dispatchRequestId ||
|
|
676
|
+
accepted.activityIdempotencyKey !== first.activityIdempotencyKey ||
|
|
677
|
+
accepted.activityIdempotencyKey !== decodedCommand.activityIdempotencyKey ||
|
|
678
|
+
accepted.occurredAt !== acceptedAt) {
|
|
679
|
+
return yield* new OrchestratorStorageError({
|
|
680
|
+
cause: {
|
|
681
|
+
acceptedAt,
|
|
682
|
+
acceptedEvent: accepted,
|
|
683
|
+
dispatchActivityIdempotencyKey: first.activityIdempotencyKey,
|
|
684
|
+
dispatchRequestId: first.dispatchRequestId
|
|
685
|
+
},
|
|
686
|
+
operation: "submit.accepted-event-mismatch"
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
if (route?.model === "gpt-5.6-sol" && workLink !== null) {
|
|
690
|
+
yield* workSql.requireDispatchHandoff({
|
|
691
|
+
dispatchRequestId: first.dispatchRequestId,
|
|
692
|
+
handoff: workLink.handoff,
|
|
693
|
+
lineage: workLink.lineage
|
|
694
|
+
}).pipe(Effect.mapError(storageError("submit.work-link.replay")));
|
|
695
|
+
}
|
|
696
|
+
return yield* Schema.decodeUnknownEffect(OrchestratorReceipt)({
|
|
697
|
+
acceptedAt,
|
|
698
|
+
dispatchRequestId: first.dispatchRequestId,
|
|
699
|
+
idempotencyKey: decodedKey,
|
|
700
|
+
status: "accepted"
|
|
701
|
+
}).pipe(Effect.mapError(storageError("submit.decode-receipt")));
|
|
702
|
+
}
|
|
703
|
+
if (route?.model === "gpt-5.6-sol" && workLink !== null) {
|
|
704
|
+
yield* workSql.acceptDispatchHandoff({
|
|
705
|
+
dispatchRequestId,
|
|
706
|
+
handoff: workLink.handoff,
|
|
707
|
+
lineage: workLink.lineage
|
|
708
|
+
}).pipe(Effect.mapError((error) => error._tag === "WorkDecisionRevisionConflictError"
|
|
709
|
+
? new OrchestratorWorkRevisionConflictError({
|
|
710
|
+
actualRevision: error.actualRevision,
|
|
711
|
+
expectedRevision: error.expectedRevision,
|
|
712
|
+
laneId: error.laneId
|
|
713
|
+
})
|
|
714
|
+
: storageError("submit.work-link")(error)));
|
|
715
|
+
}
|
|
716
|
+
if (route !== null) {
|
|
717
|
+
yield* sql `
|
|
718
|
+
INSERT INTO orchestrator_dispatch_metadata
|
|
719
|
+
(dispatch_request_id, route, work_link)
|
|
720
|
+
VALUES (${dispatchRequestId}, ${encodedRoute}, ${encodedWorkLink})
|
|
721
|
+
`.pipe(Effect.mapError(storageError("submit.insert-dispatch-metadata")));
|
|
722
|
+
}
|
|
723
|
+
yield* sql `
|
|
724
|
+
INSERT INTO orchestrator_events
|
|
725
|
+
(dispatch_request_id, sequence, type, activity_idempotency_key, occurred_at, detail, result)
|
|
726
|
+
VALUES (${dispatchRequestId}, 0, 'accepted', ${decodedCommand.activityIdempotencyKey}, ${acceptedAt}, NULL, NULL)
|
|
727
|
+
`.pipe(Effect.mapError(storageError("submit.insert-event")));
|
|
728
|
+
return {
|
|
729
|
+
acceptedAt,
|
|
730
|
+
dispatchRequestId,
|
|
731
|
+
idempotencyKey: decodedKey,
|
|
732
|
+
status: "accepted"
|
|
733
|
+
};
|
|
734
|
+
})).pipe(Effect.catchTag("SqlError", (cause) => Effect.fail(storageError("submit.transaction")(cause))));
|
|
735
|
+
});
|
|
736
|
+
const submit = Effect.fn("Orchestrator.submit")(function* (command, idempotencyKey) {
|
|
737
|
+
const decodedCommand = yield* Schema.decodeUnknownEffect(OrchestratorCommand)(command).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "command is not a typed fleet job" })));
|
|
738
|
+
const decodedKey = yield* Schema.decodeUnknownEffect(OrchestratorIdempotencyKey)(idempotencyKey).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "idempotency key is invalid" })));
|
|
739
|
+
return yield* submitInternal(decodedCommand, decodedKey, null, null);
|
|
740
|
+
});
|
|
741
|
+
const submitRouted = Effect.fn("Orchestrator.submitRouted")(function* (input) {
|
|
742
|
+
const decoded = yield* Schema.decodeUnknownEffect(OrchestratorRoutedSubmission)(input).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "routed submission is invalid" })));
|
|
743
|
+
const command = yield* Schema.decodeUnknownEffect(OrchestratorCommand)(decoded.command).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "routed command is invalid" })));
|
|
744
|
+
if (decoded.route.model === "gpt-5.6-sol" &&
|
|
745
|
+
decoded.route.linkedRequestId !== null &&
|
|
746
|
+
decoded.workLink !== null &&
|
|
747
|
+
!decoded.workLink.lineage.includes(decoded.route.linkedRequestId)) {
|
|
748
|
+
return yield* new OrchestratorValidationError({
|
|
749
|
+
detail: "Work lineage must include the linked Luna request"
|
|
750
|
+
});
|
|
751
|
+
}
|
|
752
|
+
return yield* submitInternal(command, decoded.idempotencyKey, decoded.route, decoded.workLink);
|
|
753
|
+
});
|
|
754
|
+
const submitSolEscalation = Effect.fn("Orchestrator.submitSolEscalation")(function* (input) {
|
|
755
|
+
const decoded = yield* Schema.decodeUnknownEffect(OrchestratorSolEscalationSubmission)(input).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "Sol escalation submission is invalid" })));
|
|
756
|
+
return yield* submitInternal({
|
|
757
|
+
activityIdempotencyKey: decoded.command.activityIdempotencyKey,
|
|
758
|
+
actor: decoded.command.actor,
|
|
759
|
+
kind: decoded.command.kind,
|
|
760
|
+
payload: {
|
|
761
|
+
kind: decoded.command.payload.kind,
|
|
762
|
+
mode: decoded.command.payload.mode,
|
|
763
|
+
prompt: decoded.command.payload.prompt,
|
|
764
|
+
repository: decoded.command.payload.repository
|
|
765
|
+
}
|
|
766
|
+
}, decoded.idempotencyKey, {
|
|
767
|
+
action: "dispatch",
|
|
768
|
+
linkedRequestId: decoded.reference.failedLunaRequestId,
|
|
769
|
+
model: "gpt-5.6-sol",
|
|
770
|
+
protocol: "hostd.coordinator.route.v1",
|
|
771
|
+
reason: decoded.reason,
|
|
772
|
+
reasoningEffort: "high"
|
|
773
|
+
}, decoded.reference.workLink);
|
|
774
|
+
});
|
|
775
|
+
const service = {
|
|
776
|
+
events: (dispatchRequestId) => Stream.unwrap(Schema.decodeUnknownEffect(DispatchRequestId)(dispatchRequestId).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "dispatch request ID is invalid" })), Effect.map((decodedDispatchRequestId) => Stream.succeed(undefined).pipe(Stream.concat(Stream.fromSchedule(Schedule.spaced("100 millis")).pipe(Stream.map(() => undefined))), Stream.mapEffect(() => sql.withTransaction(Effect.gen(function* () {
|
|
777
|
+
const snapshot = yield* loadValidatedEvents(decodedDispatchRequestId);
|
|
778
|
+
yield* validateDurableReadback(snapshot.dispatch, "events.work-link");
|
|
779
|
+
return snapshot;
|
|
780
|
+
})).pipe(Effect.catchTag("SqlError", (cause) => Effect.fail(storageError("events.transaction")(cause))))), Stream.mapAccum(() => -1, (lastSequence, snapshot) => {
|
|
781
|
+
const unseen = snapshot.events.filter(({ sequence }) => sequence > lastSequence);
|
|
782
|
+
const last = unseen.at(-1);
|
|
783
|
+
return [last?.sequence ?? lastSequence, unseen];
|
|
784
|
+
}), Stream.takeUntil(isTerminalEvent))))),
|
|
785
|
+
failDelivery: Effect.fn("Orchestrator.failDelivery")((dispatchRequestId, detail) => appendTransition(dispatchRequestId, "delivery_failed", detail, null)),
|
|
786
|
+
failTask: Effect.fn("Orchestrator.failTask")((dispatchRequestId, detail) => appendTransition(dispatchRequestId, "task_failed", detail, null)),
|
|
787
|
+
pending: listPending,
|
|
788
|
+
queue: Effect.fn("Orchestrator.queue")((dispatchRequestId) => appendTransition(dispatchRequestId, "queued", null, null)),
|
|
789
|
+
recover: () => Stream.paginate(undefined, Effect.fn("Orchestrator.recoverPage")(function* (state) {
|
|
790
|
+
void state;
|
|
791
|
+
const rows = yield* sql `
|
|
792
|
+
SELECT dispatch_request_id AS "dispatchRequestId", accepted_at AS "acceptedAt"
|
|
793
|
+
FROM orchestrator_dispatches
|
|
794
|
+
WHERE status = 'running'
|
|
795
|
+
ORDER BY accepted_at ASC, dispatch_request_id ASC
|
|
796
|
+
LIMIT ${recoveryPageSize}
|
|
797
|
+
`.pipe(Effect.mapError(storageError("recover.lookup")));
|
|
798
|
+
const ids = yield* Schema.decodeUnknownEffect(Schema.Array(RecoveryRow))(rows).pipe(Effect.mapError(storageError("recover.decode")));
|
|
799
|
+
const recovered = yield* Effect.forEach(ids, ({ dispatchRequestId }) => appendTransition(dispatchRequestId, "delivery_failed", "orchestrator restarted while the activity was running", null).pipe(Effect.map(Option.some), Effect.catchTag("OrchestratorTransitionError", (error) => error.from === "settled" || error.from === "delivery_failed" || error.from === "task_failed"
|
|
800
|
+
? Effect.succeed(Option.none())
|
|
801
|
+
: Effect.fail(error))));
|
|
802
|
+
const events = recovered.flatMap((event) => Option.match(event, {
|
|
803
|
+
onNone: () => [],
|
|
804
|
+
onSome: (value) => [value]
|
|
805
|
+
}));
|
|
806
|
+
const next = ids.length === 0 ? Option.none() : Option.some(undefined);
|
|
807
|
+
const page = [events, next];
|
|
808
|
+
return page;
|
|
809
|
+
})),
|
|
810
|
+
run: Effect.fn("Orchestrator.run")((dispatchRequestId) => appendTransition(dispatchRequestId, "running", null, null)),
|
|
811
|
+
settle: Effect.fn("Orchestrator.settle")((dispatchRequestId, result) => appendTransition(dispatchRequestId, "settled", null, result)),
|
|
812
|
+
submit,
|
|
813
|
+
submitRouted,
|
|
814
|
+
submitSolEscalation,
|
|
815
|
+
workerStarted,
|
|
816
|
+
request: Effect.fn("Orchestrator.request")((dispatchRequestId) => Schema.decodeUnknownEffect(DispatchRequestId)(dispatchRequestId).pipe(Effect.mapError(() => new OrchestratorValidationError({ detail: "dispatch request ID is invalid" })), Effect.flatMap(loadValidatedEvents), Effect.flatMap((snapshot) => Effect.gen(function* () {
|
|
817
|
+
yield* validateDurableReadback(snapshot.dispatch, "request.work-link");
|
|
818
|
+
const command = yield* decodeCommand(snapshot.dispatch.command);
|
|
819
|
+
return yield* Schema.decodeUnknownEffect(OrchestratorRequest)({
|
|
820
|
+
...snapshot.dispatch,
|
|
821
|
+
command
|
|
822
|
+
}).pipe(Effect.mapError(storageError("request.decode")));
|
|
823
|
+
}))))
|
|
824
|
+
};
|
|
825
|
+
return service;
|
|
826
|
+
});
|
|
827
|
+
const orchestratorLayer = Layer.effect(Orchestrator, makeOrchestrator);
|
|
828
|
+
/**
|
|
829
|
+
* Node SQLite and Crypto services used by the durable coordinator layers.
|
|
830
|
+
* Fails closed on non-POSIX paths until an ACL-backed private-directory check exists.
|
|
831
|
+
*/
|
|
832
|
+
export const sqliteLayer = (filename) => orchestratorLayer.pipe(Layer.provide(Layer.unwrap(Effect.gen(function* () {
|
|
833
|
+
const fileSystem = yield* FileSystem.FileSystem;
|
|
834
|
+
const paths = yield* Path.Path;
|
|
835
|
+
const directory = paths.dirname(filename);
|
|
836
|
+
const directoryExists = yield* fileSystem.exists(directory).pipe(Effect.mapError((cause) => new OrchestratorStorageError({ cause, operation: "sqlite.secure.directory.exists" })));
|
|
837
|
+
if (!directoryExists) {
|
|
838
|
+
yield* fileSystem.makeDirectory(directory, { recursive: true, mode: 0o700 }).pipe(Effect.mapError((cause) => new OrchestratorStorageError({ cause, operation: "sqlite.secure.directory.create" })));
|
|
839
|
+
}
|
|
840
|
+
const directoryInfo = yield* fileSystem.stat(directory).pipe(Effect.mapError((cause) => new OrchestratorStorageError({ cause, operation: "sqlite.secure.directory.stat" })));
|
|
841
|
+
const privateMode = paths.sep === "/" && (directoryInfo.mode & 0o777) === 0o700;
|
|
842
|
+
if (directoryInfo.type !== "Directory" || !privateMode) {
|
|
843
|
+
return yield* new OrchestratorStorageError({
|
|
844
|
+
cause: { directory, mode: directoryInfo.mode, type: directoryInfo.type },
|
|
845
|
+
operation: "sqlite.secure.directory.private"
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
yield* verifyPathIdentity(directory, fileSystem, paths, "sqlite.secure.directory.path-identity");
|
|
849
|
+
yield* Effect.forEach([filename, `${filename}-wal`, `${filename}-shm`], (path) => verifyPathIdentity(path, fileSystem, paths, "sqlite.secure.path-identity"), { discard: true });
|
|
850
|
+
const security = secureSqliteFiles(filename, fileSystem, paths);
|
|
851
|
+
return Layer.mergeAll(SqliteClient.layer({ filename }), NodeCrypto.layer, Layer.succeed(SqliteFileSecurity, { secure: security })).pipe(Layer.tap(() => security));
|
|
852
|
+
})).pipe(Layer.provide(Layer.mergeAll(NodeFileSystem.layer, NodePath.layer)))));
|
|
853
|
+
//# sourceMappingURL=orchestrator.js.map
|