@triagepilot/db 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/dist/availability.d.ts +164 -0
- package/dist/availability.js +974 -0
- package/dist/database.d.ts +3 -0
- package/dist/database.js +9 -0
- package/dist/decisions.d.ts +107 -0
- package/dist/decisions.js +435 -0
- package/dist/deliveries.d.ts +30 -0
- package/dist/deliveries.js +82 -0
- package/dist/heartbeat.d.ts +11 -0
- package/dist/heartbeat.js +11 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +13 -0
- package/dist/jobs.d.ts +86 -0
- package/dist/jobs.js +641 -0
- package/dist/kysely.d.ts +202 -0
- package/dist/kysely.js +1 -0
- package/dist/migrate.d.ts +1 -0
- package/dist/migrate.js +45 -0
- package/dist/operations.d.ts +77 -0
- package/dist/operations.js +256 -0
- package/dist/outbox.d.ts +61 -0
- package/dist/outbox.js +230 -0
- package/dist/provider-connections.d.ts +34 -0
- package/dist/provider-connections.js +302 -0
- package/dist/retention.d.ts +6 -0
- package/dist/retention.js +44 -0
- package/dist/routing-recovery.d.ts +55 -0
- package/dist/routing-recovery.js +175 -0
- package/dist/workspaces.d.ts +50 -0
- package/dist/workspaces.js +43 -0
- package/migrations/0001_initial.sql +79 -0
- package/migrations/0002_selected_reviewers.sql +10 -0
- package/migrations/0003_human_review_policy.sql +10 -0
- package/migrations/0004_semantic_routing_deduplication.sql +15 -0
- package/migrations/0005_reviewer_availability.sql +58 -0
- package/migrations/0005_workspace_scope.sql +198 -0
- package/migrations/0006_decision_outbox.sql +24 -0
- package/migrations/0007_workspace_reviewer_availability.sql +192 -0
- package/migrations/0008_reviewer_mutation_intents.sql +160 -0
- package/migrations/0009_provider_connection_revocations.sql +18 -0
- package/migrations/0010_provider_connection_preemptive_revocations.sql +12 -0
- package/package.json +42 -0
package/dist/jobs.js
ADDED
|
@@ -0,0 +1,641 @@
|
|
|
1
|
+
import { sql } from "kysely";
|
|
2
|
+
import { persistMutationIntentRecoveryTransaction, prepareMutationIntentTransaction, } from "./availability.js";
|
|
3
|
+
export class ReviewerMutationLeaseUnavailableError extends Error {
|
|
4
|
+
}
|
|
5
|
+
export function buildNextRunAt(now, attemptCount) {
|
|
6
|
+
const delaySeconds = Math.min(900, 5 ** Math.max(1, attemptCount));
|
|
7
|
+
return new Date(now.getTime() + delaySeconds * 1000);
|
|
8
|
+
}
|
|
9
|
+
export async function prepareClaimedReviewerMutationIntent(db, lease, input) {
|
|
10
|
+
return await withClaimedReviewerMutationLeaseTransaction(db, lease, input, async (trx) => {
|
|
11
|
+
return await prepareMutationIntentTransaction(trx, lease.workspaceId, input);
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
export async function runClaimedReviewerProviderMutation(db, lease, scope, mutation, options = {}) {
|
|
15
|
+
// The bounded transaction serializes lease/admin changes with provider writes. Aborting cannot recall a request
|
|
16
|
+
// already accepted by the provider, so the immutable intent and idempotent reconciliation remain the replay fence.
|
|
17
|
+
const timeoutMs = options.timeoutMs ?? 60_000;
|
|
18
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
|
|
19
|
+
throw new Error("reviewer provider mutation timeout must be a positive integer");
|
|
20
|
+
}
|
|
21
|
+
const controller = new AbortController();
|
|
22
|
+
const deadlineError = new ReviewerMutationLeaseUnavailableError("reviewer provider mutation authority deadline expired");
|
|
23
|
+
let providerError = noProviderError;
|
|
24
|
+
const transaction = db.transaction().execute(async (trx) => {
|
|
25
|
+
await sql `select set_config('lock_timeout', ${`${timeoutMs}ms`}, true)`.execute(trx);
|
|
26
|
+
await sql `select set_config('idle_in_transaction_session_timeout', ${`${timeoutMs + 1_000}ms`}, true)`.execute(trx);
|
|
27
|
+
await assertClaimedReviewerMutationAuthority(trx, lease, scope, true, controller.signal);
|
|
28
|
+
let result;
|
|
29
|
+
try {
|
|
30
|
+
const providerMutation = mutation({
|
|
31
|
+
signal: controller.signal,
|
|
32
|
+
assertActive: async () => {
|
|
33
|
+
await assertClaimedReviewerMutationAuthority(trx, lease, scope, false, controller.signal);
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
void providerMutation.catch(() => undefined);
|
|
37
|
+
result = await waitForReviewerMutationAuthority(providerMutation, controller.signal);
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
providerError = error;
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
await assertClaimedReviewerMutationAuthority(trx, lease, scope, false, controller.signal);
|
|
44
|
+
return result;
|
|
45
|
+
});
|
|
46
|
+
void transaction.catch(() => undefined);
|
|
47
|
+
let timeout;
|
|
48
|
+
const deadline = new Promise((_resolve, reject) => {
|
|
49
|
+
timeout = setTimeout(() => {
|
|
50
|
+
controller.abort(deadlineError);
|
|
51
|
+
reject(deadlineError);
|
|
52
|
+
}, timeoutMs);
|
|
53
|
+
});
|
|
54
|
+
try {
|
|
55
|
+
return await Promise.race([transaction, deadline]);
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
if (error instanceof ReviewerMutationLeaseUnavailableError)
|
|
59
|
+
throw error;
|
|
60
|
+
if (providerError !== noProviderError && error === providerError)
|
|
61
|
+
throw error;
|
|
62
|
+
throw new ReviewerMutationLeaseUnavailableError(`reviewer provider mutation authority was lost: ${error instanceof Error ? error.message : "unknown database error"}`);
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
if (timeout !== undefined)
|
|
66
|
+
clearTimeout(timeout);
|
|
67
|
+
controller.abort(new ReviewerMutationLeaseUnavailableError("reviewer provider mutation authority ended"));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const noProviderError = Symbol("no-provider-error");
|
|
71
|
+
async function waitForReviewerMutationAuthority(operation, signal) {
|
|
72
|
+
signal.throwIfAborted();
|
|
73
|
+
return await new Promise((resolve, reject) => {
|
|
74
|
+
const abort = () => reject(signal.reason);
|
|
75
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
76
|
+
void operation.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
async function assertClaimedReviewerMutationAuthority(trx, lease, scope, lock, signal) {
|
|
80
|
+
signal.throwIfAborted();
|
|
81
|
+
let jobQuery = trx.selectFrom("jobs").selectAll()
|
|
82
|
+
.where("id", "=", lease.jobId).where("workspace_id", "=", lease.workspaceId)
|
|
83
|
+
.where("provider", "=", lease.provider).where("provider_connection_id", "=", lease.providerConnectionId)
|
|
84
|
+
.where("status", "=", "running").where("locked_by", "=", lease.lockedBy)
|
|
85
|
+
.where("locked_at", "=", lease.lockedAt).where("attempt_count", "=", lease.attemptCount);
|
|
86
|
+
if (lock)
|
|
87
|
+
jobQuery = jobQuery.forUpdate();
|
|
88
|
+
const job = await jobQuery.executeTakeFirst();
|
|
89
|
+
const activation = job === undefined ? null : parseActivationScope(job);
|
|
90
|
+
if (activation === null || activation.absenceId !== scope.absenceId
|
|
91
|
+
|| activation.absenceRevision !== scope.absenceRevision || scope.workspaceId !== lease.workspaceId
|
|
92
|
+
|| scope.provider !== lease.provider || scope.providerConnectionId !== lease.providerConnectionId) {
|
|
93
|
+
throw new ReviewerMutationLeaseUnavailableError("reviewer provider mutation rejected stale or invalid activation lease");
|
|
94
|
+
}
|
|
95
|
+
signal.throwIfAborted();
|
|
96
|
+
let absenceQuery = trx.selectFrom("reviewer_absences").select(["revision", "status"])
|
|
97
|
+
.where("workspace_id", "=", scope.workspaceId).where("provider", "=", scope.provider)
|
|
98
|
+
.where("provider_connection_id", "=", scope.providerConnectionId).where("id", "=", scope.absenceId);
|
|
99
|
+
if (lock)
|
|
100
|
+
absenceQuery = absenceQuery.forUpdate();
|
|
101
|
+
const absence = await absenceQuery.executeTakeFirst();
|
|
102
|
+
if (absence === undefined || absence.revision !== scope.absenceRevision || absence.status !== "scheduled") {
|
|
103
|
+
throw new ReviewerMutationLeaseUnavailableError("reviewer provider mutation rejected obsolete absence authority");
|
|
104
|
+
}
|
|
105
|
+
signal.throwIfAborted();
|
|
106
|
+
let connectionQuery = trx.selectFrom("provider_connections").select("status")
|
|
107
|
+
.where("workspace_id", "=", scope.workspaceId).where("provider", "=", scope.provider)
|
|
108
|
+
.where("id", "=", scope.providerConnectionId);
|
|
109
|
+
if (lock)
|
|
110
|
+
connectionQuery = connectionQuery.forUpdate();
|
|
111
|
+
const connection = await connectionQuery.executeTakeFirst();
|
|
112
|
+
if (connection?.status !== "active") {
|
|
113
|
+
throw new ReviewerMutationLeaseUnavailableError("reviewer provider mutation rejected inactive provider connection authority");
|
|
114
|
+
}
|
|
115
|
+
signal.throwIfAborted();
|
|
116
|
+
}
|
|
117
|
+
async function withClaimedReviewerMutationLeaseTransaction(db, lease, scope, operation) {
|
|
118
|
+
return await db.transaction().execute(async (trx) => {
|
|
119
|
+
const job = await trx.selectFrom("jobs").selectAll()
|
|
120
|
+
.where("id", "=", lease.jobId).where("workspace_id", "=", lease.workspaceId)
|
|
121
|
+
.where("provider", "=", lease.provider).where("provider_connection_id", "=", lease.providerConnectionId)
|
|
122
|
+
.where("status", "=", "running").where("locked_by", "=", lease.lockedBy)
|
|
123
|
+
.where("locked_at", "=", lease.lockedAt).where("attempt_count", "=", lease.attemptCount)
|
|
124
|
+
.forUpdate().executeTakeFirst();
|
|
125
|
+
const activation = job === undefined ? null : parseActivationScope(job);
|
|
126
|
+
if (activation === null || activation.absenceId !== scope.absenceId
|
|
127
|
+
|| activation.absenceRevision !== scope.absenceRevision || scope.workspaceId !== lease.workspaceId
|
|
128
|
+
|| scope.provider !== lease.provider || scope.providerConnectionId !== lease.providerConnectionId) {
|
|
129
|
+
throw new ReviewerMutationLeaseUnavailableError("reviewer provider mutation rejected stale or invalid activation lease");
|
|
130
|
+
}
|
|
131
|
+
return await operation(trx);
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
export async function recoverStaleJobs(db, workspaceId, now, staleAfterMs = 15 * 60 * 1000) {
|
|
135
|
+
const staleBefore = new Date(now.getTime() - staleAfterMs);
|
|
136
|
+
const exhausted = await db.selectFrom("jobs")
|
|
137
|
+
.selectAll()
|
|
138
|
+
.where("workspace_id", "=", workspaceId)
|
|
139
|
+
.where("status", "=", "running")
|
|
140
|
+
.where("locked_at", "<", staleBefore)
|
|
141
|
+
.whereRef("attempt_count", ">=", "max_attempts")
|
|
142
|
+
.orderBy("created_at").orderBy("id")
|
|
143
|
+
.execute();
|
|
144
|
+
const failure = "job lease expired after maximum attempts";
|
|
145
|
+
for (const job of exhausted) {
|
|
146
|
+
if (job.locked_by === null)
|
|
147
|
+
continue;
|
|
148
|
+
const lease = {
|
|
149
|
+
jobId: job.id,
|
|
150
|
+
workspaceId: job.workspace_id,
|
|
151
|
+
provider: job.provider,
|
|
152
|
+
providerConnectionId: job.provider_connection_id,
|
|
153
|
+
lockedBy: job.locked_by,
|
|
154
|
+
lockedAt: job.locked_at,
|
|
155
|
+
attemptCount: job.attempt_count,
|
|
156
|
+
maxAttempts: job.max_attempts,
|
|
157
|
+
};
|
|
158
|
+
try {
|
|
159
|
+
await exhaustReviewerAbsenceActivationTransaction(db, lease, failure, now);
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
const detail = error instanceof Error ? error.message : "unknown database failure";
|
|
163
|
+
try {
|
|
164
|
+
await failIsolatedStaleJob(db, lease, `${failure}: recovery transaction failed: ${detail}`, now);
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Isolation is the invariant: a broken job must not abort recovery of later stale jobs.
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
await db.updateTable("jobs")
|
|
172
|
+
.set({
|
|
173
|
+
status: "queued",
|
|
174
|
+
locked_at: null,
|
|
175
|
+
locked_by: null,
|
|
176
|
+
updated_at: now,
|
|
177
|
+
})
|
|
178
|
+
.where("workspace_id", "=", workspaceId)
|
|
179
|
+
.where("status", "=", "running")
|
|
180
|
+
.where("locked_at", "<", staleBefore)
|
|
181
|
+
.whereRef("attempt_count", "<", "max_attempts")
|
|
182
|
+
.execute();
|
|
183
|
+
}
|
|
184
|
+
async function failIsolatedStaleJob(db, lease, error, now) {
|
|
185
|
+
await db.updateTable("jobs").set({
|
|
186
|
+
status: "failed", locked_at: null, locked_by: null, last_error: error, run_at: now, updated_at: now,
|
|
187
|
+
}).where("id", "=", lease.jobId).where("workspace_id", "=", lease.workspaceId)
|
|
188
|
+
.where("provider", "=", lease.provider).where("provider_connection_id", "=", lease.providerConnectionId)
|
|
189
|
+
.where("status", "=", "running").where("locked_by", "=", lease.lockedBy)
|
|
190
|
+
.where("locked_at", "=", lease.lockedAt).where("attempt_count", "=", lease.attemptCount)
|
|
191
|
+
.execute();
|
|
192
|
+
}
|
|
193
|
+
async function exhaustReviewerAbsenceActivationTransaction(db, lease, error, now) {
|
|
194
|
+
return await db.transaction().execute(async (trx) => {
|
|
195
|
+
const job = await trx.selectFrom("jobs").selectAll()
|
|
196
|
+
.where("id", "=", lease.jobId)
|
|
197
|
+
.where("workspace_id", "=", lease.workspaceId)
|
|
198
|
+
.where("provider", "=", lease.provider)
|
|
199
|
+
.where("provider_connection_id", "=", lease.providerConnectionId)
|
|
200
|
+
.where("status", "=", "running")
|
|
201
|
+
.where("locked_by", "=", lease.lockedBy)
|
|
202
|
+
.where("locked_at", "=", lease.lockedAt)
|
|
203
|
+
.where("attempt_count", "=", lease.attemptCount)
|
|
204
|
+
.forUpdate().executeTakeFirst();
|
|
205
|
+
if (job === undefined)
|
|
206
|
+
return { updated: false, reason: "stale_lease" };
|
|
207
|
+
const scope = parseActivationScope(job);
|
|
208
|
+
let sourceValid = scope !== null || job.kind !== "activate_reviewer_absence";
|
|
209
|
+
let sourceError = null;
|
|
210
|
+
if (scope !== null && isRecord(job.payload) && job.payload.reviewerReplacementFinalizerRecovery !== undefined) {
|
|
211
|
+
sourceValid = isValidRecoveryShape(job.payload.reviewerReplacementFinalizerRecovery, job, scope);
|
|
212
|
+
if (!sourceValid)
|
|
213
|
+
sourceError = "reviewer activation recovery source is invalid";
|
|
214
|
+
}
|
|
215
|
+
if (scope !== null && sourceValid) {
|
|
216
|
+
const absence = await trx.selectFrom("reviewer_absences").select("id")
|
|
217
|
+
.where("workspace_id", "=", job.workspace_id).where("provider", "=", job.provider)
|
|
218
|
+
.where("provider_connection_id", "=", job.provider_connection_id)
|
|
219
|
+
.where("id", "=", scope.absenceId).forUpdate().executeTakeFirst();
|
|
220
|
+
if (absence === undefined) {
|
|
221
|
+
sourceValid = false;
|
|
222
|
+
sourceError = "reviewer activation absence source is invalid";
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (scope !== null && sourceValid && isRecord(job.payload)
|
|
226
|
+
&& job.payload.reviewerReplacementFinalizerRecovery !== undefined) {
|
|
227
|
+
sourceValid = await recoverySourceMatches(trx, job, scope);
|
|
228
|
+
if (!sourceValid)
|
|
229
|
+
sourceError = "reviewer activation recovery source is invalid";
|
|
230
|
+
}
|
|
231
|
+
if (scope !== null && sourceValid) {
|
|
232
|
+
const intents = await trx.selectFrom("reviewer_mutation_intents").selectAll()
|
|
233
|
+
.where("workspace_id", "=", job.workspace_id)
|
|
234
|
+
.where("provider", "=", job.provider)
|
|
235
|
+
.where("provider_connection_id", "=", job.provider_connection_id)
|
|
236
|
+
.where("absence_id", "=", scope.absenceId)
|
|
237
|
+
.where("absence_revision", "=", scope.absenceRevision)
|
|
238
|
+
.orderBy("decision_id").orderBy("id").execute();
|
|
239
|
+
const unresolved = [];
|
|
240
|
+
for (const intent of intents) {
|
|
241
|
+
const history = await trx.selectFrom("reviewer_replacements").selectAll()
|
|
242
|
+
.where("workspace_id", "=", job.workspace_id)
|
|
243
|
+
.where("provider", "=", job.provider)
|
|
244
|
+
.where("provider_connection_id", "=", job.provider_connection_id)
|
|
245
|
+
.where("absence_id", "=", scope.absenceId)
|
|
246
|
+
.where("absence_revision", "=", scope.absenceRevision)
|
|
247
|
+
.where("decision_id", "=", intent.decision_id)
|
|
248
|
+
.forUpdate().executeTakeFirst();
|
|
249
|
+
if (history !== undefined) {
|
|
250
|
+
if (history.mutation_intent_id !== intent.id) {
|
|
251
|
+
sourceValid = false;
|
|
252
|
+
sourceError = "reviewer mutation intent history linkage is invalid";
|
|
253
|
+
}
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
unresolved.push(intent);
|
|
257
|
+
}
|
|
258
|
+
for (const intent of sourceValid ? unresolved : []) {
|
|
259
|
+
await persistMutationIntentRecoveryTransaction(trx, job.workspace_id, {
|
|
260
|
+
provider: intent.provider,
|
|
261
|
+
providerConnectionId: intent.provider_connection_id,
|
|
262
|
+
absenceId: intent.absence_id,
|
|
263
|
+
absenceRevision: intent.absence_revision,
|
|
264
|
+
decisionId: intent.decision_id,
|
|
265
|
+
expectedHeadRevision: intent.expected_head_revision,
|
|
266
|
+
unavailableActorId: intent.unavailable_actor_id,
|
|
267
|
+
replacementActorId: null,
|
|
268
|
+
mutationIntentId: intent.id,
|
|
269
|
+
outcome: "permanent_failure",
|
|
270
|
+
reason: error,
|
|
271
|
+
state: "permanent_failure",
|
|
272
|
+
lastError: error,
|
|
273
|
+
startedAt: now,
|
|
274
|
+
completedAt: now,
|
|
275
|
+
replaceCohort: false,
|
|
276
|
+
event: {
|
|
277
|
+
schemaVersion: 1,
|
|
278
|
+
eventType: "reviewer_replacement",
|
|
279
|
+
eventId: `reviewer-mutation-intent-exhausted:${intent.id}`,
|
|
280
|
+
occurredAt: now.toISOString(),
|
|
281
|
+
workspaceId: job.workspace_id,
|
|
282
|
+
provider: intent.provider,
|
|
283
|
+
providerConnectionId: intent.provider_connection_id,
|
|
284
|
+
absenceId: intent.absence_id,
|
|
285
|
+
absenceRevision: intent.absence_revision,
|
|
286
|
+
decisionId: intent.decision_id,
|
|
287
|
+
repositoryId: intent.repository_id,
|
|
288
|
+
changeRequestId: intent.change_request_id,
|
|
289
|
+
unavailableActor: intent.unavailable_actor_id,
|
|
290
|
+
replacementActor: null,
|
|
291
|
+
outcome: "permanent_failure",
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
if (sourceValid) {
|
|
296
|
+
const pending = await trx.selectFrom("reviewer_replacements").select("id")
|
|
297
|
+
.where("workspace_id", "=", job.workspace_id)
|
|
298
|
+
.where("provider", "=", job.provider)
|
|
299
|
+
.where("provider_connection_id", "=", job.provider_connection_id)
|
|
300
|
+
.where("absence_id", "=", scope.absenceId)
|
|
301
|
+
.where("absence_revision", "=", scope.absenceRevision)
|
|
302
|
+
.where("state", "=", "finalizer_pending")
|
|
303
|
+
.orderBy("decision_id").orderBy("id").forUpdate().execute();
|
|
304
|
+
for (const row of pending) {
|
|
305
|
+
await trx.updateTable("reviewer_replacements")
|
|
306
|
+
.set({ state: "permanent_failure", last_error: error })
|
|
307
|
+
.where("id", "=", row.id).where("state", "=", "finalizer_pending").execute();
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
const finalError = sourceError === null ? error : `${error}: ${sourceError}`;
|
|
312
|
+
await trx.updateTable("jobs").set({
|
|
313
|
+
status: "failed", run_at: now, locked_at: null, locked_by: null,
|
|
314
|
+
last_error: finalError, updated_at: now,
|
|
315
|
+
}).where("id", "=", job.id).execute();
|
|
316
|
+
return { updated: true };
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
function parseActivationScope(job) {
|
|
320
|
+
if (job.kind !== "activate_reviewer_absence" || !isRecord(job.payload))
|
|
321
|
+
return null;
|
|
322
|
+
if (job.payload.kind !== "activate_reviewer_absence"
|
|
323
|
+
|| !hasOnlyKeys(job.payload, ["kind", "workspaceId", "providerConnectionId", "absenceId", "absenceRevision",
|
|
324
|
+
"reviewerReplacementFinalizerRecovery"])
|
|
325
|
+
|| job.payload.policyCheckFailureRecovery !== undefined
|
|
326
|
+
|| job.payload.workspaceId !== job.workspace_id
|
|
327
|
+
|| job.payload.providerConnectionId !== job.provider_connection_id
|
|
328
|
+
|| !isUuid(job.payload.absenceId)
|
|
329
|
+
|| !isPositiveInteger(job.payload.absenceRevision))
|
|
330
|
+
return null;
|
|
331
|
+
return { absenceId: job.payload.absenceId, absenceRevision: job.payload.absenceRevision };
|
|
332
|
+
}
|
|
333
|
+
function isValidRecoveryShape(value, job, scope) {
|
|
334
|
+
if (!isRecord(value) || !isRecord(value.job)
|
|
335
|
+
|| !hasExactKeys(value, ["kind", "phase", "job", "provider", "unavailableActorId", "lastError", "retryable",
|
|
336
|
+
"finalizer", "replacementId", "outcome", "replacementActorId", "mutationIntentId",
|
|
337
|
+
"providerEffectsApplied", "persistence"])
|
|
338
|
+
|| !hasExactKeys(value.job, ["kind", "workspaceId", "providerConnectionId", "absenceId", "absenceRevision"])
|
|
339
|
+
|| value.kind !== "reviewer_replacement_finalizer"
|
|
340
|
+
|| !["persist_replacement", "run_finalizer", "complete_replacement"].includes(String(value.phase))
|
|
341
|
+
|| value.provider !== job.provider || !isNonBlank(value.unavailableActorId)
|
|
342
|
+
|| !isNonBlank(value.lastError) || typeof value.retryable !== "boolean"
|
|
343
|
+
|| value.job.kind !== "activate_reviewer_absence"
|
|
344
|
+
|| value.job.workspaceId !== job.workspace_id
|
|
345
|
+
|| value.job.providerConnectionId !== job.provider_connection_id
|
|
346
|
+
|| value.job.absenceId !== scope.absenceId || value.job.absenceRevision !== scope.absenceRevision
|
|
347
|
+
|| !["replaced", "skipped_policy_satisfied", "no_replacement_available", "permanent_failure"].includes(String(value.outcome))) {
|
|
348
|
+
return false;
|
|
349
|
+
}
|
|
350
|
+
const outcome = value.outcome;
|
|
351
|
+
const expectedEffect = outcome === "replaced" || outcome === "permanent_failure";
|
|
352
|
+
if (value.providerEffectsApplied !== expectedEffect)
|
|
353
|
+
return false;
|
|
354
|
+
const expectedAction = outcome === "replaced" || outcome === "skipped_policy_satisfied"
|
|
355
|
+
? "reevaluate_policy" : outcome === "no_replacement_available" ? "fail_policy" : null;
|
|
356
|
+
if (expectedAction === null) {
|
|
357
|
+
if (value.finalizer !== null)
|
|
358
|
+
return false;
|
|
359
|
+
}
|
|
360
|
+
else if (!isRecord(value.finalizer) || value.finalizer.action !== expectedAction
|
|
361
|
+
|| !hasOnlyKeys(value.finalizer, ["action", "decisionId", "summary"])
|
|
362
|
+
|| !isUuid(value.finalizer.decisionId)
|
|
363
|
+
|| (expectedAction === "reevaluate_policy" && value.finalizer.summary !== null)
|
|
364
|
+
|| (expectedAction === "fail_policy" && !isNonBlank(value.finalizer.summary)))
|
|
365
|
+
return false;
|
|
366
|
+
if (value.phase === "persist_replacement") {
|
|
367
|
+
if (value.replacementId !== null || !isRecord(value.persistence))
|
|
368
|
+
return false;
|
|
369
|
+
}
|
|
370
|
+
else if (!isUuid(value.replacementId) || value.persistence !== null)
|
|
371
|
+
return false;
|
|
372
|
+
if (outcome === "permanent_failure") {
|
|
373
|
+
if (value.phase !== "persist_replacement" || !isUuid(value.mutationIntentId)
|
|
374
|
+
|| value.replacementActorId !== null || !isRecord(value.persistence))
|
|
375
|
+
return false;
|
|
376
|
+
}
|
|
377
|
+
else {
|
|
378
|
+
if (outcome === "replaced" && (!isNonBlank(value.replacementActorId) || !isUuid(value.mutationIntentId)))
|
|
379
|
+
return false;
|
|
380
|
+
if (outcome !== "replaced" && (value.replacementActorId !== null
|
|
381
|
+
|| (value.mutationIntentId !== null && !isUuid(value.mutationIntentId))))
|
|
382
|
+
return false;
|
|
383
|
+
}
|
|
384
|
+
return value.persistence === null || isValidRecoveryPersistence(value.persistence, value, job, scope);
|
|
385
|
+
}
|
|
386
|
+
function isValidRecoveryPersistence(persistence, recovery, job, scope) {
|
|
387
|
+
if (!hasExactKeys(persistence, ["provider", "providerConnectionId", "absenceId", "absenceRevision", "decisionId",
|
|
388
|
+
"expectedHeadRevision", "unavailableActorId", "replacementActorId", "mutationIntentId", "outcome", "reason",
|
|
389
|
+
"state", "lastError", "startedAt", "completedAt", "replaceCohort", "event"])
|
|
390
|
+
|| !isRecord(persistence.event) || !isNonBlank(persistence.decisionId)
|
|
391
|
+
|| !isNonBlank(persistence.expectedHeadRevision) || !isNonBlank(persistence.unavailableActorId)
|
|
392
|
+
|| !isNonBlank(persistence.reason) || persistence.provider !== job.provider
|
|
393
|
+
|| persistence.providerConnectionId !== job.provider_connection_id
|
|
394
|
+
|| persistence.absenceId !== scope.absenceId || persistence.absenceRevision !== scope.absenceRevision
|
|
395
|
+
|| persistence.unavailableActorId !== recovery.unavailableActorId
|
|
396
|
+
|| persistence.replacementActorId !== recovery.replacementActorId
|
|
397
|
+
|| persistence.mutationIntentId !== recovery.mutationIntentId || persistence.outcome !== recovery.outcome)
|
|
398
|
+
return false;
|
|
399
|
+
const startedAt = parseCanonicalRecoveryDate(persistence.startedAt);
|
|
400
|
+
const completedAt = parseCanonicalRecoveryDate(persistence.completedAt);
|
|
401
|
+
if (startedAt === null || completedAt === null || completedAt < startedAt)
|
|
402
|
+
return false;
|
|
403
|
+
const permanent = recovery.outcome === "permanent_failure";
|
|
404
|
+
if (persistence.state !== (permanent ? "permanent_failure" : "finalizer_pending")
|
|
405
|
+
|| (permanent ? !isNonBlank(persistence.lastError) : persistence.lastError !== null)
|
|
406
|
+
|| persistence.replaceCohort !== (recovery.outcome === "replaced"))
|
|
407
|
+
return false;
|
|
408
|
+
const event = persistence.event;
|
|
409
|
+
if (!hasExactKeys(event, ["schemaVersion", "eventType", "eventId", "occurredAt", "workspaceId", "provider",
|
|
410
|
+
"providerConnectionId", "absenceId", "absenceRevision", "decisionId", "repositoryId", "changeRequestId",
|
|
411
|
+
"unavailableActor", "replacementActor", "outcome"])
|
|
412
|
+
|| event.schemaVersion !== 1 || event.eventType !== "reviewer_replacement"
|
|
413
|
+
|| !isNonBlank(event.eventId) || !isNonBlank(event.repositoryId) || !isNonBlank(event.changeRequestId)
|
|
414
|
+
|| parseCanonicalRecoveryDate(event.occurredAt)?.getTime() !== completedAt.getTime()
|
|
415
|
+
|| event.workspaceId !== job.workspace_id || event.provider !== job.provider
|
|
416
|
+
|| event.providerConnectionId !== job.provider_connection_id
|
|
417
|
+
|| event.absenceId !== scope.absenceId || event.absenceRevision !== scope.absenceRevision
|
|
418
|
+
|| event.decisionId !== persistence.decisionId || event.unavailableActor !== recovery.unavailableActorId
|
|
419
|
+
|| event.replacementActor !== recovery.replacementActorId || event.outcome !== recovery.outcome)
|
|
420
|
+
return false;
|
|
421
|
+
return !isRecord(recovery.finalizer) || persistence.decisionId === recovery.finalizer.decisionId;
|
|
422
|
+
}
|
|
423
|
+
function parseCanonicalRecoveryDate(value) {
|
|
424
|
+
if (!isNonBlank(value))
|
|
425
|
+
return null;
|
|
426
|
+
const parsed = new Date(value);
|
|
427
|
+
return Number.isFinite(parsed.getTime()) && parsed.toISOString() === value ? parsed : null;
|
|
428
|
+
}
|
|
429
|
+
function hasExactKeys(value, required) {
|
|
430
|
+
return Object.keys(value).length === required.length
|
|
431
|
+
&& required.every((key) => Object.prototype.hasOwnProperty.call(value, key));
|
|
432
|
+
}
|
|
433
|
+
function hasOnlyKeys(value, allowed) {
|
|
434
|
+
const allowedKeys = new Set(allowed);
|
|
435
|
+
return Object.keys(value).every((key) => allowedKeys.has(key));
|
|
436
|
+
}
|
|
437
|
+
async function recoverySourceMatches(trx, job, scope) {
|
|
438
|
+
const raw = job.payload.reviewerReplacementFinalizerRecovery;
|
|
439
|
+
if (!isRecord(raw) || !isRecord(raw.job)
|
|
440
|
+
|| raw.provider !== job.provider
|
|
441
|
+
|| !isNonBlank(raw.unavailableActorId)
|
|
442
|
+
|| raw.job.workspaceId !== job.workspace_id
|
|
443
|
+
|| raw.job.providerConnectionId !== job.provider_connection_id
|
|
444
|
+
|| raw.job.absenceId !== scope.absenceId
|
|
445
|
+
|| raw.job.absenceRevision !== scope.absenceRevision)
|
|
446
|
+
return false;
|
|
447
|
+
if (raw.replacementId === null) {
|
|
448
|
+
if (!isUuid(raw.mutationIntentId))
|
|
449
|
+
return false;
|
|
450
|
+
const intent = await trx.selectFrom("reviewer_mutation_intents").selectAll()
|
|
451
|
+
.where("id", "=", raw.mutationIntentId)
|
|
452
|
+
.where("workspace_id", "=", job.workspace_id)
|
|
453
|
+
.where("provider", "=", job.provider)
|
|
454
|
+
.where("provider_connection_id", "=", job.provider_connection_id)
|
|
455
|
+
.where("absence_id", "=", scope.absenceId)
|
|
456
|
+
.where("absence_revision", "=", scope.absenceRevision)
|
|
457
|
+
.executeTakeFirst();
|
|
458
|
+
if (intent === undefined || !isRecord(raw.persistence) || !isRecord(raw.persistence.event))
|
|
459
|
+
return false;
|
|
460
|
+
const replacementActorMatches = raw.outcome === "replaced"
|
|
461
|
+
? intent.replacement_actor_id === raw.replacementActorId
|
|
462
|
+
&& intent.replacement_actor_id === raw.persistence.replacementActorId
|
|
463
|
+
&& intent.replacement_actor_id === raw.persistence.event.replacementActor
|
|
464
|
+
: raw.replacementActorId === null
|
|
465
|
+
&& raw.persistence.replacementActorId === null
|
|
466
|
+
&& raw.persistence.event.replacementActor === null;
|
|
467
|
+
return intent.decision_id === raw.persistence.decisionId
|
|
468
|
+
&& intent.expected_head_revision === raw.persistence.expectedHeadRevision
|
|
469
|
+
&& intent.repository_id === raw.persistence.event.repositoryId
|
|
470
|
+
&& intent.change_request_id === raw.persistence.event.changeRequestId
|
|
471
|
+
&& intent.unavailable_actor_id === raw.unavailableActorId
|
|
472
|
+
&& intent.unavailable_actor_id === raw.persistence.unavailableActorId
|
|
473
|
+
&& intent.unavailable_actor_id === raw.persistence.event.unavailableActor
|
|
474
|
+
&& replacementActorMatches;
|
|
475
|
+
}
|
|
476
|
+
if (!isUuid(raw.replacementId) || !isRecord(raw.finalizer) || !isUuid(raw.finalizer.decisionId))
|
|
477
|
+
return false;
|
|
478
|
+
const row = await trx.selectFrom("reviewer_replacements").selectAll()
|
|
479
|
+
.where("id", "=", raw.replacementId)
|
|
480
|
+
.where("workspace_id", "=", job.workspace_id)
|
|
481
|
+
.where("provider", "=", job.provider)
|
|
482
|
+
.where("provider_connection_id", "=", job.provider_connection_id)
|
|
483
|
+
.where("absence_id", "=", scope.absenceId)
|
|
484
|
+
.where("absence_revision", "=", scope.absenceRevision)
|
|
485
|
+
.where("decision_id", "=", raw.finalizer.decisionId)
|
|
486
|
+
.executeTakeFirst();
|
|
487
|
+
return row !== undefined
|
|
488
|
+
&& row.unavailable_actor_id === raw.unavailableActorId
|
|
489
|
+
&& row.outcome === raw.outcome
|
|
490
|
+
&& row.replacement_actor_id === raw.replacementActorId
|
|
491
|
+
&& row.mutation_intent_id === raw.mutationIntentId;
|
|
492
|
+
}
|
|
493
|
+
function isRecord(value) {
|
|
494
|
+
return typeof value === "object" && value !== null;
|
|
495
|
+
}
|
|
496
|
+
function isNonBlank(value) {
|
|
497
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
498
|
+
}
|
|
499
|
+
function isUuid(value) {
|
|
500
|
+
return isNonBlank(value)
|
|
501
|
+
&& /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(value);
|
|
502
|
+
}
|
|
503
|
+
function isPositiveInteger(value) {
|
|
504
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0;
|
|
505
|
+
}
|
|
506
|
+
function toJobRecord(row) {
|
|
507
|
+
return {
|
|
508
|
+
id: row.id,
|
|
509
|
+
workspaceId: row.workspace_id,
|
|
510
|
+
provider: row.provider,
|
|
511
|
+
providerConnectionId: row.provider_connection_id,
|
|
512
|
+
kind: row.kind,
|
|
513
|
+
status: row.status,
|
|
514
|
+
payload: row.payload,
|
|
515
|
+
idempotencyKey: row.idempotency_key,
|
|
516
|
+
attemptCount: row.attempt_count,
|
|
517
|
+
maxAttempts: row.max_attempts,
|
|
518
|
+
runAt: row.run_at,
|
|
519
|
+
lockedAt: row.locked_at,
|
|
520
|
+
lockedBy: row.locked_by,
|
|
521
|
+
lastError: row.last_error,
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
export function createJobClaimer(db) {
|
|
525
|
+
return {
|
|
526
|
+
async claimNext(workerId, now) {
|
|
527
|
+
return await db.transaction().execute(async (trx) => {
|
|
528
|
+
const job = await trx
|
|
529
|
+
.selectFrom("jobs")
|
|
530
|
+
.selectAll()
|
|
531
|
+
.where("status", "=", "queued")
|
|
532
|
+
.where("run_at", "<=", now)
|
|
533
|
+
.orderBy("run_at", "asc")
|
|
534
|
+
.orderBy("created_at", "asc")
|
|
535
|
+
.orderBy("id", "asc")
|
|
536
|
+
.forUpdate()
|
|
537
|
+
.skipLocked()
|
|
538
|
+
.executeTakeFirst();
|
|
539
|
+
if (!job)
|
|
540
|
+
return null;
|
|
541
|
+
const claimed = await trx
|
|
542
|
+
.updateTable("jobs")
|
|
543
|
+
.set({
|
|
544
|
+
status: "running",
|
|
545
|
+
locked_at: now,
|
|
546
|
+
locked_by: workerId,
|
|
547
|
+
attempt_count: job.attempt_count + 1,
|
|
548
|
+
updated_at: now,
|
|
549
|
+
})
|
|
550
|
+
.where("id", "=", job.id)
|
|
551
|
+
.where("workspace_id", "=", job.workspace_id)
|
|
552
|
+
.returningAll()
|
|
553
|
+
.executeTakeFirstOrThrow();
|
|
554
|
+
return toJobRecord(claimed);
|
|
555
|
+
});
|
|
556
|
+
},
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
export function createWorkspaceJobQueue(db, workspaceId) {
|
|
560
|
+
return {
|
|
561
|
+
async enqueue(input) {
|
|
562
|
+
const inserted = await db
|
|
563
|
+
.insertInto("jobs")
|
|
564
|
+
.values({
|
|
565
|
+
workspace_id: workspaceId,
|
|
566
|
+
provider: input.provider,
|
|
567
|
+
provider_connection_id: input.providerConnectionId,
|
|
568
|
+
kind: input.kind,
|
|
569
|
+
payload: input.payload,
|
|
570
|
+
idempotency_key: input.idempotencyKey,
|
|
571
|
+
run_at: input.runAt ?? new Date(),
|
|
572
|
+
max_attempts: input.maxAttempts ?? 5,
|
|
573
|
+
})
|
|
574
|
+
.onConflict((oc) => oc.columns(["workspace_id", "idempotency_key"]).doNothing())
|
|
575
|
+
.returning(["id"])
|
|
576
|
+
.executeTakeFirst();
|
|
577
|
+
if (inserted)
|
|
578
|
+
return { inserted: true, jobId: inserted.id };
|
|
579
|
+
const existing = await db
|
|
580
|
+
.selectFrom("jobs")
|
|
581
|
+
.select(["id"])
|
|
582
|
+
.where("workspace_id", "=", workspaceId)
|
|
583
|
+
.where("idempotency_key", "=", input.idempotencyKey)
|
|
584
|
+
.executeTakeFirstOrThrow();
|
|
585
|
+
return { inserted: false, jobId: existing.id };
|
|
586
|
+
},
|
|
587
|
+
async markSucceeded(lease, now) {
|
|
588
|
+
if (lease.workspaceId !== workspaceId)
|
|
589
|
+
return { updated: false, reason: "stale_lease" };
|
|
590
|
+
const updated = await db
|
|
591
|
+
.updateTable("jobs")
|
|
592
|
+
.set({ status: "succeeded", locked_at: null, locked_by: null, updated_at: now })
|
|
593
|
+
.where("id", "=", lease.jobId)
|
|
594
|
+
.where("workspace_id", "=", workspaceId)
|
|
595
|
+
.where("provider", "=", lease.provider)
|
|
596
|
+
.where("provider_connection_id", "=", lease.providerConnectionId)
|
|
597
|
+
.where("status", "=", "running")
|
|
598
|
+
.where("locked_by", "=", lease.lockedBy)
|
|
599
|
+
.where("locked_at", "=", lease.lockedAt)
|
|
600
|
+
.where("attempt_count", "=", lease.attemptCount)
|
|
601
|
+
.returning("id")
|
|
602
|
+
.executeTakeFirst();
|
|
603
|
+
return updated ? { updated: true } : { updated: false, reason: "stale_lease" };
|
|
604
|
+
},
|
|
605
|
+
async markFailed(lease, error, now, options) {
|
|
606
|
+
if (lease.workspaceId !== workspaceId)
|
|
607
|
+
return { updated: false, reason: "stale_lease" };
|
|
608
|
+
const recovery = options.recovery;
|
|
609
|
+
const exhausted = recovery === undefined && (!options.retryable || lease.attemptCount >= lease.maxAttempts);
|
|
610
|
+
const updated = await db
|
|
611
|
+
.updateTable("jobs")
|
|
612
|
+
.set({
|
|
613
|
+
status: recovery === undefined && exhausted ? "failed" : "queued",
|
|
614
|
+
...(recovery === undefined
|
|
615
|
+
? {}
|
|
616
|
+
: { payload: recovery.payload, max_attempts: recovery.maxAttempts }),
|
|
617
|
+
last_error: error,
|
|
618
|
+
run_at: recovery === undefined && exhausted ? now : buildNextRunAt(now, lease.attemptCount),
|
|
619
|
+
locked_at: null,
|
|
620
|
+
locked_by: null,
|
|
621
|
+
updated_at: now,
|
|
622
|
+
})
|
|
623
|
+
.where("id", "=", lease.jobId)
|
|
624
|
+
.where("workspace_id", "=", workspaceId)
|
|
625
|
+
.where("provider", "=", lease.provider)
|
|
626
|
+
.where("provider_connection_id", "=", lease.providerConnectionId)
|
|
627
|
+
.where("status", "=", "running")
|
|
628
|
+
.where("locked_by", "=", lease.lockedBy)
|
|
629
|
+
.where("locked_at", "=", lease.lockedAt)
|
|
630
|
+
.where("attempt_count", "=", lease.attemptCount)
|
|
631
|
+
.returning("id")
|
|
632
|
+
.executeTakeFirst();
|
|
633
|
+
return updated ? { updated: true } : { updated: false, reason: "stale_lease" };
|
|
634
|
+
},
|
|
635
|
+
async exhaustReviewerAbsenceActivation(lease, error, now) {
|
|
636
|
+
if (lease.workspaceId !== workspaceId)
|
|
637
|
+
return { updated: false, reason: "stale_lease" };
|
|
638
|
+
return await exhaustReviewerAbsenceActivationTransaction(db, lease, error, now);
|
|
639
|
+
},
|
|
640
|
+
};
|
|
641
|
+
}
|