@sonamu-kit/tasks 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend.d.ts +4 -0
- package/dist/backend.d.ts.map +1 -1
- package/dist/backend.js.map +1 -1
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +3 -1
- package/dist/client.js.map +1 -1
- package/dist/core/retry.d.ts +35 -19
- package/dist/core/retry.d.ts.map +1 -1
- package/dist/core/retry.js +50 -14
- package/dist/core/retry.js.map +1 -1
- package/dist/core/retry.test.js +172 -11
- package/dist/core/retry.test.js.map +1 -1
- package/dist/database/backend.d.ts.map +1 -1
- package/dist/database/backend.js +114 -5
- package/dist/database/backend.js.map +1 -1
- package/dist/database/backend.testsuite.d.ts.map +1 -1
- package/dist/database/backend.testsuite.js +106 -0
- package/dist/database/backend.testsuite.js.map +1 -1
- package/dist/execution.d.ts +2 -0
- package/dist/execution.d.ts.map +1 -1
- package/dist/execution.js +17 -3
- package/dist/execution.js.map +1 -1
- package/dist/execution.test.js +104 -0
- package/dist/execution.test.js.map +1 -1
- package/dist/internal.d.ts +2 -1
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +1 -1
- package/dist/internal.js.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +2 -1
- package/dist/worker.js.map +1 -1
- package/dist/workflow.d.ts +3 -0
- package/dist/workflow.d.ts.map +1 -1
- package/dist/workflow.js.map +1 -1
- package/package.json +3 -3
- package/src/backend.ts +4 -0
- package/src/client.ts +2 -0
- package/src/core/retry.test.ts +180 -11
- package/src/core/retry.ts +95 -19
- package/src/database/backend.testsuite.ts +119 -0
- package/src/database/backend.ts +133 -5
- package/src/execution.test.ts +115 -0
- package/src/execution.ts +18 -2
- package/src/internal.ts +21 -1
- package/src/worker.ts +1 -0
- package/src/workflow.ts +3 -0
package/dist/database/backend.js
CHANGED
|
@@ -2,7 +2,7 @@ import { getLogger } from "@logtape/logtape";
|
|
|
2
2
|
import { camelize } from "inflection";
|
|
3
3
|
import knex from "knex";
|
|
4
4
|
import { DEFAULT_NAMESPACE_ID } from "../backend.js";
|
|
5
|
-
import {
|
|
5
|
+
import { mergeRetryPolicy } from "../core/retry.js";
|
|
6
6
|
import { DEFAULT_SCHEMA, migrate } from "./base.js";
|
|
7
7
|
import { PostgresPubSub } from "./pubsub.js";
|
|
8
8
|
export const DEFAULT_LISTEN_CHANNEL = "new_tasks";
|
|
@@ -12,6 +12,12 @@ const logger = getLogger([
|
|
|
12
12
|
"internal",
|
|
13
13
|
"tasks"
|
|
14
14
|
]);
|
|
15
|
+
const queryLogger = getLogger([
|
|
16
|
+
"sonamu",
|
|
17
|
+
"internal",
|
|
18
|
+
"tasks",
|
|
19
|
+
"query"
|
|
20
|
+
]);
|
|
15
21
|
/**
|
|
16
22
|
* Manages a connection to a Postgres database for workflow operations.
|
|
17
23
|
*/ export class BackendPostgres {
|
|
@@ -25,6 +31,12 @@ const logger = getLogger([
|
|
|
25
31
|
get knex() {
|
|
26
32
|
if (!this._knex) {
|
|
27
33
|
this._knex = knex(this.config);
|
|
34
|
+
this._knex.on("query", (query)=>{
|
|
35
|
+
queryLogger.debug("SQL: {query}, Values: {bindings}", {
|
|
36
|
+
query: query.sql,
|
|
37
|
+
bindings: query.bindings
|
|
38
|
+
});
|
|
39
|
+
});
|
|
28
40
|
}
|
|
29
41
|
return this._knex;
|
|
30
42
|
}
|
|
@@ -100,6 +112,15 @@ const logger = getLogger([
|
|
|
100
112
|
if (!this.initialized) {
|
|
101
113
|
throw new Error("Backend not initialized");
|
|
102
114
|
}
|
|
115
|
+
logger.info("Creating workflow run: {workflowName}:{version}", {
|
|
116
|
+
workflowName: params.workflowName,
|
|
117
|
+
version: params.version
|
|
118
|
+
});
|
|
119
|
+
// config에 retryPolicy를 포함시킵니다.
|
|
120
|
+
const configWithRetryPolicy = {
|
|
121
|
+
...typeof params.config === "object" && params.config !== null ? params.config : {},
|
|
122
|
+
retryPolicy: params.retryPolicy ?? undefined
|
|
123
|
+
};
|
|
103
124
|
const qb = this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").insert({
|
|
104
125
|
namespace_id: this.namespaceId,
|
|
105
126
|
id: crypto.randomUUID(),
|
|
@@ -107,7 +128,7 @@ const logger = getLogger([
|
|
|
107
128
|
version: params.version,
|
|
108
129
|
status: "pending",
|
|
109
130
|
idempotency_key: params.idempotencyKey,
|
|
110
|
-
config:
|
|
131
|
+
config: JSON.stringify(configWithRetryPolicy),
|
|
111
132
|
context: params.context,
|
|
112
133
|
input: params.input,
|
|
113
134
|
attempts: 0,
|
|
@@ -129,6 +150,9 @@ const logger = getLogger([
|
|
|
129
150
|
if (!this.initialized) {
|
|
130
151
|
throw new Error("Backend not initialized");
|
|
131
152
|
}
|
|
153
|
+
logger.info("Getting workflow run: {workflowRunId}", {
|
|
154
|
+
workflowRunId: params.workflowRunId
|
|
155
|
+
});
|
|
132
156
|
const workflowRun = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).select("namespace_id", "id", "workflow_name", "version", "status", "idempotency_key", "config", "context", "input", "output", "error", "attempts", "parent_step_attempt_namespace_id", "parent_step_attempt_id", "worker_id", "available_at", "deadline_at", "started_at", "finished_at", "created_at", "updated_at").first();
|
|
133
157
|
return workflowRun ?? null;
|
|
134
158
|
}
|
|
@@ -136,6 +160,10 @@ const logger = getLogger([
|
|
|
136
160
|
if (!this.initialized) {
|
|
137
161
|
throw new Error("Backend not initialized");
|
|
138
162
|
}
|
|
163
|
+
logger.info("Listing workflow runs: {after}, {before}", {
|
|
164
|
+
after: params.after,
|
|
165
|
+
before: params.before
|
|
166
|
+
});
|
|
139
167
|
const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;
|
|
140
168
|
const { after, before } = params;
|
|
141
169
|
let cursor = null;
|
|
@@ -164,6 +192,10 @@ const logger = getLogger([
|
|
|
164
192
|
if (!this.initialized) {
|
|
165
193
|
throw new Error("Backend not initialized");
|
|
166
194
|
}
|
|
195
|
+
logger.info("Claiming workflow run: {workerId}, {leaseDurationMs}", {
|
|
196
|
+
workerId: params.workerId,
|
|
197
|
+
leaseDurationMs: params.leaseDurationMs
|
|
198
|
+
});
|
|
167
199
|
const claimed = await this.knex.with("expired", (qb)=>qb.withSchema(DEFAULT_SCHEMA).table("workflow_runs").update({
|
|
168
200
|
status: "failed",
|
|
169
201
|
error: JSON.stringify({
|
|
@@ -197,6 +229,11 @@ const logger = getLogger([
|
|
|
197
229
|
if (!this.initialized) {
|
|
198
230
|
throw new Error("Backend not initialized");
|
|
199
231
|
}
|
|
232
|
+
logger.info("Extending workflow run lease: {workflowRunId}, {workerId}, {leaseDurationMs}", {
|
|
233
|
+
workflowRunId: params.workflowRunId,
|
|
234
|
+
workerId: params.workerId,
|
|
235
|
+
leaseDurationMs: params.leaseDurationMs
|
|
236
|
+
});
|
|
200
237
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).where("status", "running").where("worker_id", params.workerId).update({
|
|
201
238
|
available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),
|
|
202
239
|
updated_at: this.knex.fn.now()
|
|
@@ -213,6 +250,11 @@ const logger = getLogger([
|
|
|
213
250
|
if (!this.initialized) {
|
|
214
251
|
throw new Error("Backend not initialized");
|
|
215
252
|
}
|
|
253
|
+
logger.info("Sleeping workflow run: {workflowRunId}, {workerId}, {availableAt}", {
|
|
254
|
+
workflowRunId: params.workflowRunId,
|
|
255
|
+
workerId: params.workerId,
|
|
256
|
+
availableAt: params.availableAt
|
|
257
|
+
});
|
|
216
258
|
// 'succeeded' status is deprecated
|
|
217
259
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).whereNotIn("status", [
|
|
218
260
|
"succeeded",
|
|
@@ -237,6 +279,11 @@ const logger = getLogger([
|
|
|
237
279
|
if (!this.initialized) {
|
|
238
280
|
throw new Error("Backend not initialized");
|
|
239
281
|
}
|
|
282
|
+
logger.info("Completing workflow run: {workflowRunId}, {workerId}, {output}", {
|
|
283
|
+
workflowRunId: params.workflowRunId,
|
|
284
|
+
workerId: params.workerId,
|
|
285
|
+
output: params.output
|
|
286
|
+
});
|
|
240
287
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).where("status", "running").where("worker_id", params.workerId).update({
|
|
241
288
|
status: "completed",
|
|
242
289
|
output: JSON.stringify(params.output),
|
|
@@ -258,15 +305,47 @@ const logger = getLogger([
|
|
|
258
305
|
if (!this.initialized) {
|
|
259
306
|
throw new Error("Backend not initialized");
|
|
260
307
|
}
|
|
261
|
-
const { workflowRunId, error } = params;
|
|
262
|
-
|
|
308
|
+
const { workflowRunId, error, forceComplete, customDelayMs } = params;
|
|
309
|
+
logger.info("Failing workflow run: {workflowRunId}, {workerId}, {error}", {
|
|
310
|
+
workflowRunId: params.workflowRunId,
|
|
311
|
+
workerId: params.workerId,
|
|
312
|
+
error: params.error
|
|
313
|
+
});
|
|
314
|
+
const workflowRun = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", workflowRunId).first();
|
|
315
|
+
if (!workflowRun) {
|
|
316
|
+
throw new Error("Workflow run not found");
|
|
317
|
+
}
|
|
318
|
+
const config = typeof workflowRun.config === "string" ? JSON.parse(workflowRun.config) : workflowRun.config;
|
|
319
|
+
const savedRetryPolicy = config?.retryPolicy;
|
|
320
|
+
const retryPolicy = mergeRetryPolicy(savedRetryPolicy);
|
|
321
|
+
const { initialIntervalMs, backoffCoefficient, maximumIntervalMs, maxAttempts } = retryPolicy;
|
|
322
|
+
const currentAttempts = workflowRun.attempts ?? 0;
|
|
323
|
+
const shouldForceComplete = forceComplete || currentAttempts >= maxAttempts;
|
|
324
|
+
if (shouldForceComplete) {
|
|
325
|
+
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", workflowRunId).where("status", "running").where("worker_id", params.workerId).update({
|
|
326
|
+
status: "failed",
|
|
327
|
+
available_at: null,
|
|
328
|
+
finished_at: this.knex.fn.now(),
|
|
329
|
+
error: JSON.stringify(error),
|
|
330
|
+
worker_id: null,
|
|
331
|
+
started_at: null,
|
|
332
|
+
updated_at: this.knex.fn.now()
|
|
333
|
+
}).returning("*");
|
|
334
|
+
if (!updated) {
|
|
335
|
+
logger.error("Failed to mark workflow run failed: {params}", {
|
|
336
|
+
params
|
|
337
|
+
});
|
|
338
|
+
throw new Error("Failed to mark workflow run failed");
|
|
339
|
+
}
|
|
340
|
+
return updated;
|
|
341
|
+
}
|
|
263
342
|
// this beefy query updates a workflow's status, available_at, and
|
|
264
343
|
// finished_at based on the workflow's deadline and retry policy
|
|
265
344
|
//
|
|
266
345
|
// if the next retry would exceed the deadline, the run is marked as
|
|
267
346
|
// 'failed' and finalized, otherwise, the run is rescheduled with an updated
|
|
268
347
|
// 'available_at' timestamp for the next retry
|
|
269
|
-
const retryIntervalExpr = `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, "attempts" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;
|
|
348
|
+
const retryIntervalExpr = customDelayMs ? `${customDelayMs} * INTERVAL '1 millisecond'` : `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, "attempts" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;
|
|
270
349
|
const deadlineExceededCondition = `"deadline_at" IS NOT NULL AND NOW() + (${retryIntervalExpr}) >= "deadline_at"`;
|
|
271
350
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", workflowRunId).where("status", "running").where("worker_id", params.workerId).update({
|
|
272
351
|
status: this.knex.raw(`CASE WHEN ${deadlineExceededCondition} THEN 'failed' ELSE 'pending' END`),
|
|
@@ -289,6 +368,9 @@ const logger = getLogger([
|
|
|
289
368
|
if (!this.initialized) {
|
|
290
369
|
throw new Error("Backend not initialized");
|
|
291
370
|
}
|
|
371
|
+
logger.info("Canceling workflow run: {workflowRunId}", {
|
|
372
|
+
workflowRunId: params.workflowRunId
|
|
373
|
+
});
|
|
292
374
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).whereIn("status", [
|
|
293
375
|
"pending",
|
|
294
376
|
"running",
|
|
@@ -336,6 +418,11 @@ const logger = getLogger([
|
|
|
336
418
|
if (!this.initialized) {
|
|
337
419
|
throw new Error("Backend not initialized");
|
|
338
420
|
}
|
|
421
|
+
logger.info("Creating step attempt: {workflowRunId}, {stepName}, {kind}", {
|
|
422
|
+
workflowRunId: params.workflowRunId,
|
|
423
|
+
stepName: params.stepName,
|
|
424
|
+
kind: params.kind
|
|
425
|
+
});
|
|
339
426
|
const [stepAttempt] = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts").insert({
|
|
340
427
|
namespace_id: this.namespaceId,
|
|
341
428
|
id: crypto.randomUUID(),
|
|
@@ -361,6 +448,9 @@ const logger = getLogger([
|
|
|
361
448
|
if (!this.initialized) {
|
|
362
449
|
throw new Error("Backend not initialized");
|
|
363
450
|
}
|
|
451
|
+
logger.info("Getting step attempt: {stepAttemptId}", {
|
|
452
|
+
stepAttemptId: params.stepAttemptId
|
|
453
|
+
});
|
|
364
454
|
const stepAttempt = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts").where("namespace_id", this.namespaceId).where("id", params.stepAttemptId).first();
|
|
365
455
|
return stepAttempt ?? null;
|
|
366
456
|
}
|
|
@@ -368,6 +458,11 @@ const logger = getLogger([
|
|
|
368
458
|
if (!this.initialized) {
|
|
369
459
|
throw new Error("Backend not initialized");
|
|
370
460
|
}
|
|
461
|
+
logger.info("Listing step attempts: {workflowRunId}, {after}, {before}", {
|
|
462
|
+
workflowRunId: params.workflowRunId,
|
|
463
|
+
after: params.after,
|
|
464
|
+
before: params.before
|
|
465
|
+
});
|
|
371
466
|
const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;
|
|
372
467
|
const { after, before } = params;
|
|
373
468
|
let cursor = null;
|
|
@@ -424,10 +519,16 @@ const logger = getLogger([
|
|
|
424
519
|
}
|
|
425
520
|
};
|
|
426
521
|
}
|
|
522
|
+
// NOTE: 실제 서비스에서 이게 안 되는 것 같은데, 쿼리 등을 체크할 필요가 있음.
|
|
427
523
|
async completeStepAttempt(params) {
|
|
428
524
|
if (!this.initialized) {
|
|
429
525
|
throw new Error("Backend not initialized");
|
|
430
526
|
}
|
|
527
|
+
logger.info("Marking step attempt as completed: {workflowRunId}, {stepAttemptId}, {workerId}", {
|
|
528
|
+
workflowRunId: params.workflowRunId,
|
|
529
|
+
stepAttemptId: params.stepAttemptId,
|
|
530
|
+
workerId: params.workerId
|
|
531
|
+
});
|
|
431
532
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts as sa").update({
|
|
432
533
|
status: "completed",
|
|
433
534
|
output: JSON.stringify(params.output),
|
|
@@ -447,6 +548,14 @@ const logger = getLogger([
|
|
|
447
548
|
if (!this.initialized) {
|
|
448
549
|
throw new Error("Backend not initialized");
|
|
449
550
|
}
|
|
551
|
+
logger.info("Marking step attempt as failed: {workflowRunId}, {stepAttemptId}, {workerId}", {
|
|
552
|
+
workflowRunId: params.workflowRunId,
|
|
553
|
+
stepAttemptId: params.stepAttemptId,
|
|
554
|
+
workerId: params.workerId
|
|
555
|
+
});
|
|
556
|
+
logger.info("Error: {error.message}", {
|
|
557
|
+
error: params.error.message
|
|
558
|
+
});
|
|
450
559
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts as sa").update({
|
|
451
560
|
status: "failed",
|
|
452
561
|
output: null,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/database/backend.ts"],"sourcesContent":["import { getLogger } from \"@logtape/logtape\";\nimport { camelize } from \"inflection\";\nimport knex, { type Knex } from \"knex\";\nimport {\n type Backend,\n type CancelWorkflowRunParams,\n type ClaimWorkflowRunParams,\n type CompleteStepAttemptParams,\n type CompleteWorkflowRunParams,\n type CreateStepAttemptParams,\n type CreateWorkflowRunParams,\n DEFAULT_NAMESPACE_ID,\n type ExtendWorkflowRunLeaseParams,\n type FailStepAttemptParams,\n type FailWorkflowRunParams,\n type GetStepAttemptParams,\n type GetWorkflowRunParams,\n type ListStepAttemptsParams,\n type ListWorkflowRunsParams,\n type PaginatedResponse,\n type SleepWorkflowRunParams,\n} from \"../backend\";\nimport { DEFAULT_RETRY_POLICY } from \"../core/retry\";\nimport type { StepAttempt } from \"../core/step\";\nimport type { WorkflowRun } from \"../core/workflow\";\nimport { DEFAULT_SCHEMA, migrate } from \"./base\";\nimport { type OnSubscribed, PostgresPubSub } from \"./pubsub\";\n\nexport const DEFAULT_LISTEN_CHANNEL = \"new_tasks\" as const;\nconst DEFAULT_PAGINATION_PAGE_SIZE = 100 as const;\n\ninterface BackendPostgresOptions {\n namespaceId?: string;\n runMigrations?: boolean;\n\n // default: true\n usePubSub?: boolean;\n}\n\nconst logger = getLogger([\"sonamu\", \"internal\", \"tasks\"]);\n\n/**\n * Manages a connection to a Postgres database for workflow operations.\n */\nexport class BackendPostgres implements Backend {\n private config: Knex.Config;\n private namespaceId: string;\n private usePubSub: boolean;\n private pubsub: PostgresPubSub | null = null;\n private initialized: boolean = false;\n private runMigrations: boolean;\n\n private _knex: Knex | null = null;\n private get knex(): Knex {\n if (!this._knex) {\n this._knex = knex(this.config);\n }\n\n return this._knex;\n }\n\n constructor(config: Knex.Config, options?: BackendPostgresOptions) {\n this.config = {\n ...config,\n postProcessResponse: (result, _queryContext) => {\n if (result === null || result === undefined) {\n return result;\n }\n\n if (config?.postProcessResponse) {\n result = config.postProcessResponse(result, _queryContext);\n }\n\n const camelizeRow = (row: Record<string, unknown>) =>\n Object.fromEntries(\n Object.entries(row).map(([key, value]) => [camelize(key, true), value]),\n );\n\n if (Array.isArray(result)) {\n return result.map(camelizeRow);\n }\n\n return camelizeRow(result);\n },\n };\n\n const { namespaceId, usePubSub, runMigrations } = {\n namespaceId: DEFAULT_NAMESPACE_ID,\n usePubSub: true,\n runMigrations: true,\n ...options,\n };\n\n this.namespaceId = namespaceId;\n this.usePubSub = usePubSub;\n this.runMigrations = runMigrations;\n }\n\n async initialize() {\n if (this.initialized) {\n return;\n }\n\n if (this.runMigrations) {\n await migrate(this.config, DEFAULT_SCHEMA);\n }\n\n this.initialized = true;\n }\n\n async subscribe(callback: OnSubscribed) {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n if (!this.pubsub) {\n this.pubsub = await PostgresPubSub.create(this.knex);\n }\n\n this.pubsub.listenEvent(DEFAULT_LISTEN_CHANNEL, callback);\n }\n\n async publish(payload?: string): Promise<void> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n await this.knex.raw(\n payload\n ? `NOTIFY ${DEFAULT_LISTEN_CHANNEL}, '${payload}'`\n : `NOTIFY ${DEFAULT_LISTEN_CHANNEL}`,\n );\n }\n\n async stop(): Promise<void> {\n if (!this.initialized) {\n return;\n }\n\n await this.pubsub?.destroy();\n this.pubsub = null;\n await this.knex.destroy();\n }\n\n async createWorkflowRun(params: CreateWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_name: params.workflowName,\n version: params.version,\n status: \"pending\",\n idempotency_key: params.idempotencyKey,\n config: params.config,\n context: params.context,\n input: params.input,\n attempts: 0,\n available_at: params.availableAt ?? this.knex.fn.now(),\n deadline_at: params.deadlineAt,\n created_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n const workflowRun = await qb;\n if (!workflowRun[0]) {\n logger.error(\"Failed to create workflow run: {params}\", { params });\n throw new Error(\"Failed to create workflow run\");\n }\n\n return workflowRun[0];\n }\n\n async getWorkflowRun(params: GetWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const workflowRun = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .select(\n \"namespace_id\",\n \"id\",\n \"workflow_name\",\n \"version\",\n \"status\",\n \"idempotency_key\",\n \"config\",\n \"context\",\n \"input\",\n \"output\",\n \"error\",\n \"attempts\",\n \"parent_step_attempt_namespace_id\",\n \"parent_step_attempt_id\",\n \"worker_id\",\n \"available_at\",\n \"deadline_at\",\n \"started_at\",\n \"finished_at\",\n \"created_at\",\n \"updated_at\",\n )\n .first();\n\n return workflowRun ?? null;\n }\n\n async listWorkflowRuns(params: ListWorkflowRunsParams): Promise<PaginatedResponse<WorkflowRun>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListWorkflowRunsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListWorkflowRunsWhere(params: ListWorkflowRunsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n async claimWorkflowRun(params: ClaimWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const claimed = await this.knex\n .with(\"expired\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .update({\n status: \"failed\",\n error: JSON.stringify({ message: \"Workflow run deadline exceeded\" }),\n worker_id: null,\n available_at: null,\n finished_at: this.knex.raw(\"NOW()\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .whereNotNull(\"deadline_at\")\n .where(\"deadline_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .returning(\"id\"),\n )\n .with(\"candidate\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .select(\"id\")\n .from(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .where(\"available_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .where((qb2) => {\n qb2.whereNull(\"deadline_at\").orWhere(\"deadline_at\", \">\", this.knex.raw(\"NOW()\"));\n })\n .orderByRaw(\"CASE WHEN status = 'pending' THEN 0 ELSE 1 END\")\n .orderBy(\"available_at\", \"asc\")\n .orderBy(\"created_at\", \"asc\")\n .limit(1)\n .forUpdate()\n .skipLocked(),\n )\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs as wr\")\n .where(\"wr.namespace_id\", this.namespaceId)\n .where(\"wr.id\", this.knex.ref(\"candidate.id\"))\n .update({\n status: \"running\",\n attempts: this.knex.raw(\"wr.attempts + 1\"),\n worker_id: params.workerId,\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n started_at: this.knex.raw(\"COALESCE(wr.started_at, NOW())\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .updateFrom(\"candidate\")\n .returning(\"wr.*\");\n\n return claimed[0] ?? null;\n }\n\n async extendWorkflowRunLease(params: ExtendWorkflowRunLeaseParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to extend lease for workflow run: {params}\", { params });\n throw new Error(\"Failed to extend lease for workflow run\");\n }\n\n return updated;\n }\n\n async sleepWorkflowRun(params: SleepWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n // 'succeeded' status is deprecated\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereNotIn(\"status\", [\"succeeded\", \"completed\", \"failed\", \"canceled\"])\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"sleeping\",\n available_at: params.availableAt,\n worker_id: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to sleep workflow run: {params}\", { params });\n throw new Error(\"Failed to sleep workflow run\");\n }\n\n return updated;\n }\n\n async completeWorkflowRun(params: CompleteWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n worker_id: params.workerId,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to complete workflow run: {params}\", { params });\n throw new Error(\"Failed to complete workflow run\");\n }\n\n return updated;\n }\n\n async failWorkflowRun(params: FailWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const { workflowRunId, error } = params;\n const { initialIntervalMs, backoffCoefficient, maximumIntervalMs } = DEFAULT_RETRY_POLICY;\n\n // this beefy query updates a workflow's status, available_at, and\n // finished_at based on the workflow's deadline and retry policy\n //\n // if the next retry would exceed the deadline, the run is marked as\n // 'failed' and finalized, otherwise, the run is rescheduled with an updated\n // 'available_at' timestamp for the next retry\n const retryIntervalExpr = `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, \"attempts\" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;\n const deadlineExceededCondition = `\"deadline_at\" IS NOT NULL AND NOW() + (${retryIntervalExpr}) >= \"deadline_at\"`;\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN 'failed' ELSE 'pending' END`,\n ),\n available_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NULL ELSE NOW() + (${retryIntervalExpr}) END`,\n ),\n finished_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NOW() ELSE NULL END`,\n ),\n error: JSON.stringify(error),\n worker_id: null,\n started_at: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to mark workflow run failed: {params}\", { params });\n throw new Error(\"Failed to mark workflow run failed\");\n }\n\n return updated;\n }\n\n async cancelWorkflowRun(params: CancelWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .update({\n status: \"canceled\",\n worker_id: null,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n // workflow may already be in a terminal state\n const existing = await this.getWorkflowRun({\n workflowRunId: params.workflowRunId,\n });\n if (!existing) {\n throw new Error(`Workflow run ${params.workflowRunId} does not exist`);\n }\n\n // if already canceled, just return it\n if (existing.status === \"canceled\") {\n return existing;\n }\n\n // throw error for completed/failed workflows\n // 'succeeded' status is deprecated\n if ([\"succeeded\", \"completed\", \"failed\"].includes(existing.status)) {\n logger.error(\"Cannot cancel workflow run: {params} with status {status}\", {\n params,\n status: existing.status,\n });\n throw new Error(\n `Cannot cancel workflow run ${params.workflowRunId} with status ${existing.status}`,\n );\n }\n\n logger.error(\"Failed to cancel workflow run: {params}\", { params });\n throw new Error(\"Failed to cancel workflow run\");\n }\n\n return updated;\n }\n\n async createStepAttempt(params: CreateStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [stepAttempt] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_run_id: params.workflowRunId,\n step_name: params.stepName,\n kind: params.kind,\n status: \"running\",\n config: JSON.stringify(params.config),\n context: JSON.stringify(params.context),\n started_at: this.knex.fn.now(),\n created_at: this.knex.raw(\"date_trunc('milliseconds', NOW())\"),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!stepAttempt) {\n logger.error(\"Failed to create step attempt: {params}\", { params });\n throw new Error(\"Failed to create step attempt\");\n }\n\n return stepAttempt;\n }\n\n async getStepAttempt(params: GetStepAttemptParams): Promise<StepAttempt | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const stepAttempt = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.stepAttemptId)\n .first();\n\n return stepAttempt ?? null;\n }\n\n async listStepAttempts(params: ListStepAttemptsParams): Promise<PaginatedResponse<StepAttempt>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListStepAttemptsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListStepAttemptsWhere(params: ListStepAttemptsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"workflow_run_id\", params.workflowRunId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n private processPaginationResults<T extends Cursor>(\n rows: T[],\n limit: number,\n hasAfter: boolean,\n hasBefore: boolean,\n ): PaginatedResponse<T> {\n const data = rows;\n let hasNext = false;\n let hasPrev = false;\n\n if (hasBefore) {\n data.reverse();\n if (data.length > limit) {\n hasPrev = true;\n data.shift();\n }\n hasNext = true;\n } else {\n if (data.length > limit) {\n hasNext = true;\n data.pop();\n }\n if (hasAfter) {\n hasPrev = true;\n }\n }\n\n const lastItem = data.at(-1);\n const nextCursor = hasNext && lastItem ? encodeCursor(lastItem) : null;\n const firstItem = data[0];\n const prevCursor = hasPrev && firstItem ? encodeCursor(firstItem) : null;\n\n return {\n data,\n pagination: {\n next: nextCursor,\n prev: prevCursor,\n },\n };\n }\n\n async completeStepAttempt(params: CompleteStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt completed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt completed\");\n }\n\n return updated;\n }\n\n async failStepAttempt(params: FailStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"failed\",\n output: null,\n error: JSON.stringify(params.error),\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt failed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt failed\");\n }\n\n return updated;\n }\n}\n\n/**\n * Cursor used for pagination. Requires created_at and id fields. Because JS\n * Date does not natively support microsecond precision dates, created_at should\n * be stored with millisecond precision in paginated tables to avoid issues with\n * cursor comparisons.\n */\ninterface Cursor {\n createdAt: Date;\n id: string;\n}\n\nfunction encodeCursor(item: Cursor): string {\n const encoded = Buffer.from(\n JSON.stringify({ createdAt: item.createdAt.toISOString(), id: item.id }),\n ).toString(\"base64\");\n return encoded;\n}\n\nexport function decodeCursor(cursor: string): Cursor {\n const decoded = Buffer.from(cursor, \"base64\").toString(\"utf8\");\n const parsed = JSON.parse(decoded) as { createdAt: string; id: string };\n return {\n createdAt: new Date(parsed.createdAt),\n id: parsed.id,\n };\n}\n"],"names":["getLogger","camelize","knex","DEFAULT_NAMESPACE_ID","DEFAULT_RETRY_POLICY","DEFAULT_SCHEMA","migrate","PostgresPubSub","DEFAULT_LISTEN_CHANNEL","DEFAULT_PAGINATION_PAGE_SIZE","logger","BackendPostgres","config","namespaceId","usePubSub","pubsub","initialized","runMigrations","_knex","options","postProcessResponse","result","_queryContext","undefined","camelizeRow","row","Object","fromEntries","entries","map","key","value","Array","isArray","initialize","subscribe","callback","Error","create","listenEvent","publish","payload","raw","stop","destroy","createWorkflowRun","params","qb","withSchema","table","insert","namespace_id","id","crypto","randomUUID","workflow_name","workflowName","version","status","idempotency_key","idempotencyKey","context","input","attempts","available_at","availableAt","fn","now","deadline_at","deadlineAt","created_at","updated_at","returning","workflowRun","error","getWorkflowRun","where","workflowRunId","select","first","listWorkflowRuns","limit","after","before","cursor","decodeCursor","buildListWorkflowRunsWhere","rows","orderBy","processPaginationResults","operator","whereRaw","createdAt","toISOString","claimWorkflowRun","claimed","with","update","JSON","stringify","message","worker_id","finished_at","whereIn","whereNotNull","from","qb2","whereNull","orWhere","orderByRaw","forUpdate","skipLocked","ref","workerId","leaseDurationMs","started_at","updateFrom","extendWorkflowRunLease","updated","sleepWorkflowRun","whereNotIn","completeWorkflowRun","output","failWorkflowRun","initialIntervalMs","backoffCoefficient","maximumIntervalMs","retryIntervalExpr","deadlineExceededCondition","cancelWorkflowRun","existing","includes","createStepAttempt","stepAttempt","workflow_run_id","step_name","stepName","kind","getStepAttempt","stepAttemptId","listStepAttempts","buildListStepAttemptsWhere","hasAfter","hasBefore","data","hasNext","hasPrev","reverse","length","shift","pop","lastItem","at","nextCursor","encodeCursor","firstItem","prevCursor","pagination","next","prev","completeStepAttempt","failStepAttempt","item","encoded","Buffer","toString","decoded","parsed","parse","Date"],"mappings":"AAAA,SAASA,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,QAAQ,QAAQ,aAAa;AACtC,OAAOC,UAAyB,OAAO;AACvC,SAQEC,oBAAoB,QAUf,gBAAa;AACpB,SAASC,oBAAoB,QAAQ,mBAAgB;AAGrD,SAASC,cAAc,EAAEC,OAAO,QAAQ,YAAS;AACjD,SAA4BC,cAAc,QAAQ,cAAW;AAE7D,OAAO,MAAMC,yBAAyB,YAAqB;AAC3D,MAAMC,+BAA+B;AAUrC,MAAMC,SAASV,UAAU;IAAC;IAAU;IAAY;CAAQ;AAExD;;CAEC,GACD,OAAO,MAAMW;IACHC,OAAoB;IACpBC,YAAoB;IACpBC,UAAmB;IACnBC,SAAgC,KAAK;IACrCC,cAAuB,MAAM;IAC7BC,cAAuB;IAEvBC,QAAqB,KAAK;IAClC,IAAYhB,OAAa;QACvB,IAAI,CAAC,IAAI,CAACgB,KAAK,EAAE;YACf,IAAI,CAACA,KAAK,GAAGhB,KAAK,IAAI,CAACU,MAAM;QAC/B;QAEA,OAAO,IAAI,CAACM,KAAK;IACnB;IAEA,YAAYN,MAAmB,EAAEO,OAAgC,CAAE;QACjE,IAAI,CAACP,MAAM,GAAG;YACZ,GAAGA,MAAM;YACTQ,qBAAqB,CAACC,QAAQC;gBAC5B,IAAID,WAAW,QAAQA,WAAWE,WAAW;oBAC3C,OAAOF;gBACT;gBAEA,IAAIT,QAAQQ,qBAAqB;oBAC/BC,SAAST,OAAOQ,mBAAmB,CAACC,QAAQC;gBAC9C;gBAEA,MAAME,cAAc,CAACC,MACnBC,OAAOC,WAAW,CAChBD,OAAOE,OAAO,CAACH,KAAKI,GAAG,CAAC,CAAC,CAACC,KAAKC,MAAM,GAAK;4BAAC9B,SAAS6B,KAAK;4BAAOC;yBAAM;gBAG1E,IAAIC,MAAMC,OAAO,CAACZ,SAAS;oBACzB,OAAOA,OAAOQ,GAAG,CAACL;gBACpB;gBAEA,OAAOA,YAAYH;YACrB;QACF;QAEA,MAAM,EAAER,WAAW,EAAEC,SAAS,EAAEG,aAAa,EAAE,GAAG;YAChDJ,aAAaV;YACbW,WAAW;YACXG,eAAe;YACf,GAAGE,OAAO;QACZ;QAEA,IAAI,CAACN,WAAW,GAAGA;QACnB,IAAI,CAACC,SAAS,GAAGA;QACjB,IAAI,CAACG,aAAa,GAAGA;IACvB;IAEA,MAAMiB,aAAa;QACjB,IAAI,IAAI,CAAClB,WAAW,EAAE;YACpB;QACF;QAEA,IAAI,IAAI,CAACC,aAAa,EAAE;YACtB,MAAMX,QAAQ,IAAI,CAACM,MAAM,EAAEP;QAC7B;QAEA,IAAI,CAACW,WAAW,GAAG;IACrB;IAEA,MAAMmB,UAAUC,QAAsB,EAAE;QACtC,IAAI,CAAC,IAAI,CAACpB,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAACvB,SAAS,EAAE;YACnB;QACF;QAEA,IAAI,CAAC,IAAI,CAACC,MAAM,EAAE;YAChB,IAAI,CAACA,MAAM,GAAG,MAAMR,eAAe+B,MAAM,CAAC,IAAI,CAACpC,IAAI;QACrD;QAEA,IAAI,CAACa,MAAM,CAACwB,WAAW,CAAC/B,wBAAwB4B;IAClD;IAEA,MAAMI,QAAQC,OAAgB,EAAiB;QAC7C,IAAI,CAAC,IAAI,CAACzB,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAACvB,SAAS,EAAE;YACnB;QACF;QAEA,MAAM,IAAI,CAACZ,IAAI,CAACwC,GAAG,CACjBD,UACI,CAAC,OAAO,EAAEjC,uBAAuB,GAAG,EAAEiC,QAAQ,CAAC,CAAC,GAChD,CAAC,OAAO,EAAEjC,wBAAwB;IAE1C;IAEA,MAAMmC,OAAsB;QAC1B,IAAI,CAAC,IAAI,CAAC3B,WAAW,EAAE;YACrB;QACF;QAEA,MAAM,IAAI,CAACD,MAAM,EAAE6B;QACnB,IAAI,CAAC7B,MAAM,GAAG;QACd,MAAM,IAAI,CAACb,IAAI,CAAC0C,OAAO;IACzB;IAEA,MAAMC,kBAAkBC,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAMU,KAAK,IAAI,CAAC7C,IAAI,CACjB8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAACtC,WAAW;YAC9BuC,IAAIC,OAAOC,UAAU;YACrBC,eAAeT,OAAOU,YAAY;YAClCC,SAASX,OAAOW,OAAO;YACvBC,QAAQ;YACRC,iBAAiBb,OAAOc,cAAc;YACtChD,QAAQkC,OAAOlC,MAAM;YACrBiD,SAASf,OAAOe,OAAO;YACvBC,OAAOhB,OAAOgB,KAAK;YACnBC,UAAU;YACVC,cAAclB,OAAOmB,WAAW,IAAI,IAAI,CAAC/D,IAAI,CAACgE,EAAE,CAACC,GAAG;YACpDC,aAAatB,OAAOuB,UAAU;YAC9BC,YAAY,IAAI,CAACpE,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC5BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,MAAMC,cAAc,MAAM1B;QAC1B,IAAI,CAAC0B,WAAW,CAAC,EAAE,EAAE;YACnB/D,OAAOgE,KAAK,CAAC,2CAA2C;gBAAE5B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoC,WAAW,CAAC,EAAE;IACvB;IAEA,MAAME,eAAe7B,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAMoC,cAAc,MAAM,IAAI,CAACvE,IAAI,CAChC8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChCC,MAAM,CACL,gBACA,MACA,iBACA,WACA,UACA,mBACA,UACA,WACA,SACA,UACA,SACA,YACA,oCACA,0BACA,aACA,gBACA,eACA,cACA,eACA,cACA,cAEDC,KAAK;QAER,OAAON,eAAe;IACxB;IAEA,MAAMO,iBAAiBlC,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM4C,QAAQnC,OAAOmC,KAAK,IAAIxE;QAC9B,MAAM,EAAEyE,KAAK,EAAEC,MAAM,EAAE,GAAGrC;QAE1B,IAAIsC,SAAwB;QAC5B,IAAIF,OAAO;YACTE,SAASC,aAAaH;QACxB,OAAO,IAAIC,QAAQ;YACjBC,SAASC,aAAaF;QACxB;QAEA,MAAMpC,KAAK,IAAI,CAACuC,0BAA0B,CAACxC,QAAQsC;QACnD,MAAMG,OAAO,MAAMxC,GAChByC,OAAO,CAAC,cAAcL,SAAS,SAAS,OACxCK,OAAO,CAAC,MAAML,SAAS,SAAS,OAChCF,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACQ,wBAAwB,CAClCF,MACAN,OACA,OAAOC,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQG,2BAA2BxC,MAA8B,EAAEsC,MAAqB,EAAE;QACxF,MAAM,EAAEF,KAAK,EAAE,GAAGpC;QAClB,MAAMC,KAAK,IAAI,CAAC7C,IAAI,CACjB8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW;QAEzC,IAAIuE,QAAQ;YACV,MAAMM,WAAWR,QAAQ,MAAM;YAC/B,OAAOnC,GAAG4C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAOhC,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEA,MAAM+C,iBAAiBhD,MAA8B,EAA+B;QAClF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM0D,UAAU,MAAM,IAAI,CAAC7F,IAAI,CAC5B8F,IAAI,CAAC,WAAW,CAACjD,KAChBA,GACGC,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACNgD,MAAM,CAAC;gBACNvC,QAAQ;gBACRgB,OAAOwB,KAAKC,SAAS,CAAC;oBAAEC,SAAS;gBAAiC;gBAClEC,WAAW;gBACXrC,cAAc;gBACdsC,aAAa,IAAI,CAACpG,IAAI,CAACwC,GAAG,CAAC;gBAC3B6B,YAAY,IAAI,CAACrE,IAAI,CAACwC,GAAG,CAAC;YAC5B,GACCkC,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC0F,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpDC,YAAY,CAAC,eACb5B,KAAK,CAAC,eAAe,MAAM,IAAI,CAAC1E,IAAI,CAACwC,GAAG,CAAC,UACzC8B,SAAS,CAAC,OAEdwB,IAAI,CAAC,aAAa,CAACjD,KAClBA,GACGC,UAAU,CAAC3C,gBACXyE,MAAM,CAAC,MACP2B,IAAI,CAAC,iBACL7B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC0F,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpD3B,KAAK,CAAC,gBAAgB,MAAM,IAAI,CAAC1E,IAAI,CAACwC,GAAG,CAAC,UAC1CkC,KAAK,CAAC,CAAC8B;gBACNA,IAAIC,SAAS,CAAC,eAAeC,OAAO,CAAC,eAAe,KAAK,IAAI,CAAC1G,IAAI,CAACwC,GAAG,CAAC;YACzE,GACCmE,UAAU,CAAC,kDACXrB,OAAO,CAAC,gBAAgB,OACxBA,OAAO,CAAC,cAAc,OACtBP,KAAK,CAAC,GACN6B,SAAS,GACTC,UAAU,IAEd/D,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,uBACN2B,KAAK,CAAC,mBAAmB,IAAI,CAAC/D,WAAW,EACzC+D,KAAK,CAAC,SAAS,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,iBAC7Bf,MAAM,CAAC;YACNvC,QAAQ;YACRK,UAAU,IAAI,CAAC7D,IAAI,CAACwC,GAAG,CAAC;YACxB2D,WAAWvD,OAAOmE,QAAQ;YAC1BjD,cAAc,IAAI,CAAC9D,IAAI,CAACwC,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOoE,eAAe,CAAC,2BAA2B,CAAC;YAC1FC,YAAY,IAAI,CAACjH,IAAI,CAACwC,GAAG,CAAC;YAC1B6B,YAAY,IAAI,CAACrE,IAAI,CAACwC,GAAG,CAAC;QAC5B,GACC0E,UAAU,CAAC,aACX5C,SAAS,CAAC;QAEb,OAAOuB,OAAO,CAAC,EAAE,IAAI;IACvB;IAEA,MAAMsB,uBAAuBvE,MAAoC,EAAwB;QACvF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChCD,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNjC,cAAc,IAAI,CAAC9D,IAAI,CAACwC,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOoE,eAAe,CAAC,2BAA2B,CAAC;YAC1F3C,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,qDAAqD;gBAAE5B;YAAO;YAC3E,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMC,iBAAiBzE,MAA8B,EAAwB;QAC3E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,mCAAmC;QACnC,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChC2C,UAAU,CAAC,UAAU;YAAC;YAAa;YAAa;YAAU;SAAW,EACrE5C,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNvC,QAAQ;YACRM,cAAclB,OAAOmB,WAAW;YAChCoC,WAAW;YACX9B,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,0CAA0C;gBAAE5B;YAAO;YAChE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMG,oBAAoB3E,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChCD,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNvC,QAAQ;YACRgE,QAAQxB,KAAKC,SAAS,CAACrD,OAAO4E,MAAM;YACpChD,OAAO;YACP2B,WAAWvD,OAAOmE,QAAQ;YAC1BjD,cAAc;YACdsC,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,6CAA6C;gBAAE5B;YAAO;YACnE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMK,gBAAgB7E,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,EAAEwC,aAAa,EAAEH,KAAK,EAAE,GAAG5B;QACjC,MAAM,EAAE8E,iBAAiB,EAAEC,kBAAkB,EAAEC,iBAAiB,EAAE,GAAG1H;QAErE,kEAAkE;QAClE,gEAAgE;QAChE,EAAE;QACF,oEAAoE;QACpE,4EAA4E;QAC5E,8CAA8C;QAC9C,MAAM2H,oBAAoB,CAAC,MAAM,EAAEH,kBAAkB,SAAS,EAAEC,mBAAmB,mBAAmB,EAAEC,kBAAkB,4BAA4B,CAAC;QACvJ,MAAME,4BAA4B,CAAC,uCAAuC,EAAED,kBAAkB,kBAAkB,CAAC;QAEjH,MAAM,CAACT,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAMC,eACZD,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNvC,QAAQ,IAAI,CAACxD,IAAI,CAACwC,GAAG,CACnB,CAAC,UAAU,EAAEsF,0BAA0B,iCAAiC,CAAC;YAE3EhE,cAAc,IAAI,CAAC9D,IAAI,CAACwC,GAAG,CACzB,CAAC,UAAU,EAAEsF,0BAA0B,yBAAyB,EAAED,kBAAkB,KAAK,CAAC;YAE5FzB,aAAa,IAAI,CAACpG,IAAI,CAACwC,GAAG,CACxB,CAAC,UAAU,EAAEsF,0BAA0B,yBAAyB,CAAC;YAEnEtD,OAAOwB,KAAKC,SAAS,CAACzB;YACtB2B,WAAW;YACXc,YAAY;YACZ5C,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,gDAAgD;gBAAE5B;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMW,kBAAkBnF,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChC0B,OAAO,CAAC,UAAU;YAAC;YAAW;YAAW;SAAW,EACpDN,MAAM,CAAC;YACNvC,QAAQ;YACR2C,WAAW;YACXrC,cAAc;YACdsC,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ,8CAA8C;YAC9C,MAAMY,WAAW,MAAM,IAAI,CAACvD,cAAc,CAAC;gBACzCE,eAAe/B,OAAO+B,aAAa;YACrC;YACA,IAAI,CAACqD,UAAU;gBACb,MAAM,IAAI7F,MAAM,CAAC,aAAa,EAAES,OAAO+B,aAAa,CAAC,eAAe,CAAC;YACvE;YAEA,sCAAsC;YACtC,IAAIqD,SAASxE,MAAM,KAAK,YAAY;gBAClC,OAAOwE;YACT;YAEA,6CAA6C;YAC7C,mCAAmC;YACnC,IAAI;gBAAC;gBAAa;gBAAa;aAAS,CAACC,QAAQ,CAACD,SAASxE,MAAM,GAAG;gBAClEhD,OAAOgE,KAAK,CAAC,6DAA6D;oBACxE5B;oBACAY,QAAQwE,SAASxE,MAAM;gBACzB;gBACA,MAAM,IAAIrB,MACR,CAAC,2BAA2B,EAAES,OAAO+B,aAAa,CAAC,aAAa,EAAEqD,SAASxE,MAAM,EAAE;YAEvF;YAEAhD,OAAOgE,KAAK,CAAC,2CAA2C;gBAAE5B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMc,kBAAkBtF,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACgG,YAAY,GAAG,MAAM,IAAI,CAACnI,IAAI,CAClC8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAACtC,WAAW;YAC9BuC,IAAIC,OAAOC,UAAU;YACrBgF,iBAAiBxF,OAAO+B,aAAa;YACrC0D,WAAWzF,OAAO0F,QAAQ;YAC1BC,MAAM3F,OAAO2F,IAAI;YACjB/E,QAAQ;YACR9C,QAAQsF,KAAKC,SAAS,CAACrD,OAAOlC,MAAM;YACpCiD,SAASqC,KAAKC,SAAS,CAACrD,OAAOe,OAAO;YACtCsD,YAAY,IAAI,CAACjH,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC5BG,YAAY,IAAI,CAACpE,IAAI,CAACwC,GAAG,CAAC;YAC1B6B,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC6D,aAAa;YAChB3H,OAAOgE,KAAK,CAAC,2CAA2C;gBAAE5B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOgG;IACT;IAEA,MAAMK,eAAe5F,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAMgG,cAAc,MAAM,IAAI,CAACnI,IAAI,CAChC8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO6F,aAAa,EAChC5D,KAAK;QAER,OAAOsD,eAAe;IACxB;IAEA,MAAMO,iBAAiB9F,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM4C,QAAQnC,OAAOmC,KAAK,IAAIxE;QAC9B,MAAM,EAAEyE,KAAK,EAAEC,MAAM,EAAE,GAAGrC;QAE1B,IAAIsC,SAAwB;QAC5B,IAAIF,OAAO;YACTE,SAASC,aAAaH;QACxB,OAAO,IAAIC,QAAQ;YACjBC,SAASC,aAAaF;QACxB;QAEA,MAAMpC,KAAK,IAAI,CAAC8F,0BAA0B,CAAC/F,QAAQsC;QACnD,MAAMG,OAAO,MAAMxC,GAChByC,OAAO,CAAC,cAAcL,SAAS,SAAS,OACxCK,OAAO,CAAC,MAAML,SAAS,SAAS,OAChCF,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACQ,wBAAwB,CAClCF,MACAN,OACA,OAAOC,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQ0D,2BAA2B/F,MAA8B,EAAEsC,MAAqB,EAAE;QACxF,MAAM,EAAEF,KAAK,EAAE,GAAGpC;QAClB,MAAMC,KAAK,IAAI,CAAC7C,IAAI,CACjB8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,mBAAmB9B,OAAO+B,aAAa;QAEhD,IAAIO,QAAQ;YACV,MAAMM,WAAWR,QAAQ,MAAM;YAC/B,OAAOnC,GAAG4C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAOhC,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEQ0C,yBACNF,IAAS,EACTN,KAAa,EACb6D,QAAiB,EACjBC,SAAkB,EACI;QACtB,MAAMC,OAAOzD;QACb,IAAI0D,UAAU;QACd,IAAIC,UAAU;QAEd,IAAIH,WAAW;YACbC,KAAKG,OAAO;YACZ,IAAIH,KAAKI,MAAM,GAAGnE,OAAO;gBACvBiE,UAAU;gBACVF,KAAKK,KAAK;YACZ;YACAJ,UAAU;QACZ,OAAO;YACL,IAAID,KAAKI,MAAM,GAAGnE,OAAO;gBACvBgE,UAAU;gBACVD,KAAKM,GAAG;YACV;YACA,IAAIR,UAAU;gBACZI,UAAU;YACZ;QACF;QAEA,MAAMK,WAAWP,KAAKQ,EAAE,CAAC,CAAC;QAC1B,MAAMC,aAAaR,WAAWM,WAAWG,aAAaH,YAAY;QAClE,MAAMI,YAAYX,IAAI,CAAC,EAAE;QACzB,MAAMY,aAAaV,WAAWS,YAAYD,aAAaC,aAAa;QAEpE,OAAO;YACLX;YACAa,YAAY;gBACVC,MAAML;gBACNM,MAAMH;YACR;QACF;IACF;IAEA,MAAMI,oBAAoBlH,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,uBACNgD,MAAM,CAAC;YACNvC,QAAQ;YACRgE,QAAQxB,KAAKC,SAAS,CAACrD,OAAO4E,MAAM;YACpChD,OAAO;YACP4B,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCiD,UAAU,CAAC,GAAG/G,eAAe,oBAAoB,CAAC,EAClDuE,KAAK,CAAC,mBAAmB,IAAI,CAAC/D,WAAW,EACzC+D,KAAK,CAAC,sBAAsB9B,OAAO+B,aAAa,EAChDD,KAAK,CAAC,SAAS9B,OAAO6F,aAAa,EACnC/D,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,oBACvCpC,KAAK,CAAC,SAAS,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,uBAC7BpC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgB9B,OAAOmE,QAAQ,EACrCzC,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,mDAAmD;gBAAE5B;YAAO;YACzE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAM2C,gBAAgBnH,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,uBACNgD,MAAM,CAAC;YACNvC,QAAQ;YACRgE,QAAQ;YACRhD,OAAOwB,KAAKC,SAAS,CAACrD,OAAO4B,KAAK;YAClC4B,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCiD,UAAU,CAAC,GAAG/G,eAAe,oBAAoB,CAAC,EAClDuE,KAAK,CAAC,mBAAmB,IAAI,CAAC/D,WAAW,EACzC+D,KAAK,CAAC,sBAAsB9B,OAAO+B,aAAa,EAChDD,KAAK,CAAC,SAAS9B,OAAO6F,aAAa,EACnC/D,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,oBACvCpC,KAAK,CAAC,SAAS,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,uBAC7BpC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgB9B,OAAOmE,QAAQ,EACrCzC,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,gDAAgD;gBAAE5B;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;AACF;AAaA,SAASoC,aAAaQ,IAAY;IAChC,MAAMC,UAAUC,OAAO3D,IAAI,CACzBP,KAAKC,SAAS,CAAC;QAAEP,WAAWsE,KAAKtE,SAAS,CAACC,WAAW;QAAIzC,IAAI8G,KAAK9G,EAAE;IAAC,IACtEiH,QAAQ,CAAC;IACX,OAAOF;AACT;AAEA,OAAO,SAAS9E,aAAaD,MAAc;IACzC,MAAMkF,UAAUF,OAAO3D,IAAI,CAACrB,QAAQ,UAAUiF,QAAQ,CAAC;IACvD,MAAME,SAASrE,KAAKsE,KAAK,CAACF;IAC1B,OAAO;QACL1E,WAAW,IAAI6E,KAAKF,OAAO3E,SAAS;QACpCxC,IAAImH,OAAOnH,EAAE;IACf;AACF"}
|
|
1
|
+
{"version":3,"sources":["../../src/database/backend.ts"],"sourcesContent":["import { getLogger } from \"@logtape/logtape\";\nimport { camelize } from \"inflection\";\nimport knex, { type Knex } from \"knex\";\nimport {\n type Backend,\n type CancelWorkflowRunParams,\n type ClaimWorkflowRunParams,\n type CompleteStepAttemptParams,\n type CompleteWorkflowRunParams,\n type CreateStepAttemptParams,\n type CreateWorkflowRunParams,\n DEFAULT_NAMESPACE_ID,\n type ExtendWorkflowRunLeaseParams,\n type FailStepAttemptParams,\n type FailWorkflowRunParams,\n type GetStepAttemptParams,\n type GetWorkflowRunParams,\n type ListStepAttemptsParams,\n type ListWorkflowRunsParams,\n type PaginatedResponse,\n type SleepWorkflowRunParams,\n} from \"../backend\";\nimport { mergeRetryPolicy, type SerializableRetryPolicy } from \"../core/retry\";\nimport type { StepAttempt } from \"../core/step\";\nimport type { WorkflowRun } from \"../core/workflow\";\nimport { DEFAULT_SCHEMA, migrate } from \"./base\";\nimport { type OnSubscribed, PostgresPubSub } from \"./pubsub\";\n\nexport const DEFAULT_LISTEN_CHANNEL = \"new_tasks\" as const;\nconst DEFAULT_PAGINATION_PAGE_SIZE = 100 as const;\n\ninterface BackendPostgresOptions {\n namespaceId?: string;\n runMigrations?: boolean;\n\n // default: true\n usePubSub?: boolean;\n}\n\nconst logger = getLogger([\"sonamu\", \"internal\", \"tasks\"]);\nconst queryLogger = getLogger([\"sonamu\", \"internal\", \"tasks\", \"query\"]);\n\n/**\n * Manages a connection to a Postgres database for workflow operations.\n */\nexport class BackendPostgres implements Backend {\n private config: Knex.Config;\n private namespaceId: string;\n private usePubSub: boolean;\n private pubsub: PostgresPubSub | null = null;\n private initialized: boolean = false;\n private runMigrations: boolean;\n\n private _knex: Knex | null = null;\n private get knex(): Knex {\n if (!this._knex) {\n this._knex = knex(this.config);\n this._knex.on(\"query\", (query) => {\n queryLogger.debug(\"SQL: {query}, Values: {bindings}\", {\n query: query.sql,\n bindings: query.bindings,\n });\n });\n }\n\n return this._knex;\n }\n\n constructor(config: Knex.Config, options?: BackendPostgresOptions) {\n this.config = {\n ...config,\n postProcessResponse: (result, _queryContext) => {\n if (result === null || result === undefined) {\n return result;\n }\n\n if (config?.postProcessResponse) {\n result = config.postProcessResponse(result, _queryContext);\n }\n\n const camelizeRow = (row: Record<string, unknown>) =>\n Object.fromEntries(\n Object.entries(row).map(([key, value]) => [camelize(key, true), value]),\n );\n\n if (Array.isArray(result)) {\n return result.map(camelizeRow);\n }\n\n return camelizeRow(result);\n },\n };\n\n const { namespaceId, usePubSub, runMigrations } = {\n namespaceId: DEFAULT_NAMESPACE_ID,\n usePubSub: true,\n runMigrations: true,\n ...options,\n };\n\n this.namespaceId = namespaceId;\n this.usePubSub = usePubSub;\n this.runMigrations = runMigrations;\n }\n\n async initialize() {\n if (this.initialized) {\n return;\n }\n\n if (this.runMigrations) {\n await migrate(this.config, DEFAULT_SCHEMA);\n }\n\n this.initialized = true;\n }\n\n async subscribe(callback: OnSubscribed) {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n if (!this.pubsub) {\n this.pubsub = await PostgresPubSub.create(this.knex);\n }\n\n this.pubsub.listenEvent(DEFAULT_LISTEN_CHANNEL, callback);\n }\n\n async publish(payload?: string): Promise<void> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n await this.knex.raw(\n payload\n ? `NOTIFY ${DEFAULT_LISTEN_CHANNEL}, '${payload}'`\n : `NOTIFY ${DEFAULT_LISTEN_CHANNEL}`,\n );\n }\n\n async stop(): Promise<void> {\n if (!this.initialized) {\n return;\n }\n\n await this.pubsub?.destroy();\n this.pubsub = null;\n await this.knex.destroy();\n }\n\n async createWorkflowRun(params: CreateWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Creating workflow run: {workflowName}:{version}\", {\n workflowName: params.workflowName,\n version: params.version,\n });\n\n // config에 retryPolicy를 포함시킵니다.\n const configWithRetryPolicy = {\n ...(typeof params.config === \"object\" && params.config !== null ? params.config : {}),\n retryPolicy: params.retryPolicy ?? undefined,\n };\n\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_name: params.workflowName,\n version: params.version,\n status: \"pending\",\n idempotency_key: params.idempotencyKey,\n config: JSON.stringify(configWithRetryPolicy),\n context: params.context,\n input: params.input,\n attempts: 0,\n available_at: params.availableAt ?? this.knex.fn.now(),\n deadline_at: params.deadlineAt,\n created_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n const workflowRun = await qb;\n if (!workflowRun[0]) {\n logger.error(\"Failed to create workflow run: {params}\", { params });\n throw new Error(\"Failed to create workflow run\");\n }\n\n return workflowRun[0];\n }\n\n async getWorkflowRun(params: GetWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Getting workflow run: {workflowRunId}\", { workflowRunId: params.workflowRunId });\n const workflowRun = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .select(\n \"namespace_id\",\n \"id\",\n \"workflow_name\",\n \"version\",\n \"status\",\n \"idempotency_key\",\n \"config\",\n \"context\",\n \"input\",\n \"output\",\n \"error\",\n \"attempts\",\n \"parent_step_attempt_namespace_id\",\n \"parent_step_attempt_id\",\n \"worker_id\",\n \"available_at\",\n \"deadline_at\",\n \"started_at\",\n \"finished_at\",\n \"created_at\",\n \"updated_at\",\n )\n .first();\n\n return workflowRun ?? null;\n }\n\n async listWorkflowRuns(params: ListWorkflowRunsParams): Promise<PaginatedResponse<WorkflowRun>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Listing workflow runs: {after}, {before}\", {\n after: params.after,\n before: params.before,\n });\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListWorkflowRunsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListWorkflowRunsWhere(params: ListWorkflowRunsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n async claimWorkflowRun(params: ClaimWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Claiming workflow run: {workerId}, {leaseDurationMs}\", {\n workerId: params.workerId,\n leaseDurationMs: params.leaseDurationMs,\n });\n const claimed = await this.knex\n .with(\"expired\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .update({\n status: \"failed\",\n error: JSON.stringify({ message: \"Workflow run deadline exceeded\" }),\n worker_id: null,\n available_at: null,\n finished_at: this.knex.raw(\"NOW()\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .whereNotNull(\"deadline_at\")\n .where(\"deadline_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .returning(\"id\"),\n )\n .with(\"candidate\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .select(\"id\")\n .from(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .where(\"available_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .where((qb2) => {\n qb2.whereNull(\"deadline_at\").orWhere(\"deadline_at\", \">\", this.knex.raw(\"NOW()\"));\n })\n .orderByRaw(\"CASE WHEN status = 'pending' THEN 0 ELSE 1 END\")\n .orderBy(\"available_at\", \"asc\")\n .orderBy(\"created_at\", \"asc\")\n .limit(1)\n .forUpdate()\n .skipLocked(),\n )\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs as wr\")\n .where(\"wr.namespace_id\", this.namespaceId)\n .where(\"wr.id\", this.knex.ref(\"candidate.id\"))\n .update({\n status: \"running\",\n attempts: this.knex.raw(\"wr.attempts + 1\"),\n worker_id: params.workerId,\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n started_at: this.knex.raw(\"COALESCE(wr.started_at, NOW())\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .updateFrom(\"candidate\")\n .returning(\"wr.*\");\n\n return claimed[0] ?? null;\n }\n\n async extendWorkflowRunLease(params: ExtendWorkflowRunLeaseParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Extending workflow run lease: {workflowRunId}, {workerId}, {leaseDurationMs}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n leaseDurationMs: params.leaseDurationMs,\n });\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to extend lease for workflow run: {params}\", { params });\n throw new Error(\"Failed to extend lease for workflow run\");\n }\n\n return updated;\n }\n\n async sleepWorkflowRun(params: SleepWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Sleeping workflow run: {workflowRunId}, {workerId}, {availableAt}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n availableAt: params.availableAt,\n });\n\n // 'succeeded' status is deprecated\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereNotIn(\"status\", [\"succeeded\", \"completed\", \"failed\", \"canceled\"])\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"sleeping\",\n available_at: params.availableAt,\n worker_id: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to sleep workflow run: {params}\", { params });\n throw new Error(\"Failed to sleep workflow run\");\n }\n\n return updated;\n }\n\n async completeWorkflowRun(params: CompleteWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Completing workflow run: {workflowRunId}, {workerId}, {output}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n output: params.output,\n });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n worker_id: params.workerId,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to complete workflow run: {params}\", { params });\n throw new Error(\"Failed to complete workflow run\");\n }\n\n return updated;\n }\n\n async failWorkflowRun(params: FailWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const { workflowRunId, error, forceComplete, customDelayMs } = params;\n\n logger.info(\"Failing workflow run: {workflowRunId}, {workerId}, {error}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n error: params.error,\n });\n\n const workflowRun = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", workflowRunId)\n .first();\n\n if (!workflowRun) {\n throw new Error(\"Workflow run not found\");\n }\n\n const config =\n typeof workflowRun.config === \"string\" ? JSON.parse(workflowRun.config) : workflowRun.config;\n const savedRetryPolicy: SerializableRetryPolicy | undefined = config?.retryPolicy;\n const retryPolicy = mergeRetryPolicy(savedRetryPolicy);\n\n const { initialIntervalMs, backoffCoefficient, maximumIntervalMs, maxAttempts } = retryPolicy;\n\n const currentAttempts = workflowRun.attempts ?? 0;\n const shouldForceComplete = forceComplete || currentAttempts >= maxAttempts;\n\n if (shouldForceComplete) {\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"failed\",\n available_at: null,\n finished_at: this.knex.fn.now(),\n error: JSON.stringify(error),\n worker_id: null,\n started_at: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to mark workflow run failed: {params}\", { params });\n throw new Error(\"Failed to mark workflow run failed\");\n }\n return updated;\n }\n\n // this beefy query updates a workflow's status, available_at, and\n // finished_at based on the workflow's deadline and retry policy\n //\n // if the next retry would exceed the deadline, the run is marked as\n // 'failed' and finalized, otherwise, the run is rescheduled with an updated\n // 'available_at' timestamp for the next retry\n const retryIntervalExpr = customDelayMs\n ? `${customDelayMs} * INTERVAL '1 millisecond'`\n : `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, \"attempts\" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;\n const deadlineExceededCondition = `\"deadline_at\" IS NOT NULL AND NOW() + (${retryIntervalExpr}) >= \"deadline_at\"`;\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN 'failed' ELSE 'pending' END`,\n ),\n available_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NULL ELSE NOW() + (${retryIntervalExpr}) END`,\n ),\n finished_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NOW() ELSE NULL END`,\n ),\n error: JSON.stringify(error),\n worker_id: null,\n started_at: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to mark workflow run failed: {params}\", { params });\n throw new Error(\"Failed to mark workflow run failed\");\n }\n\n return updated;\n }\n\n async cancelWorkflowRun(params: CancelWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Canceling workflow run: {workflowRunId}\", { workflowRunId: params.workflowRunId });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .update({\n status: \"canceled\",\n worker_id: null,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n // workflow may already be in a terminal state\n const existing = await this.getWorkflowRun({\n workflowRunId: params.workflowRunId,\n });\n if (!existing) {\n throw new Error(`Workflow run ${params.workflowRunId} does not exist`);\n }\n\n // if already canceled, just return it\n if (existing.status === \"canceled\") {\n return existing;\n }\n\n // throw error for completed/failed workflows\n // 'succeeded' status is deprecated\n if ([\"succeeded\", \"completed\", \"failed\"].includes(existing.status)) {\n logger.error(\"Cannot cancel workflow run: {params} with status {status}\", {\n params,\n status: existing.status,\n });\n throw new Error(\n `Cannot cancel workflow run ${params.workflowRunId} with status ${existing.status}`,\n );\n }\n\n logger.error(\"Failed to cancel workflow run: {params}\", { params });\n throw new Error(\"Failed to cancel workflow run\");\n }\n\n return updated;\n }\n\n async createStepAttempt(params: CreateStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Creating step attempt: {workflowRunId}, {stepName}, {kind}\", {\n workflowRunId: params.workflowRunId,\n stepName: params.stepName,\n kind: params.kind,\n });\n\n const [stepAttempt] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_run_id: params.workflowRunId,\n step_name: params.stepName,\n kind: params.kind,\n status: \"running\",\n config: JSON.stringify(params.config),\n context: JSON.stringify(params.context),\n started_at: this.knex.fn.now(),\n created_at: this.knex.raw(\"date_trunc('milliseconds', NOW())\"),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!stepAttempt) {\n logger.error(\"Failed to create step attempt: {params}\", { params });\n throw new Error(\"Failed to create step attempt\");\n }\n\n return stepAttempt;\n }\n\n async getStepAttempt(params: GetStepAttemptParams): Promise<StepAttempt | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Getting step attempt: {stepAttemptId}\", { stepAttemptId: params.stepAttemptId });\n\n const stepAttempt = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.stepAttemptId)\n .first();\n\n return stepAttempt ?? null;\n }\n\n async listStepAttempts(params: ListStepAttemptsParams): Promise<PaginatedResponse<StepAttempt>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Listing step attempts: {workflowRunId}, {after}, {before}\", {\n workflowRunId: params.workflowRunId,\n after: params.after,\n before: params.before,\n });\n\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListStepAttemptsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListStepAttemptsWhere(params: ListStepAttemptsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"workflow_run_id\", params.workflowRunId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n private processPaginationResults<T extends Cursor>(\n rows: T[],\n limit: number,\n hasAfter: boolean,\n hasBefore: boolean,\n ): PaginatedResponse<T> {\n const data = rows;\n let hasNext = false;\n let hasPrev = false;\n\n if (hasBefore) {\n data.reverse();\n if (data.length > limit) {\n hasPrev = true;\n data.shift();\n }\n hasNext = true;\n } else {\n if (data.length > limit) {\n hasNext = true;\n data.pop();\n }\n if (hasAfter) {\n hasPrev = true;\n }\n }\n\n const lastItem = data.at(-1);\n const nextCursor = hasNext && lastItem ? encodeCursor(lastItem) : null;\n const firstItem = data[0];\n const prevCursor = hasPrev && firstItem ? encodeCursor(firstItem) : null;\n\n return {\n data,\n pagination: {\n next: nextCursor,\n prev: prevCursor,\n },\n };\n }\n\n // NOTE: 실제 서비스에서 이게 안 되는 것 같은데, 쿼리 등을 체크할 필요가 있음.\n async completeStepAttempt(params: CompleteStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Marking step attempt as completed: {workflowRunId}, {stepAttemptId}, {workerId}\", {\n workflowRunId: params.workflowRunId,\n stepAttemptId: params.stepAttemptId,\n workerId: params.workerId,\n });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt completed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt completed\");\n }\n\n return updated;\n }\n\n async failStepAttempt(params: FailStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Marking step attempt as failed: {workflowRunId}, {stepAttemptId}, {workerId}\", {\n workflowRunId: params.workflowRunId,\n stepAttemptId: params.stepAttemptId,\n workerId: params.workerId,\n });\n logger.info(\"Error: {error.message}\", { error: params.error.message });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"failed\",\n output: null,\n error: JSON.stringify(params.error),\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt failed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt failed\");\n }\n\n return updated;\n }\n}\n\n/**\n * Cursor used for pagination. Requires created_at and id fields. Because JS\n * Date does not natively support microsecond precision dates, created_at should\n * be stored with millisecond precision in paginated tables to avoid issues with\n * cursor comparisons.\n */\ninterface Cursor {\n createdAt: Date;\n id: string;\n}\n\nfunction encodeCursor(item: Cursor): string {\n const encoded = Buffer.from(\n JSON.stringify({ createdAt: item.createdAt.toISOString(), id: item.id }),\n ).toString(\"base64\");\n return encoded;\n}\n\nexport function decodeCursor(cursor: string): Cursor {\n const decoded = Buffer.from(cursor, \"base64\").toString(\"utf8\");\n const parsed = JSON.parse(decoded) as { createdAt: string; id: string };\n return {\n createdAt: new Date(parsed.createdAt),\n id: parsed.id,\n };\n}\n"],"names":["getLogger","camelize","knex","DEFAULT_NAMESPACE_ID","mergeRetryPolicy","DEFAULT_SCHEMA","migrate","PostgresPubSub","DEFAULT_LISTEN_CHANNEL","DEFAULT_PAGINATION_PAGE_SIZE","logger","queryLogger","BackendPostgres","config","namespaceId","usePubSub","pubsub","initialized","runMigrations","_knex","on","query","debug","sql","bindings","options","postProcessResponse","result","_queryContext","undefined","camelizeRow","row","Object","fromEntries","entries","map","key","value","Array","isArray","initialize","subscribe","callback","Error","create","listenEvent","publish","payload","raw","stop","destroy","createWorkflowRun","params","info","workflowName","version","configWithRetryPolicy","retryPolicy","qb","withSchema","table","insert","namespace_id","id","crypto","randomUUID","workflow_name","status","idempotency_key","idempotencyKey","JSON","stringify","context","input","attempts","available_at","availableAt","fn","now","deadline_at","deadlineAt","created_at","updated_at","returning","workflowRun","error","getWorkflowRun","workflowRunId","where","select","first","listWorkflowRuns","after","before","limit","cursor","decodeCursor","buildListWorkflowRunsWhere","rows","orderBy","processPaginationResults","operator","whereRaw","createdAt","toISOString","claimWorkflowRun","workerId","leaseDurationMs","claimed","with","update","message","worker_id","finished_at","whereIn","whereNotNull","from","qb2","whereNull","orWhere","orderByRaw","forUpdate","skipLocked","ref","started_at","updateFrom","extendWorkflowRunLease","updated","sleepWorkflowRun","whereNotIn","completeWorkflowRun","output","failWorkflowRun","forceComplete","customDelayMs","parse","savedRetryPolicy","initialIntervalMs","backoffCoefficient","maximumIntervalMs","maxAttempts","currentAttempts","shouldForceComplete","retryIntervalExpr","deadlineExceededCondition","cancelWorkflowRun","existing","includes","createStepAttempt","stepName","kind","stepAttempt","workflow_run_id","step_name","getStepAttempt","stepAttemptId","listStepAttempts","buildListStepAttemptsWhere","hasAfter","hasBefore","data","hasNext","hasPrev","reverse","length","shift","pop","lastItem","at","nextCursor","encodeCursor","firstItem","prevCursor","pagination","next","prev","completeStepAttempt","failStepAttempt","item","encoded","Buffer","toString","decoded","parsed","Date"],"mappings":"AAAA,SAASA,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,QAAQ,QAAQ,aAAa;AACtC,OAAOC,UAAyB,OAAO;AACvC,SAQEC,oBAAoB,QAUf,gBAAa;AACpB,SAASC,gBAAgB,QAAsC,mBAAgB;AAG/E,SAASC,cAAc,EAAEC,OAAO,QAAQ,YAAS;AACjD,SAA4BC,cAAc,QAAQ,cAAW;AAE7D,OAAO,MAAMC,yBAAyB,YAAqB;AAC3D,MAAMC,+BAA+B;AAUrC,MAAMC,SAASV,UAAU;IAAC;IAAU;IAAY;CAAQ;AACxD,MAAMW,cAAcX,UAAU;IAAC;IAAU;IAAY;IAAS;CAAQ;AAEtE;;CAEC,GACD,OAAO,MAAMY;IACHC,OAAoB;IACpBC,YAAoB;IACpBC,UAAmB;IACnBC,SAAgC,KAAK;IACrCC,cAAuB,MAAM;IAC7BC,cAAuB;IAEvBC,QAAqB,KAAK;IAClC,IAAYjB,OAAa;QACvB,IAAI,CAAC,IAAI,CAACiB,KAAK,EAAE;YACf,IAAI,CAACA,KAAK,GAAGjB,KAAK,IAAI,CAACW,MAAM;YAC7B,IAAI,CAACM,KAAK,CAACC,EAAE,CAAC,SAAS,CAACC;gBACtBV,YAAYW,KAAK,CAAC,oCAAoC;oBACpDD,OAAOA,MAAME,GAAG;oBAChBC,UAAUH,MAAMG,QAAQ;gBAC1B;YACF;QACF;QAEA,OAAO,IAAI,CAACL,KAAK;IACnB;IAEA,YAAYN,MAAmB,EAAEY,OAAgC,CAAE;QACjE,IAAI,CAACZ,MAAM,GAAG;YACZ,GAAGA,MAAM;YACTa,qBAAqB,CAACC,QAAQC;gBAC5B,IAAID,WAAW,QAAQA,WAAWE,WAAW;oBAC3C,OAAOF;gBACT;gBAEA,IAAId,QAAQa,qBAAqB;oBAC/BC,SAASd,OAAOa,mBAAmB,CAACC,QAAQC;gBAC9C;gBAEA,MAAME,cAAc,CAACC,MACnBC,OAAOC,WAAW,CAChBD,OAAOE,OAAO,CAACH,KAAKI,GAAG,CAAC,CAAC,CAACC,KAAKC,MAAM,GAAK;4BAACpC,SAASmC,KAAK;4BAAOC;yBAAM;gBAG1E,IAAIC,MAAMC,OAAO,CAACZ,SAAS;oBACzB,OAAOA,OAAOQ,GAAG,CAACL;gBACpB;gBAEA,OAAOA,YAAYH;YACrB;QACF;QAEA,MAAM,EAAEb,WAAW,EAAEC,SAAS,EAAEG,aAAa,EAAE,GAAG;YAChDJ,aAAaX;YACbY,WAAW;YACXG,eAAe;YACf,GAAGO,OAAO;QACZ;QAEA,IAAI,CAACX,WAAW,GAAGA;QACnB,IAAI,CAACC,SAAS,GAAGA;QACjB,IAAI,CAACG,aAAa,GAAGA;IACvB;IAEA,MAAMsB,aAAa;QACjB,IAAI,IAAI,CAACvB,WAAW,EAAE;YACpB;QACF;QAEA,IAAI,IAAI,CAACC,aAAa,EAAE;YACtB,MAAMZ,QAAQ,IAAI,CAACO,MAAM,EAAER;QAC7B;QAEA,IAAI,CAACY,WAAW,GAAG;IACrB;IAEA,MAAMwB,UAAUC,QAAsB,EAAE;QACtC,IAAI,CAAC,IAAI,CAACzB,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAAC5B,SAAS,EAAE;YACnB;QACF;QAEA,IAAI,CAAC,IAAI,CAACC,MAAM,EAAE;YAChB,IAAI,CAACA,MAAM,GAAG,MAAMT,eAAeqC,MAAM,CAAC,IAAI,CAAC1C,IAAI;QACrD;QAEA,IAAI,CAACc,MAAM,CAAC6B,WAAW,CAACrC,wBAAwBkC;IAClD;IAEA,MAAMI,QAAQC,OAAgB,EAAiB;QAC7C,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAAC5B,SAAS,EAAE;YACnB;QACF;QAEA,MAAM,IAAI,CAACb,IAAI,CAAC8C,GAAG,CACjBD,UACI,CAAC,OAAO,EAAEvC,uBAAuB,GAAG,EAAEuC,QAAQ,CAAC,CAAC,GAChD,CAAC,OAAO,EAAEvC,wBAAwB;IAE1C;IAEA,MAAMyC,OAAsB;QAC1B,IAAI,CAAC,IAAI,CAAChC,WAAW,EAAE;YACrB;QACF;QAEA,MAAM,IAAI,CAACD,MAAM,EAAEkC;QACnB,IAAI,CAAClC,MAAM,GAAG;QACd,MAAM,IAAI,CAACd,IAAI,CAACgD,OAAO;IACzB;IAEA,MAAMC,kBAAkBC,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,mDAAmD;YAC7DC,cAAcF,OAAOE,YAAY;YACjCC,SAASH,OAAOG,OAAO;QACzB;QAEA,+BAA+B;QAC/B,MAAMC,wBAAwB;YAC5B,GAAI,OAAOJ,OAAOvC,MAAM,KAAK,YAAYuC,OAAOvC,MAAM,KAAK,OAAOuC,OAAOvC,MAAM,GAAG,CAAC,CAAC;YACpF4C,aAAaL,OAAOK,WAAW,IAAI5B;QACrC;QAEA,MAAM6B,KAAK,IAAI,CAACxD,IAAI,CACjByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAAChD,WAAW;YAC9BiD,IAAIC,OAAOC,UAAU;YACrBC,eAAed,OAAOE,YAAY;YAClCC,SAASH,OAAOG,OAAO;YACvBY,QAAQ;YACRC,iBAAiBhB,OAAOiB,cAAc;YACtCxD,QAAQyD,KAAKC,SAAS,CAACf;YACvBgB,SAASpB,OAAOoB,OAAO;YACvBC,OAAOrB,OAAOqB,KAAK;YACnBC,UAAU;YACVC,cAAcvB,OAAOwB,WAAW,IAAI,IAAI,CAAC1E,IAAI,CAAC2E,EAAE,CAACC,GAAG;YACpDC,aAAa3B,OAAO4B,UAAU;YAC9BC,YAAY,IAAI,CAAC/E,IAAI,CAAC2E,EAAE,CAACC,GAAG;YAC5BI,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,MAAMC,cAAc,MAAM1B;QAC1B,IAAI,CAAC0B,WAAW,CAAC,EAAE,EAAE;YACnB1E,OAAO2E,KAAK,CAAC,2CAA2C;gBAAEjC;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOyC,WAAW,CAAC,EAAE;IACvB;IAEA,MAAME,eAAelC,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,yCAAyC;YAAEkC,eAAenC,OAAOmC,aAAa;QAAC;QAC3F,MAAMH,cAAc,MAAM,IAAI,CAAClF,IAAI,CAChCyD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMpC,OAAOmC,aAAa,EAChCE,MAAM,CACL,gBACA,MACA,iBACA,WACA,UACA,mBACA,UACA,WACA,SACA,UACA,SACA,YACA,oCACA,0BACA,aACA,gBACA,eACA,cACA,eACA,cACA,cAEDC,KAAK;QAER,OAAON,eAAe;IACxB;IAEA,MAAMO,iBAAiBvC,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,4CAA4C;YACtDuC,OAAOxC,OAAOwC,KAAK;YACnBC,QAAQzC,OAAOyC,MAAM;QACvB;QACA,MAAMC,QAAQ1C,OAAO0C,KAAK,IAAIrF;QAC9B,MAAM,EAAEmF,KAAK,EAAEC,MAAM,EAAE,GAAGzC;QAE1B,IAAI2C,SAAwB;QAC5B,IAAIH,OAAO;YACTG,SAASC,aAAaJ;QACxB,OAAO,IAAIC,QAAQ;YACjBE,SAASC,aAAaH;QACxB;QAEA,MAAMnC,KAAK,IAAI,CAACuC,0BAA0B,CAAC7C,QAAQ2C;QACnD,MAAMG,OAAO,MAAMxC,GAChByC,OAAO,CAAC,cAAcN,SAAS,SAAS,OACxCM,OAAO,CAAC,MAAMN,SAAS,SAAS,OAChCC,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACM,wBAAwB,CAClCF,MACAJ,OACA,OAAOF,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQI,2BAA2B7C,MAA8B,EAAE2C,MAAqB,EAAE;QACxF,MAAM,EAAEH,KAAK,EAAE,GAAGxC;QAClB,MAAMM,KAAK,IAAI,CAACxD,IAAI,CACjByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW;QAEzC,IAAIiF,QAAQ;YACV,MAAMM,WAAWT,QAAQ,MAAM;YAC/B,OAAOlC,GAAG4C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAOhC,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEA,MAAM+C,iBAAiBrD,MAA8B,EAA+B;QAClF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,wDAAwD;YAClEqD,UAAUtD,OAAOsD,QAAQ;YACzBC,iBAAiBvD,OAAOuD,eAAe;QACzC;QACA,MAAMC,UAAU,MAAM,IAAI,CAAC1G,IAAI,CAC5B2G,IAAI,CAAC,WAAW,CAACnD,KAChBA,GACGC,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACNkD,MAAM,CAAC;gBACN3C,QAAQ;gBACRkB,OAAOf,KAAKC,SAAS,CAAC;oBAAEwC,SAAS;gBAAiC;gBAClEC,WAAW;gBACXrC,cAAc;gBACdsC,aAAa,IAAI,CAAC/G,IAAI,CAAC8C,GAAG,CAAC;gBAC3BkC,YAAY,IAAI,CAAChF,IAAI,CAAC8C,GAAG,CAAC;YAC5B,GACCwC,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtCoG,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpDC,YAAY,CAAC,eACb3B,KAAK,CAAC,eAAe,MAAM,IAAI,CAACtF,IAAI,CAAC8C,GAAG,CAAC,UACzCmC,SAAS,CAAC,OAEd0B,IAAI,CAAC,aAAa,CAACnD,KAClBA,GACGC,UAAU,CAACtD,gBACXoF,MAAM,CAAC,MACP2B,IAAI,CAAC,iBACL5B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtCoG,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpD1B,KAAK,CAAC,gBAAgB,MAAM,IAAI,CAACtF,IAAI,CAAC8C,GAAG,CAAC,UAC1CwC,KAAK,CAAC,CAAC6B;gBACNA,IAAIC,SAAS,CAAC,eAAeC,OAAO,CAAC,eAAe,KAAK,IAAI,CAACrH,IAAI,CAAC8C,GAAG,CAAC;YACzE,GACCwE,UAAU,CAAC,kDACXrB,OAAO,CAAC,gBAAgB,OACxBA,OAAO,CAAC,cAAc,OACtBL,KAAK,CAAC,GACN2B,SAAS,GACTC,UAAU,IAEd/D,UAAU,CAACtD,gBACXuD,KAAK,CAAC,uBACN4B,KAAK,CAAC,mBAAmB,IAAI,CAAC1E,WAAW,EACzC0E,KAAK,CAAC,SAAS,IAAI,CAACtF,IAAI,CAACyH,GAAG,CAAC,iBAC7Bb,MAAM,CAAC;YACN3C,QAAQ;YACRO,UAAU,IAAI,CAACxE,IAAI,CAAC8C,GAAG,CAAC;YACxBgE,WAAW5D,OAAOsD,QAAQ;YAC1B/B,cAAc,IAAI,CAACzE,IAAI,CAAC8C,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOuD,eAAe,CAAC,2BAA2B,CAAC;YAC1FiB,YAAY,IAAI,CAAC1H,IAAI,CAAC8C,GAAG,CAAC;YAC1BkC,YAAY,IAAI,CAAChF,IAAI,CAAC8C,GAAG,CAAC;QAC5B,GACC6E,UAAU,CAAC,aACX1C,SAAS,CAAC;QAEb,OAAOyB,OAAO,CAAC,EAAE,IAAI;IACvB;IAEA,MAAMkB,uBAAuB1E,MAAoC,EAAwB;QACvF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,gFAAgF;YAC1FkC,eAAenC,OAAOmC,aAAa;YACnCmB,UAAUtD,OAAOsD,QAAQ;YACzBC,iBAAiBvD,OAAOuD,eAAe;QACzC;QACA,MAAM,CAACoB,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMpC,OAAOmC,aAAa,EAChCC,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAapC,OAAOsD,QAAQ,EAClCI,MAAM,CAAC;YACNnC,cAAc,IAAI,CAACzE,IAAI,CAAC8C,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOuD,eAAe,CAAC,2BAA2B,CAAC;YAC1FzB,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC4C,SAAS;YACZrH,OAAO2E,KAAK,CAAC,qDAAqD;gBAAEjC;YAAO;YAC3E,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoF;IACT;IAEA,MAAMC,iBAAiB5E,MAA8B,EAAwB;QAC3E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,qEAAqE;YAC/EkC,eAAenC,OAAOmC,aAAa;YACnCmB,UAAUtD,OAAOsD,QAAQ;YACzB9B,aAAaxB,OAAOwB,WAAW;QACjC;QAEA,mCAAmC;QACnC,MAAM,CAACmD,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMpC,OAAOmC,aAAa,EAChC0C,UAAU,CAAC,UAAU;YAAC;YAAa;YAAa;YAAU;SAAW,EACrEzC,KAAK,CAAC,aAAapC,OAAOsD,QAAQ,EAClCI,MAAM,CAAC;YACN3C,QAAQ;YACRQ,cAAcvB,OAAOwB,WAAW;YAChCoC,WAAW;YACX9B,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC4C,SAAS;YACZrH,OAAO2E,KAAK,CAAC,0CAA0C;gBAAEjC;YAAO;YAChE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoF;IACT;IAEA,MAAMG,oBAAoB9E,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,kEAAkE;YAC5EkC,eAAenC,OAAOmC,aAAa;YACnCmB,UAAUtD,OAAOsD,QAAQ;YACzByB,QAAQ/E,OAAO+E,MAAM;QACvB;QAEA,MAAM,CAACJ,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMpC,OAAOmC,aAAa,EAChCC,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAapC,OAAOsD,QAAQ,EAClCI,MAAM,CAAC;YACN3C,QAAQ;YACRgE,QAAQ7D,KAAKC,SAAS,CAACnB,OAAO+E,MAAM;YACpC9C,OAAO;YACP2B,WAAW5D,OAAOsD,QAAQ;YAC1B/B,cAAc;YACdsC,aAAa,IAAI,CAAC/G,IAAI,CAAC2E,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC4C,SAAS;YACZrH,OAAO2E,KAAK,CAAC,6CAA6C;gBAAEjC;YAAO;YACnE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoF;IACT;IAEA,MAAMK,gBAAgBhF,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEA,MAAM,EAAE4C,aAAa,EAAEF,KAAK,EAAEgD,aAAa,EAAEC,aAAa,EAAE,GAAGlF;QAE/D1C,OAAO2C,IAAI,CAAC,8DAA8D;YACxEkC,eAAenC,OAAOmC,aAAa;YACnCmB,UAAUtD,OAAOsD,QAAQ;YACzBrB,OAAOjC,OAAOiC,KAAK;QACrB;QAEA,MAAMD,cAAc,MAAM,IAAI,CAAClF,IAAI,CAChCyD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMD,eACZG,KAAK;QAER,IAAI,CAACN,aAAa;YAChB,MAAM,IAAIzC,MAAM;QAClB;QAEA,MAAM9B,SACJ,OAAOuE,YAAYvE,MAAM,KAAK,WAAWyD,KAAKiE,KAAK,CAACnD,YAAYvE,MAAM,IAAIuE,YAAYvE,MAAM;QAC9F,MAAM2H,mBAAwD3H,QAAQ4C;QACtE,MAAMA,cAAcrD,iBAAiBoI;QAErC,MAAM,EAAEC,iBAAiB,EAAEC,kBAAkB,EAAEC,iBAAiB,EAAEC,WAAW,EAAE,GAAGnF;QAElF,MAAMoF,kBAAkBzD,YAAYV,QAAQ,IAAI;QAChD,MAAMoE,sBAAsBT,iBAAiBQ,mBAAmBD;QAEhE,IAAIE,qBAAqB;YACvB,MAAM,CAACf,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMD,eACZC,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAapC,OAAOsD,QAAQ,EAClCI,MAAM,CAAC;gBACN3C,QAAQ;gBACRQ,cAAc;gBACdsC,aAAa,IAAI,CAAC/G,IAAI,CAAC2E,EAAE,CAACC,GAAG;gBAC7BO,OAAOf,KAAKC,SAAS,CAACc;gBACtB2B,WAAW;gBACXY,YAAY;gBACZ1C,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;YAC9B,GACCK,SAAS,CAAC;YAEb,IAAI,CAAC4C,SAAS;gBACZrH,OAAO2E,KAAK,CAAC,gDAAgD;oBAAEjC;gBAAO;gBACtE,MAAM,IAAIT,MAAM;YAClB;YACA,OAAOoF;QACT;QAEA,kEAAkE;QAClE,gEAAgE;QAChE,EAAE;QACF,oEAAoE;QACpE,4EAA4E;QAC5E,8CAA8C;QAC9C,MAAMgB,oBAAoBT,gBACtB,GAAGA,cAAc,2BAA2B,CAAC,GAC7C,CAAC,MAAM,EAAEG,kBAAkB,SAAS,EAAEC,mBAAmB,mBAAmB,EAAEC,kBAAkB,4BAA4B,CAAC;QACjI,MAAMK,4BAA4B,CAAC,uCAAuC,EAAED,kBAAkB,kBAAkB,CAAC;QAEjH,MAAM,CAAChB,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMD,eACZC,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAapC,OAAOsD,QAAQ,EAClCI,MAAM,CAAC;YACN3C,QAAQ,IAAI,CAACjE,IAAI,CAAC8C,GAAG,CACnB,CAAC,UAAU,EAAEgG,0BAA0B,iCAAiC,CAAC;YAE3ErE,cAAc,IAAI,CAACzE,IAAI,CAAC8C,GAAG,CACzB,CAAC,UAAU,EAAEgG,0BAA0B,yBAAyB,EAAED,kBAAkB,KAAK,CAAC;YAE5F9B,aAAa,IAAI,CAAC/G,IAAI,CAAC8C,GAAG,CACxB,CAAC,UAAU,EAAEgG,0BAA0B,yBAAyB,CAAC;YAEnE3D,OAAOf,KAAKC,SAAS,CAACc;YACtB2B,WAAW;YACXY,YAAY;YACZ1C,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC4C,SAAS;YACZrH,OAAO2E,KAAK,CAAC,gDAAgD;gBAAEjC;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoF;IACT;IAEA,MAAMkB,kBAAkB7F,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,2CAA2C;YAAEkC,eAAenC,OAAOmC,aAAa;QAAC;QAE7F,MAAM,CAACwC,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMpC,OAAOmC,aAAa,EAChC2B,OAAO,CAAC,UAAU;YAAC;YAAW;YAAW;SAAW,EACpDJ,MAAM,CAAC;YACN3C,QAAQ;YACR6C,WAAW;YACXrC,cAAc;YACdsC,aAAa,IAAI,CAAC/G,IAAI,CAAC2E,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC4C,SAAS;YACZ,8CAA8C;YAC9C,MAAMmB,WAAW,MAAM,IAAI,CAAC5D,cAAc,CAAC;gBACzCC,eAAenC,OAAOmC,aAAa;YACrC;YACA,IAAI,CAAC2D,UAAU;gBACb,MAAM,IAAIvG,MAAM,CAAC,aAAa,EAAES,OAAOmC,aAAa,CAAC,eAAe,CAAC;YACvE;YAEA,sCAAsC;YACtC,IAAI2D,SAAS/E,MAAM,KAAK,YAAY;gBAClC,OAAO+E;YACT;YAEA,6CAA6C;YAC7C,mCAAmC;YACnC,IAAI;gBAAC;gBAAa;gBAAa;aAAS,CAACC,QAAQ,CAACD,SAAS/E,MAAM,GAAG;gBAClEzD,OAAO2E,KAAK,CAAC,6DAA6D;oBACxEjC;oBACAe,QAAQ+E,SAAS/E,MAAM;gBACzB;gBACA,MAAM,IAAIxB,MACR,CAAC,2BAA2B,EAAES,OAAOmC,aAAa,CAAC,aAAa,EAAE2D,SAAS/E,MAAM,EAAE;YAEvF;YAEAzD,OAAO2E,KAAK,CAAC,2CAA2C;gBAAEjC;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoF;IACT;IAEA,MAAMqB,kBAAkBhG,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,8DAA8D;YACxEkC,eAAenC,OAAOmC,aAAa;YACnC8D,UAAUjG,OAAOiG,QAAQ;YACzBC,MAAMlG,OAAOkG,IAAI;QACnB;QAEA,MAAM,CAACC,YAAY,GAAG,MAAM,IAAI,CAACrJ,IAAI,CAClCyD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAAChD,WAAW;YAC9BiD,IAAIC,OAAOC,UAAU;YACrBuF,iBAAiBpG,OAAOmC,aAAa;YACrCkE,WAAWrG,OAAOiG,QAAQ;YAC1BC,MAAMlG,OAAOkG,IAAI;YACjBnF,QAAQ;YACRtD,QAAQyD,KAAKC,SAAS,CAACnB,OAAOvC,MAAM;YACpC2D,SAASF,KAAKC,SAAS,CAACnB,OAAOoB,OAAO;YACtCoD,YAAY,IAAI,CAAC1H,IAAI,CAAC2E,EAAE,CAACC,GAAG;YAC5BG,YAAY,IAAI,CAAC/E,IAAI,CAAC8C,GAAG,CAAC;YAC1BkC,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAACoE,aAAa;YAChB7I,OAAO2E,KAAK,CAAC,2CAA2C;gBAAEjC;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAO4G;IACT;IAEA,MAAMG,eAAetG,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,yCAAyC;YAAEsG,eAAevG,OAAOuG,aAAa;QAAC;QAE3F,MAAMJ,cAAc,MAAM,IAAI,CAACrJ,IAAI,CAChCyD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,MAAMpC,OAAOuG,aAAa,EAChCjE,KAAK;QAER,OAAO6D,eAAe;IACxB;IAEA,MAAMK,iBAAiBxG,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,6DAA6D;YACvEkC,eAAenC,OAAOmC,aAAa;YACnCK,OAAOxC,OAAOwC,KAAK;YACnBC,QAAQzC,OAAOyC,MAAM;QACvB;QAEA,MAAMC,QAAQ1C,OAAO0C,KAAK,IAAIrF;QAC9B,MAAM,EAAEmF,KAAK,EAAEC,MAAM,EAAE,GAAGzC;QAE1B,IAAI2C,SAAwB;QAC5B,IAAIH,OAAO;YACTG,SAASC,aAAaJ;QACxB,OAAO,IAAIC,QAAQ;YACjBE,SAASC,aAAaH;QACxB;QAEA,MAAMnC,KAAK,IAAI,CAACmG,0BAA0B,CAACzG,QAAQ2C;QACnD,MAAMG,OAAO,MAAMxC,GAChByC,OAAO,CAAC,cAAcN,SAAS,SAAS,OACxCM,OAAO,CAAC,MAAMN,SAAS,SAAS,OAChCC,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACM,wBAAwB,CAClCF,MACAJ,OACA,OAAOF,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQgE,2BAA2BzG,MAA8B,EAAE2C,MAAqB,EAAE;QACxF,MAAM,EAAEH,KAAK,EAAE,GAAGxC;QAClB,MAAMM,KAAK,IAAI,CAACxD,IAAI,CACjByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,iBACN4B,KAAK,CAAC,gBAAgB,IAAI,CAAC1E,WAAW,EACtC0E,KAAK,CAAC,mBAAmBpC,OAAOmC,aAAa;QAEhD,IAAIQ,QAAQ;YACV,MAAMM,WAAWT,QAAQ,MAAM;YAC/B,OAAOlC,GAAG4C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAOhC,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEQ0C,yBACNF,IAAS,EACTJ,KAAa,EACbgE,QAAiB,EACjBC,SAAkB,EACI;QACtB,MAAMC,OAAO9D;QACb,IAAI+D,UAAU;QACd,IAAIC,UAAU;QAEd,IAAIH,WAAW;YACbC,KAAKG,OAAO;YACZ,IAAIH,KAAKI,MAAM,GAAGtE,OAAO;gBACvBoE,UAAU;gBACVF,KAAKK,KAAK;YACZ;YACAJ,UAAU;QACZ,OAAO;YACL,IAAID,KAAKI,MAAM,GAAGtE,OAAO;gBACvBmE,UAAU;gBACVD,KAAKM,GAAG;YACV;YACA,IAAIR,UAAU;gBACZI,UAAU;YACZ;QACF;QAEA,MAAMK,WAAWP,KAAKQ,EAAE,CAAC,CAAC;QAC1B,MAAMC,aAAaR,WAAWM,WAAWG,aAAaH,YAAY;QAClE,MAAMI,YAAYX,IAAI,CAAC,EAAE;QACzB,MAAMY,aAAaV,WAAWS,YAAYD,aAAaC,aAAa;QAEpE,OAAO;YACLX;YACAa,YAAY;gBACVC,MAAML;gBACNM,MAAMH;YACR;QACF;IACF;IAEA,kDAAkD;IAClD,MAAMI,oBAAoB5H,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,mFAAmF;YAC7FkC,eAAenC,OAAOmC,aAAa;YACnCoE,eAAevG,OAAOuG,aAAa;YACnCjD,UAAUtD,OAAOsD,QAAQ;QAC3B;QAEA,MAAM,CAACqB,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,uBACNkD,MAAM,CAAC;YACN3C,QAAQ;YACRgE,QAAQ7D,KAAKC,SAAS,CAACnB,OAAO+E,MAAM;YACpC9C,OAAO;YACP4B,aAAa,IAAI,CAAC/G,IAAI,CAAC2E,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACC+C,UAAU,CAAC,GAAGxH,eAAe,oBAAoB,CAAC,EAClDmF,KAAK,CAAC,mBAAmB,IAAI,CAAC1E,WAAW,EACzC0E,KAAK,CAAC,sBAAsBpC,OAAOmC,aAAa,EAChDC,KAAK,CAAC,SAASpC,OAAOuG,aAAa,EACnCnE,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAACtF,IAAI,CAACyH,GAAG,CAAC,oBACvCnC,KAAK,CAAC,SAAS,IAAI,CAACtF,IAAI,CAACyH,GAAG,CAAC,uBAC7BnC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgBpC,OAAOsD,QAAQ,EACrCvB,SAAS,CAAC;QAEb,IAAI,CAAC4C,SAAS;YACZrH,OAAO2E,KAAK,CAAC,mDAAmD;gBAAEjC;YAAO;YACzE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoF;IACT;IAEA,MAAMkD,gBAAgB7H,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,gFAAgF;YAC1FkC,eAAenC,OAAOmC,aAAa;YACnCoE,eAAevG,OAAOuG,aAAa;YACnCjD,UAAUtD,OAAOsD,QAAQ;QAC3B;QACAhG,OAAO2C,IAAI,CAAC,0BAA0B;YAAEgC,OAAOjC,OAAOiC,KAAK,CAAC0B,OAAO;QAAC;QAEpE,MAAM,CAACgB,QAAQ,GAAG,MAAM,IAAI,CAAC7H,IAAI,CAC9ByD,UAAU,CAACtD,gBACXuD,KAAK,CAAC,uBACNkD,MAAM,CAAC;YACN3C,QAAQ;YACRgE,QAAQ;YACR9C,OAAOf,KAAKC,SAAS,CAACnB,OAAOiC,KAAK;YAClC4B,aAAa,IAAI,CAAC/G,IAAI,CAAC2E,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAChF,IAAI,CAAC2E,EAAE,CAACC,GAAG;QAC9B,GACC+C,UAAU,CAAC,GAAGxH,eAAe,oBAAoB,CAAC,EAClDmF,KAAK,CAAC,mBAAmB,IAAI,CAAC1E,WAAW,EACzC0E,KAAK,CAAC,sBAAsBpC,OAAOmC,aAAa,EAChDC,KAAK,CAAC,SAASpC,OAAOuG,aAAa,EACnCnE,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAACtF,IAAI,CAACyH,GAAG,CAAC,oBACvCnC,KAAK,CAAC,SAAS,IAAI,CAACtF,IAAI,CAACyH,GAAG,CAAC,uBAC7BnC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgBpC,OAAOsD,QAAQ,EACrCvB,SAAS,CAAC;QAEb,IAAI,CAAC4C,SAAS;YACZrH,OAAO2E,KAAK,CAAC,gDAAgD;gBAAEjC;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoF;IACT;AACF;AAaA,SAAS2C,aAAaQ,IAAY;IAChC,MAAMC,UAAUC,OAAOhE,IAAI,CACzB9C,KAAKC,SAAS,CAAC;QAAEgC,WAAW2E,KAAK3E,SAAS,CAACC,WAAW;QAAIzC,IAAImH,KAAKnH,EAAE;IAAC,IACtEsH,QAAQ,CAAC;IACX,OAAOF;AACT;AAEA,OAAO,SAASnF,aAAaD,MAAc;IACzC,MAAMuF,UAAUF,OAAOhE,IAAI,CAACrB,QAAQ,UAAUsF,QAAQ,CAAC;IACvD,MAAME,SAASjH,KAAKiE,KAAK,CAAC+C;IAC1B,OAAO;QACL/E,WAAW,IAAIiF,KAAKD,OAAOhF,SAAS;QACpCxC,IAAIwH,OAAOxH,EAAE;IACf;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backend.testsuite.d.ts","sourceRoot":"","sources":["../../src/database/backend.testsuite.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"backend.testsuite.d.ts","sourceRoot":"","sources":["../../src/database/backend.testsuite.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAK3C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9B;;OAEG;IACH,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/C;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,IAAI,CA43C7D"}
|
|
@@ -453,6 +453,112 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
|
453
453
|
expect(delayMs).toBeLessThan(2500);
|
|
454
454
|
await teardown(backend);
|
|
455
455
|
});
|
|
456
|
+
test("marks workflow run as failed when maxAttempts is reached", async ()=>{
|
|
457
|
+
const backend = await setup();
|
|
458
|
+
// retryPolicy에 maxAttempts: 2를 지정하여 생성
|
|
459
|
+
const retryPolicy = {
|
|
460
|
+
maxAttempts: 2,
|
|
461
|
+
initialIntervalMs: 100
|
|
462
|
+
};
|
|
463
|
+
await backend.createWorkflowRun({
|
|
464
|
+
workflowName: randomUUID(),
|
|
465
|
+
version: null,
|
|
466
|
+
idempotencyKey: null,
|
|
467
|
+
input: null,
|
|
468
|
+
config: {},
|
|
469
|
+
context: null,
|
|
470
|
+
availableAt: null,
|
|
471
|
+
deadlineAt: null,
|
|
472
|
+
retryPolicy
|
|
473
|
+
});
|
|
474
|
+
// 첫 번째 시도 - 실패하면 pending으로 스케줄링
|
|
475
|
+
let workerId = randomUUID();
|
|
476
|
+
let claimed = await backend.claimWorkflowRun({
|
|
477
|
+
workerId,
|
|
478
|
+
leaseDurationMs: 100
|
|
479
|
+
});
|
|
480
|
+
if (!claimed) throw new Error("Expected workflow run to be claimed");
|
|
481
|
+
expect(claimed.attempts).toBe(1);
|
|
482
|
+
const firstFailed = await backend.failWorkflowRun({
|
|
483
|
+
workflowRunId: claimed.id,
|
|
484
|
+
workerId,
|
|
485
|
+
error: {
|
|
486
|
+
message: "first failure"
|
|
487
|
+
}
|
|
488
|
+
});
|
|
489
|
+
expect(firstFailed.status).toBe("pending"); // 아직 maxAttempts(2) 미달
|
|
490
|
+
await sleep(150); // 100ms backoff 대기
|
|
491
|
+
// 두 번째 시도 - maxAttempts에 도달하면 failed로 종료
|
|
492
|
+
workerId = randomUUID();
|
|
493
|
+
claimed = await backend.claimWorkflowRun({
|
|
494
|
+
workerId,
|
|
495
|
+
leaseDurationMs: 100
|
|
496
|
+
});
|
|
497
|
+
if (!claimed) throw new Error("Expected workflow run to be claimed");
|
|
498
|
+
expect(claimed.attempts).toBe(2);
|
|
499
|
+
const secondFailed = await backend.failWorkflowRun({
|
|
500
|
+
workflowRunId: claimed.id,
|
|
501
|
+
workerId,
|
|
502
|
+
error: {
|
|
503
|
+
message: "second failure"
|
|
504
|
+
}
|
|
505
|
+
});
|
|
506
|
+
// maxAttempts에 도달했으므로 failed로 종료
|
|
507
|
+
expect(secondFailed.status).toBe("failed");
|
|
508
|
+
expect(secondFailed.availableAt).toBeNull();
|
|
509
|
+
expect(secondFailed.finishedAt).not.toBeNull();
|
|
510
|
+
await teardown(backend);
|
|
511
|
+
});
|
|
512
|
+
test("marks workflow run as failed immediately when forceComplete is true", async ()=>{
|
|
513
|
+
const backend = await setup();
|
|
514
|
+
await createPendingWorkflowRun(backend);
|
|
515
|
+
const workerId = randomUUID();
|
|
516
|
+
const claimed = await backend.claimWorkflowRun({
|
|
517
|
+
workerId,
|
|
518
|
+
leaseDurationMs: 100
|
|
519
|
+
});
|
|
520
|
+
if (!claimed) throw new Error("Expected workflow run to be claimed");
|
|
521
|
+
// forceComplete: true로 호출하면 재시도 없이 즉시 failed
|
|
522
|
+
const failed = await backend.failWorkflowRun({
|
|
523
|
+
workflowRunId: claimed.id,
|
|
524
|
+
workerId,
|
|
525
|
+
error: {
|
|
526
|
+
message: "forced failure"
|
|
527
|
+
},
|
|
528
|
+
forceComplete: true
|
|
529
|
+
});
|
|
530
|
+
expect(failed.status).toBe("failed");
|
|
531
|
+
expect(failed.availableAt).toBeNull();
|
|
532
|
+
expect(failed.finishedAt).not.toBeNull();
|
|
533
|
+
await teardown(backend);
|
|
534
|
+
});
|
|
535
|
+
test("stores retryPolicy in config when creating workflow run", async ()=>{
|
|
536
|
+
const backend = await setup();
|
|
537
|
+
const retryPolicy = {
|
|
538
|
+
maxAttempts: 10,
|
|
539
|
+
initialIntervalMs: 500,
|
|
540
|
+
backoffCoefficient: 1.5,
|
|
541
|
+
maximumIntervalMs: 30000
|
|
542
|
+
};
|
|
543
|
+
const created = await backend.createWorkflowRun({
|
|
544
|
+
workflowName: randomUUID(),
|
|
545
|
+
version: null,
|
|
546
|
+
idempotencyKey: null,
|
|
547
|
+
input: null,
|
|
548
|
+
config: {
|
|
549
|
+
existingKey: "existingValue"
|
|
550
|
+
},
|
|
551
|
+
context: null,
|
|
552
|
+
availableAt: null,
|
|
553
|
+
deadlineAt: null,
|
|
554
|
+
retryPolicy
|
|
555
|
+
});
|
|
556
|
+
// config에 retryPolicy가 저장되어 있는지 확인
|
|
557
|
+
const config = created.config;
|
|
558
|
+
expect(config.existingKey).toBe("existingValue");
|
|
559
|
+
expect(config.retryPolicy).toEqual(retryPolicy);
|
|
560
|
+
await teardown(backend);
|
|
561
|
+
});
|
|
456
562
|
});
|
|
457
563
|
describe("createStepAttempt()", ()=>{
|
|
458
564
|
test("creates a step attempt", async ()=>{
|