@sonamu-kit/tasks 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../src/database/backend.ts"],"names":[],"mappings":"AAEA,OAAa,EAAE,KAAK,IAAI,EAAE,MAAM,MAAM,CAAC;AACvC,OAAO,EACL,KAAK,OAAO,EACZ,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAE5B,KAAK,4BAA4B,EACjC,KAAK,qBAAqB,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC5B,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,KAAK,YAAY,EAAkB,MAAM,UAAU,CAAC;AAE7D,eAAO,MAAM,sBAAsB,EAAG,WAAoB,CAAC;AAG3D,UAAU,sBAAsB;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;
|
|
1
|
+
{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../src/database/backend.ts"],"names":[],"mappings":"AAEA,OAAa,EAAE,KAAK,IAAI,EAAE,MAAM,MAAM,CAAC;AACvC,OAAO,EACL,KAAK,OAAO,EACZ,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAE5B,KAAK,4BAA4B,EACjC,KAAK,qBAAqB,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC5B,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EAAE,KAAK,YAAY,EAAkB,MAAM,UAAU,CAAC;AAE7D,eAAO,MAAM,sBAAsB,EAAG,WAAoB,CAAC;AAG3D,UAAU,sBAAsB;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAKD;;GAEG;AACH,qBAAa,eAAgB,YAAW,OAAO;IAC7C,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,aAAa,CAAU;IAE/B,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,KAAK,IAAI,GAYf;gBAEW,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAqC3D,UAAU;IAYV,SAAS,CAAC,QAAQ,EAAE,YAAY;IAgBhC,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBxC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrB,iBAAiB,CAAC,MAAM,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IAwCxE,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAuCzE,gBAAgB,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAiC/F,OAAO,CAAC,0BAA0B;IAkB5B,gBAAgB,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAgE7E,sBAAsB,CAAC,MAAM,EAAE,4BAA4B,GAAG,OAAO,CAAC,WAAW,CAAC;IA+BlF,gBAAgB,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,CAAC;IAmCtE,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,WAAW,CAAC;IAqC5E,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,WAAW,CAAC;IAuDpE,iBAAiB,CAAC,MAAM,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IAuDxE,iBAAiB,CAAC,MAAM,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IAqCxE,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAiBzE,gBAAgB,CAAC,MAAM,EAAE,sBAAsB,GAAG,OAAO,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAmC/F,OAAO,CAAC,0BAA0B;IAmBlC,OAAO,CAAC,wBAAwB;IA0C1B,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,WAAW,CAAC;IAwC5E,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,WAAW,CAAC;CAwC3E;AAED;;;;;GAKG;AACH,UAAU,MAAM;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;CACZ;AASD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAOnD"}
|
package/dist/database/backend.js
CHANGED
|
@@ -12,6 +12,12 @@ const logger = getLogger([
|
|
|
12
12
|
"internal",
|
|
13
13
|
"tasks"
|
|
14
14
|
]);
|
|
15
|
+
const queryLogger = getLogger([
|
|
16
|
+
"sonamu",
|
|
17
|
+
"internal",
|
|
18
|
+
"tasks",
|
|
19
|
+
"query"
|
|
20
|
+
]);
|
|
15
21
|
/**
|
|
16
22
|
* Manages a connection to a Postgres database for workflow operations.
|
|
17
23
|
*/ export class BackendPostgres {
|
|
@@ -25,6 +31,12 @@ const logger = getLogger([
|
|
|
25
31
|
get knex() {
|
|
26
32
|
if (!this._knex) {
|
|
27
33
|
this._knex = knex(this.config);
|
|
34
|
+
this._knex.on("query", (query)=>{
|
|
35
|
+
queryLogger.debug("SQL: {query}, Values: {bindings}", {
|
|
36
|
+
query: query.sql,
|
|
37
|
+
bindings: query.bindings
|
|
38
|
+
});
|
|
39
|
+
});
|
|
28
40
|
}
|
|
29
41
|
return this._knex;
|
|
30
42
|
}
|
|
@@ -100,6 +112,10 @@ const logger = getLogger([
|
|
|
100
112
|
if (!this.initialized) {
|
|
101
113
|
throw new Error("Backend not initialized");
|
|
102
114
|
}
|
|
115
|
+
logger.info("Creating workflow run: {workflowName}:{version}", {
|
|
116
|
+
workflowName: params.workflowName,
|
|
117
|
+
version: params.version
|
|
118
|
+
});
|
|
103
119
|
const qb = this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").insert({
|
|
104
120
|
namespace_id: this.namespaceId,
|
|
105
121
|
id: crypto.randomUUID(),
|
|
@@ -129,6 +145,9 @@ const logger = getLogger([
|
|
|
129
145
|
if (!this.initialized) {
|
|
130
146
|
throw new Error("Backend not initialized");
|
|
131
147
|
}
|
|
148
|
+
logger.info("Getting workflow run: {workflowRunId}", {
|
|
149
|
+
workflowRunId: params.workflowRunId
|
|
150
|
+
});
|
|
132
151
|
const workflowRun = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).select("namespace_id", "id", "workflow_name", "version", "status", "idempotency_key", "config", "context", "input", "output", "error", "attempts", "parent_step_attempt_namespace_id", "parent_step_attempt_id", "worker_id", "available_at", "deadline_at", "started_at", "finished_at", "created_at", "updated_at").first();
|
|
133
152
|
return workflowRun ?? null;
|
|
134
153
|
}
|
|
@@ -136,6 +155,10 @@ const logger = getLogger([
|
|
|
136
155
|
if (!this.initialized) {
|
|
137
156
|
throw new Error("Backend not initialized");
|
|
138
157
|
}
|
|
158
|
+
logger.info("Listing workflow runs: {after}, {before}", {
|
|
159
|
+
after: params.after,
|
|
160
|
+
before: params.before
|
|
161
|
+
});
|
|
139
162
|
const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;
|
|
140
163
|
const { after, before } = params;
|
|
141
164
|
let cursor = null;
|
|
@@ -164,6 +187,10 @@ const logger = getLogger([
|
|
|
164
187
|
if (!this.initialized) {
|
|
165
188
|
throw new Error("Backend not initialized");
|
|
166
189
|
}
|
|
190
|
+
logger.info("Claiming workflow run: {workerId}, {leaseDurationMs}", {
|
|
191
|
+
workerId: params.workerId,
|
|
192
|
+
leaseDurationMs: params.leaseDurationMs
|
|
193
|
+
});
|
|
167
194
|
const claimed = await this.knex.with("expired", (qb)=>qb.withSchema(DEFAULT_SCHEMA).table("workflow_runs").update({
|
|
168
195
|
status: "failed",
|
|
169
196
|
error: JSON.stringify({
|
|
@@ -197,6 +224,11 @@ const logger = getLogger([
|
|
|
197
224
|
if (!this.initialized) {
|
|
198
225
|
throw new Error("Backend not initialized");
|
|
199
226
|
}
|
|
227
|
+
logger.info("Extending workflow run lease: {workflowRunId}, {workerId}, {leaseDurationMs}", {
|
|
228
|
+
workflowRunId: params.workflowRunId,
|
|
229
|
+
workerId: params.workerId,
|
|
230
|
+
leaseDurationMs: params.leaseDurationMs
|
|
231
|
+
});
|
|
200
232
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).where("status", "running").where("worker_id", params.workerId).update({
|
|
201
233
|
available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),
|
|
202
234
|
updated_at: this.knex.fn.now()
|
|
@@ -213,6 +245,11 @@ const logger = getLogger([
|
|
|
213
245
|
if (!this.initialized) {
|
|
214
246
|
throw new Error("Backend not initialized");
|
|
215
247
|
}
|
|
248
|
+
logger.info("Sleeping workflow run: {workflowRunId}, {workerId}, {availableAt}", {
|
|
249
|
+
workflowRunId: params.workflowRunId,
|
|
250
|
+
workerId: params.workerId,
|
|
251
|
+
availableAt: params.availableAt
|
|
252
|
+
});
|
|
216
253
|
// 'succeeded' status is deprecated
|
|
217
254
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).whereNotIn("status", [
|
|
218
255
|
"succeeded",
|
|
@@ -237,6 +274,11 @@ const logger = getLogger([
|
|
|
237
274
|
if (!this.initialized) {
|
|
238
275
|
throw new Error("Backend not initialized");
|
|
239
276
|
}
|
|
277
|
+
logger.info("Completing workflow run: {workflowRunId}, {workerId}, {output}", {
|
|
278
|
+
workflowRunId: params.workflowRunId,
|
|
279
|
+
workerId: params.workerId,
|
|
280
|
+
output: params.output
|
|
281
|
+
});
|
|
240
282
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).where("status", "running").where("worker_id", params.workerId).update({
|
|
241
283
|
status: "completed",
|
|
242
284
|
output: JSON.stringify(params.output),
|
|
@@ -268,6 +310,11 @@ const logger = getLogger([
|
|
|
268
310
|
// 'available_at' timestamp for the next retry
|
|
269
311
|
const retryIntervalExpr = `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, "attempts" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;
|
|
270
312
|
const deadlineExceededCondition = `"deadline_at" IS NOT NULL AND NOW() + (${retryIntervalExpr}) >= "deadline_at"`;
|
|
313
|
+
logger.info("Failing workflow run: {workflowRunId}, {workerId}, {error}", {
|
|
314
|
+
workflowRunId: params.workflowRunId,
|
|
315
|
+
workerId: params.workerId,
|
|
316
|
+
error: params.error
|
|
317
|
+
});
|
|
271
318
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", workflowRunId).where("status", "running").where("worker_id", params.workerId).update({
|
|
272
319
|
status: this.knex.raw(`CASE WHEN ${deadlineExceededCondition} THEN 'failed' ELSE 'pending' END`),
|
|
273
320
|
available_at: this.knex.raw(`CASE WHEN ${deadlineExceededCondition} THEN NULL ELSE NOW() + (${retryIntervalExpr}) END`),
|
|
@@ -289,6 +336,9 @@ const logger = getLogger([
|
|
|
289
336
|
if (!this.initialized) {
|
|
290
337
|
throw new Error("Backend not initialized");
|
|
291
338
|
}
|
|
339
|
+
logger.info("Canceling workflow run: {workflowRunId}", {
|
|
340
|
+
workflowRunId: params.workflowRunId
|
|
341
|
+
});
|
|
292
342
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("workflow_runs").where("namespace_id", this.namespaceId).where("id", params.workflowRunId).whereIn("status", [
|
|
293
343
|
"pending",
|
|
294
344
|
"running",
|
|
@@ -336,6 +386,11 @@ const logger = getLogger([
|
|
|
336
386
|
if (!this.initialized) {
|
|
337
387
|
throw new Error("Backend not initialized");
|
|
338
388
|
}
|
|
389
|
+
logger.info("Creating step attempt: {workflowRunId}, {stepName}, {kind}", {
|
|
390
|
+
workflowRunId: params.workflowRunId,
|
|
391
|
+
stepName: params.stepName,
|
|
392
|
+
kind: params.kind
|
|
393
|
+
});
|
|
339
394
|
const [stepAttempt] = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts").insert({
|
|
340
395
|
namespace_id: this.namespaceId,
|
|
341
396
|
id: crypto.randomUUID(),
|
|
@@ -361,6 +416,9 @@ const logger = getLogger([
|
|
|
361
416
|
if (!this.initialized) {
|
|
362
417
|
throw new Error("Backend not initialized");
|
|
363
418
|
}
|
|
419
|
+
logger.info("Getting step attempt: {stepAttemptId}", {
|
|
420
|
+
stepAttemptId: params.stepAttemptId
|
|
421
|
+
});
|
|
364
422
|
const stepAttempt = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts").where("namespace_id", this.namespaceId).where("id", params.stepAttemptId).first();
|
|
365
423
|
return stepAttempt ?? null;
|
|
366
424
|
}
|
|
@@ -368,6 +426,11 @@ const logger = getLogger([
|
|
|
368
426
|
if (!this.initialized) {
|
|
369
427
|
throw new Error("Backend not initialized");
|
|
370
428
|
}
|
|
429
|
+
logger.info("Listing step attempts: {workflowRunId}, {after}, {before}", {
|
|
430
|
+
workflowRunId: params.workflowRunId,
|
|
431
|
+
after: params.after,
|
|
432
|
+
before: params.before
|
|
433
|
+
});
|
|
371
434
|
const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;
|
|
372
435
|
const { after, before } = params;
|
|
373
436
|
let cursor = null;
|
|
@@ -424,10 +487,16 @@ const logger = getLogger([
|
|
|
424
487
|
}
|
|
425
488
|
};
|
|
426
489
|
}
|
|
490
|
+
// NOTE: 실제 서비스에서 이게 안 되는 것 같은데, 쿼리 등을 체크할 필요가 있음.
|
|
427
491
|
async completeStepAttempt(params) {
|
|
428
492
|
if (!this.initialized) {
|
|
429
493
|
throw new Error("Backend not initialized");
|
|
430
494
|
}
|
|
495
|
+
logger.info("Marking step attempt as completed: {workflowRunId}, {stepAttemptId}, {workerId}", {
|
|
496
|
+
workflowRunId: params.workflowRunId,
|
|
497
|
+
stepAttemptId: params.stepAttemptId,
|
|
498
|
+
workerId: params.workerId
|
|
499
|
+
});
|
|
431
500
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts as sa").update({
|
|
432
501
|
status: "completed",
|
|
433
502
|
output: JSON.stringify(params.output),
|
|
@@ -447,6 +516,14 @@ const logger = getLogger([
|
|
|
447
516
|
if (!this.initialized) {
|
|
448
517
|
throw new Error("Backend not initialized");
|
|
449
518
|
}
|
|
519
|
+
logger.info("Marking step attempt as failed: {workflowRunId}, {stepAttemptId}, {workerId}", {
|
|
520
|
+
workflowRunId: params.workflowRunId,
|
|
521
|
+
stepAttemptId: params.stepAttemptId,
|
|
522
|
+
workerId: params.workerId
|
|
523
|
+
});
|
|
524
|
+
logger.info("Error: {error.message}", {
|
|
525
|
+
error: params.error.message
|
|
526
|
+
});
|
|
450
527
|
const [updated] = await this.knex.withSchema(DEFAULT_SCHEMA).table("step_attempts as sa").update({
|
|
451
528
|
status: "failed",
|
|
452
529
|
output: null,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/database/backend.ts"],"sourcesContent":["import { getLogger } from \"@logtape/logtape\";\nimport { camelize } from \"inflection\";\nimport knex, { type Knex } from \"knex\";\nimport {\n type Backend,\n type CancelWorkflowRunParams,\n type ClaimWorkflowRunParams,\n type CompleteStepAttemptParams,\n type CompleteWorkflowRunParams,\n type CreateStepAttemptParams,\n type CreateWorkflowRunParams,\n DEFAULT_NAMESPACE_ID,\n type ExtendWorkflowRunLeaseParams,\n type FailStepAttemptParams,\n type FailWorkflowRunParams,\n type GetStepAttemptParams,\n type GetWorkflowRunParams,\n type ListStepAttemptsParams,\n type ListWorkflowRunsParams,\n type PaginatedResponse,\n type SleepWorkflowRunParams,\n} from \"../backend\";\nimport { DEFAULT_RETRY_POLICY } from \"../core/retry\";\nimport type { StepAttempt } from \"../core/step\";\nimport type { WorkflowRun } from \"../core/workflow\";\nimport { DEFAULT_SCHEMA, migrate } from \"./base\";\nimport { type OnSubscribed, PostgresPubSub } from \"./pubsub\";\n\nexport const DEFAULT_LISTEN_CHANNEL = \"new_tasks\" as const;\nconst DEFAULT_PAGINATION_PAGE_SIZE = 100 as const;\n\ninterface BackendPostgresOptions {\n namespaceId?: string;\n runMigrations?: boolean;\n\n // default: true\n usePubSub?: boolean;\n}\n\nconst logger = getLogger([\"sonamu\", \"internal\", \"tasks\"]);\n\n/**\n * Manages a connection to a Postgres database for workflow operations.\n */\nexport class BackendPostgres implements Backend {\n private config: Knex.Config;\n private namespaceId: string;\n private usePubSub: boolean;\n private pubsub: PostgresPubSub | null = null;\n private initialized: boolean = false;\n private runMigrations: boolean;\n\n private _knex: Knex | null = null;\n private get knex(): Knex {\n if (!this._knex) {\n this._knex = knex(this.config);\n }\n\n return this._knex;\n }\n\n constructor(config: Knex.Config, options?: BackendPostgresOptions) {\n this.config = {\n ...config,\n postProcessResponse: (result, _queryContext) => {\n if (result === null || result === undefined) {\n return result;\n }\n\n if (config?.postProcessResponse) {\n result = config.postProcessResponse(result, _queryContext);\n }\n\n const camelizeRow = (row: Record<string, unknown>) =>\n Object.fromEntries(\n Object.entries(row).map(([key, value]) => [camelize(key, true), value]),\n );\n\n if (Array.isArray(result)) {\n return result.map(camelizeRow);\n }\n\n return camelizeRow(result);\n },\n };\n\n const { namespaceId, usePubSub, runMigrations } = {\n namespaceId: DEFAULT_NAMESPACE_ID,\n usePubSub: true,\n runMigrations: true,\n ...options,\n };\n\n this.namespaceId = namespaceId;\n this.usePubSub = usePubSub;\n this.runMigrations = runMigrations;\n }\n\n async initialize() {\n if (this.initialized) {\n return;\n }\n\n if (this.runMigrations) {\n await migrate(this.config, DEFAULT_SCHEMA);\n }\n\n this.initialized = true;\n }\n\n async subscribe(callback: OnSubscribed) {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n if (!this.pubsub) {\n this.pubsub = await PostgresPubSub.create(this.knex);\n }\n\n this.pubsub.listenEvent(DEFAULT_LISTEN_CHANNEL, callback);\n }\n\n async publish(payload?: string): Promise<void> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n await this.knex.raw(\n payload\n ? `NOTIFY ${DEFAULT_LISTEN_CHANNEL}, '${payload}'`\n : `NOTIFY ${DEFAULT_LISTEN_CHANNEL}`,\n );\n }\n\n async stop(): Promise<void> {\n if (!this.initialized) {\n return;\n }\n\n await this.pubsub?.destroy();\n this.pubsub = null;\n await this.knex.destroy();\n }\n\n async createWorkflowRun(params: CreateWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_name: params.workflowName,\n version: params.version,\n status: \"pending\",\n idempotency_key: params.idempotencyKey,\n config: params.config,\n context: params.context,\n input: params.input,\n attempts: 0,\n available_at: params.availableAt ?? this.knex.fn.now(),\n deadline_at: params.deadlineAt,\n created_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n const workflowRun = await qb;\n if (!workflowRun[0]) {\n logger.error(\"Failed to create workflow run: {params}\", { params });\n throw new Error(\"Failed to create workflow run\");\n }\n\n return workflowRun[0];\n }\n\n async getWorkflowRun(params: GetWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const workflowRun = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .select(\n \"namespace_id\",\n \"id\",\n \"workflow_name\",\n \"version\",\n \"status\",\n \"idempotency_key\",\n \"config\",\n \"context\",\n \"input\",\n \"output\",\n \"error\",\n \"attempts\",\n \"parent_step_attempt_namespace_id\",\n \"parent_step_attempt_id\",\n \"worker_id\",\n \"available_at\",\n \"deadline_at\",\n \"started_at\",\n \"finished_at\",\n \"created_at\",\n \"updated_at\",\n )\n .first();\n\n return workflowRun ?? null;\n }\n\n async listWorkflowRuns(params: ListWorkflowRunsParams): Promise<PaginatedResponse<WorkflowRun>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListWorkflowRunsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListWorkflowRunsWhere(params: ListWorkflowRunsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n async claimWorkflowRun(params: ClaimWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const claimed = await this.knex\n .with(\"expired\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .update({\n status: \"failed\",\n error: JSON.stringify({ message: \"Workflow run deadline exceeded\" }),\n worker_id: null,\n available_at: null,\n finished_at: this.knex.raw(\"NOW()\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .whereNotNull(\"deadline_at\")\n .where(\"deadline_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .returning(\"id\"),\n )\n .with(\"candidate\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .select(\"id\")\n .from(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .where(\"available_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .where((qb2) => {\n qb2.whereNull(\"deadline_at\").orWhere(\"deadline_at\", \">\", this.knex.raw(\"NOW()\"));\n })\n .orderByRaw(\"CASE WHEN status = 'pending' THEN 0 ELSE 1 END\")\n .orderBy(\"available_at\", \"asc\")\n .orderBy(\"created_at\", \"asc\")\n .limit(1)\n .forUpdate()\n .skipLocked(),\n )\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs as wr\")\n .where(\"wr.namespace_id\", this.namespaceId)\n .where(\"wr.id\", this.knex.ref(\"candidate.id\"))\n .update({\n status: \"running\",\n attempts: this.knex.raw(\"wr.attempts + 1\"),\n worker_id: params.workerId,\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n started_at: this.knex.raw(\"COALESCE(wr.started_at, NOW())\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .updateFrom(\"candidate\")\n .returning(\"wr.*\");\n\n return claimed[0] ?? null;\n }\n\n async extendWorkflowRunLease(params: ExtendWorkflowRunLeaseParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to extend lease for workflow run: {params}\", { params });\n throw new Error(\"Failed to extend lease for workflow run\");\n }\n\n return updated;\n }\n\n async sleepWorkflowRun(params: SleepWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n // 'succeeded' status is deprecated\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereNotIn(\"status\", [\"succeeded\", \"completed\", \"failed\", \"canceled\"])\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"sleeping\",\n available_at: params.availableAt,\n worker_id: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to sleep workflow run: {params}\", { params });\n throw new Error(\"Failed to sleep workflow run\");\n }\n\n return updated;\n }\n\n async completeWorkflowRun(params: CompleteWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n worker_id: params.workerId,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to complete workflow run: {params}\", { params });\n throw new Error(\"Failed to complete workflow run\");\n }\n\n return updated;\n }\n\n async failWorkflowRun(params: FailWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const { workflowRunId, error } = params;\n const { initialIntervalMs, backoffCoefficient, maximumIntervalMs } = DEFAULT_RETRY_POLICY;\n\n // this beefy query updates a workflow's status, available_at, and\n // finished_at based on the workflow's deadline and retry policy\n //\n // if the next retry would exceed the deadline, the run is marked as\n // 'failed' and finalized, otherwise, the run is rescheduled with an updated\n // 'available_at' timestamp for the next retry\n const retryIntervalExpr = `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, \"attempts\" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;\n const deadlineExceededCondition = `\"deadline_at\" IS NOT NULL AND NOW() + (${retryIntervalExpr}) >= \"deadline_at\"`;\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN 'failed' ELSE 'pending' END`,\n ),\n available_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NULL ELSE NOW() + (${retryIntervalExpr}) END`,\n ),\n finished_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NOW() ELSE NULL END`,\n ),\n error: JSON.stringify(error),\n worker_id: null,\n started_at: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to mark workflow run failed: {params}\", { params });\n throw new Error(\"Failed to mark workflow run failed\");\n }\n\n return updated;\n }\n\n async cancelWorkflowRun(params: CancelWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .update({\n status: \"canceled\",\n worker_id: null,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n // workflow may already be in a terminal state\n const existing = await this.getWorkflowRun({\n workflowRunId: params.workflowRunId,\n });\n if (!existing) {\n throw new Error(`Workflow run ${params.workflowRunId} does not exist`);\n }\n\n // if already canceled, just return it\n if (existing.status === \"canceled\") {\n return existing;\n }\n\n // throw error for completed/failed workflows\n // 'succeeded' status is deprecated\n if ([\"succeeded\", \"completed\", \"failed\"].includes(existing.status)) {\n logger.error(\"Cannot cancel workflow run: {params} with status {status}\", {\n params,\n status: existing.status,\n });\n throw new Error(\n `Cannot cancel workflow run ${params.workflowRunId} with status ${existing.status}`,\n );\n }\n\n logger.error(\"Failed to cancel workflow run: {params}\", { params });\n throw new Error(\"Failed to cancel workflow run\");\n }\n\n return updated;\n }\n\n async createStepAttempt(params: CreateStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [stepAttempt] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_run_id: params.workflowRunId,\n step_name: params.stepName,\n kind: params.kind,\n status: \"running\",\n config: JSON.stringify(params.config),\n context: JSON.stringify(params.context),\n started_at: this.knex.fn.now(),\n created_at: this.knex.raw(\"date_trunc('milliseconds', NOW())\"),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!stepAttempt) {\n logger.error(\"Failed to create step attempt: {params}\", { params });\n throw new Error(\"Failed to create step attempt\");\n }\n\n return stepAttempt;\n }\n\n async getStepAttempt(params: GetStepAttemptParams): Promise<StepAttempt | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const stepAttempt = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.stepAttemptId)\n .first();\n\n return stepAttempt ?? null;\n }\n\n async listStepAttempts(params: ListStepAttemptsParams): Promise<PaginatedResponse<StepAttempt>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListStepAttemptsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListStepAttemptsWhere(params: ListStepAttemptsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"workflow_run_id\", params.workflowRunId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n private processPaginationResults<T extends Cursor>(\n rows: T[],\n limit: number,\n hasAfter: boolean,\n hasBefore: boolean,\n ): PaginatedResponse<T> {\n const data = rows;\n let hasNext = false;\n let hasPrev = false;\n\n if (hasBefore) {\n data.reverse();\n if (data.length > limit) {\n hasPrev = true;\n data.shift();\n }\n hasNext = true;\n } else {\n if (data.length > limit) {\n hasNext = true;\n data.pop();\n }\n if (hasAfter) {\n hasPrev = true;\n }\n }\n\n const lastItem = data.at(-1);\n const nextCursor = hasNext && lastItem ? encodeCursor(lastItem) : null;\n const firstItem = data[0];\n const prevCursor = hasPrev && firstItem ? encodeCursor(firstItem) : null;\n\n return {\n data,\n pagination: {\n next: nextCursor,\n prev: prevCursor,\n },\n };\n }\n\n async completeStepAttempt(params: CompleteStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt completed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt completed\");\n }\n\n return updated;\n }\n\n async failStepAttempt(params: FailStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"failed\",\n output: null,\n error: JSON.stringify(params.error),\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt failed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt failed\");\n }\n\n return updated;\n }\n}\n\n/**\n * Cursor used for pagination. Requires created_at and id fields. Because JS\n * Date does not natively support microsecond precision dates, created_at should\n * be stored with millisecond precision in paginated tables to avoid issues with\n * cursor comparisons.\n */\ninterface Cursor {\n createdAt: Date;\n id: string;\n}\n\nfunction encodeCursor(item: Cursor): string {\n const encoded = Buffer.from(\n JSON.stringify({ createdAt: item.createdAt.toISOString(), id: item.id }),\n ).toString(\"base64\");\n return encoded;\n}\n\nexport function decodeCursor(cursor: string): Cursor {\n const decoded = Buffer.from(cursor, \"base64\").toString(\"utf8\");\n const parsed = JSON.parse(decoded) as { createdAt: string; id: string };\n return {\n createdAt: new Date(parsed.createdAt),\n id: parsed.id,\n };\n}\n"],"names":["getLogger","camelize","knex","DEFAULT_NAMESPACE_ID","DEFAULT_RETRY_POLICY","DEFAULT_SCHEMA","migrate","PostgresPubSub","DEFAULT_LISTEN_CHANNEL","DEFAULT_PAGINATION_PAGE_SIZE","logger","BackendPostgres","config","namespaceId","usePubSub","pubsub","initialized","runMigrations","_knex","options","postProcessResponse","result","_queryContext","undefined","camelizeRow","row","Object","fromEntries","entries","map","key","value","Array","isArray","initialize","subscribe","callback","Error","create","listenEvent","publish","payload","raw","stop","destroy","createWorkflowRun","params","qb","withSchema","table","insert","namespace_id","id","crypto","randomUUID","workflow_name","workflowName","version","status","idempotency_key","idempotencyKey","context","input","attempts","available_at","availableAt","fn","now","deadline_at","deadlineAt","created_at","updated_at","returning","workflowRun","error","getWorkflowRun","where","workflowRunId","select","first","listWorkflowRuns","limit","after","before","cursor","decodeCursor","buildListWorkflowRunsWhere","rows","orderBy","processPaginationResults","operator","whereRaw","createdAt","toISOString","claimWorkflowRun","claimed","with","update","JSON","stringify","message","worker_id","finished_at","whereIn","whereNotNull","from","qb2","whereNull","orWhere","orderByRaw","forUpdate","skipLocked","ref","workerId","leaseDurationMs","started_at","updateFrom","extendWorkflowRunLease","updated","sleepWorkflowRun","whereNotIn","completeWorkflowRun","output","failWorkflowRun","initialIntervalMs","backoffCoefficient","maximumIntervalMs","retryIntervalExpr","deadlineExceededCondition","cancelWorkflowRun","existing","includes","createStepAttempt","stepAttempt","workflow_run_id","step_name","stepName","kind","getStepAttempt","stepAttemptId","listStepAttempts","buildListStepAttemptsWhere","hasAfter","hasBefore","data","hasNext","hasPrev","reverse","length","shift","pop","lastItem","at","nextCursor","encodeCursor","firstItem","prevCursor","pagination","next","prev","completeStepAttempt","failStepAttempt","item","encoded","Buffer","toString","decoded","parsed","parse","Date"],"mappings":"AAAA,SAASA,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,QAAQ,QAAQ,aAAa;AACtC,OAAOC,UAAyB,OAAO;AACvC,SAQEC,oBAAoB,QAUf,gBAAa;AACpB,SAASC,oBAAoB,QAAQ,mBAAgB;AAGrD,SAASC,cAAc,EAAEC,OAAO,QAAQ,YAAS;AACjD,SAA4BC,cAAc,QAAQ,cAAW;AAE7D,OAAO,MAAMC,yBAAyB,YAAqB;AAC3D,MAAMC,+BAA+B;AAUrC,MAAMC,SAASV,UAAU;IAAC;IAAU;IAAY;CAAQ;AAExD;;CAEC,GACD,OAAO,MAAMW;IACHC,OAAoB;IACpBC,YAAoB;IACpBC,UAAmB;IACnBC,SAAgC,KAAK;IACrCC,cAAuB,MAAM;IAC7BC,cAAuB;IAEvBC,QAAqB,KAAK;IAClC,IAAYhB,OAAa;QACvB,IAAI,CAAC,IAAI,CAACgB,KAAK,EAAE;YACf,IAAI,CAACA,KAAK,GAAGhB,KAAK,IAAI,CAACU,MAAM;QAC/B;QAEA,OAAO,IAAI,CAACM,KAAK;IACnB;IAEA,YAAYN,MAAmB,EAAEO,OAAgC,CAAE;QACjE,IAAI,CAACP,MAAM,GAAG;YACZ,GAAGA,MAAM;YACTQ,qBAAqB,CAACC,QAAQC;gBAC5B,IAAID,WAAW,QAAQA,WAAWE,WAAW;oBAC3C,OAAOF;gBACT;gBAEA,IAAIT,QAAQQ,qBAAqB;oBAC/BC,SAAST,OAAOQ,mBAAmB,CAACC,QAAQC;gBAC9C;gBAEA,MAAME,cAAc,CAACC,MACnBC,OAAOC,WAAW,CAChBD,OAAOE,OAAO,CAACH,KAAKI,GAAG,CAAC,CAAC,CAACC,KAAKC,MAAM,GAAK;4BAAC9B,SAAS6B,KAAK;4BAAOC;yBAAM;gBAG1E,IAAIC,MAAMC,OAAO,CAACZ,SAAS;oBACzB,OAAOA,OAAOQ,GAAG,CAACL;gBACpB;gBAEA,OAAOA,YAAYH;YACrB;QACF;QAEA,MAAM,EAAER,WAAW,EAAEC,SAAS,EAAEG,aAAa,EAAE,GAAG;YAChDJ,aAAaV;YACbW,WAAW;YACXG,eAAe;YACf,GAAGE,OAAO;QACZ;QAEA,IAAI,CAACN,WAAW,GAAGA;QACnB,IAAI,CAACC,SAAS,GAAGA;QACjB,IAAI,CAACG,aAAa,GAAGA;IACvB;IAEA,MAAMiB,aAAa;QACjB,IAAI,IAAI,CAAClB,WAAW,EAAE;YACpB;QACF;QAEA,IAAI,IAAI,CAACC,aAAa,EAAE;YACtB,MAAMX,QAAQ,IAAI,CAACM,MAAM,EAAEP;QAC7B;QAEA,IAAI,CAACW,WAAW,GAAG;IACrB;IAEA,MAAMmB,UAAUC,QAAsB,EAAE;QACtC,IAAI,CAAC,IAAI,CAACpB,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAACvB,SAAS,EAAE;YACnB;QACF;QAEA,IAAI,CAAC,IAAI,CAACC,MAAM,EAAE;YAChB,IAAI,CAACA,MAAM,GAAG,MAAMR,eAAe+B,MAAM,CAAC,IAAI,CAACpC,IAAI;QACrD;QAEA,IAAI,CAACa,MAAM,CAACwB,WAAW,CAAC/B,wBAAwB4B;IAClD;IAEA,MAAMI,QAAQC,OAAgB,EAAiB;QAC7C,IAAI,CAAC,IAAI,CAACzB,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAACvB,SAAS,EAAE;YACnB;QACF;QAEA,MAAM,IAAI,CAACZ,IAAI,CAACwC,GAAG,CACjBD,UACI,CAAC,OAAO,EAAEjC,uBAAuB,GAAG,EAAEiC,QAAQ,CAAC,CAAC,GAChD,CAAC,OAAO,EAAEjC,wBAAwB;IAE1C;IAEA,MAAMmC,OAAsB;QAC1B,IAAI,CAAC,IAAI,CAAC3B,WAAW,EAAE;YACrB;QACF;QAEA,MAAM,IAAI,CAACD,MAAM,EAAE6B;QACnB,IAAI,CAAC7B,MAAM,GAAG;QACd,MAAM,IAAI,CAACb,IAAI,CAAC0C,OAAO;IACzB;IAEA,MAAMC,kBAAkBC,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAMU,KAAK,IAAI,CAAC7C,IAAI,CACjB8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAACtC,WAAW;YAC9BuC,IAAIC,OAAOC,UAAU;YACrBC,eAAeT,OAAOU,YAAY;YAClCC,SAASX,OAAOW,OAAO;YACvBC,QAAQ;YACRC,iBAAiBb,OAAOc,cAAc;YACtChD,QAAQkC,OAAOlC,MAAM;YACrBiD,SAASf,OAAOe,OAAO;YACvBC,OAAOhB,OAAOgB,KAAK;YACnBC,UAAU;YACVC,cAAclB,OAAOmB,WAAW,IAAI,IAAI,CAAC/D,IAAI,CAACgE,EAAE,CAACC,GAAG;YACpDC,aAAatB,OAAOuB,UAAU;YAC9BC,YAAY,IAAI,CAACpE,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC5BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,MAAMC,cAAc,MAAM1B;QAC1B,IAAI,CAAC0B,WAAW,CAAC,EAAE,EAAE;YACnB/D,OAAOgE,KAAK,CAAC,2CAA2C;gBAAE5B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOoC,WAAW,CAAC,EAAE;IACvB;IAEA,MAAME,eAAe7B,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAMoC,cAAc,MAAM,IAAI,CAACvE,IAAI,CAChC8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChCC,MAAM,CACL,gBACA,MACA,iBACA,WACA,UACA,mBACA,UACA,WACA,SACA,UACA,SACA,YACA,oCACA,0BACA,aACA,gBACA,eACA,cACA,eACA,cACA,cAEDC,KAAK;QAER,OAAON,eAAe;IACxB;IAEA,MAAMO,iBAAiBlC,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM4C,QAAQnC,OAAOmC,KAAK,IAAIxE;QAC9B,MAAM,EAAEyE,KAAK,EAAEC,MAAM,EAAE,GAAGrC;QAE1B,IAAIsC,SAAwB;QAC5B,IAAIF,OAAO;YACTE,SAASC,aAAaH;QACxB,OAAO,IAAIC,QAAQ;YACjBC,SAASC,aAAaF;QACxB;QAEA,MAAMpC,KAAK,IAAI,CAACuC,0BAA0B,CAACxC,QAAQsC;QACnD,MAAMG,OAAO,MAAMxC,GAChByC,OAAO,CAAC,cAAcL,SAAS,SAAS,OACxCK,OAAO,CAAC,MAAML,SAAS,SAAS,OAChCF,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACQ,wBAAwB,CAClCF,MACAN,OACA,OAAOC,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQG,2BAA2BxC,MAA8B,EAAEsC,MAAqB,EAAE;QACxF,MAAM,EAAEF,KAAK,EAAE,GAAGpC;QAClB,MAAMC,KAAK,IAAI,CAAC7C,IAAI,CACjB8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW;QAEzC,IAAIuE,QAAQ;YACV,MAAMM,WAAWR,QAAQ,MAAM;YAC/B,OAAOnC,GAAG4C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAOhC,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEA,MAAM+C,iBAAiBhD,MAA8B,EAA+B;QAClF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM0D,UAAU,MAAM,IAAI,CAAC7F,IAAI,CAC5B8F,IAAI,CAAC,WAAW,CAACjD,KAChBA,GACGC,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACNgD,MAAM,CAAC;gBACNvC,QAAQ;gBACRgB,OAAOwB,KAAKC,SAAS,CAAC;oBAAEC,SAAS;gBAAiC;gBAClEC,WAAW;gBACXrC,cAAc;gBACdsC,aAAa,IAAI,CAACpG,IAAI,CAACwC,GAAG,CAAC;gBAC3B6B,YAAY,IAAI,CAACrE,IAAI,CAACwC,GAAG,CAAC;YAC5B,GACCkC,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC0F,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpDC,YAAY,CAAC,eACb5B,KAAK,CAAC,eAAe,MAAM,IAAI,CAAC1E,IAAI,CAACwC,GAAG,CAAC,UACzC8B,SAAS,CAAC,OAEdwB,IAAI,CAAC,aAAa,CAACjD,KAClBA,GACGC,UAAU,CAAC3C,gBACXyE,MAAM,CAAC,MACP2B,IAAI,CAAC,iBACL7B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC0F,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpD3B,KAAK,CAAC,gBAAgB,MAAM,IAAI,CAAC1E,IAAI,CAACwC,GAAG,CAAC,UAC1CkC,KAAK,CAAC,CAAC8B;gBACNA,IAAIC,SAAS,CAAC,eAAeC,OAAO,CAAC,eAAe,KAAK,IAAI,CAAC1G,IAAI,CAACwC,GAAG,CAAC;YACzE,GACCmE,UAAU,CAAC,kDACXrB,OAAO,CAAC,gBAAgB,OACxBA,OAAO,CAAC,cAAc,OACtBP,KAAK,CAAC,GACN6B,SAAS,GACTC,UAAU,IAEd/D,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,uBACN2B,KAAK,CAAC,mBAAmB,IAAI,CAAC/D,WAAW,EACzC+D,KAAK,CAAC,SAAS,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,iBAC7Bf,MAAM,CAAC;YACNvC,QAAQ;YACRK,UAAU,IAAI,CAAC7D,IAAI,CAACwC,GAAG,CAAC;YACxB2D,WAAWvD,OAAOmE,QAAQ;YAC1BjD,cAAc,IAAI,CAAC9D,IAAI,CAACwC,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOoE,eAAe,CAAC,2BAA2B,CAAC;YAC1FC,YAAY,IAAI,CAACjH,IAAI,CAACwC,GAAG,CAAC;YAC1B6B,YAAY,IAAI,CAACrE,IAAI,CAACwC,GAAG,CAAC;QAC5B,GACC0E,UAAU,CAAC,aACX5C,SAAS,CAAC;QAEb,OAAOuB,OAAO,CAAC,EAAE,IAAI;IACvB;IAEA,MAAMsB,uBAAuBvE,MAAoC,EAAwB;QACvF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChCD,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNjC,cAAc,IAAI,CAAC9D,IAAI,CAACwC,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOoE,eAAe,CAAC,2BAA2B,CAAC;YAC1F3C,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,qDAAqD;gBAAE5B;YAAO;YAC3E,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMC,iBAAiBzE,MAA8B,EAAwB;QAC3E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,mCAAmC;QACnC,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChC2C,UAAU,CAAC,UAAU;YAAC;YAAa;YAAa;YAAU;SAAW,EACrE5C,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNvC,QAAQ;YACRM,cAAclB,OAAOmB,WAAW;YAChCoC,WAAW;YACX9B,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,0CAA0C;gBAAE5B;YAAO;YAChE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMG,oBAAoB3E,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChCD,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNvC,QAAQ;YACRgE,QAAQxB,KAAKC,SAAS,CAACrD,OAAO4E,MAAM;YACpChD,OAAO;YACP2B,WAAWvD,OAAOmE,QAAQ;YAC1BjD,cAAc;YACdsC,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,6CAA6C;gBAAE5B;YAAO;YACnE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMK,gBAAgB7E,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,EAAEwC,aAAa,EAAEH,KAAK,EAAE,GAAG5B;QACjC,MAAM,EAAE8E,iBAAiB,EAAEC,kBAAkB,EAAEC,iBAAiB,EAAE,GAAG1H;QAErE,kEAAkE;QAClE,gEAAgE;QAChE,EAAE;QACF,oEAAoE;QACpE,4EAA4E;QAC5E,8CAA8C;QAC9C,MAAM2H,oBAAoB,CAAC,MAAM,EAAEH,kBAAkB,SAAS,EAAEC,mBAAmB,mBAAmB,EAAEC,kBAAkB,4BAA4B,CAAC;QACvJ,MAAME,4BAA4B,CAAC,uCAAuC,EAAED,kBAAkB,kBAAkB,CAAC;QAEjH,MAAM,CAACT,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAMC,eACZD,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAa9B,OAAOmE,QAAQ,EAClChB,MAAM,CAAC;YACNvC,QAAQ,IAAI,CAACxD,IAAI,CAACwC,GAAG,CACnB,CAAC,UAAU,EAAEsF,0BAA0B,iCAAiC,CAAC;YAE3EhE,cAAc,IAAI,CAAC9D,IAAI,CAACwC,GAAG,CACzB,CAAC,UAAU,EAAEsF,0BAA0B,yBAAyB,EAAED,kBAAkB,KAAK,CAAC;YAE5FzB,aAAa,IAAI,CAACpG,IAAI,CAACwC,GAAG,CACxB,CAAC,UAAU,EAAEsF,0BAA0B,yBAAyB,CAAC;YAEnEtD,OAAOwB,KAAKC,SAAS,CAACzB;YACtB2B,WAAW;YACXc,YAAY;YACZ5C,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,gDAAgD;gBAAE5B;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMW,kBAAkBnF,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO+B,aAAa,EAChC0B,OAAO,CAAC,UAAU;YAAC;YAAW;YAAW;SAAW,EACpDN,MAAM,CAAC;YACNvC,QAAQ;YACR2C,WAAW;YACXrC,cAAc;YACdsC,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ,8CAA8C;YAC9C,MAAMY,WAAW,MAAM,IAAI,CAACvD,cAAc,CAAC;gBACzCE,eAAe/B,OAAO+B,aAAa;YACrC;YACA,IAAI,CAACqD,UAAU;gBACb,MAAM,IAAI7F,MAAM,CAAC,aAAa,EAAES,OAAO+B,aAAa,CAAC,eAAe,CAAC;YACvE;YAEA,sCAAsC;YACtC,IAAIqD,SAASxE,MAAM,KAAK,YAAY;gBAClC,OAAOwE;YACT;YAEA,6CAA6C;YAC7C,mCAAmC;YACnC,IAAI;gBAAC;gBAAa;gBAAa;aAAS,CAACC,QAAQ,CAACD,SAASxE,MAAM,GAAG;gBAClEhD,OAAOgE,KAAK,CAAC,6DAA6D;oBACxE5B;oBACAY,QAAQwE,SAASxE,MAAM;gBACzB;gBACA,MAAM,IAAIrB,MACR,CAAC,2BAA2B,EAAES,OAAO+B,aAAa,CAAC,aAAa,EAAEqD,SAASxE,MAAM,EAAE;YAEvF;YAEAhD,OAAOgE,KAAK,CAAC,2CAA2C;gBAAE5B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAMc,kBAAkBtF,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACgG,YAAY,GAAG,MAAM,IAAI,CAACnI,IAAI,CAClC8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAACtC,WAAW;YAC9BuC,IAAIC,OAAOC,UAAU;YACrBgF,iBAAiBxF,OAAO+B,aAAa;YACrC0D,WAAWzF,OAAO0F,QAAQ;YAC1BC,MAAM3F,OAAO2F,IAAI;YACjB/E,QAAQ;YACR9C,QAAQsF,KAAKC,SAAS,CAACrD,OAAOlC,MAAM;YACpCiD,SAASqC,KAAKC,SAAS,CAACrD,OAAOe,OAAO;YACtCsD,YAAY,IAAI,CAACjH,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC5BG,YAAY,IAAI,CAACpE,IAAI,CAACwC,GAAG,CAAC;YAC1B6B,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC6D,aAAa;YAChB3H,OAAOgE,KAAK,CAAC,2CAA2C;gBAAE5B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOgG;IACT;IAEA,MAAMK,eAAe5F,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAMgG,cAAc,MAAM,IAAI,CAACnI,IAAI,CAChC8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,MAAM9B,OAAO6F,aAAa,EAChC5D,KAAK;QAER,OAAOsD,eAAe;IACxB;IAEA,MAAMO,iBAAiB9F,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM4C,QAAQnC,OAAOmC,KAAK,IAAIxE;QAC9B,MAAM,EAAEyE,KAAK,EAAEC,MAAM,EAAE,GAAGrC;QAE1B,IAAIsC,SAAwB;QAC5B,IAAIF,OAAO;YACTE,SAASC,aAAaH;QACxB,OAAO,IAAIC,QAAQ;YACjBC,SAASC,aAAaF;QACxB;QAEA,MAAMpC,KAAK,IAAI,CAAC8F,0BAA0B,CAAC/F,QAAQsC;QACnD,MAAMG,OAAO,MAAMxC,GAChByC,OAAO,CAAC,cAAcL,SAAS,SAAS,OACxCK,OAAO,CAAC,MAAML,SAAS,SAAS,OAChCF,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACQ,wBAAwB,CAClCF,MACAN,OACA,OAAOC,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQ0D,2BAA2B/F,MAA8B,EAAEsC,MAAqB,EAAE;QACxF,MAAM,EAAEF,KAAK,EAAE,GAAGpC;QAClB,MAAMC,KAAK,IAAI,CAAC7C,IAAI,CACjB8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,iBACN2B,KAAK,CAAC,gBAAgB,IAAI,CAAC/D,WAAW,EACtC+D,KAAK,CAAC,mBAAmB9B,OAAO+B,aAAa;QAEhD,IAAIO,QAAQ;YACV,MAAMM,WAAWR,QAAQ,MAAM;YAC/B,OAAOnC,GAAG4C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAOhC,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEQ0C,yBACNF,IAAS,EACTN,KAAa,EACb6D,QAAiB,EACjBC,SAAkB,EACI;QACtB,MAAMC,OAAOzD;QACb,IAAI0D,UAAU;QACd,IAAIC,UAAU;QAEd,IAAIH,WAAW;YACbC,KAAKG,OAAO;YACZ,IAAIH,KAAKI,MAAM,GAAGnE,OAAO;gBACvBiE,UAAU;gBACVF,KAAKK,KAAK;YACZ;YACAJ,UAAU;QACZ,OAAO;YACL,IAAID,KAAKI,MAAM,GAAGnE,OAAO;gBACvBgE,UAAU;gBACVD,KAAKM,GAAG;YACV;YACA,IAAIR,UAAU;gBACZI,UAAU;YACZ;QACF;QAEA,MAAMK,WAAWP,KAAKQ,EAAE,CAAC,CAAC;QAC1B,MAAMC,aAAaR,WAAWM,WAAWG,aAAaH,YAAY;QAClE,MAAMI,YAAYX,IAAI,CAAC,EAAE;QACzB,MAAMY,aAAaV,WAAWS,YAAYD,aAAaC,aAAa;QAEpE,OAAO;YACLX;YACAa,YAAY;gBACVC,MAAML;gBACNM,MAAMH;YACR;QACF;IACF;IAEA,MAAMI,oBAAoBlH,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,uBACNgD,MAAM,CAAC;YACNvC,QAAQ;YACRgE,QAAQxB,KAAKC,SAAS,CAACrD,OAAO4E,MAAM;YACpChD,OAAO;YACP4B,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCiD,UAAU,CAAC,GAAG/G,eAAe,oBAAoB,CAAC,EAClDuE,KAAK,CAAC,mBAAmB,IAAI,CAAC/D,WAAW,EACzC+D,KAAK,CAAC,sBAAsB9B,OAAO+B,aAAa,EAChDD,KAAK,CAAC,SAAS9B,OAAO6F,aAAa,EACnC/D,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,oBACvCpC,KAAK,CAAC,SAAS,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,uBAC7BpC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgB9B,OAAOmE,QAAQ,EACrCzC,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,mDAAmD;gBAAE5B;YAAO;YACzE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;IAEA,MAAM2C,gBAAgBnH,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAIqB,MAAM;QAClB;QAEA,MAAM,CAACiF,QAAQ,GAAG,MAAM,IAAI,CAACpH,IAAI,CAC9B8C,UAAU,CAAC3C,gBACX4C,KAAK,CAAC,uBACNgD,MAAM,CAAC;YACNvC,QAAQ;YACRgE,QAAQ;YACRhD,OAAOwB,KAAKC,SAAS,CAACrD,OAAO4B,KAAK;YAClC4B,aAAa,IAAI,CAACpG,IAAI,CAACgE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAACrE,IAAI,CAACgE,EAAE,CAACC,GAAG;QAC9B,GACCiD,UAAU,CAAC,GAAG/G,eAAe,oBAAoB,CAAC,EAClDuE,KAAK,CAAC,mBAAmB,IAAI,CAAC/D,WAAW,EACzC+D,KAAK,CAAC,sBAAsB9B,OAAO+B,aAAa,EAChDD,KAAK,CAAC,SAAS9B,OAAO6F,aAAa,EACnC/D,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,oBACvCpC,KAAK,CAAC,SAAS,IAAI,CAAC1E,IAAI,CAAC8G,GAAG,CAAC,uBAC7BpC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgB9B,OAAOmE,QAAQ,EACrCzC,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ5G,OAAOgE,KAAK,CAAC,gDAAgD;gBAAE5B;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOiF;IACT;AACF;AAaA,SAASoC,aAAaQ,IAAY;IAChC,MAAMC,UAAUC,OAAO3D,IAAI,CACzBP,KAAKC,SAAS,CAAC;QAAEP,WAAWsE,KAAKtE,SAAS,CAACC,WAAW;QAAIzC,IAAI8G,KAAK9G,EAAE;IAAC,IACtEiH,QAAQ,CAAC;IACX,OAAOF;AACT;AAEA,OAAO,SAAS9E,aAAaD,MAAc;IACzC,MAAMkF,UAAUF,OAAO3D,IAAI,CAACrB,QAAQ,UAAUiF,QAAQ,CAAC;IACvD,MAAME,SAASrE,KAAKsE,KAAK,CAACF;IAC1B,OAAO;QACL1E,WAAW,IAAI6E,KAAKF,OAAO3E,SAAS;QACpCxC,IAAImH,OAAOnH,EAAE;IACf;AACF"}
|
|
1
|
+
{"version":3,"sources":["../../src/database/backend.ts"],"sourcesContent":["import { getLogger } from \"@logtape/logtape\";\nimport { camelize } from \"inflection\";\nimport knex, { type Knex } from \"knex\";\nimport {\n type Backend,\n type CancelWorkflowRunParams,\n type ClaimWorkflowRunParams,\n type CompleteStepAttemptParams,\n type CompleteWorkflowRunParams,\n type CreateStepAttemptParams,\n type CreateWorkflowRunParams,\n DEFAULT_NAMESPACE_ID,\n type ExtendWorkflowRunLeaseParams,\n type FailStepAttemptParams,\n type FailWorkflowRunParams,\n type GetStepAttemptParams,\n type GetWorkflowRunParams,\n type ListStepAttemptsParams,\n type ListWorkflowRunsParams,\n type PaginatedResponse,\n type SleepWorkflowRunParams,\n} from \"../backend\";\nimport { DEFAULT_RETRY_POLICY } from \"../core/retry\";\nimport type { StepAttempt } from \"../core/step\";\nimport type { WorkflowRun } from \"../core/workflow\";\nimport { DEFAULT_SCHEMA, migrate } from \"./base\";\nimport { type OnSubscribed, PostgresPubSub } from \"./pubsub\";\n\nexport const DEFAULT_LISTEN_CHANNEL = \"new_tasks\" as const;\nconst DEFAULT_PAGINATION_PAGE_SIZE = 100 as const;\n\ninterface BackendPostgresOptions {\n namespaceId?: string;\n runMigrations?: boolean;\n\n // default: true\n usePubSub?: boolean;\n}\n\nconst logger = getLogger([\"sonamu\", \"internal\", \"tasks\"]);\nconst queryLogger = getLogger([\"sonamu\", \"internal\", \"tasks\", \"query\"]);\n\n/**\n * Manages a connection to a Postgres database for workflow operations.\n */\nexport class BackendPostgres implements Backend {\n private config: Knex.Config;\n private namespaceId: string;\n private usePubSub: boolean;\n private pubsub: PostgresPubSub | null = null;\n private initialized: boolean = false;\n private runMigrations: boolean;\n\n private _knex: Knex | null = null;\n private get knex(): Knex {\n if (!this._knex) {\n this._knex = knex(this.config);\n this._knex.on(\"query\", (query) => {\n queryLogger.debug(\"SQL: {query}, Values: {bindings}\", {\n query: query.sql,\n bindings: query.bindings,\n });\n });\n }\n\n return this._knex;\n }\n\n constructor(config: Knex.Config, options?: BackendPostgresOptions) {\n this.config = {\n ...config,\n postProcessResponse: (result, _queryContext) => {\n if (result === null || result === undefined) {\n return result;\n }\n\n if (config?.postProcessResponse) {\n result = config.postProcessResponse(result, _queryContext);\n }\n\n const camelizeRow = (row: Record<string, unknown>) =>\n Object.fromEntries(\n Object.entries(row).map(([key, value]) => [camelize(key, true), value]),\n );\n\n if (Array.isArray(result)) {\n return result.map(camelizeRow);\n }\n\n return camelizeRow(result);\n },\n };\n\n const { namespaceId, usePubSub, runMigrations } = {\n namespaceId: DEFAULT_NAMESPACE_ID,\n usePubSub: true,\n runMigrations: true,\n ...options,\n };\n\n this.namespaceId = namespaceId;\n this.usePubSub = usePubSub;\n this.runMigrations = runMigrations;\n }\n\n async initialize() {\n if (this.initialized) {\n return;\n }\n\n if (this.runMigrations) {\n await migrate(this.config, DEFAULT_SCHEMA);\n }\n\n this.initialized = true;\n }\n\n async subscribe(callback: OnSubscribed) {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n if (!this.pubsub) {\n this.pubsub = await PostgresPubSub.create(this.knex);\n }\n\n this.pubsub.listenEvent(DEFAULT_LISTEN_CHANNEL, callback);\n }\n\n async publish(payload?: string): Promise<void> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n if (!this.usePubSub) {\n return;\n }\n\n await this.knex.raw(\n payload\n ? `NOTIFY ${DEFAULT_LISTEN_CHANNEL}, '${payload}'`\n : `NOTIFY ${DEFAULT_LISTEN_CHANNEL}`,\n );\n }\n\n async stop(): Promise<void> {\n if (!this.initialized) {\n return;\n }\n\n await this.pubsub?.destroy();\n this.pubsub = null;\n await this.knex.destroy();\n }\n\n async createWorkflowRun(params: CreateWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Creating workflow run: {workflowName}:{version}\", {\n workflowName: params.workflowName,\n version: params.version,\n });\n\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_name: params.workflowName,\n version: params.version,\n status: \"pending\",\n idempotency_key: params.idempotencyKey,\n config: params.config,\n context: params.context,\n input: params.input,\n attempts: 0,\n available_at: params.availableAt ?? this.knex.fn.now(),\n deadline_at: params.deadlineAt,\n created_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n const workflowRun = await qb;\n if (!workflowRun[0]) {\n logger.error(\"Failed to create workflow run: {params}\", { params });\n throw new Error(\"Failed to create workflow run\");\n }\n\n return workflowRun[0];\n }\n\n async getWorkflowRun(params: GetWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Getting workflow run: {workflowRunId}\", { workflowRunId: params.workflowRunId });\n const workflowRun = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .select(\n \"namespace_id\",\n \"id\",\n \"workflow_name\",\n \"version\",\n \"status\",\n \"idempotency_key\",\n \"config\",\n \"context\",\n \"input\",\n \"output\",\n \"error\",\n \"attempts\",\n \"parent_step_attempt_namespace_id\",\n \"parent_step_attempt_id\",\n \"worker_id\",\n \"available_at\",\n \"deadline_at\",\n \"started_at\",\n \"finished_at\",\n \"created_at\",\n \"updated_at\",\n )\n .first();\n\n return workflowRun ?? null;\n }\n\n async listWorkflowRuns(params: ListWorkflowRunsParams): Promise<PaginatedResponse<WorkflowRun>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Listing workflow runs: {after}, {before}\", {\n after: params.after,\n before: params.before,\n });\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListWorkflowRunsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListWorkflowRunsWhere(params: ListWorkflowRunsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n async claimWorkflowRun(params: ClaimWorkflowRunParams): Promise<WorkflowRun | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Claiming workflow run: {workerId}, {leaseDurationMs}\", {\n workerId: params.workerId,\n leaseDurationMs: params.leaseDurationMs,\n });\n const claimed = await this.knex\n .with(\"expired\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .update({\n status: \"failed\",\n error: JSON.stringify({ message: \"Workflow run deadline exceeded\" }),\n worker_id: null,\n available_at: null,\n finished_at: this.knex.raw(\"NOW()\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .whereNotNull(\"deadline_at\")\n .where(\"deadline_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .returning(\"id\"),\n )\n .with(\"candidate\", (qb) =>\n qb\n .withSchema(DEFAULT_SCHEMA)\n .select(\"id\")\n .from(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .where(\"available_at\", \"<=\", this.knex.raw(\"NOW()\"))\n .where((qb2) => {\n qb2.whereNull(\"deadline_at\").orWhere(\"deadline_at\", \">\", this.knex.raw(\"NOW()\"));\n })\n .orderByRaw(\"CASE WHEN status = 'pending' THEN 0 ELSE 1 END\")\n .orderBy(\"available_at\", \"asc\")\n .orderBy(\"created_at\", \"asc\")\n .limit(1)\n .forUpdate()\n .skipLocked(),\n )\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs as wr\")\n .where(\"wr.namespace_id\", this.namespaceId)\n .where(\"wr.id\", this.knex.ref(\"candidate.id\"))\n .update({\n status: \"running\",\n attempts: this.knex.raw(\"wr.attempts + 1\"),\n worker_id: params.workerId,\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n started_at: this.knex.raw(\"COALESCE(wr.started_at, NOW())\"),\n updated_at: this.knex.raw(\"NOW()\"),\n })\n .updateFrom(\"candidate\")\n .returning(\"wr.*\");\n\n return claimed[0] ?? null;\n }\n\n async extendWorkflowRunLease(params: ExtendWorkflowRunLeaseParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Extending workflow run lease: {workflowRunId}, {workerId}, {leaseDurationMs}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n leaseDurationMs: params.leaseDurationMs,\n });\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n available_at: this.knex.raw(`NOW() + ${params.leaseDurationMs} * INTERVAL '1 millisecond'`),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to extend lease for workflow run: {params}\", { params });\n throw new Error(\"Failed to extend lease for workflow run\");\n }\n\n return updated;\n }\n\n async sleepWorkflowRun(params: SleepWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Sleeping workflow run: {workflowRunId}, {workerId}, {availableAt}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n availableAt: params.availableAt,\n });\n\n // 'succeeded' status is deprecated\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereNotIn(\"status\", [\"succeeded\", \"completed\", \"failed\", \"canceled\"])\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"sleeping\",\n available_at: params.availableAt,\n worker_id: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to sleep workflow run: {params}\", { params });\n throw new Error(\"Failed to sleep workflow run\");\n }\n\n return updated;\n }\n\n async completeWorkflowRun(params: CompleteWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Completing workflow run: {workflowRunId}, {workerId}, {output}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n output: params.output,\n });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n worker_id: params.workerId,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to complete workflow run: {params}\", { params });\n throw new Error(\"Failed to complete workflow run\");\n }\n\n return updated;\n }\n\n async failWorkflowRun(params: FailWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n const { workflowRunId, error } = params;\n const { initialIntervalMs, backoffCoefficient, maximumIntervalMs } = DEFAULT_RETRY_POLICY;\n\n // this beefy query updates a workflow's status, available_at, and\n // finished_at based on the workflow's deadline and retry policy\n //\n // if the next retry would exceed the deadline, the run is marked as\n // 'failed' and finalized, otherwise, the run is rescheduled with an updated\n // 'available_at' timestamp for the next retry\n const retryIntervalExpr = `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, \"attempts\" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;\n const deadlineExceededCondition = `\"deadline_at\" IS NOT NULL AND NOW() + (${retryIntervalExpr}) >= \"deadline_at\"`;\n\n logger.info(\"Failing workflow run: {workflowRunId}, {workerId}, {error}\", {\n workflowRunId: params.workflowRunId,\n workerId: params.workerId,\n error: params.error,\n });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", workflowRunId)\n .where(\"status\", \"running\")\n .where(\"worker_id\", params.workerId)\n .update({\n status: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN 'failed' ELSE 'pending' END`,\n ),\n available_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NULL ELSE NOW() + (${retryIntervalExpr}) END`,\n ),\n finished_at: this.knex.raw(\n `CASE WHEN ${deadlineExceededCondition} THEN NOW() ELSE NULL END`,\n ),\n error: JSON.stringify(error),\n worker_id: null,\n started_at: null,\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n logger.error(\"Failed to mark workflow run failed: {params}\", { params });\n throw new Error(\"Failed to mark workflow run failed\");\n }\n\n return updated;\n }\n\n async cancelWorkflowRun(params: CancelWorkflowRunParams): Promise<WorkflowRun> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Canceling workflow run: {workflowRunId}\", { workflowRunId: params.workflowRunId });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"workflow_runs\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.workflowRunId)\n .whereIn(\"status\", [\"pending\", \"running\", \"sleeping\"])\n .update({\n status: \"canceled\",\n worker_id: null,\n available_at: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!updated) {\n // workflow may already be in a terminal state\n const existing = await this.getWorkflowRun({\n workflowRunId: params.workflowRunId,\n });\n if (!existing) {\n throw new Error(`Workflow run ${params.workflowRunId} does not exist`);\n }\n\n // if already canceled, just return it\n if (existing.status === \"canceled\") {\n return existing;\n }\n\n // throw error for completed/failed workflows\n // 'succeeded' status is deprecated\n if ([\"succeeded\", \"completed\", \"failed\"].includes(existing.status)) {\n logger.error(\"Cannot cancel workflow run: {params} with status {status}\", {\n params,\n status: existing.status,\n });\n throw new Error(\n `Cannot cancel workflow run ${params.workflowRunId} with status ${existing.status}`,\n );\n }\n\n logger.error(\"Failed to cancel workflow run: {params}\", { params });\n throw new Error(\"Failed to cancel workflow run\");\n }\n\n return updated;\n }\n\n async createStepAttempt(params: CreateStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Creating step attempt: {workflowRunId}, {stepName}, {kind}\", {\n workflowRunId: params.workflowRunId,\n stepName: params.stepName,\n kind: params.kind,\n });\n\n const [stepAttempt] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .insert({\n namespace_id: this.namespaceId,\n id: crypto.randomUUID(),\n workflow_run_id: params.workflowRunId,\n step_name: params.stepName,\n kind: params.kind,\n status: \"running\",\n config: JSON.stringify(params.config),\n context: JSON.stringify(params.context),\n started_at: this.knex.fn.now(),\n created_at: this.knex.raw(\"date_trunc('milliseconds', NOW())\"),\n updated_at: this.knex.fn.now(),\n })\n .returning(\"*\");\n\n if (!stepAttempt) {\n logger.error(\"Failed to create step attempt: {params}\", { params });\n throw new Error(\"Failed to create step attempt\");\n }\n\n return stepAttempt;\n }\n\n async getStepAttempt(params: GetStepAttemptParams): Promise<StepAttempt | null> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Getting step attempt: {stepAttemptId}\", { stepAttemptId: params.stepAttemptId });\n\n const stepAttempt = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"id\", params.stepAttemptId)\n .first();\n\n return stepAttempt ?? null;\n }\n\n async listStepAttempts(params: ListStepAttemptsParams): Promise<PaginatedResponse<StepAttempt>> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Listing step attempts: {workflowRunId}, {after}, {before}\", {\n workflowRunId: params.workflowRunId,\n after: params.after,\n before: params.before,\n });\n\n const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;\n const { after, before } = params;\n\n let cursor: Cursor | null = null;\n if (after) {\n cursor = decodeCursor(after);\n } else if (before) {\n cursor = decodeCursor(before);\n }\n\n const qb = this.buildListStepAttemptsWhere(params, cursor);\n const rows = await qb\n .orderBy(\"created_at\", before ? \"desc\" : \"asc\")\n .orderBy(\"id\", before ? \"desc\" : \"asc\")\n .limit(limit + 1);\n\n return this.processPaginationResults(\n rows,\n limit,\n typeof after === \"string\",\n typeof before === \"string\",\n );\n }\n\n private buildListStepAttemptsWhere(params: ListStepAttemptsParams, cursor: Cursor | null) {\n const { after } = params;\n const qb = this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts\")\n .where(\"namespace_id\", this.namespaceId)\n .where(\"workflow_run_id\", params.workflowRunId);\n\n if (cursor) {\n const operator = after ? \">\" : \"<\";\n return qb.whereRaw(`(\"created_at\", \"id\") ${operator} (?, ?)`, [\n cursor.createdAt.toISOString(),\n cursor.id,\n ]);\n }\n\n return qb;\n }\n\n private processPaginationResults<T extends Cursor>(\n rows: T[],\n limit: number,\n hasAfter: boolean,\n hasBefore: boolean,\n ): PaginatedResponse<T> {\n const data = rows;\n let hasNext = false;\n let hasPrev = false;\n\n if (hasBefore) {\n data.reverse();\n if (data.length > limit) {\n hasPrev = true;\n data.shift();\n }\n hasNext = true;\n } else {\n if (data.length > limit) {\n hasNext = true;\n data.pop();\n }\n if (hasAfter) {\n hasPrev = true;\n }\n }\n\n const lastItem = data.at(-1);\n const nextCursor = hasNext && lastItem ? encodeCursor(lastItem) : null;\n const firstItem = data[0];\n const prevCursor = hasPrev && firstItem ? encodeCursor(firstItem) : null;\n\n return {\n data,\n pagination: {\n next: nextCursor,\n prev: prevCursor,\n },\n };\n }\n\n // NOTE: 실제 서비스에서 이게 안 되는 것 같은데, 쿼리 등을 체크할 필요가 있음.\n async completeStepAttempt(params: CompleteStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Marking step attempt as completed: {workflowRunId}, {stepAttemptId}, {workerId}\", {\n workflowRunId: params.workflowRunId,\n stepAttemptId: params.stepAttemptId,\n workerId: params.workerId,\n });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"completed\",\n output: JSON.stringify(params.output),\n error: null,\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt completed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt completed\");\n }\n\n return updated;\n }\n\n async failStepAttempt(params: FailStepAttemptParams): Promise<StepAttempt> {\n if (!this.initialized) {\n throw new Error(\"Backend not initialized\");\n }\n\n logger.info(\"Marking step attempt as failed: {workflowRunId}, {stepAttemptId}, {workerId}\", {\n workflowRunId: params.workflowRunId,\n stepAttemptId: params.stepAttemptId,\n workerId: params.workerId,\n });\n logger.info(\"Error: {error.message}\", { error: params.error.message });\n\n const [updated] = await this.knex\n .withSchema(DEFAULT_SCHEMA)\n .table(\"step_attempts as sa\")\n .update({\n status: \"failed\",\n output: null,\n error: JSON.stringify(params.error),\n finished_at: this.knex.fn.now(),\n updated_at: this.knex.fn.now(),\n })\n .updateFrom(`${DEFAULT_SCHEMA}.workflow_runs as wr`)\n .where(\"sa.namespace_id\", this.namespaceId)\n .where(\"sa.workflow_run_id\", params.workflowRunId)\n .where(\"sa.id\", params.stepAttemptId)\n .where(\"sa.status\", \"running\")\n .where(\"wr.namespace_id\", this.knex.ref(\"sa.namespace_id\"))\n .where(\"wr.id\", this.knex.ref(\"sa.workflow_run_id\"))\n .where(\"wr.status\", \"running\")\n .where(\"wr.worker_id\", params.workerId)\n .returning(\"sa.*\");\n\n if (!updated) {\n logger.error(\"Failed to mark step attempt failed: {params}\", { params });\n throw new Error(\"Failed to mark step attempt failed\");\n }\n\n return updated;\n }\n}\n\n/**\n * Cursor used for pagination. Requires created_at and id fields. Because JS\n * Date does not natively support microsecond precision dates, created_at should\n * be stored with millisecond precision in paginated tables to avoid issues with\n * cursor comparisons.\n */\ninterface Cursor {\n createdAt: Date;\n id: string;\n}\n\nfunction encodeCursor(item: Cursor): string {\n const encoded = Buffer.from(\n JSON.stringify({ createdAt: item.createdAt.toISOString(), id: item.id }),\n ).toString(\"base64\");\n return encoded;\n}\n\nexport function decodeCursor(cursor: string): Cursor {\n const decoded = Buffer.from(cursor, \"base64\").toString(\"utf8\");\n const parsed = JSON.parse(decoded) as { createdAt: string; id: string };\n return {\n createdAt: new Date(parsed.createdAt),\n id: parsed.id,\n };\n}\n"],"names":["getLogger","camelize","knex","DEFAULT_NAMESPACE_ID","DEFAULT_RETRY_POLICY","DEFAULT_SCHEMA","migrate","PostgresPubSub","DEFAULT_LISTEN_CHANNEL","DEFAULT_PAGINATION_PAGE_SIZE","logger","queryLogger","BackendPostgres","config","namespaceId","usePubSub","pubsub","initialized","runMigrations","_knex","on","query","debug","sql","bindings","options","postProcessResponse","result","_queryContext","undefined","camelizeRow","row","Object","fromEntries","entries","map","key","value","Array","isArray","initialize","subscribe","callback","Error","create","listenEvent","publish","payload","raw","stop","destroy","createWorkflowRun","params","info","workflowName","version","qb","withSchema","table","insert","namespace_id","id","crypto","randomUUID","workflow_name","status","idempotency_key","idempotencyKey","context","input","attempts","available_at","availableAt","fn","now","deadline_at","deadlineAt","created_at","updated_at","returning","workflowRun","error","getWorkflowRun","workflowRunId","where","select","first","listWorkflowRuns","after","before","limit","cursor","decodeCursor","buildListWorkflowRunsWhere","rows","orderBy","processPaginationResults","operator","whereRaw","createdAt","toISOString","claimWorkflowRun","workerId","leaseDurationMs","claimed","with","update","JSON","stringify","message","worker_id","finished_at","whereIn","whereNotNull","from","qb2","whereNull","orWhere","orderByRaw","forUpdate","skipLocked","ref","started_at","updateFrom","extendWorkflowRunLease","updated","sleepWorkflowRun","whereNotIn","completeWorkflowRun","output","failWorkflowRun","initialIntervalMs","backoffCoefficient","maximumIntervalMs","retryIntervalExpr","deadlineExceededCondition","cancelWorkflowRun","existing","includes","createStepAttempt","stepName","kind","stepAttempt","workflow_run_id","step_name","getStepAttempt","stepAttemptId","listStepAttempts","buildListStepAttemptsWhere","hasAfter","hasBefore","data","hasNext","hasPrev","reverse","length","shift","pop","lastItem","at","nextCursor","encodeCursor","firstItem","prevCursor","pagination","next","prev","completeStepAttempt","failStepAttempt","item","encoded","Buffer","toString","decoded","parsed","parse","Date"],"mappings":"AAAA,SAASA,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,QAAQ,QAAQ,aAAa;AACtC,OAAOC,UAAyB,OAAO;AACvC,SAQEC,oBAAoB,QAUf,gBAAa;AACpB,SAASC,oBAAoB,QAAQ,mBAAgB;AAGrD,SAASC,cAAc,EAAEC,OAAO,QAAQ,YAAS;AACjD,SAA4BC,cAAc,QAAQ,cAAW;AAE7D,OAAO,MAAMC,yBAAyB,YAAqB;AAC3D,MAAMC,+BAA+B;AAUrC,MAAMC,SAASV,UAAU;IAAC;IAAU;IAAY;CAAQ;AACxD,MAAMW,cAAcX,UAAU;IAAC;IAAU;IAAY;IAAS;CAAQ;AAEtE;;CAEC,GACD,OAAO,MAAMY;IACHC,OAAoB;IACpBC,YAAoB;IACpBC,UAAmB;IACnBC,SAAgC,KAAK;IACrCC,cAAuB,MAAM;IAC7BC,cAAuB;IAEvBC,QAAqB,KAAK;IAClC,IAAYjB,OAAa;QACvB,IAAI,CAAC,IAAI,CAACiB,KAAK,EAAE;YACf,IAAI,CAACA,KAAK,GAAGjB,KAAK,IAAI,CAACW,MAAM;YAC7B,IAAI,CAACM,KAAK,CAACC,EAAE,CAAC,SAAS,CAACC;gBACtBV,YAAYW,KAAK,CAAC,oCAAoC;oBACpDD,OAAOA,MAAME,GAAG;oBAChBC,UAAUH,MAAMG,QAAQ;gBAC1B;YACF;QACF;QAEA,OAAO,IAAI,CAACL,KAAK;IACnB;IAEA,YAAYN,MAAmB,EAAEY,OAAgC,CAAE;QACjE,IAAI,CAACZ,MAAM,GAAG;YACZ,GAAGA,MAAM;YACTa,qBAAqB,CAACC,QAAQC;gBAC5B,IAAID,WAAW,QAAQA,WAAWE,WAAW;oBAC3C,OAAOF;gBACT;gBAEA,IAAId,QAAQa,qBAAqB;oBAC/BC,SAASd,OAAOa,mBAAmB,CAACC,QAAQC;gBAC9C;gBAEA,MAAME,cAAc,CAACC,MACnBC,OAAOC,WAAW,CAChBD,OAAOE,OAAO,CAACH,KAAKI,GAAG,CAAC,CAAC,CAACC,KAAKC,MAAM,GAAK;4BAACpC,SAASmC,KAAK;4BAAOC;yBAAM;gBAG1E,IAAIC,MAAMC,OAAO,CAACZ,SAAS;oBACzB,OAAOA,OAAOQ,GAAG,CAACL;gBACpB;gBAEA,OAAOA,YAAYH;YACrB;QACF;QAEA,MAAM,EAAEb,WAAW,EAAEC,SAAS,EAAEG,aAAa,EAAE,GAAG;YAChDJ,aAAaX;YACbY,WAAW;YACXG,eAAe;YACf,GAAGO,OAAO;QACZ;QAEA,IAAI,CAACX,WAAW,GAAGA;QACnB,IAAI,CAACC,SAAS,GAAGA;QACjB,IAAI,CAACG,aAAa,GAAGA;IACvB;IAEA,MAAMsB,aAAa;QACjB,IAAI,IAAI,CAACvB,WAAW,EAAE;YACpB;QACF;QAEA,IAAI,IAAI,CAACC,aAAa,EAAE;YACtB,MAAMZ,QAAQ,IAAI,CAACO,MAAM,EAAER;QAC7B;QAEA,IAAI,CAACY,WAAW,GAAG;IACrB;IAEA,MAAMwB,UAAUC,QAAsB,EAAE;QACtC,IAAI,CAAC,IAAI,CAACzB,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAAC5B,SAAS,EAAE;YACnB;QACF;QAEA,IAAI,CAAC,IAAI,CAACC,MAAM,EAAE;YAChB,IAAI,CAACA,MAAM,GAAG,MAAMT,eAAeqC,MAAM,CAAC,IAAI,CAAC1C,IAAI;QACrD;QAEA,IAAI,CAACc,MAAM,CAAC6B,WAAW,CAACrC,wBAAwBkC;IAClD;IAEA,MAAMI,QAAQC,OAAgB,EAAiB;QAC7C,IAAI,CAAC,IAAI,CAAC9B,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEA,IAAI,CAAC,IAAI,CAAC5B,SAAS,EAAE;YACnB;QACF;QAEA,MAAM,IAAI,CAACb,IAAI,CAAC8C,GAAG,CACjBD,UACI,CAAC,OAAO,EAAEvC,uBAAuB,GAAG,EAAEuC,QAAQ,CAAC,CAAC,GAChD,CAAC,OAAO,EAAEvC,wBAAwB;IAE1C;IAEA,MAAMyC,OAAsB;QAC1B,IAAI,CAAC,IAAI,CAAChC,WAAW,EAAE;YACrB;QACF;QAEA,MAAM,IAAI,CAACD,MAAM,EAAEkC;QACnB,IAAI,CAAClC,MAAM,GAAG;QACd,MAAM,IAAI,CAACd,IAAI,CAACgD,OAAO;IACzB;IAEA,MAAMC,kBAAkBC,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,mDAAmD;YAC7DC,cAAcF,OAAOE,YAAY;YACjCC,SAASH,OAAOG,OAAO;QACzB;QAEA,MAAMC,KAAK,IAAI,CAACtD,IAAI,CACjBuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAAC9C,WAAW;YAC9B+C,IAAIC,OAAOC,UAAU;YACrBC,eAAeZ,OAAOE,YAAY;YAClCC,SAASH,OAAOG,OAAO;YACvBU,QAAQ;YACRC,iBAAiBd,OAAOe,cAAc;YACtCtD,QAAQuC,OAAOvC,MAAM;YACrBuD,SAAShB,OAAOgB,OAAO;YACvBC,OAAOjB,OAAOiB,KAAK;YACnBC,UAAU;YACVC,cAAcnB,OAAOoB,WAAW,IAAI,IAAI,CAACtE,IAAI,CAACuE,EAAE,CAACC,GAAG;YACpDC,aAAavB,OAAOwB,UAAU;YAC9BC,YAAY,IAAI,CAAC3E,IAAI,CAACuE,EAAE,CAACC,GAAG;YAC5BI,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,MAAMC,cAAc,MAAMxB;QAC1B,IAAI,CAACwB,WAAW,CAAC,EAAE,EAAE;YACnBtE,OAAOuE,KAAK,CAAC,2CAA2C;gBAAE7B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOqC,WAAW,CAAC,EAAE;IACvB;IAEA,MAAME,eAAe9B,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,yCAAyC;YAAE8B,eAAe/B,OAAO+B,aAAa;QAAC;QAC3F,MAAMH,cAAc,MAAM,IAAI,CAAC9E,IAAI,CAChCuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,MAAMhC,OAAO+B,aAAa,EAChCE,MAAM,CACL,gBACA,MACA,iBACA,WACA,UACA,mBACA,UACA,WACA,SACA,UACA,SACA,YACA,oCACA,0BACA,aACA,gBACA,eACA,cACA,eACA,cACA,cAEDC,KAAK;QAER,OAAON,eAAe;IACxB;IAEA,MAAMO,iBAAiBnC,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,4CAA4C;YACtDmC,OAAOpC,OAAOoC,KAAK;YACnBC,QAAQrC,OAAOqC,MAAM;QACvB;QACA,MAAMC,QAAQtC,OAAOsC,KAAK,IAAIjF;QAC9B,MAAM,EAAE+E,KAAK,EAAEC,MAAM,EAAE,GAAGrC;QAE1B,IAAIuC,SAAwB;QAC5B,IAAIH,OAAO;YACTG,SAASC,aAAaJ;QACxB,OAAO,IAAIC,QAAQ;YACjBE,SAASC,aAAaH;QACxB;QAEA,MAAMjC,KAAK,IAAI,CAACqC,0BAA0B,CAACzC,QAAQuC;QACnD,MAAMG,OAAO,MAAMtC,GAChBuC,OAAO,CAAC,cAAcN,SAAS,SAAS,OACxCM,OAAO,CAAC,MAAMN,SAAS,SAAS,OAChCC,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACM,wBAAwB,CAClCF,MACAJ,OACA,OAAOF,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQI,2BAA2BzC,MAA8B,EAAEuC,MAAqB,EAAE;QACxF,MAAM,EAAEH,KAAK,EAAE,GAAGpC;QAClB,MAAMI,KAAK,IAAI,CAACtD,IAAI,CACjBuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW;QAEzC,IAAI6E,QAAQ;YACV,MAAMM,WAAWT,QAAQ,MAAM;YAC/B,OAAOhC,GAAG0C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAO9B,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEA,MAAM6C,iBAAiBjD,MAA8B,EAA+B;QAClF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,wDAAwD;YAClEiD,UAAUlD,OAAOkD,QAAQ;YACzBC,iBAAiBnD,OAAOmD,eAAe;QACzC;QACA,MAAMC,UAAU,MAAM,IAAI,CAACtG,IAAI,CAC5BuG,IAAI,CAAC,WAAW,CAACjD,KAChBA,GACGC,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACNgD,MAAM,CAAC;gBACNzC,QAAQ;gBACRgB,OAAO0B,KAAKC,SAAS,CAAC;oBAAEC,SAAS;gBAAiC;gBAClEC,WAAW;gBACXvC,cAAc;gBACdwC,aAAa,IAAI,CAAC7G,IAAI,CAAC8C,GAAG,CAAC;gBAC3B8B,YAAY,IAAI,CAAC5E,IAAI,CAAC8C,GAAG,CAAC;YAC5B,GACCoC,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCkG,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpDC,YAAY,CAAC,eACb7B,KAAK,CAAC,eAAe,MAAM,IAAI,CAAClF,IAAI,CAAC8C,GAAG,CAAC,UACzC+B,SAAS,CAAC,OAEd0B,IAAI,CAAC,aAAa,CAACjD,KAClBA,GACGC,UAAU,CAACpD,gBACXgF,MAAM,CAAC,MACP6B,IAAI,CAAC,iBACL9B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCkG,OAAO,CAAC,UAAU;gBAAC;gBAAW;gBAAW;aAAW,EACpD5B,KAAK,CAAC,gBAAgB,MAAM,IAAI,CAAClF,IAAI,CAAC8C,GAAG,CAAC,UAC1CoC,KAAK,CAAC,CAAC+B;gBACNA,IAAIC,SAAS,CAAC,eAAeC,OAAO,CAAC,eAAe,KAAK,IAAI,CAACnH,IAAI,CAAC8C,GAAG,CAAC;YACzE,GACCsE,UAAU,CAAC,kDACXvB,OAAO,CAAC,gBAAgB,OACxBA,OAAO,CAAC,cAAc,OACtBL,KAAK,CAAC,GACN6B,SAAS,GACTC,UAAU,IAEd/D,UAAU,CAACpD,gBACXqD,KAAK,CAAC,uBACN0B,KAAK,CAAC,mBAAmB,IAAI,CAACtE,WAAW,EACzCsE,KAAK,CAAC,SAAS,IAAI,CAAClF,IAAI,CAACuH,GAAG,CAAC,iBAC7Bf,MAAM,CAAC;YACNzC,QAAQ;YACRK,UAAU,IAAI,CAACpE,IAAI,CAAC8C,GAAG,CAAC;YACxB8D,WAAW1D,OAAOkD,QAAQ;YAC1B/B,cAAc,IAAI,CAACrE,IAAI,CAAC8C,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOmD,eAAe,CAAC,2BAA2B,CAAC;YAC1FmB,YAAY,IAAI,CAACxH,IAAI,CAAC8C,GAAG,CAAC;YAC1B8B,YAAY,IAAI,CAAC5E,IAAI,CAAC8C,GAAG,CAAC;QAC5B,GACC2E,UAAU,CAAC,aACX5C,SAAS,CAAC;QAEb,OAAOyB,OAAO,CAAC,EAAE,IAAI;IACvB;IAEA,MAAMoB,uBAAuBxE,MAAoC,EAAwB;QACvF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,gFAAgF;YAC1F8B,eAAe/B,OAAO+B,aAAa;YACnCmB,UAAUlD,OAAOkD,QAAQ;YACzBC,iBAAiBnD,OAAOmD,eAAe;QACzC;QACA,MAAM,CAACsB,QAAQ,GAAG,MAAM,IAAI,CAAC3H,IAAI,CAC9BuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,MAAMhC,OAAO+B,aAAa,EAChCC,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAahC,OAAOkD,QAAQ,EAClCI,MAAM,CAAC;YACNnC,cAAc,IAAI,CAACrE,IAAI,CAAC8C,GAAG,CAAC,CAAC,QAAQ,EAAEI,OAAOmD,eAAe,CAAC,2BAA2B,CAAC;YAC1FzB,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZnH,OAAOuE,KAAK,CAAC,qDAAqD;gBAAE7B;YAAO;YAC3E,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOkF;IACT;IAEA,MAAMC,iBAAiB1E,MAA8B,EAAwB;QAC3E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,qEAAqE;YAC/E8B,eAAe/B,OAAO+B,aAAa;YACnCmB,UAAUlD,OAAOkD,QAAQ;YACzB9B,aAAapB,OAAOoB,WAAW;QACjC;QAEA,mCAAmC;QACnC,MAAM,CAACqD,QAAQ,GAAG,MAAM,IAAI,CAAC3H,IAAI,CAC9BuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,MAAMhC,OAAO+B,aAAa,EAChC4C,UAAU,CAAC,UAAU;YAAC;YAAa;YAAa;YAAU;SAAW,EACrE3C,KAAK,CAAC,aAAahC,OAAOkD,QAAQ,EAClCI,MAAM,CAAC;YACNzC,QAAQ;YACRM,cAAcnB,OAAOoB,WAAW;YAChCsC,WAAW;YACXhC,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZnH,OAAOuE,KAAK,CAAC,0CAA0C;gBAAE7B;YAAO;YAChE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOkF;IACT;IAEA,MAAMG,oBAAoB5E,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,kEAAkE;YAC5E8B,eAAe/B,OAAO+B,aAAa;YACnCmB,UAAUlD,OAAOkD,QAAQ;YACzB2B,QAAQ7E,OAAO6E,MAAM;QACvB;QAEA,MAAM,CAACJ,QAAQ,GAAG,MAAM,IAAI,CAAC3H,IAAI,CAC9BuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,MAAMhC,OAAO+B,aAAa,EAChCC,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAahC,OAAOkD,QAAQ,EAClCI,MAAM,CAAC;YACNzC,QAAQ;YACRgE,QAAQtB,KAAKC,SAAS,CAACxD,OAAO6E,MAAM;YACpChD,OAAO;YACP6B,WAAW1D,OAAOkD,QAAQ;YAC1B/B,cAAc;YACdwC,aAAa,IAAI,CAAC7G,IAAI,CAACuE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZnH,OAAOuE,KAAK,CAAC,6CAA6C;gBAAE7B;YAAO;YACnE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOkF;IACT;IAEA,MAAMK,gBAAgB9E,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEA,MAAM,EAAEwC,aAAa,EAAEF,KAAK,EAAE,GAAG7B;QACjC,MAAM,EAAE+E,iBAAiB,EAAEC,kBAAkB,EAAEC,iBAAiB,EAAE,GAAGjI;QAErE,kEAAkE;QAClE,gEAAgE;QAChE,EAAE;QACF,oEAAoE;QACpE,4EAA4E;QAC5E,8CAA8C;QAC9C,MAAMkI,oBAAoB,CAAC,MAAM,EAAEH,kBAAkB,SAAS,EAAEC,mBAAmB,mBAAmB,EAAEC,kBAAkB,4BAA4B,CAAC;QACvJ,MAAME,4BAA4B,CAAC,uCAAuC,EAAED,kBAAkB,kBAAkB,CAAC;QAEjH5H,OAAO2C,IAAI,CAAC,8DAA8D;YACxE8B,eAAe/B,OAAO+B,aAAa;YACnCmB,UAAUlD,OAAOkD,QAAQ;YACzBrB,OAAO7B,OAAO6B,KAAK;QACrB;QAEA,MAAM,CAAC4C,QAAQ,GAAG,MAAM,IAAI,CAAC3H,IAAI,CAC9BuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,MAAMD,eACZC,KAAK,CAAC,UAAU,WAChBA,KAAK,CAAC,aAAahC,OAAOkD,QAAQ,EAClCI,MAAM,CAAC;YACNzC,QAAQ,IAAI,CAAC/D,IAAI,CAAC8C,GAAG,CACnB,CAAC,UAAU,EAAEuF,0BAA0B,iCAAiC,CAAC;YAE3EhE,cAAc,IAAI,CAACrE,IAAI,CAAC8C,GAAG,CACzB,CAAC,UAAU,EAAEuF,0BAA0B,yBAAyB,EAAED,kBAAkB,KAAK,CAAC;YAE5FvB,aAAa,IAAI,CAAC7G,IAAI,CAAC8C,GAAG,CACxB,CAAC,UAAU,EAAEuF,0BAA0B,yBAAyB,CAAC;YAEnEtD,OAAO0B,KAAKC,SAAS,CAAC3B;YACtB6B,WAAW;YACXY,YAAY;YACZ5C,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZnH,OAAOuE,KAAK,CAAC,gDAAgD;gBAAE7B;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOkF;IACT;IAEA,MAAMW,kBAAkBpF,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,2CAA2C;YAAE8B,eAAe/B,OAAO+B,aAAa;QAAC;QAE7F,MAAM,CAAC0C,QAAQ,GAAG,MAAM,IAAI,CAAC3H,IAAI,CAC9BuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,MAAMhC,OAAO+B,aAAa,EAChC6B,OAAO,CAAC,UAAU;YAAC;YAAW;YAAW;SAAW,EACpDN,MAAM,CAAC;YACNzC,QAAQ;YACR6C,WAAW;YACXvC,cAAc;YACdwC,aAAa,IAAI,CAAC7G,IAAI,CAACuE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZ,8CAA8C;YAC9C,MAAMY,WAAW,MAAM,IAAI,CAACvD,cAAc,CAAC;gBACzCC,eAAe/B,OAAO+B,aAAa;YACrC;YACA,IAAI,CAACsD,UAAU;gBACb,MAAM,IAAI9F,MAAM,CAAC,aAAa,EAAES,OAAO+B,aAAa,CAAC,eAAe,CAAC;YACvE;YAEA,sCAAsC;YACtC,IAAIsD,SAASxE,MAAM,KAAK,YAAY;gBAClC,OAAOwE;YACT;YAEA,6CAA6C;YAC7C,mCAAmC;YACnC,IAAI;gBAAC;gBAAa;gBAAa;aAAS,CAACC,QAAQ,CAACD,SAASxE,MAAM,GAAG;gBAClEvD,OAAOuE,KAAK,CAAC,6DAA6D;oBACxE7B;oBACAa,QAAQwE,SAASxE,MAAM;gBACzB;gBACA,MAAM,IAAItB,MACR,CAAC,2BAA2B,EAAES,OAAO+B,aAAa,CAAC,aAAa,EAAEsD,SAASxE,MAAM,EAAE;YAEvF;YAEAvD,OAAOuE,KAAK,CAAC,2CAA2C;gBAAE7B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOkF;IACT;IAEA,MAAMc,kBAAkBvF,MAA+B,EAAwB;QAC7E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,8DAA8D;YACxE8B,eAAe/B,OAAO+B,aAAa;YACnCyD,UAAUxF,OAAOwF,QAAQ;YACzBC,MAAMzF,OAAOyF,IAAI;QACnB;QAEA,MAAM,CAACC,YAAY,GAAG,MAAM,IAAI,CAAC5I,IAAI,CAClCuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACNC,MAAM,CAAC;YACNC,cAAc,IAAI,CAAC9C,WAAW;YAC9B+C,IAAIC,OAAOC,UAAU;YACrBgF,iBAAiB3F,OAAO+B,aAAa;YACrC6D,WAAW5F,OAAOwF,QAAQ;YAC1BC,MAAMzF,OAAOyF,IAAI;YACjB5E,QAAQ;YACRpD,QAAQ8F,KAAKC,SAAS,CAACxD,OAAOvC,MAAM;YACpCuD,SAASuC,KAAKC,SAAS,CAACxD,OAAOgB,OAAO;YACtCsD,YAAY,IAAI,CAACxH,IAAI,CAACuE,EAAE,CAACC,GAAG;YAC5BG,YAAY,IAAI,CAAC3E,IAAI,CAAC8C,GAAG,CAAC;YAC1B8B,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCK,SAAS,CAAC;QAEb,IAAI,CAAC+D,aAAa;YAChBpI,OAAOuE,KAAK,CAAC,2CAA2C;gBAAE7B;YAAO;YACjE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOmG;IACT;IAEA,MAAMG,eAAe7F,MAA4B,EAA+B;QAC9E,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,yCAAyC;YAAE6F,eAAe9F,OAAO8F,aAAa;QAAC;QAE3F,MAAMJ,cAAc,MAAM,IAAI,CAAC5I,IAAI,CAChCuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,MAAMhC,OAAO8F,aAAa,EAChC5D,KAAK;QAER,OAAOwD,eAAe;IACxB;IAEA,MAAMK,iBAAiB/F,MAA8B,EAA2C;QAC9F,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,6DAA6D;YACvE8B,eAAe/B,OAAO+B,aAAa;YACnCK,OAAOpC,OAAOoC,KAAK;YACnBC,QAAQrC,OAAOqC,MAAM;QACvB;QAEA,MAAMC,QAAQtC,OAAOsC,KAAK,IAAIjF;QAC9B,MAAM,EAAE+E,KAAK,EAAEC,MAAM,EAAE,GAAGrC;QAE1B,IAAIuC,SAAwB;QAC5B,IAAIH,OAAO;YACTG,SAASC,aAAaJ;QACxB,OAAO,IAAIC,QAAQ;YACjBE,SAASC,aAAaH;QACxB;QAEA,MAAMjC,KAAK,IAAI,CAAC4F,0BAA0B,CAAChG,QAAQuC;QACnD,MAAMG,OAAO,MAAMtC,GAChBuC,OAAO,CAAC,cAAcN,SAAS,SAAS,OACxCM,OAAO,CAAC,MAAMN,SAAS,SAAS,OAChCC,KAAK,CAACA,QAAQ;QAEjB,OAAO,IAAI,CAACM,wBAAwB,CAClCF,MACAJ,OACA,OAAOF,UAAU,UACjB,OAAOC,WAAW;IAEtB;IAEQ2D,2BAA2BhG,MAA8B,EAAEuC,MAAqB,EAAE;QACxF,MAAM,EAAEH,KAAK,EAAE,GAAGpC;QAClB,MAAMI,KAAK,IAAI,CAACtD,IAAI,CACjBuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,iBACN0B,KAAK,CAAC,gBAAgB,IAAI,CAACtE,WAAW,EACtCsE,KAAK,CAAC,mBAAmBhC,OAAO+B,aAAa;QAEhD,IAAIQ,QAAQ;YACV,MAAMM,WAAWT,QAAQ,MAAM;YAC/B,OAAOhC,GAAG0C,QAAQ,CAAC,CAAC,qBAAqB,EAAED,SAAS,OAAO,CAAC,EAAE;gBAC5DN,OAAOQ,SAAS,CAACC,WAAW;gBAC5BT,OAAO9B,EAAE;aACV;QACH;QAEA,OAAOL;IACT;IAEQwC,yBACNF,IAAS,EACTJ,KAAa,EACb2D,QAAiB,EACjBC,SAAkB,EACI;QACtB,MAAMC,OAAOzD;QACb,IAAI0D,UAAU;QACd,IAAIC,UAAU;QAEd,IAAIH,WAAW;YACbC,KAAKG,OAAO;YACZ,IAAIH,KAAKI,MAAM,GAAGjE,OAAO;gBACvB+D,UAAU;gBACVF,KAAKK,KAAK;YACZ;YACAJ,UAAU;QACZ,OAAO;YACL,IAAID,KAAKI,MAAM,GAAGjE,OAAO;gBACvB8D,UAAU;gBACVD,KAAKM,GAAG;YACV;YACA,IAAIR,UAAU;gBACZI,UAAU;YACZ;QACF;QAEA,MAAMK,WAAWP,KAAKQ,EAAE,CAAC,CAAC;QAC1B,MAAMC,aAAaR,WAAWM,WAAWG,aAAaH,YAAY;QAClE,MAAMI,YAAYX,IAAI,CAAC,EAAE;QACzB,MAAMY,aAAaV,WAAWS,YAAYD,aAAaC,aAAa;QAEpE,OAAO;YACLX;YACAa,YAAY;gBACVC,MAAML;gBACNM,MAAMH;YACR;QACF;IACF;IAEA,kDAAkD;IAClD,MAAMI,oBAAoBnH,MAAiC,EAAwB;QACjF,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,mFAAmF;YAC7F8B,eAAe/B,OAAO+B,aAAa;YACnC+D,eAAe9F,OAAO8F,aAAa;YACnC5C,UAAUlD,OAAOkD,QAAQ;QAC3B;QAEA,MAAM,CAACuB,QAAQ,GAAG,MAAM,IAAI,CAAC3H,IAAI,CAC9BuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,uBACNgD,MAAM,CAAC;YACNzC,QAAQ;YACRgE,QAAQtB,KAAKC,SAAS,CAACxD,OAAO6E,MAAM;YACpChD,OAAO;YACP8B,aAAa,IAAI,CAAC7G,IAAI,CAACuE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCiD,UAAU,CAAC,GAAGtH,eAAe,oBAAoB,CAAC,EAClD+E,KAAK,CAAC,mBAAmB,IAAI,CAACtE,WAAW,EACzCsE,KAAK,CAAC,sBAAsBhC,OAAO+B,aAAa,EAChDC,KAAK,CAAC,SAAShC,OAAO8F,aAAa,EACnC9D,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAAClF,IAAI,CAACuH,GAAG,CAAC,oBACvCrC,KAAK,CAAC,SAAS,IAAI,CAAClF,IAAI,CAACuH,GAAG,CAAC,uBAC7BrC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgBhC,OAAOkD,QAAQ,EACrCvB,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZnH,OAAOuE,KAAK,CAAC,mDAAmD;gBAAE7B;YAAO;YACzE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOkF;IACT;IAEA,MAAM2C,gBAAgBpH,MAA6B,EAAwB;QACzE,IAAI,CAAC,IAAI,CAACnC,WAAW,EAAE;YACrB,MAAM,IAAI0B,MAAM;QAClB;QAEAjC,OAAO2C,IAAI,CAAC,gFAAgF;YAC1F8B,eAAe/B,OAAO+B,aAAa;YACnC+D,eAAe9F,OAAO8F,aAAa;YACnC5C,UAAUlD,OAAOkD,QAAQ;QAC3B;QACA5F,OAAO2C,IAAI,CAAC,0BAA0B;YAAE4B,OAAO7B,OAAO6B,KAAK,CAAC4B,OAAO;QAAC;QAEpE,MAAM,CAACgB,QAAQ,GAAG,MAAM,IAAI,CAAC3H,IAAI,CAC9BuD,UAAU,CAACpD,gBACXqD,KAAK,CAAC,uBACNgD,MAAM,CAAC;YACNzC,QAAQ;YACRgE,QAAQ;YACRhD,OAAO0B,KAAKC,SAAS,CAACxD,OAAO6B,KAAK;YAClC8B,aAAa,IAAI,CAAC7G,IAAI,CAACuE,EAAE,CAACC,GAAG;YAC7BI,YAAY,IAAI,CAAC5E,IAAI,CAACuE,EAAE,CAACC,GAAG;QAC9B,GACCiD,UAAU,CAAC,GAAGtH,eAAe,oBAAoB,CAAC,EAClD+E,KAAK,CAAC,mBAAmB,IAAI,CAACtE,WAAW,EACzCsE,KAAK,CAAC,sBAAsBhC,OAAO+B,aAAa,EAChDC,KAAK,CAAC,SAAShC,OAAO8F,aAAa,EACnC9D,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,mBAAmB,IAAI,CAAClF,IAAI,CAACuH,GAAG,CAAC,oBACvCrC,KAAK,CAAC,SAAS,IAAI,CAAClF,IAAI,CAACuH,GAAG,CAAC,uBAC7BrC,KAAK,CAAC,aAAa,WACnBA,KAAK,CAAC,gBAAgBhC,OAAOkD,QAAQ,EACrCvB,SAAS,CAAC;QAEb,IAAI,CAAC8C,SAAS;YACZnH,OAAOuE,KAAK,CAAC,gDAAgD;gBAAE7B;YAAO;YACtE,MAAM,IAAIT,MAAM;QAClB;QAEA,OAAOkF;IACT;AACF;AAaA,SAASoC,aAAaQ,IAAY;IAChC,MAAMC,UAAUC,OAAOzD,IAAI,CACzBP,KAAKC,SAAS,CAAC;QAAET,WAAWsE,KAAKtE,SAAS,CAACC,WAAW;QAAIvC,IAAI4G,KAAK5G,EAAE;IAAC,IACtE+G,QAAQ,CAAC;IACX,OAAOF;AACT;AAEA,OAAO,SAAS9E,aAAaD,MAAc;IACzC,MAAMkF,UAAUF,OAAOzD,IAAI,CAACvB,QAAQ,UAAUiF,QAAQ,CAAC;IACvD,MAAME,SAASnE,KAAKoE,KAAK,CAACF;IAC1B,OAAO;QACL1E,WAAW,IAAI6E,KAAKF,OAAO3E,SAAS;QACpCtC,IAAIiH,OAAOjH,EAAE;IACf;AACF"}
|
package/package.json
CHANGED
package/src/database/backend.ts
CHANGED
|
@@ -38,6 +38,7 @@ interface BackendPostgresOptions {
|
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
const logger = getLogger(["sonamu", "internal", "tasks"]);
|
|
41
|
+
const queryLogger = getLogger(["sonamu", "internal", "tasks", "query"]);
|
|
41
42
|
|
|
42
43
|
/**
|
|
43
44
|
* Manages a connection to a Postgres database for workflow operations.
|
|
@@ -54,6 +55,12 @@ export class BackendPostgres implements Backend {
|
|
|
54
55
|
private get knex(): Knex {
|
|
55
56
|
if (!this._knex) {
|
|
56
57
|
this._knex = knex(this.config);
|
|
58
|
+
this._knex.on("query", (query) => {
|
|
59
|
+
queryLogger.debug("SQL: {query}, Values: {bindings}", {
|
|
60
|
+
query: query.sql,
|
|
61
|
+
bindings: query.bindings,
|
|
62
|
+
});
|
|
63
|
+
});
|
|
57
64
|
}
|
|
58
65
|
|
|
59
66
|
return this._knex;
|
|
@@ -155,6 +162,11 @@ export class BackendPostgres implements Backend {
|
|
|
155
162
|
throw new Error("Backend not initialized");
|
|
156
163
|
}
|
|
157
164
|
|
|
165
|
+
logger.info("Creating workflow run: {workflowName}:{version}", {
|
|
166
|
+
workflowName: params.workflowName,
|
|
167
|
+
version: params.version,
|
|
168
|
+
});
|
|
169
|
+
|
|
158
170
|
const qb = this.knex
|
|
159
171
|
.withSchema(DEFAULT_SCHEMA)
|
|
160
172
|
.table("workflow_runs")
|
|
@@ -190,6 +202,7 @@ export class BackendPostgres implements Backend {
|
|
|
190
202
|
throw new Error("Backend not initialized");
|
|
191
203
|
}
|
|
192
204
|
|
|
205
|
+
logger.info("Getting workflow run: {workflowRunId}", { workflowRunId: params.workflowRunId });
|
|
193
206
|
const workflowRun = await this.knex
|
|
194
207
|
.withSchema(DEFAULT_SCHEMA)
|
|
195
208
|
.table("workflow_runs")
|
|
@@ -228,6 +241,10 @@ export class BackendPostgres implements Backend {
|
|
|
228
241
|
throw new Error("Backend not initialized");
|
|
229
242
|
}
|
|
230
243
|
|
|
244
|
+
logger.info("Listing workflow runs: {after}, {before}", {
|
|
245
|
+
after: params.after,
|
|
246
|
+
before: params.before,
|
|
247
|
+
});
|
|
231
248
|
const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;
|
|
232
249
|
const { after, before } = params;
|
|
233
250
|
|
|
@@ -275,6 +292,10 @@ export class BackendPostgres implements Backend {
|
|
|
275
292
|
throw new Error("Backend not initialized");
|
|
276
293
|
}
|
|
277
294
|
|
|
295
|
+
logger.info("Claiming workflow run: {workerId}, {leaseDurationMs}", {
|
|
296
|
+
workerId: params.workerId,
|
|
297
|
+
leaseDurationMs: params.leaseDurationMs,
|
|
298
|
+
});
|
|
278
299
|
const claimed = await this.knex
|
|
279
300
|
.with("expired", (qb) =>
|
|
280
301
|
qb
|
|
@@ -335,6 +356,11 @@ export class BackendPostgres implements Backend {
|
|
|
335
356
|
throw new Error("Backend not initialized");
|
|
336
357
|
}
|
|
337
358
|
|
|
359
|
+
logger.info("Extending workflow run lease: {workflowRunId}, {workerId}, {leaseDurationMs}", {
|
|
360
|
+
workflowRunId: params.workflowRunId,
|
|
361
|
+
workerId: params.workerId,
|
|
362
|
+
leaseDurationMs: params.leaseDurationMs,
|
|
363
|
+
});
|
|
338
364
|
const [updated] = await this.knex
|
|
339
365
|
.withSchema(DEFAULT_SCHEMA)
|
|
340
366
|
.table("workflow_runs")
|
|
@@ -361,6 +387,12 @@ export class BackendPostgres implements Backend {
|
|
|
361
387
|
throw new Error("Backend not initialized");
|
|
362
388
|
}
|
|
363
389
|
|
|
390
|
+
logger.info("Sleeping workflow run: {workflowRunId}, {workerId}, {availableAt}", {
|
|
391
|
+
workflowRunId: params.workflowRunId,
|
|
392
|
+
workerId: params.workerId,
|
|
393
|
+
availableAt: params.availableAt,
|
|
394
|
+
});
|
|
395
|
+
|
|
364
396
|
// 'succeeded' status is deprecated
|
|
365
397
|
const [updated] = await this.knex
|
|
366
398
|
.withSchema(DEFAULT_SCHEMA)
|
|
@@ -390,6 +422,12 @@ export class BackendPostgres implements Backend {
|
|
|
390
422
|
throw new Error("Backend not initialized");
|
|
391
423
|
}
|
|
392
424
|
|
|
425
|
+
logger.info("Completing workflow run: {workflowRunId}, {workerId}, {output}", {
|
|
426
|
+
workflowRunId: params.workflowRunId,
|
|
427
|
+
workerId: params.workerId,
|
|
428
|
+
output: params.output,
|
|
429
|
+
});
|
|
430
|
+
|
|
393
431
|
const [updated] = await this.knex
|
|
394
432
|
.withSchema(DEFAULT_SCHEMA)
|
|
395
433
|
.table("workflow_runs")
|
|
@@ -433,6 +471,12 @@ export class BackendPostgres implements Backend {
|
|
|
433
471
|
const retryIntervalExpr = `LEAST(${initialIntervalMs} * POWER(${backoffCoefficient}, "attempts" - 1), ${maximumIntervalMs}) * INTERVAL '1 millisecond'`;
|
|
434
472
|
const deadlineExceededCondition = `"deadline_at" IS NOT NULL AND NOW() + (${retryIntervalExpr}) >= "deadline_at"`;
|
|
435
473
|
|
|
474
|
+
logger.info("Failing workflow run: {workflowRunId}, {workerId}, {error}", {
|
|
475
|
+
workflowRunId: params.workflowRunId,
|
|
476
|
+
workerId: params.workerId,
|
|
477
|
+
error: params.error,
|
|
478
|
+
});
|
|
479
|
+
|
|
436
480
|
const [updated] = await this.knex
|
|
437
481
|
.withSchema(DEFAULT_SCHEMA)
|
|
438
482
|
.table("workflow_runs")
|
|
@@ -470,6 +514,8 @@ export class BackendPostgres implements Backend {
|
|
|
470
514
|
throw new Error("Backend not initialized");
|
|
471
515
|
}
|
|
472
516
|
|
|
517
|
+
logger.info("Canceling workflow run: {workflowRunId}", { workflowRunId: params.workflowRunId });
|
|
518
|
+
|
|
473
519
|
const [updated] = await this.knex
|
|
474
520
|
.withSchema(DEFAULT_SCHEMA)
|
|
475
521
|
.table("workflow_runs")
|
|
@@ -523,6 +569,12 @@ export class BackendPostgres implements Backend {
|
|
|
523
569
|
throw new Error("Backend not initialized");
|
|
524
570
|
}
|
|
525
571
|
|
|
572
|
+
logger.info("Creating step attempt: {workflowRunId}, {stepName}, {kind}", {
|
|
573
|
+
workflowRunId: params.workflowRunId,
|
|
574
|
+
stepName: params.stepName,
|
|
575
|
+
kind: params.kind,
|
|
576
|
+
});
|
|
577
|
+
|
|
526
578
|
const [stepAttempt] = await this.knex
|
|
527
579
|
.withSchema(DEFAULT_SCHEMA)
|
|
528
580
|
.table("step_attempts")
|
|
@@ -554,6 +606,8 @@ export class BackendPostgres implements Backend {
|
|
|
554
606
|
throw new Error("Backend not initialized");
|
|
555
607
|
}
|
|
556
608
|
|
|
609
|
+
logger.info("Getting step attempt: {stepAttemptId}", { stepAttemptId: params.stepAttemptId });
|
|
610
|
+
|
|
557
611
|
const stepAttempt = await this.knex
|
|
558
612
|
.withSchema(DEFAULT_SCHEMA)
|
|
559
613
|
.table("step_attempts")
|
|
@@ -569,6 +623,12 @@ export class BackendPostgres implements Backend {
|
|
|
569
623
|
throw new Error("Backend not initialized");
|
|
570
624
|
}
|
|
571
625
|
|
|
626
|
+
logger.info("Listing step attempts: {workflowRunId}, {after}, {before}", {
|
|
627
|
+
workflowRunId: params.workflowRunId,
|
|
628
|
+
after: params.after,
|
|
629
|
+
before: params.before,
|
|
630
|
+
});
|
|
631
|
+
|
|
572
632
|
const limit = params.limit ?? DEFAULT_PAGINATION_PAGE_SIZE;
|
|
573
633
|
const { after, before } = params;
|
|
574
634
|
|
|
@@ -653,11 +713,18 @@ export class BackendPostgres implements Backend {
|
|
|
653
713
|
};
|
|
654
714
|
}
|
|
655
715
|
|
|
716
|
+
// NOTE: 실제 서비스에서 이게 안 되는 것 같은데, 쿼리 등을 체크할 필요가 있음.
|
|
656
717
|
async completeStepAttempt(params: CompleteStepAttemptParams): Promise<StepAttempt> {
|
|
657
718
|
if (!this.initialized) {
|
|
658
719
|
throw new Error("Backend not initialized");
|
|
659
720
|
}
|
|
660
721
|
|
|
722
|
+
logger.info("Marking step attempt as completed: {workflowRunId}, {stepAttemptId}, {workerId}", {
|
|
723
|
+
workflowRunId: params.workflowRunId,
|
|
724
|
+
stepAttemptId: params.stepAttemptId,
|
|
725
|
+
workerId: params.workerId,
|
|
726
|
+
});
|
|
727
|
+
|
|
661
728
|
const [updated] = await this.knex
|
|
662
729
|
.withSchema(DEFAULT_SCHEMA)
|
|
663
730
|
.table("step_attempts as sa")
|
|
@@ -692,6 +759,13 @@ export class BackendPostgres implements Backend {
|
|
|
692
759
|
throw new Error("Backend not initialized");
|
|
693
760
|
}
|
|
694
761
|
|
|
762
|
+
logger.info("Marking step attempt as failed: {workflowRunId}, {stepAttemptId}, {workerId}", {
|
|
763
|
+
workflowRunId: params.workflowRunId,
|
|
764
|
+
stepAttemptId: params.stepAttemptId,
|
|
765
|
+
workerId: params.workerId,
|
|
766
|
+
});
|
|
767
|
+
logger.info("Error: {error.message}", { error: params.error.message });
|
|
768
|
+
|
|
695
769
|
const [updated] = await this.knex
|
|
696
770
|
.withSchema(DEFAULT_SCHEMA)
|
|
697
771
|
.table("step_attempts as sa")
|