@dbos-inc/dbos-sdk 4.10.8-preview → 4.10.10-preview
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/schemas/system_db_schema.d.ts +6 -0
- package/dist/schemas/system_db_schema.d.ts.map +1 -1
- package/dist/schemas/system_db_schema.js.map +1 -1
- package/dist/src/client.d.ts +4 -1
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +11 -1
- package/dist/src/client.js.map +1 -1
- package/dist/src/conductor/conductor.d.ts.map +1 -1
- package/dist/src/conductor/conductor.js +32 -0
- package/dist/src/conductor/conductor.js.map +1 -1
- package/dist/src/conductor/protocol.d.ts +28 -1
- package/dist/src/conductor/protocol.d.ts.map +1 -1
- package/dist/src/conductor/protocol.js +38 -1
- package/dist/src/conductor/protocol.js.map +1 -1
- package/dist/src/dbos-executor.d.ts +1 -1
- package/dist/src/dbos-executor.d.ts.map +1 -1
- package/dist/src/dbos-executor.js +2 -2
- package/dist/src/dbos-executor.js.map +1 -1
- package/dist/src/dbos.d.ts +15 -1
- package/dist/src/dbos.d.ts.map +1 -1
- package/dist/src/dbos.js +30 -0
- package/dist/src/dbos.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.d.ts.map +1 -1
- package/dist/src/scheduler/scheduler.js +17 -11
- package/dist/src/scheduler/scheduler.js.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.d.ts.map +1 -1
- package/dist/src/sysdb_migrations/internal/migrations.js +11 -0
- package/dist/src/sysdb_migrations/internal/migrations.js.map +1 -1
- package/dist/src/system_database.d.ts +58 -135
- package/dist/src/system_database.d.ts.map +1 -1
- package/dist/src/system_database.js +1189 -1125
- package/dist/src/system_database.js.map +1 -1
- package/dist/src/workflow.d.ts +1 -1
- package/dist/src/workflow.d.ts.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
|
@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
9
9
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.
|
|
12
|
+
exports.SystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
|
|
13
13
|
const dbos_executor_1 = require("./dbos-executor");
|
|
14
14
|
const pg_1 = require("pg");
|
|
15
15
|
const error_1 = require("./error");
|
|
@@ -146,180 +146,6 @@ class NotificationMap {
|
|
|
146
146
|
}
|
|
147
147
|
}
|
|
148
148
|
}
|
|
149
|
-
async function insertWorkflowStatus(client, initStatus, schemaName, ownerXid, incrementAttempts = false) {
|
|
150
|
-
try {
|
|
151
|
-
const { rows } = await client.query(`INSERT INTO "${schemaName}".workflow_status (
|
|
152
|
-
workflow_uuid,
|
|
153
|
-
status,
|
|
154
|
-
name,
|
|
155
|
-
class_name,
|
|
156
|
-
config_name,
|
|
157
|
-
queue_name,
|
|
158
|
-
authenticated_user,
|
|
159
|
-
assumed_role,
|
|
160
|
-
authenticated_roles,
|
|
161
|
-
request,
|
|
162
|
-
executor_id,
|
|
163
|
-
application_version,
|
|
164
|
-
application_id,
|
|
165
|
-
created_at,
|
|
166
|
-
recovery_attempts,
|
|
167
|
-
updated_at,
|
|
168
|
-
workflow_timeout_ms,
|
|
169
|
-
workflow_deadline_epoch_ms,
|
|
170
|
-
inputs,
|
|
171
|
-
deduplication_id,
|
|
172
|
-
priority,
|
|
173
|
-
queue_partition_key,
|
|
174
|
-
forked_from,
|
|
175
|
-
parent_workflow_id,
|
|
176
|
-
serialization,
|
|
177
|
-
owner_xid
|
|
178
|
-
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27)
|
|
179
|
-
ON CONFLICT (workflow_uuid)
|
|
180
|
-
DO UPDATE SET
|
|
181
|
-
recovery_attempts = CASE
|
|
182
|
-
WHEN workflow_status.status != '${workflow_1.StatusString.ENQUEUED}'
|
|
183
|
-
THEN workflow_status.recovery_attempts + $25
|
|
184
|
-
ELSE workflow_status.recovery_attempts
|
|
185
|
-
END,
|
|
186
|
-
updated_at = EXCLUDED.updated_at,
|
|
187
|
-
executor_id = CASE
|
|
188
|
-
WHEN EXCLUDED.status != '${workflow_1.StatusString.ENQUEUED}'
|
|
189
|
-
THEN EXCLUDED.executor_id
|
|
190
|
-
ELSE workflow_status.executor_id
|
|
191
|
-
END
|
|
192
|
-
RETURNING recovery_attempts, status, name, class_name, config_name, queue_name, workflow_deadline_epoch_ms, executor_id, owner_xid, serialization`, [
|
|
193
|
-
initStatus.workflowUUID,
|
|
194
|
-
initStatus.status,
|
|
195
|
-
initStatus.workflowName,
|
|
196
|
-
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
197
|
-
initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
198
|
-
initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
199
|
-
initStatus.queueName ?? null,
|
|
200
|
-
initStatus.authenticatedUser,
|
|
201
|
-
initStatus.assumedRole,
|
|
202
|
-
JSON.stringify(initStatus.authenticatedRoles),
|
|
203
|
-
JSON.stringify(initStatus.request),
|
|
204
|
-
initStatus.executorId,
|
|
205
|
-
initStatus.applicationVersion ?? null,
|
|
206
|
-
initStatus.applicationID,
|
|
207
|
-
initStatus.createdAt,
|
|
208
|
-
initStatus.status === workflow_1.StatusString.ENQUEUED ? 0 : 1,
|
|
209
|
-
initStatus.updatedAt ?? Date.now(),
|
|
210
|
-
initStatus.timeoutMS ?? null,
|
|
211
|
-
initStatus.deadlineEpochMS ?? null,
|
|
212
|
-
initStatus.input ?? null,
|
|
213
|
-
initStatus.deduplicationID ?? null,
|
|
214
|
-
initStatus.priority,
|
|
215
|
-
initStatus.queuePartitionKey ?? null,
|
|
216
|
-
initStatus.forkedFrom ?? null,
|
|
217
|
-
initStatus.parentWorkflowID ?? null,
|
|
218
|
-
(incrementAttempts ?? false) ? 1 : 0,
|
|
219
|
-
initStatus.serialization,
|
|
220
|
-
ownerXid,
|
|
221
|
-
]);
|
|
222
|
-
if (rows.length === 0) {
|
|
223
|
-
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
224
|
-
}
|
|
225
|
-
const ret = rows[0];
|
|
226
|
-
ret.class_name = ret.class_name ?? '';
|
|
227
|
-
ret.config_name = ret.config_name ?? '';
|
|
228
|
-
initStatus.serialization = ret.serialization;
|
|
229
|
-
return ret;
|
|
230
|
-
}
|
|
231
|
-
catch (error) {
|
|
232
|
-
const err = error;
|
|
233
|
-
if (err.code === '23505') {
|
|
234
|
-
throw new error_1.DBOSQueueDuplicatedError(initStatus.workflowUUID, initStatus.queueName ?? '', initStatus.deduplicationID ?? '');
|
|
235
|
-
}
|
|
236
|
-
throw error;
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
async function getWorkflowStatusValue(client, workflowID, schemaName) {
|
|
240
|
-
const { rows } = await client.query(`SELECT status FROM "${schemaName}".workflow_status WHERE workflow_uuid=$1`, [workflowID]);
|
|
241
|
-
return rows.length === 0 ? undefined : rows[0].status;
|
|
242
|
-
}
|
|
243
|
-
async function updateWorkflowStatus(client, workflowID, status, schemaName, options = {}) {
|
|
244
|
-
let setClause = `SET status=$2, updated_at=$3`;
|
|
245
|
-
let whereClause = `WHERE workflow_uuid=$1`;
|
|
246
|
-
const args = [workflowID, status, Date.now()];
|
|
247
|
-
const update = options.update ?? {};
|
|
248
|
-
if (update.output) {
|
|
249
|
-
const param = args.push(update.output);
|
|
250
|
-
setClause += `, output=$${param}`;
|
|
251
|
-
}
|
|
252
|
-
if (update.error) {
|
|
253
|
-
const param = args.push(update.error);
|
|
254
|
-
setClause += `, error=$${param}`;
|
|
255
|
-
}
|
|
256
|
-
if (update.resetRecoveryAttempts) {
|
|
257
|
-
setClause += `, recovery_attempts = 0`;
|
|
258
|
-
}
|
|
259
|
-
if (update.resetDeadline) {
|
|
260
|
-
setClause += `, workflow_deadline_epoch_ms = NULL`;
|
|
261
|
-
}
|
|
262
|
-
if (update.queueName !== undefined) {
|
|
263
|
-
const param = args.push(update.queueName ?? undefined);
|
|
264
|
-
setClause += `, queue_name=$${param}`;
|
|
265
|
-
}
|
|
266
|
-
if (update.resetDeduplicationID) {
|
|
267
|
-
setClause += `, deduplication_id = NULL`;
|
|
268
|
-
}
|
|
269
|
-
if (update.resetStartedAtEpochMs) {
|
|
270
|
-
setClause += `, started_at_epoch_ms = NULL`;
|
|
271
|
-
}
|
|
272
|
-
if (update.executorId !== undefined) {
|
|
273
|
-
const param = args.push(update.executorId ?? undefined);
|
|
274
|
-
setClause += `, executor_id=$${param}`;
|
|
275
|
-
}
|
|
276
|
-
if (update.resetNameTo !== undefined) {
|
|
277
|
-
const param = args.push(update.resetNameTo ?? undefined);
|
|
278
|
-
setClause += `, name=$${param}`;
|
|
279
|
-
}
|
|
280
|
-
const where = options.where ?? {};
|
|
281
|
-
if (where.status) {
|
|
282
|
-
const param = args.push(where.status);
|
|
283
|
-
whereClause += ` AND status=$${param}`;
|
|
284
|
-
}
|
|
285
|
-
const result = await client.query(`UPDATE "${schemaName}".workflow_status ${setClause} ${whereClause}`, args);
|
|
286
|
-
const throwOnFailure = options.throwOnFailure ?? true;
|
|
287
|
-
if (throwOnFailure && result.rowCount !== 1) {
|
|
288
|
-
throw new error_1.DBOSWorkflowConflictError(`Attempt to record transition of nonexistent workflow ${workflowID}`);
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
async function recordOperationResult(client, workflowID, functionID, functionName, checkConflict, schemaName, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
292
|
-
try {
|
|
293
|
-
const out = await client.query(`INSERT INTO ${schemaName}.operation_outputs
|
|
294
|
-
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization)
|
|
295
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
296
|
-
ON CONFLICT DO NOTHING RETURNING completed_at_epoch_ms;`, [
|
|
297
|
-
workflowID,
|
|
298
|
-
functionID,
|
|
299
|
-
options.output ?? null,
|
|
300
|
-
options.error ?? null,
|
|
301
|
-
functionName,
|
|
302
|
-
options.childWorkflowID ?? null,
|
|
303
|
-
startTimeEpochMs,
|
|
304
|
-
endTimeEpochMs,
|
|
305
|
-
options.serialization ?? null,
|
|
306
|
-
]);
|
|
307
|
-
if (checkConflict && (out?.rowCount ?? 0) > 0 && Number(out?.rows?.[0]?.completed_at_epoch_ms) !== endTimeEpochMs) {
|
|
308
|
-
dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Step output for ${workflowID}(${functionID}):${functionName} already recorded`);
|
|
309
|
-
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
catch (error) {
|
|
313
|
-
const err = error;
|
|
314
|
-
if (err.code === '40001' || err.code === '23505') {
|
|
315
|
-
// Serialization and primary key conflict (Postgres).
|
|
316
|
-
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
317
|
-
}
|
|
318
|
-
else {
|
|
319
|
-
throw err;
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
149
|
function mapWorkflowStatus(row) {
|
|
324
150
|
return {
|
|
325
151
|
workflowUUID: row.workflow_uuid,
|
|
@@ -489,10 +315,22 @@ function dbRetry(options = {}) {
|
|
|
489
315
|
return descriptor;
|
|
490
316
|
};
|
|
491
317
|
}
|
|
492
|
-
|
|
318
|
+
/**
|
|
319
|
+
* General notes:
|
|
320
|
+
* The responsibilities of the `SystemDatabase` are to store data for workflows, and
|
|
321
|
+
* associated steps, transactions, messages, and events. The system DB is
|
|
322
|
+
* also the IPC mechanism that performs notifications when things change, for
|
|
323
|
+
* example a receive is unblocked when a send occurs, or a cancel interrupts
|
|
324
|
+
* the receive.
|
|
325
|
+
* The `SystemDatabase` expects values in inputs/outputs/errors to be JSON. However,
|
|
326
|
+
* the serialization process of turning data into JSON or converting it back, should
|
|
327
|
+
* be done elsewhere (executor), as it may require application-specific logic or extensions.
|
|
328
|
+
*/
|
|
329
|
+
class SystemDatabase {
|
|
493
330
|
systemDatabaseUrl;
|
|
494
331
|
logger;
|
|
495
332
|
serializer;
|
|
333
|
+
// ==================== Lifecycle ====================
|
|
496
334
|
pool;
|
|
497
335
|
schemaName;
|
|
498
336
|
/*
|
|
@@ -574,6 +412,7 @@ class PostgresSystemDatabase {
|
|
|
574
412
|
}
|
|
575
413
|
await this.pool.end();
|
|
576
414
|
}
|
|
415
|
+
// ==================== Workflow Status ====================
|
|
577
416
|
async initWorkflowStatus(initStatus, ownerXid, options) {
|
|
578
417
|
const client = await this.pool.connect();
|
|
579
418
|
let shouldCommit = false;
|
|
@@ -581,7 +420,7 @@ class PostgresSystemDatabase {
|
|
|
581
420
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
582
421
|
// Moving from enqueued to pending asks to increment recovery attempts... rather than in the recovery process
|
|
583
422
|
// where it moves from pending back to enqueued.
|
|
584
|
-
const resRow = await insertWorkflowStatus(client, initStatus,
|
|
423
|
+
const resRow = await this.insertWorkflowStatus(client, initStatus, ownerXid, !!options?.isRecoveryRequest || !!options?.isDequeuedRequest);
|
|
585
424
|
if (resRow.name !== initStatus.workflowName) {
|
|
586
425
|
const msg = `Workflow already exists with a different function name: ${resRow.name}, but the provided function name is: ${initStatus.workflowName}`;
|
|
587
426
|
throw new error_1.DBOSConflictingWorkflowError(initStatus.workflowUUID, msg);
|
|
@@ -617,7 +456,7 @@ class PostgresSystemDatabase {
|
|
|
617
456
|
// Thus, when this number becomes equal to `maxRetries + 1`, we should mark the workflow as `MAX_RECOVERY_ATTEMPTS_EXCEEDED`.
|
|
618
457
|
const attempts = resRow.recovery_attempts;
|
|
619
458
|
if (options?.maxRetries && attempts > options?.maxRetries + 1) {
|
|
620
|
-
await updateWorkflowStatus(client, initStatus.workflowUUID, workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED,
|
|
459
|
+
await this.updateWorkflowStatus(client, initStatus.workflowUUID, workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, {
|
|
621
460
|
where: { status: workflow_1.StatusString.PENDING },
|
|
622
461
|
throwOnFailure: false,
|
|
623
462
|
});
|
|
@@ -649,7 +488,7 @@ class PostgresSystemDatabase {
|
|
|
649
488
|
async recordWorkflowOutput(workflowID, status) {
|
|
650
489
|
const client = await this.pool.connect();
|
|
651
490
|
try {
|
|
652
|
-
await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.SUCCESS,
|
|
491
|
+
await this.updateWorkflowStatus(client, workflowID, workflow_1.StatusString.SUCCESS, {
|
|
653
492
|
update: { output: status.output, resetDeduplicationID: true },
|
|
654
493
|
});
|
|
655
494
|
}
|
|
@@ -660,7 +499,7 @@ class PostgresSystemDatabase {
|
|
|
660
499
|
async recordWorkflowError(workflowID, status) {
|
|
661
500
|
const client = await this.pool.connect();
|
|
662
501
|
try {
|
|
663
|
-
await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.ERROR,
|
|
502
|
+
await this.updateWorkflowStatus(client, workflowID, workflow_1.StatusString.ERROR, {
|
|
664
503
|
update: { error: status.error, resetDeduplicationID: true },
|
|
665
504
|
});
|
|
666
505
|
}
|
|
@@ -677,23 +516,44 @@ class PostgresSystemDatabase {
|
|
|
677
516
|
queueName: i.queue_name,
|
|
678
517
|
}));
|
|
679
518
|
}
|
|
680
|
-
async
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
519
|
+
async getWorkflowStatus(workflowID, callerID, callerFN) {
|
|
520
|
+
const funcGetStatus = async () => {
|
|
521
|
+
const statuses = await this.listWorkflows({ workflowIDs: [workflowID] });
|
|
522
|
+
const status = statuses.find((s) => s.workflowUUID === workflowID);
|
|
523
|
+
return status ? JSON.stringify(status) : null;
|
|
524
|
+
};
|
|
525
|
+
if (callerID && callerFN) {
|
|
526
|
+
const client = await this.pool.connect();
|
|
527
|
+
try {
|
|
528
|
+
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
529
|
+
const json = await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_GETSTATUS, callerID, callerFN, funcGetStatus);
|
|
530
|
+
return parseStatus(json);
|
|
531
|
+
}
|
|
532
|
+
finally {
|
|
533
|
+
client.release();
|
|
534
|
+
}
|
|
687
535
|
}
|
|
688
536
|
else {
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
537
|
+
const json = await funcGetStatus();
|
|
538
|
+
return parseStatus(json);
|
|
539
|
+
}
|
|
540
|
+
function parseStatus(json) {
|
|
541
|
+
return json ? JSON.parse(json) : null;
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
// Only used in tests
|
|
545
|
+
async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
|
|
546
|
+
const client = await this.pool.connect();
|
|
547
|
+
try {
|
|
548
|
+
await this.updateWorkflowStatus(client, workflowID, status, {
|
|
549
|
+
update: { resetRecoveryAttempts, resetNameTo: internalOptions?.updateName },
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
finally {
|
|
553
|
+
client.release();
|
|
695
554
|
}
|
|
696
555
|
}
|
|
556
|
+
// ==================== Step Results ====================
|
|
697
557
|
async getOperationResultAndThrowIfCancelled(workflowID, functionID) {
|
|
698
558
|
const client = await this.pool.connect();
|
|
699
559
|
try {
|
|
@@ -711,13 +571,194 @@ class PostgresSystemDatabase {
|
|
|
711
571
|
const client = await this.pool.connect();
|
|
712
572
|
const now = Date.now();
|
|
713
573
|
try {
|
|
714
|
-
await
|
|
574
|
+
await this.recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, now, options);
|
|
715
575
|
}
|
|
716
576
|
finally {
|
|
717
577
|
client.release();
|
|
718
578
|
await (0, debugpoint_1.debugTriggerPoint)(debugpoint_1.DEBUG_TRIGGER_STEP_COMMIT);
|
|
719
579
|
}
|
|
720
580
|
}
|
|
581
|
+
async runTransactionalStep(workflowID, functionID, functionName, callback) {
|
|
582
|
+
const client = await this.pool.connect();
|
|
583
|
+
try {
|
|
584
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
585
|
+
const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
586
|
+
if (existing !== undefined) {
|
|
587
|
+
await client.query('ROLLBACK');
|
|
588
|
+
return existing;
|
|
589
|
+
}
|
|
590
|
+
const startTime = Date.now();
|
|
591
|
+
const output = await callback(client);
|
|
592
|
+
await this.recordOperationResultInternal(client, workflowID, functionID, functionName, true, startTime, Date.now(), {
|
|
593
|
+
output,
|
|
594
|
+
});
|
|
595
|
+
await client.query('COMMIT');
|
|
596
|
+
return undefined;
|
|
597
|
+
}
|
|
598
|
+
catch (e) {
|
|
599
|
+
await client.query('ROLLBACK');
|
|
600
|
+
throw e;
|
|
601
|
+
}
|
|
602
|
+
finally {
|
|
603
|
+
client.release();
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
async checkPatch(workflowID, functionID, patchName, deprecated) {
|
|
607
|
+
// Not doing a cancel check at this point.
|
|
608
|
+
if (functionID === undefined)
|
|
609
|
+
throw new TypeError('functionID must be defined');
|
|
610
|
+
patchName = `DBOS.patch-${patchName}`;
|
|
611
|
+
const { rows } = await this.pool.query(`SELECT function_name
|
|
612
|
+
FROM "${this.schemaName}".operation_outputs
|
|
613
|
+
WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
|
|
614
|
+
if (deprecated) {
|
|
615
|
+
// Deprecated does not write anything. We skip any existing matching patch marker if it matches
|
|
616
|
+
if (rows.length === 0) {
|
|
617
|
+
return { isPatched: true, hasEntry: false };
|
|
618
|
+
}
|
|
619
|
+
return { isPatched: true, hasEntry: rows[0].function_name === patchName };
|
|
620
|
+
}
|
|
621
|
+
// Nondeprecated - skip matching entry, unpatched if nonmatching entry,
|
|
622
|
+
// If there is no entry, we insert one that indicates it is patched.
|
|
623
|
+
if (rows.length !== 0) {
|
|
624
|
+
if (rows[0].function_name === patchName) {
|
|
625
|
+
return { isPatched: true, hasEntry: true };
|
|
626
|
+
}
|
|
627
|
+
return { isPatched: false, hasEntry: false };
|
|
628
|
+
}
|
|
629
|
+
// Insert a patchmarker
|
|
630
|
+
const dn = Date.now();
|
|
631
|
+
await this.pool.query(`INSERT INTO ${this.schemaName}.operation_outputs
|
|
632
|
+
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms)
|
|
633
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
634
|
+
ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn]);
|
|
635
|
+
return { isPatched: true, hasEntry: true };
|
|
636
|
+
}
|
|
637
|
+
// ==================== Workflow Management ====================
|
|
638
|
+
async cancelWorkflow(workflowID) {
|
|
639
|
+
const client = await this.pool.connect();
|
|
640
|
+
try {
|
|
641
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
642
|
+
const statusResult = await this.getWorkflowStatusValue(client, workflowID);
|
|
643
|
+
if (!statusResult) {
|
|
644
|
+
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
645
|
+
}
|
|
646
|
+
if (statusResult === workflow_1.StatusString.SUCCESS ||
|
|
647
|
+
statusResult === workflow_1.StatusString.ERROR ||
|
|
648
|
+
statusResult === workflow_1.StatusString.CANCELLED) {
|
|
649
|
+
await client.query('ROLLBACK');
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
// Set the workflow's status to CANCELLED and remove it from any queue it is on
|
|
653
|
+
await this.updateWorkflowStatus(client, workflowID, workflow_1.StatusString.CANCELLED, {
|
|
654
|
+
update: { queueName: null, resetDeduplicationID: true, resetStartedAtEpochMs: true },
|
|
655
|
+
});
|
|
656
|
+
await client.query('COMMIT');
|
|
657
|
+
}
|
|
658
|
+
catch (error) {
|
|
659
|
+
this.logger.error(error);
|
|
660
|
+
await client.query('ROLLBACK');
|
|
661
|
+
throw error;
|
|
662
|
+
}
|
|
663
|
+
finally {
|
|
664
|
+
client.release();
|
|
665
|
+
}
|
|
666
|
+
this.#setWFCancelMap(workflowID);
|
|
667
|
+
}
|
|
668
|
+
async checkIfCanceled(workflowID) {
|
|
669
|
+
const client = await this.pool.connect();
|
|
670
|
+
try {
|
|
671
|
+
await this.#checkIfCanceled(client, workflowID);
|
|
672
|
+
}
|
|
673
|
+
finally {
|
|
674
|
+
client.release();
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
async resumeWorkflow(workflowID) {
|
|
678
|
+
this.#clearWFCancelMap(workflowID);
|
|
679
|
+
const client = await this.pool.connect();
|
|
680
|
+
try {
|
|
681
|
+
await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ');
|
|
682
|
+
// Check workflow status. If it is complete, do nothing.
|
|
683
|
+
const statusResult = await this.getWorkflowStatusValue(client, workflowID);
|
|
684
|
+
if (!statusResult || statusResult === workflow_1.StatusString.SUCCESS || statusResult === workflow_1.StatusString.ERROR) {
|
|
685
|
+
await client.query('ROLLBACK');
|
|
686
|
+
if (!statusResult) {
|
|
687
|
+
if (statusResult === undefined) {
|
|
688
|
+
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
return;
|
|
692
|
+
}
|
|
693
|
+
// Set the workflow's status to ENQUEUED and reset recovery attempts and deadline.
|
|
694
|
+
await this.updateWorkflowStatus(client, workflowID, workflow_1.StatusString.ENQUEUED, {
|
|
695
|
+
update: {
|
|
696
|
+
queueName: utils_1.INTERNAL_QUEUE_NAME,
|
|
697
|
+
resetRecoveryAttempts: true,
|
|
698
|
+
resetDeadline: true,
|
|
699
|
+
resetDeduplicationID: true,
|
|
700
|
+
resetStartedAtEpochMs: true,
|
|
701
|
+
},
|
|
702
|
+
throwOnFailure: false,
|
|
703
|
+
});
|
|
704
|
+
await client.query('COMMIT');
|
|
705
|
+
}
|
|
706
|
+
catch (error) {
|
|
707
|
+
this.logger.error(error);
|
|
708
|
+
await client.query('ROLLBACK');
|
|
709
|
+
throw error;
|
|
710
|
+
}
|
|
711
|
+
finally {
|
|
712
|
+
client.release();
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
async getWorkflowChildren(workflowID) {
|
|
716
|
+
// BFS to find all descendant workflows
|
|
717
|
+
const visited = new Set([workflowID]);
|
|
718
|
+
const queue = [workflowID];
|
|
719
|
+
const children = [];
|
|
720
|
+
const client = await this.pool.connect();
|
|
721
|
+
try {
|
|
722
|
+
while (queue.length > 0) {
|
|
723
|
+
const batch = queue.splice(0, queue.length);
|
|
724
|
+
const result = await client.query(`SELECT DISTINCT child_workflow_id
|
|
725
|
+
FROM "${this.schemaName}".operation_outputs
|
|
726
|
+
WHERE workflow_uuid = ANY($1)
|
|
727
|
+
AND child_workflow_id IS NOT NULL`, [batch]);
|
|
728
|
+
for (const row of result.rows) {
|
|
729
|
+
if (!visited.has(row.child_workflow_id)) {
|
|
730
|
+
visited.add(row.child_workflow_id);
|
|
731
|
+
queue.push(row.child_workflow_id);
|
|
732
|
+
children.push(row.child_workflow_id);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
finally {
|
|
738
|
+
client.release();
|
|
739
|
+
}
|
|
740
|
+
return children;
|
|
741
|
+
}
|
|
742
|
+
async deleteWorkflow(workflowID, deleteChildren = false) {
|
|
743
|
+
let workflowsToDelete = [workflowID];
|
|
744
|
+
if (deleteChildren) {
|
|
745
|
+
const children = await this.getWorkflowChildren(workflowID);
|
|
746
|
+
workflowsToDelete = [...workflowsToDelete, ...children];
|
|
747
|
+
}
|
|
748
|
+
const client = await this.pool.connect();
|
|
749
|
+
try {
|
|
750
|
+
await client.query(`DELETE FROM "${this.schemaName}".workflow_status
|
|
751
|
+
WHERE workflow_uuid = ANY($1)`, [workflowsToDelete]);
|
|
752
|
+
}
|
|
753
|
+
finally {
|
|
754
|
+
client.release();
|
|
755
|
+
}
|
|
756
|
+
// Clean up in-memory maps
|
|
757
|
+
for (const wfid of workflowsToDelete) {
|
|
758
|
+
this.runningWorkflowMap.delete(wfid);
|
|
759
|
+
this.workflowCancellationMap.delete(wfid);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
721
762
|
async forkWorkflow(workflowID, startStep, options = {}) {
|
|
722
763
|
const newWorkflowID = options.newWorkflowID ?? (0, crypto_1.randomUUID)();
|
|
723
764
|
const workflowStatus = await this.getWorkflowStatus(workflowID);
|
|
@@ -731,7 +772,7 @@ class PostgresSystemDatabase {
|
|
|
731
772
|
try {
|
|
732
773
|
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
733
774
|
const now = Date.now();
|
|
734
|
-
await insertWorkflowStatus(client, {
|
|
775
|
+
await this.insertWorkflowStatus(client, {
|
|
735
776
|
workflowUUID: newWorkflowID,
|
|
736
777
|
status: workflow_1.StatusString.ENQUEUED,
|
|
737
778
|
workflowName: workflowStatus.workflowName,
|
|
@@ -757,7 +798,7 @@ class PostgresSystemDatabase {
|
|
|
757
798
|
queuePartitionKey: undefined,
|
|
758
799
|
forkedFrom: workflowID,
|
|
759
800
|
serialization: workflowStatus.serialization,
|
|
760
|
-
},
|
|
801
|
+
}, null);
|
|
761
802
|
if (startStep > 0) {
|
|
762
803
|
// Copy operation outputs
|
|
763
804
|
const copyOutputsQuery = `INSERT INTO "${this.schemaName}".operation_outputs
|
|
@@ -806,488 +847,6 @@ class PostgresSystemDatabase {
|
|
|
806
847
|
client.release();
|
|
807
848
|
}
|
|
808
849
|
}
|
|
809
|
-
async #runAndRecordResult(client, functionName, workflowID, functionID, func) {
|
|
810
|
-
const startTime = Date.now();
|
|
811
|
-
const result = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
812
|
-
if (result !== undefined) {
|
|
813
|
-
if (result.functionName !== functionName) {
|
|
814
|
-
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, functionName, result.functionName);
|
|
815
|
-
}
|
|
816
|
-
return result.output;
|
|
817
|
-
}
|
|
818
|
-
const output = await func();
|
|
819
|
-
await recordOperationResult(client, workflowID, functionID, functionName, true, this.schemaName, startTime, Date.now(), {
|
|
820
|
-
output,
|
|
821
|
-
});
|
|
822
|
-
return output;
|
|
823
|
-
}
|
|
824
|
-
async durableSleepms(workflowID, functionID, durationMS) {
|
|
825
|
-
let resolveNotification;
|
|
826
|
-
const cancelPromise = new Promise((resolve) => {
|
|
827
|
-
resolveNotification = resolve;
|
|
828
|
-
});
|
|
829
|
-
const cbr = this.cancelWakeupMap.registerCallback(workflowID, resolveNotification);
|
|
830
|
-
try {
|
|
831
|
-
let timeoutPromise = Promise.resolve();
|
|
832
|
-
const { promise, cancel: timeoutCancel } = await this.#durableSleep(workflowID, functionID, durationMS);
|
|
833
|
-
timeoutPromise = promise;
|
|
834
|
-
try {
|
|
835
|
-
await Promise.race([cancelPromise, timeoutPromise]);
|
|
836
|
-
}
|
|
837
|
-
finally {
|
|
838
|
-
timeoutCancel();
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
finally {
|
|
842
|
-
this.cancelWakeupMap.deregisterCallback(cbr);
|
|
843
|
-
}
|
|
844
|
-
await this.checkIfCanceled(workflowID);
|
|
845
|
-
}
|
|
846
|
-
async #durableSleep(workflowID, functionID, durationMS, maxSleepPerIteration) {
|
|
847
|
-
if (maxSleepPerIteration === undefined)
|
|
848
|
-
maxSleepPerIteration = durationMS;
|
|
849
|
-
const curTime = Date.now();
|
|
850
|
-
let endTimeMs = curTime + durationMS;
|
|
851
|
-
const client = await this.pool.connect();
|
|
852
|
-
try {
|
|
853
|
-
const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
854
|
-
if (res) {
|
|
855
|
-
if (res.functionName !== exports.DBOS_FUNCNAME_SLEEP) {
|
|
856
|
-
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, res.functionName);
|
|
857
|
-
}
|
|
858
|
-
endTimeMs = JSON.parse(res.output);
|
|
859
|
-
}
|
|
860
|
-
else {
|
|
861
|
-
await recordOperationResult(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, this.schemaName, Date.now(), Date.now(), {
|
|
862
|
-
output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
|
|
863
|
-
serialization: serialization_1.DBOSPortableJSON.name(),
|
|
864
|
-
});
|
|
865
|
-
}
|
|
866
|
-
return {
|
|
867
|
-
...(0, utils_1.cancellableSleep)(Math.max(Math.min(maxSleepPerIteration, endTimeMs - curTime), 0)),
|
|
868
|
-
endTime: endTimeMs,
|
|
869
|
-
};
|
|
870
|
-
}
|
|
871
|
-
finally {
|
|
872
|
-
client.release();
|
|
873
|
-
}
|
|
874
|
-
}
|
|
875
|
-
nullTopic = '__null__topic__';
|
|
876
|
-
async send(workflowID, functionID, destinationID, message, topic, serialization, messageUUID) {
|
|
877
|
-
topic = topic ?? this.nullTopic;
|
|
878
|
-
messageUUID = messageUUID ?? (0, crypto_1.randomUUID)();
|
|
879
|
-
const client = await this.pool.connect();
|
|
880
|
-
try {
|
|
881
|
-
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
882
|
-
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
|
|
883
|
-
await client.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
|
|
884
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
885
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
886
|
-
return undefined;
|
|
887
|
-
});
|
|
888
|
-
await client.query('COMMIT');
|
|
889
|
-
}
|
|
890
|
-
catch (error) {
|
|
891
|
-
await client.query('ROLLBACK');
|
|
892
|
-
const err = error;
|
|
893
|
-
if (err.code === '23503') {
|
|
894
|
-
// Foreign key constraint violation (only expected for the INSERT query)
|
|
895
|
-
throw new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
|
|
896
|
-
}
|
|
897
|
-
else {
|
|
898
|
-
throw err;
|
|
899
|
-
}
|
|
900
|
-
}
|
|
901
|
-
finally {
|
|
902
|
-
client.release();
|
|
903
|
-
}
|
|
904
|
-
}
|
|
905
|
-
async sendDirect(destinationID, message, topic, serialization, messageUUID) {
|
|
906
|
-
topic = topic ?? this.nullTopic;
|
|
907
|
-
messageUUID = messageUUID ?? (0, crypto_1.randomUUID)();
|
|
908
|
-
try {
|
|
909
|
-
await this.pool.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
|
|
910
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
911
|
-
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
912
|
-
}
|
|
913
|
-
catch (error) {
|
|
914
|
-
const err = error;
|
|
915
|
-
if (err.code === '23503') {
|
|
916
|
-
throw new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
|
|
917
|
-
}
|
|
918
|
-
throw err;
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
async recv(workflowID, functionID, timeoutFunctionID, topic, timeoutSeconds = dbos_executor_1.DBOSExecutor.defaultNotificationTimeoutSec) {
|
|
922
|
-
topic = topic ?? this.nullTopic;
|
|
923
|
-
const startTime = Date.now();
|
|
924
|
-
// First, check for previous executions.
|
|
925
|
-
const res = await this.getOperationResultAndThrowIfCancelled(workflowID, functionID);
|
|
926
|
-
if (res) {
|
|
927
|
-
if (res.functionName !== exports.DBOS_FUNCNAME_RECV) {
|
|
928
|
-
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.functionName);
|
|
929
|
-
}
|
|
930
|
-
return { serializedValue: res.output, serialization: res.serialization ?? null };
|
|
931
|
-
}
|
|
932
|
-
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
933
|
-
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
934
|
-
while (true) {
|
|
935
|
-
// register the key with the global notifications listener.
|
|
936
|
-
let resolveNotification;
|
|
937
|
-
const messagePromise = new Promise((resolve) => {
|
|
938
|
-
resolveNotification = resolve;
|
|
939
|
-
});
|
|
940
|
-
const payload = `${workflowID}::${topic}`;
|
|
941
|
-
const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
|
|
942
|
-
const crh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
|
|
943
|
-
resolveNotification();
|
|
944
|
-
});
|
|
945
|
-
try {
|
|
946
|
-
await this.checkIfCanceled(workflowID);
|
|
947
|
-
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
948
|
-
const initRecvRows = (await this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic])).rows;
|
|
949
|
-
if (initRecvRows.length !== 0)
|
|
950
|
-
break;
|
|
951
|
-
const ct = Date.now();
|
|
952
|
-
if (finishTime && ct > finishTime)
|
|
953
|
-
break; // Time's up
|
|
954
|
-
let timeoutPromise = Promise.resolve();
|
|
955
|
-
let timeoutCancel = () => { };
|
|
956
|
-
if (timeoutms) {
|
|
957
|
-
const { promise, cancel, endTime } = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms, this.dbPollingIntervalEventMs);
|
|
958
|
-
timeoutPromise = promise;
|
|
959
|
-
timeoutCancel = cancel;
|
|
960
|
-
finishTime = endTime;
|
|
961
|
-
}
|
|
962
|
-
else {
|
|
963
|
-
let poll = finishTime ? finishTime - ct : this.dbPollingIntervalEventMs;
|
|
964
|
-
poll = Math.min(this.dbPollingIntervalEventMs, poll);
|
|
965
|
-
const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
|
|
966
|
-
timeoutPromise = promise;
|
|
967
|
-
timeoutCancel = cancel;
|
|
968
|
-
}
|
|
969
|
-
try {
|
|
970
|
-
await Promise.race([messagePromise, timeoutPromise]);
|
|
971
|
-
}
|
|
972
|
-
finally {
|
|
973
|
-
timeoutCancel();
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
finally {
|
|
977
|
-
this.notificationsMap.deregisterCallback(cbr);
|
|
978
|
-
this.cancelWakeupMap.deregisterCallback(crh);
|
|
979
|
-
}
|
|
980
|
-
}
|
|
981
|
-
await this.checkIfCanceled(workflowID);
|
|
982
|
-
// Transactionally consume and return the message if it's in the DB, otherwise return null.
|
|
983
|
-
let message = null;
|
|
984
|
-
let serialization = null;
|
|
985
|
-
const client = await this.pool.connect();
|
|
986
|
-
try {
|
|
987
|
-
await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
|
|
988
|
-
const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
|
|
989
|
-
SET consumed = true
|
|
990
|
-
WHERE destination_uuid = $1
|
|
991
|
-
AND topic = $2
|
|
992
|
-
AND consumed = false
|
|
993
|
-
AND message_uuid = (
|
|
994
|
-
SELECT message_uuid
|
|
995
|
-
FROM "${this.schemaName}".notifications
|
|
996
|
-
WHERE destination_uuid = $1
|
|
997
|
-
AND topic = $2
|
|
998
|
-
AND consumed = false
|
|
999
|
-
ORDER BY created_at_epoch_ms ASC
|
|
1000
|
-
LIMIT 1
|
|
1001
|
-
)
|
|
1002
|
-
RETURNING notifications.message, notifications.serialization;`, [workflowID, topic])).rows;
|
|
1003
|
-
if (finalRecvRows.length > 0) {
|
|
1004
|
-
message = finalRecvRows[0].message;
|
|
1005
|
-
serialization = finalRecvRows[0].serialization;
|
|
1006
|
-
}
|
|
1007
|
-
await recordOperationResult(client, workflowID, functionID, exports.DBOS_FUNCNAME_RECV, true, this.schemaName, startTime, Date.now(), {
|
|
1008
|
-
output: message,
|
|
1009
|
-
serialization,
|
|
1010
|
-
});
|
|
1011
|
-
await client.query(`COMMIT`);
|
|
1012
|
-
}
|
|
1013
|
-
catch (e) {
|
|
1014
|
-
this.logger.error(e);
|
|
1015
|
-
await client.query(`ROLLBACK`);
|
|
1016
|
-
throw e;
|
|
1017
|
-
}
|
|
1018
|
-
finally {
|
|
1019
|
-
client.release();
|
|
1020
|
-
}
|
|
1021
|
-
return { serializedValue: message, serialization };
|
|
1022
|
-
}
|
|
1023
|
-
// Only used in tests
|
|
1024
|
-
async setWorkflowStatus(workflowID, status, resetRecoveryAttempts, internalOptions) {
|
|
1025
|
-
const client = await this.pool.connect();
|
|
1026
|
-
try {
|
|
1027
|
-
await updateWorkflowStatus(client, workflowID, status, this.schemaName, {
|
|
1028
|
-
update: { resetRecoveryAttempts, resetNameTo: internalOptions?.updateName },
|
|
1029
|
-
});
|
|
1030
|
-
}
|
|
1031
|
-
finally {
|
|
1032
|
-
client.release();
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
async setEvent(workflowID, functionID, key, message, serialization) {
|
|
1036
|
-
const client = await this.pool.connect();
|
|
1037
|
-
try {
|
|
1038
|
-
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1039
|
-
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
|
|
1040
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events (workflow_uuid, key, value, serialization)
|
|
1041
|
-
VALUES ($1, $2, $3, $4)
|
|
1042
|
-
ON CONFLICT (workflow_uuid, key)
|
|
1043
|
-
DO UPDATE SET value = $3
|
|
1044
|
-
RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
|
|
1045
|
-
// Also write to the immutable history table for fork support
|
|
1046
|
-
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
|
|
1047
|
-
VALUES ($1, $2, $3, $4, $5)
|
|
1048
|
-
ON CONFLICT (workflow_uuid, function_id, key)
|
|
1049
|
-
DO UPDATE SET value = $4;`, [workflowID, functionID, key, message, serialization]);
|
|
1050
|
-
return undefined;
|
|
1051
|
-
});
|
|
1052
|
-
await client.query('COMMIT');
|
|
1053
|
-
}
|
|
1054
|
-
catch (e) {
|
|
1055
|
-
this.logger.error(e);
|
|
1056
|
-
await client.query(`ROLLBACK`);
|
|
1057
|
-
throw e;
|
|
1058
|
-
}
|
|
1059
|
-
finally {
|
|
1060
|
-
client.release();
|
|
1061
|
-
}
|
|
1062
|
-
}
|
|
1063
|
-
async getEvent(workflowID, key, timeoutSeconds, callerWorkflow) {
|
|
1064
|
-
const startTime = Date.now();
|
|
1065
|
-
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
1066
|
-
if (callerWorkflow) {
|
|
1067
|
-
const res = await this.getOperationResultAndThrowIfCancelled(callerWorkflow.workflowID, callerWorkflow.functionID);
|
|
1068
|
-
if (res) {
|
|
1069
|
-
if (res.functionName !== exports.DBOS_FUNCNAME_GETEVENT) {
|
|
1070
|
-
throw new error_1.DBOSUnexpectedStepError(callerWorkflow.workflowID, callerWorkflow.functionID, exports.DBOS_FUNCNAME_GETEVENT, res.functionName);
|
|
1071
|
-
}
|
|
1072
|
-
return { serializedValue: res.output, serialization: null };
|
|
1073
|
-
}
|
|
1074
|
-
}
|
|
1075
|
-
// Get the return the value. if it's in the DB, otherwise return null.
|
|
1076
|
-
let value = null;
|
|
1077
|
-
let valueSer = null;
|
|
1078
|
-
const payloadKey = `${workflowID}::${key}`;
|
|
1079
|
-
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
1080
|
-
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
1081
|
-
// Register the key with the global notifications listener first... we do not want to look in the DB first
|
|
1082
|
-
// or that would cause a timing hole.
|
|
1083
|
-
while (true) {
|
|
1084
|
-
let resolveNotification;
|
|
1085
|
-
const valuePromise = new Promise((resolve) => {
|
|
1086
|
-
resolveNotification = resolve;
|
|
1087
|
-
});
|
|
1088
|
-
const cbr = this.workflowEventsMap.registerCallback(payloadKey, resolveNotification);
|
|
1089
|
-
const crh = callerWorkflow?.workflowID
|
|
1090
|
-
? this.cancelWakeupMap.registerCallback(callerWorkflow.workflowID, (_res) => {
|
|
1091
|
-
resolveNotification();
|
|
1092
|
-
})
|
|
1093
|
-
: undefined;
|
|
1094
|
-
try {
|
|
1095
|
-
if (callerWorkflow?.workflowID)
|
|
1096
|
-
await this.checkIfCanceled(callerWorkflow?.workflowID);
|
|
1097
|
-
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
1098
|
-
const initRecvRows = (await this.pool.query(`SELECT key, value, serialization
|
|
1099
|
-
FROM "${this.schemaName}".workflow_events
|
|
1100
|
-
WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
|
|
1101
|
-
if (initRecvRows.length > 0) {
|
|
1102
|
-
value = initRecvRows[0].value;
|
|
1103
|
-
valueSer = initRecvRows[0].serialization;
|
|
1104
|
-
break;
|
|
1105
|
-
}
|
|
1106
|
-
const ct = Date.now();
|
|
1107
|
-
if (finishTime && ct > finishTime)
|
|
1108
|
-
break; // Time's up
|
|
1109
|
-
// If we have a callerWorkflow, we want a durable sleep, otherwise, not
|
|
1110
|
-
let timeoutPromise = Promise.resolve();
|
|
1111
|
-
let timeoutCancel = () => { };
|
|
1112
|
-
if (callerWorkflow && timeoutms) {
|
|
1113
|
-
const { promise, cancel, endTime } = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms, this.dbPollingIntervalEventMs);
|
|
1114
|
-
timeoutPromise = promise;
|
|
1115
|
-
timeoutCancel = cancel;
|
|
1116
|
-
finishTime = endTime;
|
|
1117
|
-
}
|
|
1118
|
-
else {
|
|
1119
|
-
let poll = finishTime ? finishTime - ct : this.dbPollingIntervalEventMs;
|
|
1120
|
-
poll = Math.min(this.dbPollingIntervalEventMs, poll);
|
|
1121
|
-
const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
|
|
1122
|
-
timeoutPromise = promise;
|
|
1123
|
-
timeoutCancel = cancel;
|
|
1124
|
-
}
|
|
1125
|
-
try {
|
|
1126
|
-
await Promise.race([valuePromise, timeoutPromise]);
|
|
1127
|
-
}
|
|
1128
|
-
finally {
|
|
1129
|
-
timeoutCancel();
|
|
1130
|
-
}
|
|
1131
|
-
}
|
|
1132
|
-
finally {
|
|
1133
|
-
this.workflowEventsMap.deregisterCallback(cbr);
|
|
1134
|
-
if (crh)
|
|
1135
|
-
this.cancelWakeupMap.deregisterCallback(crh);
|
|
1136
|
-
}
|
|
1137
|
-
}
|
|
1138
|
-
// Record the output if it is inside a workflow.
|
|
1139
|
-
if (callerWorkflow) {
|
|
1140
|
-
await this.recordOperationResult(callerWorkflow.workflowID, callerWorkflow.functionID, exports.DBOS_FUNCNAME_GETEVENT, true, startTime, {
|
|
1141
|
-
output: value,
|
|
1142
|
-
serialization: valueSer,
|
|
1143
|
-
});
|
|
1144
|
-
}
|
|
1145
|
-
return { serializedValue: value, serialization: valueSer };
|
|
1146
|
-
}
|
|
1147
|
-
#setWFCancelMap(workflowID) {
|
|
1148
|
-
if (this.runningWorkflowMap.has(workflowID)) {
|
|
1149
|
-
this.workflowCancellationMap.set(workflowID, true);
|
|
1150
|
-
}
|
|
1151
|
-
this.cancelWakeupMap.callCallbacks(workflowID);
|
|
1152
|
-
}
|
|
1153
|
-
#clearWFCancelMap(workflowID) {
|
|
1154
|
-
if (this.workflowCancellationMap.has(workflowID)) {
|
|
1155
|
-
this.workflowCancellationMap.delete(workflowID);
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
async cancelWorkflow(workflowID) {
|
|
1159
|
-
const client = await this.pool.connect();
|
|
1160
|
-
try {
|
|
1161
|
-
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1162
|
-
const statusResult = await getWorkflowStatusValue(client, workflowID, this.schemaName);
|
|
1163
|
-
if (!statusResult) {
|
|
1164
|
-
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
1165
|
-
}
|
|
1166
|
-
if (statusResult === workflow_1.StatusString.SUCCESS ||
|
|
1167
|
-
statusResult === workflow_1.StatusString.ERROR ||
|
|
1168
|
-
statusResult === workflow_1.StatusString.CANCELLED) {
|
|
1169
|
-
await client.query('ROLLBACK');
|
|
1170
|
-
return;
|
|
1171
|
-
}
|
|
1172
|
-
// Set the workflow's status to CANCELLED and remove it from any queue it is on
|
|
1173
|
-
await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.CANCELLED, this.schemaName, {
|
|
1174
|
-
update: { queueName: null, resetDeduplicationID: true, resetStartedAtEpochMs: true },
|
|
1175
|
-
});
|
|
1176
|
-
await client.query('COMMIT');
|
|
1177
|
-
}
|
|
1178
|
-
catch (error) {
|
|
1179
|
-
this.logger.error(error);
|
|
1180
|
-
await client.query('ROLLBACK');
|
|
1181
|
-
throw error;
|
|
1182
|
-
}
|
|
1183
|
-
finally {
|
|
1184
|
-
client.release();
|
|
1185
|
-
}
|
|
1186
|
-
this.#setWFCancelMap(workflowID);
|
|
1187
|
-
}
|
|
1188
|
-
async #checkIfCanceled(client, workflowID) {
|
|
1189
|
-
if (this.workflowCancellationMap.get(workflowID) === true) {
|
|
1190
|
-
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
1191
|
-
}
|
|
1192
|
-
const statusValue = await getWorkflowStatusValue(client, workflowID, this.schemaName);
|
|
1193
|
-
if (statusValue === workflow_1.StatusString.CANCELLED) {
|
|
1194
|
-
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
1195
|
-
}
|
|
1196
|
-
}
|
|
1197
|
-
async checkIfCanceled(workflowID) {
|
|
1198
|
-
const client = await this.pool.connect();
|
|
1199
|
-
try {
|
|
1200
|
-
await this.#checkIfCanceled(client, workflowID);
|
|
1201
|
-
}
|
|
1202
|
-
finally {
|
|
1203
|
-
client.release();
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
async resumeWorkflow(workflowID) {
|
|
1207
|
-
this.#clearWFCancelMap(workflowID);
|
|
1208
|
-
const client = await this.pool.connect();
|
|
1209
|
-
try {
|
|
1210
|
-
await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ');
|
|
1211
|
-
// Check workflow status. If it is complete, do nothing.
|
|
1212
|
-
const statusResult = await getWorkflowStatusValue(client, workflowID, this.schemaName);
|
|
1213
|
-
if (!statusResult || statusResult === workflow_1.StatusString.SUCCESS || statusResult === workflow_1.StatusString.ERROR) {
|
|
1214
|
-
await client.query('ROLLBACK');
|
|
1215
|
-
if (!statusResult) {
|
|
1216
|
-
if (statusResult === undefined) {
|
|
1217
|
-
throw new error_1.DBOSNonExistentWorkflowError(`Workflow ${workflowID} does not exist`);
|
|
1218
|
-
}
|
|
1219
|
-
}
|
|
1220
|
-
return;
|
|
1221
|
-
}
|
|
1222
|
-
// Set the workflow's status to ENQUEUED and reset recovery attempts and deadline.
|
|
1223
|
-
await updateWorkflowStatus(client, workflowID, workflow_1.StatusString.ENQUEUED, this.schemaName, {
|
|
1224
|
-
update: {
|
|
1225
|
-
queueName: utils_1.INTERNAL_QUEUE_NAME,
|
|
1226
|
-
resetRecoveryAttempts: true,
|
|
1227
|
-
resetDeadline: true,
|
|
1228
|
-
resetDeduplicationID: true,
|
|
1229
|
-
resetStartedAtEpochMs: true,
|
|
1230
|
-
},
|
|
1231
|
-
throwOnFailure: false,
|
|
1232
|
-
});
|
|
1233
|
-
await client.query('COMMIT');
|
|
1234
|
-
}
|
|
1235
|
-
catch (error) {
|
|
1236
|
-
this.logger.error(error);
|
|
1237
|
-
await client.query('ROLLBACK');
|
|
1238
|
-
throw error;
|
|
1239
|
-
}
|
|
1240
|
-
finally {
|
|
1241
|
-
client.release();
|
|
1242
|
-
}
|
|
1243
|
-
}
|
|
1244
|
-
async getWorkflowChildren(workflowID) {
|
|
1245
|
-
// BFS to find all descendant workflows
|
|
1246
|
-
const visited = new Set([workflowID]);
|
|
1247
|
-
const queue = [workflowID];
|
|
1248
|
-
const children = [];
|
|
1249
|
-
const client = await this.pool.connect();
|
|
1250
|
-
try {
|
|
1251
|
-
while (queue.length > 0) {
|
|
1252
|
-
const batch = queue.splice(0, queue.length);
|
|
1253
|
-
const result = await client.query(`SELECT DISTINCT child_workflow_id
|
|
1254
|
-
FROM "${this.schemaName}".operation_outputs
|
|
1255
|
-
WHERE workflow_uuid = ANY($1)
|
|
1256
|
-
AND child_workflow_id IS NOT NULL`, [batch]);
|
|
1257
|
-
for (const row of result.rows) {
|
|
1258
|
-
if (!visited.has(row.child_workflow_id)) {
|
|
1259
|
-
visited.add(row.child_workflow_id);
|
|
1260
|
-
queue.push(row.child_workflow_id);
|
|
1261
|
-
children.push(row.child_workflow_id);
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
}
|
|
1265
|
-
}
|
|
1266
|
-
finally {
|
|
1267
|
-
client.release();
|
|
1268
|
-
}
|
|
1269
|
-
return children;
|
|
1270
|
-
}
|
|
1271
|
-
async deleteWorkflow(workflowID, deleteChildren = false) {
|
|
1272
|
-
let workflowsToDelete = [workflowID];
|
|
1273
|
-
if (deleteChildren) {
|
|
1274
|
-
const children = await this.getWorkflowChildren(workflowID);
|
|
1275
|
-
workflowsToDelete = [...workflowsToDelete, ...children];
|
|
1276
|
-
}
|
|
1277
|
-
const client = await this.pool.connect();
|
|
1278
|
-
try {
|
|
1279
|
-
await client.query(`DELETE FROM "${this.schemaName}".workflow_status
|
|
1280
|
-
WHERE workflow_uuid = ANY($1)`, [workflowsToDelete]);
|
|
1281
|
-
}
|
|
1282
|
-
finally {
|
|
1283
|
-
client.release();
|
|
1284
|
-
}
|
|
1285
|
-
// Clean up in-memory maps
|
|
1286
|
-
for (const wfid of workflowsToDelete) {
|
|
1287
|
-
this.runningWorkflowMap.delete(wfid);
|
|
1288
|
-
this.workflowCancellationMap.delete(wfid);
|
|
1289
|
-
}
|
|
1290
|
-
}
|
|
1291
850
|
async exportWorkflow(workflowID, exportChildren = false) {
|
|
1292
851
|
const workflowIDs = [workflowID];
|
|
1293
852
|
if (exportChildren) {
|
|
@@ -1437,6 +996,7 @@ class PostgresSystemDatabase {
|
|
|
1437
996
|
client.release();
|
|
1438
997
|
}
|
|
1439
998
|
}
|
|
999
|
+
// ==================== Awaiting Workflows ====================
|
|
1440
1000
|
registerRunningWorkflow(workflowID, workflowPromise) {
|
|
1441
1001
|
// Need to await for the workflow and capture errors.
|
|
1442
1002
|
const awaitWorkflowPromise = workflowPromise
|
|
@@ -1467,222 +1027,407 @@ class PostgresSystemDatabase {
|
|
|
1467
1027
|
//throw new Error('Message notification map is not empty - shutdown is not clean.');
|
|
1468
1028
|
}
|
|
1469
1029
|
}
|
|
1470
|
-
async
|
|
1471
|
-
const
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1030
|
+
async awaitWorkflowResult(workflowID, timeoutSeconds, callerID, timerFuncID) {
|
|
1031
|
+
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
1032
|
+
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
1033
|
+
while (true) {
|
|
1034
|
+
let resolveNotification;
|
|
1035
|
+
const statusPromise = new Promise((resolve) => {
|
|
1036
|
+
resolveNotification = resolve;
|
|
1037
|
+
});
|
|
1038
|
+
const irh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
|
|
1039
|
+
resolveNotification();
|
|
1040
|
+
});
|
|
1041
|
+
const crh = callerID
|
|
1042
|
+
? this.cancelWakeupMap.registerCallback(callerID, (_res) => {
|
|
1043
|
+
resolveNotification();
|
|
1044
|
+
})
|
|
1045
|
+
: undefined;
|
|
1478
1046
|
try {
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1047
|
+
if (callerID)
|
|
1048
|
+
await this.checkIfCanceled(callerID);
|
|
1049
|
+
try {
|
|
1050
|
+
const { rows } = await this.pool.query(`SELECT status, output, error, serialization FROM "${this.schemaName}".workflow_status
|
|
1051
|
+
WHERE workflow_uuid=$1`, [workflowID]);
|
|
1052
|
+
if (rows.length > 0) {
|
|
1053
|
+
const status = rows[0].status;
|
|
1054
|
+
if (status === workflow_1.StatusString.SUCCESS) {
|
|
1055
|
+
return { output: rows[0].output, serialization: rows[0].serialization };
|
|
1056
|
+
}
|
|
1057
|
+
else if (status === workflow_1.StatusString.ERROR) {
|
|
1058
|
+
return { error: rows[0].error, serialization: rows[0].serialization };
|
|
1059
|
+
}
|
|
1060
|
+
else if (status === workflow_1.StatusString.CANCELLED) {
|
|
1061
|
+
return { cancelled: true };
|
|
1062
|
+
}
|
|
1063
|
+
else if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
|
|
1064
|
+
return { maxRecoveryAttemptsExceeded: true };
|
|
1065
|
+
}
|
|
1066
|
+
else {
|
|
1067
|
+
// Status is not actionable
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
catch (e) {
|
|
1072
|
+
const err = e;
|
|
1073
|
+
this.logger.error(`Exception from system database: ${err}`, err);
|
|
1074
|
+
throw err;
|
|
1075
|
+
}
|
|
1076
|
+
const ct = Date.now();
|
|
1077
|
+
if (finishTime && ct > finishTime)
|
|
1078
|
+
return undefined; // Time's up
|
|
1079
|
+
let timeoutPromise = Promise.resolve();
|
|
1080
|
+
let timeoutCancel = () => { };
|
|
1081
|
+
if (timerFuncID !== undefined && callerID !== undefined && timeoutms !== undefined) {
|
|
1082
|
+
const { promise, cancel, endTime } = await this.#durableSleep(callerID, timerFuncID, timeoutms, this.dbPollingIntervalResultMs);
|
|
1083
|
+
finishTime = endTime;
|
|
1084
|
+
timeoutPromise = promise;
|
|
1085
|
+
timeoutCancel = cancel;
|
|
1086
|
+
}
|
|
1087
|
+
else {
|
|
1088
|
+
let poll = finishTime ? finishTime - ct : this.dbPollingIntervalResultMs;
|
|
1089
|
+
poll = Math.min(this.dbPollingIntervalResultMs, poll);
|
|
1090
|
+
const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
|
|
1091
|
+
timeoutPromise = promise;
|
|
1092
|
+
timeoutCancel = cancel;
|
|
1093
|
+
}
|
|
1094
|
+
try {
|
|
1095
|
+
await Promise.race([statusPromise, timeoutPromise]);
|
|
1096
|
+
}
|
|
1097
|
+
finally {
|
|
1098
|
+
timeoutCancel();
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
finally {
|
|
1102
|
+
this.cancelWakeupMap.deregisterCallback(irh);
|
|
1103
|
+
if (crh)
|
|
1104
|
+
this.cancelWakeupMap.deregisterCallback(crh);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
async awaitFirstWorkflowId(workflowIds, callerID) {
|
|
1109
|
+
const placeholders = workflowIds.map((_, i) => `$${i + 1}`).join(', ');
|
|
1110
|
+
while (true) {
|
|
1111
|
+
let resolveNotification;
|
|
1112
|
+
const wakeupPromise = new Promise((resolve) => {
|
|
1113
|
+
resolveNotification = resolve;
|
|
1114
|
+
});
|
|
1115
|
+
// Register cancel callbacks for all target workflows and the caller.
|
|
1116
|
+
const cbHandles = workflowIds.map((wfid) => this.cancelWakeupMap.registerCallback(wfid, () => resolveNotification()));
|
|
1117
|
+
const callerCbHandle = callerID
|
|
1118
|
+
? this.cancelWakeupMap.registerCallback(callerID, () => resolveNotification())
|
|
1119
|
+
: undefined;
|
|
1120
|
+
try {
|
|
1121
|
+
if (callerID)
|
|
1122
|
+
await this.checkIfCanceled(callerID);
|
|
1123
|
+
const { rows } = await this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
|
|
1124
|
+
WHERE workflow_uuid IN (${placeholders})
|
|
1125
|
+
AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}')
|
|
1126
|
+
LIMIT 1`, workflowIds);
|
|
1127
|
+
if (rows.length > 0) {
|
|
1128
|
+
return rows[0].workflow_uuid;
|
|
1129
|
+
}
|
|
1130
|
+
const { promise: sleepPromise, cancel: sleepCancel } = (0, utils_1.cancellableSleep)(this.dbPollingIntervalResultMs);
|
|
1131
|
+
try {
|
|
1132
|
+
await Promise.race([wakeupPromise, sleepPromise]);
|
|
1133
|
+
}
|
|
1134
|
+
finally {
|
|
1135
|
+
sleepCancel();
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
finally {
|
|
1139
|
+
for (const h of cbHandles) {
|
|
1140
|
+
this.cancelWakeupMap.deregisterCallback(h);
|
|
1141
|
+
}
|
|
1142
|
+
if (callerCbHandle)
|
|
1143
|
+
this.cancelWakeupMap.deregisterCallback(callerCbHandle);
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
// ==================== Sleep ====================
|
|
1148
|
+
async durableSleepms(workflowID, functionID, durationMS) {
|
|
1149
|
+
let resolveNotification;
|
|
1150
|
+
const cancelPromise = new Promise((resolve) => {
|
|
1151
|
+
resolveNotification = resolve;
|
|
1152
|
+
});
|
|
1153
|
+
const cbr = this.cancelWakeupMap.registerCallback(workflowID, resolveNotification);
|
|
1154
|
+
try {
|
|
1155
|
+
let timeoutPromise = Promise.resolve();
|
|
1156
|
+
const { promise, cancel: timeoutCancel } = await this.#durableSleep(workflowID, functionID, durationMS);
|
|
1157
|
+
timeoutPromise = promise;
|
|
1158
|
+
try {
|
|
1159
|
+
await Promise.race([cancelPromise, timeoutPromise]);
|
|
1482
1160
|
}
|
|
1483
1161
|
finally {
|
|
1484
|
-
|
|
1162
|
+
timeoutCancel();
|
|
1485
1163
|
}
|
|
1486
1164
|
}
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
return parseStatus(json);
|
|
1165
|
+
finally {
|
|
1166
|
+
this.cancelWakeupMap.deregisterCallback(cbr);
|
|
1490
1167
|
}
|
|
1491
|
-
|
|
1492
|
-
|
|
1168
|
+
await this.checkIfCanceled(workflowID);
|
|
1169
|
+
}
|
|
1170
|
+
// ==================== Messaging ====================
|
|
1171
|
+
nullTopic = '__null__topic__';
|
|
1172
|
+
async send(workflowID, functionID, destinationID, message, topic, serialization, messageUUID) {
|
|
1173
|
+
topic = topic ?? this.nullTopic;
|
|
1174
|
+
messageUUID = messageUUID ?? (0, crypto_1.randomUUID)();
|
|
1175
|
+
const client = await this.pool.connect();
|
|
1176
|
+
try {
|
|
1177
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1178
|
+
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SEND, workflowID, functionID, async () => {
|
|
1179
|
+
await client.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
|
|
1180
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
1181
|
+
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
1182
|
+
return undefined;
|
|
1183
|
+
});
|
|
1184
|
+
await client.query('COMMIT');
|
|
1185
|
+
}
|
|
1186
|
+
catch (error) {
|
|
1187
|
+
await client.query('ROLLBACK');
|
|
1188
|
+
const err = error;
|
|
1189
|
+
if (err.code === '23503') {
|
|
1190
|
+
// Foreign key constraint violation (only expected for the INSERT query)
|
|
1191
|
+
throw new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
|
|
1192
|
+
}
|
|
1193
|
+
else {
|
|
1194
|
+
throw err;
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
finally {
|
|
1198
|
+
client.release();
|
|
1493
1199
|
}
|
|
1494
1200
|
}
|
|
1495
|
-
async
|
|
1201
|
+
async sendDirect(destinationID, message, topic, serialization, messageUUID) {
|
|
1202
|
+
topic = topic ?? this.nullTopic;
|
|
1203
|
+
messageUUID = messageUUID ?? (0, crypto_1.randomUUID)();
|
|
1204
|
+
try {
|
|
1205
|
+
await this.pool.query(`INSERT INTO "${this.schemaName}".notifications (destination_uuid, topic, message, serialization, message_uuid)
|
|
1206
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
1207
|
+
ON CONFLICT (message_uuid) DO NOTHING;`, [destinationID, topic, message, serialization, messageUUID]);
|
|
1208
|
+
}
|
|
1209
|
+
catch (error) {
|
|
1210
|
+
const err = error;
|
|
1211
|
+
if (err.code === '23503') {
|
|
1212
|
+
throw new error_1.DBOSNonExistentWorkflowError(`Sent to non-existent destination workflow UUID: ${destinationID}`);
|
|
1213
|
+
}
|
|
1214
|
+
throw err;
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
async recv(workflowID, functionID, timeoutFunctionID, topic, timeoutSeconds = dbos_executor_1.DBOSExecutor.defaultNotificationTimeoutSec) {
|
|
1218
|
+
topic = topic ?? this.nullTopic;
|
|
1219
|
+
const startTime = Date.now();
|
|
1220
|
+
// First, check for previous executions.
|
|
1221
|
+
const res = await this.getOperationResultAndThrowIfCancelled(workflowID, functionID);
|
|
1222
|
+
if (res) {
|
|
1223
|
+
if (res.functionName !== exports.DBOS_FUNCNAME_RECV) {
|
|
1224
|
+
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_RECV, res.functionName);
|
|
1225
|
+
}
|
|
1226
|
+
return { serializedValue: res.output, serialization: res.serialization ?? null };
|
|
1227
|
+
}
|
|
1496
1228
|
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
1497
1229
|
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
1498
1230
|
while (true) {
|
|
1231
|
+
// register the key with the global notifications listener.
|
|
1499
1232
|
let resolveNotification;
|
|
1500
|
-
const
|
|
1233
|
+
const messagePromise = new Promise((resolve) => {
|
|
1501
1234
|
resolveNotification = resolve;
|
|
1502
1235
|
});
|
|
1503
|
-
const
|
|
1236
|
+
const payload = `${workflowID}::${topic}`;
|
|
1237
|
+
const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
|
|
1238
|
+
const crh = this.cancelWakeupMap.registerCallback(workflowID, (_res) => {
|
|
1504
1239
|
resolveNotification();
|
|
1505
1240
|
});
|
|
1506
|
-
const crh = callerID
|
|
1507
|
-
? this.cancelWakeupMap.registerCallback(callerID, (_res) => {
|
|
1508
|
-
resolveNotification();
|
|
1509
|
-
})
|
|
1510
|
-
: undefined;
|
|
1511
1241
|
try {
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
if (rows.length > 0) {
|
|
1518
|
-
const status = rows[0].status;
|
|
1519
|
-
if (status === workflow_1.StatusString.SUCCESS) {
|
|
1520
|
-
return { output: rows[0].output, serialization: rows[0].serialization };
|
|
1521
|
-
}
|
|
1522
|
-
else if (status === workflow_1.StatusString.ERROR) {
|
|
1523
|
-
return { error: rows[0].error, serialization: rows[0].serialization };
|
|
1524
|
-
}
|
|
1525
|
-
else if (status === workflow_1.StatusString.CANCELLED) {
|
|
1526
|
-
return { cancelled: true };
|
|
1527
|
-
}
|
|
1528
|
-
else if (status === workflow_1.StatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED) {
|
|
1529
|
-
return { maxRecoveryAttemptsExceeded: true };
|
|
1530
|
-
}
|
|
1531
|
-
else {
|
|
1532
|
-
// Status is not actionable
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
}
|
|
1536
|
-
catch (e) {
|
|
1537
|
-
const err = e;
|
|
1538
|
-
this.logger.error(`Exception from system database: ${err}`, err);
|
|
1539
|
-
throw err;
|
|
1540
|
-
}
|
|
1242
|
+
await this.checkIfCanceled(workflowID);
|
|
1243
|
+
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
1244
|
+
const initRecvRows = (await this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic])).rows;
|
|
1245
|
+
if (initRecvRows.length !== 0)
|
|
1246
|
+
break;
|
|
1541
1247
|
const ct = Date.now();
|
|
1542
1248
|
if (finishTime && ct > finishTime)
|
|
1543
|
-
|
|
1249
|
+
break; // Time's up
|
|
1544
1250
|
let timeoutPromise = Promise.resolve();
|
|
1545
1251
|
let timeoutCancel = () => { };
|
|
1546
|
-
if (
|
|
1547
|
-
const { promise, cancel, endTime } = await this.#durableSleep(
|
|
1548
|
-
finishTime = endTime;
|
|
1252
|
+
if (timeoutms) {
|
|
1253
|
+
const { promise, cancel, endTime } = await this.#durableSleep(workflowID, timeoutFunctionID, timeoutms, this.dbPollingIntervalEventMs);
|
|
1549
1254
|
timeoutPromise = promise;
|
|
1550
1255
|
timeoutCancel = cancel;
|
|
1256
|
+
finishTime = endTime;
|
|
1551
1257
|
}
|
|
1552
1258
|
else {
|
|
1553
|
-
let poll = finishTime ? finishTime - ct : this.
|
|
1554
|
-
poll = Math.min(this.
|
|
1259
|
+
let poll = finishTime ? finishTime - ct : this.dbPollingIntervalEventMs;
|
|
1260
|
+
poll = Math.min(this.dbPollingIntervalEventMs, poll);
|
|
1555
1261
|
const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
|
|
1556
1262
|
timeoutPromise = promise;
|
|
1557
1263
|
timeoutCancel = cancel;
|
|
1558
1264
|
}
|
|
1559
1265
|
try {
|
|
1560
|
-
await Promise.race([
|
|
1266
|
+
await Promise.race([messagePromise, timeoutPromise]);
|
|
1561
1267
|
}
|
|
1562
1268
|
finally {
|
|
1563
1269
|
timeoutCancel();
|
|
1564
1270
|
}
|
|
1565
1271
|
}
|
|
1566
1272
|
finally {
|
|
1567
|
-
this.
|
|
1568
|
-
|
|
1569
|
-
|
|
1273
|
+
this.notificationsMap.deregisterCallback(cbr);
|
|
1274
|
+
this.cancelWakeupMap.deregisterCallback(crh);
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
await this.checkIfCanceled(workflowID);
|
|
1278
|
+
// Transactionally consume and return the message if it's in the DB, otherwise return null.
|
|
1279
|
+
let message = null;
|
|
1280
|
+
let serialization = null;
|
|
1281
|
+
const client = await this.pool.connect();
|
|
1282
|
+
try {
|
|
1283
|
+
await client.query(`BEGIN ISOLATION LEVEL READ COMMITTED`);
|
|
1284
|
+
const finalRecvRows = (await client.query(`UPDATE "${this.schemaName}".notifications
|
|
1285
|
+
SET consumed = true
|
|
1286
|
+
WHERE destination_uuid = $1
|
|
1287
|
+
AND topic = $2
|
|
1288
|
+
AND consumed = false
|
|
1289
|
+
AND message_uuid = (
|
|
1290
|
+
SELECT message_uuid
|
|
1291
|
+
FROM "${this.schemaName}".notifications
|
|
1292
|
+
WHERE destination_uuid = $1
|
|
1293
|
+
AND topic = $2
|
|
1294
|
+
AND consumed = false
|
|
1295
|
+
ORDER BY created_at_epoch_ms ASC
|
|
1296
|
+
LIMIT 1
|
|
1297
|
+
)
|
|
1298
|
+
RETURNING notifications.message, notifications.serialization;`, [workflowID, topic])).rows;
|
|
1299
|
+
if (finalRecvRows.length > 0) {
|
|
1300
|
+
message = finalRecvRows[0].message;
|
|
1301
|
+
serialization = finalRecvRows[0].serialization;
|
|
1570
1302
|
}
|
|
1303
|
+
await this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_RECV, true, startTime, Date.now(), {
|
|
1304
|
+
output: message,
|
|
1305
|
+
serialization,
|
|
1306
|
+
});
|
|
1307
|
+
await client.query(`COMMIT`);
|
|
1308
|
+
}
|
|
1309
|
+
catch (e) {
|
|
1310
|
+
this.logger.error(e);
|
|
1311
|
+
await client.query(`ROLLBACK`);
|
|
1312
|
+
throw e;
|
|
1313
|
+
}
|
|
1314
|
+
finally {
|
|
1315
|
+
client.release();
|
|
1571
1316
|
}
|
|
1317
|
+
return { serializedValue: message, serialization };
|
|
1572
1318
|
}
|
|
1573
|
-
|
|
1574
|
-
|
|
1319
|
+
// ==================== Events ====================
|
|
1320
|
+
async setEvent(workflowID, functionID, key, message, serialization) {
|
|
1321
|
+
const client = await this.pool.connect();
|
|
1322
|
+
try {
|
|
1323
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1324
|
+
await this.#runAndRecordResult(client, exports.DBOS_FUNCNAME_SETEVENT, workflowID, functionID, async () => {
|
|
1325
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events (workflow_uuid, key, value, serialization)
|
|
1326
|
+
VALUES ($1, $2, $3, $4)
|
|
1327
|
+
ON CONFLICT (workflow_uuid, key)
|
|
1328
|
+
DO UPDATE SET value = $3
|
|
1329
|
+
RETURNING workflow_uuid;`, [workflowID, key, message, serialization]);
|
|
1330
|
+
// Also write to the immutable history table for fork support
|
|
1331
|
+
await client.query(`INSERT INTO "${this.schemaName}".workflow_events_history (workflow_uuid, function_id, key, value, serialization)
|
|
1332
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
1333
|
+
ON CONFLICT (workflow_uuid, function_id, key)
|
|
1334
|
+
DO UPDATE SET value = $4;`, [workflowID, functionID, key, message, serialization]);
|
|
1335
|
+
return undefined;
|
|
1336
|
+
});
|
|
1337
|
+
await client.query('COMMIT');
|
|
1338
|
+
}
|
|
1339
|
+
catch (e) {
|
|
1340
|
+
this.logger.error(e);
|
|
1341
|
+
await client.query(`ROLLBACK`);
|
|
1342
|
+
throw e;
|
|
1343
|
+
}
|
|
1344
|
+
finally {
|
|
1345
|
+
client.release();
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
async getEvent(workflowID, key, timeoutSeconds, callerWorkflow) {
|
|
1349
|
+
const startTime = Date.now();
|
|
1350
|
+
// Check if the operation has been done before for OAOO (only do this inside a workflow).
|
|
1351
|
+
if (callerWorkflow) {
|
|
1352
|
+
const res = await this.getOperationResultAndThrowIfCancelled(callerWorkflow.workflowID, callerWorkflow.functionID);
|
|
1353
|
+
if (res) {
|
|
1354
|
+
if (res.functionName !== exports.DBOS_FUNCNAME_GETEVENT) {
|
|
1355
|
+
throw new error_1.DBOSUnexpectedStepError(callerWorkflow.workflowID, callerWorkflow.functionID, exports.DBOS_FUNCNAME_GETEVENT, res.functionName);
|
|
1356
|
+
}
|
|
1357
|
+
return { serializedValue: res.output, serialization: null };
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
// Get the return the value. if it's in the DB, otherwise return null.
|
|
1361
|
+
let value = null;
|
|
1362
|
+
let valueSer = null;
|
|
1363
|
+
const payloadKey = `${workflowID}::${key}`;
|
|
1364
|
+
const timeoutms = timeoutSeconds !== undefined ? timeoutSeconds * 1000 : undefined;
|
|
1365
|
+
let finishTime = timeoutms !== undefined ? Date.now() + timeoutms : undefined;
|
|
1366
|
+
// Register the key with the global notifications listener first... we do not want to look in the DB first
|
|
1367
|
+
// or that would cause a timing hole.
|
|
1575
1368
|
while (true) {
|
|
1576
1369
|
let resolveNotification;
|
|
1577
|
-
const
|
|
1370
|
+
const valuePromise = new Promise((resolve) => {
|
|
1578
1371
|
resolveNotification = resolve;
|
|
1579
1372
|
});
|
|
1580
|
-
|
|
1581
|
-
const
|
|
1582
|
-
|
|
1583
|
-
|
|
1373
|
+
const cbr = this.workflowEventsMap.registerCallback(payloadKey, resolveNotification);
|
|
1374
|
+
const crh = callerWorkflow?.workflowID
|
|
1375
|
+
? this.cancelWakeupMap.registerCallback(callerWorkflow.workflowID, (_res) => {
|
|
1376
|
+
resolveNotification();
|
|
1377
|
+
})
|
|
1584
1378
|
: undefined;
|
|
1585
1379
|
try {
|
|
1586
|
-
if (
|
|
1587
|
-
await this.checkIfCanceled(
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
if (
|
|
1593
|
-
|
|
1380
|
+
if (callerWorkflow?.workflowID)
|
|
1381
|
+
await this.checkIfCanceled(callerWorkflow?.workflowID);
|
|
1382
|
+
// Check if the key is already in the DB, then wait for the notification if it isn't.
|
|
1383
|
+
const initRecvRows = (await this.pool.query(`SELECT key, value, serialization
|
|
1384
|
+
FROM "${this.schemaName}".workflow_events
|
|
1385
|
+
WHERE workflow_uuid=$1 AND key=$2;`, [workflowID, key])).rows;
|
|
1386
|
+
if (initRecvRows.length > 0) {
|
|
1387
|
+
value = initRecvRows[0].value;
|
|
1388
|
+
valueSer = initRecvRows[0].serialization;
|
|
1389
|
+
break;
|
|
1390
|
+
}
|
|
1391
|
+
const ct = Date.now();
|
|
1392
|
+
if (finishTime && ct > finishTime)
|
|
1393
|
+
break; // Time's up
|
|
1394
|
+
// If we have a callerWorkflow, we want a durable sleep, otherwise, not
|
|
1395
|
+
let timeoutPromise = Promise.resolve();
|
|
1396
|
+
let timeoutCancel = () => { };
|
|
1397
|
+
if (callerWorkflow && timeoutms) {
|
|
1398
|
+
const { promise, cancel, endTime } = await this.#durableSleep(callerWorkflow.workflowID, callerWorkflow.timeoutFunctionID ?? -1, timeoutms, this.dbPollingIntervalEventMs);
|
|
1399
|
+
timeoutPromise = promise;
|
|
1400
|
+
timeoutCancel = cancel;
|
|
1401
|
+
finishTime = endTime;
|
|
1402
|
+
}
|
|
1403
|
+
else {
|
|
1404
|
+
let poll = finishTime ? finishTime - ct : this.dbPollingIntervalEventMs;
|
|
1405
|
+
poll = Math.min(this.dbPollingIntervalEventMs, poll);
|
|
1406
|
+
const { promise, cancel } = (0, utils_1.cancellableSleep)(poll);
|
|
1407
|
+
timeoutPromise = promise;
|
|
1408
|
+
timeoutCancel = cancel;
|
|
1594
1409
|
}
|
|
1595
|
-
const { promise: sleepPromise, cancel: sleepCancel } = (0, utils_1.cancellableSleep)(this.dbPollingIntervalResultMs);
|
|
1596
1410
|
try {
|
|
1597
|
-
await Promise.race([
|
|
1411
|
+
await Promise.race([valuePromise, timeoutPromise]);
|
|
1598
1412
|
}
|
|
1599
1413
|
finally {
|
|
1600
|
-
|
|
1414
|
+
timeoutCancel();
|
|
1601
1415
|
}
|
|
1602
1416
|
}
|
|
1603
1417
|
finally {
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
if (callerCbHandle)
|
|
1608
|
-
this.cancelWakeupMap.deregisterCallback(callerCbHandle);
|
|
1418
|
+
this.workflowEventsMap.deregisterCallback(cbr);
|
|
1419
|
+
if (crh)
|
|
1420
|
+
this.cancelWakeupMap.deregisterCallback(crh);
|
|
1609
1421
|
}
|
|
1610
1422
|
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
const connect = async () => {
|
|
1620
|
-
const reconnect = () => {
|
|
1621
|
-
if (this.reconnectTimeout) {
|
|
1622
|
-
return;
|
|
1623
|
-
}
|
|
1624
|
-
this.reconnectTimeout = setTimeout(async () => {
|
|
1625
|
-
this.reconnectTimeout = null;
|
|
1626
|
-
await connect();
|
|
1627
|
-
}, 1000);
|
|
1628
|
-
};
|
|
1629
|
-
let client = null;
|
|
1630
|
-
try {
|
|
1631
|
-
client = await this.pool.connect();
|
|
1632
|
-
await client.query('LISTEN dbos_notifications_channel;');
|
|
1633
|
-
await client.query('LISTEN dbos_workflow_events_channel;');
|
|
1634
|
-
// Self-test: verify LISTEN actually works by sending a NOTIFY and checking it arrives.
|
|
1635
|
-
// If a transaction-mode pooler (e.g. PgBouncer pool_mode=transaction) is in the path,
|
|
1636
|
-
// LISTEN succeeds but the subscription is silently lost when the backend is released.
|
|
1637
|
-
let selfTestReceived = false;
|
|
1638
|
-
const onSelfTest = (msg) => {
|
|
1639
|
-
if (msg.channel === 'dbos_notifications_channel' && msg.payload === 'dbos_listen_selftest') {
|
|
1640
|
-
selfTestReceived = true;
|
|
1641
|
-
}
|
|
1642
|
-
};
|
|
1643
|
-
client.on('notification', onSelfTest);
|
|
1644
|
-
await this.pool.query("NOTIFY dbos_notifications_channel, 'dbos_listen_selftest'");
|
|
1645
|
-
for (let i = 0; i < 30 && !selfTestReceived; i++) {
|
|
1646
|
-
await new Promise((r) => setTimeout(r, 100));
|
|
1647
|
-
}
|
|
1648
|
-
client.removeListener('notification', onSelfTest);
|
|
1649
|
-
if (!selfTestReceived) {
|
|
1650
|
-
this.logger.warn('LISTEN/NOTIFY self-test failed: notification was not received within 3 seconds. ' +
|
|
1651
|
-
'This typically means the connection is going through a transaction-mode pooler ' +
|
|
1652
|
-
'(e.g. PgBouncer with pool_mode=transaction), which silently breaks LISTEN/NOTIFY. ' +
|
|
1653
|
-
'Workflow notifications will fall back to polling, which may increase latency.');
|
|
1654
|
-
}
|
|
1655
|
-
const handler = (msg) => {
|
|
1656
|
-
if (!this.shouldUseDBNotifications)
|
|
1657
|
-
return;
|
|
1658
|
-
if (msg.channel === 'dbos_notifications_channel' && msg.payload) {
|
|
1659
|
-
this.notificationsMap.callCallbacks(msg.payload);
|
|
1660
|
-
}
|
|
1661
|
-
else if (msg.channel === 'dbos_workflow_events_channel' && msg.payload) {
|
|
1662
|
-
this.workflowEventsMap.callCallbacks(msg.payload);
|
|
1663
|
-
}
|
|
1664
|
-
};
|
|
1665
|
-
client.on('notification', handler);
|
|
1666
|
-
client.on('error', (err) => {
|
|
1667
|
-
this.logger.warn(`Error in notifications client: ${err}`);
|
|
1668
|
-
if (client) {
|
|
1669
|
-
client.removeAllListeners();
|
|
1670
|
-
client.release(true);
|
|
1671
|
-
}
|
|
1672
|
-
reconnect();
|
|
1673
|
-
});
|
|
1674
|
-
this.notificationsClient = client;
|
|
1675
|
-
}
|
|
1676
|
-
catch (error) {
|
|
1677
|
-
this.logger.warn(`Error in notifications listener: ${String(error)}`);
|
|
1678
|
-
if (client) {
|
|
1679
|
-
client.removeAllListeners();
|
|
1680
|
-
client.release(true);
|
|
1681
|
-
}
|
|
1682
|
-
reconnect();
|
|
1683
|
-
}
|
|
1684
|
-
};
|
|
1685
|
-
await connect();
|
|
1423
|
+
// Record the output if it is inside a workflow.
|
|
1424
|
+
if (callerWorkflow) {
|
|
1425
|
+
await this.recordOperationResult(callerWorkflow.workflowID, callerWorkflow.functionID, exports.DBOS_FUNCNAME_GETEVENT, true, startTime, {
|
|
1426
|
+
output: value,
|
|
1427
|
+
serialization: valueSer,
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
return { serializedValue: value, serialization: valueSer };
|
|
1686
1431
|
}
|
|
1687
1432
|
// Event dispatcher queries / updates
|
|
1688
1433
|
async getEventDispatchState(service, workflowName, key) {
|
|
@@ -1725,125 +1470,78 @@ class PostgresSystemDatabase {
|
|
|
1725
1470
|
: undefined,
|
|
1726
1471
|
};
|
|
1727
1472
|
}
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
const
|
|
1731
|
-
|
|
1732
|
-
'
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
'
|
|
1743
|
-
'updated_at',
|
|
1744
|
-
'application_version',
|
|
1745
|
-
'application_id',
|
|
1746
|
-
'workflow_deadline_epoch_ms',
|
|
1747
|
-
'workflow_timeout_ms',
|
|
1748
|
-
'deduplication_id',
|
|
1749
|
-
'priority',
|
|
1750
|
-
'queue_partition_key',
|
|
1751
|
-
'started_at_epoch_ms',
|
|
1752
|
-
'forked_from',
|
|
1753
|
-
'parent_workflow_id',
|
|
1754
|
-
];
|
|
1755
|
-
input.loadInput = input.loadInput ?? true;
|
|
1756
|
-
input.loadOutput = input.loadOutput ?? true;
|
|
1757
|
-
if (input.loadInput) {
|
|
1758
|
-
selectColumns.push('inputs', 'request');
|
|
1473
|
+
// ==================== Streams ====================
|
|
1474
|
+
async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
|
|
1475
|
+
const client = await this.pool.connect();
|
|
1476
|
+
try {
|
|
1477
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1478
|
+
// Find the maximum offset for this workflow_uuid and key combination
|
|
1479
|
+
const maxOffsetResult = await client.query(`SELECT MAX("offset") FROM "${this.schemaName}".streams
|
|
1480
|
+
WHERE workflow_uuid = $1 AND key = $2`, [workflowID, key]);
|
|
1481
|
+
// Next offset is max + 1, or 0 if no records exist
|
|
1482
|
+
const maxOffset = maxOffsetResult.rows[0].max;
|
|
1483
|
+
const nextOffset = maxOffset !== null ? maxOffset + 1 : 0;
|
|
1484
|
+
// Insert the new stream entry
|
|
1485
|
+
await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
|
|
1486
|
+
VALUES ($1, $2, $3, $4, $5, $6)`, [workflowID, key, serializedValue, nextOffset, functionID, serialization]);
|
|
1487
|
+
await client.query('COMMIT');
|
|
1759
1488
|
}
|
|
1760
|
-
|
|
1761
|
-
|
|
1489
|
+
catch (e) {
|
|
1490
|
+
this.logger.error(e);
|
|
1491
|
+
await client.query('ROLLBACK');
|
|
1492
|
+
throw e;
|
|
1762
1493
|
}
|
|
1763
|
-
|
|
1764
|
-
|
|
1494
|
+
finally {
|
|
1495
|
+
client.release();
|
|
1765
1496
|
}
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
const
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
const
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
params.push(value);
|
|
1785
|
-
paramCounter++;
|
|
1786
|
-
}
|
|
1787
|
-
};
|
|
1788
|
-
// If queuesOnly, filter for queued workflows
|
|
1789
|
-
if (input.queuesOnly) {
|
|
1790
|
-
whereClauses.push(`queue_name IS NOT NULL`);
|
|
1791
|
-
whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1})`);
|
|
1792
|
-
params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING);
|
|
1793
|
-
paramCounter += 2;
|
|
1497
|
+
}
|
|
1498
|
+
async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
|
|
1499
|
+
const client = await this.pool.connect();
|
|
1500
|
+
try {
|
|
1501
|
+
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
1502
|
+
await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
|
|
1503
|
+
// Find the maximum offset for this workflow_uuid and key combination
|
|
1504
|
+
const maxOffsetResult = await client.query(`SELECT MAX("offset") FROM "${this.schemaName}".streams
|
|
1505
|
+
WHERE workflow_uuid = $1 AND key = $2`, [workflowID, key]);
|
|
1506
|
+
// Next offset is max + 1, or 0 if no records exist
|
|
1507
|
+
const maxOffset = maxOffsetResult.rows[0].max;
|
|
1508
|
+
const nextOffset = maxOffset !== null ? maxOffset + 1 : 0;
|
|
1509
|
+
// Insert the new stream entry
|
|
1510
|
+
await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
|
|
1511
|
+
VALUES ($1, $2, $3, $4, $5, $6)`, [workflowID, key, serializedValue, nextOffset, functionID, serialization]);
|
|
1512
|
+
return undefined;
|
|
1513
|
+
});
|
|
1514
|
+
await client.query('COMMIT');
|
|
1794
1515
|
}
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
const likeClauses = input.workflow_id_prefix.map((_, i) => `workflow_uuid LIKE $${paramCounter + i}`);
|
|
1800
|
-
whereClauses.push(`(${likeClauses.join(' OR ')})`);
|
|
1801
|
-
params.push(...input.workflow_id_prefix.map((p) => `${p}%`));
|
|
1802
|
-
paramCounter += input.workflow_id_prefix.length;
|
|
1803
|
-
}
|
|
1804
|
-
else {
|
|
1805
|
-
whereClauses.push(`workflow_uuid LIKE $${paramCounter}`);
|
|
1806
|
-
params.push(`${input.workflow_id_prefix}%`);
|
|
1807
|
-
paramCounter++;
|
|
1808
|
-
}
|
|
1516
|
+
catch (e) {
|
|
1517
|
+
this.logger.error(e);
|
|
1518
|
+
await client.query('ROLLBACK');
|
|
1519
|
+
throw e;
|
|
1809
1520
|
}
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
whereClauses.push(`workflow_uuid IN (${placeholders})`);
|
|
1813
|
-
params.push(...input.workflowIDs);
|
|
1814
|
-
paramCounter += input.workflowIDs.length;
|
|
1521
|
+
finally {
|
|
1522
|
+
client.release();
|
|
1815
1523
|
}
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1524
|
+
}
|
|
1525
|
+
async closeStream(workflowID, functionID, key) {
|
|
1526
|
+
await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL, 'portable_json', exports.DBOS_FUNCNAME_CLOSESTREAM);
|
|
1527
|
+
}
|
|
1528
|
+
async readStream(workflowID, key, offset) {
|
|
1529
|
+
const client = await this.pool.connect();
|
|
1530
|
+
try {
|
|
1531
|
+
const result = await client.query(`SELECT value, serialization FROM "${this.schemaName}".streams
|
|
1532
|
+
WHERE workflow_uuid = $1 AND key = $2 AND "offset" = $3`, [workflowID, key, offset]);
|
|
1533
|
+
if (result.rows.length === 0) {
|
|
1534
|
+
throw new Error(`No value found for workflow_uuid=${workflowID}, key=${key}, offset=${offset}`);
|
|
1535
|
+
}
|
|
1536
|
+
// Deserialize the value before returning
|
|
1537
|
+
const row = result.rows[0];
|
|
1538
|
+
return { serializedValue: row.value, serialization: row.serialization };
|
|
1823
1539
|
}
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
params.push(new Date(input.endTime).getTime());
|
|
1827
|
-
paramCounter++;
|
|
1540
|
+
finally {
|
|
1541
|
+
client.release();
|
|
1828
1542
|
}
|
|
1829
|
-
addFilter('status', input.status);
|
|
1830
|
-
addFilter('application_version', input.applicationVersion);
|
|
1831
|
-
addFilter('executor_id', input.executorId);
|
|
1832
|
-
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
|
|
1833
|
-
const orderClause = `ORDER BY created_at ${input.sortDesc ? 'DESC' : 'ASC'}`;
|
|
1834
|
-
const limitClause = input.limit ? `LIMIT ${input.limit}` : '';
|
|
1835
|
-
const offsetClause = input.offset ? `OFFSET ${input.offset}` : '';
|
|
1836
|
-
const query = `
|
|
1837
|
-
SELECT ${selectColumns.join(', ')}
|
|
1838
|
-
FROM "${schemaName}".workflow_status
|
|
1839
|
-
${whereClause}
|
|
1840
|
-
${orderClause}
|
|
1841
|
-
${limitClause}
|
|
1842
|
-
${offsetClause}
|
|
1843
|
-
`;
|
|
1844
|
-
const result = await this.pool.query(query, params);
|
|
1845
|
-
return result.rows.map(mapWorkflowStatus);
|
|
1846
1543
|
}
|
|
1544
|
+
// ==================== Queues ====================
|
|
1847
1545
|
async clearQueueAssignment(workflowID) {
|
|
1848
1546
|
// Reset the status of the task from "PENDING" to "ENQUEUED"
|
|
1849
1547
|
const wqRes = await this.pool.query(`UPDATE "${this.schemaName}".workflow_status
|
|
@@ -1982,75 +1680,125 @@ class PostgresSystemDatabase {
|
|
|
1982
1680
|
// Return the IDs of all functions we marked started
|
|
1983
1681
|
return claimedIDs;
|
|
1984
1682
|
}
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1683
|
+
// ==================== Queries & Maintenance ====================
|
|
1684
|
+
async listWorkflows(input) {
|
|
1685
|
+
const schemaName = this.schemaName;
|
|
1686
|
+
const selectColumns = [
|
|
1687
|
+
'workflow_uuid',
|
|
1688
|
+
'status',
|
|
1689
|
+
'name',
|
|
1690
|
+
'recovery_attempts',
|
|
1691
|
+
'config_name',
|
|
1692
|
+
'class_name',
|
|
1693
|
+
'authenticated_user',
|
|
1694
|
+
'authenticated_roles',
|
|
1695
|
+
'assumed_role',
|
|
1696
|
+
'queue_name',
|
|
1697
|
+
'executor_id',
|
|
1698
|
+
'created_at',
|
|
1699
|
+
'updated_at',
|
|
1700
|
+
'application_version',
|
|
1701
|
+
'application_id',
|
|
1702
|
+
'workflow_deadline_epoch_ms',
|
|
1703
|
+
'workflow_timeout_ms',
|
|
1704
|
+
'deduplication_id',
|
|
1705
|
+
'priority',
|
|
1706
|
+
'queue_partition_key',
|
|
1707
|
+
'started_at_epoch_ms',
|
|
1708
|
+
'forked_from',
|
|
1709
|
+
'parent_workflow_id',
|
|
1710
|
+
];
|
|
1711
|
+
input.loadInput = input.loadInput ?? true;
|
|
1712
|
+
input.loadOutput = input.loadOutput ?? true;
|
|
1713
|
+
if (input.loadInput) {
|
|
1714
|
+
selectColumns.push('inputs', 'request');
|
|
1999
1715
|
}
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
await client.query('ROLLBACK');
|
|
2003
|
-
throw e;
|
|
1716
|
+
if (input.loadOutput) {
|
|
1717
|
+
selectColumns.push('output', 'error');
|
|
2004
1718
|
}
|
|
2005
|
-
|
|
2006
|
-
|
|
1719
|
+
if (input.loadInput || input.loadOutput) {
|
|
1720
|
+
selectColumns.push('serialization');
|
|
2007
1721
|
}
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
const
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
const
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
1722
|
+
input.sortDesc = input.sortDesc ?? false; // By default, sort in ascending order
|
|
1723
|
+
// Build WHERE clauses
|
|
1724
|
+
const whereClauses = [];
|
|
1725
|
+
const params = [];
|
|
1726
|
+
let paramCounter = 1;
|
|
1727
|
+
// Helper: add a filter for a field that may be a single value or an array.
|
|
1728
|
+
// Uses = for a single value, IN (...) for an array.
|
|
1729
|
+
const addFilter = (column, value) => {
|
|
1730
|
+
if (!value)
|
|
1731
|
+
return;
|
|
1732
|
+
if (Array.isArray(value)) {
|
|
1733
|
+
const placeholders = value.map((_, i) => `$${paramCounter + i}`).join(', ');
|
|
1734
|
+
whereClauses.push(`${column} IN (${placeholders})`);
|
|
1735
|
+
params.push(...value);
|
|
1736
|
+
paramCounter += value.length;
|
|
1737
|
+
}
|
|
1738
|
+
else {
|
|
1739
|
+
whereClauses.push(`${column} = $${paramCounter}`);
|
|
1740
|
+
params.push(value);
|
|
1741
|
+
paramCounter++;
|
|
1742
|
+
}
|
|
1743
|
+
};
|
|
1744
|
+
// If queuesOnly, filter for queued workflows
|
|
1745
|
+
if (input.queuesOnly) {
|
|
1746
|
+
whereClauses.push(`queue_name IS NOT NULL`);
|
|
1747
|
+
whereClauses.push(`status IN ($${paramCounter}, $${paramCounter + 1})`);
|
|
1748
|
+
params.push(workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.PENDING);
|
|
1749
|
+
paramCounter += 2;
|
|
2026
1750
|
}
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
1751
|
+
addFilter('name', input.workflowName);
|
|
1752
|
+
addFilter('queue_name', input.queueName);
|
|
1753
|
+
if (input.workflow_id_prefix) {
|
|
1754
|
+
if (Array.isArray(input.workflow_id_prefix)) {
|
|
1755
|
+
const likeClauses = input.workflow_id_prefix.map((_, i) => `workflow_uuid LIKE $${paramCounter + i}`);
|
|
1756
|
+
whereClauses.push(`(${likeClauses.join(' OR ')})`);
|
|
1757
|
+
params.push(...input.workflow_id_prefix.map((p) => `${p}%`));
|
|
1758
|
+
paramCounter += input.workflow_id_prefix.length;
|
|
1759
|
+
}
|
|
1760
|
+
else {
|
|
1761
|
+
whereClauses.push(`workflow_uuid LIKE $${paramCounter}`);
|
|
1762
|
+
params.push(`${input.workflow_id_prefix}%`);
|
|
1763
|
+
paramCounter++;
|
|
1764
|
+
}
|
|
2031
1765
|
}
|
|
2032
|
-
|
|
2033
|
-
|
|
1766
|
+
if (input.workflowIDs) {
|
|
1767
|
+
const placeholders = input.workflowIDs.map((_, i) => `$${paramCounter + i}`).join(', ');
|
|
1768
|
+
whereClauses.push(`workflow_uuid IN (${placeholders})`);
|
|
1769
|
+
params.push(...input.workflowIDs);
|
|
1770
|
+
paramCounter += input.workflowIDs.length;
|
|
2034
1771
|
}
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
const result = await client.query(`SELECT value, serialization FROM "${this.schemaName}".streams
|
|
2043
|
-
WHERE workflow_uuid = $1 AND key = $2 AND "offset" = $3`, [workflowID, key, offset]);
|
|
2044
|
-
if (result.rows.length === 0) {
|
|
2045
|
-
throw new Error(`No value found for workflow_uuid=${workflowID}, key=${key}, offset=${offset}`);
|
|
2046
|
-
}
|
|
2047
|
-
// Deserialize the value before returning
|
|
2048
|
-
const row = result.rows[0];
|
|
2049
|
-
return { serializedValue: row.value, serialization: row.serialization };
|
|
1772
|
+
addFilter('authenticated_user', input.authenticatedUser);
|
|
1773
|
+
addFilter('forked_from', input.forkedFrom);
|
|
1774
|
+
addFilter('parent_workflow_id', input.parentWorkflowID);
|
|
1775
|
+
if (input.startTime) {
|
|
1776
|
+
whereClauses.push(`created_at >= $${paramCounter}`);
|
|
1777
|
+
params.push(new Date(input.startTime).getTime());
|
|
1778
|
+
paramCounter++;
|
|
2050
1779
|
}
|
|
2051
|
-
|
|
2052
|
-
|
|
1780
|
+
if (input.endTime) {
|
|
1781
|
+
whereClauses.push(`created_at <= $${paramCounter}`);
|
|
1782
|
+
params.push(new Date(input.endTime).getTime());
|
|
1783
|
+
paramCounter++;
|
|
2053
1784
|
}
|
|
1785
|
+
addFilter('status', input.status);
|
|
1786
|
+
addFilter('application_version', input.applicationVersion);
|
|
1787
|
+
addFilter('executor_id', input.executorId);
|
|
1788
|
+
const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(' AND ')}` : '';
|
|
1789
|
+
const orderClause = `ORDER BY created_at ${input.sortDesc ? 'DESC' : 'ASC'}`;
|
|
1790
|
+
const limitClause = input.limit ? `LIMIT ${input.limit}` : '';
|
|
1791
|
+
const offsetClause = input.offset ? `OFFSET ${input.offset}` : '';
|
|
1792
|
+
const query = `
|
|
1793
|
+
SELECT ${selectColumns.join(', ')}
|
|
1794
|
+
FROM "${schemaName}".workflow_status
|
|
1795
|
+
${whereClause}
|
|
1796
|
+
${orderClause}
|
|
1797
|
+
${limitClause}
|
|
1798
|
+
${offsetClause}
|
|
1799
|
+
`;
|
|
1800
|
+
const result = await this.pool.query(query, params);
|
|
1801
|
+
return result.rows.map(mapWorkflowStatus);
|
|
2054
1802
|
}
|
|
2055
1803
|
async garbageCollect(cutoffEpochTimestampMs, rowsThreshold) {
|
|
2056
1804
|
if (rowsThreshold !== undefined) {
|
|
@@ -2092,77 +1840,21 @@ class PostgresSystemDatabase {
|
|
|
2092
1840
|
value: Number(row.count),
|
|
2093
1841
|
});
|
|
2094
1842
|
}
|
|
2095
|
-
// Query step metrics
|
|
2096
|
-
const stepResult = await this.pool.query(`SELECT function_name, COUNT(*) as count
|
|
2097
|
-
FROM "${this.schemaName}".operation_outputs
|
|
2098
|
-
WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2
|
|
2099
|
-
GROUP BY function_name`, [startEpochMs, endEpochMs]);
|
|
2100
|
-
for (const row of stepResult.rows) {
|
|
2101
|
-
metrics.push({
|
|
2102
|
-
metricType: 'step_count',
|
|
2103
|
-
metricName: row.function_name,
|
|
2104
|
-
value: Number(row.count),
|
|
2105
|
-
});
|
|
2106
|
-
}
|
|
2107
|
-
return metrics;
|
|
2108
|
-
}
|
|
2109
|
-
async checkPatch(workflowID, functionID, patchName, deprecated) {
|
|
2110
|
-
// Not doing a cancel check at this point.
|
|
2111
|
-
if (functionID === undefined)
|
|
2112
|
-
throw new TypeError('functionID must be defined');
|
|
2113
|
-
patchName = `DBOS.patch-${patchName}`;
|
|
2114
|
-
const { rows } = await this.pool.query(`SELECT function_name
|
|
2115
|
-
FROM "${this.schemaName}".operation_outputs
|
|
2116
|
-
WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
|
|
2117
|
-
if (deprecated) {
|
|
2118
|
-
// Deprecated does not write anything. We skip any existing matching patch marker if it matches
|
|
2119
|
-
if (rows.length === 0) {
|
|
2120
|
-
return { isPatched: true, hasEntry: false };
|
|
2121
|
-
}
|
|
2122
|
-
return { isPatched: true, hasEntry: rows[0].function_name === patchName };
|
|
2123
|
-
}
|
|
2124
|
-
// Nondeprecated - skip matching entry, unpatched if nonmatching entry,
|
|
2125
|
-
// If there is no entry, we insert one that indicates it is patched.
|
|
2126
|
-
if (rows.length !== 0) {
|
|
2127
|
-
if (rows[0].function_name === patchName) {
|
|
2128
|
-
return { isPatched: true, hasEntry: true };
|
|
2129
|
-
}
|
|
2130
|
-
return { isPatched: false, hasEntry: false };
|
|
2131
|
-
}
|
|
2132
|
-
// Insert a patchmarker
|
|
2133
|
-
const dn = Date.now();
|
|
2134
|
-
await this.pool.query(`INSERT INTO ${this.schemaName}.operation_outputs
|
|
2135
|
-
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms)
|
|
2136
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
2137
|
-
ON CONFLICT DO NOTHING;`, [workflowID, functionID, null, null, patchName, null, dn, dn]);
|
|
2138
|
-
return { isPatched: true, hasEntry: true };
|
|
2139
|
-
}
|
|
2140
|
-
async runTransactionalStep(workflowID, functionID, functionName, callback) {
|
|
2141
|
-
const client = await this.pool.connect();
|
|
2142
|
-
try {
|
|
2143
|
-
await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
|
|
2144
|
-
const existing = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
2145
|
-
if (existing !== undefined) {
|
|
2146
|
-
await client.query('ROLLBACK');
|
|
2147
|
-
return existing;
|
|
2148
|
-
}
|
|
2149
|
-
const startTime = Date.now();
|
|
2150
|
-
const output = await callback(client);
|
|
2151
|
-
await recordOperationResult(client, workflowID, functionID, functionName, true, this.schemaName, startTime, Date.now(), {
|
|
2152
|
-
output,
|
|
2153
|
-
});
|
|
2154
|
-
await client.query('COMMIT');
|
|
2155
|
-
return undefined;
|
|
2156
|
-
}
|
|
2157
|
-
catch (e) {
|
|
2158
|
-
await client.query('ROLLBACK');
|
|
2159
|
-
throw e;
|
|
2160
|
-
}
|
|
2161
|
-
finally {
|
|
2162
|
-
client.release();
|
|
1843
|
+
// Query step metrics
|
|
1844
|
+
const stepResult = await this.pool.query(`SELECT function_name, COUNT(*) as count
|
|
1845
|
+
FROM "${this.schemaName}".operation_outputs
|
|
1846
|
+
WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2
|
|
1847
|
+
GROUP BY function_name`, [startEpochMs, endEpochMs]);
|
|
1848
|
+
for (const row of stepResult.rows) {
|
|
1849
|
+
metrics.push({
|
|
1850
|
+
metricType: 'step_count',
|
|
1851
|
+
metricName: row.function_name,
|
|
1852
|
+
value: Number(row.count),
|
|
1853
|
+
});
|
|
2163
1854
|
}
|
|
1855
|
+
return metrics;
|
|
2164
1856
|
}
|
|
2165
|
-
//
|
|
1857
|
+
// ==================== Scheduling ====================
|
|
2166
1858
|
async createSchedule(schedule, client) {
|
|
2167
1859
|
const q = client ?? this.pool;
|
|
2168
1860
|
try {
|
|
@@ -2277,150 +1969,522 @@ class PostgresSystemDatabase {
|
|
|
2277
1969
|
client.release();
|
|
2278
1970
|
}
|
|
2279
1971
|
}
|
|
1972
|
+
// ==================== Application Versions ====================
|
|
1973
|
+
async createApplicationVersion(versionName) {
|
|
1974
|
+
const versionId = (0, crypto_1.randomUUID)();
|
|
1975
|
+
await this.pool.query(`INSERT INTO "${this.schemaName}".application_versions (version_id, version_name)
|
|
1976
|
+
VALUES ($1, $2)
|
|
1977
|
+
ON CONFLICT (version_name) DO NOTHING`, [versionId, versionName]);
|
|
1978
|
+
}
|
|
1979
|
+
async updateApplicationVersionTimestamp(versionName, newTimestamp) {
|
|
1980
|
+
await this.pool.query(`UPDATE "${this.schemaName}".application_versions
|
|
1981
|
+
SET version_timestamp = $1
|
|
1982
|
+
WHERE version_name = $2`, [newTimestamp, versionName]);
|
|
1983
|
+
}
|
|
1984
|
+
async listApplicationVersions() {
|
|
1985
|
+
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at
|
|
1986
|
+
FROM "${this.schemaName}".application_versions
|
|
1987
|
+
ORDER BY version_timestamp DESC`);
|
|
1988
|
+
return rows.map((r) => ({
|
|
1989
|
+
versionId: r.version_id,
|
|
1990
|
+
versionName: r.version_name,
|
|
1991
|
+
versionTimestamp: Number(r.version_timestamp),
|
|
1992
|
+
createdAt: Number(r.created_at),
|
|
1993
|
+
}));
|
|
1994
|
+
}
|
|
1995
|
+
async getLatestApplicationVersion() {
|
|
1996
|
+
const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at
|
|
1997
|
+
FROM "${this.schemaName}".application_versions
|
|
1998
|
+
ORDER BY version_timestamp DESC
|
|
1999
|
+
LIMIT 1`);
|
|
2000
|
+
if (rows.length === 0) {
|
|
2001
|
+
throw new error_1.DBOSInitializationError('No application versions found');
|
|
2002
|
+
}
|
|
2003
|
+
const r = rows[0];
|
|
2004
|
+
return {
|
|
2005
|
+
versionId: r.version_id,
|
|
2006
|
+
versionName: r.version_name,
|
|
2007
|
+
versionTimestamp: Number(r.version_timestamp),
|
|
2008
|
+
createdAt: Number(r.created_at),
|
|
2009
|
+
};
|
|
2010
|
+
}
|
|
2011
|
+
// ==================== Internal ====================
|
|
2012
|
+
async insertWorkflowStatus(client, initStatus, ownerXid, incrementAttempts = false) {
|
|
2013
|
+
try {
|
|
2014
|
+
const { rows } = await client.query(`INSERT INTO "${this.schemaName}".workflow_status (
|
|
2015
|
+
workflow_uuid,
|
|
2016
|
+
status,
|
|
2017
|
+
name,
|
|
2018
|
+
class_name,
|
|
2019
|
+
config_name,
|
|
2020
|
+
queue_name,
|
|
2021
|
+
authenticated_user,
|
|
2022
|
+
assumed_role,
|
|
2023
|
+
authenticated_roles,
|
|
2024
|
+
request,
|
|
2025
|
+
executor_id,
|
|
2026
|
+
application_version,
|
|
2027
|
+
application_id,
|
|
2028
|
+
created_at,
|
|
2029
|
+
recovery_attempts,
|
|
2030
|
+
updated_at,
|
|
2031
|
+
workflow_timeout_ms,
|
|
2032
|
+
workflow_deadline_epoch_ms,
|
|
2033
|
+
inputs,
|
|
2034
|
+
deduplication_id,
|
|
2035
|
+
priority,
|
|
2036
|
+
queue_partition_key,
|
|
2037
|
+
forked_from,
|
|
2038
|
+
parent_workflow_id,
|
|
2039
|
+
serialization,
|
|
2040
|
+
owner_xid
|
|
2041
|
+
) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $26, $27)
|
|
2042
|
+
ON CONFLICT (workflow_uuid)
|
|
2043
|
+
DO UPDATE SET
|
|
2044
|
+
recovery_attempts = CASE
|
|
2045
|
+
WHEN workflow_status.status != '${workflow_1.StatusString.ENQUEUED}'
|
|
2046
|
+
THEN workflow_status.recovery_attempts + $25
|
|
2047
|
+
ELSE workflow_status.recovery_attempts
|
|
2048
|
+
END,
|
|
2049
|
+
updated_at = EXCLUDED.updated_at,
|
|
2050
|
+
executor_id = CASE
|
|
2051
|
+
WHEN EXCLUDED.status != '${workflow_1.StatusString.ENQUEUED}'
|
|
2052
|
+
THEN EXCLUDED.executor_id
|
|
2053
|
+
ELSE workflow_status.executor_id
|
|
2054
|
+
END
|
|
2055
|
+
RETURNING recovery_attempts, status, name, class_name, config_name, queue_name, workflow_deadline_epoch_ms, executor_id, owner_xid, serialization`, [
|
|
2056
|
+
initStatus.workflowUUID,
|
|
2057
|
+
initStatus.status,
|
|
2058
|
+
initStatus.workflowName,
|
|
2059
|
+
// For cross-language compatibility, these variables MUST be NULL in the database when not set
|
|
2060
|
+
initStatus.workflowClassName === '' ? null : initStatus.workflowClassName,
|
|
2061
|
+
initStatus.workflowConfigName === '' ? null : initStatus.workflowConfigName,
|
|
2062
|
+
initStatus.queueName ?? null,
|
|
2063
|
+
initStatus.authenticatedUser,
|
|
2064
|
+
initStatus.assumedRole,
|
|
2065
|
+
JSON.stringify(initStatus.authenticatedRoles),
|
|
2066
|
+
JSON.stringify(initStatus.request),
|
|
2067
|
+
initStatus.executorId,
|
|
2068
|
+
initStatus.applicationVersion ?? null,
|
|
2069
|
+
initStatus.applicationID,
|
|
2070
|
+
initStatus.createdAt,
|
|
2071
|
+
initStatus.status === workflow_1.StatusString.ENQUEUED ? 0 : 1,
|
|
2072
|
+
initStatus.updatedAt ?? Date.now(),
|
|
2073
|
+
initStatus.timeoutMS ?? null,
|
|
2074
|
+
initStatus.deadlineEpochMS ?? null,
|
|
2075
|
+
initStatus.input ?? null,
|
|
2076
|
+
initStatus.deduplicationID ?? null,
|
|
2077
|
+
initStatus.priority,
|
|
2078
|
+
initStatus.queuePartitionKey ?? null,
|
|
2079
|
+
initStatus.forkedFrom ?? null,
|
|
2080
|
+
initStatus.parentWorkflowID ?? null,
|
|
2081
|
+
(incrementAttempts ?? false) ? 1 : 0,
|
|
2082
|
+
initStatus.serialization,
|
|
2083
|
+
ownerXid,
|
|
2084
|
+
]);
|
|
2085
|
+
if (rows.length === 0) {
|
|
2086
|
+
throw new Error(`Attempt to insert workflow ${initStatus.workflowUUID} failed`);
|
|
2087
|
+
}
|
|
2088
|
+
const ret = rows[0];
|
|
2089
|
+
ret.class_name = ret.class_name ?? '';
|
|
2090
|
+
ret.config_name = ret.config_name ?? '';
|
|
2091
|
+
initStatus.serialization = ret.serialization;
|
|
2092
|
+
return ret;
|
|
2093
|
+
}
|
|
2094
|
+
catch (error) {
|
|
2095
|
+
const err = error;
|
|
2096
|
+
if (err.code === '23505') {
|
|
2097
|
+
throw new error_1.DBOSQueueDuplicatedError(initStatus.workflowUUID, initStatus.queueName ?? '', initStatus.deduplicationID ?? '');
|
|
2098
|
+
}
|
|
2099
|
+
throw error;
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
async getWorkflowStatusValue(client, workflowID) {
|
|
2103
|
+
const { rows } = await client.query(`SELECT status FROM "${this.schemaName}".workflow_status WHERE workflow_uuid=$1`, [workflowID]);
|
|
2104
|
+
return rows.length === 0 ? undefined : rows[0].status;
|
|
2105
|
+
}
|
|
2106
|
+
async updateWorkflowStatus(client, workflowID, status, options = {}) {
|
|
2107
|
+
let setClause = `SET status=$2, updated_at=$3`;
|
|
2108
|
+
let whereClause = `WHERE workflow_uuid=$1`;
|
|
2109
|
+
const args = [workflowID, status, Date.now()];
|
|
2110
|
+
const update = options.update ?? {};
|
|
2111
|
+
if (update.output) {
|
|
2112
|
+
const param = args.push(update.output);
|
|
2113
|
+
setClause += `, output=$${param}`;
|
|
2114
|
+
}
|
|
2115
|
+
if (update.error) {
|
|
2116
|
+
const param = args.push(update.error);
|
|
2117
|
+
setClause += `, error=$${param}`;
|
|
2118
|
+
}
|
|
2119
|
+
if (update.resetRecoveryAttempts) {
|
|
2120
|
+
setClause += `, recovery_attempts = 0`;
|
|
2121
|
+
}
|
|
2122
|
+
if (update.resetDeadline) {
|
|
2123
|
+
setClause += `, workflow_deadline_epoch_ms = NULL`;
|
|
2124
|
+
}
|
|
2125
|
+
if (update.queueName !== undefined) {
|
|
2126
|
+
const param = args.push(update.queueName ?? undefined);
|
|
2127
|
+
setClause += `, queue_name=$${param}`;
|
|
2128
|
+
}
|
|
2129
|
+
if (update.resetDeduplicationID) {
|
|
2130
|
+
setClause += `, deduplication_id = NULL`;
|
|
2131
|
+
}
|
|
2132
|
+
if (update.resetStartedAtEpochMs) {
|
|
2133
|
+
setClause += `, started_at_epoch_ms = NULL`;
|
|
2134
|
+
}
|
|
2135
|
+
if (update.executorId !== undefined) {
|
|
2136
|
+
const param = args.push(update.executorId ?? undefined);
|
|
2137
|
+
setClause += `, executor_id=$${param}`;
|
|
2138
|
+
}
|
|
2139
|
+
if (update.resetNameTo !== undefined) {
|
|
2140
|
+
const param = args.push(update.resetNameTo ?? undefined);
|
|
2141
|
+
setClause += `, name=$${param}`;
|
|
2142
|
+
}
|
|
2143
|
+
const where = options.where ?? {};
|
|
2144
|
+
if (where.status) {
|
|
2145
|
+
const param = args.push(where.status);
|
|
2146
|
+
whereClause += ` AND status=$${param}`;
|
|
2147
|
+
}
|
|
2148
|
+
const result = await client.query(`UPDATE "${this.schemaName}".workflow_status ${setClause} ${whereClause}`, args);
|
|
2149
|
+
const throwOnFailure = options.throwOnFailure ?? true;
|
|
2150
|
+
if (throwOnFailure && result.rowCount !== 1) {
|
|
2151
|
+
throw new error_1.DBOSWorkflowConflictError(`Attempt to record transition of nonexistent workflow ${workflowID}`);
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
async recordOperationResultInternal(client, workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
|
|
2155
|
+
try {
|
|
2156
|
+
const out = await client.query(`INSERT INTO ${this.schemaName}.operation_outputs
|
|
2157
|
+
(workflow_uuid, function_id, output, error, function_name, child_workflow_id, started_at_epoch_ms, completed_at_epoch_ms, serialization)
|
|
2158
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
2159
|
+
ON CONFLICT DO NOTHING RETURNING completed_at_epoch_ms;`, [
|
|
2160
|
+
workflowID,
|
|
2161
|
+
functionID,
|
|
2162
|
+
options.output ?? null,
|
|
2163
|
+
options.error ?? null,
|
|
2164
|
+
functionName,
|
|
2165
|
+
options.childWorkflowID ?? null,
|
|
2166
|
+
startTimeEpochMs,
|
|
2167
|
+
endTimeEpochMs,
|
|
2168
|
+
options.serialization ?? null,
|
|
2169
|
+
]);
|
|
2170
|
+
if (checkConflict &&
|
|
2171
|
+
(out?.rowCount ?? 0) > 0 &&
|
|
2172
|
+
Number(out?.rows?.[0]?.completed_at_epoch_ms) !== endTimeEpochMs) {
|
|
2173
|
+
dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Step output for ${workflowID}(${functionID}):${functionName} already recorded`);
|
|
2174
|
+
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
catch (error) {
|
|
2178
|
+
const err = error;
|
|
2179
|
+
if (err.code === '40001' || err.code === '23505') {
|
|
2180
|
+
// Serialization and primary key conflict (Postgres).
|
|
2181
|
+
throw new error_1.DBOSWorkflowConflictError(workflowID);
|
|
2182
|
+
}
|
|
2183
|
+
else {
|
|
2184
|
+
throw err;
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
}
|
|
2188
|
+
async #getOperationResultAndThrowIfCancelled(client, workflowID, functionID) {
|
|
2189
|
+
await this.#checkIfCanceled(client, workflowID);
|
|
2190
|
+
const { rows } = await client.query(`SELECT output, error, child_workflow_id, function_name
|
|
2191
|
+
FROM "${this.schemaName}".operation_outputs
|
|
2192
|
+
WHERE workflow_uuid=$1 AND function_id=$2`, [workflowID, functionID]);
|
|
2193
|
+
if (rows.length === 0) {
|
|
2194
|
+
return undefined;
|
|
2195
|
+
}
|
|
2196
|
+
else {
|
|
2197
|
+
return {
|
|
2198
|
+
output: rows[0].output,
|
|
2199
|
+
error: rows[0].error,
|
|
2200
|
+
childWorkflowID: rows[0].child_workflow_id,
|
|
2201
|
+
functionName: rows[0].function_name,
|
|
2202
|
+
};
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
async #runAndRecordResult(client, functionName, workflowID, functionID, func) {
|
|
2206
|
+
const startTime = Date.now();
|
|
2207
|
+
const result = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
2208
|
+
if (result !== undefined) {
|
|
2209
|
+
if (result.functionName !== functionName) {
|
|
2210
|
+
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, functionName, result.functionName);
|
|
2211
|
+
}
|
|
2212
|
+
return result.output;
|
|
2213
|
+
}
|
|
2214
|
+
const output = await func();
|
|
2215
|
+
await this.recordOperationResultInternal(client, workflowID, functionID, functionName, true, startTime, Date.now(), {
|
|
2216
|
+
output,
|
|
2217
|
+
});
|
|
2218
|
+
return output;
|
|
2219
|
+
}
|
|
2220
|
+
#setWFCancelMap(workflowID) {
|
|
2221
|
+
if (this.runningWorkflowMap.has(workflowID)) {
|
|
2222
|
+
this.workflowCancellationMap.set(workflowID, true);
|
|
2223
|
+
}
|
|
2224
|
+
this.cancelWakeupMap.callCallbacks(workflowID);
|
|
2225
|
+
}
|
|
2226
|
+
#clearWFCancelMap(workflowID) {
|
|
2227
|
+
if (this.workflowCancellationMap.has(workflowID)) {
|
|
2228
|
+
this.workflowCancellationMap.delete(workflowID);
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
async #checkIfCanceled(client, workflowID) {
|
|
2232
|
+
if (this.workflowCancellationMap.get(workflowID) === true) {
|
|
2233
|
+
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
2234
|
+
}
|
|
2235
|
+
const statusValue = await this.getWorkflowStatusValue(client, workflowID);
|
|
2236
|
+
if (statusValue === workflow_1.StatusString.CANCELLED) {
|
|
2237
|
+
throw new error_1.DBOSWorkflowCancelledError(workflowID);
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
async #durableSleep(workflowID, functionID, durationMS, maxSleepPerIteration) {
|
|
2241
|
+
if (maxSleepPerIteration === undefined)
|
|
2242
|
+
maxSleepPerIteration = durationMS;
|
|
2243
|
+
const curTime = Date.now();
|
|
2244
|
+
let endTimeMs = curTime + durationMS;
|
|
2245
|
+
const client = await this.pool.connect();
|
|
2246
|
+
try {
|
|
2247
|
+
const res = await this.#getOperationResultAndThrowIfCancelled(client, workflowID, functionID);
|
|
2248
|
+
if (res) {
|
|
2249
|
+
if (res.functionName !== exports.DBOS_FUNCNAME_SLEEP) {
|
|
2250
|
+
throw new error_1.DBOSUnexpectedStepError(workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, res.functionName);
|
|
2251
|
+
}
|
|
2252
|
+
endTimeMs = JSON.parse(res.output);
|
|
2253
|
+
}
|
|
2254
|
+
else {
|
|
2255
|
+
await this.recordOperationResultInternal(client, workflowID, functionID, exports.DBOS_FUNCNAME_SLEEP, false, Date.now(), Date.now(), {
|
|
2256
|
+
output: serialization_1.DBOSPortableJSON.stringify(endTimeMs),
|
|
2257
|
+
serialization: serialization_1.DBOSPortableJSON.name(),
|
|
2258
|
+
});
|
|
2259
|
+
}
|
|
2260
|
+
return {
|
|
2261
|
+
...(0, utils_1.cancellableSleep)(Math.max(Math.min(maxSleepPerIteration, endTimeMs - curTime), 0)),
|
|
2262
|
+
endTime: endTimeMs,
|
|
2263
|
+
};
|
|
2264
|
+
}
|
|
2265
|
+
finally {
|
|
2266
|
+
client.release();
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
/* BACKGROUND PROCESSES */
|
|
2270
|
+
/**
|
|
2271
|
+
* A background process that listens for notifications from Postgres then signals the appropriate
|
|
2272
|
+
* workflow listener by resolving its promise.
|
|
2273
|
+
*/
|
|
2274
|
+
reconnectTimeout = null;
|
|
2275
|
+
async #listenForNotifications() {
|
|
2276
|
+
const connect = async () => {
|
|
2277
|
+
const reconnect = () => {
|
|
2278
|
+
if (this.reconnectTimeout) {
|
|
2279
|
+
return;
|
|
2280
|
+
}
|
|
2281
|
+
this.reconnectTimeout = setTimeout(async () => {
|
|
2282
|
+
this.reconnectTimeout = null;
|
|
2283
|
+
await connect();
|
|
2284
|
+
}, 1000);
|
|
2285
|
+
};
|
|
2286
|
+
let client = null;
|
|
2287
|
+
try {
|
|
2288
|
+
client = await this.pool.connect();
|
|
2289
|
+
await client.query('LISTEN dbos_notifications_channel;');
|
|
2290
|
+
await client.query('LISTEN dbos_workflow_events_channel;');
|
|
2291
|
+
// Self-test: verify LISTEN actually works by sending a NOTIFY and checking it arrives.
|
|
2292
|
+
// If a transaction-mode pooler (e.g. PgBouncer pool_mode=transaction) is in the path,
|
|
2293
|
+
// LISTEN succeeds but the subscription is silently lost when the backend is released.
|
|
2294
|
+
let selfTestReceived = false;
|
|
2295
|
+
const onSelfTest = (msg) => {
|
|
2296
|
+
if (msg.channel === 'dbos_notifications_channel' && msg.payload === 'dbos_listen_selftest') {
|
|
2297
|
+
selfTestReceived = true;
|
|
2298
|
+
}
|
|
2299
|
+
};
|
|
2300
|
+
client.on('notification', onSelfTest);
|
|
2301
|
+
await this.pool.query("NOTIFY dbos_notifications_channel, 'dbos_listen_selftest'");
|
|
2302
|
+
for (let i = 0; i < 30 && !selfTestReceived; i++) {
|
|
2303
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
2304
|
+
}
|
|
2305
|
+
client.removeListener('notification', onSelfTest);
|
|
2306
|
+
if (!selfTestReceived) {
|
|
2307
|
+
this.logger.warn('LISTEN/NOTIFY self-test failed: notification was not received within 3 seconds. ' +
|
|
2308
|
+
'This typically means the connection is going through a transaction-mode pooler ' +
|
|
2309
|
+
'(e.g. PgBouncer with pool_mode=transaction), which silently breaks LISTEN/NOTIFY. ' +
|
|
2310
|
+
'Workflow notifications will fall back to polling, which may increase latency.');
|
|
2311
|
+
}
|
|
2312
|
+
const handler = (msg) => {
|
|
2313
|
+
if (!this.shouldUseDBNotifications)
|
|
2314
|
+
return;
|
|
2315
|
+
if (msg.channel === 'dbos_notifications_channel' && msg.payload) {
|
|
2316
|
+
this.notificationsMap.callCallbacks(msg.payload);
|
|
2317
|
+
}
|
|
2318
|
+
else if (msg.channel === 'dbos_workflow_events_channel' && msg.payload) {
|
|
2319
|
+
this.workflowEventsMap.callCallbacks(msg.payload);
|
|
2320
|
+
}
|
|
2321
|
+
};
|
|
2322
|
+
client.on('notification', handler);
|
|
2323
|
+
client.on('error', (err) => {
|
|
2324
|
+
this.logger.warn(`Error in notifications client: ${err}`);
|
|
2325
|
+
if (client) {
|
|
2326
|
+
client.removeAllListeners();
|
|
2327
|
+
client.release(true);
|
|
2328
|
+
}
|
|
2329
|
+
reconnect();
|
|
2330
|
+
});
|
|
2331
|
+
this.notificationsClient = client;
|
|
2332
|
+
}
|
|
2333
|
+
catch (error) {
|
|
2334
|
+
this.logger.warn(`Error in notifications listener: ${String(error)}`);
|
|
2335
|
+
if (client) {
|
|
2336
|
+
client.removeAllListeners();
|
|
2337
|
+
client.release(true);
|
|
2338
|
+
}
|
|
2339
|
+
reconnect();
|
|
2340
|
+
}
|
|
2341
|
+
};
|
|
2342
|
+
await connect();
|
|
2343
|
+
}
|
|
2280
2344
|
}
|
|
2281
|
-
exports.
|
|
2345
|
+
exports.SystemDatabase = SystemDatabase;
|
|
2282
2346
|
__decorate([
|
|
2283
2347
|
dbRetry(),
|
|
2284
2348
|
__metadata("design:type", Function),
|
|
2285
2349
|
__metadata("design:paramtypes", [Object, Object, Object]),
|
|
2286
2350
|
__metadata("design:returntype", Promise)
|
|
2287
|
-
],
|
|
2351
|
+
], SystemDatabase.prototype, "initWorkflowStatus", null);
|
|
2288
2352
|
__decorate([
|
|
2289
2353
|
dbRetry(),
|
|
2290
2354
|
__metadata("design:type", Function),
|
|
2291
2355
|
__metadata("design:paramtypes", [String, Object]),
|
|
2292
2356
|
__metadata("design:returntype", Promise)
|
|
2293
|
-
],
|
|
2357
|
+
], SystemDatabase.prototype, "recordWorkflowOutput", null);
|
|
2294
2358
|
__decorate([
|
|
2295
2359
|
dbRetry(),
|
|
2296
2360
|
__metadata("design:type", Function),
|
|
2297
2361
|
__metadata("design:paramtypes", [String, Object]),
|
|
2298
2362
|
__metadata("design:returntype", Promise)
|
|
2299
|
-
],
|
|
2363
|
+
], SystemDatabase.prototype, "recordWorkflowError", null);
|
|
2364
|
+
__decorate([
|
|
2365
|
+
dbRetry(),
|
|
2366
|
+
__metadata("design:type", Function),
|
|
2367
|
+
__metadata("design:paramtypes", [String, String, Number]),
|
|
2368
|
+
__metadata("design:returntype", Promise)
|
|
2369
|
+
], SystemDatabase.prototype, "getWorkflowStatus", null);
|
|
2300
2370
|
__decorate([
|
|
2301
2371
|
dbRetry(),
|
|
2302
2372
|
__metadata("design:type", Function),
|
|
2303
2373
|
__metadata("design:paramtypes", [String, Number]),
|
|
2304
2374
|
__metadata("design:returntype", Promise)
|
|
2305
|
-
],
|
|
2375
|
+
], SystemDatabase.prototype, "getOperationResultAndThrowIfCancelled", null);
|
|
2306
2376
|
__decorate([
|
|
2307
2377
|
dbRetry(),
|
|
2308
2378
|
__metadata("design:type", Function),
|
|
2309
2379
|
__metadata("design:paramtypes", [String, Number, String, Boolean, Number, Object]),
|
|
2310
2380
|
__metadata("design:returntype", Promise)
|
|
2311
|
-
],
|
|
2381
|
+
], SystemDatabase.prototype, "recordOperationResult", null);
|
|
2312
2382
|
__decorate([
|
|
2313
2383
|
dbRetry(),
|
|
2314
2384
|
__metadata("design:type", Function),
|
|
2315
|
-
__metadata("design:paramtypes", [String, Number,
|
|
2385
|
+
__metadata("design:paramtypes", [String, Number, String, Boolean]),
|
|
2316
2386
|
__metadata("design:returntype", Promise)
|
|
2317
|
-
],
|
|
2387
|
+
], SystemDatabase.prototype, "checkPatch", null);
|
|
2318
2388
|
__decorate([
|
|
2319
2389
|
dbRetry(),
|
|
2320
2390
|
__metadata("design:type", Function),
|
|
2321
|
-
__metadata("design:paramtypes", [String
|
|
2391
|
+
__metadata("design:paramtypes", [String]),
|
|
2322
2392
|
__metadata("design:returntype", Promise)
|
|
2323
|
-
],
|
|
2393
|
+
], SystemDatabase.prototype, "checkIfCanceled", null);
|
|
2324
2394
|
__decorate([
|
|
2325
2395
|
dbRetry(),
|
|
2326
2396
|
__metadata("design:type", Function),
|
|
2327
|
-
__metadata("design:paramtypes", [String,
|
|
2397
|
+
__metadata("design:paramtypes", [String, Number, String, Number]),
|
|
2328
2398
|
__metadata("design:returntype", Promise)
|
|
2329
|
-
],
|
|
2399
|
+
], SystemDatabase.prototype, "awaitWorkflowResult", null);
|
|
2330
2400
|
__decorate([
|
|
2331
2401
|
dbRetry(),
|
|
2332
2402
|
__metadata("design:type", Function),
|
|
2333
|
-
__metadata("design:paramtypes", [
|
|
2403
|
+
__metadata("design:paramtypes", [Array, String]),
|
|
2334
2404
|
__metadata("design:returntype", Promise)
|
|
2335
|
-
],
|
|
2405
|
+
], SystemDatabase.prototype, "awaitFirstWorkflowId", null);
|
|
2336
2406
|
__decorate([
|
|
2337
2407
|
dbRetry(),
|
|
2338
2408
|
__metadata("design:type", Function),
|
|
2339
|
-
__metadata("design:paramtypes", [String, Number,
|
|
2409
|
+
__metadata("design:paramtypes", [String, Number, Number]),
|
|
2340
2410
|
__metadata("design:returntype", Promise)
|
|
2341
|
-
],
|
|
2411
|
+
], SystemDatabase.prototype, "durableSleepms", null);
|
|
2342
2412
|
__decorate([
|
|
2343
2413
|
dbRetry(),
|
|
2344
2414
|
__metadata("design:type", Function),
|
|
2345
|
-
__metadata("design:paramtypes", [String, String,
|
|
2415
|
+
__metadata("design:paramtypes", [String, Number, String, Object, Object, Object, String]),
|
|
2346
2416
|
__metadata("design:returntype", Promise)
|
|
2347
|
-
],
|
|
2417
|
+
], SystemDatabase.prototype, "send", null);
|
|
2348
2418
|
__decorate([
|
|
2349
2419
|
dbRetry(),
|
|
2350
2420
|
__metadata("design:type", Function),
|
|
2351
|
-
__metadata("design:paramtypes", [String]),
|
|
2421
|
+
__metadata("design:paramtypes", [String, Object, Object, Object, String]),
|
|
2352
2422
|
__metadata("design:returntype", Promise)
|
|
2353
|
-
],
|
|
2423
|
+
], SystemDatabase.prototype, "sendDirect", null);
|
|
2354
2424
|
__decorate([
|
|
2355
2425
|
dbRetry(),
|
|
2356
2426
|
__metadata("design:type", Function),
|
|
2357
|
-
__metadata("design:paramtypes", [String, String, Number]),
|
|
2427
|
+
__metadata("design:paramtypes", [String, Number, Number, String, Number]),
|
|
2358
2428
|
__metadata("design:returntype", Promise)
|
|
2359
|
-
],
|
|
2429
|
+
], SystemDatabase.prototype, "recv", null);
|
|
2360
2430
|
__decorate([
|
|
2361
2431
|
dbRetry(),
|
|
2362
2432
|
__metadata("design:type", Function),
|
|
2363
|
-
__metadata("design:paramtypes", [String, Number, String,
|
|
2433
|
+
__metadata("design:paramtypes", [String, Number, String, Object, Object]),
|
|
2364
2434
|
__metadata("design:returntype", Promise)
|
|
2365
|
-
],
|
|
2435
|
+
], SystemDatabase.prototype, "setEvent", null);
|
|
2366
2436
|
__decorate([
|
|
2367
2437
|
dbRetry(),
|
|
2368
2438
|
__metadata("design:type", Function),
|
|
2369
|
-
__metadata("design:paramtypes", [
|
|
2439
|
+
__metadata("design:paramtypes", [String, String, Number, Object]),
|
|
2370
2440
|
__metadata("design:returntype", Promise)
|
|
2371
|
-
],
|
|
2441
|
+
], SystemDatabase.prototype, "getEvent", null);
|
|
2372
2442
|
__decorate([
|
|
2373
2443
|
dbRetry(),
|
|
2374
2444
|
__metadata("design:type", Function),
|
|
2375
2445
|
__metadata("design:paramtypes", [String, String, String]),
|
|
2376
2446
|
__metadata("design:returntype", Promise)
|
|
2377
|
-
],
|
|
2447
|
+
], SystemDatabase.prototype, "getEventDispatchState", null);
|
|
2378
2448
|
__decorate([
|
|
2379
2449
|
dbRetry(),
|
|
2380
2450
|
__metadata("design:type", Function),
|
|
2381
2451
|
__metadata("design:paramtypes", [Object]),
|
|
2382
2452
|
__metadata("design:returntype", Promise)
|
|
2383
|
-
],
|
|
2453
|
+
], SystemDatabase.prototype, "upsertEventDispatchState", null);
|
|
2384
2454
|
__decorate([
|
|
2385
2455
|
dbRetry(),
|
|
2386
2456
|
__metadata("design:type", Function),
|
|
2387
|
-
__metadata("design:paramtypes", [String, String]),
|
|
2457
|
+
__metadata("design:paramtypes", [String, Number, String, String, Object]),
|
|
2388
2458
|
__metadata("design:returntype", Promise)
|
|
2389
|
-
],
|
|
2459
|
+
], SystemDatabase.prototype, "writeStreamFromStep", null);
|
|
2390
2460
|
__decorate([
|
|
2391
2461
|
dbRetry(),
|
|
2392
2462
|
__metadata("design:type", Function),
|
|
2393
|
-
__metadata("design:paramtypes", [String]),
|
|
2463
|
+
__metadata("design:paramtypes", [String, Number, String, String, Object, String]),
|
|
2394
2464
|
__metadata("design:returntype", Promise)
|
|
2395
|
-
],
|
|
2465
|
+
], SystemDatabase.prototype, "writeStreamFromWorkflow", null);
|
|
2396
2466
|
__decorate([
|
|
2397
2467
|
dbRetry(),
|
|
2398
2468
|
__metadata("design:type", Function),
|
|
2399
|
-
__metadata("design:paramtypes", [String,
|
|
2469
|
+
__metadata("design:paramtypes", [String, String, Number]),
|
|
2400
2470
|
__metadata("design:returntype", Promise)
|
|
2401
|
-
],
|
|
2471
|
+
], SystemDatabase.prototype, "readStream", null);
|
|
2402
2472
|
__decorate([
|
|
2403
2473
|
dbRetry(),
|
|
2404
2474
|
__metadata("design:type", Function),
|
|
2405
|
-
__metadata("design:paramtypes", [String,
|
|
2475
|
+
__metadata("design:paramtypes", [String, String]),
|
|
2406
2476
|
__metadata("design:returntype", Promise)
|
|
2407
|
-
],
|
|
2477
|
+
], SystemDatabase.prototype, "getDeduplicatedWorkflow", null);
|
|
2408
2478
|
__decorate([
|
|
2409
2479
|
dbRetry(),
|
|
2410
2480
|
__metadata("design:type", Function),
|
|
2411
|
-
__metadata("design:paramtypes", [String
|
|
2481
|
+
__metadata("design:paramtypes", [String]),
|
|
2412
2482
|
__metadata("design:returntype", Promise)
|
|
2413
|
-
],
|
|
2483
|
+
], SystemDatabase.prototype, "getQueuePartitions", null);
|
|
2414
2484
|
__decorate([
|
|
2415
2485
|
dbRetry(),
|
|
2416
2486
|
__metadata("design:type", Function),
|
|
2417
2487
|
__metadata("design:paramtypes", [String, String]),
|
|
2418
2488
|
__metadata("design:returntype", Promise)
|
|
2419
|
-
],
|
|
2420
|
-
__decorate([
|
|
2421
|
-
dbRetry(),
|
|
2422
|
-
__metadata("design:type", Function),
|
|
2423
|
-
__metadata("design:paramtypes", [String, Number, String, Boolean]),
|
|
2424
|
-
__metadata("design:returntype", Promise)
|
|
2425
|
-
], PostgresSystemDatabase.prototype, "checkPatch", null);
|
|
2489
|
+
], SystemDatabase.prototype, "getMetrics", null);
|
|
2426
2490
|
//# sourceMappingURL=system_database.js.map
|