@dbos-inc/dbos-sdk 4.27.3-preview → 4.27.5-preview

package/dist/src/system_database.js

@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
+exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
 const dbos_executor_1 = require("./dbos-executor");
 const pg_1 = require("pg");
 const error_1 = require("./error");
@@ -31,8 +31,25 @@ exports.DBOS_FUNCNAME_SLEEP = 'DBOS.sleep';
 exports.DBOS_FUNCNAME_GETSTATUS = 'getStatus';
 exports.DBOS_FUNCNAME_WRITESTREAM = 'DBOS.writeStream';
 exports.DBOS_FUNCNAME_CLOSESTREAM = 'DBOS.closeStream';
+exports.DBOS_FUNCNAME_READSTREAM = 'DBOS.readStream';
+exports.DBOS_FUNCNAME_READSTREAMOFFSET = 'DBOS.readStreamOffset';
 exports.DEFAULT_POOL_SIZE = 10;
 exports.DBOS_STREAM_CLOSED_SENTINEL = '__DBOS_STREAM_CLOSED__';
+// The sentinel as it is stored: portable JSON, the same bytes every language writes and reads.
+exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = serialization_1.DBOSPortableJSON.stringify(exports.DBOS_STREAM_CLOSED_SENTINEL);
+/** Whether a stream value is the marker a closed stream ends with. Takes the deserialized value. */
+function isStreamClosedSentinel(value) {
+    return typeof value === 'string' && value === exports.DBOS_STREAM_CLOSED_SENTINEL;
+}
+exports.isStreamClosedSentinel = isStreamClosedSentinel;
+/**
+ * Whether a stored value is the marker as releases before the portable form wrote it: unserialized,
+ * so no deserializer parses it. Callers must test this before deserializing.
+ */
+function isLegacyClosedSentinel(serializedValue) {
+    return serializedValue === exports.DBOS_STREAM_CLOSED_SENTINEL;
+}
+exports.isLegacyClosedSentinel = isLegacyClosedSentinel;
 // LISTEN/NOTIFY channels. Streams and workflow_events are pushed by the notifier loop off the write path; notifications fires from an in-transaction DB trigger so recv is never woken before its row commits.
 exports.DBOS_NOTIFICATIONS_CHANNEL = 'dbos_notifications_channel';
 exports.DBOS_WORKFLOW_EVENTS_CHANNEL = 'dbos_workflow_events_channel';
@@ -474,6 +491,7 @@ class SystemDatabase {
     notificationsClient = null;
     dbPollingIntervalResultMs = 1000;
     dbPollingIntervalEventMs = 10000;
+    dbPollingIntervalStreamMs = 1000;
     shouldUseDBNotifications = true;
     notificationsMap = new NotificationMap();
     workflowEventsMap = new NotificationMap();
@@ -1629,7 +1647,8 @@ class SystemDatabase {
         // Need to await for the workflow and capture errors.
         const awaitWorkflowPromise = workflowPromise
             .catch((error) => {
-            this.logger.debug('Captured error in awaitWorkflowPromise: ' + error);
+            const outcome = this.#destroyed ? 'was abandoned by shutdown' : 'failed';
+            this.logger.debug(`Workflow ${workflowID} ${outcome}: ${error}`);
         })
             .finally(() => {
             onSettled();
@@ -1664,10 +1683,35 @@ class SystemDatabase {
         }
         return count;
     }
-    async awaitRunningWorkflows() {
+    /** Wait up to `timeoutMS` for locally-running workflows to finish. Without a timeout, do not wait at all. */
+    async awaitRunningWorkflows(timeoutMS) {
+        if (timeoutMS !== undefined && timeoutMS > 0) {
+            const deadline = Date.now() + timeoutMS;
+            if (this.runningWorkflowMap.size > 0) {
+                this.logger.info('Waiting for pending workflows to finish.');
+            }
+            // Each pass picks up workflows a draining workflow started, and awaits any given run only once.
+            const awaited = new Set();
+            for (;;) {
+                const pending = Array.from(this.runningWorkflowMap.values(), (entry) => entry.promise).filter((promise) => !awaited.has(promise));
+                if (pending.length === 0)
+                    break;
+                for (const promise of pending)
+                    awaited.add(promise);
+                let timer;
+                const timedOut = await Promise.race([
+                    Promise.allSettled(pending).then(() => false),
+                    new Promise((resolve) => {
+                        timer = setTimeout(() => resolve(true), Math.max(0, deadline - Date.now()));
+                    }),
+                ]);
+                clearTimeout(timer);
+                if (timedOut)
+                    break;
+            }
+        }
         if (this.runningWorkflowMap.size > 0) {
-            this.logger.info('Waiting for pending workflows to finish.');
-            await Promise.allSettled(Array.from(this.runningWorkflowMap.values(), (entry) => entry.promise));
+            this.logger.warn(`Shutting down while ${this.runningWorkflowMap.size} workflows are still running: ${Array.from(this.runningWorkflowMap.keys()).join(', ')}`);
         }
         if (this.workflowEventsMap.map.size > 0) {
             this.logger.warn('Workflow events map is not empty - shutdown is not clean.');
@@ -1696,7 +1740,8 @@ class SystemDatabase {
      * under the polling limiter so it counts against the same concurrency budget
      * as the rest of the loop's reads.
      */
-    async #checkIfCanceledLimited(workflowID) {
+    /** Cancellation check for polling waits: goes through the limiter so readers cannot starve the pool. */
+    async checkIfCanceledLimited(workflowID) {
         await this.#pollWithLimiter(() => this.#checkIfCanceled(this.pool, workflowID));
     }
     // A missing row normally means the workflow has not been inserted yet, so
@@ -1715,7 +1760,7 @@ class SystemDatabase {
         }
         while (true) {
             if (callerID)
-                await this.#checkIfCanceledLimited(callerID);
+                await this.checkIfCanceledLimited(callerID);
             let rows;
             try {
                 ({ rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT status, output, error, serialization FROM "${this.schemaName}".workflow_status
@@ -1760,7 +1805,7 @@ class SystemDatabase {
         const pollIntervalMs = pollingIntervalMs ?? this.dbPollingIntervalResultMs;
         while (true) {
             if (callerID)
-                await this.#checkIfCanceledLimited(callerID);
+                await this.checkIfCanceledLimited(callerID);
             const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
            WHERE workflow_uuid IN (${placeholders})
              AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')
@@ -1777,7 +1822,7 @@ class SystemDatabase {
         while (remainingWorkflowIds.size > 0) {
             const currentWorkflowIds = [...remainingWorkflowIds];
             if (callerID)
-                await this.#checkIfCanceledLimited(callerID);
+                await this.checkIfCanceledLimited(callerID);
             const { rows } = await this.#pollWithLimiter(() => this.pool.query(`SELECT workflow_uuid FROM "${this.schemaName}".workflow_status
            WHERE workflow_uuid = ANY($1::text[])
              AND status NOT IN ('${workflow_1.StatusString.PENDING}', '${workflow_1.StatusString.ENQUEUED}', '${workflow_1.StatusString.DELAYED}')`, [currentWorkflowIds]));
@@ -1884,7 +1929,7 @@ class SystemDatabase {
             const payload = `${workflowID}::${topic}`;
             const cbr = this.notificationsMap.registerCallback(payload, resolveNotification);
             try {
-                await this.#checkIfCanceledLimited(workflowID);
+                await this.checkIfCanceledLimited(workflowID);
                 // Check if the key is already in the DB, then wait for the notification if it isn't.
                 const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT topic FROM "${this.schemaName}".notifications WHERE destination_uuid=$1 AND topic=$2 AND consumed = false;`, [workflowID, topic]))).rows;
                 if (initRecvRows.length !== 0)
@@ -2019,7 +2064,7 @@ class SystemDatabase {
             const cbr = this.workflowEventsMap.registerCallback(payloadKey, resolveNotification);
             try {
                 if (callerWorkflow?.workflowID)
-                    await this.#checkIfCanceledLimited(callerWorkflow?.workflowID);
+                    await this.checkIfCanceledLimited(callerWorkflow?.workflowID);
                 // Check if the key is already in the DB, then wait for the notification if it isn't.
                 const initRecvRows = (await this.#pollWithLimiter(() => this.pool.query(`SELECT key, value, serialization
              FROM "${this.schemaName}".workflow_events
@@ -2098,67 +2143,79 @@ class SystemDatabase {
     }
     // ==================== Streams ====================
     async writeStreamFromStep(workflowID, functionID, key, serializedValue, serialization) {
-        const client = await this.#connect();
-        try {
-            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            // Find the maximum offset for this workflow_uuid and key combination
-            const maxOffsetResult = await client.query(`SELECT MAX("offset") FROM "${this.schemaName}".streams
-         WHERE workflow_uuid = $1 AND key = $2`, [workflowID, key]);
-            // Next offset is max + 1, or 0 if no records exist
-            const maxOffset = maxOffsetResult.rows[0].max;
-            const nextOffset = maxOffset !== null ? maxOffset + 1 : 0;
-            // Insert the new stream entry
-            await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
-         VALUES ($1, $2, $3, $4, $5, $6)`, [workflowID, key, serializedValue, nextOffset, functionID, serialization]);
-            await client.query('COMMIT');
+        while (true) {
+            try {
+                // Derives the first unused offset inside the insert; two writers can still pick the same one.
+                await this.pool.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
+           SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
+           FROM "${this.schemaName}".streams s
+           WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
+            }
+            catch (e) {
+                // Only an offset conflict resolves on retry; anything else would spin forever.
+                if (e instanceof pg_1.DatabaseError && e.code === '23505') {
+                    this.logger.warn(`Stream offset conflict for workflow ${workflowID}, key ${key}; retrying`);
+                    await (0, utils_1.sleepms)(100);
+                    continue;
+                }
+                this.logger.error(e);
+                throw e;
+            }
             // Notify only after commit, so a woken reader sees the value.
             this.#signalNotification(exports.DBOS_STREAMS_CHANNEL, `${workflowID}::${key}`);
-        }
-        catch (e) {
-            this.logger.error(e);
-            await client.query('ROLLBACK');
-            throw e;
-        }
-        finally {
-            client.release();
+            return;
         }
     }
     async writeStreamFromWorkflow(workflowID, functionID, key, serializedValue, serialization, functionName) {
         const client = await this.#connect();
         try {
-            await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
-            // Only a real insert (not a replay) should wake readers.
-            let didWrite = false;
-            await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
-                // Find the maximum offset for this workflow_uuid and key combination
-                const maxOffsetResult = await client.query(`SELECT MAX("offset") FROM "${this.schemaName}".streams
-           WHERE workflow_uuid = $1 AND key = $2`, [workflowID, key]);
-                // Next offset is max + 1, or 0 if no records exist
-                const maxOffset = maxOffsetResult.rows[0].max;
-                const nextOffset = maxOffset !== null ? maxOffset + 1 : 0;
-                // Insert the new stream entry
-                await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
-           VALUES ($1, $2, $3, $4, $5, $6)`, [workflowID, key, serializedValue, nextOffset, functionID, serialization]);
-                didWrite = true;
-                return undefined;
-            });
-            await client.query('COMMIT');
-            // Notify only after commit, so a woken reader sees the value.
-            if (didWrite) {
-                this.#signalNotification(exports.DBOS_STREAMS_CHANNEL, `${workflowID}::${key}`);
+            while (true) {
+                // Only a real insert (not a replay) should wake readers.
+                let didWrite = false;
+                try {
+                    await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
+                    await this.#runAndRecordResult(client, functionName, workflowID, functionID, async () => {
+                        // Derives the first unused offset inside the insert; two writers can still pick the same one.
+                        await client.query(`INSERT INTO "${this.schemaName}".streams (workflow_uuid, key, value, "offset", function_id, serialization)
+               SELECT $1::text, $2::text, $3::text, COALESCE(MAX(s."offset"), -1) + 1, $4::int, $5::text
+               FROM "${this.schemaName}".streams s
+               WHERE s.workflow_uuid = $1 AND s.key = $2`, [workflowID, key, serializedValue, functionID, serialization]);
+                        didWrite = true;
+                        return undefined;
+                    });
+                    await client.query('COMMIT');
+                }
+                catch (e) {
+                    // Only an offset conflict resolves on retry; anything else would spin forever.
+                    const offsetConflict = e instanceof pg_1.DatabaseError && e.code === '23505';
+                    // Log before touching the connection again: a failing ROLLBACK is what would propagate.
+                    if (!offsetConflict)
+                        this.logger.error(e);
+                    // Roll back before waiting, so a retry does not hold an aborted transaction open.
+                    await client.query('ROLLBACK');
+                    if (offsetConflict) {
+                        this.logger.warn(`Stream offset conflict for workflow ${workflowID}, key ${key}; retrying`);
+                        await (0, utils_1.sleepms)(100);
+                        continue;
+                    }
+                    throw e;
+                }
+                // Notify only after commit, so a woken reader sees the value.
+                if (didWrite) {
+                    this.#signalNotification(exports.DBOS_STREAMS_CHANNEL, `${workflowID}::${key}`);
+                }
+                return;
             }
         }
-        catch (e) {
-            this.logger.error(e);
-            await client.query('ROLLBACK');
-            throw e;
-        }
         finally {
             client.release();
         }
     }
-    async closeStream(workflowID, functionID, key) {
-        await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL, 'portable_json', exports.DBOS_FUNCNAME_CLOSESTREAM);
+    async closeStreamFromWorkflow(workflowID, functionID, key) {
+        await this.writeStreamFromWorkflow(workflowID, functionID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name(), exports.DBOS_FUNCNAME_CLOSESTREAM);
+    }
+    async closeStreamFromStep(workflowID, stepID, key) {
+        await this.writeStreamFromStep(workflowID, stepID, key, exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED, serialization_1.DBOSPortableJSON.name());
     }
     // Read the value at `offset` and the workflow's status in one query: status null = no such workflow, value undefined = nothing at that offset.
     async readStreamValue(workflowID, key, offset) {
@@ -2299,15 +2356,20 @@ class SystemDatabase {
          WHERE workflow_uuid = $1
          ORDER BY key, "offset"`, [workflowID]);
             const streams = {};
+            const closed = new Set();
             for (const row of result.rows) {
-                const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
-                if (value === exports.DBOS_STREAM_CLOSED_SENTINEL) {
+                if (closed.has(row.key)) {
                     continue;
                 }
-                if (!streams[row.key]) {
-                    streams[row.key] = [];
+                // safeParse yields the raw string for the legacy unserialized marker, which does not parse.
+                const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
+                if (isStreamClosedSentinel(value)) {
+                    // End the stream where readStream does, so the two never disagree.
+                    closed.add(row.key);
+                    streams[row.key] ??= [];
+                    continue;
                 }
-                streams[row.key].push(value);
+                (streams[row.key] ??= []).push(value);
             }
             return streams;
         }