@dbos-inc/dbos-sdk 4.27.6 → 4.28.4-preview

package/dist/src/system_database.js

@@ -9,7 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_RENAME_BATCH_SIZE = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
+exports.SystemDatabase = exports.verifySystemDatabase = exports.ensureSystemDatabase = exports.grantDbosSchemaPermissions = exports.getDbosSchemaPermissionsSql = exports.DEFAULT_GC_BATCH_SIZE = exports.DEFAULT_RENAME_BATCH_SIZE = exports.validateObservabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = exports.DEFAULT_NOTIFICATION_COALESCE_MS = exports.DBOS_STREAMS_CHANNEL = exports.DBOS_WORKFLOW_EVENTS_CHANNEL = exports.DBOS_NOTIFICATIONS_CHANNEL = exports.isLegacyClosedSentinel = exports.isStreamClosedSentinel = exports.DBOS_STREAM_CLOSED_SENTINEL_SERIALIZED = exports.DBOS_STREAM_CLOSED_SENTINEL = exports.DEFAULT_POOL_SIZE = exports.DBOS_FUNCNAME_READSTREAMOFFSET = exports.DBOS_FUNCNAME_READSTREAM = exports.DBOS_FUNCNAME_CLOSESTREAM = exports.DBOS_FUNCNAME_WRITESTREAM = exports.DBOS_FUNCNAME_GETSTATUS = exports.DBOS_FUNCNAME_SLEEP = exports.DBOS_FUNCNAME_GETEVENT = exports.DBOS_FUNCNAME_SETEVENT = exports.DBOS_FUNCNAME_RECV = exports.DBOS_FUNCNAME_SEND = void 0;
 const dbos_executor_1 = require("./dbos-executor");
 const pg_1 = require("pg");
 const error_1 = require("./error");
@@ -56,8 +56,21 @@ exports.DBOS_WORKFLOW_EVENTS_CHANNEL = 'dbos_workflow_events_channel';
 exports.DBOS_STREAMS_CHANNEL = 'dbos_streams_channel';
 // Interval for coalescing LISTEN/NOTIFY notifications off the write path; caps the rate of notifying commits regardless of write throughput.
 exports.DEFAULT_NOTIFICATION_COALESCE_MS = 10;
+// An introspection query scanning a huge table for minutes holds back xmin, stalling autovacuum database-wide.
+exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS = 30_000;
+// PostgreSQL's statement_timeout is an int32 count of milliseconds; anything larger is rejected.
+const MAX_STATEMENT_TIMEOUT_MS = 2_147_483_647;
+/** The timeout is rendered into a `SET LOCAL statement_timeout`, which takes only an in-range integer. */
+function validateObservabilityQueryTimeoutMs(value) {
+    if (value !== undefined && (!Number.isFinite(value) || value > MAX_STATEMENT_TIMEOUT_MS)) {
+        throw new Error(`observabilityQueryTimeoutMs must be a finite number no greater than ${MAX_STATEMENT_TIMEOUT_MS}, got ${value}`);
+    }
+}
+exports.validateObservabilityQueryTimeoutMs = validateObservabilityQueryTimeoutMs;
 // Workflows re-owned per transaction by a rename. Matches the GC default.
 exports.DEFAULT_RENAME_BATCH_SIZE = 10_000;
+// Workflows deleted per transaction by garbage collection.
+exports.DEFAULT_GC_BATCH_SIZE = 10_000;
 const QUEUE_COLUMN_BY_FIELD = {
     concurrency: 'concurrency',
     workerConcurrency: 'worker_concurrency',
@@ -323,6 +336,14 @@ const RETRY_SQLSTATE_PREFIXES = new Set([
 const RETRY_SQLSTATE_CODES = new Set([
     '40003', // statement_completion_unknown
 ]);
+/**
+ * Kept out of the sets above: those feed `dbRetry`, which retries forever, and the step-recording
+ * path maps 40001 to a workflow conflict. Only bulk maintenance work retries on these.
+ */
+const SERIALIZATION_SQLSTATE_CODES = new Set([
+    '40001', // serialization_failure (MVCC conflict)
+    '40P01', // deadlock_detected
+]);
 // Node.js transient network error codes (system call level)
 const RETRY_NODE_ERRNOS = new Set([
     'ECONNRESET',
@@ -412,6 +433,49 @@ function retriablePostgresException(err) {
     }
     return false;
 }
+/** 57014 is query_canceled, which is how statement_timeout cancels a query. */
+function isStatementTimeout(err) {
+    return !!err && typeof err === 'object' && err.code === '57014';
+}
+function isSerializationError(err) {
+    for (const e of unwrapErrors(err)) {
+        const anyErr = e;
+        if (isPgDatabaseError(anyErr) && !!anyErr.code && SERIALIZATION_SQLSTATE_CODES.has(anyErr.code)) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Re-run a batch that lost a deadlock or serialization race. The database already rolled it
+ * back, so replaying it is safe.
+ */
+async function retryOnSerializationError(operation) {
+    const maxAttempts = 10;
+    const maxBackoff = 2.0;
+    let backoff = 0.05;
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await operation();
+        }
+        catch (e) {
+            if (!isSerializationError(e)) {
+                throw e;
+            }
+            const message = e instanceof Error ? e.message : String(e);
+            if (attempt === maxAttempts) {
+                dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Garbage collection failed after ${maxAttempts} attempts: ${message}`);
+                throw e;
+            }
+            // Jittered backoff, so peers that collided do not collide again
+            const actualBackoff = backoff * (0.5 + Math.random());
+            dbos_executor_1.DBOSExecutor.globalInstance?.logger.warn(`Contention or deadlock detected in workflow garbage collection: ${message}. ` +
+                `Retrying in ${actualBackoff.toFixed(2)}s (attempt ${attempt})`);
+            await (0, utils_1.sleepms)(actualBackoff * 1000);
+            backoff = Math.min(backoff * 2, maxBackoff);
+        }
+    }
+}
 /**
  * If a workflow encounters a database connection issue while performing an operation,
  * block the workflow and retry the operation until it reconnects and succeeds.
@@ -499,6 +563,8 @@ class SystemDatabase {
     customPool = false;
     // Interval for coalescing LISTEN/NOTIFY notifications pushed off the write path (Postgres + L/N only).
     notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS;
+    // Statement timeout for observability reads, in ms; undefined disables the cap.
+    observabilityQueryTimeoutMs;
     // Coalesced NOTIFY payloads keyed by channel, flushed by the notifier loop; soft-private so tests can drive a flush.
     pendingNotifications = new Map();
     #notifierActive = false;
@@ -519,7 +585,7 @@ class SystemDatabase {
     #destroyed = false;
     constructor(systemDatabaseUrl, logger, serializer, sysDbPoolSize = exports.DEFAULT_POOL_SIZE, systemDatabasePool, schemaName = 'dbos', useListenNotify = true, pollingConcurrency, notificationCoalesceMs = exports.DEFAULT_NOTIFICATION_COALESCE_MS, 
     // The application this handle acts for; undefined writes unclaimed rows.
-    appName) {
+    appName, observabilityQueryTimeoutMs = exports.DEFAULT_OBSERVABILITY_QUERY_TIMEOUT_MS) {
         this.systemDatabaseUrl = systemDatabaseUrl;
         this.logger = logger;
         this.serializer = serializer;
@@ -527,6 +593,10 @@ class SystemDatabase {
         this.schemaName = schemaName;
         this.shouldUseDBNotifications = useListenNotify;
         this.notificationCoalesceMs = notificationCoalesceMs;
+        validateObservabilityQueryTimeoutMs(observabilityQueryTimeoutMs);
+        // Floor at 1ms: PostgreSQL reads 0 as "no timeout", the loosest cap rather than the tightest.
+        this.observabilityQueryTimeoutMs =
+            observabilityQueryTimeoutMs > 0 ? Math.max(1, Math.round(observabilityQueryTimeoutMs)) : undefined;
         if (systemDatabasePool) {
             this.pool = systemDatabasePool;
             this.customPool = true;
@@ -560,6 +630,40 @@ class SystemDatabase {
     #connect() {
         return borrowClient(this.pool, this.#onClientError);
     }
+    /**
+     * Cap an introspection read with a statement timeout, so one scanning a huge table cannot hold a
+     * snapshot for minutes and stall autovacuum database-wide. Soft-private so tests can assert the cap
+     * from inside the transaction.
+     *
+     * `fn` must only run queries: node-postgres leaves the unnamed portal, and the snapshot registered
+     * with it, alive until commit, so anything else it awaits extends the very hold this bounds.
+     */
+    async observabilityQuery(fn) {
+        const timeoutMs = this.observabilityQueryTimeoutMs;
+        const client = await this.#connect();
+        try {
+            if (timeoutMs === undefined) {
+                return await fn(client);
+            }
+            try {
+                // Never inherit default_transaction_isolation: a repeatable-read snapshot would outlive its statement.
+                await client.query('BEGIN ISOLATION LEVEL READ COMMITTED READ ONLY');
+                // SET LOCAL, so the cap dies with its transaction instead of riding the pooled connection into unrelated queries.
+                await client.query(`SET LOCAL statement_timeout = ${timeoutMs}`);
+                const result = await fn(client);
+                await client.query('COMMIT');
+                return result;
+            }
+            catch (e) {
+                await client.query('ROLLBACK').catch(() => { });
+                // No `cause`: dbRetry walks the cause chain and would retry 57014 forever as an operator intervention.
+                throw isStatementTimeout(e) ? new error_1.DBOSQueryTimeoutError(timeoutMs) : e;
+            }
+        }
+        finally {
+            client.release();
+        }
+    }
     getSerializer() {
         return this.serializer;
     }
@@ -1014,7 +1118,7 @@ class SystemDatabase {
             params.push(offset);
             query += ` OFFSET $${params.length}`;
         }
-        const { rows } = await this.pool.query(query, params);
+        const { rows } = await this.observabilityQuery((client) => client.query(query, params));
         return rows;
     }
     async recordOperationResult(workflowID, functionID, functionName, checkConflict, startTimeEpochMs, endTimeEpochMs, options = {}) {
@@ -2317,65 +2421,47 @@ class SystemDatabase {
     }
     // ==================== Observability: Workflow Communications ====================
     async getAllEvents(workflowID) {
-        const client = await this.#connect();
-        try {
-            const result = await client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
-         WHERE workflow_uuid = $1`, [workflowID]);
-            const events = {};
-            for (const row of result.rows) {
-                events[row.key] = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
-            }
-            return events;
-        }
-        finally {
-            client.release();
+        const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".workflow_events
+         WHERE workflow_uuid = $1`, [workflowID]));
+        const events = {};
+        for (const row of rows) {
+            events[row.key] = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
         }
+        return events;
     }
     async getAllNotifications(workflowID) {
-        const client = await this.#connect();
-        try {
-            const result = await client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
+        const { rows } = await this.observabilityQuery((client) => client.query(`SELECT topic, message, serialization, created_at_epoch_ms, consumed
          FROM "${this.schemaName}".notifications
          WHERE destination_uuid = $1
-         ORDER BY created_at_epoch_ms`, [workflowID]);
-            return await Promise.all(result.rows.map(async (row) => ({
-                topic: row.topic === this.nullTopic ? null : row.topic,
-                message: await (0, serialization_1.safeParse)(this.serializer, row.message, row.serialization),
-                createdAtEpochMs: Number(row.created_at_epoch_ms),
-                consumed: row.consumed,
-            })));
-        }
-        finally {
-            client.release();
-        }
+         ORDER BY created_at_epoch_ms`, [workflowID]));
+        return await Promise.all(rows.map(async (row) => ({
+            topic: row.topic === this.nullTopic ? null : row.topic,
+            message: await (0, serialization_1.safeParse)(this.serializer, row.message, row.serialization),
+            createdAtEpochMs: Number(row.created_at_epoch_ms),
+            consumed: row.consumed,
+        })));
     }
     async getAllStreamEntries(workflowID) {
-        const client = await this.#connect();
-        try {
-            const result = await client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
+        const { rows } = await this.observabilityQuery((client) => client.query(`SELECT key, value, serialization FROM "${this.schemaName}".streams
          WHERE workflow_uuid = $1
-         ORDER BY key, "offset"`, [workflowID]);
-            const streams = {};
-            const closed = new Set();
-            for (const row of result.rows) {
-                if (closed.has(row.key)) {
-                    continue;
-                }
-                // safeParse yields the raw string for the legacy unserialized marker, which does not parse.
-                const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
-                if (isStreamClosedSentinel(value)) {
-                    // End the stream where readStream does, so the two never disagree.
-                    closed.add(row.key);
-                    streams[row.key] ??= [];
-                    continue;
-                }
-                (streams[row.key] ??= []).push(value);
+         ORDER BY key, "offset"`, [workflowID]));
+        const streams = {};
+        const closed = new Set();
+        for (const row of rows) {
+            if (closed.has(row.key)) {
+                continue;
             }
-            return streams;
-        }
-        finally {
-            client.release();
+            // safeParse yields the raw string for the legacy unserialized marker, which does not parse.
+            const value = await (0, serialization_1.safeParse)(this.serializer, row.value, row.serialization);
+            if (isStreamClosedSentinel(value)) {
+                // End the stream where readStream does, so the two never disagree.
+                closed.add(row.key);
+                streams[row.key] ??= [];
+                continue;
+            }
+            (streams[row.key] ??= []).push(value);
         }
+        return streams;
     }
     // ==================== Queues ====================
     async transitionDelayedWorkflows() {
@@ -2911,7 +2997,10 @@ class SystemDatabase {
       ${limitClause}
       ${offsetClause}
     `;
-        const result = await this.pool.query(query, params);
+        // An ID-keyed read is bounded by its ID list, so it takes no cap: a status lookup must not fail its own caller.
+        const result = idKeyed
+            ? await this.pool.query(query, params)
+            : await this.observabilityQuery((client) => client.query(query, params));
         return result.rows.map(mapWorkflowStatus);
     }
     async getWorkflowAggregates(input) {
@@ -3060,7 +3149,7 @@ class SystemDatabase {
       ${whereClause}
       GROUP BY ${groupByClause}
     `;
-        const result = await this.pool.query(query, params);
+        const result = await this.observabilityQuery((client) => client.query(query, params));
         const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
         return result.rows.map((row) => {
             const group = {};
@@ -3172,7 +3261,7 @@ class SystemDatabase {
       ${whereClause}
       GROUP BY ${groupByClause}
     `;
-        const result = await this.pool.query(query, params);
+        const result = await this.observabilityQuery((client) => client.query(query, params));
         const toIntOrNull = (v) => (v === null || v === undefined ? null : Number(v));
         return result.rows.map((row) => {
             const group = {};
@@ -3187,8 +3276,62 @@ class SystemDatabase {
             };
         });
     }
+    /** Rows garbage collection may delete: terminal, older than the cutoff, and ours. */
+    #gcFilter(cutoffEpochTimestampMs, params) {
+        params.push(cutoffEpochTimestampMs);
+        const cutoffClause = `created_at < $${params.length}`;
+        const statuses = [workflow_1.StatusString.PENDING, workflow_1.StatusString.ENQUEUED, workflow_1.StatusString.DELAYED].map((status) => {
+            params.push(status);
+            return `$${params.length}`;
+        });
+        // Unclaimed rows included: excluding them would leak pre-upgrade rows forever.
+        const scope = this.#appNameFilter('application_name', this.appName, params);
+        return `${cutoffClause} AND status NOT IN (${statuses.join(', ')}) AND ${scope}`;
+    }
+    /**
+     * Delete one batch, returning the watermark to resume from, or undefined once the last one ran.
+     * The delete is its own transaction; it re-checks the filter, so it needs no snapshot shared
+     * with the select that bounds it.
+     */
+    async #garbageCollectBatch(cutoffEpochTimestampMs, batchSize, watermark) {
+        // Borrowed rather than pool.query'd: that releases with the error, which discards the
+        // connection on a deadlock, so the retry wrapping this would churn the pool per batch.
+        const client = await this.#connect();
+        try {
+            // The batchSize-th oldest eligible row above the watermark bounds this range
+            const stepParams = [];
+            const stepScope = this.#gcFilter(cutoffEpochTimestampMs, stepParams);
+            stepParams.push(watermark);
+            const stepResult = await client.query(`SELECT created_at
+           FROM "${this.schemaName}".workflow_status
+          WHERE ${stepScope} AND created_at > $${stepParams.length}
+          ORDER BY created_at
+          LIMIT 1 OFFSET ${batchSize - 1}`, stepParams);
+            // created_at is a bigint, so node-postgres hands it back as a string.
+            const step = stepResult.rows.length > 0 ? Number(stepResult.rows[0].created_at) : undefined;
+            const deleteParams = [];
+            let deleteScope = this.#gcFilter(cutoffEpochTimestampMs, deleteParams);
+            if (step !== undefined) {
+                // Inclusive upper bound: created_at ties may push a batch over batchSize, but never split across two.
+                deleteParams.push(watermark, step);
+                deleteScope = `${deleteScope} AND created_at > $${deleteParams.length - 1} AND created_at <= $${deleteParams.length}`;
+            }
+            // The final batch drops the watermark, so rows that appeared below it are still deleted.
+            await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
+            return step;
+        }
+        finally {
+            // No error argument: a genuinely dead connection is still evicted by the pool's own check.
+            client.release();
+        }
+    }
     // Conductor sends cleared retention thresholds as JSON null, so both params must be treated as nullish
-    async garbageCollect(cutoffEpochTimestampMs, rowsThreshold) {
+    async garbageCollect(cutoffEpochTimestampMs, rowsThreshold, options = {}) {
+        const batchSize = options.batchSize === null ? undefined : (options.batchSize ?? exports.DEFAULT_GC_BATCH_SIZE);
+        // A NaN survives a bare `< 1` test and would only fail once it reached SQL, leaving GC half-applied.
+        if (batchSize !== undefined && (!Number.isInteger(batchSize) || batchSize < 1)) {
+            throw new error_1.DBOSError(`batchSize must be a positive integer, got ${batchSize}`);
+        }
         if (rowsThreshold !== undefined && rowsThreshold !== null) {
             // Get the created_at timestamp of the rows_threshold newest row
             const params = [rowsThreshold - 1];
@@ -3211,20 +3354,32 @@ class SystemDatabase {
         if (cutoffEpochTimestampMs === undefined || cutoffEpochTimestampMs === null) {
             return;
         }
-        const deleteParams = [
-            cutoffEpochTimestampMs,
-            workflow_1.StatusString.PENDING,
-            workflow_1.StatusString.ENQUEUED,
-            workflow_1.StatusString.DELAYED,
-        ];
-        // Unclaimed rows included: excluding them would leak pre-upgrade rows forever.
-        const deleteScope = this.#appNameFilter('application_name', this.appName, deleteParams);
-        // Delete all workflows older than cutoff that are NOT PENDING, ENQUEUED, or DELAYED
-        await this.pool.query(`DELETE FROM "${this.schemaName}".workflow_status
-       WHERE created_at < $1
-         AND status NOT IN ($2, $3, $4)
-         AND ${deleteScope}`, deleteParams);
-        return;
+        // Narrowed to a constant so the closures below keep it.
+        const cutoff = cutoffEpochTimestampMs;
+        if (batchSize === undefined) {
+            await retryOnSerializationError(async () => {
+                const deleteParams = [];
+                const deleteScope = this.#gcFilter(cutoff, deleteParams);
+                const client = await this.#connect();
+                try {
+                    await client.query(`DELETE FROM "${this.schemaName}".workflow_status WHERE ${deleteScope}`, deleteParams);
+                }
+                finally {
+                    client.release();
+                }
+            });
+            return;
+        }
+        // Advance a created_at watermark, one committed transaction per batch, so a long
+        // history neither deletes in one transaction nor rescans what it already deleted.
+        let watermark = 0;
+        for (;;) {
+            const next = await retryOnSerializationError(() => this.#garbageCollectBatch(cutoff, batchSize, watermark));
+            // Fewer than a full batch remained, so that delete took the rest.
+            if (next === undefined)
+                return;
+            watermark = next;
+        }
     }
     /**
      * IDs of this application's in-flight workflows created at or before the cutoff.
@@ -3248,14 +3403,21 @@ class SystemDatabase {
     async getMetrics(startTime, endTime, applicationName) {
         const startEpochMs = new Date(startTime).getTime();
         const endEpochMs = new Date(endTime).getTime();
-        const metrics = [];
-        // Query workflow metrics
         const workflowParams = [startEpochMs, endEpochMs];
         const workflowScope = this.#observabilityFilter('application_name', applicationName, workflowParams);
-        const workflowResult = await this.pool.query(`SELECT name, COUNT(workflow_uuid) as count
-       FROM "${this.schemaName}".workflow_status
-       WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
-       GROUP BY name`, workflowParams);
+        const stepParams = [startEpochMs, endEpochMs];
+        const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
+        const [workflowResult, stepResult] = await this.observabilityQuery(async (client) => [
+            await client.query(`SELECT name, COUNT(workflow_uuid) as count
+         FROM "${this.schemaName}".workflow_status
+         WHERE created_at >= $1 AND created_at < $2 AND ${workflowScope}
+         GROUP BY name`, workflowParams),
+            await client.query(`SELECT function_name, COUNT(*) as count
+         FROM "${this.schemaName}".operation_outputs
+         WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
+         GROUP BY function_name`, stepParams),
+        ]);
+        const metrics = [];
         for (const row of workflowResult.rows) {
             metrics.push({
                 metricType: 'workflow_count',
@@ -3263,13 +3425,6 @@ class SystemDatabase {
                 value: Number(row.count),
             });
         }
-        // Query step metrics
-        const stepParams = [startEpochMs, endEpochMs];
-        const stepScope = this.#observabilityFilter('application_name', applicationName, stepParams);
-        const stepResult = await this.pool.query(`SELECT function_name, COUNT(*) as count
-       FROM "${this.schemaName}".operation_outputs
-       WHERE completed_at_epoch_ms >= $1 AND completed_at_epoch_ms < $2 AND ${stepScope}
-       GROUP BY function_name`, stepParams);
         for (const row of stepResult.rows) {
             metrics.push({
                 metricType: 'step_count',
@@ -3519,10 +3674,10 @@ class SystemDatabase {
     async listApplicationVersions() {
         const params = [];
         const scope = this.#appNameFilter('application_name', this.appName, params);
-        const { rows } = await this.pool.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
-       FROM "${this.schemaName}".application_versions
-       WHERE ${scope}
-       ORDER BY version_timestamp DESC`, params);
+        const { rows } = await this.observabilityQuery((client) => client.query(`SELECT version_id, version_name, version_timestamp, created_at, application_name
+         FROM "${this.schemaName}".application_versions
+         WHERE ${scope}
+         ORDER BY version_timestamp DESC`, params));
         return rows.map(mapVersionInfo);
     }
     /**