@loadstrike/loadstrike-sdk 1.0.30401 → 1.0.31001

package/dist/esm/runtime.js

@@ -10,6 +10,8 @@ import { buildDotnetCsvReport, buildDotnetHtmlReport, buildDotnetMarkdownReport,
 import { PortalReportingSink, cloneReportingSinkForRun } from "./sinks.js";
 import { LoadEngineV2ExecutionBudget, buildLoadEngineV2TrafficMixSeedId, LoadStrikeHistogramV1, parseLoadEngineV2HistogramArtifact, serializeLoadEngineV2HistogramArtifact, classifyLoadEngineV2Arrival, loadEngineV2FixedArrivalCount, loadEngineV2FixedDeadlineNs, loadEngineV2LatenessToleranceNs, loadEngineV2TrafficMixLaneUnitCount, loadEngineV2TrafficMixOwnedUnits, planRampingInjectionDeadlines, planRampingConstantDeadlines, planRandomInjectionDeadlines, fnv1a32 } from "./load-engine-v2.js";
 import { DEFAULT_ITERATION_OBSERVATION_SETTINGS, IterationObservationReporter, createIterationObservation, createIterationStepObservation, utcNowNs, validateIterationObservationSettings } from "./iteration-observations.js";
+import { logIterationObservationFailure, logIterationObservationRecovery } from "./iteration-observation-diagnostics.js";
+import { normalizeSinkRetryBackoffMs, normalizeSinkRetryCount, sinkRetryDelayMs, waitForSinkRetryDelay } from "./sink-retry-policy.js";
 const LOAD_ENGINE_V2_SCHEDULER_DISTRIBUTIONS = Symbol("loadstrike.load-engine-v2.scheduler-distributions");
 const PORTAL_RUN_TOKEN_PROVIDER = Symbol.for("loadstrike.internal.portal-run-token-provider");
 export const LoadStrikeNodeType = {
@@ -3427,8 +3429,8 @@ export class LoadStrikeRunner {
         }));
         const sinkErrors = [];
         const policyErrors = [];
-        const sinkRetryCount = Math.max(this.options.sinkRetryCount ?? 2, 0);
-        const sinkRetryBackoffMs = Math.max(this.options.sinkRetryBackoffMs ?? 25, 0);
+        const sinkRetryCount = normalizeSinkRetryCount(this.options.sinkRetryCount);
+        const sinkRetryBackoffMs = normalizeSinkRetryBackoffMs(this.options.sinkRetryBackoffMs);
         const policies = this.options.runtimePolicies ?? [];
         const runtimePolicyErrorMode = normalizedRuntimePolicyErrorMode(this.options.runtimePolicyErrorMode);
         const plugins = this.options.reportingSinks === undefined && this.options.workerPlugins === undefined &&
@@ -3522,14 +3524,14 @@ export class LoadStrikeRunner {
                 await init(baseContext, this.options.infraConfig ?? {});
             }
         }
-        await this.initializeSinks(sinkStates, baseContext, this.options.infraConfig ?? {}, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
+        await this.initializeSinks(sinkStates, baseContext, this.options.infraConfig ?? {}, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
         for (const plugin of plugins) {
             const start = resolveWorkerPluginStart(plugin);
             if (start) {
                 await start(sessionInfo);
             }
         }
-        await this.startSinks(sinkStates, sessionInfo, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
+        await this.startSinks(sinkStates, sessionInfo, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
         const iterationObservationRunId = String(this.internalOptions.iterationObservationRunId
             ?? sessionInfo.portalReportingRunId
             ?? sessionInfo.PortalReportingRunId
@@ -3560,7 +3562,10 @@ export class LoadStrikeRunner {
             expectedResultOwnerCount64: iterationObservationExpectedResultOwnerCount64,
             processGroup: iterationObservationProcessGroup,
             settings: resolveIterationObservationSettings(this.options),
-            sinks: reporterIterationObservationSinks
+            sinks: reporterIterationObservationSinks,
+            sinkRetryCount,
+            sinkRetryBackoffMs,
+            logger: runLogger
         });
         const emitRealtimeSnapshot = async () => {
             if (realtimeInFlight) {
@@ -3572,7 +3577,7 @@ export class LoadStrikeRunner {
                     .map((value) => value.build(scenarioDurationsMs.get(value.scenarioName) ?? Math.max(Date.now() - started.getTime(), 0)))
                     .sort((left, right) => left.sortIndex - right.sortIndex);
                 const metricsSnapshot = collectMetricStats(allRegisteredMetrics, Date.now() - started.getTime());
-                await this.emitRealtimeStats(sinkStates, snapshot, metricsSnapshot, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
+                await this.emitRealtimeStats(sinkStates, snapshot, metricsSnapshot, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
                 if (toBoolean(this.options.displayConsoleMetrics, true)) {
                     const requestCount = snapshot.reduce((sum, value) => sum + value.allRequestCount, 0);
                     const okCount = snapshot.reduce((sum, value) => sum + value.allOkCount, 0);
@@ -3763,8 +3768,8 @@ export class LoadStrikeRunner {
             finalizedResult.logFiles = mergeStringArrays(finalizedResult.logFiles, loggerSetup.logFiles);
             finalizedResult.reportFiles = this.writeReports(finalizedResult);
             finalizedResult.disabledSinks = sinkStates.filter((x) => x.disabled).map((x) => x.name);
-            await this.emitRunResult(sinkStates, finalizedResult, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
-            await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
+            await this.emitRunResult(sinkStates, finalizedResult, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
+            await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
             sinksStopped = true;
             finalizedResult.disabledSinks = sinkStates.filter((x) => x.disabled).map((x) => x.name);
             finalizedResult.sinkErrors = sinkErrors
@@ -3777,7 +3782,7 @@ export class LoadStrikeRunner {
                 await iterationObservationReporter.sealAndDrain().catch(() => { });
             }
             if (!sinksStopped) {
-                await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors);
+                await this.stopSinks(sinkStates, sinkRetryCount, sinkRetryBackoffMs, sinkErrors, testInfo.sessionId, runLogger);
             }
             if (!pluginsStopped) {
                 await this.stopPlugins(plugins, pluginLifecycleErrors, runLogger);
@@ -4085,9 +4090,9 @@ export class LoadStrikeRunner {
         }
         return filtered;
     }
-    async initializeSinks(sinkStates, context, infraConfig, retryCount, retryBackoffMs, sinkErrors) {
+    async initializeSinks(sinkStates, context, infraConfig, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
         for (const state of sinkStates) {
-            await this.invokeSinkAction(state, "init", retryCount, retryBackoffMs, sinkErrors, false, true, async () => {
+            await this.invokeSinkAction(state, "init", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, true, async () => {
                 const init = resolveSinkInit(state.sink);
                 if (init) {
                     await init(context, infraConfig);
@@ -4095,9 +4100,9 @@ export class LoadStrikeRunner {
             });
         }
     }
-    async startSinks(sinkStates, session, retryCount, retryBackoffMs, sinkErrors) {
+    async startSinks(sinkStates, session, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
         for (const state of sinkStates) {
-            await this.invokeSinkAction(state, "start", retryCount, retryBackoffMs, sinkErrors, false, true, async () => {
+            await this.invokeSinkAction(state, "start", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, true, async () => {
                 const start = resolveSinkStart(state.sink);
                 if (start) {
                     await start(session);
@@ -4105,9 +4110,9 @@ export class LoadStrikeRunner {
             });
         }
     }
-    async emitRealtimeStats(sinkStates, scenarioStats, metrics, retryCount, retryBackoffMs, sinkErrors) {
+    async emitRealtimeStats(sinkStates, scenarioStats, metrics, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
         for (const state of sinkStates) {
-            await this.invokeSinkAction(state, "realtime", retryCount, retryBackoffMs, sinkErrors, false, true, async () => {
+            await this.invokeSinkAction(state, "realtime", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, true, async () => {
                 const saveRealtimeStats = resolveSinkSaveRealtimeStats(state.sink);
                 if (saveRealtimeStats) {
                     await saveRealtimeStats(scenarioStats);
@@ -4119,10 +4124,9 @@ export class LoadStrikeRunner {
             });
         }
     }
-    async stopSinks(sinkStates, retryCount, retryBackoffMs, sinkErrors) {
-        const shutdownRetryCount = 0;
+    async stopSinks(sinkStates, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
         for (const state of sinkStates) {
-            await this.invokeSinkAction(state, "stop", shutdownRetryCount, retryBackoffMs, sinkErrors, true, true, async () => {
+            await this.invokeSinkAction(state, "stop", retryCount, retryBackoffMs, sinkErrors, runId, logger, true, true, async () => {
                 const stop = resolveSinkStop(state.sink);
                 if (stop) {
                     await stop();
@@ -4130,15 +4134,15 @@ export class LoadStrikeRunner {
             });
             const dispose = resolveSinkDispose(state.sink);
             if (dispose) {
-                await this.invokeSinkAction(state, "dispose", shutdownRetryCount, retryBackoffMs, sinkErrors, true, true, async () => {
+                await this.invokeSinkAction(state, "dispose", retryCount, retryBackoffMs, sinkErrors, runId, logger, true, true, async () => {
                     await dispose();
                 });
             }
         }
     }
-    async emitRunResult(sinkStates, result, retryCount, retryBackoffMs, sinkErrors) {
+    async emitRunResult(sinkStates, result, retryCount, retryBackoffMs, sinkErrors, runId, logger) {
         for (const state of sinkStates) {
-            await this.invokeSinkAction(state, "run-result", retryCount, retryBackoffMs, sinkErrors, false, false, async () => {
+            await this.invokeSinkAction(state, "run-result", retryCount, retryBackoffMs, sinkErrors, runId, logger, false, false, async () => {
                 const saveRunResult = resolveSinkSaveRunResult(state.sink);
                 if (saveRunResult) {
                     await saveRunResult(result);
@@ -4146,23 +4150,49 @@ export class LoadStrikeRunner {
             });
         }
     }
-    async invokeSinkAction(state, phase, retryCount, retryBackoffMs, sinkErrors, ignoreDisabled, disableOnFailure, action) {
+    async invokeSinkAction(state, phase, retryCount, retryBackoffMs, sinkErrors, runId, logger, ignoreDisabled, disableOnFailure, action) {
         if (state.disabled && !ignoreDisabled) {
             return;
         }
-        let attempts = 0;
-        while (attempts <= retryCount) {
-            attempts += 1;
+        const maximumAttempts = normalizeSinkRetryCount(retryCount) + 1;
+        const backoffMs = normalizeSinkRetryBackoffMs(retryBackoffMs);
+        for (let attempts = 1; attempts <= maximumAttempts; attempts += 1) {
             try {
                 await action();
+                if (attempts > 1) {
+                    logIterationObservationRecovery(logger, {
+                        sinkName: state.name,
+                        operation: "reporting-sink-action",
+                        phase,
+                        runId,
+                        resultOwnerId: "",
+                        observationCount: 0,
+                        attempt: attempts,
+                        maximumAttempts,
+                        nextDelayMs: 0
+                    });
+                }
                 return;
             }
             catch (error) {
-                if (attempts > retryCount) {
+                const exhausted = attempts >= maximumAttempts;
+                const nextDelayMs = exhausted ? 0 : sinkRetryDelayMs(backoffMs, attempts);
+                logIterationObservationFailure(logger, exhausted ? "error" : "warn", {
+                    sinkName: state.name,
+                    operation: "reporting-sink-action",
+                    phase,
+                    runId,
+                    resultOwnerId: "",
+                    observationCount: 0,
+                    attempt: attempts,
+                    maximumAttempts,
+                    nextDelayMs
+                }, error);
+                if (exhausted) {
                     sinkErrors.push({
                         sinkName: state.name,
                         phase,
-                        message: String(error ?? "sink action failed"),
+                        message: "The reporting sink action failed after retries.",
                         attempts
                     });
                     if (disableOnFailure) {
@@ -4170,9 +4200,7 @@ export class LoadStrikeRunner {
                     }
                     return;
                 }
-                if (retryBackoffMs > 0) {
-                    await sleep(retryBackoffMs * attempts);
-                }
+                await waitForSinkRetryDelay(nextDelayMs);
             }
         }
     }
@@ -4581,7 +4609,18 @@ async function executeScenarioRuntime(args) {
         attachScenarioContextAliases(context);
         const startedUtcNs = utcNowNs();
         const startedAtNs = process.hrtime.bigint();
-        const reply = await executeScenarioInvocation(scenario, context, operation);
+        let policyFailure;
+        let reply;
+        try {
+            reply = await executeScenarioInvocation(scenario, context, operation);
+        }
+        catch (error) {
+            if (!(error instanceof RuntimePolicyCallbackError)) {
+                throw error;
+            }
+            policyFailure = error;
+            reply = LoadStrikeResponse.fail("runtime_policy_error", "", 0);
+        }
         const observedLatencyNs = maxBigInt(process.hrtime.bigint() - startedAtNs, 0n);
         const completedUtcNs = startedUtcNs + observedLatencyNs;
         const observedLatencyMs = Number(observedLatencyNs) / 1000000;
@@ -4596,7 +4635,8 @@ async function executeScenarioRuntime(args) {
             observedLatencyUs: observedLatencyNs / 1000n,
             reportedLatencyUs: normalizeRawObservationLatencyMicroseconds(resolveRecordedLatency(reply.customLatencyMs, observedLatencyMs)),
             steps: attemptSteps,
-            recordedSteps
+            recordedSteps,
+            ...(policyFailure ? { policyFailure } : {})
         };
     };
     const captureAttemptObservation = (operation, globalOrdinal, attemptIndex, isFinalAttempt, attempt, simulationIndex, simulationKind, iterationId, globalSecondaryOrdinal = 0n) => {
@@ -4645,22 +4685,18 @@ async function executeScenarioRuntime(args) {
         const maxAttempts = 1 + (scenario.shouldRestartIterationOnFail() ? restartIterationMaxAttempts : 0);
         while (attempts < maxAttempts && !shouldStopNow()) {
             attempts += 1;
-            let attempt;
-            try {
-                attempt = await runSingleInvocation("Bombing", instanceData, instanceNumber, instanceId, false);
-            }
-            catch (error) {
-                if (error instanceof RuntimePolicyCallbackError) {
-                    stopScenario = true;
-                    scenarioAbortController.abort(error);
-                }
-                throw error;
-            }
-            const shouldRetry = !attempt.reply.isSuccess
+            const attempt = await runSingleInvocation("Bombing", instanceData, instanceNumber, instanceId, false);
+            const shouldRetry = !attempt.policyFailure
+                && !attempt.reply.isSuccess
                 && scenario.shouldRestartIterationOnFail()
                 && attempts < maxAttempts
                 && !shouldStopNow();
             captureAttemptObservation("Bombing", globalOrdinal, attempts - 1, !shouldRetry, attempt, simulationIndex, simulationKind, iterationId, explicitSecondaryOrdinal);
+            if (attempt.policyFailure) {
+                stopScenario = true;
+                scenarioAbortController.abort(attempt.policyFailure);
+                throw attempt.policyFailure;
+            }
             if (!shouldRetry) {
                 for (const step of attempt.recordedSteps) {
                     recordStepReply(step.stepName, step.reply, step.observedLatencyMs, step.sortIndex);
@@ -4687,6 +4723,11 @@ async function executeScenarioRuntime(args) {
             const globalOrdinal = nextObservationOrdinal();
             const attempt = await runSingleInvocation("WarmUp", instanceInfo.instanceData, instanceInfo.instanceNumber, instanceInfo.instanceId, false);
             captureAttemptObservation("WarmUp", globalOrdinal, 0, true, attempt, -1, "SingleInvocation");
+            if (attempt.policyFailure) {
+                stopScenario = true;
+                scenarioAbortController.abort(attempt.policyFailure);
+                throw attempt.policyFailure;
+            }
         }
     };
     const executeV2TimedConstant = async (copies, durationNs, ramping, runSimulationInvocation, segment) => {
@@ -5357,10 +5398,17 @@ async function waitForScenarioTasks(tasks, scenarioName, timeoutSeconds, logger,
         throwPolicyFailure(await settled);
         return;
     }
-    const completed = await Promise.race([
-        settled.then(() => true),
-        delayWithAbort(Math.trunc(timeoutSeconds * 1000), signal).then(() => false)
-    ]);
+    const timeoutController = new AbortController();
+    let completed;
+    try {
+        completed = await Promise.race([
+            settled.then(() => true),
+            delayWithAbort(Math.trunc(timeoutSeconds * 1000), combineAbortSignals(signal, timeoutController.signal)).then(() => false)
+        ]);
+    }
+    finally {
+        timeoutController.abort();
+    }
     if (!completed) {
         if (signal.reason instanceof RuntimePolicyCallbackError) {
             throw signal.reason;
@@ -9422,11 +9470,23 @@ function readRuntimeTrackingId(payload, selector) {
             return null;
         }
         let current = body;
-        for (const segment of selector.slice("json:".length).trim().replace(/^\$\./, "").split(".").filter(Boolean)) {
+        const path = selector.slice("json:".length).trim().replace(/^\$\./, "");
+        let segments;
+        try {
+            segments = runtimeSafeJsonPathSegments(path);
+        }
+        catch {
+            return null;
+        }
+        for (const segment of segments) {
             if (!current || typeof current !== "object" || Array.isArray(current)) {
                 return null;
             }
-            current = current[segment];
+            const record = current;
+            if (!Object.prototype.hasOwnProperty.call(record, segment)) {
+                return null;
+            }
+            current = runtimeReadOwnJsonProperty(record, segment);
         }
         return current == null ? null : String(current);
     }
@@ -9450,23 +9510,57 @@ function runtimeParseBodyAsObject(body) {
     }
 }
 function setRuntimeJsonPathValue(body, path, value) {
-    const target = body && typeof body === "object" && !Array.isArray(body)
-        ? { ...body }
-        : {};
-    const segments = path.split(".").filter(Boolean);
+    const target = runtimeCloneJsonRecord(body);
+    const segments = runtimeSafeJsonPathSegments(path);
     if (!segments.length) {
         return target;
     }
     let current = target;
     for (let i = 0; i < segments.length - 1; i += 1) {
         const segment = segments[i];
-        const next = current[segment];
+        const next = runtimeReadOwnJsonProperty(current, segment);
+        let child;
         if (!next || typeof next !== "object" || Array.isArray(next)) {
-            current[segment] = {};
+            child = {};
+        }
+        else {
+            child = runtimeCloneJsonRecord(next);
         }
-        current = current[segment];
+        runtimeDefineJsonProperty(current, segment, child);
+        current = child;
+    }
+    runtimeDefineJsonProperty(current, segments[segments.length - 1], value);
+    return target;
+}
+const FORBIDDEN_RUNTIME_JSON_PATH_SEGMENTS = new Set(["__proto__", "prototype", "constructor"]);
+function runtimeSafeJsonPathSegments(path) {
+    const segments = path.split(".").filter(Boolean);
+    const forbidden = segments.find((segment) => FORBIDDEN_RUNTIME_JSON_PATH_SEGMENTS.has(segment));
+    if (forbidden) {
+        throw new Error(`Tracking selector contains forbidden JSON path segment '${forbidden}'.`);
+    }
+    return segments;
+}
+function runtimeDefineJsonProperty(target, key, value) {
+    Object.defineProperty(target, key, {
+        configurable: true,
+        enumerable: true,
+        value,
+        writable: true
+    });
+}
+function runtimeReadOwnJsonProperty(target, key) {
+    const descriptor = Object.getOwnPropertyDescriptor(target, key);
+    return descriptor && "value" in descriptor ? descriptor.value : undefined;
+}
+function runtimeCloneJsonRecord(value) {
+    const target = {};
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return target;
+    }
+    for (const [key, entry] of Object.entries(value)) {
+        runtimeDefineJsonProperty(target, key, entry);
     }
-    current[segments[segments.length - 1]] = value;
     return target;
 }
 function asTrackingRecord(value) {
@@ -9476,8 +9570,8 @@ function asTrackingRecord(value) {
 }
 function pickTrackingValue(source, ...keys) {
     for (const key of keys) {
-        if (key in source) {
-            return source[key];
+        if (Object.prototype.hasOwnProperty.call(source, key)) {
+            return runtimeReadOwnJsonProperty(source, key);
         }
     }
     return undefined;
@@ -9731,26 +9825,10 @@ function createDefaultLogger(logFilePath) {
 function wrapLoggerWithMinimumLevel(baseLogger, minimumLogLevel) {
     const threshold = logLevelOrder(minimumLogLevel);
     return {
-        debug: (message) => {
-            if (threshold <= 0) {
-                baseLogger.debug(message);
-            }
-        },
-        info: (message) => {
-            if (threshold <= 1) {
-                baseLogger.info(message);
-            }
-        },
-        warn: (message) => {
-            if (threshold <= 2) {
-                baseLogger.warn(message);
-            }
-        },
-        error: (message) => {
-            if (threshold <= 3) {
-                baseLogger.error(message);
-            }
-        }
+        debug: (message) => threshold <= 0 ? baseLogger.debug(message) : undefined,
+        info: (message) => threshold <= 1 ? baseLogger.info(message) : undefined,
+        warn: (message) => threshold <= 2 ? baseLogger.warn(message) : undefined,
+        error: (message) => threshold <= 3 ? baseLogger.error(message) : undefined
     };
 }
 function formatDefaultLoggerLine(level, message) {
@@ -11264,6 +11342,7 @@ export const __loadstrikeTestExports = {
     parseStrictBooleanToken,
     percentile,
     pickOptionalTrackingSelectorString,
+    pickTrackingValue,
     pickTrackingNumber,
     produceOrConsumeTrackingPayload,
     readConfiguredSinkName,

package/dist/esm/sink-retry-policy.js

@@ -0,0 +1,44 @@
+export const DEFAULT_SINK_RETRY_COUNT = 3;
+export const DEFAULT_SINK_RETRY_BACKOFF_MS = 250;
+export const MAXIMUM_SINK_RETRY_COUNT = 100;
+export const MAXIMUM_TIMER_DELAY_MS = 2147483647;
+export function normalizeSinkRetryCount(value) {
+    const resolved = value === undefined ? DEFAULT_SINK_RETRY_COUNT : value;
+    if (!Number.isFinite(resolved) || resolved <= 0) {
+        return 0;
+    }
+    return Math.min(Math.trunc(resolved), MAXIMUM_SINK_RETRY_COUNT);
+}
+export function normalizeSinkRetryBackoffMs(value) {
+    const resolved = value === undefined ? DEFAULT_SINK_RETRY_BACKOFF_MS : value;
+    if (!Number.isFinite(resolved) || resolved <= 0) {
+        return 0;
+    }
+    return Math.min(Math.trunc(resolved), MAXIMUM_TIMER_DELAY_MS);
+}
+export function sinkRetryDelayMs(baseDelayMs, retryNumber) {
+    if (baseDelayMs <= 0) {
+        return 0;
+    }
+    let delayMs = Math.min(Math.trunc(baseDelayMs), MAXIMUM_TIMER_DELAY_MS);
+    const doublings = Math.max(Math.trunc(retryNumber) - 1, 0);
+    for (let index = 0; index < doublings; index += 1) {
+        if (delayMs >= Math.ceil(MAXIMUM_TIMER_DELAY_MS / 2)) {
+            return MAXIMUM_TIMER_DELAY_MS;
+        }
+        delayMs *= 2;
+    }
+    return delayMs;
+}
+export async function waitForSinkRetryDelay(delayMs) {
+    if (delayMs <= 0) {
+        await cooperativeSinkRetryYield();
+        return;
+    }
+    await new Promise((resolve) => {
+        setTimeout(resolve, Math.min(delayMs, MAXIMUM_TIMER_DELAY_MS));
+    });
+}
+export async function cooperativeSinkRetryYield() {
+    await new Promise((resolve) => setImmediate(resolve));
+}