@loadstrike/loadstrike-sdk 1.0.30201 → 1.0.31001

package/dist/cjs/iteration-observations.js

@@ -0,0 +1,884 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IterationObservationReporter = exports.ProcessIterationObservationBuffer = exports.DEFAULT_ITERATION_OBSERVATION_SETTINGS = exports.ITERATION_OBSERVATION_STREAM_COMPLETION_SCHEMA_VERSION = exports.ITERATION_OBSERVATION_BATCH_SCHEMA_VERSION = exports.ITERATION_OBSERVATION_SCHEMA_VERSION = void 0;
+exports.utcNowNs = utcNowNs;
+exports.deriveIterationId = deriveIterationId;
+exports.deriveObservationId = deriveObservationId;
+exports.deriveIterationBatchId = deriveIterationBatchId;
+exports.createIterationStepObservation = createIterationStepObservation;
+exports.createIterationObservation = createIterationObservation;
+exports.serializeIterationObservationBatchGzipJson = serializeIterationObservationBatchGzipJson;
+exports.validateIterationObservationSettings = validateIterationObservationSettings;
+const node_crypto_1 = require("node:crypto");
+const node_perf_hooks_1 = require("node:perf_hooks");
+const node_zlib_1 = require("node:zlib");
+const iteration_observation_diagnostics_js_1 = require("./iteration-observation-diagnostics.js");
+const sink_retry_policy_js_1 = require("./sink-retry-policy.js");
+exports.ITERATION_OBSERVATION_SCHEMA_VERSION = "loadstrike.iteration-observation/1";
+exports.ITERATION_OBSERVATION_BATCH_SCHEMA_VERSION = "loadstrike.iteration-batch/1";
+exports.ITERATION_OBSERVATION_STREAM_COMPLETION_SCHEMA_VERSION = "loadstrike.iteration-stream-completion/1";
+exports.DEFAULT_ITERATION_OBSERVATION_SETTINGS = Object.freeze({
+    flushIntervalMs: 5000,
+    maxBufferBytes: 256 * 1024 * 1024,
+    maxObservationsPerBatch: 50000,
+    maxBatchBytes: 8 * 1024 * 1024,
+    sinkQueueDepth: 4,
+    sinkParallelism: 2,
+    drainTimeoutMs: 30000
+});
+const BATCH_ENVELOPE_RESERVED_BYTES = 2048;
+const defaultClock = {
+    nowUtcNs: utcNowNs,
+    setInterval(callback, intervalMs) {
+        const timer = setInterval(callback, intervalMs);
+        timer.unref?.();
+        return timer;
+    },
+    clearInterval(handle) {
+        clearInterval(handle);
+    }
+};
+function utcNowNs() {
+    return BigInt(Date.now()) * 1000000n;
+}
+function deriveIterationId(runId, resultOwnerId, scenarioName, simulationIndex, globalOrdinal64) {
+    return derivePrefixedSha256("lsi_", `${runId}\n${resultOwnerId}\n${scenarioName}\n${Math.trunc(simulationIndex)}\n${decimalString(globalOrdinal64)}`);
+}
+function deriveObservationId(iterationId, attemptIndex) {
+    return derivePrefixedSha256("lso_", `${iterationId}\n${Math.max(Math.trunc(attemptIndex), 0)}`);
+}
+function deriveIterationBatchId(runId, resultOwnerId, batchSequence64) {
+    return derivePrefixedSha256("lsb_", `${runId}\n${resultOwnerId}\n${decimalString(batchSequence64)}`);
+}
+function createIterationStepObservation(input) {
+    return {
+        stepName: String(input.stepName),
+        sortIndex: Math.max(Math.trunc(input.sortIndex), 0),
+        startedUtcNs: nonNegativeDecimalString(input.startedUtcNs),
+        completedUtcNs: nonNegativeDecimalString(input.completedUtcNs),
+        observedLatencyUs64: nonNegativeDecimalString(input.observedLatencyUs64),
+        reportedLatencyUs64: nonNegativeDecimalString(input.reportedLatencyUs64),
+        isSuccess: Boolean(input.isSuccess),
+        statusCode: truncateUtf8(String(input.statusCode ?? ""), 64),
+        sizeBytes64: nonNegativeDecimalString(input.sizeBytes64)
+    };
+}
+function createIterationObservation(input) {
+    const globalOrdinal64 = nonNegativeDecimalString(input.globalOrdinal64);
+    const globalSecondaryOrdinal64 = nonNegativeDecimalString(input.globalSecondaryOrdinal64 ?? 0);
+    const simulationIndex = Math.trunc(input.simulationIndex);
+    const attemptIndex = Math.max(Math.trunc(input.attemptIndex), 0);
+    const iterationId = input.iterationId ?? deriveIterationId(String(input.runId), String(input.resultOwnerId), String(input.scenarioName), simulationIndex, globalOrdinal64);
+    if (!iterationId) {
+        throw new Error("Iteration identity must not be empty.");
+    }
+    return {
+        schemaVersion: exports.ITERATION_OBSERVATION_SCHEMA_VERSION,
+        observationId: deriveObservationId(iterationId, attemptIndex),
+        runId: String(input.runId),
+        sessionId: String(input.sessionId),
+        resultOwnerId: String(input.resultOwnerId),
+        processGroup: Math.max(Math.trunc(input.processGroup), 0),
+        scenarioName: String(input.scenarioName),
+        scenarioIndex: Math.max(Math.trunc(input.scenarioIndex), 0),
+        simulationIndex,
+        simulationKind: String(input.simulationKind || "SingleInvocation"),
+        phase: input.phase === "warmup" ? "warmup" : "bombing",
+        globalOrdinal64,
+        globalSecondaryOrdinal64,
+        shardIndex: Math.max(Math.trunc(input.shardIndex), 0),
+        shardCount: Math.max(Math.trunc(input.shardCount), 1),
+        iterationId,
+        attemptIndex,
+        isFinalAttempt: Boolean(input.isFinalAttempt),
+        startedUtcNs: nonNegativeDecimalString(input.startedUtcNs),
+        completedUtcNs: nonNegativeDecimalString(input.completedUtcNs),
+        observedLatencyUs64: nonNegativeDecimalString(input.observedLatencyUs64),
+        reportedLatencyUs64: nonNegativeDecimalString(input.reportedLatencyUs64),
+        isSuccess: Boolean(input.isSuccess),
+        statusCode: truncateUtf8(String(input.statusCode ?? ""), 64),
+        sizeBytes64: nonNegativeDecimalString(input.sizeBytes64),
+        steps: input.steps.map((step) => createIterationStepObservation(step))
+    };
+}
+function serializeIterationObservationBatchGzipJson(value) {
+    return (0, node_zlib_1.gzipSync)(Buffer.from(JSON.stringify(value), "utf8"));
+}
+class ProcessIterationObservationBuffer {
+    constructor() {
+        this.streams = new Map();
+        this.streamLimits = new Map();
+        this.totalBytes = 0;
+    }
+    get byteLength() {
+        return this.totalBytes;
+    }
+    register(streamKey, maxBufferBytes) {
+        this.streamLimits.set(streamKey, maxBufferBytes);
+    }
+    offer(streamKey, observation, maxBufferBytes, encodedBytes) {
+        this.streamLimits.set(streamKey, maxBufferBytes);
+        const processLimit = this.effectiveLimit(maxBufferBytes);
+        const jsonBytes = encodedBytes ?? Buffer.byteLength(JSON.stringify(observation), "utf8");
+        if (jsonBytes > processLimit || this.totalBytes + jsonBytes > processLimit) {
+            return false;
+        }
+        const stream = this.streams.get(streamKey) ?? [];
+        stream.push({ observation, jsonBytes });
+        this.streams.set(streamKey, stream);
+        this.totalBytes += jsonBytes;
+        return true;
+    }
+    drain(streamKey) {
+        const stream = this.streams.get(streamKey) ?? [];
+        this.streams.delete(streamKey);
+        for (const item of stream) {
+            this.totalBytes -= item.jsonBytes;
+        }
+        this.totalBytes = Math.max(this.totalBytes, 0);
+        return stream;
+    }
+    release(streamKey) {
+        this.drain(streamKey);
+    }
+    unregister(streamKey) {
+        this.drain(streamKey);
+        this.streamLimits.delete(streamKey);
+    }
+    effectiveLimit(fallback) {
+        let limit = fallback;
+        for (const value of this.streamLimits.values()) {
+            limit = Math.min(limit, value);
+        }
+        return limit;
+    }
+}
+exports.ProcessIterationObservationBuffer = ProcessIterationObservationBuffer;
+const processIterationObservationBuffer = new ProcessIterationObservationBuffer();
+class SinkDispatcher {
+    constructor(target, depth, parallelism, retryCount, retryBackoffMs, logger, runId, resultOwnerId, onDropped) {
+        this.target = target;
+        this.depth = depth;
+        this.parallelism = parallelism;
+        this.retryCount = retryCount;
+        this.retryBackoffMs = retryBackoffMs;
+        this.logger = logger;
+        this.runId = runId;
+        this.resultOwnerId = resultOwnerId;
+        this.onDropped = onDropped;
+        this.queue = [];
+        this.active = new Set();
+        this.idleWaiters = new Set();
+        this.timedOut = false;
+        this.delivered = 0n;
+        this.dropped = 0n;
+        this.drainDeadlineMs = null;
+        this.pendingRetryDelays = new Set();
+    }
+    get deliveredCount() {
+        return this.delivered;
+    }
+    get droppedCount() {
+        return this.dropped;
+    }
+    get canPublishCompletion() {
+        return !this.timedOut && this.queue.length === 0 && this.active.size === 0;
+    }
+    enqueue(batch) {
+        if (this.timedOut || this.queue.length >= this.depth) {
+            this.drop("observation_sink_overflow", batch.observations);
+            return false;
+        }
+        this.queue.push({ batch, count: BigInt(batch.observations.length) });
+        this.pump();
+        return true;
+    }
+    async waitForIdle(timeoutMs) {
+        if (this.isIdle()) {
+            return true;
+        }
+        let timer;
+        let waiter;
+        const idle = new Promise((resolve) => {
+            waiter = () => {
+                if (timer)
+                    clearTimeout(timer);
+                this.idleWaiters.delete(waiter);
+                resolve(true);
+            };
+            this.idleWaiters.add(waiter);
+        });
+        const timeout = new Promise((resolve) => {
+            timer = setTimeout(() => resolve(false), Math.max(timeoutMs, 1));
+        });
+        const completed = await Promise.race([idle, timeout]);
+        if (timer) {
+            clearTimeout(timer);
+        }
+        if (!completed) {
+            if (waiter)
+                this.idleWaiters.delete(waiter);
+            this.timeoutOutstanding();
+        }
+        return completed;
+    }
+    beginDrain(deadlineMs) {
+        this.drainDeadlineMs = deadlineMs;
+        const delaysThatCannotFit = Array.from(this.pendingRetryDelays)
+            .filter((pending) => pending.dueAtMs >= deadlineMs);
+        if (delaysThatCannotFit.length > 0) {
+            for (const pending of delaysThatCannotFit) {
+                pending.cancel();
+            }
+            this.timeoutOutstanding();
+        }
+    }
+    pump() {
+        while (!this.timedOut && this.active.size < this.parallelism && this.queue.length > 0) {
+            const item = this.queue.shift();
+            this.active.add(item);
+            Promise.resolve()
+                .then(() => this.deliver(item))
+                .then((outcome) => {
+                if (outcome === "success" && !this.timedOut) {
+                    this.delivered += item.count;
+                }
+                else if (outcome === "exhausted" && !this.timedOut) {
+                    this.drop("observation_sink_delivery_failed", item.batch.observations);
+                }
+            })
+                .catch(() => {
+                if (!this.timedOut) {
+                    this.drop("observation_sink_delivery_failed", item.batch.observations);
+                }
+            })
+                .finally(() => {
+                this.active.delete(item);
+                this.pump();
+                this.notifyIdle();
+            });
+        }
+        this.notifyIdle();
+    }
+    async deliver(item) {
+        const maximumAttempts = this.retryCount + 1;
+        for (let attempt = 1; attempt <= maximumAttempts; attempt += 1) {
+            if (this.timedOut || this.deadlineExpired()) {
+                return "deadline";
+            }
+            try {
+                await this.target.saveIterationBatch(item.batch);
+                if (attempt > 1) {
+                    (0, iteration_observation_diagnostics_js_1.logIterationObservationRecovery)(this.logger, {
+                        ...this.diagnosticContext(item, attempt, maximumAttempts),
+                        nextDelayMs: 0
+                    });
+                }
+                return "success";
+            }
+            catch (error) {
+                const exhausted = attempt >= maximumAttempts;
+                const nextDelayMs = exhausted
+                    ? 0
+                    : (0, sink_retry_policy_js_1.sinkRetryDelayMs)(this.retryBackoffMs, attempt);
+                (0, iteration_observation_diagnostics_js_1.logIterationObservationFailure)(this.logger, exhausted ? "error" : "warn", {
+                    ...this.diagnosticContext(item, attempt, maximumAttempts),
+                    nextDelayMs
+                }, error);
+                if (exhausted) {
+                    return "exhausted";
+                }
+                if (!await this.waitForRetry(attempt)) {
+                    this.timeoutOutstanding();
+                    return "deadline";
+                }
+            }
+        }
+        return "exhausted";
+    }
+    diagnosticContext(item, attempt, maximumAttempts) {
+        return {
+            sinkName: this.target.name,
+            operation: "observation-batch",
+            runId: this.runId,
+            resultOwnerId: this.resultOwnerId,
+            batchId: item.batch.batchId,
+            batchSequence64: item.batch.batchSequence64,
+            observationCount: Number(item.count),
+            attempt,
+            maximumAttempts
+        };
+    }
+    async waitForRetry(retryNumber) {
+        const delayMs = (0, sink_retry_policy_js_1.sinkRetryDelayMs)(this.retryBackoffMs, retryNumber);
+        const deadline = this.drainDeadlineMs;
+        if (deadline !== null && !delayCanFit(delayMs, deadline)) {
+            return false;
+        }
+        if (delayMs === 0) {
+            await (0, sink_retry_policy_js_1.cooperativeSinkRetryYield)();
+            return !this.timedOut && !this.deadlineExpired();
+        }
+        return await new Promise((resolve) => {
+            let settled = false;
+            let pending;
+            const finish = (completed) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                this.pendingRetryDelays.delete(pending);
+                resolve(completed && !this.timedOut && !this.deadlineExpired());
+            };
+            const timer = setTimeout(() => finish(true), delayMs);
+            const cancel = () => finish(false);
+            pending = {
+                dueAtMs: node_perf_hooks_1.performance.now() + delayMs,
+                cancel
+            };
+            this.pendingRetryDelays.add(pending);
+        });
+    }
+    deadlineExpired() {
+        return this.drainDeadlineMs !== null && node_perf_hooks_1.performance.now() >= this.drainDeadlineMs;
+    }
+    timeoutOutstanding() {
+        if (this.timedOut || this.isIdle())
+            return;
+        this.timedOut = true;
+        for (const pending of Array.from(this.pendingRetryDelays)) {
+            pending.cancel();
+        }
+        const queued = [...this.queue];
+        const active = [...this.active];
+        this.queue.length = 0;
+        for (const item of queued) {
+            this.drop("observation_sink_drain_timeout", item.batch.observations);
+        }
+        for (const item of active) {
+            // Active sink I/O cannot be cancelled safely. Warn, but settle its actual
+            // outcome before the completion marker and do not claim it was dropped.
+            this.onDropped("observation_sink_drain_timeout", this.target.name, item.batch.observations);
+        }
+        this.notifyIdle();
+    }
+    drop(code, observations) {
+        this.dropped += BigInt(observations.length);
+        this.onDropped(code, this.target.name, observations);
+    }
+    isIdle() {
+        return this.queue.length === 0 && this.active.size === 0;
+    }
+    notifyIdle(force = false) {
+        if (!force && !this.isIdle())
+            return;
+        for (const waiter of this.idleWaiters) {
+            waiter();
+        }
+        this.idleWaiters.clear();
+    }
+}
+class IterationObservationReporter {
+    constructor(options) {
+        this.options = options;
+        this.streamKey = Symbol("loadstrike-iteration-observations");
+        this.warnings = new Map();
+        this.sequence = 0n;
+        this.captured = 0n;
+        this.droppedBuffer = 0n;
+        this.unsupportedSinkCount = 0;
+        this.sealed = false;
+        this.finalResult = null;
+        this.expectedResultOwnerCount64 = positiveCanonicalDecimalString(options.expectedResultOwnerCount64 ?? "1", "Expected result owner count");
+        const requestedSettings = {
+            ...exports.DEFAULT_ITERATION_OBSERVATION_SETTINGS,
+            ...(options.settings ?? {})
+        };
+        const hasPortalSink = options.sinks.some((sink) => sink.iterationObservationPortalSink === true);
+        this.settings = hasPortalSink
+            ? {
+                ...requestedSettings,
+                maxObservationsPerBatch: Math.min(requestedSettings.maxObservationsPerBatch, 50000),
+                maxBatchBytes: Math.min(requestedSettings.maxBatchBytes, 8 * 1024 * 1024)
+            }
+            : requestedSettings;
+        if (!options.skipValidation) {
+            validateIterationObservationSettings(this.settings);
+        }
+        this.buffer = options.buffer ?? processIterationObservationBuffer;
+        this.clock = options.clock ?? defaultClock;
+        this.batchEnvelopeBytes = maximumIterationObservationBatchEnvelopeBytes(options);
+        this.dispatchers = options.sinks
+            .filter((sink) => {
+            if (typeof sink.saveIterationBatch === "function") {
+                if (sink.iterationObservationShapeLimited === true) {
+                    this.recordWarning("sink_iteration_shape_limited", sink.name, "", -1, 1n, "The reporting sink stores a metric projection rather than the complete raw iteration shape.");
+                }
+                return true;
+            }
+            this.unsupportedSinkCount += 1;
+            this.recordWarning("sink_iteration_batches_unsupported", sink.name, "", -1, 1n, "The reporting sink does not support raw iteration batches.");
+            return false;
+        })
+            .map((target) => new SinkDispatcher(target, this.settings.sinkQueueDepth, this.settings.sinkParallelism, (0, sink_retry_policy_js_1.normalizeSinkRetryCount)(options.sinkRetryCount), (0, sink_retry_policy_js_1.normalizeSinkRetryBackoffMs)(options.sinkRetryBackoffMs), options.logger, options.runId, options.resultOwnerId, (code, sinkName, observations) => this.recordSinkDrop(code, sinkName, observations)));
+        if (this.dispatchers.length > 0) {
+            this.buffer.register(this.streamKey, this.settings.maxBufferBytes);
+        }
+        this.timer = this.clock.setInterval(() => this.flushNow(), this.settings.flushIntervalMs);
+    }
+    get enabled() {
+        return this.dispatchers.length > 0 && !this.sealed;
+    }
+    capture(observation) {
+        if (!this.enabled) {
+            return false;
+        }
+        const singleObservationBudget = Math.max(this.settings.maxBatchBytes - this.batchEnvelopeBytes, 0);
+        const upperBound = jsonUtf8SizeUpperBound(observation);
+        const encodedBytes = upperBound >= Math.floor(singleObservationBudget * 0.9)
+            ? Buffer.byteLength(JSON.stringify(observation), "utf8")
+            : upperBound;
+        if (encodedBytes > singleObservationBudget) {
+            this.droppedBuffer += 1n;
+            this.recordWarning("observation_batch_oversize", undefined, observation.scenarioName, observation.simulationIndex, 1n, "The canonical raw observation cannot fit within one configured batch.", observation.scenarioIndex, observation.simulationKind);
+            return false;
+        }
+        const immutable = deepFreezeObservation(observation);
+        if (this.buffer.offer(this.streamKey, immutable, this.settings.maxBufferBytes, encodedBytes)) {
+            this.captured += 1n;
+            return true;
+        }
+        this.droppedBuffer += 1n;
+        this.recordWarning("observation_buffer_overflow", undefined, immutable.scenarioName, immutable.simulationIndex, 1n, "The process-wide raw observation buffer was full.", immutable.scenarioIndex, immutable.simulationKind);
+        return false;
+    }
+    flushNow() {
+        if (this.dispatchers.length === 0) {
+            this.buffer.release(this.streamKey);
+            return;
+        }
+        const observations = this.buffer.drain(this.streamKey);
+        if (!observations.length) {
+            return;
+        }
+        const chunks = splitBufferedObservations(observations, this.settings.maxObservationsPerBatch, this.settings.maxBatchBytes, Math.max(BATCH_ENVELOPE_RESERVED_BYTES, this.batchEnvelopeBytes));
+        for (const chunk of chunks) {
+            const batch = this.createBatch(chunk.map((item) => item.observation), this.droppedBuffer);
+            for (const dispatcher of this.dispatchers) {
+                dispatcher.enqueue(batch);
+            }
+        }
+    }
+    async sealAndDrain() {
+        if (this.finalResult) {
+            return { ...this.finalResult };
+        }
+        this.sealed = true;
+        this.clock.clearInterval(this.timer);
+        this.flushNow();
+        const started = node_perf_hooks_1.performance.now();
+        const deadline = started + this.settings.drainTimeoutMs;
+        for (const dispatcher of this.dispatchers) {
+            dispatcher.beginDrain(deadline);
+        }
+        const drainResults = new Array(this.dispatchers.length).fill(false);
+        const completionResults = await Promise.all(this.dispatchers.map(async (dispatcher, index) => {
+            const drained = await dispatcher.waitForIdle(Math.max(deadline - node_perf_hooks_1.performance.now(), 1));
+            drainResults[index] = drained;
+            if (!dispatcher.canPublishCompletion) {
+                return false;
+            }
+            const complete = dispatcher.target.completeIterationObservationStream;
+            if (!complete) {
+                this.recordWarning("sink_iteration_completion_unsupported", dispatcher.target.name, "", -1, 1n, "The reporting sink does not support raw iteration stream completion markers.");
+                return false;
+            }
+            const sinkReportingComplete = drained
+                && this.droppedBuffer === 0n
+                && dispatcher.droppedCount === 0n
+                && dispatcher.deliveredCount === this.captured;
+            const completion = this.createCompletion(dispatcher, sinkReportingComplete);
+            const outcome = await this.completeWithRetries(complete, completion, deadline, dispatcher.target.name);
+            if (outcome === "success") {
+                return true;
+            }
+            this.recordWarning("observation_stream_completion_failed", dispatcher.target.name, "", -1, 1n, outcome === "timeout"
+                ? "The reporting sink did not complete its observation stream marker before shutdown."
+                : "The reporting sink did not accept the observation stream completion marker.");
+            return false;
+        }));
+        const reportingComplete = this.dispatchers.length === 0
+            ? this.unsupportedSinkCount === 0
+            : this.unsupportedSinkCount === 0
+                && drainResults.every(Boolean)
+                && completionResults.every(Boolean)
+                && this.droppedBuffer === 0n
+                && this.dispatchers.every((dispatcher) => dispatcher.droppedCount === 0n)
+                && this.dispatchers.every((dispatcher) => dispatcher.deliveredCount === this.captured);
+        const delivered = this.dispatchers.length
+            ? this.dispatchers.reduce((minimum, dispatcher) => dispatcher.deliveredCount < minimum
+                ? dispatcher.deliveredCount
+                : minimum, this.dispatchers[0].deliveredCount)
+            : 0n;
+        const droppedSink = this.dispatchers.reduce((maximum, dispatcher) => dispatcher.droppedCount > maximum
+            ? dispatcher.droppedCount
+            : maximum, 0n);
+        this.buffer.unregister(this.streamKey);
+        this.finalResult = {
+            lastBatchSequence64: this.sequence === 0n ? "-1" : (this.sequence - 1n).toString(),
+            capturedCount64: this.captured.toString(),
+            deliveredCount64: delivered.toString(),
+            droppedBufferCount64: this.droppedBuffer.toString(),
+            droppedSinkCount64: droppedSink.toString(),
+            reportingComplete
+        };
+        return { ...this.finalResult };
+    }
+    async completeWithRetries(complete, completion, deadline, sinkName) {
+        const retryCount = (0, sink_retry_policy_js_1.normalizeSinkRetryCount)(this.options.sinkRetryCount);
+        const backoffMs = (0, sink_retry_policy_js_1.normalizeSinkRetryBackoffMs)(this.options.sinkRetryBackoffMs);
+        const maximumAttempts = retryCount + 1;
+        for (let attempt = 1; attempt <= maximumAttempts; attempt += 1) {
+            const remainingMs = deadline - node_perf_hooks_1.performance.now();
+            if (remainingMs <= 0) {
+                return "timeout";
+            }
+            let timer;
+            const outcome = await Promise.race([
+                Promise.resolve()
+                    .then(() => complete(completion))
+                    .then(() => ({ kind: "success" }), (error) => ({ kind: "failed", error })),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve({ kind: "timeout" }), Math.max(remainingMs, 1));
+                })
+            ]);
+            if (timer) {
+                clearTimeout(timer);
+            }
+            if (outcome.kind === "success") {
+                if (attempt > 1) {
+                    (0, iteration_observation_diagnostics_js_1.logIterationObservationRecovery)(this.options.logger, {
+                        sinkName,
+                        operation: "observation-stream-completion",
+                        runId: this.options.runId,
+                        resultOwnerId: this.options.resultOwnerId,
+                        observationCount: 0,
+                        attempt,
+                        maximumAttempts,
+                        nextDelayMs: 0
+                    });
+                }
+                return "success";
+            }
+            if (outcome.kind === "timeout") {
+                return "timeout";
+            }
+            const exhausted = attempt >= maximumAttempts;
+            const nextDelayMs = exhausted ? 0 : (0, sink_retry_policy_js_1.sinkRetryDelayMs)(backoffMs, attempt);
+            (0, iteration_observation_diagnostics_js_1.logIterationObservationFailure)(this.options.logger, exhausted ? "error" : "warn", {
+                sinkName,
+                operation: "observation-stream-completion",
+                runId: this.options.runId,
+                resultOwnerId: this.options.resultOwnerId,
+                observationCount: 0,
+                attempt,
+                maximumAttempts,
+                nextDelayMs
+            }, outcome.error);
+            if (exhausted) {
+                return "failed";
+            }
+            const delayMs = nextDelayMs;
+            if (!delayCanFit(delayMs, deadline)) {
+                return "timeout";
+            }
+            if (delayMs === 0) {
+                await (0, sink_retry_policy_js_1.cooperativeSinkRetryYield)();
+            }
+            else {
+                await (0, sink_retry_policy_js_1.waitForSinkRetryDelay)(delayMs);
+            }
+            if (node_perf_hooks_1.performance.now() >= deadline) {
+                return "timeout";
+            }
+        }
+        return "failed";
+    }
+    buildWarnings() {
+        return Array.from(this.warnings.values())
+            .sort((left, right) => left.code.localeCompare(right.code)
+            || String(left.sinkName ?? "").localeCompare(String(right.sinkName ?? ""))
+            || left.scenarioName.localeCompare(right.scenarioName)
+            || left.simulationIndex - right.simulationIndex)
+            .map((warning) => ({
+            code: warning.code,
+            ...(warning.sinkName ? { sinkName: warning.sinkName } : {}),
+            scenarioName: warning.scenarioName,
+            scenarioIndex: warning.scenarioIndex,
+            simulationIndex: warning.simulationIndex,
+            simulationKind: warning.simulationKind,
+            count64: warning.count.toString(),
+            message: warning.message,
+            firstObservedUtcNs: warning.firstObservedUtcNs.toString(),
+            lastObservedUtcNs: warning.lastObservedUtcNs.toString()
+        }));
+    }
+    createBatch(observations, droppedBeforeBatch) {
+        const sequence = this.sequence;
+        this.sequence += 1n;
+        const firstObservationUtcNs = observations.reduce((minimum, observation) => {
+            const value = BigInt(observation.startedUtcNs);
+            return value < minimum ? value : minimum;
+        }, BigInt(observations[0].startedUtcNs));
+        const lastObservationUtcNs = observations.reduce((maximum, observation) => {
+            const value = BigInt(observation.completedUtcNs);
+            return value > maximum ? value : maximum;
+        }, BigInt(observations[0].completedUtcNs));
+        return deepFreezeBatch({
+            schemaVersion: exports.ITERATION_OBSERVATION_BATCH_SCHEMA_VERSION,
+            batchId: deriveIterationBatchId(this.options.runId, this.options.resultOwnerId, sequence),
+            runId: this.options.runId,
+            sessionId: this.options.sessionId,
+            resultOwnerId: this.options.resultOwnerId,
+            processGroup: Math.max(Math.trunc(this.options.processGroup), 0),
+            batchSequence64: sequence.toString(),
+            createdUtcNs: this.clock.nowUtcNs().toString(),
+            firstObservationUtcNs: firstObservationUtcNs.toString(),
+            lastObservationUtcNs: lastObservationUtcNs.toString(),
+            observations,
+            capturedCount64: this.captured.toString(),
+            droppedBeforeBatch64: droppedBeforeBatch.toString(),
+            compression: "gzip-json"
+        });
+    }
+    createCompletion(dispatcher, reportingComplete) {
+        return Object.freeze({
+            schemaVersion: exports.ITERATION_OBSERVATION_STREAM_COMPLETION_SCHEMA_VERSION,
+            runId: this.options.runId,
+            sessionId: this.options.sessionId,
+            resultOwnerId: this.options.resultOwnerId,
+            expectedResultOwnerCount64: this.expectedResultOwnerCount64,
+            processGroup: Math.max(Math.trunc(this.options.processGroup), 0),
+            lastBatchSequence64: this.sequence === 0n ? "-1" : (this.sequence - 1n).toString(),
+            capturedCount64: this.captured.toString(),
+            deliveredCount64: dispatcher.deliveredCount.toString(),
+            droppedBufferCount64: this.droppedBuffer.toString(),
+            droppedSinkCount64: dispatcher.droppedCount.toString(),
+            reportingComplete,
+            completedUtcNs: this.clock.nowUtcNs().toString()
+        });
+    }
+    recordSinkDrop(code, sinkName, observations) {
+        const grouped = new Map();
+        for (const observation of observations) {
+            const key = `${observation.scenarioName}\n${observation.simulationIndex}`;
+            const group = grouped.get(key) ?? {
+                scenarioName: observation.scenarioName,
+                scenarioIndex: observation.scenarioIndex,
+                simulationIndex: observation.simulationIndex,
+                simulationKind: observation.simulationKind,
+                count: 0n
+            };
+            group.count += 1n;
+            grouped.set(key, group);
+        }
+        for (const group of grouped.values()) {
+            this.recordWarning(code, sinkName, group.scenarioName, group.simulationIndex, group.count, code === "observation_sink_overflow"
+                ? "The reporting sink raw observation queue was full."
+                : code === "observation_sink_drain_timeout"
+                    ? "The reporting sink raw observation queue did not drain before shutdown."
+                    : "The reporting sink failed to accept a raw observation batch.", group.scenarioIndex, group.simulationKind);
+        }
+    }
+    recordWarning(code, sinkName, scenarioName, simulationIndex, count, message, scenarioIndex = -1, simulationKind = "") {
+        if (count <= 0n)
+            return;
+        const key = `${code}\n${sinkName ?? ""}\n${scenarioIndex}\n${simulationIndex}\n${simulationKind}`;
+        const now = this.clock?.nowUtcNs?.() ?? utcNowNs();
+        const existing = this.warnings.get(key);
+        if (existing) {
+            existing.count += count;
+            existing.lastObservedUtcNs = now;
+            return;
+        }
+        this.warnings.set(key, {
+            code,
+            sinkName,
+            scenarioName,
+            scenarioIndex,
+            simulationIndex,
+            simulationKind,
+            count,
+            message,
+            firstObservedUtcNs: now,
+            lastObservedUtcNs: now
+        });
+    }
+}
+exports.IterationObservationReporter = IterationObservationReporter;
+function validateIterationObservationSettings(settings) {
+    validateIntegerRange(settings.flushIntervalMs, 1000, 60000, "Iteration observation flush interval");
+    validateIntegerRange(settings.maxBufferBytes, 16 * 1024 * 1024, 4 * 1024 * 1024 * 1024, "Iteration observation buffer bytes");
+    validateIntegerRange(settings.maxObservationsPerBatch, 100, 1000000, "Iteration observations per batch");
+    validateIntegerRange(settings.maxBatchBytes, 1024 * 1024, 64 * 1024 * 1024, "Iteration observation batch bytes");
+    validateIntegerRange(settings.sinkQueueDepth, 1, 64, "Iteration observation sink queue depth");
+    validateIntegerRange(settings.sinkParallelism, 1, 16, "Iteration observation sink parallelism");
+    validateIntegerRange(settings.drainTimeoutMs, 1000, 300000, "Iteration observation drain timeout");
+}
+function validateIntegerRange(value, minimum, maximum, name) {
+    if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+        throw new RangeError(`${name} must be an integer from ${minimum} through ${maximum}.`);
+    }
+}
+function splitBufferedObservations(observations, maxCount, maxBytes, envelopeBytes) {
+    const chunks = [];
+    let current = [];
+    let currentBytes = envelopeBytes;
+    for (const observation of observations) {
+        const projectedBytes = currentBytes + observation.jsonBytes + (current.length ? 1 : 0);
+        if (current.length > 0 && (current.length >= maxCount || projectedBytes > maxBytes)) {
+            chunks.push(current);
+            current = [];
+            currentBytes = envelopeBytes;
+        }
+        current.push(observation);
+        currentBytes += observation.jsonBytes + (current.length > 1 ? 1 : 0);
+    }
+    if (current.length) {
+        chunks.push(current);
+    }
+    return chunks;
+}
+function maximumIterationObservationBatchEnvelopeBytes(options) {
+    const maximumDecimal = "9223372036854775807";
+    const value = {
+        schemaVersion: exports.ITERATION_OBSERVATION_BATCH_SCHEMA_VERSION,
+        batchId: deriveIterationBatchId(options.runId, options.resultOwnerId, maximumDecimal),
+        runId: options.runId,
+        sessionId: options.sessionId,
+        resultOwnerId: options.resultOwnerId,
+        processGroup: Math.max(Math.trunc(options.processGroup), 0),
+        batchSequence64: maximumDecimal,
+        createdUtcNs: maximumDecimal,
+        firstObservationUtcNs: maximumDecimal,
+        lastObservationUtcNs: maximumDecimal,
+        observations: [],
+        capturedCount64: maximumDecimal,
+        droppedBeforeBatch64: maximumDecimal,
+        compression: "gzip-json"
+    };
+    return Buffer.byteLength(JSON.stringify(value), "utf8");
+}
+function jsonUtf8SizeUpperBound(value) {
+    if (value === null || value === undefined)
+        return 4;
+    if (typeof value === "string")
+        return 2 + escapedJsonStringUtf8Bytes(value);
+    if (typeof value === "boolean")
+        return value ? 4 : 5;
+    if (typeof value === "number")
+        return Number.isFinite(value) ? Buffer.byteLength(String(value), "utf8") : 4;
+    if (typeof value === "bigint")
+        return Buffer.byteLength(value.toString(), "utf8");
+    if (Array.isArray(value)) {
+        return 2 + value.reduce((total, item, index) => total + (index ? 1 : 0) + jsonUtf8SizeUpperBound(item), 0);
+    }
+    if (typeof value === "object") {
+        let total = 2;
+        let count = 0;
+        for (const [key, item] of Object.entries(value)) {
+            if (item === undefined || typeof item === "function" || typeof item === "symbol")
+                continue;
+            total += (count ? 1 : 0) + 2 + escapedJsonStringUtf8Bytes(key) + 1 + jsonUtf8SizeUpperBound(item);
+            count += 1;
+        }
+        return total;
+    }
+    return 4;
+}
+function escapedJsonStringUtf8Bytes(value) {
+    let bytes = 0;
+    for (let index = 0; index < value.length; index += 1) {
+        const code = value.charCodeAt(index);
+        if (code <= 0x1f)
+            bytes += 6;
+        else if (code === 0x22 || code === 0x5c)
+            bytes += 2;
+        else if (code <= 0x7f)
+            bytes += 1;
+        else if (code <= 0x7ff)
+            bytes += 2;
+        else if (code >= 0xd800 && code <= 0xdbff) {
+            const next = value.charCodeAt(index + 1);
+            if (next >= 0xdc00 && next <= 0xdfff) {
+                bytes += 4;
+                index += 1;
+            }
+            else
+                bytes += 6;
+        }
+        else if (code >= 0xdc00 && code <= 0xdfff)
+            bytes += 6;
+        else
+            bytes += 3;
+    }
+    return bytes;
+}
+function derivePrefixedSha256(prefix, material) {
+    return `${prefix}${(0, node_crypto_1.createHash)("sha256").update(material, "utf8").digest("hex").slice(0, 40)}`;
+}
+function decimalString(value) {
+    try {
+        return BigInt(String(value)).toString();
+    }
+    catch {
+        return "0";
+    }
+}
+function nonNegativeDecimalString(value) {
+    try {
+        const parsed = BigInt(String(value));
+        return (parsed < 0n ? 0n : parsed).toString();
+    }
+    catch {
+        return "0";
+    }
+}
+function positiveCanonicalDecimalString(value, label) {
+    if (typeof value === "number" && (!Number.isSafeInteger(value) || value < 1)) {
+        throw new RangeError(`${label} must be a canonical unsigned decimal greater than zero.`);
+    }
+    const text = String(value);
+    if (!/^[1-9][0-9]*$/.test(text)) {
+        throw new RangeError(`${label} must be a canonical unsigned decimal greater than zero.`);
+    }
+    return text;
+}
+function truncateUtf8(value, maxBytes) {
+    if (Buffer.byteLength(value, "utf8") <= maxBytes) {
+        return value;
+    }
+    let output = "";
+    for (const token of value) {
+        if (Buffer.byteLength(output + token, "utf8") > maxBytes) {
+            break;
+        }
+        output += token;
+    }
+    return output;
+}
+function delayCanFit(delayMs, deadlineMs) {
+    const remainingMs = deadlineMs - node_perf_hooks_1.performance.now();
+    return remainingMs > 0 && delayMs < remainingMs;
+}
+function deepFreezeObservation(observation) {
+    for (const step of observation.steps) {
+        Object.freeze(step);
+    }
+    Object.freeze(observation.steps);
+    return Object.freeze(observation);
+}
+function deepFreezeBatch(batch) {
+    Object.freeze(batch.observations);
+    return Object.freeze(batch);
+}