@loadstrike/loadstrike-sdk 1.0.30001 → 1.0.30401

package/dist/esm/sinks.js

@@ -1,9 +1,24 @@
 import { LoadStrikePluginData as LoadStrikePluginDataModel, LoadStrikePluginDataTable as LoadStrikePluginDataTableModel } from "./runtime.js";
 import { createHash, randomBytes } from "node:crypto";
-import { appendFileSync, mkdirSync } from "node:fs";
+import { appendFile, mkdir } from "node:fs/promises";
 import { dirname } from "node:path";
 import { createSocket } from "node:dgram";
+import { gzip } from "node:zlib";
 import { Pool } from "pg";
+import { serializeIterationObservationBatchGzipJson } from "./iteration-observations.js";
+const PORTAL_RUN_TOKEN_PROVIDER = Symbol.for("loadstrike.internal.portal-run-token-provider");
+function gzipJsonAsync(value) {
+    const json = Buffer.from(JSON.stringify(value), "utf8");
+    return new Promise((resolve, reject) => {
+        gzip(json, (error, compressed) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve(compressed);
+        });
+    });
+}
 const DEFAULT_INFLUX_CONFIGURATION_SECTION_PATH = "LoadStrike:ReportingSinks:InfluxDb";
 const DEFAULT_GRAFANA_LOKI_CONFIGURATION_SECTION_PATH = "LoadStrike:ReportingSinks:GrafanaLoki";
 const DEFAULT_TIMESCALEDB_CONFIGURATION_SECTION_PATH = "LoadStrike:ReportingSinks:TimescaleDb";
@@ -364,6 +379,28 @@ export class CompositeReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        for (const sink of this.sinks) {
+            const saveIterationBatch = sink.saveIterationBatch ?? sink.SaveIterationBatch;
+            if (saveIterationBatch) {
+                await saveIterationBatch.call(sink, batch);
+            }
+        }
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        for (const sink of this.sinks) {
+            const completeIterationObservationStream = sink.completeIterationObservationStream ?? sink.CompleteIterationObservationStream;
+            if (completeIterationObservationStream) {
+                await completeIterationObservationStream.call(sink, completion);
+            }
+        }
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     async stop() {
         for (const sink of this.sinks) {
             const stop = sink.stop ?? sink.Stop;
@@ -378,6 +415,7 @@ export class CompositeReportingSink {
 }
 export class PortalReportingSink {
     constructor(options = {}) {
+        this.iterationObservationPortalSink = true;
         this.sinkName = "portal";
         this.SinkName = "portal";
         this.licenseFeature = "extensions.reporting_sinks.portal";
@@ -385,6 +423,7 @@ export class PortalReportingSink {
         this.baseContext = null;
         this.session = null;
         this.runToken = "";
+        this.runTokenProvider = null;
         this.ingestUrl = "";
         this.optionsInput = { ...options };
         const source = asRecord(options);
@@ -401,7 +440,11 @@ export class PortalReportingSink {
         this.init(context, infraConfig);
     }
     start(session) {
-        this.runToken = String(session.runToken ?? session.RunToken ?? "").trim();
+        const internalProvider = session[PORTAL_RUN_TOKEN_PROVIDER];
+        this.runTokenProvider = typeof internalProvider === "function"
+            ? internalProvider
+            : () => String(session.runToken ?? session.RunToken ?? "").trim();
+        this.runToken = this.runTokenProvider();
         this.ingestUrl = String(session.portalReportingIngestUrl ?? session.PortalReportingIngestUrl ?? "").trim();
         if (!this.runToken || !this.ingestUrl) {
             throw new Error("PortalReportingSink requires a managed portal reporting session.");
@@ -431,8 +474,27 @@ export class PortalReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        const compressed = await gzipJsonAsync(batch);
+        await this.persistObservationPayload({
+            compressedObservationBatches: [{
+                    compression: "gzip-json",
+                    payloadBase64: compressed.toString("base64")
+                }]
+        });
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistObservationPayload({ observationStreamCompletions: [completion] });
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
+        this.runTokenProvider = null;
     }
     Stop() {
         this.stop();
@@ -461,12 +523,27 @@ export class PortalReportingSink {
             method: "POST",
             headers: { "Content-Type": "application/json" },
             body: JSON.stringify({
-                runToken: this.runToken,
-                events: events.map((event, index) => portalEventPayload(event, index))
+                runToken: this.currentRunToken(),
+                events: events.map((event, index) => portalEventPayload(removeSdkPercentileFields(event), index))
+            })
+        }, this.timeoutMs, "PortalReportingSink");
+        validatePortalIngestResponse(responseBody);
+    }
+    async persistObservationPayload(payload) {
+        const responseBody = await postWithTimeout(this.fetchImpl, this.ingestUrl, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                runToken: this.currentRunToken(),
+                ...payload
             })
         }, this.timeoutMs, "PortalReportingSink");
         validatePortalIngestResponse(responseBody);
     }
+    currentRunToken() {
+        const current = this.runTokenProvider?.() ?? "";
+        return current || this.runToken;
+    }
 }
 export function cloneReportingSinkForRun(sink) {
     if (sink instanceof PortalReportingSink) {
@@ -542,6 +619,18 @@ export class InfluxDbReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -651,6 +740,18 @@ export class GrafanaLokiReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -796,6 +897,18 @@ export class TimescaleDbReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     async stop() {
         this.session = null;
         if (this.pool) {
@@ -1071,6 +1184,18 @@ export class DatadogReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1193,6 +1318,18 @@ export class SplunkReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1295,6 +1432,18 @@ export class OtelCollectorReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1371,6 +1520,7 @@ class ExpandedEventReportingSink {
     constructor(sinkName, licenseFeature) {
         this.baseContext = null;
         this.session = null;
+        this.preservesRawIterationObservations = true;
         this.sinkName = sinkName;
         this.SinkName = sinkName;
         this.licenseFeature = licenseFeature;
@@ -1406,6 +1556,18 @@ class ExpandedEventReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1429,15 +1591,18 @@ class ExpandedEventReportingSink {
         return this.session;
     }
     buildPayload(events, staticTags = {}) {
+        const projectedEvents = this.preservesRawIterationObservations
+            ? events.map(removeSdkPercentileFields)
+            : events;
         return {
             sinkName: this.sinkName,
             runId: this.getSession().runId,
             sessionId: this.getSession().sessionId,
-            events: events.map((event) => ({
+            events: projectedEvents.map((event) => ({
                 ...event,
                 tags: { ...staticTags, ...event.tags }
             })),
-            metrics: createReportingSinkMetricPoints(this.sinkName, events, staticTags)
+            metrics: createReportingSinkMetricPoints(this.sinkName, projectedEvents, staticTags)
         };
     }
 }
@@ -1471,20 +1636,41 @@ class ExpandedHttpJsonReportingSink extends ExpandedEventReportingSink {
             body: JSON.stringify(this.buildPayload(events, this.staticTags))
         }, this.timeoutMs, this.constructor.name);
     }
+    async saveIterationBatch(batch) {
+        await this.persistCanonicalGzipPayload(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistCanonicalGzipPayload(completion);
+    }
+    async persistCanonicalGzipPayload(payload) {
+        const compressed = await gzipJsonAsync(payload);
+        await postWithTimeout(this.fetchImpl, `${trimTrailingSlashes(this.baseUrl)}${normalizePath(this.endpointPath)}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Content-Encoding": "gzip",
+                ...this.headers
+            },
+            body: Uint8Array.from(compressed).buffer
+        }, this.timeoutMs, this.constructor.name);
+    }
 }
 export class PrometheusRemoteWriteReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("prometheus-remote-write", "extensions.reporting_sinks.prometheus_remote_write", "/api/v1/write", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class CloudWatchReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("cloudwatch", "extensions.reporting_sinks.cloudwatch", "/loadstrike/events", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class DynatraceReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("dynatrace", "extensions.reporting_sinks.dynatrace", "/api/v2/logs/ingest", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class ElasticsearchReportingSink extends ExpandedHttpJsonReportingSink {
@@ -1500,11 +1686,25 @@ export class OpenSearchReportingSink extends ExpandedHttpJsonReportingSink {
 export class NewRelicReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("new-relic", "extensions.reporting_sinks.new_relic", "/log/v1", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class GenericWebhookReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("webhook", "extensions.reporting_sinks.webhook", "/loadstrike", options);
+        this.preservesRawIterationObservations = true;
+    }
+    async saveIterationBatch(batch) {
+        await this.persistCanonicalGzipPayload(batch);
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistCanonicalGzipPayload(completion);
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
     }
 }
 export class KafkaReportingSink extends ExpandedEventReportingSink {
@@ -1515,6 +1715,12 @@ export class KafkaReportingSink extends ExpandedEventReportingSink {
         this.staticTags = normalizeStringMap(optionRecord(source, "staticTags", "StaticTags"));
         this.publishAsync = pickRecordValue(source, "publishAsync", "PublishAsync");
     }
+    async saveIterationBatch(batch) {
+        await this.publishCanonicalValue(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.publishCanonicalValue(completion);
+    }
     async persistEvents(events) {
         if (!events.length) {
             return;
@@ -1524,10 +1730,17 @@ export class KafkaReportingSink extends ExpandedEventReportingSink {
         }
         await this.publishAsync(this.topic, JSON.stringify(this.buildPayload(events, this.staticTags)));
     }
+    async publishCanonicalValue(value) {
+        if (!this.publishAsync) {
+            throw new Error("KafkaReportingSink requires PublishAsync when a Kafka producer is not configured by the host application.");
+        }
+        await this.publishAsync(this.topic, JSON.stringify(value));
+    }
 }
 class StatsDReportingSinkBase extends ExpandedEventReportingSink {
     constructor(sinkName, licenseFeature, options = {}, dogStatsD = false) {
         super(sinkName, licenseFeature);
+        this.iterationObservationShapeLimited = true;
         const source = asRecord(options);
         this.prefix = optionString(source, "prefix", "Prefix").trim() || "loadstrike";
         this.host = optionString(source, "host", "Host").trim() || "127.0.0.1";
@@ -1536,6 +1749,47 @@ class StatsDReportingSinkBase extends ExpandedEventReportingSink {
         this.sendLineAsync = pickRecordValue(source, "sendLineAsync", "SendLineAsync");
         this.dogStatsD = dogStatsD;
     }
+    async saveIterationBatch(batch) {
+        for (const observation of batch.observations) {
+            const tags = {
+                run_id: batch.runId,
+                result_owner_id: batch.resultOwnerId,
+                scenario_name: observation.scenarioName,
+                phase: observation.phase,
+                outcome: observation.isSuccess ? "success" : "failure"
+            };
+            const occurredUtc = new Date();
+            await this.sendLine(this.formatPoint({
+                metricName: "iteration.reported_latency_us",
+                metricKind: "gauge",
+                occurredUtc,
+                value: Number(observation.reportedLatencyUs64),
+                unitOfMeasure: "us",
+                tags
+            }));
+            await this.sendLine(this.formatPoint({
+                metricName: "iteration.attempt",
+                metricKind: "count",
+                occurredUtc,
+                value: 1,
+                unitOfMeasure: "count",
+                tags
+            }));
+        }
+    }
+    async completeIterationObservationStream(completion) {
+        await this.sendLine(this.formatPoint({
+            metricName: "observation.reporting_complete",
+            metricKind: "gauge",
+            occurredUtc: new Date(),
+            value: completion.reportingComplete ? 1 : 0,
+            unitOfMeasure: "boolean",
+            tags: {
+                run_id: completion.runId,
+                result_owner_id: completion.resultOwnerId
+            }
+        }));
+    }
     async persistEvents(events) {
         const points = createReportingSinkMetricPoints(this.sinkName, events, this.tags);
         for (const point of points) {
@@ -1575,6 +1829,8 @@ export class NetdataStatsDReportingSink extends StatsDReportingSinkBase {
 export class JsonlFileReportingSink extends ExpandedEventReportingSink {
     constructor(options = {}) {
         super("jsonl", "extensions.reporting_sinks.jsonl");
+        this.preservesRawIterationObservations = true;
+        this.writeTail = Promise.resolve();
         const source = asRecord(options);
         this.filePath = optionString(source, "filePath", "FilePath").trim() || "loadstrike-reporting.jsonl";
         this.staticTags = normalizeStringMap(optionRecord(source, "staticTags", "StaticTags"));
@@ -1583,8 +1839,28 @@ export class JsonlFileReportingSink extends ExpandedEventReportingSink {
         if (!events.length) {
             return;
         }
-        mkdirSync(dirname(this.filePath), { recursive: true });
-        appendFileSync(this.filePath, `${JSON.stringify(this.buildPayload(events, this.staticTags))}\n`, "utf8");
+        await this.persistCanonicalPayload(this.buildPayload(events, this.staticTags));
+    }
+    async saveIterationBatch(batch) {
+        await this.persistCanonicalPayload(batch);
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistCanonicalPayload(completion);
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
+    persistCanonicalPayload(payload) {
+        const line = `${JSON.stringify(payload)}\n`;
+        const write = this.writeTail.then(async () => {
+            await mkdir(dirname(this.filePath), { recursive: true });
+            await appendFile(this.filePath, line, "utf8");
+        });
+        this.writeTail = write.catch(() => undefined);
+        return write;
     }
 }
 function createRealtimeStatsEvents(session, scenarios) {
@@ -1631,9 +1907,18 @@ function createRunResultEvents(session, result) {
             completed_utc: result.completedUtc,
             report_file_count: result.reportFiles.length,
             disabled_sink_count: result.disabledSinks.length,
-            sink_error_count: result.sinkErrors.length
+            sink_error_count: result.sinkErrors.length,
+            reporting_complete: result.reportingComplete ?? true,
+            observation_last_batch_sequence_64: result.observationDeliveryStats?.lastBatchSequence64 ?? "-1",
+            observation_captured_count_64: result.observationDeliveryStats?.capturedCount64 ?? "0",
+            observation_delivered_count_64: result.observationDeliveryStats?.deliveredCount64 ?? "0",
+            observation_dropped_buffer_count_64: result.observationDeliveryStats?.droppedBufferCount64 ?? "0",
+            observation_dropped_sink_count_64: result.observationDeliveryStats?.droppedSinkCount64 ?? "0"
         })
     ];
+    for (const row of result.correlationRows ?? []) {
+        events.push(createCorrelationOutcomeEvent(session, occurredUtc, row));
+    }
     for (const reportFile of result.reportFiles) {
         events.push(createReportingEvent(session, occurredUtc, "run.report.final", null, null, {
             phase: "final",
@@ -1664,8 +1949,65 @@ function createRunResultEvents(session, result) {
             attempts: sinkError.attempts
         }));
     }
+    for (const warning of result.generatorWarnings ?? []) {
+        events.push(createReportingEvent(session, occurredUtc, "run.generator-warning.final", warning.scenarioName || null, null, {
+            phase: "final",
+            entity: "generator-warning",
+            warning_code: warning.code,
+            ...(warning.sinkName ? { sink_name: warning.sinkName } : {})
+        }, {
+            warning_code: warning.code,
+            sink_name: warning.sinkName ?? "",
+            scenario_name: warning.scenarioName,
+            scenario_index: warning.scenarioIndex ?? -1,
+            simulation_index: warning.simulationIndex,
+            simulation_kind: warning.simulationKind ?? "",
+            count_64: warning.count64,
+            message: warning.message,
+            first_observed_utc_ns: warning.firstObservedUtcNs,
+            last_observed_utc_ns: warning.lastObservedUtcNs
+        }));
+    }
     return events;
 }
+function createCorrelationOutcomeEvent(session, fallbackOccurredUtc, row) {
+    const occurredToken = optionString(row, "occurredUtc", "OccurredUtc");
+    const parsedOccurredUtc = occurredToken ? new Date(occurredToken) : fallbackOccurredUtc;
+    const occurredUtc = Number.isFinite(parsedOccurredUtc.getTime())
+        ? parsedOccurredUtc
+        : fallbackOccurredUtc;
+    const scenarioName = optionString(row, "scenario", "Scenario");
+    const source = optionString(row, "source", "Source");
+    const destination = optionString(row, "destination", "Destination");
+    const runMode = optionString(row, "runMode", "RunMode");
+    const statusCode = optionString(row, "statusCode", "StatusCode");
+    const gatherByField = optionString(row, "gatherByField", "GatherByField");
+    const gatherByValue = optionString(row, "gatherByValue", "GatherByValue");
+    return createReportingEvent(session, occurredUtc, "correlation.outcome.final", scenarioName || null, null, {
+        phase: "final",
+        entity: "correlation-outcome",
+        source,
+        destination,
+        run_mode: runMode,
+        status_code: statusCode,
+        gather_by_field: gatherByField,
+        gather_by_value: gatherByValue
+    }, {
+        occurred_utc: occurredToken || occurredUtc.toISOString(),
+        source,
+        destination,
+        run_mode: runMode,
+        status_code: statusCode,
+        is_success: pickBooleanValue(row, false, "isSuccess", "IsSuccess"),
+        is_failure: pickBooleanValue(row, false, "isFailure", "IsFailure"),
+        gather_by_field: gatherByField,
+        gather_by_value: gatherByValue,
+        tracking_id: optionString(row, "trackingId", "TrackingId"),
+        event_id: optionString(row, "eventId", "EventId"),
+        latency_ms: optionNumber(row, "latencyMs", "LatencyMs") ?? 0,
+        message: optionString(row, "message", "Message")
+    });
+}
 function createNodeSummaryEvent(session, occurredUtc, stats) {
     return createReportingEvent(session, occurredUtc, "test.final", null, null, {
         phase: "final",
@@ -1851,6 +2193,123 @@ function createReportingEvent(session, occurredUtc, eventType, scenarioName, ste
         fields: eventFields
     };
 }
+function removeSdkPercentileFields(event) {
+    const fields = Object.fromEntries(Object.entries(event.fields).filter(([name]) => !/_(?:latency|bytes)_p(?:50|75|95|99)(?:_|$)/iu.test(name)));
+    return {
+        ...event,
+        fields
+    };
+}
+function createIterationObservationEvents(session, batch) {
+    const events = [];
+    for (const observation of batch.observations) {
+        const tags = {
+            run_id: batch.runId,
+            result_owner_id: batch.resultOwnerId,
+            phase: observation.phase,
+            scenario_name: observation.scenarioName,
+            simulation_kind: observation.simulationKind,
+            is_final_attempt: observation.isFinalAttempt ? "true" : "false"
+        };
+        events.push({
+            runId: batch.runId,
+            eventType: "iteration.observation",
+            occurredUtc: new Date(),
+            sessionId: batch.sessionId,
+            testSuite: session.testSuite,
+            testName: session.testName,
+            clusterId: session.clusterId,
+            nodeType: session.nodeType,
+            machineName: session.machineName,
+            scenarioName: observation.scenarioName,
+            stepName: null,
+            tags,
+            fields: {
+                schema_version: observation.schemaVersion,
+                batch_id: batch.batchId,
+                observation_id: observation.observationId,
+                iteration_id: observation.iterationId,
+                process_group: observation.processGroup,
+                scenario_index: observation.scenarioIndex,
+                simulation_index: observation.simulationIndex,
+                global_ordinal_64: observation.globalOrdinal64,
+                shard_index: observation.shardIndex,
+                shard_count: observation.shardCount,
+                attempt_index: observation.attemptIndex,
+                started_utc_ns: observation.startedUtcNs,
+                completed_utc_ns: observation.completedUtcNs,
+                observed_latency_us_64: observation.observedLatencyUs64,
+                reported_latency_us_64: observation.reportedLatencyUs64,
+                is_success: observation.isSuccess,
+                status_code: observation.statusCode,
+                size_bytes_64: observation.sizeBytes64
+            }
+        });
+        for (const step of observation.steps) {
+            events.push({
+                runId: batch.runId,
+                eventType: "iteration.step",
+                occurredUtc: new Date(),
+                sessionId: batch.sessionId,
+                testSuite: session.testSuite,
+                testName: session.testName,
+                clusterId: session.clusterId,
+                nodeType: session.nodeType,
+                machineName: session.machineName,
+                scenarioName: observation.scenarioName,
+                stepName: step.stepName,
+                tags: { ...tags },
+                fields: {
+                    schema_version: observation.schemaVersion,
+                    batch_id: batch.batchId,
+                    observation_id: observation.observationId,
+                    iteration_id: observation.iterationId,
+                    attempt_index: observation.attemptIndex,
+                    sort_index: step.sortIndex,
+                    started_utc_ns: step.startedUtcNs,
+                    completed_utc_ns: step.completedUtcNs,
+                    observed_latency_us_64: step.observedLatencyUs64,
+                    reported_latency_us_64: step.reportedLatencyUs64,
+                    is_success: step.isSuccess,
+                    status_code: step.statusCode,
+                    size_bytes_64: step.sizeBytes64
+                }
+            });
+        }
+    }
+    return events;
+}
+function createIterationCompletionEvents(session, completion) {
+    return [{
+            runId: completion.runId,
+            eventType: "observation.stream.completed",
+            occurredUtc: new Date(),
+            sessionId: completion.sessionId,
+            testSuite: session.testSuite,
+            testName: session.testName,
+            clusterId: session.clusterId,
+            nodeType: session.nodeType,
+            machineName: session.machineName,
+            scenarioName: null,
+            stepName: null,
+            tags: {
+                run_id: completion.runId,
+                result_owner_id: completion.resultOwnerId,
+                phase: "final"
+            },
+            fields: {
+                schema_version: completion.schemaVersion,
+                process_group: completion.processGroup,
+                last_batch_sequence_64: completion.lastBatchSequence64,
+                captured_count_64: completion.capturedCount64,
+                delivered_count_64: completion.deliveredCount64,
+                dropped_buffer_count_64: completion.droppedBufferCount64,
+                dropped_sink_count_64: completion.droppedSinkCount64,
+                reporting_complete: completion.reportingComplete,
+                completed_utc_ns: completion.completedUtcNs
+            }
+        }];
+}
 function portalEventPayload(event, index) {
     return {
         eventId: portalEventId(event, index),
@@ -1923,10 +2382,6 @@ function addMeasurementFields(fields, prefix, measurement) {
     fields[`${prefix}_latency_min_ms`] = latency.minMs ?? 0;
     fields[`${prefix}_latency_mean_ms`] = latency.meanMs ?? 0;
     fields[`${prefix}_latency_max_ms`] = latency.maxMs ?? 0;
-    fields[`${prefix}_latency_p50_ms`] = latency.percent50 ?? 0;
-    fields[`${prefix}_latency_p75_ms`] = latency.percent75 ?? 0;
-    fields[`${prefix}_latency_p95_ms`] = latency.percent95 ?? 0;
-    fields[`${prefix}_latency_p99_ms`] = latency.percent99 ?? 0;
     fields[`${prefix}_latency_std_dev`] = latency.stdDev ?? 0;
     fields[`${prefix}_latency_le_800_count`] = latencyCount.lessOrEq800 ?? 0;
     fields[`${prefix}_latency_gt_800_lt_1200_count`] = latencyCount.more800Less1200 ?? 0;
@@ -1935,10 +2390,6 @@ function addMeasurementFields(fields, prefix, measurement) {
     fields[`${prefix}_bytes_min`] = dataTransfer.minBytes ?? 0;
     fields[`${prefix}_bytes_mean`] = dataTransfer.meanBytes ?? 0;
     fields[`${prefix}_bytes_max`] = dataTransfer.maxBytes ?? 0;
-    fields[`${prefix}_bytes_p50`] = dataTransfer.percent50 ?? 0;
-    fields[`${prefix}_bytes_p75`] = dataTransfer.percent75 ?? 0;
-    fields[`${prefix}_bytes_p95`] = dataTransfer.percent95 ?? 0;
-    fields[`${prefix}_bytes_p99`] = dataTransfer.percent99 ?? 0;
     fields[`${prefix}_bytes_std_dev`] = dataTransfer.stdDev ?? 0;
     fields[`${prefix}_status_code_count`] = statusCodes.length;
 }
@@ -3074,6 +3525,9 @@ function deepCloneValue(value) {
     }
     return value;
 }
+function gzipJsonRequestBody(value) {
+    return Uint8Array.from(serializeIterationObservationBatchGzipJson(value)).buffer;
+}
 async function postWithTimeout(fetchImpl, url, init, timeoutMs, sinkName) {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), Math.max(timeoutMs, 1));