@loadstrike/loadstrike-sdk 1.0.30201 → 1.0.31001

package/dist/esm/sinks.js

@@ -1,9 +1,43 @@
 import { LoadStrikePluginData as LoadStrikePluginDataModel, LoadStrikePluginDataTable as LoadStrikePluginDataTableModel } from "./runtime.js";
 import { createHash, randomBytes } from "node:crypto";
-import { appendFileSync, mkdirSync } from "node:fs";
+import { appendFile, mkdir } from "node:fs/promises";
 import { dirname } from "node:path";
 import { createSocket } from "node:dgram";
+import { gzip } from "node:zlib";
 import { Pool } from "pg";
+import { serializeIterationObservationBatchGzipJson } from "./iteration-observations.js";
+import { redactIterationObservationSecrets } from "./iteration-observation-diagnostics.js";
+const PORTAL_RUN_TOKEN_PROVIDER = Symbol.for("loadstrike.internal.portal-run-token-provider");
+const MAXIMUM_HTTP_RESPONSE_BODY_BYTES = 256 * 1024;
+const MAXIMUM_HTTP_METADATA_CHARACTERS = 256;
+const HTTP_METADATA_TRUNCATION_SUFFIX = " [truncated]";
+class ReportingSinkHttpError extends Error {
+    constructor(sinkName, status, statusText, requestId) {
+        const normalizedStatus = Number.isFinite(status) ? Math.max(Math.trunc(status), 0) : 0;
+        const normalizedStatusText = normalizeHttpMetadata(statusText);
+        const normalizedRequestId = normalizeHttpMetadata(requestId);
+        super(`${sinkName} write failed with HTTP status ${normalizedStatus}`
+            + (normalizedStatusText ? ` ${normalizedStatusText}` : "")
+            + (normalizedRequestId ? ` (requestId=${normalizedRequestId})` : "")
+            + ".");
+        this.name = "ReportingSinkHttpError";
+        this.status = normalizedStatus;
+        this.statusText = normalizedStatusText;
+        this.requestId = normalizedRequestId;
+    }
+}
+function gzipJsonAsync(value) {
+    const json = Buffer.from(JSON.stringify(value), "utf8");
+    return new Promise((resolve, reject) => {
+        gzip(json, (error, compressed) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve(compressed);
+        });
+    });
+}
 const DEFAULT_INFLUX_CONFIGURATION_SECTION_PATH = "LoadStrike:ReportingSinks:InfluxDb";
 const DEFAULT_GRAFANA_LOKI_CONFIGURATION_SECTION_PATH = "LoadStrike:ReportingSinks:GrafanaLoki";
 const DEFAULT_TIMESCALEDB_CONFIGURATION_SECTION_PATH = "LoadStrike:ReportingSinks:TimescaleDb";
@@ -364,6 +398,28 @@ export class CompositeReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        for (const sink of this.sinks) {
+            const saveIterationBatch = sink.saveIterationBatch ?? sink.SaveIterationBatch;
+            if (saveIterationBatch) {
+                await saveIterationBatch.call(sink, batch);
+            }
+        }
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        for (const sink of this.sinks) {
+            const completeIterationObservationStream = sink.completeIterationObservationStream ?? sink.CompleteIterationObservationStream;
+            if (completeIterationObservationStream) {
+                await completeIterationObservationStream.call(sink, completion);
+            }
+        }
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     async stop() {
         for (const sink of this.sinks) {
             const stop = sink.stop ?? sink.Stop;
@@ -378,6 +434,7 @@ export class CompositeReportingSink {
 }
 export class PortalReportingSink {
     constructor(options = {}) {
+        this.iterationObservationPortalSink = true;
         this.sinkName = "portal";
         this.SinkName = "portal";
         this.licenseFeature = "extensions.reporting_sinks.portal";
@@ -385,6 +442,7 @@ export class PortalReportingSink {
         this.baseContext = null;
         this.session = null;
         this.runToken = "";
+        this.runTokenProvider = null;
         this.ingestUrl = "";
         this.optionsInput = { ...options };
         const source = asRecord(options);
@@ -401,7 +459,11 @@ export class PortalReportingSink {
         this.init(context, infraConfig);
     }
     start(session) {
-        this.runToken = String(session.runToken ?? session.RunToken ?? "").trim();
+        const internalProvider = session[PORTAL_RUN_TOKEN_PROVIDER];
+        this.runTokenProvider = typeof internalProvider === "function"
+            ? internalProvider
+            : () => String(session.runToken ?? session.RunToken ?? "").trim();
+        this.runToken = this.runTokenProvider();
         this.ingestUrl = String(session.portalReportingIngestUrl ?? session.PortalReportingIngestUrl ?? "").trim();
         if (!this.runToken || !this.ingestUrl) {
             throw new Error("PortalReportingSink requires a managed portal reporting session.");
@@ -431,8 +493,27 @@ export class PortalReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        const compressed = await gzipJsonAsync(batch);
+        await this.persistObservationPayload({
+            compressedObservationBatches: [{
+                    compression: "gzip-json",
+                    payloadBase64: compressed.toString("base64")
+                }]
+        });
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistObservationPayload({ observationStreamCompletions: [completion] });
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
+        this.runTokenProvider = null;
     }
     Stop() {
         this.stop();
@@ -461,12 +542,27 @@ export class PortalReportingSink {
             method: "POST",
             headers: { "Content-Type": "application/json" },
             body: JSON.stringify({
-                runToken: this.runToken,
-                events: events.map((event, index) => portalEventPayload(event, index))
+                runToken: this.currentRunToken(),
+                events: events.map((event, index) => portalEventPayload(removeSdkPercentileFields(event), index))
+            })
+        }, this.timeoutMs, "PortalReportingSink");
+        validatePortalIngestResponse(responseBody);
+    }
+    async persistObservationPayload(payload) {
+        const responseBody = await postWithTimeout(this.fetchImpl, this.ingestUrl, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                runToken: this.currentRunToken(),
+                ...payload
             })
         }, this.timeoutMs, "PortalReportingSink");
         validatePortalIngestResponse(responseBody);
     }
+    currentRunToken() {
+        const current = this.runTokenProvider?.() ?? "";
+        return current || this.runToken;
+    }
 }
 export function cloneReportingSinkForRun(sink) {
     if (sink instanceof PortalReportingSink) {
@@ -542,6 +638,18 @@ export class InfluxDbReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -651,6 +759,18 @@ export class GrafanaLokiReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -796,6 +916,18 @@ export class TimescaleDbReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     async stop() {
         this.session = null;
         if (this.pool) {
@@ -1071,6 +1203,18 @@ export class DatadogReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1193,6 +1337,18 @@ export class SplunkReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1295,6 +1451,18 @@ export class OtelCollectorReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1371,6 +1539,7 @@ class ExpandedEventReportingSink {
     constructor(sinkName, licenseFeature) {
         this.baseContext = null;
         this.session = null;
+        this.preservesRawIterationObservations = true;
         this.sinkName = sinkName;
         this.SinkName = sinkName;
         this.licenseFeature = licenseFeature;
@@ -1406,6 +1575,18 @@ class ExpandedEventReportingSink {
     async SaveRunResult(result) {
         await this.saveRunResult(result);
     }
+    async saveIterationBatch(batch) {
+        await this.persistEvents(createIterationObservationEvents(this.getSession(), batch));
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistEvents(createIterationCompletionEvents(this.getSession(), completion));
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
     stop() {
         this.session = null;
     }
@@ -1429,15 +1610,18 @@ class ExpandedEventReportingSink {
         return this.session;
     }
     buildPayload(events, staticTags = {}) {
+        const projectedEvents = this.preservesRawIterationObservations
+            ? events.map(removeSdkPercentileFields)
+            : events;
         return {
             sinkName: this.sinkName,
             runId: this.getSession().runId,
             sessionId: this.getSession().sessionId,
-            events: events.map((event) => ({
+            events: projectedEvents.map((event) => ({
                 ...event,
                 tags: { ...staticTags, ...event.tags }
             })),
-            metrics: createReportingSinkMetricPoints(this.sinkName, events, staticTags)
+            metrics: createReportingSinkMetricPoints(this.sinkName, projectedEvents, staticTags)
         };
     }
 }
@@ -1471,20 +1655,41 @@ class ExpandedHttpJsonReportingSink extends ExpandedEventReportingSink {
             body: JSON.stringify(this.buildPayload(events, this.staticTags))
         }, this.timeoutMs, this.constructor.name);
     }
+    async saveIterationBatch(batch) {
+        await this.persistCanonicalGzipPayload(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistCanonicalGzipPayload(completion);
+    }
+    async persistCanonicalGzipPayload(payload) {
+        const compressed = await gzipJsonAsync(payload);
+        await postWithTimeout(this.fetchImpl, `${trimTrailingSlashes(this.baseUrl)}${normalizePath(this.endpointPath)}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Content-Encoding": "gzip",
+                ...this.headers
+            },
+            body: Uint8Array.from(compressed).buffer
+        }, this.timeoutMs, this.constructor.name);
+    }
 }
 export class PrometheusRemoteWriteReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("prometheus-remote-write", "extensions.reporting_sinks.prometheus_remote_write", "/api/v1/write", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class CloudWatchReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("cloudwatch", "extensions.reporting_sinks.cloudwatch", "/loadstrike/events", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class DynatraceReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("dynatrace", "extensions.reporting_sinks.dynatrace", "/api/v2/logs/ingest", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class ElasticsearchReportingSink extends ExpandedHttpJsonReportingSink {
@@ -1500,11 +1705,25 @@ export class OpenSearchReportingSink extends ExpandedHttpJsonReportingSink {
 export class NewRelicReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("new-relic", "extensions.reporting_sinks.new_relic", "/log/v1", options);
+        this.iterationObservationShapeLimited = true;
     }
 }
 export class GenericWebhookReportingSink extends ExpandedHttpJsonReportingSink {
     constructor(options = {}) {
         super("webhook", "extensions.reporting_sinks.webhook", "/loadstrike", options);
+        this.preservesRawIterationObservations = true;
+    }
+    async saveIterationBatch(batch) {
+        await this.persistCanonicalGzipPayload(batch);
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistCanonicalGzipPayload(completion);
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
     }
 }
 export class KafkaReportingSink extends ExpandedEventReportingSink {
@@ -1515,6 +1734,12 @@ export class KafkaReportingSink extends ExpandedEventReportingSink {
         this.staticTags = normalizeStringMap(optionRecord(source, "staticTags", "StaticTags"));
         this.publishAsync = pickRecordValue(source, "publishAsync", "PublishAsync");
     }
+    async saveIterationBatch(batch) {
+        await this.publishCanonicalValue(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.publishCanonicalValue(completion);
+    }
     async persistEvents(events) {
         if (!events.length) {
             return;
@@ -1524,10 +1749,17 @@ export class KafkaReportingSink extends ExpandedEventReportingSink {
         }
         await this.publishAsync(this.topic, JSON.stringify(this.buildPayload(events, this.staticTags)));
     }
+    async publishCanonicalValue(value) {
+        if (!this.publishAsync) {
+            throw new Error("KafkaReportingSink requires PublishAsync when a Kafka producer is not configured by the host application.");
+        }
+        await this.publishAsync(this.topic, JSON.stringify(value));
+    }
 }
 class StatsDReportingSinkBase extends ExpandedEventReportingSink {
     constructor(sinkName, licenseFeature, options = {}, dogStatsD = false) {
         super(sinkName, licenseFeature);
+        this.iterationObservationShapeLimited = true;
         const source = asRecord(options);
         this.prefix = optionString(source, "prefix", "Prefix").trim() || "loadstrike";
         this.host = optionString(source, "host", "Host").trim() || "127.0.0.1";
@@ -1536,6 +1768,47 @@ class StatsDReportingSinkBase extends ExpandedEventReportingSink {
         this.sendLineAsync = pickRecordValue(source, "sendLineAsync", "SendLineAsync");
         this.dogStatsD = dogStatsD;
     }
+    async saveIterationBatch(batch) {
+        for (const observation of batch.observations) {
+            const tags = {
+                run_id: batch.runId,
+                result_owner_id: batch.resultOwnerId,
+                scenario_name: observation.scenarioName,
+                phase: observation.phase,
+                outcome: observation.isSuccess ? "success" : "failure"
+            };
+            const occurredUtc = new Date();
+            await this.sendLine(this.formatPoint({
+                metricName: "iteration.reported_latency_us",
+                metricKind: "gauge",
+                occurredUtc,
+                value: Number(observation.reportedLatencyUs64),
+                unitOfMeasure: "us",
+                tags
+            }));
+            await this.sendLine(this.formatPoint({
+                metricName: "iteration.attempt",
+                metricKind: "count",
+                occurredUtc,
+                value: 1,
+                unitOfMeasure: "count",
+                tags
+            }));
+        }
+    }
+    async completeIterationObservationStream(completion) {
+        await this.sendLine(this.formatPoint({
+            metricName: "observation.reporting_complete",
+            metricKind: "gauge",
+            occurredUtc: new Date(),
+            value: completion.reportingComplete ? 1 : 0,
+            unitOfMeasure: "boolean",
+            tags: {
+                run_id: completion.runId,
+                result_owner_id: completion.resultOwnerId
+            }
+        }));
+    }
     async persistEvents(events) {
         const points = createReportingSinkMetricPoints(this.sinkName, events, this.tags);
         for (const point of points) {
@@ -1575,6 +1848,8 @@ export class NetdataStatsDReportingSink extends StatsDReportingSinkBase {
 export class JsonlFileReportingSink extends ExpandedEventReportingSink {
     constructor(options = {}) {
         super("jsonl", "extensions.reporting_sinks.jsonl");
+        this.preservesRawIterationObservations = true;
+        this.writeTail = Promise.resolve();
         const source = asRecord(options);
         this.filePath = optionString(source, "filePath", "FilePath").trim() || "loadstrike-reporting.jsonl";
         this.staticTags = normalizeStringMap(optionRecord(source, "staticTags", "StaticTags"));
@@ -1583,8 +1858,28 @@ export class JsonlFileReportingSink extends ExpandedEventReportingSink {
         if (!events.length) {
             return;
         }
-        mkdirSync(dirname(this.filePath), { recursive: true });
-        appendFileSync(this.filePath, `${JSON.stringify(this.buildPayload(events, this.staticTags))}\n`, "utf8");
+        await this.persistCanonicalPayload(this.buildPayload(events, this.staticTags));
+    }
+    async saveIterationBatch(batch) {
+        await this.persistCanonicalPayload(batch);
+    }
+    async SaveIterationBatch(batch) {
+        await this.saveIterationBatch(batch);
+    }
+    async completeIterationObservationStream(completion) {
+        await this.persistCanonicalPayload(completion);
+    }
+    async CompleteIterationObservationStream(completion) {
+        await this.completeIterationObservationStream(completion);
+    }
+    persistCanonicalPayload(payload) {
+        const line = `${JSON.stringify(payload)}\n`;
+        const write = this.writeTail.then(async () => {
+            await mkdir(dirname(this.filePath), { recursive: true });
+            await appendFile(this.filePath, line, "utf8");
+        });
+        this.writeTail = write.catch(() => undefined);
+        return write;
     }
 }
 function createRealtimeStatsEvents(session, scenarios) {
@@ -1631,9 +1926,18 @@ function createRunResultEvents(session, result) {
             completed_utc: result.completedUtc,
             report_file_count: result.reportFiles.length,
             disabled_sink_count: result.disabledSinks.length,
-            sink_error_count: result.sinkErrors.length
+            sink_error_count: result.sinkErrors.length,
+            reporting_complete: result.reportingComplete ?? true,
+            observation_last_batch_sequence_64: result.observationDeliveryStats?.lastBatchSequence64 ?? "-1",
+            observation_captured_count_64: result.observationDeliveryStats?.capturedCount64 ?? "0",
+            observation_delivered_count_64: result.observationDeliveryStats?.deliveredCount64 ?? "0",
+            observation_dropped_buffer_count_64: result.observationDeliveryStats?.droppedBufferCount64 ?? "0",
+            observation_dropped_sink_count_64: result.observationDeliveryStats?.droppedSinkCount64 ?? "0"
         })
     ];
+    for (const row of result.correlationRows ?? []) {
+        events.push(createCorrelationOutcomeEvent(session, occurredUtc, row));
+    }
     for (const reportFile of result.reportFiles) {
         events.push(createReportingEvent(session, occurredUtc, "run.report.final", null, null, {
             phase: "final",
@@ -1664,8 +1968,65 @@ function createRunResultEvents(session, result) {
             attempts: sinkError.attempts
         }));
     }
+    for (const warning of result.generatorWarnings ?? []) {
+        events.push(createReportingEvent(session, occurredUtc, "run.generator-warning.final", warning.scenarioName || null, null, {
+            phase: "final",
+            entity: "generator-warning",
+            warning_code: warning.code,
+            ...(warning.sinkName ? { sink_name: warning.sinkName } : {})
+        }, {
+            warning_code: warning.code,
+            sink_name: warning.sinkName ?? "",
+            scenario_name: warning.scenarioName,
+            scenario_index: warning.scenarioIndex ?? -1,
+            simulation_index: warning.simulationIndex,
+            simulation_kind: warning.simulationKind ?? "",
+            count_64: warning.count64,
+            message: warning.message,
+            first_observed_utc_ns: warning.firstObservedUtcNs,
+            last_observed_utc_ns: warning.lastObservedUtcNs
+        }));
+    }
     return events;
 }
+function createCorrelationOutcomeEvent(session, fallbackOccurredUtc, row) {
+    const occurredToken = optionString(row, "occurredUtc", "OccurredUtc");
+    const parsedOccurredUtc = occurredToken ? new Date(occurredToken) : fallbackOccurredUtc;
+    const occurredUtc = Number.isFinite(parsedOccurredUtc.getTime())
+        ? parsedOccurredUtc
+        : fallbackOccurredUtc;
+    const scenarioName = optionString(row, "scenario", "Scenario");
+    const source = optionString(row, "source", "Source");
+    const destination = optionString(row, "destination", "Destination");
+    const runMode = optionString(row, "runMode", "RunMode");
+    const statusCode = optionString(row, "statusCode", "StatusCode");
+    const gatherByField = optionString(row, "gatherByField", "GatherByField");
+    const gatherByValue = optionString(row, "gatherByValue", "GatherByValue");
+    return createReportingEvent(session, occurredUtc, "correlation.outcome.final", scenarioName || null, null, {
+        phase: "final",
+        entity: "correlation-outcome",
+        source,
+        destination,
+        run_mode: runMode,
+        status_code: statusCode,
+        gather_by_field: gatherByField,
+        gather_by_value: gatherByValue
+    }, {
+        occurred_utc: occurredToken || occurredUtc.toISOString(),
+        source,
+        destination,
+        run_mode: runMode,
+        status_code: statusCode,
+        is_success: pickBooleanValue(row, false, "isSuccess", "IsSuccess"),
+        is_failure: pickBooleanValue(row, false, "isFailure", "IsFailure"),
+        gather_by_field: gatherByField,
+        gather_by_value: gatherByValue,
+        tracking_id: optionString(row, "trackingId", "TrackingId"),
+        event_id: optionString(row, "eventId", "EventId"),
+        latency_ms: optionNumber(row, "latencyMs", "LatencyMs") ?? 0,
+        message: optionString(row, "message", "Message")
+    });
+}
 function createNodeSummaryEvent(session, occurredUtc, stats) {
     return createReportingEvent(session, occurredUtc, "test.final", null, null, {
         phase: "final",
@@ -1851,6 +2212,125 @@ function createReportingEvent(session, occurredUtc, eventType, scenarioName, ste
         fields: eventFields
     };
 }
+function removeSdkPercentileFields(event) {
+    const fields = Object.fromEntries(Object.entries(event.fields).filter(([name]) => !/_(?:latency|bytes)_p(?:50|75|95|99)(?:_|$)/iu.test(name)));
+    return {
+        ...event,
+        fields
+    };
+}
+function createIterationObservationEvents(session, batch) {
+    const events = [];
+    const occurredUtc = dateFromUtcNanoseconds(batch.createdUtcNs);
+    for (const observation of batch.observations) {
+        const tags = {
+            run_id: batch.runId,
+            result_owner_id: batch.resultOwnerId,
+            phase: observation.phase,
+            scenario_name: observation.scenarioName,
+            simulation_kind: observation.simulationKind,
+            is_final_attempt: observation.isFinalAttempt ? "true" : "false"
+        };
+        events.push({
+            runId: batch.runId,
+            eventType: "iteration.observation",
+            occurredUtc,
+            sessionId: batch.sessionId,
+            testSuite: session.testSuite,
+            testName: session.testName,
+            clusterId: session.clusterId,
+            nodeType: session.nodeType,
+            machineName: session.machineName,
+            scenarioName: observation.scenarioName,
+            stepName: null,
+            tags,
+            fields: {
+                schema_version: observation.schemaVersion,
+                batch_id: batch.batchId,
+                observation_id: observation.observationId,
+                iteration_id: observation.iterationId,
+                process_group: observation.processGroup,
+                scenario_index: observation.scenarioIndex,
+                simulation_index: observation.simulationIndex,
+                global_ordinal_64: observation.globalOrdinal64,
+                shard_index: observation.shardIndex,
+                shard_count: observation.shardCount,
+                attempt_index: observation.attemptIndex,
+                started_utc_ns: observation.startedUtcNs,
+                completed_utc_ns: observation.completedUtcNs,
+                observed_latency_us_64: observation.observedLatencyUs64,
+                reported_latency_us_64: observation.reportedLatencyUs64,
+                is_success: observation.isSuccess,
+                status_code: observation.statusCode,
+                size_bytes_64: observation.sizeBytes64
+            }
+        });
+        for (const step of observation.steps) {
+            events.push({
+                runId: batch.runId,
+                eventType: "iteration.step",
+                occurredUtc,
+                sessionId: batch.sessionId,
+                testSuite: session.testSuite,
+                testName: session.testName,
+                clusterId: session.clusterId,
+                nodeType: session.nodeType,
+                machineName: session.machineName,
+                scenarioName: observation.scenarioName,
+                stepName: step.stepName,
+                tags: { ...tags },
+                fields: {
+                    schema_version: observation.schemaVersion,
+                    batch_id: batch.batchId,
+                    observation_id: observation.observationId,
+                    iteration_id: observation.iterationId,
+                    attempt_index: observation.attemptIndex,
+                    sort_index: step.sortIndex,
+                    started_utc_ns: step.startedUtcNs,
+                    completed_utc_ns: step.completedUtcNs,
+                    observed_latency_us_64: step.observedLatencyUs64,
+                    reported_latency_us_64: step.reportedLatencyUs64,
+                    is_success: step.isSuccess,
+                    status_code: step.statusCode,
+                    size_bytes_64: step.sizeBytes64
+                }
+            });
+        }
+    }
+    return events;
+}
+function createIterationCompletionEvents(session, completion) {
+    const occurredUtc = dateFromUtcNanoseconds(completion.completedUtcNs);
+    return [{
+            runId: completion.runId,
+            eventType: "observation.stream.completed",
+            occurredUtc,
+            sessionId: completion.sessionId,
+            testSuite: session.testSuite,
+            testName: session.testName,
+            clusterId: session.clusterId,
+            nodeType: session.nodeType,
+            machineName: session.machineName,
+            scenarioName: null,
+            stepName: null,
+            tags: {
+                run_id: completion.runId,
+                result_owner_id: completion.resultOwnerId,
+                phase: "final"
+            },
+            fields: {
+                schema_version: completion.schemaVersion,
+                process_group: completion.processGroup,
+                last_batch_sequence_64: completion.lastBatchSequence64,
+                captured_count_64: completion.capturedCount64,
+                delivered_count_64: completion.deliveredCount64,
+                dropped_buffer_count_64: completion.droppedBufferCount64,
+                dropped_sink_count_64: completion.droppedSinkCount64,
+                reporting_complete: completion.reportingComplete,
+                completed_utc_ns: completion.completedUtcNs
+            }
+        }];
+}
 function portalEventPayload(event, index) {
     return {
         eventId: portalEventId(event, index),
@@ -1883,6 +2363,22 @@ function portalEventId(event, index) {
     });
     return `lsr_${createHash("sha256").update(material).digest("hex").slice(0, 40)}`;
 }
+function dateFromUtcNanoseconds(value) {
+    try {
+        const nanoseconds = BigInt(value);
+        if (nanoseconds < 0n) {
+            return new Date(0);
+        }
+        const milliseconds = nanoseconds / 1000000n;
+        const numeric = Number(milliseconds);
+        return Number.isFinite(numeric) && numeric <= 8640000000000000
+            ? new Date(numeric)
+            : new Date(0);
+    }
+    catch {
+        return new Date(0);
+    }
+}
 function addMeasurementFields(fields, prefix, measurement) {
     const request = measurement?.request ?? { count: 0, percent: 0, rps: 0 };
     const latency = measurement?.latency ?? {
@@ -1923,10 +2419,6 @@ function addMeasurementFields(fields, prefix, measurement) {
     fields[`${prefix}_latency_min_ms`] = latency.minMs ?? 0;
     fields[`${prefix}_latency_mean_ms`] = latency.meanMs ?? 0;
     fields[`${prefix}_latency_max_ms`] = latency.maxMs ?? 0;
-    fields[`${prefix}_latency_p50_ms`] = latency.percent50 ?? 0;
-    fields[`${prefix}_latency_p75_ms`] = latency.percent75 ?? 0;
-    fields[`${prefix}_latency_p95_ms`] = latency.percent95 ?? 0;
-    fields[`${prefix}_latency_p99_ms`] = latency.percent99 ?? 0;
     fields[`${prefix}_latency_std_dev`] = latency.stdDev ?? 0;
     fields[`${prefix}_latency_le_800_count`] = latencyCount.lessOrEq800 ?? 0;
     fields[`${prefix}_latency_gt_800_lt_1200_count`] = latencyCount.more800Less1200 ?? 0;
@@ -1935,10 +2427,6 @@ function addMeasurementFields(fields, prefix, measurement) {
     fields[`${prefix}_bytes_min`] = dataTransfer.minBytes ?? 0;
     fields[`${prefix}_bytes_mean`] = dataTransfer.meanBytes ?? 0;
     fields[`${prefix}_bytes_max`] = dataTransfer.maxBytes ?? 0;
-    fields[`${prefix}_bytes_p50`] = dataTransfer.percent50 ?? 0;
-    fields[`${prefix}_bytes_p75`] = dataTransfer.percent75 ?? 0;
-    fields[`${prefix}_bytes_p95`] = dataTransfer.percent95 ?? 0;
-    fields[`${prefix}_bytes_p99`] = dataTransfer.percent99 ?? 0;
     fields[`${prefix}_bytes_std_dev`] = dataTransfer.stdDev ?? 0;
     fields[`${prefix}_status_code_count`] = statusCodes.length;
 }
@@ -3074,27 +3562,93 @@ function deepCloneValue(value) {
     }
     return value;
 }
+function gzipJsonRequestBody(value) {
+    return Uint8Array.from(serializeIterationObservationBatchGzipJson(value)).buffer;
+}
 async function postWithTimeout(fetchImpl, url, init, timeoutMs, sinkName) {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), Math.max(timeoutMs, 1));
     try {
         const response = await fetchImpl(url, { ...init, signal: controller.signal });
-        const body = await readResponseBodyText(response);
         if (!response.ok) {
-            throw new Error(`${sinkName} write failed with status ${response.status}: ${body}`);
+            throw new ReportingSinkHttpError(sinkName, response.status, readHttpResponseStatusText(response), readHttpResponseRequestId(response));
         }
-        return body;
+        return await readResponseBodyText(response);
     }
     finally {
         clearTimeout(timer);
     }
 }
 async function readResponseBodyText(response) {
-    const candidate = response;
-    if (typeof candidate.text !== "function") {
+    const body = response.body;
+    if (body && typeof body.getReader === "function") {
+        const reader = body.getReader();
+        const chunks = [];
+        let bytes = 0;
+        try {
+            while (true) {
+                const next = await reader.read();
+                if (next.done)
+                    break;
+                if (!next.value)
+                    continue;
+                if (bytes + next.value.byteLength > MAXIMUM_HTTP_RESPONSE_BODY_BYTES) {
+                    await reader.cancel();
+                    throw new Error("Reporting sink response exceeded the bounded response limit.");
+                }
+                chunks.push(next.value);
+                bytes += next.value.byteLength;
+            }
+        }
+        finally {
+            reader.releaseLock();
+        }
+        return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
+    }
+    return "";
+}
+function readHttpResponseStatusText(response) {
+    try {
+        return normalizeHttpMetadata(response.statusText);
+    }
+    catch {
+        return "";
+    }
+}
+function readHttpResponseRequestId(response) {
+    for (const name of [
+        "x-request-id",
+        "x-correlation-id",
+        "request-id",
+        "traceparent"
+    ]) {
+        try {
+            const value = response.headers?.get(name);
+            if (value) {
+                return normalizeHttpMetadata(value);
+            }
+        }
+        catch {
+            return "";
+        }
+    }
+    return "";
+}
+function normalizeHttpMetadata(value) {
+    let text;
+    try {
+        const source = String(value ?? "");
+        text = source.length <= MAXIMUM_HTTP_METADATA_CHARACTERS
+            ? source
+            : source.slice(0, MAXIMUM_HTTP_METADATA_CHARACTERS - HTTP_METADATA_TRUNCATION_SUFFIX.length) + HTTP_METADATA_TRUNCATION_SUFFIX;
+    }
+    catch {
         return "";
     }
-    return String(await candidate.text());
+    return redactIterationObservationSecrets(Buffer.from(text, "utf8").toString("utf8")
+        .replace(/[\u0000-\u001F\u007F-\u009F]/gu, " ")
+        .replace(/\s{2,}/gu, " ")
+        .trim());
 }
 function validatePortalIngestResponse(responseBody) {
     const text = String(responseBody ?? "").trim();
@@ -3147,6 +3701,8 @@ export const __loadstrikeTestExports = {
     cloneScenarioStats,
     cloneSessionStartInfo,
     createFinalStatsEvents,
+    createIterationCompletionEvents,
+    createIterationObservationEvents,
     createRealtimeStatsEvents,
     createRunResultEvents,
     createReportingEvent,
@@ -3163,6 +3719,7 @@ export const __loadstrikeTestExports = {
     normalizeStringMap,
     optionNumber,
     optionString,
+    portalEventPayload,
     pickBooleanValue,
     pickRecordValue,
     postWithTimeout,