@loadstrike/loadstrike-sdk 1.0.30401 → 1.0.31601

package/dist/esm/iteration-observations.js

@@ -1,5 +1,8 @@
 import { createHash } from "node:crypto";
+import { performance } from "node:perf_hooks";
 import { gzipSync } from "node:zlib";
+import { logIterationObservationFailure, logIterationObservationRecovery } from "./iteration-observation-diagnostics.js";
+import { cooperativeSinkRetryYield, normalizeSinkRetryBackoffMs, normalizeSinkRetryCount, sinkRetryDelayMs, waitForSinkRetryDelay } from "./sink-retry-policy.js";
 export const ITERATION_OBSERVATION_SCHEMA_VERSION = "loadstrike.iteration-observation/1";
 export const ITERATION_OBSERVATION_BATCH_SCHEMA_VERSION = "loadstrike.iteration-batch/1";
 export const ITERATION_OBSERVATION_STREAM_COMPLETION_SCHEMA_VERSION = "loadstrike.iteration-stream-completion/1";
@@ -141,10 +144,15 @@ export class ProcessIterationObservationBuffer {
 }
 const processIterationObservationBuffer = new ProcessIterationObservationBuffer();
 class SinkDispatcher {
-    constructor(target, depth, parallelism, onDropped) {
+    constructor(target, depth, parallelism, retryCount, retryBackoffMs, logger, runId, resultOwnerId, onDropped) {
         this.target = target;
         this.depth = depth;
         this.parallelism = parallelism;
+        this.retryCount = retryCount;
+        this.retryBackoffMs = retryBackoffMs;
+        this.logger = logger;
+        this.runId = runId;
+        this.resultOwnerId = resultOwnerId;
         this.onDropped = onDropped;
         this.queue = [];
         this.active = new Set();
@@ -152,6 +160,8 @@ class SinkDispatcher {
         this.timedOut = false;
         this.delivered = 0n;
         this.dropped = 0n;
+        this.drainDeadlineMs = null;
+        this.pendingRetryDelays = new Set();
     }
     get deliveredCount() {
         return this.delivered;
@@ -160,7 +170,7 @@ class SinkDispatcher {
         return this.dropped;
     }
     get canPublishCompletion() {
-        return this.queue.length === 0 && this.active.size === 0;
+        return !this.timedOut && this.queue.length === 0 && this.active.size === 0;
     }
     enqueue(batch) {
         if (this.timedOut || this.queue.length >= this.depth) {
@@ -188,7 +198,6 @@ class SinkDispatcher {
         });
         const timeout = new Promise((resolve) => {
             timer = setTimeout(() => resolve(false), Math.max(timeoutMs, 1));
-            timer.unref?.();
         });
         const completed = await Promise.race([idle, timeout]);
         if (timer) {
@@ -201,16 +210,30 @@ class SinkDispatcher {
         }
         return completed;
     }
+    beginDrain(deadlineMs) {
+        this.drainDeadlineMs = deadlineMs;
+        const delaysThatCannotFit = Array.from(this.pendingRetryDelays)
+            .filter((pending) => pending.dueAtMs >= deadlineMs);
+        if (delaysThatCannotFit.length > 0) {
+            for (const pending of delaysThatCannotFit) {
+                pending.cancel();
+            }
+            this.timeoutOutstanding();
+        }
+    }
     pump() {
         while (!this.timedOut && this.active.size < this.parallelism && this.queue.length > 0) {
             const item = this.queue.shift();
             this.active.add(item);
             Promise.resolve()
-                .then(() => this.target.saveIterationBatch(item.batch))
-                .then(() => {
-                if (!this.timedOut) {
+                .then(() => this.deliver(item))
+                .then((outcome) => {
+                if (outcome === "success" && !this.timedOut) {
                     this.delivered += item.count;
                 }
+                else if (outcome === "exhausted" && !this.timedOut) {
+                    this.drop("observation_sink_delivery_failed", item.batch.observations);
+                }
             })
                 .catch(() => {
                 if (!this.timedOut) {
@@ -225,10 +248,95 @@ class SinkDispatcher {
         }
         this.notifyIdle();
     }
+    async deliver(item) {
+        const maximumAttempts = this.retryCount + 1;
+        for (let attempt = 1; attempt <= maximumAttempts; attempt += 1) {
+            if (this.timedOut || this.deadlineExpired()) {
+                return "deadline";
+            }
+            try {
+                await this.target.saveIterationBatch(item.batch);
+                if (attempt > 1) {
+                    logIterationObservationRecovery(this.logger, {
+                        ...this.diagnosticContext(item, attempt, maximumAttempts),
+                        nextDelayMs: 0
+                    });
+                }
+                return "success";
+            }
+            catch (error) {
+                const exhausted = attempt >= maximumAttempts;
+                const nextDelayMs = exhausted
+                    ? 0
+                    : sinkRetryDelayMs(this.retryBackoffMs, attempt);
+                logIterationObservationFailure(this.logger, exhausted ? "error" : "warn", {
+                    ...this.diagnosticContext(item, attempt, maximumAttempts),
+                    nextDelayMs
+                }, error);
+                if (exhausted) {
+                    return "exhausted";
+                }
+                if (!await this.waitForRetry(attempt)) {
+                    this.timeoutOutstanding();
+                    return "deadline";
+                }
+            }
+        }
+        return "exhausted";
+    }
+    diagnosticContext(item, attempt, maximumAttempts) {
+        return {
+            sinkName: this.target.name,
+            operation: "observation-batch",
+            runId: this.runId,
+            resultOwnerId: this.resultOwnerId,
+            batchId: item.batch.batchId,
+            batchSequence64: item.batch.batchSequence64,
+            observationCount: Number(item.count),
+            attempt,
+            maximumAttempts
+        };
+    }
+    async waitForRetry(retryNumber) {
+        const delayMs = sinkRetryDelayMs(this.retryBackoffMs, retryNumber);
+        const deadline = this.drainDeadlineMs;
+        if (deadline !== null && !delayCanFit(delayMs, deadline)) {
+            return false;
+        }
+        if (delayMs === 0) {
+            await cooperativeSinkRetryYield();
+            return !this.timedOut && !this.deadlineExpired();
+        }
+        return await new Promise((resolve) => {
+            let settled = false;
+            let pending;
+            const finish = (completed) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                this.pendingRetryDelays.delete(pending);
+                resolve(completed && !this.timedOut && !this.deadlineExpired());
+            };
+            const timer = setTimeout(() => finish(true), delayMs);
+            const cancel = () => finish(false);
+            pending = {
+                dueAtMs: performance.now() + delayMs,
+                cancel
+            };
+            this.pendingRetryDelays.add(pending);
+        });
+    }
+    deadlineExpired() {
+        return this.drainDeadlineMs !== null && performance.now() >= this.drainDeadlineMs;
+    }
     timeoutOutstanding() {
         if (this.timedOut || this.isIdle())
             return;
         this.timedOut = true;
+        for (const pending of Array.from(this.pendingRetryDelays)) {
+            pending.cancel();
+        }
         const queued = [...this.queue];
         const active = [...this.active];
         this.queue.length = 0;
@@ -300,7 +408,7 @@ export class IterationObservationReporter {
             this.recordWarning("sink_iteration_batches_unsupported", sink.name, "", -1, 1n, "The reporting sink does not support raw iteration batches.");
             return false;
         })
-            .map((target) => new SinkDispatcher(target, this.settings.sinkQueueDepth, this.settings.sinkParallelism, (code, sinkName, observations) => this.recordSinkDrop(code, sinkName, observations)));
+            .map((target) => new SinkDispatcher(target, this.settings.sinkQueueDepth, this.settings.sinkParallelism, normalizeSinkRetryCount(options.sinkRetryCount), normalizeSinkRetryBackoffMs(options.sinkRetryBackoffMs), options.logger, options.runId, options.resultOwnerId, (code, sinkName, observations) => this.recordSinkDrop(code, sinkName, observations)));
         if (this.dispatchers.length > 0) {
             this.buffer.register(this.streamKey, this.settings.maxBufferBytes);
         }
@@ -356,11 +464,14 @@ export class IterationObservationReporter {
         this.sealed = true;
         this.clock.clearInterval(this.timer);
         this.flushNow();
-        const started = Date.now();
+        const started = performance.now();
         const deadline = started + this.settings.drainTimeoutMs;
+        for (const dispatcher of this.dispatchers) {
+            dispatcher.beginDrain(deadline);
+        }
         const drainResults = new Array(this.dispatchers.length).fill(false);
         const completionResults = await Promise.all(this.dispatchers.map(async (dispatcher, index) => {
-            const drained = await dispatcher.waitForIdle(Math.max(deadline - Date.now(), 1));
+            const drained = await dispatcher.waitForIdle(Math.max(deadline - performance.now(), 1));
             drainResults[index] = drained;
             if (!dispatcher.canPublishCompletion) {
                 return false;
@@ -375,19 +486,7 @@ export class IterationObservationReporter {
                 && dispatcher.droppedCount === 0n
                 && dispatcher.deliveredCount === this.captured;
             const completion = this.createCompletion(dispatcher, sinkReportingComplete);
-            let timer;
-            const outcome = await Promise.race([
-                Promise.resolve()
-                    .then(() => complete(completion))
-                    .then(() => "success", () => "failed"),
-                new Promise((resolve) => {
-                    timer = setTimeout(() => resolve("timeout"), Math.max(deadline - Date.now(), 1));
-                    timer.unref?.();
-                })
-            ]);
-            if (timer) {
-                clearTimeout(timer);
-            }
+            const outcome = await this.completeWithRetries(complete, completion, deadline, dispatcher.target.name);
             if (outcome === "success") {
                 return true;
             }
@@ -423,6 +522,76 @@ export class IterationObservationReporter {
         };
         return { ...this.finalResult };
     }
+    async completeWithRetries(complete, completion, deadline, sinkName) {
+        const retryCount = normalizeSinkRetryCount(this.options.sinkRetryCount);
+        const backoffMs = normalizeSinkRetryBackoffMs(this.options.sinkRetryBackoffMs);
+        const maximumAttempts = retryCount + 1;
+        for (let attempt = 1; attempt <= maximumAttempts; attempt += 1) {
+            const remainingMs = deadline - performance.now();
+            if (remainingMs <= 0) {
+                return "timeout";
+            }
+            let timer;
+            const outcome = await Promise.race([
+                Promise.resolve()
+                    .then(() => complete(completion))
+                    .then(() => ({ kind: "success" }), (error) => ({ kind: "failed", error })),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve({ kind: "timeout" }), Math.max(remainingMs, 1));
+                })
+            ]);
+            if (timer) {
+                clearTimeout(timer);
+            }
+            if (outcome.kind === "success") {
+                if (attempt > 1) {
+                    logIterationObservationRecovery(this.options.logger, {
+                        sinkName,
+                        operation: "observation-stream-completion",
+                        runId: this.options.runId,
+                        resultOwnerId: this.options.resultOwnerId,
+                        observationCount: 0,
+                        attempt,
+                        maximumAttempts,
+                        nextDelayMs: 0
+                    });
+                }
+                return "success";
+            }
+            if (outcome.kind === "timeout") {
+                return "timeout";
+            }
+            const exhausted = attempt >= maximumAttempts;
+            const nextDelayMs = exhausted ? 0 : sinkRetryDelayMs(backoffMs, attempt);
+            logIterationObservationFailure(this.options.logger, exhausted ? "error" : "warn", {
+                sinkName,
+                operation: "observation-stream-completion",
+                runId: this.options.runId,
+                resultOwnerId: this.options.resultOwnerId,
+                observationCount: 0,
+                attempt,
+                maximumAttempts,
+                nextDelayMs
+            }, outcome.error);
+            if (exhausted) {
+                return "failed";
+            }
+            const delayMs = nextDelayMs;
+            if (!delayCanFit(delayMs, deadline)) {
+                return "timeout";
+            }
+            if (delayMs === 0) {
+                await cooperativeSinkRetryYield();
+            }
+            else {
+                await waitForSinkRetryDelay(delayMs);
+            }
+            if (performance.now() >= deadline) {
+                return "timeout";
+            }
+        }
+        return "failed";
+    }
     buildWarnings() {
         return Array.from(this.warnings.values())
             .sort((left, right) => left.code.localeCompare(right.code)
@@ -685,6 +854,10 @@ function truncateUtf8(value, maxBytes) {
     }
     return output;
 }
+function delayCanFit(delayMs, deadlineMs) {
+    const remainingMs = deadlineMs - performance.now();
+    return remainingMs > 0 && delayMs < remainingMs;
+}
 function deepFreezeObservation(observation) {
     for (const step of observation.steps) {
         Object.freeze(step);

package/dist/esm/local-report-input.js

@@ -0,0 +1,17 @@
+export function emptyLocalReportInput() {
+    return {
+        history: {
+            status: "available",
+            points: []
+        }
+    };
+}
+export function distributedLocalReportInput() {
+    return {
+        history: {
+            status: "unavailable",
+            reasonCategory: "distributed_temporal_aggregation_unavailable",
+            points: []
+        }
+    };
+}

package/dist/esm/local.js

@@ -1378,9 +1378,20 @@ function readTrackingId(payload, selector) {
             return null;
         }
         let current = body;
-        for (const segment of path.split(".").filter(Boolean)) {
+        let segments;
+        try {
+            segments = safeJsonPathSegments(path);
+        }
+        catch {
+            return null;
+        }
+        for (const segment of segments) {
             if (current && typeof current === "object" && !Array.isArray(current)) {
-                current = current[segment];
+                const record = current;
+                if (!Object.prototype.hasOwnProperty.call(record, segment)) {
+                    return null;
+                }
+                current = record[segment];
             }
             else {
                 return null;
@@ -1413,23 +1424,57 @@ function readOptionalTrackingSelectorValue(value) {
     return undefined;
 }
 function setJsonPathValue(body, path, value) {
-    const target = body && typeof body === "object" && !Array.isArray(body)
-        ? { ...body }
-        : {};
-    const segments = path.split(".").filter(Boolean);
+    const target = cloneJsonRecord(body);
+    const segments = safeJsonPathSegments(path);
     if (!segments.length) {
         return target;
     }
     let current = target;
     for (let i = 0; i < segments.length - 1; i += 1) {
         const segment = segments[i];
-        const next = current[segment];
+        const next = readOwnJsonProperty(current, segment);
+        let child;
         if (!next || typeof next !== "object" || Array.isArray(next)) {
-            current[segment] = {};
+            child = {};
         }
-        current = current[segment];
+        else {
+            child = cloneJsonRecord(next);
+        }
+        defineJsonProperty(current, segment, child);
+        current = child;
+    }
+    defineJsonProperty(current, segments[segments.length - 1], value);
+    return target;
+}
+const FORBIDDEN_JSON_PATH_SEGMENTS = new Set(["__proto__", "prototype", "constructor"]);
+function safeJsonPathSegments(path) {
+    const segments = path.split(".").filter(Boolean);
+    const forbidden = segments.find((segment) => FORBIDDEN_JSON_PATH_SEGMENTS.has(segment));
+    if (forbidden) {
+        throw new Error(`Tracking selector contains forbidden JSON path segment '${forbidden}'.`);
+    }
+    return segments;
+}
+function defineJsonProperty(target, key, value) {
+    Object.defineProperty(target, key, {
+        configurable: true,
+        enumerable: true,
+        value,
+        writable: true
+    });
+}
+function readOwnJsonProperty(target, key) {
+    const descriptor = Object.getOwnPropertyDescriptor(target, key);
+    return descriptor && "value" in descriptor ? descriptor.value : undefined;
+}
+function cloneJsonRecord(value) {
+    const target = {};
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return target;
+    }
+    for (const [key, entry] of Object.entries(value)) {
+        defineJsonProperty(target, key, entry);
     }
-    current[segments[segments.length - 1]] = value;
     return target;
 }
 function mapCorrelationStore(tracking, runNamespace) {