@laserdata/laser-sdk 0.5.3 → 0.6.0

package/dist/stream/consumer.js

@@ -1,8 +1,11 @@
+import { INTERNAL_RETURN_DELIVERY } from "../client/internals.js";
 import { mintUlidValue } from "../runtime/ulid.js";
-import { CancelledError, FilterExecutionError, InvalidError, TransportError, UnsupportedError } from "../client/errors.js";
-const DEFAULT_BATCH_LENGTH = 100;
-const DEFAULT_POLL_INTERVAL_MS = 250;
-const DEFAULT_START_FROM = { kind: "next" };
+import { CancelledError, FilterExecutionError, InvalidError, TimeoutError, TransportError, UnsupportedError } from "../client/errors.js";
+import { Json } from "./codecs.js";
+const DEFAULT_BATCH_LENGTH = 1000;
+const DEFAULT_POLL_INTERVAL_MS = 0;
+const DEFAULT_POLLING_RETRY_INTERVAL_MS = 1_000;
+const DEFAULT_START_AT = { kind: "next" };
 function throwIfAborted(signal, message) {
     if (signal?.aborted === true)
         throw new CancelledError(message, { cause: signal.reason });
@@ -24,22 +27,52 @@ function delay(ms, signal) {
         signal?.addEventListener("abort", onAbort, { once: true });
     });
 }
+function commitInterval(policy) {
+    return "intervalMs" in policy ? policy.intervalMs : undefined;
+}
+function validateCommitPolicy(policy) {
+    const interval = commitInterval(policy);
+    if (interval !== undefined && (!Number.isFinite(interval) || interval <= 0)) {
+        throw new InvalidError("commit policy intervalMs must be a positive finite number");
+    }
+    if ("count" in policy && (!Number.isSafeInteger(policy.count) || policy.count < 1)) {
+        throw new InvalidError("commit policy count must be a positive safe integer");
+    }
+}
 /** Apply the defaults and reject values no consumer can run with. */
 export function resolveConsumerOptions(options, anonymous) {
+    const commitPolicy = options.commitPolicy ?? (anonymous ? { kind: "disabled" } : { kind: "polling" });
+    validateCommitPolicy(commitPolicy);
+    const initRetries = options.initRetries ?? { retries: 0, intervalMs: 0 };
     const resolved = {
         batchLength: options.batchLength ?? DEFAULT_BATCH_LENGTH,
-        autoCommit: options.autoCommit ?? !anonymous,
-        startFrom: options.startFrom ?? DEFAULT_START_FROM,
-        pollIntervalMs: options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS
+        autoCommit: commitPolicy.kind !== "disabled",
+        commitPolicy,
+        startAt: options.startAt ?? DEFAULT_START_AT,
+        pollIntervalMs: options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
+        autoJoinGroup: options.autoJoinGroup ?? true,
+        createGroup: options.createGroup ?? true,
+        pollingRetryIntervalMs: options.pollingRetryIntervalMs ?? DEFAULT_POLLING_RETRY_INTERVAL_MS,
+        initRetries,
+        allowReplay: options.allowReplay ?? false
     };
+    if (!Number.isFinite(resolved.pollingRetryIntervalMs) || resolved.pollingRetryIntervalMs <= 0) {
+        throw new InvalidError("consumer pollingRetryIntervalMs must be a positive finite number");
+    }
+    if (!Number.isSafeInteger(initRetries.retries) ||
+        initRetries.retries < 0 ||
+        !Number.isFinite(initRetries.intervalMs) ||
+        (options.initRetries === undefined ? initRetries.intervalMs < 0 : initRetries.intervalMs <= 0)) {
+        throw new InvalidError("consumer initRetries needs non-negative retries and a positive intervalMs");
+    }
     if (!Number.isSafeInteger(resolved.batchLength) || resolved.batchLength < 1) {
         throw new InvalidError("consumer batchLength must be a positive safe integer");
     }
     if (!Number.isFinite(resolved.pollIntervalMs) || resolved.pollIntervalMs < 0) {
         throw new InvalidError("consumer pollIntervalMs must be a non-negative finite number");
     }
-    if ((resolved.startFrom.kind === "offset" || resolved.startFrom.kind === "timestamp") &&
-        resolved.startFrom.value < 0n) {
+    if ((resolved.startAt.kind === "offset" || resolved.startAt.kind === "timestampMicros") &&
+        resolved.startAt.value < 0n) {
         throw new InvalidError("consumer start value must be non-negative");
     }
     return resolved;
@@ -59,20 +92,38 @@ export class Consumer {
     topicName;
     target;
     reader;
+    pollFailures;
     batchLength;
     autoCommit;
-    startFrom;
+    commitPolicy;
+    startAt;
     pollIntervalMs;
+    pollingRetryIntervalMs;
+    allowReplay;
     buffer = [];
+    // Native deliveries handed back, yielded again before the buffer.
+    returned = [];
+    storedOffsets = new Map();
+    yieldedSinceStore = 0;
+    commitTimer;
+    latestDelivered;
     // Group reads: the records of the current page, the record each delivered
     // message came from, and whether a delivery waits to be stored.
     records = [];
     delivered = new WeakMap();
-    handledSincePoll = false;
     lastDelivery;
+    // When a group consumer's commit policy last stored the handled prefix, the
+    // records yielded since, and the store decision a failed store left due.
+    yieldedSinceFlush = 0;
+    lastFlushAt = Date.now();
+    pendingCommit;
     moreScanned = false;
     consumedOffsets = new Map();
     explicitOffsets = new Map();
+    // The consumed offset a partition had when its server offset was deleted.
+    // No commit policy stores that offset again, so the delete stands until a
+    // newer record is delivered.
+    deletedAt = new Map();
     nextOffsets = new Map();
     assignedPartitions = new Set();
     assignmentSeen = false;
@@ -80,12 +131,16 @@ export class Consumer {
     gainedPartitions = new Set();
     started = false;
     shuttingDown = false;
-    constructor(transport, streamName, topicName, target, options = {}, reader) {
+    polling;
+    // A native group consumer leaves at shutdown only the group it joined.
+    leavesGroup;
+    constructor(transport, streamName, topicName, target, options = {}, reader, pollFailures = "retry") {
         this.transport = transport;
         this.streamName = streamName;
         this.topicName = topicName;
         this.target = target;
         this.reader = reader;
+        this.pollFailures = pollFailures;
         const anonymous = target.kind === "single" && target.name === undefined;
         if (anonymous)
             this.target = { ...target, name: `anonymous-${mintUlidValue().toString(16)}` };
@@ -96,9 +151,22 @@ export class Consumer {
         const resolved = resolveConsumerOptions(options, anonymous);
         this.batchLength = resolved.batchLength;
         this.autoCommit = resolved.autoCommit;
-        this.startFrom = resolved.startFrom;
+        this.commitPolicy = resolved.commitPolicy;
+        this.startAt = resolved.startAt;
         this.pollIntervalMs = resolved.pollIntervalMs;
+        this.pollingRetryIntervalMs = resolved.pollingRetryIntervalMs;
+        this.allowReplay = resolved.allowReplay;
+        this.leavesGroup = this.target.kind === "group" && resolved.autoJoinGroup;
     }
+    /** @internal */
+    static create(transport, streamName, topicName, target, options = {}, reader, pollFailures = "retry") {
+        return new Consumer(transport, streamName, topicName, target, options, reader, pollFailures);
+    }
+    /**
+     * Waits for the next record, bounding how long the caller sits idle. Fails
+     * with `TimeoutError` past `timeoutMs` and with `InvalidError` once the
+     * consumer was shut down, like the Rust `next_within`.
+     */
     async nextWithin(timeoutMs, options = {}) {
         if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
             throw new InvalidError("nextWithin timeout must be a non-negative finite number");
@@ -106,21 +174,23 @@ export class Consumer {
         const deadline = Date.now() + timeoutMs;
         for (;;) {
             if (this.stopped())
-                return null;
+                throw new InvalidError("the live consumer stream ended");
             throwIfAborted(options.signal, "nextWithin aborted");
-            this.handledPrevious();
+            await this.finishDelivery();
             const message = this.take();
-            if (message !== undefined)
+            if (message !== undefined) {
+                await this.afterYield(message);
                 return message;
-            await this.fillBuffer(options.signal);
+            }
+            await this.pollOnce(options.signal, deadline);
             throwIfAborted(options.signal, "nextWithin aborted");
             if (this.stopped())
-                return null;
+                throw new InvalidError("the live consumer stream ended");
             if (this.pending())
                 continue;
             const remaining = deadline - Date.now();
             if (remaining <= 0)
-                return null;
+                throw new TimeoutError("the live consumer to yield a record");
             await delay(this.moreScanned ? 0 : Math.min(this.pollIntervalMs, remaining), options.signal);
         }
     }
@@ -130,13 +200,14 @@ export class Consumer {
     async *stream(options = {}) {
         while (!this.stopped()) {
             throwIfAborted(options.signal, "consumer stream aborted");
-            this.handledPrevious();
+            await this.finishDelivery();
             const message = this.take();
             if (message !== undefined) {
+                await this.afterYield(message);
                 yield message;
                 continue;
             }
-            await this.fillBuffer(options.signal);
+            await this.pollOnce(options.signal);
             throwIfAborted(options.signal, "consumer stream aborted");
             if (this.stopped())
                 return;
@@ -145,6 +216,29 @@ export class Consumer {
             }
         }
     }
+    /**
+     * Hands back the latest delivery that the caller could not process, so the
+     * next read yields it again. Only the most recent delivery can be returned.
+     */
+    [INTERNAL_RETURN_DELIVERY](message) {
+        if (this.stopped())
+            throw new InvalidError("consumer has been shut down");
+        if (message !== this.latestDelivered) {
+            throw new InvalidError("the delivery is not the latest one from this consumer");
+        }
+        this.latestDelivered = undefined;
+        if (this.reader !== undefined) {
+            const record = this.delivered.get(message);
+            if (record === undefined) {
+                throw new InvalidError("the record was not delivered by this group consumer");
+            }
+            if (this.lastDelivery?.record === record)
+                this.lastDelivery = undefined;
+            this.records.unshift(record);
+            return;
+        }
+        this.returned.unshift(message);
+    }
     /**
      * Store a handled message's offset on the server. A group consumer on a
      * server that resolves group policies stores the contiguous prefix of the
@@ -157,18 +251,65 @@ export class Consumer {
                 throw new InvalidError("commit the message object this group consumer delivered");
             }
             await this.reader.ackThrough(record);
-            this.explicitOffsets.set(message.partitionId, message.offset);
+            this.explicitOffsets.set(message.partitionId, message.position.offset);
+            this.storedOffsets.set(message.partitionId, message.position.offset);
             return;
         }
-        await this.transport.storeOffset(this.streamName, this.topicName, this.target, message.partitionId, message.offset);
-        this.explicitOffsets.set(message.partitionId, message.offset);
+        await this.transport.storeOffset(this.streamName, this.topicName, this.target, message.partitionId, message.position.offset);
+        this.explicitOffsets.set(message.partitionId, message.position.offset);
+        this.storedOffsets.set(message.partitionId, message.position.offset);
     }
     lastConsumedOffset(partitionId) {
         return this.consumedOffsets.get(partitionId);
     }
+    /** The last offset this consumer stored for `partitionId`, by `commit`,
+     * `storeOffset`, or its commit policy. Local bookkeeping: for server-side
+     * resume, start from `next`. */
+    lastStoredOffset(partitionId) {
+        return this.storedOffsets.get(partitionId);
+    }
+    /**
+     * Stores an explicit server offset for `partitionId`, or for the partition
+     * of the latest delivery when omitted. Native only: a policy-aware group
+     * consumer refuses it, because an arbitrary offset bypasses the group's
+     * acknowledgment contract. Commit a delivered record instead.
+     */
+    async storeOffset(offset, partitionId) {
+        this.requireNative("storeOffset");
+        if (offset < 0n)
+            throw new InvalidError("offset must be non-negative");
+        const partition = this.currentPartition(partitionId);
+        await this.transport.storeOffset(this.streamName, this.topicName, this.target, partition, offset);
+        this.explicitOffsets.set(partition, offset);
+        this.storedOffsets.set(partition, offset);
+    }
+    /** Deletes the server offset for `partitionId`, or for the partition of the
+     * latest delivery when omitted. Native only, like `storeOffset`. */
+    async deleteOffset(partitionId) {
+        this.requireNative("deleteOffset");
+        if (this.transport.deleteOffset === undefined) {
+            throw new UnsupportedError("the Apache Iggy client does not expose offset deletion", {
+                surface: "stream"
+            });
+        }
+        const partition = this.currentPartition(partitionId);
+        await this.transport.deleteOffset(this.streamName, this.topicName, this.target, partition);
+        this.explicitOffsets.delete(partition);
+        this.storedOffsets.delete(partition);
+        const consumed = this.consumedOffsets.get(partition);
+        if (consumed === undefined)
+            this.deletedAt.delete(partition);
+        else
+            this.deletedAt.set(partition, consumed);
+    }
+    /** The offset the server stores for this consumer, its group or its
+     * name, on `partitionId`, with the partition head. `undefined` when nothing
+     * is stored. */
     async storedOffset(partitionId) {
         if (this.transport.getConsumerOffset === undefined) {
-            throw new UnsupportedError("the Apache Iggy client does not expose consumer offsets");
+            throw new UnsupportedError("the Apache Iggy client does not expose consumer offsets", {
+                surface: "stream"
+            });
         }
         const offsetTarget = this.target.kind === "group"
             ? { kind: "group", name: this.target.name }
@@ -188,24 +329,35 @@ export class Consumer {
         if (this.shuttingDown)
             return;
         this.shuttingDown = true;
+        if (this.commitTimer !== undefined) {
+            clearInterval(this.commitTimer);
+            this.commitTimer = undefined;
+        }
         if (this.reader !== undefined) {
-            this.handledPrevious();
-            this.records.length = 0;
-            await this.reader.close();
+            try {
+                await this.finishDelivery();
+            }
+            finally {
+                this.records.length = 0;
+                await this.reader.close();
+            }
             return;
         }
         try {
             if (this.autoCommit) {
                 for (const [partition, offset] of this.consumedOffsets) {
                     const explicit = this.explicitOffsets.get(partition);
+                    if (this.deletedAt.get(partition) === offset)
+                        continue;
                     if (explicit === undefined || explicit < offset) {
                         await this.transport.storeOffset(this.streamName, this.topicName, this.target, partition, offset);
+                        this.storedOffsets.set(partition, offset);
                     }
                 }
             }
         }
         finally {
-            if (this.target.kind === "group") {
+            if (this.leavesGroup && this.target.kind === "group") {
                 await this.transport.leaveConsumerGroup(this.streamName, this.topicName, this.target.name);
             }
         }
@@ -218,9 +370,17 @@ export class Consumer {
     // the prefix through it handled when the consumer commits on its own.
     take() {
         if (this.reader === undefined) {
-            const message = this.buffer.shift();
-            if (message !== undefined)
-                this.consumedOffsets.set(message.partitionId, message.offset);
+            const returned = this.returned.shift();
+            if (returned !== undefined) {
+                this.latestDelivered = returned;
+                return returned;
+            }
+            const polled = this.buffer.shift();
+            if (polled === undefined)
+                return undefined;
+            this.consumedOffsets.set(polled.partitionId, polled.offset);
+            const message = withJson(polled, polled.headersMalformed !== undefined);
+            this.latestDelivered = message;
             return message;
         }
         let record = this.records.shift();
@@ -229,51 +389,213 @@ export class Consumer {
         if (record === undefined)
             return undefined;
         if (this.autoCommit)
-            this.lastDelivery = record;
+            this.lastDelivery = { record, lastOfPage: this.records.length === 0 };
         this.consumedOffsets.set(record.partitionId, record.offset);
-        const message = {
-            payload: record.payload,
-            partitionId: record.partitionId,
-            offset: record.offset,
-            ...(record.timestampMicros !== undefined ? { timestampMicros: record.timestampMicros } : {}),
-            headers: record.headers
-        };
+        const message = record.message;
         this.delivered.set(message, record);
+        this.latestDelivered = message;
         return message;
     }
+    // The native commit policies that store after yielding. `polling` stores on
+    // the server during the poll itself, and a group reader stores its handled
+    // prefix on its own. A failed store never costs the caller the record: the
+    // offset stays pending, and the next yield, timer tick, or shutdown stores
+    // it, like the Rust consumer.
+    async afterYield(message) {
+        if (this.reader !== undefined)
+            return;
+        const policy = this.commitPolicy;
+        this.yieldedSinceStore += 1;
+        const store = policy.kind === "each" ||
+            policy.kind === "intervalOrEach" ||
+            ((policy.kind === "all" || policy.kind === "intervalOrAll") && this.buffer.length === 0) ||
+            ((policy.kind === "every" || policy.kind === "intervalOrEvery") &&
+                this.yieldedSinceStore >= policy.count);
+        if (!store)
+            return;
+        try {
+            await this.storeConsumed(message.partitionId);
+            this.yieldedSinceStore = 0;
+        }
+        catch {
+            // Stored on a later attempt.
+        }
+    }
+    async storeConsumed(only) {
+        for (const [partition, offset] of this.consumedOffsets) {
+            if (only !== undefined && partition !== only)
+                continue;
+            if ((this.storedOffsets.get(partition) ?? -1n) >= offset)
+                continue;
+            if (this.deletedAt.get(partition) === offset)
+                continue;
+            await this.transport.storeOffset(this.streamName, this.topicName, this.target, partition, offset);
+            this.storedOffsets.set(partition, offset);
+        }
+    }
+    startCommitTimer() {
+        const interval = commitInterval(this.commitPolicy);
+        if (interval === undefined || this.commitTimer !== undefined || this.reader !== undefined) {
+            return;
+        }
+        this.commitTimer = setInterval(() => {
+            this.storeConsumed().catch(() => undefined);
+        }, interval);
+        this.commitTimer.unref();
+    }
+    requireNative(operation) {
+        if (this.reader !== undefined) {
+            throw new InvalidError(`${operation}() is native only: a policy-aware group consumer commits delivered records`);
+        }
+    }
+    currentPartition(partitionId) {
+        const partition = partitionId ??
+            (this.target.kind === "single" ? this.target.partitionId : undefined) ??
+            this.latestDelivered?.partitionId;
+        if (partition === undefined || !Number.isSafeInteger(partition) || partition < 0) {
+            throw new InvalidError("name a partition: this consumer has no current partition yet");
+        }
+        return partition;
+    }
+    // One poll. A retryable transport failure waits the polling retry interval
+    // and reports an empty poll, so the read loop keeps going after a server
+    // restart, like the Rust consumer. The wait never runs past `deadline`.
+    async pollOnce(signal, deadline) {
+        this.startCommitTimer();
+        try {
+            this.polling ??= this.fillBuffer();
+            await this.waitForPoll(this.polling, signal, deadline);
+        }
+        catch (error) {
+            if (this.pollFailures === "propagate" ||
+                !(error instanceof TransportError) ||
+                !error.retryable ||
+                this.stopped()) {
+                throw error;
+            }
+            const wait = deadline === undefined
+                ? this.pollingRetryIntervalMs
+                : Math.min(this.pollingRetryIntervalMs, Math.max(0, deadline - Date.now()));
+            await delay(wait, signal);
+        }
+    }
+    waitForPoll(poll, signal, deadline) {
+        return new Promise((resolve, reject) => {
+            let waiting = true;
+            let timer;
+            const finish = (complete, observed) => {
+                if (!waiting)
+                    return;
+                waiting = false;
+                clearTimeout(timer);
+                signal?.removeEventListener("abort", onAbort);
+                if (observed && this.polling === poll)
+                    this.polling = undefined;
+                complete();
+            };
+            const onAbort = () => {
+                finish(() => {
+                    reject(new CancelledError("consumer poll aborted", { cause: signal?.reason }));
+                }, false);
+            };
+            poll.then(() => {
+                if (deadline !== undefined && Date.now() > deadline) {
+                    finish(() => {
+                        reject(new TimeoutError("the live consumer to yield a record"));
+                    }, true);
+                }
+                else {
+                    finish(resolve, true);
+                }
+            }, (error) => {
+                finish(() => {
+                    reject(error instanceof Error
+                        ? error
+                        : new TransportError("consumer poll failed", true, { cause: error }));
+                }, true);
+            });
+            signal?.addEventListener("abort", onAbort, { once: true });
+            if (signal?.aborted === true) {
+                onAbort();
+            }
+            else if (deadline !== undefined) {
+                const tick = () => {
+                    if (!waiting)
+                        return;
+                    const remaining = deadline - Date.now();
+                    if (remaining <= 0) {
+                        finish(() => {
+                            reject(new TimeoutError("the live consumer to yield a record"));
+                        }, false);
+                    }
+                    else {
+                        timer = setTimeout(tick, Math.min(remaining, 2_147_483_647));
+                    }
+                };
+                tick();
+            }
+        });
+    }
     pending() {
-        return this.reader === undefined ? this.buffer.length > 0 : this.records.length > 0;
+        return this.reader === undefined
+            ? this.buffer.length > 0 || this.returned.length > 0
+            : this.records.length > 0;
     }
     stopped() {
         return this.shuttingDown;
     }
     // Asking for the next record or shutting down confirms that the previous
-    // delivery reached the caller. A cancelled poll cannot complete a record
+    // delivery reached the caller, and the commit policy decides whether the
+    // handled prefix is stored now. A cancelled poll cannot complete a record
     // that the caller never received.
-    handledPrevious() {
-        if (this.lastDelivery === undefined || this.reader === undefined)
+    async finishDelivery() {
+        if (this.reader === undefined)
+            return;
+        const delivery = this.lastDelivery;
+        if (delivery !== undefined) {
+            this.lastDelivery = undefined;
+            this.reader.handled(delivery.record);
+            this.yieldedSinceFlush += 1;
+            this.pendingCommit = { kind: "delivered", lastOfPage: delivery.lastOfPage };
+        }
+        const point = this.pendingCommit;
+        if (point === undefined)
             return;
-        this.reader.handled(this.lastDelivery);
-        this.lastDelivery = undefined;
-        this.handledSincePoll = true;
+        await this.flushIfDue(this.reader, point);
+        this.pendingCommit = undefined;
     }
-    // One group poll. The delivered prefix is stored first, so a crash
-    // redelivers the current batch instead of skipping it.
-    async fillFromGroup(reader) {
-        if (this.handledSincePoll) {
-            try {
-                await reader.flushCompleted();
-            }
-            catch (error) {
-                // A policy change fences the prefix read under the old policy. Those
-                // records are delivered again from the stored offset, so nothing is
-                // lost and the consumer goes on without an error.
-                if (!(error instanceof FilterExecutionError) || error.reason !== "conflict")
-                    throw error;
-                this.records.length = 0;
-            }
-            this.handledSincePoll = false;
+    // Stores the handled prefix when the commit policy says so at this point.
+    async flushIfDue(reader, point) {
+        if (!commitDue(this.commitPolicy, this.yieldedSinceFlush, Date.now() - this.lastFlushAt, point)) {
+            return;
         }
+        try {
+            await reader.flushCompleted();
+        }
+        catch (error) {
+            const reason = error instanceof FilterExecutionError ? error.detail.reason : undefined;
+            if (reason !== "conflict" && reason !== "source_changed")
+                throw error;
+            const owned = this.records.filter((record) => reader.owns(record));
+            this.records.splice(0, this.records.length, ...owned);
+            this.pendingCommit = undefined;
+            this.yieldedSinceFlush = 0;
+            // A policy change fences the prefix read under the old policy. Those
+            // records are delivered again from the stored offset, so nothing is
+            // lost and the consumer goes on without an error.
+            if (reason !== "conflict")
+                throw error;
+            this.lastFlushAt = Date.now();
+            return;
+        }
+        this.yieldedSinceFlush = 0;
+        this.lastFlushAt = Date.now();
+    }
+    // One group poll. A policy that stores at the poll stores the delivered
+    // prefix first, so a crash redelivers the current batch instead of
+    // skipping it.
+    async fillFromGroup(reader) {
+        await this.flushIfDue(reader, { kind: "poll" });
         const [page, more] = await reader.readRound();
         this.moreScanned = more;
         if (page !== undefined)
@@ -290,7 +612,7 @@ export class Consumer {
         }
         let target = this.target;
         if (target.kind === "group" &&
-            (this.startFrom.kind !== "next" || !this.autoCommit) &&
+            (this.startAt.kind !== "next" || !this.autoCommit) &&
             this.transport.syncConsumerGroup !== undefined) {
             const assignment = await this.transport.syncConsumerGroup(this.streamName, this.topicName, target.name);
             if (assignment === undefined)
@@ -328,15 +650,76 @@ export class Consumer {
             : partition !== undefined
                 ? this.gainedPartitions.has(partition)
                     ? { kind: "next" }
-                    : this.startFrom
+                    : this.startAt
                 : this.started
                     ? { kind: "next" }
-                    : this.startFrom;
-        const polled = await this.transport.pollMessages(this.streamName, this.topicName, target, strategy, this.batchLength, this.autoCommit);
+                    : this.startAt;
+        const commitOnPoll = this.commitPolicy.kind === "polling" || this.commitPolicy.kind === "intervalOrPolling";
+        const polled = await this.transport.pollMessages(this.streamName, this.topicName, target, strategy, this.batchLength, commitOnPoll);
         this.started = true;
         for (const message of polled)
             this.nextOffsets.set(message.partitionId, message.offset + 1n);
-        this.buffer.push(...polled);
+        const fresh = this.allowReplay
+            ? polled
+            : polled.filter((message) => {
+                const consumed = this.consumedOffsets.get(message.partitionId);
+                return consumed === undefined || message.offset > consumed;
+            });
+        if (commitOnPoll) {
+            for (const message of fresh)
+                this.storedOffsets.set(message.partitionId, message.offset);
+        }
+        this.buffer.push(...fresh);
+    }
+}
+function withJson(message, headersMalformed) {
+    const json = (decodeValue) => new Json(decodeValue).decode(message.payload);
+    const position = { partitionId: message.partitionId, offset: message.offset };
+    const delivered = {
+        payload: message.payload,
+        id: position,
+        partitionId: message.partitionId,
+        headers: message.headers,
+        position,
+        messageId: message.messageId ?? 0n,
+        checksum: message.checksum ?? 0n,
+        currentOffset: message.currentOffset ?? 0n,
+        timestampMicros: message.timestampMicros ?? 0n,
+        originTimestampMicros: message.originTimestampMicros ?? 0n,
+        ...(message.userHeaders !== undefined ? { userHeaders: message.userHeaders } : {}),
+        headersMalformed
+    };
+    // Not enumerable, so a delivered message still compares equal to its plain
+    // fields.
+    return Object.defineProperty(delivered, "json", { value: json });
+}
+// Whether an automatic commit policy stores the handled prefix now, like the
+// Rust `commit_due`. `yielded` counts records delivered since the last store
+// and `elapsedMs` the time since it.
+function commitDue(policy, yielded, elapsedMs, point) {
+    const polling = point.kind === "poll" && yielded > 0;
+    const lastOfPage = point.kind === "delivered" && point.lastOfPage;
+    const delivered = point.kind === "delivered";
+    switch (policy.kind) {
+        case "disabled":
+            return false;
+        case "polling":
+            return polling;
+        case "interval":
+            return elapsedMs >= policy.intervalMs;
+        case "intervalOrPolling":
+            return polling || elapsedMs >= policy.intervalMs;
+        case "all":
+            return lastOfPage;
+        case "intervalOrAll":
+            return lastOfPage || elapsedMs >= policy.intervalMs;
+        case "each":
+        case "intervalOrEach":
+            return delivered && yielded > 0;
+        case "every":
+            return delivered && yielded >= policy.count;
+        case "intervalOrEvery":
+            return (delivered && yielded >= policy.count) || elapsedMs >= policy.intervalMs;
     }
 }
 //# sourceMappingURL=consumer.js.map