@smart-byte/leani-sdk 0.1.0-rc.3 → 0.1.0-rc.5

package/README.md

@@ -58,6 +58,10 @@ Destination writes must be idempotent because a crash between those steps can
 replay a change. Acknowledge a batch's `ackCursor` (or `acknowledgeableCursor`),
 never a change's own cursor; a backfill session's `deliveryBatches()` yields
 every such boundary with its changes, while `events()` yields changes only.
+`runBackfill(client, request, { onBatch })` runs one application-owned
+subscription to its end: it creates it, applies and acknowledges each batch,
+and deletes it once complete. After a crash or an abort, the identical request
+resumes it.
 
 The browser build assumes a same-origin deployment or a reverse proxy that
 adds the application's CORS policy. Leani does not enable cross-origin access

package/dist/backfill.d.ts

@@ -14,6 +14,16 @@ export interface BackfillStatus {
     id: string;
     owner: "subscription";
     processor: string;
+    /**
+     * Whether the node configures the processor instance `processor` now.
+     * Without it, the stream and acknowledgement routes refuse the
+     * subscription. That can be temporary; if the instance will not return,
+     * cancel the subscription and delete it with `discardUnacknowledged: true`,
+     * which discards its unacknowledged records.
+     */
+    processorConfigured: boolean;
+    /** The required consumer; tell your subscriptions apart by it, not by `id`. */
+    consumer: string | null;
     deliveryStreamId?: string;
     publicationRevision?: string;
     fromBlock: number;
@@ -332,4 +342,43 @@ export interface BackfillSubscriptionClient {
     subscribe<T = unknown>(request: MultiLaneSubscriptionRequest): Promise<MultiLaneDelivery<T>>;
 }
 export declare function createBackfillSubscriptionClient(options: BackfillSubscriptionClientOptions): BackfillSubscriptionClient;
+export type BackfillFailurePolicy = "retry" | "discard" | "throw";
+export interface RunBackfillOptions<T = unknown> {
+    /** Apply one history batch. It is acknowledged after this resolves. */
+    onBatch(batch: HistoryDeliveryBatch<T>): void | Promise<void>;
+    /** What a failed subscription does: retry it (default), delete it, or throw. */
+    onFailure?: BackfillFailurePolicy;
+    /** Retries before "retry" gives up and throws. Default 3. */
+    maxRetries?: number;
+    /** The consumer's credential; `request.consumer.credential` by default. */
+    credential?: string;
+    batching?: BackfillBatchingRequest;
+    /**
+     * Stops the run, as `request.signal` does too. The run rejects with the
+     * abort reason and keeps the subscription.
+     */
+    signal?: AbortSignal;
+}
+export interface RunBackfillResult {
+    outcome: "completed" | "cancelled" | "failed";
+    /** The subscription as it ended, before its deletion. */
+    subscription: BackfillStatus;
+    /** The completion record this run received, if it streamed one. */
+    completion?: BackfillStreamCompletion;
+    deletion: HistoricalWorkDeletion;
+}
+/**
+ * Run an application-owned history subscription to its end: create it,
+ * apply and acknowledge each history batch, and delete it once complete.
+ * Creation is idempotent by `idempotencyKey`, so running the identical
+ * request again, as after a crash, resumes the same subscription after its
+ * last acknowledged batch; a batch applied but not acknowledged is delivered
+ * again, so `onBatch` must be idempotent. After a lost connection or
+ * consumer session, it backs off, reads the subscription's state, and opens
+ * the stream again unless the subscription ended meanwhile. A failed
+ * subscription follows `onFailure`; a cancelled one is deleted with its
+ * unacknowledged records. An `onBatch` error, an abort, or any other failure
+ * closes the session and is thrown, leaving the subscription for a rerun.
+ */
+export declare function runBackfill<T = unknown>(client: BackfillSubscriptionClient, request: CreateBackfillSubscription, options: RunBackfillOptions<T>): Promise<RunBackfillResult>;
 export declare function parseNdjson(body: ReadableStream<Uint8Array>, signal?: AbortSignal): AsyncGenerator<unknown>;

package/dist/backfill.js

@@ -159,6 +159,162 @@ export function createBackfillSubscriptionClient(options) {
         subscribe: (request) => openMultiLaneDelivery(fetchImpl, baseUrl, options.token, timeoutMs, request),
     });
 }
+/**
+ * Run an application-owned history subscription to its end: create it,
+ * apply and acknowledge each history batch, and delete it once complete.
+ * Creation is idempotent by `idempotencyKey`, so running the identical
+ * request again, as after a crash, resumes the same subscription after its
+ * last acknowledged batch; a batch applied but not acknowledged is delivered
+ * again, so `onBatch` must be idempotent. After a lost connection or
+ * consumer session, it backs off, reads the subscription's state, and opens
+ * the stream again unless the subscription ended meanwhile. A failed
+ * subscription follows `onFailure`; a cancelled one is deleted with its
+ * unacknowledged records. An `onBatch` error, an abort, or any other failure
+ * closes the session and is thrown, leaving the subscription for a rerun.
+ */
+export async function runBackfill(client, request, options) {
+    const consumer = request.consumer;
+    if (consumer === undefined) {
+        throw new TypeError("runBackfill requires request.consumer, the consumer that acknowledges the history");
+    }
+    const maxRetries = options.maxRetries ?? 3;
+    if (!Number.isSafeInteger(maxRetries) || maxRetries < 0) {
+        throw new TypeError("maxRetries must be a non-negative safe integer");
+    }
+    // Either signal stops every request of the run.
+    const signal = request.signal && options.signal
+        ? AbortSignal.any([request.signal, options.signal])
+        : (options.signal ?? request.signal);
+    let subscription = await client.create({ ...request, signal });
+    let completion;
+    const finish = async (outcome, discardUnacknowledged) => ({
+        outcome,
+        subscription,
+        completion,
+        deletion: await client.delete(subscription.id, {
+            signal,
+            discardUnacknowledged,
+        }),
+    });
+    let retries = 0;
+    let delayMs = 1_000;
+    // Whether the subscription may have changed since its status was read.
+    let stale = false;
+    while (true) {
+        const { state } = subscription;
+        if (state === "complete_reclaimable" || state === "completed") {
+            return finish("completed", false);
+        }
+        if (state === "cancelled") {
+            return finish("cancelled", true);
+        }
+        if (state === "failed") {
+            if (options.onFailure === "discard") {
+                return finish("failed", true);
+            }
+            if (options.onFailure === "throw" || retries >= maxRetries) {
+                throw new BackfillStreamError({
+                    type: "error",
+                    code: "backfill_failed",
+                    message: subscription.lastError ?? "the historical subscription failed",
+                    earliestAvailableSequence: null,
+                    latestAvailableSequence: null,
+                });
+            }
+            retries += 1;
+            subscription = await client.retry(subscription.id, { signal });
+            continue;
+        }
+        try {
+            if (stale) {
+                subscription = await client.inspect(subscription.id, { signal });
+                stale = false;
+                continue;
+            }
+            const session = await client.stream(subscription.id, consumer.id, {
+                credential: options.credential ?? consumer.credential,
+                batching: options.batching,
+                signal,
+            });
+            delayMs = 1_000;
+            await applyHistory(session, options, signal, (received) => {
+                completion = received;
+            });
+        }
+        catch (error) {
+            if (error instanceof ApplicationFailure) {
+                throw error.cause;
+            }
+            const ended = error instanceof BackfillStreamError &&
+                (error.code === "backfill_failed" || error.code === "backfill_cancelled");
+            if (!ended) {
+                if (!isTransientStreamFailure(error)) {
+                    throw error;
+                }
+                await waitForReconnectDelay(delayMs, signal);
+                delayMs = Math.min(delayMs * 2, 30_000);
+            }
+            // The subscription's state decides what follows: it failed or was
+            // cancelled, or it may have ended while the stream was down, as when
+            // the node applied the completion's acknowledgement but its answer was
+            // lost. A stream opened again then would only send heartbeats.
+            stale = true;
+            continue;
+        }
+        subscription = await client.inspect(subscription.id, { signal });
+        return finish("completed", false);
+    }
+}
+/** What the application's `onBatch` threw: never a delivery failure to recover from. */
+class ApplicationFailure {
+    cause;
+    constructor(cause) {
+        this.cause = cause;
+    }
+}
+/**
+ * Apply and acknowledge each batch of a history session, then acknowledge
+ * its completion, which `received` gets first. The session is closed however
+ * this ends.
+ */
+async function applyHistory(session, options, signal, received) {
+    try {
+        for await (const batch of session.deliveryBatches()) {
+            if (batch.completion === undefined) {
+                try {
+                    await options.onBatch(batch);
+                }
+                catch (error) {
+                    throw new ApplicationFailure(error);
+                }
+            }
+            else {
+                received(batch.completion);
+            }
+            await session.acknowledge(batch.ackCursor, { signal });
+            if (batch.completion !== undefined) {
+                return;
+            }
+        }
+    }
+    finally {
+        await session.close();
+    }
+    // The caller's abort ends a session quietly.
+    signal?.throwIfAborted();
+    throw new TransportError("history stream ended before its completion");
+}
+/**
+ * Whether a history stream may be opened again: after a disconnect the SDK
+ * reports as retryable, or when the node reports, by HTTP error or stream
+ * record, that the consumer's session ended or another connection holds it.
+ */
+function isTransientStreamFailure(error) {
+    return (isRetryableDeliveryDisconnect(error) ||
+        (error instanceof BackfillStreamError &&
+            (error.code === "consumer_session_active" ||
+                error.code === "consumer_session_lost")));
+}
 async function openMultiLaneDelivery(fetchImpl, baseUrl, token, timeoutMs, request) {
     const processor = assertIdentity(request.processor, "processor");
     const consumer = assertIdentity(request.consumer, "consumer");
@@ -441,21 +597,21 @@ function isRetryableDeliveryDisconnect(error) {
             error.code === "consumer_session_lost"));
 }
 async function waitForReconnectDelay(delayMs, signal) {
-    if (signal.aborted) {
+    if (signal?.aborted) {
         throw signal.reason ?? new DOMException("Aborted", "AbortError");
     }
     await new Promise((resolve, reject) => {
         const finish = () => {
-            signal.removeEventListener("abort", abort);
+            signal?.removeEventListener("abort", abort);
             resolve();
         };
         const timeout = setTimeout(finish, delayMs);
         const abort = () => {
             clearTimeout(timeout);
-            signal.removeEventListener("abort", abort);
-            reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
+            signal?.removeEventListener("abort", abort);
+            reject(signal?.reason ?? new DOMException("Aborted", "AbortError"));
         };
-        signal.addEventListener("abort", abort, { once: true });
+        signal?.addEventListener("abort", abort, { once: true });
     });
 }
 function iterateUnifiedLanes(lanes, claim) {