effect-dynamodb 1.27.0 → 1.29.0

package/dist/change-feed/ChangeFeed.d.ts

@@ -0,0 +1,203 @@
+/**
+ * A polled change feed: an Effect `Stream` over a DynamoDB stream's records,
+ * for consumers outside Lambda and EventBridge Pipes (#156). See
+ * `docs/designs/change-feed.md`.
+ *
+ * @module
+ */
+import type { _Record } from "@aws-sdk/client-dynamodb-streams";
+import type { StreamRecordIdentifierMissing } from "@effect-dynamodb/schema/Errors.js";
+import { Duration, Effect, Stream } from "effect";
+import { type AnyEventProjection, type BatchItemFailuresResponse, type DecodableStream, type ProcessBatchesResult, type ProcessBatchOptions, type ProjectionRequirementsOf } from "../internal/EventProjection.js";
+import { CheckpointStore } from "./CheckpointStore.js";
+import { DynamoStreamsClient, type DynamoStreamsClientError } from "./DynamoStreamsClient.js";
+import { type ChangeFeedLeaseHeld, ChangeFeedLeaseLost, ChangeFeedTrimmed, type CheckpointStoreError, type ShardIteratorExpired, type StreamDataTrimmed } from "./Errors.js";
+/**
+ * A change-feed record: the SDK's `GetRecords` record, shaped like the record a
+ * Lambda event source mapping delivers (`eventID`, `eventName`,
+ * `dynamodb.SequenceNumber`, `dynamodb.NewImage` as DynamoDB JSON with `B` as
+ * bytes) — so `EventStore.processBatch` and the stream decoders take it as is.
+ */
+export type ChangeFeedRecord = _Record;
+/**
+ * One shard's records, in sequence order, read by one `GetRecords`. The shard
+ * is not read again until the batch is acknowledged with {@link
+ * ChangeFeedBatch.commit} or {@link ChangeFeedBatch.complete} — so a shard's
+ * batches are processed in order, and a consumer that never acknowledges a
+ * batch stalls its shard.
+ */
+export interface ChangeFeedBatch {
+    readonly streamArn: string;
+    readonly shardId: string;
+    /** Non-empty, in sequence order. */
+    readonly records: ReadonlyArray<ChangeFeedRecord>;
+    readonly firstSequenceNumber: string;
+    readonly lastSequenceNumber: string;
+    /**
+     * Acknowledge every record: commit the shard's checkpoint at
+     * `lastSequenceNumber`, and let the shard read on. Only the first
+     * acknowledgement that commits counts; one whose write failed can be
+     * retried (the shard waits meanwhile). Fails with `ChangeFeedLeaseLost`,
+     * writing nothing, once the feed has lost its lease.
+     */
+    readonly commit: () => Effect.Effect<void, CheckpointStoreError | ChangeFeedLeaseLost>;
+    /**
+     * Acknowledge per a partial batch response whose identifiers are
+     * `SequenceNumber`s (`source: "lambda"`): with no failures, {@link commit};
+     * otherwise commit the checkpoint at the record before the earliest failed
+     * one (none when that is the first record), and read the shard again from
+     * the failed record after a backoff. An identifier that names no record of
+     * the batch fails the whole batch, as Lambda does.
+     */
+    readonly complete: (response: BatchItemFailuresResponse) => Effect.Effect<void, CheckpointStoreError | ChangeFeedLeaseLost>;
+}
+/** Where a shard without a checkpoint starts, at a consumer's first start. */
+export type StartingPosition = "TRIM_HORIZON" | "LATEST";
+/** The feed's lease — see {@link ChangeFeedOptions.lease}. */
+export interface ChangeFeedLeaseOptions {
+    /** How long a lease lasts unless renewed. Default 30 s. */
+    readonly duration?: Duration.Input | undefined;
+    /**
+     * How often the holder renews it. Default a third of `duration`. Must be
+     * less than half of `duration` (`[EDD-9083]` otherwise): a renewal the store
+     * fails to write is retried until `renewEvery` before the lease would lapse,
+     * which leaves `duration − 2 × renewEvery` to retry in.
+     */
+    readonly renewEvery?: Duration.Input | undefined;
+    /**
+     * A lease held by another process: `"fail"` with `ChangeFeedLeaseHeld`
+     * (default), or `"wait"` — try again every `renewEvery` until it is
+     * released or expires.
+     */
+    readonly onHeld?: "fail" | "wait" | undefined;
+    /** This process's owner id. Default: a random UUID. */
+    readonly owner?: string | undefined;
+}
+/** Options of {@link changeFeed}. */
+export interface ChangeFeedOptions {
+    /** The stream to read (`TableDescription.LatestStreamArn`). */
+    readonly streamArn: string;
+    /**
+     * The application consuming the stream: checkpoints and the lease are kept
+     * per consumer, so several applications can read one stream with one
+     * checkpoint store. Default `"default"`.
+     */
+    readonly consumer?: string | undefined;
+    /**
+     * Where shards without a checkpoint start when the consumer has no
+     * checkpoints at all for the stream. `"TRIM_HORIZON"` (default): every
+     * record still retained. `"LATEST"`: the shards open now start after their
+     * latest record, and closed shards are checkpointed as finished unread.
+     * Every other shard without a checkpoint — a child found later, or any shard
+     * after a restart — starts at its trim horizon.
+     */
+    readonly startingPosition?: StartingPosition | undefined;
+    /** How long an idle shard waits between empty reads. Default 1 s. */
+    readonly pollInterval?: Duration.Input | undefined;
+    /** Records per batch (`GetRecords.Limit`, 1–1000). Default 100. */
+    readonly batchSize?: number | undefined;
+    /** How often the whole stream is described again for new shards. Default 30 s. */
+    readonly describeInterval?: Duration.Input | undefined;
+    /**
+     * How many shard reads (`GetShardIterator` / `GetRecords` calls) may be in
+     * flight at once, across all shards. Every shard keeps its own reader and
+     * takes turns, so a bound below the number of open shards slows reading
+     * rather than starving any shard. Default 16.
+     */
+    readonly maxConcurrentReads?: number | undefined;
+    /**
+     * Backoff for throttling and transient errors (retried without limit) and
+     * for re-reading a batch with failed records: jittered, exponential from
+     * `initial` (default 100 ms) up to `max` (default 10 s).
+     */
+    readonly backoff?: {
+        readonly initial?: Duration.Input | undefined;
+        readonly max?: Duration.Input | undefined;
+    } | undefined;
+    /**
+     * One lease per consumer and stream, held while the feed runs, so a second
+     * process can't consume the same feed by accident. `false` disables it.
+     * Default: enabled, with the defaults of {@link ChangeFeedLeaseOptions}.
+     */
+    readonly lease?: ChangeFeedLeaseOptions | false | undefined;
+}
+/** What a change feed fails with. */
+export type ChangeFeedError = Exclude<DynamoStreamsClientError, ShardIteratorExpired | StreamDataTrimmed> | ChangeFeedTrimmed | ChangeFeedLeaseHeld | ChangeFeedLeaseLost | CheckpointStoreError;
+/**
+ * A change feed over `options.streamArn`: a `Stream` of {@link ChangeFeedBatch}es.
+ *
+ * - **Shards** are found with `DescribeStream` (paginated), at start and every
+ *   `describeInterval`; a drained shard's children are looked up with
+ *   `ShardFilter` (`CHILD_SHARDS`), falling back to a full describe where the
+ *   filter is unsupported (DynamoDB Local) or finds nothing yet.
+ * - **A parent shard is read to its end before its children**, so one item's
+ *   records stay in order across a split. A closed shard is drained, its last
+ *   batch acknowledged, checkpointed as finished, and dropped.
+ * - **Shards are read concurrently**, each in order with one batch in flight:
+ *   a shard reads again only once its batch is acknowledged, so a slow
+ *   consumer slows polling.
+ *   `maxConcurrentReads` bounds the reads in flight across shards.
+ * - **Checkpoints** move only when a batch is acknowledged — at least once.
+ *   A shard resumes after its checkpoint (`AFTER_SEQUENCE_NUMBER`). An expired
+ *   iterator is re-acquired from the shard's position; a position past the
+ *   stream's retention fails with `ChangeFeedTrimmed`.
+ * - **Throttling and transient errors** retry with jittered backoff. An idle
+ *   shard polls every `pollInterval`.
+ * - **The lease** keeps a second process from consuming the same feed.
+ *
+ * The stream ends when the stream is disabled and every shard is drained;
+ * otherwise it runs until interrupted.
+ *
+ * Throws `[EDD-9083]` for options it can't honour: `lease.renewEvery` of half
+ * `lease.duration` or more, a `batchSize` outside 1–1000, a
+ * `maxConcurrentReads` that is not a positive integer.
+ *
+ * @example
+ * ```ts
+ * const feed = changeFeed({ streamArn, startingPosition: "TRIM_HORIZON" })
+ * yield* feed.pipe(
+ *   Stream.mapEffect((batch) => handle(batch.records).pipe(Effect.andThen(batch.commit()))),
+ *   Stream.runDrain,
+ * )
+ * ```
+ */
+export declare const changeFeed: (options: ChangeFeedOptions) => Stream.Stream<ChangeFeedBatch, ChangeFeedError, DynamoStreamsClient | CheckpointStore>;
+/** Options of {@link runChangeFeed}. */
+export type RunChangeFeedOptions<Projections extends {
+    readonly [name: string]: AnyEventProjection;
+}, Streams extends {
+    readonly [name: string]: DecodableStream;
+}, E2, R2> = ProcessBatchOptions<ChangeFeedRecord> & {
+    /** The projections to apply, as `EventStore.processBatch` takes them. */
+    readonly projections: Projections;
+    /** Streams to decode without a projection — their events come back as `unrouted`. */
+    readonly streams?: Streams | undefined;
+    /**
+     * Called with each batch and its result before the batch is acknowledged —
+     * to act on `applied` events or `unrouted` ones. Its failure ends the run
+     * before the acknowledgement, so the batch is read again on restart.
+     */
+    readonly onBatch?: ((batch: ChangeFeedBatch, result: ProcessBatchesResult<Projections, ChangeFeedRecord, Streams>) => Effect.Effect<void, E2, R2>) | undefined;
+};
+/**
+ * Run a change feed into projections: each batch goes through
+ * `EventStore.processBatch(batch.records, { projections, source: "lambda" })`,
+ * then `batch.complete(result.response)` — the checkpoint moves past the
+ * batch, or to the record before its earliest failed record, which the shard
+ * reads again. Batches of different shards are processed concurrently.
+ *
+ * Runs until the feed ends (its stream disabled and drained) or fails.
+ *
+ * @example
+ * ```ts
+ * yield* runChangeFeed(changeFeed({ streamArn }), {
+ *   projections: { summaries: OrderSummaryProjection },
+ * })
+ * ```
+ */
+export declare const runChangeFeed: <const Projections extends {
+    readonly [name: string]: AnyEventProjection;
+}, E, R, const Streams extends {
+    readonly [name: string]: DecodableStream;
+} = {}, E2 = never, R2 = never>(feed: Stream.Stream<ChangeFeedBatch, E, R>, options: RunChangeFeedOptions<Projections, Streams, E2, R2>) => Effect.Effect<void, E | E2 | CheckpointStoreError | ChangeFeedLeaseLost | StreamRecordIdentifierMissing, R | R2 | ProjectionRequirementsOf<Projections[keyof Projections]>>;
+//# sourceMappingURL=ChangeFeed.d.ts.map

package/dist/change-feed/ChangeFeed.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChangeFeed.d.ts","sourceRoot":"","sources":["../../src/change-feed/ChangeFeed.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EACV,OAAO,EAIR,MAAM,kCAAkC,CAAA;AACzC,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,mCAAmC,CAAA;AACtF,OAAO,EAIL,QAAQ,EACR,MAAM,EAQN,MAAM,EACP,MAAM,QAAQ,CAAA;AACf,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EACxB,KAAK,wBAAwB,EAE9B,MAAM,gCAAgC,CAAA;AACvC,OAAO,EAAE,eAAe,EAAqC,MAAM,sBAAsB,CAAA;AACzF,OAAO,EAAE,mBAAmB,EAAE,KAAK,wBAAwB,EAAE,MAAM,0BAA0B,CAAA;AAC7F,OAAO,EACL,KAAK,mBAAmB,EACxB,mBAAmB,EACnB,iBAAiB,EACjB,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACvB,MAAM,aAAa,CAAA;AAMpB;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAA;AAEtC;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,oCAAoC;IACpC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,gBAAgB,CAAC,CAAA;IACjD,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAA;IACpC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,oBAAoB,GAAG,mBAAmB,CAAC,CAAA;IACtF;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAQ,EAAE,CACjB,QAAQ,EAAE,yBAAyB,KAChC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,oBAAoB,GAAG,mBAAmB,CAAC,CAAA;CACrE;AAED,8EAA8E;AAC9E,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG,QAAQ,CAAA;AAExD,8DAA8D;AAC9D,MAAM,WAAW,sBAAsB;IACrC,2DAA2D;IAC3D,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IAC9C;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IAChD;;;;OAIG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;IAC7C,uDAAuD;IACvD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACpC;AAED,qCAAqC;AACrC,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC;;;;;;;OAOG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;IACxD,qEAAqE;IACrE,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IAClD,mEAAmE;IACnE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACvC,kFAAkF;IAClF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IACtD;;;;;OAKG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAChD;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EACb;QAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAC;QAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;KAAE,GAC5F,SAAS,CAAA;IACb;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,sBAAsB,GAAG,KAAK,GAAG,SAAS,CAAA;CAC5D;AAED,qCAAqC;AACrC,MAAM,MAAM,eAAe,GACvB,OAAO,CAAC,wBAAwB,EAAE,oBAAoB,GAAG,iBAAiB,CAAC,GAC3E,iBAAiB,GACjB,mBAAmB,GACnB,mBAAmB,GACnB,oBAAoB,CAAA;AA4BxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,eAAO,MAAM,UAAU,GACrB,SAAS,iBAAiB,KACzB,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,eAAe,EAAE,mBAAmB,GAAG,eAAe,CAUvF,CAAA;AA4gBD,wCAAwC;AACxC,MAAM,MAAM,oBAAoB,CAC9B,WAAW,SAAS;IAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,kBAAkB,CAAA;CAAE,EACnE,OAAO,SAAS;IAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,eAAe,CAAA;CAAE,EAC5D,EAAE,EACF,EAAE,IACA,mBAAmB,CAAC,gBAAgB,CAAC,GAAG;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IACjC,qFAAqF;IACrF,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACtC;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EACb,CAAC,CACC,KAAK,EAAE,eAAe,EACtB,MAAM,EAAE,oBAAoB,CAAC,WAAW,EAAE,gBAAgB,EAAE,OAAO,CAAC,KACjE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,GACjC,SAAS,CAAA;CACd,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,aAAa,GACxB,KAAK,CAAC,WAAW,SAAS;IAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,kBAAkB,CAAA;CAAE,EACzE,CAAC,EACD,CAAC,EACD,KAAK,CAAC,OAAO,SAAS;IAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,eAAe,CAAA;CAAE,GAAG,EAAE,EACvE,EAAE,GAAG,KAAK,EACV,EAAE,GAAG,KAAK,EAEV,MAAM,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC,EAC1C,SAAS,oBAAoB,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,CAAC,KAC1D,MAAM,CAAC,MAAM,CACd,IAAI,EACJ,CAAC,GAAG,EAAE,GAAG,oBAAoB,GAAG,mBAAmB,GAAG,6BAA6B,EACnF,CAAC,GAAG,EAAE,GAAG,wBAAwB,CAAC,WAAW,CAAC,MAAM,WAAW,CAAC,CAAC,CAuBlE,CAAA"}