@kensio/yulin 1.21.8 → 1.21.10

package/dist/service/lambda/event-source/sim-lambda-stream-retry-limits.js

@@ -23,17 +23,23 @@ const maximumRecordAgeLimit = 604_800;
 const defaultAttemptLimit = 5;
 const millisecondsPerSecond = 1000;
 /**
- * When a stream mapping stops delivering a batch its function keeps failing.
+ * What a stream mapping does with a batch its function keeps failing.
  *
- * The two limits are one thing rather than two, because they end the same
- * lifecycle: a record leaves the mapping when it has had its retries or when it
- * is too old to be worth another, whichever comes first. Both are held as
- * Lambda states them, with `-1` for no limit, so a Get or a List reports back
- * exactly what was asked for.
+ * The three settings are one thing rather than three, because they run the same
+ * lifecycle. A failing batch is split around the record that broke it where the
+ * mapping was asked to bisect, and a record leaves the mapping when it has had
+ * its retries or when it is too old to be worth another, whichever comes first.
+ * The limits are held as Lambda states them, with `-1` for no limit, so a Get
+ * or a List reports back exactly what was asked for.
  */
 export class SimLambdaStreamRetryLimits {
     maximumRetryAttempts;
     maximumRecordAgeInSeconds;
+    /**
+     * Whether a batch the function threw on is split in half and delivered again
+     * as two batches.
+     */
+    bisectBatchOnFunctionError;
     constructor(input = {}) {
         this.maximumRetryAttempts = checkedLimit({
             value: input.maximumRetryAttempts,
@@ -47,14 +53,16 @@ export class SimLambdaStreamRetryLimits {
             maximum: maximumRecordAgeLimit,
             unit: "seconds",
         });
+        this.bisectBatchOnFunctionError = input.bisectBatchOnFunctionError ?? false;
     }
     /**
-     * The two limits as a Get, a List or a create response reports them.
+     * The three settings as a Get, a List or a create response reports them.
      */
     configuration() {
         return {
             MaximumRetryAttempts: this.maximumRetryAttempts,
             MaximumRecordAgeInSeconds: this.maximumRecordAgeInSeconds,
+            BisectBatchOnFunctionError: this.bisectBatchOnFunctionError,
         };
     }
     /**

package/dist/service/lambda/event-source/stream/kinesis/sim-kinesis-event-source-streams.js

@@ -40,6 +40,7 @@ export class SimKinesisEventSourceStreams {
         }, { caller });
         return {
             records: output.Records,
+            shardId: request.shardId,
             next: { kind: "iterator", shardIterator: output.NextShardIterator },
             drained: false,
         };

package/dist/service/lambda/event-source/stream/sim-dynamodb-event-source-stream-reader.js

@@ -29,6 +29,7 @@ export class SimDynamoDbEventSourceStreamReader {
         const next = output.NextShardIterator;
         return {
             records: output.Records ?? [],
+            shardId: await this.shard.shardId(request),
             next: simDynamoDbEventSourceNextPosition(next, position),
             drained: next === undefined,
         };

package/dist/service/lambda/event-source/stream/sim-dynamodb-event-source-stream-shard.d.ts

@@ -23,5 +23,6 @@ export declare class SimDynamoDbEventSourceStreamShard {
      * is the shard.
      */
     iteratorFor(request: SimLambdaEventSourceStreamRequest, position: SimLambdaEventSourceStreamNamedPosition): Promise<string>;
+    shardId(request: SimLambdaEventSourceStreamRequest): Promise<string>;
     private describe;
 }

package/dist/service/lambda/event-source/stream/sim-dynamodb-event-source-stream-shard.js

@@ -49,6 +49,13 @@ export class SimDynamoDbEventSourceStreamShard {
         }
         return iterator.ShardIterator;
     }
+    async shardId(request) {
+        const description = await this.describe(request);
+        const shardId = description.Shards?.[0]?.ShardId;
+        if (shardId === undefined)
+            refuse(request, "reports no shard to read records from");
+        return shardId;
+    }
     async describe(request) {
         const { caller, streamArn } = request;
         const described = await this.commands.describeStream({ input: { StreamArn: streamArn } }, { caller });

package/dist/service/lambda/event-source/stream/sim-lambda-event-source-streams.d.ts

@@ -84,6 +84,7 @@ export interface SimLambdaEventSourceStreamReadRequest extends SimLambdaEventSou
 export interface SimLambdaEventSourceStreamProgressBatch {
     readonly next: SimLambdaEventSourceStreamPosition;
     readonly drained: boolean;
+    readonly shardId?: string;
 }
 /**
  * What one read of a DynamoDB stream came back with.

package/dist/service/lambda/index.d.ts

@@ -16,3 +16,4 @@ export type { SimLambdaDynamoDbAttributeValue, SimLambdaDynamoDbImage, } from ".
 export { SimLambdaStreamCascadeError } from "./event-source/stream/sim-lambda-stream-cascade.error.js";
 export type { SimLambdaDestinationRecord, SimLambdaDestinationRequestContext, SimLambdaDestinationResponseContext, } from "./destination/sim-lambda-destination-record.js";
 export type { SimLambdaInvocationCondition } from "./function/event-invoke/sim-lambda-event-invoke-config.js";
+export type { SimLambdaStreamFailureRecord, SimLambdaStreamFailureCondition, } from "./event-source/poll/sim-lambda-stream-failure-record.js";

package/dist/service/lambda/sim-lambda-commands.js

@@ -113,6 +113,7 @@ export class SimLambdaCommands {
             background,
         });
         this.eventSourceMappings = new SimLambdaEventSourceMappingCommands({
+            destinations,
             accountRegionScope,
             pollers: new SimLambdaEventSourcePollers({
                 functions: this.functionLookup,

package/docs/services/lambda/README.md

@@ -1904,7 +1904,7 @@ with more steps.
 `MaximumRetryAttempts` and `MaximumRecordAgeInSeconds` govern the same failed-batch lifecycle, and a
 stream mapping keeps both. `MaximumRetryAttempts: 0` makes one delivery and no retries, and a
 positive value allows that many retries after the first delivery. `MaximumRecordAgeInSeconds`
-discards a record once the next attempt would fall past that age. Lambda's `-1` means no limit
+discards expired records before each invocation, including the first poll. Lambda's `-1` means no limit
 (that is what a mapping naming neither reports back).
 
 ```typescript sim-lambda-stream-retry-limits
@@ -2033,6 +2033,93 @@ discarded, and the mapping reads on from behind it.
 10,000 retries, `-1` to 604,800 seconds) is a `ValidationException`. A queue mapping takes neither,
 because a message the handler never takes is left to the queue's own redrive policy.
 
+### Splitting a failed batch around the record that broke it
+
+`BisectBatchOnFunctionError` splits a batch the handler threw on in half, and delivers each half as
+its own batch. A half that fails is split again, down to a single record. That is how the records
+beside a poison record get through, and how the poison record ends up delivered on its own and
+discarded on its own.
+
+```json
+{
+  "StartingPosition": "TRIM_HORIZON",
+  "BatchSize": 100,
+  "MaximumRetryAttempts": 10,
+  "BisectBatchOnFunctionError": true
+}
+```
+
+A batch of four whose third record the handler cannot take is delivered as `1, 2, 3, 4`, then
+`1, 2`, then `3, 4`, then `3` on its own, and finally `4`. The record that broke every batch it was
+in leaves the mapping under the retry and record-age limits, and reaches the
+[failure destination](#sending-discarded-stream-batches-to-a-destination) naming one record rather
+than four.
+
+Splitting puts the retry count back to the start, so a batch always reaches a single record before
+the count decides anything. Once it is down to one record, the limits count as they do for any other
+failing batch. A batch the handler reported partial failures on is left whole, because the report
+already says which record to go back to.
+
+The halves carry the batch that failed and nothing else. A record written while the splitting is
+going on waits behind it and is delivered as part of the batch after it, and the mapping reads at its
+own batch size again once the whole of the split batch is through.
+
+`GetEventSourceMapping` and `ListEventSourceMappings` report the setting, and
+`AWS::Lambda::EventSourceMapping` takes it in a template. Both stream sources have it. A queue
+mapping is refused for naming it, the way real Lambda refuses one.
+
+### Sending discarded stream batches to a destination
+
+DynamoDB Streams and Kinesis mappings accept `DestinationConfig.OnFailure` with a standard SQS
+queue or SNS topic ARN. Supply it to `CreateEventSourceMapping` or the
+`AWS::Lambda::EventSourceMapping` resource alongside the retry and record-age limits:
+
+```json
+{
+  "MaximumRetryAttempts": 10,
+  "MaximumRecordAgeInSeconds": 3600,
+  "DestinationConfig": {
+    "OnFailure": {
+      "Destination": "arn:aws:sqs:eu-west-2:111111111111:stream-failures"
+    }
+  }
+}
+```
+
+CDK's `DynamoEventSource` and `KinesisEventSource` accept `onFailure: new SqsDlq(queue)` or
+`onFailure: new SnsDlq(topic)`. Their synthesized destination configuration is passed through to
+the simulated mapping. Create, Get, List and Delete responses preserve that configuration.
+
+The function's execution role needs `sqs:SendMessage` on the queue or `sns:Publish` on the topic.
+Delivery goes through the simulated destination service, so its IAM checks and SNS subscriptions
+apply. An IAM denial or a missing destination rejects `backgroundTasksComplete()` or the clock
+advance that exhausted the batch. The discarded records have already advanced the checkpoint.
+Yulin attempts destination delivery once and does not retry a failed send.
+
+The JSON message follows the AWS
+[DynamoDB Streams](https://docs.aws.amazon.com/lambda/latest/dg/services-dynamodb-errors.html) and
+[Kinesis](https://docs.aws.amazon.com/lambda/latest/dg/kinesis-on-failure-destination.html)
+notification formats. Import `SimLambdaStreamFailureRecord` from `@kensio/yulin/lambda` to type it.
+
+- `version` is `"1.0"` and `timestamp` is the simulated discard time in ISO 8601 format.
+- `requestContext` contains a generated `requestId`, the mapping's `functionArn`,
+  `approximateInvokeCount`, and either `RetryAttemptsExhausted` or `RecordAgeExceeded` as `condition`.
+- `responseContext` reports `statusCode: 200` and `executedVersion`. A handler error adds
+  `functionError: "Unhandled"`. A valid partial-batch failure response leaves that field out.
+  Records discarded before any invocation omit `responseContext`.
+- `DDBStreamBatchInfo` or `KinesisBatchInfo` identifies the stream ARN, shard ID, first and last
+  sequence numbers, first and last arrival times, and batch size. Arrival times are ISO 8601 strings.
+
+The original record payloads are absent. A successful batch produces no notification. After a
+partial-batch response, the notification covers the discarded suffix starting at the failed
+checkpoint. When only an older prefix expires, its notification excludes the younger records.
+Records already expired on their first poll report zero invocations. Request IDs are generated for
+the notification and are not correlated with the handler's invocation context.
+
+S3 destinations, FIFO queues or topics, and `OnSuccess` are refused. SQS source mappings cannot
+have this destination configuration. The simulator's existing five-retry cap for mappings with
+neither limit also produces a failure notification when it discards a batch.
+
 ### Reporting individual record failures
 
 A stream mapping with `FunctionResponseTypes: ["ReportBatchItemFailures"]` reads the handler's
@@ -2272,9 +2359,15 @@ this, and deploys without hand-editing. The grant CDK writes alongside it is an
 the three stream actions on the stream ARN and `dynamodb:ListStreams` on every stream, exactly what
 the mapping's execution-role check is looking for.
 
-The properties a non-default `DynamoEventSource` adds are refused by name. Those are
-`FilterCriteria`, `ParallelizationFactor`, `BisectBatchOnFunctionError`, `TumblingWindowInSeconds`
-and `DestinationConfig`.
+`bisectBatchOnError` is simulated, and is covered under
+[Splitting a failed batch around the record that broke it](#splitting-a-failed-batch-around-the-record-that-broke-it).
+The other properties a non-default `DynamoEventSource` adds are recorded rather than acted on. Those
+are `FilterCriteria`, `ParallelizationFactor` and `TumblingWindowInSeconds`.
+The mapping deploys, delivers every record whole and unfiltered, and each property it was created
+without is listed in
+[`stack.ignoredProperties`](https://yulinsim.dev/services/cloudformation/#properties-a-resource-was-created-without "Properties a Resource was created without")
+with what the mapping does in its place. `CreateEventSourceMapping` still refuses the same
+properties, since a caller naming one is asking for behaviour by hand.
 
 A hand-written template or a SAM application usually gives the function the AWS managed policy
 `AWSLambdaDynamoDBExecutionRole` instead. Simulated IAM has no model for managed policy ARNs, so
@@ -4154,8 +4247,10 @@ Current documented limitations:
 - Throttling does not drive a retry. A retry follows a handler that threw or ran out of time, and
   the `MaximumEventAgeInSeconds` a config carries is measured from when the invocation was
   accepted.
-- `DestinationConfig` on an event source mapping is left out. It is a different mechanism from the
-  asynchronous invocation destinations above.
+- Stream mappings support standard SQS and SNS `DestinationConfig.OnFailure` destinations.
+  S3 destinations and `OnSuccess` are refused. Destination delivery is attempted once. A delivery
+  failure rejects the background task after the discarded records have advanced the checkpoint.
+  Destination delivery retries and `DestinationDeliveryFailures` metrics are not simulated.
 - A settings change takes effect at once. Real Lambda reports `LastUpdateStatus: "InProgress"` while
   it rolls the change out, and neither that member nor the wait it implies is simulated.
 - A cross-account grant is only half of what admits a call. The caller's own Account has to allow
@@ -4244,11 +4339,12 @@ Current documented limitations:
   function, and `S3ObjectVersion` on `UpdateFunctionCode`. A versioned location loads the object as
   it stands.
 - SQS queues, DynamoDB streams and Kinesis streams are the only event sources. Kafka, DocumentDB and
-  Kinesis enhanced fan-out consumers are refused outright, and so are `FilterCriteria`,
-  `ScalingConfig`, `DestinationConfig`, `BisectBatchOnFunctionError`, `ParallelizationFactor`,
-  `TumblingWindowInSeconds` and the other mapping inputs this simulation has no behaviour for. An
-  `AWS::Lambda::EventSourceMapping` carrying `Tags` is the exception, and deploys with the tags
-  dropped and the property recorded.
+  Kinesis enhanced fan-out consumers are refused outright. `CreateEventSourceMapping` also refuses
+  `FilterCriteria`, `ScalingConfig`, `ParallelizationFactor`, `TumblingWindowInSeconds` and the
+  other inputs this simulation has no behaviour for. An
+  `AWS::Lambda::EventSourceMapping` naming any of them deploys instead, and records each one against
+  the Resource (see
+  [Properties a Resource was created without](https://yulinsim.dev/services/cloudformation/#properties-a-resource-was-created-without "Properties a Resource was created without")).
 - A failed stream batch waits 1, 2, 4, 8 and 16 seconds between attempts, where AWS documents no
   delay. That is deliberate. A delay of zero falls due at the instant the clock already reads, and a
   handler that always throws would leave `advanceBy` with work falling due forever. A mapping that
@@ -4256,6 +4352,10 @@ Current documented limitations:
   discards the batch, where AWS goes on until the records age out a day later. A batch item failure
   report counts against the same retries, and never starts them again for the records it rewound
   to.
+- Splitting a batch under `BisectBatchOnFunctionError` puts the retry count back to the start, where
+  AWS counts a bisected batch's deliveries against the same quota. Without that, the simulator's own
+  cap of five attempts would discard a batch of a hundred long before it was down to one record. The
+  splitting still ends on its own, because a batch halves at every step.
 - A handler writing into the table whose stream invoked it is refused with
   `SimLambdaStreamCascadeError` rather than being delivered its own writes forever. Real Lambda runs
   that loop.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kensio/yulin",
-  "version": "1.21.8",
+  "version": "1.21.10",
   "description": "AWS system behaviour simulation for isolated unit testing",
   "repository": "https://github.com/KensioSoftware/yulin",
   "homepage": "https://yulinsim.dev/",