@kensio/yulin 1.21.9 → 1.21.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/dist/service/lambda/cfn/event-source-mapping/sim-cfn-lambda-event-source-mapping-properties.d.ts +1 -1
  2. package/dist/service/lambda/cfn/event-source-mapping/sim-cfn-lambda-event-source-mapping-properties.js +4 -3
  3. package/dist/service/lambda/cfn/event-source-mapping/sim-cfn-lambda-event-source-mapping-property-names.d.ts +30 -0
  4. package/dist/service/lambda/cfn/event-source-mapping/sim-cfn-lambda-event-source-mapping-property-names.js +91 -0
  5. package/dist/service/lambda/cfn/event-source-mapping/sim-cfn-lambda-event-source-mapping-property-rules.d.ts +7 -12
  6. package/dist/service/lambda/cfn/event-source-mapping/sim-cfn-lambda-event-source-mapping-property-rules.js +12 -92
  7. package/dist/service/lambda/command/event-source-mapping/create-event-source-mapping-input.js +1 -0
  8. package/dist/service/lambda/command/event-source-mapping/create-event-source-mapping-refusals.js +0 -5
  9. package/dist/service/lambda/event-source/poll/kinesis/sim-lambda-kinesis-shard-poller.js +1 -1
  10. package/dist/service/lambda/event-source/poll/sim-lambda-dynamodb-stream-polling.js +1 -1
  11. package/dist/service/lambda/event-source/poll/sim-lambda-stream-bisect.d.ts +53 -0
  12. package/dist/service/lambda/event-source/poll/sim-lambda-stream-bisect.js +79 -0
  13. package/dist/service/lambda/event-source/poll/sim-lambda-stream-progress-state.d.ts +2 -0
  14. package/dist/service/lambda/event-source/poll/sim-lambda-stream-progress-state.js +3 -0
  15. package/dist/service/lambda/event-source/poll/sim-lambda-stream-progress.d.ts +14 -1
  16. package/dist/service/lambda/event-source/poll/sim-lambda-stream-progress.js +24 -2
  17. package/dist/service/lambda/event-source/sim-lambda-event-source-retry-limits.d.ts +8 -7
  18. package/dist/service/lambda/event-source/sim-lambda-event-source-retry-limits.js +6 -3
  19. package/dist/service/lambda/event-source/sim-lambda-stream-retry-limits.d.ts +15 -7
  20. package/dist/service/lambda/event-source/sim-lambda-stream-retry-limits.js +15 -7
  21. package/docs/services/lambda/README.md +54 -7
  22. package/package.json +1 -1
@@ -19,7 +19,7 @@ export declare class SimCfnLambdaEventSourceMappingProperties {
19
19
  private readonly parser;
20
20
  constructor(properties: SimCfnLambdaEventSourceMappingPropertiesProperties);
21
21
  /**
22
- * The create input this Resource asks for, refusing what is not simulated.
22
+ * The create input this Resource asks for, recording what is left out of it.
23
23
  *
24
24
  * The event source ARN is read before anything else is judged, as the command
25
25
  * reads it first for the same reason: what a mapping may ask for depends on
@@ -1,6 +1,6 @@
1
1
  import { simLambdaStreamDestinationConfig } from "../../event-source/sim-lambda-stream-destination-config.js";
2
2
  import { SimCfnLambdaPropertyParser } from "../function/sim-cfn-lambda-property-parser.js";
3
- import { assertSimulatedEventSourceMappingProperties } from "./sim-cfn-lambda-event-source-mapping-property-rules.js";
3
+ import { recordUnsimulatedEventSourceMappingProperties } from "./sim-cfn-lambda-event-source-mapping-property-rules.js";
4
4
  import { simCfnLambdaTargetFunctionName } from "../function/sim-cfn-lambda-target-function.js";
5
5
  /**
6
6
  * What a CloudFormation Unix time seconds value is multiplied by to reach the
@@ -24,7 +24,7 @@ export class SimCfnLambdaEventSourceMappingProperties {
24
24
  this.properties = properties.properties;
25
25
  }
26
26
  /**
27
- * The create input this Resource asks for, refusing what is not simulated.
27
+ * The create input this Resource asks for, recording what is left out of it.
28
28
  *
29
29
  * The event source ARN is read before anything else is judged, as the command
30
30
  * reads it first for the same reason: what a mapping may ask for depends on
@@ -32,7 +32,7 @@ export class SimCfnLambdaEventSourceMappingProperties {
32
32
  */
33
33
  createInput() {
34
34
  const eventSourceArn = this.parser.requiredString(this.resource, this.properties["EventSourceArn"], "EventSourceArn");
35
- assertSimulatedEventSourceMappingProperties(this.resource, this.properties);
35
+ recordUnsimulatedEventSourceMappingProperties(this.resource, this.properties);
36
36
  return {
37
37
  EventSourceArn: eventSourceArn,
38
38
  DestinationConfig: simLambdaStreamDestinationConfig(this.properties["DestinationConfig"]),
@@ -42,6 +42,7 @@ export class SimCfnLambdaEventSourceMappingProperties {
42
42
  MaximumBatchingWindowInSeconds: this.parser.optionalNumber(this.resource, this.properties["MaximumBatchingWindowInSeconds"], "MaximumBatchingWindowInSeconds"),
43
43
  MaximumRetryAttempts: this.parser.optionalNumber(this.resource, this.properties["MaximumRetryAttempts"], "MaximumRetryAttempts"),
44
44
  MaximumRecordAgeInSeconds: this.parser.optionalNumber(this.resource, this.properties["MaximumRecordAgeInSeconds"], "MaximumRecordAgeInSeconds"),
45
+ BisectBatchOnFunctionError: this.parser.optionalBoolean(this.resource, this.properties["BisectBatchOnFunctionError"], "BisectBatchOnFunctionError"),
45
46
  StartingPosition: this.parser.optionalString(this.resource, this.properties["StartingPosition"], "StartingPosition"),
46
47
  StartingPositionTimestamp: this.startingPositionTimestamp(),
47
48
  FunctionResponseTypes: this.functionResponseTypes(),
@@ -0,0 +1,30 @@
1
+ /**
2
+ * The AWS::Lambda::EventSourceMapping properties this simulation acts on.
3
+ *
4
+ * The two starting position properties are here rather than among the
5
+ * unsimulated ones because whether a mapping may carry one is the event
6
+ * source's own rule: a stream has to be given a position and a queue is refused
7
+ * for naming one. That is decided by CreateEventSourceMapping, which knows
8
+ * which source the ARN names, so both are read and passed on. The two
9
+ * failed-batch limits are read for the same reason: a stream mapping keeps them
10
+ * and a queue mapping is refused for naming one.
11
+ */
12
+ export declare const simulatedPropertyNames: ReadonlySet<string>;
13
+ /**
14
+ * Real AWS::Lambda::EventSourceMapping properties this simulation does not act
15
+ * on, and what the mapping does in place of each.
16
+ *
17
+ * Every one of them changes what the function sees or when it sees it. That is
18
+ * what the record against the Resource is for. A mapping created without one
19
+ * still carries the records the template asked for to the function the
20
+ * template named, and the difference is there to be read.
21
+ */
22
+ export declare const unsimulatedPropertyReasons: ReadonlyMap<string, string>;
23
+ /**
24
+ * Why a property name this simulation has never heard of is recorded.
25
+ *
26
+ * A typo and a property AWS added after this simulation read the docs look
27
+ * identical from here. A mapping that deploys with the unread name reported is
28
+ * more use than a stack that fails over either.
29
+ */
30
+ export declare const unknownPropertyReason: string;
@@ -0,0 +1,91 @@
1
+ /**
2
+ * The AWS::Lambda::EventSourceMapping properties this simulation acts on.
3
+ *
4
+ * The two starting position properties are here rather than among the
5
+ * unsimulated ones because whether a mapping may carry one is the event
6
+ * source's own rule: a stream has to be given a position and a queue is refused
7
+ * for naming one. That is decided by CreateEventSourceMapping, which knows
8
+ * which source the ARN names, so both are read and passed on. The two
9
+ * failed-batch limits are read for the same reason: a stream mapping keeps them
10
+ * and a queue mapping is refused for naming one.
11
+ */
12
+ export const simulatedPropertyNames = new Set([
13
+ "BatchSize",
14
+ "Enabled",
15
+ "EventSourceArn",
16
+ "FunctionName",
17
+ "FunctionResponseTypes",
18
+ "MaximumBatchingWindowInSeconds",
19
+ "MaximumRecordAgeInSeconds",
20
+ "MaximumRetryAttempts",
21
+ "BisectBatchOnFunctionError",
22
+ "DestinationConfig",
23
+ "StartingPosition",
24
+ "StartingPositionTimestamp",
25
+ ]);
26
+ /**
27
+ * The event sources a mapping property can only be configuring, none of which
28
+ * this simulation has.
29
+ *
30
+ * A mapping naming one of these has an event source ARN to match, and that ARN
31
+ * is what the mapping is refused on. The property beside it is recorded.
32
+ */
33
+ const absentEventSource = "the mapping polls the source its EventSourceArn names, and this " +
34
+ "simulation has SQS queues, DynamoDB streams and Kinesis streams";
35
+ /**
36
+ * Real AWS::Lambda::EventSourceMapping properties this simulation does not act
37
+ * on, and what the mapping does in place of each.
38
+ *
39
+ * Every one of them changes what the function sees or when it sees it. That is
40
+ * what the record against the Resource is for. A mapping created without one
41
+ * still carries the records the template asked for to the function the
42
+ * template named, and the difference is there to be read.
43
+ */
44
+ export const unsimulatedPropertyReasons = new Map([
45
+ ["AmazonManagedKafkaEventSourceConfig", absentEventSource],
46
+ ["DocumentDBEventSourceConfig", absentEventSource],
47
+ [
48
+ "FilterCriteria",
49
+ "every record on the event source is delivered to the function, unfiltered",
50
+ ],
51
+ [
52
+ "KmsKeyArn",
53
+ "the key encrypts filter criteria, and filter criteria are not simulated",
54
+ ],
55
+ ["MetricsConfig", "the mapping publishes no CloudWatch metrics"],
56
+ [
57
+ "ParallelizationFactor",
58
+ "polling concurrency is not simulated, and each shard of a stream is read " +
59
+ "by one reader",
60
+ ],
61
+ ["ProvisionedPollerConfig", "polling concurrency is not simulated"],
62
+ ["Queues", absentEventSource],
63
+ ["ScalingConfig", "polling concurrency is not simulated"],
64
+ ["SelfManagedEventSource", absentEventSource],
65
+ ["SelfManagedKafkaEventSourceConfig", absentEventSource],
66
+ [
67
+ "SourceAccessConfigurations",
68
+ "event source authentication is not simulated",
69
+ ],
70
+ [
71
+ "Tags",
72
+ "the mapping is created without them, and nothing reads them back or " +
73
+ "groups or bills by them",
74
+ ],
75
+ ["Topics", absentEventSource],
76
+ [
77
+ "TumblingWindowInSeconds",
78
+ "a batch carries no window state from the batch before it, and the " +
79
+ "function is handed no state to return",
80
+ ],
81
+ ]);
82
+ /**
83
+ * Why a property name this simulation has never heard of is recorded.
84
+ *
85
+ * A typo and a property AWS added after this simulation read the docs look
86
+ * identical from here. A mapping that deploys with the unread name reported is
87
+ * more use than a stack that fails over either.
88
+ */
89
+ export const unknownPropertyReason = "it is not an AWS::Lambda::EventSourceMapping property this simulation " +
90
+ "knows, whether because AWS added it or because the template misspelled " +
91
+ "something";
@@ -1,17 +1,12 @@
1
1
  import type { SimCfnResource } from "../../../cloudformation/resource/sim-cfn-resource.js";
2
2
  import type { SimCfnTemplateValueRecord } from "../../../cloudformation/template/value/sim-cfn-template-value.js";
3
3
  /**
4
- * Build the error a property of an AWS::Lambda::EventSourceMapping Resource is
5
- * refused with.
4
+ * Record everything about an AWS::Lambda::EventSourceMapping Resource that the
5
+ * mapping is created without.
6
6
  *
7
- * The wording matters: sim CloudFormation skips a Resource whose error reads as
8
- * an unsupported Resource type, and skipping is the wrong answer for a mapping
9
- * that cannot be created as the template asks. The stack would deploy with the
10
- * queue and the function and nothing between them.
7
+ * Nothing here fails the Resource. A mapping missing one of these settings
8
+ * still carries records from the source to the function, and a stack that
9
+ * refused it would take every other Resource down over one line of a template.
10
+ * What the mapping does instead is in `stack.ignoredProperties`.
11
11
  */
12
- export declare function eventSourceMappingPropertyError(logicalId: string, reason: string): Error;
13
- /**
14
- * Refuse everything about an AWS::Lambda::EventSourceMapping Resource that is
15
- * not simulated, and record what the mapping is created without.
16
- */
17
- export declare function assertSimulatedEventSourceMappingProperties(resource: SimCfnResource, properties: SimCfnTemplateValueRecord): void;
12
+ export declare function recordUnsimulatedEventSourceMappingProperties(resource: SimCfnResource, properties: SimCfnTemplateValueRecord): void;
@@ -1,100 +1,20 @@
1
+ import { simulatedPropertyNames, unknownPropertyReason, unsimulatedPropertyReasons, } from "./sim-cfn-lambda-event-source-mapping-property-names.js";
1
2
  /**
2
- * The AWS::Lambda::EventSourceMapping properties this simulation acts on.
3
+ * Record everything about an AWS::Lambda::EventSourceMapping Resource that the
4
+ * mapping is created without.
3
5
  *
4
- * The two starting position properties are here rather than among the
5
- * unsimulated ones because whether a mapping may carry one is the event
6
- * source's own rule: a stream has to be given a position and a queue is refused
7
- * for naming one. That is decided by CreateEventSourceMapping, which knows
8
- * which source the ARN names, so both are read and passed on. The two
9
- * failed-batch limits are read for the same reason: a stream mapping keeps them
10
- * and a queue mapping is refused for naming one.
6
+ * Nothing here fails the Resource. A mapping missing one of these settings
7
+ * still carries records from the source to the function, and a stack that
8
+ * refused it would take every other Resource down over one line of a template.
9
+ * What the mapping does instead is in `stack.ignoredProperties`.
11
10
  */
12
- const simulatedPropertyNames = new Set([
13
- "BatchSize",
14
- "Enabled",
15
- "EventSourceArn",
16
- "FunctionName",
17
- "FunctionResponseTypes",
18
- "MaximumBatchingWindowInSeconds",
19
- "MaximumRecordAgeInSeconds",
20
- "MaximumRetryAttempts",
21
- "DestinationConfig",
22
- "StartingPosition",
23
- "StartingPositionTimestamp",
24
- ]);
25
- /**
26
- * Real AWS::Lambda::EventSourceMapping properties this simulation does not
27
- * model.
28
- *
29
- * Every one of them changes what the function sees or when it sees it, so a
30
- * template asking for one fails the Resource rather than deploying a mapping
31
- * that quietly ignores it.
32
- */
33
- const unsimulatedPropertyNames = new Set([
34
- "AmazonManagedKafkaEventSourceConfig",
35
- "BisectBatchOnFunctionError",
36
- "DocumentDBEventSourceConfig",
37
- "FilterCriteria",
38
- "KmsKeyArn",
39
- "MetricsConfig",
40
- "ParallelizationFactor",
41
- "ProvisionedPollerConfig",
42
- "Queues",
43
- "ScalingConfig",
44
- "SelfManagedEventSource",
45
- "SelfManagedKafkaEventSourceConfig",
46
- "SourceAccessConfigurations",
47
- "Topics",
48
- "TumblingWindowInSeconds",
49
- ]);
50
- /**
51
- * Real AWS::Lambda::EventSourceMapping properties this simulation has nothing
52
- * to act on and no reason to fail a stack over.
53
- *
54
- * `Tags` is the whole list. The mapping is created without them and the
55
- * omission is recorded against the Resource. A mapping delivers the same
56
- * records with a tag and without one, and a CDK app calling
57
- * `Tags.of(app).add(...)` tags every mapping in it.
58
- */
59
- const ignoredPropertyReasons = new Map([
60
- [
61
- "Tags",
62
- "AWS::Lambda::EventSourceMapping property Tags is not simulated, so the " +
63
- "mapping is created without them. Nothing reads them back and nothing " +
64
- "is grouped or billed by them.",
65
- ],
66
- ]);
67
- /**
68
- * Build the error a property of an AWS::Lambda::EventSourceMapping Resource is
69
- * refused with.
70
- *
71
- * The wording matters: sim CloudFormation skips a Resource whose error reads as
72
- * an unsupported Resource type, and skipping is the wrong answer for a mapping
73
- * that cannot be created as the template asks. The stack would deploy with the
74
- * queue and the function and nothing between them.
75
- */
76
- export function eventSourceMappingPropertyError(logicalId, reason) {
77
- return new Error(`Invalid AWS::Lambda::EventSourceMapping Resource ${logicalId}: ${reason}`);
78
- }
79
- /**
80
- * Refuse everything about an AWS::Lambda::EventSourceMapping Resource that is
81
- * not simulated, and record what the mapping is created without.
82
- */
83
- export function assertSimulatedEventSourceMappingProperties(resource, properties) {
84
- const logicalId = resource.logicalId;
11
+ export function recordUnsimulatedEventSourceMappingProperties(resource, properties) {
85
12
  for (const name of Object.keys(properties)) {
86
- const ignoredReason = ignoredPropertyReasons.get(name);
87
- if (ignoredReason !== undefined) {
88
- resource.ignoreProperty(name, ignoredReason);
13
+ if (simulatedPropertyNames.has(name)) {
89
14
  continue;
90
15
  }
91
- if (unsimulatedPropertyNames.has(name)) {
92
- throw eventSourceMappingPropertyError(logicalId, `${name} is a real AWS::Lambda::EventSourceMapping property that ` +
93
- "this simulation does not simulate, so it is refused rather than " +
94
- "ignored");
95
- }
96
- if (!simulatedPropertyNames.has(name)) {
97
- throw eventSourceMappingPropertyError(logicalId, `${name} is not an AWS::Lambda::EventSourceMapping property`);
98
- }
16
+ const reason = unsimulatedPropertyReasons.get(name) ?? unknownPropertyReason;
17
+ resource.ignoreProperty(name, `AWS::Lambda::EventSourceMapping property ${name} is not simulated: ` +
18
+ `${reason}.`);
99
19
  }
100
20
  }
@@ -68,6 +68,7 @@ export class SimLambdaEventSourceMappingInput {
68
68
  streamRetryLimits: eventSourceArn.retryLimitRules.limitsIn({
69
69
  maximumRetryAttempts: input.MaximumRetryAttempts,
70
70
  maximumRecordAgeInSeconds: input.MaximumRecordAgeInSeconds,
71
+ bisectBatchOnFunctionError: input.BisectBatchOnFunctionError,
71
72
  }),
72
73
  });
73
74
  }
@@ -12,11 +12,6 @@ const unsimulatedInputs = new Map([
12
12
  ["FilterCriteria", "event filtering is not simulated"],
13
13
  ["ScalingConfig", "polling concurrency is not simulated"],
14
14
  ["ProvisionedPollerConfig", "polling concurrency is not simulated"],
15
- [
16
- "BisectBatchOnFunctionError",
17
- "batch bisection is not simulated, so a failing stream batch is retried " +
18
- "whole",
19
- ],
20
15
  [
21
16
  "ParallelizationFactor",
22
17
  "polling concurrency is not simulated, so each shard of a stream is read " +
@@ -66,7 +66,7 @@ export class SimLambdaKinesisShardPoller {
66
66
  const { progress } = this;
67
67
  // Reading is done as the function's execution role, as on real Lambda, so
68
68
  // simulated IAM decides whether this mapping may read its stream.
69
- const batch = await this.shard.read(progress.position, this.mapping.batchSize);
69
+ const batch = await this.shard.read(progress.position, progress.batchSizeWithin(this.mapping.batchSize));
70
70
  await processSimLambdaStreamBatch({
71
71
  batch,
72
72
  progress,
@@ -15,7 +15,7 @@ export class SimLambdaDynamoDbStreamPolling {
15
15
  const { progress, stream, mapping, delivery } = this.properties;
16
16
  // Reading is done as the function's execution role, as on real Lambda, so
17
17
  // simulated IAM decides whether this mapping may read its stream.
18
- const batch = await stream.read(simFunction.roleArn, progress.position, mapping.batchSize);
18
+ const batch = await stream.read(simFunction.roleArn, progress.position, progress.batchSizeWithin(mapping.batchSize));
19
19
  await processSimLambdaStreamBatch({
20
20
  batch,
21
21
  progress,
@@ -0,0 +1,53 @@
1
+ import type { SimLambdaStreamBatchOutcome } from "./sim-lambda-stream-batch-outcome.js";
2
+ /**
3
+ * How much of a failing batch the next delivery may carry.
4
+ *
5
+ * A mapping asked to bisect halves a batch its function threw on, and halves
6
+ * again on each further error, until a failing batch holds one record. That is
7
+ * how the record that broke the batch ends up delivered on its own, and how the
8
+ * records beside it get through.
9
+ *
10
+ * A split is progress rather than another attempt at the same work, so it puts
11
+ * the retry count back to the start. Without that the simulator's own cap of
12
+ * five attempts would discard a batch of a hundred long before it was down to
13
+ * one record. A batch halves at every step, so the splitting ends on its own,
14
+ * and the single record it ends on is then counted and discarded as any failing
15
+ * batch is.
16
+ *
17
+ * Two numbers say what the next read may take. The half is how many records a
18
+ * delivery may carry, and the remainder is how many records of the batch being
19
+ * split are still to get through. Reading the lower of the two is what keeps a
20
+ * record written while the splitting is going on out of it: the halves are the
21
+ * batch that failed, and a record that arrived afterwards belongs to the batch
22
+ * after it. The mapping's own batch size comes back once the whole of the split
23
+ * batch is through.
24
+ */
25
+ export declare class SimLambdaStreamBisect {
26
+ #private;
27
+ private readonly enabled;
28
+ constructor(enabled: boolean);
29
+ /**
30
+ * How many records the next read may take, within the mapping's batch size.
31
+ */
32
+ sizeWithin(batchSize: number): number;
33
+ /**
34
+ * Split a batch the function threw on, answering with whether it split.
35
+ *
36
+ * A batch the function reported partial failures on is left alone. Lambda
37
+ * bisects around an error the handler raised, and a report already says which
38
+ * record to go back to.
39
+ */
40
+ split(outcome: SimLambdaStreamBatchOutcome): boolean;
41
+ /**
42
+ * Take records of the batch being split that the mapping is finished with,
43
+ * whether the function handled them, they were discarded, or they aged out.
44
+ *
45
+ * The whole of the split batch being through is what puts the mapping back on
46
+ * full batches.
47
+ */
48
+ finished(count: number): void;
49
+ /**
50
+ * Read whole batches again.
51
+ */
52
+ reset(): void;
53
+ }
@@ -0,0 +1,79 @@
1
+ /**
2
+ * How much of a failing batch the next delivery may carry.
3
+ *
4
+ * A mapping asked to bisect halves a batch its function threw on, and halves
5
+ * again on each further error, until a failing batch holds one record. That is
6
+ * how the record that broke the batch ends up delivered on its own, and how the
7
+ * records beside it get through.
8
+ *
9
+ * A split is progress rather than another attempt at the same work, so it puts
10
+ * the retry count back to the start. Without that the simulator's own cap of
11
+ * five attempts would discard a batch of a hundred long before it was down to
12
+ * one record. A batch halves at every step, so the splitting ends on its own,
13
+ * and the single record it ends on is then counted and discarded as any failing
14
+ * batch is.
15
+ *
16
+ * Two numbers say what the next read may take. The half is how many records a
17
+ * delivery may carry, and the remainder is how many records of the batch being
18
+ * split are still to get through. Reading the lower of the two is what keeps a
19
+ * record written while the splitting is going on out of it: the halves are the
20
+ * batch that failed, and a record that arrived afterwards belongs to the batch
21
+ * after it. The mapping's own batch size comes back once the whole of the split
22
+ * batch is through.
23
+ */
24
+ export class SimLambdaStreamBisect {
25
+ enabled;
26
+ #half;
27
+ #remaining = 0;
28
+ constructor(enabled) {
29
+ this.enabled = enabled;
30
+ }
31
+ /**
32
+ * How many records the next read may take, within the mapping's batch size.
33
+ */
34
+ sizeWithin(batchSize) {
35
+ const half = this.#half;
36
+ return half === undefined
37
+ ? batchSize
38
+ : Math.min(half, this.#remaining, batchSize);
39
+ }
40
+ /**
41
+ * Split a batch the function threw on, answering with whether it split.
42
+ *
43
+ * A batch the function reported partial failures on is left alone. Lambda
44
+ * bisects around an error the handler raised, and a report already says which
45
+ * record to go back to.
46
+ */
47
+ split(outcome) {
48
+ const count = outcome.records.length;
49
+ if (!this.enabled || !outcome.functionError || count < 2) {
50
+ return false;
51
+ }
52
+ this.#half = Math.ceil(count / 2);
53
+ this.#remaining = count;
54
+ return true;
55
+ }
56
+ /**
57
+ * Take records of the batch being split that the mapping is finished with,
58
+ * whether the function handled them, they were discarded, or they aged out.
59
+ *
60
+ * The whole of the split batch being through is what puts the mapping back on
61
+ * full batches.
62
+ */
63
+ finished(count) {
64
+ if (this.#half === undefined || count <= 0) {
65
+ return;
66
+ }
67
+ this.#remaining -= count;
68
+ if (this.#remaining <= 0) {
69
+ this.reset();
70
+ }
71
+ }
72
+ /**
73
+ * Read whole batches again.
74
+ */
75
+ reset() {
76
+ this.#half = undefined;
77
+ this.#remaining = 0;
78
+ }
79
+ }
@@ -2,6 +2,7 @@ import { SimLambdaStreamExpiry } from "./sim-lambda-stream-expiry.js";
2
2
  import { SimLambdaStreamFailureDestination } from "./sim-lambda-stream-failure-destination.js";
3
3
  import { SimLambdaStreamCursor } from "./sim-lambda-stream-cursor.js";
4
4
  import { SimLambdaStreamRetry } from "./sim-lambda-stream-retry.js";
5
+ import { SimLambdaStreamBisect } from "./sim-lambda-stream-bisect.js";
5
6
  import type { BackgroundScheduler, BackgroundTask } from "../../../../util/background/background.js";
6
7
  import type { SimLambdaEventSourceMapping } from "../sim-lambda-event-source-mapping.js";
7
8
  import type { SimLambdaEventSourceStreamProgressBatch } from "../stream/sim-lambda-event-source-streams.js";
@@ -16,6 +17,7 @@ export declare class SimLambdaStreamProgressState {
16
17
  readonly failures: SimLambdaStreamFailureDestination;
17
18
  readonly cursor: SimLambdaStreamCursor;
18
19
  readonly retry: SimLambdaStreamRetry;
20
+ readonly bisect: SimLambdaStreamBisect;
19
21
  constructor(properties: SimLambdaStreamProgressProperties);
20
22
  handled(batch: SimLambdaEventSourceStreamProgressBatch): void;
21
23
  }
@@ -3,6 +3,7 @@ import { SimLambdaStreamBatchAge } from "./sim-lambda-stream-batch-age.js";
3
3
  import { SimLambdaStreamFailureDestination } from "./sim-lambda-stream-failure-destination.js";
4
4
  import { SimLambdaStreamCursor } from "./sim-lambda-stream-cursor.js";
5
5
  import { SimLambdaStreamRetry } from "./sim-lambda-stream-retry.js";
6
+ import { SimLambdaStreamBisect } from "./sim-lambda-stream-bisect.js";
6
7
  import { PollSchedule } from "../../../../util/background/poll-schedule.js";
7
8
  import { assertDefined } from "../../../../util/type-guard/defined.js";
8
9
  /** Own the per-shard retry state and its expiry and destination collaborators. */
@@ -11,6 +12,7 @@ export class SimLambdaStreamProgressState {
11
12
  failures;
12
13
  cursor;
13
14
  retry;
15
+ bisect;
14
16
  constructor(properties) {
15
17
  const { mapping, background } = properties;
16
18
  this.failures = new SimLambdaStreamFailureDestination(mapping);
@@ -22,6 +24,7 @@ export class SimLambdaStreamProgressState {
22
24
  assertDefined(streamRetryLimits, "stream mapping retry limits");
23
25
  this.cursor = new SimLambdaStreamCursor(start, new PollSchedule(properties));
24
26
  this.retry = new SimLambdaStreamRetry(streamRetryLimits, background);
27
+ this.bisect = new SimLambdaStreamBisect(streamRetryLimits.bisectBatchOnFunctionError);
25
28
  this.expiry = new SimLambdaStreamExpiry({
26
29
  age: new SimLambdaStreamBatchAge(streamRetryLimits, background),
27
30
  cursor: this.cursor,
@@ -15,6 +15,11 @@ export declare class SimLambdaStreamProgress {
15
15
  * Where the next read starts.
16
16
  */
17
17
  get position(): SimLambdaEventSourceStreamPosition;
18
+ /**
19
+ * How many records the next read may take, which is the mapping's batch size
20
+ * until a batch is being bisected.
21
+ */
22
+ batchSizeWithin(batchSize: number): number;
18
23
  /**
19
24
  * Take a read that came back with nothing.
20
25
  *
@@ -23,9 +28,17 @@ export declare class SimLambdaStreamProgress {
23
28
  * than working out a new one each time.
24
29
  */
25
30
  caughtUp(batch: SimLambdaEventSourceStreamProgressBatch): void;
31
+ /**
32
+ * Drop the records at the front of a batch that are too old to hand over,
33
+ * answering with where the live ones start.
34
+ *
35
+ * Records that aged out are as finished with as records the function took, so
36
+ * a batch being split counts them off its remainder too.
37
+ */
26
38
  before(records: readonly SimLambdaStreamRecordTime[], batch: SimLambdaEventSourceStreamProgressBatch, simFunction: SimLambdaFunction): Promise<number>;
27
39
  /**
28
- * Take what became of a batch: move past it, or wait and try it again.
40
+ * Take what became of a batch: move past it, split it, or wait and try it
41
+ * again.
29
42
  */
30
43
  after(outcome: SimLambdaStreamBatchOutcome, batch: SimLambdaEventSourceStreamProgressBatch, simFunction: SimLambdaFunction): Promise<void>;
31
44
  /**
@@ -17,6 +17,13 @@ export class SimLambdaStreamProgress {
17
17
  get position() {
18
18
  return this.state.cursor.position;
19
19
  }
20
+ /**
21
+ * How many records the next read may take, which is the mapping's batch size
22
+ * until a batch is being bisected.
23
+ */
24
+ batchSizeWithin(batchSize) {
25
+ return this.state.bisect.sizeWithin(batchSize);
26
+ }
20
27
  /**
21
28
  * Take a read that came back with nothing.
22
29
  *
@@ -27,21 +34,36 @@ export class SimLambdaStreamProgress {
27
34
  caughtUp(batch) {
28
35
  this.state.cursor.advanceTo(batch.next);
29
36
  }
37
+ /**
38
+ * Drop the records at the front of a batch that are too old to hand over,
39
+ * answering with where the live ones start.
40
+ *
41
+ * Records that aged out are as finished with as records the function took, so
42
+ * a batch being split counts them off its remainder too.
43
+ */
30
44
  async before(records, batch, simFunction) {
31
- return await this.state.expiry.before(records, batch, simFunction);
45
+ const firstLive = await this.state.expiry.before(records, batch, simFunction);
46
+ this.state.bisect.finished(firstLive);
47
+ return firstLive;
32
48
  }
33
49
  /**
34
- * Take what became of a batch: move past it, or wait and try it again.
50
+ * Take what became of a batch: move past it, split it, or wait and try it
51
+ * again.
35
52
  */
36
53
  async after(outcome, batch, simFunction) {
37
54
  this.state.failures.invoked(outcome);
38
55
  if (outcome.isHandled) {
56
+ this.state.bisect.finished(outcome.records.length);
39
57
  this.handled(batch);
40
58
  return;
41
59
  }
60
+ if (this.state.bisect.split(outcome)) {
61
+ this.state.retry.reset();
62
+ }
42
63
  const again = this.state.retry.after(outcome, this.position);
43
64
  const notification = this.state.failures.deliver(this.state.retry.discarded, batch, simFunction);
44
65
  if (again === undefined) {
66
+ this.state.bisect.finished(outcome.records.length);
45
67
  this.handled(batch);
46
68
  }
47
69
  else {
@@ -1,12 +1,13 @@
1
1
  import { type SimLambdaEventSourceRetryLimitsInput, SimLambdaStreamRetryLimits } from "./sim-lambda-stream-retry-limits.js";
2
2
  /**
3
- * Whether the event source a mapping names has failed-batch limits at all.
3
+ * Whether the event source a mapping names decides for itself what becomes of
4
+ * a failed batch.
4
5
  *
5
- * A stream does: a record stays on the stream whatever the function makes of
6
- * it, so the mapping is the only thing that decides when to stop trying. A
7
- * queue does not, because a message the function never handles goes to the
8
- * queue's own redrive policy, and real Lambda refuses a request that names
9
- * either limit for one.
6
+ * A stream leaves it to the mapping: a record stays on the stream whatever the
7
+ * function makes of it, so the mapping is the only thing that decides when to
8
+ * stop trying and whether to split the batch first. A queue does not, because a
9
+ * message the function never handles goes to the queue's own redrive policy,
10
+ * and real Lambda refuses a request that names any of the three for one.
10
11
  */
11
12
  export interface SimLambdaEventSourceRetryLimitRules {
12
13
  /**
@@ -23,7 +24,7 @@ export declare class SimLambdaNoRetryLimits implements SimLambdaEventSourceRetry
23
24
  private readonly insteadDescription;
24
25
  constructor(sourceDescription: string, insteadDescription: string);
25
26
  /**
26
- * Refuse both limits, since this source is not the one counting.
27
+ * Refuse all three, since this source is not the one counting.
27
28
  */
28
29
  limitsIn(input: SimLambdaEventSourceRetryLimitsInput): SimLambdaStreamRetryLimits | undefined;
29
30
  }
@@ -11,7 +11,7 @@ export class SimLambdaNoRetryLimits {
11
11
  this.insteadDescription = insteadDescription;
12
12
  }
13
13
  /**
14
- * Refuse both limits, since this source is not the one counting.
14
+ * Refuse all three, since this source is not the one counting.
15
15
  */
16
16
  limitsIn(input) {
17
17
  const named = namedLimit(input);
@@ -40,7 +40,10 @@ function namedLimit(input) {
40
40
  if (input.maximumRetryAttempts !== undefined) {
41
41
  return "MaximumRetryAttempts";
42
42
  }
43
- return input.maximumRecordAgeInSeconds === undefined
43
+ if (input.maximumRecordAgeInSeconds !== undefined) {
44
+ return "MaximumRecordAgeInSeconds";
45
+ }
46
+ return input.bisectBatchOnFunctionError === undefined
44
47
  ? undefined
45
- : "MaximumRecordAgeInSeconds";
48
+ : "BisectBatchOnFunctionError";
46
49
  }
@@ -9,6 +9,7 @@
9
9
  export interface SimLambdaEventSourceRetryLimitsInput {
10
10
  readonly maximumRetryAttempts?: number | undefined;
11
11
  readonly maximumRecordAgeInSeconds?: number | undefined;
12
+ readonly bisectBatchOnFunctionError?: boolean | undefined;
12
13
  }
13
14
  /**
14
15
  * The failed-batch limits a mapping reports back, as Lambda names them.
@@ -16,22 +17,29 @@ export interface SimLambdaEventSourceRetryLimitsInput {
16
17
  export interface SimLambdaStreamRetryLimitsConfiguration {
17
18
  readonly MaximumRetryAttempts: number;
18
19
  readonly MaximumRecordAgeInSeconds: number;
20
+ readonly BisectBatchOnFunctionError: boolean;
19
21
  }
20
22
  /**
21
- * When a stream mapping stops delivering a batch its function keeps failing.
23
+ * What a stream mapping does with a batch its function keeps failing.
22
24
  *
23
- * The two limits are one thing rather than two, because they end the same
24
- * lifecycle: a record leaves the mapping when it has had its retries or when it
25
- * is too old to be worth another, whichever comes first. Both are held as
26
- * Lambda states them, with `-1` for no limit, so a Get or a List reports back
27
- * exactly what was asked for.
25
+ * The three settings are one thing rather than three, because they run the same
26
+ * lifecycle. A failing batch is split around the record that broke it where the
27
+ * mapping was asked to bisect, and a record leaves the mapping when it has had
28
+ * its retries or when it is too old to be worth another, whichever comes first.
29
+ * The limits are held as Lambda states them, with `-1` for no limit, so a Get
30
+ * or a List reports back exactly what was asked for.
28
31
  */
29
32
  export declare class SimLambdaStreamRetryLimits {
30
33
  readonly maximumRetryAttempts: number;
31
34
  readonly maximumRecordAgeInSeconds: number;
35
+ /**
36
+ * Whether a batch the function threw on is split in half and delivered again
37
+ * as two batches.
38
+ */
39
+ readonly bisectBatchOnFunctionError: boolean;
32
40
  constructor(input?: SimLambdaEventSourceRetryLimitsInput);
33
41
  /**
34
- * The two limits as a Get, a List or a create response reports them.
42
+ * The three settings as a Get, a List or a create response reports them.
35
43
  */
36
44
  configuration(): SimLambdaStreamRetryLimitsConfiguration;
37
45
  /**
@@ -23,17 +23,23 @@ const maximumRecordAgeLimit = 604_800;
23
23
  const defaultAttemptLimit = 5;
24
24
  const millisecondsPerSecond = 1000;
25
25
  /**
26
- * When a stream mapping stops delivering a batch its function keeps failing.
26
+ * What a stream mapping does with a batch its function keeps failing.
27
27
  *
28
- * The two limits are one thing rather than two, because they end the same
29
- * lifecycle: a record leaves the mapping when it has had its retries or when it
30
- * is too old to be worth another, whichever comes first. Both are held as
31
- * Lambda states them, with `-1` for no limit, so a Get or a List reports back
32
- * exactly what was asked for.
28
+ * The three settings are one thing rather than three, because they run the same
29
+ * lifecycle. A failing batch is split around the record that broke it where the
30
+ * mapping was asked to bisect, and a record leaves the mapping when it has had
31
+ * its retries or when it is too old to be worth another, whichever comes first.
32
+ * The limits are held as Lambda states them, with `-1` for no limit, so a Get
33
+ * or a List reports back exactly what was asked for.
33
34
  */
34
35
  export class SimLambdaStreamRetryLimits {
35
36
  maximumRetryAttempts;
36
37
  maximumRecordAgeInSeconds;
38
+ /**
39
+ * Whether a batch the function threw on is split in half and delivered again
40
+ * as two batches.
41
+ */
42
+ bisectBatchOnFunctionError;
37
43
  constructor(input = {}) {
38
44
  this.maximumRetryAttempts = checkedLimit({
39
45
  value: input.maximumRetryAttempts,
@@ -47,14 +53,16 @@ export class SimLambdaStreamRetryLimits {
47
53
  maximum: maximumRecordAgeLimit,
48
54
  unit: "seconds",
49
55
  });
56
+ this.bisectBatchOnFunctionError = input.bisectBatchOnFunctionError ?? false;
50
57
  }
51
58
  /**
52
- * The two limits as a Get, a List or a create response reports them.
59
+ * The three settings as a Get, a List or a create response reports them.
53
60
  */
54
61
  configuration() {
55
62
  return {
56
63
  MaximumRetryAttempts: this.maximumRetryAttempts,
57
64
  MaximumRecordAgeInSeconds: this.maximumRecordAgeInSeconds,
65
+ BisectBatchOnFunctionError: this.bisectBatchOnFunctionError,
58
66
  };
59
67
  }
60
68
  /**
@@ -2033,6 +2033,41 @@ discarded, and the mapping reads on from behind it.
2033
2033
  10,000 retries, `-1` to 604,800 seconds) is a `ValidationException`. A queue mapping takes neither,
2034
2034
  because a message the handler never takes is left to the queue's own redrive policy.
2035
2035
 
2036
+ ### Splitting a failed batch around the record that broke it
2037
+
2038
+ `BisectBatchOnFunctionError` splits a batch the handler threw on in half, and delivers each half as
2039
+ its own batch. A half that fails is split again, down to a single record. That is how the records
2040
+ beside a poison record get through, and how the poison record ends up delivered on its own and
2041
+ discarded on its own.
2042
+
2043
+ ```json
2044
+ {
2045
+ "StartingPosition": "TRIM_HORIZON",
2046
+ "BatchSize": 100,
2047
+ "MaximumRetryAttempts": 10,
2048
+ "BisectBatchOnFunctionError": true
2049
+ }
2050
+ ```
2051
+
2052
+ A batch of four whose third record the handler cannot take is delivered as `1, 2, 3, 4`, then
2053
+ `1, 2`, then `3, 4`, then `3` on its own, and finally `4`. The record that broke every batch it was
2054
+ in leaves the mapping under the retry and record-age limits, and reaches the
2055
+ [failure destination](#sending-discarded-stream-batches-to-a-destination) naming one record rather
2056
+ than four.
2057
+
2058
+ Splitting puts the retry count back to the start, so a batch always reaches a single record before
2059
+ the count decides anything. Once it is down to one record, the limits count as they do for any other
2060
+ failing batch. A batch the handler reported partial failures on is left whole, because the report
2061
+ already says which record to go back to.
2062
+
2063
+ The halves carry the batch that failed and nothing else. A record written while the splitting is
2064
+ going on waits behind it and is delivered as part of the batch after it, and the mapping reads at its
2065
+ own batch size again once the whole of the split batch is through.
2066
+
2067
+ `GetEventSourceMapping` and `ListEventSourceMappings` report the setting, and
2068
+ `AWS::Lambda::EventSourceMapping` takes it in a template. Both stream sources have it. A queue
2069
+ mapping is refused for naming it, the way real Lambda refuses one.
2070
+
2036
2071
  ### Sending discarded stream batches to a destination
2037
2072
 
2038
2073
  DynamoDB Streams and Kinesis mappings accept `DestinationConfig.OnFailure` with a standard SQS
@@ -2324,8 +2359,15 @@ this, and deploys without hand-editing. The grant CDK writes alongside it is an
2324
2359
  the three stream actions on the stream ARN and `dynamodb:ListStreams` on every stream, exactly what
2325
2360
  the mapping's execution-role check is looking for.
2326
2361
 
2327
- The properties a non-default `DynamoEventSource` adds are refused by name. Those are
2328
- `FilterCriteria`, `ParallelizationFactor`, `BisectBatchOnFunctionError` and `TumblingWindowInSeconds`.
2362
+ `bisectBatchOnError` is simulated, and is covered under
2363
+ [Splitting a failed batch around the record that broke it](#splitting-a-failed-batch-around-the-record-that-broke-it).
2364
+ The other properties a non-default `DynamoEventSource` adds are recorded rather than acted on. Those
2365
+ are `FilterCriteria`, `ParallelizationFactor` and `TumblingWindowInSeconds`.
2366
+ The mapping deploys, delivers every record whole and unfiltered, and each property it was created
2367
+ without is listed in
2368
+ [`stack.ignoredProperties`](https://yulinsim.dev/services/cloudformation/#properties-a-resource-was-created-without "Properties a Resource was created without")
2369
+ with what the mapping does in its place. `CreateEventSourceMapping` still refuses the same
2370
+ properties, since a caller naming one is asking for behaviour by hand.
2329
2371
 
2330
2372
  A hand-written template or a SAM application usually gives the function the AWS managed policy
2331
2373
  `AWSLambdaDynamoDBExecutionRole` instead. Simulated IAM has no model for managed policy ARNs, so
@@ -4297,11 +4339,12 @@ Current documented limitations:
4297
4339
  function, and `S3ObjectVersion` on `UpdateFunctionCode`. A versioned location loads the object as
4298
4340
  it stands.
4299
4341
  - SQS queues, DynamoDB streams and Kinesis streams are the only event sources. Kafka, DocumentDB and
4300
- Kinesis enhanced fan-out consumers are refused outright, and so are `FilterCriteria`,
4301
- `ScalingConfig`, `BisectBatchOnFunctionError`, `ParallelizationFactor`,
4302
- `TumblingWindowInSeconds` and the other mapping inputs this simulation has no behaviour for. An
4303
- `AWS::Lambda::EventSourceMapping` carrying `Tags` is the exception, and deploys with the tags
4304
- dropped and the property recorded.
4342
+ Kinesis enhanced fan-out consumers are refused outright. `CreateEventSourceMapping` also refuses
4343
+ `FilterCriteria`, `ScalingConfig`, `ParallelizationFactor`, `TumblingWindowInSeconds` and the
4344
+ other inputs this simulation has no behaviour for. An
4345
+ `AWS::Lambda::EventSourceMapping` naming any of them deploys instead, and records each one against
4346
+ the Resource (see
4347
+ [Properties a Resource was created without](https://yulinsim.dev/services/cloudformation/#properties-a-resource-was-created-without "Properties a Resource was created without")).
4305
4348
  - A failed stream batch waits 1, 2, 4, 8 and 16 seconds between attempts, where AWS documents no
4306
4349
  delay. That is deliberate. A delay of zero falls due at the instant the clock already reads, and a
4307
4350
  handler that always throws would leave `advanceBy` with work falling due forever. A mapping that
@@ -4309,6 +4352,10 @@ Current documented limitations:
4309
4352
  discards the batch, where AWS goes on until the records age out a day later. A batch item failure
4310
4353
  report counts against the same retries, and never starts them again for the records it rewound
4311
4354
  to.
4355
+ - Splitting a batch under `BisectBatchOnFunctionError` puts the retry count back to the start, where
4356
+ AWS counts a bisected batch's deliveries against the same quota. Without that, the simulator's own
4357
+ cap of five attempts would discard a batch of a hundred long before it was down to one record. The
4358
+ splitting still ends on its own, because a batch halves at every step.
4312
4359
  - A handler writing into the table whose stream invoked it is refused with
4313
4360
  `SimLambdaStreamCascadeError` rather than being delivered its own writes forever. Real Lambda runs
4314
4361
  that loop.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kensio/yulin",
3
- "version": "1.21.9",
3
+ "version": "1.21.10",
4
4
  "description": "AWS system behaviour simulation for isolated unit testing",
5
5
  "repository": "https://github.com/KensioSoftware/yulin",
6
6
  "homepage": "https://yulinsim.dev/",