@saidsef/aws-kinesis-local 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/handler.d.ts +0 -1
- package/dist/handler.js +1 -1
- package/dist/operations/common.d.ts +0 -3
- package/dist/operations/common.js +3 -3
- package/dist/operations/consumers.d.ts +2 -4
- package/dist/operations/consumers.js +1 -1
- package/dist/operations/records.d.ts +0 -1
- package/dist/operations/records.js +7 -5
- package/dist/operations/shards.d.ts +0 -4
- package/dist/operations/shards.js +3 -3
- package/dist/operations/streams.d.ts +0 -3
- package/dist/operations/streams.js +2 -2
- package/dist/operations/subscribe.d.ts +0 -1
- package/dist/operations/subscribe.js +3 -8
- package/dist/protocol/cbor.d.ts +0 -2
- package/dist/protocol/cbor.js +1 -1
- package/dist/store/types.d.ts +3 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# AWS Kinesis Local [](https://github.com/saidsef/aws-kinesis-local/actions/workflows/ci.yml) [](https://github.com/saidsef/aws-kinesis-local/actions/workflows/release.yml) [](https://www.npmjs.com/package/@saidsef/aws-kinesis-local)
|
|
2
2
|
|
|
3
|
-
aws-kinesis-local serves the Amazon Kinesis Data Streams API on your machine, for development and CI. It implements all 45 Kinesis operations, including enhanced fan-out, and serves JSON and CBOR over HTTP/1.1 and HTTP/2 on
|
|
3
|
+
aws-kinesis-local serves the Amazon Kinesis Data Streams API on your machine, for development and CI. It implements all 45 Kinesis operations, including enhanced fan-out, and serves JSON and CBOR over HTTP/1.1 and HTTP/2 on the same port, so the AWS CLI and the AWS SDKs work with their default settings. It ships as an npm package, a Docker image and a Helm chart.
|
|
4
4
|
|
|
5
5
|
## Quick start
|
|
6
6
|
|
package/dist/handler.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
2
|
import type { Http2ServerRequest, Http2ServerResponse } from 'node:http2';
|
|
3
3
|
import type { Store } from './store/store.ts';
|
|
4
|
-
export declare const MAX_REQUEST_BYTES: number;
|
|
5
4
|
type Request = IncomingMessage | Http2ServerRequest;
|
|
6
5
|
type Response = ServerResponse | Http2ServerResponse;
|
|
7
6
|
export declare function createHandler(store: Store): (req: Request, res: Response) => void;
|
package/dist/handler.js
CHANGED
|
@@ -7,7 +7,7 @@ import { Subscription } from './operations/subscribe.js';
|
|
|
7
7
|
import { decodeCbor } from './protocol/cbor.js';
|
|
8
8
|
import { AMZ_CBOR, AMZ_JSON, encodeBody, serializeShape } from './protocol/serialize.js';
|
|
9
9
|
import { coerceStructure, decodeBlobs, validateStructure } from './validation/validate.js';
|
|
10
|
-
|
|
10
|
+
const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
|
|
11
11
|
const EVENT_STREAM = 'application/vnd.amazon.eventstream';
|
|
12
12
|
const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
|
|
13
13
|
function header(req, name) {
|
|
@@ -8,7 +8,6 @@ export interface Context {
|
|
|
8
8
|
}
|
|
9
9
|
export type Handler = (context: Context, data: Input) => unknown;
|
|
10
10
|
export declare const DEFAULT_MAX_RECORD_SIZE_KIB = 1024;
|
|
11
|
-
export declare const MAX_TAGS_PER_RESOURCE = 50;
|
|
12
11
|
export declare const MAX_RETENTION_HOURS = 8760;
|
|
13
12
|
export declare const MIN_RETENTION_HOURS = 24;
|
|
14
13
|
export declare const ON_DEMAND_SHARD_COUNT = 4;
|
|
@@ -25,7 +24,6 @@ export declare function parseConsumerArn(arn: string): {
|
|
|
25
24
|
} | undefined;
|
|
26
25
|
export declare function isChannelArn(arn: string): boolean;
|
|
27
26
|
export declare function streamNameOf(data: Input): string;
|
|
28
|
-
export declare function notFound(context: Context, name: string): import("../errors.ts").KinesisError;
|
|
29
27
|
export declare function getStream(context: Context, name: string): StreamDoc;
|
|
30
28
|
export declare function isOpen(shard: ShardDoc): boolean;
|
|
31
29
|
export declare function streamMode(stream: StreamDoc): import("../store/types.ts").StreamMode;
|
|
@@ -33,7 +31,6 @@ export declare function provisionedOpenShards(context: Context): number;
|
|
|
33
31
|
export declare function shardLimitMessage(context: Context, current: number, additional: number): string;
|
|
34
32
|
export declare function shardCreateTime(shard: ShardDoc): number;
|
|
35
33
|
export declare function nowSeconds(): number;
|
|
36
|
-
export declare function checkTagCharacters(values: string[], action: string, resource: string, context: Context): void;
|
|
37
34
|
export declare function applyTags(existing: Record<string, string>, tags: Record<string, string>, kind: string, name: string, context: Context): void;
|
|
38
35
|
export declare function removeTags(existing: Record<string, string>, keys: string[], kind: string, name: string, context: Context): void;
|
|
39
36
|
export declare function tagList(tags: Record<string, string>): {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { invalidArgument, resourceNotFound } from '../errors.js';
|
|
2
2
|
import { parseSequence } from '../kinesis/sequence.js';
|
|
3
3
|
export const DEFAULT_MAX_RECORD_SIZE_KIB = 1024;
|
|
4
|
-
|
|
4
|
+
const MAX_TAGS_PER_RESOURCE = 50;
|
|
5
5
|
export const MAX_RETENTION_HOURS = 8760;
|
|
6
6
|
export const MIN_RETENTION_HOURS = 24;
|
|
7
7
|
export const ON_DEMAND_SHARD_COUNT = 4;
|
|
@@ -37,7 +37,7 @@ export function streamNameOf(data) {
|
|
|
37
37
|
}
|
|
38
38
|
return (data.StreamName ?? fromArn);
|
|
39
39
|
}
|
|
40
|
-
|
|
40
|
+
function notFound(context, name) {
|
|
41
41
|
return resourceNotFound(`Stream ${name} under account ${account(context)} not found.`);
|
|
42
42
|
}
|
|
43
43
|
export function getStream(context, name) {
|
|
@@ -75,7 +75,7 @@ export function nowSeconds() {
|
|
|
75
75
|
return Math.floor(Date.now() / 1000);
|
|
76
76
|
}
|
|
77
77
|
const INVALID_TAG_CHARACTERS = /[^À-Ⰰ-\w./\-=+_ @%]/;
|
|
78
|
-
|
|
78
|
+
function checkTagCharacters(values, action, resource, context) {
|
|
79
79
|
if (values.some(value => INVALID_TAG_CHARACTERS.test(value))) {
|
|
80
80
|
throw invalidArgument('Some tags contain invalid characters. Valid characters: Unicode letters, digits, white space, _ . / = + - % @.');
|
|
81
81
|
}
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import type { Store } from '../store/store.ts';
|
|
2
|
-
import type { ConsumerDoc } from '../store/types.ts';
|
|
3
2
|
import { type Context, type Input } from './common.ts';
|
|
4
3
|
export declare function consumerNotFound(context: Context, description: string): import("../errors.ts").KinesisError;
|
|
5
|
-
export declare function findConsumer(context: Context, data: Input): ConsumerDoc;
|
|
6
4
|
export declare function registerStreamConsumer(context: Context, data: Input): {
|
|
7
5
|
Consumer: {
|
|
8
6
|
ConsumerName: string;
|
|
9
7
|
ConsumerARN: string;
|
|
10
|
-
ConsumerStatus:
|
|
8
|
+
ConsumerStatus: "ACTIVE" | "CREATING" | "DELETING";
|
|
11
9
|
ConsumerCreationTimestamp: number;
|
|
12
10
|
};
|
|
13
11
|
};
|
|
@@ -15,7 +13,7 @@ export declare function describeStreamConsumer(context: Context, data: Input): {
|
|
|
15
13
|
ConsumerDescription: {
|
|
16
14
|
ConsumerName: string;
|
|
17
15
|
ConsumerARN: string;
|
|
18
|
-
ConsumerStatus:
|
|
16
|
+
ConsumerStatus: "ACTIVE" | "CREATING" | "DELETING";
|
|
19
17
|
ConsumerCreationTimestamp: number;
|
|
20
18
|
StreamARN: string;
|
|
21
19
|
};
|
|
@@ -16,7 +16,7 @@ function streamConsumers(context, streamName) {
|
|
|
16
16
|
export function consumerNotFound(context, description) {
|
|
17
17
|
return resourceNotFound(`Consumer ${description} under account ${account(context)} not found.`);
|
|
18
18
|
}
|
|
19
|
-
|
|
19
|
+
function findConsumer(context, data) {
|
|
20
20
|
if (data.ConsumerARN != null) {
|
|
21
21
|
const consumer = context.store.consumers.get(data.ConsumerARN);
|
|
22
22
|
if (!consumer)
|
|
@@ -4,7 +4,6 @@ import { type Context, type Input } from './common.ts';
|
|
|
4
4
|
export declare function dryRun(data: Input): void;
|
|
5
5
|
export declare function putRecord(context: Context, data: Input): Record<string, unknown>;
|
|
6
6
|
export declare function putRecords(context: Context, data: Input): Record<string, unknown>;
|
|
7
|
-
export declare function latestSequence(stream: StreamDoc, shardIx: number, now?: number): string;
|
|
8
7
|
export interface StartingPosition {
|
|
9
8
|
Type: string;
|
|
10
9
|
SequenceNumber?: string;
|
|
@@ -136,7 +136,7 @@ export function putRecords(context, data) {
|
|
|
136
136
|
result.EncryptionType = encryptionType;
|
|
137
137
|
return result;
|
|
138
138
|
}
|
|
139
|
-
|
|
139
|
+
function latestSequence(stream, shardIx, now = Date.now()) {
|
|
140
140
|
const shard = stream.Shards[shardIx];
|
|
141
141
|
const start = parseSequence(shard.SequenceNumberRange.StartingSequenceNumber);
|
|
142
142
|
return stringifySequence({
|
|
@@ -259,15 +259,17 @@ export function readShard(context, stream, shardIx, sequence, sequenceText, limi
|
|
|
259
259
|
const defaultTime = sequence.seqTime > now ? sequence.seqTime : now;
|
|
260
260
|
const nextSequence = incrementSequence(last ?? sequence, last ? undefined : defaultTime);
|
|
261
261
|
let shardEnded = false;
|
|
262
|
-
let millisBehindLatest = 0;
|
|
263
262
|
const shard = stream.Shards[shardIx];
|
|
264
263
|
if (!records.length && shard.SequenceNumberRange.EndingSequenceNumber) {
|
|
265
264
|
const end = parseSequence(shard.SequenceNumberRange.EndingSequenceNumber);
|
|
266
|
-
if (sequence.seqTime >= end.seqTime)
|
|
265
|
+
if (sequence.seqTime >= end.seqTime)
|
|
267
266
|
shardEnded = true;
|
|
268
|
-
millisBehindLatest = Math.max(0, now - end.seqTime);
|
|
269
|
-
}
|
|
270
267
|
}
|
|
268
|
+
const lastRow = rows.at(-1);
|
|
269
|
+
const newest = store.lastRecord(stream.StreamName, shardIx);
|
|
270
|
+
const millisBehindLatest = lastRow && newest && newest.seq > lastRow.SequenceNumber
|
|
271
|
+
? Math.max(1, now - Math.round(lastRow.ApproximateArrivalTimestamp * 1000))
|
|
272
|
+
: 0;
|
|
271
273
|
if (expired.length)
|
|
272
274
|
setImmediate(() => store.deleteRecords(stream.StreamName, shardIx, expired));
|
|
273
275
|
return { records, nextSequence, shardEnded, millisBehindLatest };
|
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
import type { Store } from '../store/store.ts';
|
|
2
|
-
import type { StreamDoc } from '../store/types.ts';
|
|
3
2
|
import { type Context, type Input } from './common.ts';
|
|
4
3
|
export declare function listShards(context: Context, data: Input): Record<string, unknown>;
|
|
5
|
-
export declare function applySplit(stream: StreamDoc, shardIx: number, hashKey: bigint, now: number): void;
|
|
6
|
-
export declare function applyMerge(stream: StreamDoc, shardIx: number, adjacentShardIx: number, now: number): void;
|
|
7
4
|
export declare function splitShard(context: Context, data: Input): void;
|
|
8
5
|
export declare function mergeShards(context: Context, data: Input): void;
|
|
9
|
-
export declare function reshard(stream: StreamDoc, target: number, now: number): void;
|
|
10
6
|
export declare function updateShardCount(context: Context, data: Input): {
|
|
11
7
|
StreamName: string;
|
|
12
8
|
CurrentShardCount: number;
|
|
@@ -89,7 +89,7 @@ function closeShard(shard, shardIx, now) {
|
|
|
89
89
|
function newShardSequence(stream, now) {
|
|
90
90
|
return stringifySequence({ shardCreateTime: now + 1000, shardIx: stream.Shards.length });
|
|
91
91
|
}
|
|
92
|
-
|
|
92
|
+
function applySplit(stream, shardIx, hashKey, now) {
|
|
93
93
|
const shard = stream.Shards[shardIx];
|
|
94
94
|
closeShard(shard, shardIx, now);
|
|
95
95
|
stream.Shards.push({
|
|
@@ -105,7 +105,7 @@ export function applySplit(stream, shardIx, hashKey, now) {
|
|
|
105
105
|
ShardId: shardIdName(stream.Shards.length),
|
|
106
106
|
});
|
|
107
107
|
}
|
|
108
|
-
|
|
108
|
+
function applyMerge(stream, shardIx, adjacentShardIx, now) {
|
|
109
109
|
const shard = stream.Shards[shardIx];
|
|
110
110
|
const adjacent = stream.Shards[adjacentShardIx];
|
|
111
111
|
closeShard(shard, shardIx, now);
|
|
@@ -175,7 +175,7 @@ export function mergeShards(context, data) {
|
|
|
175
175
|
context.store.saveStream(stream);
|
|
176
176
|
context.store.schedule('stream', name, 'mergeShards', context.store.options.updateStreamMs, { shardIxs: [first.shardIx, second.shardIx] });
|
|
177
177
|
}
|
|
178
|
-
|
|
178
|
+
function reshard(stream, target, now) {
|
|
179
179
|
const width = POW_2_128 / BigInt(target);
|
|
180
180
|
const starts = Array.from({ length: target }, (_, i) => width * BigInt(i));
|
|
181
181
|
for (const boundary of starts.slice(1)) {
|
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
import type { Store } from '../store/store.ts';
|
|
2
|
-
import type { ShardDoc, StreamDoc } from '../store/types.ts';
|
|
3
2
|
import { type Context, type Input } from './common.ts';
|
|
4
|
-
export declare function uniformShards(count: number, createTime: number, firstIndex?: number): ShardDoc[];
|
|
5
3
|
export declare function requireBillingCommitment(context: Context, parameter: string): void;
|
|
6
4
|
export declare function createStream(context: Context, data: Input): void;
|
|
7
5
|
export declare function deleteStream(context: Context, data: Input): void;
|
|
8
|
-
export declare function describeStreamView(stream: StreamDoc, shards: ShardDoc[], hasMoreShards: boolean): Record<string, unknown>;
|
|
9
6
|
export declare function describeStream(context: Context, data: Input): {
|
|
10
7
|
StreamDescription: Record<string, unknown>;
|
|
11
8
|
};
|
|
@@ -2,7 +2,7 @@ import { invalidArgument, limitExceeded, resourceInUse } from '../errors.js';
|
|
|
2
2
|
import { POW_2_128, shardIdName, stringifySequence } from '../kinesis/sequence.js';
|
|
3
3
|
import { ON_DEMAND_SHARD_COUNT, ON_DEMAND_STREAM_LIMIT, account, applyTags, decodeToken, encodeToken, getStream, isOpen, provisionedOpenShards, publicView, shardLimitMessage, streamArn, streamMode, streamNameOf, } from './common.js';
|
|
4
4
|
const SEQ_ADJUST_MS = 2000;
|
|
5
|
-
|
|
5
|
+
function uniformShards(count, createTime, firstIndex = 0) {
|
|
6
6
|
const width = POW_2_128 / BigInt(count);
|
|
7
7
|
const shards = [];
|
|
8
8
|
for (let i = 0; i < count; i++) {
|
|
@@ -106,7 +106,7 @@ export function deleteStream(context, data) {
|
|
|
106
106
|
store.events.emit('stream-deleted', name);
|
|
107
107
|
store.schedule('stream', name, 'deleteStream', store.options.deleteStreamMs);
|
|
108
108
|
}
|
|
109
|
-
|
|
109
|
+
function describeStreamView(stream, shards, hasMoreShards) {
|
|
110
110
|
const view = publicView(stream);
|
|
111
111
|
view.Shards = shards;
|
|
112
112
|
view.HasMoreShards = hasMoreShards;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { KinesisError } from '../errors.ts';
|
|
2
2
|
import { type Protocol } from '../protocol/serialize.ts';
|
|
3
3
|
import { type Context, type Input } from './common.ts';
|
|
4
|
-
export declare const SUBSCRIPTION_MS = 300000;
|
|
5
4
|
export interface EventSink {
|
|
6
5
|
write(chunk: Buffer): void;
|
|
7
6
|
end(): void;
|
|
@@ -5,7 +5,7 @@ import { AMZ_CBOR, encodeBody, serializeShape } from '../protocol/serialize.js';
|
|
|
5
5
|
import { account, getStream } from './common.js';
|
|
6
6
|
import { consumerNotFound } from './consumers.js';
|
|
7
7
|
import { childShards, dryRun, readShard, startingSequence } from './records.js';
|
|
8
|
-
|
|
8
|
+
const SUBSCRIPTION_MS = 300000;
|
|
9
9
|
const RESUBSCRIBE_WINDOW_MS = 5000;
|
|
10
10
|
const HEARTBEAT_MS = 5000;
|
|
11
11
|
const POLL_MS = 1000;
|
|
@@ -110,17 +110,12 @@ export class Subscription {
|
|
|
110
110
|
return;
|
|
111
111
|
}
|
|
112
112
|
const now = Date.now();
|
|
113
|
-
const event = { Records: result.records, MillisBehindLatest:
|
|
114
|
-
if (result.records.length)
|
|
113
|
+
const event = { Records: result.records, MillisBehindLatest: result.millisBehindLatest };
|
|
114
|
+
if (result.records.length)
|
|
115
115
|
this.continuation = result.records[result.records.length - 1].SequenceNumber;
|
|
116
|
-
if (result.records.length === EVENT_RECORD_LIMIT) {
|
|
117
|
-
event.MillisBehindLatest = Math.max(0, now - Math.round(result.records[result.records.length - 1].ApproximateArrivalTimestamp * 1000));
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
116
|
this.position = result.nextSequence;
|
|
121
117
|
if (result.shardEnded) {
|
|
122
118
|
event.ChildShards = childShards(stream, this.shardId);
|
|
123
|
-
event.MillisBehindLatest = 0;
|
|
124
119
|
this.send(event);
|
|
125
120
|
this.stop();
|
|
126
121
|
return;
|
package/dist/protocol/cbor.d.ts
CHANGED
package/dist/protocol/cbor.js
CHANGED
package/dist/store/types.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export interface ShardDoc {
|
|
|
12
12
|
};
|
|
13
13
|
ShardId: string;
|
|
14
14
|
}
|
|
15
|
-
|
|
15
|
+
type StreamStatus = 'CREATING' | 'ACTIVE' | 'UPDATING' | 'DELETING';
|
|
16
16
|
export type StreamMode = 'PROVISIONED' | 'ON_DEMAND';
|
|
17
17
|
export interface Pending {
|
|
18
18
|
action: string;
|
|
@@ -45,7 +45,7 @@ export interface StreamDoc {
|
|
|
45
45
|
_tags: Record<string, string>;
|
|
46
46
|
_pending?: Pending;
|
|
47
47
|
}
|
|
48
|
-
|
|
48
|
+
type ConsumerStatus = 'CREATING' | 'ACTIVE' | 'DELETING';
|
|
49
49
|
export interface ConsumerDoc {
|
|
50
50
|
ConsumerName: string;
|
|
51
51
|
ConsumerARN: string;
|
|
@@ -94,3 +94,4 @@ export interface NewRecord {
|
|
|
94
94
|
arrival: number;
|
|
95
95
|
encryptionType: string | null;
|
|
96
96
|
}
|
|
97
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@saidsef/aws-kinesis-local",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.2",
|
|
4
4
|
"description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"aws",
|