@saidsef/aws-kinesis-local 1.0.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # AWS Kinesis Local [![CI](https://github.com/saidsef/aws-kinesis-local/actions/workflows/ci.yml/badge.svg)](https://github.com/saidsef/aws-kinesis-local/actions/workflows/ci.yml) [![Release](https://github.com/saidsef/aws-kinesis-local/actions/workflows/release.yml/badge.svg)](https://github.com/saidsef/aws-kinesis-local/actions/workflows/release.yml) [![npm](https://img.shields.io/npm/v/@saidsef/aws-kinesis-local)](https://www.npmjs.com/package/@saidsef/aws-kinesis-local)
2
2
 
3
- aws-kinesis-local serves the Amazon Kinesis Data Streams API on your machine, for development and CI. It implements all 45 Kinesis operations, including enhanced fan-out, and serves JSON and CBOR over HTTP/1.1 and HTTP/2 on one port, so the AWS CLI and the AWS SDKs work with their default settings. It ships as an npm package, a Docker image and a Helm chart.
3
+ aws-kinesis-local serves the Amazon Kinesis Data Streams API on your machine, for development and CI. It implements all 45 Kinesis operations, including enhanced fan-out, and serves JSON and CBOR over HTTP/1.1 and HTTP/2 on the same port, so the AWS CLI and the AWS SDKs work with their default settings. It ships as an npm package, a Docker image and a Helm chart.
4
4
 
5
5
  ## Quick start
6
6
 
package/dist/handler.d.ts CHANGED
@@ -1,7 +1,6 @@
1
1
  import type { IncomingMessage, ServerResponse } from 'node:http';
2
2
  import type { Http2ServerRequest, Http2ServerResponse } from 'node:http2';
3
3
  import type { Store } from './store/store.ts';
4
- export declare const MAX_REQUEST_BYTES: number;
5
4
  type Request = IncomingMessage | Http2ServerRequest;
6
5
  type Response = ServerResponse | Http2ServerResponse;
7
6
  export declare function createHandler(store: Store): (req: Request, res: Response) => void;
package/dist/handler.js CHANGED
@@ -7,7 +7,7 @@ import { Subscription } from './operations/subscribe.js';
7
7
  import { decodeCbor } from './protocol/cbor.js';
8
8
  import { AMZ_CBOR, AMZ_JSON, encodeBody, serializeShape } from './protocol/serialize.js';
9
9
  import { coerceStructure, decodeBlobs, validateStructure } from './validation/validate.js';
10
- export const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
10
+ const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
11
11
  const EVENT_STREAM = 'application/vnd.amazon.eventstream';
12
12
  const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
13
13
  function header(req, name) {
@@ -8,7 +8,6 @@ export interface Context {
8
8
  }
9
9
  export type Handler = (context: Context, data: Input) => unknown;
10
10
  export declare const DEFAULT_MAX_RECORD_SIZE_KIB = 1024;
11
- export declare const MAX_TAGS_PER_RESOURCE = 50;
12
11
  export declare const MAX_RETENTION_HOURS = 8760;
13
12
  export declare const MIN_RETENTION_HOURS = 24;
14
13
  export declare const ON_DEMAND_SHARD_COUNT = 4;
@@ -25,7 +24,6 @@ export declare function parseConsumerArn(arn: string): {
25
24
  } | undefined;
26
25
  export declare function isChannelArn(arn: string): boolean;
27
26
  export declare function streamNameOf(data: Input): string;
28
- export declare function notFound(context: Context, name: string): import("../errors.ts").KinesisError;
29
27
  export declare function getStream(context: Context, name: string): StreamDoc;
30
28
  export declare function isOpen(shard: ShardDoc): boolean;
31
29
  export declare function streamMode(stream: StreamDoc): import("../store/types.ts").StreamMode;
@@ -33,7 +31,6 @@ export declare function provisionedOpenShards(context: Context): number;
33
31
  export declare function shardLimitMessage(context: Context, current: number, additional: number): string;
34
32
  export declare function shardCreateTime(shard: ShardDoc): number;
35
33
  export declare function nowSeconds(): number;
36
- export declare function checkTagCharacters(values: string[], action: string, resource: string, context: Context): void;
37
34
  export declare function applyTags(existing: Record<string, string>, tags: Record<string, string>, kind: string, name: string, context: Context): void;
38
35
  export declare function removeTags(existing: Record<string, string>, keys: string[], kind: string, name: string, context: Context): void;
39
36
  export declare function tagList(tags: Record<string, string>): {
@@ -1,7 +1,7 @@
1
1
  import { invalidArgument, resourceNotFound } from '../errors.js';
2
2
  import { parseSequence } from '../kinesis/sequence.js';
3
3
  export const DEFAULT_MAX_RECORD_SIZE_KIB = 1024;
4
- export const MAX_TAGS_PER_RESOURCE = 50;
4
+ const MAX_TAGS_PER_RESOURCE = 50;
5
5
  export const MAX_RETENTION_HOURS = 8760;
6
6
  export const MIN_RETENTION_HOURS = 24;
7
7
  export const ON_DEMAND_SHARD_COUNT = 4;
@@ -37,7 +37,7 @@ export function streamNameOf(data) {
37
37
  }
38
38
  return (data.StreamName ?? fromArn);
39
39
  }
40
- export function notFound(context, name) {
40
+ function notFound(context, name) {
41
41
  return resourceNotFound(`Stream ${name} under account ${account(context)} not found.`);
42
42
  }
43
43
  export function getStream(context, name) {
@@ -75,7 +75,7 @@ export function nowSeconds() {
75
75
  return Math.floor(Date.now() / 1000);
76
76
  }
77
77
  const INVALID_TAG_CHARACTERS = /[^À-῿Ⰰ-퟿\w./\-=+_ @%]/;
78
- export function checkTagCharacters(values, action, resource, context) {
78
+ function checkTagCharacters(values, action, resource, context) {
79
79
  if (values.some(value => INVALID_TAG_CHARACTERS.test(value))) {
80
80
  throw invalidArgument('Some tags contain invalid characters. Valid characters: Unicode letters, digits, white space, _ . / = + - % @.');
81
81
  }
@@ -1,13 +1,11 @@
1
1
  import type { Store } from '../store/store.ts';
2
- import type { ConsumerDoc } from '../store/types.ts';
3
2
  import { type Context, type Input } from './common.ts';
4
3
  export declare function consumerNotFound(context: Context, description: string): import("../errors.ts").KinesisError;
5
- export declare function findConsumer(context: Context, data: Input): ConsumerDoc;
6
4
  export declare function registerStreamConsumer(context: Context, data: Input): {
7
5
  Consumer: {
8
6
  ConsumerName: string;
9
7
  ConsumerARN: string;
10
- ConsumerStatus: import("../store/types.ts").ConsumerStatus;
8
+ ConsumerStatus: "ACTIVE" | "CREATING" | "DELETING";
11
9
  ConsumerCreationTimestamp: number;
12
10
  };
13
11
  };
@@ -15,7 +13,7 @@ export declare function describeStreamConsumer(context: Context, data: Input): {
15
13
  ConsumerDescription: {
16
14
  ConsumerName: string;
17
15
  ConsumerARN: string;
18
- ConsumerStatus: import("../store/types.ts").ConsumerStatus;
16
+ ConsumerStatus: "ACTIVE" | "CREATING" | "DELETING";
19
17
  ConsumerCreationTimestamp: number;
20
18
  StreamARN: string;
21
19
  };
@@ -16,7 +16,7 @@ function streamConsumers(context, streamName) {
16
16
  export function consumerNotFound(context, description) {
17
17
  return resourceNotFound(`Consumer ${description} under account ${account(context)} not found.`);
18
18
  }
19
- export function findConsumer(context, data) {
19
+ function findConsumer(context, data) {
20
20
  if (data.ConsumerARN != null) {
21
21
  const consumer = context.store.consumers.get(data.ConsumerARN);
22
22
  if (!consumer)
@@ -4,7 +4,6 @@ import { type Context, type Input } from './common.ts';
4
4
  export declare function dryRun(data: Input): void;
5
5
  export declare function putRecord(context: Context, data: Input): Record<string, unknown>;
6
6
  export declare function putRecords(context: Context, data: Input): Record<string, unknown>;
7
- export declare function latestSequence(stream: StreamDoc, shardIx: number, now?: number): string;
8
7
  export interface StartingPosition {
9
8
  Type: string;
10
9
  SequenceNumber?: string;
@@ -136,7 +136,7 @@ export function putRecords(context, data) {
136
136
  result.EncryptionType = encryptionType;
137
137
  return result;
138
138
  }
139
- export function latestSequence(stream, shardIx, now = Date.now()) {
139
+ function latestSequence(stream, shardIx, now = Date.now()) {
140
140
  const shard = stream.Shards[shardIx];
141
141
  const start = parseSequence(shard.SequenceNumberRange.StartingSequenceNumber);
142
142
  return stringifySequence({
@@ -259,15 +259,17 @@ export function readShard(context, stream, shardIx, sequence, sequenceText, limi
259
259
  const defaultTime = sequence.seqTime > now ? sequence.seqTime : now;
260
260
  const nextSequence = incrementSequence(last ?? sequence, last ? undefined : defaultTime);
261
261
  let shardEnded = false;
262
- let millisBehindLatest = 0;
263
262
  const shard = stream.Shards[shardIx];
264
263
  if (!records.length && shard.SequenceNumberRange.EndingSequenceNumber) {
265
264
  const end = parseSequence(shard.SequenceNumberRange.EndingSequenceNumber);
266
- if (sequence.seqTime >= end.seqTime) {
265
+ if (sequence.seqTime >= end.seqTime)
267
266
  shardEnded = true;
268
- millisBehindLatest = Math.max(0, now - end.seqTime);
269
- }
270
267
  }
268
+ const lastRow = rows.at(-1);
269
+ const newest = store.lastRecord(stream.StreamName, shardIx);
270
+ const millisBehindLatest = lastRow && newest && newest.seq > lastRow.SequenceNumber
271
+ ? Math.max(1, now - Math.round(lastRow.ApproximateArrivalTimestamp * 1000))
272
+ : 0;
271
273
  if (expired.length)
272
274
  setImmediate(() => store.deleteRecords(stream.StreamName, shardIx, expired));
273
275
  return { records, nextSequence, shardEnded, millisBehindLatest };
@@ -1,12 +1,8 @@
1
1
  import type { Store } from '../store/store.ts';
2
- import type { StreamDoc } from '../store/types.ts';
3
2
  import { type Context, type Input } from './common.ts';
4
3
  export declare function listShards(context: Context, data: Input): Record<string, unknown>;
5
- export declare function applySplit(stream: StreamDoc, shardIx: number, hashKey: bigint, now: number): void;
6
- export declare function applyMerge(stream: StreamDoc, shardIx: number, adjacentShardIx: number, now: number): void;
7
4
  export declare function splitShard(context: Context, data: Input): void;
8
5
  export declare function mergeShards(context: Context, data: Input): void;
9
- export declare function reshard(stream: StreamDoc, target: number, now: number): void;
10
6
  export declare function updateShardCount(context: Context, data: Input): {
11
7
  StreamName: string;
12
8
  CurrentShardCount: number;
@@ -89,7 +89,7 @@ function closeShard(shard, shardIx, now) {
89
89
  function newShardSequence(stream, now) {
90
90
  return stringifySequence({ shardCreateTime: now + 1000, shardIx: stream.Shards.length });
91
91
  }
92
- export function applySplit(stream, shardIx, hashKey, now) {
92
+ function applySplit(stream, shardIx, hashKey, now) {
93
93
  const shard = stream.Shards[shardIx];
94
94
  closeShard(shard, shardIx, now);
95
95
  stream.Shards.push({
@@ -105,7 +105,7 @@ export function applySplit(stream, shardIx, hashKey, now) {
105
105
  ShardId: shardIdName(stream.Shards.length),
106
106
  });
107
107
  }
108
- export function applyMerge(stream, shardIx, adjacentShardIx, now) {
108
+ function applyMerge(stream, shardIx, adjacentShardIx, now) {
109
109
  const shard = stream.Shards[shardIx];
110
110
  const adjacent = stream.Shards[adjacentShardIx];
111
111
  closeShard(shard, shardIx, now);
@@ -175,7 +175,7 @@ export function mergeShards(context, data) {
175
175
  context.store.saveStream(stream);
176
176
  context.store.schedule('stream', name, 'mergeShards', context.store.options.updateStreamMs, { shardIxs: [first.shardIx, second.shardIx] });
177
177
  }
178
- export function reshard(stream, target, now) {
178
+ function reshard(stream, target, now) {
179
179
  const width = POW_2_128 / BigInt(target);
180
180
  const starts = Array.from({ length: target }, (_, i) => width * BigInt(i));
181
181
  for (const boundary of starts.slice(1)) {
@@ -1,11 +1,8 @@
1
1
  import type { Store } from '../store/store.ts';
2
- import type { ShardDoc, StreamDoc } from '../store/types.ts';
3
2
  import { type Context, type Input } from './common.ts';
4
- export declare function uniformShards(count: number, createTime: number, firstIndex?: number): ShardDoc[];
5
3
  export declare function requireBillingCommitment(context: Context, parameter: string): void;
6
4
  export declare function createStream(context: Context, data: Input): void;
7
5
  export declare function deleteStream(context: Context, data: Input): void;
8
- export declare function describeStreamView(stream: StreamDoc, shards: ShardDoc[], hasMoreShards: boolean): Record<string, unknown>;
9
6
  export declare function describeStream(context: Context, data: Input): {
10
7
  StreamDescription: Record<string, unknown>;
11
8
  };
@@ -2,7 +2,7 @@ import { invalidArgument, limitExceeded, resourceInUse } from '../errors.js';
2
2
  import { POW_2_128, shardIdName, stringifySequence } from '../kinesis/sequence.js';
3
3
  import { ON_DEMAND_SHARD_COUNT, ON_DEMAND_STREAM_LIMIT, account, applyTags, decodeToken, encodeToken, getStream, isOpen, provisionedOpenShards, publicView, shardLimitMessage, streamArn, streamMode, streamNameOf, } from './common.js';
4
4
  const SEQ_ADJUST_MS = 2000;
5
- export function uniformShards(count, createTime, firstIndex = 0) {
5
+ function uniformShards(count, createTime, firstIndex = 0) {
6
6
  const width = POW_2_128 / BigInt(count);
7
7
  const shards = [];
8
8
  for (let i = 0; i < count; i++) {
@@ -106,7 +106,7 @@ export function deleteStream(context, data) {
106
106
  store.events.emit('stream-deleted', name);
107
107
  store.schedule('stream', name, 'deleteStream', store.options.deleteStreamMs);
108
108
  }
109
- export function describeStreamView(stream, shards, hasMoreShards) {
109
+ function describeStreamView(stream, shards, hasMoreShards) {
110
110
  const view = publicView(stream);
111
111
  view.Shards = shards;
112
112
  view.HasMoreShards = hasMoreShards;
@@ -1,7 +1,6 @@
1
1
  import { KinesisError } from '../errors.ts';
2
2
  import { type Protocol } from '../protocol/serialize.ts';
3
3
  import { type Context, type Input } from './common.ts';
4
- export declare const SUBSCRIPTION_MS = 300000;
5
4
  export interface EventSink {
6
5
  write(chunk: Buffer): void;
7
6
  end(): void;
@@ -5,7 +5,7 @@ import { AMZ_CBOR, encodeBody, serializeShape } from '../protocol/serialize.js';
5
5
  import { account, getStream } from './common.js';
6
6
  import { consumerNotFound } from './consumers.js';
7
7
  import { childShards, dryRun, readShard, startingSequence } from './records.js';
8
- export const SUBSCRIPTION_MS = 300000;
8
+ const SUBSCRIPTION_MS = 300000;
9
9
  const RESUBSCRIBE_WINDOW_MS = 5000;
10
10
  const HEARTBEAT_MS = 5000;
11
11
  const POLL_MS = 1000;
@@ -110,17 +110,12 @@ export class Subscription {
110
110
  return;
111
111
  }
112
112
  const now = Date.now();
113
- const event = { Records: result.records, MillisBehindLatest: 0 };
114
- if (result.records.length) {
113
+ const event = { Records: result.records, MillisBehindLatest: result.millisBehindLatest };
114
+ if (result.records.length)
115
115
  this.continuation = result.records[result.records.length - 1].SequenceNumber;
116
- if (result.records.length === EVENT_RECORD_LIMIT) {
117
- event.MillisBehindLatest = Math.max(0, now - Math.round(result.records[result.records.length - 1].ApproximateArrivalTimestamp * 1000));
118
- }
119
- }
120
116
  this.position = result.nextSequence;
121
117
  if (result.shardEnded) {
122
118
  event.ChildShards = childShards(stream, this.shardId);
123
- event.MillisBehindLatest = 0;
124
119
  this.send(event);
125
120
  this.stop();
126
121
  return;
@@ -1,4 +1,2 @@
1
- export declare class CborError extends Error {
2
- }
3
1
  export declare function decodeCbor(buffer: Buffer): unknown;
4
2
  export declare function encodeCbor(value: unknown): Buffer;
@@ -1,5 +1,5 @@
1
1
  const MILLISECOND_THRESHOLD = 1e11;
2
- export class CborError extends Error {
2
+ class CborError extends Error {
3
3
  }
4
4
  class Reader {
5
5
  buffer;
@@ -12,7 +12,7 @@ export interface ShardDoc {
12
12
  };
13
13
  ShardId: string;
14
14
  }
15
- export type StreamStatus = 'CREATING' | 'ACTIVE' | 'UPDATING' | 'DELETING';
15
+ type StreamStatus = 'CREATING' | 'ACTIVE' | 'UPDATING' | 'DELETING';
16
16
  export type StreamMode = 'PROVISIONED' | 'ON_DEMAND';
17
17
  export interface Pending {
18
18
  action: string;
@@ -45,7 +45,7 @@ export interface StreamDoc {
45
45
  _tags: Record<string, string>;
46
46
  _pending?: Pending;
47
47
  }
48
- export type ConsumerStatus = 'CREATING' | 'ACTIVE' | 'DELETING';
48
+ type ConsumerStatus = 'CREATING' | 'ACTIVE' | 'DELETING';
49
49
  export interface ConsumerDoc {
50
50
  ConsumerName: string;
51
51
  ConsumerARN: string;
@@ -94,3 +94,4 @@ export interface NewRecord {
94
94
  arrival: number;
95
95
  encryptionType: string | null;
96
96
  }
97
+ export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@saidsef/aws-kinesis-local",
3
- "version": "1.0.0",
3
+ "version": "1.0.2",
4
4
  "description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
5
5
  "keywords": [
6
6
  "aws",