@saidsef/aws-kinesis-local 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +4 -4
- package/dist/handler.js +19 -0
- package/dist/operations/subscribe.js +7 -1
- package/dist/server.d.ts +1 -0
- package/dist/server.js +18 -12
- package/dist/store/store.d.ts +1 -0
- package/dist/store/store.js +26 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -5,15 +5,15 @@ const HELP = [
|
|
|
5
5
|
'',
|
|
6
6
|
'Usage: aws-kinesis-local [--port <port>] [--path <path>] [--ssl] [options]',
|
|
7
7
|
'',
|
|
8
|
-
'Serves the Amazon Kinesis Data Streams API in JSON and CBOR over HTTP/1.1 and HTTP/2, optionally backed by SQLite',
|
|
8
|
+
'Serves the Amazon Kinesis Data Streams API in JSON and CBOR over HTTP/1.1 and HTTP/2, with or without TLS, optionally backed by SQLite',
|
|
9
9
|
'',
|
|
10
10
|
'Options:',
|
|
11
11
|
'--help Display this help message and exit',
|
|
12
12
|
'--port <port> The port to listen on (default: 4567)',
|
|
13
13
|
'--path <path> The directory for the SQLite store (in-memory by default)',
|
|
14
|
-
'--ssl
|
|
15
|
-
'--key <file> PEM private key for
|
|
16
|
-
'--cert <file> PEM certificate for
|
|
14
|
+
'--ssl Accept TLS connections only (default: TLS and plain HTTP on the same port)',
|
|
15
|
+
'--key <file> PEM private key for TLS connections (default: a generated self-signed key)',
|
|
16
|
+
'--cert <file> PEM certificate for TLS connections (default: a generated self-signed certificate)',
|
|
17
17
|
'--createStreamMs <ms> Amount of time streams stay in CREATING state (default: 500)',
|
|
18
18
|
'--deleteStreamMs <ms> Amount of time streams stay in DELETING state (default: 500)',
|
|
19
19
|
'--updateStreamMs <ms> Amount of time streams stay in UPDATING state (default: 500)',
|
package/dist/handler.js
CHANGED
|
@@ -10,6 +10,7 @@ import { coerceStructure, decodeBlobs, validateStructure } from './validation/va
|
|
|
10
10
|
const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
|
|
11
11
|
const EVENT_STREAM = 'application/vnd.amazon.eventstream';
|
|
12
12
|
const HEALTH_PATHS = ['/healthz', '/readyz'];
|
|
13
|
+
const RESET_PATH = '/_control/reset';
|
|
13
14
|
const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
|
|
14
15
|
function header(req, name) {
|
|
15
16
|
const value = req.headers[name];
|
|
@@ -43,6 +44,22 @@ function sendHealth(store, exchange, path) {
|
|
|
43
44
|
exchange.res.setHeader('Content-Type', 'text/plain');
|
|
44
45
|
sendRaw(exchange, healthy ? 'ok\n' : 'unavailable\n', healthy ? 200 : 503);
|
|
45
46
|
}
|
|
47
|
+
function sendReset(store, exchange, body) {
|
|
48
|
+
let input = {};
|
|
49
|
+
try {
|
|
50
|
+
if (body)
|
|
51
|
+
input = JSON.parse(body.toString('utf8'));
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
input = undefined;
|
|
55
|
+
}
|
|
56
|
+
const stream = input?.stream;
|
|
57
|
+
exchange.contentType = 'application/json';
|
|
58
|
+
if (typeof input !== 'object' || input === null || Array.isArray(input) || (stream !== undefined && typeof stream !== 'string')) {
|
|
59
|
+
return sendResponse(exchange, { message: 'The body must be empty or a JSON object whose stream member is a string.' }, 400);
|
|
60
|
+
}
|
|
61
|
+
sendResponse(exchange, { streams: store.reset(stream) });
|
|
62
|
+
}
|
|
46
63
|
function validationContext(store, data) {
|
|
47
64
|
let name = typeof data.StreamName === 'string' ? data.StreamName : undefined;
|
|
48
65
|
if (name === undefined && typeof data.StreamARN === 'string')
|
|
@@ -155,6 +172,8 @@ function respond(store, exchange, body) {
|
|
|
155
172
|
const path = req.url?.split('?')[0] ?? '';
|
|
156
173
|
if (req.method === 'GET' && HEALTH_PATHS.includes(path))
|
|
157
174
|
return sendHealth(store, exchange, path);
|
|
175
|
+
if (req.method === 'POST' && path === RESET_PATH)
|
|
176
|
+
return sendReset(store, exchange, body);
|
|
158
177
|
if (req.method !== 'POST') {
|
|
159
178
|
return sendError(exchange, false, 'AccessDeniedException', 'Unable to determine service/operation name to be authorized');
|
|
160
179
|
}
|
|
@@ -14,7 +14,13 @@ const states = new WeakMap();
|
|
|
14
14
|
function stateOf(store) {
|
|
15
15
|
let state = states.get(store);
|
|
16
16
|
if (!state) {
|
|
17
|
-
|
|
17
|
+
const lastSubscribed = new Map();
|
|
18
|
+
store.events.on('consumer-removed', (arn) => {
|
|
19
|
+
for (const key of lastSubscribed.keys())
|
|
20
|
+
if (key.startsWith(`${arn}|`))
|
|
21
|
+
lastSubscribed.delete(key);
|
|
22
|
+
});
|
|
23
|
+
state = { lastSubscribed, active: new Map() };
|
|
18
24
|
states.set(store, state);
|
|
19
25
|
}
|
|
20
26
|
return state;
|
package/dist/server.d.ts
CHANGED
package/dist/server.js
CHANGED
|
@@ -7,6 +7,7 @@ import { createHandler } from './handler.js';
|
|
|
7
7
|
import { registerTransitions } from './operations/index.js';
|
|
8
8
|
import { Store } from './store/store.js';
|
|
9
9
|
const HTTP2_PREFACE = Buffer.from('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n', 'latin1');
|
|
10
|
+
const TLS_HANDSHAKE = 0x16;
|
|
10
11
|
export function defaultOptions() {
|
|
11
12
|
return {
|
|
12
13
|
port: 4567,
|
|
@@ -21,22 +22,26 @@ export function defaultOptions() {
|
|
|
21
22
|
shutdownMs: 5000,
|
|
22
23
|
};
|
|
23
24
|
}
|
|
24
|
-
function routeByProtocol(socket,
|
|
25
|
+
function routeByProtocol(socket, servers) {
|
|
25
26
|
let buffered = Buffer.alloc(0);
|
|
26
27
|
const onData = (chunk) => {
|
|
27
28
|
buffered = Buffer.concat([buffered, chunk]);
|
|
29
|
+
const isTls = buffered[0] === TLS_HANDSHAKE;
|
|
28
30
|
const length = Math.min(buffered.length, HTTP2_PREFACE.length);
|
|
29
|
-
const isHttp2 = buffered.subarray(0, length).equals(HTTP2_PREFACE.subarray(0, length));
|
|
31
|
+
const isHttp2 = !isTls && buffered.subarray(0, length).equals(HTTP2_PREFACE.subarray(0, length));
|
|
30
32
|
if (isHttp2 && buffered.length < HTTP2_PREFACE.length)
|
|
31
33
|
return;
|
|
32
34
|
socket.off('data', onData);
|
|
33
35
|
socket.pause();
|
|
34
36
|
socket.unshift(buffered);
|
|
35
|
-
if (
|
|
36
|
-
|
|
37
|
+
if (isTls) {
|
|
38
|
+
servers.tls.emit('connection', socket);
|
|
39
|
+
}
|
|
40
|
+
else if (isHttp2) {
|
|
41
|
+
servers.http2.emit('connection', socket);
|
|
37
42
|
}
|
|
38
43
|
else {
|
|
39
|
-
http1.emit('connection', socket);
|
|
44
|
+
servers.http1.emit('connection', socket);
|
|
40
45
|
socket.resume();
|
|
41
46
|
}
|
|
42
47
|
};
|
|
@@ -49,6 +54,9 @@ function hostname(address) {
|
|
|
49
54
|
}
|
|
50
55
|
export async function start(overrides = {}) {
|
|
51
56
|
const options = { ...defaultOptions(), ...overrides };
|
|
57
|
+
const credentials = options.keyFile && options.certFile
|
|
58
|
+
? { key: readFileSync(options.keyFile), cert: readFileSync(options.certFile) }
|
|
59
|
+
: selfSignedCertificate();
|
|
52
60
|
const store = new Store(options);
|
|
53
61
|
registerTransitions(store);
|
|
54
62
|
store.startExpiry();
|
|
@@ -74,22 +82,19 @@ export async function start(overrides = {}) {
|
|
|
74
82
|
sessions.add(session);
|
|
75
83
|
session.once('close', () => sessions.delete(session));
|
|
76
84
|
};
|
|
85
|
+
const secure = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
|
|
86
|
+
secure.on('session', trackSession);
|
|
77
87
|
let server;
|
|
78
88
|
if (options.ssl) {
|
|
79
|
-
const credentials = options.keyFile && options.certFile
|
|
80
|
-
? { key: readFileSync(options.keyFile), cert: readFileSync(options.certFile) }
|
|
81
|
-
: selfSignedCertificate();
|
|
82
|
-
const secure = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
|
|
83
|
-
secure.on('session', trackSession);
|
|
84
89
|
server = secure;
|
|
85
90
|
}
|
|
86
91
|
else {
|
|
87
|
-
const http1 = createHttpServer(handler);
|
|
88
92
|
const http2 = createHttp2Server(handler);
|
|
89
93
|
http2.on('session', trackSession);
|
|
94
|
+
const servers = { http1: createHttpServer(handler), http2: http2, tls: secure };
|
|
90
95
|
server = createNetServer(socket => {
|
|
91
96
|
socket.on('error', () => socket.destroy());
|
|
92
|
-
routeByProtocol(socket,
|
|
97
|
+
routeByProtocol(socket, servers);
|
|
93
98
|
});
|
|
94
99
|
}
|
|
95
100
|
server.on('connection', (socket) => {
|
|
@@ -135,6 +140,7 @@ export async function start(overrides = {}) {
|
|
|
135
140
|
endpoint: `${protocol}://${hostname(address)}:${address.port}`,
|
|
136
141
|
address,
|
|
137
142
|
protocol,
|
|
143
|
+
reset: stream => store.reset(stream),
|
|
138
144
|
close: () => closing ??= shutdown(),
|
|
139
145
|
};
|
|
140
146
|
}
|
package/dist/store/store.d.ts
CHANGED
|
@@ -42,6 +42,7 @@ export declare class Store {
|
|
|
42
42
|
savePolicy(arn: string, policy: string): void;
|
|
43
43
|
removePolicy(arn: string): void;
|
|
44
44
|
saveSettings(settings: AccountSettings): void;
|
|
45
|
+
reset(name?: string): string[];
|
|
45
46
|
insertRecords(stream: string, records: NewRecord[]): void;
|
|
46
47
|
readRecords(stream: string, shardIx: number, fromSequence: string, limit: number): StoredRecord[];
|
|
47
48
|
firstSequenceAtOrAfter(stream: string, shardIx: number, timestamp: number): string | undefined;
|
package/dist/store/store.js
CHANGED
|
@@ -5,6 +5,7 @@ import { DatabaseSync } from 'node:sqlite';
|
|
|
5
5
|
import { parseSequence } from '../kinesis/sequence.js';
|
|
6
6
|
export const DATABASE_FILE = 'aws-kinesis-local.sqlite';
|
|
7
7
|
const EXPIRY_BATCH_SIZE = 500;
|
|
8
|
+
const defaultSettings = () => ({ MinimumThroughputBillingCommitment: { Status: 'DISABLED' } });
|
|
8
9
|
export class Store {
|
|
9
10
|
options;
|
|
10
11
|
streams = new Map();
|
|
@@ -12,7 +13,7 @@ export class Store {
|
|
|
12
13
|
channels = new Map();
|
|
13
14
|
policies = new Map();
|
|
14
15
|
events = new EventEmitter();
|
|
15
|
-
settings =
|
|
16
|
+
settings = defaultSettings();
|
|
16
17
|
draining = false;
|
|
17
18
|
db;
|
|
18
19
|
transitions = new Map();
|
|
@@ -147,6 +148,30 @@ export class Store {
|
|
|
147
148
|
this.settings = settings;
|
|
148
149
|
this.write('settings', 'account', settings);
|
|
149
150
|
}
|
|
151
|
+
reset(name) {
|
|
152
|
+
const names = name === undefined ? [...this.streams.keys()].sort() : this.streams.has(name) ? [name] : [];
|
|
153
|
+
const arns = new Set(names.map(stream => this.streams.get(stream).StreamARN));
|
|
154
|
+
const consumers = [...this.consumers.values()].filter(consumer => names.includes(consumer.StreamName));
|
|
155
|
+
const channels = [...this.channels.values()].filter(channel => name === undefined || channel.StreamConfigurationList.some(entry => arns.has(entry.StreamARN)));
|
|
156
|
+
this.transaction(() => {
|
|
157
|
+
for (const stream of names) {
|
|
158
|
+
this.events.emit('stream-deleted', stream);
|
|
159
|
+
this.removePolicy(this.streams.get(stream).StreamARN);
|
|
160
|
+
this.removeStream(stream);
|
|
161
|
+
}
|
|
162
|
+
for (const consumer of consumers)
|
|
163
|
+
this.removeConsumer(consumer.ConsumerARN);
|
|
164
|
+
for (const channel of channels)
|
|
165
|
+
this.removeChannel(channel.ChannelARN);
|
|
166
|
+
if (name === undefined) {
|
|
167
|
+
for (const arn of [...this.policies.keys()])
|
|
168
|
+
this.removePolicy(arn);
|
|
169
|
+
this.settings = defaultSettings();
|
|
170
|
+
this.erase('settings', 'account');
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
return names;
|
|
174
|
+
}
|
|
150
175
|
insertRecords(stream, records) {
|
|
151
176
|
for (const record of records) {
|
|
152
177
|
this.statements.putRecord.run(stream, record.shardIx, record.sequenceNumber, record.partitionKey, record.data, record.arrival, record.encryptionType);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@saidsef/aws-kinesis-local",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"aws",
|