@saidsef/aws-kinesis-local 1.0.3 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/handler.js +9 -0
- package/dist/operations/subscribe.d.ts +1 -1
- package/dist/operations/subscribe.js +6 -4
- package/dist/server.d.ts +1 -0
- package/dist/server.js +46 -8
- package/dist/store/store.d.ts +3 -0
- package/dist/store/store.js +20 -1
- package/package.json +2 -3
package/README.md
CHANGED
|
@@ -57,7 +57,7 @@ Read the Docs publishes the pages under [`docs/`](docs) with MkDocs Material at
|
|
|
57
57
|
| [Kubernetes](docs/kubernetes.md) | Helm, Kustomize and Argo CD installs |
|
|
58
58
|
| [Supported operations](docs/supported-operations.md) | Implemented operations, differences from AWS and KCL setup |
|
|
59
59
|
| [Development](docs/development.md) | Builds, tests, CI workflows, releases and image tags |
|
|
60
|
-
| [Specifications](docs/specs/architecture.md) | Architecture, request validation, storage, enhanced fan-out
|
|
60
|
+
| [Specifications](docs/specs/architecture.md) | Architecture, request validation, storage, enhanced fan-out and testing |
|
|
61
61
|
| [Chart values](charts/kinesis/README.md) | Every Helm value, generated by helm-docs |
|
|
62
62
|
|
|
63
63
|
## Contributing
|
package/dist/handler.js
CHANGED
|
@@ -9,6 +9,7 @@ import { AMZ_CBOR, AMZ_JSON, encodeBody, serializeShape } from './protocol/seria
|
|
|
9
9
|
import { coerceStructure, decodeBlobs, validateStructure } from './validation/validate.js';
|
|
10
10
|
const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
|
|
11
11
|
const EVENT_STREAM = 'application/vnd.amazon.eventstream';
|
|
12
|
+
const HEALTH_PATHS = ['/healthz', '/readyz'];
|
|
12
13
|
const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
|
|
13
14
|
function header(req, name) {
|
|
14
15
|
const value = req.headers[name];
|
|
@@ -37,6 +38,11 @@ function sendError(exchange, contentValid, type, message) {
|
|
|
37
38
|
return sendRaw(exchange, `<${type}/>\n`, message);
|
|
38
39
|
return sendRaw(exchange, `<${type}>\n <Message>${message}</Message>\n</${type}>\n`, 403);
|
|
39
40
|
}
|
|
41
|
+
function sendHealth(store, exchange, path) {
|
|
42
|
+
const healthy = store.healthy() && (path === '/healthz' || !store.draining);
|
|
43
|
+
exchange.res.setHeader('Content-Type', 'text/plain');
|
|
44
|
+
sendRaw(exchange, healthy ? 'ok\n' : 'unavailable\n', healthy ? 200 : 503);
|
|
45
|
+
}
|
|
40
46
|
function validationContext(store, data) {
|
|
41
47
|
let name = typeof data.StreamName === 'string' ? data.StreamName : undefined;
|
|
42
48
|
if (name === undefined && typeof data.StreamARN === 'string')
|
|
@@ -146,6 +152,9 @@ function respond(store, exchange, body) {
|
|
|
146
152
|
}
|
|
147
153
|
res.setHeader('Access-Control-Expose-Headers', EXPOSED_HEADERS);
|
|
148
154
|
}
|
|
155
|
+
const path = req.url?.split('?')[0] ?? '';
|
|
156
|
+
if (req.method === 'GET' && HEALTH_PATHS.includes(path))
|
|
157
|
+
return sendHealth(store, exchange, path);
|
|
149
158
|
if (req.method !== 'POST') {
|
|
150
159
|
return sendError(exchange, false, 'AccessDeniedException', 'Unable to determine service/operation name to be authorized');
|
|
151
160
|
}
|
|
@@ -23,7 +23,7 @@ export declare class Subscription {
|
|
|
23
23
|
private readonly onRecords;
|
|
24
24
|
private readonly onConsumerGone;
|
|
25
25
|
private readonly onStreamDeleted;
|
|
26
|
-
private readonly
|
|
26
|
+
private readonly onDrain;
|
|
27
27
|
constructor(context: Context, consumerArn: string, streamName: string, shardIx: number, shardId: string, position: string);
|
|
28
28
|
get protocol(): Protocol;
|
|
29
29
|
get payloadType(): "application/json" | "application/x-amz-cbor-1.1";
|
|
@@ -44,7 +44,7 @@ export class Subscription {
|
|
|
44
44
|
onRecords;
|
|
45
45
|
onConsumerGone;
|
|
46
46
|
onStreamDeleted;
|
|
47
|
-
|
|
47
|
+
onDrain;
|
|
48
48
|
constructor(context, consumerArn, streamName, shardIx, shardId, position) {
|
|
49
49
|
this.context = context;
|
|
50
50
|
this.key = `${consumerArn}|${shardId}`;
|
|
@@ -66,7 +66,7 @@ export class Subscription {
|
|
|
66
66
|
if (name === this.streamName)
|
|
67
67
|
this.fail(resourceNotFound(`Stream ${name} under account ${account(this.context)} not found.`));
|
|
68
68
|
};
|
|
69
|
-
this.
|
|
69
|
+
this.onDrain = () => this.stop();
|
|
70
70
|
}
|
|
71
71
|
get protocol() {
|
|
72
72
|
return this.context.protocol;
|
|
@@ -81,9 +81,11 @@ export class Subscription {
|
|
|
81
81
|
events.on('consumer-deleting', this.onConsumerGone);
|
|
82
82
|
events.on('consumer-removed', this.onConsumerGone);
|
|
83
83
|
events.on('stream-deleted', this.onStreamDeleted);
|
|
84
|
-
events.on('
|
|
84
|
+
events.on('drain', this.onDrain);
|
|
85
85
|
sink.onClose(() => this.stop());
|
|
86
86
|
sink.write(encodeEvent({ ':event-type': 'initial-response', ':content-type': this.payloadType, ':message-type': 'event' }, encodeBody({}, this.protocol)));
|
|
87
|
+
if (this.context.store.draining)
|
|
88
|
+
return this.stop();
|
|
87
89
|
this.deadline = setTimeout(() => this.stop(), SUBSCRIPTION_MS);
|
|
88
90
|
this.schedule(0);
|
|
89
91
|
}
|
|
@@ -153,7 +155,7 @@ export class Subscription {
|
|
|
153
155
|
events.off('consumer-deleting', this.onConsumerGone);
|
|
154
156
|
events.off('consumer-removed', this.onConsumerGone);
|
|
155
157
|
events.off('stream-deleted', this.onStreamDeleted);
|
|
156
|
-
events.off('
|
|
158
|
+
events.off('drain', this.onDrain);
|
|
157
159
|
const { active } = stateOf(this.context.store);
|
|
158
160
|
if (active.get(this.key) === this)
|
|
159
161
|
active.delete(this.key);
|
package/dist/server.d.ts
CHANGED
package/dist/server.js
CHANGED
|
@@ -18,6 +18,7 @@ export function defaultOptions() {
|
|
|
18
18
|
shardLimit: 10,
|
|
19
19
|
accountId: (process.env.AWS_ACCOUNT_ID || '0000-0000-0000').replace(/[^\d]/g, ''),
|
|
20
20
|
region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
|
|
21
|
+
shutdownMs: 5000,
|
|
21
22
|
};
|
|
22
23
|
}
|
|
23
24
|
function routeByProtocol(socket, http1, http2) {
|
|
@@ -52,18 +53,40 @@ export async function start(overrides = {}) {
|
|
|
52
53
|
registerTransitions(store);
|
|
53
54
|
store.startExpiry();
|
|
54
55
|
store.resume();
|
|
55
|
-
const
|
|
56
|
+
const handle = createHandler(store);
|
|
56
57
|
const sockets = new Set();
|
|
58
|
+
const sessions = new Set();
|
|
59
|
+
let inFlight = 0;
|
|
60
|
+
let onIdle = () => { };
|
|
61
|
+
const handler = (req, res) => {
|
|
62
|
+
const response = res;
|
|
63
|
+
inFlight++;
|
|
64
|
+
response.once('close', () => {
|
|
65
|
+
inFlight--;
|
|
66
|
+
if (inFlight === 0)
|
|
67
|
+
onIdle();
|
|
68
|
+
});
|
|
69
|
+
if (store.draining && req.httpVersionMajor < 2)
|
|
70
|
+
response.setHeader('Connection', 'close');
|
|
71
|
+
handle(req, res);
|
|
72
|
+
};
|
|
73
|
+
const trackSession = (session) => {
|
|
74
|
+
sessions.add(session);
|
|
75
|
+
session.once('close', () => sessions.delete(session));
|
|
76
|
+
};
|
|
57
77
|
let server;
|
|
58
78
|
if (options.ssl) {
|
|
59
79
|
const credentials = options.keyFile && options.certFile
|
|
60
80
|
? { key: readFileSync(options.keyFile), cert: readFileSync(options.certFile) }
|
|
61
81
|
: selfSignedCertificate();
|
|
62
|
-
|
|
82
|
+
const secure = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
|
|
83
|
+
secure.on('session', trackSession);
|
|
84
|
+
server = secure;
|
|
63
85
|
}
|
|
64
86
|
else {
|
|
65
87
|
const http1 = createHttpServer(handler);
|
|
66
88
|
const http2 = createHttp2Server(handler);
|
|
89
|
+
http2.on('session', trackSession);
|
|
67
90
|
server = createNetServer(socket => {
|
|
68
91
|
socket.on('error', () => socket.destroy());
|
|
69
92
|
routeByProtocol(socket, http1, http2);
|
|
@@ -88,15 +111,30 @@ export async function start(overrides = {}) {
|
|
|
88
111
|
}
|
|
89
112
|
const address = server.address();
|
|
90
113
|
const protocol = options.ssl ? 'https' : 'http';
|
|
114
|
+
const shutdown = async () => {
|
|
115
|
+
store.drain();
|
|
116
|
+
const stopped = new Promise(resolve => server.close(() => resolve()));
|
|
117
|
+
for (const session of sessions)
|
|
118
|
+
session.close();
|
|
119
|
+
await new Promise(resolve => {
|
|
120
|
+
const timer = setTimeout(resolve, options.shutdownMs);
|
|
121
|
+
onIdle = () => {
|
|
122
|
+
clearTimeout(timer);
|
|
123
|
+
resolve();
|
|
124
|
+
};
|
|
125
|
+
if (inFlight === 0)
|
|
126
|
+
onIdle();
|
|
127
|
+
});
|
|
128
|
+
store.close();
|
|
129
|
+
for (const socket of sockets)
|
|
130
|
+
socket.destroy();
|
|
131
|
+
await stopped;
|
|
132
|
+
};
|
|
133
|
+
let closing;
|
|
91
134
|
return {
|
|
92
135
|
endpoint: `${protocol}://${hostname(address)}:${address.port}`,
|
|
93
136
|
address,
|
|
94
137
|
protocol,
|
|
95
|
-
close: () =>
|
|
96
|
-
store.close();
|
|
97
|
-
for (const socket of sockets)
|
|
98
|
-
socket.destroy();
|
|
99
|
-
server.close(() => resolve());
|
|
100
|
-
}),
|
|
138
|
+
close: () => closing ??= shutdown(),
|
|
101
139
|
};
|
|
102
140
|
}
|
package/dist/store/store.d.ts
CHANGED
|
@@ -21,6 +21,7 @@ export declare class Store {
|
|
|
21
21
|
readonly policies: Map<string, string>;
|
|
22
22
|
readonly events: EventEmitter<any>;
|
|
23
23
|
settings: AccountSettings;
|
|
24
|
+
draining: boolean;
|
|
24
25
|
private readonly db;
|
|
25
26
|
private readonly transitions;
|
|
26
27
|
private readonly timers;
|
|
@@ -62,6 +63,8 @@ export declare class Store {
|
|
|
62
63
|
private fire;
|
|
63
64
|
cancel(kind: Kind, key: string): void;
|
|
64
65
|
resume(): void;
|
|
66
|
+
healthy(): boolean;
|
|
67
|
+
drain(): void;
|
|
65
68
|
close(): void;
|
|
66
69
|
}
|
|
67
70
|
export {};
|
package/dist/store/store.js
CHANGED
|
@@ -13,6 +13,7 @@ export class Store {
|
|
|
13
13
|
policies = new Map();
|
|
14
14
|
events = new EventEmitter();
|
|
15
15
|
settings = { MinimumThroughputBillingCommitment: { Status: 'DISABLED' } };
|
|
16
|
+
draining = false;
|
|
16
17
|
db;
|
|
17
18
|
transitions = new Map();
|
|
18
19
|
timers = new Map();
|
|
@@ -57,6 +58,7 @@ export class Store {
|
|
|
57
58
|
oldestSequences: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? ORDER BY seq LIMIT ?'),
|
|
58
59
|
deleteRecord: this.db.prepare('DELETE FROM records WHERE stream = ? AND shard = ? AND seq = ?'),
|
|
59
60
|
deleteStreamRecords: this.db.prepare('DELETE FROM records WHERE stream = ?'),
|
|
61
|
+
schemaVersion: this.db.prepare("SELECT value FROM metadata WHERE key = 'schema'"),
|
|
60
62
|
};
|
|
61
63
|
this.load();
|
|
62
64
|
}
|
|
@@ -277,13 +279,30 @@ export class Store {
|
|
|
277
279
|
}
|
|
278
280
|
}
|
|
279
281
|
}
|
|
282
|
+
healthy() {
|
|
283
|
+
if (this.closed)
|
|
284
|
+
return false;
|
|
285
|
+
try {
|
|
286
|
+
this.statements.schemaVersion.get();
|
|
287
|
+
return true;
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
drain() {
|
|
294
|
+
if (this.draining)
|
|
295
|
+
return;
|
|
296
|
+
this.draining = true;
|
|
297
|
+
this.events.emit('drain');
|
|
298
|
+
}
|
|
280
299
|
close() {
|
|
281
300
|
if (this.closed)
|
|
282
301
|
return;
|
|
302
|
+
this.drain();
|
|
283
303
|
for (const timer of this.timers.values())
|
|
284
304
|
clearTimeout(timer);
|
|
285
305
|
this.timers.clear();
|
|
286
|
-
this.events.emit('close');
|
|
287
306
|
this.closed = true;
|
|
288
307
|
this.db.close();
|
|
289
308
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@saidsef/aws-kinesis-local",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"aws",
|
|
@@ -10,8 +10,7 @@
|
|
|
10
10
|
"mock",
|
|
11
11
|
"testing",
|
|
12
12
|
"kcl",
|
|
13
|
-
"enhanced-fan-out"
|
|
14
|
-
"kinesalite"
|
|
13
|
+
"enhanced-fan-out"
|
|
15
14
|
],
|
|
16
15
|
"author": "Said Sef <saidsef@gmail.com>",
|
|
17
16
|
"license": "MIT",
|