@saidsef/aws-kinesis-local 1.0.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/handler.js CHANGED
@@ -9,6 +9,7 @@ import { AMZ_CBOR, AMZ_JSON, encodeBody, serializeShape } from './protocol/seria
9
9
  import { coerceStructure, decodeBlobs, validateStructure } from './validation/validate.js';
10
10
  const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
11
11
  const EVENT_STREAM = 'application/vnd.amazon.eventstream';
12
+ const HEALTH_PATHS = ['/healthz', '/readyz'];
12
13
  const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
13
14
  function header(req, name) {
14
15
  const value = req.headers[name];
@@ -37,6 +38,11 @@ function sendError(exchange, contentValid, type, message) {
37
38
  return sendRaw(exchange, `<${type}/>\n`, message);
38
39
  return sendRaw(exchange, `<${type}>\n <Message>${message}</Message>\n</${type}>\n`, 403);
39
40
  }
41
+ function sendHealth(store, exchange, path) {
42
+ const healthy = store.healthy() && (path === '/healthz' || !store.draining);
43
+ exchange.res.setHeader('Content-Type', 'text/plain');
44
+ sendRaw(exchange, healthy ? 'ok\n' : 'unavailable\n', healthy ? 200 : 503);
45
+ }
40
46
  function validationContext(store, data) {
41
47
  let name = typeof data.StreamName === 'string' ? data.StreamName : undefined;
42
48
  if (name === undefined && typeof data.StreamARN === 'string')
@@ -146,6 +152,9 @@ function respond(store, exchange, body) {
146
152
  }
147
153
  res.setHeader('Access-Control-Expose-Headers', EXPOSED_HEADERS);
148
154
  }
155
+ const path = req.url?.split('?')[0] ?? '';
156
+ if (req.method === 'GET' && HEALTH_PATHS.includes(path))
157
+ return sendHealth(store, exchange, path);
149
158
  if (req.method !== 'POST') {
150
159
  return sendError(exchange, false, 'AccessDeniedException', 'Unable to determine service/operation name to be authorized');
151
160
  }
@@ -23,7 +23,7 @@ export declare class Subscription {
23
23
  private readonly onRecords;
24
24
  private readonly onConsumerGone;
25
25
  private readonly onStreamDeleted;
26
- private readonly onClose;
26
+ private readonly onDrain;
27
27
  constructor(context: Context, consumerArn: string, streamName: string, shardIx: number, shardId: string, position: string);
28
28
  get protocol(): Protocol;
29
29
  get payloadType(): "application/json" | "application/x-amz-cbor-1.1";
@@ -44,7 +44,7 @@ export class Subscription {
44
44
  onRecords;
45
45
  onConsumerGone;
46
46
  onStreamDeleted;
47
- onClose;
47
+ onDrain;
48
48
  constructor(context, consumerArn, streamName, shardIx, shardId, position) {
49
49
  this.context = context;
50
50
  this.key = `${consumerArn}|${shardId}`;
@@ -66,7 +66,7 @@ export class Subscription {
66
66
  if (name === this.streamName)
67
67
  this.fail(resourceNotFound(`Stream ${name} under account ${account(this.context)} not found.`));
68
68
  };
69
- this.onClose = () => this.stop();
69
+ this.onDrain = () => this.stop();
70
70
  }
71
71
  get protocol() {
72
72
  return this.context.protocol;
@@ -81,9 +81,11 @@ export class Subscription {
81
81
  events.on('consumer-deleting', this.onConsumerGone);
82
82
  events.on('consumer-removed', this.onConsumerGone);
83
83
  events.on('stream-deleted', this.onStreamDeleted);
84
- events.on('close', this.onClose);
84
+ events.on('drain', this.onDrain);
85
85
  sink.onClose(() => this.stop());
86
86
  sink.write(encodeEvent({ ':event-type': 'initial-response', ':content-type': this.payloadType, ':message-type': 'event' }, encodeBody({}, this.protocol)));
87
+ if (this.context.store.draining)
88
+ return this.stop();
87
89
  this.deadline = setTimeout(() => this.stop(), SUBSCRIPTION_MS);
88
90
  this.schedule(0);
89
91
  }
@@ -153,7 +155,7 @@ export class Subscription {
153
155
  events.off('consumer-deleting', this.onConsumerGone);
154
156
  events.off('consumer-removed', this.onConsumerGone);
155
157
  events.off('stream-deleted', this.onStreamDeleted);
156
- events.off('close', this.onClose);
158
+ events.off('drain', this.onDrain);
157
159
  const { active } = stateOf(this.context.store);
158
160
  if (active.get(this.key) === this)
159
161
  active.delete(this.key);
package/dist/server.d.ts CHANGED
@@ -13,6 +13,7 @@ export interface StartOptions {
13
13
  shardLimit: number;
14
14
  accountId: string;
15
15
  region: string;
16
+ shutdownMs: number;
16
17
  }
17
18
  export interface KinesisLocal {
18
19
  endpoint: string;
package/dist/server.js CHANGED
@@ -18,6 +18,7 @@ export function defaultOptions() {
18
18
  shardLimit: 10,
19
19
  accountId: (process.env.AWS_ACCOUNT_ID || '0000-0000-0000').replace(/[^\d]/g, ''),
20
20
  region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
21
+ shutdownMs: 5000,
21
22
  };
22
23
  }
23
24
  function routeByProtocol(socket, http1, http2) {
@@ -52,18 +53,40 @@ export async function start(overrides = {}) {
52
53
  registerTransitions(store);
53
54
  store.startExpiry();
54
55
  store.resume();
55
- const handler = createHandler(store);
56
+ const handle = createHandler(store);
56
57
  const sockets = new Set();
58
+ const sessions = new Set();
59
+ let inFlight = 0;
60
+ let onIdle = () => { };
61
+ const handler = (req, res) => {
62
+ const response = res;
63
+ inFlight++;
64
+ response.once('close', () => {
65
+ inFlight--;
66
+ if (inFlight === 0)
67
+ onIdle();
68
+ });
69
+ if (store.draining && req.httpVersionMajor < 2)
70
+ response.setHeader('Connection', 'close');
71
+ handle(req, res);
72
+ };
73
+ const trackSession = (session) => {
74
+ sessions.add(session);
75
+ session.once('close', () => sessions.delete(session));
76
+ };
57
77
  let server;
58
78
  if (options.ssl) {
59
79
  const credentials = options.keyFile && options.certFile
60
80
  ? { key: readFileSync(options.keyFile), cert: readFileSync(options.certFile) }
61
81
  : selfSignedCertificate();
62
- server = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
82
+ const secure = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
83
+ secure.on('session', trackSession);
84
+ server = secure;
63
85
  }
64
86
  else {
65
87
  const http1 = createHttpServer(handler);
66
88
  const http2 = createHttp2Server(handler);
89
+ http2.on('session', trackSession);
67
90
  server = createNetServer(socket => {
68
91
  socket.on('error', () => socket.destroy());
69
92
  routeByProtocol(socket, http1, http2);
@@ -88,15 +111,30 @@ export async function start(overrides = {}) {
88
111
  }
89
112
  const address = server.address();
90
113
  const protocol = options.ssl ? 'https' : 'http';
114
+ const shutdown = async () => {
115
+ store.drain();
116
+ const stopped = new Promise(resolve => server.close(() => resolve()));
117
+ for (const session of sessions)
118
+ session.close();
119
+ await new Promise(resolve => {
120
+ const timer = setTimeout(resolve, options.shutdownMs);
121
+ onIdle = () => {
122
+ clearTimeout(timer);
123
+ resolve();
124
+ };
125
+ if (inFlight === 0)
126
+ onIdle();
127
+ });
128
+ store.close();
129
+ for (const socket of sockets)
130
+ socket.destroy();
131
+ await stopped;
132
+ };
133
+ let closing;
91
134
  return {
92
135
  endpoint: `${protocol}://${hostname(address)}:${address.port}`,
93
136
  address,
94
137
  protocol,
95
- close: () => new Promise(resolve => {
96
- store.close();
97
- for (const socket of sockets)
98
- socket.destroy();
99
- server.close(() => resolve());
100
- }),
138
+ close: () => closing ??= shutdown(),
101
139
  };
102
140
  }
@@ -21,6 +21,7 @@ export declare class Store {
21
21
  readonly policies: Map<string, string>;
22
22
  readonly events: EventEmitter<any>;
23
23
  settings: AccountSettings;
24
+ draining: boolean;
24
25
  private readonly db;
25
26
  private readonly transitions;
26
27
  private readonly timers;
@@ -62,6 +63,8 @@ export declare class Store {
62
63
  private fire;
63
64
  cancel(kind: Kind, key: string): void;
64
65
  resume(): void;
66
+ healthy(): boolean;
67
+ drain(): void;
65
68
  close(): void;
66
69
  }
67
70
  export {};
@@ -13,6 +13,7 @@ export class Store {
13
13
  policies = new Map();
14
14
  events = new EventEmitter();
15
15
  settings = { MinimumThroughputBillingCommitment: { Status: 'DISABLED' } };
16
+ draining = false;
16
17
  db;
17
18
  transitions = new Map();
18
19
  timers = new Map();
@@ -57,6 +58,7 @@ export class Store {
57
58
  oldestSequences: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? ORDER BY seq LIMIT ?'),
58
59
  deleteRecord: this.db.prepare('DELETE FROM records WHERE stream = ? AND shard = ? AND seq = ?'),
59
60
  deleteStreamRecords: this.db.prepare('DELETE FROM records WHERE stream = ?'),
61
+ schemaVersion: this.db.prepare("SELECT value FROM metadata WHERE key = 'schema'"),
60
62
  };
61
63
  this.load();
62
64
  }
@@ -277,13 +279,30 @@ export class Store {
277
279
  }
278
280
  }
279
281
  }
282
+ healthy() {
283
+ if (this.closed)
284
+ return false;
285
+ try {
286
+ this.statements.schemaVersion.get();
287
+ return true;
288
+ }
289
+ catch {
290
+ return false;
291
+ }
292
+ }
293
+ drain() {
294
+ if (this.draining)
295
+ return;
296
+ this.draining = true;
297
+ this.events.emit('drain');
298
+ }
280
299
  close() {
281
300
  if (this.closed)
282
301
  return;
302
+ this.drain();
283
303
  for (const timer of this.timers.values())
284
304
  clearTimeout(timer);
285
305
  this.timers.clear();
286
- this.events.emit('close');
287
306
  this.closed = true;
288
307
  this.db.close();
289
308
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@saidsef/aws-kinesis-local",
3
- "version": "1.0.3",
3
+ "version": "1.1.0",
4
4
  "description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
5
5
  "keywords": [
6
6
  "aws",