@saidsef/aws-kinesis-local 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/handler.js CHANGED
@@ -9,6 +9,7 @@ import { AMZ_CBOR, AMZ_JSON, encodeBody, serializeShape } from './protocol/seria
9
9
  import { coerceStructure, decodeBlobs, validateStructure } from './validation/validate.js';
10
10
  const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
11
11
  const EVENT_STREAM = 'application/vnd.amazon.eventstream';
12
+ const HEALTH_PATHS = ['/healthz', '/readyz'];
12
13
  const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
13
14
  function header(req, name) {
14
15
  const value = req.headers[name];
@@ -37,6 +38,11 @@ function sendError(exchange, contentValid, type, message) {
37
38
  return sendRaw(exchange, `<${type}/>\n`, message);
38
39
  return sendRaw(exchange, `<${type}>\n <Message>${message}</Message>\n</${type}>\n`, 403);
39
40
  }
41
+ function sendHealth(store, exchange, path) {
42
+ const healthy = store.healthy() && (path === '/healthz' || !store.draining);
43
+ exchange.res.setHeader('Content-Type', 'text/plain');
44
+ sendRaw(exchange, healthy ? 'ok\n' : 'unavailable\n', healthy ? 200 : 503);
45
+ }
40
46
  function validationContext(store, data) {
41
47
  let name = typeof data.StreamName === 'string' ? data.StreamName : undefined;
42
48
  if (name === undefined && typeof data.StreamARN === 'string')
@@ -146,6 +152,9 @@ function respond(store, exchange, body) {
146
152
  }
147
153
  res.setHeader('Access-Control-Expose-Headers', EXPOSED_HEADERS);
148
154
  }
155
+ const path = req.url?.split('?')[0] ?? '';
156
+ if (req.method === 'GET' && HEALTH_PATHS.includes(path))
157
+ return sendHealth(store, exchange, path);
149
158
  if (req.method !== 'POST') {
150
159
  return sendError(exchange, false, 'AccessDeniedException', 'Unable to determine service/operation name to be authorized');
151
160
  }
@@ -23,7 +23,7 @@ export declare class Subscription {
23
23
  private readonly onRecords;
24
24
  private readonly onConsumerGone;
25
25
  private readonly onStreamDeleted;
26
- private readonly onClose;
26
+ private readonly onDrain;
27
27
  constructor(context: Context, consumerArn: string, streamName: string, shardIx: number, shardId: string, position: string);
28
28
  get protocol(): Protocol;
29
29
  get payloadType(): "application/json" | "application/x-amz-cbor-1.1";
@@ -44,7 +44,7 @@ export class Subscription {
44
44
  onRecords;
45
45
  onConsumerGone;
46
46
  onStreamDeleted;
47
- onClose;
47
+ onDrain;
48
48
  constructor(context, consumerArn, streamName, shardIx, shardId, position) {
49
49
  this.context = context;
50
50
  this.key = `${consumerArn}|${shardId}`;
@@ -66,7 +66,7 @@ export class Subscription {
66
66
  if (name === this.streamName)
67
67
  this.fail(resourceNotFound(`Stream ${name} under account ${account(this.context)} not found.`));
68
68
  };
69
- this.onClose = () => this.stop();
69
+ this.onDrain = () => this.stop();
70
70
  }
71
71
  get protocol() {
72
72
  return this.context.protocol;
@@ -81,9 +81,11 @@ export class Subscription {
81
81
  events.on('consumer-deleting', this.onConsumerGone);
82
82
  events.on('consumer-removed', this.onConsumerGone);
83
83
  events.on('stream-deleted', this.onStreamDeleted);
84
- events.on('close', this.onClose);
84
+ events.on('drain', this.onDrain);
85
85
  sink.onClose(() => this.stop());
86
86
  sink.write(encodeEvent({ ':event-type': 'initial-response', ':content-type': this.payloadType, ':message-type': 'event' }, encodeBody({}, this.protocol)));
87
+ if (this.context.store.draining)
88
+ return this.stop();
87
89
  this.deadline = setTimeout(() => this.stop(), SUBSCRIPTION_MS);
88
90
  this.schedule(0);
89
91
  }
@@ -153,7 +155,7 @@ export class Subscription {
153
155
  events.off('consumer-deleting', this.onConsumerGone);
154
156
  events.off('consumer-removed', this.onConsumerGone);
155
157
  events.off('stream-deleted', this.onStreamDeleted);
156
- events.off('close', this.onClose);
158
+ events.off('drain', this.onDrain);
157
159
  const { active } = stateOf(this.context.store);
158
160
  if (active.get(this.key) === this)
159
161
  active.delete(this.key);
package/dist/server.d.ts CHANGED
@@ -9,9 +9,11 @@ export interface StartOptions {
9
9
  createStreamMs: number;
10
10
  deleteStreamMs: number;
11
11
  updateStreamMs: number;
12
+ expiryIntervalMs: number;
12
13
  shardLimit: number;
13
14
  accountId: string;
14
15
  region: string;
16
+ shutdownMs: number;
15
17
  }
16
18
  export interface KinesisLocal {
17
19
  endpoint: string;
package/dist/server.js CHANGED
@@ -14,9 +14,11 @@ export function defaultOptions() {
14
14
  createStreamMs: 500,
15
15
  deleteStreamMs: 500,
16
16
  updateStreamMs: 500,
17
+ expiryIntervalMs: 60000,
17
18
  shardLimit: 10,
18
19
  accountId: (process.env.AWS_ACCOUNT_ID || '0000-0000-0000').replace(/[^\d]/g, ''),
19
20
  region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
21
+ shutdownMs: 5000,
20
22
  };
21
23
  }
22
24
  function routeByProtocol(socket, http1, http2) {
@@ -49,19 +51,42 @@ export async function start(overrides = {}) {
49
51
  const options = { ...defaultOptions(), ...overrides };
50
52
  const store = new Store(options);
51
53
  registerTransitions(store);
54
+ store.startExpiry();
52
55
  store.resume();
53
- const handler = createHandler(store);
56
+ const handle = createHandler(store);
54
57
  const sockets = new Set();
58
+ const sessions = new Set();
59
+ let inFlight = 0;
60
+ let onIdle = () => { };
61
+ const handler = (req, res) => {
62
+ const response = res;
63
+ inFlight++;
64
+ response.once('close', () => {
65
+ inFlight--;
66
+ if (inFlight === 0)
67
+ onIdle();
68
+ });
69
+ if (store.draining && req.httpVersionMajor < 2)
70
+ response.setHeader('Connection', 'close');
71
+ handle(req, res);
72
+ };
73
+ const trackSession = (session) => {
74
+ sessions.add(session);
75
+ session.once('close', () => sessions.delete(session));
76
+ };
55
77
  let server;
56
78
  if (options.ssl) {
57
79
  const credentials = options.keyFile && options.certFile
58
80
  ? { key: readFileSync(options.keyFile), cert: readFileSync(options.certFile) }
59
81
  : selfSignedCertificate();
60
- server = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
82
+ const secure = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
83
+ secure.on('session', trackSession);
84
+ server = secure;
61
85
  }
62
86
  else {
63
87
  const http1 = createHttpServer(handler);
64
88
  const http2 = createHttp2Server(handler);
89
+ http2.on('session', trackSession);
65
90
  server = createNetServer(socket => {
66
91
  socket.on('error', () => socket.destroy());
67
92
  routeByProtocol(socket, http1, http2);
@@ -86,15 +111,30 @@ export async function start(overrides = {}) {
86
111
  }
87
112
  const address = server.address();
88
113
  const protocol = options.ssl ? 'https' : 'http';
114
+ const shutdown = async () => {
115
+ store.drain();
116
+ const stopped = new Promise(resolve => server.close(() => resolve()));
117
+ for (const session of sessions)
118
+ session.close();
119
+ await new Promise(resolve => {
120
+ const timer = setTimeout(resolve, options.shutdownMs);
121
+ onIdle = () => {
122
+ clearTimeout(timer);
123
+ resolve();
124
+ };
125
+ if (inFlight === 0)
126
+ onIdle();
127
+ });
128
+ store.close();
129
+ for (const socket of sockets)
130
+ socket.destroy();
131
+ await stopped;
132
+ };
133
+ let closing;
89
134
  return {
90
135
  endpoint: `${protocol}://${hostname(address)}:${address.port}`,
91
136
  address,
92
137
  protocol,
93
- close: () => new Promise(resolve => {
94
- store.close();
95
- for (const socket of sockets)
96
- socket.destroy();
97
- server.close(() => resolve());
98
- }),
138
+ close: () => closing ??= shutdown(),
99
139
  };
100
140
  }
@@ -6,6 +6,7 @@ export interface StoreOptions {
6
6
  createStreamMs: number;
7
7
  deleteStreamMs: number;
8
8
  updateStreamMs: number;
9
+ expiryIntervalMs: number;
9
10
  shardLimit: number;
10
11
  accountId: string;
11
12
  region: string;
@@ -20,6 +21,7 @@ export declare class Store {
20
21
  readonly policies: Map<string, string>;
21
22
  readonly events: EventEmitter<any>;
22
23
  settings: AccountSettings;
24
+ draining: boolean;
23
25
  private readonly db;
24
26
  private readonly transitions;
25
27
  private readonly timers;
@@ -49,6 +51,11 @@ export declare class Store {
49
51
  } | undefined;
50
52
  deleteRecords(stream: string, shardIx: number, sequenceNumbers: string[]): void;
51
53
  deleteStreamRecords(stream: string): void;
54
+ startExpiry(): void;
55
+ private scheduleExpiry;
56
+ private expireBatch;
57
+ private expiryBatches;
58
+ private expireOldest;
52
59
  registerTransition(action: string, transition: Transition): void;
53
60
  schedule(kind: Kind, key: string, action: string, delayMs: number, data?: unknown): void;
54
61
  private persist;
@@ -56,6 +63,8 @@ export declare class Store {
56
63
  private fire;
57
64
  cancel(kind: Kind, key: string): void;
58
65
  resume(): void;
66
+ healthy(): boolean;
67
+ drain(): void;
59
68
  close(): void;
60
69
  }
61
70
  export {};
@@ -2,7 +2,9 @@ import { EventEmitter } from 'node:events';
2
2
  import { mkdirSync } from 'node:fs';
3
3
  import { join } from 'node:path';
4
4
  import { DatabaseSync } from 'node:sqlite';
5
+ import { parseSequence } from '../kinesis/sequence.js';
5
6
  export const DATABASE_FILE = 'aws-kinesis-local.sqlite';
7
+ const EXPIRY_BATCH_SIZE = 500;
6
8
  export class Store {
7
9
  options;
8
10
  streams = new Map();
@@ -11,6 +13,7 @@ export class Store {
11
13
  policies = new Map();
12
14
  events = new EventEmitter();
13
15
  settings = { MinimumThroughputBillingCommitment: { Status: 'DISABLED' } };
16
+ draining = false;
14
17
  db;
15
18
  transitions = new Map();
16
19
  timers = new Map();
@@ -52,8 +55,10 @@ export class Store {
52
55
  readRecords: this.db.prepare('SELECT seq, partition_key, data, arrival, encryption FROM records WHERE stream = ? AND shard = ? AND seq >= ? ORDER BY seq LIMIT ?'),
53
56
  firstAtOrAfter: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? AND arrival >= ? ORDER BY seq LIMIT 1'),
54
57
  lastRecord: this.db.prepare('SELECT seq, arrival FROM records WHERE stream = ? AND shard = ? ORDER BY seq DESC LIMIT 1'),
58
+ oldestSequences: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? ORDER BY seq LIMIT ?'),
55
59
  deleteRecord: this.db.prepare('DELETE FROM records WHERE stream = ? AND shard = ? AND seq = ?'),
56
60
  deleteStreamRecords: this.db.prepare('DELETE FROM records WHERE stream = ?'),
61
+ schemaVersion: this.db.prepare("SELECT value FROM metadata WHERE key = 'schema'"),
57
62
  };
58
63
  this.load();
59
64
  }
@@ -173,6 +178,47 @@ export class Store {
173
178
  return;
174
179
  this.statements.deleteStreamRecords.run(stream);
175
180
  }
181
+ startExpiry() {
182
+ this.scheduleExpiry(0, this.expiryBatches());
183
+ }
184
+ scheduleExpiry(delayMs, batches) {
185
+ this.timers.set('expiry', setTimeout(() => this.expireBatch(batches), delayMs));
186
+ }
187
+ expireBatch(batches) {
188
+ let done = true;
189
+ try {
190
+ done = batches.next().done;
191
+ }
192
+ catch (error) {
193
+ console.error(error instanceof Error ? error.stack : error);
194
+ }
195
+ if (done)
196
+ this.scheduleExpiry(this.options.expiryIntervalMs, this.expiryBatches());
197
+ else
198
+ this.scheduleExpiry(0, batches);
199
+ }
200
+ *expiryBatches() {
201
+ for (const stream of [...this.streams.values()]) {
202
+ for (let shardIx = 0; shardIx < stream.Shards.length; shardIx++) {
203
+ while (this.expireOldest(stream, shardIx) === EXPIRY_BATCH_SIZE)
204
+ yield;
205
+ yield;
206
+ }
207
+ }
208
+ }
209
+ expireOldest(stream, shardIx) {
210
+ const cutoff = Date.now() - stream.RetentionPeriodHours * 60 * 60 * 1000;
211
+ const oldest = this.statements.oldestSequences.all(stream.StreamName, shardIx, EXPIRY_BATCH_SIZE);
212
+ const expired = [];
213
+ for (const { seq } of oldest) {
214
+ if (parseSequence(seq).seqTime >= cutoff)
215
+ break;
216
+ expired.push(seq);
217
+ }
218
+ if (expired.length)
219
+ this.transaction(() => this.deleteRecords(stream.StreamName, shardIx, expired));
220
+ return expired.length;
221
+ }
176
222
  registerTransition(action, transition) {
177
223
  this.transitions.set(action, transition);
178
224
  }
@@ -233,13 +279,30 @@ export class Store {
233
279
  }
234
280
  }
235
281
  }
282
+ healthy() {
283
+ if (this.closed)
284
+ return false;
285
+ try {
286
+ this.statements.schemaVersion.get();
287
+ return true;
288
+ }
289
+ catch {
290
+ return false;
291
+ }
292
+ }
293
+ drain() {
294
+ if (this.draining)
295
+ return;
296
+ this.draining = true;
297
+ this.events.emit('drain');
298
+ }
236
299
  close() {
237
300
  if (this.closed)
238
301
  return;
302
+ this.drain();
239
303
  for (const timer of this.timers.values())
240
304
  clearTimeout(timer);
241
305
  this.timers.clear();
242
- this.events.emit('close');
243
306
  this.closed = true;
244
307
  this.db.close();
245
308
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@saidsef/aws-kinesis-local",
3
- "version": "1.0.2",
3
+ "version": "1.1.0",
4
4
  "description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
5
5
  "keywords": [
6
6
  "aws",