@saidsef/aws-kinesis-local 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handler.js +9 -0
- package/dist/operations/subscribe.d.ts +1 -1
- package/dist/operations/subscribe.js +6 -4
- package/dist/server.d.ts +2 -0
- package/dist/server.js +48 -8
- package/dist/store/store.d.ts +9 -0
- package/dist/store/store.js +64 -1
- package/package.json +1 -1
package/dist/handler.js
CHANGED
|
@@ -9,6 +9,7 @@ import { AMZ_CBOR, AMZ_JSON, encodeBody, serializeShape } from './protocol/seria
|
|
|
9
9
|
import { coerceStructure, decodeBlobs, validateStructure } from './validation/validate.js';
|
|
10
10
|
const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
|
|
11
11
|
const EVENT_STREAM = 'application/vnd.amazon.eventstream';
|
|
12
|
+
const HEALTH_PATHS = ['/healthz', '/readyz'];
|
|
12
13
|
const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
|
|
13
14
|
function header(req, name) {
|
|
14
15
|
const value = req.headers[name];
|
|
@@ -37,6 +38,11 @@ function sendError(exchange, contentValid, type, message) {
|
|
|
37
38
|
return sendRaw(exchange, `<${type}/>\n`, message);
|
|
38
39
|
return sendRaw(exchange, `<${type}>\n <Message>${message}</Message>\n</${type}>\n`, 403);
|
|
39
40
|
}
|
|
41
|
+
function sendHealth(store, exchange, path) {
|
|
42
|
+
const healthy = store.healthy() && (path === '/healthz' || !store.draining);
|
|
43
|
+
exchange.res.setHeader('Content-Type', 'text/plain');
|
|
44
|
+
sendRaw(exchange, healthy ? 'ok\n' : 'unavailable\n', healthy ? 200 : 503);
|
|
45
|
+
}
|
|
40
46
|
function validationContext(store, data) {
|
|
41
47
|
let name = typeof data.StreamName === 'string' ? data.StreamName : undefined;
|
|
42
48
|
if (name === undefined && typeof data.StreamARN === 'string')
|
|
@@ -146,6 +152,9 @@ function respond(store, exchange, body) {
|
|
|
146
152
|
}
|
|
147
153
|
res.setHeader('Access-Control-Expose-Headers', EXPOSED_HEADERS);
|
|
148
154
|
}
|
|
155
|
+
const path = req.url?.split('?')[0] ?? '';
|
|
156
|
+
if (req.method === 'GET' && HEALTH_PATHS.includes(path))
|
|
157
|
+
return sendHealth(store, exchange, path);
|
|
149
158
|
if (req.method !== 'POST') {
|
|
150
159
|
return sendError(exchange, false, 'AccessDeniedException', 'Unable to determine service/operation name to be authorized');
|
|
151
160
|
}
|
|
@@ -23,7 +23,7 @@ export declare class Subscription {
|
|
|
23
23
|
private readonly onRecords;
|
|
24
24
|
private readonly onConsumerGone;
|
|
25
25
|
private readonly onStreamDeleted;
|
|
26
|
-
private readonly
|
|
26
|
+
private readonly onDrain;
|
|
27
27
|
constructor(context: Context, consumerArn: string, streamName: string, shardIx: number, shardId: string, position: string);
|
|
28
28
|
get protocol(): Protocol;
|
|
29
29
|
get payloadType(): "application/json" | "application/x-amz-cbor-1.1";
|
|
@@ -44,7 +44,7 @@ export class Subscription {
|
|
|
44
44
|
onRecords;
|
|
45
45
|
onConsumerGone;
|
|
46
46
|
onStreamDeleted;
|
|
47
|
-
|
|
47
|
+
onDrain;
|
|
48
48
|
constructor(context, consumerArn, streamName, shardIx, shardId, position) {
|
|
49
49
|
this.context = context;
|
|
50
50
|
this.key = `${consumerArn}|${shardId}`;
|
|
@@ -66,7 +66,7 @@ export class Subscription {
|
|
|
66
66
|
if (name === this.streamName)
|
|
67
67
|
this.fail(resourceNotFound(`Stream ${name} under account ${account(this.context)} not found.`));
|
|
68
68
|
};
|
|
69
|
-
this.
|
|
69
|
+
this.onDrain = () => this.stop();
|
|
70
70
|
}
|
|
71
71
|
get protocol() {
|
|
72
72
|
return this.context.protocol;
|
|
@@ -81,9 +81,11 @@ export class Subscription {
|
|
|
81
81
|
events.on('consumer-deleting', this.onConsumerGone);
|
|
82
82
|
events.on('consumer-removed', this.onConsumerGone);
|
|
83
83
|
events.on('stream-deleted', this.onStreamDeleted);
|
|
84
|
-
events.on('
|
|
84
|
+
events.on('drain', this.onDrain);
|
|
85
85
|
sink.onClose(() => this.stop());
|
|
86
86
|
sink.write(encodeEvent({ ':event-type': 'initial-response', ':content-type': this.payloadType, ':message-type': 'event' }, encodeBody({}, this.protocol)));
|
|
87
|
+
if (this.context.store.draining)
|
|
88
|
+
return this.stop();
|
|
87
89
|
this.deadline = setTimeout(() => this.stop(), SUBSCRIPTION_MS);
|
|
88
90
|
this.schedule(0);
|
|
89
91
|
}
|
|
@@ -153,7 +155,7 @@ export class Subscription {
|
|
|
153
155
|
events.off('consumer-deleting', this.onConsumerGone);
|
|
154
156
|
events.off('consumer-removed', this.onConsumerGone);
|
|
155
157
|
events.off('stream-deleted', this.onStreamDeleted);
|
|
156
|
-
events.off('
|
|
158
|
+
events.off('drain', this.onDrain);
|
|
157
159
|
const { active } = stateOf(this.context.store);
|
|
158
160
|
if (active.get(this.key) === this)
|
|
159
161
|
active.delete(this.key);
|
package/dist/server.d.ts
CHANGED
|
@@ -9,9 +9,11 @@ export interface StartOptions {
|
|
|
9
9
|
createStreamMs: number;
|
|
10
10
|
deleteStreamMs: number;
|
|
11
11
|
updateStreamMs: number;
|
|
12
|
+
expiryIntervalMs: number;
|
|
12
13
|
shardLimit: number;
|
|
13
14
|
accountId: string;
|
|
14
15
|
region: string;
|
|
16
|
+
shutdownMs: number;
|
|
15
17
|
}
|
|
16
18
|
export interface KinesisLocal {
|
|
17
19
|
endpoint: string;
|
package/dist/server.js
CHANGED
|
@@ -14,9 +14,11 @@ export function defaultOptions() {
|
|
|
14
14
|
createStreamMs: 500,
|
|
15
15
|
deleteStreamMs: 500,
|
|
16
16
|
updateStreamMs: 500,
|
|
17
|
+
expiryIntervalMs: 60000,
|
|
17
18
|
shardLimit: 10,
|
|
18
19
|
accountId: (process.env.AWS_ACCOUNT_ID || '0000-0000-0000').replace(/[^\d]/g, ''),
|
|
19
20
|
region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
|
|
21
|
+
shutdownMs: 5000,
|
|
20
22
|
};
|
|
21
23
|
}
|
|
22
24
|
function routeByProtocol(socket, http1, http2) {
|
|
@@ -49,19 +51,42 @@ export async function start(overrides = {}) {
|
|
|
49
51
|
const options = { ...defaultOptions(), ...overrides };
|
|
50
52
|
const store = new Store(options);
|
|
51
53
|
registerTransitions(store);
|
|
54
|
+
store.startExpiry();
|
|
52
55
|
store.resume();
|
|
53
|
-
const
|
|
56
|
+
const handle = createHandler(store);
|
|
54
57
|
const sockets = new Set();
|
|
58
|
+
const sessions = new Set();
|
|
59
|
+
let inFlight = 0;
|
|
60
|
+
let onIdle = () => { };
|
|
61
|
+
const handler = (req, res) => {
|
|
62
|
+
const response = res;
|
|
63
|
+
inFlight++;
|
|
64
|
+
response.once('close', () => {
|
|
65
|
+
inFlight--;
|
|
66
|
+
if (inFlight === 0)
|
|
67
|
+
onIdle();
|
|
68
|
+
});
|
|
69
|
+
if (store.draining && req.httpVersionMajor < 2)
|
|
70
|
+
response.setHeader('Connection', 'close');
|
|
71
|
+
handle(req, res);
|
|
72
|
+
};
|
|
73
|
+
const trackSession = (session) => {
|
|
74
|
+
sessions.add(session);
|
|
75
|
+
session.once('close', () => sessions.delete(session));
|
|
76
|
+
};
|
|
55
77
|
let server;
|
|
56
78
|
if (options.ssl) {
|
|
57
79
|
const credentials = options.keyFile && options.certFile
|
|
58
80
|
? { key: readFileSync(options.keyFile), cert: readFileSync(options.certFile) }
|
|
59
81
|
: selfSignedCertificate();
|
|
60
|
-
|
|
82
|
+
const secure = createSecureServer({ ...credentials, allowHTTP1: true }, handler);
|
|
83
|
+
secure.on('session', trackSession);
|
|
84
|
+
server = secure;
|
|
61
85
|
}
|
|
62
86
|
else {
|
|
63
87
|
const http1 = createHttpServer(handler);
|
|
64
88
|
const http2 = createHttp2Server(handler);
|
|
89
|
+
http2.on('session', trackSession);
|
|
65
90
|
server = createNetServer(socket => {
|
|
66
91
|
socket.on('error', () => socket.destroy());
|
|
67
92
|
routeByProtocol(socket, http1, http2);
|
|
@@ -86,15 +111,30 @@ export async function start(overrides = {}) {
|
|
|
86
111
|
}
|
|
87
112
|
const address = server.address();
|
|
88
113
|
const protocol = options.ssl ? 'https' : 'http';
|
|
114
|
+
const shutdown = async () => {
|
|
115
|
+
store.drain();
|
|
116
|
+
const stopped = new Promise(resolve => server.close(() => resolve()));
|
|
117
|
+
for (const session of sessions)
|
|
118
|
+
session.close();
|
|
119
|
+
await new Promise(resolve => {
|
|
120
|
+
const timer = setTimeout(resolve, options.shutdownMs);
|
|
121
|
+
onIdle = () => {
|
|
122
|
+
clearTimeout(timer);
|
|
123
|
+
resolve();
|
|
124
|
+
};
|
|
125
|
+
if (inFlight === 0)
|
|
126
|
+
onIdle();
|
|
127
|
+
});
|
|
128
|
+
store.close();
|
|
129
|
+
for (const socket of sockets)
|
|
130
|
+
socket.destroy();
|
|
131
|
+
await stopped;
|
|
132
|
+
};
|
|
133
|
+
let closing;
|
|
89
134
|
return {
|
|
90
135
|
endpoint: `${protocol}://${hostname(address)}:${address.port}`,
|
|
91
136
|
address,
|
|
92
137
|
protocol,
|
|
93
|
-
close: () =>
|
|
94
|
-
store.close();
|
|
95
|
-
for (const socket of sockets)
|
|
96
|
-
socket.destroy();
|
|
97
|
-
server.close(() => resolve());
|
|
98
|
-
}),
|
|
138
|
+
close: () => closing ??= shutdown(),
|
|
99
139
|
};
|
|
100
140
|
}
|
package/dist/store/store.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export interface StoreOptions {
|
|
|
6
6
|
createStreamMs: number;
|
|
7
7
|
deleteStreamMs: number;
|
|
8
8
|
updateStreamMs: number;
|
|
9
|
+
expiryIntervalMs: number;
|
|
9
10
|
shardLimit: number;
|
|
10
11
|
accountId: string;
|
|
11
12
|
region: string;
|
|
@@ -20,6 +21,7 @@ export declare class Store {
|
|
|
20
21
|
readonly policies: Map<string, string>;
|
|
21
22
|
readonly events: EventEmitter<any>;
|
|
22
23
|
settings: AccountSettings;
|
|
24
|
+
draining: boolean;
|
|
23
25
|
private readonly db;
|
|
24
26
|
private readonly transitions;
|
|
25
27
|
private readonly timers;
|
|
@@ -49,6 +51,11 @@ export declare class Store {
|
|
|
49
51
|
} | undefined;
|
|
50
52
|
deleteRecords(stream: string, shardIx: number, sequenceNumbers: string[]): void;
|
|
51
53
|
deleteStreamRecords(stream: string): void;
|
|
54
|
+
startExpiry(): void;
|
|
55
|
+
private scheduleExpiry;
|
|
56
|
+
private expireBatch;
|
|
57
|
+
private expiryBatches;
|
|
58
|
+
private expireOldest;
|
|
52
59
|
registerTransition(action: string, transition: Transition): void;
|
|
53
60
|
schedule(kind: Kind, key: string, action: string, delayMs: number, data?: unknown): void;
|
|
54
61
|
private persist;
|
|
@@ -56,6 +63,8 @@ export declare class Store {
|
|
|
56
63
|
private fire;
|
|
57
64
|
cancel(kind: Kind, key: string): void;
|
|
58
65
|
resume(): void;
|
|
66
|
+
healthy(): boolean;
|
|
67
|
+
drain(): void;
|
|
59
68
|
close(): void;
|
|
60
69
|
}
|
|
61
70
|
export {};
|
package/dist/store/store.js
CHANGED
|
@@ -2,7 +2,9 @@ import { EventEmitter } from 'node:events';
|
|
|
2
2
|
import { mkdirSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
+
import { parseSequence } from '../kinesis/sequence.js';
|
|
5
6
|
export const DATABASE_FILE = 'aws-kinesis-local.sqlite';
|
|
7
|
+
const EXPIRY_BATCH_SIZE = 500;
|
|
6
8
|
export class Store {
|
|
7
9
|
options;
|
|
8
10
|
streams = new Map();
|
|
@@ -11,6 +13,7 @@ export class Store {
|
|
|
11
13
|
policies = new Map();
|
|
12
14
|
events = new EventEmitter();
|
|
13
15
|
settings = { MinimumThroughputBillingCommitment: { Status: 'DISABLED' } };
|
|
16
|
+
draining = false;
|
|
14
17
|
db;
|
|
15
18
|
transitions = new Map();
|
|
16
19
|
timers = new Map();
|
|
@@ -52,8 +55,10 @@ export class Store {
|
|
|
52
55
|
readRecords: this.db.prepare('SELECT seq, partition_key, data, arrival, encryption FROM records WHERE stream = ? AND shard = ? AND seq >= ? ORDER BY seq LIMIT ?'),
|
|
53
56
|
firstAtOrAfter: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? AND arrival >= ? ORDER BY seq LIMIT 1'),
|
|
54
57
|
lastRecord: this.db.prepare('SELECT seq, arrival FROM records WHERE stream = ? AND shard = ? ORDER BY seq DESC LIMIT 1'),
|
|
58
|
+
oldestSequences: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? ORDER BY seq LIMIT ?'),
|
|
55
59
|
deleteRecord: this.db.prepare('DELETE FROM records WHERE stream = ? AND shard = ? AND seq = ?'),
|
|
56
60
|
deleteStreamRecords: this.db.prepare('DELETE FROM records WHERE stream = ?'),
|
|
61
|
+
schemaVersion: this.db.prepare("SELECT value FROM metadata WHERE key = 'schema'"),
|
|
57
62
|
};
|
|
58
63
|
this.load();
|
|
59
64
|
}
|
|
@@ -173,6 +178,47 @@ export class Store {
|
|
|
173
178
|
return;
|
|
174
179
|
this.statements.deleteStreamRecords.run(stream);
|
|
175
180
|
}
|
|
181
|
+
startExpiry() {
|
|
182
|
+
this.scheduleExpiry(0, this.expiryBatches());
|
|
183
|
+
}
|
|
184
|
+
scheduleExpiry(delayMs, batches) {
|
|
185
|
+
this.timers.set('expiry', setTimeout(() => this.expireBatch(batches), delayMs));
|
|
186
|
+
}
|
|
187
|
+
expireBatch(batches) {
|
|
188
|
+
let done = true;
|
|
189
|
+
try {
|
|
190
|
+
done = batches.next().done;
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
console.error(error instanceof Error ? error.stack : error);
|
|
194
|
+
}
|
|
195
|
+
if (done)
|
|
196
|
+
this.scheduleExpiry(this.options.expiryIntervalMs, this.expiryBatches());
|
|
197
|
+
else
|
|
198
|
+
this.scheduleExpiry(0, batches);
|
|
199
|
+
}
|
|
200
|
+
*expiryBatches() {
|
|
201
|
+
for (const stream of [...this.streams.values()]) {
|
|
202
|
+
for (let shardIx = 0; shardIx < stream.Shards.length; shardIx++) {
|
|
203
|
+
while (this.expireOldest(stream, shardIx) === EXPIRY_BATCH_SIZE)
|
|
204
|
+
yield;
|
|
205
|
+
yield;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
expireOldest(stream, shardIx) {
|
|
210
|
+
const cutoff = Date.now() - stream.RetentionPeriodHours * 60 * 60 * 1000;
|
|
211
|
+
const oldest = this.statements.oldestSequences.all(stream.StreamName, shardIx, EXPIRY_BATCH_SIZE);
|
|
212
|
+
const expired = [];
|
|
213
|
+
for (const { seq } of oldest) {
|
|
214
|
+
if (parseSequence(seq).seqTime >= cutoff)
|
|
215
|
+
break;
|
|
216
|
+
expired.push(seq);
|
|
217
|
+
}
|
|
218
|
+
if (expired.length)
|
|
219
|
+
this.transaction(() => this.deleteRecords(stream.StreamName, shardIx, expired));
|
|
220
|
+
return expired.length;
|
|
221
|
+
}
|
|
176
222
|
registerTransition(action, transition) {
|
|
177
223
|
this.transitions.set(action, transition);
|
|
178
224
|
}
|
|
@@ -233,13 +279,30 @@ export class Store {
|
|
|
233
279
|
}
|
|
234
280
|
}
|
|
235
281
|
}
|
|
282
|
+
healthy() {
|
|
283
|
+
if (this.closed)
|
|
284
|
+
return false;
|
|
285
|
+
try {
|
|
286
|
+
this.statements.schemaVersion.get();
|
|
287
|
+
return true;
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
drain() {
|
|
294
|
+
if (this.draining)
|
|
295
|
+
return;
|
|
296
|
+
this.draining = true;
|
|
297
|
+
this.events.emit('drain');
|
|
298
|
+
}
|
|
236
299
|
close() {
|
|
237
300
|
if (this.closed)
|
|
238
301
|
return;
|
|
302
|
+
this.drain();
|
|
239
303
|
for (const timer of this.timers.values())
|
|
240
304
|
clearTimeout(timer);
|
|
241
305
|
this.timers.clear();
|
|
242
|
-
this.events.emit('close');
|
|
243
306
|
this.closed = true;
|
|
244
307
|
this.db.close();
|
|
245
308
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@saidsef/aws-kinesis-local",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"aws",
|