@saidsef/aws-kinesis-local 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +8 -0
- package/dist/handler.d.ts +2 -1
- package/dist/handler.js +90 -6
- package/dist/logger.d.ts +10 -0
- package/dist/logger.js +27 -0
- package/dist/operations/subscribe.d.ts +1 -0
- package/dist/operations/subscribe.js +8 -1
- package/dist/server.d.ts +3 -0
- package/dist/server.js +6 -2
- package/dist/store/store.d.ts +4 -1
- package/dist/store/store.js +30 -3
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { numberOption, parseArguments, stringOption } from './arguments.js';
|
|
3
|
+
import { isLogLevel, LOG_LEVELS } from './logger.js';
|
|
3
4
|
import { defaultOptions, start } from './server.js';
|
|
4
5
|
const HELP = [
|
|
5
6
|
'',
|
|
@@ -18,6 +19,7 @@ const HELP = [
|
|
|
18
19
|
'--deleteStreamMs <ms> Amount of time streams stay in DELETING state (default: 500)',
|
|
19
20
|
'--updateStreamMs <ms> Amount of time streams stay in UPDATING state (default: 500)',
|
|
20
21
|
'--shardLimit <limit> Shard limit for error reporting (default: 10)',
|
|
22
|
+
'--logLevel <level> Least severe level to log: error, warn, info, debug or trace (default: warn)',
|
|
21
23
|
'',
|
|
22
24
|
'Report bugs at github.com/saidsef/aws-kinesis-local/issues',
|
|
23
25
|
].join('\n');
|
|
@@ -27,6 +29,11 @@ if (args.help) {
|
|
|
27
29
|
}
|
|
28
30
|
else {
|
|
29
31
|
const defaults = defaultOptions();
|
|
32
|
+
const logLevel = stringOption(args.logLevel) ?? defaults.logLevel;
|
|
33
|
+
if (!isLogLevel(logLevel)) {
|
|
34
|
+
console.error(`Unknown log level ${logLevel}. Use ${LOG_LEVELS.join(', ')}.`);
|
|
35
|
+
process.exit(1);
|
|
36
|
+
}
|
|
30
37
|
const server = await start({
|
|
31
38
|
port: numberOption(args.port, defaults.port) || defaults.port,
|
|
32
39
|
path: stringOption(args.path),
|
|
@@ -37,6 +44,7 @@ else {
|
|
|
37
44
|
deleteStreamMs: numberOption(args.deleteStreamMs, defaults.deleteStreamMs),
|
|
38
45
|
updateStreamMs: numberOption(args.updateStreamMs, defaults.updateStreamMs),
|
|
39
46
|
shardLimit: numberOption(args.shardLimit, defaults.shardLimit),
|
|
47
|
+
logLevel,
|
|
40
48
|
});
|
|
41
49
|
const { address, port } = server.address;
|
|
42
50
|
console.log('Listening at %s://%s:%s', server.protocol, address, port);
|
package/dist/handler.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
2
2
|
import type { Http2ServerRequest, Http2ServerResponse } from 'node:http2';
|
|
3
|
+
import type { Logger } from './logger.ts';
|
|
3
4
|
import type { Store } from './store/store.ts';
|
|
4
5
|
type Request = IncomingMessage | Http2ServerRequest;
|
|
5
6
|
type Response = ServerResponse | Http2ServerResponse;
|
|
6
|
-
export declare function createHandler(store: Store): (req: Request, res: Response) => void;
|
|
7
|
+
export declare function createHandler(store: Store, logger: Logger): (req: Request, res: Response) => void;
|
|
7
8
|
export {};
|
package/dist/handler.js
CHANGED
|
@@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { KinesisError } from './errors.js';
|
|
3
3
|
import { operations, serviceTarget } from './generated/model.js';
|
|
4
4
|
import { handlers } from './operations/index.js';
|
|
5
|
-
import { DEFAULT_MAX_RECORD_SIZE_KIB, streamNameFromArn } from './operations/common.js';
|
|
5
|
+
import { DEFAULT_MAX_RECORD_SIZE_KIB, parseConsumerArn, streamNameFromArn } from './operations/common.js';
|
|
6
6
|
import { Subscription } from './operations/subscribe.js';
|
|
7
7
|
import { decodeCbor } from './protocol/cbor.js';
|
|
8
8
|
import { AMZ_CBOR, AMZ_JSON, encodeBody, serializeShape } from './protocol/serialize.js';
|
|
@@ -10,6 +10,7 @@ import { coerceStructure, decodeBlobs, validateStructure } from './validation/va
|
|
|
10
10
|
const MAX_REQUEST_BYTES = 16 * 1024 * 1024;
|
|
11
11
|
const EVENT_STREAM = 'application/vnd.amazon.eventstream';
|
|
12
12
|
const HEALTH_PATHS = ['/healthz', '/readyz'];
|
|
13
|
+
const RESET_PATH = '/_control/reset';
|
|
13
14
|
const EXPOSED_HEADERS = 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date';
|
|
14
15
|
function header(req, name) {
|
|
15
16
|
const value = req.headers[name];
|
|
@@ -19,6 +20,8 @@ function sendRaw(exchange, body, statusCode = 200) {
|
|
|
19
20
|
const { req, res } = exchange;
|
|
20
21
|
req.removeAllListeners('data');
|
|
21
22
|
res.statusCode = statusCode;
|
|
23
|
+
if (typeof body === 'string')
|
|
24
|
+
exchange.output = body;
|
|
22
25
|
if (body === null)
|
|
23
26
|
return res.end();
|
|
24
27
|
res.setHeader('Content-Length', Buffer.byteLength(body));
|
|
@@ -27,22 +30,52 @@ function sendRaw(exchange, body, statusCode = 200) {
|
|
|
27
30
|
function protocolOf(exchange) {
|
|
28
31
|
return exchange.contentType === AMZ_CBOR ? 'cbor' : 'json';
|
|
29
32
|
}
|
|
33
|
+
function noteError(exchange, data) {
|
|
34
|
+
const body = (data?.Output ?? data ?? {});
|
|
35
|
+
exchange.errorType = body.__type;
|
|
36
|
+
exchange.errorMessage = (body.message ?? body.Message);
|
|
37
|
+
}
|
|
30
38
|
function sendResponse(exchange, data, statusCode = 200) {
|
|
39
|
+
exchange.output = data;
|
|
40
|
+
if (statusCode >= 400)
|
|
41
|
+
noteError(exchange, data);
|
|
31
42
|
exchange.res.setHeader('Content-Type', exchange.contentType);
|
|
32
43
|
sendRaw(exchange, encodeBody(data, protocolOf(exchange)), statusCode);
|
|
33
44
|
}
|
|
34
45
|
function sendError(exchange, contentValid, type, message) {
|
|
35
46
|
if (contentValid)
|
|
36
47
|
return sendResponse(exchange, { __type: type, message }, 400);
|
|
48
|
+
exchange.errorType = type;
|
|
37
49
|
if (typeof message === 'number')
|
|
38
50
|
return sendRaw(exchange, `<${type}/>\n`, message);
|
|
51
|
+
exchange.errorMessage = message;
|
|
39
52
|
return sendRaw(exchange, `<${type}>\n <Message>${message}</Message>\n</${type}>\n`, 403);
|
|
40
53
|
}
|
|
41
54
|
function sendHealth(store, exchange, path) {
|
|
42
55
|
const healthy = store.healthy() && (path === '/healthz' || !store.draining);
|
|
56
|
+
exchange.level = 'debug';
|
|
43
57
|
exchange.res.setHeader('Content-Type', 'text/plain');
|
|
44
58
|
sendRaw(exchange, healthy ? 'ok\n' : 'unavailable\n', healthy ? 200 : 503);
|
|
45
59
|
}
|
|
60
|
+
function sendReset(store, exchange, body) {
|
|
61
|
+
let input = {};
|
|
62
|
+
try {
|
|
63
|
+
if (body)
|
|
64
|
+
input = JSON.parse(body.toString('utf8'));
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
input = undefined;
|
|
68
|
+
}
|
|
69
|
+
const stream = input?.stream;
|
|
70
|
+
exchange.contentType = 'application/json';
|
|
71
|
+
exchange.input = input;
|
|
72
|
+
if (typeof stream === 'string')
|
|
73
|
+
exchange.stream = stream;
|
|
74
|
+
if (typeof input !== 'object' || input === null || Array.isArray(input) || (stream !== undefined && typeof stream !== 'string')) {
|
|
75
|
+
return sendResponse(exchange, { message: 'The body must be empty or a JSON object whose stream member is a string.' }, 400);
|
|
76
|
+
}
|
|
77
|
+
sendResponse(exchange, { streams: store.reset(stream) });
|
|
78
|
+
}
|
|
46
79
|
function validationContext(store, data) {
|
|
47
80
|
let name = typeof data.StreamName === 'string' ? data.StreamName : undefined;
|
|
48
81
|
if (name === undefined && typeof data.StreamARN === 'string')
|
|
@@ -53,6 +86,19 @@ function validationContext(store, data) {
|
|
|
53
86
|
maxRecordBytes: (stream?.MaxRecordSizeInKiB ?? DEFAULT_MAX_RECORD_SIZE_KIB) * 1024,
|
|
54
87
|
};
|
|
55
88
|
}
|
|
89
|
+
function streamOf(data) {
|
|
90
|
+
if (typeof data.StreamName === 'string')
|
|
91
|
+
return data.StreamName;
|
|
92
|
+
for (const member of ['StreamARN', 'ConsumerARN', 'ResourceARN']) {
|
|
93
|
+
const arn = data[member];
|
|
94
|
+
if (typeof arn !== 'string')
|
|
95
|
+
continue;
|
|
96
|
+
const name = parseConsumerArn(arn)?.streamName ?? streamNameFromArn(arn);
|
|
97
|
+
if (name !== undefined)
|
|
98
|
+
return name;
|
|
99
|
+
}
|
|
100
|
+
return undefined;
|
|
101
|
+
}
|
|
56
102
|
function checkAuthentication(exchange) {
|
|
57
103
|
const { req } = exchange;
|
|
58
104
|
const authHeader = header(req, 'authorization');
|
|
@@ -111,6 +157,10 @@ function streamEvents(exchange, subscription) {
|
|
|
111
157
|
if (!ended)
|
|
112
158
|
res.write(chunk);
|
|
113
159
|
},
|
|
160
|
+
fail: error => {
|
|
161
|
+
exchange.errorType = error.type;
|
|
162
|
+
exchange.errorMessage = error.message;
|
|
163
|
+
},
|
|
114
164
|
end: () => {
|
|
115
165
|
if (!ended) {
|
|
116
166
|
ended = true;
|
|
@@ -135,7 +185,6 @@ function dispatch(store, exchange, operation, input) {
|
|
|
135
185
|
}
|
|
136
186
|
function respond(store, exchange, body) {
|
|
137
187
|
const { req, res } = exchange;
|
|
138
|
-
res.setHeader('x-amzn-RequestId', randomUUID());
|
|
139
188
|
if (req.method !== 'OPTIONS' || !req.headers.origin)
|
|
140
189
|
res.setHeader('x-amz-id-2', randomBytes(72).toString('base64'));
|
|
141
190
|
if (req.headers.origin) {
|
|
@@ -148,6 +197,7 @@ function respond(store, exchange, body) {
|
|
|
148
197
|
if (requestMethod)
|
|
149
198
|
res.setHeader('Access-Control-Allow-Methods', requestMethod);
|
|
150
199
|
res.setHeader('Access-Control-Max-Age', 172800);
|
|
200
|
+
exchange.level = 'debug';
|
|
151
201
|
return sendRaw(exchange, '');
|
|
152
202
|
}
|
|
153
203
|
res.setHeader('Access-Control-Expose-Headers', EXPOSED_HEADERS);
|
|
@@ -155,12 +205,16 @@ function respond(store, exchange, body) {
|
|
|
155
205
|
const path = req.url?.split('?')[0] ?? '';
|
|
156
206
|
if (req.method === 'GET' && HEALTH_PATHS.includes(path))
|
|
157
207
|
return sendHealth(store, exchange, path);
|
|
208
|
+
if (req.method === 'POST' && path === RESET_PATH)
|
|
209
|
+
return sendReset(store, exchange, body);
|
|
158
210
|
if (req.method !== 'POST') {
|
|
159
211
|
return sendError(exchange, false, 'AccessDeniedException', 'Unable to determine service/operation name to be authorized');
|
|
160
212
|
}
|
|
161
213
|
const contentType = header(req, 'content-type').split(';')[0].trim();
|
|
162
214
|
const contentValid = [AMZ_JSON, AMZ_CBOR, 'application/json'].includes(contentType);
|
|
163
215
|
const [service, operation] = header(req, 'x-amz-target').split('.');
|
|
216
|
+
if (operation)
|
|
217
|
+
exchange.operation = operation;
|
|
164
218
|
const serviceValid = !!service && service === serviceTarget;
|
|
165
219
|
const operationValid = !!operation && Object.hasOwn(handlers, operation);
|
|
166
220
|
if (!body) {
|
|
@@ -187,6 +241,8 @@ function respond(store, exchange, body) {
|
|
|
187
241
|
}
|
|
188
242
|
return sendResponse(exchange, { __type: 'SerializationException' }, 400);
|
|
189
243
|
}
|
|
244
|
+
exchange.input = data;
|
|
245
|
+
exchange.stream = streamOf(data);
|
|
190
246
|
if (contentType === 'application/json') {
|
|
191
247
|
return sendResponse(exchange, { Output: { __type: 'com.amazon.coral.service#UnknownOperationException' }, Version: '1.0' }, 404);
|
|
192
248
|
}
|
|
@@ -201,16 +257,43 @@ function respond(store, exchange, body) {
|
|
|
201
257
|
catch (error) {
|
|
202
258
|
if (error instanceof KinesisError)
|
|
203
259
|
return sendResponse(exchange, error.body, error.statusCode);
|
|
204
|
-
|
|
260
|
+
exchange.error = error;
|
|
205
261
|
return sendResponse(exchange, { __type: 'InternalFailure' }, 500);
|
|
206
262
|
}
|
|
207
263
|
}
|
|
208
|
-
|
|
264
|
+
function logRequest(logger, exchange, started, body) {
|
|
265
|
+
const { req, res } = exchange;
|
|
266
|
+
const status = res.statusCode;
|
|
267
|
+
const level = status >= 500 ? 'error' : status >= 400 || exchange.errorType ? 'warn' : exchange.level;
|
|
268
|
+
if (!logger.enabled(level))
|
|
269
|
+
return;
|
|
270
|
+
const trace = logger.enabled('trace');
|
|
271
|
+
logger.log(level, {
|
|
272
|
+
requestId: exchange.requestId,
|
|
273
|
+
method: req.method,
|
|
274
|
+
path: req.url?.split('?')[0],
|
|
275
|
+
operation: exchange.operation,
|
|
276
|
+
stream: exchange.stream,
|
|
277
|
+
status,
|
|
278
|
+
errorType: exchange.errorType,
|
|
279
|
+
errorMessage: exchange.errorMessage,
|
|
280
|
+
durationMs: Math.round((performance.now() - started) * 1000) / 1000,
|
|
281
|
+
aborted: res.writableEnded ? undefined : true,
|
|
282
|
+
error: exchange.error instanceof Error ? exchange.error.stack : exchange.error,
|
|
283
|
+
request: trace ? exchange.input ?? body?.toString('utf8') : undefined,
|
|
284
|
+
response: trace ? exchange.output : undefined,
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
export function createHandler(store, logger) {
|
|
209
288
|
return (req, res) => {
|
|
210
|
-
const
|
|
289
|
+
const started = performance.now();
|
|
290
|
+
const exchange = { req, res: res, contentType: AMZ_JSON, requestId: randomUUID(), level: 'info' };
|
|
211
291
|
const chunks = [];
|
|
212
292
|
let size = 0;
|
|
213
293
|
let rejected = false;
|
|
294
|
+
let body;
|
|
295
|
+
exchange.res.setHeader('x-amzn-RequestId', exchange.requestId);
|
|
296
|
+
exchange.res.once('close', () => logRequest(logger, exchange, started, body));
|
|
214
297
|
req.on('error', () => res.destroy());
|
|
215
298
|
req.on('data', (chunk) => {
|
|
216
299
|
if (rejected)
|
|
@@ -228,7 +311,8 @@ export function createHandler(store) {
|
|
|
228
311
|
req.on('end', () => {
|
|
229
312
|
if (rejected)
|
|
230
313
|
return;
|
|
231
|
-
|
|
314
|
+
body = size ? Buffer.concat(chunks, size) : undefined;
|
|
315
|
+
respond(store, exchange, body);
|
|
232
316
|
});
|
|
233
317
|
};
|
|
234
318
|
}
|
package/dist/logger.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare const LOG_LEVELS: readonly ['error', 'warn', 'info', 'debug', 'trace'];
|
|
2
|
+
export type LogLevel = typeof LOG_LEVELS[number];
|
|
3
|
+
export declare function isLogLevel(value: unknown): value is LogLevel;
|
|
4
|
+
export declare class Logger {
|
|
5
|
+
readonly level: LogLevel;
|
|
6
|
+
private readonly write;
|
|
7
|
+
constructor(level: LogLevel, write?: (line: string) => void);
|
|
8
|
+
enabled(level: LogLevel): boolean;
|
|
9
|
+
log(level: LogLevel, fields: Record<string, unknown>): void;
|
|
10
|
+
}
|
package/dist/logger.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export const LOG_LEVELS = ['error', 'warn', 'info', 'debug', 'trace'];
|
|
2
|
+
export function isLogLevel(value) {
|
|
3
|
+
return LOG_LEVELS.includes(value);
|
|
4
|
+
}
|
|
5
|
+
function replacer(key, value) {
|
|
6
|
+
const raw = this[key];
|
|
7
|
+
if (raw instanceof Uint8Array)
|
|
8
|
+
return Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength).toString('base64');
|
|
9
|
+
return typeof value === 'bigint' ? value.toString() : value;
|
|
10
|
+
}
|
|
11
|
+
export class Logger {
|
|
12
|
+
level;
|
|
13
|
+
write;
|
|
14
|
+
constructor(level, write = line => process.stdout.write(line)) {
|
|
15
|
+
if (!isLogLevel(level))
|
|
16
|
+
throw new Error(`Unknown log level ${level}. Use ${LOG_LEVELS.join(', ')}.`);
|
|
17
|
+
this.level = level;
|
|
18
|
+
this.write = write;
|
|
19
|
+
}
|
|
20
|
+
enabled(level) {
|
|
21
|
+
return LOG_LEVELS.indexOf(level) <= LOG_LEVELS.indexOf(this.level);
|
|
22
|
+
}
|
|
23
|
+
log(level, fields) {
|
|
24
|
+
if (this.enabled(level))
|
|
25
|
+
this.write(`${JSON.stringify({ time: new Date().toISOString(), level, ...fields }, replacer)}\n`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -14,7 +14,13 @@ const states = new WeakMap();
|
|
|
14
14
|
function stateOf(store) {
|
|
15
15
|
let state = states.get(store);
|
|
16
16
|
if (!state) {
|
|
17
|
-
|
|
17
|
+
const lastSubscribed = new Map();
|
|
18
|
+
store.events.on('consumer-removed', (arn) => {
|
|
19
|
+
for (const key of lastSubscribed.keys())
|
|
20
|
+
if (key.startsWith(`${arn}|`))
|
|
21
|
+
lastSubscribed.delete(key);
|
|
22
|
+
});
|
|
23
|
+
state = { lastSubscribed, active: new Map() };
|
|
18
24
|
states.set(store, state);
|
|
19
25
|
}
|
|
20
26
|
return state;
|
|
@@ -141,6 +147,7 @@ export class Subscription {
|
|
|
141
147
|
fail(error) {
|
|
142
148
|
if (this.finished || !this.sink)
|
|
143
149
|
return;
|
|
150
|
+
this.sink.fail(error);
|
|
144
151
|
this.sink.write(encodeEvent({ ':exception-type': error.type, ':content-type': this.payloadType, ':message-type': 'exception' }, encodeBody({ message: error.message }, this.protocol)));
|
|
145
152
|
this.stop();
|
|
146
153
|
}
|
package/dist/server.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type AddressInfo } from 'node:net';
|
|
2
|
+
import { type LogLevel } from './logger.ts';
|
|
2
3
|
export interface StartOptions {
|
|
3
4
|
port: number;
|
|
4
5
|
host?: string | undefined;
|
|
@@ -14,11 +15,13 @@ export interface StartOptions {
|
|
|
14
15
|
accountId: string;
|
|
15
16
|
region: string;
|
|
16
17
|
shutdownMs: number;
|
|
18
|
+
logLevel: LogLevel;
|
|
17
19
|
}
|
|
18
20
|
export interface KinesisLocal {
|
|
19
21
|
endpoint: string;
|
|
20
22
|
address: AddressInfo;
|
|
21
23
|
protocol: 'http' | 'https';
|
|
24
|
+
reset(stream?: string): string[];
|
|
22
25
|
close(): Promise<void>;
|
|
23
26
|
}
|
|
24
27
|
export declare function defaultOptions(): StartOptions;
|
package/dist/server.js
CHANGED
|
@@ -4,6 +4,7 @@ import { createServer as createHttp2Server, createSecureServer } from 'node:http
|
|
|
4
4
|
import { createServer as createNetServer } from 'node:net';
|
|
5
5
|
import { selfSignedCertificate } from './certificate.js';
|
|
6
6
|
import { createHandler } from './handler.js';
|
|
7
|
+
import { Logger } from './logger.js';
|
|
7
8
|
import { registerTransitions } from './operations/index.js';
|
|
8
9
|
import { Store } from './store/store.js';
|
|
9
10
|
const HTTP2_PREFACE = Buffer.from('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n', 'latin1');
|
|
@@ -20,6 +21,7 @@ export function defaultOptions() {
|
|
|
20
21
|
accountId: (process.env.AWS_ACCOUNT_ID || '0000-0000-0000').replace(/[^\d]/g, ''),
|
|
21
22
|
region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
|
|
22
23
|
shutdownMs: 5000,
|
|
24
|
+
logLevel: 'warn',
|
|
23
25
|
};
|
|
24
26
|
}
|
|
25
27
|
function routeByProtocol(socket, servers) {
|
|
@@ -54,14 +56,15 @@ function hostname(address) {
|
|
|
54
56
|
}
|
|
55
57
|
export async function start(overrides = {}) {
|
|
56
58
|
const options = { ...defaultOptions(), ...overrides };
|
|
59
|
+
const logger = new Logger(options.logLevel);
|
|
57
60
|
const credentials = options.keyFile && options.certFile
|
|
58
61
|
? { key: readFileSync(options.keyFile), cert: readFileSync(options.certFile) }
|
|
59
62
|
: selfSignedCertificate();
|
|
60
|
-
const store = new Store(options);
|
|
63
|
+
const store = new Store(options, logger);
|
|
61
64
|
registerTransitions(store);
|
|
62
65
|
store.startExpiry();
|
|
63
66
|
store.resume();
|
|
64
|
-
const handle = createHandler(store);
|
|
67
|
+
const handle = createHandler(store, logger);
|
|
65
68
|
const sockets = new Set();
|
|
66
69
|
const sessions = new Set();
|
|
67
70
|
let inFlight = 0;
|
|
@@ -140,6 +143,7 @@ export async function start(overrides = {}) {
|
|
|
140
143
|
endpoint: `${protocol}://${hostname(address)}:${address.port}`,
|
|
141
144
|
address,
|
|
142
145
|
protocol,
|
|
146
|
+
reset: stream => store.reset(stream),
|
|
143
147
|
close: () => closing ??= shutdown(),
|
|
144
148
|
};
|
|
145
149
|
}
|
package/dist/store/store.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
+
import type { Logger } from '../logger.ts';
|
|
2
3
|
import type { AccountSettings, ChannelDoc, ConsumerDoc, NewRecord, StoredRecord, StreamDoc } from './types.ts';
|
|
3
4
|
export declare const DATABASE_FILE = "aws-kinesis-local.sqlite";
|
|
4
5
|
export interface StoreOptions {
|
|
@@ -15,6 +16,7 @@ type Kind = 'stream' | 'consumer' | 'channel';
|
|
|
15
16
|
export type Transition = (store: Store, key: string, data: unknown) => void;
|
|
16
17
|
export declare class Store {
|
|
17
18
|
readonly options: StoreOptions;
|
|
19
|
+
private readonly logger;
|
|
18
20
|
readonly streams: Map<string, StreamDoc>;
|
|
19
21
|
readonly consumers: Map<string, ConsumerDoc>;
|
|
20
22
|
readonly channels: Map<string, ChannelDoc>;
|
|
@@ -27,7 +29,7 @@ export declare class Store {
|
|
|
27
29
|
private readonly timers;
|
|
28
30
|
private readonly statements;
|
|
29
31
|
private closed;
|
|
30
|
-
constructor(options: StoreOptions);
|
|
32
|
+
constructor(options: StoreOptions, logger: Logger);
|
|
31
33
|
private load;
|
|
32
34
|
transaction<T>(work: () => T): T;
|
|
33
35
|
private collection;
|
|
@@ -42,6 +44,7 @@ export declare class Store {
|
|
|
42
44
|
savePolicy(arn: string, policy: string): void;
|
|
43
45
|
removePolicy(arn: string): void;
|
|
44
46
|
saveSettings(settings: AccountSettings): void;
|
|
47
|
+
reset(name?: string): string[];
|
|
45
48
|
insertRecords(stream: string, records: NewRecord[]): void;
|
|
46
49
|
readRecords(stream: string, shardIx: number, fromSequence: string, limit: number): StoredRecord[];
|
|
47
50
|
firstSequenceAtOrAfter(stream: string, shardIx: number, timestamp: number): string | undefined;
|
package/dist/store/store.js
CHANGED
|
@@ -5,22 +5,25 @@ import { DatabaseSync } from 'node:sqlite';
|
|
|
5
5
|
import { parseSequence } from '../kinesis/sequence.js';
|
|
6
6
|
export const DATABASE_FILE = 'aws-kinesis-local.sqlite';
|
|
7
7
|
const EXPIRY_BATCH_SIZE = 500;
|
|
8
|
+
const defaultSettings = () => ({ MinimumThroughputBillingCommitment: { Status: 'DISABLED' } });
|
|
8
9
|
export class Store {
|
|
9
10
|
options;
|
|
11
|
+
logger;
|
|
10
12
|
streams = new Map();
|
|
11
13
|
consumers = new Map();
|
|
12
14
|
channels = new Map();
|
|
13
15
|
policies = new Map();
|
|
14
16
|
events = new EventEmitter();
|
|
15
|
-
settings =
|
|
17
|
+
settings = defaultSettings();
|
|
16
18
|
draining = false;
|
|
17
19
|
db;
|
|
18
20
|
transitions = new Map();
|
|
19
21
|
timers = new Map();
|
|
20
22
|
statements;
|
|
21
23
|
closed = false;
|
|
22
|
-
constructor(options) {
|
|
24
|
+
constructor(options, logger) {
|
|
23
25
|
this.options = options;
|
|
26
|
+
this.logger = logger;
|
|
24
27
|
this.events.setMaxListeners(0);
|
|
25
28
|
let file = ':memory:';
|
|
26
29
|
if (options.path) {
|
|
@@ -147,6 +150,30 @@ export class Store {
|
|
|
147
150
|
this.settings = settings;
|
|
148
151
|
this.write('settings', 'account', settings);
|
|
149
152
|
}
|
|
153
|
+
reset(name) {
|
|
154
|
+
const names = name === undefined ? [...this.streams.keys()].sort() : this.streams.has(name) ? [name] : [];
|
|
155
|
+
const arns = new Set(names.map(stream => this.streams.get(stream).StreamARN));
|
|
156
|
+
const consumers = [...this.consumers.values()].filter(consumer => names.includes(consumer.StreamName));
|
|
157
|
+
const channels = [...this.channels.values()].filter(channel => name === undefined || channel.StreamConfigurationList.some(entry => arns.has(entry.StreamARN)));
|
|
158
|
+
this.transaction(() => {
|
|
159
|
+
for (const stream of names) {
|
|
160
|
+
this.events.emit('stream-deleted', stream);
|
|
161
|
+
this.removePolicy(this.streams.get(stream).StreamARN);
|
|
162
|
+
this.removeStream(stream);
|
|
163
|
+
}
|
|
164
|
+
for (const consumer of consumers)
|
|
165
|
+
this.removeConsumer(consumer.ConsumerARN);
|
|
166
|
+
for (const channel of channels)
|
|
167
|
+
this.removeChannel(channel.ChannelARN);
|
|
168
|
+
if (name === undefined) {
|
|
169
|
+
for (const arn of [...this.policies.keys()])
|
|
170
|
+
this.removePolicy(arn);
|
|
171
|
+
this.settings = defaultSettings();
|
|
172
|
+
this.erase('settings', 'account');
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
return names;
|
|
176
|
+
}
|
|
150
177
|
insertRecords(stream, records) {
|
|
151
178
|
for (const record of records) {
|
|
152
179
|
this.statements.putRecord.run(stream, record.shardIx, record.sequenceNumber, record.partitionKey, record.data, record.arrival, record.encryptionType);
|
|
@@ -190,7 +217,7 @@ export class Store {
|
|
|
190
217
|
done = batches.next().done;
|
|
191
218
|
}
|
|
192
219
|
catch (error) {
|
|
193
|
-
|
|
220
|
+
this.logger.log('error', { message: 'Record expiry failed', error: error instanceof Error ? error.stack : error });
|
|
194
221
|
}
|
|
195
222
|
if (done)
|
|
196
223
|
this.scheduleExpiry(this.options.expiryIntervalMs, this.expiryBatches());
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@saidsef/aws-kinesis-local",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"aws",
|
|
@@ -60,7 +60,7 @@
|
|
|
60
60
|
"start": "node src/cli.ts"
|
|
61
61
|
},
|
|
62
62
|
"devDependencies": {
|
|
63
|
-
"@aws-sdk/client-kinesis": "3.
|
|
63
|
+
"@aws-sdk/client-kinesis": "3.1165.0",
|
|
64
64
|
"@smithy/eventstream-codec": "4.5.2",
|
|
65
65
|
"@smithy/node-http-handler": "4.14.2",
|
|
66
66
|
"@types/node": "26.6.6",
|