@saidsef/aws-kinesis-local 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server.d.ts +1 -0
- package/dist/server.js +2 -0
- package/dist/store/store.d.ts +6 -0
- package/dist/store/store.js +44 -0
- package/package.json +1 -1
package/dist/server.d.ts
CHANGED
package/dist/server.js
CHANGED
|
@@ -14,6 +14,7 @@ export function defaultOptions() {
|
|
|
14
14
|
createStreamMs: 500,
|
|
15
15
|
deleteStreamMs: 500,
|
|
16
16
|
updateStreamMs: 500,
|
|
17
|
+
expiryIntervalMs: 60000,
|
|
17
18
|
shardLimit: 10,
|
|
18
19
|
accountId: (process.env.AWS_ACCOUNT_ID || '0000-0000-0000').replace(/[^\d]/g, ''),
|
|
19
20
|
region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1',
|
|
@@ -49,6 +50,7 @@ export async function start(overrides = {}) {
|
|
|
49
50
|
const options = { ...defaultOptions(), ...overrides };
|
|
50
51
|
const store = new Store(options);
|
|
51
52
|
registerTransitions(store);
|
|
53
|
+
store.startExpiry();
|
|
52
54
|
store.resume();
|
|
53
55
|
const handler = createHandler(store);
|
|
54
56
|
const sockets = new Set();
|
package/dist/store/store.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export interface StoreOptions {
|
|
|
6
6
|
createStreamMs: number;
|
|
7
7
|
deleteStreamMs: number;
|
|
8
8
|
updateStreamMs: number;
|
|
9
|
+
expiryIntervalMs: number;
|
|
9
10
|
shardLimit: number;
|
|
10
11
|
accountId: string;
|
|
11
12
|
region: string;
|
|
@@ -49,6 +50,11 @@ export declare class Store {
|
|
|
49
50
|
} | undefined;
|
|
50
51
|
deleteRecords(stream: string, shardIx: number, sequenceNumbers: string[]): void;
|
|
51
52
|
deleteStreamRecords(stream: string): void;
|
|
53
|
+
startExpiry(): void;
|
|
54
|
+
private scheduleExpiry;
|
|
55
|
+
private expireBatch;
|
|
56
|
+
private expiryBatches;
|
|
57
|
+
private expireOldest;
|
|
52
58
|
registerTransition(action: string, transition: Transition): void;
|
|
53
59
|
schedule(kind: Kind, key: string, action: string, delayMs: number, data?: unknown): void;
|
|
54
60
|
private persist;
|
package/dist/store/store.js
CHANGED
|
@@ -2,7 +2,9 @@ import { EventEmitter } from 'node:events';
|
|
|
2
2
|
import { mkdirSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
+
import { parseSequence } from '../kinesis/sequence.js';
|
|
5
6
|
export const DATABASE_FILE = 'aws-kinesis-local.sqlite';
|
|
7
|
+
const EXPIRY_BATCH_SIZE = 500;
|
|
6
8
|
export class Store {
|
|
7
9
|
options;
|
|
8
10
|
streams = new Map();
|
|
@@ -52,6 +54,7 @@ export class Store {
|
|
|
52
54
|
readRecords: this.db.prepare('SELECT seq, partition_key, data, arrival, encryption FROM records WHERE stream = ? AND shard = ? AND seq >= ? ORDER BY seq LIMIT ?'),
|
|
53
55
|
firstAtOrAfter: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? AND arrival >= ? ORDER BY seq LIMIT 1'),
|
|
54
56
|
lastRecord: this.db.prepare('SELECT seq, arrival FROM records WHERE stream = ? AND shard = ? ORDER BY seq DESC LIMIT 1'),
|
|
57
|
+
oldestSequences: this.db.prepare('SELECT seq FROM records WHERE stream = ? AND shard = ? ORDER BY seq LIMIT ?'),
|
|
55
58
|
deleteRecord: this.db.prepare('DELETE FROM records WHERE stream = ? AND shard = ? AND seq = ?'),
|
|
56
59
|
deleteStreamRecords: this.db.prepare('DELETE FROM records WHERE stream = ?'),
|
|
57
60
|
};
|
|
@@ -173,6 +176,47 @@ export class Store {
|
|
|
173
176
|
return;
|
|
174
177
|
this.statements.deleteStreamRecords.run(stream);
|
|
175
178
|
}
|
|
179
|
+
startExpiry() {
|
|
180
|
+
this.scheduleExpiry(0, this.expiryBatches());
|
|
181
|
+
}
|
|
182
|
+
scheduleExpiry(delayMs, batches) {
|
|
183
|
+
this.timers.set('expiry', setTimeout(() => this.expireBatch(batches), delayMs));
|
|
184
|
+
}
|
|
185
|
+
expireBatch(batches) {
|
|
186
|
+
let done = true;
|
|
187
|
+
try {
|
|
188
|
+
done = batches.next().done;
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
console.error(error instanceof Error ? error.stack : error);
|
|
192
|
+
}
|
|
193
|
+
if (done)
|
|
194
|
+
this.scheduleExpiry(this.options.expiryIntervalMs, this.expiryBatches());
|
|
195
|
+
else
|
|
196
|
+
this.scheduleExpiry(0, batches);
|
|
197
|
+
}
|
|
198
|
+
*expiryBatches() {
|
|
199
|
+
for (const stream of [...this.streams.values()]) {
|
|
200
|
+
for (let shardIx = 0; shardIx < stream.Shards.length; shardIx++) {
|
|
201
|
+
while (this.expireOldest(stream, shardIx) === EXPIRY_BATCH_SIZE)
|
|
202
|
+
yield;
|
|
203
|
+
yield;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
expireOldest(stream, shardIx) {
|
|
208
|
+
const cutoff = Date.now() - stream.RetentionPeriodHours * 60 * 60 * 1000;
|
|
209
|
+
const oldest = this.statements.oldestSequences.all(stream.StreamName, shardIx, EXPIRY_BATCH_SIZE);
|
|
210
|
+
const expired = [];
|
|
211
|
+
for (const { seq } of oldest) {
|
|
212
|
+
if (parseSequence(seq).seqTime >= cutoff)
|
|
213
|
+
break;
|
|
214
|
+
expired.push(seq);
|
|
215
|
+
}
|
|
216
|
+
if (expired.length)
|
|
217
|
+
this.transaction(() => this.deleteRecords(stream.StreamName, shardIx, expired));
|
|
218
|
+
return expired.length;
|
|
219
|
+
}
|
|
176
220
|
registerTransition(action, transition) {
|
|
177
221
|
this.transitions.set(action, transition);
|
|
178
222
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@saidsef/aws-kinesis-local",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"description": "Local Amazon Kinesis Data Streams server for development and CI, with all 45 operations, enhanced fan-out, JSON and CBOR over HTTP/1.1 and HTTP/2",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"aws",
|