@netacea/kinesisingest 1.5.72 → 1.5.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/KinesisIngest.d.ts +16 -3
- package/dist/KinesisIngest.js +25 -5
- package/dist/KinesisIngest.js.map +1 -1
- package/package.json +2 -2
- package/package.json.bak +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,22 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
## 1.5.74 (2024-09-27)
|
|
7
|
+
|
|
8
|
+
**Note:** Version bump only for package @netacea/kinesisingest
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
## 1.5.73 (2024-09-24)
|
|
15
|
+
|
|
16
|
+
**Note:** Version bump only for package @netacea/kinesisingest
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
6
22
|
## 1.5.72 (2024-09-23)
|
|
7
23
|
|
|
8
24
|
**Note:** Version bump only for package @netacea/kinesisingest
|
package/dist/KinesisIngest.d.ts
CHANGED
|
@@ -17,16 +17,29 @@ export interface KinesisIngestConfigArgs {
|
|
|
17
17
|
}
|
|
18
18
|
export interface KinesisIngestArgs extends KinesisIngestConfigArgs {
|
|
19
19
|
apiKey: string;
|
|
20
|
+
/**
|
|
21
|
+
* Gradually increase the batch size to reduce risk of lost batches.
|
|
22
|
+
* This is intended for user on aws lambdas, which have an unpredictable lifetime.
|
|
23
|
+
*/
|
|
24
|
+
rampUpBatchSize?: boolean;
|
|
25
|
+
/**
|
|
26
|
+
* If a request triggest a POST to kinesis, then this is how maximum time
|
|
27
|
+
* the ingest method will wait for that request to finish.
|
|
28
|
+
* If undefined, then ingest will wait as long as needed ,until a batch is ingested.
|
|
29
|
+
*/
|
|
30
|
+
maxAwaitTimePerIngestCallMs?: number;
|
|
20
31
|
}
|
|
21
32
|
export default class KinesisIngest {
|
|
22
33
|
protected readonly kinesisStreamName: string;
|
|
23
34
|
protected readonly kinesisAccessKey?: string;
|
|
24
35
|
protected readonly kinesisSecretKey?: string;
|
|
25
|
-
protected readonly
|
|
36
|
+
protected readonly maxLogBatchSize: number;
|
|
26
37
|
protected readonly maxLogAgeSeconds: number;
|
|
38
|
+
protected logBatchSize: number;
|
|
39
|
+
protected maxAwaitTimePerIngestCallMs: undefined | number;
|
|
27
40
|
protected logCache: KinesisIngestWebLog[];
|
|
28
41
|
private intervalSet;
|
|
29
|
-
constructor({ kinesisStreamName, kinesisAccessKey, kinesisSecretKey, maxLogAgeSeconds, logBatchSize }: KinesisIngestArgs);
|
|
42
|
+
constructor({ kinesisStreamName, kinesisAccessKey, kinesisSecretKey, maxLogAgeSeconds, logBatchSize, rampUpBatchSize, maxAwaitTimePerIngestCallMs }: KinesisIngestArgs);
|
|
30
43
|
putToKinesis<MakeRequest extends KinesisMakeRequest>(makeRequest: MakeRequest): Promise<void>;
|
|
31
|
-
ingest<LogFormat extends KinesisIngestWebLog, MakeRequest extends KinesisMakeRequest>(log: LogFormat, makeRequest: MakeRequest): Promise<
|
|
44
|
+
ingest<LogFormat extends KinesisIngestWebLog, MakeRequest extends KinesisMakeRequest>(log: LogFormat, makeRequest: MakeRequest): Promise<void>;
|
|
32
45
|
}
|
package/dist/KinesisIngest.js
CHANGED
|
@@ -7,22 +7,31 @@ async function sleep(ms) {
|
|
|
7
7
|
});
|
|
8
8
|
}
|
|
9
9
|
class KinesisIngest {
|
|
10
|
-
constructor({ kinesisStreamName, kinesisAccessKey, kinesisSecretKey, maxLogAgeSeconds, logBatchSize }) {
|
|
11
|
-
this.
|
|
10
|
+
constructor({ kinesisStreamName, kinesisAccessKey, kinesisSecretKey, maxLogAgeSeconds, logBatchSize, rampUpBatchSize, maxAwaitTimePerIngestCallMs }) {
|
|
11
|
+
this.maxLogBatchSize = 20;
|
|
12
12
|
this.maxLogAgeSeconds = 10;
|
|
13
|
+
this.logBatchSize = 20;
|
|
13
14
|
this.logCache = [];
|
|
14
15
|
this.intervalSet = false;
|
|
15
16
|
this.kinesisStreamName = kinesisStreamName;
|
|
16
17
|
this.kinesisAccessKey = kinesisAccessKey;
|
|
17
18
|
this.kinesisSecretKey = kinesisSecretKey;
|
|
19
|
+
this.maxAwaitTimePerIngestCallMs = maxAwaitTimePerIngestCallMs;
|
|
18
20
|
if (maxLogAgeSeconds !== undefined &&
|
|
19
21
|
maxLogAgeSeconds < this.maxLogAgeSeconds &&
|
|
20
22
|
maxLogAgeSeconds > 0) {
|
|
21
23
|
this.maxLogAgeSeconds = maxLogAgeSeconds;
|
|
22
24
|
}
|
|
23
25
|
if (logBatchSize !== undefined) {
|
|
24
|
-
this.
|
|
26
|
+
this.maxLogBatchSize = logBatchSize;
|
|
25
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* If we are following a ramp up strategy, then start at 1,
|
|
30
|
+
* otherwise start at the maximum.
|
|
31
|
+
*/
|
|
32
|
+
this.logBatchSize = rampUpBatchSize === true
|
|
33
|
+
? 1
|
|
34
|
+
: this.maxLogBatchSize;
|
|
26
35
|
}
|
|
27
36
|
async putToKinesis(makeRequest) {
|
|
28
37
|
if (this.logCache.length === 0) {
|
|
@@ -43,6 +52,9 @@ class KinesisIngest {
|
|
|
43
52
|
path: data.path,
|
|
44
53
|
body: data.body
|
|
45
54
|
});
|
|
55
|
+
if (this.logBatchSize !== this.maxLogBatchSize) {
|
|
56
|
+
this.logBatchSize = Math.min(this.maxLogBatchSize, this.logBatchSize * 2);
|
|
57
|
+
}
|
|
46
58
|
}
|
|
47
59
|
catch (e) {
|
|
48
60
|
this.logCache.push(...localCache);
|
|
@@ -52,17 +64,25 @@ class KinesisIngest {
|
|
|
52
64
|
async ingest(log, makeRequest) {
|
|
53
65
|
this.logCache.push(log);
|
|
54
66
|
if (this.logCache.length >= this.logBatchSize) {
|
|
55
|
-
|
|
67
|
+
const promises = [];
|
|
68
|
+
promises.push(this.putToKinesis(makeRequest));
|
|
69
|
+
if (this.maxAwaitTimePerIngestCallMs !== undefined) {
|
|
70
|
+
promises.push(sleep(this.maxAwaitTimePerIngestCallMs));
|
|
71
|
+
}
|
|
72
|
+
await Promise.race(promises);
|
|
56
73
|
}
|
|
57
74
|
else if (!this.intervalSet) {
|
|
58
75
|
this.intervalSet = true;
|
|
59
|
-
|
|
76
|
+
const batchPromise = sleep(this.maxLogAgeSeconds * 1000)
|
|
60
77
|
.then(async () => {
|
|
61
78
|
await this.putToKinesis(makeRequest);
|
|
62
79
|
this.intervalSet = false;
|
|
63
80
|
}).catch(() => {
|
|
64
81
|
// swallow error, http errors are handled in putToKinesis
|
|
65
82
|
});
|
|
83
|
+
if (this.maxAwaitTimePerIngestCallMs === undefined) {
|
|
84
|
+
await batchPromise;
|
|
85
|
+
}
|
|
66
86
|
}
|
|
67
87
|
}
|
|
68
88
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"KinesisIngest.js","sourceRoot":"","sources":["../src/KinesisIngest.ts"],"names":[],"mappings":";;AAAA,uCAAuC;AAkBvC,KAAK,UAAU,KAAK,CAAE,EAAU;IAC9B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;QAC1B,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IACzB,CAAC,CAAC,CAAA;AACJ,CAAC;
|
|
1
|
+
{"version":3,"file":"KinesisIngest.js","sourceRoot":"","sources":["../src/KinesisIngest.ts"],"names":[],"mappings":";;AAAA,uCAAuC;AAkBvC,KAAK,UAAU,KAAK,CAAE,EAAU;IAC9B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;QAC1B,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IACzB,CAAC,CAAC,CAAA;AACJ,CAAC;AAgBD,MAAqB,aAAa;IAWhC,YAAa,EACX,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,eAAe,EACf,2BAA2B,EACT;QAfD,oBAAe,GAAW,EAAE,CAAA;QAC5B,qBAAgB,GAAW,EAAE,CAAA;QACtC,iBAAY,GAAW,EAAE,CAAA;QAEzB,aAAQ,GAA0B,EAAE,CAAA;QACtC,gBAAW,GAAY,KAAK,CAAA;QAWlC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAA;QAC1C,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAA;QAE9D,IACE,gBAAgB,KAAK,SAAS;YAC9B,gBAAgB,GAAG,IAAI,CAAC,gBAAgB;YACxC,gBAAgB,GAAG,CAAC,EACpB,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAC1C,CAAC;QAED,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,CAAC,eAAe,GAAG,YAAY,CAAA;QACrC,CAAC;QAED;;;WAGG;QACH,IAAI,CAAC,YAAY,GAAG,eAAe,KAAK,IAAI;YAC1C,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,IAAI,CAAC,eAAe,CAAA;IAC1B,CAAC;IAEM,KAAK,CAAC,YAAY,CAA0C,WAAwB;QACzF,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAM;QACR,CAAC;QACD,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAA;QACrC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;QAClB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAA,qBAAW,EAAC;gBACvB,UAAU,EAAE,IAAI,CAAC,iBAAiB;gBAClC,WAAW,EAAE,IAAI,CAAC,gBAAgB;gBAClC,eAAe,EAAE,IAAI,CAAC,gBAAgB;aACvC,EAAE,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAEjC,MAAM,WAAW,CAAC;gBAChB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,IAAI,EAAE,WAAW,IAAI,CAAC,QAAQ,EAAE;gBAChC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAA;YAEF,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC/C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAA;YAC3E,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;YACjC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QAClB,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,MAAM,CAEjB,GAAc,EACd,WAAwB;QAExB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAEvB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAA4B,EAAE,CAAA;YAC5C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAA;YAE7C,IAAI,IAAI,CAAC,2BAA2B,KAAK,SAAS,EAAE,CAAC;gBACnD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAA;YACxD,CAAC;YACD,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC9B,CAAC;aAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YAEvB,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;iBACrD,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAA;gBACpC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;YAC1B,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;gBACZ,yDAAyD;YAC3D,CAAC,CAAC,CAAA;YAEJ,IAAI,IAAI,CAAC,2BAA2B,KAAK,SAAS,EAAE,CAAC;gBACnD,MAAM,YAAY,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;CACF;AA3GD,gCA2GC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@netacea/kinesisingest",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.74",
|
|
4
4
|
"description": "Kinesis Ingest module for Netacea.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -16,5 +16,5 @@
|
|
|
16
16
|
"dependencies": {
|
|
17
17
|
"aws4": "1.11.0"
|
|
18
18
|
},
|
|
19
|
-
"gitHead": "
|
|
19
|
+
"gitHead": "3631732cf87f8e0abc170bdf03cef29a934d3b01"
|
|
20
20
|
}
|
package/package.json.bak
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@netacea/kinesisingest",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.74",
|
|
4
4
|
"description": "Kinesis Ingest module for Netacea.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -33,5 +33,5 @@
|
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"aws4": "1.11.0"
|
|
35
35
|
},
|
|
36
|
-
"gitHead": "
|
|
36
|
+
"gitHead": "3631732cf87f8e0abc170bdf03cef29a934d3b01"
|
|
37
37
|
}
|