@mhmdhammoud/meritt-utils 1.6.1 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReleaseNotes.md +17 -1
- package/dist/__tests__/elastic-transport.test.d.ts +1 -0
- package/dist/__tests__/elastic-transport.test.js +78 -0
- package/dist/__tests__/logger.test.js +21 -0
- package/dist/lib/elastic-transport.d.ts +5 -1
- package/dist/lib/elastic-transport.js +49 -11
- package/dist/lib/logger.js +11 -6
- package/dist/types/logger.d.ts +2 -1
- package/package.json +9 -3
- package/.github/workflows/npm-publish.yml +0 -84
- package/.github/workflows/push.yml +0 -83
- package/.husky/pre-commit +0 -62
- package/.prettierrc +0 -8
- package/.prettierrc.json +0 -8
- package/ISSUES.md +0 -0
- package/eslint.config.mjs +0 -64
- package/example.env +0 -6
- package/jest.config.js +0 -13
- package/src/__tests__/colorful.test.ts +0 -77
- package/src/__tests__/formatter.test.ts +0 -96
- package/src/__tests__/logger.test.ts +0 -122
- package/src/env.d.ts +0 -9
- package/src/index.ts +0 -1
- package/src/lib/colorful.ts +0 -167
- package/src/lib/cypto.ts +0 -296
- package/src/lib/elastic-transport.ts +0 -307
- package/src/lib/formatter.ts +0 -125
- package/src/lib/imagefull.ts +0 -41
- package/src/lib/index.ts +0 -12
- package/src/lib/logger.ts +0 -590
- package/src/lib/pdf.ts +0 -25
- package/src/lib/trace-store.ts +0 -46
- package/src/types/index.ts +0 -1
- package/src/types/logger.ts +0 -66
- package/src/utilities/axios.ts +0 -4
- package/src/utilities/index.ts +0 -1
- package/tsconfig.json +0 -16
package/src/lib/cypto.ts
DELETED
|
@@ -1,296 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
Author : Omar Sawwas https://github.com/omarsawwas
|
|
3
|
-
Date : 2023-06-17
|
|
4
|
-
Description : RSA algorithm
|
|
5
|
-
Version : 1.0
|
|
6
|
-
*/
|
|
7
|
-
class Crypto {
|
|
8
|
-
// Default variables
|
|
9
|
-
private _primeNumberP = 17
|
|
10
|
-
private _primeNumberQ = 19
|
|
11
|
-
private _phiN = (this._primeNumberP - 1) * (this._primeNumberQ - 1)
|
|
12
|
-
private _moduloDenominator = this._primeNumberP * this._primeNumberQ
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
*
|
|
16
|
-
* @param number - The number to check if prime
|
|
17
|
-
* @returns The boolean answer of prime checking operation
|
|
18
|
-
* @example
|
|
19
|
-
* ```typescript
|
|
20
|
-
* this._isPrime(5)
|
|
21
|
-
* ```
|
|
22
|
-
* */
|
|
23
|
-
|
|
24
|
-
private _isPrime = (number: number): boolean => {
|
|
25
|
-
if (number <= 1) {
|
|
26
|
-
return false
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
if (number <= 3) {
|
|
30
|
-
return true
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
if (number % 2 === 0 || number % 3 === 0) {
|
|
34
|
-
return false
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
for (let i = 5; i * i <= number; i += 6) {
|
|
38
|
-
if (number % i === 0 || number % (i + 2) === 0) {
|
|
39
|
-
return false
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
return true
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Set the P prime number for the RSA algorithm
|
|
48
|
-
* @param primeP - The prime number P of type string
|
|
49
|
-
* @returns void
|
|
50
|
-
* @example
|
|
51
|
-
* ```typescript
|
|
52
|
-
* crypto.setPrimeP(17)
|
|
53
|
-
* ```
|
|
54
|
-
* */
|
|
55
|
-
|
|
56
|
-
public setPrimeP(primeP: number) {
|
|
57
|
-
if (isNaN(primeP) || primeP === 0) {
|
|
58
|
-
throw new Error('Provide a valid number')
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
if (!this._isPrime(primeP)) {
|
|
62
|
-
throw new Error('The number is not prime')
|
|
63
|
-
}
|
|
64
|
-
this._primeNumberP = primeP
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Set the Q prime number for the RSA algorithm
|
|
69
|
-
* @param primeQ - The prime number Q of type string
|
|
70
|
-
* @returns void
|
|
71
|
-
* @example
|
|
72
|
-
* ```typescript
|
|
73
|
-
* crypto.setPrimeQ(19)
|
|
74
|
-
* ```
|
|
75
|
-
* */
|
|
76
|
-
|
|
77
|
-
public setPrimeQ(primeQ: number) {
|
|
78
|
-
if (isNaN(primeQ) || primeQ === 0) {
|
|
79
|
-
throw new Error('Provide a valid number')
|
|
80
|
-
}
|
|
81
|
-
if (!this._isPrime(primeQ)) {
|
|
82
|
-
throw new Error('The number is not prime')
|
|
83
|
-
}
|
|
84
|
-
this._primeNumberQ = primeQ
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
*
|
|
89
|
-
* @param base - The base number of type number
|
|
90
|
-
* @param exponent - The exponent number of type number
|
|
91
|
-
* @returns The modular exponentiation of the base and the exponent
|
|
92
|
-
* @example
|
|
93
|
-
* ```typescript
|
|
94
|
-
* this.modularExponentiation(2, 3)
|
|
95
|
-
* ```
|
|
96
|
-
* */
|
|
97
|
-
|
|
98
|
-
private _modularExponentiation = (base: number, exponent: number) => {
|
|
99
|
-
if (isNaN(base) || isNaN(exponent)) {
|
|
100
|
-
throw new Error('Provide a valid number')
|
|
101
|
-
}
|
|
102
|
-
if (this._moduloDenominator === 1) {
|
|
103
|
-
return 0
|
|
104
|
-
}
|
|
105
|
-
let result = 1
|
|
106
|
-
base = base % this._moduloDenominator
|
|
107
|
-
while (exponent > 0) {
|
|
108
|
-
if (exponent % 2 === 1) {
|
|
109
|
-
result = (result * base) % this._moduloDenominator
|
|
110
|
-
}
|
|
111
|
-
exponent = Math.floor(exponent / 2)
|
|
112
|
-
base = (base * base) % this._moduloDenominator
|
|
113
|
-
}
|
|
114
|
-
return result
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
*
|
|
119
|
-
* @param temporaryVarriableOne - The number one
|
|
120
|
-
* @param temporaryVarriableTwo- The number two (phi n)
|
|
121
|
-
* @returns The boolean answer of relatively prime checking operation with phi n
|
|
122
|
-
* @example
|
|
123
|
-
* ```typescript
|
|
124
|
-
* this._areRelativelyPrime(5,9)
|
|
125
|
-
* ```
|
|
126
|
-
* */
|
|
127
|
-
|
|
128
|
-
private _areRelativelyPrime = (
|
|
129
|
-
temporaryVarriableOne: number,
|
|
130
|
-
temporaryVarriableTwo: number
|
|
131
|
-
) => {
|
|
132
|
-
while (temporaryVarriableTwo !== 0) {
|
|
133
|
-
const temp = temporaryVarriableTwo
|
|
134
|
-
temporaryVarriableTwo = temporaryVarriableOne % temporaryVarriableTwo
|
|
135
|
-
temporaryVarriableOne = temp
|
|
136
|
-
}
|
|
137
|
-
return temporaryVarriableOne === 1
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
*
|
|
142
|
-
* @returns The public and private key number which is a random number generated to match conditions set by RSA algorithm
|
|
143
|
-
* @example
|
|
144
|
-
* ```typescript
|
|
145
|
-
* crypto.generateKeys()
|
|
146
|
-
* ```
|
|
147
|
-
* */
|
|
148
|
-
|
|
149
|
-
public generateKeys = (): Record<'publicKey' | 'privateKey', number> => {
|
|
150
|
-
const temporaryArray: number[] = []
|
|
151
|
-
for (let i = 2; i < this._phiN; i++) {
|
|
152
|
-
if (this._areRelativelyPrime(i, this._phiN)) {
|
|
153
|
-
temporaryArray.push(i)
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
if (temporaryArray.length === 0)
|
|
157
|
-
throw new Error('No public key options found!')
|
|
158
|
-
const randomIndex = Math.floor(Math.random() * temporaryArray.length)
|
|
159
|
-
// return temporaryArray[randomIndex]
|
|
160
|
-
const publicKey = temporaryArray[randomIndex]
|
|
161
|
-
const privateKey = this._generatePrivateKey(publicKey)
|
|
162
|
-
if (privateKey === publicKey) {
|
|
163
|
-
return this.generateKeys()
|
|
164
|
-
}
|
|
165
|
-
return {
|
|
166
|
-
privateKey: publicKey,
|
|
167
|
-
publicKey: privateKey,
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
/**
|
|
172
|
-
*
|
|
173
|
-
* @param publicKey - The public key number
|
|
174
|
-
* @returns The private key number which is the modulus inverse with repect to modulo denominator
|
|
175
|
-
* @example
|
|
176
|
-
* ```typescript
|
|
177
|
-
* crypto.generatePrivateKey(5)
|
|
178
|
-
* ```
|
|
179
|
-
* */
|
|
180
|
-
|
|
181
|
-
private _generatePrivateKey = (publicKey: number): number => {
|
|
182
|
-
if (publicKey <= 1 || isNaN(publicKey)) {
|
|
183
|
-
throw new Error('Public key should be a number greater than 1')
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
if (!this._areRelativelyPrime(publicKey, this._phiN)) {
|
|
187
|
-
throw new Error(
|
|
188
|
-
'Public key should be relatively prime with respect to phi N'
|
|
189
|
-
)
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
let t1 = 0
|
|
193
|
-
let t2 = 1
|
|
194
|
-
let r1 = this._phiN
|
|
195
|
-
let r2 = publicKey
|
|
196
|
-
|
|
197
|
-
while (r2 !== 0) {
|
|
198
|
-
const quotient = Math.floor(r1 / r2)
|
|
199
|
-
const temp1 = t1
|
|
200
|
-
const temp2 = r1
|
|
201
|
-
|
|
202
|
-
t1 = t2
|
|
203
|
-
r1 = r2
|
|
204
|
-
|
|
205
|
-
t2 = temp1 - quotient * t2
|
|
206
|
-
r2 = temp2 - quotient * r2
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
if (r1 > 1) {
|
|
210
|
-
throw new Error('The number does not have a modular inverse.')
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
if (t1 < 0) {
|
|
214
|
-
t1 += this._phiN
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
return t1
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
/**
|
|
221
|
-
* Encrypt the message using the public key
|
|
222
|
-
* @param message - The message to encrypt of type string|number|Record
|
|
223
|
-
* @param publicKey - The public key to encrypt the message with of type number
|
|
224
|
-
* @returns The encrypted message of type number[]
|
|
225
|
-
* @example
|
|
226
|
-
* ```typescript
|
|
227
|
-
* crypto.encrypt('Hello World', 7)
|
|
228
|
-
* ```
|
|
229
|
-
* */
|
|
230
|
-
|
|
231
|
-
public encrypt = (
|
|
232
|
-
message: string | number | Record<any, any>,
|
|
233
|
-
publicKey: number
|
|
234
|
-
): number[] => {
|
|
235
|
-
if (isNaN(publicKey)) {
|
|
236
|
-
throw new Error('Public key should be a number')
|
|
237
|
-
}
|
|
238
|
-
if (publicKey <= 0) {
|
|
239
|
-
throw new Error(
|
|
240
|
-
'Private key should be a positive number different than 0'
|
|
241
|
-
)
|
|
242
|
-
}
|
|
243
|
-
if (typeof message === 'number') {
|
|
244
|
-
message = message.toString()
|
|
245
|
-
}
|
|
246
|
-
if (typeof message === 'object') {
|
|
247
|
-
message = JSON.stringify(message)
|
|
248
|
-
}
|
|
249
|
-
const encryptedMessage: number[] = []
|
|
250
|
-
for (let i = 0; i < message.length; i++) {
|
|
251
|
-
const a = message.charCodeAt(i)
|
|
252
|
-
encryptedMessage.push(this._modularExponentiation(a, publicKey))
|
|
253
|
-
}
|
|
254
|
-
return encryptedMessage
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
/**
|
|
258
|
-
* Decrypt the message using the private key
|
|
259
|
-
* @param encryptedMessage - The encryptedMessage to decrypt of type number[]
|
|
260
|
-
* @param privateKey - The private key to decrypt the message with of type number
|
|
261
|
-
* @returns The decrypted message of type string
|
|
262
|
-
* @example
|
|
263
|
-
* ```typescript
|
|
264
|
-
* crypto.decrypt([72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100], 23)
|
|
265
|
-
* ```
|
|
266
|
-
* */
|
|
267
|
-
public decrypt = (encryptedMessage: number[], privateKey: number): string => {
|
|
268
|
-
if (isNaN(privateKey)) {
|
|
269
|
-
throw new Error('Public key should be a number')
|
|
270
|
-
}
|
|
271
|
-
if (privateKey <= 0) {
|
|
272
|
-
throw new Error(
|
|
273
|
-
'Private key should be a positive number different than 0'
|
|
274
|
-
)
|
|
275
|
-
}
|
|
276
|
-
if (!Array.isArray(encryptedMessage)) {
|
|
277
|
-
throw new Error('Encrypted message should be an array')
|
|
278
|
-
}
|
|
279
|
-
const allValidNumbers = encryptedMessage.every((number) => !isNaN(number))
|
|
280
|
-
if (!allValidNumbers) {
|
|
281
|
-
throw new Error('Encrypted message should be an array of numbers')
|
|
282
|
-
}
|
|
283
|
-
const decryptedAsciiMessage: number[] = []
|
|
284
|
-
for (let i = 0; i < encryptedMessage.length; i++) {
|
|
285
|
-
const a = encryptedMessage[i]
|
|
286
|
-
decryptedAsciiMessage.push(this._modularExponentiation(a, privateKey))
|
|
287
|
-
}
|
|
288
|
-
let finalString = ''
|
|
289
|
-
for (let i = 0; i < decryptedAsciiMessage.length; i++) {
|
|
290
|
-
finalString += String.fromCharCode(decryptedAsciiMessage[i])
|
|
291
|
-
}
|
|
292
|
-
return finalString
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
const crypto = new Crypto()
|
|
296
|
-
export default crypto
|
|
@@ -1,307 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Elasticsearch transport for Pino with connection lifecycle resilience.
|
|
3
|
-
*
|
|
4
|
-
* Based on pino-elasticsearch with a fix for GitHub issue #140:
|
|
5
|
-
* When maxRetries are exceeded and Elasticsearch nodes are DEAD, the bulk helper
|
|
6
|
-
* destroys the splitter stream, causing logs to stop permanently until restart.
|
|
7
|
-
*
|
|
8
|
-
* This implementation overrides splitter.destroy to BOTH resurrect the connection
|
|
9
|
-
* pool AND reinitialize the bulk handler, so logging continues after ES recovers.
|
|
10
|
-
*
|
|
11
|
-
* @see https://github.com/pinojs/pino-elasticsearch/issues/140
|
|
12
|
-
* @see https://github.com/pinojs/pino-elasticsearch/issues/72
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
16
|
-
const split = require('split2') as (
|
|
17
|
-
fn: (line: string) => unknown,
|
|
18
|
-
opts?: { autoDestroy?: boolean }
|
|
19
|
-
) => NodeJS.ReadWriteStream
|
|
20
|
-
import { Client } from '@elastic/elasticsearch'
|
|
21
|
-
import type { ClientOptions } from '@elastic/elasticsearch'
|
|
22
|
-
|
|
23
|
-
export interface ElasticTransportOptions extends Pick<
|
|
24
|
-
ClientOptions,
|
|
25
|
-
| 'node'
|
|
26
|
-
| 'auth'
|
|
27
|
-
| 'cloud'
|
|
28
|
-
| 'caFingerprint'
|
|
29
|
-
| 'Connection'
|
|
30
|
-
| 'ConnectionPool'
|
|
31
|
-
| 'maxRetries'
|
|
32
|
-
| 'requestTimeout'
|
|
33
|
-
> {
|
|
34
|
-
sniffOnConnectionFault?: boolean
|
|
35
|
-
index?: string | ((logTime: string) => string)
|
|
36
|
-
flushBytes?: number
|
|
37
|
-
'flush-bytes'?: number
|
|
38
|
-
flushInterval?: number
|
|
39
|
-
'flush-interval'?: number
|
|
40
|
-
esVersion?: number
|
|
41
|
-
'es-version'?: number
|
|
42
|
-
rejectUnauthorized?: boolean
|
|
43
|
-
tls?: ClientOptions['tls']
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
interface LogDocument {
|
|
47
|
-
time?: string
|
|
48
|
-
'@timestamp'?: string
|
|
49
|
-
[k: string]: unknown
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
function setDateTimeString(value: unknown): string {
|
|
53
|
-
if (value !== null && typeof value === 'object' && 'time' in value) {
|
|
54
|
-
const t = (value as { time: unknown }).time
|
|
55
|
-
if (
|
|
56
|
-
(typeof t === 'string' && t.length > 0) ||
|
|
57
|
-
(typeof t === 'number' && t >= 0)
|
|
58
|
-
) {
|
|
59
|
-
return new Date(t).toISOString()
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
return new Date().toISOString()
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function getIndexName(
|
|
66
|
-
index: string | ((logTime: string) => string),
|
|
67
|
-
time: string
|
|
68
|
-
): string {
|
|
69
|
-
if (typeof index === 'function') {
|
|
70
|
-
return index(time)
|
|
71
|
-
}
|
|
72
|
-
return index.replace('%{DATE}', time.substring(0, 10))
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
function createBulkSender(
|
|
76
|
-
opts: ElasticTransportOptions,
|
|
77
|
-
client: Client,
|
|
78
|
-
splitter: NodeJS.ReadWriteStream
|
|
79
|
-
): {
|
|
80
|
-
add: (doc: unknown) => void
|
|
81
|
-
flush: () => Promise<void>
|
|
82
|
-
close: () => Promise<void>
|
|
83
|
-
} {
|
|
84
|
-
const esVersion = Number(opts.esVersion ?? opts['es-version'] ?? 7)
|
|
85
|
-
const index = opts.index ?? 'pino'
|
|
86
|
-
const buildIndexName = typeof index === 'function' ? index : null
|
|
87
|
-
const opType = esVersion >= 7 ? undefined : undefined
|
|
88
|
-
const flushBytes = opts.flushBytes ?? opts['flush-bytes'] ?? 1000
|
|
89
|
-
const flushInterval = opts.flushInterval ?? opts['flush-interval'] ?? 3000
|
|
90
|
-
|
|
91
|
-
let buffer: unknown[] = []
|
|
92
|
-
let bufferedBytes = 0
|
|
93
|
-
let timer: NodeJS.Timeout | undefined
|
|
94
|
-
let isFlushing = false
|
|
95
|
-
let flushAgain = false
|
|
96
|
-
|
|
97
|
-
const indexName = (time = new Date().toISOString()) =>
|
|
98
|
-
buildIndexName ? buildIndexName(time) : getIndexName(index as string, time)
|
|
99
|
-
|
|
100
|
-
const clearFlushTimer = () => {
|
|
101
|
-
if (timer) {
|
|
102
|
-
clearTimeout(timer)
|
|
103
|
-
timer = undefined
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
const scheduleFlush = () => {
|
|
108
|
-
if (timer || buffer.length === 0) {
|
|
109
|
-
return
|
|
110
|
-
}
|
|
111
|
-
timer = setTimeout(() => {
|
|
112
|
-
timer = undefined
|
|
113
|
-
void flush()
|
|
114
|
-
}, flushInterval)
|
|
115
|
-
timer.unref?.()
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
const buildOperation = (doc: unknown): [Record<string, unknown>, unknown] => {
|
|
119
|
-
try {
|
|
120
|
-
const d = doc as LogDocument
|
|
121
|
-
const date = d.time ?? d['@timestamp'] ?? new Date().toISOString()
|
|
122
|
-
if (opType === 'create') {
|
|
123
|
-
d['@timestamp'] = date
|
|
124
|
-
}
|
|
125
|
-
return [
|
|
126
|
-
{
|
|
127
|
-
index: {
|
|
128
|
-
_index: indexName(date),
|
|
129
|
-
op_type: opType,
|
|
130
|
-
},
|
|
131
|
-
},
|
|
132
|
-
doc,
|
|
133
|
-
]
|
|
134
|
-
} catch {
|
|
135
|
-
return [
|
|
136
|
-
{
|
|
137
|
-
index: {
|
|
138
|
-
_index: indexName(),
|
|
139
|
-
op_type: opType,
|
|
140
|
-
},
|
|
141
|
-
},
|
|
142
|
-
doc,
|
|
143
|
-
]
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
const emitDroppedDocument = (doc: unknown, cause?: unknown) => {
|
|
148
|
-
const error = new Error('Dropped document') as Error & {
|
|
149
|
-
document: unknown
|
|
150
|
-
cause?: unknown
|
|
151
|
-
}
|
|
152
|
-
error.document = doc
|
|
153
|
-
error.cause = cause
|
|
154
|
-
splitter.emit('insertError', error)
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
const flush = async (): Promise<void> => {
|
|
158
|
-
if (isFlushing) {
|
|
159
|
-
flushAgain = true
|
|
160
|
-
return
|
|
161
|
-
}
|
|
162
|
-
clearFlushTimer()
|
|
163
|
-
if (buffer.length === 0) {
|
|
164
|
-
return
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
isFlushing = true
|
|
168
|
-
const batch = buffer
|
|
169
|
-
buffer = []
|
|
170
|
-
bufferedBytes = 0
|
|
171
|
-
|
|
172
|
-
try {
|
|
173
|
-
const operations = batch.flatMap(buildOperation)
|
|
174
|
-
const response = await client.bulk({
|
|
175
|
-
operations,
|
|
176
|
-
refresh: false,
|
|
177
|
-
timeout: opts.requestTimeout ? `${opts.requestTimeout}ms` : undefined,
|
|
178
|
-
})
|
|
179
|
-
|
|
180
|
-
const body = response as {
|
|
181
|
-
errors?: boolean
|
|
182
|
-
items?: Array<Record<string, { error?: unknown }>>
|
|
183
|
-
}
|
|
184
|
-
if (body.errors && Array.isArray(body.items)) {
|
|
185
|
-
body.items.forEach((item, index) => {
|
|
186
|
-
const result = Object.values(item)[0]
|
|
187
|
-
if (result?.error) {
|
|
188
|
-
emitDroppedDocument(batch[index], result.error)
|
|
189
|
-
}
|
|
190
|
-
})
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
splitter.emit('insert', {
|
|
194
|
-
successful: batch.length,
|
|
195
|
-
failed: body.errors ? (body.items?.length ?? 0) : 0,
|
|
196
|
-
})
|
|
197
|
-
} catch (err) {
|
|
198
|
-
splitter.emit('error', err)
|
|
199
|
-
// Drop the failed batch instead of wedging the stream. The next log line
|
|
200
|
-
// creates a fresh bulk request and can recover without a process restart.
|
|
201
|
-
batch.forEach((doc) => emitDroppedDocument(doc, err))
|
|
202
|
-
} finally {
|
|
203
|
-
isFlushing = false
|
|
204
|
-
if (flushAgain || buffer.length > 0) {
|
|
205
|
-
flushAgain = false
|
|
206
|
-
scheduleFlush()
|
|
207
|
-
if (bufferedBytes >= flushBytes) {
|
|
208
|
-
void flush()
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
return {
|
|
215
|
-
add(doc: unknown) {
|
|
216
|
-
buffer.push(doc)
|
|
217
|
-
bufferedBytes += Buffer.byteLength(JSON.stringify(doc))
|
|
218
|
-
if (bufferedBytes >= flushBytes) {
|
|
219
|
-
void flush()
|
|
220
|
-
return
|
|
221
|
-
}
|
|
222
|
-
scheduleFlush()
|
|
223
|
-
},
|
|
224
|
-
flush,
|
|
225
|
-
async close() {
|
|
226
|
-
clearFlushTimer()
|
|
227
|
-
await flush()
|
|
228
|
-
},
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
export const createElasticTransport = (
|
|
233
|
-
opts: ElasticTransportOptions = {}
|
|
234
|
-
): NodeJS.ReadWriteStream => {
|
|
235
|
-
const splitter = split(
|
|
236
|
-
function (this: NodeJS.ReadWriteStream, line: string) {
|
|
237
|
-
let value: unknown
|
|
238
|
-
|
|
239
|
-
try {
|
|
240
|
-
value = JSON.parse(line) as unknown
|
|
241
|
-
} catch (error) {
|
|
242
|
-
this.emit('unknown', line, error)
|
|
243
|
-
return
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
if (typeof value === 'boolean') {
|
|
247
|
-
this.emit('unknown', line, 'Boolean value ignored')
|
|
248
|
-
return
|
|
249
|
-
}
|
|
250
|
-
if (value === null) {
|
|
251
|
-
this.emit('unknown', line, 'Null value ignored')
|
|
252
|
-
return
|
|
253
|
-
}
|
|
254
|
-
if (typeof value !== 'object') {
|
|
255
|
-
value = { data: value, time: setDateTimeString(value) }
|
|
256
|
-
} else {
|
|
257
|
-
const obj = value as Record<string, unknown>
|
|
258
|
-
if (obj['@timestamp'] === undefined) {
|
|
259
|
-
;(obj as LogDocument).time = setDateTimeString(obj)
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
return value
|
|
263
|
-
},
|
|
264
|
-
{ autoDestroy: true }
|
|
265
|
-
)
|
|
266
|
-
|
|
267
|
-
const clientOpts: ClientOptions = {
|
|
268
|
-
node: opts.node,
|
|
269
|
-
auth: opts.auth,
|
|
270
|
-
cloud: opts.cloud,
|
|
271
|
-
tls: { rejectUnauthorized: opts.rejectUnauthorized, ...opts.tls },
|
|
272
|
-
maxRetries: opts.maxRetries,
|
|
273
|
-
requestTimeout: opts.requestTimeout,
|
|
274
|
-
sniffOnConnectionFault: opts.sniffOnConnectionFault,
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
if (opts.caFingerprint) {
|
|
278
|
-
clientOpts.caFingerprint = opts.caFingerprint
|
|
279
|
-
}
|
|
280
|
-
if (opts.Connection) {
|
|
281
|
-
clientOpts.Connection = opts.Connection
|
|
282
|
-
}
|
|
283
|
-
if (opts.ConnectionPool) {
|
|
284
|
-
clientOpts.ConnectionPool = opts.ConnectionPool
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
const client = new Client(clientOpts)
|
|
288
|
-
const bulkSender = createBulkSender(opts, client, splitter)
|
|
289
|
-
|
|
290
|
-
splitter.on('data', (doc) => {
|
|
291
|
-
bulkSender.add(doc)
|
|
292
|
-
})
|
|
293
|
-
splitter.on('finish', () => {
|
|
294
|
-
void bulkSender.close()
|
|
295
|
-
})
|
|
296
|
-
|
|
297
|
-
const splitterWithDestroy = splitter as NodeJS.ReadWriteStream & {
|
|
298
|
-
destroy: (err?: Error) => void
|
|
299
|
-
}
|
|
300
|
-
const originalDestroy = splitterWithDestroy.destroy.bind(splitterWithDestroy)
|
|
301
|
-
splitterWithDestroy.destroy = function (err?: Error) {
|
|
302
|
-
void bulkSender.close()
|
|
303
|
-
originalDestroy(err)
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
return splitter
|
|
307
|
-
}
|