bellboy 4.2.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +344 -270
- package/lib/destinations/http-destination.d.ts +2 -3
- package/lib/destinations/http-destination.js +2 -2
- package/lib/processors/delimited-processor.d.ts +1 -0
- package/lib/processors/delimited-processor.js +16 -104
- package/lib/processors/excel-processor.d.ts +1 -0
- package/lib/processors/excel-processor.js +2 -0
- package/lib/processors/http-processor.d.ts +7 -3
- package/lib/processors/http-processor.js +51 -22
- package/lib/types.d.ts +10 -5
- package/lib/utils.d.ts +12 -0
- package/lib/utils.js +93 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
# bellboy [](https://travis-ci.com/Claviz/bellboy) [](https://codecov.io/gh/Claviz/bellboy) 
|
|
2
2
|
|
|
3
3
|
Highly performant JavaScript data stream ETL engine.
|
|
4
4
|
|
|
5
5
|
## How it works?
|
|
6
|
+
|
|
6
7
|
Bellboy streams input data row by row. Every row, in turn, goes through user-defined function where it can be transformed. When enough data is collected in batch, it is being loaded to destination.
|
|
7
8
|
|
|
8
9
|
## Installation
|
|
@@ -20,57 +21,57 @@ This example shows how `bellboy` can extract rows from the [Excel file](#excel-p
|
|
|
20
21
|
Just in five simple steps.
|
|
21
22
|
|
|
22
23
|
```javascript
|
|
23
|
-
const bellboy = require(
|
|
24
|
-
const fs = require(
|
|
25
|
-
const path = require(
|
|
24
|
+
const bellboy = require("bellboy");
|
|
25
|
+
const fs = require("fs");
|
|
26
|
+
const path = require("path");
|
|
26
27
|
|
|
27
28
|
(async () => {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
29
|
+
const srcPath = `C:/source`;
|
|
30
|
+
|
|
31
|
+
// 1. create a processor which will process
|
|
32
|
+
// Excel files in the folder one by one
|
|
33
|
+
const processor = new bellboy.ExcelProcessor({
|
|
34
|
+
path: srcPath,
|
|
35
|
+
hasHeader: true,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// 2. create a destination which will add a new 'status'
|
|
39
|
+
// field to each row and load processed data into a Postgres database
|
|
40
|
+
const destination = new bellboy.PostgresDestination({
|
|
41
|
+
connection: {
|
|
42
|
+
user: "user",
|
|
43
|
+
password: "password",
|
|
44
|
+
host: "localhost",
|
|
45
|
+
database: "bellboy",
|
|
46
|
+
},
|
|
47
|
+
table: "stats",
|
|
48
|
+
recordGenerator: async function* (record) {
|
|
49
|
+
yield {
|
|
50
|
+
...record.raw.obj,
|
|
51
|
+
status: "done",
|
|
52
|
+
};
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
// 3. create a job which will glue the processor and the destination together
|
|
57
|
+
const job = new bellboy.Job(processor, [destination]);
|
|
58
|
+
|
|
59
|
+
// 4. tell bellboy to move the file away as soon as it was processed
|
|
60
|
+
job.on("endProcessingStream", async (file) => {
|
|
61
|
+
const filePath = path.join(srcPath, file);
|
|
62
|
+
const newFilePath = path.join(`./destination`, file);
|
|
63
|
+
await fs.renameSync(filePath, newFilePath);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// 5. Log all error events
|
|
67
|
+
job.onAny(async (eventName, ...args) => {
|
|
68
|
+
if (eventName.includes("Error")) {
|
|
69
|
+
console.log(args);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
// 6. run your job
|
|
74
|
+
await job.run();
|
|
74
75
|
})();
|
|
75
76
|
```
|
|
76
77
|
|
|
@@ -80,124 +81,158 @@ A job in `bellboy` is a relationship link between [processor](#processors) and [
|
|
|
80
81
|
|
|
81
82
|
#### Initialization
|
|
82
83
|
|
|
83
|
-
To initialize a Job instance, pass [processor](#processors) and some [destination(s)](#destinations).
|
|
84
|
+
To initialize a Job instance, pass [processor](#processors) and some [destination(s)](#destinations).
|
|
85
|
+
|
|
84
86
|
<!-- and [options](#job-options) if needed. -->
|
|
85
87
|
|
|
86
88
|
```javascript
|
|
87
|
-
const job = new bellboy.Job(
|
|
89
|
+
const job = new bellboy.Job(
|
|
90
|
+
processor_instance,
|
|
91
|
+
[destination_instance],
|
|
92
|
+
(job_options = {})
|
|
93
|
+
);
|
|
88
94
|
```
|
|
89
95
|
|
|
90
96
|
#### Options
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
97
|
+
|
|
98
|
+
- **reporters** `Reporter[]`\
|
|
99
|
+
Array of [reporters](#reporters).
|
|
100
|
+
- **jobName** `string`\
|
|
101
|
+
Optional user-defined name of the job. Can become handy if used in combination with [extended events](#extended-event) to distinguish events from different jobs.
|
|
95
102
|
|
|
96
103
|
#### Instance methods
|
|
97
104
|
|
|
98
|
-
|
|
99
|
-
Starts processing data.
|
|
100
|
-
|
|
101
|
-
Add specific [event listener](#events).
|
|
102
|
-
|
|
103
|
-
Add [any event listener](#any-event).
|
|
104
|
-
|
|
105
|
-
Stops job execution. If `errorMessage` is passed, job will throw an error with this message.
|
|
105
|
+
- **run** `async function()`\
|
|
106
|
+
Starts processing data.
|
|
107
|
+
- **on** `function(event, async function listener)`\
|
|
108
|
+
Add specific [event listener](#events).
|
|
109
|
+
- **onAny** `function(async function listener)`\
|
|
110
|
+
Add [any event listener](#any-event).
|
|
111
|
+
- **stop** `function(errorMessage?)`\
|
|
112
|
+
Stops job execution. If `errorMessage` is passed, job will throw an error with this message.
|
|
106
113
|
|
|
107
114
|
#### Events and event listeners <div id='events'/>
|
|
108
115
|
|
|
109
116
|
Event listeners, which can be registered with `job.on` or `job.onAny` method, allow you to listen to specific events in the job lifecycle and interact with it.
|
|
110
117
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
118
|
+
- Multiple listeners for one event will be executed in the order they were registered.
|
|
119
|
+
- Job always waits code inside listener to complete.
|
|
120
|
+
- Any error thrown inside listener will be ignored and warning message will be printed out.
|
|
121
|
+
- `job.stop()` method can be used inside listener to stop job execution and throw an error if needed.
|
|
115
122
|
|
|
116
123
|
```ts
|
|
117
|
-
job.on(
|
|
124
|
+
job.on(
|
|
125
|
+
"startProcessing",
|
|
126
|
+
async (processor: IProcessor, destinations: IDestination[]) => {
|
|
118
127
|
// Job has started execution.
|
|
119
|
-
}
|
|
128
|
+
}
|
|
129
|
+
);
|
|
120
130
|
```
|
|
131
|
+
|
|
121
132
|
```ts
|
|
122
|
-
job.on(
|
|
123
|
-
|
|
124
|
-
|
|
133
|
+
job.on("startProcessingStream", async (...args: any) => {
|
|
134
|
+
// Stream processing has been started.
|
|
135
|
+
// Passed parameters may vary based on specific processor.
|
|
125
136
|
});
|
|
126
137
|
```
|
|
138
|
+
|
|
127
139
|
```ts
|
|
128
140
|
job.on('startProcessingRow', async (row: any)) => {
|
|
129
141
|
// Row has been received and is about to be processed inside `recordGenerator` method.
|
|
130
142
|
});
|
|
131
143
|
```
|
|
144
|
+
|
|
132
145
|
```ts
|
|
133
146
|
job.on('rowGenerated', async (destinationIndex: number, generatedRow: any)) => {
|
|
134
147
|
// Row has been generated using `recordGenerator` method.
|
|
135
148
|
});
|
|
136
149
|
```
|
|
150
|
+
|
|
137
151
|
```ts
|
|
138
152
|
job.on('rowGenerationError', async (destinationIndex: number, row: any, error: any)) => {
|
|
139
153
|
// Record generation (`recordGenerator` method) has thrown an error.
|
|
140
154
|
});
|
|
141
155
|
```
|
|
156
|
+
|
|
142
157
|
```ts
|
|
143
158
|
job.on('endProcessingRow', async ()) => {
|
|
144
159
|
// Row has been processed.
|
|
145
160
|
});
|
|
146
161
|
```
|
|
162
|
+
|
|
147
163
|
```ts
|
|
148
|
-
job.on(
|
|
149
|
-
|
|
164
|
+
job.on("transformingBatch", async (destinationIndex: number, rows: any[]) => {
|
|
165
|
+
// Batch is about to be transformed inside `batchTransformer` method.
|
|
150
166
|
});
|
|
151
167
|
```
|
|
168
|
+
|
|
152
169
|
```ts
|
|
153
|
-
job.on(
|
|
170
|
+
job.on(
|
|
171
|
+
"transformedBatch",
|
|
172
|
+
async (destinationIndex: number, transformedRows: any) => {
|
|
154
173
|
// Batch has been transformed using`batchTransformer` method.
|
|
155
|
-
}
|
|
174
|
+
}
|
|
175
|
+
);
|
|
156
176
|
```
|
|
177
|
+
|
|
157
178
|
```ts
|
|
158
|
-
job.on(
|
|
179
|
+
job.on(
|
|
180
|
+
"transformingBatchError",
|
|
181
|
+
async (destinationIndex: number, rows: any[], error: any) => {
|
|
159
182
|
// Batch transformation (`batchTransformer` method) has thrown an error.
|
|
160
|
-
}
|
|
183
|
+
}
|
|
184
|
+
);
|
|
161
185
|
```
|
|
186
|
+
|
|
162
187
|
```ts
|
|
163
|
-
job.on(
|
|
164
|
-
|
|
188
|
+
job.on("endTransformingBatch", async (destinationIndex: number) => {
|
|
189
|
+
// Batch has been transformed.
|
|
165
190
|
});
|
|
166
191
|
```
|
|
192
|
+
|
|
167
193
|
```ts
|
|
168
|
-
job.on(
|
|
169
|
-
|
|
194
|
+
job.on("loadingBatch", async (destinationIndex: number, data: any[]) => {
|
|
195
|
+
// Batch is about to be loaded into destination.
|
|
170
196
|
});
|
|
171
197
|
```
|
|
198
|
+
|
|
172
199
|
```ts
|
|
173
|
-
job.on(
|
|
174
|
-
|
|
200
|
+
job.on("loadedBatch", async (destinationIndex: number, data: any[]) => {
|
|
201
|
+
// Batch has been loaded into destination.
|
|
175
202
|
});
|
|
176
203
|
```
|
|
204
|
+
|
|
177
205
|
```ts
|
|
178
|
-
job.on(
|
|
206
|
+
job.on(
|
|
207
|
+
"loadingBatchError",
|
|
208
|
+
async (destinationIndex: number, data: any[], error: any) => {
|
|
179
209
|
// Batch load has failed.
|
|
180
|
-
}
|
|
210
|
+
}
|
|
211
|
+
);
|
|
181
212
|
```
|
|
213
|
+
|
|
182
214
|
```ts
|
|
183
|
-
job.on(
|
|
184
|
-
|
|
215
|
+
job.on("endLoadingBatch", async (destinationIndex: number) => {
|
|
216
|
+
// Batch load has finished .
|
|
185
217
|
});
|
|
186
218
|
```
|
|
219
|
+
|
|
187
220
|
```ts
|
|
188
|
-
job.on(
|
|
189
|
-
|
|
190
|
-
|
|
221
|
+
job.on("endProcessingStream", async (...args: any) => {
|
|
222
|
+
// Stream processing has finished.
|
|
223
|
+
// Passed parameters may vary based on specific processor.
|
|
191
224
|
});
|
|
192
225
|
```
|
|
226
|
+
|
|
193
227
|
```ts
|
|
194
|
-
job.on(
|
|
195
|
-
|
|
228
|
+
job.on("processingError", async (error: any) => {
|
|
229
|
+
// Unexpected error has occured.
|
|
196
230
|
});
|
|
197
231
|
```
|
|
232
|
+
|
|
198
233
|
```ts
|
|
199
|
-
job.on(
|
|
200
|
-
|
|
234
|
+
job.on("endProcessing", async () => {
|
|
235
|
+
// Job has finished execution.
|
|
201
236
|
});
|
|
202
237
|
```
|
|
203
238
|
|
|
@@ -207,7 +242,7 @@ Special listener can be registered using `job.onAny` method which will listen fo
|
|
|
207
242
|
|
|
208
243
|
```ts
|
|
209
244
|
job.onAny(async (eventName: string, ...args: any) => {
|
|
210
|
-
|
|
245
|
+
// An event has been fired.
|
|
211
246
|
});
|
|
212
247
|
```
|
|
213
248
|
|
|
@@ -220,63 +255,69 @@ This information can be obtained by registering an async function as a third par
|
|
|
220
255
|
For example,
|
|
221
256
|
|
|
222
257
|
```ts
|
|
223
|
-
job.on(
|
|
224
|
-
|
|
225
|
-
|
|
258
|
+
job.on("rowGenerated", undefined, async (event: IBellboyEvent) => {
|
|
259
|
+
// Row has been generated using `recordGenerator` method.
|
|
260
|
+
console.log(
|
|
261
|
+
`${event.jobName} has generated row for #${event.eventArguments.destinationIndex} destination`
|
|
262
|
+
);
|
|
226
263
|
});
|
|
227
264
|
```
|
|
265
|
+
|
|
228
266
|
or
|
|
267
|
+
|
|
229
268
|
```ts
|
|
230
269
|
job.onAny(undefined, async (event: IBellboyEvent) => {
|
|
231
|
-
|
|
270
|
+
console.log(`${event.jobName} has fired ${event.jobEvent}`);
|
|
232
271
|
});
|
|
233
272
|
```
|
|
234
273
|
|
|
235
274
|
#### Extended event (IBellboyEvent) fields
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
275
|
+
|
|
276
|
+
- **eventName** `string`\
|
|
277
|
+
Name of the event.
|
|
278
|
+
- **eventArguments** `any`\
|
|
279
|
+
Arguments of the event.
|
|
280
|
+
- **jobName** `string?`\
|
|
281
|
+
User-defined name of the job.
|
|
282
|
+
- **jobId** `string`\
|
|
283
|
+
Unique ID of the job.
|
|
284
|
+
- **eventId** `string`\
|
|
285
|
+
Unique ID of the event.
|
|
286
|
+
- **timestamp** `number`\
|
|
287
|
+
High resolution timestamp of the event.
|
|
288
|
+
- **jobStopped** `boolean`\
|
|
289
|
+
Whether the job is stopped or not.
|
|
250
290
|
|
|
251
291
|
## Processors <div id='processors'/>
|
|
252
292
|
|
|
253
293
|
Each processor in `bellboy` is a class which has a single responsibility of processing data of specific type -
|
|
254
294
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
295
|
+
- [MqttProcessor](#mqtt-processor) processes **MQTT** protocol messages.
|
|
296
|
+
- [HttpProcessor](#http-processor) processes data received from a **HTTP** call.
|
|
297
|
+
- [ExcelProcessor](#excel-processor) processes **XLSX** file data from the file system.
|
|
298
|
+
- [JsonProcessor](#json-processor) processes **JSON** file data from the file system.
|
|
299
|
+
- [DelimitedProcessor](#delimited-processor) processes files with **delimited data** from the file system.
|
|
300
|
+
- [PostgresProcessor](#database-processors) processes data received from a **PostgreSQL** SELECT.
|
|
301
|
+
- [MssqlProcessor](#database-processors) processes data received from a **MSSQL** SELECT.
|
|
302
|
+
- [DynamicProcessor](#dynamic-processor) processes **dynamically generated** data.
|
|
303
|
+
- [TailProcessor](#tail-processor) processes **new lines** added to the file.
|
|
264
304
|
|
|
265
305
|
### Options <div id='processor-options'/>
|
|
266
306
|
|
|
267
|
-
|
|
268
|
-
Number of records to be processed before stopping processor. If not specified or `0` is passed, all records will be processed.
|
|
307
|
+
- **rowLimit** `number`\
|
|
308
|
+
Number of records to be processed before stopping processor. If not specified or `0` is passed, all records will be processed.
|
|
269
309
|
|
|
270
310
|
### MqttProcessor <div id='mqtt-processor'/>
|
|
271
311
|
|
|
272
312
|
[Usage examples](tests/mqtt-source.spec.ts)
|
|
273
313
|
|
|
274
|
-
Listens for messages and processes them one by one. It also handles backpressure by queuing messages, so all messages can be eventually processed.
|
|
314
|
+
Listens for messages and processes them one by one. It also handles backpressure by queuing messages, so all messages can be eventually processed.
|
|
275
315
|
|
|
276
316
|
#### Options
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
317
|
+
|
|
318
|
+
- [Processor options](#processor-options)
|
|
319
|
+
- **url** `string` `required`
|
|
320
|
+
- **topics** `string[]` `required`
|
|
280
321
|
|
|
281
322
|
### HttpProcessor <div id='http-processor'/>
|
|
282
323
|
|
|
@@ -284,44 +325,58 @@ Listens for messages and processes them one by one. It also handles backpressure
|
|
|
284
325
|
|
|
285
326
|
Processes data received from a HTTP call. Can process `JSON` as well as `delimited` data. Can handle pagination by using `nextRequest` function.
|
|
286
327
|
|
|
328
|
+
For delimited data produces rows described [here](#delimited-produced-row).
|
|
329
|
+
|
|
287
330
|
#### Options
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
331
|
+
|
|
332
|
+
- [Processor options](#processor-options)
|
|
333
|
+
- **connection** `object` `required`\
|
|
334
|
+
Options from [axios](https://github.com/axios/axios) library.
|
|
335
|
+
- **dataFormat** `delimited | json` `required`
|
|
336
|
+
- **rowSeparator** `string` `required for delimited`
|
|
337
|
+
- **delimiter** `string` `only for delimited`\
|
|
338
|
+
A symbol separating fields of the row.
|
|
339
|
+
- **hasHeader** `boolean` `only for delimited`\
|
|
340
|
+
If `true`, first row will be processed as a header.
|
|
341
|
+
- **qualifier** `string` `only for delimited`\
|
|
342
|
+
Symbol placed around a field to signify that it is the same field.
|
|
343
|
+
- **trimQualifier** `boolean` `only for delimited`\
|
|
344
|
+
Trim qualifier from produced row item.
|
|
345
|
+
- **jsonPath** `RegExp`\
|
|
346
|
+
Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
|
|
347
|
+
- **nextRequest** `async function(header)`\
|
|
348
|
+
Function which must return `connection` for the next request or `null` if the next request is not needed.
|
|
349
|
+
|
|
297
350
|
```javascript
|
|
298
351
|
const processor = new bellboy.HttpProcessor({
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
352
|
+
nextRequest: async function () {
|
|
353
|
+
if (currentPage < pageCount) {
|
|
354
|
+
return {
|
|
355
|
+
...connection,
|
|
356
|
+
url: `${url}¤t_page=${currentPage + 1}`,
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
return null;
|
|
360
|
+
},
|
|
361
|
+
// ...
|
|
309
362
|
});
|
|
310
363
|
```
|
|
311
364
|
|
|
312
365
|
### Directory processors <div id='directory-processors'/>
|
|
366
|
+
|
|
313
367
|
Used for streaming text data from files in directory. There are currently four types of directory processors - `ExcelProcessor`, `JsonProcessor`, `DelimitedProcessor` and `TailProcessor`. Such processors search for the files in the source directory and process them one by one.
|
|
314
368
|
|
|
315
369
|
File name (`file`) and full file path (`filePath`) parameters will be passed to `startProcessingStream` event.
|
|
316
370
|
|
|
317
371
|
#### Options <div id='directory-processor-options'/>
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
372
|
+
|
|
373
|
+
- [Processor options](#processor-options)
|
|
374
|
+
- **path** `string`\
|
|
375
|
+
Path to the directory where files are located. Current directory by default.
|
|
376
|
+
- **filePattern** `RegExp`\
|
|
377
|
+
Regex pattern for the files to be processed. If not specified, all files in the directory will be matched.
|
|
378
|
+
- **files** `string[]`\
|
|
379
|
+
Array of file names. If not specified, all files in the directory will be matched against `filePattern` regex and processed in alphabetical order.
|
|
325
380
|
|
|
326
381
|
### ExcelProcessor <div id='excel-processor'/>
|
|
327
382
|
|
|
@@ -330,31 +385,34 @@ Array of file names. If not specified, all files in the directory will be matche
|
|
|
330
385
|
Processes `XLSX` files in the directory.
|
|
331
386
|
|
|
332
387
|
#### Options
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
388
|
+
|
|
389
|
+
- [Directory processor options](#directory-processor-options)
|
|
390
|
+
- **hasHeader** `boolean` | `number`\
|
|
391
|
+
Whether the worksheet has a header or not, `false` by default. 0-based row location can be passed to this option if header is not located on the first row.
|
|
392
|
+
- **fillMergedCells** `boolean`\
|
|
393
|
+
If `true`, merged cells wil have the same value (by default, only the first cell of merged cells is filled with value). \
|
|
394
|
+
**Warning!** Enabling this feature may increase streaming time because file must be processed to detect merged cells before actual stream. `false` by default.
|
|
395
|
+
- **ignoreEmpty** `boolean`\
|
|
396
|
+
Whether to ignore empty rows or not, `true` by default.
|
|
397
|
+
- **sheets** `(string | number)[] | async function(sheets)`\
|
|
398
|
+
Array of sheet names and/or sheet indexes or async function, which accepts array of all sheets and must return another array of sheet names that needs to be processed. If not specified, first sheet will be processed.
|
|
399
|
+
|
|
343
400
|
```javascript
|
|
344
401
|
const processor = new bellboy.ExcelProcessor({
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
402
|
+
// process last sheet
|
|
403
|
+
sheets: async (sheets) => {
|
|
404
|
+
const sheet = sheets[sheets.length - 1];
|
|
405
|
+
return [sheet.name];
|
|
406
|
+
},
|
|
407
|
+
// ...
|
|
351
408
|
});
|
|
352
409
|
```
|
|
410
|
+
|
|
353
411
|
<!-- * **sheetName** `string`
|
|
354
412
|
* **sheetIndex** `number`\
|
|
355
413
|
Starts from `0`.
|
|
356
414
|
* **sheetGetter** `async function(sheets)`\
|
|
357
|
-
Function which has array of `sheets` as a parameter and must return required name of the sheet.
|
|
415
|
+
Function which has array of `sheets` as a parameter and must return required name of the sheet.
|
|
358
416
|
```javascript
|
|
359
417
|
const processor = new bellboy.ExcelProcessor({
|
|
360
418
|
// returns last sheet name
|
|
@@ -367,6 +425,7 @@ const processor = new bellboy.ExcelProcessor({
|
|
|
367
425
|
If no `sheetName` specified, value of the `sheetIndex` will be used. If it isn't specified either, `sheetGetter` function will be called. If none options are specified, first sheet will be processed. -->
|
|
368
426
|
|
|
369
427
|
#### Produced row
|
|
428
|
+
|
|
370
429
|
To see how processed row will look like, proceed to [xlstream](https://github.com/Claviz/xlstream) library documentation which is used for Excel processing.
|
|
371
430
|
|
|
372
431
|
### JsonProcessor <div id='json-processor'/>
|
|
@@ -374,12 +433,13 @@ To see how processed row will look like, proceed to [xlstream](https://github.co
|
|
|
374
433
|
Processes `JSON` files in the directory.
|
|
375
434
|
|
|
376
435
|
#### Options
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
`
|
|
436
|
+
|
|
437
|
+
- [Directory processor options](#directory-processor-options)
|
|
438
|
+
- **jsonPath** `RegExp`\
|
|
439
|
+
Path to the array to be streamed. Internally when JSON is streamed, current path is joined together using `.` as separator and then tested against provided regular expression. If not specified, a root array will be streamed. As an example, if you have this JSON object:\
|
|
440
|
+
`{ "animals": { "dogs": [ "pug", "bulldog", "poodle" ] } }`\
|
|
441
|
+
And want to stream `dogs` array, path you will need to use is `/animals.dogs.(\d+)/`.\
|
|
442
|
+
`(\d+)` is used here because each index of the array is a number.
|
|
383
443
|
|
|
384
444
|
### DelimitedProcessor <div id='delimited-processor'/>
|
|
385
445
|
|
|
@@ -388,24 +448,28 @@ And want to stream `dogs` array, path you will need to use is `/animals.dogs.(\d
|
|
|
388
448
|
Processes files with delimited data in the directory.
|
|
389
449
|
|
|
390
450
|
#### Options
|
|
391
|
-
* [Directory processor options](#directory-processor-options)
|
|
392
|
-
* **rowSeparator** `string` `required`
|
|
393
|
-
* **delimiter** `string`\
|
|
394
|
-
A symbol separating fields of the row.
|
|
395
|
-
* **hasHeader** `boolean`\
|
|
396
|
-
If `true`, first row will be processed as a header.
|
|
397
|
-
* **qualifier** `string` \
|
|
398
|
-
Symbol placed around a field to signify that it is the same field.
|
|
399
451
|
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
452
|
+
- [Directory processor options](#directory-processor-options)
|
|
453
|
+
- **rowSeparator** `string` `required`
|
|
454
|
+
- **delimiter** `string`\
|
|
455
|
+
A symbol separating fields of the row.
|
|
456
|
+
- **hasHeader** `boolean`\
|
|
457
|
+
If `true`, first row will be processed as a header.
|
|
458
|
+
- **qualifier** `string` \
|
|
459
|
+
Symbol placed around a field to signify that it is the same field.
|
|
460
|
+
- **trimQualifier** `boolean` \
|
|
461
|
+
Trim qualifier from produced row item.
|
|
462
|
+
|
|
463
|
+
#### Produced row <div id='delimited-produced-row'/>
|
|
464
|
+
|
|
465
|
+
- **header** `string[]`\
|
|
466
|
+
If `hasHeader` is `true`, first row will appear here.
|
|
467
|
+
- **arr** `string`\
|
|
468
|
+
Row split by `delimiter` and `qualifier`.
|
|
469
|
+
- **obj** `string`\
|
|
470
|
+
If `hasHeader` is `true`, object with header elements as keys will appear here.
|
|
471
|
+
- **row** `string`\
|
|
472
|
+
Received raw row.
|
|
409
473
|
|
|
410
474
|
### TailProcessor <div id='tail-processor'/>
|
|
411
475
|
|
|
@@ -414,52 +478,58 @@ Received raw row.
|
|
|
414
478
|
Watches for file changes and outputs last part of file as soon as new lines are added to the file.
|
|
415
479
|
|
|
416
480
|
#### Options
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
481
|
+
|
|
482
|
+
- [Directory processor options](#directory-processor-options)
|
|
483
|
+
- **fromBeginning** `boolean`\
|
|
484
|
+
In addition to emitting new lines, emits lines from the beginning of file, `false` by default.
|
|
420
485
|
|
|
421
486
|
#### Produced row
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
487
|
+
|
|
488
|
+
- **file** `string`\
|
|
489
|
+
Name of the file the data came from.
|
|
490
|
+
- **data** `string`
|
|
425
491
|
|
|
426
492
|
### Database processors <div id='database-processors'/>
|
|
493
|
+
|
|
427
494
|
Processes `SELECT` query row by row. There are two database processors - `PostgresProcessor` ([usage examples](tests/postgres-source.spec.ts)) and `MssqlProcessor` ([usage examples](tests/mssql-source.spec.ts)). Both of them are having the same options.
|
|
428
495
|
|
|
429
496
|
#### Options
|
|
430
497
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
Query to execute.
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
498
|
+
- [Processor options](#processor-options)
|
|
499
|
+
- **query** `string` `required`\
|
|
500
|
+
Query to execute.
|
|
501
|
+
- **connection** `object` `required`
|
|
502
|
+
- **user**
|
|
503
|
+
- **password**
|
|
504
|
+
- **server**\
|
|
438
505
|
Used with `MssqlProcessor`.
|
|
439
|
-
|
|
506
|
+
- **host**
|
|
440
507
|
Used with `PostgresProcessor`.
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
508
|
+
- **port**
|
|
509
|
+
- **database**
|
|
510
|
+
- **schema**\
|
|
444
511
|
Currently available only for `PostgresProcessor`.
|
|
445
|
-
|
|
512
|
+
- **driver**\
|
|
446
513
|
Available only for `MssqlProcessor`. Defines which driver to use - `tedious` (used by default) or `msnodesqlv8`.
|
|
447
514
|
|
|
448
515
|
### DynamicProcessor <div id='dynamic-processor'/>
|
|
516
|
+
|
|
449
517
|
Processor which generates records on the fly. Can be used to define custom data processors.
|
|
450
518
|
|
|
451
519
|
#### Options
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
520
|
+
|
|
521
|
+
- [Processor options](#processor-options)
|
|
522
|
+
- **generator** `async generator function` `required`\
|
|
523
|
+
[Generator](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/function*) function which must yield records to process.
|
|
524
|
+
|
|
455
525
|
```javascript
|
|
456
526
|
// processor which generates 10 records dynamically
|
|
457
527
|
const processor = new bellboy.DynamicProcessor({
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
528
|
+
generator: async function* () {
|
|
529
|
+
for (let i = 0; i < 10; i++) {
|
|
530
|
+
yield i;
|
|
531
|
+
}
|
|
532
|
+
},
|
|
463
533
|
});
|
|
464
534
|
```
|
|
465
535
|
|
|
@@ -467,30 +537,31 @@ const processor = new bellboy.DynamicProcessor({
|
|
|
467
537
|
|
|
468
538
|
Every [job](#job) can have as many destinations (outputs) as needed. For example, one job can load processed data into a [database](#postgres-destination), log this data to [stdout](#stdout-destination) and post it by [HTTP](#http-destination) simultaneously.
|
|
469
539
|
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
540
|
+
- [StdoutDestination](#stdout-destination) logs data to **console**.
|
|
541
|
+
- [HttpDestination](#http-destination) executes **HTTP** request calls.
|
|
542
|
+
- [PostgresDestination](#postgres-destination) inserts/upserts data to **PostgreSQL** database.
|
|
543
|
+
- [MssqlDestination](#mssql-destination) inserts data to **MSSQL** database.
|
|
474
544
|
|
|
475
545
|
### Options <div id='destination-options'/>
|
|
476
546
|
|
|
477
|
-
|
|
478
|
-
If `true`, no data will be loaded to the destination. In combination with [reporters](#reporters), this option can become handy during testing process.
|
|
479
|
-
|
|
480
|
-
Number of records to be processed before loading them to the destination. If not specified or `0` is passed, all records will be processed.
|
|
481
|
-
|
|
482
|
-
Function which receives produced row by processor and can apply transformations to it.
|
|
483
|
-
|
|
484
|
-
Function which receives whole batch of rows. This function is being called after row count reaches `batchSize`. Data is being loaded to destination immediately after this function has been executed.
|
|
547
|
+
- **disableLoad** `boolean`\
|
|
548
|
+
If `true`, no data will be loaded to the destination. In combination with [reporters](#reporters), this option can become handy during testing process.
|
|
549
|
+
- **batchSize** `number`\
|
|
550
|
+
Number of records to be processed before loading them to the destination. If not specified or `0` is passed, all records will be processed.
|
|
551
|
+
- **recordGenerator** `async generator function(row)`\
|
|
552
|
+
Function which receives produced row by processor and can apply transformations to it.
|
|
553
|
+
- **batchTransformer** `async function(rows)`\
|
|
554
|
+
Function which receives whole batch of rows. This function is being called after row count reaches `batchSize`. Data is being loaded to destination immediately after this function has been executed.
|
|
485
555
|
|
|
486
556
|
### StdoutDestination <div id='stdout-destination'/>
|
|
487
557
|
|
|
488
558
|
Logs out all data to stdout (console).
|
|
489
559
|
|
|
490
560
|
#### Options
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
561
|
+
|
|
562
|
+
- [General destination options](#destination-options)
|
|
563
|
+
- **asTable** `boolean`\
|
|
564
|
+
If set to `true`, data will be printed as table.
|
|
494
565
|
|
|
495
566
|
### HttpDestination <div id='http-destination'/>
|
|
496
567
|
|
|
@@ -499,9 +570,10 @@ If set to `true`, data will be printed as table.
|
|
|
499
570
|
Puts processed data one by one in `body` and executes specified HTTP request.
|
|
500
571
|
|
|
501
572
|
#### Options
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
573
|
+
|
|
574
|
+
- [General destination options](#destination-options)
|
|
575
|
+
- **request** `required`\
|
|
576
|
+
Options from [axios](https://github.com/axios/axios) library.
|
|
505
577
|
|
|
506
578
|
### PostgresDestination <div id='postgres-destination'/>
|
|
507
579
|
|
|
@@ -510,17 +582,18 @@ Options from [request](https://github.com/request/request#requestoptions-callbac
|
|
|
510
582
|
Inserts data to PostgreSQL.
|
|
511
583
|
|
|
512
584
|
#### Options
|
|
513
|
-
|
|
514
|
-
|
|
585
|
+
|
|
586
|
+
- [General destination options](#destination-options)
|
|
587
|
+
- **table** `string` `required`\
|
|
515
588
|
Table name.
|
|
516
|
-
|
|
589
|
+
- **upsertConstraints** `string[]`\
|
|
517
590
|
If specified, `UPSERT` command will be executed based on provided constraints.
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
591
|
+
- **connection** `object` `required`
|
|
592
|
+
- **user**
|
|
593
|
+
- **password**
|
|
594
|
+
- **host**
|
|
595
|
+
- **database**
|
|
596
|
+
- **schema**
|
|
524
597
|
|
|
525
598
|
### MssqlDestination <div id='mssql-destination'/>
|
|
526
599
|
|
|
@@ -529,14 +602,15 @@ Inserts data to PostgreSQL.
|
|
|
529
602
|
Inserts data to MSSQL.
|
|
530
603
|
|
|
531
604
|
#### Options
|
|
532
|
-
|
|
533
|
-
|
|
605
|
+
|
|
606
|
+
- [General destination options](#destination-options)
|
|
607
|
+
- **table** `string` `required`\
|
|
534
608
|
Table name.
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
609
|
+
- **connection** `object` `required`
|
|
610
|
+
- **user**
|
|
611
|
+
- **password**
|
|
612
|
+
- **server**
|
|
613
|
+
- **database**
|
|
540
614
|
|
|
541
615
|
## Extendability
|
|
542
616
|
|
|
@@ -548,14 +622,14 @@ New [processors](#processors) and [destinations](#destinations) can be made by e
|
|
|
548
622
|
|
|
549
623
|
To create a new processor, you must extend `Processor` class and implement async `process` function. This function accepts one parameter:
|
|
550
624
|
|
|
551
|
-
|
|
625
|
+
- **processStream** `async function(readStream, ...args)` `required`\
|
|
552
626
|
Callback function which accepts [Readable stream](https://nodejs.org/api/stream.html#stream_class_stream_readable). After calling this function, `job` instance will handle passed stream internally. Passed parameters (`args`) will be emitted with `startProcessingStream` event during job execution.
|
|
553
627
|
|
|
554
628
|
```javascript
|
|
555
629
|
class CustomProcessor extends bellboy.Processor {
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
630
|
+
async process(processStream) {
|
|
631
|
+
// await processStream(readStream, 'hello', 'world');
|
|
632
|
+
}
|
|
559
633
|
}
|
|
560
634
|
```
|
|
561
635
|
|
|
@@ -565,14 +639,14 @@ class CustomProcessor extends bellboy.Processor {
|
|
|
565
639
|
|
|
566
640
|
To create a new destination, you must extend `Destination` class and implement async `loadBatch` function. This function accepts one parameter:
|
|
567
641
|
|
|
568
|
-
|
|
642
|
+
- **data** `any[]` `required`\
|
|
569
643
|
Array of some processed data that needs to be loaded.
|
|
570
644
|
|
|
571
645
|
```javascript
|
|
572
646
|
class CustomDestination extends bellboy.Destination {
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
647
|
+
async loadBatch(data) {
|
|
648
|
+
console.log(data);
|
|
649
|
+
}
|
|
576
650
|
}
|
|
577
651
|
```
|
|
578
652
|
|
|
@@ -582,16 +656,16 @@ class CustomDestination extends bellboy.Destination {
|
|
|
582
656
|
|
|
583
657
|
Reporter is a job wrapper which can operate with [job instance](#job) (for example, listen to events using job `on` method). To create a new reporter, you must extend `Reporter` class and implement `report` function, which will be executed during job instance initialization. This function accepts one parameter:
|
|
584
658
|
|
|
585
|
-
|
|
659
|
+
- **job** `Job` `required`\
|
|
586
660
|
[Job](#job) instance
|
|
587
661
|
|
|
588
662
|
```javascript
|
|
589
663
|
class CustomReporter extends bellboy.Reporter {
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
664
|
+
report(job) {
|
|
665
|
+
job.on("startProcessing", undefined, async ({ jobName }) => {
|
|
666
|
+
console.log(`Job ${jobName} has been started.`);
|
|
667
|
+
});
|
|
668
|
+
}
|
|
595
669
|
}
|
|
596
670
|
```
|
|
597
671
|
|