leo-connector-common 5.0.0 → 5.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,149 +1,149 @@
1
- const exec = require('child_process').exec;
2
- const fs = require("fs");
3
- const path = require("path");
4
- const merge = require("lodash/merge");
5
- const PassThrough = require("stream").PassThrough;
6
- const leo = require("leo-sdk");
7
- const ls = leo.streams;
8
- const transform = require("./transform.js");
9
- const async = require("async");
10
- const crypto = require("crypto");
11
-
12
- module.exports = function(tableIds, opts) {
13
- let streams = {};
14
- let count = 0;
15
-
16
- opts = Object.assign({
17
- dateFormat: d => d.toISOString().slice(0, 19).replace('T', ' ')
18
- });
19
- let dateFormat = opts.dateFormat;
20
-
21
- return ls.through((obj, done) => {
22
- count++;
23
- if (count % 10000 == 0) {
24
- console.log(count, obj.eid);
25
- }
26
- let payload = obj.payload;
27
- let table = transform.parseTable(payload);
28
- if (table == undefined || tableIds[table] == undefined) {
29
- return done(null);
30
- }
31
-
32
- let values = transform.parseValues(payload.data, dateFormat);
33
-
34
- let stream = streams[table];
35
- if (!stream) {
36
- let unsortedFile = `/tmp/leo_dw_${table}`;
37
- stream = streams[table] = {
38
- table: table,
39
- fields: {},
40
- unsortedFile: unsortedFile,
41
- sortedFile: unsortedFile + "_sorted",
42
- stream: fs.createWriteStream(unsortedFile)
43
- };
44
- }
45
- Object.keys(values).forEach(f => stream.fields[f] = 1);
46
- let id = crypto.createHash('md5');
47
- id.update(tableIds[table].map(f => values[f]).join(','));
48
-
49
- if (!stream.stream.write(`${id.digest('hex')}-${("00000000"+count).slice(-9)}` + JSON.stringify(values) + "\n")) {
50
- stream.stream.once('drain', () => {
51
- done(null);
52
- });
53
- } else {
54
- done(null);
55
- }
56
- }, function(done) {
57
- let tasks = [];
58
- let tables = {};
59
-
60
- Object.keys(streams).forEach((t) => {
61
- tasks.push((done) => {
62
- let table = streams[t];
63
- table.stream.end((err) => {
64
- if (err) {
65
- return done(err);
66
- }
67
- tables[t] = {
68
- table: t,
69
- fields: Object.keys(streams[t].fields),
70
- stream: combine(table.unsortedFile)
71
- };
72
- done();
73
- });
74
- });
75
- });
76
- async.parallelLimit(tasks, 4, err => {
77
- if (!err) {
78
- this.push(tables);
79
- }
80
- done(err);
81
- });
82
- });
83
- };
84
-
85
-
86
-
87
- function combine(file) {
88
- let pass = new PassThrough({
89
- objectMode: true
90
- });
91
- file = path.resolve(file);
92
- console.time("Sorted File " + file);
93
- var sortedFile = path.resolve(file + "_sorted");
94
- exec(`sort -S 1G ${file} > ${sortedFile}`, {
95
- env: {
96
- LC_ALL: 'C'
97
- }
98
- }, function(error) {
99
- if (error) {
100
- pass.emit("error", error);
101
- pass.end();
102
- return;
103
- }
104
- console.timeEnd("Sorted File " + file);
105
- fs.unlinkSync(file);
106
-
107
- var lastObj = null;
108
- var lastId = null;
109
- console.time("Merged File " + sortedFile);
110
- ls.pipe(fs.createReadStream(sortedFile), ls.split(), ls.through((line, done, push) => {
111
- try {
112
- var id = line.substr(0, 32);
113
- var data = JSON.parse(line.substr(42));
114
- } catch (e) {
115
- console.log(e);
116
- console.log(file);
117
- console.log(line.toString());
118
- process.exit();
119
- }
120
- if (lastObj && id === lastId) {
121
- if (data.__leo_delete__) {
122
- lastObj = data;
123
- } else {
124
- lastObj = merge(lastObj, data);
125
- }
126
- } else {
127
- if (lastObj) {
128
- push(lastObj);
129
- }
130
- lastObj = data;
131
- }
132
- lastId = id;
133
- done();
134
- },
135
- function(done) {
136
- if (lastObj) {
137
- done(null, lastObj);
138
- }
139
- }), pass, (err) => {
140
- if (err) {
141
- pass.emit('error', err);
142
- } else {
143
- console.timeEnd("Merged File " + sortedFile);
144
- fs.unlinkSync(sortedFile);
145
- }
146
- });
147
- });
148
- return pass;
149
- }
1
+ const exec = require('child_process').exec;
2
+ const fs = require("fs");
3
+ const path = require("path");
4
+ const merge = require("lodash/merge");
5
+ const PassThrough = require("stream").PassThrough;
6
+ const leo = require("leo-sdk");
7
+ const ls = leo.streams;
8
+ const transform = require("./transform.js");
9
+ const async = require("async");
10
+ const crypto = require("crypto");
11
+
12
+ module.exports = function(tableIds, opts) {
13
+ let streams = {};
14
+ let count = 0;
15
+
16
+ opts = Object.assign({
17
+ dateFormat: d => d.toISOString().slice(0, 19).replace('T', ' ')
18
+ });
19
+ let dateFormat = opts.dateFormat;
20
+
21
+ return ls.through((obj, done) => {
22
+ count++;
23
+ if (count % 10000 == 0) {
24
+ console.log(count, obj.eid);
25
+ }
26
+ let payload = obj.payload;
27
+ let table = transform.parseTable(payload);
28
+ if (table == undefined || tableIds[table] == undefined) {
29
+ return done(null);
30
+ }
31
+
32
+ let values = transform.parseValues(payload.data, dateFormat);
33
+
34
+ let stream = streams[table];
35
+ if (!stream) {
36
+ let unsortedFile = `/tmp/leo_dw_${table}`;
37
+ stream = streams[table] = {
38
+ table: table,
39
+ fields: {},
40
+ unsortedFile: unsortedFile,
41
+ sortedFile: unsortedFile + "_sorted",
42
+ stream: fs.createWriteStream(unsortedFile)
43
+ };
44
+ }
45
+ Object.keys(values).forEach(f => stream.fields[f] = 1);
46
+ let id = crypto.createHash('md5');
47
+ id.update(tableIds[table].map(f => values[f]).join(','));
48
+
49
+ if (!stream.stream.write(`${id.digest('hex')}-${("00000000"+count).slice(-9)}` + JSON.stringify(values) + "\n")) {
50
+ stream.stream.once('drain', () => {
51
+ done(null);
52
+ });
53
+ } else {
54
+ done(null);
55
+ }
56
+ }, function(done) {
57
+ let tasks = [];
58
+ let tables = {};
59
+
60
+ Object.keys(streams).forEach((t) => {
61
+ tasks.push((done) => {
62
+ let table = streams[t];
63
+ table.stream.end((err) => {
64
+ if (err) {
65
+ return done(err);
66
+ }
67
+ tables[t] = {
68
+ table: t,
69
+ fields: Object.keys(streams[t].fields),
70
+ stream: combine(table.unsortedFile)
71
+ };
72
+ done();
73
+ });
74
+ });
75
+ });
76
+ async.parallelLimit(tasks, 4, err => {
77
+ if (!err) {
78
+ this.push(tables);
79
+ }
80
+ done(err);
81
+ });
82
+ });
83
+ };
84
+
85
+
86
+
87
+ function combine(file) {
88
+ let pass = new PassThrough({
89
+ objectMode: true
90
+ });
91
+ file = path.resolve(file);
92
+ console.time("Sorted File " + file);
93
+ var sortedFile = path.resolve(file + "_sorted");
94
+ exec(`sort -S 1G ${file} > ${sortedFile}`, {
95
+ env: {
96
+ LC_ALL: 'C'
97
+ }
98
+ }, function(error) {
99
+ if (error) {
100
+ pass.emit("error", error);
101
+ pass.end();
102
+ return;
103
+ }
104
+ console.timeEnd("Sorted File " + file);
105
+ fs.unlinkSync(file);
106
+
107
+ var lastObj = null;
108
+ var lastId = null;
109
+ console.time("Merged File " + sortedFile);
110
+ ls.pipe(fs.createReadStream(sortedFile), ls.split(), ls.through((line, done, push) => {
111
+ try {
112
+ var id = line.substr(0, 32);
113
+ var data = JSON.parse(line.substr(42));
114
+ } catch (e) {
115
+ console.log(e);
116
+ console.log(file);
117
+ console.log(line.toString());
118
+ process.exit();
119
+ }
120
+ if (lastObj && id === lastId) {
121
+ if (data.__leo_delete__) {
122
+ lastObj = data;
123
+ } else {
124
+ lastObj = merge(lastObj, data);
125
+ }
126
+ } else {
127
+ if (lastObj) {
128
+ push(lastObj);
129
+ }
130
+ lastObj = data;
131
+ }
132
+ lastId = id;
133
+ done();
134
+ },
135
+ function(done) {
136
+ if (lastObj) {
137
+ done(null, lastObj);
138
+ }
139
+ }), pass, (err) => {
140
+ if (err) {
141
+ pass.emit('error', err);
142
+ } else {
143
+ console.timeEnd("Merged File " + sortedFile);
144
+ fs.unlinkSync(sortedFile);
145
+ }
146
+ });
147
+ });
148
+ return pass;
149
+ }