leo-connector-common 5.0.0 → 5.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/base.js +45 -45
- package/checksum/index.js +793 -793
- package/checksum/lib/basicConnector.js +330 -330
- package/checksum/lib/checksumNibbler.js +401 -401
- package/checksum/lib/handler.js +75 -75
- package/checksum/lib/nibbler.js +168 -168
- package/checksum/lib/test.js +205 -205
- package/checksum/tests/body-too-large.utest.js +296 -296
- package/datawarehouse/combine.js +149 -149
- package/datawarehouse/load.js +366 -366
- package/datawarehouse/transform.js +64 -64
- package/dol.js +733 -733
- package/package.json +38 -38
- package/sql/nibbler.js +164 -164
- package/sql/snapshotter.js +287 -287
- package/sql/test/test.js +97 -97
- package/test/checksum/index.test.js +10 -10
- package/utils/validation.js +127 -127
package/checksum/lib/handler.js
CHANGED
|
@@ -1,76 +1,76 @@
|
|
|
1
|
-
module.exports = function(checksumlib) {
|
|
2
|
-
return {
|
|
3
|
-
handler: function(event, context, callback) {
|
|
4
|
-
context.callbackWaitsForEmptyEventLoop = false;
|
|
5
|
-
var method = event.params.querystring.method;
|
|
6
|
-
|
|
7
|
-
if (method == "batch") {
|
|
8
|
-
delete event.body.checksum;
|
|
9
|
-
checksumlib.batch(event.body, (err, data) => {
|
|
10
|
-
callback(err, {
|
|
11
|
-
response: data,
|
|
12
|
-
session: event.body.session
|
|
13
|
-
});
|
|
14
|
-
});
|
|
15
|
-
} else if (method == "individual") {
|
|
16
|
-
delete event.body.individual;
|
|
17
|
-
checksumlib.individual(event.body, (err, data) => {
|
|
18
|
-
callback(err, {
|
|
19
|
-
response: data,
|
|
20
|
-
session: event.body.session
|
|
21
|
-
});
|
|
22
|
-
});
|
|
23
|
-
} else if (method == "delete") {
|
|
24
|
-
delete event.body.delete;
|
|
25
|
-
checksumlib.delete(event.body, (err, data) => {
|
|
26
|
-
callback(err, {
|
|
27
|
-
response: data,
|
|
28
|
-
session: event.body.session
|
|
29
|
-
});
|
|
30
|
-
});
|
|
31
|
-
} else if (method == "sample") {
|
|
32
|
-
delete event.body.sample;
|
|
33
|
-
checksumlib.sample(event.body, (err, data) => {
|
|
34
|
-
callback(err, {
|
|
35
|
-
response: data,
|
|
36
|
-
session: event.body.session
|
|
37
|
-
});
|
|
38
|
-
});
|
|
39
|
-
} else if (method == "nibble") {
|
|
40
|
-
delete event.body.nibble;
|
|
41
|
-
checksumlib.nibble(event.body, (err, data) => {
|
|
42
|
-
callback(err, {
|
|
43
|
-
response: data,
|
|
44
|
-
session: event.body.session
|
|
45
|
-
});
|
|
46
|
-
});
|
|
47
|
-
} else if (method == "range") {
|
|
48
|
-
delete event.body.range;
|
|
49
|
-
checksumlib.range(event.body, (err, data) => {
|
|
50
|
-
callback(err, {
|
|
51
|
-
response: data,
|
|
52
|
-
session: event.body.session
|
|
53
|
-
});
|
|
54
|
-
});
|
|
55
|
-
} else if (method == "initialize") {
|
|
56
|
-
delete event.body.initialize;
|
|
57
|
-
checksumlib.initialize(event.body, (err, data) => {
|
|
58
|
-
callback(err, {
|
|
59
|
-
response: data,
|
|
60
|
-
session: event.body.session
|
|
61
|
-
});
|
|
62
|
-
});
|
|
63
|
-
} else if (method == "destroy") {
|
|
64
|
-
delete event.body.destroy;
|
|
65
|
-
checksumlib.destroy(event.body, (err, data) => {
|
|
66
|
-
callback(err, {
|
|
67
|
-
response: data,
|
|
68
|
-
session: event.body.session
|
|
69
|
-
});
|
|
70
|
-
});
|
|
71
|
-
} else {
|
|
72
|
-
callback("Invalid Method: " + method);
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
};
|
|
1
|
+
module.exports = function(checksumlib) {
|
|
2
|
+
return {
|
|
3
|
+
handler: function(event, context, callback) {
|
|
4
|
+
context.callbackWaitsForEmptyEventLoop = false;
|
|
5
|
+
var method = event.params.querystring.method;
|
|
6
|
+
|
|
7
|
+
if (method == "batch") {
|
|
8
|
+
delete event.body.checksum;
|
|
9
|
+
checksumlib.batch(event.body, (err, data) => {
|
|
10
|
+
callback(err, {
|
|
11
|
+
response: data,
|
|
12
|
+
session: event.body.session
|
|
13
|
+
});
|
|
14
|
+
});
|
|
15
|
+
} else if (method == "individual") {
|
|
16
|
+
delete event.body.individual;
|
|
17
|
+
checksumlib.individual(event.body, (err, data) => {
|
|
18
|
+
callback(err, {
|
|
19
|
+
response: data,
|
|
20
|
+
session: event.body.session
|
|
21
|
+
});
|
|
22
|
+
});
|
|
23
|
+
} else if (method == "delete") {
|
|
24
|
+
delete event.body.delete;
|
|
25
|
+
checksumlib.delete(event.body, (err, data) => {
|
|
26
|
+
callback(err, {
|
|
27
|
+
response: data,
|
|
28
|
+
session: event.body.session
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
} else if (method == "sample") {
|
|
32
|
+
delete event.body.sample;
|
|
33
|
+
checksumlib.sample(event.body, (err, data) => {
|
|
34
|
+
callback(err, {
|
|
35
|
+
response: data,
|
|
36
|
+
session: event.body.session
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
} else if (method == "nibble") {
|
|
40
|
+
delete event.body.nibble;
|
|
41
|
+
checksumlib.nibble(event.body, (err, data) => {
|
|
42
|
+
callback(err, {
|
|
43
|
+
response: data,
|
|
44
|
+
session: event.body.session
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
} else if (method == "range") {
|
|
48
|
+
delete event.body.range;
|
|
49
|
+
checksumlib.range(event.body, (err, data) => {
|
|
50
|
+
callback(err, {
|
|
51
|
+
response: data,
|
|
52
|
+
session: event.body.session
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
} else if (method == "initialize") {
|
|
56
|
+
delete event.body.initialize;
|
|
57
|
+
checksumlib.initialize(event.body, (err, data) => {
|
|
58
|
+
callback(err, {
|
|
59
|
+
response: data,
|
|
60
|
+
session: event.body.session
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
} else if (method == "destroy") {
|
|
64
|
+
delete event.body.destroy;
|
|
65
|
+
checksumlib.destroy(event.body, (err, data) => {
|
|
66
|
+
callback(err, {
|
|
67
|
+
response: data,
|
|
68
|
+
session: event.body.session
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
} else {
|
|
72
|
+
callback("Invalid Method: " + method);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
76
|
};
|
package/checksum/lib/nibbler.js
CHANGED
|
@@ -1,169 +1,169 @@
|
|
|
1
|
-
const async = require("async");
|
|
2
|
-
const logger = require("leo-logger")("leo-nibbler");
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Interface
|
|
6
|
-
* nibbler.sync({
|
|
7
|
-
* onInit: function(nibble) { }, // setup stuff for sync; Async is not allowed
|
|
8
|
-
* whilst: function(nibble) { return true; }, // return true to continue; Async is not allowed
|
|
9
|
-
* onBite: function(nibble, done) { done(); }, // do something with the nibble data
|
|
10
|
-
* onError: function(err, data, nibble, done) { nibble.limit = nibble.limit / 2; done(); }, // Fix the error by addressing nibble settings or nibble.move() or settings to end
|
|
11
|
-
* onEnd: function(err, nibble, done) { done(); }
|
|
12
|
-
*
|
|
13
|
-
* }, callback);
|
|
14
|
-
*/
|
|
15
|
-
|
|
16
|
-
module.exports = function(connector, opts) {
|
|
17
|
-
var nibble = {};
|
|
18
|
-
|
|
19
|
-
var logTimeout = null;
|
|
20
|
-
//@todo: Update all this to use the log-update node module
|
|
21
|
-
function clearLog() {
|
|
22
|
-
if (opts.showOutput) {
|
|
23
|
-
process.stdout.write("\r\x1b[K");
|
|
24
|
-
}
|
|
25
|
-
if (logTimeout) clearInterval(logTimeout);
|
|
26
|
-
}
|
|
27
|
-
var log = function() {
|
|
28
|
-
clearLog();
|
|
29
|
-
var percent = (nibble.progress / nibble.total) * 100;
|
|
30
|
-
var fixed = percent.toFixed(2);
|
|
31
|
-
|
|
32
|
-
if (fixed == "100.00" && percent < 100) {
|
|
33
|
-
fixed = "99.99";
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
logger.log(fixed + "% :", Object.keys(arguments).map(k => arguments[k]).join(", "));
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function timeLog(message) {
|
|
40
|
-
clearLog();
|
|
41
|
-
var time = new Date();
|
|
42
|
-
|
|
43
|
-
function writeMessage() {
|
|
44
|
-
if (opts.showOutput) {
|
|
45
|
-
process.stdout.write("\r\x1b[K");
|
|
46
|
-
process.stdout.write(((new Date() - time) / 1000).toFixed(1) + "s : " + message);
|
|
47
|
-
} else {
|
|
48
|
-
logger.log(message);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
writeMessage();
|
|
52
|
-
|
|
53
|
-
if (opts.showOutput) {
|
|
54
|
-
logTimeout = setInterval(writeMessage, 200);
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
function normalLog(message) {
|
|
59
|
-
clearLog();
|
|
60
|
-
logger.log(message);
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
return {
|
|
64
|
-
log: log,
|
|
65
|
-
timeLog: timeLog,
|
|
66
|
-
normalLog: normalLog,
|
|
67
|
-
sync: function(opts, callback) {
|
|
68
|
-
if (typeof opts == "function") {
|
|
69
|
-
callback = opts;
|
|
70
|
-
opts = {};
|
|
71
|
-
}
|
|
72
|
-
var opts = Object.assign({
|
|
73
|
-
time: 1,
|
|
74
|
-
limit: 9000,
|
|
75
|
-
maxLimit: 1000000,
|
|
76
|
-
sample: false,
|
|
77
|
-
reverse: false,
|
|
78
|
-
}, opts || {});
|
|
79
|
-
|
|
80
|
-
var cb = callback;
|
|
81
|
-
callback = (err, data) => {
|
|
82
|
-
clearLog();
|
|
83
|
-
cb(err, data);
|
|
84
|
-
}
|
|
85
|
-
connector.range(opts).then((range) => {
|
|
86
|
-
// logger.log("nibbler.sync", JSON.stringify(opts, null, 2))
|
|
87
|
-
//Now let's nibble our way through it.
|
|
88
|
-
nibble = {
|
|
89
|
-
start: opts.start && opts.start > range.min ? opts.start : range.min,
|
|
90
|
-
end: opts.end && opts.end < range.max ? opts.end : range.max,
|
|
91
|
-
limit: opts.limit,
|
|
92
|
-
next: null,
|
|
93
|
-
min: range.min,
|
|
94
|
-
max: range.max,
|
|
95
|
-
total: range.total,
|
|
96
|
-
progress: 0,
|
|
97
|
-
reverse: opts.reverse,
|
|
98
|
-
hadRecentErrors: 0,
|
|
99
|
-
move: function() {
|
|
100
|
-
if (!this.reverse)
|
|
101
|
-
this.start = this.next;
|
|
102
|
-
else
|
|
103
|
-
this.end = this.next;
|
|
104
|
-
}
|
|
105
|
-
};
|
|
106
|
-
|
|
107
|
-
if (opts.onInit) {
|
|
108
|
-
opts.onInit(nibble);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
log(`Starting. Total: ${nibble.total}, Min: ${nibble.min}, Max: ${nibble.max}, Start: ${nibble.start}, End: ${nibble.end}`);
|
|
112
|
-
//var hadRecentErrors = 0;
|
|
113
|
-
async.doWhilst((done) => {
|
|
114
|
-
timeLog("Nibbling the next set of Data from Locale");
|
|
115
|
-
var forward = !nibble.reverse;
|
|
116
|
-
nibble.start = forward ? nibble.start : nibble.min;
|
|
117
|
-
nibble.end = forward ? nibble.max : nibble.end;
|
|
118
|
-
|
|
119
|
-
connector.nibble(nibble).then((n) => {
|
|
120
|
-
if (n.current != undefined) {
|
|
121
|
-
if (!nibble.reverse) {
|
|
122
|
-
n.end = n.current ? n.current : nibble.max;
|
|
123
|
-
} else {
|
|
124
|
-
n.start = n.current ? n.current : nibble.min;
|
|
125
|
-
}
|
|
126
|
-
delete n.current;
|
|
127
|
-
}
|
|
128
|
-
Object.assign(nibble, n);
|
|
129
|
-
|
|
130
|
-
opts.onBite(nibble, (err, result) => {
|
|
131
|
-
if (err) {
|
|
132
|
-
opts.onError(err, result, nibble, done);
|
|
133
|
-
nibble.hadRecentErrors = opts.errorAllowance || 1;
|
|
134
|
-
} else {
|
|
135
|
-
nibble.move();
|
|
136
|
-
|
|
137
|
-
//we had no errors this time nor last time, so lets up the limit
|
|
138
|
-
if (nibble.hadRecentErrors) {
|
|
139
|
-
nibble.hadRecentErrors--;
|
|
140
|
-
} else {
|
|
141
|
-
nibble.limit = Math.min(
|
|
142
|
-
nibble.limit * 5,
|
|
143
|
-
opts.maxLimit,
|
|
144
|
-
(result && result.duration) ? (Math.round((1000 / Math.max(result.duration)) * nibble.limit)) : opts.maxLimit
|
|
145
|
-
);
|
|
146
|
-
}
|
|
147
|
-
done(null, result);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
});
|
|
151
|
-
}, callback);
|
|
152
|
-
},
|
|
153
|
-
function() {
|
|
154
|
-
return (!nibble.reverse ? nibble.start != null : nibble.end != null) && (!opts.whilst || opts.whilst(nibble));
|
|
155
|
-
},
|
|
156
|
-
function(err, data) {
|
|
157
|
-
//clearLog();
|
|
158
|
-
if (opts.onEnd) {
|
|
159
|
-
opts.onEnd(err, nibble, function(err2, data) {
|
|
160
|
-
callback(err || err2, data);
|
|
161
|
-
});
|
|
162
|
-
} else {
|
|
163
|
-
callback(err);
|
|
164
|
-
}
|
|
165
|
-
});
|
|
166
|
-
}, callback).catch(callback);
|
|
167
|
-
}
|
|
168
|
-
};
|
|
1
|
+
const async = require("async");
|
|
2
|
+
const logger = require("leo-logger")("leo-nibbler");
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Interface
|
|
6
|
+
* nibbler.sync({
|
|
7
|
+
* onInit: function(nibble) { }, // setup stuff for sync; Async is not allowed
|
|
8
|
+
* whilst: function(nibble) { return true; }, // return true to continue; Async is not allowed
|
|
9
|
+
* onBite: function(nibble, done) { done(); }, // do something with the nibble data
|
|
10
|
+
* onError: function(err, data, nibble, done) { nibble.limit = nibble.limit / 2; done(); }, // Fix the error by addressing nibble settings or nibble.move() or settings to end
|
|
11
|
+
* onEnd: function(err, nibble, done) { done(); }
|
|
12
|
+
*
|
|
13
|
+
* }, callback);
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
module.exports = function(connector, opts) {
|
|
17
|
+
var nibble = {};
|
|
18
|
+
|
|
19
|
+
var logTimeout = null;
|
|
20
|
+
//@todo: Update all this to use the log-update node module
|
|
21
|
+
function clearLog() {
|
|
22
|
+
if (opts.showOutput) {
|
|
23
|
+
process.stdout.write("\r\x1b[K");
|
|
24
|
+
}
|
|
25
|
+
if (logTimeout) clearInterval(logTimeout);
|
|
26
|
+
}
|
|
27
|
+
var log = function() {
|
|
28
|
+
clearLog();
|
|
29
|
+
var percent = (nibble.progress / nibble.total) * 100;
|
|
30
|
+
var fixed = percent.toFixed(2);
|
|
31
|
+
|
|
32
|
+
if (fixed == "100.00" && percent < 100) {
|
|
33
|
+
fixed = "99.99";
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
logger.log(fixed + "% :", Object.keys(arguments).map(k => arguments[k]).join(", "));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function timeLog(message) {
|
|
40
|
+
clearLog();
|
|
41
|
+
var time = new Date();
|
|
42
|
+
|
|
43
|
+
function writeMessage() {
|
|
44
|
+
if (opts.showOutput) {
|
|
45
|
+
process.stdout.write("\r\x1b[K");
|
|
46
|
+
process.stdout.write(((new Date() - time) / 1000).toFixed(1) + "s : " + message);
|
|
47
|
+
} else {
|
|
48
|
+
logger.log(message);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
writeMessage();
|
|
52
|
+
|
|
53
|
+
if (opts.showOutput) {
|
|
54
|
+
logTimeout = setInterval(writeMessage, 200);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function normalLog(message) {
|
|
59
|
+
clearLog();
|
|
60
|
+
logger.log(message);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return {
|
|
64
|
+
log: log,
|
|
65
|
+
timeLog: timeLog,
|
|
66
|
+
normalLog: normalLog,
|
|
67
|
+
sync: function(opts, callback) {
|
|
68
|
+
if (typeof opts == "function") {
|
|
69
|
+
callback = opts;
|
|
70
|
+
opts = {};
|
|
71
|
+
}
|
|
72
|
+
var opts = Object.assign({
|
|
73
|
+
time: 1,
|
|
74
|
+
limit: 9000,
|
|
75
|
+
maxLimit: 1000000,
|
|
76
|
+
sample: false,
|
|
77
|
+
reverse: false,
|
|
78
|
+
}, opts || {});
|
|
79
|
+
|
|
80
|
+
var cb = callback;
|
|
81
|
+
callback = (err, data) => {
|
|
82
|
+
clearLog();
|
|
83
|
+
cb(err, data);
|
|
84
|
+
}
|
|
85
|
+
connector.range(opts).then((range) => {
|
|
86
|
+
// logger.log("nibbler.sync", JSON.stringify(opts, null, 2))
|
|
87
|
+
//Now let's nibble our way through it.
|
|
88
|
+
nibble = {
|
|
89
|
+
start: opts.start && opts.start > range.min ? opts.start : range.min,
|
|
90
|
+
end: opts.end && opts.end < range.max ? opts.end : range.max,
|
|
91
|
+
limit: opts.limit,
|
|
92
|
+
next: null,
|
|
93
|
+
min: range.min,
|
|
94
|
+
max: range.max,
|
|
95
|
+
total: range.total,
|
|
96
|
+
progress: 0,
|
|
97
|
+
reverse: opts.reverse,
|
|
98
|
+
hadRecentErrors: 0,
|
|
99
|
+
move: function() {
|
|
100
|
+
if (!this.reverse)
|
|
101
|
+
this.start = this.next;
|
|
102
|
+
else
|
|
103
|
+
this.end = this.next;
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
if (opts.onInit) {
|
|
108
|
+
opts.onInit(nibble);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
log(`Starting. Total: ${nibble.total}, Min: ${nibble.min}, Max: ${nibble.max}, Start: ${nibble.start}, End: ${nibble.end}`);
|
|
112
|
+
//var hadRecentErrors = 0;
|
|
113
|
+
async.doWhilst((done) => {
|
|
114
|
+
timeLog("Nibbling the next set of Data from Locale");
|
|
115
|
+
var forward = !nibble.reverse;
|
|
116
|
+
nibble.start = forward ? nibble.start : nibble.min;
|
|
117
|
+
nibble.end = forward ? nibble.max : nibble.end;
|
|
118
|
+
|
|
119
|
+
connector.nibble(nibble).then((n) => {
|
|
120
|
+
if (n.current != undefined) {
|
|
121
|
+
if (!nibble.reverse) {
|
|
122
|
+
n.end = n.current ? n.current : nibble.max;
|
|
123
|
+
} else {
|
|
124
|
+
n.start = n.current ? n.current : nibble.min;
|
|
125
|
+
}
|
|
126
|
+
delete n.current;
|
|
127
|
+
}
|
|
128
|
+
Object.assign(nibble, n);
|
|
129
|
+
|
|
130
|
+
opts.onBite(nibble, (err, result) => {
|
|
131
|
+
if (err) {
|
|
132
|
+
opts.onError(err, result, nibble, done);
|
|
133
|
+
nibble.hadRecentErrors = opts.errorAllowance || 1;
|
|
134
|
+
} else {
|
|
135
|
+
nibble.move();
|
|
136
|
+
|
|
137
|
+
//we had no errors this time nor last time, so lets up the limit
|
|
138
|
+
if (nibble.hadRecentErrors) {
|
|
139
|
+
nibble.hadRecentErrors--;
|
|
140
|
+
} else {
|
|
141
|
+
nibble.limit = Math.min(
|
|
142
|
+
nibble.limit * 5,
|
|
143
|
+
opts.maxLimit,
|
|
144
|
+
(result && result.duration) ? (Math.round((1000 / Math.max(result.duration)) * nibble.limit)) : opts.maxLimit
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
done(null, result);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
});
|
|
151
|
+
}, callback);
|
|
152
|
+
},
|
|
153
|
+
function() {
|
|
154
|
+
return (!nibble.reverse ? nibble.start != null : nibble.end != null) && (!opts.whilst || opts.whilst(nibble));
|
|
155
|
+
},
|
|
156
|
+
function(err, data) {
|
|
157
|
+
//clearLog();
|
|
158
|
+
if (opts.onEnd) {
|
|
159
|
+
opts.onEnd(err, nibble, function(err2, data) {
|
|
160
|
+
callback(err || err2, data);
|
|
161
|
+
});
|
|
162
|
+
} else {
|
|
163
|
+
callback(err);
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
}, callback).catch(callback);
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
169
|
};
|