leo-connector-common 5.0.0 → 5.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/base.js +45 -45
- package/checksum/index.js +793 -793
- package/checksum/lib/basicConnector.js +330 -330
- package/checksum/lib/checksumNibbler.js +401 -401
- package/checksum/lib/handler.js +75 -75
- package/checksum/lib/nibbler.js +168 -168
- package/checksum/lib/test.js +205 -205
- package/checksum/tests/body-too-large.utest.js +296 -296
- package/datawarehouse/combine.js +149 -149
- package/datawarehouse/load.js +366 -366
- package/datawarehouse/transform.js +64 -64
- package/dol.js +733 -733
- package/package.json +38 -38
- package/sql/nibbler.js +164 -164
- package/sql/snapshotter.js +287 -287
- package/sql/test/test.js +97 -97
- package/test/checksum/index.test.js +10 -10
- package/utils/validation.js +127 -127
package/checksum/lib/test.js
CHANGED
|
@@ -1,206 +1,206 @@
|
|
|
1
|
-
var db1 = {};
|
|
2
|
-
var db2 = {};
|
|
3
|
-
[{
|
|
4
|
-
id: 1,
|
|
5
|
-
id2: 99,
|
|
6
|
-
name: "Clint",
|
|
7
|
-
}, {
|
|
8
|
-
id: 2,
|
|
9
|
-
id2: 673,
|
|
10
|
-
name: "Stephen",
|
|
11
|
-
}, {
|
|
12
|
-
id: 3,
|
|
13
|
-
id2: 234,
|
|
14
|
-
name: "Francis",
|
|
15
|
-
}, {
|
|
16
|
-
id: 4,
|
|
17
|
-
id2: 98,
|
|
18
|
-
name: "Joseph",
|
|
19
|
-
}, {
|
|
20
|
-
id: 5,
|
|
21
|
-
id2: 0,
|
|
22
|
-
name: "Lance",
|
|
23
|
-
}, {
|
|
24
|
-
id: 6,
|
|
25
|
-
id2: 443,
|
|
26
|
-
name: "Blaine",
|
|
27
|
-
}, {
|
|
28
|
-
id: 7,
|
|
29
|
-
id2: 2456,
|
|
30
|
-
name: "Steve",
|
|
31
|
-
}].map(r => {
|
|
32
|
-
db1[r.id] = r;
|
|
33
|
-
db2[r.id2] = r;
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
require("../").basicConnector("connector_id", {
|
|
38
|
-
id_column: "id"
|
|
39
|
-
}, {
|
|
40
|
-
// Respond to start and end
|
|
41
|
-
// Return Stream, Array, or a Hash
|
|
42
|
-
batch: function(start, end, done) {
|
|
43
|
-
console.log(start, end, this.settings);
|
|
44
|
-
let data = [];
|
|
45
|
-
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
46
|
-
for (let i = start; i <= end; i++) {
|
|
47
|
-
let v = db[i];
|
|
48
|
-
if (v !== undefined) {
|
|
49
|
-
data.push(v);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
//return Promise.resolve(data);
|
|
53
|
-
done(null, data);
|
|
54
|
-
},
|
|
55
|
-
|
|
56
|
-
// Respond to start and end
|
|
57
|
-
// Return Stream, Array, or a Hash
|
|
58
|
-
individual: function(start, end) {
|
|
59
|
-
|
|
60
|
-
console.log(start, end, this.settings);
|
|
61
|
-
let data = [];
|
|
62
|
-
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
63
|
-
for (let i = start; i <= end; i++) {
|
|
64
|
-
let v = db[i];
|
|
65
|
-
if (v !== undefined) {
|
|
66
|
-
data.push(v);
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
return Promise.resolve(data);
|
|
70
|
-
},
|
|
71
|
-
|
|
72
|
-
// Respond to ids
|
|
73
|
-
// Return Stream, Array
|
|
74
|
-
sample: function(ids) {
|
|
75
|
-
let data = [];
|
|
76
|
-
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
77
|
-
ids.map(id => {
|
|
78
|
-
let v = db[id];
|
|
79
|
-
if (v !== undefined) {
|
|
80
|
-
data.push(v);
|
|
81
|
-
console.log(v);
|
|
82
|
-
}
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
return Promise.resolve(data);
|
|
86
|
-
},
|
|
87
|
-
|
|
88
|
-
// Respond to start and end -- options
|
|
89
|
-
// Return object with min, max, total
|
|
90
|
-
range: function(start, end) {
|
|
91
|
-
console.log(start, end, this.settings);
|
|
92
|
-
let min = null;
|
|
93
|
-
let max = null;
|
|
94
|
-
let total = 0;
|
|
95
|
-
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
96
|
-
Object.keys(db).map(id => {
|
|
97
|
-
id = db[id][this.settings.id_column];
|
|
98
|
-
if ((start == undefined || id >= start) && (end == undefined || id <= end)) {
|
|
99
|
-
total++;
|
|
100
|
-
if (min == null || id < min) {
|
|
101
|
-
min = id;
|
|
102
|
-
}
|
|
103
|
-
if (max == null || id > max) {
|
|
104
|
-
max = id;
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
});
|
|
108
|
-
return Promise.resolve({
|
|
109
|
-
min,
|
|
110
|
-
max,
|
|
111
|
-
total
|
|
112
|
-
});
|
|
113
|
-
},
|
|
114
|
-
|
|
115
|
-
// Responds to start, end, limit, reverse
|
|
116
|
-
// Returns object with next, current
|
|
117
|
-
nibble: function(start, end, limit, reverse) {
|
|
118
|
-
console.log(start, end, this.settings);
|
|
119
|
-
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
120
|
-
let current = null;
|
|
121
|
-
let next = null;
|
|
122
|
-
let dir = 1;
|
|
123
|
-
let ostart = start;
|
|
124
|
-
let oend = end;
|
|
125
|
-
if (reverse) {
|
|
126
|
-
start = end;
|
|
127
|
-
end = ostart;
|
|
128
|
-
dir = -1;
|
|
129
|
-
}
|
|
130
|
-
let cnt = 0;
|
|
131
|
-
for (let i = start; i >= ostart && i <= oend; i += dir) {
|
|
132
|
-
let v = db[i];
|
|
133
|
-
if (v !== undefined) {
|
|
134
|
-
cnt++;
|
|
135
|
-
if (cnt >= limit) {
|
|
136
|
-
if (!current) {
|
|
137
|
-
current = v[this.settings.id_column];
|
|
138
|
-
} else {
|
|
139
|
-
next = v[this.settings.id_column];
|
|
140
|
-
break;
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
return Promise.resolve({
|
|
147
|
-
current,
|
|
148
|
-
next
|
|
149
|
-
});
|
|
150
|
-
},
|
|
151
|
-
|
|
152
|
-
// Called With data
|
|
153
|
-
// Returns a session
|
|
154
|
-
initialize: function(data) {
|
|
155
|
-
return Promise.resolve({});
|
|
156
|
-
},
|
|
157
|
-
|
|
158
|
-
// Called With data
|
|
159
|
-
destroy: function(data) {
|
|
160
|
-
return Promise.resolve();
|
|
161
|
-
},
|
|
162
|
-
|
|
163
|
-
// Respond to ids
|
|
164
|
-
// No return
|
|
165
|
-
delete: function(ids) {
|
|
166
|
-
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
167
|
-
ids.map(id => {
|
|
168
|
-
if (id in db) {
|
|
169
|
-
delete db[id];
|
|
170
|
-
}
|
|
171
|
-
});
|
|
172
|
-
return Promise.resolve();
|
|
173
|
-
}
|
|
174
|
-
}).delete({
|
|
175
|
-
//reverse: true,
|
|
176
|
-
limit: 3,
|
|
177
|
-
start: 0,
|
|
178
|
-
end: 100000,
|
|
179
|
-
ids: [1]
|
|
180
|
-
}).then(data => {
|
|
181
|
-
console.log("cool", data);
|
|
182
|
-
}).catch(err => {
|
|
183
|
-
console.log(err);
|
|
184
|
-
});
|
|
185
|
-
// .handler({
|
|
186
|
-
// params: {
|
|
187
|
-
// querystring: {
|
|
188
|
-
// method: "range"
|
|
189
|
-
// }
|
|
190
|
-
// },
|
|
191
|
-
// body: {
|
|
192
|
-
// session: {},
|
|
193
|
-
// settings: {
|
|
194
|
-
// id_column: "id",
|
|
195
|
-
// },
|
|
196
|
-
// data: {
|
|
197
|
-
// reverse: true,
|
|
198
|
-
// limit: 3,
|
|
199
|
-
// //start: 0,
|
|
200
|
-
// //end: 1,
|
|
201
|
-
// ids: [1, 5, 7, 99, 2456]
|
|
202
|
-
// }
|
|
203
|
-
// }
|
|
204
|
-
// }, {}, (err, data) => {
|
|
205
|
-
// console.log(err, data && data.response);
|
|
1
|
+
var db1 = {};
|
|
2
|
+
var db2 = {};
|
|
3
|
+
[{
|
|
4
|
+
id: 1,
|
|
5
|
+
id2: 99,
|
|
6
|
+
name: "Clint",
|
|
7
|
+
}, {
|
|
8
|
+
id: 2,
|
|
9
|
+
id2: 673,
|
|
10
|
+
name: "Stephen",
|
|
11
|
+
}, {
|
|
12
|
+
id: 3,
|
|
13
|
+
id2: 234,
|
|
14
|
+
name: "Francis",
|
|
15
|
+
}, {
|
|
16
|
+
id: 4,
|
|
17
|
+
id2: 98,
|
|
18
|
+
name: "Joseph",
|
|
19
|
+
}, {
|
|
20
|
+
id: 5,
|
|
21
|
+
id2: 0,
|
|
22
|
+
name: "Lance",
|
|
23
|
+
}, {
|
|
24
|
+
id: 6,
|
|
25
|
+
id2: 443,
|
|
26
|
+
name: "Blaine",
|
|
27
|
+
}, {
|
|
28
|
+
id: 7,
|
|
29
|
+
id2: 2456,
|
|
30
|
+
name: "Steve",
|
|
31
|
+
}].map(r => {
|
|
32
|
+
db1[r.id] = r;
|
|
33
|
+
db2[r.id2] = r;
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
require("../").basicConnector("connector_id", {
|
|
38
|
+
id_column: "id"
|
|
39
|
+
}, {
|
|
40
|
+
// Respond to start and end
|
|
41
|
+
// Return Stream, Array, or a Hash
|
|
42
|
+
batch: function(start, end, done) {
|
|
43
|
+
console.log(start, end, this.settings);
|
|
44
|
+
let data = [];
|
|
45
|
+
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
46
|
+
for (let i = start; i <= end; i++) {
|
|
47
|
+
let v = db[i];
|
|
48
|
+
if (v !== undefined) {
|
|
49
|
+
data.push(v);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//return Promise.resolve(data);
|
|
53
|
+
done(null, data);
|
|
54
|
+
},
|
|
55
|
+
|
|
56
|
+
// Respond to start and end
|
|
57
|
+
// Return Stream, Array, or a Hash
|
|
58
|
+
individual: function(start, end) {
|
|
59
|
+
|
|
60
|
+
console.log(start, end, this.settings);
|
|
61
|
+
let data = [];
|
|
62
|
+
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
63
|
+
for (let i = start; i <= end; i++) {
|
|
64
|
+
let v = db[i];
|
|
65
|
+
if (v !== undefined) {
|
|
66
|
+
data.push(v);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return Promise.resolve(data);
|
|
70
|
+
},
|
|
71
|
+
|
|
72
|
+
// Respond to ids
|
|
73
|
+
// Return Stream, Array
|
|
74
|
+
sample: function(ids) {
|
|
75
|
+
let data = [];
|
|
76
|
+
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
77
|
+
ids.map(id => {
|
|
78
|
+
let v = db[id];
|
|
79
|
+
if (v !== undefined) {
|
|
80
|
+
data.push(v);
|
|
81
|
+
console.log(v);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
return Promise.resolve(data);
|
|
86
|
+
},
|
|
87
|
+
|
|
88
|
+
// Respond to start and end -- options
|
|
89
|
+
// Return object with min, max, total
|
|
90
|
+
range: function(start, end) {
|
|
91
|
+
console.log(start, end, this.settings);
|
|
92
|
+
let min = null;
|
|
93
|
+
let max = null;
|
|
94
|
+
let total = 0;
|
|
95
|
+
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
96
|
+
Object.keys(db).map(id => {
|
|
97
|
+
id = db[id][this.settings.id_column];
|
|
98
|
+
if ((start == undefined || id >= start) && (end == undefined || id <= end)) {
|
|
99
|
+
total++;
|
|
100
|
+
if (min == null || id < min) {
|
|
101
|
+
min = id;
|
|
102
|
+
}
|
|
103
|
+
if (max == null || id > max) {
|
|
104
|
+
max = id;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
return Promise.resolve({
|
|
109
|
+
min,
|
|
110
|
+
max,
|
|
111
|
+
total
|
|
112
|
+
});
|
|
113
|
+
},
|
|
114
|
+
|
|
115
|
+
// Responds to start, end, limit, reverse
|
|
116
|
+
// Returns object with next, current
|
|
117
|
+
nibble: function(start, end, limit, reverse) {
|
|
118
|
+
console.log(start, end, this.settings);
|
|
119
|
+
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
120
|
+
let current = null;
|
|
121
|
+
let next = null;
|
|
122
|
+
let dir = 1;
|
|
123
|
+
let ostart = start;
|
|
124
|
+
let oend = end;
|
|
125
|
+
if (reverse) {
|
|
126
|
+
start = end;
|
|
127
|
+
end = ostart;
|
|
128
|
+
dir = -1;
|
|
129
|
+
}
|
|
130
|
+
let cnt = 0;
|
|
131
|
+
for (let i = start; i >= ostart && i <= oend; i += dir) {
|
|
132
|
+
let v = db[i];
|
|
133
|
+
if (v !== undefined) {
|
|
134
|
+
cnt++;
|
|
135
|
+
if (cnt >= limit) {
|
|
136
|
+
if (!current) {
|
|
137
|
+
current = v[this.settings.id_column];
|
|
138
|
+
} else {
|
|
139
|
+
next = v[this.settings.id_column];
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return Promise.resolve({
|
|
147
|
+
current,
|
|
148
|
+
next
|
|
149
|
+
});
|
|
150
|
+
},
|
|
151
|
+
|
|
152
|
+
// Called With data
|
|
153
|
+
// Returns a session
|
|
154
|
+
initialize: function(data) {
|
|
155
|
+
return Promise.resolve({});
|
|
156
|
+
},
|
|
157
|
+
|
|
158
|
+
// Called With data
|
|
159
|
+
destroy: function(data) {
|
|
160
|
+
return Promise.resolve();
|
|
161
|
+
},
|
|
162
|
+
|
|
163
|
+
// Respond to ids
|
|
164
|
+
// No return
|
|
165
|
+
delete: function(ids) {
|
|
166
|
+
let db = this.settings.id_column == "id2" ? db2 : db1;
|
|
167
|
+
ids.map(id => {
|
|
168
|
+
if (id in db) {
|
|
169
|
+
delete db[id];
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
return Promise.resolve();
|
|
173
|
+
}
|
|
174
|
+
}).delete({
|
|
175
|
+
//reverse: true,
|
|
176
|
+
limit: 3,
|
|
177
|
+
start: 0,
|
|
178
|
+
end: 100000,
|
|
179
|
+
ids: [1]
|
|
180
|
+
}).then(data => {
|
|
181
|
+
console.log("cool", data);
|
|
182
|
+
}).catch(err => {
|
|
183
|
+
console.log(err);
|
|
184
|
+
});
|
|
185
|
+
// .handler({
|
|
186
|
+
// params: {
|
|
187
|
+
// querystring: {
|
|
188
|
+
// method: "range"
|
|
189
|
+
// }
|
|
190
|
+
// },
|
|
191
|
+
// body: {
|
|
192
|
+
// session: {},
|
|
193
|
+
// settings: {
|
|
194
|
+
// id_column: "id",
|
|
195
|
+
// },
|
|
196
|
+
// data: {
|
|
197
|
+
// reverse: true,
|
|
198
|
+
// limit: 3,
|
|
199
|
+
// //start: 0,
|
|
200
|
+
// //end: 1,
|
|
201
|
+
// ids: [1, 5, 7, 99, 2456]
|
|
202
|
+
// }
|
|
203
|
+
// }
|
|
204
|
+
// }, {}, (err, data) => {
|
|
205
|
+
// console.log(err, data && data.response);
|
|
206
206
|
// });
|