@l-v-yonsama/multi-platform-database-drivers 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +15 -0
- package/.eslintignore +1 -0
- package/.eslintrc.json +30 -0
- package/.github/workflows/npm-publish-github-packages.yml +50 -0
- package/.prettierrc +12 -0
- package/README.md +7 -0
- package/__tests__/db/drivers/AwsS3Driver.test.ts +97 -0
- package/__tests__/db/drivers/MySQLDriver.test.ts +140 -0
- package/__tests__/db/drivers/PostgresDriver.test.ts +139 -0
- package/__tests__/db/drivers/RedisDriver.test.ts +156 -0
- package/jest.config.js +18 -0
- package/package.json +76 -0
- package/src/db/DBError.ts +26 -0
- package/src/db/drivers/AwsS3Driver.ts +539 -0
- package/src/db/drivers/BaseDriver.ts +256 -0
- package/src/db/drivers/MongoDriver.js +110 -0
- package/src/db/drivers/MySQLDriver.ts +318 -0
- package/src/db/drivers/PostgresDriver.ts +363 -0
- package/src/db/drivers/RedisDriver.ts +277 -0
- package/src/db/manager.ts +95 -0
- package/src/db/resource/DbResource.ts +579 -0
- package/src/db/resource/MultipleResultSetDataHolder.ts +119 -0
- package/src/db/resource/ResourceUtil.ts +366 -0
- package/src/db/resource/ResultSetDataHolder.ts +806 -0
- package/src/db/resource/types/DBType.ts +43 -0
- package/src/db/resource/types/GeneralColumnType.ts +321 -0
- package/src/db/resource/types/MySQLColumnType.ts +134 -0
- package/src/db/resource/types/ODBCVendorType.ts +3 -0
- package/src/db/resource/types/PostgresColumnType.ts +94 -0
- package/src/db/resource/types/RedisKeyType.ts +48 -0
- package/src/db/resource/types/ResourceType.ts +10 -0
- package/src/main.ts +1 -0
- package/src/service/request/general_db_request.ts +20 -0
- package/src/service/request/redis_request.ts +21 -0
- package/src/service/request/s3_request.ts +15 -0
- package/src/service/response/GeneralReponse.ts +10 -0
- package/src/types/DateModifiedType.ts +75 -0
- package/src/types/FileKindType.ts +80 -0
- package/src/util/file_util.ts +178 -0
- package/tsconfig.json +23 -0
- package/tsconfig.release.json +23 -0
- package/unit-test.yml +41 -0
|
@@ -0,0 +1,806 @@
|
|
|
1
|
+
import { DbResource } from './DbResource';
|
|
2
|
+
import { default as listit } from 'list-it';
|
|
3
|
+
import { GeneralColumnType } from './types/GeneralColumnType';
|
|
4
|
+
import { EnumValues } from 'enum-values';
|
|
5
|
+
import * as ss from 'simple-statistics';
|
|
6
|
+
import ShuffleArray from 'shuffle-array';
|
|
7
|
+
|
|
8
|
+
export class SampleClassPair {
|
|
9
|
+
public clazz_value: any;
|
|
10
|
+
public sample_values: any[] = [];
|
|
11
|
+
}
|
|
12
|
+
export class SampleGroupByClass {
|
|
13
|
+
public pairs: SampleClassPair[] = [];
|
|
14
|
+
public clazz_key: string;
|
|
15
|
+
public sample_keys: string[] = [];
|
|
16
|
+
public is_shuffled = false;
|
|
17
|
+
|
|
18
|
+
constructor(clazz_key: string, sample_keys: string[]) {
|
|
19
|
+
this.sample_keys = sample_keys;
|
|
20
|
+
this.clazz_key = clazz_key;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export class RdhKey {
|
|
24
|
+
public name: string;
|
|
25
|
+
public comment: string;
|
|
26
|
+
public type = GeneralColumnType.UNKNOWN;
|
|
27
|
+
public width?: number;
|
|
28
|
+
public meta?: {
|
|
29
|
+
is_image?: boolean;
|
|
30
|
+
is_hyperlink?: boolean;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
constructor(name: string, type = GeneralColumnType.UNKNOWN, comment = '') {
|
|
34
|
+
this.name = name;
|
|
35
|
+
this.type = type;
|
|
36
|
+
this.comment = comment;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
public toString(): string {
|
|
40
|
+
return `${this.name} [${EnumValues.getNameFromValue(
|
|
41
|
+
GeneralColumnType,
|
|
42
|
+
this.type,
|
|
43
|
+
)}:${GeneralColumnType.parseHandsonType(this.type)}]`;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
export enum AnnotationType {
|
|
47
|
+
Del,
|
|
48
|
+
Upd,
|
|
49
|
+
Add,
|
|
50
|
+
Err,
|
|
51
|
+
Lnt,
|
|
52
|
+
Stl,
|
|
53
|
+
}
|
|
54
|
+
export interface AnnotationOptions {
|
|
55
|
+
message?: string;
|
|
56
|
+
result?: any;
|
|
57
|
+
style?: AnnotationStyleOptions;
|
|
58
|
+
}
|
|
59
|
+
export interface AnnotationStyleOptions {
|
|
60
|
+
f?: { s: number; n: string };
|
|
61
|
+
a?: { h?: string; v?: string };
|
|
62
|
+
b?: string;
|
|
63
|
+
fmt?: string;
|
|
64
|
+
}
|
|
65
|
+
export class CellAnnotation {
|
|
66
|
+
type: AnnotationType;
|
|
67
|
+
options?: AnnotationOptions;
|
|
68
|
+
styles?: AnnotationStyleOptions;
|
|
69
|
+
constructor(type: AnnotationType, options?: AnnotationOptions) {
|
|
70
|
+
this.type = type;
|
|
71
|
+
this.options = options;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export interface MergedCell {
|
|
75
|
+
row: number;
|
|
76
|
+
col: number;
|
|
77
|
+
rowspan: number;
|
|
78
|
+
colspan: number;
|
|
79
|
+
}
|
|
80
|
+
export class RdhRow {
|
|
81
|
+
public meta: any;
|
|
82
|
+
public values: any;
|
|
83
|
+
|
|
84
|
+
constructor(meta: any, values: any) {
|
|
85
|
+
this.meta = meta;
|
|
86
|
+
this.values = values;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
pushAnnotation(
|
|
90
|
+
key: string,
|
|
91
|
+
type: AnnotationType,
|
|
92
|
+
options?: AnnotationOptions,
|
|
93
|
+
): void {
|
|
94
|
+
if (this.meta[key] === undefined) {
|
|
95
|
+
this.meta[key] = new Array<CellAnnotation>();
|
|
96
|
+
}
|
|
97
|
+
this.meta[key].push(new CellAnnotation(type, options));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
public clearAnnotations(type: AnnotationType): void {
|
|
101
|
+
if (this.meta) {
|
|
102
|
+
const meta_keys = Object.keys(this.meta);
|
|
103
|
+
if (meta_keys && meta_keys.length > 0) {
|
|
104
|
+
for (let i = 0; i < meta_keys.length; i++) {
|
|
105
|
+
const annotations: CellAnnotation[] = this.meta[meta_keys[i]];
|
|
106
|
+
if (annotations) {
|
|
107
|
+
for (let j = 0; j < annotations.length; j++) {
|
|
108
|
+
if (annotations[j].type === type) {
|
|
109
|
+
annotations.splice(j, 1);
|
|
110
|
+
j--;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
public getAnnotations(type: AnnotationType): CellAnnotation[] {
|
|
120
|
+
const r = new Array<CellAnnotation>();
|
|
121
|
+
if (this.meta) {
|
|
122
|
+
const meta_keys = Object.keys(this.meta);
|
|
123
|
+
if (meta_keys && meta_keys.length > 0) {
|
|
124
|
+
for (let i = 0; i < meta_keys.length; i++) {
|
|
125
|
+
const annotations: CellAnnotation[] = this.meta[meta_keys[i]];
|
|
126
|
+
if (annotations) {
|
|
127
|
+
annotations
|
|
128
|
+
.filter((a) => a.type === type)
|
|
129
|
+
.forEach((a) => r.push(a));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return r;
|
|
135
|
+
}
|
|
136
|
+
public getFirstAnnotationsOf(
|
|
137
|
+
key: string,
|
|
138
|
+
type: AnnotationType,
|
|
139
|
+
): CellAnnotation | undefined {
|
|
140
|
+
if (this.meta && this.meta[key]) {
|
|
141
|
+
const annotations: CellAnnotation[] = this.meta[key];
|
|
142
|
+
if (annotations) {
|
|
143
|
+
return annotations.find((a) => a.type === type);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
public getAnnotationsOf(key: string, type: AnnotationType): CellAnnotation[] {
|
|
149
|
+
const r = new Array<CellAnnotation>();
|
|
150
|
+
if (this.meta && this.meta[key]) {
|
|
151
|
+
const annotations: CellAnnotation[] = this.meta[key];
|
|
152
|
+
if (annotations) {
|
|
153
|
+
annotations.filter((a) => a.type === type).forEach((a) => r.push(a));
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return r;
|
|
157
|
+
}
|
|
158
|
+
public hasAnnotation(type: AnnotationType): boolean {
|
|
159
|
+
if (this.meta) {
|
|
160
|
+
const meta_keys = Object.keys(this.meta);
|
|
161
|
+
if (meta_keys && meta_keys.length > 0) {
|
|
162
|
+
for (let i = 0; i < meta_keys.length; i++) {
|
|
163
|
+
const annotations: CellAnnotation[] = this.meta[meta_keys[i]];
|
|
164
|
+
if (annotations) {
|
|
165
|
+
if (annotations.some((annotation) => annotation.type === type)) {
|
|
166
|
+
return true;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export default class ResultSetDataHolder {
|
|
177
|
+
static create(err?: Error): ResultSetDataHolder {
|
|
178
|
+
const rdh = new ResultSetDataHolder([]);
|
|
179
|
+
if (err) {
|
|
180
|
+
rdh.errorMessage = err.message;
|
|
181
|
+
}
|
|
182
|
+
return rdh;
|
|
183
|
+
}
|
|
184
|
+
is_empty = false;
|
|
185
|
+
keys: RdhKey[];
|
|
186
|
+
rows: Array<RdhRow>;
|
|
187
|
+
dbRes: DbResource | undefined;
|
|
188
|
+
sqlStatement: string | undefined;
|
|
189
|
+
errorMessage: string | undefined;
|
|
190
|
+
shuffledIndexes?: number[];
|
|
191
|
+
shuffledNextCounter?: number;
|
|
192
|
+
mergeCells?: MergedCell[];
|
|
193
|
+
|
|
194
|
+
constructor(keys: Array<string | RdhKey>, dbRes?: DbResource) {
|
|
195
|
+
this.keys = [];
|
|
196
|
+
this.rows = [];
|
|
197
|
+
this.setKeys(keys);
|
|
198
|
+
this.dbRes = dbRes;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
static createEmpty(): ResultSetDataHolder {
|
|
202
|
+
const rdh = new ResultSetDataHolder(['message']);
|
|
203
|
+
rdh.is_empty = true;
|
|
204
|
+
rdh.addRow({ message: 'empty result set' });
|
|
205
|
+
return rdh;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
static from(list: any, i_titles?: string | string[]): ResultSetDataHolder {
|
|
209
|
+
if (list === undefined || list === null || list === '') {
|
|
210
|
+
throw new Error(typeof list + ' has no value.');
|
|
211
|
+
}
|
|
212
|
+
const constructor = list.constructor.name;
|
|
213
|
+
if (constructor === 'ResultSetDataHolder') {
|
|
214
|
+
return list;
|
|
215
|
+
}
|
|
216
|
+
const t = typeof list;
|
|
217
|
+
let ret = ResultSetDataHolder.createEmpty();
|
|
218
|
+
// console.log('outputToSpread, list=', list, t, list.constructor.name)
|
|
219
|
+
const str_titles = new Array<string>();
|
|
220
|
+
|
|
221
|
+
if (i_titles) {
|
|
222
|
+
console.log('has i_titles', i_titles);
|
|
223
|
+
if (typeof i_titles === 'string') {
|
|
224
|
+
str_titles.push(i_titles);
|
|
225
|
+
}
|
|
226
|
+
if (i_titles instanceof Array) {
|
|
227
|
+
i_titles.forEach((s) => {
|
|
228
|
+
str_titles.push(s);
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (list instanceof Array) {
|
|
233
|
+
if (list.length > 0) {
|
|
234
|
+
let elm = list[0];
|
|
235
|
+
|
|
236
|
+
if (elm instanceof Array) {
|
|
237
|
+
// number[][]
|
|
238
|
+
let i = str_titles.length + 1;
|
|
239
|
+
while (str_titles.length < elm.length) {
|
|
240
|
+
str_titles.push(`K${i++}`);
|
|
241
|
+
}
|
|
242
|
+
ret = new ResultSetDataHolder(str_titles);
|
|
243
|
+
|
|
244
|
+
for (let r = 0; r < list.length; r++) {
|
|
245
|
+
elm = list[r];
|
|
246
|
+
const values: any = {};
|
|
247
|
+
for (let c = 0; c < elm.length; c++) {
|
|
248
|
+
values[str_titles[c]] = elm[c];
|
|
249
|
+
}
|
|
250
|
+
ret.addRow(values);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
} else {
|
|
255
|
+
// console.log('list.keys=', list.keys);
|
|
256
|
+
// console.log('list.rows=', list.rows);
|
|
257
|
+
// console.log('list.is_empty=', list.is_empty);
|
|
258
|
+
if (list.keys && list.rows && list.is_empty !== undefined) {
|
|
259
|
+
// rdh
|
|
260
|
+
const keyList = new Array<RdhKey>();
|
|
261
|
+
list.keys.forEach((k: RdhKey) => {
|
|
262
|
+
const new_key = new RdhKey(k.name, k.type, k.comment);
|
|
263
|
+
new_key.width = k.width;
|
|
264
|
+
keyList.push(new_key);
|
|
265
|
+
});
|
|
266
|
+
const rdh = new ResultSetDataHolder(keyList);
|
|
267
|
+
rdh.is_empty = list.is_empty;
|
|
268
|
+
list.rows.forEach((r: RdhRow) => {
|
|
269
|
+
rdh.rows.push(new RdhRow(r.meta, r.values));
|
|
270
|
+
});
|
|
271
|
+
rdh.dbRes = list.dbRes;
|
|
272
|
+
rdh.sqlStatement = list.sqlStatement;
|
|
273
|
+
rdh.mergeCells = list.mergeCells;
|
|
274
|
+
rdh.errorMessage = list.errorMessage;
|
|
275
|
+
return rdh;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
switch (t) {
|
|
279
|
+
case 'object':
|
|
280
|
+
if (str_titles.length !== 2) {
|
|
281
|
+
str_titles.splice(0, str_titles.length);
|
|
282
|
+
str_titles.push('KEY');
|
|
283
|
+
str_titles.push('VALUE');
|
|
284
|
+
}
|
|
285
|
+
ret = new ResultSetDataHolder(str_titles);
|
|
286
|
+
Object.keys(list).forEach((k: string) => {
|
|
287
|
+
const v = list[k];
|
|
288
|
+
const values: any = {};
|
|
289
|
+
values[str_titles[0]] = k;
|
|
290
|
+
values[str_titles[1]] = v;
|
|
291
|
+
|
|
292
|
+
ret.addRow(values);
|
|
293
|
+
});
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
ret.resetKeyTypeByRows();
|
|
298
|
+
return ret;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* The correlation is a measure of how correlated two datasets are, between -1 and 1
|
|
303
|
+
* @param key_x first input
|
|
304
|
+
* @param key_y first input
|
|
305
|
+
* @returns sample correlation
|
|
306
|
+
*/
|
|
307
|
+
sampleCorrelation(key_x: string, key_y: string): number {
|
|
308
|
+
const x: number[] = this.toVector(key_x, true);
|
|
309
|
+
const y: number[] = this.toVector(key_y, true);
|
|
310
|
+
return ss.sampleCorrelation(x, y);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
describe(): ResultSetDataHolder {
|
|
314
|
+
// # a b
|
|
315
|
+
// # count 4.000000 4.000000
|
|
316
|
+
// # mean 1.750000 0.600000
|
|
317
|
+
// # std 0.957427 0.439697
|
|
318
|
+
// # min 1.000000 0.100000
|
|
319
|
+
// # 25% 1.000000 0.325000
|
|
320
|
+
// # 50% 1.500000 0.600000
|
|
321
|
+
// # 75% 2.250000 0.875000
|
|
322
|
+
// # max 3.000000 1.100000
|
|
323
|
+
const desc_keys = new Array<RdhKey>();
|
|
324
|
+
this.keys
|
|
325
|
+
.filter((k) => GeneralColumnType.isNumericLike(k.type))
|
|
326
|
+
.forEach((k) => {
|
|
327
|
+
desc_keys.push(new RdhKey(k.name, k.type, k.comment));
|
|
328
|
+
});
|
|
329
|
+
desc_keys.unshift(new RdhKey('stat', GeneralColumnType.TEXT));
|
|
330
|
+
const ret = new ResultSetDataHolder(desc_keys);
|
|
331
|
+
|
|
332
|
+
const count_values: any = { stat: 'count' };
|
|
333
|
+
const mean_values: any = { stat: 'mean' };
|
|
334
|
+
const std_values: any = { stat: 'std' };
|
|
335
|
+
const min_values: any = { stat: 'min' };
|
|
336
|
+
const quatile25_values: any = { stat: '25%' };
|
|
337
|
+
const median_values: any = { stat: '50%' };
|
|
338
|
+
const quatile75_values: any = { stat: '75%' };
|
|
339
|
+
const max_values: any = { stat: 'max' };
|
|
340
|
+
this.keys.forEach((key) => {
|
|
341
|
+
const num_list: number[] = this.toVector(key.name, true);
|
|
342
|
+
count_values[key.name] = num_list.length;
|
|
343
|
+
mean_values[key.name] = num_list.length === 0 ? '-' : ss.mean(num_list);
|
|
344
|
+
std_values[key.name] =
|
|
345
|
+
num_list.length === 0 ? '-' : ss.standardDeviation(num_list);
|
|
346
|
+
min_values[key.name] = num_list.length === 0 ? '-' : ss.min(num_list);
|
|
347
|
+
quatile25_values[key.name] =
|
|
348
|
+
num_list.length === 0 ? '-' : ss.quantile(num_list, 0.25);
|
|
349
|
+
median_values[key.name] =
|
|
350
|
+
num_list.length === 0 ? '-' : ss.median(num_list);
|
|
351
|
+
quatile75_values[key.name] =
|
|
352
|
+
num_list.length === 0 ? '-' : ss.quantile(num_list, 0.75);
|
|
353
|
+
max_values[key.name] = num_list.length === 0 ? '-' : ss.max(num_list);
|
|
354
|
+
});
|
|
355
|
+
ret.addRow(count_values);
|
|
356
|
+
ret.addRow(mean_values);
|
|
357
|
+
ret.addRow(std_values);
|
|
358
|
+
ret.addRow(min_values);
|
|
359
|
+
ret.addRow(quatile25_values);
|
|
360
|
+
ret.addRow(median_values);
|
|
361
|
+
ret.addRow(quatile75_values);
|
|
362
|
+
ret.addRow(max_values);
|
|
363
|
+
|
|
364
|
+
return ret;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
splitRows(
|
|
368
|
+
test_percentage: number,
|
|
369
|
+
with_shuffle = false,
|
|
370
|
+
): [ResultSetDataHolder, ResultSetDataHolder] {
|
|
371
|
+
// Split the data into training and testing portions.
|
|
372
|
+
if (test_percentage >= 100) {
|
|
373
|
+
test_percentage = 100;
|
|
374
|
+
}
|
|
375
|
+
if (test_percentage < 0) {
|
|
376
|
+
test_percentage = 0;
|
|
377
|
+
}
|
|
378
|
+
const numTestExamples = Math.round(
|
|
379
|
+
(this.rows.length * test_percentage) / 100,
|
|
380
|
+
);
|
|
381
|
+
const numTrainExamples = this.rows.length - numTestExamples;
|
|
382
|
+
const train = new ResultSetDataHolder(this.keys);
|
|
383
|
+
const test = new ResultSetDataHolder(this.keys);
|
|
384
|
+
const cloned_rows = this.rows.slice();
|
|
385
|
+
if (with_shuffle) {
|
|
386
|
+
ShuffleArray(cloned_rows);
|
|
387
|
+
}
|
|
388
|
+
train.rows = cloned_rows.slice(0, numTrainExamples);
|
|
389
|
+
test.rows = cloned_rows.slice(numTrainExamples);
|
|
390
|
+
return [train, test];
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* Sample a data from each class of this resutlsets.
|
|
395
|
+
*
|
|
396
|
+
* @param {number} numExamplesPerClass Number of examples per class.
|
|
397
|
+
*/
|
|
398
|
+
sampleXKeysGroupByClass(
|
|
399
|
+
numExamplesPerClass: number,
|
|
400
|
+
xKeys: string[],
|
|
401
|
+
clazz: string,
|
|
402
|
+
shuffle = false,
|
|
403
|
+
): SampleGroupByClass {
|
|
404
|
+
const ret = new SampleGroupByClass(clazz, xKeys);
|
|
405
|
+
const localIndexes = new Array<number>();
|
|
406
|
+
for (let i = 0; i < this.rows.length; i++) {
|
|
407
|
+
localIndexes.push(i);
|
|
408
|
+
}
|
|
409
|
+
if (shuffle) {
|
|
410
|
+
ret.is_shuffled = true;
|
|
411
|
+
ShuffleArray(localIndexes);
|
|
412
|
+
}
|
|
413
|
+
const uniqClassValueSets = new Set<any>();
|
|
414
|
+
for (let j = 0; j < this.rows.length; j++) {
|
|
415
|
+
const row = this.rows[localIndexes[j]];
|
|
416
|
+
const clazz_value: any = (<any>row.values)[clazz];
|
|
417
|
+
uniqClassValueSets.add(clazz_value);
|
|
418
|
+
}
|
|
419
|
+
const uniqClassValues = Array.from(uniqClassValueSets).sort();
|
|
420
|
+
uniqClassValues.forEach((uniq) => {
|
|
421
|
+
const pair = new SampleClassPair();
|
|
422
|
+
pair.clazz_value = uniq;
|
|
423
|
+
ret.pairs.push(pair);
|
|
424
|
+
});
|
|
425
|
+
for (let j = 0; j < this.rows.length; j++) {
|
|
426
|
+
const row = this.rows[localIndexes[j]];
|
|
427
|
+
const innerX = new Array<any>();
|
|
428
|
+
xKeys.forEach((k) => {
|
|
429
|
+
innerX.push((<any>row.values)[k]);
|
|
430
|
+
});
|
|
431
|
+
const clazz_value: any = (<any>row.values)[clazz];
|
|
432
|
+
const pair = ret.pairs.find((pair) => pair.clazz_value === clazz_value);
|
|
433
|
+
if (pair && pair.sample_values.length < numExamplesPerClass) {
|
|
434
|
+
pair.sample_values.push(innerX);
|
|
435
|
+
}
|
|
436
|
+
let num_full_clazz = 0;
|
|
437
|
+
ret.pairs.forEach((pair) => {
|
|
438
|
+
if (pair.sample_values.length >= numExamplesPerClass) {
|
|
439
|
+
num_full_clazz++;
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
if (num_full_clazz >= uniqClassValues.length) {
|
|
443
|
+
break;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
return ret;
|
|
447
|
+
}
|
|
448
|
+
nextXYBatch(
|
|
449
|
+
batch_size: number,
|
|
450
|
+
xKeys: string[],
|
|
451
|
+
yKey: string,
|
|
452
|
+
): [any[][], any[]] {
|
|
453
|
+
if (this.shuffledNextCounter === undefined) {
|
|
454
|
+
this.shuffledNextCounter = 0;
|
|
455
|
+
this.shuffledIndexes = new Array<number>();
|
|
456
|
+
for (let i = 0; i < this.rows.length; i++) {
|
|
457
|
+
this.shuffledIndexes.push(i);
|
|
458
|
+
}
|
|
459
|
+
ShuffleArray(this.shuffledIndexes);
|
|
460
|
+
}
|
|
461
|
+
const x = new Array<any[]>();
|
|
462
|
+
const y = new Array<any>();
|
|
463
|
+
for (let j = 0; j < batch_size; j++) {
|
|
464
|
+
const row = this.rows[this.shuffledIndexes[this.shuffledNextCounter]];
|
|
465
|
+
const innerX = new Array<any>();
|
|
466
|
+
xKeys.forEach((k) => {
|
|
467
|
+
innerX.push((<any>row.values)[k]);
|
|
468
|
+
});
|
|
469
|
+
x.push(innerX);
|
|
470
|
+
y.push((<any>row.values)[yKey]);
|
|
471
|
+
this.shuffledNextCounter++;
|
|
472
|
+
if (this.rows.length <= this.shuffledNextCounter) {
|
|
473
|
+
ShuffleArray(this.shuffledIndexes);
|
|
474
|
+
this.shuffledNextCounter = 0;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
return [x, y];
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
hasKey(key: string): boolean {
|
|
481
|
+
return this.keys.some((k) => k.name === key);
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
drop(key: string): void {
|
|
485
|
+
if (this.hasKey(key)) {
|
|
486
|
+
this.rows.forEach((v) => {
|
|
487
|
+
delete v.values[key];
|
|
488
|
+
});
|
|
489
|
+
const idx = this.keys.findIndex((k) => k.name === key);
|
|
490
|
+
this.keys.splice(idx, 1);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
assign(key: string, list: any): void {
|
|
495
|
+
if (list === undefined || list === null || list === '') {
|
|
496
|
+
throw new Error(typeof list + ' has no value.');
|
|
497
|
+
}
|
|
498
|
+
const constructor = list.constructor.name;
|
|
499
|
+
this.drop(key);
|
|
500
|
+
if (constructor === 'Float32Array' || constructor === 'Float64Array') {
|
|
501
|
+
list = Array.from(list);
|
|
502
|
+
this.keys.push(new RdhKey(key, GeneralColumnType.NUMERIC));
|
|
503
|
+
} else if (
|
|
504
|
+
constructor === 'Int8Array' ||
|
|
505
|
+
constructor === 'Int16Array' ||
|
|
506
|
+
constructor === 'Int32Array'
|
|
507
|
+
) {
|
|
508
|
+
list = Array.from(list);
|
|
509
|
+
this.keys.push(new RdhKey(key, GeneralColumnType.INTEGER));
|
|
510
|
+
} else {
|
|
511
|
+
const types = new Set<string>();
|
|
512
|
+
list.forEach((v: any) => {
|
|
513
|
+
if (v !== '' && v !== null) {
|
|
514
|
+
types.add(typeof v);
|
|
515
|
+
}
|
|
516
|
+
});
|
|
517
|
+
if (types.size === 1 && types.has('number')) {
|
|
518
|
+
this.keys.push(new RdhKey(key, GeneralColumnType.NUMERIC));
|
|
519
|
+
} else {
|
|
520
|
+
this.keys.push(new RdhKey(key, GeneralColumnType.UNKNOWN));
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
list.forEach((v: any, i: number) => {
|
|
524
|
+
this.rows[i].values[key] = v;
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
assignFromDictionary(
|
|
529
|
+
new_key: string,
|
|
530
|
+
existing_key: string,
|
|
531
|
+
dictionary: string[],
|
|
532
|
+
): void {
|
|
533
|
+
this.drop(new_key);
|
|
534
|
+
this.keys.push(new RdhKey(new_key, GeneralColumnType.TEXT));
|
|
535
|
+
this.rows.forEach((row: any) => {
|
|
536
|
+
const existing_val = row.values[existing_key];
|
|
537
|
+
if (
|
|
538
|
+
existing_val === undefined ||
|
|
539
|
+
existing_val === '' ||
|
|
540
|
+
existing_val === null
|
|
541
|
+
) {
|
|
542
|
+
row.values[new_key] = '';
|
|
543
|
+
} else {
|
|
544
|
+
if (dictionary.length <= existing_val || existing_val < 0) {
|
|
545
|
+
row.values[new_key] = '';
|
|
546
|
+
} else {
|
|
547
|
+
row.values[new_key] = dictionary[existing_val];
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
toVector(key_name: string, is_only_number = false): Array<any> {
|
|
554
|
+
const retList = new Array<any>();
|
|
555
|
+
this.rows.forEach((row: RdhRow) => {
|
|
556
|
+
const v = (<any>row.values)[key_name];
|
|
557
|
+
if (is_only_number) {
|
|
558
|
+
if (isFinite(v) && v !== '' && v !== null) {
|
|
559
|
+
retList.push(v);
|
|
560
|
+
}
|
|
561
|
+
} else {
|
|
562
|
+
retList.push(v);
|
|
563
|
+
}
|
|
564
|
+
});
|
|
565
|
+
return retList;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
toMatrixArray(key_names?: string[]): Array<Array<any>> {
|
|
569
|
+
const retList = new Array<Array<any>>();
|
|
570
|
+
this.rows.forEach((row: RdhRow) => {
|
|
571
|
+
const retRow = new Array<any>();
|
|
572
|
+
if (key_names && key_names.length > 0) {
|
|
573
|
+
key_names.forEach((key_name: any) => {
|
|
574
|
+
retRow.push((<any>row.values)[key_name]);
|
|
575
|
+
});
|
|
576
|
+
} else {
|
|
577
|
+
this.keys.forEach((key: any) => {
|
|
578
|
+
retRow.push((<any>row.values)[key.name]);
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
retList.push(retRow);
|
|
582
|
+
});
|
|
583
|
+
return retList;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
toCsv(config?: { key_names?: string[] }): string {
|
|
587
|
+
if (config === undefined) {
|
|
588
|
+
config = {};
|
|
589
|
+
}
|
|
590
|
+
const retList = new Array<string>();
|
|
591
|
+
if (config.key_names && config.key_names.length > 0) {
|
|
592
|
+
retList.push(config.key_names.join(','));
|
|
593
|
+
} else {
|
|
594
|
+
retList.push(this.keys.map((k) => k.name).join(','));
|
|
595
|
+
}
|
|
596
|
+
this.rows.forEach((row: RdhRow) => {
|
|
597
|
+
const retRow = new Array<any>();
|
|
598
|
+
if (config.key_names && config.key_names.length > 0) {
|
|
599
|
+
config.key_names.forEach((key_name: any) => {
|
|
600
|
+
retRow.push((<any>row.values)[key_name]);
|
|
601
|
+
});
|
|
602
|
+
} else {
|
|
603
|
+
this.keys.forEach((key: any) => {
|
|
604
|
+
retRow.push((<any>row.values)[key.name]);
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
retList.push(retRow.join(','));
|
|
608
|
+
});
|
|
609
|
+
return retList.join('\r\n');
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
addRow(recordData: any, default_meta?: any): void {
|
|
613
|
+
let meta = {};
|
|
614
|
+
if (default_meta) {
|
|
615
|
+
meta = default_meta;
|
|
616
|
+
}
|
|
617
|
+
const values = <any>{};
|
|
618
|
+
this.keys.forEach((key) => {
|
|
619
|
+
values[key.name] = recordData[key.name];
|
|
620
|
+
});
|
|
621
|
+
this.rows.push(new RdhRow(meta, values));
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
clearRows(): void {
|
|
625
|
+
this.rows.splice(0, this.rows.length);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
copyFrom(that: ResultSetDataHolder): void {
|
|
629
|
+
this.clearRows();
|
|
630
|
+
this.keys.splice(0, this.keys.length);
|
|
631
|
+
this.setKeys(that.keys);
|
|
632
|
+
that.rows.forEach((v) => {
|
|
633
|
+
this.addRow(v);
|
|
634
|
+
});
|
|
635
|
+
}
|
|
636
|
+
setSqlStatement(sqlStatement: string): void {
|
|
637
|
+
this.sqlStatement = sqlStatement;
|
|
638
|
+
}
|
|
639
|
+
setDbRes(dbRes: DbResource): void {
|
|
640
|
+
this.dbRes = dbRes;
|
|
641
|
+
}
|
|
642
|
+
public hasAnnotation(type: AnnotationType): boolean {
|
|
643
|
+
let r = false;
|
|
644
|
+
for (let i = 0; i < this.rows.length; i++) {
|
|
645
|
+
if (this.rows[i].hasAnnotation(type)) {
|
|
646
|
+
r = true;
|
|
647
|
+
break;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
return r;
|
|
651
|
+
}
|
|
652
|
+
fillnull(how: 'mean' | 'median'): void {
|
|
653
|
+
this.keys
|
|
654
|
+
.filter((k) => GeneralColumnType.isNumericLike(k.type))
|
|
655
|
+
.forEach((k) => {
|
|
656
|
+
const num_list = new Array<number>();
|
|
657
|
+
for (let i = 0; i < this.rows.length; i++) {
|
|
658
|
+
const v = this.rows[i].values[k.name];
|
|
659
|
+
if (isFinite(v) && v !== '') {
|
|
660
|
+
num_list.push(v);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
let new_value = 0;
|
|
664
|
+
switch (how) {
|
|
665
|
+
case 'mean':
|
|
666
|
+
new_value = ss.mean(num_list);
|
|
667
|
+
break;
|
|
668
|
+
case 'median':
|
|
669
|
+
new_value = ss.median(num_list);
|
|
670
|
+
break;
|
|
671
|
+
}
|
|
672
|
+
for (let i = 0; i < this.rows.length; i++) {
|
|
673
|
+
if (this.rows[i].values[k.name] === null) {
|
|
674
|
+
this.rows[i].values[k.name] = new_value;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
resetKeyTypeByRows(): void {
|
|
680
|
+
this.keys.forEach((k) => {
|
|
681
|
+
const length = this.rows.length;
|
|
682
|
+
const types = new Set<string>();
|
|
683
|
+
for (let i = 0; i < length; i++) {
|
|
684
|
+
const v = this.rows[i].values[k.name];
|
|
685
|
+
if (v === '' || v === null) {
|
|
686
|
+
continue;
|
|
687
|
+
}
|
|
688
|
+
types.add(typeof v);
|
|
689
|
+
}
|
|
690
|
+
if (types.size === 1 && types.has('number')) {
|
|
691
|
+
k.type = GeneralColumnType.NUMERIC;
|
|
692
|
+
for (let i = 0; i < length; i++) {
|
|
693
|
+
if (this.rows[i].values[k.name] === '') {
|
|
694
|
+
this.rows[i].values[k.name] = null;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
setKeys(keys: Array<string | RdhKey>): void {
|
|
701
|
+
keys.forEach((k) => {
|
|
702
|
+
if (k instanceof RdhKey) {
|
|
703
|
+
this.keys.push(k);
|
|
704
|
+
} else {
|
|
705
|
+
this.keys.push(new RdhKey(k));
|
|
706
|
+
}
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
public keynames(is_only_numeric_like = false): string[] {
|
|
710
|
+
if (is_only_numeric_like) {
|
|
711
|
+
return this.keys
|
|
712
|
+
.filter((k) => GeneralColumnType.isNumericLike(k.type))
|
|
713
|
+
.map((k) => k.name);
|
|
714
|
+
}
|
|
715
|
+
return this.keys.map((k) => k.name);
|
|
716
|
+
}
|
|
717
|
+
// FieldPacket {
|
|
718
|
+
// catalog: 'def',
|
|
719
|
+
// db: 'com',
|
|
720
|
+
// table: 'nten',
|
|
721
|
+
// orgTable: 'nten',
|
|
722
|
+
// name: 'ration_tim',
|
|
723
|
+
// orgName: 'ration_tim',
|
|
724
|
+
// charsetNr: 63,
|
|
725
|
+
// length: 11,
|
|
726
|
+
// type: 3,
|
|
727
|
+
// flags: 0,
|
|
728
|
+
// decimals: 0,
|
|
729
|
+
// default: undefined,
|
|
730
|
+
// zeroFill: false,
|
|
731
|
+
// protocol41: true }
|
|
732
|
+
addKey(k: string | RdhKey): void {
|
|
733
|
+
if (this.keys === undefined) {
|
|
734
|
+
this.keys = [];
|
|
735
|
+
}
|
|
736
|
+
if (k instanceof RdhKey) {
|
|
737
|
+
this.keys.push(k);
|
|
738
|
+
} else {
|
|
739
|
+
this.keys.push(new RdhKey(k));
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
private toShortString(o: any): string {
|
|
744
|
+
const s = '' + o;
|
|
745
|
+
if (s.length > 48) {
|
|
746
|
+
return s.substring(0, 48) + '..';
|
|
747
|
+
} else {
|
|
748
|
+
return s;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
public toString(num_of_print = 8): string {
|
|
753
|
+
if (this.keys.length < 0) {
|
|
754
|
+
if (this.errorMessage) {
|
|
755
|
+
return 'No Keys. Error:' + this.errorMessage;
|
|
756
|
+
} else {
|
|
757
|
+
return 'No Keys. There is nothing error.';
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
const buf = listit.buffer();
|
|
761
|
+
buf.d('ROW');
|
|
762
|
+
this.keys.forEach((k) => {
|
|
763
|
+
buf.d(this.toShortString(k.name));
|
|
764
|
+
});
|
|
765
|
+
buf.nl();
|
|
766
|
+
buf.d(
|
|
767
|
+
EnumValues.getNameFromValue(GeneralColumnType, GeneralColumnType.INTEGER),
|
|
768
|
+
);
|
|
769
|
+
this.keys.forEach((k) => {
|
|
770
|
+
buf.d(EnumValues.getNameFromValue(GeneralColumnType, k.type));
|
|
771
|
+
});
|
|
772
|
+
buf.nl();
|
|
773
|
+
|
|
774
|
+
if (this.rows.length <= num_of_print) {
|
|
775
|
+
this.rows.forEach((v, idx) => {
|
|
776
|
+
buf.d(idx + 1);
|
|
777
|
+
this.keys.forEach((k) => {
|
|
778
|
+
buf.d(this.toShortString(v.values[k.name]));
|
|
779
|
+
});
|
|
780
|
+
buf.nl();
|
|
781
|
+
});
|
|
782
|
+
} else {
|
|
783
|
+
const num_of_head = Math.ceil(num_of_print / 2);
|
|
784
|
+
this.rows.slice(0, num_of_head).forEach((v, idx) => {
|
|
785
|
+
buf.d(idx + 1);
|
|
786
|
+
this.keys.forEach((k) => {
|
|
787
|
+
buf.d(this.toShortString(v.values[k.name]));
|
|
788
|
+
});
|
|
789
|
+
buf.nl();
|
|
790
|
+
});
|
|
791
|
+
this.rows
|
|
792
|
+
.slice(this.rows.length - num_of_head, this.rows.length)
|
|
793
|
+
.forEach((v, idx) => {
|
|
794
|
+
buf.d(this.rows.length - num_of_head + idx + 1);
|
|
795
|
+
this.keys.forEach((k) => {
|
|
796
|
+
buf.d(this.toShortString(v.values[k.name]));
|
|
797
|
+
});
|
|
798
|
+
buf.nl();
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
return (
|
|
802
|
+
`RDH NumOfRows:${this.rows.length} Error:${this.errorMessage}\n` +
|
|
803
|
+
buf.toString()
|
|
804
|
+
);
|
|
805
|
+
}
|
|
806
|
+
}
|