@naturalcycles/db-lib 8.12.0 → 8.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/adapter/cachedb/cache.db.d.ts +3 -3
- package/dist/adapter/cachedb/cache.db.js +3 -3
- package/dist/adapter/file/file.db.d.ts +2 -2
- package/dist/adapter/file/file.db.js +4 -2
- package/dist/adapter/inmemory/inMemory.db.d.ts +3 -4
- package/dist/adapter/inmemory/inMemory.db.js +6 -4
- package/dist/base.common.db.d.ts +3 -3
- package/dist/base.common.db.js +9 -3
- package/dist/common.db.d.ts +3 -3
- package/dist/commondao/common.dao.d.ts +3 -4
- package/dist/commondao/common.dao.js +1 -1
- package/dist/index.d.ts +2 -4
- package/dist/index.js +1 -5
- package/dist/pipeline/dbPipelineBackup.d.ts +0 -1
- package/dist/pipeline/dbPipelineBackup.js +2 -13
- package/dist/pipeline/dbPipelineRestore.js +1 -1
- package/dist/testing/dbTest.js +1 -1
- package/dist/testing/test.model.d.ts +2 -2
- package/dist/testing/test.model.js +13 -2
- package/package.json +1 -1
- package/readme.md +5 -4
- package/src/adapter/cachedb/cache.db.ts +9 -5
- package/src/adapter/file/file.db.ts +7 -4
- package/src/adapter/inmemory/inMemory.db.ts +20 -7
- package/src/base.common.db.ts +10 -4
- package/src/common.db.ts +3 -3
- package/src/commondao/common.dao.ts +4 -4
- package/src/index.ts +0 -7
- package/src/pipeline/dbPipelineBackup.ts +2 -15
- package/src/pipeline/dbPipelineRestore.ts +1 -1
- package/src/testing/dbTest.ts +1 -1
- package/src/testing/test.model.ts +16 -4
- package/dist/schema/common.schema.d.ts +0 -38
- package/dist/schema/common.schema.js +0 -18
- package/dist/schema/commonSchemaGenerator.d.ts +0 -35
- package/dist/schema/commonSchemaGenerator.js +0 -151
- package/src/schema/common.schema.ts +0 -49
- package/src/schema/commonSchemaGenerator.ts +0 -200
|
@@ -2,6 +2,7 @@ import {
|
|
|
2
2
|
AppError,
|
|
3
3
|
AsyncMapper,
|
|
4
4
|
ErrorMode,
|
|
5
|
+
JsonSchemaObject,
|
|
5
6
|
_filterNullishValues,
|
|
6
7
|
_passthroughPredicate,
|
|
7
8
|
_since,
|
|
@@ -27,7 +28,6 @@ import {
|
|
|
27
28
|
import { DBLibError } from '../cnst'
|
|
28
29
|
import { DBModelType, ObjectWithId, RunQueryResult, Saved } from '../db.model'
|
|
29
30
|
import { DBQuery, RunnableDBQuery } from '../query/dbQuery'
|
|
30
|
-
import { CommonSchema } from '../schema/common.schema'
|
|
31
31
|
import {
|
|
32
32
|
CommonDaoCfg,
|
|
33
33
|
CommonDaoCreateOptions,
|
|
@@ -874,13 +874,13 @@ export class CommonDao<
|
|
|
874
874
|
return convertedValue
|
|
875
875
|
}
|
|
876
876
|
|
|
877
|
-
async getTableSchema(): Promise<
|
|
877
|
+
async getTableSchema(): Promise<JsonSchemaObject<DBM>> {
|
|
878
878
|
return await this.cfg.db.getTableSchema<DBM>(this.cfg.table)
|
|
879
879
|
}
|
|
880
880
|
|
|
881
|
-
async createTable(schema:
|
|
881
|
+
async createTable(schema: JsonSchemaObject<DBM>, opt?: CommonDaoCreateOptions): Promise<void> {
|
|
882
882
|
this.requireWriteAccess()
|
|
883
|
-
await this.cfg.db.createTable(schema, opt)
|
|
883
|
+
await this.cfg.db.createTable(this.cfg.table, schema as any, opt)
|
|
884
884
|
}
|
|
885
885
|
|
|
886
886
|
/**
|
package/src/index.ts
CHANGED
|
@@ -67,8 +67,6 @@ import {
|
|
|
67
67
|
DBQueryOrder,
|
|
68
68
|
RunnableDBQuery,
|
|
69
69
|
} from './query/dbQuery'
|
|
70
|
-
import { CommonSchema, CommonSchemaField, DATA_TYPE } from './schema/common.schema'
|
|
71
|
-
import { CommonSchemaGenerator, CommonSchemaGeneratorCfg } from './schema/commonSchemaGenerator'
|
|
72
70
|
import { DBTransaction, RunnableDBTransaction } from './transaction/dbTransaction'
|
|
73
71
|
import { commitDBTransactionSimple, mergeDBOperations } from './transaction/dbTransaction.util'
|
|
74
72
|
|
|
@@ -110,9 +108,6 @@ export type {
|
|
|
110
108
|
DBPipelineBackupOptions,
|
|
111
109
|
DBPipelineRestoreOptions,
|
|
112
110
|
DBPipelineCopyOptions,
|
|
113
|
-
CommonSchema,
|
|
114
|
-
CommonSchemaField,
|
|
115
|
-
CommonSchemaGeneratorCfg,
|
|
116
111
|
CommonDBAdapter,
|
|
117
112
|
DBOperation,
|
|
118
113
|
DBSaveBatchOperation,
|
|
@@ -143,7 +138,6 @@ export {
|
|
|
143
138
|
dbPipelineBackup,
|
|
144
139
|
dbPipelineRestore,
|
|
145
140
|
dbPipelineCopy,
|
|
146
|
-
DATA_TYPE,
|
|
147
141
|
getDB,
|
|
148
142
|
DBLibError,
|
|
149
143
|
BaseCommonDB,
|
|
@@ -151,6 +145,5 @@ export {
|
|
|
151
145
|
RunnableDBTransaction,
|
|
152
146
|
mergeDBOperations,
|
|
153
147
|
commitDBTransactionSimple,
|
|
154
|
-
CommonSchemaGenerator,
|
|
155
148
|
CommonKeyValueDao,
|
|
156
149
|
}
|
|
@@ -15,7 +15,6 @@ import * as fs from 'fs-extra'
|
|
|
15
15
|
import { createGzip, ZlibOptions } from 'zlib'
|
|
16
16
|
import { CommonDB } from '../common.db'
|
|
17
17
|
import { DBQuery } from '../index'
|
|
18
|
-
import { CommonSchemaGenerator } from '../schema/commonSchemaGenerator'
|
|
19
18
|
|
|
20
19
|
export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
|
|
21
20
|
/**
|
|
@@ -102,7 +101,7 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
|
|
|
102
101
|
* @default false
|
|
103
102
|
* If true - will use CommonSchemaGenerator to detect schema from input data.
|
|
104
103
|
*/
|
|
105
|
-
emitSchemaFromData?: boolean
|
|
104
|
+
// emitSchemaFromData?: boolean
|
|
106
105
|
|
|
107
106
|
/**
|
|
108
107
|
* @default false
|
|
@@ -140,7 +139,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
|
|
|
140
139
|
transformMapOptions,
|
|
141
140
|
errorMode = ErrorMode.SUPPRESS,
|
|
142
141
|
emitSchemaFromDB = false,
|
|
143
|
-
emitSchemaFromData = false,
|
|
144
142
|
sortObjects = false,
|
|
145
143
|
} = opt
|
|
146
144
|
const strict = errorMode !== ErrorMode.SUPPRESS
|
|
@@ -193,10 +191,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
|
|
|
193
191
|
console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
|
|
194
192
|
}
|
|
195
193
|
|
|
196
|
-
const schemaGen = emitSchemaFromData
|
|
197
|
-
? new CommonSchemaGenerator({ table, sortedFields: true })
|
|
198
|
-
: undefined
|
|
199
|
-
|
|
200
194
|
await _pipeline([
|
|
201
195
|
db.streamQuery(q),
|
|
202
196
|
transformLogProgress({
|
|
@@ -210,9 +204,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
|
|
|
210
204
|
...transformMapOptions,
|
|
211
205
|
metric: table,
|
|
212
206
|
}),
|
|
213
|
-
transformTap(
|
|
207
|
+
transformTap(() => {
|
|
214
208
|
rows++
|
|
215
|
-
if (schemaGen) schemaGen.add(row)
|
|
216
209
|
}),
|
|
217
210
|
transformToNDJson({ strict, sortObjects }),
|
|
218
211
|
...(gzip ? [createGzip(zlibOptions)] : []), // optional gzip
|
|
@@ -229,12 +222,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
|
|
|
229
222
|
|
|
230
223
|
console.log(`>> ${grey(filePath)}\n` + stats.toPretty())
|
|
231
224
|
|
|
232
|
-
if (schemaGen) {
|
|
233
|
-
const schema = schemaGen.generate()
|
|
234
|
-
await fs.writeJson(schemaFilePath, schema, { spaces: 2 })
|
|
235
|
-
console.log(`>> ${grey(schemaFilePath)} saved (generated from data)`)
|
|
236
|
-
}
|
|
237
|
-
|
|
238
225
|
statsPerTable[table] = stats
|
|
239
226
|
},
|
|
240
227
|
{ concurrency, errorMode },
|
|
@@ -179,7 +179,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
|
|
|
179
179
|
}
|
|
180
180
|
|
|
181
181
|
const schema = await fs.readJson(schemaFilePath)
|
|
182
|
-
await db.createTable(schema, { dropIfExists: true })
|
|
182
|
+
await db.createTable(table, schema, { dropIfExists: true })
|
|
183
183
|
})
|
|
184
184
|
}
|
|
185
185
|
|
package/src/testing/dbTest.ts
CHANGED
|
@@ -97,7 +97,7 @@ export function runCommonDBTest(
|
|
|
97
97
|
// CREATE TABLE, DROP
|
|
98
98
|
if (createTable) {
|
|
99
99
|
test('createTable, dropIfExists=true', async () => {
|
|
100
|
-
await db.createTable(getTestItemSchema(), { dropIfExists: true })
|
|
100
|
+
await db.createTable(TEST_TABLE, getTestItemSchema(), { dropIfExists: true })
|
|
101
101
|
})
|
|
102
102
|
}
|
|
103
103
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { jsonSchema, _range } from '@naturalcycles/js-lib'
|
|
1
|
+
import { jsonSchema, JsonSchemaObject, _range } from '@naturalcycles/js-lib'
|
|
2
2
|
import {
|
|
3
3
|
binarySchema,
|
|
4
4
|
booleanSchema,
|
|
@@ -6,7 +6,6 @@ import {
|
|
|
6
6
|
objectSchema,
|
|
7
7
|
stringSchema,
|
|
8
8
|
} from '@naturalcycles/nodejs-lib'
|
|
9
|
-
import { CommonSchema, CommonSchemaGenerator } from '..'
|
|
10
9
|
import { BaseDBEntity, baseDBEntitySchema, Saved, savedDBEntitySchema } from '../db.model'
|
|
11
10
|
|
|
12
11
|
const MOCK_TS_2018_06_21 = 1529539200
|
|
@@ -95,6 +94,19 @@ export function createTestItemsBM(count = 1): Saved<TestItemBM>[] {
|
|
|
95
94
|
return _range(1, count + 1).map(num => createTestItemBM(num))
|
|
96
95
|
}
|
|
97
96
|
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
const testItemJsonSchema = jsonSchema
|
|
98
|
+
.object<TestItemDBM>({
|
|
99
|
+
id: jsonSchema.string(),
|
|
100
|
+
k1: jsonSchema.string(),
|
|
101
|
+
k2: jsonSchema.string(),
|
|
102
|
+
k3: jsonSchema.number(),
|
|
103
|
+
even: jsonSchema.boolean(),
|
|
104
|
+
created: jsonSchema.unixTimestamp(),
|
|
105
|
+
updated: jsonSchema.unixTimestamp(),
|
|
106
|
+
})
|
|
107
|
+
.build()
|
|
108
|
+
|
|
109
|
+
export function getTestItemSchema(): JsonSchemaObject<TestItemDBM> {
|
|
110
|
+
// return CommonSchemaGenerator.generateFromRows({ table: TEST_TABLE }, createTestItemsDBM())
|
|
111
|
+
return testItemJsonSchema
|
|
100
112
|
}
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
export interface CommonSchema<ROW = any> {
|
|
2
|
-
/**
|
|
3
|
-
* Name of the Table
|
|
4
|
-
*/
|
|
5
|
-
table: string;
|
|
6
|
-
fields: CommonSchemaField[];
|
|
7
|
-
}
|
|
8
|
-
export interface CommonSchemaField {
|
|
9
|
-
name: string;
|
|
10
|
-
type: DATA_TYPE;
|
|
11
|
-
/**
|
|
12
|
-
* If it's an Array - then parentType should tell "array of what?"
|
|
13
|
-
*/
|
|
14
|
-
arrayOf?: CommonSchemaField;
|
|
15
|
-
/**
|
|
16
|
-
* If it's an Object - defined the CommonSchemaField of that object
|
|
17
|
-
*/
|
|
18
|
-
objectFields?: CommonSchemaField[];
|
|
19
|
-
notNull?: boolean;
|
|
20
|
-
/**
|
|
21
|
-
* Applicable to certain fields, e.g String, to be able to autodetect limits for certain Databases
|
|
22
|
-
*/
|
|
23
|
-
maxLen?: number;
|
|
24
|
-
minLen?: number;
|
|
25
|
-
}
|
|
26
|
-
export declare enum DATA_TYPE {
|
|
27
|
-
UNKNOWN = "UNKNOWN",
|
|
28
|
-
NULL = "NULL",
|
|
29
|
-
STRING = "STRING",
|
|
30
|
-
INT = "INT",
|
|
31
|
-
FLOAT = "FLOAT",
|
|
32
|
-
BOOLEAN = "BOOLEAN",
|
|
33
|
-
BINARY = "BINARY",
|
|
34
|
-
LOCAL_DATE = "LOCAL_DATE",
|
|
35
|
-
TIMESTAMP = "TIMESTAMP",
|
|
36
|
-
ARRAY = "ARRAY",
|
|
37
|
-
OBJECT = "OBJECT"
|
|
38
|
-
}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DATA_TYPE = void 0;
|
|
4
|
-
var DATA_TYPE;
|
|
5
|
-
(function (DATA_TYPE) {
|
|
6
|
-
DATA_TYPE["UNKNOWN"] = "UNKNOWN";
|
|
7
|
-
DATA_TYPE["NULL"] = "NULL";
|
|
8
|
-
DATA_TYPE["STRING"] = "STRING";
|
|
9
|
-
DATA_TYPE["INT"] = "INT";
|
|
10
|
-
DATA_TYPE["FLOAT"] = "FLOAT";
|
|
11
|
-
DATA_TYPE["BOOLEAN"] = "BOOLEAN";
|
|
12
|
-
DATA_TYPE["BINARY"] = "BINARY";
|
|
13
|
-
DATA_TYPE["LOCAL_DATE"] = "LOCAL_DATE";
|
|
14
|
-
DATA_TYPE["TIMESTAMP"] = "TIMESTAMP";
|
|
15
|
-
// Semi-structured
|
|
16
|
-
DATA_TYPE["ARRAY"] = "ARRAY";
|
|
17
|
-
DATA_TYPE["OBJECT"] = "OBJECT";
|
|
18
|
-
})(DATA_TYPE = exports.DATA_TYPE || (exports.DATA_TYPE = {}));
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
import { ErrorMode } from '@naturalcycles/js-lib';
|
|
2
|
-
import { CommonSchema } from './common.schema';
|
|
3
|
-
export interface CommonSchemaGeneratorCfg {
|
|
4
|
-
/**
|
|
5
|
-
* Name of the Table
|
|
6
|
-
*/
|
|
7
|
-
table: string;
|
|
8
|
-
/**
|
|
9
|
-
* @default SUPPRESS
|
|
10
|
-
*
|
|
11
|
-
* On error (field has multiple types):
|
|
12
|
-
* if ErrorMode.THROW_IMMEDIATE - will throw on .add()
|
|
13
|
-
* if ErrorMode.THROW_AGGREGATED - will throw on .generate()
|
|
14
|
-
*/
|
|
15
|
-
errorMode?: ErrorMode;
|
|
16
|
-
/**
|
|
17
|
-
* @default false
|
|
18
|
-
* If true - fields will be sorted by name alphabetically (also inside objects)
|
|
19
|
-
*/
|
|
20
|
-
sortedFields?: boolean;
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Class that helps to generate CommonSchema by processing ALL rows through it.
|
|
24
|
-
*/
|
|
25
|
-
export declare class CommonSchemaGenerator<ROW = any> {
|
|
26
|
-
cfg: CommonSchemaGeneratorCfg;
|
|
27
|
-
constructor(cfg: CommonSchemaGeneratorCfg);
|
|
28
|
-
private fieldByName;
|
|
29
|
-
private nullableFields;
|
|
30
|
-
add(row: ROW): void;
|
|
31
|
-
private mergeFields;
|
|
32
|
-
private detectType;
|
|
33
|
-
generate(): CommonSchema<ROW>;
|
|
34
|
-
static generateFromRows<ROW>(cfg: CommonSchemaGeneratorCfg, rows?: ROW[]): CommonSchema<ROW>;
|
|
35
|
-
}
|
|
@@ -1,151 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.CommonSchemaGenerator = void 0;
|
|
4
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
|
-
const common_schema_1 = require("./common.schema");
|
|
6
|
-
const LOCAL_DATE_PATTERN = new RegExp(/[0-9]{4}-[01][0-9]-[0-3][0-9]/);
|
|
7
|
-
/**
|
|
8
|
-
* Class that helps to generate CommonSchema by processing ALL rows through it.
|
|
9
|
-
*/
|
|
10
|
-
class CommonSchemaGenerator {
|
|
11
|
-
constructor(cfg) {
|
|
12
|
-
this.cfg = cfg;
|
|
13
|
-
this.fieldByName = {};
|
|
14
|
-
this.nullableFields = new Set();
|
|
15
|
-
}
|
|
16
|
-
// private fieldsWithMultipleTypes: Record<string, DATA_TYPE[]> = {}
|
|
17
|
-
add(row) {
|
|
18
|
-
if (!row)
|
|
19
|
-
return; // safety
|
|
20
|
-
Object.entries(row).forEach(([fieldName, value]) => {
|
|
21
|
-
this.fieldByName[fieldName] = this.mergeFields(this.fieldByName[fieldName], this.detectType(fieldName, value));
|
|
22
|
-
if (this.fieldByName[fieldName].type === common_schema_1.DATA_TYPE.NULL) {
|
|
23
|
-
this.nullableFields.add(fieldName);
|
|
24
|
-
}
|
|
25
|
-
});
|
|
26
|
-
// missing fields
|
|
27
|
-
Object.keys(this.fieldByName).forEach(fieldName => {
|
|
28
|
-
if (!(fieldName in row)) {
|
|
29
|
-
// missing field!
|
|
30
|
-
this.nullableFields.add(fieldName);
|
|
31
|
-
}
|
|
32
|
-
});
|
|
33
|
-
}
|
|
34
|
-
mergeFields(existing, newField) {
|
|
35
|
-
if (!existing)
|
|
36
|
-
return newField;
|
|
37
|
-
if (newField.type === common_schema_1.DATA_TYPE.UNKNOWN || newField.type === common_schema_1.DATA_TYPE.NULL) {
|
|
38
|
-
return {
|
|
39
|
-
...existing,
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
if (existing.type === common_schema_1.DATA_TYPE.UNKNOWN || existing.type === common_schema_1.DATA_TYPE.NULL) {
|
|
43
|
-
return {
|
|
44
|
-
...newField,
|
|
45
|
-
};
|
|
46
|
-
}
|
|
47
|
-
let type = existing.type;
|
|
48
|
-
if (existing.type !== newField.type) {
|
|
49
|
-
const [type1, type2] = [existing.type, newField.type].sort();
|
|
50
|
-
if (type1 === common_schema_1.DATA_TYPE.FLOAT && type2 === common_schema_1.DATA_TYPE.INT) {
|
|
51
|
-
type = common_schema_1.DATA_TYPE.FLOAT;
|
|
52
|
-
}
|
|
53
|
-
else if (type1 === common_schema_1.DATA_TYPE.LOCAL_DATE && type2 === common_schema_1.DATA_TYPE.STRING) {
|
|
54
|
-
type = common_schema_1.DATA_TYPE.STRING;
|
|
55
|
-
}
|
|
56
|
-
else {
|
|
57
|
-
// Type mismatch! Oj!
|
|
58
|
-
return newField; // currently just use "latest" type
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
// Type is same
|
|
62
|
-
const minLen = existing.minLen !== undefined
|
|
63
|
-
? Math.min(...[existing.minLen, newField.minLen].filter(Boolean))
|
|
64
|
-
: undefined;
|
|
65
|
-
const maxLen = existing.maxLen !== undefined
|
|
66
|
-
? Math.max(...[existing.maxLen, newField.maxLen].filter(Boolean))
|
|
67
|
-
: undefined;
|
|
68
|
-
// todo: recursively merge/compare array/object schemas
|
|
69
|
-
return (0, js_lib_1._filterNullishValues)({
|
|
70
|
-
...newField,
|
|
71
|
-
type,
|
|
72
|
-
minLen,
|
|
73
|
-
maxLen,
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
detectType(name, value, level = 1) {
|
|
77
|
-
// Null
|
|
78
|
-
if (value === undefined || value === null) {
|
|
79
|
-
return {
|
|
80
|
-
name,
|
|
81
|
-
type: common_schema_1.DATA_TYPE.NULL,
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
// String
|
|
85
|
-
if (typeof value === 'string') {
|
|
86
|
-
// LocalDate
|
|
87
|
-
if (LOCAL_DATE_PATTERN.test(value)) {
|
|
88
|
-
return { name, type: common_schema_1.DATA_TYPE.LOCAL_DATE };
|
|
89
|
-
}
|
|
90
|
-
return { name, type: common_schema_1.DATA_TYPE.STRING, minLen: value.length, maxLen: value.length };
|
|
91
|
-
}
|
|
92
|
-
// Int, Float
|
|
93
|
-
if (typeof value === 'number') {
|
|
94
|
-
// Cannot really detect TIMESTAMP
|
|
95
|
-
return {
|
|
96
|
-
name,
|
|
97
|
-
type: Number.isInteger(value) ? common_schema_1.DATA_TYPE.INT : common_schema_1.DATA_TYPE.FLOAT,
|
|
98
|
-
minLen: value,
|
|
99
|
-
maxLen: value,
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
// Boolean
|
|
103
|
-
if (typeof value === 'boolean') {
|
|
104
|
-
return { name, type: common_schema_1.DATA_TYPE.BOOLEAN };
|
|
105
|
-
}
|
|
106
|
-
// Binary
|
|
107
|
-
if (Buffer.isBuffer(value)) {
|
|
108
|
-
return { name, type: common_schema_1.DATA_TYPE.BINARY, minLen: value.length, maxLen: value.length };
|
|
109
|
-
}
|
|
110
|
-
// Array
|
|
111
|
-
if (Array.isArray(value)) {
|
|
112
|
-
return {
|
|
113
|
-
name,
|
|
114
|
-
type: common_schema_1.DATA_TYPE.ARRAY,
|
|
115
|
-
arrayOf: this.detectType('', value[0], level + 1),
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
// Object
|
|
119
|
-
if (typeof value === 'object') {
|
|
120
|
-
return {
|
|
121
|
-
name,
|
|
122
|
-
type: common_schema_1.DATA_TYPE.OBJECT,
|
|
123
|
-
objectFields: Object.entries(value).map(([k, v]) => this.detectType(k, v, level + 1)),
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
return { name, type: common_schema_1.DATA_TYPE.UNKNOWN };
|
|
127
|
-
}
|
|
128
|
-
generate() {
|
|
129
|
-
// set nullability
|
|
130
|
-
Object.keys(this.fieldByName).forEach(fieldName => {
|
|
131
|
-
if (!this.nullableFields.has(fieldName)) {
|
|
132
|
-
this.fieldByName[fieldName].notNull = true;
|
|
133
|
-
}
|
|
134
|
-
});
|
|
135
|
-
const { table, sortedFields } = this.cfg;
|
|
136
|
-
const fieldNames = Object.keys(this.fieldByName);
|
|
137
|
-
if (sortedFields)
|
|
138
|
-
fieldNames.sort(); // mutates
|
|
139
|
-
// todo: sort object fields too
|
|
140
|
-
return {
|
|
141
|
-
table,
|
|
142
|
-
fields: fieldNames.map(name => this.fieldByName[name]),
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
static generateFromRows(cfg, rows = []) {
|
|
146
|
-
const gen = new CommonSchemaGenerator(cfg);
|
|
147
|
-
rows.forEach(r => gen.add(r));
|
|
148
|
-
return gen.generate();
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
exports.CommonSchemaGenerator = CommonSchemaGenerator;
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
// eslint-disable-next-line unused-imports/no-unused-vars
|
|
2
|
-
export interface CommonSchema<ROW = any> {
|
|
3
|
-
/**
|
|
4
|
-
* Name of the Table
|
|
5
|
-
*/
|
|
6
|
-
table: string
|
|
7
|
-
|
|
8
|
-
fields: CommonSchemaField[]
|
|
9
|
-
// can possibly have `meta` with table meta-information
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
export interface CommonSchemaField {
|
|
13
|
-
name: string
|
|
14
|
-
type: DATA_TYPE
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* If it's an Array - then parentType should tell "array of what?"
|
|
18
|
-
*/
|
|
19
|
-
arrayOf?: CommonSchemaField
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* If it's an Object - defined the CommonSchemaField of that object
|
|
23
|
-
*/
|
|
24
|
-
objectFields?: CommonSchemaField[]
|
|
25
|
-
|
|
26
|
-
notNull?: boolean
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Applicable to certain fields, e.g String, to be able to autodetect limits for certain Databases
|
|
30
|
-
*/
|
|
31
|
-
maxLen?: number
|
|
32
|
-
minLen?: number
|
|
33
|
-
// avgLen?: number
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
export enum DATA_TYPE {
|
|
37
|
-
UNKNOWN = 'UNKNOWN',
|
|
38
|
-
NULL = 'NULL',
|
|
39
|
-
STRING = 'STRING',
|
|
40
|
-
INT = 'INT',
|
|
41
|
-
FLOAT = 'FLOAT',
|
|
42
|
-
BOOLEAN = 'BOOLEAN',
|
|
43
|
-
BINARY = 'BINARY',
|
|
44
|
-
LOCAL_DATE = 'LOCAL_DATE', // ISO
|
|
45
|
-
TIMESTAMP = 'TIMESTAMP', // unix timestamp
|
|
46
|
-
// Semi-structured
|
|
47
|
-
ARRAY = 'ARRAY',
|
|
48
|
-
OBJECT = 'OBJECT',
|
|
49
|
-
}
|
|
@@ -1,200 +0,0 @@
|
|
|
1
|
-
import { ErrorMode, _filterNullishValues } from '@naturalcycles/js-lib'
|
|
2
|
-
import { CommonSchema, CommonSchemaField, DATA_TYPE } from './common.schema'
|
|
3
|
-
|
|
4
|
-
export interface CommonSchemaGeneratorCfg {
|
|
5
|
-
/**
|
|
6
|
-
* Name of the Table
|
|
7
|
-
*/
|
|
8
|
-
table: string
|
|
9
|
-
|
|
10
|
-
/**
|
|
11
|
-
* @default SUPPRESS
|
|
12
|
-
*
|
|
13
|
-
* On error (field has multiple types):
|
|
14
|
-
* if ErrorMode.THROW_IMMEDIATE - will throw on .add()
|
|
15
|
-
* if ErrorMode.THROW_AGGREGATED - will throw on .generate()
|
|
16
|
-
*/
|
|
17
|
-
errorMode?: ErrorMode
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* @default false
|
|
21
|
-
* If true - fields will be sorted by name alphabetically (also inside objects)
|
|
22
|
-
*/
|
|
23
|
-
sortedFields?: boolean
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const LOCAL_DATE_PATTERN = new RegExp(/[0-9]{4}-[01][0-9]-[0-3][0-9]/)
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Class that helps to generate CommonSchema by processing ALL rows through it.
|
|
30
|
-
*/
|
|
31
|
-
export class CommonSchemaGenerator<ROW = any> {
|
|
32
|
-
constructor(public cfg: CommonSchemaGeneratorCfg) {}
|
|
33
|
-
|
|
34
|
-
private fieldByName: Record<string, CommonSchemaField> = {}
|
|
35
|
-
private nullableFields = new Set<string>()
|
|
36
|
-
// private fieldsWithMultipleTypes: Record<string, DATA_TYPE[]> = {}
|
|
37
|
-
|
|
38
|
-
add(row: ROW): void {
|
|
39
|
-
if (!row) return // safety
|
|
40
|
-
|
|
41
|
-
Object.entries(row).forEach(([fieldName, value]) => {
|
|
42
|
-
this.fieldByName[fieldName] = this.mergeFields(
|
|
43
|
-
this.fieldByName[fieldName],
|
|
44
|
-
this.detectType(fieldName, value),
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
if (this.fieldByName[fieldName]!.type === DATA_TYPE.NULL) {
|
|
48
|
-
this.nullableFields.add(fieldName)
|
|
49
|
-
}
|
|
50
|
-
})
|
|
51
|
-
|
|
52
|
-
// missing fields
|
|
53
|
-
Object.keys(this.fieldByName).forEach(fieldName => {
|
|
54
|
-
if (!(fieldName in row)) {
|
|
55
|
-
// missing field!
|
|
56
|
-
this.nullableFields.add(fieldName)
|
|
57
|
-
}
|
|
58
|
-
})
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
private mergeFields(
|
|
62
|
-
existing: CommonSchemaField | undefined,
|
|
63
|
-
newField: CommonSchemaField,
|
|
64
|
-
): CommonSchemaField {
|
|
65
|
-
if (!existing) return newField
|
|
66
|
-
|
|
67
|
-
if (newField.type === DATA_TYPE.UNKNOWN || newField.type === DATA_TYPE.NULL) {
|
|
68
|
-
return {
|
|
69
|
-
...existing,
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
if (existing.type === DATA_TYPE.UNKNOWN || existing.type === DATA_TYPE.NULL) {
|
|
74
|
-
return {
|
|
75
|
-
...newField,
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
let type = existing.type
|
|
80
|
-
|
|
81
|
-
if (existing.type !== newField.type) {
|
|
82
|
-
const [type1, type2] = [existing.type, newField.type].sort()
|
|
83
|
-
if (type1 === DATA_TYPE.FLOAT && type2 === DATA_TYPE.INT) {
|
|
84
|
-
type = DATA_TYPE.FLOAT
|
|
85
|
-
} else if (type1 === DATA_TYPE.LOCAL_DATE && type2 === DATA_TYPE.STRING) {
|
|
86
|
-
type = DATA_TYPE.STRING
|
|
87
|
-
} else {
|
|
88
|
-
// Type mismatch! Oj!
|
|
89
|
-
return newField // currently just use "latest" type
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// Type is same
|
|
94
|
-
const minLen =
|
|
95
|
-
existing.minLen !== undefined
|
|
96
|
-
? Math.min(...[existing.minLen, newField.minLen!].filter(Boolean))
|
|
97
|
-
: undefined
|
|
98
|
-
const maxLen =
|
|
99
|
-
existing.maxLen !== undefined
|
|
100
|
-
? Math.max(...[existing.maxLen, newField.maxLen!].filter(Boolean))
|
|
101
|
-
: undefined
|
|
102
|
-
|
|
103
|
-
// todo: recursively merge/compare array/object schemas
|
|
104
|
-
|
|
105
|
-
return _filterNullishValues({
|
|
106
|
-
...newField,
|
|
107
|
-
type,
|
|
108
|
-
minLen,
|
|
109
|
-
maxLen,
|
|
110
|
-
})
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
private detectType(name: string, value: any, level = 1): CommonSchemaField {
|
|
114
|
-
// Null
|
|
115
|
-
if (value === undefined || value === null) {
|
|
116
|
-
return {
|
|
117
|
-
name,
|
|
118
|
-
type: DATA_TYPE.NULL,
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// String
|
|
123
|
-
if (typeof value === 'string') {
|
|
124
|
-
// LocalDate
|
|
125
|
-
if (LOCAL_DATE_PATTERN.test(value)) {
|
|
126
|
-
return { name, type: DATA_TYPE.LOCAL_DATE }
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
return { name, type: DATA_TYPE.STRING, minLen: value.length, maxLen: value.length }
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
// Int, Float
|
|
133
|
-
if (typeof value === 'number') {
|
|
134
|
-
// Cannot really detect TIMESTAMP
|
|
135
|
-
return {
|
|
136
|
-
name,
|
|
137
|
-
type: Number.isInteger(value) ? DATA_TYPE.INT : DATA_TYPE.FLOAT,
|
|
138
|
-
minLen: value,
|
|
139
|
-
maxLen: value,
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
// Boolean
|
|
144
|
-
if (typeof value === 'boolean') {
|
|
145
|
-
return { name, type: DATA_TYPE.BOOLEAN }
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
// Binary
|
|
149
|
-
if (Buffer.isBuffer(value)) {
|
|
150
|
-
return { name, type: DATA_TYPE.BINARY, minLen: value.length, maxLen: value.length }
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// Array
|
|
154
|
-
if (Array.isArray(value)) {
|
|
155
|
-
return {
|
|
156
|
-
name,
|
|
157
|
-
type: DATA_TYPE.ARRAY,
|
|
158
|
-
arrayOf: this.detectType('', value[0], level + 1),
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// Object
|
|
163
|
-
if (typeof value === 'object') {
|
|
164
|
-
return {
|
|
165
|
-
name,
|
|
166
|
-
type: DATA_TYPE.OBJECT,
|
|
167
|
-
objectFields: Object.entries(value).map(([k, v]) => this.detectType(k, v, level + 1)),
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
return { name, type: DATA_TYPE.UNKNOWN }
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
generate(): CommonSchema<ROW> {
|
|
175
|
-
// set nullability
|
|
176
|
-
Object.keys(this.fieldByName).forEach(fieldName => {
|
|
177
|
-
if (!this.nullableFields.has(fieldName)) {
|
|
178
|
-
this.fieldByName[fieldName]!.notNull = true
|
|
179
|
-
}
|
|
180
|
-
})
|
|
181
|
-
|
|
182
|
-
const { table, sortedFields } = this.cfg
|
|
183
|
-
|
|
184
|
-
const fieldNames = Object.keys(this.fieldByName)
|
|
185
|
-
if (sortedFields) fieldNames.sort() // mutates
|
|
186
|
-
|
|
187
|
-
// todo: sort object fields too
|
|
188
|
-
|
|
189
|
-
return {
|
|
190
|
-
table,
|
|
191
|
-
fields: fieldNames.map(name => this.fieldByName[name]!),
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
static generateFromRows<ROW>(cfg: CommonSchemaGeneratorCfg, rows: ROW[] = []): CommonSchema<ROW> {
|
|
196
|
-
const gen = new CommonSchemaGenerator(cfg)
|
|
197
|
-
rows.forEach(r => gen.add(r))
|
|
198
|
-
return gen.generate()
|
|
199
|
-
}
|
|
200
|
-
}
|