@naturalcycles/db-lib 8.12.0 → 8.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/dist/adapter/cachedb/cache.db.d.ts +3 -3
  3. package/dist/adapter/cachedb/cache.db.js +3 -3
  4. package/dist/adapter/file/file.db.d.ts +2 -2
  5. package/dist/adapter/file/file.db.js +4 -2
  6. package/dist/adapter/inmemory/inMemory.db.d.ts +3 -4
  7. package/dist/adapter/inmemory/inMemory.db.js +6 -4
  8. package/dist/base.common.db.d.ts +3 -3
  9. package/dist/base.common.db.js +9 -3
  10. package/dist/common.db.d.ts +3 -3
  11. package/dist/commondao/common.dao.d.ts +3 -4
  12. package/dist/commondao/common.dao.js +1 -1
  13. package/dist/index.d.ts +2 -4
  14. package/dist/index.js +1 -5
  15. package/dist/pipeline/dbPipelineBackup.d.ts +0 -1
  16. package/dist/pipeline/dbPipelineBackup.js +2 -13
  17. package/dist/pipeline/dbPipelineRestore.js +1 -1
  18. package/dist/testing/dbTest.js +1 -1
  19. package/dist/testing/test.model.d.ts +2 -2
  20. package/dist/testing/test.model.js +13 -2
  21. package/package.json +1 -1
  22. package/readme.md +5 -4
  23. package/src/adapter/cachedb/cache.db.ts +9 -5
  24. package/src/adapter/file/file.db.ts +7 -4
  25. package/src/adapter/inmemory/inMemory.db.ts +20 -7
  26. package/src/base.common.db.ts +10 -4
  27. package/src/common.db.ts +3 -3
  28. package/src/commondao/common.dao.ts +4 -4
  29. package/src/index.ts +0 -7
  30. package/src/pipeline/dbPipelineBackup.ts +2 -15
  31. package/src/pipeline/dbPipelineRestore.ts +1 -1
  32. package/src/testing/dbTest.ts +1 -1
  33. package/src/testing/test.model.ts +16 -4
  34. package/dist/schema/common.schema.d.ts +0 -38
  35. package/dist/schema/common.schema.js +0 -18
  36. package/dist/schema/commonSchemaGenerator.d.ts +0 -35
  37. package/dist/schema/commonSchemaGenerator.js +0 -151
  38. package/src/schema/common.schema.ts +0 -49
  39. package/src/schema/commonSchemaGenerator.ts +0 -200
@@ -2,6 +2,7 @@ import {
2
2
  AppError,
3
3
  AsyncMapper,
4
4
  ErrorMode,
5
+ JsonSchemaObject,
5
6
  _filterNullishValues,
6
7
  _passthroughPredicate,
7
8
  _since,
@@ -27,7 +28,6 @@ import {
27
28
  import { DBLibError } from '../cnst'
28
29
  import { DBModelType, ObjectWithId, RunQueryResult, Saved } from '../db.model'
29
30
  import { DBQuery, RunnableDBQuery } from '../query/dbQuery'
30
- import { CommonSchema } from '../schema/common.schema'
31
31
  import {
32
32
  CommonDaoCfg,
33
33
  CommonDaoCreateOptions,
@@ -874,13 +874,13 @@ export class CommonDao<
874
874
  return convertedValue
875
875
  }
876
876
 
877
- async getTableSchema(): Promise<CommonSchema> {
877
+ async getTableSchema(): Promise<JsonSchemaObject<DBM>> {
878
878
  return await this.cfg.db.getTableSchema<DBM>(this.cfg.table)
879
879
  }
880
880
 
881
- async createTable(schema: CommonSchema, opt?: CommonDaoCreateOptions): Promise<void> {
881
+ async createTable(schema: JsonSchemaObject<DBM>, opt?: CommonDaoCreateOptions): Promise<void> {
882
882
  this.requireWriteAccess()
883
- await this.cfg.db.createTable(schema, opt)
883
+ await this.cfg.db.createTable(this.cfg.table, schema as any, opt)
884
884
  }
885
885
 
886
886
  /**
package/src/index.ts CHANGED
@@ -67,8 +67,6 @@ import {
67
67
  DBQueryOrder,
68
68
  RunnableDBQuery,
69
69
  } from './query/dbQuery'
70
- import { CommonSchema, CommonSchemaField, DATA_TYPE } from './schema/common.schema'
71
- import { CommonSchemaGenerator, CommonSchemaGeneratorCfg } from './schema/commonSchemaGenerator'
72
70
  import { DBTransaction, RunnableDBTransaction } from './transaction/dbTransaction'
73
71
  import { commitDBTransactionSimple, mergeDBOperations } from './transaction/dbTransaction.util'
74
72
 
@@ -110,9 +108,6 @@ export type {
110
108
  DBPipelineBackupOptions,
111
109
  DBPipelineRestoreOptions,
112
110
  DBPipelineCopyOptions,
113
- CommonSchema,
114
- CommonSchemaField,
115
- CommonSchemaGeneratorCfg,
116
111
  CommonDBAdapter,
117
112
  DBOperation,
118
113
  DBSaveBatchOperation,
@@ -143,7 +138,6 @@ export {
143
138
  dbPipelineBackup,
144
139
  dbPipelineRestore,
145
140
  dbPipelineCopy,
146
- DATA_TYPE,
147
141
  getDB,
148
142
  DBLibError,
149
143
  BaseCommonDB,
@@ -151,6 +145,5 @@ export {
151
145
  RunnableDBTransaction,
152
146
  mergeDBOperations,
153
147
  commitDBTransactionSimple,
154
- CommonSchemaGenerator,
155
148
  CommonKeyValueDao,
156
149
  }
@@ -15,7 +15,6 @@ import * as fs from 'fs-extra'
15
15
  import { createGzip, ZlibOptions } from 'zlib'
16
16
  import { CommonDB } from '../common.db'
17
17
  import { DBQuery } from '../index'
18
- import { CommonSchemaGenerator } from '../schema/commonSchemaGenerator'
19
18
 
20
19
  export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
21
20
  /**
@@ -102,7 +101,7 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
102
101
  * @default false
103
102
  * If true - will use CommonSchemaGenerator to detect schema from input data.
104
103
  */
105
- emitSchemaFromData?: boolean
104
+ // emitSchemaFromData?: boolean
106
105
 
107
106
  /**
108
107
  * @default false
@@ -140,7 +139,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
140
139
  transformMapOptions,
141
140
  errorMode = ErrorMode.SUPPRESS,
142
141
  emitSchemaFromDB = false,
143
- emitSchemaFromData = false,
144
142
  sortObjects = false,
145
143
  } = opt
146
144
  const strict = errorMode !== ErrorMode.SUPPRESS
@@ -193,10 +191,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
193
191
  console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
194
192
  }
195
193
 
196
- const schemaGen = emitSchemaFromData
197
- ? new CommonSchemaGenerator({ table, sortedFields: true })
198
- : undefined
199
-
200
194
  await _pipeline([
201
195
  db.streamQuery(q),
202
196
  transformLogProgress({
@@ -210,9 +204,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
210
204
  ...transformMapOptions,
211
205
  metric: table,
212
206
  }),
213
- transformTap(row => {
207
+ transformTap(() => {
214
208
  rows++
215
- if (schemaGen) schemaGen.add(row)
216
209
  }),
217
210
  transformToNDJson({ strict, sortObjects }),
218
211
  ...(gzip ? [createGzip(zlibOptions)] : []), // optional gzip
@@ -229,12 +222,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
229
222
 
230
223
  console.log(`>> ${grey(filePath)}\n` + stats.toPretty())
231
224
 
232
- if (schemaGen) {
233
- const schema = schemaGen.generate()
234
- await fs.writeJson(schemaFilePath, schema, { spaces: 2 })
235
- console.log(`>> ${grey(schemaFilePath)} saved (generated from data)`)
236
- }
237
-
238
225
  statsPerTable[table] = stats
239
226
  },
240
227
  { concurrency, errorMode },
@@ -179,7 +179,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
179
179
  }
180
180
 
181
181
  const schema = await fs.readJson(schemaFilePath)
182
- await db.createTable(schema, { dropIfExists: true })
182
+ await db.createTable(table, schema, { dropIfExists: true })
183
183
  })
184
184
  }
185
185
 
@@ -97,7 +97,7 @@ export function runCommonDBTest(
97
97
  // CREATE TABLE, DROP
98
98
  if (createTable) {
99
99
  test('createTable, dropIfExists=true', async () => {
100
- await db.createTable(getTestItemSchema(), { dropIfExists: true })
100
+ await db.createTable(TEST_TABLE, getTestItemSchema(), { dropIfExists: true })
101
101
  })
102
102
  }
103
103
 
@@ -1,4 +1,4 @@
1
- import { jsonSchema, _range } from '@naturalcycles/js-lib'
1
+ import { jsonSchema, JsonSchemaObject, _range } from '@naturalcycles/js-lib'
2
2
  import {
3
3
  binarySchema,
4
4
  booleanSchema,
@@ -6,7 +6,6 @@ import {
6
6
  objectSchema,
7
7
  stringSchema,
8
8
  } from '@naturalcycles/nodejs-lib'
9
- import { CommonSchema, CommonSchemaGenerator } from '..'
10
9
  import { BaseDBEntity, baseDBEntitySchema, Saved, savedDBEntitySchema } from '../db.model'
11
10
 
12
11
  const MOCK_TS_2018_06_21 = 1529539200
@@ -95,6 +94,19 @@ export function createTestItemsBM(count = 1): Saved<TestItemBM>[] {
95
94
  return _range(1, count + 1).map(num => createTestItemBM(num))
96
95
  }
97
96
 
98
- export function getTestItemSchema(): CommonSchema<TestItemDBM> {
99
- return CommonSchemaGenerator.generateFromRows({ table: TEST_TABLE }, createTestItemsDBM())
97
+ const testItemJsonSchema = jsonSchema
98
+ .object<TestItemDBM>({
99
+ id: jsonSchema.string(),
100
+ k1: jsonSchema.string(),
101
+ k2: jsonSchema.string(),
102
+ k3: jsonSchema.number(),
103
+ even: jsonSchema.boolean(),
104
+ created: jsonSchema.unixTimestamp(),
105
+ updated: jsonSchema.unixTimestamp(),
106
+ })
107
+ .build()
108
+
109
+ export function getTestItemSchema(): JsonSchemaObject<TestItemDBM> {
110
+ // return CommonSchemaGenerator.generateFromRows({ table: TEST_TABLE }, createTestItemsDBM())
111
+ return testItemJsonSchema
100
112
  }
@@ -1,38 +0,0 @@
1
- export interface CommonSchema<ROW = any> {
2
- /**
3
- * Name of the Table
4
- */
5
- table: string;
6
- fields: CommonSchemaField[];
7
- }
8
- export interface CommonSchemaField {
9
- name: string;
10
- type: DATA_TYPE;
11
- /**
12
- * If it's an Array - then parentType should tell "array of what?"
13
- */
14
- arrayOf?: CommonSchemaField;
15
- /**
16
- * If it's an Object - defined the CommonSchemaField of that object
17
- */
18
- objectFields?: CommonSchemaField[];
19
- notNull?: boolean;
20
- /**
21
- * Applicable to certain fields, e.g String, to be able to autodetect limits for certain Databases
22
- */
23
- maxLen?: number;
24
- minLen?: number;
25
- }
26
- export declare enum DATA_TYPE {
27
- UNKNOWN = "UNKNOWN",
28
- NULL = "NULL",
29
- STRING = "STRING",
30
- INT = "INT",
31
- FLOAT = "FLOAT",
32
- BOOLEAN = "BOOLEAN",
33
- BINARY = "BINARY",
34
- LOCAL_DATE = "LOCAL_DATE",
35
- TIMESTAMP = "TIMESTAMP",
36
- ARRAY = "ARRAY",
37
- OBJECT = "OBJECT"
38
- }
@@ -1,18 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.DATA_TYPE = void 0;
4
- var DATA_TYPE;
5
- (function (DATA_TYPE) {
6
- DATA_TYPE["UNKNOWN"] = "UNKNOWN";
7
- DATA_TYPE["NULL"] = "NULL";
8
- DATA_TYPE["STRING"] = "STRING";
9
- DATA_TYPE["INT"] = "INT";
10
- DATA_TYPE["FLOAT"] = "FLOAT";
11
- DATA_TYPE["BOOLEAN"] = "BOOLEAN";
12
- DATA_TYPE["BINARY"] = "BINARY";
13
- DATA_TYPE["LOCAL_DATE"] = "LOCAL_DATE";
14
- DATA_TYPE["TIMESTAMP"] = "TIMESTAMP";
15
- // Semi-structured
16
- DATA_TYPE["ARRAY"] = "ARRAY";
17
- DATA_TYPE["OBJECT"] = "OBJECT";
18
- })(DATA_TYPE = exports.DATA_TYPE || (exports.DATA_TYPE = {}));
@@ -1,35 +0,0 @@
1
- import { ErrorMode } from '@naturalcycles/js-lib';
2
- import { CommonSchema } from './common.schema';
3
- export interface CommonSchemaGeneratorCfg {
4
- /**
5
- * Name of the Table
6
- */
7
- table: string;
8
- /**
9
- * @default SUPPRESS
10
- *
11
- * On error (field has multiple types):
12
- * if ErrorMode.THROW_IMMEDIATE - will throw on .add()
13
- * if ErrorMode.THROW_AGGREGATED - will throw on .generate()
14
- */
15
- errorMode?: ErrorMode;
16
- /**
17
- * @default false
18
- * If true - fields will be sorted by name alphabetically (also inside objects)
19
- */
20
- sortedFields?: boolean;
21
- }
22
- /**
23
- * Class that helps to generate CommonSchema by processing ALL rows through it.
24
- */
25
- export declare class CommonSchemaGenerator<ROW = any> {
26
- cfg: CommonSchemaGeneratorCfg;
27
- constructor(cfg: CommonSchemaGeneratorCfg);
28
- private fieldByName;
29
- private nullableFields;
30
- add(row: ROW): void;
31
- private mergeFields;
32
- private detectType;
33
- generate(): CommonSchema<ROW>;
34
- static generateFromRows<ROW>(cfg: CommonSchemaGeneratorCfg, rows?: ROW[]): CommonSchema<ROW>;
35
- }
@@ -1,151 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.CommonSchemaGenerator = void 0;
4
- const js_lib_1 = require("@naturalcycles/js-lib");
5
- const common_schema_1 = require("./common.schema");
6
- const LOCAL_DATE_PATTERN = new RegExp(/[0-9]{4}-[01][0-9]-[0-3][0-9]/);
7
- /**
8
- * Class that helps to generate CommonSchema by processing ALL rows through it.
9
- */
10
- class CommonSchemaGenerator {
11
- constructor(cfg) {
12
- this.cfg = cfg;
13
- this.fieldByName = {};
14
- this.nullableFields = new Set();
15
- }
16
- // private fieldsWithMultipleTypes: Record<string, DATA_TYPE[]> = {}
17
- add(row) {
18
- if (!row)
19
- return; // safety
20
- Object.entries(row).forEach(([fieldName, value]) => {
21
- this.fieldByName[fieldName] = this.mergeFields(this.fieldByName[fieldName], this.detectType(fieldName, value));
22
- if (this.fieldByName[fieldName].type === common_schema_1.DATA_TYPE.NULL) {
23
- this.nullableFields.add(fieldName);
24
- }
25
- });
26
- // missing fields
27
- Object.keys(this.fieldByName).forEach(fieldName => {
28
- if (!(fieldName in row)) {
29
- // missing field!
30
- this.nullableFields.add(fieldName);
31
- }
32
- });
33
- }
34
- mergeFields(existing, newField) {
35
- if (!existing)
36
- return newField;
37
- if (newField.type === common_schema_1.DATA_TYPE.UNKNOWN || newField.type === common_schema_1.DATA_TYPE.NULL) {
38
- return {
39
- ...existing,
40
- };
41
- }
42
- if (existing.type === common_schema_1.DATA_TYPE.UNKNOWN || existing.type === common_schema_1.DATA_TYPE.NULL) {
43
- return {
44
- ...newField,
45
- };
46
- }
47
- let type = existing.type;
48
- if (existing.type !== newField.type) {
49
- const [type1, type2] = [existing.type, newField.type].sort();
50
- if (type1 === common_schema_1.DATA_TYPE.FLOAT && type2 === common_schema_1.DATA_TYPE.INT) {
51
- type = common_schema_1.DATA_TYPE.FLOAT;
52
- }
53
- else if (type1 === common_schema_1.DATA_TYPE.LOCAL_DATE && type2 === common_schema_1.DATA_TYPE.STRING) {
54
- type = common_schema_1.DATA_TYPE.STRING;
55
- }
56
- else {
57
- // Type mismatch! Oj!
58
- return newField; // currently just use "latest" type
59
- }
60
- }
61
- // Type is same
62
- const minLen = existing.minLen !== undefined
63
- ? Math.min(...[existing.minLen, newField.minLen].filter(Boolean))
64
- : undefined;
65
- const maxLen = existing.maxLen !== undefined
66
- ? Math.max(...[existing.maxLen, newField.maxLen].filter(Boolean))
67
- : undefined;
68
- // todo: recursively merge/compare array/object schemas
69
- return (0, js_lib_1._filterNullishValues)({
70
- ...newField,
71
- type,
72
- minLen,
73
- maxLen,
74
- });
75
- }
76
- detectType(name, value, level = 1) {
77
- // Null
78
- if (value === undefined || value === null) {
79
- return {
80
- name,
81
- type: common_schema_1.DATA_TYPE.NULL,
82
- };
83
- }
84
- // String
85
- if (typeof value === 'string') {
86
- // LocalDate
87
- if (LOCAL_DATE_PATTERN.test(value)) {
88
- return { name, type: common_schema_1.DATA_TYPE.LOCAL_DATE };
89
- }
90
- return { name, type: common_schema_1.DATA_TYPE.STRING, minLen: value.length, maxLen: value.length };
91
- }
92
- // Int, Float
93
- if (typeof value === 'number') {
94
- // Cannot really detect TIMESTAMP
95
- return {
96
- name,
97
- type: Number.isInteger(value) ? common_schema_1.DATA_TYPE.INT : common_schema_1.DATA_TYPE.FLOAT,
98
- minLen: value,
99
- maxLen: value,
100
- };
101
- }
102
- // Boolean
103
- if (typeof value === 'boolean') {
104
- return { name, type: common_schema_1.DATA_TYPE.BOOLEAN };
105
- }
106
- // Binary
107
- if (Buffer.isBuffer(value)) {
108
- return { name, type: common_schema_1.DATA_TYPE.BINARY, minLen: value.length, maxLen: value.length };
109
- }
110
- // Array
111
- if (Array.isArray(value)) {
112
- return {
113
- name,
114
- type: common_schema_1.DATA_TYPE.ARRAY,
115
- arrayOf: this.detectType('', value[0], level + 1),
116
- };
117
- }
118
- // Object
119
- if (typeof value === 'object') {
120
- return {
121
- name,
122
- type: common_schema_1.DATA_TYPE.OBJECT,
123
- objectFields: Object.entries(value).map(([k, v]) => this.detectType(k, v, level + 1)),
124
- };
125
- }
126
- return { name, type: common_schema_1.DATA_TYPE.UNKNOWN };
127
- }
128
- generate() {
129
- // set nullability
130
- Object.keys(this.fieldByName).forEach(fieldName => {
131
- if (!this.nullableFields.has(fieldName)) {
132
- this.fieldByName[fieldName].notNull = true;
133
- }
134
- });
135
- const { table, sortedFields } = this.cfg;
136
- const fieldNames = Object.keys(this.fieldByName);
137
- if (sortedFields)
138
- fieldNames.sort(); // mutates
139
- // todo: sort object fields too
140
- return {
141
- table,
142
- fields: fieldNames.map(name => this.fieldByName[name]),
143
- };
144
- }
145
- static generateFromRows(cfg, rows = []) {
146
- const gen = new CommonSchemaGenerator(cfg);
147
- rows.forEach(r => gen.add(r));
148
- return gen.generate();
149
- }
150
- }
151
- exports.CommonSchemaGenerator = CommonSchemaGenerator;
@@ -1,49 +0,0 @@
1
- // eslint-disable-next-line unused-imports/no-unused-vars
2
- export interface CommonSchema<ROW = any> {
3
- /**
4
- * Name of the Table
5
- */
6
- table: string
7
-
8
- fields: CommonSchemaField[]
9
- // can possibly have `meta` with table meta-information
10
- }
11
-
12
- export interface CommonSchemaField {
13
- name: string
14
- type: DATA_TYPE
15
-
16
- /**
17
- * If it's an Array - then parentType should tell "array of what?"
18
- */
19
- arrayOf?: CommonSchemaField
20
-
21
- /**
22
- * If it's an Object - defined the CommonSchemaField of that object
23
- */
24
- objectFields?: CommonSchemaField[]
25
-
26
- notNull?: boolean
27
-
28
- /**
29
- * Applicable to certain fields, e.g String, to be able to autodetect limits for certain Databases
30
- */
31
- maxLen?: number
32
- minLen?: number
33
- // avgLen?: number
34
- }
35
-
36
- export enum DATA_TYPE {
37
- UNKNOWN = 'UNKNOWN',
38
- NULL = 'NULL',
39
- STRING = 'STRING',
40
- INT = 'INT',
41
- FLOAT = 'FLOAT',
42
- BOOLEAN = 'BOOLEAN',
43
- BINARY = 'BINARY',
44
- LOCAL_DATE = 'LOCAL_DATE', // ISO
45
- TIMESTAMP = 'TIMESTAMP', // unix timestamp
46
- // Semi-structured
47
- ARRAY = 'ARRAY',
48
- OBJECT = 'OBJECT',
49
- }
@@ -1,200 +0,0 @@
1
- import { ErrorMode, _filterNullishValues } from '@naturalcycles/js-lib'
2
- import { CommonSchema, CommonSchemaField, DATA_TYPE } from './common.schema'
3
-
4
- export interface CommonSchemaGeneratorCfg {
5
- /**
6
- * Name of the Table
7
- */
8
- table: string
9
-
10
- /**
11
- * @default SUPPRESS
12
- *
13
- * On error (field has multiple types):
14
- * if ErrorMode.THROW_IMMEDIATE - will throw on .add()
15
- * if ErrorMode.THROW_AGGREGATED - will throw on .generate()
16
- */
17
- errorMode?: ErrorMode
18
-
19
- /**
20
- * @default false
21
- * If true - fields will be sorted by name alphabetically (also inside objects)
22
- */
23
- sortedFields?: boolean
24
- }
25
-
26
- const LOCAL_DATE_PATTERN = new RegExp(/[0-9]{4}-[01][0-9]-[0-3][0-9]/)
27
-
28
- /**
29
- * Class that helps to generate CommonSchema by processing ALL rows through it.
30
- */
31
- export class CommonSchemaGenerator<ROW = any> {
32
- constructor(public cfg: CommonSchemaGeneratorCfg) {}
33
-
34
- private fieldByName: Record<string, CommonSchemaField> = {}
35
- private nullableFields = new Set<string>()
36
- // private fieldsWithMultipleTypes: Record<string, DATA_TYPE[]> = {}
37
-
38
- add(row: ROW): void {
39
- if (!row) return // safety
40
-
41
- Object.entries(row).forEach(([fieldName, value]) => {
42
- this.fieldByName[fieldName] = this.mergeFields(
43
- this.fieldByName[fieldName],
44
- this.detectType(fieldName, value),
45
- )
46
-
47
- if (this.fieldByName[fieldName]!.type === DATA_TYPE.NULL) {
48
- this.nullableFields.add(fieldName)
49
- }
50
- })
51
-
52
- // missing fields
53
- Object.keys(this.fieldByName).forEach(fieldName => {
54
- if (!(fieldName in row)) {
55
- // missing field!
56
- this.nullableFields.add(fieldName)
57
- }
58
- })
59
- }
60
-
61
- private mergeFields(
62
- existing: CommonSchemaField | undefined,
63
- newField: CommonSchemaField,
64
- ): CommonSchemaField {
65
- if (!existing) return newField
66
-
67
- if (newField.type === DATA_TYPE.UNKNOWN || newField.type === DATA_TYPE.NULL) {
68
- return {
69
- ...existing,
70
- }
71
- }
72
-
73
- if (existing.type === DATA_TYPE.UNKNOWN || existing.type === DATA_TYPE.NULL) {
74
- return {
75
- ...newField,
76
- }
77
- }
78
-
79
- let type = existing.type
80
-
81
- if (existing.type !== newField.type) {
82
- const [type1, type2] = [existing.type, newField.type].sort()
83
- if (type1 === DATA_TYPE.FLOAT && type2 === DATA_TYPE.INT) {
84
- type = DATA_TYPE.FLOAT
85
- } else if (type1 === DATA_TYPE.LOCAL_DATE && type2 === DATA_TYPE.STRING) {
86
- type = DATA_TYPE.STRING
87
- } else {
88
- // Type mismatch! Oj!
89
- return newField // currently just use "latest" type
90
- }
91
- }
92
-
93
- // Type is same
94
- const minLen =
95
- existing.minLen !== undefined
96
- ? Math.min(...[existing.minLen, newField.minLen!].filter(Boolean))
97
- : undefined
98
- const maxLen =
99
- existing.maxLen !== undefined
100
- ? Math.max(...[existing.maxLen, newField.maxLen!].filter(Boolean))
101
- : undefined
102
-
103
- // todo: recursively merge/compare array/object schemas
104
-
105
- return _filterNullishValues({
106
- ...newField,
107
- type,
108
- minLen,
109
- maxLen,
110
- })
111
- }
112
-
113
- private detectType(name: string, value: any, level = 1): CommonSchemaField {
114
- // Null
115
- if (value === undefined || value === null) {
116
- return {
117
- name,
118
- type: DATA_TYPE.NULL,
119
- }
120
- }
121
-
122
- // String
123
- if (typeof value === 'string') {
124
- // LocalDate
125
- if (LOCAL_DATE_PATTERN.test(value)) {
126
- return { name, type: DATA_TYPE.LOCAL_DATE }
127
- }
128
-
129
- return { name, type: DATA_TYPE.STRING, minLen: value.length, maxLen: value.length }
130
- }
131
-
132
- // Int, Float
133
- if (typeof value === 'number') {
134
- // Cannot really detect TIMESTAMP
135
- return {
136
- name,
137
- type: Number.isInteger(value) ? DATA_TYPE.INT : DATA_TYPE.FLOAT,
138
- minLen: value,
139
- maxLen: value,
140
- }
141
- }
142
-
143
- // Boolean
144
- if (typeof value === 'boolean') {
145
- return { name, type: DATA_TYPE.BOOLEAN }
146
- }
147
-
148
- // Binary
149
- if (Buffer.isBuffer(value)) {
150
- return { name, type: DATA_TYPE.BINARY, minLen: value.length, maxLen: value.length }
151
- }
152
-
153
- // Array
154
- if (Array.isArray(value)) {
155
- return {
156
- name,
157
- type: DATA_TYPE.ARRAY,
158
- arrayOf: this.detectType('', value[0], level + 1),
159
- }
160
- }
161
-
162
- // Object
163
- if (typeof value === 'object') {
164
- return {
165
- name,
166
- type: DATA_TYPE.OBJECT,
167
- objectFields: Object.entries(value).map(([k, v]) => this.detectType(k, v, level + 1)),
168
- }
169
- }
170
-
171
- return { name, type: DATA_TYPE.UNKNOWN }
172
- }
173
-
174
- generate(): CommonSchema<ROW> {
175
- // set nullability
176
- Object.keys(this.fieldByName).forEach(fieldName => {
177
- if (!this.nullableFields.has(fieldName)) {
178
- this.fieldByName[fieldName]!.notNull = true
179
- }
180
- })
181
-
182
- const { table, sortedFields } = this.cfg
183
-
184
- const fieldNames = Object.keys(this.fieldByName)
185
- if (sortedFields) fieldNames.sort() // mutates
186
-
187
- // todo: sort object fields too
188
-
189
- return {
190
- table,
191
- fields: fieldNames.map(name => this.fieldByName[name]!),
192
- }
193
- }
194
-
195
- static generateFromRows<ROW>(cfg: CommonSchemaGeneratorCfg, rows: ROW[] = []): CommonSchema<ROW> {
196
- const gen = new CommonSchemaGenerator(cfg)
197
- rows.forEach(r => gen.add(r))
198
- return gen.generate()
199
- }
200
- }