@naturalcycles/db-lib 8.11.0 → 8.13.0

package/src/testing/keyValueDaoTest.ts

@@ -40,8 +40,10 @@ export function runCommonKeyValueDaoTest(dao: CommonKeyValueDao<Buffer>): void {
 
   test('streamIds limited', async () => {
     const idsLimited = await readableToArray(dao.streamIds(2))
-    idsLimited.sort()
-    expect(idsLimited).toEqual(testIds.slice(0, 2))
+    // Order is non-deterministic, so, cannot compare values
+    // idsLimited.sort()
+    // expect(idsLimited).toEqual(testIds.slice(0, 2))
+    expect(idsLimited.length).toBe(2)
   })
 
   test('streamValues', async () => {
@@ -52,8 +54,9 @@ export function runCommonKeyValueDaoTest(dao: CommonKeyValueDao<Buffer>): void {
 
   test('streamValues limited', async () => {
     const valuesLimited = await readableToArray(dao.streamValues(2))
-    valuesLimited.sort()
-    expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
+    // valuesLimited.sort()
+    // expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
+    expect(valuesLimited.length).toBe(2)
   })
 
   test('streamEntries', async () => {
@@ -64,8 +67,9 @@ export function runCommonKeyValueDaoTest(dao: CommonKeyValueDao<Buffer>): void {
 
   test('streamEntries limited', async () => {
     const entriesLimited = await readableToArray(dao.streamEntries(2))
-    entriesLimited.sort()
-    expect(entriesLimited).toEqual(testEntries.slice(0, 2))
+    // entriesLimited.sort()
+    // expect(entriesLimited).toEqual(testEntries.slice(0, 2))
+    expect(entriesLimited.length).toBe(2)
   })
 
   test('deleteByIds should clear', async () => {

package/src/testing/test.model.ts

@@ -1,4 +1,4 @@
-import { _range } from '@naturalcycles/js-lib'
+import { jsonSchema, JsonSchemaObject, _range } from '@naturalcycles/js-lib'
 import {
   binarySchema,
   booleanSchema,
@@ -6,7 +6,6 @@ import {
   objectSchema,
   stringSchema,
 } from '@naturalcycles/nodejs-lib'
-import { CommonSchema, CommonSchemaGenerator } from '..'
 import { BaseDBEntity, baseDBEntitySchema, Saved, savedDBEntitySchema } from '../db.model'
 
 const MOCK_TS_2018_06_21 = 1529539200
@@ -49,6 +48,28 @@ export const testItemTMSchema = objectSchema<TestItemTM>({
   even: booleanSchema.optional(),
 })
 
+export const testItemBMJsonSchema = jsonSchema
+  .rootObject<TestItemBM>({
+    k1: jsonSchema.string(),
+    k2: jsonSchema.string().optional(),
+    k3: jsonSchema.number().optional(),
+    even: jsonSchema.boolean().optional(),
+    b1: jsonSchema.buffer().optional(),
+  })
+  .baseDBEntity()
+
+export const testItemDBMJsonSchema = jsonSchema.rootObject<TestItemDBM>({
+  // todo: figure out how to not copy-paste these 3 fields
+  id: jsonSchema.string(),
+  created: jsonSchema.unixTimestamp(),
+  updated: jsonSchema.unixTimestamp(),
+  k1: jsonSchema.string(),
+  k2: jsonSchema.string().optional(),
+  k3: jsonSchema.number().optional(),
+  even: jsonSchema.boolean().optional(),
+  b1: jsonSchema.buffer().optional(),
+})
+
 export function createTestItemDBM(num = 1): TestItemDBM {
   return {
     id: `id${num}`,
@@ -73,6 +94,19 @@ export function createTestItemsBM(count = 1): Saved<TestItemBM>[] {
   return _range(1, count + 1).map(num => createTestItemBM(num))
 }
 
-export function getTestItemSchema(): CommonSchema<TestItemDBM> {
-  return CommonSchemaGenerator.generateFromRows({ table: TEST_TABLE }, createTestItemsDBM())
+const testItemJsonSchema = jsonSchema
+  .object<TestItemDBM>({
+    id: jsonSchema.string(),
+    k1: jsonSchema.string(),
+    k2: jsonSchema.string(),
+    k3: jsonSchema.number(),
+    even: jsonSchema.boolean(),
+    created: jsonSchema.unixTimestamp(),
+    updated: jsonSchema.unixTimestamp(),
+  })
+  .build()
+
+export function getTestItemSchema(): JsonSchemaObject<TestItemDBM> {
+  // return CommonSchemaGenerator.generateFromRows({ table: TEST_TABLE }, createTestItemsDBM())
+  return testItemJsonSchema
 }

package/dist/schema/common.schema.d.ts

@@ -1,38 +0,0 @@
-export interface CommonSchema<ROW = any> {
-    /**
-     * Name of the Table
-     */
-    table: string;
-    fields: CommonSchemaField[];
-}
-export interface CommonSchemaField {
-    name: string;
-    type: DATA_TYPE;
-    /**
-     * If it's an Array - then parentType should tell "array of what?"
-     */
-    arrayOf?: CommonSchemaField;
-    /**
-     * If it's an Object - defined the CommonSchemaField of that object
-     */
-    objectFields?: CommonSchemaField[];
-    notNull?: boolean;
-    /**
-     * Applicable to certain fields, e.g String, to be able to autodetect limits for certain Databases
-     */
-    maxLen?: number;
-    minLen?: number;
-}
-export declare enum DATA_TYPE {
-    UNKNOWN = "UNKNOWN",
-    NULL = "NULL",
-    STRING = "STRING",
-    INT = "INT",
-    FLOAT = "FLOAT",
-    BOOLEAN = "BOOLEAN",
-    BINARY = "BINARY",
-    LOCAL_DATE = "LOCAL_DATE",
-    TIMESTAMP = "TIMESTAMP",
-    ARRAY = "ARRAY",
-    OBJECT = "OBJECT"
-}

package/dist/schema/common.schema.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DATA_TYPE = void 0;
-var DATA_TYPE;
-(function (DATA_TYPE) {
-    DATA_TYPE["UNKNOWN"] = "UNKNOWN";
-    DATA_TYPE["NULL"] = "NULL";
-    DATA_TYPE["STRING"] = "STRING";
-    DATA_TYPE["INT"] = "INT";
-    DATA_TYPE["FLOAT"] = "FLOAT";
-    DATA_TYPE["BOOLEAN"] = "BOOLEAN";
-    DATA_TYPE["BINARY"] = "BINARY";
-    DATA_TYPE["LOCAL_DATE"] = "LOCAL_DATE";
-    DATA_TYPE["TIMESTAMP"] = "TIMESTAMP";
-    // Semi-structured
-    DATA_TYPE["ARRAY"] = "ARRAY";
-    DATA_TYPE["OBJECT"] = "OBJECT";
-})(DATA_TYPE = exports.DATA_TYPE || (exports.DATA_TYPE = {}));

package/dist/schema/commonSchemaGenerator.d.ts

@@ -1,35 +0,0 @@
-import { ErrorMode } from '@naturalcycles/js-lib';
-import { CommonSchema } from './common.schema';
-export interface CommonSchemaGeneratorCfg {
-    /**
-     * Name of the Table
-     */
-    table: string;
-    /**
-     * @default SUPPRESS
-     *
-     * On error (field has multiple types):
-     * if ErrorMode.THROW_IMMEDIATE - will throw on .add()
-     * if ErrorMode.THROW_AGGREGATED - will throw on .generate()
-     */
-    errorMode?: ErrorMode;
-    /**
-     * @default false
-     * If true - fields will be sorted by name alphabetically (also inside objects)
-     */
-    sortedFields?: boolean;
-}
-/**
- * Class that helps to generate CommonSchema by processing ALL rows through it.
- */
-export declare class CommonSchemaGenerator<ROW = any> {
-    cfg: CommonSchemaGeneratorCfg;
-    constructor(cfg: CommonSchemaGeneratorCfg);
-    private fieldByName;
-    private nullableFields;
-    add(row: ROW): void;
-    private mergeFields;
-    private detectType;
-    generate(): CommonSchema<ROW>;
-    static generateFromRows<ROW>(cfg: CommonSchemaGeneratorCfg, rows?: ROW[]): CommonSchema<ROW>;
-}

package/dist/schema/commonSchemaGenerator.js

@@ -1,151 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CommonSchemaGenerator = void 0;
-const js_lib_1 = require("@naturalcycles/js-lib");
-const common_schema_1 = require("./common.schema");
-const LOCAL_DATE_PATTERN = new RegExp(/[0-9]{4}-[01][0-9]-[0-3][0-9]/);
-/**
- * Class that helps to generate CommonSchema by processing ALL rows through it.
- */
-class CommonSchemaGenerator {
-    constructor(cfg) {
-        this.cfg = cfg;
-        this.fieldByName = {};
-        this.nullableFields = new Set();
-    }
-    // private fieldsWithMultipleTypes: Record<string, DATA_TYPE[]> = {}
-    add(row) {
-        if (!row)
-            return; // safety
-        Object.entries(row).forEach(([fieldName, value]) => {
-            this.fieldByName[fieldName] = this.mergeFields(this.fieldByName[fieldName], this.detectType(fieldName, value));
-            if (this.fieldByName[fieldName].type === common_schema_1.DATA_TYPE.NULL) {
-                this.nullableFields.add(fieldName);
-            }
-        });
-        // missing fields
-        Object.keys(this.fieldByName).forEach(fieldName => {
-            if (!(fieldName in row)) {
-                // missing field!
-                this.nullableFields.add(fieldName);
-            }
-        });
-    }
-    mergeFields(existing, newField) {
-        if (!existing)
-            return newField;
-        if (newField.type === common_schema_1.DATA_TYPE.UNKNOWN || newField.type === common_schema_1.DATA_TYPE.NULL) {
-            return {
-                ...existing,
-            };
-        }
-        if (existing.type === common_schema_1.DATA_TYPE.UNKNOWN || existing.type === common_schema_1.DATA_TYPE.NULL) {
-            return {
-                ...newField,
-            };
-        }
-        let type = existing.type;
-        if (existing.type !== newField.type) {
-            const [type1, type2] = [existing.type, newField.type].sort();
-            if (type1 === common_schema_1.DATA_TYPE.FLOAT && type2 === common_schema_1.DATA_TYPE.INT) {
-                type = common_schema_1.DATA_TYPE.FLOAT;
-            }
-            else if (type1 === common_schema_1.DATA_TYPE.LOCAL_DATE && type2 === common_schema_1.DATA_TYPE.STRING) {
-                type = common_schema_1.DATA_TYPE.STRING;
-            }
-            else {
-                // Type mismatch! Oj!
-                return newField; // currently just use "latest" type
-            }
-        }
-        // Type is same
-        const minLen = existing.minLen !== undefined
-            ? Math.min(...[existing.minLen, newField.minLen].filter(Boolean))
-            : undefined;
-        const maxLen = existing.maxLen !== undefined
-            ? Math.max(...[existing.maxLen, newField.maxLen].filter(Boolean))
-            : undefined;
-        // todo: recursively merge/compare array/object schemas
-        return js_lib_1._filterNullishValues({
-            ...newField,
-            type,
-            minLen,
-            maxLen,
-        });
-    }
-    detectType(name, value, level = 1) {
-        // Null
-        if (value === undefined || value === null) {
-            return {
-                name,
-                type: common_schema_1.DATA_TYPE.NULL,
-            };
-        }
-        // String
-        if (typeof value === 'string') {
-            // LocalDate
-            if (LOCAL_DATE_PATTERN.test(value)) {
-                return { name, type: common_schema_1.DATA_TYPE.LOCAL_DATE };
-            }
-            return { name, type: common_schema_1.DATA_TYPE.STRING, minLen: value.length, maxLen: value.length };
-        }
-        // Int, Float
-        if (typeof value === 'number') {
-            // Cannot really detect TIMESTAMP
-            return {
-                name,
-                type: Number.isInteger(value) ? common_schema_1.DATA_TYPE.INT : common_schema_1.DATA_TYPE.FLOAT,
-                minLen: value,
-                maxLen: value,
-            };
-        }
-        // Boolean
-        if (typeof value === 'boolean') {
-            return { name, type: common_schema_1.DATA_TYPE.BOOLEAN };
-        }
-        // Binary
-        if (Buffer.isBuffer(value)) {
-            return { name, type: common_schema_1.DATA_TYPE.BINARY, minLen: value.length, maxLen: value.length };
-        }
-        // Array
-        if (Array.isArray(value)) {
-            return {
-                name,
-                type: common_schema_1.DATA_TYPE.ARRAY,
-                arrayOf: this.detectType('', value[0], level + 1),
-            };
-        }
-        // Object
-        if (typeof value === 'object') {
-            return {
-                name,
-                type: common_schema_1.DATA_TYPE.OBJECT,
-                objectFields: Object.entries(value).map(([k, v]) => this.detectType(k, v, level + 1)),
-            };
-        }
-        return { name, type: common_schema_1.DATA_TYPE.UNKNOWN };
-    }
-    generate() {
-        // set nullability
-        Object.keys(this.fieldByName).forEach(fieldName => {
-            if (!this.nullableFields.has(fieldName)) {
-                this.fieldByName[fieldName].notNull = true;
-            }
-        });
-        const { table, sortedFields } = this.cfg;
-        const fieldNames = Object.keys(this.fieldByName);
-        if (sortedFields)
-            fieldNames.sort(); // mutates
-        // todo: sort object fields too
-        return {
-            table,
-            fields: fieldNames.map(name => this.fieldByName[name]),
-        };
-    }
-    static generateFromRows(cfg, rows = []) {
-        const gen = new CommonSchemaGenerator(cfg);
-        rows.forEach(r => gen.add(r));
-        return gen.generate();
-    }
-}
-exports.CommonSchemaGenerator = CommonSchemaGenerator;

package/src/schema/common.schema.ts

@@ -1,49 +0,0 @@
-// eslint-disable-next-line unused-imports/no-unused-vars
-export interface CommonSchema<ROW = any> {
-  /**
-   * Name of the Table
-   */
-  table: string
-
-  fields: CommonSchemaField[]
-  // can possibly have `meta` with table meta-information
-}
-
-export interface CommonSchemaField {
-  name: string
-  type: DATA_TYPE
-
-  /**
-   * If it's an Array - then parentType should tell "array of what?"
-   */
-  arrayOf?: CommonSchemaField
-
-  /**
-   * If it's an Object - defined the CommonSchemaField of that object
-   */
-  objectFields?: CommonSchemaField[]
-
-  notNull?: boolean
-
-  /**
-   * Applicable to certain fields, e.g String, to be able to autodetect limits for certain Databases
-   */
-  maxLen?: number
-  minLen?: number
-  // avgLen?: number
-}
-
-export enum DATA_TYPE {
-  UNKNOWN = 'UNKNOWN',
-  NULL = 'NULL',
-  STRING = 'STRING',
-  INT = 'INT',
-  FLOAT = 'FLOAT',
-  BOOLEAN = 'BOOLEAN',
-  BINARY = 'BINARY',
-  LOCAL_DATE = 'LOCAL_DATE', // ISO
-  TIMESTAMP = 'TIMESTAMP', // unix timestamp
-  // Semi-structured
-  ARRAY = 'ARRAY',
-  OBJECT = 'OBJECT',
-}

package/src/schema/commonSchemaGenerator.ts

@@ -1,200 +0,0 @@
-import { ErrorMode, _filterNullishValues } from '@naturalcycles/js-lib'
-import { CommonSchema, CommonSchemaField, DATA_TYPE } from './common.schema'
-
-export interface CommonSchemaGeneratorCfg {
-  /**
-   * Name of the Table
-   */
-  table: string
-
-  /**
-   * @default SUPPRESS
-   *
-   * On error (field has multiple types):
-   * if ErrorMode.THROW_IMMEDIATE - will throw on .add()
-   * if ErrorMode.THROW_AGGREGATED - will throw on .generate()
-   */
-  errorMode?: ErrorMode
-
-  /**
-   * @default false
-   * If true - fields will be sorted by name alphabetically (also inside objects)
-   */
-  sortedFields?: boolean
-}
-
-const LOCAL_DATE_PATTERN = new RegExp(/[0-9]{4}-[01][0-9]-[0-3][0-9]/)
-
-/**
- * Class that helps to generate CommonSchema by processing ALL rows through it.
- */
-export class CommonSchemaGenerator<ROW = any> {
-  constructor(public cfg: CommonSchemaGeneratorCfg) {}
-
-  private fieldByName: Record<string, CommonSchemaField> = {}
-  private nullableFields = new Set<string>()
-  // private fieldsWithMultipleTypes: Record<string, DATA_TYPE[]> = {}
-
-  add(row: ROW): void {
-    if (!row) return // safety
-
-    Object.entries(row).forEach(([fieldName, value]) => {
-      this.fieldByName[fieldName] = this.mergeFields(
-        this.fieldByName[fieldName],
-        this.detectType(fieldName, value),
-      )
-
-      if (this.fieldByName[fieldName]!.type === DATA_TYPE.NULL) {
-        this.nullableFields.add(fieldName)
-      }
-    })
-
-    // missing fields
-    Object.keys(this.fieldByName).forEach(fieldName => {
-      if (!(fieldName in row)) {
-        // missing field!
-        this.nullableFields.add(fieldName)
-      }
-    })
-  }
-
-  private mergeFields(
-    existing: CommonSchemaField | undefined,
-    newField: CommonSchemaField,
-  ): CommonSchemaField {
-    if (!existing) return newField
-
-    if (newField.type === DATA_TYPE.UNKNOWN || newField.type === DATA_TYPE.NULL) {
-      return {
-        ...existing,
-      }
-    }
-
-    if (existing.type === DATA_TYPE.UNKNOWN || existing.type === DATA_TYPE.NULL) {
-      return {
-        ...newField,
-      }
-    }
-
-    let type = existing.type
-
-    if (existing.type !== newField.type) {
-      const [type1, type2] = [existing.type, newField.type].sort()
-      if (type1 === DATA_TYPE.FLOAT && type2 === DATA_TYPE.INT) {
-        type = DATA_TYPE.FLOAT
-      } else if (type1 === DATA_TYPE.LOCAL_DATE && type2 === DATA_TYPE.STRING) {
-        type = DATA_TYPE.STRING
-      } else {
-        // Type mismatch! Oj!
-        return newField // currently just use "latest" type
-      }
-    }
-
-    // Type is same
-    const minLen =
-      existing.minLen !== undefined
-        ? Math.min(...[existing.minLen, newField.minLen!].filter(Boolean))
-        : undefined
-    const maxLen =
-      existing.maxLen !== undefined
-        ? Math.max(...[existing.maxLen, newField.maxLen!].filter(Boolean))
-        : undefined
-
-    // todo: recursively merge/compare array/object schemas
-
-    return _filterNullishValues({
-      ...newField,
-      type,
-      minLen,
-      maxLen,
-    })
-  }
-
-  private detectType(name: string, value: any, level = 1): CommonSchemaField {
-    // Null
-    if (value === undefined || value === null) {
-      return {
-        name,
-        type: DATA_TYPE.NULL,
-      }
-    }
-
-    // String
-    if (typeof value === 'string') {
-      // LocalDate
-      if (LOCAL_DATE_PATTERN.test(value)) {
-        return { name, type: DATA_TYPE.LOCAL_DATE }
-      }
-
-      return { name, type: DATA_TYPE.STRING, minLen: value.length, maxLen: value.length }
-    }
-
-    // Int, Float
-    if (typeof value === 'number') {
-      // Cannot really detect TIMESTAMP
-      return {
-        name,
-        type: Number.isInteger(value) ? DATA_TYPE.INT : DATA_TYPE.FLOAT,
-        minLen: value,
-        maxLen: value,
-      }
-    }
-
-    // Boolean
-    if (typeof value === 'boolean') {
-      return { name, type: DATA_TYPE.BOOLEAN }
-    }
-
-    // Binary
-    if (Buffer.isBuffer(value)) {
-      return { name, type: DATA_TYPE.BINARY, minLen: value.length, maxLen: value.length }
-    }
-
-    // Array
-    if (Array.isArray(value)) {
-      return {
-        name,
-        type: DATA_TYPE.ARRAY,
-        arrayOf: this.detectType('', value[0], level + 1),
-      }
-    }
-
-    // Object
-    if (typeof value === 'object') {
-      return {
-        name,
-        type: DATA_TYPE.OBJECT,
-        objectFields: Object.entries(value).map(([k, v]) => this.detectType(k, v, level + 1)),
-      }
-    }
-
-    return { name, type: DATA_TYPE.UNKNOWN }
-  }
-
-  generate(): CommonSchema<ROW> {
-    // set nullability
-    Object.keys(this.fieldByName).forEach(fieldName => {
-      if (!this.nullableFields.has(fieldName)) {
-        this.fieldByName[fieldName]!.notNull = true
-      }
-    })
-
-    const { table, sortedFields } = this.cfg
-
-    const fieldNames = Object.keys(this.fieldByName)
-    if (sortedFields) fieldNames.sort() // mutates
-
-    // todo: sort object fields too
-
-    return {
-      table,
-      fields: fieldNames.map(name => this.fieldByName[name]!),
-    }
-  }
-
-  static generateFromRows<ROW>(cfg: CommonSchemaGeneratorCfg, rows: ROW[] = []): CommonSchema<ROW> {
-    const gen = new CommonSchemaGenerator(cfg)
-    rows.forEach(r => gen.add(r))
-    return gen.generate()
-  }
-}