@naturalcycles/db-lib 8.11.0 → 8.13.0

package/dist/transaction/dbTransaction.util.js

@@ -29,10 +29,10 @@ function mergeDBOperations(ops) {
         }
     });
     const resultOps = [];
-    js_lib_1._stringMapEntries(saveMapByTable).forEach(([table, saveMap]) => {
+    (0, js_lib_1._stringMapEntries)(saveMapByTable).forEach(([table, saveMap]) => {
         const rowsToSave = [];
         const idsToDelete = [];
-        js_lib_1._stringMapEntries(saveMap).forEach(([id, r]) => {
+        (0, js_lib_1._stringMapEntries)(saveMap).forEach(([id, r]) => {
             if (r === null) {
                 idsToDelete.push(id);
             }

package/dist/validation/index.js

@@ -3,30 +3,30 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.dbQuerySchema = exports.dbQueryOrderSchema = exports.dbQueryFilterSchema = exports.dbQueryFilterOperatorSchema = exports.commonDBSaveOptionsSchema = exports.commonDBOptionsSchema = void 0;
 const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const dbQuery_1 = require("../query/dbQuery");
-exports.commonDBOptionsSchema = nodejs_lib_1.objectSchema({
+exports.commonDBOptionsSchema = (0, nodejs_lib_1.objectSchema)({
     onlyCache: nodejs_lib_1.booleanSchema.optional(),
     skipCache: nodejs_lib_1.booleanSchema.optional(),
 });
-exports.commonDBSaveOptionsSchema = nodejs_lib_1.objectSchema({
-    excludeFromIndexes: nodejs_lib_1.arraySchema(nodejs_lib_1.stringSchema).optional(),
+exports.commonDBSaveOptionsSchema = (0, nodejs_lib_1.objectSchema)({
+    excludeFromIndexes: (0, nodejs_lib_1.arraySchema)(nodejs_lib_1.stringSchema).optional(),
 }).concat(exports.commonDBOptionsSchema);
 exports.dbQueryFilterOperatorSchema = nodejs_lib_1.stringSchema.valid(...dbQuery_1.DBQueryFilterOperatorValues);
-exports.dbQueryFilterSchema = nodejs_lib_1.objectSchema({
+exports.dbQueryFilterSchema = (0, nodejs_lib_1.objectSchema)({
     name: nodejs_lib_1.stringSchema,
     op: exports.dbQueryFilterOperatorSchema,
     val: nodejs_lib_1.anySchema,
 });
-exports.dbQueryOrderSchema = nodejs_lib_1.objectSchema({
+exports.dbQueryOrderSchema = (0, nodejs_lib_1.objectSchema)({
     name: nodejs_lib_1.stringSchema,
     descending: nodejs_lib_1.booleanSchema.optional(),
 });
-exports.dbQuerySchema = nodejs_lib_1.objectSchema({
+exports.dbQuerySchema = (0, nodejs_lib_1.objectSchema)({
     table: nodejs_lib_1.stringSchema,
-    _filters: nodejs_lib_1.arraySchema(exports.dbQueryFilterSchema).optional(),
+    _filters: (0, nodejs_lib_1.arraySchema)(exports.dbQueryFilterSchema).optional(),
     _limitValue: nodejs_lib_1.integerSchema.min(0).optional(),
     _offsetValue: nodejs_lib_1.integerSchema.min(0).optional(),
-    _orders: nodejs_lib_1.arraySchema(exports.dbQueryOrderSchema).optional(),
+    _orders: (0, nodejs_lib_1.arraySchema)(exports.dbQueryOrderSchema).optional(),
     _startCursor: nodejs_lib_1.stringSchema.optional(),
     _endCursor: nodejs_lib_1.stringSchema.optional(),
-    _selectedFieldNames: nodejs_lib_1.arraySchema(nodejs_lib_1.stringSchema).optional(),
+    _selectedFieldNames: (0, nodejs_lib_1.arraySchema)(nodejs_lib_1.stringSchema).optional(),
 });

package/package.json

@@ -42,7 +42,7 @@
   "engines": {
     "node": ">=12.13"
   },
-  "version": "8.11.0",
+  "version": "8.13.0",
   "description": "Lowest Common Denominator API to supported Databases",
   "keywords": [
     "db",

package/readme.md

@@ -216,19 +216,20 @@ await db.getTables()
 
 ###### getTableSchema
 
-`getTableSchema(table: string): Promise<CommonSchema>`
+`getTableSchema(table: string): Promise<JsonSchemaObject>`
 
 ```typescript
 await db.getTableSchema('table1')
-// todo: describe CommonSchema example
 ```
 
+Returns a JsonSchema, generated from the table.
+
 ###### createTable
 
-`createTable(schema: CommonSchema): Promise<void>`
+`createTable(table: string, schema: JsonSchemaObject): Promise<void>`
 
 Applicable to Relational DBs, like MySQL. Will invoke smth like `create table Table1 ... ;`. Takes a
-`CommonSchema` as an argument.
+`JsonSchema` as an argument.
 
 # DBQuery
 

package/src/adapter/cachedb/cache.db.ts

@@ -1,10 +1,10 @@
+import { JsonSchemaObject } from '@naturalcycles/js-lib'
 import { Debug, IDebugger } from '@naturalcycles/nodejs-lib'
 import { Readable } from 'stream'
 import { BaseCommonDB } from '../../base.common.db'
 import { CommonDB } from '../../common.db'
 import { ObjectWithId, RunQueryResult } from '../../db.model'
 import { DBQuery } from '../../query/dbQuery'
-import { CommonSchema } from '../../schema/common.schema'
 import {
   CacheDBCfg,
   CacheDBCreateOptions,
@@ -45,17 +45,21 @@ export class CacheDB extends BaseCommonDB implements CommonDB {
 
   override async getTableSchema<ROW extends ObjectWithId>(
     table: string,
-  ): Promise<CommonSchema<ROW>> {
+  ): Promise<JsonSchemaObject<ROW>> {
     return await this.cfg.downstreamDB.getTableSchema<ROW>(table)
   }
 
-  override async createTable(schema: CommonSchema, opt: CacheDBCreateOptions = {}): Promise<void> {
+  override async createTable(
+    table: string,
+    schema: JsonSchemaObject,
+    opt: CacheDBCreateOptions = {},
+  ): Promise<void> {
     if (!opt.onlyCache && !this.cfg.onlyCache) {
-      await this.cfg.downstreamDB.createTable(schema, opt)
+      await this.cfg.downstreamDB.createTable(table, schema, opt)
     }
 
     if (!opt.skipCache && !this.cfg.skipCache) {
-      await this.cfg.cacheDB.createTable(schema, opt)
+      await this.cfg.cacheDB.createTable(table, schema, opt)
     }
   }
 

package/src/adapter/file/file.db.ts

@@ -1,4 +1,6 @@
 import {
+  generateJsonSchemaFromData,
+  JsonSchemaObject,
   pMap,
   StringMap,
   _by,
@@ -12,8 +14,6 @@ import {
 import { Debug, readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { dimGrey } from '@naturalcycles/nodejs-lib/dist/colors'
 import { BaseCommonDB, DBSaveBatchOperation, ObjectWithId, queryInMemory } from '../..'
-import { CommonSchema } from '../..'
-import { CommonSchemaGenerator } from '../..'
 import { CommonDB } from '../../common.db'
 import {
   CommonDBOptions,
@@ -213,9 +213,12 @@ export class FileDB extends BaseCommonDB implements CommonDB {
 
   override async getTableSchema<ROW extends ObjectWithId>(
     table: string,
-  ): Promise<CommonSchema<ROW>> {
+  ): Promise<JsonSchemaObject<ROW>> {
     const rows = await this.loadFile(table)
-    return CommonSchemaGenerator.generateFromRows({ table }, rows)
+    return {
+      ...generateJsonSchemaFromData(rows),
+      $id: `${table}.schema.json`,
+    } as JsonSchemaObject<ROW>
   }
 
   // wrapper, to handle logging

package/src/adapter/inmemory/inMemory.db.ts

@@ -1,4 +1,12 @@
-import { pMap, StringMap, _by, _since, _sortObjectDeep } from '@naturalcycles/js-lib'
+import {
+  generateJsonSchemaFromData,
+  JsonSchemaObject,
+  pMap,
+  StringMap,
+  _by,
+  _since,
+  _sortObjectDeep,
+} from '@naturalcycles/js-lib'
 import {
   bufferReviver,
   Debug,
@@ -14,8 +22,6 @@ import * as fs from 'fs-extra'
 import { Readable } from 'stream'
 import { createGzip, createUnzip } from 'zlib'
 import { CommonDB, DBTransaction, ObjectWithId, queryInMemory } from '../..'
-import { CommonSchema } from '../..'
-import { CommonSchemaGenerator } from '../..'
 import {
   CommonDBCreateOptions,
   CommonDBOptions,
@@ -105,13 +111,20 @@ export class InMemoryDB implements CommonDB {
     return Object.keys(this.data).filter(t => t.startsWith(this.cfg.tablesPrefix))
   }
 
-  async getTableSchema<ROW extends ObjectWithId>(_table: string): Promise<CommonSchema<ROW>> {
+  async getTableSchema<ROW extends ObjectWithId>(_table: string): Promise<JsonSchemaObject<ROW>> {
     const table = this.cfg.tablesPrefix + _table
-    return CommonSchemaGenerator.generateFromRows({ table }, Object.values(this.data[table] || {}))
+    return {
+      ...generateJsonSchemaFromData(Object.values(this.data[table] || {})),
+      $id: `${table}.schema.json`,
+    } as JsonSchemaObject<ROW>
   }
 
-  async createTable(schema: CommonSchema, opt: CommonDBCreateOptions = {}): Promise<void> {
-    const table = this.cfg.tablesPrefix + schema.table
+  async createTable(
+    _table: string,
+    _schema: JsonSchemaObject,
+    opt: CommonDBCreateOptions = {},
+  ): Promise<void> {
+    const table = this.cfg.tablesPrefix + _table
     if (opt.dropIfExists) {
       this.data[table] = {}
     } else {

package/src/base.common.db.ts

@@ -1,9 +1,9 @@
+import { JsonSchemaObject } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { Readable } from 'stream'
 import { CommonDB } from './common.db'
 import { CommonDBSaveOptions, ObjectWithId, RunQueryResult } from './db.model'
 import { DBQuery } from './query/dbQuery'
-import { CommonSchema } from './schema/common.schema'
 import { DBTransaction } from './transaction/dbTransaction'
 import { commitDBTransactionSimple } from './transaction/dbTransaction.util'
 
@@ -18,11 +18,17 @@ export class BaseCommonDB implements CommonDB {
     return []
   }
 
-  async getTableSchema<ROW>(table: string): Promise<CommonSchema<ROW>> {
-    return { table, fields: [] }
+  async getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaObject<ROW>> {
+    return {
+      $id: `${table}.schema.json`,
+      type: 'object',
+      additionalProperties: true,
+      properties: {} as any,
+      required: [],
+    }
   }
 
-  async createTable(_schema: CommonSchema): Promise<void> {}
+  async createTable(_table: string, _schema: JsonSchemaObject): Promise<void> {}
 
   async deleteByIds(_table: string, _ids: string[]): Promise<number> {
     return 0

package/src/common.db.ts

@@ -1,3 +1,4 @@
+import { JsonSchemaObject } from '@naturalcycles/js-lib'
 import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import {
   CommonDBCreateOptions,
@@ -8,7 +9,6 @@ import {
   RunQueryResult,
 } from './db.model'
 import { DBQuery } from './query/dbQuery'
-import { CommonSchema } from './schema/common.schema'
 import { DBTransaction } from './transaction/dbTransaction'
 
 export interface CommonDB {
@@ -25,13 +25,13 @@ export interface CommonDB {
    */
   getTables(): Promise<string[]>
 
-  getTableSchema<ROW extends ObjectWithId>(table: string): Promise<CommonSchema<ROW>>
+  getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaObject<ROW>>
 
   /**
    * Will do like `create table ...` for mysql.
    * Caution! dropIfExists defaults to false. If set to true - will actually DROP the table!
    */
-  createTable(schema: CommonSchema, opt?: CommonDBCreateOptions): Promise<void>
+  createTable(table: string, schema: JsonSchemaObject, opt?: CommonDBCreateOptions): Promise<void>
 
   // GET
   /**

package/src/commondao/common.dao.model.ts

@@ -1,8 +1,11 @@
 import { ErrorMode } from '@naturalcycles/js-lib'
 import {
+  AjvSchema,
+  AjvValidationError,
   JoiValidationError,
   ObjectSchemaTyped,
   TransformLogProgressOptions,
+  TransformMapOptions,
 } from '@naturalcycles/nodejs-lib'
 import { CommonDB } from '../common.db'
 import {
@@ -16,11 +19,11 @@ import {
 export type CommonDaoCreateIdHook<BM, DBM> = (obj: DBM | BM) => string
 export type CommonDaoParseNaturalIdHook<DBM> = (id: string) => Partial<DBM>
 export type CommonDaoBeforeCreateHook<BM> = (bm: Partial<BM>) => BM
-export type CommonDaoBeforeDBMValidateHook<DBM> = (dbm: DBM) => DBM
-export type CommonDaoBeforeDBMToBMHook<BM, DBM> = (dbm: DBM) => BM
-export type CommonDaoBeforeBMToDBMHook<BM, DBM> = (bm: BM) => DBM
-export type CommonDaoBeforeTMToBMHook<BM, TM> = (tm: TM) => BM
-export type CommonDaoBeforeBMToTMHook<BM, TM> = (bm: BM) => TM
+export type CommonDaoBeforeDBMValidateHook<DBM> = (dbm: Partial<DBM>) => Partial<DBM>
+export type CommonDaoBeforeDBMToBMHook<BM, DBM> = (dbm: DBM) => Partial<BM>
+export type CommonDaoBeforeBMToDBMHook<BM, DBM> = (bm: BM) => Partial<DBM>
+export type CommonDaoBeforeTMToBMHook<BM, TM> = (tm: TM) => Partial<BM>
+export type CommonDaoBeforeBMToTMHook<BM, TM> = (bm: BM) => Partial<TM>
 export type CommonDaoAnonymizeHook<DBM> = (dbm: DBM) => DBM
 
 interface CommonDaoHooks<BM, DBM, TM> {
@@ -33,6 +36,14 @@ interface CommonDaoHooks<BM, DBM, TM> {
   beforeTMToBM: CommonDaoBeforeTMToBMHook<BM, TM>
   beforeBMToTM: CommonDaoBeforeBMToTMHook<BM, TM>
   anonymize: CommonDaoAnonymizeHook<DBM>
+
+  /**
+   * If hook is defined - allows to prevent or modify the error thrown.
+   * Return `false` to prevent throwing an error.
+   * Return original `err` to pass the error through (will be thrown in CommonDao).
+   * Return modified/new `Error` if needed.
+   */
+  onValidationError: (err: JoiValidationError | AjvValidationError) => Error | false
 }
 
 export enum CommonDaoLogLevel {
@@ -48,9 +59,13 @@ export enum CommonDaoLogLevel {
 export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
   db: CommonDB
   table: string
-  dbmSchema?: ObjectSchemaTyped<DBM>
-  bmSchema?: ObjectSchemaTyped<BM>
-  tmSchema?: ObjectSchemaTyped<TM>
+
+  /**
+   * Joi or AjvSchema are supported.
+   */
+  dbmSchema?: ObjectSchemaTyped<DBM> | AjvSchema<DBM>
+  bmSchema?: ObjectSchemaTyped<BM> | AjvSchema<BM>
+  tmSchema?: ObjectSchemaTyped<TM> | AjvSchema<TM>
 
   excludeFromIndexes?: string[]
 
@@ -70,22 +85,6 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends Obje
    */
   logStarted?: boolean
 
-  /**
-   * @default false
-   */
-  throwOnEntityValidationError?: boolean
-
-  /**
-   * @default to throwOnEntityValidationError setting
-   */
-  throwOnDaoCreateObject?: boolean
-
-  /**
-   * Called when validation error occurs.
-   * Called ONLY when error is NOT thrown (when throwOnEntityValidationError is off)
-   */
-  onValidationError?: (err: JoiValidationError) => any
-
   // Hooks are designed with inspiration from got/ky interface
   hooks?: Partial<CommonDaoHooks<BM, DBM, TM>>
 
@@ -127,11 +126,6 @@ export interface CommonDaoOptions extends CommonDBOptions {
    */
   raw?: boolean
 
-  /**
-   * @default inherited from CommonDaoCfg.throwOnEntityValidationError
-   */
-  throwOnError?: boolean
-
   /**
    * @default false
    */
@@ -170,6 +164,7 @@ export interface CommonDaoSaveOptions extends CommonDaoOptions, CommonDBSaveOpti
 
 export interface CommonDaoStreamForEachOptions<IN>
   extends CommonDaoStreamOptions,
+    TransformMapOptions<IN, any>,
     TransformLogProgressOptions<IN> {}
 
 export interface CommonDaoStreamOptions extends CommonDaoOptions {

package/src/commondao/common.dao.ts

@@ -2,14 +2,18 @@ import {
   AppError,
   AsyncMapper,
   ErrorMode,
+  JsonSchemaObject,
   _filterNullishValues,
   _passthroughPredicate,
   _since,
   _truncate,
 } from '@naturalcycles/js-lib'
 import {
+  AjvSchema,
+  AjvValidationError,
   Debug,
   getValidationResult,
+  JoiValidationError,
   ObjectSchemaTyped,
   ReadableTyped,
   stringId,
@@ -24,7 +28,6 @@ import {
 import { DBLibError } from '../cnst'
 import { DBModelType, ObjectWithId, RunQueryResult, Saved } from '../db.model'
 import { DBQuery, RunnableDBQuery } from '../query/dbQuery'
-import { CommonSchema } from '../schema/common.schema'
 import {
   CommonDaoCfg,
   CommonDaoCreateOptions,
@@ -66,6 +69,7 @@ export class CommonDao<
         beforeTMToBM: tm => tm as any,
         beforeBMToTM: bm => bm as any,
         anonymize: dbm => dbm,
+        onValidationError: err => err,
         ...cfg.hooks,
       },
     }
@@ -73,10 +77,6 @@ export class CommonDao<
 
   // CREATE
   create(input: Partial<BM>, opt: CommonDaoOptions = {}): Saved<BM> {
-    if (opt.throwOnError === undefined) {
-      opt.throwOnError = this.cfg.throwOnDaoCreateObject
-    }
-
     let bm = this.cfg.hooks!.beforeCreate!(input)
     bm = this.validateAndConvert(bm, this.cfg.bmSchema, DBModelType.BM, opt)
 
@@ -804,7 +804,7 @@ export class CommonDao<
     // tm = this.validateAndConvert(tm, this.cfg.tmSchema, DBModelType.TM, opt)
 
     // TM > BM
-    const bm = this.cfg.hooks!.beforeTMToBM!(tm)
+    const bm = this.cfg.hooks!.beforeTMToBM!(tm) as BM
 
     // Validate/convert BM
     return this.validateAndConvert<BM>(bm, this.cfg.bmSchema, DBModelType.BM, opt)
@@ -818,11 +818,12 @@ export class CommonDao<
   /**
    * Returns *converted value*.
    * Validates (unless `skipValidation=true` passed).
-   * Throws only if `throwOnError=true` passed OR if `env().throwOnEntityValidationError`
+   *
+   * Does NOT mutate the object.
    */
-  validateAndConvert<IN = any, OUT = IN>(
+  validateAndConvert<IN, OUT = IN>(
     obj: IN,
-    schema?: ObjectSchemaTyped<IN>,
+    schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>,
     modelType?: DBModelType,
     opt: CommonDaoOptions = {},
   ): OUT {
@@ -844,32 +845,42 @@ export class CommonDao<
 
     // This will Convert and Validate
     const table = opt.table || this.cfg.table
-    const { value, error } = getValidationResult<IN, OUT>(obj, schema, table + (modelType || ''))
+    const objectName = table + (modelType || '')
+
+    let error: JoiValidationError | AjvValidationError | undefined
+    let convertedValue: any
+
+    if (schema instanceof AjvSchema) {
+      // Ajv schema
+      convertedValue = obj // because Ajv mutates original object
+
+      error = schema.getValidationError(obj, {
+        objectName,
+      })
+    } else {
+      // Joi
+      const vr = getValidationResult<IN, OUT>(obj, schema, objectName)
+      error = vr.error
+      convertedValue = vr.value
+    }
 
     // If we care about validation and there's an error
     if (error && !opt.skipValidation) {
-      if (
-        opt.throwOnError ||
-        (this.cfg.throwOnEntityValidationError && opt.throwOnError === undefined)
-      ) {
-        throw error
-      } else {
-        // capture by Sentry and ignore the error
-        // It will still *convert* the value and return.
-        this.cfg.onValidationError?.(error)
-      }
+      const processedError = this.cfg.hooks!.onValidationError!(error)
+
+      if (processedError) throw processedError
     }
 
-    return value // converted value
+    return convertedValue
   }
 
-  async getTableSchema(): Promise<CommonSchema> {
+  async getTableSchema(): Promise<JsonSchemaObject<DBM>> {
     return await this.cfg.db.getTableSchema<DBM>(this.cfg.table)
   }
 
-  async createTable(schema: CommonSchema, opt?: CommonDaoCreateOptions): Promise<void> {
+  async createTable(schema: JsonSchemaObject<DBM>, opt?: CommonDaoCreateOptions): Promise<void> {
     this.requireWriteAccess()
-    await this.cfg.db.createTable(schema, opt)
+    await this.cfg.db.createTable(this.cfg.table, schema as any, opt)
   }
 
   /**

package/src/index.ts

@@ -67,8 +67,6 @@ import {
   DBQueryOrder,
   RunnableDBQuery,
 } from './query/dbQuery'
-import { CommonSchema, CommonSchemaField, DATA_TYPE } from './schema/common.schema'
-import { CommonSchemaGenerator, CommonSchemaGeneratorCfg } from './schema/commonSchemaGenerator'
 import { DBTransaction, RunnableDBTransaction } from './transaction/dbTransaction'
 import { commitDBTransactionSimple, mergeDBOperations } from './transaction/dbTransaction.util'
 
@@ -110,9 +108,6 @@ export type {
   DBPipelineBackupOptions,
   DBPipelineRestoreOptions,
   DBPipelineCopyOptions,
-  CommonSchema,
-  CommonSchemaField,
-  CommonSchemaGeneratorCfg,
   CommonDBAdapter,
   DBOperation,
   DBSaveBatchOperation,
@@ -143,7 +138,6 @@ export {
   dbPipelineBackup,
   dbPipelineRestore,
   dbPipelineCopy,
-  DATA_TYPE,
   getDB,
   DBLibError,
   BaseCommonDB,
@@ -151,6 +145,5 @@ export {
   RunnableDBTransaction,
   mergeDBOperations,
   commitDBTransactionSimple,
-  CommonSchemaGenerator,
   CommonKeyValueDao,
 }

package/src/pipeline/dbPipelineBackup.ts

@@ -15,7 +15,6 @@ import * as fs from 'fs-extra'
 import { createGzip, ZlibOptions } from 'zlib'
 import { CommonDB } from '../common.db'
 import { DBQuery } from '../index'
-import { CommonSchemaGenerator } from '../schema/commonSchemaGenerator'
 
 export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
   /**
@@ -102,7 +101,7 @@ export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
    * @default false
    * If true - will use CommonSchemaGenerator to detect schema from input data.
    */
-  emitSchemaFromData?: boolean
+  // emitSchemaFromData?: boolean
 
   /**
    * @default false
@@ -140,7 +139,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
     transformMapOptions,
     errorMode = ErrorMode.SUPPRESS,
     emitSchemaFromDB = false,
-    emitSchemaFromData = false,
     sortObjects = false,
   } = opt
   const strict = errorMode !== ErrorMode.SUPPRESS
@@ -193,10 +191,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
         console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
       }
 
-      const schemaGen = emitSchemaFromData
-        ? new CommonSchemaGenerator({ table, sortedFields: true })
-        : undefined
-
       await _pipeline([
         db.streamQuery(q),
         transformLogProgress({
@@ -210,9 +204,8 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
           ...transformMapOptions,
           metric: table,
         }),
-        transformTap(row => {
+        transformTap(() => {
           rows++
-          if (schemaGen) schemaGen.add(row)
         }),
         transformToNDJson({ strict, sortObjects }),
         ...(gzip ? [createGzip(zlibOptions)] : []), // optional gzip
@@ -229,12 +222,6 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
 
       console.log(`>> ${grey(filePath)}\n` + stats.toPretty())
 
-      if (schemaGen) {
-        const schema = schemaGen.generate()
-        await fs.writeJson(schemaFilePath, schema, { spaces: 2 })
-        console.log(`>> ${grey(schemaFilePath)} saved (generated from data)`)
-      }
-
       statsPerTable[table] = stats
     },
     { concurrency, errorMode },

package/src/pipeline/dbPipelineRestore.ts

@@ -179,7 +179,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
       }
 
       const schema = await fs.readJson(schemaFilePath)
-      await db.createTable(schema, { dropIfExists: true })
+      await db.createTable(table, schema, { dropIfExists: true })
     })
   }
 

package/src/testing/dbTest.ts

@@ -97,7 +97,7 @@ export function runCommonDBTest(
   // CREATE TABLE, DROP
   if (createTable) {
     test('createTable, dropIfExists=true', async () => {
-      await db.createTable(getTestItemSchema(), { dropIfExists: true })
+      await db.createTable(TEST_TABLE, getTestItemSchema(), { dropIfExists: true })
     })
   }
 

package/src/testing/keyValueDBTest.ts

@@ -41,8 +41,10 @@ export function runCommonKeyValueDBTest(db: CommonKeyValueDB): void {
 
   test('streamIds limited', async () => {
     const idsLimited = await readableToArray(db.streamIds(TEST_TABLE, 2))
-    idsLimited.sort()
-    expect(idsLimited).toEqual(testIds.slice(0, 2))
+    // Order is non-deterministic, so, cannot compare values
+    // idsLimited.sort()
+    // expect(idsLimited).toEqual(testIds.slice(0, 2))
+    expect(idsLimited.length).toBe(2)
   })
 
   test('streamValues', async () => {
@@ -53,8 +55,9 @@ export function runCommonKeyValueDBTest(db: CommonKeyValueDB): void {
 
   test('streamValues limited', async () => {
     const valuesLimited = await readableToArray(db.streamValues(TEST_TABLE, 2))
-    valuesLimited.sort()
-    expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
+    // valuesLimited.sort()
+    // expect(valuesLimited).toEqual(testEntries.map(e => e[1]).slice(0, 2))
+    expect(valuesLimited.length).toBe(2)
   })
 
   test('streamEntries', async () => {
@@ -65,8 +68,9 @@ export function runCommonKeyValueDBTest(db: CommonKeyValueDB): void {
 
   test('streamEntries limited', async () => {
     const entriesLimited = await readableToArray(db.streamEntries(TEST_TABLE, 2))
-    entriesLimited.sort()
-    expect(entriesLimited).toEqual(testEntries.slice(0, 2))
+    // entriesLimited.sort()
+    // expect(entriesLimited).toEqual(testEntries.slice(0, 2))
+    expect(entriesLimited.length).toBe(2)
   })
 
   test('deleteByIds should clear', async () => {