@naturalcycles/db-lib 8.55.0 → 8.55.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.FileDB = void 0;
4
4
  const js_lib_1 = require("@naturalcycles/js-lib");
5
5
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
6
- const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
7
6
  const __1 = require("../..");
8
7
  /**
9
8
  * Provides barebone implementation for "whole file" based CommonDB.
@@ -195,7 +194,7 @@ class FileDB extends __1.BaseCommonDB {
195
194
  logFinished(started, op) {
196
195
  if (!this.cfg.logFinished)
197
196
  return;
198
- this.cfg.logger?.log(`<< ${op} ${(0, nodejs_lib_2.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
197
+ this.cfg.logger?.log(`<< ${op} ${(0, nodejs_lib_1.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
199
198
  }
200
199
  }
201
200
  exports.FileDB = FileDB;
@@ -3,8 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.LocalFilePersistencePlugin = void 0;
4
4
  const tslib_1 = require("tslib");
5
5
  const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
6
- const node_stream_1 = require("node:stream");
7
6
  const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
7
+ const node_stream_1 = require("node:stream");
8
8
  const node_zlib_1 = require("node:zlib");
9
9
  const js_lib_1 = require("@naturalcycles/js-lib");
10
10
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
@@ -8,7 +8,6 @@ const node_stream_1 = require("node:stream");
8
8
  const node_zlib_1 = require("node:zlib");
9
9
  const js_lib_1 = require("@naturalcycles/js-lib");
10
10
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
11
- const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
12
11
  const __1 = require("../..");
13
12
  const dbQuery_1 = require("../../query/dbQuery");
14
13
  class InMemoryDB {
@@ -185,7 +184,7 @@ class InMemoryDB {
185
184
  node_fs_1.default.createWriteStream(fname),
186
185
  ]);
187
186
  });
188
- this.cfg.logger.log(`flushToDisk took ${(0, nodejs_lib_2.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, nodejs_lib_2.yellow)(tables)} tables`);
187
+ this.cfg.logger.log(`flushToDisk took ${(0, nodejs_lib_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, nodejs_lib_1.yellow)(tables)} tables`);
189
188
  }
190
189
  /**
191
190
  * Restores all tables (all namespaces) at once.
@@ -212,7 +211,7 @@ class InMemoryDB {
212
211
  ]);
213
212
  this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
214
213
  });
215
- this.cfg.logger.log(`restoreFromDisk took ${(0, nodejs_lib_2.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, nodejs_lib_2.yellow)(files.length)} tables`);
214
+ this.cfg.logger.log(`restoreFromDisk took ${(0, nodejs_lib_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, nodejs_lib_1.yellow)(files.length)} tables`);
216
215
  }
217
216
  }
218
217
  exports.InMemoryDB = InMemoryDB;
@@ -7,7 +7,6 @@ const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
7
7
  const node_zlib_1 = require("node:zlib");
8
8
  const js_lib_1 = require("@naturalcycles/js-lib");
9
9
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
10
- const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
11
10
  const index_1 = require("../index");
12
11
  /**
13
12
  * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
@@ -23,11 +22,11 @@ async function dbPipelineBackup(opt) {
23
22
  const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
24
23
  const gzip = opt.gzip !== false; // default to true
25
24
  let { tables } = opt;
26
- const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
27
- console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineBackup')} started in ${(0, nodejs_lib_2.grey)(outputDirPath)}...${sinceUpdatedStr}`);
25
+ const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
26
+ console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineBackup')} started in ${(0, nodejs_lib_1.grey)(outputDirPath)}...${sinceUpdatedStr}`);
28
27
  (0, nodejs_lib_1._ensureDirSync)(outputDirPath);
29
28
  tables ||= await db.getTables();
30
- console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n` + tables.join('\n'));
29
+ console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
31
30
  const statsPerTable = {};
32
31
  await (0, js_lib_1.pMap)(tables, async (table) => {
33
32
  let q = index_1.DBQuery.create(table).limit(limit);
@@ -42,11 +41,11 @@ async function dbPipelineBackup(opt) {
42
41
  const started = Date.now();
43
42
  let rows = 0;
44
43
  (0, nodejs_lib_1._ensureFileSync)(filePath);
45
- console.log(`>> ${(0, nodejs_lib_2.grey)(filePath)} started...`);
44
+ console.log(`>> ${(0, nodejs_lib_1.grey)(filePath)} started...`);
46
45
  if (emitSchemaFromDB) {
47
46
  const schema = await db.getTableSchema(table);
48
47
  await (0, nodejs_lib_1._writeJson)(schemaFilePath, schema, { spaces: 2 });
49
- console.log(`>> ${(0, nodejs_lib_2.grey)(schemaFilePath)} saved (generated from DB)`);
48
+ console.log(`>> ${(0, nodejs_lib_1.grey)(schemaFilePath)} saved (generated from DB)`);
50
49
  }
51
50
  await (0, nodejs_lib_1._pipeline)([
52
51
  db.streamQuery(q),
@@ -74,7 +73,7 @@ async function dbPipelineBackup(opt) {
74
73
  rows,
75
74
  sizeBytes,
76
75
  });
77
- console.log(`>> ${(0, nodejs_lib_2.grey)(filePath)}\n` + stats.toPretty());
76
+ console.log(`>> ${(0, nodejs_lib_1.grey)(filePath)}\n` + stats.toPretty());
78
77
  statsPerTable[table] = stats;
79
78
  }, { concurrency, errorMode });
80
79
  const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.dbPipelineCopy = void 0;
4
4
  const js_lib_1 = require("@naturalcycles/js-lib");
5
5
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
6
- const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
7
6
  const dbQuery_1 = require("../query/dbQuery");
8
7
  /**
9
8
  * Pipeline from input stream(s) to CommonDB .saveBatch().
@@ -14,10 +13,10 @@ const dbQuery_1 = require("../query/dbQuery");
14
13
  async function dbPipelineCopy(opt) {
15
14
  const { batchSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
16
15
  let { tables } = opt;
17
- const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
18
- console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
16
+ const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
17
+ console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
19
18
  tables ||= await dbInput.getTables();
20
- console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n` + tables.join('\n'));
19
+ console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
21
20
  const statsPerTable = {};
22
21
  await (0, js_lib_1.pMap)(tables, async (table) => {
23
22
  let q = dbQuery_1.DBQuery.create(table).limit(limit);
@@ -53,7 +52,7 @@ async function dbPipelineCopy(opt) {
53
52
  rows,
54
53
  sizeBytes: 0, // n/a
55
54
  });
56
- console.log(`>> ${(0, nodejs_lib_2.grey)(table)}\n` + stats.toPretty());
55
+ console.log(`>> ${(0, nodejs_lib_1.grey)(table)}\n` + stats.toPretty());
57
56
  statsPerTable[table] = stats;
58
57
  }, { concurrency, errorMode });
59
58
  const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
@@ -6,7 +6,6 @@ const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
6
6
  const node_zlib_1 = require("node:zlib");
7
7
  const js_lib_1 = require("@naturalcycles/js-lib");
8
8
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
9
- const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
10
9
  /**
11
10
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
12
11
  * Allows to define a mapper and a predicate to map/filter objects between input and output.
@@ -18,8 +17,8 @@ async function dbPipelineRestore(opt) {
18
17
  const { db, concurrency = 16, batchSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
19
18
  const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
20
19
  const onlyTables = opt.tables && new Set(opt.tables);
21
- const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
22
- console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineRestore')} started in ${(0, nodejs_lib_2.grey)(inputDirPath)}...${sinceUpdatedStr}`);
20
+ const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
21
+ console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineRestore')} started in ${(0, nodejs_lib_1.grey)(inputDirPath)}...${sinceUpdatedStr}`);
23
22
  (0, nodejs_lib_1._ensureDirSync)(inputDirPath);
24
23
  const tablesToGzip = new Set();
25
24
  const sizeByTable = {};
@@ -46,7 +45,7 @@ async function dbPipelineRestore(opt) {
46
45
  sizeByTable[table] = node_fs_1.default.statSync(`${inputDirPath}/${f}`).size;
47
46
  });
48
47
  const sizeStrByTable = (0, js_lib_1._mapValues)(sizeByTable, (_k, b) => (0, js_lib_1._hb)(b));
49
- console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n`, sizeStrByTable);
48
+ console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n`, sizeStrByTable);
50
49
  // const schemaByTable: Record<string, CommonSchema> = {}
51
50
  if (recreateTables) {
52
51
  await (0, js_lib_1.pMap)(tables, async (table) => {
@@ -66,7 +65,7 @@ async function dbPipelineRestore(opt) {
66
65
  const started = Date.now();
67
66
  let rows = 0;
68
67
  const sizeBytes = sizeByTable[table];
69
- console.log(`<< ${(0, nodejs_lib_2.grey)(filePath)} ${(0, nodejs_lib_2.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
68
+ console.log(`<< ${(0, nodejs_lib_1.grey)(filePath)} ${(0, nodejs_lib_1.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
70
69
  await (0, nodejs_lib_1._pipeline)([
71
70
  node_fs_1.default.createReadStream(filePath),
72
71
  ...(gzip ? [(0, node_zlib_1.createUnzip)()] : []),
@@ -98,7 +97,7 @@ async function dbPipelineRestore(opt) {
98
97
  rows,
99
98
  sizeBytes,
100
99
  });
101
- console.log(`<< ${(0, nodejs_lib_2.grey)(filePath)}\n` + stats.toPretty());
100
+ console.log(`<< ${(0, nodejs_lib_1.grey)(filePath)}\n` + stats.toPretty());
102
101
  statsPerTable[table] = stats;
103
102
  }, { concurrency, errorMode });
104
103
  const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
@@ -205,7 +205,7 @@ function runCommonDaoTest(db, features = {}, quirks = {}) {
205
205
  // cleanup
206
206
  await dao.query().deleteByQuery();
207
207
  // Test that id, created, updated are created
208
- const now = (0, js_lib_1.localTime)().unix();
208
+ const now = (0, js_lib_1.localTimeNow)().unix();
209
209
  await dao.runInTransaction([dao.tx.save((0, js_lib_1._omit)(item1, ['id', 'created', 'updated']))]);
210
210
  const loaded = await dao.query().runQuery();
211
211
  expect(loaded.length).toBe(1);
package/package.json CHANGED
@@ -40,7 +40,7 @@
40
40
  "engines": {
41
41
  "node": ">=18.12"
42
42
  },
43
- "version": "8.55.0",
43
+ "version": "8.55.1",
44
44
  "description": "Lowest Common Denominator API to supported Databases",
45
45
  "keywords": [
46
46
  "db",
@@ -17,8 +17,7 @@ import {
17
17
  _stringMapEntries,
18
18
  Saved,
19
19
  } from '@naturalcycles/js-lib'
20
- import { readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
21
- import { dimGrey } from '@naturalcycles/nodejs-lib'
20
+ import { readableCreate, ReadableTyped, dimGrey } from '@naturalcycles/nodejs-lib'
22
21
  import { BaseCommonDB, DBSaveBatchOperation, queryInMemory } from '../..'
23
22
  import { CommonDB } from '../../common.db'
24
23
  import {
@@ -1,6 +1,6 @@
1
1
  import fs from 'node:fs'
2
- import { Readable } from 'node:stream'
3
2
  import fsp from 'node:fs/promises'
3
+ import { Readable } from 'node:stream'
4
4
  import { createGzip, createUnzip } from 'node:zlib'
5
5
  import { pMap, ObjectWithId } from '@naturalcycles/js-lib'
6
6
  import {
@@ -27,8 +27,9 @@ import {
27
27
  _pipeline,
28
28
  _emptyDir,
29
29
  _ensureDir,
30
+ dimGrey,
31
+ yellow,
30
32
  } from '@naturalcycles/nodejs-lib'
31
- import { dimGrey, yellow } from '@naturalcycles/nodejs-lib'
32
33
  import { CommonDB, DBIncrement, DBPatch, DBTransaction, queryInMemory } from '../..'
33
34
  import {
34
35
  CommonDBCreateOptions,
package/src/model.util.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { stringId } from '@naturalcycles/nodejs-lib'
2
1
  import { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib'
2
+ import { stringId } from '@naturalcycles/nodejs-lib'
3
3
 
4
4
  export function createdUpdatedFields(
5
5
  existingObject?: Partial<CreatedUpdated> | null,
@@ -22,8 +22,11 @@ import {
22
22
  _pathExistsSync,
23
23
  _ensureFileSync,
24
24
  _writeJson,
25
+ boldWhite,
26
+ dimWhite,
27
+ grey,
28
+ yellow,
25
29
  } from '@naturalcycles/nodejs-lib'
26
- import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
27
30
  import { CommonDB } from '../common.db'
28
31
  import { DBQuery } from '../index'
29
32
 
@@ -9,8 +9,11 @@ import {
9
9
  transformTap,
10
10
  writableForEach,
11
11
  _pipeline,
12
+ boldWhite,
13
+ dimWhite,
14
+ grey,
15
+ yellow,
12
16
  } from '@naturalcycles/nodejs-lib'
13
- import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
14
17
  import { CommonDB } from '../common.db'
15
18
  import { CommonDBSaveOptions } from '../db.model'
16
19
  import { DBQuery } from '../query/dbQuery'
@@ -27,8 +27,11 @@ import {
27
27
  _pipeline,
28
28
  _ensureDirSync,
29
29
  _readJson,
30
+ boldWhite,
31
+ dimWhite,
32
+ grey,
33
+ yellow,
30
34
  } from '@naturalcycles/nodejs-lib'
31
- import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
32
35
  import { CommonDB } from '../common.db'
33
36
  import { CommonDBSaveOptions } from '../index'
34
37
 
@@ -1,5 +1,5 @@
1
1
  import { Readable } from 'node:stream'
2
- import { pDelay, _deepCopy, _pick, _sortBy, _omit, localTime } from '@naturalcycles/js-lib'
2
+ import { pDelay, _deepCopy, _pick, _sortBy, _omit, localTimeNow } from '@naturalcycles/js-lib'
3
3
  import { _pipeline, readableToArray, transformNoOp } from '@naturalcycles/nodejs-lib'
4
4
  import { CommonDaoLogLevel, DBQuery } from '..'
5
5
  import { CommonDB } from '../common.db'
@@ -289,7 +289,7 @@ export function runCommonDaoTest(
289
289
  await dao.query().deleteByQuery()
290
290
 
291
291
  // Test that id, created, updated are created
292
- const now = localTime().unix()
292
+ const now = localTimeNow().unix()
293
293
  await dao.runInTransaction([dao.tx.save(_omit(item1, ['id', 'created', 'updated']))])
294
294
 
295
295
  const loaded = await dao.query().runQuery()
@@ -1,7 +1,7 @@
1
1
  import { _range, _sortBy } from '@naturalcycles/js-lib'
2
2
  import { readableToArray } from '@naturalcycles/nodejs-lib'
3
- import { CommonKeyValueDao } from '../kv/commonKeyValueDao'
4
3
  import { KeyValueDBTuple } from '../kv/commonKeyValueDB'
4
+ import { CommonKeyValueDao } from '../kv/commonKeyValueDao'
5
5
 
6
6
  const testIds = _range(1, 4).map(n => `id${n}`)
7
7
  const testEntries: KeyValueDBTuple[] = testIds.map(id => [id, Buffer.from(`${id}value`)])