@naturalcycles/db-lib 8.54.4 → 8.54.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.FileDB = void 0;
4
4
  const js_lib_1 = require("@naturalcycles/js-lib");
5
5
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
6
- const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
6
+ const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
7
7
  const __1 = require("../..");
8
8
  /**
9
9
  * Provides barebone implementation for "whole file" based CommonDB.
@@ -195,7 +195,7 @@ class FileDB extends __1.BaseCommonDB {
195
195
  logFinished(started, op) {
196
196
  if (!this.cfg.logFinished)
197
197
  return;
198
- this.cfg.logger?.log(`<< ${op} ${(0, colors_1.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
198
+ this.cfg.logger?.log(`<< ${op} ${(0, nodejs_lib_2.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
199
199
  }
200
200
  }
201
201
  exports.FileDB = FileDB;
@@ -7,7 +7,7 @@ const node_stream_1 = require("node:stream");
7
7
  const node_zlib_1 = require("node:zlib");
8
8
  const js_lib_1 = require("@naturalcycles/js-lib");
9
9
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
10
- const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
10
+ const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
11
11
  const __1 = require("../..");
12
12
  const dbQuery_1 = require("../../query/dbQuery");
13
13
  class InMemoryDB {
@@ -184,7 +184,7 @@ class InMemoryDB {
184
184
  fs.createWriteStream(fname),
185
185
  ]);
186
186
  });
187
- this.cfg.logger.log(`flushToDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, colors_1.yellow)(tables)} tables`);
187
+ this.cfg.logger.log(`flushToDisk took ${(0, nodejs_lib_2.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, nodejs_lib_2.yellow)(tables)} tables`);
188
188
  }
189
189
  /**
190
190
  * Restores all tables (all namespaces) at once.
@@ -211,7 +211,7 @@ class InMemoryDB {
211
211
  ]);
212
212
  this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
213
213
  });
214
- this.cfg.logger.log(`restoreFromDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, colors_1.yellow)(files.length)} tables`);
214
+ this.cfg.logger.log(`restoreFromDisk took ${(0, nodejs_lib_2.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, nodejs_lib_2.yellow)(files.length)} tables`);
215
215
  }
216
216
  }
217
217
  exports.InMemoryDB = InMemoryDB;
@@ -6,7 +6,7 @@ const fsp = require("node:fs/promises");
6
6
  const node_zlib_1 = require("node:zlib");
7
7
  const js_lib_1 = require("@naturalcycles/js-lib");
8
8
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
9
- const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
9
+ const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
10
10
  const index_1 = require("../index");
11
11
  /**
12
12
  * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
@@ -22,11 +22,11 @@ async function dbPipelineBackup(opt) {
22
22
  const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
23
23
  const gzip = opt.gzip !== false; // default to true
24
24
  let { tables } = opt;
25
- const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, colors_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
26
- console.log(`>> ${(0, colors_1.dimWhite)('dbPipelineBackup')} started in ${(0, colors_1.grey)(outputDirPath)}...${sinceUpdatedStr}`);
25
+ const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
26
+ console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineBackup')} started in ${(0, nodejs_lib_2.grey)(outputDirPath)}...${sinceUpdatedStr}`);
27
27
  (0, nodejs_lib_1._ensureDirSync)(outputDirPath);
28
28
  tables ||= await db.getTables();
29
- console.log(`${(0, colors_1.yellow)(tables.length)} ${(0, colors_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
29
+ console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n` + tables.join('\n'));
30
30
  const statsPerTable = {};
31
31
  await (0, js_lib_1.pMap)(tables, async (table) => {
32
32
  let q = index_1.DBQuery.create(table).limit(limit);
@@ -41,11 +41,11 @@ async function dbPipelineBackup(opt) {
41
41
  const started = Date.now();
42
42
  let rows = 0;
43
43
  (0, nodejs_lib_1._ensureFileSync)(filePath);
44
- console.log(`>> ${(0, colors_1.grey)(filePath)} started...`);
44
+ console.log(`>> ${(0, nodejs_lib_2.grey)(filePath)} started...`);
45
45
  if (emitSchemaFromDB) {
46
46
  const schema = await db.getTableSchema(table);
47
47
  await (0, nodejs_lib_1._writeJson)(schemaFilePath, schema, { spaces: 2 });
48
- console.log(`>> ${(0, colors_1.grey)(schemaFilePath)} saved (generated from DB)`);
48
+ console.log(`>> ${(0, nodejs_lib_2.grey)(schemaFilePath)} saved (generated from DB)`);
49
49
  }
50
50
  await (0, nodejs_lib_1._pipeline)([
51
51
  db.streamQuery(q),
@@ -73,7 +73,7 @@ async function dbPipelineBackup(opt) {
73
73
  rows,
74
74
  sizeBytes,
75
75
  });
76
- console.log(`>> ${(0, colors_1.grey)(filePath)}\n` + stats.toPretty());
76
+ console.log(`>> ${(0, nodejs_lib_2.grey)(filePath)}\n` + stats.toPretty());
77
77
  statsPerTable[table] = stats;
78
78
  }, { concurrency, errorMode });
79
79
  const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.dbPipelineCopy = void 0;
4
4
  const js_lib_1 = require("@naturalcycles/js-lib");
5
5
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
6
- const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
6
+ const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
7
7
  const dbQuery_1 = require("../query/dbQuery");
8
8
  /**
9
9
  * Pipeline from input stream(s) to CommonDB .saveBatch().
@@ -14,10 +14,10 @@ const dbQuery_1 = require("../query/dbQuery");
14
14
  async function dbPipelineCopy(opt) {
15
15
  const { batchSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
16
16
  let { tables } = opt;
17
- const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, colors_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
18
- console.log(`>> ${(0, colors_1.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
17
+ const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
18
+ console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
19
19
  tables ||= await dbInput.getTables();
20
- console.log(`${(0, colors_1.yellow)(tables.length)} ${(0, colors_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
20
+ console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n` + tables.join('\n'));
21
21
  const statsPerTable = {};
22
22
  await (0, js_lib_1.pMap)(tables, async (table) => {
23
23
  let q = dbQuery_1.DBQuery.create(table).limit(limit);
@@ -53,7 +53,7 @@ async function dbPipelineCopy(opt) {
53
53
  rows,
54
54
  sizeBytes: 0, // n/a
55
55
  });
56
- console.log(`>> ${(0, colors_1.grey)(table)}\n` + stats.toPretty());
56
+ console.log(`>> ${(0, nodejs_lib_2.grey)(table)}\n` + stats.toPretty());
57
57
  statsPerTable[table] = stats;
58
58
  }, { concurrency, errorMode });
59
59
  const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
@@ -5,7 +5,7 @@ const fs = require("node:fs");
5
5
  const node_zlib_1 = require("node:zlib");
6
6
  const js_lib_1 = require("@naturalcycles/js-lib");
7
7
  const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
8
- const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
8
+ const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
9
9
  /**
10
10
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
11
11
  * Allows to define a mapper and a predicate to map/filter objects between input and output.
@@ -17,8 +17,8 @@ async function dbPipelineRestore(opt) {
17
17
  const { db, concurrency = 16, batchSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
18
18
  const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
19
19
  const onlyTables = opt.tables && new Set(opt.tables);
20
- const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, colors_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
21
- console.log(`>> ${(0, colors_1.dimWhite)('dbPipelineRestore')} started in ${(0, colors_1.grey)(inputDirPath)}...${sinceUpdatedStr}`);
20
+ const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
21
+ console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineRestore')} started in ${(0, nodejs_lib_2.grey)(inputDirPath)}...${sinceUpdatedStr}`);
22
22
  (0, nodejs_lib_1._ensureDirSync)(inputDirPath);
23
23
  const tablesToGzip = new Set();
24
24
  const sizeByTable = {};
@@ -45,7 +45,7 @@ async function dbPipelineRestore(opt) {
45
45
  sizeByTable[table] = fs.statSync(`${inputDirPath}/${f}`).size;
46
46
  });
47
47
  const sizeStrByTable = (0, js_lib_1._mapValues)(sizeByTable, (_k, b) => (0, js_lib_1._hb)(b));
48
- console.log(`${(0, colors_1.yellow)(tables.length)} ${(0, colors_1.boldWhite)('table(s)')}:\n`, sizeStrByTable);
48
+ console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n`, sizeStrByTable);
49
49
  // const schemaByTable: Record<string, CommonSchema> = {}
50
50
  if (recreateTables) {
51
51
  await (0, js_lib_1.pMap)(tables, async (table) => {
@@ -54,7 +54,7 @@ async function dbPipelineRestore(opt) {
54
54
  console.warn(`${schemaFilePath} does not exist!`);
55
55
  return;
56
56
  }
57
- const schema = await (0, nodejs_lib_1._readJsonFile)(schemaFilePath);
57
+ const schema = await (0, nodejs_lib_1._readJson)(schemaFilePath);
58
58
  await db.createTable(table, schema, { dropIfExists: true });
59
59
  });
60
60
  }
@@ -65,7 +65,7 @@ async function dbPipelineRestore(opt) {
65
65
  const started = Date.now();
66
66
  let rows = 0;
67
67
  const sizeBytes = sizeByTable[table];
68
- console.log(`<< ${(0, colors_1.grey)(filePath)} ${(0, colors_1.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
68
+ console.log(`<< ${(0, nodejs_lib_2.grey)(filePath)} ${(0, nodejs_lib_2.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
69
69
  await (0, nodejs_lib_1._pipeline)([
70
70
  fs.createReadStream(filePath),
71
71
  ...(gzip ? [(0, node_zlib_1.createUnzip)()] : []),
@@ -97,7 +97,7 @@ async function dbPipelineRestore(opt) {
97
97
  rows,
98
98
  sizeBytes,
99
99
  });
100
- console.log(`<< ${(0, colors_1.grey)(filePath)}\n` + stats.toPretty());
100
+ console.log(`<< ${(0, nodejs_lib_2.grey)(filePath)}\n` + stats.toPretty());
101
101
  statsPerTable[table] = stats;
102
102
  }, { concurrency, errorMode });
103
103
  const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
package/package.json CHANGED
@@ -40,7 +40,7 @@
40
40
  "engines": {
41
41
  "node": ">=18.12"
42
42
  },
43
- "version": "8.54.4",
43
+ "version": "8.54.5",
44
44
  "description": "Lowest Common Denominator API to supported Databases",
45
45
  "keywords": [
46
46
  "db",
@@ -18,7 +18,7 @@ import {
18
18
  Saved,
19
19
  } from '@naturalcycles/js-lib'
20
20
  import { readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
21
- import { dimGrey } from '@naturalcycles/nodejs-lib/dist/colors'
21
+ import { dimGrey } from '@naturalcycles/nodejs-lib'
22
22
  import { BaseCommonDB, DBSaveBatchOperation, queryInMemory } from '../..'
23
23
  import { CommonDB } from '../../common.db'
24
24
  import {
@@ -28,7 +28,7 @@ import {
28
28
  _emptyDir,
29
29
  _ensureDir,
30
30
  } from '@naturalcycles/nodejs-lib'
31
- import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
31
+ import { dimGrey, yellow } from '@naturalcycles/nodejs-lib'
32
32
  import { CommonDB, DBIncrement, DBPatch, DBTransaction, queryInMemory } from '../..'
33
33
  import {
34
34
  CommonDBCreateOptions,
@@ -23,7 +23,7 @@ import {
23
23
  _ensureFileSync,
24
24
  _writeJson,
25
25
  } from '@naturalcycles/nodejs-lib'
26
- import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
26
+ import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
27
27
  import { CommonDB } from '../common.db'
28
28
  import { DBQuery } from '../index'
29
29
 
@@ -10,7 +10,7 @@ import {
10
10
  writableForEach,
11
11
  _pipeline,
12
12
  } from '@naturalcycles/nodejs-lib'
13
- import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
13
+ import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
14
14
  import { CommonDB } from '../common.db'
15
15
  import { CommonDBSaveOptions } from '../db.model'
16
16
  import { DBQuery } from '../query/dbQuery'
@@ -26,9 +26,9 @@ import {
26
26
  writableForEach,
27
27
  _pipeline,
28
28
  _ensureDirSync,
29
- _readJsonFile,
29
+ _readJson,
30
30
  } from '@naturalcycles/nodejs-lib'
31
- import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
31
+ import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
32
32
  import { CommonDB } from '../common.db'
33
33
  import { CommonDBSaveOptions } from '../index'
34
34
 
@@ -182,7 +182,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
182
182
  return
183
183
  }
184
184
 
185
- const schema = await _readJsonFile<JsonSchemaObject<any>>(schemaFilePath)
185
+ const schema = await _readJson<JsonSchemaObject<any>>(schemaFilePath)
186
186
  await db.createTable(table, schema, { dropIfExists: true })
187
187
  })
188
188
  }