@naturalcycles/db-lib 8.54.4 → 8.54.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/file/file.db.js +2 -2
- package/dist/adapter/inmemory/inMemory.db.js +3 -3
- package/dist/pipeline/dbPipelineBackup.js +7 -7
- package/dist/pipeline/dbPipelineCopy.js +5 -5
- package/dist/pipeline/dbPipelineRestore.js +7 -7
- package/package.json +1 -1
- package/src/adapter/file/file.db.ts +1 -1
- package/src/adapter/inmemory/inMemory.db.ts +1 -1
- package/src/pipeline/dbPipelineBackup.ts +1 -1
- package/src/pipeline/dbPipelineCopy.ts +1 -1
- package/src/pipeline/dbPipelineRestore.ts +3 -3
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.FileDB = void 0;
|
|
4
4
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
5
|
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
6
|
-
const
|
|
6
|
+
const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
|
|
7
7
|
const __1 = require("../..");
|
|
8
8
|
/**
|
|
9
9
|
* Provides barebone implementation for "whole file" based CommonDB.
|
|
@@ -195,7 +195,7 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
195
195
|
logFinished(started, op) {
|
|
196
196
|
if (!this.cfg.logFinished)
|
|
197
197
|
return;
|
|
198
|
-
this.cfg.logger?.log(`<< ${op} ${(0,
|
|
198
|
+
this.cfg.logger?.log(`<< ${op} ${(0, nodejs_lib_2.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
|
|
199
199
|
}
|
|
200
200
|
}
|
|
201
201
|
exports.FileDB = FileDB;
|
|
@@ -7,7 +7,7 @@ const node_stream_1 = require("node:stream");
|
|
|
7
7
|
const node_zlib_1 = require("node:zlib");
|
|
8
8
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
9
9
|
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
10
|
-
const
|
|
10
|
+
const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
|
|
11
11
|
const __1 = require("../..");
|
|
12
12
|
const dbQuery_1 = require("../../query/dbQuery");
|
|
13
13
|
class InMemoryDB {
|
|
@@ -184,7 +184,7 @@ class InMemoryDB {
|
|
|
184
184
|
fs.createWriteStream(fname),
|
|
185
185
|
]);
|
|
186
186
|
});
|
|
187
|
-
this.cfg.logger.log(`flushToDisk took ${(0,
|
|
187
|
+
this.cfg.logger.log(`flushToDisk took ${(0, nodejs_lib_2.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, nodejs_lib_2.yellow)(tables)} tables`);
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
190
|
* Restores all tables (all namespaces) at once.
|
|
@@ -211,7 +211,7 @@ class InMemoryDB {
|
|
|
211
211
|
]);
|
|
212
212
|
this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
|
|
213
213
|
});
|
|
214
|
-
this.cfg.logger.log(`restoreFromDisk took ${(0,
|
|
214
|
+
this.cfg.logger.log(`restoreFromDisk took ${(0, nodejs_lib_2.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, nodejs_lib_2.yellow)(files.length)} tables`);
|
|
215
215
|
}
|
|
216
216
|
}
|
|
217
217
|
exports.InMemoryDB = InMemoryDB;
|
|
@@ -6,7 +6,7 @@ const fsp = require("node:fs/promises");
|
|
|
6
6
|
const node_zlib_1 = require("node:zlib");
|
|
7
7
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
8
8
|
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
9
|
-
const
|
|
9
|
+
const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
|
|
10
10
|
const index_1 = require("../index");
|
|
11
11
|
/**
|
|
12
12
|
* Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
|
|
@@ -22,11 +22,11 @@ async function dbPipelineBackup(opt) {
|
|
|
22
22
|
const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
|
|
23
23
|
const gzip = opt.gzip !== false; // default to true
|
|
24
24
|
let { tables } = opt;
|
|
25
|
-
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0,
|
|
26
|
-
console.log(`>> ${(0,
|
|
25
|
+
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
|
|
26
|
+
console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineBackup')} started in ${(0, nodejs_lib_2.grey)(outputDirPath)}...${sinceUpdatedStr}`);
|
|
27
27
|
(0, nodejs_lib_1._ensureDirSync)(outputDirPath);
|
|
28
28
|
tables ||= await db.getTables();
|
|
29
|
-
console.log(`${(0,
|
|
29
|
+
console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n` + tables.join('\n'));
|
|
30
30
|
const statsPerTable = {};
|
|
31
31
|
await (0, js_lib_1.pMap)(tables, async (table) => {
|
|
32
32
|
let q = index_1.DBQuery.create(table).limit(limit);
|
|
@@ -41,11 +41,11 @@ async function dbPipelineBackup(opt) {
|
|
|
41
41
|
const started = Date.now();
|
|
42
42
|
let rows = 0;
|
|
43
43
|
(0, nodejs_lib_1._ensureFileSync)(filePath);
|
|
44
|
-
console.log(`>> ${(0,
|
|
44
|
+
console.log(`>> ${(0, nodejs_lib_2.grey)(filePath)} started...`);
|
|
45
45
|
if (emitSchemaFromDB) {
|
|
46
46
|
const schema = await db.getTableSchema(table);
|
|
47
47
|
await (0, nodejs_lib_1._writeJson)(schemaFilePath, schema, { spaces: 2 });
|
|
48
|
-
console.log(`>> ${(0,
|
|
48
|
+
console.log(`>> ${(0, nodejs_lib_2.grey)(schemaFilePath)} saved (generated from DB)`);
|
|
49
49
|
}
|
|
50
50
|
await (0, nodejs_lib_1._pipeline)([
|
|
51
51
|
db.streamQuery(q),
|
|
@@ -73,7 +73,7 @@ async function dbPipelineBackup(opt) {
|
|
|
73
73
|
rows,
|
|
74
74
|
sizeBytes,
|
|
75
75
|
});
|
|
76
|
-
console.log(`>> ${(0,
|
|
76
|
+
console.log(`>> ${(0, nodejs_lib_2.grey)(filePath)}\n` + stats.toPretty());
|
|
77
77
|
statsPerTable[table] = stats;
|
|
78
78
|
}, { concurrency, errorMode });
|
|
79
79
|
const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.dbPipelineCopy = void 0;
|
|
4
4
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
5
|
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
6
|
-
const
|
|
6
|
+
const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
|
|
7
7
|
const dbQuery_1 = require("../query/dbQuery");
|
|
8
8
|
/**
|
|
9
9
|
* Pipeline from input stream(s) to CommonDB .saveBatch().
|
|
@@ -14,10 +14,10 @@ const dbQuery_1 = require("../query/dbQuery");
|
|
|
14
14
|
async function dbPipelineCopy(opt) {
|
|
15
15
|
const { batchSize = 100, dbInput, dbOutput, concurrency = 16, limit = 0, sinceUpdated, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, } = opt;
|
|
16
16
|
let { tables } = opt;
|
|
17
|
-
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0,
|
|
18
|
-
console.log(`>> ${(0,
|
|
17
|
+
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
|
|
18
|
+
console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineCopy')} started...${sinceUpdatedStr}`);
|
|
19
19
|
tables ||= await dbInput.getTables();
|
|
20
|
-
console.log(`${(0,
|
|
20
|
+
console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n` + tables.join('\n'));
|
|
21
21
|
const statsPerTable = {};
|
|
22
22
|
await (0, js_lib_1.pMap)(tables, async (table) => {
|
|
23
23
|
let q = dbQuery_1.DBQuery.create(table).limit(limit);
|
|
@@ -53,7 +53,7 @@ async function dbPipelineCopy(opt) {
|
|
|
53
53
|
rows,
|
|
54
54
|
sizeBytes: 0, // n/a
|
|
55
55
|
});
|
|
56
|
-
console.log(`>> ${(0,
|
|
56
|
+
console.log(`>> ${(0, nodejs_lib_2.grey)(table)}\n` + stats.toPretty());
|
|
57
57
|
statsPerTable[table] = stats;
|
|
58
58
|
}, { concurrency, errorMode });
|
|
59
59
|
const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
|
|
@@ -5,7 +5,7 @@ const fs = require("node:fs");
|
|
|
5
5
|
const node_zlib_1 = require("node:zlib");
|
|
6
6
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
7
7
|
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
8
|
-
const
|
|
8
|
+
const nodejs_lib_2 = require("@naturalcycles/nodejs-lib");
|
|
9
9
|
/**
|
|
10
10
|
* Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
|
|
11
11
|
* Allows to define a mapper and a predicate to map/filter objects between input and output.
|
|
@@ -17,8 +17,8 @@ async function dbPipelineRestore(opt) {
|
|
|
17
17
|
const { db, concurrency = 16, batchSize = 100, limit, sinceUpdated, inputDirPath, mapperPerTable = {}, saveOptionsPerTable = {}, transformMapOptions, errorMode = js_lib_1.ErrorMode.SUPPRESS, recreateTables = false, } = opt;
|
|
18
18
|
const strict = errorMode !== js_lib_1.ErrorMode.SUPPRESS;
|
|
19
19
|
const onlyTables = opt.tables && new Set(opt.tables);
|
|
20
|
-
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0,
|
|
21
|
-
console.log(`>> ${(0,
|
|
20
|
+
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_2.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
|
|
21
|
+
console.log(`>> ${(0, nodejs_lib_2.dimWhite)('dbPipelineRestore')} started in ${(0, nodejs_lib_2.grey)(inputDirPath)}...${sinceUpdatedStr}`);
|
|
22
22
|
(0, nodejs_lib_1._ensureDirSync)(inputDirPath);
|
|
23
23
|
const tablesToGzip = new Set();
|
|
24
24
|
const sizeByTable = {};
|
|
@@ -45,7 +45,7 @@ async function dbPipelineRestore(opt) {
|
|
|
45
45
|
sizeByTable[table] = fs.statSync(`${inputDirPath}/${f}`).size;
|
|
46
46
|
});
|
|
47
47
|
const sizeStrByTable = (0, js_lib_1._mapValues)(sizeByTable, (_k, b) => (0, js_lib_1._hb)(b));
|
|
48
|
-
console.log(`${(0,
|
|
48
|
+
console.log(`${(0, nodejs_lib_2.yellow)(tables.length)} ${(0, nodejs_lib_2.boldWhite)('table(s)')}:\n`, sizeStrByTable);
|
|
49
49
|
// const schemaByTable: Record<string, CommonSchema> = {}
|
|
50
50
|
if (recreateTables) {
|
|
51
51
|
await (0, js_lib_1.pMap)(tables, async (table) => {
|
|
@@ -54,7 +54,7 @@ async function dbPipelineRestore(opt) {
|
|
|
54
54
|
console.warn(`${schemaFilePath} does not exist!`);
|
|
55
55
|
return;
|
|
56
56
|
}
|
|
57
|
-
const schema = await (0, nodejs_lib_1.
|
|
57
|
+
const schema = await (0, nodejs_lib_1._readJson)(schemaFilePath);
|
|
58
58
|
await db.createTable(table, schema, { dropIfExists: true });
|
|
59
59
|
});
|
|
60
60
|
}
|
|
@@ -65,7 +65,7 @@ async function dbPipelineRestore(opt) {
|
|
|
65
65
|
const started = Date.now();
|
|
66
66
|
let rows = 0;
|
|
67
67
|
const sizeBytes = sizeByTable[table];
|
|
68
|
-
console.log(`<< ${(0,
|
|
68
|
+
console.log(`<< ${(0, nodejs_lib_2.grey)(filePath)} ${(0, nodejs_lib_2.dimWhite)((0, js_lib_1._hb)(sizeBytes))} started...`);
|
|
69
69
|
await (0, nodejs_lib_1._pipeline)([
|
|
70
70
|
fs.createReadStream(filePath),
|
|
71
71
|
...(gzip ? [(0, node_zlib_1.createUnzip)()] : []),
|
|
@@ -97,7 +97,7 @@ async function dbPipelineRestore(opt) {
|
|
|
97
97
|
rows,
|
|
98
98
|
sizeBytes,
|
|
99
99
|
});
|
|
100
|
-
console.log(`<< ${(0,
|
|
100
|
+
console.log(`<< ${(0, nodejs_lib_2.grey)(filePath)}\n` + stats.toPretty());
|
|
101
101
|
statsPerTable[table] = stats;
|
|
102
102
|
}, { concurrency, errorMode });
|
|
103
103
|
const statsTotal = nodejs_lib_1.NDJsonStats.createCombined(Object.values(statsPerTable));
|
package/package.json
CHANGED
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
Saved,
|
|
19
19
|
} from '@naturalcycles/js-lib'
|
|
20
20
|
import { readableCreate, ReadableTyped } from '@naturalcycles/nodejs-lib'
|
|
21
|
-
import { dimGrey } from '@naturalcycles/nodejs-lib
|
|
21
|
+
import { dimGrey } from '@naturalcycles/nodejs-lib'
|
|
22
22
|
import { BaseCommonDB, DBSaveBatchOperation, queryInMemory } from '../..'
|
|
23
23
|
import { CommonDB } from '../../common.db'
|
|
24
24
|
import {
|
|
@@ -28,7 +28,7 @@ import {
|
|
|
28
28
|
_emptyDir,
|
|
29
29
|
_ensureDir,
|
|
30
30
|
} from '@naturalcycles/nodejs-lib'
|
|
31
|
-
import { dimGrey, yellow } from '@naturalcycles/nodejs-lib
|
|
31
|
+
import { dimGrey, yellow } from '@naturalcycles/nodejs-lib'
|
|
32
32
|
import { CommonDB, DBIncrement, DBPatch, DBTransaction, queryInMemory } from '../..'
|
|
33
33
|
import {
|
|
34
34
|
CommonDBCreateOptions,
|
|
@@ -23,7 +23,7 @@ import {
|
|
|
23
23
|
_ensureFileSync,
|
|
24
24
|
_writeJson,
|
|
25
25
|
} from '@naturalcycles/nodejs-lib'
|
|
26
|
-
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib
|
|
26
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
|
|
27
27
|
import { CommonDB } from '../common.db'
|
|
28
28
|
import { DBQuery } from '../index'
|
|
29
29
|
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
writableForEach,
|
|
11
11
|
_pipeline,
|
|
12
12
|
} from '@naturalcycles/nodejs-lib'
|
|
13
|
-
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib
|
|
13
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
|
|
14
14
|
import { CommonDB } from '../common.db'
|
|
15
15
|
import { CommonDBSaveOptions } from '../db.model'
|
|
16
16
|
import { DBQuery } from '../query/dbQuery'
|
|
@@ -26,9 +26,9 @@ import {
|
|
|
26
26
|
writableForEach,
|
|
27
27
|
_pipeline,
|
|
28
28
|
_ensureDirSync,
|
|
29
|
-
|
|
29
|
+
_readJson,
|
|
30
30
|
} from '@naturalcycles/nodejs-lib'
|
|
31
|
-
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib
|
|
31
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib'
|
|
32
32
|
import { CommonDB } from '../common.db'
|
|
33
33
|
import { CommonDBSaveOptions } from '../index'
|
|
34
34
|
|
|
@@ -182,7 +182,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
|
|
|
182
182
|
return
|
|
183
183
|
}
|
|
184
184
|
|
|
185
|
-
const schema = await
|
|
185
|
+
const schema = await _readJson<JsonSchemaObject<any>>(schemaFilePath)
|
|
186
186
|
await db.createTable(table, schema, { dropIfExists: true })
|
|
187
187
|
})
|
|
188
188
|
}
|