@naturalcycles/db-lib 8.59.0 → 8.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/file/localFile.persistence.plugin.js +4 -4
- package/dist/adapter/inmemory/inMemory.db.js +3 -3
- package/dist/common.db.d.ts +1 -1
- package/dist/commondao/common.dao.d.ts +23 -23
- package/dist/commondao/common.dao.js +2 -4
- package/dist/commondao/common.dao.model.d.ts +6 -10
- package/dist/model.util.d.ts +1 -1
- package/dist/pipeline/dbPipelineBackup.js +5 -5
- package/dist/pipeline/dbPipelineRestore.js +3 -3
- package/dist/query/dbQuery.d.ts +6 -6
- package/dist/testing/test.model.js +1 -1
- package/dist/timeseries/commonTimeSeriesDao.js +4 -4
- package/dist/validation/index.d.ts +2 -2
- package/package.json +1 -1
- package/src/adapter/file/localFile.persistence.plugin.ts +4 -5
- package/src/adapter/inmemory/inMemory.db.ts +3 -4
- package/src/common.db.ts +1 -1
- package/src/commondao/common.dao.model.ts +7 -18
- package/src/commondao/common.dao.ts +51 -46
- package/src/model.util.ts +2 -2
- package/src/pipeline/dbPipelineBackup.ts +5 -8
- package/src/pipeline/dbPipelineRestore.ts +3 -4
- package/src/query/dbQuery.ts +7 -8
|
@@ -26,17 +26,17 @@ class LocalFilePersistencePlugin {
|
|
|
26
26
|
.map(f => f.split('.ndjson')[0]);
|
|
27
27
|
}
|
|
28
28
|
async loadFile(table) {
|
|
29
|
-
await
|
|
29
|
+
await nodejs_lib_1.fs2.ensureDirAsync(this.cfg.storagePath);
|
|
30
30
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
|
|
31
31
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
32
|
-
if (!(await
|
|
32
|
+
if (!(await nodejs_lib_1.fs2.pathExistsAsync(filePath)))
|
|
33
33
|
return [];
|
|
34
34
|
const transformUnzip = this.cfg.gzip ? [(0, node_zlib_1.createUnzip)()] : [];
|
|
35
35
|
const rows = [];
|
|
36
36
|
await (0, nodejs_lib_1._pipeline)([
|
|
37
37
|
node_fs_1.default.createReadStream(filePath),
|
|
38
38
|
...transformUnzip,
|
|
39
|
-
(0, nodejs_lib_1.transformSplit)(),
|
|
39
|
+
(0, nodejs_lib_1.transformSplit)(), // splits by \n
|
|
40
40
|
(0, nodejs_lib_1.transformJsonParse)(),
|
|
41
41
|
(0, nodejs_lib_1.writablePushToArray)(rows),
|
|
42
42
|
]);
|
|
@@ -46,7 +46,7 @@ class LocalFilePersistencePlugin {
|
|
|
46
46
|
await (0, js_lib_1.pMap)(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 16 });
|
|
47
47
|
}
|
|
48
48
|
async saveFile(table, rows) {
|
|
49
|
-
await
|
|
49
|
+
await nodejs_lib_1.fs2.ensureDirAsync(this.cfg.storagePath);
|
|
50
50
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
|
|
51
51
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
52
52
|
const transformZip = this.cfg.gzip ? [(0, node_zlib_1.createGzip)()] : [];
|
|
@@ -167,7 +167,7 @@ class InMemoryDB {
|
|
|
167
167
|
(0, js_lib_1._assert)(this.cfg.persistenceEnabled, 'flushToDisk() called but persistenceEnabled=false');
|
|
168
168
|
const { persistentStoragePath, persistZip } = this.cfg;
|
|
169
169
|
const started = Date.now();
|
|
170
|
-
await
|
|
170
|
+
await nodejs_lib_1.fs2.emptyDirAsync(persistentStoragePath);
|
|
171
171
|
const transformZip = persistZip ? [(0, node_zlib_1.createGzip)()] : [];
|
|
172
172
|
let tables = 0;
|
|
173
173
|
// infinite concurrency for now
|
|
@@ -193,7 +193,7 @@ class InMemoryDB {
|
|
|
193
193
|
(0, js_lib_1._assert)(this.cfg.persistenceEnabled, 'restoreFromDisk() called but persistenceEnabled=false');
|
|
194
194
|
const { persistentStoragePath } = this.cfg;
|
|
195
195
|
const started = Date.now();
|
|
196
|
-
await
|
|
196
|
+
await nodejs_lib_1.fs2.ensureDirAsync(persistentStoragePath);
|
|
197
197
|
this.data = {}; // empty it in the beginning!
|
|
198
198
|
const files = (await promises_1.default.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'));
|
|
199
199
|
// infinite concurrency for now
|
|
@@ -205,7 +205,7 @@ class InMemoryDB {
|
|
|
205
205
|
await (0, nodejs_lib_1._pipeline)([
|
|
206
206
|
node_fs_1.default.createReadStream(fname),
|
|
207
207
|
...transformUnzip,
|
|
208
|
-
(0, nodejs_lib_1.transformSplit)(),
|
|
208
|
+
(0, nodejs_lib_1.transformSplit)(), // splits by \n
|
|
209
209
|
(0, nodejs_lib_1.transformJsonParse)(),
|
|
210
210
|
(0, nodejs_lib_1.writablePushToArray)(rows),
|
|
211
211
|
]);
|
package/dist/common.db.d.ts
CHANGED
|
@@ -31,7 +31,7 @@ export interface CommonDB {
|
|
|
31
31
|
* Order of items returned is not guaranteed to match order of ids.
|
|
32
32
|
* (Such limitation exists because Datastore doesn't support it).
|
|
33
33
|
*/
|
|
34
|
-
getByIds: <ROW extends ObjectWithId>(table: string, ids:
|
|
34
|
+
getByIds: <ROW extends ObjectWithId>(table: string, ids: string[], opt?: CommonDBOptions) => Promise<ROW[]>;
|
|
35
35
|
/**
|
|
36
36
|
* Order by 'id' is not supported by all implementations (for example, Datastore doesn't support it).
|
|
37
37
|
*/
|
|
@@ -12,22 +12,22 @@ import { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveBa
|
|
|
12
12
|
* BM = Backend model (optimized for API access)
|
|
13
13
|
* TM = Transport model (optimized to be sent over the wire)
|
|
14
14
|
*/
|
|
15
|
-
export declare class CommonDao<BM extends Partial<ObjectWithId
|
|
16
|
-
cfg: CommonDaoCfg<BM, DBM, TM
|
|
17
|
-
constructor(cfg: CommonDaoCfg<BM, DBM, TM
|
|
15
|
+
export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM extends AnyObject = BM> {
|
|
16
|
+
cfg: CommonDaoCfg<BM, DBM, TM>;
|
|
17
|
+
constructor(cfg: CommonDaoCfg<BM, DBM, TM>);
|
|
18
18
|
create(part?: Partial<BM>, opt?: CommonDaoOptions): Saved<BM>;
|
|
19
19
|
getById(id: undefined | null, opt?: CommonDaoOptions): Promise<null>;
|
|
20
|
-
getById(id?:
|
|
21
|
-
getByIdOrEmpty(id:
|
|
22
|
-
getByIdAsDBMOrEmpty(id:
|
|
20
|
+
getById(id?: string | null, opt?: CommonDaoOptions): Promise<Saved<BM> | null>;
|
|
21
|
+
getByIdOrEmpty(id: string, part?: Partial<BM>, opt?: CommonDaoOptions): Promise<Saved<BM>>;
|
|
22
|
+
getByIdAsDBMOrEmpty(id: string, part?: Partial<BM>, opt?: CommonDaoOptions): Promise<DBM>;
|
|
23
23
|
getByIdAsDBM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>;
|
|
24
|
-
getByIdAsDBM(id?:
|
|
24
|
+
getByIdAsDBM(id?: string | null, opt?: CommonDaoOptions): Promise<DBM | null>;
|
|
25
25
|
getByIdAsTM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>;
|
|
26
|
-
getByIdAsTM(id?:
|
|
27
|
-
getByIds(ids:
|
|
28
|
-
getByIdsAsDBM(ids:
|
|
29
|
-
requireById(id:
|
|
30
|
-
requireByIdAsDBM(id:
|
|
26
|
+
getByIdAsTM(id?: string | null, opt?: CommonDaoOptions): Promise<TM | null>;
|
|
27
|
+
getByIds(ids: string[], opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
|
|
28
|
+
getByIdsAsDBM(ids: string[], opt?: CommonDaoOptions): Promise<DBM[]>;
|
|
29
|
+
requireById(id: string, opt?: CommonDaoOptions): Promise<Saved<BM>>;
|
|
30
|
+
requireByIdAsDBM(id: string, opt?: CommonDaoOptions): Promise<DBM>;
|
|
31
31
|
private throwRequiredError;
|
|
32
32
|
/**
|
|
33
33
|
* Throws if readOnly is true
|
|
@@ -44,7 +44,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
|
|
|
44
44
|
/**
|
|
45
45
|
* Pass `table` to override table
|
|
46
46
|
*/
|
|
47
|
-
query(table?: string): RunnableDBQuery<BM, DBM, TM
|
|
47
|
+
query(table?: string): RunnableDBQuery<BM, DBM, TM>;
|
|
48
48
|
runQuery(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
|
|
49
49
|
runQuerySingleColumn<T = any>(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<T[]>;
|
|
50
50
|
/**
|
|
@@ -75,9 +75,9 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
|
|
|
75
75
|
* You can do `.pipe(transformNoOp)` to make it "valid again".
|
|
76
76
|
*/
|
|
77
77
|
streamQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<Saved<BM>>): ReadableTyped<Saved<BM>>;
|
|
78
|
-
queryIds(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<
|
|
79
|
-
streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<
|
|
80
|
-
streamQueryIdsForEach(q: DBQuery<DBM>, mapper: AsyncMapper<
|
|
78
|
+
queryIds(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<string[]>;
|
|
79
|
+
streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<string>): ReadableTyped<string>;
|
|
80
|
+
streamQueryIdsForEach(q: DBQuery<DBM>, mapper: AsyncMapper<string, void>, opt?: CommonDaoStreamForEachOptions<string>): Promise<void>;
|
|
81
81
|
/**
|
|
82
82
|
* Mutates!
|
|
83
83
|
* "Returns", just to have a type of "Saved"
|
|
@@ -88,8 +88,8 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
|
|
|
88
88
|
tx: {
|
|
89
89
|
save: (bm: Unsaved<BM>, opt?: CommonDaoSaveBatchOptions<DBM>) => Promise<DBSaveBatchOperation | undefined>;
|
|
90
90
|
saveBatch: (bms: Unsaved<BM>[], opt?: CommonDaoSaveBatchOptions<DBM>) => Promise<DBSaveBatchOperation | undefined>;
|
|
91
|
-
deleteByIds: (ids:
|
|
92
|
-
deleteById: (id:
|
|
91
|
+
deleteByIds: (ids: string[], opt?: CommonDaoOptions) => Promise<DBDeleteByIdsOperation | undefined>;
|
|
92
|
+
deleteById: (id: string | null | undefined, opt?: CommonDaoOptions) => Promise<DBDeleteByIdsOperation | undefined>;
|
|
93
93
|
};
|
|
94
94
|
/**
|
|
95
95
|
* Mutates with id, created, updated
|
|
@@ -113,7 +113,7 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
|
|
|
113
113
|
* 2. Applies the patch on top of loaded data.
|
|
114
114
|
* 3. Saves (as fast as possible since the read) with the Patch applied, but only if the data has changed.
|
|
115
115
|
*/
|
|
116
|
-
patchById(id:
|
|
116
|
+
patchById(id: string, patch: Partial<BM>, opt?: CommonDaoSaveBatchOptions<DBM>): Promise<Saved<BM>>;
|
|
117
117
|
/**
|
|
118
118
|
* Same as patchById, but takes the whole object as input.
|
|
119
119
|
* This "whole object" is mutated with the patch and returned.
|
|
@@ -135,16 +135,16 @@ export declare class CommonDao<BM extends Partial<ObjectWithId<ID>>, DBM extends
|
|
|
135
135
|
* @returns number of deleted items
|
|
136
136
|
*/
|
|
137
137
|
deleteById(id: undefined | null, opt?: CommonDaoOptions): Promise<0>;
|
|
138
|
-
deleteById(id?:
|
|
139
|
-
deleteByIds(ids:
|
|
138
|
+
deleteById(id?: string | null, opt?: CommonDaoOptions): Promise<number>;
|
|
139
|
+
deleteByIds(ids: string[], opt?: CommonDaoOptions): Promise<number>;
|
|
140
140
|
/**
|
|
141
141
|
* Pass `stream: true` option to use Streaming: it will Stream the query, batch by 500, and execute
|
|
142
142
|
* `deleteByIds` for each batch concurrently (infinite concurrency).
|
|
143
143
|
* This is expected to be more memory-efficient way of deleting big numbers of rows.
|
|
144
144
|
*/
|
|
145
145
|
deleteByQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
|
|
146
|
-
updateById(id:
|
|
147
|
-
updateByIds(ids:
|
|
146
|
+
updateById(id: string, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
|
|
147
|
+
updateByIds(ids: string[], patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
|
|
148
148
|
updateByQuery(q: DBQuery<DBM>, patch: DBPatch<DBM>, opt?: CommonDaoOptions): Promise<number>;
|
|
149
149
|
dbmToBM(_dbm: undefined, opt?: CommonDaoOptions): Promise<undefined>;
|
|
150
150
|
dbmToBM(_dbm?: DBM, opt?: CommonDaoOptions): Promise<Saved<BM>>;
|
|
@@ -56,7 +56,7 @@ class CommonDao {
|
|
|
56
56
|
return {
|
|
57
57
|
type: 'deleteByIds',
|
|
58
58
|
table: this.cfg.table,
|
|
59
|
-
ids
|
|
59
|
+
ids,
|
|
60
60
|
opt,
|
|
61
61
|
};
|
|
62
62
|
},
|
|
@@ -76,7 +76,6 @@ class CommonDao {
|
|
|
76
76
|
// otherwise to log Operations
|
|
77
77
|
// e.g in Dev (local machine), Test - it will log operations (useful for debugging)
|
|
78
78
|
logLevel: isGAE || isCI ? common_dao_model_1.CommonDaoLogLevel.NONE : common_dao_model_1.CommonDaoLogLevel.OPERATIONS,
|
|
79
|
-
idType: 'string',
|
|
80
79
|
createId: true,
|
|
81
80
|
assignGeneratedIds: false,
|
|
82
81
|
created: true,
|
|
@@ -96,7 +95,6 @@ class CommonDao {
|
|
|
96
95
|
},
|
|
97
96
|
};
|
|
98
97
|
if (this.cfg.createId) {
|
|
99
|
-
(0, js_lib_1._assert)(this.cfg.idType === 'string', 'db-lib: automatic generation of non-string ids is not supported');
|
|
100
98
|
this.cfg.hooks.createRandomId ||= () => (0, nodejs_lib_1.stringId)();
|
|
101
99
|
}
|
|
102
100
|
else {
|
|
@@ -875,7 +873,7 @@ class CommonDao {
|
|
|
875
873
|
// LogProgress should be AFTER the mapper, to be able to report correct stats
|
|
876
874
|
(0, nodejs_lib_1.transformLogProgress)({
|
|
877
875
|
metric: q.table,
|
|
878
|
-
logEvery: 2,
|
|
876
|
+
logEvery: 2, // 500 * 2 === 1000
|
|
879
877
|
batchSize,
|
|
880
878
|
...opt,
|
|
881
879
|
}),
|
|
@@ -2,24 +2,24 @@ import { CommonLogger, ErrorMode, ObjectWithId, Promisable, Saved, ZodError, Zod
|
|
|
2
2
|
import { AjvSchema, AjvValidationError, JoiValidationError, ObjectSchema, TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib';
|
|
3
3
|
import { CommonDB } from '../common.db';
|
|
4
4
|
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model';
|
|
5
|
-
export interface CommonDaoHooks<BM extends Partial<ObjectWithId
|
|
5
|
+
export interface CommonDaoHooks<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
|
|
6
6
|
/**
|
|
7
7
|
* Allows to override the id generation function.
|
|
8
8
|
* By default it uses `stringId` from nodejs-lib
|
|
9
9
|
* (which uses lowercase alphanumberic alphabet and the size of 16).
|
|
10
10
|
*/
|
|
11
|
-
createRandomId: () =>
|
|
11
|
+
createRandomId: () => string;
|
|
12
12
|
/**
|
|
13
13
|
* createNaturalId hook is called (tried) first.
|
|
14
14
|
* If it doesn't exist - createRandomId is called.
|
|
15
15
|
*/
|
|
16
|
-
createNaturalId: (obj: DBM | BM) =>
|
|
16
|
+
createNaturalId: (obj: DBM | BM) => string;
|
|
17
17
|
/**
|
|
18
18
|
* It's a counter-part of `createNaturalId`.
|
|
19
19
|
* Allows to provide a parser function to parse "natural id" into
|
|
20
20
|
* DBM components (e.g accountId and some other property that is part of the id).
|
|
21
21
|
*/
|
|
22
|
-
parseNaturalId: (id:
|
|
22
|
+
parseNaturalId: (id: string) => Partial<DBM>;
|
|
23
23
|
/**
|
|
24
24
|
* It is called only on `dao.create` method.
|
|
25
25
|
* Dao.create method is called in:
|
|
@@ -104,7 +104,7 @@ export declare enum CommonDaoLogLevel {
|
|
|
104
104
|
*/
|
|
105
105
|
DATA_FULL = 30
|
|
106
106
|
}
|
|
107
|
-
export interface CommonDaoCfg<BM extends Partial<ObjectWithId
|
|
107
|
+
export interface CommonDaoCfg<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM = BM> {
|
|
108
108
|
db: CommonDB;
|
|
109
109
|
table: string;
|
|
110
110
|
/**
|
|
@@ -141,11 +141,7 @@ export interface CommonDaoCfg<BM extends Partial<ObjectWithId<ID>>, DBM extends
|
|
|
141
141
|
* @default false
|
|
142
142
|
*/
|
|
143
143
|
logStarted?: boolean;
|
|
144
|
-
hooks?: Partial<CommonDaoHooks<BM, DBM, TM
|
|
145
|
-
/**
|
|
146
|
-
* Defaults to 'string'
|
|
147
|
-
*/
|
|
148
|
-
idType?: 'string' | 'number';
|
|
144
|
+
hooks?: Partial<CommonDaoHooks<BM, DBM, TM>>;
|
|
149
145
|
/**
|
|
150
146
|
* Defaults to true.
|
|
151
147
|
* Set to false to disable auto-generation of `id`.
|
package/dist/model.util.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib';
|
|
2
2
|
export declare function createdUpdatedFields(existingObject?: Partial<CreatedUpdated> | null): CreatedUpdated;
|
|
3
|
-
export declare function createdUpdatedIdFields(existingObject?: Partial<CreatedUpdatedId
|
|
3
|
+
export declare function createdUpdatedIdFields(existingObject?: Partial<CreatedUpdatedId> | null): CreatedUpdatedId;
|
|
4
4
|
export declare function deserializeJsonField<T = any>(f?: string): T;
|
|
5
5
|
export declare function serializeJsonField(f: any): string | undefined;
|
|
@@ -23,7 +23,7 @@ async function dbPipelineBackup(opt) {
|
|
|
23
23
|
const gzip = opt.gzip !== false; // default to true
|
|
24
24
|
let { tables } = opt;
|
|
25
25
|
console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineBackup')} started in ${(0, nodejs_lib_1.grey)(outputDirPath)}...`);
|
|
26
|
-
|
|
26
|
+
nodejs_lib_1.fs2.ensureDir(outputDirPath);
|
|
27
27
|
tables ||= await db.getTables();
|
|
28
28
|
console.log(`${(0, nodejs_lib_1.yellow)(tables.length)} ${(0, nodejs_lib_1.boldWhite)('table(s)')}:\n` + tables.join('\n'));
|
|
29
29
|
const statsPerTable = {};
|
|
@@ -46,16 +46,16 @@ async function dbPipelineBackup(opt) {
|
|
|
46
46
|
}
|
|
47
47
|
const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '');
|
|
48
48
|
const schemaFilePath = `${outputDirPath}/${table}.schema.json`;
|
|
49
|
-
if (protectFromOverwrite &&
|
|
49
|
+
if (protectFromOverwrite && nodejs_lib_1.fs2.pathExists(filePath)) {
|
|
50
50
|
throw new js_lib_1.AppError(`dbPipelineBackup: output file exists: ${filePath}`);
|
|
51
51
|
}
|
|
52
52
|
const started = Date.now();
|
|
53
53
|
let rows = 0;
|
|
54
|
-
|
|
54
|
+
nodejs_lib_1.fs2.ensureFile(filePath);
|
|
55
55
|
// console.log(`>> ${grey(filePath)} started...`)
|
|
56
56
|
if (emitSchemaFromDB) {
|
|
57
57
|
const schema = await db.getTableSchema(table);
|
|
58
|
-
await
|
|
58
|
+
await nodejs_lib_1.fs2.writeJsonAsync(schemaFilePath, schema, { spaces: 2 });
|
|
59
59
|
console.log(`>> ${(0, nodejs_lib_1.grey)(schemaFilePath)} saved (generated from DB)`);
|
|
60
60
|
}
|
|
61
61
|
await (0, nodejs_lib_1._pipeline)([
|
|
@@ -75,7 +75,7 @@ async function dbPipelineBackup(opt) {
|
|
|
75
75
|
rows++;
|
|
76
76
|
}),
|
|
77
77
|
(0, nodejs_lib_1.transformToNDJson)({ strict, sortObjects }),
|
|
78
|
-
...(gzip ? [(0, node_zlib_1.createGzip)(zlibOptions)] : []),
|
|
78
|
+
...(gzip ? [(0, node_zlib_1.createGzip)(zlibOptions)] : []), // optional gzip
|
|
79
79
|
node_fs_1.default.createWriteStream(filePath),
|
|
80
80
|
]);
|
|
81
81
|
const { size: sizeBytes } = await promises_1.default.stat(filePath);
|
|
@@ -19,7 +19,7 @@ async function dbPipelineRestore(opt) {
|
|
|
19
19
|
const onlyTables = opt.tables && new Set(opt.tables);
|
|
20
20
|
const sinceUpdatedStr = sinceUpdated ? ' since ' + (0, nodejs_lib_1.grey)((0, js_lib_1.localTime)(sinceUpdated).toPretty()) : '';
|
|
21
21
|
console.log(`>> ${(0, nodejs_lib_1.dimWhite)('dbPipelineRestore')} started in ${(0, nodejs_lib_1.grey)(inputDirPath)}...${sinceUpdatedStr}`);
|
|
22
|
-
|
|
22
|
+
nodejs_lib_1.fs2.ensureDir(inputDirPath);
|
|
23
23
|
const tablesToGzip = new Set();
|
|
24
24
|
const sizeByTable = {};
|
|
25
25
|
const statsPerTable = {};
|
|
@@ -54,7 +54,7 @@ async function dbPipelineRestore(opt) {
|
|
|
54
54
|
console.warn(`${schemaFilePath} does not exist!`);
|
|
55
55
|
return;
|
|
56
56
|
}
|
|
57
|
-
const schema = await
|
|
57
|
+
const schema = await nodejs_lib_1.fs2.readJsonAsync(schemaFilePath);
|
|
58
58
|
await db.createTable(table, schema, { dropIfExists: true });
|
|
59
59
|
});
|
|
60
60
|
}
|
|
@@ -69,7 +69,7 @@ async function dbPipelineRestore(opt) {
|
|
|
69
69
|
await (0, nodejs_lib_1._pipeline)([
|
|
70
70
|
node_fs_1.default.createReadStream(filePath),
|
|
71
71
|
...(gzip ? [(0, node_zlib_1.createUnzip)()] : []),
|
|
72
|
-
(0, nodejs_lib_1.transformSplit)(),
|
|
72
|
+
(0, nodejs_lib_1.transformSplit)(), // splits by \n
|
|
73
73
|
(0, nodejs_lib_1.transformJsonParse)({ strict }),
|
|
74
74
|
(0, nodejs_lib_1.transformTap)(() => rows++),
|
|
75
75
|
(0, nodejs_lib_1.transformLogProgress)({
|
package/dist/query/dbQuery.d.ts
CHANGED
|
@@ -84,12 +84,12 @@ export declare class DBQuery<ROW extends ObjectWithId = AnyObjectWithId> {
|
|
|
84
84
|
/**
|
|
85
85
|
* DBQuery that has additional method to support Fluent API style.
|
|
86
86
|
*/
|
|
87
|
-
export declare class RunnableDBQuery<BM extends Partial<ObjectWithId
|
|
88
|
-
dao: CommonDao<BM, DBM, TM
|
|
87
|
+
export declare class RunnableDBQuery<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId = Saved<BM>, TM extends AnyObject = BM> extends DBQuery<DBM> {
|
|
88
|
+
dao: CommonDao<BM, DBM, TM>;
|
|
89
89
|
/**
|
|
90
90
|
* Pass `table` to override table.
|
|
91
91
|
*/
|
|
92
|
-
constructor(dao: CommonDao<BM, DBM, TM
|
|
92
|
+
constructor(dao: CommonDao<BM, DBM, TM>, table?: string);
|
|
93
93
|
runQuery(opt?: CommonDaoOptions): Promise<Saved<BM>[]>;
|
|
94
94
|
runQuerySingleColumn<T = any>(opt?: CommonDaoOptions): Promise<T[]>;
|
|
95
95
|
runQueryAsDBM(opt?: CommonDaoOptions): Promise<DBM[]>;
|
|
@@ -103,8 +103,8 @@ export declare class RunnableDBQuery<BM extends Partial<ObjectWithId<ID>>, DBM e
|
|
|
103
103
|
streamQueryAsDBMForEach(mapper: AsyncMapper<DBM, void>, opt?: CommonDaoStreamForEachOptions<DBM>): Promise<void>;
|
|
104
104
|
streamQuery(opt?: CommonDaoStreamOptions<Saved<BM>>): ReadableTyped<Saved<BM>>;
|
|
105
105
|
streamQueryAsDBM(opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM>;
|
|
106
|
-
queryIds(opt?: CommonDaoOptions): Promise<
|
|
107
|
-
streamQueryIds(opt?: CommonDaoStreamOptions<
|
|
108
|
-
streamQueryIdsForEach(mapper: AsyncMapper<
|
|
106
|
+
queryIds(opt?: CommonDaoOptions): Promise<string[]>;
|
|
107
|
+
streamQueryIds(opt?: CommonDaoStreamOptions<string>): ReadableTyped<string>;
|
|
108
|
+
streamQueryIdsForEach(mapper: AsyncMapper<string, void>, opt?: CommonDaoStreamForEachOptions<string>): Promise<void>;
|
|
109
109
|
deleteByQuery(opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
|
|
110
110
|
}
|
|
@@ -25,7 +25,7 @@ exports.testItemTMSchema = (0, nodejs_lib_1.objectSchema)({
|
|
|
25
25
|
});
|
|
26
26
|
exports.testItemBMJsonSchema = js_lib_1.jsonSchema
|
|
27
27
|
.rootObject({
|
|
28
|
-
id: js_lib_1.jsonSchema.string(),
|
|
28
|
+
id: js_lib_1.jsonSchema.string(), // todo: not strictly needed here
|
|
29
29
|
k1: js_lib_1.jsonSchema.string(),
|
|
30
30
|
k2: js_lib_1.jsonSchema.oneOf([js_lib_1.jsonSchema.string(), js_lib_1.jsonSchema.null()]).optional(),
|
|
31
31
|
k3: js_lib_1.jsonSchema.number().optional(),
|
|
@@ -31,8 +31,8 @@ class CommonTimeSeriesDao {
|
|
|
31
31
|
if (!dataPoints.length)
|
|
32
32
|
return;
|
|
33
33
|
const rows = dataPoints.map(([ts, v]) => ({
|
|
34
|
-
id: String(ts),
|
|
35
|
-
ts,
|
|
34
|
+
id: String(ts), // Convert Number id into String id, as per CommonDB
|
|
35
|
+
ts, // to allow querying by ts, since querying by id is not always available (Datastore is one example)
|
|
36
36
|
v,
|
|
37
37
|
}));
|
|
38
38
|
await this.cfg.db.saveBatch(`${series}${_TIMESERIES_RAW}`, rows);
|
|
@@ -46,8 +46,8 @@ class CommonTimeSeriesDao {
|
|
|
46
46
|
const tx = __1.DBTransaction.create();
|
|
47
47
|
ops.forEach(op => {
|
|
48
48
|
const rows = op.dataPoints.map(([ts, v]) => ({
|
|
49
|
-
id: String(ts),
|
|
50
|
-
ts,
|
|
49
|
+
id: String(ts), // Convert Number id into String id, as per CommonDB
|
|
50
|
+
ts, // to allow querying by ts, since querying by id is not always available (Datastore is one example)
|
|
51
51
|
v,
|
|
52
52
|
}));
|
|
53
53
|
tx.saveBatch(`${op.series}${_TIMESERIES_RAW}`, rows);
|
|
@@ -3,6 +3,6 @@ import { DBQuery, DBQueryFilter, DBQueryFilterOperator, DBQueryOrder } from '../
|
|
|
3
3
|
export declare const commonDBOptionsSchema: import("joi").ObjectSchema<CommonDBOptions>;
|
|
4
4
|
export declare const commonDBSaveOptionsSchema: import("joi").ObjectSchema<CommonDBSaveOptions<any>>;
|
|
5
5
|
export declare const dbQueryFilterOperatorSchema: import("@naturalcycles/nodejs-lib/dist/validation/joi/string.extensions").StringSchema<DBQueryFilterOperator>;
|
|
6
|
-
export declare const dbQueryFilterSchema: import("joi").ObjectSchema<DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId
|
|
7
|
-
export declare const dbQueryOrderSchema: import("joi").ObjectSchema<DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId
|
|
6
|
+
export declare const dbQueryFilterSchema: import("joi").ObjectSchema<DBQueryFilter<import("@naturalcycles/js-lib").AnyObjectWithId>>;
|
|
7
|
+
export declare const dbQueryOrderSchema: import("joi").ObjectSchema<DBQueryOrder<import("@naturalcycles/js-lib").AnyObjectWithId>>;
|
|
8
8
|
export declare const dbQuerySchema: import("joi").ObjectSchema<DBQuery<any>>;
|
package/package.json
CHANGED
|
@@ -9,8 +9,7 @@ import {
|
|
|
9
9
|
transformToNDJson,
|
|
10
10
|
writablePushToArray,
|
|
11
11
|
_pipeline,
|
|
12
|
-
|
|
13
|
-
_pathExists,
|
|
12
|
+
fs2,
|
|
14
13
|
} from '@naturalcycles/nodejs-lib'
|
|
15
14
|
import { DBSaveBatchOperation } from '../../db.model'
|
|
16
15
|
import { FileDBPersistencePlugin } from './file.db.model'
|
|
@@ -50,11 +49,11 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
|
|
|
50
49
|
}
|
|
51
50
|
|
|
52
51
|
async loadFile<ROW extends ObjectWithId>(table: string): Promise<ROW[]> {
|
|
53
|
-
await
|
|
52
|
+
await fs2.ensureDirAsync(this.cfg.storagePath)
|
|
54
53
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
|
|
55
54
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`
|
|
56
55
|
|
|
57
|
-
if (!(await
|
|
56
|
+
if (!(await fs2.pathExistsAsync(filePath))) return []
|
|
58
57
|
|
|
59
58
|
const transformUnzip = this.cfg.gzip ? [createUnzip()] : []
|
|
60
59
|
|
|
@@ -76,7 +75,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
|
|
|
76
75
|
}
|
|
77
76
|
|
|
78
77
|
async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {
|
|
79
|
-
await
|
|
78
|
+
await fs2.ensureDirAsync(this.cfg.storagePath)
|
|
80
79
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
|
|
81
80
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`
|
|
82
81
|
const transformZip = this.cfg.gzip ? [createGzip()] : []
|
|
@@ -25,10 +25,9 @@ import {
|
|
|
25
25
|
transformToNDJson,
|
|
26
26
|
writablePushToArray,
|
|
27
27
|
_pipeline,
|
|
28
|
-
_emptyDir,
|
|
29
|
-
_ensureDir,
|
|
30
28
|
dimGrey,
|
|
31
29
|
yellow,
|
|
30
|
+
fs2,
|
|
32
31
|
} from '@naturalcycles/nodejs-lib'
|
|
33
32
|
import { CommonDB, DBIncrement, DBPatch, DBTransaction, queryInMemory } from '../..'
|
|
34
33
|
import {
|
|
@@ -283,7 +282,7 @@ export class InMemoryDB implements CommonDB {
|
|
|
283
282
|
|
|
284
283
|
const started = Date.now()
|
|
285
284
|
|
|
286
|
-
await
|
|
285
|
+
await fs2.emptyDirAsync(persistentStoragePath)
|
|
287
286
|
|
|
288
287
|
const transformZip = persistZip ? [createGzip()] : []
|
|
289
288
|
let tables = 0
|
|
@@ -318,7 +317,7 @@ export class InMemoryDB implements CommonDB {
|
|
|
318
317
|
|
|
319
318
|
const started = Date.now()
|
|
320
319
|
|
|
321
|
-
await
|
|
320
|
+
await fs2.ensureDirAsync(persistentStoragePath)
|
|
322
321
|
|
|
323
322
|
this.data = {} // empty it in the beginning!
|
|
324
323
|
|
package/src/common.db.ts
CHANGED
|
@@ -18,31 +18,26 @@ import {
|
|
|
18
18
|
import { CommonDB } from '../common.db'
|
|
19
19
|
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model'
|
|
20
20
|
|
|
21
|
-
export interface CommonDaoHooks<
|
|
22
|
-
BM extends Partial<ObjectWithId<ID>>,
|
|
23
|
-
DBM extends ObjectWithId<ID>,
|
|
24
|
-
TM,
|
|
25
|
-
ID extends string | number,
|
|
26
|
-
> {
|
|
21
|
+
export interface CommonDaoHooks<BM extends Partial<ObjectWithId>, DBM extends ObjectWithId, TM> {
|
|
27
22
|
/**
|
|
28
23
|
* Allows to override the id generation function.
|
|
29
24
|
* By default it uses `stringId` from nodejs-lib
|
|
30
25
|
* (which uses lowercase alphanumberic alphabet and the size of 16).
|
|
31
26
|
*/
|
|
32
|
-
createRandomId: () =>
|
|
27
|
+
createRandomId: () => string
|
|
33
28
|
|
|
34
29
|
/**
|
|
35
30
|
* createNaturalId hook is called (tried) first.
|
|
36
31
|
* If it doesn't exist - createRandomId is called.
|
|
37
32
|
*/
|
|
38
|
-
createNaturalId: (obj: DBM | BM) =>
|
|
33
|
+
createNaturalId: (obj: DBM | BM) => string
|
|
39
34
|
|
|
40
35
|
/**
|
|
41
36
|
* It's a counter-part of `createNaturalId`.
|
|
42
37
|
* Allows to provide a parser function to parse "natural id" into
|
|
43
38
|
* DBM components (e.g accountId and some other property that is part of the id).
|
|
44
39
|
*/
|
|
45
|
-
parseNaturalId: (id:
|
|
40
|
+
parseNaturalId: (id: string) => Partial<DBM>
|
|
46
41
|
|
|
47
42
|
/**
|
|
48
43
|
* It is called only on `dao.create` method.
|
|
@@ -137,10 +132,9 @@ export enum CommonDaoLogLevel {
|
|
|
137
132
|
}
|
|
138
133
|
|
|
139
134
|
export interface CommonDaoCfg<
|
|
140
|
-
BM extends Partial<ObjectWithId
|
|
141
|
-
DBM extends ObjectWithId
|
|
135
|
+
BM extends Partial<ObjectWithId>,
|
|
136
|
+
DBM extends ObjectWithId = Saved<BM>,
|
|
142
137
|
TM = BM,
|
|
143
|
-
ID extends string | number = string,
|
|
144
138
|
> {
|
|
145
139
|
db: CommonDB
|
|
146
140
|
table: string
|
|
@@ -187,12 +181,7 @@ export interface CommonDaoCfg<
|
|
|
187
181
|
logStarted?: boolean
|
|
188
182
|
|
|
189
183
|
// Hooks are designed with inspiration from got/ky interface
|
|
190
|
-
hooks?: Partial<CommonDaoHooks<BM, DBM, TM
|
|
191
|
-
|
|
192
|
-
/**
|
|
193
|
-
* Defaults to 'string'
|
|
194
|
-
*/
|
|
195
|
-
idType?: 'string' | 'number'
|
|
184
|
+
hooks?: Partial<CommonDaoHooks<BM, DBM, TM>>
|
|
196
185
|
|
|
197
186
|
/**
|
|
198
187
|
* Defaults to true.
|
|
@@ -78,18 +78,16 @@ const isCI = !!process.env['CI']
|
|
|
78
78
|
* TM = Transport model (optimized to be sent over the wire)
|
|
79
79
|
*/
|
|
80
80
|
export class CommonDao<
|
|
81
|
-
BM extends Partial<ObjectWithId
|
|
82
|
-
DBM extends ObjectWithId
|
|
81
|
+
BM extends Partial<ObjectWithId>,
|
|
82
|
+
DBM extends ObjectWithId = Saved<BM>,
|
|
83
83
|
TM extends AnyObject = BM,
|
|
84
|
-
ID extends string | number = NonNullable<BM['id']>,
|
|
85
84
|
> {
|
|
86
|
-
constructor(public cfg: CommonDaoCfg<BM, DBM, TM
|
|
85
|
+
constructor(public cfg: CommonDaoCfg<BM, DBM, TM>) {
|
|
87
86
|
this.cfg = {
|
|
88
87
|
// Default is to NOT log in AppEngine and in CI,
|
|
89
88
|
// otherwise to log Operations
|
|
90
89
|
// e.g in Dev (local machine), Test - it will log operations (useful for debugging)
|
|
91
90
|
logLevel: isGAE || isCI ? CommonDaoLogLevel.NONE : CommonDaoLogLevel.OPERATIONS,
|
|
92
|
-
idType: 'string',
|
|
93
91
|
createId: true,
|
|
94
92
|
assignGeneratedIds: false,
|
|
95
93
|
created: true,
|
|
@@ -106,16 +104,11 @@ export class CommonDao<
|
|
|
106
104
|
anonymize: dbm => dbm,
|
|
107
105
|
onValidationError: err => err,
|
|
108
106
|
...cfg.hooks,
|
|
109
|
-
} satisfies Partial<CommonDaoHooks<BM, DBM, TM
|
|
107
|
+
} satisfies Partial<CommonDaoHooks<BM, DBM, TM>>,
|
|
110
108
|
}
|
|
111
109
|
|
|
112
110
|
if (this.cfg.createId) {
|
|
113
|
-
|
|
114
|
-
this.cfg.idType === 'string',
|
|
115
|
-
'db-lib: automatic generation of non-string ids is not supported',
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
this.cfg.hooks!.createRandomId ||= () => stringId() as ID
|
|
111
|
+
this.cfg.hooks!.createRandomId ||= () => stringId()
|
|
119
112
|
} else {
|
|
120
113
|
delete this.cfg.hooks!.createRandomId
|
|
121
114
|
}
|
|
@@ -131,8 +124,8 @@ export class CommonDao<
|
|
|
131
124
|
|
|
132
125
|
// GET
|
|
133
126
|
async getById(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
|
|
134
|
-
async getById(id?:
|
|
135
|
-
async getById(id?:
|
|
127
|
+
async getById(id?: string | null, opt?: CommonDaoOptions): Promise<Saved<BM> | null>
|
|
128
|
+
async getById(id?: string | null, opt: CommonDaoOptions = {}): Promise<Saved<BM> | null> {
|
|
136
129
|
if (!id) return null
|
|
137
130
|
const op = `getById(${id})`
|
|
138
131
|
const table = opt.table || this.cfg.table
|
|
@@ -148,14 +141,22 @@ export class CommonDao<
|
|
|
148
141
|
return bm || null
|
|
149
142
|
}
|
|
150
143
|
|
|
151
|
-
async getByIdOrEmpty(
|
|
144
|
+
async getByIdOrEmpty(
|
|
145
|
+
id: string,
|
|
146
|
+
part: Partial<BM> = {},
|
|
147
|
+
opt?: CommonDaoOptions,
|
|
148
|
+
): Promise<Saved<BM>> {
|
|
152
149
|
const bm = await this.getById(id, opt)
|
|
153
150
|
if (bm) return bm
|
|
154
151
|
|
|
155
152
|
return this.create({ ...part, id }, opt)
|
|
156
153
|
}
|
|
157
154
|
|
|
158
|
-
async getByIdAsDBMOrEmpty(
|
|
155
|
+
async getByIdAsDBMOrEmpty(
|
|
156
|
+
id: string,
|
|
157
|
+
part: Partial<BM> = {},
|
|
158
|
+
opt?: CommonDaoOptions,
|
|
159
|
+
): Promise<DBM> {
|
|
159
160
|
const dbm = await this.getByIdAsDBM(id, opt)
|
|
160
161
|
if (dbm) return dbm
|
|
161
162
|
|
|
@@ -164,8 +165,8 @@ export class CommonDao<
|
|
|
164
165
|
}
|
|
165
166
|
|
|
166
167
|
async getByIdAsDBM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
|
|
167
|
-
async getByIdAsDBM(id?:
|
|
168
|
-
async getByIdAsDBM(id?:
|
|
168
|
+
async getByIdAsDBM(id?: string | null, opt?: CommonDaoOptions): Promise<DBM | null>
|
|
169
|
+
async getByIdAsDBM(id?: string | null, opt: CommonDaoOptions = {}): Promise<DBM | null> {
|
|
169
170
|
if (!id) return null
|
|
170
171
|
const op = `getByIdAsDBM(${id})`
|
|
171
172
|
const table = opt.table || this.cfg.table
|
|
@@ -183,8 +184,8 @@ export class CommonDao<
|
|
|
183
184
|
}
|
|
184
185
|
|
|
185
186
|
async getByIdAsTM(id: undefined | null, opt?: CommonDaoOptions): Promise<null>
|
|
186
|
-
async getByIdAsTM(id?:
|
|
187
|
-
async getByIdAsTM(id?:
|
|
187
|
+
async getByIdAsTM(id?: string | null, opt?: CommonDaoOptions): Promise<TM | null>
|
|
188
|
+
async getByIdAsTM(id?: string | null, opt: CommonDaoOptions = {}): Promise<TM | null> {
|
|
188
189
|
if (!id) return null
|
|
189
190
|
const op = `getByIdAsTM(${id})`
|
|
190
191
|
const table = opt.table || this.cfg.table
|
|
@@ -204,7 +205,7 @@ export class CommonDao<
|
|
|
204
205
|
return tm || null
|
|
205
206
|
}
|
|
206
207
|
|
|
207
|
-
async getByIds(ids:
|
|
208
|
+
async getByIds(ids: string[], opt: CommonDaoOptions = {}): Promise<Saved<BM>[]> {
|
|
208
209
|
if (!ids.length) return []
|
|
209
210
|
const op = `getByIds ${ids.length} id(s) (${_truncate(ids.slice(0, 10).join(', '), 50)})`
|
|
210
211
|
const table = opt.table || this.cfg.table
|
|
@@ -221,7 +222,7 @@ export class CommonDao<
|
|
|
221
222
|
return bms
|
|
222
223
|
}
|
|
223
224
|
|
|
224
|
-
async getByIdsAsDBM(ids:
|
|
225
|
+
async getByIdsAsDBM(ids: string[], opt: CommonDaoOptions = {}): Promise<DBM[]> {
|
|
225
226
|
if (!ids.length) return []
|
|
226
227
|
const op = `getByIdsAsDBM ${ids.length} id(s) (${_truncate(ids.slice(0, 10).join(', '), 50)})`
|
|
227
228
|
const table = opt.table || this.cfg.table
|
|
@@ -237,7 +238,7 @@ export class CommonDao<
|
|
|
237
238
|
return dbms
|
|
238
239
|
}
|
|
239
240
|
|
|
240
|
-
async requireById(id:
|
|
241
|
+
async requireById(id: string, opt: CommonDaoOptions = {}): Promise<Saved<BM>> {
|
|
241
242
|
const r = await this.getById(id, opt)
|
|
242
243
|
if (!r) {
|
|
243
244
|
this.throwRequiredError(id, opt)
|
|
@@ -245,7 +246,7 @@ export class CommonDao<
|
|
|
245
246
|
return r
|
|
246
247
|
}
|
|
247
248
|
|
|
248
|
-
async requireByIdAsDBM(id:
|
|
249
|
+
async requireByIdAsDBM(id: string, opt: CommonDaoOptions = {}): Promise<DBM> {
|
|
249
250
|
const r = await this.getByIdAsDBM(id, opt)
|
|
250
251
|
if (!r) {
|
|
251
252
|
this.throwRequiredError(id, opt)
|
|
@@ -253,7 +254,7 @@ export class CommonDao<
|
|
|
253
254
|
return r
|
|
254
255
|
}
|
|
255
256
|
|
|
256
|
-
private throwRequiredError(id:
|
|
257
|
+
private throwRequiredError(id: string, opt: CommonDaoOptions): never {
|
|
257
258
|
const table = opt.table || this.cfg.table
|
|
258
259
|
throw new AppError(`DB row required, but not found in ${table}`, {
|
|
259
260
|
table,
|
|
@@ -311,8 +312,8 @@ export class CommonDao<
|
|
|
311
312
|
/**
|
|
312
313
|
* Pass `table` to override table
|
|
313
314
|
*/
|
|
314
|
-
query(table?: string): RunnableDBQuery<BM, DBM, TM
|
|
315
|
-
return new RunnableDBQuery<BM, DBM, TM
|
|
315
|
+
query(table?: string): RunnableDBQuery<BM, DBM, TM> {
|
|
316
|
+
return new RunnableDBQuery<BM, DBM, TM>(this, table)
|
|
316
317
|
}
|
|
317
318
|
|
|
318
319
|
async runQuery(q: DBQuery<DBM>, opt?: CommonDaoOptions): Promise<Saved<BM>[]> {
|
|
@@ -614,21 +615,21 @@ export class CommonDao<
|
|
|
614
615
|
)
|
|
615
616
|
}
|
|
616
617
|
|
|
617
|
-
async queryIds(q: DBQuery<DBM>, opt: CommonDaoOptions = {}): Promise<
|
|
618
|
+
async queryIds(q: DBQuery<DBM>, opt: CommonDaoOptions = {}): Promise<string[]> {
|
|
618
619
|
q.table = opt.table || q.table
|
|
619
620
|
const { rows } = await this.cfg.db.runQuery(q.select(['id']), opt)
|
|
620
621
|
return rows.map(r => r.id)
|
|
621
622
|
}
|
|
622
623
|
|
|
623
|
-
streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<
|
|
624
|
+
streamQueryIds(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<string> = {}): ReadableTyped<string> {
|
|
624
625
|
q.table = opt.table || q.table
|
|
625
626
|
opt.errorMode ||= ErrorMode.SUPPRESS
|
|
626
627
|
|
|
627
|
-
const stream: ReadableTyped<
|
|
628
|
+
const stream: ReadableTyped<string> = this.cfg.db
|
|
628
629
|
.streamQuery<DBM>(q.select(['id']), opt)
|
|
629
630
|
.on('error', err => stream.emit('error', err))
|
|
630
631
|
.pipe(
|
|
631
|
-
transformMapSimple<DBM,
|
|
632
|
+
transformMapSimple<DBM, string>(objectWithId => objectWithId.id, {
|
|
632
633
|
errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
|
|
633
634
|
}),
|
|
634
635
|
)
|
|
@@ -638,8 +639,8 @@ export class CommonDao<
|
|
|
638
639
|
|
|
639
640
|
async streamQueryIdsForEach(
|
|
640
641
|
q: DBQuery<DBM>,
|
|
641
|
-
mapper: AsyncMapper<
|
|
642
|
-
opt: CommonDaoStreamForEachOptions<
|
|
642
|
+
mapper: AsyncMapper<string, void>,
|
|
643
|
+
opt: CommonDaoStreamForEachOptions<string> = {},
|
|
643
644
|
): Promise<void> {
|
|
644
645
|
q.table = opt.table || q.table
|
|
645
646
|
opt.errorMode ||= ErrorMode.SUPPRESS
|
|
@@ -650,11 +651,11 @@ export class CommonDao<
|
|
|
650
651
|
|
|
651
652
|
await _pipeline([
|
|
652
653
|
this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
|
|
653
|
-
transformMapSimple<DBM,
|
|
654
|
+
transformMapSimple<DBM, string>(objectWithId => {
|
|
654
655
|
count++
|
|
655
656
|
return objectWithId.id
|
|
656
657
|
}),
|
|
657
|
-
transformMap<
|
|
658
|
+
transformMap<string, void>(mapper, {
|
|
658
659
|
...opt,
|
|
659
660
|
predicate: _passthroughPredicate,
|
|
660
661
|
}),
|
|
@@ -734,26 +735,26 @@ export class CommonDao<
|
|
|
734
735
|
}
|
|
735
736
|
},
|
|
736
737
|
deleteByIds: async (
|
|
737
|
-
ids:
|
|
738
|
+
ids: string[],
|
|
738
739
|
opt: CommonDaoOptions = {},
|
|
739
740
|
): Promise<DBDeleteByIdsOperation | undefined> => {
|
|
740
741
|
if (!ids.length) return
|
|
741
742
|
return {
|
|
742
743
|
type: 'deleteByIds',
|
|
743
744
|
table: this.cfg.table,
|
|
744
|
-
ids
|
|
745
|
+
ids,
|
|
745
746
|
opt,
|
|
746
747
|
}
|
|
747
748
|
},
|
|
748
749
|
deleteById: async (
|
|
749
|
-
id:
|
|
750
|
+
id: string | null | undefined,
|
|
750
751
|
opt: CommonDaoOptions = {},
|
|
751
752
|
): Promise<DBDeleteByIdsOperation | undefined> => {
|
|
752
753
|
if (!id) return
|
|
753
754
|
return {
|
|
754
755
|
type: 'deleteByIds',
|
|
755
756
|
table: this.cfg.table,
|
|
756
|
-
ids: [id
|
|
757
|
+
ids: [id],
|
|
757
758
|
opt,
|
|
758
759
|
}
|
|
759
760
|
},
|
|
@@ -848,7 +849,7 @@ export class CommonDao<
|
|
|
848
849
|
* 3. Saves (as fast as possible since the read) with the Patch applied, but only if the data has changed.
|
|
849
850
|
*/
|
|
850
851
|
async patchById(
|
|
851
|
-
id:
|
|
852
|
+
id: string,
|
|
852
853
|
patch: Partial<BM>,
|
|
853
854
|
opt: CommonDaoSaveBatchOptions<DBM> = {},
|
|
854
855
|
): Promise<Saved<BM>> {
|
|
@@ -1109,8 +1110,8 @@ export class CommonDao<
|
|
|
1109
1110
|
* @returns number of deleted items
|
|
1110
1111
|
*/
|
|
1111
1112
|
async deleteById(id: undefined | null, opt?: CommonDaoOptions): Promise<0>
|
|
1112
|
-
async deleteById(id?:
|
|
1113
|
-
async deleteById(id?:
|
|
1113
|
+
async deleteById(id?: string | null, opt?: CommonDaoOptions): Promise<number>
|
|
1114
|
+
async deleteById(id?: string | null, opt: CommonDaoOptions = {}): Promise<number> {
|
|
1114
1115
|
if (!id) return 0
|
|
1115
1116
|
this.requireWriteAccess()
|
|
1116
1117
|
this.requireObjectMutability(opt)
|
|
@@ -1122,7 +1123,7 @@ export class CommonDao<
|
|
|
1122
1123
|
return count
|
|
1123
1124
|
}
|
|
1124
1125
|
|
|
1125
|
-
async deleteByIds(ids:
|
|
1126
|
+
async deleteByIds(ids: string[], opt: CommonDaoOptions = {}): Promise<number> {
|
|
1126
1127
|
if (!ids.length) return 0
|
|
1127
1128
|
this.requireWriteAccess()
|
|
1128
1129
|
this.requireObjectMutability(opt)
|
|
@@ -1155,7 +1156,7 @@ export class CommonDao<
|
|
|
1155
1156
|
|
|
1156
1157
|
await _pipeline([
|
|
1157
1158
|
this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
|
|
1158
|
-
transformMapSimple<DBM,
|
|
1159
|
+
transformMapSimple<DBM, string>(objectWithId => objectWithId.id, {
|
|
1159
1160
|
errorMode: ErrorMode.SUPPRESS,
|
|
1160
1161
|
}),
|
|
1161
1162
|
transformBuffer<string>({ batchSize }),
|
|
@@ -1188,11 +1189,15 @@ export class CommonDao<
|
|
|
1188
1189
|
return deleted
|
|
1189
1190
|
}
|
|
1190
1191
|
|
|
1191
|
-
async updateById(id:
|
|
1192
|
+
async updateById(id: string, patch: DBPatch<DBM>, opt: CommonDaoOptions = {}): Promise<number> {
|
|
1192
1193
|
return await this.updateByQuery(this.query().filterEq('id', id), patch, opt)
|
|
1193
1194
|
}
|
|
1194
1195
|
|
|
1195
|
-
async updateByIds(
|
|
1196
|
+
async updateByIds(
|
|
1197
|
+
ids: string[],
|
|
1198
|
+
patch: DBPatch<DBM>,
|
|
1199
|
+
opt: CommonDaoOptions = {},
|
|
1200
|
+
): Promise<number> {
|
|
1196
1201
|
if (!ids.length) return 0
|
|
1197
1202
|
return await this.updateByQuery(this.query().filterIn('id', ids), patch, opt)
|
|
1198
1203
|
}
|
package/src/model.util.ts
CHANGED
|
@@ -12,8 +12,8 @@ export function createdUpdatedFields(
|
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
export function createdUpdatedIdFields(
|
|
15
|
-
existingObject?: Partial<CreatedUpdatedId
|
|
16
|
-
): CreatedUpdatedId
|
|
15
|
+
existingObject?: Partial<CreatedUpdatedId> | null,
|
|
16
|
+
): CreatedUpdatedId {
|
|
17
17
|
const now = Math.floor(Date.now() / 1000)
|
|
18
18
|
return {
|
|
19
19
|
created: existingObject?.created || now,
|
|
@@ -20,14 +20,11 @@ import {
|
|
|
20
20
|
transformTap,
|
|
21
21
|
transformToNDJson,
|
|
22
22
|
_pipeline,
|
|
23
|
-
_ensureDirSync,
|
|
24
|
-
_pathExistsSync,
|
|
25
|
-
_ensureFileSync,
|
|
26
|
-
_writeJson,
|
|
27
23
|
boldWhite,
|
|
28
24
|
dimWhite,
|
|
29
25
|
grey,
|
|
30
26
|
yellow,
|
|
27
|
+
fs2,
|
|
31
28
|
} from '@naturalcycles/nodejs-lib'
|
|
32
29
|
import { CommonDB } from '../common.db'
|
|
33
30
|
import { DBQuery } from '../index'
|
|
@@ -180,7 +177,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
|
|
|
180
177
|
|
|
181
178
|
console.log(`>> ${dimWhite('dbPipelineBackup')} started in ${grey(outputDirPath)}...`)
|
|
182
179
|
|
|
183
|
-
|
|
180
|
+
fs2.ensureDir(outputDirPath)
|
|
184
181
|
|
|
185
182
|
tables ||= await db.getTables()
|
|
186
183
|
|
|
@@ -213,20 +210,20 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
|
|
|
213
210
|
const filePath = `${outputDirPath}/${table}.ndjson` + (gzip ? '.gz' : '')
|
|
214
211
|
const schemaFilePath = `${outputDirPath}/${table}.schema.json`
|
|
215
212
|
|
|
216
|
-
if (protectFromOverwrite &&
|
|
213
|
+
if (protectFromOverwrite && fs2.pathExists(filePath)) {
|
|
217
214
|
throw new AppError(`dbPipelineBackup: output file exists: ${filePath}`)
|
|
218
215
|
}
|
|
219
216
|
|
|
220
217
|
const started = Date.now()
|
|
221
218
|
let rows = 0
|
|
222
219
|
|
|
223
|
-
|
|
220
|
+
fs2.ensureFile(filePath)
|
|
224
221
|
|
|
225
222
|
// console.log(`>> ${grey(filePath)} started...`)
|
|
226
223
|
|
|
227
224
|
if (emitSchemaFromDB) {
|
|
228
225
|
const schema = await db.getTableSchema(table)
|
|
229
|
-
await
|
|
226
|
+
await fs2.writeJsonAsync(schemaFilePath, schema, { spaces: 2 })
|
|
230
227
|
console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
|
|
231
228
|
}
|
|
232
229
|
|
|
@@ -25,12 +25,11 @@ import {
|
|
|
25
25
|
transformTap,
|
|
26
26
|
writableForEach,
|
|
27
27
|
_pipeline,
|
|
28
|
-
_ensureDirSync,
|
|
29
|
-
_readJson,
|
|
30
28
|
boldWhite,
|
|
31
29
|
dimWhite,
|
|
32
30
|
grey,
|
|
33
31
|
yellow,
|
|
32
|
+
fs2,
|
|
34
33
|
} from '@naturalcycles/nodejs-lib'
|
|
35
34
|
import { CommonDB } from '../common.db'
|
|
36
35
|
import { CommonDBSaveOptions } from '../index'
|
|
@@ -145,7 +144,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
|
|
|
145
144
|
`>> ${dimWhite('dbPipelineRestore')} started in ${grey(inputDirPath)}...${sinceUpdatedStr}`,
|
|
146
145
|
)
|
|
147
146
|
|
|
148
|
-
|
|
147
|
+
fs2.ensureDir(inputDirPath)
|
|
149
148
|
|
|
150
149
|
const tablesToGzip = new Set<string>()
|
|
151
150
|
const sizeByTable: Record<string, number> = {}
|
|
@@ -185,7 +184,7 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
|
|
|
185
184
|
return
|
|
186
185
|
}
|
|
187
186
|
|
|
188
|
-
const schema = await
|
|
187
|
+
const schema = await fs2.readJsonAsync<JsonSchemaObject<any>>(schemaFilePath)
|
|
189
188
|
await db.createTable(table, schema, { dropIfExists: true })
|
|
190
189
|
})
|
|
191
190
|
}
|
package/src/query/dbQuery.ts
CHANGED
|
@@ -237,16 +237,15 @@ export class DBQuery<ROW extends ObjectWithId = AnyObjectWithId> {
|
|
|
237
237
|
* DBQuery that has additional method to support Fluent API style.
|
|
238
238
|
*/
|
|
239
239
|
export class RunnableDBQuery<
|
|
240
|
-
BM extends Partial<ObjectWithId
|
|
241
|
-
DBM extends ObjectWithId
|
|
240
|
+
BM extends Partial<ObjectWithId>,
|
|
241
|
+
DBM extends ObjectWithId = Saved<BM>,
|
|
242
242
|
TM extends AnyObject = BM,
|
|
243
|
-
ID extends string | number = string,
|
|
244
243
|
> extends DBQuery<DBM> {
|
|
245
244
|
/**
|
|
246
245
|
* Pass `table` to override table.
|
|
247
246
|
*/
|
|
248
247
|
constructor(
|
|
249
|
-
public dao: CommonDao<BM, DBM, TM
|
|
248
|
+
public dao: CommonDao<BM, DBM, TM>,
|
|
250
249
|
table?: string,
|
|
251
250
|
) {
|
|
252
251
|
super(table || dao.cfg.table)
|
|
@@ -310,17 +309,17 @@ export class RunnableDBQuery<
|
|
|
310
309
|
return this.dao.streamQueryAsDBM(this, opt)
|
|
311
310
|
}
|
|
312
311
|
|
|
313
|
-
async queryIds(opt?: CommonDaoOptions): Promise<
|
|
312
|
+
async queryIds(opt?: CommonDaoOptions): Promise<string[]> {
|
|
314
313
|
return await this.dao.queryIds(this, opt)
|
|
315
314
|
}
|
|
316
315
|
|
|
317
|
-
streamQueryIds(opt?: CommonDaoStreamOptions<
|
|
316
|
+
streamQueryIds(opt?: CommonDaoStreamOptions<string>): ReadableTyped<string> {
|
|
318
317
|
return this.dao.streamQueryIds(this, opt)
|
|
319
318
|
}
|
|
320
319
|
|
|
321
320
|
async streamQueryIdsForEach(
|
|
322
|
-
mapper: AsyncMapper<
|
|
323
|
-
opt?: CommonDaoStreamForEachOptions<
|
|
321
|
+
mapper: AsyncMapper<string, void>,
|
|
322
|
+
opt?: CommonDaoStreamForEachOptions<string>,
|
|
324
323
|
): Promise<void> {
|
|
325
324
|
await this.dao.streamQueryIdsForEach(this, mapper, opt)
|
|
326
325
|
}
|