@naturalcycles/db-lib 10.5.0 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/cachedb/cache.db.d.ts +4 -3
- package/dist/adapter/cachedb/cache.db.js +2 -2
- package/dist/adapter/cachedb/cache.db.model.d.ts +3 -2
- package/dist/adapter/file/file.db.d.ts +4 -3
- package/dist/adapter/file/file.db.js +10 -5
- package/dist/adapter/file/file.db.model.d.ts +2 -1
- package/dist/adapter/file/inMemory.persistence.plugin.d.ts +1 -1
- package/dist/adapter/file/localFile.persistence.plugin.d.ts +1 -1
- package/dist/adapter/file/localFile.persistence.plugin.js +5 -5
- package/dist/adapter/file/noop.persistence.plugin.d.ts +1 -1
- package/dist/commondao/common.dao.d.ts +3 -1
- package/dist/commondao/common.dao.js +7 -1
- package/dist/commondao/common.dao.model.d.ts +4 -2
- package/dist/commondao/index.d.ts +2 -0
- package/dist/commondao/index.js +2 -0
- package/dist/{base.common.db.d.ts → commondb/base.common.db.d.ts} +4 -3
- package/dist/{base.common.db.js → commondb/base.common.db.js} +1 -1
- package/dist/{common.db.d.ts → commondb/common.db.d.ts} +4 -3
- package/dist/db.model.d.ts +2 -2
- package/dist/index.d.ts +2 -13
- package/dist/index.js +2 -13
- package/dist/{adapter/inmemory → inmemory}/inMemory.db.d.ts +7 -5
- package/dist/{adapter/inmemory → inmemory}/inMemory.db.js +13 -7
- package/dist/{adapter/inmemory → inmemory}/inMemoryKeyValueDB.d.ts +3 -3
- package/dist/{adapter/inmemory → inmemory}/inMemoryKeyValueDB.js +1 -1
- package/dist/inmemory/index.d.ts +3 -0
- package/dist/inmemory/index.js +3 -0
- package/dist/inmemory/queryInMemory.d.ts +3 -0
- package/dist/{adapter/inmemory → inmemory}/queryInMemory.js +1 -1
- package/dist/kv/commonKeyValueDB.d.ts +1 -1
- package/dist/kv/commonKeyValueDao.d.ts +2 -1
- package/dist/kv/commonKeyValueDao.js +3 -2
- package/dist/kv/commonKeyValueDaoMemoCache.d.ts +3 -2
- package/dist/kv/commonKeyValueDaoMemoCache.js +2 -1
- package/dist/kv/index.d.ts +3 -0
- package/dist/kv/index.js +3 -0
- package/dist/model.util.d.ts +1 -1
- package/dist/model.util.js +1 -1
- package/dist/pipeline/dbPipelineBackup.d.ts +4 -4
- package/dist/pipeline/dbPipelineBackup.js +8 -4
- package/dist/pipeline/dbPipelineCopy.d.ts +3 -3
- package/dist/pipeline/dbPipelineCopy.js +5 -2
- package/dist/pipeline/dbPipelineRestore.d.ts +4 -4
- package/dist/pipeline/dbPipelineRestore.js +10 -4
- package/dist/pipeline/index.d.ts +3 -0
- package/dist/pipeline/index.js +3 -0
- package/dist/query/dbQuery.d.ts +1 -1
- package/dist/query/dbQuery.js +2 -1
- package/dist/testing/commonDBTest.d.ts +1 -1
- package/dist/testing/commonDBTest.js +5 -2
- package/dist/testing/commonDaoTest.d.ts +1 -1
- package/dist/testing/commonDaoTest.js +3 -1
- package/dist/testing/test.model.d.ts +2 -1
- package/dist/testing/test.model.js +2 -1
- package/dist/timeseries/timeSeries.model.d.ts +1 -1
- package/dist/transaction/dbTransaction.util.d.ts +2 -2
- package/package.json +8 -4
- package/src/adapter/cachedb/cache.db.model.ts +3 -2
- package/src/adapter/cachedb/cache.db.ts +5 -9
- package/src/adapter/file/file.db.model.ts +2 -1
- package/src/adapter/file/file.db.ts +14 -16
- package/src/adapter/file/inMemory.persistence.plugin.ts +1 -1
- package/src/adapter/file/localFile.persistence.plugin.ts +10 -6
- package/src/adapter/file/noop.persistence.plugin.ts +1 -1
- package/src/commondao/common.dao.model.ts +4 -4
- package/src/commondao/common.dao.ts +15 -22
- package/src/commondao/index.ts +2 -0
- package/src/{base.common.db.ts → commondb/base.common.db.ts} +7 -11
- package/src/{common.db.ts → commondb/common.db.ts} +4 -8
- package/src/db.model.ts +2 -2
- package/src/index.ts +2 -13
- package/src/{adapter/inmemory → inmemory}/inMemory.db.ts +30 -27
- package/src/{adapter/inmemory → inmemory}/inMemoryKeyValueDB.ts +4 -8
- package/src/inmemory/index.ts +3 -0
- package/src/{adapter/inmemory → inmemory}/queryInMemory.ts +3 -3
- package/src/kv/commonKeyValueDB.ts +1 -1
- package/src/kv/commonKeyValueDao.ts +5 -3
- package/src/kv/commonKeyValueDaoMemoCache.ts +4 -2
- package/src/kv/index.ts +3 -0
- package/src/model.util.ts +2 -2
- package/src/pipeline/dbPipelineBackup.ts +13 -9
- package/src/pipeline/dbPipelineCopy.ts +7 -4
- package/src/pipeline/dbPipelineRestore.ts +15 -20
- package/src/pipeline/index.ts +3 -0
- package/src/query/dbQuery.ts +3 -2
- package/src/testing/commonDBTest.ts +6 -3
- package/src/testing/commonDaoTest.ts +4 -2
- package/src/testing/keyValueDBTest.ts +1 -1
- package/src/testing/test.model.ts +4 -2
- package/src/timeseries/commonTimeSeriesDao.ts +1 -1
- package/src/timeseries/timeSeries.model.ts +1 -1
- package/src/transaction/dbTransaction.util.ts +2 -2
- package/dist/adapter/inmemory/queryInMemory.d.ts +0 -3
- /package/dist/{common.db.js → commondb/common.db.js} +0 -0
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { JsonSchemaObject, JsonSchemaRootObject
|
|
1
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
2
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
|
-
import { BaseCommonDB } from '../../base.common.db.js';
|
|
4
|
-
import type { CommonDB, CommonDBSupport } from '../../common.db.js';
|
|
4
|
+
import { BaseCommonDB } from '../../commondb/base.common.db.js';
|
|
5
|
+
import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js';
|
|
5
6
|
import type { RunQueryResult } from '../../db.model.js';
|
|
6
7
|
import type { DBQuery } from '../../query/dbQuery.js';
|
|
7
8
|
import type { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBSaveOptions, CacheDBStreamOptions } from './cache.db.model.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { _isTruthy } from '@naturalcycles/js-lib';
|
|
3
|
-
import { BaseCommonDB } from '../../base.common.db.js';
|
|
4
|
-
import { commonDBFullSupport } from '../../common.db.js';
|
|
3
|
+
import { BaseCommonDB } from '../../commondb/base.common.db.js';
|
|
4
|
+
import { commonDBFullSupport } from '../../commondb/common.db.js';
|
|
5
5
|
/**
|
|
6
6
|
* CommonDB implementation that proxies requests to downstream CommonDB
|
|
7
7
|
* and does in-memory caching.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { CommonLogger
|
|
2
|
-
import type {
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
2
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
3
|
+
import type { CommonDB } from '../../commondb/common.db.js';
|
|
3
4
|
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model.js';
|
|
4
5
|
export interface CacheDBCfg {
|
|
5
6
|
name: string;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { JsonSchemaRootObject
|
|
1
|
+
import type { JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
2
|
+
import { type ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
|
-
import { BaseCommonDB } from '../../base.common.db.js';
|
|
4
|
-
import type { CommonDB, CommonDBSupport } from '../../common.db.js';
|
|
4
|
+
import { BaseCommonDB } from '../../commondb/base.common.db.js';
|
|
5
|
+
import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js';
|
|
5
6
|
import type { CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, DBSaveBatchOperation, RunQueryResult } from '../../db.model.js';
|
|
6
7
|
import type { DBQuery } from '../../query/dbQuery.js';
|
|
7
8
|
import type { FileDBCfg } from './file.db.model.js';
|
|
@@ -1,9 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { _by, _sortBy } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime';
|
|
3
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
4
|
+
import { generateJsonSchemaFromData } from '@naturalcycles/js-lib/json-schema';
|
|
5
|
+
import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object';
|
|
6
|
+
import { _stringMapValues, } from '@naturalcycles/js-lib/types';
|
|
7
|
+
import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
|
|
3
8
|
import { readableCreate } from '@naturalcycles/nodejs-lib/stream';
|
|
4
|
-
import { BaseCommonDB } from '../../base.common.db.js';
|
|
5
|
-
import { commonDBFullSupport } from '../../common.db.js';
|
|
6
|
-
import { queryInMemory } from '
|
|
9
|
+
import { BaseCommonDB } from '../../commondb/base.common.db.js';
|
|
10
|
+
import { commonDBFullSupport } from '../../commondb/common.db.js';
|
|
11
|
+
import { queryInMemory } from '../../inmemory/queryInMemory.js';
|
|
7
12
|
/**
|
|
8
13
|
* Provides barebone implementation for "whole file" based CommonDB.
|
|
9
14
|
* "whole file" means that the persistence layer doesn't allow any querying,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { CommonLogger
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
2
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
4
|
import type { DBQueryOrder } from '../../query/dbQuery.js';
|
|
4
5
|
export interface FileDBPersistencePlugin {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
|
|
2
2
|
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
3
|
import type { FileDBPersistencePlugin } from './file.db.model.js';
|
|
4
4
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ObjectWithId } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
2
|
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
3
|
import type { FileDBPersistencePlugin } from './file.db.model.js';
|
|
4
4
|
export interface LocalFilePersistencePluginCfg {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import { pMap } from '@naturalcycles/js-lib';
|
|
3
|
-
import { fs2 } from '@naturalcycles/nodejs-lib/
|
|
4
|
-
import { _pipeline } from '@naturalcycles/nodejs-lib/stream';
|
|
2
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
3
|
+
import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
|
|
4
|
+
import { _pipeline, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
|
|
5
5
|
/**
|
|
6
6
|
* Persists in local filesystem as ndjson.
|
|
7
7
|
*/
|
|
@@ -26,7 +26,7 @@ export class LocalFilePersistencePlugin {
|
|
|
26
26
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
27
27
|
if (!(await fs2.pathExistsAsync(filePath)))
|
|
28
28
|
return [];
|
|
29
|
-
return await
|
|
29
|
+
return await createReadStreamAsNDJSON(filePath).toArray();
|
|
30
30
|
}
|
|
31
31
|
async saveFiles(ops) {
|
|
32
32
|
await pMap(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 32 });
|
|
@@ -35,6 +35,6 @@ export class LocalFilePersistencePlugin {
|
|
|
35
35
|
await fs2.ensureDirAsync(this.cfg.storagePath);
|
|
36
36
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
|
|
37
37
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
38
|
-
await _pipeline([Readable.from(rows), ...
|
|
38
|
+
await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(filePath)]);
|
|
39
39
|
}
|
|
40
40
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ObjectWithId } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
2
|
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
3
|
import type { FileDBPersistencePlugin } from './file.db.model.js';
|
|
4
4
|
export declare class NoopPersistencePlugin implements FileDBPersistencePlugin {
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { Transform } from 'node:stream';
|
|
2
|
-
import type {
|
|
2
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
3
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
4
|
+
import type { AsyncIndexedMapper, BaseDBEntity, StringMap, UnixTimestampMillis, Unsaved } from '@naturalcycles/js-lib/types';
|
|
3
5
|
import { ZodType } from '@naturalcycles/js-lib/zod';
|
|
4
6
|
import { AjvSchema } from '@naturalcycles/nodejs-lib/ajv';
|
|
5
7
|
import { type ObjectSchema } from '@naturalcycles/nodejs-lib/joi';
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { _isTruthy, _uniqBy } from '@naturalcycles/js-lib';
|
|
2
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime';
|
|
3
|
+
import { _assert, AppError, ErrorMode } from '@naturalcycles/js-lib/error';
|
|
4
|
+
import { _deepCopy, _deepJsonEquals, _filterUndefinedValues, _objectAssignExact, } from '@naturalcycles/js-lib/object';
|
|
5
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
6
|
+
import { _truncate } from '@naturalcycles/js-lib/string';
|
|
7
|
+
import { _passthroughPredicate, _typeCast, SKIP } from '@naturalcycles/js-lib/types';
|
|
2
8
|
import { ZodType, zSafeValidate } from '@naturalcycles/js-lib/zod';
|
|
3
9
|
import { stringId } from '@naturalcycles/nodejs-lib';
|
|
4
10
|
import { AjvSchema } from '@naturalcycles/nodejs-lib/ajv';
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ErrorMode } from '@naturalcycles/js-lib/error';
|
|
2
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
3
|
+
import type { BaseDBEntity, NumberOfMilliseconds, Promisable, UnixTimestamp } from '@naturalcycles/js-lib/types';
|
|
2
4
|
import type { ZodType, ZodValidationError } from '@naturalcycles/js-lib/zod';
|
|
3
5
|
import type { AjvSchema, AjvValidationError } from '@naturalcycles/nodejs-lib/ajv';
|
|
4
6
|
import type { JoiValidationError, ObjectSchema } from '@naturalcycles/nodejs-lib/joi';
|
|
5
7
|
import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
|
|
6
|
-
import type { CommonDB } from '../common.db.js';
|
|
8
|
+
import type { CommonDB } from '../commondb/common.db.js';
|
|
7
9
|
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model.js';
|
|
8
10
|
export interface CommonDaoHooks<BM extends BaseDBEntity, DBM extends BaseDBEntity, ID = BM['id']> {
|
|
9
11
|
/**
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import type { JsonSchemaObject, JsonSchemaRootObject
|
|
1
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
2
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
4
|
+
import type { CommonDBOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBTransaction, DBTransactionFn, RunQueryResult } from '../db.model.js';
|
|
5
|
+
import type { DBQuery } from '../query/dbQuery.js';
|
|
3
6
|
import type { CommonDB, CommonDBSupport } from './common.db.js';
|
|
4
7
|
import { CommonDBType } from './common.db.js';
|
|
5
|
-
import type { CommonDBOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBTransaction, DBTransactionFn, RunQueryResult } from './db.model.js';
|
|
6
|
-
import type { DBQuery } from './query/dbQuery.js';
|
|
7
8
|
/**
|
|
8
9
|
* No-op implementation of CommonDB interface.
|
|
9
10
|
* To be extended by actual implementations.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { FakeDBTransaction } from '../transaction/dbTransaction.util.js';
|
|
1
2
|
import { CommonDBType } from './common.db.js';
|
|
2
|
-
import { FakeDBTransaction } from './transaction/dbTransaction.util.js';
|
|
3
3
|
/**
|
|
4
4
|
* No-op implementation of CommonDB interface.
|
|
5
5
|
* To be extended by actual implementations.
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { JsonSchemaObject, JsonSchemaRootObject
|
|
1
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
2
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
|
-
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBStreamOptions, CommonDBTransactionOptions, DBTransaction, DBTransactionFn, RunQueryResult } from '
|
|
4
|
-
import type { DBQuery } from '
|
|
4
|
+
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBStreamOptions, CommonDBTransactionOptions, DBTransaction, DBTransactionFn, RunQueryResult } from '../db.model.js';
|
|
5
|
+
import type { DBQuery } from '../query/dbQuery.js';
|
|
5
6
|
export declare enum CommonDBType {
|
|
6
7
|
'document' = "document",
|
|
7
8
|
'relational' = "relational"
|
package/dist/db.model.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ObjectWithId, UnixTimestamp } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { CommonDB } from './common.db.js';
|
|
1
|
+
import type { ObjectWithId, UnixTimestamp } from '@naturalcycles/js-lib/types';
|
|
2
|
+
import type { CommonDB } from './commondb/common.db.js';
|
|
3
3
|
/**
|
|
4
4
|
* Similar to SQL INSERT, UPDATE.
|
|
5
5
|
* Insert will fail if row already exists.
|
package/dist/index.d.ts
CHANGED
|
@@ -1,17 +1,6 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './adapter/inmemory/queryInMemory.js';
|
|
4
|
-
export * from './base.common.db.js';
|
|
5
|
-
export * from './common.db.js';
|
|
6
|
-
export * from './commondao/common.dao.js';
|
|
7
|
-
export * from './commondao/common.dao.model.js';
|
|
1
|
+
export * from './commondb/base.common.db.js';
|
|
2
|
+
export * from './commondb/common.db.js';
|
|
8
3
|
export * from './db.model.js';
|
|
9
|
-
export * from './kv/commonKeyValueDao.js';
|
|
10
|
-
export * from './kv/commonKeyValueDaoMemoCache.js';
|
|
11
|
-
export * from './kv/commonKeyValueDB.js';
|
|
12
4
|
export * from './model.util.js';
|
|
13
|
-
export * from './pipeline/dbPipelineBackup.js';
|
|
14
|
-
export * from './pipeline/dbPipelineCopy.js';
|
|
15
|
-
export * from './pipeline/dbPipelineRestore.js';
|
|
16
5
|
export * from './query/dbQuery.js';
|
|
17
6
|
export * from './transaction/dbTransaction.util.js';
|
package/dist/index.js
CHANGED
|
@@ -1,17 +1,6 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './adapter/inmemory/queryInMemory.js';
|
|
4
|
-
export * from './base.common.db.js';
|
|
5
|
-
export * from './common.db.js';
|
|
6
|
-
export * from './commondao/common.dao.js';
|
|
7
|
-
export * from './commondao/common.dao.model.js';
|
|
1
|
+
export * from './commondb/base.common.db.js';
|
|
2
|
+
export * from './commondb/common.db.js';
|
|
8
3
|
export * from './db.model.js';
|
|
9
|
-
export * from './kv/commonKeyValueDao.js';
|
|
10
|
-
export * from './kv/commonKeyValueDaoMemoCache.js';
|
|
11
|
-
export * from './kv/commonKeyValueDB.js';
|
|
12
4
|
export * from './model.util.js';
|
|
13
|
-
export * from './pipeline/dbPipelineBackup.js';
|
|
14
|
-
export * from './pipeline/dbPipelineCopy.js';
|
|
15
|
-
export * from './pipeline/dbPipelineRestore.js';
|
|
16
5
|
export * from './query/dbQuery.js';
|
|
17
6
|
export * from './transaction/dbTransaction.util.js';
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type JsonSchemaObject, type JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
2
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
3
|
+
import { type AnyObjectWithId, type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
|
|
2
4
|
import { type ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
|
-
import type { CommonDB, CommonDBSupport } from '
|
|
4
|
-
import { CommonDBType } from '
|
|
5
|
-
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBOperation, DBTransaction, DBTransactionFn, RunQueryResult } from '
|
|
6
|
-
import type { DBQuery } from '
|
|
5
|
+
import type { CommonDB, CommonDBSupport } from '../commondb/common.db.js';
|
|
6
|
+
import { CommonDBType } from '../commondb/common.db.js';
|
|
7
|
+
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBOperation, DBTransaction, DBTransactionFn, RunQueryResult } from '../db.model.js';
|
|
8
|
+
import type { DBQuery } from '../query/dbQuery.js';
|
|
7
9
|
export interface InMemoryDBCfg {
|
|
8
10
|
/**
|
|
9
11
|
* @default ''
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
2
|
+
import { _by, _isEmptyObject } from '@naturalcycles/js-lib';
|
|
3
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime';
|
|
4
|
+
import { _assert } from '@naturalcycles/js-lib/error';
|
|
5
|
+
import { generateJsonSchemaFromData, } from '@naturalcycles/js-lib/json-schema';
|
|
6
|
+
import { _deepCopy, _sortObjectDeep } from '@naturalcycles/js-lib/object';
|
|
7
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
8
|
+
import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
|
|
9
|
+
import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/colors';
|
|
10
|
+
import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
|
|
11
|
+
import { _pipeline, bufferReviver, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
|
|
12
|
+
import { commonDBFullSupport, CommonDBType } from '../commondb/common.db.js';
|
|
7
13
|
import { queryInMemory } from './queryInMemory.js';
|
|
8
14
|
export class InMemoryDB {
|
|
9
15
|
dbType = CommonDBType.document;
|
|
@@ -183,7 +189,7 @@ export class InMemoryDB {
|
|
|
183
189
|
return; // 0 rows
|
|
184
190
|
tables++;
|
|
185
191
|
const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`;
|
|
186
|
-
await _pipeline([Readable.from(rows), ...
|
|
192
|
+
await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(fname)]);
|
|
187
193
|
});
|
|
188
194
|
this.cfg.logger.log(`flushToDisk took ${dimGrey(_since(started))} to save ${yellow(tables)} tables`);
|
|
189
195
|
}
|
|
@@ -201,7 +207,7 @@ export class InMemoryDB {
|
|
|
201
207
|
await pMap(files, async (file) => {
|
|
202
208
|
const fname = `${persistentStoragePath}/${file}`;
|
|
203
209
|
const table = file.split('.ndjson')[0];
|
|
204
|
-
const rows = await
|
|
210
|
+
const rows = await createReadStreamAsNDJSON(fname).toArray();
|
|
205
211
|
this.data[table] = _by(rows, r => r.id);
|
|
206
212
|
});
|
|
207
213
|
this.cfg.logger.log(`restoreFromDisk took ${dimGrey(_since(started))} to read ${yellow(files.length)} tables`);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { StringMap } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { StringMap } from '@naturalcycles/js-lib/types';
|
|
2
2
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
|
-
import type { CommonDBCreateOptions } from '
|
|
4
|
-
import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '
|
|
3
|
+
import type { CommonDBCreateOptions } from '../db.model.js';
|
|
4
|
+
import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '../kv/commonKeyValueDB.js';
|
|
5
5
|
export interface InMemoryKeyValueDBCfg {
|
|
6
6
|
}
|
|
7
7
|
export declare class InMemoryKeyValueDB implements CommonKeyValueDB {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Integer, UnixTimestamp } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { Integer, UnixTimestamp } from '@naturalcycles/js-lib/types';
|
|
2
2
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
3
|
import type { CommonDBCreateOptions } from '../db.model.js';
|
|
4
4
|
/**
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { CommonLogger
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
2
|
+
import type { KeyValueTuple } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
4
|
import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js';
|
|
4
5
|
import type { CommonDBCreateOptions } from '../db.model.js';
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { AppError
|
|
2
|
-
import {
|
|
1
|
+
import { AppError } from '@naturalcycles/js-lib/error';
|
|
2
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
3
|
+
import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip';
|
|
3
4
|
export const commonKeyValueDaoDeflatedJsonTransformer = {
|
|
4
5
|
valueToBuffer: async (v) => await deflateString(JSON.stringify(v)),
|
|
5
6
|
bufferToValue: async (buf) => JSON.parse(await inflateToString(buf)),
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { AsyncMemoCache
|
|
2
|
-
import {
|
|
1
|
+
import type { AsyncMemoCache } from '@naturalcycles/js-lib/decorators';
|
|
2
|
+
import type { NumberOfSeconds } from '@naturalcycles/js-lib/types';
|
|
3
|
+
import { MISS } from '@naturalcycles/js-lib/types';
|
|
3
4
|
import type { CommonKeyValueDao } from './commonKeyValueDao.js';
|
|
4
5
|
export interface CommonKeyValueDaoMemoCacheCfg<VALUE> {
|
|
5
6
|
dao: CommonKeyValueDao<string, VALUE>;
|
package/dist/kv/index.js
ADDED
package/dist/model.util.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib/types';
|
|
2
2
|
export declare function createdUpdatedFields(existingObject?: Partial<CreatedUpdated> | null): CreatedUpdated;
|
|
3
3
|
export declare function createdUpdatedIdFields(existingObject?: Partial<CreatedUpdatedId> | null): CreatedUpdatedId;
|
|
4
4
|
export declare function deserializeJsonField<T = any>(f?: string): T;
|
package/dist/model.util.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import type
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error';
|
|
2
|
+
import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-lib/types';
|
|
3
|
+
import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
|
|
4
4
|
import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
|
|
5
|
-
import type { CommonDB } from '../common.db.js';
|
|
5
|
+
import type { CommonDB } from '../commondb/common.db.js';
|
|
6
6
|
import { DBQuery } from '../query/dbQuery.js';
|
|
7
7
|
export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
|
|
8
8
|
/**
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime';
|
|
2
|
+
import { AppError, ErrorMode } from '@naturalcycles/js-lib/error';
|
|
3
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
4
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib/types';
|
|
5
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
|
|
6
|
+
import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
|
|
7
|
+
import { createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
|
|
4
8
|
import { _pipeline, NDJsonStats, transformLogProgress, transformMap, transformTap, } from '@naturalcycles/nodejs-lib/stream';
|
|
5
9
|
import { DBQuery } from '../query/dbQuery.js';
|
|
6
10
|
/**
|
|
@@ -68,7 +72,7 @@ export async function dbPipelineBackup(opt) {
|
|
|
68
72
|
transformTap(() => {
|
|
69
73
|
rows++;
|
|
70
74
|
}),
|
|
71
|
-
...
|
|
75
|
+
...createWriteStreamAsNDJSON(filePath),
|
|
72
76
|
]);
|
|
73
77
|
const { size: sizeBytes } = await fs2.statAsync(filePath);
|
|
74
78
|
const stats = NDJsonStats.create({
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error';
|
|
2
|
+
import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
|
|
3
3
|
import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
|
|
4
4
|
import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
|
|
5
|
-
import type { CommonDB } from '../common.db.js';
|
|
5
|
+
import type { CommonDB } from '../commondb/common.db.js';
|
|
6
6
|
import type { CommonDBSaveOptions } from '../db.model.js';
|
|
7
7
|
export interface DBPipelineCopyOptions extends TransformLogProgressOptions {
|
|
8
8
|
dbInput: CommonDB;
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime';
|
|
2
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error';
|
|
3
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
4
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib/types';
|
|
5
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
|
|
3
6
|
import { _pipeline, NDJsonStats, transformChunk, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
|
|
4
7
|
import { DBQuery } from '../query/dbQuery.js';
|
|
5
8
|
/**
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import type
|
|
1
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error';
|
|
2
|
+
import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
|
|
3
|
+
import { type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
|
|
4
4
|
import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
|
|
5
|
-
import type { CommonDB } from '../common.db.js';
|
|
5
|
+
import type { CommonDB } from '../commondb/common.db.js';
|
|
6
6
|
import type { CommonDBSaveOptions } from '../db.model.js';
|
|
7
7
|
export interface DBPipelineRestoreOptions extends TransformLogProgressOptions {
|
|
8
8
|
/**
|
|
@@ -1,6 +1,12 @@
|
|
|
1
|
-
import { _hb
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { _hb } from '@naturalcycles/js-lib';
|
|
2
|
+
import { localTime } from '@naturalcycles/js-lib/datetime';
|
|
3
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error';
|
|
4
|
+
import { _mapValues } from '@naturalcycles/js-lib/object';
|
|
5
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
6
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib/types';
|
|
7
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
|
|
8
|
+
import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
|
|
9
|
+
import { createReadStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
|
|
4
10
|
import { _pipeline, NDJsonStats, transformChunk, transformFilterSync, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
|
|
5
11
|
/**
|
|
6
12
|
* Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
|
|
@@ -62,7 +68,7 @@ export async function dbPipelineRestore(opt) {
|
|
|
62
68
|
const sizeBytes = sizeByTable[table];
|
|
63
69
|
console.log(`<< ${grey(filePath)} ${dimWhite(_hb(sizeBytes))} started...`);
|
|
64
70
|
await _pipeline([
|
|
65
|
-
|
|
71
|
+
createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
|
|
66
72
|
transformTap(() => rows++),
|
|
67
73
|
transformLogProgress({
|
|
68
74
|
logEvery: 1000,
|
package/dist/query/dbQuery.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AsyncIndexedMapper, BaseDBEntity, ObjectWithId } from '@naturalcycles/js-lib';
|
|
1
|
+
import type { AsyncIndexedMapper, BaseDBEntity, ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
2
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
3
3
|
import type { CommonDao } from '../commondao/common.dao.js';
|
|
4
4
|
import type { CommonDaoOptions, CommonDaoReadOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions } from '../commondao/common.dao.model.js';
|
package/dist/query/dbQuery.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { _sortBy } from '@naturalcycles/js-lib';
|
|
2
|
+
import { localTime } from '@naturalcycles/js-lib/datetime';
|
|
3
|
+
import { _deepFreeze, _filterObject, _pick } from '@naturalcycles/js-lib/object';
|
|
4
|
+
import { pMap } from '@naturalcycles/js-lib/promise';
|
|
5
|
+
import { CommonDBType } from '../commondb/common.db.js';
|
|
3
6
|
import { DBQuery } from '../query/dbQuery.js';
|
|
4
7
|
import { createTestItemDBM, createTestItemsDBM, TEST_TABLE, testItemBMJsonSchema, } from './test.model.js';
|
|
5
8
|
export async function runCommonDBTest(db, quirks = {}) {
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type { CommonDB } from '../common.db.js';
|
|
1
|
+
import type { CommonDB } from '../commondb/common.db.js';
|
|
2
2
|
import type { CommonDBImplementationQuirks } from './commonDBTest.js';
|
|
3
3
|
export declare function runCommonDaoTest(db: CommonDB, quirks?: CommonDBImplementationQuirks): Promise<void>;
|