@naturalcycles/db-lib 10.5.0 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/cachedb/cache.db.d.ts +4 -3
- package/dist/adapter/cachedb/cache.db.js +2 -2
- package/dist/adapter/cachedb/cache.db.model.d.ts +3 -2
- package/dist/adapter/file/file.db.d.ts +4 -3
- package/dist/adapter/file/file.db.js +10 -5
- package/dist/adapter/file/file.db.model.d.ts +2 -1
- package/dist/adapter/file/inMemory.persistence.plugin.d.ts +1 -1
- package/dist/adapter/file/localFile.persistence.plugin.d.ts +1 -1
- package/dist/adapter/file/localFile.persistence.plugin.js +5 -5
- package/dist/adapter/file/noop.persistence.plugin.d.ts +1 -1
- package/dist/commondao/common.dao.d.ts +3 -1
- package/dist/commondao/common.dao.js +7 -1
- package/dist/commondao/common.dao.model.d.ts +4 -2
- package/dist/commondao/index.d.ts +2 -0
- package/dist/commondao/index.js +2 -0
- package/dist/{base.common.db.d.ts → commondb/base.common.db.d.ts} +4 -3
- package/dist/{base.common.db.js → commondb/base.common.db.js} +1 -1
- package/dist/{common.db.d.ts → commondb/common.db.d.ts} +4 -3
- package/dist/db.model.d.ts +2 -2
- package/dist/index.d.ts +2 -13
- package/dist/index.js +2 -13
- package/dist/{adapter/inmemory → inmemory}/inMemory.db.d.ts +7 -5
- package/dist/{adapter/inmemory → inmemory}/inMemory.db.js +13 -7
- package/dist/{adapter/inmemory → inmemory}/inMemoryKeyValueDB.d.ts +3 -3
- package/dist/{adapter/inmemory → inmemory}/inMemoryKeyValueDB.js +1 -1
- package/dist/inmemory/index.d.ts +3 -0
- package/dist/inmemory/index.js +3 -0
- package/dist/inmemory/queryInMemory.d.ts +3 -0
- package/dist/{adapter/inmemory → inmemory}/queryInMemory.js +1 -1
- package/dist/kv/commonKeyValueDB.d.ts +1 -1
- package/dist/kv/commonKeyValueDao.d.ts +2 -1
- package/dist/kv/commonKeyValueDao.js +3 -2
- package/dist/kv/commonKeyValueDaoMemoCache.d.ts +3 -2
- package/dist/kv/commonKeyValueDaoMemoCache.js +2 -1
- package/dist/kv/index.d.ts +3 -0
- package/dist/kv/index.js +3 -0
- package/dist/model.util.d.ts +1 -1
- package/dist/model.util.js +1 -1
- package/dist/pipeline/dbPipelineBackup.d.ts +4 -4
- package/dist/pipeline/dbPipelineBackup.js +8 -4
- package/dist/pipeline/dbPipelineCopy.d.ts +3 -3
- package/dist/pipeline/dbPipelineCopy.js +5 -2
- package/dist/pipeline/dbPipelineRestore.d.ts +4 -4
- package/dist/pipeline/dbPipelineRestore.js +10 -4
- package/dist/pipeline/index.d.ts +3 -0
- package/dist/pipeline/index.js +3 -0
- package/dist/query/dbQuery.d.ts +1 -1
- package/dist/query/dbQuery.js +2 -1
- package/dist/testing/commonDBTest.d.ts +1 -1
- package/dist/testing/commonDBTest.js +5 -2
- package/dist/testing/commonDaoTest.d.ts +1 -1
- package/dist/testing/commonDaoTest.js +3 -1
- package/dist/testing/test.model.d.ts +2 -1
- package/dist/testing/test.model.js +2 -1
- package/dist/timeseries/timeSeries.model.d.ts +1 -1
- package/dist/transaction/dbTransaction.util.d.ts +2 -2
- package/package.json +8 -4
- package/src/adapter/cachedb/cache.db.model.ts +3 -2
- package/src/adapter/cachedb/cache.db.ts +5 -9
- package/src/adapter/file/file.db.model.ts +2 -1
- package/src/adapter/file/file.db.ts +14 -16
- package/src/adapter/file/inMemory.persistence.plugin.ts +1 -1
- package/src/adapter/file/localFile.persistence.plugin.ts +10 -6
- package/src/adapter/file/noop.persistence.plugin.ts +1 -1
- package/src/commondao/common.dao.model.ts +4 -4
- package/src/commondao/common.dao.ts +15 -22
- package/src/commondao/index.ts +2 -0
- package/src/{base.common.db.ts → commondb/base.common.db.ts} +7 -11
- package/src/{common.db.ts → commondb/common.db.ts} +4 -8
- package/src/db.model.ts +2 -2
- package/src/index.ts +2 -13
- package/src/{adapter/inmemory → inmemory}/inMemory.db.ts +30 -27
- package/src/{adapter/inmemory → inmemory}/inMemoryKeyValueDB.ts +4 -8
- package/src/inmemory/index.ts +3 -0
- package/src/{adapter/inmemory → inmemory}/queryInMemory.ts +3 -3
- package/src/kv/commonKeyValueDB.ts +1 -1
- package/src/kv/commonKeyValueDao.ts +5 -3
- package/src/kv/commonKeyValueDaoMemoCache.ts +4 -2
- package/src/kv/index.ts +3 -0
- package/src/model.util.ts +2 -2
- package/src/pipeline/dbPipelineBackup.ts +13 -9
- package/src/pipeline/dbPipelineCopy.ts +7 -4
- package/src/pipeline/dbPipelineRestore.ts +15 -20
- package/src/pipeline/index.ts +3 -0
- package/src/query/dbQuery.ts +3 -2
- package/src/testing/commonDBTest.ts +6 -3
- package/src/testing/commonDaoTest.ts +4 -2
- package/src/testing/keyValueDBTest.ts +1 -1
- package/src/testing/test.model.ts +4 -2
- package/src/timeseries/commonTimeSeriesDao.ts +1 -1
- package/src/timeseries/timeSeries.model.ts +1 -1
- package/src/transaction/dbTransaction.util.ts +2 -2
- package/dist/adapter/inmemory/queryInMemory.d.ts +0 -3
- /package/dist/{common.db.js → commondb/common.db.js} +0 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import {
|
|
2
|
+
import { _sortBy } from '@naturalcycles/js-lib';
|
|
3
|
+
import { localTime } from '@naturalcycles/js-lib/datetime';
|
|
4
|
+
import { _deepCopy, _filterObject, _omit, _pick } from '@naturalcycles/js-lib/object';
|
|
3
5
|
import { _pipeline } from '@naturalcycles/nodejs-lib/stream';
|
|
4
6
|
import { CommonDao } from '../commondao/common.dao.js';
|
|
5
7
|
import { CommonDaoLogLevel } from '../commondao/common.dao.model.js';
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { JsonSchemaObject } from '@naturalcycles/js-lib/json-schema';
|
|
2
|
+
import type { BaseDBEntity } from '@naturalcycles/js-lib/types';
|
|
2
3
|
import { type ObjectSchema } from '@naturalcycles/nodejs-lib/joi';
|
|
3
4
|
export declare const TEST_TABLE = "TEST_TABLE";
|
|
4
5
|
export interface TestItemBM extends BaseDBEntity {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { _range
|
|
1
|
+
import { _range } from '@naturalcycles/js-lib';
|
|
2
|
+
import { jsonSchema } from '@naturalcycles/js-lib/json-schema';
|
|
2
3
|
import { baseDBEntitySchema, binarySchema, booleanSchema, numberSchema, objectSchema, stringSchema, } from '@naturalcycles/nodejs-lib/joi';
|
|
3
4
|
const MOCK_TS_2018_06_21 = 1529539200;
|
|
4
5
|
export const TEST_TABLE = 'TEST_TABLE';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import type { CommonDB } from '../common.db.js';
|
|
1
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
|
+
import type { CommonDB } from '../commondb/common.db.js';
|
|
3
3
|
import type { CommonDBOptions, CommonDBSaveOptions, DBTransaction } from '../db.model.js';
|
|
4
4
|
/**
|
|
5
5
|
* Optimizes the Transaction (list of DBOperations) to do less operations.
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/db-lib",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "10.
|
|
4
|
+
"version": "10.7.0",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@naturalcycles/js-lib": "^15",
|
|
7
7
|
"@naturalcycles/nodejs-lib": "^15"
|
|
8
8
|
},
|
|
9
9
|
"devDependencies": {
|
|
10
|
-
"@naturalcycles/dev-lib": "19.
|
|
10
|
+
"@naturalcycles/dev-lib": "19.8.0"
|
|
11
11
|
},
|
|
12
12
|
"files": [
|
|
13
13
|
"dist",
|
|
@@ -23,8 +23,12 @@
|
|
|
23
23
|
".": "./dist/index.js",
|
|
24
24
|
"./testing": "./dist/testing/index.js",
|
|
25
25
|
"./validation": "./dist/validation/index.js",
|
|
26
|
-
"./
|
|
27
|
-
"./
|
|
26
|
+
"./pipeline": "./dist/pipeline/index.js",
|
|
27
|
+
"./kv": "./dist/kv/index.js",
|
|
28
|
+
"./dao": "./dist/commondao/index.js",
|
|
29
|
+
"./cachedb": "./dist/adapter/cachedb/index.js",
|
|
30
|
+
"./file": "./dist/adapter/file/index.js",
|
|
31
|
+
"./inmemory": "./dist/inmemory/index.js"
|
|
28
32
|
},
|
|
29
33
|
"publishConfig": {
|
|
30
34
|
"access": "public"
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { CommonLogger
|
|
2
|
-
import type {
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
2
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types'
|
|
3
|
+
import type { CommonDB } from '../../commondb/common.db.js'
|
|
3
4
|
import type {
|
|
4
5
|
CommonDBCreateOptions,
|
|
5
6
|
CommonDBOptions,
|
|
@@ -1,15 +1,11 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
|
-
import type {
|
|
3
|
-
JsonSchemaObject,
|
|
4
|
-
JsonSchemaRootObject,
|
|
5
|
-
ObjectWithId,
|
|
6
|
-
StringMap,
|
|
7
|
-
} from '@naturalcycles/js-lib'
|
|
8
2
|
import { _isTruthy } from '@naturalcycles/js-lib'
|
|
3
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
|
|
4
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
|
|
9
5
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
10
|
-
import { BaseCommonDB } from '../../base.common.db.js'
|
|
11
|
-
import type { CommonDB, CommonDBSupport } from '../../common.db.js'
|
|
12
|
-
import { commonDBFullSupport } from '../../common.db.js'
|
|
6
|
+
import { BaseCommonDB } from '../../commondb/base.common.db.js'
|
|
7
|
+
import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js'
|
|
8
|
+
import { commonDBFullSupport } from '../../commondb/common.db.js'
|
|
13
9
|
import type { RunQueryResult } from '../../db.model.js'
|
|
14
10
|
import type { DBQuery } from '../../query/dbQuery.js'
|
|
15
11
|
import type {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { CommonLogger
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
2
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types'
|
|
2
3
|
import type { DBSaveBatchOperation } from '../../db.model.js'
|
|
3
4
|
import type { DBQueryOrder } from '../../query/dbQuery.js'
|
|
4
5
|
|
|
@@ -1,22 +1,20 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { _by, _sortBy } from '@naturalcycles/js-lib'
|
|
2
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime'
|
|
3
|
+
import { _assert } from '@naturalcycles/js-lib/error'
|
|
4
|
+
import type { JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
|
|
5
|
+
import { generateJsonSchemaFromData } from '@naturalcycles/js-lib/json-schema'
|
|
6
|
+
import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object'
|
|
2
7
|
import {
|
|
3
|
-
_assert,
|
|
4
|
-
_by,
|
|
5
|
-
_deepEquals,
|
|
6
|
-
_filterUndefinedValues,
|
|
7
|
-
_since,
|
|
8
|
-
_sortBy,
|
|
9
|
-
_sortObjectDeep,
|
|
10
8
|
_stringMapValues,
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
} from '@naturalcycles/js-lib'
|
|
14
|
-
import { dimGrey } from '@naturalcycles/nodejs-lib'
|
|
9
|
+
type ObjectWithId,
|
|
10
|
+
type UnixTimestampMillis,
|
|
11
|
+
} from '@naturalcycles/js-lib/types'
|
|
12
|
+
import { dimGrey } from '@naturalcycles/nodejs-lib/colors'
|
|
15
13
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
16
14
|
import { readableCreate } from '@naturalcycles/nodejs-lib/stream'
|
|
17
|
-
import { BaseCommonDB } from '../../base.common.db.js'
|
|
18
|
-
import type { CommonDB, CommonDBSupport } from '../../common.db.js'
|
|
19
|
-
import { commonDBFullSupport } from '../../common.db.js'
|
|
15
|
+
import { BaseCommonDB } from '../../commondb/base.common.db.js'
|
|
16
|
+
import type { CommonDB, CommonDBSupport } from '../../commondb/common.db.js'
|
|
17
|
+
import { commonDBFullSupport } from '../../commondb/common.db.js'
|
|
20
18
|
import type {
|
|
21
19
|
CommonDBOptions,
|
|
22
20
|
CommonDBSaveOptions,
|
|
@@ -24,8 +22,8 @@ import type {
|
|
|
24
22
|
DBSaveBatchOperation,
|
|
25
23
|
RunQueryResult,
|
|
26
24
|
} from '../../db.model.js'
|
|
25
|
+
import { queryInMemory } from '../../inmemory/queryInMemory.js'
|
|
27
26
|
import type { DBQuery } from '../../query/dbQuery.js'
|
|
28
|
-
import { queryInMemory } from '../inmemory/queryInMemory.js'
|
|
29
27
|
import type { FileDBCfg } from './file.db.model.js'
|
|
30
28
|
|
|
31
29
|
/**
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib'
|
|
2
1
|
import { _by } from '@naturalcycles/js-lib'
|
|
2
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
|
|
3
3
|
import type { DBSaveBatchOperation } from '../../db.model.js'
|
|
4
4
|
import type { FileDBPersistencePlugin } from './file.db.model.js'
|
|
5
5
|
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
4
|
-
import { fs2 } from '@naturalcycles/nodejs-lib/
|
|
5
|
-
import {
|
|
2
|
+
import { pMap } from '@naturalcycles/js-lib/promise'
|
|
3
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types'
|
|
4
|
+
import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
|
|
5
|
+
import {
|
|
6
|
+
_pipeline,
|
|
7
|
+
createReadStreamAsNDJSON,
|
|
8
|
+
createWriteStreamAsNDJSON,
|
|
9
|
+
} from '@naturalcycles/nodejs-lib/stream'
|
|
6
10
|
import type { DBSaveBatchOperation } from '../../db.model.js'
|
|
7
11
|
import type { FileDBPersistencePlugin } from './file.db.model.js'
|
|
8
12
|
|
|
@@ -47,7 +51,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
|
|
|
47
51
|
|
|
48
52
|
if (!(await fs2.pathExistsAsync(filePath))) return []
|
|
49
53
|
|
|
50
|
-
return await
|
|
54
|
+
return await createReadStreamAsNDJSON(filePath).toArray()
|
|
51
55
|
}
|
|
52
56
|
|
|
53
57
|
async saveFiles(ops: DBSaveBatchOperation<any>[]): Promise<void> {
|
|
@@ -59,6 +63,6 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
|
|
|
59
63
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
|
|
60
64
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`
|
|
61
65
|
|
|
62
|
-
await _pipeline([Readable.from(rows), ...
|
|
66
|
+
await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(filePath)])
|
|
63
67
|
}
|
|
64
68
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
+
import type { ErrorMode } from '@naturalcycles/js-lib/error'
|
|
2
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
1
3
|
import type {
|
|
2
4
|
BaseDBEntity,
|
|
3
|
-
CommonLogger,
|
|
4
|
-
ErrorMode,
|
|
5
5
|
NumberOfMilliseconds,
|
|
6
6
|
Promisable,
|
|
7
7
|
UnixTimestamp,
|
|
8
|
-
} from '@naturalcycles/js-lib'
|
|
8
|
+
} from '@naturalcycles/js-lib/types'
|
|
9
9
|
import type { ZodType, ZodValidationError } from '@naturalcycles/js-lib/zod'
|
|
10
10
|
import type { AjvSchema, AjvValidationError } from '@naturalcycles/nodejs-lib/ajv'
|
|
11
11
|
import type { JoiValidationError, ObjectSchema } from '@naturalcycles/nodejs-lib/joi'
|
|
@@ -13,7 +13,7 @@ import type {
|
|
|
13
13
|
TransformLogProgressOptions,
|
|
14
14
|
TransformMapOptions,
|
|
15
15
|
} from '@naturalcycles/nodejs-lib/stream'
|
|
16
|
-
import type { CommonDB } from '../common.db.js'
|
|
16
|
+
import type { CommonDB } from '../commondb/common.db.js'
|
|
17
17
|
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions } from '../db.model.js'
|
|
18
18
|
|
|
19
19
|
export interface CommonDaoHooks<BM extends BaseDBEntity, DBM extends BaseDBEntity, ID = BM['id']> {
|
|
@@ -1,33 +1,26 @@
|
|
|
1
1
|
import type { Transform } from 'node:stream'
|
|
2
|
+
import { _isTruthy, _uniqBy } from '@naturalcycles/js-lib'
|
|
3
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime'
|
|
4
|
+
import { _assert, AppError, ErrorMode } from '@naturalcycles/js-lib/error'
|
|
5
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
|
|
6
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
7
|
+
import {
|
|
8
|
+
_deepCopy,
|
|
9
|
+
_deepJsonEquals,
|
|
10
|
+
_filterUndefinedValues,
|
|
11
|
+
_objectAssignExact,
|
|
12
|
+
} from '@naturalcycles/js-lib/object'
|
|
13
|
+
import { pMap } from '@naturalcycles/js-lib/promise'
|
|
14
|
+
import { _truncate } from '@naturalcycles/js-lib/string'
|
|
2
15
|
import type {
|
|
3
16
|
AsyncIndexedMapper,
|
|
4
17
|
BaseDBEntity,
|
|
5
|
-
CommonLogger,
|
|
6
|
-
JsonSchemaObject,
|
|
7
|
-
JsonSchemaRootObject,
|
|
8
18
|
ObjectWithId,
|
|
9
19
|
StringMap,
|
|
10
20
|
UnixTimestampMillis,
|
|
11
21
|
Unsaved,
|
|
12
|
-
} from '@naturalcycles/js-lib'
|
|
13
|
-
import {
|
|
14
|
-
_assert,
|
|
15
|
-
_deepCopy,
|
|
16
|
-
_deepJsonEquals,
|
|
17
|
-
_filterUndefinedValues,
|
|
18
|
-
_isTruthy,
|
|
19
|
-
_objectAssignExact,
|
|
20
|
-
_passthroughPredicate,
|
|
21
|
-
_since,
|
|
22
|
-
_truncate,
|
|
23
|
-
_typeCast,
|
|
24
|
-
_uniqBy,
|
|
25
|
-
AppError,
|
|
26
|
-
ErrorMode,
|
|
27
|
-
localTime,
|
|
28
|
-
pMap,
|
|
29
|
-
SKIP,
|
|
30
|
-
} from '@naturalcycles/js-lib'
|
|
22
|
+
} from '@naturalcycles/js-lib/types'
|
|
23
|
+
import { _passthroughPredicate, _typeCast, SKIP } from '@naturalcycles/js-lib/types'
|
|
31
24
|
import type { ZodValidationError } from '@naturalcycles/js-lib/zod'
|
|
32
25
|
import { ZodType, zSafeValidate } from '@naturalcycles/js-lib/zod'
|
|
33
26
|
import { stringId } from '@naturalcycles/nodejs-lib'
|
|
@@ -1,12 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
JsonSchemaRootObject,
|
|
4
|
-
ObjectWithId,
|
|
5
|
-
StringMap,
|
|
6
|
-
} from '@naturalcycles/js-lib'
|
|
1
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
|
|
2
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
|
|
7
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
8
|
-
import type { CommonDB, CommonDBSupport } from './common.db.js'
|
|
9
|
-
import { CommonDBType } from './common.db.js'
|
|
10
4
|
import type {
|
|
11
5
|
CommonDBOptions,
|
|
12
6
|
CommonDBSaveOptions,
|
|
@@ -14,9 +8,11 @@ import type {
|
|
|
14
8
|
DBTransaction,
|
|
15
9
|
DBTransactionFn,
|
|
16
10
|
RunQueryResult,
|
|
17
|
-
} from '
|
|
18
|
-
import type { DBQuery } from '
|
|
19
|
-
import { FakeDBTransaction } from '
|
|
11
|
+
} from '../db.model.js'
|
|
12
|
+
import type { DBQuery } from '../query/dbQuery.js'
|
|
13
|
+
import { FakeDBTransaction } from '../transaction/dbTransaction.util.js'
|
|
14
|
+
import type { CommonDB, CommonDBSupport } from './common.db.js'
|
|
15
|
+
import { CommonDBType } from './common.db.js'
|
|
20
16
|
|
|
21
17
|
/**
|
|
22
18
|
* No-op implementation of CommonDB interface.
|
|
@@ -1,9 +1,5 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
JsonSchemaRootObject,
|
|
4
|
-
ObjectWithId,
|
|
5
|
-
StringMap,
|
|
6
|
-
} from '@naturalcycles/js-lib'
|
|
1
|
+
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema'
|
|
2
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
|
|
7
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
8
4
|
import type {
|
|
9
5
|
CommonDBCreateOptions,
|
|
@@ -15,8 +11,8 @@ import type {
|
|
|
15
11
|
DBTransaction,
|
|
16
12
|
DBTransactionFn,
|
|
17
13
|
RunQueryResult,
|
|
18
|
-
} from '
|
|
19
|
-
import type { DBQuery } from '
|
|
14
|
+
} from '../db.model.js'
|
|
15
|
+
import type { DBQuery } from '../query/dbQuery.js'
|
|
20
16
|
|
|
21
17
|
export enum CommonDBType {
|
|
22
18
|
'document' = 'document',
|
package/src/db.model.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ObjectWithId, UnixTimestamp } from '@naturalcycles/js-lib'
|
|
2
|
-
import type { CommonDB } from './common.db.js'
|
|
1
|
+
import type { ObjectWithId, UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
2
|
+
import type { CommonDB } from './commondb/common.db.js'
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Similar to SQL INSERT, UPDATE.
|
package/src/index.ts
CHANGED
|
@@ -1,17 +1,6 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './adapter/inmemory/queryInMemory.js'
|
|
4
|
-
export * from './base.common.db.js'
|
|
5
|
-
export * from './common.db.js'
|
|
6
|
-
export * from './commondao/common.dao.js'
|
|
7
|
-
export * from './commondao/common.dao.model.js'
|
|
1
|
+
export * from './commondb/base.common.db.js'
|
|
2
|
+
export * from './commondb/common.db.js'
|
|
8
3
|
export * from './db.model.js'
|
|
9
|
-
export * from './kv/commonKeyValueDao.js'
|
|
10
|
-
export * from './kv/commonKeyValueDaoMemoCache.js'
|
|
11
|
-
export * from './kv/commonKeyValueDB.js'
|
|
12
4
|
export * from './model.util.js'
|
|
13
|
-
export * from './pipeline/dbPipelineBackup.js'
|
|
14
|
-
export * from './pipeline/dbPipelineCopy.js'
|
|
15
|
-
export * from './pipeline/dbPipelineRestore.js'
|
|
16
5
|
export * from './query/dbQuery.js'
|
|
17
6
|
export * from './transaction/dbTransaction.util.js'
|
|
@@ -1,30 +1,33 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
} from '@naturalcycles/js-lib'
|
|
2
|
+
import { _by, _isEmptyObject } from '@naturalcycles/js-lib'
|
|
3
|
+
import { _since, localTime } from '@naturalcycles/js-lib/datetime'
|
|
4
|
+
import { _assert } from '@naturalcycles/js-lib/error'
|
|
5
|
+
import {
|
|
6
|
+
generateJsonSchemaFromData,
|
|
7
|
+
type JsonSchemaObject,
|
|
8
|
+
type JsonSchemaRootObject,
|
|
9
|
+
} from '@naturalcycles/js-lib/json-schema'
|
|
10
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
11
|
+
import { _deepCopy, _sortObjectDeep } from '@naturalcycles/js-lib/object'
|
|
12
|
+
import { pMap } from '@naturalcycles/js-lib/promise'
|
|
10
13
|
import {
|
|
11
|
-
_assert,
|
|
12
|
-
_by,
|
|
13
|
-
_deepCopy,
|
|
14
|
-
_isEmptyObject,
|
|
15
|
-
_since,
|
|
16
|
-
_sortObjectDeep,
|
|
17
14
|
_stringMapEntries,
|
|
18
15
|
_stringMapValues,
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
} from '@naturalcycles/js-lib'
|
|
23
|
-
import { dimGrey, yellow } from '@naturalcycles/nodejs-lib'
|
|
24
|
-
import { fs2 } from '@naturalcycles/nodejs-lib/
|
|
25
|
-
import {
|
|
26
|
-
|
|
27
|
-
|
|
16
|
+
type AnyObjectWithId,
|
|
17
|
+
type ObjectWithId,
|
|
18
|
+
type StringMap,
|
|
19
|
+
} from '@naturalcycles/js-lib/types'
|
|
20
|
+
import { dimGrey, yellow } from '@naturalcycles/nodejs-lib/colors'
|
|
21
|
+
import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
|
|
22
|
+
import {
|
|
23
|
+
_pipeline,
|
|
24
|
+
bufferReviver,
|
|
25
|
+
createReadStreamAsNDJSON,
|
|
26
|
+
createWriteStreamAsNDJSON,
|
|
27
|
+
type ReadableTyped,
|
|
28
|
+
} from '@naturalcycles/nodejs-lib/stream'
|
|
29
|
+
import type { CommonDB, CommonDBSupport } from '../commondb/common.db.js'
|
|
30
|
+
import { commonDBFullSupport, CommonDBType } from '../commondb/common.db.js'
|
|
28
31
|
import type {
|
|
29
32
|
CommonDBCreateOptions,
|
|
30
33
|
CommonDBOptions,
|
|
@@ -34,8 +37,8 @@ import type {
|
|
|
34
37
|
DBTransaction,
|
|
35
38
|
DBTransactionFn,
|
|
36
39
|
RunQueryResult,
|
|
37
|
-
} from '
|
|
38
|
-
import type { DBQuery } from '
|
|
40
|
+
} from '../db.model.js'
|
|
41
|
+
import type { DBQuery } from '../query/dbQuery.js'
|
|
39
42
|
import { queryInMemory } from './queryInMemory.js'
|
|
40
43
|
|
|
41
44
|
export interface InMemoryDBCfg {
|
|
@@ -329,7 +332,7 @@ export class InMemoryDB implements CommonDB {
|
|
|
329
332
|
tables++
|
|
330
333
|
const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`
|
|
331
334
|
|
|
332
|
-
await _pipeline([Readable.from(rows), ...
|
|
335
|
+
await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(fname)])
|
|
333
336
|
})
|
|
334
337
|
|
|
335
338
|
this.cfg.logger!.log(
|
|
@@ -357,7 +360,7 @@ export class InMemoryDB implements CommonDB {
|
|
|
357
360
|
const fname = `${persistentStoragePath}/${file}`
|
|
358
361
|
const table = file.split('.ndjson')[0]!
|
|
359
362
|
|
|
360
|
-
const rows = await
|
|
363
|
+
const rows = await createReadStreamAsNDJSON(fname).toArray()
|
|
361
364
|
|
|
362
365
|
this.data[table] = _by(rows, r => r.id)
|
|
363
366
|
})
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
|
-
import type { StringMap } from '@naturalcycles/js-lib'
|
|
2
|
+
import type { StringMap } from '@naturalcycles/js-lib/types'
|
|
3
3
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
4
|
-
import type { CommonDBCreateOptions } from '
|
|
5
|
-
import type {
|
|
6
|
-
|
|
7
|
-
IncrementTuple,
|
|
8
|
-
KeyValueDBTuple,
|
|
9
|
-
} from '../../kv/commonKeyValueDB.js'
|
|
10
|
-
import { commonKeyValueDBFullSupport } from '../../kv/commonKeyValueDB.js'
|
|
4
|
+
import type { CommonDBCreateOptions } from '../db.model.js'
|
|
5
|
+
import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '../kv/commonKeyValueDB.js'
|
|
6
|
+
import { commonKeyValueDBFullSupport } from '../kv/commonKeyValueDB.js'
|
|
11
7
|
|
|
12
8
|
export interface InMemoryKeyValueDBCfg {}
|
|
13
9
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import type { DBQuery, DBQueryFilterOperator } from '
|
|
1
|
+
import { _get, _pick } from '@naturalcycles/js-lib/object'
|
|
2
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types'
|
|
3
|
+
import type { DBQuery, DBQueryFilterOperator } from '../query/dbQuery.js'
|
|
4
4
|
|
|
5
5
|
type FilterFn = (v: any, val: any) => boolean
|
|
6
6
|
const FILTER_FNS: Record<DBQueryFilterOperator, FilterFn> = {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Integer, UnixTimestamp } from '@naturalcycles/js-lib'
|
|
1
|
+
import type { Integer, UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
2
2
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
3
3
|
import type { CommonDBCreateOptions } from '../db.model.js'
|
|
4
4
|
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { AppError } from '@naturalcycles/js-lib/error'
|
|
2
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
3
|
+
import { pMap } from '@naturalcycles/js-lib/promise'
|
|
4
|
+
import type { KeyValueTuple } from '@naturalcycles/js-lib/types'
|
|
4
5
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
6
|
+
import { deflateString, inflateToString } from '@naturalcycles/nodejs-lib/zip'
|
|
5
7
|
import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js'
|
|
6
8
|
import type { CommonDBCreateOptions } from '../db.model.js'
|
|
7
9
|
import type {
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime'
|
|
2
|
+
import type { AsyncMemoCache } from '@naturalcycles/js-lib/decorators'
|
|
3
|
+
import type { NumberOfSeconds, UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
4
|
+
import { MISS } from '@naturalcycles/js-lib/types'
|
|
3
5
|
import type { CommonKeyValueDao } from './commonKeyValueDao.js'
|
|
4
6
|
|
|
5
7
|
export interface CommonKeyValueDaoMemoCacheCfg<VALUE> {
|
package/src/kv/index.ts
ADDED
package/src/model.util.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime'
|
|
2
|
+
import type { CreatedUpdated, CreatedUpdatedId } from '@naturalcycles/js-lib/types'
|
|
3
3
|
import { stringId } from '@naturalcycles/nodejs-lib'
|
|
4
4
|
|
|
5
5
|
export function createdUpdatedFields(
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime'
|
|
2
|
+
import { AppError, ErrorMode } from '@naturalcycles/js-lib/error'
|
|
3
|
+
import { pMap } from '@naturalcycles/js-lib/promise'
|
|
4
|
+
import type { AsyncMapper, StringMap, UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
5
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib/types'
|
|
6
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
|
|
7
|
+
import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
|
|
8
|
+
import {
|
|
9
|
+
createWriteStreamAsNDJSON,
|
|
10
|
+
type TransformLogProgressOptions,
|
|
11
|
+
type TransformMapOptions,
|
|
8
12
|
} from '@naturalcycles/nodejs-lib/stream'
|
|
9
13
|
import {
|
|
10
14
|
_pipeline,
|
|
@@ -13,7 +17,7 @@ import {
|
|
|
13
17
|
transformMap,
|
|
14
18
|
transformTap,
|
|
15
19
|
} from '@naturalcycles/nodejs-lib/stream'
|
|
16
|
-
import type { CommonDB } from '../common.db.js'
|
|
20
|
+
import type { CommonDB } from '../commondb/common.db.js'
|
|
17
21
|
import { DBQuery } from '../query/dbQuery.js'
|
|
18
22
|
|
|
19
23
|
export interface DBPipelineBackupOptions extends TransformLogProgressOptions {
|
|
@@ -223,7 +227,7 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
|
|
|
223
227
|
transformTap(() => {
|
|
224
228
|
rows++
|
|
225
229
|
}),
|
|
226
|
-
...
|
|
230
|
+
...createWriteStreamAsNDJSON(filePath),
|
|
227
231
|
])
|
|
228
232
|
|
|
229
233
|
const { size: sizeBytes } = await fs2.statAsync(filePath)
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { localTime } from '@naturalcycles/js-lib/datetime'
|
|
2
|
+
import { ErrorMode } from '@naturalcycles/js-lib/error'
|
|
3
|
+
import { pMap } from '@naturalcycles/js-lib/promise'
|
|
4
|
+
import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
|
|
5
|
+
import { _passthroughMapper } from '@naturalcycles/js-lib/types'
|
|
6
|
+
import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
|
|
4
7
|
import type {
|
|
5
8
|
TransformLogProgressOptions,
|
|
6
9
|
TransformMapOptions,
|
|
@@ -14,7 +17,7 @@ import {
|
|
|
14
17
|
transformTap,
|
|
15
18
|
writableForEach,
|
|
16
19
|
} from '@naturalcycles/nodejs-lib/stream'
|
|
17
|
-
import type { CommonDB } from '../common.db.js'
|
|
20
|
+
import type { CommonDB } from '../commondb/common.db.js'
|
|
18
21
|
import type { CommonDBSaveOptions } from '../db.model.js'
|
|
19
22
|
import { DBQuery } from '../query/dbQuery.js'
|
|
20
23
|
|