@verdant-web/server 3.1.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/storage/sqlShard/_testData/unifiedData.d.ts +4 -4
- package/dist/esm/storage/sqlShard/_testData/unifiedData.js +8 -8
- package/dist/esm/storage/sqlShard/_testData/unifiedData.js.map +1 -1
- package/dist/esm/storage/sqlShard/transfer.d.ts +2 -1
- package/dist/esm/storage/sqlShard/transfer.js +21 -7
- package/dist/esm/storage/sqlShard/transfer.js.map +1 -1
- package/dist/esm/storage/sqlShard/transfer.test.js +35 -20
- package/dist/esm/storage/sqlShard/transfer.test.js.map +1 -1
- package/package.json +1 -1
- package/src/storage/sqlShard/_testData/unifiedData.ts +8 -15
- package/src/storage/sqlShard/transfer.test.ts +39 -20
- package/src/storage/sqlShard/transfer.ts +27 -11
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare function randomOperations(libraryId: string): {
|
|
1
|
+
export declare function randomOperations(libraryId: string, count?: number): {
|
|
2
2
|
oid: string;
|
|
3
3
|
timestamp: string;
|
|
4
4
|
data: string;
|
|
@@ -6,20 +6,20 @@ export declare function randomOperations(libraryId: string): {
|
|
|
6
6
|
replicaId: string;
|
|
7
7
|
libraryId: string;
|
|
8
8
|
}[];
|
|
9
|
-
export declare function randomBaselines(libraryId: string): {
|
|
9
|
+
export declare function randomBaselines(libraryId: string, count?: number): {
|
|
10
10
|
oid: string;
|
|
11
11
|
snapshot: string;
|
|
12
12
|
timestamp: string;
|
|
13
13
|
libraryId: string;
|
|
14
14
|
}[];
|
|
15
|
-
export declare function randomFileMetadata(libraryId: string): {
|
|
15
|
+
export declare function randomFileMetadata(libraryId: string, count?: number): {
|
|
16
16
|
libraryId: string;
|
|
17
17
|
fileId: string;
|
|
18
18
|
name: string;
|
|
19
19
|
type: string;
|
|
20
20
|
pendingDeleteAt: number;
|
|
21
21
|
}[];
|
|
22
|
-
export declare function randomReplicaInfo(libraryId: string): {
|
|
22
|
+
export declare function randomReplicaInfo(libraryId: string, count?: number): {
|
|
23
23
|
libraryId: string;
|
|
24
24
|
id: string;
|
|
25
25
|
clientId: string;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
function random() {
|
|
2
2
|
return Math.random().toString(36).substring(7);
|
|
3
3
|
}
|
|
4
|
-
export function randomOperations(libraryId) {
|
|
5
|
-
return Array.from({ length:
|
|
4
|
+
export function randomOperations(libraryId, count = 10) {
|
|
5
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
6
6
|
oid: `oid-${i}-${random()}`,
|
|
7
7
|
timestamp: `timestamp-${i}-${random()}`,
|
|
8
8
|
data: `data-${i}-${random()}`,
|
|
@@ -11,16 +11,16 @@ export function randomOperations(libraryId) {
|
|
|
11
11
|
libraryId,
|
|
12
12
|
}));
|
|
13
13
|
}
|
|
14
|
-
export function randomBaselines(libraryId) {
|
|
15
|
-
return Array.from({ length:
|
|
14
|
+
export function randomBaselines(libraryId, count = 10) {
|
|
15
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
16
16
|
oid: `oid-${i}-${random()}`,
|
|
17
17
|
snapshot: `snapshot-${i}-${random()}`,
|
|
18
18
|
timestamp: `timestamp-${i}-${random()}`,
|
|
19
19
|
libraryId,
|
|
20
20
|
}));
|
|
21
21
|
}
|
|
22
|
-
export function randomFileMetadata(libraryId) {
|
|
23
|
-
return Array.from({ length:
|
|
22
|
+
export function randomFileMetadata(libraryId, count = 10) {
|
|
23
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
24
24
|
libraryId,
|
|
25
25
|
fileId: `fileId-${i}-${random()}`,
|
|
26
26
|
name: `name-${i}-${random()}`,
|
|
@@ -28,8 +28,8 @@ export function randomFileMetadata(libraryId) {
|
|
|
28
28
|
pendingDeleteAt: i,
|
|
29
29
|
}));
|
|
30
30
|
}
|
|
31
|
-
export function randomReplicaInfo(libraryId) {
|
|
32
|
-
return Array.from({ length:
|
|
31
|
+
export function randomReplicaInfo(libraryId, count = 10) {
|
|
32
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
33
33
|
libraryId,
|
|
34
34
|
id: `replicaId-${i}`,
|
|
35
35
|
clientId: `clientId-${i}-${random()}`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unifiedData.js","sourceRoot":"","sources":["../../../../../src/storage/sqlShard/_testData/unifiedData.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"unifiedData.js","sourceRoot":"","sources":["../../../../../src/storage/sqlShard/_testData/unifiedData.ts"],"names":[],"mappings":"AAAA,SAAS,MAAM;IACd,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,KAAK,GAAG,EAAE;IAC7D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,GAAG,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,EAAE;QAC3B,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,EAAE,EAAE;QACvC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,EAAE;QAC7B,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,aAAa,CAAC,EAAE;QAC3B,SAAS;KACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAiB,EAAE,KAAK,GAAG,EAAE;IAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,GAAG,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,EAAE;QAC3B,QAAQ,EAAE,YAAY,CAAC,IAAI,MAAM,EAAE,EAAE;QACrC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,EAAE,EAAE;QACvC,SAAS;KACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,KAAK,GAAG,EAAE;IAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,SAAS;QACT,MAAM,EAAE,UAAU,CAAC,IAAI,MAAM,EAAE,EAAE;QACjC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,EAAE;QAC7B,IAAI,EAAE,QAAQ,CAAC,EAAE;QACjB,eAAe,EAAE,CAAC;KAClB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,KAAK,GAAG,EAAE;IAC9D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,SAAS;QACT,EAAE,EAAE,aAAa,CAAC,EAAE;QACpB,QAAQ,EAAE,YAAY,CAAC,IAAI,MAAM,EAAE,EAAE;QACrC,qBAAqB,EAAE,CAAC;QACxB,gBAAgB,EAAE,oBAAoB,CAAC,IAAI,MAAM,EAAE,EAAE;QACrD,IAAI,EAAE,CAAC;QACP,gBAAgB,EAAE,CAAC;KACnB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Kysely } from 'kysely';
|
|
2
2
|
import { Database } from './tables.js';
|
|
3
|
-
export declare function transferToShards({ file, directory, }: {
|
|
3
|
+
export declare function transferToShards({ file, directory, batchSize, }: {
|
|
4
4
|
file: string;
|
|
5
5
|
directory: string;
|
|
6
|
+
batchSize?: number;
|
|
6
7
|
}): Promise<Record<string, Kysely<Database>>>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { openDatabase as openUnifiedDatabase } from '../sql/database.js';
|
|
2
2
|
import { openDatabase } from './database.js';
|
|
3
|
-
|
|
3
|
+
const DEFAULT_BATCH_SIZE = 1000;
|
|
4
|
+
export async function transferToShards({ file, directory, batchSize = DEFAULT_BATCH_SIZE, }) {
|
|
4
5
|
console.info(`Transferring from unified database (${file}) to shards (${directory})...`);
|
|
5
6
|
const { db: unifiedDb, ready } = openUnifiedDatabase(file);
|
|
6
7
|
await ready;
|
|
@@ -11,7 +12,7 @@ export async function transferToShards({ file, directory, }) {
|
|
|
11
12
|
.execute();
|
|
12
13
|
const dbs = await Promise.all(libraries.map(async ({ libraryId }) => {
|
|
13
14
|
const db = await openDatabase(directory, libraryId);
|
|
14
|
-
await copyLibrary(libraryId, unifiedDb, db);
|
|
15
|
+
await copyLibrary(libraryId, unifiedDb, db, batchSize);
|
|
15
16
|
return db;
|
|
16
17
|
}));
|
|
17
18
|
return libraries.reduce((acc, { libraryId }, i) => {
|
|
@@ -19,7 +20,7 @@ export async function transferToShards({ file, directory, }) {
|
|
|
19
20
|
return acc;
|
|
20
21
|
}, {});
|
|
21
22
|
}
|
|
22
|
-
async function copyLibrary(libraryId, source, dest) {
|
|
23
|
+
async function copyLibrary(libraryId, source, dest, batchSize) {
|
|
23
24
|
const operations = await source
|
|
24
25
|
.selectFrom('OperationHistory')
|
|
25
26
|
.where('libraryId', '=', libraryId)
|
|
@@ -58,17 +59,30 @@ async function copyLibrary(libraryId, source, dest) {
|
|
|
58
59
|
}
|
|
59
60
|
await dest.transaction().execute(async (trx) => {
|
|
60
61
|
const actions = [];
|
|
62
|
+
// batch insert data to avoid maximum vars limits
|
|
61
63
|
if (operations.length) {
|
|
62
|
-
|
|
64
|
+
for (let i = 0; i < operations.length; i += batchSize) {
|
|
65
|
+
const toInsert = operations.slice(i, i + batchSize);
|
|
66
|
+
actions.push(trx.insertInto('OperationHistory').values(toInsert).execute());
|
|
67
|
+
}
|
|
63
68
|
}
|
|
64
69
|
if (baselines.length) {
|
|
65
|
-
|
|
70
|
+
for (let i = 0; i < baselines.length; i += batchSize) {
|
|
71
|
+
const toInsert = baselines.slice(i, i + batchSize);
|
|
72
|
+
actions.push(trx.insertInto('DocumentBaseline').values(toInsert).execute());
|
|
73
|
+
}
|
|
66
74
|
}
|
|
67
75
|
if (replicas.length) {
|
|
68
|
-
|
|
76
|
+
for (let i = 0; i < replicas.length; i += batchSize) {
|
|
77
|
+
const toInsert = replicas.slice(i, i + batchSize);
|
|
78
|
+
actions.push(trx.insertInto('ReplicaInfo').values(toInsert).execute());
|
|
79
|
+
}
|
|
69
80
|
}
|
|
70
81
|
if (fileMetadata.length) {
|
|
71
|
-
|
|
82
|
+
for (let i = 0; i < fileMetadata.length; i += batchSize) {
|
|
83
|
+
const toInsert = fileMetadata.slice(i, i + batchSize);
|
|
84
|
+
actions.push(trx.insertInto('FileMetadata').values(toInsert).execute());
|
|
85
|
+
}
|
|
72
86
|
}
|
|
73
87
|
await Promise.all(actions);
|
|
74
88
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../../../src/storage/sqlShard/transfer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EACtC,IAAI,EACJ,SAAS,
|
|
1
|
+
{"version":3,"file":"transfer.js","sourceRoot":"","sources":["../../../../src/storage/sqlShard/transfer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzE,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EACtC,IAAI,EACJ,SAAS,EACT,SAAS,GAAG,kBAAkB,GAK9B;IACA,OAAO,CAAC,IAAI,CACX,uCAAuC,IAAI,gBAAgB,SAAS,MAAM,CAC1E,CAAC;IACF,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC3D,MAAM,KAAK,CAAC;IACZ,MAAM,SAAS,GAAG,MAAM,SAAS;SAC/B,UAAU,CAAC,aAAa,CAAC;SACzB,MAAM,CAAC,uBAAuB,CAAC;SAC/B,QAAQ,EAAE;SACV,OAAO,EAAE,CAAC;IAEZ,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;QACrC,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,WAAW,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC;QACvD,OAAO,EAAE,CAAC;IACX,CAAC,CAAC,CACF,CAAC;IAEF,OAAO,SAAS,CAAC,MAAM,CACtB,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;QACzB,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACxB,OAAO,GAAG,CAAC;IACZ,CAAC,EACD,EAAsC,CACtC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACzB,SAAiB,EACjB,MAA+B,EAC/B,IAAsB,EACtB,SAAiB;IAEjB,MAAM,UAAU,GAAG,MAAM,MAAM;SAC7B,UAAU,CAAC,kBAAkB,CAAC;SAC9B,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,SAAS,CAAC;SAClC,SAAS,EAAE;SACX,OAAO,EAAE,CAAC;IACZ,MAAM,SAAS,GAAG,MAAM,MAAM;SAC5B,UAAU,CAAC,kBAAkB,CAAC;SAC9B,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,SAAS,CAAC;SAClC,SAAS,EAAE;SACX,OAAO,EAAE,CAAC;IACZ,MAAM,QAAQ,GAAG,MAAM,MAAM;SAC3B,UAAU,CAAC,aAAa,CAAC;SACzB,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,SAAS,CAAC;SAClC,SAAS,EAAE;SACX,OAAO,EAAE,CAAC;IACZ,MAAM,YAAY,GAAG,MAAM,MAAM;SAC/B,UAAU,CAAC,cAAc,CAAC;SAC1B,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,SAAS,CAAC;SAClC,SAAS,EAAE;SACX,OAAO,EAAE,CAAC;IAEZ,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,mBAAmB;QACnB,OAAO,SAAS,CAAC,SAAS,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,mBAAmB;QACnB,OAAO,QAAQ,CAAC,SAAS,CAAC;IAC3B,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,mBAAmB;QACnB,OAAO,OAAO,CAAC,SAAS,CAAC;IAC1B,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;QACrC,mBAAmB;QACnB,OAAO,QAAQ,CAAC,SAAS,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAC9C,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,iDAAiD;QACjD,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBACpD,OAAO,CAAC,IAAI,CACX,GAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAC7D,CAAC;YACH,CAAC;QACF,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBACnD,OAAO,CAAC,IAAI,CACX,GAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAC7D,CAAC;YACH,CAAC;QACF,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACrD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAClD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACxE,CAAC;QACF,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;YACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACzE,CAAC;QACF,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -17,9 +17,15 @@ describe('sql-shard storage transfer utility', () => {
|
|
|
17
17
|
await mkdir(testTempDir, { recursive: true });
|
|
18
18
|
});
|
|
19
19
|
afterAll(async () => {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
try {
|
|
21
|
+
await Promise.all(databases.map((db) => db.destroy()));
|
|
22
|
+
// empty the temp dir
|
|
23
|
+
await rm(testTempDir, { recursive: true });
|
|
24
|
+
}
|
|
25
|
+
catch (err) {
|
|
26
|
+
// what can you do...
|
|
27
|
+
console.error(err);
|
|
28
|
+
}
|
|
23
29
|
});
|
|
24
30
|
it('should transfer data from unified database to shards', async () => {
|
|
25
31
|
const unifiedFile = join(testTempDir, 'unified.sqlite');
|
|
@@ -30,27 +36,36 @@ describe('sql-shard storage transfer utility', () => {
|
|
|
30
36
|
const randomData = {};
|
|
31
37
|
for (const libraryId of libraryIds) {
|
|
32
38
|
randomData[libraryId] = {
|
|
33
|
-
|
|
39
|
+
// test a lot of them - more than max sqlite var count.
|
|
40
|
+
// only doing operations here to reduce test time.
|
|
41
|
+
operations: randomOperations(libraryId, 6000),
|
|
34
42
|
baselines: randomBaselines(libraryId),
|
|
35
43
|
replicas: randomReplicaInfo(libraryId),
|
|
36
44
|
fileMetadata: randomFileMetadata(libraryId),
|
|
37
45
|
};
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
.
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
46
|
+
// have to batch these...
|
|
47
|
+
for (let i = 0; i < randomData[libraryId].operations.length; i += 1000) {
|
|
48
|
+
const toInsert = randomData[libraryId].operations.slice(i, i + 1000);
|
|
49
|
+
await unifiedDb
|
|
50
|
+
.insertInto('OperationHistory')
|
|
51
|
+
.values(toInsert)
|
|
52
|
+
.execute();
|
|
53
|
+
}
|
|
54
|
+
for (let i = 0; i < randomData[libraryId].baselines.length; i += 1000) {
|
|
55
|
+
const toInsert = randomData[libraryId].baselines.slice(i, i + 1000);
|
|
56
|
+
await unifiedDb
|
|
57
|
+
.insertInto('DocumentBaseline')
|
|
58
|
+
.values(toInsert)
|
|
59
|
+
.execute();
|
|
60
|
+
}
|
|
61
|
+
for (let i = 0; i < randomData[libraryId].fileMetadata.length; i += 1000) {
|
|
62
|
+
const toInsert = randomData[libraryId].fileMetadata.slice(i, i + 1000);
|
|
63
|
+
await unifiedDb.insertInto('FileMetadata').values(toInsert).execute();
|
|
64
|
+
}
|
|
65
|
+
for (let i = 0; i < randomData[libraryId].replicas.length; i += 1000) {
|
|
66
|
+
const toInsert = randomData[libraryId].replicas.slice(i, i + 1000);
|
|
67
|
+
await unifiedDb.insertInto('ReplicaInfo').values(toInsert).execute();
|
|
68
|
+
}
|
|
54
69
|
}
|
|
55
70
|
const shards = await transferToShards({
|
|
56
71
|
directory: testTempDir,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transfer.test.js","sourceRoot":"","sources":["../../../../src/storage/sqlShard/transfer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACnE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EACN,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GACjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AASpC,MAAM,WAAW,GAAG,IAAI,CACvB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACvC,WAAW,EACX,KAAK,CACL,CAAC;AAEF,QAAQ,CAAC,oCAAoC,EAAE,GAAG,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,KAAK,EAAe,CAAC;IAC3C,SAAS,CAAC,KAAK,IAAI,EAAE;QACpB,IAAI,CAAC;YACJ,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;QAChB,oBAAoB;QACpB,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"transfer.test.js","sourceRoot":"","sources":["../../../../src/storage/sqlShard/transfer.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACnE,OAAO,EAAE,YAAY,IAAI,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EACN,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GACjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AASpC,MAAM,WAAW,GAAG,IAAI,CACvB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACvC,WAAW,EACX,KAAK,CACL,CAAC;AAEF,QAAQ,CAAC,oCAAoC,EAAE,GAAG,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,KAAK,EAAe,CAAC;IAC3C,SAAS,CAAC,KAAK,IAAI,EAAE;QACpB,IAAI,CAAC;YACJ,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC,CAAA,CAAC;QAChB,oBAAoB;QACpB,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,KAAK,IAAI,EAAE;QACnB,IAAI,CAAC;YACJ,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACvD,qBAAqB;YACrB,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,qBAAqB;YACrB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACxD,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAClE,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1B,MAAM,KAAK,CAAC;QACZ,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,EAQlB,CAAC;QACF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,UAAU,CAAC,SAAS,CAAC,GAAG;gBACvB,uDAAuD;gBACvD,kDAAkD;gBAClD,UAAU,EAAE,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC;gBAC7C,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC;gBACrC,QAAQ,EAAE,iBAAiB,CAAC,SAAS,CAAC;gBACtC,YAAY,EAAE,kBAAkB,CAAC,SAAS,CAAC;aAC3C,CAAC;YAEF,yBAAyB;YACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACxE,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACrE,MAAM,SAAS;qBACb,UAAU,CAAC,kBAAkB,CAAC;qBAC9B,MAAM,CAAC,QAAQ,CAAC;qBAChB,OAAO,EAAE,CAAC;YACb,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACvE,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACpE,MAAM,SAAS;qBACb,UAAU,CAAC,kBAAkB,CAAC;qBAC9B,MAAM,CAAC,QAAQ,CAAC;qBAChB,OAAO,EAAE,CAAC;YACb,CAAC;YACD,KACC,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,MAAM,EAC7C,CAAC,IAAI,IAAI,EACR,CAAC;gBACF,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACvE,MAAM,SAAS,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;YACvE,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACtE,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBACnE,MAAM,SAAS,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;YACtE,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC;YACrC,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,WAAW;SACjB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAE9D,KAAK,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACtD,MAAM,UAAU,GAAG,MAAM,EAAE;iBACzB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YACZ,MAAM,SAAS,GAAG,MAAM,EAAE;iBACxB,UAAU,CAAC,kBAAkB,CAAC;iBAC9B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,CAAC;YAC1E,MAAM,YAAY,GAAG,MAAM,EAAE;iBAC3B,UAAU,CAAC,cAAc,CAAC;iBAC1B,SAAS,EAAE;iBACX,OAAO,EAAE,CAAC;YAEZ,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACxE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACtE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/B,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,MAAM,CACzC,CAAC;YAEF,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,CAAC;gBACxD,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/D,CAAC;YACD,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,SAAS,EAAE,CAAC;gBACvD,MAAM,CACL,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,KAAK,GAAG,CAAC,CAClD,CAAC,WAAW,EAAE,CAAC;YACjB,CAAC;YACD,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACrD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACrE,CAAC;YACD,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC,YAAY,EAAE,CAAC;gBAC7D,MAAM,CACL,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CACnD,CAAC,WAAW,EAAE,CAAC;YACjB,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,16 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
OperationHistoryRow,
|
|
3
|
-
DocumentBaselineRow,
|
|
4
|
-
FileMetadataRow,
|
|
5
|
-
ReplicaInfoRow,
|
|
6
|
-
} from '../../sql/tables.js';
|
|
7
|
-
|
|
8
1
|
function random() {
|
|
9
2
|
return Math.random().toString(36).substring(7);
|
|
10
3
|
}
|
|
11
4
|
|
|
12
|
-
export function randomOperations(libraryId: string) {
|
|
13
|
-
return Array.from({ length:
|
|
5
|
+
export function randomOperations(libraryId: string, count = 10) {
|
|
6
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
14
7
|
oid: `oid-${i}-${random()}`,
|
|
15
8
|
timestamp: `timestamp-${i}-${random()}`,
|
|
16
9
|
data: `data-${i}-${random()}`,
|
|
@@ -20,8 +13,8 @@ export function randomOperations(libraryId: string) {
|
|
|
20
13
|
}));
|
|
21
14
|
}
|
|
22
15
|
|
|
23
|
-
export function randomBaselines(libraryId: string) {
|
|
24
|
-
return Array.from({ length:
|
|
16
|
+
export function randomBaselines(libraryId: string, count = 10) {
|
|
17
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
25
18
|
oid: `oid-${i}-${random()}`,
|
|
26
19
|
snapshot: `snapshot-${i}-${random()}`,
|
|
27
20
|
timestamp: `timestamp-${i}-${random()}`,
|
|
@@ -29,8 +22,8 @@ export function randomBaselines(libraryId: string) {
|
|
|
29
22
|
}));
|
|
30
23
|
}
|
|
31
24
|
|
|
32
|
-
export function randomFileMetadata(libraryId: string) {
|
|
33
|
-
return Array.from({ length:
|
|
25
|
+
export function randomFileMetadata(libraryId: string, count = 10) {
|
|
26
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
34
27
|
libraryId,
|
|
35
28
|
fileId: `fileId-${i}-${random()}`,
|
|
36
29
|
name: `name-${i}-${random()}`,
|
|
@@ -39,8 +32,8 @@ export function randomFileMetadata(libraryId: string) {
|
|
|
39
32
|
}));
|
|
40
33
|
}
|
|
41
34
|
|
|
42
|
-
export function randomReplicaInfo(libraryId: string) {
|
|
43
|
-
return Array.from({ length:
|
|
35
|
+
export function randomReplicaInfo(libraryId: string, count = 10) {
|
|
36
|
+
return Array.from({ length: count }, (_, i) => ({
|
|
44
37
|
libraryId,
|
|
45
38
|
id: `replicaId-${i}`,
|
|
46
39
|
clientId: `clientId-${i}-${random()}`,
|
|
@@ -34,9 +34,14 @@ describe('sql-shard storage transfer utility', () => {
|
|
|
34
34
|
await mkdir(testTempDir, { recursive: true });
|
|
35
35
|
});
|
|
36
36
|
afterAll(async () => {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
try {
|
|
38
|
+
await Promise.all(databases.map((db) => db.destroy()));
|
|
39
|
+
// empty the temp dir
|
|
40
|
+
await rm(testTempDir, { recursive: true });
|
|
41
|
+
} catch (err) {
|
|
42
|
+
// what can you do...
|
|
43
|
+
console.error(err);
|
|
44
|
+
}
|
|
40
45
|
});
|
|
41
46
|
|
|
42
47
|
it('should transfer data from unified database to shards', async () => {
|
|
@@ -56,27 +61,41 @@ describe('sql-shard storage transfer utility', () => {
|
|
|
56
61
|
>;
|
|
57
62
|
for (const libraryId of libraryIds) {
|
|
58
63
|
randomData[libraryId] = {
|
|
59
|
-
|
|
64
|
+
// test a lot of them - more than max sqlite var count.
|
|
65
|
+
// only doing operations here to reduce test time.
|
|
66
|
+
operations: randomOperations(libraryId, 6000),
|
|
60
67
|
baselines: randomBaselines(libraryId),
|
|
61
68
|
replicas: randomReplicaInfo(libraryId),
|
|
62
69
|
fileMetadata: randomFileMetadata(libraryId),
|
|
63
70
|
};
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
.
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
71
|
+
|
|
72
|
+
// have to batch these...
|
|
73
|
+
for (let i = 0; i < randomData[libraryId].operations.length; i += 1000) {
|
|
74
|
+
const toInsert = randomData[libraryId].operations.slice(i, i + 1000);
|
|
75
|
+
await unifiedDb
|
|
76
|
+
.insertInto('OperationHistory')
|
|
77
|
+
.values(toInsert)
|
|
78
|
+
.execute();
|
|
79
|
+
}
|
|
80
|
+
for (let i = 0; i < randomData[libraryId].baselines.length; i += 1000) {
|
|
81
|
+
const toInsert = randomData[libraryId].baselines.slice(i, i + 1000);
|
|
82
|
+
await unifiedDb
|
|
83
|
+
.insertInto('DocumentBaseline')
|
|
84
|
+
.values(toInsert)
|
|
85
|
+
.execute();
|
|
86
|
+
}
|
|
87
|
+
for (
|
|
88
|
+
let i = 0;
|
|
89
|
+
i < randomData[libraryId].fileMetadata.length;
|
|
90
|
+
i += 1000
|
|
91
|
+
) {
|
|
92
|
+
const toInsert = randomData[libraryId].fileMetadata.slice(i, i + 1000);
|
|
93
|
+
await unifiedDb.insertInto('FileMetadata').values(toInsert).execute();
|
|
94
|
+
}
|
|
95
|
+
for (let i = 0; i < randomData[libraryId].replicas.length; i += 1000) {
|
|
96
|
+
const toInsert = randomData[libraryId].replicas.slice(i, i + 1000);
|
|
97
|
+
await unifiedDb.insertInto('ReplicaInfo').values(toInsert).execute();
|
|
98
|
+
}
|
|
80
99
|
}
|
|
81
100
|
|
|
82
101
|
const shards = await transferToShards({
|
|
@@ -4,12 +4,16 @@ import { Database as UnifiedDatabase } from '../sql/tables.js';
|
|
|
4
4
|
import { Database } from './tables.js';
|
|
5
5
|
import { openDatabase } from './database.js';
|
|
6
6
|
|
|
7
|
+
const DEFAULT_BATCH_SIZE = 1000;
|
|
8
|
+
|
|
7
9
|
export async function transferToShards({
|
|
8
10
|
file,
|
|
9
11
|
directory,
|
|
12
|
+
batchSize = DEFAULT_BATCH_SIZE,
|
|
10
13
|
}: {
|
|
11
14
|
file: string;
|
|
12
15
|
directory: string;
|
|
16
|
+
batchSize?: number;
|
|
13
17
|
}) {
|
|
14
18
|
console.info(
|
|
15
19
|
`Transferring from unified database (${file}) to shards (${directory})...`,
|
|
@@ -25,7 +29,7 @@ export async function transferToShards({
|
|
|
25
29
|
const dbs = await Promise.all(
|
|
26
30
|
libraries.map(async ({ libraryId }) => {
|
|
27
31
|
const db = await openDatabase(directory, libraryId);
|
|
28
|
-
await copyLibrary(libraryId, unifiedDb, db);
|
|
32
|
+
await copyLibrary(libraryId, unifiedDb, db, batchSize);
|
|
29
33
|
return db;
|
|
30
34
|
}),
|
|
31
35
|
);
|
|
@@ -43,6 +47,7 @@ async function copyLibrary(
|
|
|
43
47
|
libraryId: string,
|
|
44
48
|
source: Kysely<UnifiedDatabase>,
|
|
45
49
|
dest: Kysely<Database>,
|
|
50
|
+
batchSize: number,
|
|
46
51
|
) {
|
|
47
52
|
const operations = await source
|
|
48
53
|
.selectFrom('OperationHistory')
|
|
@@ -84,23 +89,34 @@ async function copyLibrary(
|
|
|
84
89
|
|
|
85
90
|
await dest.transaction().execute(async (trx) => {
|
|
86
91
|
const actions: Promise<any>[] = [];
|
|
92
|
+
// batch insert data to avoid maximum vars limits
|
|
87
93
|
if (operations.length) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
94
|
+
for (let i = 0; i < operations.length; i += batchSize) {
|
|
95
|
+
const toInsert = operations.slice(i, i + batchSize);
|
|
96
|
+
actions.push(
|
|
97
|
+
trx.insertInto('OperationHistory').values(toInsert).execute(),
|
|
98
|
+
);
|
|
99
|
+
}
|
|
91
100
|
}
|
|
92
101
|
if (baselines.length) {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
102
|
+
for (let i = 0; i < baselines.length; i += batchSize) {
|
|
103
|
+
const toInsert = baselines.slice(i, i + batchSize);
|
|
104
|
+
actions.push(
|
|
105
|
+
trx.insertInto('DocumentBaseline').values(toInsert).execute(),
|
|
106
|
+
);
|
|
107
|
+
}
|
|
96
108
|
}
|
|
97
109
|
if (replicas.length) {
|
|
98
|
-
|
|
110
|
+
for (let i = 0; i < replicas.length; i += batchSize) {
|
|
111
|
+
const toInsert = replicas.slice(i, i + batchSize);
|
|
112
|
+
actions.push(trx.insertInto('ReplicaInfo').values(toInsert).execute());
|
|
113
|
+
}
|
|
99
114
|
}
|
|
100
115
|
if (fileMetadata.length) {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
116
|
+
for (let i = 0; i < fileMetadata.length; i += batchSize) {
|
|
117
|
+
const toInsert = fileMetadata.slice(i, i + batchSize);
|
|
118
|
+
actions.push(trx.insertInto('FileMetadata').values(toInsert).execute());
|
|
119
|
+
}
|
|
104
120
|
}
|
|
105
121
|
await Promise.all(actions);
|
|
106
122
|
});
|