gmongo 3.0.1 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +26 -0
- package/examples/collectionCopy.js +75 -0
- package/examples/encryptDataAtRest.js +16 -16
- package/mongo.js +19 -19
- package/package.json +4 -3
- package/services/aes.js +9 -9
package/.env.example
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Source Database Configuration
|
|
2
|
+
FROM_ATLAS=false
|
|
3
|
+
FROM_DB=source_database
|
|
4
|
+
FROM_IP=localhost
|
|
5
|
+
FROM_PORT=27017
|
|
6
|
+
FROM_USER=
|
|
7
|
+
FROM_PASS=
|
|
8
|
+
FROM_X509=false
|
|
9
|
+
|
|
10
|
+
# Destination Database Configuration
|
|
11
|
+
TO_ATLAS=false
|
|
12
|
+
TO_DB=destination_database
|
|
13
|
+
TO_IP=localhost
|
|
14
|
+
TO_PORT=27017
|
|
15
|
+
TO_USER=
|
|
16
|
+
TO_PASS=
|
|
17
|
+
TO_X509=false
|
|
18
|
+
|
|
19
|
+
# Connection Settings
|
|
20
|
+
CONNECTION_TIMEOUT_MS=10000
|
|
21
|
+
|
|
22
|
+
# Batch Settings
|
|
23
|
+
INSERT_BATCH_SIZE=1000
|
|
24
|
+
|
|
25
|
+
# Collections to Copy (comma-separated list)
|
|
26
|
+
COLLECTIONS_TO_COPY=collection1,collection2,collection3
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
require('dotenv').config()
|
|
2
|
+
const MGO = require('../mongo.js')
|
|
3
|
+
|
|
4
|
+
function splitIntoBatches(array, batchSize) {
|
|
5
|
+
const batches = []
|
|
6
|
+
for (let i = 0; i < array.length; i += batchSize) {
|
|
7
|
+
batches.push(array.slice(i, i + batchSize))
|
|
8
|
+
}
|
|
9
|
+
return batches
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async function copyCollections() {
|
|
13
|
+
const Source = {
|
|
14
|
+
ATLAS: process.env.FROM_ATLAS === 'true',
|
|
15
|
+
DB: process.env.FROM_DB,
|
|
16
|
+
IP: process.env.FROM_IP,
|
|
17
|
+
PORT: process.env.FROM_PORT,
|
|
18
|
+
USER: process.env.FROM_USER,
|
|
19
|
+
PASS: process.env.FROM_PASS,
|
|
20
|
+
X509: process.env.FROM_X509 === 'true',
|
|
21
|
+
}
|
|
22
|
+
const Destination = {
|
|
23
|
+
ATLAS: process.env.TO_ATLAS === 'true',
|
|
24
|
+
DB: process.env.TO_DB,
|
|
25
|
+
IP: process.env.TO_IP,
|
|
26
|
+
PORT: process.env.TO_PORT,
|
|
27
|
+
USER: process.env.TO_USER,
|
|
28
|
+
PASS: process.env.TO_PASS,
|
|
29
|
+
X509: process.env.TO_X509 === 'true',
|
|
30
|
+
}
|
|
31
|
+
const timeoutInMS = parseInt(process.env.CONNECTION_TIMEOUT_MS) || 10000
|
|
32
|
+
const insertBatchSize = parseInt(process.env.INSERT_BATCH_SIZE) || 1000
|
|
33
|
+
|
|
34
|
+
await MGO.start(
|
|
35
|
+
Source.ATLAS,
|
|
36
|
+
Source.DB,
|
|
37
|
+
Source.IP,
|
|
38
|
+
Source.PORT,
|
|
39
|
+
Source.USER,
|
|
40
|
+
Source.PASS,
|
|
41
|
+
Source.X509,
|
|
42
|
+
timeoutInMS,
|
|
43
|
+
)
|
|
44
|
+
await MGO.start(
|
|
45
|
+
Destination.ATLAS,
|
|
46
|
+
Destination.DB,
|
|
47
|
+
Destination.IP,
|
|
48
|
+
Destination.PORT,
|
|
49
|
+
Destination.USER,
|
|
50
|
+
Destination.PASS,
|
|
51
|
+
Destination.X509,
|
|
52
|
+
timeoutInMS,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
const CollectionsToCopy = process.env.COLLECTIONS_TO_COPY.split(',').map(
|
|
56
|
+
(c) => c.trim(),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
try {
|
|
60
|
+
for (const collectionName of CollectionsToCopy) {
|
|
61
|
+
const fromCollection = await MGO.query(Source.DB, collectionName, {})
|
|
62
|
+
const insertList = splitIntoBatches(fromCollection, insertBatchSize)
|
|
63
|
+
for (const batch of insertList) {
|
|
64
|
+
await MGO.insert(Destination.DB, collectionName, batch)
|
|
65
|
+
}
|
|
66
|
+
console.log(
|
|
67
|
+
`Copied collection: ${collectionName} with ${fromCollection.length} documents`,
|
|
68
|
+
)
|
|
69
|
+
}
|
|
70
|
+
} catch (error) {
|
|
71
|
+
console.error('Error copying collections:', error)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
copyCollections()
|
|
@@ -4,7 +4,7 @@ const keys = {
|
|
|
4
4
|
test: {
|
|
5
5
|
columns: ['name', 'dob'],
|
|
6
6
|
key: MGO.aes.format.key(
|
|
7
|
-
'6233616364623234393935616165333332633466376134663031633833643262'
|
|
7
|
+
'6233616364623234393935616165333332633466376134663031633833643262',
|
|
8
8
|
),
|
|
9
9
|
iv: MGO.aes.format.iv('62323166666532663034393535346233'),
|
|
10
10
|
},
|
|
@@ -35,8 +35,8 @@ const start = async (_) => {
|
|
|
35
35
|
data,
|
|
36
36
|
['test'],
|
|
37
37
|
keys.test.key,
|
|
38
|
-
keys.test.iv
|
|
39
|
-
)
|
|
38
|
+
keys.test.iv,
|
|
39
|
+
),
|
|
40
40
|
)
|
|
41
41
|
console.log(data)
|
|
42
42
|
}
|
|
@@ -45,7 +45,7 @@ const start = async (_) => {
|
|
|
45
45
|
console.log(
|
|
46
46
|
await MGO.delete(DBName, 'TestTable', {
|
|
47
47
|
_id: MGO.id('655003e5d765319d61037ceb'),
|
|
48
|
-
})
|
|
48
|
+
}),
|
|
49
49
|
)
|
|
50
50
|
}
|
|
51
51
|
if (false) {
|
|
@@ -60,8 +60,8 @@ const start = async (_) => {
|
|
|
60
60
|
{ test: 'My favourite data' },
|
|
61
61
|
['test'],
|
|
62
62
|
keys.test.key,
|
|
63
|
-
keys.test.iv
|
|
64
|
-
)
|
|
63
|
+
keys.test.iv,
|
|
64
|
+
),
|
|
65
65
|
)
|
|
66
66
|
}
|
|
67
67
|
if (false) {
|
|
@@ -73,8 +73,8 @@ const start = async (_) => {
|
|
|
73
73
|
{},
|
|
74
74
|
['test'],
|
|
75
75
|
keys.test.key,
|
|
76
|
-
keys.test.iv
|
|
77
|
-
)
|
|
76
|
+
keys.test.iv,
|
|
77
|
+
),
|
|
78
78
|
)
|
|
79
79
|
}
|
|
80
80
|
if (false) {
|
|
@@ -88,8 +88,8 @@ const start = async (_) => {
|
|
|
88
88
|
},
|
|
89
89
|
['test'],
|
|
90
90
|
keys.test.key,
|
|
91
|
-
keys.test.iv
|
|
92
|
-
)
|
|
91
|
+
keys.test.iv,
|
|
92
|
+
),
|
|
93
93
|
)
|
|
94
94
|
}
|
|
95
95
|
if (false) {
|
|
@@ -102,8 +102,8 @@ const start = async (_) => {
|
|
|
102
102
|
{},
|
|
103
103
|
['test'],
|
|
104
104
|
keys.test.key,
|
|
105
|
-
keys.test.iv
|
|
106
|
-
)
|
|
105
|
+
keys.test.iv,
|
|
106
|
+
),
|
|
107
107
|
)
|
|
108
108
|
}
|
|
109
109
|
if (false) {
|
|
@@ -117,8 +117,8 @@ const start = async (_) => {
|
|
|
117
117
|
{},
|
|
118
118
|
['test'],
|
|
119
119
|
keys.test.key,
|
|
120
|
-
keys.test.iv
|
|
121
|
-
)
|
|
120
|
+
keys.test.iv,
|
|
121
|
+
),
|
|
122
122
|
)
|
|
123
123
|
}
|
|
124
124
|
if (false) {
|
|
@@ -135,8 +135,8 @@ const start = async (_) => {
|
|
|
135
135
|
{},
|
|
136
136
|
['test'],
|
|
137
137
|
keys.test.key,
|
|
138
|
-
keys.test.iv
|
|
139
|
-
)
|
|
138
|
+
keys.test.iv,
|
|
139
|
+
),
|
|
140
140
|
)
|
|
141
141
|
}
|
|
142
142
|
|
package/mongo.js
CHANGED
|
@@ -67,7 +67,7 @@ const db = {
|
|
|
67
67
|
rowOrRows[i][columsOfRowOrRowsToEncrypt[n]] = aes.encrypt(
|
|
68
68
|
key,
|
|
69
69
|
iv,
|
|
70
|
-
rowOrRows[i][columsOfRowOrRowsToEncrypt[n]]
|
|
70
|
+
rowOrRows[i][columsOfRowOrRowsToEncrypt[n]],
|
|
71
71
|
)
|
|
72
72
|
}
|
|
73
73
|
}
|
|
@@ -76,7 +76,7 @@ const db = {
|
|
|
76
76
|
rowOrRows[columsOfRowOrRowsToEncrypt[n]] = aes.encrypt(
|
|
77
77
|
key,
|
|
78
78
|
iv,
|
|
79
|
-
rowOrRows[columsOfRowOrRowsToEncrypt[n]]
|
|
79
|
+
rowOrRows[columsOfRowOrRowsToEncrypt[n]],
|
|
80
80
|
)
|
|
81
81
|
}
|
|
82
82
|
}
|
|
@@ -88,7 +88,7 @@ const db = {
|
|
|
88
88
|
Array.isArray(rowOrRows) ? 'insertMany' : 'insertOne'
|
|
89
89
|
](rowOrRows),
|
|
90
90
|
resolve,
|
|
91
|
-
reject
|
|
91
|
+
reject,
|
|
92
92
|
)
|
|
93
93
|
} else {
|
|
94
94
|
reject('No Connection')
|
|
@@ -110,7 +110,7 @@ const db = {
|
|
|
110
110
|
newData,
|
|
111
111
|
columsOfRowOrRowsToEncrypt,
|
|
112
112
|
key,
|
|
113
|
-
iv
|
|
113
|
+
iv,
|
|
114
114
|
) =>
|
|
115
115
|
new Promise(async (resolve, reject) => {
|
|
116
116
|
if (db.databaseList[dbName]) {
|
|
@@ -119,7 +119,7 @@ const db = {
|
|
|
119
119
|
newData[columsOfRowOrRowsToEncrypt[n]] = aes.encrypt(
|
|
120
120
|
key,
|
|
121
121
|
iv,
|
|
122
|
-
newData[columsOfRowOrRowsToEncrypt[n]]
|
|
122
|
+
newData[columsOfRowOrRowsToEncrypt[n]],
|
|
123
123
|
)
|
|
124
124
|
}
|
|
125
125
|
}
|
|
@@ -128,7 +128,7 @@ const db = {
|
|
|
128
128
|
null,
|
|
129
129
|
await collection.updateMany(dataToUpdate, { $set: newData }),
|
|
130
130
|
resolve,
|
|
131
|
-
reject
|
|
131
|
+
reject,
|
|
132
132
|
)
|
|
133
133
|
} else {
|
|
134
134
|
reject('No Connection')
|
|
@@ -145,7 +145,7 @@ const db = {
|
|
|
145
145
|
? decrypt([result], columnsToDecrypt, key, iv)[0]
|
|
146
146
|
: result,
|
|
147
147
|
resolve,
|
|
148
|
-
reject
|
|
148
|
+
reject,
|
|
149
149
|
)
|
|
150
150
|
} else {
|
|
151
151
|
reject('No Connection')
|
|
@@ -166,7 +166,7 @@ const db = {
|
|
|
166
166
|
? decrypt(result, columnsToDecrypt, key, iv)
|
|
167
167
|
: result,
|
|
168
168
|
resolve,
|
|
169
|
-
reject
|
|
169
|
+
reject,
|
|
170
170
|
)
|
|
171
171
|
} else {
|
|
172
172
|
reject('No Connection')
|
|
@@ -187,7 +187,7 @@ const db = {
|
|
|
187
187
|
? decrypt(result, columnsToDecrypt, key, iv)
|
|
188
188
|
: result,
|
|
189
189
|
resolve,
|
|
190
|
-
reject
|
|
190
|
+
reject,
|
|
191
191
|
)
|
|
192
192
|
} else {
|
|
193
193
|
reject('No Connection')
|
|
@@ -201,7 +201,7 @@ const db = {
|
|
|
201
201
|
query,
|
|
202
202
|
columnsToDecrypt,
|
|
203
203
|
key,
|
|
204
|
-
iv
|
|
204
|
+
iv,
|
|
205
205
|
) =>
|
|
206
206
|
new Promise(async (resolve, reject) => {
|
|
207
207
|
if (db.databaseList[dbName]) {
|
|
@@ -219,7 +219,7 @@ const db = {
|
|
|
219
219
|
? decrypt(result, columnsToDecrypt, key, iv)
|
|
220
220
|
: result,
|
|
221
221
|
resolve,
|
|
222
|
-
reject
|
|
222
|
+
reject,
|
|
223
223
|
)
|
|
224
224
|
} else {
|
|
225
225
|
reject('No Connection')
|
|
@@ -236,7 +236,7 @@ const db = {
|
|
|
236
236
|
query,
|
|
237
237
|
columnsToDecrypt,
|
|
238
238
|
key,
|
|
239
|
-
iv
|
|
239
|
+
iv,
|
|
240
240
|
) =>
|
|
241
241
|
new Promise(async (resolve, reject) => {
|
|
242
242
|
if (db.databaseList[dbName]) {
|
|
@@ -259,8 +259,8 @@ const db = {
|
|
|
259
259
|
],
|
|
260
260
|
columnsToDecrypt,
|
|
261
261
|
key,
|
|
262
|
-
iv
|
|
263
|
-
)
|
|
262
|
+
iv,
|
|
263
|
+
),
|
|
264
264
|
)
|
|
265
265
|
} else {
|
|
266
266
|
reject('No Connection')
|
|
@@ -274,7 +274,7 @@ const db = {
|
|
|
274
274
|
query,
|
|
275
275
|
columnsToDecrypt,
|
|
276
276
|
key,
|
|
277
|
-
iv
|
|
277
|
+
iv,
|
|
278
278
|
) =>
|
|
279
279
|
new Promise(async (resolve, reject) => {
|
|
280
280
|
if (db.databaseList[dbName]) {
|
|
@@ -299,8 +299,8 @@ const db = {
|
|
|
299
299
|
aggregrateList,
|
|
300
300
|
columnsToDecrypt,
|
|
301
301
|
key,
|
|
302
|
-
iv
|
|
303
|
-
)
|
|
302
|
+
iv,
|
|
303
|
+
),
|
|
304
304
|
)
|
|
305
305
|
} else {
|
|
306
306
|
reject('No Connection')
|
|
@@ -314,7 +314,7 @@ const db = {
|
|
|
314
314
|
aggregates,
|
|
315
315
|
columnsToDecrypt,
|
|
316
316
|
key,
|
|
317
|
-
iv
|
|
317
|
+
iv,
|
|
318
318
|
) =>
|
|
319
319
|
new Promise(async (resolve, reject) => {
|
|
320
320
|
if (db.databaseList[dbName]) {
|
|
@@ -336,7 +336,7 @@ const db = {
|
|
|
336
336
|
? decrypt(result, columnsToDecrypt, key, iv)
|
|
337
337
|
: result,
|
|
338
338
|
resolve,
|
|
339
|
-
reject
|
|
339
|
+
reject,
|
|
340
340
|
)
|
|
341
341
|
} else {
|
|
342
342
|
reject('No Connection')
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gmongo",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.2",
|
|
4
4
|
"description": "MongoDB Connection Class",
|
|
5
5
|
"main": "mongo.js",
|
|
6
6
|
"scripts": {
|
|
@@ -17,9 +17,10 @@
|
|
|
17
17
|
},
|
|
18
18
|
"homepage": "https://github.com/GnomeGroup/gmongo#readme",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"eslint": "^8.
|
|
20
|
+
"eslint": "^8.57.0",
|
|
21
21
|
"mongodb": "^6.2.0",
|
|
22
|
-
"prettier": "^2.
|
|
22
|
+
"prettier": "^3.2.5",
|
|
23
|
+
"prettier-eslint": "^16.3.0"
|
|
23
24
|
},
|
|
24
25
|
"prettier": {
|
|
25
26
|
"singleQuote": true,
|
package/services/aes.js
CHANGED
|
@@ -11,36 +11,36 @@ module.exports = {
|
|
|
11
11
|
const encipher = crypto.createCipheriv(
|
|
12
12
|
'aes-256-cbc',
|
|
13
13
|
Buffer.from(key, 'hex'),
|
|
14
|
-
Buffer.from(iv, 'hex')
|
|
14
|
+
Buffer.from(iv, 'hex'),
|
|
15
15
|
)
|
|
16
16
|
return Buffer.from(
|
|
17
17
|
encipher.update(plainData, 'utf8', 'binary') + encipher.final('binary'),
|
|
18
|
-
'binary'
|
|
18
|
+
'binary',
|
|
19
19
|
).toString('base64')
|
|
20
20
|
},
|
|
21
21
|
decrypt: (key, iv, encrypted) => {
|
|
22
22
|
const decipher = crypto.createDecipheriv(
|
|
23
23
|
'aes-256-cbc',
|
|
24
24
|
Buffer.from(key, 'hex'),
|
|
25
|
-
Buffer.from(iv, 'hex')
|
|
25
|
+
Buffer.from(iv, 'hex'),
|
|
26
26
|
)
|
|
27
27
|
return (
|
|
28
28
|
decipher.update(
|
|
29
29
|
Buffer.from(encrypted, 'base64').toString('binary'),
|
|
30
30
|
'binary',
|
|
31
|
-
'utf8'
|
|
31
|
+
'utf8',
|
|
32
32
|
) + decipher.final('utf8')
|
|
33
33
|
)
|
|
34
34
|
},
|
|
35
|
-
makeIv: _ => crypto.randomBytes(16).toString('hex'),
|
|
36
|
-
makeKey: _ =>
|
|
35
|
+
makeIv: (_) => crypto.randomBytes(16).toString('hex'),
|
|
36
|
+
makeKey: (_) =>
|
|
37
37
|
crypto
|
|
38
38
|
.createHash('sha256')
|
|
39
39
|
.update(crypto.randomBytes(256))
|
|
40
40
|
.digest()
|
|
41
41
|
.toString('hex'),
|
|
42
42
|
format: {
|
|
43
|
-
iv: value => getBufferFromData(16, value),
|
|
44
|
-
key: value => getBufferFromData(32, value)
|
|
45
|
-
}
|
|
43
|
+
iv: (value) => getBufferFromData(16, value),
|
|
44
|
+
key: (value) => getBufferFromData(32, value),
|
|
45
|
+
},
|
|
46
46
|
}
|