@pi-r/gcp 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/index.js +50 -37
- package/download/index.js +37 -42
- package/package.json +6 -6
- package/upload/index.js +38 -37
package/client/index.js
CHANGED
|
@@ -13,6 +13,7 @@ exports.setBucketWebsite = setBucketWebsite;
|
|
|
13
13
|
exports.deleteObjects = deleteObjects;
|
|
14
14
|
exports.deleteObjectsV2 = deleteObjectsV2;
|
|
15
15
|
exports.deleteObjectsV3 = deleteObjectsV3;
|
|
16
|
+
exports.copyObject = copyObject;
|
|
16
17
|
exports.executeQuery = executeQuery;
|
|
17
18
|
exports.executeBatchQuery = executeBatchQuery;
|
|
18
19
|
exports.setPublicRead = setPublicRead;
|
|
@@ -98,15 +99,11 @@ async function hasBucket(client) {
|
|
|
98
99
|
}
|
|
99
100
|
async function updatePredefinedAcl(acl, values, metadata = {}) {
|
|
100
101
|
const items = [];
|
|
101
|
-
let projectNumber;
|
|
102
|
-
if (
|
|
103
|
-
|
|
104
|
-
(
|
|
105
|
-
|
|
106
|
-
const target = defaultObjectAcl.find(item => item.projectTeam?.projectNumber);
|
|
107
|
-
if (target !== undefined) {
|
|
108
|
-
projectNumber = target.projectTeam.projectNumber;
|
|
109
|
-
}
|
|
102
|
+
let { projectNumber, defaultObjectAcl } = metadata;
|
|
103
|
+
if (projectNumber === undefined && (0, types_1.isArray)(defaultObjectAcl)) {
|
|
104
|
+
const target = defaultObjectAcl.find(item => item.projectTeam?.projectNumber);
|
|
105
|
+
if (target !== undefined) {
|
|
106
|
+
projectNumber = target.projectTeam.projectNumber;
|
|
110
107
|
}
|
|
111
108
|
}
|
|
112
109
|
for (const target of acl.default ? [acl, acl.default] : [acl]) {
|
|
@@ -141,7 +138,7 @@ async function setUniformBucketLevel(bucket, enabled) {
|
|
|
141
138
|
const keyLimit = (limit) => limit > 0 ? `_${limit}` : '_0';
|
|
142
139
|
const asFieldPath = (params) => Array.isArray(params) && params.every(item => typeof item === 'string') ? new firestore_1.FieldPath(...params) : params;
|
|
143
140
|
const getFirebasePackageName = (credential) => credential.apiKey && credential.authDomain && credential ? 'firebase' : 'firebase-admin';
|
|
144
|
-
const isErrorConflict = (err) =>
|
|
141
|
+
const isErrorConflict = (err) => (0, types_1.isErrorCode)(err, 409);
|
|
145
142
|
const hasGoogleAuth = (credential) => (0, types_1.isString)(credential.keyFilename || credential.keyFile || process.env.GOOGLE_APPLICATION_CREDENTIALS) || (0, types_1.isObject)(credential.credentials) || !!credential.authClient;
|
|
146
143
|
const isSpanner = (data) => data.product === "spanner" || !data.product && !!(data.name && data.database && (data.query || data.table));
|
|
147
144
|
const isBigtable = (data) => data.product === "bigtable" || !data.product && !!(data.name && data.table && !data.database);
|
|
@@ -206,10 +203,10 @@ function createDatabaseClient(credential, data) {
|
|
|
206
203
|
}
|
|
207
204
|
return new firestore_1.Firestore(credential);
|
|
208
205
|
}
|
|
209
|
-
async function createBucket(credential, bucket, publicRead, service =
|
|
206
|
+
async function createBucket(credential, bucket, publicRead, service = "gcp") {
|
|
210
207
|
return createBucketV2.call(this, credential, bucket, publicRead ? 'publicRead' : undefined, createBucketRequest(credential), service);
|
|
211
208
|
}
|
|
212
|
-
async function createBucketV2(credential, bucket, acl, options, service =
|
|
209
|
+
async function createBucketV2(credential, bucket, acl, options, service = "gcp") {
|
|
213
210
|
if (isFirebaseApp(credential, true, true)) {
|
|
214
211
|
return false;
|
|
215
212
|
}
|
|
@@ -222,7 +219,7 @@ async function createBucketV2(credential, bucket, acl, options, service = 'gcp')
|
|
|
222
219
|
}
|
|
223
220
|
return instance.createBucket(bucket, options)
|
|
224
221
|
.then(async ([target]) => {
|
|
225
|
-
this.formatMessage(64, service, ["Bucket created", bucket], '',
|
|
222
|
+
this.formatMessage(64, service, ["Bucket created", bucket], '', Cloud.optionsLogMessage('COMMAND'));
|
|
226
223
|
if (acl) {
|
|
227
224
|
await setBucketPolicy.call(this, credential, bucket, { acl, bucket: target }, service);
|
|
228
225
|
}
|
|
@@ -232,17 +229,17 @@ async function createBucketV2(credential, bucket, acl, options, service = 'gcp')
|
|
|
232
229
|
if (isErrorConflict(err)) {
|
|
233
230
|
return acl ? await setBucketPolicy.call(this, credential, bucket, { acl }, service) : true;
|
|
234
231
|
}
|
|
235
|
-
this.formatFail(64, service, ["Unable to create bucket", bucket], err,
|
|
232
|
+
this.formatFail(64, service, ["Unable to create bucket", bucket], err, Cloud.optionsLogMessage('FAIL'));
|
|
236
233
|
return false;
|
|
237
234
|
});
|
|
238
235
|
}
|
|
239
|
-
async function setBucketPolicy(credential, bucket, options, service =
|
|
236
|
+
async function setBucketPolicy(credential, bucket, options, service = "gcp") {
|
|
240
237
|
if (isFirebaseApp(credential, true, true)) {
|
|
241
238
|
return false;
|
|
242
239
|
}
|
|
243
240
|
const client = createStorageClient.call(this, credential, bucket).bucket(bucket);
|
|
244
241
|
if (!await hasBucket(client)) {
|
|
245
|
-
this.formatFail(64, service, ["Bucket does not exist", bucket], (0, types_1.errorMessage)(service, "Bucket does not exist"),
|
|
242
|
+
this.formatFail(64, service, ["Bucket does not exist", bucket], (0, types_1.errorMessage)(service, "Bucket does not exist"), Cloud.optionsLogMessage('FAIL', { fatal: false }));
|
|
246
243
|
return false;
|
|
247
244
|
}
|
|
248
245
|
const errors = [];
|
|
@@ -283,13 +280,13 @@ async function setBucketPolicy(credential, bucket, options, service = 'gcp') {
|
|
|
283
280
|
errors.push(...await updatePredefinedAcl(client.acl, options.acl, options.metadata));
|
|
284
281
|
}
|
|
285
282
|
if (errors.length > 0) {
|
|
286
|
-
this.formatFail(64, service, ["Unable to update bucket policy", bucket], errors[0],
|
|
283
|
+
this.formatFail(64, service, ["Unable to update bucket policy", bucket], errors[0], Cloud.optionsLogMessage('FAIL', { fatal: false }));
|
|
287
284
|
return false;
|
|
288
285
|
}
|
|
289
|
-
this.formatMessage(64, service, ["Bucket ACL configured", bucket], (0, types_1.isString)(acl) ? acl : '',
|
|
286
|
+
this.formatMessage(64, service, ["Bucket ACL configured", bucket], (0, types_1.isString)(acl) ? acl : '', Cloud.optionsLogMessage('COMMAND'));
|
|
290
287
|
return true;
|
|
291
288
|
}
|
|
292
|
-
async function setBucketTagging(credential, bucket, labels, service =
|
|
289
|
+
async function setBucketTagging(credential, bucket, labels, service = "gcp") {
|
|
293
290
|
if (isFirebaseApp(credential, true, true) || !(0, types_1.isPlainObject)(labels)) {
|
|
294
291
|
return false;
|
|
295
292
|
}
|
|
@@ -307,15 +304,15 @@ async function setBucketTagging(credential, bucket, labels, service = 'gcp') {
|
|
|
307
304
|
}
|
|
308
305
|
return client.setMetadata({ labels })
|
|
309
306
|
.then(([metadata]) => {
|
|
310
|
-
this.formatMessage(64, service, [deleting ? "Tags deleted" : "Tags created", bucket], ((0, types_1.isPlainObject)(metadata.labels) ? Object.keys(metadata.labels).length : 0) + ' labels (current)',
|
|
307
|
+
this.formatMessage(64, service, [deleting ? "Tags deleted" : "Tags created", bucket], ((0, types_1.isPlainObject)(metadata.labels) ? Object.keys(metadata.labels).length : 0) + ' labels (current)', Cloud.optionsLogMessage('COMMAND'));
|
|
311
308
|
return true;
|
|
312
309
|
})
|
|
313
310
|
.catch((err) => {
|
|
314
|
-
this.formatFail(64, service, ["Unable to update bucket tagging", bucket], err,
|
|
311
|
+
this.formatFail(64, service, ["Unable to update bucket tagging", bucket], err, Cloud.optionsLogMessage('FAIL', { fatal: false }));
|
|
315
312
|
return false;
|
|
316
313
|
});
|
|
317
314
|
}
|
|
318
|
-
async function setBucketWebsite(credential, bucket, options, service =
|
|
315
|
+
async function setBucketWebsite(credential, bucket, options, service = "gcp") {
|
|
319
316
|
if (isFirebaseApp(credential, true, true)) {
|
|
320
317
|
return false;
|
|
321
318
|
}
|
|
@@ -327,13 +324,14 @@ async function setBucketWebsite(credential, bucket, options, service = 'gcp') {
|
|
|
327
324
|
if ((0, types_1.isString)(errorPage)) {
|
|
328
325
|
website.notFoundPage = errorPage;
|
|
329
326
|
}
|
|
330
|
-
|
|
327
|
+
const client = createStorageClient.call(this, credential, bucket).bucket(bucket);
|
|
328
|
+
return client.setMetadata({ website })
|
|
331
329
|
.then(() => {
|
|
332
|
-
this.formatMessage(64, service, ["Bucket website configured", bucket], website,
|
|
330
|
+
this.formatMessage(64, service, ["Bucket website configured", bucket], website, Cloud.optionsLogMessage('COMMAND'));
|
|
333
331
|
return true;
|
|
334
332
|
})
|
|
335
333
|
.catch((err) => {
|
|
336
|
-
this.formatFail(64, service, ["Unable to set bucket website", bucket], err,
|
|
334
|
+
this.formatFail(64, service, ["Unable to set bucket website", bucket], err, Cloud.optionsLogMessage('FAIL', { fatal: false }));
|
|
337
335
|
return false;
|
|
338
336
|
});
|
|
339
337
|
}
|
|
@@ -343,10 +341,10 @@ async function deleteObjects(credential, bucket, service, sdk, recursive = true)
|
|
|
343
341
|
async function deleteObjectsV2(credential, bucket, recursive = true, service) {
|
|
344
342
|
return deleteObjectsV3.call(this, credential, bucket, { recursive }, service);
|
|
345
343
|
}
|
|
346
|
-
async function deleteObjectsV3(credential, bucket, options = {}, service =
|
|
347
|
-
options = { ...options };
|
|
344
|
+
async function deleteObjectsV3(credential, bucket, options = {}, service = "gcp") {
|
|
348
345
|
let recursive = true;
|
|
349
346
|
if ('recursive' in options) {
|
|
347
|
+
options = { ...options };
|
|
350
348
|
recursive = options.recursive;
|
|
351
349
|
delete options.recursive;
|
|
352
350
|
}
|
|
@@ -381,7 +379,7 @@ async function deleteObjectsV3(credential, bucket, options = {}, service = 'gcp'
|
|
|
381
379
|
if (await hasBucket(client)) {
|
|
382
380
|
if (recursive) {
|
|
383
381
|
return client.deleteFiles({ ...options, force: true }).then(() => {
|
|
384
|
-
this.formatMessage(64, service, "Bucket emptied" + ' (recursive)', bucket,
|
|
382
|
+
this.formatMessage(64, service, "Bucket emptied" + ' (recursive)', bucket, Cloud.optionsLogMessage('COMMAND'));
|
|
385
383
|
});
|
|
386
384
|
}
|
|
387
385
|
const tasks = [];
|
|
@@ -390,7 +388,7 @@ async function deleteObjectsV3(credential, bucket, options = {}, service = 'gcp'
|
|
|
390
388
|
for (const { name } of result[0]) {
|
|
391
389
|
if (!name.includes('/')) {
|
|
392
390
|
tasks.push(client.file(name).delete({ ignoreNotFound: true }).catch((err) => {
|
|
393
|
-
this.formatFail(64, service, ["Delete failed", name], err,
|
|
391
|
+
this.formatFail(64, service, ["Delete failed", name], err, Cloud.optionsLogMessage('FAIL', { fatal: false }));
|
|
394
392
|
}));
|
|
395
393
|
}
|
|
396
394
|
}
|
|
@@ -405,7 +403,7 @@ async function deleteObjectsV3(credential, bucket, options = {}, service = 'gcp'
|
|
|
405
403
|
}
|
|
406
404
|
const checkErrors = () => {
|
|
407
405
|
if (errors.length > 0) {
|
|
408
|
-
this.formatFail(64, service, ["Unable to empty bucket" + (recursive ? ' (recursive)' : ''), errors.length + ' errors'], errors[0],
|
|
406
|
+
this.formatFail(64, service, ["Unable to empty bucket" + (recursive ? ' (recursive)' : ''), errors.length + ' errors'], errors[0], Cloud.optionsLogMessage('FAIL', { fatal: false }));
|
|
409
407
|
return true;
|
|
410
408
|
}
|
|
411
409
|
return false;
|
|
@@ -415,7 +413,7 @@ async function deleteObjectsV3(credential, bucket, options = {}, service = 'gcp'
|
|
|
415
413
|
errors.push(...result.filter(item => item.status === 'rejected').map((item) => item.reason));
|
|
416
414
|
if (!checkErrors()) {
|
|
417
415
|
const files = result.length + ' files';
|
|
418
|
-
this.formatMessage(64, service, ["Bucket emptied" + ` (${recursive ? 'recursive' : files})`, bucket], recursive ? files : '',
|
|
416
|
+
this.formatMessage(64, service, ["Bucket emptied" + ` (${recursive ? 'recursive' : files})`, bucket], recursive ? files : '', Cloud.optionsLogMessage('COMMAND'));
|
|
419
417
|
}
|
|
420
418
|
})
|
|
421
419
|
.catch((err) => {
|
|
@@ -424,6 +422,21 @@ async function deleteObjectsV3(credential, bucket, options = {}, service = 'gcp'
|
|
|
424
422
|
}
|
|
425
423
|
checkErrors();
|
|
426
424
|
}
|
|
425
|
+
async function copyObject(credential, bucketSource, keySource, bucket, key, options = {}, service = "gcp") {
|
|
426
|
+
if (isFirebaseApp(credential, true, true)) {
|
|
427
|
+
throw (0, types_1.errorValue)("Copy object not supported", 'firebase');
|
|
428
|
+
}
|
|
429
|
+
const instance = createStorageClient.call(this, credential);
|
|
430
|
+
const source = (0, types_1.isString)(keySource) ? instance.bucket(bucketSource).file(keySource) : keySource;
|
|
431
|
+
const destination = (0, types_1.isString)(key) ? instance.bucket(bucket).file(key) : key;
|
|
432
|
+
return source.copy(destination, options)
|
|
433
|
+
.then(() => {
|
|
434
|
+
this.formatMessage(64, service, ["Copy success", bucket], source.bucket.name + '/' + source.name, Cloud.optionsLogMessage('COMMAND'));
|
|
435
|
+
})
|
|
436
|
+
.catch((err) => {
|
|
437
|
+
this.formatFail(64, service, ["Copy failed", destination.bucket.name + '/' + destination.name], err, Cloud.optionsLogMessage('FAIL', { fatal: false }));
|
|
438
|
+
});
|
|
439
|
+
}
|
|
427
440
|
async function executeQuery(credential, data, sessionKey) {
|
|
428
441
|
return (await executeBatchQuery.call(this, credential, [data], sessionKey))[0] || [];
|
|
429
442
|
}
|
|
@@ -1043,20 +1056,20 @@ async function executeBatchQuery(credential, batch, sessionKey) {
|
|
|
1043
1056
|
closeClient();
|
|
1044
1057
|
return result;
|
|
1045
1058
|
}
|
|
1046
|
-
function setPublicRead(acl, pathname, requested, service =
|
|
1059
|
+
function setPublicRead(acl, pathname, requested, service = "gcp") {
|
|
1047
1060
|
acl.add({ entity: 'allUsers', role: 'READER' })
|
|
1048
1061
|
.then(() => {
|
|
1049
1062
|
if (requested) {
|
|
1050
|
-
this.formatMessage(64, service, "Grant public-read", pathname,
|
|
1063
|
+
this.formatMessage(64, service, "Grant public-read", pathname, Cloud.optionsLogMessage('COMMAND'));
|
|
1051
1064
|
}
|
|
1052
1065
|
})
|
|
1053
1066
|
.catch((err) => {
|
|
1054
1067
|
if (requested) {
|
|
1055
|
-
this.formatMessage(64, service, ["Unable to grant public-read", pathname], err,
|
|
1068
|
+
this.formatMessage(64, service, ["Unable to grant public-read", pathname], err, Cloud.optionsLogMessage('WARN'));
|
|
1056
1069
|
}
|
|
1057
1070
|
});
|
|
1058
1071
|
}
|
|
1059
|
-
async function setPredefinedAcl(target, value, bucket, pathname = '', metadata = {}, requested = true, service =
|
|
1072
|
+
async function setPredefinedAcl(target, value, bucket, pathname = '', metadata = {}, requested = true, service = "gcp") {
|
|
1060
1073
|
let revoke = false;
|
|
1061
1074
|
if (metadata === true) {
|
|
1062
1075
|
revoke = true;
|
|
@@ -1126,10 +1139,10 @@ async function setPredefinedAcl(target, value, bucket, pathname = '', metadata =
|
|
|
1126
1139
|
}
|
|
1127
1140
|
return updatePredefinedAcl(target, items, metadata).then(errors => {
|
|
1128
1141
|
if (errors.length > 0) {
|
|
1129
|
-
this.formatMessage(64, service, [`Unable to ${revoke ? 'revoke' : 'grant'} ` + value, bucket && pathname ? bucket.name
|
|
1142
|
+
this.formatMessage(64, service, [`Unable to ${revoke ? 'revoke' : 'grant'} ` + value, bucket && pathname ? Cloud.joinPath(bucket.name, pathname) : bucket ? bucket.name : pathname], errors[0], Cloud.optionsLogMessage('WARN'));
|
|
1130
1143
|
}
|
|
1131
1144
|
else if (requested && (bucket || pathname)) {
|
|
1132
|
-
this.formatMessage(64, service, [(revoke ? 'Revoke' : 'Grant') + ' ' + value, bucket?.name], pathname,
|
|
1145
|
+
this.formatMessage(64, service, [(revoke ? 'Revoke' : 'Grant') + ' ' + value, bucket?.name], pathname, Cloud.optionsLogMessage('COMMAND'));
|
|
1133
1146
|
}
|
|
1134
1147
|
});
|
|
1135
1148
|
}
|
package/download/index.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
const path = require("node:path");
|
|
3
3
|
const fs = require("node:fs");
|
|
4
|
-
const
|
|
4
|
+
const stream = require("node:stream");
|
|
5
5
|
const storage_1 = require("@google-cloud/storage");
|
|
6
6
|
const Cloud = require("@e-mc/cloud");
|
|
7
7
|
const types_1 = require("@e-mc/types");
|
|
8
|
+
const util_1 = require("@e-mc/cloud/util");
|
|
8
9
|
const client_1 = require("@pi-r/gcp");
|
|
9
|
-
function download(credential, service =
|
|
10
|
+
function download(credential, service = "gcp") {
|
|
10
11
|
const storage = client_1.createStorageClient.call(this, credential);
|
|
11
12
|
return (data, callback) => {
|
|
12
13
|
const { bucket: Bucket, download: target } = data;
|
|
@@ -15,56 +16,50 @@ function download(credential, service = 'gcp') {
|
|
|
15
16
|
callback((0, types_1.errorValue)('Missing property', !Bucket ? 'Bucket' : 'Key'));
|
|
16
17
|
return;
|
|
17
18
|
}
|
|
18
|
-
const tempDir = (0, types_1.getTempDir)(true, service);
|
|
19
19
|
const location = Cloud.joinPath(Bucket, Key);
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
this.
|
|
20
|
+
const deleteMessage = () => this.formatMessage(64, service, "Delete success", location, Cloud.optionsLogMessage('DELETE'));
|
|
21
|
+
const deleteFailed = (err) => {
|
|
22
|
+
this.formatFail(64, service, ["Delete failed", location], err, Cloud.optionsLogMessage('FAIL', { fatal: !!target.active }));
|
|
23
23
|
};
|
|
24
24
|
if ((0, client_1.isFirebaseApp)(credential, true, true)) {
|
|
25
|
-
const { ref,
|
|
25
|
+
const { ref, getStream, deleteObject } = require('@firebase/storage');
|
|
26
26
|
const objectRef = ref(storage, Key);
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
.then(deleteMessage)
|
|
38
|
-
.catch((err) => {
|
|
39
|
-
this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active });
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
})
|
|
43
|
-
.on('error', callback));
|
|
44
|
-
}
|
|
45
|
-
else {
|
|
46
|
-
callback((0, types_1.errorMessage)(statusCode, 'Invalid HTTP request', location));
|
|
47
|
-
}
|
|
48
|
-
});
|
|
49
|
-
});
|
|
27
|
+
(0, util_1.readableAsBuffer)(stream.Readable.from(getStream(objectRef)))
|
|
28
|
+
.then(buffer => {
|
|
29
|
+
callback(null, buffer);
|
|
30
|
+
if (target.deleteObject) {
|
|
31
|
+
deleteObject(objectRef)
|
|
32
|
+
.then(deleteMessage)
|
|
33
|
+
.catch(deleteFailed);
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
.catch(callback);
|
|
50
37
|
return;
|
|
51
38
|
}
|
|
52
39
|
const { flags = 0 } = target;
|
|
40
|
+
const tempDir = (0, types_1.getTempDir)(true, service);
|
|
41
|
+
const destination = path.join(tempDir, path.basename(target.filename || Key));
|
|
53
42
|
const bucket = storage.bucket(Bucket);
|
|
54
43
|
const file = bucket.file(Key, { generation: target.versionId });
|
|
55
44
|
const chunkSizeBytes = flags & 4 ? (0, types_1.alignSize)(target.chunkSize, 1024) : 0;
|
|
56
|
-
const
|
|
45
|
+
const complete = async () => {
|
|
57
46
|
callback(null, destination);
|
|
47
|
+
const copyTo = (0, util_1.intoArray)(target.copyObject);
|
|
48
|
+
if (copyTo) {
|
|
49
|
+
const tasks = [];
|
|
50
|
+
for (const { bucket: bucketName, pathname, filename, options } of copyTo) {
|
|
51
|
+
const keyObject = filename ? Cloud.joinPath(pathname, filename, true) : Key;
|
|
52
|
+
tasks.push(client_1.copyObject
|
|
53
|
+
.call(this, credential, Bucket, file, bucketName, keyObject, options)
|
|
54
|
+
.catch((0, util_1.createErrorHandler)(this, service, Bucket)));
|
|
55
|
+
}
|
|
56
|
+
await Promise.all(tasks);
|
|
57
|
+
}
|
|
58
58
|
const deleteObject = target.deleteObject;
|
|
59
59
|
if (deleteObject) {
|
|
60
|
-
file.delete((0, types_1.isObject)(deleteObject) ? deleteObject : { ignoreNotFound: true }
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active });
|
|
66
|
-
}
|
|
67
|
-
});
|
|
60
|
+
file.delete((0, types_1.isObject)(deleteObject) ? deleteObject : { ignoreNotFound: true })
|
|
61
|
+
.then(deleteMessage)
|
|
62
|
+
.catch(deleteFailed);
|
|
68
63
|
}
|
|
69
64
|
};
|
|
70
65
|
if (chunkSizeBytes > 0) {
|
|
@@ -73,18 +68,18 @@ function download(credential, service = 'gcp') {
|
|
|
73
68
|
chunkSizeBytes,
|
|
74
69
|
concurrencyLimit: target.chunkLimit || target.options?.concurrencyLimit
|
|
75
70
|
})
|
|
76
|
-
.then(
|
|
71
|
+
.then(complete)
|
|
77
72
|
.catch(callback);
|
|
78
73
|
}
|
|
79
74
|
else if (target.chunkLimit === 1 && !target.chunkSize) {
|
|
80
75
|
file.createReadStream()
|
|
81
76
|
.pipe(fs.createWriteStream(destination))
|
|
82
|
-
.on('finish',
|
|
77
|
+
.on('finish', complete)
|
|
83
78
|
.on('error', callback);
|
|
84
79
|
}
|
|
85
80
|
else {
|
|
86
81
|
file.download({ destination })
|
|
87
|
-
.then(
|
|
82
|
+
.then(complete)
|
|
88
83
|
.catch(callback);
|
|
89
84
|
}
|
|
90
85
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pi-r/gcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Google (GCP) cloud functions for E-mc.",
|
|
5
5
|
"main": "client/index.js",
|
|
6
6
|
"publishConfig": {
|
|
@@ -19,10 +19,10 @@
|
|
|
19
19
|
"license": "MIT",
|
|
20
20
|
"homepage": "https://github.com/anpham6/pi-r#readme",
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@e-mc/cloud": "^0.
|
|
23
|
-
"@e-mc/module": "^0.
|
|
24
|
-
"@e-mc/types": "^0.
|
|
25
|
-
"@google-cloud/firestore": "^7.11.
|
|
26
|
-
"@google-cloud/storage": "^7.
|
|
22
|
+
"@e-mc/cloud": "^0.13.0",
|
|
23
|
+
"@e-mc/module": "^0.13.0",
|
|
24
|
+
"@e-mc/types": "^0.13.0",
|
|
25
|
+
"@google-cloud/firestore": "^7.11.6",
|
|
26
|
+
"@google-cloud/storage": "^7.17.2"
|
|
27
27
|
}
|
|
28
28
|
}
|
package/upload/index.js
CHANGED
|
@@ -10,14 +10,14 @@ const client_1 = require("@pi-r/gcp");
|
|
|
10
10
|
const BUCKET_SESSION = new Set();
|
|
11
11
|
const BUCKET_RESPONSE = {};
|
|
12
12
|
const getBucketKey = (credential, bucket, acl = '') => Cloud.asString(credential, true) + bucket + '_' + acl;
|
|
13
|
-
function upload(credential, service =
|
|
13
|
+
function upload(credential, service = "gcp") {
|
|
14
14
|
const storage = client_1.createStorageClient.call(this, credential);
|
|
15
15
|
return async (data, callback) => {
|
|
16
16
|
const firebase = (0, client_1.isFirebaseApp)(credential, true, true);
|
|
17
17
|
const { bucket, localUri } = data;
|
|
18
|
-
const { pathname
|
|
18
|
+
const { pathname, flags = 0, fileGroup, contentType, metadata, endpoint, active, publicRead, acl, admin = {}, overwrite, options } = data.upload;
|
|
19
19
|
let filename = data.upload.filename || path.basename(localUri), chunkSizeBytes = 0, bucketClient, bucketKey, listFiles;
|
|
20
|
-
const
|
|
20
|
+
const complete = (err, url) => {
|
|
21
21
|
if (!firebase) {
|
|
22
22
|
BUCKET_SESSION.delete(bucket);
|
|
23
23
|
if (bucketKey) {
|
|
@@ -26,11 +26,7 @@ function upload(credential, service = 'gcp') {
|
|
|
26
26
|
}
|
|
27
27
|
callback(err, url);
|
|
28
28
|
};
|
|
29
|
-
const addLog = (
|
|
30
|
-
if (err instanceof Error) {
|
|
31
|
-
this.addLog(this.statusType.WARN, err, service, bucket);
|
|
32
|
-
}
|
|
33
|
-
};
|
|
29
|
+
const addLog = (0, util_1.createErrorHandler)(this, bucket, service);
|
|
34
30
|
if (firebase) {
|
|
35
31
|
const { ref, listAll } = require('@firebase/storage');
|
|
36
32
|
bucketClient = ref(storage, bucket);
|
|
@@ -39,7 +35,7 @@ function upload(credential, service = 'gcp') {
|
|
|
39
35
|
listFiles = (await listAll(bucketClient)).items.map(item => item.fullPath);
|
|
40
36
|
}
|
|
41
37
|
catch (err) {
|
|
42
|
-
|
|
38
|
+
complete(err);
|
|
43
39
|
return;
|
|
44
40
|
}
|
|
45
41
|
}
|
|
@@ -50,7 +46,7 @@ function upload(credential, service = 'gcp') {
|
|
|
50
46
|
const bucketAcl = admin.publicRead ? "publicRead" : admin.acl;
|
|
51
47
|
const response = BUCKET_RESPONSE[bucketKey = getBucketKey(credential, bucket, bucketAcl)] ||= client_1.createBucketV2.call(this, credential, bucket, bucketAcl, configBucket?.create);
|
|
52
48
|
if (!await response) {
|
|
53
|
-
|
|
49
|
+
complete(null);
|
|
54
50
|
return;
|
|
55
51
|
}
|
|
56
52
|
BUCKET_SESSION.add(bucket);
|
|
@@ -59,7 +55,7 @@ function upload(credential, service = 'gcp') {
|
|
|
59
55
|
if (configBucket) {
|
|
60
56
|
let { cors, lifecycle, retentionPolicy } = configBucket;
|
|
61
57
|
const commandMessage = (feature, message) => {
|
|
62
|
-
this.formatMessage(64, service, ["Bucket configured" + ` (${feature})`, bucket], message,
|
|
58
|
+
this.formatMessage(64, service, ["Bucket configured" + ` (${feature})`, bucket], message, Cloud.optionsLogMessage('COMMAND'));
|
|
63
59
|
};
|
|
64
60
|
if (typeof retentionPolicy === 'number') {
|
|
65
61
|
(retentionPolicy === 0 ? bucketClient.removeRetentionPeriod() : bucketClient.setRetentionPeriod(retentionPolicy))
|
|
@@ -78,16 +74,11 @@ function upload(credential, service = 'gcp') {
|
|
|
78
74
|
}
|
|
79
75
|
if ((0, types_1.isObject)(lifecycle)) {
|
|
80
76
|
let append = false;
|
|
81
|
-
if (
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
const arg = lifecycle[lifecycle.length - 1];
|
|
87
|
-
if (typeof arg === 'boolean') {
|
|
88
|
-
append = arg;
|
|
89
|
-
lifecycle = lifecycle.slice(0, lifecycle.length - 1);
|
|
90
|
-
}
|
|
77
|
+
if ((0, types_1.isArray)(lifecycle)) {
|
|
78
|
+
const arg = lifecycle.at(-1);
|
|
79
|
+
if (typeof arg === 'boolean') {
|
|
80
|
+
append = arg;
|
|
81
|
+
lifecycle = lifecycle.slice(0, lifecycle.length - 1);
|
|
91
82
|
}
|
|
92
83
|
}
|
|
93
84
|
bucketClient.addLifecycleRule(lifecycle, { append })
|
|
@@ -116,7 +107,7 @@ function upload(credential, service = 'gcp') {
|
|
|
116
107
|
break;
|
|
117
108
|
}
|
|
118
109
|
}
|
|
119
|
-
const name =
|
|
110
|
+
const name = Cloud.joinPath(pathname, filename, true);
|
|
120
111
|
if (firebase) {
|
|
121
112
|
exists = listFiles.includes(name);
|
|
122
113
|
}
|
|
@@ -125,7 +116,7 @@ function upload(credential, service = 'gcp') {
|
|
|
125
116
|
}
|
|
126
117
|
} while (exists && ++i);
|
|
127
118
|
if (i > 0) {
|
|
128
|
-
this.formatMessage(64, service, ["File renamed", current], filename,
|
|
119
|
+
this.formatMessage(64, service, ["File renamed", current], filename, Cloud.optionsLogMessage('WARN'));
|
|
129
120
|
}
|
|
130
121
|
}
|
|
131
122
|
const getURL = (value) => endpoint ? Cloud.joinPath(endpoint, value) : Cloud.joinPath("https://storage.googleapis.com", bucket, value);
|
|
@@ -160,7 +151,7 @@ function upload(credential, service = 'gcp') {
|
|
|
160
151
|
}
|
|
161
152
|
}
|
|
162
153
|
catch (err) {
|
|
163
|
-
|
|
154
|
+
complete(err);
|
|
164
155
|
return;
|
|
165
156
|
}
|
|
166
157
|
}
|
|
@@ -193,7 +184,7 @@ function upload(credential, service = 'gcp') {
|
|
|
193
184
|
}
|
|
194
185
|
}
|
|
195
186
|
catch (err) {
|
|
196
|
-
|
|
187
|
+
complete(err);
|
|
197
188
|
return;
|
|
198
189
|
}
|
|
199
190
|
}
|
|
@@ -220,11 +211,11 @@ function upload(credential, service = 'gcp') {
|
|
|
220
211
|
const first = i === 0;
|
|
221
212
|
if (this.aborted) {
|
|
222
213
|
if (first) {
|
|
223
|
-
|
|
214
|
+
complete((0, types_1.createAbortError)());
|
|
224
215
|
}
|
|
225
216
|
return;
|
|
226
217
|
}
|
|
227
|
-
const destination =
|
|
218
|
+
const destination = Cloud.joinPath(pathname, Key[i], true);
|
|
228
219
|
if (firebase) {
|
|
229
220
|
const { ref, uploadBytes } = require('@firebase/storage');
|
|
230
221
|
const params = { ...options };
|
|
@@ -234,7 +225,7 @@ function upload(credential, service = 'gcp') {
|
|
|
234
225
|
Body[0] = fs.readFileSync(localUri);
|
|
235
226
|
}
|
|
236
227
|
catch (err) {
|
|
237
|
-
|
|
228
|
+
complete(err);
|
|
238
229
|
return;
|
|
239
230
|
}
|
|
240
231
|
}
|
|
@@ -249,14 +240,14 @@ function upload(credential, service = 'gcp') {
|
|
|
249
240
|
uploadBytes(ref(bucketClient, destination), Body[i], params)
|
|
250
241
|
.then(() => {
|
|
251
242
|
const url = getURL(destination);
|
|
252
|
-
this.formatMessage(64, service, "Upload success", url,
|
|
243
|
+
this.formatMessage(64, service, "Upload success", url, Cloud.optionsLogMessage('UPLOAD'));
|
|
253
244
|
if (first) {
|
|
254
|
-
|
|
245
|
+
complete(null, url);
|
|
255
246
|
}
|
|
256
247
|
})
|
|
257
248
|
.catch((err) => {
|
|
258
249
|
if (first) {
|
|
259
|
-
|
|
250
|
+
complete(err);
|
|
260
251
|
}
|
|
261
252
|
else {
|
|
262
253
|
addLog(err);
|
|
@@ -323,14 +314,24 @@ function upload(credential, service = 'gcp') {
|
|
|
323
314
|
uploadClient
|
|
324
315
|
.then(async () => {
|
|
325
316
|
const url = getURL(file.name);
|
|
326
|
-
this.formatMessage(64, service, "Upload success", url,
|
|
317
|
+
this.formatMessage(64, service, "Upload success", url, Cloud.optionsLogMessage('UPLOAD'));
|
|
327
318
|
if (first) {
|
|
328
|
-
|
|
319
|
+
const copyTo = (0, util_1.intoArray)(data.upload.copyObject);
|
|
320
|
+
if (copyTo) {
|
|
321
|
+
const tasks = [];
|
|
322
|
+
for (const { bucket: bucketName, pathname: pathObject = pathname, filename: fileObject, options: copyOptions } of copyTo) {
|
|
323
|
+
const keyObject = fileObject ? Cloud.joinPath(pathObject, fileObject, true) : file.name;
|
|
324
|
+
tasks.push(client_1.copyObject
|
|
325
|
+
.call(this, credential, bucket, file, bucketName, keyObject, copyOptions)
|
|
326
|
+
.catch(addLog));
|
|
327
|
+
}
|
|
328
|
+
void Promise.all(tasks);
|
|
329
|
+
}
|
|
330
|
+
complete(null, url);
|
|
329
331
|
}
|
|
330
332
|
if (Path[i]) {
|
|
331
|
-
file.setMetadata(params)
|
|
332
|
-
addLog
|
|
333
|
-
});
|
|
333
|
+
file.setMetadata(params)
|
|
334
|
+
.catch(addLog);
|
|
334
335
|
}
|
|
335
336
|
if (ACL) {
|
|
336
337
|
void client_1.setPredefinedAcl.call(this, file.acl, ACL, file.bucket, file.name, revoke || (await file.getMetadata())[0], true);
|
|
@@ -338,7 +339,7 @@ function upload(credential, service = 'gcp') {
|
|
|
338
339
|
})
|
|
339
340
|
.catch((err) => {
|
|
340
341
|
if (first) {
|
|
341
|
-
|
|
342
|
+
complete(err);
|
|
342
343
|
}
|
|
343
344
|
else {
|
|
344
345
|
addLog(err);
|