@pi-r/azure 0.10.2 → 0.11.0

package/client/index.js

@@ -10,6 +10,7 @@ exports.setBucketWebsite = setBucketWebsite;
 exports.deleteObjects = deleteObjects;
 exports.deleteObjectsV2 = deleteObjectsV2;
 exports.deleteObjectsV3 = deleteObjectsV3;
+exports.copyObject = copyObject;
 exports.executeQuery = executeQuery;
 exports.executeBatchQuery = executeBatchQuery;
 const path = require("node:path");
@@ -19,11 +20,15 @@ const identity = require("@azure/identity");
 const Cloud = require("@e-mc/cloud");
 const types_1 = require("@e-mc/types");
 const util_1 = require("@e-mc/cloud/util");
-function validateStorage(credential) {
+function isEnvironmentCredential() {
     const env = process.env;
-    if (credential.accountName && (credential.accountKey || credential.accountSas) || credential.connectionString || credential.sharedAccessSignature || env.AZURE_TENANT_ID && env.AZURE_CLIENT_ID && env.AZURE_CLIENT_SECRET) {
+    return !!(env.AZURE_TENANT_ID && env.AZURE_CLIENT_ID && (env.AZURE_CLIENT_SECRET || env.AZURE_CLIENT_CERTIFICATE_PATH));
+}
+function validateStorage(credential) {
+    if (credential.accountName && (credential.accountKey || credential.accountSas) || credential.connectionString || credential.sharedAccessSignature || isEnvironmentCredential()) {
         return true;
     }
+    const env = process.env;
     return Cloud.enabled("process.env.apply") && !!((credential.connectionString = env.AZURE_STORAGE_CONNECTION_STRING) || (credential.sharedAccessSignature = env.AZURE_STORAGE_SAS_TOKEN) || (credential.accountName = env.AZURE_STORAGE_ACCOUNT) && (credential.accountKey = env.AZURE_STORAGE_KEY));
 }
 function validateDatabase(credential, data) {
@@ -59,6 +64,9 @@ function createStorageClient(credential) {
     else if (accountName && accountKey) {
         serviceCredential = new storage.StorageSharedKeyCredential(accountName, accountKey);
     }
+    else if (isEnvironmentCredential()) {
+        serviceCredential = new identity.EnvironmentCredential();
+    }
     else {
         serviceCredential = new identity.DefaultAzureCredential(credential.defaultAzureOptions);
     }
@@ -73,12 +81,15 @@ function createDatabaseClient(credential) {
         delete credential.tenantId;
         delete credential.clientId;
     }
+    else if (isEnvironmentCredential()) {
+        credential.aadCredentials = new identity.EnvironmentCredential();
+    }
     return new cosmos.CosmosClient(credential);
 }
-async function createBucket(credential, bucket, publicRead, service = 'azure') {
+async function createBucket(credential, bucket, publicRead, service = "azure") {
     return createBucketV2.call(this, credential, bucket, publicRead ? 'blob' : 'container', undefined, service);
 }
-async function createBucketV2(credential, bucket, access, options, service = 'azure') {
+async function createBucketV2(credential, bucket, access, options, service = "azure") {
     switch (access) {
         case 'blob':
         case 'container':
@@ -87,10 +98,10 @@ async function createBucketV2(credential, bucket, access, options, service = 'az
             access = undefined;
             break;
     }
-    const containerClient = createStorageClient.call(this, credential).getContainerClient(bucket);
+    const containerClient = createStorageClient(credential).getContainerClient(bucket);
     const setAccessPolicy = async () => {
         return containerClient.setAccessPolicy(access).catch((err) => {
-            this.formatMessage(64, service, ["Unable to configure container", bucket], err, { ...Cloud.LOG_CLOUD_WARN });
+            this.formatMessage(64, service, ["Unable to configure container", bucket], err, Cloud.optionsLogMessage('WARN'));
         });
     };
     if (await containerClient.exists().catch(() => false)) {
@@ -108,25 +119,25 @@ async function createBucketV2(credential, bucket, access, options, service = 'az
         .then(response => {
         const { requestId, errorCode } = response;
         if (errorCode) {
-            this.formatMessage(64, service, ['Container created with errors', bucket], (0, types_1.errorMessage)("Error code", errorCode), { ...Cloud.LOG_CLOUD_WARN });
+            this.formatMessage(64, service, ['Container created with errors', bucket], (0, types_1.errorMessage)("Error code", errorCode), Cloud.optionsLogMessage('WARN'));
         }
         else {
-            this.formatMessage(64, service, ['Container created', bucket], requestId, { ...Cloud.LOG_CLOUD_COMMAND });
+            this.formatMessage(64, service, ['Container created', bucket], requestId, Cloud.optionsLogMessage('COMMAND'));
         }
         return true;
     })
         .catch(async (err) => {
-        if (err instanceof Error && err.code === 'ContainerAlreadyExists') {
+        if ((0, types_1.isErrorCode)(err, 'ContainerAlreadyExists')) {
             if (access) {
                 await setAccessPolicy();
             }
             return true;
         }
-        this.formatFail(64, service, ["Unable to create container", bucket], err, { ...Cloud.LOG_CLOUD_FAIL });
+        this.formatFail(64, service, ["Unable to create container", bucket], err, Cloud.optionsLogMessage('FAIL'));
         return false;
     });
 }
-async function setBucketWebsite(credential, bucket, options, service = 'azure') {
+async function setBucketWebsite(credential, bucket, options, service = "azure") {
     const staticWebsite = { enabled: true };
     const { indexPage, indexPath, errorPath } = options;
     if ((0, types_1.isString)(indexPage)) {
@@ -138,19 +149,19 @@ async function setBucketWebsite(credential, bucket, options, service = 'azure')
     if ((0, types_1.isString)(errorPath)) {
         staticWebsite.errorDocument404Path = Cloud.joinPath(bucket, errorPath);
     }
-    return createStorageClient.call(this, credential).setProperties({ staticWebsite })
+    return createStorageClient(credential).setProperties({ staticWebsite })
         .then(response => {
         const { requestId, errorCode } = response;
         if (errorCode) {
-            this.formatMessage(64, service, ["Bucket website configured" + ' with errors', bucket], (0, types_1.errorMessage)("Error code", errorCode), { ...Cloud.LOG_CLOUD_WARN });
+            this.formatMessage(64, service, ["Bucket website configured" + ' with errors', bucket], (0, types_1.errorMessage)("Error code", errorCode), Cloud.optionsLogMessage('WARN'));
         }
         else {
-            this.formatMessage(64, service, ["Bucket website configured", bucket], requestId, { ...Cloud.LOG_CLOUD_COMMAND });
+            this.formatMessage(64, service, ["Bucket website configured", bucket], requestId, Cloud.optionsLogMessage('COMMAND'));
         }
         return true;
     })
         .catch((err) => {
-        this.formatFail(64, service, ["Unable to set bucket website", bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
+        this.formatFail(64, service, ["Unable to set bucket website", bucket], err, Cloud.optionsLogMessage('FAIL', { fatal: false }));
         return false;
     });
 }
@@ -160,8 +171,8 @@ async function deleteObjects(credential, bucket, service, sdk, recursive = true)
 async function deleteObjectsV2(credential, bucket, recursive = true, service) {
     return deleteObjectsV3.call(this, credential, bucket, { recursive }, service);
 }
-async function deleteObjectsV3(credential, bucket, options = {}, service = 'azure') {
-    const containerClient = createStorageClient.call(this, credential).getContainerClient(bucket);
+async function deleteObjectsV3(credential, bucket, options = {}, service = "azure") {
+    const containerClient = createStorageClient(credential).getContainerClient(bucket);
     if (await containerClient.exists().catch(() => false)) {
         try {
             options = { ...options };
@@ -170,6 +181,7 @@ async function deleteObjectsV3(credential, bucket, options = {}, service = 'azur
                 recursive = options.recursive;
                 delete options.recursive;
             }
+            options.abortSignal = this.signal;
             options.includeUncommitedBlobs ??= true;
             const tasks = [];
             let fileCount = 0;
@@ -182,14 +194,32 @@ async function deleteObjectsV3(credential, bucket, options = {}, service = 'azur
             }
             fileCount = tasks.length;
             await this.allSettled(tasks, ["Unable to delete blob", bucket], 64).then(() => {
-                this.formatMessage(64, service, ['Container emptied' + (recursive ? ' (recursive)' : ''), bucket], fileCount + ' files', { ...Cloud.LOG_CLOUD_COMMAND });
+                this.formatMessage(64, service, ['Container emptied' + (recursive ? ' (recursive)' : ''), bucket], fileCount + ' files', Cloud.optionsLogMessage('COMMAND'));
             });
         }
         catch (err) {
-            this.formatFail(64, service, ["Unable to list bucket", bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
+            this.formatFail(64, service, ["Unable to list bucket", bucket], err, Cloud.optionsLogMessage('FAIL', { fatal: false }));
         }
     }
 }
+async function copyObject(credential, bucketSource, keySource, bucket, key, options = {}, service = "azure") {
+    const storageClient = createStorageClient(credential);
+    const client = storageClient.getContainerClient(bucketSource).getBlobClient(keySource);
+    const destination = storageClient.getContainerClient(bucket).getBlobClient(key);
+    const poller = await destination.beginCopyFromURL(client.url, { ...options, abortSignal: this.signal });
+    return poller.pollUntilDone()
+        .then(response => {
+        if (response.errorCode) {
+            this.formatMessage(64, service, ["Copy success" + ' with errors', bucket], (0, types_1.errorMessage)("Error code", response.errorCode, bucketSource), Cloud.optionsLogMessage('WARN'));
+        }
+        else {
+            this.formatMessage(64, service, ["Copy success", bucket], bucketSource + '/' + keySource, Cloud.optionsLogMessage('COMMAND'));
+        }
+    })
+        .catch((err) => {
+        this.formatFail(64, service, ["Copy failed", bucket + '/' + key], err, Cloud.optionsLogMessage('FAIL', { fatal: false }));
+    });
+}
 async function executeQuery(credential, data, sessionKey) {
     return (await executeBatchQuery.call(this, credential, [data], sessionKey))[0] || [];
 }
@@ -200,7 +230,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
     const cacheValue = { value: this.valueOfKey(credential, 'cache'), sessionKey };
     let client;
     const createClient = (name, table) => {
-        client ||= createDatabaseClient.call(this, credential);
+        client ||= createDatabaseClient(credential);
         return client.database(name).container(table);
     };
     const attachSignal = (opts) => {

package/download/index.js

@@ -2,44 +2,55 @@
 const path = require("node:path");
 const Cloud = require("@e-mc/cloud");
 const types_1 = require("@e-mc/types");
+const util_1 = require("@e-mc/cloud/util");
 const client_1 = require("@pi-r/azure");
-function download(credential, service = 'azure') {
-    const blobServiceClient = client_1.createStorageClient.call(this, credential);
+function download(credential, service = "azure") {
+    const blobServiceClient = (0, client_1.createStorageClient)(credential);
     return (data, callback) => {
-        const { bucket: Bucket, download: target } = data;
-        const Key = target.keyname || target.filename;
-        if (!Bucket || !Key) {
-            callback((0, types_1.errorValue)('Missing property', !Bucket ? 'Container' : 'Name'));
+        const { bucket, download: target } = data;
+        const key = target.keyname || target.filename;
+        if (!bucket || !key) {
+            callback((0, types_1.errorValue)('Missing property', !bucket ? 'Container' : 'Name'));
             return;
         }
         const { flags = 0 } = target;
-        const containerClient = blobServiceClient.getContainerClient(Bucket);
-        const blobClient = containerClient.getBlockBlobClient(Key);
+        const containerClient = blobServiceClient.getContainerClient(bucket);
+        const blobClient = containerClient.getBlockBlobClient(key);
         const blockSize = (flags & 4) && (0, types_1.alignSize)(target.chunkSize, 1024) || 0;
         const tempDir = blockSize >= 2097152000 ? (0, types_1.getTempDir)(true, service) : null;
         let diskClient, destination;
         if (tempDir) {
-            destination = path.join(tempDir, path.basename(target.filename || Key));
+            destination = path.join(tempDir, path.basename(target.filename || key));
             diskClient = blobClient.downloadToFile(destination, 0, undefined, { snapshot: target.versionId, abortSignal: this.signal });
         }
         else {
             diskClient = blobClient.downloadToBuffer(0, 0, { ...target.options, blockSize, concurrency: blockSize > 0 ? target.chunkLimit : undefined, abortSignal: this.signal });
         }
         diskClient
-            .then((value) => {
+            .then(async (value) => {
             callback(null, destination || value);
+            const copyTo = (0, util_1.intoArray)(target.copyObject);
+            if (copyTo) {
+                const tasks = [];
+                for (const { bucket: bucketName, pathname, filename, options } of copyTo) {
+                    const keyObject = filename ? Cloud.joinPath(pathname, filename, true) : key;
+                    tasks.push(client_1.copyObject
+                        .call(this, credential, bucket, key, bucketName, keyObject, options)
+                        .catch((0, util_1.createErrorHandler)(this, service, bucket)));
+                }
+                await Promise.all(tasks);
+            }
             const deleteObject = target.deleteObject;
             if (deleteObject) {
-                const location = Cloud.joinPath(Bucket, Key);
-                containerClient.delete((0, types_1.isPlainObject)(deleteObject) ? (deleteObject.abortSignal = this.signal, deleteObject) : { abortSignal: this.signal })
+                const location = Cloud.joinPath(bucket, key);
+                containerClient.delete((0, types_1.isPlainObject)(deleteObject) ? { ...deleteObject, abortSignal: this.signal } : { abortSignal: this.signal })
                     .then(() => {
-                    this.formatMessage(64, service, "Delete success", location, { ...Cloud.LOG_CLOUD_DELETE });
+                    this.formatMessage(64, service, "Delete success", location, Cloud.optionsLogMessage('DELETE'));
                 })
                     .catch((err) => {
-                    if (err instanceof Error && err.code === 'BlobNotFound') {
-                        return;
+                    if (!(0, types_1.isErrorCode)(err, 'BlobNotFound')) {
+                        this.formatFail(64, service, ["Delete failed", location], err, Cloud.optionsLogMessage('FAIL', { fatal: !!target.active }));
                     }
-                    this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active });
                 });
             }
         })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-r/azure",
-  "version": "0.10.2",
+  "version": "0.11.0",
   "description": "Azure cloud functions for E-mc.",
   "main": "client/index.js",
   "publishConfig": {
@@ -19,11 +19,11 @@
   "license": "MIT",
   "homepage": "https://github.com/anpham6/pi-r#readme",
   "dependencies": {
-    "@e-mc/cloud": "^0.12.5",
-    "@e-mc/module": "^0.12.5",
-    "@e-mc/types": "^0.12.5",
-    "@azure/cosmos": "^4.5.0",
+    "@e-mc/cloud": "^0.13.0",
+    "@e-mc/module": "^0.13.0",
+    "@e-mc/types": "^0.13.0",
+    "@azure/cosmos": "^4.6.0",
     "@azure/identity": "*",
-    "@azure/storage-blob": "^12.28.0"
+    "@azure/storage-blob": "^12.29.1"
   }
 }

package/upload/index.js

@@ -9,43 +9,33 @@ const client_1 = require("@pi-r/azure");
 const BUCKET_SESSION = new Set();
 const BUCKET_RESPONSE = {};
 const getBucketKey = (credential, bucket, acl = '') => Cloud.asString(credential, true) + bucket + '_' + acl;
-function upload(credential, service = 'azure') {
-    const blobServiceClient = client_1.createStorageClient.call(this, credential);
+function upload(credential, service = "azure") {
+    const blobServiceClient = (0, client_1.createStorageClient)(credential);
     return async (data, callback) => {
         const { bucket, localUri } = data;
-        const { pathname = '', flags = 0, fileGroup, contentType, metadata, tags, endpoint, admin = {}, overwrite, options } = data.upload;
+        const { pathname, flags = 0, fileGroup, contentType, metadata, tags, endpoint, admin = {}, overwrite, options } = data.upload;
         const containerClient = blobServiceClient.getContainerClient(bucket);
         let filename = data.upload.filename || path.basename(localUri), bucketKey;
-        const cleanup = () => {
+        const complete = (err, url) => {
             BUCKET_SESSION.delete(bucket);
             if (bucketKey) {
                 delete BUCKET_RESPONSE[bucketKey];
             }
+            callback(err, url);
         };
-        const errorResponse = (err) => {
-            cleanup();
-            callback(err);
-        };
-        const addLog = (err) => {
-            if (err instanceof Error) {
-                this.addLog(this.statusType.WARN, err, service, bucket);
-            }
-        };
+        const addLog = (0, util_1.createErrorHandler)(this, bucket, service);
         const configBucket = admin.configBucket;
         if (!BUCKET_SESSION.has(bucket)) {
             const bucketAcl = admin.publicRead ? 'blob' : admin.acl;
             const response = BUCKET_RESPONSE[bucketKey = getBucketKey(credential, bucket, bucketAcl)] ||= client_1.createBucketV2.call(this, credential, bucket, bucketAcl, configBucket?.create);
             if (!await response) {
-                errorResponse(null);
+                complete(null);
                 return;
             }
             BUCKET_SESSION.add(bucket);
         }
-        let retentionPolicy = configBucket?.retentionPolicy;
+        const retentionPolicy = (0, util_1.intoArray)(configBucket?.retentionPolicy);
         if (retentionPolicy) {
-            if (!Array.isArray(retentionPolicy)) {
-                retentionPolicy = [retentionPolicy];
-            }
             try {
                 if (this.hasCoerce(service, 'options')) {
                     for (const policy of retentionPolicy) {
@@ -58,7 +48,7 @@ function upload(credential, service = 'azure') {
             containerClient.setAccessPolicy('container', retentionPolicy)
                 .then(() => {
                 const { expiresOn, startsOn, permissions } = retentionPolicy[0].accessPolicy;
-                this.formatMessage(64, service, ["Bucket configured" + ' (retention policy)', bucket], expiresOn ? expiresOn.toLocaleString() + ' (expires)' : startsOn ? startsOn.toLocaleString() + ' (starts)' : permissions, { ...Cloud.LOG_CLOUD_COMMAND });
+                this.formatMessage(64, service, ["Bucket configured" + ' (retention policy)', bucket], expiresOn ? expiresOn.toLocaleString() + ' (expires)' : startsOn ? startsOn.toLocaleString() + ' (starts)' : permissions, Cloud.optionsLogMessage('COMMAND'));
             })
                 .catch(addLog);
         }
@@ -74,7 +64,7 @@ function upload(credential, service = 'azure') {
                             break;
                         }
                     }
-                    const name = pathname ? Cloud.joinPath(pathname, filename) : filename;
+                    const name = Cloud.joinPath(pathname, filename, true);
                     for await (const blob of containerClient.listBlobsFlat({ includeUncommitedBlobs: true })) {
                         if (blob.name === name) {
                             exists = true;
@@ -83,11 +73,11 @@ function upload(credential, service = 'azure') {
                     }
                 } while (exists && ++i);
                 if (i > 0) {
-                    this.formatMessage(64, service, ["File renamed", current], filename, { ...Cloud.LOG_CLOUD_WARN });
+                    this.formatMessage(64, service, ["File renamed", current], filename, Cloud.optionsLogMessage('WARN'));
                 }
             }
             catch (err) {
-                errorResponse(err);
+                complete(err);
                 return;
             }
         }
@@ -112,7 +102,7 @@ function upload(credential, service = 'azure') {
                 }
             }
             catch (err) {
-                errorResponse(err);
+                complete(err);
                 return;
             }
         }
@@ -140,11 +130,11 @@ function upload(credential, service = 'azure') {
             const first = i === 0;
             if (this.aborted) {
                 if (first) {
-                    errorResponse((0, types_1.createAbortError)());
+                    complete((0, types_1.createAbortError)());
                 }
                 return;
             }
-            const blobName = Cloud.joinPath(pathname, Key[i]);
+            const blobName = Cloud.joinPath(pathname, Key[i], true);
             const params = { ...options, abortSignal: this.signal };
             const headers = params.blobHTTPHeaders ||= {};
             if (first) {
@@ -181,15 +171,25 @@ function upload(credential, service = 'azure') {
                 .then(result => {
                 let requestUrl = result._response.request.url;
                 const url = endpoint ? Cloud.joinPath(endpoint, blobName) : Cloud.isFile(requestUrl = decodeURIComponent(requestUrl), 'http/s') ? requestUrl : Cloud.joinPath(`https://${credential.accountName}.blob.core.windows.net`, bucket, blobName);
-                this.formatMessage(64, service, "Upload success", url, { ...Cloud.LOG_CLOUD_UPLOAD });
+                this.formatMessage(64, service, "Upload success", url, Cloud.optionsLogMessage('UPLOAD'));
                 if (first) {
-                    cleanup();
-                    callback(null, url);
+                    const copyTo = (0, util_1.intoArray)(data.upload.copyObject);
+                    if (copyTo) {
+                        const tasks = [];
+                        for (const { bucket: bucketName, pathname: pathObject = pathname, filename: fileObject, options: copyOptions } of copyTo) {
+                            const keyObject = fileObject ? Cloud.joinPath(pathObject, fileObject, true) : blobName;
+                            tasks.push(client_1.copyObject
+                                .call(this, credential, bucket, blobName, bucketName, keyObject, copyOptions)
+                                .catch(addLog));
+                        }
+                        void Promise.all(tasks);
+                    }
+                    complete(null, url);
                 }
             })
                 .catch((err) => {
                 if (first) {
-                    errorResponse(err);
+                    complete(err);
                 }
                 else {
                     addLog(err);