@pi-r/azure 0.6.6 → 0.7.0

package/README.md

@@ -1,10 +1,7 @@
-# @pi-r/azure
+### @pi-r/azure
 
-## Documentation
+https://e-mc.readthedocs.io/en/latest/cloud/azure.html
 
-- [E-mc](https://e-mc.readthedocs.io/en/latest/cloud/azure.html)
-- [squared](https://squared.readthedocs.io)
-
-## LICENSE
+### LICENSE
 
 MIT

package/client/index.js

@@ -1,5 +1,5 @@
 "use strict";
-exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = void 0;
+exports.CLOUD_DOWNLOAD_CHUNK = exports.CLOUD_UPLOAD_CHUNK = exports.CLOUD_UPLOAD_STREAM = exports.executeBatchQuery = exports.executeQuery = exports.deleteObjectsV2 = exports.deleteObjects = exports.setBucketWebsite = exports.createBucketV2 = exports.createBucket = exports.createDatabaseClient = exports.createStorageClient = exports.validateDatabase = exports.validateStorage = void 0;
 const path = require("path");
 const storage = require("@azure/storage-blob");
 const cosmos = require("@azure/cosmos");
@@ -35,7 +35,7 @@ function createStorageClient(credential) {
             }
         }
         else {
-            credential.accountName || (credential.accountName = /^https:\/\/([a-z\d]+)\./.exec(sharedAccessSignature)?.[1]);
+            credential.accountName ||= /^https:\/\/([a-z\d]+)\./.exec(sharedAccessSignature)?.[1];
         }
         return new storage.BlobServiceClient(sharedAccessSignature);
     }
@@ -68,7 +68,7 @@ async function createBucketV2(credential, bucket, access, options, service = 'az
             break;
     }
     const containerClient = createStorageClient.call(this, credential).getContainerClient(bucket);
-    const setAccessPolicy = async () => containerClient.setAccessPolicy(access).catch(err => this.formatMessage(64, service, ["Unable to configure container", bucket], err, { ...Cloud.LOG_CLOUD_WARN }));
+    const setAccessPolicy = async () => containerClient.setAccessPolicy(access).catch((err) => this.formatMessage(64, service, ["Unable to configure container", bucket], err, { ...Cloud.LOG_CLOUD_WARN }));
     if (await containerClient.exists().catch(() => false)) {
         if (access) {
             await setAccessPolicy();
@@ -119,15 +119,15 @@ async function setBucketWebsite(credential, bucket, options, service = 'azure')
         .then(response => {
         const { requestId, errorCode } = response;
         if (errorCode) {
-            this.formatMessage(64, service, ["Bucket configured" + ' with errors', bucket], (0, types_1.errorMessage)("Error code", errorCode), { ...Cloud.LOG_CLOUD_WARN });
+            this.formatMessage(64, service, ["Bucket website configured" + ' with errors', bucket], (0, types_1.errorMessage)("Error code", errorCode), { ...Cloud.LOG_CLOUD_WARN });
         }
         else {
-            this.formatMessage(64, service, ["Bucket configured", bucket], requestId, { ...Cloud.LOG_CLOUD_COMMAND });
+            this.formatMessage(64, service, ["Bucket website configured", bucket], requestId, { ...Cloud.LOG_CLOUD_COMMAND });
         }
         return true;
     })
-        .catch(err => {
-        this.formatFail(64, service, ["Unable to configure bucket", bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
+        .catch((err) => {
+        this.formatFail(64, service, ["Unable to set bucket website", bucket], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: false });
         return false;
     });
 }
@@ -143,11 +143,12 @@ async function deleteObjectsV2(credential, bucket, recursive = true, service = '
             const tasks = [];
             let fileCount = 0;
             for await (const blob of containerClient.listBlobsFlat({ includeUncommitedBlobs: true })) {
-                if (recursive || !blob.name.includes('/')) {
-                    tasks.push(containerClient.deleteBlob(blob.name, { versionId: blob.versionId }).catch(() => {
-                        --fileCount;
-                    }));
+                if (!recursive && blob.name.includes('/')) {
+                    continue;
                 }
+                tasks.push(containerClient.deleteBlob(blob.name, { versionId: blob.versionId }).catch(() => {
+                    --fileCount;
+                }));
             }
             fileCount = tasks.length;
             return this.allSettled(tasks, ["Unable to delete blob", bucket], 64).then(() => {
@@ -171,7 +172,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
     const cacheValue = { value: this.valueOfKey(credential, 'cache'), sessionKey };
     let client;
     const createClient = (name, table) => {
-        client || (client = createDatabaseClient.call(this, credential));
+        client ||= createDatabaseClient.call(this, credential);
         return client.database(name).container(table);
     };
     const closeClient = () => client?.dispose();
@@ -192,7 +193,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
         let rows, queryString = caching && ignoreCache !== true ? name + '_' + table + '_' : '';
         if (storedProcedureId && params) {
             const { options } = item;
-            if (queryString && (rows = getCache(queryString += Cloud.asString(partitionKey, true) + '_' + storedProcedureId + Cloud.asString(params, true) + Cloud.asString(options, true)))) {
+            if (queryString && (rows = getCache(queryString += (partitionKey ? Cloud.asString(partitionKey, true) : '') + '_' + storedProcedureId + '_' + Cloud.asString(params, true) + Cloud.asString(options, true)))) {
                 result[i] = rows;
                 continue;
             }
@@ -204,7 +205,7 @@ async function executeBatchQuery(credential, batch, sessionKey) {
         else if (id) {
             const { options } = item;
             if (queryString) {
-                queryString += Cloud.asString(partitionKey, true) + '_' + id + Cloud.asString(options, true);
+                queryString += (partitionKey ? Cloud.asString(partitionKey, true) : '') + '_' + id + '_' + Cloud.asString(options, true);
                 if (!update && (rows = getCache(queryString))) {
                     result[i] = rows;
                     continue;
@@ -219,10 +220,10 @@ async function executeBatchQuery(credential, batch, sessionKey) {
         else {
             let { options } = item;
             if (partitionKey) {
-                (options || (options = {})).partitionKey = partitionKey;
+                (options ||= {}).partitionKey = partitionKey;
             }
             if (limit > 0) {
-                (options || (options = {})).maxItemCount = limit;
+                (options ||= {}).maxItemCount = limit;
             }
             if (query || (0, types_1.isPlainObject)(options)) {
                 if (queryString && (rows = getCache(queryString += (query ? Cloud.asString(query, true) : '') + Cloud.asString(options, true)))) {
@@ -243,3 +244,6 @@ async function executeBatchQuery(credential, batch, sessionKey) {
     return result;
 }
 exports.executeBatchQuery = executeBatchQuery;
+exports.CLOUD_UPLOAD_STREAM = true;
+exports.CLOUD_UPLOAD_CHUNK = true;
+exports.CLOUD_DOWNLOAD_CHUNK = true;

package/download/index.js

@@ -1,26 +1,42 @@
 "use strict";
 const types_1 = require("@e-mc/types");
-const Module = require("@e-mc/module");
+const path = require("path");
 const Cloud = require("@e-mc/cloud");
 const client_1 = require("@pi-r/azure");
 module.exports = function download(credential, service = 'azure') {
     const blobServiceClient = client_1.createStorageClient.call(this, credential);
     return (data, callback) => {
         const { bucket: Bucket, download: target } = data;
-        const Key = target.filename;
+        const Key = target.keyname || target.filename;
         if (!Bucket || !Key) {
             callback((0, types_1.errorValue)('Missing property', !Bucket ? 'Container' : 'Name'));
             return;
         }
-        const location = Module.joinPath(Bucket, Key);
-        const blobClient = blobServiceClient.getContainerClient(Bucket);
-        blobClient.getBlockBlobClient(Key).downloadToBuffer()
-            .then(value => {
-            callback(null, value);
+        let tempDir;
+        const downloadResponse = (err, url) => {
+            callback(err, url);
+            if (tempDir) {
+                queueMicrotask(() => Cloud.removeDir(tempDir));
+            }
+        };
+        const { flags = 0 } = target;
+        const containerClient = blobServiceClient.getContainerClient(Bucket);
+        const blobClient = containerClient.getBlockBlobClient(Key);
+        const blockSize = (flags & 4) && (0, types_1.alignSize)(target.chunkSize, 1024) || 0;
+        let diskClient, destination;
+        if (blockSize >= 2097152000 && (tempDir = this.getTempDir({ uuidDir: true }))) {
+            diskClient = blobClient.downloadToFile(destination = path.join(tempDir, path.basename(target.filename || Key)), 0, undefined, { snapshot: target.versionId, abortSignal: this.signal });
+        }
+        (diskClient || blobClient.downloadToBuffer(0, 0, { ...target.options, blockSize, concurrency: blockSize > 0 ? target.chunkLimit : undefined, abortSignal: this.signal }))
+            .then((value) => {
+            downloadResponse(null, diskClient && destination || value);
             const deleteObject = target.deleteObject;
             if (deleteObject) {
-                blobClient.delete((0, types_1.isPlainObject)(deleteObject) ? (deleteObject.abortSignal = this.signal, deleteObject) : { abortSignal: this.signal })
-                    .then(() => this.formatMessage(64, service, "Delete success", location, { ...Cloud.LOG_CLOUD_DELETE }))
+                const location = Cloud.joinPath(Bucket, Key);
+                containerClient.delete((0, types_1.isPlainObject)(deleteObject) ? (deleteObject.abortSignal = this.signal, deleteObject) : { abortSignal: this.signal })
+                    .then(() => {
+                    this.formatMessage(64, service, "Delete success", location, { ...Cloud.LOG_CLOUD_DELETE });
+                })
                     .catch((err) => {
                     if (!(err instanceof Error && err.code === 'BlobNotFound')) {
                         this.formatFail(64, service, ["Delete failed", location], err, { ...Cloud.LOG_CLOUD_FAIL, fatal: !!target.active });
@@ -28,6 +44,6 @@ module.exports = function download(credential, service = 'azure') {
                 });
             }
         })
-            .catch(err => callback(err));
+            .catch((err) => downloadResponse(err));
     };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-r/azure",
-  "version": "0.6.6",
+  "version": "0.7.0",
   "description": "Azure cloud functions for E-mc.",
   "main": "client/index.js",
   "publishConfig": {
@@ -20,11 +20,11 @@
   "license": "MIT",
   "homepage": "https://github.com/anpham6/pi-r#readme",
   "dependencies": {
-    "@e-mc/cloud": "^0.8.7",
-    "@e-mc/module": "^0.8.7",
-    "@e-mc/types": "^0.8.7",
+    "@e-mc/cloud": "^0.9.0",
+    "@e-mc/module": "^0.9.0",
+    "@e-mc/types": "^0.9.0",
     "@azure/cosmos": "^4.0.0",
-    "@azure/identity": "^4.0.1",
+    "@azure/identity": "^4.1.0",
     "@azure/storage-blob": "^12.17.0"
   }
 }

package/types/index.d.ts

@@ -0,0 +1,24 @@
+import type { CloudDatabase, CloudStorage } from '@e-mc/types/lib/cloud';
+
+import type { CosmosClientOptions, FeedOptions, PartitionKey, PatchRequestBody, RequestOptions, SqlQuerySpec } from '@azure/cosmos';
+
+export type AzureStorage = CloudStorage<AzureStorageCredential, "azure" | "az">;
+
+export interface AzureStorageCredential {
+    accountName?: string;
+    accountKey?: string;
+    connectionString?: string;
+    sharedAccessSignature?: string;
+}
+
+export interface AzureDatabaseQuery<T = FeedOptions | RequestOptions> extends CloudDatabase<string | SqlQuerySpec, T, PatchRequestBody, unknown[], AzureDatabaseCredential> {
+    source: "cloud";
+    service: "azure" | "az";
+    partitionKey?: PartitionKey;
+    storedProcedureId?: string;
+}
+
+export interface AzureDatabaseCredential extends AuthValue, CosmosClientOptions {
+    tenantId?: string;
+    clientId?: string;
+}

package/upload/index.js

@@ -1,19 +1,19 @@
 "use strict";
 const path = require("path");
+const fs = require("fs");
+const stream = require("stream");
 const util_1 = require("@e-mc/cloud/util");
 const types_1 = require("@e-mc/types");
-const Module = require("@e-mc/module");
 const Cloud = require("@e-mc/cloud");
 const client_1 = require("@pi-r/azure");
 const BUCKET_SESSION = new Set();
 const BUCKET_RESPONSE = {};
-const getBucketKey = (credential, bucket, acl = '') => Module.asString(credential, true) + bucket + '_' + acl;
+const getBucketKey = (credential, bucket, acl = '') => Cloud.asString(credential, true) + bucket + '_' + acl;
 module.exports = function upload(credential, service = 'azure') {
     const blobServiceClient = client_1.createStorageClient.call(this, credential);
     return async (data, callback) => {
-        var _a, _b;
         const { bucket, localUri } = data;
-        const { pathname = '', fileGroup, contentType, metadata, tags, endpoint, admin = {}, overwrite, options } = data.upload;
+        const { pathname = '', flags = 0, fileGroup, contentType, metadata, tags, endpoint, admin = {}, overwrite, options } = data.upload;
         const containerClient = blobServiceClient.getContainerClient(bucket);
         let filename = data.upload.filename || path.basename(localUri), bucketKey;
         const cleanup = () => {
@@ -30,7 +30,7 @@ module.exports = function upload(credential, service = 'azure') {
         const configBucket = admin.configBucket;
         if (!BUCKET_SESSION.has(bucket)) {
             const bucketAcl = admin.publicRead ? 'blob' : admin.acl;
-            const response = BUCKET_RESPONSE[_a = bucketKey = getBucketKey(credential, bucket, bucketAcl)] || (BUCKET_RESPONSE[_a] = client_1.createBucketV2.call(this, credential, bucket, bucketAcl, configBucket?.create));
+            const response = BUCKET_RESPONSE[bucketKey = getBucketKey(credential, bucket, bucketAcl)] ||= client_1.createBucketV2.call(this, credential, bucket, bucketAcl, configBucket?.create);
             if (!await response) {
                 errorResponse(null);
                 return;
@@ -42,12 +42,19 @@ module.exports = function upload(credential, service = 'azure') {
             if (!Array.isArray(retentionPolicy)) {
                 retentionPolicy = [retentionPolicy];
             }
+            try {
+                if (this.hasCoerce(service, 'options')) {
+                    retentionPolicy.forEach(policy => (0, types_1.coerceObject)(policy));
+                }
+            }
+            catch {
+            }
             containerClient.setAccessPolicy('container', retentionPolicy)
                 .then(() => {
                 const { expiresOn, startsOn, permissions } = retentionPolicy[0].accessPolicy;
                 this.formatMessage(64, service, ["Bucket configured" + ' (retention policy)', bucket], expiresOn ? expiresOn.toLocaleString() + ' (expires)' : startsOn ? startsOn.toLocaleString() + ' (starts)' : permissions, { ...Cloud.LOG_CLOUD_COMMAND });
             })
-                .catch(err => addLog(err));
+                .catch((err) => addLog(err));
         }
         if (!overwrite) {
             try {
@@ -61,7 +68,7 @@ module.exports = function upload(credential, service = 'azure') {
                             break;
                         }
                     }
-                    const name = pathname ? Module.joinPath(pathname, filename) : filename;
+                    const name = pathname ? Cloud.joinPath(pathname, filename) : filename;
                     for await (const blob of containerClient.listBlobsFlat({ includeUncommitedBlobs: true })) {
                         if (blob.name === name) {
                             exists = true;
@@ -78,14 +85,50 @@ module.exports = function upload(credential, service = 'azure') {
                 return;
             }
         }
+        const chunkSize = flags & 4 ? (0, types_1.alignSize)(data.upload.chunkSize, 1024) : 0;
+        const maxSingleShotSize = Math.max(chunkSize, 8388608);
         const Key = [filename];
         const Body = [data.buffer];
+        const Path = [];
+        const Stream = [];
         const ContentType = [contentType];
+        const getChunkSize = () => chunkSize > 0 ? maxSingleShotSize : undefined;
+        if (Body[0].length === 0) {
+            try {
+                if (chunkSize > 0 && fs.statSync(localUri).size > maxSingleShotSize) {
+                    Path[0] = localUri;
+                }
+                else if (flags & 2) {
+                    Stream[0] = fs.createReadStream(localUri, { highWaterMark: getChunkSize(), signal: this.signal });
+                }
+                else {
+                    Body[0] = fs.readFileSync(localUri);
+                }
+            }
+            catch (err) {
+                errorResponse(err);
+                return;
+            }
+        }
+        else if ((flags & 2) || chunkSize > 0) {
+            Stream[0] = stream.Readable.from(Body[0], { highWaterMark: getChunkSize() });
+        }
         if (fileGroup) {
-            const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, addLog);
+            const [key, body, type] = (0, util_1.createKeyAndBody)(filename, fileGroup, chunkSize > 0 ? maxSingleShotSize : 0, addLog, flags);
             Key.push(...key);
-            Body.push(...body);
             ContentType.push(...type);
+            for (let i = 0; i < body.length; ++i) {
+                const item = body[i];
+                if (typeof item === 'string') {
+                    Path[i + 1] = item;
+                }
+                else if (Buffer.isBuffer(item)) {
+                    Body[i + 1] = item;
+                }
+                else {
+                    Stream[i + 1] = item;
+                }
+            }
         }
         for (let i = 0; i < Key.length; ++i) {
             const first = i === 0;
@@ -95,28 +138,47 @@ module.exports = function upload(credential, service = 'azure') {
                 }
                 return;
             }
-            const blobName = Module.joinPath(pathname, Key[i]);
+            const blobName = Cloud.joinPath(pathname, Key[i]);
             const params = { ...options, abortSignal: this.signal };
+            const headers = params.blobHTTPHeaders ||= {};
             if (first) {
-                if ((0, types_1.isPlainObject)(metadata)) {
+                if ((0, types_1.isObject)(metadata)) {
                     params.metadata = metadata;
                 }
-                if ((0, types_1.isPlainObject)(tags)) {
+                if ((0, types_1.isObject)(tags)) {
                     params.tags = tags;
                 }
+                headers.blobContentType ||= ContentType[i];
+            }
+            else {
+                headers.blobContentType = ContentType[i];
+            }
+            const blobClient = containerClient.getBlockBlobClient(blobName);
+            let uploadClient;
+            if (Stream[i]) {
+                uploadClient = blobClient.uploadStream(Stream[i], getChunkSize(), chunkSize > 0 ? data.upload.chunkLimit : undefined, params);
+            }
+            else if (Path[i]) {
+                params.maxSingleShotSize = maxSingleShotSize;
+                params.concurrency = data.upload.chunkLimit;
+                uploadClient = blobClient.uploadFile(Path[i], params);
+            }
+            else if (chunkSize > 0 && Body[i].byteLength > maxSingleShotSize) {
+                params.maxSingleShotSize = maxSingleShotSize;
+                params.concurrency = data.upload.chunkLimit;
+                uploadClient = blobClient.uploadData(Body[i], params);
             }
-            (_b = (params.blobHTTPHeaders || (params.blobHTTPHeaders = {}))).blobContentType || (_b.blobContentType = ContentType[i]);
-            containerClient.getBlockBlobClient(blobName).upload(Body[i], Body[i].byteLength, params)
+            (uploadClient || blobClient.upload(Body[i], Body[i].byteLength, params))
                 .then(result => {
                 let requestUrl = result._response.request.url;
-                const url = endpoint ? Module.joinPath(endpoint, blobName) : Module.isFile(requestUrl = decodeURIComponent(requestUrl), 'http/s') ? requestUrl : Module.joinPath(`https://${credential.accountName}.blob.core.windows.net`, bucket, blobName);
+                const url = endpoint ? Cloud.joinPath(endpoint, blobName) : Cloud.isFile(requestUrl = decodeURIComponent(requestUrl), 'http/s') ? requestUrl : Cloud.joinPath(`https://${credential.accountName}.blob.core.windows.net`, bucket, blobName);
                 this.formatMessage(64, service, "Upload success", url, { ...Cloud.LOG_CLOUD_UPLOAD });
                 if (first) {
                     cleanup();
                     callback(null, url);
                 }
             })
-                .catch(err => {
+                .catch((err) => {
                 if (first) {
                     errorResponse(err);
                 }