@golemio/core 1.17.3-dev.1581832818 → 1.17.3-dev.1582234527
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
/// <reference types="node" />
|
|
3
3
|
/// <reference types="node" />
|
|
4
4
|
import { ILogger } from "../../logger";
|
|
5
|
-
import { Readable } from "stream";
|
|
5
|
+
import { Readable } from "node:stream";
|
|
6
6
|
import { AbstractStorageService } from "./AbstractStorageService";
|
|
7
7
|
export interface IAzureBlobStorageConfig {
|
|
8
8
|
clientId: string;
|
|
@@ -13,13 +13,14 @@ export interface IAzureBlobStorageConfig {
|
|
|
13
13
|
uploadTimeoutInSeconds: number;
|
|
14
14
|
}
|
|
15
15
|
export declare class AzureBlobStorageService extends AbstractStorageService {
|
|
16
|
+
private config;
|
|
16
17
|
private readonly log;
|
|
17
|
-
private readonly containerClient;
|
|
18
18
|
private readonly uploadTimeoutInMs;
|
|
19
19
|
constructor(config: IAzureBlobStorageConfig, log: ILogger);
|
|
20
20
|
uploadStream(stream: Readable, pathPrefix: string, ext?: string, fileName?: string): Promise<void>;
|
|
21
21
|
uploadFile(data: Buffer | string, pathPrefix: string, ext?: string, fileName?: string): Promise<void>;
|
|
22
22
|
deleteFile(fileKey: string): Promise<void>;
|
|
23
23
|
getFileStream(fileKey: string): Promise<NodeJS.ReadableStream | undefined>;
|
|
24
|
+
private instantiateContainerClient;
|
|
24
25
|
private handleUploadProgress;
|
|
25
26
|
}
|
|
@@ -14,17 +14,12 @@ const _helpers_1 = require("../..");
|
|
|
14
14
|
const identity_1 = require("@azure/identity");
|
|
15
15
|
const storage_blob_1 = require("@azure/storage-blob");
|
|
16
16
|
const AbstractStorageService_1 = require("./AbstractStorageService");
|
|
17
|
+
const UPLOAD_STREAM_BUFFER_SIZE = 4 * 1024 * 1024; // 4 MB
|
|
17
18
|
class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageService {
|
|
18
19
|
constructor(config, log) {
|
|
19
20
|
super();
|
|
21
|
+
this.config = config;
|
|
20
22
|
this.log = log;
|
|
21
|
-
const credentials = new identity_1.ClientSecretCredential(config.tenantId, config.clientId, config.clientSecret);
|
|
22
|
-
const blobServiceClient = new storage_blob_1.BlobServiceClient(`https://${config.account}.blob.core.windows.net`, credentials);
|
|
23
|
-
this.containerClient = blobServiceClient.getContainerClient(config.containerName);
|
|
24
|
-
this.containerClient.createIfNotExists().catch((err) => {
|
|
25
|
-
// This is fine in the production environment
|
|
26
|
-
this.log.debug(err, `[${this.constructor.name}] Could not initialize container "${config.containerName}"`);
|
|
27
|
-
});
|
|
28
23
|
this.uploadTimeoutInMs = config.uploadTimeoutInSeconds * 1000;
|
|
29
24
|
}
|
|
30
25
|
uploadStream(stream, pathPrefix, ext = "", fileName) {
|
|
@@ -33,34 +28,15 @@ class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageSe
|
|
|
33
28
|
const fileNamePrefix = fileName !== null && fileName !== void 0 ? fileName : `${now.format("yyyy-MM-dd")}/${now.format("HH_mm_ss.SSS")}`;
|
|
34
29
|
const fileNameKey = `${pathPrefix}/${fileNamePrefix}`;
|
|
35
30
|
const fileNameExt = ext ? `.${ext}` : "";
|
|
36
|
-
const controller = new AbortController();
|
|
37
31
|
try {
|
|
38
32
|
const options = {
|
|
39
33
|
blobHTTPHeaders: { blobContentType: "application/octet-stream" },
|
|
40
34
|
onProgress: this.handleUploadProgress(fileNameKey + fileNameExt),
|
|
41
|
-
abortSignal:
|
|
35
|
+
abortSignal: AbortSignal.timeout(this.uploadTimeoutInMs),
|
|
42
36
|
};
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
if (!controller.signal.aborted) {
|
|
47
|
-
controller.abort();
|
|
48
|
-
}
|
|
49
|
-
});
|
|
50
|
-
stream.on("end", () => {
|
|
51
|
-
this.log.verbose(`[${this.constructor.name}] Read stream ended (${fileNameKey}${fileNameExt})`);
|
|
52
|
-
});
|
|
53
|
-
stream.on("close", () => {
|
|
54
|
-
this.log.verbose(`[${this.constructor.name}] Read stream closed (${fileNameKey}${fileNameExt})`);
|
|
55
|
-
});
|
|
56
|
-
//#endregion
|
|
57
|
-
setTimeout(() => {
|
|
58
|
-
if (!controller.signal.aborted) {
|
|
59
|
-
controller.abort();
|
|
60
|
-
}
|
|
61
|
-
}, this.uploadTimeoutInMs);
|
|
62
|
-
const blobClient = this.containerClient.getBlockBlobClient(`${fileNameKey}${fileNameExt}`);
|
|
63
|
-
yield blobClient.uploadStream(stream, undefined, undefined, options);
|
|
37
|
+
const containerClient = this.instantiateContainerClient();
|
|
38
|
+
const blobClient = containerClient.getBlockBlobClient(`${fileNameKey}${fileNameExt}`);
|
|
39
|
+
yield blobClient.uploadStream(stream, UPLOAD_STREAM_BUFFER_SIZE, undefined, options);
|
|
64
40
|
this.log.verbose(`[${this.constructor.name}] Data stream uploaded (${fileNameKey}${fileNameExt})`);
|
|
65
41
|
}
|
|
66
42
|
catch (err) {
|
|
@@ -83,8 +59,10 @@ class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageSe
|
|
|
83
59
|
if (!Buffer.isBuffer(data)) {
|
|
84
60
|
data = Buffer.from(data);
|
|
85
61
|
}
|
|
86
|
-
const
|
|
62
|
+
const containerClient = this.instantiateContainerClient();
|
|
63
|
+
const blobClient = containerClient.getBlockBlobClient(`${fileNameKey}${fileNameExt}`);
|
|
87
64
|
yield blobClient.uploadData(data, options);
|
|
65
|
+
this.log.verbose(`[${this.constructor.name}] Data uploaded (${fileNameKey}${fileNameExt})`);
|
|
88
66
|
}
|
|
89
67
|
catch (err) {
|
|
90
68
|
this.log.error(err, `[${this.constructor.name}] Saving of data failed (${fileNameKey}${fileNameExt})`);
|
|
@@ -97,8 +75,10 @@ class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageSe
|
|
|
97
75
|
const options = {
|
|
98
76
|
deleteSnapshots: "include",
|
|
99
77
|
};
|
|
100
|
-
const
|
|
78
|
+
const containerClient = this.instantiateContainerClient();
|
|
79
|
+
const blobClient = containerClient.getBlockBlobClient(fileKey);
|
|
101
80
|
yield blobClient.delete(options);
|
|
81
|
+
this.log.verbose(`[${this.constructor.name}] Data deleted (${fileKey})`);
|
|
102
82
|
}
|
|
103
83
|
catch (err) {
|
|
104
84
|
this.log.error(err, `[${this.constructor.name}] Deletion of data failed (${fileKey})`);
|
|
@@ -108,7 +88,8 @@ class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageSe
|
|
|
108
88
|
getFileStream(fileKey) {
|
|
109
89
|
return __awaiter(this, void 0, void 0, function* () {
|
|
110
90
|
try {
|
|
111
|
-
const
|
|
91
|
+
const containerClient = this.instantiateContainerClient();
|
|
92
|
+
const blobClient = containerClient.getBlockBlobClient(fileKey);
|
|
112
93
|
const downloadResponse = yield blobClient.download();
|
|
113
94
|
return downloadResponse.readableStreamBody;
|
|
114
95
|
}
|
|
@@ -117,6 +98,12 @@ class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageSe
|
|
|
117
98
|
}
|
|
118
99
|
});
|
|
119
100
|
}
|
|
101
|
+
instantiateContainerClient() {
|
|
102
|
+
// TODO: use @azure/identity-cache-persistence if Microsoft starts to reject auth requests
|
|
103
|
+
const credentials = new identity_1.ClientSecretCredential(this.config.tenantId, this.config.clientId, this.config.clientSecret);
|
|
104
|
+
const blobServiceClient = new storage_blob_1.BlobServiceClient(`https://${this.config.account}.blob.core.windows.net`, credentials);
|
|
105
|
+
return blobServiceClient.getContainerClient(this.config.containerName);
|
|
106
|
+
}
|
|
120
107
|
handleUploadProgress(filePath, dataByteLength) {
|
|
121
108
|
return (progress) => {
|
|
122
109
|
this.log.verbose(`[${this.constructor.name}] File upload ` +
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AzureBlobStorageService.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/storage/AzureBlobStorageService.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,oCAAoC;AAGpC,8CAAyD;AACzD,sDAA4H;AAE5H,qEAAkE;AAWlE,
|
|
1
|
+
{"version":3,"file":"AzureBlobStorageService.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/storage/AzureBlobStorageService.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,oCAAoC;AAGpC,8CAAyD;AACzD,sDAA4H;AAE5H,qEAAkE;AAWlE,MAAM,yBAAyB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;AAE1D,MAAa,uBAAwB,SAAQ,+CAAsB;IAG/D,YAAoB,MAA+B,EAAmB,GAAY;QAC9E,KAAK,EAAE,CAAC;QADQ,WAAM,GAAN,MAAM,CAAyB;QAAmB,QAAG,GAAH,GAAG,CAAS;QAE9E,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAClE,CAAC;IAEY,YAAY,CAAC,MAAgB,EAAE,UAAkB,EAAE,MAAc,EAAE,EAAE,QAAiB;;YAC/F,MAAM,GAAG,GAAG,IAAA,mBAAQ,EAAC,IAAI,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;YAChE,MAAM,cAAc,GAAG,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/F,MAAM,WAAW,GAAG,GAAG,UAAU,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAEzC,IAAI;gBACA,MAAM,OAAO,GAAmC;oBAC5C,eAAe,EAAE,EAAE,eAAe,EAAE,0BAA0B,EAAE;oBAChE,UAAU,EAAE,IAAI,CAAC,oBAAoB,CAAC,WAAW,GAAG,WAAW,CAAC;oBAChE,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC;iBAC3D,CAAC;gBAEF,MAAM,eAAe,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAC1D,MAAM,UAAU,GAAG,eAAe,CAAC,kBAAkB,CAAC,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC,CAAC;gBAEtF,MAAM,UAAU,CAAC,YAAY,CAAC,MAAM,EAAE,yBAAyB,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;gBACrF,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,2BAA2B,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC;aACtG;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,mCAAmC,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC;aACjH;QACL,CAAC;KAAA;IAEY,UAAU,CAAC,IAAqB,EAAE,UAAkB,EAAE,MAAc,EAAE,EAAE,QAAiB;;YAClG,MAAM,GAAG,GAAG,IAAA,mBAAQ,EAAC,IAAI,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;YAChE,MAAM,cAAc,GAAG,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/F,MAAM,WAAW,GAAG,GAAG,UAAU,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAEzC,IAAI;gBACA,MAAM,OAAO,GAAmC;oBAC5C,eAAe,EAAE,EAAE,eAAe,EAAE,0BAA0B,EAAE;oBAChE,UAAU,EAAE,IAAI,CAAC,oBAAoB,CAAC,WAAW,GAAG,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;oBACzF,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC;iBAC3D,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBACxB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC5B;gBAED,MAAM,eAAe,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAC1D,MAAM,UAAU,GAAG,eAAe,CAAC,kBAAkB,CAAC,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC,CAAC;gBAEtF,MAAM,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,oBAAoB,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC;aAC/F;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,4BAA4B,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC;aAC1G;QACL,CAAC;KAAA;IAEY,UAAU,CAAC,OAAe;;YACnC,IAAI;gBACA,MAAM,OAAO,GAAsB;oBAC/B,eAAe,EAAE,SAAS;iBAC7B,CAAC;gBAEF,MAAM,eAAe,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAC1D,MAAM,UAAU,GAAG,eAAe,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAE/D,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACjC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,mBAAmB,OAAO,GAAG,CAAC,CAAC;aAC5E;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,8BAA8B,OAAO,GAAG,CAAC,CAAC;aAC1F;QACL,CAAC;KAAA;IAEY,aAAa,CAAC,OAAe;;YACtC,IAAI;gBACA,MAAM,eAAe,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAC1D,MAAM,UAAU,GAAG,eAAe,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAE/D,MAAM,gBAAgB,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACrD,OAAO,gBAAgB,CAAC,kBAAkB,CAAC;aAC9C;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,0BAA0B,OAAO,GAAG,CAAC,CAAC;aACtF;QACL,CAAC;KAAA;IAEO,0BAA0B;QAC9B,0FAA0F;QAC1F,MAAM,WAAW,GAAG,IAAI,iCAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACrH,MAAM,iBAAiB,GAAG,IAAI,gCAAiB,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,OAAO,wBAAwB,EAAE,WAAW,CAAC,CAAC;QACrH,OAAO,iBAAiB,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAC3E,CAAC;IAEO,oBAAoB,CAAC,QAAgB,EAAE,cAAuB;QAClE,OAAO,CAAC,QAA+B,EAAE,EAAE;YACvC,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gBAAgB;gBACrC,GAAG,QAAQ,KAAK,QAAQ,CAAC,WAAW,MAAM,cAAc,aAAd,cAAc,cAAd,cAAc,GAAI,SAAS,EAAE,CAC9E,CAAC;QACN,CAAC,CAAC;IACN,CAAC;CACJ;AArGD,0DAqGC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@golemio/core",
|
|
3
|
-
"version": "1.17.3-dev.
|
|
3
|
+
"version": "1.17.3-dev.1582234527",
|
|
4
4
|
"description": "Golemio Core Module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
"sequelize": "^6.29.0",
|
|
126
126
|
"ssh2-sftp-client": "^7.1.0",
|
|
127
127
|
"tsyringe": "^4.8.0",
|
|
128
|
-
"undici": "^6.
|
|
128
|
+
"undici": "^6.21.0",
|
|
129
129
|
"xml2js": "^0.5.0",
|
|
130
130
|
"yauzl": "^2.10.0"
|
|
131
131
|
},
|