@e-mc/cloud 0.8.10 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/LICENSE +7 -3
  2. package/README.md +125 -18
  3. package/index.js +170 -114
  4. package/package.json +5 -5
  5. package/types/index.d.ts +20 -16
  6. package/util.d.ts +3 -7
  7. package/util.js +76 -24
package/LICENSE CHANGED
@@ -1,7 +1,11 @@
1
1
  Copyright 2024 An Pham
2
2
 
3
- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
3
+ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
4
4
 
5
- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
5
+ 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
6
6
 
7
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
7
+ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
8
+
9
+ 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
10
+
11
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # @e-mc/cloud
2
2
 
3
- * NodeJS 14
4
- * ES2019
3
+ * NodeJS 16
4
+ * ES2020
5
5
 
6
6
  ## General Usage
7
7
 
@@ -9,28 +9,29 @@
9
9
 
10
10
  ## Interface
11
11
 
12
- - https://www.unpkg.com/@e-mc/types@0.8.10/lib/index.d.ts
12
+ * [View Source](https://www.unpkg.com/@e-mc/types@0.9.1/lib/index.d.ts)
13
13
 
14
14
  ```typescript
15
15
  import type { IHost, IScopeOrigin } from "./index";
16
16
  import type { ExternalAsset } from "./asset";
17
- import type { BucketWebsiteOptions, CloudDatabase, CloudFeatures, CloudFunctions, CloudService, CloudStorage, CloudStorageDownload, CloudStorageUpload } from "./cloud";
17
+ import type { BucketWebsiteOptions, CloudDatabase, CloudFeatures, CloudFunctions, CloudService, CloudStorage, CloudStorageDownload, CloudStorageUpload, UploadAssetOptions } from "./cloud";
18
18
  import type { ClientDbConstructor, IClientDb } from "./core";
19
19
  import type { BatchQueryResult, QueryResult } from "./db";
20
20
  import type { LogMessageOptions } from "./logger";
21
- import type { CloudModule, CloudServiceOptions, CloudSettings, DbCoerceSettings } from "./settings";
21
+ import type { CloudAuthSettings, CloudModule, CloudServiceOptions, CloudSettings, DbCoerceSettings } from "./settings";
22
22
 
23
- interface ICloud extends IClientDb<IHost, CloudModule, CloudDatabase, CloudServiceOptions, DbCoerceSettings> {
23
+ interface ICloud extends IClientDb<IHost, CloudModule, CloudDatabase, CloudServiceOptions, DbCoerceSettings & CloudAuthSettings> {
24
24
  module: CloudModule;
25
25
  readonly uploaded: string[];
26
26
  readonly downloaded: string[];
27
27
  createBucket(service: string, credential: unknown, bucket: string, acl?: unknown, options?: unknown): Promise<boolean>;
28
28
  createBucket(service: string, credential: unknown, bucket: string, publicRead?: boolean): Promise<boolean>;
29
29
  setBucketPolicy(service: string, credential: unknown, bucket: string, options: unknown): Promise<boolean>;
30
+ setBucketTagging(service: string, credential: unknown, bucket: string, options: unknown): Promise<boolean>;
30
31
  setBucketWebsite(service: string, credential: unknown, bucket: string, options: BucketWebsiteOptions): Promise<boolean>;
31
32
  deleteObjects(service: string, credential: unknown, bucket: string, recursive?: boolean): Promise<void>;
32
- uploadObject(service: string, credential: unknown, bucket: string, upload: CloudStorageUpload, localUri: string, beforeResolve?: (value: string) => Promise<void> | void): Promise<string>;
33
- downloadObject(service: string, credential: unknown, bucket: string, download: CloudStorageDownload, beforeResolve?: (value: Buffer | string | null) => Promise<string | undefined> | void): Promise<Buffer | string>;
33
+ uploadObject(service: string, credential: unknown, bucket: string, upload: CloudStorageUpload, localUri: string, beforeResolve?: ((value: string) => Promise<void> | void)): Promise<string>;
34
+ downloadObject(service: string, credential: unknown, bucket: string, download: CloudStorageDownload, beforeResolve?: ((value: Buffer | string | null) => Promise<string | undefined> | void)): Promise<Buffer | string>;
34
35
  getStorage(action: CloudFunctions, data: CloudStorage[] | undefined): CloudStorage | undefined;
35
36
  hasStorage(action: CloudFunctions, storage: CloudStorage): CloudStorageUpload | false;
36
37
  getDatabaseRows(item: CloudDatabase, ignoreErrors: boolean, sessionKey?: string): Promise<QueryResult>;
@@ -40,8 +41,8 @@ interface ICloud extends IClientDb<IHost, CloudModule, CloudDatabase, CloudServi
40
41
  hasCredential(feature: CloudFeatures, data: CloudService, credential?: unknown): boolean;
41
42
  getCredential(item: CloudService, unused?: boolean): Record<string | number | symbol, unknown>;
42
43
  getSettings(service: string): Record<string, unknown> | undefined;
43
- settingsOf(service: string, name: "cache"): unknown;
44
44
  settingsOf(service: string, name: "coerce", component: keyof DbCoerceSettings): unknown;
45
+ settingsOf(service: string, name: "auth", component: keyof CloudAuthSettings): unknown;
45
46
  getUploadHandler(service: string, credential: unknown): (...args: unknown[]) => void;
46
47
  getDownloadHandler(service: string, credential: unknown): (...args: unknown[]) => void;
47
48
  resolveService(service: string, folder?: string): string;
@@ -57,23 +58,129 @@ interface CloudConstructor extends ClientDbConstructor<IHost> {
57
58
  LOG_CLOUD_DELETE: LogMessageOptions;
58
59
  LOG_CLOUD_DELAYED: LogMessageOptions;
59
60
  finalize(this: IHost, instance: ICloud): Promise<unknown>;
60
- uploadAsset(state: IScopeOrigin<IFileManager, ICloud<IFileManager>>, file: ExternalAsset, ignoreProcess: boolean): Promise<unknown>[];
61
- uploadAsset(state: IScopeOrigin<IFileManager, ICloud<IFileManager>>, file: ExternalAsset, contentType?: string, ignoreProcess?: boolean): Promise<unknown>[];
61
+ uploadAsset(state: IScopeOrigin<IFileManager, ICloud>, file: ExternalAsset, options: UploadAssetOptions): Promise<unknown>[];
62
+ uploadAsset(state: IScopeOrigin<IFileManager, ICloud>, file: ExternalAsset, ignoreProcess: boolean): Promise<unknown>[];
63
+ uploadAsset(state: IScopeOrigin<IFileManager, ICloud>, file: ExternalAsset, contentType?: string, ignoreProcess?: boolean): Promise<unknown>[];
62
64
  sanitizeAssets(assets: ExternalAsset[]): ExternalAsset[];
63
65
  readonly prototype: ICloud;
64
66
  new(module?: CloudModule, database?: CloudDatabase[], ...args: unknown[]): ICloud;
65
67
  }
68
+
69
+ interface ICloudServiceClient {
70
+ CLOUD_SERVICE_NAME: string;
71
+ CLOUD_UPLOAD_DISK?: boolean;
72
+ CLOUD_UPLOAD_STREAM?: boolean;
73
+ CLOUD_UPLOAD_CHUNK?: boolean;
74
+ CLOUD_DOWNLOAD_CHUNK?: boolean;
75
+ validateStorage?(credential: unknown, data?: CloudService): boolean;
76
+ validateDatabase?(credential: unknown, data?: CloudService): boolean;
77
+ createStorageClient?(this: IModule, credential: unknown, service?: string): unknown;
78
+ createDatabaseClient?(this: IModule, credential: unknown, data?: CloudService): unknown;
79
+ createBucket?(this: IModule, credential: unknown, bucket: string, publicRead?: boolean, service?: string, sdk?: string): Promise<boolean>;
80
+ createBucketV2?(this: IModule, credential: unknown, bucket: string, acl?: unknown, options?: unknown, service?: string, sdk?: string): Promise<boolean>;
81
+ setBucketPolicy?(this: IModule, credential: unknown, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
82
+ setBucketTagging?(this: IModule, credential: unknown, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
83
+ setBucketWebsite?(this: IModule, credential: unknown, bucket: string, options: BucketWebsiteOptions, service?: string, sdk?: string): Promise<boolean>;
84
+ deleteObjects?(this: IModule, credential: unknown, bucket: string, service?: string, sdk?: string, recursive?: boolean): Promise<void>;
85
+ deleteObjectsV2?(this: IModule, credential: unknown, bucket: string, recursive?: boolean, service?: string, sdk?: string): Promise<void>;
86
+ executeQuery?(this: ICloud, credential: unknown, data: CloudDatabase, sessionKey?: string): Promise<QueryResult>;
87
+ executeBatchQuery?(this: ICloud, credential: unknown, batch: CloudDatabase[], sessionKey?: string): Promise<BatchQueryResult>;
88
+ }
89
+ ```
90
+
91
+ ## Settings
92
+
93
+ ```typescript
94
+ import type { PermittedDirectories } from "./core";
95
+ import type { CloudServiceOptions, DbSourceOptions, PurgeComponent } from "./settings";
96
+
97
+ interface CloudModule {
98
+ // handler: "@e-mc/cloud";
99
+ extensions?: string[];
100
+ atlas?: CloudStoredCredentials;
101
+ aws?: CloudStoredCredentials;
102
+ "aws-v3"?: CloudStoredCredentials;
103
+ azure?: CloudStoredCredentials; // az
104
+ gcp?: CloudStoredCredentials; // gcloud
105
+ ibm?: CloudStoredCredentials;
106
+ oci?: CloudStoredCredentials;
107
+ minio?: CloudStoredCredentials;
108
+ settings?: {
109
+ broadcast_id?: string | string[];
110
+ users?: Record<string, Record<string, unknown>>;
111
+ cache_dir?: string;
112
+ session_expires?: number;
113
+ user_key?: Record<string, DbSourceOptions>;
114
+ imports?: StringMap;
115
+ purge?: PurgeComponent;
116
+ atlas?: CloudServiceOptions;
117
+ aws?: CloudServiceOptions;
118
+ "aws-v3"?: CloudServiceOptions;
119
+ azure?: CloudServiceOptions;
120
+ gcp?: CloudServiceOptions;
121
+ ibm?: CloudServiceOptions;
122
+ oci?: CloudServiceOptions;
123
+ minio?: CloudServiceOptions;
124
+ };
125
+ permission?: PermittedDirectories;
126
+ }
127
+
128
+ type CloudStoredCredentials = Record<string, Record<string, unknown>>;
129
+ ```
130
+
131
+ ### Example usage
132
+
133
+ ```javascript
134
+ const Cloud = require("@e-mc/cloud"); // Using @pi-r/aws
135
+
136
+ const instance = new Cloud({
137
+ aws: {
138
+ main: {
139
+ accessKeyId: "**********",
140
+ secretAccessKey: "**********"
141
+ }
142
+ },
143
+ "aws-v3": {
144
+ main: {
145
+ credentials: {
146
+ accessKeyId: "**********",
147
+ secretAccessKey: "**********",
148
+ region: "ap-northeast-1"
149
+ }
150
+ }
151
+ }
152
+ });
153
+ // instance.host = new Host();
154
+ instance.init();
155
+
156
+ const options = {
157
+ contentType: "application/tar",
158
+ acl: "authenticated-read",
159
+ chunkSize: "8mb",
160
+ overwrite: false, // Default
161
+ tags: { key_1: "value", key_2: "value" }
162
+ };
163
+ Promise.all([
164
+ // nodejs-001/archive.tar
165
+ instance.uploadObject("aws", "main", "nodejs-001", options, "/tmp/archive.tar"),
166
+ // nodejs-001/2024/01-01.tar
167
+ instance.uploadObject("aws", "main", "nodejs-001", { ...options, publicRead: true, pathname: "2024", filename: "01-01.tar" }, "/tmp/archive.tar"),
168
+ // nodejs-001/archive_1.tar
169
+ instance.uploadObject("aws", { accessKeyId: "*****", secretAccessKey: "*****" }, "nodejs-001", { overwrite: false }, "/tmp/archive.tar")
170
+ ]);
171
+
172
+ const rows = await instance.getDatabaseRows({ service: "aws-v3", credential: "main", table: "demo", key: { id: 1 } });
66
173
  ```
67
174
 
68
175
  ## References
69
176
 
70
- - https://www.unpkg.com/@e-mc/types@0.8.10/lib/asset.d.ts
71
- - https://www.unpkg.com/@e-mc/types@0.8.10/lib/cloud.d.ts
72
- - https://www.unpkg.com/@e-mc/types@0.8.10/lib/core.d.ts
73
- - https://www.unpkg.com/@e-mc/types@0.8.10/lib/db.d.ts
74
- - https://www.unpkg.com/@e-mc/types@0.8.10/lib/logger.d.ts
75
- - https://www.unpkg.com/@e-mc/types@0.8.10/lib/settings.d.ts
177
+ - https://www.unpkg.com/@e-mc/types@0.9.1/lib/asset.d.ts
178
+ - https://www.unpkg.com/@e-mc/types@0.9.1/lib/cloud.d.ts
179
+ - https://www.unpkg.com/@e-mc/types@0.9.1/lib/core.d.ts
180
+ - https://www.unpkg.com/@e-mc/types@0.9.1/lib/db.d.ts
181
+ - https://www.unpkg.com/@e-mc/types@0.9.1/lib/logger.d.ts
182
+ - https://www.unpkg.com/@e-mc/types@0.9.1/lib/settings.d.ts
76
183
 
77
184
  ## LICENSE
78
185
 
79
- MIT
186
+ BSD 3-Clause
package/index.js CHANGED
@@ -1,5 +1,4 @@
1
1
  "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
2
  const path = require("path");
4
3
  const fs = require("fs");
5
4
  const types_1 = require("@e-mc/types");
@@ -50,10 +49,8 @@ class Cloud extends core_1.ClientDb {
50
49
  this.downloaded = [];
51
50
  }
52
51
  static async finalize(instance) {
53
- var _a, _b, _c, _d;
54
- var _e, _f;
55
52
  if (instance.aborted) {
56
- return Promise.reject((0, types_1.createAbortError)());
53
+ return (0, types_1.createAbortError)(true);
57
54
  }
58
55
  Cloud.sanitizeAssets(this.assets);
59
56
  const localStorage = new Map();
@@ -61,6 +58,7 @@ class Cloud extends core_1.ClientDb {
61
58
  const state = { host: this, instance, bucketGroup, localStorage };
62
59
  const bucketDelete = {};
63
60
  const bucketPolicy = {};
61
+ const bucketTagging = {};
64
62
  const rawFiles = [];
65
63
  const startTime = process.hrtime();
66
64
  let tasks = [], downloadMap;
@@ -68,7 +66,7 @@ class Cloud extends core_1.ClientDb {
68
66
  instance.writeTimeElapsed(instance.moduleName, "Transactions were committed", startTime, { type: 64, ...Cloud.LOG_STYLE_SUCCESS });
69
67
  }
70
68
  for (const { instance: document } of this.Document) {
71
- (_a = document.cloudInit) === null || _a === void 0 ? void 0 : _a.call(document, state);
69
+ document.cloudInit?.(state);
72
70
  }
73
71
  for (const item of this.assets) {
74
72
  const cloudStorage = item.cloudStorage;
@@ -77,11 +75,11 @@ class Cloud extends core_1.ClientDb {
77
75
  cloudStorage.forEach(storage => instance.formatMessage(64, storage.service, ["Upload failed", storage.bucket], (0, types_1.errorValue)("File not found", item.uri || item.filename || "Unknown"), { ...Cloud.LOG_CLOUD_WARN }));
78
76
  continue;
79
77
  }
80
- ignore: {
78
+ resume: {
81
79
  if (item.localUri || item.torrentFiles) {
82
80
  for (const { instance: document } of this.Document) {
83
- if ((_b = document.cloudObject) === null || _b === void 0 ? void 0 : _b.call(document, state, item)) {
84
- break ignore;
81
+ if (document.cloudObject?.(state, item)) {
82
+ break resume;
85
83
  }
86
84
  }
87
85
  if (item.compress) {
@@ -93,20 +91,29 @@ class Cloud extends core_1.ClientDb {
93
91
  for (const storage of cloudStorage) {
94
92
  const { admin, bucket } = storage;
95
93
  if (admin && bucket && instance.hasCredential('storage', storage)) {
96
- const policy = (_c = admin.configBucket) === null || _c === void 0 ? void 0 : _c.policy;
94
+ const name = storage.service;
95
+ const credential = instance.getCredential(storage, true);
96
+ const configBucket = admin.configBucket;
97
97
  if (admin.emptyBucket) {
98
- const service = bucketDelete[_e = storage.service] || (bucketDelete[_e] = {});
98
+ const service = bucketDelete[name] || (bucketDelete[name] = {});
99
99
  const items = service[bucket];
100
100
  if (!items) {
101
- service[bucket] = [instance.getCredential(storage), admin.recursive];
101
+ service[bucket] = [{ ...credential }, admin.recursive];
102
102
  }
103
103
  else if (admin.recursive === false) {
104
104
  items[1] = false;
105
105
  }
106
106
  }
107
- if (policy) {
108
- const service = bucketPolicy[_f = storage.service] || (bucketPolicy[_f] = {});
109
- service[bucket] = [storage.service, instance.getCredential(storage), bucket, policy];
107
+ if (configBucket) {
108
+ const { policy, tags } = configBucket;
109
+ if (policy) {
110
+ const service = bucketPolicy[name] || (bucketPolicy[name] = {});
111
+ service[bucket] = [{ ...credential }, policy];
112
+ }
113
+ if (tags) {
114
+ const service = bucketTagging[name] || (bucketTagging[name] = {});
115
+ service[bucket] = [{ ...credential }, tags];
116
+ }
110
117
  }
111
118
  }
112
119
  }
@@ -115,7 +122,7 @@ class Cloud extends core_1.ClientDb {
115
122
  if (tasks.length) {
116
123
  await instance.allSettled(tasks, ['Compress files', instance.moduleName]);
117
124
  if (instance.aborted) {
118
- return Promise.reject((0, types_1.createAbortError)());
125
+ return (0, types_1.createAbortError)(true);
119
126
  }
120
127
  tasks = [];
121
128
  }
@@ -123,43 +130,53 @@ class Cloud extends core_1.ClientDb {
123
130
  const map = bucketDelete[service];
124
131
  for (const bucket in map) {
125
132
  const [credential, recursive] = map[bucket];
126
- tasks.push(instance.deleteObjects(service, credential, bucket, recursive).catch(err => instance.writeFail(["Unable to empty bucket", service + ': ' + bucket], err, { type: 64, startTime })));
133
+ tasks.push(instance.deleteObjects(service, credential, bucket, recursive).catch((err) => instance.writeFail(["Unable to empty bucket", service + ': ' + bucket], err, { type: 64, startTime })));
127
134
  }
128
135
  }
129
136
  if (tasks.length) {
130
137
  await instance.allSettled(tasks, ['Empty bucket', instance.moduleName]);
131
138
  if (instance.aborted) {
132
- return Promise.reject((0, types_1.createAbortError)());
139
+ return (0, types_1.createAbortError)(true);
133
140
  }
134
141
  tasks = [];
135
142
  }
136
- rawFiles.forEach(item => tasks.push(...Cloud.uploadAsset(state, item)));
143
+ if (rawFiles.length) {
144
+ const options = { preferBuffer: process.env.EMC_CLOUD_UPLOAD_BUFFER === 'true' };
145
+ rawFiles.forEach(item => tasks.push(...Cloud.uploadAsset(state, item, options)));
146
+ }
137
147
  if (tasks.length) {
138
148
  await instance.allSettled(tasks, ['Upload raw assets', instance.moduleName]);
139
149
  if (instance.aborted) {
140
- return Promise.reject((0, types_1.createAbortError)());
150
+ return (0, types_1.createAbortError)(true);
141
151
  }
142
152
  tasks = [];
143
153
  }
144
154
  for (const service in bucketPolicy) {
145
155
  const map = bucketPolicy[service];
146
156
  for (const bucket in map) {
147
- const params = map[bucket];
148
- tasks.push(instance.setBucketPolicy(...params).catch(err => instance.writeFail(["Unable to update bucket policy", params[0] + ': ' + params[2]], err, { type: 64, startTime })));
157
+ const [credential, options] = map[bucket];
158
+ tasks.push(instance.setBucketPolicy(service, credential, bucket, options).catch((err) => instance.writeFail(["Unable to update bucket policy", service + ': ' + bucket], err, { type: 64, startTime })));
159
+ }
160
+ }
161
+ for (const service in bucketTagging) {
162
+ const map = bucketTagging[service];
163
+ for (const bucket in map) {
164
+ const [credential, options] = map[bucket];
165
+ tasks.push(instance.setBucketTagging(service, credential, bucket, options).catch((err) => instance.writeFail(["Unable to update bucket tagging", service + ': ' + bucket], err, { type: 64, startTime })));
149
166
  }
150
167
  }
151
168
  if (tasks.length) {
152
169
  await instance.allSettled(tasks, ['Configure bucket', instance.moduleName]);
153
170
  if (instance.aborted) {
154
- return Promise.reject((0, types_1.createAbortError)());
171
+ return (0, types_1.createAbortError)(true);
155
172
  }
156
173
  tasks = [];
157
174
  }
158
175
  for (const { instance: document } of this.Document) {
159
176
  if (document.cloudFinalize) {
160
- await document.cloudFinalize(state).catch(err => document.writeFail(["Handled rejection", document.moduleName], err, { type: 64, startTime }));
177
+ await document.cloudFinalize(state).catch((err) => document.writeFail(["Handled rejection", document.moduleName], err, { type: 64, startTime }));
161
178
  if (document.aborted) {
162
- return Promise.reject((0, types_1.createAbortError)());
179
+ return (0, types_1.createAbortError)(true);
163
180
  }
164
181
  }
165
182
  }
@@ -188,7 +205,7 @@ class Cloud extends core_1.ClientDb {
188
205
  }
189
206
  }
190
207
  else {
191
- downloadUri = pathname ? path.join(this.baseDirectory, pathname.replace(/^([A-Z]:)?[\\/]+/i, ''), filename) : path.join(((_d = data.admin) === null || _d === void 0 ? void 0 : _d.preservePath) && localUri ? path.dirname(localUri) : this.baseDirectory, filename);
208
+ downloadUri = pathname ? path.join(this.baseDirectory, pathname.replace(/^([A-Z]:)?[\\/]+/i, ''), filename) : path.join(data.admin?.preservePath && localUri ? path.dirname(localUri) : this.baseDirectory, filename);
192
209
  }
193
210
  const destDir = path.dirname(downloadUri);
194
211
  if (Cloud.isPath(downloadUri)) {
@@ -209,7 +226,8 @@ class Cloud extends core_1.ClientDb {
209
226
  active = false;
210
227
  }
211
228
  }
212
- const location = data.service + data.bucket + filename;
229
+ const service = data.service;
230
+ const location = service + '_' + data.bucket + '_' + (download.keyname || filename);
213
231
  let pending = (downloadMap || (downloadMap = {}))[location];
214
232
  if (pending) {
215
233
  pending.add(downloadUri);
@@ -217,45 +235,47 @@ class Cloud extends core_1.ClientDb {
217
235
  }
218
236
  pending = new Set([downloadUri]);
219
237
  download.admin = data.admin;
220
- const task = instance.downloadObject(data.service, instance.getCredential(data), data.bucket, download, async (value) => {
238
+ download.flags = SERVICE_CLIENT.get(service).CLOUD_DOWNLOAD_CHUNK && (0, types_1.alignSize)(download.chunkSize) > 0 ? 4 : 0;
239
+ const task = instance.downloadObject(service, instance.getCredential(data), data.bucket, download, async (value) => {
240
+ if (instance.aborted || !value) {
241
+ return;
242
+ }
243
+ const items = Array.from(pending);
221
244
  let result;
222
- if (value && !instance.aborted) {
223
- const items = Array.from(pending);
224
- for (let i = 0, length = items.length, size, copy; i < length; ++i) {
225
- const destUri = items[i];
226
- try {
227
- if (typeof value === 'string') {
228
- if (!copy && i === length - 1) {
229
- fs.renameSync(value, destUri);
230
- }
231
- else {
232
- fs.copyFileSync(value, destUri);
233
- }
234
- size = this.addDownload(destUri);
245
+ for (let i = 0, length = items.length, size, copy; i < length; ++i) {
246
+ const destUri = items[i];
247
+ try {
248
+ if (typeof value === 'string') {
249
+ if (!copy && i === length - 1) {
250
+ fs.renameSync(value, destUri);
235
251
  }
236
252
  else {
237
- fs.writeFileSync(destUri, value);
238
- this.addDownload(size = value.length);
253
+ fs.copyFileSync(value, destUri);
239
254
  }
240
- this.add(destUri);
241
- this.formatMessage(64, data.service, ["Download success", (0, types_1.formatSize)(size)], destUri, { ...Cloud.LOG_CLOUD_DOWNLOAD });
242
- result || (result = destUri);
255
+ size = this.addDownload(destUri);
243
256
  }
244
- catch (err) {
245
- if (!copy && core_1.ClientDb.isErrorCode(err, 'EXDEV')) {
246
- copy = true;
247
- --i;
248
- }
249
- else {
250
- instance.writeFail(["Unable to write file", path.basename(destUri)], err, { type: 32, fatal: !!active, startTime });
251
- }
257
+ else {
258
+ fs.writeFileSync(destUri, value);
259
+ this.addDownload(size = value.length);
260
+ }
261
+ this.add(destUri);
262
+ this.formatMessage(64, service, ["Download success", (0, types_1.formatSize)(size)], destUri, { ...Cloud.LOG_CLOUD_DOWNLOAD });
263
+ result || (result = destUri);
264
+ }
265
+ catch (err) {
266
+ if (!copy && core_1.ClientDb.isErrorCode(err, 'EXDEV')) {
267
+ copy = true;
268
+ --i;
269
+ }
270
+ else {
271
+ instance.writeFail(["Unable to write file", path.basename(destUri)], err, { type: 32, fatal: !!active, startTime });
252
272
  }
253
273
  }
254
274
  }
255
275
  return result;
256
276
  })
257
- .catch(err => instance.writeFail(["Download failed", path.basename(downloadUri)], err, { type: 64, startTime }));
258
- if (active || waitStatus || this.incremental === 'staging') {
277
+ .catch((err) => instance.writeFail(["Download failed", path.basename(downloadUri)], err, { type: 64, startTime }));
278
+ if (active || waitStatus || this.incremental === "staging") {
259
279
  tasks.push(task);
260
280
  }
261
281
  downloadMap[location] = pending;
@@ -266,11 +286,16 @@ class Cloud extends core_1.ClientDb {
266
286
  return instance.allSettled(tasks, ['Download objects', instance.moduleName]);
267
287
  }
268
288
  }
269
- static uploadAsset(state, file, contentType = file.mimeType, ignoreProcess) {
270
- if (typeof contentType === 'boolean') {
289
+ static uploadAsset(state, file, contentType, ignoreProcess) {
290
+ let preferBuffer;
291
+ if ((0, types_1.isObject)(contentType)) {
292
+ ({ contentType, ignoreProcess, preferBuffer } = contentType);
293
+ }
294
+ else if (typeof contentType === 'boolean') {
271
295
  ignoreProcess = contentType;
272
- contentType = file.mimeType;
296
+ contentType = undefined;
273
297
  }
298
+ contentType || (contentType = file.mimeType);
274
299
  const { host, instance } = state;
275
300
  const cloudStorage = file.cloudStorage;
276
301
  if (instance.aborted || !Array.isArray(cloudStorage)) {
@@ -281,8 +306,18 @@ class Cloud extends core_1.ClientDb {
281
306
  if (!instance.hasStorage('upload', storage)) {
282
307
  continue;
283
308
  }
309
+ const client = SERVICE_CLIENT.get(storage.service);
284
310
  const upload = storage.upload;
285
311
  const active = storage === instance.getStorage('upload', cloudStorage);
312
+ let flags = client.CLOUD_UPLOAD_DISK ? 1 : 0;
313
+ if (!preferBuffer) {
314
+ if (client.CLOUD_UPLOAD_STREAM && upload.minStreamSize !== -1) {
315
+ flags |= 2;
316
+ }
317
+ if (client.CLOUD_UPLOAD_CHUNK && (0, types_1.alignSize)(upload.chunkSize) > 0) {
318
+ flags |= 4;
319
+ }
320
+ }
286
321
  if (active && upload.localStorage === false) {
287
322
  state.localStorage.set(file, upload);
288
323
  }
@@ -297,14 +332,10 @@ class Cloud extends core_1.ClientDb {
297
332
  };
298
333
  const task = new Promise(resolve => {
299
334
  const { service, bucket = state.bucketGroup, admin } = storage;
300
- let minStreamSize = upload.minStreamSize;
301
- if ((0, types_1.isString)(minStreamSize)) {
302
- minStreamSize = (0, types_1.formatSize)(minStreamSize);
303
- }
304
335
  const credential = instance.getCredential(storage, true);
336
+ const minStreamSize = (0, types_1.alignSize)(upload.minStreamSize);
305
337
  const uploading = [];
306
338
  getFiles(file, upload).forEach(async (group, index) => {
307
- var _a;
308
339
  let fileGroup;
309
340
  if (index === 0) {
310
341
  if (!group[0]) {
@@ -312,7 +343,7 @@ class Cloud extends core_1.ClientDb {
312
343
  return;
313
344
  }
314
345
  if (group.length > 1) {
315
- if ((_a = SERVICE_CLIENT.get(service)) === null || _a === void 0 ? void 0 : _a.CLOUD_UPLOAD_FROMDISK) {
346
+ if (flags > 0) {
316
347
  fileGroup = group.slice(1).filter(value => this.isPath(value, true)).map(value => [value, path.extname(value), value]);
317
348
  }
318
349
  else {
@@ -321,7 +352,7 @@ class Cloud extends core_1.ClientDb {
321
352
  const value = group[i];
322
353
  if (this.isPath(value, true)) {
323
354
  try {
324
- fileGroup.push([typeof minStreamSize === 'number' ? await this.streamFile(value, { minStreamSize, cache: false, signal: instance.signal }) : fs.readFileSync(value), path.extname(value), value]);
355
+ fileGroup.push([!isNaN(minStreamSize) ? await this.streamFile(value, { minStreamSize, cache: false, signal: instance.signal }) : fs.readFileSync(value), path.extname(value), value]);
325
356
  }
326
357
  catch (err) {
327
358
  instance.writeFail(["Unable to read file", path.basename(value)], err, { type: 32, fatal: false });
@@ -336,7 +367,7 @@ class Cloud extends core_1.ClientDb {
336
367
  const localUri = group[i];
337
368
  const exists = index === 0 || this.isPath(localUri, true);
338
369
  if (!exists || !instance.canRead(localUri, { ownPermissionOnly: true })) {
339
- instance.writeFail(["Unable to read file", path.basename(localUri)], (0, types_1.errorValue)(exists ? "Not permitted to read file" : "File not found", localUri), { type: 64, fatal: i === 0 && index === 0 });
370
+ instance.writeFail(["Unable to read file", path.basename(localUri)], (0, types_1.errorValue)(!exists ? "File not found" : "Not permitted to read file", localUri), { type: 64, fatal: i === 0 && index === 0 });
340
371
  continue;
341
372
  }
342
373
  let buffer, filename;
@@ -350,17 +381,32 @@ class Cloud extends core_1.ClientDb {
350
381
  else if (upload.overwrite) {
351
382
  filename = path.basename(localUri);
352
383
  }
353
- buffer = file.sourceUTF8 ? Buffer.from(file.sourceUTF8, file.encoding) : typeof minStreamSize === 'number' ? await host.getBuffer(file, minStreamSize).catch(() => host.getBuffer(file)) : host.getBuffer(file);
384
+ if (file.sourceUTF8) {
385
+ buffer = Buffer.from(file.sourceUTF8, file.encoding);
386
+ }
387
+ else if (!(buffer = file.buffer) && (flags & 2) === 0 && ((flags & 4) === 0 || !isNaN(minStreamSize))) {
388
+ if (!isNaN(minStreamSize)) {
389
+ buffer = await host.getBuffer(file, minStreamSize).catch(() => {
390
+ if (flags & 1) {
391
+ return Buffer.alloc(0);
392
+ }
393
+ return null;
394
+ });
395
+ }
396
+ else {
397
+ buffer = host.getBuffer(file);
398
+ }
399
+ }
354
400
  }
355
401
  else {
356
402
  contentType = this.lookupMime(path.basename(localUri)) || file.mimeType;
357
403
  }
358
- const options = { ...upload, buffer, filename, fileGroup, admin };
404
+ const options = { ...upload, buffer, filename, fileGroup, admin, flags };
359
405
  if (index > 0 || !options.contentType) {
360
406
  options.contentType = contentType;
361
407
  }
362
408
  uploading.push(instance.uploadObject(service, { ...credential }, bucket, options, localUri, callback)
363
- .catch(err => instance.writeFail(["Upload failed", path.basename(localUri)], err, { type: 64, fatal: i === 0 && index === 0 })));
409
+ .catch((err) => instance.writeFail(["Upload failed", path.basename(localUri)], err, { type: 64, fatal: i === 0 && index === 0 })));
364
410
  }
365
411
  });
366
412
  instance.allSettled(uploading, [`Upload file "${contentType || "Unknown"}"`, storage.service + ': ' + path.basename(file.localUri)]).then(() => resolve());
@@ -372,7 +418,6 @@ class Cloud extends core_1.ClientDb {
372
418
  return tasks;
373
419
  }
374
420
  static sanitizeAssets(assets) {
375
- var _a;
376
421
  const storage = [];
377
422
  for (const item of assets) {
378
423
  const cloudStorage = item.cloudStorage;
@@ -383,7 +428,7 @@ class Cloud extends core_1.ClientDb {
383
428
  if (upload.filename) {
384
429
  setUploadFilename(upload, this.toPosix(upload.filename));
385
430
  }
386
- const pathname = upload.pathname || ((_a = data.admin) === null || _a === void 0 ? void 0 : _a.preservePath) && item.pathname;
431
+ const pathname = upload.pathname || data.admin?.preservePath && item.pathname;
387
432
  if (pathname) {
388
433
  upload.pathname = this.toPosix(pathname).replace(/^\/+/, '') + '/';
389
434
  }
@@ -451,21 +496,20 @@ class Cloud extends core_1.ClientDb {
451
496
  return super.setQueryResult(service, credential, queryString, result, sessionKey);
452
497
  }
453
498
  async createBucket(service, credential, bucket, publicRead, options) {
454
- var _a, _b;
455
499
  if (this.aborted) {
456
- return Promise.reject((0, types_1.createAbortError)());
500
+ return (0, types_1.createAbortError)(true);
457
501
  }
458
502
  try {
459
503
  const client = this.getClient(service);
460
504
  try {
461
505
  if (publicRead === undefined || typeof publicRead === 'boolean') {
462
- const handler = (_a = client.createBucket) === null || _a === void 0 ? void 0 : _a.bind(this);
506
+ const handler = client.createBucket?.bind(this);
463
507
  if (handler) {
464
508
  return handler.call(this, credential, bucket, publicRead);
465
509
  }
466
510
  }
467
511
  else {
468
- const handler = (_b = client.createBucketV2) === null || _b === void 0 ? void 0 : _b.bind(this);
512
+ const handler = client.createBucketV2?.bind(this);
469
513
  if (handler) {
470
514
  return handler.call(this, credential, bucket, publicRead, options);
471
515
  }
@@ -482,12 +526,11 @@ class Cloud extends core_1.ClientDb {
482
526
  }
483
527
  }
484
528
  async setBucketPolicy(service, credential, bucket, options) {
485
- var _a;
486
529
  if (this.aborted) {
487
- return Promise.reject((0, types_1.createAbortError)());
530
+ return (0, types_1.createAbortError)(true);
488
531
  }
489
532
  try {
490
- const handler = (_a = this.getClient(service).setBucketPolicy) === null || _a === void 0 ? void 0 : _a.bind(this);
533
+ const handler = this.getClient(service).setBucketPolicy?.bind(this);
491
534
  if (handler) {
492
535
  try {
493
536
  return handler.call(this, credential, bucket, options);
@@ -503,13 +546,33 @@ class Cloud extends core_1.ClientDb {
503
546
  return Promise.reject(err);
504
547
  }
505
548
  }
549
+ async setBucketTagging(service, credential, bucket, options) {
550
+ if (this.aborted) {
551
+ return (0, types_1.createAbortError)(true);
552
+ }
553
+ try {
554
+ const handler = this.getClient(service).setBucketTagging?.bind(this);
555
+ if (handler) {
556
+ try {
557
+ return handler.call(this, credential, bucket, options);
558
+ }
559
+ catch (err) {
560
+ this.formatMessage(64, service, ["Unable to update bucket tagging", bucket], err, { ...Cloud.LOG_CLOUD_WARN });
561
+ return Promise.reject(err);
562
+ }
563
+ }
564
+ return Promise.reject((0, types_1.errorMessage)(service, "Bucket tagging not supported"));
565
+ }
566
+ catch (err) {
567
+ return Promise.reject(err);
568
+ }
569
+ }
506
570
  async setBucketWebsite(service, credential, bucket, options) {
507
- var _a;
508
571
  if (this.aborted) {
509
- return Promise.reject((0, types_1.createAbortError)());
572
+ return (0, types_1.createAbortError)(true);
510
573
  }
511
574
  try {
512
- const handler = (_a = this.getClient(service).setBucketWebsite) === null || _a === void 0 ? void 0 : _a.bind(this);
575
+ const handler = this.getClient(service).setBucketWebsite?.bind(this);
513
576
  if (handler) {
514
577
  try {
515
578
  return handler.call(this, credential, bucket, options);
@@ -526,20 +589,19 @@ class Cloud extends core_1.ClientDb {
526
589
  }
527
590
  }
528
591
  async deleteObjects(service, credential, bucket, recursive = true) {
529
- var _a, _b;
530
592
  if (this.aborted) {
531
- return Promise.reject((0, types_1.createAbortError)());
593
+ return (0, types_1.createAbortError)(true);
532
594
  }
533
595
  try {
534
596
  const errorResponse = (err) => this.formatMessage(64, service, ["Unable to empty bucket", bucket], err, { ...Cloud.LOG_CLOUD_WARN });
535
597
  const client = this.getClient(service);
536
- const handlerV2 = (_a = client.deleteObjectsV2) === null || _a === void 0 ? void 0 : _a.bind(this);
598
+ const handlerV2 = client.deleteObjectsV2?.bind(this);
537
599
  if (handlerV2) {
538
- return handlerV2.call(this, credential, bucket, recursive, service).catch(err => errorResponse(err));
600
+ return handlerV2.call(this, credential, bucket, recursive, service).catch((err) => errorResponse(err));
539
601
  }
540
- const handlerV1 = (_b = client.deleteObjects) === null || _b === void 0 ? void 0 : _b.bind(this);
602
+ const handlerV1 = client.deleteObjects?.bind(this);
541
603
  if (handlerV1) {
542
- return handlerV1.call(this, credential, bucket, service, undefined, recursive).catch(err => errorResponse(err));
604
+ return handlerV1.call(this, credential, bucket, service, undefined, recursive).catch((err) => errorResponse(err));
543
605
  }
544
606
  return Promise.reject((0, types_1.errorMessage)(service, "Delete objects not supported"));
545
607
  }
@@ -549,7 +611,7 @@ class Cloud extends core_1.ClientDb {
549
611
  }
550
612
  async uploadObject(service, credential, bucket, upload, localUri, beforeResolve) {
551
613
  if (this.aborted) {
552
- return Promise.reject((0, types_1.createAbortError)());
614
+ return (0, types_1.createAbortError)(true);
553
615
  }
554
616
  let handler;
555
617
  try {
@@ -561,7 +623,8 @@ class Cloud extends core_1.ClientDb {
561
623
  }
562
624
  return new Promise((resolve, reject) => {
563
625
  try {
564
- handler({ bucket, upload, buffer: upload.buffer || fs.readFileSync(localUri), localUri }, async (err, value) => {
626
+ const flags = upload.flags || 0;
627
+ handler({ bucket, upload, buffer: upload.buffer || ((flags & 1) === 0 && (flags & 2) === 0 && (flags & 4) === 0 ? fs.readFileSync(localUri) : Buffer.alloc(0)), localUri }, async (err, value) => {
565
628
  if (err) {
566
629
  reject(errorObject(err, service, "Upload failed"));
567
630
  }
@@ -569,7 +632,7 @@ class Cloud extends core_1.ClientDb {
569
632
  if (beforeResolve) {
570
633
  await beforeResolve(value);
571
634
  }
572
- this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> upload -> ' + value);
635
+ this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> uploadObject', bucket, value);
573
636
  resolve(value);
574
637
  }
575
638
  else {
@@ -584,7 +647,7 @@ class Cloud extends core_1.ClientDb {
584
647
  }
585
648
  async downloadObject(service, credential, bucket, download, beforeResolve) {
586
649
  if (this.aborted) {
587
- return Promise.reject((0, types_1.createAbortError)());
650
+ return (0, types_1.createAbortError)(true);
588
651
  }
589
652
  if ((service === 'gcp' || service === 'gcloud') && (0, types_1.isPlainObject)(credential)) {
590
653
  credential.storageBucket || (credential.storageBucket = bucket);
@@ -611,7 +674,7 @@ class Cloud extends core_1.ClientDb {
611
674
  }
612
675
  }
613
676
  if (typeof value === 'string' && path.isAbsolute(value)) {
614
- this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> download -> ' + value);
677
+ this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> downloadObject', bucket, value);
615
678
  this.downloaded.push(value);
616
679
  }
617
680
  resolve(value);
@@ -628,7 +691,7 @@ class Cloud extends core_1.ClientDb {
628
691
  }
629
692
  async getDatabaseRows(item, ignoreErrors, sessionKey) {
630
693
  if (this.aborted) {
631
- return Promise.reject((0, types_1.createAbortError)());
694
+ return (0, types_1.createAbortError)(true);
632
695
  }
633
696
  if (typeof ignoreErrors === 'string') {
634
697
  sessionKey = ignoreErrors;
@@ -637,9 +700,9 @@ class Cloud extends core_1.ClientDb {
637
700
  const service = item.service;
638
701
  let client;
639
702
  if (this.hasCredential('database', item) && (client = this.getClient(item.service))) {
640
- if (client === null || client === void 0 ? void 0 : client.executeQuery) {
703
+ if (client?.executeQuery) {
641
704
  const credential = this.getCredential(item);
642
- if (item.options && this.hasCoerce(service, 'options', null, credential)) {
705
+ if (item.options && this.hasCoerce(service, 'options', credential)) {
643
706
  (0, types_1.coerceObject)(item.options);
644
707
  }
645
708
  if (ignoreErrors) {
@@ -659,7 +722,7 @@ class Cloud extends core_1.ClientDb {
659
722
  }
660
723
  async getDatabaseBatchRows(batch, ignoreErrors, sessionKey) {
661
724
  if (this.aborted) {
662
- return Promise.reject((0, types_1.createAbortError)());
725
+ return (0, types_1.createAbortError)(true);
663
726
  }
664
727
  if (typeof ignoreErrors === 'string') {
665
728
  sessionKey = ignoreErrors;
@@ -669,9 +732,9 @@ class Cloud extends core_1.ClientDb {
669
732
  const service = data.service;
670
733
  let client;
671
734
  if (this.hasCredential('database', data) && (client = this.getClient(service))) {
672
- if (client === null || client === void 0 ? void 0 : client.executeBatchQuery) {
735
+ if (client?.executeBatchQuery) {
673
736
  const credential = this.getCredential(data);
674
- if (this.hasCoerce(service, 'options', null, credential)) {
737
+ if (this.hasCoerce(service, 'options', credential)) {
675
738
  batch.forEach(item => item.options && (0, types_1.coerceObject)(item.options));
676
739
  }
677
740
  if (ignoreErrors) {
@@ -741,9 +804,9 @@ class Cloud extends core_1.ClientDb {
741
804
  const client = this.getClient(data.service);
742
805
  switch (feature) {
743
806
  case 'storage':
744
- return typeof client.validateStorage === 'function' && client.validateStorage(credential, data);
807
+ return typeof client.validateStorage === 'function' && client.validateStorage(credential, data) || this.settingsOf(data.service, 'auth', 'storage') === false;
745
808
  case 'database':
746
- return typeof client.validateDatabase === 'function' && client.validateDatabase(credential, data);
809
+ return typeof client.validateDatabase === 'function' && client.validateDatabase(credential, data) || this.settingsOf(data.service, 'auth', 'database') === false;
747
810
  }
748
811
  }
749
812
  catch (err) {
@@ -758,28 +821,25 @@ class Cloud extends core_1.ClientDb {
758
821
  return (SERVICE_DOWNLOAD[service] || (SERVICE_DOWNLOAD[service] = require(this.resolveService(service, 'download')))).call(this, credential, service);
759
822
  }
760
823
  resolveService(service, folder) {
761
- var _a;
762
824
  let result;
763
825
  if (!service.startsWith('@')) {
764
- result = ((_a = this.settings.imports) === null || _a === void 0 ? void 0 : _a[service]) || util_1.IMPORTS[service];
826
+ result = this.settings.imports?.[service] || util_1.IMPORTS[service];
765
827
  }
766
- else if (!folder && !service.startsWith("@squared-functions/")) {
767
- folder = 'client';
828
+ else {
829
+ folder || (folder = 'client');
768
830
  }
769
831
  return (result || service) + (folder ? '/' + folder : '');
770
832
  }
771
833
  settingsOf(service, name, component) {
772
- var _a;
773
- const result = (_a = this.settings[service]) === null || _a === void 0 ? void 0 : _a[name];
834
+ const result = this.settings[service]?.[name];
774
835
  return component ? (0, types_1.isObject)(result) ? result[component] : undefined : result;
775
836
  }
776
837
  async commit() {
777
838
  if (this.aborted) {
778
- return Promise.reject((0, types_1.createAbortError)());
839
+ return (0, types_1.createAbortError)(true);
779
840
  }
780
841
  const items = this.pending.filter(item => !item.document).map(async (data) => {
781
- var _a;
782
- (_a = data.ignoreCache) !== null && _a !== void 0 ? _a : (data.ignoreCache = true);
842
+ data.ignoreCache ?? (data.ignoreCache = true);
783
843
  return this.getDatabaseRows(data, true);
784
844
  });
785
845
  return items.length === 0 ? false : this.allSettled(items, ["Execute unassigned queries", this.moduleName]).then(result => result.length > 0).catch(() => false);
@@ -810,9 +870,5 @@ Cloud.LOG_CLOUD_UPLOAD = Object.freeze({ titleColor: 'green' });
810
870
  Cloud.LOG_CLOUD_DOWNLOAD = Object.freeze({ titleColor: 'cyan' });
811
871
  Cloud.LOG_CLOUD_DELETE = Object.freeze({ titleColor: 'grey' });
812
872
  Cloud.LOG_CLOUD_DELAYED = Object.freeze({ titleColor: 'grey' });
813
- exports.default = Cloud;
814
873
 
815
- if (exports.default) {
816
- module.exports = exports.default;
817
- module.exports.default = exports.default;
818
- }
874
+ module.exports = Cloud;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@e-mc/cloud",
3
- "version": "0.8.10",
3
+ "version": "0.9.1",
4
4
  "description": "Cloud constructor for E-mc.",
5
5
  "main": "index.js",
6
6
  "types": "index.d.ts",
@@ -17,12 +17,12 @@
17
17
  "squared-functions"
18
18
  ],
19
19
  "author": "An Pham <anpham6@gmail.com>",
20
- "license": "MIT",
20
+ "license": "BSD 3-Clause",
21
21
  "homepage": "https://github.com/anpham6/e-mc#readme",
22
22
  "dependencies": {
23
- "@e-mc/core": "0.8.10",
24
- "@e-mc/db": "0.8.10",
25
- "@e-mc/types": "0.8.10",
23
+ "@e-mc/core": "0.9.1",
24
+ "@e-mc/db": "0.9.1",
25
+ "@e-mc/types": "0.9.1",
26
26
  "mime-types": "^2.1.35"
27
27
  }
28
28
  }
package/types/index.d.ts CHANGED
@@ -2,21 +2,25 @@ import type { ICloud, IFileManager, IModule, IScopeOrigin } from '../../types/li
2
2
  import type { BucketWebsiteOptions, CloudAsset, CloudDatabase, CloudService, CloudStorageUpload, DownloadData, UploadData } from '../../types/lib/cloud';
3
3
  import type { BatchQueryResult, QueryResult } from '../../types/lib/db';
4
4
 
5
- export interface ICloudServiceClient<T extends CloudDatabase = CloudDatabase> {
5
+ export interface ICloudServiceClient<T extends CloudDatabase = CloudDatabase, U = unknown, V = unknown> {
6
6
  CLOUD_SERVICE_NAME: string;
7
- CLOUD_UPLOAD_FROMDISK?: boolean;
8
- validateStorage?(credential: unknown, data?: CloudService): boolean;
9
- validateDatabase?(credential: unknown, data?: CloudService): boolean;
10
- createStorageClient?<U>(this: IModule, credential: unknown, service?: string): U;
11
- createDatabaseClient?<U>(this: IModule, credential: unknown, data?: CloudService): U;
12
- createBucket?(this: IModule, credential: unknown, bucket: string, publicRead?: boolean, service?: string, sdk?: string): Promise<boolean>;
13
- createBucketV2?(this: IModule, credential: unknown, bucket: string, acl?: unknown, options?: unknown, service?: string, sdk?: string): Promise<boolean>;
14
- setBucketPolicy?(this: IModule, credential: unknown, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
15
- setBucketWebsite?(this: IModule, credential: unknown, bucket: string, options: BucketWebsiteOptions, service?: string, sdk?: string): Promise<boolean>;
16
- deleteObjects?(this: IModule, credential: unknown, bucket: string, service?: string, sdk?: string, recursive?: boolean): Promise<void>;
17
- deleteObjectsV2?(this: IModule, credential: unknown, bucket: string, recursive?: boolean, service?: string, sdk?: string): Promise<void>;
18
- executeQuery?(this: ICloud, credential: unknown, data: T, sessionKey?: string): Promise<QueryResult>;
19
- executeBatchQuery?(this: ICloud, credential: unknown, batch: T[], sessionKey?: string): Promise<BatchQueryResult>;
7
+ CLOUD_UPLOAD_DISK?: boolean;
8
+ CLOUD_UPLOAD_STREAM?: boolean;
9
+ CLOUD_UPLOAD_CHUNK?: boolean;
10
+ CLOUD_DOWNLOAD_CHUNK?: boolean;
11
+ validateStorage?(credential: U, data?: CloudService): boolean;
12
+ validateDatabase?(credential: V, data?: CloudService): boolean;
13
+ createStorageClient?<W>(this: IModule, credential: U, service?: string): W;
14
+ createDatabaseClient?<W>(this: IModule, credential: V, data?: CloudService): W;
15
+ createBucket?(this: IModule, credential: U, bucket: string, publicRead?: boolean, service?: string, sdk?: string): Promise<boolean>;
16
+ createBucketV2?(this: IModule, credential: U, bucket: string, acl?: unknown, options?: unknown, service?: string, sdk?: string): Promise<boolean>;
17
+ setBucketPolicy?(this: IModule, credential: U, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
18
+ setBucketTagging?(this: IModule, credential: U, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
19
+ setBucketWebsite?(this: IModule, credential: U, bucket: string, options: BucketWebsiteOptions, service?: string, sdk?: string): Promise<boolean>;
20
+ deleteObjects?(this: IModule, credential: U, bucket: string, service?: string, sdk?: string, recursive?: boolean): Promise<void>;
21
+ deleteObjectsV2?(this: IModule, credential: U, bucket: string, recursive?: boolean, service?: string, sdk?: string): Promise<void>;
22
+ executeQuery?(this: ICloud, credential: V, data: T, sessionKey?: string): Promise<QueryResult>;
23
+ executeBatchQuery?(this: ICloud, credential: V, batch: T[], sessionKey?: string): Promise<BatchQueryResult>;
20
24
  }
21
25
 
22
26
  export interface CloudScopeOrigin<T extends IFileManager<U>, U extends CloudAsset = CloudAsset, V extends ICloud = ICloud<T>> extends Required<IScopeOrigin<T, V>> {
@@ -24,8 +28,8 @@ export interface CloudScopeOrigin<T extends IFileManager<U>, U extends CloudAsse
24
28
  localStorage: Map<U, CloudStorageUpload>;
25
29
  }
26
30
 
27
- export type ServiceHost<T> = (this: IModule, credential: unknown, service?: string, sdk?: string) => T;
31
+ export type ServiceHost<T, U = unknown> = (this: IModule, credential: U, service?: string, sdk?: string) => T;
28
32
  export type UploadCallback = (data: UploadData, callback: (err: unknown, value?: string) => void) => void;
29
- export type DownloadCallback = (data: DownloadData, callback: (err: unknown, value?: Null<BufferContent>) => void) => void;
33
+ export type DownloadCallback = (data: DownloadData, callback: (err: unknown, value?: Null<Bufferable>) => void) => void;
30
34
  export type UploadHost = ServiceHost<UploadCallback>;
31
35
  export type DownloadHost = ServiceHost<DownloadCallback>;
package/util.d.ts CHANGED
@@ -1,16 +1,12 @@
1
1
  import type { UploadContent } from '../types/lib/cloud';
2
+ import type { AuthValue } from '../types/lib/http';
2
3
 
3
4
  import type { Readable } from 'stream';
4
5
 
5
- interface AuthValue {
6
- username?: string;
7
- password?: string;
8
- }
9
-
10
6
  declare namespace util {
11
7
  const IMPORTS: Record<string, string | undefined>;
12
- function readableAsBuffer(stream: Readable): Promise<Buffer | null>;
13
- function createKeyAndBody(filename: string, items: UploadContent[], mimeType?: string | FunctionType<void>, errorCallback?: FunctionType<void>): [string[], Buffer[], string[]];
8
+ function readableAsBuffer(from: Readable): Promise<Buffer | null>;
9
+ function createKeyAndBody<T = Buffer>(filename: string, items: UploadContent[], chunkSize?: number | string | FunctionType<void>, errorCallback?: FunctionType<void> | number, flags?: number): [string[], T[], string[]];
14
10
  function generateFilename(filename: string): (i: number) => [string, boolean];
15
11
  function getBasicAuth(auth: AuthValue): string;
16
12
  function getBasicAuth(username: unknown, password?: unknown): string;
package/util.js CHANGED
@@ -1,8 +1,8 @@
1
1
  "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
2
  exports.hasBasicAuth = exports.getBasicAuth = exports.formatError = exports.generateFilename = exports.createKeyAndBody = exports.readableAsBuffer = exports.IMPORTS = void 0;
4
3
  const path = require("path");
5
4
  const fs = require("fs");
5
+ const stream = require("stream");
6
6
  const mime = require("mime-types");
7
7
  const types_1 = require("@e-mc/types");
8
8
  const util_1 = require("@e-mc/db/util");
@@ -20,11 +20,13 @@ exports.IMPORTS = {
20
20
  "minio": "@pi-r/minio",
21
21
  "oci": "@pi-r/oci"
22
22
  };
23
- async function readableAsBuffer(stream) {
23
+ async function readableAsBuffer(from) {
24
24
  return new Promise((resolve, reject) => {
25
25
  let result = null;
26
- stream
27
- .on('data', chunk => {
26
+ from.on('data', chunk => {
27
+ if (!Buffer.isBuffer(chunk)) {
28
+ chunk = Buffer.from(chunk);
29
+ }
28
30
  result = result ? Buffer.concat([result, chunk]) : chunk;
29
31
  })
30
32
  .on('end', () => resolve(result))
@@ -33,38 +35,88 @@ async function readableAsBuffer(stream) {
33
35
  });
34
36
  }
35
37
  exports.readableAsBuffer = readableAsBuffer;
36
- function createKeyAndBody(filename, items, mimeType, errorCallback) {
37
- if (typeof mimeType === 'function') {
38
- errorCallback = mimeType;
39
- mimeType = undefined;
38
+ function createKeyAndBody(filename, items, chunkSize = 0, errorCallback, flags = 0) {
39
+ let mimeType;
40
+ switch (typeof chunkSize) {
41
+ case 'function':
42
+ errorCallback = chunkSize;
43
+ chunkSize = 0;
44
+ break;
45
+ case 'string':
46
+ mimeType = chunkSize;
47
+ chunkSize = 0;
48
+ break;
49
+ }
50
+ if (typeof errorCallback === 'number') {
51
+ flags = errorCallback;
52
+ errorCallback = undefined;
40
53
  }
41
54
  const key = [];
42
55
  const body = [];
43
56
  const type = [];
44
57
  for (let [content, ext, localFile] of items) {
45
- let buffer;
46
- if (typeof content === 'string') {
47
- try {
48
- if (content === localFile || fs.existsSync(content)) {
49
- buffer = fs.readFileSync(content);
58
+ try {
59
+ let target;
60
+ if (chunkSize > 0) {
61
+ if (Buffer.isBuffer(content)) {
62
+ if (flags & 2) {
63
+ target = stream.Readable.from(content);
64
+ }
65
+ else {
66
+ target = content;
67
+ if (localFile && content.length > chunkSize) {
68
+ try {
69
+ fs.writeFileSync(localFile, content);
70
+ target = localFile;
71
+ }
72
+ catch {
73
+ }
74
+ }
75
+ }
76
+ }
77
+ else {
78
+ if (localFile) {
79
+ target = localFile;
80
+ }
81
+ else if (path.isAbsolute(content) && fs.existsSync(content)) {
82
+ target = content;
83
+ }
84
+ try {
85
+ if (target && fs.statSync(target).size <= chunkSize) {
86
+ const buffer = flags & 2 ? fs.createReadStream(target) : fs.readFileSync(target);
87
+ localFile = target;
88
+ target = buffer;
89
+ }
90
+ }
91
+ catch {
92
+ continue;
93
+ }
94
+ }
95
+ }
96
+ else if (typeof content === 'string') {
97
+ if (content === localFile || path.isAbsolute(content) && fs.existsSync(content)) {
98
+ target = flags & 2 ? fs.createReadStream(content) : fs.readFileSync(content);
50
99
  localFile = content;
51
100
  }
52
101
  else {
53
- buffer = Buffer.from(content);
102
+ target = Buffer.from(content);
103
+ if (flags & 2) {
104
+ target = stream.Readable.from(target);
105
+ }
54
106
  }
55
107
  }
56
- catch (err) {
57
- errorCallback === null || errorCallback === void 0 ? void 0 : errorCallback(err);
108
+ else if (Buffer.isBuffer(content)) {
109
+ target = flags & 2 ? stream.Readable.from(content) : content;
110
+ }
111
+ if (target) {
112
+ const output = filename + ext;
113
+ key.push(ext === '.map' && localFile ? path.basename(localFile) : output);
114
+ body.push(target);
115
+ type.push(ext !== '.map' && mime.lookup(output) || mimeType || 'application/octet-stream');
58
116
  }
59
117
  }
60
- else if (Buffer.isBuffer(content)) {
61
- buffer = content;
62
- }
63
- if (buffer) {
64
- const output = filename + ext;
65
- key.push(ext === '.map' && localFile ? path.basename(localFile) : output);
66
- body.push(buffer);
67
- type.push(ext !== '.map' && mime.lookup(output) || mimeType || 'application/octet-stream');
118
+ catch (err) {
119
+ errorCallback?.(err);
68
120
  }
69
121
  }
70
122
  return [key, body, type];