@e-mc/cloud 0.8.6 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/LICENSE +10 -10
  2. package/README.md +139 -16
  3. package/index.d.ts +5 -5
  4. package/index.js +155 -89
  5. package/package.json +4 -4
  6. package/types/index.d.ts +20 -16
  7. package/util.d.ts +3 -7
  8. package/util.js +76 -24
package/LICENSE CHANGED
@@ -1,11 +1,11 @@
1
- Copyright 2024 An Pham
2
-
3
- Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
4
-
5
- 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
6
-
7
- 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
8
-
9
- 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
10
-
1
+ Copyright 2024 An Pham
2
+
3
+ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
4
+
5
+ 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
6
+
7
+ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
8
+
9
+ 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
10
+
11
11
  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # @e-mc/cloud
2
2
 
3
- * NodeJS 14
3
+ * NodeJS 14/16
4
4
  * ES2020
5
5
 
6
6
  ## General Usage
@@ -9,28 +9,29 @@
9
9
 
10
10
  ## Interface
11
11
 
12
- - https://www.unpkg.com/@e-mc/types@0.8.6/lib/index.d.ts
12
+ * [View Source](https://www.unpkg.com/@e-mc/types@0.9.0/lib/index.d.ts)
13
13
 
14
14
  ```typescript
15
15
  import type { IHost, IScopeOrigin } from "./index";
16
16
  import type { ExternalAsset } from "./asset";
17
- import type { BucketWebsiteOptions, CloudDatabase, CloudFeatures, CloudFunctions, CloudService, CloudStorage, CloudStorageDownload, CloudStorageUpload } from "./cloud";
17
+ import type { BucketWebsiteOptions, CloudDatabase, CloudFeatures, CloudFunctions, CloudService, CloudStorage, CloudStorageDownload, CloudStorageUpload, UploadAssetOptions } from "./cloud";
18
18
  import type { ClientDbConstructor, IClientDb } from "./core";
19
19
  import type { BatchQueryResult, QueryResult } from "./db";
20
20
  import type { LogMessageOptions } from "./logger";
21
- import type { CloudModule, CloudServiceOptions, CloudSettings, DbCoerceSettings } from "./settings";
21
+ import type { CloudAuthSettings, CloudModule, CloudServiceOptions, CloudSettings, DbCoerceSettings } from "./settings";
22
22
 
23
- interface ICloud extends IClientDb<IHost, CloudModule, CloudDatabase, CloudServiceOptions, DbCoerceSettings> {
23
+ interface ICloud extends IClientDb<IHost, CloudModule, CloudDatabase, CloudServiceOptions, DbCoerceSettings & CloudAuthSettings> {
24
24
  module: CloudModule;
25
25
  readonly uploaded: string[];
26
26
  readonly downloaded: string[];
27
27
  createBucket(service: string, credential: unknown, bucket: string, acl?: unknown, options?: unknown): Promise<boolean>;
28
28
  createBucket(service: string, credential: unknown, bucket: string, publicRead?: boolean): Promise<boolean>;
29
29
  setBucketPolicy(service: string, credential: unknown, bucket: string, options: unknown): Promise<boolean>;
30
+ setBucketTagging(service: string, credential: unknown, bucket: string, options: unknown): Promise<boolean>;
30
31
  setBucketWebsite(service: string, credential: unknown, bucket: string, options: BucketWebsiteOptions): Promise<boolean>;
31
32
  deleteObjects(service: string, credential: unknown, bucket: string, recursive?: boolean): Promise<void>;
32
- uploadObject(service: string, credential: unknown, bucket: string, upload: CloudStorageUpload, localUri: string, beforeResolve?: (value: string) => Promise<void> | void): Promise<string>;
33
- downloadObject(service: string, credential: unknown, bucket: string, download: CloudStorageDownload, beforeResolve?: (value: Buffer | string | null) => Promise<string | undefined> | void): Promise<Buffer | string>;
33
+ uploadObject(service: string, credential: unknown, bucket: string, upload: CloudStorageUpload, localUri: string, beforeResolve?: ((value: string) => Promise<void> | void)): Promise<string>;
34
+ downloadObject(service: string, credential: unknown, bucket: string, download: CloudStorageDownload, beforeResolve?: ((value: Buffer | string | null) => Promise<string | undefined> | void)): Promise<Buffer | string>;
34
35
  getStorage(action: CloudFunctions, data: CloudStorage[] | undefined): CloudStorage | undefined;
35
36
  hasStorage(action: CloudFunctions, storage: CloudStorage): CloudStorageUpload | false;
36
37
  getDatabaseRows(item: CloudDatabase, ignoreErrors: boolean, sessionKey?: string): Promise<QueryResult>;
@@ -40,8 +41,8 @@ interface ICloud extends IClientDb<IHost, CloudModule, CloudDatabase, CloudServi
40
41
  hasCredential(feature: CloudFeatures, data: CloudService, credential?: unknown): boolean;
41
42
  getCredential(item: CloudService, unused?: boolean): Record<string | number | symbol, unknown>;
42
43
  getSettings(service: string): Record<string, unknown> | undefined;
43
- settingsOf(service: string, name: "cache"): unknown;
44
44
  settingsOf(service: string, name: "coerce", component: keyof DbCoerceSettings): unknown;
45
+ settingsOf(service: string, name: "auth", component: keyof CloudAuthSettings): unknown;
45
46
  getUploadHandler(service: string, credential: unknown): (...args: unknown[]) => void;
46
47
  getDownloadHandler(service: string, credential: unknown): (...args: unknown[]) => void;
47
48
  resolveService(service: string, folder?: string): string;
@@ -57,22 +58,144 @@ interface CloudConstructor extends ClientDbConstructor<IHost> {
57
58
  LOG_CLOUD_DELETE: LogMessageOptions;
58
59
  LOG_CLOUD_DELAYED: LogMessageOptions;
59
60
  finalize(this: IHost, instance: ICloud): Promise<unknown>;
60
- uploadAsset(state: IScopeOrigin<IFileManager, ICloud<IFileManager>>, file: ExternalAsset, ignoreProcess: boolean): Promise<unknown>[];
61
- uploadAsset(state: IScopeOrigin<IFileManager, ICloud<IFileManager>>, file: ExternalAsset, contentType?: string, ignoreProcess?: boolean): Promise<unknown>[];
61
+ uploadAsset(state: IScopeOrigin<IFileManager, ICloud>, file: ExternalAsset, options: UploadAssetOptions): Promise<unknown>[];
62
+ uploadAsset(state: IScopeOrigin<IFileManager, ICloud>, file: ExternalAsset, ignoreProcess: boolean): Promise<unknown>[];
63
+ uploadAsset(state: IScopeOrigin<IFileManager, ICloud>, file: ExternalAsset, contentType?: string, ignoreProcess?: boolean): Promise<unknown>[];
62
64
  sanitizeAssets(assets: ExternalAsset[]): ExternalAsset[];
63
65
  readonly prototype: ICloud;
64
66
  new(module?: CloudModule, database?: CloudDatabase[], ...args: unknown[]): ICloud;
65
67
  }
68
+
69
+ interface ICloudServiceClient {
70
+ CLOUD_SERVICE_NAME: string;
71
+ CLOUD_UPLOAD_DISK?: boolean;
72
+ CLOUD_UPLOAD_STREAM?: boolean;
73
+ CLOUD_UPLOAD_CHUNK?: boolean;
74
+ CLOUD_DOWNLOAD_CHUNK?: boolean;
75
+ validateStorage?(credential: unknown, data?: CloudService): boolean;
76
+ validateDatabase?(credential: unknown, data?: CloudService): boolean;
77
+ createStorageClient?(this: IModule, credential: unknown, service?: string): unknown;
78
+ createDatabaseClient?(this: IModule, credential: unknown, data?: CloudService): unknown;
79
+ createBucket?(this: IModule, credential: unknown, bucket: string, publicRead?: boolean, service?: string, sdk?: string): Promise<boolean>;
80
+ createBucketV2?(this: IModule, credential: unknown, bucket: string, acl?: unknown, options?: unknown, service?: string, sdk?: string): Promise<boolean>;
81
+ setBucketPolicy?(this: IModule, credential: unknown, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
82
+ setBucketTagging?(this: IModule, credential: unknown, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
83
+ setBucketWebsite?(this: IModule, credential: unknown, bucket: string, options: BucketWebsiteOptions, service?: string, sdk?: string): Promise<boolean>;
84
+ deleteObjects?(this: IModule, credential: unknown, bucket: string, service?: string, sdk?: string, recursive?: boolean): Promise<void>;
85
+ deleteObjectsV2?(this: IModule, credential: unknown, bucket: string, recursive?: boolean, service?: string, sdk?: string): Promise<void>;
86
+ executeQuery?(this: ICloud, credential: unknown, data: CloudDatabase, sessionKey?: string): Promise<QueryResult>;
87
+ executeBatchQuery?(this: ICloud, credential: unknown, batch: CloudDatabase[], sessionKey?: string): Promise<BatchQueryResult>;
88
+ }
89
+ ```
90
+
91
+ ## Settings
92
+
93
+ ```typescript
94
+ import type { PermittedDirectories } from "./core";
95
+ import type { CloudServiceOptions, DbSourceOptions, PurgeComponent } from "./settings";
96
+
97
+ interface CloudModule {
98
+ // handler: "@e-mc/cloud";
99
+ extensions?: string[];
100
+ atlas?: CloudStoredCredentials;
101
+ aws?: CloudStoredCredentials;
102
+ "aws-v3"?: CloudStoredCredentials;
103
+ azure?: CloudStoredCredentials; // az
104
+ gcp?: CloudStoredCredentials; // gcloud
105
+ ibm?: CloudStoredCredentials;
106
+ oci?: CloudStoredCredentials;
107
+ minio?: CloudStoredCredentials;
108
+ settings?: {
109
+ broadcast_id?: string | string[];
110
+ users?: Record<string, Record<string, unknown>>;
111
+ cache_dir?: string;
112
+ session_expires?: number;
113
+ user_key?: Record<string, DbSourceOptions>;
114
+ imports?: StringMap;
115
+ purge?: PurgeComponent;
116
+ atlas?: CloudServiceOptions;
117
+ aws?: CloudServiceOptions;
118
+ "aws-v3"?: CloudServiceOptions;
119
+ azure?: CloudServiceOptions;
120
+ gcp?: CloudServiceOptions;
121
+ ibm?: CloudServiceOptions;
122
+ oci?: CloudServiceOptions;
123
+ minio?: CloudServiceOptions;
124
+ };
125
+ permission?: PermittedDirectories;
126
+ }
127
+
128
+ type CloudStoredCredentials = Record<string, Record<string, unknown>>;
129
+ ```
130
+
131
+ ### Example usage
132
+
133
+ ```javascript
134
+ const Cloud = require("@e-mc/cloud"); // Using @pi-r/aws
135
+
136
+ const instance = new Cloud({
137
+ aws: {
138
+ main: {
139
+ accessKeyId: "**********",
140
+ secretAccessKey: "**********"
141
+ }
142
+ },
143
+ "aws-v3": {
144
+ main: {
145
+ credentials: {
146
+ accessKeyId: "**********",
147
+ secretAccessKey: "**********",
148
+ region: "ap-northeast-1"
149
+ }
150
+ }
151
+ }
152
+ });
153
+ // instance.host = new Host();
154
+ instance.init();
155
+
156
+ const options = {
157
+ contentType: "application/tar",
158
+ acl: "authenticated-read",
159
+ chunkSize: "8mb",
160
+ overwrite: false, // Default
161
+ tags: { key_1: "value", key_2: "value" }
162
+ };
163
+ Promise.all([
164
+ // nodejs-001/archive.tar
165
+ instance.uploadObject("aws", "main", "nodejs-001", options, "/tmp/archive.tar"),
166
+ // nodejs-001/2024/01-01.tar
167
+ instance.uploadObject("aws", "main", "nodejs-001", { ...options, publicRead: true, pathname: "2024", filename: "01-01.tar" }, "/tmp/archive.tar"),
168
+ // nodejs-001/archive_1.tar
169
+ instance.uploadObject("aws", { accessKeyId: "*****", secretAccessKey: "*****" }, "nodejs-001", { overwrite: false }, "/tmp/archive.tar")
170
+ ]);
171
+
172
+ const rows = await instance.getDatabaseRows({ service: "aws-v3", credential: "main", table: "demo", key: { id: 1 } });
173
+ ```
174
+
175
+ ## NodeJS 14 LTS
176
+
177
+ Any optional fail safe dependencies were removed as of `E-mc 0.9`. The code itself will still be *ES2020* and will continue to work equivalently when self-installing these dependencies:
178
+
179
+ ### Under 15.4 + 16.0
180
+
181
+ ```sh
182
+ npm i abort-controller event-target-shim
183
+ ```
184
+
185
+ ### Under 14.17 + 15.6
186
+
187
+ ```sh
188
+ npm i uuid
66
189
  ```
67
190
 
68
191
  ## References
69
192
 
70
- - https://www.unpkg.com/@e-mc/types@0.8.6/lib/asset.d.ts
71
- - https://www.unpkg.com/@e-mc/types@0.8.6/lib/cloud.d.ts
72
- - https://www.unpkg.com/@e-mc/types@0.8.6/lib/core.d.ts
73
- - https://www.unpkg.com/@e-mc/types@0.8.6/lib/db.d.ts
74
- - https://www.unpkg.com/@e-mc/types@0.8.6/lib/logger.d.ts
75
- - https://www.unpkg.com/@e-mc/types@0.8.6/lib/settings.d.ts
193
+ - https://www.unpkg.com/@e-mc/types@0.9.0/lib/asset.d.ts
194
+ - https://www.unpkg.com/@e-mc/types@0.9.0/lib/cloud.d.ts
195
+ - https://www.unpkg.com/@e-mc/types@0.9.0/lib/core.d.ts
196
+ - https://www.unpkg.com/@e-mc/types@0.9.0/lib/db.d.ts
197
+ - https://www.unpkg.com/@e-mc/types@0.9.0/lib/logger.d.ts
198
+ - https://www.unpkg.com/@e-mc/types@0.9.0/lib/settings.d.ts
76
199
 
77
200
  ## LICENSE
78
201
 
package/index.d.ts CHANGED
@@ -1,6 +1,6 @@
1
- import type { CloudConstructor, IFileManager } from '../types/lib';
2
- import type { CloudAsset } from '../types/lib/cloud';
3
-
4
- declare const Cloud: CloudConstructor<IFileManager<CloudAsset>>;
5
-
1
+ import type { CloudConstructor, IFileManager } from '../types/lib';
2
+ import type { CloudAsset } from '../types/lib/cloud';
3
+
4
+ declare const Cloud: CloudConstructor<IFileManager<CloudAsset>>;
5
+
6
6
  export = Cloud;
package/index.js CHANGED
@@ -1,5 +1,4 @@
1
1
  "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
2
  const path = require("path");
4
3
  const fs = require("fs");
5
4
  const types_1 = require("@e-mc/types");
@@ -50,9 +49,8 @@ class Cloud extends core_1.ClientDb {
50
49
  this.downloaded = [];
51
50
  }
52
51
  static async finalize(instance) {
53
- var _a, _b;
54
52
  if (instance.aborted) {
55
- return Promise.reject((0, types_1.createAbortError)());
53
+ return (0, types_1.createAbortError)(true);
56
54
  }
57
55
  Cloud.sanitizeAssets(this.assets);
58
56
  const localStorage = new Map();
@@ -60,6 +58,7 @@ class Cloud extends core_1.ClientDb {
60
58
  const state = { host: this, instance, bucketGroup, localStorage };
61
59
  const bucketDelete = {};
62
60
  const bucketPolicy = {};
61
+ const bucketTagging = {};
63
62
  const rawFiles = [];
64
63
  const startTime = process.hrtime();
65
64
  let tasks = [], downloadMap;
@@ -76,11 +75,11 @@ class Cloud extends core_1.ClientDb {
76
75
  cloudStorage.forEach(storage => instance.formatMessage(64, storage.service, ["Upload failed", storage.bucket], (0, types_1.errorValue)("File not found", item.uri || item.filename || "Unknown"), { ...Cloud.LOG_CLOUD_WARN }));
77
76
  continue;
78
77
  }
79
- ignore: {
78
+ resume: {
80
79
  if (item.localUri || item.torrentFiles) {
81
80
  for (const { instance: document } of this.Document) {
82
81
  if (document.cloudObject?.(state, item)) {
83
- break ignore;
82
+ break resume;
84
83
  }
85
84
  }
86
85
  if (item.compress) {
@@ -92,20 +91,29 @@ class Cloud extends core_1.ClientDb {
92
91
  for (const storage of cloudStorage) {
93
92
  const { admin, bucket } = storage;
94
93
  if (admin && bucket && instance.hasCredential('storage', storage)) {
95
- const policy = admin.configBucket?.policy;
94
+ const name = storage.service;
95
+ const credential = instance.getCredential(storage, true);
96
+ const configBucket = admin.configBucket;
96
97
  if (admin.emptyBucket) {
97
- const service = bucketDelete[_a = storage.service] || (bucketDelete[_a] = {});
98
+ const service = bucketDelete[name] || (bucketDelete[name] = {});
98
99
  const items = service[bucket];
99
100
  if (!items) {
100
- service[bucket] = [instance.getCredential(storage), admin.recursive];
101
+ service[bucket] = [{ ...credential }, admin.recursive];
101
102
  }
102
103
  else if (admin.recursive === false) {
103
104
  items[1] = false;
104
105
  }
105
106
  }
106
- if (policy) {
107
- const service = bucketPolicy[_b = storage.service] || (bucketPolicy[_b] = {});
108
- service[bucket] = [storage.service, instance.getCredential(storage), bucket, policy];
107
+ if (configBucket) {
108
+ const { policy, tags } = configBucket;
109
+ if (policy) {
110
+ const service = bucketPolicy[name] || (bucketPolicy[name] = {});
111
+ service[bucket] = [{ ...credential }, policy];
112
+ }
113
+ if (tags) {
114
+ const service = bucketTagging[name] || (bucketTagging[name] = {});
115
+ service[bucket] = [{ ...credential }, tags];
116
+ }
109
117
  }
110
118
  }
111
119
  }
@@ -114,7 +122,7 @@ class Cloud extends core_1.ClientDb {
114
122
  if (tasks.length) {
115
123
  await instance.allSettled(tasks, ['Compress files', instance.moduleName]);
116
124
  if (instance.aborted) {
117
- return Promise.reject((0, types_1.createAbortError)());
125
+ return (0, types_1.createAbortError)(true);
118
126
  }
119
127
  tasks = [];
120
128
  }
@@ -122,43 +130,53 @@ class Cloud extends core_1.ClientDb {
122
130
  const map = bucketDelete[service];
123
131
  for (const bucket in map) {
124
132
  const [credential, recursive] = map[bucket];
125
- tasks.push(instance.deleteObjects(service, credential, bucket, recursive).catch(err => instance.writeFail(["Unable to empty bucket", service + ': ' + bucket], err, { type: 64, startTime })));
133
+ tasks.push(instance.deleteObjects(service, credential, bucket, recursive).catch((err) => instance.writeFail(["Unable to empty bucket", service + ': ' + bucket], err, { type: 64, startTime })));
126
134
  }
127
135
  }
128
136
  if (tasks.length) {
129
137
  await instance.allSettled(tasks, ['Empty bucket', instance.moduleName]);
130
138
  if (instance.aborted) {
131
- return Promise.reject((0, types_1.createAbortError)());
139
+ return (0, types_1.createAbortError)(true);
132
140
  }
133
141
  tasks = [];
134
142
  }
135
- rawFiles.forEach(item => tasks.push(...Cloud.uploadAsset(state, item)));
143
+ if (rawFiles.length) {
144
+ const options = { preferBuffer: process.env.EMC_CLOUD_UPLOAD_BUFFER === 'true' };
145
+ rawFiles.forEach(item => tasks.push(...Cloud.uploadAsset(state, item, options)));
146
+ }
136
147
  if (tasks.length) {
137
148
  await instance.allSettled(tasks, ['Upload raw assets', instance.moduleName]);
138
149
  if (instance.aborted) {
139
- return Promise.reject((0, types_1.createAbortError)());
150
+ return (0, types_1.createAbortError)(true);
140
151
  }
141
152
  tasks = [];
142
153
  }
143
154
  for (const service in bucketPolicy) {
144
155
  const map = bucketPolicy[service];
145
156
  for (const bucket in map) {
146
- const params = map[bucket];
147
- tasks.push(instance.setBucketPolicy(...params).catch(err => instance.writeFail(["Unable to update bucket policy", params[0] + ': ' + params[2]], err, { type: 64, startTime })));
157
+ const [credential, options] = map[bucket];
158
+ tasks.push(instance.setBucketPolicy(service, credential, bucket, options).catch((err) => instance.writeFail(["Unable to update bucket policy", service + ': ' + bucket], err, { type: 64, startTime })));
159
+ }
160
+ }
161
+ for (const service in bucketTagging) {
162
+ const map = bucketTagging[service];
163
+ for (const bucket in map) {
164
+ const [credential, options] = map[bucket];
165
+ tasks.push(instance.setBucketTagging(service, credential, bucket, options).catch((err) => instance.writeFail(["Unable to update bucket tagging", service + ': ' + bucket], err, { type: 64, startTime })));
148
166
  }
149
167
  }
150
168
  if (tasks.length) {
151
169
  await instance.allSettled(tasks, ['Configure bucket', instance.moduleName]);
152
170
  if (instance.aborted) {
153
- return Promise.reject((0, types_1.createAbortError)());
171
+ return (0, types_1.createAbortError)(true);
154
172
  }
155
173
  tasks = [];
156
174
  }
157
175
  for (const { instance: document } of this.Document) {
158
176
  if (document.cloudFinalize) {
159
- await document.cloudFinalize(state).catch(err => document.writeFail(["Handled rejection", document.moduleName], err, { type: 64, startTime }));
177
+ await document.cloudFinalize(state).catch((err) => document.writeFail(["Handled rejection", document.moduleName], err, { type: 64, startTime }));
160
178
  if (document.aborted) {
161
- return Promise.reject((0, types_1.createAbortError)());
179
+ return (0, types_1.createAbortError)(true);
162
180
  }
163
181
  }
164
182
  }
@@ -208,7 +226,8 @@ class Cloud extends core_1.ClientDb {
208
226
  active = false;
209
227
  }
210
228
  }
211
- const location = data.service + data.bucket + filename;
229
+ const service = data.service;
230
+ const location = service + '_' + data.bucket + '_' + (download.keyname || filename);
212
231
  let pending = (downloadMap || (downloadMap = {}))[location];
213
232
  if (pending) {
214
233
  pending.add(downloadUri);
@@ -216,45 +235,47 @@ class Cloud extends core_1.ClientDb {
216
235
  }
217
236
  pending = new Set([downloadUri]);
218
237
  download.admin = data.admin;
219
- const task = instance.downloadObject(data.service, instance.getCredential(data), data.bucket, download, async (value) => {
238
+ download.flags = SERVICE_CLIENT.get(service).CLOUD_DOWNLOAD_CHUNK && (0, types_1.alignSize)(download.chunkSize) > 0 ? 4 : 0;
239
+ const task = instance.downloadObject(service, instance.getCredential(data), data.bucket, download, async (value) => {
240
+ if (instance.aborted || !value) {
241
+ return;
242
+ }
243
+ const items = Array.from(pending);
220
244
  let result;
221
- if (value && !instance.aborted) {
222
- const items = Array.from(pending);
223
- for (let i = 0, length = items.length, size, copy; i < length; ++i) {
224
- const destUri = items[i];
225
- try {
226
- if (typeof value === 'string') {
227
- if (!copy && i === length - 1) {
228
- fs.renameSync(value, destUri);
229
- }
230
- else {
231
- fs.copyFileSync(value, destUri);
232
- }
233
- size = this.addDownload(destUri);
245
+ for (let i = 0, length = items.length, size, copy; i < length; ++i) {
246
+ const destUri = items[i];
247
+ try {
248
+ if (typeof value === 'string') {
249
+ if (!copy && i === length - 1) {
250
+ fs.renameSync(value, destUri);
234
251
  }
235
252
  else {
236
- fs.writeFileSync(destUri, value);
237
- this.addDownload(size = value.length);
253
+ fs.copyFileSync(value, destUri);
238
254
  }
239
- this.add(destUri);
240
- this.formatMessage(64, data.service, ["Download success", (0, types_1.formatSize)(size)], destUri, { ...Cloud.LOG_CLOUD_DOWNLOAD });
241
- result || (result = destUri);
255
+ size = this.addDownload(destUri);
242
256
  }
243
- catch (err) {
244
- if (!copy && core_1.ClientDb.isErrorCode(err, 'EXDEV')) {
245
- copy = true;
246
- --i;
247
- }
248
- else {
249
- instance.writeFail(["Unable to write file", path.basename(destUri)], err, { type: 32, fatal: !!active, startTime });
250
- }
257
+ else {
258
+ fs.writeFileSync(destUri, value);
259
+ this.addDownload(size = value.length);
260
+ }
261
+ this.add(destUri);
262
+ this.formatMessage(64, service, ["Download success", (0, types_1.formatSize)(size)], destUri, { ...Cloud.LOG_CLOUD_DOWNLOAD });
263
+ result || (result = destUri);
264
+ }
265
+ catch (err) {
266
+ if (!copy && core_1.ClientDb.isErrorCode(err, 'EXDEV')) {
267
+ copy = true;
268
+ --i;
269
+ }
270
+ else {
271
+ instance.writeFail(["Unable to write file", path.basename(destUri)], err, { type: 32, fatal: !!active, startTime });
251
272
  }
252
273
  }
253
274
  }
254
275
  return result;
255
276
  })
256
- .catch(err => instance.writeFail(["Download failed", path.basename(downloadUri)], err, { type: 64, startTime }));
257
- if (active || waitStatus || this.incremental === 'staging') {
277
+ .catch((err) => instance.writeFail(["Download failed", path.basename(downloadUri)], err, { type: 64, startTime }));
278
+ if (active || waitStatus || this.incremental === "staging") {
258
279
  tasks.push(task);
259
280
  }
260
281
  downloadMap[location] = pending;
@@ -265,11 +286,16 @@ class Cloud extends core_1.ClientDb {
265
286
  return instance.allSettled(tasks, ['Download objects', instance.moduleName]);
266
287
  }
267
288
  }
268
- static uploadAsset(state, file, contentType = file.mimeType, ignoreProcess) {
269
- if (typeof contentType === 'boolean') {
289
+ static uploadAsset(state, file, contentType, ignoreProcess) {
290
+ let preferBuffer;
291
+ if ((0, types_1.isObject)(contentType)) {
292
+ ({ contentType, ignoreProcess, preferBuffer } = contentType);
293
+ }
294
+ else if (typeof contentType === 'boolean') {
270
295
  ignoreProcess = contentType;
271
- contentType = file.mimeType;
296
+ contentType = undefined;
272
297
  }
298
+ contentType || (contentType = file.mimeType);
273
299
  const { host, instance } = state;
274
300
  const cloudStorage = file.cloudStorage;
275
301
  if (instance.aborted || !Array.isArray(cloudStorage)) {
@@ -280,8 +306,18 @@ class Cloud extends core_1.ClientDb {
280
306
  if (!instance.hasStorage('upload', storage)) {
281
307
  continue;
282
308
  }
309
+ const client = SERVICE_CLIENT.get(storage.service);
283
310
  const upload = storage.upload;
284
311
  const active = storage === instance.getStorage('upload', cloudStorage);
312
+ let flags = client.CLOUD_UPLOAD_DISK ? 1 : 0;
313
+ if (!preferBuffer) {
314
+ if (client.CLOUD_UPLOAD_STREAM && upload.minStreamSize !== -1) {
315
+ flags |= 2;
316
+ }
317
+ if (client.CLOUD_UPLOAD_CHUNK && (0, types_1.alignSize)(upload.chunkSize) > 0) {
318
+ flags |= 4;
319
+ }
320
+ }
285
321
  if (active && upload.localStorage === false) {
286
322
  state.localStorage.set(file, upload);
287
323
  }
@@ -296,11 +332,8 @@ class Cloud extends core_1.ClientDb {
296
332
  };
297
333
  const task = new Promise(resolve => {
298
334
  const { service, bucket = state.bucketGroup, admin } = storage;
299
- let minStreamSize = upload.minStreamSize;
300
- if ((0, types_1.isString)(minStreamSize)) {
301
- minStreamSize = (0, types_1.formatSize)(minStreamSize);
302
- }
303
335
  const credential = instance.getCredential(storage, true);
336
+ const minStreamSize = (0, types_1.alignSize)(upload.minStreamSize);
304
337
  const uploading = [];
305
338
  getFiles(file, upload).forEach(async (group, index) => {
306
339
  let fileGroup;
@@ -310,7 +343,7 @@ class Cloud extends core_1.ClientDb {
310
343
  return;
311
344
  }
312
345
  if (group.length > 1) {
313
- if (SERVICE_CLIENT.get(service)?.CLOUD_UPLOAD_FROMDISK) {
346
+ if (flags > 0) {
314
347
  fileGroup = group.slice(1).filter(value => this.isPath(value, true)).map(value => [value, path.extname(value), value]);
315
348
  }
316
349
  else {
@@ -319,7 +352,7 @@ class Cloud extends core_1.ClientDb {
319
352
  const value = group[i];
320
353
  if (this.isPath(value, true)) {
321
354
  try {
322
- fileGroup.push([typeof minStreamSize === 'number' ? await this.streamFile(value, { minStreamSize, cache: false, signal: instance.signal }) : fs.readFileSync(value), path.extname(value), value]);
355
+ fileGroup.push([!isNaN(minStreamSize) ? await this.streamFile(value, { minStreamSize, cache: false, signal: instance.signal }) : fs.readFileSync(value), path.extname(value), value]);
323
356
  }
324
357
  catch (err) {
325
358
  instance.writeFail(["Unable to read file", path.basename(value)], err, { type: 32, fatal: false });
@@ -334,7 +367,7 @@ class Cloud extends core_1.ClientDb {
334
367
  const localUri = group[i];
335
368
  const exists = index === 0 || this.isPath(localUri, true);
336
369
  if (!exists || !instance.canRead(localUri, { ownPermissionOnly: true })) {
337
- instance.writeFail(["Unable to read file", path.basename(localUri)], (0, types_1.errorValue)(exists ? "Not permitted to read file" : "File not found", localUri), { type: 64, fatal: i === 0 && index === 0 });
370
+ instance.writeFail(["Unable to read file", path.basename(localUri)], (0, types_1.errorValue)(!exists ? "File not found" : "Not permitted to read file", localUri), { type: 64, fatal: i === 0 && index === 0 });
338
371
  continue;
339
372
  }
340
373
  let buffer, filename;
@@ -348,17 +381,32 @@ class Cloud extends core_1.ClientDb {
348
381
  else if (upload.overwrite) {
349
382
  filename = path.basename(localUri);
350
383
  }
351
- buffer = file.sourceUTF8 ? Buffer.from(file.sourceUTF8, file.encoding) : typeof minStreamSize === 'number' ? await host.getBuffer(file, minStreamSize).catch(() => host.getBuffer(file)) : host.getBuffer(file);
384
+ if (file.sourceUTF8) {
385
+ buffer = Buffer.from(file.sourceUTF8, file.encoding);
386
+ }
387
+ else if (!(buffer = file.buffer) && (flags & 2) === 0 && ((flags & 4) === 0 || !isNaN(minStreamSize))) {
388
+ if (!isNaN(minStreamSize)) {
389
+ buffer = await host.getBuffer(file, minStreamSize).catch(() => {
390
+ if (flags & 1) {
391
+ return Buffer.alloc(0);
392
+ }
393
+ return null;
394
+ });
395
+ }
396
+ else {
397
+ buffer = host.getBuffer(file);
398
+ }
399
+ }
352
400
  }
353
401
  else {
354
402
  contentType = this.lookupMime(path.basename(localUri)) || file.mimeType;
355
403
  }
356
- const options = { ...upload, buffer, filename, fileGroup, admin };
404
+ const options = { ...upload, buffer, filename, fileGroup, admin, flags };
357
405
  if (index > 0 || !options.contentType) {
358
406
  options.contentType = contentType;
359
407
  }
360
408
  uploading.push(instance.uploadObject(service, { ...credential }, bucket, options, localUri, callback)
361
- .catch(err => instance.writeFail(["Upload failed", path.basename(localUri)], err, { type: 64, fatal: i === 0 && index === 0 })));
409
+ .catch((err) => instance.writeFail(["Upload failed", path.basename(localUri)], err, { type: 64, fatal: i === 0 && index === 0 })));
362
410
  }
363
411
  });
364
412
  instance.allSettled(uploading, [`Upload file "${contentType || "Unknown"}"`, storage.service + ': ' + path.basename(file.localUri)]).then(() => resolve());
@@ -449,7 +497,7 @@ class Cloud extends core_1.ClientDb {
449
497
  }
450
498
  async createBucket(service, credential, bucket, publicRead, options) {
451
499
  if (this.aborted) {
452
- return Promise.reject((0, types_1.createAbortError)());
500
+ return (0, types_1.createAbortError)(true);
453
501
  }
454
502
  try {
455
503
  const client = this.getClient(service);
@@ -479,7 +527,7 @@ class Cloud extends core_1.ClientDb {
479
527
  }
480
528
  async setBucketPolicy(service, credential, bucket, options) {
481
529
  if (this.aborted) {
482
- return Promise.reject((0, types_1.createAbortError)());
530
+ return (0, types_1.createAbortError)(true);
483
531
  }
484
532
  try {
485
533
  const handler = this.getClient(service).setBucketPolicy?.bind(this);
@@ -498,9 +546,30 @@ class Cloud extends core_1.ClientDb {
498
546
  return Promise.reject(err);
499
547
  }
500
548
  }
549
+ async setBucketTagging(service, credential, bucket, options) {
550
+ if (this.aborted) {
551
+ return (0, types_1.createAbortError)(true);
552
+ }
553
+ try {
554
+ const handler = this.getClient(service).setBucketTagging?.bind(this);
555
+ if (handler) {
556
+ try {
557
+ return handler.call(this, credential, bucket, options);
558
+ }
559
+ catch (err) {
560
+ this.formatMessage(64, service, ["Unable to update bucket tagging", bucket], err, { ...Cloud.LOG_CLOUD_WARN });
561
+ return Promise.reject(err);
562
+ }
563
+ }
564
+ return Promise.reject((0, types_1.errorMessage)(service, "Bucket tagging not supported"));
565
+ }
566
+ catch (err) {
567
+ return Promise.reject(err);
568
+ }
569
+ }
501
570
  async setBucketWebsite(service, credential, bucket, options) {
502
571
  if (this.aborted) {
503
- return Promise.reject((0, types_1.createAbortError)());
572
+ return (0, types_1.createAbortError)(true);
504
573
  }
505
574
  try {
506
575
  const handler = this.getClient(service).setBucketWebsite?.bind(this);
@@ -521,18 +590,18 @@ class Cloud extends core_1.ClientDb {
521
590
  }
522
591
  async deleteObjects(service, credential, bucket, recursive = true) {
523
592
  if (this.aborted) {
524
- return Promise.reject((0, types_1.createAbortError)());
593
+ return (0, types_1.createAbortError)(true);
525
594
  }
526
595
  try {
527
596
  const errorResponse = (err) => this.formatMessage(64, service, ["Unable to empty bucket", bucket], err, { ...Cloud.LOG_CLOUD_WARN });
528
597
  const client = this.getClient(service);
529
598
  const handlerV2 = client.deleteObjectsV2?.bind(this);
530
599
  if (handlerV2) {
531
- return handlerV2.call(this, credential, bucket, recursive, service).catch(err => errorResponse(err));
600
+ return handlerV2.call(this, credential, bucket, recursive, service).catch((err) => errorResponse(err));
532
601
  }
533
602
  const handlerV1 = client.deleteObjects?.bind(this);
534
603
  if (handlerV1) {
535
- return handlerV1.call(this, credential, bucket, service, undefined, recursive).catch(err => errorResponse(err));
604
+ return handlerV1.call(this, credential, bucket, service, undefined, recursive).catch((err) => errorResponse(err));
536
605
  }
537
606
  return Promise.reject((0, types_1.errorMessage)(service, "Delete objects not supported"));
538
607
  }
@@ -542,7 +611,7 @@ class Cloud extends core_1.ClientDb {
542
611
  }
543
612
  async uploadObject(service, credential, bucket, upload, localUri, beforeResolve) {
544
613
  if (this.aborted) {
545
- return Promise.reject((0, types_1.createAbortError)());
614
+ return (0, types_1.createAbortError)(true);
546
615
  }
547
616
  let handler;
548
617
  try {
@@ -554,7 +623,8 @@ class Cloud extends core_1.ClientDb {
554
623
  }
555
624
  return new Promise((resolve, reject) => {
556
625
  try {
557
- handler({ bucket, upload, buffer: upload.buffer || fs.readFileSync(localUri), localUri }, async (err, value) => {
626
+ const flags = upload.flags || 0;
627
+ handler({ bucket, upload, buffer: upload.buffer || ((flags & 1) === 0 && (flags & 2) === 0 && (flags & 4) === 0 ? fs.readFileSync(localUri) : Buffer.alloc(0)), localUri }, async (err, value) => {
558
628
  if (err) {
559
629
  reject(errorObject(err, service, "Upload failed"));
560
630
  }
@@ -562,7 +632,7 @@ class Cloud extends core_1.ClientDb {
562
632
  if (beforeResolve) {
563
633
  await beforeResolve(value);
564
634
  }
565
- this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> upload -> ' + value);
635
+ this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> uploadObject', bucket, value);
566
636
  resolve(value);
567
637
  }
568
638
  else {
@@ -577,7 +647,7 @@ class Cloud extends core_1.ClientDb {
577
647
  }
578
648
  async downloadObject(service, credential, bucket, download, beforeResolve) {
579
649
  if (this.aborted) {
580
- return Promise.reject((0, types_1.createAbortError)());
650
+ return (0, types_1.createAbortError)(true);
581
651
  }
582
652
  if ((service === 'gcp' || service === 'gcloud') && (0, types_1.isPlainObject)(credential)) {
583
653
  credential.storageBucket || (credential.storageBucket = bucket);
@@ -604,7 +674,7 @@ class Cloud extends core_1.ClientDb {
604
674
  }
605
675
  }
606
676
  if (typeof value === 'string' && path.isAbsolute(value)) {
607
- this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> download -> ' + value);
677
+ this.addLog(types_1.STATUS_TYPE.INFO, service + ' -> downloadObject', bucket, value);
608
678
  this.downloaded.push(value);
609
679
  }
610
680
  resolve(value);
@@ -621,7 +691,7 @@ class Cloud extends core_1.ClientDb {
621
691
  }
622
692
  async getDatabaseRows(item, ignoreErrors, sessionKey) {
623
693
  if (this.aborted) {
624
- return Promise.reject((0, types_1.createAbortError)());
694
+ return (0, types_1.createAbortError)(true);
625
695
  }
626
696
  if (typeof ignoreErrors === 'string') {
627
697
  sessionKey = ignoreErrors;
@@ -632,7 +702,7 @@ class Cloud extends core_1.ClientDb {
632
702
  if (this.hasCredential('database', item) && (client = this.getClient(item.service))) {
633
703
  if (client?.executeQuery) {
634
704
  const credential = this.getCredential(item);
635
- if (item.options && this.hasCoerce(service, 'options', null, credential)) {
705
+ if (item.options && this.hasCoerce(service, 'options', credential)) {
636
706
  (0, types_1.coerceObject)(item.options);
637
707
  }
638
708
  if (ignoreErrors) {
@@ -652,7 +722,7 @@ class Cloud extends core_1.ClientDb {
652
722
  }
653
723
  async getDatabaseBatchRows(batch, ignoreErrors, sessionKey) {
654
724
  if (this.aborted) {
655
- return Promise.reject((0, types_1.createAbortError)());
725
+ return (0, types_1.createAbortError)(true);
656
726
  }
657
727
  if (typeof ignoreErrors === 'string') {
658
728
  sessionKey = ignoreErrors;
@@ -664,7 +734,7 @@ class Cloud extends core_1.ClientDb {
664
734
  if (this.hasCredential('database', data) && (client = this.getClient(service))) {
665
735
  if (client?.executeBatchQuery) {
666
736
  const credential = this.getCredential(data);
667
- if (this.hasCoerce(service, 'options', null, credential)) {
737
+ if (this.hasCoerce(service, 'options', credential)) {
668
738
  batch.forEach(item => item.options && (0, types_1.coerceObject)(item.options));
669
739
  }
670
740
  if (ignoreErrors) {
@@ -734,9 +804,9 @@ class Cloud extends core_1.ClientDb {
734
804
  const client = this.getClient(data.service);
735
805
  switch (feature) {
736
806
  case 'storage':
737
- return typeof client.validateStorage === 'function' && client.validateStorage(credential, data);
807
+ return typeof client.validateStorage === 'function' && client.validateStorage(credential, data) || this.settingsOf(data.service, 'auth', 'storage') === false;
738
808
  case 'database':
739
- return typeof client.validateDatabase === 'function' && client.validateDatabase(credential, data);
809
+ return typeof client.validateDatabase === 'function' && client.validateDatabase(credential, data) || this.settingsOf(data.service, 'auth', 'database') === false;
740
810
  }
741
811
  }
742
812
  catch (err) {
@@ -755,8 +825,8 @@ class Cloud extends core_1.ClientDb {
755
825
  if (!service.startsWith('@')) {
756
826
  result = this.settings.imports?.[service] || util_1.IMPORTS[service];
757
827
  }
758
- else if (!folder && !service.startsWith("@squared-functions/")) {
759
- folder = 'client';
828
+ else {
829
+ folder || (folder = 'client');
760
830
  }
761
831
  return (result || service) + (folder ? '/' + folder : '');
762
832
  }
@@ -766,7 +836,7 @@ class Cloud extends core_1.ClientDb {
766
836
  }
767
837
  async commit() {
768
838
  if (this.aborted) {
769
- return Promise.reject((0, types_1.createAbortError)());
839
+ return (0, types_1.createAbortError)(true);
770
840
  }
771
841
  const items = this.pending.filter(item => !item.document).map(async (data) => {
772
842
  data.ignoreCache ?? (data.ignoreCache = true);
@@ -800,9 +870,5 @@ Cloud.LOG_CLOUD_UPLOAD = Object.freeze({ titleColor: 'green' });
800
870
  Cloud.LOG_CLOUD_DOWNLOAD = Object.freeze({ titleColor: 'cyan' });
801
871
  Cloud.LOG_CLOUD_DELETE = Object.freeze({ titleColor: 'grey' });
802
872
  Cloud.LOG_CLOUD_DELAYED = Object.freeze({ titleColor: 'grey' });
803
- exports.default = Cloud;
804
873
 
805
- if (exports.default) {
806
- module.exports = exports.default;
807
- module.exports.default = exports.default;
808
- }
874
+ module.exports = Cloud;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@e-mc/cloud",
3
- "version": "0.8.6",
3
+ "version": "0.9.0",
4
4
  "description": "Cloud constructor for E-mc.",
5
5
  "main": "index.js",
6
6
  "types": "index.d.ts",
@@ -20,9 +20,9 @@
20
20
  "license": "BSD 3-Clause",
21
21
  "homepage": "https://github.com/anpham6/e-mc#readme",
22
22
  "dependencies": {
23
- "@e-mc/core": "0.8.6",
24
- "@e-mc/db": "0.8.6",
25
- "@e-mc/types": "0.8.6",
23
+ "@e-mc/core": "0.9.0",
24
+ "@e-mc/db": "0.9.0",
25
+ "@e-mc/types": "0.9.0",
26
26
  "mime-types": "^2.1.35"
27
27
  }
28
28
  }
package/types/index.d.ts CHANGED
@@ -2,21 +2,25 @@ import type { ICloud, IFileManager, IModule, IScopeOrigin } from '../../types/li
2
2
  import type { BucketWebsiteOptions, CloudAsset, CloudDatabase, CloudService, CloudStorageUpload, DownloadData, UploadData } from '../../types/lib/cloud';
3
3
  import type { BatchQueryResult, QueryResult } from '../../types/lib/db';
4
4
 
5
- export interface ICloudServiceClient<T extends CloudDatabase = CloudDatabase> {
5
+ export interface ICloudServiceClient<T extends CloudDatabase = CloudDatabase, U = unknown, V = unknown> {
6
6
  CLOUD_SERVICE_NAME: string;
7
- CLOUD_UPLOAD_FROMDISK?: boolean;
8
- validateStorage?(credential: unknown, data?: CloudService): boolean;
9
- validateDatabase?(credential: unknown, data?: CloudService): boolean;
10
- createStorageClient?<U>(this: IModule, credential: unknown, service?: string): U;
11
- createDatabaseClient?<U>(this: IModule, credential: unknown, data?: CloudService): U;
12
- createBucket?(this: IModule, credential: unknown, bucket: string, publicRead?: boolean, service?: string, sdk?: string): Promise<boolean>;
13
- createBucketV2?(this: IModule, credential: unknown, bucket: string, acl?: unknown, options?: unknown, service?: string, sdk?: string): Promise<boolean>;
14
- setBucketPolicy?(this: IModule, credential: unknown, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
15
- setBucketWebsite?(this: IModule, credential: unknown, bucket: string, options: BucketWebsiteOptions, service?: string, sdk?: string): Promise<boolean>;
16
- deleteObjects?(this: IModule, credential: unknown, bucket: string, service?: string, sdk?: string, recursive?: boolean): Promise<void>;
17
- deleteObjectsV2?(this: IModule, credential: unknown, bucket: string, recursive?: boolean, service?: string, sdk?: string): Promise<void>;
18
- executeQuery?(this: ICloud, credential: unknown, data: T, sessionKey?: string): Promise<QueryResult>;
19
- executeBatchQuery?(this: ICloud, credential: unknown, batch: T[], sessionKey?: string): Promise<BatchQueryResult>;
7
+ CLOUD_UPLOAD_DISK?: boolean;
8
+ CLOUD_UPLOAD_STREAM?: boolean;
9
+ CLOUD_UPLOAD_CHUNK?: boolean;
10
+ CLOUD_DOWNLOAD_CHUNK?: boolean;
11
+ validateStorage?(credential: U, data?: CloudService): boolean;
12
+ validateDatabase?(credential: V, data?: CloudService): boolean;
13
+ createStorageClient?<W>(this: IModule, credential: U, service?: string): W;
14
+ createDatabaseClient?<W>(this: IModule, credential: V, data?: CloudService): W;
15
+ createBucket?(this: IModule, credential: U, bucket: string, publicRead?: boolean, service?: string, sdk?: string): Promise<boolean>;
16
+ createBucketV2?(this: IModule, credential: U, bucket: string, acl?: unknown, options?: unknown, service?: string, sdk?: string): Promise<boolean>;
17
+ setBucketPolicy?(this: IModule, credential: U, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
18
+ setBucketTagging?(this: IModule, credential: U, bucket: string, options: unknown, service?: string, sdk?: string): Promise<boolean>;
19
+ setBucketWebsite?(this: IModule, credential: U, bucket: string, options: BucketWebsiteOptions, service?: string, sdk?: string): Promise<boolean>;
20
+ deleteObjects?(this: IModule, credential: U, bucket: string, service?: string, sdk?: string, recursive?: boolean): Promise<void>;
21
+ deleteObjectsV2?(this: IModule, credential: U, bucket: string, recursive?: boolean, service?: string, sdk?: string): Promise<void>;
22
+ executeQuery?(this: ICloud, credential: V, data: T, sessionKey?: string): Promise<QueryResult>;
23
+ executeBatchQuery?(this: ICloud, credential: V, batch: T[], sessionKey?: string): Promise<BatchQueryResult>;
20
24
  }
21
25
 
22
26
  export interface CloudScopeOrigin<T extends IFileManager<U>, U extends CloudAsset = CloudAsset, V extends ICloud = ICloud<T>> extends Required<IScopeOrigin<T, V>> {
@@ -24,8 +28,8 @@ export interface CloudScopeOrigin<T extends IFileManager<U>, U extends CloudAsse
24
28
  localStorage: Map<U, CloudStorageUpload>;
25
29
  }
26
30
 
27
- export type ServiceHost<T> = (this: IModule, credential: unknown, service?: string, sdk?: string) => T;
31
+ export type ServiceHost<T, U = unknown> = (this: IModule, credential: U, service?: string, sdk?: string) => T;
28
32
  export type UploadCallback = (data: UploadData, callback: (err: unknown, value?: string) => void) => void;
29
- export type DownloadCallback = (data: DownloadData, callback: (err: unknown, value?: Null<BufferContent>) => void) => void;
33
+ export type DownloadCallback = (data: DownloadData, callback: (err: unknown, value?: Null<Bufferable>) => void) => void;
30
34
  export type UploadHost = ServiceHost<UploadCallback>;
31
35
  export type DownloadHost = ServiceHost<DownloadCallback>;
package/util.d.ts CHANGED
@@ -1,16 +1,12 @@
1
1
  import type { UploadContent } from '../types/lib/cloud';
2
+ import type { AuthValue } from '../types/lib/http';
2
3
 
3
4
  import type { Readable } from 'stream';
4
5
 
5
- interface AuthValue {
6
- username?: string;
7
- password?: string;
8
- }
9
-
10
6
  declare namespace util {
11
7
  const IMPORTS: Record<string, string | undefined>;
12
- function readableAsBuffer(stream: Readable): Promise<Buffer | null>;
13
- function createKeyAndBody(filename: string, items: UploadContent[], mimeType?: string | FunctionType<void>, errorCallback?: FunctionType<void>): [string[], Buffer[], string[]];
8
+ function readableAsBuffer(from: Readable): Promise<Buffer | null>;
9
+ function createKeyAndBody<T = Buffer>(filename: string, items: UploadContent[], chunkSize?: number | string | FunctionType<void>, errorCallback?: FunctionType<void> | number, flags?: number): [string[], T[], string[]];
14
10
  function generateFilename(filename: string): (i: number) => [string, boolean];
15
11
  function getBasicAuth(auth: AuthValue): string;
16
12
  function getBasicAuth(username: unknown, password?: unknown): string;
package/util.js CHANGED
@@ -1,8 +1,8 @@
1
1
  "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
2
  exports.hasBasicAuth = exports.getBasicAuth = exports.formatError = exports.generateFilename = exports.createKeyAndBody = exports.readableAsBuffer = exports.IMPORTS = void 0;
4
3
  const path = require("path");
5
4
  const fs = require("fs");
5
+ const stream = require("stream");
6
6
  const mime = require("mime-types");
7
7
  const types_1 = require("@e-mc/types");
8
8
  const util_1 = require("@e-mc/db/util");
@@ -20,11 +20,13 @@ exports.IMPORTS = {
20
20
  "minio": "@pi-r/minio",
21
21
  "oci": "@pi-r/oci"
22
22
  };
23
- async function readableAsBuffer(stream) {
23
+ async function readableAsBuffer(from) {
24
24
  return new Promise((resolve, reject) => {
25
25
  let result = null;
26
- stream
27
- .on('data', chunk => {
26
+ from.on('data', chunk => {
27
+ if (!Buffer.isBuffer(chunk)) {
28
+ chunk = Buffer.from(chunk);
29
+ }
28
30
  result = result ? Buffer.concat([result, chunk]) : chunk;
29
31
  })
30
32
  .on('end', () => resolve(result))
@@ -33,38 +35,88 @@ async function readableAsBuffer(stream) {
33
35
  });
34
36
  }
35
37
  exports.readableAsBuffer = readableAsBuffer;
36
- function createKeyAndBody(filename, items, mimeType, errorCallback) {
37
- if (typeof mimeType === 'function') {
38
- errorCallback = mimeType;
39
- mimeType = undefined;
38
+ function createKeyAndBody(filename, items, chunkSize = 0, errorCallback, flags = 0) {
39
+ let mimeType;
40
+ switch (typeof chunkSize) {
41
+ case 'function':
42
+ errorCallback = chunkSize;
43
+ chunkSize = 0;
44
+ break;
45
+ case 'string':
46
+ mimeType = chunkSize;
47
+ chunkSize = 0;
48
+ break;
49
+ }
50
+ if (typeof errorCallback === 'number') {
51
+ flags = errorCallback;
52
+ errorCallback = undefined;
40
53
  }
41
54
  const key = [];
42
55
  const body = [];
43
56
  const type = [];
44
57
  for (let [content, ext, localFile] of items) {
45
- let buffer;
46
- if (typeof content === 'string') {
47
- try {
48
- if (content === localFile || fs.existsSync(content)) {
49
- buffer = fs.readFileSync(content);
58
+ try {
59
+ let target;
60
+ if (chunkSize > 0) {
61
+ if (Buffer.isBuffer(content)) {
62
+ if (flags & 2) {
63
+ target = stream.Readable.from(content);
64
+ }
65
+ else {
66
+ target = content;
67
+ if (localFile && content.length > chunkSize) {
68
+ try {
69
+ fs.writeFileSync(localFile, content);
70
+ target = localFile;
71
+ }
72
+ catch {
73
+ }
74
+ }
75
+ }
76
+ }
77
+ else {
78
+ if (localFile) {
79
+ target = localFile;
80
+ }
81
+ else if (path.isAbsolute(content) && fs.existsSync(content)) {
82
+ target = content;
83
+ }
84
+ try {
85
+ if (target && fs.statSync(target).size <= chunkSize) {
86
+ const buffer = flags & 2 ? fs.createReadStream(target) : fs.readFileSync(target);
87
+ localFile = target;
88
+ target = buffer;
89
+ }
90
+ }
91
+ catch {
92
+ continue;
93
+ }
94
+ }
95
+ }
96
+ else if (typeof content === 'string') {
97
+ if (content === localFile || path.isAbsolute(content) && fs.existsSync(content)) {
98
+ target = flags & 2 ? fs.createReadStream(content) : fs.readFileSync(content);
50
99
  localFile = content;
51
100
  }
52
101
  else {
53
- buffer = Buffer.from(content);
102
+ target = Buffer.from(content);
103
+ if (flags & 2) {
104
+ target = stream.Readable.from(target);
105
+ }
54
106
  }
55
107
  }
56
- catch (err) {
57
- errorCallback?.(err);
108
+ else if (Buffer.isBuffer(content)) {
109
+ target = flags & 2 ? stream.Readable.from(content) : content;
110
+ }
111
+ if (target) {
112
+ const output = filename + ext;
113
+ key.push(ext === '.map' && localFile ? path.basename(localFile) : output);
114
+ body.push(target);
115
+ type.push(ext !== '.map' && mime.lookup(output) || mimeType || 'application/octet-stream');
58
116
  }
59
117
  }
60
- else if (Buffer.isBuffer(content)) {
61
- buffer = content;
62
- }
63
- if (buffer) {
64
- const output = filename + ext;
65
- key.push(ext === '.map' && localFile ? path.basename(localFile) : output);
66
- body.push(buffer);
67
- type.push(ext !== '.map' && mime.lookup(output) || mimeType || 'application/octet-stream');
118
+ catch (err) {
119
+ errorCallback?.(err);
68
120
  }
69
121
  }
70
122
  return [key, body, type];