cloud-bucket 0.3.15 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bucket.d.ts +2 -2
- package/dist/bucket.js +24 -38
- package/dist/driver-aws.d.ts +3 -3
- package/dist/driver-aws.js +24 -18
- package/dist/driver-gcp.js +13 -11
- package/dist/driver-minio.d.ts +2 -2
- package/dist/driver-minio.js +5 -9
- package/dist/driver.d.ts +1 -1
- package/dist/driver.js +1 -2
- package/dist/index.d.ts +6 -6
- package/dist/index.js +11 -18
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -2
- package/dist/url-signer.js +3 -27
- package/package.json +11 -10
- package/src/bucket.ts +4 -3
- package/src/driver-aws.ts +7 -7
- package/src/driver-gcp.ts +1 -1
- package/src/driver-minio.ts +6 -5
- package/src/driver.ts +1 -1
- package/src/index.ts +8 -8
- package/src/types.ts +1 -1
package/dist/bucket.d.ts
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
import { Bucket as GoogleBucket } from '@google-cloud/storage';
|
|
3
3
|
import { S3 } from 'aws-sdk';
|
|
4
4
|
import { Readable, Writable } from 'stream';
|
|
5
|
-
import { Driver, ListCloudFilesOptions } from './driver';
|
|
6
|
-
import { BucketFile, BucketFileDeleted, BucketType, ListArg, ListOptions, ListResult } from './types';
|
|
5
|
+
import { Driver, ListCloudFilesOptions } from './driver.js';
|
|
6
|
+
import { BucketFile, BucketFileDeleted, BucketType, ListArg, ListOptions, ListResult } from './types.js';
|
|
7
7
|
export interface BucketOptions {
|
|
8
8
|
driver: Driver;
|
|
9
9
|
log: boolean;
|
package/dist/bucket.js
CHANGED
|
@@ -1,34 +1,24 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
if (k2 === undefined) k2 = k;
|
|
7
|
-
o[k2] = m[k];
|
|
8
|
-
}));
|
|
9
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
-
}) : function(o, v) {
|
|
12
|
-
o["default"] = v;
|
|
13
|
-
});
|
|
14
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
-
if (mod && mod.__esModule) return mod;
|
|
16
|
-
var result = {};
|
|
17
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
-
__setModuleDefault(result, mod);
|
|
19
|
-
return result;
|
|
20
|
-
};
|
|
21
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
-
exports.parsePrefixOrGlob = exports.parseListOptions = exports.getContentType = exports.buildFullDestPath = exports.newBucket = void 0;
|
|
23
|
-
const fs_extra_plus_1 = require("fs-extra-plus");
|
|
24
|
-
const mime_types_1 = require("mime-types");
|
|
25
|
-
const Path = __importStar(require("path"));
|
|
26
|
-
function newBucket(opts) {
|
|
1
|
+
import { glob } from 'fs-extra-plus';
|
|
2
|
+
import { lookup } from 'mime-types';
|
|
3
|
+
import * as Path from 'path';
|
|
4
|
+
const { mkdirp } = (await import('fs-extra')).default;
|
|
5
|
+
export function newBucket(opts) {
|
|
27
6
|
return new BucketImpl(opts);
|
|
28
7
|
}
|
|
29
|
-
exports.newBucket = newBucket;
|
|
30
8
|
class BucketImpl {
|
|
31
9
|
constructor(opts) {
|
|
10
|
+
Object.defineProperty(this, "driver", {
|
|
11
|
+
enumerable: true,
|
|
12
|
+
configurable: true,
|
|
13
|
+
writable: true,
|
|
14
|
+
value: void 0
|
|
15
|
+
});
|
|
16
|
+
Object.defineProperty(this, "log", {
|
|
17
|
+
enumerable: true,
|
|
18
|
+
configurable: true,
|
|
19
|
+
writable: true,
|
|
20
|
+
value: void 0
|
|
21
|
+
});
|
|
32
22
|
this.driver = opts.driver;
|
|
33
23
|
this.log = opts.log;
|
|
34
24
|
}
|
|
@@ -135,7 +125,7 @@ class BucketImpl {
|
|
|
135
125
|
const remotePath = this.driver.getPath(cf);
|
|
136
126
|
const localFilePath = (isLocalPathDir) ? getDestPath(baseDir, remotePath, localPath) : localPath;
|
|
137
127
|
const localPathDir = Path.dirname(localFilePath);
|
|
138
|
-
await
|
|
128
|
+
await mkdirp(localPathDir);
|
|
139
129
|
if (this.log) {
|
|
140
130
|
process.stdout.write(`Downloading ${this.type}://${this.name}/${remotePath} to ${localFilePath}`);
|
|
141
131
|
}
|
|
@@ -167,7 +157,7 @@ class BucketImpl {
|
|
|
167
157
|
}
|
|
168
158
|
const isLocalGlob = localPath.includes('*');
|
|
169
159
|
const { baseDir } = parsePrefixOrGlob(localPath);
|
|
170
|
-
const localFiles = await
|
|
160
|
+
const localFiles = await glob(localPath);
|
|
171
161
|
for (const localPath of localFiles) {
|
|
172
162
|
// if we have an localFileExpression (globs), then, we build the fullDestPath relative to the baseDir of the glob (last / before the first *)
|
|
173
163
|
const fullDestPath = (isLocalGlob) ? getDestPath(baseDir, localPath, remotePath) : buildFullDestPath(localPath, remotePath);
|
|
@@ -256,7 +246,7 @@ function getDestPath(baseDir, remotePath, destPathDir) {
|
|
|
256
246
|
*
|
|
257
247
|
* @throws exception if destPath is not present.
|
|
258
248
|
*/
|
|
259
|
-
function buildFullDestPath(localPath, destPath) {
|
|
249
|
+
export function buildFullDestPath(localPath, destPath) {
|
|
260
250
|
// we do not have a dest path, throw error
|
|
261
251
|
if (!destPath) {
|
|
262
252
|
throw new Error('No depthPath');
|
|
@@ -273,14 +263,12 @@ function buildFullDestPath(localPath, destPath) {
|
|
|
273
263
|
}
|
|
274
264
|
return fullDestPath;
|
|
275
265
|
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
let ct = (0, mime_types_1.lookup)(path);
|
|
266
|
+
export function getContentType(path) {
|
|
267
|
+
let ct = lookup(path);
|
|
279
268
|
let contentType = (ct) ? ct : undefined;
|
|
280
269
|
return contentType;
|
|
281
270
|
}
|
|
282
|
-
|
|
283
|
-
function parseListOptions(optsOrPrefix) {
|
|
271
|
+
export function parseListOptions(optsOrPrefix) {
|
|
284
272
|
if (optsOrPrefix == null) {
|
|
285
273
|
optsOrPrefix = ''; // for now, default
|
|
286
274
|
}
|
|
@@ -289,7 +277,6 @@ function parseListOptions(optsOrPrefix) {
|
|
|
289
277
|
const { directory, limit, marker } = opts;
|
|
290
278
|
return { prefix, glob, directory, limit, marker };
|
|
291
279
|
}
|
|
292
|
-
exports.parseListOptions = parseListOptions;
|
|
293
280
|
/**
|
|
294
281
|
* Return a clean prefix and glob when defined in the string. Clean prefix, meaning, glob less one,
|
|
295
282
|
* that can be passed to most cloud storage api.
|
|
@@ -300,7 +287,7 @@ exports.parseListOptions = parseListOptions;
|
|
|
300
287
|
* - glob is prefixOrGlob value if it is a glob, otherwise undefined.
|
|
301
288
|
* - baseDir is the eventual longest directory path without any glob char (ending with '/')
|
|
302
289
|
*/
|
|
303
|
-
function parsePrefixOrGlob(prefixOrGlob) {
|
|
290
|
+
export function parsePrefixOrGlob(prefixOrGlob) {
|
|
304
291
|
let glob;
|
|
305
292
|
let prefix;
|
|
306
293
|
let baseDir;
|
|
@@ -324,5 +311,4 @@ function parsePrefixOrGlob(prefixOrGlob) {
|
|
|
324
311
|
}
|
|
325
312
|
return { prefix, glob, baseDir };
|
|
326
313
|
}
|
|
327
|
-
exports.parsePrefixOrGlob = parsePrefixOrGlob;
|
|
328
314
|
//#endregion ---------- /Utils ----------
|
package/dist/driver-aws.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { S3 } from 'aws-sdk';
|
|
2
|
+
import type { S3 as S3_TYPE } from 'aws-sdk';
|
|
3
3
|
import { Readable, Writable } from "stream";
|
|
4
4
|
import { Driver, ListCloudFilesOptions, ListCloudFilesResult } from "./driver";
|
|
5
5
|
import { BucketFile, BucketType } from './types';
|
|
6
|
-
declare type AwsFile =
|
|
6
|
+
declare type AwsFile = S3_TYPE.Object & {
|
|
7
7
|
ContentType?: string;
|
|
8
8
|
};
|
|
9
9
|
export interface S3DriverCfg {
|
|
@@ -17,7 +17,7 @@ export declare class S3Driver implements Driver<AwsFile> {
|
|
|
17
17
|
private baseParams;
|
|
18
18
|
get type(): BucketType;
|
|
19
19
|
get name(): string;
|
|
20
|
-
constructor(s3:
|
|
20
|
+
constructor(s3: S3_TYPE, bucketName: string);
|
|
21
21
|
toFile(awsFile: AwsFile): Omit<BucketFile, 'bucket'>;
|
|
22
22
|
getPath(obj: AwsFile): string;
|
|
23
23
|
exists(path: string): Promise<boolean>;
|
package/dist/driver-aws.js
CHANGED
|
@@ -1,22 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
const micromatch = require("micromatch");
|
|
8
|
-
async function getS3Driver(cfg) {
|
|
9
|
-
const credentials = new aws_sdk_1.Credentials(cfg.access_key_id, cfg.access_key_secret);
|
|
1
|
+
import { PassThrough } from "stream";
|
|
2
|
+
const micromatch = (await import('micromatch')).default;
|
|
3
|
+
const { createReadStream, createWriteStream } = (await import('fs-extra')).default;
|
|
4
|
+
const { Credentials, S3 } = (await import('aws-sdk')).default;
|
|
5
|
+
export async function getS3Driver(cfg) {
|
|
6
|
+
const credentials = new Credentials(cfg.access_key_id, cfg.access_key_secret);
|
|
10
7
|
// Create S3 service object
|
|
11
|
-
const s3 = new
|
|
8
|
+
const s3 = new S3({ apiVersion: '2006-03-01', credentials });
|
|
12
9
|
return new S3Driver(s3, cfg.bucketName);
|
|
13
10
|
}
|
|
14
|
-
exports.getS3Driver = getS3Driver;
|
|
15
11
|
/**
|
|
16
12
|
* Custom Writable to trigger finish/close event manually on upload
|
|
17
13
|
* TODO: Needs to check if this create some side effect.
|
|
18
14
|
*/
|
|
19
|
-
class S3UploadWriteStream extends
|
|
15
|
+
class S3UploadWriteStream extends PassThrough {
|
|
20
16
|
emit(event) {
|
|
21
17
|
if (event !== 'finish' && event !== 'close') {
|
|
22
18
|
super.emit(event);
|
|
@@ -31,8 +27,20 @@ class S3UploadWriteStream extends stream_1.PassThrough {
|
|
|
31
27
|
super.emit('close');
|
|
32
28
|
}
|
|
33
29
|
}
|
|
34
|
-
class S3Driver {
|
|
30
|
+
export class S3Driver {
|
|
35
31
|
constructor(s3, bucketName) {
|
|
32
|
+
Object.defineProperty(this, "s3", {
|
|
33
|
+
enumerable: true,
|
|
34
|
+
configurable: true,
|
|
35
|
+
writable: true,
|
|
36
|
+
value: void 0
|
|
37
|
+
});
|
|
38
|
+
Object.defineProperty(this, "baseParams", {
|
|
39
|
+
enumerable: true,
|
|
40
|
+
configurable: true,
|
|
41
|
+
writable: true,
|
|
42
|
+
value: void 0
|
|
43
|
+
});
|
|
36
44
|
this.s3 = s3;
|
|
37
45
|
this.baseParams = { Bucket: bucketName };
|
|
38
46
|
}
|
|
@@ -87,7 +95,6 @@ class S3Driver {
|
|
|
87
95
|
}
|
|
88
96
|
}
|
|
89
97
|
async listCloudFiles(opts) {
|
|
90
|
-
var _a;
|
|
91
98
|
const { prefix, glob, directory, limit, marker } = opts;
|
|
92
99
|
// build the list params
|
|
93
100
|
let listParams = {};
|
|
@@ -113,7 +120,7 @@ class S3Driver {
|
|
|
113
120
|
let dirs = undefined;
|
|
114
121
|
if (directory && awsResult.CommonPrefixes) {
|
|
115
122
|
// Note: for now, match the gcp driver, undefined if empty
|
|
116
|
-
const prefixes =
|
|
123
|
+
const prefixes = awsResult.CommonPrefixes?.map(cp => cp.Prefix);
|
|
117
124
|
if (prefixes != null && prefixes.length > 0) {
|
|
118
125
|
dirs = prefixes;
|
|
119
126
|
}
|
|
@@ -141,7 +148,7 @@ class S3Driver {
|
|
|
141
148
|
const remotePath = rawFile.Key;
|
|
142
149
|
const params = { ...this.baseParams, ...{ Key: remotePath } };
|
|
143
150
|
const remoteReadStream = this.s3.getObject(params).createReadStream();
|
|
144
|
-
const localWriteStream =
|
|
151
|
+
const localWriteStream = createWriteStream(localPath);
|
|
145
152
|
const writePromise = new Promise((resolve, reject) => {
|
|
146
153
|
localWriteStream.once('close', () => {
|
|
147
154
|
resolve();
|
|
@@ -154,7 +161,7 @@ class S3Driver {
|
|
|
154
161
|
await writePromise;
|
|
155
162
|
}
|
|
156
163
|
async uploadCloudFile(localPath, remoteFilePath, contentType) {
|
|
157
|
-
const readable =
|
|
164
|
+
const readable = createReadStream(localPath);
|
|
158
165
|
const awsResult = await this.s3.putObject({ ...this.baseParams, ...{ Key: remoteFilePath, Body: readable, ContentType: contentType } }).promise();
|
|
159
166
|
// TODO: probably check the awsResult that match remoteFilePath
|
|
160
167
|
return { Key: remoteFilePath };
|
|
@@ -202,4 +209,3 @@ class S3Driver {
|
|
|
202
209
|
}
|
|
203
210
|
}
|
|
204
211
|
}
|
|
205
|
-
exports.S3Driver = S3Driver;
|
package/dist/driver-gcp.js
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const stream_1 = require("stream");
|
|
6
|
-
const micromatch = require("micromatch");
|
|
7
|
-
async function getGsDriver(cfg) {
|
|
1
|
+
import { Storage as GoogleStorage } from '@google-cloud/storage';
|
|
2
|
+
import { Readable } from "stream";
|
|
3
|
+
const micromatch = (await import('micromatch')).default;
|
|
4
|
+
export async function getGsDriver(cfg) {
|
|
8
5
|
// TODO: valid cfg
|
|
9
6
|
const googleStorageConf = {
|
|
10
7
|
projectId: cfg.project_id,
|
|
@@ -13,13 +10,18 @@ async function getGsDriver(cfg) {
|
|
|
13
10
|
private_key: cfg.private_key
|
|
14
11
|
}
|
|
15
12
|
};
|
|
16
|
-
const storage = new
|
|
13
|
+
const storage = new GoogleStorage(googleStorageConf);
|
|
17
14
|
const googleBucket = storage.bucket(cfg.bucketName);
|
|
18
15
|
return new GcpDriver(googleBucket);
|
|
19
16
|
}
|
|
20
|
-
exports.getGsDriver = getGsDriver;
|
|
21
17
|
class GcpDriver {
|
|
22
18
|
constructor(googleBucket) {
|
|
19
|
+
Object.defineProperty(this, "googleBucket", {
|
|
20
|
+
enumerable: true,
|
|
21
|
+
configurable: true,
|
|
22
|
+
writable: true,
|
|
23
|
+
value: void 0
|
|
24
|
+
});
|
|
23
25
|
this.googleBucket = googleBucket;
|
|
24
26
|
}
|
|
25
27
|
get type() {
|
|
@@ -87,7 +89,7 @@ class GcpDriver {
|
|
|
87
89
|
// get the dirs
|
|
88
90
|
const dirs = (directory && meta.prefixes) ? meta.prefixes : undefined;
|
|
89
91
|
// get pageToken (nextMarker)
|
|
90
|
-
const nextMarker = info
|
|
92
|
+
const nextMarker = info?.pageToken;
|
|
91
93
|
// if glob, filter the data further
|
|
92
94
|
let files = (!glob) ? gfList : gfList.filter(gf => micromatch.isMatch(gf.name, glob));
|
|
93
95
|
return { files, dirs, nextMarker };
|
|
@@ -139,7 +141,7 @@ class GcpDriver {
|
|
|
139
141
|
}
|
|
140
142
|
async uploadCloudContent(path, content, contentType) {
|
|
141
143
|
const googleFile = this.googleBucket.file(path);
|
|
142
|
-
const uploadReadable = new
|
|
144
|
+
const uploadReadable = new Readable();
|
|
143
145
|
return new Promise(function (resolve, reject) {
|
|
144
146
|
uploadReadable
|
|
145
147
|
.pipe(googleFile.createWriteStream({ contentType }))
|
package/dist/driver-minio.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { S3Driver, S3DriverCfg } from './driver-aws';
|
|
2
|
-
import { BucketType } from './types';
|
|
1
|
+
import { S3Driver, S3DriverCfg } from './driver-aws.js';
|
|
2
|
+
import { BucketType } from './types.js';
|
|
3
3
|
export interface MinioDriverCfg extends S3DriverCfg {
|
|
4
4
|
minio_endpoint: string;
|
|
5
5
|
}
|
package/dist/driver-minio.js
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const aws_sdk_1 = require("aws-sdk");
|
|
5
|
-
const driver_aws_1 = require("./driver-aws");
|
|
6
|
-
async function getMinioDriver(cfg) {
|
|
1
|
+
import { S3Driver } from './driver-aws.js';
|
|
2
|
+
const { S3 } = (await import('aws-sdk')).default;
|
|
3
|
+
export async function getMinioDriver(cfg) {
|
|
7
4
|
// const credentials = new Credentials(cfg.access_key_id, cfg.access_key_secret);
|
|
8
|
-
const s3 = new
|
|
5
|
+
const s3 = new S3({
|
|
9
6
|
accessKeyId: cfg.access_key_id,
|
|
10
7
|
secretAccessKey: cfg.access_key_secret,
|
|
11
8
|
endpoint: cfg.minio_endpoint,
|
|
@@ -19,8 +16,7 @@ async function getMinioDriver(cfg) {
|
|
|
19
16
|
// Create S3 service object
|
|
20
17
|
return new MinioDriver(s3, cfg.bucketName);
|
|
21
18
|
}
|
|
22
|
-
|
|
23
|
-
class MinioDriver extends driver_aws_1.S3Driver {
|
|
19
|
+
class MinioDriver extends S3Driver {
|
|
24
20
|
get type() {
|
|
25
21
|
return 'minio';
|
|
26
22
|
}
|
package/dist/driver.d.ts
CHANGED
package/dist/driver.js
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { Bucket } from './bucket';
|
|
2
|
-
import { S3DriverCfg } from './driver-aws';
|
|
3
|
-
import { GsDriverCfg } from './driver-gcp';
|
|
4
|
-
import { MinioDriverCfg } from './driver-minio';
|
|
5
|
-
import { BucketFile, ListOptions, ListResult } from './types';
|
|
6
|
-
export { signUrl, SignUrlOptions, urlSigner } from './url-signer';
|
|
1
|
+
import { Bucket } from './bucket.js';
|
|
2
|
+
import { S3DriverCfg } from './driver-aws.js';
|
|
3
|
+
import { GsDriverCfg } from './driver-gcp.js';
|
|
4
|
+
import { MinioDriverCfg } from './driver-minio.js';
|
|
5
|
+
import { BucketFile, ListOptions, ListResult } from './types.js';
|
|
6
|
+
export { signUrl, SignUrlOptions, urlSigner } from './url-signer.js';
|
|
7
7
|
export { Bucket, BucketFile, ListOptions, ListResult };
|
|
8
8
|
declare type GetBucketOptions = {
|
|
9
9
|
log?: boolean;
|
package/dist/index.js
CHANGED
|
@@ -1,34 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
const driver_minio_1 = require("./driver-minio");
|
|
8
|
-
var url_signer_1 = require("./url-signer");
|
|
9
|
-
Object.defineProperty(exports, "signUrl", { enumerable: true, get: function () { return url_signer_1.signUrl; } });
|
|
10
|
-
Object.defineProperty(exports, "urlSigner", { enumerable: true, get: function () { return url_signer_1.urlSigner; } });
|
|
11
|
-
async function getBucket(options) {
|
|
12
|
-
var _a;
|
|
1
|
+
import { newBucket } from './bucket.js';
|
|
2
|
+
import { getS3Driver } from './driver-aws.js';
|
|
3
|
+
import { getGsDriver } from './driver-gcp.js';
|
|
4
|
+
import { getMinioDriver } from './driver-minio.js';
|
|
5
|
+
export { signUrl, urlSigner } from './url-signer.js';
|
|
6
|
+
export async function getBucket(options) {
|
|
13
7
|
if (options == null) {
|
|
14
8
|
throw new Error(`ERROR - cloud-bucket - Cannot getBucket with options ${options}`);
|
|
15
9
|
}
|
|
16
|
-
const log =
|
|
10
|
+
const log = options.log ?? false; // by default, false.
|
|
17
11
|
// if has .project_id, assume GcpBucket
|
|
18
12
|
const driver = await getDriver(options);
|
|
19
|
-
const bucket =
|
|
13
|
+
const bucket = newBucket({ driver, log });
|
|
20
14
|
return bucket;
|
|
21
15
|
}
|
|
22
|
-
exports.getBucket = getBucket;
|
|
23
16
|
async function getDriver(driverCfg) {
|
|
24
17
|
if (isGsDriverCfg(driverCfg)) {
|
|
25
|
-
return
|
|
18
|
+
return getGsDriver(driverCfg);
|
|
26
19
|
}
|
|
27
20
|
else if (isMinioDriverCfg(driverCfg)) { // IMPORTANT MUST be before S3Driver, because same access_key...
|
|
28
|
-
return
|
|
21
|
+
return getMinioDriver(driverCfg);
|
|
29
22
|
}
|
|
30
23
|
else if (isS3DriverCfg(driverCfg)) {
|
|
31
|
-
return
|
|
24
|
+
return getS3Driver(driverCfg);
|
|
32
25
|
}
|
|
33
26
|
else {
|
|
34
27
|
throw new Error(`bucket config does not seem to be valid (only support Gcp and Aws for now)`);
|
package/dist/types.d.ts
CHANGED
package/dist/types.js
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
package/dist/url-signer.js
CHANGED
|
@@ -1,27 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
5
|
-
}) : (function(o, m, k, k2) {
|
|
6
|
-
if (k2 === undefined) k2 = k;
|
|
7
|
-
o[k2] = m[k];
|
|
8
|
-
}));
|
|
9
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
-
}) : function(o, v) {
|
|
12
|
-
o["default"] = v;
|
|
13
|
-
});
|
|
14
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
-
if (mod && mod.__esModule) return mod;
|
|
16
|
-
var result = {};
|
|
17
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
-
__setModuleDefault(result, mod);
|
|
19
|
-
return result;
|
|
20
|
-
};
|
|
21
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
-
exports.urlSigner = exports.signUrl = void 0;
|
|
23
|
-
const crypto = __importStar(require("crypto"));
|
|
24
|
-
function signUrl(url, opts) {
|
|
1
|
+
import * as crypto from 'crypto';
|
|
2
|
+
export function signUrl(url, opts) {
|
|
25
3
|
if (opts.type === 's3') {
|
|
26
4
|
return s3_sign_url(url, opts);
|
|
27
5
|
}
|
|
@@ -35,7 +13,6 @@ function signUrl(url, opts) {
|
|
|
35
13
|
throw new Error(`cdnSign does not support type ${opts.type} for now`);
|
|
36
14
|
}
|
|
37
15
|
}
|
|
38
|
-
exports.signUrl = signUrl;
|
|
39
16
|
/**
|
|
40
17
|
* Return a path signer based on a baseUrl and the sign option. This is the optimal way to sign many urls and should be typically used over the signUrl method.
|
|
41
18
|
*
|
|
@@ -49,7 +26,7 @@ exports.signUrl = signUrl;
|
|
|
49
26
|
* - s3 - This takes full advantage of the aws 'directory signing' like urlSigner('https://.../some/dir/*', opts) will create one signature for the folder and apply it to each sub path.
|
|
50
27
|
* - gs - While google storage does not have the same capability, there are small benefits as well on some base64 object creation (not much though). However, because of GCP small key, the signature is much faster than s3 (about 10x)
|
|
51
28
|
*/
|
|
52
|
-
function urlSigner(baseUrl, opts) {
|
|
29
|
+
export function urlSigner(baseUrl, opts) {
|
|
53
30
|
if (opts.type === 's3') {
|
|
54
31
|
return s3_urlSigner(baseUrl, opts);
|
|
55
32
|
}
|
|
@@ -66,7 +43,6 @@ function urlSigner(baseUrl, opts) {
|
|
|
66
43
|
throw new Error('urlSigner only supported for s3');
|
|
67
44
|
}
|
|
68
45
|
}
|
|
69
|
-
exports.urlSigner = urlSigner;
|
|
70
46
|
//#region ---------- S3 Signer ----------
|
|
71
47
|
function s3_urlSigner(baseUrl, opts) {
|
|
72
48
|
const isWildPolicy = baseUrl.endsWith('*');
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cloud-bucket",
|
|
3
|
-
"
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "0.4.1",
|
|
4
5
|
"description": "Simple multi cloud (Google Storage and AWS S3) bucket API",
|
|
5
6
|
"author": "Jeremy Chone <jeremy.chone@gmail.com>",
|
|
6
7
|
"main": "dist/index.js",
|
|
@@ -12,24 +13,24 @@
|
|
|
12
13
|
"node": ">=14"
|
|
13
14
|
},
|
|
14
15
|
"dependencies": {
|
|
15
|
-
"@google-cloud/storage": "^5.
|
|
16
|
+
"@google-cloud/storage": "^5.18.0",
|
|
16
17
|
"@types/micromatch": "^4.0.2",
|
|
17
18
|
"@types/mime-types": "^2.1.1",
|
|
18
|
-
"aws-sdk": "^2.
|
|
19
|
-
"fs-extra-plus": "^0.
|
|
19
|
+
"aws-sdk": "^2.1061.0",
|
|
20
|
+
"fs-extra-plus": "^0.6.0",
|
|
20
21
|
"micromatch": "^4.0.4",
|
|
21
|
-
"mime-types": "^2.1.
|
|
22
|
+
"mime-types": "^2.1.34"
|
|
22
23
|
},
|
|
23
24
|
"devDependencies": {
|
|
24
|
-
"@types/js-yaml": "^4.0.
|
|
25
|
-
"@types/mocha": "^9.
|
|
25
|
+
"@types/js-yaml": "^4.0.5",
|
|
26
|
+
"@types/mocha": "^9.1.0",
|
|
26
27
|
"js-yaml": "^4.1.0",
|
|
27
|
-
"mocha": "^9.1.
|
|
28
|
+
"mocha": "^9.1.4",
|
|
28
29
|
"ts-node": "^10.4.0",
|
|
29
|
-
"typescript": "^4.
|
|
30
|
+
"typescript": "^4.5.5"
|
|
30
31
|
},
|
|
31
32
|
"scripts": {
|
|
32
|
-
"test": "
|
|
33
|
+
"test": "TS_NODE_PROJECT='test/tsconfig.json' ./node_modules/.bin/mocha",
|
|
33
34
|
"testw": "TS_NODE_FILES=true ./node_modules/.bin/mocha --watch",
|
|
34
35
|
"testd": "TS_NODE_FILES=true ./node_modules/.bin/mocha --inspect-brk",
|
|
35
36
|
"prepare": "rm -Rf dist/ && ./node_modules/.bin/tsc"
|
package/src/bucket.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { Bucket as GoogleBucket } from '@google-cloud/storage';
|
|
2
2
|
import { S3 } from 'aws-sdk';
|
|
3
|
-
import { glob
|
|
3
|
+
import { glob } from 'fs-extra-plus';
|
|
4
4
|
import { lookup } from 'mime-types';
|
|
5
5
|
import * as Path from 'path';
|
|
6
6
|
import { Readable, Writable } from 'stream';
|
|
7
|
-
import { Driver, ListCloudFilesOptions } from './driver';
|
|
8
|
-
import { BucketFile, BucketFileDeleted, BucketType, ListArg, ListOptions, ListResult } from './types';
|
|
7
|
+
import { Driver, ListCloudFilesOptions } from './driver.js';
|
|
8
|
+
import { BucketFile, BucketFileDeleted, BucketType, ListArg, ListOptions, ListResult } from './types.js';
|
|
9
|
+
const { mkdirp } = (await import('fs-extra')).default;
|
|
9
10
|
|
|
10
11
|
export interface BucketOptions {
|
|
11
12
|
driver: Driver;
|
package/src/driver-aws.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { S3 as S3_TYPE } from 'aws-sdk';
|
|
2
2
|
import { ListObjectsV2Request } from 'aws-sdk/clients/s3';
|
|
3
|
-
import { createReadStream, createWriteStream } from 'fs-extra-plus';
|
|
4
3
|
import { PassThrough, Readable, Writable } from "stream";
|
|
5
4
|
import { Driver, ListCloudFilesOptions, ListCloudFilesResult } from "./driver";
|
|
6
5
|
import { BucketFile, BucketType } from './types';
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
const micromatch = (await import('micromatch')).default;
|
|
7
|
+
const { createReadStream, createWriteStream } = (await import('fs-extra')).default;
|
|
8
|
+
const { Credentials, S3 } = (await import('aws-sdk')).default;
|
|
9
9
|
|
|
10
10
|
// import {Object as AwsFile} from 'aws-sdk';
|
|
11
11
|
|
|
12
12
|
// type S3 = AWS.S3;
|
|
13
|
-
type AwsFile =
|
|
13
|
+
type AwsFile = S3_TYPE.Object & { ContentType?: string };
|
|
14
14
|
|
|
15
15
|
export interface S3DriverCfg {
|
|
16
16
|
bucketName: string;
|
|
@@ -46,7 +46,7 @@ class S3UploadWriteStream extends PassThrough {
|
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
export class S3Driver implements Driver<AwsFile> {
|
|
49
|
-
private s3:
|
|
49
|
+
private s3: S3_TYPE;
|
|
50
50
|
private baseParams: { Bucket: string };
|
|
51
51
|
|
|
52
52
|
get type(): BucketType {
|
|
@@ -57,7 +57,7 @@ export class S3Driver implements Driver<AwsFile> {
|
|
|
57
57
|
return this.baseParams.Bucket;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
constructor(s3:
|
|
60
|
+
constructor(s3: S3_TYPE, bucketName: string) {
|
|
61
61
|
this.s3 = s3;
|
|
62
62
|
this.baseParams = { Bucket: bucketName };
|
|
63
63
|
}
|
package/src/driver-gcp.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Bucket as GoogleBucket, File as GoogleFile, GetFilesOptions, Storage as
|
|
|
2
2
|
import { Readable, Writable } from "stream";
|
|
3
3
|
import { Driver, ListCloudFilesOptions, ListCloudFilesResult } from "./driver";
|
|
4
4
|
import { BucketFile, BucketType } from './types';
|
|
5
|
-
|
|
5
|
+
const micromatch = (await import('micromatch')).default;
|
|
6
6
|
|
|
7
7
|
export async function getGsDriver(cfg: GsDriverCfg) {
|
|
8
8
|
// TODO: valid cfg
|
package/src/driver-minio.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { S3 } from 'aws-sdk';
|
|
2
|
-
import { S3Driver, S3DriverCfg } from './driver-aws';
|
|
3
|
-
import { BucketType } from './types';
|
|
1
|
+
import type { S3 as S3_TYPE } from 'aws-sdk';
|
|
2
|
+
import { S3Driver, S3DriverCfg } from './driver-aws.js';
|
|
3
|
+
import { BucketType } from './types.js';
|
|
4
|
+
const { S3 } = (await import('aws-sdk')).default;
|
|
4
5
|
|
|
5
6
|
export interface MinioDriverCfg extends S3DriverCfg {
|
|
6
7
|
minio_endpoint: string;
|
|
@@ -33,7 +34,7 @@ class MinioDriver extends S3Driver {
|
|
|
33
34
|
|
|
34
35
|
|
|
35
36
|
|
|
36
|
-
async function bucketExists(s3:
|
|
37
|
+
async function bucketExists(s3: S3_TYPE, bucketName: string) {
|
|
37
38
|
|
|
38
39
|
return new Promise((res, rej) => {
|
|
39
40
|
s3.headBucket({
|
|
@@ -45,7 +46,7 @@ async function bucketExists(s3: S3, bucketName: string) {
|
|
|
45
46
|
|
|
46
47
|
}
|
|
47
48
|
|
|
48
|
-
async function createBucket(s3:
|
|
49
|
+
async function createBucket(s3: S3_TYPE, bucketName: string) {
|
|
49
50
|
|
|
50
51
|
// create the bucket
|
|
51
52
|
await new Promise((res, rej) => {
|
package/src/driver.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { Bucket, newBucket } from './bucket';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { BucketFile, ListOptions, ListResult } from './types';
|
|
7
|
-
|
|
8
|
-
export { signUrl, SignUrlOptions, urlSigner } from './url-signer';
|
|
1
|
+
import { Bucket, newBucket } from './bucket.js';
|
|
2
|
+
import { getS3Driver, S3DriverCfg } from './driver-aws.js';
|
|
3
|
+
import { getGsDriver, GsDriverCfg } from './driver-gcp.js';
|
|
4
|
+
import { getMinioDriver, MinioDriverCfg } from './driver-minio.js';
|
|
5
|
+
import { Driver } from './driver.js';
|
|
6
|
+
import { BucketFile, ListOptions, ListResult } from './types.js';
|
|
7
|
+
|
|
8
|
+
export { signUrl, SignUrlOptions, urlSigner } from './url-signer.js';
|
|
9
9
|
export { Bucket, BucketFile, ListOptions, ListResult };
|
|
10
10
|
|
|
11
11
|
type GetBucketOptions = { log?: boolean } & (GsDriverCfg | S3DriverCfg | MinioDriverCfg);
|
package/src/types.ts
CHANGED