@naturalcycles/cloud-storage-lib 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,13 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.runCommonStorageTest = exports.InMemoryCommonStorage = exports.CommonStorageBucket = exports.CommonStorageKeyValueDB = exports.CloudStorage = void 0;
4
+ const cloudStorage_1 = require("./cloudStorage");
5
+ Object.defineProperty(exports, "CloudStorage", { enumerable: true, get: function () { return cloudStorage_1.CloudStorage; } });
6
+ const commonStorageBucket_1 = require("./commonStorageBucket");
7
+ Object.defineProperty(exports, "CommonStorageBucket", { enumerable: true, get: function () { return commonStorageBucket_1.CommonStorageBucket; } });
8
+ const commonStorageKeyValueDB_1 = require("./commonStorageKeyValueDB");
9
+ Object.defineProperty(exports, "CommonStorageKeyValueDB", { enumerable: true, get: function () { return commonStorageKeyValueDB_1.CommonStorageKeyValueDB; } });
10
+ const inMemoryCommonStorage_1 = require("./inMemoryCommonStorage");
11
+ Object.defineProperty(exports, "InMemoryCommonStorage", { enumerable: true, get: function () { return inMemoryCommonStorage_1.InMemoryCommonStorage; } });
12
+ const commonStorageTest_1 = require("./testing/commonStorageTest");
13
+ Object.defineProperty(exports, "runCommonStorageTest", { enumerable: true, get: function () { return commonStorageTest_1.runCommonStorageTest; } });
@@ -0,0 +1,5 @@
1
+ export interface GCPServiceAccount {
2
+ client_email: string;
3
+ private_key: string;
4
+ project_id: string;
5
+ }
package/dist/model.js ADDED
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1,5 @@
1
+ import { CommonStorage } from '../commonStorage';
2
+ /**
3
+ * This test suite must be idempotent.
4
+ */
5
+ export declare function runCommonStorageTest(storage: CommonStorage, bucketName: string): void;
@@ -0,0 +1,108 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.runCommonStorageTest = void 0;
4
+ const js_lib_1 = require("@naturalcycles/js-lib");
5
+ const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
6
+ const TEST_FOLDER = 'test/subdir';
7
+ const TEST_ITEMS = (0, js_lib_1._range)(10).map(n => ({
8
+ id: `id_${n + 1}`,
9
+ n,
10
+ even: n % 2 === 0,
11
+ }));
12
+ const TEST_ITEMS2 = (0, js_lib_1._range)(10).map(n => ({
13
+ fileType: 2,
14
+ id: `id_${n + 1}`,
15
+ n,
16
+ even: n % 2 === 0,
17
+ }));
18
+ const TEST_ITEMS3 = (0, js_lib_1._range)(10).map(n => ({
19
+ fileType: 3,
20
+ id: `id_${n + 1}`,
21
+ n,
22
+ even: n % 2 === 0,
23
+ }));
24
+ const TEST_FILES = [TEST_ITEMS, TEST_ITEMS2, TEST_ITEMS3].map((obj, i) => ({
25
+ filePath: `${TEST_FOLDER}/file_${i + 1}.json`,
26
+ content: Buffer.from(JSON.stringify(obj)),
27
+ }));
28
+ /**
29
+ * This test suite must be idempotent.
30
+ */
31
+ function runCommonStorageTest(storage, bucketName) {
32
+ // test('createBucket', async () => {
33
+ // await storage.createBucket(bucketName)
34
+ // })
35
+ test('ping', async () => {
36
+ await storage.ping();
37
+ });
38
+ test('listBuckets', async () => {
39
+ const buckets = await storage.getBucketNames();
40
+ console.log(buckets);
41
+ });
42
+ test('streamBuckets', async () => {
43
+ const buckets = await (0, nodejs_lib_1.readableToArray)(storage.getBucketNamesStream());
44
+ console.log(buckets);
45
+ });
46
+ test('prepare: clear bucket', async () => {
47
+ await (0, js_lib_1.pMap)(TEST_FILES.map(f => f.filePath), async (filePath) => await storage.deletePath(bucketName, filePath));
48
+ });
49
+ test('listFileNames on root should return empty', async () => {
50
+ const fileNames = await storage.getFileNames(bucketName, '');
51
+ expect(fileNames).toEqual([]);
52
+ });
53
+ test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
54
+ const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER);
55
+ expect(fileNames).toEqual([]);
56
+ });
57
+ test('streamFileNames on root should return empty', async () => {
58
+ const fileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, ''));
59
+ expect(fileNames).toEqual([]);
60
+ });
61
+ test(`exists should return empty array`, async () => {
62
+ await (0, js_lib_1.pMap)(TEST_FILES, async (f) => {
63
+ const exists = await storage.fileExists(bucketName, f.filePath);
64
+ expect(exists).toBe(false);
65
+ });
66
+ });
67
+ test(`saveFiles, then listFileNames, streamFileNames and getFiles should return just saved files`, async () => {
68
+ const testFilesMap = Object.fromEntries(TEST_FILES.map(f => [f.filePath, f.content]));
69
+ // It's done in the same test to ensure "strong consistency"
70
+ await (0, js_lib_1.pMap)(TEST_FILES, async (f) => await storage.saveFile(bucketName, f.filePath, f.content));
71
+ const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER);
72
+ expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
73
+ const streamedFileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, TEST_FOLDER));
74
+ expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
75
+ const filesMap = {};
76
+ await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
77
+ filesMap[filePath] = (await storage.getFile(bucketName, filePath));
78
+ });
79
+ expect(filesMap).toEqual(testFilesMap);
80
+ await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
81
+ const exists = await storage.fileExists(bucketName, filePath);
82
+ expect(exists).toBe(true);
83
+ });
84
+ });
85
+ test('cleanup', async () => {
86
+ await storage.deletePath(bucketName, '');
87
+ });
88
+ // Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access
89
+ /*
90
+ test(`get/set FilesVisibility`, async () => {
91
+ const fileNames = TEST_FILES.map(f => f.filePath)
92
+
93
+ let map = await storage.getFilesVisibility(bucketName, fileNames)
94
+ expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
95
+
96
+ await storage.setFilesVisibility(bucketName, fileNames, true)
97
+
98
+ map = await storage.getFilesVisibility(bucketName, fileNames)
99
+ expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, true])))
100
+
101
+ await storage.setFilesVisibility(bucketName, fileNames, false)
102
+
103
+ map = await storage.getFilesVisibility(bucketName, fileNames)
104
+ expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
105
+ })
106
+ */
107
+ }
108
+ exports.runCommonStorageTest = runCommonStorageTest;
package/package.json ADDED
@@ -0,0 +1,41 @@
1
+ {
2
+ "name": "@naturalcycles/cloud-storage-lib",
3
+ "scripts": {
4
+ "prepare": "husky install"
5
+ },
6
+ "dependencies": {
7
+ "@google-cloud/storage": "^5.14.0",
8
+ "@naturalcycles/db-lib": "^8.25.0",
9
+ "@naturalcycles/js-lib": "^14.41.0",
10
+ "@naturalcycles/nodejs-lib": "^12.31.0"
11
+ },
12
+ "devDependencies": {
13
+ "@naturalcycles/dev-lib": "^12.1.3",
14
+ "@types/node": "^17.0.5",
15
+ "jest": "^27.1.0"
16
+ },
17
+ "files": [
18
+ "dist",
19
+ "src",
20
+ "!src/test",
21
+ "!src/**/*.test.ts",
22
+ "!src/**/__snapshots__",
23
+ "!src/**/__exclude"
24
+ ],
25
+ "main": "dist/index.js",
26
+ "types": "dist/index.d.ts",
27
+ "publishConfig": {
28
+ "access": "public"
29
+ },
30
+ "repository": {
31
+ "type": "git",
32
+ "url": "https://github.com/NaturalCycles/cloud-storage-lib"
33
+ },
34
+ "engines": {
35
+ "node": ">=14.16.0"
36
+ },
37
+ "version": "1.0.0",
38
+ "description": "",
39
+ "author": "Natural Cycles Team",
40
+ "license": "MIT"
41
+ }
package/readme.md ADDED
@@ -0,0 +1,20 @@
1
+ ## @naturalcycles/cloud-storage-lib
2
+
3
+ > CommonStorage implementation based on Google Cloud Storage
4
+
5
+ [![npm](https://img.shields.io/npm/v/@naturalcycles/cloud-storage-lib/latest.svg)](https://www.npmjs.com/package/@naturalcycles/cloud-storage-lib)
6
+ [![min.gz size](https://badgen.net/bundlephobia/minzip/@naturalcycles/cloud-storage-lib)](https://bundlephobia.com/result?p=@naturalcycles/cloud-storage-lib)
7
+ [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
8
+
9
+ Implements:
10
+
11
+ - `CommonStorage`
12
+ - `CommonKeyValueDB`
13
+
14
+ Exports:
15
+
16
+ - `CommonStorage`
17
+ - `CommonStorageBucket`
18
+ - `CloudStorage`
19
+ - `CommonStorageKeyValueDB`
20
+ - `InMemoryCommonStorage`
@@ -0,0 +1,171 @@
1
+ import * as Buffer from 'buffer'
2
+ import { Readable, Writable } from 'stream'
3
+ import { Bucket, File, Storage } from '@google-cloud/storage'
4
+ import { ReadableTyped, transformMap, transformMapSimple } from '@naturalcycles/nodejs-lib'
5
+ import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
6
+ import { GCPServiceAccount } from './model'
7
+
8
+ /**
9
+ * This object is intentionally made to NOT extend StorageOptions,
10
+ * because StorageOptions is complicated and provides just too many ways
11
+ * to configure credentials.
12
+ *
13
+ * Here we define the minimum simple set of credentials needed.
14
+ * All of these properties are available from the "service account" json file
15
+ * (either personal one or non-personal).
16
+ */
17
+ export interface CloudStorageCfg {
18
+ credentials: GCPServiceAccount
19
+ }
20
+
21
+ export class CloudStorage implements CommonStorage {
22
+ constructor(public cfg: CloudStorageCfg) {
23
+ this.storage = new Storage({
24
+ credentials: cfg.credentials,
25
+ // Explicitly passing it here to fix this error:
26
+ // Error: Unable to detect a Project Id in the current environment.
27
+ // To learn more about authentication and Google APIs, visit:
28
+ // https://cloud.google.com/docs/authentication/getting-started
29
+ // at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
30
+ projectId: cfg.credentials.project_id,
31
+ })
32
+ }
33
+
34
+ storage: Storage
35
+
36
+ // async createBucket(bucketName: string): Promise<void> {
37
+ // const bucket = await this.storage.createBucket(bucketName)
38
+ // console.log(bucket) // debugging
39
+ // }
40
+
41
+ async ping(bucketName?: string): Promise<void> {
42
+ await this.storage.bucket(bucketName || 'non-existing-for-sure').exists()
43
+ }
44
+
45
+ async getBucketNames(opt: CommonStorageGetOptions = {}): Promise<string[]> {
46
+ const [buckets] = await this.storage.getBuckets({
47
+ maxResults: opt.limit,
48
+ })
49
+
50
+ return buckets.map(b => b.name)
51
+ }
52
+
53
+ getBucketNamesStream(): ReadableTyped<string> {
54
+ return this.storage.getBucketsStream().pipe(transformMapSimple<Bucket, string>(b => b.name))
55
+ }
56
+
57
+ async deletePath(bucketName: string, prefix: string): Promise<void> {
58
+ await this.storage.bucket(bucketName).deleteFiles({
59
+ prefix,
60
+ // to keep going in case error occurs, similar to THROW_AGGREGATED
61
+ force: true,
62
+ })
63
+ }
64
+
65
+ async fileExists(bucketName: string, filePath: string): Promise<boolean> {
66
+ const [exists] = await this.storage.bucket(bucketName).file(filePath).exists()
67
+ return exists
68
+ }
69
+
70
+ async getFileNames(bucketName: string, prefix: string): Promise<string[]> {
71
+ const [files] = await this.storage.bucket(bucketName).getFiles({
72
+ prefix,
73
+ })
74
+ return files.map(f => f.name)
75
+ }
76
+
77
+ getFileNamesStream(
78
+ bucketName: string,
79
+ prefix: string,
80
+ opt: CommonStorageGetOptions = {},
81
+ ): ReadableTyped<string> {
82
+ return this.storage
83
+ .bucket(bucketName)
84
+ .getFilesStream({
85
+ prefix,
86
+ maxResults: opt.limit,
87
+ })
88
+ .pipe(transformMapSimple<File, string>(f => f.name))
89
+ }
90
+
91
+ getFilesStream(
92
+ bucketName: string,
93
+ prefix: string,
94
+ opt: CommonStorageGetOptions = {},
95
+ ): ReadableTyped<FileEntry> {
96
+ return this.storage
97
+ .bucket(bucketName)
98
+ .getFilesStream({
99
+ prefix,
100
+ maxResults: opt.limit,
101
+ })
102
+ .pipe(
103
+ transformMap<File, FileEntry>(async f => {
104
+ const [content] = await f.download()
105
+ return { filePath: f.name, content }
106
+ }),
107
+ )
108
+ }
109
+
110
+ async getFile(bucketName: string, filePath: string): Promise<Buffer | null> {
111
+ const [buf] = await this.storage
112
+ .bucket(bucketName)
113
+ .file(filePath)
114
+ .download()
115
+ .catch(err => {
116
+ if (err?.code === 404) return [null] // file not found
117
+ throw err // rethrow otherwise
118
+ })
119
+
120
+ return buf
121
+ }
122
+
123
+ /**
124
+ * Returns a Readable that is NOT object mode,
125
+ * so you can e.g pipe it to fs.createWriteStream()
126
+ */
127
+ getFileReadStream(bucketName: string, filePath: string): Readable {
128
+ return this.storage.bucket(bucketName).file(filePath).createReadStream()
129
+ }
130
+
131
+ async saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void> {
132
+ await this.storage.bucket(bucketName).file(filePath).save(content)
133
+ }
134
+
135
+ getFileWriteStream(bucketName: string, filePath: string): Writable {
136
+ return this.storage.bucket(bucketName).file(filePath).createWriteStream()
137
+ }
138
+
139
+ async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
140
+ await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']()
141
+ }
142
+
143
+ async getFileVisibility(bucketName: string, filePath: string): Promise<boolean> {
144
+ const [isPublic] = await this.storage.bucket(bucketName).file(filePath).isPublic()
145
+ return isPublic
146
+ }
147
+
148
+ async copyFile(
149
+ fromBucket: string,
150
+ fromPath: string,
151
+ toPath: string,
152
+ toBucket?: string,
153
+ ): Promise<void> {
154
+ await this.storage
155
+ .bucket(fromBucket)
156
+ .file(fromPath)
157
+ .copy(this.storage.bucket(toBucket || fromBucket).file(toPath))
158
+ }
159
+
160
+ async moveFile(
161
+ fromBucket: string,
162
+ fromPath: string,
163
+ toPath: string,
164
+ toBucket?: string,
165
+ ): Promise<void> {
166
+ await this.storage
167
+ .bucket(fromBucket)
168
+ .file(fromPath)
169
+ .move(this.storage.bucket(toBucket || fromBucket).file(toPath))
170
+ }
171
+ }
@@ -0,0 +1,103 @@
1
+ import { Readable, Writable } from 'stream'
2
+ import { ReadableTyped } from '@naturalcycles/nodejs-lib'
3
+
4
+ export interface FileEntry {
5
+ filePath: string
6
+ content: Buffer
7
+ }
8
+
9
+ // TODO: move it away to a separate repo
10
+
11
+ export interface CommonStorageGetOptions {
12
+ /**
13
+ * Will filter resulting files based on `prefix`.
14
+ */
15
+ prefix?: string
16
+
17
+ /**
18
+ * Limits the number of results.
19
+ *
20
+ * By default it's unlimited.
21
+ */
22
+ limit?: number
23
+ }
24
+
25
+ /**
26
+ * Common denominator interface for File Storage.
27
+ * Modelled after GCP Cloud Storage, Firebase Storage.
28
+ *
29
+ * Uses the concept of Bucket (identified by string name) and Path within the Bucket.
30
+ *
31
+ * Path MUST NOT start with a slash !
32
+ *
33
+ * Similarly to CommonDB, Bucket is like a Table, and Path is like an `id`.
34
+ */
35
+ export interface CommonStorage {
36
+ /**
37
+ * Ensure that the credentials are correct and the connection is working.
38
+ * Idempotent.
39
+ *
40
+ * Pass `bucketName` in case you only have permissions to operate on that bucket.
41
+ */
42
+ ping(bucketName?: string): Promise<void>
43
+
44
+ /**
45
+ * Often needs a special permission.
46
+ */
47
+ getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>
48
+
49
+ getBucketNamesStream(): ReadableTyped<string>
50
+
51
+ /**
52
+ * Creates a new bucket by given name.
53
+ * todo: check what to do if it already exists
54
+ */
55
+ // createBucket(bucketName: string): Promise<void>
56
+
57
+ fileExists(bucketName: string, filePath: string): Promise<boolean>
58
+
59
+ getFile(bucketName: string, filePath: string): Promise<Buffer | null>
60
+
61
+ saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>
62
+
63
+ /**
64
+ * Should recursively delete all files in a folder, if path is a folder.
65
+ */
66
+ deletePath(bucketName: string, prefix: string): Promise<void>
67
+
68
+ /**
69
+ * Returns an array of strings which are file paths.
70
+ * Files that are not found by the path are not present in the map.
71
+ *
72
+ * Second argument is called `prefix` (same as `path`) to explain how
73
+ * listing works (it filters all files by `startsWith`). Also, to match
74
+ * GCP Cloud Storage API.
75
+ *
76
+ * Important difference between `prefix` and `path` is that `prefix` will
77
+ * return all files from sub-directories too!
78
+ */
79
+ getFileNames(bucketName: string, prefix: string): Promise<string[]>
80
+
81
+ getFileNamesStream(
82
+ bucketName: string,
83
+ prefix: string,
84
+ opt?: CommonStorageGetOptions,
85
+ ): ReadableTyped<string>
86
+
87
+ getFilesStream(
88
+ bucketName: string,
89
+ prefix: string,
90
+ opt?: CommonStorageGetOptions,
91
+ ): ReadableTyped<FileEntry>
92
+
93
+ getFileReadStream(bucketName: string, filePath: string): Readable
94
+
95
+ getFileWriteStream(bucketName: string, filePath: string): Writable
96
+
97
+ setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>
98
+
99
+ getFileVisibility(bucketName: string, filePath: string): Promise<boolean>
100
+
101
+ copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>
102
+ moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>
103
+ }
@@ -0,0 +1,171 @@
1
+ import { Readable, Writable } from 'stream'
2
+ import { AppError, pMap } from '@naturalcycles/js-lib'
3
+ import { ReadableTyped } from '@naturalcycles/nodejs-lib'
4
+ import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
5
+
6
+ export interface CommonStorageBucketCfg {
7
+ storage: CommonStorage
8
+ bucketName: string
9
+ }
10
+
11
+ /**
12
+ * Convenience wrapper around CommonStorage for a given Bucket.
13
+ *
14
+ * Similar to what CommonDao is to CommonDB.
15
+ */
16
+ export class CommonStorageBucket {
17
+ constructor(public cfg: CommonStorageBucketCfg) {}
18
+
19
+ async ping(bucketName?: string): Promise<void> {
20
+ await this.cfg.storage.ping(bucketName)
21
+ }
22
+
23
+ async fileExists(filePath: string): Promise<boolean> {
24
+ return await this.cfg.storage.fileExists(this.cfg.bucketName, filePath)
25
+ }
26
+
27
+ async getFile(filePath: string): Promise<Buffer | null> {
28
+ return await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
29
+ }
30
+
31
+ async getFileAsString(filePath: string): Promise<string | null> {
32
+ const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
33
+ return buf?.toString() || null
34
+ }
35
+
36
+ async getFileAsJson<T = any>(filePath: string): Promise<T | null> {
37
+ const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
38
+ if (!buf) return null
39
+ return JSON.parse(buf.toString())
40
+ }
41
+
42
+ async requireFile(filePath: string): Promise<Buffer> {
43
+ const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
44
+ if (!buf) this.throwRequiredError(filePath)
45
+ return buf
46
+ }
47
+
48
+ async requireFileAsString(filePath: string): Promise<string> {
49
+ const s = await this.getFileAsString(filePath)
50
+ return s ?? this.throwRequiredError(filePath)
51
+ }
52
+
53
+ async requireFileAsJson<T = any>(filePath: string): Promise<T> {
54
+ const v = await this.getFileAsJson<T>(filePath)
55
+ return v ?? this.throwRequiredError(filePath)
56
+ }
57
+
58
+ private throwRequiredError(filePath: string): never {
59
+ throw new AppError(`File required, but not found: ${this.cfg.bucketName}/${filePath}`, {
60
+ code: 'FILE_REQUIRED',
61
+ })
62
+ }
63
+
64
+ async getFileContents(paths: string[]): Promise<Buffer[]> {
65
+ return (
66
+ await pMap(
67
+ paths,
68
+ async filePath => (await this.cfg.storage.getFile(this.cfg.bucketName, filePath))!,
69
+ )
70
+ ).filter(Boolean)
71
+ }
72
+
73
+ async getFileContentsAsJson<T = any>(paths: string[]): Promise<T[]> {
74
+ return (
75
+ await pMap(paths, async filePath => {
76
+ const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
77
+ return buf ? JSON.parse(buf.toString()) : null
78
+ })
79
+ ).filter(Boolean)
80
+ }
81
+
82
+ async getFileEntries(paths: string[]): Promise<FileEntry[]> {
83
+ return (
84
+ await pMap(paths, async filePath => {
85
+ const content = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
86
+ return { filePath, content: content! }
87
+ })
88
+ ).filter(f => f.content)
89
+ }
90
+
91
+ async getFileEntriesAsJson<T = any>(
92
+ paths: string[],
93
+ ): Promise<{ filePath: string; content: T }[]> {
94
+ return (
95
+ await pMap(paths, async filePath => {
96
+ const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
97
+ return buf ? { filePath, content: JSON.parse(buf.toString()) } : (null as any)
98
+ })
99
+ ).filter(Boolean)
100
+ }
101
+
102
+ async saveFile(filePath: string, content: Buffer): Promise<void> {
103
+ await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content)
104
+ }
105
+
106
+ async saveFiles(entries: FileEntry[]): Promise<void> {
107
+ await pMap(entries, async f => {
108
+ await this.cfg.storage.saveFile(this.cfg.bucketName, f.filePath, f.content)
109
+ })
110
+ }
111
+
112
+ /**
113
+ * Should recursively delete all files in a folder, if path is a folder.
114
+ */
115
+ async deletePath(prefix: string): Promise<void> {
116
+ return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix)
117
+ }
118
+
119
+ async deletePaths(prefixes: string[]): Promise<void> {
120
+ await pMap(prefixes, async prefix => {
121
+ return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix)
122
+ })
123
+ }
124
+
125
+ /**
126
+ * Returns an array of strings which are file paths.
127
+ * Files that are not found by the path are not present in the map.
128
+ *
129
+ * Second argument is called `prefix` (same as `path`) to explain how
130
+ * listing works (it filters all files by `startsWith`). Also, to match
131
+ * GCP Cloud Storage API.
132
+ *
133
+ * Important difference between `prefix` and `path` is that `prefix` will
134
+ * return all files from sub-directories too!
135
+ */
136
+ async getFileNames(prefix: string): Promise<string[]> {
137
+ return await this.cfg.storage.getFileNames(this.cfg.bucketName, prefix)
138
+ }
139
+
140
+ getFileNamesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string> {
141
+ return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, prefix, opt)
142
+ }
143
+
144
+ getFilesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry> {
145
+ return this.cfg.storage.getFilesStream(this.cfg.bucketName, prefix, opt)
146
+ }
147
+
148
+ getFileReadStream(filePath: string): Readable {
149
+ return this.cfg.storage.getFileReadStream(this.cfg.bucketName, filePath)
150
+ }
151
+
152
+ getFileWriteStream(filePath: string): Writable {
153
+ return this.cfg.storage.getFileWriteStream(this.cfg.bucketName, filePath)
154
+ }
155
+
156
+ async setFileVisibility(filePath: string, isPublic: boolean): Promise<void> {
157
+ await this.cfg.storage.setFileVisibility(this.cfg.bucketName, filePath, isPublic)
158
+ }
159
+
160
+ async getFileVisibility(filePath: string): Promise<boolean> {
161
+ return await this.cfg.storage.getFileVisibility(this.cfg.bucketName, filePath)
162
+ }
163
+
164
+ async copyFile(fromPath: string, toPath: string, toBucket?: string): Promise<void> {
165
+ await this.cfg.storage.copyFile(this.cfg.bucketName, fromPath, toPath, toBucket)
166
+ }
167
+
168
+ async moveFile(fromPath: string, toPath: string, toBucket?: string): Promise<void> {
169
+ await this.cfg.storage.moveFile(this.cfg.bucketName, fromPath, toPath, toBucket)
170
+ }
171
+ }