@naturalcycles/cloud-storage-lib 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cloudStorage.d.ts +43 -0
- package/dist/cloudStorage.js +118 -0
- package/dist/commonStorage.d.ts +74 -0
- package/dist/commonStorage.js +2 -0
- package/dist/commonStorageBucket.d.ts +60 -0
- package/dist/commonStorageBucket.js +131 -0
- package/dist/commonStorageKeyValueDB.d.ts +32 -0
- package/dist/commonStorageKeyValueDB.js +87 -0
- package/dist/inMemoryCommonStorage.d.ts +28 -0
- package/dist/inMemoryCommonStorage.js +82 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +13 -0
- package/dist/model.d.ts +5 -0
- package/dist/model.js +2 -0
- package/dist/testing/commonStorageTest.d.ts +5 -0
- package/dist/testing/commonStorageTest.js +108 -0
- package/package.json +41 -0
- package/readme.md +20 -0
- package/src/cloudStorage.ts +171 -0
- package/src/commonStorage.ts +103 -0
- package/src/commonStorageBucket.ts +171 -0
- package/src/commonStorageKeyValueDB.ts +109 -0
- package/src/inMemoryCommonStorage.ts +115 -0
- package/src/index.ts +24 -0
- package/src/model.ts +5 -0
- package/src/testing/commonStorageTest.ts +134 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runCommonStorageTest = exports.InMemoryCommonStorage = exports.CommonStorageBucket = exports.CommonStorageKeyValueDB = exports.CloudStorage = void 0;
|
|
4
|
+
const cloudStorage_1 = require("./cloudStorage");
|
|
5
|
+
Object.defineProperty(exports, "CloudStorage", { enumerable: true, get: function () { return cloudStorage_1.CloudStorage; } });
|
|
6
|
+
const commonStorageBucket_1 = require("./commonStorageBucket");
|
|
7
|
+
Object.defineProperty(exports, "CommonStorageBucket", { enumerable: true, get: function () { return commonStorageBucket_1.CommonStorageBucket; } });
|
|
8
|
+
const commonStorageKeyValueDB_1 = require("./commonStorageKeyValueDB");
|
|
9
|
+
Object.defineProperty(exports, "CommonStorageKeyValueDB", { enumerable: true, get: function () { return commonStorageKeyValueDB_1.CommonStorageKeyValueDB; } });
|
|
10
|
+
const inMemoryCommonStorage_1 = require("./inMemoryCommonStorage");
|
|
11
|
+
Object.defineProperty(exports, "InMemoryCommonStorage", { enumerable: true, get: function () { return inMemoryCommonStorage_1.InMemoryCommonStorage; } });
|
|
12
|
+
const commonStorageTest_1 = require("./testing/commonStorageTest");
|
|
13
|
+
Object.defineProperty(exports, "runCommonStorageTest", { enumerable: true, get: function () { return commonStorageTest_1.runCommonStorageTest; } });
|
package/dist/model.d.ts
ADDED
package/dist/model.js
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runCommonStorageTest = void 0;
|
|
4
|
+
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
5
|
+
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
6
|
+
const TEST_FOLDER = 'test/subdir';
|
|
7
|
+
const TEST_ITEMS = (0, js_lib_1._range)(10).map(n => ({
|
|
8
|
+
id: `id_${n + 1}`,
|
|
9
|
+
n,
|
|
10
|
+
even: n % 2 === 0,
|
|
11
|
+
}));
|
|
12
|
+
const TEST_ITEMS2 = (0, js_lib_1._range)(10).map(n => ({
|
|
13
|
+
fileType: 2,
|
|
14
|
+
id: `id_${n + 1}`,
|
|
15
|
+
n,
|
|
16
|
+
even: n % 2 === 0,
|
|
17
|
+
}));
|
|
18
|
+
const TEST_ITEMS3 = (0, js_lib_1._range)(10).map(n => ({
|
|
19
|
+
fileType: 3,
|
|
20
|
+
id: `id_${n + 1}`,
|
|
21
|
+
n,
|
|
22
|
+
even: n % 2 === 0,
|
|
23
|
+
}));
|
|
24
|
+
const TEST_FILES = [TEST_ITEMS, TEST_ITEMS2, TEST_ITEMS3].map((obj, i) => ({
|
|
25
|
+
filePath: `${TEST_FOLDER}/file_${i + 1}.json`,
|
|
26
|
+
content: Buffer.from(JSON.stringify(obj)),
|
|
27
|
+
}));
|
|
28
|
+
/**
|
|
29
|
+
* This test suite must be idempotent.
|
|
30
|
+
*/
|
|
31
|
+
function runCommonStorageTest(storage, bucketName) {
|
|
32
|
+
// test('createBucket', async () => {
|
|
33
|
+
// await storage.createBucket(bucketName)
|
|
34
|
+
// })
|
|
35
|
+
test('ping', async () => {
|
|
36
|
+
await storage.ping();
|
|
37
|
+
});
|
|
38
|
+
test('listBuckets', async () => {
|
|
39
|
+
const buckets = await storage.getBucketNames();
|
|
40
|
+
console.log(buckets);
|
|
41
|
+
});
|
|
42
|
+
test('streamBuckets', async () => {
|
|
43
|
+
const buckets = await (0, nodejs_lib_1.readableToArray)(storage.getBucketNamesStream());
|
|
44
|
+
console.log(buckets);
|
|
45
|
+
});
|
|
46
|
+
test('prepare: clear bucket', async () => {
|
|
47
|
+
await (0, js_lib_1.pMap)(TEST_FILES.map(f => f.filePath), async (filePath) => await storage.deletePath(bucketName, filePath));
|
|
48
|
+
});
|
|
49
|
+
test('listFileNames on root should return empty', async () => {
|
|
50
|
+
const fileNames = await storage.getFileNames(bucketName, '');
|
|
51
|
+
expect(fileNames).toEqual([]);
|
|
52
|
+
});
|
|
53
|
+
test(`listFileNames on ${TEST_FOLDER} should return empty`, async () => {
|
|
54
|
+
const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER);
|
|
55
|
+
expect(fileNames).toEqual([]);
|
|
56
|
+
});
|
|
57
|
+
test('streamFileNames on root should return empty', async () => {
|
|
58
|
+
const fileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, ''));
|
|
59
|
+
expect(fileNames).toEqual([]);
|
|
60
|
+
});
|
|
61
|
+
test(`exists should return empty array`, async () => {
|
|
62
|
+
await (0, js_lib_1.pMap)(TEST_FILES, async (f) => {
|
|
63
|
+
const exists = await storage.fileExists(bucketName, f.filePath);
|
|
64
|
+
expect(exists).toBe(false);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
test(`saveFiles, then listFileNames, streamFileNames and getFiles should return just saved files`, async () => {
|
|
68
|
+
const testFilesMap = Object.fromEntries(TEST_FILES.map(f => [f.filePath, f.content]));
|
|
69
|
+
// It's done in the same test to ensure "strong consistency"
|
|
70
|
+
await (0, js_lib_1.pMap)(TEST_FILES, async (f) => await storage.saveFile(bucketName, f.filePath, f.content));
|
|
71
|
+
const fileNames = await storage.getFileNames(bucketName, TEST_FOLDER);
|
|
72
|
+
expect(fileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
|
|
73
|
+
const streamedFileNames = await (0, nodejs_lib_1.readableToArray)(storage.getFileNamesStream(bucketName, TEST_FOLDER));
|
|
74
|
+
expect(streamedFileNames.sort()).toEqual(TEST_FILES.map(f => f.filePath).sort());
|
|
75
|
+
const filesMap = {};
|
|
76
|
+
await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
|
|
77
|
+
filesMap[filePath] = (await storage.getFile(bucketName, filePath));
|
|
78
|
+
});
|
|
79
|
+
expect(filesMap).toEqual(testFilesMap);
|
|
80
|
+
await (0, js_lib_1.pMap)(fileNames, async (filePath) => {
|
|
81
|
+
const exists = await storage.fileExists(bucketName, filePath);
|
|
82
|
+
expect(exists).toBe(true);
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
test('cleanup', async () => {
|
|
86
|
+
await storage.deletePath(bucketName, '');
|
|
87
|
+
});
|
|
88
|
+
// Cannot update access control for an object when uniform bucket-level access is enabled. Read more at https://cloud.google.com/storage/docs/uniform-bucket-level-access
|
|
89
|
+
/*
|
|
90
|
+
test(`get/set FilesVisibility`, async () => {
|
|
91
|
+
const fileNames = TEST_FILES.map(f => f.filePath)
|
|
92
|
+
|
|
93
|
+
let map = await storage.getFilesVisibility(bucketName, fileNames)
|
|
94
|
+
expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
|
|
95
|
+
|
|
96
|
+
await storage.setFilesVisibility(bucketName, fileNames, true)
|
|
97
|
+
|
|
98
|
+
map = await storage.getFilesVisibility(bucketName, fileNames)
|
|
99
|
+
expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, true])))
|
|
100
|
+
|
|
101
|
+
await storage.setFilesVisibility(bucketName, fileNames, false)
|
|
102
|
+
|
|
103
|
+
map = await storage.getFilesVisibility(bucketName, fileNames)
|
|
104
|
+
expect(map).toEqual(Object.fromEntries(fileNames.map(f => [f, false])))
|
|
105
|
+
})
|
|
106
|
+
*/
|
|
107
|
+
}
|
|
108
|
+
exports.runCommonStorageTest = runCommonStorageTest;
|
package/package.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@naturalcycles/cloud-storage-lib",
|
|
3
|
+
"scripts": {
|
|
4
|
+
"prepare": "husky install"
|
|
5
|
+
},
|
|
6
|
+
"dependencies": {
|
|
7
|
+
"@google-cloud/storage": "^5.14.0",
|
|
8
|
+
"@naturalcycles/db-lib": "^8.25.0",
|
|
9
|
+
"@naturalcycles/js-lib": "^14.41.0",
|
|
10
|
+
"@naturalcycles/nodejs-lib": "^12.31.0"
|
|
11
|
+
},
|
|
12
|
+
"devDependencies": {
|
|
13
|
+
"@naturalcycles/dev-lib": "^12.1.3",
|
|
14
|
+
"@types/node": "^17.0.5",
|
|
15
|
+
"jest": "^27.1.0"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"dist",
|
|
19
|
+
"src",
|
|
20
|
+
"!src/test",
|
|
21
|
+
"!src/**/*.test.ts",
|
|
22
|
+
"!src/**/__snapshots__",
|
|
23
|
+
"!src/**/__exclude"
|
|
24
|
+
],
|
|
25
|
+
"main": "dist/index.js",
|
|
26
|
+
"types": "dist/index.d.ts",
|
|
27
|
+
"publishConfig": {
|
|
28
|
+
"access": "public"
|
|
29
|
+
},
|
|
30
|
+
"repository": {
|
|
31
|
+
"type": "git",
|
|
32
|
+
"url": "https://github.com/NaturalCycles/cloud-storage-lib"
|
|
33
|
+
},
|
|
34
|
+
"engines": {
|
|
35
|
+
"node": ">=14.16.0"
|
|
36
|
+
},
|
|
37
|
+
"version": "1.0.0",
|
|
38
|
+
"description": "",
|
|
39
|
+
"author": "Natural Cycles Team",
|
|
40
|
+
"license": "MIT"
|
|
41
|
+
}
|
package/readme.md
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
## @naturalcycles/cloud-storage-lib
|
|
2
|
+
|
|
3
|
+
> CommonStorage implementation based on Google Cloud Storage
|
|
4
|
+
|
|
5
|
+
[](https://www.npmjs.com/package/@naturalcycles/cloud-storage-lib)
|
|
6
|
+
[](https://bundlephobia.com/result?p=@naturalcycles/cloud-storage-lib)
|
|
7
|
+
[](https://github.com/prettier/prettier)
|
|
8
|
+
|
|
9
|
+
Implements:
|
|
10
|
+
|
|
11
|
+
- `CommonStorage`
|
|
12
|
+
- `CommonKeyValueDB`
|
|
13
|
+
|
|
14
|
+
Exports:
|
|
15
|
+
|
|
16
|
+
- `CommonStorage`
|
|
17
|
+
- `CommonStorageBucket`
|
|
18
|
+
- `CloudStorage`
|
|
19
|
+
- `CommonStorageKeyValueDB`
|
|
20
|
+
- `InMemoryCommonStorage`
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import * as Buffer from 'buffer'
|
|
2
|
+
import { Readable, Writable } from 'stream'
|
|
3
|
+
import { Bucket, File, Storage } from '@google-cloud/storage'
|
|
4
|
+
import { ReadableTyped, transformMap, transformMapSimple } from '@naturalcycles/nodejs-lib'
|
|
5
|
+
import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
|
|
6
|
+
import { GCPServiceAccount } from './model'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* This object is intentionally made to NOT extend StorageOptions,
|
|
10
|
+
* because StorageOptions is complicated and provides just too many ways
|
|
11
|
+
* to configure credentials.
|
|
12
|
+
*
|
|
13
|
+
* Here we define the minimum simple set of credentials needed.
|
|
14
|
+
* All of these properties are available from the "service account" json file
|
|
15
|
+
* (either personal one or non-personal).
|
|
16
|
+
*/
|
|
17
|
+
export interface CloudStorageCfg {
|
|
18
|
+
credentials: GCPServiceAccount
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class CloudStorage implements CommonStorage {
|
|
22
|
+
constructor(public cfg: CloudStorageCfg) {
|
|
23
|
+
this.storage = new Storage({
|
|
24
|
+
credentials: cfg.credentials,
|
|
25
|
+
// Explicitly passing it here to fix this error:
|
|
26
|
+
// Error: Unable to detect a Project Id in the current environment.
|
|
27
|
+
// To learn more about authentication and Google APIs, visit:
|
|
28
|
+
// https://cloud.google.com/docs/authentication/getting-started
|
|
29
|
+
// at /root/repo/node_modules/google-auth-library/build/src/auth/googleauth.js:95:31
|
|
30
|
+
projectId: cfg.credentials.project_id,
|
|
31
|
+
})
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
storage: Storage
|
|
35
|
+
|
|
36
|
+
// async createBucket(bucketName: string): Promise<void> {
|
|
37
|
+
// const bucket = await this.storage.createBucket(bucketName)
|
|
38
|
+
// console.log(bucket) // debugging
|
|
39
|
+
// }
|
|
40
|
+
|
|
41
|
+
async ping(bucketName?: string): Promise<void> {
|
|
42
|
+
await this.storage.bucket(bucketName || 'non-existing-for-sure').exists()
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async getBucketNames(opt: CommonStorageGetOptions = {}): Promise<string[]> {
|
|
46
|
+
const [buckets] = await this.storage.getBuckets({
|
|
47
|
+
maxResults: opt.limit,
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
return buckets.map(b => b.name)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
getBucketNamesStream(): ReadableTyped<string> {
|
|
54
|
+
return this.storage.getBucketsStream().pipe(transformMapSimple<Bucket, string>(b => b.name))
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async deletePath(bucketName: string, prefix: string): Promise<void> {
|
|
58
|
+
await this.storage.bucket(bucketName).deleteFiles({
|
|
59
|
+
prefix,
|
|
60
|
+
// to keep going in case error occurs, similar to THROW_AGGREGATED
|
|
61
|
+
force: true,
|
|
62
|
+
})
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async fileExists(bucketName: string, filePath: string): Promise<boolean> {
|
|
66
|
+
const [exists] = await this.storage.bucket(bucketName).file(filePath).exists()
|
|
67
|
+
return exists
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async getFileNames(bucketName: string, prefix: string): Promise<string[]> {
|
|
71
|
+
const [files] = await this.storage.bucket(bucketName).getFiles({
|
|
72
|
+
prefix,
|
|
73
|
+
})
|
|
74
|
+
return files.map(f => f.name)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
getFileNamesStream(
|
|
78
|
+
bucketName: string,
|
|
79
|
+
prefix: string,
|
|
80
|
+
opt: CommonStorageGetOptions = {},
|
|
81
|
+
): ReadableTyped<string> {
|
|
82
|
+
return this.storage
|
|
83
|
+
.bucket(bucketName)
|
|
84
|
+
.getFilesStream({
|
|
85
|
+
prefix,
|
|
86
|
+
maxResults: opt.limit,
|
|
87
|
+
})
|
|
88
|
+
.pipe(transformMapSimple<File, string>(f => f.name))
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
getFilesStream(
|
|
92
|
+
bucketName: string,
|
|
93
|
+
prefix: string,
|
|
94
|
+
opt: CommonStorageGetOptions = {},
|
|
95
|
+
): ReadableTyped<FileEntry> {
|
|
96
|
+
return this.storage
|
|
97
|
+
.bucket(bucketName)
|
|
98
|
+
.getFilesStream({
|
|
99
|
+
prefix,
|
|
100
|
+
maxResults: opt.limit,
|
|
101
|
+
})
|
|
102
|
+
.pipe(
|
|
103
|
+
transformMap<File, FileEntry>(async f => {
|
|
104
|
+
const [content] = await f.download()
|
|
105
|
+
return { filePath: f.name, content }
|
|
106
|
+
}),
|
|
107
|
+
)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async getFile(bucketName: string, filePath: string): Promise<Buffer | null> {
|
|
111
|
+
const [buf] = await this.storage
|
|
112
|
+
.bucket(bucketName)
|
|
113
|
+
.file(filePath)
|
|
114
|
+
.download()
|
|
115
|
+
.catch(err => {
|
|
116
|
+
if (err?.code === 404) return [null] // file not found
|
|
117
|
+
throw err // rethrow otherwise
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
return buf
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Returns a Readable that is NOT object mode,
|
|
125
|
+
* so you can e.g pipe it to fs.createWriteStream()
|
|
126
|
+
*/
|
|
127
|
+
getFileReadStream(bucketName: string, filePath: string): Readable {
|
|
128
|
+
return this.storage.bucket(bucketName).file(filePath).createReadStream()
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void> {
|
|
132
|
+
await this.storage.bucket(bucketName).file(filePath).save(content)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
getFileWriteStream(bucketName: string, filePath: string): Writable {
|
|
136
|
+
return this.storage.bucket(bucketName).file(filePath).createWriteStream()
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void> {
|
|
140
|
+
await this.storage.bucket(bucketName).file(filePath)[isPublic ? 'makePublic' : 'makePrivate']()
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async getFileVisibility(bucketName: string, filePath: string): Promise<boolean> {
|
|
144
|
+
const [isPublic] = await this.storage.bucket(bucketName).file(filePath).isPublic()
|
|
145
|
+
return isPublic
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async copyFile(
|
|
149
|
+
fromBucket: string,
|
|
150
|
+
fromPath: string,
|
|
151
|
+
toPath: string,
|
|
152
|
+
toBucket?: string,
|
|
153
|
+
): Promise<void> {
|
|
154
|
+
await this.storage
|
|
155
|
+
.bucket(fromBucket)
|
|
156
|
+
.file(fromPath)
|
|
157
|
+
.copy(this.storage.bucket(toBucket || fromBucket).file(toPath))
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async moveFile(
|
|
161
|
+
fromBucket: string,
|
|
162
|
+
fromPath: string,
|
|
163
|
+
toPath: string,
|
|
164
|
+
toBucket?: string,
|
|
165
|
+
): Promise<void> {
|
|
166
|
+
await this.storage
|
|
167
|
+
.bucket(fromBucket)
|
|
168
|
+
.file(fromPath)
|
|
169
|
+
.move(this.storage.bucket(toBucket || fromBucket).file(toPath))
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { Readable, Writable } from 'stream'
|
|
2
|
+
import { ReadableTyped } from '@naturalcycles/nodejs-lib'
|
|
3
|
+
|
|
4
|
+
export interface FileEntry {
|
|
5
|
+
filePath: string
|
|
6
|
+
content: Buffer
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
// TODO: move it away to a separate repo
|
|
10
|
+
|
|
11
|
+
export interface CommonStorageGetOptions {
|
|
12
|
+
/**
|
|
13
|
+
* Will filter resulting files based on `prefix`.
|
|
14
|
+
*/
|
|
15
|
+
prefix?: string
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Limits the number of results.
|
|
19
|
+
*
|
|
20
|
+
* By default it's unlimited.
|
|
21
|
+
*/
|
|
22
|
+
limit?: number
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Common denominator interface for File Storage.
|
|
27
|
+
* Modelled after GCP Cloud Storage, Firebase Storage.
|
|
28
|
+
*
|
|
29
|
+
* Uses the concept of Bucket (identified by string name) and Path within the Bucket.
|
|
30
|
+
*
|
|
31
|
+
* Path MUST NOT start with a slash !
|
|
32
|
+
*
|
|
33
|
+
* Similarly to CommonDB, Bucket is like a Table, and Path is like an `id`.
|
|
34
|
+
*/
|
|
35
|
+
export interface CommonStorage {
|
|
36
|
+
/**
|
|
37
|
+
* Ensure that the credentials are correct and the connection is working.
|
|
38
|
+
* Idempotent.
|
|
39
|
+
*
|
|
40
|
+
* Pass `bucketName` in case you only have permissions to operate on that bucket.
|
|
41
|
+
*/
|
|
42
|
+
ping(bucketName?: string): Promise<void>
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Often needs a special permission.
|
|
46
|
+
*/
|
|
47
|
+
getBucketNames(opt?: CommonStorageGetOptions): Promise<string[]>
|
|
48
|
+
|
|
49
|
+
getBucketNamesStream(): ReadableTyped<string>
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Creates a new bucket by given name.
|
|
53
|
+
* todo: check what to do if it already exists
|
|
54
|
+
*/
|
|
55
|
+
// createBucket(bucketName: string): Promise<void>
|
|
56
|
+
|
|
57
|
+
fileExists(bucketName: string, filePath: string): Promise<boolean>
|
|
58
|
+
|
|
59
|
+
getFile(bucketName: string, filePath: string): Promise<Buffer | null>
|
|
60
|
+
|
|
61
|
+
saveFile(bucketName: string, filePath: string, content: Buffer): Promise<void>
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Should recursively delete all files in a folder, if path is a folder.
|
|
65
|
+
*/
|
|
66
|
+
deletePath(bucketName: string, prefix: string): Promise<void>
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Returns an array of strings which are file paths.
|
|
70
|
+
* Files that are not found by the path are not present in the map.
|
|
71
|
+
*
|
|
72
|
+
* Second argument is called `prefix` (same as `path`) to explain how
|
|
73
|
+
* listing works (it filters all files by `startsWith`). Also, to match
|
|
74
|
+
* GCP Cloud Storage API.
|
|
75
|
+
*
|
|
76
|
+
* Important difference between `prefix` and `path` is that `prefix` will
|
|
77
|
+
* return all files from sub-directories too!
|
|
78
|
+
*/
|
|
79
|
+
getFileNames(bucketName: string, prefix: string): Promise<string[]>
|
|
80
|
+
|
|
81
|
+
getFileNamesStream(
|
|
82
|
+
bucketName: string,
|
|
83
|
+
prefix: string,
|
|
84
|
+
opt?: CommonStorageGetOptions,
|
|
85
|
+
): ReadableTyped<string>
|
|
86
|
+
|
|
87
|
+
getFilesStream(
|
|
88
|
+
bucketName: string,
|
|
89
|
+
prefix: string,
|
|
90
|
+
opt?: CommonStorageGetOptions,
|
|
91
|
+
): ReadableTyped<FileEntry>
|
|
92
|
+
|
|
93
|
+
getFileReadStream(bucketName: string, filePath: string): Readable
|
|
94
|
+
|
|
95
|
+
getFileWriteStream(bucketName: string, filePath: string): Writable
|
|
96
|
+
|
|
97
|
+
setFileVisibility(bucketName: string, filePath: string, isPublic: boolean): Promise<void>
|
|
98
|
+
|
|
99
|
+
getFileVisibility(bucketName: string, filePath: string): Promise<boolean>
|
|
100
|
+
|
|
101
|
+
copyFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>
|
|
102
|
+
moveFile(fromBucket: string, fromPath: string, toPath: string, toBucket?: string): Promise<void>
|
|
103
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { Readable, Writable } from 'stream'
|
|
2
|
+
import { AppError, pMap } from '@naturalcycles/js-lib'
|
|
3
|
+
import { ReadableTyped } from '@naturalcycles/nodejs-lib'
|
|
4
|
+
import { CommonStorage, CommonStorageGetOptions, FileEntry } from './commonStorage'
|
|
5
|
+
|
|
6
|
+
export interface CommonStorageBucketCfg {
|
|
7
|
+
storage: CommonStorage
|
|
8
|
+
bucketName: string
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Convenience wrapper around CommonStorage for a given Bucket.
|
|
13
|
+
*
|
|
14
|
+
* Similar to what CommonDao is to CommonDB.
|
|
15
|
+
*/
|
|
16
|
+
export class CommonStorageBucket {
|
|
17
|
+
constructor(public cfg: CommonStorageBucketCfg) {}
|
|
18
|
+
|
|
19
|
+
async ping(bucketName?: string): Promise<void> {
|
|
20
|
+
await this.cfg.storage.ping(bucketName)
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async fileExists(filePath: string): Promise<boolean> {
|
|
24
|
+
return await this.cfg.storage.fileExists(this.cfg.bucketName, filePath)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async getFile(filePath: string): Promise<Buffer | null> {
|
|
28
|
+
return await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async getFileAsString(filePath: string): Promise<string | null> {
|
|
32
|
+
const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
|
|
33
|
+
return buf?.toString() || null
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async getFileAsJson<T = any>(filePath: string): Promise<T | null> {
|
|
37
|
+
const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
|
|
38
|
+
if (!buf) return null
|
|
39
|
+
return JSON.parse(buf.toString())
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async requireFile(filePath: string): Promise<Buffer> {
|
|
43
|
+
const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
|
|
44
|
+
if (!buf) this.throwRequiredError(filePath)
|
|
45
|
+
return buf
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async requireFileAsString(filePath: string): Promise<string> {
|
|
49
|
+
const s = await this.getFileAsString(filePath)
|
|
50
|
+
return s ?? this.throwRequiredError(filePath)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async requireFileAsJson<T = any>(filePath: string): Promise<T> {
|
|
54
|
+
const v = await this.getFileAsJson<T>(filePath)
|
|
55
|
+
return v ?? this.throwRequiredError(filePath)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
private throwRequiredError(filePath: string): never {
|
|
59
|
+
throw new AppError(`File required, but not found: ${this.cfg.bucketName}/${filePath}`, {
|
|
60
|
+
code: 'FILE_REQUIRED',
|
|
61
|
+
})
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async getFileContents(paths: string[]): Promise<Buffer[]> {
|
|
65
|
+
return (
|
|
66
|
+
await pMap(
|
|
67
|
+
paths,
|
|
68
|
+
async filePath => (await this.cfg.storage.getFile(this.cfg.bucketName, filePath))!,
|
|
69
|
+
)
|
|
70
|
+
).filter(Boolean)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async getFileContentsAsJson<T = any>(paths: string[]): Promise<T[]> {
|
|
74
|
+
return (
|
|
75
|
+
await pMap(paths, async filePath => {
|
|
76
|
+
const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
|
|
77
|
+
return buf ? JSON.parse(buf.toString()) : null
|
|
78
|
+
})
|
|
79
|
+
).filter(Boolean)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async getFileEntries(paths: string[]): Promise<FileEntry[]> {
|
|
83
|
+
return (
|
|
84
|
+
await pMap(paths, async filePath => {
|
|
85
|
+
const content = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
|
|
86
|
+
return { filePath, content: content! }
|
|
87
|
+
})
|
|
88
|
+
).filter(f => f.content)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async getFileEntriesAsJson<T = any>(
|
|
92
|
+
paths: string[],
|
|
93
|
+
): Promise<{ filePath: string; content: T }[]> {
|
|
94
|
+
return (
|
|
95
|
+
await pMap(paths, async filePath => {
|
|
96
|
+
const buf = await this.cfg.storage.getFile(this.cfg.bucketName, filePath)
|
|
97
|
+
return buf ? { filePath, content: JSON.parse(buf.toString()) } : (null as any)
|
|
98
|
+
})
|
|
99
|
+
).filter(Boolean)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async saveFile(filePath: string, content: Buffer): Promise<void> {
|
|
103
|
+
await this.cfg.storage.saveFile(this.cfg.bucketName, filePath, content)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async saveFiles(entries: FileEntry[]): Promise<void> {
|
|
107
|
+
await pMap(entries, async f => {
|
|
108
|
+
await this.cfg.storage.saveFile(this.cfg.bucketName, f.filePath, f.content)
|
|
109
|
+
})
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Should recursively delete all files in a folder, if path is a folder.
|
|
114
|
+
*/
|
|
115
|
+
async deletePath(prefix: string): Promise<void> {
|
|
116
|
+
return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix)
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async deletePaths(prefixes: string[]): Promise<void> {
|
|
120
|
+
await pMap(prefixes, async prefix => {
|
|
121
|
+
return await this.cfg.storage.deletePath(this.cfg.bucketName, prefix)
|
|
122
|
+
})
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Returns an array of strings which are file paths.
|
|
127
|
+
* Files that are not found by the path are not present in the map.
|
|
128
|
+
*
|
|
129
|
+
* Second argument is called `prefix` (same as `path`) to explain how
|
|
130
|
+
* listing works (it filters all files by `startsWith`). Also, to match
|
|
131
|
+
* GCP Cloud Storage API.
|
|
132
|
+
*
|
|
133
|
+
* Important difference between `prefix` and `path` is that `prefix` will
|
|
134
|
+
* return all files from sub-directories too!
|
|
135
|
+
*/
|
|
136
|
+
async getFileNames(prefix: string): Promise<string[]> {
|
|
137
|
+
return await this.cfg.storage.getFileNames(this.cfg.bucketName, prefix)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
getFileNamesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<string> {
|
|
141
|
+
return this.cfg.storage.getFileNamesStream(this.cfg.bucketName, prefix, opt)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
getFilesStream(prefix: string, opt?: CommonStorageGetOptions): ReadableTyped<FileEntry> {
|
|
145
|
+
return this.cfg.storage.getFilesStream(this.cfg.bucketName, prefix, opt)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
getFileReadStream(filePath: string): Readable {
|
|
149
|
+
return this.cfg.storage.getFileReadStream(this.cfg.bucketName, filePath)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
getFileWriteStream(filePath: string): Writable {
|
|
153
|
+
return this.cfg.storage.getFileWriteStream(this.cfg.bucketName, filePath)
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async setFileVisibility(filePath: string, isPublic: boolean): Promise<void> {
|
|
157
|
+
await this.cfg.storage.setFileVisibility(this.cfg.bucketName, filePath, isPublic)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async getFileVisibility(filePath: string): Promise<boolean> {
|
|
161
|
+
return await this.cfg.storage.getFileVisibility(this.cfg.bucketName, filePath)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async copyFile(fromPath: string, toPath: string, toBucket?: string): Promise<void> {
|
|
165
|
+
await this.cfg.storage.copyFile(this.cfg.bucketName, fromPath, toPath, toBucket)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async moveFile(fromPath: string, toPath: string, toBucket?: string): Promise<void> {
|
|
169
|
+
await this.cfg.storage.moveFile(this.cfg.bucketName, fromPath, toPath, toBucket)
|
|
170
|
+
}
|
|
171
|
+
}
|