firebase-tools 10.2.2 → 10.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands/deploy.js +1 -1
- package/lib/commands/experimental-functions-shell.js +1 -1
- package/lib/commands/ext-configure.js +68 -7
- package/lib/commands/ext-export.js +10 -9
- package/lib/commands/ext-install.js +73 -9
- package/lib/commands/ext-uninstall.js +9 -0
- package/lib/commands/ext-update.js +58 -3
- package/lib/commands/functions-config-export.js +2 -2
- package/lib/commands/functions-shell.js +1 -1
- package/lib/commands/hosting-channel-create.js +2 -2
- package/lib/commands/hosting-channel-delete.js +2 -2
- package/lib/commands/hosting-channel-deploy.js +2 -2
- package/lib/commands/hosting-channel-list.js +2 -2
- package/lib/commands/hosting-channel-open.js +2 -2
- package/lib/commands/hosting-sites-delete.js +2 -2
- package/lib/commands/serve.js +1 -1
- package/lib/commands/target-apply.js +2 -2
- package/lib/commands/target-clear.js +2 -2
- package/lib/commands/target-remove.js +2 -2
- package/lib/commands/target.js +2 -2
- package/lib/config.js +9 -3
- package/lib/deploy/extensions/planner.js +15 -9
- package/lib/deploy/functions/backend.js +10 -1
- package/lib/deploy/functions/checkIam.js +4 -4
- package/lib/deploy/functions/prepare.js +2 -1
- package/lib/deploy/functions/release/fabricator.js +4 -4
- package/lib/deploy/functions/release/planner.js +34 -20
- package/lib/deploy/functions/runtimes/discovery/v1alpha1.js +6 -1
- package/lib/deploy/functions/runtimes/node/index.js +27 -0
- package/lib/deploy/functions/runtimes/node/parseTriggers.js +36 -13
- package/lib/deploy/functions/services/firebaseAlerts.js +30 -0
- package/lib/deploy/functions/services/index.js +9 -1
- package/lib/deploy/functions/services/storage.js +10 -4
- package/lib/deploy/functions/triggerRegionHelper.js +1 -1
- package/lib/emulator/auth/apiSpec.js +37 -0
- package/lib/emulator/commandUtils.js +2 -2
- package/lib/emulator/constants.js +1 -0
- package/lib/emulator/controller.js +9 -7
- package/lib/emulator/extensions/validation.js +37 -2
- package/lib/emulator/extensionsEmulator.js +47 -9
- package/lib/emulator/functionsEmulator.js +17 -12
- package/lib/emulator/functionsEmulatorShared.js +34 -11
- package/lib/emulator/storage/apis/firebase.js +316 -341
- package/lib/emulator/storage/apis/gcloud.js +238 -113
- package/lib/emulator/storage/crc.js +5 -1
- package/lib/emulator/storage/errors.js +9 -0
- package/lib/emulator/storage/files.js +161 -304
- package/lib/emulator/storage/index.js +25 -74
- package/lib/emulator/storage/metadata.js +63 -49
- package/lib/emulator/storage/multipart.js +62 -0
- package/lib/emulator/storage/persistence.js +78 -0
- package/lib/emulator/storage/rules/config.js +34 -0
- package/lib/emulator/storage/rules/manager.js +98 -0
- package/lib/emulator/storage/rules/runtime.js +4 -0
- package/lib/emulator/storage/rules/utils.js +48 -0
- package/lib/emulator/storage/server.js +2 -2
- package/lib/emulator/storage/upload.js +106 -0
- package/lib/extensions/askUserForParam.js +77 -28
- package/lib/extensions/emulator/optionsHelper.js +35 -3
- package/lib/extensions/extensionsHelper.js +19 -10
- package/lib/extensions/manifest.js +142 -14
- package/lib/extensions/paramHelper.js +32 -9
- package/lib/fsutils.js +14 -1
- package/lib/functions/env.js +4 -6
- package/lib/functions/events/v2.js +11 -0
- package/lib/gcp/cloudfunctions.js +20 -7
- package/lib/gcp/cloudfunctionsv2.js +30 -12
- package/lib/gcp/resourceManager.js +4 -4
- package/lib/requireConfig.js +11 -9
- package/lib/serve/functions.js +2 -1
- package/lib/utils.js +14 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/lib/deploy/extensions/params.js +0 -42
- package/lib/deploy/functions/eventTypes.js +0 -10
- package/lib/prepareUpload.js +0 -44
|
@@ -7,19 +7,18 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
|
7
7
|
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
8
8
|
};
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.
|
|
10
|
+
exports.StorageLayer = exports.StoredFile = void 0;
|
|
11
11
|
const fs_1 = require("fs");
|
|
12
|
-
const os_1 = require("os");
|
|
13
|
-
const uuid_1 = require("uuid");
|
|
14
12
|
const list_1 = require("./list");
|
|
15
13
|
const metadata_1 = require("./metadata");
|
|
14
|
+
const errors_1 = require("./errors");
|
|
16
15
|
const path = require("path");
|
|
17
|
-
const fs = require("fs");
|
|
18
16
|
const fse = require("fs-extra");
|
|
19
|
-
const rimraf = require("rimraf");
|
|
20
17
|
const cloudFunctions_1 = require("./cloudFunctions");
|
|
21
18
|
const logger_1 = require("../../logger");
|
|
22
19
|
const adminSdkConfig_1 = require("../adminSdkConfig");
|
|
20
|
+
const types_1 = require("./rules/types");
|
|
21
|
+
const upload_1 = require("./upload");
|
|
23
22
|
class StoredFile {
|
|
24
23
|
constructor(metadata, path) {
|
|
25
24
|
this.metadata = metadata;
|
|
@@ -39,73 +38,17 @@ class StoredFile {
|
|
|
39
38
|
}
|
|
40
39
|
}
|
|
41
40
|
exports.StoredFile = StoredFile;
|
|
42
|
-
class ResumableUpload {
|
|
43
|
-
constructor(bucketId, objectId, uploadId, contentType, metadata, authorization) {
|
|
44
|
-
this._currentBytesUploaded = 0;
|
|
45
|
-
this._status = UploadStatus.ACTIVE;
|
|
46
|
-
this._bucketId = bucketId;
|
|
47
|
-
this._objectId = objectId;
|
|
48
|
-
this._uploadId = uploadId;
|
|
49
|
-
this._contentType = contentType;
|
|
50
|
-
this._metadata = metadata;
|
|
51
|
-
this._authorization = authorization;
|
|
52
|
-
this._fileLocation = encodeURIComponent(`${uploadId}_b_${bucketId}_o_${objectId}`);
|
|
53
|
-
this._currentBytesUploaded = 0;
|
|
54
|
-
}
|
|
55
|
-
get uploadId() {
|
|
56
|
-
return this._uploadId;
|
|
57
|
-
}
|
|
58
|
-
get metadata() {
|
|
59
|
-
return this._metadata;
|
|
60
|
-
}
|
|
61
|
-
get bucketId() {
|
|
62
|
-
return this._bucketId;
|
|
63
|
-
}
|
|
64
|
-
get objectId() {
|
|
65
|
-
return this._objectId;
|
|
66
|
-
}
|
|
67
|
-
get contentType() {
|
|
68
|
-
return this._contentType;
|
|
69
|
-
}
|
|
70
|
-
set contentType(contentType) {
|
|
71
|
-
this._contentType = contentType;
|
|
72
|
-
}
|
|
73
|
-
get authorization() {
|
|
74
|
-
return this._authorization;
|
|
75
|
-
}
|
|
76
|
-
get currentBytesUploaded() {
|
|
77
|
-
return this._currentBytesUploaded;
|
|
78
|
-
}
|
|
79
|
-
set currentBytesUploaded(value) {
|
|
80
|
-
this._currentBytesUploaded = value;
|
|
81
|
-
}
|
|
82
|
-
set status(status) {
|
|
83
|
-
this._status = status;
|
|
84
|
-
}
|
|
85
|
-
get status() {
|
|
86
|
-
return this._status;
|
|
87
|
-
}
|
|
88
|
-
get fileLocation() {
|
|
89
|
-
return this._fileLocation;
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
exports.ResumableUpload = ResumableUpload;
|
|
93
|
-
var UploadStatus;
|
|
94
|
-
(function (UploadStatus) {
|
|
95
|
-
UploadStatus[UploadStatus["ACTIVE"] = 0] = "ACTIVE";
|
|
96
|
-
UploadStatus[UploadStatus["CANCELLED"] = 1] = "CANCELLED";
|
|
97
|
-
UploadStatus[UploadStatus["FINISHED"] = 2] = "FINISHED";
|
|
98
|
-
})(UploadStatus = exports.UploadStatus || (exports.UploadStatus = {}));
|
|
99
41
|
class StorageLayer {
|
|
100
|
-
constructor(_projectId) {
|
|
42
|
+
constructor(_projectId, _rulesValidator, _adminCredsValidator, _persistence) {
|
|
101
43
|
this._projectId = _projectId;
|
|
44
|
+
this._rulesValidator = _rulesValidator;
|
|
45
|
+
this._adminCredsValidator = _adminCredsValidator;
|
|
46
|
+
this._persistence = _persistence;
|
|
102
47
|
this.reset();
|
|
103
48
|
this._cloudFunctions = new cloudFunctions_1.StorageCloudFunctions(this._projectId);
|
|
104
49
|
}
|
|
105
50
|
reset() {
|
|
106
51
|
this._files = new Map();
|
|
107
|
-
this._persistence = new Persistence(`${(0, os_1.tmpdir)()}/firebase/storage/blobs`);
|
|
108
|
-
this._uploads = new Map();
|
|
109
52
|
this._buckets = new Map();
|
|
110
53
|
}
|
|
111
54
|
createBucket(id) {
|
|
@@ -123,6 +66,22 @@ class StorageLayer {
|
|
|
123
66
|
}
|
|
124
67
|
return [...this._buckets.values()];
|
|
125
68
|
}
|
|
69
|
+
async handleGetObject(request, skipAuth = false) {
|
|
70
|
+
var _a;
|
|
71
|
+
const metadata = this.getMetadata(request.bucketId, request.decodedObjectId);
|
|
72
|
+
const hasValidDownloadToken = ((metadata === null || metadata === void 0 ? void 0 : metadata.downloadTokens) || []).includes((_a = request.downloadToken) !== null && _a !== void 0 ? _a : "");
|
|
73
|
+
let authorized = skipAuth || hasValidDownloadToken;
|
|
74
|
+
if (!authorized) {
|
|
75
|
+
authorized = await this._rulesValidator.validate(["b", request.bucketId, "o", request.decodedObjectId].join("/"), request.bucketId, types_1.RulesetOperationMethod.GET, { before: metadata === null || metadata === void 0 ? void 0 : metadata.asRulesResource() }, request.authorization);
|
|
76
|
+
}
|
|
77
|
+
if (!authorized) {
|
|
78
|
+
throw new errors_1.ForbiddenError("Failed auth");
|
|
79
|
+
}
|
|
80
|
+
if (!metadata) {
|
|
81
|
+
throw new errors_1.NotFoundError("File not found");
|
|
82
|
+
}
|
|
83
|
+
return { metadata: metadata, data: this.getBytes(request.bucketId, request.decodedObjectId) };
|
|
84
|
+
}
|
|
126
85
|
getMetadata(bucket, object) {
|
|
127
86
|
const key = this.path(bucket, object);
|
|
128
87
|
const val = this._files.get(key);
|
|
@@ -131,16 +90,6 @@ class StorageLayer {
|
|
|
131
90
|
}
|
|
132
91
|
return;
|
|
133
92
|
}
|
|
134
|
-
createMetadata(upload) {
|
|
135
|
-
const bytes = this._persistence.readBytes(upload.fileLocation, upload.currentBytesUploaded);
|
|
136
|
-
return new metadata_1.StoredFileMetadata({
|
|
137
|
-
name: upload.objectId,
|
|
138
|
-
bucket: upload.bucketId,
|
|
139
|
-
contentType: "",
|
|
140
|
-
contentEncoding: upload.metadata.contentEncoding,
|
|
141
|
-
customMetadata: upload.metadata.metadata,
|
|
142
|
-
}, this._cloudFunctions, bytes);
|
|
143
|
-
}
|
|
144
93
|
getBytes(bucket, object, size, offset) {
|
|
145
94
|
const key = this.path(bucket, object);
|
|
146
95
|
const val = this._files.get(key);
|
|
@@ -150,33 +99,17 @@ class StorageLayer {
|
|
|
150
99
|
}
|
|
151
100
|
return undefined;
|
|
152
101
|
}
|
|
153
|
-
|
|
154
|
-
this.
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
this._uploads.set(uploadId, upload);
|
|
160
|
-
return upload;
|
|
161
|
-
}
|
|
162
|
-
queryUpload(uploadId) {
|
|
163
|
-
return this._uploads.get(uploadId);
|
|
164
|
-
}
|
|
165
|
-
cancelUpload(upload) {
|
|
166
|
-
if (upload.status === UploadStatus.ACTIVE) {
|
|
167
|
-
this._persistence.deleteFile(upload.fileLocation);
|
|
168
|
-
upload.status = UploadStatus.CANCELLED;
|
|
102
|
+
async handleDeleteObject(request, skipAuth = false) {
|
|
103
|
+
const storedMetadata = this.getMetadata(request.bucketId, request.decodedObjectId);
|
|
104
|
+
const authorized = skipAuth ||
|
|
105
|
+
(await this._rulesValidator.validate(["b", request.bucketId, "o", request.decodedObjectId].join("/"), request.bucketId, types_1.RulesetOperationMethod.DELETE, { before: storedMetadata === null || storedMetadata === void 0 ? void 0 : storedMetadata.asRulesResource() }, request.authorization));
|
|
106
|
+
if (!authorized) {
|
|
107
|
+
throw new errors_1.ForbiddenError();
|
|
169
108
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (!upload) {
|
|
175
|
-
return undefined;
|
|
176
|
-
}
|
|
177
|
-
this._persistence.appendBytes(upload.fileLocation, bytes, upload.currentBytesUploaded);
|
|
178
|
-
upload.currentBytesUploaded += bytes.byteLength;
|
|
179
|
-
return upload;
|
|
109
|
+
if (!storedMetadata) {
|
|
110
|
+
throw new errors_1.NotFoundError();
|
|
111
|
+
}
|
|
112
|
+
this.deleteFile(request.bucketId, request.decodedObjectId);
|
|
180
113
|
}
|
|
181
114
|
deleteFile(bucketId, objectId) {
|
|
182
115
|
const isFolder = objectId.toLowerCase().endsWith("%2f");
|
|
@@ -198,77 +131,129 @@ class StorageLayer {
|
|
|
198
131
|
return true;
|
|
199
132
|
}
|
|
200
133
|
}
|
|
201
|
-
async
|
|
202
|
-
|
|
134
|
+
async handleUpdateObjectMetadata(request, skipAuth = false) {
|
|
135
|
+
const storedMetadata = this.getMetadata(request.bucketId, request.decodedObjectId);
|
|
136
|
+
const authorized = skipAuth ||
|
|
137
|
+
(await this._rulesValidator.validate(["b", request.bucketId, "o", request.decodedObjectId].join("/"), request.bucketId, types_1.RulesetOperationMethod.UPDATE, {
|
|
138
|
+
before: storedMetadata === null || storedMetadata === void 0 ? void 0 : storedMetadata.asRulesResource(),
|
|
139
|
+
after: storedMetadata === null || storedMetadata === void 0 ? void 0 : storedMetadata.asRulesResource(request.metadata),
|
|
140
|
+
}, request.authorization));
|
|
141
|
+
if (!authorized) {
|
|
142
|
+
throw new errors_1.ForbiddenError();
|
|
143
|
+
}
|
|
144
|
+
if (!storedMetadata) {
|
|
145
|
+
throw new errors_1.NotFoundError();
|
|
146
|
+
}
|
|
147
|
+
storedMetadata.update(request.metadata);
|
|
148
|
+
return storedMetadata;
|
|
203
149
|
}
|
|
204
|
-
|
|
205
|
-
upload.status
|
|
206
|
-
|
|
150
|
+
async handleUploadObject(upload, skipAuth = false) {
|
|
151
|
+
if (upload.status !== upload_1.UploadStatus.FINISHED) {
|
|
152
|
+
throw new Error(`Unexpected upload status encountered: ${upload.status}.`);
|
|
153
|
+
}
|
|
207
154
|
const filePath = this.path(upload.bucketId, upload.objectId);
|
|
208
|
-
const
|
|
209
|
-
|
|
155
|
+
const metadata = new metadata_1.StoredFileMetadata({
|
|
156
|
+
name: upload.objectId,
|
|
157
|
+
bucket: upload.bucketId,
|
|
158
|
+
contentType: upload.metadata.contentType || "application/octet-stream",
|
|
159
|
+
contentDisposition: upload.metadata.contentDisposition,
|
|
160
|
+
contentEncoding: upload.metadata.contentEncoding,
|
|
161
|
+
contentLanguage: upload.metadata.contentLanguage,
|
|
162
|
+
cacheControl: upload.metadata.cacheControl,
|
|
163
|
+
customMetadata: upload.metadata.metadata,
|
|
164
|
+
}, this._cloudFunctions, this._persistence.readBytes(upload.path, upload.size));
|
|
165
|
+
const authorized = skipAuth ||
|
|
166
|
+
(await this._rulesValidator.validate(["b", upload.bucketId, "o", upload.objectId].join("/"), upload.bucketId, types_1.RulesetOperationMethod.CREATE, { after: metadata === null || metadata === void 0 ? void 0 : metadata.asRulesResource() }, upload.authorization));
|
|
167
|
+
if (!authorized) {
|
|
168
|
+
this._persistence.deleteFile(upload.path);
|
|
169
|
+
throw new errors_1.ForbiddenError();
|
|
170
|
+
}
|
|
210
171
|
this._persistence.deleteFile(filePath, true);
|
|
211
|
-
this._persistence.renameFile(upload.
|
|
212
|
-
this.
|
|
213
|
-
|
|
172
|
+
this._persistence.renameFile(upload.path, filePath);
|
|
173
|
+
this._files.set(filePath, new StoredFile(metadata, this._persistence.getDiskPath(filePath)));
|
|
174
|
+
this._cloudFunctions.dispatch("finalize", new metadata_1.CloudStorageObjectMetadata(metadata));
|
|
175
|
+
return metadata;
|
|
214
176
|
}
|
|
215
|
-
|
|
216
|
-
const filePath = this.path(
|
|
177
|
+
copyFile(sourceFile, destinationBucket, destinationObject, incomingMetadata) {
|
|
178
|
+
const filePath = this.path(destinationBucket, destinationObject);
|
|
217
179
|
this._persistence.deleteFile(filePath, true);
|
|
180
|
+
const bytes = this.getBytes(sourceFile.bucket, sourceFile.name);
|
|
218
181
|
this._persistence.appendBytes(filePath, bytes);
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
}
|
|
226
|
-
|
|
182
|
+
const newMetadata = Object.assign(Object.assign(Object.assign({}, sourceFile), { metadata: sourceFile.customMetadata }), incomingMetadata);
|
|
183
|
+
if (sourceFile.downloadTokens.length &&
|
|
184
|
+
!((incomingMetadata === null || incomingMetadata === void 0 ? void 0 : incomingMetadata.metadata) && Object.keys(incomingMetadata === null || incomingMetadata === void 0 ? void 0 : incomingMetadata.metadata).length)) {
|
|
185
|
+
if (!newMetadata.metadata)
|
|
186
|
+
newMetadata.metadata = {};
|
|
187
|
+
newMetadata.metadata.firebaseStorageDownloadTokens = sourceFile.downloadTokens.join(",");
|
|
188
|
+
}
|
|
189
|
+
if (newMetadata.metadata) {
|
|
190
|
+
for (const [k, v] of Object.entries(newMetadata.metadata)) {
|
|
191
|
+
if (v === null)
|
|
192
|
+
newMetadata.metadata[k] = "";
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const copiedFileMetadata = new metadata_1.StoredFileMetadata({
|
|
196
|
+
name: destinationObject,
|
|
197
|
+
bucket: destinationBucket,
|
|
198
|
+
contentType: newMetadata.contentType || "application/octet-stream",
|
|
199
|
+
contentDisposition: newMetadata.contentDisposition,
|
|
200
|
+
contentEncoding: newMetadata.contentEncoding,
|
|
201
|
+
contentLanguage: newMetadata.contentLanguage,
|
|
202
|
+
cacheControl: newMetadata.cacheControl,
|
|
203
|
+
customMetadata: newMetadata.metadata,
|
|
204
|
+
}, this._cloudFunctions, bytes, incomingMetadata);
|
|
205
|
+
const file = new StoredFile(copiedFileMetadata, this._persistence.getDiskPath(filePath));
|
|
227
206
|
this._files.set(filePath, file);
|
|
228
207
|
this._cloudFunctions.dispatch("finalize", new metadata_1.CloudStorageObjectMetadata(file.metadata));
|
|
229
208
|
return file.metadata;
|
|
230
209
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
if (!
|
|
236
|
-
|
|
237
|
-
}
|
|
238
|
-
if (!prefix.endsWith(delimiter)) {
|
|
239
|
-
prefix += delimiter;
|
|
240
|
-
}
|
|
241
|
-
if (!prefix.startsWith(delimiter)) {
|
|
242
|
-
prefix = delimiter + prefix;
|
|
210
|
+
async handleListObjects(request, skipAuth = false) {
|
|
211
|
+
var _a, _b, _c;
|
|
212
|
+
const authorized = skipAuth ||
|
|
213
|
+
(await this._rulesValidator.validate(["b", request.bucketId, "o", request.prefix].join("/"), request.bucketId, types_1.RulesetOperationMethod.LIST, {}, request.authorization));
|
|
214
|
+
if (!authorized) {
|
|
215
|
+
throw new errors_1.ForbiddenError();
|
|
243
216
|
}
|
|
217
|
+
const itemsResults = this.listItems(request.bucketId, request.prefix, request.delimiter, request.pageToken, request.maxResults);
|
|
218
|
+
return new list_1.ListResponse((_a = itemsResults.prefixes) !== null && _a !== void 0 ? _a : [], (_c = (_b = itemsResults.items) === null || _b === void 0 ? void 0 : _b.map((i) => new list_1.ListItem(i.name, i.bucket))) !== null && _c !== void 0 ? _c : [], itemsResults.nextPageToken);
|
|
219
|
+
}
|
|
220
|
+
listItems(bucket, prefix, delimiter, pageToken, maxResults) {
|
|
244
221
|
let items = [];
|
|
245
222
|
const prefixes = new Set();
|
|
246
223
|
for (const [, file] of this._files) {
|
|
247
224
|
if (file.metadata.bucket !== bucket) {
|
|
248
225
|
continue;
|
|
249
226
|
}
|
|
250
|
-
|
|
227
|
+
const name = file.metadata.name;
|
|
251
228
|
if (!name.startsWith(prefix)) {
|
|
252
229
|
continue;
|
|
253
230
|
}
|
|
254
|
-
|
|
255
|
-
if (
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
items.push(file.metadata.name);
|
|
231
|
+
let includeMetadata = true;
|
|
232
|
+
if (delimiter) {
|
|
233
|
+
const delimiterIdx = name.indexOf(delimiter);
|
|
234
|
+
const delimiterAfterPrefixIdx = name.indexOf(delimiter, prefix.length);
|
|
235
|
+
includeMetadata = delimiterIdx === -1 || delimiterAfterPrefixIdx === -1;
|
|
236
|
+
if (delimiterAfterPrefixIdx !== -1) {
|
|
237
|
+
prefixes.add(name.slice(0, delimiterAfterPrefixIdx + delimiter.length));
|
|
262
238
|
}
|
|
263
239
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
prefixes.add(prefixPath);
|
|
240
|
+
if (includeMetadata) {
|
|
241
|
+
items.push(file.metadata);
|
|
267
242
|
}
|
|
268
243
|
}
|
|
269
|
-
items.sort()
|
|
244
|
+
items.sort((a, b) => {
|
|
245
|
+
if (a.name === b.name) {
|
|
246
|
+
return 0;
|
|
247
|
+
}
|
|
248
|
+
else if (a.name < b.name) {
|
|
249
|
+
return -1;
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
return 1;
|
|
253
|
+
}
|
|
254
|
+
});
|
|
270
255
|
if (pageToken) {
|
|
271
|
-
const idx = items.findIndex((v) => v === pageToken);
|
|
256
|
+
const idx = items.findIndex((v) => v.name === pageToken);
|
|
272
257
|
if (idx !== -1) {
|
|
273
258
|
items = items.slice(idx);
|
|
274
259
|
}
|
|
@@ -278,81 +263,39 @@ class StorageLayer {
|
|
|
278
263
|
}
|
|
279
264
|
let nextPageToken = undefined;
|
|
280
265
|
if (items.length > maxResults) {
|
|
281
|
-
nextPageToken = items[maxResults];
|
|
266
|
+
nextPageToken = items[maxResults].name;
|
|
282
267
|
items = items.slice(0, maxResults);
|
|
283
268
|
}
|
|
284
|
-
return new list_1.ListResponse([...prefixes].sort(), items.map((i) => new list_1.ListItem(i, bucket)), nextPageToken);
|
|
285
|
-
}
|
|
286
|
-
listItems(bucket, prefix, delimiter, pageToken, maxResults) {
|
|
287
|
-
if (!delimiter) {
|
|
288
|
-
delimiter = "/";
|
|
289
|
-
}
|
|
290
|
-
if (!prefix) {
|
|
291
|
-
prefix = "";
|
|
292
|
-
}
|
|
293
|
-
if (!prefix.endsWith(delimiter)) {
|
|
294
|
-
prefix += delimiter;
|
|
295
|
-
}
|
|
296
|
-
let items = [];
|
|
297
|
-
for (const [, file] of this._files) {
|
|
298
|
-
if (file.metadata.bucket !== bucket) {
|
|
299
|
-
continue;
|
|
300
|
-
}
|
|
301
|
-
let name = file.metadata.name;
|
|
302
|
-
if (!name.startsWith(prefix)) {
|
|
303
|
-
continue;
|
|
304
|
-
}
|
|
305
|
-
name = name.substring(prefix.length);
|
|
306
|
-
if (name.startsWith(delimiter)) {
|
|
307
|
-
name = name.substring(prefix.length);
|
|
308
|
-
}
|
|
309
|
-
items.push(this.path(file.metadata.bucket, file.metadata.name));
|
|
310
|
-
}
|
|
311
|
-
items.sort();
|
|
312
|
-
if (pageToken) {
|
|
313
|
-
const idx = items.findIndex((v) => v === pageToken);
|
|
314
|
-
if (idx !== -1) {
|
|
315
|
-
items = items.slice(idx);
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
if (!maxResults) {
|
|
319
|
-
maxResults = 1000;
|
|
320
|
-
}
|
|
321
269
|
return {
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
if (!storedFile) {
|
|
326
|
-
return console.warn(`No file ${item}`);
|
|
327
|
-
}
|
|
328
|
-
return new metadata_1.CloudStorageObjectMetadata(storedFile.metadata);
|
|
329
|
-
}),
|
|
270
|
+
nextPageToken,
|
|
271
|
+
prefixes: prefixes.size > 0 ? [...prefixes].sort() : undefined,
|
|
272
|
+
items: items.length > 0 ? items.map((item) => new metadata_1.CloudStorageObjectMetadata(item)) : undefined,
|
|
330
273
|
};
|
|
331
274
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
275
|
+
handleCreateDownloadToken(request) {
|
|
276
|
+
if (!this._adminCredsValidator.validate(request.authorization)) {
|
|
277
|
+
throw new errors_1.ForbiddenError();
|
|
278
|
+
}
|
|
279
|
+
const metadata = this.getMetadata(request.bucketId, request.decodedObjectId);
|
|
280
|
+
if (!metadata) {
|
|
281
|
+
throw new errors_1.NotFoundError();
|
|
337
282
|
}
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
return md;
|
|
283
|
+
metadata.addDownloadToken();
|
|
284
|
+
return metadata;
|
|
341
285
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
286
|
+
handleDeleteDownloadToken(request) {
|
|
287
|
+
if (!this._adminCredsValidator.validate(request.authorization)) {
|
|
288
|
+
throw new errors_1.ForbiddenError();
|
|
289
|
+
}
|
|
290
|
+
const metadata = this.getMetadata(request.bucketId, request.decodedObjectId);
|
|
291
|
+
if (!metadata) {
|
|
292
|
+
throw new errors_1.NotFoundError();
|
|
347
293
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
return md;
|
|
294
|
+
metadata.deleteDownloadToken(request.token);
|
|
295
|
+
return metadata;
|
|
351
296
|
}
|
|
352
297
|
path(bucket, object) {
|
|
353
|
-
|
|
354
|
-
const filename = path.basename(object) + (object.endsWith("/") ? "/" : "");
|
|
355
|
-
return path.join(bucket, directory, encodeURIComponent(filename));
|
|
298
|
+
return path.join(bucket, object);
|
|
356
299
|
}
|
|
357
300
|
get dirPath() {
|
|
358
301
|
return this._persistence.dirPath;
|
|
@@ -375,9 +318,7 @@ class StorageLayer {
|
|
|
375
318
|
try {
|
|
376
319
|
for (var _b = __asyncValues(this._files.entries()), _c; _c = await _b.next(), !_c.done;) {
|
|
377
320
|
const [p, file] = _c.value;
|
|
378
|
-
const metadataExportPath = path.join(metadataDirPath, p) + ".json";
|
|
379
|
-
const metadataExportDirPath = path.dirname(metadataExportPath);
|
|
380
|
-
await fse.ensureDir(metadataExportDirPath);
|
|
321
|
+
const metadataExportPath = path.join(metadataDirPath, encodeURIComponent(p)) + ".json";
|
|
381
322
|
await fse.writeFile(metadataExportPath, metadata_1.StoredFileMetadata.toJSON(file.metadata));
|
|
382
323
|
}
|
|
383
324
|
}
|
|
@@ -391,7 +332,7 @@ class StorageLayer {
|
|
|
391
332
|
}
|
|
392
333
|
import(storageExportPath) {
|
|
393
334
|
const bucketsFile = path.join(storageExportPath, "buckets.json");
|
|
394
|
-
const bucketsList = JSON.parse(
|
|
335
|
+
const bucketsList = JSON.parse((0, fs_1.readFileSync)(bucketsFile, "utf-8"));
|
|
395
336
|
for (const b of bucketsList.buckets) {
|
|
396
337
|
const bucketMetadata = new metadata_1.CloudStorageBucketMetadata(b.id);
|
|
397
338
|
this._buckets.set(b.id, bucketMetadata);
|
|
@@ -405,11 +346,11 @@ class StorageLayer {
|
|
|
405
346
|
logger_1.logger.debug(`Skipping unexpected storage metadata file: ${f}`);
|
|
406
347
|
continue;
|
|
407
348
|
}
|
|
408
|
-
const metadata = metadata_1.StoredFileMetadata.fromJSON(
|
|
349
|
+
const metadata = metadata_1.StoredFileMetadata.fromJSON((0, fs_1.readFileSync)(f, "utf-8"), this._cloudFunctions);
|
|
409
350
|
const metadataRelPath = path.relative(metadataDir, f);
|
|
410
351
|
const blobPath = metadataRelPath.substring(0, metadataRelPath.length - dotJson.length);
|
|
411
352
|
const blobAbsPath = path.join(blobsDir, blobPath);
|
|
412
|
-
if (!
|
|
353
|
+
if (!(0, fs_1.existsSync)(blobAbsPath)) {
|
|
413
354
|
logger_1.logger.warn(`Could not find file "${blobPath}" in storage export.`);
|
|
414
355
|
continue;
|
|
415
356
|
}
|
|
@@ -419,10 +360,10 @@ class StorageLayer {
|
|
|
419
360
|
fse.copySync(blobsDir, this.dirPath);
|
|
420
361
|
}
|
|
421
362
|
*walkDirSync(dir) {
|
|
422
|
-
const files =
|
|
363
|
+
const files = (0, fs_1.readdirSync)(dir);
|
|
423
364
|
for (const file of files) {
|
|
424
365
|
const p = path.join(dir, file);
|
|
425
|
-
if (
|
|
366
|
+
if ((0, fs_1.statSync)(p).isDirectory()) {
|
|
426
367
|
yield* this.walkDirSync(p);
|
|
427
368
|
}
|
|
428
369
|
else {
|
|
@@ -432,87 +373,3 @@ class StorageLayer {
|
|
|
432
373
|
}
|
|
433
374
|
}
|
|
434
375
|
exports.StorageLayer = StorageLayer;
|
|
435
|
-
class Persistence {
|
|
436
|
-
constructor(dirPath) {
|
|
437
|
-
this._dirPath = dirPath;
|
|
438
|
-
if (!(0, fs_1.existsSync)(dirPath)) {
|
|
439
|
-
(0, fs_1.mkdirSync)(dirPath, {
|
|
440
|
-
recursive: true,
|
|
441
|
-
});
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
get dirPath() {
|
|
445
|
-
return this._dirPath;
|
|
446
|
-
}
|
|
447
|
-
appendBytes(fileName, bytes, fileOffset) {
|
|
448
|
-
const filepath = this.getDiskPath(fileName);
|
|
449
|
-
const encodedSlashIndex = filepath.toLowerCase().lastIndexOf("%2f");
|
|
450
|
-
const dirPath = encodedSlashIndex >= 0 ? filepath.substring(0, encodedSlashIndex) : path.dirname(filepath);
|
|
451
|
-
if (!(0, fs_1.existsSync)(dirPath)) {
|
|
452
|
-
(0, fs_1.mkdirSync)(dirPath, {
|
|
453
|
-
recursive: true,
|
|
454
|
-
});
|
|
455
|
-
}
|
|
456
|
-
let fd;
|
|
457
|
-
try {
|
|
458
|
-
fs.appendFileSync(filepath, bytes);
|
|
459
|
-
return filepath;
|
|
460
|
-
}
|
|
461
|
-
finally {
|
|
462
|
-
if (fd) {
|
|
463
|
-
(0, fs_1.closeSync)(fd);
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
readBytes(fileName, size, fileOffset) {
|
|
468
|
-
const path = this.getDiskPath(fileName);
|
|
469
|
-
let fd;
|
|
470
|
-
try {
|
|
471
|
-
fd = (0, fs_1.openSync)(path, "r");
|
|
472
|
-
const buf = Buffer.alloc(size);
|
|
473
|
-
const offset = fileOffset && fileOffset > 0 ? fileOffset : 0;
|
|
474
|
-
(0, fs_1.readSync)(fd, buf, 0, size, offset);
|
|
475
|
-
return buf;
|
|
476
|
-
}
|
|
477
|
-
finally {
|
|
478
|
-
if (fd) {
|
|
479
|
-
(0, fs_1.closeSync)(fd);
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
deleteFile(fileName, failSilently = false) {
|
|
484
|
-
try {
|
|
485
|
-
(0, fs_1.unlinkSync)(this.getDiskPath(fileName));
|
|
486
|
-
}
|
|
487
|
-
catch (err) {
|
|
488
|
-
if (!failSilently) {
|
|
489
|
-
throw err;
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
deleteAll() {
|
|
494
|
-
return new Promise((resolve, reject) => {
|
|
495
|
-
rimraf(this._dirPath, (err) => {
|
|
496
|
-
if (err) {
|
|
497
|
-
reject(err);
|
|
498
|
-
}
|
|
499
|
-
else {
|
|
500
|
-
resolve();
|
|
501
|
-
}
|
|
502
|
-
});
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
renameFile(oldName, newName) {
|
|
506
|
-
const dirPath = this.getDiskPath(path.dirname(newName));
|
|
507
|
-
if (!(0, fs_1.existsSync)(dirPath)) {
|
|
508
|
-
(0, fs_1.mkdirSync)(dirPath, {
|
|
509
|
-
recursive: true,
|
|
510
|
-
});
|
|
511
|
-
}
|
|
512
|
-
(0, fs_1.renameSync)(this.getDiskPath(oldName), this.getDiskPath(newName));
|
|
513
|
-
}
|
|
514
|
-
getDiskPath(fileName) {
|
|
515
|
-
return path.join(this._dirPath, fileName);
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
exports.Persistence = Persistence;
|