@google-cloud/nodejs-common 1.7.9-alpha → 1.8.1-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/apis/ads_data_hub.js +50 -87
- package/src/apis/base/auth_restful_api.js +59 -0
- package/src/apis/base/restful_api_base.js +143 -0
- package/src/apis/index.js +16 -1
- package/src/apis/sendgrid.js +65 -0
- package/src/components/cloudfunctions_utils.js +32 -103
- package/src/components/firestore/access_base.js +23 -19
- package/src/components/firestore/data_access_object.js +12 -22
- package/src/components/firestore/datastore_mode_access.js +60 -66
- package/src/components/firestore/native_mode_access.js +48 -56
- package/src/components/storage.js +44 -70
- package/src/components/utils.js +5 -7
|
@@ -70,51 +70,44 @@ class NativeModeAccess {
|
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
/** @override */
|
|
73
|
-
getObject(id) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
console.error(error);
|
|
86
|
-
});
|
|
73
|
+
async getObject(id) {
|
|
74
|
+
try {
|
|
75
|
+
const documentSnapshot = await this.getDocumentReference(id).get();
|
|
76
|
+
if (documentSnapshot.exists) {
|
|
77
|
+
this.logger.debug(`Get ${this.path}/${id}:`, documentSnapshot);
|
|
78
|
+
return documentSnapshot.data();
|
|
79
|
+
} else {
|
|
80
|
+
this.logger.info(`Failed to find doc: ${this.path}/${id}`);
|
|
81
|
+
}
|
|
82
|
+
} catch (error) {
|
|
83
|
+
this.logger.error(error);
|
|
84
|
+
}
|
|
87
85
|
}
|
|
88
86
|
|
|
89
87
|
/** @override */
|
|
90
|
-
saveObject(data, id = undefined) {
|
|
88
|
+
async saveObject(data, id = undefined) {
|
|
91
89
|
this.logger.debug(`Start to save doc ${this.path}/${id}`, data);
|
|
92
90
|
if (id) {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
return id;
|
|
97
|
-
});
|
|
91
|
+
const result = await this.getDocumentReference(id).set(data);
|
|
92
|
+
this.logger.debug(`Result of saving doc ${this.path}/${id}: `, result);
|
|
93
|
+
return id;
|
|
98
94
|
} else {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
});
|
|
95
|
+
this.logger.info(`Create new doc under ${this.path}`);
|
|
96
|
+
const documentReference = await this.collection.add(data);
|
|
97
|
+
this.logger.debug(
|
|
98
|
+
`Saved ${JSON.stringify(data)} as:`, documentReference);
|
|
99
|
+
return documentReference.id;
|
|
105
100
|
}
|
|
106
101
|
}
|
|
107
102
|
|
|
108
103
|
/** @override */
|
|
109
|
-
deleteObject(id) {
|
|
104
|
+
async deleteObject(id) {
|
|
110
105
|
const documentReference = this.getDocumentReference(id);
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
});
|
|
117
|
-
});
|
|
106
|
+
const documentSnapshot = await documentReference.get();
|
|
107
|
+
if (!documentSnapshot.exists) return false;
|
|
108
|
+
const writeResult = await documentReference.delete();
|
|
109
|
+
this.logger.debug(`Delete ${this.path}/${id}: `, writeResult);
|
|
110
|
+
return true;
|
|
118
111
|
}
|
|
119
112
|
|
|
120
113
|
/**
|
|
@@ -123,7 +116,7 @@ class NativeModeAccess {
|
|
|
123
116
|
* It doesn't work when filter and order have different property names.
|
|
124
117
|
* @override
|
|
125
118
|
*/
|
|
126
|
-
queryObjects(filters, order, limit, offset) {
|
|
119
|
+
async queryObjects(filters, order, limit, offset) {
|
|
127
120
|
let query = this.collection;
|
|
128
121
|
if (filters) {
|
|
129
122
|
filters.forEach((filter) => {
|
|
@@ -137,22 +130,21 @@ class NativeModeAccess {
|
|
|
137
130
|
}
|
|
138
131
|
if (limit) query = query.limit(limit);
|
|
139
132
|
if (offset) query = query.offset(offset);
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
});
|
|
133
|
+
try {
|
|
134
|
+
const snapshot = await query.get(); //type: QuerySnapshot
|
|
135
|
+
const result = [];
|
|
136
|
+
if (snapshot.empty) {
|
|
137
|
+
this.logger.debug('No matching documents.');
|
|
138
|
+
return result;
|
|
139
|
+
}
|
|
140
|
+
snapshot.forEach((document) => {
|
|
141
|
+
result.push({ id: document.id, entity: document.data() });
|
|
142
|
+
});
|
|
143
|
+
return result;
|
|
144
|
+
} catch (error) {
|
|
145
|
+
this.logger.error('Error getting documents', error);
|
|
146
|
+
throw error;
|
|
147
|
+
}
|
|
156
148
|
}
|
|
157
149
|
|
|
158
150
|
/** @override */
|
|
@@ -162,12 +154,12 @@ class NativeModeAccess {
|
|
|
162
154
|
|
|
163
155
|
/** @override */
|
|
164
156
|
wrapInTransaction(id, transactionOperation) {
|
|
165
|
-
return (transaction) => {
|
|
166
|
-
|
|
157
|
+
return async (transaction) => {
|
|
158
|
+
this.logger.info(`Transaction starts for ${this.path}/${id}.`);
|
|
167
159
|
const documentReference = this.getDocumentReference(id);
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
160
|
+
const documentSnapshot = await transaction.get(documentReference);
|
|
161
|
+
return transactionOperation(
|
|
162
|
+
documentSnapshot, documentReference, transaction);
|
|
171
163
|
};
|
|
172
164
|
}
|
|
173
165
|
}
|
|
@@ -59,10 +59,9 @@ class StorageFile {
|
|
|
59
59
|
* Gets the file size.
|
|
60
60
|
* @return {!Promise<number>}
|
|
61
61
|
*/
|
|
62
|
-
getFileSize() {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
});
|
|
62
|
+
async getFileSize() {
|
|
63
|
+
const fileResponse = await this.file.get();
|
|
64
|
+
return parseInt(fileResponse[1].size, 10);
|
|
66
65
|
};
|
|
67
66
|
|
|
68
67
|
/**
|
|
@@ -70,11 +69,10 @@ class StorageFile {
|
|
|
70
69
|
* @param {string=} prefix The file name prefix.
|
|
71
70
|
* @return {!Promise<!Array<string>>} Array of file names.
|
|
72
71
|
*/
|
|
73
|
-
listFiles(prefix = this.fileName) {
|
|
72
|
+
async listFiles(prefix = this.fileName) {
|
|
74
73
|
const options = {prefix: prefix, delimiter: '/'};
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
);
|
|
74
|
+
const [files] = await this.bucket.getFiles(options);
|
|
75
|
+
return files.map((file) => file.name);
|
|
78
76
|
}
|
|
79
77
|
|
|
80
78
|
/**
|
|
@@ -84,32 +82,25 @@ class StorageFile {
|
|
|
84
82
|
* @param {?number} end End position, default is the end of the file.
|
|
85
83
|
* @return {!Promise<string>} contents File content between the start and end.
|
|
86
84
|
*/
|
|
87
|
-
loadContent(start = 0, end) {
|
|
85
|
+
async loadContent(start = 0, end) {
|
|
88
86
|
if (start < 0) {
|
|
89
87
|
console.log(`GCS load 'start' before 0 [${start}], move it to 0.`);
|
|
90
88
|
start = 0;
|
|
91
89
|
}
|
|
92
90
|
if (end < start) {
|
|
93
91
|
console.log(`GCS load for [${start}, ${end}], returns empty string.`);
|
|
94
|
-
return
|
|
92
|
+
return '';
|
|
95
93
|
}
|
|
96
|
-
const option = {
|
|
97
|
-
start: start,
|
|
98
|
-
end: end,
|
|
99
|
-
};
|
|
94
|
+
const option = { start, end };
|
|
100
95
|
const stream = this.file.createReadStream(option);
|
|
101
96
|
return new Promise((resolve, reject) => {
|
|
102
97
|
const chunks = [];
|
|
103
|
-
stream.on('data', (chunk) =>
|
|
104
|
-
chunks.push(chunk);
|
|
105
|
-
});
|
|
98
|
+
stream.on('data', (chunk) => void chunks.push(chunk));
|
|
106
99
|
stream.on('end', () => {
|
|
107
100
|
console.log(`Get [${this.fileName}] from ${start} to ${end}`);
|
|
108
101
|
resolve(chunks.join(''));
|
|
109
102
|
});
|
|
110
|
-
stream.on('error', (error) =>
|
|
111
|
-
reject(error);
|
|
112
|
-
});
|
|
103
|
+
stream.on('error', (error) => void reject(error));
|
|
113
104
|
});
|
|
114
105
|
};
|
|
115
106
|
|
|
@@ -157,20 +148,15 @@ class StorageFile {
|
|
|
157
148
|
* @param {number=} index The start point of this round split.
|
|
158
149
|
* @return {!Promise<!Array<!Array<number,number>>>}
|
|
159
150
|
*/
|
|
160
|
-
getSplitRanges(fileSize, splitSize, index = 0) {
|
|
151
|
+
async getSplitRanges(fileSize, splitSize, index = 0) {
|
|
161
152
|
if (index + splitSize >= fileSize) {
|
|
162
|
-
return
|
|
153
|
+
return [[index, fileSize - 1]];
|
|
163
154
|
} else {
|
|
164
155
|
const end = index + splitSize - 1;
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
return this.getSplitRanges(fileSize, splitSize, realEnd + 1)
|
|
170
|
-
.then((splits) => {
|
|
171
|
-
return Promise.resolve(piece.concat(splits));
|
|
172
|
-
});
|
|
173
|
-
});
|
|
156
|
+
const realEnd = await this.getLastLineBreaker(index, end);
|
|
157
|
+
const piece = [[index, realEnd]];
|
|
158
|
+
const splits = await this.getSplitRanges(fileSize, splitSize, realEnd + 1);
|
|
159
|
+
return piece.concat(splits);
|
|
174
160
|
}
|
|
175
161
|
}
|
|
176
162
|
|
|
@@ -185,13 +171,13 @@ class StorageFile {
|
|
|
185
171
|
copyRangeToFile(start, end, croppedFileName) {
|
|
186
172
|
const outputFile = this.bucket.file(croppedFileName);
|
|
187
173
|
return new Promise((resolve) => {
|
|
188
|
-
this.file.createReadStream({start, end,})
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
174
|
+
this.file.createReadStream({ start, end, })
|
|
175
|
+
.pipe(outputFile.createWriteStream())
|
|
176
|
+
.on('finish', async () => {
|
|
177
|
+
const [{ contentType }] = await this.file.getMetadata();
|
|
178
|
+
const [file] = await outputFile.setMetadata({ contentType });
|
|
179
|
+
return file.name;
|
|
180
|
+
});
|
|
195
181
|
});
|
|
196
182
|
}
|
|
197
183
|
|
|
@@ -209,28 +195,19 @@ class StorageFile {
|
|
|
209
195
|
* @param {string=} outputName File name for output.
|
|
210
196
|
* @return {!Promise<string>} Output file name.
|
|
211
197
|
*/
|
|
212
|
-
addHeader(
|
|
198
|
+
async addHeader(
|
|
213
199
|
header, sourceName = this.fileName,
|
|
214
200
|
outputName = sourceName + '_w_header') {
|
|
215
201
|
const headerFile = this.bucket.file('_header_' + (new Date()).getTime());
|
|
216
202
|
if (!header.endsWith(LINE_BREAKER)) header = header + LINE_BREAKER;
|
|
203
|
+
await headerFile.save(header);
|
|
217
204
|
const sourceFile = this.bucket.file(sourceName);
|
|
218
|
-
const
|
|
219
|
-
|
|
220
|
-
headerFile.
|
|
221
|
-
];
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
.combine([headerFile, sourceFile], this.bucket.file(outputName))
|
|
225
|
-
.then(([file]) => {
|
|
226
|
-
const promises = [
|
|
227
|
-
file.setMetadata({contentType}),
|
|
228
|
-
headerFile.delete(),
|
|
229
|
-
];
|
|
230
|
-
return Promise.all(promises);
|
|
231
|
-
})
|
|
232
|
-
.then(([[file]]) => file.name);
|
|
233
|
-
});
|
|
205
|
+
const [{ contentType }] = await sourceFile.getMetadata();
|
|
206
|
+
const [outputFile] = await this.bucket.combine(
|
|
207
|
+
[headerFile, sourceFile], this.bucket.file(outputName));
|
|
208
|
+
const [file] = await outputFile.setMetadata({ contentType });
|
|
209
|
+
await headerFile.delete();
|
|
210
|
+
return file.name;
|
|
234
211
|
}
|
|
235
212
|
|
|
236
213
|
/**
|
|
@@ -242,21 +219,18 @@ class StorageFile {
|
|
|
242
219
|
* Default value DEFAULT_SPLIT_SIZE.
|
|
243
220
|
* @return {!Promise<!Array<string>>} The filenames of the output files.
|
|
244
221
|
*/
|
|
245
|
-
split(splitSize = DEFAULT_SPLIT_SIZE) {
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
});
|
|
258
|
-
}
|
|
259
|
-
});
|
|
222
|
+
async split(splitSize = DEFAULT_SPLIT_SIZE) {
|
|
223
|
+
const size = await this.getFileSize();
|
|
224
|
+
if (size <= splitSize) { // No need to split.
|
|
225
|
+
return [this.fileName];
|
|
226
|
+
}
|
|
227
|
+
// Trying to split the big file.
|
|
228
|
+
console.log(`Get file size: ${size}, split size: ${splitSize}.`);
|
|
229
|
+
const splitRanges = await this.getSplitRanges(size, splitSize);
|
|
230
|
+
return Promise.all(splitRanges.map(([start, end], index) => {
|
|
231
|
+
const newFile = `${this.fileName}-${index}-of-${splitRanges.length}`;
|
|
232
|
+
return this.copyRangeToFile(start, end, newFile);
|
|
233
|
+
}));
|
|
260
234
|
}
|
|
261
235
|
|
|
262
236
|
/**
|
package/src/components/utils.js
CHANGED
|
@@ -387,16 +387,14 @@ const getProperValue = (value, defaultValue, capped = true) => {
|
|
|
387
387
|
* time.
|
|
388
388
|
* @return {!Promise<!Object>}
|
|
389
389
|
*/
|
|
390
|
-
const wait = (time, value = '') => {
|
|
390
|
+
const wait = async (time, value = '') => {
|
|
391
391
|
let timeoutId;
|
|
392
|
-
|
|
392
|
+
await new Promise((resolve) => {
|
|
393
393
|
timeoutId = setTimeout(resolve, time);
|
|
394
394
|
});
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
return Promise.resolve(value);
|
|
399
|
-
});
|
|
395
|
+
clearTimeout(timeoutId);
|
|
396
|
+
timeoutId = null;
|
|
397
|
+
return value;
|
|
400
398
|
};
|
|
401
399
|
|
|
402
400
|
/**
|