@google-cloud/nodejs-common 2.3.0 → 2.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/install_functions.sh +0 -6
- package/package.json +13 -13
- package/src/apis/auth_client.js +10 -10
- package/src/apis/base/ads_api_common.js +15 -3
- package/src/apis/base/google_api_client.js +18 -5
- package/src/apis/doubleclick_search.js +0 -132
- package/src/apis/google_ads_api.js +17 -17
- package/src/apis/search_ads.js +82 -17
- package/src/apis/spreadsheets.js +28 -6
- package/src/components/firestore/access_base.js +3 -3
- package/src/components/firestore/data_access_object.js +1 -1
- package/src/components/firestore/datastore_mode_access.js +3 -3
- package/src/components/pubsub.js +1 -1
- package/src/components/scheduler.js +3 -3
- package/src/components/storage.js +1 -1
- package/src/components/utils.js +15 -1
- package/bin/apps_scripts.sh +0 -209
- package/bin/bigquery.sh +0 -53
- package/bin/google_ads.sh +0 -115
package/bin/install_functions.sh
CHANGED
|
@@ -2018,10 +2018,4 @@ join_string_array() {
|
|
|
2018
2018
|
printf %s "$first" "${@/#/$separator}" | sed -e "s/\\$separator/$separator/g"
|
|
2019
2019
|
}
|
|
2020
2020
|
|
|
2021
|
-
# Import other bash files.
|
|
2022
|
-
_SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
2023
|
-
source "${_SELF}/google_ads.sh"
|
|
2024
|
-
source "${_SELF}/bigquery.sh"
|
|
2025
|
-
source "${_SELF}/apps_scripts.sh"
|
|
2026
|
-
|
|
2027
2021
|
printf '%s\n' "Common Bash Library is loaded."
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@google-cloud/nodejs-common",
|
|
3
|
-
"version": "2.3.
|
|
3
|
+
"version": "2.3.9",
|
|
4
4
|
"description": "A NodeJs common library for solutions based on Cloud Functions",
|
|
5
5
|
"author": "Google Inc.",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -16,22 +16,22 @@
|
|
|
16
16
|
},
|
|
17
17
|
"homepage": "https://github.com/GoogleCloudPlatform/cloud-for-marketing/blob/master/marketing-analytics/activation/common-libs/nodejs-common/README.md",
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"@google-cloud/aiplatform": "^3.
|
|
19
|
+
"@google-cloud/aiplatform": "^3.34.0",
|
|
20
20
|
"@google-cloud/automl": "^4.0.1",
|
|
21
|
-
"@google-cloud/bigquery": "^7.
|
|
22
|
-
"@google-cloud/datastore": "^9.1
|
|
23
|
-
"@google-cloud/firestore": "^7.
|
|
21
|
+
"@google-cloud/bigquery": "^7.9.1",
|
|
22
|
+
"@google-cloud/datastore": "^9.2.1",
|
|
23
|
+
"@google-cloud/firestore": "^7.10.0",
|
|
24
24
|
"@google-cloud/logging-winston": "^6.0.0",
|
|
25
|
-
"@google-cloud/pubsub": "^4.
|
|
26
|
-
"@google-cloud/storage": "^7.
|
|
25
|
+
"@google-cloud/pubsub": "^4.9.0",
|
|
26
|
+
"@google-cloud/storage": "^7.14.0",
|
|
27
27
|
"@google-cloud/scheduler": "^4.3.0",
|
|
28
28
|
"@google-cloud/secret-manager": "^5.6.0",
|
|
29
|
-
"gaxios": "^6.7.
|
|
30
|
-
"google-ads-nodejs-client": "
|
|
31
|
-
"google-auth-library": "^9.
|
|
32
|
-
"googleapis": "^
|
|
33
|
-
"winston": "^3.
|
|
34
|
-
"@grpc/grpc-js": "^1.
|
|
29
|
+
"gaxios": "^6.7.1",
|
|
30
|
+
"google-ads-nodejs-client": "^18.0.1",
|
|
31
|
+
"google-auth-library": "^9.15.0",
|
|
32
|
+
"googleapis": "^144.0.0",
|
|
33
|
+
"winston": "^3.17.0",
|
|
34
|
+
"@grpc/grpc-js": "^1.12.2",
|
|
35
35
|
"lodash": "^4.17.21"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
package/src/apis/auth_client.js
CHANGED
|
@@ -48,23 +48,23 @@ const DEFAULT_ENV_KEYFILE = 'API_SERVICE_ACCOUNT';
|
|
|
48
48
|
*
|
|
49
49
|
* There are two use cases for this authentication helper class:
|
|
50
50
|
* 1. The user only has OAuth access due to some reasons, so ADC can't be used;
|
|
51
|
-
* 2. User-managed service account is
|
|
51
|
+
* 2. User-managed service account is required for external APIs for some
|
|
52
52
|
* specific considerations, e.g. security. In this case, a file based key file
|
|
53
53
|
* can be used to generate a JWT auth client.
|
|
54
54
|
*
|
|
55
55
|
* To solve these challenges, this class tries to probe the settings from
|
|
56
|
-
*
|
|
56
|
+
* environment variables, starts from the name of secret (Secret Manager), OAuth
|
|
57
57
|
* token file (deprecated), then service account key file (deprecated). It will
|
|
58
58
|
* fallback to ADC if those probing failed.
|
|
59
|
-
* Note, Secret Manager is the
|
|
59
|
+
* Note, Secret Manager is the recommended way to store tokens because it is a
|
|
60
60
|
* secure and convenient central storage system to manage access across Google
|
|
61
61
|
* Cloud.
|
|
62
62
|
*
|
|
63
63
|
* The recommended environment variable is:
|
|
64
64
|
* SECRET_NAME: the name of secret. The secret can be a oauth token file or a
|
|
65
65
|
* service account key file. This env var is used to offer a global auth for a
|
|
66
|
-
* solution. If different
|
|
67
|
-
* set
|
|
66
|
+
* solution. If different authentications are required, the value of passed
|
|
67
|
+
* `env` can be set by the runtime.
|
|
68
68
|
*
|
|
69
69
|
* Alternative environment variable but not recommended for prod environment:
|
|
70
70
|
* OAUTH2_TOKEN_JSON : the oauth token key files, refresh token and proper API
|
|
@@ -88,7 +88,7 @@ class AuthClient {
|
|
|
88
88
|
|
|
89
89
|
/**
|
|
90
90
|
* Prepares the `oauthToken` object and/or `serviceAccountKey` based on the
|
|
91
|
-
* settings in
|
|
91
|
+
* settings in environment object.
|
|
92
92
|
* A secret name is preferred to offer the token of the OAuth or key of a
|
|
93
93
|
* service account.
|
|
94
94
|
* To be compatible, this function also checks the env for oauth token file
|
|
@@ -100,10 +100,10 @@ class AuthClient {
|
|
|
100
100
|
return;
|
|
101
101
|
}
|
|
102
102
|
if (this.env[DEFAULT_ENV_SECRET]) {
|
|
103
|
-
const
|
|
103
|
+
const secretManager = new SecretManager({
|
|
104
104
|
projectId: this.env.GCP_PROJECT,
|
|
105
105
|
});
|
|
106
|
-
const secret = await
|
|
106
|
+
const secret = await secretManager.access(this.env[DEFAULT_ENV_SECRET]);
|
|
107
107
|
if (secret) {
|
|
108
108
|
const secretObj = JSON.parse(secret);
|
|
109
109
|
if (secretObj.token) this.oauthToken = secretObj;
|
|
@@ -127,7 +127,7 @@ class AuthClient {
|
|
|
127
127
|
}
|
|
128
128
|
|
|
129
129
|
/**
|
|
130
|
-
* Factory method to offer a prepared AuthClient
|
|
130
|
+
* Factory method to offer a prepared AuthClient instance in an async way.
|
|
131
131
|
* @param {string|!Array<string>|!ReadonlyArray<string>} scopes
|
|
132
132
|
* @param {!Object<string,string>=} env The environment object to hold env
|
|
133
133
|
* variables.
|
|
@@ -163,7 +163,7 @@ class AuthClient {
|
|
|
163
163
|
* steps:
|
|
164
164
|
* 1. Checks environment variable GOOGLE_APPLICATION_CREDENTIALS to get
|
|
165
165
|
* service account. Returns a JWT if it exists;
|
|
166
|
-
* 2. Uses default service account of Computer Engine/
|
|
166
|
+
* 2. Uses default service account of Computer Engine/AppEngine/Cloud
|
|
167
167
|
* Functions
|
|
168
168
|
* 3. Otherwise, an error occurs.
|
|
169
169
|
* @see https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually
|
|
@@ -38,21 +38,26 @@ const END_TAG = '"requestId"';
|
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
40
|
* A stream.Transform that can extract properties and convert naming of the
|
|
41
|
-
*
|
|
41
|
+
* response of Google/Search Ads report from REST interface.
|
|
42
42
|
*/
|
|
43
43
|
class RestSearchStreamTransform extends Transform {
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
46
|
* @constructor
|
|
47
47
|
* @param {boolean=} snakeCase Whether or not output JSON in snake naming.
|
|
48
|
+
* @param {function|undefined} postProcessFn An optional function to process
|
|
49
|
+
* the data after the default process. The default process includes
|
|
50
|
+
* filtering out fields based on `fieldMask` and adjusting the naming
|
|
51
|
+
* convention.
|
|
48
52
|
*/
|
|
49
|
-
constructor(snakeCase = false) {
|
|
53
|
+
constructor(snakeCase = false, postProcessFn) {
|
|
50
54
|
super({ objectMode: true });
|
|
51
55
|
this.snakeCase = snakeCase;
|
|
52
56
|
this.chunks = [Buffer.from('')];
|
|
53
57
|
this.processFn; // The function to process a row of the report.
|
|
54
58
|
this.logger = getLogger('ADS.STREAM.T');
|
|
55
59
|
this.stopwatch = Date.now();
|
|
60
|
+
this.postProcessFn = postProcessFn;
|
|
56
61
|
}
|
|
57
62
|
|
|
58
63
|
_transform(chunk, encoding, callback) {
|
|
@@ -68,7 +73,14 @@ class RestSearchStreamTransform extends Transform {
|
|
|
68
73
|
.substring(maskIndex + FIELD_MASK_TAG.length, rawString.indexOf(END_TAG))
|
|
69
74
|
.split('"')[1];
|
|
70
75
|
this.logger.debug(`Got fieldMask: ${fieldMask}`);
|
|
71
|
-
|
|
76
|
+
const processFn = getFilterAndStringifyFn(fieldMask, this.snakeCase);
|
|
77
|
+
if (this.postProcessFn) {
|
|
78
|
+
this.processFn = (obj) => {
|
|
79
|
+
return this.postProcessFn(processFn(obj));
|
|
80
|
+
}
|
|
81
|
+
} else {
|
|
82
|
+
this.processFn = processFn;
|
|
83
|
+
}
|
|
72
84
|
}
|
|
73
85
|
const resultsWithTailing = rawString.substring(startIndex, maskIndex);
|
|
74
86
|
const results = resultsWithTailing.substring(
|
|
@@ -37,17 +37,30 @@ class GoogleApiClient extends AuthRestfulApi {
|
|
|
37
37
|
*/
|
|
38
38
|
getVersion() { }
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Gets the default options to initialize an Api object in Google Api client
|
|
42
|
+
* library. It contains the Api version and auth information.
|
|
43
|
+
* @param {object|undefined} initOptions
|
|
44
|
+
* @return {object}
|
|
45
|
+
*/
|
|
46
|
+
async getApiClientInitOptions(initOptions) {
|
|
47
|
+
return {
|
|
48
|
+
version: this.getVersion(),
|
|
49
|
+
auth: await this.getAuth(),
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
40
53
|
/**
|
|
41
54
|
* Returns the Api instance.
|
|
55
|
+
* This function expects an instance name for the object in Google Api client
|
|
56
|
+
* library, e.g. searchads360 for 'Search Ads 360'.
|
|
42
57
|
* @return {!Promise<object>} The Api instance.
|
|
43
58
|
*/
|
|
44
|
-
async getApiClient() {
|
|
59
|
+
async getApiClient(initOptions = {}) {
|
|
45
60
|
if (this.apiClient) return this.apiClient;
|
|
46
61
|
this.logger.info(`Initialized ${this.constructor.name} instance.`);
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
auth: await this.getAuth(),
|
|
50
|
-
});
|
|
62
|
+
const options = await this.getApiClientInitOptions(initOptions);
|
|
63
|
+
this.apiClient = google[this.googleApi](options);
|
|
51
64
|
return this.apiClient;
|
|
52
65
|
}
|
|
53
66
|
}
|
|
@@ -19,7 +19,6 @@
|
|
|
19
19
|
|
|
20
20
|
'use strict';
|
|
21
21
|
|
|
22
|
-
const {request} = require('gaxios');
|
|
23
22
|
const { GoogleApiClient } = require('./base/google_api_client.js');
|
|
24
23
|
const {
|
|
25
24
|
getLogger,
|
|
@@ -66,47 +65,6 @@ let InsertConversionsConfig;
|
|
|
66
65
|
*/
|
|
67
66
|
let AvailabilityConfig;
|
|
68
67
|
|
|
69
|
-
/**
|
|
70
|
-
* Nodejs Google API client library doesn't export this type, so here is a
|
|
71
|
-
* partial typedef of Report Request which only contains essential properties.
|
|
72
|
-
* For complete definition, see:
|
|
73
|
-
* https://developers.google.com/search-ads/v2/reference/reports/request
|
|
74
|
-
* @typedef {{
|
|
75
|
-
* reportScope: {
|
|
76
|
-
* agencyId: string,
|
|
77
|
-
* advertiserId: string,
|
|
78
|
-
* },
|
|
79
|
-
* reportType: string,
|
|
80
|
-
* columns: Array<{
|
|
81
|
-
* columnName: string,
|
|
82
|
-
* headerText: string,
|
|
83
|
-
* startDate: string,
|
|
84
|
-
* endDate: string
|
|
85
|
-
* }>,
|
|
86
|
-
* filters: Array<{
|
|
87
|
-
* columnName: string,
|
|
88
|
-
* headerText: string,
|
|
89
|
-
* startDate: string,
|
|
90
|
-
* endDate: string
|
|
91
|
-
* }>|undefined,
|
|
92
|
-
* timeRange: {
|
|
93
|
-
* startDate: string,
|
|
94
|
-
* endDate: string,
|
|
95
|
-
* changedMetricsSinceTimestamp: datetime|undefined,
|
|
96
|
-
* changedAttributesSinceTimestamp: datetime|undefined,
|
|
97
|
-
* },
|
|
98
|
-
* downloadFormat: 'CSV'|'TSV',
|
|
99
|
-
* statisticsCurrency: 'usd'|'agency'|'advertiser'|'account',
|
|
100
|
-
* maxRowsPerFile: integer,
|
|
101
|
-
* includeDeletedEntities: boolean|undefined,
|
|
102
|
-
* includeRemovedEntities: boolean|undefined,
|
|
103
|
-
* verifySingleTimeZone: boolean|undefined,
|
|
104
|
-
* startRow: integer|undefined,
|
|
105
|
-
* rowCount: integer|undefined,
|
|
106
|
-
* }}
|
|
107
|
-
*/
|
|
108
|
-
let ReportRequest;
|
|
109
|
-
|
|
110
68
|
/**
|
|
111
69
|
* DoubleClick Search (DS) Ads 360 API v2 stub.
|
|
112
70
|
* See: https://developers.google.com/search-ads/v2/reference/
|
|
@@ -293,102 +251,12 @@ class DoubleClickSearch extends GoogleApiClient {
|
|
|
293
251
|
});
|
|
294
252
|
batchResult.errors = Array.from(errors);
|
|
295
253
|
}
|
|
296
|
-
|
|
297
|
-
/**
|
|
298
|
-
* There are three steps to get asynchronous reports in SA360:
|
|
299
|
-
* 1. Call Reports.request() to specify the type of data for the report.
|
|
300
|
-
* 2. Call Reports.get() with the report id to check whether it's ready.
|
|
301
|
-
* 3. Call Reports.getFile() to the download the report files.
|
|
302
|
-
* @see https://developers.google.com/search-ads/v2/how-tos/reporting/asynchronous-requests
|
|
303
|
-
*
|
|
304
|
-
* This is the first step.
|
|
305
|
-
* @param {!ReportRequest} requestBody
|
|
306
|
-
* @return {!Promise<string>}
|
|
307
|
-
*/
|
|
308
|
-
async requestReports(requestBody) {
|
|
309
|
-
const doubleclicksearch = await this.getApiClient();
|
|
310
|
-
const { status, data } = await doubleclicksearch.reports.request({ requestBody });
|
|
311
|
-
if (status >= 200 && status < 300) {
|
|
312
|
-
return data.id;
|
|
313
|
-
}
|
|
314
|
-
const errorMsg = `Fail to request reports: ${JSON.stringify(requestBody)}`;
|
|
315
|
-
this.logger.error(errorMsg, data);
|
|
316
|
-
throw new Error(errorMsg);
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
/**
|
|
320
|
-
* Returns the report file links if the report is ready or undefined if not.
|
|
321
|
-
* @param {string} reportId
|
|
322
|
-
* @return {!Promise<undefined|Array<{
|
|
323
|
-
* url:string,
|
|
324
|
-
* byteCount:string,
|
|
325
|
-
* }>>}
|
|
326
|
-
*/
|
|
327
|
-
async getReportUrls(reportId) {
|
|
328
|
-
const doubleclicksearch = await this.getApiClient();
|
|
329
|
-
const { status, data } = await doubleclicksearch.reports.get({ reportId });
|
|
330
|
-
switch (status) {
|
|
331
|
-
case 200:
|
|
332
|
-
const {rowCount, files} = data;
|
|
333
|
-
this.logger.info(
|
|
334
|
-
`Report[${reportId}] has ${rowCount} rows and ${files.length} files.`);
|
|
335
|
-
return files;
|
|
336
|
-
case 202:
|
|
337
|
-
this.logger.info(`Report[${reportId}] is not ready.`);
|
|
338
|
-
break;
|
|
339
|
-
default:
|
|
340
|
-
const errorMsg =
|
|
341
|
-
`Error in get reports: ${reportId} with status code: ${status}`;
|
|
342
|
-
this.logger.error(errorMsg, data);
|
|
343
|
-
throw new Error(errorMsg);
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
/**
|
|
348
|
-
* Get the given part of a report.
|
|
349
|
-
* @param {string} reportId
|
|
350
|
-
* @param {number} reportFragment The index (based 0) of report files.
|
|
351
|
-
* @return {!Promise<string>}
|
|
352
|
-
*/
|
|
353
|
-
async getReportFile(reportId, reportFragment) {
|
|
354
|
-
const doubleclicksearch = await this.getApiClient();
|
|
355
|
-
const response = await doubleclicksearch.reports.getFile(
|
|
356
|
-
{reportId, reportFragment});
|
|
357
|
-
if (response.status === 200) {
|
|
358
|
-
return response.data;
|
|
359
|
-
}
|
|
360
|
-
const errorMsg =
|
|
361
|
-
`Error in get file from reports: ${reportFragment}@${reportId}`;
|
|
362
|
-
this.logger.error(errorMsg, response);
|
|
363
|
-
throw new Error(errorMsg);
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
/**
|
|
367
|
-
* Returns a readable stream of the report.
|
|
368
|
-
* In case of the report is large, use stream directly to write out to fit in
|
|
369
|
-
* the resource limited environment, e.g. Cloud Functions.
|
|
370
|
-
* @param {string} url
|
|
371
|
-
* @return {!Promise<ReadableStream>}
|
|
372
|
-
*/
|
|
373
|
-
async getReportFileStream(url) {
|
|
374
|
-
const auth = await this.getAuth();
|
|
375
|
-
const headers = await auth.getRequestHeaders();
|
|
376
|
-
const response = await request({
|
|
377
|
-
method: 'GET',
|
|
378
|
-
headers,
|
|
379
|
-
url,
|
|
380
|
-
responseType: 'stream',
|
|
381
|
-
});
|
|
382
|
-
return response.data;
|
|
383
|
-
}
|
|
384
|
-
|
|
385
254
|
}
|
|
386
255
|
|
|
387
256
|
module.exports = {
|
|
388
257
|
DoubleClickSearch,
|
|
389
258
|
InsertConversionsConfig,
|
|
390
259
|
AvailabilityConfig,
|
|
391
|
-
ReportRequest,
|
|
392
260
|
API_VERSION,
|
|
393
261
|
API_SCOPES,
|
|
394
262
|
};
|
|
@@ -211,7 +211,7 @@ const MAX_IDENTIFIERS = {
|
|
|
211
211
|
};
|
|
212
212
|
|
|
213
213
|
/**
|
|
214
|
-
* Configuration for uploading click conversions, call
|
|
214
|
+
* Configuration for uploading click conversions, call conversions or conversion
|
|
215
215
|
* adjustments for Google Ads, includes:
|
|
216
216
|
* gclid, conversionAction, conversionDateTime, conversionValue,
|
|
217
217
|
* currencyCode, orderId, externalAttributionData,
|
|
@@ -598,7 +598,7 @@ class GoogleAdsApi {
|
|
|
598
598
|
* e.g. wrong conversion Id, wrong gclid, etc.
|
|
599
599
|
* `Status` has a property `message` that has the general information of the
|
|
600
600
|
* error, however, the detailed information (e.g. the failed conversions and
|
|
601
|
-
* their reasons) lies in the property named `
|
|
601
|
+
* their reasons) lies in the property named `details` (an array of
|
|
602
602
|
* `GoogleAdsFailure`). Each `GoogleAdsFailure` is related to a failed
|
|
603
603
|
* conversion. The function `extraFailedLines_` is used to extract the
|
|
604
604
|
* details.
|
|
@@ -612,8 +612,8 @@ class GoogleAdsApi {
|
|
|
612
612
|
* @param {string} customerId
|
|
613
613
|
* @param {string} loginCustomerId Login customer account ID (Mcc Account id).
|
|
614
614
|
* @param {!ConversionConfig} conversionConfig Default conversion parameters.
|
|
615
|
-
* @param {string} functionName The name of sending
|
|
616
|
-
* be `uploadClickConversions`, `uploadCallConversions` or
|
|
615
|
+
* @param {string} functionName The name of sending conversions function,
|
|
616
|
+
* could be `uploadClickConversions`, `uploadCallConversions` or
|
|
617
617
|
* `uploadConversionAdjustments`.
|
|
618
618
|
* @param {string} propertyForDebug The name of property for debug info.
|
|
619
619
|
* @return {!SendSingleBatch} Function which can send a batch of hits to
|
|
@@ -945,8 +945,8 @@ class GoogleAdsApi {
|
|
|
945
945
|
AND user_list.membership_status = OPEN
|
|
946
946
|
AND user_list.crm_based_user_list.upload_key_type = ${uploadKeyType}
|
|
947
947
|
`;
|
|
948
|
-
const
|
|
949
|
-
return
|
|
948
|
+
const userLists = await this.getReport(customerId, loginCustomerId, query);
|
|
949
|
+
return userLists.length === 0 ? undefined : userLists[0].userList.id;
|
|
950
950
|
}
|
|
951
951
|
|
|
952
952
|
/**
|
|
@@ -954,8 +954,8 @@ class GoogleAdsApi {
|
|
|
954
954
|
* Id. The user list would be a CRM_BASED type.
|
|
955
955
|
* Trying to create a list with an used name will fail.
|
|
956
956
|
* The Google Ads service behind this function (UserListService) supports
|
|
957
|
-
* `partial_failure`. Here, we only create one
|
|
958
|
-
* `partial_failure` is disabled to simplified the error
|
|
957
|
+
* `partial_failure`. Here, we only create one user list at one time, so
|
|
958
|
+
* `partial_failure` is disabled to simplified the error handling process.
|
|
959
959
|
* @see getUploadConversionFnBase_ for more details of error handling.
|
|
960
960
|
* @param {!CustomerMatchConfig} customerMatchConfig
|
|
961
961
|
* @return {number} The created user list id. Note this is not the resource
|
|
@@ -973,7 +973,7 @@ class GoogleAdsApi {
|
|
|
973
973
|
customerId: getCleanCid(customerId),
|
|
974
974
|
operations: [{ create: userList }],
|
|
975
975
|
validateOnly: this.debugMode, // when true makes no changes
|
|
976
|
-
partialFailure: false, // Simplify error handling in creating
|
|
976
|
+
partialFailure: false, // Simplify error handling in creating user list
|
|
977
977
|
});
|
|
978
978
|
const client = await this.getUserListServiceClient_();
|
|
979
979
|
const options = this.getCallOptions_(loginCustomerId);
|
|
@@ -984,7 +984,7 @@ class GoogleAdsApi {
|
|
|
984
984
|
*/
|
|
985
985
|
const [response] = await client.mutateUserLists(request, options);
|
|
986
986
|
const { results } = response; // No `partialFailureError` here.
|
|
987
|
-
this.logger.debug(`Created crm
|
|
987
|
+
this.logger.debug(`Created crm user list from`, customerMatchConfig);
|
|
988
988
|
if (!results[0]) {
|
|
989
989
|
if (this.debugMode) {
|
|
990
990
|
throw new Error('No UserList was created in DEBUG mode.');
|
|
@@ -1139,7 +1139,7 @@ class GoogleAdsApi {
|
|
|
1139
1139
|
/**
|
|
1140
1140
|
* Creates a OfflineUserDataJob and returns resource name.
|
|
1141
1141
|
* @param {OfflineUserDataJobConfig} offlineUserDataJobConfig
|
|
1142
|
-
* @return {string} The
|
|
1142
|
+
* @return {string} The resource name of the created job.
|
|
1143
1143
|
*/
|
|
1144
1144
|
async createOfflineUserDataJob(offlineUserDataJobConfig) {
|
|
1145
1145
|
const config = this.getCamelConfig_(offlineUserDataJobConfig);
|
|
@@ -1231,7 +1231,7 @@ class GoogleAdsApi {
|
|
|
1231
1231
|
this.logger.debug(`Add operation to job batch[${batchId}]`, response);
|
|
1232
1232
|
const { partialFailureError: failed } = response;
|
|
1233
1233
|
if (failed) {
|
|
1234
|
-
this.logger.info(`Job[${jobResourceName}]
|
|
1234
|
+
this.logger.info(`Job[${jobResourceName}] fail:`, failed.message);
|
|
1235
1235
|
const failures = failed.details.map(
|
|
1236
1236
|
({ value }) => GoogleAdsFailure.decode(value));
|
|
1237
1237
|
this.extraFailedLines_(batchResult, failures, lines, 0);
|
|
@@ -1331,7 +1331,7 @@ class GoogleAdsApi {
|
|
|
1331
1331
|
|
|
1332
1332
|
/**
|
|
1333
1333
|
* Returns a HTTP header object contains the authentication information for
|
|
1334
|
-
* Google Ads API, include: `developer-token` and `
|
|
1334
|
+
* Google Ads API, include: `developer-token` and `login-customer-id`.
|
|
1335
1335
|
* @param {string} loginCustomerId
|
|
1336
1336
|
* @return {object} The HTTP header object.
|
|
1337
1337
|
* @private
|
|
@@ -1360,7 +1360,7 @@ class GoogleAdsApi {
|
|
|
1360
1360
|
}
|
|
1361
1361
|
|
|
1362
1362
|
/**
|
|
1363
|
-
* Prepares the
|
|
1363
|
+
* Prepares the fetch data service client instance.
|
|
1364
1364
|
* @return {!GoogleAdsServiceClient}
|
|
1365
1365
|
* @private
|
|
1366
1366
|
*/
|
|
@@ -1447,8 +1447,8 @@ class GoogleAdsApi {
|
|
|
1447
1447
|
}
|
|
1448
1448
|
|
|
1449
1449
|
/**
|
|
1450
|
-
* Returns an
|
|
1451
|
-
* @param {Object} record An object contains user
|
|
1450
|
+
* Returns an array of UserIdentifier object based the given JSON object.
|
|
1451
|
+
* @param {Object} record An object contains user identifier information.
|
|
1452
1452
|
* @param {!Array<string>} identifierTypes An list of user identifier types that
|
|
1453
1453
|
* are supported by the target service.
|
|
1454
1454
|
* @param {number} maximumNumOfIdentifiers The maximum number of user
|
|
@@ -1541,7 +1541,7 @@ const buildConversionJsonList = (lines, config, conversionFields,
|
|
|
1541
1541
|
})
|
|
1542
1542
|
}
|
|
1543
1543
|
/**
|
|
1544
|
-
* Returns an
|
|
1544
|
+
* Returns an array of UserData object based on the given arran of JSON strings.
|
|
1545
1545
|
* @param {!Array<string>} lines An array of JSON strings of UserData.
|
|
1546
1546
|
* @param {{
|
|
1547
1547
|
* additionalAttributes: (!UserIdentifierSource|undefined)
|
package/src/apis/search_ads.js
CHANGED
|
@@ -21,7 +21,8 @@
|
|
|
21
21
|
const { request: gaxiosRequest } = require('gaxios');
|
|
22
22
|
const { google } = require('googleapis');
|
|
23
23
|
const { GoogleApiClient } = require('./base/google_api_client.js');
|
|
24
|
-
const { getLogger }
|
|
24
|
+
const { changeStringToBigQuerySafe, getLogger }
|
|
25
|
+
= require('../components/utils.js');
|
|
25
26
|
const { getCleanCid, RestSearchStreamTransform }
|
|
26
27
|
= require('./base/ads_api_common.js');
|
|
27
28
|
|
|
@@ -59,7 +60,7 @@ class SearchAds extends GoogleApiClient {
|
|
|
59
60
|
}
|
|
60
61
|
|
|
61
62
|
/**
|
|
62
|
-
*
|
|
63
|
+
* Returns the options to initialize Search Ads 360 Reporting API instance.
|
|
63
64
|
* OAuth 2.0 application credentials is required for calling this API.
|
|
64
65
|
* For Search Ads Reporting API calls made by a manager to a client account,
|
|
65
66
|
* a HTTP header named `login-customer-id` is required in the request. This
|
|
@@ -67,19 +68,17 @@ class SearchAds extends GoogleApiClient {
|
|
|
67
68
|
* API call. Be sure to remove any hyphens (—), for example: 1234567890, not
|
|
68
69
|
* 123-456-7890.
|
|
69
70
|
* @see https://developers.google.com/search-ads/reporting/api/reference/rest/auth
|
|
70
|
-
* @return {!
|
|
71
|
-
* @
|
|
71
|
+
* @return {!Object}
|
|
72
|
+
* @override
|
|
72
73
|
*/
|
|
73
|
-
async
|
|
74
|
-
|
|
75
|
-
const
|
|
76
|
-
version: this.getVersion(),
|
|
77
|
-
auth: await this.getAuth(),
|
|
78
|
-
};
|
|
74
|
+
async getApiClientInitOptions(initOptions) {
|
|
75
|
+
const options = await super.getApiClientInitOptions(initOptions);
|
|
76
|
+
const { loginCustomerId } = initOptions;
|
|
79
77
|
if (loginCustomerId) {
|
|
78
|
+
this.logger.debug(`initialized with loginCustomerId: ${loginCustomerId}`);
|
|
80
79
|
options.headers = { 'login-customer-id': getCleanCid(loginCustomerId) };
|
|
81
80
|
}
|
|
82
|
-
return
|
|
81
|
+
return options;
|
|
83
82
|
}
|
|
84
83
|
|
|
85
84
|
/**
|
|
@@ -96,7 +95,7 @@ class SearchAds extends GoogleApiClient {
|
|
|
96
95
|
* @see https://developers.google.com/search-ads/reporting/api/reference/rpc/google.ads.searchads360.v0.services#searchsearchads360response
|
|
97
96
|
*/
|
|
98
97
|
async getPaginatedReport(customerId, loginCustomerId, query, options = {}) {
|
|
99
|
-
const searchads = await this.getApiClient(loginCustomerId);
|
|
98
|
+
const searchads = await this.getApiClient({ loginCustomerId });
|
|
100
99
|
const requestBody = Object.assign({
|
|
101
100
|
query,
|
|
102
101
|
pageSize: 10000,
|
|
@@ -146,17 +145,83 @@ class SearchAds extends GoogleApiClient {
|
|
|
146
145
|
* @param {string} loginCustomerId Login customer account ID (Mcc Account id).
|
|
147
146
|
* @param {string} query A Google Ads Query string.
|
|
148
147
|
* @param {boolean=} snakeCase Output JSON objects in snake_case.
|
|
148
|
+
* @param {boolean=} rawCustomColumns Keeps the raw custom column values as
|
|
149
|
+
* an array.
|
|
149
150
|
* @return {!Promise<stream>}
|
|
150
151
|
*/
|
|
151
152
|
async cleanedRestStreamReport(customerId, loginCustomerId, query,
|
|
152
|
-
snakeCase = false) {
|
|
153
|
-
|
|
153
|
+
snakeCase = false, rawCustomColumns = false) {
|
|
154
|
+
let postProcessFn;
|
|
155
|
+
if (!rawCustomColumns && query.indexOf('custom_columns.id[') > -1) {
|
|
156
|
+
const convertor = await this.getCustomColumnsConvertor(
|
|
157
|
+
customerId, loginCustomerId, query);
|
|
158
|
+
postProcessFn = (str) => {
|
|
159
|
+
const source = JSON.parse(str);
|
|
160
|
+
source.customColumns = convertor(source.customColumns);
|
|
161
|
+
return JSON.stringify(source);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const transform = new RestSearchStreamTransform(snakeCase, postProcessFn);
|
|
154
165
|
const stream =
|
|
155
166
|
await this.restStreamReport(customerId, loginCustomerId, query);
|
|
156
167
|
return stream.on('error', (error) => transform.emit('error', error))
|
|
157
168
|
.pipe(transform);
|
|
158
169
|
}
|
|
159
170
|
|
|
171
|
+
/**
|
|
172
|
+
* Gets all custom columns information from the report query. By default, it
|
|
173
|
+
* will load columns for the manager account. If there are some columns
|
|
174
|
+
* missed, it will load columns for the customerId.
|
|
175
|
+
* @param {string} customerId - The ID of the customer.
|
|
176
|
+
* @param {string} loginCustomerId - The ID of the manager.
|
|
177
|
+
* @param {string} query - The report query.
|
|
178
|
+
* @return {!Array<object>}
|
|
179
|
+
*/
|
|
180
|
+
async getCustomColumnsFromQuery(customerId, loginCustomerId, query) {
|
|
181
|
+
const pattern = /custom_columns\.id\[(\d+)\]/g;
|
|
182
|
+
const columnIds = Array.from(query.matchAll(pattern), m => m[1]);
|
|
183
|
+
const selectedColumns = {};
|
|
184
|
+
const getSelectedColumn = (column) => {
|
|
185
|
+
if (columnIds.indexOf(column.id) > -1) {
|
|
186
|
+
selectedColumns[column.id] = column;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
(await this.listCustomColumns(loginCustomerId, loginCustomerId))
|
|
190
|
+
.forEach(getSelectedColumn);
|
|
191
|
+
if (Object.keys(selectedColumns).length < columnIds.length) {
|
|
192
|
+
this.logger.warn('Missing custom columns in MCC, try CID now');
|
|
193
|
+
(await this.listCustomColumns(customerId, loginCustomerId))
|
|
194
|
+
.forEach(getSelectedColumn);
|
|
195
|
+
}
|
|
196
|
+
return columnIds.map((columnId) => selectedColumns[columnId]);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Returns the function to convert the array values of custom columns to
|
|
201
|
+
* an object with the column names as the property keys. The column names will
|
|
202
|
+
* be proceeded to be BigQuery naming safe.
|
|
203
|
+
* @param {string} customerId - The ID of the customer.
|
|
204
|
+
* @param {string} loginCustomerId - The ID of the manager.
|
|
205
|
+
* @param {string} query - The report query.
|
|
206
|
+
* @return {function}
|
|
207
|
+
*/
|
|
208
|
+
async getCustomColumnsConvertor(customerId, loginCustomerId, query) {
|
|
209
|
+
const customerColumns = await this.getCustomColumnsFromQuery(
|
|
210
|
+
customerId, loginCustomerId, query);
|
|
211
|
+
const columnConvertors = customerColumns.map(({ name, valueType }) => {
|
|
212
|
+
return (obj) => {
|
|
213
|
+
const key = changeStringToBigQuerySafe(name);
|
|
214
|
+
const value = obj[`${valueType.toLowerCase()}Value`];
|
|
215
|
+
return { [key]: value };
|
|
216
|
+
};
|
|
217
|
+
});
|
|
218
|
+
const convertor = (customerColumns) => {
|
|
219
|
+
return Object.assign(
|
|
220
|
+
...customerColumns.map((obj, index) => columnConvertors[index](obj)));
|
|
221
|
+
};
|
|
222
|
+
return convertor;
|
|
223
|
+
}
|
|
224
|
+
|
|
160
225
|
/**
|
|
161
226
|
* Returns the requested field or resource (artifact) used by SearchAds360Service.
|
|
162
227
|
* This service doesn't require `login-customer-id` HTTP header.
|
|
@@ -203,9 +268,9 @@ class SearchAds extends GoogleApiClient {
|
|
|
203
268
|
* @see https://developers.google.com/search-ads/reporting/api/reference/rest/v0/customers.customColumns#CustomColumn
|
|
204
269
|
*/
|
|
205
270
|
async listCustomColumns(customerId, loginCustomerId) {
|
|
206
|
-
const searchads = await this.getApiClient(loginCustomerId);
|
|
271
|
+
const searchads = await this.getApiClient({ loginCustomerId });
|
|
207
272
|
const response = await searchads.customers.customColumns.list({ customerId });
|
|
208
|
-
return response.data.customColumns;
|
|
273
|
+
return response.data.customColumns || [];
|
|
209
274
|
}
|
|
210
275
|
|
|
211
276
|
/**
|
|
@@ -219,7 +284,7 @@ class SearchAds extends GoogleApiClient {
|
|
|
219
284
|
*/
|
|
220
285
|
async getCustomColumn(columnId, customerId, loginCustomerId) {
|
|
221
286
|
const resourceName = `customers/${customerId}/customColumns/${columnId}`;
|
|
222
|
-
const searchads = await this.getApiClient(loginCustomerId);
|
|
287
|
+
const searchads = await this.getApiClient({ loginCustomerId });
|
|
223
288
|
const response = await searchads.customers.customColumns.get({ resourceName });
|
|
224
289
|
return response.data;
|
|
225
290
|
}
|
package/src/apis/spreadsheets.js
CHANGED
|
@@ -67,15 +67,19 @@ let DimensionRange;
|
|
|
67
67
|
class Spreadsheets extends GoogleApiClient {
|
|
68
68
|
/**
|
|
69
69
|
* Init Spreadsheets API client.
|
|
70
|
-
* @param {string}
|
|
70
|
+
* @param {string} spreadsheetIdOrUrl
|
|
71
71
|
* @param {!Object<string,string>=} env The environment object to hold env
|
|
72
72
|
* variables.
|
|
73
73
|
*/
|
|
74
|
-
constructor(
|
|
74
|
+
constructor(spreadsheetIdOrUrl, env = process.env) {
|
|
75
75
|
super(env);
|
|
76
76
|
this.googleApi = 'sheets';
|
|
77
77
|
/** @const {string} */
|
|
78
|
-
|
|
78
|
+
if (spreadsheetIdOrUrl.startsWith('https://')) {
|
|
79
|
+
this.spreadsheetId = /spreadsheets\/d\/([^/]*)/.exec(spreadsheetIdOrUrl)[1];
|
|
80
|
+
} else {
|
|
81
|
+
this.spreadsheetId = spreadsheetIdOrUrl;
|
|
82
|
+
}
|
|
79
83
|
this.logger = getLogger('API.GS');
|
|
80
84
|
}
|
|
81
85
|
|
|
@@ -109,7 +113,7 @@ class Spreadsheets extends GoogleApiClient {
|
|
|
109
113
|
}
|
|
110
114
|
|
|
111
115
|
/**
|
|
112
|
-
* Returns whether the sheet with
|
|
116
|
+
* Returns whether the sheet with specified name exists.
|
|
113
117
|
* @param {string} sheetName
|
|
114
118
|
* @return {boolean}
|
|
115
119
|
*/
|
|
@@ -122,7 +126,7 @@ class Spreadsheets extends GoogleApiClient {
|
|
|
122
126
|
}
|
|
123
127
|
|
|
124
128
|
/**
|
|
125
|
-
* Creates a sheet with the
|
|
129
|
+
* Creates a sheet with the given name.
|
|
126
130
|
*
|
|
127
131
|
* @param {string} sheetName
|
|
128
132
|
*/
|
|
@@ -137,7 +141,7 @@ class Spreadsheets extends GoogleApiClient {
|
|
|
137
141
|
}
|
|
138
142
|
|
|
139
143
|
/**
|
|
140
|
-
* Deletes a sheet with the
|
|
144
|
+
* Deletes a sheet with the given name.
|
|
141
145
|
*
|
|
142
146
|
* @param {string} sheetName
|
|
143
147
|
*/
|
|
@@ -289,6 +293,24 @@ class Spreadsheets extends GoogleApiClient {
|
|
|
289
293
|
}
|
|
290
294
|
return batchResult;
|
|
291
295
|
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Gets values of first row of the specified sheet.
|
|
299
|
+
* @param {string} sheetName
|
|
300
|
+
* @param {number} skipLeadingRows
|
|
301
|
+
* @return {!Array<string>}
|
|
302
|
+
*/
|
|
303
|
+
async getHeadline(sheetName, skipLeadingRows = 1) {
|
|
304
|
+
const request = {
|
|
305
|
+
spreadsheetId: this.spreadsheetId,
|
|
306
|
+
range: `'${sheetName}'!${skipLeadingRows}:${skipLeadingRows}`,
|
|
307
|
+
};
|
|
308
|
+
const sheets = await this.getApiClient();
|
|
309
|
+
const response = await sheets.spreadsheets.values.get(request);
|
|
310
|
+
const data = response.data;
|
|
311
|
+
this.logger.debug(`Get: `, data);
|
|
312
|
+
return data.values[0];
|
|
313
|
+
}
|
|
292
314
|
}
|
|
293
315
|
|
|
294
316
|
module.exports = {
|
|
@@ -65,7 +65,7 @@ let Filter;
|
|
|
65
65
|
*/
|
|
66
66
|
class DatastoreDocumentFacade {
|
|
67
67
|
/**
|
|
68
|
-
* Initializes DocumentFacade for
|
|
68
|
+
* Initializes DocumentFacade for Datastore.
|
|
69
69
|
* @param {!DatastoreModeEntity} entity
|
|
70
70
|
*/
|
|
71
71
|
constructor(entity) {
|
|
@@ -87,7 +87,7 @@ class DatastoreDocumentFacade {
|
|
|
87
87
|
*/
|
|
88
88
|
class DatastoreTransactionFacade {
|
|
89
89
|
/**
|
|
90
|
-
* Initializes
|
|
90
|
+
* Initializes Firestore TransactionFacade for Datastore Transaction.
|
|
91
91
|
* @param {!DatastoreModeTransaction} transaction
|
|
92
92
|
* @param {!DatastoreModeEntity} entity
|
|
93
93
|
*/
|
|
@@ -145,7 +145,7 @@ let Database;
|
|
|
145
145
|
* interface offers unified operations on the data objects in both of these two
|
|
146
146
|
* modes.
|
|
147
147
|
*
|
|
148
|
-
* Firestore Native mode ('Firestore') and Firestore Datastore mode ('
|
|
148
|
+
* Firestore Native mode ('Firestore') and Firestore Datastore mode ('Datastore')
|
|
149
149
|
* have different interfaces:
|
|
150
150
|
* 1. 'Firestore' has two kinds objects: 'document' stands for an object (data
|
|
151
151
|
* entity) and 'collection' stands for a group of 'documents'. The
|
|
@@ -34,7 +34,7 @@ const DatastoreModeAccess = require('./datastore_mode_access.js');
|
|
|
34
34
|
/**
|
|
35
35
|
* This is data access object base class on Firestore. It seals the details of
|
|
36
36
|
* different underlying databases, Firestore and Datastore.
|
|
37
|
-
* This class relies on an initial parameter in
|
|
37
|
+
* This class relies on an initial parameter in constructor to indicate the
|
|
38
38
|
* Firestore type.
|
|
39
39
|
*
|
|
40
40
|
* Firestore and Datastore have different transaction APIs. In that case,
|
|
@@ -33,7 +33,7 @@ const {
|
|
|
33
33
|
} = require('./access_base.js');
|
|
34
34
|
const {getLogger, wait} = require('../utils.js');
|
|
35
35
|
|
|
36
|
-
/** @const {number} Max retry times when commit failed in a
|
|
36
|
+
/** @const {number} Max retry times when commit failed in a transaction. */
|
|
37
37
|
const MAX_RETRY_TIMES = 5;
|
|
38
38
|
|
|
39
39
|
/**
|
|
@@ -66,7 +66,7 @@ class DatastoreModeAccess {
|
|
|
66
66
|
* Datastore uses 'Key' to identify entities. A 'Key' composes of Id, entity
|
|
67
67
|
* kind and namespace. The 'id' can be 'undefined' if the next operation is
|
|
68
68
|
* creating a new entity.
|
|
69
|
-
* The default Id of Datastore is
|
|
69
|
+
* The default Id of Datastore is an integer. However, Pub/sub can only send
|
|
70
70
|
* attributes with string values. This will cause the Datastore Ids to be
|
|
71
71
|
* converted to strings. So here will try to change the id back to number if
|
|
72
72
|
* possible.
|
|
@@ -106,7 +106,7 @@ class DatastoreModeAccess {
|
|
|
106
106
|
async waitUntilGetObject(id) {
|
|
107
107
|
const entity = await this.getObject(id);
|
|
108
108
|
if (entity) return id;
|
|
109
|
-
this.logger.debug(`Wait 1 more second until the
|
|
109
|
+
this.logger.debug(`Wait 1 more second until the entity@${id} is ready`);
|
|
110
110
|
return wait(1000, this.waitUntilGetObject(id));
|
|
111
111
|
}
|
|
112
112
|
|
package/src/components/pubsub.js
CHANGED
|
@@ -129,7 +129,7 @@ class EnhancedPubSub {
|
|
|
129
129
|
|
|
130
130
|
/**
|
|
131
131
|
* Using `SubscriberClient` to acknowledge messages.
|
|
132
|
-
* 2022.11.02 The
|
|
132
|
+
* 2022.11.02 The method `ack()` in Message doesn't work properly due to a
|
|
133
133
|
* unknown reason. Use this function to acknowledge a message for now.
|
|
134
134
|
*
|
|
135
135
|
* @param {string} subscription Subscription name.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// limitations under the License.
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
|
-
* @fileoverview Cloud Scheduler wrapper class, including: pause/
|
|
16
|
+
* @fileoverview Cloud Scheduler wrapper class, including: pause/resume a job.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
'use strict';
|
|
@@ -137,9 +137,9 @@ class CloudScheduler {
|
|
|
137
137
|
* @private
|
|
138
138
|
*/
|
|
139
139
|
async getJobs_(name, targetLocations = undefined) {
|
|
140
|
-
const
|
|
140
|
+
const jobName = `/jobs/${name}`;
|
|
141
141
|
const allJobs = await this.listJobs_(targetLocations);
|
|
142
|
-
const jobs = allJobs.filter((job) =>
|
|
142
|
+
const jobs = allJobs.filter((job) => job.endsWith(jobName));
|
|
143
143
|
if (jobs.length === 0) console.error(`Can not find job: ${name}`);
|
|
144
144
|
return jobs;
|
|
145
145
|
}
|
|
@@ -116,7 +116,7 @@ class StorageFile {
|
|
|
116
116
|
*/
|
|
117
117
|
async getLastLineBreaker(start, end, checkPoint = -1) {
|
|
118
118
|
/**
|
|
119
|
-
* How many characters to look back to find a
|
|
119
|
+
* How many characters to look back to find a possible line breaker. If no
|
|
120
120
|
* link break in this range, it will extend to find the last one.
|
|
121
121
|
*/
|
|
122
122
|
const possibleLineBreakRange = 1000;
|
package/src/components/utils.js
CHANGED
|
@@ -29,7 +29,7 @@ const lodash = require('lodash');
|
|
|
29
29
|
* data that will be sent out in one single request.
|
|
30
30
|
* Some APIs allows partial failure: it will take those correct data and
|
|
31
31
|
* response with reasons for those failed ones. 'groupedFailed' uses error
|
|
32
|
-
* message as the key, and
|
|
32
|
+
* message as the key, and the array of related failed lines(records) as value.
|
|
33
33
|
* Some APIs upload whole file. In this case, there will be not 'numberOfLines'
|
|
34
34
|
* or 'failedLines', etc.
|
|
35
35
|
* @typedef {{
|
|
@@ -593,6 +593,19 @@ const changeObjectNamingFromLowerCamelToSnake = (obj) => {
|
|
|
593
593
|
}
|
|
594
594
|
};
|
|
595
595
|
|
|
596
|
+
/**
|
|
597
|
+
* Converts a string to be a safe column name for BigQuery table.
|
|
598
|
+
* Only keep letters, digits and underscore and replace all other characters
|
|
599
|
+
* with underscores. If the string starts with a digit, add a leading
|
|
600
|
+
* underscore to it.
|
|
601
|
+
*
|
|
602
|
+
* @param {string} str
|
|
603
|
+
* @return {string}
|
|
604
|
+
*/
|
|
605
|
+
const changeStringToBigQuerySafe = (str) => {
|
|
606
|
+
return str.trim().replace(/[^a-zA-Z0-9_]/g, '_').replace(/^([0-9])/, '_$1');
|
|
607
|
+
}
|
|
608
|
+
|
|
596
609
|
/**
|
|
597
610
|
* Generates a function that can convert a given JSON object to a JSON string
|
|
598
611
|
* with only specified fields(fieldMask), in specified naming convention.
|
|
@@ -658,6 +671,7 @@ module.exports = {
|
|
|
658
671
|
changeNamingFromLowerCamelToSnake,
|
|
659
672
|
changeObjectNamingFromSnakeToLowerCamel,
|
|
660
673
|
changeObjectNamingFromLowerCamelToSnake,
|
|
674
|
+
changeStringToBigQuerySafe,
|
|
661
675
|
getFilterAndStringifyFn,
|
|
662
676
|
requestWithRetry,
|
|
663
677
|
};
|
package/bin/apps_scripts.sh
DELETED
|
@@ -1,209 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
#
|
|
3
|
-
# Copyright 2022 Google Inc.
|
|
4
|
-
#
|
|
5
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
-
# you may not use this file except in compliance with the License.
|
|
7
|
-
# You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
|
|
17
|
-
# Const the folder name of Apps Script.
|
|
18
|
-
DEFAULT_APPS_SCRIPT_FOLDER="apps_script"
|
|
19
|
-
|
|
20
|
-
#######################################
|
|
21
|
-
# Clasp login.
|
|
22
|
-
# Globals:
|
|
23
|
-
# None
|
|
24
|
-
# Arguments:
|
|
25
|
-
# None
|
|
26
|
-
#######################################
|
|
27
|
-
clasp_login() {
|
|
28
|
-
while :; do
|
|
29
|
-
local claspLogin=$(clasp login --status)
|
|
30
|
-
if [[ "${claspLogin}" != "You are not logged in." ]]; then
|
|
31
|
-
printf '%s' "${claspLogin} Would you like to continue with it? [Y/n]"
|
|
32
|
-
local logout
|
|
33
|
-
read -r logout
|
|
34
|
-
logout=${logout:-"Y"}
|
|
35
|
-
if [[ ${logout} == "Y" || ${logout} == "y" ]]; then
|
|
36
|
-
break
|
|
37
|
-
else
|
|
38
|
-
clasp logout
|
|
39
|
-
fi
|
|
40
|
-
fi
|
|
41
|
-
clasp login --no-localhost
|
|
42
|
-
done
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
#######################################
|
|
46
|
-
# Initialize a AppsScript project. Usually it involves following steps:
|
|
47
|
-
# 1. Create a AppsScript project within a new Google Sheet.
|
|
48
|
-
# 2. Prompt to update the Google Cloud Project number of the AppsScript project
|
|
49
|
-
# to enable external APIs for this AppsScript project.
|
|
50
|
-
# 3. Prompt to grant the access of Cloud Functions' default service account to
|
|
51
|
-
# this Google Sheet, so the Cloud Functions can query this Sheet later.
|
|
52
|
-
# 4. Initialize the Sheet based on requests.
|
|
53
|
-
# Globals:
|
|
54
|
-
# None
|
|
55
|
-
# Arguments:
|
|
56
|
-
# The Google Sheet name.
|
|
57
|
-
# The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
|
|
58
|
-
#######################################
|
|
59
|
-
clasp_initialize() {
|
|
60
|
-
((STEP += 1))
|
|
61
|
-
printf '%s\n' "Step ${STEP}: Starting to create Google Sheets..."
|
|
62
|
-
local sheetName="${1}"
|
|
63
|
-
local apps_script_src="${2-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
|
|
64
|
-
clasp_login
|
|
65
|
-
while :; do
|
|
66
|
-
local claspStatus=$(
|
|
67
|
-
clasp status -P "${apps_script_src}" >/dev/null 2>&1
|
|
68
|
-
echo $?
|
|
69
|
-
)
|
|
70
|
-
if [[ $claspStatus -gt 0 ]]; then
|
|
71
|
-
clasp create --type sheets --title "${sheetName}" --rootDir "${apps_script_src}"
|
|
72
|
-
local createResult=$?
|
|
73
|
-
if [[ $createResult -gt 0 ]]; then
|
|
74
|
-
printf '%s' "Press any key to continue after you enable the Google \
|
|
75
|
-
Apps Script API: https://script.google.com/home/usersettings..."
|
|
76
|
-
local any
|
|
77
|
-
read -n1 -s any
|
|
78
|
-
printf '\n\n'
|
|
79
|
-
continue
|
|
80
|
-
fi
|
|
81
|
-
break
|
|
82
|
-
else
|
|
83
|
-
printf '%s' "AppsScript project exists. Would you like to continue with \
|
|
84
|
-
it? [Y/n]"
|
|
85
|
-
local useCurrent
|
|
86
|
-
read -r useCurrent
|
|
87
|
-
useCurrent=${useCurrent:-"Y"}
|
|
88
|
-
if [[ ${useCurrent} = "Y" || ${useCurrent} = "y" ]]; then
|
|
89
|
-
break
|
|
90
|
-
else
|
|
91
|
-
printf '%s' "Would you like to delete current AppsScript and create a \
|
|
92
|
-
new one? [N/y]"
|
|
93
|
-
local deleteCurrent
|
|
94
|
-
read -r deleteCurrent
|
|
95
|
-
deleteCurrent=${deleteCurrent:-"N"}
|
|
96
|
-
if [[ ${deleteCurrent} = "Y" || ${deleteCurrent} = "y" ]]; then
|
|
97
|
-
rm "${apps_script_src}/.clasp.json"
|
|
98
|
-
continue
|
|
99
|
-
fi
|
|
100
|
-
fi
|
|
101
|
-
fi
|
|
102
|
-
done
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
#######################################
|
|
106
|
-
# Copy GCP project configuration file to AppsScript codes as a constant named
|
|
107
|
-
# `GCP_CONFIG`.
|
|
108
|
-
# Globals:
|
|
109
|
-
# None
|
|
110
|
-
# Arguments:
|
|
111
|
-
# The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
|
|
112
|
-
#######################################
|
|
113
|
-
generate_config_js_for_apps_script() {
|
|
114
|
-
local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
|
|
115
|
-
local generated_file="${apps_script_src}/.generated_config.js"
|
|
116
|
-
if [[ -f "${CONFIG_FILE}" ]]; then
|
|
117
|
-
echo '// Copyright 2022 Google Inc.
|
|
118
|
-
//
|
|
119
|
-
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
120
|
-
// you may not use this file except in compliance with the License.
|
|
121
|
-
// You may obtain a copy of the License at
|
|
122
|
-
//
|
|
123
|
-
// http://www.apache.org/licenses/LICENSE-2.0
|
|
124
|
-
//
|
|
125
|
-
// Unless required by applicable law or agreed to in writing, software
|
|
126
|
-
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
127
|
-
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
128
|
-
// See the License for the specific language governing permissions and
|
|
129
|
-
// limitations under the License.
|
|
130
|
-
|
|
131
|
-
/** @fileoverview Auto-generated configuration file for Apps Script. */
|
|
132
|
-
'>"${generated_file}"
|
|
133
|
-
echo -n "const GCP_CONFIG = " >>"${generated_file}"
|
|
134
|
-
cat "${CONFIG_FILE}" >>"${generated_file}"
|
|
135
|
-
else
|
|
136
|
-
printf '%s\n' "Couldn't find ${CONFIG_FILE}."
|
|
137
|
-
fi
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
#######################################
|
|
141
|
-
# Clasp pushes AppsScript codes.
|
|
142
|
-
# Globals:
|
|
143
|
-
# None
|
|
144
|
-
# Arguments:
|
|
145
|
-
# The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
|
|
146
|
-
#######################################
|
|
147
|
-
clasp_push_codes() {
|
|
148
|
-
((STEP += 1))
|
|
149
|
-
printf '%s\n' "Step ${STEP}: Starting to push codes to the Google Sheets..."
|
|
150
|
-
local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
|
|
151
|
-
clasp status -P "${apps_script_src}" >>/dev/null
|
|
152
|
-
local project_status=$?
|
|
153
|
-
if [[ ${project_status} -gt 0 ]]; then
|
|
154
|
-
return ${project_status}
|
|
155
|
-
else
|
|
156
|
-
generate_config_js_for_apps_script "${apps_script_src}"
|
|
157
|
-
clasp push --force -P "${apps_script_src}"
|
|
158
|
-
fi
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
#######################################
|
|
162
|
-
# Ask user to update the GCP number of this AppsScript.
|
|
163
|
-
# Globals:
|
|
164
|
-
# GCP_PROJECT
|
|
165
|
-
# Arguments:
|
|
166
|
-
# The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
|
|
167
|
-
#######################################
|
|
168
|
-
clasp_update_project_number() {
|
|
169
|
-
((STEP += 1))
|
|
170
|
-
local projectNumber=$(get_project_number)
|
|
171
|
-
local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
|
|
172
|
-
printf '%s\n' "Step ${STEP}: Update Google Cloud Platform (GCP) Project for \
|
|
173
|
-
Apps Script."
|
|
174
|
-
printf '%s' " "
|
|
175
|
-
clasp open -P "${apps_script_src}"
|
|
176
|
-
printf '%s\n' " On the open tab of Apps Script, use 'Project \
|
|
177
|
-
Settings' to set the Google Cloud Platform (GCP) Project as: ${projectNumber}"
|
|
178
|
-
printf '%s' "Press any key to continue after you update the GCP number..."
|
|
179
|
-
local any
|
|
180
|
-
read -n1 -s any
|
|
181
|
-
printf '\n'
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
#######################################
|
|
185
|
-
# Ask user to grant the access to CF's default service account.
|
|
186
|
-
# Note: the target GCP needs to have OAuth consent screen.
|
|
187
|
-
# Globals:
|
|
188
|
-
# SHEET_URL
|
|
189
|
-
# Arguments:
|
|
190
|
-
# The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
|
|
191
|
-
#######################################
|
|
192
|
-
grant_access_to_service_account() {
|
|
193
|
-
((STEP += 1))
|
|
194
|
-
local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
|
|
195
|
-
local defaultServiceAccount=$(get_cloud_functions_service_account \
|
|
196
|
-
"${PROJECT_NAMESPACE}_main")
|
|
197
|
-
local parentId=$(get_value_from_json_file "${apps_script_src}"/.clasp.json \
|
|
198
|
-
parentId|cut -d\" -f2)
|
|
199
|
-
printf '%s\n' "Step ${STEP}: Share the Google Sheet with ${SOLUTION_NAME}."
|
|
200
|
-
|
|
201
|
-
printf '%s\n' " Open Google Sheet: \
|
|
202
|
-
https://drive.google.com/open?id=${parentId}"
|
|
203
|
-
printf '%s\n' " Click 'Share' and grant the Viewer access to: \
|
|
204
|
-
${defaultServiceAccount}"
|
|
205
|
-
printf '%s' "Press any key to continue after you grant the access..."
|
|
206
|
-
local any
|
|
207
|
-
read -n1 -s any
|
|
208
|
-
printf '\n'
|
|
209
|
-
}
|
package/bin/bigquery.sh
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
#
|
|
3
|
-
# Copyright 2022 Google Inc.
|
|
4
|
-
#
|
|
5
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
-
# you may not use this file except in compliance with the License.
|
|
7
|
-
# You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
|
|
17
|
-
#######################################
|
|
18
|
-
# Checks whether the BigQuery object (table or view) exists.
|
|
19
|
-
# Globals:
|
|
20
|
-
# GCP_PROJECT
|
|
21
|
-
# DATASET
|
|
22
|
-
# BIGQUERY_LOG_TABLE
|
|
23
|
-
# Arguments:
|
|
24
|
-
# None
|
|
25
|
-
#######################################
|
|
26
|
-
check_existence_in_bigquery() {
|
|
27
|
-
bq show "${1}" >/dev/null 2>&1
|
|
28
|
-
printf '%d' $?
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
#######################################
|
|
32
|
-
# Creates or updates the BigQuery view.
|
|
33
|
-
# Globals:
|
|
34
|
-
# GCP_PROJECT
|
|
35
|
-
# DATASET
|
|
36
|
-
# Arguments:
|
|
37
|
-
# The name of view.
|
|
38
|
-
# The query of view.
|
|
39
|
-
#######################################
|
|
40
|
-
create_or_update_view() {
|
|
41
|
-
local viewName viewQuery
|
|
42
|
-
viewName="${1}"
|
|
43
|
-
viewQuery=${2}
|
|
44
|
-
local action="mk"
|
|
45
|
-
if [[ $(check_existence_in_bigquery "${DATASET}.${viewName}") -eq 0 ]]; then
|
|
46
|
-
action="update"
|
|
47
|
-
fi
|
|
48
|
-
bq "${action}" \
|
|
49
|
-
--use_legacy_sql=false \
|
|
50
|
-
--view "${viewQuery}" \
|
|
51
|
-
--project_id ${GCP_PROJECT} \
|
|
52
|
-
"${DATASET}.${viewName}"
|
|
53
|
-
}
|
package/bin/google_ads.sh
DELETED
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
#
|
|
3
|
-
# Copyright 2022 Google Inc.
|
|
4
|
-
#
|
|
5
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
-
# you may not use this file except in compliance with the License.
|
|
7
|
-
# You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
-
# See the License for the specific language governing permissions and
|
|
15
|
-
# limitations under the License.
|
|
16
|
-
|
|
17
|
-
# Google Ads API version
|
|
18
|
-
GOOGLE_ADS_API_VERSION=16
|
|
19
|
-
|
|
20
|
-
#######################################
|
|
21
|
-
# Verify whether the current OAuth token, CID and developer token can work.
|
|
22
|
-
# Globals:
|
|
23
|
-
# None
|
|
24
|
-
# Arguments:
|
|
25
|
-
# MCC CID
|
|
26
|
-
# Developer token
|
|
27
|
-
#######################################
|
|
28
|
-
validate_googleads_account() {
|
|
29
|
-
local cid developerToken accessToken request response
|
|
30
|
-
cid=${1}
|
|
31
|
-
developerToken=${2}
|
|
32
|
-
accessToken=$(get_oauth_access_token)
|
|
33
|
-
request=(
|
|
34
|
-
-H "Accept: application/json"
|
|
35
|
-
-H "Content-Type: application/json"
|
|
36
|
-
-H "developer-token: ${developerToken}"
|
|
37
|
-
-H "Authorization: Bearer ${accessToken}"
|
|
38
|
-
-X POST "https://googleads.googleapis.com/v${GOOGLE_ADS_API_VERSION}/customers/${cid}/googleAds:search"
|
|
39
|
-
-d '{"query": "SELECT customer.id FROM customer"}'
|
|
40
|
-
)
|
|
41
|
-
response=$(curl "${request[@]}" 2>/dev/null)
|
|
42
|
-
local errorCode errorMessage
|
|
43
|
-
errorCode=$(get_value_from_json_string "${response}" "error.code")
|
|
44
|
-
if [[ -n "${errorCode}" ]]; then
|
|
45
|
-
errorMessage=$(get_value_from_json_string "${response}" "error.message")
|
|
46
|
-
printf '%s\n' "Validate failed: ${errorMessage}" >&2
|
|
47
|
-
return 1
|
|
48
|
-
fi
|
|
49
|
-
return 0
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
#######################################
|
|
53
|
-
# Let user input MCC CID and developer token for cronjob(s).
|
|
54
|
-
# Globals:
|
|
55
|
-
# MCC_CIDS
|
|
56
|
-
# DEVELOPER_TOKEN
|
|
57
|
-
# Arguments:
|
|
58
|
-
# None
|
|
59
|
-
#######################################
|
|
60
|
-
set_google_ads_account() {
|
|
61
|
-
printf '%s\n' "Setting up Google Ads account information..."
|
|
62
|
-
local developToken mccCids
|
|
63
|
-
while :; do
|
|
64
|
-
# Developer token
|
|
65
|
-
while [[ -z ${developToken} ]]; do
|
|
66
|
-
printf '%s' " Enter the developer token[${DEVELOPER_TOKEN}]: "
|
|
67
|
-
read -r input
|
|
68
|
-
developToken="${input:-${DEVELOPER_TOKEN}}"
|
|
69
|
-
done
|
|
70
|
-
DEVELOPER_TOKEN="${developToken}"
|
|
71
|
-
mccCids=""
|
|
72
|
-
# MCC CIDs
|
|
73
|
-
while :; do
|
|
74
|
-
printf '%s' " Enter the MCC CID: "
|
|
75
|
-
read -r input
|
|
76
|
-
if [[ -z ${input} ]]; then
|
|
77
|
-
continue
|
|
78
|
-
fi
|
|
79
|
-
input="$(printf '%s' "${input}" | sed -r 's/-//g')"
|
|
80
|
-
printf '%s' " validating ${input}...... "
|
|
81
|
-
validate_googleads_account ${input} ${DEVELOPER_TOKEN}
|
|
82
|
-
if [[ $? -eq 1 ]]; then
|
|
83
|
-
printf '%s\n' "failed.
|
|
84
|
-
Press 'd' to re-enter developer token ["${developToken}"] or
|
|
85
|
-
'C' to continue with this MCC CID or
|
|
86
|
-
any other key to enter another MCC CID..."
|
|
87
|
-
local any
|
|
88
|
-
read -n1 -s any
|
|
89
|
-
if [[ "${any}" == "d" ]]; then
|
|
90
|
-
developToken=""
|
|
91
|
-
continue 2
|
|
92
|
-
elif [[ "${any}" == "C" ]]; then
|
|
93
|
-
printf '%s\n' "WARNING! Continue with FAILED MCC ${input}."
|
|
94
|
-
else
|
|
95
|
-
continue
|
|
96
|
-
fi
|
|
97
|
-
else
|
|
98
|
-
printf '%s\n' "succeeded."
|
|
99
|
-
fi
|
|
100
|
-
mccCids+=",${input}"
|
|
101
|
-
printf '%s' " Do you want to add another MCC CID? [Y/n]: "
|
|
102
|
-
read -r input
|
|
103
|
-
if [[ ${input} == 'n' || ${input} == 'N' ]]; then
|
|
104
|
-
break
|
|
105
|
-
fi
|
|
106
|
-
done
|
|
107
|
-
# Left Shift one position to remove the first comma.
|
|
108
|
-
# After shifting, MCC_CIDS would like "11111,22222".
|
|
109
|
-
MCC_CIDS="${mccCids:1}"
|
|
110
|
-
printf '%s\n' "Using Google Ads MCC CIDs: ${MCC_CIDS}."
|
|
111
|
-
break
|
|
112
|
-
done
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
printf '%s\n' "Google Ads Bash Library is loaded."
|