@google-cloud/nodejs-common 2.3.1 → 2.4.0

package/README.md

@@ -67,9 +67,11 @@ and [Data Tasks Coordinator]. This library includes:
      an adapter to wrap a Node8 Cloud Functions into Node6 and Node8 compatible
      functions.~~ (This has been removed since v1.9.0)
 
-4. A share library for [Bash] to facilitate installation tasks.
+4. ~~A share library for [Bash] to facilitate installation tasks.~~ (This has
+ been deprecated since v2.4.0. [Sheets Based Installer] is preferred.)
 
 [gmp and google ads connector]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/gmp-googleads-connector
 [data tasks coordinator]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/data-tasks-coordinator
+[sheets based installer]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/main/marketing-analytics/activation/sheets-based-installer
 [comparison]: https://cloud.google.com/datastore/docs/firestore-or-datastore
 [bash]: https://www.gnu.org/software/bash/

package/bin/install_functions.sh

@@ -2018,10 +2018,4 @@ join_string_array() {
   printf %s "$first" "${@/#/$separator}" | sed -e "s/\\$separator/$separator/g"
 }
 
-# Import other bash files.
-_SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${_SELF}/google_ads.sh"
-source "${_SELF}/bigquery.sh"
-source "${_SELF}/apps_scripts.sh"
-
 printf '%s\n' "Common Bash Library is loaded."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -16,22 +16,22 @@
   },
   "homepage": "https://github.com/GoogleCloudPlatform/cloud-for-marketing/blob/master/marketing-analytics/activation/common-libs/nodejs-common/README.md",
   "dependencies": {
-    "@google-cloud/aiplatform": "^3.25.0",
+    "@google-cloud/aiplatform": "^3.34.0",
     "@google-cloud/automl": "^4.0.1",
-    "@google-cloud/bigquery": "^7.8.0",
-    "@google-cloud/datastore": "^9.1.0",
-    "@google-cloud/firestore": "^7.9.0",
+    "@google-cloud/bigquery": "^7.9.1",
+    "@google-cloud/datastore": "^9.2.1",
+    "@google-cloud/firestore": "^7.10.0",
     "@google-cloud/logging-winston": "^6.0.0",
-    "@google-cloud/pubsub": "^4.5.0",
-    "@google-cloud/storage": "^7.12.0",
+    "@google-cloud/pubsub": "^4.9.0",
+    "@google-cloud/storage": "^7.14.0",
     "@google-cloud/scheduler": "^4.3.0",
     "@google-cloud/secret-manager": "^5.6.0",
-    "gaxios": "^6.7.0",
-    "google-ads-nodejs-client": "16.0.0",
-    "google-auth-library": "^9.11.0",
-    "googleapis": "^140.0.1",
-    "winston": "^3.13.1",
-    "@grpc/grpc-js": "^1.11.1",
+    "gaxios": "^6.7.1",
+    "google-ads-nodejs-client": "^18.0.1",
+    "google-auth-library": "^9.15.0",
+    "googleapis": "^144.0.0",
+    "winston": "^3.17.0",
+    "@grpc/grpc-js": "^1.12.2",
     "lodash": "^4.17.21"
   },
   "devDependencies": {

package/src/apis/base/ads_api_common.js

@@ -45,14 +45,19 @@ class RestSearchStreamTransform extends Transform {
   /**
    * @constructor
    * @param {boolean=} snakeCase Whether or not output JSON in snake naming.
+   * @param {function|undefined} postProcessFn An optional function to process
+   *     the data after the default process. The default process includes
+   *     filtering out fields based on `fieldMask` and adjusting the naming
+   *     convention.
    */
-  constructor(snakeCase = false) {
+  constructor(snakeCase = false, postProcessFn) {
     super({ objectMode: true });
     this.snakeCase = snakeCase;
     this.chunks = [Buffer.from('')];
     this.processFn; // The function to process a row of the report.
     this.logger = getLogger('ADS.STREAM.T');
     this.stopwatch = Date.now();
+    this.postProcessFn = postProcessFn;
   }
 
   _transform(chunk, encoding, callback) {
@@ -68,7 +73,14 @@ class RestSearchStreamTransform extends Transform {
           .substring(maskIndex + FIELD_MASK_TAG.length, rawString.indexOf(END_TAG))
           .split('"')[1];
         this.logger.debug(`Got fieldMask: ${fieldMask}`);
-        this.processFn = getFilterAndStringifyFn(fieldMask, this.snakeCase);
+        const processFn = getFilterAndStringifyFn(fieldMask, this.snakeCase);
+        if (this.postProcessFn) {
+          this.processFn = (obj) => {
+            return this.postProcessFn(processFn(obj));
+          }
+        } else {
+          this.processFn = processFn;
+        }
       }
       const resultsWithTailing = rawString.substring(startIndex, maskIndex);
       const results = resultsWithTailing.substring(

package/src/apis/base/google_api_client.js

@@ -37,17 +37,30 @@ class GoogleApiClient extends AuthRestfulApi {
    */
   getVersion() { }
 
+  /**
+   * Gets the default options to initialize an Api object in Google Api client
+   * library. It contains the Api version and auth information.
+   * @param {object|undefined} initOptions
+   * @return {object}
+   */
+  async getApiClientInitOptions(initOptions) {
+    return {
+      version: this.getVersion(),
+      auth: await this.getAuth(),
+    };
+  }
+
   /**
    * Returns the Api instance.
+   * This function expects an instance name for the object in Google Api client
+   * library, e.g. searchads360 for 'Search Ads 360'.
    * @return {!Promise<object>} The Api instance.
    */
-  async getApiClient() {
+  async getApiClient(initOptions = {}) {
     if (this.apiClient) return this.apiClient;
     this.logger.info(`Initialized ${this.constructor.name} instance.`);
-    this.apiClient = google[this.googleApi]({
-      version: this.getVersion(),
-      auth: await this.getAuth(),
-    });
+    const options = await this.getApiClientInitOptions(initOptions);
+    this.apiClient = google[this.googleApi](options);
     return this.apiClient;
   }
 }

package/src/apis/doubleclick_search.js

@@ -19,7 +19,6 @@
 
 'use strict';
 
-const {request} = require('gaxios');
 const { GoogleApiClient } = require('./base/google_api_client.js');
 const {
   getLogger,
@@ -66,47 +65,6 @@ let InsertConversionsConfig;
  */
 let AvailabilityConfig;
 
-/**
- * Nodejs Google API client library doesn't export this type, so here is a
- * partial typedef of Report Request which only contains essential properties.
- * For complete definition, see:
- * https://developers.google.com/search-ads/v2/reference/reports/request
- * @typedef {{
- *    reportScope: {
- *      agencyId: string,
- *      advertiserId: string,
- *    },
- *    reportType: string,
- *    columns: Array<{
- *      columnName: string,
- *      headerText: string,
- *      startDate: string,
- *      endDate: string
- *    }>,
- *    filters: Array<{
- *      columnName: string,
- *      headerText: string,
- *      startDate: string,
- *      endDate: string
- *    }>|undefined,
- *    timeRange: {
- *      startDate: string,
- *      endDate: string,
- *      changedMetricsSinceTimestamp: datetime|undefined,
- *      changedAttributesSinceTimestamp: datetime|undefined,
- *    },
- *    downloadFormat: 'CSV'|'TSV',
- *    statisticsCurrency: 'usd'|'agency'|'advertiser'|'account',
- *    maxRowsPerFile: integer,
- *    includeDeletedEntities: boolean|undefined,
- *    includeRemovedEntities: boolean|undefined,
- *    verifySingleTimeZone: boolean|undefined,
- *    startRow: integer|undefined,
- *    rowCount: integer|undefined,
- * }}
- */
-let ReportRequest;
-
 /**
  * DoubleClick Search (DS) Ads 360 API v2 stub.
  * See: https://developers.google.com/search-ads/v2/reference/
@@ -293,102 +251,12 @@ class DoubleClickSearch extends GoogleApiClient {
     });
     batchResult.errors = Array.from(errors);
   }
-
-  /**
-   * There are three steps to get asynchronous reports in SA360:
-   * 1. Call Reports.request() to specify the type of data for the report.
-   * 2. Call Reports.get() with the report id to check whether it's ready.
-   * 3. Call Reports.getFile() to the download the report files.
-   * @see https://developers.google.com/search-ads/v2/how-tos/reporting/asynchronous-requests
-   *
-   * This is the first step.
-   * @param {!ReportRequest} requestBody
-   * @return {!Promise<string>}
-   */
-  async requestReports(requestBody) {
-    const doubleclicksearch = await this.getApiClient();
-    const { status, data } = await doubleclicksearch.reports.request({ requestBody });
-    if (status >= 200 && status < 300) {
-      return data.id;
-    }
-    const errorMsg = `Fail to request reports: ${JSON.stringify(requestBody)}`;
-    this.logger.error(errorMsg, data);
-    throw new Error(errorMsg);
-  }
-
-  /**
-   * Returns the report file links if the report is ready or undefined if not.
-   * @param {string} reportId
-   * @return {!Promise<undefined|Array<{
-   *   url:string,
-   *   byteCount:string,
-   * }>>}
-   */
-  async getReportUrls(reportId) {
-    const doubleclicksearch = await this.getApiClient();
-    const { status, data } = await doubleclicksearch.reports.get({ reportId });
-    switch (status) {
-      case 200:
-        const {rowCount, files} = data;
-        this.logger.info(
-            `Report[${reportId}] has ${rowCount} rows and ${files.length} files.`);
-        return files;
-      case 202:
-        this.logger.info(`Report[${reportId}] is not ready.`);
-        break;
-      default:
-        const errorMsg =
-            `Error in get reports: ${reportId} with status code: ${status}`;
-        this.logger.error(errorMsg, data);
-        throw new Error(errorMsg);
-    }
-  }
-
-  /**
-   * Get the given part of a report.
-   * @param {string} reportId
-   * @param {number} reportFragment The index (based 0) of report files.
-   * @return {!Promise<string>}
-   */
-  async getReportFile(reportId, reportFragment) {
-    const doubleclicksearch = await this.getApiClient();
-    const response = await doubleclicksearch.reports.getFile(
-        {reportId, reportFragment});
-    if (response.status === 200) {
-      return response.data;
-    }
-    const errorMsg =
-        `Error in get file from reports: ${reportFragment}@${reportId}`;
-    this.logger.error(errorMsg, response);
-    throw new Error(errorMsg);
-  }
-
-  /**
-   * Returns a readable stream of the report.
-   * In case of the report is large, use stream directly to write out to fit in
-   * the resource limited environment, e.g. Cloud Functions.
-   * @param {string} url
-   * @return {!Promise<ReadableStream>}
-   */
-  async getReportFileStream(url) {
-    const auth = await this.getAuth();
-    const headers = await auth.getRequestHeaders();
-    const response = await request({
-      method: 'GET',
-      headers,
-      url,
-      responseType: 'stream',
-    });
-    return response.data;
-  }
-
 }
 
 module.exports = {
   DoubleClickSearch,
   InsertConversionsConfig,
   AvailabilityConfig,
-  ReportRequest,
   API_VERSION,
   API_SCOPES,
 };

package/src/apis/search_ads.js

@@ -21,7 +21,8 @@
 const { request: gaxiosRequest } = require('gaxios');
 const { google } = require('googleapis');
 const { GoogleApiClient } = require('./base/google_api_client.js');
-const { getLogger } = require('../components/utils.js');
+const { changeStringToBigQuerySafe, getLogger }
+  = require('../components/utils.js');
 const { getCleanCid, RestSearchStreamTransform }
   = require('./base/ads_api_common.js');
 
@@ -59,7 +60,7 @@ class SearchAds extends GoogleApiClient {
   }
 
   /**
-   * Prepares the Search Ads 360 Reporting API instance.
+   * Returns the options to initialize Search Ads 360 Reporting API instance.
    * OAuth 2.0 application credentials is required for calling this API.
    * For Search Ads Reporting API calls made by a manager to a client account,
    * a HTTP header named `login-customer-id` is required in the request. This
@@ -67,19 +68,17 @@ class SearchAds extends GoogleApiClient {
    * API call. Be sure to remove any hyphens (—), for example: 1234567890, not
    * 123-456-7890.
    * @see https://developers.google.com/search-ads/reporting/api/reference/rest/auth
-   * @return {!google.searchads360}
-   * @private
+   * @return {!Object}
+   * @override
    */
-  async getApiClient(loginCustomerId) {
-    this.logger.debug(`Initialized SA reporting for ${loginCustomerId}`);
-    const options = {
-      version: this.getVersion(),
-      auth: await this.getAuth(),
-    };
+  async getApiClientInitOptions(initOptions) {
+    const options = await super.getApiClientInitOptions(initOptions);
+    const { loginCustomerId } = initOptions;
     if (loginCustomerId) {
+      this.logger.debug(`initialized with loginCustomerId: ${loginCustomerId}`);
       options.headers = { 'login-customer-id': getCleanCid(loginCustomerId) };
     }
-    return google.searchads360(options);
+    return options;
   }
 
   /**
@@ -96,7 +95,7 @@ class SearchAds extends GoogleApiClient {
    * @see https://developers.google.com/search-ads/reporting/api/reference/rpc/google.ads.searchads360.v0.services#searchsearchads360response
    */
   async getPaginatedReport(customerId, loginCustomerId, query, options = {}) {
-    const searchads = await this.getApiClient(loginCustomerId);
+    const searchads = await this.getApiClient({ loginCustomerId });
     const requestBody = Object.assign({
       query,
       pageSize: 10000,
@@ -146,17 +145,83 @@ class SearchAds extends GoogleApiClient {
    * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
    * @param {string} query A Google Ads Query string.
    * @param {boolean=} snakeCase Output JSON objects in snake_case.
+   * @param {boolean=} rawCustomColumns Keeps the raw custom column values as
+   *     an array.
    * @return {!Promise<stream>}
    */
   async cleanedRestStreamReport(customerId, loginCustomerId, query,
-    snakeCase = false) {
-    const transform = new RestSearchStreamTransform(snakeCase);
+    snakeCase = false, rawCustomColumns = false) {
+    let postProcessFn;
+    if (!rawCustomColumns && query.indexOf('custom_columns.id[') > -1) {
+      const convertor = await this.getCustomColumnsConvertor(
+        customerId, loginCustomerId, query);
+      postProcessFn = (str) => {
+        const source = JSON.parse(str);
+        source.customColumns = convertor(source.customColumns);
+        return JSON.stringify(source);
+      }
+    }
+    const transform = new RestSearchStreamTransform(snakeCase, postProcessFn);
     const stream =
       await this.restStreamReport(customerId, loginCustomerId, query);
     return stream.on('error', (error) => transform.emit('error', error))
       .pipe(transform);
   }
 
+  /**
+   * Gets all custom columns information from the report query. By default, it
+   * will load columns for the manager account. If there are some columns
+   * missed, it will load columns for the customerId.
+   * @param {string} customerId - The ID of the customer.
+   * @param {string} loginCustomerId - The ID of the manager.
+   * @param {string} query - The report query.
+   * @return {!Array<object>}
+   */
+  async getCustomColumnsFromQuery(customerId, loginCustomerId, query) {
+    const pattern = /custom_columns\.id\[(\d+)\]/g;
+    const columnIds = Array.from(query.matchAll(pattern), m => m[1]);
+    const selectedColumns = {};
+    const getSelectedColumn = (column) => {
+      if (columnIds.indexOf(column.id) > -1) {
+        selectedColumns[column.id] = column;
+      }
+    }
+    (await this.listCustomColumns(loginCustomerId, loginCustomerId))
+      .forEach(getSelectedColumn);
+    if (Object.keys(selectedColumns).length < columnIds.length) {
+      this.logger.warn('Missing custom columns in MCC, try CID now');
+      (await this.listCustomColumns(customerId, loginCustomerId))
+        .forEach(getSelectedColumn);
+    }
+    return columnIds.map((columnId) => selectedColumns[columnId]);
+  }
+
+  /**
+   * Returns the function to convert the array values of custom columns to
+   * an object with the column names as the property keys. The column names will
+   * be proceeded to be BigQuery naming safe.
+   * @param {string} customerId - The ID of the customer.
+   * @param {string} loginCustomerId - The ID of the manager.
+   * @param {string} query - The report query.
+   * @return {function}
+   */
+  async getCustomColumnsConvertor(customerId, loginCustomerId, query) {
+    const customerColumns = await this.getCustomColumnsFromQuery(
+      customerId, loginCustomerId, query);
+    const columnConvertors = customerColumns.map(({ name, valueType }) => {
+      return (obj) => {
+        const key = changeStringToBigQuerySafe(name);
+        const value = obj[`${valueType.toLowerCase()}Value`];
+        return { [key]: value };
+      };
+    });
+    const convertor = (customerColumns) => {
+      return Object.assign(
+        ...customerColumns.map((obj, index) => columnConvertors[index](obj)));
+    };
+    return convertor;
+  }
+
   /**
    * Returns the requested field or resource (artifact) used by SearchAds360Service.
    * This service doesn't require `login-customer-id` HTTP header.
@@ -203,9 +268,9 @@ class SearchAds extends GoogleApiClient {
    * @see https://developers.google.com/search-ads/reporting/api/reference/rest/v0/customers.customColumns#CustomColumn
    */
   async listCustomColumns(customerId, loginCustomerId) {
-    const searchads = await this.getApiClient(loginCustomerId);
+    const searchads = await this.getApiClient({ loginCustomerId });
     const response = await searchads.customers.customColumns.list({ customerId });
-    return response.data.customColumns;
+    return response.data.customColumns || [];
   }
 
   /**
@@ -219,7 +284,7 @@ class SearchAds extends GoogleApiClient {
    */
   async getCustomColumn(columnId, customerId, loginCustomerId) {
     const resourceName = `customers/${customerId}/customColumns/${columnId}`;
-    const searchads = await this.getApiClient(loginCustomerId);
+    const searchads = await this.getApiClient({ loginCustomerId });
     const response = await searchads.customers.customColumns.get({ resourceName });
     return response.data;
   }

package/src/apis/spreadsheets.js

@@ -297,12 +297,13 @@ class Spreadsheets extends GoogleApiClient {
   /**
    * Gets values of first row of the specified sheet.
    * @param {string} sheetName
+   * @param {number} skipLeadingRows
    * @return {!Array<string>}
    */
-  async getHeadline(sheetName) {
+  async getHeadline(sheetName, skipLeadingRows = 1) {
     const request = {
       spreadsheetId: this.spreadsheetId,
-      range: `'${sheetName}'!1:1`,
+      range: `'${sheetName}'!${skipLeadingRows}:${skipLeadingRows}`,
     };
     const sheets = await this.getApiClient();
     const response = await sheets.spreadsheets.values.get(request);

package/src/components/scheduler.js

@@ -137,9 +137,9 @@ class CloudScheduler {
    * @private
    */
   async getJobs_(name, targetLocations = undefined) {
-    const regex = new RegExp(`/jobs/${name}$`);
+    const jobName = `/jobs/${name}`;
     const allJobs = await this.listJobs_(targetLocations);
-    const jobs = allJobs.filter((job) => regex.test(job));
+    const jobs = allJobs.filter((job) => job.endsWith(jobName));
     if (jobs.length === 0) console.error(`Can not find job: ${name}`);
     return jobs;
   }

package/src/components/utils.js

@@ -593,6 +593,19 @@ const changeObjectNamingFromLowerCamelToSnake = (obj) => {
   }
 };
 
+/**
+ * Converts a string to be a safe column name for BigQuery table.
+ * Only keep letters, digits and underscore and replace all other characters
+ * with underscores. If the string starts with a digit, add a leading
+ * underscore to it.
+ *
+ * @param {string} str
+ * @return {string}
+ */
+const changeStringToBigQuerySafe = (str) => {
+  return str.trim().replace(/[^a-zA-Z0-9_]/g, '_').replace(/^([0-9])/, '_$1');
+}
+
 /**
  * Generates a function that can convert a given JSON object to a JSON string
  * with only specified fields(fieldMask), in specified naming convention.
@@ -658,6 +671,7 @@ module.exports = {
   changeNamingFromLowerCamelToSnake,
   changeObjectNamingFromSnakeToLowerCamel,
   changeObjectNamingFromLowerCamelToSnake,
+  changeStringToBigQuerySafe,
   getFilterAndStringifyFn,
   requestWithRetry,
 };

package/bin/apps_scripts.sh

@@ -1,209 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright 2022 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Const the folder name of Apps Script.
-DEFAULT_APPS_SCRIPT_FOLDER="apps_script"
-
-#######################################
-# Clasp login.
-# Globals:
-#   None
-# Arguments:
-#   None
-#######################################
-clasp_login() {
-  while :; do
-    local claspLogin=$(clasp login --status)
-    if [[ "${claspLogin}" != "You are not logged in." ]]; then
-      printf '%s' "${claspLogin} Would you like to continue with it? [Y/n]"
-      local logout
-      read -r logout
-      logout=${logout:-"Y"}
-      if [[ ${logout} == "Y" || ${logout} == "y" ]]; then
-        break
-      else
-        clasp logout
-      fi
-    fi
-    clasp login --no-localhost
-  done
-}
-
-#######################################
-# Initialize a AppsScript project. Usually it involves following steps:
-# 1. Create a AppsScript project within a new Google Sheet.
-# 2. Prompt to update the Google Cloud Project number of the AppsScript project
-#    to enable external APIs for this AppsScript project.
-# 3. Prompt to grant the access of Cloud Functions' default service account to
-#    this Google Sheet, so the Cloud Functions can query this Sheet later.
-# 4. Initialize the Sheet based on requests.
-# Globals:
-#   None
-# Arguments:
-#   The Google Sheet name.
-#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
-#######################################
-clasp_initialize() {
-  ((STEP += 1))
-  printf '%s\n' "Step ${STEP}: Starting to create Google Sheets..."
-  local sheetName="${1}"
-  local apps_script_src="${2-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
-  clasp_login
-  while :; do
-    local claspStatus=$(
-      clasp status -P "${apps_script_src}" >/dev/null 2>&1
-      echo $?
-    )
-    if [[ $claspStatus -gt 0 ]]; then
-      clasp create --type sheets --title "${sheetName}" --rootDir "${apps_script_src}"
-      local createResult=$?
-      if [[ $createResult -gt 0 ]]; then
-        printf '%s' "Press any key to continue after you enable the Google \
-Apps Script API: https://script.google.com/home/usersettings..."
-        local any
-        read -n1 -s any
-        printf '\n\n'
-        continue
-      fi
-      break
-    else
-      printf '%s' "AppsScript project exists. Would you like to continue with \
-it? [Y/n]"
-      local useCurrent
-      read -r useCurrent
-      useCurrent=${useCurrent:-"Y"}
-      if [[ ${useCurrent} = "Y" || ${useCurrent} = "y" ]]; then
-        break
-      else
-        printf '%s' "Would you like to delete current AppsScript and create a \
-new one? [N/y]"
-        local deleteCurrent
-        read -r deleteCurrent
-        deleteCurrent=${deleteCurrent:-"N"}
-        if [[ ${deleteCurrent} = "Y" || ${deleteCurrent} = "y" ]]; then
-          rm "${apps_script_src}/.clasp.json"
-          continue
-        fi
-      fi
-    fi
-  done
-}
-
-#######################################
-# Copy GCP project configuration file to AppsScript codes as a constant named
-# `GCP_CONFIG`.
-# Globals:
-#   None
-# Arguments:
-#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
-#######################################
-generate_config_js_for_apps_script() {
-  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
-  local generated_file="${apps_script_src}/.generated_config.js"
-  if [[ -f "${CONFIG_FILE}" ]]; then
-    echo '// Copyright 2022 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/** @fileoverview Auto-generated configuration file for Apps Script. */
-'>"${generated_file}"
-    echo -n "const GCP_CONFIG = " >>"${generated_file}"
-    cat "${CONFIG_FILE}" >>"${generated_file}"
-  else
-    printf '%s\n' "Couldn't find ${CONFIG_FILE}."
-  fi
-}
-
-#######################################
-# Clasp pushes AppsScript codes.
-# Globals:
-#   None
-# Arguments:
-#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
-#######################################
-clasp_push_codes() {
-  ((STEP += 1))
-  printf '%s\n' "Step ${STEP}: Starting to push codes to the Google Sheets..."
-  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
-  clasp status -P "${apps_script_src}" >>/dev/null
-  local project_status=$?
-  if [[ ${project_status} -gt 0 ]]; then
-    return ${project_status}
-  else
-    generate_config_js_for_apps_script "${apps_script_src}"
-    clasp push --force -P "${apps_script_src}"
-  fi
-}
-
-#######################################
-# Ask user to update the GCP number of this AppsScript.
-# Globals:
-#   GCP_PROJECT
-# Arguments:
-#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
-#######################################
-clasp_update_project_number() {
-  ((STEP += 1))
-  local projectNumber=$(get_project_number)
-  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
-  printf '%s\n' "Step ${STEP}: Update Google Cloud Platform (GCP) Project for \
-Apps Script."
-  printf '%s' "  "
-  clasp open -P "${apps_script_src}"
-  printf '%s\n' "  On the open tab of Apps Script, use 'Project \
-Settings' to set the Google Cloud Platform (GCP) Project as: ${projectNumber}"
-  printf '%s' "Press any key to continue after you update the GCP number..."
-  local any
-  read -n1 -s any
-  printf '\n'
-}
-
-#######################################
-# Ask user to grant the access to CF's default service account.
-# Note: the target GCP needs to have OAuth consent screen.
-# Globals:
-#   SHEET_URL
-# Arguments:
-#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
-#######################################
-grant_access_to_service_account() {
-  ((STEP += 1))
-  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
-  local defaultServiceAccount=$(get_cloud_functions_service_account \
-    "${PROJECT_NAMESPACE}_main")
-  local parentId=$(get_value_from_json_file "${apps_script_src}"/.clasp.json \
-    parentId|cut -d\" -f2)
-  printf '%s\n' "Step ${STEP}: Share the Google Sheet with ${SOLUTION_NAME}."
-
-  printf '%s\n' "  Open Google Sheet: \
-https://drive.google.com/open?id=${parentId}"
-  printf '%s\n' "  Click 'Share' and grant the Viewer access to: \
-${defaultServiceAccount}"
-  printf '%s' "Press any key to continue after you grant the access..."
-  local any
-  read -n1 -s any
-  printf '\n'
-}

package/bin/bigquery.sh

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright 2022 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#######################################
-# Checks whether the BigQuery object (table or view) exists.
-# Globals:
-#  GCP_PROJECT
-#  DATASET
-#  BIGQUERY_LOG_TABLE
-# Arguments:
-#  None
-#######################################
-check_existence_in_bigquery() {
-  bq show "${1}" >/dev/null 2>&1
-  printf '%d' $?
-}
-
-#######################################
-# Creates or updates the BigQuery view.
-# Globals:
-#  GCP_PROJECT
-#  DATASET
-# Arguments:
-#  The name of view.
-#  The query of view.
-#######################################
-create_or_update_view() {
-  local viewName viewQuery
-  viewName="${1}"
-  viewQuery=${2}
-  local action="mk"
-  if [[ $(check_existence_in_bigquery "${DATASET}.${viewName}") -eq 0 ]]; then
-    action="update"
-  fi
-  bq "${action}" \
-    --use_legacy_sql=false \
-    --view "${viewQuery}" \
-    --project_id ${GCP_PROJECT} \
-    "${DATASET}.${viewName}"
-}

package/bin/google_ads.sh

@@ -1,115 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright 2022 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Google Ads API version
-GOOGLE_ADS_API_VERSION=16
-
-#######################################
-# Verify whether the current OAuth token, CID and developer token can work.
-# Globals:
-#   None
-# Arguments:
-#   MCC CID
-#   Developer token
-#######################################
-validate_googleads_account() {
-  local cid developerToken accessToken request response
-  cid=${1}
-  developerToken=${2}
-  accessToken=$(get_oauth_access_token)
-  request=(
-    -H "Accept: application/json"
-    -H "Content-Type: application/json"
-    -H "developer-token: ${developerToken}"
-    -H "Authorization: Bearer ${accessToken}"
-    -X POST "https://googleads.googleapis.com/v${GOOGLE_ADS_API_VERSION}/customers/${cid}/googleAds:search"
-    -d '{"query": "SELECT customer.id FROM customer"}'
-  )
-  response=$(curl "${request[@]}" 2>/dev/null)
-  local errorCode errorMessage
-  errorCode=$(get_value_from_json_string "${response}" "error.code")
-  if [[ -n "${errorCode}" ]]; then
-    errorMessage=$(get_value_from_json_string "${response}" "error.message")
-    printf '%s\n' "Validate failed: ${errorMessage}" >&2
-    return 1
-  fi
-  return 0
-}
-
-#######################################
-# Let user input MCC CID and developer token for cronjob(s).
-# Globals:
-#   MCC_CIDS
-#   DEVELOPER_TOKEN
-# Arguments:
-#   None
-#######################################
-set_google_ads_account() {
-  printf '%s\n' "Setting up Google Ads account information..."
-  local developToken mccCids
-  while :; do
-    # Developer token
-    while [[ -z ${developToken} ]]; do
-      printf '%s' "  Enter the developer token[${DEVELOPER_TOKEN}]: "
-      read -r input
-      developToken="${input:-${DEVELOPER_TOKEN}}"
-    done
-    DEVELOPER_TOKEN="${developToken}"
-    mccCids=""
-    # MCC CIDs
-    while :; do
-      printf '%s' "  Enter the MCC CID: "
-      read -r input
-      if [[ -z ${input} ]]; then
-        continue
-      fi
-      input="$(printf '%s' "${input}" | sed -r 's/-//g')"
-      printf '%s' "    validating ${input}...... "
-      validate_googleads_account ${input} ${DEVELOPER_TOKEN}
-      if [[ $? -eq 1 ]]; then
-        printf '%s\n' "failed.
-      Press 'd' to re-enter developer token ["${developToken}"] or
-            'C' to continue with this MCC CID or
-            any other key to enter another MCC CID..."
-        local any
-        read -n1 -s any
-        if [[ "${any}" == "d" ]]; then
-          developToken=""
-          continue 2
-        elif [[ "${any}" == "C" ]]; then
-          printf '%s\n' "WARNING! Continue with FAILED MCC ${input}."
-        else
-          continue
-        fi
-      else
-        printf '%s\n' "succeeded."
-      fi
-      mccCids+=",${input}"
-      printf '%s' "  Do you want to add another MCC CID? [Y/n]: "
-      read -r input
-      if [[ ${input} == 'n' || ${input} == 'N' ]]; then
-        break
-      fi
-    done
-    # Left Shift one position to remove the first comma.
-    # After shifting, MCC_CIDS would like "11111,22222".
-    MCC_CIDS="${mccCids:1}"
-    printf '%s\n' "Using Google Ads MCC CIDs: ${MCC_CIDS}."
-    break
-  done
-}
-
-printf '%s\n' "Google Ads Bash Library is loaded."