@google-cloud/nodejs-common 1.0.5 → 1.2.0

package/README.md

@@ -1,6 +1,6 @@
 # NodeJS Common Library
 
-<!--* freshness: { owner: 'lushu' reviewed: '2021-12-02' } *-->
+<!--* freshness: { owner: 'lushu' reviewed: '2022-04-07' } *-->
 
 A NodeJs common library for other projects, e.g. [GMP and Google Ads Connector]
 and [Data Tasks Coordinator]. This library includes:

package/bin/apps_scripts.sh

@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Const the folder name of Apps Script.
+DEFAULT_APPS_SCRIPT_FOLDER="apps_script"
+
+#######################################
+# Clasp login.
+# Globals:
+#   None
+# Arguments:
+#   None
+#######################################
+clasp_login() {
+  while :; do
+    local claspLogin=$(clasp login --status)
+    if [[ "${claspLogin}" != "You are not logged in." ]]; then
+      printf '%s' "${claspLogin} Would you like to continue with it? [Y/n]"
+      local logout
+      read -r logout
+      logout=${logout:-"Y"}
+      if [[ ${logout} == "Y" || ${logout} == "y" ]]; then
+        break
+      else
+        clasp logout
+      fi
+    fi
+    clasp login --no-localhost
+  done
+}
+
+#######################################
+# Initialize a AppsScript project. Usually it involves following steps:
+# 1. Create a AppsScript project within a new Google Sheet.
+# 2. Prompt to update the Google Cloud Project number of the AppsScript project
+#    to enable external APIs for this AppsScript project.
+# 3. Prompt to grant the access of Cloud Functions' default service account to
+#    this Google Sheet, so the Cloud Functions can query this Sheet later.
+# 4. Initialize the Sheet based on requests.
+# Globals:
+#   None
+# Arguments:
+#   The Google Sheet name.
+#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
+#######################################
+clasp_initialize() {
+  ((STEP += 1))
+  printf '%s\n' "Step ${STEP}: Starting to create Google Sheets..."
+  local sheetName="${1}"
+  local apps_script_src="${2-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
+  clasp_login
+  while :; do
+    local claspStatus=$(
+      clasp status -P "${apps_script_src}" >/dev/null 2>&1
+      echo $?
+    )
+    if [[ $claspStatus -gt 0 ]]; then
+      clasp create --type sheets --title "${sheetName}" --rootDir "${apps_script_src}"
+      local createResult=$?
+      if [[ $createResult -gt 0 ]]; then
+        printf '%s' "Press any key to continue after you enable the Google \
+Apps Script API: https://script.google.com/home/usersettings..."
+        local any
+        read -n1 -s any
+        printf '\n\n'
+        continue
+      fi
+      break
+    else
+      printf '%s' "AppsScript project exists. Would you like to continue with \
+it? [Y/n]"
+      local useCurrent
+      read -r useCurrent
+      useCurrent=${useCurrent:-"Y"}
+      if [[ ${useCurrent} = "Y" || ${useCurrent} = "y" ]]; then
+        break
+      else
+        printf '%s' "Would you like to delete current AppsScript and create a \
+new one? [N/y]"
+        local deleteCurrent
+        read -r deleteCurrent
+        deleteCurrent=${deleteCurrent:-"N"}
+        if [[ ${deleteCurrent} = "Y" || ${deleteCurrent} = "y" ]]; then
+          rm "${apps_script_src}/.clasp.json"
+          continue
+        fi
+      fi
+    fi
+  done
+}
+
+#######################################
+# Copy GCP project configuration file to AppsScript codes as a constant named
+# `GCP_CONFIG`.
+# Globals:
+#   None
+# Arguments:
+#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
+#######################################
+generate_config_js_for_apps_script() {
+  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
+  local generated_file="${apps_script_src}/.generated_config.js"
+  if [[ -f "${CONFIG_FILE}" ]]; then
+    echo '// Copyright 2022 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/** @fileoverview Auto-generated configuration file for Apps Script. */
+'>"${generated_file}"
+    echo -n "const GCP_CONFIG = " >>"${generated_file}"
+    cat "${CONFIG_FILE}" >>"${generated_file}"
+  else
+    printf '%s\n' "Couldn't find ${CONFIG_FILE}."
+  fi
+}
+
+#######################################
+# Clasp pushes AppsScript codes.
+# Globals:
+#   None
+# Arguments:
+#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
+#######################################
+clasp_push_codes() {
+  ((STEP += 1))
+  printf '%s\n' "Step ${STEP}: Starting to push codes to the Google Sheets..."
+  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
+  clasp status -P "${apps_script_src}" >>/dev/null
+  local project_status=$?
+  if [[ ${project_status} -gt 0 ]]; then
+    return ${project_status}
+  else
+    generate_config_js_for_apps_script "${apps_script_src}"
+    clasp push --force -P "${apps_script_src}"
+  fi
+}
+
+#######################################
+# Ask user to update the GCP number of this AppsScript.
+# Globals:
+#   GCP_PROJECT
+# Arguments:
+#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
+#######################################
+clasp_update_project_number() {
+  ((STEP += 1))
+  local projectNumber=$(get_project_number)
+  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
+  printf '%s\n' "Step ${STEP}: Update Google Cloud Platform (GCP) Project for \
+Apps Script."
+  printf '%s' "  "
+  clasp open -P "${apps_script_src}"
+  printf '%s\n' "  On the open tab of Apps Script, use 'Project \
+Settings' to set the Google Cloud Platform (GCP) Project as: ${projectNumber}"
+  printf '%s' "Press any key to continue after you update the GCP number..."
+  local any
+  read -n1 -s any
+  printf '\n'
+}
+
+#######################################
+# Ask user to grant the access to CF's default service account.
+# Note: the target GCP needs to have OAuth consent screen.
+# Globals:
+#   SHEET_URL
+# Arguments:
+#   The folder for Apps Script code, default value ${DEFAULT_APPS_SCRIPT_FOLDER}
+#######################################
+grant_access_to_service_account() {
+  ((STEP += 1))
+  local apps_script_src="${1-"${DEFAULT_APPS_SCRIPT_FOLDER}"}"
+  local defaultServiceAccount=$(get_cloud_functions_service_account \
+    "${PROJECT_NAMESPACE}_main")
+  local parentId=$(get_value_from_json_file "${apps_script_src}"/.clasp.json \
+    parentId|cut -d\" -f2)
+  printf '%s\n' "Step ${STEP}: Share the Google Sheet with ${SOLUTION_NAME}."
+
+  printf '%s\n' "  Open Google Sheet: \
+https://drive.google.com/open?id=${parentId}"
+  printf '%s\n' "  Click 'Share' and grant the Viewer access to: \
+${defaultServiceAccount}"
+  printf '%s' "Press any key to continue after you grant the access..."
+  local any
+  read -n1 -s any
+  printf '\n'
+}

package/bin/bigquery.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#######################################
+# Checks whether the BigQuery object (table or view) exists.
+# Globals:
+#  GCP_PROJECT
+#  DATASET
+#  BIGQUERY_LOG_TABLE
+# Arguments:
+#  None
+#######################################
+check_existence_in_bigquery() {
+  bq show "${1}" >/dev/null 2>&1
+  printf '%d' $?
+}
+
+#######################################
+# Creates or updates the BigQuery view.
+# Globals:
+#  GCP_PROJECT
+#  DATASET
+# Arguments:
+#  The name of view.
+#  The query of view.
+#######################################
+create_or_update_view() {
+  local viewName viewQuery
+  viewName="${1}"
+  viewQuery=${2}
+  local action="mk"
+  if [[ $(check_existence_in_bigquery "${DATASET}.${viewName}") -eq 0 ]]; then
+    action="update"
+  fi
+  bq "${action}" \
+    --use_legacy_sql=false \
+    --view "${viewQuery}" \
+    --project_id ${GCP_PROJECT} \
+    "${DATASET}.${viewName}"
+}

package/bin/google_ads.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Google Ads API version
+GOOGLE_ADS_API_VERSION=10
+
+#######################################
+# Verify whether the current OAuth token, CID and developer token can work.
+# Globals:
+#   None
+# Arguments:
+#   MCC CID
+#   Developer token
+#######################################
+validate_googleads_account() {
+  local cid developerToken accessToken request response
+  cid=${1}
+  developerToken=${2}
+  accessToken=$(get_oauth_access_token)
+  request=(
+    -H "Accept: application/json"
+    -H "Content-Type: application/json"
+    -H "developer-token: ${developerToken}"
+    -H "Authorization: Bearer ${accessToken}"
+    -X POST "https://googleads.googleapis.com/v${GOOGLE_ADS_API_VERSION}/customers/${cid}/googleAds:search"
+    -d '{"query": "SELECT customer.id FROM customer"}'
+  )
+  response=$(curl "${request[@]}" 2>/dev/null)
+  local errorCode errorMessage
+  errorCode=$(get_value_from_json_string "${response}" "error.code")
+  if [[ -n "${errorCode}" ]]; then
+    errorMessage=$(get_value_from_json_string "${response}" "error.message")
+    printf '%s\n' "Validate failed: ${errorMessage}" >&2
+    return 1
+  fi
+  return 0
+}
+
+#######################################
+# Let user input MCC CID and developer token for cronjob(s).
+# Globals:
+#   MCC_CIDS
+#   DEVELOPER_TOKEN
+# Arguments:
+#   None
+#######################################
+set_google_ads_account() {
+  printf '%s\n' "Setting up Google Ads account information..."
+  local developToken mccCids
+  while :; do
+    # Developer token
+    while [[ -z ${developToken} ]]; do
+      printf '%s' "  Enter the developer token[${DEVELOPER_TOKEN}]: "
+      read -r input
+      developToken="${input:-${DEVELOPER_TOKEN}}"
+    done
+    DEVELOPER_TOKEN="${developToken}"
+    mccCids=""
+    # MCC CIDs
+    while :; do
+      printf '%s' "  Enter the MCC CID: "
+      read -r input
+      if [[ -z ${input} ]]; then
+        continue
+      fi
+      input="$(printf '%s' "${input}" | sed -r 's/-//g')"
+      printf '%s' "    validating ${input}...... "
+      validate_googleads_account ${input} ${DEVELOPER_TOKEN}
+      if [[ $? -eq 1 ]]; then
+        printf '%s\n' "failed.
+      Press 'd' to re-enter developer token ["${developToken}"] or
+            'C' to continue with this MCC CID or
+            any other key to enter another MCC CID..."
+        local any
+        read -n1 -s any
+        if [[ "${any}" == "d" ]]; then
+          developToken=""
+          continue 2
+        elif [[ "${any}" == "C" ]]; then
+          printf '%s\n' "WARNING! Continue with FAILED MCC ${input}."
+        else
+          continue
+        fi
+      else
+        printf '%s\n' "succeeded."
+      fi
+      mccCids+=",${input}"
+      printf '%s' "  Do you want to add another MCC CID? [Y/n]: "
+      read -r input
+      if [[ ${input} == 'n' || ${input} == 'N' ]]; then
+        break
+      fi
+    done
+    # Left Shift one position to remove the first comma.
+    # After shifting, MCC_CIDS would like "11111,22222".
+    MCC_CIDS="${mccCids:1}"
+    printf '%s\n' "Using Google Ads MCC CIDs: ${MCC_CIDS}."
+    break
+  done
+}
+
+printf '%s\n' "Google Ads Bash Library is loaded."

package/bin/install_functions.sh

@@ -573,7 +573,7 @@ API(s):"
     printf '%s\n'  "OK. OAuth is required by selected API(s)."
     return 0
   fi
-  printf '%s\n' "Selected API(s) require authentication. Please select \
+  printf '%s\n' "Selected API(s) require authentication. Choose the \
 authentication method:"
   local auths=("Service Account (recommended)" "OAuth")
   select auth in "${auths[@]}"; do
@@ -604,28 +604,40 @@ check_permissions() {
   (( STEP += 1 ))
   printf '%s\n' "Step ${STEP}: Checking current user's access..."
   local role error
-  for role in "${!GOOGLE_CLOUD_PERMISSIONS[@]}"; do
-    printf '%s'  "  Checking permissions for ${role}... "
-    local permissions
-    permissions=(${GOOGLE_CLOUD_PERMISSIONS[${role}]})
-    local missed
-    missed=$(get_number_of_missed_permissions "${permissions[@]}")
-    if [[ $missed -gt 0 ]]; then
-      message="missed ${missed}, failed"
-      error=1
+  while :; do
+    error=0
+    for role in "${!GOOGLE_CLOUD_PERMISSIONS[@]}"; do
+      printf '%s'  "  Checking permissions for ${role}... "
+      local permissions
+      permissions=(${GOOGLE_CLOUD_PERMISSIONS[${role}]})
+      local missed
+      missed=$(get_number_of_missed_permissions "${permissions[@]}")
+      if [[ $missed -gt 0 ]]; then
+        message="missed ${missed}, failed"
+        error=1
+      else
+        message='successfully'
+      fi
+      printf '%s\n' " ${message}."
+    done
+    if [[ ${error} -gt 0 ]]; then
+      printf '%s\n' "Permissions check failed."
+      printf '%s\n' "Would you like to login with another account? [Y/n]: "
+      local reLogin
+      read -r reLogin
+      reLogin=${reLogin:-"Y"}
+      if [[ ${reLogin} == "Y" || ${reLogin} == "y" ]]; then
+        gcloud auth login
+        continue
+      else
+        return 1
+      fi
     else
-      message='successfully'
+      echo "OK. Permissions check passed for Google Cloud project \
+  [${GCP_PROJECT}]."
+      return 0
     fi
-    printf '%s\n' " ${message}."
   done
-  if [[ ${error} -gt 0 ]]; then
-    printf '%s\n' "Permissions check failed."
-    return 1
-  else
-    echo "OK. Permissions check passed for Google Cloud project \
-[${GCP_PROJECT}]."
-    return 0
-  fi
 }
 
 #######################################
@@ -1031,13 +1043,24 @@ refresh the list:"
 # Globals:
 #   GCP_PROJECT
 # Arguments:
-#   Bucket name var, default value 'GCS_BUCKET'
+#   Bucket name var with optional usage, e.g. 'GCS_BUCKET_REPORT:reports'.
+#     The default value is 'GCS_BUCKET'.
 #   Location var name, default value 'REGION'
 #   If the second location var is unset, use this var as default value
 #######################################
 confirm_located_bucket() {
-  local gcsName defaultValue defaultBucketName locationName location
-  gcsName="${1:-"GCS_BUCKET"}"
+  local gcsName usage defaultValue defaultBucketName locationName location
+  if [[ -z "${1}" ]]; then
+    gcsName="GCS_BUCKET"
+  elif [[ "${1}" == *":"* ]]; then
+    gcsName=$(echo "${1}" | cut -d\: -f1)
+    usage=$(echo "${1}" | cut -d\: -f2)
+    if [[ -n "${usage}" ]]; then
+      usage=" for ${usage}"
+    fi
+  else
+    gcsName="${1}"
+  fi
   locationName="${2:-"REGION"}"
   defaultValue="${!gcsName}"
   defaultBucketName=$(get_default_bucket_name "${GCP_PROJECT}")
@@ -1052,7 +1075,8 @@ confirm_located_bucket() {
 
   (( STEP += 1 ))
   if [[ -z "${location}" ]]; then
-    printf '%s\n' "Step ${STEP}: Checking or creating a Cloud Storage Bucket..."
+    printf '%s\n' "Step ${STEP}: Checking or creating a Cloud Storage \
+Bucket${usage}..."
     if [[ "${locationName}" == "REGION" ]]; then
       select_functions_location ${locationName}
     else
@@ -1060,8 +1084,8 @@ confirm_located_bucket() {
     fi
     location="${!locationName}"
   else
-    printf '%s\n' "Step ${STEP}: Checking or creating a Cloud Storage Bucket in \
-location [${location}] ..."
+    printf '%s\n' "Step ${STEP}: Checking or creating a Cloud Storage \
+Bucket${usage} in location [${location}] ..."
   fi
   declare -g "${locationName}=${location}"
   while :; do
@@ -1129,6 +1153,7 @@ Project. Continuing to enter another bucket..."
     fi
   done
   declare -g "${gcsName}=${bucket}"
+  confirm_bucket_lifecycle "${bucket}"
 }
 
 #######################################
@@ -1149,6 +1174,58 @@ confirm_bucket_with_location() {
   confirm_located_bucket ${gcsName} ${locationName}
 }
 
+#######################################
+# Manage the lifecycle of a GCS bucket: setting or removing the GCS lifecycle
+# rule of 'age'.
+# See: https://cloud.google.com/storage/docs/lifecycle#age
+# Arguments:
+#   Bucket name
+#######################################
+confirm_bucket_lifecycle() {
+  local bucket bucketMetadata lifecycle
+  bucket="${1}"
+  bucketMetadata="$(get_bucket_metadata "${bucket}")"
+  lifecycle="$(get_value_from_json_string "${bucketMetadata}" "lifecycle")"
+  if [[ -n "${lifecycle}" ]]; then
+    printf '%s\n' "There are lifecycle rules in this bucket: ${lifecycle}."
+    printf '%s' "Would you like to overwrite it? [N/y]:"
+    local confirmContinue
+    read -r confirmContinue
+    confirmContinue=${confirmContinue:-"N"}
+    if [[ ${confirmContinue} == "N" || ${confirmContinue} == "n" ]]; then
+      return 0
+    fi
+  else
+    printf '%s\n' "There is no lifecycle rules in this bucket."
+    printf '%s' "Would you like to create it? [Y/n]:"
+    local confirmContinue
+    read -r confirmContinue
+    confirmContinue=${confirmContinue:-"Y"}
+    if [[ ${confirmContinue} == "N" || ${confirmContinue} == "n" ]]; then
+      return 0
+    fi
+  fi
+  while :; do
+    printf '%s' "Enter the number of days that a file will be kept before it \
+is automatically removed in the bucket[${bucket}]. (enter 0 to remove all \
+existing lifecycle rules): "
+    local days
+    read -r days
+    days=${days}
+    if [[ "${days}" =~ ^[0-9]+$ ]]; then
+      local lifecycle
+      if [[ "${days}" == "0" ]]; then
+        lifecycle="{}"
+      else
+        lifecycle='{"rule":[{"action":{"type":"Delete"},"condition":{"age":'\
+"${days}}}]}"
+      fi
+      gsutil lifecycle set /dev/stdin gs://${bucket} <<< ${lifecycle}
+      return $?
+    fi
+  done
+}
+
 #######################################
 # Confirm the monitored folder.
 # Globals:
@@ -1409,20 +1486,18 @@ EOF
 "client_id=${client_id}" "scope=${scope}")
     local auth_url
     auth_url="${OAUTH_BASE_URL}auth?${parameters}"
-    printf '%s\n' "3. Open the link in browser and finish authentication: \
-${auth_url}"
+    printf '%s\n' "3. Open the link in your browser and finish authentication. \
+    Do not close the redirected page: ${auth_url}"
     cat <<EOF
   Note:
+    The succeeded OAuth flow will land the browser on an error page - \
+"This site can't be reached". This is expected behavior. Copy the whole URL and continue.
     If the OAuth client is not for a native application, there will be an \
 "Error 400: redirect_uri_mismatch" shown up on the page. In this case, press \
 "Enter" to start again with a native application OAuth client ID.
-    If there is no local web server serving at ${REDIRECT_URI}, the \
-succeeded OAuth flow will land the browser on an error page ("This site can't \
-be reached"). This is an expected behavior. Copy the whole URL and continue.
 
 EOF
-    printf '%s' "4. Copy the authorization code or complete url from browser \
-and paste here: "
+    printf '%s' "4. Copy the complete URL from your browser and paste here: "
     read -r auth_code
     if [[ -z ${auth_code} ]]; then
       printf '%s\n\n' "No authorization code. Starting from beginning again..."
@@ -1507,6 +1582,7 @@ set_authentication_env_for_cloud_functions() {
 create_or_update_cloud_scheduler_for_pubsub(){
   check_authentication
   quit_if_failed $?
+  local location_flag="--location=${REGION}"
   local scheduler_flag=()
   scheduler_flag+=(--schedule="$2")
   scheduler_flag+=(--time-zone="$3")
@@ -1514,13 +1590,13 @@ create_or_update_cloud_scheduler_for_pubsub(){
   scheduler_flag+=(--message-body="$5")
   local exist_job
   exist_job=($(gcloud scheduler jobs list --filter="name~${1}" \
---format="value(state)"))
+--format="value(state)" "${location_flag}"))
   local action needPause
   if [[ ${#exist_job[@]} -gt 0 ]]; then
     action="update"
     scheduler_flag+=(--update-attributes=$6)
     if [[ "${exist_job[0]}" == "PAUSED" ]]; then
-      gcloud scheduler jobs resume "${1}"
+      gcloud scheduler jobs resume "${1}" "${location_flag}"
       if [[ $? -gt 0 ]]; then
         printf '%s\n' "Failed to resume paused Cloud Scheduler job [${1}]."
         return 1
@@ -1531,9 +1607,9 @@ create_or_update_cloud_scheduler_for_pubsub(){
     action="create"
     scheduler_flag+=(--attributes=$6)
   fi
-  gcloud scheduler jobs ${action} pubsub "$1" "${scheduler_flag[@]}"
+  gcloud scheduler jobs ${action} pubsub "$1" "${scheduler_flag[@]}" "${location_flag}"
   if [[ "${needPause}" == "true" ]]; then
-      gcloud scheduler jobs pause "${1}"
+      gcloud scheduler jobs pause "${1}" "${location_flag}"
   fi
 }
 
@@ -1613,7 +1689,9 @@ customized_install() {
   local tasks=("$@")
   local task
   for task in "${tasks[@]}"; do
-    ${task}
+    local cmd
+    eval "cmd=(${task})"
+    "${cmd[@]}"
     quit_if_failed $?
   done
 }
@@ -1921,3 +1999,11 @@ join_string_array() {
   shift
   printf %s "$first" "${@/#/$separator}"
 }
+
+# Import other bash files.
+_SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${_SELF}/google_ads.sh"
+source "${_SELF}/bigquery.sh"
+source "${_SELF}/apps_scripts.sh"
+
+printf '%s\n' "Common Bash Library is loaded."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "1.0.5",
+  "version": "1.2.0",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -16,25 +16,25 @@
   },
   "homepage": "https://github.com/GoogleCloudPlatform/cloud-for-marketing/blob/master/marketing-analytics/activation/common-libs/nodejs-common/README.md",
   "dependencies": {
-    "@google-cloud/aiplatform": "^1.13.0",
-    "@google-cloud/automl": "^2.5.1",
-    "@google-cloud/bigquery": "^5.9.2",
+    "@google-cloud/aiplatform": "^1.19.0",
+    "@google-cloud/automl": "^2.5.2",
+    "@google-cloud/bigquery": "^6.0.0",
     "@google-cloud/datastore": "^6.6.2",
-    "@google-cloud/firestore": "^4.15.1",
-    "@google-cloud/logging-winston": "^4.1.1",
-    "@google-cloud/pubsub": "^2.18.2",
-    "@google-cloud/storage": "^5.16.0",
-    "gaxios": "^4.3.2",
-    "google-ads-api": "^9.0.0",
-    "google-ads-node": "^7.0.0",
-    "google-auth-library": "^7.10.2",
-    "googleapis": "^91.0.0",
-    "soap": "^0.43.0",
-    "winston": "^3.3.3",
+    "@google-cloud/firestore": "^5.0.2",
+    "@google-cloud/logging-winston": "^5.1.0",
+    "@google-cloud/pubsub": "^3.0.1",
+    "@google-cloud/storage": "^6.0.1",
+    "@google-cloud/scheduler": "^3.0.0",
+    "gaxios": "^5.0.0",
+    "google-ads-api": "^10.0.1",
+    "google-ads-node":"^8.0.1",
+    "google-auth-library": "^8.0.2",
+    "googleapis": "^100.0.0",
+    "winston": "^3.7.2",
     "lodash": "^4.17.21"
   },
   "devDependencies": {
-    "jasmine": "^3.5.0"
+    "jasmine": "^4.1.0"
   },
   "scripts": {
     "test": "node node_modules/jasmine/bin/jasmine"

package/src/components/automl.js

@@ -17,15 +17,9 @@
  */
 
 'use strict';
-const {request} = require('gaxios');
-const {JWT, Compute} = require('google-auth-library');
+const { ClientOptions } = require('google-gax');
 const automl = require('@google-cloud/automl');
 const google = automl.protos.google;
-const AuthClient = require('../apis/auth_client.js');
-
-const API_SCOPES = Object.freeze([
-  'https://www.googleapis.com/auth/cloud-platform',
-]);
 
 /**
  * For version `v1beta1`, BigQuery can be the source and destination.
@@ -40,46 +34,49 @@ const API_SCOPES = Object.freeze([
  * @type {string}
  */
 const API_VERSION = 'v1beta1';
-const PredictionServiceClient = automl[API_VERSION].PredictionServiceClient;
+const { PredictionServiceClient } = automl[API_VERSION];
 
 /**
- * AutoML Tables API doesn't have fully cloud client library support yet. This
- * class acts as the single client class for different API requests of AutoML
- * Tables API, including:
- * 1. 'batch predict' based on Google Cloud Client Library,
- * https://googleapis.dev/nodejs/automl/latest/index.html
- * 2. 'get Operations' based on REST API, see:
- * https://cloud.google.com/automl/docs/reference/rest/v1/projects.locations.operations/get
- * https://cloud.google.com/automl-tables/docs/long-operations#get-operation
+ * AutoML Tables API class for:
+ * 1. start a batch predict job:
+ * @see https://cloud.google.com/nodejs/docs/reference/automl/latest/automl/v1beta1.predictionserviceclient#_google_cloud_automl_v1beta1_PredictionServiceClient_batchPredict_member_3_
+ * 2. get the status of an operation:
+ * @see https://cloud.google.com/nodejs/docs/reference/automl/latest/automl/v1beta1.predictionserviceclient#_google_cloud_automl_v1beta1_PredictionServiceClient_checkBatchPredictProgress_member_1_
  */
 class AutoMl {
-
   /**
    * Initialize an instance.
-   * @param {{keyFilename:(string|undefined)}} options
+   * @param {ClientOptions=} options
    */
   constructor(options = {}) {
-    this.options = options;
+    this.client = new PredictionServiceClient(options);
   }
 
   /**
    * Batch predicts based on Google Cloud Client Library.
-   * See https://googleapis.dev/nodejs/automl/latest/index.html
    * @param {string} projectId
    * @param {string} computeRegion
    * @param {string} modelId
-   * @param {google.cloud.automl.v1.IBatchPredictInputConfig} inputConfig
-   * @param {google.cloud.automl.v1.IBatchPredictOutputConfig} outputConfig
+   * @param {google.cloud.automl.v1beta1.IBatchPredictInputConfig} inputConfig
+   * @param {google.cloud.automl.v1beta1.IBatchPredictOutputConfig} outputConfig
    * @return {Promise<string>} Predict operation name.
    */
-  async batchPredict(projectId, computeRegion, modelId, inputConfig,
-      outputConfig) {
-    const client = new PredictionServiceClient(this.options);
-    const modelFullId = client.modelPath(projectId, computeRegion, modelId);
-    const responses = await client.batchPredict({
+  async batchPredict(
+    projectId,
+    computeRegion,
+    modelId,
+    inputConfig,
+    outputConfig
+  ) {
+    const modelFullId = this.client.modelPath(
+      projectId,
+      computeRegion,
+      modelId
+    );
+    const responses = await this.client.batchPredict({
       name: modelFullId,
-      inputConfig: inputConfig,
-      outputConfig: outputConfig,
+      inputConfig,
+      outputConfig,
     });
     const operation = responses[1];
     console.log(`Operation name: ${operation.name}`);
@@ -87,37 +84,20 @@ class AutoMl {
   }
 
   /**
-   * Gets Operations based on REST API, see:
-   * https://cloud.google.com/automl/docs/reference/rest/v1/projects.locations.operations/get
-   * https://cloud.google.com/automl-tables/docs/long-operations#get-operation
+   * Gets status of an operation.
    * @param {string} operationName
-   * @return {Promise<google.longrunning.Operation>}
+   * @return {Promise<{{
+   *   done: boolean,
+   *   error: Error|undefined,
+   *   metadata: OperationMetadata,
+   *   name: string,
+   * }}>}
    */
   async getOperation(operationName) {
-    const url = `https://automl.googleapis.com/${API_VERSION}/${operationName}`;
-    const headers = await this.getAuthClient_().getRequestHeaders();
-    const requestOptions = {
-      method: 'GET',
-      headers,
-      url,
-    };
-    const response = await request(requestOptions);
-    return response.data;
+    const response = await this.client.checkBatchPredictProgress(operationName);
+    const { done, error, metadata, name } = response;
+    return { done, error, metadata, name };
   }
-
-  /**
-   * Gets authentication client of AutoML API.
-   * This API belongs to GCP, however it is not fully supported by Google Cloud
-   * Client Library. So we need to manage some functions on its REST API,
-   * in the way like we invoke other external APIs.
-   * @returns {(!JWT|!Compute)}
-   * @private
-   */
-  getAuthClient_() {
-    /** @const {!AuthClient} */ const authClient = new AuthClient(API_SCOPES);
-    return authClient.getApplicationDefaultCredentials();
-  }
-
 }
 
 exports.AutoMl = AutoMl;

package/src/components/scheduler.js

@@ -30,11 +30,36 @@ const API_VERSION = 'v1';
  * to get/pause/resume a job.
  */
 class CloudScheduler {
-  constructor(projectId = process.env['GCP_PROJECT']) {
-    /** @const {!AuthClient} */
-    const authClient = new AuthClient(API_SCOPES);
-    this.auth = authClient.getApplicationDefaultCredentials();
-    this.projectId = projectId;
+  constructor(env = process.env, options = {}) {
+    if (!options.authClient) {
+      /** @const {!AuthClient} */
+      const authClient = new AuthClient(API_SCOPES, env);
+      /**
+       * By default, `AuthClient` (getDefaultAuth()) will return an auth client
+       * based on the settings in ENV while the OAuth is the most preferred.
+       * This works for most of the external API clients (in the '../apis'
+       * folder), however this won't work in the Cloud Functions, as those OAuth
+       * token usually won't have enough permission to invoke Google Cloud API.
+       * Using the method `getApplicationDefaultCredentials` to force
+       * `AuthClient` return an ADC auth client, which will work in the Cloud.
+       *
+       * Cloud Scheduler API Client Library is used here as Cloud Client Library
+       * is still at beta stage. (For the difference, see
+       * https://cloud.google.com/apis/docs/client-libraries-explained)
+       * Eventually, when we migrate this to the cloud client library, which
+       * automatically takes ADC as the authentication method, the 'AuthClient'
+       * is not required here and can be removed.
+       */
+      this.auth = authClient.getApplicationDefaultCredentials();
+    } else {
+      /**
+       * `authClient` can be consumed by cloud client library as the auth
+       * client. By passing this in, we can offer more flexible auth clients in
+       * test cases for API client library and cloud client library in future.
+       */
+      this.auth = options.authClient;
+    }
+    this.projectId = env['GCP_PROJECT'];
     this.instance = cloudscheduler({
       version: API_VERSION,
       auth: this.auth,
@@ -113,8 +138,8 @@ class CloudScheduler {
    */
   async getJobs_(name, targetLocations = undefined) {
     const regex = new RegExp(`/jobs/${name}$`);
-    const jobs = (await this.listJobs_(targetLocations)).filter(
-        (job) => regex.test(job));
+    const allJobs = await this.listJobs_(targetLocations);
+    const jobs = allJobs.filter((job) => regex.test(job));
     if (jobs.length === 0) console.error(`Can not find job: ${name}`);
     return jobs;
   }
@@ -134,23 +159,10 @@ class CloudScheduler {
     const projectId = await this.getProjectId_();
     const requestPrefix = `projects/${projectId}/locations`;
     const jobs = locations.map(async (location) => {
-      const request = {parent: `${requestPrefix}/${location}`};
-      try {
-        const response = await this.instance.projects.locations.jobs.list(
-            request);
-        return response.data.jobs.map((job) => job.name);
-      } catch (error) {
-        // Currently, listLocations always returns an array with one location.
-        // Not sure whether this will be changed or not in future. If one day
-        // the Cloud Scheduler let users to select a location for a job, then
-        // it may return multiple locations when listLocation, however the
-        // target job may exist in one of the locations. In this case, when we
-        // iterate the job with all possible locations to get the complete job
-        // path to operate, it will likely generate an error for those wrong
-        // location(s). So set the try-catch here to handle this situation.
-        console.error(error.message);
-        return [];
-      }
+      const request = { parent: `${requestPrefix}/${location}` };
+      const response = await this.instance.projects.locations.jobs.list(request);
+      if (!response.data.jobs) return [];
+      return response.data.jobs.map((job) => job.name);
     });
     // Waits for all jobs names and flattens nested job name arrays, however
     // there is no 'flat' available in current Cloud Functions runtime.

package/src/components/vertex_ai.js

@@ -18,6 +18,7 @@
 
 'use strict';
 
+const { ClientOptions } = require('google-gax');
 const {JobServiceClient} = require('@google-cloud/aiplatform');
 
 /**
@@ -29,7 +30,7 @@ class VertexAi {
 
   /**
    * Initialize an instance.
-   * @param {{keyFilename:(string|undefined)}} options
+   * @param {ClientOptions=} options
    */
   constructor(options = {}) {
     this.options = options;
@@ -105,9 +106,9 @@ class VertexAi {
    * @private
    */
   getJobServiceClient_(location) {
-    const clientOptions = {
+    const clientOptions = Object.assign({}, this.options, {
       apiEndpoint: this.getServiceEndpoint_(location),
-    };
+    });
     return new JobServiceClient(clientOptions);
   }
 }