@google-cloud/nodejs-common 1.0.4 → 1.1.1-beta

package/README.md

@@ -1,6 +1,6 @@
 # NodeJS Common Library
 
-<!--* freshness: { owner: 'lushu' reviewed: '2021-12-02' } *-->
+<!--* freshness: { owner: 'lushu' reviewed: '2022-04-07' } *-->
 
 A NodeJs common library for other projects, e.g. [GMP and Google Ads Connector]
 and [Data Tasks Coordinator]. This library includes:
@@ -11,68 +11,65 @@ and [Data Tasks Coordinator]. This library includes:
 1. Wrapper for some Google APIs for integration, mainly
    for [GMP and Google Ads Connector]:
 
-    * Google Analytics data import
-    * Google Analytics measurement protocol
-    * Campaign Manager offline conversion upload
-    * Search Ads 360 conversions upload
-    * Google Ads click conversions upload
-    * Google Ads customer match upload
-    * Google Ads conversions scheduled uploads based on Google Sheets
-    * Measurement Protocol Google Analytics 4
+   - Google Analytics data import
+   - Google Analytics measurement protocol
+   - Campaign Manager offline conversion upload
+   - Search Ads 360 conversions upload
+   - Google Ads click conversions upload
+   - Google Ads customer match upload
+   - Google Ads enhanced conversions upload
+   - Google Ads conversions scheduled uploads based on Google Sheets
+   - Measurement Protocol Google Analytics 4
 
 1. Wrapper for some Google APIs for reporting, mainly
    for [Data Tasks Coordinator]:
 
-    * Google Ads reporting
-    * Campaign Manager reporting
-    * Search Ads 360 reporting
-    * Display and Video 360 reporting
-    * YouTube Data API
-    * Ads Data Hub querying
+   - Google Ads reporting
+   - Campaign Manager reporting
+   - Search Ads 360 reporting
+   - Display and Video 360 reporting
+   - YouTube Data API
+   - Ads Data Hub querying
 
 1. Utilities wrapper class for Google Cloud Products:
 
-    * **Firestore Access Object**: Firestore has two modes which are excluded to
-      each other and can't be changed once selected in a Cloud Project[[2]].
-      This class, with its two successors offer a unified interface to operate
-      data objects on either Firestore or Datastore.
+   - **Firestore Access Object**: Firestore has two modes which are excluded to
+     each other and can't be changed once selected in a Cloud Project[[2]].
+     This class, with its two successors offer a unified interface to operate
+     data objects on either Firestore or Datastore.
 
-    * **AutoMl Tables API**: Offers a unified entry to use this API based on
-      Google Cloud client library combined with REST requests to service
-      directly due to some functionalities missed in the client library.
+   - **AutoMl Tables API**: Offers a unified entry to use this API based on
+     Google Cloud client library combined with REST requests to service
+     directly due to some functionalities missed in the client library.
 
-    * **Vertex AI API**: Offers a unified entry to use this API based on Google
-      Cloud client library.
+   - **Vertex AI API**: Offers a unified entry to use this API based on Google
+     Cloud client library.
 
-    * **Pub/Sub Utilities**: Offers utilities functions to create topics and
-      subscriptions for Pub/Sub, as well as the convenient way to publish a
-      message.
+   - **Pub/Sub Utilities**: Offers utilities functions to create topics and
+     subscriptions for Pub/Sub, as well as the convenient way to publish a
+     message.
 
-    * **Storage Utilities**: Offers functions to manipulate the files on Cloud
-      Storage. The main functions are:
+   - **Storage Utilities**: Offers functions to manipulate the files on Cloud
+     Storage. The main functions are:
 
-        * Reading a given length (or slightly less) content without breaking a
-          line;
-        * Splitting a file into multiple files with the given length (or
-          slightly less) without breaking a line;
-        * Merging files into one file.
+     - Reading a given length (or slightly less) content without breaking a
+       line;
+     - Splitting a file into multiple files with the given length (or
+       slightly less) without breaking a line;
+     - Merging files into one file.
 
-    * **Cloud Scheduler Adapter**: A wrapper to pause and resume Cloud Scheduler
-      jobs.
+   - **Cloud Scheduler Adapter**: A wrapper to pause and resume Cloud Scheduler
+     jobs.
 
-    * **Cloud Functions Adapter**: Cloud Functions have different parameters in
-      different environments, e.g. Node6 vs Node8[[1]]. This utility file offers
-      an adapter to wrap a Node8 Cloud Functions into Node6 and Node8 compatible
-      functions.
+   - **Cloud Functions Adapter**: Cloud Functions have different parameters in
+     different environments, e.g. Node6 vs Node8[[1]]. This utility file offers
+     an adapter to wrap a Node8 Cloud Functions into Node6 and Node8 compatible
+     functions.
 
 1. A share library for [Bash] to facilitate installation tasks.
 
-[GMP and Google Ads Connector]:https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/gmp-googleads-connector
-
-[Data Tasks Coordinator]:https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/data-tasks-coordinator
-
-[1]:https://cloud.google.com/functions/docs/writing/background#functions-writing-background-hello-pubsub-node8-10
-
-[2]:https://cloud.google.com/datastore/docs/concepts/overview#comparison_with_traditional_databases
-
-[Bash]:https://www.gnu.org/software/bash/
+[gmp and google ads connector]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/gmp-googleads-connector
+[data tasks coordinator]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/data-tasks-coordinator
+[1]: https://cloud.google.com/functions/docs/writing/background#functions-writing-background-hello-pubsub-node8-10
+[2]: https://cloud.google.com/datastore/docs/concepts/overview#comparison_with_traditional_databases
+[bash]: https://www.gnu.org/software/bash/

package/bin/bigquery.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#######################################
+# Checks whether the BigQuery object (table or view) exists.
+# Globals:
+#  GCP_PROJECT
+#  DATASET
+#  BIGQUERY_LOG_TABLE
+# Arguments:
+#  None
+#######################################
+check_existence_in_bigquery() {
+  bq show "${1}" >/dev/null 2>&1
+  printf '%d' $?
+}
+
+#######################################
+# Creates or updates the BigQuery view.
+# Globals:
+#  GCP_PROJECT
+#  DATASET
+# Arguments:
+#  The name of view.
+#  The query of view.
+#######################################
+create_or_update_view() {
+  local viewName viewQuery
+  viewName="${1}"
+  viewQuery=${2}
+  local action="mk"
+  if [[ $(check_existence_in_bigquery "${DATASET}.${viewName}") -eq 0 ]]; then
+    action="update"
+  fi
+  bq "${action}" \
+    --use_legacy_sql=false \
+    --view "${viewQuery}" \
+    --project_id ${GCP_PROJECT} \
+    "${DATASET}.${viewName}"
+}

package/bin/google_ads.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Google Ads API version
+GOOGLE_ADS_API_VERSION=10
+
+#######################################
+# Verify whether the current OAuth token, CID and developer token can work.
+# Globals:
+#   None
+# Arguments:
+#   MCC CID
+#   Developer token
+#######################################
+validate_googleads_account() {
+  local cid developerToken accessToken request response
+  cid=${1}
+  developerToken=${2}
+  accessToken=$(get_oauth_access_token)
+  request=(
+    -H "Accept: application/json"
+    -H "Content-Type: application/json"
+    -H "developer-token: ${developerToken}"
+    -H "Authorization: Bearer ${accessToken}"
+    -X POST "https://googleads.googleapis.com/v${GOOGLE_ADS_API_VERSION}/customers/${cid}/googleAds:search"
+    -d '{"query": "SELECT customer.id FROM customer"}'
+  )
+  response=$(curl "${request[@]}" 2>/dev/null)
+  local errorCode errorMessage
+  errorCode=$(get_value_from_json_string "${response}" "error.code")
+  if [[ -n "${errorCode}" ]]; then
+    errorMessage=$(get_value_from_json_string "${response}" "error.message")
+    printf '%s\n' "Validate failed: ${errorMessage}" >&2
+    return 1
+  fi
+  return 0
+}
+
+#######################################
+# Let user input MCC CID and developer token for cronjob(s).
+# Globals:
+#   MCC_CIDS
+#   DEVELOPER_TOKEN
+# Arguments:
+#   None
+#######################################
+set_google_ads_account() {
+  printf '%s\n' "Setting up Google Ads account information..."
+  local developToken mccCids
+  while :; do
+    # Developer token
+    while [[ -z ${developToken} ]]; do
+      printf '%s' "  Enter the developer token[${DEVELOPER_TOKEN}]: "
+      read -r input
+      developToken="${input:-${DEVELOPER_TOKEN}}"
+    done
+    DEVELOPER_TOKEN="${developToken}"
+    mccCids=""
+    # MCC CIDs
+    while :; do
+      printf '%s' "  Enter the MCC CID: "
+      read -r input
+      if [[ -z ${input} ]]; then
+        continue
+      fi
+      input="$(printf '%s' "${input}" | sed -r 's/-//g')"
+      printf '%s' "    validating ${input}...... "
+      validate_googleads_account ${input} ${DEVELOPER_TOKEN}
+      if [[ $? -eq 1 ]]; then
+        printf '%s\n' "failed.
+      Press 'd' to re-enter developer token ["${developToken}"] or
+            'C' to continue with this MCC CID or
+            any other key to enter another MCC CID..."
+        local any
+        read -n1 -s any
+        if [[ "${any}" == "d" ]]; then
+          developToken=""
+          continue 2
+        elif [[ "${any}" == "C" ]]; then
+          printf '%s\n' "WARNING! Continue with FAILED MCC ${input}."
+        else
+          continue
+        fi
+      else
+        printf '%s\n' "succeeded."
+      fi
+      mccCids+=",${input}"
+      printf '%s' "  Do you want to add another MCC CID? [Y/n]: "
+      read -r input
+      if [[ ${input} == 'n' || ${input} == 'N' ]]; then
+        break
+      fi
+    done
+    # Left Shift one position to remove the first comma.
+    # After shifting, MCC_CIDS would like "11111,22222".
+    MCC_CIDS="${mccCids:1}"
+    printf '%s\n' "Using Google Ads MCC CIDs: ${MCC_CIDS}."
+    break
+  done
+}
+
+printf '%s\n' "Google Ads Bash Library is loaded."

package/bin/install_functions.sh

@@ -173,7 +173,7 @@ project ID: [${current_project}]"
         project=${project:-"${current_project}"}
         target_project=${project}
       fi
-      if [[ -n ${current_project} && \
+      if [[ -z ${current_project} || \
 ${current_project} != "${target_project}" ]];then
         printf '%s' "Setting the Google Cloud project to [${target_project}]..."
         gcloud config set project "${target_project}"
@@ -573,7 +573,7 @@ API(s):"
     printf '%s\n'  "OK. OAuth is required by selected API(s)."
     return 0
   fi
-  printf '%s\n' "Selected API(s) require authentication. Please select \
+  printf '%s\n' "Selected API(s) require authentication. Choose the \
 authentication method:"
   local auths=("Service Account (recommended)" "OAuth")
   select auth in "${auths[@]}"; do
@@ -604,28 +604,40 @@ check_permissions() {
   (( STEP += 1 ))
   printf '%s\n' "Step ${STEP}: Checking current user's access..."
   local role error
-  for role in "${!GOOGLE_CLOUD_PERMISSIONS[@]}"; do
-    printf '%s'  "  Checking permissions for ${role}... "
-    local permissions
-    permissions=(${GOOGLE_CLOUD_PERMISSIONS[${role}]})
-    local missed
-    missed=$(get_number_of_missed_permissions "${permissions[@]}")
-    if [[ $missed -gt 0 ]]; then
-      message="missed ${missed}, failed"
-      error=1
+  while :; do
+    error=0
+    for role in "${!GOOGLE_CLOUD_PERMISSIONS[@]}"; do
+      printf '%s'  "  Checking permissions for ${role}... "
+      local permissions
+      permissions=(${GOOGLE_CLOUD_PERMISSIONS[${role}]})
+      local missed
+      missed=$(get_number_of_missed_permissions "${permissions[@]}")
+      if [[ $missed -gt 0 ]]; then
+        message="missed ${missed}, failed"
+        error=1
+      else
+        message='successfully'
+      fi
+      printf '%s\n' " ${message}."
+    done
+    if [[ ${error} -gt 0 ]]; then
+      printf '%s\n' "Permissions check failed."
+      printf '%s\n' "Would you like to login with another account? [Y/n]: "
+      local reLogin
+      read -r reLogin
+      reLogin=${reLogin:-"Y"}
+      if [[ ${reLogin} == "Y" || ${reLogin} == "y" ]]; then
+        gcloud auth login
+        continue
+      else
+        return 1
+      fi
     else
-      message='successfully'
+      echo "OK. Permissions check passed for Google Cloud project \
+  [${GCP_PROJECT}]."
+      return 0
     fi
-    printf '%s\n' " ${message}."
   done
-  if [[ ${error} -gt 0 ]]; then
-    printf '%s\n' "Permissions check failed."
-    return 1
-  else
-    echo "OK. Permissions check passed for Google Cloud project \
-[${GCP_PROJECT}]."
-    return 0
-  fi
 }
 
 #######################################
@@ -1129,6 +1141,7 @@ Project. Continuing to enter another bucket..."
     fi
   done
   declare -g "${gcsName}=${bucket}"
+  confirm_bucket_lifecycle "${bucket}"
 }
 
 #######################################
@@ -1149,6 +1162,58 @@ confirm_bucket_with_location() {
   confirm_located_bucket ${gcsName} ${locationName}
 }
 
+#######################################
+# Manage the lifecycle of a GCS bucket: setting or removing the GCS lifecycle
+# rule of 'age'.
+# See: https://cloud.google.com/storage/docs/lifecycle#age
+# Arguments:
+#   Bucket name
+#######################################
+confirm_bucket_lifecycle() {
+  local bucket bucketMetadata lifecycle
+  bucket="${1}"
+  bucketMetadata="$(get_bucket_metadata "${bucket}")"
+  lifecycle="$(get_value_from_json_string "${bucketMetadata}" "lifecycle")"
+  if [[ -n "${lifecycle}" ]]; then
+    printf '%s\n' "There are lifecycle rules in this bucket: ${lifecycle}."
+    printf '%s' "Would you like to overwrite it? [N/y]:"
+    local confirmContinue
+    read -r confirmContinue
+    confirmContinue=${confirmContinue:-"N"}
+    if [[ ${confirmContinue} == "N" || ${confirmContinue} == "n" ]]; then
+      return 0
+    fi
+  else
+    printf '%s\n' "There is no lifecycle rules in this bucket."
+    printf '%s' "Would you like to create it? [Y/n]:"
+    local confirmContinue
+    read -r confirmContinue
+    confirmContinue=${confirmContinue:-"Y"}
+    if [[ ${confirmContinue} == "N" || ${confirmContinue} == "n" ]]; then
+      return 0
+    fi
+  fi
+  while :; do
+    printf '%s' "Enter the number of days that a file will be kept before it \
+is automatically removed in the bucket[${bucket}]. (enter 0 to remove all \
+existing lifecycle rules): "
+    local days
+    read -r days
+    days=${days}
+    if [[ "${days}" =~ ^[0-9]+$ ]]; then
+      local lifecycle
+      if [[ "${days}" == "0" ]]; then
+        lifecycle="{}"
+      else
+        lifecycle='{"rule":[{"action":{"type":"Delete"},"condition":{"age":'\
+"${days}}}]}"
+      fi
+      gsutil lifecycle set /dev/stdin gs://${bucket} <<< ${lifecycle}
+      return $?
+    fi
+  done
+}
+
 #######################################
 # Confirm the monitored folder.
 # Globals:
@@ -1507,6 +1572,7 @@ set_authentication_env_for_cloud_functions() {
 create_or_update_cloud_scheduler_for_pubsub(){
   check_authentication
   quit_if_failed $?
+  local location_flag="--location=${REGION}"
   local scheduler_flag=()
   scheduler_flag+=(--schedule="$2")
   scheduler_flag+=(--time-zone="$3")
@@ -1514,13 +1580,13 @@ create_or_update_cloud_scheduler_for_pubsub(){
   scheduler_flag+=(--message-body="$5")
   local exist_job
   exist_job=($(gcloud scheduler jobs list --filter="name~${1}" \
---format="value(state)"))
+--format="value(state)" "${location_flag}"))
   local action needPause
   if [[ ${#exist_job[@]} -gt 0 ]]; then
     action="update"
     scheduler_flag+=(--update-attributes=$6)
     if [[ "${exist_job[0]}" == "PAUSED" ]]; then
-      gcloud scheduler jobs resume "${1}"
+      gcloud scheduler jobs resume "${1}" "${location_flag}"
       if [[ $? -gt 0 ]]; then
         printf '%s\n' "Failed to resume paused Cloud Scheduler job [${1}]."
         return 1
@@ -1531,9 +1597,9 @@ create_or_update_cloud_scheduler_for_pubsub(){
     action="create"
     scheduler_flag+=(--attributes=$6)
   fi
-  gcloud scheduler jobs ${action} pubsub "$1" "${scheduler_flag[@]}"
+  gcloud scheduler jobs ${action} pubsub "$1" "${scheduler_flag[@]}" "${location_flag}"
   if [[ "${needPause}" == "true" ]]; then
-      gcloud scheduler jobs pause "${1}"
+      gcloud scheduler jobs pause "${1}" "${location_flag}"
   fi
 }
 
@@ -1921,3 +1987,10 @@ join_string_array() {
   shift
   printf %s "$first" "${@/#/$separator}"
 }
+
+# Import other bash files.
+_SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${_SELF}/google_ads.sh"
+source "${_SELF}/bigquery.sh"
+
+printf '%s\n' "Common Bash Library is loaded."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "1.0.4",
+  "version": "1.1.1-beta",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -16,25 +16,26 @@
   },
   "homepage": "https://github.com/GoogleCloudPlatform/cloud-for-marketing/blob/master/marketing-analytics/activation/common-libs/nodejs-common/README.md",
   "dependencies": {
-    "@google-cloud/aiplatform": "^1.13.0",
-    "@google-cloud/automl": "^2.5.1",
-    "@google-cloud/bigquery": "^5.9.2",
+    "@google-cloud/aiplatform": "^1.19.0",
+    "@google-cloud/automl": "^2.5.2",
+    "@google-cloud/bigquery": "^5.12.0",
     "@google-cloud/datastore": "^6.6.2",
-    "@google-cloud/firestore": "^4.15.1",
-    "@google-cloud/logging-winston": "^4.1.1",
-    "@google-cloud/pubsub": "^2.18.2",
-    "@google-cloud/storage": "^5.16.0",
+    "@google-cloud/firestore": "^5.0.2",
+    "@google-cloud/logging-winston": "^4.2.2",
+    "@google-cloud/pubsub": "^2.19.0",
+    "@google-cloud/storage": "^5.18.3",
+    "@google-cloud/scheduler": "^2.3.0",
     "gaxios": "^4.3.2",
-    "google-ads-api": "^9.0.0",
-    "google-ads-node": "^7.0.0",
-    "google-auth-library": "^7.10.2",
-    "googleapis": "^91.0.0",
+    "google-ads-api": "^10.0.1",
+    "google-ads-node":"^8.0.1",
+    "google-auth-library": "^7.14.1",
+    "googleapis": "^100.0.0",
     "soap": "^0.43.0",
-    "winston": "^3.3.3",
+    "winston": "^3.7.2",
     "lodash": "^4.17.21"
   },
   "devDependencies": {
-    "jasmine": "^3.5.0"
+    "jasmine": "^4.0.2"
   },
   "scripts": {
     "test": "node node_modules/jasmine/bin/jasmine"

package/src/apis/ads_data_hub.js

@@ -37,9 +37,11 @@ class AdsDataHub {
    *
    * @param {GaxiosOptions|undefined=} options Used to setup for tests.
    * @param {string|undefined=} customerId ADH customer id.
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
    */
-  constructor(options, customerId = undefined) {
-    const authClient = new AuthClient(API_SCOPES);
+  constructor(options, customerId = undefined, env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     this.auth = authClient.getDefaultAuth();
     /** @const{GaxiosOptions} */ this.options = options || {};
     /** @const{string|undefined=} */ this.customerId = customerId;

package/src/apis/cloud_platform_apis.js

@@ -32,11 +32,11 @@ const API_VERSION = 'v1';
  * Google cloud client libraries.
  */
 class CloudPlatformApis {
-  constructor(projectId = process.env['GCP_PROJECT']) {
+  constructor(env = process.env) {
     /** @const {!AuthClient} */
-    const authClient = new AuthClient(API_SCOPES);
-    this.auth = authClient.getApplicationDefaultCredentials();
-    this.projectId = projectId;
+    const authClient = new AuthClient(API_SCOPES, env);
+    this.auth = authClient.getDefaultAuth();
+    this.projectId = env['GCP_PROJECT'];
   }
 
   /**

package/src/apis/doubleclick_bidmanager.js

@@ -56,8 +56,13 @@ let RequestBody;
  * https://developers.google.com/bid-manager/how-tos/authorizing
  */
 class DoubleClickBidManager {
-  constructor() {
-    const authClient = new AuthClient(API_SCOPES);
+  /**
+   * @constructor
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
+   */
+  constructor(env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     const auth = authClient.getDefaultAuth();
     /** @const {!google.doubleclickbidmanager} */
     this.instance = google.doubleclickbidmanager({
@@ -104,6 +109,20 @@ class DoubleClickBidManager {
     return response.data.queryId;
   }
 
+  /**
+   * Deletes a query.
+   * @param {number} queryId
+   * @return {!Promise<boolean>} Whether the query was deleted.
+   */
+  async deleteQuery(queryId) {
+    try {
+      await this.instance.queries.deletequery({ queryId });
+      return true;
+    } catch (error) {
+      console.error(error);
+      return false;
+    }
+  }
 }
 
 module.exports = {

package/src/apis/google_ads.js

@@ -31,6 +31,8 @@ const {
   services: {
     UploadClickConversionsRequest,
     UploadClickConversionsResponse,
+    UploadConversionAdjustmentsRequest,
+    UploadConversionAdjustmentsResponse,
     UploadUserDataRequest,
     UploadUserDataResponse,
     UserDataOperation,
@@ -51,8 +53,9 @@ const API_SCOPES = Object.freeze(['https://www.googleapis.com/auth/adwords',]);
 
 /**
  * List of properties that will be taken from the data file as elements of a
- * conversion.
+ * conversion or a conversion adjustment.
  * @see https://developers.google.com/google-ads/api/reference/rpc/latest/ClickConversion
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/ConversionAdjustment
  * @type {Array<string>}
  */
 const PICKED_PROPERTIES = [
@@ -65,6 +68,10 @@ const PICKED_PROPERTIES = [
   'conversion_value',
   'currency_code',
   'order_id',
+  'adjustment_type',
+  'adjustment_date_time',
+  'user_agent',
+  'gclid_date_time_pair',
 ];
 
 /**
@@ -88,9 +95,11 @@ const IDENTIFIERS = [
 const MAX_IDENTIFIERS_PER_USER = 20;
 
 /**
- * Configuration for uploading click conversions for Google Ads, includes:
+ * Configuration for uploading click conversions or conversion adjustments for
+ * Google Ads, includes:
  * gclid, conversion_action, conversion_date_time, conversion_value,
- * currency_code, order_id, external_attribution_data, etc.
+ * currency_code, order_id, external_attribution_data,
+ * adjustment_type, adjustment_date_time, user_agent, gclid_date_time_pair, etc.
  * @see PICKED_PROPERTIES
  *
  * Other properties that will be used to build the conversions but not picked by
@@ -116,12 +125,15 @@ const MAX_IDENTIFIERS_PER_USER = 20;
  *   conversion_value: number,
  *   currency_code:(string|undefined),
  *   order_id: (string|undefined),
- *   user_identifier_source:(UserIdentifierSource|undefined),
+ *   adjustment_type: (string|undefined),
+ *   adjustment_date_time: (!ConversionAdjustmentType|undefined),
+ *   user_agent: (string|undefined),
+ *   user_identifier_source:(!UserIdentifierSource|undefined),
  *   custom_variable_tags:(!Array<string>|undefined),
  *   customVariables:(!Object<string,string>|undefined),
  * }}
  */
-let ClickConversionConfig;
+let ConversionConfig;
 
 /**
  * Configuration for uploading customer match to Google Ads, includes:
@@ -255,7 +267,7 @@ class GoogleAds {
    * of click conversions.
    * @param {string} customerId
    * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
-   * @param {!ClickConversionConfig} adsConfig Default click conversion params
+   * @param {!ConversionConfig} adsConfig Default click conversion params
    * @return {!SendSingleBatch} Function which can send a batch of hits to
    *     Google Ads API.
    */
@@ -268,7 +280,7 @@ class GoogleAds {
      * @return {!BatchResult}
      */
     return async (lines, batchId) => {
-      /** @type {!Array<ClickConversionConfig>} */
+      /** @type {!Array<ConversionConfig>} */
       const conversions = lines.map(
           (line) => buildClickConversionFromLine(line, adsConfig, customerId));
       /** @const {BatchResult} */
@@ -300,6 +312,57 @@ class GoogleAds {
     }
   }
 
+  /**
+   * Returns the function to send out a request to Google Ads API with a batch
+   * of conversion adjustments.
+   * @param {string} customerId
+   * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
+   * @param {!ConversionConfig} adsConfig Default conversion adjustments
+   *     params.
+   * @return {!SendSingleBatch} Function which can send a batch of hits to
+   *     Google Ads API.
+   */
+  getUploadConversionAdjustmentFn(customerId, loginCustomerId, adsConfig) {
+    /**
+     * Sends a batch of hits to Google Ads API.
+     * @param {!Array<string>} lines Data for single request. It should be
+     *     guaranteed that it doesn't exceed quota limitation.
+     * @param {string} batchId The tag for log.
+     * @return {!BatchResult}
+     */
+    return async (lines, batchId) => {
+      /** @type {!Array<ConversionConfig>} */
+      const conversions = lines.map(
+          (line) => buildClickConversionFromLine(line, adsConfig, customerId));
+      /** @const {BatchResult} */
+      const batchResult = {
+        result: true,
+        numberOfLines: lines.length,
+      };
+      try {
+        const response = await this.uploadConversionAdjustments(conversions,
+            customerId, loginCustomerId);
+        const {results, partial_failure_error: failed} = response;
+        if (this.logger.isDebugEnabled()) {
+          const orderId = results.map((conversion) => conversion.order_id);
+          this.logger.debug('Uploaded order_id:', orderId);
+        }
+        if (failed) {
+          this.logger.info('partial_failure_error:', failed.message);
+          const failures = failed.details.map(
+              ({value}) => GoogleAdsFailure.decode(value));
+          this.extraFailedLines_(batchResult, failures, lines, 0);
+        }
+        return batchResult;
+      } catch (error) {
+        this.logger.error(
+            `Error in upload conversion adjustments batch: ${batchId}`, error);
+        this.updateBatchResultWithError_(batchResult, error, lines, 0);
+        return batchResult;
+      }
+    }
+  }
+
   /**
    * Updates the BatchResult based on errors.
    *
@@ -434,7 +497,7 @@ class GoogleAds {
    * Uploads click conversions to google ads account.
    * It requires an array of click conversions and customer id.
    * In DEBUG mode, this function will only validate the conversions.
-   * @param {Array<ClickConversionConfig>} clickConversions ClickConversions
+   * @param {Array<ConversionConfig>} clickConversions ClickConversions
    * @param {string} customerId
    * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
    * @return {!Promise<!UploadClickConversionsResponse>}
@@ -451,6 +514,31 @@ class GoogleAds {
     return customer.conversionUploads.uploadClickConversions(request);
   }
 
+  /**
+   * Uploads conversion adjustments to google ads account.
+   * It requires an array of conversion adjustments and customer id.
+   * In DEBUG mode, this function will only validate the conversion adjustments.
+   * @param {Array<ConversionAdjustment>} conversionAdjustments Conversion
+   *     adjustments.
+   * @param {string} customerId
+   * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
+   * @return {!Promise<!UploadConversionAdjustmentsResponse>}
+   */
+  uploadConversionAdjustments(conversionAdjustments, customerId,
+      loginCustomerId) {
+    this.logger.debug('Upload conversion adjustments for customerId:',
+        customerId);
+    const customer = this.getGoogleAdsApiCustomer_(loginCustomerId, customerId);
+    const request = new UploadConversionAdjustmentsRequest({
+      conversion_adjustments: conversionAdjustments,
+      customer_id: customerId,
+      validate_only: this.debugMode, // when true makes no changes
+      partial_failure: true, // Will still create the non-failed entities
+    });
+    return customer.conversionAdjustmentUploads.uploadConversionAdjustments(
+        request);
+  }
+
   /**
    * Returns the id of Conversion Custom Variable with the given tag.
    * @param {string} tag Custom Variable tag.
@@ -622,7 +710,7 @@ class GoogleAds {
 /**
  * Returns a conversion object based the given config and line data.
  * @param {string} line A JSON string of a conversion data.
- * @param {ClickConversionConfig} config Default click conversion params
+ * @param {ConversionConfig} config Default click conversion params
  * @param {string} customerId
  * @return {object} A conversion
  */
@@ -657,7 +745,7 @@ const buildClickConversionFromLine = (line, config, customerId) => {
 }
 
 module.exports = {
-  ClickConversionConfig,
+  ConversionConfig,
   CustomerMatchRecord,
   CustomerMatchConfig,
   GoogleAds,

package/src/apis/index.js

@@ -88,7 +88,7 @@ exports.bigquery = require('./bigquery.js');
  * APIs integration class for Google Ads.
  * @const {{
  *   GoogleAds:!GoogleAds,
- *   ClickConversionConfig:!ClickConversionConfig,
+ *   ConversionConfig:!ConversionConfig,
  *   CustomerMatchConfig: !CustomerMatchConfig,
  *   CustomerMatchRecord: !CustomerMatchRecord,
  *   ReportQueryConfig:!ReportQueryConfig,

package/src/components/automl.js

@@ -17,15 +17,9 @@
  */
 
 'use strict';
-const {request} = require('gaxios');
-const {JWT, Compute} = require('google-auth-library');
+const { ClientOptions } = require('google-gax');
 const automl = require('@google-cloud/automl');
 const google = automl.protos.google;
-const AuthClient = require('../apis/auth_client.js');
-
-const API_SCOPES = Object.freeze([
-  'https://www.googleapis.com/auth/cloud-platform',
-]);
 
 /**
  * For version `v1beta1`, BigQuery can be the source and destination.
@@ -40,46 +34,49 @@ const API_SCOPES = Object.freeze([
  * @type {string}
  */
 const API_VERSION = 'v1beta1';
-const PredictionServiceClient = automl[API_VERSION].PredictionServiceClient;
+const { PredictionServiceClient } = automl[API_VERSION];
 
 /**
- * AutoML Tables API doesn't have fully cloud client library support yet. This
- * class acts as the single client class for different API requests of AutoML
- * Tables API, including:
- * 1. 'batch predict' based on Google Cloud Client Library,
- * https://googleapis.dev/nodejs/automl/latest/index.html
- * 2. 'get Operations' based on REST API, see:
- * https://cloud.google.com/automl/docs/reference/rest/v1/projects.locations.operations/get
- * https://cloud.google.com/automl-tables/docs/long-operations#get-operation
+ * AutoML Tables API class for:
+ * 1. start a batch predict job:
+ * @see https://cloud.google.com/nodejs/docs/reference/automl/latest/automl/v1beta1.predictionserviceclient#_google_cloud_automl_v1beta1_PredictionServiceClient_batchPredict_member_3_
+ * 2. get the status of an operation:
+ * @see https://cloud.google.com/nodejs/docs/reference/automl/latest/automl/v1beta1.predictionserviceclient#_google_cloud_automl_v1beta1_PredictionServiceClient_checkBatchPredictProgress_member_1_
  */
 class AutoMl {
-
   /**
    * Initialize an instance.
-   * @param {{keyFilename:(string|undefined)}} options
+   * @param {ClientOptions=} options
    */
   constructor(options = {}) {
-    this.options = options;
+    this.client = new PredictionServiceClient(options);
   }
 
   /**
    * Batch predicts based on Google Cloud Client Library.
-   * See https://googleapis.dev/nodejs/automl/latest/index.html
    * @param {string} projectId
    * @param {string} computeRegion
    * @param {string} modelId
-   * @param {google.cloud.automl.v1.IBatchPredictInputConfig} inputConfig
-   * @param {google.cloud.automl.v1.IBatchPredictOutputConfig} outputConfig
+   * @param {google.cloud.automl.v1beta1.IBatchPredictInputConfig} inputConfig
+   * @param {google.cloud.automl.v1beta1.IBatchPredictOutputConfig} outputConfig
    * @return {Promise<string>} Predict operation name.
    */
-  async batchPredict(projectId, computeRegion, modelId, inputConfig,
-      outputConfig) {
-    const client = new PredictionServiceClient(this.options);
-    const modelFullId = client.modelPath(projectId, computeRegion, modelId);
-    const responses = await client.batchPredict({
+  async batchPredict(
+    projectId,
+    computeRegion,
+    modelId,
+    inputConfig,
+    outputConfig
+  ) {
+    const modelFullId = this.client.modelPath(
+      projectId,
+      computeRegion,
+      modelId
+    );
+    const responses = await this.client.batchPredict({
       name: modelFullId,
-      inputConfig: inputConfig,
-      outputConfig: outputConfig,
+      inputConfig,
+      outputConfig,
     });
     const operation = responses[1];
     console.log(`Operation name: ${operation.name}`);
@@ -87,37 +84,20 @@ class AutoMl {
   }
 
   /**
-   * Gets Operations based on REST API, see:
-   * https://cloud.google.com/automl/docs/reference/rest/v1/projects.locations.operations/get
-   * https://cloud.google.com/automl-tables/docs/long-operations#get-operation
+   * Gets status of an operation.
    * @param {string} operationName
-   * @return {Promise<google.longrunning.Operation>}
+   * @return {Promise<{{
+   *   done: boolean,
+   *   error: Error|undefined,
+   *   metadata: OperationMetadata,
+   *   name: string,
+   * }}>}
    */
   async getOperation(operationName) {
-    const url = `https://automl.googleapis.com/${API_VERSION}/${operationName}`;
-    const headers = await this.getAuthClient_().getRequestHeaders();
-    const requestOptions = {
-      method: 'GET',
-      headers,
-      url,
-    };
-    const response = await request(requestOptions);
-    return response.data;
+    const response = await this.client.checkBatchPredictProgress(operationName);
+    const { done, error, metadata, name } = response;
+    return { done, error, metadata, name };
   }
-
-  /**
-   * Gets authentication client of AutoML API.
-   * This API belongs to GCP, however it is not fully supported by Google Cloud
-   * Client Library. So we need to manage some functions on its REST API,
-   * in the way like we invoke other external APIs.
-   * @returns {(!JWT|!Compute)}
-   * @private
-   */
-  getAuthClient_() {
-    /** @const {!AuthClient} */ const authClient = new AuthClient(API_SCOPES);
-    return authClient.getApplicationDefaultCredentials();
-  }
-
 }
 
 exports.AutoMl = AutoMl;

package/src/components/scheduler.js

@@ -30,11 +30,36 @@ const API_VERSION = 'v1';
  * to get/pause/resume a job.
  */
 class CloudScheduler {
-  constructor(projectId = process.env['GCP_PROJECT']) {
-    /** @const {!AuthClient} */
-    const authClient = new AuthClient(API_SCOPES);
-    this.auth = authClient.getApplicationDefaultCredentials();
-    this.projectId = projectId;
+  constructor(env = process.env, options = {}) {
+    if (!options.authClient) {
+      /** @const {!AuthClient} */
+      const authClient = new AuthClient(API_SCOPES, env);
+      /**
+       * By default, `AuthClient` (getDefaultAuth()) will return an auth client
+       * based on the settings in ENV while the OAuth is the most preferred.
+       * This works for most of the external API clients (in the '../apis'
+       * folder), however this won't work in the Cloud Functions, as those OAuth
+       * token usually won't have enough permission to invoke Google Cloud API.
+       * Using the method `getApplicationDefaultCredentials` to force
+       * `AuthClient` return an ADC auth client, which will work in the Cloud.
+       *
+       * Cloud Scheduler API Client Library is used here as Cloud Client Library
+       * is still at beta stage. (For the difference, see
+       * https://cloud.google.com/apis/docs/client-libraries-explained)
+       * Eventually, when we migrate this to the cloud client library, which
+       * automatically takes ADC as the authentication method, the 'AuthClient'
+       * is not required here and can be removed.
+       */
+      this.auth = authClient.getApplicationDefaultCredentials();
+    } else {
+      /**
+       * `authClient` can be consumed by cloud client library as the auth
+       * client. By passing this in, we can offer more flexible auth clients in
+       * test cases for API client library and cloud client library in future.
+       */
+      this.auth = options.authClient;
+    }
+    this.projectId = env['GCP_PROJECT'];
     this.instance = cloudscheduler({
       version: API_VERSION,
       auth: this.auth,
@@ -113,8 +138,8 @@ class CloudScheduler {
    */
   async getJobs_(name, targetLocations = undefined) {
     const regex = new RegExp(`/jobs/${name}$`);
-    const jobs = (await this.listJobs_(targetLocations)).filter(
-        (job) => regex.test(job));
+    const allJobs = await this.listJobs_(targetLocations);
+    const jobs = allJobs.filter((job) => regex.test(job));
     if (jobs.length === 0) console.error(`Can not find job: ${name}`);
     return jobs;
   }
@@ -134,23 +159,10 @@ class CloudScheduler {
     const projectId = await this.getProjectId_();
     const requestPrefix = `projects/${projectId}/locations`;
     const jobs = locations.map(async (location) => {
-      const request = {parent: `${requestPrefix}/${location}`};
-      try {
-        const response = await this.instance.projects.locations.jobs.list(
-            request);
-        return response.data.jobs.map((job) => job.name);
-      } catch (error) {
-        // Currently, listLocations always returns an array with one location.
-        // Not sure whether this will be changed or not in future. If one day
-        // the Cloud Scheduler let users to select a location for a job, then
-        // it may return multiple locations when listLocation, however the
-        // target job may exist in one of the locations. In this case, when we
-        // iterate the job with all possible locations to get the complete job
-        // path to operate, it will likely generate an error for those wrong
-        // location(s). So set the try-catch here to handle this situation.
-        console.error(error.message);
-        return [];
-      }
+      const request = { parent: `${requestPrefix}/${location}` };
+      const response = await this.instance.projects.locations.jobs.list(request);
+      if (!response.data.jobs) return [];
+      return response.data.jobs.map((job) => job.name);
     });
     // Waits for all jobs names and flattens nested job name arrays, however
     // there is no 'flat' available in current Cloud Functions runtime.

package/src/components/utils.js

@@ -20,8 +20,7 @@
 
 const winston = require('winston');
 const {inspect} = require('util');
-const {LoggingWinston} = require('@google-cloud/logging-winston');
-const {CloudPlatformApis} = require('../apis/cloud_platform_apis.js');
+const { LoggingWinston } = require('@google-cloud/logging-winston');
 
 /**
  * The result of a batch of data sent to target API. The batch here means the
@@ -498,37 +497,6 @@ const extractObject = (paths) => {
   };
 };
 
-/**
- * Checks whether the permissions are granted for current authentication.
- * This function will be invoked during the deployment of a specific solution,
- * e.g. Tentacles, to make sure the operator has the proper permissions to
- * carry on. If the operator doesn't have enough permissions, it will exit with
- * status code 1 to let the invoker (the Bash installation script) know that it
- * doesn't pass.
- * @param {!Array<string>} permissions Array of permissions to check.
- * @param {string} projectId The Id of Cloud project.
- * @return {!Promise<undefined>}
- */
-const checkPermissions = (permissions,
-    projectId = process.env['GCP_PROJECT']) => {
-  const cloudPlatformApis = new CloudPlatformApis(projectId);
-  return cloudPlatformApis.testIamPermissions(permissions)
-      .then((grantedPermissions) => {
-        console.log(grantedPermissions);
-        grantedPermissions = grantedPermissions || [];
-        if (grantedPermissions.length < permissions.length) {
-          const missedPermissions = permissions.filter(
-              (permission) => grantedPermissions.indexOf(permission) === -1);
-          console.error(`[MISSED] ${missedPermissions.join(',')}.`);
-          process.exit(1);
-        }
-      })
-      .catch((error) => {
-        console.error(`[ERROR] ${error.message}`);
-        process.exit(1);
-      });
-};
-
 /**
  * For more details, see:
  * https://developers.google.com/google-ads/api/docs/rest/design/json-mappings
@@ -565,7 +533,6 @@ module.exports = {
   replaceParameters,
   getFilterFunction,
   extractObject,
-  checkPermissions,
   changeNamingFromSnakeToUpperCamel,
   changeNamingFromSnakeToLowerCamel,
 };

package/src/components/vertex_ai.js

@@ -18,6 +18,7 @@
 
 'use strict';
 
+const { ClientOptions } = require('google-gax');
 const {JobServiceClient} = require('@google-cloud/aiplatform');
 
 /**
@@ -29,7 +30,7 @@ class VertexAi {
 
   /**
    * Initialize an instance.
-   * @param {{keyFilename:(string|undefined)}} options
+   * @param {ClientOptions=} options
    */
   constructor(options = {}) {
     this.options = options;
@@ -105,9 +106,9 @@ class VertexAi {
    * @private
    */
   getJobServiceClient_(location) {
-    const clientOptions = {
+    const clientOptions = Object.assign({}, this.options, {
       apiEndpoint: this.getServiceEndpoint_(location),
-    };
+    });
     return new JobServiceClient(clientOptions);
   }
 }