@google-cloud/nodejs-common 0.8.3 → 0.9.2-beta2

package/bin/install_functions.sh

@@ -138,13 +138,6 @@ EXTERNAL_API_SCOPES=(
 # Enabled APIs' OAuth scopes.
 ENABLED_OAUTH_SCOPES=()
 
-# https://cloud.google.com/iam/docs/understanding-roles#service-accounts-roles
-declare -A GOOGLE_SERVICE_ACCOUNT_PERMISSIONS
-GOOGLE_SERVICE_ACCOUNT_PERMISSIONS=(
-  ["Service Account Admin"]="iam.serviceAccounts.create"
-  ["Service Account Key Admin"]="iam.serviceAccounts.create"
-)
-
 # Preparation functions.
 #######################################
 # Mimic a Cloud Shell environment to enable local running.
@@ -437,8 +430,8 @@ select_functions_location() {
   local regionVariableName defaultValue locations
   regionVariableName="${1:-"REGION"}"
   defaultValue="${!regionVariableName}"
-  locations=($(gcloud functions regions list | grep "projects"|sed \
-    's/projects\/.*\/locations\///'))
+  locations=($(gcloud functions regions list --format="csv[no-heading](name)"| \
+    sed 's/projects\/.*\/locations\///'))
   local region
   while :; do
     local exist_functions
@@ -561,7 +554,6 @@ need to be enabled."
 #   NEED_OAUTH
 #   NEED_SERVICE_ACCOUNT
 #   GOOGLE_CLOUD_PERMISSIONS
-#   GOOGLE_SERVICE_ACCOUNT_PERMISSIONS
 # Arguments:
 #   None
 #######################################
@@ -590,11 +582,6 @@ authentication method:"
     printf '%s\n' "... OAuth is selected."
   else
     NEED_SERVICE_ACCOUNT="true"
-    local role
-    for role in "${!GOOGLE_SERVICE_ACCOUNT_PERMISSIONS[@]}"; do
-      GOOGLE_CLOUD_PERMISSIONS["${role}"]=\
-"${GOOGLE_SERVICE_ACCOUNT_PERMISSIONS["${role}"]}"
-    done
     printf '%s\n' "... Service Account is selected."
   fi
 }
@@ -1192,6 +1179,87 @@ EOF
   printf '%s\n' "OK. Continue with monitored folder [${folder}]."
 }
 
+#######################################
+# Create or update a Log router sink.
+# Globals:
+#   None
+# Arguments:
+#   Name of the sink
+#   Filter conditions
+#   Sink destination
+#######################################
+create_or_update_sink() {
+  local sinkName=${1}
+  local logFilter=${2}
+  local sinkDestAndFlags=(${3})
+  local existingFilter
+  existingFilter=$(gcloud logging sinks list --filter="name:${sinkName}" \
+--format="value(filter)")
+  if [[ "${existingFilter}" != "${logFilter}" ]]; then
+    local action
+    if [[ -z "${existingFilter}" ]]; then
+      action="create"
+      printf '%s\n' "  Logging Export [${sinkName}] doesn't exist. Creating..."
+    else
+      action="update"
+      printf '%s\n' "  Logging Export [${sinkName}] exists with a different \
+filter. Updating..."
+    fi
+    gcloud -q logging sinks ${action} "${sinkName}" "${sinkDestAndFlags[@]}" \
+--log-filter="${logFilter}"
+  else
+    printf '%s\n' "  Logging Export [${sinkName}] exists. Continue..."
+  fi
+  if [[ $? -gt 0 ]];then
+    printf '%s\n' "Failed to create or update Logs router sink."
+    return 1
+  fi
+}
+
+#######################################
+# Confirm the service account of the given sink has proper permission to dump
+# the logs.
+# Globals:
+#   None
+# Arguments:
+#   Name of the sink
+#   Bindings role, e.g. "pubsub.publisher" or "bigquery.dataEditor"
+#   Role name, e.g. "Pub/Sub Publisher" or "BigQuery Data Editor"
+#######################################
+confirm_sink_service_account_permission() {
+  local sinkName=${1}
+  local bindingsRole=${2}
+  local roleName=${3}
+  local serviceAccount existingRole
+  serviceAccount=$(gcloud logging sinks describe "${sinkName}" \
+--format="get(writerIdentity)")
+  while :;do
+    printf '%s\n' "  Checking the role of the sink's service account \
+[${serviceAccount}]..."
+    existingRole=$(gcloud projects get-iam-policy "${GCP_PROJECT}" \
+--flatten=bindings --filter="bindings.members:${serviceAccount} AND \
+bindings.role:roles/${bindingsRole}" --format="get(bindings.members)")
+    if [[ -z "${existingRole}" ]];then
+      printf '%s\n'  "  Granting Role '${roleName}' to the service \
+account..."
+      gcloud -q projects add-iam-policy-binding "${GCP_PROJECT}" --member \
+"${serviceAccount}" --role roles/${bindingsRole}
+      if [[ $? -gt 0 ]];then
+        printf '%s\n' "Failed to grant the role. Use this link \
+https://console.cloud.google.com/iam-admin/iam?project=${GCP_PROJECT} to \
+manually grant Role '${roleName}' to ${serviceAccount}."
+        printf '%s' "Press any key to continue after you grant the access..."
+        local any
+        read -n1 -s any
+        continue
+      fi
+    else
+      printf '%s\n'  "  The role has already been granted."
+      return 0
+    fi
+  done
+}
+
 #######################################
 # Save the configuration to a local file.
 # Globals:
@@ -1228,9 +1296,15 @@ save_config() {
 
 #######################################
 # Based on the authentication method, guide user complete authentication
-# process. For service account, confirm to use/create a service account and
-# download key file; for OAuth 2.0, guide user to complete OAuth authentication
-# and save the refresh token.
+# process.
+# For service account, given that Cloud Functions can extend the authorized API
+# scopes now, it will use the default service account rather than an explicit
+# service account with the key file downloaded. By doing so, we can reduce the
+# risk of leaking service account key. If there is a service key in the current
+# folder from previous installation, the code will continue using it, otherwise
+# it will use the Cloud Functions' default service account.
+# For OAuth 2.0, guide user to complete OAuth authentication and save the
+# refresh token.
 # Globals:
 #   NEED_SERVICE_ACCOUNT
 #   NEED_OAUTH
@@ -1238,152 +1312,11 @@ save_config() {
 #   None
 #######################################
 do_authentication(){
-  if [[ ${NEED_SERVICE_ACCOUNT} == "true" ]]; then
-    download_service_account_key
-  fi
   if [[ ${NEED_OAUTH} == "true" ]]; then
     do_oauth
   fi
 }
 
-#######################################
-# Download a service account key file and save as `$SA_KEY_FILE`.
-# Globals:
-#   SA_NAME
-#   GCP_PROJECT
-#   SA_KEY_FILE
-# Arguments:
-#   None
-# Returns:
-#   0 if service key files exists or created, non-zero on error.
-#######################################
-download_service_account_key() {
-  (( STEP += 1 ))
-  printf '%s\n' "Step ${STEP}: Downloading the key file for the service \
-account..."
-  if [[ -z ${SA_NAME} ]];then
-    confirm_service_account
-  fi
-  local suffix exist
-  suffix=$(get_sa_domain_from_gcp_id "${GCP_PROJECT}")
-  local email="${SA_NAME}@${suffix}"
-  local prompt="Would you like to download the key file for [${email}] and \
-save it as ${SA_KEY_FILE}? [Y/n]: "
-  local default_value="y"
-  if [[ -f "${SA_KEY_FILE}" && -s "${SA_KEY_FILE}" ]]; then
-    exist=$(get_value_from_json_file ${SA_KEY_FILE} 'client_email' 2>&1)
-    if [[ ${exist} =~ .*("@${suffix}") ]]; then
-      prompt="A key file for [${exist}] with the key ID '\
-$(get_value_from_json_file ${SA_KEY_FILE} 'private_key_id') already exists'. \
-Would you like to create a new key to overwrite it? [N/y]: "
-      default_value="n"
-    fi
-  fi
-  printf '%s' "${prompt}"
-  local input
-  read -r input
-  input=${input:-"${default_value}"}
-  if [[ ${input} == 'y' || ${input} == 'Y' ]];then
-    printf '%s\n' "Downloading a new key file for [${email}]..."
-    gcloud iam service-accounts keys create "${SA_KEY_FILE}" --iam-account \
-"${email}"
-    if [[ $? -gt 0 ]]; then
-      printf '%s\n' "Failed to download new key files for [${email}]."
-      return 1
-    else
-      printf '%s\n' "OK. New key file is saved at [${SA_KEY_FILE}]."
-      return 0
-    fi
-  else
-    printf '%s\n' "Skipped downloading new key file. See \
-https://cloud.google.com/iam/docs/creating-managing-service-account-keys \
-to learn more about service account key files."
-    return 0
-  fi
-}
-
-#######################################
-# Make sure a service account for this integration exists and set the email of
-# the service account to the global variable `SA_NAME`.
-# Globals:
-#   GCP_PROJECT
-#   SA_KEY_FILE
-#   SA_NAME
-#   DEFAULT_SERVICE_ACCOUNT
-# Arguments:
-#   None
-#######################################
-confirm_service_account() {
-  cat <<EOF
-  Some external APIs might require authentication based on OAuth or \
-JWT(service account), for example, Google Analytics or Campaign Manager. \
-In this step, you prepare the service account. For more information, see \
-https://cloud.google.com/iam/docs/creating-managing-service-accounts
-EOF
-
-  local suffix
-  suffix=$(get_sa_domain_from_gcp_id "${GCP_PROJECT}")
-  local email
-  if [[ -f "${SA_KEY_FILE}" && -s "${SA_KEY_FILE}" ]]; then
-    email=$(get_value_from_json_file "${SA_KEY_FILE}" 'client_email')
-    if [[ ${email} =~ .*("@${suffix}") ]]; then
-      printf '%s' "A key file for service account [${email}] already exists. \
-Would you like to create a new service account? [N/y]: "
-      local input
-      read -r input
-      if [[ ${input} != 'y' && ${input} != 'Y' ]]; then
-        printf '%s\n' "OK. Will use existing service account [${email}]."
-        SA_NAME=$(printf "${email}" | cut -d@ -f1)
-        return 0
-      fi
-    fi
-  fi
-
-  SA_NAME="${SA_NAME:-"${PROJECT_NAMESPACE}-api"}"
-  while :; do
-    printf '%s' "Enter the name of service account [${SA_NAME}]: "
-    local input sa_elements=() sa
-    read -r input
-    input=${input:-"${SA_NAME}"}
-    IFS='@' read -a sa_elements <<< "${input}"
-    if [[ ${#sa_elements[@]} == 1 ]]; then
-      echo "  Append default suffix to service account name and get: ${email}"
-      sa="${input}"
-      email="${sa}@${suffix}"
-    else
-      if [[ ${sa_elements[1]} != "${suffix}" ]]; then
-        printf '%s\n' "  Error: Service account domain name ${sa_elements[1]} \
-doesn't belong to the current project. The service account domain name for the \
-current project should be: ${suffix}."
-        continue
-      fi
-      sa="${sa_elements[0]}"
-      email="${input}"
-    fi
-
-    printf '%s\n' "Checking the existence of the service account [${email}]..."
-    if ! result=$(gcloud iam service-accounts describe "${email}" 2>&1); then
-      printf '%s\n' "  Service account [${email}] does not exist. Trying to \
-create..."
-      gcloud iam service-accounts create "${sa}" --display-name \
-"Tentacles API requester"
-      if [[ $? -gt 0 ]]; then
-        printf '%s\n' "Creating the service account [${email}] failed. Please \
-try again..."
-      else
-        printf '%s\n' "The service account [${email}] was successfully created."
-        SA_NAME=${sa}
-        break
-      fi
-    else
-      printf ' found.\n'
-      SA_NAME=${sa}
-      break
-    fi
-  done
-  printf '%s\n' "OK. Service account [${SA_NAME}] is ready."
-}
-
 #######################################
 # Guide an OAuth process and save the token to file.
 # The whole process will be:
@@ -1393,8 +1326,8 @@ try again..."
 # 3. Prompt user to enter the OAuth Client secret.
 # 4. Print the OAuth authentication URL which users should open in a browser \
 # and complete the process.
-# 5. Copy the authentication code from the browser and paste here.
-# 6. Use the authentication code to redeem an OAuth token and save it.
+# 5. Copy the authorization code from the browser and paste here.
+# 6. Use the authorization code to redeem an OAuth token and save it.
 # Globals:
 #   ENABLED_OAUTH_SCOPES
 #   OAUTH2_TOKEN_JSON
@@ -1480,10 +1413,10 @@ ${auth_url}"
 "Error 400: redirect_uri_mismatch" shown up on the page. In this case, press \
 "Enter" to start again with a native application OAuth client ID.
 EOF
-    printf '%s' "4. Copy the authentication code from browser and paste here: "
+    printf '%s' "4. Copy the authorization code from browser and paste here: "
     read -r auth_code
     if [[ -z ${auth_code} ]]; then
-      printf '%s\n\n' "No authentication code. Starting from beginning again..."
+      printf '%s\n\n' "No authorization code. Starting from beginning again..."
       continue
     fi
     auth_response=$(curl -s -d "code=${auth_code}" -d "client_id=${client_id}" \
@@ -1492,7 +1425,7 @@ EOF
     auth_error=$(node -e "console.log(!!JSON.parse(process.argv[1]).error)" \
 "${auth_response}")
     if [[ ${auth_error} == "true" ]]; then
-      printf '%s\n' "Error happened in redeem the authentication code: \
+      printf '%s\n' "Error happened in redeem the authorization code: \
 ${auth_response}"
       continue
     fi
@@ -1549,7 +1482,7 @@ set_authentication_env_for_cloud_functions() {
 }
 
 #######################################
-# Create or update a Cloud Schdduleer
+# Create or update a Cloud Scheduler
 # Globals:
 #   None
 # Arguments:
@@ -1635,8 +1568,27 @@ print_finished(){
 print_service_account(){
   printf '%s\n' "=========================="
   local email
-  email=$(get_value_from_json_file "${SA_KEY_FILE}" 'client_email')
-  printf '%s\n' "The email address of the current service account is ${email}."
+  email=$(get_service_account)
+  printf '%s\n' "The email address of the current service account is: ${email}."
+}
+
+#######################################
+# Returns the email address of the service account. If there is a key file of
+# service account, it returns the email of that service account; otherwise it
+# it will get the default service account of Cloud Functions.
+# Globals:
+#   SA_KEY_FILE
+# Arguments:
+#   None
+#######################################
+get_service_account(){
+  local email
+  if [[ -f "${SA_KEY_FILE}" ]]; then
+    email=$(get_value_from_json_file "${SA_KEY_FILE}" 'client_email')
+  else
+    email=$(get_cloud_functions_service_account)
+  fi
+  printf '%s' "${email}"
 }
 
 #######################################
@@ -1843,25 +1795,25 @@ sed  -r 's/^([^:]*):(.*)$/\2.\1/').iam.gserviceaccount.com"
 
 #######################################
 # Returns the default service account of the given Cloud Functions.
-# Use cases:
-# 1. When BigQuery query Google Sheet based external tables, this service
-#    account needs to be added to the Google Sheet as a viewer.
 # Globals:
 #   None
 # Arguments:
-#   None
+#   The name of Cloud Functions, default value is a random one starts with the
+#     namespace.
 # Returns:
 #   The default service account of the given Cloud Functions.
 #######################################
 get_cloud_functions_service_account() {
-  local region=$(gcloud functions list --format="csv[no-heading](name,REGION)" \
-| grep "${1}" | cut -d, -f2 | uniq)
-  if [[ -z ${region} ]]; then
-    printf '%s\n' "Cloud Functions [$1] doesn't exist."
+  local cf=($(gcloud functions list --format="csv[no-heading,separator=\
+' '](name,REGION)" | grep "${1:-"${PROJECT_NAMESPACE}"}" | head -1))
+  if [[ ${#cf[@]} -lt 1 ]]; then
+    printf '%s\n' "Cloud Functions [${1}] doesn't exist."
   else
-    local service_account=$(gcloud functions describe "$1" \
+    local name="${cf[0]}"
+    local region="${cf[1]}"
+    local service_account=$(gcloud functions describe "${name}" \
 --region="${region}" --format="get(serviceAccountEmail)")
-    printf '%s' "$service_account"
+    printf '%s' "${service_account}"
   fi
 }
 
@@ -1903,7 +1855,7 @@ EOF
 #######################################
 quit_if_failed() {
   printf '\n'
-  if [[ $1 -gt 0 ]];then
+  if [[ ${1} -gt 0 ]];then
     printf '%s\n' "[Error] Quit."
     exit 1
   fi

package/index.js

@@ -26,3 +26,4 @@ exports.pubsub = require('./src/components/pubsub.js');
 exports.scheduler = require('./src/components/scheduler.js');
 exports.storage = require('./src/components/storage.js');
 exports.utils = require('./src/components/utils.js');
+exports.vertexai = require('./src/components/vertex_ai.js');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "0.8.3",
+  "version": "0.9.2-beta2",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -16,6 +16,7 @@
   },
   "homepage": "https://github.com/GoogleCloudPlatform/cloud-for-marketing/blob/master/marketing-analytics/activation/common-libs/nodejs-common/README.md",
   "dependencies": {
+    "@google-cloud/aiplatform": "^1.10.1",
     "@google-cloud/automl": "^2.4.2",
     "@google-cloud/bigquery": "^5.6.0",
     "@google-cloud/datastore": "^6.4.0",
@@ -24,8 +25,8 @@
     "@google-cloud/pubsub": "^2.12.0",
     "@google-cloud/storage": "^5.8.5",
     "gaxios": "^4.3.0",
-    "google-ads-api": "^5.2.0",
-    "google-ads-node": "^5.0.0",
+    "google-ads-api": "^8.1.0",
+    "google-ads-node": "^6.1.3",
     "google-auth-library": "^7.1.0",
     "googleapis": "^74.2.0",
     "soap": "^0.38.0",

package/src/apis/analytics.js

@@ -23,7 +23,7 @@ const stream = require('stream');
 const {google} = require('googleapis');
 const {Schema$Upload} = google.analytics;
 const AuthClient = require('./auth_client.js');
-const {wait, getLogger} = require('../components/utils.js');
+const {wait, getLogger, BatchResult} = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
   'https://www.googleapis.com/auth/analytics',
@@ -56,8 +56,13 @@ let DataImportClearConfig;
  * Google Analytics API v3 stub.
  */
 class Analytics {
-  constructor() {
-    const authClient = new AuthClient(API_SCOPES);
+  /**
+   * @constructor
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
+   */
+  constructor(env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     const auth = authClient.getDefaultAuth();
     /** @type {!google.analytics} */
     this.instance = google.analytics({
@@ -76,8 +81,7 @@ class Analytics {
    * @param {string|!stream.Readable} data A string or a stream to be uploaded.
    * @param {!DataImportConfig} config GA data import configuration.
    * @param {string=} batchId A tag for log.
-   * @return {!Promise<boolean>} Promise returning whether data import
-   *     succeeded.
+   * @return {!Promise<!BatchResult>}
    */
   async uploadData(data, config, batchId = 'unnamed') {
     const uploadConfig = Object.assign(
@@ -95,28 +99,38 @@ class Analytics {
     this.logger.debug('Response: ', response);
     const job = /** @type {Schema$Upload} */ response.data;
     const uploadId = (/** @type {Schema$Upload} */job).id;
-    console.log(`Task [${batchId}] creates GA Data import job: ${uploadId}`);
+    this.logger.info(
+        `Task [${batchId}] creates GA Data import job: ${uploadId}`);
     const jobConfig = Object.assign({uploadId}, config);
     const result = await Promise.race([
       this.checkJobStatus(jobConfig),
       wait(8 * 60 * 1000, job),  // wait up to 8 minutes here
     ]);
+    /** @type {BatchResult} */ const batchResult = {};
     switch ((/** @type {Schema$Upload} */ result).status) {
       case 'FAILED':
-        this.logger.error('GA Data Import failed', job);
-        return false;
+        this.logger.error('GA Data Import failed', result);
+        batchResult.result = false;
+        batchResult.errors = result.errors
+            || [`Unknown reason. ID: ${uploadId}`];
+        break;
       case 'COMPLETED':
         this.logger.info(`GA Data Import job[${uploadId}] completed.`);
-        this.logger.debug('Response: ', job);
-        return true;
+        this.logger.debug('Response: ', result);
+        batchResult.result = true;
+        break;
       case 'PENDING':
-        this.logger.info('GA Data Import pending.', job);
+        this.logger.info('GA Data Import pending.', result);
         this.logger.info('Still will return true here.');
-        return true;
+        batchResult.result = true;
+        break;
       default:
-        this.logger.error('Unknown results of GA Data Import: ', job);
-        return false;
+        this.logger.error('Unknown results of GA Data Import: ', result);
+        batchResult.result = false;
+        batchResult.errors = [`Unknown status. ID: ${uploadId}`];
     }
+    console.log(batchResult);
+    return batchResult;
   }
 
   /**

package/src/apis/auth_client.js

@@ -55,11 +55,13 @@ class AuthClient {
   /**
    * Create a new instance with given API scopes.
    * @param {string|!Array<string>|!ReadonlyArray<string>} scopes
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
    */
-  constructor(scopes) {
+  constructor(scopes, env = process.env) {
     this.scopes = scopes;
-    this.oauthTokenFile = getFullPathForEnv(DEFAULT_ENV_OAUTH);
-    this.serviceAccountKeyFile = getFullPathForEnv(DEFAULT_ENV_KEYFILE);
+    this.oauthTokenFile = getFullPathForEnv(DEFAULT_ENV_OAUTH, env);
+    this.serviceAccountKeyFile = getFullPathForEnv(DEFAULT_ENV_KEYFILE, env);
   }
 
   /**
@@ -109,7 +111,7 @@ class AuthClient {
    */
   getOAuth2Client(keyFile = this.oauthTokenFile) {
     const key = JSON.parse(fs.readFileSync(keyFile).toString());
-    console.log(`Get OAuth token with Email: ${key.client_id}`);
+    console.log(`Get OAuth token with client Id: ${key.client_id}`);
     const oAuth2Client = new OAuth2Client(key.client_id, key.client_secret);
     oAuth2Client.setCredentials({refresh_token: key.token.refresh_token});
     return oAuth2Client;
@@ -152,15 +154,18 @@ class AuthClient {
  * Returns the full path of a existent file whose path (relative or
  * absolute) is the value of the given environment variable.
  * @param {string} envName The name of environment variable for the file path.
+ * @param {!Object<string,string>=} env The environment object to hold env
+ *     variables.
  * @return {?string} Full path of the file what set as an environment variable.
  */
-function getFullPathForEnv(envName) {
-  if (typeof process.env[envName] === 'undefined') {
+function getFullPathForEnv(envName, env) {
+  const envValue = env[envName];
+  if (typeof envValue === 'undefined') {
     console.log(`Env[${envName}] doesn't have a value.`);
   } else {
-    const fullPath = process.env[envName].startsWith('/') ?
-        process.env[envName] :
-        path.join(__dirname, process.env[envName]);
+    const fullPath = envValue.startsWith('/') ?
+        envValue :
+        path.join(__dirname, envValue);
     if (fs.existsSync(fullPath)) {
       console.log(`Find file '${fullPath}' set in env as [${envName}].`);
       return fullPath;

package/src/apis/cloud_platform_apis.js

@@ -32,11 +32,11 @@ const API_VERSION = 'v1';
  * Google cloud client libraries.
  */
 class CloudPlatformApis {
-  constructor() {
+  constructor(projectId = process.env['GCP_PROJECT']) {
     /** @const {!AuthClient} */
     const authClient = new AuthClient(API_SCOPES);
     this.auth = authClient.getApplicationDefaultCredentials();
-    this.projectId = process.env['GCP_PROJECT'];
+    this.projectId = projectId;
   }
 
   /**

package/src/apis/google_ads.js

@@ -18,24 +18,23 @@
 'use strict';
 
 const {protos: {google: {ads: {googleads}}}} = require('google-ads-node');
+const googleAdsLib = googleads[Object.keys(googleads)[0]];
 const {
-  v7: {
-    common: {
-      UserData,
-      UserIdentifier,
-      CustomerMatchUserListMetadata,
-    },
-    resources: {
-      GoogleAdsField,
-    },
-    services: {
-      UploadClickConversionsRequest,
-      UploadUserDataRequest,
-      UserDataOperation,
-      SearchGoogleAdsFieldsRequest,
-    },
-  }
-} = googleads;
+  common: {
+    UserData,
+    UserIdentifier,
+    CustomerMatchUserListMetadata,
+  },
+  resources: {
+    GoogleAdsField,
+  },
+  services: {
+    UploadClickConversionsRequest,
+    UploadUserDataRequest,
+    UserDataOperation,
+    SearchGoogleAdsFieldsRequest,
+  },
+} = googleAdsLib;
 const {GoogleAdsApi} = require('google-ads-api');
 const AuthClient = require('./auth_client.js');
 const {getLogger} = require('../components/utils.js');

package/src/apis/measurement_protocol.js

@@ -64,7 +64,7 @@ class MeasurementProtocol {
      * @param {string} batchId The tag for log.
      * @return {!Promise<boolean>}
      */
-    return (lines, batchId) => {
+    return async (lines, batchId) => {
       const payload =
           lines
               .map((line) => {
@@ -85,24 +85,37 @@ class MeasurementProtocol {
         body: payload,
         headers: {'User-Agent': 'Tentacles/MeasurementProtocol-v1'}
       };
-      return request(requestOptions).then((response) => {
-        if (response.status < 200 || response.status >= 300) {
-          const errorMessages = [
-            `Measurement Protocol [${batchId}] didn't succeed.`,
-            `Get response code: ${response.status}`,
-            `response: ${response.data}`,
-          ];
-          console.error(errorMessages.join('\n'));
-          throw new Error(`Status code not 2XX`);
-        }
-        this.logger.debug(`Configuration:`, config);
-        this.logger.debug(`Input Data:   `, lines);
-        this.logger.debug(`Batch[${batchId}] status: ${response.status}`);
-        this.logger.debug(response.data);
-        // There is not enough information from the non-debug mode.
-        if (!this.debugMode) return true;
-        return response.data.hitParsingResult.every((result) => result.valid);
-      });
+      const response = await request(requestOptions);
+      console.log(response);
+      if (response.status < 200 || response.status >= 300) {
+        const errorMessages = [
+          `Measurement Protocol [${batchId}] didn't succeed.`,
+          `Get response code: ${response.status}`,
+          `response: ${response.data}`,
+        ];
+        console.error(errorMessages.join('\n'));
+        throw new Error(`Status code not 2XX`);
+      }
+      this.logger.debug(`Configuration:`, config);
+      this.logger.debug(`Input Data:   `, lines);
+      this.logger.debug(`Batch[${batchId}] status: ${response.status}`);
+      this.logger.debug(response.data);
+      // There is not enough information from the non-debug mode.
+      if (!this.debugMode) return true;
+      const failedHits = [];
+      const failedReasons = new Set();
+      const errorMessage = response.data.hitParsingResult
+          .forEach((result, index) => {
+            if (!result.valid) {
+              failedHits.push(lines[index]);
+              result.parserMessage.forEach(({description}) => {
+                failedReasons.add(description);
+              })
+            }
+          });
+      console.log(failedHits);
+      console.log(failedReasons);
+      return errorMessage;
     };
   };
 

package/src/components/firestore/access_base.js

@@ -217,11 +217,14 @@ class FirestoreAccessBase {
    * Returns whether the mode of Firestore is 'Native'.
    * @return {!Promise<boolean>}
    */
-  static isNativeMode() {
-    return new Firestore().listCollections().then(() => true).catch((error) => {
+  static async isNativeMode(projectId = process.env['GCP_PROJECT']) {
+    try {
+      await new Firestore({projectId}).listCollections();
+      return true;
+    } catch (error) {
       console.log(`In detecting Firestore mode: `, error.message);
       return false;
-    });
+    }
   };
 }
 

package/src/components/firestore/data_access_object.js

@@ -54,19 +54,22 @@ class DataAccessObject {
    * @param {string} kind The data model name.
    * @param {string} namespace The namespace of the data.
    * @param {!DataSource} dataSource The data source type.
+   * @param {string} projectId The Id of Cloud project.
    */
   constructor(kind, namespace,
-      dataSource = DataAccessObject.getDataSourceFromEnvironment()) {
+      dataSource = DataAccessObject.getDataSourceFromEnvironment(),
+      projectId = process.env['GCP_PROJECT']) {
     /** @const {string} */ this.namespace = namespace;
     /** @const {!DataSource} */ this.dataSource = dataSource;
     /** @type {!FirestoreAccessBase} */ this.accessObject = undefined;
     switch (this.dataSource) {
       case DataSource.FIRESTORE:
         this.accessObject = new NativeModeAccess(
-            `${this.namespace}/database/${kind}`);
+            `${this.namespace}/database/${kind}`, projectId);
         break;
       case DataSource.DATASTORE:
-        this.accessObject = new DatastoreModeAccess(this.namespace, kind);
+        this.accessObject = new DatastoreModeAccess(this.namespace, kind,
+            projectId);
         break;
       default:
         throw new Error(`Unknown DataSource item: ${this.dataSource}.`);

package/src/components/firestore/datastore_mode_access.js

@@ -46,10 +46,11 @@ class DatastoreModeAccess {
    * Initializes DatastoreModeAccess instance.
    * @param {string} namespace The namespace for data.
    * @param {string} kind The kind of this entity.
+   * @param {string} projectId The Id of Cloud project.
    */
-  constructor(namespace, kind) {
+  constructor(namespace, kind, projectId = process.env['GCP_PROJECT']) {
     /** @type{Datastore} */
-    this.datastore = new Datastore();
+    this.datastore = new Datastore({projectId});
     this.kind = kind;
     this.namespace = namespace;
     this.logger = getLogger('DS.ACC');

package/src/components/firestore/native_mode_access.js

@@ -41,10 +41,11 @@ class NativeModeAccess {
    * way. This constructor will check the path to make sure it presents a
    * 'collection', otherwise an Error will be thrown.
    * @param {string} path Path for the 'collection'.
+   * @param {string} projectId The Id of Cloud project.
    */
-  constructor(path) {
+  constructor(path, projectId = process.env['GCP_PROJECT']) {
     /** @type {!Firestore} */
-    this.firestore = new Firestore();
+    this.firestore = new Firestore({projectId});
     if (path.split('/').length % 2 === 0) {
       throw new Error(`Invalid path for Collection: ${path}`);
     }

package/src/components/scheduler.js

@@ -30,11 +30,11 @@ const API_VERSION = 'v1';
  * to get/pause/resume a job.
  */
 class CloudScheduler {
-  constructor() {
+  constructor(projectId = process.env['GCP_PROJECT']) {
     /** @const {!AuthClient} */
     const authClient = new AuthClient(API_SCOPES);
     this.auth = authClient.getApplicationDefaultCredentials();
-    this.projectId = process.env['GCP_PROJECT'];
+    this.projectId = projectId;
     this.instance = cloudscheduler({
       version: API_VERSION,
       auth: this.auth,

package/src/components/utils.js

@@ -23,12 +23,23 @@ const {inspect} = require('util');
 const {LoggingWinston} = require('@google-cloud/logging-winston');
 const {CloudPlatformApis} = require('../apis/cloud_platform_apis.js');
 
+/**
+ * The result of a batch of data sent to target API.
+ *
+ * @typedef {{
+ *   result: boolean,
+ *   errors: (Array<string>|undefined),
+ *   output: (Array<string>|undefined),
+ * }}
+ */
+let BatchResult;
+
 /**
  * Function which sends a batch of data. Takes two parameters:
  * {!Array<string>} Data for single request. It should be guaranteed that it
  *     doesn't exceed quota limitation.
  * {string} The tag for log.
- * @typedef {function(!Array<string>,string): !Promise<boolean>}
+ * @typedef {function(!Array<string>,string): !Promise<!BatchResult>}
  */
 let SendSingleBatch;
 
@@ -414,10 +425,12 @@ const extractObject = (paths) => {
  * status code 1 to let the invoker (the Bash installation script) know that it
  * doesn't pass.
  * @param {!Array<string>} permissions Array of permissions to check.
+ * @param {string} projectId The Id of Cloud project.
  * @return {!Promise<undefined>}
  */
-const checkPermissions = (permissions) => {
-  const cloudPlatformApis = new CloudPlatformApis();
+const checkPermissions = (permissions,
+    projectId = process.env['GCP_PROJECT']) => {
+  const cloudPlatformApis = new CloudPlatformApis(projectId);
   return cloudPlatformApis.testIamPermissions(permissions)
       .then((grantedPermissions) => {
         console.log(grantedPermissions);
@@ -461,6 +474,7 @@ const changeNamingFromSnakeToLowerCamel = (name) => {
 module.exports = {
   getLogger,
   wait,
+  BatchResult,
   SendSingleBatch,
   apiSpeedControl,
   splitArray,

package/src/components/vertex_ai.js

@@ -0,0 +1,115 @@
+// Copyright 2021 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @fileoverview Google Cloud Vertex AI API helper.
+ */
+
+'use strict';
+
+const {JobServiceClient} = require('@google-cloud/aiplatform');
+
+/**
+ * Wrapper class for AI Platform (Vertex AI) API.
+ * 1. Create or get batch prediction jobs.
+ * @see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchPredictionJob
+ */
+class VertexAi {
+
+  /**
+   * Initialize an instance.
+   * @param {{keyFilename:(string|undefined)}} options
+   */
+  constructor(options = {}) {
+    this.options = options;
+  }
+
+  /**
+   * Starts a BatchPredictionJob and returns the job name.
+   * @see https://googleapis.dev/nodejs/aiplatform/latest/v1.JobServiceClient.html#createBatchPredictionJob
+   * @see https://googleapis.dev/nodejs/aiplatform/latest/google.cloud.aiplatform.v1.BatchPredictionJob.html
+   * @param {string} projectId
+   * @param {string} location
+   * @param {string} modelId
+   * @param {google.cloud.aiplatform.v1.BatchPredictionJob.IInputConfig} inputConfig
+   * @param {google.cloud.aiplatform.v1.BatchPredictionJob.IOutputConfig} outputConfig
+   * @param {string=} displayName
+   * @return {Promise<string>} BatchPredictionJob name.
+   */
+  async batchPredict(projectId, location, modelId, inputConfig,
+      outputConfig, displayName = 'Batch Prediction Job') {
+    const jobServiceClient = this.getJobServiceClient_(location);
+    const parent = `projects/${projectId}/locations/${location}`;
+    const model = jobServiceClient.modelPath(projectId, location, modelId);
+    const batchPredictionJob = {
+      displayName,
+      model,
+      inputConfig,
+      outputConfig,
+    };
+    const request = {
+      parent,
+      batchPredictionJob,
+    };
+    const [response] = await jobServiceClient.createBatchPredictionJob(request);
+    return response.name;
+  }
+
+  /**
+   * Gets a BatchPredictionJob.
+   * @see https://googleapis.dev/nodejs/aiplatform/latest/v1.JobServiceClient.html#getBatchPredictionJob
+   * @param {string} jobName
+   * @param {(string|undefined)=} explicitLocation Location of the service
+   *     endpoint. A location will be extracted from jobName if omitted.
+   * @return {Promise<google.cloud.aiplatform.v1.BatchPredictionJob>}
+   */
+  async getBatchPredictionJob(jobName, explicitLocation = undefined) {
+    let location = explicitLocation;
+    if (!location) {
+      location = /projects\/[^\/]*\/locations\/([^\/]*)\/.*/.exec(jobName)[1];
+    }
+    const jobServiceClient = this.getJobServiceClient_(location);
+    const request = {name: jobName};
+    const [response] = await jobServiceClient.getBatchPredictionJob(request);
+    return response;
+  }
+
+  /**
+   * Gets the service endpoint for Vertex AI.
+   * @see https://cloud.google.com/vertex-ai/docs/general/locations
+   * @see https://cloud.google.com/vertex-ai/docs/reference/rest#service-endpoint
+   * @param {string} location
+   * @return {string} The service endpoint.
+   * @private
+   */
+  getServiceEndpoint_(location) {
+    return `${location}-aiplatform.googleapis.com`;
+  }
+
+  /**
+   * Gets the service for creating and managing AI Platform's jobs.
+   * @see https://googleapis.dev/nodejs/aiplatform/latest/v1.JobServiceClient.html
+   * @param {string} location
+   * @return {!JobServiceClient}
+   * @private
+   */
+  getJobServiceClient_(location) {
+    const clientOptions = {
+      apiEndpoint: this.getServiceEndpoint_(location),
+    };
+    return new JobServiceClient(clientOptions);
+  }
+}
+
+exports.VertexAi = VertexAi;