@google-cloud/nodejs-common 1.1.0 → 1.1.1-beta

package/bin/bigquery.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#######################################
+# Checks whether the BigQuery object (table or view) exists.
+# Globals:
+#  GCP_PROJECT
+#  DATASET
+#  BIGQUERY_LOG_TABLE
+# Arguments:
+#  None
+#######################################
+check_existence_in_bigquery() {
+  bq show "${1}" >/dev/null 2>&1
+  printf '%d' $?
+}
+
+#######################################
+# Creates or updates the BigQuery view.
+# Globals:
+#  GCP_PROJECT
+#  DATASET
+# Arguments:
+#  The name of view.
+#  The query of view.
+#######################################
+create_or_update_view() {
+  local viewName viewQuery
+  viewName="${1}"
+  viewQuery=${2}
+  local action="mk"
+  if [[ $(check_existence_in_bigquery "${DATASET}.${viewName}") -eq 0 ]]; then
+    action="update"
+  fi
+  bq "${action}" \
+    --use_legacy_sql=false \
+    --view "${viewQuery}" \
+    --project_id ${GCP_PROJECT} \
+    "${DATASET}.${viewName}"
+}

package/bin/google_ads.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Google Ads API version
+GOOGLE_ADS_API_VERSION=10
+
+#######################################
+# Verify whether the current OAuth token, CID and developer token can work.
+# Globals:
+#   None
+# Arguments:
+#   MCC CID
+#   Developer token
+#######################################
+validate_googleads_account() {
+  local cid developerToken accessToken request response
+  cid=${1}
+  developerToken=${2}
+  accessToken=$(get_oauth_access_token)
+  request=(
+    -H "Accept: application/json"
+    -H "Content-Type: application/json"
+    -H "developer-token: ${developerToken}"
+    -H "Authorization: Bearer ${accessToken}"
+    -X POST "https://googleads.googleapis.com/v${GOOGLE_ADS_API_VERSION}/customers/${cid}/googleAds:search"
+    -d '{"query": "SELECT customer.id FROM customer"}'
+  )
+  response=$(curl "${request[@]}" 2>/dev/null)
+  local errorCode errorMessage
+  errorCode=$(get_value_from_json_string "${response}" "error.code")
+  if [[ -n "${errorCode}" ]]; then
+    errorMessage=$(get_value_from_json_string "${response}" "error.message")
+    printf '%s\n' "Validate failed: ${errorMessage}" >&2
+    return 1
+  fi
+  return 0
+}
+
+#######################################
+# Let user input MCC CID and developer token for cronjob(s).
+# Globals:
+#   MCC_CIDS
+#   DEVELOPER_TOKEN
+# Arguments:
+#   None
+#######################################
+set_google_ads_account() {
+  printf '%s\n' "Setting up Google Ads account information..."
+  local developToken mccCids
+  while :; do
+    # Developer token
+    while [[ -z ${developToken} ]]; do
+      printf '%s' "  Enter the developer token[${DEVELOPER_TOKEN}]: "
+      read -r input
+      developToken="${input:-${DEVELOPER_TOKEN}}"
+    done
+    DEVELOPER_TOKEN="${developToken}"
+    mccCids=""
+    # MCC CIDs
+    while :; do
+      printf '%s' "  Enter the MCC CID: "
+      read -r input
+      if [[ -z ${input} ]]; then
+        continue
+      fi
+      input="$(printf '%s' "${input}" | sed -r 's/-//g')"
+      printf '%s' "    validating ${input}...... "
+      validate_googleads_account ${input} ${DEVELOPER_TOKEN}
+      if [[ $? -eq 1 ]]; then
+        printf '%s\n' "failed.
+      Press 'd' to re-enter developer token ["${developToken}"] or
+            'C' to continue with this MCC CID or
+            any other key to enter another MCC CID..."
+        local any
+        read -n1 -s any
+        if [[ "${any}" == "d" ]]; then
+          developToken=""
+          continue 2
+        elif [[ "${any}" == "C" ]]; then
+          printf '%s\n' "WARNING! Continue with FAILED MCC ${input}."
+        else
+          continue
+        fi
+      else
+        printf '%s\n' "succeeded."
+      fi
+      mccCids+=",${input}"
+      printf '%s' "  Do you want to add another MCC CID? [Y/n]: "
+      read -r input
+      if [[ ${input} == 'n' || ${input} == 'N' ]]; then
+        break
+      fi
+    done
+    # Left Shift one position to remove the first comma.
+    # After shifting, MCC_CIDS would like "11111,22222".
+    MCC_CIDS="${mccCids:1}"
+    printf '%s\n' "Using Google Ads MCC CIDs: ${MCC_CIDS}."
+    break
+  done
+}
+
+printf '%s\n' "Google Ads Bash Library is loaded."

package/bin/install_functions.sh

@@ -573,7 +573,7 @@ API(s):"
     printf '%s\n'  "OK. OAuth is required by selected API(s)."
     return 0
   fi
-  printf '%s\n' "Selected API(s) require authentication. Please select \
+  printf '%s\n' "Selected API(s) require authentication. Choose the \
 authentication method:"
   local auths=("Service Account (recommended)" "OAuth")
   select auth in "${auths[@]}"; do
@@ -604,28 +604,40 @@ check_permissions() {
   (( STEP += 1 ))
   printf '%s\n' "Step ${STEP}: Checking current user's access..."
   local role error
-  for role in "${!GOOGLE_CLOUD_PERMISSIONS[@]}"; do
-    printf '%s'  "  Checking permissions for ${role}... "
-    local permissions
-    permissions=(${GOOGLE_CLOUD_PERMISSIONS[${role}]})
-    local missed
-    missed=$(get_number_of_missed_permissions "${permissions[@]}")
-    if [[ $missed -gt 0 ]]; then
-      message="missed ${missed}, failed"
-      error=1
+  while :; do
+    error=0
+    for role in "${!GOOGLE_CLOUD_PERMISSIONS[@]}"; do
+      printf '%s'  "  Checking permissions for ${role}... "
+      local permissions
+      permissions=(${GOOGLE_CLOUD_PERMISSIONS[${role}]})
+      local missed
+      missed=$(get_number_of_missed_permissions "${permissions[@]}")
+      if [[ $missed -gt 0 ]]; then
+        message="missed ${missed}, failed"
+        error=1
+      else
+        message='successfully'
+      fi
+      printf '%s\n' " ${message}."
+    done
+    if [[ ${error} -gt 0 ]]; then
+      printf '%s\n' "Permissions check failed."
+      printf '%s\n' "Would you like to login with another account? [Y/n]: "
+      local reLogin
+      read -r reLogin
+      reLogin=${reLogin:-"Y"}
+      if [[ ${reLogin} == "Y" || ${reLogin} == "y" ]]; then
+        gcloud auth login
+        continue
+      else
+        return 1
+      fi
     else
-      message='successfully'
+      echo "OK. Permissions check passed for Google Cloud project \
+  [${GCP_PROJECT}]."
+      return 0
     fi
-    printf '%s\n' " ${message}."
   done
-  if [[ ${error} -gt 0 ]]; then
-    printf '%s\n' "Permissions check failed."
-    return 1
-  else
-    echo "OK. Permissions check passed for Google Cloud project \
-[${GCP_PROJECT}]."
-    return 0
-  fi
 }
 
 #######################################
@@ -1129,6 +1141,7 @@ Project. Continuing to enter another bucket..."
     fi
   done
   declare -g "${gcsName}=${bucket}"
+  confirm_bucket_lifecycle "${bucket}"
 }
 
 #######################################
@@ -1149,6 +1162,58 @@ confirm_bucket_with_location() {
   confirm_located_bucket ${gcsName} ${locationName}
 }
 
+#######################################
+# Manage the lifecycle of a GCS bucket: setting or removing the GCS lifecycle
+# rule of 'age'.
+# See: https://cloud.google.com/storage/docs/lifecycle#age
+# Arguments:
+#   Bucket name
+#######################################
+confirm_bucket_lifecycle() {
+  local bucket bucketMetadata lifecycle
+  bucket="${1}"
+  bucketMetadata="$(get_bucket_metadata "${bucket}")"
+  lifecycle="$(get_value_from_json_string "${bucketMetadata}" "lifecycle")"
+  if [[ -n "${lifecycle}" ]]; then
+    printf '%s\n' "There are lifecycle rules in this bucket: ${lifecycle}."
+    printf '%s' "Would you like to overwrite it? [N/y]:"
+    local confirmContinue
+    read -r confirmContinue
+    confirmContinue=${confirmContinue:-"N"}
+    if [[ ${confirmContinue} == "N" || ${confirmContinue} == "n" ]]; then
+      return 0
+    fi
+  else
+    printf '%s\n' "There is no lifecycle rules in this bucket."
+    printf '%s' "Would you like to create it? [Y/n]:"
+    local confirmContinue
+    read -r confirmContinue
+    confirmContinue=${confirmContinue:-"Y"}
+    if [[ ${confirmContinue} == "N" || ${confirmContinue} == "n" ]]; then
+      return 0
+    fi
+  fi
+  while :; do
+    printf '%s' "Enter the number of days that a file will be kept before it \
+is automatically removed in the bucket[${bucket}]. (enter 0 to remove all \
+existing lifecycle rules): "
+    local days
+    read -r days
+    days=${days}
+    if [[ "${days}" =~ ^[0-9]+$ ]]; then
+      local lifecycle
+      if [[ "${days}" == "0" ]]; then
+        lifecycle="{}"
+      else
+        lifecycle='{"rule":[{"action":{"type":"Delete"},"condition":{"age":'\
+"${days}}}]}"
+      fi
+      gsutil lifecycle set /dev/stdin gs://${bucket} <<< ${lifecycle}
+      return $?
+    fi
+  done
+}
+
 #######################################
 # Confirm the monitored folder.
 # Globals:
@@ -1507,6 +1572,7 @@ set_authentication_env_for_cloud_functions() {
 create_or_update_cloud_scheduler_for_pubsub(){
   check_authentication
   quit_if_failed $?
+  local location_flag="--location=${REGION}"
   local scheduler_flag=()
   scheduler_flag+=(--schedule="$2")
   scheduler_flag+=(--time-zone="$3")
@@ -1514,13 +1580,13 @@ create_or_update_cloud_scheduler_for_pubsub(){
   scheduler_flag+=(--message-body="$5")
   local exist_job
   exist_job=($(gcloud scheduler jobs list --filter="name~${1}" \
---format="value(state)"))
+--format="value(state)" "${location_flag}"))
   local action needPause
   if [[ ${#exist_job[@]} -gt 0 ]]; then
     action="update"
     scheduler_flag+=(--update-attributes=$6)
     if [[ "${exist_job[0]}" == "PAUSED" ]]; then
-      gcloud scheduler jobs resume "${1}"
+      gcloud scheduler jobs resume "${1}" "${location_flag}"
       if [[ $? -gt 0 ]]; then
         printf '%s\n' "Failed to resume paused Cloud Scheduler job [${1}]."
         return 1
@@ -1531,9 +1597,9 @@ create_or_update_cloud_scheduler_for_pubsub(){
     action="create"
     scheduler_flag+=(--attributes=$6)
   fi
-  gcloud scheduler jobs ${action} pubsub "$1" "${scheduler_flag[@]}"
+  gcloud scheduler jobs ${action} pubsub "$1" "${scheduler_flag[@]}" "${location_flag}"
   if [[ "${needPause}" == "true" ]]; then
-      gcloud scheduler jobs pause "${1}"
+      gcloud scheduler jobs pause "${1}" "${location_flag}"
   fi
 }
 
@@ -1921,3 +1987,10 @@ join_string_array() {
   shift
   printf %s "$first" "${@/#/$separator}"
 }
+
+# Import other bash files.
+_SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${_SELF}/google_ads.sh"
+source "${_SELF}/bigquery.sh"
+
+printf '%s\n' "Common Bash Library is loaded."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "1.1.0",
+  "version": "1.1.1-beta",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -24,6 +24,7 @@
     "@google-cloud/logging-winston": "^4.2.2",
     "@google-cloud/pubsub": "^2.19.0",
     "@google-cloud/storage": "^5.18.3",
+    "@google-cloud/scheduler": "^2.3.0",
     "gaxios": "^4.3.2",
     "google-ads-api": "^10.0.1",
     "google-ads-node":"^8.0.1",

package/src/components/automl.js

@@ -17,15 +17,9 @@
  */
 
 'use strict';
-const {request} = require('gaxios');
-const {JWT, Compute} = require('google-auth-library');
+const { ClientOptions } = require('google-gax');
 const automl = require('@google-cloud/automl');
 const google = automl.protos.google;
-const AuthClient = require('../apis/auth_client.js');
-
-const API_SCOPES = Object.freeze([
-  'https://www.googleapis.com/auth/cloud-platform',
-]);
 
 /**
  * For version `v1beta1`, BigQuery can be the source and destination.
@@ -40,46 +34,49 @@ const API_SCOPES = Object.freeze([
  * @type {string}
  */
 const API_VERSION = 'v1beta1';
-const PredictionServiceClient = automl[API_VERSION].PredictionServiceClient;
+const { PredictionServiceClient } = automl[API_VERSION];
 
 /**
- * AutoML Tables API doesn't have fully cloud client library support yet. This
- * class acts as the single client class for different API requests of AutoML
- * Tables API, including:
- * 1. 'batch predict' based on Google Cloud Client Library,
- * https://googleapis.dev/nodejs/automl/latest/index.html
- * 2. 'get Operations' based on REST API, see:
- * https://cloud.google.com/automl/docs/reference/rest/v1/projects.locations.operations/get
- * https://cloud.google.com/automl-tables/docs/long-operations#get-operation
+ * AutoML Tables API class for:
+ * 1. start a batch predict job:
+ * @see https://cloud.google.com/nodejs/docs/reference/automl/latest/automl/v1beta1.predictionserviceclient#_google_cloud_automl_v1beta1_PredictionServiceClient_batchPredict_member_3_
+ * 2. get the status of an operation:
+ * @see https://cloud.google.com/nodejs/docs/reference/automl/latest/automl/v1beta1.predictionserviceclient#_google_cloud_automl_v1beta1_PredictionServiceClient_checkBatchPredictProgress_member_1_
  */
 class AutoMl {
-
   /**
    * Initialize an instance.
-   * @param {{keyFilename:(string|undefined)}} options
+   * @param {ClientOptions=} options
    */
   constructor(options = {}) {
-    this.options = options;
+    this.client = new PredictionServiceClient(options);
   }
 
   /**
    * Batch predicts based on Google Cloud Client Library.
-   * See https://googleapis.dev/nodejs/automl/latest/index.html
    * @param {string} projectId
    * @param {string} computeRegion
    * @param {string} modelId
-   * @param {google.cloud.automl.v1.IBatchPredictInputConfig} inputConfig
-   * @param {google.cloud.automl.v1.IBatchPredictOutputConfig} outputConfig
+   * @param {google.cloud.automl.v1beta1.IBatchPredictInputConfig} inputConfig
+   * @param {google.cloud.automl.v1beta1.IBatchPredictOutputConfig} outputConfig
    * @return {Promise<string>} Predict operation name.
    */
-  async batchPredict(projectId, computeRegion, modelId, inputConfig,
-      outputConfig) {
-    const client = new PredictionServiceClient(this.options);
-    const modelFullId = client.modelPath(projectId, computeRegion, modelId);
-    const responses = await client.batchPredict({
+  async batchPredict(
+    projectId,
+    computeRegion,
+    modelId,
+    inputConfig,
+    outputConfig
+  ) {
+    const modelFullId = this.client.modelPath(
+      projectId,
+      computeRegion,
+      modelId
+    );
+    const responses = await this.client.batchPredict({
       name: modelFullId,
-      inputConfig: inputConfig,
-      outputConfig: outputConfig,
+      inputConfig,
+      outputConfig,
     });
     const operation = responses[1];
     console.log(`Operation name: ${operation.name}`);
@@ -87,37 +84,20 @@ class AutoMl {
   }
 
   /**
-   * Gets Operations based on REST API, see:
-   * https://cloud.google.com/automl/docs/reference/rest/v1/projects.locations.operations/get
-   * https://cloud.google.com/automl-tables/docs/long-operations#get-operation
+   * Gets status of an operation.
    * @param {string} operationName
-   * @return {Promise<google.longrunning.Operation>}
+   * @return {Promise<{{
+   *   done: boolean,
+   *   error: Error|undefined,
+   *   metadata: OperationMetadata,
+   *   name: string,
+   * }}>}
    */
   async getOperation(operationName) {
-    const url = `https://automl.googleapis.com/${API_VERSION}/${operationName}`;
-    const headers = await this.getAuthClient_().getRequestHeaders();
-    const requestOptions = {
-      method: 'GET',
-      headers,
-      url,
-    };
-    const response = await request(requestOptions);
-    return response.data;
+    const response = await this.client.checkBatchPredictProgress(operationName);
+    const { done, error, metadata, name } = response;
+    return { done, error, metadata, name };
   }
-
-  /**
-   * Gets authentication client of AutoML API.
-   * This API belongs to GCP, however it is not fully supported by Google Cloud
-   * Client Library. So we need to manage some functions on its REST API,
-   * in the way like we invoke other external APIs.
-   * @returns {(!JWT|!Compute)}
-   * @private
-   */
-  getAuthClient_() {
-    /** @const {!AuthClient} */ const authClient = new AuthClient(API_SCOPES);
-    return authClient.getApplicationDefaultCredentials();
-  }
-
 }
 
 exports.AutoMl = AutoMl;

package/src/components/scheduler.js

@@ -30,11 +30,36 @@ const API_VERSION = 'v1';
  * to get/pause/resume a job.
  */
 class CloudScheduler {
-  constructor(projectId = process.env['GCP_PROJECT']) {
-    /** @const {!AuthClient} */
-    const authClient = new AuthClient(API_SCOPES);
-    this.auth = authClient.getApplicationDefaultCredentials();
-    this.projectId = projectId;
+  constructor(env = process.env, options = {}) {
+    if (!options.authClient) {
+      /** @const {!AuthClient} */
+      const authClient = new AuthClient(API_SCOPES, env);
+      /**
+       * By default, `AuthClient` (getDefaultAuth()) will return an auth client
+       * based on the settings in ENV while the OAuth is the most preferred.
+       * This works for most of the external API clients (in the '../apis'
+       * folder), however this won't work in the Cloud Functions, as those OAuth
+       * token usually won't have enough permission to invoke Google Cloud API.
+       * Using the method `getApplicationDefaultCredentials` to force
+       * `AuthClient` return an ADC auth client, which will work in the Cloud.
+       *
+       * Cloud Scheduler API Client Library is used here as Cloud Client Library
+       * is still at beta stage. (For the difference, see
+       * https://cloud.google.com/apis/docs/client-libraries-explained)
+       * Eventually, when we migrate this to the cloud client library, which
+       * automatically takes ADC as the authentication method, the 'AuthClient'
+       * is not required here and can be removed.
+       */
+      this.auth = authClient.getApplicationDefaultCredentials();
+    } else {
+      /**
+       * `authClient` can be consumed by cloud client library as the auth
+       * client. By passing this in, we can offer more flexible auth clients in
+       * test cases for API client library and cloud client library in future.
+       */
+      this.auth = options.authClient;
+    }
+    this.projectId = env['GCP_PROJECT'];
     this.instance = cloudscheduler({
       version: API_VERSION,
       auth: this.auth,
@@ -113,8 +138,8 @@ class CloudScheduler {
    */
   async getJobs_(name, targetLocations = undefined) {
     const regex = new RegExp(`/jobs/${name}$`);
-    const jobs = (await this.listJobs_(targetLocations)).filter(
-        (job) => regex.test(job));
+    const allJobs = await this.listJobs_(targetLocations);
+    const jobs = allJobs.filter((job) => regex.test(job));
     if (jobs.length === 0) console.error(`Can not find job: ${name}`);
     return jobs;
   }
@@ -134,23 +159,10 @@ class CloudScheduler {
     const projectId = await this.getProjectId_();
     const requestPrefix = `projects/${projectId}/locations`;
     const jobs = locations.map(async (location) => {
-      const request = {parent: `${requestPrefix}/${location}`};
-      try {
-        const response = await this.instance.projects.locations.jobs.list(
-            request);
-        return response.data.jobs.map((job) => job.name);
-      } catch (error) {
-        // Currently, listLocations always returns an array with one location.
-        // Not sure whether this will be changed or not in future. If one day
-        // the Cloud Scheduler let users to select a location for a job, then
-        // it may return multiple locations when listLocation, however the
-        // target job may exist in one of the locations. In this case, when we
-        // iterate the job with all possible locations to get the complete job
-        // path to operate, it will likely generate an error for those wrong
-        // location(s). So set the try-catch here to handle this situation.
-        console.error(error.message);
-        return [];
-      }
+      const request = { parent: `${requestPrefix}/${location}` };
+      const response = await this.instance.projects.locations.jobs.list(request);
+      if (!response.data.jobs) return [];
+      return response.data.jobs.map((job) => job.name);
     });
     // Waits for all jobs names and flattens nested job name arrays, however
     // there is no 'flat' available in current Cloud Functions runtime.

package/src/components/vertex_ai.js

@@ -18,6 +18,7 @@
 
 'use strict';
 
+const { ClientOptions } = require('google-gax');
 const {JobServiceClient} = require('@google-cloud/aiplatform');
 
 /**
@@ -29,7 +30,7 @@ class VertexAi {
 
   /**
    * Initialize an instance.
-   * @param {{keyFilename:(string|undefined)}} options
+   * @param {ClientOptions=} options
    */
   constructor(options = {}) {
     this.options = options;
@@ -105,9 +106,9 @@ class VertexAi {
    * @private
    */
   getJobServiceClient_(location) {
-    const clientOptions = {
+    const clientOptions = Object.assign({}, this.options, {
       apiEndpoint: this.getServiceEndpoint_(location),
-    };
+    });
     return new JobServiceClient(clientOptions);
   }
 }