@google-cloud/nodejs-common 0.9.2-beta → 0.9.2-beta2

package/bin/install_functions.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Cloud Functions Runtime Environment.
-CF_RUNTIME="${CF_RUNTIME:=nodejs14}"
+CF_RUNTIME="${CF_RUNTIME:=nodejs10}"
 
 # Counter for steps.
 STEP=0
@@ -138,14 +138,6 @@ EXTERNAL_API_SCOPES=(
 # Enabled APIs' OAuth scopes.
 ENABLED_OAUTH_SCOPES=()
 
-# No explicit service account is required.
-# https://cloud.google.com/iam/docs/understanding-roles#service-accounts-roles
-#declare -A GOOGLE_SERVICE_ACCOUNT_PERMISSIONS
-#GOOGLE_SERVICE_ACCOUNT_PERMISSIONS=(
-#  ["Service Account Admin"]="iam.serviceAccounts.create"
-#  ["Service Account Key Admin"]="iam.serviceAccounts.create"
-#)
-
 # Preparation functions.
 #######################################
 # Mimic a Cloud Shell environment to enable local running.
@@ -590,11 +582,6 @@ authentication method:"
     printf '%s\n' "... OAuth is selected."
   else
     NEED_SERVICE_ACCOUNT="true"
-#    local role
-#    for role in "${!GOOGLE_SERVICE_ACCOUNT_PERMISSIONS[@]}"; do
-#      GOOGLE_CLOUD_PERMISSIONS["${role}"]=\
-#"${GOOGLE_SERVICE_ACCOUNT_PERMISSIONS["${role}"]}"
-#    done
     printf '%s\n' "... Service Account is selected."
   fi
 }
@@ -1192,6 +1179,87 @@ EOF
   printf '%s\n' "OK. Continue with monitored folder [${folder}]."
 }
 
+#######################################
+# Create or update a Log router sink.
+# Globals:
+#   None
+# Arguments:
+#   Name of the sink
+#   Filter conditions
+#   Sink destination
+#######################################
+create_or_update_sink() {
+  local sinkName=${1}
+  local logFilter=${2}
+  local sinkDestAndFlags=(${3})
+  local existingFilter
+  existingFilter=$(gcloud logging sinks list --filter="name:${sinkName}" \
+--format="value(filter)")
+  if [[ "${existingFilter}" != "${logFilter}" ]]; then
+    local action
+    if [[ -z "${existingFilter}" ]]; then
+      action="create"
+      printf '%s\n' "  Logging Export [${sinkName}] doesn't exist. Creating..."
+    else
+      action="update"
+      printf '%s\n' "  Logging Export [${sinkName}] exists with a different \
+filter. Updating..."
+    fi
+    gcloud -q logging sinks ${action} "${sinkName}" "${sinkDestAndFlags[@]}" \
+--log-filter="${logFilter}"
+  else
+    printf '%s\n' "  Logging Export [${sinkName}] exists. Continue..."
+  fi
+  if [[ $? -gt 0 ]];then
+    printf '%s\n' "Failed to create or update Logs router sink."
+    return 1
+  fi
+}
+
+#######################################
+# Confirm the service account of the given sink has proper permission to dump
+# the logs.
+# Globals:
+#   None
+# Arguments:
+#   Name of the sink
+#   Bindings role, e.g. "pubsub.publisher" or "bigquery.dataEditor"
+#   Role name, e.g. "Pub/Sub Publisher" or "BigQuery Data Editor"
+#######################################
+confirm_sink_service_account_permission() {
+  local sinkName=${1}
+  local bindingsRole=${2}
+  local roleName=${3}
+  local serviceAccount existingRole
+  serviceAccount=$(gcloud logging sinks describe "${sinkName}" \
+--format="get(writerIdentity)")
+  while :;do
+    printf '%s\n' "  Checking the role of the sink's service account \
+[${serviceAccount}]..."
+    existingRole=$(gcloud projects get-iam-policy "${GCP_PROJECT}" \
+--flatten=bindings --filter="bindings.members:${serviceAccount} AND \
+bindings.role:roles/${bindingsRole}" --format="get(bindings.members)")
+    if [[ -z "${existingRole}" ]];then
+      printf '%s\n'  "  Granting Role '${roleName}' to the service \
+account..."
+      gcloud -q projects add-iam-policy-binding "${GCP_PROJECT}" --member \
+"${serviceAccount}" --role roles/${bindingsRole}
+      if [[ $? -gt 0 ]];then
+        printf '%s\n' "Failed to grant the role. Use this link \
+https://console.cloud.google.com/iam-admin/iam?project=${GCP_PROJECT} to \
+manually grant Role '${roleName}' to ${serviceAccount}."
+        printf '%s' "Press any key to continue after you grant the access..."
+        local any
+        read -n1 -s any
+        continue
+      fi
+    else
+      printf '%s\n'  "  The role has already been granted."
+      return 0
+    fi
+  done
+}
+
 #######################################
 # Save the configuration to a local file.
 # Globals:
@@ -1232,7 +1300,9 @@ save_config() {
 # For service account, given that Cloud Functions can extend the authorized API
 # scopes now, it will use the default service account rather than an explicit
 # service account with the key file downloaded. By doing so, we can reduce the
-# risk of leaking service account key.
+# risk of leaking service account key. If there is a service key in the current
+# folder from previous installation, the code will continue using it, otherwise
+# it will use the Cloud Functions' default service account.
 # For OAuth 2.0, guide user to complete OAuth authentication and save the
 # refresh token.
 # Globals:
@@ -1242,155 +1312,11 @@ save_config() {
 #   None
 #######################################
 do_authentication(){
-# If there is a service key in the current folder from previous installation,
-# the code will continue using it, otherwise it will use the Cloud Functions'
-# default service account.
-#  if [[ ${NEED_SERVICE_ACCOUNT} == "true" ]]; then
-#    download_service_account_key
-#  fi
   if [[ ${NEED_OAUTH} == "true" ]]; then
     do_oauth
   fi
 }
 
-#######################################
-# Download a service account key file and save as `$SA_KEY_FILE`.
-# Globals:
-#   SA_NAME
-#   GCP_PROJECT
-#   SA_KEY_FILE
-# Arguments:
-#   None
-# Returns:
-#   0 if service key files exists or created, non-zero on error.
-#######################################
-download_service_account_key() {
-  (( STEP += 1 ))
-  printf '%s\n' "Step ${STEP}: Downloading the key file for the service \
-account..."
-  if [[ -z ${SA_NAME} ]];then
-    confirm_service_account
-  fi
-  local suffix exist
-  suffix=$(get_sa_domain_from_gcp_id "${GCP_PROJECT}")
-  local email="${SA_NAME}@${suffix}"
-  local prompt="Would you like to download the key file for [${email}] and \
-save it as ${SA_KEY_FILE}? [Y/n]: "
-  local default_value="y"
-  if [[ -f "${SA_KEY_FILE}" && -s "${SA_KEY_FILE}" ]]; then
-    exist=$(get_value_from_json_file ${SA_KEY_FILE} 'client_email' 2>&1)
-    if [[ ${exist} =~ .*("@${suffix}") ]]; then
-      prompt="A key file for [${exist}] with the key ID '\
-$(get_value_from_json_file ${SA_KEY_FILE} 'private_key_id') already exists'. \
-Would you like to create a new key to overwrite it? [N/y]: "
-      default_value="n"
-    fi
-  fi
-  printf '%s' "${prompt}"
-  local input
-  read -r input
-  input=${input:-"${default_value}"}
-  if [[ ${input} == 'y' || ${input} == 'Y' ]];then
-    printf '%s\n' "Downloading a new key file for [${email}]..."
-    gcloud iam service-accounts keys create "${SA_KEY_FILE}" --iam-account \
-"${email}"
-    if [[ $? -gt 0 ]]; then
-      printf '%s\n' "Failed to download new key files for [${email}]."
-      return 1
-    else
-      printf '%s\n' "OK. New key file is saved at [${SA_KEY_FILE}]."
-      return 0
-    fi
-  else
-    printf '%s\n' "Skipped downloading new key file. See \
-https://cloud.google.com/iam/docs/creating-managing-service-account-keys \
-to learn more about service account key files."
-    return 0
-  fi
-}
-
-#######################################
-# Make sure a service account for this integration exists and set the email of
-# the service account to the global variable `SA_NAME`.
-# Globals:
-#   GCP_PROJECT
-#   SA_KEY_FILE
-#   SA_NAME
-#   DEFAULT_SERVICE_ACCOUNT
-# Arguments:
-#   None
-#######################################
-confirm_service_account() {
-  cat <<EOF
-  Some external APIs might require authentication based on OAuth or \
-JWT(service account), for example, Google Analytics or Campaign Manager. \
-In this step, you prepare the service account. For more information, see \
-https://cloud.google.com/iam/docs/creating-managing-service-accounts
-EOF
-
-  local suffix
-  suffix=$(get_sa_domain_from_gcp_id "${GCP_PROJECT}")
-  local email
-  if [[ -f "${SA_KEY_FILE}" && -s "${SA_KEY_FILE}" ]]; then
-    email=$(get_value_from_json_file "${SA_KEY_FILE}" 'client_email')
-    if [[ ${email} =~ .*("@${suffix}") ]]; then
-      printf '%s' "A key file for service account [${email}] already exists. \
-Would you like to create a new service account? [N/y]: "
-      local input
-      read -r input
-      if [[ ${input} != 'y' && ${input} != 'Y' ]]; then
-        printf '%s\n' "OK. Will use existing service account [${email}]."
-        SA_NAME=$(printf "${email}" | cut -d@ -f1)
-        return 0
-      fi
-    fi
-  fi
-
-  SA_NAME="${SA_NAME:-"${PROJECT_NAMESPACE}-api"}"
-  while :; do
-    printf '%s' "Enter the name of service account [${SA_NAME}]: "
-    local input sa_elements=() sa
-    read -r input
-    input=${input:-"${SA_NAME}"}
-    IFS='@' read -a sa_elements <<< "${input}"
-    if [[ ${#sa_elements[@]} == 1 ]]; then
-      echo "  Append default suffix to service account name and get: ${email}"
-      sa="${input}"
-      email="${sa}@${suffix}"
-    else
-      if [[ ${sa_elements[1]} != "${suffix}" ]]; then
-        printf '%s\n' "  Error: Service account domain name ${sa_elements[1]} \
-doesn't belong to the current project. The service account domain name for the \
-current project should be: ${suffix}."
-        continue
-      fi
-      sa="${sa_elements[0]}"
-      email="${input}"
-    fi
-
-    printf '%s\n' "Checking the existence of the service account [${email}]..."
-    if ! result=$(gcloud iam service-accounts describe "${email}" 2>&1); then
-      printf '%s\n' "  Service account [${email}] does not exist. Trying to \
-create..."
-      gcloud iam service-accounts create "${sa}" --display-name \
-"Tentacles API requester"
-      if [[ $? -gt 0 ]]; then
-        printf '%s\n' "Creating the service account [${email}] failed. Please \
-try again..."
-      else
-        printf '%s\n' "The service account [${email}] was successfully created."
-        SA_NAME=${sa}
-        break
-      fi
-    else
-      printf ' found.\n'
-      SA_NAME=${sa}
-      break
-    fi
-  done
-  printf '%s\n' "OK. Service account [${SA_NAME}] is ready."
-}
-
 #######################################
 # Guide an OAuth process and save the token to file.
 # The whole process will be:
@@ -1517,87 +1443,6 @@ EOF
   done
 }
 
-#######################################
-# Create or update a Log router sink.
-# Globals:
-#   None
-# Arguments:
-#   Name of the sink
-#   Filter conditions
-#   Sink destination
-#######################################
-create_or_update_sink() {
-  local sinkName=${1}
-  local logFilter=${2}
-  local sinkDestAndFlags=(${3})
-  local existingFilter
-  existingFilter=$(gcloud logging sinks list --filter="name:${sinkName}" \
---format="value(filter)")
-  if [[ "${existingFilter}" != "${logFilter}" ]]; then
-    local action
-    if [[ -z "${existingFilter}" ]]; then
-      action="create"
-      printf '%s\n' "  Logging Export [${sinkName}] doesn't exist. Creating..."
-    else
-      action="update"
-      printf '%s\n' "  Logging Export [${sinkName}] exists with a different \
-filter. Updating..."
-    fi
-    gcloud -q logging sinks ${action} "${sinkName}" "${sinkDestAndFlags[@]}" \
---log-filter="${logFilter}"
-  else
-    printf '%s\n' "  Logging Export [${sinkName}] exists. Continue..."
-  fi
-  if [[ $? -gt 0 ]];then
-    printf '%s\n' "Failed to create or update Logs router sink."
-    return 1
-  fi
-}
-
-#######################################
-# Confirm the service account of the given sink has proper permission to dump
-# the logs.
-# Globals:
-#   None
-# Arguments:
-#   Name of the sink
-#   Bindings role, e.g. "pubsub.publisher" or "bigquery.dataEditor"
-#   Role name, e.g. "Pub/Sub Publisher" or "BigQuery Data Editor"
-#######################################
-confirm_sink_service_account_permission() {
-  local sinkName=${1}
-  local bindingsRole=${2}
-  local roleName=${3}
-  local serviceAccount existingRole
-  serviceAccount=$(gcloud logging sinks describe "${sinkName}" \
---format="get(writerIdentity)")
-  while :;do
-    printf '%s\n' "  Checking the role of the sink's service account \
-[${serviceAccount}]..."
-    existingRole=$(gcloud projects get-iam-policy "${GCP_PROJECT}" \
---flatten=bindings --filter="bindings.members:${serviceAccount} AND \
-bindings.role:roles/${bindingsRole}" --format="get(bindings.members)")
-    if [[ -z "${existingRole}" ]];then
-      printf '%s\n'  "  Granting Role '${roleName}' to the service \
-account..."
-      gcloud -q projects add-iam-policy-binding "${GCP_PROJECT}" --member \
-"${serviceAccount}" --role roles/bigquery.dataEditor
-      if [[ $? -gt 0 ]];then
-        printf '%s\n' "Failed to grant the role. Use this link \
-https://console.cloud.google.com/iam-admin/iam?project=${GCP_PROJECT} to \
-manually grant Role '${roleName}' to ${serviceAccount}."
-        printf '%s' "Press any key to continue after you grant the access..."
-        local any
-        read -n1 -s any
-        continue
-      fi
-    else
-      printf '%s\n'  "  The role has already been granted."
-      return 0
-    fi
-  done
-}
-
 #######################################
 # Set default configuration for a Cloud Functions.
 # Globals:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "0.9.2-beta",
+  "version": "0.9.2-beta2",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",

package/src/apis/analytics.js

@@ -23,7 +23,7 @@ const stream = require('stream');
 const {google} = require('googleapis');
 const {Schema$Upload} = google.analytics;
 const AuthClient = require('./auth_client.js');
-const {wait, getLogger} = require('../components/utils.js');
+const {wait, getLogger, BatchResult} = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
   'https://www.googleapis.com/auth/analytics',
@@ -81,8 +81,7 @@ class Analytics {
    * @param {string|!stream.Readable} data A string or a stream to be uploaded.
    * @param {!DataImportConfig} config GA data import configuration.
    * @param {string=} batchId A tag for log.
-   * @return {!Promise<boolean>} Promise returning whether data import
-   *     succeeded.
+   * @return {!Promise<!BatchResult>}
    */
   async uploadData(data, config, batchId = 'unnamed') {
     const uploadConfig = Object.assign(
@@ -100,29 +99,38 @@ class Analytics {
     this.logger.debug('Response: ', response);
     const job = /** @type {Schema$Upload} */ response.data;
     const uploadId = (/** @type {Schema$Upload} */job).id;
-    console.log(`Task [${batchId}] creates GA Data import job: ${uploadId}`);
+    this.logger.info(
+        `Task [${batchId}] creates GA Data import job: ${uploadId}`);
     const jobConfig = Object.assign({uploadId}, config);
     const result = await Promise.race([
       this.checkJobStatus(jobConfig),
       wait(8 * 60 * 1000, job),  // wait up to 8 minutes here
     ]);
+    /** @type {BatchResult} */ const batchResult = {};
     switch ((/** @type {Schema$Upload} */ result).status) {
       case 'FAILED':
         this.logger.error('GA Data Import failed', result);
-        const errors = result.errors || [`Unknown reason. ID: ${uploadId}`];
-        throw new Error(errors.join('\n'));
+        batchResult.result = false;
+        batchResult.errors = result.errors
+            || [`Unknown reason. ID: ${uploadId}`];
+        break;
       case 'COMPLETED':
         this.logger.info(`GA Data Import job[${uploadId}] completed.`);
         this.logger.debug('Response: ', result);
-        return true;
+        batchResult.result = true;
+        break;
       case 'PENDING':
         this.logger.info('GA Data Import pending.', result);
         this.logger.info('Still will return true here.');
-        return true;
+        batchResult.result = true;
+        break;
       default:
         this.logger.error('Unknown results of GA Data Import: ', result);
-        throw new Error(`Unknown status. ID: ${uploadId}`);
+        batchResult.result = false;
+        batchResult.errors = [`Unknown status. ID: ${uploadId}`];
     }
+    console.log(batchResult);
+    return batchResult;
   }
 
   /**

package/src/apis/auth_client.js

@@ -55,6 +55,8 @@ class AuthClient {
   /**
    * Create a new instance with given API scopes.
    * @param {string|!Array<string>|!ReadonlyArray<string>} scopes
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
    */
   constructor(scopes, env = process.env) {
     this.scopes = scopes;
@@ -152,7 +154,7 @@ class AuthClient {
  * Returns the full path of a existent file whose path (relative or
  * absolute) is the value of the given environment variable.
  * @param {string} envName The name of environment variable for the file path.
- * @param {!Object<string,string>} env The environment object to hold env
+ * @param {!Object<string,string>=} env The environment object to hold env
  *     variables.
  * @return {?string} Full path of the file what set as an environment variable.
  */

package/src/apis/measurement_protocol.js

@@ -64,7 +64,7 @@ class MeasurementProtocol {
      * @param {string} batchId The tag for log.
      * @return {!Promise<boolean>}
      */
-    return (lines, batchId) => {
+    return async (lines, batchId) => {
       const payload =
           lines
               .map((line) => {
@@ -85,24 +85,37 @@ class MeasurementProtocol {
         body: payload,
         headers: {'User-Agent': 'Tentacles/MeasurementProtocol-v1'}
       };
-      return request(requestOptions).then((response) => {
-        if (response.status < 200 || response.status >= 300) {
-          const errorMessages = [
-            `Measurement Protocol [${batchId}] didn't succeed.`,
-            `Get response code: ${response.status}`,
-            `response: ${response.data}`,
-          ];
-          console.error(errorMessages.join('\n'));
-          throw new Error(`Status code not 2XX`);
-        }
-        this.logger.debug(`Configuration:`, config);
-        this.logger.debug(`Input Data:   `, lines);
-        this.logger.debug(`Batch[${batchId}] status: ${response.status}`);
-        this.logger.debug(response.data);
-        // There is not enough information from the non-debug mode.
-        if (!this.debugMode) return true;
-        return response.data.hitParsingResult.every((result) => result.valid);
-      });
+      const response = await request(requestOptions);
+      console.log(response);
+      if (response.status < 200 || response.status >= 300) {
+        const errorMessages = [
+          `Measurement Protocol [${batchId}] didn't succeed.`,
+          `Get response code: ${response.status}`,
+          `response: ${response.data}`,
+        ];
+        console.error(errorMessages.join('\n'));
+        throw new Error(`Status code not 2XX`);
+      }
+      this.logger.debug(`Configuration:`, config);
+      this.logger.debug(`Input Data:   `, lines);
+      this.logger.debug(`Batch[${batchId}] status: ${response.status}`);
+      this.logger.debug(response.data);
+      // There is not enough information from the non-debug mode.
+      if (!this.debugMode) return true;
+      const failedHits = [];
+      const failedReasons = new Set();
+      const errorMessage = response.data.hitParsingResult
+          .forEach((result, index) => {
+            if (!result.valid) {
+              failedHits.push(lines[index]);
+              result.parserMessage.forEach(({description}) => {
+                failedReasons.add(description);
+              })
+            }
+          });
+      console.log(failedHits);
+      console.log(failedReasons);
+      return errorMessage;
     };
   };
 

package/src/components/utils.js

@@ -23,12 +23,23 @@ const {inspect} = require('util');
 const {LoggingWinston} = require('@google-cloud/logging-winston');
 const {CloudPlatformApis} = require('../apis/cloud_platform_apis.js');
 
+/**
+ * The result of a batch of data sent to target API.
+ *
+ * @typedef {{
+ *   result: boolean,
+ *   errors: (Array<string>|undefined),
+ *   output: (Array<string>|undefined),
+ * }}
+ */
+let BatchResult;
+
 /**
  * Function which sends a batch of data. Takes two parameters:
  * {!Array<string>} Data for single request. It should be guaranteed that it
  *     doesn't exceed quota limitation.
  * {string} The tag for log.
- * @typedef {function(!Array<string>,string): !Promise<boolean>}
+ * @typedef {function(!Array<string>,string): !Promise<!BatchResult>}
  */
 let SendSingleBatch;
 
@@ -463,6 +474,7 @@ const changeNamingFromSnakeToLowerCamel = (name) => {
 module.exports = {
   getLogger,
   wait,
+  BatchResult,
   SendSingleBatch,
   apiSpeedControl,
   splitArray,