@google-cloud/nodejs-common 0.9.4-beta2 → 0.9.5-beta2

package/src/apis/google_ads.js

@@ -30,14 +30,19 @@ const {
   },
   services: {
     UploadClickConversionsRequest,
+    UploadClickConversionsResponse,
     UploadUserDataRequest,
+    UploadUserDataResponse,
     UserDataOperation,
     SearchGoogleAdsFieldsRequest,
   },
+  errors: {
+    GoogleAdsFailure,
+  },
 } = googleAdsLib;
 const {GoogleAdsApi} = require('google-ads-api');
 const AuthClient = require('./auth_client.js');
-const {getLogger} = require('../components/utils.js');
+const {getLogger, BatchResult,} = require('../components/utils.js');
 
 /** @type {!ReadonlyArray<string>} */
 const API_SCOPES = Object.freeze(['https://www.googleapis.com/auth/adwords',]);
@@ -46,7 +51,7 @@ const API_SCOPES = Object.freeze(['https://www.googleapis.com/auth/adwords',]);
  * Configuration for uploading click conversions for Google Ads, includes:
  * gclid, conversion_action, conversion_date_time, conversion_value,
  * currency_code, order_id, external_attribution_data
- * @see https://developers.google.com/google-ads/api/reference/rpc/v7/ClickConversion
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/ClickConversion
  * @typedef {{
  *   gclid: string,
  *   conversion_action: string,
@@ -65,7 +70,7 @@ let ClickConversionConfig;
  * list_type must be one of the following: hashed_email,
  * hashed_phone_number, mobile_id, third_party_user_id or address_info;
  * operation must be one of the two: 'create' or 'remove';
- * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserDataOperation
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserDataOperation
  * @typedef {{
  *   customer_id: string,
  *   login_customer_id: string,
@@ -80,7 +85,7 @@ let CustomerMatchConfig;
 /**
  * Configuration for uploading customer match data for Google Ads, includes one of:
  * hashed_email, hashed_phone_number, mobile_id, third_party_user_id or address_info
- * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserIdentifier
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserIdentifier
  * @typedef {{
  *   hashed_email: string,
  * }|{
@@ -123,17 +128,21 @@ let CustomerMatchRecord;
 let ReportQueryConfig;
 
 /**
- * Google Ads API v6.1 stub.
+ * Google Ads API class based on Opteo's Nodejs library.
  * see https://opteo.com/dev/google-ads-api/#features
  */
 class GoogleAds {
   /**
    * Note: Rate limits is set by the access level of Developer token.
    * @param {string} developerToken Developer token to access the API.
+   * @param {boolean=} debugMode This is used to set ONLY validate conversions
+   *     but not real uploading.
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
    */
-  constructor(developerToken) {
-    this.debug = process.env['DEBUG'] === 'true';
-    const oauthClient = new AuthClient(API_SCOPES).getOAuth2Token();
+  constructor(developerToken, debugMode = false, env = process.env) {
+    this.debugMode = debugMode;
+    const oauthClient = new AuthClient(API_SCOPES, env).getOAuth2Token();
     /** @const {GoogleAdsApi} */ this.apiClient = new GoogleAdsApi({
       client_id: oauthClient.clientId,
       client_secret: oauthClient.clientSecret,
@@ -206,7 +215,7 @@ class GoogleAds {
      * @param {!Array<string>} lines Data for single request. It should be
      *     guaranteed that it doesn't exceed quota limitation.
      * @param {string} batchId The tag for log.
-     * @return {!Promise<boolean>}
+     * @return {!BatchResult}
      */
     return async (lines, batchId) => {
       /** @type {!Array<ClickConversionConfig>} */
@@ -214,17 +223,166 @@ class GoogleAds {
         const record = JSON.parse(line);
         return Object.assign({}, adsConfig, record);
       });
+      /** @const {BatchResult} */
+      const batchResult = {
+        result: true,
+        numberOfLines: lines.length,
+      };
       try {
-        return await this.uploadClickConversions(conversions, customerId,
-            loginCustomerId);
+        const response = await this.uploadClickConversions(conversions,
+            customerId, loginCustomerId);
+        const {results, partial_failure_error: failed} = response;
+        if (this.logger.isDebugEnabled()) {
+          const gclids = results.map((conversion) => conversion.gclid);
+          this.logger.debug('Uploaded gclids:', gclids);
+        }
+        if (failed) {
+          this.logger.info('partial_failure_error:', failed.message);
+          const failures = failed.details.map(
+              ({value}) => GoogleAdsFailure.decode(value));
+          this.extraFailedLines_(batchResult, failures, lines, 0);
+        }
+        return batchResult;
       } catch (error) {
-        this.logger.info(
-            `Error in getUploadConFn in batchId: ${batchId}`, error);
-        return false;
+        this.logger.error(
+            `Error in upload conversions batch: ${batchId}`, error);
+        this.updateBatchResultWithError_(batchResult, error, lines, 0);
+        return batchResult;
       }
     }
   }
 
+  /**
+   * Updates the BatchResult based on errors.
+   *
+   * There are 2 types of errors here:
+   * 1. Normal JavaScript Error object. It happens when the whole process fails
+   * (not partial failure), so there is no detailed failed lines.
+   * 2. GoogleAdsFailure. It is a Google Ads' own error object which has an
+   * array of GoogleAdsError (property name 'errors'). GoogleAdsError contains
+   * the detailed failed data if it is a line-error. For example, a wrong
+   * encoded user identifier is a line-error, while a wrong user list id is not.
+   * GoogleAdsFailure: https://developers.google.com/google-ads/api/reference/rpc/latest/GoogleAdsFailure
+   * GoogleAdsError: https://developers.google.com/google-ads/api/reference/rpc/latest/GoogleAdsError
+   *
+   * For Customer Match data uploading, there is not partial failure, so the
+   * result can be either succeeded or a thrown error. The thrown error will be
+   * used to build the returned result here.
+   * For Conversions uploading (partial failure enabled), if there is an error
+   * fails the whole process, the error will also be thrown and handled here.
+   * Otherwise, the errors will be wrapped in the response as the property named
+   * 'partial_failure_error' which contains an array of GoogleAdsFailure. This
+   * kind of failure doesn't fail the process, while line-errors can be
+   * extracted from it.
+   * For more information, see the function `extraFailedLines_`.
+   *
+   * An example of 'GoogleAdsFailure' is:
+   * GoogleAdsFailure {
+   *   errors: [
+   *     GoogleAdsError {
+   *       error_code: ErrorCode { offline_user_data_job_error: 25 },
+   *       message: 'The SHA256 encoded value is malformed.',
+   *       location: ErrorLocation {
+   *         field_path_elements: [
+   *           FieldPathElement { field_name: 'operations', index: 0 },
+   *           FieldPathElement { field_name: 'create' },
+   *           FieldPathElement { field_name: 'user_identifiers', index: 0 },
+   *           FieldPathElement { field_name: 'hashed_email' }
+   *         ]
+   *       }
+   *     }
+   *   ],
+   *   request_id: 'xxxxxxxxxxxxxxx'
+   * }
+   *
+   * @param {!BatchResult} batchResult
+   * @param {(!GoogleAdsFailure|!Error)} error
+   * @param {!Array<string>} lines The original input data.
+   * @param {number} fieldPathIndex The index of 'FieldPathElement' in the array
+   *     'field_path_elements'. This is used to get the original line related to
+   *     this GoogleAdsError.
+   * @private
+   */
+  updateBatchResultWithError_(batchResult, error, lines, fieldPathIndex) {
+    batchResult.result = false;
+    if (error.errors) { //GoogleAdsFailure
+      this.extraFailedLines_(batchResult, [error], lines, fieldPathIndex);
+    } else {
+      batchResult.errors = [error.message || error.toString()];
+    }
+  }
+
+  /**
+   * Extras failed lines based on the GoogleAdsFailures.
+   *
+   * Different errors have different 'fieldPathIndex' which is the index of
+   * failed lines in original input data (an array of a string).
+   *
+   * For conversions, the ErrorLocation is like:
+   * ErrorLocation {
+   *   field_path_elements: [
+   *     FieldPathElement { field_name: 'operations', index: 0 },
+   *     FieldPathElement { field_name: 'create' }
+   *   ]
+   * }
+   * So the index is 0, index of 'operations'.
+   *
+   * For customer match upload, the ErrorLocation is like:
+   * ErrorLocation {
+   *   field_path_elements: [
+   *     FieldPathElement { field_name: 'operations', index: 0 },
+   *     FieldPathElement { field_name: 'create' },
+   *     FieldPathElement { field_name: 'user_identifiers', index: 0 },
+   *     FieldPathElement { field_name: 'hashed_email' }
+   *   ]
+   * }
+   * The index should be 2, index of 'user_identifiers'.
+   *
+   * With this we can get errors and failed lines. The function will set
+   * following for the given BatchResult object:
+   *   result - false
+   *   errors - de-duplicated error reasons
+   *   failedLines - failed lines, an array of string. Without the reason of
+   *     failure.
+   *   groupedFailed - a hashmap of failed the lines. The key is the reason, the
+   *     value is the array of failed lines due to this reason.
+   *     TODO: Confirm how to surface and handle groupedFailed.
+   * @param {!BatchResult} batchResult
+   * @param {!Array<!GoogleAdsFailure>} failures
+   * @param {!Array<string>} lines The original input data.
+   * @param {number} fieldPathIndex The index of 'FieldPathElement' in the array
+   *     'field_path_elements'. This is used to get the original line related to
+   *     this GoogleAdsError.
+   * @private
+   */
+  extraFailedLines_(batchResult, failures, lines, fieldPathIndex) {
+    batchResult.result = false;
+    batchResult.failedLines = [];
+    batchResult.groupedFailed = {};
+    const errors = new Set();
+    failures.forEach((failure) => {
+      failure.errors.forEach(({message, location}) => {
+        errors.add(message);
+        if (location && location.field_path_elements[fieldPathIndex]) {
+          const {index} = location.field_path_elements[fieldPathIndex];
+          if (typeof index === 'undefined') {
+            this.logger.warn(`Unknown field path index: ${fieldPathIndex}`,
+                location.field_path_elements);
+          } else {
+            const groupedFailed = batchResult.groupedFailed[message] || [];
+            const failedLine = lines[index];
+            batchResult.failedLines.push(failedLine);
+            groupedFailed.push(failedLine);
+            if (groupedFailed.length === 1) {
+              batchResult.groupedFailed[message] = groupedFailed;
+            }
+          }
+        }
+      });
+    });
+    batchResult.errors = Array.from(errors);
+  }
+
   /**
    * Uploads click conversions to google ads account.
    * It requires an array of click conversions and customer id.
@@ -232,30 +390,18 @@ class GoogleAds {
    * @param {Array<ClickConversionConfig>} clickConversions ClickConversions
    * @param {string} customerId
    * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
-   * @return {!Promise<boolean>}
+   * @return {!Promise<!UploadClickConversionsResponse>}
    */
-  async uploadClickConversions(clickConversions, customerId, loginCustomerId) {
+  uploadClickConversions(clickConversions, customerId, loginCustomerId) {
     this.logger.debug('Upload click conversions for customerId:', customerId);
     const customer = this.getGoogleAdsApiCustomer_(loginCustomerId, customerId);
     const request = new UploadClickConversionsRequest({
       conversions: clickConversions,
       customer_id: customerId,
-      validate_only: this.debug, // when true makes no changes
+      validate_only: this.debugMode, // when true makes no changes
       partial_failure: true, // Will still create the non-failed entities
     });
-    const result = await customer.conversionUploads.uploadClickConversions(
-        request);
-    const {results, partial_failure_error: failed} = result;
-    const response = results.map((conversion) => conversion.gclid);
-    this.logger.debug('Uploaded gclids:', response);
-    // const failed = result.partial_failure_error;
-    // Note: the response is different from previous version. current 'message'
-    // only contains partial failed conversions. The other field 'details'
-    // contains more information with the type of Array<Buffer>.
-    if (failed) {
-      this.logger.info('Errors:', failed.message);
-    }
-    return !failed;
+    return customer.conversionUploads.uploadClickConversions(request);
   }
 
   /**
@@ -271,29 +417,36 @@ class GoogleAds {
      * @param {!Array<string>} lines Data for single request. It should be
      *     guaranteed that it doesn't exceed quota limitation.
      * @param {string} batchId The tag for log.
-     * @return {!Promise<boolean>}
+     * @return {!Promise<BatchResult>}
      */
     return async (lines, batchId) => {
       /** @type {Array<CustomerMatchRecord>} */
       const userIds = lines.map((line) => JSON.parse(line));
+      /** @const {BatchResult} */ const batchResult = {
+        result: true,
+        numberOfLines: lines.length,
+      };
       try {
-        return await this.uploadUserDataToUserList(userIds,
+        const response = await this.uploadUserDataToUserList(userIds,
             customerMatchConfig);
+        this.logger.debug(`Customer Match upload batch[${batchId}]`, response);
+        return batchResult;
       } catch (error) {
         this.logger.error(
-            `Error in getUploadCustomerMatchFn in batchId: ${batchId}`, error);
-        return false;
+            `Error in Customer Match upload batch[${batchId}]`, error);
+        this.updateBatchResultWithError_(batchResult, error, lines, 2);
+        return batchResult;
       }
     }
   }
 
   /**
    * Uploads a user data to a user list (aka customer match).
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserDataService
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserDataOperation
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserData
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserIdentifier
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/CustomerMatchUserListMetadata
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserDataService
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserDataOperation
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserData
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserIdentifier
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/CustomerMatchUserListMetadata
    * Please note: The UserDataService has a limit of 10 UserDataOperations
    * and 100 user IDs per request
    * @see https://developers.google.com/google-ads/api/docs/migration/user-data-service#rate_limits
@@ -302,7 +455,7 @@ class GoogleAds {
    * customer_id, login_customer_id, list_id, list_type which can be one of the following
    * hashed_email, hashed_phone_number, mobile_id, third_party_user_id or address_info and
    * operation which can be either 'create' or 'remove'
-   * @return {!Promise<boolean>}
+   * @return {!Promise<UploadUserDataResponse>}
    */
   async uploadUserDataToUserList(customerMatchRecords, customerMatchConfig) {
     const customerId = customerMatchConfig.customer_id.replace(/-/g, '');
@@ -323,16 +476,15 @@ class GoogleAds {
       customer_match_user_list_metadata: metadata,
     });
     const response = await customer.userData.uploadUserData(request);
-    this.logger.debug('Uploaded CM users:', response);
-    return true;
+    return response;
   }
 
   /**
    * Builds a list of UserDataOperations.
    * Since v6 you can set a user_attribute
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserData
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserIdentifier
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/UserDataOperation
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserData
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserIdentifier
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserDataOperation
    * @param {string} operationType either 'create' or 'remove'
    * @param {Array<CustomerMatchRecord>} customerMatchRecords userIds
    * @param {string} userListType One of the following hashed_email, hashed_phone_number,
@@ -351,7 +503,7 @@ class GoogleAds {
 
   /**
    * Creates CustomerMatchUserListMetadata.
-   * @see https://developers.google.com/google-ads/api/reference/rpc/v7/CustomerMatchUserListMetadata
+   * @see https://developers.google.com/google-ads/api/reference/rpc/latest/CustomerMatchUserListMetadata
    * @param {string} customerId part of the ResourceName to be mutated
    * @param {string} userListId part of the ResourceName to be mutated
    * @return {!CustomerMatchUserListMetadata}

package/src/apis/index.js

@@ -110,3 +110,13 @@ exports.adsdatahub = require('./ads_data_hub.js');
  * }}
  */
 exports.measurementprotocolga4 = require('./measurement_protocol_ga4.js');
+
+/**
+ * APIs integration class for YouTube.
+ * @const {{
+ *   YouTube:!YouTube,
+ *   ListChannelsConfig: !ListChannelsConfig,
+ *   ListVideosConfig: !ListVideosConfig,
+ * }}
+ */
+exports.youtube = require('./youtube.js');

package/src/apis/measurement_protocol.js

@@ -69,18 +69,14 @@ class MeasurementProtocol {
      * @return {!Promise<BatchResult>}
      */
     return async (lines, batchId) => {
-      const payload =
-          lines
-              .map((line) => {
-                const record = JSON.parse(line);
-                const hit = Object.assign({}, config, record);
-                return Object.keys(hit)
-                    .map((key) => {
-                      return `${key}=${encodeURIComponent(hit[key])}`;
-                    })
-                    .join('&');
-              })
-              .join('\n');
+      const payload = lines.map((line) => {
+            const record = JSON.parse(line);
+            const hit = Object.assign({}, config, record);
+            return Object.keys(hit).map(
+                (key) => `${key}=${encodeURIComponent(hit[key])}`)
+                .join('&');
+          })
+          .join('\n');
       // In debug mode, the path is fixed to '/debug/collect'.
       const path = (this.debugMode) ? '/debug/collect' : '/batch';
       const requestOptions = {
@@ -112,24 +108,48 @@ class MeasurementProtocol {
       if (!this.debugMode) {
         batchResult.result = true;
       } else {
-        const failedHits = [];
-        const failedReasons = new Set();
-        response.data.hitParsingResult.forEach((result, index) => {
-          if (!result.valid) {
-            failedHits.push(lines[index]);
-            result.parserMessage.forEach(({description}) => {
-              failedReasons.add(description);
-            });
-          }
-        });
-        batchResult.result = failedHits.length === 0;
-        batchResult.failedLines = failedHits;
-        batchResult.errors = Array.from(failedReasons);
+        this.extraFailedLines_(batchResult, response.data.hitParsingResult,
+            lines);
       }
       return batchResult;
     };
   };
 
+  /**
+   * Extras failed lines based on the hitParsingResult, see:
+   * https://developers.google.com/analytics/devguides/collection/protocol/v1/validating-hits
+   *
+   * Note, only in 'debug' mode, Google Analytics will return this part of data.
+   *
+   * @param {!BatchResult} batchResult
+   * @param {!Array<!Object>} hitParsingResults
+   * @param {!Array<string>} lines The original input data.
+   * @private
+   */
+  extraFailedLines_(batchResult, hitParsingResults, lines) {
+    batchResult.failedLines = [];
+    batchResult.groupedFailed = {};
+    const errors = new Set();
+    hitParsingResults.forEach((result, index) => {
+      if (!result.valid) {
+        const failedLine = lines[index];
+        batchResult.failedLines.push(failedLine);
+        result.parserMessage.forEach(({description: error, messageType}) => {
+          this.logger.info(`[${messageType}]: ${error} for ${failedLine}`);
+          if (messageType === 'ERROR') {
+            errors.add(error);
+            const groupedFailed = batchResult.groupedFailed[error] || [];
+            groupedFailed.push(failedLine);
+            if (groupedFailed.length === 1) {
+              batchResult.groupedFailed[error] = groupedFailed;
+            }
+          }
+        });
+      }
+    });
+    batchResult.result = batchResult.failedLines.length === 0;
+    batchResult.errors = Array.from(errors);
+  }
 }
 
 module.exports = {MeasurementProtocol};

package/src/apis/measurement_protocol_ga4.js

@@ -20,6 +20,7 @@
 'use strict';
 
 const {request} = require('gaxios');
+const lodash = require('lodash');
 const {
   getLogger,
   SendSingleBatch,
@@ -93,7 +94,11 @@ class MeasurementProtocolGA4 {
      */
     return async (lines, batchId) => {
       const line = lines[0]; // Each request contains one record only.
-      const hit = Object.assign({}, config.requestBody, JSON.parse(line));
+      if (lines.length > 1) {
+        this.logger.warn(
+            "Only one line data expected. Will only send the first line.");
+      }
+      const hit = lodash.merge({}, config.requestBody, JSON.parse(line));
 
       const requestOptions = {
         method: 'POST',
@@ -135,6 +140,7 @@ class MeasurementProtocolGA4 {
           batchResult.failedLines = lines;
           batchResult.errors = response.data.validationMessages.map(
               ({description}) => description);
+          batchResult.groupedFailed = {[batchResult.errors.join()]: [lines[0]]};
         }
       }
       return batchResult;

package/src/apis/spreadsheets.js

@@ -67,11 +67,13 @@ class Spreadsheets {
   /**
    * Init Spreadsheets API client.
    * @param {string} spreadsheetId
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
    */
-  constructor(spreadsheetId) {
+  constructor(spreadsheetId, env = process.env) {
     /** @const {string} */
     this.spreadsheetId = spreadsheetId;
-    const authClient = new AuthClient(API_SCOPES);
+    const authClient = new AuthClient(API_SCOPES, env);
     const auth = authClient.getDefaultAuth();
     /** @const {!!google.sheets} */
     this.instance = google.sheets({
@@ -108,7 +110,6 @@ class Spreadsheets {
    * see:
    * https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/clear
    * @param {string} sheetName Name of the Sheet.
-   * @return {!Promise<boolean>} Whether the operation succeeded.
    */
   async clearSheet(sheetName) {
     const request = {
@@ -119,10 +120,9 @@ class Spreadsheets {
       const response = await this.instance.spreadsheets.values.clear(request);
       const data = response.data;
       this.logger.debug(`Clear sheet[${sheetName}}]: `, data);
-      return true;
     } catch (error) {
-      this.logger.error(error);
-      return false;
+      this.logger.error(error.toString());
+      throw error;
     }
   }
 
@@ -162,7 +162,6 @@ class Spreadsheets {
    * @param {string} sheetName Name of the Sheet.
    * @param {number} targetRows Loaded data rows number.
    * @param {number} targetColumns Loaded data columns number.
-   * @return {!Promise<boolean>} Whether the operation succeeded.
    */
   async reshape(sheetName, targetRows, targetColumns) {
     const request =  /** @type{Params$Resource$Spreadsheets$Get} */ {
@@ -194,13 +193,12 @@ class Spreadsheets {
       if (requests.resource.requests.length > 0) {
         const {data} = await this.instance.spreadsheets.batchUpdate(requests);
         this.logger.debug(`Reshape Sheet [${sheetName}]: `, data);
-        return true;
+      } else {
+        this.logger.debug('No need to reshape.');
       }
-      this.logger.debug('No need to reshape.');
-      return true;
     } catch (error) {
-      console.error(error);
-      return false;
+      this.logger.error(error.toString());
+      throw error;
     }
   }
 
@@ -218,7 +216,9 @@ class Spreadsheets {
       spreadsheetId: this.spreadsheetId,
       resource: {requests: [{pasteData}]},
     };
-    /** @type {BatchResult} */ const batchResult = {};
+    /** @type {BatchResult} */ const batchResult = {
+      numberOfLines: data.trim().split('\n').length,
+    };
     try {
       const response = await this.instance.spreadsheets.batchUpdate(request);
       const data = response.data;