@google-cloud/nodejs-common 0.9.4-beta2 → 0.9.5-beta2

package/bin/install_functions.sh

@@ -108,6 +108,7 @@ declare EXTERNAL_APIS=(
   "sheets.googleapis.com"
   "googleads.googleapis.com"
   "adsdatahub.googleapis.com"
+  "youtube.googleapis.com"
 )
 
 # Some APIs only support OAuth which needs an process to generate refresh token.
@@ -115,6 +116,7 @@ declare EXTERNAL_APIS=(
 declare EXTERNAL_APIS_OAUTH_ONLY=(
   "googleads.googleapis.com"
   "doubleclickbidmanager.googleapis.com"
+  "youtube.googleapis.com"
 )
 
 # API scopes for OAuth authentication.
@@ -133,6 +135,8 @@ EXTERNAL_API_SCOPES=(
   ["sheets.googleapis.com"]="https://www.googleapis.com/auth/spreadsheets"
   ["googleads.googleapis.com"]="https://www.googleapis.com/auth/adwords"
   ["adsdatahub.googleapis.com"]="https://www.googleapis.com/auth/adsdatahub"
+  ["youtube.googleapis.com"]=\
+"https://www.googleapis.com/auth/youtube.force-ssl"
 )
 
 # Enabled APIs' OAuth scopes.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "0.9.4-beta2",
+  "version": "0.9.5-beta2",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -30,7 +30,8 @@
     "google-auth-library": "^7.1.0",
     "googleapis": "^74.2.0",
     "soap": "^0.38.0",
-    "winston": "^3.3.3"
+    "winston": "^3.3.3",
+    "lodash": "^4.17.21"
   },
   "devDependencies": {
     "jasmine": "^3.5.0"

package/src/apis/auth_client.js

@@ -110,6 +110,7 @@ class AuthClient {
    * @return {!OAuth2Client}
    */
   getOAuth2Client(keyFile = this.oauthTokenFile) {
+    this.ensureFileExisting_(keyFile, 'OAUTH token');
     const key = JSON.parse(fs.readFileSync(keyFile).toString());
     console.log(`Get OAuth token with client Id: ${key.client_id}`);
     const oAuth2Client = new OAuth2Client(key.client_id, key.client_secret);
@@ -123,6 +124,7 @@ class AuthClient {
    * @return {!JWT}
    */
   getServiceAccount(keyFile = this.serviceAccountKeyFile) {
+    this.ensureFileExisting_(keyFile, 'Service Account key');
     console.log(`Get Service Account's key file: ${keyFile}`);
     return new JWT({keyFile, scopes: this.scopes,});
   }
@@ -141,6 +143,7 @@ class AuthClient {
    * }}
    */
   getOAuth2Token(keyFile = this.oauthTokenFile) {
+    this.ensureFileExisting_(keyFile, 'OAuth token');
     const key = JSON.parse(fs.readFileSync(keyFile).toString());
     return {
       clientId: key.client_id,
@@ -148,6 +151,18 @@ class AuthClient {
       refreshToken: key.token.refresh_token,
     };
   }
+
+  /**
+   * Some APIs only support one authorization type, so we have to designate a
+   * specific auth method. If the related key/token file is not available, the
+   * auth will fail. The function throws errors with more meaningful message.
+   * @param {string} file Target file path
+   * @param {string} type Key type name, 'OAuth token' or 'Service Account key'
+   * @private
+   */
+  ensureFileExisting_(file, type) {
+    if (!file) throw new Error(`Required ${type} file doesn't exist.`);
+  }
 }
 
 /**

package/src/apis/dfa_reporting.js

@@ -26,6 +26,7 @@ const {
   getLogger,
   getFilterFunction,
   SendSingleBatch,
+  BatchResult,
 } = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
@@ -64,7 +65,7 @@ let InsertConversionsConfig;
 /**
  * List of properties that will be take from the data file as elements of a
  * conversion.
- * See https://developers.google.com/doubleclick-advertisers/v3.3/conversions
+ * See https://developers.google.com/doubleclick-advertisers/rest/v3.5/Conversion
  * @type {Array<string>}
  */
 const PICKED_PROPERTIES = [
@@ -79,8 +80,14 @@ const PICKED_PROPERTIES = [
  * see https://developers.google.com/doubleclick-advertisers/service_accounts
  */
 class DfaReporting {
-  constructor() {
-    const authClient = new AuthClient(API_SCOPES);
+
+  /**
+   * @constructor
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
+   */
+  constructor(env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     this.auth = authClient.getDefaultAuth();
     /** @const {!google.dfareporting} */
     this.instance = google.dfareporting({
@@ -126,7 +133,7 @@ class DfaReporting {
      * @param {!Array<string>} lines Data for single request. It should be
      *     guaranteed that it doesn't exceed quota limitation.
      * @param {string} batchId The tag for log.
-     * @return {!Promise<boolean>}
+     * @return {!Promise<BatchResult>}
      */
     return async (lines, batchId) => {
       /** @type {function} Gets the conversion elements from the data object. */
@@ -153,6 +160,11 @@ class DfaReporting {
       if (config.idType === 'encryptedUserId') {
         requestBody.encryptionInfo = config.encryptionInfo;
       }
+      /** @const {BatchResult} */
+      const batchResult = {
+        result: true,
+        numberOfLines: lines.length,
+      };
       try {
         const response = await this.instance.conversions.batchinsert({
           profileId: config.profileId,
@@ -160,25 +172,61 @@ class DfaReporting {
         });
         const failed = response.data.hasFailures;
         if (failed) {
-          console.error(`Dfareporting[${batchId}] has failures.`);
-          const errorMessages = response.data.status
-              .filter((record) => typeof record.errors !== 'undefined')
-              .map((record) => {
-                this.logger.debug(record);
-                return record.errors.map((error) => error.message).join(',');
-              });
-          console.error(errorMessages.join('\n'));
+          this.logger.warn(`CM [${batchId}] has failures.`);
+          this.extraFailedLines_(batchResult, response.data.status, lines);
         }
         this.logger.debug('Configuration: ', config);
         this.logger.debug('Response: ', response);
-        return !failed;
+        return batchResult;
       } catch (error) {
-        console.error(`Dfareporting[${batchId}] failed.`, error);
-        return false;
+        this.logger.error(`CM[${batchId}] failed.`, error);
+        batchResult.result = false;
+        batchResult.errors = [error.message || error.toString()];
+        return batchResult;
       }
     };
   };
 
+  /**
+   * Campaign Manager API returns an array of ConversionStatus for the status of
+   * uploaded conversions. If there are errors related to the conversion, then
+   * an array of 'ConversionError' named 'errors' will be available in the
+   * ConversionStatus object. This function extras failed lines and error
+   * messages based on the 'errors'.
+   * For 'ConversionStatus', see:
+   *   https://developers.google.com/doubleclick-advertisers/rest/v3.5/ConversionStatus
+   * For 'ConversionError', see:
+   *   https://developers.google.com/doubleclick-advertisers/rest/v3.5/ConversionStatus#ConversionError
+   * @param {!BatchResult} batchResult
+   * @param {!Array<!Schema$ConversionStatus>} statuses
+   * @param {!Array<string>} lines The original input data.
+   * @private
+   */
+  extraFailedLines_(batchResult, statuses, lines) {
+    batchResult.result = false;
+    batchResult.failedLines = [];
+    batchResult.groupedFailed = {};
+    const errors = new Set();
+    statuses.forEach((conversionStatus, index) => {
+      if (conversionStatus.errors) {
+        const failedLine = lines[index];
+        batchResult.failedLines.push(failedLine);
+        conversionStatus.errors.forEach(({message}) => {
+          // error messages have detailed IDs. Need to generalize them.
+          const generalMessage = message.replace(/.*error: /, '');
+          errors.add(generalMessage);
+          const groupedFailed = batchResult.groupedFailed[generalMessage]
+              || [];
+          groupedFailed.push(failedLine);
+          if (groupedFailed.length === 1) {
+            batchResult.groupedFailed[generalMessage] = groupedFailed;
+          }
+        });
+      }
+      batchResult.errors = Array.from(errors);
+    });
+  }
+
   /**
    * Lists all UserProfiles.
    * @return {!Promise<!Array<string>>}
@@ -203,8 +251,8 @@ class DfaReporting {
    * @return {!Promise<string>} Profile Id.
    * @private
    */
-  getProfileForOperation_(config) {
-    if (config.profileId) return Promise.resolve(config.profileId);
+  async getProfileForOperation_(config) {
+    if (config.profileId) return config.profileId;
     if (config.accountId) return this.getProfileId(config.accountId);
     throw new Error('There is no profileId or accountId in the configuration.');
   }
@@ -213,7 +261,7 @@ class DfaReporting {
    * Runs a report and return the file Id. As an asynchronized process, the
    * returned file Id will be a placeholder until the status changes to
    * 'REPORT_AVAILABLE' in the response of `getFile`.
-   * @see https://developers.google.com/doubleclick-advertisers/v3.3/reports/run
+   * @see https://developers.google.com/doubleclick-advertisers/rest/v3.5/reports/run
    *
    * @param {{
    *   accountId:(string|undefined),
@@ -236,7 +284,7 @@ class DfaReporting {
    * Returns file url from a report. If the report status is 'REPORT_AVAILABLE',
    * then return the apiUrl from the response; if the status is 'PROCESSING',
    * returns undefined; otherwise throws an error.
-   * @see https://developers.google.com/doubleclick-advertisers/v3.3/reports/files/get
+   * @see https://developers.google.com/doubleclick-advertisers/rest/v3.5/reports/get
    *
    * @param {{
    *   accountId:(string|undefined),

package/src/apis/doubleclick_search.js

@@ -22,7 +22,11 @@
 const {google} = require('googleapis');
 const {request} = require('gaxios');
 const AuthClient = require('./auth_client.js');
-const {getLogger, SendSingleBatch} = require('../components/utils.js');
+const {
+  getLogger,
+  SendSingleBatch,
+  BatchResult,
+} = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
   'https://www.googleapis.com/auth/doubleclicksearch',
@@ -115,8 +119,14 @@ let ReportRequest;
  * https://support.google.com/adsihc/answer/6346075?hl=en
  */
 class DoubleClickSearch {
-  constructor() {
-    const authClient = new AuthClient(API_SCOPES);
+
+  /**
+   * @constructor
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
+   */
+  constructor(env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     this.auth = authClient.getDefaultAuth();
     /** @const {!google.doubleclicksearch} */
     this.instance = google.doubleclicksearch({
@@ -138,22 +148,21 @@ class DoubleClickSearch {
    *     availabilities array.
    * @return {!Promise<boolean>} Update result.
    */
-  updateAvailability(availabilities) {
+  async updateAvailability(availabilities) {
     const availabilityTimestamp = Date.now();
-    for (const availability of availabilities) {
+    availabilities.forEach((availability) => {
       availability.availabilityTimestamp = availabilityTimestamp;
-    }
+    });
     this.logger.debug('Sending out availabilities', availabilities);
-    return this.instance.conversion
-        .updateAvailability({requestBody: {availabilities}})
-        .then((response) => {
-          this.logger.debug('Get response: ', response);
-          return response.status === 200;
-        })
-        .catch((error) => {
-          console.error(error);
-          return false;
-        });
+    try {
+      const response = await this.instance.conversion.updateAvailability(
+          {requestBody: {availabilities}});
+      this.logger.debug('Get response: ', response);
+      return response.status === 200;
+    } catch (e) {
+      this.logger.error(e);
+      return false;
+    }
   }
 
   /**
@@ -170,16 +179,16 @@ class DoubleClickSearch {
      * @param {!Array<string>} lines Data for single request. It should be
      *     guaranteed that it doesn't exceed quota limitation.
      * @param {string} batchId The tag for log.
-     * @return {!Promise<boolean>}
+     * @return {!Promise<BatchResult>}
      */
-    return (lines, batchId) => {
+    return async (lines, batchId) => {
       const conversionTimestamp = new Date().getTime();
       const conversions = lines.map((line, index) => {
         const record = JSON.parse(line);
         return Object.assign(
             {
               // Default value, can be overwritten by the exported data.
-              conversionTimestamp: conversionTimestamp,
+              conversionTimestamp,
               // Default conversion Id should be unique in a single request.
               // See error code 0x0000011F at here:
               // https://developers.google.com/search-ads/v2/troubleshooting#conversion-upload-errors
@@ -188,33 +197,100 @@ class DoubleClickSearch {
             config, record);
       });
       this.logger.debug('Configuration: ', config);
-      return this.instance.conversion
-          .insert({requestBody: {conversion: conversions}})
-          .then((response) => {
-            this.logger.debug('Response: ', response);
-            console.log(`SA[${batchId}] Insert ${
-                response.data.conversion.length} conversions`);
-            return true;
-          })
-          .catch((error) => {
-            if (error.code === 400 && error.errors) {//requestValidation error
-              const messages = error.errors.map((singleError) => {
-                return singleError.message.replace(
-                    'The request was not valid. Details: ', '');
-              });
-              console.log(`SA[${batchId}] partially failed.\n`,
-                  messages.join(',\n'));
-              return false;
-            }
-            console.error(
-                `SA insert conversions [${batchId}] failed.`, error.message);
-            this.logger.debug(
-                'Errors in response:', error.response.data.error);
-            return false;
-          });
+      /** @const {BatchResult} */
+      const batchResult = {
+        result: true,
+        numberOfLines: lines.length,
+      };
+      try {
+        const response = await this.instance.conversion.insert(
+            {requestBody: {conversion: conversions}}
+        );
+        this.logger.debug('Response: ', response);
+        const insertedConversions = response.data.conversion.length;
+        if (lines.length !== insertedConversions) {
+          const errorMessage =
+              `Conversions input/inserted: ${lines.length}/${insertedConversions}`;
+          this.logger.warn(errorMessage);
+          batchResult.result = false;
+          batchResult.numberOfLines = insertedConversions;
+          batchResult.errors = [errorMessage];
+        }
+        this.logger.info(
+            `SA[${batchId}] Insert ${insertedConversions} conversions.`);
+        return batchResult;
+      } catch (error) {
+        this.updateBatchResultWithError_(batchResult, error, lines);
+        return batchResult;
+      }
     };
   }
 
+  /**
+   * Updates the BatchResult based on errors.
+   * There are 3 types of errors here:
+   * The first two errors are from 'Standard Error Responses';
+   * The last one is normal Javascript Error object.
+   *
+   * For more details of 'Standard Error Responses', see:
+   * https://developers.google.com/search-ads/v2/standard-error-responses
+   *
+   * Error 1. error code is not 400, e.g. 403 for no access to SA360 API. This
+   *     error fail the whole process and no need to extract detailed lines.
+   * Error 2. error code 400 and 'errors' has one or more lines. Each line might
+   *     have different failure reason. Failed lines can be extracted to give
+   *     users more information.
+   *     Note, some failure reason may fail every line, e.g. wrong
+   *     'segmentationName' in the config (Error code '0x0000010E', see:
+   *     https://developers.google.com/search-ads/v2/troubleshooting#conversion-upload-errors )
+   * Error 3. normal JavaScript Error object. No property 'errors'.
+   *
+   * @param {!BatchResult} batchResult
+   * @param {(!GoogleAdsFailure|!Error)} error
+   * @param {!Array<string>} lines The original input data.
+   * @private
+   */
+  updateBatchResultWithError_(batchResult, error, lines) {
+    batchResult.result = false;
+    // Error 3.
+    if (!error.errors) {
+      batchResult.errors = [error.message || error.toString()];
+      return;
+    }
+    const errorMessages = error.errors.map(({message}) => message);
+    // Error 1.
+    if (error.code !== 400) {
+      batchResult.errors = errorMessages;
+      return;
+    }
+    // Error 2.
+    batchResult.failedLines = [];
+    batchResult.groupedFailed = {};
+    const errors = new Set();
+    const messageReg = /.*Details: \[(.*) index=\d+ conversionId=.*/;
+    const indexReg = /.*index=(\d*) .*/;
+    errorMessages.forEach((message) => {
+      const errorMessage = messageReg.exec(message);
+      if (errorMessage) {
+        const index = indexReg.exec(message);
+        const failedLine = lines[index[1]];
+        batchResult.failedLines.push(failedLine);
+        // error messages have detailed IDs. Need to generalize them.
+        const generalMessage =
+            errorMessage[1].replace(/ \'[^\']*\'/, '');
+        errors.add(generalMessage);
+        const groupedFailed = batchResult.groupedFailed[generalMessage] || [];
+        groupedFailed.push(failedLine);
+        if (groupedFailed.length === 1) {
+          batchResult.groupedFailed[generalMessage] = groupedFailed;
+        }
+      } else {
+        errors.add(message);
+      }
+    });
+    batchResult.errors = Array.from(errors);
+  }
+
   /**
    * There are three steps to get asynchronous reports in SA360:
    * 1. Call Reports.request() to specify the type of data for the report.
@@ -226,15 +302,14 @@ class DoubleClickSearch {
    * @param {!ReportRequest} requestBody
    * @return {!Promise<string>}
    */
-  requestReports(requestBody) {
-    return this.instance.reports.request({requestBody})
-        .then(({status, data}) => {
-          if (status >= 200 && status < 300) return data.id;
-          const errorMsg =
-              `Fail to request reports: ${JSON.stringify(requestBody)}`;
-          console.error(errorMsg, data);
-          throw new Error(errorMsg);
-        });
+  async requestReports(requestBody) {
+    const {status, data} = await this.instance.reports.request({requestBody});
+    if (status >= 200 && status < 300) {
+      return data.id;
+    }
+    const errorMsg = `Fail to request reports: ${JSON.stringify(requestBody)}`;
+    this.logger.error(errorMsg, data);
+    throw new Error(errorMsg);
   }
 
   /**
@@ -245,24 +320,23 @@ class DoubleClickSearch {
    *   byteCount:string,
    * }>>}
    */
-  getReportUrls(reportId) {
-    return this.instance.reports.get({reportId})
-        .then(({status, data}) => {
-          switch (status) {
-            case 200:
-              console.log(
-                  `Report[${reportId}] has ${data.rowCount} rows and ${data.files.length} files.`);
-              return data.files;
-            case 202:
-              console.log(`Report[${reportId}] is not ready.`);
-              break;
-            default:
-              const errorMsg =
-                  `Error in get reports: ${reportId} with status code: ${status}`;
-              console.error(errorMsg, data);
-              throw new Error(errorMsg);
-          }
-        });
+  async getReportUrls(reportId) {
+    const {status, data} = await this.instance.reports.get({reportId});
+    switch (status) {
+      case 200:
+        const {rowCount, files} = data;
+        this.logger.info(
+            `Report[${reportId}] has ${rowCount} rows and ${files.length} files.`);
+        return files;
+      case 202:
+        this.logger.info(`Report[${reportId}] is not ready.`);
+        break;
+      default:
+        const errorMsg =
+            `Error in get reports: ${reportId} with status code: ${status}`;
+        this.logger.error(errorMsg, data);
+        throw new Error(errorMsg);
+    }
   }
 
   /**
@@ -271,15 +345,14 @@ class DoubleClickSearch {
    * @param {number} reportFragment The index (based 0) of report files.
    * @return {!Promise<string>}
    */
-  getReportFile(reportId, reportFragment) {
-    return this.instance.reports.getFile({reportId, reportFragment})
-        .then((response) => {
-          if (response.status === 200) return response.data;
-          const errorMsg =
-              `Error in get file from reports: ${reportFragment}@${reportId}`;
-          console.error(errorMsg, response);
-          throw new Error(errorMsg);
-        });
+  async getReportFile(reportId, reportFragment) {
+    const response = await this.instance.reports.getFile(
+        {reportId, reportFragment});
+    if (response.status === 200) return response.data;
+    const errorMsg =
+        `Error in get file from reports: ${reportFragment}@${reportId}`;
+    this.logger.error(errorMsg, response);
+    throw new Error(errorMsg);
   }
 
   /**
@@ -289,16 +362,15 @@ class DoubleClickSearch {
    * @param {string} url
    * @return {!Promise<ReadableStream>}
    */
-  getReportFileStream(url) {
-    return this.auth.getRequestHeaders()
-        .then((headers) => {
-          return request({
-            method: 'GET',
-            headers,
-            url,
-            responseType: 'stream',
-          });
-        }).then((response) => response.data);
+  async getReportFileStream(url) {
+    const headers = await this.auth.getRequestHeaders();
+    const response = await request({
+      method: 'GET',
+      headers,
+      url,
+      responseType: 'stream',
+    });
+    return response.data;
   }
 
 }