@google-cloud/nodejs-common 1.0.3 → 1.1.0

package/README.md

@@ -1,6 +1,6 @@
 # NodeJS Common Library
 
-<!--* freshness: { owner: 'lushu' reviewed: '2021-12-02' } *-->
+<!--* freshness: { owner: 'lushu' reviewed: '2022-04-07' } *-->
 
 A NodeJs common library for other projects, e.g. [GMP and Google Ads Connector]
 and [Data Tasks Coordinator]. This library includes:
@@ -11,68 +11,65 @@ and [Data Tasks Coordinator]. This library includes:
 1. Wrapper for some Google APIs for integration, mainly
    for [GMP and Google Ads Connector]:
 
-    * Google Analytics data import
-    * Google Analytics measurement protocol
-    * Campaign Manager offline conversion upload
-    * Search Ads 360 conversions upload
-    * Google Ads click conversions upload
-    * Google Ads customer match upload
-    * Google Ads conversions scheduled uploads based on Google Sheets
-    * Measurement Protocol Google Analytics 4
+   - Google Analytics data import
+   - Google Analytics measurement protocol
+   - Campaign Manager offline conversion upload
+   - Search Ads 360 conversions upload
+   - Google Ads click conversions upload
+   - Google Ads customer match upload
+   - Google Ads enhanced conversions upload
+   - Google Ads conversions scheduled uploads based on Google Sheets
+   - Measurement Protocol Google Analytics 4
 
 1. Wrapper for some Google APIs for reporting, mainly
    for [Data Tasks Coordinator]:
 
-    * Google Ads reporting
-    * Campaign Manager reporting
-    * Search Ads 360 reporting
-    * Display and Video 360 reporting
-    * YouTube Data API
-    * Ads Data Hub querying
+   - Google Ads reporting
+   - Campaign Manager reporting
+   - Search Ads 360 reporting
+   - Display and Video 360 reporting
+   - YouTube Data API
+   - Ads Data Hub querying
 
 1. Utilities wrapper class for Google Cloud Products:
 
-    * **Firestore Access Object**: Firestore has two modes which are excluded to
-      each other and can't be changed once selected in a Cloud Project[[2]].
-      This class, with its two successors offer a unified interface to operate
-      data objects on either Firestore or Datastore.
+   - **Firestore Access Object**: Firestore has two modes which are excluded to
+     each other and can't be changed once selected in a Cloud Project[[2]].
+     This class, with its two successors offer a unified interface to operate
+     data objects on either Firestore or Datastore.
 
-    * **AutoMl Tables API**: Offers a unified entry to use this API based on
-      Google Cloud client library combined with REST requests to service
-      directly due to some functionalities missed in the client library.
+   - **AutoMl Tables API**: Offers a unified entry to use this API based on
+     Google Cloud client library combined with REST requests to service
+     directly due to some functionalities missed in the client library.
 
-    * **Vertex AI API**: Offers a unified entry to use this API based on Google
-      Cloud client library.
+   - **Vertex AI API**: Offers a unified entry to use this API based on Google
+     Cloud client library.
 
-    * **Pub/Sub Utilities**: Offers utilities functions to create topics and
-      subscriptions for Pub/Sub, as well as the convenient way to publish a
-      message.
+   - **Pub/Sub Utilities**: Offers utilities functions to create topics and
+     subscriptions for Pub/Sub, as well as the convenient way to publish a
+     message.
 
-    * **Storage Utilities**: Offers functions to manipulate the files on Cloud
-      Storage. The main functions are:
+   - **Storage Utilities**: Offers functions to manipulate the files on Cloud
+     Storage. The main functions are:
 
-        * Reading a given length (or slightly less) content without breaking a
-          line;
-        * Splitting a file into multiple files with the given length (or
-          slightly less) without breaking a line;
-        * Merging files into one file.
+     - Reading a given length (or slightly less) content without breaking a
+       line;
+     - Splitting a file into multiple files with the given length (or
+       slightly less) without breaking a line;
+     - Merging files into one file.
 
-    * **Cloud Scheduler Adapter**: A wrapper to pause and resume Cloud Scheduler
-      jobs.
+   - **Cloud Scheduler Adapter**: A wrapper to pause and resume Cloud Scheduler
+     jobs.
 
-    * **Cloud Functions Adapter**: Cloud Functions have different parameters in
-      different environments, e.g. Node6 vs Node8[[1]]. This utility file offers
-      an adapter to wrap a Node8 Cloud Functions into Node6 and Node8 compatible
-      functions.
+   - **Cloud Functions Adapter**: Cloud Functions have different parameters in
+     different environments, e.g. Node6 vs Node8[[1]]. This utility file offers
+     an adapter to wrap a Node8 Cloud Functions into Node6 and Node8 compatible
+     functions.
 
 1. A share library for [Bash] to facilitate installation tasks.
 
-[GMP and Google Ads Connector]:https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/gmp-googleads-connector
-
-[Data Tasks Coordinator]:https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/data-tasks-coordinator
-
-[1]:https://cloud.google.com/functions/docs/writing/background#functions-writing-background-hello-pubsub-node8-10
-
-[2]:https://cloud.google.com/datastore/docs/concepts/overview#comparison_with_traditional_databases
-
-[Bash]:https://www.gnu.org/software/bash/
+[gmp and google ads connector]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/gmp-googleads-connector
+[data tasks coordinator]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/data-tasks-coordinator
+[1]: https://cloud.google.com/functions/docs/writing/background#functions-writing-background-hello-pubsub-node8-10
+[2]: https://cloud.google.com/datastore/docs/concepts/overview#comparison_with_traditional_databases
+[bash]: https://www.gnu.org/software/bash/

package/bin/install_functions.sh

@@ -173,7 +173,7 @@ project ID: [${current_project}]"
         project=${project:-"${current_project}"}
         target_project=${project}
       fi
-      if [[ -n ${current_project} && \
+      if [[ -z ${current_project} || \
 ${current_project} != "${target_project}" ]];then
         printf '%s' "Setting the Google Cloud project to [${target_project}]..."
         gcloud config set project "${target_project}"
@@ -1337,10 +1337,13 @@ do_authentication(){
 do_oauth(){
   # Base url of Google OAuth service.
   OAUTH_BASE_URL="https://accounts.google.com/o/oauth2/"
+  # Redirect uri.
+  # Must be the same for requests of authorization code and refresh token.
+  REDIRECT_URI="http%3A//127.0.0.1%3A8887"
   # Defaulat parameters of OAuth requests.
   OAUTH_PARAMETERS=(
     "access_type=offline"
-    "redirect_uri=urn:ietf:wg:oauth:2.0:oob"
+    "redirect_uri=${REDIRECT_URI}"
     "response_type=code"
   )
   while [ ${#ENABLED_OAUTH_SCOPES[@]} -eq 0 ]; do
@@ -1409,18 +1412,26 @@ EOF
     printf '%s\n' "3. Open the link in browser and finish authentication: \
 ${auth_url}"
     cat <<EOF
-  Note: if the OAuth client is not for a native application, there will be an \
+  Note:
+    If the OAuth client is not for a native application, there will be an \
 "Error 400: redirect_uri_mismatch" shown up on the page. In this case, press \
 "Enter" to start again with a native application OAuth client ID.
+    If there is no local web server serving at ${REDIRECT_URI}, the \
+succeeded OAuth flow will land the browser on an error page ("This site can't \
+be reached"). This is an expected behavior. Copy the whole URL and continue.
+
 EOF
-    printf '%s' "4. Copy the authorization code from browser and paste here: "
+    printf '%s' "4. Copy the authorization code or complete url from browser \
+and paste here: "
     read -r auth_code
     if [[ -z ${auth_code} ]]; then
       printf '%s\n\n' "No authorization code. Starting from beginning again..."
       continue
     fi
+    auth_code=$(printf "%s" "${auth_code}" | sed 's/^.*code=//;s/&.*//')
+    printf '%s\n' "Got authorization code: ${auth_code}"
     auth_response=$(curl -s -d "code=${auth_code}" -d "client_id=${client_id}" \
--d "grant_type=authorization_code" -d "redirect_uri=urn:ietf:wg:oauth:2.0:oob" \
+-d "grant_type=authorization_code" -d "redirect_uri=${REDIRECT_URI}" \
 -d "client_secret=${client_secret}" "${OAUTH_BASE_URL}token")
     auth_error=$(node -e "console.log(!!JSON.parse(process.argv[1]).error)" \
 "${auth_response}")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -16,25 +16,25 @@
   },
   "homepage": "https://github.com/GoogleCloudPlatform/cloud-for-marketing/blob/master/marketing-analytics/activation/common-libs/nodejs-common/README.md",
   "dependencies": {
-    "@google-cloud/aiplatform": "^1.13.0",
-    "@google-cloud/automl": "^2.5.1",
-    "@google-cloud/bigquery": "^5.9.2",
+    "@google-cloud/aiplatform": "^1.19.0",
+    "@google-cloud/automl": "^2.5.2",
+    "@google-cloud/bigquery": "^5.12.0",
     "@google-cloud/datastore": "^6.6.2",
-    "@google-cloud/firestore": "^4.15.1",
-    "@google-cloud/logging-winston": "^4.1.1",
-    "@google-cloud/pubsub": "^2.18.2",
-    "@google-cloud/storage": "^5.16.0",
+    "@google-cloud/firestore": "^5.0.2",
+    "@google-cloud/logging-winston": "^4.2.2",
+    "@google-cloud/pubsub": "^2.19.0",
+    "@google-cloud/storage": "^5.18.3",
     "gaxios": "^4.3.2",
-    "google-ads-api": "^9.0.0",
-    "google-ads-node": "^7.0.0",
-    "google-auth-library": "^7.10.2",
-    "googleapis": "^91.0.0",
+    "google-ads-api": "^10.0.1",
+    "google-ads-node":"^8.0.1",
+    "google-auth-library": "^7.14.1",
+    "googleapis": "^100.0.0",
     "soap": "^0.43.0",
-    "winston": "^3.3.3",
+    "winston": "^3.7.2",
     "lodash": "^4.17.21"
   },
   "devDependencies": {
-    "jasmine": "^3.5.0"
+    "jasmine": "^4.0.2"
   },
   "scripts": {
     "test": "node node_modules/jasmine/bin/jasmine"

package/src/apis/ads_data_hub.js

@@ -37,9 +37,11 @@ class AdsDataHub {
    *
    * @param {GaxiosOptions|undefined=} options Used to setup for tests.
    * @param {string|undefined=} customerId ADH customer id.
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
    */
-  constructor(options, customerId = undefined) {
-    const authClient = new AuthClient(API_SCOPES);
+  constructor(options, customerId = undefined, env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     this.auth = authClient.getDefaultAuth();
     /** @const{GaxiosOptions} */ this.options = options || {};
     /** @const{string|undefined=} */ this.customerId = customerId;

package/src/apis/cloud_platform_apis.js

@@ -32,11 +32,11 @@ const API_VERSION = 'v1';
  * Google cloud client libraries.
  */
 class CloudPlatformApis {
-  constructor(projectId = process.env['GCP_PROJECT']) {
+  constructor(env = process.env) {
     /** @const {!AuthClient} */
-    const authClient = new AuthClient(API_SCOPES);
-    this.auth = authClient.getApplicationDefaultCredentials();
-    this.projectId = projectId;
+    const authClient = new AuthClient(API_SCOPES, env);
+    this.auth = authClient.getDefaultAuth();
+    this.projectId = env['GCP_PROJECT'];
   }
 
   /**

package/src/apis/doubleclick_bidmanager.js

@@ -56,8 +56,13 @@ let RequestBody;
  * https://developers.google.com/bid-manager/how-tos/authorizing
  */
 class DoubleClickBidManager {
-  constructor() {
-    const authClient = new AuthClient(API_SCOPES);
+  /**
+   * @constructor
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
+   */
+  constructor(env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     const auth = authClient.getDefaultAuth();
     /** @const {!google.doubleclickbidmanager} */
     this.instance = google.doubleclickbidmanager({
@@ -104,6 +109,20 @@ class DoubleClickBidManager {
     return response.data.queryId;
   }
 
+  /**
+   * Deletes a query.
+   * @param {number} queryId
+   * @return {!Promise<boolean>} Whether the query was deleted.
+   */
+  async deleteQuery(queryId) {
+    try {
+      await this.instance.queries.deletequery({ queryId });
+      return true;
+    } catch (error) {
+      console.error(error);
+      return false;
+    }
+  }
 }
 
 module.exports = {

package/src/apis/google_ads.js

@@ -31,6 +31,8 @@ const {
   services: {
     UploadClickConversionsRequest,
     UploadClickConversionsResponse,
+    UploadConversionAdjustmentsRequest,
+    UploadConversionAdjustmentsResponse,
     UploadUserDataRequest,
     UploadUserDataResponse,
     UserDataOperation,
@@ -51,8 +53,9 @@ const API_SCOPES = Object.freeze(['https://www.googleapis.com/auth/adwords',]);
 
 /**
  * List of properties that will be taken from the data file as elements of a
- * conversion.
+ * conversion or a conversion adjustment.
  * @see https://developers.google.com/google-ads/api/reference/rpc/latest/ClickConversion
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/ConversionAdjustment
  * @type {Array<string>}
  */
 const PICKED_PROPERTIES = [
@@ -65,6 +68,10 @@ const PICKED_PROPERTIES = [
   'conversion_value',
   'currency_code',
   'order_id',
+  'adjustment_type',
+  'adjustment_date_time',
+  'user_agent',
+  'gclid_date_time_pair',
 ];
 
 /**
@@ -88,9 +95,11 @@ const IDENTIFIERS = [
 const MAX_IDENTIFIERS_PER_USER = 20;
 
 /**
- * Configuration for uploading click conversions for Google Ads, includes:
+ * Configuration for uploading click conversions or conversion adjustments for
+ * Google Ads, includes:
  * gclid, conversion_action, conversion_date_time, conversion_value,
- * currency_code, order_id, external_attribution_data, etc.
+ * currency_code, order_id, external_attribution_data,
+ * adjustment_type, adjustment_date_time, user_agent, gclid_date_time_pair, etc.
  * @see PICKED_PROPERTIES
  *
  * Other properties that will be used to build the conversions but not picked by
@@ -116,12 +125,15 @@ const MAX_IDENTIFIERS_PER_USER = 20;
  *   conversion_value: number,
  *   currency_code:(string|undefined),
  *   order_id: (string|undefined),
- *   user_identifier_source:(UserIdentifierSource|undefined),
+ *   adjustment_type: (string|undefined),
+ *   adjustment_date_time: (!ConversionAdjustmentType|undefined),
+ *   user_agent: (string|undefined),
+ *   user_identifier_source:(!UserIdentifierSource|undefined),
  *   custom_variable_tags:(!Array<string>|undefined),
  *   customVariables:(!Object<string,string>|undefined),
  * }}
  */
-let ClickConversionConfig;
+let ConversionConfig;
 
 /**
  * Configuration for uploading customer match to Google Ads, includes:
@@ -255,7 +267,7 @@ class GoogleAds {
    * of click conversions.
    * @param {string} customerId
    * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
-   * @param {!ClickConversionConfig} adsConfig Default click conversion params
+   * @param {!ConversionConfig} adsConfig Default click conversion params
    * @return {!SendSingleBatch} Function which can send a batch of hits to
    *     Google Ads API.
    */
@@ -268,7 +280,7 @@ class GoogleAds {
      * @return {!BatchResult}
      */
     return async (lines, batchId) => {
-      /** @type {!Array<ClickConversionConfig>} */
+      /** @type {!Array<ConversionConfig>} */
       const conversions = lines.map(
           (line) => buildClickConversionFromLine(line, adsConfig, customerId));
       /** @const {BatchResult} */
@@ -300,6 +312,57 @@ class GoogleAds {
     }
   }
 
+  /**
+   * Returns the function to send out a request to Google Ads API with a batch
+   * of conversion adjustments.
+   * @param {string} customerId
+   * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
+   * @param {!ConversionConfig} adsConfig Default conversion adjustments
+   *     params.
+   * @return {!SendSingleBatch} Function which can send a batch of hits to
+   *     Google Ads API.
+   */
+  getUploadConversionAdjustmentFn(customerId, loginCustomerId, adsConfig) {
+    /**
+     * Sends a batch of hits to Google Ads API.
+     * @param {!Array<string>} lines Data for single request. It should be
+     *     guaranteed that it doesn't exceed quota limitation.
+     * @param {string} batchId The tag for log.
+     * @return {!BatchResult}
+     */
+    return async (lines, batchId) => {
+      /** @type {!Array<ConversionConfig>} */
+      const conversions = lines.map(
+          (line) => buildClickConversionFromLine(line, adsConfig, customerId));
+      /** @const {BatchResult} */
+      const batchResult = {
+        result: true,
+        numberOfLines: lines.length,
+      };
+      try {
+        const response = await this.uploadConversionAdjustments(conversions,
+            customerId, loginCustomerId);
+        const {results, partial_failure_error: failed} = response;
+        if (this.logger.isDebugEnabled()) {
+          const orderId = results.map((conversion) => conversion.order_id);
+          this.logger.debug('Uploaded order_id:', orderId);
+        }
+        if (failed) {
+          this.logger.info('partial_failure_error:', failed.message);
+          const failures = failed.details.map(
+              ({value}) => GoogleAdsFailure.decode(value));
+          this.extraFailedLines_(batchResult, failures, lines, 0);
+        }
+        return batchResult;
+      } catch (error) {
+        this.logger.error(
+            `Error in upload conversion adjustments batch: ${batchId}`, error);
+        this.updateBatchResultWithError_(batchResult, error, lines, 0);
+        return batchResult;
+      }
+    }
+  }
+
   /**
    * Updates the BatchResult based on errors.
    *
@@ -434,7 +497,7 @@ class GoogleAds {
    * Uploads click conversions to google ads account.
    * It requires an array of click conversions and customer id.
    * In DEBUG mode, this function will only validate the conversions.
-   * @param {Array<ClickConversionConfig>} clickConversions ClickConversions
+   * @param {Array<ConversionConfig>} clickConversions ClickConversions
    * @param {string} customerId
    * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
    * @return {!Promise<!UploadClickConversionsResponse>}
@@ -451,6 +514,31 @@ class GoogleAds {
     return customer.conversionUploads.uploadClickConversions(request);
   }
 
+  /**
+   * Uploads conversion adjustments to google ads account.
+   * It requires an array of conversion adjustments and customer id.
+   * In DEBUG mode, this function will only validate the conversion adjustments.
+   * @param {Array<ConversionAdjustment>} conversionAdjustments Conversion
+   *     adjustments.
+   * @param {string} customerId
+   * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
+   * @return {!Promise<!UploadConversionAdjustmentsResponse>}
+   */
+  uploadConversionAdjustments(conversionAdjustments, customerId,
+      loginCustomerId) {
+    this.logger.debug('Upload conversion adjustments for customerId:',
+        customerId);
+    const customer = this.getGoogleAdsApiCustomer_(loginCustomerId, customerId);
+    const request = new UploadConversionAdjustmentsRequest({
+      conversion_adjustments: conversionAdjustments,
+      customer_id: customerId,
+      validate_only: this.debugMode, // when true makes no changes
+      partial_failure: true, // Will still create the non-failed entities
+    });
+    return customer.conversionAdjustmentUploads.uploadConversionAdjustments(
+        request);
+  }
+
   /**
    * Returns the id of Conversion Custom Variable with the given tag.
    * @param {string} tag Custom Variable tag.
@@ -622,7 +710,7 @@ class GoogleAds {
 /**
  * Returns a conversion object based the given config and line data.
  * @param {string} line A JSON string of a conversion data.
- * @param {ClickConversionConfig} config Default click conversion params
+ * @param {ConversionConfig} config Default click conversion params
  * @param {string} customerId
  * @return {object} A conversion
  */
@@ -657,7 +745,7 @@ const buildClickConversionFromLine = (line, config, customerId) => {
 }
 
 module.exports = {
-  ClickConversionConfig,
+  ConversionConfig,
   CustomerMatchRecord,
   CustomerMatchConfig,
   GoogleAds,

package/src/apis/index.js

@@ -88,7 +88,7 @@ exports.bigquery = require('./bigquery.js');
  * APIs integration class for Google Ads.
  * @const {{
  *   GoogleAds:!GoogleAds,
- *   ClickConversionConfig:!ClickConversionConfig,
+ *   ConversionConfig:!ConversionConfig,
  *   CustomerMatchConfig: !CustomerMatchConfig,
  *   CustomerMatchRecord: !CustomerMatchRecord,
  *   ReportQueryConfig:!ReportQueryConfig,

package/src/components/utils.js

@@ -20,8 +20,7 @@
 
 const winston = require('winston');
 const {inspect} = require('util');
-const {LoggingWinston} = require('@google-cloud/logging-winston');
-const {CloudPlatformApis} = require('../apis/cloud_platform_apis.js');
+const { LoggingWinston } = require('@google-cloud/logging-winston');
 
 /**
  * The result of a batch of data sent to target API. The batch here means the
@@ -498,37 +497,6 @@ const extractObject = (paths) => {
   };
 };
 
-/**
- * Checks whether the permissions are granted for current authentication.
- * This function will be invoked during the deployment of a specific solution,
- * e.g. Tentacles, to make sure the operator has the proper permissions to
- * carry on. If the operator doesn't have enough permissions, it will exit with
- * status code 1 to let the invoker (the Bash installation script) know that it
- * doesn't pass.
- * @param {!Array<string>} permissions Array of permissions to check.
- * @param {string} projectId The Id of Cloud project.
- * @return {!Promise<undefined>}
- */
-const checkPermissions = (permissions,
-    projectId = process.env['GCP_PROJECT']) => {
-  const cloudPlatformApis = new CloudPlatformApis(projectId);
-  return cloudPlatformApis.testIamPermissions(permissions)
-      .then((grantedPermissions) => {
-        console.log(grantedPermissions);
-        grantedPermissions = grantedPermissions || [];
-        if (grantedPermissions.length < permissions.length) {
-          const missedPermissions = permissions.filter(
-              (permission) => grantedPermissions.indexOf(permission) === -1);
-          console.error(`[MISSED] ${missedPermissions.join(',')}.`);
-          process.exit(1);
-        }
-      })
-      .catch((error) => {
-        console.error(`[ERROR] ${error.message}`);
-        process.exit(1);
-      });
-};
-
 /**
  * For more details, see:
  * https://developers.google.com/google-ads/api/docs/rest/design/json-mappings
@@ -565,7 +533,6 @@ module.exports = {
   replaceParameters,
   getFilterFunction,
   extractObject,
-  checkPermissions,
   changeNamingFromSnakeToUpperCamel,
   changeNamingFromSnakeToLowerCamel,
 };