@google-cloud/nodejs-common 2.0.16-beta → 2.1.1

package/README.md

@@ -18,10 +18,11 @@ and [Data Tasks Coordinator]. This library includes:
    - Google Ads click conversions upload
    - Google Ads customer match upload
    - Google Ads enhanced conversions upload
+   - Google Ads offline userdata job data upload
    - Google Ads conversions scheduled uploads based on Google Sheets
    - Measurement Protocol Google Analytics 4
 
-1. Wrapper for some Google APIs for reporting, mainly
+2. Wrapper for some Google APIs for reporting, mainly
    for [Data Tasks Coordinator]:
 
    - Google Ads reporting
@@ -31,7 +32,7 @@ and [Data Tasks Coordinator]. This library includes:
    - YouTube Data API
    - Ads Data Hub querying
 
-1. Utilities wrapper class for Google Cloud Products:
+3. Utilities wrapper class for Google Cloud Products:
 
    - **Firestore Access Object**: Firestore has two modes[[comparison]] which
      have different API. This class, with its two successors, offer a unified
@@ -66,7 +67,7 @@ and [Data Tasks Coordinator]. This library includes:
      an adapter to wrap a Node8 Cloud Functions into Node6 and Node8 compatible
      functions.~~ (This has been removed since v1.9.0)
 
-1. A share library for [Bash] to facilitate installation tasks.
+4. A share library for [Bash] to facilitate installation tasks.
 
 [gmp and google ads connector]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/gmp-googleads-connector
 [data tasks coordinator]: https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/data-tasks-coordinator

package/bin/google_ads.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Google Ads API version
-GOOGLE_ADS_API_VERSION=13
+GOOGLE_ADS_API_VERSION=16
 
 #######################################
 # Verify whether the current OAuth token, CID and developer token can work.

package/bin/install_functions.sh

@@ -1560,6 +1560,7 @@ set_cloud_functions_default_settings() {
   default_cf_flag+=(--set-env-vars=DEBUG="${DEBUG}")
   default_cf_flag+=(--set-env-vars=IN_GCP="${IN_GCP}")
   default_cf_flag+=(--set-env-vars=DATABASE_ID="${DATABASE_ID}")
+  default_cf_flag+=(--set-env-vars=DATABASE_MODE="${DATABASE_MODE}")
 }
 
 #######################################

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "2.0.16-beta",
+  "version": "2.1.1",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",

package/src/apis/analytics.js

@@ -21,6 +21,7 @@
 
 const stream = require('stream');
 const {google} = require('googleapis');
+const { GoogleApiClient } = require('./base/google_api_client.js');
 const {Schema$Upload} = google.analytics;
 const AuthClient = require('./auth_client.js');
 const {wait, getLogger, BatchResult} = require('../components/utils.js');
@@ -55,31 +56,26 @@ let DataImportClearConfig;
 /**
  * Google Analytics API v3 stub.
  */
-class Analytics {
+class Analytics extends GoogleApiClient {
   /**
    * @constructor
    * @param {!Object<string,string>=} env The environment object to hold env
    *     variables.
    */
   constructor(env = process.env) {
-    this.authClient = new AuthClient(API_SCOPES, env);
+    super(env);
+    this.googleApi = 'analytics';
     this.logger = getLogger('API.GA');
   }
 
-  /**
-   * Prepares the Google Analytics instance.
-   * @return {!google.analytics}
-   * @private
-   */
-  async getApiClient_() {
-    if (this.analytics) return this.analytics;
-    await this.authClient.prepareCredentials();
-    this.logger.debug(`Initialized ${this.constructor.name} instance.`);
-    this.analytics = google.analytics({
-      version: API_VERSION,
-      auth: this.authClient.getDefaultAuth(),
-    });
-    return this.analytics;
+  /** @override */
+  getScope() {
+    return API_SCOPES;
+  }
+
+  /** @override */
+  getVersion() {
+    return API_VERSION;
   }
 
   /**
@@ -102,7 +98,7 @@ class Analytics {
         },
         config);
 
-    const analytics = await this.getApiClient_();
+    const analytics = await this.getApiClient();
     const response = await analytics.management.uploads.uploadData(
         uploadConfig);
     this.logger.debug('Configuration: ', config);
@@ -151,7 +147,7 @@ class Analytics {
    * @return {!Promise<!Schema$Upload>} Updated data import Job status.
    */
   async checkJobStatus(jobConfig) {
-    const analytics = await this.getApiClient_();
+    const analytics = await this.getApiClient();
     const { data: job } = await analytics.management.uploads.get(jobConfig);
     if (job.status !== 'PENDING') return job;
     this.logger.debug(
@@ -169,7 +165,7 @@ class Analytics {
    * @return {!Promise<!Array<string>>}
    */
   async listAccounts() {
-    const analytics = await this.getApiClient_();
+    const analytics = await this.getApiClient();
     const response = await analytics.management.accounts.list();
     return response.data.items.map(
         (account) => `Account id: ${account.name}[${account.id}]`
@@ -182,7 +178,7 @@ class Analytics {
    * @return {!Promise<!Array<Object>>}
    */
   async listUploads(config) {
-    const analytics = await this.getApiClient_();
+    const analytics = await this.getApiClient();
     const response = await analytics.management.uploads.list(config);
     return response.data.items;
   }
@@ -203,7 +199,7 @@ class Analytics {
     const request = Object.assign({}, config, {
       resource: {customDataImportUids},
     });
-    const analytics = await this.getApiClient_();
+    const analytics = await this.getApiClient();
     await analytics.management.uploads.deleteUploadData(request);
     this.logger.debug('Delete uploads: ', customDataImportUids);
   }

package/src/apis/base/ads_api_common.js

@@ -0,0 +1,99 @@
+// Copyright 2024 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this fileAccessObject except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+/**
+ * @fileoverview Common functions for Google Ads and Search Ads API classes.
+ */
+
+'use strict';
+
+const { Transform } = require('stream');
+const {
+  getFilterAndStringifyFn,
+  getLogger,
+} = require('../../components/utils.js');
+
+/**
+ * Returns a integer format CID by removing dashes and spaces.
+ * @param {string} cid
+ * @return {string}
+ */
+function getCleanCid(cid) {
+  return cid.toString().trim().replace(/-/g, '');
+}
+
+const START_TAG = '"results":';
+const FIELD_MASK_TAG = '"fieldMask"';
+const END_TAG = '"requestId"';
+
+/**
+ * A stream.Transform that can extract properties and convert naming of the
+ * reponse of Google/Search Ads report from REST interface.
+ */
+class RestSearchStreamTransform extends Transform {
+
+  /**
+   * @constructor
+   * @param {boolean=} snakeCase Whether or not output JSON in snake naming.
+   */
+  constructor(snakeCase = false) {
+    super({ objectMode: true });
+    this.snakeCase = snakeCase;
+    this.chunks = [Buffer.from('')];
+    this.processFn; // The function to process a row of the report.
+    this.logger = getLogger('ADS.STREAM.T');
+    this.stopwatch = Date.now();
+  }
+
+  _transform(chunk, encoding, callback) {
+    const latest = Buffer.concat([this.chunks[this.chunks.length - 1], chunk]);
+    const endIndex = latest.indexOf(END_TAG);
+    if (endIndex > -1) {
+      this.chunks.push(chunk);
+      const rawString = Buffer.concat(this.chunks).toString();
+      const startIndex = rawString.indexOf(START_TAG) + START_TAG.length;
+      const maskIndex = rawString.lastIndexOf(FIELD_MASK_TAG);
+      if (!this.processFn) {
+        const fieldMask = rawString
+          .substring(maskIndex + FIELD_MASK_TAG.length, rawString.indexOf(END_TAG))
+          .split('"')[1];
+        this.logger.debug(`Got fieldMask: ${fieldMask}`);
+        this.processFn = getFilterAndStringifyFn(fieldMask, this.snakeCase);
+      }
+      const resultsWithTailing = rawString.substring(startIndex, maskIndex);
+      const results = resultsWithTailing.substring(
+        0, resultsWithTailing.lastIndexOf(','));
+      const rows = JSON.parse(results);
+      const data = rows.map(this.processFn).join('\n') + '\n';
+      // Clear cached chunks.
+      this.chunks = [latest.subarray(latest.indexOf(END_TAG) + END_TAG.length)];
+
+      this.logger.debug(`Got ${rows.length} rows. Process time:`,
+        Date.now() - this.stopwatch);
+      this.stopwatch = Date.now();
+      callback(null, data);
+    } else {
+      if (chunk.length < END_TAG.length) {// Update latest chunk for short chunk
+        this.chunks[this.chunks.length - 1] = latest;
+      } else {
+        this.chunks.push(chunk);
+      }
+      callback();
+    }
+  }
+}
+
+module.exports = {
+  getCleanCid,
+  RestSearchStreamTransform,
+};

package/src/apis/base/google_api_client.js

@@ -0,0 +1,55 @@
+// Copyright 2023 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @fileoverview A base class for Google Api client library class.
+ */
+const { google } = require('googleapis');
+const { getLogger, getObjectByPath } = require('../../components/utils.js');
+const { AuthRestfulApi } = require('./auth_restful_api.js');
+
+/**
+ * A Google Api client library class.
+ */
+class GoogleApiClient extends AuthRestfulApi {
+
+  /** @constructor */
+  constructor(env = process.env, options = {}) {
+    super(env, options);
+    this.logger = getLogger('API.default');
+  }
+
+  /**
+   * Returns the Api version of the Api in the current library.
+   * @return {string}
+   * @abstract
+   */
+  getVersion() { }
+
+  /**
+   * Returns the Api instance.
+   * @return {!Promise<object>} The Api instance.
+   */
+  async getApiClient() {
+    if (this.apiClient) return this.apiClient;
+    this.logger.info(`Initialized ${this.constructor.name} instance.`);
+    this.apiClient = google[this.googleApi]({
+      version: this.getVersion(),
+      auth: await this.getAuth(),
+    });
+    return this.apiClient;
+  }
+}
+
+module.exports = { GoogleApiClient };

package/src/apis/dfa_reporting.js

@@ -19,9 +19,8 @@
 
 'use strict';
 
-const {google} = require('googleapis');
-const {request} = require('gaxios');
-const AuthClient = require('./auth_client.js');
+const { request } = require('gaxios');
+const { GoogleApiClient } = require('./base/google_api_client.js');
 const {
   getLogger,
   getFilterFunction,
@@ -94,7 +93,7 @@ const MAX_IDENTIFIERS_PER_USER = 5;
  * Google DfaReport API v3.0 stub.
  * see https://developers.google.com/doubleclick-advertisers/service_accounts
  */
-class DfaReporting {
+class DfaReporting extends GoogleApiClient {
 
   /**
    * @constructor
@@ -102,34 +101,19 @@ class DfaReporting {
    *     variables.
    */
   constructor(env = process.env) {
-    this.authClient = new AuthClient(API_SCOPES, env);
+    super(env);
+    this.googleApi = 'dfareporting';
     this.logger = getLogger('API.CM');
   }
 
-  /**
-    * Prepares the Google DfaReport API instance.
-    * @return {!google.dfareporting}
-    * @private
-    */
-  async getApiClient_() {
-    if (this.dfareporting) return this.dfareporting;
-    this.logger.debug(`Initialized ${this.constructor.name} instance.`);
-    this.dfareporting = google.dfareporting({
-      version: API_VERSION,
-      auth: await this.getAuth_(),
-    });
-    return this.dfareporting;
+  /** @override */
+  getScope() {
+    return API_SCOPES;
   }
 
-  /**
-   * Gets the auth object.
-   * @return {!Promise<{!OAuth2Client|!JWT|!Compute}>}
-   */
-  async getAuth_() {
-    if (this.auth) return this.auth;
-    await this.authClient.prepareCredentials();
-    this.auth = this.authClient.getDefaultAuth();
-    return this.auth;
+  /** @override */
+  getVersion() {
+    return API_VERSION;
   }
 
   /**
@@ -140,7 +124,7 @@ class DfaReporting {
    * @return {!Promise<string>}
    */
   async getProfileId(accountId) {
-    const dfareporting = await this.getApiClient_();
+    const dfareporting = await this.getApiClient();
     const { data: { items } } = await dfareporting.userProfiles.list();
     const profiles = items.filter(
         (profile) => profile.accountId === accountId
@@ -237,7 +221,7 @@ class DfaReporting {
         numberOfLines: lines.length,
       };
       try {
-        const dfareporting = await this.getApiClient_();
+        const dfareporting = await this.getApiClient();
         const response = await dfareporting.conversions[operation]({
           profileId: config.profileId,
           requestBody: requestBody,
@@ -304,7 +288,7 @@ class DfaReporting {
    * @return {!Promise<!Array<string>>}
    */
   async listUserProfiles() {
-    const dfareporting = await this.getApiClient_();
+    const dfareporting = await this.getApiClient();
     const { data: { items } } = await dfareporting.userProfiles.list();
     return items.map(({profileId, userName, accountId, accountName}) => {
       return `Profile: ${profileId}[${userName}] `
@@ -349,7 +333,7 @@ class DfaReporting {
    */
   async runReport(config) {
     const profileId = await this.getProfileForOperation_(config);
-    const dfareporting = await this.getApiClient_();
+    const dfareporting = await this.getApiClient();
     const { startDate, endDate } = config;
     if (startDate && endDate) {
       const { data: report } = await dfareporting.reports.get({
@@ -392,7 +376,7 @@ class DfaReporting {
    */
   async getReportFileUrl(config) {
     const profileId = await this.getProfileForOperation_(config);
-    const dfareporting = await this.getApiClient_();
+    const dfareporting = await this.getApiClient();
     const response = await dfareporting.reports.files.get({
       profileId,
       reportId: config.reportId,
@@ -410,7 +394,7 @@ class DfaReporting {
    * @return {!Promise<stream>}
    */
   async getReportFileStream(url) {
-    const auth = await this.getAuth_();
+    const auth = await this.getAuth();
     const headers = await auth.getRequestHeaders();
     const response = await request({
       method: 'GET',

package/src/apis/display_video.js

@@ -19,14 +19,8 @@
 
 'use strict';
 
-const { google } = require('googleapis');
-const AuthClient = require('./auth_client.js');
-const {
-  getLogger,
-  getObjectByPath,
-  SendSingleBatch,
-  BatchResult,
-} = require('../components/utils.js');
+const { GoogleApiClient } = require('./base/google_api_client.js');
+const { getLogger } = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
   'https://www.googleapis.com/auth/display-video',
@@ -34,13 +28,13 @@ const API_SCOPES = Object.freeze([
 const API_VERSION = 'v3';
 
 /**
- * Display and Video 360 API v2 stub.
- * @see https://developers.google.com/display-video/api/reference/rest/v2
+ * Display and Video 360 API v3 stub.
+ * @see https://developers.google.com/display-video/api/reference/rest/v3
  * This is not the same to Reports Display & Video 360 API which is from Google
  * Bid Manager API.
  * @see https://developers.google.com/bid-manager/reference/rest
  */
-class DisplayVideo {
+class DisplayVideo extends GoogleApiClient {
 
   /**
    * @constructor
@@ -48,44 +42,19 @@ class DisplayVideo {
    *     variables.
    */
   constructor(env = process.env) {
-    this.authClient = new AuthClient(API_SCOPES, env);
+    super(env);
+    this.googleApi = 'displayvideo';
     this.logger = getLogger('API.DV3API');
   }
 
-  /**
-   * Prepares the Google DV3 instance.
-   * @return {!google.displayvideo}
-   * @private
-   */
-  async getApiClient_() {
-    if (this.displayvideo) return this.displayvideo;
-    this.logger.debug(`Initialized ${this.constructor.name} instance.`);
-    this.displayvideo = google.displayvideo({
-      version: API_VERSION,
-      auth: await this.getAuth_(),
-    });
-    return this.displayvideo;
-  }
-
-  /**
-   * Gets the auth object.
-   * @return {!Promise<{!OAuth2Client|!JWT|!Compute}>}
-   */
-  async getAuth_() {
-    if (this.auth) return this.auth;
-    await this.authClient.prepareCredentials();
-    this.auth = this.authClient.getDefaultAuth();
-    return this.auth;
+  /** @override */
+  getScope() {
+    return API_SCOPES;
   }
 
-  /**
-   * Gets the instance of function object based on Google API client library.
-   * @param {string|undefined} path
-   * @return {Object}
-   */
-  async getFunctionObject(path) {
-    const instance = await this.getApiClient_();
-    return getObjectByPath(instance, path);
+  /** @override */
+  getVersion() {
+    return API_VERSION;
   }
 
 }

package/src/apis/doubleclick_bidmanager.js

@@ -19,8 +19,7 @@
 
 'use strict';
 
-const {google} = require('googleapis');
-const AuthClient = require('./auth_client.js');
+const { GoogleApiClient } = require('./base/google_api_client.js');
 const { getLogger } = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
@@ -28,25 +27,24 @@ const API_SCOPES = Object.freeze([
 ]);
 const API_VERSION = 'v2';
 
-/**
- * The returned information of get a query.
- * @typedef {{
- *   running:boolean,
- *   latestReportRunTimeMs:string,
- *   googleCloudStoragePathForLatestReport:string,
- * }}
- */
-let QueryResource;
-
 /**
  * RequestBody controls the data range of reports.
  * see:
- * https://developers.google.com/bid-manager/v1.1/queries/runquery#request-body
+ * https://developers.google.com/bid-manager/reference/rest/v2/queries/run#RunQueryRequest
  * @typedef {{
- *   dataRange: string,
- *   reportDataStartTimeMs: long,
- *   reportDataEndTimeMs: long,
- *   timezoneCode: string
+ *   dataRange: {
+ *     range: Range,
+ *     customStartDate: {
+ *       year: integer,
+ *       month: integer,
+ *       day: integer,
+ *     },
+ *     customEndDate: {
+*       year: integer,
+*       month: integer,
+*       day: integer,
+*     },
+ *   }
  * }}
  */
 let RequestBody;
@@ -56,31 +54,26 @@ let RequestBody;
  * Note: DV360 report API only support OAuth 2.0, see:
  * https://developers.google.com/bid-manager/how-tos/authorizing
  */
-class DoubleClickBidManager {
+class DoubleClickBidManager extends GoogleApiClient {
   /**
    * @constructor
    * @param {!Object<string,string>=} env The environment object to hold env
    *     variables.
    */
   constructor(env = process.env) {
-    this.authClient = new AuthClient(API_SCOPES, env);
+    super(env);
+    this.googleApi = 'doubleclickbidmanager';
     this.logger = getLogger('API.DV3');
   }
 
-  /**
-   * Prepares the Google DBM instance.
-   * @return {!google.doubleclickbidmanager}
-   * @private
-   */
-  async getApiClient_() {
-    if (this.doubleclickbidmanager) return this.doubleclickbidmanager;
-    await this.authClient.prepareCredentials();
-    this.logger.debug(`Initialized ${this.constructor.name} instance.`);
-    this.doubleclickbidmanager = google.doubleclickbidmanager({
-      version: API_VERSION,
-      auth: this.authClient.getDefaultAuth(),
-    });
-    return this.doubleclickbidmanager;
+  /** @override */
+  getScope() {
+    return API_SCOPES;
+  }
+
+  /** @override */
+  getVersion() {
+    return API_VERSION;
   }
 
   /**
@@ -91,21 +84,21 @@ class DoubleClickBidManager {
    * @return {!Promise<number>} Report Id.
    */
   async runQuery(queryId, requestBody = undefined) {
-    const doubleclickbidmanager = await this.getApiClient_();
+    const doubleclickbidmanager = await this.getApiClient();
     const response = await doubleclickbidmanager.queries.run(
         {queryId, requestBody});
     return response.data.key.reportId;
   }
 
   /**
-   * Gets a query resource.
+   * Gets a query metadata.
    * See https://developers.google.com/bid-manager/reference/rest/v2/queries/get
    * @param {number} queryId Id of the query.
-   * @return {!Promise<!QueryResource>} Query resource, see
-   *     https://developers.google.com/bid-manager/reference/rest/v2/queries#Query
+   * @return {!Promise<!QueryMetadata>} Query metadata, see
+   *     https://developers.google.com/bid-manager/reference/rest/v2/queries#QueryMetadata
    */
   async getQuery(queryId) {
-    const doubleclickbidmanager = await this.getApiClient_();
+    const doubleclickbidmanager = await this.getApiClient();
     const response = await doubleclickbidmanager.queries.get({ queryId });
     return response.data.metadata;
   }
@@ -117,14 +110,14 @@ class DoubleClickBidManager {
    * @return {!Promise<number>} Id of created query.
    */
   async createQuery(query) {
-    const doubleclickbidmanager = await this.getApiClient_();
+    const doubleclickbidmanager = await this.getApiClient();
     const response = await doubleclickbidmanager.queries.create(
         {requestBody: query});
     return response.data.queryId;
   }
 
   async getQueryReport(queryId, reportId) {
-    const doubleclickbidmanager = await this.getApiClient_();
+    const doubleclickbidmanager = await this.getApiClient();
     const response = await doubleclickbidmanager.queries.reports.get(
       { queryId, reportId });
     return response.data.metadata;
@@ -136,7 +129,7 @@ class DoubleClickBidManager {
    * @return {!Promise<boolean>} Whether the query was deleted.
    */
   async deleteQuery(queryId) {
-    const doubleclickbidmanager = await this.getApiClient_();
+    const doubleclickbidmanager = await this.getApiClient();
     try {
       const { status } = await doubleclickbidmanager.queries.delete({ queryId });
       return status === 200;
@@ -148,7 +141,6 @@ class DoubleClickBidManager {
 }
 
 module.exports = {
-  QueryResource,
   RequestBody,
   DoubleClickBidManager,
 };