@google-cloud/nodejs-common 0.9.5-alpha → 1.0.0

package/README.md

@@ -1,69 +1,78 @@
 # NodeJS Common Library
 
-<!--* freshness: { owner: 'lushu' reviewed: '2021-06-01' } *-->
-
-A NodeJs common library for other projects, e.g. [GMP and Google Ads 
-Connector] and [Data Tasks Coordinator]. This library includes:
-
-1.  Authentication wrapper based on google auth library to support OAuth, JWT
-    and ADC authentication;
-
-1.  Wrapper for some Google APIs for integration, mainly for [GMP and Google Ads 
- Connector]:
-
-    *   Google Analytics data import
-    *   Google Analytics measurement protocol
-    *   Campaign Manager offline conversion upload    
-    *   Search Ads 360 conversions upload
-    *   Google Ads click conversions upload
-    *   Google Ads customer match upload
-    *   Google Ads conversions scheduled uploads based on Google Sheets
-    *   Measurement Protocol Google Analytics 4
-
-1.  Wrapper for some Google APIs for reporting, mainly for [Data Tasks Coordinator]:
-
-    *   Google Ads reporting
-    *   Campaign Manager reporting
-    *   Search Ads 360 reporting
-    *   Display and Video 360 reporting
-    *   Ads Data Hub querying
-
-1.  Utilities wrapper class for Google Cloud Products:
-
-    *   **Firestore Access Object**: Firestore has two modes which are excluded
-        to each other and can't be changed once selected in a Cloud
-        Project[[2]]. This class, with its two successors offer a unified
-        interface to operate data objects on either Firestore or Datastore.
-
-    *  **AutoMl Tables API**: Offers a unified entry to use this API based on 
-        Google Cloud client library combined with REST requests to service 
-        directly due to some functionalities missed in the client library.
-
-    *   **Pub/Sub Utilities**: Offers utilities functions to create topics and
-        subscriptions for Pub/Sub, as well as the convenient way to publish a
-        message.
-
-    *   **Storage Utilities**: Offers functions to manipulate the files on Cloud
-        Storage. The main functions are:
-
-        *   Reading a given length (or slightly less) content without breaking a
-            line;
-        *   Splitting a file into multiple files with the given length (or
-            slightly less) without breaking a line;
-        *   Merging files into one file.
-    
-    *  **Cloud Scheduler Adapter**: A wrapper to pause and resume Cloud 
-       Scheduler jobs.
-    
-    *  **Cloud Functions Adapter**: Cloud Functions have different parameters
-       in different environments, e.g. Node6 vs Node8[[1]]. This utility file
-       offers an adapter to wrap a Node8 Cloud Functions into Node6 and Node8
-       compatible functions.
-       
+<!--* freshness: { owner: 'lushu' reviewed: '2021-12-02' } *-->
+
+A NodeJs common library for other projects, e.g. [GMP and Google Ads Connector]
+and [Data Tasks Coordinator]. This library includes:
+
+1. Authentication wrapper based on google auth library to support OAuth, JWT and
+   ADC authentication;
+
+1. Wrapper for some Google APIs for integration, mainly
+   for [GMP and Google Ads Connector]:
+
+    * Google Analytics data import
+    * Google Analytics measurement protocol
+    * Campaign Manager offline conversion upload
+    * Search Ads 360 conversions upload
+    * Google Ads click conversions upload
+    * Google Ads customer match upload
+    * Google Ads conversions scheduled uploads based on Google Sheets
+    * Measurement Protocol Google Analytics 4
+
+1. Wrapper for some Google APIs for reporting, mainly
+   for [Data Tasks Coordinator]:
+
+    * Google Ads reporting
+    * Campaign Manager reporting
+    * Search Ads 360 reporting
+    * Display and Video 360 reporting
+    * YouTube Data API
+    * Ads Data Hub querying
+
+1. Utilities wrapper class for Google Cloud Products:
+
+    * **Firestore Access Object**: Firestore has two modes which are excluded to
+      each other and can't be changed once selected in a Cloud Project[[2]].
+      This class, with its two successors offer a unified interface to operate
+      data objects on either Firestore or Datastore.
+
+    * **AutoMl Tables API**: Offers a unified entry to use this API based on
+      Google Cloud client library combined with REST requests to service
+      directly due to some functionalities missed in the client library.
+
+    * **Vertex AI API**: Offers a unified entry to use this API based on Google
+      Cloud client library.
+
+    * **Pub/Sub Utilities**: Offers utilities functions to create topics and
+      subscriptions for Pub/Sub, as well as the convenient way to publish a
+      message.
+
+    * **Storage Utilities**: Offers functions to manipulate the files on Cloud
+      Storage. The main functions are:
+
+        * Reading a given length (or slightly less) content without breaking a
+          line;
+        * Splitting a file into multiple files with the given length (or
+          slightly less) without breaking a line;
+        * Merging files into one file.
+
+    * **Cloud Scheduler Adapter**: A wrapper to pause and resume Cloud Scheduler
+      jobs.
+
+    * **Cloud Functions Adapter**: Cloud Functions have different parameters in
+      different environments, e.g. Node6 vs Node8[[1]]. This utility file offers
+      an adapter to wrap a Node8 Cloud Functions into Node6 and Node8 compatible
+      functions.
+
 1. A share library for [Bash] to facilitate installation tasks.
 
 [GMP and Google Ads Connector]:https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/gmp-googleads-connector
+
 [Data Tasks Coordinator]:https://github.com/GoogleCloudPlatform/cloud-for-marketing/tree/master/marketing-analytics/activation/data-tasks-coordinator
+
 [1]:https://cloud.google.com/functions/docs/writing/background#functions-writing-background-hello-pubsub-node8-10
+
 [2]:https://cloud.google.com/datastore/docs/concepts/overview#comparison_with_traditional_databases
+
 [Bash]:https://www.gnu.org/software/bash/

package/bin/install_functions.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Cloud Functions Runtime Environment.
-CF_RUNTIME="${CF_RUNTIME:=nodejs10}"
+CF_RUNTIME="${CF_RUNTIME:=nodejs14}"
 
 # Counter for steps.
 STEP=0
@@ -743,8 +743,8 @@ select_dataset_location() {
     "ASIA_PACIFIC"
   )
   local MULTI_REGIONAL=(
-    "Data centers within member states of the European Union (EU)"
-    "Data centers in the United States (US)"
+    "Data centers within member states of the European Union (eu)"
+    "Data centers in the United States (us)"
   )
   local AMERICAS=(
     "${NORTH_AMERICA[@]}"
@@ -1199,21 +1199,17 @@ create_or_update_sink() {
   local existingFilter
   existingFilter=$(gcloud logging sinks list --filter="name:${sinkName}" \
 --format="value(filter)")
-  if [[ "${existingFilter}" != "${logFilter}" ]]; then
-    local action
-    if [[ -z "${existingFilter}" ]]; then
-      action="create"
-      printf '%s\n' "  Logging Export [${sinkName}] doesn't exist. Creating..."
-    else
-      action="update"
-      printf '%s\n' "  Logging Export [${sinkName}] exists with a different \
-filter. Updating..."
-    fi
-    gcloud -q logging sinks ${action} "${sinkName}" "${sinkDestAndFlags[@]}" \
---log-filter="${logFilter}"
+  local action
+  if [[ -z "${existingFilter}" ]]; then
+    action="create"
+    printf '%s\n' "  Logging Export [${sinkName}] doesn't exist. Creating..."
   else
-    printf '%s\n' "  Logging Export [${sinkName}] exists. Continue..."
+    action="update"
+    printf '%s\n' "  Logging Export [${sinkName}] exists with a different \
+filter. Updating..."
   fi
+  gcloud -q logging sinks ${action} "${sinkName}" "${sinkDestAndFlags[@]}" \
+--log-filter="${logFilter}"
   if [[ $? -gt 0 ]];then
     printf '%s\n' "Failed to create or update Logs router sink."
     return 1
@@ -1833,9 +1829,8 @@ get_cloud_functions_service_account() {
 #   None.
 #######################################
 check_firestore_existence() {
-  local firestore_status
-  firestore_status=$(gcloud firestore operations list 2>&1)
-  while [[ ${firestore_status} =~ .*NOT_FOUND.* ]]; do
+  gcloud firestore indexes fields list >/dev/null 2>&1
+  while [[ $? -gt 0 ]]; do
     cat <<EOF
 Cannot find Firestore or Datastore in current project. Please visit \
 https://console.cloud.google.com/firestore?project=${GCP_PROJECT} to create a \
@@ -1846,7 +1841,7 @@ EOF
     local any
     read -n1 -s any
     printf '\n'
-    firestore_status=$(gcloud firestore operations list 2>&1)
+    gcloud firestore indexes fields list >/dev/null 2>&1
   done
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "0.9.5-alpha",
+  "version": "1.0.0",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -16,21 +16,22 @@
   },
   "homepage": "https://github.com/GoogleCloudPlatform/cloud-for-marketing/blob/master/marketing-analytics/activation/common-libs/nodejs-common/README.md",
   "dependencies": {
-    "@google-cloud/aiplatform": "^1.10.1",
-    "@google-cloud/automl": "^2.4.2",
-    "@google-cloud/bigquery": "^5.6.0",
-    "@google-cloud/datastore": "^6.4.0",
-    "@google-cloud/firestore": "^4.12.2",
-    "@google-cloud/logging-winston": "^4.0.5",
-    "@google-cloud/pubsub": "^2.12.0",
-    "@google-cloud/storage": "^5.8.5",
-    "gaxios": "^4.3.0",
-    "google-ads-api": "^8.1.0",
-    "google-ads-node": "^6.1.3",
-    "google-auth-library": "^7.1.0",
-    "googleapis": "^74.2.0",
-    "soap": "^0.38.0",
-    "winston": "^3.3.3"
+    "@google-cloud/aiplatform": "^1.13.0",
+    "@google-cloud/automl": "^2.5.1",
+    "@google-cloud/bigquery": "^5.9.2",
+    "@google-cloud/datastore": "^6.6.2",
+    "@google-cloud/firestore": "^4.15.1",
+    "@google-cloud/logging-winston": "^4.1.1",
+    "@google-cloud/pubsub": "^2.18.2",
+    "@google-cloud/storage": "^5.16.0",
+    "gaxios": "^4.3.2",
+    "google-ads-api": "^9.0.0",
+    "google-ads-node": "^7.0.0",
+    "google-auth-library": "^7.10.2",
+    "googleapis": "^91.0.0",
+    "soap": "^0.43.0",
+    "winston": "^3.3.3",
+    "lodash": "^4.17.21"
   },
   "devDependencies": {
     "jasmine": "^3.5.0"

package/src/apis/doubleclick_search.js

@@ -267,7 +267,7 @@ class DoubleClickSearch {
     batchResult.failedLines = [];
     batchResult.groupedFailed = {};
     const errors = new Set();
-    const messageReg = /.*Details: \[(.*) index=\d* conversionId=.*/;
+    const messageReg = /.*Details: \[(.*) index=\d+ conversionId=.*/;
     const indexReg = /.*index=(\d*) .*/;
     errorMessages.forEach((message) => {
       const errorMessage = messageReg.exec(message);

package/src/apis/google_ads.js

@@ -41,25 +41,77 @@ const {
   },
 } = googleAdsLib;
 const {GoogleAdsApi} = require('google-ads-api');
+const lodash = require('lodash');
+
 const AuthClient = require('./auth_client.js');
 const {getLogger, BatchResult,} = require('../components/utils.js');
 
 /** @type {!ReadonlyArray<string>} */
 const API_SCOPES = Object.freeze(['https://www.googleapis.com/auth/adwords',]);
 
+/**
+ * List of properties that will be taken from the data file as elements of a
+ * conversion.
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/ClickConversion
+ * @type {Array<string>}
+ */
+const PICKED_PROPERTIES = [
+  'external_attribution_data',
+  'cart_data',
+  'user_identifiers',
+  'gclid',
+  'conversion_action',
+  'conversion_date_time',
+  'conversion_value',
+  'currency_code',
+  'order_id',
+];
+
+/**
+ * Kinds of UserIdentifier.
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserIdentifier
+ * @type {Array<string>}
+ */
+const IDENTIFIERS = [
+  'hashed_email',
+  'hashed_phone_number',
+  'mobile_id',
+  'third_party_user_id',
+  'address_info',
+];
+
 /**
  * Configuration for uploading click conversions for Google Ads, includes:
  * gclid, conversion_action, conversion_date_time, conversion_value,
- * currency_code, order_id, external_attribution_data
+ * currency_code, order_id, external_attribution_data, etc.
+ * @see PICKED_PROPERTIES
+ *
+ * Other properties that will be used to build the conversions but not picked by
+ * the value directly including:
+ * 1. 'user_identifier_source', source of the user identifier. If there is user
+ * identifiers information in the conversion, this property should be set as
+ * 'FIRST_PARTY'.
+ * @see IDENTIFIERS
+ * @see https://developers.google.com/google-ads/api/reference/rpc/latest/UserIdentifier?hl=en
+ * 2. 'custom_variable_tags', the tags of conversion custom variables. To upload
+ * custom variables, 'conversion_custom_variable_id' is required rather than the
+ * 'tag'. So the invoker is expected to use the function
+ * 'getConversionCustomVariableId' to get the ids and pass in as a
+ * map(customVariables) of <tag, id> pairs before uploading conversions.
+ *
  * @see https://developers.google.com/google-ads/api/reference/rpc/latest/ClickConversion
  * @typedef {{
+ *   external_attribution_data: (GoogleAdsApi.ExternalAttributionData|undefined),
+ *   cart_data: (object|undefined),
  *   gclid: string,
  *   conversion_action: string,
  *   conversion_date_time: string,
  *   conversion_value: number,
  *   currency_code:(string|undefined),
  *   order_id: (string|undefined),
- *   external_attribution_data: (GoogleAdsApi.ExternalAttributionData|undefined),
+ *   user_identifier_source:(UserIdentifierSource|undefined),
+ *   custom_variable_tags:(!Array<string>|undefined),
+ *   customVariables:(!object<string,string>|undefined),
  * }}
  */
 let ClickConversionConfig;
@@ -219,10 +271,8 @@ class GoogleAds {
      */
     return async (lines, batchId) => {
       /** @type {!Array<ClickConversionConfig>} */
-      const conversions = lines.map((line) => {
-        const record = JSON.parse(line);
-        return Object.assign({}, adsConfig, record);
-      });
+      const conversions = lines.map(
+          (line) => buildClickConversionFromLine(line, adsConfig, customerId));
       /** @const {BatchResult} */
       const batchResult = {
         result: true,
@@ -346,7 +396,6 @@ class GoogleAds {
    *     failure.
    *   groupedFailed - a hashmap of failed the lines. The key is the reason, the
    *     value is the array of failed lines due to this reason.
-   *     TODO: Confirm how to surface and handle groupedFailed.
    * @param {!BatchResult} batchResult
    * @param {!Array<!GoogleAdsFailure>} failures
    * @param {!Array<string>} lines The original input data.
@@ -404,6 +453,26 @@ class GoogleAds {
     return customer.conversionUploads.uploadClickConversions(request);
   }
 
+  /**
+   * Returns the id of Conversion Custom Variable with the given tag.
+   * @param {string} tag Custom Variable tag.
+   * @param {string} customerId
+   * @param {string} loginCustomerId Login customer account ID (Mcc Account id).
+   * @return {Promise<number|undefined>} Returns undefined if can't find tag.
+   */
+  async getConversionCustomVariableId(tag, customerId, loginCustomerId) {
+    const customer = this.getGoogleAdsApiCustomer_(loginCustomerId, customerId);
+    const customVariables = await customer.query(`
+        SELECT conversion_custom_variable.id,
+               conversion_custom_variable.tag
+        FROM conversion_custom_variable
+        WHERE conversion_custom_variable.tag = "${tag}" LIMIT 1
+    `);
+    if (customVariables.length > 0) {
+      return customVariables[0].conversion_custom_variable.id;
+    }
+  }
+
   /**
    * Returns the function to send out a request to Google Ads API with
    * user ids for Customer Match upload
@@ -535,6 +604,43 @@ class GoogleAds {
 
 }
 
+/**
+ * Returns a conversion object based the given config and line data.
+ * @param {string} line A JSON string of a conversion data.
+ * @param {ClickConversionConfig} config Default click conversion params
+ * @param {string} customerId
+ * @return {object} A conversion
+ */
+const buildClickConversionFromLine = (line, config, customerId) => {
+  const {customVariables, user_identifier_source} = config;
+  const record = JSON.parse(line);
+  const conversion = lodash.merge(lodash.pick(config, PICKED_PROPERTIES),
+      lodash.pick(record, PICKED_PROPERTIES));
+  if (customVariables) {
+    const tags = Object.keys(customVariables);
+    conversion.custom_variables = tags.map((tag) => {
+      return {
+        conversion_custom_variable:
+            `customers/${customerId}/conversionCustomVariables/${customVariables[tag]}`,
+        value: record[tag],
+      };
+    });
+  }
+  const user_identifiers = [];
+  IDENTIFIERS.forEach((identifier) => {
+    if (record[identifier]) {
+      user_identifiers.push({
+        user_identifier_source,
+        [identifier]: record[identifier],
+      });
+    }
+  });
+  if (user_identifiers.length > 0) {
+    conversion.user_identifiers = user_identifiers;
+  }
+  return conversion;
+}
+
 module.exports = {
   ClickConversionConfig,
   CustomerMatchRecord,
@@ -542,4 +648,5 @@ module.exports = {
   GoogleAds,
   ReportQueryConfig,
   GoogleAdsField,
+  buildClickConversionFromLine,
 };

package/src/apis/measurement_protocol_ga4.js

@@ -20,6 +20,7 @@
 'use strict';
 
 const {request} = require('gaxios');
+const lodash = require('lodash');
 const {
   getLogger,
   SendSingleBatch,
@@ -97,7 +98,7 @@ class MeasurementProtocolGA4 {
         this.logger.warn(
             "Only one line data expected. Will only send the first line.");
       }
-      const hit = Object.assign({}, config.requestBody, JSON.parse(line));
+      const hit = lodash.merge({}, config.requestBody, JSON.parse(line));
 
       const requestOptions = {
         method: 'POST',

package/src/apis/youtube.js

@@ -19,7 +19,13 @@
 'use strict';
 
 const {google} = require('googleapis');
-const {Schema$Channel, Schema$Video, Schema$CommentThread} = google.youtube;
+const {
+  Schema$Channel,
+  Schema$Video,
+  Schema$CommentThread,
+  Schema$Playlist,
+  Schema$Search,
+} = google.youtube;
 const AuthClient = require('./auth_client.js');
 const {getLogger} = require('../components/utils.js');
 
@@ -83,6 +89,62 @@ let ListVideosConfig;
  */
 let ListCommentThreadsConfig;
 
+/**
+ * Configuration for listing youtube play list.
+ * @see https://developers.google.com/youtube/v3/docs/Playlists/list
+ * @typedef {{
+ *   part: Array<string>,
+ *   channelId:	(string|undefined),
+ *   id: (string|undefined),
+ *   mine:	(boolean|undefined),
+ *   hl:	(string|undefined),
+ *   maxResults: (unsigned integer|undefined),
+ *   onBehalfOfContentOwner: (string|undefined),
+ *   onBehalfOfContentOwnerChannel: (string|undefined),
+ *   pageToken: (string|undefined)
+ * }}
+ */
+let ListPlaylistConfig;
+
+/**
+ * Configuration for listing youtube search results.
+ * @see https://developers.google.com/youtube/v3/docs/search/list
+ * @typedef {{
+ *   part: Array<string>,
+ *   forContentOwner:	(boolean|undefined),
+ *   forDeveloper:	(boolean|undefined),
+ *   forMine:	(boolean|undefined),
+ *   relatedToVideoId:	(string|undefined),
+ *   channelId:	(string|undefined),
+ *   channelType:	(string|undefined),
+ *   eventType:	(string|undefined),
+ *   location:	(string|undefined),
+ *   locationRadius:	(string|undefined),
+ *   maxResults: (unsigned integer|undefined),
+ *   onBehalfOfContentOwner:	(string|undefined),
+ *   order:	(string|undefined),
+ *   pageToken:	(string|undefined),
+ *   publishedAfter: (datetime|undefined),
+ *   publishedBefore: (datetime|undefined),
+ *   q:	(string|undefined),
+ *   regionCode:	(string|undefined),
+ *   relevanceLanguage:	(string|undefined),
+ *   safeSearch:	(string|undefined),
+ *   topicId:	(string|undefined),
+ *   type:	(string|undefined),
+ *   videoCaption:	(string|undefined),
+ *   videoCategoryId:	(string|undefined),
+ *   videoDefinition:	(string|undefined),
+ *   videoDimension:	(string|undefined),
+ *   videoDuration:	(string|undefined),
+ *   videoEmbeddable:	(string|undefined),
+ *   videoLicense:	(string|undefined),
+ *   videoSyndicated:	(string|undefined),
+ *   videoType:	(string|undefined)
+ * }}
+ */
+let ListSearchConfig;
+
 /**
  * Youtube API v3 stub.
  * See: https://developers.google.com/youtube/v3/docs
@@ -90,10 +152,21 @@ let ListCommentThreadsConfig;
  * https://developers.google.com/youtube/v3/docs/channels/list
  * Video list type definition, see:
  * https://developers.google.com/youtube/v3/docs/videos/list
+ * CommentThread list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/commentThreads/list
+ * Playlist list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/playlists/list
+ * Search list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/search/list
  */
 class YouTube {
-  constructor() {
-    const authClient = new AuthClient(API_SCOPES);
+  /**
+   * @constructor
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
+   */
+  constructor(env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
     this.auth = authClient.getDefaultAuth();
     /** @const {!google.youtube} */
     this.instance = google.youtube({
@@ -179,6 +252,96 @@ class YouTube {
       throw new Error(errorMsg);
     }
   }
+
+  /**
+   * Returns a collection of playlists that match the API request parameters.
+   * @see https://developers.google.com/youtube/v3/docs/playlists/list
+   * @param {!ListPlaylistConfig} config List playlist configuration.
+   * @param {number} resultLimit The limit of the number of results.
+   * @param {string} pageToken Token to identify a specific page in the result.
+   * @return {!Promise<Array<Schema$Playlist>>}
+   */
+  async listPlaylists(config, resultLimit = 1000, pageToken = null) {
+    if (resultLimit <= 0) return [];
+
+    const playlistsRequest = Object.assign({
+        auth: this.auth,
+    }, config, {
+        pageToken
+    });
+
+    if (Array.isArray(playlistsRequest.part)) {
+      playlistsRequest.part = playlistsRequest.part.join(',');
+    }
+
+    try {
+      const response = await this.instance.playlists.list(
+        playlistsRequest);
+      this.logger.debug('Response: ', response.data);
+      if (response.data.nextPageToken) {
+        this.logger.debug(
+          'Call youtube playlist:list API agian with Token: ',
+          response.data.nextPageToken);
+        const nextResponse = await this.listPlaylists(
+          config,
+          resultLimit - response.data.items.length,
+          response.data.nextPageToken);
+        return response.data.items.concat(nextResponse);
+      }
+      return response.data.items;
+    } catch (error) {
+      const errorMsg =
+        `Fail to list playlists: ${JSON.stringify(playlistsRequest)}`;
+      this.logger.error('YouTube list playlists failed.', error.message);
+      this.logger.debug('Errors in response:', error);
+      throw new Error(errorMsg);
+    }
+  }
+
+  /**
+   * Returns a collection of search results that match the query parameters
+   * specified in the API request.
+   * @see https://developers.google.com/youtube/v3/docs/search/list
+   * @param {!ListSearchConfig} config List search result configuration.
+   * @param {number} resultLimit The limit of the number of results.
+   * @param {string} pageToken Token to identify a specific page in the result.
+   * @return {!Promise<Array<Schema$Search>>}
+   */
+  async listSearchResults(config, resultLimit = 1000, pageToken = null) {
+    if (resultLimit <= 0) return [];
+
+    const searchRequest = Object.assign({
+        auth: this.auth,
+    }, config, {
+        pageToken
+    });
+
+    if (Array.isArray(searchRequest.part)) {
+      searchRequest.part = searchRequest.part.join(',');
+    }
+
+    try {
+      const response = await this.instance.search.list(searchRequest);
+      this.logger.debug('Response: ', response.data);
+      if (response.data.nextPageToken) {
+        this.logger.debug(
+          'Call youtube search:list API agian with Token: ',
+          response.data.nextPageToken);
+        const nextResponse = await this.listSearchResults(
+          config,
+          resultLimit - response.data.items.length,
+          response.data.nextPageToken);
+        return response.data.items.concat(nextResponse);
+      }
+      return response.data.items;
+    } catch (error) {
+      const errorMsg =
+        `Fail to list search results: ${JSON.stringify(searchRequest)}`;
+      this.logger.error('YouTube list search results failed.', error.message);
+      this.logger.debug('Errors in response:', error);
+      throw new Error(errorMsg);
+    }
+  }
 }
 
 module.exports = {
@@ -186,6 +349,8 @@ module.exports = {
   ListChannelsConfig,
   ListVideosConfig,
   ListCommentThreadsConfig,
+  ListPlaylistConfig,
+  ListSearchConfig,
   API_VERSION,
   API_SCOPES,
 };

package/src/components/utils.js

@@ -26,8 +26,11 @@ const {CloudPlatformApis} = require('../apis/cloud_platform_apis.js');
 /**
  * The result of a batch of data sent to target API. The batch here means the
  * data that will be sent out in one single request.
+ * Some APIs allows partial failure: it will take those correct data and
+ * response with reasons for those failed ones. 'groupedFailed' uses error
+ * message as the key, and tthe array of related failed lines(records) as value.
  * Some APIs upload whole file. In this case, there will be not 'numberOfLines'
- * or 'failedLines'.
+ * or 'failedLines', etc.
  * @typedef {{
  *   result: boolean,
  *   numberOfLines: (number|undefined),
@@ -177,6 +180,27 @@ const splitArray = (records, splitSize) => {
   return results;
 };
 
+/**
+ * Merges an object of 'groupedFailed into the object 'mergedResult'
+ * @param {!BatchResult} mergedResult
+ * @param {!BatchResult} groupedFailed
+ * @private
+ */
+const mergeGroupedFailed_ = (mergedResult, groupedFailed) => {
+  if (groupedFailed) {
+    const mergedKeys = Object.keys(mergedResult.groupedFailed);
+    mergedKeys.forEach((key) => {
+      mergedResult.groupedFailed[key] =
+          mergedResult.groupedFailed[key].concat(groupedFailed[key]);
+    });
+    Object.keys(groupedFailed)
+        .filter((key) => mergedKeys.indexOf(key) < 0)
+        .forEach((key) => {
+          mergedResult.groupedFailed[key] = groupedFailed[key];
+        });
+  }
+}
+
 /**
  * Merges an array of API results (BatchResult) in to a single one.
  *
@@ -198,8 +222,8 @@ const mergeBatchResults = (batchResults, batchPrefix) => {
     const {
       result,
       numberOfLines,
-      failedLines,
-      errors,
+      failedLines = [],
+      errors = [],
       groupedFailed,
     } = batchResult;
     if (logger.isDebugEnabled()) {
@@ -213,22 +237,13 @@ const mergeBatchResults = (batchResults, batchPrefix) => {
     }
     mergedResult.result = mergedResult.result && result;
     mergedResult.numberOfLines += numberOfLines;
-    mergedResult.failedLines = mergedResult.failedLines.concat(
-        failedLines || []);
-    mergedResult.errors = mergedResult.errors.concat(errors || []);
-    if (groupedFailed) {
-      const mergedKeys = Object.keys(mergedResult.groupedFailed);
-      mergedKeys.forEach((key) => {
-        mergedResult.groupedFailed[key] =
-            mergedResult.groupedFailed[key].concat(groupedFailed[key]);
-      });
-      Object.keys(groupedFailed)
-          .filter((key) => mergedKeys.indexOf(key) < 0)
-          .forEach((key) => {
-            mergedResult.groupedFailed[key] = groupedFailed[key];
-          })
-    }
-
+    mergedResult.failedLines = mergedResult.failedLines.concat(failedLines);
+    errors.forEach((error) => {
+      if (mergedResult.errors.indexOf(error) === -1) {
+        mergedResult.errors.push(error);
+      }
+    });
+    mergeGroupedFailed_(mergedResult, groupedFailed);
   });
   return mergedResult;
 };