@google-cloud/nodejs-common 0.9.4-beta3 → 0.9.9-alpha

package/src/apis/spreadsheets.js

@@ -67,11 +67,13 @@ class Spreadsheets {
   /**
    * Init Spreadsheets API client.
    * @param {string} spreadsheetId
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
    */
-  constructor(spreadsheetId) {
+  constructor(spreadsheetId, env = process.env) {
     /** @const {string} */
     this.spreadsheetId = spreadsheetId;
-    const authClient = new AuthClient(API_SCOPES);
+    const authClient = new AuthClient(API_SCOPES, env);
     const auth = authClient.getDefaultAuth();
     /** @const {!!google.sheets} */
     this.instance = google.sheets({
@@ -108,7 +110,6 @@ class Spreadsheets {
    * see:
    * https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/clear
    * @param {string} sheetName Name of the Sheet.
-   * @return {!Promise<boolean>} Whether the operation succeeded.
    */
   async clearSheet(sheetName) {
     const request = {
@@ -119,10 +120,9 @@ class Spreadsheets {
       const response = await this.instance.spreadsheets.values.clear(request);
       const data = response.data;
       this.logger.debug(`Clear sheet[${sheetName}}]: `, data);
-      return true;
     } catch (error) {
-      this.logger.error(error);
-      return false;
+      this.logger.error(error.toString());
+      throw error;
     }
   }
 
@@ -162,7 +162,6 @@ class Spreadsheets {
    * @param {string} sheetName Name of the Sheet.
    * @param {number} targetRows Loaded data rows number.
    * @param {number} targetColumns Loaded data columns number.
-   * @return {!Promise<boolean>} Whether the operation succeeded.
    */
   async reshape(sheetName, targetRows, targetColumns) {
     const request =  /** @type{Params$Resource$Spreadsheets$Get} */ {
@@ -194,13 +193,12 @@ class Spreadsheets {
       if (requests.resource.requests.length > 0) {
         const {data} = await this.instance.spreadsheets.batchUpdate(requests);
         this.logger.debug(`Reshape Sheet [${sheetName}]: `, data);
-        return true;
+      } else {
+        this.logger.debug('No need to reshape.');
       }
-      this.logger.debug('No need to reshape.');
-      return true;
     } catch (error) {
-      console.error(error);
-      return false;
+      this.logger.error(error.toString());
+      throw error;
     }
   }
 
@@ -218,7 +216,9 @@ class Spreadsheets {
       spreadsheetId: this.spreadsheetId,
       resource: {requests: [{pasteData}]},
     };
-    /** @type {BatchResult} */ const batchResult = {};
+    /** @type {BatchResult} */ const batchResult = {
+      numberOfLines: data.trim().split('\n').length,
+    };
     try {
       const response = await this.instance.spreadsheets.batchUpdate(request);
       const data = response.data;

package/src/apis/youtube.js

@@ -0,0 +1,356 @@
+// Copyright 2021 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @fileoverview Youtube API Client Library.
+ */
+
+'use strict';
+
+const {google} = require('googleapis');
+const {
+  Schema$Channel,
+  Schema$Video,
+  Schema$CommentThread,
+  Schema$Playlist,
+  Schema$Search,
+} = google.youtube;
+const AuthClient = require('./auth_client.js');
+const {getLogger} = require('../components/utils.js');
+
+const API_SCOPES = Object.freeze([
+  'https://www.googleapis.com/auth/youtube.force-ssl'
+]);
+const API_VERSION = 'v3';
+
+/**
+ * Configuration for listing youtube channels.
+ * @see https://developers.google.com/youtube/v3/docs/channels/list
+ * @typedef {{
+ *   part: Array<string>,
+ *   categoryId: (string|undefined),
+ *   forUsername: (string|undefined),
+ *   id: (string|undefined),
+ *   managedByMe: (boolean|undefined),
+ *   mine: (boolean|undefined),
+ *   hl: (string|undefined),
+ *   onBehalfOfContentOwner: (string|undefined),
+ *   pageToken: (string|undefined),
+ * }}
+ */
+let ListChannelsConfig;
+
+/**
+ * Configuration for listing youtube videos.
+ * @see https://developers.google.com/youtube/v3/docs/videos/list
+ * @typedef {{
+ *   part: Array<string>,
+ *   id: (string|undefined),
+ *   chart: (string|undefined),
+ *   hl: (string|undefined),
+ *   maxHeight: (unsigned integer|undefined),
+ *   maxResults: (unsigned integer|undefined),
+ *   maxWidth: (unsigned integer|undefined),
+ *   onBehalfOfContentOwner: (string|undefined),
+ *   pageToken: (string|undefined),
+ *   regionCode: (string|undefined),
+ * }}
+ */
+let ListVideosConfig;
+
+/**
+ * Configuration for listing youtube comment threads by the given id.
+ * @see https://developers.google.com/youtube/v3/docs/commentThreads/list
+ * @typedef {{
+ *   part: Array<string>,
+ *   id: (string|undefined),
+ *   allThreadsRelatedToChannelId:	(string|undefined),
+ *   channelId:	(string|undefined),
+ *   videoId:	(string|undefined),
+ *   maxResults: (unsigned integer|undefined),
+ *   moderationStatus: (string|undefined),
+ *   order: (string|undefined),
+ *   pageToken: (string|undefined),
+ *   searchTerms: (string|undefined),
+ *   textFormat: (string|undefined),
+ *   limit: (unsigned integer|undefined),
+ * }}
+ */
+let ListCommentThreadsConfig;
+
+/**
+ * Configuration for listing youtube play list.
+ * @see https://developers.google.com/youtube/v3/docs/Playlists/list
+ * @typedef {{
+ *   part: Array<string>,
+ *   channelId:	(string|undefined),
+ *   id: (string|undefined),
+ *   mine:	(boolean|undefined),
+ *   hl:	(string|undefined),
+ *   maxResults: (unsigned integer|undefined),
+ *   onBehalfOfContentOwner: (string|undefined),
+ *   onBehalfOfContentOwnerChannel: (string|undefined),
+ *   pageToken: (string|undefined)
+ * }}
+ */
+let ListPlaylistConfig;
+
+/**
+ * Configuration for listing youtube search results.
+ * @see https://developers.google.com/youtube/v3/docs/search/list
+ * @typedef {{
+ *   part: Array<string>,
+ *   forContentOwner:	(boolean|undefined),
+ *   forDeveloper:	(boolean|undefined),
+ *   forMine:	(boolean|undefined),
+ *   relatedToVideoId:	(string|undefined),
+ *   channelId:	(string|undefined),
+ *   channelType:	(string|undefined),
+ *   eventType:	(string|undefined),
+ *   location:	(string|undefined),
+ *   locationRadius:	(string|undefined),
+ *   maxResults: (unsigned integer|undefined),
+ *   onBehalfOfContentOwner:	(string|undefined),
+ *   order:	(string|undefined),
+ *   pageToken:	(string|undefined),
+ *   publishedAfter: (datetime|undefined),
+ *   publishedBefore: (datetime|undefined),
+ *   q:	(string|undefined),
+ *   regionCode:	(string|undefined),
+ *   relevanceLanguage:	(string|undefined),
+ *   safeSearch:	(string|undefined),
+ *   topicId:	(string|undefined),
+ *   type:	(string|undefined),
+ *   videoCaption:	(string|undefined),
+ *   videoCategoryId:	(string|undefined),
+ *   videoDefinition:	(string|undefined),
+ *   videoDimension:	(string|undefined),
+ *   videoDuration:	(string|undefined),
+ *   videoEmbeddable:	(string|undefined),
+ *   videoLicense:	(string|undefined),
+ *   videoSyndicated:	(string|undefined),
+ *   videoType:	(string|undefined)
+ * }}
+ */
+let ListSearchConfig;
+
+/**
+ * Youtube API v3 stub.
+ * See: https://developers.google.com/youtube/v3/docs
+ * Channel list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/channels/list
+ * Video list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/videos/list
+ * CommentThread list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/commentThreads/list
+ * Playlist list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/playlists/list
+ * Search list type definition, see:
+ * https://developers.google.com/youtube/v3/docs/search/list
+ */
+class YouTube {
+  /**
+   * @constructor
+   * @param {!Object<string,string>=} env The environment object to hold env
+   *     variables.
+   */
+  constructor(env = process.env) {
+    const authClient = new AuthClient(API_SCOPES, env);
+    this.auth = authClient.getDefaultAuth();
+    /** @const {!google.youtube} */
+    this.instance = google.youtube({
+      version: API_VERSION,
+    });
+    /**
+     * Logger object from 'log4js' package where this type is not exported.
+     */
+    this.logger = getLogger('API.YT');
+  }
+
+  /**
+   * Returns a collection of zero or more channel resources that match the
+   * request criteria.
+   * @see https://developers.google.com/youtube/v3/docs/channels/list
+   * @param {!ListChannelsConfig} config List channels configuration.
+   * @return {!Promise<Array<Schema$Channel>>}
+   */
+  async listChannels(config) {
+    const channelListRequest = Object.assign({
+      auth: this.auth,
+    }, config);
+    channelListRequest.part = channelListRequest.part.join(',')
+    try {
+      const response = await this.instance.channels.list(channelListRequest);
+      this.logger.debug('Response: ', response);
+      return response.data.items;
+    } catch (error) {
+      const errorMsg =
+          `Fail to list channels: ${JSON.stringify(channelListRequest)}`;
+      this.logger.error('YouTube list channels failed.', error.message);
+      this.logger.debug('Errors in response:', error);
+      throw new Error(errorMsg);
+    }
+  }
+
+  /**
+   * Returns a list of videos that match the API request parameters.
+   * @see https://developers.google.com/youtube/v3/docs/videos/list
+   * @param {!ListVideosConfig} config List videos configuration.
+   * @return {!Promise<Array<Schema$Video>>}
+   */
+  async listVideos(config) {
+    const videoListRequest = Object.assign({
+      auth: this.auth,
+    }, config);
+    videoListRequest.part = videoListRequest.part.join(',')
+    try {
+      const response = await this.instance.videos.list(videoListRequest);
+      this.logger.debug('Response: ', response);
+      return response.data.items;
+    } catch (error) {
+      const errorMsg =
+          `Fail to list videos: ${JSON.stringify(videoListRequest)}`;
+      this.logger.error('YouTube list videos failed.', error.message);
+      this.logger.debug('Errors in response:', error);
+      throw new Error(errorMsg);
+    }
+  }
+
+  /**
+   * Returns a list of comment threads that match the API request parameters.
+   * @see https://developers.google.com/youtube/v3/docs/videos/list
+   * @param {!ListCommentThreadsConfig} config List comment threads
+   *   configuration.
+   * @return {!Promise<Array<Schema$CommentThread>>}
+   */
+  async listCommentThreads(config) {
+    const commentThreadsRequest = Object.assign({
+      auth: this.auth,
+    }, config);
+    commentThreadsRequest.part = commentThreadsRequest.part.join(',')
+    try {
+      const response = await this.instance.commentThreads.list(
+        commentThreadsRequest);
+      this.logger.debug('Response: ', response.data);
+      return response.data.items;
+    } catch (error) {
+      const errorMsg =
+        `Fail to list comment threads: ${JSON.stringify(commentThreadsRequest)}`;
+      this.logger.error('YouTube list comment threads failed.', error.message);
+      this.logger.debug('Errors in response:', error);
+      throw new Error(errorMsg);
+    }
+  }
+
+  /**
+   * Returns a collection of playlists that match the API request parameters.
+   * @see https://developers.google.com/youtube/v3/docs/playlists/list
+   * @param {!ListPlaylistConfig} config List playlist configuration.
+   * @param {number} resultLimit The limit of the number of results.
+   * @param {string} pageToken Token to identify a specific page in the result.
+   * @return {!Promise<Array<Schema$Playlist>>}
+   */
+  async listPlaylists(config, resultLimit = 1000, pageToken = null) {
+    if (resultLimit <= 0) return [];
+
+    const playlistsRequest = Object.assign({
+        auth: this.auth,
+    }, config, {
+        pageToken
+    });
+
+    if (Array.isArray(playlistsRequest.part)) {
+      playlistsRequest.part = playlistsRequest.part.join(',');
+    }
+
+    try {
+      const response = await this.instance.playlists.list(
+        playlistsRequest);
+      this.logger.debug('Response: ', response.data);
+      if (response.data.nextPageToken) {
+        this.logger.debug(
+          'Call youtube playlist:list API agian with Token: ',
+          response.data.nextPageToken);
+        const nextResponse = await this.listPlaylists(
+          config,
+          resultLimit - response.data.items.length,
+          response.data.nextPageToken);
+        return response.data.items.concat(nextResponse);
+      }
+      return response.data.items;
+    } catch (error) {
+      const errorMsg =
+        `Fail to list playlists: ${JSON.stringify(playlistsRequest)}`;
+      this.logger.error('YouTube list playlists failed.', error.message);
+      this.logger.debug('Errors in response:', error);
+      throw new Error(errorMsg);
+    }
+  }
+
+  /**
+   * Returns a collection of search results that match the query parameters
+   * specified in the API request.
+   * @see https://developers.google.com/youtube/v3/docs/search/list
+   * @param {!ListSearchConfig} config List search result configuration.
+   * @param {number} resultLimit The limit of the number of results.
+   * @param {string} pageToken Token to identify a specific page in the result.
+   * @return {!Promise<Array<Schema$Search>>}
+   */
+  async listSearchResults(config, resultLimit = 1000, pageToken = null) {
+    if (resultLimit <= 0) return [];
+
+    const searchRequest = Object.assign({
+        auth: this.auth,
+    }, config, {
+        pageToken
+    });
+
+    if (Array.isArray(searchRequest.part)) {
+      searchRequest.part = searchRequest.part.join(',');
+    }
+
+    try {
+      const response = await this.instance.search.list(searchRequest);
+      this.logger.debug('Response: ', response.data);
+      if (response.data.nextPageToken) {
+        this.logger.debug(
+          'Call youtube search:list API agian with Token: ',
+          response.data.nextPageToken);
+        const nextResponse = await this.listSearchResults(
+          config,
+          resultLimit - response.data.items.length,
+          response.data.nextPageToken);
+        return response.data.items.concat(nextResponse);
+      }
+      return response.data.items;
+    } catch (error) {
+      const errorMsg =
+        `Fail to list search results: ${JSON.stringify(searchRequest)}`;
+      this.logger.error('YouTube list search results failed.', error.message);
+      this.logger.debug('Errors in response:', error);
+      throw new Error(errorMsg);
+    }
+  }
+}
+
+module.exports = {
+  YouTube,
+  ListChannelsConfig,
+  ListVideosConfig,
+  ListCommentThreadsConfig,
+  ListPlaylistConfig,
+  ListSearchConfig,
+  API_VERSION,
+  API_SCOPES,
+};

package/src/components/utils.js

@@ -26,14 +26,17 @@ const {CloudPlatformApis} = require('../apis/cloud_platform_apis.js');
 /**
  * The result of a batch of data sent to target API. The batch here means the
  * data that will be sent out in one single request.
+ * Some APIs allows partial failure: it will take those correct data and
+ * response with reasons for those failed ones. 'groupedFailed' uses error
+ * message as the key, and tthe array of related failed lines(records) as value.
  * Some APIs upload whole file. In this case, there will be not 'numberOfLines'
- * or 'failedLines'.
+ * or 'failedLines', etc.
  * @typedef {{
  *   result: boolean,
  *   numberOfLines: (number|undefined),
  *   failedLines: (Array<string>|undefined),
  *   errors: (Array<string>|undefined),
- *   output: (Array<string>|undefined),
+ *   groupedFailed: (Object<string,Array<string>>|undefined),
  * }}
  */
 let BatchResult;
@@ -177,6 +180,27 @@ const splitArray = (records, splitSize) => {
   return results;
 };
 
+/**
+ * Merges an object of 'groupedFailed into the object 'mergedResult'
+ * @param {!BatchResult} mergedResult
+ * @param {!BatchResult} groupedFailed
+ * @private
+ */
+const mergeGroupedFailed_ = (mergedResult, groupedFailed) => {
+  if (groupedFailed) {
+    const mergedKeys = Object.keys(mergedResult.groupedFailed);
+    mergedKeys.forEach((key) => {
+      mergedResult.groupedFailed[key] =
+          mergedResult.groupedFailed[key].concat(groupedFailed[key]);
+    });
+    Object.keys(groupedFailed)
+        .filter((key) => mergedKeys.indexOf(key) < 0)
+        .forEach((key) => {
+          mergedResult.groupedFailed[key] = groupedFailed[key];
+        });
+  }
+}
+
 /**
  * Merges an array of API results (BatchResult) in to a single one.
  *
@@ -191,11 +215,17 @@ const mergeBatchResults = (batchResults, batchPrefix) => {
     numberOfLines: 0,
     failedLines: [],
     errors: [],
-    output: []
+    groupedFailed: {},
   };
   batchResults.forEach((batchResult, index) => {
     const batchId = `${batchPrefix}-${index}`;
-    const {result, numberOfLines, failedLines, errors, output} = batchResult;
+    const {
+      result,
+      numberOfLines,
+      failedLines = [],
+      errors = [],
+      groupedFailed,
+    } = batchResult;
     if (logger.isDebugEnabled()) {
       logger.debug(
           `  Task [${batchId}] has ${numberOfLines} lines: ${result
@@ -207,10 +237,13 @@ const mergeBatchResults = (batchResults, batchPrefix) => {
     }
     mergedResult.result = mergedResult.result && result;
     mergedResult.numberOfLines += numberOfLines;
-    mergedResult.failedLines = mergedResult.failedLines.concat(
-        failedLines || []);
-    mergedResult.errors = mergedResult.errors.concat(errors || []);
-    mergedResult.output = mergedResult.output.concat(output || []);
+    mergedResult.failedLines = mergedResult.failedLines.concat(failedLines);
+    errors.forEach((error) => {
+      if (mergedResult.errors.indexOf(error) === -1) {
+        mergedResult.errors.push(error);
+      }
+    });
+    mergeGroupedFailed_(mergedResult, groupedFailed);
   });
   return mergedResult;
 };
@@ -289,12 +322,13 @@ const apiSpeedControl = (recordSize = 1, numberOfThreads = 1, qps = 1,
     const records = Array.isArray(data) ?
         data :
         data.split('\n').filter((line) => line.trim() !== '');
-    if (maxRecords < data.length) {
+    if (maxRecords < records.length) {
       const error = `Predicted timeout: ${records.length} records with config: `
           + `${recordSize} records/request and ${qps} QPS in ${timeout} sec.`;
       logger.error(error);
       return {
         result: false,
+        numberOfLines: records.length,
         errors: [error],
       };
     }
@@ -523,6 +557,7 @@ module.exports = {
   wait,
   BatchResult,
   SendSingleBatch,
+  mergeBatchResults,
   sendSingleRound,
   apiSpeedControl,
   splitArray,