@google-cloud/nodejs-common 0.9.3-beta → 0.9.4-beta3

package/bin/install_functions.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # Cloud Functions Runtime Environment.
-CF_RUNTIME="${CF_RUNTIME:=nodejs10}"
+CF_RUNTIME="${CF_RUNTIME:=nodejs14}"
 
 # Counter for steps.
 STEP=0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "0.9.3-beta",
+  "version": "0.9.4-beta3",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",

package/src/apis/analytics.js

@@ -23,7 +23,7 @@ const stream = require('stream');
 const {google} = require('googleapis');
 const {Schema$Upload} = google.analytics;
 const AuthClient = require('./auth_client.js');
-const {wait, getLogger} = require('../components/utils.js');
+const {wait, getLogger, BatchResult} = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
   'https://www.googleapis.com/auth/analytics',
@@ -81,8 +81,7 @@ class Analytics {
    * @param {string|!stream.Readable} data A string or a stream to be uploaded.
    * @param {!DataImportConfig} config GA data import configuration.
    * @param {string=} batchId A tag for log.
-   * @return {!Promise<boolean>} Promise returning whether data import
-   *     succeeded.
+   * @return {!BatchResult}
    */
   async uploadData(data, config, batchId = 'unnamed') {
     const uploadConfig = Object.assign(
@@ -100,29 +99,37 @@ class Analytics {
     this.logger.debug('Response: ', response);
     const job = /** @type {Schema$Upload} */ response.data;
     const uploadId = (/** @type {Schema$Upload} */job).id;
-    console.log(`Task [${batchId}] creates GA Data import job: ${uploadId}`);
+    this.logger.info(
+        `Task [${batchId}] creates GA Data import job: ${uploadId}`);
     const jobConfig = Object.assign({uploadId}, config);
     const result = await Promise.race([
       this.checkJobStatus(jobConfig),
       wait(8 * 60 * 1000, job),  // wait up to 8 minutes here
     ]);
+    /** @type {BatchResult} */ const batchResult = {};
     switch ((/** @type {Schema$Upload} */ result).status) {
       case 'FAILED':
         this.logger.error('GA Data Import failed', result);
-        const errors = result.errors || [`Unknown reason. ID: ${uploadId}`];
-        throw new Error(errors.join('\n'));
+        batchResult.result = false;
+        batchResult.errors = result.errors
+            || [`Unknown reason. ID: ${uploadId}`];
+        break;
       case 'COMPLETED':
         this.logger.info(`GA Data Import job[${uploadId}] completed.`);
         this.logger.debug('Response: ', result);
-        return true;
+        batchResult.result = true;
+        break;
       case 'PENDING':
         this.logger.info('GA Data Import pending.', result);
         this.logger.info('Still will return true here.');
-        return true;
+        batchResult.result = true;
+        break;
       default:
         this.logger.error('Unknown results of GA Data Import: ', result);
-        throw new Error(`Unknown status. ID: ${uploadId}`);
+        batchResult.result = false;
+        batchResult.errors = [`Unknown status. ID: ${uploadId}`];
     }
+    return batchResult;
   }
 
   /**

package/src/apis/index.js

@@ -110,13 +110,3 @@ exports.adsdatahub = require('./ads_data_hub.js');
  * }}
  */
 exports.measurementprotocolga4 = require('./measurement_protocol_ga4.js');
-
-/**
- * APIs integration class for YouTube.
- * @const {{
- *   YouTube:!YouTube,
- *   ListChannelsConfig: !ListChannelsConfig,
- *   ListVideosConfig: !ListVideosConfig,
- * }}
- */
-exports.youtube = require('./youtube.js');

package/src/apis/measurement_protocol.js

@@ -20,7 +20,11 @@
 'use strict';
 
 const {request} = require('gaxios');
-const {getLogger, SendSingleBatch} = require('../components/utils.js');
+const {
+  getLogger,
+  SendSingleBatch,
+  BatchResult,
+} = require('../components/utils.js');
 /** Base URL for Google Analytics service. */
 const BASE_URL = 'https://www.google-analytics.com';
 
@@ -62,9 +66,9 @@ class MeasurementProtocol {
      * @param {!Array<string>} lines Data for single request. It should be
      *     guaranteed that it doesn't exceed quota limitation.
      * @param {string} batchId The tag for log.
-     * @return {!Promise<boolean>}
+     * @return {!Promise<BatchResult>}
      */
-    return (lines, batchId) => {
+    return async (lines, batchId) => {
       const payload =
           lines
               .map((line) => {
@@ -85,30 +89,47 @@ class MeasurementProtocol {
         body: payload,
         headers: {'User-Agent': 'Tentacles/MeasurementProtocol-v1'}
       };
-      return request(requestOptions).then((response) => {
-        if (response.status < 200 || response.status >= 300) {
-          const errorMessages = [
-            `Measurement Protocol [${batchId}] didn't succeed.`,
-            `Get response code: ${response.status}`,
-            `response: ${response.data}`,
-          ];
-          console.error(errorMessages.join('\n'));
-          throw new Error(`Status code not 2XX`);
-        }
-        this.logger.debug(`Configuration:`, config);
-        this.logger.debug(`Input Data:   `, lines);
-        this.logger.debug(`Batch[${batchId}] status: ${response.status}`);
-        this.logger.debug(response.data);
-        // There is not enough information from the non-debug mode.
-        if (!this.debugMode) return true;
-        return response.data.hitParsingResult.every((result) => result.valid);
-      });
+      const response = await request(requestOptions);
+      /** @type {BatchResult} */ const batchResult = {
+        numberOfLines: lines.length,
+      };
+      if (response.status < 200 || response.status >= 300) {
+        const errorMessages = [
+          `Measurement Protocol [${batchId}] didn't succeed.`,
+          `Get response code: ${response.status}`,
+          `response: ${response.data}`,
+        ];
+        this.logger.error(errorMessages.join('\n'));
+        batchResult.errors = errorMessages;
+        batchResult.result = false;
+        return batchResult;
+      }
+      this.logger.debug(`Configuration:`, config);
+      this.logger.debug(`Input Data:   `, lines);
+      this.logger.debug(`Batch[${batchId}] status: ${response.status}`);
+      this.logger.debug(response.data);
+      // There is not enough information from the non-debug mode.
+      if (!this.debugMode) {
+        batchResult.result = true;
+      } else {
+        const failedHits = [];
+        const failedReasons = new Set();
+        response.data.hitParsingResult.forEach((result, index) => {
+          if (!result.valid) {
+            failedHits.push(lines[index]);
+            result.parserMessage.forEach(({description}) => {
+              failedReasons.add(description);
+            });
+          }
+        });
+        batchResult.result = failedHits.length === 0;
+        batchResult.failedLines = failedHits;
+        batchResult.errors = Array.from(failedReasons);
+      }
+      return batchResult;
     };
   };
 
-  static getInstance() {
-    return new MeasurementProtocol(process.env['DEBUG'] === 'true');
-  }
 }
 
 module.exports = {MeasurementProtocol};

package/src/apis/measurement_protocol_ga4.js

@@ -20,7 +20,11 @@
 'use strict';
 
 const {request} = require('gaxios');
-const {getLogger, SendSingleBatch} = require('../components/utils.js');
+const {
+  getLogger,
+  SendSingleBatch,
+  BatchResult,
+} = require('../components/utils.js');
 /** Base URL for Google Analytics service. */
 const BASE_URL = 'https://www.google-analytics.com';
 
@@ -85,7 +89,7 @@ class MeasurementProtocolGA4 {
      * @param {!Array<string>} lines Data for single request. It should be
      *     guaranteed that it doesn't exceed quota limitation.
      * @param {string} batchId The tag for log.
-     * @return {!Promise<boolean>}
+     * @return {!BatchResult}
      */
     return async (lines, batchId) => {
       const line = lines[0]; // Each request contains one record only.
@@ -104,22 +108,36 @@ class MeasurementProtocolGA4 {
         headers: {'User-Agent': 'Tentacles/MeasurementProtocol-GA4'}
       };
       const response = await request(requestOptions);
+      /** @type {BatchResult} */ const batchResult = {
+        numberOfLines: lines.length,
+      };
       if (response.status < 200 || response.status >= 300) {
         const errorMessages = [
           `Measurement Protocol GA4 [${batchId}] didn't succeed.`,
           `Get response code: ${response.status}`,
           `response: ${response.data}`,
         ];
-        console.error(errorMessages.join('\n'));
-        throw new Error(`Status code not 2XX`);
+        this.logger.error(errorMessages.join('\n'));
+        batchResult.errors = errorMessages;
+        batchResult.result = false;
+        return batchResult;
       }
       this.logger.debug('Configuration:', config);
       this.logger.debug('Input Data:   ', lines);
       this.logger.debug(`Batch[${batchId}] status: ${response.status}`);
-      this.logger.debug('Response:     ', response.data);
+      this.logger.debug(response.data);
       // There is not enough information from the non-debug mode.
-      if (!this.debugMode) return true;
-      return response.data.validationMessages.length === 0;
+      if (!this.debugMode) {
+        batchResult.result = true;
+      } else {
+        batchResult.result = response.data.validationMessages.length === 0;
+        if (!batchResult.result) {
+          batchResult.failedLines = lines;
+          batchResult.errors = response.data.validationMessages.map(
+              ({description}) => description);
+        }
+      }
+      return batchResult;
     };
   };
 

package/src/apis/spreadsheets.js

@@ -21,7 +21,7 @@
 const {google} = require('googleapis');
 const {Params$Resource$Spreadsheets$Get} = google.sheets;
 const AuthClient = require('./auth_client.js');
-const {getLogger} = require('../components/utils.js');
+const {getLogger, BatchResult} = require('../components/utils.js');
 
 const API_SCOPES = Object.freeze([
   'https://www.googleapis.com/auth/spreadsheets',
@@ -92,16 +92,15 @@ class Spreadsheets {
    * @param {string} sheetName Name of the Sheet.
    * @return {!Promise<number>} Sheet Id.
    */
-  getSheetId(sheetName) {
+  async getSheetId(sheetName) {
     const request = /** @type{Params$Resource$Spreadsheets$Get} */ {
       spreadsheetId: this.spreadsheetId,
       ranges: sheetName,
     };
-    return this.instance.spreadsheets.get(request).then((response) => {
-      const sheet = response.data.sheets[0];
-      this.logger.debug(`Get sheet[${sheetName}]: `, sheet);
-      return sheet.properties.sheetId;
-    });
+    const response = await this.instance.spreadsheets.get(request);
+    const sheet = response.data.sheets[0];
+    this.logger.debug(`Get sheet[${sheetName}]: `, sheet);
+    return sheet.properties.sheetId;
   }
 
   /**
@@ -111,21 +110,20 @@ class Spreadsheets {
    * @param {string} sheetName Name of the Sheet.
    * @return {!Promise<boolean>} Whether the operation succeeded.
    */
-  clearSheet(sheetName) {
+  async clearSheet(sheetName) {
     const request = {
       spreadsheetId: this.spreadsheetId,
       range: sheetName,
     };
-    return this.instance.spreadsheets.values.clear(request)
-        .then((response) => {
-          const data = response.data;
-          this.logger.debug(`Clear sheet[${sheetName}}]: `, data);
-          return true;
-        })
-        .catch((error) => {
-          console.error(error);
-          return false;
-        });
+    try {
+      const response = await this.instance.spreadsheets.values.clear(request);
+      const data = response.data;
+      this.logger.debug(`Clear sheet[${sheetName}}]: `, data);
+      return true;
+    } catch (error) {
+      this.logger.error(error);
+      return false;
+    }
   }
 
   /**
@@ -147,26 +145,13 @@ class Spreadsheets {
    * @private
    */
   getChangeDimensionRequest_(sheetId, dimension, current, target) {
-    if (current === target) {
-      return;
-    }
+    if (current === target) return;
     if (current < target) {  // Appends dimension.
-      return {
-        appendDimension: {
-          sheetId: sheetId,
-          dimension: dimension,
-          length: target - current,
-        }
-      };
+      return {appendDimension: {sheetId, dimension, length: target - current}};
     } else {  // Deletes dimension.
       return {
         deleteDimension: {
-          range: {
-            sheetId: sheetId,
-            dimension: dimension,
-            startIndex: target,
-            endIndex: current,
-          }
+          range: {sheetId, dimension, startIndex: target, endIndex: current},
         }
       };
     }
@@ -179,49 +164,44 @@ class Spreadsheets {
    * @param {number} targetColumns Loaded data columns number.
    * @return {!Promise<boolean>} Whether the operation succeeded.
    */
-  reshape(sheetName, targetRows, targetColumns) {
+  async reshape(sheetName, targetRows, targetColumns) {
     const request =  /** @type{Params$Resource$Spreadsheets$Get} */ {
       spreadsheetId: this.spreadsheetId,
       ranges: sheetName,
     };
-    return this.instance.spreadsheets.get(request)
-        .then((response) => {
-          const sheet = response.data.sheets[0];
-          const sheetId = sheet.properties.sheetId;
-          const rowCount = sheet.properties.gridProperties.rowCount;
-          const columnCount = sheet.properties.gridProperties.columnCount;
-          this.logger.debug(`Get sheet[${sheetName}]: `, sheet);
-          const requests = {
-            spreadsheetId: this.spreadsheetId,
-            resource: {requests: []},
-          };
-          if (rowCount !== targetRows) {
-            requests.resource.requests.push(this.getChangeDimensionRequest_(
-                sheetId, 'ROWS', rowCount, targetRows));
-          }
-          if (columnCount !== targetColumns) {
-            requests.resource.requests.push(this.getChangeDimensionRequest_(
-                sheetId, 'COLUMNS', columnCount, targetColumns));
-          }
-          this.logger.debug(
-              `Reshape Sheet from [${rowCount}, ${columnCount}] to [${
-                  targetRows}, ${targetColumns}]`,
-              JSON.stringify(requests.resource.requests));
-          if (requests.resource.requests.length > 0) {
-            return this.instance.spreadsheets.batchUpdate(requests).then(
-                (response) => {
-                  const data = response.data;
-                  console.log(`Reshape Sheet [${sheetName}]: `, data);
-                  return true;
-                });
-          }
-          this.logger.debug('No need to reshape.');
-          return true;
-        })
-        .catch((error) => {
-          console.error(error);
-          return false;
-        });
+    try {
+      const response = await this.instance.spreadsheets.get(request);
+      const sheet = response.data.sheets[0];
+      const sheetId = sheet.properties.sheetId;
+      const rowCount = sheet.properties.gridProperties.rowCount;
+      const columnCount = sheet.properties.gridProperties.columnCount;
+      this.logger.debug(`Get sheet[${sheetName}]: `, sheet);
+      const requests = {
+        spreadsheetId: this.spreadsheetId,
+        resource: {requests: []},
+      };
+      if (rowCount !== targetRows) {
+        requests.resource.requests.push(this.getChangeDimensionRequest_(
+            sheetId, 'ROWS', rowCount, targetRows));
+      }
+      if (columnCount !== targetColumns) {
+        requests.resource.requests.push(this.getChangeDimensionRequest_(
+            sheetId, 'COLUMNS', columnCount, targetColumns));
+      }
+      this.logger.debug(`Reshape Sheet from [${rowCount}, ${
+              columnCount}] to [${targetRows}, ${targetColumns}]`,
+          JSON.stringify(requests.resource.requests));
+      if (requests.resource.requests.length > 0) {
+        const {data} = await this.instance.spreadsheets.batchUpdate(requests);
+        this.logger.debug(`Reshape Sheet [${sheetName}]: `, data);
+        return true;
+      }
+      this.logger.debug('No need to reshape.');
+      return true;
+    } catch (error) {
+      console.error(error);
+      return false;
+    }
   }
 
   /**
@@ -230,25 +210,26 @@ class Spreadsheets {
    * @param {!ParseDataRequest} config A ParseDataRequest object template. The
    * data will be put in before it is send out through Sheets batchUpdate.
    * @param {string=} batchId The tag for log.
-   * @return {!Promise<boolean>} Promise returning whether the operation
-   *     succeeded.
+   * @return {!BatchResult}
    */
-  loadData(data, config, batchId = 'unnamed') {
-    const pasteData = Object.assign({}, config, {data: data});
+  async loadData(data, config, batchId = 'unnamed') {
+    const pasteData = Object.assign({}, config, {data});
     const request = {
       spreadsheetId: this.spreadsheetId,
-      resource: {requests: [{pasteData: pasteData}]},
+      resource: {requests: [{pasteData}]},
     };
-    return this.instance.spreadsheets.batchUpdate(request)
-        .then((response) => {
-          const data = response.data;
-          console.log(`Batch[${batchId}] uploaded: `, data);
-          return true;
-        })
-        .catch((error) => {
-          console.error(error.errors);
-          return false;
-        });
+    /** @type {BatchResult} */ const batchResult = {};
+    try {
+      const response = await this.instance.spreadsheets.batchUpdate(request);
+      const data = response.data;
+      this.logger.debug(`Batch[${batchId}] uploaded: `, data);
+      batchResult.result = true;
+    } catch (error) {
+      this.logger.error(error);
+      batchResult.result = false;
+      batchResult.errors = [error.toString()];
+    }
+    return batchResult;
   }
 }
 

package/src/components/utils.js

@@ -23,12 +23,27 @@ const {inspect} = require('util');
 const {LoggingWinston} = require('@google-cloud/logging-winston');
 const {CloudPlatformApis} = require('../apis/cloud_platform_apis.js');
 
+/**
+ * The result of a batch of data sent to target API. The batch here means the
+ * data that will be sent out in one single request.
+ * Some APIs upload whole file. In this case, there will be not 'numberOfLines'
+ * or 'failedLines'.
+ * @typedef {{
+ *   result: boolean,
+ *   numberOfLines: (number|undefined),
+ *   failedLines: (Array<string>|undefined),
+ *   errors: (Array<string>|undefined),
+ *   output: (Array<string>|undefined),
+ * }}
+ */
+let BatchResult;
+
 /**
  * Function which sends a batch of data. Takes two parameters:
  * {!Array<string>} Data for single request. It should be guaranteed that it
  *     doesn't exceed quota limitation.
  * {string} The tag for log.
- * @typedef {function(!Array<string>,string): !Promise<boolean>}
+ * @typedef {function(!Array<string>,string): !BatchResult}
  */
 let SendSingleBatch;
 
@@ -162,6 +177,44 @@ const splitArray = (records, splitSize) => {
   return results;
 };
 
+/**
+ * Merges an array of API results (BatchResult) in to a single one.
+ *
+ * @param {!Array<!BatchResult>} batchResults
+ * @param {string} batchPrefix Prefix tag for this batch, used for logging.
+ * @return {!BatchResult}
+ */
+const mergeBatchResults = (batchResults, batchPrefix) => {
+  const logger = getLogger('SPEED_CTL');
+  /** @const {!BatchResult} */ const mergedResult = {
+    result: true,
+    numberOfLines: 0,
+    failedLines: [],
+    errors: [],
+    output: []
+  };
+  batchResults.forEach((batchResult, index) => {
+    const batchId = `${batchPrefix}-${index}`;
+    const {result, numberOfLines, failedLines, errors, output} = batchResult;
+    if (logger.isDebugEnabled()) {
+      logger.debug(
+          `  Task [${batchId}] has ${numberOfLines} lines: ${result
+              ? 'succeeded' : 'failed'}.`);
+      if (!result) {
+        logger.debug(`  Errors: ${errors.join('\n')}`);
+        logger.debug(`  Failed lines: ${failedLines.join('\n')}`);
+      }
+    }
+    mergedResult.result = mergedResult.result && result;
+    mergedResult.numberOfLines += numberOfLines;
+    mergedResult.failedLines = mergedResult.failedLines.concat(
+        failedLines || []);
+    mergedResult.errors = mergedResult.errors.concat(errors || []);
+    mergedResult.output = mergedResult.output.concat(output || []);
+  });
+  return mergedResult;
+};
+
 /**
  * Sends a round of data in multiple batches (requests). Number of records in
  * every batch is defined by 'recordSize'. All these requests will be send out
@@ -175,45 +228,28 @@ const splitArray = (records, splitSize) => {
  *     single request.
  * @param {number} qps Queries per second.
  * @param {string} roundId Round ID for log.
- * @return {!Promise<boolean>}
+ * @return {!Promise<!Array<!BatchResult>>}
  * @private
  */
-const sendSingleRound = (sendingFn, sliced, recordSize, qps, roundId) => {
+const sendSingleRound = async (sendingFn, sliced, recordSize, qps, roundId) => {
   const logger = getLogger('SPEED_CTL');
   const batchArray = splitArray(sliced, recordSize);
   const securedDefer = Math.ceil(batchArray.length / qps * 1000);
-  logger.debug(`Task round[${roundId}] has ${
-      batchArray.length} requests/batches. Start:`);
-  const deferPromise = wait(securedDefer).then(() => {
+  logger.debug(
+      `Task round[${roundId}] has ${batchArray.length} requests/batches:`);
+  const deferPromise = async () => {
+    await wait(securedDefer);
     logger.debug(`Task round[${roundId}] is secured for ${securedDefer} ms.`);
-    return true;
-  });
-  const batchPromises = [deferPromise];
-  batchArray.forEach((batch, index) => {
-    const batchId = `${roundId}-${index}`;
-    batchPromises.push(sendingFn(batch, batchId).then((batchResult) => {
-      logger.debug(`Task batch[${batchId}] has ${batch.length} records: ${
-          batchResult ? 'succeeded' : 'failed'}.`);
-      return batchResult;
-    }));
-  });
-  return Promise.all(batchPromises).then((results) => {
-    const roundResult = !results.includes(false);
-    logger.debug(
-        `Task round[${roundId}] ${roundResult ? 'succeeded' : 'failed'}.`);
-    if (!roundResult) {
-      console.log(
-          `Task round[${roundId}] has batch(es) failed batches:`, results);
-    }
-    return roundResult;
-  });
+  };
+  const batchPromises = batchArray.map(
+      (batch, index) => sendingFn(batch, `${roundId}-${index}`));
+  const [batchResults] = await Promise.all([
+    Promise.all(batchPromises),
+    deferPromise(),
+  ]);
+  return batchResults;
 };
-//TODO refactor the returned value here to give more information of results.
-// Ideally return every error line's result.
-// pseudo code:
-// p.then((previousResult)=>{
-//   return thisPromise().then((newValue) => value_merge_newValue_with_previousResult);
-// }
+
 /**
  * Some APIs will have the limitations for:
  * 1. The number of conversions for each request;
@@ -225,11 +261,18 @@ const sendSingleRound = (sendingFn, sliced, recordSize, qps, roundId) => {
  * @param {number=} numberOfThreads The number of requests will be fired
  *     simultaneously.
  * @param {number=} qps Queries per second.
+ * @param {function(!Array<!BatchResult>, string):!BatchResult} mergeFn
+ *     The function to merge the array of result of each request.
+ * @param {number} timeout The time (seconds) that this function can run. This
+ *     is used to evaluate whether the given data could be processed with the
+ *     given 'qps' and 'recordSize' in time.
  * @return {function(!SendSingleBatch,(string|!Array<string>),string=):
- *     !Promise<boolean>} Speed and content managed sending function.
+ *     !BatchResult} Speed and content managed sending function.
  */
-const apiSpeedControl = (recordSize = 1, numberOfThreads = 1, qps = 1) => {
+const apiSpeedControl = (recordSize = 1, numberOfThreads = 1, qps = 1,
+    mergeFn = mergeBatchResults, timeout = 540) => {
   const roundSize = recordSize * numberOfThreads;
+  const maxRecords = qps * recordSize * timeout;
   /**
    * Returns a sending function with speed and content managed.
    * @param {!SendSingleBatch} sendingFn Function to send out a
@@ -239,27 +282,42 @@ const apiSpeedControl = (recordSize = 1, numberOfThreads = 1, qps = 1) => {
    *     time, the element in Array<string> is expected to be in the format of
    *     JSON object.
    * @param {string=} taskId Task ID for log.
-   * @return {!Promise<boolean>}
+   * @return {!BatchResult}
    */
-  return (sendingFn, data, taskId = 'unnamed') => {
+  return async (sendingFn, data, taskId = 'unnamed') => {
+    const logger = getLogger('SPEED_CTL');
     const records = Array.isArray(data) ?
         data :
         data.split('\n').filter((line) => line.trim() !== '');
+    if (maxRecords < data.length) {
+      const error = `Predicted timeout: ${records.length} records with config: `
+          + `${recordSize} records/request and ${qps} QPS in ${timeout} sec.`;
+      logger.error(error);
+      return {
+        result: false,
+        errors: [error],
+      };
+    }
     const roundArray = splitArray(records, roundSize);
-    console.log(`Task[${taskId}] has ${records.length} records in ${
-        roundArray.length} rounds.`);
-    let roundPromise = Promise.resolve(true);
-    roundArray.forEach((round, index) => {
+    logger.debug(
+        `Task [${taskId}] has ${records.length} records in ${roundArray.length} rounds.`);
+    const reduceFn = async (previous, roundData, index) => {
+      const results = await previous;
       const roundId = `${taskId}-${index}`;
-      roundPromise = roundPromise.then((previousResult) => {
-        return sendSingleRound(sendingFn, round, recordSize, qps, roundId)
-            .then(roundResult => previousResult && roundResult);
-      });
-    });
-    return roundPromise.then((taskResult) => {
-      console.log(`Task[${taskId}] ${taskResult ? 'succeeded' : 'failed'}.`);
-      return taskResult;
-    });
+      /** @const {!Array<!BatchResult>} */ const roundResult =
+          await sendSingleRound(sendingFn, roundData, recordSize, qps, roundId);
+      /** @const {!BatchResult} */
+      const mergedRoundResult = mergeFn(roundResult, roundId);
+      return results.concat(mergedRoundResult);
+    };
+    /** @const {!Array<!BatchResult>} */
+    const taskResult = await roundArray.reduce(reduceFn, []);
+    /** @const {!BatchResult} */
+    const mergedTaskResult = mergeBatchResults(taskResult, taskId);
+    logger.debug(
+        `Task [${taskId}]: ${mergedTaskResult.result ? 'succeeded' : 'failed'}.`
+    );
+    return mergedTaskResult;
   };
 };
 
@@ -463,7 +521,9 @@ const changeNamingFromSnakeToLowerCamel = (name) => {
 module.exports = {
   getLogger,
   wait,
+  BatchResult,
   SendSingleBatch,
+  sendSingleRound,
   apiSpeedControl,
   splitArray,
   getProperValue,

package/src/apis/youtube.js

@@ -1,145 +0,0 @@
-// Copyright 2021 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * @fileoverview Youtube API Client Library.
- */
-
-'use strict';
-
-const {google} = require('googleapis');
-const {Schema$Channel, Schema$Video} = google.youtube;
-const AuthClient = require('./auth_client.js');
-const {getLogger} = require('../components/utils.js');
-
-const API_SCOPES = Object.freeze([
-  'https://www.googleapis.com/auth/youtube.readonly'
-]);
-const API_VERSION = 'v3';
-
-/**
- * Configuration for listing youtube channels.
- * @see https://developers.google.com/youtube/v3/docs/channels/list
- * @typedef {{
- *   part: Array<string>,
- *   categoryId: (string|undefined),
- *   forUsername: (string|undefined),
- *   id: (string|undefined),
- *   managedByMe: (boolean|undefined),
- *   mine: (boolean|undefined),
- *   hl: (string|undefined),
- *   onBehalfOfContentOwner: (string|undefined),
- *   pageToken: (string|undefined),
- * }}
- */
-let ListChannelsConfig;
-
-/**
- * Configuration for listing youtube videos.
- * @see https://developers.google.com/youtube/v3/docs/videos/list
- * @typedef {{
- *   part: Array<string>,
- *   id: (string|undefined),
- *   chart: (string|undefined),
- *   hl: (string|undefined),
- *   maxHeight: (unsigned integer|undefined),
- *   maxResults: (unsigned integer|undefined),
- *   maxWidth: (unsigned integer|undefined),
- *   onBehalfOfContentOwner: (string|undefined),
- *   pageToken: (string|undefined),
- *   regionCode: (string|undefined),
- * }}
- */
-let ListVideosConfig;
-
-/**
- * Youtube API v3 stub.
- * See: https://developers.google.com/youtube/v3/docs
- * Channel list type definition, see:
- * https://developers.google.com/youtube/v3/docs/channels/list
- * Video list type definition, see:
- * https://developers.google.com/youtube/v3/docs/videos/list
- */
-class YouTube {
-  constructor() {
-    const authClient = new AuthClient(API_SCOPES);
-    this.auth = authClient.getDefaultAuth();
-    /** @const {!google.youtube} */
-    this.instance = google.youtube({
-      version: API_VERSION,
-    });
-    /**
-     * Logger object from 'log4js' package where this type is not exported.
-     */
-    this.logger = getLogger('API.YT');
-  }
-
-  /**
-   * Returns a collection of zero or more channel resources that match the
-   * request criteria.
-   * @see https://developers.google.com/youtube/v3/docs/channels/list
-   * @param {!ListChannelsConfig} config List channels configuration.
-   * @return {!Promise<Array<Schema$Channel>>}
-   */
-  async listChannels(config) {
-    const channelListRequest = Object.assign({
-      auth: this.auth,
-    }, config);
-    channelListRequest.part = channelListRequest.part.join(',')
-    try {
-      const response = await this.instance.channels.list(channelListRequest);
-      this.logger.debug('Response: ', response);
-      return response.data.items;
-    } catch (error) {
-      const errorMsg =
-          `Fail to list channels: ${JSON.stringify(channelListRequest)}`;
-      this.logger.error('YouTube list channels failed.', error.message);
-      this.logger.debug('Errors in response:', error);
-      throw new Error(errorMsg);
-    }
-  }
-
-  /**
-   * Returns a list of videos that match the API request parameters.
-   * @see https://developers.google.com/youtube/v3/docs/videos/list
-   * @param {!ListVideosConfig} config List videos configuration.
-   * @return {!Promise<Array<Schema$Video>>}
-   */
-  async listVideos(config) {
-    const videoListRequest = Object.assign({
-      auth: this.auth,
-    }, config);
-    videoListRequest.part = videoListRequest.part.join(',')
-    try {
-      const response = await this.instance.videos.list(videoListRequest);
-      this.logger.debug('Response: ', response);
-      return response.data.items;
-    } catch (error) {
-      const errorMsg =
-          `Fail to list videos: ${JSON.stringify(videoListRequest)}`;
-      this.logger.error('YouTube list videos failed.', error.message);
-      this.logger.debug('Errors in response:', error);
-      throw new Error(errorMsg);
-    }
-  }
-
-}
-
-module.exports = {
-  YouTube,
-  ListChannelsConfig,
-  ListVideosConfig,
-  API_VERSION,
-  API_SCOPES,
-};