@google-cloud/nodejs-common 1.9.0 → 1.9.2-alpha

package/bin/install_functions.sh

@@ -1933,7 +1933,8 @@ check_firestore_existence() {
   local firestore mode appRegion
   mode="${1}"
   appRegion="${2}"
-  firestore=$(gcloud firestore databases list --format="csv[no-heading](databases.type)")
+  firestore=$(gcloud firestore databases list \
+--filter="name:(default)" --format="csv[no-heading](type)")
   if [[ -z "${firestore}" ]]; then
     printf '%s\n' "Firestore is not ready. Creating a new Firestore database\
 is an irreversible operation, so read carefully before continue:"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@google-cloud/nodejs-common",
-  "version": "1.9.0",
+  "version": "1.9.2-alpha",
   "description": "A NodeJs common library for solutions based on Cloud Functions",
   "author": "Google Inc.",
   "license": "Apache-2.0",
@@ -19,7 +19,7 @@
     "@google-cloud/aiplatform": "^3.0.0",
     "@google-cloud/automl": "^4.0.0",
     "@google-cloud/bigquery": "^7.2.0",
-    "@google-cloud/datastore": "^8.0.0",
+    "@google-cloud/datastore": "^8.2.2",
     "@google-cloud/firestore": "^6.7.0",
     "@google-cloud/logging-winston": "^6.0.0",
     "@google-cloud/pubsub": "^4.0.2",

package/src/apis/ads_data_hub.js

@@ -108,7 +108,9 @@ class AdsDataHub extends AuthRestfulApi {
    * to the specified BigQuery destination table. The returned operation name
    * can be used to poll for query completion status.
    * @see https://developers.google.com/ads-data-hub/reference/rest/v1/customers.analysisQueries/startTransient
-   * @param {string} queryText The content of the query.
+   * @param {!AnalysisQuery} query An analysis query that can be executed within
+   *     Ads Data Hub. Its properties include 'queryText', 'parameterTypes', etc.
+   *     @see https://developers.google.com/ads-data-hub/reference/rest/v1/customers.analysisQueries#AnalysisQuery
    * @param {Object} spec Defines the query execution parameters.
    *     @see https://developers.google.com/ads-data-hub/reference/rest/v1/QueryExecutionSpec
    * @param {string} destTable Destination BigQuery table for query results with
@@ -121,9 +123,9 @@ class AdsDataHub extends AuthRestfulApi {
    * @return {!Promise<Object>} Promised operation object.
    *     @see https://developers.google.com/ads-data-hub/reference/rest/v1/operations#Operation
    */
-  async startTransientQuery(queryText, spec, destTable, customerId = this.customerId) {
+  async startTransientQuery(query, spec, destTable, customerId = this.customerId) {
     const path = `customers/${customerId}/analysisQueries:startTransient`;
-    const data = { query: { queryText }, spec, destTable };
+    const data = { query, spec, destTable };
     const response = await this.request(path, 'POST', data);
     return response.data;
   }

package/src/apis/base/auth_restful_api.js

@@ -16,13 +16,25 @@
  * @fileoverview A base class for RESTful API with authorization client.
  */
 
-const { RestfuleApiBase } = require('./restful_api_base');
+const { RestfulApiBase } = require('./restful_api_base');
 const AuthClient = require('../auth_client.js');
 
 /**
  * A RESTful API class with authorization client.
  */
-class AuthRestfulApi extends RestfuleApiBase {
+class AuthRestfulApi extends RestfulApiBase {
+
+  constructor(env = process.env, options = {}) {
+    super(env);
+    /**
+     * `authClient` can be consumed by cloud client library as the auth
+     * client. By passing this in, we can offer more flexible auth clients in
+     * test cases for API client library and cloud client library in future.
+     */
+    if (options.authClient) {
+      this.auth = options.authClient;
+    }
+  }
 
   /**
    * Returns the Api scope for authorization.

package/src/apis/base/restful_api_base.js

@@ -22,6 +22,8 @@ const {
   GaxiosOptions,
 } = require('gaxios');
 
+const { getLogger } = require('../../components/utils.js');
+
 /**
  * A type for an Api response which is independent to the 'request' library.
  * @typedef {{
@@ -35,10 +37,11 @@ let Response;
  * Base class for RESTful API based on gaxios.
  * @see https://github.com/googleapis/gaxios
  */
-class RestfuleApiBase {
+class RestfulApiBase {
 
   constructor(env = process.env) {
     this.env = env;
+    this.logger = getLogger(this.constructor.name);
   }
 
   /**
@@ -46,7 +49,7 @@ class RestfuleApiBase {
    * @return {string}
    * @abstract
    */
-  getBaseUrl() { }
+  async getBaseUrl() { }
 
   /**
    * Returns default HTTP headers.
@@ -93,7 +96,7 @@ class RestfuleApiBase {
     const options = Object.assign(
       this.getRequesterOptions(),
       {
-        url: this.getRequestUrl(uri),
+        url: await this.getRequestUrl(uri),
         method,
         headers: headers || await this.getDefaultHeaders(),
         data,
@@ -111,8 +114,8 @@ class RestfuleApiBase {
    * @param {string} requestUri The URI of the specific resource to request.
    * @return {string} representing the fully-qualified API URL.
    */
-  getRequestUrl(requestUri) {
-    const baseUrl = this.getBaseUrl();
+  async getRequestUrl(requestUri) {
+    const baseUrl = await this.getBaseUrl();
     if (requestUri) {
       if (requestUri.toLocaleLowerCase().startsWith('http')) return requestUri;
       return `${baseUrl}/${requestUri}`;
@@ -139,5 +142,5 @@ class RestfuleApiBase {
 
 module.exports = {
   Response,
-  RestfuleApiBase,
+  RestfulApiBase,
 };

package/src/apis/cloud/google_cloud_api.js

@@ -0,0 +1,79 @@
+// Copyright 2023 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * @fileoverview The base class of Google Cloud Apis.
+ */
+const AuthClient = require('../auth_client.js');
+const { AuthRestfulApi } = require('../base/auth_restful_api.js');
+
+/**
+ * A RESTful API class for Google Cloud Platform API with ADC as the
+ * authorization method.
+ */
+class GoogleCloudApi extends AuthRestfulApi {
+
+  constructor(env = process.env, options = {}) {
+    super(env, options);
+    this.projectId = options.projectId || env['GCP_PROJECT'];
+  }
+
+  /** @override */
+  getScope() {
+    return ['https://www.googleapis.com/auth/cloud-platform'];
+  }
+
+  /**
+   * @override
+   * Returns the `GoogleAuth` object for a Google Cloud component object.
+   * If the component is based on Google Cloud Client Library, it will use the
+   * `ADC` auth object by itself. However, some components are based on Google
+   * API Client Library, which is based on API Scope and an explicit auth
+   * object. In this situation, this function can help to generate such an auth
+   * object.
+   * By default, `AuthClient` (getDefaultAuth()) will return an auth client
+   * based on the settings in ENV while the OAuth is the most preferred.
+   * This works for most of the external API clients (in the '../apis'
+   * folder), however this won't work in the Cloud Functions, as those OAuth
+   * token usually won't have enough permission to invoke Google Cloud API.
+   * Using the method `getApplicationDefaultCredentials` to force
+   * `AuthClient` return an ADC auth client, which will work in the Cloud.
+   */
+  async getAuth() {
+    if (this.auth) return this.auth;
+    this.authClient = new AuthClient(this.getScope(), this.env);
+    // Not required for ADC: await this.authClient.prepareCredentials();
+    this.auth = this.authClient.getApplicationDefaultCredentials();
+    return this.auth;
+  }
+
+  /**
+   * Gets the GCP project Id. In Cloud Functions, it *should* be passed in
+   * through environment variable during the deployment. But if it doesn't exist
+   * (for example, in local unit tests), this function will fallback to ADC
+   * (Application Default Credential) auth's asynchronous function to get the
+   * project Id.
+   * @return {string}
+   */
+  async getProjectId() {
+    if (!this.projectId) {
+      const auth = await this.getAuth();
+      this.projectId = await auth.getProjectId();
+    }
+    return this.projectId;
+  }
+
+}
+
+module.exports = { GoogleCloudApi };

package/src/apis/dfa_reporting.js

@@ -391,18 +391,18 @@ class DfaReporting {
     throw new Error(`Unsupported report status: ${data.status}`);
   }
 
-  //TODO(lushu) check the response for very big file.
   /**
-   * Downloads the report file.
+   * Gets the stream of the report file.
    * @param {string} url
-   * @return {!Promise<string>}
+   * @return {!Promise<stream>}
    */
-  async downloadReportFile(url) {
+  async getReportFileStream(url) {
     const auth = await this.getAuth_();
     const headers = await auth.getRequestHeaders();
     const response = await request({
       method: 'GET',
       headers,
+      responseType: 'stream',
       url,
     });
     return response.data;

package/src/apis/sendgrid.js

@@ -18,7 +18,7 @@
 
 'use strict';
 
-const { RestfuleApiBase } = require('./base/restful_api_base.js');
+const { RestfulApiBase } = require('./base/restful_api_base.js');
 const { getLogger } = require('../components/utils.js');
 
 const API_VERSION = 'v3';
@@ -28,7 +28,7 @@ const API_ENDPOINT = 'https://api.sendgrid.com';
  * SendGrid API access class.
  * @see https://docs.sendgrid.com/api-reference/how-to-use-the-sendgrid-v3-api/authentication
  */
-class SendGrid extends RestfuleApiBase {
+class SendGrid extends RestfulApiBase {
 
   constructor(apiKey, env = process.env) {
     super(env);

package/src/components/firestore/access_base.js

@@ -27,6 +27,10 @@ const {
   DocumentData,
   Transaction: NativeModeTransaction,
 } = require('@google-cloud/firestore');
+const {
+  FirestoreApi,
+  DEFAULT_DATABASE,
+} = require('./firestore_api.js');
 
 /**
  * Document in Native mode or Entity in Datastore mode.
@@ -122,6 +126,17 @@ const DataSource = Object.freeze({
   DATASTORE: 'datastore',
 });
 
+/**
+ * Definition of a database. Currently, it supports:
+ * 1. Firestore Native mode
+ * 2. Firestore Datastore mode
+ * @typedef {(
+ *   source: !DataSource,
+ *   id: string,
+ * )}
+ */
+let Database;
+
 /**
  * Firestore has two modes: 'Native' and 'Datastore'. Only one of them can be
  * set in a Cloud Project and it can not be changed in the Cloud Project once
@@ -216,10 +231,33 @@ class FirestoreAccessBase {
 
 }
 
+/**
+ * Returns whether the mode of Firestore is 'Native'.
+ * @param {string} projectId GCP project Id.
+ * @param {string=} databaseId GCP project Id.
+ * @return {!Promise<!Database>}
+ */
+async function getFirestoreDatabase(projectId, databaseId = DEFAULT_DATABASE) {
+  try {
+    const firestore = new FirestoreApi(process.env, { projectId, databaseId });
+    const type = await firestore.getFirestoreMode();
+    console.log(`Get Firestore ${databaseId}@${projectId}, mode: ${type}`);
+    return {
+      source: DataSource[type],
+      id: firestore.databaseId,
+    };
+  } catch (error) {
+    console.error(`Failed to get Firestore ${databaseId}@${projectId}: `,
+      error.message);
+    throw error;
+  }
+};
+
 /**
  * Returns whether the mode of Firestore is 'Native'.
  * @param {string=} projectId GCP project Id.
  * @return {!Promise<boolean>}
+ * @deprecated
  */
 async function isNativeMode(projectId = process.env['GCP_PROJECT']) {
   try {
@@ -237,8 +275,12 @@ module.exports = {
   Transaction,
   Filter,
   TransactionOperation,
+  Database,
   DatastoreDocumentFacade,
   DatastoreTransactionFacade,
   FirestoreAccessBase,
+  getFirestoreDatabase,
+  DEFAULT_DATABASE,
+  FirestoreApi,
   isNativeMode,
 };

package/src/components/firestore/data_access_object.js

@@ -25,6 +25,8 @@ const {
   Filter,
   TemplatedTxFunction,
   FirestoreAccessBase,
+  Database,
+  DEFAULT_DATABASE,
 } = require('./access_base.js');
 const NativeModeAccess = require('./native_mode_access.js');
 const DatastoreModeAccess = require('./datastore_mode_access.js');
@@ -33,8 +35,7 @@ const DatastoreModeAccess = require('./datastore_mode_access.js');
  * This is data access object base class on Firestore. It seals the details of
  * different underlying databases, Firestore and Datastore.
  * This class relies on an initial parameter in construtor to indicate the
- * Firestore type. If omitted, this parameter will take the value of the
- * environment variable named 'FIRESTORE_TYPE'.
+ * Firestore type.
  *
  * Firestore and Datastore have different transaction APIs. In that case,
  * separate DAOs are required to offer different functions in each mode.
@@ -45,22 +46,24 @@ class DataAccessObject {
    * Initializes the instance based on given data source.
    * @param {string} kind The data model name.
    * @param {string} namespace The namespace of the data.
-   * @param {!DataSource} dataSource The data source type.
+   * @param {!Database|!DataSource} database The database or data base type.
    * @param {string} projectId The Id of Cloud project.
    */
-  constructor(kind, namespace, dataSource = process.env['FIRESTORE_TYPE'],
-      projectId = process.env['GCP_PROJECT']) {
+  constructor(kind, namespace, database,
+    projectId = process.env['GCP_PROJECT']) {
     /** @const {string} */ this.namespace = namespace;
-    /** @const {!DataSource} */ this.dataSource = dataSource;
+    /** @const {!DataSource} */ this.dataSource =
+      typeof database === 'string' ? database : database.source;
+    const databaseId = database.id || DEFAULT_DATABASE;
     /** @type {!FirestoreAccessBase} */ this.accessObject = undefined;
     switch (this.dataSource) {
       case DataSource.FIRESTORE:
         this.accessObject = new NativeModeAccess(
-            `${this.namespace}/database/${kind}`, projectId);
+          `${this.namespace}/database/${kind}`, projectId, databaseId);
         break;
       case DataSource.DATASTORE:
         this.accessObject = new DatastoreModeAccess(this.namespace, kind,
-            projectId);
+          projectId, databaseId);
         break;
       default:
         throw new Error(`Unknown DataSource item: ${this.dataSource}.`);

package/src/components/firestore/datastore_mode_access.js

@@ -29,6 +29,7 @@ const {
   DatastoreDocumentFacade,
   DatastoreTransactionFacade,
   FirestoreAccessBase,
+  DEFAULT_DATABASE,
 } = require('./access_base.js');
 const {getLogger, wait} = require('../utils.js');
 
@@ -46,11 +47,15 @@ class DatastoreModeAccess {
    * Initializes DatastoreModeAccess instance.
    * @param {string} namespace The namespace for data.
    * @param {string} kind The kind of this entity.
-   * @param {string} projectId The Id of Cloud project.
+   * @param {string=} projectId The Id of Cloud project.
+   * @param {string=} databaseId The Id of Firestore database.
    */
-  constructor(namespace, kind, projectId = process.env['GCP_PROJECT']) {
+  constructor(namespace, kind, projectId = process.env['GCP_PROJECT'],
+    databaseId = process.env['DATABASE_ID'] || DEFAULT_DATABASE) {
+    // '(default)' is not allowed, use empty string '' to refer the default database.
+    const idForDatastore = databaseId === DEFAULT_DATABASE ? '' : databaseId;
     /** @type{Datastore} */
-    this.datastore = new Datastore({projectId});
+    this.datastore = new Datastore({ projectId, databaseId: idForDatastore });
     this.kind = kind;
     this.namespace = namespace;
     this.logger = getLogger('DS.ACC');

package/src/components/firestore/firestore_api.js

@@ -0,0 +1,89 @@
+// Copyright 2023 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const { GoogleCloudApi } = require('../../apis/cloud/google_cloud_api');
+
+/**
+ * The name of default database.
+ * @see https://cloud.google.com/firestore/docs/reference/rest/v1/projects.databases#resource:-database
+ * @const {string}
+ */
+const DEFAULT_DATABASE = '(default)';
+
+/**
+ * The modes of Firestore. Mode changes are only allowed if the database is empty.
+ * @see https://cloud.google.com/datastore/docs/firestore-or-datastore
+ * @see https://cloud.google.com/firestore/docs/reference/rest/v1/projects.databases#DatabaseType
+ * @enum {string}
+ */
+const FIRESTORE_MODE = Object.freeze({
+  FIRESTORE_NATIVE: 'FIRESTORE',
+  DATASTORE_MODE: 'DATASTORE',
+});
+
+class FirestoreApi extends GoogleCloudApi {
+
+  constructor(env = process.env, options = {}) {
+    super(env, options);
+    this.apiUrl = 'https://firestore.googleapis.com';
+    this.version = 'v1';
+    this.databaseId = options.databaseId || env.DATABASE_ID || DEFAULT_DATABASE;
+  }
+
+  /** @override */
+  async getBaseUrl() {
+    const projectId = await this.getProjectId();
+    return `${this.apiUrl}/${this.version}/projects/${projectId}`;
+  }
+
+  /**
+   * Gets the information of the default database.
+   * If the database is ready, will return:
+   * {
+   *   data: {
+   *     name:...
+   *     type:...
+   *   },
+   *   code: 200
+   * }
+   * If the database is not created, will throw an error: "Request failed with
+   * status code 404"
+   * @see https://cloud.google.com/firestore/docs/reference/rest/v1/projects.databases/get
+   * @return {Database}
+   * @see https://cloud.google.com/firestore/docs/reference/rest/v1/projects.databases#Database
+   */
+  async getDatabase(databaseId = this.databaseId) {
+    return super.request(`databases/${databaseId}`);
+  }
+
+  /**
+   * Gets the mode of the Firestore database.
+   * @param {string=} databaseId
+   * @return {'FIRESTORE'|'DATASTORE'} Mode of the Firestore.
+   */
+  async getFirestoreMode(databaseId = this.databaseId) {
+    const response = await this.getDatabase(databaseId);
+    if (!response.data) {
+      this.logger.error(`Fail to find database ${databaseId}`, response);
+      throw new Error(`Fail to find database ${databaseId}`);
+    }
+    this.logger.debug(`Get ${databaseId} in mode ${response.data.type}.`);
+    return FIRESTORE_MODE[response.data.type];
+  }
+}
+
+module.exports = {
+  DEFAULT_DATABASE,
+  FirestoreApi,
+};

package/src/components/firestore/native_mode_access.js

@@ -24,7 +24,7 @@ const {
   CollectionReference,
   OrderByDirection,
 } = require('@google-cloud/firestore');
-const {FirestoreAccessBase} = require('./access_base.js');
+const { FirestoreAccessBase, DEFAULT_DATABASE } = require('./access_base.js');
 const {getLogger} = require('../utils.js');
 
 /**
@@ -41,11 +41,13 @@ class NativeModeAccess {
    * way. This constructor will check the path to make sure it presents a
    * 'collection', otherwise an Error will be thrown.
    * @param {string} path Path for the 'collection'.
-   * @param {string} projectId The Id of Cloud project.
+   * @param {string=} projectId The Id of Cloud project.
+   * @param {string=} databaseId The Id of Firestore database.
    */
-  constructor(path, projectId = process.env['GCP_PROJECT']) {
+  constructor(path, projectId = process.env['GCP_PROJECT'],
+    databaseId = process.env['DATABASE_ID'] || DEFAULT_DATABASE) {
     /** @type {!Firestore} */
-    this.firestore = new Firestore({projectId});
+    this.firestore = new Firestore({ projectId, databaseId });
     if (path.split('/').length % 2 === 0) {
       throw new Error(`Invalid path for Collection: ${path}`);
     }