@causa/workspace-google 0.3.1 → 0.4.0

package/README.md

@@ -114,3 +114,11 @@ The Google module provides several infrastructure processors, which can be used
 [GoogleServicesEnable](./src/functions/google-services-enable.ts) is the same underlying function as the `cs google enableServices` command. It enables GCP services before preparing or deploying the infrastructure.
 
 Although infrastructure as code tools usually expose this feature as well (e.g. the [`google_project_service`](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_service) Terraform resource), it might be more convenient to enable all the required services before running those tools. It avoids having to define dependencies between the services and all the actual resources being deployed.
+
+### `GoogleSpannerWriteDatabases`
+
+[GoogleSpannerWriteDatabases](./src/functions/google-spanner-write-databases.ts) writes a configuration file for each Spanner database, such that it can be picked up by the Causa Spanner Terraform module. This allows automatic setup of Spanner databases and their DDLs.
+
+### `GooglePubSubWriteTopics`
+
+[GooglePubSubWriteTopics](./src/functions/google-pubsub-write-topics.ts) writes a configuration file for each event topic, such that it can be picked up by the Causa Pub/Sub Terraform module. This allows automatic setup of Pub/Sub topics, and optionally of the corresponding BigQuery tables.

package/dist/configurations/google.d.ts

@@ -153,6 +153,19 @@ export type GoogleConfiguration = {
                  */
                 readonly containerName?: string;
             };
+            /**
+             * The directory where the topic configuration files are written by the `GooglePubSubWriteTopics` processor.
+             */
+            readonly topicConfigurationsDirectory?: string;
+            /**
+             * Configuration for the storage of Pub/Sub events in BigQuery.
+             */
+            readonly bigQueryStorage?: {
+                /**
+                 * The ID of the BigQuery dataset where raw Pub/Sub events should be stored.
+                 */
+                readonly rawEventsDatasetId?: string;
+            };
         };
         /**
          * Configuration for the Spanner service.
@@ -175,6 +188,25 @@ export type GoogleConfiguration = {
                  */
                 readonly instanceName?: string;
             };
+            /**
+             * Configuration for the Spanner instance.
+             */
+            readonly instance?: {
+                /**
+                 * The name of the Spanner instance.
+                 */
+                readonly name?: string;
+                /**
+                 * The instance geographic configuration.
+                 * See https://cloud.google.com/spanner/docs/instance-configurations for more details.
+                 */
+                readonly configuration?: string;
+                /**
+                 * The compute capacity of the instance.
+                 * See https://cloud.google.com/spanner/docs/compute-capacity for more details.
+                 */
+                readonly processingUnits?: number;
+            };
             /**
              * Defines how DDLs are found for the Spanner databases in the workspace.
              */
@@ -192,6 +224,11 @@ export type GoogleConfiguration = {
                  */
                 readonly regularExpression?: string;
             };
+            /**
+             * The directory where the database configuration files are written by the `GoogleSpannerWriteDatabases`
+             * processor.
+             */
+            readonly databaseConfigurationsDirectory?: string;
         };
         /**
          * Configuration for Cloud Functions.
@@ -210,6 +247,41 @@ export type GoogleConfiguration = {
              * The Docker repository where Cloud Run containers should be uploaded.
              */
             readonly dockerRepository?: string;
+            /**
+             * The location where Cloud Run services should be deployed.
+             */
+            readonly location?: string;
+            /**
+             * A map where keys are the name of environment variables, and values are secret versions IDs (e.g.
+             * `projects/my-project/secrets/my-secret/versions/1`).
+             */
+            readonly secretEnvironmentVariables?: Record<string, string>;
+            /**
+             * Whether the container is running between requests and can perform background operations.
+             */
+            readonly cpuAlwaysAllocated?: boolean;
+            /**
+             * The max duration the instance is allowed for responding to a request, in seconds.
+             */
+            readonly timeout?: number;
+            /**
+             * The maximum allowed in-flight (concurrent) requests per container.
+             */
+            readonly requestConcurrency?: number;
+            /**
+             * The type of allowed ingress that can reach the container.
+             * See https://cloud.google.com/run/docs/securing/ingress#settings for more details.
+             */
+            readonly ingress?: 'all' | 'internal' | 'internal-and-cloud-load-balancing';
+            /**
+             * The name of the VPC access connector through which egress traffic should be routed.
+             */
+            readonly vpcAccessConnector?: string;
+            /**
+             * The setting for egress traffic that goes through the VPC access connector.
+             * See https://cloud.google.com/run/docs/configuring/connecting-vpc#manage for more details.
+             */
+            readonly vpcAccessConnectorEgressSettings?: 'all-traffic' | 'private-ranges-only';
         };
     };
 };

package/dist/functions/emulator-start-spanner.js

@@ -1,6 +1,6 @@
 import { DockerEmulatorService, EmulatorStart, } from '@causa/workspace-core';
 import { Spanner } from '@google-cloud/spanner';
-import { grpc } from 'google-gax';
+import { credentials } from '@grpc/grpc-js';
 import { getLocalGcpProject, } from '../configurations/index.js';
 import { SPANNER_EMULATOR_NAME, SPANNER_GRPC_PORT, SPANNER_HTTP_PORT, SPANNER_IMAGE, getSpannerContainerName, } from '../emulators/index.js';
 import { GoogleSpannerListDatabases } from './google-spanner-list-databases.js';
@@ -75,7 +75,7 @@ export class EmulatorStartForSpanner extends EmulatorStart {
             servicePath: '127.0.0.1',
             port: SPANNER_GRPC_PORT,
             projectId: getLocalGcpProject(context),
-            sslCreds: grpc.credentials.createInsecure(),
+            sslCreds: credentials.createInsecure(),
         });
         const [instance, operation] = await spanner.createInstance(instanceName, {
             config: 'emulator-config',

package/dist/functions/google-pubsub-write-topics.d.ts

@@ -0,0 +1,22 @@
+import { ProcessorResult, WorkspaceContext, WorkspaceFunction } from '@causa/workspace';
+import { InfrastructureProcessor } from '@causa/workspace-core';
+/**
+ * A function that uses {@link EventTopicList} to find all the topics in the workspace, and writes their configurations
+ * to a directory.
+ * The `google.pubSub.topicConfigurationsDirectory` configuration can be used to specify the output location of the
+ * topic configurations.
+ * This function returns a partial configuration, such that it can be used as a processor.
+ */
+export declare class GooglePubSubWriteTopics extends WorkspaceFunction<Promise<ProcessorResult>> implements InfrastructureProcessor {
+    readonly tearDown?: boolean;
+    /**
+     * Returns the path to the directory where Pub/Sub topic configurations should be written.
+     * It is either fetched from the workspace configuration, or the default value is used.
+     *
+     * @param context The {@link WorkspaceContext}.
+     * @returns The path to the directory where topic configurations should be written.
+     */
+    private getConfigurationsDirectory;
+    _call(context: WorkspaceContext): Promise<ProcessorResult>;
+    _supports(): boolean;
+}

package/dist/functions/google-pubsub-write-topics.js

@@ -0,0 +1,80 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { WorkspaceFunction, } from '@causa/workspace';
+import { EventTopicList, } from '@causa/workspace-core';
+import { CAUSA_FOLDER } from '@causa/workspace/initialization';
+import { AllowMissing } from '@causa/workspace/validation';
+import { IsBoolean } from 'class-validator';
+import { mkdir, rm, writeFile } from 'fs/promises';
+import { join } from 'path';
+/**
+ * The default directory where Pub/Sub topic configurations are written, relative to the workspace root.
+ */
+const DEFAULT_TOPIC_CONFIGURATIONS_DIRECTORY = join(CAUSA_FOLDER, 'pubsub-topics');
+/**
+ * A function that uses {@link EventTopicList} to find all the topics in the workspace, and writes their configurations
+ * to a directory.
+ * The `google.pubSub.topicConfigurationsDirectory` configuration can be used to specify the output location of the
+ * topic configurations.
+ * This function returns a partial configuration, such that it can be used as a processor.
+ */
+export class GooglePubSubWriteTopics extends WorkspaceFunction {
+    tearDown;
+    /**
+     * Returns the path to the directory where Pub/Sub topic configurations should be written.
+     * It is either fetched from the workspace configuration, or the default value is used.
+     *
+     * @param context The {@link WorkspaceContext}.
+     * @returns The path to the directory where topic configurations should be written.
+     */
+    getConfigurationsDirectory(context) {
+        return (context
+            .asConfiguration()
+            .get('google.pubSub.topicConfigurationsDirectory') ??
+            DEFAULT_TOPIC_CONFIGURATIONS_DIRECTORY);
+    }
+    async _call(context) {
+        const topicConfigurationsDirectory = this.getConfigurationsDirectory(context);
+        const absoluteDir = join(context.rootPath, topicConfigurationsDirectory);
+        await rm(absoluteDir, { recursive: true, force: true });
+        if (this.tearDown) {
+            context.logger.debug(`📫 Tore down Pub/Sub topic configurations directory '${absoluteDir}'.`);
+            return { configuration: {} };
+        }
+        context.logger.info('️📫 Listing and writing Pub/Sub topic configurations.');
+        const topics = await context.call(EventTopicList, {});
+        context.logger.debug(`📫 Writing configurations for Pub/Sub topics: ${topics
+            .map((d) => `'${d.id}'`)
+            .join(', ')}.`);
+        await mkdir(absoluteDir, { recursive: true });
+        await Promise.all(topics.map(async (topic) => {
+            const topicConfiguration = {
+                ...topic,
+                bigQueryTableName: topic.id.replace(/[-\.]/g, '_'),
+            };
+            const topicFile = join(absoluteDir, `${topic.id}.json`);
+            await writeFile(topicFile, JSON.stringify(topicConfiguration));
+        }));
+        context.logger.debug(`️📫 Wrote Pub/Sub topic configurations in '${absoluteDir}'.`);
+        return {
+            configuration: {
+                google: { pubSub: { topicConfigurationsDirectory } },
+            },
+        };
+    }
+    _supports() {
+        return true;
+    }
+}
+__decorate([
+    IsBoolean(),
+    AllowMissing(),
+    __metadata("design:type", Boolean)
+], GooglePubSubWriteTopics.prototype, "tearDown", void 0);

package/dist/functions/google-spanner-write-databases.d.ts

@@ -0,0 +1,22 @@
+import { ProcessorResult, WorkspaceContext, WorkspaceFunction } from '@causa/workspace';
+import { InfrastructureProcessor } from '@causa/workspace-core';
+/**
+ * A function that uses {@link GoogleSpannerListDatabases} to find all the Spanner databases in the workspace, and
+ * writes their configurations to a directory.
+ * The `google.spanner.databaseConfigurationsDirectory` configuration can be used to specify the output location of the
+ * database configurations.
+ * This function returns a partial configuration, such that it can be used as a processor.
+ */
+export declare class GoogleSpannerWriteDatabases extends WorkspaceFunction<Promise<ProcessorResult>> implements InfrastructureProcessor {
+    readonly tearDown?: boolean;
+    /**
+     * Returns the path to the directory where Spanner databases configurations should be written.
+     * It is either fetched from the workspace configuration, or the default value is used.
+     *
+     * @param context The {@link WorkspaceContext}.
+     * @returns The path to the directory where database configurations should be written.
+     */
+    private getConfigurationsDirectory;
+    _call(context: WorkspaceContext): Promise<ProcessorResult>;
+    _supports(): boolean;
+}

package/dist/functions/google-spanner-write-databases.js

@@ -0,0 +1,73 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import { WorkspaceFunction, } from '@causa/workspace';
+import { CAUSA_FOLDER } from '@causa/workspace/initialization';
+import { AllowMissing } from '@causa/workspace/validation';
+import { IsBoolean } from 'class-validator';
+import { mkdir, rm, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { GoogleSpannerListDatabases } from './google-spanner-list-databases.js';
+/**
+ * The default directory where Spanner database configurations are written, relative to the workspace root.
+ */
+const DEFAULT_DATABASE_CONFIGURATIONS_DIRECTORY = join(CAUSA_FOLDER, 'spanner-databases');
+/**
+ * A function that uses {@link GoogleSpannerListDatabases} to find all the Spanner databases in the workspace, and
+ * writes their configurations to a directory.
+ * The `google.spanner.databaseConfigurationsDirectory` configuration can be used to specify the output location of the
+ * database configurations.
+ * This function returns a partial configuration, such that it can be used as a processor.
+ */
+export class GoogleSpannerWriteDatabases extends WorkspaceFunction {
+    tearDown;
+    /**
+     * Returns the path to the directory where Spanner databases configurations should be written.
+     * It is either fetched from the workspace configuration, or the default value is used.
+     *
+     * @param context The {@link WorkspaceContext}.
+     * @returns The path to the directory where database configurations should be written.
+     */
+    getConfigurationsDirectory(context) {
+        return (context
+            .asConfiguration()
+            .get('google.spanner.databaseConfigurationsDirectory') ??
+            DEFAULT_DATABASE_CONFIGURATIONS_DIRECTORY);
+    }
+    async _call(context) {
+        const databaseConfigurationsDirectory = this.getConfigurationsDirectory(context);
+        const absoluteDir = join(context.rootPath, databaseConfigurationsDirectory);
+        await rm(absoluteDir, { recursive: true, force: true });
+        if (this.tearDown) {
+            context.logger.debug(`️🗃️ Tore down Spanner database configurations directory '${absoluteDir}'.`);
+            return { configuration: {} };
+        }
+        context.logger.info('🗃️ Listing and writing Spanner database configurations.');
+        const databases = await context.call(GoogleSpannerListDatabases, {});
+        await mkdir(absoluteDir, { recursive: true });
+        context.logger.debug(`🗃️ Writing configurations for Spanner databases: ${databases
+            .map((d) => `'${d.id}'`)
+            .join(', ')}.`);
+        await Promise.all(databases.map((database) => writeFile(join(absoluteDir, `${database.id}.json`), JSON.stringify(database))));
+        context.logger.debug(`🗃️ Wrote Spanner database configurations in '${absoluteDir}'.`);
+        return {
+            configuration: {
+                google: { spanner: { databaseConfigurationsDirectory } },
+            },
+        };
+    }
+    _supports() {
+        return true;
+    }
+}
+__decorate([
+    IsBoolean(),
+    AllowMissing(),
+    __metadata("design:type", Boolean)
+], GoogleSpannerWriteDatabases.prototype, "tearDown", void 0);

package/dist/functions/index.js

@@ -13,12 +13,14 @@ import { GoogleFirebaseStorageMergeRules } from './google-firebase-storage-merge
 import { GoogleFirestoreMergeRules } from './google-firestore-merge-rules.js';
 import { GoogleIdentityPlatformGenerateCustomToken } from './google-identity-platform-generate-custom-token.js';
 import { GoogleIdentityPlatformGenerateToken } from './google-identity-platform-generate-token.js';
+import { GooglePubSubWriteTopics } from './google-pubsub-write-topics.js';
 import { GoogleServicesEnable } from './google-services-enable.js';
 import { GoogleSpannerListDatabases } from './google-spanner-list-databases.js';
+import { GoogleSpannerWriteDatabases } from './google-spanner-write-databases.js';
 import { ProjectGetArtefactDestinationForCloudFunctions } from './project-get-artefact-destination-cloud-functions.js';
 import { ProjectGetArtefactDestinationForCloudRun } from './project-get-artefact-destination-cloud-run.js';
 import { ProjectPushArtefactForCloudFunctions } from './project-push-artefact-cloud-functions.js';
 import { SecretFetchForGoogleSecretManager } from './secret-fetch-secret-manager.js';
 export function registerFunctions(context) {
-    context.registerFunctionImplementations(EmulatorStartForFirebaseStorage, EmulatorStartForFirestore, EmulatorStartForIdentityPlatform, EmulatorStartForPubSub, EmulatorStartForSpanner, EmulatorStopForFirebaseStorage, EmulatorStopForFirestore, EmulatorStopForIdentityPlatform, EmulatorStopForPubSub, EmulatorStopForSpanner, GoogleAppCheckGenerateToken, GoogleFirebaseStorageMergeRules, GoogleFirestoreMergeRules, GoogleIdentityPlatformGenerateCustomToken, GoogleIdentityPlatformGenerateToken, GoogleServicesEnable, GoogleSpannerListDatabases, ProjectGetArtefactDestinationForCloudFunctions, ProjectGetArtefactDestinationForCloudRun, ProjectPushArtefactForCloudFunctions, SecretFetchForGoogleSecretManager);
+    context.registerFunctionImplementations(EmulatorStartForFirebaseStorage, EmulatorStartForFirestore, EmulatorStartForIdentityPlatform, EmulatorStartForPubSub, EmulatorStartForSpanner, EmulatorStopForFirebaseStorage, EmulatorStopForFirestore, EmulatorStopForIdentityPlatform, EmulatorStopForPubSub, EmulatorStopForSpanner, GoogleAppCheckGenerateToken, GoogleFirebaseStorageMergeRules, GoogleFirestoreMergeRules, GoogleIdentityPlatformGenerateCustomToken, GoogleIdentityPlatformGenerateToken, GooglePubSubWriteTopics, GoogleServicesEnable, GoogleSpannerListDatabases, GoogleSpannerWriteDatabases, ProjectGetArtefactDestinationForCloudFunctions, ProjectGetArtefactDestinationForCloudRun, ProjectPushArtefactForCloudFunctions, SecretFetchForGoogleSecretManager);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@causa/workspace-google",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "The Causa workspace module providing many functionalities related to GCP and its services.",
   "repository": "github:causa-io/workspace-module-google",
   "license": "ISC",
@@ -30,38 +30,38 @@
   },
   "dependencies": {
     "@causa/cli": ">= 0.4.0 < 1.0.0",
-    "@causa/workspace": ">= 0.10.0 < 1.0.0",
-    "@causa/workspace-core": ">= 0.7.0 < 1.0.0",
+    "@causa/workspace": ">= 0.12.0 < 1.0.0",
+    "@causa/workspace-core": ">= 0.8.0 < 1.0.0",
     "@google-cloud/apikeys": "^0.2.2",
     "@google-cloud/iam-credentials": "^2.0.4",
-    "@google-cloud/pubsub": "^3.7.1",
+    "@google-cloud/pubsub": "^3.7.3",
     "@google-cloud/secret-manager": "^4.2.2",
     "@google-cloud/service-usage": "^2.2.2",
-    "@google-cloud/spanner": "^6.11.0",
-    "@google-cloud/storage": "^6.11.0",
+    "@google-cloud/spanner": "^6.14.0",
+    "@google-cloud/storage": "^6.12.0",
+    "@grpc/grpc-js": "^1.8.20",
     "class-validator": "^0.14.0",
-    "firebase": "^9.22.2",
-    "firebase-admin": "^11.9.0",
-    "globby": "^13.1.4",
-    "google-gax": "^3.6.0",
-    "googleapis": "^118.0.0",
+    "firebase": "^10.1.0",
+    "firebase-admin": "^11.10.1",
+    "globby": "^13.2.2",
+    "googleapis": "^123.0.0",
     "uuid": "^9.0.0"
   },
   "devDependencies": {
-    "@tsconfig/node18": "^2.0.1",
-    "@types/jest": "^29.5.2",
-    "@types/node": "^18.16.16",
+    "@tsconfig/node18": "^18.2.0",
+    "@types/jest": "^29.5.3",
+    "@types/node": "^18.17.1",
     "@types/uuid": "^9.0.2",
-    "@typescript-eslint/eslint-plugin": "^5.59.9",
+    "@typescript-eslint/eslint-plugin": "^6.2.0",
     "copyfiles": "^2.4.1",
-    "eslint": "^8.42.0",
+    "eslint": "^8.45.0",
     "eslint-config-prettier": "^8.8.0",
-    "eslint-plugin-prettier": "^4.2.1",
-    "jest": "^29.5.0",
+    "eslint-plugin-prettier": "^5.0.0",
+    "jest": "^29.6.1",
     "jest-extended": "^4.0.0",
     "rimraf": "^5.0.1",
-    "ts-jest": "^29.1.0",
+    "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
-    "typescript": "^5.1.3"
+    "typescript": "^5.1.6"
   }
 }