@causa/workspace-google 1.3.1 → 1.4.0-beta.1

package/dist/graph/rules/terraform/cloud-run-pubsub-triggers.js

@@ -0,0 +1,103 @@
+import {} from '@causa/workspace-core';
+import { brokerSubscriptionId, brokerTopicId, ModelFact, nodeId, triggerId, triggerLocator, } from '@causa/workspace-core/graph';
+import { terraformDeploys } from '@causa/workspace-terraform';
+import { CloudRunFact } from '../../cloud-run.js';
+import { blockResource } from '../../terraform-modules.js';
+/**
+ * A push `brokerSubscription` per Pub/Sub trigger of a project, for each Cloud Run module block deploying it with
+ * Pub/Sub triggers enabled. The locator is the trigger's. The module names the subscription `run-<service>-<trigger>`.
+ *
+ * Each subscription `realizes` its trigger, `targets` the service its module block deploys, and is delivered to by the
+ * broker topic of its trigger. A topic that no event schema of the workspace defines is created outside of it, and
+ * gets its own `brokerTopic` node.
+ *
+ * @param graph The context of the extraction.
+ * @returns The subscriptions, the external broker topics, and their edges.
+ */
+export async function cloudRunPubSubTriggers(graph) {
+    const { locator } = graph;
+    const [cloudRun, model] = await Promise.all([
+        graph.get(CloudRunFact),
+        graph.get(ModelFact),
+    ]);
+    const nodes = [];
+    const edges = [];
+    const warnings = [];
+    const subscriptions = new Set();
+    for (const service of cloudRun) {
+        const source = service.block.declaration;
+        for (const [name, trigger] of service.pubsubTriggers) {
+            const subscriptionLocator = triggerLocator(service.project.directory, name);
+            const subscriptionId = brokerSubscriptionId(subscriptionLocator);
+            if (subscriptions.has(subscriptionLocator)) {
+                warnings.push({
+                    message: `Trigger '${name}' of '${service.project.name}' is deployed with Pub/Sub triggers by several module blocks. A single subscription node stands for several subscriptions.`,
+                    sources: [source],
+                });
+            }
+            subscriptions.add(subscriptionLocator);
+            const filter = trigger['google.pubSub']?.filter;
+            const subscription = `run-${service.name}-${name}`;
+            const data = {
+                filter: typeof filter === 'string' ? filter : undefined,
+                resource: await blockResource(graph, service.block, {
+                    type: 'pubsub.googleapis.com/Subscription',
+                    scope: service.project,
+                    id: [
+                        'projects/',
+                        service.projectPart,
+                        `/subscriptions/${subscription}`,
+                    ],
+                }, warnings),
+            };
+            nodes.push({
+                layer: 'infrastructure',
+                type: 'brokerSubscription',
+                locator: subscriptionLocator,
+                name: subscription,
+                sources: [
+                    source,
+                    await locator.configurationSource(service.project.context, [
+                        'serviceContainer',
+                        'triggers',
+                        name,
+                    ]),
+                ],
+                data,
+            });
+            edges.push(...terraformDeploys(service.block, nodeId('brokerSubscription', subscriptionLocator)), {
+                type: 'realizes',
+                from: subscriptionId,
+                to: triggerId(service.project.directory, name),
+                sources: [source],
+            }, {
+                type: 'targets',
+                from: subscriptionId,
+                to: service.id,
+                label: trigger.endpoint?.path,
+                sources: [source],
+            });
+            if (typeof trigger.topic !== 'string') {
+                continue;
+            }
+            const topicSource = await locator.configurationSource(service.project.context, ['serviceContainer', 'triggers', name, 'topic']);
+            edges.push({
+                type: 'delivers',
+                from: brokerTopicId(trigger.topic),
+                to: subscriptionId,
+                sources: [source, topicSource],
+            });
+            if (!model.topics.has(trigger.topic)) {
+                nodes.push({
+                    layer: 'infrastructure',
+                    type: 'brokerTopic',
+                    locator: trigger.topic,
+                    name: trigger.topic,
+                    description: 'A topic no event schema of the workspace defines, created outside of it.',
+                    sources: [topicSource],
+                });
+            }
+        }
+    }
+    return { nodes, edges, warnings };
+}

package/dist/graph/rules/terraform/cloud-run-services.d.ts

@@ -0,0 +1,12 @@
+import { type GraphRuleOutput } from '@causa/workspace-core';
+import { type GraphContext } from '@causa/workspace-core/graph';
+/**
+ * A `service` node per block of the Cloud Run module, and a `realizes` edge from it to the project whose configuration
+ * file the block reads. The service name is the block's `name` argument or the project name, and `platform` is the
+ * project's `serviceContainer.platform`. Blocks whose project cannot be found, and arguments that cannot be evaluated,
+ * are reported.
+ *
+ * @param graph The context of the extraction.
+ * @returns The services, and their edges.
+ */
+export declare function cloudRunServices(graph: GraphContext): Promise<GraphRuleOutput>;

package/dist/graph/rules/terraform/cloud-run-services.js

@@ -0,0 +1,61 @@
+import {} from '@causa/workspace-core';
+import { projectId, serviceLocator, } from '@causa/workspace-core/graph';
+import { terraformDeploys } from '@causa/workspace-terraform';
+import { CloudRunFact } from '../../cloud-run.js';
+import { blockResource } from '../../terraform-modules.js';
+/**
+ * A `service` node per block of the Cloud Run module, and a `realizes` edge from it to the project whose configuration
+ * file the block reads. The service name is the block's `name` argument or the project name, and `platform` is the
+ * project's `serviceContainer.platform`. Blocks whose project cannot be found, and arguments that cannot be evaluated,
+ * are reported.
+ *
+ * @param graph The context of the extraction.
+ * @returns The services, and their edges.
+ */
+export async function cloudRunServices(graph) {
+    const { locator } = graph;
+    const cloudRun = await graph.get(CloudRunFact);
+    const nodes = [];
+    const edges = [];
+    const warnings = [];
+    for (const service of cloudRun) {
+        const source = service.block.declaration;
+        const data = {
+            platform: service.project.context
+                .asConfiguration()
+                .get('serviceContainer.platform'),
+            resource: await blockResource(graph, service.block, {
+                type: 'run.googleapis.com/Service',
+                scope: service.project,
+                id: [
+                    'projects/',
+                    service.projectPart,
+                    '/locations/',
+                    service.locationPart,
+                    `/services/${service.name}`,
+                ],
+            }, warnings),
+        };
+        nodes.push({
+            layer: 'infrastructure',
+            type: 'service',
+            locator: serviceLocator(service.project.directory, service.name),
+            name: service.name,
+            sources: [
+                source,
+                await locator.configurationSource(service.project.context, [
+                    'project',
+                    'name',
+                ]),
+            ],
+            data,
+        });
+        edges.push(...terraformDeploys(service.block, service.id), {
+            type: 'realizes',
+            from: service.id,
+            to: projectId(service.project.directory),
+            sources: [source],
+        });
+    }
+    return { nodes, edges, warnings };
+}

package/dist/graph/rules/terraform/cloud-run-tasks-triggers.d.ts

@@ -0,0 +1,14 @@
+import { type GraphRuleOutput } from '@causa/workspace-core';
+import { type GraphContext } from '@causa/workspace-core/graph';
+/**
+ * The resources a Cloud Run module block creates for the task triggers of a project, when tasks triggers are enabled:
+ *
+ * - A `queue` node per `queue` of the task triggers. The module suffixes the queue name with a random string, such that
+ *   only a prefix of the resource name is known. The locator is the declared name. The queue `realizes` the task
+ *   triggers declaring it, and `targets` the service the block deploys, which is the HTTP target of the tasks.
+ * - An `enqueues` edge from every service of the project to each queue.
+ *
+ * @param graph The context of the extraction.
+ * @returns The queues, and their edges.
+ */
+export declare function cloudRunTasksTriggers(graph: GraphContext): Promise<GraphRuleOutput>;

package/dist/graph/rules/terraform/cloud-run-tasks-triggers.js

@@ -0,0 +1,97 @@
+import {} from '@causa/workspace-core';
+import { nodeId, queueId, triggerId, } from '@causa/workspace-core/graph';
+import { terraformDeploys } from '@causa/workspace-terraform';
+import { CloudRunFact } from '../../cloud-run.js';
+import { blockResource } from '../../terraform-modules.js';
+/**
+ * The resources a Cloud Run module block creates for the task triggers of a project, when tasks triggers are enabled:
+ *
+ * - A `queue` node per `queue` of the task triggers. The module suffixes the queue name with a random string, such that
+ *   only a prefix of the resource name is known. The locator is the declared name. The queue `realizes` the task
+ *   triggers declaring it, and `targets` the service the block deploys, which is the HTTP target of the tasks.
+ * - An `enqueues` edge from every service of the project to each queue.
+ *
+ * @param graph The context of the extraction.
+ * @returns The queues, and their edges.
+ */
+export async function cloudRunTasksTriggers(graph) {
+    const { locator } = graph;
+    const cloudRun = await graph.get(CloudRunFact);
+    const nodes = [];
+    const edges = [];
+    const warnings = [];
+    for (const service of cloudRun) {
+        const source = service.block.declaration;
+        for (const [name, trigger] of service.tasksTriggers) {
+            const queueSource = await locator.configurationSource(service.project.context, ['serviceContainer', 'triggers', name, 'queue']);
+            if (typeof trigger.queue !== 'string') {
+                warnings.push({
+                    message: `Task trigger '${name}' has no queue.`,
+                    sources: [queueSource],
+                });
+                continue;
+            }
+            const resource = await blockResource(graph, service.block, {
+                type: 'cloudtasks.googleapis.com/Queue',
+                scope: service.project,
+                prefix: true,
+                id: [
+                    'projects/',
+                    service.projectPart,
+                    '/locations/',
+                    service.locationPart,
+                    `/queues/${trigger.queue}-`,
+                ],
+            }, warnings);
+            nodes.push({
+                layer: 'infrastructure',
+                type: 'queue',
+                locator: trigger.queue,
+                name: trigger.queue,
+                sources: [source, queueSource],
+                data: { resource },
+            });
+            edges.push(...terraformDeploys(service.block, nodeId('queue', trigger.queue)), {
+                type: 'realizes',
+                from: queueId(trigger.queue),
+                to: triggerId(service.project.directory, name),
+                sources: [source],
+            }, {
+                type: 'targets',
+                from: queueId(trigger.queue),
+                to: service.id,
+                label: trigger.endpoint?.path,
+                sources: [source],
+            });
+        }
+    }
+    for (const services of Map.groupBy(cloudRun, (s) => s.project).values()) {
+        for (const granting of services.filter((s) => s.setTasksPermissions)) {
+            const queues = new Map();
+            for (const [name, trigger] of granting.tasksTriggers) {
+                if (typeof trigger.queue === 'string' && !queues.has(trigger.queue)) {
+                    queues.set(trigger.queue, name);
+                }
+            }
+            for (const [queue, trigger] of queues) {
+                for (const service of services) {
+                    edges.push({
+                        type: 'enqueues',
+                        from: service.id,
+                        to: queueId(queue),
+                        sources: [
+                            granting.block.declaration,
+                            await locator.configurationSource(granting.project.context, [
+                                'serviceContainer',
+                                'triggers',
+                                trigger,
+                                'queue',
+                            ]),
+                        ],
+                    });
+                }
+            }
+        }
+    }
+    return { nodes, edges, warnings };
+}

package/dist/graph/rules/terraform/event-topics.d.ts

@@ -0,0 +1,18 @@
+import { type GraphRule } from '@causa/workspace-core';
+/**
+ * The source of the Causa Terraform module creating the Pub/Sub topics, and the raw events BigQuery tables.
+ */
+export declare const EVENT_TOPICS_MODULE_SOURCE = "causa-io/event-topics-pubsub/google";
+/**
+ * The resources created by the event topics module block.
+ *
+ * - A `brokerTopic` node per event topic, named by the topic ID: the module creates one Pub/Sub topic per topic listed
+ *   by `EventTopicList`. It `realizes` the event topic of the same ID.
+ * - When the block has a raw events dataset (`bigquery_raw_events_dataset`, or
+ *   `google.pubSub.bigQueryStorage.rawEventsDatasetId`), a `database` node for the BigQuery dataset, and for each
+ *   topic:
+ *   - A `table` node, named by the topic ID with `.` and `-` replaced by `_`.
+ *   - A BigQuery `brokerSubscription` node: the module creates one `bq-<topic>` subscription per topic, writing events
+ *     to the topic's raw table. The broker topic `delivers` to it, and it `targets` the table.
+ */
+export declare const eventTopicsModule: GraphRule;

package/dist/graph/rules/terraform/event-topics.js

@@ -0,0 +1,193 @@
+import { GraphOriginKind, } from '@causa/workspace-core';
+import { brokerSubscriptionId, brokerTopicId, databaseId, ModelFact, nodeId, tableId, tableLocator, topicId, } from '@causa/workspace-core/graph';
+import { terraformDeploys } from '@causa/workspace-terraform';
+import { relative } from 'path';
+import { rawEventsTableName } from '../../../functions/google-pubsub/index.js';
+import { BIGQUERY_ENGINE, bigQueryDatasetLocator, rawEventsSubscriptionLocator, } from '../../ids.js';
+import { blockResource, infrastructureModules, argumentOrConfiguration, } from '../../terraform-modules.js';
+/**
+ * The source of the Causa Terraform module creating the Pub/Sub topics, and the raw events BigQuery tables.
+ */
+export const EVENT_TOPICS_MODULE_SOURCE = 'causa-io/event-topics-pubsub/google';
+/**
+ * The configuration defining the raw events BigQuery dataset, when the block does not set it.
+ */
+const RAW_EVENTS_DATASET_CONFIGURATION = 'google.pubSub.bigQueryStorage.rawEventsDatasetId';
+/**
+ * Returns the raw events dataset of an event topics module block: its `bigquery_raw_events_dataset` argument, or the
+ * `google.pubSub.bigQueryStorage.rawEventsDatasetId` configuration.
+ *
+ * @param graph The context of the extraction.
+ * @param module The event topics module block.
+ * @param warnings The warnings of the rule, to which the one about a computed argument is added.
+ * @returns The dataset, or `undefined` if raw events are not stored.
+ */
+async function rawEventsDataset(graph, module, warnings) {
+    const part = argumentOrConfiguration(module.block, 'bigquery_raw_events_dataset', RAW_EVENTS_DATASET_CONFIGURATION, warnings);
+    if (typeof part === 'string') {
+        return {
+            dataset: part,
+            part,
+            locator: bigQueryDatasetLocator(part),
+            sources: [],
+        };
+    }
+    const dataset = module.context
+        .asConfiguration()
+        .get(RAW_EVENTS_DATASET_CONFIGURATION);
+    if (typeof dataset !== 'string') {
+        return undefined;
+    }
+    return {
+        dataset,
+        part,
+        locator: bigQueryDatasetLocator(dataset),
+        sources: [
+            await graph.locator.configurationSource(module.context, RAW_EVENTS_DATASET_CONFIGURATION.split('.')),
+        ],
+    };
+}
+/**
+ * The resources created by the event topics module block.
+ *
+ * - A `brokerTopic` node per event topic, named by the topic ID: the module creates one Pub/Sub topic per topic listed
+ *   by `EventTopicList`. It `realizes` the event topic of the same ID.
+ * - When the block has a raw events dataset (`bigquery_raw_events_dataset`, or
+ *   `google.pubSub.bigQueryStorage.rawEventsDatasetId`), a `database` node for the BigQuery dataset, and for each
+ *   topic:
+ *   - A `table` node, named by the topic ID with `.` and `-` replaced by `_`.
+ *   - A BigQuery `brokerSubscription` node: the module creates one `bq-<topic>` subscription per topic, writing events
+ *     to the topic's raw table. The broker topic `delivers` to it, and it `targets` the table.
+ */
+export const eventTopicsModule = {
+    name: 'eventTopicsModule',
+    kind: GraphOriginKind.Declared,
+    description: 'One `brokerTopic` per event topic for the first event topics module block, realizing the topic. When the block has a raw events BigQuery dataset, one `database` node for the dataset, and for each topic one `table` node, named by the topic ID with `.` and `-` replaced by `_`, and one `brokerSubscription` (`rawEvents/<topic>`) delivered to by the broker topic and targeting the table. Each resource is linked by a `deploys` edge from the projects applying the block.',
+    async run(graph) {
+        const { context } = graph;
+        const [eventTopics, model] = await Promise.all([
+            infrastructureModules(graph, EVENT_TOPICS_MODULE_SOURCE),
+            graph.get(ModelFact),
+        ]);
+        const nodes = [];
+        const edges = [];
+        const warnings = [];
+        const [module, ...skipped] = eventTopics;
+        if (!module) {
+            if (model.topics.size > 0) {
+                warnings.push({
+                    message: 'There is no event topics module block: no broker topics are created for the event topics.',
+                });
+            }
+            return { warnings };
+        }
+        warnings.push(...skipped.map((m) => ({
+            message: 'The event topics module block is skipped: only the first one is processed, as several blocks would create resources with the same IDs.',
+            sources: [m.block.declaration],
+        })));
+        const source = module.block.declaration;
+        const projectPart = argumentOrConfiguration(module.block, 'gcp_project_id', 'google.project', warnings);
+        const rawEvents = await rawEventsDataset(graph, module, warnings);
+        if (rawEvents) {
+            const data = {
+                engine: BIGQUERY_ENGINE,
+                resource: await blockResource(graph, module.block, {
+                    type: 'bigquery.googleapis.com/Dataset',
+                    scope: module.project,
+                    id: ['projects/', projectPart, '/datasets/', rawEvents.part],
+                }, warnings),
+            };
+            nodes.push({
+                layer: 'infrastructure',
+                type: 'database',
+                locator: rawEvents.locator,
+                name: rawEvents.dataset,
+                description: 'Raw events from Pub/Sub, one table per topic.',
+                sources: [source, ...rawEvents.sources],
+                data,
+            });
+            edges.push(...terraformDeploys(module.block, nodeId('database', rawEvents.locator)));
+        }
+        for (const topic of model.topics.values()) {
+            const schemaSource = {
+                path: relative(context.rootPath, topic.schemaFilePath),
+            };
+            nodes.push({
+                layer: 'infrastructure',
+                type: 'brokerTopic',
+                locator: topic.id,
+                name: topic.id,
+                sources: [source, schemaSource],
+                data: {
+                    resource: await blockResource(graph, module.block, {
+                        type: 'pubsub.googleapis.com/Topic',
+                        scope: module.project,
+                        id: ['projects/', projectPart, `/topics/${topic.id}`],
+                    }, warnings),
+                },
+            });
+            edges.push(...terraformDeploys(module.block, nodeId('brokerTopic', topic.id)), {
+                type: 'realizes',
+                from: brokerTopicId(topic.id),
+                to: topicId(topic.id),
+                sources: [source],
+            });
+            if (!rawEvents) {
+                continue;
+            }
+            const tableName = rawEventsTableName(topic.id);
+            const rawTableLocator = tableLocator(rawEvents.locator, tableName);
+            nodes.push({
+                layer: 'infrastructure',
+                type: 'table',
+                locator: rawTableLocator,
+                parent: databaseId(rawEvents.locator),
+                name: tableName,
+                description: `Raw \`${topic.id}\` events.`,
+                sources: [source, schemaSource],
+                data: {
+                    resource: await blockResource(graph, module.block, {
+                        type: 'bigquery.googleapis.com/Table',
+                        scope: module.project,
+                        id: [
+                            'projects/',
+                            projectPart,
+                            '/datasets/',
+                            rawEvents.part,
+                            `/tables/${tableName}`,
+                        ],
+                    }, warnings),
+                },
+            });
+            edges.push(...terraformDeploys(module.block, nodeId('table', rawTableLocator)));
+            const subscriptionLocator = rawEventsSubscriptionLocator(topic.id);
+            const subscription = `bq-${topic.id}`;
+            nodes.push({
+                layer: 'infrastructure',
+                type: 'brokerSubscription',
+                locator: subscriptionLocator,
+                name: subscription,
+                sources: [source, ...rawEvents.sources],
+                data: {
+                    resource: await blockResource(graph, module.block, {
+                        type: 'pubsub.googleapis.com/Subscription',
+                        scope: module.project,
+                        id: ['projects/', projectPart, `/subscriptions/${subscription}`],
+                    }, warnings),
+                },
+            });
+            edges.push(...terraformDeploys(module.block, nodeId('brokerSubscription', subscriptionLocator)), {
+                type: 'delivers',
+                from: brokerTopicId(topic.id),
+                to: brokerSubscriptionId(subscriptionLocator),
+                sources: [source],
+            }, {
+                type: 'targets',
+                from: brokerSubscriptionId(subscriptionLocator),
+                to: tableId(rawEvents.locator, tableName),
+                sources: [source],
+            });
+        }
+        return { nodes, edges, warnings };
+    },
+};

package/dist/graph/rules/terraform/index.d.ts

@@ -0,0 +1,5 @@
+import type { GraphRule } from '@causa/workspace-core';
+/**
+ * The Google rules mirroring the Causa Terraform modules, grouped by module.
+ */
+export declare const GOOGLE_TERRAFORM_GRAPH_RULES: readonly GraphRule[];

package/dist/graph/rules/terraform/index.js

@@ -0,0 +1,13 @@
+import { apiRouterModule } from './api-router.js';
+import { cloudRunModule } from './cloud-run-module.js';
+import { eventTopicsModule } from './event-topics.js';
+import { spannerDatabasesModule } from './spanner-databases.js';
+/**
+ * The Google rules mirroring the Causa Terraform modules, grouped by module.
+ */
+export const GOOGLE_TERRAFORM_GRAPH_RULES = [
+    cloudRunModule,
+    apiRouterModule,
+    eventTopicsModule,
+    spannerDatabasesModule,
+];

package/dist/graph/rules/terraform/spanner-databases.d.ts

@@ -0,0 +1,17 @@
+import { type GraphRule } from '@causa/workspace-core';
+/**
+ * The source of the Causa Terraform module creating the Spanner instance and databases.
+ */
+export declare const SPANNER_DATABASES_MODULE_SOURCE = "causa-io/spanner-databases/google";
+/**
+ * The resources created by the blocks of the Spanner databases module:
+ *
+ * - A `google.spanner.instance` node per block, which creates the instance from `google.spanner.instance.name`.
+ * - A `database` node per Spanner database, as `GoogleSpannerListDatabases` lists them. The locator is
+ *   `google.spanner/<instance>.<database>`, and the parent is the instance.
+ * - A `table` node per `CREATE TABLE` statement in the DDL files of each database. Tables no schema declares (e.g. an
+ *   outbox) are still nodes, realizing nothing.
+ * - A `realizes` edge from a table to the entity whose binding names it. The table is the one created by the DDL of a
+ *   database, disambiguated by domain when several databases create a table with the same name.
+ */
+export declare const spannerDatabasesModule: GraphRule;