@causa/workspace-google 1.3.0 → 1.4.0-beta.1

package/dist/graph/rules/terraform/spanner-databases.js

@@ -0,0 +1,180 @@
+import { GraphOriginKind, } from '@causa/workspace-core';
+import { databaseId, tableId, tableLocator, DomainsFact, ModelFact, nodeId, } from '@causa/workspace-core/graph';
+import { terraformDeploys } from '@causa/workspace-terraform';
+import { SPANNER_ENGINE, spannerInstanceId } from '../../ids.js';
+import { ddlSource, resolveSpannerTable, SpannerFact } from '../../spanner.js';
+import { blockResource, infrastructureModules, argumentOrConfiguration, } from '../../terraform-modules.js';
+/**
+ * The source of the Causa Terraform module creating the Spanner instance and databases.
+ */
+export const SPANNER_DATABASES_MODULE_SOURCE = 'causa-io/spanner-databases/google';
+/**
+ * Returns the GCP project and instance parts of the resource names created by a Spanner databases module block.
+ */
+function nameParts(module, warnings) {
+    return {
+        project: argumentOrConfiguration(module.block, 'gcp_project_id', 'google.project', warnings),
+        instance: argumentOrConfiguration(module.block, 'instance_name', 'google.spanner.instance.name', warnings),
+    };
+}
+/**
+ * The resources created by the blocks of the Spanner databases module:
+ *
+ * - A `google.spanner.instance` node per block, which creates the instance from `google.spanner.instance.name`.
+ * - A `database` node per Spanner database, as `GoogleSpannerListDatabases` lists them. The locator is
+ *   `google.spanner/<instance>.<database>`, and the parent is the instance.
+ * - A `table` node per `CREATE TABLE` statement in the DDL files of each database. Tables no schema declares (e.g. an
+ *   outbox) are still nodes, realizing nothing.
+ * - A `realizes` edge from a table to the entity whose binding names it. The table is the one created by the DDL of a
+ *   database, disambiguated by domain when several databases create a table with the same name.
+ */
+export const spannerDatabasesModule = {
+    name: 'spannerDatabasesModule',
+    kind: GraphOriginKind.Declared,
+    description: 'One `google.spanner.instance` node per Spanner databases module block, named by `google.spanner.instance.name`. One `database` node per Spanner database listed by `GoogleSpannerListDatabases` from the DDL files, and one `table` node per `CREATE TABLE` in them, realizing the entity whose `googleSpannerTable` binding names it, disambiguated by domain. Each resource is linked by a `deploys` edge from the projects applying the block.',
+    async run(graph) {
+        const { locator, context } = graph;
+        const [domains, model, spanner, spannerDatabases] = await Promise.all([
+            graph.get(DomainsFact),
+            graph.get(ModelFact),
+            graph.get(SpannerFact),
+            infrastructureModules(graph, SPANNER_DATABASES_MODULE_SOURCE),
+        ]);
+        const nodes = [];
+        const edges = [];
+        const warnings = [];
+        for (const module of spannerDatabases) {
+            const source = module.block.declaration;
+            if (!spanner.instance) {
+                warnings.push({
+                    message: "A Spanner databases module block exists, but 'google.spanner.instance.name' is not set.",
+                    sources: [source],
+                });
+                continue;
+            }
+            const { project, instance } = nameParts(module, warnings);
+            if (module.block.arguments['instance_name'] !== undefined) {
+                warnings.push({
+                    message: `The module overrides the instance name. Locators use the configured '${spanner.instance}'.`,
+                    sources: [source],
+                });
+            }
+            nodes.push({
+                layer: 'infrastructure',
+                type: 'google.spanner.instance',
+                locator: spanner.instance,
+                name: spanner.instance,
+                sources: [
+                    await locator.configurationSource(module.context, [
+                        'google',
+                        'spanner',
+                        'instance',
+                        'name',
+                    ]),
+                    source,
+                ],
+                data: {
+                    resource: await blockResource(graph, module.block, {
+                        type: 'spanner.googleapis.com/Instance',
+                        scope: module.project,
+                        id: ['projects/', project, '/instances/', instance],
+                    }, warnings),
+                },
+            });
+            edges.push(...terraformDeploys(module.block, nodeId('google.spanner.instance', spanner.instance)));
+            for (const database of spanner.databases) {
+                const data = {
+                    engine: SPANNER_ENGINE,
+                    resource: await blockResource(graph, module.block, {
+                        type: 'spanner.googleapis.com/Database',
+                        scope: module.project,
+                        id: [
+                            'projects/',
+                            project,
+                            '/instances/',
+                            instance,
+                            `/databases/${database.id}`,
+                        ],
+                    }, warnings),
+                };
+                nodes.push({
+                    layer: 'infrastructure',
+                    type: 'database',
+                    locator: database.locator,
+                    parent: spannerInstanceId(spanner.instance),
+                    name: database.id,
+                    sources: [
+                        { path: database.directory },
+                        await locator.configurationSource(context, [
+                            'google',
+                            'spanner',
+                            'ddls',
+                        ]),
+                        source,
+                    ],
+                    data,
+                });
+                edges.push(...terraformDeploys(module.block, nodeId('database', database.locator)));
+            }
+        }
+        if (spanner.databases.length > 0 && spannerDatabases.length === 0) {
+            warnings.push({
+                message: 'Spanner DDL files exist but no Spanner databases module block creates the databases.',
+            });
+        }
+        if (!spanner.instance || spannerDatabases.length === 0) {
+            return { nodes, edges, warnings };
+        }
+        const blocks = spannerDatabases.map((m) => m.block.declaration);
+        for (const database of spanner.databases) {
+            for (const table of database.tables) {
+                const nodeLocator = tableLocator(database.locator, table.name);
+                nodes.push({
+                    layer: 'infrastructure',
+                    type: 'table',
+                    locator: nodeLocator,
+                    parent: databaseId(database.locator),
+                    name: table.name,
+                    sources: [ddlSource(table), ...blocks],
+                });
+                edges.push(...spannerDatabases.flatMap((module) => terraformDeploys(module.block, nodeId('table', nodeLocator))));
+            }
+        }
+        for (const entity of model.entities.values()) {
+            const resolution = resolveSpannerTable(spanner, entity, domains);
+            if (!resolution) {
+                continue;
+            }
+            const { file, segments, definition } = entity.schema;
+            const binding = await locator.source(file, [
+                ...segments,
+                'causa',
+                'googleSpannerTable',
+            ]);
+            const table = definition.databases.find((d) => d.engine === SPANNER_ENGINE)?.table;
+            if (resolution.kind === 'missing') {
+                warnings.push({
+                    message: `Entity '${definition.name}' binds to Spanner table '${table}', which no DDL creates.`,
+                    sources: [binding],
+                });
+                continue;
+            }
+            if (resolution.kind === 'ambiguous') {
+                warnings.push({
+                    message: `Entity '${definition.name}' binds to Spanner table '${table}', which is created in several databases: ${resolution.candidates
+                        .map((c) => `'${c.id}'`)
+                        .join(', ')}.`,
+                    sources: [binding],
+                });
+                continue;
+            }
+            edges.push({
+                type: 'realizes',
+                from: tableId(resolution.database.locator, resolution.table.name),
+                to: entity.id,
+                sources: [binding, ddlSource(resolution.table)],
+            });
+        }
+        return { nodes, edges, warnings };
+    },
+};

package/dist/graph/spanner.d.ts

@@ -0,0 +1,106 @@
+import type { GraphOriginSource } from '@causa/workspace-core';
+import { GraphFact, type EntityFacts, type GraphContext, type GraphFactOutput, type WorkspaceDomain } from '@causa/workspace-core/graph';
+/**
+ * A table created by a `CREATE TABLE` statement in a Spanner DDL file.
+ */
+export type SpannerTable = {
+    /**
+     * The name of the table.
+     */
+    readonly name: string;
+    /**
+     * The DDL file declaring the table, relative to the workspace root.
+     */
+    readonly file: string;
+    /**
+     * The line of the statement in the file.
+     */
+    readonly line: number;
+};
+/**
+ * A Spanner database of the workspace, and the tables its DDL files create.
+ */
+export type SpannerDatabaseFacts = {
+    /**
+     * The ID of the database, as `GoogleSpannerListDatabases` returns it.
+     */
+    readonly id: string;
+    /**
+     * The name of the database within its engine, `<instance>.<database>`, as `outputs['google.spanner']` lists it.
+     */
+    readonly name: string;
+    /**
+     * The locator of the `database` node.
+     */
+    readonly locator: string;
+    /**
+     * The directory holding the DDL files, relative to the workspace root.
+     */
+    readonly directory: string;
+    /**
+     * The domain the DDL files belong to.
+     */
+    readonly domain: WorkspaceDomain | undefined;
+    /**
+     * The tables created by the DDL files.
+     */
+    readonly tables: SpannerTable[];
+};
+/**
+ * The Spanner databases of the workspace.
+ */
+export type SpannerFacts = {
+    /**
+     * The Spanner instance, from `google.spanner.instance.name`.
+     */
+    readonly instance: string | undefined;
+    /**
+     * The databases, as `GoogleSpannerListDatabases` lists them.
+     */
+    readonly databases: SpannerDatabaseFacts[];
+};
+/**
+ * The Spanner databases of the workspace, as `GoogleSpannerListDatabases` lists them, and the tables their DDL files
+ * create.
+ */
+export declare class SpannerFact extends GraphFact<SpannerFacts> {
+    compute(graph: GraphContext): Promise<GraphFactOutput<SpannerFacts>>;
+    /**
+     * Parses the tables created by the `CREATE TABLE` statements of DDL files.
+     *
+     * @param context The workspace context.
+     * @param files The DDL files, relative to the workspace root.
+     * @returns The tables, in the order of the files and statements.
+     */
+    private parseTables;
+}
+/**
+ * The resolution of an entity's Spanner binding to a table.
+ */
+export type SpannerTableResolution = {
+    readonly kind: 'resolved';
+    readonly database: SpannerDatabaseFacts;
+    readonly table: SpannerTable;
+} | {
+    readonly kind: 'ambiguous';
+    readonly candidates: SpannerDatabaseFacts[];
+} | {
+    readonly kind: 'missing';
+};
+/**
+ * Resolves the Spanner binding of an entity to the database whose DDL creates a table of that name. When several
+ * databases declare the name, the one in the entity's domain is chosen.
+ *
+ * @param spanner The Spanner databases.
+ * @param entity The entity.
+ * @param domains The domains of the workspace, in which the domain of the entity is looked up.
+ * @returns The resolution, or `undefined` if the entity is not persisted in Spanner.
+ */
+export declare function resolveSpannerTable(spanner: SpannerFacts, entity: EntityFacts, domains: readonly WorkspaceDomain[]): SpannerTableResolution | undefined;
+/**
+ * Returns the origin source of a `CREATE TABLE` statement.
+ *
+ * @param table The table.
+ * @returns The origin source.
+ */
+export declare function ddlSource(table: SpannerTable): GraphOriginSource;

package/dist/graph/spanner.js

@@ -0,0 +1,118 @@
+import { domainAt, domainOfFile, DomainsFact, GraphFact, } from '@causa/workspace-core/graph';
+import { readFile } from 'fs/promises';
+import { dirname, join, relative } from 'path';
+import { GoogleSpannerListDatabases } from '../functions/google-spanner/index.js';
+import { SPANNER_ENGINE, spannerDatabaseLocator, spannerDatabaseName, } from './ids.js';
+/**
+ * The Spanner databases of the workspace, as `GoogleSpannerListDatabases` lists them, and the tables their DDL files
+ * create.
+ */
+export class SpannerFact extends GraphFact {
+    async compute(graph) {
+        const { context } = graph;
+        const instance = context
+            .asConfiguration()
+            .get('google.spanner.instance.name');
+        const [listed, domains] = await Promise.all([
+            context.call(GoogleSpannerListDatabases, {}),
+            graph.get(DomainsFact),
+        ]);
+        const databases = await Promise.all(listed.map(async ({ id, ddlFiles }) => {
+            const files = ddlFiles.map((f) => relative(context.rootPath, f)).sort();
+            const tables = await this.parseTables(context, files);
+            const directory = files[0] ? dirname(files[0]) : '';
+            return {
+                id,
+                name: spannerDatabaseName(instance ?? '', id),
+                locator: spannerDatabaseLocator(instance ?? '', id),
+                directory,
+                domain: directory ? domainAt(domains, directory) : undefined,
+                tables,
+            };
+        }));
+        databases.sort((a, b) => a.id.localeCompare(b.id));
+        const warnings = [];
+        for (const database of databases) {
+            const names = new Set();
+            for (const table of database.tables) {
+                if (names.has(table.name)) {
+                    warnings.push({
+                        message: `Table '${table.name}' is created twice in database '${database.id}'.`,
+                        sources: [ddlSource(table)],
+                    });
+                }
+                names.add(table.name);
+            }
+        }
+        return { value: { instance, databases }, warnings };
+    }
+    /**
+     * Parses the tables created by the `CREATE TABLE` statements of DDL files.
+     *
+     * @param context The workspace context.
+     * @param files The DDL files, relative to the workspace root.
+     * @returns The tables, in the order of the files and statements.
+     */
+    async parseTables(context, files) {
+        const tables = [];
+        for (const file of files) {
+            const content = await readFile(join(context.rootPath, file), 'utf-8');
+            const withoutComments = content.replace(/--.*$/gm, (m) => ' '.repeat(m.length));
+            const statements = withoutComments.matchAll(/\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[`"']?([A-Za-z_]\w*)[`"']?/gi);
+            for (const match of statements) {
+                tables.push({
+                    name: match[1],
+                    file,
+                    line: withoutComments.slice(0, match.index).split('\n').length,
+                });
+            }
+        }
+        return tables;
+    }
+}
+/**
+ * Resolves the Spanner binding of an entity to the database whose DDL creates a table of that name. When several
+ * databases declare the name, the one in the entity's domain is chosen.
+ *
+ * @param spanner The Spanner databases.
+ * @param entity The entity.
+ * @param domains The domains of the workspace, in which the domain of the entity is looked up.
+ * @returns The resolution, or `undefined` if the entity is not persisted in Spanner.
+ */
+export function resolveSpannerTable(spanner, entity, domains) {
+    const { definition, file } = entity.schema;
+    const table = definition.databases.find((d) => d.engine === SPANNER_ENGINE)?.table;
+    if (!table) {
+        return undefined;
+    }
+    const candidates = spanner.databases.filter((d) => d.tables.some((t) => t.name === table));
+    const pick = (database) => ({
+        kind: 'resolved',
+        database,
+        table: database.tables.find((t) => t.name === table),
+    });
+    if (candidates.length === 1) {
+        return pick(candidates[0]);
+    }
+    if (candidates.length === 0) {
+        return { kind: 'missing' };
+    }
+    const domain = domainOfFile(domains, file);
+    const sameDomain = candidates.filter((d) => domain && d.domain?.directory === domain.directory);
+    return sameDomain.length === 1
+        ? pick(sameDomain[0])
+        : { kind: 'ambiguous', candidates };
+}
+/**
+ * Returns the origin source of a `CREATE TABLE` statement.
+ *
+ * @param table The table.
+ * @returns The origin source.
+ */
+export function ddlSource(table) {
+    return {
+        path: table.file,
+        pointer: `CREATE TABLE ${table.name}`,
+        location: { start: { line: table.line } },
+    };
+}

package/dist/graph/terraform-modules.d.ts

@@ -0,0 +1,87 @@
+import type { WorkspaceContext } from '@causa/workspace';
+import { type GraphContext, type GraphResource, type GraphTemplatePart, type GraphWarning, type WorkspaceProject } from '@causa/workspace-core/graph';
+import { type TerraformArgument, type TerraformBlock, type TerraformModuleBlock } from '@causa/workspace-terraform';
+/**
+ * Finds the project whose configuration file a block reads. The argument is a path ending with `<project name>.json`,
+ * the file written by `ProjectWriteConfigurations`.
+ *
+ * @param argument The argument of the module block referencing the configuration file.
+ * @param projects The projects of the workspace.
+ * @returns The project, or `undefined` if the argument does not name a project of the workspace.
+ */
+export declare function projectOfConfigurationFile(argument: TerraformArgument | undefined, projects: readonly WorkspaceProject[]): WorkspaceProject | undefined;
+/**
+ * A block of a workspace-level module, which reads the configuration of an infrastructure project
+ * (`infrastructure_configuration_file`).
+ */
+export type InfrastructureModule = {
+    /**
+     * The module block.
+     */
+    readonly block: TerraformModuleBlock;
+    /**
+     * The infrastructure project whose configuration the module reads.
+     */
+    readonly project: WorkspaceProject | undefined;
+    /**
+     * The context whose configuration the module reads: the project's if it is found, otherwise the workspace's.
+     */
+    readonly context: WorkspaceContext;
+};
+/**
+ * Returns the blocks of a workspace-level module, along with the context whose configuration each reads. The workspace
+ * context is used when the infrastructure project cannot be found.
+ *
+ * @param graph The context of the extraction.
+ * @param source The source of the module, e.g. `causa-io/event-topics-pubsub/google`.
+ * @returns The blocks of the module.
+ */
+export declare function infrastructureModules(graph: GraphContext, source: string): Promise<InfrastructureModule[]>;
+/**
+ * Returns a string argument of a module block, or a reference to the configuration the module falls back to, mirroring
+ * the `coalesce(var.<argument>, <configuration>)` logic of the Causa Terraform modules. When the block passes the
+ * argument but it cannot be evaluated as a string, the configuration is assumed, and a warning is raised.
+ *
+ * @param block The module block.
+ * @param argument The name of the module variable.
+ * @param configuration The path to the configuration value the module falls back to.
+ * @param warnings The warnings to which the one about a computed argument is added.
+ * @returns The literal value of the argument, or a reference to the configuration.
+ */
+export declare function argumentOrConfiguration(block: TerraformModuleBlock, argument: string, configuration: string, warnings: GraphWarning[]): GraphTemplatePart;
+/**
+ * The infrastructure resource created by a Terraform block, before its identifier is built.
+ */
+export type BlockResource = {
+    /**
+     * The type of the resource, e.g. `pubsub.googleapis.com/Topic`.
+     */
+    readonly type: string;
+    /**
+     * The project whose configuration the block reads, if any. Otherwise, the workspace configuration is used.
+     */
+    readonly scope?: WorkspaceProject;
+    /**
+     * The parts of the identifier of the resource.
+     */
+    readonly id: GraphTemplatePart[];
+    /**
+     * Whether {@link BlockResource.id} is only a prefix of the identifier, when the full identifier cannot be known from
+     * the workspace.
+     */
+    readonly prefix?: boolean;
+};
+/**
+ * Returns the infrastructure resource a Terraform block creates.
+ * A block applied by the environment project creates the resource once per environment: its identifier references
+ * the configuration, to be rendered for a given environment, using the configuration of the scope. Other
+ * infrastructure projects do not depend on the environment: the identifier is rendered, and the resource has no
+ * scope. If it cannot be rendered, a warning is raised and the identifier references the configuration.
+ *
+ * @param graph The context of the extraction.
+ * @param block The block creating the resource.
+ * @param resource The resource.
+ * @param warnings The warnings to which the one about an identifier that cannot be rendered is added.
+ * @returns The {@link GraphResource}.
+ */
+export declare function blockResource(graph: GraphContext, block: TerraformBlock, resource: BlockResource, warnings: GraphWarning[]): Promise<GraphResource>;

package/dist/graph/terraform-modules.js

@@ -0,0 +1,107 @@
+import { graphTemplate, projectId, ProjectsFact, renderGraphTemplate, } from '@causa/workspace-core/graph';
+import { modulesWithSource, TerraformProjectsFact, } from '@causa/workspace-terraform';
+import { basename } from 'path';
+/**
+ * Finds the project whose configuration file a block reads. The argument is a path ending with `<project name>.json`,
+ * the file written by `ProjectWriteConfigurations`.
+ *
+ * @param argument The argument of the module block referencing the configuration file.
+ * @param projects The projects of the workspace.
+ * @returns The project, or `undefined` if the argument does not name a project of the workspace.
+ */
+export function projectOfConfigurationFile(argument, projects) {
+    if (argument === undefined) {
+        return undefined;
+    }
+    const path = 'value' in argument ? argument.value : argument.expression;
+    if (typeof path !== 'string') {
+        return undefined;
+    }
+    const file = basename(path);
+    if (!/^[^${}]+\.json$/.test(file)) {
+        return undefined;
+    }
+    const name = file.slice(0, -'.json'.length);
+    return projects.find((p) => p.name === name);
+}
+/**
+ * Returns the blocks of a workspace-level module, along with the context whose configuration each reads. The workspace
+ * context is used when the infrastructure project cannot be found.
+ *
+ * @param graph The context of the extraction.
+ * @param source The source of the module, e.g. `causa-io/event-topics-pubsub/google`.
+ * @returns The blocks of the module.
+ */
+export async function infrastructureModules(graph, source) {
+    const [terraformProjects, projects] = await Promise.all([
+        graph.get(TerraformProjectsFact),
+        graph.get(ProjectsFact),
+    ]);
+    return modulesWithSource(terraformProjects, source).map((block) => {
+        const project = projectOfConfigurationFile(block.arguments['infrastructure_configuration_file'], projects);
+        return { block, project, context: project?.context ?? graph.context };
+    });
+}
+/**
+ * Returns a string argument of a module block, or a reference to the configuration the module falls back to, mirroring
+ * the `coalesce(var.<argument>, <configuration>)` logic of the Causa Terraform modules. When the block passes the
+ * argument but it cannot be evaluated as a string, the configuration is assumed, and a warning is raised.
+ *
+ * @param block The module block.
+ * @param argument The name of the module variable.
+ * @param configuration The path to the configuration value the module falls back to.
+ * @param warnings The warnings to which the one about a computed argument is added.
+ * @returns The literal value of the argument, or a reference to the configuration.
+ */
+export function argumentOrConfiguration(block, argument, configuration, warnings) {
+    const value = block.arguments[argument];
+    if (value === undefined ||
+        ('value' in value && (value.value === null || value.value === ''))) {
+        return { configuration };
+    }
+    if ('value' in value && typeof value.value === 'string') {
+        return value.value;
+    }
+    const expression = 'expression' in value ? value.expression : JSON.stringify(value.value);
+    warnings.push({
+        message: `The argument '${argument}' is computed (${expression}). Assuming the '${configuration}' configuration.`,
+        sources: [block.declaration],
+    });
+    return { configuration };
+}
+/**
+ * Returns the infrastructure resource a Terraform block creates.
+ * A block applied by the environment project creates the resource once per environment: its identifier references
+ * the configuration, to be rendered for a given environment, using the configuration of the scope. Other
+ * infrastructure projects do not depend on the environment: the identifier is rendered, and the resource has no
+ * scope. If it cannot be rendered, a warning is raised and the identifier references the configuration.
+ *
+ * @param graph The context of the extraction.
+ * @param block The block creating the resource.
+ * @param resource The resource.
+ * @param warnings The warnings to which the one about an identifier that cannot be rendered is added.
+ * @returns The {@link GraphResource}.
+ */
+export async function blockResource(graph, block, resource, warnings) {
+    const { type, scope, id, prefix } = resource;
+    const build = (id, scope) => ({
+        type,
+        ...(prefix ? { idPrefix: id } : { id }),
+        ...(scope ? { scope } : {}),
+    });
+    const templated = build(graphTemplate(...id), scope && projectId(scope.directory));
+    if (block.module.project.environment) {
+        return templated;
+    }
+    try {
+        const context = scope?.context ?? graph.context;
+        return build(await renderGraphTemplate(context, ...id));
+    }
+    catch (error) {
+        warnings.push({
+            message: `The identifier of the '${type}' resource cannot be rendered, although the block is not applied by the environment project: ${error.message}`,
+            sources: [block.declaration],
+        });
+        return templated;
+    }
+}