@causa/workspace-google 1.3.1 → 1.4.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/functions/event-topic/create-backfill-source-bigquery.call.js +2 -2
- package/dist/functions/event-topic/query-events-bigquery.call.js +2 -2
- package/dist/functions/google-app-check/generate-token.js +3 -1
- package/dist/functions/google-firebase-storage/merge-rules.js +3 -1
- package/dist/functions/google-firestore/generate-typescript-decorators.js +2 -1
- package/dist/functions/google-firestore/merge-rules.js +3 -1
- package/dist/functions/google-identity-platform/generate-custom-token.js +7 -2
- package/dist/functions/google-pubsub/index.d.ts +1 -1
- package/dist/functions/google-pubsub/index.js +1 -1
- package/dist/functions/google-pubsub/write-topics.d.ts +7 -0
- package/dist/functions/google-pubsub/write-topics.js +17 -3
- package/dist/functions/google-services/enable.js +3 -1
- package/dist/functions/google-spanner/generate-typescript-decorators.js +2 -1
- package/dist/functions/google-spanner/list-databases.d.ts +1 -1
- package/dist/functions/google-spanner/write-databases.js +7 -2
- package/dist/functions/graph/fetch-environment-metric-series.call.d.ts +3 -0
- package/dist/functions/graph/fetch-environment-metric-series.call.js +107 -0
- package/dist/functions/graph/fetch-environment-metric-series.d.ts +23 -0
- package/dist/functions/graph/fetch-environment-metric-series.js +40 -0
- package/dist/functions/graph/get-environment-provider.d.ts +11 -0
- package/dist/functions/graph/get-environment-provider.js +20 -0
- package/dist/functions/graph/index.d.ts +3 -0
- package/dist/functions/graph/index.js +3 -0
- package/dist/functions/graph/list-rules.d.ts +11 -0
- package/dist/functions/graph/list-rules.js +16 -0
- package/dist/functions/index.js +2 -1
- package/dist/graph/cloud-run.d.ts +98 -0
- package/dist/graph/cloud-run.js +102 -0
- package/dist/graph/environment/fetchers/cloud-monitoring-alerts.d.ts +6 -0
- package/dist/graph/environment/fetchers/cloud-monitoring-alerts.js +106 -0
- package/dist/graph/environment/fetchers/cloud-monitoring.d.ts +5 -0
- package/dist/graph/environment/fetchers/cloud-monitoring.js +13 -0
- package/dist/graph/environment/fetchers/cloud-run.d.ts +5 -0
- package/dist/graph/environment/fetchers/cloud-run.js +88 -0
- package/dist/graph/environment/fetchers/cloud-scheduler.d.ts +5 -0
- package/dist/graph/environment/fetchers/cloud-scheduler.js +95 -0
- package/dist/graph/environment/fetchers/console-links.d.ts +14 -0
- package/dist/graph/environment/fetchers/console-links.js +132 -0
- package/dist/graph/environment/fetchers/error-reporting.d.ts +5 -0
- package/dist/graph/environment/fetchers/error-reporting.js +98 -0
- package/dist/graph/environment/fetchers/index.d.ts +5 -0
- package/dist/graph/environment/fetchers/index.js +17 -0
- package/dist/graph/environment/index.d.ts +9 -0
- package/dist/graph/environment/index.js +9 -0
- package/dist/graph/environment/load-balancer-backends.d.ts +82 -0
- package/dist/graph/environment/load-balancer-backends.js +126 -0
- package/dist/graph/environment/monitoring-alerts.d.ts +60 -0
- package/dist/graph/environment/monitoring-alerts.js +67 -0
- package/dist/graph/environment/monitoring-catalog.d.ts +95 -0
- package/dist/graph/environment/monitoring-catalog.js +356 -0
- package/dist/graph/environment/monitoring-mappings.d.ts +133 -0
- package/dist/graph/environment/monitoring-mappings.js +97 -0
- package/dist/graph/environment/monitoring-query.d.ts +116 -0
- package/dist/graph/environment/monitoring-query.js +314 -0
- package/dist/graph/environment/monitoring-values.d.ts +41 -0
- package/dist/graph/environment/monitoring-values.js +119 -0
- package/dist/graph/environment/resource-resolvers.d.ts +7 -0
- package/dist/graph/environment/resource-resolvers.js +57 -0
- package/dist/graph/environment/resources.d.ts +45 -0
- package/dist/graph/environment/resources.js +49 -0
- package/dist/graph/ids.d.ts +35 -0
- package/dist/graph/ids.js +44 -0
- package/dist/graph/index.d.ts +7 -0
- package/dist/graph/index.js +7 -0
- package/dist/graph/rules/configuration/enqueues-assumed-from-google-task-trigger.d.ts +6 -0
- package/dist/graph/rules/configuration/enqueues-assumed-from-google-task-trigger.js +39 -0
- package/dist/graph/rules/configuration/firestore.d.ts +11 -0
- package/dist/graph/rules/configuration/firestore.js +93 -0
- package/dist/graph/rules/configuration/index.d.ts +5 -0
- package/dist/graph/rules/configuration/index.js +11 -0
- package/dist/graph/rules/configuration/service-container.d.ts +13 -0
- package/dist/graph/rules/configuration/service-container.js +123 -0
- package/dist/graph/rules/terraform/api-router.d.ts +15 -0
- package/dist/graph/rules/terraform/api-router.js +117 -0
- package/dist/graph/rules/terraform/cloud-run-cron-triggers.d.ts +11 -0
- package/dist/graph/rules/terraform/cloud-run-cron-triggers.js +67 -0
- package/dist/graph/rules/terraform/cloud-run-grants.d.ts +16 -0
- package/dist/graph/rules/terraform/cloud-run-grants.js +118 -0
- package/dist/graph/rules/terraform/cloud-run-module.d.ts +6 -0
- package/dist/graph/rules/terraform/cloud-run-module.js +29 -0
- package/dist/graph/rules/terraform/cloud-run-pubsub-triggers.d.ts +14 -0
- package/dist/graph/rules/terraform/cloud-run-pubsub-triggers.js +103 -0
- package/dist/graph/rules/terraform/cloud-run-services.d.ts +12 -0
- package/dist/graph/rules/terraform/cloud-run-services.js +61 -0
- package/dist/graph/rules/terraform/cloud-run-tasks-triggers.d.ts +14 -0
- package/dist/graph/rules/terraform/cloud-run-tasks-triggers.js +97 -0
- package/dist/graph/rules/terraform/event-topics.d.ts +18 -0
- package/dist/graph/rules/terraform/event-topics.js +193 -0
- package/dist/graph/rules/terraform/index.d.ts +5 -0
- package/dist/graph/rules/terraform/index.js +13 -0
- package/dist/graph/rules/terraform/spanner-databases.d.ts +17 -0
- package/dist/graph/rules/terraform/spanner-databases.js +180 -0
- package/dist/graph/spanner.d.ts +106 -0
- package/dist/graph/spanner.js +118 -0
- package/dist/graph/terraform-modules.d.ts +87 -0
- package/dist/graph/terraform-modules.js +107 -0
- package/package.json +30 -29
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { GraphOriginKind, } from '@causa/workspace-core';
|
|
2
|
+
import { brokerSubscriptionId, brokerTopicId, databaseId, ModelFact, nodeId, tableId, tableLocator, topicId, } from '@causa/workspace-core/graph';
|
|
3
|
+
import { terraformDeploys } from '@causa/workspace-terraform';
|
|
4
|
+
import { relative } from 'path';
|
|
5
|
+
import { rawEventsTableName } from '../../../functions/google-pubsub/index.js';
|
|
6
|
+
import { BIGQUERY_ENGINE, bigQueryDatasetLocator, rawEventsSubscriptionLocator, } from '../../ids.js';
|
|
7
|
+
import { blockResource, infrastructureModules, argumentOrConfiguration, } from '../../terraform-modules.js';
|
|
8
|
+
/**
|
|
9
|
+
* The source of the Causa Terraform module creating the Pub/Sub topics, and the raw events BigQuery tables.
|
|
10
|
+
*/
|
|
11
|
+
export const EVENT_TOPICS_MODULE_SOURCE = 'causa-io/event-topics-pubsub/google';
|
|
12
|
+
/**
|
|
13
|
+
* The configuration defining the raw events BigQuery dataset, when the block does not set it.
|
|
14
|
+
*/
|
|
15
|
+
const RAW_EVENTS_DATASET_CONFIGURATION = 'google.pubSub.bigQueryStorage.rawEventsDatasetId';
|
|
16
|
+
/**
|
|
17
|
+
* Returns the raw events dataset of an event topics module block: its `bigquery_raw_events_dataset` argument, or the
|
|
18
|
+
* `google.pubSub.bigQueryStorage.rawEventsDatasetId` configuration.
|
|
19
|
+
*
|
|
20
|
+
* @param graph The context of the extraction.
|
|
21
|
+
* @param module The event topics module block.
|
|
22
|
+
* @param warnings The warnings of the rule, to which the one about a computed argument is added.
|
|
23
|
+
* @returns The dataset, or `undefined` if raw events are not stored.
|
|
24
|
+
*/
|
|
25
|
+
async function rawEventsDataset(graph, module, warnings) {
|
|
26
|
+
const part = argumentOrConfiguration(module.block, 'bigquery_raw_events_dataset', RAW_EVENTS_DATASET_CONFIGURATION, warnings);
|
|
27
|
+
if (typeof part === 'string') {
|
|
28
|
+
return {
|
|
29
|
+
dataset: part,
|
|
30
|
+
part,
|
|
31
|
+
locator: bigQueryDatasetLocator(part),
|
|
32
|
+
sources: [],
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
const dataset = module.context
|
|
36
|
+
.asConfiguration()
|
|
37
|
+
.get(RAW_EVENTS_DATASET_CONFIGURATION);
|
|
38
|
+
if (typeof dataset !== 'string') {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
return {
|
|
42
|
+
dataset,
|
|
43
|
+
part,
|
|
44
|
+
locator: bigQueryDatasetLocator(dataset),
|
|
45
|
+
sources: [
|
|
46
|
+
await graph.locator.configurationSource(module.context, RAW_EVENTS_DATASET_CONFIGURATION.split('.')),
|
|
47
|
+
],
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The resources created by the event topics module block.
|
|
52
|
+
*
|
|
53
|
+
* - A `brokerTopic` node per event topic, named by the topic ID: the module creates one Pub/Sub topic per topic listed
|
|
54
|
+
* by `EventTopicList`. It `realizes` the event topic of the same ID.
|
|
55
|
+
* - When the block has a raw events dataset (`bigquery_raw_events_dataset`, or
|
|
56
|
+
* `google.pubSub.bigQueryStorage.rawEventsDatasetId`), a `database` node for the BigQuery dataset, and for each
|
|
57
|
+
* topic:
|
|
58
|
+
* - A `table` node, named by the topic ID with `.` and `-` replaced by `_`.
|
|
59
|
+
* - A BigQuery `brokerSubscription` node: the module creates one `bq-<topic>` subscription per topic, writing events
|
|
60
|
+
* to the topic's raw table. The broker topic `delivers` to it, and it `targets` the table.
|
|
61
|
+
*/
|
|
62
|
+
export const eventTopicsModule = {
|
|
63
|
+
name: 'eventTopicsModule',
|
|
64
|
+
kind: GraphOriginKind.Declared,
|
|
65
|
+
description: 'One `brokerTopic` per event topic for the first event topics module block, realizing the topic. When the block has a raw events BigQuery dataset, one `database` node for the dataset, and for each topic one `table` node, named by the topic ID with `.` and `-` replaced by `_`, and one `brokerSubscription` (`rawEvents/<topic>`) delivered to by the broker topic and targeting the table. Each resource is linked by a `deploys` edge from the projects applying the block.',
|
|
66
|
+
async run(graph) {
|
|
67
|
+
const { context } = graph;
|
|
68
|
+
const [eventTopics, model] = await Promise.all([
|
|
69
|
+
infrastructureModules(graph, EVENT_TOPICS_MODULE_SOURCE),
|
|
70
|
+
graph.get(ModelFact),
|
|
71
|
+
]);
|
|
72
|
+
const nodes = [];
|
|
73
|
+
const edges = [];
|
|
74
|
+
const warnings = [];
|
|
75
|
+
const [module, ...skipped] = eventTopics;
|
|
76
|
+
if (!module) {
|
|
77
|
+
if (model.topics.size > 0) {
|
|
78
|
+
warnings.push({
|
|
79
|
+
message: 'There is no event topics module block: no broker topics are created for the event topics.',
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
return { warnings };
|
|
83
|
+
}
|
|
84
|
+
warnings.push(...skipped.map((m) => ({
|
|
85
|
+
message: 'The event topics module block is skipped: only the first one is processed, as several blocks would create resources with the same IDs.',
|
|
86
|
+
sources: [m.block.declaration],
|
|
87
|
+
})));
|
|
88
|
+
const source = module.block.declaration;
|
|
89
|
+
const projectPart = argumentOrConfiguration(module.block, 'gcp_project_id', 'google.project', warnings);
|
|
90
|
+
const rawEvents = await rawEventsDataset(graph, module, warnings);
|
|
91
|
+
if (rawEvents) {
|
|
92
|
+
const data = {
|
|
93
|
+
engine: BIGQUERY_ENGINE,
|
|
94
|
+
resource: await blockResource(graph, module.block, {
|
|
95
|
+
type: 'bigquery.googleapis.com/Dataset',
|
|
96
|
+
scope: module.project,
|
|
97
|
+
id: ['projects/', projectPart, '/datasets/', rawEvents.part],
|
|
98
|
+
}, warnings),
|
|
99
|
+
};
|
|
100
|
+
nodes.push({
|
|
101
|
+
layer: 'infrastructure',
|
|
102
|
+
type: 'database',
|
|
103
|
+
locator: rawEvents.locator,
|
|
104
|
+
name: rawEvents.dataset,
|
|
105
|
+
description: 'Raw events from Pub/Sub, one table per topic.',
|
|
106
|
+
sources: [source, ...rawEvents.sources],
|
|
107
|
+
data,
|
|
108
|
+
});
|
|
109
|
+
edges.push(...terraformDeploys(module.block, nodeId('database', rawEvents.locator)));
|
|
110
|
+
}
|
|
111
|
+
for (const topic of model.topics.values()) {
|
|
112
|
+
const schemaSource = {
|
|
113
|
+
path: relative(context.rootPath, topic.schemaFilePath),
|
|
114
|
+
};
|
|
115
|
+
nodes.push({
|
|
116
|
+
layer: 'infrastructure',
|
|
117
|
+
type: 'brokerTopic',
|
|
118
|
+
locator: topic.id,
|
|
119
|
+
name: topic.id,
|
|
120
|
+
sources: [source, schemaSource],
|
|
121
|
+
data: {
|
|
122
|
+
resource: await blockResource(graph, module.block, {
|
|
123
|
+
type: 'pubsub.googleapis.com/Topic',
|
|
124
|
+
scope: module.project,
|
|
125
|
+
id: ['projects/', projectPart, `/topics/${topic.id}`],
|
|
126
|
+
}, warnings),
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
edges.push(...terraformDeploys(module.block, nodeId('brokerTopic', topic.id)), {
|
|
130
|
+
type: 'realizes',
|
|
131
|
+
from: brokerTopicId(topic.id),
|
|
132
|
+
to: topicId(topic.id),
|
|
133
|
+
sources: [source],
|
|
134
|
+
});
|
|
135
|
+
if (!rawEvents) {
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
const tableName = rawEventsTableName(topic.id);
|
|
139
|
+
const rawTableLocator = tableLocator(rawEvents.locator, tableName);
|
|
140
|
+
nodes.push({
|
|
141
|
+
layer: 'infrastructure',
|
|
142
|
+
type: 'table',
|
|
143
|
+
locator: rawTableLocator,
|
|
144
|
+
parent: databaseId(rawEvents.locator),
|
|
145
|
+
name: tableName,
|
|
146
|
+
description: `Raw \`${topic.id}\` events.`,
|
|
147
|
+
sources: [source, schemaSource],
|
|
148
|
+
data: {
|
|
149
|
+
resource: await blockResource(graph, module.block, {
|
|
150
|
+
type: 'bigquery.googleapis.com/Table',
|
|
151
|
+
scope: module.project,
|
|
152
|
+
id: [
|
|
153
|
+
'projects/',
|
|
154
|
+
projectPart,
|
|
155
|
+
'/datasets/',
|
|
156
|
+
rawEvents.part,
|
|
157
|
+
`/tables/${tableName}`,
|
|
158
|
+
],
|
|
159
|
+
}, warnings),
|
|
160
|
+
},
|
|
161
|
+
});
|
|
162
|
+
edges.push(...terraformDeploys(module.block, nodeId('table', rawTableLocator)));
|
|
163
|
+
const subscriptionLocator = rawEventsSubscriptionLocator(topic.id);
|
|
164
|
+
const subscription = `bq-${topic.id}`;
|
|
165
|
+
nodes.push({
|
|
166
|
+
layer: 'infrastructure',
|
|
167
|
+
type: 'brokerSubscription',
|
|
168
|
+
locator: subscriptionLocator,
|
|
169
|
+
name: subscription,
|
|
170
|
+
sources: [source, ...rawEvents.sources],
|
|
171
|
+
data: {
|
|
172
|
+
resource: await blockResource(graph, module.block, {
|
|
173
|
+
type: 'pubsub.googleapis.com/Subscription',
|
|
174
|
+
scope: module.project,
|
|
175
|
+
id: ['projects/', projectPart, `/subscriptions/${subscription}`],
|
|
176
|
+
}, warnings),
|
|
177
|
+
},
|
|
178
|
+
});
|
|
179
|
+
edges.push(...terraformDeploys(module.block, nodeId('brokerSubscription', subscriptionLocator)), {
|
|
180
|
+
type: 'delivers',
|
|
181
|
+
from: brokerTopicId(topic.id),
|
|
182
|
+
to: brokerSubscriptionId(subscriptionLocator),
|
|
183
|
+
sources: [source],
|
|
184
|
+
}, {
|
|
185
|
+
type: 'targets',
|
|
186
|
+
from: brokerSubscriptionId(subscriptionLocator),
|
|
187
|
+
to: tableId(rawEvents.locator, tableName),
|
|
188
|
+
sources: [source],
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
return { nodes, edges, warnings };
|
|
192
|
+
},
|
|
193
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { apiRouterModule } from './api-router.js';
|
|
2
|
+
import { cloudRunModule } from './cloud-run-module.js';
|
|
3
|
+
import { eventTopicsModule } from './event-topics.js';
|
|
4
|
+
import { spannerDatabasesModule } from './spanner-databases.js';
|
|
5
|
+
/**
|
|
6
|
+
* The Google rules mirroring the Causa Terraform modules, grouped by module.
|
|
7
|
+
*/
|
|
8
|
+
export const GOOGLE_TERRAFORM_GRAPH_RULES = [
|
|
9
|
+
cloudRunModule,
|
|
10
|
+
apiRouterModule,
|
|
11
|
+
eventTopicsModule,
|
|
12
|
+
spannerDatabasesModule,
|
|
13
|
+
];
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type GraphRule } from '@causa/workspace-core';
|
|
2
|
+
/**
|
|
3
|
+
* The source of the Causa Terraform module creating the Spanner instance and databases.
|
|
4
|
+
*/
|
|
5
|
+
export declare const SPANNER_DATABASES_MODULE_SOURCE = "causa-io/spanner-databases/google";
|
|
6
|
+
/**
|
|
7
|
+
* The resources created by the blocks of the Spanner databases module:
|
|
8
|
+
*
|
|
9
|
+
* - A `google.spanner.instance` node per block, which creates the instance from `google.spanner.instance.name`.
|
|
10
|
+
* - A `database` node per Spanner database, as `GoogleSpannerListDatabases` lists them. The locator is
|
|
11
|
+
* `google.spanner/<instance>.<database>`, and the parent is the instance.
|
|
12
|
+
* - A `table` node per `CREATE TABLE` statement in the DDL files of each database. Tables no schema declares (e.g. an
|
|
13
|
+
* outbox) are still nodes, realizing nothing.
|
|
14
|
+
* - A `realizes` edge from a table to the entity whose binding names it. The table is the one created by the DDL of a
|
|
15
|
+
* database, disambiguated by domain when several databases create a table with the same name.
|
|
16
|
+
*/
|
|
17
|
+
export declare const spannerDatabasesModule: GraphRule;
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { GraphOriginKind, } from '@causa/workspace-core';
|
|
2
|
+
import { databaseId, tableId, tableLocator, DomainsFact, ModelFact, nodeId, } from '@causa/workspace-core/graph';
|
|
3
|
+
import { terraformDeploys } from '@causa/workspace-terraform';
|
|
4
|
+
import { SPANNER_ENGINE, spannerInstanceId } from '../../ids.js';
|
|
5
|
+
import { ddlSource, resolveSpannerTable, SpannerFact } from '../../spanner.js';
|
|
6
|
+
import { blockResource, infrastructureModules, argumentOrConfiguration, } from '../../terraform-modules.js';
|
|
7
|
+
/**
|
|
8
|
+
* The source of the Causa Terraform module creating the Spanner instance and databases.
|
|
9
|
+
*/
|
|
10
|
+
export const SPANNER_DATABASES_MODULE_SOURCE = 'causa-io/spanner-databases/google';
|
|
11
|
+
/**
|
|
12
|
+
* Returns the GCP project and instance parts of the resource names created by a Spanner databases module block.
|
|
13
|
+
*/
|
|
14
|
+
function nameParts(module, warnings) {
|
|
15
|
+
return {
|
|
16
|
+
project: argumentOrConfiguration(module.block, 'gcp_project_id', 'google.project', warnings),
|
|
17
|
+
instance: argumentOrConfiguration(module.block, 'instance_name', 'google.spanner.instance.name', warnings),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The resources created by the blocks of the Spanner databases module:
|
|
22
|
+
*
|
|
23
|
+
* - A `google.spanner.instance` node per block, which creates the instance from `google.spanner.instance.name`.
|
|
24
|
+
* - A `database` node per Spanner database, as `GoogleSpannerListDatabases` lists them. The locator is
|
|
25
|
+
* `google.spanner/<instance>.<database>`, and the parent is the instance.
|
|
26
|
+
* - A `table` node per `CREATE TABLE` statement in the DDL files of each database. Tables no schema declares (e.g. an
|
|
27
|
+
* outbox) are still nodes, realizing nothing.
|
|
28
|
+
* - A `realizes` edge from a table to the entity whose binding names it. The table is the one created by the DDL of a
|
|
29
|
+
* database, disambiguated by domain when several databases create a table with the same name.
|
|
30
|
+
*/
|
|
31
|
+
export const spannerDatabasesModule = {
|
|
32
|
+
name: 'spannerDatabasesModule',
|
|
33
|
+
kind: GraphOriginKind.Declared,
|
|
34
|
+
description: 'One `google.spanner.instance` node per Spanner databases module block, named by `google.spanner.instance.name`. One `database` node per Spanner database listed by `GoogleSpannerListDatabases` from the DDL files, and one `table` node per `CREATE TABLE` in them, realizing the entity whose `googleSpannerTable` binding names it, disambiguated by domain. Each resource is linked by a `deploys` edge from the projects applying the block.',
|
|
35
|
+
async run(graph) {
|
|
36
|
+
const { locator, context } = graph;
|
|
37
|
+
const [domains, model, spanner, spannerDatabases] = await Promise.all([
|
|
38
|
+
graph.get(DomainsFact),
|
|
39
|
+
graph.get(ModelFact),
|
|
40
|
+
graph.get(SpannerFact),
|
|
41
|
+
infrastructureModules(graph, SPANNER_DATABASES_MODULE_SOURCE),
|
|
42
|
+
]);
|
|
43
|
+
const nodes = [];
|
|
44
|
+
const edges = [];
|
|
45
|
+
const warnings = [];
|
|
46
|
+
for (const module of spannerDatabases) {
|
|
47
|
+
const source = module.block.declaration;
|
|
48
|
+
if (!spanner.instance) {
|
|
49
|
+
warnings.push({
|
|
50
|
+
message: "A Spanner databases module block exists, but 'google.spanner.instance.name' is not set.",
|
|
51
|
+
sources: [source],
|
|
52
|
+
});
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
const { project, instance } = nameParts(module, warnings);
|
|
56
|
+
if (module.block.arguments['instance_name'] !== undefined) {
|
|
57
|
+
warnings.push({
|
|
58
|
+
message: `The module overrides the instance name. Locators use the configured '${spanner.instance}'.`,
|
|
59
|
+
sources: [source],
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
nodes.push({
|
|
63
|
+
layer: 'infrastructure',
|
|
64
|
+
type: 'google.spanner.instance',
|
|
65
|
+
locator: spanner.instance,
|
|
66
|
+
name: spanner.instance,
|
|
67
|
+
sources: [
|
|
68
|
+
await locator.configurationSource(module.context, [
|
|
69
|
+
'google',
|
|
70
|
+
'spanner',
|
|
71
|
+
'instance',
|
|
72
|
+
'name',
|
|
73
|
+
]),
|
|
74
|
+
source,
|
|
75
|
+
],
|
|
76
|
+
data: {
|
|
77
|
+
resource: await blockResource(graph, module.block, {
|
|
78
|
+
type: 'spanner.googleapis.com/Instance',
|
|
79
|
+
scope: module.project,
|
|
80
|
+
id: ['projects/', project, '/instances/', instance],
|
|
81
|
+
}, warnings),
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
edges.push(...terraformDeploys(module.block, nodeId('google.spanner.instance', spanner.instance)));
|
|
85
|
+
for (const database of spanner.databases) {
|
|
86
|
+
const data = {
|
|
87
|
+
engine: SPANNER_ENGINE,
|
|
88
|
+
resource: await blockResource(graph, module.block, {
|
|
89
|
+
type: 'spanner.googleapis.com/Database',
|
|
90
|
+
scope: module.project,
|
|
91
|
+
id: [
|
|
92
|
+
'projects/',
|
|
93
|
+
project,
|
|
94
|
+
'/instances/',
|
|
95
|
+
instance,
|
|
96
|
+
`/databases/${database.id}`,
|
|
97
|
+
],
|
|
98
|
+
}, warnings),
|
|
99
|
+
};
|
|
100
|
+
nodes.push({
|
|
101
|
+
layer: 'infrastructure',
|
|
102
|
+
type: 'database',
|
|
103
|
+
locator: database.locator,
|
|
104
|
+
parent: spannerInstanceId(spanner.instance),
|
|
105
|
+
name: database.id,
|
|
106
|
+
sources: [
|
|
107
|
+
{ path: database.directory },
|
|
108
|
+
await locator.configurationSource(context, [
|
|
109
|
+
'google',
|
|
110
|
+
'spanner',
|
|
111
|
+
'ddls',
|
|
112
|
+
]),
|
|
113
|
+
source,
|
|
114
|
+
],
|
|
115
|
+
data,
|
|
116
|
+
});
|
|
117
|
+
edges.push(...terraformDeploys(module.block, nodeId('database', database.locator)));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (spanner.databases.length > 0 && spannerDatabases.length === 0) {
|
|
121
|
+
warnings.push({
|
|
122
|
+
message: 'Spanner DDL files exist but no Spanner databases module block creates the databases.',
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
if (!spanner.instance || spannerDatabases.length === 0) {
|
|
126
|
+
return { nodes, edges, warnings };
|
|
127
|
+
}
|
|
128
|
+
const blocks = spannerDatabases.map((m) => m.block.declaration);
|
|
129
|
+
for (const database of spanner.databases) {
|
|
130
|
+
for (const table of database.tables) {
|
|
131
|
+
const nodeLocator = tableLocator(database.locator, table.name);
|
|
132
|
+
nodes.push({
|
|
133
|
+
layer: 'infrastructure',
|
|
134
|
+
type: 'table',
|
|
135
|
+
locator: nodeLocator,
|
|
136
|
+
parent: databaseId(database.locator),
|
|
137
|
+
name: table.name,
|
|
138
|
+
sources: [ddlSource(table), ...blocks],
|
|
139
|
+
});
|
|
140
|
+
edges.push(...spannerDatabases.flatMap((module) => terraformDeploys(module.block, nodeId('table', nodeLocator))));
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
for (const entity of model.entities.values()) {
|
|
144
|
+
const resolution = resolveSpannerTable(spanner, entity, domains);
|
|
145
|
+
if (!resolution) {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const { file, segments, definition } = entity.schema;
|
|
149
|
+
const binding = await locator.source(file, [
|
|
150
|
+
...segments,
|
|
151
|
+
'causa',
|
|
152
|
+
'googleSpannerTable',
|
|
153
|
+
]);
|
|
154
|
+
const table = definition.databases.find((d) => d.engine === SPANNER_ENGINE)?.table;
|
|
155
|
+
if (resolution.kind === 'missing') {
|
|
156
|
+
warnings.push({
|
|
157
|
+
message: `Entity '${definition.name}' binds to Spanner table '${table}', which no DDL creates.`,
|
|
158
|
+
sources: [binding],
|
|
159
|
+
});
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (resolution.kind === 'ambiguous') {
|
|
163
|
+
warnings.push({
|
|
164
|
+
message: `Entity '${definition.name}' binds to Spanner table '${table}', which is created in several databases: ${resolution.candidates
|
|
165
|
+
.map((c) => `'${c.id}'`)
|
|
166
|
+
.join(', ')}.`,
|
|
167
|
+
sources: [binding],
|
|
168
|
+
});
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
edges.push({
|
|
172
|
+
type: 'realizes',
|
|
173
|
+
from: tableId(resolution.database.locator, resolution.table.name),
|
|
174
|
+
to: entity.id,
|
|
175
|
+
sources: [binding, ddlSource(resolution.table)],
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
return { nodes, edges, warnings };
|
|
179
|
+
},
|
|
180
|
+
};
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { GraphOriginSource } from '@causa/workspace-core';
|
|
2
|
+
import { GraphExtractionFact, type EntityFacts, type GraphContext, type GraphFactOutput, type WorkspaceDomain } from '@causa/workspace-core/graph';
|
|
3
|
+
/**
|
|
4
|
+
* A table created by a `CREATE TABLE` statement in a Spanner DDL file.
|
|
5
|
+
*/
|
|
6
|
+
export type SpannerTable = {
|
|
7
|
+
/**
|
|
8
|
+
* The name of the table.
|
|
9
|
+
*/
|
|
10
|
+
readonly name: string;
|
|
11
|
+
/**
|
|
12
|
+
* The DDL file declaring the table, relative to the workspace root.
|
|
13
|
+
*/
|
|
14
|
+
readonly file: string;
|
|
15
|
+
/**
|
|
16
|
+
* The line of the statement in the file.
|
|
17
|
+
*/
|
|
18
|
+
readonly line: number;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* A Spanner database of the workspace, and the tables its DDL files create.
|
|
22
|
+
*/
|
|
23
|
+
export type SpannerDatabaseFacts = {
|
|
24
|
+
/**
|
|
25
|
+
* The ID of the database, as `GoogleSpannerListDatabases` returns it.
|
|
26
|
+
*/
|
|
27
|
+
readonly id: string;
|
|
28
|
+
/**
|
|
29
|
+
* The name of the database within its engine, `<instance>.<database>`, as `outputs['google.spanner']` lists it.
|
|
30
|
+
*/
|
|
31
|
+
readonly name: string;
|
|
32
|
+
/**
|
|
33
|
+
* The locator of the `database` node.
|
|
34
|
+
*/
|
|
35
|
+
readonly locator: string;
|
|
36
|
+
/**
|
|
37
|
+
* The directory holding the DDL files, relative to the workspace root.
|
|
38
|
+
*/
|
|
39
|
+
readonly directory: string;
|
|
40
|
+
/**
|
|
41
|
+
* The domain the DDL files belong to.
|
|
42
|
+
*/
|
|
43
|
+
readonly domain: WorkspaceDomain | undefined;
|
|
44
|
+
/**
|
|
45
|
+
* The tables created by the DDL files.
|
|
46
|
+
*/
|
|
47
|
+
readonly tables: SpannerTable[];
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* The Spanner databases of the workspace.
|
|
51
|
+
*/
|
|
52
|
+
export type SpannerFacts = {
|
|
53
|
+
/**
|
|
54
|
+
* The Spanner instance, from `google.spanner.instance.name`.
|
|
55
|
+
*/
|
|
56
|
+
readonly instance: string | undefined;
|
|
57
|
+
/**
|
|
58
|
+
* The databases, as `GoogleSpannerListDatabases` lists them.
|
|
59
|
+
*/
|
|
60
|
+
readonly databases: SpannerDatabaseFacts[];
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* The Spanner databases of the workspace, as `GoogleSpannerListDatabases` lists them, and the tables their DDL files
|
|
64
|
+
* create.
|
|
65
|
+
*/
|
|
66
|
+
export declare class SpannerFact extends GraphExtractionFact<SpannerFacts> {
|
|
67
|
+
compute(graph: GraphContext): Promise<GraphFactOutput<SpannerFacts>>;
|
|
68
|
+
/**
|
|
69
|
+
* Parses the tables created by the `CREATE TABLE` statements of DDL files.
|
|
70
|
+
*
|
|
71
|
+
* @param context The workspace context.
|
|
72
|
+
* @param files The DDL files, relative to the workspace root.
|
|
73
|
+
* @returns The tables, in the order of the files and statements.
|
|
74
|
+
*/
|
|
75
|
+
private parseTables;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The resolution of an entity's Spanner binding to a table.
|
|
79
|
+
*/
|
|
80
|
+
export type SpannerTableResolution = {
|
|
81
|
+
readonly kind: 'resolved';
|
|
82
|
+
readonly database: SpannerDatabaseFacts;
|
|
83
|
+
readonly table: SpannerTable;
|
|
84
|
+
} | {
|
|
85
|
+
readonly kind: 'ambiguous';
|
|
86
|
+
readonly candidates: SpannerDatabaseFacts[];
|
|
87
|
+
} | {
|
|
88
|
+
readonly kind: 'missing';
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Resolves the Spanner binding of an entity to the database whose DDL creates a table of that name. When several
|
|
92
|
+
* databases declare the name, the one in the entity's domain is chosen.
|
|
93
|
+
*
|
|
94
|
+
* @param spanner The Spanner databases.
|
|
95
|
+
* @param entity The entity.
|
|
96
|
+
* @param domains The domains of the workspace, in which the domain of the entity is looked up.
|
|
97
|
+
* @returns The resolution, or `undefined` if the entity is not persisted in Spanner.
|
|
98
|
+
*/
|
|
99
|
+
export declare function resolveSpannerTable(spanner: SpannerFacts, entity: EntityFacts, domains: readonly WorkspaceDomain[]): SpannerTableResolution | undefined;
|
|
100
|
+
/**
|
|
101
|
+
* Returns the origin source of a `CREATE TABLE` statement.
|
|
102
|
+
*
|
|
103
|
+
* @param table The table.
|
|
104
|
+
* @returns The origin source.
|
|
105
|
+
*/
|
|
106
|
+
export declare function ddlSource(table: SpannerTable): GraphOriginSource;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { domainAt, domainOfFile, DomainsFact, GraphExtractionFact, } from '@causa/workspace-core/graph';
|
|
2
|
+
import { readFile } from 'fs/promises';
|
|
3
|
+
import { dirname, join, relative } from 'path';
|
|
4
|
+
import { GoogleSpannerListDatabases } from '../functions/google-spanner/index.js';
|
|
5
|
+
import { SPANNER_ENGINE, spannerDatabaseLocator, spannerDatabaseName, } from './ids.js';
|
|
6
|
+
/**
|
|
7
|
+
* The Spanner databases of the workspace, as `GoogleSpannerListDatabases` lists them, and the tables their DDL files
|
|
8
|
+
* create.
|
|
9
|
+
*/
|
|
10
|
+
export class SpannerFact extends GraphExtractionFact {
|
|
11
|
+
async compute(graph) {
|
|
12
|
+
const { context } = graph;
|
|
13
|
+
const instance = context
|
|
14
|
+
.asConfiguration()
|
|
15
|
+
.get('google.spanner.instance.name');
|
|
16
|
+
const [listed, domains] = await Promise.all([
|
|
17
|
+
context.call(GoogleSpannerListDatabases, {}),
|
|
18
|
+
graph.get(DomainsFact),
|
|
19
|
+
]);
|
|
20
|
+
const databases = await Promise.all(listed.map(async ({ id, ddlFiles }) => {
|
|
21
|
+
const files = ddlFiles.map((f) => relative(context.rootPath, f)).sort();
|
|
22
|
+
const tables = await this.parseTables(context, files);
|
|
23
|
+
const directory = files[0] ? dirname(files[0]) : '';
|
|
24
|
+
return {
|
|
25
|
+
id,
|
|
26
|
+
name: spannerDatabaseName(instance ?? '', id),
|
|
27
|
+
locator: spannerDatabaseLocator(instance ?? '', id),
|
|
28
|
+
directory,
|
|
29
|
+
domain: directory ? domainAt(domains, directory) : undefined,
|
|
30
|
+
tables,
|
|
31
|
+
};
|
|
32
|
+
}));
|
|
33
|
+
databases.sort((a, b) => a.id.localeCompare(b.id));
|
|
34
|
+
const warnings = [];
|
|
35
|
+
for (const database of databases) {
|
|
36
|
+
const names = new Set();
|
|
37
|
+
for (const table of database.tables) {
|
|
38
|
+
if (names.has(table.name)) {
|
|
39
|
+
warnings.push({
|
|
40
|
+
message: `Table '${table.name}' is created twice in database '${database.id}'.`,
|
|
41
|
+
sources: [ddlSource(table)],
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
names.add(table.name);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return { value: { instance, databases }, warnings };
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Parses the tables created by the `CREATE TABLE` statements of DDL files.
|
|
51
|
+
*
|
|
52
|
+
* @param context The workspace context.
|
|
53
|
+
* @param files The DDL files, relative to the workspace root.
|
|
54
|
+
* @returns The tables, in the order of the files and statements.
|
|
55
|
+
*/
|
|
56
|
+
async parseTables(context, files) {
|
|
57
|
+
const tables = [];
|
|
58
|
+
for (const file of files) {
|
|
59
|
+
const content = await readFile(join(context.rootPath, file), 'utf-8');
|
|
60
|
+
const withoutComments = content.replace(/--.*$/gm, (m) => ' '.repeat(m.length));
|
|
61
|
+
const statements = withoutComments.matchAll(/\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[`"']?([A-Za-z_]\w*)[`"']?/gi);
|
|
62
|
+
for (const match of statements) {
|
|
63
|
+
tables.push({
|
|
64
|
+
name: match[1],
|
|
65
|
+
file,
|
|
66
|
+
line: withoutComments.slice(0, match.index).split('\n').length,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return tables;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Resolves the Spanner binding of an entity to the database whose DDL creates a table of that name. When several
|
|
75
|
+
* databases declare the name, the one in the entity's domain is chosen.
|
|
76
|
+
*
|
|
77
|
+
* @param spanner The Spanner databases.
|
|
78
|
+
* @param entity The entity.
|
|
79
|
+
* @param domains The domains of the workspace, in which the domain of the entity is looked up.
|
|
80
|
+
* @returns The resolution, or `undefined` if the entity is not persisted in Spanner.
|
|
81
|
+
*/
|
|
82
|
+
export function resolveSpannerTable(spanner, entity, domains) {
|
|
83
|
+
const { definition, file } = entity.schema;
|
|
84
|
+
const table = definition.databases.find((d) => d.engine === SPANNER_ENGINE)?.table;
|
|
85
|
+
if (!table) {
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
88
|
+
const candidates = spanner.databases.filter((d) => d.tables.some((t) => t.name === table));
|
|
89
|
+
const pick = (database) => ({
|
|
90
|
+
kind: 'resolved',
|
|
91
|
+
database,
|
|
92
|
+
table: database.tables.find((t) => t.name === table),
|
|
93
|
+
});
|
|
94
|
+
if (candidates.length === 1) {
|
|
95
|
+
return pick(candidates[0]);
|
|
96
|
+
}
|
|
97
|
+
if (candidates.length === 0) {
|
|
98
|
+
return { kind: 'missing' };
|
|
99
|
+
}
|
|
100
|
+
const domain = domainOfFile(domains, file);
|
|
101
|
+
const sameDomain = candidates.filter((d) => domain && d.domain?.directory === domain.directory);
|
|
102
|
+
return sameDomain.length === 1
|
|
103
|
+
? pick(sameDomain[0])
|
|
104
|
+
: { kind: 'ambiguous', candidates };
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Returns the origin source of a `CREATE TABLE` statement.
|
|
108
|
+
*
|
|
109
|
+
* @param table The table.
|
|
110
|
+
* @returns The origin source.
|
|
111
|
+
*/
|
|
112
|
+
export function ddlSource(table) {
|
|
113
|
+
return {
|
|
114
|
+
path: table.file,
|
|
115
|
+
pointer: `CREATE TABLE ${table.name}`,
|
|
116
|
+
location: { start: { line: table.line } },
|
|
117
|
+
};
|
|
118
|
+
}
|