@causa/workspace-google 1.4.0-beta.1 → 1.4.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +19 -3
  2. package/dist/functions/event-topic/count-events-bigquery.call.d.ts +3 -0
  3. package/dist/functions/event-topic/count-events-bigquery.call.js +75 -0
  4. package/dist/functions/event-topic/count-events-bigquery.d.ts +11 -0
  5. package/dist/functions/event-topic/count-events-bigquery.js +19 -0
  6. package/dist/functions/event-topic/create-backfill-source-bigquery.call.js +6 -0
  7. package/dist/functions/event-topic/create-backfill-source-bigquery.d.ts +1 -1
  8. package/dist/functions/event-topic/create-backfill-source-bigquery.js +1 -1
  9. package/dist/functions/event-topic/index.d.ts +1 -0
  10. package/dist/functions/event-topic/index.js +1 -0
  11. package/dist/functions/event-topic/query-events-bigquery.call.js +2 -0
  12. package/dist/functions/event-topic/query-events-bigquery.d.ts +1 -0
  13. package/dist/functions/event-topic/query-events-bigquery.js +1 -0
  14. package/dist/functions/google-app-check/generate-token.js +3 -1
  15. package/dist/functions/google-firebase-storage/merge-rules.js +3 -1
  16. package/dist/functions/google-firestore/merge-rules.js +3 -1
  17. package/dist/functions/google-identity-platform/generate-custom-token.js +7 -2
  18. package/dist/functions/google-pubsub/write-topics.js +7 -2
  19. package/dist/functions/google-services/enable.js +3 -1
  20. package/dist/functions/google-spanner/count-records.call.d.ts +16 -0
  21. package/dist/functions/google-spanner/count-records.call.js +120 -0
  22. package/dist/functions/google-spanner/count-records.d.ts +15 -0
  23. package/dist/functions/google-spanner/count-records.js +21 -0
  24. package/dist/functions/google-spanner/index.d.ts +3 -1
  25. package/dist/functions/google-spanner/index.js +3 -1
  26. package/dist/functions/google-spanner/list-databases.d.ts +22 -1
  27. package/dist/functions/google-spanner/list-databases.js +27 -15
  28. package/dist/functions/google-spanner/resolve-table.d.ts +25 -0
  29. package/dist/functions/google-spanner/resolve-table.js +44 -0
  30. package/dist/functions/google-spanner/write-databases.js +7 -2
  31. package/dist/functions/graph/fetch-environment-metric-series.call.d.ts +3 -0
  32. package/dist/functions/graph/fetch-environment-metric-series.call.js +107 -0
  33. package/dist/functions/graph/fetch-environment-metric-series.d.ts +23 -0
  34. package/dist/functions/graph/fetch-environment-metric-series.js +40 -0
  35. package/dist/functions/graph/get-environment-provider.d.ts +11 -0
  36. package/dist/functions/graph/get-environment-provider.js +20 -0
  37. package/dist/functions/graph/index.d.ts +2 -0
  38. package/dist/functions/graph/index.js +2 -0
  39. package/dist/functions/index.js +4 -4
  40. package/dist/graph/cloud-run.d.ts +2 -2
  41. package/dist/graph/cloud-run.js +2 -2
  42. package/dist/graph/environment/fetchers/cloud-monitoring-alerts.d.ts +6 -0
  43. package/dist/graph/environment/fetchers/cloud-monitoring-alerts.js +106 -0
  44. package/dist/graph/environment/fetchers/cloud-monitoring.d.ts +5 -0
  45. package/dist/graph/environment/fetchers/cloud-monitoring.js +13 -0
  46. package/dist/graph/environment/fetchers/cloud-run.d.ts +5 -0
  47. package/dist/graph/environment/fetchers/cloud-run.js +88 -0
  48. package/dist/graph/environment/fetchers/cloud-scheduler.d.ts +5 -0
  49. package/dist/graph/environment/fetchers/cloud-scheduler.js +95 -0
  50. package/dist/graph/environment/fetchers/console-links.d.ts +14 -0
  51. package/dist/graph/environment/fetchers/console-links.js +132 -0
  52. package/dist/graph/environment/fetchers/error-reporting.d.ts +5 -0
  53. package/dist/graph/environment/fetchers/error-reporting.js +98 -0
  54. package/dist/graph/environment/fetchers/index.d.ts +5 -0
  55. package/dist/graph/environment/fetchers/index.js +17 -0
  56. package/dist/graph/environment/index.d.ts +9 -0
  57. package/dist/graph/environment/index.js +9 -0
  58. package/dist/graph/environment/load-balancer-backends.d.ts +82 -0
  59. package/dist/graph/environment/load-balancer-backends.js +126 -0
  60. package/dist/graph/environment/monitoring-alerts.d.ts +60 -0
  61. package/dist/graph/environment/monitoring-alerts.js +67 -0
  62. package/dist/graph/environment/monitoring-catalog.d.ts +95 -0
  63. package/dist/graph/environment/monitoring-catalog.js +356 -0
  64. package/dist/graph/environment/monitoring-mappings.d.ts +133 -0
  65. package/dist/graph/environment/monitoring-mappings.js +97 -0
  66. package/dist/graph/environment/monitoring-query.d.ts +116 -0
  67. package/dist/graph/environment/monitoring-query.js +314 -0
  68. package/dist/graph/environment/monitoring-values.d.ts +41 -0
  69. package/dist/graph/environment/monitoring-values.js +119 -0
  70. package/dist/graph/environment/resource-resolvers.d.ts +7 -0
  71. package/dist/graph/environment/resource-resolvers.js +57 -0
  72. package/dist/graph/environment/resources.d.ts +45 -0
  73. package/dist/graph/environment/resources.js +49 -0
  74. package/dist/graph/index.d.ts +1 -0
  75. package/dist/graph/index.js +1 -0
  76. package/dist/graph/rules/configuration/service-container.js +4 -4
  77. package/dist/graph/rules/terraform/spanner-databases.js +4 -4
  78. package/dist/graph/spanner.d.ts +11 -60
  79. package/dist/graph/spanner.js +25 -69
  80. package/package.json +24 -24
package/README.md CHANGED
@@ -60,7 +60,7 @@ Backfilling is supported when `google.pubSub` is set as the `events.broker`. Tem
60
60
  - As a raw URI, using the format `[[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger`.
61
61
  - As a project-scoped trigger, using the format `<projectPath>#<triggerName>[?<options>]`. The referenced project must be of type `serviceContainer` with `google.cloudRun` as `serviceContainer.platform`. The HTTP path is read from `serviceContainer.triggers[<triggerName>].endpoint` (whose `type` must be `http`), and the `options` query string is forwarded as URL query parameters. The target Cloud Run service name defaults to `project.name`, and can be overridden by setting `google.cloudRun.eventBackfillServiceName`.
62
62
 
63
- When no source is specified, the default is to fetch events to backfill from the BigQuery dataset configured in `google.pubSub.bigQueryStorage`. A custom BigQuery table can also be set as source using the `bq://<projectId>.<datasetId>.<tableId>` format. It should have the `data` and `attributes` columns.
63
+ When no source is specified, the default is to fetch events to backfill from the BigQuery dataset configured in `google.pubSub.bigQueryStorage`, once per Pub/Sub message ID. A custom BigQuery table can also be set as source using the `bq://<projectId>.<datasetId>.<tableId>` format. It should have the `data` and `attributes` columns.
64
64
 
65
65
  The Pub/Sub publish options used when backfilling can be tuned through `google.pubSub.backfillPublishOptions`, which exposes the `maxOutstandingBytes`, `maxOutstandingMessages`, `maxBytes`, `maxMessages`, and `maxMilliseconds` fields.
66
66
 
@@ -68,11 +68,13 @@ By default, project-scoped triggers send backfill events to the live Cloud Run s
68
68
 
69
69
  ### Querying
70
70
 
71
- This module implements the `DatabaseQueryRecords`, `ServiceContainerQueryLogs`, and `EventTopicQueryEvents` workspace functions, which expose read-only access to data stored in or emitted by deployed services:
71
+ This module implements the `DatabaseQueryRecords`, `DatabaseCountRecords`, `ServiceContainerQueryLogs`, `EventTopicQueryEvents`, and `EventTopicCountEvents` workspace functions, which expose read-only access to data stored in or emitted by deployed services:
72
72
 
73
73
  - `DatabaseQueryRecords` is supported for the `google.spanner` and `google.firestore` engines. For Spanner, the function takes a database name and a SQL query, runs it on the configured Spanner instance, and returns the matching rows (capped to 100000). For Firestore, the query is the path to a single document, fetched from the default database (or the specified one). The document data is returned as a single-element array. An empty array is returned when the document does not exist.
74
+ - `DatabaseCountRecords` is supported for the `google.spanner` engine. It counts the rows of the table in the schema's `googleSpannerTable` binding, in the database whose DDL creates it, or the one in the domain of the schema when several databases create a table with that name. Properties are read from the column named by their `googleSpannerColumn` extension, or from the column with the same name, and nested properties are read from JSON columns, where a value only equals a constant of the same JSON type.
74
75
  - `ServiceContainerQueryLogs` is supported when `serviceContainer.platform` is `google.cloudRun`. It queries Cloud Logging for log entries emitted by the service's Cloud Run revisions. By default, only the last hour of logs and up to 1 000 entries are returned.
75
- - `EventTopicQueryEvents` is supported when `events.broker` is `google.pubSub`, `events.format` is `json`, and `google.pubSub.bigQueryStorage.rawEventsDatasetId` is set. It reads events from the topic's BigQuery raw events table. Same defaults as for log queries: last hour and up to 1 000 events.
76
+ - `EventTopicQueryEvents` is supported when `events.broker` is `google.pubSub`, `events.format` is `json`, and `google.pubSub.bigQueryStorage.rawEventsDatasetId` is set. It reads events from the topic's BigQuery raw events table, once per Pub/Sub message ID, as the BigQuery subscription may write a message several times. Same defaults as for log queries: last hour and up to 1 000 events.
77
+ - `EventTopicCountEvents` is supported in the same cases as `EventTopicQueryEvents`. It counts the events published within the time range in the topic's BigQuery raw events table, whose `data` column should be of the `JSON` type, once per Pub/Sub message ID. A value only equals a constant of the same JSON type.
76
78
 
77
79
  ### Architecture graph
78
80
 
@@ -83,6 +85,20 @@ This module contributes to the architecture graph extracted by `cs graph extract
83
85
 
84
86
  Infrastructure nodes carry the Google Cloud resource they stand for in `data.resource`: its `type` (e.g. `pubsub.googleapis.com/Topic`), its `id`, i.e. its full resource name (or `idPrefix` when the full name is not known), and the `scope`, i.e. the project whose configuration should be used to render the identifier.
85
87
 
88
+ When the graph is enriched with environment data, this module resolves the names of Cloud Tasks queues by listing the queues of the environment. Queues that cannot be resolved are removed from their nodes. This module's fetchers then:
89
+
90
+ - read metrics of services, API routers, Pub/Sub topics and subscriptions, Cloud Tasks queues, and Spanner, Firestore, and BigQuery databases from Cloud Monitoring
91
+ - add the `deployment` of Cloud Run services, and the `lastAttempt` and `nextRun` of Cloud Scheduler jobs. They describe services and jobs as they are when the graph is enriched, with a warning when they changed after `at`.
92
+ - add `links` to the Google Cloud console for each resource
93
+ - attach the alerts open in Cloud Monitoring to the nodes of their resources. Matching an alert to a node requires the condition of its policy to keep the labels making up the resource name (e.g. `project_id`, `location`, and `service_name`), along with `monitored_resource` when the metric is written against several resource types, like `logging.googleapis.com/log_entry_count`.
94
+ - attach the open Error Reporting groups of Cloud Run services to their nodes, as alerts with the `error` severity. A group is open when it has occurrences during the last 30 days, and is not acknowledged, resolved, or muted.
95
+
96
+ The requests and latencies of API routers are also grouped by the `node` the requests were routed to, by mapping the serverless network endpoint groups of their load balancers to Cloud Run services. They are listed using the Compute Engine API (`compute.networkEndpointGroups.list`) in the projects of the API routers.
97
+
98
+ `GraphFetchEnvironmentMetricSeries` is implemented for the metrics read from Cloud Monitoring.
99
+
100
+ Cloud Monitoring exposes complete data with a delay of a few minutes. Metrics, including their time series, are read over windows ending at most 5 minutes ago.
101
+
86
102
  ### Secrets backend
87
103
 
88
104
  This module implements the `google.secretManager` secret backend, allowing fetching secrets from the Google Secret Manager service. Here are some example of how secrets with the `google.secretManager` backend should be defined:
@@ -0,0 +1,3 @@
1
+ import type { RecordCounts } from '@causa/workspace-core';
2
+ import type { EventTopicCountEventsForBigQuery } from './count-events-bigquery.js';
3
+ export default function call(this: EventTopicCountEventsForBigQuery): Promise<RecordCounts>;
@@ -0,0 +1,75 @@
1
+ import { BigQueryService } from '../../services/index.js';
2
+ import { rawEventsTableName } from '../google-pubsub/index.js';
3
+ /**
4
+ * Returns the function converting a JSON value to the type of the given value, without failing for other types.
5
+ *
6
+ * @param value The value to which the JSON value is compared.
7
+ * @returns The name of the BigQuery conversion function.
8
+ */
9
+ function laxConversion(value) {
10
+ switch (typeof value) {
11
+ case 'string':
12
+ return 'LAX_STRING';
13
+ case 'number':
14
+ return 'LAX_FLOAT64';
15
+ case 'boolean':
16
+ return 'LAX_BOOL';
17
+ }
18
+ }
19
+ /**
20
+ * Builds the condition on the event payload, stored in the `data` JSON column, for a {@link RecordCondition}.
21
+ *
22
+ * @param condition The condition.
23
+ * @param addParam A function adding a parameter to the query, and returning its name.
24
+ * @returns The SQL condition.
25
+ */
26
+ function sqlCondition(condition, addParam) {
27
+ const value = `data${condition.path.map((key) => `[${JSON.stringify(key)}]`).join('')}`;
28
+ switch (condition.operator) {
29
+ case 'isNull':
30
+ return `(${value} IS NULL OR JSON_TYPE(${value}) = 'null')`;
31
+ case 'isNotNull':
32
+ return `JSON_TYPE(${value}) != 'null'`;
33
+ case 'equals':
34
+ // The type is checked as lax conversions also convert values of other types, e.g. the string `"false"` to `FALSE`.
35
+ return `JSON_TYPE(${value}) = '${typeof condition.value}' AND ${laxConversion(condition.value)}(${value}) = @${addParam(condition.value)}`;
36
+ }
37
+ }
38
+ export default async function call() {
39
+ const googleConf = this._context.asConfiguration();
40
+ const projectId = googleConf.getOrThrow('google.project');
41
+ const rawEventsDatasetId = googleConf.getOrThrow('google.pubSub.bigQueryStorage.rawEventsDatasetId');
42
+ const tableId = `${projectId}.${rawEventsDatasetId}.${rawEventsTableName(this.topic)}`;
43
+ const names = Object.keys(this.filters);
44
+ const params = {
45
+ from: this.from.toISOString(),
46
+ to: this.to.toISOString(),
47
+ };
48
+ let paramCount = 0;
49
+ const addParam = (value) => {
50
+ const param = `p${paramCount++}`;
51
+ params[param] = value;
52
+ return param;
53
+ };
54
+ const counts = names.map((name, index) => {
55
+ const conditions = this.filters[name].map((c) => sqlCondition(c, addParam));
56
+ return `COUNTIF(${conditions.join(' AND ') || 'TRUE'}) AS f${index}`;
57
+ });
58
+ // Events written several times to the table by the BigQuery subscription are only counted once.
59
+ const query = `
60
+ WITH events AS (
61
+ SELECT data
62
+ FROM \`${tableId}\`
63
+ WHERE publish_time >= @from AND publish_time < @to
64
+ QUALIFY ROW_NUMBER() OVER (PARTITION BY message_id ORDER BY publish_time) = 1
65
+ )
66
+ SELECT ${['COUNT(*) AS total', ...counts].join(', ')}
67
+ FROM events`;
68
+ this._context.logger.debug(`Counting events in BigQuery:\n${query}`);
69
+ const bigQuery = this._context.service(BigQueryService).bigQuery;
70
+ const [[row]] = await bigQuery.query({ query, params });
71
+ return {
72
+ total: Number(row.total),
73
+ counts: Object.fromEntries(names.map((name, index) => [name, Number(row[`f${index}`])])),
74
+ };
75
+ }
@@ -0,0 +1,11 @@
1
+ import { EventTopicCountEvents, type RecordCounts } from '@causa/workspace-core';
2
+ /**
3
+ * Implements {@link EventTopicCountEvents} for Pub/Sub topics whose events are stored in BigQuery via a BigQuery
4
+ * subscription. The events are read from the dataset configured at `google.pubSub.bigQueryStorage.rawEventsDatasetId`,
5
+ * whose `data` column is expected to be of the `JSON` type.
6
+ * Events are counted once per Pub/Sub message ID, and the time range applies to their publish time.
7
+ */
8
+ export declare class EventTopicCountEventsForBigQuery extends EventTopicCountEvents {
9
+ _call(): Promise<RecordCounts>;
10
+ _supports(): boolean;
11
+ }
@@ -0,0 +1,19 @@
1
+ import { callDeferred } from '@causa/workspace';
2
+ import { EventTopicCountEvents, } from '@causa/workspace-core';
3
+ /**
4
+ * Implements {@link EventTopicCountEvents} for Pub/Sub topics whose events are stored in BigQuery via a BigQuery
5
+ * subscription. The events are read from the dataset configured at `google.pubSub.bigQueryStorage.rawEventsDatasetId`,
6
+ * whose `data` column is expected to be of the `JSON` type.
7
+ * Events are counted once per Pub/Sub message ID, and the time range applies to their publish time.
8
+ */
9
+ export class EventTopicCountEventsForBigQuery extends EventTopicCountEvents {
10
+ async _call() {
11
+ return await callDeferred(this, import.meta.url);
12
+ }
13
+ _supports() {
14
+ const conf = this._context.asConfiguration();
15
+ return (conf.get('events.broker') === 'google.pubSub' &&
16
+ conf.get('events.format') === 'json' &&
17
+ !!conf.get('google.pubSub.bigQueryStorage.rawEventsDatasetId'));
18
+ }
19
+ }
@@ -34,6 +34,12 @@ export default async function call() {
34
34
  WHERE
35
35
  (${this.filter})`;
36
36
  }
37
+ if (!this.source) {
38
+ // Events written several times to the default storage by the BigQuery subscription are only backfilled once.
39
+ query += `
40
+ QUALIFY
41
+ ROW_NUMBER() OVER (PARTITION BY message_id ORDER BY publish_time) = 1`;
42
+ }
37
43
  this._context.logger.debug(`Creating BigQuery job from query '${query}'.`);
38
44
  const bigQuery = this._context.service(BigQueryService).bigQuery;
39
45
  const [job] = await bigQuery.createQueryJob(query);
@@ -9,7 +9,7 @@ export declare const BIGQUERY_SOURCE_REGEX: RegExp;
9
9
  * Two cases are supported:
10
10
  * - The `source` is a `bq://<projectId>.<datasetId>.<tableId>` URI.
11
11
  * - The `source` is omitted and the broker is `google.pubSub` with `google.pubSub.bigQueryStorage.rawEventsDatasetId`
12
- * configured: events are read from the default BigQuery storage for the topic.
12
+ * configured: events are read from the default BigQuery storage for the topic, once per Pub/Sub message ID.
13
13
  *
14
14
  * The returned async iterable pages through the query results and yields each row as a {@link BackfillEvent}.
15
15
  */
@@ -10,7 +10,7 @@ export const BIGQUERY_SOURCE_REGEX = /^bq:\/\/(?<tableId>.+)$/;
10
10
  * Two cases are supported:
11
11
  * - The `source` is a `bq://<projectId>.<datasetId>.<tableId>` URI.
12
12
  * - The `source` is omitted and the broker is `google.pubSub` with `google.pubSub.bigQueryStorage.rawEventsDatasetId`
13
- * configured: events are read from the default BigQuery storage for the topic.
13
+ * configured: events are read from the default BigQuery storage for the topic, once per Pub/Sub message ID.
14
14
  *
15
15
  * The returned async iterable pages through the query results and yields each row as a {@link BackfillEvent}.
16
16
  */
@@ -8,5 +8,6 @@ export { EventTopicBrokerDeleteTriggerResourceForServiceAccount } from './broker
8
8
  export { EventTopicBrokerGetTopicIdForPubSub } from './broker-get-topic-id-pubsub.js';
9
9
  export { EventTopicBrokerPublishEventsForGoogle } from './broker-publish-events-google.js';
10
10
  export { EventTopicBrokerWaitForProcessingForPubSub } from './broker-wait-for-processing-pubsub.js';
11
+ export { EventTopicCountEventsForBigQuery } from './count-events-bigquery.js';
11
12
  export { EventTopicCreateBackfillSourceForBigQuery } from './create-backfill-source-bigquery.js';
12
13
  export { EventTopicQueryEventsForBigQuery } from './query-events-bigquery.js';
@@ -8,5 +8,6 @@ export { EventTopicBrokerDeleteTriggerResourceForServiceAccount } from './broker
8
8
  export { EventTopicBrokerGetTopicIdForPubSub } from './broker-get-topic-id-pubsub.js';
9
9
  export { EventTopicBrokerPublishEventsForGoogle } from './broker-publish-events-google.js';
10
10
  export { EventTopicBrokerWaitForProcessingForPubSub } from './broker-wait-for-processing-pubsub.js';
11
+ export { EventTopicCountEventsForBigQuery } from './count-events-bigquery.js';
11
12
  export { EventTopicCreateBackfillSourceForBigQuery } from './create-backfill-source-bigquery.js';
12
13
  export { EventTopicQueryEventsForBigQuery } from './query-events-bigquery.js';
@@ -24,10 +24,12 @@ export default async function call() {
24
24
  if (this.filter) {
25
25
  conditions.push(`(${this.filter})`);
26
26
  }
27
+ // Events written several times to the table by the BigQuery subscription are only returned once.
27
28
  const query = `
28
29
  SELECT message_id, publish_time, data, attributes
29
30
  FROM \`${tableId}\`
30
31
  WHERE ${conditions.join(' AND ')}
32
+ QUALIFY ROW_NUMBER() OVER (PARTITION BY message_id ORDER BY publish_time) = 1
31
33
  ORDER BY publish_time ASC
32
34
  LIMIT ${limit}`;
33
35
  this._context.logger.debug(`Querying BigQuery for events:\n${query}`);
@@ -2,6 +2,7 @@ import { EventTopicQueryEvents, type QueriedEvent } from '@causa/workspace-core'
2
2
  /**
3
3
  * Implements {@link EventTopicQueryEvents} for Pub/Sub topics whose events are stored in BigQuery via a BigQuery
4
4
  * subscription. The events are read from the dataset configured at `google.pubSub.bigQueryStorage.rawEventsDatasetId`.
5
+ * Events are returned once per Pub/Sub message ID.
5
6
  */
6
7
  export declare class EventTopicQueryEventsForBigQuery extends EventTopicQueryEvents {
7
8
  _call(): Promise<QueriedEvent[]>;
@@ -3,6 +3,7 @@ import { EventTopicQueryEvents, } from '@causa/workspace-core';
3
3
  /**
4
4
  * Implements {@link EventTopicQueryEvents} for Pub/Sub topics whose events are stored in BigQuery via a BigQuery
5
5
  * subscription. The events are read from the dataset configured at `google.pubSub.bigQueryStorage.rawEventsDatasetId`.
6
+ * Events are returned once per Pub/Sub message ID.
6
7
  */
7
8
  export class EventTopicQueryEventsForBigQuery extends EventTopicQueryEvents {
8
9
  async _call() {
@@ -9,6 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  };
10
10
  import { CliCommand, CliOption } from '@causa/cli';
11
11
  import { callDeferred, WorkspaceFunction } from '@causa/workspace';
12
+ import { PassArgumentsByReference } from '@causa/workspace/function-registry';
12
13
  import { AllowMissing } from '@causa/workspace/validation';
13
14
  import { IsString } from 'class-validator';
14
15
  import { appCheckCommandDefinition } from '../../cli/index.js';
@@ -48,6 +49,7 @@ If the Firebase app ID is not specified, it will:
48
49
  - If not present in the configuration, will be automatically looked up from the API.`,
49
50
  summary: 'Generates an AppCheck token.',
50
51
  outputFn: (token) => console.log(token),
51
- })
52
+ }),
53
+ PassArgumentsByReference()
52
54
  ], GoogleAppCheckGenerateToken);
53
55
  export { GoogleAppCheckGenerateToken };
@@ -9,6 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  };
10
10
  import { CliCommand } from '@causa/cli';
11
11
  import { WorkspaceFunction } from '@causa/workspace';
12
+ import { PassArgumentsByReference } from '@causa/workspace/function-registry';
12
13
  import { AllowMissing } from '@causa/workspace/validation';
13
14
  import { IsBoolean } from 'class-validator';
14
15
  import { firebaseStorageCommandDefinition } from '../../cli/index.js';
@@ -67,6 +68,7 @@ GoogleFirebaseStorageMergeRules = __decorate([
67
68
  Input files are looked for in the workspace using the globs defined in google.firebaseStorage.securityRulesFiles.`,
68
69
  summary: 'Merge Firebase Storage security rules into a single file.',
69
70
  outputFn: ({ securityRuleFile }) => console.log(securityRuleFile),
70
- })
71
+ }),
72
+ PassArgumentsByReference()
71
73
  ], GoogleFirebaseStorageMergeRules);
72
74
  export { GoogleFirebaseStorageMergeRules };
@@ -9,6 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  };
10
10
  import { CliCommand } from '@causa/cli';
11
11
  import { WorkspaceFunction } from '@causa/workspace';
12
+ import { PassArgumentsByReference } from '@causa/workspace/function-registry';
12
13
  import { AllowMissing } from '@causa/workspace/validation';
13
14
  import { IsBoolean } from 'class-validator';
14
15
  import { firestoreCommandDefinition } from '../../cli/index.js';
@@ -67,6 +68,7 @@ GoogleFirestoreMergeRules = __decorate([
67
68
  Input files are looked for in the workspace using the globs defined in google.firestore.securityRulesFiles.`,
68
69
  summary: 'Merge Firestore security rules into a single file.',
69
70
  outputFn: ({ securityRuleFile }) => console.log(securityRuleFile),
70
- })
71
+ }),
72
+ PassArgumentsByReference()
71
73
  ], GoogleFirestoreMergeRules);
72
74
  export { GoogleFirestoreMergeRules };
@@ -8,6 +8,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
8
8
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
9
  };
10
10
  import { callDeferred, WorkspaceFunction } from '@causa/workspace';
11
+ import { PassArgumentsByReference } from '@causa/workspace/function-registry';
11
12
  import { AllowMissing } from '@causa/workspace/validation';
12
13
  import { IsObject, IsString } from 'class-validator';
13
14
  /**
@@ -17,7 +18,7 @@ import { IsObject, IsString } from 'class-validator';
17
18
  * If `google.firebase.adminServiceAccount` is set, the service account will be used to sign the token. Otherwise, an
18
19
  * attempt will be made to find a valid service account in the `google.project`.
19
20
  */
20
- export class GoogleIdentityPlatformGenerateCustomToken extends WorkspaceFunction {
21
+ let GoogleIdentityPlatformGenerateCustomToken = class GoogleIdentityPlatformGenerateCustomToken extends WorkspaceFunction {
21
22
  /**
22
23
  * The ID of the user for which the custom token should be generated.
23
24
  */
@@ -32,7 +33,7 @@ export class GoogleIdentityPlatformGenerateCustomToken extends WorkspaceFunction
32
33
  _supports() {
33
34
  return true;
34
35
  }
35
- }
36
+ };
36
37
  __decorate([
37
38
  IsString(),
38
39
  __metadata("design:type", String)
@@ -42,3 +43,7 @@ __decorate([
42
43
  AllowMissing(),
43
44
  __metadata("design:type", Object)
44
45
  ], GoogleIdentityPlatformGenerateCustomToken.prototype, "claims", void 0);
46
+ GoogleIdentityPlatformGenerateCustomToken = __decorate([
47
+ PassArgumentsByReference()
48
+ ], GoogleIdentityPlatformGenerateCustomToken);
49
+ export { GoogleIdentityPlatformGenerateCustomToken };
@@ -9,6 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  };
10
10
  import { WorkspaceFunction } from '@causa/workspace';
11
11
  import { EventTopicList, } from '@causa/workspace-core';
12
+ import { PassArgumentsByReference } from '@causa/workspace/function-registry';
12
13
  import { CAUSA_FOLDER } from '@causa/workspace/initialization';
13
14
  import { AllowMissing } from '@causa/workspace/validation';
14
15
  import { IsBoolean } from 'class-validator';
@@ -34,7 +35,7 @@ export function rawEventsTableName(topic) {
34
35
  * topic configurations.
35
36
  * This function returns a partial configuration, such that it can be used as a processor.
36
37
  */
37
- export class GooglePubSubWriteTopics extends WorkspaceFunction {
38
+ let GooglePubSubWriteTopics = class GooglePubSubWriteTopics extends WorkspaceFunction {
38
39
  tearDown;
39
40
  /**
40
41
  * Returns the path to the directory where Pub/Sub topic configurations should be written.
@@ -80,9 +81,13 @@ export class GooglePubSubWriteTopics extends WorkspaceFunction {
80
81
  _supports() {
81
82
  return true;
82
83
  }
83
- }
84
+ };
84
85
  __decorate([
85
86
  IsBoolean(),
86
87
  AllowMissing(),
87
88
  __metadata("design:type", Boolean)
88
89
  ], GooglePubSubWriteTopics.prototype, "tearDown", void 0);
90
+ GooglePubSubWriteTopics = __decorate([
91
+ PassArgumentsByReference()
92
+ ], GooglePubSubWriteTopics);
93
+ export { GooglePubSubWriteTopics };
@@ -9,6 +9,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
9
9
  };
10
10
  import { CliCommand } from '@causa/cli';
11
11
  import { callDeferred, WorkspaceFunction } from '@causa/workspace';
12
+ import { PassArgumentsByReference } from '@causa/workspace/function-registry';
12
13
  import { AllowMissing } from '@causa/workspace/validation';
13
14
  import { IsBoolean } from 'class-validator';
14
15
  import { googleCommandDefinition } from '../../cli/index.js';
@@ -38,6 +39,7 @@ Required GCP services should be defined in the 'google.services' configuration.
38
39
  They will be enabled in the 'google.project' GCP project.`,
39
40
  summary: 'Enables the GCP services used by the current project.',
40
41
  outputFn: ({ services }) => console.log(services.join('\n')),
41
- })
42
+ }),
43
+ PassArgumentsByReference()
42
44
  ], GoogleServicesEnable);
43
45
  export { GoogleServicesEnable };
@@ -0,0 +1,16 @@
1
+ import type { ObjectSchema, RecordCondition, RecordCounts } from '@causa/workspace-core';
2
+ import type { DatabaseCountRecordsForSpanner } from './count-records.js';
3
+ /**
4
+ * Builds the SQL query counting the rows of a table, and those matching each filter.
5
+ *
6
+ * @param table The name of the table.
7
+ * @param schema The schema persisted in the table.
8
+ * @param filters The conditions of each filter, in order.
9
+ * @returns The SQL query and its parameters. The counts are returned in the `total` column, and the `f<index>` column
10
+ * for each filter.
11
+ */
12
+ export declare function buildCountQuery(table: string, schema: ObjectSchema, filters: RecordCondition[][]): {
13
+ sql: string;
14
+ params: Record<string, string | number | boolean>;
15
+ };
16
+ export default function call(this: DatabaseCountRecordsForSpanner): Promise<RecordCounts>;
@@ -0,0 +1,120 @@
1
+ import { splitSchemaPath } from '@causa/workspace-core/jsonschema';
2
+ import { Spanner } from '@google-cloud/spanner';
3
+ import { GoogleSpannerListDatabases } from './list-databases.js';
4
+ import { SPANNER_ENGINE } from './query-records.js';
5
+ import { resolveSpannerTable } from './resolve-table.js';
6
+ /**
7
+ * Finds the database whose DDL creates a table, using {@link resolveSpannerTable}.
8
+ *
9
+ * @param context The {@link WorkspaceContext}.
10
+ * @param table The name of the table.
11
+ * @param schema The schema persisted in the table.
12
+ * @returns The ID of the database.
13
+ */
14
+ async function findDatabase(context, table, schema) {
15
+ const [databases, domainPaths] = await Promise.all([
16
+ context.call(GoogleSpannerListDatabases, {}),
17
+ context.listDomainPaths(),
18
+ ]);
19
+ const { file } = splitSchemaPath(schema.path);
20
+ const resolution = resolveSpannerTable(databases, table, file, domainPaths);
21
+ switch (resolution.kind) {
22
+ case 'resolved':
23
+ return resolution.database.id;
24
+ case 'missing':
25
+ throw new Error(`No Spanner DDL creates the table '${table}'.`);
26
+ case 'ambiguous':
27
+ throw new Error(`The table '${table}' is created in several Spanner databases: ${resolution.candidates
28
+ .map(({ id }) => `'${id}'`)
29
+ .join(', ')}.`);
30
+ }
31
+ }
32
+ /**
33
+ * Returns the SQL expression reading the value at a path in the records of a schema.
34
+ *
35
+ * @param schema The schema persisted in the table.
36
+ * @param path The names of the properties leading to the value.
37
+ * @returns The SQL expression, and the JSON value when it reads a nested value from a JSON column.
38
+ */
39
+ function valueExpression(schema, path) {
40
+ const [name, ...nested] = path;
41
+ const property = schema.properties.find((p) => p.name === name);
42
+ if (!property) {
43
+ throw new Error(`'${name}' is not a property of '${schema.name}'.`);
44
+ }
45
+ const columnExtension = property.extensions.googleSpannerColumn;
46
+ const column = typeof columnExtension?.name === 'string' ? columnExtension.name : name;
47
+ const expression = `\`${column}\``;
48
+ if (nested.length === 0) {
49
+ return { expression };
50
+ }
51
+ const json = `${expression}${nested.map((key) => `[${JSON.stringify(key)}]`).join('')}`;
52
+ return { expression: `JSON_VALUE(${json})`, json };
53
+ }
54
+ /**
55
+ * Builds the SQL query counting the rows of a table, and those matching each filter.
56
+ *
57
+ * @param table The name of the table.
58
+ * @param schema The schema persisted in the table.
59
+ * @param filters The conditions of each filter, in order.
60
+ * @returns The SQL query and its parameters. The counts are returned in the `total` column, and the `f<index>` column
61
+ * for each filter.
62
+ */
63
+ export function buildCountQuery(table, schema, filters) {
64
+ const params = {};
65
+ const condition = (c) => {
66
+ const { expression, json } = valueExpression(schema, c.path);
67
+ if (c.operator !== 'equals') {
68
+ return `${expression} ${c.operator === 'isNull' ? 'IS NULL' : 'IS NOT NULL'}`;
69
+ }
70
+ const param = `p${Object.keys(params).length}`;
71
+ if (!json) {
72
+ params[param] = c.value;
73
+ return `${expression} = @${param}`;
74
+ }
75
+ // Nested values are read as strings from JSON columns, and only match values of the same JSON type.
76
+ const type = `JSON_TYPE(${json}) = '${typeof c.value}'`;
77
+ if (typeof c.value === 'number') {
78
+ params[param] = c.value;
79
+ return `${type} AND SAFE_CAST(${expression} AS FLOAT64) = @${param}`;
80
+ }
81
+ params[param] = String(c.value);
82
+ return `${type} AND ${expression} = @${param}`;
83
+ };
84
+ const counts = filters.map((conditions, index) => `COUNTIF(${conditions.map(condition).join(' AND ') || 'TRUE'}) AS f${index}`);
85
+ const sql = `SELECT ${['COUNT(*) AS total', ...counts].join(', ')} FROM \`${table}\``;
86
+ return { sql, params };
87
+ }
88
+ export default async function call() {
89
+ const table = this.schema.databases.find((d) => d.engine === SPANNER_ENGINE)?.table;
90
+ if (!table) {
91
+ throw new Error(`The schema '${this.schema.name}' is not persisted in a Spanner table.`);
92
+ }
93
+ const names = Object.keys(this.filters);
94
+ const { sql, params } = buildCountQuery(table, this.schema, names.map((n) => this.filters[n]));
95
+ const databaseId = await findDatabase(this._context, table, this.schema);
96
+ const googleConf = this._context.asConfiguration();
97
+ const projectId = googleConf.getOrThrow('google.project');
98
+ const instanceName = googleConf.getOrThrow('google.spanner.instance.name');
99
+ this._context.logger.debug(`🗃️ Counting rows in Spanner database '${databaseId}' in instance '${instanceName}' (project '${projectId}'):\n${sql}`);
100
+ const spanner = new Spanner({ projectId });
101
+ let database;
102
+ try {
103
+ database = spanner.instance(instanceName).database(databaseId);
104
+ const [rows] = await database.run({
105
+ sql,
106
+ params,
107
+ json: true,
108
+ jsonOptions: { wrapNumbers: false },
109
+ });
110
+ const row = rows[0];
111
+ return {
112
+ total: Number(row.total),
113
+ counts: Object.fromEntries(names.map((name, index) => [name, Number(row[`f${index}`])])),
114
+ };
115
+ }
116
+ finally {
117
+ await database?.close();
118
+ spanner.close();
119
+ }
120
+ }
@@ -0,0 +1,15 @@
1
+ import { DatabaseCountRecords, type RecordCounts } from '@causa/workspace-core';
2
+ /**
3
+ * Implements {@link DatabaseCountRecords} for Spanner databases.
4
+ *
5
+ * The table is the one in the schema's `google.spanner` binding, in the database whose DDL creates it. When several
6
+ * databases create a table with that name, the one in the domain of the schema is chosen. The Spanner instance and GCP
7
+ * project are read from the workspace configuration.
8
+ *
9
+ * Properties are read from the column named by their `googleSpannerColumn` extension, or from the column with the same
10
+ * name. Nested properties are read from JSON columns.
11
+ */
12
+ export declare class DatabaseCountRecordsForSpanner extends DatabaseCountRecords {
13
+ _call(): Promise<RecordCounts>;
14
+ _supports(): boolean;
15
+ }
@@ -0,0 +1,21 @@
1
+ import { callDeferred } from '@causa/workspace';
2
+ import { DatabaseCountRecords } from '@causa/workspace-core';
3
+ import { SPANNER_ENGINE } from './query-records.js';
4
+ /**
5
+ * Implements {@link DatabaseCountRecords} for Spanner databases.
6
+ *
7
+ * The table is the one in the schema's `google.spanner` binding, in the database whose DDL creates it. When several
8
+ * databases create a table with that name, the one in the domain of the schema is chosen. The Spanner instance and GCP
9
+ * project are read from the workspace configuration.
10
+ *
11
+ * Properties are read from the column named by their `googleSpannerColumn` extension, or from the column with the same
12
+ * name. Nested properties are read from JSON columns.
13
+ */
14
+ export class DatabaseCountRecordsForSpanner extends DatabaseCountRecords {
15
+ async _call() {
16
+ return await callDeferred(this, import.meta.url);
17
+ }
18
+ _supports() {
19
+ return this.engine === SPANNER_ENGINE;
20
+ }
21
+ }
@@ -1,5 +1,7 @@
1
+ export { DatabaseCountRecordsForSpanner } from './count-records.js';
1
2
  export { ModelSchemaExtractDatabaseForGoogleSpanner } from './extract-database.js';
2
3
  export { ModelGenerateTypeScriptDecoratorsForGoogleSpanner } from './generate-typescript-decorators.js';
3
- export { GoogleSpannerListDatabases } from './list-databases.js';
4
+ export { GoogleSpannerListDatabases, type SpannerDatabase, type SpannerTable, } from './list-databases.js';
4
5
  export { DatabaseQueryRecordsForSpanner } from './query-records.js';
6
+ export { resolveSpannerTable, type SpannerTableResolution, } from './resolve-table.js';
5
7
  export { GoogleSpannerWriteDatabases } from './write-databases.js';
@@ -1,5 +1,7 @@
1
+ export { DatabaseCountRecordsForSpanner } from './count-records.js';
1
2
  export { ModelSchemaExtractDatabaseForGoogleSpanner } from './extract-database.js';
2
3
  export { ModelGenerateTypeScriptDecoratorsForGoogleSpanner } from './generate-typescript-decorators.js';
3
- export { GoogleSpannerListDatabases } from './list-databases.js';
4
+ export { GoogleSpannerListDatabases, } from './list-databases.js';
4
5
  export { DatabaseQueryRecordsForSpanner } from './query-records.js';
6
+ export { resolveSpannerTable, } from './resolve-table.js';
5
7
  export { GoogleSpannerWriteDatabases } from './write-databases.js';
@@ -1,4 +1,21 @@
1
1
  import { WorkspaceFunction } from '@causa/workspace';
2
+ /**
3
+ * A table created by a `CREATE TABLE` statement in a Spanner DDL file.
4
+ */
5
+ export type SpannerTable = {
6
+ /**
7
+ * The name of the table.
8
+ */
9
+ readonly name: string;
10
+ /**
11
+ * The absolute path of the DDL file creating the table.
12
+ */
13
+ readonly file: string;
14
+ /**
15
+ * The line of the statement in the file.
16
+ */
17
+ readonly line: number;
18
+ };
2
19
  /**
3
20
  * The definition of a Spanner database with its DDL statements.
4
21
  */
@@ -15,6 +32,10 @@ export type SpannerDatabase = {
15
32
  * The DDL statements for the database.
16
33
  */
17
34
  ddls: string[];
35
+ /**
36
+ * The tables created by the `CREATE TABLE` statements of the DDL files.
37
+ */
38
+ tables: SpannerTable[];
18
39
  };
19
40
  /**
20
41
  * A function that lists Spanner databases defined in the workspace, and returns their corresponding DDLs.
@@ -26,7 +47,7 @@ export declare class GoogleSpannerListDatabases extends WorkspaceFunction<Promis
26
47
  * End-of-the-line comments are removed, and statements are split using semicolons.
27
48
  *
28
49
  * @param path The path to the SQL file containing DDL statements.
29
- * @returns The list of DDL statements.
50
+ * @returns The list of DDL statements, along with the file and the line at which each statement starts.
30
51
  */
31
52
  private parseDdlFile;
32
53
  _supports(): boolean;