@causa/workspace-google 0.14.1 → 0.16.0

package/README.md

@@ -55,7 +55,10 @@ The following emulators are implemented:
 
 ### Backfilling
 
-Backfilling is supported when `google.pubSub` is set as the `events.broker`. Temporary triggers can be created for Cloud Run services, by referencing them using the format `[[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger`.
+Backfilling is supported when `google.pubSub` is set as the `events.broker`. Temporary triggers can be created for Cloud Run services in two ways:
+
+- As a raw URI, using the format `[[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger`.
+- As a project-scoped trigger, using the format `<projectPath>#<triggerName>[?<options>]`. The referenced project must be of type `serviceContainer` with `google.cloudRun` as `serviceContainer.platform`. The HTTP path is read from `serviceContainer.triggers[<triggerName>].endpoint` (whose `type` must be `http`), and the `options` query string is forwarded as URL query parameters. The target Cloud Run service name defaults to `project.name`, and can be overridden by setting `google.cloudRun.eventBackfillServiceName`.
 
 When no source is specified, the default is to fetch events to backfill from the BigQuery dataset configured in `google.pubSub.bigQueryStorage`. A custom BigQuery table can also be set as source using the `bq://<projectId>.<datasetId>.<tableId>` format. It should have the `data` and `attributes` columns.
 

package/dist/configurations/generated.d.ts

@@ -51,8 +51,8 @@ export declare enum GoogleCloudRunIngress {
  * See https://cloud.google.com/run/docs/configuring/connecting-vpc#manage for more details.
  */
 export declare enum GoogleCloudRunVpcAccessConnectorEgressSettings {
-    AllTraffic = "all-traffic",
-    PrivateRangesOnly = "private-ranges-only"
+    AllTraffic = "ALL_TRAFFIC",
+    PrivateRangesOnly = "PRIVATE_RANGES_ONLY"
 }
 /**
  * Configuration for Cloud Run containers.
@@ -67,6 +67,12 @@ export declare class GoogleCloudRun {
      * The Docker repository where Cloud Run containers should be uploaded.
      */
     readonly dockerRepository?: string;
+    /**
+     * For `serviceContainer` projects deployed on Cloud Run, the name of the Cloud Run service that receives
+     * events during a backfill targeting a project-scoped trigger. Defaults to the project's name
+     * (`project.name`) when unset.
+     */
+    readonly eventBackfillServiceName?: string;
     /**
      * The type of allowed ingress that can reach the container.
      * See https://cloud.google.com/run/docs/securing/ingress#settings for more details.
@@ -597,9 +603,20 @@ export declare class Causa {
     readonly secrets?: Secrets;
     [property: string]: any;
 }
-export declare class Secret {
-    constructor(init: Secret);
+/**
+ * A secret that returns a Google access token for the current user or service account.
+ * This backend does not require any additional configuration.
+ */
+export declare class GoogleSecretConfiguration {
+    constructor(init: GoogleSecretConfiguration);
     readonly backend?: string;
+    /**
+     * The reference to the secret in Google Secret Manager.
+     * Supported formats: `<secret-name>`, `projects/<project>/secrets/<secret-name>`, `projects/<project>/secrets/<secret-name>/versions/<version>`.
+     * If the project is not specified, `google.secretManager.project` or `google.project` will be used.
+     * If the version is not specified, `latest` will be used.
+     */
+    readonly id?: string;
     [property: string]: any;
 }
 /**
@@ -609,7 +626,7 @@ export declare class GoogleSecretsConfiguration {
     constructor(init: GoogleSecretsConfiguration);
     readonly causa?: Causa;
     readonly secrets?: {
-        [key: string]: Secret;
+        [key: string]: GoogleSecretConfiguration;
     };
     [property: string]: any;
 }

package/dist/configurations/generated.js

@@ -92,8 +92,8 @@ export var GoogleCloudRunIngress;
  */
 export var GoogleCloudRunVpcAccessConnectorEgressSettings;
 (function (GoogleCloudRunVpcAccessConnectorEgressSettings) {
-    GoogleCloudRunVpcAccessConnectorEgressSettings["AllTraffic"] = "all-traffic";
-    GoogleCloudRunVpcAccessConnectorEgressSettings["PrivateRangesOnly"] = "private-ranges-only";
+    GoogleCloudRunVpcAccessConnectorEgressSettings["AllTraffic"] = "ALL_TRAFFIC";
+    GoogleCloudRunVpcAccessConnectorEgressSettings["PrivateRangesOnly"] = "PRIVATE_RANGES_ONLY";
 })(GoogleCloudRunVpcAccessConnectorEgressSettings || (GoogleCloudRunVpcAccessConnectorEgressSettings = {}));
 /**
  * Configuration for Cloud Run containers.
@@ -110,6 +110,12 @@ export class GoogleCloudRun {
      * The Docker repository where Cloud Run containers should be uploaded.
      */
     dockerRepository;
+    /**
+     * For `serviceContainer` projects deployed on Cloud Run, the name of the Cloud Run service that receives
+     * events during a backfill targeting a project-scoped trigger. Defaults to the project's name
+     * (`project.name`) when unset.
+     */
+    eventBackfillServiceName;
     /**
      * The type of allowed ingress that can reach the container.
      * See https://cloud.google.com/run/docs/securing/ingress#settings for more details.
@@ -156,6 +162,11 @@ __decorate([
     IsString(),
     __metadata("design:type", String)
 ], GoogleCloudRun.prototype, "dockerRepository", void 0);
+__decorate([
+    AllowMissing(),
+    IsString(),
+    __metadata("design:type", String)
+], GoogleCloudRun.prototype, "eventBackfillServiceName", void 0);
 __decorate([
     AllowMissing(),
     IsIn(['all', 'internal', 'internal-and-cloud-load-balancing']),
@@ -193,7 +204,7 @@ __decorate([
 ], GoogleCloudRun.prototype, "vpcAccessConnector", void 0);
 __decorate([
     AllowMissing(),
-    IsIn(['all-traffic', 'private-ranges-only']),
+    IsIn(['ALL_TRAFFIC', 'PRIVATE_RANGES_ONLY']),
     __metadata("design:type", String)
 ], GoogleCloudRun.prototype, "vpcAccessConnectorEgressSettings", void 0);
 /**
@@ -1073,17 +1084,33 @@ __decorate([
     AllowMissing(),
     __metadata("design:type", Secrets)
 ], Causa.prototype, "secrets", void 0);
-export class Secret {
+/**
+ * A secret that returns a Google access token for the current user or service account.
+ * This backend does not require any additional configuration.
+ */
+export class GoogleSecretConfiguration {
     constructor(init) {
         Object.assign(this, init);
     }
     backend;
+    /**
+     * The reference to the secret in Google Secret Manager.
+     * Supported formats: `<secret-name>`, `projects/<project>/secrets/<secret-name>`, `projects/<project>/secrets/<secret-name>/versions/<version>`.
+     * If the project is not specified, `google.secretManager.project` or `google.project` will be used.
+     * If the version is not specified, `latest` will be used.
+     */
+    id;
 }
 __decorate([
     AllowMissing(),
     IsString(),
     __metadata("design:type", String)
-], Secret.prototype, "backend", void 0);
+], GoogleSecretConfiguration.prototype, "backend", void 0);
+__decorate([
+    AllowMissing(),
+    IsString(),
+    __metadata("design:type", String)
+], GoogleSecretConfiguration.prototype, "id", void 0);
 /**
  * Google-specific refinement of the secrets configuration.
  */

package/dist/configurations/schemas/google.yaml

@@ -358,6 +358,16 @@ properties:
           location:
             type: string
             description: The location where Cloud Run services should be deployed.
+          eventBackfillServiceName:
+            type: string
+            description: >-
+              For `serviceContainer` projects deployed on Cloud Run, the name of
+              the Cloud Run service that receives
+
+              events during a backfill targeting a project-scoped trigger.
+              Defaults to the project's name
+
+              (`project.name`) when unset.
           secretEnvironmentVariables:
             type: object
             description: >-
@@ -404,8 +414,8 @@ properties:
             title: GoogleCloudRunVpcAccessConnectorEgressSettings
             type: string
             enum:
-              - all-traffic
-              - private-ranges-only
+              - ALL_TRAFFIC
+              - PRIVATE_RANGES_ONLY
             description: >-
               The setting for egress traffic that goes through the VPC access
               connector.

package/dist/functions/event-topic/broker-create-trigger-cloud-run.call.d.ts

@@ -1,3 +1,3 @@
 import type { WorkspaceContext } from '@causa/workspace';
-import type { EventTopicBrokerCreateTriggerForCloudRun } from './broker-create-trigger-cloud-run.js';
+import { type EventTopicBrokerCreateTriggerForCloudRun } from './broker-create-trigger-cloud-run.js';
 export default function call(this: EventTopicBrokerCreateTriggerForCloudRun, context: WorkspaceContext): Promise<string[]>;

package/dist/functions/event-topic/broker-create-trigger-cloud-run.call.js

@@ -1,23 +1,53 @@
 import { CloudRunPubSubTriggerService } from '../../services/cloud-run-pubsub-trigger.js';
+import { CLOUD_RUN_TRIGGER_ID_REGEX, } from './broker-create-trigger-cloud-run.js';
 /**
- * The regular expression used to match Cloud Run trigger IDs, consisting of the service ID and the path to the trigger.
- * [[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger
+ * Resolves the Cloud Run service ID and HTTP path for a raw URI trigger string.
  */
-const CLOUD_RUN_TRIGGER_ID_REGEX = /^(?:(?:projects\/(?<projectId>[\w-]+)\/)?locations\/(?<location>[\w-]+)\/)?services\/(?<name>[\w-]+)(?<path>\/.*)$/;
-export default async function call(context) {
-    const match = this.trigger.match(CLOUD_RUN_TRIGGER_ID_REGEX);
+function resolveFromString(context, trigger) {
+    const match = trigger.match(CLOUD_RUN_TRIGGER_ID_REGEX);
     if (!match?.groups) {
-        throw new Error('Oops.');
+        throw new Error(`Invalid Cloud Run trigger '${trigger}'.`);
     }
     const googleConf = context.asConfiguration();
     const projectId = match.groups.projectId ?? googleConf.getOrThrow('google.project');
     const location = match.groups.location ?? googleConf.getOrThrow('google.cloudRun.location');
-    const name = match.groups.name;
-    const path = match.groups.path;
-    if (!name || !path) {
-        throw new Error('Oops.');
+    return {
+        serviceId: `projects/${projectId}/locations/${location}/services/${match.groups.name}`,
+        path: match.groups.path,
+    };
+}
+/**
+ * Resolves the Cloud Run service ID and HTTP path for a project-scoped trigger object.
+ * The project is expected to be a `serviceContainer` deployed on Cloud Run. The service name defaults to the project's
+ * name, and is overridden by `google.cloudRun.eventBackfillServiceName` when set. The trigger path is read from
+ * `serviceContainer.triggers[name].endpoint`, which must be of type `http`. Options are serialized as a URL query
+ * string.
+ *
+ * @param context The {@link WorkspaceContext}.
+ * @param trigger The trigger object.
+ * @returns The resolved service ID and path.
+ */
+function resolveFromObject(context, trigger) {
+    const conf = context.asConfiguration();
+    const projectId = conf.getOrThrow('google.project');
+    const location = conf.getOrThrow('google.cloudRun.location');
+    const serviceName = conf.get('google.cloudRun.eventBackfillServiceName') ??
+        conf.getOrThrow('project.name');
+    const endpoint = conf.get(`serviceContainer.triggers.${trigger.name}.endpoint`);
+    if (!endpoint || endpoint.type !== 'http' || !endpoint.path) {
+        throw new Error(`Trigger '${trigger.name}' does not exist or does not define an HTTP endpoint.`);
     }
-    const serviceId = `projects/${projectId}/locations/${location}/services/${name}`;
+    const query = new URLSearchParams(trigger.options).toString();
+    const path = query ? `${endpoint.path}?${query}` : endpoint.path;
+    return {
+        serviceId: `projects/${projectId}/locations/${location}/services/${serviceName}`,
+        path,
+    };
+}
+export default async function call(context) {
+    const { serviceId, path } = typeof this.trigger === 'string'
+        ? resolveFromString(context, this.trigger)
+        : resolveFromObject(context, this.trigger);
     return await context
         .service(CloudRunPubSubTriggerService)
         .create(this.backfillId, this.topicId, serviceId, path);

package/dist/functions/event-topic/broker-create-trigger-cloud-run.d.ts

@@ -1,9 +1,18 @@
 import { WorkspaceContext } from '@causa/workspace';
 import { EventTopicBrokerCreateTrigger } from '@causa/workspace-core';
+/**
+ * The regular expression used to match Cloud Run trigger IDs, consisting of the service ID and the path to the trigger.
+ * [[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger
+ */
+export declare const CLOUD_RUN_TRIGGER_ID_REGEX: RegExp;
 /**
  * Implements {@link EventTopicBrokerCreateTrigger} for Cloud Run endpoints being triggered by Pub/Sub messages.
- * The `id` argument should be a Cloud Run service ID, followed by the path to the trigger, e.g.
- * `[[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger`.
+ *
+ * Two forms of trigger are supported:
+ * - A string in the format `[[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger`.
+ * - An object `{ name, options }` passed when the function is called on a project-scoped context. In that case the
+ *   project must be a `serviceContainer` deployed on Cloud Run. The trigger's `name` must match an entry in
+ *   `serviceContainer.triggers` whose endpoint is of type `http`. Options are forwarded as URL query parameters.
  */
 export declare class EventTopicBrokerCreateTriggerForCloudRun extends EventTopicBrokerCreateTrigger {
     _call(context: WorkspaceContext): Promise<string[]>;

package/dist/functions/event-topic/broker-create-trigger-cloud-run.js

@@ -4,19 +4,31 @@ import { EventTopicBrokerCreateTrigger, } from '@causa/workspace-core';
  * The regular expression used to match Cloud Run trigger IDs, consisting of the service ID and the path to the trigger.
  * [[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger
  */
-const CLOUD_RUN_TRIGGER_ID_REGEX = /^(?:(?:projects\/(?<projectId>[\w-]+)\/)?locations\/(?<location>[\w-]+)\/)?services\/(?<name>[\w-]+)(?<path>\/.*)$/;
+export const CLOUD_RUN_TRIGGER_ID_REGEX = /^(?:(?:projects\/(?<projectId>[\w-]+)\/)?locations\/(?<location>[\w-]+)\/)?services\/(?<name>[\w-]+)(?<path>\/.*)$/;
 /**
  * Implements {@link EventTopicBrokerCreateTrigger} for Cloud Run endpoints being triggered by Pub/Sub messages.
- * The `id` argument should be a Cloud Run service ID, followed by the path to the trigger, e.g.
- * `[[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger`.
+ *
+ * Two forms of trigger are supported:
+ * - A string in the format `[[projects/<projectId>/]locations/<location>/]services/<name>/path-to-trigger`.
+ * - An object `{ name, options }` passed when the function is called on a project-scoped context. In that case the
+ *   project must be a `serviceContainer` deployed on Cloud Run. The trigger's `name` must match an entry in
+ *   `serviceContainer.triggers` whose endpoint is of type `http`. Options are forwarded as URL query parameters.
  */
 export class EventTopicBrokerCreateTriggerForCloudRun extends EventTopicBrokerCreateTrigger {
     async _call(context) {
         return await callDeferred(this, context, import.meta.url);
     }
     _supports(context) {
-        return (this.trigger.match(CLOUD_RUN_TRIGGER_ID_REGEX) != null &&
-            context.asConfiguration().get('events.broker') ===
-                'google.pubSub');
+        const isPubSub = context.asConfiguration().get('events.broker') ===
+            'google.pubSub';
+        if (!isPubSub) {
+            return false;
+        }
+        if (typeof this.trigger === 'string') {
+            return CLOUD_RUN_TRIGGER_ID_REGEX.test(this.trigger);
+        }
+        const conf = context.asConfiguration();
+        return (conf.get('project.type') === 'serviceContainer' &&
+            conf.get('serviceContainer.platform') === 'google.cloudRun');
     }
 }

package/dist/functions/event-topic/broker-publish-events-google.call.js

@@ -1,29 +1,45 @@
-import { JsonFilesEventSource, } from '@causa/workspace-core/backfill';
-import { BigQueryEventsSource, PubSubBackfillEventPublisher, } from '../../backfilling/index.js';
-async function createSource(self, context) {
-    if (!self.source) {
-        const googleConf = context.asConfiguration();
-        const projectId = googleConf.getOrThrow('google.project');
-        const rawEventsDatasetId = googleConf.get('google.pubSub.bigQueryStorage.rawEventsDatasetId');
-        if (!rawEventsDatasetId) {
-            throw new Error('Cannot use the default event source because BigQuery storage is not configured.');
+import { FlowControlledPublisher } from '@google-cloud/pubsub/build/src/publisher/flow-publisher.js';
+import { PubSubService } from '../../services/index.js';
+/**
+ * The number of events between each progress log line.
+ */
+const PUBLISH_PROGRESS_LOG_INTERVAL = 10000;
+/**
+ * Options when publishing events in batches to Pub/Sub.
+ */
+const PUBLISH_OPTIONS = {
+    flowControlOptions: {
+        maxOutstandingBytes: 10 * 1024 * 1024,
+        maxOutstandingMessages: 1000,
+    },
+    batching: {
+        maxBytes: 10 * 1024 * 1024,
+        maxMessages: 1000,
+        maxMilliseconds: 1000,
+    },
+};
+export default async function call(context) {
+    const publisher = context
+        .service(PubSubService)
+        .pubSub.topic(this.topicId, PUBLISH_OPTIONS)
+        .flowControlled();
+    context.logger.info('📫 Publishing events.');
+    let numEvents = 0;
+    for await (const event of this.source()) {
+        numEvents += 1;
+        const wait = publisher.publish({
+            data: event.data,
+            attributes: event.attributes,
+            orderingKey: event.key,
+        });
+        if (wait) {
+            context.logger.debug('Waiting for the publisher to catch up before resuming publishing.');
+            await wait;
+        }
+        if (numEvents % PUBLISH_PROGRESS_LOG_INTERVAL === 0) {
+            context.logger.debug(`Published ${numEvents} events.`);
         }
-        const tableName = self.eventTopic.replace(/[-\.]/g, '_');
-        const tableId = `${projectId}.${rawEventsDatasetId}.${tableName}`;
-        return new BigQueryEventsSource(context, tableId);
-    }
-    const bqSource = await BigQueryEventsSource.fromSourceAndFilter(context, self.source, self.filter);
-    if (bqSource) {
-        return bqSource;
-    }
-    const jsonFilesSource = await JsonFilesEventSource.fromSourceAndFilter(context, self.source, self.filter);
-    if (jsonFilesSource) {
-        return jsonFilesSource;
     }
-    throw new Error(`The event source '${self.source}' is not supported.`);
-}
-export default async function call(context) {
-    const source = await createSource(this, context);
-    const publisher = new PubSubBackfillEventPublisher(context, this.topicId);
-    await publisher.publishFromSource(source);
+    await publisher.all();
+    context.logger.info(`📫 Finished publishing ${numEvents} events.`);
 }

package/dist/functions/event-topic/create-backfill-source-bigquery.call.d.ts

@@ -0,0 +1,4 @@
+import type { WorkspaceContext } from '@causa/workspace';
+import type { BackfillEvent } from '@causa/workspace-core';
+import { type EventTopicCreateBackfillSourceForBigQuery } from './create-backfill-source-bigquery.js';
+export default function call(this: EventTopicCreateBackfillSourceForBigQuery, context: WorkspaceContext): Promise<AsyncIterable<BackfillEvent>>;

package/dist/functions/event-topic/create-backfill-source-bigquery.call.js

@@ -0,0 +1,48 @@
+import { BigQueryService } from '../../services/index.js';
+import { BIGQUERY_SOURCE_REGEX, } from './create-backfill-source-bigquery.js';
+/**
+ * Resolves the full BigQuery table ID from the function arguments and configuration.
+ *
+ * @param self The function instance.
+ * @param context The {@link WorkspaceContext}.
+ * @returns The full BigQuery table ID, in the format `projectId.datasetId.tableId`.
+ */
+function resolveTableId(self, context) {
+    if (self.source) {
+        const match = self.source.match(BIGQUERY_SOURCE_REGEX);
+        if (!match?.groups?.tableId) {
+            throw new Error(`The source '${self.source}' is not a valid BigQuery source.`);
+        }
+        return match.groups.tableId;
+    }
+    const googleConf = context.asConfiguration();
+    const projectId = googleConf.getOrThrow('google.project');
+    const rawEventsDatasetId = googleConf.getOrThrow('google.pubSub.bigQueryStorage.rawEventsDatasetId');
+    const tableName = self.eventTopic.replace(/[-\.]/g, '_');
+    return `${projectId}.${rawEventsDatasetId}.${tableName}`;
+}
+export default async function call(context) {
+    const tableId = resolveTableId(this, context);
+    let query = `
+    SELECT
+      data,
+      attributes,
+    FROM
+      \`${tableId}\``;
+    if (this.filter) {
+        query += `
+    WHERE
+      (${this.filter})`;
+    }
+    context.logger.debug(`Creating BigQuery job from query '${query}'.`);
+    const bigQuery = context.service(BigQueryService).bigQuery;
+    const [job] = await bigQuery.createQueryJob(query);
+    return (async function* () {
+        for await (const row of job.getQueryResultsStream()) {
+            yield {
+                data: Buffer.from(row.data),
+                attributes: JSON.parse(row.attributes),
+            };
+        }
+    })();
+}

package/dist/functions/event-topic/create-backfill-source-bigquery.d.ts

@@ -0,0 +1,20 @@
+import { WorkspaceContext } from '@causa/workspace';
+import { type BackfillEvent, EventTopicCreateBackfillSource } from '@causa/workspace-core';
+/**
+ * The regular expression used to parse a `bq://<tableId>` source string.
+ */
+export declare const BIGQUERY_SOURCE_REGEX: RegExp;
+/**
+ * Implements {@link EventTopicCreateBackfillSource} for BigQuery tables.
+ *
+ * Two cases are supported:
+ * - The `source` is a `bq://<projectId>.<datasetId>.<tableId>` URI.
+ * - The `source` is omitted and the broker is `google.pubSub` with `google.pubSub.bigQueryStorage.rawEventsDatasetId`
+ *   configured: events are read from the default BigQuery storage for the topic.
+ *
+ * The returned async iterable pages through the query results and yields each row as a {@link BackfillEvent}.
+ */
+export declare class EventTopicCreateBackfillSourceForBigQuery extends EventTopicCreateBackfillSource {
+    _call(context: WorkspaceContext): Promise<AsyncIterable<BackfillEvent>>;
+    _supports(context: WorkspaceContext): boolean;
+}

package/dist/functions/event-topic/create-backfill-source-bigquery.js

@@ -0,0 +1,32 @@
+import { callDeferred, WorkspaceContext } from '@causa/workspace';
+import { EventTopicCreateBackfillSource, } from '@causa/workspace-core';
+/**
+ * The regular expression used to parse a `bq://<tableId>` source string.
+ */
+export const BIGQUERY_SOURCE_REGEX = /^bq:\/\/(?<tableId>.+)$/;
+/**
+ * Implements {@link EventTopicCreateBackfillSource} for BigQuery tables.
+ *
+ * Two cases are supported:
+ * - The `source` is a `bq://<projectId>.<datasetId>.<tableId>` URI.
+ * - The `source` is omitted and the broker is `google.pubSub` with `google.pubSub.bigQueryStorage.rawEventsDatasetId`
+ *   configured: events are read from the default BigQuery storage for the topic.
+ *
+ * The returned async iterable pages through the query results and yields each row as a {@link BackfillEvent}.
+ */
+export class EventTopicCreateBackfillSourceForBigQuery extends EventTopicCreateBackfillSource {
+    async _call(context) {
+        return await callDeferred(this, context, import.meta.url);
+    }
+    _supports(context) {
+        if (this.source?.match(BIGQUERY_SOURCE_REGEX)) {
+            return true;
+        }
+        if (this.source) {
+            return false;
+        }
+        const conf = context.asConfiguration();
+        return (conf.get('events.broker') === 'google.pubSub' &&
+            !!conf.get('google.pubSub.bigQueryStorage.rawEventsDatasetId'));
+    }
+}

package/dist/functions/event-topic/index.d.ts

@@ -6,3 +6,4 @@ export { EventTopicBrokerDeleteTriggerResourceForPubSubSubscription } from './br
 export { EventTopicBrokerDeleteTriggerResourceForServiceAccount } from './broker-delete-trigger-resource-service-account.js';
 export { EventTopicBrokerGetTopicIdForPubSub } from './broker-get-topic-id-pubsub.js';
 export { EventTopicBrokerPublishEventsForGoogle } from './broker-publish-events-google.js';
+export { EventTopicCreateBackfillSourceForBigQuery } from './create-backfill-source-bigquery.js';

package/dist/functions/event-topic/index.js

@@ -6,3 +6,4 @@ export { EventTopicBrokerDeleteTriggerResourceForPubSubSubscription } from './br
 export { EventTopicBrokerDeleteTriggerResourceForServiceAccount } from './broker-delete-trigger-resource-service-account.js';
 export { EventTopicBrokerGetTopicIdForPubSub } from './broker-get-topic-id-pubsub.js';
 export { EventTopicBrokerPublishEventsForGoogle } from './broker-publish-events-google.js';
+export { EventTopicCreateBackfillSourceForBigQuery } from './create-backfill-source-bigquery.js';

package/dist/functions/index.js

@@ -1,6 +1,6 @@
 import { CausaListConfigurationSchemasForGoogle } from './causa/index.js';
 import { EmulatorStartForFirebaseStorage, EmulatorStartForFirestore, EmulatorStartForIdentityPlatform, EmulatorStartForPubSub, EmulatorStartForSpanner, EmulatorStopForFirebaseStorage, EmulatorStopForFirestore, EmulatorStopForIdentityPlatform, EmulatorStopForPubSub, EmulatorStopForSpanner, } from './emulator/index.js';
-import { EventTopicBrokerCreateTopicForPubSub, EventTopicBrokerCreateTriggerForCloudRun, EventTopicBrokerDeleteTopicForPubSub, EventTopicBrokerDeleteTriggerResourceForCloudRunInvokerRole, EventTopicBrokerDeleteTriggerResourceForPubSubSubscription, EventTopicBrokerDeleteTriggerResourceForServiceAccount, EventTopicBrokerGetTopicIdForPubSub, EventTopicBrokerPublishEventsForGoogle, } from './event-topic/index.js';
+import { EventTopicBrokerCreateTopicForPubSub, EventTopicBrokerCreateTriggerForCloudRun, EventTopicBrokerDeleteTopicForPubSub, EventTopicBrokerDeleteTriggerResourceForCloudRunInvokerRole, EventTopicBrokerDeleteTriggerResourceForPubSubSubscription, EventTopicBrokerDeleteTriggerResourceForServiceAccount, EventTopicBrokerGetTopicIdForPubSub, EventTopicBrokerPublishEventsForGoogle, EventTopicCreateBackfillSourceForBigQuery, } from './event-topic/index.js';
 import { GoogleAppCheckGenerateToken } from './google-app-check/index.js';
 import { GoogleFirebaseStorageMergeRules } from './google-firebase-storage/index.js';
 import { GoogleFirestoreMergeRules } from './google-firestore/index.js';
@@ -12,5 +12,5 @@ import { ProjectGetArtefactDestinationForCloudFunctions, ProjectGetArtefactDesti
 import { SecretFetchForGoogleAccessToken, SecretFetchForGoogleSecretManager, } from './secret/index.js';
 import { TypeScriptGetDecoratorRendererForGoogleFirestore, TypeScriptGetDecoratorRendererForGoogleSpanner, } from './typescript/index.js';
 export function registerFunctions(context) {
-    context.registerFunctionImplementations(CausaListConfigurationSchemasForGoogle, EmulatorStartForFirebaseStorage, EmulatorStartForFirestore, EmulatorStartForIdentityPlatform, EmulatorStartForPubSub, EmulatorStartForSpanner, EmulatorStopForFirebaseStorage, EmulatorStopForFirestore, EmulatorStopForIdentityPlatform, EmulatorStopForPubSub, EmulatorStopForSpanner, EventTopicBrokerCreateTopicForPubSub, EventTopicBrokerCreateTriggerForCloudRun, EventTopicBrokerDeleteTopicForPubSub, EventTopicBrokerDeleteTriggerResourceForCloudRunInvokerRole, EventTopicBrokerDeleteTriggerResourceForPubSubSubscription, EventTopicBrokerDeleteTriggerResourceForServiceAccount, EventTopicBrokerGetTopicIdForPubSub, EventTopicBrokerPublishEventsForGoogle, GoogleAppCheckGenerateToken, GoogleFirebaseStorageMergeRules, GoogleFirestoreMergeRules, GoogleIdentityPlatformGenerateCustomToken, GoogleIdentityPlatformGenerateToken, GooglePubSubWriteTopics, GoogleServicesEnable, GoogleSpannerListDatabases, GoogleSpannerWriteDatabases, ProjectGetArtefactDestinationForCloudFunctions, ProjectGetArtefactDestinationForCloudRun, ProjectPushArtefactForCloudFunctions, SecretFetchForGoogleAccessToken, SecretFetchForGoogleSecretManager, TypeScriptGetDecoratorRendererForGoogleFirestore, TypeScriptGetDecoratorRendererForGoogleSpanner);
+    context.registerFunctionImplementations(CausaListConfigurationSchemasForGoogle, EmulatorStartForFirebaseStorage, EmulatorStartForFirestore, EmulatorStartForIdentityPlatform, EmulatorStartForPubSub, EmulatorStartForSpanner, EmulatorStopForFirebaseStorage, EmulatorStopForFirestore, EmulatorStopForIdentityPlatform, EmulatorStopForPubSub, EmulatorStopForSpanner, EventTopicBrokerCreateTopicForPubSub, EventTopicBrokerCreateTriggerForCloudRun, EventTopicBrokerDeleteTopicForPubSub, EventTopicBrokerDeleteTriggerResourceForCloudRunInvokerRole, EventTopicBrokerDeleteTriggerResourceForPubSubSubscription, EventTopicBrokerDeleteTriggerResourceForServiceAccount, EventTopicBrokerGetTopicIdForPubSub, EventTopicBrokerPublishEventsForGoogle, EventTopicCreateBackfillSourceForBigQuery, GoogleAppCheckGenerateToken, GoogleFirebaseStorageMergeRules, GoogleFirestoreMergeRules, GoogleIdentityPlatformGenerateCustomToken, GoogleIdentityPlatformGenerateToken, GooglePubSubWriteTopics, GoogleServicesEnable, GoogleSpannerListDatabases, GoogleSpannerWriteDatabases, ProjectGetArtefactDestinationForCloudFunctions, ProjectGetArtefactDestinationForCloudRun, ProjectPushArtefactForCloudFunctions, SecretFetchForGoogleAccessToken, SecretFetchForGoogleSecretManager, TypeScriptGetDecoratorRendererForGoogleFirestore, TypeScriptGetDecoratorRendererForGoogleSpanner);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@causa/workspace-google",
-  "version": "0.14.1",
+  "version": "0.16.0",
   "description": "The Causa workspace module providing many functionalities related to GCP and its services.",
   "repository": {
     "type": "git",
@@ -9,7 +9,7 @@
   "license": "ISC",
   "type": "module",
   "engines": {
-    "node": ">=20"
+    "node": ">=22"
   },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -35,10 +35,10 @@
     "generateConfigurationSchemas": "cs model genCode && perl -pi -e 's|\\@causa/runtime|\\@causa/workspace/validation|g' src/configurations/generated.ts"
   },
   "dependencies": {
-    "@causa/cli": ">= 0.6.6 < 1.0.0",
-    "@causa/workspace": ">= 0.22.2 < 1.0.0",
-    "@causa/workspace-core": ">= 0.28.1 < 1.0.0",
-    "@causa/workspace-typescript": ">= 0.19.0 < 1.0.0",
+    "@causa/cli": ">= 0.7.0 < 1.0.0",
+    "@causa/workspace": ">= 0.24.0 < 1.0.0",
+    "@causa/workspace-core": ">= 0.32.0 < 1.0.0",
+    "@causa/workspace-typescript": ">= 0.22.0 < 1.0.0",
     "@google-cloud/apikeys": "^2.2.1",
     "@google-cloud/bigquery": "^8.1.1",
     "@google-cloud/iam-credentials": "^4.2.1",
@@ -50,10 +50,10 @@
     "@google-cloud/spanner": "^8.6.0",
     "@google-cloud/storage": "^7.19.0",
     "class-validator": "^0.15.1",
-    "firebase": "^12.10.0",
-    "firebase-admin": "^13.7.0",
-    "globby": "^16.1.1",
-    "google-auth-library": "^10.6.1",
+    "firebase": "^12.12.1",
+    "firebase-admin": "^13.8.0",
+    "globby": "^16.2.0",
+    "google-auth-library": "^10.6.2",
     "google-gax": "5.0.6",
     "googleapis": "^171.4.0",
     "micromatch": "^4.0.8",
@@ -61,20 +61,20 @@
     "quicktype-core": "^23.2.6"
   },
   "devDependencies": {
-    "@swc/core": "^1.15.18",
+    "@swc/core": "^1.15.30",
     "@swc/jest": "^0.2.39",
-    "@tsconfig/node20": "^20.1.9",
+    "@tsconfig/node22": "^22.0.5",
     "@types/jest": "^30.0.0",
     "@types/micromatch": "^4.0.10",
-    "@types/node": "^20.19.37",
+    "@types/node": "^20.19.39",
     "copyfiles": "^2.4.1",
-    "eslint": "^10.0.3",
+    "eslint": "^10.2.1",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.5",
     "jest": "^30.3.0",
     "jest-extended": "^7.0.0",
     "rimraf": "^6.1.3",
-    "typescript": "^5.9.3",
-    "typescript-eslint": "^8.57.0"
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.59.0"
   }
 }

package/dist/backfilling/bigquery.d.ts

@@ -1,53 +0,0 @@
-import { WorkspaceContext } from '@causa/workspace';
-import type { BackfillEvent, BackfillEventsSource } from '@causa/workspace-core/backfill';
-/**
- * An events source that reads events from a BigQuery table.
- */
-export declare class BigQueryEventsSource implements BackfillEventsSource {
-    readonly tableId: string;
-    readonly filter?: string | undefined;
-    /**
-     * The BigQuery client.
-     */
-    private readonly bigQuery;
-    /**
-     * The job from which the events are read.
-     */
-    private job;
-    /**
-     * The request for the next page of results.
-     * If this is `null` after initializing the job, there are no more results.
-     */
-    private nextPageRequest;
-    /**
-     * The logger.
-     */
-    private readonly logger;
-    /**
-     * Creates a new events source that reads events from a BigQuery table.
-     *
-     * @param context The {@link WorkspaceContext}.
-     * @param tableId The ID of the BigQuery table from which the events should be read.
-     * @param filter A SQL boolean expression that filters the events from the table.
-     */
-    constructor(context: WorkspaceContext, tableId: string, filter?: string | undefined);
-    /**
-     * Creates a BigQuery job that reads the events from the table.
-     * The `data` and `attributes` columns are read to get the corresponding event properties.
-     * If a filter was specified, it is applied to the query as a where clause.
-     *
-     * @returns The BigQuery job that reads the events from the table.
-     */
-    private createJob;
-    getBatch(): Promise<BackfillEvent[] | null>;
-    dispose(): Promise<void>;
-    /**
-     * Tries to create a new {@link BigQueryEventsSource} from the given source string.
-     *
-     * @param context The {@link WorkspaceContext}.
-     * @param source The string representing the source, in the format `bq://<projectId>.<datasetId>.<tableId>`.
-     * @param filter A SQL boolean expression that filters the events from the table.
-     * @returns A new {@link BigQueryEventsSource} if the source string is valid, otherwise `null`.
-     */
-    static fromSourceAndFilter(context: WorkspaceContext, source: string, filter?: string): Promise<BigQueryEventsSource | null>;
-}

package/dist/backfilling/bigquery.js

@@ -1,101 +0,0 @@
-import { WorkspaceContext } from '@causa/workspace';
-import { BigQuery, Job } from '@google-cloud/bigquery';
-import { BigQueryService } from '../services/index.js';
-/**
- * The number of rows (events) in a single batch.
- */
-const BIGQUERY_MAX_RESULTS = 10000;
-/**
- * An events source that reads events from a BigQuery table.
- */
-export class BigQueryEventsSource {
-    tableId;
-    filter;
-    /**
-     * The BigQuery client.
-     */
-    bigQuery;
-    /**
-     * The job from which the events are read.
-     */
-    job = null;
-    /**
-     * The request for the next page of results.
-     * If this is `null` after initializing the job, there are no more results.
-     */
-    nextPageRequest = null;
-    /**
-     * The logger.
-     */
-    logger;
-    /**
-     * Creates a new events source that reads events from a BigQuery table.
-     *
-     * @param context The {@link WorkspaceContext}.
-     * @param tableId The ID of the BigQuery table from which the events should be read.
-     * @param filter A SQL boolean expression that filters the events from the table.
-     */
-    constructor(context, tableId, filter) {
-        this.tableId = tableId;
-        this.filter = filter;
-        this.bigQuery = context.service(BigQueryService).bigQuery;
-        this.logger = context.logger;
-    }
-    /**
-     * Creates a BigQuery job that reads the events from the table.
-     * The `data` and `attributes` columns are read to get the corresponding event properties.
-     * If a filter was specified, it is applied to the query as a where clause.
-     *
-     * @returns The BigQuery job that reads the events from the table.
-     */
-    async createJob() {
-        let query = `
-      SELECT
-        data,
-        attributes,
-      FROM
-        \`${this.tableId}\``;
-        if (this.filter) {
-            query += `
-      WHERE
-        (${this.filter})`;
-        }
-        this.logger.debug(`Creating BigQuery job from query '${query}'.`);
-        const [job] = await this.bigQuery.createQueryJob(query);
-        return job;
-    }
-    async getBatch() {
-        if (!this.job) {
-            this.job = await this.createJob();
-        }
-        else if (!this.nextPageRequest) {
-            return null;
-        }
-        const [rows, nextPageRequest] = await this.job.getQueryResults(this.nextPageRequest ?? {
-            autoPaginate: false,
-            maxResults: BIGQUERY_MAX_RESULTS,
-        });
-        this.nextPageRequest = nextPageRequest ?? null;
-        return rows.map((row) => ({
-            data: Buffer.from(row.data),
-            attributes: JSON.parse(row.attributes),
-        }));
-    }
-    async dispose() { }
-    /**
-     * Tries to create a new {@link BigQueryEventsSource} from the given source string.
-     *
-     * @param context The {@link WorkspaceContext}.
-     * @param source The string representing the source, in the format `bq://<projectId>.<datasetId>.<tableId>`.
-     * @param filter A SQL boolean expression that filters the events from the table.
-     * @returns A new {@link BigQueryEventsSource} if the source string is valid, otherwise `null`.
-     */
-    static async fromSourceAndFilter(context, source, filter) {
-        const match = source.match(/^bq:\/\/(?<tableId>.+)$/);
-        if (!match?.groups?.tableId) {
-            return null;
-        }
-        const tableId = match.groups.tableId;
-        return new BigQueryEventsSource(context, tableId, filter);
-    }
-}

package/dist/backfilling/index.d.ts

@@ -1,2 +0,0 @@
-export { BigQueryEventsSource } from './bigquery.js';
-export { PubSubBackfillEventPublisher } from './pubsub.js';

package/dist/backfilling/index.js

@@ -1,2 +0,0 @@
-export { BigQueryEventsSource } from './bigquery.js';
-export { PubSubBackfillEventPublisher } from './pubsub.js';

package/dist/backfilling/pubsub.d.ts

@@ -1,34 +0,0 @@
-import { WorkspaceContext } from '@causa/workspace';
-import { type BackfillEvent, BackfillEventPublisher } from '@causa/workspace-core/backfill';
-/**
- * A publisher that publishes events to backfill to a Pub/Sub topic.
- */
-export declare class PubSubBackfillEventPublisher extends BackfillEventPublisher {
-    readonly topicId: string;
-    /**
-     * The actual publisher, which handles batching and flow control.
-     */
-    private readonly publisher;
-    /**
-     * Creates a new {@link PubSubBackfillEventPublisher}.
-     *
-     * @param context The {@link WorkspaceContext}.
-     * @param topicId The ID of the Pub/Sub topic to publish to.
-     *   This should be a full Pub/Sub topic ID, e.g. `projects/<projectId>/topics/<topicId>`.
-     */
-    constructor(context: WorkspaceContext, topicId: string);
-    /**
-     * Publishes an event to the Pub/Sub topic.
-     * Publishing actually occurs asynchronously. {@link PubSubBackfillEventPublisher.flush} must be called to wait for
-     * all events to be published.
-     *
-     * @param event The event to publish.
-     * @returns A promise that resolves when the publisher has caught up with publishing, or `null` if other events can be
-     *   published immediately.
-     */
-    protected publishEvent(event: BackfillEvent): Promise<void> | null;
-    /**
-     * Waits for all events to be published.
-     */
-    protected flush(): Promise<void>;
-}

package/dist/backfilling/pubsub.js

@@ -1,65 +0,0 @@
-import { WorkspaceContext } from '@causa/workspace';
-import { BackfillEventPublisher, } from '@causa/workspace-core/backfill';
-import { FlowControlledPublisher } from '@google-cloud/pubsub/build/src/publisher/flow-publisher.js';
-import { PubSubService } from '../services/index.js';
-/**
- * Options when publishing events in batches to Pub/Sub.
- */
-const PUBLISH_OPTIONS = {
-    flowControlOptions: {
-        maxOutstandingBytes: 10 * 1024 * 1024,
-        maxOutstandingMessages: 1000,
-    },
-    batching: {
-        maxBytes: 10 * 1024 * 1024,
-        maxMessages: 1000,
-        maxMilliseconds: 1000,
-    },
-};
-/**
- * A publisher that publishes events to backfill to a Pub/Sub topic.
- */
-export class PubSubBackfillEventPublisher extends BackfillEventPublisher {
-    topicId;
-    /**
-     * The actual publisher, which handles batching and flow control.
-     */
-    publisher;
-    /**
-     * Creates a new {@link PubSubBackfillEventPublisher}.
-     *
-     * @param context The {@link WorkspaceContext}.
-     * @param topicId The ID of the Pub/Sub topic to publish to.
-     *   This should be a full Pub/Sub topic ID, e.g. `projects/<projectId>/topics/<topicId>`.
-     */
-    constructor(context, topicId) {
-        super(context);
-        this.topicId = topicId;
-        this.publisher = context
-            .service(PubSubService)
-            .pubSub.topic(topicId, PUBLISH_OPTIONS)
-            .flowControlled();
-    }
-    /**
-     * Publishes an event to the Pub/Sub topic.
-     * Publishing actually occurs asynchronously. {@link PubSubBackfillEventPublisher.flush} must be called to wait for
-     * all events to be published.
-     *
-     * @param event The event to publish.
-     * @returns A promise that resolves when the publisher has caught up with publishing, or `null` if other events can be
-     *   published immediately.
-     */
-    publishEvent(event) {
-        return this.publisher.publish({
-            data: event.data,
-            attributes: event.attributes,
-            orderingKey: event.key,
-        });
-    }
-    /**
-     * Waits for all events to be published.
-     */
-    async flush() {
-        await this.publisher.all();
-    }
-}