@shushed/helpers 0.0.27 → 0.0.28

package/dist/index.d.ts

@@ -5,6 +5,7 @@ import { PubSub, Topic, Subscription } from '@google-cloud/pubsub';
 import * as _google_cloud_scheduler_build_protos_protos from '@google-cloud/scheduler/build/protos/protos';
 import { CloudSchedulerClient } from '@google-cloud/scheduler';
 import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
+import { BigQuery } from '@google-cloud/bigquery';
 
 declare const schema$l: {
     readonly $schema: "http://json-schema.org/draft-07/schema#";
@@ -28743,12 +28744,14 @@ declare class PubSubHelper extends Runtime {
     createOrUpdate(opts: {
         topicName: string;
         dlq: boolean;
+        bigQueryTableId?: string | null;
         retryLimit: number;
         retryDelay: number;
         ackDeadline: number;
     }): Promise<{
         topic: Topic;
         dlqTopic: Topic | null;
+        bqSubscription: Subscription | null;
         dlqSubscription: Subscription | null;
         subscription: Subscription;
     }>;
@@ -28799,7 +28802,7 @@ declare class SchedulerHelper extends Runtime {
         timezone: string;
     }): Promise<{
         status: _google_cloud_scheduler_build_protos_protos.google.rpc.IStatus | null | undefined;
-        state: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "STATE_UNSPECIFIED" | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
+        state: "STATE_UNSPECIFIED" | _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
         name: string | null | undefined;
         httpTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IHttpTarget | null | undefined;
         pubsubTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IPubsubTarget | null | undefined;
@@ -28809,7 +28812,7 @@ declare class SchedulerHelper extends Runtime {
         timezone: string;
     }): Promise<{
         status: _google_cloud_scheduler_build_protos_protos.google.rpc.IStatus | null | undefined;
-        state: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "STATE_UNSPECIFIED" | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
+        state: "STATE_UNSPECIFIED" | _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
         name: string | null | undefined;
         httpTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IHttpTarget | null | undefined;
         pubsubTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IPubsubTarget | null | undefined;
@@ -28833,6 +28836,21 @@ declare class JWKSHelper extends Runtime {
     private fetchJWKS;
 }
 
+interface ColumnSchema {
+    name: string;
+    type: string;
+    mode?: string;
+}
+declare class BigQueryHelper extends Runtime {
+    private bigQuery;
+    constructor(opts: Opts, bigQuery: BigQuery);
+    createOrUpdate(opts: {
+        datasetId: string;
+        tableName: string;
+        incomingSchema: ColumnSchema[];
+    }): Promise<string>;
+}
+
 type TriggerOnExecuteOptions = {
     req: {
         status?: number;
@@ -28979,6 +28997,8 @@ type TriggerOnCreateOptions = {
     };
 };
 
+type index_BigQueryHelper = BigQueryHelper;
+declare const index_BigQueryHelper: typeof BigQueryHelper;
 type index_EnvEngine = EnvEngine;
 declare const index_EnvEngine: typeof EnvEngine;
 type index_JWKSHelper = JWKSHelper;
@@ -29000,7 +29020,7 @@ declare const index_sanitizeToString: typeof sanitizeToString;
 declare const index_validate: typeof validate;
 declare const index_validateGoogleAuth: typeof validateGoogleAuth;
 declare namespace index {
-  export { index_EnvEngine as EnvEngine, index_JWKSHelper as JWKSHelper, index_PubSubHelper as PubSubHelper, index_Runtime as Runtime, index_SchedulerHelper as SchedulerHelper, index_Secrets as Secrets, type index_TriggerOnCreateOptions as TriggerOnCreateOptions, type index_TriggerOnExecuteOptions as TriggerOnExecuteOptions, index_isCronMessage as isCronMessage, index_isPubSubRequest as isPubSubRequest, index_sanitize as sanitize, index_sanitizeToString as sanitizeToString, index_validate as validate, index_validateGoogleAuth as validateGoogleAuth };
+  export { index_BigQueryHelper as BigQueryHelper, index_EnvEngine as EnvEngine, index_JWKSHelper as JWKSHelper, index_PubSubHelper as PubSubHelper, index_Runtime as Runtime, index_SchedulerHelper as SchedulerHelper, index_Secrets as Secrets, type index_TriggerOnCreateOptions as TriggerOnCreateOptions, type index_TriggerOnExecuteOptions as TriggerOnExecuteOptions, index_isCronMessage as isCronMessage, index_isPubSubRequest as isPubSubRequest, index_sanitize as sanitize, index_sanitizeToString as sanitizeToString, index_validate as validate, index_validateGoogleAuth as validateGoogleAuth };
 }
 
 export { index as lib, index$9 as schema, index$1 as types };

package/dist/index.js

@@ -280,6 +280,7 @@ var shipped_exports2 = {};
 // src-public/index.ts
 var src_public_exports = {};
 __export(src_public_exports, {
+  BigQueryHelper: () => bigquery_default,
   EnvEngine: () => env_default,
   JWKSHelper: () => jwks_default,
   PubSubHelper: () => pubsub_default,
@@ -631,6 +632,9 @@ var AES256GCM = class {
 var env_default = EnvEngine;
 
 // src-public/pubsub.ts
+function isSubscriptionOptionsPush(opts) {
+  return "endpointUrl" in opts;
+}
 var PubSubHelper = class extends Runtime {
   constructor(opts, pubSub) {
     super(opts);
@@ -648,6 +652,18 @@ var PubSubHelper = class extends Runtime {
       dlqSubscription = await this.ensureDLQSubscriptionExists(dlqSubscriptionName, {
         topic: dlqTopic
       });
+    } else {
+      await this.deleteSubscription(`${subscriptionName}-dlq`);
+    }
+    let bqSubscription = null;
+    if (opts.bigQueryTableId) {
+      const bigQuerySubscriptionName = `${subscriptionName}-bq`;
+      bqSubscription = await this.ensureSubscribtionExists(bigQuerySubscriptionName, {
+        topic,
+        tableId: opts.bigQueryTableId
+      });
+    } else {
+      await this.deleteSubscription(`${subscriptionName}-bq`);
     }
     const subscription = await this.ensureSubscribtionExists(subscriptionName, {
       topic,
@@ -661,6 +677,7 @@ var PubSubHelper = class extends Runtime {
     return {
       topic,
       dlqTopic,
+      bqSubscription,
       dlqSubscription,
       subscription
     };
@@ -689,40 +706,58 @@ var PubSubHelper = class extends Runtime {
     let subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName);
     let endpointHasChanged = false;
     let retryLimitHasChanged = false;
+    let ackDeadlineHasChanged = false;
     let retryDelayHasChanged = false;
     let shouldRecreate = false;
     let dlqHasChanged = false;
+    let tableIdHasChanged = false;
     if (subscription) {
       const [subscriptionMetaData] = await subscription.getMetadata();
-      endpointHasChanged = subscriptionMetaData.pushConfig?.pushEndpoint != opts.endpointUrl;
-      retryDelayHasChanged = subscriptionMetaData.retryPolicy?.minimumBackoff?.seconds != opts.retryMinDelay;
-      retryLimitHasChanged = subscriptionMetaData.deadLetterPolicy?.maxDeliveryAttempts != opts.retryLimit;
-      dlqHasChanged = !subscriptionMetaData.deadLetterPolicy && !!opts.dlqTopic || !!subscriptionMetaData.deadLetterPolicy?.deadLetterTopic && !opts.dlqTopic;
-      shouldRecreate = endpointHasChanged || retryDelayHasChanged || retryLimitHasChanged || dlqHasChanged;
+      if (!isSubscriptionOptionsPush(opts)) {
+        tableIdHasChanged = (subscriptionMetaData.bigqueryConfig?.table || null) !== (opts.tableId || null);
+      } else {
+        endpointHasChanged = subscriptionMetaData.pushConfig?.pushEndpoint != opts.endpointUrl;
+        dlqHasChanged = !subscriptionMetaData.deadLetterPolicy && !!opts.dlqTopic || !!subscriptionMetaData.deadLetterPolicy?.deadLetterTopic && !opts.dlqTopic;
+        retryDelayHasChanged = subscriptionMetaData.retryPolicy?.minimumBackoff?.seconds != opts.retryMinDelay;
+        retryLimitHasChanged = subscriptionMetaData.deadLetterPolicy?.maxDeliveryAttempts != opts.retryLimit;
+        ackDeadlineHasChanged = subscriptionMetaData.ackDeadlineSeconds !== (opts.ackDeadline || 90);
+      }
+      shouldRecreate = endpointHasChanged || retryDelayHasChanged || retryLimitHasChanged || dlqHasChanged || tableIdHasChanged || ackDeadlineHasChanged;
     }
     if (subscription && shouldRecreate) {
       await subscription.delete();
     }
-    if (!subscription || shouldRecreate) {
-      await opts.topic.createSubscription(subscriptionName, Object.assign({
-        ackDeadlineSeconds: opts.ackDeadline || 90,
-        pushConfig: {
-          pushEndpoint: opts.endpointUrl,
-          oidcToken: {
-            serviceAccountEmail: `runtime@${process.env.GCLOUD_PROJECT}.iam.gserviceaccount.com`,
-            audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId
-          }
-        },
-        retryPolicy: {
-          minimumBackoff: { seconds: opts.retryMinDelay },
-          maximumBackoff: { seconds: opts.retryMaxDelay }
+    const payload = {};
+    if (!isSubscriptionOptionsPush(opts)) {
+      payload.ackDeadlineSeconds = 60;
+      payload.bigqueryConfig = {
+        table: opts.tableId,
+        writeMetadata: true,
+        dropUnknownFields: true,
+        useTableSchema: true
+      };
+    } else {
+      payload.ackDeadlineSeconds = opts.ackDeadline || 90;
+      payload.pushConfig = {
+        pushEndpoint: opts.endpointUrl,
+        oidcToken: {
+          serviceAccountEmail: `runtime@${process.env.GCLOUD_PROJECT}.iam.gserviceaccount.com`,
+          audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId
         }
-      }, opts.dlqTopic ? {
-        deadLetterPolicy: {
+      };
+      payload.retryPolicy = {
+        minimumBackoff: { seconds: opts.retryMinDelay },
+        maximumBackoff: { seconds: opts.retryMaxDelay }
+      };
+      if (opts.dlqTopic) {
+        payload.deadLetterPolicy = {
           deadLetterTopic: opts.dlqTopic.name,
           maxDeliveryAttempts: opts.retryLimit
-        }
-      } : {}));
+        };
+      }
+    }
+    if (!subscription || shouldRecreate) {
+      await opts.topic.createSubscription(subscriptionName, payload);
       [subscriptions] = await opts.topic.getSubscriptions();
       subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName);
     }
@@ -942,6 +977,64 @@ var Secrets = class extends Runtime {
   }
 };
 var secret_default = Secrets;
+
+// src-public/bigquery.ts
+var BigQueryHelper = class extends Runtime {
+  constructor(opts, bigQuery) {
+    super(opts);
+    this.bigQuery = bigQuery;
+  }
+  async createOrUpdate(opts) {
+    const { datasetId, tableName, incomingSchema } = opts;
+    const dataset = this.bigQuery.dataset(datasetId);
+    const table = dataset.table(tableName);
+    const fullTableId = `${this.bigQuery.projectId}.${datasetId}.${tableName}`;
+    const [tableExists] = await table.exists();
+    if (!tableExists) {
+      throw new Error(`Table ${fullTableId} does not exist.`);
+    }
+    const [metadata] = await table.getMetadata();
+    const existingSchema = metadata.schema.fields;
+    const alterStatements = [];
+    const existingColumnsMap = new Map(
+      existingSchema.map((col) => [col.name.toLowerCase(), col])
+    );
+    const incomingColumnsMap = new Map(
+      incomingSchema.map((col) => [col.name.toLowerCase(), col])
+    );
+    const currentDate = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10).replace(/-/g, "");
+    for (const incomingCol of incomingSchema) {
+      const existingCol = existingColumnsMap.get(incomingCol.name.toLowerCase());
+      if (!existingCol) {
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` ADD COLUMN \`${incomingCol.name}\` ${incomingCol.type}${incomingCol.mode === "REPEATED" ? " REPEATED" : ""};`
+        );
+      } else if (existingCol.type !== incomingCol.type || incomingCol.mode && existingCol.mode !== incomingCol.mode) {
+        const renamed = `${incomingCol.name}${currentDate}`;
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` RENAME COLUMN \`${incomingCol.name}\` TO \`${renamed}\`;`
+        );
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` ADD COLUMN \`${incomingCol.name}\` ${incomingCol.type}${incomingCol.mode === "REPEATED" ? " REPEATED" : ""};`
+        );
+      }
+    }
+    for (const existingCol of existingSchema) {
+      if (!incomingColumnsMap.has(existingCol.name.toLowerCase())) {
+        const renamed = `${existingCol.name}${currentDate}`;
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` RENAME COLUMN \`${existingCol.name}\` TO \`${renamed}\`;`
+        );
+      }
+    }
+    for (const sql of alterStatements) {
+      console.log(`Executing SQL: ${sql}`);
+      await this.bigQuery.query({ query: sql });
+    }
+    return fullTableId;
+  }
+};
+var bigquery_default = BigQueryHelper;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   lib,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shushed/helpers",
-  "version": "0.0.27",
+  "version": "0.0.28",
   "author": "",
   "license": "UNLICENSED",
   "description": "",