@shushed/helpers 0.0.27 → 0.0.29

package/dist/index.d.ts

@@ -2,9 +2,11 @@ import { JSONSchemaType } from 'ajv';
 import { DeepRedact } from '@hackylabs/deep-redact';
 import { Firestore, DocumentReference } from '@google-cloud/firestore';
 import { PubSub, Topic, Subscription } from '@google-cloud/pubsub';
+import { Context } from 'co-body';
 import * as _google_cloud_scheduler_build_protos_protos from '@google-cloud/scheduler/build/protos/protos';
 import { CloudSchedulerClient } from '@google-cloud/scheduler';
 import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
+import { BigQuery } from '@google-cloud/bigquery';
 
 declare const schema$l: {
     readonly $schema: "http://json-schema.org/draft-07/schema#";
@@ -28720,6 +28722,8 @@ declare const isCronMessage: (headers: Record<string, string>) => boolean;
 declare const validateGoogleAuth: (opts: Opts & {
     serviceAccount?: string | null;
 }, headers: Record<string, string>, firestore: Firestore) => Promise<void>;
+declare const parseDateOrDefault: (date: string | null | Date, defaultValue?: Date | null) => Date | null;
+declare function getEventTime(x: any, defaultValue: Date | null): Date | null;
 
 type Level = 'env' | 'workflow' | 'trigger';
 declare class EnvEngine extends Runtime {
@@ -28737,18 +28741,48 @@ declare class EnvEngine extends Runtime {
     get(key: string, level?: Level, decryptWithValue?: string): Promise<string>;
 }
 
+type Message = {
+    buildshipId: string;
+    body: string;
+    sourceSystem: string;
+    targetSystem: string;
+    processStartedAt: Date | null;
+    recordModifiedAt: Date | null;
+    originalPublishTime: Date | null;
+    publishTime: Date | null;
+};
+type LastModifiedKeys = 'last_modified_date_time' | 'last_modified_date' | 'last_modified_datetime' | 'record_modified_at' | 'lastModifiedDateTime' | 'lastModifiedAt' | 'last_modified_at' | 'SystemModifiedAt' | 'system_modified_at' | 'lastUpdatedAt' | 'modified_at' | 'last_updated_at' | 'modifiedAt';
+type AtLeastOne<T, Keys extends keyof T = keyof T> = Partial<T> & {
+    [K in Keys]-?: Required<Pick<T, K>> & Partial<Record<Exclude<Keys, K>, never>>;
+}[Keys];
+type MessageBody = {
+    [K in LastModifiedKeys]?: Date | string;
+} & {
+    [key: string]: any;
+};
 declare class PubSubHelper extends Runtime {
     private pubSub;
     constructor(opts: Opts, pubSub: PubSub);
+    static decodeMessageFromRequest(req: Context): Promise<Message>;
+    publish(options: Partial<Message> & {
+        topicName: string;
+        sourceSystem: string;
+    }, payload: AtLeastOne<MessageBody, LastModifiedKeys>[], orgMessage?: Message): Promise<string[]>;
+    publish(options: Partial<Message> & {
+        topicName: string;
+        sourceSystem: string;
+    }, payload: AtLeastOne<MessageBody, LastModifiedKeys>, orgMessage?: Message): Promise<string[]>;
     createOrUpdate(opts: {
         topicName: string;
         dlq: boolean;
+        bigQueryTableId?: string | null;
         retryLimit: number;
         retryDelay: number;
         ackDeadline: number;
     }): Promise<{
         topic: Topic;
         dlqTopic: Topic | null;
+        bqSubscription: Subscription | null;
         dlqSubscription: Subscription | null;
         subscription: Subscription;
     }>;
@@ -28799,7 +28833,7 @@ declare class SchedulerHelper extends Runtime {
         timezone: string;
     }): Promise<{
         status: _google_cloud_scheduler_build_protos_protos.google.rpc.IStatus | null | undefined;
-        state: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "STATE_UNSPECIFIED" | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
+        state: "STATE_UNSPECIFIED" | _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
         name: string | null | undefined;
         httpTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IHttpTarget | null | undefined;
         pubsubTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IPubsubTarget | null | undefined;
@@ -28809,7 +28843,7 @@ declare class SchedulerHelper extends Runtime {
         timezone: string;
     }): Promise<{
         status: _google_cloud_scheduler_build_protos_protos.google.rpc.IStatus | null | undefined;
-        state: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "STATE_UNSPECIFIED" | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
+        state: "STATE_UNSPECIFIED" | _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.Job.State | "ENABLED" | "PAUSED" | "DISABLED" | "UPDATE_FAILED" | null | undefined;
         name: string | null | undefined;
         httpTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IHttpTarget | null | undefined;
         pubsubTarget: _google_cloud_scheduler_build_protos_protos.google.cloud.scheduler.v1.IPubsubTarget | null | undefined;
@@ -28833,6 +28867,21 @@ declare class JWKSHelper extends Runtime {
     private fetchJWKS;
 }
 
+interface ColumnSchema {
+    name: string;
+    type: string;
+    mode?: string;
+}
+declare class BigQueryHelper extends Runtime {
+    private bigQuery;
+    constructor(opts: Opts, bigQuery: BigQuery);
+    createOrUpdate(opts: {
+        datasetId: string;
+        tableName: string;
+        incomingSchema: ColumnSchema[];
+    }): Promise<string>;
+}
+
 type TriggerOnExecuteOptions = {
     req: {
         status?: number;
@@ -28979,6 +29028,8 @@ type TriggerOnCreateOptions = {
     };
 };
 
+type index_BigQueryHelper = BigQueryHelper;
+declare const index_BigQueryHelper: typeof BigQueryHelper;
 type index_EnvEngine = EnvEngine;
 declare const index_EnvEngine: typeof EnvEngine;
 type index_JWKSHelper = JWKSHelper;
@@ -28993,14 +29044,16 @@ type index_Secrets = Secrets;
 declare const index_Secrets: typeof Secrets;
 type index_TriggerOnCreateOptions = TriggerOnCreateOptions;
 type index_TriggerOnExecuteOptions = TriggerOnExecuteOptions;
+declare const index_getEventTime: typeof getEventTime;
 declare const index_isCronMessage: typeof isCronMessage;
 declare const index_isPubSubRequest: typeof isPubSubRequest;
+declare const index_parseDateOrDefault: typeof parseDateOrDefault;
 declare const index_sanitize: typeof sanitize;
 declare const index_sanitizeToString: typeof sanitizeToString;
 declare const index_validate: typeof validate;
 declare const index_validateGoogleAuth: typeof validateGoogleAuth;
 declare namespace index {
-  export { index_EnvEngine as EnvEngine, index_JWKSHelper as JWKSHelper, index_PubSubHelper as PubSubHelper, index_Runtime as Runtime, index_SchedulerHelper as SchedulerHelper, index_Secrets as Secrets, type index_TriggerOnCreateOptions as TriggerOnCreateOptions, type index_TriggerOnExecuteOptions as TriggerOnExecuteOptions, index_isCronMessage as isCronMessage, index_isPubSubRequest as isPubSubRequest, index_sanitize as sanitize, index_sanitizeToString as sanitizeToString, index_validate as validate, index_validateGoogleAuth as validateGoogleAuth };
+  export { index_BigQueryHelper as BigQueryHelper, index_EnvEngine as EnvEngine, index_JWKSHelper as JWKSHelper, index_PubSubHelper as PubSubHelper, index_Runtime as Runtime, index_SchedulerHelper as SchedulerHelper, index_Secrets as Secrets, type index_TriggerOnCreateOptions as TriggerOnCreateOptions, type index_TriggerOnExecuteOptions as TriggerOnExecuteOptions, index_getEventTime as getEventTime, index_isCronMessage as isCronMessage, index_isPubSubRequest as isPubSubRequest, index_parseDateOrDefault as parseDateOrDefault, index_sanitize as sanitize, index_sanitizeToString as sanitizeToString, index_validate as validate, index_validateGoogleAuth as validateGoogleAuth };
 }
 
 export { index as lib, index$9 as schema, index$1 as types };

package/dist/index.js

@@ -280,14 +280,17 @@ var shipped_exports2 = {};
 // src-public/index.ts
 var src_public_exports = {};
 __export(src_public_exports, {
+  BigQueryHelper: () => bigquery_default,
   EnvEngine: () => env_default,
   JWKSHelper: () => jwks_default,
   PubSubHelper: () => pubsub_default,
   Runtime: () => Runtime,
   SchedulerHelper: () => scheduler_default,
   Secrets: () => secret_default,
+  getEventTime: () => getEventTime,
   isCronMessage: () => isCronMessage,
   isPubSubRequest: () => isPubSubRequest,
+  parseDateOrDefault: () => parseDateOrDefault,
   sanitize: () => sanitize,
   sanitizeToString: () => sanitizeToString,
   validate: () => validate,
@@ -485,6 +488,49 @@ var validateGoogleAuth = async (opts, headers, firestore) => {
     throw new Error("Invalid authorization header " + err.message);
   }
 };
+var parseDateOrDefault = (date, defaultValue = null) => {
+  if (!date) {
+    return defaultValue;
+  }
+  if (date instanceof Date) {
+    if (isNaN(date.getTime())) {
+      return defaultValue;
+    }
+    return date;
+  }
+  const dateObj = new Date(date);
+  if (isNaN(dateObj.getTime())) {
+    return defaultValue;
+  }
+  return dateObj;
+};
+function getEventTime(x, defaultValue) {
+  const possibleKeys = [
+    "last_modified_date_time",
+    "record_modified_at",
+    "last_modified_date",
+    "last_modified_datetime",
+    "last_modified_date_time",
+    "lastModifiedDateTime",
+    "lastModifiedAt",
+    "last_modified_at",
+    "SystemModifiedAt",
+    "system_modified_at",
+    "lastUpdatedAt",
+    "modified_at",
+    "last_updated_at",
+    "modifiedAt"
+  ];
+  for (let i = 0; i < possibleKeys.length; i++) {
+    if (x[possibleKeys[i]]) {
+      const possibleDateTime = new Date(x[possibleKeys[i]]);
+      if (!isNaN(possibleDateTime.getTime())) {
+        return possibleDateTime;
+      }
+    }
+  }
+  return defaultValue;
+}
 
 // src-public/env.ts
 var crypto = __toESM(require("crypto"));
@@ -631,11 +677,69 @@ var AES256GCM = class {
 var env_default = EnvEngine;
 
 // src-public/pubsub.ts
+var import_co_body = __toESM(require("co-body"));
+function isSubscriptionOptionsPush(opts) {
+  return "endpointUrl" in opts;
+}
 var PubSubHelper = class extends Runtime {
   constructor(opts, pubSub) {
     super(opts);
     this.pubSub = pubSub;
   }
+  static async decodeMessageFromRequest(req) {
+    const rawMessage = await import_co_body.default.json(req);
+    const rawMessageText = Buffer.from(rawMessage.message?.data, "base64").toString("utf-8");
+    let messageBody;
+    try {
+      messageBody = JSON.parse(rawMessageText);
+    } catch (err) {
+      throw new Error("Could not parse the message body. Original body: " + rawMessageText + " . Error: " + err.message);
+    }
+    const message = {
+      sourceSystem: rawMessage.attributes?.source_system || "unknown",
+      targetSystem: rawMessage.attributes?.target_system || "*",
+      processStartedAt: parseDateOrDefault(rawMessage.attributes?.process_started_at),
+      recordModifiedAt: parseDateOrDefault(getEventTime(messageBody, rawMessage.attributes?.record_modified_at)),
+      originalPublishTime: parseDateOrDefault(rawMessage.attributes?.original_publish_time, parseDateOrDefault(rawMessage.publish_time)),
+      buildshipId: rawMessage.attributes?.buildship_id || "unknown",
+      publishTime: parseDateOrDefault(rawMessage.publish_time),
+      body: messageBody
+    };
+    return message;
+  }
+  async publish(options, payload, orgMessage) {
+    const topic = this.pubSub.topic(options.topicName, {
+      batching: {
+        maxMessages: 20,
+        maxMilliseconds: 2 * 1e3
+      }
+    });
+    let normPayloads = [];
+    if (!Array.isArray(payload)) {
+      normPayloads = [payload];
+    } else {
+      normPayloads = payload;
+    }
+    const promises = [];
+    for (let i = 0; i < normPayloads.length; i++) {
+      const normPayload = normPayloads[i];
+      promises.push((async () => {
+        const result = await topic.publishMessage({
+          data: Buffer.from(JSON.stringify(normPayload)),
+          attributes: {
+            source_system: options.sourceSystem,
+            target_system: options.targetSystem || orgMessage?.targetSystem || "*",
+            process_started_at: (options.processStartedAt || orgMessage?.processStartedAt || /* @__PURE__ */ new Date()).toISOString(),
+            record_modified_at: (getEventTime(normPayload, null) || /* @__PURE__ */ new Date()).toISOString(),
+            original_publish_time: (orgMessage?.originalPublishTime || orgMessage?.publishTime || /* @__PURE__ */ new Date())?.toDateString() || "",
+            buildship_id: this.systemEnvName + "-" + this.workflowId + "-" + this.triggerId
+          }
+        });
+        return result;
+      })());
+    }
+    return Promise.all(promises);
+  }
   async createOrUpdate(opts) {
     const subscriptionName = `${this.envName === "prod" ? "" : `${this.envName}---`}${this.triggerId}`;
     const topic = await this.ensureTopicExists(opts.topicName);
@@ -648,6 +752,18 @@ var PubSubHelper = class extends Runtime {
       dlqSubscription = await this.ensureDLQSubscriptionExists(dlqSubscriptionName, {
         topic: dlqTopic
       });
+    } else {
+      await this.deleteSubscription(`${subscriptionName}-dlq`);
+    }
+    let bqSubscription = null;
+    if (opts.bigQueryTableId) {
+      const bigQuerySubscriptionName = `${subscriptionName}-bq`;
+      bqSubscription = await this.ensureSubscribtionExists(bigQuerySubscriptionName, {
+        topic,
+        tableId: opts.bigQueryTableId
+      });
+    } else {
+      await this.deleteSubscription(`${subscriptionName}-bq`);
     }
     const subscription = await this.ensureSubscribtionExists(subscriptionName, {
       topic,
@@ -661,6 +777,7 @@ var PubSubHelper = class extends Runtime {
     return {
       topic,
       dlqTopic,
+      bqSubscription,
       dlqSubscription,
       subscription
     };
@@ -689,40 +806,58 @@ var PubSubHelper = class extends Runtime {
     let subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName);
     let endpointHasChanged = false;
     let retryLimitHasChanged = false;
+    let ackDeadlineHasChanged = false;
     let retryDelayHasChanged = false;
     let shouldRecreate = false;
     let dlqHasChanged = false;
+    let tableIdHasChanged = false;
     if (subscription) {
       const [subscriptionMetaData] = await subscription.getMetadata();
-      endpointHasChanged = subscriptionMetaData.pushConfig?.pushEndpoint != opts.endpointUrl;
-      retryDelayHasChanged = subscriptionMetaData.retryPolicy?.minimumBackoff?.seconds != opts.retryMinDelay;
-      retryLimitHasChanged = subscriptionMetaData.deadLetterPolicy?.maxDeliveryAttempts != opts.retryLimit;
-      dlqHasChanged = !subscriptionMetaData.deadLetterPolicy && !!opts.dlqTopic || !!subscriptionMetaData.deadLetterPolicy?.deadLetterTopic && !opts.dlqTopic;
-      shouldRecreate = endpointHasChanged || retryDelayHasChanged || retryLimitHasChanged || dlqHasChanged;
+      if (!isSubscriptionOptionsPush(opts)) {
+        tableIdHasChanged = (subscriptionMetaData.bigqueryConfig?.table || null) !== (opts.tableId || null);
+      } else {
+        endpointHasChanged = subscriptionMetaData.pushConfig?.pushEndpoint != opts.endpointUrl;
+        dlqHasChanged = !subscriptionMetaData.deadLetterPolicy && !!opts.dlqTopic || !!subscriptionMetaData.deadLetterPolicy?.deadLetterTopic && !opts.dlqTopic;
+        retryDelayHasChanged = subscriptionMetaData.retryPolicy?.minimumBackoff?.seconds != opts.retryMinDelay;
+        retryLimitHasChanged = subscriptionMetaData.deadLetterPolicy?.maxDeliveryAttempts != opts.retryLimit;
+        ackDeadlineHasChanged = subscriptionMetaData.ackDeadlineSeconds !== (opts.ackDeadline || 90);
+      }
+      shouldRecreate = endpointHasChanged || retryDelayHasChanged || retryLimitHasChanged || dlqHasChanged || tableIdHasChanged || ackDeadlineHasChanged;
     }
     if (subscription && shouldRecreate) {
       await subscription.delete();
     }
-    if (!subscription || shouldRecreate) {
-      await opts.topic.createSubscription(subscriptionName, Object.assign({
-        ackDeadlineSeconds: opts.ackDeadline || 90,
-        pushConfig: {
-          pushEndpoint: opts.endpointUrl,
-          oidcToken: {
-            serviceAccountEmail: `runtime@${process.env.GCLOUD_PROJECT}.iam.gserviceaccount.com`,
-            audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId
-          }
-        },
-        retryPolicy: {
-          minimumBackoff: { seconds: opts.retryMinDelay },
-          maximumBackoff: { seconds: opts.retryMaxDelay }
+    const payload = {};
+    if (!isSubscriptionOptionsPush(opts)) {
+      payload.ackDeadlineSeconds = 60;
+      payload.bigqueryConfig = {
+        table: opts.tableId,
+        writeMetadata: true,
+        dropUnknownFields: true,
+        useTableSchema: true
+      };
+    } else {
+      payload.ackDeadlineSeconds = opts.ackDeadline || 90;
+      payload.pushConfig = {
+        pushEndpoint: opts.endpointUrl,
+        oidcToken: {
+          serviceAccountEmail: `runtime@${process.env.GCLOUD_PROJECT}.iam.gserviceaccount.com`,
+          audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId
         }
-      }, opts.dlqTopic ? {
-        deadLetterPolicy: {
+      };
+      payload.retryPolicy = {
+        minimumBackoff: { seconds: opts.retryMinDelay },
+        maximumBackoff: { seconds: opts.retryMaxDelay }
+      };
+      if (opts.dlqTopic) {
+        payload.deadLetterPolicy = {
           deadLetterTopic: opts.dlqTopic.name,
           maxDeliveryAttempts: opts.retryLimit
-        }
-      } : {}));
+        };
+      }
+    }
+    if (!subscription || shouldRecreate) {
+      await opts.topic.createSubscription(subscriptionName, payload);
       [subscriptions] = await opts.topic.getSubscriptions();
       subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName);
     }
@@ -942,6 +1077,64 @@ var Secrets = class extends Runtime {
   }
 };
 var secret_default = Secrets;
+
+// src-public/bigquery.ts
+var BigQueryHelper = class extends Runtime {
+  constructor(opts, bigQuery) {
+    super(opts);
+    this.bigQuery = bigQuery;
+  }
+  async createOrUpdate(opts) {
+    const { datasetId, tableName, incomingSchema } = opts;
+    const dataset = this.bigQuery.dataset(datasetId);
+    const table = dataset.table(tableName);
+    const fullTableId = `${this.bigQuery.projectId}.${datasetId}.${tableName}`;
+    const [tableExists] = await table.exists();
+    if (!tableExists) {
+      throw new Error(`Table ${fullTableId} does not exist.`);
+    }
+    const [metadata] = await table.getMetadata();
+    const existingSchema = metadata.schema.fields;
+    const alterStatements = [];
+    const existingColumnsMap = new Map(
+      existingSchema.map((col) => [col.name.toLowerCase(), col])
+    );
+    const incomingColumnsMap = new Map(
+      incomingSchema.map((col) => [col.name.toLowerCase(), col])
+    );
+    const currentDate = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10).replace(/-/g, "");
+    for (const incomingCol of incomingSchema) {
+      const existingCol = existingColumnsMap.get(incomingCol.name.toLowerCase());
+      if (!existingCol) {
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` ADD COLUMN \`${incomingCol.name}\` ${incomingCol.type}${incomingCol.mode === "REPEATED" ? " REPEATED" : ""};`
+        );
+      } else if (existingCol.type !== incomingCol.type || incomingCol.mode && existingCol.mode !== incomingCol.mode) {
+        const renamed = `${incomingCol.name}${currentDate}`;
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` RENAME COLUMN \`${incomingCol.name}\` TO \`${renamed}\`;`
+        );
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` ADD COLUMN \`${incomingCol.name}\` ${incomingCol.type}${incomingCol.mode === "REPEATED" ? " REPEATED" : ""};`
+        );
+      }
+    }
+    for (const existingCol of existingSchema) {
+      if (!incomingColumnsMap.has(existingCol.name.toLowerCase())) {
+        const renamed = `${existingCol.name}${currentDate}`;
+        alterStatements.push(
+          `ALTER TABLE \`${fullTableId}\` RENAME COLUMN \`${existingCol.name}\` TO \`${renamed}\`;`
+        );
+      }
+    }
+    for (const sql of alterStatements) {
+      console.log(`Executing SQL: ${sql}`);
+      await this.bigQuery.query({ query: sql });
+    }
+    return fullTableId;
+  }
+};
+var bigquery_default = BigQueryHelper;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   lib,

package/dist/package.json

@@ -8,6 +8,7 @@
     "@hackylabs/deep-redact": "^2.2.1",
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
+    "co-body": "^6.2.0",
     "jose": "^6.0.11",
     "lodash.clonedeep": "^4.5.0"
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shushed/helpers",
-  "version": "0.0.27",
+  "version": "0.0.29",
   "author": "",
   "license": "UNLICENSED",
   "description": "",
@@ -8,6 +8,7 @@
     "@hackylabs/deep-redact": "^2.2.1",
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
+    "co-body": "^6.2.0",
     "jose": "^6.0.11",
     "lodash.clonedeep": "^4.5.0"
   },