gcf-common-lib 0.23.18 → 0.23.20

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gcf-common-lib",
   "description": "",
-  "version": "0.23.18",
+  "version": "0.23.20",
   "publishConfig": {
     "access": "public",
     "branches": [
@@ -21,6 +21,8 @@
     "@google-cloud/pubsub": "^3.3.0",
     "@google-cloud/secret-manager": "^4.2.1",
     "@google-cloud/storage": "^6.9.4",
+    "@types/amqplib": "^0.10.1",
+    "amqplib": "^0.10.3",
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
     "mongodb": "^4.14.0",

package/src/index.js

@@ -38,7 +38,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.GcfCommon = exports.secretClient = exports.storage = exports.pubSub = exports._mongodb = exports._rxjs = exports._secret_manager = exports._pubsub = exports._storage = void 0;
+exports.GcfCommon = exports.secretClient = exports.storage = exports.pubSub = exports.MongoDb = exports.RxJs = exports.SecretManager = exports.PubSub = exports.Storage = void 0;
 const pubsub_1 = require("@google-cloud/pubsub");
 const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
 const noop_1 = __importDefault(require("lodash/noop"));
@@ -46,11 +46,11 @@ const utils_1 = require("./utils");
 const storage_1 = require("@google-cloud/storage");
 const secret_manager_1 = require("@google-cloud/secret-manager");
 const lodash_1 = require("lodash");
-exports._storage = __importStar(require("@google-cloud/storage"));
-exports._pubsub = __importStar(require("@google-cloud/pubsub"));
-exports._secret_manager = __importStar(require("@google-cloud/secret-manager"));
-exports._rxjs = __importStar(require("rxjs"));
-exports._mongodb = __importStar(require("mongodb"));
+exports.Storage = __importStar(require("@google-cloud/storage"));
+exports.PubSub = __importStar(require("@google-cloud/pubsub"));
+exports.SecretManager = __importStar(require("@google-cloud/secret-manager"));
+exports.RxJs = __importStar(require("rxjs"));
+exports.MongoDb = __importStar(require("mongodb"));
 __exportStar(require("./types"), exports);
 __exportStar(require("./utils"), exports);
 exports.pubSub = new pubsub_1.PubSub();
@@ -88,14 +88,50 @@ class GcfCommon {
     }
     static publish(event, context, json, attributes) {
         return __awaiter(this, void 0, void 0, function* () {
-            const { topic, appId, env, requestId } = yield this.getTopic(event, context);
-            if (!(0, isEmpty_1.default)(topic)) {
+            const { topic, queue, appId, env, requestId } = yield this.getTopicAndQueue(event, context);
+            if (topic && !(0, isEmpty_1.default)(topic)) {
                 console.log('publish:', topic, appId, env, json, attributes);
                 return exports.pubSub.topic(topic).publishMessage({
                     json: json !== null && json !== void 0 ? json : {},
                     attributes: Object.assign(Object.assign({}, (0, lodash_1.mapValues)(attributes !== null && attributes !== void 0 ? attributes : {}, (v) => '' + v)), { requestId: requestId !== null && requestId !== void 0 ? requestId : '', appId: appId !== null && appId !== void 0 ? appId : '', env: env !== null && env !== void 0 ? env : '', type: 'response', response: '1' }),
                 });
             }
+            if (queue && !(0, isEmpty_1.default)(queue)) {
+                console.log('send:', queue, appId, env, json, attributes);
+                return yield (0, utils_1.withAmqpCh)((ch) => __awaiter(this, void 0, void 0, function* () {
+                    const payload = Buffer.from(JSON.stringify(json !== null && json !== void 0 ? json : {}));
+                    yield ch.assertQueue(queue, this.amqpOptions.assertOptions);
+                    ch.sendToQueue(queue, payload, this.amqpOptions.publishOptions);
+                }), this.amqpOptions.url);
+            }
+        });
+    }
+    static getTopicAndQueue(event, context) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            let { topic, queue, appId, env, requestId } = this.getMetadata(event, context);
+            if (!topic && !queue && ((_a = context === null || context === void 0 ? void 0 : context.resource) === null || _a === void 0 ? void 0 : _a.type) === 'storage#object') {
+                if ((context === null || context === void 0 ? void 0 : context.eventType) === 'google.storage.object.finalize') {
+                    const gsEvent = event;
+                    const [meta] = yield exports.storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+                    ({ topic, queue, appId, env, requestId } = (_b = meta === null || meta === void 0 ? void 0 : meta.metadata) !== null && _b !== void 0 ? _b : {});
+                }
+            }
+            return { topic, queue, appId, env, requestId };
+        });
+    }
+    static getQueue(event, context) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            let { queue, appId, env, requestId } = this.getMetadata(event, context);
+            if (!queue && ((_a = context === null || context === void 0 ? void 0 : context.resource) === null || _a === void 0 ? void 0 : _a.type) === 'storage#object') {
+                if ((context === null || context === void 0 ? void 0 : context.eventType) === 'google.storage.object.finalize') {
+                    const gsEvent = event;
+                    const [meta] = yield exports.storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+                    ({ queue, appId, env, requestId } = (_b = meta === null || meta === void 0 ? void 0 : meta.metadata) !== null && _b !== void 0 ? _b : {});
+                }
+            }
+            return { queue, appId, env, requestId };
         });
     }
     static getTopic(event, context) {
@@ -159,3 +195,7 @@ class GcfCommon {
     }
 }
 exports.GcfCommon = GcfCommon;
+GcfCommon.amqpOptions = {
+    assertOptions: { durable: true },
+    publishOptions: { persistent: true },
+};

package/src/index.ts

@@ -1,18 +1,19 @@
 import { PubSub } from '@google-cloud/pubsub';
 import isEmpty from 'lodash/isEmpty';
 import noop from 'lodash/noop';
-import { timeoutAfter } from './utils';
+import { timeoutAfter, withAmqpCh } from './utils';
 import { Storage } from '@google-cloud/storage';
 import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
 import { TContext, TEvent, TGSEvent, TMetadata, TPSEvent, TResponse } from './types';
 import { mapValues } from 'lodash';
+import { Options } from 'amqplib/properties';
 import Dict = NodeJS.Dict;
 
-export * as _storage from '@google-cloud/storage';
-export * as _pubsub from '@google-cloud/pubsub';
-export * as _secret_manager from '@google-cloud/secret-manager';
-export * as _rxjs from 'rxjs';
-export * as _mongodb from 'mongodb';
+export * as Storage from '@google-cloud/storage';
+export * as PubSub from '@google-cloud/pubsub';
+export * as SecretManager from '@google-cloud/secret-manager';
+export * as RxJs from 'rxjs';
+export * as MongoDb from 'mongodb';
 export * from './types';
 export * from './utils';
 
@@ -21,6 +22,15 @@ export const storage = new Storage();
 export const secretClient = new SecretManagerServiceClient();
 
 export class GcfCommon {
+  static amqpOptions: {
+    url?: string;
+    assertOptions?: Options.AssertQueue;
+    publishOptions?: Options.Publish;
+  } = {
+    assertOptions: { durable: true },
+    publishOptions: { persistent: true },
+  };
+
   /**
    *
    * @param {!TEvent} event Event payload.
@@ -54,11 +64,11 @@ export class GcfCommon {
   }
 
   static async publish(event: TEvent, context: TContext, json?: TResponse, attributes?: Dict<any>) {
-    const { topic, appId, env, requestId } = await this.getTopic(event, context);
+    const { topic, queue, appId, env, requestId } = await this.getTopicAndQueue(event, context);
 
-    if (!isEmpty(topic)) {
+    if (topic && !isEmpty(topic)) {
       console.log('publish:', topic, appId, env, json, attributes);
-      return pubSub.topic(topic as string).publishMessage({
+      return pubSub.topic(topic).publishMessage({
         json: json ?? {},
         attributes: {
           ...mapValues(attributes ?? {}, (v) => '' + v),
@@ -70,6 +80,44 @@ export class GcfCommon {
         },
       });
     }
+
+    if (queue && !isEmpty(queue)) {
+      console.log('send:', queue, appId, env, json, attributes);
+      return await withAmqpCh(async ch => {
+        const payload = Buffer.from(JSON.stringify(json ?? {}));
+        await ch.assertQueue(queue, this.amqpOptions.assertOptions);
+        ch.sendToQueue(queue, payload, this.amqpOptions.publishOptions);
+      }, this.amqpOptions.url as string);
+    }
+  }
+
+
+  static async getTopicAndQueue(event: TEvent, context: TContext) {
+    let { topic, queue, appId, env, requestId } = this.getMetadata(event, context);
+
+    if (!topic && !queue && context?.resource?.type === 'storage#object') {
+      if (context?.eventType === 'google.storage.object.finalize') {
+        const gsEvent = event as TGSEvent;
+        const [meta] = await storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+        ({ topic, queue, appId, env, requestId } = meta?.metadata ?? {});
+      }
+    }
+
+    return { topic, queue, appId, env, requestId };
+  }
+
+  static async getQueue(event: TEvent, context: TContext) {
+    let { queue, appId, env, requestId } = this.getMetadata(event, context);
+
+    if (!queue && context?.resource?.type === 'storage#object') {
+      if (context?.eventType === 'google.storage.object.finalize') {
+        const gsEvent = event as TGSEvent;
+        const [meta] = await storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+        ({ queue, appId, env, requestId } = meta?.metadata ?? {});
+      }
+    }
+
+    return { queue, appId, env, requestId };
   }
 
   static async getTopic(event: TEvent, context: TContext) {

package/src/types.ts

@@ -1,7 +1,8 @@
 import Dict = NodeJS.Dict;
 
 export type TMetadata = {
-  topic?: string; // response topic [t_{GUID}__{YYYY-MM-DD}]
+  topic?: string; // response PubSub topic [t_{GUID}__{YYYY-MM-DD}]
+  queue?: string; // response amqp queue
   requestId?: string; // for rpc response [GUID]
   env?: string; // app environment
   appId?: string; // app id

package/src/utils.js

@@ -9,7 +9,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.A1ToColNum = exports.colNumToA1 = exports.A1ToIndex = exports.indexToA1 = exports.delay = exports.timeoutAfter = void 0;
+exports.withAmqpCh = exports.withAmqpConn = exports.A1ToColNum = exports.colNumToA1 = exports.A1ToIndex = exports.indexToA1 = exports.delay = exports.timeoutAfter = void 0;
+const amqplib_1 = require("amqplib");
 /**
  *
  * @param seconds Google function v1 timeout limit (max: 9 min)
@@ -76,3 +77,23 @@ function A1ToColNum(value) {
     return result;
 }
 exports.A1ToColNum = A1ToColNum;
+function withAmqpConn(fn, url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const amqpConn = yield (0, amqplib_1.connect)(url);
+        amqpConn.on('close', () => console.info('Amqp connection closed!'));
+        return yield fn(amqpConn).finally(() => __awaiter(this, void 0, void 0, function* () { return yield amqpConn.close(); }));
+    });
+}
+exports.withAmqpConn = withAmqpConn;
+function withAmqpCh(fn, url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return yield withAmqpConn((conn) => __awaiter(this, void 0, void 0, function* () {
+            const ch = yield conn.createChannel();
+            return yield fn(ch).finally(() => __awaiter(this, void 0, void 0, function* () {
+                yield delay(1000);
+                yield ch.close();
+            }));
+        }), url);
+    });
+}
+exports.withAmqpCh = withAmqpCh;

package/src/utils.ts

@@ -1,3 +1,5 @@
+import { Channel, connect, Connection } from 'amqplib';
+
 /**
  *
  * @param seconds Google function v1 timeout limit (max: 9 min)
@@ -63,3 +65,19 @@ export function A1ToColNum(value: string) {
   }
   return result;
 }
+
+export async function withAmqpConn(fn: (conn: Connection) => Promise<any>, url: string) {
+  const amqpConn = await connect(url);
+  amqpConn.on('close', () => console.info('Amqp connection closed!'));
+  return await fn(amqpConn).finally(async () => await amqpConn.close());
+}
+
+export async function withAmqpCh(fn: (ch: Channel) => Promise<any>, url: string) {
+  return await withAmqpConn(async conn => {
+    const ch = await conn.createChannel();
+    return await fn(ch).finally(async () => {
+      await delay(1000);
+      await ch.close();
+    });
+  }, url);
+}