gcf-common-lib 0.23.14 → 0.23.16

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gcf-common-lib",
   "description": "",
-  "version": "0.23.14",
+  "version": "0.23.16",
   "publishConfig": {
     "access": "public",
     "branches": [
@@ -20,7 +20,7 @@
   "dependencies": {
     "@google-cloud/pubsub": "^3.3.0",
     "@google-cloud/secret-manager": "^4.2.1",
-    "@google-cloud/storage": "^6.9.3",
+    "@google-cloud/storage": "^6.9.4",
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
     "mongodb": "^4.14.0",
@@ -29,8 +29,8 @@
   "devDependencies": {
     "@tsconfig/node14": "^1.0.3",
     "@types/lodash": "^4.14.191",
-    "@types/node": "^14.18.36"
+    "@types/node": "^14.18.37"
   },
   "author": "alert83@gmail.com",
-  "license": "MIT"
+  "license": ""
 }

package/src/index.js

@@ -14,12 +14,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GcfCommon = exports.secretClient = exports.storage = exports.pubSub = void 0;
 const pubsub_1 = require("@google-cloud/pubsub");
-const fromPairs_1 = __importDefault(require("lodash/fromPairs"));
 const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
 const noop_1 = __importDefault(require("lodash/noop"));
 const utils_1 = require("./utils");
 const storage_1 = require("@google-cloud/storage");
 const secret_manager_1 = require("@google-cloud/secret-manager");
+const lodash_1 = require("lodash");
 exports.pubSub = new pubsub_1.PubSub();
 exports.storage = new storage_1.Storage();
 exports.secretClient = new secret_manager_1.SecretManagerServiceClient();
@@ -55,64 +55,60 @@ class GcfCommon {
     }
     static publish(event, context, json, attributes) {
         return __awaiter(this, void 0, void 0, function* () {
-            const { topic, requestId, env } = yield this.getTopic(event, context);
-            console.log('publish:', topic, env, json, attributes);
+            const { topic, appId, env, requestId } = yield this.getTopic(event, context);
             if (!(0, isEmpty_1.default)(topic)) {
+                console.log('publish:', topic, appId, env, json, attributes);
                 return exports.pubSub.topic(topic).publishMessage({
                     json: json !== null && json !== void 0 ? json : {},
-                    attributes: Object.assign(Object.assign({}, (0, fromPairs_1.default)(Object.entries(attributes !== null && attributes !== void 0 ? attributes : {}).map(([k, v]) => [k, '' + v]))), { env: env !== null && env !== void 0 ? env : '', requestId: requestId !== null && requestId !== void 0 ? requestId : '', type: 'response', response: '1' }),
+                    attributes: Object.assign(Object.assign({}, (0, lodash_1.mapValues)(attributes !== null && attributes !== void 0 ? attributes : {}, (v) => '' + v)), { requestId: requestId !== null && requestId !== void 0 ? requestId : '', appId: appId !== null && appId !== void 0 ? appId : '', env: env !== null && env !== void 0 ? env : '', type: 'response', response: '1' }),
                 });
             }
         });
     }
     static getTopic(event, context) {
-        var _a, _b, _c, _d;
+        var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
-            /** t_{GUID}__{YYYY-MM-DD} */
-            let topic;
-            let requestId;
-            let env;
-            switch (context === null || context === void 0 ? void 0 : context.eventType) {
-                case 'google.storage.object.finalize': {
+            let { topic, appId, env, requestId } = this.getMetadata(event, context);
+            if (!topic && ((_a = context === null || context === void 0 ? void 0 : context.resource) === null || _a === void 0 ? void 0 : _a.type) === 'storage#object') {
+                if ((context === null || context === void 0 ? void 0 : context.eventType) === 'google.storage.object.finalize') {
                     const gsEvent = event;
-                    ({ topic, requestId, env } = (_a = gsEvent === null || gsEvent === void 0 ? void 0 : gsEvent.metadata) !== null && _a !== void 0 ? _a : {});
-                    if (!topic && ((_b = context === null || context === void 0 ? void 0 : context.resource) === null || _b === void 0 ? void 0 : _b.type) === 'storage#object') {
-                        const [meta] = yield exports.storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
-                        ({ topic, requestId, env } = (_c = meta === null || meta === void 0 ? void 0 : meta.metadata) !== null && _c !== void 0 ? _c : {});
-                    }
-                    break;
-                }
-                case 'google.pubsub.topic.publish': {
-                    const psEvent = event;
-                    ({ topic, requestId, env } = (_d = psEvent === null || psEvent === void 0 ? void 0 : psEvent.attributes) !== null && _d !== void 0 ? _d : {});
-                    break;
+                    const [meta] = yield exports.storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+                    ({ topic, appId, env, requestId } = (_b = meta === null || meta === void 0 ? void 0 : meta.metadata) !== null && _b !== void 0 ? _b : {});
                 }
             }
-            return { topic, requestId, env };
+            return { topic, appId, env, requestId };
         });
     }
     static getOptions(event, context) {
-        var _a, _b, _c, _d, _e, _f, _g;
+        var _a;
         return __awaiter(this, void 0, void 0, function* () {
-            switch (context === null || context === void 0 ? void 0 : context.eventType) {
-                case 'google.storage.object.finalize': {
+            let { options } = this.getMetadata(event, context);
+            if (!options) {
+                if ((context === null || context === void 0 ? void 0 : context.eventType) === 'google.storage.object.finalize') {
                     const gsEvent = event;
-                    if ((_a = gsEvent === null || gsEvent === void 0 ? void 0 : gsEvent.metadata) === null || _a === void 0 ? void 0 : _a.options) {
-                        return JSON.parse((_c = (_b = gsEvent === null || gsEvent === void 0 ? void 0 : gsEvent.metadata) === null || _b === void 0 ? void 0 : _b.options) !== null && _c !== void 0 ? _c : '{}');
-                    }
-                    else {
-                        const file = exports.storage.bucket(gsEvent.bucket).file(gsEvent.name);
-                        const [meta] = yield file.getMetadata();
-                        return JSON.parse((_e = (_d = meta === null || meta === void 0 ? void 0 : meta.metadata) === null || _d === void 0 ? void 0 : _d.options) !== null && _e !== void 0 ? _e : '{}');
-                    }
-                }
-                case 'google.pubsub.topic.publish': {
-                    const psEvent = event;
-                    return JSON.parse((_g = (_f = psEvent === null || psEvent === void 0 ? void 0 : psEvent.attributes) === null || _f === void 0 ? void 0 : _f.options) !== null && _g !== void 0 ? _g : '{}');
+                    const [meta] = yield exports.storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+                    options = (_a = meta === null || meta === void 0 ? void 0 : meta.metadata) === null || _a === void 0 ? void 0 : _a.options;
                 }
             }
+            return JSON.parse(options !== null && options !== void 0 ? options : '{}');
         });
     }
+    static getMetadata(event, context) {
+        let metadata;
+        switch (context === null || context === void 0 ? void 0 : context.eventType) {
+            case 'google.storage.object.finalize': {
+                const gsEvent = event;
+                metadata = gsEvent === null || gsEvent === void 0 ? void 0 : gsEvent.metadata;
+                break;
+            }
+            case 'google.pubsub.topic.publish': {
+                const psEvent = event;
+                metadata = psEvent === null || psEvent === void 0 ? void 0 : psEvent.attributes;
+                break;
+            }
+        }
+        return metadata !== null && metadata !== void 0 ? metadata : {};
+    }
     static getSecret(name, version) {
         var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {

package/src/index.ts

@@ -1,73 +1,13 @@
 import { PubSub } from '@google-cloud/pubsub';
-import fromPairs from 'lodash/fromPairs';
 import isEmpty from 'lodash/isEmpty';
 import noop from 'lodash/noop';
 import { timeoutAfter } from './utils';
-import { File, Storage } from '@google-cloud/storage';
+import { Storage } from '@google-cloud/storage';
 import { SecretManagerServiceClient } from '@google-cloud/secret-manager';
+import { TContext, TEvent, TGSEvent, TMetadata, TPSEvent, TResponse } from './types';
+import { mapValues } from 'lodash';
 import Dict = NodeJS.Dict;
 
-export type TGSEvent = {
-  bucket: string;
-  contentType: string;
-  crc32c: string;
-  etag: string;
-  generation: string;
-  id: string;
-  kind: string; // 'storage#object'
-  md5Hash: string;
-  mediaLink: string;
-  metadata: {
-    // [key: string]: string,
-    topic: string;
-    requestId: string;
-    env: string;
-    options?: any;
-  };
-  metageneration: string;
-  name: string;
-  selfLink: string;
-  size: string;
-  storageClass: string;
-  timeCreated: string;
-  timeStorageClassUpdated: string;
-  updated: string;
-};
-
-export type TPSEvent = {
-  '@type': string; // 'type.googleapis.com/google.pubsub.v1.PubsubMessage'
-  attributes?: {
-    // [k: string]: string,
-    topic: string;
-    requestId: string;
-    env: string;
-    options?: any;
-  };
-  data?: string | Dict<any>;
-};
-
-export type TEvent = TGSEvent | TPSEvent;
-
-export type TContext = {
-  eventId: string;
-  timestamp: string;
-  eventType: 'google.storage.object.finalize' | 'google.pubsub.topic.publish';
-  resource: {
-    service: 'storage.googleapis.com' | 'pubsub.googleapis.com';
-    type: 'storage#object' | 'type.googleapis.com/google.pubsub.v1.PubsubMessage';
-    name: string;
-  };
-};
-
-export type TResponse = {
-  [prop: string]: any;
-  error?: {
-    name: string;
-    message: string;
-    stack?: string;
-  };
-};
-
 export const pubSub = new PubSub();
 export const storage = new Storage();
 export const secretClient = new SecretManagerServiceClient();
@@ -106,16 +46,17 @@ export class GcfCommon {
   }
 
   static async publish(event: TEvent, context: TContext, json?: TResponse, attributes?: Dict<any>) {
-    const { topic, requestId, env } = await this.getTopic(event, context);
-    console.log('publish:', topic, env, json, attributes);
+    const { topic, appId, env, requestId } = await this.getTopic(event, context);
 
     if (!isEmpty(topic)) {
+      console.log('publish:', topic, appId, env, json, attributes);
       return pubSub.topic(topic as string).publishMessage({
         json: json ?? {},
         attributes: {
-          ...fromPairs(Object.entries(attributes ?? {}).map(([k, v]) => [k, '' + v])),
-          env: env ?? '',
+          ...mapValues(attributes ?? {}, (v) => '' + v),
           requestId: requestId ?? '',
+          appId: appId ?? '',
+          env: env ?? '',
           type: 'response',
           response: '1',
         },
@@ -124,48 +65,50 @@ export class GcfCommon {
   }
 
   static async getTopic(event: TEvent, context: TContext) {
-    /** t_{GUID}__{YYYY-MM-DD} */
-    let topic: string | undefined;
-    let requestId: string | undefined;
-    let env: string | undefined;
+    let { topic, appId, env, requestId } = this.getMetadata(event, context);
 
-    switch (context?.eventType) {
-      case 'google.storage.object.finalize': {
+    if (!topic && context?.resource?.type === 'storage#object') {
+      if (context?.eventType === 'google.storage.object.finalize') {
         const gsEvent = event as TGSEvent;
-        ({ topic, requestId, env } = gsEvent?.metadata ?? {});
-        if (!topic && context?.resource?.type === 'storage#object') {
-          const [meta] = await storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
-          ({ topic, requestId, env } = meta?.metadata ?? {});
-        }
-        break;
-      }
-      case 'google.pubsub.topic.publish': {
-        const psEvent = event as TPSEvent;
-        ({ topic, requestId, env } = psEvent?.attributes ?? ({} as any));
-        break;
+        const [meta] = await storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+        ({ topic, appId, env, requestId } = meta?.metadata ?? {});
       }
     }
 
-    return { topic, requestId, env };
+    return { topic, appId, env, requestId };
   }
 
   static async getOptions(event: TEvent, context: TContext): Promise<Dict<any>> {
+    let { options } = this.getMetadata(event, context);
+
+    if (!options) {
+      if (context?.eventType === 'google.storage.object.finalize') {
+        const gsEvent = event as TGSEvent;
+        const [meta] = await storage.bucket(gsEvent.bucket).file(gsEvent.name).getMetadata();
+        options = meta?.metadata?.options;
+      }
+    }
+
+    return JSON.parse(options ?? '{}');
+  }
+
+  static getMetadata(event: TEvent, context: TContext) {
+    let metadata: TMetadata | undefined;
+
     switch (context?.eventType) {
       case 'google.storage.object.finalize': {
         const gsEvent = event as TGSEvent;
-        if (gsEvent?.metadata?.options) {
-          return JSON.parse(gsEvent?.metadata?.options ?? '{}');
-        } else {
-          const file: File = storage.bucket(gsEvent.bucket).file(gsEvent.name);
-          const [meta] = await file.getMetadata();
-          return JSON.parse(meta?.metadata?.options ?? '{}');
-        }
+        metadata = gsEvent?.metadata;
+        break;
       }
       case 'google.pubsub.topic.publish': {
         const psEvent = event as TPSEvent;
-        return JSON.parse(psEvent?.attributes?.options ?? '{}');
+        metadata = psEvent?.attributes;
+        break;
       }
     }
+
+    return metadata ?? {};
   }
 
   static async getSecret(name: string, version?: string) {

package/src/mongo-lock.js

@@ -74,7 +74,7 @@ class MongoLock {
                 // console.log(locked);
                 if (!locked)
                     throw new Error('Could not acquire lock');
-            }), wait ? (0, rxjs_1.retry)({ count, delay }) : rxjs_1.identity, (0, rxjs_1.switchMap)(() => __awaiter(this, void 0, void 0, function* () { return fn().finally(() => this.releaseLock(name).catch(noop_1.default)); }))));
+            }), wait ? (0, rxjs_1.retry)({ count, delay }) : (0, rxjs_1.tap)(rxjs_1.identity), (0, rxjs_1.switchMap)(() => __awaiter(this, void 0, void 0, function* () { return fn().finally(() => this.releaseLock(name).catch(noop_1.default)); }))));
         });
     }
     checkLock(name, waitSeconds = 0) {
@@ -86,7 +86,7 @@ class MongoLock {
                 // console.log(lock);
                 if (lock)
                     throw lock;
-            }), waitSeconds > 0 ? (0, rxjs_1.retry)({ count, delay }) : rxjs_1.identity));
+            }), waitSeconds > 0 ? (0, rxjs_1.retry)({ count, delay }) : (0, rxjs_1.tap)(rxjs_1.identity)));
         });
     }
 }

package/src/mongo-lock.ts

@@ -1,7 +1,7 @@
 import { MongoClient, MongoError } from 'mongodb';
 import moment from 'moment';
 import noop from 'lodash/noop';
-import { defer, from, identity, lastValueFrom, map, retry, switchMap } from 'rxjs';
+import { defer, from, identity, lastValueFrom, map, of, retry, switchMap, tap } from 'rxjs';
 import { MongoHelper } from './mongo-helper';
 
 export class MongoLock {
@@ -61,7 +61,7 @@ export class MongoLock {
           // console.log(locked);
           if (!locked) throw new Error('Could not acquire lock');
         }),
-        wait ? retry({ count, delay }) : identity,
+        wait ? retry({ count, delay }) : tap(identity),
         switchMap(async () => fn().finally(() => this.releaseLock(name).catch(noop))),
       ),
     );
@@ -78,7 +78,7 @@ export class MongoLock {
           // console.log(lock);
           if (lock) throw lock;
         }),
-        waitSeconds > 0 ? retry({ count, delay }) : identity,
+        waitSeconds > 0 ? retry({ count, delay }) : tap(identity),
       ),
     );
   }

package/src/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/src/types.ts

@@ -0,0 +1,58 @@
+import Dict = NodeJS.Dict;
+
+export type TMetadata = {
+  topic?: string; // response topic [t_{GUID}__{YYYY-MM-DD}]
+  requestId?: string; // for rpc response [GUID]
+  env?: string; // app environment
+  appId?: string; // app id
+  options?: string;
+}
+
+export type TGSEvent = {
+  bucket: string;
+  contentType: string;
+  crc32c: string;
+  etag: string;
+  generation: string;
+  id: string;
+  kind: string; // 'storage#object'
+  md5Hash: string;
+  mediaLink: string;
+  metadata: TMetadata;
+  metageneration: string;
+  name: string;
+  selfLink: string;
+  size: string;
+  storageClass: string;
+  timeCreated: string;
+  timeStorageClassUpdated: string;
+  updated: string;
+};
+
+export type TPSEvent = {
+  '@type': string; // 'type.googleapis.com/google.pubsub.v1.PubsubMessage'
+  attributes?: TMetadata;
+  data?: string | Dict<any>;
+};
+
+export type TEvent = TGSEvent | TPSEvent;
+
+export type TContext = {
+  eventId: string;
+  timestamp: string;
+  eventType: 'google.storage.object.finalize' | 'google.pubsub.topic.publish';
+  resource: {
+    service: 'storage.googleapis.com' | 'pubsub.googleapis.com';
+    type: 'storage#object' | 'type.googleapis.com/google.pubsub.v1.PubsubMessage';
+    name: string;
+  };
+};
+
+export type TResponse = {
+  [prop: string]: any;
+  error?: {
+    name: string;
+    message: string;
+    stack?: string;
+  };
+};

package/src/utils.js

@@ -9,21 +9,27 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.A1ToColNum = exports.colNumToA1 = exports.A1ToIndex = exports.indexToA1 = exports.timeoutAfter = void 0;
+exports.A1ToColNum = exports.colNumToA1 = exports.A1ToIndex = exports.indexToA1 = exports.delay = exports.timeoutAfter = void 0;
 /**
  *
- * @param seconds Google function timeout limit (max: 9 min)
+ * @param seconds Google function v1 timeout limit (max: 9 min)
  */
 function timeoutAfter(seconds = 540) {
     return __awaiter(this, void 0, void 0, function* () {
         return new Promise((resolve, reject) => {
-            setTimeout(() => {
-                reject(new Error(`${seconds} seconds timeout exceeded`));
-            }, seconds * 1000);
+            setTimeout(() => reject(new Error(`${seconds} seconds timeout exceeded`)), seconds * 1000);
         });
     });
 }
 exports.timeoutAfter = timeoutAfter;
+function delay(seconds) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return new Promise((resolve, reject) => {
+            setTimeout(() => resolve(undefined), seconds * 1000);
+        });
+    });
+}
+exports.delay = delay;
 //
 function indexToA1(idx) {
     return colNumToA1(idx + 1);

package/src/utils.ts

@@ -1,12 +1,16 @@
 /**
  *
- * @param seconds Google function timeout limit (max: 9 min)
+ * @param seconds Google function v1 timeout limit (max: 9 min)
  */
 export async function timeoutAfter(seconds: number = 540) {
   return new Promise<undefined>((resolve, reject) => {
-    setTimeout(() => {
-      reject(new Error(`${seconds} seconds timeout exceeded`));
-    }, seconds * 1000);
+    setTimeout(() => reject(new Error(`${seconds} seconds timeout exceeded`)), seconds * 1000);
+  });
+}
+
+export async function delay(seconds: number) {
+  return new Promise<undefined>((resolve, reject) => {
+    setTimeout(() => resolve(undefined), seconds * 1000);
   });
 }