@contrail/util 1.3.0 → 1.3.1

package/lib/dynamodb-util/dynamodb-util.methods.d.ts

@@ -0,0 +1,26 @@
+import { DynamoDBItem, FlatDynamoItem } from './dynamodb-util.types';
+export declare type Environment = 'dev' | 'prod';
+export declare function getAll(scanOperations: AsyncGenerator<{
+    [key: string]: any;
+}[]>): Promise<{
+    [key: string]: any;
+}[]>;
+interface CacheOptions {
+    useCache: boolean;
+    overwriteCache: boolean;
+    cacheName: string;
+}
+export declare function getAllWithCaching<T>(getAllFunc: () => Promise<T[]>, options: CacheOptions): Promise<T[]>;
+declare type DyanmoQuery = {
+    TableName: string;
+    ExpressionAttributeValues: {
+        [key: string]: any;
+    };
+    FilterExpression: string;
+};
+export declare function scanDynamo(query: DyanmoQuery): AsyncGenerator<{
+    [key: string]: any;
+}[]>;
+export declare function pushData<T>(data: DynamoDBItem<T>[], tableName: string): Promise<void>;
+export declare function flattenDynamoItem<T>(item: DynamoDBItem<T>): FlatDynamoItem<T>;
+export {};

package/lib/dynamodb-util/dynamodb-util.methods.js

@@ -0,0 +1,192 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncDelegator = (this && this.__asyncDelegator) || function (o) {
+    var i, p;
+    return i = {}, verb("next"), verb("throw", function (e) { throw e; }), verb("return"), i[Symbol.iterator] = function () { return this; }, i;
+    function verb(n, f) { i[n] = o[n] ? function (v) { return (p = !p) ? { value: __await(o[n](v)), done: n === "return" } : f ? f(v) : v; } : f; }
+};
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.flattenDynamoItem = exports.pushData = exports.scanDynamo = exports.getAllWithCaching = exports.getAll = void 0;
+const aws = require('aws-sdk');
+const ddt = require('dynamodb-data-types');
+const fs_1 = require("fs");
+const file_util_1 = require("../file-util/file-util");
+const dynamo = new aws.DynamoDB.DocumentClient({
+    region: 'us-east-1',
+});
+const CACHE_DIR = 'dynamodb_cache';
+ddt.preserveArrays();
+const attr = ddt.AttributeValue;
+function getAllAppsFromDynamo(environment) {
+    return () => __awaiter(this, void 0, void 0, function* () {
+        return getAll(scanForApps(environment));
+    });
+}
+function getAll(scanOperations) {
+    var e_1, _a;
+    return __awaiter(this, void 0, void 0, function* () {
+        const scan = scanOperations;
+        const allItems = [];
+        try {
+            for (var scan_1 = __asyncValues(scan), scan_1_1; scan_1_1 = yield scan_1.next(), !scan_1_1.done;) {
+                const items = scan_1_1.value;
+                allItems.push(...items);
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (scan_1_1 && !scan_1_1.done && (_a = scan_1.return)) yield _a.call(scan_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return allItems;
+    });
+}
+exports.getAll = getAll;
+const getAllApps = (environment, options) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a, _b;
+    return getAllWithCaching(getAllAppsFromDynamo(environment), {
+        useCache: (_a = options.useCache) !== null && _a !== void 0 ? _a : false,
+        overwriteCache: (_b = options.overwriteCache) !== null && _b !== void 0 ? _b : false,
+        cacheName: `${environment}-apps`,
+    });
+});
+function getAllWithCaching(getAllFunc, options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { useCache, overwriteCache, cacheName } = options;
+        if (useCache) {
+            console.log('Attempt to use cache:', cacheName);
+            if (yield (0, file_util_1.doesFileExist)(getCacheName(cacheName))) {
+                return readCachedResults(cacheName);
+            }
+            else {
+                console.log('Cache not found:', cacheName);
+            }
+        }
+        console.log('Fetching from DynamoDB:', cacheName);
+        const data = yield getAllFunc();
+        if (overwriteCache) {
+            console.log('Caching results:', cacheName);
+            yield cacheResultsInJSON(cacheName, data);
+        }
+        return data;
+    });
+}
+exports.getAllWithCaching = getAllWithCaching;
+const cacheResultsInJSON = (cacheName, data) => __awaiter(void 0, void 0, void 0, function* () {
+    yield (0, file_util_1.createDirectoryIfNotExists)(CACHE_DIR);
+    yield fs_1.promises.writeFile(getCacheName(cacheName), JSON.stringify(data, null, 2), 'utf8');
+});
+const readCachedResults = (cacheName) => __awaiter(void 0, void 0, void 0, function* () {
+    const data = yield fs_1.promises.readFile(getCacheName(cacheName), 'utf8');
+    return JSON.parse(data);
+});
+const getCacheName = (cacheName) => `${CACHE_DIR}/${cacheName}.json`;
+function scanForApps(environment) {
+    return __asyncGenerator(this, arguments, function* scanForApps_1() {
+        const query = {
+            TableName: `${environment}-apps`,
+            ExpressionAttributeValues: {
+                ':pk': 'app:',
+                ':sk': 'app:',
+            },
+            FilterExpression: 'begins_with(PK, :pk) AND begins_with(SK, :sk)',
+        };
+        yield __await(yield* __asyncDelegator(__asyncValues(scanDynamo(query))));
+    });
+}
+function scanDynamo(query) {
+    return __asyncGenerator(this, arguments, function* scanDynamo_1() {
+        let lastEvaluatedKey = null;
+        const params = Object.assign(Object.assign({}, query), { ExclusiveStartKey: null });
+        do {
+            const response = (yield __await(dynamo.scan(params).promise()));
+            yield yield __await(response.Items);
+            lastEvaluatedKey = response.LastEvaluatedKey;
+            params.ExclusiveStartKey = lastEvaluatedKey;
+        } while (lastEvaluatedKey);
+    });
+}
+exports.scanDynamo = scanDynamo;
+function pushData(data, tableName) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const ddb = new aws.DynamoDB({ apiVersion: '2012-08-10', region: 'us-east-1' });
+        let itemsToPush = [];
+        for (let index = 0; index < data.length; index++) {
+            const element = data[index];
+            const item = {
+                PutRequest: {
+                    Item: attr.wrap(element),
+                },
+            };
+            itemsToPush.push(item);
+            if (itemsToPush.length === 25 || index === data.length - 1) {
+                const params = {
+                    RequestItems: {},
+                };
+                params.RequestItems[tableName] = itemsToPush;
+                ddb.batchWriteItem(params, function (err, data) {
+                    if (err) {
+                        console.log('Error', err);
+                    }
+                    else {
+                        console.log('Success', data);
+                    }
+                });
+                yield delay(1000);
+                console.log('pushed', index + 1, 'of', data.length);
+                itemsToPush = [];
+            }
+        }
+    });
+}
+exports.pushData = pushData;
+function delay(ms) {
+    return __awaiter(this, void 0, void 0, function* () {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    });
+}
+function flattenDynamoItem(item) {
+    const { properties } = item, rest = __rest(item, ["properties"]);
+    return Object.assign(Object.assign({}, rest), properties);
+}
+exports.flattenDynamoItem = flattenDynamoItem;

package/lib/dynamodb-util/dynamodb-util.types.d.ts

@@ -0,0 +1,20 @@
+export declare type DynamoDBItem<T> = {
+    PK: string;
+    SK: string;
+    ALT_PK1?: string;
+    ALT_SK1?: string;
+    ALT_PK2?: string;
+    ALT_SK2?: string;
+    ALT_PK3?: string;
+    ALT_SK3?: string;
+    properties: BaseProperties & T;
+};
+declare type BaseProperties = {
+    id: string;
+    createdById: string;
+    createdOn: string;
+    updatedById: string;
+    updatedOn: string;
+};
+export declare type FlatDynamoItem<T> = Omit<DynamoDBItem<T>, 'properties'> & BaseProperties & T;
+export {};

package/lib/dynamodb-util/dynamodb-util.types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/dynamodb-util/index.d.ts

@@ -0,0 +1,2 @@
+export * from './dynamodb-util.types';
+export * from './dynamodb-util.methods';

package/lib/dynamodb-util/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./dynamodb-util.types"), exports);
+__exportStar(require("./dynamodb-util.methods"), exports);

package/lib/file-util/file-util.d.ts

@@ -0,0 +1,7 @@
+export declare function doesFileExist(filePath: any): Promise<boolean>;
+export declare const createDirectoryIfNotExists: (directoryPath: string) => Promise<void>;
+export declare const deleteDirectoryAndContentsIfExists: (directoryPath: string) => Promise<void>;
+export declare const copyFile: (options: {
+    sourcePath: string;
+    destinationPath: string;
+}) => Promise<void>;

package/lib/file-util/file-util.js

@@ -0,0 +1,46 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.copyFile = exports.deleteDirectoryAndContentsIfExists = exports.createDirectoryIfNotExists = exports.doesFileExist = void 0;
+const fs_1 = require("fs");
+function doesFileExist(filePath) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            yield fs_1.promises.access(filePath);
+            return true;
+        }
+        catch (error) {
+            return false;
+        }
+    });
+}
+exports.doesFileExist = doesFileExist;
+const createDirectoryIfNotExists = (directoryPath) => __awaiter(void 0, void 0, void 0, function* () {
+    yield fs_1.promises.mkdir(directoryPath, { recursive: true }).catch((err) => {
+        if (err.code !== 'EEXIST') {
+            console.error(err);
+        }
+    });
+});
+exports.createDirectoryIfNotExists = createDirectoryIfNotExists;
+const deleteDirectoryAndContentsIfExists = (directoryPath) => __awaiter(void 0, void 0, void 0, function* () {
+    yield fs_1.promises.rmdir(directoryPath, { recursive: true }).catch((err) => {
+        if (err.code !== 'ENOENT') {
+            console.error(err);
+        }
+    });
+});
+exports.deleteDirectoryAndContentsIfExists = deleteDirectoryAndContentsIfExists;
+const copyFile = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { sourcePath, destinationPath } = options;
+    yield fs_1.promises.copyFile(sourcePath, destinationPath);
+});
+exports.copyFile = copyFile;

package/lib/object-util/cloneDeep/cloneDeep.js

@@ -1,8 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.cloneDeep = cloneDeep;
+const cloneDeepArray_1 = require("./cloneDeepArray");
 function cloneDeep(obj) {
     if (obj == null)
         return null;
+    if (Array.isArray(obj)) {
+        return (0, cloneDeepArray_1.cloneDeepArray)(obj, jsonClone);
+    }
+    return jsonClone(obj);
+}
+function jsonClone(obj) {
     return JSON.parse(JSON.stringify(obj));
 }

package/lib/object-util/cloneDeep/cloneDeepArray.d.ts

@@ -0,0 +1 @@
+export declare function cloneDeepArray<T, R>(arr: T[], cloneFn: (chunk: T[]) => R[]): R[];

package/lib/object-util/cloneDeep/cloneDeepArray.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cloneDeepArray = cloneDeepArray;
+const OPTIMAL_CHUNK_SIZE_BYTES = 10 * 1024 * 1024;
+const MIN_ARRAY_LENGTH_BEFORE_CHUNKING = 1024;
+function cloneDeepArray(arr, cloneFn) {
+    var _a, _b;
+    if (!arr || arr.length === 0)
+        return [];
+    if (arr.length <= MIN_ARRAY_LENGTH_BEFORE_CHUNKING) {
+        return cloneFn(arr);
+    }
+    const sampleSize = ((_b = (_a = JSON.stringify(arr[0])) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 2) * 2;
+    const elementsPerChunk = Math.max(1, Math.floor(OPTIMAL_CHUNK_SIZE_BYTES / sampleSize));
+    if (elementsPerChunk >= arr.length) {
+        return cloneFn(arr);
+    }
+    const result = [];
+    for (let i = 0; i < arr.length; i += elementsPerChunk) {
+        const clonedChunk = cloneFn(arr.slice(i, i + elementsPerChunk));
+        for (let j = 0; j < clonedChunk.length; j++) {
+            result.push(clonedChunk[j]);
+        }
+    }
+    return result;
+}

package/lib/object-util/cloneDeepPreserveDates/cloneDeepPreserveDates.js

@@ -4,9 +4,16 @@ exports.cloneDeepPreserveDates = cloneDeepPreserveDates;
 exports.tagDates = tagDates;
 exports.reviveTaggedDates = reviveTaggedDates;
 exports.isTaggedDate = isTaggedDate;
+const cloneDeepArray_1 = require("../cloneDeep/cloneDeepArray");
 function cloneDeepPreserveDates(obj) {
     if (obj == null)
         return null;
+    if (Array.isArray(obj)) {
+        return (0, cloneDeepArray_1.cloneDeepArray)(obj, jsonClonePreserveDates);
+    }
+    return jsonClonePreserveDates(obj);
+}
+function jsonClonePreserveDates(obj) {
     const tagged = tagDates(obj);
     const json = JSON.stringify(tagged);
     return JSON.parse(json, reviveTaggedDates);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@contrail/util",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "General JavaScript utilities",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",