ag-common 0.0.345 → 0.0.347

package/dist/api/helpers/dynamo.js

@@ -80,7 +80,7 @@ let batchWriteRaw = (req) => __awaiter(void 0, void 0, void 0, function* () {
 const batchWrite = (tableName, itemsIn, breakOnError = false) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;
     let items = JSON.parse(JSON.stringify(itemsIn));
-    (0, log_1.info)(`push to dynamo:${tableName} - count=${itemsIn.length}`);
+    (0, log_1.debug)(`push to dynamo:${tableName} - count=${itemsIn.length}`);
     const error = [];
     while (items.length > 0) {
         const { part, rest } = (0, array_1.take)(items, 25);
@@ -320,15 +320,21 @@ const wipeTable = (tableName) => __awaiter(void 0, void 0, void 0, function* ()
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
     let keyHash = infoV.Table.KeySchema.find((k) => k.KeyType === 'HASH').AttributeName;
-    let all = yield dbRaw
-        .scan({
+    const params = {
         TableName: tableName,
-    })
-        .promise();
-    (0, log_1.warn)(`will delete ${(_h = all === null || all === void 0 ? void 0 : all.Items) === null || _h === void 0 ? void 0 : _h.length} items from ${tableName}`);
+        ExclusiveStartKey: undefined,
+    };
+    let all = [];
+    let working;
+    do {
+        working = yield dbRaw.scan(params).promise();
+        (_h = working.Items) === null || _h === void 0 ? void 0 : _h.forEach((item) => all.push(item));
+        params.ExclusiveStartKey = working.LastEvaluatedKey;
+    } while (typeof working.LastEvaluatedKey !== 'undefined');
+    (0, log_1.warn)(`will delete ${all === null || all === void 0 ? void 0 : all.length} items from ${tableName}`);
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
-    const datagr = (0, array_1.chunk)(all.Items, 25);
+    const datagr = (0, array_1.chunk)(all, 25);
     let res = yield Promise.all(datagr.map((group) => dbRaw
         .batchWriteItem({
         RequestItems: {

package/dist/api/helpers/s3.d.ts

@@ -1,13 +1,26 @@
 /// <reference types="node" />
 import { Blob } from 'aws-sdk/lib/dynamodb/document_client';
 import S3 from 'aws-sdk/clients/s3';
+import { PromiseResult } from 'aws-sdk/lib/request';
+import { AWSError } from 'aws-sdk/lib/core';
+import { error } from '../../common/helpers/log';
 export declare const setS3: (region: string) => void;
 export declare const getS3Object: ({ fileurl, }: {
     fileurl: {
         Bucket: string;
         Key: string;
     };
-}) => Promise<import("aws-sdk/lib/request").PromiseResult<S3.GetObjectOutput, import("aws-sdk/lib/error").AWSError>>;
+}) => Promise<PromiseResult<S3.GetObjectOutput, AWSError>>;
+export interface IS3Object {
+    bucket: string;
+    key: string;
+    content: S3.Body | undefined;
+}
+/** function generator to get s3 files */
+export declare function getS3Objects({ bucket, keys, }: {
+    bucket: string;
+    keys: string[];
+}): AsyncGenerator<IS3Object, void, unknown>;
 export declare const putS3Object: ({ Body, Bucket, Key, ContentType, }: {
     ContentType?: string | undefined;
     Body: string | Blob;
@@ -27,3 +40,10 @@ export declare const deleteFile: ({ Bucket, Key, }: {
 }) => Promise<{
     error?: string;
 }>;
+export declare const deleteFiles: ({ Bucket, Keys, }: {
+    Bucket: string;
+    Keys: string[];
+}) => Promise<{
+    error?: string;
+}>;
+export declare function listFiles(bucketName: string): Promise<string[]>;

package/dist/api/helpers/s3.js

@@ -8,22 +8,55 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.deleteFile = exports.uploadFile = exports.putS3Object = exports.getS3Object = exports.setS3 = void 0;
+exports.listFiles = exports.deleteFiles = exports.deleteFile = exports.uploadFile = exports.putS3Object = exports.getS3Objects = exports.getS3Object = exports.setS3 = void 0;
 const s3_1 = __importDefault(require("aws-sdk/clients/s3"));
+const log_1 = require("../../common/helpers/log");
+const array_1 = require("../../common/helpers/array");
 let s3 = new s3_1.default();
 const setS3 = (region) => {
     s3 = new s3_1.default({ region });
 };
 exports.setS3 = setS3;
-const getS3Object = ({ fileurl, }) => __awaiter(void 0, void 0, void 0, function* () {
-    const content = yield s3.getObject(fileurl).promise();
-    return content;
-});
+const getS3Object = ({ fileurl, }) => __awaiter(void 0, void 0, void 0, function* () { return s3.getObject(fileurl).promise(); });
 exports.getS3Object = getS3Object;
+/** function generator to get s3 files */
+function getS3Objects({ bucket, keys, }) {
+    return __asyncGenerator(this, arguments, function* getS3Objects_1() {
+        let toProcess = keys.map((Key) => ({
+            Bucket: bucket,
+            Key,
+        }));
+        while (toProcess.length > 0) {
+            const { part, rest } = (0, array_1.take)(toProcess, 1);
+            toProcess = rest;
+            const fileurl = part[0];
+            const content = yield __await((0, exports.getS3Object)({ fileurl }));
+            const ret1 = {
+                bucket: fileurl.Bucket,
+                key: fileurl.Key,
+                content: content.Body,
+            };
+            yield yield __await(ret1);
+        }
+    });
+}
+exports.getS3Objects = getS3Objects;
 const putS3Object = ({ Body, Bucket, Key, ContentType, }) => __awaiter(void 0, void 0, void 0, function* () {
     const r = yield s3
         .putObject({
@@ -52,3 +85,50 @@ const deleteFile = ({ Bucket, Key, }) => __awaiter(void 0, void 0, void 0, funct
     return {};
 });
 exports.deleteFile = deleteFile;
+const deleteFiles = ({ Bucket, Keys, }) => __awaiter(void 0, void 0, void 0, function* () {
+    let toDelete = Keys.map((Key) => ({ Key }));
+    while (toDelete.length > 0) {
+        const { part, rest } = (0, array_1.take)(toDelete, 900);
+        toDelete = rest;
+        const res = yield s3
+            .deleteObjects({
+            Bucket,
+            Delete: { Objects: part },
+        })
+            .promise();
+        if (res.$response.error) {
+            return { error: res.$response.error.message };
+        }
+    }
+    return {};
+});
+exports.deleteFiles = deleteFiles;
+function listFiles(bucketName) {
+    var _a, _b;
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            const ret = [];
+            let response = {
+                IsTruncated: true,
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            };
+            while (response.IsTruncated) {
+                response = yield s3
+                    .listObjectsV2({
+                    Bucket: bucketName,
+                    ContinuationToken: response.NextContinuationToken,
+                })
+                    .promise();
+                (_b = (_a = response.Contents) === null || _a === void 0 ? void 0 : _a.filter((r) => r.Key)) === null || _b === void 0 ? void 0 : _b.map((c) => {
+                    ret.push(c.Key);
+                });
+            }
+            return (0, array_1.distinct)(ret.filter((r) => r));
+        }
+        catch (err) {
+            (0, log_1.error)('Error', err);
+            return [];
+        }
+    });
+}
+exports.listFiles = listFiles;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ag-common",
-  "version": "0.0.345",
+  "version": "0.0.347",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "author": "Andrei Gec <@andreigec> (https://gec.dev/)",