@transcend-io/cli 6.23.1 → 6.25.0

package/build/requests/skipPreflightJobs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skipPreflightJobs.d.ts","sourceRoot":"","sources":["../../src/requests/skipPreflightJobs.ts"],"names":[],"mappings":"AAWA,OAAO,EAEL,aAAa,EACd,MAAM,6BAA6B,CAAC;AAGrC;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,WAAW,EACX,IAAI,EACJ,WAAiB,EACjB,YAAoC,GACrC,EAAE;IACD,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,eAAe,CAAC,EAAE,aAAa,EAAE,CAAC;CACnC,GAAG,OAAO,CAAC,MAAM,CAAC,CAuFlB"}

package/build/requests/skipPreflightJobs.js

@@ -0,0 +1,73 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.skipPreflightJobs = void 0;
+const bluebird_1 = require("bluebird");
+const colors_1 = __importDefault(require("colors"));
+const logger_1 = require("../logger");
+const graphql_1 = require("../graphql");
+const cli_progress_1 = __importDefault(require("cli-progress"));
+const privacy_types_1 = require("@transcend-io/privacy-types");
+const constants_1 = require("../constants");
+/**
+ * Given an enricher ID, mark all open request enrichers as skipped
+ *
+ * @param options - Options
+ * @returns Number of items skipped
+ */
+async function skipPreflightJobs({ enricherIds, auth, concurrency = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, }) {
+    // Find all requests made before createdAt that are in a removing data state
+    const client = (0, graphql_1.buildTranscendGraphQLClient)(transcendUrl, auth);
+    // Time duration
+    const t0 = new Date().getTime();
+    // fetch all RequestDataSilos that are open
+    const requests = await (0, graphql_1.fetchAllRequests)(client, {
+        statuses: [privacy_types_1.RequestStatus.Enriching],
+    });
+    // Notify Transcend
+    logger_1.logger.info(colors_1.default.magenta(`Processing enricher: "${enricherIds.join(',')}" fetched "${requests.length}" in enriching status.`));
+    // create a new progress bar instance and use shades_classic theme
+    const progressBar = new cli_progress_1.default.SingleBar({}, cli_progress_1.default.Presets.shades_classic);
+    let total = 0;
+    progressBar.start(requests.length, 0);
+    let totalSkipped = 0;
+    await (0, bluebird_1.map)(requests, async (request) => {
+        // FIXME dont pull all in
+        const requestEnrichers = await (0, graphql_1.fetchAllRequestEnrichers)(client, {
+            requestId: request.id,
+        });
+        const requestEnrichersFiltered = requestEnrichers.filter((enricher) => enricherIds.includes(enricher.enricher.id) &&
+            ![
+                privacy_types_1.RequestEnricherStatus.Resolved,
+                privacy_types_1.RequestEnricherStatus.Skipped,
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ].includes(enricher.status));
+        // FIXME
+        if (requestEnrichersFiltered.length > 0) {
+            await (0, bluebird_1.mapSeries)(requestEnrichersFiltered, async (requestEnricher) => {
+                try {
+                    await (0, graphql_1.makeGraphQLRequest)(client, graphql_1.SKIP_REQUEST_ENRICHER, {
+                        requestEnricherId: requestEnricher.id,
+                    });
+                    totalSkipped += 1;
+                }
+                catch (err) {
+                    if (!err.message.includes('Client error: Cannot skip Request enricher because it has already completed')) {
+                        throw err;
+                    }
+                }
+            });
+        }
+        total += 1;
+        progressBar.update(total);
+    }, { concurrency });
+    progressBar.stop();
+    const t1 = new Date().getTime();
+    const totalTime = t1 - t0;
+    logger_1.logger.info(colors_1.default.green(`Successfully skipped "${totalSkipped}" for  "${requests.length}" requests in "${totalTime / 1000}" seconds!`));
+    return requests.length;
+}
+exports.skipPreflightJobs = skipPreflightJobs;
+//# sourceMappingURL=skipPreflightJobs.js.map

package/build/requests/skipPreflightJobs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skipPreflightJobs.js","sourceRoot":"","sources":["../../src/requests/skipPreflightJobs.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA0C;AAC1C,oDAA4B;AAC5B,sCAAmC;AACnC,wCAMoB;AACpB,gEAAuC;AACvC,+DAGqC;AACrC,4CAAqD;AAErD;;;;;GAKG;AACI,KAAK,UAAU,iBAAiB,CAAC,EACtC,WAAW,EACX,IAAI,EACJ,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,iCAAqB,GAYrC;IACC,4EAA4E;IAC5E,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAE/D,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAEhC,2CAA2C;IAC3C,MAAM,QAAQ,GAAG,MAAM,IAAA,0BAAgB,EAAC,MAAM,EAAE;QAC9C,QAAQ,EAAE,CAAC,6BAAa,CAAC,SAAS,CAAC;KACpC,CAAC,CAAC;IAEH,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,yBAAyB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,cAC5C,QAAQ,CAAC,MACX,wBAAwB,CACzB,CACF,CAAC;IAEF,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,IAAA,cAAG,EACP,QAAQ,EACR,KAAK,EAAE,OAAO,EAAE,EAAE;QAChB,yBAAyB;QACzB,MAAM,gBAAgB,GAAG,MAAM,IAAA,kCAAwB,EAAC,MAAM,EAAE;YAC9D,SAAS,EAAE,OAAO,CAAC,EAAE;SACtB,CAAC,CAAC;QACH,MAAM,wBAAwB,GAAG,gBAAgB,CAAC,MAAM,CACtD,CAAC,QAAQ,EAAE,EAAE,CACX,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,CAAC;gBACC,qCAAqB,CAAC,QAAQ;gBAC9B,qCAAqB,CAAC,OAAO;gBAC7B,8DAA8D;aAC/D,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAa,CAAC,CACrC,CAAC;QAEF,QAAQ;QACR,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE;YACvC,MAAM,IAAA,oBAAS,EAAC,wBAAwB,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE;gBAClE,IAAI;oBACF,MAAM,IAAA,4BAAkB,EAGrB,MAAM,EAAE,+BAAqB,EAAE;wBAChC,iBAAiB,EAAE,eAAe,CAAC,EAAE;qBACtC,CAAC,CAAC;oBACH,YAAY,IAAI,CAAC,CAAC;iBACnB;gBAAC,OAAO,GAAG,EAAE;oBACZ,IACE,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CACnB,6EAA6E,CAC9E,EACD;wBACA,MAAM,GAAG,CAAC;qBACX;iBACF;YACH,CAAC,CAAC,CAAC;SACJ;QACD,KAAK,IAAI,CAAC,CAAC;QACX,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAC;IAEF,WAAW,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,yBAAyB,YAAY,WACnC,QAAQ,CAAC,MACX,kBAAkB,SAAS,GAAG,IAAI,YAAY,CAC/C,CACF,CAAC;IACF,OAAO,QAAQ,CAAC,MAAM,CAAC;AACzB,CAAC;AAvGD,8CAuGC"}

package/build/requests/streamPrivacyRequestFiles.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+import { RequestFileMetadata } from './getFileMetadataForPrivacyRequests';
+import { Got } from 'got/dist/source';
+/**
+ * This function will take in a set of file metadata for privacy requests
+ * call the Transcend API to stream the file metadata for these requests
+ * and pass that through a callback function
+ *
+ * @param fileMetadata - Metadata to download
+ * @param options - Options for the request
+ */
+export declare function streamPrivacyRequestFiles(fileMetadata: RequestFileMetadata[], { requestId, sombra, onFileDownloaded, concurrency, }: {
+    /** Request ID for logging */
+    requestId: string;
+    /** Sombra got instance */
+    sombra: Got;
+    /** Handler on each file */
+    onFileDownloaded: (metadata: RequestFileMetadata, stream: Buffer) => void;
+    /** Concurrent downloads at once */
+    concurrency?: number;
+}): Promise<void>;
+//# sourceMappingURL=streamPrivacyRequestFiles.d.ts.map

package/build/requests/streamPrivacyRequestFiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamPrivacyRequestFiles.d.ts","sourceRoot":"","sources":["../../src/requests/streamPrivacyRequestFiles.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAGtC;;;;;;;GAOG;AACH,wBAAsB,yBAAyB,CAC7C,YAAY,EAAE,mBAAmB,EAAE,EACnC,EACE,SAAS,EACT,MAAM,EACN,gBAAgB,EAChB,WAAgB,GACjB,EAAE;IACD,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,MAAM,EAAE,GAAG,CAAC;IACZ,2BAA2B;IAC3B,gBAAgB,EAAE,CAAC,QAAQ,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1E,mCAAmC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAsCf"}

package/build/requests/streamPrivacyRequestFiles.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.streamPrivacyRequestFiles = void 0;
+const bluebird_1 = require("bluebird");
+const colors_1 = __importDefault(require("colors"));
+const logger_1 = require("../logger");
+/**
+ * This function will take in a set of file metadata for privacy requests
+ * call the Transcend API to stream the file metadata for these requests
+ * and pass that through a callback function
+ *
+ * @param fileMetadata - Metadata to download
+ * @param options - Options for the request
+ */
+async function streamPrivacyRequestFiles(fileMetadata, { requestId, sombra, onFileDownloaded, concurrency = 20, }) {
+    // Loop over each file
+    await (0, bluebird_1.map)(fileMetadata, async (metadata) => {
+        var _a, _b, _c;
+        try {
+            // Construct the stream
+            await sombra
+                .get('v1/files', {
+                searchParams: {
+                    downloadKey: metadata.downloadKey,
+                },
+            })
+                .buffer()
+                .then((fileResponse) => onFileDownloaded(metadata, fileResponse));
+        }
+        catch (err) {
+            if ((_b = (_a = err === null || err === void 0 ? void 0 : err.response) === null || _a === void 0 ? void 0 : _a.body) === null || _b === void 0 ? void 0 : _b.includes('fileMetadata#verify')) {
+                logger_1.logger.error(colors_1.default.red(`Failed to pull file for: ${metadata.fileName} (request:${requestId}) - JWT expired. ` +
+                    'This likely means that the file is no longer available. ' +
+                    'Try restarting the request from scratch in Transcend Admin Dashboard. ' +
+                    'Skipping the download of this file.'));
+                return;
+            }
+            throw new Error(`Received an error from server: ${((_c = err === null || err === void 0 ? void 0 : err.response) === null || _c === void 0 ? void 0 : _c.body) || (err === null || err === void 0 ? void 0 : err.message)}`);
+        }
+    }, {
+        concurrency,
+    });
+}
+exports.streamPrivacyRequestFiles = streamPrivacyRequestFiles;
+//# sourceMappingURL=streamPrivacyRequestFiles.js.map

package/build/requests/streamPrivacyRequestFiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamPrivacyRequestFiles.js","sourceRoot":"","sources":["../../src/requests/streamPrivacyRequestFiles.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA+B;AAC/B,oDAA4B;AAG5B,sCAAmC;AAEnC;;;;;;;GAOG;AACI,KAAK,UAAU,yBAAyB,CAC7C,YAAmC,EACnC,EACE,SAAS,EACT,MAAM,EACN,gBAAgB,EAChB,WAAW,GAAG,EAAE,GAUjB;IAED,sBAAsB;IACtB,MAAM,IAAA,cAAG,EACP,YAAY,EACZ,KAAK,EAAE,QAAQ,EAAE,EAAE;;QACjB,IAAI;YACF,uBAAuB;YACvB,MAAM,MAAM;iBACT,GAAG,CAAC,UAAU,EAAE;gBACf,YAAY,EAAE;oBACZ,WAAW,EAAE,QAAQ,CAAC,WAAW;iBAClC;aACF,CAAC;iBACD,MAAM,EAAE;iBACR,IAAI,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;SACrE;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,MAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,QAAQ,0CAAE,IAAI,0CAAE,QAAQ,CAAC,qBAAqB,CAAC,EAAE;gBACxD,eAAM,CAAC,KAAK,CACV,gBAAM,CAAC,GAAG,CACR,4BAA4B,QAAQ,CAAC,QAAQ,aAAa,SAAS,mBAAmB;oBACpF,0DAA0D;oBAC1D,wEAAwE;oBACxE,qCAAqC,CACxC,CACF,CAAC;gBACF,OAAO;aACR;YACD,MAAM,IAAI,KAAK,CACb,kCACE,CAAA,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,QAAQ,0CAAE,IAAI,MAAI,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CACrC,EAAE,CACH,CAAC;SACH;IACH,CAAC,EACD;QACE,WAAW;KACZ,CACF,CAAC;AACJ,CAAC;AAvDD,8DAuDC"}