@transcend-io/cli 6.22.1 → 6.24.0

package/build/requests/skipPreflightJobs.d.ts

@@ -0,0 +1,20 @@
+import { RequestStatus } from '@transcend-io/privacy-types';
+/**
+ * Given an enricher ID, mark all open request enrichers as skipped
+ *
+ * @param options - Options
+ * @returns Number of items skipped
+ */
+export declare function skipPreflightJobs({ enricherIds, auth, concurrency, transcendUrl, }: {
+    /** Transcend API key authentication */
+    auth: string;
+    /** Enricher IDs to pull down jobs for */
+    enricherIds: string[];
+    /** Upload concurrency */
+    concurrency?: number;
+    /** API URL for Transcend backend */
+    transcendUrl?: string;
+    /** Request statuses to mark as completed */
+    requestStatuses?: RequestStatus[];
+}): Promise<number>;
+//# sourceMappingURL=skipPreflightJobs.d.ts.map

package/build/requests/skipPreflightJobs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"skipPreflightJobs.d.ts","sourceRoot":"","sources":["../../src/requests/skipPreflightJobs.ts"],"names":[],"mappings":"AAWA,OAAO,EAEL,aAAa,EACd,MAAM,6BAA6B,CAAC;AAGrC;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,WAAW,EACX,IAAI,EACJ,WAAiB,EACjB,YAAoC,GACrC,EAAE;IACD,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,yCAAyC;IACzC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,yBAAyB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,eAAe,CAAC,EAAE,aAAa,EAAE,CAAC;CACnC,GAAG,OAAO,CAAC,MAAM,CAAC,CAuFlB"}

package/build/requests/skipPreflightJobs.js

@@ -0,0 +1,73 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.skipPreflightJobs = void 0;
+const bluebird_1 = require("bluebird");
+const colors_1 = __importDefault(require("colors"));
+const logger_1 = require("../logger");
+const graphql_1 = require("../graphql");
+const cli_progress_1 = __importDefault(require("cli-progress"));
+const privacy_types_1 = require("@transcend-io/privacy-types");
+const constants_1 = require("../constants");
+/**
+ * Given an enricher ID, mark all open request enrichers as skipped
+ *
+ * @param options - Options
+ * @returns Number of items skipped
+ */
+async function skipPreflightJobs({ enricherIds, auth, concurrency = 100, transcendUrl = constants_1.DEFAULT_TRANSCEND_API, }) {
+    // Find all requests made before createdAt that are in a removing data state
+    const client = (0, graphql_1.buildTranscendGraphQLClient)(transcendUrl, auth);
+    // Time duration
+    const t0 = new Date().getTime();
+    // fetch all RequestDataSilos that are open
+    const requests = await (0, graphql_1.fetchAllRequests)(client, {
+        statuses: [privacy_types_1.RequestStatus.Enriching],
+    });
+    // Notify Transcend
+    logger_1.logger.info(colors_1.default.magenta(`Processing enricher: "${enricherIds.join(',')}" fetched "${requests.length}" in enriching status.`));
+    // create a new progress bar instance and use shades_classic theme
+    const progressBar = new cli_progress_1.default.SingleBar({}, cli_progress_1.default.Presets.shades_classic);
+    let total = 0;
+    progressBar.start(requests.length, 0);
+    let totalSkipped = 0;
+    await (0, bluebird_1.map)(requests, async (request) => {
+        // FIXME dont pull all in
+        const requestEnrichers = await (0, graphql_1.fetchAllRequestEnrichers)(client, {
+            requestId: request.id,
+        });
+        const requestEnrichersFiltered = requestEnrichers.filter((enricher) => enricherIds.includes(enricher.enricher.id) &&
+            ![
+                privacy_types_1.RequestEnricherStatus.Resolved,
+                privacy_types_1.RequestEnricherStatus.Skipped,
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            ].includes(enricher.status));
+        // FIXME
+        if (requestEnrichersFiltered.length > 0) {
+            await (0, bluebird_1.mapSeries)(requestEnrichersFiltered, async (requestEnricher) => {
+                try {
+                    await (0, graphql_1.makeGraphQLRequest)(client, graphql_1.SKIP_REQUEST_ENRICHER, {
+                        requestEnricherId: requestEnricher.id,
+                    });
+                    totalSkipped += 1;
+                }
+                catch (err) {
+                    if (!err.message.includes('Client error: Cannot skip Request enricher because it has already completed')) {
+                        throw err;
+                    }
+                }
+            });
+        }
+        total += 1;
+        progressBar.update(total);
+    }, { concurrency });
+    progressBar.stop();
+    const t1 = new Date().getTime();
+    const totalTime = t1 - t0;
+    logger_1.logger.info(colors_1.default.green(`Successfully skipped "${totalSkipped}" for  "${requests.length}" requests in "${totalTime / 1000}" seconds!`));
+    return requests.length;
+}
+exports.skipPreflightJobs = skipPreflightJobs;
+//# sourceMappingURL=skipPreflightJobs.js.map

package/build/requests/skipPreflightJobs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skipPreflightJobs.js","sourceRoot":"","sources":["../../src/requests/skipPreflightJobs.ts"],"names":[],"mappings":";;;;;;AAAA,uCAA0C;AAC1C,oDAA4B;AAC5B,sCAAmC;AACnC,wCAMoB;AACpB,gEAAuC;AACvC,+DAGqC;AACrC,4CAAqD;AAErD;;;;;GAKG;AACI,KAAK,UAAU,iBAAiB,CAAC,EACtC,WAAW,EACX,IAAI,EACJ,WAAW,GAAG,GAAG,EACjB,YAAY,GAAG,iCAAqB,GAYrC;IACC,4EAA4E;IAC5E,MAAM,MAAM,GAAG,IAAA,qCAA2B,EAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAE/D,gBAAgB;IAChB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAEhC,2CAA2C;IAC3C,MAAM,QAAQ,GAAG,MAAM,IAAA,0BAAgB,EAAC,MAAM,EAAE;QAC9C,QAAQ,EAAE,CAAC,6BAAa,CAAC,SAAS,CAAC;KACpC,CAAC,CAAC;IAEH,mBAAmB;IACnB,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,OAAO,CACZ,yBAAyB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,cAC5C,QAAQ,CAAC,MACX,wBAAwB,CACzB,CACF,CAAC;IAEF,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAC3C,EAAE,EACF,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IAEF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,IAAA,cAAG,EACP,QAAQ,EACR,KAAK,EAAE,OAAO,EAAE,EAAE;QAChB,yBAAyB;QACzB,MAAM,gBAAgB,GAAG,MAAM,IAAA,kCAAwB,EAAC,MAAM,EAAE;YAC9D,SAAS,EAAE,OAAO,CAAC,EAAE;SACtB,CAAC,CAAC;QACH,MAAM,wBAAwB,GAAG,gBAAgB,CAAC,MAAM,CACtD,CAAC,QAAQ,EAAE,EAAE,CACX,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,CAAC;gBACC,qCAAqB,CAAC,QAAQ;gBAC9B,qCAAqB,CAAC,OAAO;gBAC7B,8DAA8D;aAC/D,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAa,CAAC,CACrC,CAAC;QAEF,QAAQ;QACR,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE;YACvC,MAAM,IAAA,oBAAS,EAAC,wBAAwB,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE;gBAClE,IAAI;oBACF,MAAM,IAAA,4BAAkB,EAGrB,MAAM,EAAE,+BAAqB,EAAE;wBAChC,iBAAiB,EAAE,eAAe,CAAC,EAAE;qBACtC,CAAC,CAAC;oBACH,YAAY,IAAI,CAAC,CAAC;iBACnB;gBAAC,OAAO,GAAG,EAAE;oBACZ,IACE,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CACnB,6EAA6E,CAC9E,EACD;wBACA,MAAM,GAAG,CAAC;qBACX;iBACF;YACH,CAAC,CAAC,CAAC;SACJ;QACD,KAAK,IAAI,CAAC,CAAC;QACX,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC,EACD,EAAE,WAAW,EAAE,CAChB,CAAC;IAEF,WAAW,CAAC,IAAI,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,CAAC;IAE1B,eAAM,CAAC,IAAI,CACT,gBAAM,CAAC,KAAK,CACV,yBAAyB,YAAY,WACnC,QAAQ,CAAC,MACX,kBAAkB,SAAS,GAAG,IAAI,YAAY,CAC/C,CACF,CAAC;IACF,OAAO,QAAQ,CAAC,MAAM,CAAC;AACzB,CAAC;AAvGD,8CAuGC"}