@8ms/helpers 1.40.6 → 1.40.9

package/initClients.js

@@ -0,0 +1,131 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const initClient_1 = __importDefault(require("./aws/athenaExpress/initClient"));
+const initClient_2 = require("./aws/ec2/initClient");
+const initClient_3 = require("./aws/ecs/initClient");
+const initClient_4 = __importDefault(require("./aws/glue/initClient"));
+const initClient_5 = __importDefault(require("./aws/lambda/initClient"));
+const initClient_6 = __importDefault(require("./aws/s3/initClient"));
+const initClient_7 = __importDefault(require("./aws/ses/initClient"));
+const getParameter_1 = __importDefault(require("./aws/ssm/getParameter"));
+const initClient_8 = __importDefault(require("./aws/ssm/initClient"));
+const initClient_9 = __importDefault(require("./lumar/api/initClient"));
+const initClient_10 = __importDefault(require("./lumar/graphql/initClient"));
+const initClient_11 = __importDefault(require("./google/bigQuery/initClient"));
+const initClient_12 = __importDefault(require("./google/sheets/initClient"));
+const initClient_13 = __importDefault(require("./google/storage/initClient"));
+const initClient_14 = __importDefault(require("./googleAds/initClient"));
+const initClient_15 = __importDefault(require("./prisma/initClient"));
+/**
+ * Function to simplify initialising the clients.
+ */
+const initClients = async (props) => {
+    if (props?.athenaExpress) {
+        (0, initClient_1.default)({
+            config: props.awsConfig,
+            database: props.athenaExpress.database,
+            s3Bucket: props.athenaExpress.s3Bucket,
+            s3Key: props.athenaExpress.s3Key,
+        });
+    }
+    if (props?.awsEc2) {
+        (0, initClient_2.initClient)({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsEcs) {
+        (0, initClient_3.initClient)({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsGlue) {
+        (0, initClient_4.default)({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsLambda) {
+        (0, initClient_5.default)({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsS3) {
+        (0, initClient_6.default)({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsSes) {
+        (0, initClient_7.default)({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsSsm || props?.lumarApi || props?.lumarGraphql || props?.googleAds || props?.googleConfig) {
+        (0, initClient_8.default)({
+            config: props.awsConfig,
+        });
+    }
+    // Lumar (API) authentication using SSM Parameter
+    if (props?.lumarApi) {
+        const lumarApiParameter = await (0, getParameter_1.default)({
+            name: props.lumarApi.parameterName,
+        });
+        await (0, initClient_9.default)({
+            auth: lumarApiParameter
+        });
+    }
+    // Lumar (GraphQL) authentication using SSM Parameter
+    if (props?.lumarGraphql) {
+        const lumarGraphqlParameter = await (0, getParameter_1.default)({
+            name: props.lumarGraphql.parameterName,
+        });
+        await (0, initClient_10.default)({
+            auth: lumarGraphqlParameter
+        });
+    }
+    // Google authentication using SSM
+    if (props?.googleConfig) {
+        // Google Parameter
+        const googleParameter = await (0, getParameter_1.default)({
+            name: props.googleConfig.parameterName,
+        });
+        // BigQuery
+        if (props?.googleBigQuery && props?.googleConfig?.projectId) {
+            (0, initClient_11.default)({
+                parameter: googleParameter,
+                projectId: props.googleConfig.projectId,
+            });
+        }
+        // Google Sheets
+        if (props?.googleSheets) {
+            (0, initClient_12.default)({
+                parameter: googleParameter,
+            });
+        }
+        // Google Storage
+        if (props?.googleStorage && props?.googleConfig?.projectId) {
+            (0, initClient_13.default)({
+                parameter: googleParameter,
+                projectId: props.googleConfig.projectId,
+            });
+        }
+    }
+    // Google Ads authentication using SSM Parameter
+    if (props?.googleAds) {
+        const googleAdsParameter = await (0, getParameter_1.default)({
+            name: props.googleAds.parameterName,
+        });
+        await (0, initClient_14.default)({
+            parameter: googleAdsParameter,
+        });
+    }
+    // Initialise Prisma
+    if (props?.prisma) {
+        (0, initClient_15.default)({
+            debug: props?.prisma?.debug ?? false,
+            planetScale: props?.prisma?.planetScale ?? false,
+        });
+    }
+};
+exports.default = initClients;

package/json/getJsonNewline.d.ts

@@ -0,0 +1,5 @@
+type GetGlueJson = {
+    data: any;
+};
+declare const GetJsonNewline: ({ data }: GetGlueJson) => string;
+export default GetJsonNewline;

package/json/getJsonNewline.js

@@ -0,0 +1,17 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const getArray_1 = __importDefault(require("../array/getArray"));
+const GetJsonNewline = ({ data }) => {
+    // Must be an array
+    const inputArray = (0, getArray_1.default)({
+        input: data
+    });
+    // For each array row convert the row
+    const stringRows = inputArray.map(row => JSON.stringify(row));
+    // Combine each row with a line separator
+    return stringRows.join("\n");
+};
+exports.default = GetJsonNewline;

package/json/isJson.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Is the provided input JSON?
+ * https://stackoverflow.com/a/33369954
+ */
+declare const isJson: ({ input }: {
+    input: any;
+}) => boolean;
+export default isJson;

package/json/isJson.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * Is the provided input JSON?
+ * https://stackoverflow.com/a/33369954
+ */
+const isJson = ({ input }) => {
+    input = 'string' === typeof input ? input : JSON.stringify(input);
+    try {
+        input = JSON.parse(input);
+    }
+    catch (e) {
+        return false;
+    }
+    if ('object' === typeof input && null !== input) {
+        return true;
+    }
+    return false;
+};
+exports.default = isJson;

package/littleWarden/getUrlStatus.d.ts

@@ -0,0 +1,9 @@
+type GetUrlStatus = {
+    apiKey: string;
+    urlId: string;
+};
+/**
+ * Get the URL Status for a given URL ID from Little Warden.
+ */
+declare const getUrlStatus: ({ apiKey, urlId }: GetUrlStatus) => Promise<any>;
+export default getUrlStatus;

package/littleWarden/getUrlStatus.js

@@ -0,0 +1,21 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const get_1 = __importDefault(require("../axios/get"));
+/**
+ * Get the URL Status for a given URL ID from Little Warden.
+ */
+const getUrlStatus = async ({ apiKey, urlId }) => {
+    const response = await (0, get_1.default)({
+        config: {
+            headers: {
+                'Authorization': `Token token="${apiKey}"`,
+            },
+        },
+        url: `https://littlewarden.com/api/sites/${urlId}`,
+    });
+    return response.getBodyDefaultTo({ defaultValue: {}, keys: ['data', 'site'] });
+};
+exports.default = getUrlStatus;

package/littleWarden/index.d.ts

@@ -0,0 +1,5 @@
+export declare enum State {
+    ALL_GOOD = "all_good",
+    DANGER = "danger",
+    WARNING = "warning"
+}

package/littleWarden/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.State = void 0;
+var State;
+(function (State) {
+    State["ALL_GOOD"] = "all_good";
+    State["DANGER"] = "danger";
+    State["WARNING"] = "warning";
+})(State || (exports.State = State = {}));

package/littleWarden/isAllGood.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Check if the field is All Good.
+ */
+declare const isAllGood: ({ field }: {
+    field: any;
+}) => boolean;
+export default isAllGood;

package/littleWarden/isAllGood.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const _1 = require("./");
+/**
+ * Check if the field is All Good.
+ */
+const isAllGood = ({ field }) => {
+    let response = false;
+    // Check that the status field exists
+    if (undefined !== field && undefined !== field["status"]) {
+        response = _1.State.ALL_GOOD === field["status"];
+    }
+    return response;
+};
+exports.default = isAllGood;

package/littleWarden/isDanger.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Check if the field is Danger.
+ */
+declare const isDanger: ({ field }: {
+    field: any;
+}) => boolean;
+export default isDanger;

package/littleWarden/isDanger.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const _1 = require("./");
+/**
+ * Check if the field is Danger.
+ */
+const isDanger = ({ field }) => {
+    let response = false;
+    // Check that the status field exists
+    if (undefined !== field && undefined !== field['status']) {
+        response = _1.State.DANGER === field['status'];
+    }
+    return response;
+};
+exports.default = isDanger;

package/littleWarden/isWarning.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Check if the field is Warning.
+ */
+declare const isWarning: ({ field }: {
+    field: any;
+}) => boolean;
+export default isWarning;

package/littleWarden/isWarning.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const _1 = require("./");
+/**
+ * Check if the field is Warning.
+ */
+const isWarning = ({ field }) => {
+    let response = false;
+    // Check that the status field exists
+    if (undefined !== field && undefined !== field['status']) {
+        response = _1.State.WARNING === field['status'];
+    }
+    return response;
+};
+exports.default = isWarning;

package/lumar/api/buildRequest.d.ts

@@ -0,0 +1,15 @@
+import { Filter } from './filter';
+import { Order } from './orders';
+export type Request = {
+    crawlId?: number;
+    customPath?: string;
+    filters?: Filter[];
+    limit?: number;
+    orders?: Order[];
+    page?: number;
+    projectId?: number;
+    report?: string;
+    subReport?: string;
+};
+declare const buildRequest: ({ crawlId, customPath, filters, limit, orders, page, projectId, report, subReport, }: Request) => string;
+export default buildRequest;

package/lumar/api/buildRequest.js

@@ -0,0 +1,83 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const reports_1 = __importDefault(require("./reports"));
+const orders_1 = __importDefault(require("./orders"));
+const buildRequest = ({ crawlId, customPath, filters, limit, orders, page, projectId, report, subReport, }) => {
+    let url = `https://api.deepcrawl.com/accounts/37204`;
+    // Limit must be greater than 0 but also less than 200
+    if (undefined !== limit) {
+        if (limit > 200) {
+            throw `Maximum row limit is 200.`;
+        }
+        else if (0 === limit) {
+            throw `Row limit must be greater than 0.`;
+        }
+    }
+    // Set limit to 200 by default
+    else {
+        limit = 200;
+    }
+    // If a report other than List Crawls is selected, Crawl ID must be defined
+    if (undefined !== report && reports_1.default.LIST_CRAWLS !== report && undefined == crawlId) {
+        throw `Crawl ID must be defined.`;
+    }
+    switch (report) {
+        case reports_1.default.LIST_CRAWLS:
+            url += `/projects/${projectId}/crawls`;
+            break;
+        case reports_1.default.LIST_PROJECTS:
+            url += `/projects`;
+            break;
+        case reports_1.default.CUSTOM:
+            url += customPath;
+            url = url.replace('[PROJECT]', projectId.toString());
+            url = url.replace('[CRAWL]', crawlId.toString());
+            break;
+        case reports_1.default.CRAWL_STATISTICS:
+            url += `/projects/${projectId}/crawls/${crawlId}/statistics`;
+            break;
+        case reports_1.default.LIST_REPORTS:
+            url += `/projects/${projectId}/crawls/${crawlId}/reports`;
+            break;
+        case reports_1.default.REPORT_ROWS:
+        case reports_1.default.REPORT_STATISTICS:
+        case reports_1.default.REPORT_SUMMARY:
+            url += `/projects/${projectId}/crawls/${crawlId}/reports/${subReport}`;
+            if (reports_1.default.REPORT_ROWS === report) {
+                url += `/report_rows`;
+            }
+            else if (reports_1.default.REPORT_STATISTICS === report) {
+                url += `/statistics`;
+            }
+            break;
+    }
+    // Specify the limit
+    url += -1 === url.indexOf('?') ? '?' : '&';
+    url += `per_page=${limit}`;
+    // Set the page number
+    if (undefined !== page) {
+        url += -1 === url.indexOf('?') ? '?' : '&';
+        url += `page=${page}`;
+    }
+    // Apply any filters
+    if (undefined !== filters) {
+        filters.map(filter => {
+            url += -1 === url.indexOf('?') ? '?' : '&';
+            url += `q[${filter.field}_${filter.operand}]=${filter.value}`;
+        });
+    }
+    // Order the data
+    if (undefined !== orders) {
+        url += -1 === url.indexOf('?') ? '?' : '&';
+        orders.map((order, index) => {
+            url += 0 === index ? 'sort=' : ',';
+            url += orders_1.default.DESCENDING === order.direction ? '-' : '';
+            url += order.field;
+        });
+    }
+    return url;
+};
+exports.default = buildRequest;

package/lumar/api/filter.d.ts

@@ -0,0 +1,5 @@
+export type Filter = {
+    field: string;
+    operand: string;
+    value: string;
+};

package/lumar/api/filter.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lumar/api/getData.d.ts

@@ -0,0 +1,6 @@
+import ApiResponse from '../../api/ApiResponse';
+type GetData = {
+    url: string;
+};
+declare const getData: ({ url }: GetData) => Promise<ApiResponse>;
+export default getData;

package/lumar/api/getData.js

@@ -0,0 +1,18 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const get_1 = __importDefault(require("../../axios/get"));
+const getData = async ({ url }) => {
+    const apiResponse = await (0, get_1.default)({
+        config: {
+            headers: {
+                'X-AUTH-TOKEN': global.deepcrawlApi.token,
+            },
+        },
+        url,
+    });
+    return apiResponse;
+};
+exports.default = getData;

package/lumar/api/initClient.d.ts

@@ -0,0 +1,13 @@
+type InitClient = {
+    auth: {
+        apiId: string;
+        apiKey: string;
+    };
+};
+/**
+ * http://api-docs.deepcrawl.com/
+ * Renew the Lumar user token.
+ * Token lasts for 6 hours.
+ */
+declare const initClient: ({ auth }: InitClient) => Promise<void>;
+export default initClient;

package/lumar/api/initClient.js

@@ -0,0 +1,38 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const post_1 = __importDefault(require("../../axios/post"));
+global.lumarApi = {
+    token: null,
+    updatedAt: null,
+};
+/**
+ * http://api-docs.deepcrawl.com/
+ * Renew the Lumar user token.
+ * Token lasts for 6 hours.
+ */
+const initClient = async ({ auth }) => {
+    // todo: Or was more than 6 hours ago
+    if (null === global.lumarApi.token) {
+        const apiResponse = await (0, post_1.default)({
+            config: {
+                auth: {
+                    username: auth.apiId,
+                    password: auth.apiKey,
+                },
+            },
+            url: `https://api.deepcrawl.com/sessions`,
+        });
+        // Ensure the value exists
+        const token = apiResponse.getBodyDefaultTo({
+            defaultValue: null,
+            keys: ['token'],
+        });
+        if (token) {
+            global.lumarApi.token = token;
+        }
+    }
+};
+exports.default = initClient;

package/lumar/api/operands.d.ts

@@ -0,0 +1,11 @@
+declare const operands: {
+    EQUALS: string;
+    CONTAINS: string;
+    LESS_THAN: string;
+    LESS_THAN_OR_EQUAL: string;
+    GREATER_THAN: string;
+    GREATER_THAN_OR_EQUAL: string;
+    REGEX: string;
+    NOT_REGEX: string;
+};
+export default operands;

package/lumar/api/operands.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const operands = {
+    EQUALS: 'eql',
+    CONTAINS: 'cont',
+    LESS_THAN: 'lt',
+    LESS_THAN_OR_EQUAL: 'lte',
+    GREATER_THAN: 'gt',
+    GREATER_THAN_OR_EQUAL: 'gte',
+    REGEX: 'rgx',
+    NOT_REGEX: 'nrgx',
+};
+exports.default = operands;

package/lumar/api/orders.d.ts

@@ -0,0 +1,9 @@
+export type Order = {
+    direction: string;
+    field: string;
+};
+declare const orders: {
+    ASCENDING: string;
+    DESCENDING: string;
+};
+export default orders;

package/lumar/api/orders.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const orders = {
+    ASCENDING: 'asc',
+    DESCENDING: 'desc',
+};
+exports.default = orders;

package/lumar/api/reports.d.ts

@@ -0,0 +1,11 @@
+declare const reports: {
+    CRAWL_STATISTICS: string;
+    CUSTOM: string;
+    LIST_PROJECTS: string;
+    LIST_CRAWLS: string;
+    LIST_REPORTS: string;
+    REPORT_ROWS: string;
+    REPORT_STATISTICS: string;
+    REPORT_SUMMARY: string;
+};
+export default reports;

package/lumar/api/reports.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const reports = {
+    CRAWL_STATISTICS: 'crawl_statistics',
+    CUSTOM: 'custom',
+    LIST_PROJECTS: 'list_projects',
+    LIST_CRAWLS: 'list_crawls',
+    LIST_REPORTS: 'list_reports',
+    REPORT_ROWS: 'report_rows',
+    REPORT_STATISTICS: 'report_statistics',
+    REPORT_SUMMARY: 'report_summary',
+};
+exports.default = reports;

package/lumar/graphql/columns.d.ts

@@ -0,0 +1,29 @@
+declare const columns: {
+    CUSTOM_EXTRACTION_1: string;
+    CUSTOM_EXTRACTION_2: string;
+    DEEP_RANK: string;
+    DESCRIPTION: string;
+    DUPLICATE_PAGE: string;
+    DUPLICATE_DESCRIPTION_FROM: string;
+    DUPLICATE_TITLE_FROM: string;
+    FOUND_AT_URL: string;
+    FOUND_IN_CRAWL: string;
+    FOUND_IN_SEARCH_CONSOLE: string;
+    FOUND_IN_SITEMAP: string;
+    LEVEL: string;
+    PAGE_TITLE: string;
+    PRIMARY_PAGE: string;
+    PRIMARY_URL_FROM: string;
+    PRIMARY_URL_FROM_STATUS_CODE: string;
+    REDIRECTED_TO_URL: string;
+    REDIRECTED_TO_STATUS_CODE: string;
+    REDIRECTION_CHAIN: string;
+    SOURCE_URL: string;
+    STATUS_CODE: string;
+    TARGET_STATUS_CODE: string;
+    URL: string;
+    URL_COUNT: string;
+    URL_TO: string;
+    URL_TO_STATUS_CODE: string;
+};
+export default columns;

package/lumar/graphql/columns.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const columns = {
+    CUSTOM_EXTRACTION_1: 'customExtraction1',
+    CUSTOM_EXTRACTION_2: 'customExtraction2',
+    DEEP_RANK: 'deepRank',
+    DESCRIPTION: 'description',
+    DUPLICATE_PAGE: 'duplicatePage',
+    DUPLICATE_DESCRIPTION_FROM: 'duplicateDescriptionPrimaryUrl',
+    DUPLICATE_TITLE_FROM: 'duplicateTitlePrimaryUrl',
+    FOUND_AT_URL: 'foundAtUrl',
+    FOUND_IN_CRAWL: 'foundInWebCrawl',
+    FOUND_IN_SEARCH_CONSOLE: 'foundInGoogleSearchConsole',
+    FOUND_IN_SITEMAP: 'foundInSitemap',
+    LEVEL: 'level',
+    PAGE_TITLE: 'pageTitle',
+    PRIMARY_PAGE: 'primaryPage',
+    PRIMARY_URL_FROM: 'primaryUrlFrom',
+    PRIMARY_URL_FROM_STATUS_CODE: 'primaryUrlFromStatusCode',
+    REDIRECTED_TO_URL: 'redirectedToUrl',
+    REDIRECTED_TO_STATUS_CODE: 'redirectedToStatusCode',
+    REDIRECTION_CHAIN: 'redirectionChain',
+    SOURCE_URL: 'sourceUrl',
+    STATUS_CODE: 'httpStatusCode',
+    TARGET_STATUS_CODE: 'targetStatusCode',
+    URL: 'url',
+    URL_COUNT: 'urlCount',
+    URL_TO: 'urlTo',
+    URL_TO_STATUS_CODE: 'urlToStatusCode',
+};
+exports.default = columns;

package/lumar/graphql/getData.d.ts

@@ -0,0 +1,9 @@
+import ApiResponse from '../../api/ApiResponse';
+type GetData = {
+    query: string;
+    variables?: {
+        [key: string]: any;
+    };
+};
+declare const getData: ({ query, variables }: GetData) => Promise<ApiResponse>;
+export default getData;