@8ms/helpers 1.41.1 → 1.42.0

package/dist/esm/initClients.d.ts

@@ -0,0 +1,43 @@
+import { Config as AwsConfig } from "./aws/getConfig";
+type InitClients = {
+    athenaExpress?: {
+        database: string;
+        s3Bucket: string;
+        s3Key: string;
+    };
+    awsConfig?: AwsConfig;
+    awsEc2?: boolean;
+    awsEcs?: boolean;
+    awsGlue?: boolean;
+    awsLambda?: boolean;
+    awsS3?: boolean;
+    awsSes?: boolean;
+    awsSsm?: boolean;
+    lumarApi?: {
+        parameterName: string;
+    };
+    lumarGraphql?: {
+        parameterName: string;
+    };
+    eskimi?: boolean;
+    googleAds?: {
+        parameterName: string;
+    };
+    googleConfig?: {
+        parameterName: string;
+        projectId: string;
+    };
+    googleBigQuery?: boolean;
+    googleSheets?: boolean;
+    googleStorage?: boolean;
+    prisma?: {
+        debug?: boolean;
+        planetScale?: boolean;
+    };
+    myTarget?: boolean;
+};
+/**
+ * Function to simplify initialising the clients.
+ */
+declare const initClients: (props: InitClients) => Promise<void>;
+export default initClients;

package/dist/esm/initClients.js

@@ -0,0 +1,126 @@
+import initAthenaExpress from "./aws/athenaExpress/initClient";
+import { initClient as initAwsEc2 } from "./aws/ec2/initClient";
+import { initClient as initAwsEcs } from "./aws/ecs/initClient";
+import initAwsGlue from "./aws/glue/initClient";
+import initAwsLambda from "./aws/lambda/initClient";
+import initAwsS3 from "./aws/s3/initClient";
+import initAwsSes from "./aws/ses/initClient";
+import getParameter from "./aws/ssm/getParameter";
+import initAwsSsm from "./aws/ssm/initClient";
+import initLumarApi from "./lumar/api/initClient";
+import initLumarGraphql from "./lumar/graphql/initClient";
+import initGoogleBigQuery from "./google/bigQuery/initClient";
+import initGoogleSheets from "./google/sheets/initClient";
+import initGoogleStorage from "./google/storage/initClient";
+import initGoogleAds from "./googleAds/initClient";
+import initPrisma from "./prisma/initClient";
+/**
+ * Function to simplify initialising the clients.
+ */
+const initClients = async (props) => {
+    if (props?.athenaExpress) {
+        initAthenaExpress({
+            config: props.awsConfig,
+            database: props.athenaExpress.database,
+            s3Bucket: props.athenaExpress.s3Bucket,
+            s3Key: props.athenaExpress.s3Key,
+        });
+    }
+    if (props?.awsEc2) {
+        initAwsEc2({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsEcs) {
+        initAwsEcs({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsGlue) {
+        initAwsGlue({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsLambda) {
+        initAwsLambda({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsS3) {
+        initAwsS3({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsSes) {
+        initAwsSes({
+            config: props.awsConfig,
+        });
+    }
+    if (props?.awsSsm || props?.lumarApi || props?.lumarGraphql || props?.googleAds || props?.googleConfig) {
+        initAwsSsm({
+            config: props.awsConfig,
+        });
+    }
+    // Lumar (API) authentication using SSM Parameter
+    if (props?.lumarApi) {
+        const lumarApiParameter = await getParameter({
+            name: props.lumarApi.parameterName,
+        });
+        await initLumarApi({
+            auth: lumarApiParameter
+        });
+    }
+    // Lumar (GraphQL) authentication using SSM Parameter
+    if (props?.lumarGraphql) {
+        const lumarGraphqlParameter = await getParameter({
+            name: props.lumarGraphql.parameterName,
+        });
+        await initLumarGraphql({
+            auth: lumarGraphqlParameter
+        });
+    }
+    // Google authentication using SSM
+    if (props?.googleConfig) {
+        // Google Parameter
+        const googleParameter = await getParameter({
+            name: props.googleConfig.parameterName,
+        });
+        // BigQuery
+        if (props?.googleBigQuery && props?.googleConfig?.projectId) {
+            initGoogleBigQuery({
+                parameter: googleParameter,
+                projectId: props.googleConfig.projectId,
+            });
+        }
+        // Google Sheets
+        if (props?.googleSheets) {
+            initGoogleSheets({
+                parameter: googleParameter,
+            });
+        }
+        // Google Storage
+        if (props?.googleStorage && props?.googleConfig?.projectId) {
+            initGoogleStorage({
+                parameter: googleParameter,
+                projectId: props.googleConfig.projectId,
+            });
+        }
+    }
+    // Google Ads authentication using SSM Parameter
+    if (props?.googleAds) {
+        const googleAdsParameter = await getParameter({
+            name: props.googleAds.parameterName,
+        });
+        await initGoogleAds({
+            parameter: googleAdsParameter,
+        });
+    }
+    // Initialise Prisma
+    if (props?.prisma) {
+        initPrisma({
+            debug: props?.prisma?.debug ?? false,
+            planetScale: props?.prisma?.planetScale ?? false,
+        });
+    }
+};
+export default initClients;

package/dist/esm/json/getJsonNewline.d.ts

@@ -0,0 +1,5 @@
+type GetGlueJson = {
+    data: any;
+};
+declare const GetJsonNewline: ({ data }: GetGlueJson) => string;
+export default GetJsonNewline;

package/dist/esm/json/getJsonNewline.js

@@ -0,0 +1,12 @@
+import getArray from '../array/getArray';
+const GetJsonNewline = ({ data }) => {
+    // Must be an array
+    const inputArray = getArray({
+        input: data
+    });
+    // For each array row convert the row
+    const stringRows = inputArray.map(row => JSON.stringify(row));
+    // Combine each row with a line separator
+    return stringRows.join("\n");
+};
+export default GetJsonNewline;

package/dist/esm/json/isJson.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Is the provided input JSON?
+ * https://stackoverflow.com/a/33369954
+ */
+declare const isJson: ({ input }: {
+    input: any;
+}) => boolean;
+export default isJson;

package/dist/esm/json/isJson.js

@@ -0,0 +1,18 @@
+/**
+ * Is the provided input JSON?
+ * https://stackoverflow.com/a/33369954
+ */
+const isJson = ({ input }) => {
+    input = 'string' === typeof input ? input : JSON.stringify(input);
+    try {
+        input = JSON.parse(input);
+    }
+    catch (e) {
+        return false;
+    }
+    if ('object' === typeof input && null !== input) {
+        return true;
+    }
+    return false;
+};
+export default isJson;

package/dist/esm/littleWarden/getUrlStatus.d.ts

@@ -0,0 +1,9 @@
+type GetUrlStatus = {
+    apiKey: string;
+    urlId: string;
+};
+/**
+ * Get the URL Status for a given URL ID from Little Warden.
+ */
+declare const getUrlStatus: ({ apiKey, urlId }: GetUrlStatus) => Promise<any>;
+export default getUrlStatus;

package/dist/esm/littleWarden/getUrlStatus.js

@@ -0,0 +1,16 @@
+import get from '../axios/get';
+/**
+ * Get the URL Status for a given URL ID from Little Warden.
+ */
+const getUrlStatus = async ({ apiKey, urlId }) => {
+    const response = await get({
+        config: {
+            headers: {
+                'Authorization': `Token token="${apiKey}"`,
+            },
+        },
+        url: `https://littlewarden.com/api/sites/${urlId}`,
+    });
+    return response.getBodyDefaultTo({ defaultValue: {}, keys: ['data', 'site'] });
+};
+export default getUrlStatus;

package/dist/esm/littleWarden/index.d.ts

@@ -0,0 +1,5 @@
+export declare enum State {
+    ALL_GOOD = "all_good",
+    DANGER = "danger",
+    WARNING = "warning"
+}

package/dist/esm/littleWarden/index.js

@@ -0,0 +1,6 @@
+export var State;
+(function (State) {
+    State["ALL_GOOD"] = "all_good";
+    State["DANGER"] = "danger";
+    State["WARNING"] = "warning";
+})(State || (State = {}));

package/dist/esm/littleWarden/isAllGood.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Check if the field is All Good.
+ */
+declare const isAllGood: ({ field }: {
+    field: any;
+}) => boolean;
+export default isAllGood;

package/dist/esm/littleWarden/isAllGood.js

@@ -0,0 +1,13 @@
+import { State } from "./";
+/**
+ * Check if the field is All Good.
+ */
+const isAllGood = ({ field }) => {
+    let response = false;
+    // Check that the status field exists
+    if (undefined !== field && undefined !== field["status"]) {
+        response = State.ALL_GOOD === field["status"];
+    }
+    return response;
+};
+export default isAllGood;

package/dist/esm/littleWarden/isDanger.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Check if the field is Danger.
+ */
+declare const isDanger: ({ field }: {
+    field: any;
+}) => boolean;
+export default isDanger;

package/dist/esm/littleWarden/isDanger.js

@@ -0,0 +1,13 @@
+import { State } from "./";
+/**
+ * Check if the field is Danger.
+ */
+const isDanger = ({ field }) => {
+    let response = false;
+    // Check that the status field exists
+    if (undefined !== field && undefined !== field['status']) {
+        response = State.DANGER === field['status'];
+    }
+    return response;
+};
+export default isDanger;

package/dist/esm/littleWarden/isWarning.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Check if the field is Warning.
+ */
+declare const isWarning: ({ field }: {
+    field: any;
+}) => boolean;
+export default isWarning;

package/dist/esm/littleWarden/isWarning.js

@@ -0,0 +1,13 @@
+import { State } from "./";
+/**
+ * Check if the field is Warning.
+ */
+const isWarning = ({ field }) => {
+    let response = false;
+    // Check that the status field exists
+    if (undefined !== field && undefined !== field['status']) {
+        response = State.WARNING === field['status'];
+    }
+    return response;
+};
+export default isWarning;

package/dist/esm/lumar/api/buildRequest.d.ts

@@ -0,0 +1,15 @@
+import { Filter } from './filter';
+import { Order } from './orders';
+export type Request = {
+    crawlId?: number;
+    customPath?: string;
+    filters?: Filter[];
+    limit?: number;
+    orders?: Order[];
+    page?: number;
+    projectId?: number;
+    report?: string;
+    subReport?: string;
+};
+declare const buildRequest: ({ crawlId, customPath, filters, limit, orders, page, projectId, report, subReport, }: Request) => string;
+export default buildRequest;

package/dist/esm/lumar/api/buildRequest.js

@@ -0,0 +1,78 @@
+import reports from './reports';
+import ordersConstants from './orders';
+const buildRequest = ({ crawlId, customPath, filters, limit, orders, page, projectId, report, subReport, }) => {
+    let url = `https://api.deepcrawl.com/accounts/37204`;
+    // Limit must be greater than 0 but also less than 200
+    if (undefined !== limit) {
+        if (limit > 200) {
+            throw `Maximum row limit is 200.`;
+        }
+        else if (0 === limit) {
+            throw `Row limit must be greater than 0.`;
+        }
+    }
+    // Set limit to 200 by default
+    else {
+        limit = 200;
+    }
+    // If a report other than List Crawls is selected, Crawl ID must be defined
+    if (undefined !== report && reports.LIST_CRAWLS !== report && undefined == crawlId) {
+        throw `Crawl ID must be defined.`;
+    }
+    switch (report) {
+        case reports.LIST_CRAWLS:
+            url += `/projects/${projectId}/crawls`;
+            break;
+        case reports.LIST_PROJECTS:
+            url += `/projects`;
+            break;
+        case reports.CUSTOM:
+            url += customPath;
+            url = url.replace('[PROJECT]', projectId.toString());
+            url = url.replace('[CRAWL]', crawlId.toString());
+            break;
+        case reports.CRAWL_STATISTICS:
+            url += `/projects/${projectId}/crawls/${crawlId}/statistics`;
+            break;
+        case reports.LIST_REPORTS:
+            url += `/projects/${projectId}/crawls/${crawlId}/reports`;
+            break;
+        case reports.REPORT_ROWS:
+        case reports.REPORT_STATISTICS:
+        case reports.REPORT_SUMMARY:
+            url += `/projects/${projectId}/crawls/${crawlId}/reports/${subReport}`;
+            if (reports.REPORT_ROWS === report) {
+                url += `/report_rows`;
+            }
+            else if (reports.REPORT_STATISTICS === report) {
+                url += `/statistics`;
+            }
+            break;
+    }
+    // Specify the limit
+    url += -1 === url.indexOf('?') ? '?' : '&';
+    url += `per_page=${limit}`;
+    // Set the page number
+    if (undefined !== page) {
+        url += -1 === url.indexOf('?') ? '?' : '&';
+        url += `page=${page}`;
+    }
+    // Apply any filters
+    if (undefined !== filters) {
+        filters.map(filter => {
+            url += -1 === url.indexOf('?') ? '?' : '&';
+            url += `q[${filter.field}_${filter.operand}]=${filter.value}`;
+        });
+    }
+    // Order the data
+    if (undefined !== orders) {
+        url += -1 === url.indexOf('?') ? '?' : '&';
+        orders.map((order, index) => {
+            url += 0 === index ? 'sort=' : ',';
+            url += ordersConstants.DESCENDING === order.direction ? '-' : '';
+            url += order.field;
+        });
+    }
+    return url;
+};
+export default buildRequest;

package/dist/esm/lumar/api/filter.d.ts

@@ -0,0 +1,5 @@
+export type Filter = {
+    field: string;
+    operand: string;
+    value: string;
+};

package/dist/esm/lumar/api/filter.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/lumar/api/getData.d.ts

@@ -0,0 +1,6 @@
+import ApiResponse from '../../api/ApiResponse';
+type GetData = {
+    url: string;
+};
+declare const getData: ({ url }: GetData) => Promise<ApiResponse>;
+export default getData;

package/dist/esm/lumar/api/getData.js

@@ -0,0 +1,13 @@
+import get from '../../axios/get';
+const getData = async ({ url }) => {
+    const apiResponse = await get({
+        config: {
+            headers: {
+                'X-AUTH-TOKEN': global.deepcrawlApi.token,
+            },
+        },
+        url,
+    });
+    return apiResponse;
+};
+export default getData;

package/dist/esm/lumar/api/initClient.d.ts

@@ -0,0 +1,13 @@
+type InitClient = {
+    auth: {
+        apiId: string;
+        apiKey: string;
+    };
+};
+/**
+ * http://api-docs.deepcrawl.com/
+ * Renew the Lumar user token.
+ * Token lasts for 6 hours.
+ */
+declare const initClient: ({ auth }: InitClient) => Promise<void>;
+export default initClient;

package/dist/esm/lumar/api/initClient.js

@@ -0,0 +1,33 @@
+import post from '../../axios/post';
+global.lumarApi = {
+    token: null,
+    updatedAt: null,
+};
+/**
+ * http://api-docs.deepcrawl.com/
+ * Renew the Lumar user token.
+ * Token lasts for 6 hours.
+ */
+const initClient = async ({ auth }) => {
+    // todo: Or was more than 6 hours ago
+    if (null === global.lumarApi.token) {
+        const apiResponse = await post({
+            config: {
+                auth: {
+                    username: auth.apiId,
+                    password: auth.apiKey,
+                },
+            },
+            url: `https://api.deepcrawl.com/sessions`,
+        });
+        // Ensure the value exists
+        const token = apiResponse.getBodyDefaultTo({
+            defaultValue: null,
+            keys: ['token'],
+        });
+        if (token) {
+            global.lumarApi.token = token;
+        }
+    }
+};
+export default initClient;

package/dist/esm/lumar/api/operands.d.ts

@@ -0,0 +1,11 @@
+declare const operands: {
+    EQUALS: string;
+    CONTAINS: string;
+    LESS_THAN: string;
+    LESS_THAN_OR_EQUAL: string;
+    GREATER_THAN: string;
+    GREATER_THAN_OR_EQUAL: string;
+    REGEX: string;
+    NOT_REGEX: string;
+};
+export default operands;

package/dist/esm/lumar/api/operands.js

@@ -0,0 +1,11 @@
+const operands = {
+    EQUALS: 'eql',
+    CONTAINS: 'cont',
+    LESS_THAN: 'lt',
+    LESS_THAN_OR_EQUAL: 'lte',
+    GREATER_THAN: 'gt',
+    GREATER_THAN_OR_EQUAL: 'gte',
+    REGEX: 'rgx',
+    NOT_REGEX: 'nrgx',
+};
+export default operands;

package/dist/esm/lumar/api/orders.d.ts

@@ -0,0 +1,9 @@
+export type Order = {
+    direction: string;
+    field: string;
+};
+declare const orders: {
+    ASCENDING: string;
+    DESCENDING: string;
+};
+export default orders;

package/dist/esm/lumar/api/orders.js

@@ -0,0 +1,5 @@
+const orders = {
+    ASCENDING: 'asc',
+    DESCENDING: 'desc',
+};
+export default orders;

package/dist/esm/lumar/api/reports.d.ts

@@ -0,0 +1,11 @@
+declare const reports: {
+    CRAWL_STATISTICS: string;
+    CUSTOM: string;
+    LIST_PROJECTS: string;
+    LIST_CRAWLS: string;
+    LIST_REPORTS: string;
+    REPORT_ROWS: string;
+    REPORT_STATISTICS: string;
+    REPORT_SUMMARY: string;
+};
+export default reports;

package/dist/esm/lumar/api/reports.js

@@ -0,0 +1,11 @@
+const reports = {
+    CRAWL_STATISTICS: 'crawl_statistics',
+    CUSTOM: 'custom',
+    LIST_PROJECTS: 'list_projects',
+    LIST_CRAWLS: 'list_crawls',
+    LIST_REPORTS: 'list_reports',
+    REPORT_ROWS: 'report_rows',
+    REPORT_STATISTICS: 'report_statistics',
+    REPORT_SUMMARY: 'report_summary',
+};
+export default reports;

package/dist/esm/lumar/graphql/columns.d.ts

@@ -0,0 +1,29 @@
+declare const columns: {
+    CUSTOM_EXTRACTION_1: string;
+    CUSTOM_EXTRACTION_2: string;
+    DEEP_RANK: string;
+    DESCRIPTION: string;
+    DUPLICATE_PAGE: string;
+    DUPLICATE_DESCRIPTION_FROM: string;
+    DUPLICATE_TITLE_FROM: string;
+    FOUND_AT_URL: string;
+    FOUND_IN_CRAWL: string;
+    FOUND_IN_SEARCH_CONSOLE: string;
+    FOUND_IN_SITEMAP: string;
+    LEVEL: string;
+    PAGE_TITLE: string;
+    PRIMARY_PAGE: string;
+    PRIMARY_URL_FROM: string;
+    PRIMARY_URL_FROM_STATUS_CODE: string;
+    REDIRECTED_TO_URL: string;
+    REDIRECTED_TO_STATUS_CODE: string;
+    REDIRECTION_CHAIN: string;
+    SOURCE_URL: string;
+    STATUS_CODE: string;
+    TARGET_STATUS_CODE: string;
+    URL: string;
+    URL_COUNT: string;
+    URL_TO: string;
+    URL_TO_STATUS_CODE: string;
+};
+export default columns;

package/dist/esm/lumar/graphql/columns.js

@@ -0,0 +1,29 @@
+const columns = {
+    CUSTOM_EXTRACTION_1: 'customExtraction1',
+    CUSTOM_EXTRACTION_2: 'customExtraction2',
+    DEEP_RANK: 'deepRank',
+    DESCRIPTION: 'description',
+    DUPLICATE_PAGE: 'duplicatePage',
+    DUPLICATE_DESCRIPTION_FROM: 'duplicateDescriptionPrimaryUrl',
+    DUPLICATE_TITLE_FROM: 'duplicateTitlePrimaryUrl',
+    FOUND_AT_URL: 'foundAtUrl',
+    FOUND_IN_CRAWL: 'foundInWebCrawl',
+    FOUND_IN_SEARCH_CONSOLE: 'foundInGoogleSearchConsole',
+    FOUND_IN_SITEMAP: 'foundInSitemap',
+    LEVEL: 'level',
+    PAGE_TITLE: 'pageTitle',
+    PRIMARY_PAGE: 'primaryPage',
+    PRIMARY_URL_FROM: 'primaryUrlFrom',
+    PRIMARY_URL_FROM_STATUS_CODE: 'primaryUrlFromStatusCode',
+    REDIRECTED_TO_URL: 'redirectedToUrl',
+    REDIRECTED_TO_STATUS_CODE: 'redirectedToStatusCode',
+    REDIRECTION_CHAIN: 'redirectionChain',
+    SOURCE_URL: 'sourceUrl',
+    STATUS_CODE: 'httpStatusCode',
+    TARGET_STATUS_CODE: 'targetStatusCode',
+    URL: 'url',
+    URL_COUNT: 'urlCount',
+    URL_TO: 'urlTo',
+    URL_TO_STATUS_CODE: 'urlToStatusCode',
+};
+export default columns;

package/dist/esm/lumar/graphql/getData.d.ts

@@ -0,0 +1,9 @@
+import ApiResponse from '../../api/ApiResponse';
+type GetData = {
+    query: string;
+    variables?: {
+        [key: string]: any;
+    };
+};
+declare const getData: ({ query, variables }: GetData) => Promise<ApiResponse>;
+export default getData;

package/dist/esm/lumar/graphql/getData.js

@@ -0,0 +1,21 @@
+import post from '../../axios/post';
+const getData = async ({ query, variables }) => {
+    let queryClean = query.trim();
+    queryClean = queryClean.replace(/\n/g, ' ');
+    queryClean = queryClean.replace(/\t/g, ' ');
+    queryClean = queryClean.replace(/  /g, ' ');
+    const apiResponse = await post({
+        config: {
+            headers: {
+                'x-auth-token': global.lumarGraphql.token,
+            },
+        },
+        data: {
+            query: queryClean,
+            variables,
+        },
+        url: 'https://api.lumar.io/graphql',
+    });
+    return apiResponse;
+};
+export default getData;