@xapp/arachne-cli 0.9.1 → 1.1.0

package/lib/analyze.d.ts

@@ -0,0 +1,11 @@
+/*! Copyright (c) 2023, XAPP AI */
+import "isomorphic-fetch";
+export interface AnalyzeArgs {
+    u?: string;
+    url: string;
+    h?: boolean;
+    headless?: boolean;
+    v?: boolean;
+    verbose?: boolean;
+}
+export declare function analyze(argv: AnalyzeArgs): Promise<void>;

package/lib/analyze.js

@@ -0,0 +1,89 @@
+/*! Copyright (c) 2023, XAPP AI */
+/* eslint-disable no-console */
+import "isomorphic-fetch";
+import { ArachneWebService, hexToRGB } from "@xapp/arachne-web-service";
+import terminalImage from "terminal-image";
+import terminalLink from 'terminal-link';
+import { readConfig, tempDirectory } from "./config.js";
+export async function analyze(argv) {
+    const pageWait = 8000;
+    const url = argv.url;
+    console.log(`Analyzing ${url}`);
+    console.log(`This will take approximately ${pageWait} ms`);
+    const config = await readConfig();
+    const openAIKey = config?.key;
+    if (!openAIKey) {
+        console.log(`No OPENAI_API_KEY environment variable found, limiting what we can do.`);
+    }
+    const verbose = argv.v;
+    if (verbose) {
+        process.env.STENTOR_LOG_LEVEL = "debug";
+        console.log(`VERBOSE ON!`);
+    }
+    const service = new ArachneWebService();
+    // Ensure the temp directory exists and get its path
+    const tempDir = await tempDirectory();
+    if (tempDir) {
+        console.log(`Saving to temp directory ${tempDir}`);
+    }
+    const data = await service.getWebsiteData(url, {
+        screenshotsPath: tempDir,
+        markdownPath: tempDir,
+        pageWait,
+        pageHandlerTimeout: 120000,
+        openAIKey,
+        headless: false,
+        viewportSize: { width: 1080, height: 800 }
+    });
+    const business = data.business;
+    if (verbose) {
+        console.log(JSON.stringify(data, undefined, 2));
+    }
+    console.log(`Business:`);
+    console.log(`\tname: ${business.name}`);
+    console.log(`\tcategory: ${business.category}`);
+    console.log(``);
+    const site = data.site;
+    // site.schema
+    const primaryPreview = '\x1b[48;2;' + hexToRGB(site.primaryColor) + 'm      \x1b[0m';
+    const secondaryPreview = '\x1b[48;2;' + hexToRGB(site.secondaryColor) + 'm      \x1b[0m';
+    const vibrantPreview = '\x1b[48;2;' + hexToRGB(site.vibrantColor) + 'm      \x1b[0m';
+    const darkVibrantPreview = '\x1b[48;2;' + hexToRGB(site.vibrantColor) + 'm      \x1b[0m';
+    console.log('Site:');
+    console.log(`\tdescription: ${site.description}`);
+    console.log(`\tvibrantColor: ${site.darkVibrantColor} ${vibrantPreview}`);
+    console.log(`\tdarkVibrantColor: ${site.darkVibrantColor} ${darkVibrantPreview}`);
+    console.log(`\tprimaryColor: ${site.primaryColor} ${primaryPreview}`);
+    console.log(`\tsecondaryColor: ${site.secondaryColor} ${secondaryPreview}`);
+    if (site.logo) {
+        const link = terminalLink('logo', site.logo, { fallback: (text, url) => `${text} (${url})` });
+        console.log(`\t${link}: ${site.logo}`);
+        try {
+            const response = await fetch(site.logo);
+            // Ensure the request was successful
+            if (!response.ok) {
+                throw new Error(`Failed to fetch image: ${response.statusText}`);
+            }
+            const arrayBuffer = await response.arrayBuffer();
+            const imageBuffer = Buffer.from(arrayBuffer);
+            const image = await terminalImage.buffer(imageBuffer, { preserveAspectRatio: true, width: 20, height: 20 });
+            console.log(image);
+        }
+        catch (e) {
+            console.log(`Unable to load image ${site.logo} ${e.message}`);
+        }
+    }
+    console.log(``);
+    console.log(`Technologies:`);
+    const installed = data.detectedTechnologies.filter((technology) => technology.installed);
+    if (installed.length === 0) {
+        console.log(`\tnone detected`);
+    }
+    else {
+        const installedList = installed.reduce((previous, current) => {
+            return `${previous}${current.technology}, `;
+        }, "");
+        console.log(`\t${installedList}`);
+    }
+}
+//# sourceMappingURL=analyze.js.map

package/lib/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,+BAA+B;AAC/B,OAAO,kBAAkB,CAAC;AAC1B,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACxE,OAAO,aAAa,MAAM,gBAAgB,CAAC;AAC3C,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAaxD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAiB;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC;IAEtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,gCAAgC,QAAQ,KAAK,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;IAClC,MAAM,SAAS,GAAG,MAAM,EAAE,GAAG,CAAC;IAE9B,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,wEAAwE,CAAC,CAAA;IACzF,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IACvB,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,OAAO,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAExC,oDAAoD;IACpD,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IAEtC,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,GAAG,EAAE;QAC3C,eAAe,EAAE,OAAO;QACxB,YAAY,EAAE,OAAO;QACrB,QAAQ;QACR,kBAAkB,EAAE,MAAM;QAC1B,SAAS;QACT,QAAQ,EAAE,KAAK;QACf,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;KAC7C,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAE/B,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IAEvB,cAAc;IACd,MAAM,cAAc,GAAG,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAC;IACrF,MAAM,gBAAgB,GAAG,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,gBAAgB,CAAC;IACzF,MAAM,cAAc,GAAG,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAA;IACpF,MAAM,kBAAkB,GAAG,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAA;IAExF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,gBAAgB,IAAI,cAAc,EAAE,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,CAAC,gBAAgB,IAAI,kBAAkB,EAAE,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,YAAY,IAAI,cAAc,EAAE,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,cAAc,IAAI,gBAAgB,EAAE,CAAC,CAAC;IAE5E,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC,CAAC;QAC9F,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAExC,oCAAoC;YACpC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACjD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAE5G,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;QACjE,CAAC;IACL,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAEzF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACJ,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE;YACzD,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC;QAChD,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,OAAO,CAAC,GAAG,CAAC,KAAK,aAAa,EAAE,CAAC,CAAC;IACtC,CAAC;AACL,CAAC"}

package/lib/config.d.ts

@@ -0,0 +1,27 @@
+/*! Copyright (c) 2023, XAPP AI */
+export interface ConfigArgs {
+    key: string;
+    k?: string;
+}
+export interface ArachneConfig {
+    /**
+     * OpenAI API Key
+     */
+    key?: string;
+}
+/**
+ * Stores a configuration key in the local machine's appropriate directory.
+ * @param argv The arguments containing the key to store.
+ */
+export declare function config(argv: ConfigArgs): Promise<void>;
+/**
+ * Returns the temp directory and ensures that the temp directory exists.
+ *
+ * @returns The path to the temp directory.
+ */
+export declare function tempDirectory(): Promise<string>;
+/**
+ * Reads the configuration from the config file.
+ * @returns The stored configuration key, or undefined if not found.
+ */
+export declare function readConfig(): Promise<ArachneConfig | undefined>;

package/lib/config.js

@@ -0,0 +1,90 @@
+/*! Copyright (c) 2023, XAPP AI */
+import * as os from 'os';
+import * as path from 'path';
+import * as fs from 'fs/promises';
+/**
+ * Stores a configuration key in the local machine's appropriate directory.
+ * @param argv The arguments containing the key to store.
+ */
+export async function config(argv) {
+    const keyToStore = argv.key || argv.k;
+    if (!keyToStore) {
+        throw new Error('No key provided for configuration.');
+    }
+    const configDir = getConfigDirectory();
+    const configFile = path.join(configDir, 'arachne.config.json');
+    try {
+        await fs.mkdir(configDir, { recursive: true });
+    }
+    catch (error) {
+        // Type guard to check if error is an instance of NodeJS.ErrnoException
+        if (error instanceof Error && 'code' in error) {
+            const errnoError = error;
+            if (errnoError.code !== 'EEXIST') {
+                throw errnoError; // Rethrow if error is not related to directory existence
+            }
+        }
+        else {
+            throw error; // Rethrow if error is not an ErrnoException
+        }
+    }
+    const configData = { key: keyToStore };
+    await fs.writeFile(configFile, JSON.stringify(configData, null, 2));
+}
+/**
+ * Gets the directory path where configuration files are stored based on the OS.
+ * @returns The path to the configuration directory.
+ */
+function getConfigDirectory() {
+    const homeDir = os.homedir();
+    switch (os.platform()) {
+        case 'win32':
+            return path.join(homeDir, 'AppData', 'Local', 'arachne');
+        case 'darwin':
+        case 'linux':
+            return path.join(homeDir, '.config', 'arachne');
+        default:
+            throw new Error('Unsupported platform');
+    }
+}
+/**
+ * Returns the temp directory and ensures that the temp directory exists.
+ *
+ * @returns The path to the temp directory.
+ */
+export async function tempDirectory() {
+    const storageDir = getConfigDirectory();
+    const tempDir = path.join(storageDir, 'temp');
+    try {
+        await fs.mkdir(tempDir, { recursive: true });
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error) {
+            const errnoError = error;
+            if (errnoError.code !== 'EEXIST') {
+                throw errnoError; // Rethrow if error is not related to directory existence
+            }
+        }
+        else {
+            throw error; // Rethrow if error is not an ErrnoException
+        }
+    }
+    return tempDir;
+}
+/**
+ * Reads the configuration from the config file.
+ * @returns The stored configuration key, or undefined if not found.
+ */
+export async function readConfig() {
+    try {
+        const configDir = getConfigDirectory();
+        const configFile = path.join(configDir, 'arachne.config.json');
+        const configData = JSON.parse(await fs.readFile(configFile, 'utf8'));
+        return configData;
+    }
+    catch (error) {
+        console.error('Error reading the config file:', error);
+        return undefined;
+    }
+}
+//# sourceMappingURL=config.js.map

package/lib/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAElC,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAclC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAgB;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC;IACtC,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,SAAS,GAAG,kBAAkB,EAAE,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;IAE/D,IAAI,CAAC;QACD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,uEAAuE;QACvE,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,KAA8B,CAAC;YAClD,IAAI,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,UAAU,CAAC,CAAC,yDAAyD;YAC/E,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,CAAC,CAAC,4CAA4C;QAC7D,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAkB,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;IACtD,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAExE,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB;IACvB,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IAE7B,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;QACpB,KAAK,OAAO;YACR,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAC7D,KAAK,QAAQ,CAAC;QACd,KAAK,OAAO;YACR,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACpD;YACI,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAChD,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa;IAC/B,MAAM,UAAU,GAAG,kBAAkB,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAE9C,IAAI,CAAC;QACD,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,KAA8B,CAAC;YAClD,IAAI,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,UAAU,CAAC,CAAC,yDAAyD;YAC/E,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,CAAC,CAAC,4CAA4C;QAC7D,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU;IAC5B,IAAI,CAAC;QACD,MAAM,SAAS,GAAG,kBAAkB,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QACrE,OAAO,UAAU,CAAC;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACvD,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC"}

package/lib/crawl.d.ts

@@ -0,0 +1,30 @@
+export interface CrawlArgs {
+    url: string;
+    u?: string;
+    a?: boolean;
+    faq?: boolean;
+    d?: string;
+    dir?: string;
+    e?: string;
+    executablePath?: string;
+    h?: boolean;
+    headless?: boolean;
+    m?: number;
+    max?: number;
+    o?: number;
+    timeout?: number;
+    b?: number;
+    break?: number;
+    settle?: number;
+    p?: number;
+    pages?: number;
+    s?: boolean;
+    stealth?: boolean;
+    t?: string;
+    text?: string;
+    w?: string;
+    whitelist?: string;
+    x?: boolean;
+    metadata?: boolean;
+}
+export declare function crawl(argv: CrawlArgs): Promise<void>;

package/lib/crawl.js

@@ -0,0 +1,196 @@
+/*! Copyright (c) 2023, XAPP AI */
+/* eslint-disable no-console */
+import { Arachne, ArachneURLPattern, autoScroll, enqueueLinks, MemoryRequestQueue, } from "@xapp/arachne";
+import { ArachneJSONLD, ArachneHTMLtoText, isAnswer } from "@xapp/arachne-parsers";
+import { URL } from "url";
+import { existsSync, mkdirSync, writeFileSync, readFileSync } from "fs";
+import { resolve } from "path";
+export async function crawl(argv) {
+    const inputURL = argv.url;
+    const url = new URL(inputURL);
+    const cleanHost = url.host.replace(/\./g, "_");
+    const cleanPath = url.pathname.replace(/\//g, "_");
+    let patterns;
+    const whiteListPath = argv.w;
+    if (whiteListPath) {
+        console.log(whiteListPath);
+        const resolvedPath = resolve(whiteListPath);
+        const contents = readFileSync(resolvedPath, "utf-8");
+        patterns = contents.split("\n");
+    }
+    else {
+        patterns = inputURL.endsWith("/") ? [`${inputURL}[.*]`] : [`${inputURL}/[.*]`];
+    }
+    const directory = Array.isArray(argv.d) ? argv.d[1] : argv.d;
+    const path = directory ? resolve(directory) : undefined;
+    if (path) {
+        // Make sure it exists
+        if (!existsSync(path)) {
+            throw new Error(`Path ${directory} does not exist.  Please provide an existing path to save the HTML`);
+        }
+    }
+    if (typeof argv.b === "number") {
+        // make sure it is less than the timeout
+        const timeout = argv.o || 4000;
+        if (argv.b > timeout) {
+            throw new Error(`Settling time is greater than page handler timeout, please adjust your inputs. -o ${argv.b + 4000}  `);
+        }
+    }
+    let saveDir;
+    if (path) {
+        const exportDirName = `${cleanHost}${cleanPath}-${new Date().getTime()}`;
+        saveDir = resolve(path, exportDirName);
+        mkdirSync(saveDir);
+    }
+    const queue = new MemoryRequestQueue();
+    queue.addRequest({ url: inputURL });
+    const crawlPatterns = patterns.map((pattern) => new ArachneURLPattern(pattern));
+    const answers = new Map();
+    console.log(`Starting crawl at ${inputURL}`);
+    const crawler = Arachne.crawler({
+        maxConcurrentPages: argv.p,
+        maxCrawlAttempts: argv.m,
+        pageHandlerTimeout: argv.o,
+        stealth: argv.s,
+        launchOptions: { headless: argv.h, timeout: 5000, executablePath: argv.e }, /* timeout set to 5 seconds, default of 30 is too long */
+        queue,
+        pageHandler: async (page, request, response) => {
+            const title = await page.title();
+            console.log(`Loaded ${request.url} : ${title}`);
+            if (typeof argv.b === "number") {
+                const pageWaitTime = argv.b;
+                console.log(`Waiting for ${pageWaitTime} ms for page to completely load....`);
+                await autoScroll(page, pageWaitTime);
+                console.log('...and done waiting.');
+            }
+            // Get the raw HTML
+            // It is important to use page.content() versus response.text() since
+            // page.content() gets you the HTML after it has been modified by the JavaScript
+            // on the page while response.text() only gives you the HTML from the original GET
+            const text = await page.content();
+            // Pull out the raw html
+            if (saveDir) {
+                const requestURL = new URL(request.url);
+                const cleanHost = requestURL.host.replace(/(\.|\/)/g, "_").replace(/_+$/g, "").replace(/^_+/g, "");
+                const cleanRequestPath = requestURL.pathname.replace(/\//g, "_").replace(".html", "").replace(/_+$/g, "").replace(/^_+/g, "");
+                const cleanFileName = `${cleanHost}_${cleanRequestPath}`.replace(/_+$/g, "");
+                const htmlName = `${cleanFileName}.html`;
+                console.log(`File Name: ${htmlName}`);
+                const filePath = resolve(saveDir, htmlName);
+                console.log(`Saving to ${filePath}`);
+                writeFileSync(filePath, text);
+                if (argv.x) {
+                    const date = new Date().toISOString();
+                    // Doc title
+                    const metadata = {
+                        Attributes: {
+                            _created_at: date,
+                            _source_uri: request.url,
+                            _last_updated_at: date
+                        },
+                        Title: title,
+                        ContentType: "HTML"
+                    };
+                    const metadataFileName = `${htmlName}.metadata.json`;
+                    const metadataFilePath = resolve(saveDir, metadataFileName);
+                    console.log(`Saving metadata to ${metadataFilePath}`);
+                    const metadataText = JSON.stringify(metadata, undefined, 2);
+                    writeFileSync(metadataFilePath, metadataText);
+                }
+                if (typeof argv.t === "string") {
+                    const toMD = argv.t === "markdown" ? true : false;
+                    // Write some MARKDOWN!
+                    const convertedHTML = new ArachneHTMLtoText().convert(text, toMD);
+                    const extension = toMD ? `md` : `txt`;
+                    const markdownName = `${cleanFileName}.${extension}`;
+                    const markdownFilePath = resolve(saveDir, markdownName);
+                    console.log(`Saving markdown to ${markdownFilePath}`);
+                    writeFileSync(markdownFilePath, convertedHTML);
+                }
+            }
+            // Let's see if we can find some structured data.
+            const questions = new ArachneJSONLD(text).getQuestions();
+            if (questions.length > 0) {
+                console.log(`${questions.length} questions found on page.`);
+                // Clean the answer
+                questions.forEach((question) => {
+                    // name has the question text
+                    // acceptedAnswer has the answer text
+                    const questionText = typeof question.name === "string" ? question.name : "";
+                    const acceptedAnswer = question.acceptedAnswer;
+                    if (acceptedAnswer && isAnswer(acceptedAnswer)) {
+                        const answerText = typeof acceptedAnswer.text === "string" ? acceptedAnswer.text : "";
+                        if (answerText.length > 0) {
+                            // Clean it
+                            const cleanAnswer = new ArachneHTMLtoText().convert(answerText);
+                            let existingAnswer = answers.get(cleanAnswer);
+                            if (existingAnswer) {
+                                existingAnswer.questions.push(questionText);
+                            }
+                            else {
+                                existingAnswer = {
+                                    answer: cleanAnswer,
+                                    questions: [questionText],
+                                    url: request.url
+                                };
+                            }
+                            // Add it back!
+                            answers.set(cleanAnswer, existingAnswer);
+                        }
+                    }
+                });
+            }
+            await enqueueLinks(page, queue, crawlPatterns);
+            const info = await queue.getInfo();
+            console.log(`Completed ${info.completeRequests} of ${info.totalRequests}, ${info.inProgressRequests} are in progress with ${info.pendingRequests} pending and ${info.failedRequests} failed.`);
+        }
+    });
+    try {
+        await crawler.run();
+    }
+    catch (e) {
+        console.log('Unable to start the crawler.');
+        console.error(e);
+    }
+    console.log('Crawler complete.');
+    const info = await queue.getInfo();
+    console.log(`Final count, completed ${info.completeRequests} of ${info.totalRequests}, ${info.failedRequests} failed.`);
+    const failed = queue.getFailed();
+    if (failed.length > 0) {
+        console.log('Failures....');
+        failed.forEach((failedRequest) => {
+            console.log(`\t${failedRequest.url} failed after ${failedRequest.requestAttempts} attempts with final error: ${failedRequest.errorMessages ? failedRequest.errorMessages[failedRequest.errorMessages.length - 1] : "unknown"}`);
+        });
+    }
+    console.log(`Found ${answers.size} answers.`);
+    if (argv.faq && saveDir) {
+        // This follows the JSON format https://docs.aws.amazon.com/kendra/latest/dg/in-creating-faq.html#faq-custom-json
+        const qaJSON = {
+            "SchemaVersion": 1,
+            "FaqDocuments": []
+        };
+        let qaCSV = `"Question","Answer","URL"`;
+        answers.forEach((answer) => {
+            // For the questions on the answer, add a line
+            answer.questions.forEach((question) => {
+                qaJSON.FaqDocuments.push({
+                    Question: question,
+                    Answer: answer.answer,
+                    Attributes: {
+                        _source_uri: answer.url
+                    }
+                });
+                qaCSV += `\r\n"${question}","${answer.answer}","${answer.url}"`;
+            });
+        });
+        const questionAnswerName = `question-answers.json`;
+        const qaFilePath = resolve(saveDir, questionAnswerName);
+        console.log(`Saving FAQ JSON to ${qaFilePath}`);
+        writeFileSync(qaFilePath, JSON.stringify(qaJSON, undefined, 2));
+        const csvQuestionName = `question-answers.csv`;
+        const csvQaFilePath = resolve(saveDir, csvQuestionName);
+        console.log(`Saving FAQ CSV to ${csvQaFilePath}`);
+        writeFileSync(csvQaFilePath, qaCSV);
+    }
+}
+//# sourceMappingURL=crawl.js.map

package/lib/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../src/crawl.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,+BAA+B;AAC/B,OAAO,EACH,OAAO,EAIP,iBAAiB,EACjB,UAAU,EACV,YAAY,EACZ,kBAAkB,GACrB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACH,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACX,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAqE/B,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAe;IAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAa,CAAC;IACpC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAEnD,IAAI,QAAkB,CAAC;IAEvB,MAAM,aAAa,GAAuB,IAAI,CAAC,CAAC,CAAC;IACjD,IAAI,aAAa,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAE3B,MAAM,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACrD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,CAAC;SAAM,CAAC;QACJ,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,OAAO,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,SAAS,GAAuB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjF,MAAM,IAAI,GAAuB,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5E,IAAI,IAAI,EAAE,CAAC;QACP,sBAAsB;QACtB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,QAAQ,SAAS,oEAAoE,CAAC,CAAC;QAC3G,CAAC;IACL,CAAC;IAED,IAAI,OAAO,IAAI,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC7B,wCAAwC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC;QAC/B,IAAI,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,qFAAqF,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;QAC3H,CAAC;IACL,CAAC;IAED,IAAI,OAA2B,CAAC;IAEhC,IAAI,IAAI,EAAE,CAAC;QACP,MAAM,aAAa,GAAG,GAAG,SAAS,GAAG,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QACzE,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QACvC,SAAS,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAGD,MAAM,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;IACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEpC,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;IAQhF,MAAM,OAAO,GAAwB,IAAI,GAAG,EAAE,CAAC;IAE/C,OAAO,CAAC,GAAG,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IAE7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC5B,kBAAkB,EAAE,IAAI,CAAC,CAAC;QAC1B,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACxB,kBAAkB,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,EAAE,IAAI,CAAC,CAAC;QACf,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,yDAAyD;QACrI,KAAK;QACL,WAAW,EAAE,KAAK,EAAE,IAAiB,EAAE,OAAuB,EAAE,QAAyB,EAAE,EAAE;YAEzF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;YAEhD,IAAI,OAAO,IAAI,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,eAAe,YAAY,qCAAqC,CAAC,CAAC;gBAC9E,MAAM,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC;YAED,mBAAmB;YACnB,qEAAqE;YACrE,gFAAgF;YAChF,kFAAkF;YAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAElC,wBAAwB;YACxB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAExC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACnG,MAAM,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAC9H,MAAM,aAAa,GAAG,GAAG,SAAS,IAAI,gBAAgB,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAC7E,MAAM,QAAQ,GAAG,GAAG,aAAa,OAAO,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAE5C,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;gBACrC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAE9B,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC;oBAET,MAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;oBACtC,YAAY;oBACZ,MAAM,QAAQ,GAAuB;wBACjC,UAAU,EAAE;4BACR,WAAW,EAAE,IAAI;4BACjB,WAAW,EAAE,OAAO,CAAC,GAAG;4BACxB,gBAAgB,EAAE,IAAI;yBACzB;wBACD,KAAK,EAAE,KAAK;wBACZ,WAAW,EAAE,MAAM;qBACtB,CAAA;oBAED,MAAM,gBAAgB,GAAG,GAAG,QAAQ,gBAAgB,CAAC;oBACrD,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;oBAE5D,OAAO,CAAC,GAAG,CAAC,sBAAsB,gBAAgB,EAAE,CAAC,CAAC;oBACtD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;oBAC5D,aAAa,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;gBAClD,CAAC;gBAED,IAAI,OAAO,IAAI,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;oBAC7B,MAAM,IAAI,GAAY,IAAI,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;oBAC1D,uBAAuB;oBACvB,MAAM,aAAa,GAAG,IAAI,iBAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAClE,MAAM,SAAS,GAAW,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC9C,MAAM,YAAY,GAAG,GAAG,aAAa,IAAI,SAAS,EAAE,CAAC;oBACrD,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;oBACxD,OAAO,CAAC,GAAG,CAAC,sBAAsB,gBAAgB,EAAE,CAAC,CAAC;oBACtD,aAAa,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;gBACnD,CAAC;YACL,CAAC;YAED,iDAAiD;YAEjD,MAAM,SAAS,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YACzD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,2BAA2B,CAAC,CAAC;gBAC5D,mBAAmB;gBACnB,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;oBAC3B,6BAA6B;oBAC7B,qCAAqC;oBACrC,MAAM,YAAY,GAAW,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpF,MAAM,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC;oBAC/C,IAAI,cAAc,IAAI,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;wBAC7C,MAAM,UAAU,GAAW,OAAO,cAAc,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC9F,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACxB,WAAW;4BACX,MAAM,WAAW,GAAG,IAAI,iBAAiB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;4BAEhE,IAAI,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;4BAC9C,IAAI,cAAc,EAAE,CAAC;gCACjB,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;4BAChD,CAAC;iCAAM,CAAC;gCACJ,cAAc,GAAG;oCACb,MAAM,EAAE,WAAW;oCACnB,SAAS,EAAE,CAAC,YAAY,CAAC;oCACzB,GAAG,EAAE,OAAO,CAAC,GAAG;iCACnB,CAAA;4BACL,CAAC;4BACD,eAAe;4BACf,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;wBAC7C,CAAC;oBACL,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;YACD,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,kBAAkB,yBAAyB,IAAI,CAAC,eAAe,gBAAgB,IAAI,CAAC,cAAc,UAAU,CAAC,CAAC;QACnM,CAAC;KACJ,CAAC,CAAC;IAEH,IAAI,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,cAAc,UAAU,CAAC,CAAC;IAExH,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;YAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,aAAa,CAAC,GAAG,iBAAiB,aAAa,CAAC,eAAe,+BAA+B,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAA;QACnO,CAAC,CAAC,CAAA;IACN,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,IAAI,WAAW,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE,CAAC;QAEtB,iHAAiH;QACjH,MAAM,MAAM,GAAkB;YAC1B,eAAe,EAAE,CAAC;YAClB,cAAc,EAAE,EACf;SACJ,CAAC;QAEF,IAAI,KAAK,GAAW,2BAA2B,CAAC;QAEhD,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACvB,8CAA8C;YAC9C,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAClC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC;oBACrB,QAAQ,EAAE,QAAQ;oBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,UAAU,EAAE;wBACR,WAAW,EAAE,MAAM,CAAC,GAAG;qBAC1B;iBACJ,CAAC,CAAC;gBAEH,KAAK,IAAI,QAAQ,QAAQ,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,GAAG,GAAG,CAAC;YACpE,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAG,uBAAuB,CAAC;QACnD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QAExD,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAChD,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhE,MAAM,eAAe,GAAG,sBAAsB,CAAC;QAC/C,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,aAAa,EAAE,CAAC,CAAC;QAClD,aAAa,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;AACL,CAAC"}

package/lib/index.js

@@ -1,233 +1,50 @@
 #!/usr/bin/env node
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
 /*! Copyright (c) 2020, XAPP AI */
 /* eslint-disable no-console */
-const arachne_1 = require("@xapp/arachne");
-const arachne_parsers_1 = require("@xapp/arachne-parsers");
-const url_1 = require("url");
-const fs_1 = require("fs");
-const path_1 = require("path");
-const yargs_1 = __importDefault(require("yargs/yargs"));
-const helpers_1 = require("yargs/helpers");
-const argv = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
-    .command('crawl <url>', 'Crawl the provided URL', (yargs) => {
+import yargs from 'yargs/yargs';
+import { hideBin } from 'yargs/helpers';
+import { crawl } from './crawl.js';
+import { analyze } from './analyze.js';
+import { config } from './config.js';
+const yargsInstance = yargs(hideBin(process.argv));
+yargsInstance.command('crawl <url>', 'Crawl the provided URL', (yargs) => {
+    yargs.positional("url", { type: 'string', alias: ["u"], demandOption: true, describe: "The URL to crawl" });
+    yargs.options({
+        a: { type: 'boolean', alias: ["faq"], default: false, describe: "Generate a JSON file with FAQs, AWS Kendra format, found in JSON+LD while crawling." },
+        d: { type: 'string', alias: ["dir"], describe: "The directory to save the HTML source to" },
+        e: { type: 'string', alias: ["executablePath"], describe: "Executable path Chrome" },
+        h: { type: 'boolean', alias: ["headless"], default: false },
+        m: { type: 'number', alias: ["max"], default: 1000, describe: "The max number of total crawls." },
+        o: { type: 'number', alias: ["timeout"], default: 4000, describe: "The timeout for each individual page load." },
+        b: { type: 'number', alias: ["break", "settle"], describe: "Settle time, in MS, to wait for the page to finish loading.  By default it only waits for page load." },
+        p: { type: 'number', alias: ["pages"], default: 10, describe: "The max number of pages (tabs) running simultaneously." },
+        s: { type: 'boolean', alias: ["stealth"], describe: "Turns on stealth mode" },
+        t: { type: 'string', alias: ["text"], default: undefined, describe: "Possible values are 'text' or 'markdown', requires a directory to be provided as well.  Will save the output to the directory." },
+        w: { type: 'string', alias: ["whitelist"], describe: "A text file where each line is a whitelisted pattern" },
+        x: { type: 'boolean', alias: ["metadata"], describe: "Generate a JSON metadata file for Amazon Kendra" }
+    });
+}, async (argv) => {
+    await crawl(argv);
+});
+yargsInstance.command('analyze <url>', 'Performs an analysis on the provided URL', (yargs) => {
     yargs.positional("url", { type: 'string', alias: ["u"], demandOption: true, describe: "The URL to crawl" });
-})
-    .options({
-    a: { type: 'boolean', alias: ["faq"], default: false, describe: "Generate a JSON file with FAQs, AWS Kendra format, found in JSON+LD while crawling." },
-    d: { type: 'string', alias: ["dir"], describe: "The directory to save the HTML source to" },
-    e: { type: 'string', alias: ["executablePath"], describe: "Executable path Chrome" },
-    h: { type: 'boolean', alias: ["headless"], default: false },
-    m: { type: 'number', alias: ["max"], default: 1000, describe: "The max number of total crawls." },
-    o: { type: 'number', alias: ["timeout"], default: 4000, describe: "The timeout for each individual page load." },
-    b: { type: 'number', alias: ["break", "settle"], describe: "Settle time, in MS, to wait for the page to finish loading.  By default it only waits for page load." },
-    p: { type: 'number', alias: ["pages"], default: 10, describe: "The max number of pages (tabs) running simultaneously." },
-    s: { type: 'boolean', alias: ["stealth"], describe: "Turns on stealth mode" },
-    t: { type: 'string', alias: ["text"], default: undefined, describe: "Possible values are 'text' or 'markdown', requires a directory to be provided as well.  Will save the output to the directory." },
-    w: { type: 'string', alias: ["whitelist"], describe: "A text file where each line is a whitelisted pattern" },
-    x: { type: 'boolean', alias: ["metadata"], describe: "Generate a JSON metadata file for Amazon Kendra" }
-})
-    .nargs('u', 1)
-    .parseSync();
-const inputURL = argv.url;
-const url = new url_1.URL(inputURL);
-const cleanHost = url.host.replace(/\./g, "_");
-const cleanPath = url.pathname.replace(/\//g, "_");
-let patterns;
-const whiteListPath = argv.w;
-if (whiteListPath) {
-    console.log(whiteListPath);
-    const resolvedPath = (0, path_1.resolve)(whiteListPath);
-    const contents = (0, fs_1.readFileSync)(resolvedPath, "utf-8");
-    patterns = contents.split("\n");
-}
-else {
-    patterns = inputURL.endsWith("/") ? [`${inputURL}[.*]`] : [`${inputURL}/[.*]`];
-}
-const directory = Array.isArray(argv.d) ? argv.d[1] : argv.d;
-const path = directory ? (0, path_1.resolve)(directory) : undefined;
-if (path) {
-    // Make sure it exists
-    if (!(0, fs_1.existsSync)(path)) {
-        throw new Error(`Path ${directory} does not exist.  Please provide an existing path to save the HTML`);
-    }
-}
-if (typeof argv.b === "number") {
-    // make sure it is less than the timeout
-    const timeout = argv.o || 4000;
-    if (argv.b > timeout) {
-        throw new Error(`Settling time is greater than page handler timeout, please adjust your inputs. -o ${argv.b + 4000}  `);
-    }
-}
-let saveDir;
-if (path) {
-    const exportDirName = `${cleanHost}${cleanPath}-${new Date().getTime()}`;
-    saveDir = (0, path_1.resolve)(path, exportDirName);
-    (0, fs_1.mkdirSync)(saveDir);
-}
-(() => __awaiter(void 0, void 0, void 0, function* () {
-    const queue = new arachne_1.MemoryRequestQueue();
-    queue.addRequest({ url: inputURL });
-    const crawlPatterns = patterns.map((pattern) => new arachne_1.ArachneURLPattern(pattern));
-    const answers = new Map();
-    console.log(`Starting crawl at ${inputURL}`);
-    const crawler = arachne_1.Arachne.crawler({
-        maxConcurrentPages: argv.p,
-        maxCrawlAttempts: argv.m,
-        pageHandlerTimeout: argv.o,
-        stealth: argv.s,
-        launchOptions: { headless: argv.h, timeout: 5000, executablePath: argv.e },
-        queue,
-        pageHandler: (page, request, response) => __awaiter(void 0, void 0, void 0, function* () {
-            const title = yield page.title();
-            console.log(`Loaded ${request.url} : ${title}`);
-            if (typeof argv.b === "number") {
-                const pageWaitTime = argv.b;
-                console.log(`Waiting for ${pageWaitTime} ms for page to completely load....`);
-                yield (0, arachne_1.autoScroll)(page, pageWaitTime);
-                console.log('...and done waiting.');
-            }
-            // Get the raw HTML
-            // It is important to use page.content() versus response.text() since
-            // page.content() gets you the HTML after it has been modified by the JavaScript
-            // on the page while response.text() only gives you the HTML from the original GET
-            const text = yield page.content();
-            // Pull out the raw html
-            if (saveDir) {
-                const requestURL = new url_1.URL(request.url);
-                const cleanHost = requestURL.host.replace(/(\.|\/)/g, "_").replace(/_+$/g, "").replace(/^_+/g, "");
-                const cleanRequestPath = requestURL.pathname.replace(/\//g, "_").replace(".html", "").replace(/_+$/g, "").replace(/^_+/g, "");
-                const cleanFileName = `${cleanHost}_${cleanRequestPath}`.replace(/_+$/g, "");
-                const htmlName = `${cleanFileName}.html`;
-                console.log(`File Name: ${htmlName}`);
-                const filePath = (0, path_1.resolve)(saveDir, htmlName);
-                console.log(`Saving to ${filePath}`);
-                (0, fs_1.writeFileSync)(filePath, text);
-                if (argv.x) {
-                    const date = new Date().toISOString();
-                    // Doc title
-                    const metadata = {
-                        Attributes: {
-                            _created_at: date,
-                            _source_uri: request.url,
-                            _last_updated_at: date
-                        },
-                        Title: title,
-                        ContentType: "HTML"
-                    };
-                    const metadataFileName = `${htmlName}.metadata.json`;
-                    const metadataFilePath = (0, path_1.resolve)(saveDir, metadataFileName);
-                    console.log(`Saving metadata to ${metadataFilePath}`);
-                    const metadataText = JSON.stringify(metadata, undefined, 2);
-                    (0, fs_1.writeFileSync)(metadataFilePath, metadataText);
-                }
-                if (typeof argv.t === "string") {
-                    const toMD = argv.t === "markdown" ? true : false;
-                    // Write some MARKDOWN!
-                    const convertedHTML = new arachne_parsers_1.ArachneHTMLtoText().convert(text, toMD);
-                    const extension = toMD ? `md` : `txt`;
-                    const markdownName = `${cleanFileName}.${extension}`;
-                    const markdownFilePath = (0, path_1.resolve)(saveDir, markdownName);
-                    console.log(`Saving markdown to ${markdownFilePath}`);
-                    (0, fs_1.writeFileSync)(markdownFilePath, convertedHTML);
-                }
-            }
-            // Let's see if we can find some structured data.
-            const questions = new arachne_parsers_1.ArachneJSONLD(text).getQuestions();
-            if (questions.length > 0) {
-                console.log(`${questions.length} questions found on page.`);
-                // Clean the answer
-                questions.forEach((question) => {
-                    // name has the question text
-                    // acceptedAnswer has the answer text
-                    const questionText = typeof question.name === "string" ? question.name : "";
-                    const acceptedAnswer = question.acceptedAnswer;
-                    if (acceptedAnswer && (0, arachne_parsers_1.isAnswer)(acceptedAnswer)) {
-                        const answerText = typeof acceptedAnswer.text === "string" ? acceptedAnswer.text : "";
-                        if (answerText.length > 0) {
-                            // Clean it
-                            const cleanAnswer = new arachne_parsers_1.ArachneHTMLtoText().convert(answerText);
-                            let existingAnswer = answers.get(cleanAnswer);
-                            if (existingAnswer) {
-                                existingAnswer.questions.push(questionText);
-                            }
-                            else {
-                                existingAnswer = {
-                                    answer: cleanAnswer,
-                                    questions: [questionText],
-                                    url: request.url
-                                };
-                            }
-                            // Add it back!
-                            answers.set(cleanAnswer, existingAnswer);
-                        }
-                    }
-                });
-            }
-            yield (0, arachne_1.enqueueLinks)(page, queue, crawlPatterns);
-            const info = yield queue.getInfo();
-            console.log(`Completed ${info.completeRequests} of ${info.totalRequests}, ${info.inProgressRequests} are in progress with ${info.pendingRequests} pending and ${info.failedRequests} failed.`);
-        })
+    yargs.options({
+        h: { type: 'boolean', alias: ["headless"], default: false },
+        v: { type: 'boolean', alias: ["verbose"], default: false }
+    });
+}, async (argv) => {
+    await analyze(argv);
+});
+yargsInstance.command('config', 'Configures the CLI', (yargs) => {
+    return yargs
+        .option('key', {
+        describe: 'The configuration key for OpenAI',
+        type: 'string',
+        demandOption: true, // Makes sure the key is provided
+        alias: 'k' // Allows the use of -k as an alias for --key
     });
-    try {
-        yield crawler.run();
-    }
-    catch (e) {
-        console.log('Unable to start the crawler.');
-        console.error(e);
-    }
-    console.log('Crawler complete.');
-    const info = yield queue.getInfo();
-    console.log(`Final count, completed ${info.completeRequests} of ${info.totalRequests}, ${info.failedRequests} failed.`);
-    const failed = queue.getFailed();
-    if (failed.length > 0) {
-        console.log('Failures....');
-        failed.forEach((failedRequest) => {
-            console.log(`\t${failedRequest.url} failed after ${failedRequest.requestAttempts} attempts with final error: ${failedRequest.errorMessages ? failedRequest.errorMessages[failedRequest.errorMessages.length - 1] : "unknown"}`);
-        });
-    }
-    console.log(`Found ${answers.size} answers.`);
-    if (argv.faq && saveDir) {
-        // This follows the JSON format https://docs.aws.amazon.com/kendra/latest/dg/in-creating-faq.html#faq-custom-json
-        const qaJSON = {
-            "SchemaVersion": 1,
-            "FaqDocuments": []
-        };
-        let qaCSV = `"Question","Answer","URL"`;
-        answers.forEach((answer) => {
-            // For the questions on the answer, add a line
-            answer.questions.forEach((question) => {
-                qaJSON.FaqDocuments.push({
-                    Question: question,
-                    Answer: answer.answer,
-                    Attributes: {
-                        _source_uri: answer.url
-                    }
-                });
-                qaCSV += `\r\n"${question}","${answer.answer}","${answer.url}"`;
-            });
-        });
-        const questionAnswerName = `question-answers.json`;
-        const qaFilePath = (0, path_1.resolve)(saveDir, questionAnswerName);
-        console.log(`Saving FAQ JSON to ${qaFilePath}`);
-        (0, fs_1.writeFileSync)(qaFilePath, JSON.stringify(qaJSON, undefined, 2));
-        const csvQuestionName = `question-answers.csv`;
-        const csvQaFilePath = (0, path_1.resolve)(saveDir, csvQuestionName);
-        console.log(`Saving FAQ CSV to ${csvQaFilePath}`);
-        (0, fs_1.writeFileSync)(csvQaFilePath, qaCSV);
-    }
-}))();
+}, async (argv) => {
+    await config(argv);
+});
+yargsInstance.parse();
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,kCAAkC;AAClC,+BAA+B;AAC/B,2CASuB;AAEvB,2DAAmF;AAEnF,6BAA0B;AAC1B,2BAAwE;AACxE,+BAA+B;AAE/B,wDAAgC;AAChC,2CAAuC;AAqCvC,MAAM,IAAI,GAAG,IAAA,eAAK,EAAC,IAAA,iBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KACpC,OAAO,CAAC,aAAa,EAAE,wBAAwB,EAAE,CAAC,KAAK,EAAE,EAAE;IACxD,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAA;AAC/G,CAAC,CAAC;KACD,OAAO,CAAC;IACL,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,qFAAqF,EAAE;IACvJ,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,0CAA0C,EAAE;IAC3F,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,gBAAgB,CAAC,EAAE,QAAQ,EAAE,wBAAwB,EAAE;IACpF,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;IAC3D,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,iCAAiC,EAAE;IACjG,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,4CAA4C,EAAE;IAChH,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,sGAAsG,EAAE;IACnK,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,wDAAwD,EAAE;IACxH,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,uBAAuB,EAAE;IAC7E,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gIAAgI,EAAE;IACtM,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,sDAAsD,EAAE;IAC7G,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,iDAAiD,EAAE;CAC3G,CAAC;KACD,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;KACb,SAAS,EAAE,CAAC;AAEjB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAa,CAAC;AACpC,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,QAAQ,CAAC,CAAC;AAC9B,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AAEnD,IAAI,QAAkB,CAAC;AAEvB,MAAM,aAAa,GAAuB,IAAI,CAAC,CAAC,CAAC;AACjD,IAAI,aAAa,EAAE;IACf,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAE3B,MAAM,YAAY,GAAG,IAAA,cAAO,EAAC,aAAa,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAA,iBAAY,EAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACrD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;CAEnC;KAAM;IACH,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,OAAO,CAAC,CAAC;CAClF;AAED,MAAM,SAAS,GAAuB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEjF,MAAM,IAAI,GAAuB,SAAS,CAAC,CAAC,CAAC,IAAA,cAAO,EAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAE5E,IAAI,IAAI,EAAE;IACN,sBAAsB;IACtB,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,QAAQ,SAAS,oEAAoE,CAAC,CAAC;KAC1G;CACJ;AAED,IAAI,OAAO,IAAI,CAAC,CAAC,KAAK,QAAQ,EAAE;IAC5B,wCAAwC;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC;IAC/B,IAAI,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE;QAClB,MAAM,IAAI,KAAK,CAAC,qFAAqF,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAA;KAC1H;CACJ;AAED,IAAI,OAA2B,CAAC;AAEhC,IAAI,IAAI,EAAE;IACN,MAAM,aAAa,GAAG,GAAG,SAAS,GAAG,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;IACzE,OAAO,GAAG,IAAA,cAAO,EAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACvC,IAAA,cAAS,EAAC,OAAO,CAAC,CAAC;CACtB;AAED,CAAC,GAAS,EAAE;IAER,MAAM,KAAK,GAAG,IAAI,4BAAkB,EAAE,CAAC;IACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEpC,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,2BAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;IAQhF,MAAM,OAAO,GAAwB,IAAI,GAAG,EAAE,CAAC;IAE/C,OAAO,CAAC,GAAG,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IAE7C,MAAM,OAAO,GAAG,iBAAO,CAAC,OAAO,CAAC;QAC5B,kBAAkB,EAAE,IAAI,CAAC,CAAC;QAC1B,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACxB,kBAAkB,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,EAAE,IAAI,CAAC,CAAC;QACf,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC,EAAE;QAC1E,KAAK;QACL,WAAW,EAAE,CAAO,IAAiB,EAAE,OAAuB,EAAE,QAAyB,EAAE,EAAE;YAEzF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;YAEhD,IAAI,OAAO,IAAI,CAAC,CAAC,KAAK,QAAQ,EAAE;gBAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,eAAe,YAAY,qCAAqC,CAAC,CAAC;gBAC9E,MAAM,IAAA,oBAAU,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;aACvC;YAED,mBAAmB;YACnB,qEAAqE;YACrE,gFAAgF;YAChF,kFAAkF;YAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAElC,wBAAwB;YACxB,IAAI,OAAO,EAAE;gBACT,MAAM,UAAU,GAAG,IAAI,SAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAExC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACnG,MAAM,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAC9H,MAAM,aAAa,GAAG,GAAG,SAAS,IAAI,gBAAgB,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAC7E,MAAM,QAAQ,GAAG,GAAG,aAAa,OAAO,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAE5C,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;gBACrC,IAAA,kBAAa,EAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAE9B,IAAI,IAAI,CAAC,CAAC,EAAE;oBAER,MAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;oBACtC,YAAY;oBACZ,MAAM,QAAQ,GAAuB;wBACjC,UAAU,EAAE;4BACR,WAAW,EAAE,IAAI;4BACjB,WAAW,EAAE,OAAO,CAAC,GAAG;4BACxB,gBAAgB,EAAE,IAAI;yBACzB;wBACD,KAAK,EAAE,KAAK;wBACZ,WAAW,EAAE,MAAM;qBACtB,CAAA;oBAED,MAAM,gBAAgB,GAAG,GAAG,QAAQ,gBAAgB,CAAC;oBACrD,MAAM,gBAAgB,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;oBAE5D,OAAO,CAAC,GAAG,CAAC,sBAAsB,gBAAgB,EAAE,CAAC,CAAC;oBACtD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;oBAC5D,IAAA,kBAAa,EAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;iBACjD;gBAED,IAAI,OAAO,IAAI,CAAC,CAAC,KAAK,QAAQ,EAAE;oBAC5B,MAAM,IAAI,GAAY,IAAI,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;oBAC1D,uBAAuB;oBACvB,MAAM,aAAa,GAAG,IAAI,mCAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAClE,MAAM,SAAS,GAAW,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC9C,MAAM,YAAY,GAAG,GAAG,aAAa,IAAI,SAAS,EAAE,CAAC;oBACrD,MAAM,gBAAgB,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,YAAY,CAAC,CAAC;oBACxD,OAAO,CAAC,GAAG,CAAC,sBAAsB,gBAAgB,EAAE,CAAC,CAAC;oBACtD,IAAA,kBAAa,EAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;iBAClD;aACJ;YAED,iDAAiD;YAEjD,MAAM,SAAS,GAAG,IAAI,+BAAa,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YACzD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;gBACtB,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,2BAA2B,CAAC,CAAC;gBAC5D,mBAAmB;gBACnB,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;oBAC3B,6BAA6B;oBAC7B,qCAAqC;oBACrC,MAAM,YAAY,GAAW,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpF,MAAM,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC;oBAC/C,IAAI,cAAc,IAAI,IAAA,0BAAQ,EAAC,cAAc,CAAC,EAAE;wBAC5C,MAAM,UAAU,GAAW,OAAO,cAAc,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC9F,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;4BACvB,WAAW;4BACX,MAAM,WAAW,GAAG,IAAI,mCAAiB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;4BAEhE,IAAI,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;4BAC9C,IAAI,cAAc,EAAE;gCAChB,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;6BAC/C;iCAAM;gCACH,cAAc,GAAG;oCACb,MAAM,EAAE,WAAW;oCACnB,SAAS,EAAE,CAAC,YAAY,CAAC;oCACzB,GAAG,EAAE,OAAO,CAAC,GAAG;iCACnB,CAAA;6BACJ;4BACD,eAAe;4BACf,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;yBAC5C;qBACJ;gBACL,CAAC,CAAC,CAAC;aACN;YACD,MAAM,IAAA,sBAAY,EAAC,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,kBAAkB,yBAAyB,IAAI,CAAC,eAAe,gBAAgB,IAAI,CAAC,cAAc,UAAU,CAAC,CAAC;QACnM,CAAC,CAAA;KACJ,CAAC,CAAC;IAEH,IAAI;QACA,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;KACvB;IAAC,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACpB;IAED,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,cAAc,UAAU,CAAC,CAAC;IAExH,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACnB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;YAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,aAAa,CAAC,GAAG,iBAAiB,aAAa,CAAC,eAAe,+BAA+B,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAA;QACnO,CAAC,CAAC,CAAA;KACL;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,IAAI,WAAW,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE;QAErB,iHAAiH;QACjH,MAAM,MAAM,GAAkB;YAC1B,eAAe,EAAE,CAAC;YAClB,cAAc,EAAE,EACf;SACJ,CAAC;QAEF,IAAI,KAAK,GAAW,2BAA2B,CAAC;QAEhD,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACvB,8CAA8C;YAC9C,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAClC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC;oBACrB,QAAQ,EAAE,QAAQ;oBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,UAAU,EAAE;wBACR,WAAW,EAAE,MAAM,CAAC,GAAG;qBAC1B;iBACJ,CAAC,CAAC;gBAEH,KAAK,IAAI,QAAQ,QAAQ,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,GAAG,GAAG,CAAC;YACpE,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAG,uBAAuB,CAAC;QACnD,MAAM,UAAU,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QAExD,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAChD,IAAA,kBAAa,EAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhE,MAAM,eAAe,GAAG,sBAAsB,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,aAAa,EAAE,CAAC,CAAC;QAClD,IAAA,kBAAa,EAAC,aAAa,EAAE,KAAK,CAAC,CAAC;KACvC;AACL,CAAC,CAAA,CAAC,EAAE,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,kCAAkC;AAClC,+BAA+B;AAC/B,OAAO,KAAK,MAAM,aAAa,CAAC;AAChC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,KAAK,EAAa,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAe,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,MAAM,EAAc,MAAM,aAAa,CAAC;AAEjD,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAEnD,aAAa,CAAC,OAAO,CACjB,aAAa,EACb,wBAAwB,EACxB,CAAC,KAAK,EAAE,EAAE;IACN,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC5G,KAAK,CAAC,OAAO,CAAC;QACV,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,qFAAqF,EAAE;QACvJ,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,0CAA0C,EAAE;QAC3F,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,gBAAgB,CAAC,EAAE,QAAQ,EAAE,wBAAwB,EAAE;QACpF,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;QAC3D,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,iCAAiC,EAAE;QACjG,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,4CAA4C,EAAE;QAChH,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,sGAAsG,EAAE;QACnK,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,wDAAwD,EAAE;QACxH,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,uBAAuB,EAAE;QAC7E,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gIAAgI,EAAE;QACtM,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,sDAAsD,EAAE;QAC7G,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,iDAAiD,EAAE;KAC3G,CAAC,CAAA;AACN,CAAC,EACD,KAAK,EAAE,IAAI,EAAE,EAAE;IACX,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC,CACJ,CAAC;AAEF,aAAa,CAAC,OAAO,CACjB,eAAe,EACf,0CAA0C,EAC1C,CAAC,KAAK,EAAE,EAAE;IACN,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC5G,KAAK,CAAC,OAAO,CAAC;QACV,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;QAC3D,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;KAC7D,CAAC,CAAA;AACN,CAAC,EACD,KAAK,EAAE,IAAI,EAAE,EAAE;IACX,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC,CACJ,CAAC;AAEF,aAAa,CAAC,OAAO,CACjB,QAAQ,EACR,oBAAoB,EACpB,CAAC,KAAK,EAAE,EAAE;IACN,OAAO,KAAK;SACP,MAAM,CAAC,KAAK,EAAE;QACX,QAAQ,EAAE,kCAAkC;QAC5C,IAAI,EAAE,QAAQ;QACd,YAAY,EAAE,IAAI,EAAE,iCAAiC;QACrD,KAAK,EAAE,GAAG,CAAC,6CAA6C;KAC3D,CAAC,CAAC;AACX,CAAC,EACD,KAAK,EAAE,IAAI,EAAE,EAAE;IACX,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;AACvB,CAAC,CACJ,CAAC;AAEF,aAAa,CAAC,KAAK,EAAE,CAAC"}

package/lib/utils/index.d.ts

@@ -0,0 +1,2 @@
+/*! Copyright (c) 2023, XAPP AI */
+export declare function createClickableLink(url: string, linkText: string): string;

package/lib/utils/index.js

@@ -0,0 +1,5 @@
+/*! Copyright (c) 2023, XAPP AI */
+export function createClickableLink(url, linkText) {
+    return `\x1b]8;;${url}\a${linkText}\x1b]8;;\a`;
+}
+//# sourceMappingURL=index.js.map

package/lib/utils/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAElC,MAAM,UAAU,mBAAmB,CAAC,GAAW,EAAE,QAAgB;IAC7D,OAAO,WAAW,GAAG,KAAK,QAAQ,YAAY,CAAC;AACnD,CAAC"}

package/package.json

@@ -1,9 +1,10 @@
 {
     "name": "@xapp/arachne-cli",
     "preferGlobal": true,
-    "version": "0.9.1",
+    "version": "1.1.0",
     "types": "lib/index",
     "main": "lib/index",
+    "type": "module",
     "files": [
         "lib"
     ],
@@ -18,27 +19,31 @@
         "node": "^14 || ^16 || ^18"
     },
     "devDependencies": {
-        "@microsoft/api-extractor": "7.36.4",
-        "@types/chai": "4.3.5",
+        "@microsoft/api-extractor": "7.39.1",
+        "@types/chai": "4.3.11",
         "@types/generic-pool": "3.1.11",
-        "@types/mocha": "10.0.1",
-        "@types/node": "18.17.9",
-        "@types/sinon": "10.0.16",
-        "@types/sinon-chai": "3.2.9",
-        "@types/yargs": "17.0.24",
+        "@types/mocha": "10.0.6",
+        "@types/node": "18.19.4",
+        "@types/sinon": "10.0.20",
+        "@types/sinon-chai": "3.2.12",
+        "@types/yargs": "17.0.32",
         "@xapp/config": "0.2.3",
-        "chai": "4.3.8",
+        "chai": "4.3.10",
         "mocha": "10.2.0",
         "sinon": "15.2.0",
         "sinon-chai": "3.7.0",
-        "ts-node": "10.9.1",
+        "ts-node": "10.9.2",
         "tslib": "2.6.2",
-        "typescript": "5.1.6"
+        "typescript": "5.3.3"
     },
     "dependencies": {
-        "@xapp/arachne": "0.9.1",
-        "@xapp/arachne-parsers": "0.9.0",
-        "puppeteer": "21.1.1",
+        "@xapp/arachne": "1.1.0",
+        "@xapp/arachne-parsers": "1.1.0",
+        "@xapp/arachne-web-service": "1.1.0",
+        "isomorphic-fetch": "3.0.0",
+        "puppeteer": "21.6.1",
+        "terminal-image": "2.0.0",
+        "terminal-link": "3.0.0",
         "yargs": "17.7.2"
     },
     "scripts": {
@@ -46,5 +51,5 @@
         "build": "tsc -d true -p .",
         "clean": "rm -rf ./lib/*"
     },
-    "gitHead": "b0d58f73fb5461a0ed43d2dd659b3ae3c1ed6f19"
+    "gitHead": "caa638425218d19c06de48d2b3cad2d78e3fca67"
 }