@xapp/arachne-cli 0.0.2 → 0.4.0

package/LICENSE.md

@@ -0,0 +1,176 @@
+                               Apache License
+                         Version 2.0, January 2004
+                      http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+    "License" shall mean the terms and conditions for use, reproduction,
+    and distribution as defined by Sections 1 through 9 of this document.
+
+    "Licensor" shall mean the copyright owner or entity authorized by
+    the copyright owner that is granting the License.
+
+    "Legal Entity" shall mean the union of the acting entity and all
+    other entities that control, are controlled by, or are under common
+    control with that entity. For the purposes of this definition,
+    "control" means (i) the power, direct or indirect, to cause the
+    direction or management of such entity, whether by contract or
+    otherwise, or (ii) ownership of fifty percent (50%) or more of the
+    outstanding shares, or (iii) beneficial ownership of such entity.
+
+    "You" (or "Your") shall mean an individual or Legal Entity
+    exercising permissions granted by this License.
+
+    "Source" form shall mean the preferred form for making modifications,
+    including but not limited to software source code, documentation
+    source, and configuration files.
+
+    "Object" form shall mean any form resulting from mechanical
+    transformation or translation of a Source form, including but
+    not limited to compiled object code, generated documentation,
+    and conversions to other media types.
+
+    "Work" shall mean the work of authorship, whether in Source or
+    Object form, made available under the License, as indicated by a
+    copyright notice that is included in or attached to the work
+    (an example is provided in the Appendix below).
+
+    "Derivative Works" shall mean any work, whether in Source or Object
+    form, that is based on (or derived from) the Work and for which the
+    editorial revisions, annotations, elaborations, or other modifications
+    represent, as a whole, an original work of authorship. For the purposes
+    of this License, Derivative Works shall not include works that remain
+    separable from, or merely link (or bind by name) to the interfaces of,
+    the Work and Derivative Works thereof.
+
+    "Contribution" shall mean any work of authorship, including
+    the original version of the Work and any modifications or additions
+    to that Work or Derivative Works thereof, that is intentionally
+    submitted to Licensor for inclusion in the Work by the copyright owner
+    or by an individual or Legal Entity authorized to submit on behalf of
+    the copyright owner. For the purposes of this definition, "submitted"
+    means any form of electronic, verbal, or written communication sent
+    to the Licensor or its representatives, including but not limited to
+    communication on electronic mailing lists, source code control systems,
+    and issue tracking systems that are managed by, or on behalf of, the
+    Licensor for the purpose of discussing and improving the Work, but
+    excluding communication that is conspicuously marked or otherwise
+    designated in writing by the copyright owner as "Not a Contribution."
+
+    "Contributor" shall mean Licensor and any individual or Legal Entity
+    on behalf of whom a Contribution has been received by Licensor and
+    subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    copyright license to reproduce, prepare Derivative Works of,
+    publicly display, publicly perform, sublicense, and distribute the
+    Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    (except as stated in this section) patent license to make, have made,
+    use, offer to sell, sell, import, and otherwise transfer the Work,
+    where such license applies only to those patent claims licensable
+    by such Contributor that are necessarily infringed by their
+    Contribution(s) alone or by combination of their Contribution(s)
+    with the Work to which such Contribution(s) was submitted. If You
+    institute patent litigation against any entity (including a
+    cross-claim or counterclaim in a lawsuit) alleging that the Work
+    or a Contribution incorporated within the Work constitutes direct
+    or contributory patent infringement, then any patent licenses
+    granted to You under this License for that Work shall terminate
+    as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+    Work or Derivative Works thereof in any medium, with or without
+    modifications, and in Source or Object form, provided that You
+    meet the following conditions:
+
+    (a) You must give any other recipients of the Work or
+        Derivative Works a copy of this License; and
+
+    (b) You must cause any modified files to carry prominent notices
+        stating that You changed the files; and
+
+    (c) You must retain, in the Source form of any Derivative Works
+        that You distribute, all copyright, patent, trademark, and
+        attribution notices from the Source form of the Work,
+        excluding those notices that do not pertain to any part of
+        the Derivative Works; and
+
+    (d) If the Work includes a "NOTICE" text file as part of its
+        distribution, then any Derivative Works that You distribute must
+        include a readable copy of the attribution notices contained
+        within such NOTICE file, excluding those notices that do not
+        pertain to any part of the Derivative Works, in at least one
+        of the following places: within a NOTICE text file distributed
+        as part of the Derivative Works; within the Source form or
+        documentation, if provided along with the Derivative Works; or,
+        within a display generated by the Derivative Works, if and
+        wherever such third-party notices normally appear. The contents
+        of the NOTICE file are for informational purposes only and
+        do not modify the License. You may add Your own attribution
+        notices within Derivative Works that You distribute, alongside
+        or as an addendum to the NOTICE text from the Work, provided
+        that such additional attribution notices cannot be construed
+        as modifying the License.
+
+    You may add Your own copyright statement to Your modifications and
+    may provide additional or different license terms and conditions
+    for use, reproduction, or distribution of Your modifications, or
+    for any such Derivative Works as a whole, provided Your use,
+    reproduction, and distribution of the Work otherwise complies with
+    the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+    any Contribution intentionally submitted for inclusion in the Work
+    by You to the Licensor shall be under the terms and conditions of
+    this License, without any additional terms or conditions.
+    Notwithstanding the above, nothing herein shall supersede or modify
+    the terms of any separate license agreement you may have executed
+    with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+    names, trademarks, service marks, or product names of the Licensor,
+    except as required for reasonable and customary use in describing the
+    origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+    agreed to in writing, Licensor provides the Work (and each
+    Contributor provides its Contributions) on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+    implied, including, without limitation, any warranties or conditions
+    of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+    PARTICULAR PURPOSE. You are solely responsible for determining the
+    appropriateness of using or redistributing the Work and assume any
+    risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+    whether in tort (including negligence), contract, or otherwise,
+    unless required by applicable law (such as deliberate and grossly
+    negligent acts) or agreed to in writing, shall any Contributor be
+    liable to You for damages, including any direct, indirect, special,
+    incidental, or consequential damages of any character arising as a
+    result of this License or out of the use or inability to use the
+    Work (including but not limited to damages for loss of goodwill,
+    work stoppage, computer failure or malfunction, or any and all
+    other commercial damages or losses), even if such Contributor
+    has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+    the Work or Derivative Works thereof, You may choose to offer,
+    and charge a fee for, acceptance of support, warranty, indemnity,
+    or other liability obligations and/or rights consistent with this
+    License. However, in accepting such obligations, You may act only
+    on Your own behalf and on Your sole responsibility, not on behalf
+    of any other Contributor, and only if You agree to indemnify,
+    defend, and hold each Contributor harmless for any liability
+    incurred by, or claims asserted against, such Contributor by reason
+    of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS

package/README.md

@@ -0,0 +1,29 @@
+## @xapp/arachne-cli
+
+### Example Usage
+
+To crawl a site and save the pages to a local ./temp directory
+```
+node lib crawl http://www.thecoffeefaq.com/ -d ./temp
+```
+
+To also save markdown and schema.org FAQs
+```
+node lib crawl http://www.thecoffeefaq.com/ -a -t -d ./temp
+```
+
+With a whitelisted patterns file
+```
+node lib crawl http://www.thecoffeefaq.com/ -a -t -d ./temp -w ./temp/whitelist.md
+```
+
+### Windows & WSL2 Notes
+
+Follow the instructions here to setup: https://github.com/puppeteer/puppeteer/issues/1837#issuecomment-689006806
+
+You __will__ need to start XLaunch before running the CLI, select multiple windows, no client, turn off access control.
+
+If the normal commands don't work, you might need to pass in the executablePath (-e) and run headless (-h).
+```
+node lib crawl http://www.thecoffeefaq.com/ -e  /usr/bin/google-chrome -h
+```

package/lib/index.d.ts

@@ -1 +1,2 @@
+#!/usr/bin/env node
 export {};

package/lib/index.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 "use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
@@ -8,27 +9,209 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+/*! Copyright (c) 2020, XAPP AI */
+/* eslint-disable no-console */
 const arachne_1 = require("@xapp/arachne");
+const arachne_parsers_1 = require("@xapp/arachne-parsers");
+const url_1 = require("url");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const yargs_1 = __importDefault(require("yargs/yargs"));
+const helpers_1 = require("yargs/helpers");
+const argv = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
+    .command('crawl <url>', 'Crawl the provided URL', (yargs) => {
+    yargs.positional("url", { type: 'string', alias: ["u"], demandOption: true, describe: "The URL to crawl" });
+})
+    .options({
+    a: { type: 'boolean', alias: ["faq"], default: false, describe: "Generate a JSON file with FAQs, AWS Kendra format, found in JSON+LD while crawling." },
+    d: { type: 'string', alias: ["dir"], describe: "The directory to save the HTML source to" },
+    e: { type: 'string', alias: ["executablePath"], describe: "Executable path Chrome" },
+    h: { type: 'boolean', alias: ["headless"], default: false },
+    m: { type: 'number', alias: ["max"], default: 1000, describe: "The max number of total crawls." },
+    o: { type: 'number', alias: ["timeout"], default: 4000, describe: "The timeout for each individual page load." },
+    p: { type: 'number', alias: ["pages"], default: 10, describe: "The max number of pages (tabs) running simultaneously." },
+    s: { type: 'boolean', alias: ["stealth"], describe: "Turns on stealth mode" },
+    t: { type: 'boolean', alias: ["markdown", "text"], default: false, describe: "When a directory is provided, this will also save a markdown version of the HTML." },
+    w: { type: 'string', alias: ["whitelist"], describe: "A text file where each line is a whitelisted pattern" },
+    x: { type: 'boolean', alias: ["metadata"], describe: "Generate a JSON metadata file for Amazon Kendra" }
+})
+    .nargs('u', 1)
+    .parseSync();
+argv;
+const inputURL = argv.url;
+const url = new url_1.URL(inputURL);
+const cleanHost = url.host.replace(/\./g, "_");
+const cleanPath = url.pathname.replace(/\//g, "_");
+let patterns;
+const whiteListPath = argv.w;
+if (whiteListPath) {
+    console.log(whiteListPath);
+    const resolvedPath = (0, path_1.resolve)(whiteListPath);
+    const contents = (0, fs_1.readFileSync)(resolvedPath, "utf-8");
+    patterns = contents.split("\n");
+}
+else {
+    patterns = inputURL.endsWith("/") ? [`${inputURL}[.*]`] : [`${inputURL}/[.*]`];
+}
+const directory = Array.isArray(argv.d) ? argv.d[1] : argv.d;
+const path = directory ? (0, path_1.resolve)(directory) : undefined;
+if (path) {
+    // Make sure it exists
+    if (!(0, fs_1.existsSync)(path)) {
+        throw new Error(`Path ${directory} does not exist.  Please provide an existing path to save the HTML`);
+    }
+}
+let saveDir;
+if (path) {
+    const exportDirName = `${cleanHost}${cleanPath}-${new Date().getTime()}`;
+    saveDir = (0, path_1.resolve)(path, exportDirName);
+    (0, fs_1.mkdirSync)(saveDir);
+}
 (() => __awaiter(void 0, void 0, void 0, function* () {
     const queue = new arachne_1.MemoryRequestQueue();
-    queue.addRequest({ url: 'https://www.dhs.gov/how-do-i/' });
-    const patterns = [new arachne_1.ArachneURLPattern('https://www.dhs.gov/how-do-i/[.*]')];
+    queue.addRequest({ url: inputURL });
+    const crawlPatterns = patterns.map((pattern) => new arachne_1.ArachneURLPattern(pattern));
+    const answers = new Map();
+    console.log(`Starting crawl at ${inputURL}`);
     const crawler = arachne_1.Arachne.crawler({
-        maxConcurrentPages: 10,
-        maxCrawlAttempts: 1000,
-        launchOptions: { headless: false },
+        maxConcurrentPages: argv.p,
+        maxCrawlAttempts: argv.m,
+        pageHandlerTimeout: argv.o,
+        stealth: argv.s,
+        launchOptions: { headless: argv.h, timeout: 5000, executablePath: argv.e },
         queue,
-        pageHandler: (request, page) => __awaiter(void 0, void 0, void 0, function* () {
+        pageHandler: (page, request, response) => __awaiter(void 0, void 0, void 0, function* () {
             const title = yield page.title();
-            console.log(`Loaded ${request.url}: ${title}`);
-            yield arachne_1.enqueueLinks(page, queue, patterns);
+            console.log(`Loaded ${request.url} : ${title}`);
+            // Get the raw HTML
+            // It is important to use page.content() versus response.text() since
+            // page.content() gets you the HTML after it has been modified by the JavaScript
+            // on the page while response.text() only gives you the HTML from the original GET
+            const text = yield page.content();
+            // Pull out the raw html
+            if (saveDir) {
+                const requestURL = new url_1.URL(request.url);
+                const cleanHost = requestURL.host.replace(/(\.|\/)/g, "_");
+                const cleanRequestPath = requestURL.pathname.replace(/\//g, "_").replace(".html", "");
+                const htmlName = `${cleanHost}_${cleanRequestPath}.html`;
+                console.log(`File Name: ${htmlName}`);
+                const filePath = (0, path_1.resolve)(saveDir, htmlName);
+                console.log(`Saving to ${filePath}`);
+                (0, fs_1.writeFileSync)(filePath, text);
+                if (argv.x) {
+                    const date = new Date().toISOString();
+                    // Doc title
+                    const metadata = {
+                        Attributes: {
+                            _created_at: date,
+                            _source_uri: request.url,
+                            _last_updated_at: date
+                        },
+                        Title: title,
+                        ContentType: "HTML"
+                    };
+                    const metadataFileName = `${htmlName}.metadata.json`;
+                    const metadataFilePath = (0, path_1.resolve)(saveDir, metadataFileName);
+                    console.log(`Saving metadata to ${metadataFilePath}`);
+                    const metadataText = JSON.stringify(metadata, undefined, 2);
+                    (0, fs_1.writeFileSync)(metadataFilePath, metadataText);
+                }
+                if (argv.t) {
+                    // Write some MARKDOWN!
+                    const markdown = new arachne_parsers_1.ArachneHTMLtoText().convert(text, true);
+                    const markdownName = `${cleanRequestPath}.md`;
+                    const markdownFilePath = (0, path_1.resolve)(saveDir, markdownName);
+                    console.log(`Saving markdown to ${markdownFilePath}`);
+                    (0, fs_1.writeFileSync)(markdownFilePath, markdown);
+                }
+            }
+            // Let's see if we can find some structured data.
+            const questions = new arachne_parsers_1.ArachneJSONLD(text).getQuestions();
+            if (questions.length > 0) {
+                console.log(`${questions.length} questions found on page.`);
+                // Clean the answer
+                questions.forEach((question) => {
+                    // name has the question text
+                    // acceptedAnswer has the answer text
+                    const questionText = typeof question.name === "string" ? question.name : "";
+                    const acceptedAnswer = question.acceptedAnswer;
+                    if (acceptedAnswer && (0, arachne_parsers_1.isAnswer)(acceptedAnswer)) {
+                        const answerText = typeof acceptedAnswer.text === "string" ? acceptedAnswer.text : "";
+                        if (answerText.length > 0) {
+                            // Clean it
+                            const cleanAnswer = new arachne_parsers_1.ArachneHTMLtoText().convert(answerText);
+                            let existingAnswer = answers.get(cleanAnswer);
+                            if (existingAnswer) {
+                                existingAnswer.questions.push(questionText);
+                            }
+                            else {
+                                existingAnswer = {
+                                    answer: cleanAnswer,
+                                    questions: [questionText],
+                                    url: request.url
+                                };
+                            }
+                            // Add it back!
+                            answers.set(cleanAnswer, existingAnswer);
+                        }
+                    }
+                });
+            }
+            yield (0, arachne_1.enqueueLinks)(page, queue, crawlPatterns);
             const info = yield queue.getInfo();
-            console.log(`Completed ${info.completeRequests} of ${info.totalRequests}`);
+            console.log(`Completed ${info.completeRequests} of ${info.totalRequests}, ${info.inProgressRequests} are in progress with ${info.pendingRequests} pending and ${info.failedRequests} failed.`);
         })
     });
-    yield crawler.run();
+    try {
+        yield crawler.run();
+    }
+    catch (e) {
+        console.log('Unable to start the crawler.');
+        console.error(e);
+    }
+    console.log('Crawler complete.');
     const info = yield queue.getInfo();
-    console.log(`Final count, completed ${info.completeRequests} of ${info.totalRequests}`);
+    console.log(`Final count, completed ${info.completeRequests} of ${info.totalRequests}, ${info.failedRequests} failed.`);
+    const failed = queue.getFailed();
+    if (failed.length > 0) {
+        console.log('Failures....');
+        failed.forEach((failedRequest) => {
+            console.log(`\t${failedRequest.url} failed after ${failedRequest.requestAttempts} attempts with final error: ${failedRequest.errorMessages ? failedRequest.errorMessages[failedRequest.errorMessages.length - 1] : "unknown"}`);
+        });
+    }
+    console.log(`Found ${answers.size} answers.`);
+    if (argv.faq && saveDir) {
+        // This follows the JSON format https://docs.aws.amazon.com/kendra/latest/dg/in-creating-faq.html#faq-custom-json
+        const qaJSON = {
+            "SchemaVersion": 1,
+            "FaqDocuments": []
+        };
+        let qaCSV = `"Question","Answer","URL"`;
+        answers.forEach((answer) => {
+            // For the questions on the answer, add a line
+            answer.questions.forEach((question) => {
+                qaJSON.FaqDocuments.push({
+                    Question: question,
+                    Answer: answer.answer,
+                    Attributes: {
+                        _source_uri: answer.url
+                    }
+                });
+                qaCSV += `\r\n"${question}","${answer.answer}","${answer.url}"`;
+            });
+        });
+        const questionAnswerName = `question-answers.json`;
+        const qaFilePath = (0, path_1.resolve)(saveDir, questionAnswerName);
+        console.log(`Saving FAQ JSON to ${qaFilePath}`);
+        (0, fs_1.writeFileSync)(qaFilePath, JSON.stringify(qaJSON, undefined, 2));
+        const csvQuestionName = `question-answers.csv`;
+        const csvQaFilePath = (0, path_1.resolve)(saveDir, csvQuestionName);
+        console.log(`Saving FAQ CSV to ${csvQaFilePath}`);
+        (0, fs_1.writeFileSync)(csvQaFilePath, qaCSV);
+    }
 }))();
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;AAAA,2CAA0H;AAE1H,CAAC,GAAS,EAAE;IAER,MAAM,KAAK,GAAG,IAAI,4BAAkB,EAAE,CAAC;IACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,+BAA+B,EAAE,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,CAAC,IAAI,2BAAiB,CAAC,mCAAmC,CAAC,CAAC,CAAC;IAE9E,MAAM,OAAO,GAAG,iBAAO,CAAC,OAAO,CAAC;QAC5B,kBAAkB,EAAE,EAAE;QACtB,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;QAClC,KAAK;QACL,WAAW,EAAE,CAAO,OAAuB,EAAE,IAAiB,EAAG,EAAE;YAE/D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC;YAE/C,MAAM,sBAAY,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YAE1C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;QAC9E,CAAC,CAAA;KACJ,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;IAEpB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;AAC3F,CAAC,CAAA,CAAC,EAAE,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,kCAAkC;AAClC,+BAA+B;AAC/B,2CAQuB;AAEvB,2DAAmF;AAEnF,6BAA0B;AAC1B,2BAAwE;AACxE,+BAA+B;AAE/B,wDAAgC;AAChC,2CAAuC;AAqCvC,MAAM,IAAI,GAAG,IAAA,eAAK,EAAC,IAAA,iBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;KACpC,OAAO,CAAC,aAAa,EAAE,wBAAwB,EAAE,CAAC,KAAK,EAAE,EAAE;IACxD,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAA;AAC/G,CAAC,CAAC;KACD,OAAO,CAAC;IACL,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,qFAAqF,EAAE;IACvJ,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,0CAA0C,EAAE;IAC3F,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,gBAAgB,CAAC,EAAE,QAAQ,EAAE,wBAAwB,EAAE;IACpF,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;IAC3D,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,iCAAiC,EAAE;IACjG,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,4CAA4C,EAAE;IAChH,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,wDAAwD,EAAE;IACxH,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,uBAAuB,EAAE;IAC7E,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,mFAAmF,EAAE;IAClK,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,sDAAsD,EAAE;IAC7G,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,iDAAiD,EAAE;CAC3G,CAAC;KACD,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;KACb,SAAS,EAAE,CAAC;AAEjB,IAAI,CAAA;AAEJ,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAa,CAAC;AACpC,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,QAAQ,CAAC,CAAC;AAC9B,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AAEnD,IAAI,QAAkB,CAAC;AAEvB,MAAM,aAAa,GAAuB,IAAI,CAAC,CAAC,CAAC;AACjD,IAAI,aAAa,EAAE;IACf,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAE3B,MAAM,YAAY,GAAG,IAAA,cAAO,EAAC,aAAa,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAA,iBAAY,EAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACrD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;CAEnC;KAAM;IACH,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,OAAO,CAAC,CAAC;CAClF;AAED,MAAM,SAAS,GAAuB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEjF,MAAM,IAAI,GAAuB,SAAS,CAAC,CAAC,CAAC,IAAA,cAAO,EAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAE5E,IAAI,IAAI,EAAE;IACN,sBAAsB;IACtB,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,QAAQ,SAAS,oEAAoE,CAAC,CAAC;KAC1G;CACJ;AAED,IAAI,OAA2B,CAAC;AAEhC,IAAI,IAAI,EAAE;IACN,MAAM,aAAa,GAAG,GAAG,SAAS,GAAG,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;IACzE,OAAO,GAAG,IAAA,cAAO,EAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACvC,IAAA,cAAS,EAAC,OAAO,CAAC,CAAC;CACtB;AAED,CAAC,GAAS,EAAE;IAER,MAAM,KAAK,GAAG,IAAI,4BAAkB,EAAE,CAAC;IACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEpC,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,2BAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;IAQhF,MAAM,OAAO,GAAwB,IAAI,GAAG,EAAE,CAAC;IAE/C,OAAO,CAAC,GAAG,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IAE7C,MAAM,OAAO,GAAG,iBAAO,CAAC,OAAO,CAAC;QAC5B,kBAAkB,EAAE,IAAI,CAAC,CAAC;QAC1B,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACxB,kBAAkB,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,EAAE,IAAI,CAAC,CAAC;QACf,aAAa,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC,EAAE;QAC1E,KAAK;QACL,WAAW,EAAE,CAAO,IAAiB,EAAE,OAAuB,EAAE,QAAyB,EAAE,EAAE;YAEzF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;YAEhD,mBAAmB;YACnB,qEAAqE;YACrE,gFAAgF;YAChF,kFAAkF;YAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAElC,wBAAwB;YACxB,IAAI,OAAO,EAAE;gBACT,MAAM,UAAU,GAAG,IAAI,SAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;gBAC1D,MAAM,gBAAgB,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,gBAAgB,OAAO,CAAC;gBACzD,OAAO,CAAC,GAAG,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAE5C,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;gBACrC,IAAA,kBAAa,EAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAE9B,IAAI,IAAI,CAAC,CAAC,EAAE;oBAER,MAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;oBACtC,YAAY;oBACZ,MAAM,QAAQ,GAAuB;wBACjC,UAAU,EAAE;4BACR,WAAW,EAAE,IAAI;4BACjB,WAAW,EAAE,OAAO,CAAC,GAAG;4BACxB,gBAAgB,EAAE,IAAI;yBACzB;wBACD,KAAK,EAAE,KAAK;wBACZ,WAAW,EAAE,MAAM;qBACtB,CAAA;oBAED,MAAM,gBAAgB,GAAG,GAAG,QAAQ,gBAAgB,CAAC;oBACrD,MAAM,gBAAgB,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;oBAE5D,OAAO,CAAC,GAAG,CAAC,sBAAsB,gBAAgB,EAAE,CAAC,CAAC;oBACtD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;oBAC5D,IAAA,kBAAa,EAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;iBACjD;gBAED,IAAI,IAAI,CAAC,CAAC,EAAE;oBACR,uBAAuB;oBACvB,MAAM,QAAQ,GAAG,IAAI,mCAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC7D,MAAM,YAAY,GAAG,GAAG,gBAAgB,KAAK,CAAC;oBAC9C,MAAM,gBAAgB,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,YAAY,CAAC,CAAC;oBACxD,OAAO,CAAC,GAAG,CAAC,sBAAsB,gBAAgB,EAAE,CAAC,CAAC;oBACtD,IAAA,kBAAa,EAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;iBAC7C;aACJ;YAED,iDAAiD;YAEjD,MAAM,SAAS,GAAG,IAAI,+BAAa,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YACzD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;gBACtB,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,2BAA2B,CAAC,CAAC;gBAC5D,mBAAmB;gBACnB,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;oBAC3B,6BAA6B;oBAC7B,qCAAqC;oBACrC,MAAM,YAAY,GAAW,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpF,MAAM,cAAc,GAAG,QAAQ,CAAC,cAAc,CAAC;oBAC/C,IAAI,cAAc,IAAI,IAAA,0BAAQ,EAAC,cAAc,CAAC,EAAE;wBAC5C,MAAM,UAAU,GAAW,OAAO,cAAc,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC9F,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;4BACvB,WAAW;4BACX,MAAM,WAAW,GAAG,IAAI,mCAAiB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;4BAEhE,IAAI,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;4BAC9C,IAAI,cAAc,EAAE;gCAChB,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;6BAC/C;iCAAM;gCACH,cAAc,GAAG;oCACb,MAAM,EAAE,WAAW;oCACnB,SAAS,EAAE,CAAC,YAAY,CAAC;oCACzB,GAAG,EAAE,OAAO,CAAC,GAAG;iCACnB,CAAA;6BACJ;4BACD,eAAe;4BACf,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;yBAC5C;qBACJ;gBACL,CAAC,CAAC,CAAC;aACN;YACD,MAAM,IAAA,sBAAY,EAAC,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,kBAAkB,yBAAyB,IAAI,CAAC,eAAe,gBAAgB,IAAI,CAAC,cAAc,UAAU,CAAC,CAAA;QAClM,CAAC,CAAA;KACJ,CAAC,CAAC;IAEH,IAAI;QACA,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;KACvB;IAAC,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACpB;IAED,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,gBAAgB,OAAO,IAAI,CAAC,aAAa,KAAK,IAAI,CAAC,cAAc,UAAU,CAAC,CAAC;IAExH,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;IACjC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACnB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE;YAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,aAAa,CAAC,GAAG,iBAAiB,aAAa,CAAC,eAAe,+BAA+B,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAA;QACnO,CAAC,CAAC,CAAA;KACL;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,OAAO,CAAC,IAAI,WAAW,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE;QAErB,iHAAiH;QACjH,MAAM,MAAM,GAAkB;YAC1B,eAAe,EAAE,CAAC;YAClB,cAAc,EAAE,EACf;SACJ,CAAC;QAEF,IAAI,KAAK,GAAW,2BAA2B,CAAC;QAEhD,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACvB,8CAA8C;YAC9C,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAClC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC;oBACrB,QAAQ,EAAE,QAAQ;oBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,UAAU,EAAE;wBACR,WAAW,EAAE,MAAM,CAAC,GAAG;qBAC1B;iBACJ,CAAC,CAAC;gBAEH,KAAK,IAAI,QAAQ,QAAQ,MAAM,MAAM,CAAC,MAAM,MAAM,MAAM,CAAC,GAAG,GAAG,CAAC;YACpE,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAG,uBAAuB,CAAC;QACnD,MAAM,UAAU,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QAExD,OAAO,CAAC,GAAG,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAChD,IAAA,kBAAa,EAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhE,MAAM,eAAe,GAAG,sBAAsB,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAA,cAAO,EAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,qBAAqB,aAAa,EAAE,CAAC,CAAC;QAClD,IAAA,kBAAa,EAAC,aAAa,EAAE,KAAK,CAAC,CAAC;KACvC;AACL,CAAC,CAAA,CAAC,EAAE,CAAC"}

package/package.json

@@ -1,44 +1,51 @@
 {
     "name": "@xapp/arachne-cli",
-    "version": "0.0.2",
+    "preferGlobal": true,
+    "version": "0.4.0",
     "types": "lib/index",
     "main": "lib/index",
     "files": [
         "lib"
     ],
+    "bin": {
+        "arachne": "lib/index"
+    },
+    "license": "Apache-2.0",
     "private": false,
     "publishConfig": {
         "access": "public"
     },
     "engines": {
-        "node": "10.x"
+        "node": "^10 || ^12 || ^14"
     },
     "devDependencies": {
-        "@microsoft/api-extractor": "7.9.15",
-        "@types/chai": "4.2.12",
-        "@types/generic-pool": "3.1.2",
-        "@types/mocha": "8.0.3",
-        "@types/node": "13.13.16",
-        "@types/puppeteer": "1.3.0",
-        "@types/sinon": "9.0.5",
-        "@types/sinon-chai": "3.2.4",
+        "@microsoft/api-extractor": "7.22.2",
+        "@types/chai": "4.3.1",
+        "@types/generic-pool": "3.1.10",
+        "@types/mocha": "9.1.1",
+        "@types/node": "17.0.25",
+        "@types/puppeteer": "5.4.5",
+        "@types/sinon": "10.0.11",
+        "@types/sinon-chai": "3.2.8",
+        "@types/yargs": "17.0.10",
         "@xapp/config": "0.2.3",
-        "chai": "4.2.0",
-        "mocha": "8.1.3",
-        "sinon": "9.0.3",
-        "sinon-chai": "3.5.0",
-        "ts-node": "9.0.0",
-        "tslib": "2.0.1",
-        "typescript": "4.0.2"
+        "chai": "4.3.6",
+        "mocha": "9.2.2",
+        "sinon": "13.0.2",
+        "sinon-chai": "3.7.0",
+        "ts-node": "10.7.0",
+        "tslib": "2.4.0",
+        "typescript": "4.6.3"
     },
     "dependencies": {
-        "@xapp/arachne": "^0.0.2"
+        "@xapp/arachne": "0.4.0",
+        "@xapp/arachne-parsers": "0.4.0",
+        "yargs": "17.4.1"
     },
     "scripts": {
         "api": "api-extractor run --local",
         "build": "tsc -d true -p .",
-        "clean": "rm -rf ./lib/*",
-        "test": "mocha --recursive -r ts-node/register \"./src/**/*.test.ts\""
+        "clean": "rm -rf ./lib/*"
     },
-    "gitHead": "ce4c30928679f5c84dba0dee2998a5cb13e904fd"
+    "gitHead": "5fb12fa2701e8a62a892c3c1a3d6865eb5aadcc5"
 }