pagerts 0.1.9 → 0.2.0

package/bin/package.json

@@ -0,0 +1,40 @@
+{
+    "name": "pagerts",
+    "description": "A tool for viewing external relations in a webpage",
+    "version": "0.1.9",
+    "main": "main.js",
+    "bin": {
+        "pagerts": "bin/main.js"
+    },
+    "scripts": {
+        "test": "jest",
+        "build": "esbuild src/main.ts --external:jsdom --bundle --outdir=bin --minify --sourcemap --platform=node",
+        "lint": "tsc",
+        "start": "node ./bin/main.js",
+        "dev": "npx tsx src/main.ts"
+    },
+    "keywords": [
+        "webpage",
+        "hierarchy",
+        "management"
+    ],
+    "author": "Kirill kn253 Nevzorov",
+    "license": "MIT",
+    "bugs": {
+        "url": "https://github.com/akinevz0/pagerts/issues"
+    },
+    "homepage": "https://github.com/akinevz0/pagerts",
+    "dependencies": {
+        "blessed": "^0.1.81",
+        "commander": "^12.1.0",
+        "dotenv": "^16.4.5",
+        "jsdom": "^26.0.0"
+    },
+    "devDependencies": {
+        "@types/blessed": "^0.1.25",
+        "@types/jsdom": "^21.1.7",
+        "@types/node": "^22.8.2",
+        "esbuild": "^0.25.1",
+        "ts-node": "^10.9.2"
+    }
+}

package/bin/src/extractors/AbstractExtractor.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractExtractor = void 0;
+class AbstractExtractor {
+    name;
+    constructor(name) {
+        this.name = name;
+    }
+}
+exports.AbstractExtractor = AbstractExtractor;
+//# sourceMappingURL=AbstractExtractor.js.map

package/bin/src/extractors/AbstractExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractExtractor.js","sourceRoot":"","sources":["../../../src/extractors/AbstractExtractor.ts"],"names":[],"mappings":";;;AACA,MAAsB,iBAAiB;IACd;IAArB,YAAqB,IAAW;QAAX,SAAI,GAAJ,IAAI,CAAO;IAAI,CAAC;CAExC;AAHD,8CAGC"}

package/bin/src/extractors/PageExtractor.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PageExtractor = void 0;
+const AbstractExtractor_1 = require("./AbstractExtractor");
+class PageExtractor extends AbstractExtractor_1.AbstractExtractor {
+    constructor() { super("page-extractor"); }
+    async extract(value) {
+        const { window: { document: { title, location: { href: url } } } } = value;
+        return { title, url };
+    }
+}
+exports.PageExtractor = PageExtractor;
+//# sourceMappingURL=PageExtractor.js.map

package/bin/src/extractors/PageExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PageExtractor.js","sourceRoot":"","sources":["../../../src/extractors/PageExtractor.ts"],"names":[],"mappings":";;;AAEA,2DAAwD;AAExD,MAAa,aAAc,SAAQ,qCAA8B;IAC7D,gBAAgB,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAE1C,KAAK,CAAC,OAAO,CAAC,KAAY;QACtB,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,KAAK,CAAA;QAC1E,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAA;IACzB,CAAC;CACJ;AAPD,sCAOC"}

package/bin/src/extractors/ResourceExtractor.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResourceExtractor = void 0;
+const resource_1 = require("../resource");
+const AbstractExtractor_1 = require("./AbstractExtractor");
+class ResourceExtractor extends AbstractExtractor_1.AbstractExtractor {
+    tags;
+    constructor(tags) {
+        super("page-extractor");
+        this.tags = tags;
+    }
+    async extract(value) {
+        const { document } = value.window;
+        const externalResources = [];
+        for (const tag of this.tags) {
+            const selector = document.querySelectorAll(tag);
+            const elements = Array.from(selector);
+            for (const element of elements) {
+                const text = (0, resource_1.findResourceText)(element);
+                const link = (0, resource_1.findResourceLink)(element);
+                if (!text || !link)
+                    continue;
+                if (!link.url.startsWith("http"))
+                    continue;
+                externalResources.push({ text, link });
+            }
+        }
+        return externalResources;
+    }
+}
+exports.ResourceExtractor = ResourceExtractor;
+//# sourceMappingURL=ResourceExtractor.js.map

package/bin/src/extractors/ResourceExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ResourceExtractor.js","sourceRoot":"","sources":["../../../src/extractors/ResourceExtractor.ts"],"names":[],"mappings":";;;AACA,0CAAiH;AACjH,2DAAwD;AAExD,MAAa,iBAAkB,SAAQ,qCAA4C;IAClD;IAA7B,YAA6B,IAAW;QACpC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QADC,SAAI,GAAJ,IAAI,CAAO;IAExC,CAAC;IACD,KAAK,CAAC,OAAO,CAAC,KAAY;QACtB,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,iBAAiB,GAAuB,EAAE,CAAC;QACjD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAW,GAAG,CAAC,CAAA;YACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACrC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,IAAI,GAAG,IAAA,2BAAgB,EAAC,OAAO,CAAC,CAAC;gBACvC,MAAM,IAAI,GAAG,IAAA,2BAAgB,EAAC,OAAO,CAAC,CAAC;gBACvC,IAAG,CAAC,IAAI,IAAI,CAAC,IAAI;oBAAE,SAAQ;gBAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;oBAAE,SAAQ;gBAC1C,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1C,CAAC;QACL,CAAC;QACD,OAAO,iBAAiB,CAAC;IAC7B,CAAC;CACJ;AApBD,8CAoBC"}

package/bin/src/main.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const package_json_1 = require("../package.json");
+const PageExtractor_1 = require("./extractors/PageExtractor");
+const ResourceExtractor_1 = require("./extractors/ResourceExtractor");
+const PageFetcher_1 = require("./page/PageFetcher");
+const JSONStylePrinter_1 = require("./printers/JSONStylePrinter");
+const program = new commander_1.Command();
+const url = (0, commander_1.createArgument)("<url | file...>", "remote https://URL or local file://resource.html to extract from");
+(async () => {
+    await program
+        .name(package_json_1.name)
+        .version(package_json_1.version, "-v, --version")
+        .description(package_json_1.description)
+        .addArgument(url)
+        .action(async (urls) => {
+        const printer = new JSONStylePrinter_1.JSONStylePrinter();
+        // simple log style printer
+        // const printer = new LogStylePrinter();
+        const pageFetcher = new PageFetcher_1.PageFetcher();
+        const pageExtractor = new PageExtractor_1.PageExtractor();
+        const resourceExtractor = new ResourceExtractor_1.ResourceExtractor(["a", "meta", "link", "embed"]);
+        const pageResponses = await pageFetcher.fetchAll(urls);
+        const pageMetadatas = [];
+        for (const { content, url, error } of pageResponses) {
+            const resources = error in (content) ? [] : await resourceExtractor.extract(content);
+            const descriptor = error in content ? { url, error } : await pageExtractor.extract(content);
+            pageMetadatas.push({ ...descriptor, resources });
+        }
+        await printer.print(...pageMetadatas);
+    })
+        .parseAsync(process.argv);
+})();
+//# sourceMappingURL=main.js.map

package/bin/src/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/main.ts"],"names":[],"mappings":";;;AACA,yCAAoD;AAEpD,kDAA6D;AAC7D,8DAA2D;AAC3D,sEAAmE;AACnE,oDAAiD;AAEjD,kEAA+D;AAG/D,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,MAAM,GAAG,GAAG,IAAA,0BAAc,EAAC,iBAAiB,EAAE,kEAAkE,CAAC,CAAC;AAElH,CAAC,KAAK,IAAI,EAAE;IACV,MAAM,OAAO;SACV,IAAI,CAAC,mBAAI,CAAC;SACV,OAAO,CAAC,sBAAO,EAAE,eAAe,CAAC;SACjC,WAAW,CAAC,0BAAW,CAAC;SACxB,WAAW,CAAC,GAAG,CAAC;SAChB,MAAM,CAAC,KAAK,EAAE,IAAc,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,IAAI,mCAAgB,EAAE,CAAC;QACvC,2BAA2B;QAC3B,yCAAyC;QAEzC,MAAM,WAAW,GAAG,IAAI,yBAAW,EAAE,CAAA;QACrC,MAAM,aAAa,GAAG,IAAI,6BAAa,EAAE,CAAA;QACzC,MAAM,iBAAiB,GAAG,IAAI,qCAAiB,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;QAE/E,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,aAAa,GAAmB,EAAE,CAAC;QAEzC,KAAK,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,aAAa,EAAE,CAAC;YACpD,MAAM,SAAS,GAAG,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACrF,MAAM,UAAU,GAAG,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC5F,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,aAAa,CAAC,CAAC;IACxC,CAAC,CAAC;SACD,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC,CAAC,EAAE,CAAC"}

package/bin/src/page/Page.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPage = exports.isError = void 0;
+const isError = (page) => 'error' in page;
+exports.isError = isError;
+const isPage = (page) => "resources" in page && Array.isArray(page.resources);
+exports.isPage = isPage;
+//# sourceMappingURL=Page.js.map

package/bin/src/page/Page.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Page.js","sourceRoot":"","sources":["../../../src/page/Page.ts"],"names":[],"mappings":";;;AAgBO,MAAM,OAAO,GAAG,CAAC,IAAkB,EAA6B,EAAE,CAAC,OAAO,IAAI,IAAI,CAAC;AAA7E,QAAA,OAAO,WAAsE;AACnF,MAAM,MAAM,GAAG,CAAC,IAAS,EAAgB,EAAE,CAC9C,WAAW,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AAD5C,QAAA,MAAM,UACsC"}

package/bin/src/page/PageFetcher.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PageFetcher = void 0;
+const jsdom_1 = require("jsdom");
+class PageFetcher {
+    async fetchPage(url) {
+        let dom;
+        const virtualConsole = new jsdom_1.VirtualConsole().on('jsdomError', (error) => {
+            process.stderr.write(`Error parsing ${url}:${error.message}\n`);
+        });
+        if (url.startsWith("file://")) {
+            dom = jsdom_1.JSDOM.fromFile(url, { virtualConsole });
+        }
+        else {
+            dom = jsdom_1.JSDOM.fromURL(url, { virtualConsole });
+        }
+        return dom.then(content => ({ url, content }))
+            .catch(({ message }) => ({ url, error: `JSDOM failed to parse: ${message}` }));
+    }
+    async fetchAll(urls) {
+        const responses = await Promise.all(urls.map(url => this.fetchPage(url)));
+        return responses.filter(response => response.content !== undefined);
+    }
+}
+exports.PageFetcher = PageFetcher;
+//# sourceMappingURL=PageFetcher.js.map

package/bin/src/page/PageFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PageFetcher.js","sourceRoot":"","sources":["../../../src/page/PageFetcher.ts"],"names":[],"mappings":";;;AAAA,iCAA8C;AAS9C,MAAa,WAAW;IACZ,KAAK,CAAC,SAAS,CAAC,GAAW;QAC/B,IAAI,GAAmB,CAAC;QACxB,MAAM,cAAc,GAAG,IAAI,sBAAc,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE;YACnE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5B,GAAG,GAAG,aAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,GAAG,GAAG,aAAK,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;aACzC,KAAK,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,0BAA0B,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACvF,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,IAAc;QACzB,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,OAAO,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;IACxE,CAAC;CAEJ;AApBD,kCAoBC"}

package/bin/src/printers/AbstractResourcePrinter.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractResourcePrinter = void 0;
+class AbstractResourcePrinter {
+    constructor() { }
+}
+exports.AbstractResourcePrinter = AbstractResourcePrinter;
+//# sourceMappingURL=AbstractResourcePrinter.js.map

package/bin/src/printers/AbstractResourcePrinter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AbstractResourcePrinter.js","sourceRoot":"","sources":["../../../src/printers/AbstractResourcePrinter.ts"],"names":[],"mappings":";;;AAEA,MAAsB,uBAAuB;IACzC,gBAAiB,CAAC;CAErB;AAHD,0DAGC"}

package/bin/src/printers/JSONStylePrinter.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JSONStylePrinter = void 0;
+const AbstractResourcePrinter_1 = require("./AbstractResourcePrinter");
+class JSONStylePrinter extends AbstractResourcePrinter_1.AbstractResourcePrinter {
+    print(...pages) {
+        const json = JSON.stringify(pages);
+        process.stdout.write(json + "\n");
+    }
+}
+exports.JSONStylePrinter = JSONStylePrinter;
+//# sourceMappingURL=JSONStylePrinter.js.map

package/bin/src/printers/JSONStylePrinter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JSONStylePrinter.js","sourceRoot":"","sources":["../../../src/printers/JSONStylePrinter.ts"],"names":[],"mappings":";;;AACA,uEAAoE;AAGpE,MAAa,gBAAiB,SAAQ,iDAAuB;IACzD,KAAK,CAAC,GAAG,KAAqB;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;IACrC,CAAC;CAGJ;AAPD,4CAOC"}

package/bin/src/printers/LogStylePrinter.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LogStylePrinter = void 0;
+const Page_1 = require("../page/Page");
+const AbstractResourcePrinter_1 = require("./AbstractResourcePrinter");
+class LogStylePrinter extends AbstractResourcePrinter_1.AbstractResourcePrinter {
+    write(str) {
+        process.stdout.write(str);
+    }
+    async print(...pages) {
+        for (const page of pages) {
+            if (!(0, Page_1.isPage)(page)) {
+                this.write(page.error);
+                continue;
+            }
+            const { resources, title, url } = page;
+            this.write(`Title: ${title}\n`);
+            this.write(`URL: ${url}\n\n`);
+            for (const resource of resources) {
+                const { link: { url }, text: { value } } = resource;
+                this.write(`${value}: ${url}\n`);
+            }
+        }
+    }
+}
+exports.LogStylePrinter = LogStylePrinter;
+//# sourceMappingURL=LogStylePrinter.js.map

package/bin/src/printers/LogStylePrinter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LogStylePrinter.js","sourceRoot":"","sources":["../../../src/printers/LogStylePrinter.ts"],"names":[],"mappings":";;;AAAA,uCAAoE;AACpE,uEAAoE;AAEpE,MAAa,eAAgB,SAAQ,iDAAuB;IAExD,KAAK,CAAC,GAAW;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAG,KAAqB;QAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,IAAA,aAAM,EAAC,IAAI,CAAC,EAAE,CAAC;gBAChB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBACtB,SAAQ;YACZ,CAAC;YAED,MAAM,EAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;YAErC,IAAI,CAAC,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC,CAAA;YAC/B,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAA;YAE7B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBAC/B,MAAM,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,GAAG,QAAQ,CAAA;gBACnD,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,GAAG,IAAI,CAAC,CAAA;YACpC,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAxBD,0CAwBC"}

package/bin/src/resource.js

@@ -0,0 +1,56 @@
+"use strict";
+/**
+ * @license MIT
+ * We are interested in visualising a page as a collection of tags.
+ *
+ * We wish to work with tags that can be compactly previewed on a webpage.
+ * Here we must declare all of the element types that can be used to represent
+ * a resource that can be hyperlinked off a webpage.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isKeyDefined = exports.isResourceKey = exports.RESOURCE_LINK_KEYS = exports.RESOURCE_DISPLAYABLE_KEYS = void 0;
+exports.findResourceText = findResourceText;
+exports.findResourceLink = findResourceLink;
+function findDefinedKey(element, keys) {
+    for (const key of keys) {
+        if ((0, exports.isKeyDefined)(key, element)) {
+            return key;
+        }
+    }
+}
+exports.RESOURCE_DISPLAYABLE_KEYS = [
+    'id',
+    'innerText',
+    'textContent',
+    'class',
+    'ariaLabel',
+    'ariaDescription',
+    'alt',
+    'rel'
+];
+exports.RESOURCE_LINK_KEYS = [
+    "href",
+    "data-src",
+    "target",
+    "action",
+    "src",
+    "url"
+];
+function findResourceText(element) {
+    for (const key of exports.RESOURCE_DISPLAYABLE_KEYS) {
+        const value = element[key];
+        if (value && typeof value === 'string' && value.trim() !== '')
+            return { key, value };
+    }
+}
+function findResourceLink(element) {
+    const key = findDefinedKey(element, [...exports.RESOURCE_LINK_KEYS]);
+    const url = element[key];
+    if (url && typeof url === 'string' && url.trim() !== '')
+        return { key, url };
+}
+const isResourceKey = (key) => key in exports.RESOURCE_LINK_KEYS;
+exports.isResourceKey = isResourceKey;
+const isKeyDefined = (key, element) => key in element && element[key] !== undefined;
+exports.isKeyDefined = isKeyDefined;
+//# sourceMappingURL=resource.js.map

package/bin/src/resource.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resource.js","sourceRoot":"","sources":["../../src/resource.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;AA8CH,4CAMC;AAED,4CAKC;AAvDD,SAAS,cAAc,CAAC,OAAiB,EAAE,IAAe;IACtD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,IAAA,oBAAY,EAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YAC7B,OAAO,GAAG,CAAC;QACf,CAAC;IACL,CAAC;AACL,CAAC;AAEY,QAAA,yBAAyB,GAAG;IACrC,IAAI;IACJ,WAAW;IACX,aAAa;IACb,OAAO;IACP,WAAW;IACX,iBAAiB;IACjB,KAAK;IACL,KAAK;CACC,CAAC;AASE,QAAA,kBAAkB,GAAG;IAC9B,MAAM;IACN,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,KAAK;IACL,KAAK;CACC,CAAC;AASX,SAAgB,gBAAgB,CAAC,OAAiB;IAC9C,KAAK,MAAM,GAAG,IAAI,iCAAyB,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1B,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;YACzD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;AACL,CAAC;AAED,SAAgB,gBAAgB,CAAC,OAAiB;IAC9C,MAAM,GAAG,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,GAAG,0BAAkB,CAAC,CAAC,CAAC;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;QACnD,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AAC5B,CAAC;AAOM,MAAM,aAAa,GAAG,CAAC,GAAW,EAAkB,EAAE,CAAC,GAAG,IAAI,0BAAkB,CAAC;AAA3E,QAAA,aAAa,iBAA8D;AAEjF,MAAM,YAAY,GAAG,CAA6B,GAAW,EAAE,OAAU,EAAW,EAAE,CACzF,GAAG,IAAI,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC;AADpC,QAAA,YAAY,gBACwB"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pagerts",
   "description": "A tool for viewing external relations in a webpage",
-  "version": "0.1.9",
+  "version": "0.2.0",
   "main": "main.js",
   "bin": {
     "pagerts": "bin/main.js"
@@ -9,7 +9,7 @@
   "scripts": {
     "test": "jest",
     "build": "esbuild src/main.ts --external:jsdom --bundle --outdir=bin --minify --sourcemap --platform=node",
-    "tsc": "tsc -b",
+    "lint": "tsc",
     "start": "node ./bin/main.js",
     "dev": "npx tsx src/main.ts"
   },

package/src/extractors/PageExtractor.ts

@@ -1,21 +1,12 @@
 import { isError, type Page } from '../page/Page';
 import { JSDOM } from 'jsdom';
 import { AbstractExtractor } from './AbstractExtractor';
-import { isJSDOM, isMetadata, type PageResponse } from '../page/PageFetcher';
 
-export class PageExtractor extends AbstractExtractor<PageResponse, Page> {
+export class PageExtractor extends AbstractExtractor<JSDOM, Page> {
     constructor() { super("page-extractor"); }
 
-    async extract(value: PageResponse): Promise<Page> {
-        if (isJSDOM(value)) {
-            const { window: { document: { title, location: { href: url } } } } = value
-            return { title, url }
-        }
-        if (isError(value)) {
-            const { error } = value
-            throw new Error(error)
-        }
-        const { title, url } = value
+    async extract(value: JSDOM): Promise<Page> {
+        const { window: { document: { title, location: { href: url } } } } = value
         return { title, url }
     }
 }

package/src/main.ts

@@ -4,14 +4,14 @@ import { Command, createArgument } from "commander";
 import { description, name, version } from '../package.json';
 import { PageExtractor } from "./extractors/PageExtractor";
 import { ResourceExtractor } from "./extractors/ResourceExtractor";
-import { isJSDOM, PageFetcher, type PageResponse } from "./page/PageFetcher";
+import { PageFetcher } from "./page/PageFetcher";
 import type { Page, PageMetadata } from "./page/Page";
 import { JSONStylePrinter } from "./printers/JSONStylePrinter";
 import { LogStylePrinter } from "./printers/LogStylePrinter";
 
 const program = new Command();
 
-const url = createArgument("<url|file...>", "remote URL or local file to extract remote resources from");
+const url = createArgument("<url | file...>", "remote https://URL or local file://resource.html to extract from");
 
 (async () => {
   await program
@@ -28,20 +28,15 @@ const url = createArgument("<url|file...>", "remote URL or local file to extract
       const pageExtractor = new PageExtractor()
       const resourceExtractor = new ResourceExtractor(["a", "meta", "link", "embed"])
 
-      const pageResponses: PageResponse[] = await pageFetcher.fetchAll(urls);
-      const pageMetadatas: PageMetadata[] = []
-      for (const page of pageResponses) {
-        // check if page has an error
-        if (isJSDOM(page)) {
-          const resources = await resourceExtractor.extract(page);
-          const descriptor = await pageExtractor.extract(page);
-          pageMetadatas.push({
-            ...descriptor, resources
-          });
-        } else pageMetadatas.push({
-            ...page, resources: []
-          });
+      const pageResponses = await pageFetcher.fetchAll(urls);
+      const pageMetadatas: PageMetadata[] = [];
+
+      for (const { content, url, error } of pageResponses) {
+        const resources = error in (content) ? [] : await resourceExtractor.extract(content);
+        const descriptor = error in content ? { url, error } : await pageExtractor.extract(content);
+        pageMetadatas.push({ ...descriptor, resources });
       }
+
       await printer.print(...pageMetadatas);
     })
     .parseAsync(process.argv);

package/src/page/Page.ts

@@ -1,5 +1,4 @@
 import type { ExternalResource } from "../resource";
-import type { PageResponse } from "./PageFetcher";
 
 type hasTitle = {
     title: string;

package/src/page/PageFetcher.ts

@@ -1,24 +1,30 @@
 import { JSDOM, VirtualConsole } from 'jsdom';
 import type { Page, PageMetadata } from './Page';
 
-export type PageResponse = JSDOM | PageMetadata
-
-export const isJSDOM = (page: PageResponse): page is JSDOM => 'window' in page;
-export const isMetadata = (page: PageResponse): page is PageMetadata => 'url' in page && 'title' in page;
+interface PageResponse {
+    url: string;
+    content?: JSDOM;
+    error?: string;
+}
 
 export class PageFetcher {
-    async fetchPage(url: string): Promise<PageResponse> {
-        const dom = JSDOM.fromURL(url, {
-            virtualConsole: new VirtualConsole().on('jsdomError', (error) => {
-                process.stderr.write(`Error parsing ${url}:${error.message}\n`);
-            })
+    private async fetchPage(url: string): Promise<PageResponse> {
+        let dom: Promise<JSDOM>;
+        const virtualConsole = new VirtualConsole().on('jsdomError', (error) => {
+            process.stderr.write(`Error parsing ${url}:${error.message}\n`);
         });
-        return dom.catch(({ message }) => {
-            return { url, error: `Not an HTML page: ${message}` };
-        })
+        if (url.startsWith("file://")) {
+            dom = JSDOM.fromFile(url, { virtualConsole });
+        } else {
+            dom = JSDOM.fromURL(url, { virtualConsole });
+        }
+
+        return dom.then(content => ({ url, content }))
+            .catch(({ message }) => ({ url, error: `JSDOM failed to parse: ${message}` }));
     }
     async fetchAll(urls: string[]): Promise<PageResponse[]> {
-        return await Promise.all(urls.map(url => this.fetchPage(url)));
+        const responses = await Promise.all(urls.map(url => this.fetchPage(url)));
+        return responses.filter(response => response.content !== undefined);
     }
-    constructor() { }
+
 }