pagerts 0.1.5 → 0.2.0

package/src/main.ts

@@ -1,47 +1,43 @@
-#!/usr/bin/env node
-import { Command, createArgument } from "commander";
-
-import { description, name, version } from '../package.json';
-import { PageExtractor } from "./extractors/PageExtractor";
-import { ResourceExtractor } from "./extractors/ResourceExtractor";
-import { isJSDOM, PageFetcher, type PageResponse } from "./page/PageFetcher";
-import { type Page } from "./page/PageMetadata";
-import { JSONStylePrinter } from "./printers/JSONStylePrinter";
-
-const program = new Command();
-
-const url = createArgument("<url|file...>", "remote URL or local file to extract remote resources from");
-
-(async () => {
-  await program
-    .name(name)
-    .version(version, "-v, --version")
-    .description(description)
-    .addArgument(url)
-    .action(async (urls: string[]) => {
-      const printer = new JSONStylePrinter();
-      try {
-        const pageFetcher = new PageFetcher()
-        const pageExtractor = new PageExtractor()
-        const resourceExtractor = new ResourceExtractor(["a", "meta", "link", "embed"])
-        const pagesFetched: PageResponse[] = await pageFetcher.fetchAll(urls);
-        const metadataPages: Page[] = []
-        for (const page of pagesFetched) {
-          // check if page has an error
-          if (isJSDOM(page)) {
-            const resources = await resourceExtractor.extract(page);
-            const descriptor = await pageExtractor.extract(page);
-            metadataPages.push({
-              ...descriptor, resources
-            });
-          } else metadataPages.push(page);
-        }
-        await printer.print(...metadataPages);
-      } catch (error) {
-        await printer.print({
-          error: error.message
-        })
-      }
-    })
-    .parseAsync(process.argv);
-})();
+#!/usr/bin/env node
+import { Command, createArgument } from "commander";
+
+import { description, name, version } from '../package.json';
+import { PageExtractor } from "./extractors/PageExtractor";
+import { ResourceExtractor } from "./extractors/ResourceExtractor";
+import { PageFetcher } from "./page/PageFetcher";
+import type { Page, PageMetadata } from "./page/Page";
+import { JSONStylePrinter } from "./printers/JSONStylePrinter";
+import { LogStylePrinter } from "./printers/LogStylePrinter";
+
+const program = new Command();
+
+const url = createArgument("<url | file...>", "remote https://URL or local file://resource.html to extract from");
+
+(async () => {
+  await program
+    .name(name)
+    .version(version, "-v, --version")
+    .description(description)
+    .addArgument(url)
+    .action(async (urls: string[]) => {
+      const printer = new JSONStylePrinter();
+      // simple log style printer
+      // const printer = new LogStylePrinter();
+
+      const pageFetcher = new PageFetcher()
+      const pageExtractor = new PageExtractor()
+      const resourceExtractor = new ResourceExtractor(["a", "meta", "link", "embed"])
+
+      const pageResponses = await pageFetcher.fetchAll(urls);
+      const pageMetadatas: PageMetadata[] = [];
+
+      for (const { content, url, error } of pageResponses) {
+        const resources = error in (content) ? [] : await resourceExtractor.extract(content);
+        const descriptor = error in content ? { url, error } : await pageExtractor.extract(content);
+        pageMetadatas.push({ ...descriptor, resources });
+      }
+
+      await printer.print(...pageMetadatas);
+    })
+    .parseAsync(process.argv);
+})();

package/src/page/Page.ts

@@ -1,3 +1,19 @@
-import { JSDOM } from 'jsdom';
-
-export type Page = JSDOM
+import type { ExternalResource } from "../resource";
+
+type hasTitle = {
+    title: string;
+};
+
+type hasUrl = {
+    url: string;
+};
+
+type hasResources = {
+    resources: ExternalResource[];
+};
+
+export type Page = hasTitle & hasUrl
+export type PageMetadata = (Page & hasResources) | { error: string }
+export const isError = (page: PageMetadata): page is { error: string } => 'error' in page;
+export const isPage = (page: any): page is Page =>
+    "resources" in page && Array.isArray(page.resources);

package/src/page/PageFetcher.ts

@@ -1,25 +1,30 @@
-import { JSDOM, VirtualConsole } from 'jsdom';
-import type { Page } from './PageMetadata';
-
-export type PageResponse = JSDOM | Page
-
-export const isJSDOM = (page: PageResponse): page is JSDOM => 'window' in page;
-
-export class PageFetcher {
-    async fetchPage(url: string): Promise<PageResponse> {
-        try {
-            const dom = await JSDOM.fromURL(url, {
-                virtualConsole: new VirtualConsole().on('jsdomError', (error) => {
-                    console.error(`Error parsing ${url}:`, error.message);
-                })
-            });
-            return dom;
-        } catch (error) {
-            throw error;
-        }
-    }
-    async fetchAll(urls: string[]): Promise<PageResponse[]> {
-        return await Promise.all(urls.map(url => this.fetchPage(url)));
-    }
-    constructor() { }
-}
+import { JSDOM, VirtualConsole } from 'jsdom';
+import type { Page, PageMetadata } from './Page';
+
+interface PageResponse {
+    url: string;
+    content?: JSDOM;
+    error?: string;
+}
+
+export class PageFetcher {
+    private async fetchPage(url: string): Promise<PageResponse> {
+        let dom: Promise<JSDOM>;
+        const virtualConsole = new VirtualConsole().on('jsdomError', (error) => {
+            process.stderr.write(`Error parsing ${url}:${error.message}\n`);
+        });
+        if (url.startsWith("file://")) {
+            dom = JSDOM.fromFile(url, { virtualConsole });
+        } else {
+            dom = JSDOM.fromURL(url, { virtualConsole });
+        }
+
+        return dom.then(content => ({ url, content }))
+            .catch(({ message }) => ({ url, error: `JSDOM failed to parse: ${message}` }));
+    }
+    async fetchAll(urls: string[]): Promise<PageResponse[]> {
+        const responses = await Promise.all(urls.map(url => this.fetchPage(url)));
+        return responses.filter(response => response.content !== undefined);
+    }
+
+}

package/src/printers/AbstractResourcePrinter.ts

@@ -1,6 +1,6 @@
-import type { Page } from "../page/PageMetadata";
-
-export abstract class AbstractResourcePrinter {
-    constructor() {  }
-    abstract print(...pages: Page[]): void | Promise<void>;
-}
+import type { Page, PageMetadata  } from "../page/Page";
+
+export abstract class AbstractResourcePrinter {
+    constructor() {  }
+    abstract print(...pages: PageMetadata[]): void | Promise<void>;
+}

package/src/printers/JSONStylePrinter.ts

@@ -1,12 +1,12 @@
-import type { Page } from "../page/PageMetadata";
-import { AbstractResourcePrinter } from "./AbstractResourcePrinter";
-
-
-export class JSONStylePrinter extends AbstractResourcePrinter {
-    print(...pages: Page[]): void | Promise<void> {
-        const json = JSON.stringify(pages);
-        process.stdout.write(json + "\n")
-    }
-   
-
-}
+import type { PageMetadata } from "../page/Page";
+import { AbstractResourcePrinter } from "./AbstractResourcePrinter";
+
+
+export class JSONStylePrinter extends AbstractResourcePrinter {
+    print(...pages: PageMetadata[]): void | Promise<void> {
+        const json = JSON.stringify(pages);
+        process.stdout.write(json + "\n")
+    }
+   
+
+}

package/src/printers/LogStylePrinter.ts

@@ -1,23 +1,28 @@
-import type { PageMetadata } from '../page/PageMetadata';
-import { AbstractResourcePrinter } from './AbstractResourcePrinter';
-
-export class LogStylePrinter extends AbstractResourcePrinter {
-
-    write(str: string): void {
-        process.stdout.write(str)
-    }
-
-    async print(...pages: PageMetadata[]): Promise<void> {
-        for (const page of pages) {
-            const { resources, title, url } = page
-
-            this.write(`Title: ${title}\n`)
-            this.write(`URL: ${url}\n\n`)
-
-            for (const resource of resources) {
-                const { link: { url }, text: { value } } = resource
-                this.write(`${value}: ${url}\n`)
-            }
-        }
-    }
-}
+import { isPage, type Page, type PageMetadata } from '../page/Page';
+import { AbstractResourcePrinter } from './AbstractResourcePrinter';
+
+export class LogStylePrinter extends AbstractResourcePrinter {
+
+    write(str: string): void {
+        process.stdout.write(str)
+    }
+
+    async print(...pages: PageMetadata[]): Promise<void> {
+        for (const page of pages) {
+            if (!isPage(page)) {
+                this.write(page.error)
+                continue
+            }
+
+            const {resources, title, url } = page
+
+            this.write(`Title: ${title}\n`)
+            this.write(`URL: ${url}\n\n`)
+
+            for (const resource of resources) {
+                const { link: { url }, text: { value } } = resource
+                this.write(`${value}: ${url}\n`)
+            }
+        }
+    }
+}

package/src/resource.ts

@@ -1,96 +1,96 @@
-/**
- * @license MIT
- * We are interested in visualising a page as a collection of tags.
- * 
- * We wish to work with tags that can be compactly previewed on a webpage.
- * Here we must declare all of the element types that can be used to represent
- * a resource that can be hyperlinked off a webpage.
- */
-
-type Tags = HTMLElementTagNameMap
-
-function findDefinedKey(element: Resource, keys: LinkKey[]): LinkKey | undefined {
-    for (const key of keys) {
-        if (isKeyDefined(key, element)) {
-            return key;
-        }
-    }
-}
-
-export const RESOURCE_DISPLAYABLE_KEYS = [
-    'id',
-    'innerText',
-    'textContent',
-    'class',
-    'ariaLabel',
-    'ariaDescription',
-    'alt',
-    'rel'
-] as const;
-
-export type DisplayableKey = (typeof RESOURCE_DISPLAYABLE_KEYS)[number];
-
-export type ResourceKey = {
-    key: DisplayableKey;
-    value: string;
-};
-
-export const RESOURCE_LINK_KEYS = [
-    "href",
-    "data-src",
-    "target",
-    "action",
-    "src",
-    "url"
-] as const;
-
-export type LinkKey = typeof RESOURCE_LINK_KEYS[number];
-
-export type ResourceLink = {
-    key: LinkKey;
-    url: string;
-}
-
-export function findResourceText(element: Resource): ResourceKey | undefined {
-    for (const key of RESOURCE_DISPLAYABLE_KEYS) {
-        const value = element[key]
-        if (value && typeof value === 'string' && value.trim() !== '')
-            return { key, value };
-    }
-}
-
-export function findResourceLink(element: Resource): ResourceLink | undefined {
-    const key = findDefinedKey(element, [...RESOURCE_LINK_KEYS]);
-    const url = element[key];
-    if (url && typeof url === 'string' && url.trim() !== '')
-        return { key, url };
-}
-
-export type ExternalResource = {
-    text: ResourceKey;
-    link: ResourceLink;
-};
-
-export const isResourceKey = (key: string): key is LinkKey => key in RESOURCE_LINK_KEYS;
-
-export const isKeyDefined = <E extends Tags[keyof Tags]>(key: string, element: E): boolean =>
-    key in element && element[key] !== undefined;
-
-export type ResourceElement<T, U> = {
-    [K in keyof T]: U extends keyof T[K] ? T[K] : never
-}[keyof T];
-
-export type Tag = keyof Tags
-
-export type Resource = ResourceElement<Tags, (typeof RESOURCE_LINK_KEYS)[number]>;
-
-export type ResourceByName<T extends keyof Tags> = Tags[T]
-
-/** tests **/
-
-type test1 = HTMLAnchorElement extends Resource ? true : false // true
-type test2 = HTMLImageElement extends Resource ? true : false // true
-type test3 = HTMLDivElement extends Resource ? true : false // false
-
-type test4 = ResourceElement<Tags, "src">
-
+/**
+ * @license MIT
+ * We are interested in visualising a page as a collection of tags.
+ * 
+ * We wish to work with tags that can be compactly previewed on a webpage.
+ * Here we must declare all of the element types that can be used to represent
+ * a resource that can be hyperlinked off a webpage.
+ */
+
+type Tags = HTMLElementTagNameMap
+
+function findDefinedKey(element: Resource, keys: LinkKey[]): LinkKey | undefined {
+    for (const key of keys) {
+        if (isKeyDefined(key, element)) {
+            return key;
+        }
+    }
+}
+
+export const RESOURCE_DISPLAYABLE_KEYS = [
+    'id',
+    'innerText',
+    'textContent',
+    'class',
+    'ariaLabel',
+    'ariaDescription',
+    'alt',
+    'rel'
+] as const;
+
+export type DisplayableKey = (typeof RESOURCE_DISPLAYABLE_KEYS)[number];
+
+export type ResourceKey = {
+    key: DisplayableKey;
+    value: string;
+};
+
+export const RESOURCE_LINK_KEYS = [
+    "href",
+    "data-src",
+    "target",
+    "action",
+    "src",
+    "url"
+] as const;
+
+export type LinkKey = typeof RESOURCE_LINK_KEYS[number];
+
+export type ResourceLink = {
+    key: LinkKey;
+    url: string;
+}
+
+export function findResourceText(element: Resource): ResourceKey | undefined {
+    for (const key of RESOURCE_DISPLAYABLE_KEYS) {
+        const value = element[key]
+        if (value && typeof value === 'string' && value.trim() !== '')
+            return { key, value };
+    }
+}
+
+export function findResourceLink(element: Resource): ResourceLink | undefined {
+    const key = findDefinedKey(element, [...RESOURCE_LINK_KEYS]);
+    const url = element[key];
+    if (url && typeof url === 'string' && url.trim() !== '')
+        return { key, url };
+}
+
+export type ExternalResource = {
+    text: ResourceKey;
+    link: ResourceLink;
+};
+
+export const isResourceKey = (key: string): key is LinkKey => key in RESOURCE_LINK_KEYS;
+
+export const isKeyDefined = <E extends Tags[keyof Tags]>(key: string, element: E): boolean =>
+    key in element && element[key] !== undefined;
+
+export type ResourceElement<T, U> = {
+    [K in keyof T]: U extends keyof T[K] ? T[K] : never
+}[keyof T];
+
+export type Tag = keyof Tags
+
+export type Resource = ResourceElement<Tags, (typeof RESOURCE_LINK_KEYS)[number]>;
+
+export type ResourceByName<T extends keyof Tags> = Tags[T]
+
+/** tests **/
+
+type test1 = HTMLAnchorElement extends Resource ? true : false // true
+type test2 = HTMLImageElement extends Resource ? true : false // true
+type test3 = HTMLDivElement extends Resource ? true : false // false
+
+type test4 = ResourceElement<Tags, "src">
+

package/tsconfig.json

@@ -1,12 +1,12 @@
-{
-    "compilerOptions": {
-        "module": "NodeNext",
-        "target": "ESNext",
-        "resolveJsonModule": true,
-        "outDir": "dist",
-        "sourceMap": true,
-    },
-    "include": [
-        "src/**.*",
-    ],
+{
+    "compilerOptions": {
+        "module": "NodeNext",
+        "target": "ESNext",
+        "resolveJsonModule": true,
+        "outDir": "bin",
+        "sourceMap": true,
+    },
+    "include": [
+        "src/**.*",
+    ],
 }

package/.vscode/launch.json

@@ -1,22 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "node",
-            "args": ["https://google.com/"],
-            "request": "launch",
-            "name": "Launch Program",
-            "skipFiles": [
-                "<node_internals>/**"
-            ],
-            "program": "${workspaceFolder}/src/main.ts",
-            "preLaunchTask": "tsc: build - tsconfig.json",
-            "outFiles": [
-                "${workspaceFolder}/dist/**/*.js"
-            ]
-        }
-    ]
-}

package/src/page/PageMetadata.ts

@@ -1,23 +0,0 @@
-import type { ExternalResource } from "../resource";
-import type { PageResponse } from "./PageFetcher";
-
-type hasTitle = {
-    title: string;
-};
-
-type hasUrl = {
-    url: string;
-};
-
-export type PageDescriptor = hasTitle & hasUrl 
-
-export type PageMetadata = {
-    resources: ExternalResource[];
-} 
-
-export type Page = (PageDescriptor & PageMetadata) | {
-    error: Error
-}
-
-export const isPage = (page: any): page is Page => 
-  "resources" in page && Array.isArray(page.resources);