pagerts 0.2.0 → 0.4.1

package/src/main.ts

@@ -1,43 +1,71 @@
-#!/usr/bin/env node
-import { Command, createArgument } from "commander";
-
-import { description, name, version } from '../package.json';
-import { PageExtractor } from "./extractors/PageExtractor";
-import { ResourceExtractor } from "./extractors/ResourceExtractor";
-import { PageFetcher } from "./page/PageFetcher";
-import type { Page, PageMetadata } from "./page/Page";
-import { JSONStylePrinter } from "./printers/JSONStylePrinter";
-import { LogStylePrinter } from "./printers/LogStylePrinter";
-
-const program = new Command();
-
-const url = createArgument("<url | file...>", "remote https://URL or local file://resource.html to extract from");
-
-(async () => {
-  await program
-    .name(name)
-    .version(version, "-v, --version")
-    .description(description)
-    .addArgument(url)
-    .action(async (urls: string[]) => {
-      const printer = new JSONStylePrinter();
-      // simple log style printer
-      // const printer = new LogStylePrinter();
-
-      const pageFetcher = new PageFetcher()
-      const pageExtractor = new PageExtractor()
-      const resourceExtractor = new ResourceExtractor(["a", "meta", "link", "embed"])
-
-      const pageResponses = await pageFetcher.fetchAll(urls);
-      const pageMetadatas: PageMetadata[] = [];
-
-      for (const { content, url, error } of pageResponses) {
-        const resources = error in (content) ? [] : await resourceExtractor.extract(content);
-        const descriptor = error in content ? { url, error } : await pageExtractor.extract(content);
-        pageMetadatas.push({ ...descriptor, resources });
-      }
-
-      await printer.print(...pageMetadatas);
-    })
-    .parseAsync(process.argv);
-})();
+#!/usr/bin/env node
+import { Command, createArgument } from 'commander';
+
+import pkg from '../package.json';
+import { PageExtractor, ResourceExtractor } from './extractors/index.js';
+import { PageFetcher, type PageMetadata } from './page/index.js';
+import { JSONStylePrinter } from './printers/index.js';
+import { validateUrls } from './security.js';
+
+const { description, name, version } = pkg;
+
+const program = new Command();
+
+const url = createArgument(
+  '<url | file...>',
+  'remote https://URL or local file://resource.html to extract from'
+);
+
+(async (): Promise<void> => {
+  await program
+    .name(name)
+    .version(version, '-v, --version')
+    .description(description)
+    .addArgument(url)
+    .action(async (urls: string[]) => {
+      try {
+        // Validate URLs first
+        const { validUrls, errors } = validateUrls(urls);
+
+        // Report validation errors
+        if (errors.length > 0) {
+          console.error('\n❌ URL Validation Errors:');
+          errors.forEach(({ url: invalidUrl, error }) => {
+            console.error(`  - ${invalidUrl}: ${error}`);
+          });
+        }
+
+        // Exit if no valid URLs
+        if (validUrls.length === 0) {
+          console.error('\n❌ No valid URLs to process. Exiting.');
+          process.exit(1);
+        }
+
+        console.error(`\n✅ Processing ${validUrls.length} valid URL(s)...`);
+
+        const printer = new JSONStylePrinter();
+        const pageFetcher = new PageFetcher();
+        const pageExtractor = new PageExtractor();
+        const resourceExtractor = new ResourceExtractor(['a', 'meta', 'link', 'embed']);
+
+        const pageResponses = await pageFetcher.fetchAll(validUrls);
+        const pageMetadatas: PageMetadata[] = [];
+
+        for (const { content, url: responseUrl, error } of pageResponses) {
+          const resources =
+            error !== undefined || !content ? [] : await resourceExtractor.extract(content);
+          const descriptor =
+            error !== undefined || !content
+              ? { url: responseUrl, error: error ?? 'Unknown error', resources }
+              : await pageExtractor.extract(content);
+          pageMetadatas.push({ ...descriptor, resources });
+        }
+
+        await printer.print(...pageMetadatas);
+      } catch (error) {
+        console.error('\n❌ An error occurred:', error instanceof Error ? error.message : error);
+        process.exit(1);
+      }
+    })
+    .parseAsync(process.argv);
+})();

package/src/page/Page.ts

@@ -1,19 +1,24 @@
-import type { ExternalResource } from "../resource";
-
-type hasTitle = {
-    title: string;
-};
-
-type hasUrl = {
-    url: string;
-};
-
-type hasResources = {
-    resources: ExternalResource[];
-};
-
-export type Page = hasTitle & hasUrl
-export type PageMetadata = (Page & hasResources) | { error: string }
-export const isError = (page: PageMetadata): page is { error: string } => 'error' in page;
-export const isPage = (page: any): page is Page =>
-    "resources" in page && Array.isArray(page.resources);
+import type { ExternalResource } from '../resource.js';
+
+type hasTitle = {
+  title: string;
+};
+
+type hasUrl = {
+  url: string;
+};
+
+type hasResources = {
+  resources: ExternalResource[];
+};
+
+export type Page = hasTitle & hasUrl;
+
+export type PageSuccess = Page & hasResources;
+export type PageFailure = hasUrl & hasResources & { error: string };
+export type PageMetadata = PageSuccess | PageFailure;
+
+export const isError = (page: PageMetadata): page is PageFailure => 'error' in page;
+
+export const isPage = (page: PageMetadata): page is PageSuccess =>
+  'title' in page && typeof page.title === 'string' && Array.isArray(page.resources);

package/src/page/PageFetcher.ts

@@ -1,30 +1,81 @@
-import { JSDOM, VirtualConsole } from 'jsdom';
-import type { Page, PageMetadata } from './Page';
-
-interface PageResponse {
-    url: string;
-    content?: JSDOM;
-    error?: string;
-}
-
-export class PageFetcher {
-    private async fetchPage(url: string): Promise<PageResponse> {
-        let dom: Promise<JSDOM>;
-        const virtualConsole = new VirtualConsole().on('jsdomError', (error) => {
-            process.stderr.write(`Error parsing ${url}:${error.message}\n`);
-        });
-        if (url.startsWith("file://")) {
-            dom = JSDOM.fromFile(url, { virtualConsole });
-        } else {
-            dom = JSDOM.fromURL(url, { virtualConsole });
-        }
-
-        return dom.then(content => ({ url, content }))
-            .catch(({ message }) => ({ url, error: `JSDOM failed to parse: ${message}` }));
-    }
-    async fetchAll(urls: string[]): Promise<PageResponse[]> {
-        const responses = await Promise.all(urls.map(url => this.fetchPage(url)));
-        return responses.filter(response => response.content !== undefined);
-    }
-
-}
+import { JSDOM, VirtualConsole } from 'jsdom';
+
+interface PageResponse {
+  url: string;
+  content?: JSDOM;
+  error?: string;
+}
+
+export class PageFetcher {
+  private readonly timeout: number;
+  private readonly maxRetries: number;
+
+  constructor(timeout = 10000, maxRetries = 2) {
+    this.timeout = timeout;
+    this.maxRetries = maxRetries;
+  }
+
+  private async fetchPage(url: string, retryCount = 0): Promise<PageResponse> {
+    const virtualConsole = new VirtualConsole().on('jsdomError', (error: Error) => {
+      process.stderr.write(`Error parsing ${url}: ${error.message}\n`);
+    });
+
+    try {
+      let dom: Promise<JSDOM>;
+
+      if (url.startsWith('file://')) {
+        dom = JSDOM.fromFile(url.substring(7), { virtualConsole });
+      } else {
+        // Add timeout and security options for remote URLs
+        dom = JSDOM.fromURL(url, {
+          virtualConsole,
+          resources: 'usable',
+          runScripts: 'outside-only', // More secure - don't execute page scripts
+          beforeParse(window) {
+            // Prevent infinite loops and resource exhaustion
+            window.setTimeout = (() => {
+              throw new Error('setTimeout disabled for security');
+            }) as typeof window.setTimeout;
+            window.setInterval = (() => {
+              throw new Error('setInterval disabled for security');
+            }) as typeof window.setInterval;
+          },
+        });
+      }
+
+      const content = await Promise.race([
+        dom,
+        new Promise<never>((_, reject) =>
+          setTimeout(() => reject(new Error('Request timeout')), this.timeout)
+        ),
+      ]);
+
+      return { url, content };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : 'Unknown error';
+
+      // Retry logic for transient errors
+      if (retryCount < this.maxRetries && this.isRetryableError(message)) {
+        process.stderr.write(`Retrying ${url} (attempt ${retryCount + 1}/${this.maxRetries})...\n`);
+        await this.delay(1000 * (retryCount + 1)); // Exponential backoff
+        return this.fetchPage(url, retryCount + 1);
+      }
+
+      return { url, error: `Failed to fetch: ${message}` };
+    }
+  }
+
+  private isRetryableError(message: string): boolean {
+    const retryablePatterns = [/timeout/i, /ECONNRESET/i, /ETIMEDOUT/i, /ENOTFOUND/i, /network/i];
+    return retryablePatterns.some((pattern) => pattern.test(message));
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  async fetchAll(urls: string[]): Promise<PageResponse[]> {
+    const responses = await Promise.all(urls.map((url) => this.fetchPage(url)));
+    return responses.filter((response) => response.content !== undefined || response.error);
+  }
+}

package/src/page/index.ts

@@ -0,0 +1,3 @@
+export { PageFetcher } from './PageFetcher.js';
+export type { Page, PageMetadata } from './Page.js';
+export { isPage, isError } from './Page.js';

package/src/printers/AbstractResourcePrinter.ts

@@ -1,6 +1,6 @@
-import type { Page, PageMetadata  } from "../page/Page";
-
-export abstract class AbstractResourcePrinter {
-    constructor() {  }
-    abstract print(...pages: PageMetadata[]): void | Promise<void>;
-}
+import type { PageMetadata } from '../page/index.js';
+
+export abstract class AbstractResourcePrinter {
+  constructor() {}
+  abstract print(...pages: PageMetadata[]): void | Promise<void>;
+}

package/src/printers/JSONStylePrinter.ts

@@ -1,12 +1,9 @@
-import type { PageMetadata } from "../page/Page";
-import { AbstractResourcePrinter } from "./AbstractResourcePrinter";
-
-
-export class JSONStylePrinter extends AbstractResourcePrinter {
-    print(...pages: PageMetadata[]): void | Promise<void> {
-        const json = JSON.stringify(pages);
-        process.stdout.write(json + "\n")
-    }
-   
-
-}
+import type { PageMetadata } from '../page/index.js';
+import { AbstractResourcePrinter } from './AbstractResourcePrinter.js';
+
+export class JSONStylePrinter extends AbstractResourcePrinter {
+  print(...pages: PageMetadata[]): void | Promise<void> {
+    const json = JSON.stringify(pages);
+    process.stdout.write(json + '\n');
+  }
+}

package/src/printers/LogStylePrinter.ts

@@ -1,28 +1,30 @@
-import { isPage, type Page, type PageMetadata } from '../page/Page';
-import { AbstractResourcePrinter } from './AbstractResourcePrinter';
-
-export class LogStylePrinter extends AbstractResourcePrinter {
-
-    write(str: string): void {
-        process.stdout.write(str)
-    }
-
-    async print(...pages: PageMetadata[]): Promise<void> {
-        for (const page of pages) {
-            if (!isPage(page)) {
-                this.write(page.error)
-                continue
-            }
-
-            const {resources, title, url } = page
-
-            this.write(`Title: ${title}\n`)
-            this.write(`URL: ${url}\n\n`)
-
-            for (const resource of resources) {
-                const { link: { url }, text: { value } } = resource
-                this.write(`${value}: ${url}\n`)
-            }
-        }
-    }
-}
+import { isPage, type PageMetadata } from '../page/index.js';
+import { AbstractResourcePrinter } from './AbstractResourcePrinter.js';
+
+export class LogStylePrinter extends AbstractResourcePrinter {
+  write(str: string): void {
+    process.stdout.write(str);
+  }
+
+  async print(...pages: PageMetadata[]): Promise<void> {
+    for (const page of pages) {
+      if (!isPage(page)) {
+        this.write(page.error);
+        continue;
+      }
+
+      const { resources, title, url } = page;
+
+      this.write(`Title: ${title}\n`);
+      this.write(`URL: ${url}\n\n`);
+
+      for (const resource of resources) {
+        const {
+          link: { url },
+          text: { value },
+        } = resource;
+        this.write(`${value}: ${url}\n`);
+      }
+    }
+  }
+}

package/src/printers/index.ts

@@ -0,0 +1,3 @@
+export { AbstractResourcePrinter } from './AbstractResourcePrinter.js';
+export { JSONStylePrinter } from './JSONStylePrinter.js';
+export { LogStylePrinter } from './LogStylePrinter.js';

package/src/resource.ts

@@ -1,96 +1,88 @@
-/**
- * @license MIT
- * We are interested in visualising a page as a collection of tags.
- * 
- * We wish to work with tags that can be compactly previewed on a webpage.
- * Here we must declare all of the element types that can be used to represent
- * a resource that can be hyperlinked off a webpage.
- */
-
-type Tags = HTMLElementTagNameMap
-
-function findDefinedKey(element: Resource, keys: LinkKey[]): LinkKey | undefined {
-    for (const key of keys) {
-        if (isKeyDefined(key, element)) {
-            return key;
-        }
-    }
-}
-
-export const RESOURCE_DISPLAYABLE_KEYS = [
-    'id',
-    'innerText',
-    'textContent',
-    'class',
-    'ariaLabel',
-    'ariaDescription',
-    'alt',
-    'rel'
-] as const;
-
-export type DisplayableKey = (typeof RESOURCE_DISPLAYABLE_KEYS)[number];
-
-export type ResourceKey = {
-    key: DisplayableKey;
-    value: string;
-};
-
-export const RESOURCE_LINK_KEYS = [
-    "href",
-    "data-src",
-    "target",
-    "action",
-    "src",
-    "url"
-] as const;
-
-export type LinkKey = typeof RESOURCE_LINK_KEYS[number];
-
-export type ResourceLink = {
-    key: LinkKey;
-    url: string;
-}
-
-export function findResourceText(element: Resource): ResourceKey | undefined {
-    for (const key of RESOURCE_DISPLAYABLE_KEYS) {
-        const value = element[key]
-        if (value && typeof value === 'string' && value.trim() !== '')
-            return { key, value };
-    }
-}
-
-export function findResourceLink(element: Resource): ResourceLink | undefined {
-    const key = findDefinedKey(element, [...RESOURCE_LINK_KEYS]);
-    const url = element[key];
-    if (url && typeof url === 'string' && url.trim() !== '')
-        return { key, url };
-}
-
-export type ExternalResource = {
-    text: ResourceKey;
-    link: ResourceLink;
-};
-
-export const isResourceKey = (key: string): key is LinkKey => key in RESOURCE_LINK_KEYS;
-
-export const isKeyDefined = <E extends Tags[keyof Tags]>(key: string, element: E): boolean =>
-    key in element && element[key] !== undefined;
-
-export type ResourceElement<T, U> = {
-    [K in keyof T]: U extends keyof T[K] ? T[K] : never
-}[keyof T];
-
-export type Tag = keyof Tags
-
-export type Resource = ResourceElement<Tags, (typeof RESOURCE_LINK_KEYS)[number]>;
-
-export type ResourceByName<T extends keyof Tags> = Tags[T]
-
-/** tests **/
-
-type test1 = HTMLAnchorElement extends Resource ? true : false // true
-type test2 = HTMLImageElement extends Resource ? true : false // true
-type test3 = HTMLDivElement extends Resource ? true : false // false
-
-type test4 = ResourceElement<Tags, "src">
-
+/**
+ * @license MIT
+ * We are interested in visualising a page as a collection of tags.
+ *
+ * We wish to work with tags that can be compactly previewed on a webpage.
+ * Here we must declare all of the element types that can be used to represent
+ * a resource that can be hyperlinked off a webpage.
+ */
+type Tags = HTMLElementTagNameMap;
+
+function findDefinedKey(element: Resource, keys: LinkKey[]): LinkKey | undefined {
+  for (const key of keys) {
+    if (isKeyDefined(key, element)) {
+      return key;
+    }
+  }
+
+  return undefined;
+}
+
+export const RESOURCE_DISPLAYABLE_KEYS = [
+  'id',
+  'innerText',
+  'textContent',
+  'class',
+  'ariaLabel',
+  'ariaDescription',
+  'alt',
+] as const;
+
+export type DisplayableKey = (typeof RESOURCE_DISPLAYABLE_KEYS)[number];
+
+export type ResourceKey = {
+  key: DisplayableKey;
+  value: string;
+};
+
+export const RESOURCE_LINK_KEYS = ['href', 'data-src', 'target', 'action', 'src', 'url'] as const;
+
+export type LinkKey = (typeof RESOURCE_LINK_KEYS)[number];
+
+export type ResourceLink = {
+  key: LinkKey;
+  url: string;
+};
+
+export function findResourceText(element: Resource): ResourceKey | undefined {
+  for (const key of RESOURCE_DISPLAYABLE_KEYS) {
+    const value = element[key];
+    if (value && typeof value === 'string' && value.trim() !== '') return { key, value };
+  }
+
+  return undefined;
+}
+
+export function findResourceLink(element: Resource): ResourceLink | undefined {
+  const key = findDefinedKey(element, [...RESOURCE_LINK_KEYS]);
+  if (!key) {
+    return undefined;
+  }
+
+  const url = element[key];
+  if (url && typeof url === 'string' && url.trim() !== '') return { key, url };
+
+  return undefined;
+}
+
+export type ExternalResource = {
+  text: ResourceKey;
+  link: ResourceLink;
+};
+
+export const isResourceKey = (key: string): key is LinkKey => key in RESOURCE_LINK_KEYS;
+
+export const isKeyDefined = (key: DisplayableKey | LinkKey, element: Resource): boolean =>
+  key in element && element[key] !== undefined;
+
+export type ResourceElement<T, U> = {
+  [K in keyof T]: U extends keyof T[K] ? T[K] : never;
+}[keyof T];
+
+export type Tag = keyof Tags;
+
+export type Resource = HTMLElement & {
+  [K in DisplayableKey | LinkKey]?: string | null;
+};
+
+export type ResourceByName<T extends keyof Tags> = Tags[T];