pagerts 0.4.1 → 1.0.2

package/jest.config.cjs

@@ -1,213 +0,0 @@
-/**
- * For a detailed explanation regarding each configuration property, visit:
- * https://jestjs.io/docs/configuration
- */
-
-/** @type {import('jest').Config} */
-const config = {
-  preset: 'ts-jest',
-  testEnvironment: 'node',
-
-  // Support for ES modules
-  extensionsToTreatAsEsm: ['.ts'],
-
-  // Module name mapper for package.json imports
-  moduleNameMapper: {
-    '^(\\.{1,2}/.*)\\.js$': '$1',
-  },
-
-  // Automatically clear mock calls, instances, contexts and results before every test
-  clearMocks: true,
-
-  // Indicates whether the coverage information should be collected while executing the test
-  collectCoverage: true,
-
-  // The directory where Jest should output its coverage files
-  coverageDirectory: 'coverage',
-
-  // Indicates which provider should be used to instrument code for coverage
-  coverageProvider: 'v8',
-
-  // An array of glob patterns indicating a set of files for which coverage information should be collected
-  collectCoverageFrom: ['src/**/*.ts', '!src/**/*.test.ts', '!src/**/*.spec.ts'],
-
-  // Coverage thresholds - realistic for current state, will improve over time
-  coverageThreshold: {
-    global: {
-      branches: 30,
-      functions: 35,
-      lines: 30,
-      statements: 30,
-    },
-  },
-
-  // Test match patterns
-  testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'],
-
-  // Module file extensions
-  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
-
-  // Transform files with ts-jest
-  transform: {
-    '^.+\\.ts$': [
-      'ts-jest',
-      {
-        useESM: true,
-        tsconfig: {
-          module: 'ES2022',
-          target: 'ES2022',
-          esModuleInterop: true,
-          moduleResolution: 'node',
-        },
-      },
-    ],
-  },
-
-  // Make calling deprecated APIs throw helpful error messages
-  // errorOnDeprecated: false,
-
-  // The default configuration for fake timers
-  // fakeTimers: {
-  //   "enableGlobally": false
-  // },
-
-  // Force coverage collection from ignored files using an array of glob patterns
-  // forceCoverageMatch: [],
-
-  // A path to a module which exports an async function that is triggered once before all test suites
-  // globalSetup: undefined,
-
-  // A path to a module which exports an async function that is triggered once after all test suites
-  // globalTeardown: undefined,
-
-  // A set of global variables that need to be available in all test environments
-  // globals: {},
-
-  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
-  // maxWorkers: "50%",
-
-  // An array of directory names to be searched recursively up from the requiring module's location
-  // moduleDirectories: [
-  //   "node_modules"
-  // ],
-
-  // An array of file extensions your modules use
-  // moduleFileExtensions: [
-  //   "js",
-  //   "mjs",
-  //   "cjs",
-  //   "jsx",
-  //   "ts",
-  //   "tsx",
-  //   "json",
-  //   "node"
-  // ],
-
-  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
-  // moduleNameMapper: {},
-
-  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
-  // modulePathIgnorePatterns: [],
-
-  // Activates notifications for test results
-  // notify: false,
-
-  // An enum that specifies notification mode. Requires { notify: true }
-  // notifyMode: "failure-change",
-
-  // A preset that is used as a base for Jest's configuration
-  // preset: undefined,
-
-  // Run tests from one or more projects
-  // projects: undefined,
-
-  // Use this configuration option to add custom reporters to Jest
-  // reporters: undefined,
-
-  // Automatically reset mock state before every test
-  // resetMocks: false,
-
-  // Reset the module registry before running each individual test
-  // resetModules: false,
-
-  // A path to a custom resolver
-  // resolver: undefined,
-
-  // Automatically restore mock state and implementation before every test
-  // restoreMocks: false,
-
-  // The root directory that Jest should scan for tests and modules within
-  // rootDir: undefined,
-
-  // A list of paths to directories that Jest should use to search for files in
-  // roots: [
-  //   "<rootDir>"
-  // ],
-
-  // Allows you to use a custom runner instead of Jest's default test runner
-  // runner: "jest-runner",
-
-  // The paths to modules that run some code to configure or set up the testing environment before each test
-  // setupFiles: [],
-
-  // A list of paths to modules that run some code to configure or set up the testing framework before each test
-  // setupFilesAfterEnv: [],
-
-  // The number of seconds after which a test is considered as slow and reported as such in the results.
-  // slowTestThreshold: 5,
-
-  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
-  // snapshotSerializers: [],
-
-  // The test environment that will be used for testing
-  // testEnvironment: "jest-environment-node",
-
-  // Options that will be passed to the testEnvironment
-  // testEnvironmentOptions: {},
-
-  // Adds a location field to test results
-  // testLocationInResults: false,
-
-  // The glob patterns Jest uses to detect test files
-  // testMatch: [
-  //   "**/__tests__/**/*.[jt]s?(x)",
-  //   "**/?(*.)+(spec|test).[tj]s?(x)"
-  // ],
-
-  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
-  // testPathIgnorePatterns: [
-  //   "/node_modules/"
-  // ],
-
-  // The regexp pattern or array of patterns that Jest uses to detect test files
-  // testRegex: [],
-
-  // This option allows the use of a custom results processor
-  // testResultsProcessor: undefined,
-
-  // This option allows use of a custom test runner
-  // testRunner: "jest-circus/runner",
-
-  // A map from regular expressions to paths to transformers
-  // transform: undefined,
-
-  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
-  // transformIgnorePatterns: [
-  //   "/node_modules/",
-  //   "\\.pnp\\.[^\\/]+$"
-  // ],
-
-  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
-  // unmockedModulePathPatterns: undefined,
-
-  // Indicates whether each individual test should be reported during the run
-  // verbose: undefined,
-
-  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
-  // watchPathIgnorePatterns: [],
-
-  // Whether to use watchman for file crawling
-  // watchman: true,
-};
-
-module.exports = config;

package/src/__tests__/PageFetcher.test.ts

@@ -1,48 +0,0 @@
-import { PageFetcher } from '../page/PageFetcher';
-
-describe('PageFetcher', () => {
-  let pageFetcher: PageFetcher;
-
-  beforeEach(() => {
-    pageFetcher = new PageFetcher();
-  });
-
-  describe('fetchAll', () => {
-    it('should fetch valid URLs', async () => {
-      const urls = ['https://example.com'];
-      const responses = await pageFetcher.fetchAll(urls);
-
-      expect(responses.length).toBeGreaterThan(0);
-      expect(responses[0].url).toBe('https://example.com');
-    });
-
-    it('should handle invalid URLs gracefully', async () => {
-      const urls = ['https://this-domain-definitely-does-not-exist-12345.com'];
-      const responses = await pageFetcher.fetchAll(urls);
-
-      expect(responses.length).toBeGreaterThan(0);
-      if (responses[0].error) {
-        expect(responses[0].error).toContain('Failed to fetch');
-      }
-    });
-
-    it('should handle multiple URLs', async () => {
-      const urls = ['https://example.com', 'https://example.org'];
-      const responses = await pageFetcher.fetchAll(urls);
-
-      expect(responses.length).toBe(2);
-    });
-
-    it('should have timeout for slow requests', async () => {
-      const slowFetcher = new PageFetcher(100, 0); // 100ms timeout, no retries
-      const urls = ['https://httpbin.org/delay/5']; // This will timeout
-
-      const responses = await slowFetcher.fetchAll(urls);
-      expect(responses.length).toBeGreaterThan(0);
-
-      if (responses[0].error) {
-        expect(responses[0].error).toContain('timeout');
-      }
-    }, 10000);
-  });
-});

package/src/__tests__/security.test.ts

@@ -1,153 +0,0 @@
-import { validateUrl, validateUrls, RateLimiter, sanitizeText } from '../security';
-
-describe('Security Module', () => {
-  describe('validateUrl', () => {
-    it('should validate a proper HTTPS URL', () => {
-      const result = validateUrl('https://example.com');
-      expect(result.isValid).toBe(true);
-      expect(result.sanitizedUrl).toBe('https://example.com/');
-    });
-
-    it('should validate a proper HTTP URL', () => {
-      const result = validateUrl('http://example.com');
-      expect(result.isValid).toBe(true);
-    });
-
-    it('should validate a file:// URL', () => {
-      const result = validateUrl('file:///path/to/file.html');
-      expect(result.isValid).toBe(true);
-    });
-
-    it('should reject empty URLs', () => {
-      const result = validateUrl('');
-      expect(result.isValid).toBe(false);
-      expect(result.error).toContain('empty');
-    });
-
-    it('should reject URLs with javascript: protocol', () => {
-      const result = validateUrl('javascript:alert(1)');
-      expect(result.isValid).toBe(false);
-      expect(result.error).toContain('suspicious');
-    });
-
-    it('should reject URLs with data: protocol', () => {
-      const result = validateUrl('data:text/html,<script>alert(1)</script>');
-      expect(result.isValid).toBe(false);
-      expect(result.error).toContain('suspicious');
-    });
-
-    it('should reject URLs exceeding maximum length', () => {
-      const longUrl = 'https://example.com/' + 'a'.repeat(3000);
-      const result = validateUrl(longUrl);
-      expect(result.isValid).toBe(false);
-      expect(result.error).toContain('exceeds maximum length');
-    });
-
-    it('should reject URLs with script tags', () => {
-      const result = validateUrl('https://example.com/<script>alert(1)</script>');
-      expect(result.isValid).toBe(false);
-      expect(result.error).toContain('suspicious');
-    });
-
-    it('should reject invalid URL formats', () => {
-      const result = validateUrl('not-a-valid-url');
-      expect(result.isValid).toBe(false);
-      expect(result.error).toContain('Invalid URL format');
-    });
-
-    it('should trim whitespace from URLs', () => {
-      const result = validateUrl('  https://example.com  ');
-      expect(result.isValid).toBe(true);
-      expect(result.sanitizedUrl).toBe('https://example.com/');
-    });
-  });
-
-  describe('validateUrls', () => {
-    it('should validate multiple URLs and separate valid from invalid', () => {
-      const urls = ['https://example.com', 'javascript:alert(1)', 'http://test.com', 'invalid-url'];
-      const result = validateUrls(urls);
-
-      expect(result.validUrls.length).toBe(2);
-      expect(result.errors.length).toBe(2);
-      expect(result.validUrls).toContain('https://example.com/');
-      expect(result.validUrls).toContain('http://test.com/');
-    });
-
-    it('should return empty arrays for empty input', () => {
-      const result = validateUrls([]);
-      expect(result.validUrls.length).toBe(0);
-      expect(result.errors.length).toBe(0);
-    });
-  });
-
-  describe('RateLimiter', () => {
-    it('should allow requests within the limit', () => {
-      const limiter = new RateLimiter(5, 1000);
-
-      for (let i = 0; i < 5; i++) {
-        expect(limiter.isAllowed()).toBe(true);
-      }
-    });
-
-    it('should block requests exceeding the limit', () => {
-      const limiter = new RateLimiter(3, 1000);
-
-      // Use up all allowed requests
-      for (let i = 0; i < 3; i++) {
-        limiter.isAllowed();
-      }
-
-      // Next request should be blocked
-      expect(limiter.isAllowed()).toBe(false);
-    });
-
-    it('should reset after the time window', async () => {
-      const limiter = new RateLimiter(2, 100); // 100ms window
-
-      limiter.isAllowed();
-      limiter.isAllowed();
-      expect(limiter.isAllowed()).toBe(false);
-
-      // Wait for window to expire
-      await new Promise((resolve) => setTimeout(resolve, 150));
-
-      // Should be allowed again
-      expect(limiter.isAllowed()).toBe(true);
-    });
-
-    it('should correctly report remaining requests', () => {
-      const limiter = new RateLimiter(5, 1000);
-
-      expect(limiter.getRemainingRequests()).toBe(5);
-      limiter.isAllowed();
-      expect(limiter.getRemainingRequests()).toBe(4);
-      limiter.isAllowed();
-      expect(limiter.getRemainingRequests()).toBe(3);
-    });
-  });
-
-  describe('sanitizeText', () => {
-    it('should sanitize HTML special characters', () => {
-      const input = '<script>alert("XSS")</script>';
-      const output = sanitizeText(input);
-      expect(output).toBe('&lt;script&gt;alert(&quot;XSS&quot;)&lt;&#x2F;script&gt;');
-    });
-
-    it('should handle empty strings', () => {
-      expect(sanitizeText('')).toBe('');
-    });
-
-    it('should escape quotes and apostrophes', () => {
-      const input = `It's a "test"`;
-      const output = sanitizeText(input);
-      expect(output).toContain('&#x27;');
-      expect(output).toContain('&quot;');
-    });
-
-    it('should escape forward slashes', () => {
-      const input = '</script>';
-      const output = sanitizeText(input);
-      expect(output).toBe('&lt;&#x2F;script&gt;');
-    });
-  });
-});

package/src/extractors/AbstractExtractor.ts

@@ -1,4 +0,0 @@
-export abstract class AbstractExtractor<V, R> {
-  constructor(readonly name: string) {}
-  abstract extract(value: V): Promise<R>;
-}

package/src/extractors/PageExtractor.ts

@@ -1,21 +0,0 @@
-import type { Page } from '../page/index.js';
-import { JSDOM } from 'jsdom';
-import { AbstractExtractor } from './AbstractExtractor.js';
-
-export class PageExtractor extends AbstractExtractor<JSDOM, Page> {
-  constructor() {
-    super('page-extractor');
-  }
-
-  async extract(value: JSDOM): Promise<Page> {
-    const {
-      window: {
-        document: {
-          title,
-          location: { href: url },
-        },
-      },
-    } = value;
-    return { title, url };
-  }
-}

package/src/extractors/ResourceExtractor.ts

@@ -1,31 +0,0 @@
-import type { JSDOM } from 'jsdom';
-import {
-  findResourceLink,
-  findResourceText,
-  type ExternalResource,
-  type Resource,
-  type Tag,
-} from '../resource.js';
-import { AbstractExtractor } from './AbstractExtractor.js';
-
-export class ResourceExtractor extends AbstractExtractor<JSDOM, ExternalResource[]> {
-  constructor(private readonly tags: Tag[]) {
-    super('page-extractor');
-  }
-  async extract(value: JSDOM): Promise<ExternalResource[]> {
-    const { document } = value.window;
-    const externalResources: ExternalResource[] = [];
-    for (const tag of this.tags) {
-      const selector = document.querySelectorAll<Resource>(tag);
-      const elements = Array.from(selector);
-      for (const element of elements) {
-        const text = findResourceText(element);
-        const link = findResourceLink(element);
-        if (!text || !link) continue;
-        if (!link.url.startsWith('http')) continue;
-        externalResources.push({ text, link });
-      }
-    }
-    return externalResources;
-  }
-}

package/src/extractors/TagExtractor.ts

@@ -1,13 +0,0 @@
-import { JSDOM } from 'jsdom';
-import type { Resource, Tag } from '../resource.js';
-import { AbstractExtractor } from './AbstractExtractor.js';
-
-export class TagExtractor<T extends Tag> extends AbstractExtractor<JSDOM, Resource[]> {
-  extract(value: JSDOM): Promise<Resource[]> {
-    const linkNodes = value.window.document.querySelectorAll<Resource>(this.tagName);
-    return Promise.resolve(Array.from(linkNodes));
-  }
-  constructor(private readonly tagName: T) {
-    super(`extract <${tagName}>`);
-  }
-}

package/src/extractors/index.ts

@@ -1,4 +0,0 @@
-export { AbstractExtractor } from './AbstractExtractor.js';
-export { PageExtractor } from './PageExtractor.js';
-export { ResourceExtractor } from './ResourceExtractor.js';
-export { TagExtractor } from './TagExtractor.js';

package/src/main.ts

@@ -1,71 +0,0 @@
-#!/usr/bin/env node
-import { Command, createArgument } from 'commander';
-
-import pkg from '../package.json';
-import { PageExtractor, ResourceExtractor } from './extractors/index.js';
-import { PageFetcher, type PageMetadata } from './page/index.js';
-import { JSONStylePrinter } from './printers/index.js';
-import { validateUrls } from './security.js';
-
-const { description, name, version } = pkg;
-
-const program = new Command();
-
-const url = createArgument(
-  '<url | file...>',
-  'remote https://URL or local file://resource.html to extract from'
-);
-
-(async (): Promise<void> => {
-  await program
-    .name(name)
-    .version(version, '-v, --version')
-    .description(description)
-    .addArgument(url)
-    .action(async (urls: string[]) => {
-      try {
-        // Validate URLs first
-        const { validUrls, errors } = validateUrls(urls);
-
-        // Report validation errors
-        if (errors.length > 0) {
-          console.error('\n❌ URL Validation Errors:');
-          errors.forEach(({ url: invalidUrl, error }) => {
-            console.error(`  - ${invalidUrl}: ${error}`);
-          });
-        }
-
-        // Exit if no valid URLs
-        if (validUrls.length === 0) {
-          console.error('\n❌ No valid URLs to process. Exiting.');
-          process.exit(1);
-        }
-
-        console.error(`\n✅ Processing ${validUrls.length} valid URL(s)...`);
-
-        const printer = new JSONStylePrinter();
-        const pageFetcher = new PageFetcher();
-        const pageExtractor = new PageExtractor();
-        const resourceExtractor = new ResourceExtractor(['a', 'meta', 'link', 'embed']);
-
-        const pageResponses = await pageFetcher.fetchAll(validUrls);
-        const pageMetadatas: PageMetadata[] = [];
-
-        for (const { content, url: responseUrl, error } of pageResponses) {
-          const resources =
-            error !== undefined || !content ? [] : await resourceExtractor.extract(content);
-          const descriptor =
-            error !== undefined || !content
-              ? { url: responseUrl, error: error ?? 'Unknown error', resources }
-              : await pageExtractor.extract(content);
-          pageMetadatas.push({ ...descriptor, resources });
-        }
-
-        await printer.print(...pageMetadatas);
-      } catch (error) {
-        console.error('\n❌ An error occurred:', error instanceof Error ? error.message : error);
-        process.exit(1);
-      }
-    })
-    .parseAsync(process.argv);
-})();

package/src/page/Page.ts

@@ -1,24 +0,0 @@
-import type { ExternalResource } from '../resource.js';
-
-type hasTitle = {
-  title: string;
-};
-
-type hasUrl = {
-  url: string;
-};
-
-type hasResources = {
-  resources: ExternalResource[];
-};
-
-export type Page = hasTitle & hasUrl;
-
-export type PageSuccess = Page & hasResources;
-export type PageFailure = hasUrl & hasResources & { error: string };
-export type PageMetadata = PageSuccess | PageFailure;
-
-export const isError = (page: PageMetadata): page is PageFailure => 'error' in page;
-
-export const isPage = (page: PageMetadata): page is PageSuccess =>
-  'title' in page && typeof page.title === 'string' && Array.isArray(page.resources);

package/src/page/PageFetcher.ts

@@ -1,81 +0,0 @@
-import { JSDOM, VirtualConsole } from 'jsdom';
-
-interface PageResponse {
-  url: string;
-  content?: JSDOM;
-  error?: string;
-}
-
-export class PageFetcher {
-  private readonly timeout: number;
-  private readonly maxRetries: number;
-
-  constructor(timeout = 10000, maxRetries = 2) {
-    this.timeout = timeout;
-    this.maxRetries = maxRetries;
-  }
-
-  private async fetchPage(url: string, retryCount = 0): Promise<PageResponse> {
-    const virtualConsole = new VirtualConsole().on('jsdomError', (error: Error) => {
-      process.stderr.write(`Error parsing ${url}: ${error.message}\n`);
-    });
-
-    try {
-      let dom: Promise<JSDOM>;
-
-      if (url.startsWith('file://')) {
-        dom = JSDOM.fromFile(url.substring(7), { virtualConsole });
-      } else {
-        // Add timeout and security options for remote URLs
-        dom = JSDOM.fromURL(url, {
-          virtualConsole,
-          resources: 'usable',
-          runScripts: 'outside-only', // More secure - don't execute page scripts
-          beforeParse(window) {
-            // Prevent infinite loops and resource exhaustion
-            window.setTimeout = (() => {
-              throw new Error('setTimeout disabled for security');
-            }) as typeof window.setTimeout;
-            window.setInterval = (() => {
-              throw new Error('setInterval disabled for security');
-            }) as typeof window.setInterval;
-          },
-        });
-      }
-
-      const content = await Promise.race([
-        dom,
-        new Promise<never>((_, reject) =>
-          setTimeout(() => reject(new Error('Request timeout')), this.timeout)
-        ),
-      ]);
-
-      return { url, content };
-    } catch (error) {
-      const message = error instanceof Error ? error.message : 'Unknown error';
-
-      // Retry logic for transient errors
-      if (retryCount < this.maxRetries && this.isRetryableError(message)) {
-        process.stderr.write(`Retrying ${url} (attempt ${retryCount + 1}/${this.maxRetries})...\n`);
-        await this.delay(1000 * (retryCount + 1)); // Exponential backoff
-        return this.fetchPage(url, retryCount + 1);
-      }
-
-      return { url, error: `Failed to fetch: ${message}` };
-    }
-  }
-
-  private isRetryableError(message: string): boolean {
-    const retryablePatterns = [/timeout/i, /ECONNRESET/i, /ETIMEDOUT/i, /ENOTFOUND/i, /network/i];
-    return retryablePatterns.some((pattern) => pattern.test(message));
-  }
-
-  private delay(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
-  async fetchAll(urls: string[]): Promise<PageResponse[]> {
-    const responses = await Promise.all(urls.map((url) => this.fetchPage(url)));
-    return responses.filter((response) => response.content !== undefined || response.error);
-  }
-}

package/src/page/index.ts

@@ -1,3 +0,0 @@
-export { PageFetcher } from './PageFetcher.js';
-export type { Page, PageMetadata } from './Page.js';
-export { isPage, isError } from './Page.js';

package/src/printers/AbstractResourcePrinter.ts

@@ -1,6 +0,0 @@
-import type { PageMetadata } from '../page/index.js';
-
-export abstract class AbstractResourcePrinter {
-  constructor() {}
-  abstract print(...pages: PageMetadata[]): void | Promise<void>;
-}