pagerts 0.2.0 → 0.4.1

package/eslint.config.mjs

@@ -0,0 +1,83 @@
+import eslint from '@eslint/js';
+import tseslint from '@typescript-eslint/eslint-plugin';
+import tsparser from '@typescript-eslint/parser';
+import security from 'eslint-plugin-security';
+import prettier from 'eslint-config-prettier';
+
+export default [
+  {
+    ignores: ['bin/**', 'coverage/**', 'node_modules/**'],
+  },
+  eslint.configs.recommended,
+  {
+    files: ['src/**/*.ts'],
+    languageOptions: {
+      parser: tsparser,
+      parserOptions: {
+        ecmaVersion: 2022,
+        sourceType: 'module',
+        project: './tsconfig.eslint.json',
+      },
+      globals: {
+        console: 'readonly',
+        process: 'readonly',
+        __dirname: 'readonly',
+        __filename: 'readonly',
+        Buffer: 'readonly',
+      },
+    },
+    plugins: {
+      '@typescript-eslint': tseslint,
+      security: security,
+    },
+    rules: {
+      // Disable base JS rules in favor of TS-aware equivalents
+      'no-unused-vars': 'off',
+      'no-undef': 'off',
+
+      // TypeScript rules
+      '@typescript-eslint/no-explicit-any': 'error',
+      '@typescript-eslint/explicit-function-return-type': 'warn',
+      '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+      '@typescript-eslint/no-non-null-assertion': 'error',
+      '@typescript-eslint/prefer-nullish-coalescing': 'warn',
+      '@typescript-eslint/prefer-optional-chain': 'warn',
+
+      // Security rules
+      'security/detect-object-injection': 'warn',
+      'security/detect-non-literal-regexp': 'warn',
+      'security/detect-unsafe-regex': 'error',
+      'security/detect-buffer-noassert': 'error',
+      'security/detect-child-process': 'warn',
+      'security/detect-disable-mustache-escape': 'error',
+      'security/detect-eval-with-expression': 'error',
+      'security/detect-no-csrf-before-method-override': 'error',
+      'security/detect-non-literal-fs-filename': 'warn',
+      'security/detect-non-literal-require': 'warn',
+      'security/detect-possible-timing-attacks': 'warn',
+      'security/detect-pseudoRandomBytes': 'error',
+
+      // General rules
+      'no-console': ['warn', { allow: ['warn', 'error'] }],
+      'no-debugger': 'error',
+      'no-eval': 'error',
+      'no-implied-eval': 'error',
+      'no-new-func': 'error',
+      'prefer-const': 'error',
+      'no-var': 'error',
+    },
+  },
+  {
+    files: ['src/__tests__/**/*.ts'],
+    languageOptions: {
+      globals: {
+        describe: 'readonly',
+        it: 'readonly',
+        expect: 'readonly',
+        beforeEach: 'readonly',
+        setTimeout: 'readonly',
+      },
+    },
+  },
+  prettier,
+];

package/{jest.config.js → jest.config.cjs}

@@ -5,48 +5,63 @@
 
 /** @type {import('jest').Config} */
 const config = {
-  // All imported modules in your tests should be mocked automatically
-  // automock: false,
+  preset: 'ts-jest',
+  testEnvironment: 'node',
 
-  // Stop running tests after `n` failures
-  // bail: 0,
+  // Support for ES modules
+  extensionsToTreatAsEsm: ['.ts'],
 
-  // The directory where Jest should store its cached dependency information
-  // cacheDirectory: "/tmp/jest_rs",
+  // Module name mapper for package.json imports
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
 
   // Automatically clear mock calls, instances, contexts and results before every test
-  // clearMocks: false,
+  clearMocks: true,
 
   // Indicates whether the coverage information should be collected while executing the test
   collectCoverage: true,
 
-  // An array of glob patterns indicating a set of files for which coverage information should be collected
-  // collectCoverageFrom: undefined,
-
   // The directory where Jest should output its coverage files
-  coverageDirectory: "coverage",
-
-  // An array of regexp pattern strings used to skip coverage collection
-  // coveragePathIgnorePatterns: [
-  //   "/node_modules/"
-  // ],
+  coverageDirectory: 'coverage',
 
   // Indicates which provider should be used to instrument code for coverage
-  coverageProvider: "v8",
+  coverageProvider: 'v8',
 
-  // A list of reporter names that Jest uses when writing coverage reports
-  // coverageReporters: [
-  //   "json",
-  //   "text",
-  //   "lcov",
-  //   "clover"
-  // ],
-
-  // An object that configures minimum threshold enforcement for coverage results
-  // coverageThreshold: undefined,
-
-  // A path to a custom dependency extractor
-  // dependencyExtractor: undefined,
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  collectCoverageFrom: ['src/**/*.ts', '!src/**/*.test.ts', '!src/**/*.spec.ts'],
+
+  // Coverage thresholds - realistic for current state, will improve over time
+  coverageThreshold: {
+    global: {
+      branches: 30,
+      functions: 35,
+      lines: 30,
+      statements: 30,
+    },
+  },
+
+  // Test match patterns
+  testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'],
+
+  // Module file extensions
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+
+  // Transform files with ts-jest
+  transform: {
+    '^.+\\.ts$': [
+      'ts-jest',
+      {
+        useESM: true,
+        tsconfig: {
+          module: 'ES2022',
+          target: 'ES2022',
+          esModuleInterop: true,
+          moduleResolution: 'node',
+        },
+      },
+    ],
+  },
 
   // Make calling deprecated APIs throw helpful error messages
   // errorOnDeprecated: false,

package/package.json

@@ -1,22 +1,36 @@
 {
   "name": "pagerts",
   "description": "A tool for viewing external relations in a webpage",
-  "version": "0.2.0",
+  "version": "0.4.1",
   "main": "main.js",
   "bin": {
     "pagerts": "bin/main.js"
   },
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "scripts": {
-    "test": "jest",
-    "build": "esbuild src/main.ts --external:jsdom --bundle --outdir=bin --minify --sourcemap --platform=node",
-    "lint": "tsc",
+    "test": "jest --coverage",
+    "test:watch": "jest --watch",
+    "build": "esbuild src/main.ts --external:jsdom --bundle --outdir=bin --minify --sourcemap --platform=node --format=esm",
+    "lint": "eslint src/**/*.ts",
+    "lint:fix": "eslint src/**/*.ts --fix",
+    "type-check": "tsc --noEmit",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "format:check": "prettier --check \"src/**/*.ts\"",
+    "security:audit": "npm audit --audit-level=moderate",
+    "security:check": "npm run security:audit && npm run lint",
     "start": "node ./bin/main.js",
-    "dev": "npx tsx src/main.ts"
+    "dev": "tsx src/main.ts",
+    "prepare": "npm run build"
   },
   "keywords": [
     "webpage",
     "hierarchy",
-    "management"
+    "management",
+    "web-scraping",
+    "cli",
+    "url-extraction"
   ],
   "author": "Kirill kn253 Nevzorov",
   "license": "MIT",
@@ -25,16 +39,23 @@
   },
   "homepage": "https://github.com/akinevz0/pagerts",
   "dependencies": {
-    "blessed": "^0.1.81",
     "commander": "^12.1.0",
-    "dotenv": "^16.4.5",
-    "jsdom": "^26.0.0"
+    "jsdom": "^25.0.1"
   },
   "devDependencies": {
-    "@types/blessed": "^0.1.25",
+    "@types/jest": "^29.5.14",
     "@types/jsdom": "^21.1.7",
-    "@types/node": "^22.8.2",
+    "@types/node": "^22.10.5",
+    "@typescript-eslint/eslint-plugin": "^8.20.0",
+    "@typescript-eslint/parser": "^8.20.0",
     "esbuild": "^0.25.1",
-    "ts-node": "^10.9.2"
+    "eslint": "^9.18.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-security": "^3.0.1",
+    "jest": "^29.7.0",
+    "prettier": "^3.4.2",
+    "ts-jest": "^29.2.5",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2"
   }
-}
+}

package/src/__tests__/PageFetcher.test.ts

@@ -0,0 +1,48 @@
+import { PageFetcher } from '../page/PageFetcher';
+
+describe('PageFetcher', () => {
+  let pageFetcher: PageFetcher;
+
+  beforeEach(() => {
+    pageFetcher = new PageFetcher();
+  });
+
+  describe('fetchAll', () => {
+    it('should fetch valid URLs', async () => {
+      const urls = ['https://example.com'];
+      const responses = await pageFetcher.fetchAll(urls);
+
+      expect(responses.length).toBeGreaterThan(0);
+      expect(responses[0].url).toBe('https://example.com');
+    });
+
+    it('should handle invalid URLs gracefully', async () => {
+      const urls = ['https://this-domain-definitely-does-not-exist-12345.com'];
+      const responses = await pageFetcher.fetchAll(urls);
+
+      expect(responses.length).toBeGreaterThan(0);
+      if (responses[0].error) {
+        expect(responses[0].error).toContain('Failed to fetch');
+      }
+    });
+
+    it('should handle multiple URLs', async () => {
+      const urls = ['https://example.com', 'https://example.org'];
+      const responses = await pageFetcher.fetchAll(urls);
+
+      expect(responses.length).toBe(2);
+    });
+
+    it('should have timeout for slow requests', async () => {
+      const slowFetcher = new PageFetcher(100, 0); // 100ms timeout, no retries
+      const urls = ['https://httpbin.org/delay/5']; // This will timeout
+
+      const responses = await slowFetcher.fetchAll(urls);
+      expect(responses.length).toBeGreaterThan(0);
+
+      if (responses[0].error) {
+        expect(responses[0].error).toContain('timeout');
+      }
+    }, 10000);
+  });
+});

package/src/__tests__/security.test.ts

@@ -0,0 +1,153 @@
+import { validateUrl, validateUrls, RateLimiter, sanitizeText } from '../security';
+
+describe('Security Module', () => {
+  describe('validateUrl', () => {
+    it('should validate a proper HTTPS URL', () => {
+      const result = validateUrl('https://example.com');
+      expect(result.isValid).toBe(true);
+      expect(result.sanitizedUrl).toBe('https://example.com/');
+    });
+
+    it('should validate a proper HTTP URL', () => {
+      const result = validateUrl('http://example.com');
+      expect(result.isValid).toBe(true);
+    });
+
+    it('should validate a file:// URL', () => {
+      const result = validateUrl('file:///path/to/file.html');
+      expect(result.isValid).toBe(true);
+    });
+
+    it('should reject empty URLs', () => {
+      const result = validateUrl('');
+      expect(result.isValid).toBe(false);
+      expect(result.error).toContain('empty');
+    });
+
+    it('should reject URLs with javascript: protocol', () => {
+      const result = validateUrl('javascript:alert(1)');
+      expect(result.isValid).toBe(false);
+      expect(result.error).toContain('suspicious');
+    });
+
+    it('should reject URLs with data: protocol', () => {
+      const result = validateUrl('data:text/html,<script>alert(1)</script>');
+      expect(result.isValid).toBe(false);
+      expect(result.error).toContain('suspicious');
+    });
+
+    it('should reject URLs exceeding maximum length', () => {
+      const longUrl = 'https://example.com/' + 'a'.repeat(3000);
+      const result = validateUrl(longUrl);
+      expect(result.isValid).toBe(false);
+      expect(result.error).toContain('exceeds maximum length');
+    });
+
+    it('should reject URLs with script tags', () => {
+      const result = validateUrl('https://example.com/<script>alert(1)</script>');
+      expect(result.isValid).toBe(false);
+      expect(result.error).toContain('suspicious');
+    });
+
+    it('should reject invalid URL formats', () => {
+      const result = validateUrl('not-a-valid-url');
+      expect(result.isValid).toBe(false);
+      expect(result.error).toContain('Invalid URL format');
+    });
+
+    it('should trim whitespace from URLs', () => {
+      const result = validateUrl('  https://example.com  ');
+      expect(result.isValid).toBe(true);
+      expect(result.sanitizedUrl).toBe('https://example.com/');
+    });
+  });
+
+  describe('validateUrls', () => {
+    it('should validate multiple URLs and separate valid from invalid', () => {
+      const urls = ['https://example.com', 'javascript:alert(1)', 'http://test.com', 'invalid-url'];
+      const result = validateUrls(urls);
+
+      expect(result.validUrls.length).toBe(2);
+      expect(result.errors.length).toBe(2);
+      expect(result.validUrls).toContain('https://example.com/');
+      expect(result.validUrls).toContain('http://test.com/');
+    });
+
+    it('should return empty arrays for empty input', () => {
+      const result = validateUrls([]);
+      expect(result.validUrls.length).toBe(0);
+      expect(result.errors.length).toBe(0);
+    });
+  });
+
+  describe('RateLimiter', () => {
+    it('should allow requests within the limit', () => {
+      const limiter = new RateLimiter(5, 1000);
+
+      for (let i = 0; i < 5; i++) {
+        expect(limiter.isAllowed()).toBe(true);
+      }
+    });
+
+    it('should block requests exceeding the limit', () => {
+      const limiter = new RateLimiter(3, 1000);
+
+      // Use up all allowed requests
+      for (let i = 0; i < 3; i++) {
+        limiter.isAllowed();
+      }
+
+      // Next request should be blocked
+      expect(limiter.isAllowed()).toBe(false);
+    });
+
+    it('should reset after the time window', async () => {
+      const limiter = new RateLimiter(2, 100); // 100ms window
+
+      limiter.isAllowed();
+      limiter.isAllowed();
+      expect(limiter.isAllowed()).toBe(false);
+
+      // Wait for window to expire
+      await new Promise((resolve) => setTimeout(resolve, 150));
+
+      // Should be allowed again
+      expect(limiter.isAllowed()).toBe(true);
+    });
+
+    it('should correctly report remaining requests', () => {
+      const limiter = new RateLimiter(5, 1000);
+
+      expect(limiter.getRemainingRequests()).toBe(5);
+      limiter.isAllowed();
+      expect(limiter.getRemainingRequests()).toBe(4);
+      limiter.isAllowed();
+      expect(limiter.getRemainingRequests()).toBe(3);
+    });
+  });
+
+  describe('sanitizeText', () => {
+    it('should sanitize HTML special characters', () => {
+      const input = '<script>alert("XSS")</script>';
+      const output = sanitizeText(input);
+      expect(output).toBe('&lt;script&gt;alert(&quot;XSS&quot;)&lt;&#x2F;script&gt;');
+    });
+
+    it('should handle empty strings', () => {
+      expect(sanitizeText('')).toBe('');
+    });
+
+    it('should escape quotes and apostrophes', () => {
+      const input = `It's a "test"`;
+      const output = sanitizeText(input);
+      expect(output).toContain('&#x27;');
+      expect(output).toContain('&quot;');
+    });
+
+    it('should escape forward slashes', () => {
+      const input = '</script>';
+      const output = sanitizeText(input);
+      expect(output).toBe('&lt;&#x2F;script&gt;');
+    });
+  });
+});

package/src/extractors/AbstractExtractor.ts

@@ -1,5 +1,4 @@
-
-export abstract class AbstractExtractor<V, R> {
-    constructor(readonly name:string) { }
-    abstract extract(value: V): Promise<R>;
-}
+export abstract class AbstractExtractor<V, R> {
+  constructor(readonly name: string) {}
+  abstract extract(value: V): Promise<R>;
+}

package/src/extractors/PageExtractor.ts

@@ -1,12 +1,21 @@
-import { isError, type Page } from '../page/Page';
-import { JSDOM } from 'jsdom';
-import { AbstractExtractor } from './AbstractExtractor';
-
-export class PageExtractor extends AbstractExtractor<JSDOM, Page> {
-    constructor() { super("page-extractor"); }
-
-    async extract(value: JSDOM): Promise<Page> {
-        const { window: { document: { title, location: { href: url } } } } = value
-        return { title, url }
-    }
-}
+import type { Page } from '../page/index.js';
+import { JSDOM } from 'jsdom';
+import { AbstractExtractor } from './AbstractExtractor.js';
+
+export class PageExtractor extends AbstractExtractor<JSDOM, Page> {
+  constructor() {
+    super('page-extractor');
+  }
+
+  async extract(value: JSDOM): Promise<Page> {
+    const {
+      window: {
+        document: {
+          title,
+          location: { href: url },
+        },
+      },
+    } = value;
+    return { title, url };
+  }
+}

package/src/extractors/ResourceExtractor.ts

@@ -1,25 +1,31 @@
-import type { JSDOM } from "jsdom";
-import { findResourceLink, findResourceText, type ExternalResource, type Resource, type Tag } from "../resource";
-import { AbstractExtractor } from './AbstractExtractor';
-
-export class ResourceExtractor extends AbstractExtractor<JSDOM, ExternalResource[]> {
-    constructor(private readonly tags: Tag[]) {
-        super("page-extractor");
-    }
-    async extract(value: JSDOM): Promise<ExternalResource[]> {
-        const { document } = value.window;
-        const externalResources: ExternalResource[] = [];
-        for (const tag of this.tags) {
-            const selector = document.querySelectorAll<Resource>(tag)
-            const elements = Array.from(selector)
-            for (const element of elements) {
-                const text = findResourceText(element);
-                const link = findResourceLink(element);
-                if(!text || !link) continue
-                if (!link.url.startsWith("http")) continue
-                externalResources.push({ text, link })
-            }
-        }
-        return externalResources;
-    }
-}
+import type { JSDOM } from 'jsdom';
+import {
+  findResourceLink,
+  findResourceText,
+  type ExternalResource,
+  type Resource,
+  type Tag,
+} from '../resource.js';
+import { AbstractExtractor } from './AbstractExtractor.js';
+
+export class ResourceExtractor extends AbstractExtractor<JSDOM, ExternalResource[]> {
+  constructor(private readonly tags: Tag[]) {
+    super('page-extractor');
+  }
+  async extract(value: JSDOM): Promise<ExternalResource[]> {
+    const { document } = value.window;
+    const externalResources: ExternalResource[] = [];
+    for (const tag of this.tags) {
+      const selector = document.querySelectorAll<Resource>(tag);
+      const elements = Array.from(selector);
+      for (const element of elements) {
+        const text = findResourceText(element);
+        const link = findResourceLink(element);
+        if (!text || !link) continue;
+        if (!link.url.startsWith('http')) continue;
+        externalResources.push({ text, link });
+      }
+    }
+    return externalResources;
+  }
+}

package/src/extractors/TagExtractor.ts

@@ -1,14 +1,13 @@
-import { JSDOM } from 'jsdom';
-import type { Resource, Tag } from '../resource';
-import { AbstractExtractor } from './AbstractExtractor';
-
-export class TagExtractor<T extends Tag> extends AbstractExtractor<JSDOM, Resource[]> {
-    extract(value: JSDOM): Promise<Resource[]> {
-        const linkNodes = value.window.document.querySelectorAll<Resource>(this.tagName);
-        return Promise.resolve(Array.from(linkNodes));
-    }
-    constructor(private readonly tagName: T) {
-        super(`extract <${tagName}>`)
-    };
-
-}
+import { JSDOM } from 'jsdom';
+import type { Resource, Tag } from '../resource.js';
+import { AbstractExtractor } from './AbstractExtractor.js';
+
+export class TagExtractor<T extends Tag> extends AbstractExtractor<JSDOM, Resource[]> {
+  extract(value: JSDOM): Promise<Resource[]> {
+    const linkNodes = value.window.document.querySelectorAll<Resource>(this.tagName);
+    return Promise.resolve(Array.from(linkNodes));
+  }
+  constructor(private readonly tagName: T) {
+    super(`extract <${tagName}>`);
+  }
+}

package/src/extractors/index.ts

@@ -0,0 +1,4 @@
+export { AbstractExtractor } from './AbstractExtractor.js';
+export { PageExtractor } from './PageExtractor.js';
+export { ResourceExtractor } from './ResourceExtractor.js';
+export { TagExtractor } from './TagExtractor.js';