pagerts 0.1.0

package/jest.config.js

@@ -0,0 +1,198 @@
+/**
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+/** @type {import('jest').Config} */
+const config = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/tmp/jest_rs",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  // clearMocks: false,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  collectCoverage: true,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  coverageDirectory: "coverage",
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: "v8",
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "mjs",
+  //   "cjs",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "json",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: undefined,
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  // setupFilesAfterEnv: [],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  // testEnvironment: "jest-environment-node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).[tj]s?(x)"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  // transform: undefined,
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/",
+  //   "\\.pnp\\.[^\\/]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};
+
+module.exports = config;

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "pagerts",
+  "description": "A tool for viewing external relations in a webpage",
+  "version": "0.1.0",
+  "main": "./bin/main.js",
+  "bin": {
+    "pagerts": "./bin/main.js"
+  },
+  "scripts": {
+    "test": "jest",
+    "build": "esbuild src/main.ts --external:jsdom --bundle --outdir=bin --minify --sourcemap --platform=node",
+    "tscbuild": "tsc",
+    "start": "node ./bin/main.js",
+    "dev": "npx tsx src/main.ts"
+  },
+  "keywords": [
+    "webpage",
+    "hierarchy",
+    "management"
+  ],
+  "author": "Kirill kn253 Nevzorov",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/akinevz0/treepage/issues"
+  },
+  "homepage": "https://github.com/akinevz0/treepage",
+  "dependencies": {
+    "blessed": "^0.1.81",
+    "commander": "^12.1.0",
+    "dotenv": "^16.4.5",
+    "jsdom": "^26.0.0"
+  },
+  "devDependencies": {
+    "@types/blessed": "^0.1.25",
+    "@types/jsdom": "^21.1.7",
+    "@types/node": "^22.8.2",
+    "esbuild": "^0.25.1",
+    "ts-node": "^10.9.2"
+  }
+}

package/src/extractors/AbstractExtractor.ts

@@ -0,0 +1,5 @@
+
+export abstract class AbstractExtractor<V, R> {
+    constructor(readonly name:string) { }
+    abstract extract(value: V): Promise<R>;
+}

package/src/extractors/PageExtractor.ts

@@ -0,0 +1,15 @@
+import { Page } from '../page/Page';
+import type { PageDescriptor } from '../page/PageMetadata';
+import { AbstractExtractor } from './AbstractExtractor';
+
+export class PageExtractor extends AbstractExtractor<Page, PageDescriptor> {
+    constructor() { super("page-extractor"); }
+
+    async extract(value: Page): Promise<PageDescriptor> {
+        const { title, location: { href: url } } = value.window.document
+        return {
+            url,
+            title,
+        };
+    }
+}

package/src/extractors/ResourceExtractor.ts

@@ -0,0 +1,25 @@
+import { Page } from '../page/Page';
+import { findResourceLink, findResourceText, type ExternalResource, type Resource, type Tag } from "../resource";
+import { AbstractExtractor } from './AbstractExtractor';
+
+export class ResourceExtractor extends AbstractExtractor<Page, ExternalResource[]> {
+    constructor(private readonly tags: Tag[]) {
+        super("page-extractor");
+    }
+    async extract(value: Page): Promise<ExternalResource[]> {
+        const { document } = value.window;
+        const externalResources: ExternalResource[] = [];
+        for (const tag of this.tags) {
+            const selector = document.querySelectorAll<Resource>(tag)
+            const elements = Array.from(selector)
+            for (const element of elements) {
+                const text = findResourceText(element);
+                const link = findResourceLink(element);
+                if(!text || !link) continue
+                const externalResource: ExternalResource = {text,link}
+                externalResources.push(externalResource)
+            }
+        }
+        return externalResources;
+    }
+}

package/src/extractors/TagExtractor.ts

@@ -0,0 +1,14 @@
+import { JSDOM } from 'jsdom';
+import type { Resource, Tag } from '../resource';
+import { AbstractExtractor } from './AbstractExtractor';
+
+export class TagExtractor<T extends Tag> extends AbstractExtractor<JSDOM, Resource[]> {
+    extract(value: JSDOM): Promise<Resource[]> {
+        const linkNodes = value.window.document.querySelectorAll<Resource>(this.tagName);
+        return Promise.resolve(Array.from(linkNodes));
+    }
+    constructor(private readonly tagName: T) {
+        super(`extract <${tagName}>`)
+    };
+
+}

package/src/main.ts

@@ -0,0 +1,37 @@
+import { Command, createArgument } from "commander";
+
+import { description, name, version } from '../package.json';
+import { PageExtractor } from "./extractors/PageExtractor";
+import { ResourceExtractor } from "./extractors/ResourceExtractor";
+import { PageFetcher } from "./page/PageFetcher";
+import type { PageMetadata } from "./page/PageMetadata";
+import { JSONStylePrinter } from "./printers/JSONStylePrinter";
+
+const program = new Command();
+
+const url = createArgument("<url|file...>", "remote URL or local file to extract remote resources from");
+
+(async () => {
+  await program
+    .name(name)
+    .version(version, "-v, --version")
+    .description(description)
+    .addArgument(url)
+    .action(async (urls: string[]) => {
+      const printer = new JSONStylePrinter();
+      const pageFetcher = new PageFetcher()
+      const pageExtractor = new PageExtractor()
+      const resourceExtractor = new ResourceExtractor(["a", "meta", "link", "embed"])
+      const pagesFetched = await pageFetcher.fetchAll(urls);
+      const metadataPages:PageMetadata[] = []
+      for (const page of pagesFetched) {
+        const resources = await resourceExtractor.extract(page);
+        const descriptor = await pageExtractor.extract(page);
+        metadataPages.push({
+            ...descriptor, resources
+          });
+      }
+      await printer.print(...metadataPages);
+    })
+    .parseAsync(process.argv);
+})();

package/src/page/Page.ts

@@ -0,0 +1,3 @@
+import { JSDOM } from 'jsdom';
+
+export type Page = JSDOM

package/src/page/PageFetcher.ts

@@ -0,0 +1,16 @@
+import { JSDOM, VirtualConsole } from 'jsdom';
+
+export class PageFetcher {
+    async fetchPage(url: string): Promise<JSDOM> {
+        const dom = await JSDOM.fromURL(url, {
+            virtualConsole: new VirtualConsole().on('jsdomError', (error) => {
+                console.error(`Error parsing ${url}:`, error.message);
+            })
+        });
+        return dom;
+    }
+    async fetchAll(urls:string[]): Promise<JSDOM[]> {
+        return await Promise.all(urls.map(url => this.fetchPage(url)));
+    }
+    constructor() {}
+}

package/src/page/PageMetadata.ts

@@ -0,0 +1,15 @@
+import type { ExternalResource } from "../resource";
+
+type hasTitle = {
+    title: string;
+};
+
+type hasUrl = {
+    url: string;
+};
+
+export type PageDescriptor = hasTitle & hasUrl;
+
+export type PageMetadata = {
+    resources: ExternalResource[];
+} & PageDescriptor;

package/src/printers/AbstractResourcePrinter.ts

@@ -0,0 +1,6 @@
+import type { PageMetadata } from "../page/PageMetadata";
+
+export abstract class AbstractResourcePrinter {
+    constructor() {  }
+    abstract print(...pages: PageMetadata[]): void | Promise<void>;
+}

package/src/printers/JSONStylePrinter.ts

@@ -0,0 +1,12 @@
+import type { PageMetadata } from "../page/PageMetadata";
+import { AbstractResourcePrinter } from "./AbstractResourcePrinter";
+
+
+export class JSONStylePrinter extends AbstractResourcePrinter {
+    print(...pages: PageMetadata[]): void | Promise<void> {
+        const json = JSON.stringify(pages);
+        process.stdout.write(json + "\n")
+    }
+   
+
+}

package/src/printers/LogStylePrinter.ts

@@ -0,0 +1,23 @@
+import type { PageMetadata } from '../page/PageMetadata';
+import { AbstractResourcePrinter } from './AbstractResourcePrinter';
+
+export class LogStylePrinter extends AbstractResourcePrinter {
+
+    write(str: string): void {
+        process.stdout.write(str)
+    }
+
+    async print(...pages: PageMetadata[]): Promise<void> {
+        for (const page of pages) {
+            const { resources, title, url } = page
+
+            this.write(`Title: ${title}\n`)
+            this.write(`URL: ${url}\n\n`)
+
+            for (const resource of resources) {
+                const { link: { url }, text: { value } } = resource
+                this.write(`${value}: ${url}\n`)
+            }
+        }
+    }
+}

package/src/resource.ts

@@ -0,0 +1,96 @@
+/**
+ * @license MIT
+ * We are interested in visualising a page as a collection of tags.
+ * 
+ * We wish to work with tags that can be compactly previewed on a webpage.
+ * Here we must declare all of the element types that can be used to represent
+ * a resource that can be hyperlinked off a webpage.
+ */
+
+type Tags = HTMLElementTagNameMap
+
+function findDefinedKey(element: Resource, keys: LinkKey[]): LinkKey | undefined {
+    for (const key of keys) {
+        if (isKeyDefined(key, element)) {
+            return key;
+        }
+    }
+}
+
+export const RESOURCE_DISPLAYABLE_KEYS = [
+    'id',
+    'innerText',
+    'textContent',
+    'class',
+    'ariaLabel',
+    'ariaDescription',
+    'alt',
+    'rel'
+] as const;
+
+export type DisplayableKey = (typeof RESOURCE_DISPLAYABLE_KEYS)[number];
+
+export type ResourceKey = {
+    key: DisplayableKey;
+    value: string;
+};
+
+export const RESOURCE_LINK_KEYS = [
+    "href",
+    "data-src",
+    "target",
+    "action",
+    "src",
+    "url"
+] as const;
+
+export type LinkKey = typeof RESOURCE_LINK_KEYS[number];
+
+export type ResourceLink = {
+    key: LinkKey;
+    url: string;
+}
+
+export function findResourceText(element: Resource): ResourceKey | undefined {
+    for (const key of RESOURCE_DISPLAYABLE_KEYS) {
+        const value = element[key]
+        if (value && typeof value === 'string' && value.trim() !== '')
+            return { key, value };
+    }
+}
+
+export function findResourceLink(element: Resource): ResourceLink | undefined {
+    const key = findDefinedKey(element, [...RESOURCE_LINK_KEYS]);
+    const url = element[key];
+    if (url && typeof url === 'string' && url.trim() !== '')
+        return { key, url };
+}
+
+export type ExternalResource = {
+    text: ResourceKey;
+    link: ResourceLink;
+};
+
+export const isResourceKey = (key: string): key is LinkKey => key in RESOURCE_LINK_KEYS;
+
+export const isKeyDefined = <E extends Tags[keyof Tags]>(key: string, element: E): boolean =>
+    key in element && element[key] !== undefined;
+
+export type ResourceElement<T, U> = {
+    [K in keyof T]: U extends keyof T[K] ? T[K] : never
+}[keyof T];
+
+export type Tag = keyof Tags
+
+export type Resource = ResourceElement<Tags, (typeof RESOURCE_LINK_KEYS)[number]>;
+
+export type ResourceByName<T extends keyof Tags> = Tags[T]
+
+/** tests **/
+
+type test1 = HTMLAnchorElement extends Resource ? true : false // true
+type test2 = HTMLImageElement extends Resource ? true : false // true
+type test3 = HTMLDivElement extends Resource ? true : false // false
+
+type test4 = ResourceElement<Tags, "src">
+

package/tsconfig.json

@@ -0,0 +1,12 @@
+{
+    "compilerOptions": {
+        "module": "NodeNext",
+        "target": "ESNext",
+        "resolveJsonModule": true,
+        "outDir": "dist",
+        "sourceMap": true,
+    },
+    "include": [
+        "src/**.*",
+    ],
+}