@dancrumb/web-crawler 1.0.0

package/.claude/settings.local.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(pnpm --version)"
+    ]
+  }
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@dancrumb/web-crawler",
+  "version": "1.0.0",
+  "description": "",
+  "export": {
+    "./": "./dist/WebCrawler.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest --run"
+  },
+  "keywords": [],
+  "author": "Dan Rumney<dancrumb@gmail.com>",
+  "license": "MIT",
+  "type": "module",
+  "devDependencies": {
+    "@types/debug": "^4.1.13",
+    "@types/node": "^25.9.1",
+    "git-cz": "^4.9.0",
+    "supports-color": "^10.2.2",
+    "tsx": "^4.22.3",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.7"
+  },
+  "dependencies": {
+    "@dancrumb/fpish": "^8.0.0",
+    "axios": "^1.16.1",
+    "cheerio": "^1.2.0",
+    "debug": "^4.4.3",
+    "fpish": "^3.4.3"
+  },
+  "config": {
+    "commitizen": {
+      "path": "./node_modules/git-cz"
+    }
+  }
+}

package/src/WebCralwer.ts

@@ -0,0 +1,281 @@
+import type { AxiosResponse } from "axios";
+import * as cheerio from "cheerio";
+import debug from "debug";
+import assert from "node:assert";
+import EventEmitter from "node:events";
+import { httpGet, httpHead } from "./http-operations.js";
+import { popMap } from "./pop-map.js";
+
+const logDebug = debug("crawler:main");
+
+type Document = {
+  url: string;
+  canonicalUrl: string;
+  content: string | Buffer;
+  contentType: string;
+  title: string | null;
+};
+
+export class WebCrawler extends EventEmitter<{
+  progress: [progress: string | null];
+}> {
+  private crawled = new Map<string, number>();
+  private uncrawled = new Map<string, number>();
+  private documents = new Map<string, Document>();
+  private roots = new Set<string>();
+  private maxDepth: number = 0;
+
+  private reportProgress() {
+    if (this.uncrawled.size === 0) {
+      this.emit("progress", null);
+    }
+    const progress = (
+      (this.crawled.size / (this.uncrawled.size + this.crawled.size)) *
+      100
+    ).toFixed(1);
+    this.emit("progress", progress);
+  }
+
+  private addUncrawled(url: string, depth: number) {
+    this.uncrawled.set(url, depth);
+    this.reportProgress();
+  }
+
+  private markCrawled(url: string, urlDepth?: number) {
+    let depth = urlDepth ?? this.uncrawled.get(url);
+    if (depth === undefined) {
+      return;
+    }
+    this.crawled.set(url, depth);
+    this.uncrawled.delete(url);
+    this.reportProgress();
+  }
+
+  private async loadDocument(url: string): Promise<string | Buffer> {
+    const { data, headers } = await httpGet(url);
+    const contentType =
+      this.getContentType(headers) ?? "application/octet-stream";
+    const content = data;
+    this.documents.set(url, {
+      url,
+      canonicalUrl: "",
+      content,
+      title: null,
+      contentType,
+    });
+    const canonicalUrl = await this.getCanonical(url);
+    if (canonicalUrl !== url) {
+      this.documents.delete(url);
+      this.documents.set(canonicalUrl, {
+        url,
+        canonicalUrl,
+        content,
+        title: null,
+        contentType,
+      });
+    }
+    return data;
+  }
+
+  private async getDocument(url: string): Promise<string | Buffer> {
+    if (this.documents.has(url)) {
+      return this.documents.get(url)!.content;
+    }
+    return this.loadDocument(url);
+  }
+
+  private isUnderRoot(url: string) {
+    return [...this.roots.values()].some((root) => url.startsWith(root));
+  }
+
+  private async getAllLinks(url: string): Promise<string[]> {
+    try {
+      logDebug(`Getting links from ${url}`);
+
+      const { data } = await httpGet(url);
+      const $ = cheerio.load(data);
+
+      const anchorLinks: string[] = [];
+
+      for (const anchor of $("a")) {
+        anchorLinks.push(anchor.attribs["href"] ?? "");
+      }
+
+      return anchorLinks;
+    } catch (e) {
+      logDebug(`Got "${(e as Error).message}" while trying to get links`);
+      return [];
+    }
+  }
+
+  private async getCanonical(url: string): Promise<string> {
+    try {
+      const data = await this.getDocument(url);
+      const $ = cheerio.load(data);
+      for (const link of $("link")) {
+        if (link.attribs["rel"] === "canonical") {
+          return link.attribs["href"]!;
+        }
+      }
+      return url;
+    } catch (e) {
+      logDebug(
+        `Got "${(e as Error).message}" while trying to get canonical url`,
+      );
+      return url;
+    }
+  }
+
+  private async resolveLink(link: string, root: string) {
+    let absolute = link;
+    if (link.startsWith("http")) {
+      absolute = link;
+    } else {
+      absolute = new URL(link, root).href.replace(/\/$/, "");
+    }
+    return await this.getCanonical(absolute);
+  }
+
+  private async fetchHead(
+    link: string,
+  ): Promise<AxiosResponse["headers"] | null> {
+    let linkUrl: URL;
+    try {
+      linkUrl = new URL(link);
+    } catch (e) {
+      logDebug(`${link} is invalid, so skipping`);
+      return null;
+    }
+
+    if (!linkUrl.protocol.startsWith("http")) {
+      logDebug(`Protocol: '${linkUrl.protocol}' not supported, so skipping`);
+      return null;
+    }
+
+    logDebug(`HEAD ${link}`);
+    try {
+      const pageHead = await httpHead(link);
+
+      if (pageHead.status === 200) {
+        return pageHead.headers;
+      }
+      logDebug(
+        `Got ${pageHead.status}: ${pageHead.statusText} when requesting ${link}`,
+      );
+    } catch (e) {
+      logDebug(`Got ${(e as Error).message} when requesting ${link}`);
+    }
+
+    return null;
+  }
+
+  private getContentType(headers: AxiosResponse["headers"]): string | null {
+    const contentType =
+      headers["content-type"] ?? headers["Content-Type"] ?? null;
+    if (contentType === null) {
+      return contentType;
+    }
+    if (typeof contentType !== "string") {
+      return null;
+    }
+    return contentType;
+  }
+
+  async crawlLink(initialLink: string, maxDepth: number = this.maxDepth) {
+    const root = await this.getCanonical(initialLink);
+    this.addRoot(root);
+    this.maxDepth = maxDepth;
+
+    let [url, depth]: [string, number] | [undefined, undefined] = [root, 0];
+
+    while (url !== undefined) {
+      assert(depth !== undefined);
+      logDebug("\n*********\n");
+      logDebug(
+        `Crawled: ${this.crawled.size}\nUncrawled: ${this.uncrawled.size}`,
+      );
+      logDebug(`URL: ${url}\nDepth:${depth}\n`);
+      if (depth <= maxDepth) {
+        this.markCrawled(url, depth);
+        logDebug(`Crawling ${url}...`);
+        const links = await this.getAllLinks(url);
+        for (const link of links) {
+          const resolved = await this.resolveLink(link, url);
+          if (!this.isUnderRoot(resolved)) {
+            logDebug(`Out of scope: ${resolved} (${root})`);
+            continue;
+          }
+          logDebug(
+            `${resolved} is under one of ${[...this.roots.entries()].join(",")}`,
+          );
+          if ((this.crawled.get(resolved) ?? maxDepth) < depth) {
+            logDebug(
+              `Already crawled ${resolved} at depth ${this.crawled.get(resolved)}`,
+            );
+            continue;
+          }
+          if (!this.uncrawled.has(resolved)) {
+            const head = await this.fetchHead(resolved);
+            if (head === null) {
+              continue;
+            }
+            const contentType = this.getContentType(head);
+            if (contentType === null || !contentType.startsWith("text/html")) {
+              logDebug(
+                `Page at ${resolved} has MIME type ${contentType}, so skipping`,
+              );
+              continue;
+            }
+            logDebug(`Adding ${resolved} to list of uncrawled`);
+            this.addUncrawled(resolved, depth + 1);
+          }
+        }
+      }
+      [url, depth] = popMap(this.uncrawled);
+    }
+
+    return [[...this.crawled.keys()], [...this.uncrawled.keys()]];
+  }
+
+  clear() {
+    this.crawled.clear();
+    this.uncrawled.clear();
+    this.documents.clear();
+    this.roots.clear();
+  }
+
+  async addRoot(root: string): Promise<string> {
+    const canonical = await this.getCanonical(root);
+    this.roots.add(canonical);
+    return canonical;
+  }
+
+  /**
+   * Sets the maximum crawl depth.
+   *
+   * If the number is negative or infinite, it sets the maxDepth to 0
+   * Otherwise, it truncates the number to an integer (if necessary) and sets the maxDepth to that number
+   *
+   * Returns the number that was saved
+   */
+  setMaxDepth(depth: number) {
+    if (depth < 0) {
+      this.maxDepth = 0;
+    } else if (Number.isFinite(depth)) {
+      this.maxDepth = Math.trunc(depth);
+    } else {
+      this.maxDepth = 0;
+    }
+    return this.maxDepth;
+  }
+
+  getCrawledUrls() {
+    return [...this.crawled.keys()];
+  }
+
+  getDocuments(): Document[] {
+    return this.getCrawledUrls()
+      .map((url) => this.documents.get(url))
+      .filter((d) => d !== undefined);
+  }
+}

package/src/http-operations.ts

@@ -0,0 +1,33 @@
+import { default as _axios } from "axios";
+import debug from "debug";
+
+const logDebug = debug("crawler:http");
+
+export async function httpHead(link: string) {
+  const aborter = new AbortController();
+
+  const abortTimeout = setTimeout(() => {
+    logDebug(`HEAD ${link} Timed Out`);
+    aborter.abort();
+  }, 5000);
+  logDebug(`HEAD ${link}`);
+  const head = await _axios.head(link, { signal: aborter.signal });
+  clearTimeout(abortTimeout);
+  return head;
+}
+
+export async function httpGet(link: string) {
+  const aborter = new AbortController();
+
+  const abortTimeout = setTimeout(() => {
+    logDebug(`GET ${link} Timed Out`);
+    aborter.abort();
+  }, 5000);
+
+  logDebug(`GET ${link}`);
+  const got = await _axios.get<string | Buffer>(link, {
+    signal: aborter.signal,
+  });
+  clearTimeout(abortTimeout);
+  return got;
+}

package/src/ndnu.ts

@@ -0,0 +1,14 @@
+import { inspect } from "node:util";
+import { WebCrawler } from "./WebCralwer.js";
+
+const NDNU = "https://ndnu.edu/financial-aid/";
+
+const crawler = new WebCrawler();
+crawler.setMaxDepth(1);
+crawler.on("progress", (progress) =>
+  console.log(`Progress: ${progress ?? "-"}%`),
+);
+
+await crawler.crawlLink(NDNU);
+console.log("Crawled");
+console.log(crawler.getDocuments().map(({ content: _, ...d }) => inspect(d)));

package/src/pop-map.ts

@@ -0,0 +1,30 @@
+/**
+ * "Pops" a value from a Map
+ *
+ * This relies on the native ordering of the Map implementation, so it does not guarantee
+ * that this is the least- or most-recent-value.
+ *
+ * It always returns a two-value tuple; either the key and value or undefined, undefined.
+ *
+ * This make it easier on calling code to determine what was returned
+ *
+ * @example
+ * ```
+ * const [key,value] = popMap(someMap);
+ * if(key === defined) {
+ *  // The Map was Empty
+ *}
+ *
+ * // value !== undefined (unless the definition of the map's keys allow for this)
+ *
+ * @param map
+ * @returns
+ */
+export function popMap<K, V>(map: Map<K, V>): [K, V] | [undefined, undefined] {
+  const entry = map.entries().next().value ?? [undefined, undefined];
+  if (entry[0]) {
+    map.delete(entry[0]);
+  }
+
+  return entry;
+}

package/src/popMap.test.ts

@@ -0,0 +1,39 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import { popMap } from "./pop-map.js";
+
+describe("popMap", () => {
+  const testMap = new Map<string, number>();
+
+  beforeEach(() => {
+    testMap.clear();
+    testMap.set("a", 1);
+    testMap.set("b", 2);
+    testMap.set("c", 3);
+  });
+
+  it("pops a value from a map", () => {
+    const entry = popMap(testMap);
+    expect(entry).toBeDefined();
+    expect(testMap.size).toBe(2);
+    expect(entry[0]).toBeOneOf(["a", "b", "c"]);
+  });
+
+  it("handles an empty map", () => {
+    popMap(testMap);
+    expect(testMap.size).toBe(2);
+    popMap(testMap);
+    expect(testMap.size).toBe(1);
+    popMap(testMap);
+    expect(testMap.size).toBe(0);
+    popMap(testMap);
+    expect(testMap.size).toBe(0);
+  });
+
+  it("handles additions to the map", () => {
+    popMap(testMap);
+    expect(testMap.size).toBe(2);
+    testMap.set("q", 42);
+    popMap(testMap);
+    expect(testMap.size).toBe(2);
+  });
+});

package/tsconfig.json

@@ -0,0 +1,37 @@
+{
+  "exclude": [
+    "./src/ndnu.ts",
+    "./src/*.test.*",
+    "./dist"
+  ],
+  // Visit https://aka.ms/tsconfig to read more about this file
+  "compilerOptions": {
+    // File Layout
+    "rootDir": "./src",
+    "outDir": "./dist",
+    // Environment Settings
+    // See also https://aka.ms/tsconfig/module
+    "module": "nodenext",
+    "target": "esnext",
+    "types": [
+      "node"
+    ],
+    // For nodejs:
+    // "lib": ["esnext"],
+    // "types": ["node"],
+    // and npm install -D @types/node
+    // Other Outputs
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+    // Stricter Typechecking Options
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "strict": true,
+    "verbatimModuleSyntax": true,
+    "isolatedModules": true,
+    "noUncheckedSideEffectImports": true,
+    "moduleDetection": "force",
+    "skipLibCheck": true,
+  }
+}