@dancrumb/web-crawler 1.0.3 → 1.3.0

package/dist/WebCrawler.d.ts

@@ -0,0 +1,44 @@
+import EventEmitter from "node:events";
+type Document = {
+    url: string;
+    canonicalUrl: string;
+    content: string | Buffer;
+    contentType: string;
+    title: string | null;
+};
+export declare class WebCrawler extends EventEmitter<{
+    progress: [progress: string | null];
+}> {
+    private crawled;
+    private uncrawled;
+    private documents;
+    private roots;
+    private maxDepth;
+    private reportProgress;
+    private addUncrawled;
+    private markCrawled;
+    private loadDocument;
+    private getDocument;
+    private isUnderRoot;
+    private getAllLinks;
+    private getCanonical;
+    private resolveLink;
+    private fetchHead;
+    private getContentType;
+    crawlLink(initialLink: string, maxDepth?: number): Promise<string[][]>;
+    clear(): void;
+    addRoot(root: string): Promise<string>;
+    /**
+     * Sets the maximum crawl depth.
+     *
+     * If the number is negative or infinite, it sets the maxDepth to 0
+     * Otherwise, it truncates the number to an integer (if necessary) and sets the maxDepth to that number
+     *
+     * Returns the number that was saved
+     */
+    setMaxDepth(depth: number): number;
+    getCrawledUrls(): string[];
+    getDocuments(): Document[];
+}
+export {};
+//# sourceMappingURL=WebCrawler.d.ts.map

package/dist/WebCrawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebCrawler.d.ts","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AAIA,OAAO,YAAY,MAAM,aAAa,CAAC;AAMvC,KAAK,QAAQ,GAAG;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,CAAC;AAEF,qBAAa,UAAW,SAAQ,YAAY,CAAC;IAC3C,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;CACrC,CAAC;IACA,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,QAAQ,CAAa;IAE7B,OAAO,CAAC,cAAc;IAWtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,WAAW;YAUL,YAAY;YA0BZ,WAAW;IAOzB,OAAO,CAAC,WAAW;YAIL,WAAW;YAoBX,YAAY;YAkBZ,WAAW;YAUX,SAAS;IAiCvB,OAAO,CAAC,cAAc;IAYhB,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAsB;IAwDrE,KAAK;IAOC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAM5C;;;;;;;OAOG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM;IAWzB,cAAc;IAId,YAAY,IAAI,QAAQ,EAAE;CAK3B"}

package/dist/WebCrawler.js

@@ -0,0 +1,230 @@
+import * as cheerio from "cheerio";
+import debug from "debug";
+import assert from "node:assert";
+import EventEmitter from "node:events";
+import { httpGet, httpHead } from "./http-operations.js";
+import { popMap } from "./pop-map.js";
+const logDebug = debug("crawler:main");
+export class WebCrawler extends EventEmitter {
+    crawled = new Map();
+    uncrawled = new Map();
+    documents = new Map();
+    roots = new Set();
+    maxDepth = 0;
+    reportProgress() {
+        if (this.uncrawled.size === 0) {
+            this.emit("progress", null);
+        }
+        const progress = ((this.crawled.size / (this.uncrawled.size + this.crawled.size)) *
+            100).toFixed(1);
+        this.emit("progress", progress);
+    }
+    addUncrawled(url, depth) {
+        this.uncrawled.set(url, depth);
+        this.reportProgress();
+    }
+    markCrawled(url, urlDepth) {
+        let depth = urlDepth ?? this.uncrawled.get(url);
+        if (depth === undefined) {
+            return;
+        }
+        this.crawled.set(url, depth);
+        this.uncrawled.delete(url);
+        this.reportProgress();
+    }
+    async loadDocument(url) {
+        const { data, headers } = await httpGet(url);
+        const contentType = this.getContentType(headers) ?? "application/octet-stream";
+        const content = data;
+        this.documents.set(url, {
+            url,
+            canonicalUrl: "",
+            content,
+            title: null,
+            contentType,
+        });
+        const canonicalUrl = await this.getCanonical(url);
+        if (canonicalUrl !== url) {
+            this.documents.delete(url);
+            this.documents.set(canonicalUrl, {
+                url,
+                canonicalUrl,
+                content,
+                title: null,
+                contentType,
+            });
+        }
+        return data;
+    }
+    async getDocument(url) {
+        if (this.documents.has(url)) {
+            return this.documents.get(url).content;
+        }
+        return this.loadDocument(url);
+    }
+    isUnderRoot(url) {
+        return [...this.roots.values()].some((root) => url.startsWith(root));
+    }
+    async getAllLinks(url) {
+        try {
+            logDebug(`Getting links from ${url}`);
+            const { data } = await httpGet(url);
+            const $ = cheerio.load(data);
+            const anchorLinks = [];
+            for (const anchor of $("a")) {
+                anchorLinks.push(anchor.attribs["href"] ?? "");
+            }
+            return anchorLinks;
+        }
+        catch (e) {
+            logDebug(`Got "${e.message}" while trying to get links`);
+            return [];
+        }
+    }
+    async getCanonical(url) {
+        try {
+            const data = await this.getDocument(url);
+            const $ = cheerio.load(data);
+            for (const link of $("link")) {
+                if (link.attribs["rel"] === "canonical") {
+                    return link.attribs["href"];
+                }
+            }
+            return url;
+        }
+        catch (e) {
+            logDebug(`Got "${e.message}" while trying to get canonical url`);
+            return url;
+        }
+    }
+    async resolveLink(link, root) {
+        let absolute = link;
+        if (link.startsWith("http")) {
+            absolute = link;
+        }
+        else {
+            absolute = new URL(link, root).href.replace(/\/$/, "");
+        }
+        return await this.getCanonical(absolute);
+    }
+    async fetchHead(link) {
+        let linkUrl;
+        try {
+            linkUrl = new URL(link);
+        }
+        catch (e) {
+            logDebug(`${link} is invalid, so skipping`);
+            return null;
+        }
+        if (!linkUrl.protocol.startsWith("http")) {
+            logDebug(`Protocol: '${linkUrl.protocol}' not supported, so skipping`);
+            return null;
+        }
+        logDebug(`HEAD ${link}`);
+        try {
+            const pageHead = await httpHead(link);
+            if (pageHead.status === 200) {
+                return pageHead.headers;
+            }
+            logDebug(`Got ${pageHead.status}: ${pageHead.statusText} when requesting ${link}`);
+        }
+        catch (e) {
+            logDebug(`Got ${e.message} when requesting ${link}`);
+        }
+        return null;
+    }
+    getContentType(headers) {
+        const contentType = headers["content-type"] ?? headers["Content-Type"] ?? null;
+        if (contentType === null) {
+            return contentType;
+        }
+        if (typeof contentType !== "string") {
+            return null;
+        }
+        return contentType;
+    }
+    async crawlLink(initialLink, maxDepth = this.maxDepth) {
+        const root = await this.getCanonical(initialLink);
+        this.addRoot(root);
+        this.maxDepth = maxDepth;
+        let [url, depth] = [root, 0];
+        while (url !== undefined) {
+            assert(depth !== undefined);
+            logDebug("\n*********\n");
+            logDebug(`Crawled: ${this.crawled.size}\nUncrawled: ${this.uncrawled.size}`);
+            logDebug(`URL: ${url}\nDepth:${depth}\n`);
+            if (depth <= maxDepth) {
+                this.markCrawled(url, depth);
+                logDebug(`Crawling ${url}...`);
+                const links = await this.getAllLinks(url);
+                for (const link of links) {
+                    const resolved = await this.resolveLink(link, url);
+                    if (!this.isUnderRoot(resolved)) {
+                        logDebug(`Out of scope: ${resolved} (${root})`);
+                        continue;
+                    }
+                    logDebug(`${resolved} is under one of ${[...this.roots.entries()].join(",")}`);
+                    if ((this.crawled.get(resolved) ?? maxDepth) < depth) {
+                        logDebug(`Already crawled ${resolved} at depth ${this.crawled.get(resolved)}`);
+                        continue;
+                    }
+                    if (!this.uncrawled.has(resolved)) {
+                        const head = await this.fetchHead(resolved);
+                        if (head === null) {
+                            continue;
+                        }
+                        const contentType = this.getContentType(head);
+                        if (contentType === null || !contentType.startsWith("text/html")) {
+                            logDebug(`Page at ${resolved} has MIME type ${contentType}, so skipping`);
+                            continue;
+                        }
+                        logDebug(`Adding ${resolved} to list of uncrawled`);
+                        this.addUncrawled(resolved, depth + 1);
+                    }
+                }
+            }
+            [url, depth] = popMap(this.uncrawled);
+        }
+        return [[...this.crawled.keys()], [...this.uncrawled.keys()]];
+    }
+    clear() {
+        this.crawled.clear();
+        this.uncrawled.clear();
+        this.documents.clear();
+        this.roots.clear();
+    }
+    async addRoot(root) {
+        const canonical = await this.getCanonical(root);
+        this.roots.add(canonical);
+        return canonical;
+    }
+    /**
+     * Sets the maximum crawl depth.
+     *
+     * If the number is negative or infinite, it sets the maxDepth to 0
+     * Otherwise, it truncates the number to an integer (if necessary) and sets the maxDepth to that number
+     *
+     * Returns the number that was saved
+     */
+    setMaxDepth(depth) {
+        if (depth < 0) {
+            this.maxDepth = 0;
+        }
+        else if (Number.isFinite(depth)) {
+            this.maxDepth = Math.trunc(depth);
+        }
+        else {
+            this.maxDepth = 0;
+        }
+        return this.maxDepth;
+    }
+    getCrawledUrls() {
+        return [...this.crawled.keys()];
+    }
+    getDocuments() {
+        return this.getCrawledUrls()
+            .map((url) => this.documents.get(url))
+            .filter((d) => d !== undefined);
+    }
+}
+//# sourceMappingURL=WebCrawler.js.map

package/dist/WebCrawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebCrawler.js","sourceRoot":"","sources":["../src/WebCrawler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,YAAY,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;AAUvC,MAAM,OAAO,UAAW,SAAQ,YAE9B;IACQ,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;IACxC,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1B,QAAQ,GAAW,CAAC,CAAC;IAErB,cAAc;QACpB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,MAAM,QAAQ,GAAG,CACf,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC/D,GAAG,CACJ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAEO,YAAY,CAAC,GAAW,EAAE,KAAa;QAC7C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAEO,WAAW,CAAC,GAAW,EAAE,QAAiB;QAChD,IAAI,KAAK,GAAG,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,cAAc,EAAE,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,0BAA0B,CAAC;QAC7D,MAAM,OAAO,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG;YACH,YAAY,EAAE,EAAE;YAChB,OAAO;YACP,KAAK,EAAE,IAAI;YACX,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,YAAY,KAAK,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,EAAE;gBAC/B,GAAG;gBACH,YAAY;gBACZ,OAAO;gBACP,KAAK,EAAE,IAAI;gBACX,WAAW;aACZ,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,OAAO,CAAC;QAC1C,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEO,WAAW,CAAC,GAAW;QAC7B,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,GAAW;QACnC,IAAI,CAAC;YACH,QAAQ,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;YAEtC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE7B,MAAM,WAAW,GAAa,EAAE,CAAC;YAEjC,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,QAAS,CAAW,CAAC,OAAO,6BAA6B,CAAC,CAAC;YACpE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAW;QACpC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,WAAW,EAAE,CAAC;oBACxC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAE,CAAC;gBAC/B,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CACN,QAAS,CAAW,CAAC,OAAO,qCAAqC,CAClE,CAAC;YACF,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,IAAY;QAClD,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,IAAY;QAEZ,IAAI,OAAY,CAAC;QACjB,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,GAAG,IAAI,0BAA0B,CAAC,CAAC;YAC5C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,cAAc,OAAO,CAAC,QAAQ,8BAA8B,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;YAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,OAAO,QAAQ,CAAC,OAAO,CAAC;YAC1B,CAAC;YACD,QAAQ,CACN,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,oBAAoB,IAAI,EAAE,CACzE,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,QAAQ,CAAC,OAAQ,CAAW,CAAC,OAAO,oBAAoB,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,OAAiC;QACtD,MAAM,WAAW,GACf,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC;QAC7D,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzB,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,WAAmB,EAAE,WAAmB,IAAI,CAAC,QAAQ;QACnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,GAA8C,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAExE,OAAO,GAAG,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;YAC5B,QAAQ,CAAC,eAAe,CAAC,CAAC;YAC1B,QAAQ,CACN,YAAY,IAAI,CAAC,OAAO,CAAC,IAAI,gBAAgB,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CACnE,CAAC;YACF,QAAQ,CAAC,QAAQ,GAAG,WAAW,KAAK,IAAI,CAAC,CAAC;YAC1C,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC7B,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;gBAC/B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBACnD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChC,QAAQ,CAAC,iBAAiB,QAAQ,KAAK,IAAI,GAAG,CAAC,CAAC;wBAChD,SAAS;oBACX,CAAC;oBACD,QAAQ,CACN,GAAG,QAAQ,oBAAoB,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CACrE,CAAC;oBACF,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC;wBACrD,QAAQ,CACN,mBAAmB,QAAQ,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CACrE,CAAC;wBACF,SAAS;oBACX,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAClC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;wBAC5C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;4BAClB,SAAS;wBACX,CAAC;wBACD,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;wBAC9C,IAAI,WAAW,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;4BACjE,QAAQ,CACN,WAAW,QAAQ,kBAAkB,WAAW,eAAe,CAChE,CAAC;4BACF,SAAS;wBACX,CAAC;wBACD,QAAQ,CAAC,UAAU,QAAQ,uBAAuB,CAAC,CAAC;wBACpD,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;YACH,CAAC;YACD,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAY;QACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,KAAa;QACvB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,cAAc;QACZ,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,cAAc,EAAE;aACzB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IACpC,CAAC;CACF"}

package/dist/http-operations.d.ts

@@ -0,0 +1,3 @@
+export declare function httpHead(link: string): Promise<import("axios").AxiosResponse<any, any, {}>>;
+export declare function httpGet(link: string): Promise<import("axios").AxiosResponse<string | Buffer<ArrayBufferLike>, any, {}>>;
+//# sourceMappingURL=http-operations.d.ts.map

package/dist/http-operations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-operations.d.ts","sourceRoot":"","sources":["../src/http-operations.ts"],"names":[],"mappings":"AAKA,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM,wDAW1C;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,qFAczC"}

package/dist/http-operations.js

@@ -0,0 +1,28 @@
+import { default as _axios } from "axios";
+import debug from "debug";
+const logDebug = debug("crawler:http");
+export async function httpHead(link) {
+    const aborter = new AbortController();
+    const abortTimeout = setTimeout(() => {
+        logDebug(`HEAD ${link} Timed Out`);
+        aborter.abort();
+    }, 5000);
+    logDebug(`HEAD ${link}`);
+    const head = await _axios.head(link, { signal: aborter.signal });
+    clearTimeout(abortTimeout);
+    return head;
+}
+export async function httpGet(link) {
+    const aborter = new AbortController();
+    const abortTimeout = setTimeout(() => {
+        logDebug(`GET ${link} Timed Out`);
+        aborter.abort();
+    }, 5000);
+    logDebug(`GET ${link}`);
+    const got = await _axios.get(link, {
+        signal: aborter.signal,
+    });
+    clearTimeout(abortTimeout);
+    return got;
+}
+//# sourceMappingURL=http-operations.js.map

package/dist/http-operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-operations.js","sourceRoot":"","sources":["../src/http-operations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,MAAM,EAAE,MAAM,OAAO,CAAC;AAC1C,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;AAEvC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAEtC,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QACnC,QAAQ,CAAC,QAAQ,IAAI,YAAY,CAAC,CAAC;QACnC,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC,EAAE,IAAI,CAAC,CAAC;IACT,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IACzB,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACjE,YAAY,CAAC,YAAY,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAEtC,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QACnC,QAAQ,CAAC,OAAO,IAAI,YAAY,CAAC,CAAC;QAClC,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC,EAAE,IAAI,CAAC,CAAC;IAET,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IACxB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,CAAkB,IAAI,EAAE;QAClD,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IACH,YAAY,CAAC,YAAY,CAAC,CAAC;IAC3B,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/pop-map.d.ts

@@ -0,0 +1,24 @@
+/**
+ * "Pops" a value from a Map
+ *
+ * This relies on the native ordering of the Map implementation, so it does not guarantee
+ * that this is the least- or most-recent-value.
+ *
+ * It always returns a two-value tuple; either the key and value or undefined, undefined.
+ *
+ * This make it easier on calling code to determine what was returned
+ *
+ * @example
+ * ```
+ * const [key,value] = popMap(someMap);
+ * if(key === defined) {
+ *  // The Map was Empty
+ *}
+ *
+ * // value !== undefined (unless the definition of the map's keys allow for this)
+ *
+ * @param map
+ * @returns
+ */
+export declare function popMap<K, V>(map: Map<K, V>): [K, V] | [undefined, undefined];
+//# sourceMappingURL=pop-map.d.ts.map

package/dist/pop-map.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pop-map.d.ts","sourceRoot":"","sources":["../src/pop-map.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAO5E"}

package/{src/pop-map.ts → dist/pop-map.js}

@@ -20,11 +20,11 @@
  * @param map
  * @returns
  */
-export function popMap<K, V>(map: Map<K, V>): [K, V] | [undefined, undefined] {
-  const entry = map.entries().next().value ?? [undefined, undefined];
-  if (entry[0]) {
-    map.delete(entry[0]);
-  }
-
-  return entry;
+export function popMap(map) {
+    const entry = map.entries().next().value ?? [undefined, undefined];
+    if (entry[0]) {
+        map.delete(entry[0]);
+    }
+    return entry;
 }
+//# sourceMappingURL=pop-map.js.map

package/dist/pop-map.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pop-map.js","sourceRoot":"","sources":["../src/pop-map.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,MAAM,CAAO,GAAc;IACzC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACnE,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACb,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}

package/package.json

@@ -1,9 +1,12 @@
 {
   "name": "@dancrumb/web-crawler",
-  "version": "1.0.3",
+  "version": "1.3.0",
   "description": "",
-  "export": {
-    "./": "./dist/WebCrawler.js"
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    "/": "./dist/WebCrawler.js"
   },
   "scripts": {
     "build": "tsc",

package/.claude/settings.local.json

@@ -1,7 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(pnpm --version)"
-    ]
-  }
-}

package/src/WebCralwer.ts

@@ -1,281 +0,0 @@
-import type { AxiosResponse } from "axios";
-import * as cheerio from "cheerio";
-import debug from "debug";
-import assert from "node:assert";
-import EventEmitter from "node:events";
-import { httpGet, httpHead } from "./http-operations.js";
-import { popMap } from "./pop-map.js";
-
-const logDebug = debug("crawler:main");
-
-type Document = {
-  url: string;
-  canonicalUrl: string;
-  content: string | Buffer;
-  contentType: string;
-  title: string | null;
-};
-
-export class WebCrawler extends EventEmitter<{
-  progress: [progress: string | null];
-}> {
-  private crawled = new Map<string, number>();
-  private uncrawled = new Map<string, number>();
-  private documents = new Map<string, Document>();
-  private roots = new Set<string>();
-  private maxDepth: number = 0;
-
-  private reportProgress() {
-    if (this.uncrawled.size === 0) {
-      this.emit("progress", null);
-    }
-    const progress = (
-      (this.crawled.size / (this.uncrawled.size + this.crawled.size)) *
-      100
-    ).toFixed(1);
-    this.emit("progress", progress);
-  }
-
-  private addUncrawled(url: string, depth: number) {
-    this.uncrawled.set(url, depth);
-    this.reportProgress();
-  }
-
-  private markCrawled(url: string, urlDepth?: number) {
-    let depth = urlDepth ?? this.uncrawled.get(url);
-    if (depth === undefined) {
-      return;
-    }
-    this.crawled.set(url, depth);
-    this.uncrawled.delete(url);
-    this.reportProgress();
-  }
-
-  private async loadDocument(url: string): Promise<string | Buffer> {
-    const { data, headers } = await httpGet(url);
-    const contentType =
-      this.getContentType(headers) ?? "application/octet-stream";
-    const content = data;
-    this.documents.set(url, {
-      url,
-      canonicalUrl: "",
-      content,
-      title: null,
-      contentType,
-    });
-    const canonicalUrl = await this.getCanonical(url);
-    if (canonicalUrl !== url) {
-      this.documents.delete(url);
-      this.documents.set(canonicalUrl, {
-        url,
-        canonicalUrl,
-        content,
-        title: null,
-        contentType,
-      });
-    }
-    return data;
-  }
-
-  private async getDocument(url: string): Promise<string | Buffer> {
-    if (this.documents.has(url)) {
-      return this.documents.get(url)!.content;
-    }
-    return this.loadDocument(url);
-  }
-
-  private isUnderRoot(url: string) {
-    return [...this.roots.values()].some((root) => url.startsWith(root));
-  }
-
-  private async getAllLinks(url: string): Promise<string[]> {
-    try {
-      logDebug(`Getting links from ${url}`);
-
-      const { data } = await httpGet(url);
-      const $ = cheerio.load(data);
-
-      const anchorLinks: string[] = [];
-
-      for (const anchor of $("a")) {
-        anchorLinks.push(anchor.attribs["href"] ?? "");
-      }
-
-      return anchorLinks;
-    } catch (e) {
-      logDebug(`Got "${(e as Error).message}" while trying to get links`);
-      return [];
-    }
-  }
-
-  private async getCanonical(url: string): Promise<string> {
-    try {
-      const data = await this.getDocument(url);
-      const $ = cheerio.load(data);
-      for (const link of $("link")) {
-        if (link.attribs["rel"] === "canonical") {
-          return link.attribs["href"]!;
-        }
-      }
-      return url;
-    } catch (e) {
-      logDebug(
-        `Got "${(e as Error).message}" while trying to get canonical url`,
-      );
-      return url;
-    }
-  }
-
-  private async resolveLink(link: string, root: string) {
-    let absolute = link;
-    if (link.startsWith("http")) {
-      absolute = link;
-    } else {
-      absolute = new URL(link, root).href.replace(/\/$/, "");
-    }
-    return await this.getCanonical(absolute);
-  }
-
-  private async fetchHead(
-    link: string,
-  ): Promise<AxiosResponse["headers"] | null> {
-    let linkUrl: URL;
-    try {
-      linkUrl = new URL(link);
-    } catch (e) {
-      logDebug(`${link} is invalid, so skipping`);
-      return null;
-    }
-
-    if (!linkUrl.protocol.startsWith("http")) {
-      logDebug(`Protocol: '${linkUrl.protocol}' not supported, so skipping`);
-      return null;
-    }
-
-    logDebug(`HEAD ${link}`);
-    try {
-      const pageHead = await httpHead(link);
-
-      if (pageHead.status === 200) {
-        return pageHead.headers;
-      }
-      logDebug(
-        `Got ${pageHead.status}: ${pageHead.statusText} when requesting ${link}`,
-      );
-    } catch (e) {
-      logDebug(`Got ${(e as Error).message} when requesting ${link}`);
-    }
-
-    return null;
-  }
-
-  private getContentType(headers: AxiosResponse["headers"]): string | null {
-    const contentType =
-      headers["content-type"] ?? headers["Content-Type"] ?? null;
-    if (contentType === null) {
-      return contentType;
-    }
-    if (typeof contentType !== "string") {
-      return null;
-    }
-    return contentType;
-  }
-
-  async crawlLink(initialLink: string, maxDepth: number = this.maxDepth) {
-    const root = await this.getCanonical(initialLink);
-    this.addRoot(root);
-    this.maxDepth = maxDepth;
-
-    let [url, depth]: [string, number] | [undefined, undefined] = [root, 0];
-
-    while (url !== undefined) {
-      assert(depth !== undefined);
-      logDebug("\n*********\n");
-      logDebug(
-        `Crawled: ${this.crawled.size}\nUncrawled: ${this.uncrawled.size}`,
-      );
-      logDebug(`URL: ${url}\nDepth:${depth}\n`);
-      if (depth <= maxDepth) {
-        this.markCrawled(url, depth);
-        logDebug(`Crawling ${url}...`);
-        const links = await this.getAllLinks(url);
-        for (const link of links) {
-          const resolved = await this.resolveLink(link, url);
-          if (!this.isUnderRoot(resolved)) {
-            logDebug(`Out of scope: ${resolved} (${root})`);
-            continue;
-          }
-          logDebug(
-            `${resolved} is under one of ${[...this.roots.entries()].join(",")}`,
-          );
-          if ((this.crawled.get(resolved) ?? maxDepth) < depth) {
-            logDebug(
-              `Already crawled ${resolved} at depth ${this.crawled.get(resolved)}`,
-            );
-            continue;
-          }
-          if (!this.uncrawled.has(resolved)) {
-            const head = await this.fetchHead(resolved);
-            if (head === null) {
-              continue;
-            }
-            const contentType = this.getContentType(head);
-            if (contentType === null || !contentType.startsWith("text/html")) {
-              logDebug(
-                `Page at ${resolved} has MIME type ${contentType}, so skipping`,
-              );
-              continue;
-            }
-            logDebug(`Adding ${resolved} to list of uncrawled`);
-            this.addUncrawled(resolved, depth + 1);
-          }
-        }
-      }
-      [url, depth] = popMap(this.uncrawled);
-    }
-
-    return [[...this.crawled.keys()], [...this.uncrawled.keys()]];
-  }
-
-  clear() {
-    this.crawled.clear();
-    this.uncrawled.clear();
-    this.documents.clear();
-    this.roots.clear();
-  }
-
-  async addRoot(root: string): Promise<string> {
-    const canonical = await this.getCanonical(root);
-    this.roots.add(canonical);
-    return canonical;
-  }
-
-  /**
-   * Sets the maximum crawl depth.
-   *
-   * If the number is negative or infinite, it sets the maxDepth to 0
-   * Otherwise, it truncates the number to an integer (if necessary) and sets the maxDepth to that number
-   *
-   * Returns the number that was saved
-   */
-  setMaxDepth(depth: number) {
-    if (depth < 0) {
-      this.maxDepth = 0;
-    } else if (Number.isFinite(depth)) {
-      this.maxDepth = Math.trunc(depth);
-    } else {
-      this.maxDepth = 0;
-    }
-    return this.maxDepth;
-  }
-
-  getCrawledUrls() {
-    return [...this.crawled.keys()];
-  }
-
-  getDocuments(): Document[] {
-    return this.getCrawledUrls()
-      .map((url) => this.documents.get(url))
-      .filter((d) => d !== undefined);
-  }
-}

package/src/http-operations.ts

@@ -1,33 +0,0 @@
-import { default as _axios } from "axios";
-import debug from "debug";
-
-const logDebug = debug("crawler:http");
-
-export async function httpHead(link: string) {
-  const aborter = new AbortController();
-
-  const abortTimeout = setTimeout(() => {
-    logDebug(`HEAD ${link} Timed Out`);
-    aborter.abort();
-  }, 5000);
-  logDebug(`HEAD ${link}`);
-  const head = await _axios.head(link, { signal: aborter.signal });
-  clearTimeout(abortTimeout);
-  return head;
-}
-
-export async function httpGet(link: string) {
-  const aborter = new AbortController();
-
-  const abortTimeout = setTimeout(() => {
-    logDebug(`GET ${link} Timed Out`);
-    aborter.abort();
-  }, 5000);
-
-  logDebug(`GET ${link}`);
-  const got = await _axios.get<string | Buffer>(link, {
-    signal: aborter.signal,
-  });
-  clearTimeout(abortTimeout);
-  return got;
-}

package/src/ndnu.ts

@@ -1,14 +0,0 @@
-import { inspect } from "node:util";
-import { WebCrawler } from "./WebCralwer.js";
-
-const NDNU = "https://ndnu.edu/financial-aid/";
-
-const crawler = new WebCrawler();
-crawler.setMaxDepth(1);
-crawler.on("progress", (progress) =>
-  console.log(`Progress: ${progress ?? "-"}%`),
-);
-
-await crawler.crawlLink(NDNU);
-console.log("Crawled");
-console.log(crawler.getDocuments().map(({ content: _, ...d }) => inspect(d)));

package/src/popMap.test.ts

@@ -1,39 +0,0 @@
-import { beforeEach, describe, expect, it } from "vitest";
-import { popMap } from "./pop-map.js";
-
-describe("popMap", () => {
-  const testMap = new Map<string, number>();
-
-  beforeEach(() => {
-    testMap.clear();
-    testMap.set("a", 1);
-    testMap.set("b", 2);
-    testMap.set("c", 3);
-  });
-
-  it("pops a value from a map", () => {
-    const entry = popMap(testMap);
-    expect(entry).toBeDefined();
-    expect(testMap.size).toBe(2);
-    expect(entry[0]).toBeOneOf(["a", "b", "c"]);
-  });
-
-  it("handles an empty map", () => {
-    popMap(testMap);
-    expect(testMap.size).toBe(2);
-    popMap(testMap);
-    expect(testMap.size).toBe(1);
-    popMap(testMap);
-    expect(testMap.size).toBe(0);
-    popMap(testMap);
-    expect(testMap.size).toBe(0);
-  });
-
-  it("handles additions to the map", () => {
-    popMap(testMap);
-    expect(testMap.size).toBe(2);
-    testMap.set("q", 42);
-    popMap(testMap);
-    expect(testMap.size).toBe(2);
-  });
-});

package/tsconfig.json

@@ -1,37 +0,0 @@
-{
-  "exclude": [
-    "./src/ndnu.ts",
-    "./src/*.test.*",
-    "./dist"
-  ],
-  // Visit https://aka.ms/tsconfig to read more about this file
-  "compilerOptions": {
-    // File Layout
-    "rootDir": "./src",
-    "outDir": "./dist",
-    // Environment Settings
-    // See also https://aka.ms/tsconfig/module
-    "module": "nodenext",
-    "target": "esnext",
-    "types": [
-      "node"
-    ],
-    // For nodejs:
-    // "lib": ["esnext"],
-    // "types": ["node"],
-    // and npm install -D @types/node
-    // Other Outputs
-    "sourceMap": true,
-    "declaration": true,
-    "declarationMap": true,
-    // Stricter Typechecking Options
-    "noUncheckedIndexedAccess": true,
-    "exactOptionalPropertyTypes": true,
-    "strict": true,
-    "verbatimModuleSyntax": true,
-    "isolatedModules": true,
-    "noUncheckedSideEffectImports": true,
-    "moduleDetection": "force",
-    "skipLibCheck": true,
-  }
-}