jopijs 0.0.1

package/dist/@crawler/core.d.ts

@@ -0,0 +1,87 @@
+import { type OnCrawlingFinishedInfos, type WebSiteCrawlerOptions } from "./common.ts";
+export declare class WebSiteCrawler {
+    private readonly urlDone;
+    private readonly newWebSite_basePath;
+    private readonly newWebSite_lcBasePath;
+    private readonly newWebSite_urlInfos;
+    private readonly requiredPrefix;
+    private readonly requiredPrefix2;
+    private isStarted;
+    private readonly options;
+    private readonly cache?;
+    private currentGroup;
+    private readonly groupStack;
+    private urlCount;
+    /**
+     * Create a new crawler instance.
+     *
+     * @param sourceWebSite
+     *      The url of the website to crawl.
+     * @param options
+     *      Options for complex cases.
+     */
+    constructor(sourceWebSite: string, options?: WebSiteCrawlerOptions);
+    /**
+     * Start the processing
+     */
+    start(entryPoint?: string): Promise<OnCrawlingFinishedInfos>;
+    /**
+     * Take an url and clean this url.
+     * - Resolve relative url.
+     * - Exclude special url ("mailto:", "tel:", ...)
+     * - Exclude anchor url (starts with #).
+     */
+    private _cleanUpUrl;
+    /**
+     * Is like cleanUpUrl but with a special case for CSS.
+     *
+     * Url in CSS is related to the dir of the CSS file.
+     * If I have "myImage.jpg" then it's https//my/css/dir/myImage.jpg.
+     */
+    private cleanUpCssUrl;
+    private cleanUpUrlAux;
+    /**
+     * Is called when we want to add an url to the processing queue.
+     * A call to cleanUpUrl must have been done before.
+     */
+    private pushUrl;
+    private processStack;
+    /**
+     * Will fetch an url and process the result.
+     * If the result is HTML, it will be analyzed.
+     * Also, if it's CSS.
+     */
+    private processGroup;
+    private isResource;
+    private processUrl;
+    /**
+     * Process an HTML file, which consiste:
+     * - Extracting the url.
+     * - Replacing this url inside the HTML to convert them.
+     */
+    private processHtml;
+    /**
+     * Allow rewriting the url from a source site (where we take our pages)
+     *  to transform this url to a local url (our website).
+     */
+    rewriteSourceSiteUrl(url: string): string;
+    /**
+     * Allow transforming an url found by the HTML parser.
+     */
+    transformFoundUrl(url: string, enableRelocatable?: boolean): string;
+    /**
+     * Clean up the url to make it compatible with the file-system.
+     * Will remove the query-string and the anchors part.
+     * And make url relatif (with "../.." as a prefix).
+     *
+     * Why does relatif url are required?
+     *      For example, I have a HTML page: file://folderA/webSiteRoot/myPage/index.html
+     *          And now a css: /my/css/folder/style.css
+     *      Here the final url will be:  file://folderA/webSiteRoot/myPage/my/css/folder/style.css
+     *          and not:                 file://folderA/webSiteRoot/my/css/folder/style.css
+     *      It's why                     my/css/folder/style.css
+     *      must be transformed as    ../my/css/folder/style.css
+     *      (only inside this page)
+     */
+    urlTool_buildFileSystemUrl(url: string): string;
+}

package/dist/@crawler/core.js

@@ -0,0 +1,586 @@
+/// <reference path="../@core/cheerio.d.ts" />
+import { DirectFileCache } from "./directFileCache.js";
+import { UrlMapping } from "./urlMapping.js";
+import * as cheerio from 'cheerio';
+import { getErrorMessage } from "jopi-toolkit/jk_tools";
+// @ts-ignore no ts definition
+import parseCssUrls from "css-url-parser";
+import { applyDefaults, tick } from "./utils.js";
+import { ProcessUrlResult, UrlSortTools } from "./common.js";
+export class WebSiteCrawler {
+    urlDone = [];
+    newWebSite_basePath;
+    newWebSite_lcBasePath;
+    newWebSite_urlInfos;
+    requiredPrefix;
+    requiredPrefix2;
+    isStarted = false;
+    options;
+    cache;
+    currentGroup = { url: "", stack: [] };
+    groupStack = [];
+    urlCount = 1;
+    /**
+     * Create a new crawler instance.
+     *
+     * @param sourceWebSite
+     *      The url of the website to crawl.
+     * @param options
+     *      Options for complex cases.
+     */
+    constructor(sourceWebSite, options) {
+        options = applyDefaults(options, {
+            requireRelocatableUrl: true,
+        });
+        options = this.options = { ...options };
+        let newWebSiteUrl = new URL(options.newWebSiteUrl || sourceWebSite).origin;
+        this.newWebSite_basePath = newWebSiteUrl;
+        this.newWebSite_lcBasePath = newWebSiteUrl.toLowerCase();
+        const urlInfos = new URL(newWebSiteUrl);
+        this.newWebSite_urlInfos = urlInfos;
+        this.requiredPrefix = urlInfos.origin + "/";
+        this.requiredPrefix2 = "//" + urlInfos.hostname;
+        let sourceWebSiteOrigin = new URL(sourceWebSite).origin;
+        if (sourceWebSiteOrigin !== newWebSiteUrl) {
+            if (!options.rewriteThisUrls)
+                options.rewriteThisUrls = [];
+            // Allow rewriting the url.
+            if (!options.rewriteThisUrls.includes(sourceWebSiteOrigin)) {
+                options.rewriteThisUrls.push(sourceWebSiteOrigin);
+            }
+        }
+        if (!options.urlMapping) {
+            options.urlMapping = new UrlMapping(sourceWebSite);
+        }
+        else {
+            if (!options.rewriteThisUrls)
+                options.rewriteThisUrls = [];
+            const knownOrigin = options.urlMapping.getKnownOrigins();
+            knownOrigin.forEach(origin => {
+                if (!options.rewriteThisUrls.includes(origin)) {
+                    options.rewriteThisUrls.push(origin);
+                }
+            });
+        }
+        if (options.cache) {
+            this.cache = options.cache;
+        }
+        else if (options.outputDir) {
+            this.cache = new DirectFileCache(options.outputDir);
+        }
+    }
+    /**
+     * Start the processing
+     */
+    async start(entryPoint) {
+        if (!entryPoint) {
+            entryPoint = this.newWebSite_basePath;
+        }
+        const newGroup = { url: entryPoint, stack: [] };
+        this.groupStack.push(newGroup);
+        this.currentGroup = newGroup;
+        if (this.options.scanThisUrls) {
+            for (let i = 0; i < this.options.scanThisUrls.length; i++) {
+                this.pushUrl(this.options.scanThisUrls[i]);
+            }
+        }
+        await this.processStack();
+        const finishedInfos = {
+            remainingStack: this.groupStack.map(g => g.url)
+        };
+        if (this.options.onFinished) {
+            this.options.onFinished(finishedInfos);
+        }
+        return finishedInfos;
+    }
+    /**
+     * Take an url and clean this url.
+     * - Resolve relative url.
+     * - Exclude special url ("mailto:", "tel:", ...)
+     * - Exclude anchor url (starts with #).
+     */
+    _cleanUpUrl(url) {
+        return this.cleanUpUrlAux(url, false);
+    }
+    /**
+     * Is like cleanUpUrl but with a special case for CSS.
+     *
+     * Url in CSS is related to the dir of the CSS file.
+     * If I have "myImage.jpg" then it's https//my/css/dir/myImage.jpg.
+     */
+    cleanUpCssUrl(url, baseUrl) {
+        return this.cleanUpUrlAux(url, true, baseUrl);
+    }
+    cleanUpUrlAux(url, isCss, currentUrl) {
+        if (!url)
+            return null;
+        url = url.trim();
+        if (!url)
+            return null;
+        if (url[0] === '#')
+            return null;
+        // Convert to an absolute url.
+        if (!url.includes("://")) {
+            if (url[0] === "?") {
+                let currentUrl = this.currentGroup.url;
+                let idx = currentUrl.indexOf("?");
+                if (idx !== -1)
+                    currentUrl = currentUrl.substring(0, idx);
+                url = currentUrl + url;
+            }
+            else if (url.includes(":")) {
+                if (url.startsWith("data:"))
+                    return null;
+                if (url.startsWith("javascript:"))
+                    return null;
+                if (url.startsWith("mailto:"))
+                    return null;
+                if (url.startsWith("tel:"))
+                    return null;
+                if (url.startsWith("sms:"))
+                    return null;
+                if (url.startsWith("ftp:"))
+                    return null;
+            }
+            if (url.startsWith("//")) {
+                if (!url.toLowerCase().startsWith(this.requiredPrefix2))
+                    return null;
+                url = this.newWebSite_urlInfos.protocol + url;
+            }
+            else if (url[0] === "/") {
+                url = resolveRelativeUrl(url, this.newWebSite_urlInfos);
+            }
+            else {
+                if (isCss) {
+                    url = resolveRelativeUrl(url, new URL(currentUrl));
+                }
+                else {
+                    url = resolveRelativeUrl(url, this.newWebSite_urlInfos);
+                }
+            }
+        }
+        else {
+            url = this.rewriteSourceSiteUrl(url);
+        }
+        if (!url.toLowerCase().startsWith(this.requiredPrefix)) {
+            return null;
+        }
+        return url.trim();
+    }
+    /**
+     * Is called when we want to add an url to the processing queue.
+     * A call to cleanUpUrl must have been done before.
+     */
+    pushUrl(url) {
+        if (!url)
+            return "";
+        url = this._cleanUpUrl(url);
+        if (!url)
+            return "";
+        if (this.urlDone.includes(url))
+            return url;
+        this.urlDone.push(url);
+        if (this.options.canDownload) {
+            if (!this.options.canDownload(url.substring(this.requiredPrefix.length), this.isResource(url))) {
+                return url;
+            }
+        }
+        if (!this.currentGroup.stack)
+            this.currentGroup.stack = [];
+        this.currentGroup.stack.push(url);
+        return url;
+    }
+    async processStack() {
+        if (this.isStarted)
+            return;
+        this.isStarted = true;
+        while (true) {
+            const group = this.groupStack.shift();
+            if (!group)
+                break;
+            if (!await this.processGroup(group))
+                break;
+        }
+        this.isStarted = false;
+    }
+    /**
+     * Will fetch an url and process the result.
+     * If the result is HTML, it will be analyzed.
+     * Also, if it's CSS.
+     */
+    async processGroup(group) {
+        this.currentGroup = group;
+        // Process the group main url.
+        const processResponse = await this.processUrl(group.url);
+        // Process the resource inside the group.
+        if (group.stack) {
+            let isResource;
+            let isNotResource;
+            group.stack.forEach(url => {
+                if (this.isResource(url)) {
+                    if (!isResource)
+                        isResource = [];
+                    isResource.push(url);
+                }
+                else {
+                    if (!isNotResource)
+                        isNotResource = [];
+                    isNotResource.push(url);
+                }
+            });
+            group.stack = undefined;
+            // Stack the pages coming from the resources.
+            if (isNotResource) {
+                if ((isNotResource.length > 1) && this.options.sortPagesToDownload) {
+                    const sortTools = new UrlSortTools(isNotResource);
+                    this.options.sortPagesToDownload(sortTools);
+                    isNotResource = sortTools.result();
+                }
+                isNotResource.forEach(url => {
+                    this.groupStack.push({ url });
+                });
+            }
+            // Process the resources now.
+            // Allow the page to be completely loaded.
+            //
+            while (isResource) {
+                const resources = isResource;
+                isResource = undefined;
+                for (let i = 0; i < resources.length; i++) {
+                    const resUrl = resources[i];
+                    const resState = await this.processUrl(resUrl);
+                    if (this.options.onResourceDownloaded) {
+                        this.options.onResourceDownloaded(resUrl, resState);
+                    }
+                }
+                // Come from CSS.
+                if (group.stack) {
+                    isResource = group.stack;
+                    group.stack = undefined;
+                }
+            }
+        }
+        if (this.options.onPageFullyDownloaded) {
+            const res = this.options.onPageFullyDownloaded(group.url, processResponse);
+            if (res instanceof Promise)
+                await res;
+            if (res === false)
+                return false;
+        }
+        return true;
+    }
+    isResource(u) {
+        const url = new URL(u);
+        u = url.pathname;
+        let idx = u.lastIndexOf(".");
+        if (idx === -1)
+            return false;
+        let ext = u.substring(idx);
+        return gExtensionForResourceType.includes(ext);
+    }
+    async processUrl(sourceUrl) {
+        const sendSignal = (state) => {
+            if (this.options.onUrlProcessed) {
+                const date = Date.now();
+                const elapsed = date - now;
+                const cacheKey = this.cache?.getKey(transformedUrl);
+                this.options.onUrlProcessed({
+                    sourceUrl, requestedByUrl,
+                    state, retryCount,
+                    transformedUrl,
+                    localUrl,
+                    cacheKey,
+                    urlCount: this.urlCount,
+                    date, elapsed
+                });
+            }
+            return state;
+        };
+        let retryCount = 0;
+        const localUrl = sourceUrl.substring(this.newWebSite_basePath.length);
+        const now = Date.now();
+        const partialUrl = sourceUrl.substring(this.newWebSite_basePath.length);
+        const requestedByUrl = this.currentGroup.url;
+        const mappingResult = this.options.urlMapping.resolveURL(partialUrl);
+        if (!mappingResult)
+            return ProcessUrlResult.IGNORED;
+        let transformedUrl = sourceUrl;
+        if (this.cache) {
+            transformedUrl = this.transformFoundUrl(sourceUrl, false);
+        }
+        if (this.cache && this.options.canIgnoreIfAlreadyCrawled) {
+            const isInCache = await this.cache.hasInCache(transformedUrl, requestedByUrl);
+            if (isInCache && this.options.canIgnoreIfAlreadyCrawled(sourceUrl.substring(this.newWebSite_basePath.length), { sourceUrl: mappingResult.url })) {
+                return sendSignal(ProcessUrlResult.IGNORED);
+            }
+        }
+        if (mappingResult.wakeUpServer) {
+            await mappingResult.wakeUpServer();
+        }
+        this.urlCount++;
+        if (this.options.pauseDuration_ms) {
+            await tick(this.options.pauseDuration_ms);
+        }
+        while (true) {
+            try {
+                let res;
+                if (this.options.doFetch) {
+                    res = await this.options.doFetch(this, mappingResult.url, requestedByUrl);
+                }
+                else {
+                    // noinspection JSUnusedGlobalSymbols
+                    res = await fetch(mappingResult.url, {
+                        // > This option allows avoiding SSL certificate check.
+                        // @ts-ignore
+                        rejectUnauthorized: false,
+                        requestCert: false,
+                        tls: {
+                            rejectUnauthorized: false,
+                            checkServerIdentity: () => { return undefined; }
+                        },
+                        // Allow avoiding automatic redirections.
+                        // @ts-ignore
+                        redirect: 'manual',
+                        headers: {
+                            "referer": requestedByUrl
+                        }
+                        //verbose: true
+                    });
+                }
+                if (res.status !== 200) {
+                    if (res.status >= 300 && res.status < 400) {
+                        const location = res.headers.get("Location");
+                        if (location)
+                            this.pushUrl(location);
+                        return sendSignal(ProcessUrlResult.REDIRECTED);
+                    }
+                    else {
+                        let canContinue = false;
+                        if (this.options.onInvalidResponseCodeFound) {
+                            let what = this.options.onInvalidResponseCodeFound(sourceUrl, retryCount, res);
+                            if (what instanceof Promise)
+                                what = await what;
+                            canContinue = what;
+                        }
+                        else if (retryCount < 3) {
+                            // Retry 3 times, with a longer pause each time.
+                            await tick(1000 * retryCount);
+                            canContinue = true;
+                        }
+                        if (!canContinue) {
+                            return sendSignal(ProcessUrlResult.ERROR);
+                        }
+                        retryCount++;
+                        // Will retry automatically.
+                        continue;
+                    }
+                }
+                const contentType = res.headers.get("content-type");
+                if (contentType) {
+                    if (contentType.startsWith("text/html")) {
+                        let html = await res.text();
+                        if (this.options.rewriteHtmlBeforeProcessing) {
+                            let res = this.options.rewriteHtmlBeforeProcessing(html, sourceUrl.substring(this.newWebSite_basePath.length), mappingResult.url);
+                            if (res instanceof Promise)
+                                res = await res;
+                            html = res;
+                        }
+                        html = await this.processHtml(html);
+                        if (this.options.rewriteHtmlBeforeStoring) {
+                            let res = this.options.rewriteHtmlBeforeStoring(html, sourceUrl.substring(this.newWebSite_basePath.length), mappingResult.url);
+                            if (res instanceof Promise)
+                                res = await res;
+                            html = res;
+                        }
+                        res = new Response(html, { status: 200, headers: res.headers });
+                    }
+                    else if (contentType.startsWith("text/css")) {
+                        const content = await res.text();
+                        const cssUrls = parseCssUrls(content);
+                        if (cssUrls.length) {
+                            cssUrls.forEach(u => {
+                                const cleanedUrl = this.cleanUpCssUrl(u, sourceUrl);
+                                if (cleanedUrl)
+                                    this.pushUrl(cleanedUrl);
+                            });
+                        }
+                        res = new Response(content, { status: 200, headers: res.headers });
+                    }
+                }
+                if (this.cache) {
+                    let hRes = new Response(res.body, { status: res.status, headers: res.headers });
+                    await this.cache.addToCache(transformedUrl, hRes, requestedByUrl);
+                }
+                return sendSignal(ProcessUrlResult.OK);
+            }
+            catch (e) {
+                console.error("Crawler - Error while fetching:", sourceUrl);
+                console.error("|--> Message:", getErrorMessage(e));
+                console.log(e);
+                return sendSignal(ProcessUrlResult.ERROR);
+            }
+        }
+    }
+    /**
+     * Process an HTML file, which consiste:
+     * - Extracting the url.
+     * - Replacing this url inside the HTML to convert them.
+     */
+    async processHtml(html) {
+        // Extract all url and rewrite them inside the html.
+        // Will emit calls to addUrl for each url found.
+        const $ = cheerio.load(html);
+        $("img, script, iframe, source").each((_i, node) => {
+            let url = node.attribs["src"];
+            if (url) {
+                url = this.pushUrl(url);
+                if (url.length)
+                    node.attribs["src"] = this.transformFoundUrl(url);
+            }
+        });
+        $("a, link").each((_i, node) => {
+            let url = node.attribs["href"];
+            if (url) {
+                url = this.pushUrl(url);
+                if (url.length) {
+                    node.attribs["href"] = this.transformFoundUrl(url);
+                }
+            }
+        });
+        $("img").each((_i, node) => {
+            let srcset = node.attribs["srcset"];
+            if (!srcset)
+                return;
+            const parts = srcset.split(",");
+            let newSrcset = "";
+            parts.forEach(p => {
+                p = p.trim();
+                const idx = p.indexOf(" ");
+                if (idx === -1)
+                    return;
+                let url = p.substring(0, idx);
+                const size = p.substring(idx + 1);
+                let newUrl = this.pushUrl(url);
+                if (url.length)
+                    url = newUrl;
+                url = this.transformFoundUrl(url);
+                newSrcset += "," + url + " " + size;
+            });
+            node.attribs["srcset"] = newSrcset.substring(1);
+        });
+        html = $.html();
+        // Security for residual urls.
+        if (html.includes(this.newWebSite_basePath + "/"))
+            html = html.replaceAll(this.newWebSite_basePath, "");
+        if (html.includes(this.newWebSite_basePath))
+            html = html.replaceAll(this.newWebSite_basePath, "/");
+        return html;
+    }
+    /**
+     * Allow rewriting the url from a source site (where we take our pages)
+     *  to transform this url to a local url (our website).
+     */
+    rewriteSourceSiteUrl(url) {
+        if (this.options.rewriteThisUrls) {
+            for (let i = 0; i < this.options.rewriteThisUrls.length; i++) {
+                const prefix = this.options.rewriteThisUrls[i];
+                if (url.startsWith(prefix)) {
+                    url = this.newWebSite_basePath + url.substring(prefix.length);
+                    return url;
+                }
+            }
+        }
+        return url;
+    }
+    /**
+     * Allow transforming an url found by the HTML parser.
+     */
+    transformFoundUrl(url, enableRelocatable = true) {
+        if (this.options.transformUrl) {
+            url = this.options.transformUrl(url, {
+                crawler: this,
+                comeFromPage: this.currentGroup.url,
+                requireRelocatableUrl: this.options.requireRelocatableUrl
+            });
+        }
+        if (enableRelocatable && this.options.requireRelocatableUrl) {
+            url = this.urlTool_buildFileSystemUrl(url);
+        }
+        return url;
+    }
+    /**
+     * Clean up the url to make it compatible with the file-system.
+     * Will remove the query-string and the anchors part.
+     * And make url relatif (with "../.." as a prefix).
+     *
+     * Why does relatif url are required?
+     *      For example, I have a HTML page: file://folderA/webSiteRoot/myPage/index.html
+     *          And now a css: /my/css/folder/style.css
+     *      Here the final url will be:  file://folderA/webSiteRoot/myPage/my/css/folder/style.css
+     *          and not:                 file://folderA/webSiteRoot/my/css/folder/style.css
+     *      It's why                     my/css/folder/style.css
+     *      must be transformed as    ../my/css/folder/style.css
+     *      (only inside this page)
+     */
+    urlTool_buildFileSystemUrl(url) {
+        // Allow to not always check.
+        if (!this.options.requireRelocatableUrl)
+            return url;
+        let idx = url.indexOf("?");
+        if (idx !== -1)
+            url = url.substring(0, idx);
+        idx = url.indexOf("#");
+        if (idx !== -1)
+            url = url.substring(0, idx);
+        // > If not a file, then it a directory.
+        //   Transform it to be a /index.html file.
+        if (url.endsWith("/")) {
+            url += "index.html";
+        }
+        else {
+            const lastSlash = url.lastIndexOf("/");
+            const lastSegment = lastSlash === -1 ? url : url.substring(lastSlash + 1);
+            if (!lastSegment.includes(".")) {
+                url += "/index.html";
+            }
+        }
+        // Make the url relatif.
+        //
+        if (url.startsWith(this.newWebSite_lcBasePath)) {
+            url = url.substring(this.newWebSite_lcBasePath.length + 1);
+            let currentUrl = this.currentGroup.url.substring(this.newWebSite_lcBasePath.length + 1);
+            if (!currentUrl)
+                return url;
+            if (url === currentUrl)
+                return url;
+            let backCount = currentUrl.split("/").length;
+            if (currentUrl.endsWith("/"))
+                backCount--;
+            for (let i = 0; i < backCount; i++)
+                url = "../" + url;
+        }
+        return url;
+    }
+}
+function resolveRelativeUrl(url, baseUrl) {
+    if (url[0] === "/") {
+        if (url[1] === "/") {
+            const urlInfos = new URL(url);
+            urlInfos.protocol = baseUrl.protocol;
+            urlInfos.port = baseUrl.port;
+            return urlInfos.toString();
+        }
+        else {
+            return baseUrl.toString() + url.substring(1);
+        }
+    }
+    else if (url[0] === ".") {
+        return new URL(url, baseUrl).toString();
+    }
+    return url;
+}
+const gExtensionForResourceType = [
+    ".css", ".js", ".jpg", ".png", ".jpeg", ".gif", ".svg", ".webp",
+    ".woff", ".woff2", ".ttf", ".txt", ".avif", ".ico"
+];
+//# sourceMappingURL=core.js.map