jopijs 0.0.1

package/src/@core/serverImpl/server_nodejs.ts

@@ -0,0 +1,312 @@
+import http from "node:http";
+import https from "node:https";
+import {WebSocketServer} from "ws";
+import * as jk_fs from "jopi-toolkit/jk_fs";
+import fs from "node:fs";
+import {addBrowserCacheControlHeaders, type TryReturnFileParams} from "../browserCacheControl.ts";
+
+import type {
+    CoreServer,
+    ServerSocketAddress,
+    SseEvent,
+    SseEventController,
+    StartServerOptions,
+    WebSocketConnectionInfos
+} from "../jopiServer.ts";
+import type {
+    HttpMethod,
+    JopiWebSocket,
+    JopiWsRouteHandler,
+    WebSiteImpl,
+    WebSiteRouteInfos
+} from "../jopiWebSite.tsx";
+import {SBPE_MustReturnWithoutResponseException} from "../jopiWebSite.tsx";
+import type {ServerInstanceBuilder} from "../serverInstanceBuilder.ts";
+import {addRoute, createRouter, findRoute, type RouterContext} from "rou3";
+import React from "react";
+
+class NodeServerInstance implements CoreServer {
+    private readonly server: http.Server<typeof http.IncomingMessage, typeof http.ServerResponse>;
+
+    constructor(private options: StartServerOptions) {
+        let isHttps = options.tls !== undefined;
+
+        async function handler(req: http.IncomingMessage, res: http.ServerResponse) {
+            const headers = new Headers(req.headers as any);
+
+            const method = req.method!;
+            const body = (method == "GET" || method === "HEAD") ? undefined : jk_fs.nodeStreamToWebStream(req);
+
+            // req doesn't allow knowing if we are http or https.
+            // TODO BUG: https
+            const webReq = new Request((isHttps ? "https://" : "http://") + req.headers.host! + req.url!, {
+                body, headers, method,
+                // @ts-ignore
+                duplex: "half"
+            });
+
+            // @ts-ignore
+            webReq.nodeJsReq = req;
+            // @ts-ignore
+            webReq.nodeJsRes = res;
+
+            let webRes = reqFetch(webReq);
+            if (webRes instanceof Promise) webRes = await webRes;
+            if (webRes===undefined) return;
+
+            let resHeaders = webRes.headers;
+            let asJson: any = {};
+            resHeaders.forEach((value, key) => asJson[key] = value);
+
+            res.writeHead(webRes.status, asJson);
+
+            if (webRes.body) {
+                const asNodeStream = jk_fs.webStreamToNodeStream(webRes.body);
+                asNodeStream.pipe(res);
+            } else {
+                res.end("");
+            }
+        }
+
+        const reqFetch = options.fetch;
+
+        if (options.tls) {
+            let key = "", cert = "";
+
+            if (options.tls instanceof Array) {
+                for (const tls of options.tls) {
+                    key += tls.key;
+                    cert += tls.cert;
+                }
+            } else {
+                key = options.tls.key;
+                cert = options.tls.cert;
+            }
+
+            this.server = https.createServer({key, cert}, handler);
+        }
+        else {
+            this.server = http.createServer(handler);
+        }
+
+        const onWebSocketConnection = options.onWebSocketConnection;
+
+        if (onWebSocketConnection) {
+            const wss = new WebSocketServer({ server: this.server });
+
+            wss.on('connection', (ws, req) => {
+                onWebSocketConnection(ws as unknown as WebSocket, {
+                    url: "https://" + req.headers.host! + req.url!,
+                    headers: new Headers(req.headers as any)
+                });
+            });
+        }
+    }
+
+    requestIP(req: Request): ServerSocketAddress | null {
+        // @ts-ignore
+        let nodeReq: http.IncomingMessage = req.nodeJsReq;
+
+        return {
+            address: nodeReq.socket.remoteAddress!,
+            port: nodeReq.socket.remotePort!,
+            family: nodeReq.socket.remoteFamily as "IPv4" | "IPv6"
+        };
+    }
+
+    async stop(_closeActiveConnections: boolean): Promise<void> {
+        this.server.close();
+    }
+
+    timeout(_req: Request, _seconds: number): void {
+        // Timeout is managed globally for all the requests.
+    }
+
+    start() {
+        this.server.listen(this.options.port);
+    }
+}
+
+//region SSE Events
+
+interface NodeSseEvent extends SseEvent {
+    clients: http.ServerResponse[];
+}
+
+export async function onSseEvent(sseEvent: SseEvent, rawReq: any): Promise<Response> {
+    const req = rawReq.nodeJsReq as unknown as http.IncomingMessage;
+    const res = rawReq.nodeJsRes as unknown as http.ServerResponse;
+
+    res.writeHead(200, {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive'
+    });
+
+    res.write(`data: ${sseEvent.getWelcomeMessage()}\n\n`);
+
+    const nodeSseEvent = sseEvent as NodeSseEvent;
+
+    if (!nodeSseEvent.clients) {
+        nodeSseEvent.clients = [];
+
+        let controller: SseEventController = {
+            send(eventName: string, data: string) {
+                let toSend = `event: ${eventName}\ndata: ${ JSON.stringify({message: data}) }\n\n`;
+                nodeSseEvent.clients.forEach(res => { res.write(toSend) });
+            },
+
+            close() {
+                nodeSseEvent.clients.forEach(res => {
+                    if (!res.closed) {
+                        res.end();
+                    }
+                });
+
+                nodeSseEvent.clients = [];
+            }
+        }
+
+        nodeSseEvent.handler(controller);
+    }
+
+    nodeSseEvent.clients.push(res);
+
+    req.on('close', () => {
+        nodeSseEvent.clients = nodeSseEvent.clients.filter(client => client !== res);
+    });
+
+    // Allow bubbling up.
+    throw new SBPE_MustReturnWithoutResponseException();
+}
+
+//endregion
+
+//region ServerInstanceProvider
+
+export class NodeJsServerInstanceBuilder implements ServerInstanceBuilder {
+    private readonly router: RouterContext<WebSiteRouteInfos> = createRouter<WebSiteRouteInfos>();
+    private readonly wsRouter: RouterContext<JopiWsRouteHandler> = createRouter<JopiWsRouteHandler>();
+
+    constructor(private readonly webSite: WebSiteImpl) {
+    }
+
+    addRoute(verb: HttpMethod, path: string, routeInfos: WebSiteRouteInfos) {
+        addRoute(this.router, verb, path, routeInfos);
+    }
+
+    addWsRoute(path: string, handler: (ws: JopiWebSocket, infos: WebSocketConnectionInfos) => void) {
+        addRoute(this.wsRouter, "ws", path, handler);
+    }
+
+    addSseEVent(path: string, handler: SseEvent): void {
+        handler = {...handler};
+
+        this.addRoute("GET", path, {
+            route: path,
+
+            handler: async req => {
+                return onSseEvent(handler, req.coreRequest);
+            }
+        });
+    }
+
+    async startServer(params: { port: number; tls: any }): Promise<CoreServer> {
+        async function doFetch(req: Request): Promise<Response|undefined> {
+            const urlInfos = new URL(req.url);
+            const matched = findRoute(router, req.method, urlInfos.pathname);
+
+            if (matched) {
+                return await webSite.processRequest(matched.data.handler, matched.params, matched.data, urlInfos, req, server!);
+            } else {
+                return await webSite.processRequest(undefined, {}, undefined, urlInfos, req, server!);
+            }
+        }
+
+        const webSite = this.webSite;
+        const router = this.router;
+
+        const server = new NodeServerInstance({
+            port: String(params.port),
+            tls: params.tls,
+            fetch: doFetch,
+
+            onWebSocketConnection: (ws: WebSocket, infos: WebSocketConnectionInfos) => {
+                //const urlInfos = new URL(infos.url);
+
+                //const jws = new JopiWebSocket(this.webSite, server, ws);
+                //TODO
+                //webSite.declareNewWebSocketConnection(jws, infos, urlInfos);
+            }
+        });
+
+        server.start();
+
+        return server;
+    }
+
+    updateTlsCertificate(certificate: any) {
+        // Not available for node.js
+    }
+
+    addPage(path: string, pageKey: string, reactComponent: React.FC<any>, routeInfos: WebSiteRouteInfos) {
+        const mustRemoveTrailingSlashs = this.webSite.mustRemoveTrailingSlashs;
+
+        routeInfos.handler = async (req) => {
+            let path = req.urlInfos.pathname;
+
+            if (path!=="/") {
+                // Node.js router don't distinguish url ending with / and without.
+                // It's why we must check it.
+                //
+                if (mustRemoveTrailingSlashs) {
+                    if (path.endsWith("/")) {
+                        req.urlInfos.pathname = req.urlInfos.pathname.slice(0, -1);
+                        let redirectUrl = req.urlInfos.href;
+                        return Response.redirect(redirectUrl, 302);
+                    }
+                } else {
+                    if (!path.endsWith("/")) {
+                        req.urlInfos.pathname += "/";
+                        let redirectUrl = req.urlInfos.href;
+                        return Response.redirect(redirectUrl, 302);
+                    }
+                }
+            }
+
+            return req.react_fromPage(pageKey, reactComponent);
+        };
+
+        routeInfos.handler = this.webSite.applyMiddlewares("GET", path, routeInfos.handler, true);
+        this.addRoute("GET", path, routeInfos);
+    }
+
+    async tryReturnFile(params: TryReturnFileParams): Promise<Response|undefined> {
+        let stats = params.validationInfos.fileState;
+        if (!stats) return undefined;
+
+        // @ts-ignore
+        const nodeJsRes = params.req.coreRequest.nodeJsRes as http.ServerResponse;
+
+        const headers: any = {
+            "content-type": jk_fs.getMimeTypeFromName(params.filePath),
+            "content-length": stats.size
+        };
+
+        if (params.contentEncoding) {
+            headers["content-encoding"] = params.contentEncoding;
+        }
+
+        addBrowserCacheControlHeaders(headers, params);
+
+        // Envoyer les headers
+        nodeJsRes.writeHead(200, headers);
+
+        const readStream = fs.createReadStream(params.filePath);
+        readStream.pipe(nodeJsRes);
+
+        throw new SBPE_MustReturnWithoutResponseException();
+    }
+}
+
+//endregion

package/src/@core/serverInstanceBuilder.ts

@@ -0,0 +1,36 @@
+import type {CoreServer, SseEvent, WebSocketConnectionInfos} from "./jopiServer.ts";
+import {isBunJS} from "jopi-toolkit/jk_what";
+import {BunJsServerInstanceBuilder} from "./serverImpl/server_bunjs.tsx";
+import {NodeJsServerInstanceBuilder} from "./serverImpl/server_nodejs.ts";
+import {
+    type HttpMethod,
+    JopiWebSocket,
+    WebSiteImpl,
+    type WebSiteRouteInfos
+} from "./jopiWebSite.tsx";
+import React from "react";
+import type {TryReturnFileParams} from "./browserCacheControl.ts";
+
+export interface ServerInstanceBuilder {
+    addRoute(verb: HttpMethod, path: string, routeInfos: WebSiteRouteInfos): void;
+
+    addWsRoute(path: string, handler: (ws: JopiWebSocket, infos: WebSocketConnectionInfos) => void): void;
+
+    addSseEVent(path: string, handler: SseEvent): void;
+
+    startServer(params: { port: number; tls: any }): Promise<CoreServer>;
+
+    updateTlsCertificate(certificate: any): void;
+
+    addPage(path: string, pageKey: string, reactComponent: React.FC<any>, routeInfos: WebSiteRouteInfos): void;
+
+    tryReturnFile(params: TryReturnFileParams): Promise<Response|undefined>;
+}
+
+export function getNewServerInstanceBuilder(webSite: WebSiteImpl): ServerInstanceBuilder {
+    if (isBunJS) {
+        return new BunJsServerInstanceBuilder(webSite);
+    } else {
+        return new NodeJsServerInstanceBuilder(webSite);
+    }
+}

package/src/@core/userStores.ts

@@ -0,0 +1,34 @@
+import {type UserInfos, type WebSite} from "./jopiWebSite.tsx";
+
+export interface UserLoginPassword {
+    login: string;
+    password: string;
+}
+
+export interface UserInfos_WithLoginPassword extends UserLoginPassword {
+    userInfos: UserInfos;
+}
+
+export class UserStore_WithLoginPassword {
+    public readonly users: UserInfos_WithLoginPassword[] = [];
+
+    add(infos: UserInfos_WithLoginPassword) {
+        this.users.push(infos);
+    }
+
+    setAuthHandler(webSite: WebSite) {
+        webSite.setAuthHandler<UserLoginPassword>(loginInfo => {
+            let foundUser = this.users.find(e => e.login===loginInfo.login);
+
+            if (!foundUser) {
+                return {isOk: false, errorMessage: "Unknown user"};
+            }
+
+            if (loginInfo.password!==foundUser.password) {
+                return {isOk: false, errorMessage: "Wrong password"};
+            }
+
+            return {isOk: true, userInfos: foundUser.userInfos};
+        })
+    }
+}

package/src/@crawler/README.md

@@ -0,0 +1 @@
+Contains the crawler engine to crawl / download / transform a website.

package/src/@crawler/common.ts

@@ -0,0 +1,275 @@
+import {UrlMapping} from "./urlMapping.ts";
+import type {WebSiteCrawler} from "./core.ts";
+
+export interface CrawlerCache {
+    hasInCache(url: string, requestedByUrl: string): Promise<boolean>;
+    addToCache(url: string, response: Response, requestedByUrl: string): Promise<void>;
+    getKey(url: string): string;
+}
+
+export interface CrawlerTransformUrlInfos {
+    /**
+     * The local url of the page from which this url has been found.
+     */
+    comeFromPage: string;
+
+    /**
+     * The instance of the crawler.
+     */
+    crawler: WebSiteCrawler;
+
+    /**
+     * If true, this mean we need a final url which is ok with
+     * opening the page directly from the file-system.
+     */
+    requireRelocatableUrl: boolean;
+}
+
+export interface CrawlerCanIgnoreIfAlreadyCrawled {
+    /**
+     * The url which will be fetched.
+     */
+    sourceUrl: string;
+}
+
+export interface WebSiteCrawlerOptions {
+    /**
+     * If set, then will save the page inside this directory.
+     * Warning: will replace the cache value.
+     */
+    outputDir?: string;
+
+    /**
+     * Allow using a cache to save the HTML pages and resource
+     * and get theses that are already in the cache.
+     */
+    cache?: CrawlerCache;
+
+    /**
+     * If defined, then allow rewriting an url found by the HTML analyzer.
+     *
+     * @param url
+     *      The url found, converted to local site url.
+     * @param infos
+     *      Information about the context.
+     */
+    transformUrl?(url: string, infos: CrawlerTransformUrlInfos): string;
+
+    /**
+     * Is called when an URL is found and the content is HTML.
+     * Allow altering the HTML which will be processed or post-process URL.
+     */
+    rewriteHtmlBeforeProcessing?: (html: string, url: string, sourceUrl: string) => string|Promise<string>;
+
+    /**
+     * Is called when an URL is found and the content is HTML.
+     * Allow altering the final HTML.
+     * Is called after URL extraction.
+     */
+    rewriteHtmlBeforeStoring?: (html: string, url: string, sourceUrl: string) => string|Promise<string>;
+
+    /**
+     * Allow ignoring an entry if already crawled.
+     * The function takes and url (without a base path) and
+     * returns true if the page can be ignored, or false if it must crawl.
+     */
+    canIgnoreIfAlreadyCrawled?: (url: string, infos: CrawlerCanIgnoreIfAlreadyCrawled) => boolean;
+
+    /**
+     * Alter the final HTML to make the URL relocatable.
+     * This means we can copy and paste the website without attachement to the website name.
+     * Default is true.
+     */
+    requireRelocatableUrl?: boolean;
+
+    /**
+     * A list of url which must be replaced.
+     * If one of these urls is found as an url prefix,
+     * then replace it by the baseUrl.
+     */
+    rewriteThisUrls?: string[];
+
+    /**
+     * A list of urls to scan.
+     * Allow including forgotten urls (which mainly come from CSS or JavaScript).
+     * Ex: ["/my-style.css"].
+     */
+    scanThisUrls?: string[];
+
+    /**
+     * A mapper that allows knowing where to get data.
+     * Allow things like:
+     *      "/documentation/docA" --> "https://my-docsite.local/documentaiton/docA".
+     *      "/blog/my-blog-entry" --> "https://my-blog.local/my-blog-entry".
+     */
+    urlMapping?: UrlMapping;
+
+    /**
+     * The url of the new website, if downloading.
+     */
+    newWebSiteUrl?: string;
+
+    /**
+     * Allow the crawler to do a pause between two call to the server.
+     * The default value is 0: no pause.
+     */
+    pauseDuration_ms?: number;
+
+    /**
+     * Is called once a page is entirely downloaded.
+     * This means the page himself and all the links starting from this page.
+     * Will allow stopping the downloading by returning false.
+     */
+    onPageFullyDownloaded?: (url: string, state: ProcessUrlResult) => void|undefined|boolean|Promise<boolean>;
+
+    /**
+     * Is called when a resource is downloaded (.js, .css, .png, ...)
+     */
+    onResourceDownloaded?(url: string, state: ProcessUrlResult): void;
+
+    /**
+     * Allow sorting (and filtering) the pages we must download.
+     * The main use case is to prioritize some pages when there is a large breadcrumb/pager/menu.
+     */
+    sortPagesToDownload?(allUrls: UrlSortTools): void;
+
+    /**
+     * Is called when a resource returns a code which isn't 200 (ok) or a redirect.
+     * Return true if retry to download, false to stop.
+     */
+    onInvalidResponseCodeFound?: (url: string, retryCount: number, response: CrawlerFetchResponse) => boolean|Promise<boolean>;
+
+    /**
+     * Is called to know if this url can be downloaded.
+     *
+     * @param url
+     *      The local url of the page (ex: /my-page).
+     * @param isResource
+     *      true is the url is pointing to a resource (.css,.png,...)
+     *      false otherwise.
+     */
+    canDownload?(url: string, isResource: boolean): boolean;
+
+    /**
+     * Is called when a URL is processed.
+     * Allow building stats or listing all urls found.
+     */
+    onUrlProcessed?(infos: UrlProcessedInfos): void;
+
+    /**
+     * Is called when the crawling is finished.
+     * @param infos
+     */
+    onFinished?(infos: OnCrawlingFinishedInfos): void;
+
+    /**
+     * Allow replacing the fetch function with our own fetch.
+     */
+    doFetch?: CrawlerFetch;
+}
+
+export type CrawlerFetch = (crawler: WebSiteCrawler, url: string, referer: string) => Promise<CrawlerFetchResponse>;
+
+export interface CrawlerFetchResponse {
+    status: number;
+    headers: Headers;
+    body: ReadableStream<Uint8Array> | null;
+    text(): Promise<string>;
+}
+
+export interface OnCrawlingFinishedInfos {
+    remainingStack: string[]
+}
+
+export interface UrlProcessedInfos {
+    sourceUrl: string;
+    urlCount: number;
+    localUrl: string;
+    transformedUrl: string;
+    requestedByUrl: string;
+    state: ProcessUrlResult;
+    retryCount: number;
+
+    date: number;
+    elapsed: number;
+
+    /**
+     * The path inside the cache.
+     */
+    cacheKey?: string;
+}
+
+export enum ProcessUrlResult {
+    /**
+     * The resource has been downloaded.
+     */
+    OK = "ok",
+
+    /**
+     * The resource was a redirection.
+     */
+    REDIRECTED = "redirected",
+
+    /**
+     * An error occurred while processing the resource
+     * or the resources is an error page.
+     */
+    ERROR = "error",
+
+    /**
+     * The resource has been ignored.
+     * Probably because it was already downloaded.
+     */
+    IGNORED = "ignored"
+}
+
+export class UrlSortTools {
+    constructor(allUrls: string[]) {
+        this.allUrl = allUrls;
+    }
+
+    /**
+     * Remove the urls for which the filter response true
+     * and return an array with the extracted urls.
+     */
+    remove(filter: (url: string) => boolean): UrlSortTools {
+        const removed: string[] = [];
+        const others: string[] = [];
+
+        this.allUrl.forEach(url => {
+            if (filter(url)) removed.push(url);
+            else others.push(url);
+        });
+
+        this.removed = removed;
+        this.allUrl = others;
+
+        return this;
+    }
+
+    sortAsc(): UrlSortTools {
+        this.allUrl = this.allUrl.sort();
+        return this;
+    }
+
+    addRemovedBefore(): UrlSortTools {
+        if (!this.removed) return this;
+        this.allUrl = [...this.removed, ...this.allUrl];
+        this.removed = undefined;
+        return this;
+    }
+
+    addRemovedAfter(): UrlSortTools {
+        if (!this.removed) return this;
+        this.allUrl = [...this.allUrl, ...this.removed];
+        this.removed = undefined;
+        return this;
+    }
+
+    result(): string[] {
+        return this.allUrl;
+    }
+
+    removed?: string[];
+    allUrl: string[];
+}