jopijs 0.0.1

package/src/@core/letsEncrypt.ts

@@ -0,0 +1,243 @@
+import * as acme from 'acme-client';
+import {type SslCertificatePath, type WebSite, WebSiteImpl} from "./jopiWebSite.tsx";
+import path from "node:path";
+import * as jk_timer from "jopi-toolkit/jk_timer";
+import * as jk_fs from "jopi-toolkit/jk_fs";
+
+export type OnTimeoutError = (webSite: WebSite, isRenew: boolean) => void;
+
+export interface LetsEncryptParams {
+    email: string;
+
+    certificateDir?: string;
+
+    log?: boolean;
+    isProduction?: boolean;
+    forceRenew?: boolean;
+    
+    expireAfter_days?: number;
+
+    /**
+     * Allow stopping if the certificate isn't renewed after this delay.
+     * Will to an error with error.code="TIME_OUT".
+     */
+    timout_sec?: number;
+
+    /**
+     * Is called if there is an error.
+     */
+    onTimeoutError?: OnTimeoutError;
+}
+
+enum CertificateState {
+    DontExist, IsExpired, IsOk
+}
+
+async function getCertificateState(certPaths: SslCertificatePath, params: LetsEncryptParams): Promise<CertificateState> {
+    if (!await jk_fs.isFile(certPaths.cert)) return CertificateState.DontExist;
+    if (!await jk_fs.isFile(certPaths.key)) return CertificateState.DontExist;
+
+    let proofFile = path.join(path.dirname(certPaths.cert), "_updateDate.txt");
+    if (!await jk_fs.isFile(proofFile)) return CertificateState.DontExist;
+
+    // Using a file allows copying/paste the file or store it in GitHub.
+    // It's better than checking his update date.
+    //
+    const sDate = await jk_fs.readTextFromFile(proofFile);
+    if (!sDate) return CertificateState.DontExist;
+
+    const now = new Date();
+    let updateDate = parseInt(sDate);
+    const diffDays = (now.getTime() - updateDate) / (1000 * 60 * 60 * 24);
+
+    if (diffDays > params.expireAfter_days!) {
+        return CertificateState.IsExpired;
+    }
+
+    return CertificateState.IsOk;
+}
+
+function getCertificateDir(certDirPath: string, hostName: string): SslCertificatePath {
+    const sslDirPath = path.resolve(certDirPath, hostName);
+
+    return {
+        key: path.join(sslDirPath, "certificate.key"),
+        cert:path.join(sslDirPath, "certificate.crt.key")
+    };
+}
+
+async function saveCertificate(certPaths: SslCertificatePath, key: string, cert: string): Promise<void> {
+    await jk_fs.mkDir(path.dirname(certPaths.cert));
+    await jk_fs.writeTextToFile(certPaths.key, key);
+    await jk_fs.writeTextToFile(certPaths.cert, cert);
+
+    let proofFile = path.join(path.dirname(certPaths.cert), "_updateDate.txt");
+    await jk_fs.writeTextToFile(proofFile, Date.now().toString());
+}
+
+/**
+ * Download a LetsEncrypt certificate.
+ * Will be renewed if no current certificat or if the current one is perempted.
+ */
+export async function getLetsEncryptCertificate(httpsWebSite: WebSite, params: LetsEncryptParams): Promise<void> {
+    return checkWebSite(httpsWebSite, params, false);
+}
+
+export async function checkWebSite(httpsWebSite: WebSite, params: LetsEncryptParams, isFromCron: boolean): Promise<void> {
+    /**
+     * Write a proof.
+     */
+    async function challengeCreateFn(_auth: acme.Authorization, challenge: any, keyAuthorization: string): Promise<void> {
+        vChallengeToken = challenge.token;
+        vKeyAuthorization = keyAuthorization;
+    }
+
+    /**
+     * Remove the proof.
+     */
+    async function challengeRemoveFn(_auth: acme.Authorization, _challenge: any, _keyAuthorization: string) {
+        if (canLog) console.log("LetsEncrypt - removing challenge");
+    }
+    
+    if (!params.certificateDir) params.certificateDir = "certs";
+    if (params.log===undefined) params.log = true;
+
+    // Use 80 and not 90, to have a grace period.
+    if (!params.expireAfter_days) params.expireAfter_days = 80;
+
+    if (!params.timout_sec) params.timout_sec = 30;
+
+    if (!params.isProduction===undefined) {
+        // Need: NODE_ENV=production node app.js
+        params.isProduction = process.env.NODE_ENV === 'production';
+    }
+
+    if (!params.isProduction) {
+        console.warn("LetsEncrypt - Requesting as development");
+    }
+
+    // Will allow checking and replacing the certificate.
+    //
+    if (!isFromCron) {
+        registerToCron(httpsWebSite, params);
+    }
+
+    // ACME challenge requires port 80 of the server.
+    const webSite80 = httpsWebSite.getOrCreateHttpRedirectWebsite();
+
+    const certPaths = getCertificateDir(params.certificateDir, (webSite80 as WebSiteImpl).host);
+    let canLog = params.log;
+
+    let certState = await getCertificateState(certPaths, params);
+
+    if (certState===CertificateState.IsOk) {
+        if (!params.forceRenew) {
+            if (canLog) console.log("LetsEncrypt - certificat is already valid");
+            return;
+        }
+    }
+
+    let vChallengeToken = "";
+    let vKeyAuthorization = "";
+
+    const host = new URL((webSite80 as WebSiteImpl).welcomeUrl).host;
+
+    if (canLog) {
+        if (certState==CertificateState.IsExpired) console.log("LetsEncrypt - Will renew certificate for", host);
+        else console.log("LetsEncrypt - Requesting initial certificate for", host);
+    }
+    
+    // Must be on port 80.
+    webSite80.onGET("/.well-known/acme-challenge/**", async req => {
+        console.log("LetsEncrypt - requested ", req.url);
+        
+        if (req.url.endsWith(vChallengeToken)) {
+            console.log("LetsEncrypt - returning the auth", vKeyAuthorization);
+            return req.res_textResponse(vKeyAuthorization);
+        }
+
+        return req.res_returnError404_NotFound();
+    });
+
+    const client = new acme.Client({
+        directoryUrl: params.isProduction ? acme.directory.letsencrypt.production : acme.directory.letsencrypt.staging,
+        accountKey: await acme.crypto.createPrivateKey()
+    });
+
+    // CSR: Certificate Signing Request
+    const [key, csr] = await acme.crypto.createCsr({commonName: host});
+    
+    const options: acme.ClientAutoOptions = {
+        csr,
+        email: params.email,
+        termsOfServiceAgreed: true,
+        challengePriority: ['http-01'],
+
+        challengeCreateFn,
+        challengeRemoveFn,
+    };
+
+    let isResolved = false;
+
+    const timeoutPromise = new Promise<never>((_, reject) => {
+        setTimeout(() => {
+            if (isResolved) return;
+
+            const error = new Error(`Let's Encrypt certificate request timed out after ${params.timout_sec} seconds`);
+            (error as any).code = "TIME_OUT";
+            reject(error);
+        }, params.timout_sec! * 1000);
+    });
+
+    try {
+        const cert = await Promise.race([
+            client.auto(options),
+            timeoutPromise
+        ]);
+
+        isResolved = true;
+        if (canLog) console.log("LetsEncrypt - Certificate received for", host);
+        
+        await saveCertificate(certPaths, key.toString(), cert);
+    } catch (error: any) {
+        if (error.code === "TIME_OUT") {
+            if (params.onTimeoutError) {
+                params.onTimeoutError(httpsWebSite, isFromCron);
+            } else {
+                if (canLog) console.error("LetsEncrypt - Request is time-out");
+                throw error;
+            }
+        }
+        // Re-lancer l'erreur originale si ce n'est pas un timeout
+        throw error;
+    }
+
+    if (isFromCron) {
+        webSite80.updateSslCertificate(certPaths);
+    }
+}
+
+interface CronEntry {
+    webSite: WebSite;
+    params: LetsEncryptParams
+}
+
+let gIsCronStarted = false;
+const gWebsiteToCheck: CronEntry[] = [];
+
+function startCron() {
+    if (gIsCronStarted) return;
+    gIsCronStarted = true;
+
+    jk_timer.newInterval(jk_timer.ONE_DAY, () => {
+        gWebsiteToCheck.forEach(ce => checkWebSite(ce.webSite, ce.params, true));
+    });
+}
+
+function registerToCron(webSite: WebSite, params: LetsEncryptParams) {
+    if (!gIsCronStarted) {
+        startCron();
+    }
+
+    gWebsiteToCheck.push({webSite, params});
+}

package/src/@core/linker.ts

@@ -0,0 +1,42 @@
+import {type InstallFunction, loadServerInstall, getBrowserInstallFunction, getDefaultLinkerConfig, compile} from "jopijs/linker";
+import {ModuleInitContext} from "jopijs/ui";
+import * as jk_events from "jopi-toolkit/jk_events";
+import {JopiEasyWebSite, type WebSite} from "jopijs";
+import {logServer_linker} from "./_logs.ts";
+import {DontCallBeforeElapsed} from "jopi-toolkit/jk_tools";
+
+let gBrowserInstallFunction: InstallFunction<ModuleInitContext>;
+let gIsInit = false;
+
+export async function initLinker(webSite: JopiEasyWebSite, onWebSiteCreate: (h: (webSite: WebSite) => void|Promise<void>) => void) {
+    if (gIsInit) return;
+    gIsInit = true;
+
+    const endLog = logServer_linker.beginInfo("Rebuild linker on start");
+    await compile(import.meta, getDefaultLinkerConfig());
+    endLog();
+
+    gBrowserInstallFunction = await getBrowserInstallFunction();
+
+    await loadServerInstall(webSite, onWebSiteCreate);
+}
+
+export function executeBrowserInstall(ctx: ModuleInitContext) {
+    if (!gIsInit) return;
+    gBrowserInstallFunction(ctx);
+}
+
+const gLimitCompileCalls = new DontCallBeforeElapsed(2000);
+
+// Will allows updating shared components and composites.
+jk_events.addListener("@jopi.bundler.watch.beforeRebuild", async () => {
+    /**
+     * This event can occur multiple times with the single-page mode.
+     * It's why we limit the number of calls.
+     */
+    if (!gLimitCompileCalls.check()) return;
+
+    const endLog = logServer_linker.beginInfo("Rebuild linker on change");
+    await compile(import.meta, getDefaultLinkerConfig(), true /* is refreshing */);
+    endLog();
+});

package/src/@core/loadBalancing.ts

@@ -0,0 +1,152 @@
+import type {ServerFetch} from "./serverFetch.ts";
+import {type SendingBody} from "./jopiWebSite.tsx";
+import {JopiRequest} from "./jopiRequest.js";
+import * as jk_timer from "jopi-toolkit/jk_timer";
+
+const newInterval = jk_timer.newInterval;
+
+export class LoadBalancer {
+    private readonly servers: Server[] = [];
+    private totalWeight = 0;
+    private head?: Server;
+    private tail?: Server;
+    private lastUsedServer?: Server;
+    private isTimerStarted = false;
+
+    addServer<T>(server: ServerFetch<T>, weight?: number) {
+        if (server.loadBalancer) throw Error("Server already added to a load balancer");
+        server.loadBalancer = this;
+
+        if (!weight) weight = 1;
+        else if (weight>1) weight = 1;
+        else if (weight<0) weight = 0;
+
+        const lbServer = {fetcher: server, weight} as Server;
+
+        if (!this.head) this.head = lbServer;
+        lbServer.next = this.head;
+
+        if (this.tail) this.tail.next = lbServer;
+        this.tail = lbServer;
+
+        this.servers.push(lbServer);
+        this.totalWeight += lbServer.weight;
+    }
+
+    replaceServer<T>(oldServer: ServerFetch<T>, newServer: ServerFetch<T>, weight?: number) {
+        const lbServer = this.servers.find(s => s.fetcher===oldServer);
+        if (!lbServer) return;
+
+        newServer.loadBalancer = this;
+
+        lbServer.isServerDown = false;
+        lbServer.fetcher = newServer as ServerFetch<unknown>;
+        if (weight!==undefined) lbServer.weight = weight;
+    }
+
+    private selectServer(): Server|undefined {
+        if (this.servers.length === 0) return undefined;
+
+        const lastUsedServer = this.lastUsedServer! || this.tail!;
+        let cursor = lastUsedServer.next;
+        let round = 0;
+
+        let random = Math.random();
+
+        while (true) {
+            if (cursor===lastUsedServer) {
+                round++;
+
+                if (round===1) {
+                    // If we are here, it's mean we have tested all the server but no one is ok.
+                    // Then reduce the random value to 0 to include a server with weight 0.
+                    //
+                    random = 0;
+                } else if (round===2) {
+                    // No server despite that?
+                    return undefined;
+                }
+            }
+
+            if (cursor.isServerDown) continue;
+            if (random < cursor.weight) break;
+
+            cursor = cursor.next;
+        }
+
+        this.lastUsedServer = cursor;
+        return cursor;
+    }
+
+    async fetch(method: string, url: URL, body?: SendingBody, headers?: any): Promise<Response> {
+        const server = this.selectServer();
+        if (!server) return new Response("", {status: 521});
+
+        const res = await server.fetcher.fetch(method, url, body, headers);
+
+        if (res.status===521) {
+            this.declareServerDown(server);
+
+            // We retry with the next server.
+            // It's ok since the body isn't consumed yey.
+            return this.fetch(method, url, body, headers);
+        }
+
+        return res;
+    }
+
+    async directProxy(serverRequest: JopiRequest): Promise<Response> {
+        const server = this.selectServer();
+        if (!server) return new Response("", {status: 521});
+
+        const res = await server.fetcher.directProxy(serverRequest);
+
+        if (res.status===521) {
+            this.declareServerDown(server);
+        }
+
+        return res;
+    }
+
+    private declareServerDown(server: Server) {
+        if (server.isServerDown) return;
+        server.isServerDown = true;
+
+        // Will try to restart the server automatically.
+        //
+        if (!this.isTimerStarted) {
+            newInterval(2000, () => this.onTimer())
+        }
+    }
+
+    onTimer() {
+        let hasServerDown = false;
+
+        this.servers.forEach(async server => {
+            if (!server.isServerDown) return;
+
+            if (await server.fetcher.checkIfServerOk()) {
+                server.isServerDown = false;
+            } else {
+                hasServerDown = true;
+            }
+        });
+
+        if (!hasServerDown) {
+            this.isTimerStarted = false;
+
+            // Returning false will stop the timer.
+            return false;
+        }
+
+        // The timer will continue.
+        return true;
+    }
+}
+
+interface Server {
+    fetcher: ServerFetch<unknown>;
+    weight: number;
+    next: Server;
+    isServerDown?: boolean;
+}

package/src/@core/middlewares/CorsMiddleware.ts

@@ -0,0 +1,23 @@
+import type {JopiPostMiddleware} from "../jopiWebSite.tsx";
+
+export interface CorsMiddlewareOptions {
+    accessControlAllowOrigin?: string[];
+}
+
+export default function(options: CorsMiddlewareOptions): JopiPostMiddleware {
+    let accessControlAllowOrigin: string|undefined;
+    if (options.accessControlAllowOrigin) {
+        // Make it easier when setting urls.
+        const values = options.accessControlAllowOrigin.map(v => new URL(v).origin);
+
+        accessControlAllowOrigin = values.join(",");
+    }
+
+    return (req, res) => {
+        if (accessControlAllowOrigin) {
+            res.headers.set("Access-Control-Allow-Origin", accessControlAllowOrigin);
+        }
+
+        return res;
+    };
+}

package/src/@core/middlewares/DdosProtection.ts

@@ -0,0 +1,120 @@
+import {JopiRequest} from "../jopiRequest.tsx";
+import type {JopiMiddleware} from "../jopiWebSite.tsx";
+import {getServerStartOptions} from "../jopiServer.ts";
+import * as jk_tools from "jopi-toolkit/jk_tools";
+import * as jk_timer from "jopi-toolkit/jk_timer";
+
+const newInterval = jk_timer.newInterval;
+const applyDefaults = jk_tools.applyDefaults;
+
+// slowhttptest -c 1000 -H -i 10 -r 200 -t GET -u http://my-server -x 24 -p 3
+
+export interface DdosProtectionOptions {
+    /**
+     * If the request takes more than n-milliseconds to send his headers, then we reject this request.
+     * Warning: this value is global to all websites. Setting it will affect all of them.
+     * Default: 500 ms.
+     */
+    sendHeadersTimeout_ms?: number;
+
+    /**
+     * Delay in millisecondes after which the request timeout.
+     * Default: 60 seconds.
+     */
+    requestTimeout_sec?: number;
+
+    /**
+     * We will limit the number of calls allowed during an interval.
+     * Here it's the size of this interval in milliseconds.
+     * Default is 1000 ms (one second).
+     *
+     * The default behaviors are that you can do more than 10 calls to the same second with the same IP.
+     * If you need an exception for an IP, use onBlackRequest
+     */
+    timeInterval_ms?: number;
+
+    /**
+     * We will limit the number of calls allowed during an interval.
+     * Here's the number of connections allowed during this interval of time.
+     * Default is 10.
+     *
+     * The default behaviors are that you can do more than 10 calls to the same second with the same IP.
+     * If you need an exception for an IP, use onBlackRequest
+     */
+    connectionLimit?: number;
+
+    /**
+     * Is call if a request is detected as an anomaly.
+     */
+    onBlackRequest?: JopiMiddleware;
+}
+
+let gGlobalBlackRequestListener: JopiMiddleware|undefined;
+
+/**
+ * Allow setting a listener which is called when a black request is detected.
+ * It'd mainly used to add his IP to a banned IP list.
+ */
+export function setGlobalBlackRequestListener(listener: JopiMiddleware) {
+    gGlobalBlackRequestListener = listener;
+}
+
+export default function(options?: DdosProtectionOptions): JopiMiddleware {
+    options = applyDefaults<DdosProtectionOptions>(options, {
+        sendHeadersTimeout_ms: 500,
+        requestTimeout_sec: 60,
+
+        timeInterval_ms: 1000,
+        connectionLimit: 10,
+
+        onBlackRequest: () => {
+            return new Response("Too many request", { status: 429 });
+        }
+    });
+
+    // Here it's a common value for all servers.
+    getServerStartOptions().timeout = options.sendHeadersTimeout_ms!;
+
+    const mapConnectionsPerIP = new Map<String, number>;
+    const connectionLimit = options.connectionLimit!;
+
+    let mustReset = false;
+
+    // The values here are reset after a delay.
+    //
+    newInterval(options.timeInterval_ms!, () => {
+        if (mustReset) {
+            mapConnectionsPerIP.clear();
+        }
+    });
+
+    return (req: JopiRequest) => {
+        // If we are here, it's mean the request take less than 1 second to send his headers.
+        // It's why we can extend the time-out delay (60 secondes).
+        //
+        req.req_extendTimeout_sec(options.requestTimeout_sec!);
+
+        // > Check how many-time this request has reached the server recently.
+
+        const clientIP = req.requestIP?.address;
+
+        if (clientIP) {
+            const currentConnections = (mapConnectionsPerIP.get(clientIP) || 0) + 1;
+
+            if (currentConnections > connectionLimit) {
+                if (gGlobalBlackRequestListener) {
+                    const res = gGlobalBlackRequestListener(req);
+                    if (res!==null) return res;
+                }
+
+                return options.onBlackRequest!(req);
+            }
+
+            mapConnectionsPerIP.set(clientIP, currentConnections);
+            mustReset = true;
+        }
+
+        // Allow continuing to the next middleware.
+        return null;
+    }
+};

package/src/@core/middlewares/index.ts

@@ -0,0 +1,16 @@
+import DdosProtection from "./DdosProtection.ts";
+import CorsMiddleware from "./CorsMiddleware.ts";
+import {JopiRequest} from "../jopiRequest.js";
+
+export const Middlewares = {
+    ddosProtection: DdosProtection,
+
+    requestTimeout_sec: (timeInSec:number) => (req: JopiRequest) => {
+        req.req_extendTimeout_sec(timeInSec);
+        return null;
+    }
+};
+
+export const PostMiddlewares = {
+    cors: CorsMiddleware
+};

package/src/@core/publicTools.ts

@@ -0,0 +1,14 @@
+import type {HttpMethod} from "./jopiWebSite.tsx";
+
+export function octetToMo(size: number) {
+    const res = size / ONE_MEGA_OCTET;
+    return Math.trunc(res * 100) / 100;
+}
+
+export const ONE_MINUTE = 1000 * 60;
+export const ONE_HOUR = ONE_MINUTE * 60;
+export const ONE_DAY = ONE_HOUR * 24;
+export const ONE_KILO_OCTET = 1024;
+export const ONE_MEGA_OCTET = 1024 * ONE_KILO_OCTET;
+
+export const HTTP_VERBS: HttpMethod[] = ["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"];