jopijs 0.0.1

package/src/@crawler/directFileCache.ts

@@ -0,0 +1,69 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import type {CrawlerCache} from "./common.ts";
+import * as jk_fs from "jopi-toolkit/jk_fs";
+
+export class DirectFileCache implements CrawlerCache {
+    public readonly rootDir: string;
+
+    constructor(rootDir: string) {
+        if (!rootDir) rootDir = ".";
+        if (!path.isAbsolute(rootDir)) rootDir = path.resolve(process.cwd(), rootDir);
+        this.rootDir = rootDir;
+    }
+
+    private calKey(url: string): string {
+        const url2 = new URL(url);
+        url2.hostname = "localhost";
+        url2.port = "";
+        url2.protocol = "file:";
+
+        const sURL = url2.toString();
+        return jk_fs.fileURLToPath(sURL);
+    }
+
+    private calcFilePath(url: string): string {
+        let fp = path.join(this.rootDir, this.calKey(url));
+
+        if (fp.endsWith("/")) {
+            fp += "index.html";
+        } else {
+            const ext = path.extname(fp);
+            if (!ext) fp += "/index.html";
+        }
+
+        return fp;
+    }
+
+    getKey(url: string): string {
+        return this.calcFilePath(url);
+    }
+
+    async addToCache(url: string, response: Response, _requestedByUrl: string): Promise<void> {
+        // We don't store 404 and others.
+        if (response.status !== 200) return;
+
+        const filePath = this.calcFilePath(url);
+        await fs.mkdir(path.dirname(filePath), {recursive: true});
+
+        try {
+            await jk_fs.writeResponseToFile(response, filePath);
+        }
+        catch (e) {
+            console.error(e);
+        }
+    }
+
+    async hasInCache(url: string): Promise<boolean> {
+        const filePath = this.calcFilePath(url);
+
+        try {
+            const stat = await jk_fs.getFileStat(filePath);
+            return (stat!==undefined) && stat.isFile();
+        }
+        catch {
+            return false;
+        }
+
+    }
+}

package/src/@crawler/index.ts

@@ -0,0 +1,4 @@
+export * from "./common.ts";
+export * from "./core.ts";
+export * from "./utils.ts";
+export * from "./urlMapping.ts";

package/src/@crawler/urlMapping.ts

@@ -0,0 +1,67 @@
+import {addRoute, createRouter, findRoute} from "rou3";
+
+export interface UrlResolver {
+    resolveURL(url: string): UrlMappingResult|undefined;
+}
+
+export interface UrlMappingResult {
+    url: string;
+    wakeUpServer?: ()=>Promise<void>;
+}
+
+/**
+ * Allows knowing how to map the url.
+ * This allows mixing more than one website.
+ */
+export class UrlMapping implements UrlResolver {
+    private readonly router = createRouter<UrlMappingResult>();
+    private readonly defaultTarget: string;
+    private readonly allOrigins: string[] = [];
+
+    constructor(defaultTarget: string) {
+        this.defaultTarget = this.cleanUpRoute(defaultTarget);
+    }
+
+    private cleanUpRoute(route: string): string {
+        if (route.endsWith("/*")) return route.slice(0, -2);
+        if (route.endsWith("/**")) return route.slice(0, -3);
+        if (route.endsWith("/")) return route.slice(0, -1);
+        return route;
+    }
+
+    public getKnownOrigins(): string[] {
+        return this.allOrigins;
+    }
+
+    public mapURL(route: string, mapTo: string, wakeUpServer?: ()=>Promise<void>): UrlMapping {
+        // Avoid errors.
+        mapTo = this.cleanUpRoute(mapTo);
+        route = this.cleanUpRoute(route) + "/**";
+
+        let mapToOrigin = new URL(mapTo).origin;
+        if (!this.allOrigins.includes(mapToOrigin)) this.allOrigins.push(mapToOrigin);
+
+        addRoute(this.router, "GET", route, {
+            url: mapTo,
+            wakeUpServer: wakeUpServer
+        });
+
+        return this;
+    }
+
+    public resolveURL(url: string): UrlMappingResult|undefined {
+        const matched = findRoute(this.router, "GET", url);
+        let target: string;
+
+        let wakeUpServer: (()=>Promise<void>)|undefined;
+
+        if (!matched) {
+            target = this.defaultTarget;
+        } else {
+            target = matched.data.url;
+            wakeUpServer = matched.data.wakeUpServer;
+        }
+
+        return {url: target + url, wakeUpServer};
+    }
+}

package/src/@crawler/utils.ts

@@ -0,0 +1,13 @@
+export function applyDefaults<T>(source: T|undefined, defaults: T): T {
+    if (!source) source = {} as T;
+    return {...defaults, ...source};
+}
+
+export function tick(timeInMs: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, timeInMs));
+}
+
+export const ONE_SECOND = 1000;
+export const ONE_MINUTE = ONE_SECOND * 60;
+export const ONE_HOUR = ONE_MINUTE * 60;
+export const ONE_DAY = ONE_HOUR * 24;

package/src/@generated/index.ts

@@ -0,0 +1,134 @@
+import React from "react";
+import {type HttpMethod, JopiRequest, WebSiteImpl, type WebSiteRouteInfos} from "jopijs";
+import * as jk_crypto from "jopi-toolkit/jk_crypto";
+import * as jk_events from "jopi-toolkit/jk_events";
+import {PriorityLevel} from "jopi-toolkit/jk_tools";
+
+export interface RouteAttributs {
+    needRoles?: Record<string, string[]>;
+    disableCache?: boolean;
+    priority?: PriorityLevel;
+    configFile?: string;
+
+    /**
+     * When doing a catch-all, a slug can be set.
+     * Here we store this slug name.
+     *
+     * Catch-all: /my/route/[...]
+     * Catch-all with slug: /my/route/[...slugName]
+     */
+    catchAllSlug?: string;
+}
+
+type RouteHandler = (req: JopiRequest) => Promise<Response>;
+
+function applyAttributes(infos: WebSiteRouteInfos, attributes: RouteAttributs, verb: string) {
+    if (attributes.needRoles) {
+        infos.requiredRoles = attributes.needRoles[verb];
+
+        let allRoles = attributes.needRoles["all"];
+        if (allRoles) infos.requiredRoles = infos.requiredRoles?.concat(allRoles);
+    }
+}
+
+interface RouteBindPageParams {
+    route: string;
+    attributs: RouteAttributs;
+    filePath: string;
+}
+
+export async function routeBindPage(webSite: WebSiteImpl, reactComponent: React.FC<any>, params: RouteBindPageParams) {
+    const pageKey = "page_" + jk_crypto.fastHash(params.route);
+    let infos: WebSiteRouteInfos;
+
+    if (params.route.endsWith("*")) {
+        infos = webSite.onPage(params.route, pageKey, reactComponent);
+    }
+    else {
+        let specialPageHandler: (() => void) | undefined;
+
+        if (params.route.startsWith("/error")) {
+            switch (params.route) {
+                case "/error404":
+                    specialPageHandler = () => {
+                        webSite.on404_NotFound(async (req) => {
+                            return req.react_fromPage(pageKey, reactComponent);
+                        });
+                    }
+
+                    break;
+
+                case "/error500":
+                    specialPageHandler = () => {
+                        webSite.on500_Error(async (req) => {
+                            return req.react_fromPage(pageKey, reactComponent);
+                        });
+                    }
+
+                    break;
+
+                case "/error401":
+                    specialPageHandler = () => {
+                        webSite.on401_Unauthorized(async (req) => {
+                            return req.react_fromPage(pageKey, reactComponent);
+                        });
+                    }
+
+                    break;
+            }
+        }
+
+        // The page must not be visible if it's a special page.
+        //
+        if (!specialPageHandler) {
+            //const REDIRECT_CODE = 301; // definitive
+            const REDIRECT_CODE = 302; // temporary
+
+            if (params.route === "/") {
+                infos = webSite.onPage(params.route, pageKey, reactComponent);
+            } else {
+                if (webSite.mustRemoveTrailingSlashs) {
+                    infos = webSite.onPage(params.route, pageKey, reactComponent);
+
+                    // Note: with node.js, the router doesn't distinguish with and without /
+                    // It's why the redirection is done internally.
+                    //
+                    webSite.onGET(params.route + "/", async (req) => {
+                        req.urlInfos.pathname = req.urlInfos.pathname.slice(0, -1);
+                        return Response.redirect(req.urlInfos.href, REDIRECT_CODE);
+                    });
+                } else {
+                    infos = webSite.onPage(params.route + "/", pageKey, reactComponent);
+
+                    // Note: with node.js, the router doesn't distinguish with and without /
+                    // It's why the redirection is done internally.
+                    //
+                    webSite.onGET(params.route, async (req) => {
+                        req.urlInfos.pathname += "/";
+                        return Response.redirect(req.urlInfos.href, REDIRECT_CODE);
+                    });
+                }
+
+                applyAttributes(infos, params.attributs, "PAGE");
+            }
+        } else {
+            specialPageHandler();
+        }
+    }
+
+    await jk_events.sendAsyncEvent("@jopi.route.newPage", params);
+}
+
+interface RouteBindVerbParams extends RouteBindPageParams {
+    verb: HttpMethod;
+}
+
+export async function routeBindVerb(webSite: WebSiteImpl, handler: RouteHandler, params: RouteBindVerbParams) {
+    let infos = webSite.onVerb(params.verb, params.route, handler);
+    applyAttributes(infos, params.attributs, params.verb);
+
+    if (params.route!=="/") {
+        infos = webSite.onVerb(params.verb, params.route + "/", handler);
+        applyAttributes(infos, params.attributs, params.verb);
+    }
+}

package/src/@linker/arobaseTypes.ts

@@ -0,0 +1,325 @@
+import * as jk_fs from "jopi-toolkit/jk_fs";
+import * as jk_tools from "jopi-toolkit/jk_tools";
+import * as jk_events from "jopi-toolkit/jk_events";
+
+import {
+    type ProcessDirItemParams,
+    getSortedDirItem,
+    type TransformItemParams,
+    PriorityLevel,
+    type RegistryItem,
+    ArobaseType,
+    CodeGenWriter
+} from "./engine.ts";
+
+// region ArobaseList
+
+export interface ArobaseList extends RegistryItem {
+    listName: string;
+    allDirPath: string[];
+    items: ArobaseListItem[];
+    itemsType: string;
+    conditions?: Set<string>;
+}
+
+export interface ArobaseListItem {
+    ref?: string;
+    entryPoint?: string;
+    priority: PriorityLevel;
+    sortKey: string;
+}
+
+export class Type_ArobaseList extends ArobaseType {
+    protected async onListItem(item: ArobaseListItem, list: ArobaseListItem[], _dirPath: string): Promise<void> {
+        list.push(item);
+    }
+
+    protected mergeIntoList(list: ArobaseList, items: ArobaseListItem[]) {
+        let currentItems = list.items;
+        currentItems.push(...items);
+        currentItems.sort((a, b) => a.sortKey.localeCompare(b.sortKey));
+    }
+
+    processDir(p: { moduleDir: string; arobaseDir: string; genDir: string; }) {
+        return this.processList(p.arobaseDir);
+    }
+
+    protected async processList(listDirPath: string): Promise<void> {
+        await this.dir_recurseOnDir({
+            dirToScan: listDirPath,
+            expectFsType: "dir",
+
+            rules: {
+                nameConstraint: "canBeUid",
+                requireRefFile: false,
+                requirePriority: false,
+                allowConditions: false,
+                rootDirName: jk_fs.basename(listDirPath),
+                transform: (p) => this.processGroup(p)
+            }
+        });
+    }
+
+    protected async processGroup(p: TransformItemParams) {
+        let listId = this.typeName + "!" + p.itemName!;
+        const listName = p.itemName;
+
+        // > Extract the list items.
+
+        const dirItems = await getSortedDirItem(p.itemPath);
+        let listItems: ArobaseListItem[] = [];
+
+        const params: ProcessDirItemParams = {
+            rootDirName: p.parentDirName,
+            nameConstraint: "canBeUid",
+            requirePriority: false,
+            requireRefFile: false,
+            allowConditions: false,
+
+            filesToResolve: {
+                "entryPoint": ["index.tsx", "index.ts"]
+            },
+
+            transform: async (item) => {
+                const listItem: ArobaseListItem = {
+                    priority: item.priority,
+                    sortKey: item.itemName,
+                    ref: item.refTarget,
+                    entryPoint: item.resolved.entryPoint
+                };
+
+                const eventData = {itemPath: item.itemPath, item: listItem, list: listItems, mustSkip: false};
+                await jk_events.sendAsyncEvent("@jopi.linker.onNewListItem." + this.typeName, eventData);
+                if (!eventData.mustSkip) await this.onListItem(listItem, listItems, item.itemPath);
+            }
+        };
+
+        for (let dirItem of dirItems) {
+            if (!dirItem.isDirectory) continue;
+
+            if (dirItem.name === "_") {
+                let uid = jk_tools.generateUUIDv4();
+                let newPath = jk_fs.join(jk_fs.dirname(dirItem.fullPath), uid);
+                await jk_fs.rename(dirItem.fullPath, newPath);
+
+                dirItem.name = uid;
+                dirItem.fullPath = newPath;
+            }
+
+            if ((dirItem.name[0] === "_") || (dirItem.name[0] === ".")) continue;
+
+            await this.dir_processItem(dirItem, params);
+        }
+
+        // > Add the list.
+
+        let current = this.registry_getItem<ArobaseList>(listId, this);
+
+        if (!current) {
+            const newItem: ArobaseList = {
+                listName, conditions: p.conditions,
+                arobaseType: this, itemPath: p.itemPath,
+                items: listItems, itemsType: p.parentDirName, allDirPath: [p.itemPath]
+            };
+
+            this.registry_addItem(listId, newItem);
+        } else {
+            if (current.itemsType !== p.parentDirName) {
+                throw this.declareError(`The list ${listId} is already defined and has a different type: ${current.itemsType}`, p.itemPath);
+            }
+
+            // Merge the items into the current one.
+            this.mergeIntoList(current, listItems);
+
+            // The list of event declaration locations.
+            current.allDirPath.push(p.itemPath);
+        }
+    }
+
+    protected getGenOutputDir(_list: ArobaseList) {
+        return this.typeName;
+    }
+
+    protected resolveEntryPointFor(list: ArobaseList, item: ArobaseListItem): string {
+        let entryPoint = item.entryPoint!;
+
+        if (!entryPoint) {
+            let d = this.registry_requireItem<ArobaseChunk>(item.ref!);
+            if (d.itemType!==list.itemsType) {
+                throw this.declareError(`Type mismatch. Expect ${list.itemsType}`, d.itemPath)
+            }
+
+            if (!d.entryPoint) {
+                throw this.declareError(`Item if missing index.ts/index.tsx file`, d.itemPath)
+            }
+
+            entryPoint = d.entryPoint;
+        }
+
+        return entryPoint;
+    }
+
+    async generateCodeForItem(writer: CodeGenWriter, key: string, rItem: RegistryItem) {
+        function sortByPriority(items: ArobaseListItem[]): ArobaseListItem[] {
+            function addPriority(priority: PriorityLevel) {
+                let e = byPriority[priority];
+                if (e) items.push(...e);
+            }
+
+            const byPriority: any = {};
+
+            for (let item of items) {
+                if (!byPriority[item.priority]) byPriority[item.priority] = [];
+                byPriority[item.priority].push(item);
+            }
+
+            items = [];
+
+            addPriority(PriorityLevel.veryHigh);
+            addPriority(PriorityLevel.high);
+            addPriority(PriorityLevel.default);
+            addPriority(PriorityLevel.low);
+            addPriority(PriorityLevel.veryLow);
+
+            return items;
+        }
+
+        const list = rItem as ArobaseList;
+        list.items = sortByPriority(list.items);
+
+        await this.generateCodeForList(writer, key, list);
+    }
+
+    protected async generateCodeForList(writer: CodeGenWriter, key: string, list: ArobaseList): Promise<void> {
+        let count = 1;
+        let outDir_innerPath = this.getGenOutputDir(list);
+        let outDir_fullPath = jk_fs.join(writer.dir.output_src, outDir_innerPath);
+
+        let srcCode = writer.AI_INSTRUCTIONS + this.codeGen_generateImports();
+        let distCode = srcCode;
+
+        for (let item of list.items) {
+            let entryPoint = this.resolveEntryPointFor(list, item);
+            let relPath = jk_fs.getRelativePath(outDir_fullPath, entryPoint);
+
+            srcCode += `import I${count} from "${relPath}";\n`;
+            distCode += `import I${count} from "${writer.toJavascriptFileName(relPath)}";\n`;
+
+            count++;
+        }
+
+        let array = "";
+        let max = list.items.length;
+        for (let i = 1; i <= max; i++) array += `I${i},`;
+
+        let toAdd = "\n" + this.codeGen_generateExports("[" + array + "]", list.listName);
+        srcCode += toAdd;
+        distCode += toAdd;
+
+        let fileName = key.substring(key.indexOf("!") + 1);
+
+        await writer.writeCodeFile({
+            fileInnerPath: jk_fs.join(outDir_innerPath, fileName),
+            declarationFile: this.codeGen_createDeclarationTypes(),
+            srcFileContent: srcCode,
+            distFileContent: distCode,
+            useTypescriptForSource: true
+        });
+    }
+
+    protected codeGen_generateImports() {
+        return "";
+    }
+
+    protected codeGen_generateExports(listAsArray: string, listName: string) {
+        return "export default " + listAsArray + ";";
+    }
+
+    /**
+     * Allow creating content for the .d.ts file.
+     * @protected
+     */
+    protected codeGen_createDeclarationTypes() {
+        return `const list: any[]; export default list;`
+    }
+}
+
+//endregion
+
+//region ArobaseChunk
+
+export interface ArobaseChunk extends RegistryItem {
+    entryPoint: string;
+    itemType: string;
+}
+
+export class Type_ArobaseChunk extends ArobaseType {
+    async onChunk(chunk: ArobaseChunk, key: string, _dirPath: string) {
+        this.registry_addItem(key, chunk);
+    }
+
+    async processDir(p: { moduleDir: string; arobaseDir: string; genDir: string; }) {
+        await this.dir_recurseOnDir({
+            dirToScan: p.arobaseDir,
+            expectFsType: "dir",
+
+            rules: {
+                rootDirName: jk_fs.basename(p.arobaseDir),
+                nameConstraint: "canBeUid",
+                requireRefFile: false,
+                requirePriority: true,
+
+                filesToResolve: {
+                    "info": ["info.json"],
+                    "entryPoint": ["index.tsx", "index.ts"]
+                },
+
+                transform: async (props) => {
+                    if (!props.resolved?.entryPoint) {
+                        throw this.declareError("No 'index.ts' or 'index.tsx' file found", props.itemPath);
+                    }
+
+                    const chunk: ArobaseChunk = {
+                        arobaseType: this,
+                        entryPoint: props.resolved.entryPoint,
+                        itemType: props.parentDirName,
+                        itemPath: props.itemPath,
+                        priority: props.priority
+                    };
+
+                    const key = this.typeName + "!" + props.itemName;
+                    const eventData = {mustSkip: false, key, chunk, itemPath: props.itemPath};
+                    await jk_events.sendAsyncEvent("@jopi.linker.onNewChunk." + this.typeName, eventData);
+
+                    if (!eventData.mustSkip) {
+                        await this.onChunk(chunk, key, props.itemPath);
+                    }
+                }
+            }
+        });
+    }
+
+    async generateCodeForItem(writer: CodeGenWriter, key: string, rItem: RegistryItem) {
+        const item = rItem as ArobaseChunk;
+
+        let targetName = key.substring(key.indexOf("!") + 1);
+        let outDir = jk_fs.join(writer.dir.output_src, this.getGenOutputDir(item));
+        let entryPoint = jk_fs.getRelativePath(outDir, item.entryPoint);
+
+        let srcCode = writer.AI_INSTRUCTIONS + `import C from "${entryPoint}";\nexport default C;`;
+        let distCode = writer.AI_INSTRUCTIONS + `import C from "${writer.toJavascriptFileName(entryPoint)}";\nexport default C;`;
+
+        await writer.writeCodeFile({
+            fileInnerPath: jk_fs.join(this.getGenOutputDir(item), targetName),
+            srcFileContent: srcCode,
+            distFileContent: distCode,
+            useTypescriptForSource: true
+        });
+    }
+
+    protected getGenOutputDir(_chunk: ArobaseChunk) {
+        return this.typeName;
+    }
+}
+
+//endregion

package/src/@linker/changeProof.ts

@@ -0,0 +1,53 @@
+import * as fs from 'fs/promises';
+import * as fss from 'fs';
+import * as path from 'path';
+import * as jk_crypto from "jopi-toolkit/jk_crypto";
+import * as jk_fs from "jopi-toolkit/jk_fs";
+
+/**
+ * Recursive internal function to collect file/folder information strings.
+ * All paths in the resulting strings are relative to 'basePath'.
+ */
+export async function calculateDirectoryProof(rootDir: string): Promise<string> {
+    async function collectProofString(currentPath: string): Promise<string> {
+        let dirItems: fss.Dirent[];
+
+        try {
+            dirItems = await fs.readdir(currentPath, {withFileTypes: true});
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'EACCES') return "";
+            throw error;
+        }
+
+        dirItems.sort((a, b) => a.name.localeCompare(b.name));
+
+        let chunksString = "";
+
+        for (const dirItem of dirItems) {
+            const fullPath = path.join(currentPath, dirItem.name);
+            const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, '/');
+            const stats = await fs.lstat(fullPath);
+
+            if (stats.isSymbolicLink()) {
+                let target = await fs.readlink(fullPath);
+                target = path.relative(rootDir, fullPath).replace(/\\/g, '/');
+                chunksString += `SYM_LINK:${relativePath}:${target}`;
+            } else if (dirItem.isDirectory()) {
+                chunksString += await collectProofString(fullPath);
+            } else if (dirItem.isFile()) {
+                try {
+                    let md5 = jk_crypto.md5(await jk_fs.readTextFromFile(fullPath));
+                    chunksString += `FILE:${relativePath}:${md5}`;
+                }
+                catch (e) {
+                    chunksString += `FILE:${relativePath}:???`;
+                }
+            }
+        }
+
+        return chunksString;
+    }
+
+    const chunksString = await collectProofString(rootDir);
+    return jk_crypto.md5(chunksString);
+}