@eighty4/dank 0.0.2 → 0.0.4-0

package/lib_js/html.js

@@ -1,143 +1,258 @@
-import { readFile, writeFile } from 'node:fs/promises';
-import { dirname, join, relative } from 'node:path';
+import EventEmitter from 'node:events';
+import { readFile } from 'node:fs/promises';
+import { dirname, join, relative, resolve } from 'node:path';
 import { extname } from 'node:path/posix';
 import { defaultTreeAdapter, parse, parseFragment, serialize, } from 'parse5';
-// unenforced but necessary sequence:
-//  injectPartials
-//  collectScripts
-//  rewriteHrefs
-//  writeTo
-export class HtmlEntrypoint {
-    static async readFrom(urlPath, fsPath) {
-        let html;
-        try {
-            html = await readFile(fsPath, 'utf-8');
-        }
-        catch (e) {
-            console.log(`\u001b[31merror:\u001b[0m`, fsPath, 'does not exist');
-            process.exit(1);
-        }
-        return new HtmlEntrypoint(urlPath, html, fsPath);
-    }
-    #document;
+export class HtmlEntrypoint extends EventEmitter {
+    #build;
+    #decorations;
+    #document = defaultTreeAdapter.createDocument();
+    // todo cache entrypoints set for quicker diffing
+    // #entrypoints: Set<string> = new Set()
     #fsPath;
     #partials = [];
     #scripts = [];
+    #update = Object();
     #url;
-    constructor(url, html, fsPath) {
+    constructor(build, url, fsPath, decorations) {
+        super({ captureRejections: true });
+        this.#build = build;
+        this.#decorations = decorations;
         this.#url = url;
-        this.#document = parse(html);
         this.#fsPath = fsPath;
+        this.on('change', this.#onChange);
+        this.emit('change');
     }
-    async injectPartials() {
-        this.#collectPartials(this.#document);
-        await this.#injectPartials();
+    get fsPath() {
+        return this.#fsPath;
     }
-    collectScripts() {
-        this.#collectScripts(this.#document);
-        return this.#scripts;
+    get url() {
+        return this.#url;
     }
-    // rewrites hrefs to content hashed urls
-    // call without hrefs to rewrite tsx? ext to js
-    rewriteHrefs(hrefs) {
-        for (const importScript of this.#scripts) {
-            const rewriteTo = hrefs ? hrefs[importScript.in] : null;
-            if (importScript.type === 'script') {
-                if (importScript.in.endsWith('.tsx') ||
-                    importScript.in.endsWith('.ts')) {
-                    importScript.elem.attrs.find(attr => attr.name === 'src').value = rewriteTo || `/${importScript.out}`;
-                }
+    async #html() {
+        try {
+            return await readFile(join(this.#build.dirs.pages, this.#fsPath), 'utf8');
+        }
+        catch (e) {
+            // todo error handling
+            errorExit(this.#fsPath + ' does not exist');
+        }
+    }
+    // todo if partial changes, hot swap content in page
+    #onChange = async (_partial) => {
+        const update = (this.#update = Object());
+        const html = await this.#html();
+        const document = parse(html);
+        const imports = {
+            partials: [],
+            scripts: [],
+        };
+        this.#collectImports(document, imports);
+        const partials = await this.#resolvePartialContent(imports.partials);
+        if (update !== this.#update) {
+            // another update has started so aborting this one
+            return;
+        }
+        this.#addDecorations(document);
+        this.#update = update;
+        this.#document = document;
+        this.#partials = partials;
+        this.#scripts = imports.scripts;
+        const entrypoints = mergeEntrypoints(imports, ...partials.map(p => p.imports));
+        // this.#entrypoints = new Set(entrypoints.map(entrypoint => entrypoint.in))
+        this.emit('entrypoints', entrypoints);
+        this.emit('partials', this.#partials.map(p => p.fsPath));
+        if (this.listenerCount('output')) {
+            this.emit('output', this.output());
+        }
+    };
+    // Emits `partial` on detecting a partial reference for `dank serve` file watches
+    // to respond to dependent changes
+    // todo safeguard recursive partials that cause circular imports
+    async #resolvePartialContent(partials) {
+        return await Promise.all(partials.map(async (p) => {
+            this.emit('partial', p.fsPath);
+            const html = await readFile(join(this.#build.dirs.pages, p.fsPath), 'utf8');
+            const fragment = parseFragment(html);
+            const imports = {
+                partials: [],
+                scripts: [],
+            };
+            this.#collectImports(fragment, imports, node => {
+                this.#rewritePartialRelativePaths(node, p.fsPath);
+            });
+            if (imports.partials.length) {
+                // todo recursive partials?
+                // await this.#resolvePartialContent(imports.partials)
+                errorExit(`partial ${p.fsPath} cannot recursively import partials`);
+            }
+            const content = {
+                ...p,
+                fragment,
+                imports,
+            };
+            return content;
+        }));
+    }
+    // rewrite hrefs in a partial to be relative to the html entrypoint instead of the partial
+    #rewritePartialRelativePaths(elem, partialPath) {
+        let rewritePath = null;
+        if (elem.nodeName === 'script') {
+            rewritePath = 'src';
+        }
+        else if (elem.nodeName === 'link' &&
+            hasAttr(elem, 'rel', 'stylesheet')) {
+            rewritePath = 'href';
+        }
+        if (rewritePath !== null) {
+            const attr = getAttr(elem, rewritePath);
+            if (attr) {
+                attr.value = join(relative(dirname(this.#fsPath), dirname(partialPath)), attr.value);
             }
-            else if (importScript.type === 'style') {
-                importScript.elem.attrs.find(attr => attr.name === 'href').value = rewriteTo || `/${importScript.out}`;
+        }
+    }
+    #addDecorations(document) {
+        if (!this.#decorations?.length) {
+            return;
+        }
+        for (const decoration of this.#decorations) {
+            switch (decoration.type) {
+                case 'script':
+                    const scriptNode = parseFragment(`<script type="module">${decoration.js}</script>`).childNodes[0];
+                    const htmlNode = document.childNodes.find(node => node.nodeName === 'html');
+                    const headNode = htmlNode.childNodes.find(node => node.nodeName === 'head');
+                    defaultTreeAdapter.appendChild(headNode || htmlNode, scriptNode);
+                    break;
             }
         }
     }
-    appendScript(clientJS) {
-        const scriptNode = parseFragment(`<script type="module">${clientJS}</script>`).childNodes[0];
-        const htmlNode = this.#document.childNodes.find(node => node.nodeName === 'html');
-        const headNode = htmlNode.childNodes.find(node => node.nodeName === 'head');
-        defaultTreeAdapter.appendChild(headNode || htmlNode, scriptNode);
+    output(hrefs) {
+        this.#injectPartials();
+        this.#rewriteHrefs(hrefs);
+        return serialize(this.#document);
     }
-    async writeTo(buildDir) {
-        await writeFile(join(buildDir, this.#url, 'index.html'), serialize(this.#document));
+    // rewrites hrefs to content hashed urls
+    // call without hrefs to rewrite tsx? ext to js
+    #rewriteHrefs(hrefs) {
+        rewriteHrefs(this.#scripts, hrefs);
+        for (const partial of this.#partials) {
+            rewriteHrefs(partial.imports.scripts, hrefs);
+        }
     }
     async #injectPartials() {
-        for (const commentNode of this.#partials) {
-            const pp = commentNode.data
-                .match(/\{\{(?<pp>.+)\}\}/)
-                .groups.pp.trim();
-            const fragment = parseFragment(await readFile(pp, 'utf-8'));
+        for (const { commentNode, fragment } of this.#partials) {
+            if (!this.#build.production) {
+                defaultTreeAdapter.insertBefore(commentNode.parentNode, defaultTreeAdapter.createCommentNode(commentNode.data), commentNode);
+            }
             for (const node of fragment.childNodes) {
-                if (node.nodeName === 'script') {
-                    this.#rewritePathFromPartial(pp, node, 'src');
-                }
-                else if (node.nodeName === 'link' &&
-                    hasAttr(node, 'rel', 'stylesheet')) {
-                    this.#rewritePathFromPartial(pp, node, 'href');
-                }
                 defaultTreeAdapter.insertBefore(commentNode.parentNode, node, commentNode);
             }
-            defaultTreeAdapter.detachNode(commentNode);
-        }
-    }
-    // rewrite a ts or css href relative to an html partial to be relative to the html entrypoint
-    #rewritePathFromPartial(pp, elem, attrName) {
-        const attr = getAttr(elem, attrName);
-        if (attr) {
-            attr.value = join(relative(dirname(this.#fsPath), dirname(pp)), attr.value);
+            if (this.#build.production) {
+                defaultTreeAdapter.detachNode(commentNode);
+            }
         }
     }
-    #collectPartials(node) {
+    #collectImports(node, collection, forEach) {
         for (const childNode of node.childNodes) {
+            if (forEach && 'tagName' in childNode) {
+                forEach(childNode);
+            }
             if (childNode.nodeName === '#comment' && 'data' in childNode) {
-                if (/\{\{.+\}\}/.test(childNode.data)) {
-                    this.#partials.push(childNode);
+                const partialMatch = childNode.data.match(/\{\{(?<pp>.+)\}\}/);
+                if (partialMatch) {
+                    const partialSpecifier = partialMatch.groups.pp.trim();
+                    if (partialSpecifier.startsWith('/')) {
+                        errorExit(`partial ${partialSpecifier} in webpage ${this.#fsPath} cannot be an absolute path`);
+                    }
+                    const partialPath = join(dirname(this.#fsPath), partialSpecifier);
+                    if (!isPagesSubpathInPagesDir(this.#build, partialPath)) {
+                        errorExit(`partial ${partialSpecifier} in webpage ${this.#fsPath} cannot be outside of the pages directory`);
+                    }
+                    collection.partials.push({
+                        fsPath: partialPath,
+                        commentNode: childNode,
+                    });
                 }
             }
-            else if ('childNodes' in childNode) {
-                this.#collectPartials(childNode);
-            }
-        }
-    }
-    #collectScripts(node) {
-        for (const childNode of node.childNodes) {
-            if (childNode.nodeName === 'script') {
+            else if (childNode.nodeName === 'script') {
                 const srcAttr = childNode.attrs.find(attr => attr.name === 'src');
                 if (srcAttr) {
-                    this.#addScript('script', srcAttr.value, childNode);
+                    collection.scripts.push(this.#parseImport('script', srcAttr.value, childNode));
                 }
             }
             else if (childNode.nodeName === 'link' &&
                 hasAttr(childNode, 'rel', 'stylesheet')) {
                 const hrefAttr = getAttr(childNode, 'href');
                 if (hrefAttr) {
-                    this.#addScript('style', hrefAttr.value, childNode);
+                    collection.scripts.push(this.#parseImport('style', hrefAttr.value, childNode));
                 }
             }
             else if ('childNodes' in childNode) {
-                this.#collectScripts(childNode);
+                this.#collectImports(childNode, collection);
             }
         }
     }
-    #addScript(type, href, elem) {
-        const inPath = join(dirname(this.#fsPath), href);
-        let outPath = inPath.replace(/^pages\//, '');
+    #parseImport(type, href, elem) {
+        const inPath = join(this.#build.dirs.pages, dirname(this.#fsPath), href);
+        if (!isPathInPagesDir(this.#build, inPath)) {
+            errorExit(`href ${href} in webpage ${this.#fsPath} cannot reference sources outside of the pages directory`);
+        }
+        let outPath = join(dirname(this.#fsPath), href);
         if (type === 'script' && !outPath.endsWith('.js')) {
             outPath = outPath.replace(new RegExp(extname(outPath).substring(1) + '$'), 'js');
         }
-        this.#scripts.push({
+        return {
             type,
             href,
             elem,
-            in: inPath,
-            out: outPath,
-        });
+            entrypoint: {
+                in: inPath,
+                out: outPath,
+            },
+        };
     }
 }
+// check if relative dir is a subpath of pages dir when joined with pages dir
+// used if the joined pages dir path is only used for the pages dir check
+function isPagesSubpathInPagesDir(build, subpath) {
+    return isPathInPagesDir(build, join(build.dirs.pages, subpath));
+}
+// check if subpath joined with pages dir is a subpath of pages dir
+function isPathInPagesDir(build, p) {
+    return resolve(p).startsWith(build.dirs.pagesResolved);
+}
 function getAttr(elem, name) {
     return elem.attrs.find(attr => attr.name === name);
 }
 function hasAttr(elem, name, value) {
     return elem.attrs.some(attr => attr.name === name && attr.value === value);
 }
+function mergeEntrypoints(...imports) {
+    const entrypoints = [];
+    for (const { scripts } of imports) {
+        for (const script of scripts) {
+            entrypoints.push(script.entrypoint);
+        }
+    }
+    return entrypoints;
+}
+function rewriteHrefs(scripts, hrefs) {
+    for (const { elem, entrypoint, type } of scripts) {
+        const rewriteTo = hrefs ? hrefs.mappedHref(entrypoint.in) : null;
+        if (type === 'script') {
+            if (entrypoint.in.endsWith('.tsx') ||
+                entrypoint.in.endsWith('.ts')) {
+                elem.attrs.find(attr => attr.name === 'src').value =
+                    rewriteTo || `/${entrypoint.out}`;
+            }
+        }
+        else if (type === 'style') {
+            elem.attrs.find(attr => attr.name === 'href').value =
+                rewriteTo || `/${entrypoint.out}`;
+        }
+    }
+}
+function errorExit(msg) {
+    console.log(`\u001b[31merror:\u001b[0m`, msg);
+    process.exit(1);
+}

package/lib_js/http.js

@@ -2,59 +2,144 @@ import { createReadStream } from 'node:fs';
 import { stat } from 'node:fs/promises';
 import { createServer, } from 'node:http';
 import { extname, join } from 'node:path';
+import { Readable } from 'node:stream';
 import mime from 'mime';
-export function createWebServer(port, frontendFetcher) {
-    const serverAddress = 'http://localhost:' + port;
-    return createServer((req, res) => {
+export function startWebServer(serve, frontendFetcher, httpServices, pageRoutes) {
+    const serverAddress = 'http://localhost:' + serve.dankPort;
+    const handler = (req, res) => {
         if (!req.url || !req.method) {
             res.end();
         }
         else {
             const url = new URL(serverAddress + req.url);
-            if (req.method !== 'GET') {
-                res.writeHead(405);
-                res.end();
+            const headers = convertHeadersToFetch(req.headers);
+            frontendFetcher(url, headers, res, () => onNotFound(req, url, headers, httpServices, pageRoutes, serve, res));
+        }
+    };
+    createServer(serve.logHttp ? createLogWrapper(handler) : handler).listen(serve.dankPort);
+    console.log(serve.preview ? 'preview' : 'dev', `server is live at http://127.0.0.1:${serve.dankPort}`);
+}
+async function onNotFound(req, url, headers, httpServices, pageRoutes, serve, res) {
+    if (req.method === 'GET' && extname(url.pathname) === '') {
+        const urlRewrite = tryUrlRewrites(url, pageRoutes, serve);
+        if (urlRewrite) {
+            streamFile(urlRewrite, res);
+            return;
+        }
+    }
+    const fetchResponse = await tryHttpServices(req, url, headers, httpServices);
+    if (fetchResponse) {
+        sendFetchResponse(res, fetchResponse);
+    }
+    else {
+        res.writeHead(404);
+        res.end();
+    }
+}
+async function sendFetchResponse(res, fetchResponse) {
+    res.writeHead(fetchResponse.status, undefined, convertHeadersFromFetch(fetchResponse.headers));
+    if (fetchResponse.body) {
+        Readable.fromWeb(fetchResponse.body).pipe(res);
+    }
+    else {
+        res.end();
+    }
+}
+function tryUrlRewrites(url, pageRoutes, serve) {
+    const urlRewrite = pageRoutes.urlRewrites.find(urlRewrite => urlRewrite.pattern.test(url.pathname));
+    return urlRewrite
+        ? join(serve.dirs.buildWatch, urlRewrite.url, 'index.html')
+        : null;
+}
+async function tryHttpServices(req, url, headers, httpServices) {
+    if (url.pathname.startsWith('/.well-known/')) {
+        return null;
+    }
+    const body = await collectReqBody(req);
+    const { running } = httpServices;
+    for (const httpService of running) {
+        const proxyUrl = new URL(url);
+        proxyUrl.port = `${httpService.port}`;
+        try {
+            const response = await retryFetchWithTimeout(proxyUrl, {
+                body,
+                headers,
+                method: req.method,
+                redirect: 'manual',
+            });
+            if (response.status === 404 || response.status === 405) {
+                continue;
             }
             else {
-                frontendFetcher(url, convertHeadersToFetch(req.headers), res);
+                return response;
             }
         }
-    });
+        catch (e) {
+            if (e === 'retrytimeout') {
+                continue;
+            }
+            else {
+                errorExit(`unexpected error http proxying to port ${httpService.port}: ${e.message}`);
+            }
+        }
+    }
+    return null;
 }
-export function createBuiltDistFilesFetcher(dir, files) {
-    return (url, _headers, res) => {
-        if (!files.has(url.pathname)) {
-            res.writeHead(404);
-            res.end();
+function collectReqBody(req) {
+    let body = '';
+    req.on('data', data => (body += data.toString()));
+    return new Promise(res => req.on('end', () => res(body.length ? body : null)));
+}
+function createLogWrapper(handler) {
+    return (req, res) => {
+        console.log('  > ', req.method, req.url);
+        res.on('close', () => {
+            console.log('', res.statusCode, req.method, req.url);
+        });
+        handler(req, res);
+    };
+}
+export function createBuiltDistFilesFetcher(dir, manifest) {
+    return (url, _headers, res, notFound) => {
+        if (manifest.pageUrls.has(url.pathname)) {
+            streamFile(join(dir, url.pathname, 'index.html'), res);
+        }
+        else if (manifest.files.has(url.pathname)) {
+            streamFile(join(dir, url.pathname), res);
         }
         else {
-            const p = extname(url.pathname) === ''
-                ? join(dir, url.pathname, 'index.html')
-                : join(dir, url.pathname);
-            streamFile(p, res);
+            notFound();
         }
     };
 }
-export function createDevServeFilesFetcher(opts) {
-    const proxyAddress = 'http://127.0.0.1:' + opts.proxyPort;
-    return (url, _headers, res) => {
-        if (opts.pages[url.pathname]) {
-            streamFile(join(opts.pagesDir, url.pathname + 'index.html'), res);
+// todo replace PageRouteState with WebsiteRegistry
+export function createDevServeFilesFetcher(pageRoutes, serve) {
+    const proxyAddress = 'http://127.0.0.1:' + serve.esbuildPort;
+    return (url, _headers, res, notFound) => {
+        if (pageRoutes.urls.includes(url.pathname)) {
+            streamFile(join(serve.dirs.buildWatch, url.pathname, 'index.html'), res);
         }
         else {
-            const maybePublicPath = join(opts.publicDir, url.pathname);
-            exists(join(opts.publicDir, url.pathname)).then(fromPublic => {
+            const maybePublicPath = join(serve.dirs.public, url.pathname);
+            exists(maybePublicPath).then(fromPublic => {
                 if (fromPublic) {
                     streamFile(maybePublicPath, res);
                 }
                 else {
                     retryFetchWithTimeout(proxyAddress + url.pathname)
                         .then(fetchResponse => {
-                        res.writeHead(fetchResponse.status, convertHeadersFromFetch(fetchResponse.headers));
-                        fetchResponse.bytes().then(data => res.end(data));
+                        if (fetchResponse.status === 404) {
+                            notFound();
+                        }
+                        else {
+                            res.writeHead(fetchResponse.status, convertHeadersFromFetch(fetchResponse.headers));
+                            fetchResponse
+                                .bytes()
+                                .then(data => res.end(data));
+                        }
                     })
                         .catch(e => {
-                        if (e === 'retrytimeout') {
+                        if (isFetchRetryTimeout(e)) {
                             res.writeHead(504);
                         }
                         else {
@@ -70,11 +155,11 @@ export function createDevServeFilesFetcher(opts) {
 }
 const PROXY_FETCH_RETRY_INTERVAL = 27;
 const PROXY_FETCH_RETRY_TIMEOUT = 1000;
-async function retryFetchWithTimeout(url) {
+async function retryFetchWithTimeout(url, requestInit) {
     let timeout = Date.now() + PROXY_FETCH_RETRY_TIMEOUT;
     while (true) {
         try {
-            return await fetch(url);
+            return await fetch(url, requestInit);
         }
         catch (e) {
             if (isNodeFailedFetch(e) || isBunFailedFetch(e)) {
@@ -91,6 +176,9 @@ async function retryFetchWithTimeout(url) {
         }
     }
 }
+function isFetchRetryTimeout(e) {
+    return e === 'retrytimeout';
+}
 function isBunFailedFetch(e) {
     return e.code === 'ConnectionRefused';
 }
@@ -136,3 +224,7 @@ function convertHeadersToFetch(from) {
     }
     return to;
 }
+function errorExit(msg) {
+    console.log(`\u001b[31merror:\u001b[0m`, msg);
+    process.exit(1);
+}