@eighty4/dank 0.0.1-0

package/lib/manifest.ts

@@ -0,0 +1,61 @@
+import { writeFile } from 'node:fs/promises'
+import { extname, join } from 'node:path'
+
+// catalog of build's filesystem output
+export type BuildManifest = {
+    buildTag: string
+    files: Array<string>
+}
+
+// functional data for service worker
+export type CacheManifest = {
+    apiRoutes: Array<string>
+    buildTag: string
+    files: Array<string>
+}
+
+export async function writeBuildManifest(buildTag: string, files: Set<string>) {
+    await writeJsonToBuildDir('manifest.json', {
+        buildTag,
+        files: Array.from(files).map(f =>
+            extname(f).length
+                ? f
+                : f === '/'
+                  ? '/index.html'
+                  : f + '/index.html',
+        ),
+    })
+}
+
+export async function writeJsonToBuildDir(
+    filename: `${string}.json`,
+    json: any,
+) {
+    await writeFile(join('./build', filename), JSON.stringify(json, null, 4))
+}
+
+export async function writeMetafile(filename: `${string}.json`, json: any) {
+    await writeJsonToBuildDir(
+        join('metafiles', filename) as `${string}.json`,
+        json,
+    )
+}
+
+export async function writeCacheManifest(buildTag: string, files: Set<string>) {
+    await writeJsonToBuildDir('cache.json', {
+        apiRoutes: [],
+        buildTag,
+        files: Array.from(files).map(filenameToWebappPath),
+    })
+}
+
+// drops index.html from path
+function filenameToWebappPath(p: string): string {
+    if (p === '/index.html') {
+        return '/'
+    } else if (p.endsWith('/index.html')) {
+        return p.substring(0, p.length - '/index.html'.length) as `/${string}`
+    } else {
+        return p
+    }
+}

package/lib/public.ts

@@ -0,0 +1,48 @@
+import { copyFile, mkdir, readdir, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+
+export async function copyAssets(
+    outRoot: string,
+): Promise<Array<string> | null> {
+    try {
+        const stats = await stat('public')
+        if (stats.isDirectory()) {
+            await mkdir(outRoot, { recursive: true })
+            return await recursiveCopyAssets(outRoot)
+        } else {
+            throw Error('./public cannot be a file')
+        }
+    } catch (e) {
+        return null
+    }
+}
+
+async function recursiveCopyAssets(
+    outRoot: string,
+    dir: string = '',
+): Promise<Array<string>> {
+    const copied: Array<string> = []
+    const to = join(outRoot, dir)
+    let madeDir = dir === ''
+    for (const p of await readdir(join('public', dir))) {
+        try {
+            const stats = await stat(join('public', dir, p))
+            if (stats.isDirectory()) {
+                copied.push(
+                    ...(await recursiveCopyAssets(outRoot, join(dir, p))),
+                )
+            } else {
+                if (!madeDir) {
+                    await mkdir(join(outRoot, dir))
+                    madeDir = true
+                }
+                await copyFile(join('public', dir, p), join(to, p))
+                copied.push('/' + join(dir, p).replaceAll('\\', '/'))
+            }
+        } catch (e) {
+            console.error('stat error', e)
+            process.exit(1)
+        }
+    }
+    return copied
+}

package/lib/serve.ts

@@ -0,0 +1,96 @@
+import { mkdir, rm } from 'node:fs/promises'
+import { join } from 'node:path'
+import { buildWebsite } from './build.ts'
+import type { DankConfig } from './dank.ts'
+import { createGlobalDefinitions } from './define.ts'
+import { esbuildDevContext } from './esbuild.ts'
+import { isPreviewBuild } from './flags.ts'
+import { HtmlEntrypoint } from './html.ts'
+import {
+    createBuiltDistFilesFetcher,
+    createLocalProxyFilesFetcher,
+    createWebServer,
+    type FrontendFetcher,
+} from './http.ts'
+import { copyAssets } from './public.ts'
+
+const isPreview = isPreviewBuild()
+
+// alternate port for --preview bc of service worker
+const PORT = isPreview ? 4000 : 3000
+
+// port for esbuild.serve
+const ESBUILD_PORT = 2999
+
+export async function serveWebsite(c: DankConfig): Promise<never> {
+    await rm('build', { force: true, recursive: true })
+    let frontend: FrontendFetcher
+    if (isPreview) {
+        const { dir, files } = await buildWebsite(c)
+        frontend = createBuiltDistFilesFetcher(dir, files)
+    } else {
+        const { port } = await startEsbuildWatch(c)
+        frontend = createLocalProxyFilesFetcher(port)
+    }
+    createWebServer(PORT, frontend).listen(PORT)
+    console.log(
+        isPreview ? 'preview' : 'dev server',
+        `is live at http://127.0.0.1:${PORT}`,
+    )
+    return new Promise(() => {})
+}
+
+async function startEsbuildWatch(c: DankConfig): Promise<{ port: number }> {
+    const watchDir = join('build', 'watch')
+    await mkdir(watchDir, { recursive: true })
+    await copyAssets(watchDir)
+
+    const entryPointUrls: Set<string> = new Set()
+    const entryPoints: Array<{ in: string; out: string }> = []
+
+    await Promise.all(
+        Object.entries(c.pages).map(async ([url, srcPath]) => {
+            const html = await HtmlEntrypoint.readFrom(
+                url,
+                join('pages', srcPath),
+            )
+            await html.injectPartials()
+            if (url !== '/') {
+                await mkdir(join(watchDir, url), { recursive: true })
+            }
+            html.collectScripts()
+                .filter(scriptImport => !entryPointUrls.has(scriptImport.in))
+                .forEach(scriptImport => {
+                    entryPointUrls.add(scriptImport.in)
+                    entryPoints.push({
+                        in: scriptImport.in,
+                        out: scriptImport.out,
+                    })
+                })
+            html.rewriteHrefs()
+            await html.writeTo(watchDir)
+            return html
+        }),
+    )
+
+    console.log(entryPoints)
+
+    const ctx = await esbuildDevContext(
+        createGlobalDefinitions(),
+        entryPoints,
+        watchDir,
+    )
+
+    await ctx.watch()
+
+    await ctx.serve({
+        host: '127.0.0.1',
+        port: ESBUILD_PORT,
+        servedir: watchDir,
+        cors: {
+            origin: 'http://127.0.0.1:' + PORT,
+        },
+    })
+
+    return { port: ESBUILD_PORT }
+}

package/lib/tag.ts

@@ -0,0 +1,25 @@
+import { exec } from 'node:child_process'
+import { isProductionBuild } from './flags.ts'
+
+export async function createBuildTag(): Promise<string> {
+    const now = new Date()
+    const ms =
+        now.getUTCMilliseconds() +
+        now.getUTCSeconds() * 1000 +
+        now.getUTCMinutes() * 1000 * 60 +
+        now.getUTCHours() * 1000 * 60 * 60
+    const date = now.toISOString().substring(0, 10)
+    const time = String(ms).padStart(8, '0')
+    const when = `${date}-${time}`
+    if (isProductionBuild()) {
+        const gitHash = await new Promise((res, rej) =>
+            exec('git rev-parse --short HEAD', (err, stdout) => {
+                if (err) rej(err)
+                res(stdout.trim())
+            }),
+        )
+        return `${when}-${gitHash}`
+    } else {
+        return when
+    }
+}

package/lib_js/bin.js

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+import { buildWebsite } from "./build.js";
+import { loadConfig } from "./config.js";
+import { serveWebsite } from "./serve.js";
+function printHelp(task) {
+    if (!task || task === 'build') {
+        console.log('dank build [--minify] [--production]');
+    }
+    if (!task || task === 'serve') {
+        console.log(
+        // 'dank serve [--minify] [--preview] [--production]',
+        'dank serve [--minify] [--production]');
+    }
+    console.log('\nOPTIONS:');
+    console.log('  --minify       minify sources');
+    // if (!task || task === 'serve') {
+    // console.log('  --preview      pre-bundle and build ServiceWorker')
+    // }
+    console.log('  --production   build for production release');
+    if (task) {
+        console.log();
+        console.log('use `dank -h` for details on all commands');
+    }
+    process.exit(1);
+}
+const args = (function collectProgramArgs() {
+    const programNames = ['dank', 'bin.js', 'bin.ts'];
+    let args = [...process.argv];
+    while (true) {
+        const shifted = args.shift();
+        if (!shifted || programNames.some(name => shifted.endsWith(name))) {
+            return args;
+        }
+    }
+})();
+const task = (function resolveTask() {
+    const showHelp = args.some(arg => arg === '-h' || arg === '--help');
+    const task = (() => {
+        while (true) {
+            const shifted = args.shift();
+            switch (shifted) {
+                case '-h':
+                case '--help':
+                    break;
+                case 'build':
+                    return 'build';
+                case 'dev':
+                case 'serve':
+                    return 'serve';
+                default:
+                    if (showHelp) {
+                        printHelp();
+                    }
+                    else if (typeof shifted === 'undefined') {
+                        printError('missing command');
+                        printHelp();
+                    }
+                    else {
+                        printError(shifted + " isn't a command");
+                        printHelp();
+                    }
+            }
+        }
+    })();
+    if (showHelp) {
+        printHelp(task);
+    }
+    return task;
+})();
+const c = await loadConfig();
+try {
+    switch (task) {
+        case 'build':
+            await buildWebsite(c);
+            console.log(green('done'));
+            process.exit(0);
+        case 'serve':
+            await serveWebsite(c);
+    }
+}
+catch (e) {
+    errorExit(e);
+}
+function printError(e) {
+    if (e !== null) {
+        if (typeof e === 'string') {
+            console.error(red('error:'), e);
+        }
+        else if (e instanceof Error) {
+            console.error(red('error:'), e.message);
+            if (e.stack) {
+                console.error(e.stack);
+            }
+        }
+    }
+}
+function green(s) {
+    return `\u001b[32m${s}\u001b[0m`;
+}
+function red(s) {
+    return `\u001b[31m${s}\u001b[0m`;
+}
+function errorExit(e) {
+    printError(e);
+    process.exit(1);
+}

package/lib_js/build.js

@@ -0,0 +1,70 @@
+import { mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { isProductionBuild, willMinify } from "./flags.js";
+import { copyAssets } from "./public.js";
+import { createBuildTag } from "./tag.js";
+import { writeBuildManifest, writeMetafile } from "./manifest.js";
+import { createGlobalDefinitions } from "./define.js";
+import { HtmlEntrypoint } from "./html.js";
+import { esbuildWebpages } from "./esbuild.js";
+export async function buildWebsite(c) {
+    const buildDir = 'build';
+    const distDir = join(buildDir, 'dist');
+    const buildTag = await createBuildTag();
+    console.log(willMinify()
+        ? isProductionBuild()
+            ? 'minified production'
+            : 'minified'
+        : 'unminified', 'build', buildTag, 'building in ./build/dist');
+    await rm(buildDir, { recursive: true, force: true });
+    await mkdir(distDir, { recursive: true });
+    await mkdir(join(buildDir, 'metafiles'), { recursive: true });
+    const staticAssets = await copyAssets(distDir);
+    const buildUrls = [];
+    buildUrls.push(...(await buildWebpages(distDir, createGlobalDefinitions(), c.pages)));
+    if (staticAssets) {
+        buildUrls.push(...staticAssets);
+    }
+    const result = new Set(buildUrls);
+    await writeBuildManifest(buildTag, result);
+    return {
+        dir: buildDir,
+        files: result,
+    };
+}
+async function buildWebpages(distDir, define, pages) {
+    const entryPointUrls = new Set();
+    const entryPoints = [];
+    const htmlEntrypoints = await Promise.all(Object.entries(pages).map(async ([urlPath, fsPath]) => {
+        const html = await HtmlEntrypoint.readFrom(urlPath, join('pages', fsPath));
+        await html.injectPartials();
+        if (urlPath !== '/') {
+            await mkdir(join(distDir, urlPath), { recursive: true });
+        }
+        html.collectScripts()
+            .filter(scriptImport => !entryPointUrls.has(scriptImport.in))
+            .forEach(scriptImport => {
+            entryPointUrls.add(scriptImport.in);
+            entryPoints.push({
+                in: scriptImport.in,
+                out: scriptImport.out,
+            });
+        });
+        return html;
+    }));
+    const metafile = await esbuildWebpages(define, entryPoints, distDir);
+    await writeMetafile(`pages.json`, metafile);
+    // todo these hrefs would have \ path separators on windows
+    const buildUrls = [...Object.keys(pages)];
+    const mapInToOutHrefs = {};
+    for (const [outputFile, { entryPoint }] of Object.entries(metafile.outputs)) {
+        const outputUrl = outputFile.replace(/^build\/dist/, '');
+        buildUrls.push(outputUrl);
+        mapInToOutHrefs[entryPoint] = outputUrl;
+    }
+    await Promise.all(htmlEntrypoints.map(async (html) => {
+        html.rewriteHrefs(mapInToOutHrefs);
+        await html.writeTo(distDir);
+    }));
+    return buildUrls;
+}

package/lib_js/config.js

@@ -0,0 +1,22 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { isAbsolute, resolve } from 'node:path';
+const CFG_P = './dank.config.ts';
+export async function loadConfig(path = CFG_P) {
+    const module = await import(__rewriteRelativeImportExtension(resolveConfigPath(path)));
+    return await module.default;
+}
+export function resolveConfigPath(path) {
+    if (isAbsolute(path)) {
+        return path;
+    }
+    else {
+        return resolve(process.cwd(), path);
+    }
+}

package/lib_js/dank.js

@@ -0,0 +1,11 @@
+export async function defineConfig(c) {
+    if (typeof c.pages === 'undefined' || Object.keys(c.pages).length === 0) {
+        throw Error('DankConfig.pages is required');
+    }
+    for (const [urlPath, htmlPath] of Object.entries(c.pages)) {
+        if (typeof htmlPath !== 'string' || !htmlPath.endsWith('.html')) {
+            throw Error(`DankConfig.pages['${urlPath}'] must configure an html file`);
+        }
+    }
+    return c;
+}

package/lib_js/define.js

@@ -0,0 +1,8 @@
+import { isProductionBuild } from "./flags.js";
+export function createGlobalDefinitions() {
+    const isProduction = isProductionBuild();
+    return {
+        'dank.IS_DEV': JSON.stringify(!isProduction),
+        'dank.IS_PROD': JSON.stringify(isProduction),
+    };
+}

package/lib_js/esbuild.js

@@ -0,0 +1,64 @@
+import esbuild, {} from 'esbuild';
+import { willMinify } from "./flags.js";
+const jsBuildOptions = {
+    bundle: true,
+    metafile: true,
+    minify: willMinify(),
+    platform: 'browser',
+    splitting: false,
+    treeShaking: true,
+    write: true,
+};
+const webpageBuildOptions = {
+    assetNames: 'assets/[name]-[hash]',
+    format: 'esm',
+    ...jsBuildOptions,
+};
+export async function esbuildDevContext(define, entryPoints, outdir) {
+    return await esbuild.context({
+        define,
+        entryNames: '[dir]/[name]',
+        entryPoints,
+        outdir,
+        ...webpageBuildOptions,
+    });
+}
+export async function esbuildWebpages(define, entryPoints, outdir) {
+    const buildResult = await esbuild.build({
+        define,
+        entryNames: '[dir]/[name]-[hash]',
+        entryPoints: removeEntryPointOutExt(entryPoints),
+        outdir,
+        ...webpageBuildOptions,
+    });
+    esbuildResultChecks(buildResult);
+    return buildResult.metafile;
+}
+function removeEntryPointOutExt(entryPoints) {
+    return entryPoints.map(entryPoint => {
+        return {
+            in: entryPoint.in,
+            out: entryPoint.out.replace(/\.(tsx?|jsx?|css)$/, ''),
+        };
+    });
+}
+function esbuildResultChecks(buildResult) {
+    if (buildResult.errors.length) {
+        buildResult.errors.forEach(msg => esbuildPrintMessage(msg, 'warning'));
+        process.exit(1);
+    }
+    if (buildResult.warnings.length) {
+        buildResult.warnings.forEach(msg => esbuildPrintMessage(msg, 'warning'));
+    }
+}
+function esbuildPrintMessage(msg, category) {
+    const location = msg.location
+        ? ` (${msg.location.file}L${msg.location.line}:${msg.location.column})`
+        : '';
+    console.error(`esbuild ${category}${location}:`, msg.text);
+    msg.notes.forEach(note => {
+        console.error('  ', note.text);
+        if (note.location)
+            console.error('   ', note.location);
+    });
+}

package/lib_js/flags.js

@@ -0,0 +1,11 @@
+// `dank serve` will pre-bundle and use service worker
+export const isPreviewBuild = () => process.env.PREVIEW === 'true' || process.argv.includes('--preview');
+// `dank build` will minify sources and append git release tag to build tag
+// `dank serve` will pre-bundle with service worker and minify
+export const isProductionBuild = () => process.env.PRODUCTION === 'true' || process.argv.includes('--production');
+export const willMinify = () => isProductionBuild() ||
+    process.env.MINIFY === 'true' ||
+    process.argv.includes('--minify');
+export const willTsc = () => isProductionBuild() ||
+    process.env.TSC === 'true' ||
+    process.argv.includes('--tsc');

package/lib_js/html.js

@@ -0,0 +1,132 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { dirname, join, relative } from 'node:path';
+import { defaultTreeAdapter, parse, parseFragment, serialize, } from 'parse5';
+// unenforced but necessary sequence:
+//  injectPartials
+//  collectScripts
+//  rewriteHrefs
+//  writeTo
+export class HtmlEntrypoint {
+    static async readFrom(urlPath, fsPath) {
+        let html;
+        try {
+            html = await readFile(fsPath, 'utf-8');
+        }
+        catch (e) {
+            console.log(`\u001b[31merror:\u001b[0m`, fsPath, 'does not exist');
+            process.exit(1);
+        }
+        return new HtmlEntrypoint(urlPath, html, fsPath);
+    }
+    #document;
+    #fsPath;
+    #partials = [];
+    #scripts = [];
+    #url;
+    constructor(url, html, fsPath) {
+        this.#url = url;
+        this.#document = parse(html);
+        this.#fsPath = fsPath;
+    }
+    async injectPartials() {
+        this.#collectPartials(this.#document);
+        await this.#injectPartials();
+    }
+    collectScripts() {
+        this.#collectScripts(this.#document);
+        return this.#scripts;
+    }
+    // rewrites hrefs to content hashed urls
+    // call without hrefs to rewrite tsx? ext to js
+    rewriteHrefs(hrefs) {
+        for (const importScript of this.#scripts) {
+            const rewriteTo = hrefs ? hrefs[importScript.in] : null;
+            if (importScript.type === 'script') {
+                if (importScript.in.endsWith('.tsx') ||
+                    importScript.in.endsWith('.ts')) {
+                    importScript.elem.attrs.find(attr => attr.name === 'src').value = rewriteTo || `/${importScript.out}.js`;
+                }
+            }
+            else if (importScript.type === 'style') {
+                importScript.elem.attrs.find(attr => attr.name === 'href').value = rewriteTo || `/${importScript.out}.css`;
+            }
+        }
+    }
+    async writeTo(buildDir) {
+        await writeFile(join(buildDir, this.#url, 'index.html'), serialize(this.#document));
+    }
+    async #injectPartials() {
+        for (const commentNode of this.#partials) {
+            const pp = commentNode.data
+                .match(/\{\{(?<pp>.+)\}\}/)
+                .groups.pp.trim();
+            const fragment = parseFragment(await readFile(pp, 'utf-8'));
+            for (const node of fragment.childNodes) {
+                if (node.nodeName === 'script') {
+                    this.#rewritePathFromPartial(pp, node, 'src');
+                }
+                else if (node.nodeName === 'link' &&
+                    hasAttr(node, 'rel', 'stylesheet')) {
+                    this.#rewritePathFromPartial(pp, node, 'href');
+                }
+                defaultTreeAdapter.insertBefore(commentNode.parentNode, node, commentNode);
+            }
+            defaultTreeAdapter.detachNode(commentNode);
+        }
+    }
+    // rewrite a ts or css href relative to an html partial to be relative to the html entrypoint
+    #rewritePathFromPartial(pp, elem, attrName) {
+        const attr = getAttr(elem, attrName);
+        if (attr) {
+            attr.value = join(relative(dirname(this.#fsPath), dirname(pp)), attr.value);
+        }
+    }
+    #collectPartials(node) {
+        for (const childNode of node.childNodes) {
+            if (childNode.nodeName === '#comment' && 'data' in childNode) {
+                if (/\{\{.+\}\}/.test(childNode.data)) {
+                    this.#partials.push(childNode);
+                }
+            }
+            else if ('childNodes' in childNode) {
+                this.#collectPartials(childNode);
+            }
+        }
+    }
+    #collectScripts(node) {
+        for (const childNode of node.childNodes) {
+            if (childNode.nodeName === 'script') {
+                const srcAttr = childNode.attrs.find(attr => attr.name === 'src');
+                if (srcAttr) {
+                    this.#addScript('script', srcAttr.value, childNode);
+                }
+            }
+            else if (childNode.nodeName === 'link' &&
+                hasAttr(childNode, 'rel', 'stylesheet')) {
+                const hrefAttr = getAttr(childNode, 'href');
+                if (hrefAttr) {
+                    this.#addScript('style', hrefAttr.value, childNode);
+                }
+            }
+            else if ('childNodes' in childNode) {
+                this.#collectScripts(childNode);
+            }
+        }
+    }
+    #addScript(type, href, elem) {
+        const inPath = join(dirname(this.#fsPath), href);
+        this.#scripts.push({
+            type,
+            href,
+            elem,
+            in: inPath,
+            out: inPath.replace(/^pages\//, ''),
+        });
+    }
+}
+function getAttr(elem, name) {
+    return elem.attrs.find(attr => attr.name === name);
+}
+function hasAttr(elem, name, value) {
+    return elem.attrs.some(attr => attr.name === name && attr.value === value);
+}