@eighty4/dank 0.0.2 → 0.0.4-0

package/lib_js/build.js

@@ -1,70 +1,95 @@
-import { mkdir, rm } from 'node:fs/promises';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { isProductionBuild, willMinify } from "./flags.js";
-import { copyAssets } from "./public.js";
-import { createBuildTag } from "./tag.js";
-import { writeBuildManifest, writeMetafile } from "./manifest.js";
+import { createBuildTag } from "./build_tag.js";
 import { createGlobalDefinitions } from "./define.js";
+import { esbuildWebpages, esbuildWorkers } from "./esbuild.js";
+import { resolveBuildFlags } from "./flags.js";
 import { HtmlEntrypoint } from "./html.js";
-import { esbuildWebpages } from "./esbuild.js";
-export async function buildWebsite(c) {
-    const buildDir = 'build';
-    const distDir = join(buildDir, 'dist');
-    const buildTag = await createBuildTag();
-    console.log(willMinify()
-        ? isProductionBuild()
+import { WebsiteRegistry } from "./metadata.js";
+import { copyAssets } from "./public.js";
+export async function buildWebsite(c, build = resolveBuildFlags()) {
+    const buildTag = await createBuildTag(build);
+    console.log(build.minify
+        ? build.production
             ? 'minified production'
             : 'minified'
         : 'unminified', 'build', buildTag, 'building in ./build/dist');
-    await rm(buildDir, { recursive: true, force: true });
-    await mkdir(distDir, { recursive: true });
-    await mkdir(join(buildDir, 'metafiles'), { recursive: true });
-    const staticAssets = await copyAssets(distDir);
-    const buildUrls = [];
-    buildUrls.push(...(await buildWebpages(distDir, createGlobalDefinitions(), c.pages)));
-    if (staticAssets) {
-        buildUrls.push(...staticAssets);
+    await rm(build.dirs.buildRoot, { recursive: true, force: true });
+    await mkdir(build.dirs.buildDist, { recursive: true });
+    for (const subdir of Object.keys(c.pages).filter(url => url !== '/')) {
+        await mkdir(join(build.dirs.buildDist, subdir), { recursive: true });
     }
-    const result = new Set(buildUrls);
-    await writeBuildManifest(buildTag, result);
-    return {
-        dir: distDir,
-        files: result,
-    };
+    await mkdir(join(build.dirs.buildRoot, 'metafiles'), { recursive: true });
+    const registry = new WebsiteRegistry(build);
+    registry.pageUrls = Object.keys(c.pages);
+    registry.copiedAssets = await copyAssets(build);
+    await buildWebpages(c, registry, build, createGlobalDefinitions(build));
+    return await registry.writeManifest(buildTag);
 }
-async function buildWebpages(distDir, define, pages) {
-    const entryPointUrls = new Set();
-    const entryPoints = [];
-    const htmlEntrypoints = await Promise.all(Object.entries(pages).map(async ([urlPath, fsPath]) => {
-        const html = await HtmlEntrypoint.readFrom(urlPath, join('pages', fsPath));
-        await html.injectPartials();
-        if (urlPath !== '/') {
-            await mkdir(join(distDir, urlPath), { recursive: true });
+// builds all webpage entrypoints in one esbuild.build context
+// to support code splitting
+// returns all built assets URLs and webpage URLs from DankConfig.pages
+async function buildWebpages(c, registry, build, define) {
+    // create HtmlEntrypoint for each webpage and collect awaitable esbuild entrypoints
+    const loadingEntryPoints = [];
+    const htmlEntrypoints = [];
+    for (const [urlPath, mapping] of Object.entries(c.pages)) {
+        const fsPath = typeof mapping === 'string' ? mapping : mapping.webpage;
+        const html = new HtmlEntrypoint(build, urlPath, fsPath);
+        loadingEntryPoints.push(new Promise(res => html.on('entrypoints', res)));
+        htmlEntrypoints.push(html);
+    }
+    // collect esbuild entrypoints from every HtmlEntrypoint
+    const uniqueEntryPoints = new Set();
+    const buildEntryPoints = [];
+    for (const pageEntryPoints of await Promise.all(loadingEntryPoints)) {
+        for (const entryPoint of pageEntryPoints) {
+            if (!uniqueEntryPoints.has(entryPoint.in)) {
+                buildEntryPoints.push(entryPoint);
+            }
         }
-        html.collectScripts()
-            .filter(scriptImport => !entryPointUrls.has(scriptImport.in))
-            .forEach(scriptImport => {
-            entryPointUrls.add(scriptImport.in);
-            entryPoints.push({
-                in: scriptImport.in,
-                out: scriptImport.out,
-            });
-        });
-        return html;
-    }));
-    const metafile = await esbuildWebpages(define, entryPoints, distDir);
-    await writeMetafile(`pages.json`, metafile);
-    // todo these hrefs would have \ path separators on windows
-    const buildUrls = [...Object.keys(pages)];
-    const mapInToOutHrefs = {};
-    for (const [outputFile, { entryPoint }] of Object.entries(metafile.outputs)) {
-        const outputUrl = outputFile.replace(/^build\/dist/, '');
-        buildUrls.push(outputUrl);
-        mapInToOutHrefs[entryPoint] = outputUrl;
     }
+    await esbuildWebpages(build, registry, define, buildEntryPoints, c.esbuild);
+    // todo recursively build workers on building workers that create workers
+    const workerEntryPoints = registry.workerEntryPoints();
+    if (workerEntryPoints?.length) {
+        await esbuildWorkers(build, registry, define, workerEntryPoints, c.esbuild);
+    }
+    await rewriteWorkerUrls(build, registry);
+    // write out html output with rewritten hrefs
     await Promise.all(htmlEntrypoints.map(async (html) => {
-        html.rewriteHrefs(mapInToOutHrefs);
-        await html.writeTo(distDir);
+        await writeFile(join(build.dirs.buildDist, html.url, 'index.html'), html.output(registry));
+    }));
+}
+export async function rewriteWorkerUrls(build, registry) {
+    const workers = registry.workers();
+    if (!workers) {
+        return;
+    }
+    const dependentBundlePaths = workers.map(w => registry.mappedHref(w.dependentEntryPoint));
+    const bundleOutputs = {};
+    // collect all js file contents concurrently
+    const readingFiles = Promise.all(dependentBundlePaths.map(async (p) => {
+        bundleOutputs[p] = await readFile(join(build.dirs.projectRootAbs, build.dirs.buildDist, p), 'utf8');
+    }));
+    // build regex replacements during file reads
+    const rewriteChains = {};
+    for (const p of dependentBundlePaths)
+        rewriteChains[p] = [];
+    for (const w of workers) {
+        rewriteChains[registry.mappedHref(w.dependentEntryPoint)].push(s => s.replace(createWorkerRegex(w.workerUrlPlaceholder), `new Worker('${registry.mappedHref(w.workerEntryPoint)}')`));
+    }
+    // wait for file reads
+    await readingFiles;
+    // run rewrite regex chain and write back to dist
+    await Promise.all(Object.entries(bundleOutputs).map(async ([p, content]) => {
+        let result = content;
+        for (const rewriteFn of rewriteChains[p]) {
+            result = rewriteFn(result);
+        }
+        await writeFile(join(build.dirs.projectRootAbs, build.dirs.buildDist, p), result);
     }));
-    return buildUrls;
+}
+export function createWorkerRegex(workerUrl) {
+    return new RegExp(`new(?:\\s|\\r?\\n)+Worker(?:\\s|\\r?\\n)*\\((?:\\s|\\r?\\n)*['"]${workerUrl}['"](?:\\s|\\r?\\n)*\\)`, 'g');
 }

package/lib_js/{tag.js → build_tag.js}

@@ -1,6 +1,5 @@
 import { exec } from 'node:child_process';
-import { isProductionBuild } from "./flags.js";
-export async function createBuildTag() {
+export async function createBuildTag(build) {
     const now = new Date();
     const ms = now.getUTCMilliseconds() +
         now.getUTCSeconds() * 1000 +
@@ -9,7 +8,7 @@ export async function createBuildTag() {
     const date = now.toISOString().substring(0, 10);
     const time = String(ms).padStart(8, '0');
     const when = `${date}-${time}`;
-    if (isProductionBuild()) {
+    if (build.production) {
         const gitHash = await new Promise((res, rej) => exec('git rev-parse --short HEAD', (err, stdout) => {
             if (err)
                 rej(err);

package/lib_js/dank.js

@@ -1,21 +1,76 @@
 export async function defineConfig(c) {
+    if (c.port !== null && typeof c.port !== 'undefined') {
+        if (typeof c.port !== 'number') {
+            throw Error('DankConfig.port must be a number');
+        }
+    }
+    if (c.previewPort !== null && typeof c.previewPort !== 'undefined') {
+        if (typeof c.previewPort !== 'number') {
+            throw Error('DankConfig.previewPort must be a number');
+        }
+    }
     validatePages(c.pages);
     validateDevServices(c.services);
+    validateEsbuildConfig(c.esbuild);
     normalizePagePaths(c.pages);
     return c;
 }
+function validateEsbuildConfig(esbuild) {
+    if (esbuild?.loaders !== null && typeof esbuild?.loaders !== 'undefined') {
+        if (typeof esbuild.loaders !== 'object') {
+            throw Error('DankConfig.esbuild.loaders must be a map of extensions to esbuild loaders');
+        }
+        else {
+            for (const [ext, loader] of Object.entries(esbuild.loaders)) {
+                if (typeof loader !== 'string') {
+                    throw Error(`DankConfig.esbuild.loaders['${ext}'] must be a string of a loader name`);
+                }
+            }
+        }
+    }
+    if (esbuild?.plugins !== null && typeof esbuild?.plugins !== 'undefined') {
+        if (!Array.isArray(esbuild.plugins)) {
+            throw Error('DankConfig.esbuild.plugins must be an array of esbuild plugins');
+        }
+    }
+    if (esbuild?.port !== null && typeof esbuild?.port !== 'undefined') {
+        if (typeof esbuild.port !== 'number') {
+            throw Error('DankConfig.esbuild.port must be a number');
+        }
+    }
+}
 function validatePages(pages) {
     if (pages === null ||
         typeof pages === 'undefined' ||
         Object.keys(pages).length === 0) {
         throw Error('DankConfig.pages is required');
     }
-    for (const [urlPath, htmlPath] of Object.entries(pages)) {
-        if (typeof htmlPath !== 'string' || !htmlPath.endsWith('.html')) {
-            throw Error(`DankConfig.pages['${urlPath}'] must configure an html file`);
+    for (const [urlPath, mapping] of Object.entries(pages)) {
+        if (typeof mapping === 'string' && mapping.endsWith('.html')) {
+            continue;
+        }
+        if (typeof mapping === 'object') {
+            validatePageMapping(urlPath, mapping);
+            continue;
         }
+        throw Error(`DankConfig.pages['${urlPath}'] must configure an html file`);
     }
 }
+function validatePageMapping(urlPath, mapping) {
+    if (mapping.webpage === null ||
+        typeof mapping.webpage !== 'string' ||
+        !mapping.webpage.endsWith('.html')) {
+        throw Error(`DankConfig.pages['${urlPath}'].webpage must configure an html file`);
+    }
+    if (mapping.pattern === null || typeof mapping.pattern === 'undefined') {
+        return;
+    }
+    if (typeof mapping.pattern === 'object' &&
+        mapping.pattern.constructor.name === 'RegExp') {
+        return;
+    }
+    throw Error(`DankConfig.pages['${urlPath}'].pattern must be a RegExp`);
+}
 function validateDevServices(services) {
     if (services === null || typeof services === 'undefined') {
         return;
@@ -46,10 +101,23 @@ function validateDevServices(services) {
                 }
             }
         }
+        if (s.http !== null && typeof s.http !== 'undefined') {
+            if (typeof s.http.port !== 'number') {
+                throw Error(`DankConfig.services[${i}].http.port must be a number`);
+            }
+        }
     }
 }
 function normalizePagePaths(pages) {
-    for (const urlPath of Object.keys(pages)) {
-        pages[urlPath] = pages[urlPath].replace(/^\.\//, '');
+    for (const [pageUrl, mapping] of Object.entries(pages)) {
+        if (typeof mapping === 'string') {
+            pages[pageUrl] = normalizePagePath(mapping);
+        }
+        else {
+            mapping.webpage = normalizePagePath(mapping.webpage);
+        }
     }
 }
+function normalizePagePath(p) {
+    return p.replace(/^\.\//, '');
+}

package/lib_js/define.js

@@ -1,8 +1,6 @@
-import { isProductionBuild } from "./flags.js";
-export function createGlobalDefinitions() {
-    const isProduction = isProductionBuild();
+export function createGlobalDefinitions(build) {
     return {
-        'dank.IS_DEV': JSON.stringify(!isProduction),
-        'dank.IS_PROD': JSON.stringify(isProduction),
+        'dank.IS_DEV': JSON.stringify(!build.production),
+        'dank.IS_PROD': JSON.stringify(build.production),
     };
 }

package/lib_js/esbuild.js

@@ -1,50 +1,67 @@
+import { readFile } from 'node:fs/promises';
 import esbuild, {} from 'esbuild';
-import { willMinify } from "./flags.js";
-const jsBuildOptions = {
-    bundle: true,
-    metafile: true,
-    minify: willMinify(),
-    platform: 'browser',
-    splitting: false,
-    treeShaking: true,
-    write: true,
-};
-const webpageBuildOptions = {
-    assetNames: 'assets/[name]-[hash]',
-    format: 'esm',
-    loader: {
-        '.tff': 'file',
-        '.woff': 'file',
-        '.woff2': 'file',
-    },
-    ...jsBuildOptions,
-};
-export async function esbuildDevContext(define, entryPoints, outdir) {
+export async function esbuildDevContext(b, r, define, entryPoints, c) {
     return await esbuild.context({
         define,
         entryNames: '[dir]/[name]',
-        entryPoints: removeEntryPointOutExt(entryPoints),
-        outdir,
-        ...webpageBuildOptions,
-        metafile: false,
+        entryPoints: mapEntryPointPaths(entryPoints),
+        outdir: b.dirs.buildWatch,
+        ...commonBuildOptions(b, r, c),
+        splitting: false,
         write: false,
     });
 }
-export async function esbuildWebpages(define, entryPoints, outdir) {
-    const buildResult = await esbuild.build({
+export async function esbuildWebpages(b, r, define, entryPoints, c) {
+    const result = await esbuild.build({
+        define,
+        entryNames: '[dir]/[name]-[hash]',
+        entryPoints: mapEntryPointPaths(entryPoints),
+        outdir: b.dirs.buildDist,
+        ...commonBuildOptions(b, r, c),
+    });
+    esbuildResultChecks(result);
+}
+export async function esbuildWorkers(b, r, define, entryPoints, c) {
+    const result = await esbuild.build({
         define,
         entryNames: '[dir]/[name]-[hash]',
-        entryPoints: removeEntryPointOutExt(entryPoints),
-        outdir,
-        ...webpageBuildOptions,
+        entryPoints: mapEntryPointPaths(entryPoints),
+        outdir: b.dirs.buildDist,
+        ...commonBuildOptions(b, r, c),
+        splitting: false,
+        metafile: true,
+        write: true,
+        assetNames: 'assets/[name]-[hash]',
     });
-    esbuildResultChecks(buildResult);
-    return buildResult.metafile;
+    esbuildResultChecks(result);
+}
+function commonBuildOptions(b, r, c) {
+    const p = workersPlugin(r.buildRegistry());
+    return {
+        absWorkingDir: b.dirs.projectRootAbs,
+        assetNames: 'assets/[name]-[hash]',
+        bundle: true,
+        format: 'esm',
+        loader: c?.loaders || defaultLoaders(),
+        metafile: true,
+        minify: b.minify,
+        platform: 'browser',
+        plugins: c?.plugins?.length ? [p, ...c.plugins] : [p],
+        splitting: true,
+        treeShaking: true,
+        write: true,
+    };
+}
+function defaultLoaders() {
+    return {
+        '.woff': 'file',
+        '.woff2': 'file',
+    };
 }
 // esbuild will append the .js or .css to output filenames
 // keeping extension on entryPoints data for consistency
-// and removing and mapping entryPoints to pass to esbuild
-function removeEntryPointOutExt(entryPoints) {
+// and only trimming when creating esbuild opts
+function mapEntryPointPaths(entryPoints) {
     return entryPoints.map(entryPoint => {
         return {
             in: entryPoint.in,
@@ -72,3 +89,91 @@ function esbuildPrintMessage(msg, category) {
             console.error('   ', note.location);
     });
 }
+const WORKER_CTOR_REGEX = /new(?:\s|\r?\n)+Worker(?:\s|\r?\n)*\((?:\s|\r?\n)*(?<url>.*)(?:\s|\r?\n)*\)/g;
+const WORKER_URL_REGEX = /^('.*'|".*")$/;
+export function workersPlugin(r) {
+    return {
+        name: '@eighty4/dank/esbuild/workers',
+        setup(build) {
+            if (!build.initialOptions.absWorkingDir)
+                throw TypeError('plugin requires absWorkingDir');
+            if (!build.initialOptions.metafile)
+                throw TypeError('plugin requires metafile');
+            const { absWorkingDir } = build.initialOptions;
+            build.onLoad({ filter: /\.(t|m?j)s$/ }, async (args) => {
+                let contents = await readFile(args.path, 'utf8');
+                let offset = 0;
+                let errors = undefined;
+                for (const workerCtorMatch of contents.matchAll(WORKER_CTOR_REGEX)) {
+                    const workerUrlString = workerCtorMatch.groups.url;
+                    if (WORKER_URL_REGEX.test(workerUrlString)) {
+                        const preamble = contents.substring(0, workerCtorMatch.index);
+                        const lineIndex = preamble.lastIndexOf('\n') || 0;
+                        const lineCommented = /\/\//.test(preamble.substring(lineIndex));
+                        if (lineCommented)
+                            continue;
+                        const blockCommentIndex = preamble.lastIndexOf('/*');
+                        const blockCommented = blockCommentIndex !== -1 &&
+                            preamble
+                                .substring(blockCommentIndex)
+                                .indexOf('*/') === -1;
+                        if (blockCommented)
+                            continue;
+                        const clientScript = args.path
+                            .replace(absWorkingDir, '')
+                            .substring(1);
+                        const workerUrl = workerUrlString.substring(1, workerUrlString.length - 1);
+                        // todo out of bounds error on path resolve
+                        const workerEntryPoint = r.resolve(clientScript, workerUrl);
+                        const workerUrlPlaceholder = workerEntryPoint
+                            .replace(/^pages/, '')
+                            .replace(/\.(t|m?j)s$/, '.js');
+                        const workerCtorReplacement = `new Worker('${workerUrlPlaceholder}')`;
+                        contents =
+                            contents.substring(0, workerCtorMatch.index + offset) +
+                                workerCtorReplacement +
+                                contents.substring(workerCtorMatch.index +
+                                    workerCtorMatch[0].length +
+                                    offset);
+                        offset +=
+                            workerCtorReplacement.length -
+                                workerCtorMatch[0].length;
+                        r.addWorker({
+                            clientScript,
+                            workerEntryPoint,
+                            workerUrl,
+                            workerUrlPlaceholder,
+                        });
+                    }
+                    else {
+                        if (!errors)
+                            errors = [];
+                        const preamble = contents.substring(0, workerCtorMatch.index);
+                        const line = preamble.match(/\n/g)?.length || 0;
+                        const lineIndex = preamble.lastIndexOf('\n') || 0;
+                        const column = preamble.length - lineIndex;
+                        const lineText = contents.substring(lineIndex, contents.indexOf('\n', lineIndex) ||
+                            contents.length);
+                        errors.push({
+                            id: 'worker-url-unresolvable',
+                            location: {
+                                lineText,
+                                line,
+                                column,
+                                file: args.path,
+                                length: workerCtorMatch[0].length,
+                            },
+                        });
+                    }
+                }
+                const loader = args.path.endsWith('ts') ? 'ts' : 'js';
+                return { contents, errors, loader };
+            });
+            build.onEnd((result) => {
+                if (result.metafile) {
+                    r.completeBuild(result);
+                }
+            });
+        },
+    };
+}

package/lib_js/flags.js

@@ -1,11 +1,123 @@
+import { join, resolve } from 'node:path';
+import { cwd } from 'node:process';
+export function resolveBuildFlags() {
+    const flags = {
+        dirs: defaultProjectDirs(cwd()),
+        minify: willMinify(),
+        production: isProductionBuild(),
+    };
+    return {
+        get dirs() {
+            return flags.dirs;
+        },
+        get minify() {
+            return flags.minify;
+        },
+        get production() {
+            return flags.production;
+        },
+    };
+}
+export function resolveServeFlags(c) {
+    const preview = isPreviewBuild();
+    const flags = {
+        dirs: defaultProjectDirs(cwd()),
+        dankPort: dankPort(c, preview),
+        esbuildPort: esbuildPort(c),
+        logHttp: willLogHttp(),
+        minify: willMinify(),
+        preview,
+        production: isProductionBuild(),
+    };
+    return {
+        get dirs() {
+            return flags.dirs;
+        },
+        get dankPort() {
+            return flags.dankPort;
+        },
+        get esbuildPort() {
+            return flags.esbuildPort;
+        },
+        get logHttp() {
+            return flags.logHttp;
+        },
+        get minify() {
+            return flags.minify;
+        },
+        get preview() {
+            return flags.preview;
+        },
+        get production() {
+            return flags.production;
+        },
+    };
+}
 // `dank serve` will pre-bundle and use service worker
-export const isPreviewBuild = () => process.env.PREVIEW === 'true' || process.argv.includes('--preview');
+const isPreviewBuild = () => process.env.PREVIEW === 'true' || process.argv.includes('--preview');
 // `dank build` will minify sources and append git release tag to build tag
 // `dank serve` will pre-bundle with service worker and minify
-export const isProductionBuild = () => process.env.PRODUCTION === 'true' || process.argv.includes('--production');
-export const willMinify = () => isProductionBuild() ||
+const isProductionBuild = () => process.env.PRODUCTION === 'true' || process.argv.includes('--production');
+// `dank serve` dank port for frontend webserver
+// alternate --preview port for service worker builds
+function dankPort(c, preview) {
+    if (process.env.DANK_PORT?.length) {
+        return parsePortEnvVar('DANK_PORT');
+    }
+    return preview ? c.previewPort || c.port || 4000 : c.port || 3000;
+}
+// `dank serve` esbuild port for bundler integration
+function esbuildPort(c) {
+    if (process.env.ESBUILD_PORT?.length) {
+        return parsePortEnvVar('ESBUILD_PORT');
+    }
+    return c.esbuild?.port || 3995;
+}
+function parsePortEnvVar(name) {
+    const port = parseInt(process.env[name], 10);
+    if (isNaN(port)) {
+        throw Error(`env var ${name}=${port} must be a valid port number`);
+    }
+    else {
+        return port;
+    }
+}
+export function defaultProjectDirs(projectRootAbs) {
+    const dirs = {
+        buildRoot: 'build',
+        buildDist: join('build', 'dist'),
+        buildWatch: join('build', 'watch'),
+        pages: 'pages',
+        pagesResolved: resolve(join(projectRootAbs, 'pages')),
+        projectRootAbs,
+        public: 'public',
+    };
+    return {
+        get buildRoot() {
+            return dirs.buildRoot;
+        },
+        get buildDist() {
+            return dirs.buildDist;
+        },
+        get buildWatch() {
+            return dirs.buildWatch;
+        },
+        get pages() {
+            return dirs.pages;
+        },
+        get pagesResolved() {
+            return dirs.pagesResolved;
+        },
+        get projectRootAbs() {
+            return dirs.projectRootAbs;
+        },
+        get public() {
+            return dirs.public;
+        },
+    };
+}
+const willMinify = () => isProductionBuild() ||
     process.env.MINIFY === 'true' ||
     process.argv.includes('--minify');
-export const willTsc = () => isProductionBuild() ||
-    process.env.TSC === 'true' ||
-    process.argv.includes('--tsc');
+// `dank serve` will print http access logs to console
+const willLogHttp = () => process.env.LOG_HTTP === 'true' || process.argv.includes('--log-http');