@eighty4/dank 0.0.3 → 0.0.4-1

package/lib_js/metadata.js

@@ -0,0 +1,210 @@
+import EventEmitter from 'node:events';
+import { writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+class ResolverImpl {
+    #dirs;
+    constructor(dirs) {
+        this.#dirs = dirs;
+    }
+    isProjectSubpathInPagesDir(p) {
+        return resolve(join(this.#dirs.projectResolved, p)).startsWith(this.#dirs.pagesResolved);
+    }
+    isPagesSubpathInPagesDir(p) {
+        return this.isProjectSubpathInPagesDir(join(this.#dirs.pages, p));
+    }
+    resolveHrefInPagesDir(from, href) {
+        const p = join(dirname(from), href);
+        if (this.isProjectSubpathInPagesDir(p)) {
+            return p;
+        }
+        else {
+            return 'outofbounds';
+        }
+    }
+}
+// manages website resources during `dank build` and `dank serve`
+export class WebsiteRegistry extends EventEmitter {
+    #build;
+    // paths of bundled esbuild outputs
+    #bundles = new Set();
+    // public dir assets
+    #copiedAssets = null;
+    // map of entrypoints to their output path
+    #entrypointHrefs = {};
+    #pageUrls = [];
+    #resolver;
+    #workers = null;
+    constructor(build) {
+        super();
+        this.#build = build;
+        this.#resolver = new ResolverImpl(build.dirs);
+    }
+    set copiedAssets(copiedAssets) {
+        this.#copiedAssets =
+            copiedAssets === null ? null : new Set(copiedAssets);
+    }
+    set pageUrls(pageUrls) {
+        this.#pageUrls = pageUrls;
+    }
+    get resolver() {
+        return this.#resolver;
+    }
+    // bundleOutputs(type?: 'css' | 'js'): Array<string> {
+    //     if (!type) {
+    //         return Array.from(this.#bundles)
+    //     } else {
+    //         return Array.from(this.#bundles).filter(p => p.endsWith(type))
+    //     }
+    // }
+    buildRegistry() {
+        return new BuildRegistry(this.#build, this.#onBuildManifest);
+    }
+    files() {
+        const files = new Set();
+        for (const pageUrl of this.#pageUrls)
+            files.add(pageUrl === '/' ? '/index.html' : `${pageUrl}/index.html`);
+        for (const f of this.#bundles)
+            files.add(f);
+        if (this.#copiedAssets)
+            for (const f of this.#copiedAssets)
+                files.add(f);
+        return files;
+    }
+    mappedHref(lookup) {
+        const found = this.#entrypointHrefs[lookup];
+        if (found) {
+            return found;
+        }
+        else {
+            throw Error(`mapped href for ${lookup} not found`);
+        }
+    }
+    workerEntryPoints() {
+        return (this.#workers?.map(({ workerEntryPoint }) => ({
+            in: workerEntryPoint,
+            out: workerEntryPoint
+                .replace(/^pages[\//]/, '')
+                .replace(/\.(mj|t)s$/, '.js'),
+        })) || null);
+    }
+    workers() {
+        return this.#workers;
+    }
+    async writeManifest(buildTag) {
+        const manifest = this.#manifest(buildTag);
+        await writeFile(join(this.#build.dirs.projectRootAbs, this.#build.dirs.buildRoot, 'website.json'), JSON.stringify({
+            buildTag,
+            files: Array.from(manifest.files),
+            pageUrls: Array.from(manifest.pageUrls),
+        }, null, 4));
+        return manifest;
+    }
+    #manifest(buildTag) {
+        return {
+            buildTag,
+            files: this.files(),
+            pageUrls: new Set(this.#pageUrls),
+        };
+    }
+    #onBuildManifest = (build) => {
+        // collect built bundle entrypoint hrefs
+        for (const [outPath, entrypoint] of Object.entries(build.bundles)) {
+            this.#bundles.add(outPath);
+            if (entrypoint) {
+                this.#entrypointHrefs[entrypoint] = outPath;
+            }
+        }
+        // determine if worker entrypoints have changed
+        let updatedWorkerEntrypoints = false;
+        const previousWorkers = this.#workers === null
+            ? null
+            : new Set(this.#workers.map(w => w.workerEntryPoint));
+        if (build.workers) {
+            if (!previousWorkers ||
+                previousWorkers.size !==
+                    new Set(build.workers.map(w => w.workerEntryPoint)).size) {
+                updatedWorkerEntrypoints = true;
+            }
+            else {
+                updatedWorkerEntrypoints = !build.workers.every(w => previousWorkers.has(w.workerEntryPoint));
+            }
+        }
+        else if (previousWorkers) {
+            updatedWorkerEntrypoints = true;
+        }
+        // merge unique entrypoints from built workers with registry state
+        // todo filtering out unique occurrences of clientScript and workerUrl
+        //  drops reporting/summary/debugging capabilities, but currently
+        //  this.#workers is used for unique worker/client entrypoints
+        if (build.workers) {
+            if (!this.#workers) {
+                this.#workers = build.workers;
+            }
+            else {
+                for (const w of build.workers) {
+                    const found = this.#workers.find(w2 => {
+                        return (w.dependentEntryPoint === w2.dependentEntryPoint &&
+                            w.workerEntryPoint === w2.workerEntryPoint);
+                    });
+                    if (!found) {
+                        this.#workers.push(w);
+                    }
+                }
+            }
+        }
+        if (updatedWorkerEntrypoints) {
+            this.emit('workers');
+        }
+    };
+}
+// result accumulator of an esbuild `build` or `Context.rebuild`
+export class BuildRegistry {
+    #onComplete;
+    #resolver;
+    #workers = null;
+    constructor(build, onComplete) {
+        this.#onComplete = onComplete;
+        this.#resolver = new ResolverImpl(build.dirs);
+    }
+    get resolver() {
+        return this.#resolver;
+    }
+    // resolve web worker imported by a webpage module
+    addWorker(worker) {
+        // todo normalize path
+        if (!this.#workers) {
+            this.#workers = [worker];
+        }
+        else {
+            this.#workers.push(worker);
+        }
+    }
+    completeBuild(result) {
+        const bundles = {};
+        for (const [outPath, output] of Object.entries(result.metafile.outputs)) {
+            bundles[outPath.replace(/^build[/\\]dist/, '')] =
+                output.entryPoint || null;
+        }
+        let workers = null;
+        if (this.#workers) {
+            workers = [];
+            for (const output of Object.values(result.metafile.outputs)) {
+                if (!output.entryPoint)
+                    continue;
+                const inputs = Object.keys(output.inputs);
+                for (const worker of this.#workers) {
+                    if (inputs.includes(worker.clientScript)) {
+                        workers.push({
+                            ...worker,
+                            dependentEntryPoint: output.entryPoint,
+                        });
+                    }
+                }
+            }
+        }
+        this.#onComplete({
+            bundles,
+            workers,
+        });
+    }
+}

package/lib_js/public.js

@@ -1,11 +1,12 @@
 import { copyFile, mkdir, readdir, stat } from 'node:fs/promises';
+import { platform } from 'node:os';
 import { join } from 'node:path';
-export async function copyAssets(outRoot) {
+export async function copyAssets(build) {
     try {
-        const stats = await stat('public');
+        const stats = await stat(build.dirs.public);
         if (stats.isDirectory()) {
-            await mkdir(outRoot, { recursive: true });
-            return await recursiveCopyAssets(outRoot);
+            await mkdir(build.dirs.buildDist, { recursive: true });
+            return await recursiveCopyAssets(build);
         }
         else {
             throw Error('./public cannot be a file');
@@ -15,22 +16,29 @@ export async function copyAssets(outRoot) {
         return null;
     }
 }
-async function recursiveCopyAssets(outRoot, dir = '') {
+const IGNORE = platform() === 'darwin' ? ['.DS_Store'] : [];
+async function recursiveCopyAssets(build, dir = '') {
     const copied = [];
-    const to = join(outRoot, dir);
+    const to = join(build.dirs.buildDist, dir);
     let madeDir = dir === '';
-    for (const p of await readdir(join('public', dir))) {
+    const listingDir = join(build.dirs.public, dir);
+    for (const p of await readdir(listingDir)) {
+        if (IGNORE.includes(p)) {
+            continue;
+        }
         try {
-            const stats = await stat(join('public', dir, p));
+            const stats = await stat(join(listingDir, p));
             if (stats.isDirectory()) {
-                copied.push(...(await recursiveCopyAssets(outRoot, join(dir, p))));
+                copied.push(...(await recursiveCopyAssets(build, join(dir, p))));
             }
             else {
                 if (!madeDir) {
-                    await mkdir(join(outRoot, dir));
+                    await mkdir(join(build.dirs.buildDist, dir), {
+                        recursive: true,
+                    });
                     madeDir = true;
                 }
-                await copyFile(join('public', dir, p), join(to, p));
+                await copyFile(join(listingDir, p), join(to, p));
                 copied.push('/' + join(dir, p).replaceAll('\\', '/'));
             }
         }

package/lib_js/serve.js

@@ -1,43 +1,57 @@
-import { mkdir, readFile, rm, watch as _watch } from 'node:fs/promises';
+import { mkdir, readFile, rm, watch as _watch, writeFile, } from 'node:fs/promises';
 import { extname, join, resolve } from 'node:path';
 import { buildWebsite } from "./build.js";
 import { loadConfig } from "./config.js";
 import { createGlobalDefinitions } from "./define.js";
 import { esbuildDevContext } from "./esbuild.js";
-import { isPreviewBuild } from "./flags.js";
+import { resolveServeFlags } from "./flags.js";
 import { HtmlEntrypoint } from "./html.js";
-import { createBuiltDistFilesFetcher, createDevServeFilesFetcher, createWebServer, } from "./http.js";
+import { createBuiltDistFilesFetcher, createDevServeFilesFetcher, startWebServer, } from "./http.js";
+import { WebsiteRegistry } from "./metadata.js";
 import { startDevServices, updateDevServices } from "./services.js";
-const isPreview = isPreviewBuild();
-// alternate port for --preview bc of service worker
-const PORT = isPreview ? 4000 : 3000;
-// port for esbuild.serve
-const ESBUILD_PORT = 2999;
 export async function serveWebsite(c) {
-    await rm('build', { force: true, recursive: true });
-    if (isPreview) {
-        await startPreviewMode(c);
+    const serve = resolveServeFlags(c);
+    await rm(serve.dirs.buildRoot, { force: true, recursive: true });
+    const abortController = new AbortController();
+    process.once('exit', () => abortController.abort());
+    if (serve.preview) {
+        await startPreviewMode(c, serve, abortController.signal);
     }
     else {
-        const abortController = new AbortController();
-        await startDevMode(c, abortController.signal);
+        await startDevMode(c, serve, abortController.signal);
     }
     return new Promise(() => { });
 }
-async function startPreviewMode(c) {
-    const { dir, files } = await buildWebsite(c);
-    const frontend = createBuiltDistFilesFetcher(dir, files);
-    createWebServer(PORT, frontend).listen(PORT);
-    console.log(`preview is live at http://127.0.0.1:${PORT}`);
+async function startPreviewMode(c, serve, signal) {
+    const manifest = await buildWebsite(c, serve);
+    const frontend = createBuiltDistFilesFetcher(serve.dirs.buildDist, manifest);
+    const devServices = startDevServices(c, signal);
+    startWebServer(serve, frontend, devServices.http, {
+        urls: Object.keys(c.pages),
+        urlRewrites: collectUrlRewrites(c),
+    });
+}
+function collectUrlRewrites(c) {
+    return Object.keys(c.pages)
+        .sort()
+        .map(url => {
+        const mapping = c.pages[url];
+        return typeof mapping !== 'object' || !mapping.pattern
+            ? null
+            : { url, pattern: mapping.pattern };
+    })
+        .filter(mapping => mapping !== null);
 }
 // todo changing partials triggers update on html pages
-async function startDevMode(c, signal) {
-    const watchDir = join('build', 'watch');
-    await mkdir(watchDir, { recursive: true });
-    const clientJS = await loadClientJS();
+async function startDevMode(c, serve, signal) {
+    await mkdir(serve.dirs.buildWatch, { recursive: true });
+    const registry = new WebsiteRegistry(serve);
+    const clientJS = await loadClientJS(serve.esbuildPort);
     const pagesByUrlPath = {};
+    const partialsByUrlPath = {};
     const entryPointsByUrlPath = {};
     let buildContext = null;
+    registry.on('workers', resetBuildContext);
     watch('dank.config.ts', signal, async () => {
         let updated;
         try {
@@ -47,17 +61,18 @@ async function startDevMode(c, signal) {
             return;
         }
         const prevPages = new Set(Object.keys(pagesByUrlPath));
-        await Promise.all(Object.entries(updated.pages).map(async ([urlPath, srcPath]) => {
-            c.pages[urlPath] = srcPath;
-            if (pagesByUrlPath[urlPath]) {
+        await Promise.all(Object.entries(updated.pages).map(async ([urlPath, mapping]) => {
+            c.pages[urlPath] = mapping;
+            const srcPath = typeof mapping === 'string' ? mapping : mapping.webpage;
+            if (!pagesByUrlPath[urlPath]) {
+                await addPage(urlPath, srcPath);
+            }
+            else {
                 prevPages.delete(urlPath);
-                if (pagesByUrlPath[urlPath].srcPath !== srcPath) {
+                if (pagesByUrlPath[urlPath].fsPath !== srcPath) {
                     await updatePage(urlPath);
                 }
             }
-            else {
-                await addPage(urlPath, srcPath);
-            }
         }));
         for (const prevPage of Array.from(prevPages)) {
             delete c.pages[prevPage];
@@ -65,84 +80,101 @@ async function startDevMode(c, signal) {
         }
         updateDevServices(updated);
     });
-    watch('pages', signal, filename => {
+    watch(serve.dirs.pages, signal, filename => {
         if (extname(filename) === '.html') {
             for (const [urlPath, srcPath] of Object.entries(c.pages)) {
                 if (srcPath === filename) {
                     updatePage(urlPath);
                 }
             }
+            for (const [urlPath, partials] of Object.entries(partialsByUrlPath)) {
+                if (partials.includes(filename)) {
+                    updatePage(urlPath, filename);
+                }
+            }
         }
     });
-    await Promise.all(Object.entries(c.pages).map(([urlPath, srcPath]) => addPage(urlPath, srcPath)));
+    await Promise.all(Object.entries(c.pages).map(async ([urlPath, mapping]) => {
+        const srcPath = typeof mapping === 'string' ? mapping : mapping.webpage;
+        await addPage(urlPath, srcPath);
+        return new Promise(res => pagesByUrlPath[urlPath].once('entrypoints', res));
+    }));
     async function addPage(urlPath, srcPath) {
-        const metadata = await processWebpage({
-            clientJS,
-            outDir: watchDir,
-            pagesDir: 'pages',
-            srcPath,
-            urlPath,
+        await mkdir(join(serve.dirs.buildWatch, urlPath), { recursive: true });
+        const htmlEntrypoint = (pagesByUrlPath[urlPath] = new HtmlEntrypoint(serve, registry.resolver, urlPath, srcPath, [{ type: 'script', js: clientJS }]));
+        htmlEntrypoint.on('entrypoints', entrypoints => {
+            const pathsIn = new Set(entrypoints.map(e => e.in));
+            if (!entryPointsByUrlPath[urlPath] ||
+                !matchingEntrypoints(entryPointsByUrlPath[urlPath].pathsIn, pathsIn)) {
+                entryPointsByUrlPath[urlPath] = { entrypoints, pathsIn };
+                resetBuildContext();
+            }
         });
-        pagesByUrlPath[urlPath] = metadata;
-        entryPointsByUrlPath[urlPath] = new Set(metadata.entryPoints.map(e => e.in));
-        if (buildContext !== null) {
-            resetBuildContext();
-        }
+        htmlEntrypoint.on('partial', partial => {
+            if (!partialsByUrlPath[urlPath]) {
+                partialsByUrlPath[urlPath] = [];
+            }
+            partialsByUrlPath[urlPath].push(partial);
+        });
+        htmlEntrypoint.on('partials', partials => (partialsByUrlPath[urlPath] = partials));
+        htmlEntrypoint.on('output', html => writeFile(join(serve.dirs.buildWatch, urlPath, 'index.html'), html));
     }
     function deletePage(urlPath) {
+        pagesByUrlPath[urlPath].removeAllListeners();
         delete pagesByUrlPath[urlPath];
         delete entryPointsByUrlPath[urlPath];
         resetBuildContext();
     }
-    async function updatePage(urlPath) {
-        const update = await processWebpage({
-            clientJS,
-            outDir: watchDir,
-            pagesDir: 'pages',
-            srcPath: c.pages[urlPath],
-            urlPath,
-        });
-        const entryPointUrls = new Set(update.entryPoints.map(e => e.in));
-        if (!hasSameValues(entryPointUrls, entryPointsByUrlPath[urlPath])) {
-            entryPointsByUrlPath[urlPath] = entryPointUrls;
-            resetBuildContext();
-        }
+    async function updatePage(urlPath, partial) {
+        pagesByUrlPath[urlPath].emit('change', partial);
     }
     function collectEntrypoints() {
-        const sources = new Set();
-        return Object.values(pagesByUrlPath)
-            .flatMap(({ entryPoints }) => entryPoints)
+        const unique = new Set();
+        const pageBundles = Object.values(entryPointsByUrlPath)
+            .flatMap(entrypointState => entrypointState.entrypoints)
             .filter(entryPoint => {
-            if (sources.has(entryPoint.in)) {
+            if (unique.has(entryPoint.in)) {
                 return false;
             }
             else {
-                sources.add(entryPoint.in);
+                unique.add(entryPoint.in);
                 return true;
             }
         });
+        const workerBundles = registry.workerEntryPoints();
+        if (workerBundles) {
+            return [...pageBundles, ...workerBundles];
+        }
+        else {
+            return pageBundles;
+        }
     }
     function resetBuildContext() {
-        if (buildContext === 'starting' || buildContext === 'dirty') {
-            buildContext = 'dirty';
-            return;
-        }
-        if (buildContext === 'disposing') {
-            return;
+        switch (buildContext) {
+            case 'starting':
+                buildContext = 'dirty';
+                return;
+            case 'dirty':
+            case 'disposing':
+                return;
         }
         if (buildContext !== null) {
-            const prev = buildContext;
+            const disposing = buildContext.dispose();
             buildContext = 'disposing';
-            prev.dispose().then(() => {
+            disposing.then(() => {
                 buildContext = null;
                 resetBuildContext();
             });
         }
         else {
-            startEsbuildWatch(collectEntrypoints()).then(ctx => {
+            buildContext = 'starting';
+            startEsbuildWatch(c, registry, serve, collectEntrypoints()).then(ctx => {
                 if (buildContext === 'dirty') {
-                    buildContext = null;
-                    resetBuildContext();
+                    buildContext = 'disposing';
+                    ctx.dispose().then(() => {
+                        buildContext = null;
+                        resetBuildContext();
+                    });
                 }
                 else {
                     buildContext = ctx;
@@ -150,18 +182,32 @@ async function startDevMode(c, signal) {
             });
         }
     }
-    buildContext = await startEsbuildWatch(collectEntrypoints());
-    const frontend = createDevServeFilesFetcher({
-        pages: c.pages,
-        pagesDir: watchDir,
-        proxyPort: ESBUILD_PORT,
-        publicDir: 'public',
-    });
-    createWebServer(PORT, frontend).listen(PORT);
-    console.log(`dev server is live at http://127.0.0.1:${PORT}`);
-    startDevServices(c, signal);
+    // function removePartialFromPage(partial: string, urlPath: string) {
+    //     const deleteIndex = urlPathsByPartials[partial].indexOf(urlPath)
+    //     if (deleteIndex !== -1) {
+    //         if (urlPathsByPartials[partial].length === 1) {
+    //             delete urlPathsByPartials[partial]
+    //         } else {
+    //             urlPathsByPartials[partial].splice(deleteIndex, 1)
+    //         }
+    //     }
+    // }
+    // inital start of esbuild ctx
+    resetBuildContext();
+    // todo this page route state could be built on change and reused
+    const pageRoutes = {
+        get urls() {
+            return Object.keys(c.pages);
+        },
+        get urlRewrites() {
+            return collectUrlRewrites(c);
+        },
+    };
+    const frontend = createDevServeFilesFetcher(pageRoutes, serve);
+    const devServices = startDevServices(c, signal);
+    startWebServer(serve, frontend, devServices.http, pageRoutes);
 }
-function hasSameValues(a, b) {
+function matchingEntrypoints(a, b) {
     if (a.size !== b.size) {
         return false;
     }
@@ -172,42 +218,21 @@ function hasSameValues(a, b) {
     }
     return true;
 }
-async function processWebpage(inputs) {
-    const html = await HtmlEntrypoint.readFrom(inputs.urlPath, join(inputs.pagesDir, inputs.srcPath));
-    await html.injectPartials();
-    if (inputs.urlPath !== '/') {
-        await mkdir(join(inputs.outDir, inputs.urlPath), { recursive: true });
-    }
-    const entryPoints = [];
-    html.collectScripts().forEach(scriptImport => {
-        entryPoints.push({
-            in: scriptImport.in,
-            out: scriptImport.out,
-        });
-    });
-    html.rewriteHrefs();
-    html.appendScript(inputs.clientJS);
-    await html.writeTo(inputs.outDir);
-    return {
-        entryPoints,
-        srcPath: inputs.srcPath,
-        urlPath: inputs.urlPath,
-    };
-}
-async function startEsbuildWatch(entryPoints) {
-    const ctx = await esbuildDevContext(createGlobalDefinitions(), entryPoints, 'build/watch');
+async function startEsbuildWatch(c, registry, serve, entryPoints) {
+    const ctx = await esbuildDevContext(serve, registry, createGlobalDefinitions(serve), entryPoints, c.esbuild);
     await ctx.watch();
     await ctx.serve({
         host: '127.0.0.1',
-        port: ESBUILD_PORT,
+        port: serve.esbuildPort,
         cors: {
-            origin: 'http://127.0.0.1:' + PORT,
+            origin: ['127.0.0.1', 'localhost'].map(hostname => `http://${hostname}:${serve.dankPort}`),
         },
     });
     return ctx;
 }
-async function loadClientJS() {
-    return await readFile(resolve(import.meta.dirname, join('..', 'client', 'esbuild.js')), 'utf-8');
+async function loadClientJS(esbuildPort) {
+    const clientJS = await readFile(resolve(import.meta.dirname, join('..', 'client', 'esbuild.js')), 'utf-8');
+    return clientJS.replace('3995', `${esbuildPort}`);
 }
 async function watch(p, signal, fire) {
     const delayFire = 90;

package/lib_js/services.js

@@ -12,6 +12,13 @@ export function startDevServices(c, _signal) {
             running.push({ s, process: startService(s) });
         }
     }
+    return {
+        http: {
+            get running() {
+                return running.map(({ s }) => s.http).filter(http => !!http);
+            },
+        },
+    };
 }
 export function updateDevServices(c) {
     if (!c.services?.length) {
@@ -130,8 +137,12 @@ function startService(s) {
     const stderrLabel = logLabel(s.cwd, cmd, args, 31);
     spawned.stderr.on('data', chunk => printChunk(stderrLabel, chunk));
     spawned.on('error', e => {
-        const cause = 'code' in e && e.code === 'ENOENT' ? 'program not found' : e.message;
-        opPrint(s, 'error: ' + cause);
+        if (e.name !== 'AbortError') {
+            const cause = 'code' in e && e.code === 'ENOENT'
+                ? 'program not found'
+                : e.message;
+            opPrint(s, 'error: ' + cause);
+        }
         removeFromRunning(s);
     });
     spawned.on('exit', () => {

package/lib_types/dank.d.ts

@@ -1,10 +1,27 @@
+import type { Plugin as EsbuildPlugin } from 'esbuild';
 export type DankConfig = {
-    pages: Record<`/${string}`, `${string}.html`>;
+    esbuild?: EsbuildConfig;
+    pages: Record<`/${string}`, `${string}.html` | PageMapping>;
+    port?: number;
+    previewPort?: number;
     services?: Array<DevService>;
 };
+export type PageMapping = {
+    pattern?: RegExp;
+    webpage: `${string}.html`;
+};
 export type DevService = {
     command: string;
     cwd?: string;
     env?: Record<string, string>;
+    http?: {
+        port: number;
+    };
+};
+export type EsbuildConfig = {
+    loaders?: Record<`.${string}`, EsbuildLoader>;
+    plugins?: Array<EsbuildPlugin>;
+    port?: number;
 };
+export type EsbuildLoader = 'base64' | 'binary' | 'copy' | 'dataurl' | 'empty' | 'file' | 'json' | 'text';
 export declare function defineConfig(c: Partial<DankConfig>): Promise<DankConfig>;