@eighty4/dank 0.0.1-4 → 0.0.1

package/lib_js/http.js

@@ -1,6 +1,7 @@
 import { createReadStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
 import { createServer, } from 'node:http';
-import { extname, join as fsJoin } from 'node:path';
+import { extname, join } from 'node:path';
 import { isProductionBuild } from "./flags.js";
 export function createWebServer(port, frontendFetcher) {
     const serverAddress = 'http://localhost:' + port;
@@ -27,32 +28,98 @@ export function createBuiltDistFilesFetcher(dir, files) {
             res.end();
         }
         else {
-            const mimeType = resolveMimeType(url);
-            res.setHeader('Content-Type', mimeType);
-            const reading = createReadStream(mimeType === 'text/html'
-                ? fsJoin(dir, url.pathname, 'index.html')
-                : fsJoin(dir, url.pathname));
-            reading.pipe(res);
-            reading.on('error', err => {
-                console.error(`${url.pathname} file read ${reading.path} error ${err.message}`);
-                res.statusCode = 500;
-                res.end();
-            });
+            const p = extname(url.pathname) === ''
+                ? join(dir, url.pathname, 'index.html')
+                : join(dir, url.pathname);
+            streamFile(p, res);
         }
     };
 }
-export function createLocalProxyFilesFetcher(port) {
-    const proxyAddress = 'http://127.0.0.1:' + port;
+export function createDevServeFilesFetcher(opts) {
+    const proxyAddress = 'http://127.0.0.1:' + opts.proxyPort;
     return (url, _headers, res) => {
-        fetch(proxyAddress + url.pathname).then(fetchResponse => {
-            res.writeHead(fetchResponse.status, convertHeadersFromFetch(fetchResponse.headers));
-            fetchResponse.bytes().then(data => res.end(data));
-        });
+        if (opts.pages[url.pathname]) {
+            streamFile(join(opts.pagesDir, url.pathname + 'index.html'), res);
+        }
+        else {
+            const maybePublicPath = join(opts.publicDir, url.pathname);
+            exists(join(opts.publicDir, url.pathname)).then(fromPublic => {
+                if (fromPublic) {
+                    streamFile(maybePublicPath, res);
+                }
+                else {
+                    retryFetchWithTimeout(proxyAddress + url.pathname)
+                        .then(fetchResponse => {
+                        res.writeHead(fetchResponse.status, convertHeadersFromFetch(fetchResponse.headers));
+                        fetchResponse.bytes().then(data => res.end(data));
+                    })
+                        .catch(e => {
+                        if (e === 'retrytimeout') {
+                            res.writeHead(504);
+                        }
+                        else {
+                            console.error('unknown frontend proxy fetch error:', e);
+                            res.writeHead(502);
+                        }
+                        res.end();
+                    });
+                }
+            });
+        }
     };
 }
-function resolveMimeType(url) {
-    switch (extname(url.pathname)) {
-        case '':
+const PROXY_FETCH_RETRY_INTERVAL = 27;
+const PROXY_FETCH_RETRY_TIMEOUT = 1000;
+async function retryFetchWithTimeout(url) {
+    let timeout = Date.now() + PROXY_FETCH_RETRY_TIMEOUT;
+    while (true) {
+        try {
+            return await fetch(url);
+        }
+        catch (e) {
+            if (isNodeFailedFetch(e) || isBunFailedFetch(e)) {
+                if (timeout < Date.now()) {
+                    throw 'retrytimeout';
+                }
+                else {
+                    await new Promise(res => setTimeout(res, PROXY_FETCH_RETRY_INTERVAL));
+                }
+            }
+            else {
+                throw e;
+            }
+        }
+    }
+}
+function isBunFailedFetch(e) {
+    return e.code === 'ConnectionRefused';
+}
+function isNodeFailedFetch(e) {
+    return e.message === 'fetch failed';
+}
+async function exists(p) {
+    try {
+        const maybe = stat(p);
+        return (await maybe).isFile();
+    }
+    catch (ignore) {
+        return false;
+    }
+}
+function streamFile(p, res) {
+    const mimeType = resolveMimeType(p);
+    res.setHeader('Content-Type', mimeType);
+    const reading = createReadStream(p);
+    reading.pipe(res);
+    reading.on('error', err => {
+        console.error(`file read ${reading.path} error ${err.message}`);
+        res.statusCode = 500;
+        res.end();
+    });
+}
+function resolveMimeType(p) {
+    switch (extname(p)) {
+        case '.html':
             return 'text/html';
         case '.js':
             return 'text/javascript';
@@ -71,7 +138,7 @@ function resolveMimeType(url) {
         case '.woff2':
             return 'font/woff2';
         default:
-            console.warn('? mime type for', url.pathname);
+            console.warn('? mime type for', p);
             if (!isProductionBuild())
                 process.exit(1);
             return 'application/octet-stream';

package/lib_js/serve.js

@@ -1,13 +1,13 @@
-import { mkdir, readFile, rm } from 'node:fs/promises';
-import { join, resolve } from 'node:path';
+import { mkdir, readFile, rm, watch as _watch } from 'node:fs/promises';
+import { extname, join, resolve } from 'node:path';
 import { buildWebsite } from "./build.js";
+import { loadConfig } from "./config.js";
 import { createGlobalDefinitions } from "./define.js";
 import { esbuildDevContext } from "./esbuild.js";
 import { isPreviewBuild } from "./flags.js";
 import { HtmlEntrypoint } from "./html.js";
-import { createBuiltDistFilesFetcher, createLocalProxyFilesFetcher, createWebServer, } from "./http.js";
-import { copyAssets } from "./public.js";
-import { startDevServices } from "./services.js";
+import { createBuiltDistFilesFetcher, createDevServeFilesFetcher, createWebServer, } from "./http.js";
+import { startDevServices, updateDevServices } from "./services.js";
 const isPreview = isPreviewBuild();
 // alternate port for --preview bc of service worker
 const PORT = isPreview ? 4000 : 3000;
@@ -15,56 +15,229 @@ const PORT = isPreview ? 4000 : 3000;
 const ESBUILD_PORT = 2999;
 export async function serveWebsite(c) {
     await rm('build', { force: true, recursive: true });
-    let frontend;
     if (isPreview) {
-        const { dir, files } = await buildWebsite(c);
-        frontend = createBuiltDistFilesFetcher(dir, files);
+        await startPreviewMode(c);
     }
     else {
-        const { port } = await startEsbuildWatch(c);
-        frontend = createLocalProxyFilesFetcher(port);
+        const abortController = new AbortController();
+        await startDevMode(c, abortController.signal);
     }
-    createWebServer(PORT, frontend).listen(PORT);
-    console.log(isPreview ? 'preview' : 'dev server', `is live at http://127.0.0.1:${PORT}`);
-    startDevServices(c);
     return new Promise(() => { });
 }
-async function startEsbuildWatch(c) {
+async function startPreviewMode(c) {
+    const { dir, files } = await buildWebsite(c);
+    const frontend = createBuiltDistFilesFetcher(dir, files);
+    createWebServer(PORT, frontend).listen(PORT);
+    console.log(`preview is live at http://127.0.0.1:${PORT}`);
+}
+// todo changing partials triggers update on html pages
+async function startDevMode(c, signal) {
     const watchDir = join('build', 'watch');
     await mkdir(watchDir, { recursive: true });
-    await copyAssets(watchDir);
-    const clientJS = await readFile(resolve(import.meta.dirname, join('..', 'client', 'esbuild.js')), 'utf-8');
-    const entryPointUrls = new Set();
-    const entryPoints = [];
-    await Promise.all(Object.entries(c.pages).map(async ([url, srcPath]) => {
-        const html = await HtmlEntrypoint.readFrom(url, join('pages', srcPath));
-        await html.injectPartials();
-        if (url !== '/') {
-            await mkdir(join(watchDir, url), { recursive: true });
+    const clientJS = await loadClientJS();
+    const pagesByUrlPath = {};
+    const entryPointsByUrlPath = {};
+    let buildContext = null;
+    watch('dank.config.ts', signal, async () => {
+        let updated;
+        try {
+            updated = await loadConfig();
+        }
+        catch (ignore) {
+            return;
+        }
+        const prevPages = new Set(Object.keys(pagesByUrlPath));
+        await Promise.all(Object.entries(updated.pages).map(async ([urlPath, srcPath]) => {
+            c.pages[urlPath] = srcPath;
+            if (pagesByUrlPath[urlPath]) {
+                prevPages.delete(urlPath);
+                if (pagesByUrlPath[urlPath].srcPath !== srcPath) {
+                    await updatePage(urlPath);
+                }
+            }
+            else {
+                await addPage(urlPath, srcPath);
+            }
+        }));
+        for (const prevPage of Array.from(prevPages)) {
+            delete c.pages[prevPage];
+            deletePage(prevPage);
+        }
+        updateDevServices(updated);
+    });
+    watch('pages', signal, filename => {
+        if (extname(filename) === '.html') {
+            for (const [urlPath, srcPath] of Object.entries(c.pages)) {
+                if (srcPath === filename) {
+                    updatePage(urlPath);
+                }
+            }
+        }
+    });
+    await Promise.all(Object.entries(c.pages).map(([urlPath, srcPath]) => addPage(urlPath, srcPath)));
+    async function addPage(urlPath, srcPath) {
+        const metadata = await processWebpage({
+            clientJS,
+            outDir: watchDir,
+            pagesDir: 'pages',
+            srcPath,
+            urlPath,
+        });
+        pagesByUrlPath[urlPath] = metadata;
+        entryPointsByUrlPath[urlPath] = new Set(metadata.entryPoints.map(e => e.in));
+        if (buildContext !== null) {
+            resetBuildContext();
         }
-        html.collectScripts()
-            .filter(scriptImport => !entryPointUrls.has(scriptImport.in))
-            .forEach(scriptImport => {
-            entryPointUrls.add(scriptImport.in);
-            entryPoints.push({
-                in: scriptImport.in,
-                out: scriptImport.out,
+    }
+    function deletePage(urlPath) {
+        delete pagesByUrlPath[urlPath];
+        delete entryPointsByUrlPath[urlPath];
+        resetBuildContext();
+    }
+    async function updatePage(urlPath) {
+        const update = await processWebpage({
+            clientJS,
+            outDir: watchDir,
+            pagesDir: 'pages',
+            srcPath: c.pages[urlPath],
+            urlPath,
+        });
+        const entryPointUrls = new Set(update.entryPoints.map(e => e.in));
+        if (!hasSameValues(entryPointUrls, entryPointsByUrlPath[urlPath])) {
+            entryPointsByUrlPath[urlPath] = entryPointUrls;
+            resetBuildContext();
+        }
+    }
+    function collectEntrypoints() {
+        const sources = new Set();
+        return Object.values(pagesByUrlPath)
+            .flatMap(({ entryPoints }) => entryPoints)
+            .filter(entryPoint => {
+            if (sources.has(entryPoint.in)) {
+                return false;
+            }
+            else {
+                sources.add(entryPoint.in);
+                return true;
+            }
+        });
+    }
+    function resetBuildContext() {
+        if (buildContext === 'starting' || buildContext === 'dirty') {
+            buildContext = 'dirty';
+            return;
+        }
+        if (buildContext === 'disposing') {
+            return;
+        }
+        if (buildContext !== null) {
+            const prev = buildContext;
+            buildContext = 'disposing';
+            prev.dispose().then(() => {
+                buildContext = null;
+                resetBuildContext();
             });
+        }
+        else {
+            startEsbuildWatch(collectEntrypoints()).then(ctx => {
+                if (buildContext === 'dirty') {
+                    buildContext = null;
+                    resetBuildContext();
+                }
+                else {
+                    buildContext = ctx;
+                }
+            });
+        }
+    }
+    buildContext = await startEsbuildWatch(collectEntrypoints());
+    const frontend = createDevServeFilesFetcher({
+        pages: c.pages,
+        pagesDir: watchDir,
+        proxyPort: ESBUILD_PORT,
+        publicDir: 'public',
+    });
+    createWebServer(PORT, frontend).listen(PORT);
+    console.log(`dev server is live at http://127.0.0.1:${PORT}`);
+    startDevServices(c, signal);
+}
+function hasSameValues(a, b) {
+    if (a.size !== b.size) {
+        return false;
+    }
+    for (const v in a) {
+        if (!b.has(v)) {
+            return false;
+        }
+    }
+    return true;
+}
+async function processWebpage(inputs) {
+    const html = await HtmlEntrypoint.readFrom(inputs.urlPath, join(inputs.pagesDir, inputs.srcPath));
+    await html.injectPartials();
+    if (inputs.urlPath !== '/') {
+        await mkdir(join(inputs.outDir, inputs.urlPath), { recursive: true });
+    }
+    const entryPoints = [];
+    html.collectScripts().forEach(scriptImport => {
+        entryPoints.push({
+            in: scriptImport.in,
+            out: scriptImport.out,
         });
-        html.rewriteHrefs();
-        html.appendScript(clientJS);
-        await html.writeTo(watchDir);
-        return html;
-    }));
-    const ctx = await esbuildDevContext(createGlobalDefinitions(), entryPoints, watchDir);
+    });
+    html.rewriteHrefs();
+    html.appendScript(inputs.clientJS);
+    await html.writeTo(inputs.outDir);
+    return {
+        entryPoints,
+        srcPath: inputs.srcPath,
+        urlPath: inputs.urlPath,
+    };
+}
+async function startEsbuildWatch(entryPoints) {
+    const ctx = await esbuildDevContext(createGlobalDefinitions(), entryPoints, 'build/watch');
     await ctx.watch();
     await ctx.serve({
         host: '127.0.0.1',
         port: ESBUILD_PORT,
-        servedir: watchDir,
         cors: {
             origin: 'http://127.0.0.1:' + PORT,
         },
     });
-    return { port: ESBUILD_PORT };
+    return ctx;
+}
+async function loadClientJS() {
+    return await readFile(resolve(import.meta.dirname, join('..', 'client', 'esbuild.js')), 'utf-8');
+}
+async function watch(p, signal, fire) {
+    const delayFire = 90;
+    const timeout = 100;
+    let changes = {};
+    try {
+        for await (const { filename } of _watch(p, {
+            recursive: true,
+            signal,
+        })) {
+            if (filename) {
+                if (!changes[filename]) {
+                    const now = Date.now();
+                    changes[filename] = now + delayFire;
+                    setTimeout(() => {
+                        const now = Date.now();
+                        for (const [filename, then] of Object.entries(changes)) {
+                            if (then <= now) {
+                                fire(filename);
+                                delete changes[filename];
+                            }
+                        }
+                    }, timeout);
+                }
+            }
+        }
+    }
+    catch (e) {
+        if (e.name !== 'AbortError') {
+            throw e;
+        }
+    }
 }

package/lib_js/services.js

@@ -1,20 +1,120 @@
 import { spawn } from 'node:child_process';
 import { basename, isAbsolute, resolve } from 'node:path';
-export function startDevServices(c) {
+// up to date representation of dank.config.ts services
+const running = [];
+let signal;
+// batch of services that must be stopped before starting new services
+let updating = null;
+export function startDevServices(c, _signal) {
+    signal = _signal;
     if (c.services?.length) {
-        const ac = new AbortController();
-        try {
-            for (const s of c.services) {
-                startService(s, ac.signal);
+        for (const s of c.services) {
+            running.push({ s, process: startService(s) });
+        }
+    }
+}
+export function updateDevServices(c) {
+    if (!c.services?.length) {
+        if (running.length) {
+            if (updating === null) {
+                updating = { stopping: [], starting: [] };
+            }
+            running.forEach(({ s, process }) => {
+                if (process) {
+                    stopService(s, process);
+                }
+                else {
+                    removeFromUpdating(s);
+                }
+            });
+            running.length = 0;
+        }
+    }
+    else {
+        if (updating === null) {
+            updating = { stopping: [], starting: [] };
+        }
+        const keep = [];
+        const next = [];
+        for (const s of c.services) {
+            let found = false;
+            for (let i = 0; i < running.length; i++) {
+                const p = running[i].s;
+                if (matchingConfig(s, p)) {
+                    found = true;
+                    keep.push(i);
+                    break;
+                }
+            }
+            if (!found) {
+                next.push(s);
             }
         }
-        catch (e) {
-            ac.abort();
-            throw e;
+        for (let i = running.length - 1; i >= 0; i--) {
+            if (!keep.includes(i)) {
+                const { s, process } = running[i];
+                if (process) {
+                    stopService(s, process);
+                }
+                else {
+                    removeFromUpdating(s);
+                }
+                running.splice(i, 1);
+            }
+        }
+        if (updating.stopping.length) {
+            for (const s of next) {
+                if (!updating.starting.find(queued => matchingConfig(queued, s))) {
+                    updating.starting.push(s);
+                }
+            }
+        }
+        else {
+            updating = null;
+            for (const s of next) {
+                running.push({ s, process: startService(s) });
+            }
         }
     }
 }
-function startService(s, signal) {
+function stopService(s, process) {
+    opPrint(s, 'stopping');
+    updating.stopping.push(s);
+    process.kill();
+}
+function matchingConfig(a, b) {
+    if (a.command !== b.command) {
+        return false;
+    }
+    if (a.cwd !== b.cwd) {
+        return false;
+    }
+    if (!a.env && !b.env) {
+        return true;
+    }
+    else if (a.env && !b.env) {
+        return false;
+    }
+    else if (!a.env && b.env) {
+        return false;
+    }
+    else if (Object.keys(a.env).length !== Object.keys(b.env).length) {
+        return false;
+    }
+    else {
+        for (const k of Object.keys(a.env)) {
+            if (!b.env[k]) {
+                return false;
+            }
+            else if (a.env[k] !== b.env[k]) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+function startService(s) {
+    opPrint(s, 'starting');
     const splitCmdAndArgs = s.command.split(/\s+/);
     const cmd = splitCmdAndArgs[0];
     const args = splitCmdAndArgs.length === 1 ? [] : splitCmdAndArgs.slice(1);
@@ -29,9 +129,39 @@ function startService(s, signal) {
     spawned.stdout.on('data', chunk => printChunk(stdoutLabel, chunk));
     const stderrLabel = logLabel(s.cwd, cmd, args, 31);
     spawned.stderr.on('data', chunk => printChunk(stderrLabel, chunk));
+    spawned.on('error', e => {
+        const cause = 'code' in e && e.code === 'ENOENT' ? 'program not found' : e.message;
+        opPrint(s, 'error: ' + cause);
+        removeFromRunning(s);
+    });
     spawned.on('exit', () => {
-        console.log(`[${s.command}]`, 'exit');
+        opPrint(s, 'exited');
+        removeFromRunning(s);
+        removeFromUpdating(s);
     });
+    return spawned;
+}
+function removeFromRunning(s) {
+    for (let i = 0; i < running.length; i++) {
+        if (matchingConfig(running[i].s, s)) {
+            running.splice(i, 1);
+            return;
+        }
+    }
+}
+function removeFromUpdating(s) {
+    if (updating !== null) {
+        for (let i = 0; i < updating.stopping.length; i++) {
+            if (matchingConfig(updating.stopping[i], s)) {
+                updating.stopping.splice(i, 1);
+                if (!updating.stopping.length) {
+                    updating.starting.forEach(startService);
+                    updating = null;
+                    return;
+                }
+            }
+        }
+    }
 }
 function printChunk(label, c) {
     for (const l of parseChunk(c))
@@ -51,6 +181,12 @@ function resolveCwd(p) {
         return resolve(process.cwd(), p);
     }
 }
+function opPrint(s, msg) {
+    console.log(opLabel(s), msg);
+}
+function opLabel(s) {
+    return `\`${s.cwd ? s.cwd + ' ' : ''}${s.command}\``;
+}
 function logLabel(cwd, cmd, args, ansiColor) {
     cwd = !cwd
         ? './'

package/lib_types/dank.d.ts

@@ -1,5 +1,5 @@
 export type DankConfig = {
-    pages: Record<`/${string}`, `./${string}.html`>;
+    pages: Record<`/${string}`, `${string}.html`>;
     services?: Array<DevService>;
 };
 export type DevService = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eighty4/dank",
-  "version": "0.0.1-4",
+  "version": "0.0.1",
   "type": "module",
   "description": "Multi-page development system for CDN-deployed websites",
   "keywords": [