@eighty4/dank 0.0.3 → 0.0.4-0

package/lib/http.ts

@@ -8,34 +8,162 @@ import {
     type ServerResponse,
 } from 'node:http'
 import { extname, join } from 'node:path'
+import { Readable } from 'node:stream'
 import mime from 'mime'
-import { isLogHttp } from './flags.ts'
+import type { DankServe } from './flags.ts'
+import type { WebsiteManifest } from './metadata.ts'
+import type { HttpServices } from './services.ts'
 
 export type FrontendFetcher = (
     url: URL,
     headers: Headers,
     res: ServerResponse,
+    notFound: () => void,
 ) => void
 
-export function createWebServer(
-    port: number,
+// state needed to eval url rewriting after FrontendFetcher and before HttpServices
+export type PageRouteState = {
+    // urls of html entrypoints
+    urls: Array<string>
+    urlRewrites: Array<UrlRewrite>
+}
+
+export type UrlRewrite = {
+    pattern: RegExp
+    url: string
+}
+
+export function startWebServer(
+    serve: DankServe,
     frontendFetcher: FrontendFetcher,
-): ReturnType<typeof createServer> {
-    const serverAddress = 'http://localhost:' + port
+    httpServices: HttpServices,
+    pageRoutes: PageRouteState,
+) {
+    const serverAddress = 'http://localhost:' + serve.dankPort
     const handler = (req: IncomingMessage, res: ServerResponse) => {
         if (!req.url || !req.method) {
             res.end()
         } else {
             const url = new URL(serverAddress + req.url)
-            if (req.method !== 'GET') {
-                res.writeHead(405)
-                res.end()
+            const headers = convertHeadersToFetch(req.headers)
+            frontendFetcher(url, headers, res, () =>
+                onNotFound(
+                    req,
+                    url,
+                    headers,
+                    httpServices,
+                    pageRoutes,
+                    serve,
+                    res,
+                ),
+            )
+        }
+    }
+    createServer(serve.logHttp ? createLogWrapper(handler) : handler).listen(
+        serve.dankPort,
+    )
+    console.log(
+        serve.preview ? 'preview' : 'dev',
+        `server is live at http://127.0.0.1:${serve.dankPort}`,
+    )
+}
+
+async function onNotFound(
+    req: IncomingMessage,
+    url: URL,
+    headers: Headers,
+    httpServices: HttpServices,
+    pageRoutes: PageRouteState,
+    serve: DankServe,
+    res: ServerResponse,
+) {
+    if (req.method === 'GET' && extname(url.pathname) === '') {
+        const urlRewrite = tryUrlRewrites(url, pageRoutes, serve)
+        if (urlRewrite) {
+            streamFile(urlRewrite, res)
+            return
+        }
+    }
+    const fetchResponse = await tryHttpServices(req, url, headers, httpServices)
+    if (fetchResponse) {
+        sendFetchResponse(res, fetchResponse)
+    } else {
+        res.writeHead(404)
+        res.end()
+    }
+}
+
+async function sendFetchResponse(res: ServerResponse, fetchResponse: Response) {
+    res.writeHead(
+        fetchResponse.status,
+        undefined,
+        convertHeadersFromFetch(fetchResponse.headers),
+    )
+    if (fetchResponse.body) {
+        Readable.fromWeb(fetchResponse.body).pipe(res)
+    } else {
+        res.end()
+    }
+}
+
+function tryUrlRewrites(
+    url: URL,
+    pageRoutes: PageRouteState,
+    serve: DankServe,
+): string | null {
+    const urlRewrite = pageRoutes.urlRewrites.find(urlRewrite =>
+        urlRewrite.pattern.test(url.pathname),
+    )
+    return urlRewrite
+        ? join(serve.dirs.buildWatch, urlRewrite.url, 'index.html')
+        : null
+}
+
+async function tryHttpServices(
+    req: IncomingMessage,
+    url: URL,
+    headers: Headers,
+    httpServices: HttpServices,
+): Promise<Response | null> {
+    if (url.pathname.startsWith('/.well-known/')) {
+        return null
+    }
+    const body = await collectReqBody(req)
+    const { running } = httpServices
+    for (const httpService of running) {
+        const proxyUrl = new URL(url)
+        proxyUrl.port = `${httpService.port}`
+        try {
+            const response = await retryFetchWithTimeout(proxyUrl, {
+                body,
+                headers,
+                method: req.method,
+                redirect: 'manual',
+            })
+            if (response.status === 404 || response.status === 405) {
+                continue
             } else {
-                frontendFetcher(url, convertHeadersToFetch(req.headers), res)
+                return response
+            }
+        } catch (e: any) {
+            if (e === 'retrytimeout') {
+                continue
+            } else {
+                errorExit(
+                    `unexpected error http proxying to port ${httpService.port}: ${e.message}`,
+                )
             }
         }
     }
-    return createServer(isLogHttp() ? createLogWrapper(handler) : handler)
+    return null
+}
+
+function collectReqBody(req: IncomingMessage): Promise<string | null> {
+    let body = ''
+    req.on('data', data => (body += data.toString()))
+    return new Promise(res =>
+        req.on('end', () => res(body.length ? body : null)),
+    )
 }
 
 type RequestListener = (req: IncomingMessage, res: ServerResponse) => void
@@ -51,57 +179,65 @@ function createLogWrapper(handler: RequestListener): RequestListener {
 
 export function createBuiltDistFilesFetcher(
     dir: string,
-    files: Set<string>,
+    manifest: WebsiteManifest,
 ): FrontendFetcher {
-    return (url: URL, _headers: Headers, res: ServerResponse) => {
-        if (!files.has(url.pathname)) {
-            res.writeHead(404)
-            res.end()
+    return (
+        url: URL,
+        _headers: Headers,
+        res: ServerResponse,
+        notFound: () => void,
+    ) => {
+        if (manifest.pageUrls.has(url.pathname)) {
+            streamFile(join(dir, url.pathname, 'index.html'), res)
+        } else if (manifest.files.has(url.pathname)) {
+            streamFile(join(dir, url.pathname), res)
         } else {
-            const p =
-                extname(url.pathname) === ''
-                    ? join(dir, url.pathname, 'index.html')
-                    : join(dir, url.pathname)
-            streamFile(p, res)
+            notFound()
         }
     }
 }
 
-type DevServeOpts = {
-    // ref of original DankConfig['pages'] mapping
-    // updated incrementally instead of replacing
-    pages: Record<string, string>
-    // dir processed html files are written to
-    pagesDir: string
-    // port to esbuild dev server
-    proxyPort: number
-    // dir of public assets
-    publicDir: string
-}
-
+// todo replace PageRouteState with WebsiteRegistry
 export function createDevServeFilesFetcher(
-    opts: DevServeOpts,
+    pageRoutes: PageRouteState,
+    serve: DankServe,
 ): FrontendFetcher {
-    const proxyAddress = 'http://127.0.0.1:' + opts.proxyPort
-    return (url: URL, _headers: Headers, res: ServerResponse) => {
-        if (opts.pages[url.pathname]) {
-            streamFile(join(opts.pagesDir, url.pathname, 'index.html'), res)
+    const proxyAddress = 'http://127.0.0.1:' + serve.esbuildPort
+    return (
+        url: URL,
+        _headers: Headers,
+        res: ServerResponse,
+        notFound: () => void,
+    ) => {
+        if (pageRoutes.urls.includes(url.pathname)) {
+            streamFile(
+                join(serve.dirs.buildWatch, url.pathname, 'index.html'),
+                res,
+            )
         } else {
-            const maybePublicPath = join(opts.publicDir, url.pathname)
-            exists(join(opts.publicDir, url.pathname)).then(fromPublic => {
+            const maybePublicPath = join(serve.dirs.public, url.pathname)
+            exists(maybePublicPath).then(fromPublic => {
                 if (fromPublic) {
                     streamFile(maybePublicPath, res)
                 } else {
                     retryFetchWithTimeout(proxyAddress + url.pathname)
                         .then(fetchResponse => {
-                            res.writeHead(
-                                fetchResponse.status,
-                                convertHeadersFromFetch(fetchResponse.headers),
-                            )
-                            fetchResponse.bytes().then(data => res.end(data))
+                            if (fetchResponse.status === 404) {
+                                notFound()
+                            } else {
+                                res.writeHead(
+                                    fetchResponse.status,
+                                    convertHeadersFromFetch(
+                                        fetchResponse.headers,
+                                    ),
+                                )
+                                fetchResponse
+                                    .bytes()
+                                    .then(data => res.end(data))
+                            }
                         })
                         .catch(e => {
-                            if (e === 'retrytimeout') {
+                            if (isFetchRetryTimeout(e)) {
                                 res.writeHead(504)
                             } else {
                                 console.error(
@@ -121,11 +257,14 @@ export function createDevServeFilesFetcher(
 const PROXY_FETCH_RETRY_INTERVAL = 27
 const PROXY_FETCH_RETRY_TIMEOUT = 1000
 
-async function retryFetchWithTimeout(url: string): Promise<Response> {
+async function retryFetchWithTimeout(
+    url: URL | string,
+    requestInit?: RequestInit,
+): Promise<Response> {
     let timeout = Date.now() + PROXY_FETCH_RETRY_TIMEOUT
     while (true) {
         try {
-            return await fetch(url)
+            return await fetch(url, requestInit)
         } catch (e: any) {
             if (isNodeFailedFetch(e) || isBunFailedFetch(e)) {
                 if (timeout < Date.now()) {
@@ -142,6 +281,10 @@ async function retryFetchWithTimeout(url: string): Promise<Response> {
     }
 }
 
+function isFetchRetryTimeout(e: any): boolean {
+    return e === 'retrytimeout'
+}
+
 function isBunFailedFetch(e: any): boolean {
     return e.code === 'ConnectionRefused'
 }
@@ -189,3 +332,8 @@ function convertHeadersToFetch(from: IncomingHttpHeaders): Headers {
     }
     return to
 }
+
+function errorExit(msg: string): never {
+    console.log(`\u001b[31merror:\u001b[0m`, msg)
+    process.exit(1)
+}

package/lib/metadata.ts

@@ -0,0 +1,284 @@
+import EventEmitter from 'node:events'
+import { writeFile } from 'node:fs/promises'
+import { dirname, join, resolve, sep } from 'node:path'
+import type { BuildResult } from 'esbuild'
+import type { EntryPoint } from './esbuild.ts'
+import type { DankBuild } from './flags.ts'
+
+export type Resolver = {
+    resolve(from: string, href: string): string | 'outofbounds'
+}
+
+// summary of a website build
+export type WebsiteManifest = {
+    buildTag: string
+    files: Set<string>
+    pageUrls: Set<string>
+}
+
+// result of an esbuild build from the context of the config's entrypoints
+// path of entrypoint is the reference point to lookup from a dependent page
+export type BuildManifest = {
+    // css and js bundles mapping output path to
+    // entrypoint path or null for code splitting chunks
+    bundles: Record<string, string | null>
+
+    // web worker urls mapped by dependent entrypoints
+    // to be built with a subsequent esbuild context
+    workers: Array<WorkerManifest> | null
+}
+
+type OnBuildComplete = (manifest: BuildManifest) => void
+
+type WorkerManifest = {
+    // path to module dependent on worker
+    clientScript: string
+    // path to bundled entrypoint dependent on `clientScript`
+    dependentEntryPoint: string
+    workerEntryPoint: string
+    workerUrl: string
+    workerUrlPlaceholder: string
+}
+
+export type WebsiteRegistryEvents = {
+    workers: []
+}
+
+// manages website resources during `dank build` and `dank serve`
+export class WebsiteRegistry
+    extends EventEmitter<WebsiteRegistryEvents>
+    implements Resolver
+{
+    #build: DankBuild
+    // paths of bundled esbuild outputs
+    #bundles: Set<string> = new Set()
+    // public dir assets
+    #copiedAssets: Set<string> | null = null
+    // map of entrypoints to their output path
+    #entrypointHrefs: Record<string, string | null> = {}
+    #pageUrls: Array<string> = []
+    #workers: Array<WorkerManifest> | null = null
+
+    constructor(build: DankBuild) {
+        super()
+        this.#build = build
+    }
+
+    // bundleOutputs(type?: 'css' | 'js'): Array<string> {
+    //     if (!type) {
+    //         return Array.from(this.#bundles)
+    //     } else {
+    //         return Array.from(this.#bundles).filter(p => p.endsWith(type))
+    //     }
+    // }
+
+    buildRegistry(): BuildRegistry {
+        return new BuildRegistry(this.#build, this.#onBuildManifest)
+    }
+
+    files(): Set<string> {
+        const files = new Set<string>()
+        for (const pageUrl of this.#pageUrls)
+            files.add(pageUrl === '/' ? '/index.html' : `${pageUrl}/index.html`)
+        for (const f of this.#bundles) files.add(f)
+        if (this.#copiedAssets) for (const f of this.#copiedAssets) files.add(f)
+        return files
+    }
+
+    mappedHref(lookup: string): string {
+        const found = this.#entrypointHrefs[lookup]
+        if (found) {
+            return found
+        } else {
+            throw Error(`mapped href for ${lookup} not found`)
+        }
+    }
+
+    resolve(from: string, href: string): string {
+        return resolveImpl(this.#build, from, href)
+    }
+
+    workerEntryPoints(): Array<EntryPoint> | null {
+        return (
+            this.#workers?.map(({ workerEntryPoint }) => ({
+                in: workerEntryPoint,
+                out: workerEntryPoint
+                    .replace(/^pages[\//]/, '')
+                    .replace(/\.(mj|t)s$/, '.js'),
+            })) || null
+        )
+    }
+
+    workers(): Array<WorkerManifest> | null {
+        return this.#workers
+    }
+
+    async writeManifest(buildTag: string): Promise<WebsiteManifest> {
+        const manifest = this.#manifest(buildTag)
+        await writeFile(
+            join(
+                this.#build.dirs.projectRootAbs,
+                this.#build.dirs.buildRoot,
+                'website.json',
+            ),
+            JSON.stringify(
+                {
+                    buildTag,
+                    files: Array.from(manifest.files),
+                    pageUrls: Array.from(manifest.pageUrls),
+                },
+                null,
+                4,
+            ),
+        )
+        return manifest
+    }
+
+    set copiedAssets(copiedAssets: Array<string> | null) {
+        this.#copiedAssets =
+            copiedAssets === null ? null : new Set(copiedAssets)
+    }
+
+    set pageUrls(pageUrls: Array<string>) {
+        this.#pageUrls = pageUrls
+    }
+
+    #manifest(buildTag: string): WebsiteManifest {
+        return {
+            buildTag,
+            files: this.files(),
+            pageUrls: new Set(this.#pageUrls),
+        }
+    }
+
+    #onBuildManifest: OnBuildComplete = (build: BuildManifest) => {
+        // collect built bundle entrypoint hrefs
+        for (const [outPath, entrypoint] of Object.entries(build.bundles)) {
+            this.#bundles.add(outPath)
+            if (entrypoint) {
+                this.#entrypointHrefs[entrypoint] = outPath
+            }
+        }
+
+        // determine if worker entrypoints have changed
+        let updatedWorkerEntrypoints = false
+        const previousWorkers =
+            this.#workers === null
+                ? null
+                : new Set(this.#workers.map(w => w.workerEntryPoint))
+        if (build.workers) {
+            if (
+                !previousWorkers ||
+                previousWorkers.size !==
+                    new Set(build.workers.map(w => w.workerEntryPoint)).size
+            ) {
+                updatedWorkerEntrypoints = true
+            } else {
+                updatedWorkerEntrypoints = !build.workers.every(w =>
+                    previousWorkers.has(w.workerEntryPoint),
+                )
+            }
+        } else if (previousWorkers) {
+            updatedWorkerEntrypoints = true
+        }
+
+        // merge unique entrypoints from built workers with registry state
+        // todo filtering out unique occurrences of clientScript and workerUrl
+        //  drops reporting/summary/debugging capabilities, but currently
+        //  this.#workers is used for unique worker/client entrypoints
+        if (build.workers) {
+            if (!this.#workers) {
+                this.#workers = build.workers
+            } else {
+                for (const w of build.workers) {
+                    const found = this.#workers.find(w2 => {
+                        return (
+                            w.dependentEntryPoint === w2.dependentEntryPoint &&
+                            w.workerEntryPoint === w2.workerEntryPoint
+                        )
+                    })
+                    if (!found) {
+                        this.#workers.push(w)
+                    }
+                }
+            }
+        }
+
+        if (updatedWorkerEntrypoints) {
+            this.emit('workers')
+        }
+    }
+}
+
+// result accumulator of an esbuild `build` or `Context.rebuild`
+export class BuildRegistry implements Resolver {
+    #build: DankBuild
+    #onComplete: OnBuildComplete
+    #workers: Array<Omit<WorkerManifest, 'dependentEntryPoint'>> | null = null
+
+    constructor(
+        build: DankBuild,
+        onComplete: (manifest: BuildManifest) => void,
+    ) {
+        this.#build = build
+        this.#onComplete = onComplete
+    }
+
+    // resolve web worker imported by a webpage module
+    addWorker(worker: Omit<WorkerManifest, 'dependentEntryPoint'>) {
+        // todo normalize path
+        if (!this.#workers) {
+            this.#workers = [worker]
+        } else {
+            this.#workers.push(worker)
+        }
+    }
+
+    completeBuild(result: BuildResult<{ metafile: true }>) {
+        const bundles: Record<string, string | null> = {}
+        for (const [outPath, output] of Object.entries(
+            result.metafile.outputs,
+        )) {
+            bundles[outPath.replace(/^build[/\\]dist/, '')] =
+                output.entryPoint || null
+        }
+        let workers: BuildManifest['workers'] = null
+        if (this.#workers) {
+            workers = []
+            for (const output of Object.values(result.metafile.outputs)) {
+                if (!output.entryPoint) continue
+                const inputs = Object.keys(output.inputs)
+                for (const worker of this.#workers) {
+                    if (inputs.includes(worker.clientScript)) {
+                        workers.push({
+                            ...worker,
+                            dependentEntryPoint: output.entryPoint,
+                        })
+                    }
+                }
+            }
+        }
+
+        this.#onComplete({
+            bundles,
+            workers,
+        })
+    }
+
+    resolve(from: string, href: string): string {
+        return resolveImpl(this.#build, from, href)
+    }
+}
+
+function resolveImpl(build: DankBuild, from: string, href: string): string {
+    const { pagesResolved, projectRootAbs } = build.dirs
+    const fromDir = dirname(from)
+    const resolvedFromProjectRoot = join(projectRootAbs, fromDir, href)
+    if (!resolve(resolvedFromProjectRoot).startsWith(pagesResolved)) {
+        throw Error(
+            `href ${href} cannot be resolved from pages${sep}${from} to a path outside of the pages directory`,
+        )
+    } else {
+        return join(fromDir, href)
+    }
+}

package/lib/public.ts

@@ -1,14 +1,16 @@
 import { copyFile, mkdir, readdir, stat } from 'node:fs/promises'
+import { platform } from 'node:os'
 import { join } from 'node:path'
+import type { DankBuild } from './flags.ts'
 
 export async function copyAssets(
-    outRoot: string,
+    build: DankBuild,
 ): Promise<Array<string> | null> {
     try {
-        const stats = await stat('public')
+        const stats = await stat(build.dirs.public)
         if (stats.isDirectory()) {
-            await mkdir(outRoot, { recursive: true })
-            return await recursiveCopyAssets(outRoot)
+            await mkdir(build.dirs.buildDist, { recursive: true })
+            return await recursiveCopyAssets(build)
         } else {
             throw Error('./public cannot be a file')
         }
@@ -17,26 +19,32 @@ export async function copyAssets(
     }
 }
 
+const IGNORE = platform() === 'darwin' ? ['.DS_Store'] : []
+
 async function recursiveCopyAssets(
-    outRoot: string,
+    build: DankBuild,
     dir: string = '',
 ): Promise<Array<string>> {
     const copied: Array<string> = []
-    const to = join(outRoot, dir)
+    const to = join(build.dirs.buildDist, dir)
     let madeDir = dir === ''
-    for (const p of await readdir(join('public', dir))) {
+    const listingDir = join(build.dirs.public, dir)
+    for (const p of await readdir(listingDir)) {
+        if (IGNORE.includes(p)) {
+            continue
+        }
         try {
-            const stats = await stat(join('public', dir, p))
+            const stats = await stat(join(listingDir, p))
             if (stats.isDirectory()) {
-                copied.push(
-                    ...(await recursiveCopyAssets(outRoot, join(dir, p))),
-                )
+                copied.push(...(await recursiveCopyAssets(build, join(dir, p))))
             } else {
                 if (!madeDir) {
-                    await mkdir(join(outRoot, dir))
+                    await mkdir(join(build.dirs.buildDist, dir), {
+                        recursive: true,
+                    })
                     madeDir = true
                 }
-                await copyFile(join('public', dir, p), join(to, p))
+                await copyFile(join(listingDir, p), join(to, p))
                 copied.push('/' + join(dir, p).replaceAll('\\', '/'))
             }
         } catch (e) {