@pathmx/core 0.5.0

package/publish/build.ts

@@ -0,0 +1,402 @@
+import {
+  anonymousPrincipal,
+  type CredentialSession,
+} from "../authority/model.ts"
+import type { Engine } from "../host/engine.ts"
+import { renderUncachedPage } from "../host/render.ts"
+import { AGENT_BROWSING_PLUGIN_ID } from "../server/protocols.ts"
+import { handleRequestAtPath } from "../server/http.ts"
+import { publicPath } from "../server/router.ts"
+import type { PublishContext } from "../plugins/types.ts"
+import type { Source } from "../source/index.ts"
+import type {
+  PublicationBundle,
+  PublicationRepresentation,
+  PublicationStore,
+} from "./model.ts"
+import { mapConcurrent } from "./concurrent.ts"
+import {
+  canonicalPublishedPath,
+  comparePaths,
+  exactArtifact,
+  fileRepresentation,
+  pageArtifact,
+  PublicationError,
+  PublicationPlan,
+  responseHeaders,
+} from "./plan.ts"
+import { validatePublication } from "./validate.ts"
+import {
+  createPublicationLocation,
+  type PublicationLocation,
+} from "./location.ts"
+
+const anonymousSession = Object.freeze({
+  type: "anonymous",
+  signIn: null,
+}) satisfies CredentialSession
+
+const RESERVED_ROUTES = ["/actions", "/api"] as const
+
+const memoryStore: PublicationStore = Object.freeze({
+  async put(_path: string, body: Blob) {
+    return Object.freeze({
+      size: body.size,
+      read: async () => body,
+    })
+  },
+})
+
+function isReserved(pathname: string) {
+  return RESERVED_ROUTES.some(
+    (root) => pathname === root || pathname.startsWith(`${root}/`),
+  )
+}
+
+function publicSource(app: Engine, source: Source) {
+  return app.readAccess(anonymousPrincipal, source) === "public"
+}
+
+function contentType(response: Response, fallback: string) {
+  return response.headers.get("content-type") || fallback
+}
+
+async function responseBlob(response: Response, pathname: string) {
+  if (!response.ok) {
+    throw new PublicationError(
+      `Publication route ${pathname} returned ${response.status}.`,
+    )
+  }
+  return response.blob()
+}
+
+async function collectPluginRoutes(app: Engine) {
+  const routes = new Set<string>()
+  const issues: string[] = []
+  for (const plugin of app.plugins) {
+    if (!plugin.publish) continue
+    const ctx: PublishContext = Object.freeze({
+      route(pathname) {
+        routes.add(canonicalPublishedPath(pathname))
+      },
+      unsupported(source, message) {
+        issues.push(`${plugin.id}: ${source.path}: ${message}`)
+      },
+    })
+    await plugin.publish(ctx, app.appContext(plugin.id))
+  }
+  if (issues.length) {
+    throw new PublicationError(
+      `Static publication is unsupported:\n${issues.map((issue) => `- ${issue}`).join("\n")}`,
+    )
+  }
+  return [...routes].sort(comparePaths)
+}
+
+async function addPublishedRoutes(
+  app: Engine,
+  plan: PublicationPlan,
+  location: PublicationLocation,
+  pathnames: readonly string[],
+) {
+  for (const pathname of pathnames) {
+    const mounted = location.pathname(pathname)
+    const response = await handleRequestAtPath(
+      app,
+      new Request(location.url(pathname), {
+        headers: { Accept: "*/*" },
+      }),
+      pathname,
+    )
+    const type = contentType(response, "application/octet-stream")
+    const artifact = exactArtifact(mounted)
+    const body = await location.relocate(
+      await responseBlob(response, pathname),
+      pathname,
+      type,
+    )
+    plan.addArtifact({
+      path: artifact,
+      contentType: type,
+      read: async () => body,
+    })
+    plan.addRoute({
+      pathname: mounted,
+      type: "artifact",
+      representations: [
+        fileRepresentation(artifact, type, responseHeaders(response)),
+      ],
+    })
+  }
+}
+
+async function addSources(
+  app: Engine,
+  plan: PublicationPlan,
+  store: PublicationStore,
+  location: PublicationLocation,
+) {
+  const sources = app.repo
+    .all()
+    .filter((source) => publicSource(app, source))
+    .sort((left, right) => comparePaths(left.path, right.path))
+  const markdownArtifacts = new Map<string, string>()
+
+  for (const source of sources) {
+    const pathname = location.pathname(source.path)
+    if (plan.hasRoute(pathname) || isReserved(source.path)) continue
+    const artifact = exactArtifact(pathname)
+    const type = "text/markdown; charset=utf-8"
+    const content = source.content
+    plan.addArtifact({
+      path: artifact,
+      contentType: type,
+      read: async () => new Blob([content], { type }),
+    })
+    plan.addRoute({
+      pathname,
+      type: "source",
+      sourcePath: source.path,
+      representations: [
+        Object.freeze({
+          type: "markdown",
+          default: true,
+          artifact,
+          contentType: type,
+        }),
+      ],
+    })
+    markdownArtifacts.set(source.path, artifact)
+  }
+
+  const pages = sources.filter((source) => {
+    if (!app.isPage(source)) return false
+    const pathname = publicPath(source)
+    const mounted = location.pathname(pathname)
+    return (
+      !plan.hasRoute(mounted) &&
+      !isReserved(pathname) &&
+      app.resolve(pathname) === source
+    )
+  })
+  await mapConcurrent(pages, async (source) => {
+    const pathname = publicPath(source)
+    const mounted = location.pathname(pathname)
+    const response = await renderUncachedPage(
+      app,
+      anonymousSession,
+      anonymousPrincipal,
+      source,
+    )
+    const type = contentType(response, "text/html; charset=utf-8")
+    const body = await location.relocate(
+      await responseBlob(response, pathname),
+      pathname,
+      type,
+    )
+    const artifact = pageArtifact(mounted)
+    plan.addArtifact({
+      path: artifact,
+      contentType: type,
+      read: async () => body,
+    })
+    const representations: PublicationRepresentation[] = [
+      Object.freeze({
+        type: "html",
+        default: true,
+        artifact,
+        contentType: type,
+      }),
+    ]
+    const markdown = markdownArtifacts.get(source.path)
+    if (markdown && app.hasPlugin(AGENT_BROWSING_PLUGIN_ID)) {
+      representations.push(
+        Object.freeze({
+          type: "markdown",
+          accept: "text/markdown",
+          artifact: markdown,
+          contentType: "text/markdown; charset=utf-8",
+          alternatePath: location.pathname(source.path),
+        }),
+      )
+    }
+    plan.addRoute({
+      pathname: mounted,
+      type: "page",
+      sourcePath: source.path,
+      representations,
+    })
+    await plan.stagePage(artifact, store)
+  })
+}
+
+async function addAuthoredAssets(
+  app: Engine,
+  plan: PublicationPlan,
+  location: PublicationLocation,
+) {
+  const storage = app.environment.storage
+  const files = [...(await storage.list(app.repo.rootPath))].sort(comparePaths)
+  for (const relative of files) {
+    if (relative.endsWith(".md")) continue
+    const pathname = app.repo.graphPath(relative)
+    const mounted = location.pathname(pathname)
+    if (plan.hasRoute(mounted) || isReserved(pathname)) continue
+    if (app.readAccess(anonymousPrincipal, pathname) !== "public") continue
+    const storagePath = app.repo.storagePath(pathname)
+    if (!storagePath) continue
+    const body = await storage.readBlob(storagePath)
+    const type = body.type || "application/octet-stream"
+    const artifact = exactArtifact(mounted)
+    plan.addArtifact({
+      path: artifact,
+      contentType: type,
+      read: async () =>
+        location.relocate(await storage.readBlob(storagePath), pathname, type),
+    })
+    plan.addRoute({
+      pathname: mounted,
+      type: "asset",
+      representations: [fileRepresentation(artifact, type)],
+    })
+  }
+}
+
+function addGeneratedAssets(
+  app: Engine,
+  plan: PublicationPlan,
+  location: PublicationLocation,
+) {
+  for (const [pathname, asset] of app.generatedAssetEntries()) {
+    const mounted = location.pathname(pathname)
+    if (plan.hasRoute(mounted)) {
+      throw new PublicationError(`Generated asset route collision: ${pathname}`)
+    }
+    const artifact = exactArtifact(mounted)
+    plan.addArtifact({
+      path: artifact,
+      contentType: asset.contentType,
+      read: async () => {
+        const body =
+          typeof asset.body === "string"
+            ? new Blob([asset.body], { type: asset.contentType })
+            : asset.body
+        return location.relocate(body, pathname, asset.contentType)
+      },
+    })
+    plan.addRoute({
+      pathname: mounted,
+      type: "asset",
+      representations: [
+        fileRepresentation(artifact, asset.contentType, {
+          "cache-control": "public, max-age=31536000, immutable",
+        }),
+      ],
+    })
+  }
+}
+
+function addPageMap(
+  plan: PublicationPlan,
+  build: string,
+  location: PublicationLocation,
+) {
+  const pathname = location.pathname(`/.pmx/pages-${build}.json`)
+  const artifact = exactArtifact(pathname)
+  const type = "application/json; charset=utf-8"
+  const body = new Blob(
+    [
+      JSON.stringify({
+        version: 1,
+        build,
+        pages: plan.pageLeaves(),
+      }) + "\n",
+    ],
+    { type },
+  )
+  plan.addArtifact({
+    path: artifact,
+    contentType: type,
+    size: body.size,
+    read: async () => body,
+  })
+  plan.addRoute({
+    pathname,
+    type: "artifact",
+    representations: [
+      fileRepresentation(artifact, type, {
+        "cache-control": "public, max-age=31536000, immutable",
+      }),
+    ],
+  })
+  return pathname
+}
+
+function addBuildRecord(
+  plan: PublicationPlan,
+  build: string,
+  pages: string,
+  location: PublicationLocation,
+) {
+  const pathname = location.pathname("/.pmx/build.json")
+  const artifact = exactArtifact(pathname)
+  const type = "application/json; charset=utf-8"
+  const body = new Blob([JSON.stringify({ version: build, pages }) + "\n"], {
+    type,
+  })
+  plan.addArtifact({
+    path: artifact,
+    contentType: type,
+    size: body.size,
+    read: async () => body,
+  })
+  plan.addRoute({
+    pathname,
+    type: "artifact",
+    representations: [
+      fileRepresentation(artifact, type, { "cache-control": "no-store" }),
+    ],
+  })
+}
+
+/** Build a canonical public route bundle without selecting a hosting platform. */
+export async function createPublication(
+  app: Engine,
+  options: Readonly<{
+    origin: string | URL
+    basePath?: string
+    store?: PublicationStore
+  }>,
+): Promise<PublicationBundle> {
+  if (app.runtime.target !== "static") {
+    throw new PublicationError(
+      "Static publication requires runtime.target to be static.",
+    )
+  }
+  const location = createPublicationLocation(
+    app,
+    options.origin,
+    options.basePath,
+  )
+  const store = options.store ?? memoryStore
+  const plan = new PublicationPlan()
+  const published = await collectPluginRoutes(app)
+  await addPublishedRoutes(app, plan, location, published)
+  await addSources(app, plan, store, location)
+  await addAuthoredAssets(app, plan, location)
+  addGeneratedAssets(app, plan, location)
+  await plan.materialize(store)
+  const build = await plan.buildRoot(location.origin, location.basePath)
+  const pages = addPageMap(plan, build, location)
+  await plan.materialize(store)
+  addBuildRecord(plan, build, pages, location)
+  await plan.materialize(store)
+  const publication = Object.freeze({
+    manifest: plan.manifest(build, location.origin, location.basePath),
+    artifacts: Object.freeze(plan.artifactList()),
+  })
+  await validatePublication(app, publication)
+  return publication
+}
+
+export { PublicationError } from "./plan.ts"

package/publish/concurrent.ts

@@ -0,0 +1,32 @@
+export const PUBLICATION_CONCURRENCY = 16
+
+/** Map publication work concurrently while preserving input order. */
+export async function mapConcurrent<T, Result>(
+  items: readonly T[],
+  map: (item: T) => Promise<Result>,
+) {
+  const results = new Array<Result>(items.length)
+  let cursor = 0
+  let failed = false
+  let failure: unknown
+  const worker = async () => {
+    while (!failed) {
+      const index = cursor++
+      if (index >= items.length) return
+      try {
+        results[index] = await map(items[index]!)
+      } catch (error) {
+        failed = true
+        failure = error
+      }
+    }
+  }
+  await Promise.all(
+    Array.from(
+      { length: Math.min(PUBLICATION_CONCURRENCY, items.length) },
+      worker,
+    ),
+  )
+  if (failed) throw failure
+  return results
+}

package/publish/identity.ts

@@ -0,0 +1,63 @@
+import { blobFingerprint } from "../assets.ts"
+import {
+  PUBLICATION_VERSION,
+  type PublicationArtifactRecord,
+  type PublicationRoute,
+} from "./model.ts"
+import { mapConcurrent } from "./concurrent.ts"
+
+const ARTIFACT_DOMAIN = "application/vnd.pathmx.artifact-leaf"
+const CONTRACT_DOMAIN = "application/vnd.pathmx.publication-contract"
+const NODE_DOMAIN = "application/vnd.pathmx.merkle-node"
+
+export function artifactLeaf(path: string, contentType: string, body: Blob) {
+  return blobFingerprint(body, `${ARTIFACT_DOMAIN}\0${path}\0${contentType}`)
+}
+
+function contractLeaf(
+  origin: URL,
+  basePath: string,
+  routes: readonly PublicationRoute[],
+) {
+  return blobFingerprint(
+    new Blob([
+      JSON.stringify({
+        version: PUBLICATION_VERSION,
+        origin: origin.origin,
+        basePath,
+        routes,
+      }),
+    ]),
+    CONTRACT_DOMAIN,
+  )
+}
+
+function parentLeaf(left: string, right: string) {
+  return blobFingerprint(new Blob([left, "\0", right]), NODE_DOMAIN)
+}
+
+async function merkleRoot(leaves: readonly string[]) {
+  let level = [...leaves]
+  while (level.length > 1) {
+    const previous = level
+    level = await mapConcurrent(
+      Array.from(
+        { length: Math.ceil(previous.length / 2) },
+        (_, index) => index * 2,
+      ),
+      (index) =>
+        parentLeaf(previous[index]!, previous[index + 1] ?? previous[index]!),
+    )
+  }
+  return level[0]!
+}
+
+export async function publicationRoot(
+  origin: URL,
+  basePath: string,
+  routes: readonly PublicationRoute[],
+  artifacts: readonly PublicationArtifactRecord[],
+) {
+  const contract = await contractLeaf(origin, basePath, routes)
+  return merkleRoot([contract, ...artifacts.map((artifact) => artifact.leaf)])
+}

package/publish/index.ts

@@ -0,0 +1,11 @@
+export { createPublication, PublicationError } from "./build.ts"
+export type {
+  PublicationArtifact,
+  PublicationArtifactRecord,
+  PublicationBundle,
+  PublicationManifest,
+  PublicationRepresentation,
+  PublicationRoute,
+  PublicationStore,
+  PublicationStoredBody,
+} from "./model.ts"

package/publish/location.ts

@@ -0,0 +1,167 @@
+import type { Engine } from "../host/engine.ts"
+import {
+  canonicalBasePath,
+  mountedPublishedPath,
+  PublicationError,
+} from "./plan.ts"
+
+const NETWORK_URL = /^(?:[a-z][a-z\d+.-]*:|\/\/)/i
+const CSS_URL = /url\(\s*(?:(["'])(.*?)\1|([^)'"\s][^)]*?))\s*\)/gi
+const CSS_IMPORT = /(@import\s+)(["'])(.*?)\2/gi
+
+export type PublicationLocation = Readonly<{
+  origin: URL
+  basePath: string
+  pathname(pathname: string): string
+  url(pathname: string): URL
+  relocate(body: Blob, route: string, contentType: string): Promise<Blob>
+}>
+
+function canonicalOrigin(value: string | URL) {
+  let origin: URL
+  try {
+    origin = new URL(value)
+  } catch {
+    throw new PublicationError(`Invalid publication origin: ${value}`)
+  }
+  if (
+    (origin.protocol !== "http:" && origin.protocol !== "https:") ||
+    origin.username ||
+    origin.password ||
+    origin.pathname !== "/" ||
+    origin.search ||
+    origin.hash
+  ) {
+    throw new PublicationError(
+      `Publication origin must be an HTTP(S) origin without a path: ${origin}`,
+    )
+  }
+  return origin
+}
+
+function relocatedUrl(
+  value: string,
+  origin: URL,
+  basePath: string,
+  route: string,
+) {
+  const input = value.trim()
+  if (!input || input.startsWith("#") || NETWORK_URL.test(input)) return
+  try {
+    const resolved = new URL(input, new URL(route, origin))
+    if (resolved.origin !== origin.origin) return
+    return `${mountedPublishedPath(basePath, resolved.pathname)}${resolved.search}${resolved.hash}`
+  } catch {
+    return
+  }
+}
+
+function relocateCss(
+  css: string,
+  origin: URL,
+  basePath: string,
+  route: string,
+) {
+  const relocate = (value: string) =>
+    relocatedUrl(value, origin, basePath, route)
+  return css
+    .replace(CSS_URL, (match, quote, quoted, bare) => {
+      const value = quoted ?? bare?.trim()
+      const relocated = value ? relocate(value) : undefined
+      return relocated ? `url(${quote ?? ""}${relocated}${quote ?? ""})` : match
+    })
+    .replace(CSS_IMPORT, (match, prefix, quote, value) => {
+      const relocated = relocate(value)
+      return relocated ? `${prefix}${quote}${relocated}${quote}` : match
+    })
+}
+
+function relocateHtml(
+  app: Engine,
+  html: string,
+  origin: URL,
+  basePath: string,
+  route: string,
+) {
+  const mountedRoute = mountedPublishedPath(basePath, route)
+  const relocated = app.environment.rewriteHtml(html, [
+    [
+      "html",
+      {
+        element(element) {
+          element.setAttribute("data-pmx-base", basePath)
+        },
+      },
+    ],
+    [
+      "[data-pmx-route]",
+      {
+        element(element) {
+          element.setAttribute("data-pmx-route", mountedRoute)
+        },
+      },
+    ],
+    [
+      "[href], [src], [action], [poster], object[data]",
+      {
+        element(element) {
+          for (const attribute of ["href", "src", "action", "poster", "data"]) {
+            const value = element.getAttribute(attribute)
+            const relocated = value
+              ? relocatedUrl(value, origin, basePath, route)
+              : undefined
+            if (relocated) element.setAttribute(attribute, relocated)
+          }
+        },
+      },
+    ],
+    [
+      "[style]",
+      {
+        element(element) {
+          const value = element.getAttribute("style")
+          if (value) {
+            element.setAttribute(
+              "style",
+              relocateCss(value, origin, basePath, route),
+            )
+          }
+        },
+      },
+    ],
+  ])
+  return relocated.replace(
+    /(<style\b[^>]*>)([\s\S]*?)(<\/style\s*>)/gi,
+    (_match, open, css, close) =>
+      `${open}${relocateCss(css, origin, basePath, route)}${close}`,
+  )
+}
+
+export function createPublicationLocation(
+  app: Engine,
+  originValue: string | URL,
+  basePathValue?: string,
+): PublicationLocation {
+  const origin = canonicalOrigin(originValue)
+  const basePath = canonicalBasePath(basePathValue)
+  const pathname = (path: string) => mountedPublishedPath(basePath, path)
+  return Object.freeze({
+    origin,
+    basePath,
+    pathname,
+    url: (path: string) => new URL(pathname(path), origin),
+    async relocate(body: Blob, route: string, contentType: string) {
+      if (
+        !contentType.includes("text/html") &&
+        !contentType.includes("text/css")
+      ) {
+        return body
+      }
+      const text = await body.text()
+      const relocated = contentType.includes("text/html")
+        ? relocateHtml(app, text, origin, basePath, route)
+        : relocateCss(text, origin, basePath, route)
+      return new Blob([relocated], { type: contentType })
+    },
+  })
+}