@pathmx/core 0.5.0

package/environments/bun.ts

@@ -0,0 +1,223 @@
+import { mkdir, unlink } from "node:fs/promises"
+import { dirname, join, posix } from "node:path"
+import { fileURLToPath } from "node:url"
+import { Database } from "bun:sqlite"
+import { YAML } from "bun"
+import type {
+  SqlConnection,
+  SqlDatabase,
+  SqlParams,
+  SqlResult,
+  SqlValue,
+} from "../database.ts"
+import type {
+  BundledAsset,
+  HtmlRewrite,
+  PathMXEnvironment,
+} from "../environment.ts"
+
+export { publishToDirectory, writePublication } from "./bun-publish.ts"
+
+function positional(params: SqlParams): params is readonly SqlValue[] {
+  return Array.isArray(params)
+}
+
+function isBusy(error: unknown): error is { code: "SQLITE_BUSY" } {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    error.code === "SQLITE_BUSY"
+  )
+}
+
+async function retryBusy<T>(run: () => T, timeoutMs = 5_000): Promise<T> {
+  const deadline = Date.now() + timeoutMs
+  while (true) {
+    try {
+      return run()
+    } catch (error) {
+      if (!isBusy(error) || Date.now() >= deadline) throw error
+      await new Promise((resolve) => setTimeout(resolve, 10))
+    }
+  }
+}
+
+function bunDatabase(path: string): SqlConnection {
+  const database = new Database(path, { create: true, strict: true })
+  database.exec("PRAGMA foreign_keys = ON")
+  database.exec("PRAGMA journal_mode = WAL")
+  // A synchronous busy wait blocks the JS thread and can deadlock the async
+  // transaction holding the competing connection. Retry by yielding instead.
+  database.exec("PRAGMA busy_timeout = 0")
+
+  let tail = Promise.resolve()
+  const enqueue = <T>(run: () => T | Promise<T>) => {
+    const result = tail.then(run, run)
+    tail = result.then(
+      () => undefined,
+      () => undefined,
+    )
+    return result
+  }
+  const direct = (allowTransaction: boolean): SqlDatabase =>
+    Object.freeze({
+      all: async <Row>(sql: string, params?: SqlParams) => {
+        return retryBusy(() => {
+          if (!params) return database.query<Row, []>(sql).all()
+          return positional(params)
+            ? database.query<Row, SqlValue[]>(sql).all(...params)
+            : database
+                .query<Row, Record<string, SqlValue>>(sql)
+                .all({ ...params })
+        })
+      },
+      get: async <Row>(sql: string, params?: SqlParams) => {
+        const row = await retryBusy(() =>
+          !params
+            ? database.query<Row, []>(sql).get()
+            : positional(params)
+              ? database.query<Row, SqlValue[]>(sql).get(...params)
+              : database
+                  .query<Row, Record<string, SqlValue>>(sql)
+                  .get({ ...params }),
+        )
+        return row ?? undefined
+      },
+      run: async (sql: string, params?: SqlParams): Promise<SqlResult> => {
+        const result = await retryBusy(() =>
+          !params
+            ? database.query<never, []>(sql).run()
+            : positional(params)
+              ? database.query<never, SqlValue[]>(sql).run(...params)
+              : database
+                  .query<never, Record<string, SqlValue>>(sql)
+                  .run({ ...params }),
+        )
+        return {
+          changes: result.changes,
+          ...(result.lastInsertRowid != null
+            ? { lastInsertRowId: result.lastInsertRowid }
+            : {}),
+        }
+      },
+      exec: async (sql: string) => {
+        database.exec(sql)
+      },
+      transaction: async <T>(run: (database: SqlDatabase) => Promise<T>) => {
+        if (!allowTransaction) {
+          throw new Error("Nested database transactions are not supported.")
+        }
+        await retryBusy(() => database.exec("BEGIN IMMEDIATE"))
+        try {
+          const result = await run(direct(false))
+          await retryBusy(() => database.exec("COMMIT"))
+          return result
+        } catch (error) {
+          database.exec("ROLLBACK")
+          throw error
+        }
+      },
+    })
+  const raw = direct(true)
+  return Object.freeze({
+    all: <Row>(sql: string, params?: SqlParams) =>
+      enqueue(() => raw.all<Row>(sql, params)),
+    get: <Row>(sql: string, params?: SqlParams) =>
+      enqueue(() => raw.get<Row>(sql, params)),
+    run: (sql: string, params?: SqlParams) =>
+      enqueue(() => raw.run(sql, params)),
+    exec: (sql: string) => enqueue(() => raw.exec(sql)),
+    transaction: <T>(run: (database: SqlDatabase) => Promise<T>) =>
+      enqueue(() => raw.transaction(run)),
+    close: () =>
+      enqueue(() => {
+        database.close(false)
+      }),
+  })
+}
+
+function bundledAssetType(contentType: string) {
+  if (contentType.includes("javascript")) return "script" as const
+  if (contentType.includes("css")) return "style" as const
+  return "file" as const
+}
+
+function rewriteHtml(input: string, rules: Parameters<HtmlRewrite>[1]): string
+function rewriteHtml(
+  input: Response,
+  rules: Parameters<HtmlRewrite>[1],
+): Response
+function rewriteHtml(
+  input: string | Response,
+  rules: Parameters<HtmlRewrite>[1],
+) {
+  const rewriter = new HTMLRewriter()
+  for (const [selector, handler] of rules) {
+    rewriter.on(
+      selector,
+      handler as HTMLRewriterTypes.HTMLRewriterElementContentHandlers,
+    )
+  }
+  if (typeof input === "string") return rewriter.transform(input)
+  return rewriter.transform(input)
+}
+
+export const bunEnvironment: PathMXEnvironment = {
+  renderMarkdown: (markdown) =>
+    Bun.markdown.html(markdown, {
+      headings: { ids: true },
+      noIndentedCodeBlocks: true,
+    }),
+  rewriteHtml,
+  loadAsset: (url) => Bun.file(url),
+  bundle: async (entrypoint) => {
+    const result = await Bun.build({
+      entrypoints: [
+        entrypoint instanceof URL ? fileURLToPath(entrypoint) : entrypoint,
+      ],
+      target: "browser",
+      minify: true,
+      define: { "process.env.NODE_ENV": JSON.stringify("production") },
+      throw: true,
+    })
+    return Promise.all(
+      result.outputs.map(async (artifact): Promise<BundledAsset> => {
+        const path = artifact.path.replace(/^\.\//, "")
+        const type = bundledAssetType(artifact.type)
+        return type === "file"
+          ? { path, type, blob: artifact }
+          : { path, type, text: await artifact.text() }
+      }),
+    )
+  },
+  loadModule: async (specifier) => {
+    if (!(await Bun.file(specifier).exists())) return
+    return import(specifier)
+  },
+  path: posix,
+  yaml: {
+    parse: YAML.parse,
+    stringify: (value) => YAML.stringify(value, null, 2),
+  },
+  storage: {
+    list: (root) => [
+      ...new Bun.Glob("**/*").scanSync({ cwd: root, onlyFiles: true }),
+    ],
+    readText: (path) => Bun.file(path).text(),
+    readBlob: (path) => Bun.file(path),
+    writeText: async (path, content) => {
+      await mkdir(dirname(path), { recursive: true })
+      await Bun.write(path, content)
+    },
+    delete: (path) => unlink(path),
+    exists: (path) => Bun.file(path).exists(),
+  },
+  database: {
+    path: join,
+    open: async (path) => {
+      await mkdir(dirname(path), { recursive: true })
+      return bunDatabase(path)
+    },
+  },
+}

package/globals.d.ts

@@ -0,0 +1,14 @@
+declare module "*.css" {
+  const text: string
+  export default text
+}
+
+declare module "*.html" {
+  const text: string
+  export default text
+}
+
+declare module "*.js" {
+  const text: string
+  export default text
+}

package/graph.ts

@@ -0,0 +1,153 @@
+import type { Repository } from "./repository"
+import { resolveRoute } from "./server/router"
+import {
+  interpolateText,
+  isExternalHref,
+  type Source,
+  type LinkDirective,
+  type SourceLinkType,
+  type SourceNode,
+} from "./source"
+
+function splitLinkTarget(href: string) {
+  const hash = href.indexOf("#")
+  const withoutFragment = hash === -1 ? href : href.slice(0, hash)
+  const query = withoutFragment.indexOf("?")
+  return {
+    pathname: query === -1 ? withoutFragment : withoutFragment.slice(0, query),
+    query: query === -1 ? undefined : withoutFragment.slice(query + 1),
+    fragment: hash === -1 ? undefined : href.slice(hash + 1),
+  }
+}
+
+export type GraphEdge = {
+  from: string
+  href: string
+  type: SourceLinkType
+  label?: string
+  directive?: LinkDirective
+  to?: string
+  fragment?: string
+}
+
+type LinkResolution = {
+  type: SourceLinkType
+  targetPath?: string
+  fragment?: string
+}
+
+export class Graph {
+  private repository: Repository
+  nodes = new Map<string, SourceNode>()
+  private outgoing = new Map<string, GraphEdge[]>()
+  private incoming = new Map<string, GraphEdge[]>()
+
+  constructor(repository: Repository) {
+    this.repository = repository
+  }
+
+  /** Classify authored Source/Block links and rebuild adjacency. */
+  async rebuild() {
+    this.nodes.clear()
+    this.outgoing.clear()
+    this.incoming.clear()
+    for (const source of this.repository.all()) {
+      await this.indexSource(source)
+    }
+  }
+
+  async indexSource(source: Source) {
+    await this.indexNode(source)
+    for (const block of source.blocks) this.nodes.set(block.path, block)
+  }
+
+  linksFrom(path: string) {
+    return this.outgoing.get(path) ?? []
+  }
+
+  linksTo(path: string) {
+    return this.incoming.get(path) ?? []
+  }
+
+  unresolvedLinks() {
+    return [...this.outgoing.values()]
+      .flat()
+      .filter((edge) => edge.type === "unresolved")
+  }
+
+  neighbors(path: string) {
+    return this.linksFrom(path)
+      .map((edge) => (edge.to ? this.nodes.get(edge.to) : undefined))
+      .filter((node): node is SourceNode => node != null)
+  }
+
+  backlinks(path: string) {
+    return this.linksTo(path)
+      .map((edge) => this.nodes.get(edge.from))
+      .filter((node): node is SourceNode => node != null)
+  }
+
+  private async indexNode(node: SourceNode) {
+    this.nodes.set(node.path, node)
+    const edges: GraphEdge[] = []
+    for (const link of node.links) {
+      const resolved = await this.resolveLink(
+        node,
+        interpolateText(link.href, node.data),
+      )
+      const edge: GraphEdge = {
+        from: node.path,
+        href: link.href,
+        type: resolved.type,
+        label: link.label,
+        directive: link.directive,
+        to: resolved.targetPath,
+        fragment: resolved.fragment,
+      }
+      edges.push(edge)
+      if (edge.type === "source" && edge.to) this.addIncoming(edge.to, edge)
+    }
+    this.outgoing.set(node.path, edges)
+  }
+
+  private pageSourceAt(targetPath: string) {
+    const route = resolveRoute(this.repository, targetPath)
+    return route?.type === "page" ? route.source : undefined
+  }
+
+  private addIncoming(path: string, edge: GraphEdge) {
+    const list = this.incoming.get(path)
+    if (list) list.push(edge)
+    else this.incoming.set(path, [edge])
+  }
+
+  private async resolveLink(
+    source: SourceNode,
+    href: string,
+  ): Promise<LinkResolution> {
+    if (isExternalHref(href)) return { type: "external" }
+
+    if (href.startsWith("~/")) return { type: "actor-relative" }
+
+    const { pathname, query, fragment } = splitLinkTarget(href)
+
+    if (!pathname && (query !== undefined || fragment !== undefined))
+      return { type: "fragment", targetPath: source.path, fragment }
+    if (!pathname) return { type: "unresolved" }
+
+    const targetPath = pathname.startsWith("/")
+      ? this.repository.environment.path.normalize(pathname)
+      : this.repository.environment.path.join(
+          this.repository.environment.path.dirname(source.path),
+          pathname,
+        )
+    // match URL semantics: directory links resolve to their index source
+    const linkedSource =
+      this.repository.atPath(targetPath) ?? this.pageSourceAt(targetPath)
+    if (linkedSource)
+      return { type: "source", targetPath: linkedSource.path, fragment }
+    if (await this.repository.isFile(targetPath))
+      return { type: "asset", targetPath, fragment }
+    return { type: "unresolved", targetPath, fragment }
+  }
+}

package/host/compiler.ts

@@ -0,0 +1,314 @@
+import type {
+  CompiledPage,
+  CompileContext,
+  PageContext,
+} from "../plugins/types.ts"
+import {
+  deriveSourceMeta,
+  interpolate,
+  type Source,
+  type SourceBlock,
+} from "../source/index.ts"
+import {
+  dependencyKey,
+  DependencyIndex,
+  PageDependencies,
+  type Dependency,
+  type DependencyContext,
+} from "../dependencies.ts"
+import {
+  applyHeadingPermalinks,
+  headingIdsForBlock,
+  preserveTaskInstructions,
+  unwrapBlockFlow,
+  wrapBlockSection,
+  wrapPageArticle,
+} from "../document.ts"
+import { isPageSource, publicPath } from "../server/router.ts"
+import { escapeHtml } from "../util.ts"
+import { elementTagNames } from "../view/syntax.ts"
+import type { Engine } from "./engine.ts"
+import type { PluginHost, CompiledViewPlan } from "./plugin-host.ts"
+
+/** Owns compilation state, page caches, dependencies, and View plans. */
+export class PageCompiler {
+  readonly pageCache = new Map<string, CompiledPage>()
+  private compiling = new Set<string>()
+  private pageBuilds = new WeakMap<Source, Promise<CompiledPage>>()
+  private dependencies = new DependencyIndex()
+  private viewPlans = new Map<string, CompiledViewPlan>()
+  private sourceElementTags = new Map<string, ReadonlySet<string>>()
+
+  constructor(
+    private engine: Engine,
+    private plugins: PluginHost,
+  ) {}
+
+  indexSource(source: Source) {
+    this.sourceElementTags.set(source.path, elementTagNames(source.content))
+  }
+
+  unindexSource(path: string) {
+    this.sourceElementTags.delete(path)
+  }
+
+  dependentsOf(dependency: string | Dependency) {
+    return this.dependencies.dependentsOf(
+      typeof dependency === "string" ? dependency : dependencyKey(dependency),
+    )
+  }
+
+  dependenciesOf(path: string) {
+    return this.dependencies.dependenciesOf(path)
+  }
+
+  planFor(path: string) {
+    return this.viewPlans.get(path)
+  }
+
+  isPage(source: Source) {
+    return isPageSource(source)
+  }
+
+  compilePage(source: Source) {
+    return this.engine.perf.measureAsync(
+      "compile.page",
+      async () => {
+        const dependencies = new PageDependencies()
+        let html = wrapPageArticle(
+          source,
+          await this.compilePageBody(source, dependencies),
+        )
+        const { assets, addAsset, addBrowserBundle } =
+          this.engine.documentAssetBag()
+        const ctx: PageContext = {
+          source,
+          addAsset,
+          addBrowserBundle,
+          depend: dependencies.depend,
+        }
+        html = await this.plugins.page(html, ctx, this.engine)
+        const compiled = await this.plugins.compileViewComponents(
+          html,
+          source,
+          ctx,
+          this.engine,
+        )
+        html = compiled.html
+        this.dependencies.replaceDependent(source.path, dependencies.keys)
+        if (compiled.plan) {
+          this.viewPlans.set(source.path, compiled.plan)
+        } else {
+          this.viewPlans.delete(source.path)
+        }
+        return Object.freeze({
+          html,
+          assets: Object.freeze([...assets]),
+          sourceId: source.id,
+          route: publicPath(source),
+        })
+      },
+      source.path,
+    )
+  }
+
+  /** Return the cached page, compiling it once on the first request. */
+  ensurePage(source: Source) {
+    const cached = this.pageCache.get(source.path)
+    if (cached) return Promise.resolve(cached)
+    return this.cachePage(source)
+  }
+
+  private cachePage(source: Source) {
+    const existing = this.pageBuilds.get(source)
+    if (existing) return existing
+
+    const promise = this.compilePage(source)
+      .then((page) => {
+        if (this.engine.repo.atPath(source.path) === source) {
+          this.pageCache.set(source.path, page)
+        }
+        return page
+      })
+      .finally(() => this.pageBuilds.delete(source))
+    this.pageBuilds.set(source, promise)
+    return promise
+  }
+
+  private async compilePageBody(
+    source: Source,
+    dependencies: DependencyContext,
+  ) {
+    const parts: string[] = []
+    if (source.blocks.length) {
+      for (const block of source.blocks) {
+        parts.push(await this.compileBlock(source, block, dependencies))
+      }
+    } else {
+      parts.push(
+        wrapBlockSection(
+          {
+            id: "block-0",
+            path: source.path + "#block-0",
+            type: "block",
+          },
+          await this.compileContent(
+            source,
+            source.body,
+            source.data,
+            undefined,
+            dependencies,
+          ),
+        ),
+      )
+    }
+    return parts.filter(Boolean).join("\n")
+  }
+
+  /** Compile a Source as a document without recursively applying its caller. */
+  async compileDocument(
+    source: Source,
+    dependencies: DependencyContext,
+    excludedPluginId: string,
+  ) {
+    const parts: string[] = []
+    if (source.blocks.length) {
+      for (const block of source.blocks) {
+        parts.push(await this.compileBlockContent(source, block, dependencies))
+      }
+    } else {
+      parts.push(
+        await this.compileContent(
+          source,
+          source.body,
+          source.data,
+          undefined,
+          dependencies,
+        ),
+      )
+    }
+    let html = parts.filter(Boolean).join("\n")
+    const { assets, addAsset, addBrowserBundle } =
+      this.engine.documentAssetBag()
+    const ctx: PageContext = {
+      source,
+      addAsset,
+      addBrowserBundle,
+      depend: dependencies.depend.bind(dependencies),
+    }
+    html = await this.plugins.page(html, ctx, this.engine, excludedPluginId)
+    return { html, assets }
+  }
+
+  async compileBlock(
+    source: Source,
+    block: SourceBlock,
+    dependencies: DependencyContext = new PageDependencies(),
+    options?: { id?: string },
+    scope: CompileContext["scope"] = "page",
+  ) {
+    if (this.compiling.has(block.path)) {
+      return "<p>[Circular Include: " + escapeHtml(block.path) + "]</p>"
+    }
+    this.compiling.add(block.path)
+    try {
+      const inner = await this.compileBlockContent(
+        source,
+        block,
+        dependencies,
+        scope,
+      )
+      const flow = unwrapBlockFlow(inner)
+      if (flow !== undefined) return flow
+      return wrapBlockSection(block, inner, options)
+    } finally {
+      this.compiling.delete(block.path)
+    }
+  }
+
+  private compileBlockContent(
+    source: Source,
+    block: SourceBlock,
+    dependencies: DependencyContext,
+    scope: CompileContext["scope"] = "page",
+  ) {
+    return this.compileContent(
+      source,
+      block.body,
+      { ...source.data, ...block.data },
+      block,
+      dependencies,
+      scope,
+    )
+  }
+
+  /** Compile-time interpolation context: authored data plus source fields. */
+  private interpolationContext(source: Source, data: Source["data"]) {
+    return {
+      ...data,
+      source: {
+        id: source.id,
+        path: source.path,
+        name: source.name,
+        type: source.type,
+        href: publicPath(source),
+        ...deriveSourceMeta(source),
+      },
+    }
+  }
+
+  private async compileContent(
+    source: Source,
+    markdown: string,
+    data: Source["data"],
+    block?: SourceBlock,
+    dependencies: DependencyContext = new PageDependencies(),
+    scope: CompileContext["scope"] = "page",
+  ) {
+    const ctx: CompileContext = {
+      source,
+      block,
+      scope,
+      slots: [],
+      elements: this.sourceElementTags.get(source.path) ?? new Set(),
+      depend: dependencies.depend,
+    }
+    let content = this.plugins.compileVariables(markdown, ctx)
+    content = interpolate(content, this.interpolationContext(source, data))
+    content = await this.plugins.precompile(content, ctx, this.engine)
+    let html = this.engine.environment.renderMarkdown(content)
+    html = preserveTaskInstructions(html)
+    if (block) {
+      const headings = headingIdsForBlock(source, block)
+      html = applyHeadingPermalinks(html, headings, block.headings)
+    }
+    return this.plugins.postcompile(html, ctx, this.engine)
+  }
+
+  buildCache() {
+    return this.engine.perf.measureAsync("cache.build", async () => {
+      for (const source of this.engine.repo.all()) {
+        if (this.isPage(source)) await this.ensurePage(source)
+      }
+    })
+  }
+
+  async recompilePages(paths: Iterable<string>) {
+    const seen = new Set<string>()
+    for (const pagePath of paths) {
+      if (seen.has(pagePath)) continue
+      seen.add(pagePath)
+      const page =
+        this.engine.repo.atPath(pagePath) ?? this.engine.repo.get(pagePath)
+      if (!page || !this.isPage(page)) continue
+      if (!this.pageCache.has(page.path)) continue
+      await this.cachePage(page)
+    }
+  }
+
+  forgetCompiled(path: string) {
+    this.pageCache.delete(path)
+    this.viewPlans.delete(path)
+    this.dependencies.deleteDependent(path)
+  }
+}