@pathmx/core 0.5.0

package/source/markdown/lists.ts

@@ -0,0 +1,223 @@
+import { isFenceClose, matchFenceOpen } from "./fences.ts"
+import { withoutHtmlComments, type HtmlCommentState } from "./comments.ts"
+
+const LIST_ITEM = /^( *)([-+*]|(\d+)[.)])[ \t]+(.+?)[ \t]*$/
+const TASK_ITEM = /^\[([ xX])\](?:[ \t]+(.*))?$/
+const TASK_MARK =
+  /(?:^|[ \t]+)\{#([A-Za-z][\w-]*)((?:[ \t]+\.[A-Za-z][\w-]*)*)\}$/
+const ITEM_LINK =
+  /^\[([^\]\r\n]+)\]\(\s*(?:<([^>\r\n]+)>|([^\s)]+))(?:[ \t]+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)$/
+
+export type MarkdownList = Readonly<{
+  type: "ordered" | "unordered"
+  start?: number
+  items: readonly MarkdownListItem[]
+}>
+
+export type MarkdownListItem = Readonly<{
+  line: number
+  text: string
+  children: readonly MarkdownList[]
+}>
+
+export type MarkdownListLink = Readonly<{
+  label: string
+  href: string
+}>
+
+export type MarkdownTask = Readonly<{
+  line: number
+  text: string
+  checked: boolean
+  id?: string
+  tags: readonly string[]
+}>
+
+export type MarkdownListParse = Readonly<{
+  lists: readonly MarkdownList[]
+  tasks: readonly MarkdownTask[]
+  other: readonly Readonly<{ line: number; text: string }>[]
+}>
+
+type MutableList = {
+  type: MarkdownList["type"]
+  start?: number
+  items: MutableListItem[]
+}
+
+type MutableListItem = {
+  line: number
+  text: string
+  children: MutableList[]
+}
+
+const EMPTY_CHILDREN = Object.freeze([]) as unknown as MutableList[]
+
+function couldBeListItem(line: string) {
+  let index = 0
+  while (line.charCodeAt(index) === 32) index++
+  const marker = line.charCodeAt(index)
+  if (marker === 45 || marker === 43 || marker === 42) {
+    const separator = line.charCodeAt(index + 1)
+    return separator === 32 || separator === 9
+  }
+  if (marker < 48 || marker > 57) return false
+  while (line.charCodeAt(index) >= 48 && line.charCodeAt(index) <= 57) index++
+  const terminator = line.charCodeAt(index)
+  if (terminator !== 46 && terminator !== 41) return false
+  const separator = line.charCodeAt(index + 1)
+  return separator === 32 || separator === 9
+}
+
+function freezeLists(lists: MutableList[]): readonly MarkdownList[] {
+  for (const list of lists) {
+    for (const item of list.items) {
+      if (item.children !== EMPTY_CHILDREN) freezeLists(item.children)
+      Object.freeze(item)
+    }
+    Object.freeze(list.items)
+    Object.freeze(list)
+  }
+  return Object.freeze(lists)
+}
+
+function markdownTask(item: MarkdownListItem): MarkdownTask | undefined {
+  const task = item.text.match(TASK_ITEM)
+  if (!task) return
+  const label = task[2] ?? ""
+  const mark = label.match(TASK_MARK)
+  const tags = mark?.[2]
+    ? mark[2]
+        .trim()
+        .split(/[ \t]+/)
+        .map((tag) => tag.slice(1))
+    : []
+  return Object.freeze({
+    line: item.line,
+    text: mark ? label.slice(0, mark.index).trimEnd() : label,
+    checked: task[1]!.toLowerCase() === "x",
+    ...(mark ? { id: mark[1]! } : {}),
+    tags: Object.freeze(tags),
+  })
+}
+
+function tasksFromLists(lists: readonly MarkdownList[]) {
+  const tasks: MarkdownTask[] = []
+  const collect = (nested: readonly MarkdownList[]) => {
+    for (const list of nested) {
+      for (const item of list.items) {
+        const task = markdownTask(item)
+        if (task) tasks.push(task)
+        collect(item.children)
+      }
+    }
+  }
+  collect(lists)
+  return Object.freeze(tasks)
+}
+
+/** Incrementally build nested Markdown lists from already-visible lines. */
+export class MarkdownListCollector {
+  readonly lists: MutableList[] = []
+  readonly other: Array<{ line: number; text: string }> = []
+  private levels: Array<{ indent: number; list: MutableList }> = []
+  private finished?: readonly MarkdownList[]
+
+  constructor(private collectOther = false) {}
+
+  separate() {
+    this.levels.length = 0
+  }
+
+  add(line: string, lineNumber: number) {
+    if (!line.trim()) return
+    const match = couldBeListItem(line) ? line.match(LIST_ITEM) : null
+    if (!match) {
+      this.separate()
+      if (this.collectOther) this.other.push({ line: lineNumber, text: line })
+      return
+    }
+
+    const indent = match[1]!.length
+    const type = match[3] ? "ordered" : "unordered"
+    const start = match[3] ? Number(match[3]) : undefined
+    while (this.levels.length && indent < this.levels.at(-1)!.indent) {
+      this.levels.pop()
+    }
+
+    let list: MutableList
+    const current = this.levels.at(-1)
+    if (current?.indent === indent && current.list.type === type) {
+      list = current.list
+    } else {
+      if (current?.indent === indent) this.levels.pop()
+      const parent = this.levels.at(-1)?.list.items.at(-1)
+      list = {
+        type,
+        ...(start !== undefined ? { start } : {}),
+        items: [],
+      }
+      if (parent) {
+        if (parent.children === EMPTY_CHILDREN) parent.children = []
+        parent.children.push(list)
+      } else this.lists.push(list)
+      this.levels.push({ indent, list })
+    }
+
+    list.items.push({
+      line: lineNumber,
+      text: match[4]!,
+      children: EMPTY_CHILDREN,
+    })
+  }
+
+  finish() {
+    return (this.finished ??= freezeLists(this.lists))
+  }
+}
+
+/** Parse list structure without building a complete Markdown AST. */
+export function parseMarkdownLists(markdown: string): MarkdownListParse {
+  const collector = new MarkdownListCollector(true)
+  const hasHtmlComments = markdown.includes("<!--")
+  let marker: string | undefined
+  let inHtmlComment = false
+  for (const [index, authoredLine] of markdown.split(/\r?\n/).entries()) {
+    if (marker) {
+      if (isFenceClose(authoredLine, marker)) marker = undefined
+      continue
+    }
+    const visible: HtmlCommentState | undefined =
+      hasHtmlComments || inHtmlComment
+        ? withoutHtmlComments(authoredLine, inHtmlComment)
+        : undefined
+    if (visible) inHtmlComment = visible.inComment
+    const visibleLine = visible?.line ?? authoredLine
+    const fence = matchFenceOpen(visibleLine)
+    if (fence) {
+      marker = fence.marker
+      collector.add(visibleLine, index + 1)
+      continue
+    }
+    collector.add(visibleLine, index + 1)
+  }
+  const lists = collector.finish()
+  let tasks: readonly MarkdownTask[] | undefined
+  return {
+    lists,
+    /** Derive the flat projection only when a consumer requests tasks. */
+    get tasks() {
+      return (tasks ??= tasksFromLists(lists))
+    },
+    other: collector.other,
+  }
+}
+
+/** Return the one exact inline Markdown link authored by a list item. */
+export function markdownListItemLink(
+  item: Pick<MarkdownListItem, "text">,
+): MarkdownListLink | undefined {
+  const match = item.text.match(ITEM_LINK)
+  if (!match) return
+  return { label: match[1]!, href: (match[2] ?? match[3] ?? "").trim() }
+}

package/source/markdown/prose.ts

@@ -0,0 +1,74 @@
+import { withoutHtmlComments } from "./comments.ts"
+import { isFenceClose, matchFenceOpen } from "./fences.ts"
+
+const HTML_COMMENT_SOURCE = "<!--[\\s\\S]*?(?:-->|$)"
+const INLINE_CODE_SOURCE = "(`+)[\\s\\S]*?\\1"
+
+function transformOutside(
+  markdown: string,
+  protectedSource: string,
+  transform: (prose: string) => string,
+) {
+  const protectedSyntax = new RegExp(protectedSource, "g")
+  let cursor = 0
+  let output = ""
+  for (const match of markdown.matchAll(protectedSyntax)) {
+    output += transform(markdown.slice(cursor, match.index))
+    output += match[0]
+    cursor = match.index + match[0].length
+  }
+  return output + transform(markdown.slice(cursor))
+}
+
+/** Transform rendered HTML text without touching preserved comments. */
+export function transformOutsideHtmlComments(
+  html: string,
+  transform: (visible: string) => string,
+) {
+  if (!html.includes("<!--")) return transform(html)
+  return transformOutside(html, HTML_COMMENT_SOURCE, transform)
+}
+
+/** Transform visible Markdown prose without touching comments or code. */
+export function transformMarkdownProse(
+  markdown: string,
+  transform: (prose: string) => string,
+) {
+  const lines = markdown.split(/\r?\n/)
+  const output: string[] = []
+  const prose: string[] = []
+  let fence: string | undefined
+  let inHtmlComment = false
+
+  const flush = () => {
+    if (!prose.length) return
+    output.push(
+      transformOutside(
+        prose.join("\n"),
+        `${HTML_COMMENT_SOURCE}|${INLINE_CODE_SOURCE}`,
+        transform,
+      ),
+    )
+    prose.length = 0
+  }
+
+  for (const line of lines) {
+    if (fence) {
+      output.push(line)
+      if (isFenceClose(line, fence)) fence = undefined
+      continue
+    }
+    const visible = withoutHtmlComments(line, inHtmlComment)
+    inHtmlComment = visible.inComment
+    const opening = matchFenceOpen(visible.line)
+    if (opening) {
+      flush()
+      fence = opening.marker
+      output.push(line)
+      continue
+    }
+    prose.push(line)
+  }
+  flush()
+  return output.join("\n")
+}

package/source/markdown/scan.ts

@@ -0,0 +1,79 @@
+import { headingFromLine, type MarkdownHeading } from "./headings.ts"
+import {
+  isFenceClose,
+  matchFenceOpen,
+  type MarkdownCodeFence,
+} from "./fences.ts"
+import { withoutHtmlComments, type HtmlCommentState } from "./comments.ts"
+
+export type MarkdownStructure = Readonly<{
+  /** Ordered, fence-free paragraphs that may contain authored links. */
+  linkFragments: string[]
+  headings: MarkdownHeading[]
+  fences: MarkdownCodeFence[]
+}>
+
+/** Scan fence boundaries once and share that context with structural indexes. */
+export function scanMarkdownStructure(markdown: string): MarkdownStructure {
+  const lines = markdown.split(/\r?\n/)
+  const headings: MarkdownHeading[] = []
+  const fences: MarkdownCodeFence[] = []
+  const linkFragments: string[] = []
+  let linkLines: string[] = []
+  const headingCounts = new Map<string, number>()
+  const hasHtmlComments = markdown.includes("<!--")
+  let inHtmlComment = false
+  let open:
+    { marker: string; lang: string; info: string; body: string[] } | undefined
+  const flushLinkLines = () => {
+    if (linkLines.some((line) => line.includes("[") || /<a\b/i.test(line))) {
+      linkFragments.push(linkLines.join("\n"))
+    }
+    linkLines = []
+  }
+
+  for (let index = 0; index < lines.length; index++) {
+    const line = lines[index] ?? ""
+    if (open) {
+      if (!isFenceClose(line, open.marker)) {
+        open.body.push(line)
+        continue
+      }
+      fences.push({
+        lang: open.lang,
+        info: open.info,
+        body: open.body.join("\n"),
+      })
+      open = undefined
+      continue
+    }
+
+    const visible: HtmlCommentState | undefined =
+      hasHtmlComments || inHtmlComment
+        ? withoutHtmlComments(line, inHtmlComment)
+        : undefined
+    if (visible) inHtmlComment = visible.inComment
+    const visibleLine = visible?.line ?? line
+
+    const fence = matchFenceOpen(visibleLine)
+    if (fence) {
+      flushLinkLines()
+      open = {
+        marker: fence.marker,
+        lang: fence.info.split(/\s+/, 1)[0] ?? "",
+        info: fence.info,
+        body: [],
+      }
+      continue
+    }
+
+    const heading = headingFromLine(visibleLine, headingCounts)
+    if (heading) headings.push(heading)
+
+    if (visibleLine.trim()) linkLines.push(visibleLine)
+    else flushLinkLines()
+  }
+
+  flushLinkLines()
+  return { linkFragments, headings, fences }
+}

package/source/markdown/text.ts

@@ -0,0 +1,82 @@
+import { decodeHtmlEntities } from "../../util.ts"
+import { isFenceClose, matchFenceOpen } from "./fences.ts"
+
+/** Collapse inline markdown to plain text: drop images, keep link labels, strip tags and emphasis. */
+export function cleanInlineText(text: string) {
+  return decodeHtmlEntities(text)
+    .replace(/!\[[^\]\n]*\]\([^)\n]*\)/g, " ")
+    .replace(/\[([^\]\n]+)\]\([^)\n]*\)/g, "$1")
+    .replace(/<[^>]*>/g, " ")
+    .replace(/[`*_~]/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+function isStructuralLine(line: string) {
+  return (
+    line.startsWith("#") ||
+    line.startsWith("<") ||
+    line.startsWith(">") ||
+    line.startsWith("|") ||
+    /^([-*_]\s*){3,}$/.test(line) ||
+    /^[-*+]\s+\S/.test(line) ||
+    /^\d+[.)]\s+\S/.test(line) ||
+    /^!\[[^\]\n]*\]\([^)\n]*\)\s*$/.test(line) ||
+    /^\[[^\]\n]+\]\([^)\n]*\)\s*$/.test(line) ||
+    /^\[[^\]\n]+\]:/.test(line)
+  )
+}
+
+function truncateAtWord(text: string, maxLength: number) {
+  if (text.length <= maxLength) return text
+  const trimmed = text
+    .slice(0, maxLength)
+    .replace(/\s+\S*$/, "")
+    .trim()
+  return `${trimmed || text.slice(0, maxLength).trim()}...`
+}
+
+/**
+ * First prose paragraph of a markdown body: contiguous plain-text lines
+ * outside fences, skipping comments, headings, markup, lists, tables, and
+ * link/image-only lines. Short scraps (under `minLength`) are skipped and
+ * long text is truncated at a word boundary.
+ */
+export function firstParagraph(
+  markdown: string,
+  { minLength = 12, maxLength = 160 } = {},
+) {
+  const lines = markdown.replace(/<!--[\s\S]*?-->/g, "").split(/\r?\n/)
+  let paragraph: string[] = []
+  let marker: string | undefined
+
+  const flush = () => {
+    if (!paragraph.length) return
+    const text = cleanInlineText(paragraph.join(" "))
+    paragraph = []
+    if (text.length >= minLength) return truncateAtWord(text, maxLength)
+  }
+
+  for (const line of lines) {
+    if (marker) {
+      if (isFenceClose(line, marker)) marker = undefined
+      continue
+    }
+    const fence = matchFenceOpen(line)
+    if (fence) {
+      marker = fence.marker
+      const text = flush()
+      if (text) return text
+      continue
+    }
+    const trimmed = line.trim()
+    if (!trimmed || isStructuralLine(trimmed)) {
+      const text = flush()
+      if (text) return text
+      continue
+    }
+    paragraph.push(trimmed)
+  }
+
+  return flush() ?? ""
+}

package/source/meta.ts

@@ -0,0 +1,87 @@
+import { posixPath } from "../path.ts"
+import { interpolateText } from "./interpolate.ts"
+import { parseLinks } from "./markdown/links.ts"
+import { cleanInlineText, firstParagraph } from "./markdown/text.ts"
+import type { Source, SourceNode } from "./source.ts"
+import type { SourceBlock } from "./blocks.ts"
+
+export type SourceMeta = {
+  title: string
+  description: string
+  image: string
+}
+
+export type BlockMeta = SourceMeta
+
+const sourceCache = new WeakMap<Source, SourceMeta>()
+const blockCache = new WeakMap<SourceBlock, BlockMeta>()
+
+/** Resolve a relative image href against the source folder; leave absolute, external, protocol, fragment, and query hrefs alone. */
+function resolveImage(sourcePath: string, href: string) {
+  const trimmed = href.trim()
+  if (
+    !trimmed ||
+    trimmed.startsWith("/") ||
+    trimmed.startsWith("#") ||
+    trimmed.startsWith("?") ||
+    /^(?:\/\/|[a-z][a-z0-9+.-]*:)/i.test(trimmed)
+  ) {
+    return trimmed
+  }
+  return posixPath.resolve(posixPath.dirname(sourcePath), trimmed)
+}
+
+function firstImageHref(node: SourceNode) {
+  // Parse after token fill so `{{ isbn }}` inside a URL is a complete href.
+  const filled = interpolateText(node.body, node.data)
+  return (
+    parseLinks(filled).find((link) => link.kind === "image")?.href ??
+    node.links.find((link) => link.kind === "image")?.href ??
+    ""
+  )
+}
+
+function authored(value: unknown) {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined
+}
+
+function deriveNodeMeta(
+  node: SourceNode,
+  sourcePath: string,
+  fallbackTitle = "",
+): SourceMeta {
+  const headingTitle = cleanInlineText(node.headings[0]?.text ?? "")
+  const fill = (text: string) => interpolateText(text, node.data).trim()
+  const imageHref = fill(authored(node.data.image) ?? firstImageHref(node))
+  return {
+    title: fill(authored(node.data.title) ?? (headingTitle || fallbackTitle)),
+    description: fill(
+      authored(node.data.description) ?? firstParagraph(node.body),
+    ),
+    image: imageHref ? resolveImage(sourcePath, imageHref) : "",
+  }
+}
+
+/**
+ * Source meta with frontmatter overrides and content fallbacks:
+ * title from the first heading, description from the first paragraph,
+ * image from the first image link (resolved against the source folder).
+ */
+export function deriveSourceMeta(source: Source): SourceMeta {
+  const cached = sourceCache.get(source)
+  if (cached) return cached
+
+  const meta = deriveNodeMeta(source, source.path, source.name)
+  if (!meta.title) meta.title = source.name
+  sourceCache.set(source, meta)
+  return meta
+}
+
+/** Block-local metadata using the same derivation rules without inheriting Source data. */
+export function deriveBlockMeta(block: SourceBlock, source: Source): BlockMeta {
+  const cached = blockCache.get(block)
+  if (cached) return cached
+  const meta = deriveNodeMeta(block, source.path)
+  blockCache.set(block, meta)
+  return meta
+}

package/source/source.ts

@@ -0,0 +1,121 @@
+import { posixPath } from "../path.ts"
+import type { PathMXYaml } from "../environment.ts"
+import { createBlock, type SourceBlock } from "./blocks.ts"
+import {
+  assignHeadingIds,
+  splitMarkdownBlocks,
+  type MarkdownCodeFence,
+  type MarkdownHeading,
+  type MarkdownList,
+  type MarkdownTask,
+  type SourceLink,
+} from "./markdown/index.ts"
+import type { Topmatter } from "./topmatter.ts"
+import { splitFrontmatter } from "./topmatter.ts"
+import { parseSourceTypeAndQualifiers } from "./utils.ts"
+
+/** Shared Source / Block document fields. */
+export type SourceNode = {
+  id: string
+  path: string
+  /** Source: path hint or frontmatter. Block: authored topmatter type, or `"block"`. */
+  type: string
+  /** Authored local metadata only. Source `data` is markdown frontmatter; Block `data` is Block topmatter. */
+  data: Topmatter
+  links: SourceLink[]
+  headings: MarkdownHeading[]
+  fences: MarkdownCodeFence[]
+  lists: readonly MarkdownList[]
+  tasks: readonly MarkdownTask[]
+  /** Raw serialization: Source file text, or the Block section including topmatter. */
+  content: string
+  /** Compile input: Source after frontmatter, or Block after topmatter. */
+  body: string
+}
+
+export type Source = SourceNode & {
+  name: string
+  qualifiers: readonly string[]
+  blocks: SourceBlock[]
+}
+
+export const DEFAULT_SOURCE_TYPE = "page"
+
+export function sourceIdFromPath(path: string, type = DEFAULT_SOURCE_TYPE) {
+  if (posixPath.extname(path) !== ".md") return path
+  const id = path.slice(0, -3)
+  const { typeHint } = parseSourceTypeAndQualifiers(path)
+  return typeHint ? id : `${id}.${type}`
+}
+
+export function resolveSourceId(path: string, data: Topmatter) {
+  if (typeof data.id === "string" && data.id.trim()) return data.id.trim()
+  const type =
+    typeof data.type === "string" && data.type.trim()
+      ? data.type.trim()
+      : DEFAULT_SOURCE_TYPE
+  return sourceIdFromPath(path, type)
+}
+
+export function createSource(
+  path: string,
+  content: string,
+  yaml?: PathMXYaml,
+): Source {
+  const { data, body } = splitFrontmatter(content, yaml)
+  const { name, typeHint, qualifiers } = parseSourceTypeAndQualifiers(path)
+  const authoredType =
+    typeof data.type === "string" && data.type.trim()
+      ? data.type.trim()
+      : undefined
+  const type = typeHint ?? authoredType ?? DEFAULT_SOURCE_TYPE
+  const blocks: SourceBlock[] = []
+  return {
+    id: resolveSourceId(path, data),
+    path,
+    name,
+    type,
+    qualifiers,
+    data,
+    links: [],
+    headings: [],
+    fences: [],
+    /** Source-level list access remains lazy across its indexed Blocks. */
+    get lists() {
+      return blocks.flatMap((block) => block.lists)
+    },
+    /** Source-level tasks aggregate the same lazy Block structures. */
+    get tasks() {
+      return blocks.flatMap((block) => block.tasks)
+    },
+    content,
+    body,
+    blocks,
+  }
+}
+
+/** Fill Source indexes from Block parses. Heading ids are unique across the Source. */
+export function rollupSourceIndexes(source: Source) {
+  const links = new Map<string, SourceLink>()
+  const headings: MarkdownHeading[] = []
+  const fences: MarkdownCodeFence[] = []
+  for (const block of source.blocks) {
+    for (const link of block.links) {
+      if (!links.has(link.href)) links.set(link.href, link)
+    }
+    headings.push(...block.headings)
+    fences.push(...block.fences)
+  }
+  source.links = [...links.values()]
+  source.headings = assignHeadingIds(headings)
+  source.fences = fences
+}
+
+/** Rebuild the canonical Block tree and derived Source indexes from authored Markdown. */
+export function indexSourceDocument(source: Source, yaml: PathMXYaml) {
+  const blocks = splitMarkdownBlocks(source.body).map((content, ordinal) =>
+    createBlock(source, ordinal, content, {}, yaml),
+  )
+  source.blocks.splice(0, source.blocks.length, ...blocks)
+  rollupSourceIndexes(source)
+}