@brett_lamy/docstream 0.4.0 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brett_lamy/docstream",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "GitBook-aware readonly markdown and AI stream renderer.",
   "type": "module",
   "scripts": {
@@ -65,14 +65,14 @@
     "./styles.css": "./src/styles.css"
   },
   "dependencies": {
+    "@brett_lamy/viz-engine": ">=0.2.0",
     "hast-util-to-html": "^9.0.5",
     "lowlight": "^3.3.0",
     "lucide-react": "^1.17.0",
     "mermaid": "^11.15.0",
     "rrweb": "2.0.0-alpha.20",
     "streamdown": "^2.5.0",
-    "yaml": "^2.9.0",
-    "@brett_lamy/viz-engine": "^0.2.0"
+    "yaml": "^2.9.0"
   },
   "peerDependencies": {
     "@agent-wasm/core": ">=0.4.0",

package/src/docs/DocsRenderer.tsx

@@ -16,6 +16,7 @@ import { VideoEmbed } from "../video"
 import { OpenApiOperation } from "../openapi/OpenApiOperation"
 import { Mermaid } from "./Mermaid"
 import { HighlightedCode } from "./HighlightedCode"
+import { CitationSources, InlineReference } from "./reference"
 
 export interface LivePreviewProps {
   files: Record<string, string>
@@ -31,6 +32,7 @@ function InlineText({ nodes }: { nodes: Inline[] }) {
   return (
     <>
       {nodes.map((n, i) => {
+        if (n.type === "reference") return <InlineReference key={i} node={n} />
         if (n.type === "image") {
           const img = (
             <img
@@ -359,9 +361,11 @@ export function MarkdownContent({
   sourceRenderer,
   className,
 }: { markdown: string; className?: string } & Renderers) {
+  const doc = parseMarkdown(markdown)
   return (
     <div data-docstream="" className={className}>
-      <Blocks blocks={parseMarkdown(markdown).children} liveRenderer={liveRenderer} sourceRenderer={sourceRenderer} />
+      <Blocks blocks={doc.children} liveRenderer={liveRenderer} sourceRenderer={sourceRenderer} />
+      <CitationSources citations={doc.citations} />
     </div>
   )
 }
@@ -370,6 +374,7 @@ export function DocsRenderer({ doc, liveRenderer, sourceRenderer }: { doc: Docum
   return (
     <article className="docs-article">
       <Blocks blocks={doc.children} liveRenderer={liveRenderer} sourceRenderer={sourceRenderer} />
+      <CitationSources citations={doc.citations} />
     </article>
   )
 }

package/src/docs/reference.tsx

@@ -0,0 +1,95 @@
+import { createContext, useContext, useState, type ReactNode } from "react"
+import { AtSign, Globe, Hash } from "lucide-react"
+
+import type { CitationDef, ReferenceNode } from "../gitbook/ast"
+
+export interface DocstreamRefContextValue {
+  /** Called when a mention/tag chip (or a citation, if it has no url) is clicked. */
+  onReferenceClick?: (ref: ReferenceNode) => void
+  /** Full override for chip rendering; return null to fall back to the default. */
+  renderReference?: (ref: ReferenceNode) => ReactNode | null
+}
+
+export const DocstreamRefContext = createContext<DocstreamRefContextValue>({})
+
+export function InlineReference({ node }: { node: ReferenceNode }) {
+  const { onReferenceClick, renderReference } = useContext(DocstreamRefContext)
+  const custom = renderReference?.(node)
+  if (custom != null) return <>{custom}</>
+
+  if (node.kind === "citation") {
+    // Citations default to linking their source; hosts can still observe clicks.
+    if (node.url) {
+      return (
+        <a
+          className="docs-cite"
+          href={node.url}
+          target="_blank"
+          rel="noreferrer"
+          title={node.label ?? node.url}
+          onClick={() => onReferenceClick?.(node)}
+        >
+          {node.id}
+        </a>
+      )
+    }
+    // Definition missing (not yet streamed, or standalone parseInline call).
+    return (
+      <span className="docs-cite" onClick={() => onReferenceClick?.(node)}>
+        {node.id}
+      </span>
+    )
+  }
+
+  const Icon = node.kind === "mention" ? AtSign : Hash
+  return (
+    <button
+      type="button"
+      className={`docs-ref docs-ref-${node.kind}`}
+      onClick={() => onReferenceClick?.(node)}
+    >
+      <Icon className="docs-ref-icon" />
+      {node.id}
+    </button>
+  )
+}
+
+function hostnameOf(url: string): string | null {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "")
+  } catch {
+    return null
+  }
+}
+
+function SourceChip({ def }: { def: CitationDef }) {
+  const [faviconFailed, setFaviconFailed] = useState(false)
+  const host = hostnameOf(def.url)
+  return (
+    <a className="docs-source-chip" href={def.url} target="_blank" rel="noreferrer" title={def.label ?? def.url}>
+      {host && !faviconFailed ? (
+        <img
+          className="docs-source-favicon"
+          src={`https://www.google.com/s2/favicons?sz=32&domain=${host}`}
+          alt=""
+          onError={() => setFaviconFailed(true)}
+        />
+      ) : (
+        <Globe className="docs-source-favicon" />
+      )}
+      {host ?? def.url}
+    </a>
+  )
+}
+
+/** Row of source-domain pills for a document's citation definitions. */
+export function CitationSources({ citations }: { citations?: CitationDef[] }) {
+  if (!citations?.length) return null
+  return (
+    <div className="docs-sources">
+      {citations.map((def) => (
+        <SourceChip key={def.id} def={def} />
+      ))}
+    </div>
+  )
+}

package/src/gitbook/ast.ts

@@ -23,7 +23,29 @@ export interface InlineImageNode {
   link?: string
 }
 
-export type Inline = TextNode | InlineImageNode
+export type ReferenceKind = "mention" | "tag" | "citation"
+
+/**
+ * Inline reference chip: `@mention`, `#tag`, or footnote-style citation `[^id]`.
+ * `id` carries no sigil. Citations resolve `url`/`label` from their
+ * `[^id]: url "Label"` definition at parse time when one exists.
+ */
+export interface ReferenceNode {
+  type: "reference"
+  kind: ReferenceKind
+  id: string
+  url?: string
+  label?: string
+}
+
+/** A footnote citation definition (`[^id]: url "Label"`), collected doc-level. */
+export interface CitationDef {
+  id: string
+  url: string
+  label?: string
+}
+
+export type Inline = TextNode | InlineImageNode | ReferenceNode
 
 export interface ParagraphNode {
   type: "paragraph"
@@ -214,6 +236,8 @@ export type Block =
 export interface DocumentNode {
   type: "doc"
   children: Block[]
+  /** Footnote citation definitions, in definition order. */
+  citations?: CitationDef[]
 }
 
 export const text = (t: string, marks: Partial<Omit<TextNode, "type" | "text">> = {}): TextNode => ({

package/src/gitbook/index.ts

@@ -1,6 +1,6 @@
 // Pure GitBook markdown engine (no React, no mermaid/streamdown). Safe to import
 // in non-DOM environments such as a server-side AI agent or a Cloudflare Worker.
 export type * from "./ast"
-export { parseMarkdown, parseBlocks } from "./parse"
+export { parseMarkdown, parseBlocks, trimPartialInlineToken } from "./parse"
 export { serializeBlocks, serializeMarkdown } from "./serialize"
-export { parseInline, plainText, refDefinitions, serializeInline } from "./inline"
+export { footnoteDefinitions, parseInline, plainText, refDefinitions, serializeInline, serializeReference } from "./inline"

package/src/gitbook/inline.ts

@@ -1,4 +1,4 @@
-import type { Inline, InlineImageNode, TextNode } from "./ast"
+import type { Inline, InlineImageNode, ReferenceNode, TextNode } from "./ast"
 
 type Marks = Partial<Omit<TextNode, "type" | "text">>
 
@@ -6,6 +6,17 @@ type Marks = Partial<Omit<TextNode, "type" | "text">>
 // and consumed here for [text][ref] / [text][] forms.
 export const refDefinitions = new Map<string, string>()
 
+// Footnote citation definitions ([^id]: url "Label"), populated by
+// parseMarkdown and consumed here to resolve [^id] markers.
+export const footnoteDefinitions = new Map<string, { url: string; label?: string }>()
+
+// @mention / #tag body: letter/underscore start, then word chars, dots, dashes.
+const REFERENCE_BODY_RE = /^[@#]([A-Za-z_][\w.-]*)/
+// Chips are only recognized at start-of-input or after whitespace/open brackets,
+// so brett@replay.io and C# stay plain text.
+const isReferenceBoundary = (prev: string | undefined) =>
+  prev === undefined || /[\s([{]/.test(prev)
+
 function imgAttrs(attrStr: string): Omit<InlineImageNode, "type" | "link"> {
   const attr = (name: string) => attrStr.match(new RegExp(`${name}="([^"]*)"`, "i"))?.[1]
   const out: Omit<InlineImageNode, "type" | "link"> = { src: attr("src") ?? "" }
@@ -110,6 +121,30 @@ export function parseInline(src: string, marks: Marks = {}): Inline[] {
     }
     if (matched) continue
 
+    // [^id] — footnote citation marker (must precede [text](url) / [text][ref])
+    const cite = rest.match(/^\[\^([^\]\s]+)\]/)
+    if (cite) {
+      flush()
+      const def = footnoteDefinitions.get(cite[1])
+      out.push({ type: "reference", kind: "citation", id: cite[1], ...def })
+      i += cite[0].length
+      continue
+    }
+
+    // @mention / #tag chips, only at a word boundary
+    if ((rest[0] === "@" || rest[0] === "#") && isReferenceBoundary(src[i - 1])) {
+      const m = rest.match(REFERENCE_BODY_RE)
+      if (m) {
+        const id = m[1].replace(/[.-]+$/, "")
+        if (id) {
+          flush()
+          out.push({ type: "reference", kind: rest[0] === "@" ? "mention" : "tag", id })
+          i += 1 + id.length
+          continue
+        }
+      }
+    }
+
     // [text](url)
     if (rest[0] === "[") {
       const m = rest.match(/^\[([^\]]*)\]\(([^)\s]+)\)/)
@@ -162,7 +197,18 @@ export function parseInline(src: string, marks: Marks = {}): Inline[] {
   return out
 }
 
-const escapeText = (t: string) => t.replace(/([*_~`[\]\\])/g, "\\$1")
+const escapeText = (t: string) =>
+  t
+    .replace(/([*_~`[\]\\])/g, "\\$1")
+    // Escape @/# only at a chip boundary so emails/C# survive untouched.
+    // Note: node-local — a boundary formed across adjacent inline nodes
+    // (previous node ending in whitespace) is not caught; rare, accepted.
+    .replace(/(^|[\s([{])([@#])(?=[A-Za-z_])/g, "$1\\$2")
+
+export function serializeReference(n: ReferenceNode): string {
+  if (n.kind === "citation") return `[^${n.id}]`
+  return `${n.kind === "mention" ? "@" : "#"}${n.id}`
+}
 
 function serializeImage(n: InlineImageNode): string {
   const attrs = [
@@ -182,6 +228,7 @@ export function serializeInline(nodes: Inline[]): string {
   return nodes
     .map((n) => {
       if (n.type === "image") return serializeImage(n)
+      if (n.type === "reference") return serializeReference(n)
       // bare autolink: text identical to the URL, no other marks
       if (n.link && n.text === n.link && !n.bold && !n.italic && !n.strike && !n.code) {
         return n.link
@@ -198,5 +245,14 @@ export function serializeInline(nodes: Inline[]): string {
 }
 
 export function plainText(nodes: Inline[]): string {
-  return nodes.map((n) => (n.type === "text" ? n.text : (n.alt ?? ""))).join("")
+  return nodes
+    .map((n) => {
+      if (n.type === "text") return n.text
+      if (n.type === "reference") {
+        // Citation superscripts are visual noise in plain text (titles etc.).
+        return n.kind === "citation" ? "" : serializeReference(n)
+      }
+      return n.alt ?? ""
+    })
+    .join("")
 }

package/src/gitbook/parse.ts

@@ -9,7 +9,7 @@ import type {
   TabNode,
   UpdateNode,
 } from "./ast"
-import { parseInline, plainText, refDefinitions } from "./inline"
+import { footnoteDefinitions, parseInline, plainText, refDefinitions } from "./inline"
 
 // Minimal HTML-inline → markdown-inline bridge for HTML table cells.
 function htmlToInlineMd(html: string): string {
@@ -119,13 +119,46 @@ const HINT_STYLES: HintStyle[] = ["info", "success", "warning", "danger"]
 export function parseMarkdown(src: string): DocumentNode {
   const lines = src.split(/\r?\n/)
   refDefinitions.clear()
+  footnoteDefinitions.clear()
   const content: string[] = []
+  let inFence = false
   for (const line of lines) {
-    const def = line.match(/^\[([^\]]+)\]:\s*(\S+)\s*$/)
-    if (def) refDefinitions.set(def[1].toLowerCase(), def[2])
-    else content.push(line)
+    // Definition lines inside code fences are content, not definitions.
+    if (/^\s*(?:```|~~~)/.test(line)) inFence = !inFence
+    if (!inFence) {
+      // Footnote citation definitions first: [^id]: url "Optional Label"
+      const foot = line.match(/^\[\^([^\]\s]+)\]:\s*(\S+)(?:\s+"([^"]*)")?\s*$/)
+      if (foot) {
+        footnoteDefinitions.set(foot[1], { url: foot[2], ...(foot[3] ? { label: foot[3] } : {}) })
+        continue
+      }
+      const def = line.match(/^\[([^\]]+)\]:\s*(\S+)\s*$/)
+      // Keys starting with ^ are reserved for footnotes and must never
+      // shadow reference-style link definitions.
+      if (def && !def[1].startsWith("^")) {
+        refDefinitions.set(def[1].toLowerCase(), def[2])
+        continue
+      }
+    }
+    content.push(line)
+  }
+  const doc: DocumentNode = { type: "doc", children: parseBlocks(content) }
+  if (footnoteDefinitions.size) {
+    doc.citations = [...footnoteDefinitions.entries()].map(([id, d]) => ({ id, ...d }))
   }
-  return { type: "doc", children: parseBlocks(content) }
+  return doc
+}
+
+// Strips an in-progress inline token (`[^1`, `@bre`, a lone `#`, a trailing
+// `\`) from the end of a partially streamed document so chips don't flash as
+// plain text mid-stream. No-op inside an unterminated code fence.
+export function trimPartialInlineToken(md: string): string {
+  const fences = md.match(/^\s*(?:```|~~~)/gm)
+  if (fences && fences.length % 2 === 1) return md
+  return md
+    .replace(/\[\^[^\]]*$/, "")
+    .replace(/(^|[\s([{])[@#][\w.-]*$/, "$1")
+    .replace(/\\$/, "")
 }
 
 export function parseBlocks(lines: string[]): Block[] {

package/src/gitbook/serialize.ts

@@ -1,8 +1,15 @@
 import type { Block, DocumentNode, Inline, ListItemNode } from "./ast"
-import { serializeInline } from "./inline"
+import { serializeInline, serializeReference } from "./inline"
 
 export function serializeMarkdown(doc: DocumentNode): string {
-  return serializeBlocks(doc.children).trimEnd() + "\n"
+  let out = serializeBlocks(doc.children).trimEnd()
+  if (doc.citations?.length) {
+    const defs = doc.citations
+      .map((c) => `[^${c.id}]: ${c.url}${c.label ? ` "${c.label}"` : ""}`)
+      .join("\n")
+    out = out ? `${out}\n\n${defs}` : defs
+  }
+  return out + "\n"
 }
 
 export function serializeBlocks(blocks: Block[]): string {
@@ -156,6 +163,9 @@ function inlineToHtml(nodes: Inline[]): string {
         const img = `<img src="${n.src}"${n.alt ? ` alt="${n.alt}"` : ""}>`
         return n.link ? `<a href="${n.link}">${img}</a>` : img
       }
+      // HTML card tables round-trip through htmlToInlineMd, which strips
+      // tags — plain markers are the only representation that survives.
+      if (n.type === "reference") return serializeReference(n)
       let s = n.text
       if (n.code) s = `<code>${s}</code>`
       if (n.bold) s = `<strong>${s}</strong>`

package/src/index.ts

@@ -2,6 +2,8 @@ export { PlaygroundStreamdown as GitbookStreamdown } from "./playground/Playgrou
 export type { PlaygroundStreamdownProps as GitbookStreamdownProps } from "./playground/PlaygroundStreamdown"
 export { DocsRenderer, MarkdownContent } from "./docs/DocsRenderer"
 export type { LivePreviewProps, LivePreviewRenderer, SourceReferenceRenderer } from "./docs/DocsRenderer"
+export { CitationSources, DocstreamRefContext, InlineReference } from "./docs/reference"
+export type { DocstreamRefContextValue } from "./docs/reference"
 export { ReplayEmbed, ReplayPreview } from "./replay"
 export type {
   ReplayEventsSource,
@@ -42,9 +44,16 @@ export type {
   SourceProvenance,
   SourceReferenceClient,
 } from "./source"
-export { parseMarkdown, parseBlocks } from "./gitbook/parse"
+export { parseMarkdown, parseBlocks, trimPartialInlineToken } from "./gitbook/parse"
 export { serializeBlocks, serializeMarkdown } from "./gitbook/serialize"
-export { parseInline, plainText, refDefinitions, serializeInline } from "./gitbook/inline"
+export {
+  footnoteDefinitions,
+  parseInline,
+  plainText,
+  refDefinitions,
+  serializeInline,
+  serializeReference,
+} from "./gitbook/inline"
 export { Streamdown } from "streamdown"
 export type { StreamdownProps } from "streamdown"
 export type * from "./gitbook/ast"

package/src/playground/PlaygroundStreamdown.tsx

@@ -1,7 +1,7 @@
 import { useMemo } from "react"
 
 import { DocsRenderer, type LivePreviewRenderer } from "../docs/DocsRenderer"
-import { parseMarkdown } from "../gitbook/parse"
+import { parseMarkdown, trimPartialInlineToken } from "../gitbook/parse"
 import { ReactCodePreview } from "./ReactCodePreview"
 
 export interface PlaygroundStreamdownProps {
@@ -25,7 +25,10 @@ export function PlaygroundStreamdown({
   markdown,
 }: PlaygroundStreamdownProps) {
   const content = markdown ?? children ?? ""
-  const doc = useMemo(() => parseMarkdown(content), [content])
+  const doc = useMemo(
+    () => parseMarkdown(isStreaming ? trimPartialInlineToken(content) : content),
+    [content, isStreaming]
+  )
   const liveRenderer: LivePreviewRenderer = ({ files, entry, title }) => (
     <ReactCodePreview files={files} entry={entry} title={title} />
   )

package/src/streamdown.tsx

@@ -1,6 +1,6 @@
 import { useMemo } from "react"
 import { DocsRenderer } from "./docs/DocsRenderer"
-import { parseMarkdown } from "./gitbook/parse"
+import { parseMarkdown, trimPartialInlineToken } from "./gitbook/parse"
 
 export interface GitbookStreamdownProps {
   children?: string
@@ -18,7 +18,10 @@ export function GitbookStreamdown({
   markdown,
 }: GitbookStreamdownProps) {
   const content = markdown ?? children ?? ""
-  const gitbookDoc = useMemo(() => parseMarkdown(content), [content])
+  const gitbookDoc = useMemo(
+    () => parseMarkdown(isStreaming ? trimPartialInlineToken(content) : content),
+    [content, isStreaming]
+  )
 
   return (
     <div

package/src/styles.css

@@ -953,3 +953,94 @@
   outline: 2px solid var(--docs-react-demo-primary);
   outline-offset: 2px;
 }
+
+/* ---- Inline reference chips (@mention, #tag) and citations ---- */
+
+.docs-ref {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  padding: 1px 8px;
+  border-radius: 999px;
+  font-family: inherit;
+  font-size: 0.85em;
+  font-weight: 500;
+  line-height: 1.5;
+  vertical-align: baseline;
+  cursor: pointer;
+}
+
+.docs-ref-icon {
+  width: 12px;
+  height: 12px;
+}
+
+.docs-ref-mention {
+  border: 1px solid rgba(59, 130, 246, 0.45);
+  background: rgba(59, 130, 246, 0.1);
+  color: var(--gb-info);
+}
+
+.docs-ref-mention:hover {
+  background: rgba(59, 130, 246, 0.18);
+}
+
+.docs-ref-tag {
+  border: 1px solid var(--gb-border);
+  background: var(--gb-muted);
+  color: var(--gb-muted-foreground);
+}
+
+.docs-ref-tag:hover {
+  background: var(--gb-accent);
+  color: var(--gb-accent-foreground);
+}
+
+.docs-cite {
+  display: inline-block;
+  padding: 1px 5px;
+  border-radius: 5px;
+  background: rgba(59, 130, 246, 0.14);
+  color: var(--gb-info) !important;
+  font-size: 10px;
+  font-weight: 700;
+  line-height: 1.4;
+  vertical-align: super;
+  text-decoration: none !important;
+  cursor: pointer;
+}
+
+.docs-cite:hover {
+  background: rgba(59, 130, 246, 0.22);
+}
+
+.docs-sources {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  margin-top: 16px;
+}
+
+.docs-source-chip {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  padding: 4px 12px;
+  border: 1px solid var(--gb-border);
+  border-radius: 999px;
+  font-size: 12.5px;
+  font-weight: 500;
+  color: var(--gb-muted-foreground) !important;
+  text-decoration: none !important;
+}
+
+.docs-source-chip:hover {
+  background: var(--gb-accent);
+  color: var(--gb-accent-foreground) !important;
+}
+
+.docs-source-favicon {
+  width: 14px;
+  height: 14px;
+  border-radius: 4px;
+}