@brett_lamy/docstream-editor 0.5.8 → 0.6.0

package/README.md

@@ -10,7 +10,7 @@ TipTap editor components for Docstream GitBook-style markdown documents.
 - TipTap extensions for common writing flows.
 - Conversion helpers between the Docstream AST and TipTap JSON.
 - Slash menu for inserting supported blocks.
-- Code block support through `lowlight`.
+- Code block syntax highlighting through [`gpu-lexer`](https://gpu-lexer.vercel.app/) (language-agnostic, WebGPU; plain text where WebGPU is unavailable).
 - Table, task list, heading, quote, and inline formatting support.
 - Preserves Docstream/GitBook block semantics when converting back to markdown.
 - Preserves mounted source-file/export provenance and can write edits to the real file through Vite.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brett_lamy/docstream-editor",
-  "version": "0.5.8",
+  "version": "0.6.0",
   "description": "TipTap editor for Docstream GitBook-style markdown documents.",
   "type": "module",
   "scripts": {
@@ -34,12 +34,12 @@
   },
   "dependencies": {
     "@brett_lamy/docstream": "0.5.7",
-    "lowlight": "^3.3.0",
+    "gpu-lexer": "0.0.2",
     "lucide-react": "^1.17.0"
   },
   "peerDependencies": {
     "@tiptap/core": "^3.26.1",
-    "@tiptap/extension-code-block-lowlight": "^3.26.1",
+    "@tiptap/extension-code-block": "^3.26.1",
     "@tiptap/extension-list": "^3.26.1",
     "@tiptap/extension-placeholder": "^3.26.1",
     "@tiptap/extension-table": "^3.26.1",
@@ -56,7 +56,7 @@
   "devDependencies": {
     "@agent-wasm/core": "^0.4.0",
     "@tiptap/core": "^3.26.1",
-    "@tiptap/extension-code-block-lowlight": "^3.26.1",
+    "@tiptap/extension-code-block": "^3.26.1",
     "@tiptap/extension-list": "^3.26.1",
     "@tiptap/extension-placeholder": "^3.26.1",
     "@tiptap/extension-table": "^3.26.1",

package/src/editor/gpu-highlight.ts

@@ -0,0 +1,139 @@
+import type { Node as PmNode } from "@tiptap/pm/model"
+import { Plugin, PluginKey } from "@tiptap/pm/state"
+import { Decoration, DecorationSet } from "@tiptap/pm/view"
+
+/**
+ * Code-block syntax highlighting through gpu-lexer — the same language-agnostic WebGPU
+ * lexer docstream's renderer uses, so edit and preview agree. gpu-lexer is async, so
+ * spans are cached by block text: decorations are built synchronously from the cache,
+ * missing blocks are tokenized in the background, and a meta transaction re-decorates
+ * when results land. Without WebGPU `parse` rejects and blocks stay plain.
+ */
+
+type SpanType =
+  | "plain"
+  | "comment"
+  | "string"
+  | "number"
+  | "keyword"
+  | "type"
+  | "function"
+  | "constant"
+  | "operator"
+
+interface Span {
+  type: SpanType
+  start: number
+  end: number
+}
+
+const MAX_CODE_UNITS = 128 * 1024
+const CACHE_LIMIT = 256
+
+let lexer: Promise<(code: string) => Promise<Span[]>> | undefined
+const cache = new Map<string, Span[] | null>()
+const pending = new Map<string, Promise<void>>()
+
+function remember(code: string, spans: Span[] | null) {
+  cache.set(code, spans)
+  if (cache.size > CACHE_LIMIT) cache.delete(cache.keys().next().value as string)
+}
+
+function tokenize(code: string): Promise<void> {
+  const inflight = pending.get(code)
+  if (inflight) return inflight
+  // Loaded on first code block so the model stays out of the editor's initial bundle.
+  lexer ??= import("gpu-lexer").then((module) => module.parse as (code: string) => Promise<Span[]>)
+  const run = lexer
+    .then((parse) => parse(code))
+    .then(
+      (spans) => remember(code, spans),
+      () => remember(code, null),
+    )
+    .finally(() => pending.delete(code))
+  pending.set(code, run)
+  return run
+}
+
+function highlightable(code: string) {
+  return code.length > 0 && code.length <= MAX_CODE_UNITS
+}
+
+/**
+ * Decorations from the span cache. A block whose new text is not tokenized yet keeps its
+ * previous decorations mapped through the edit, so typing never flickers to plain.
+ */
+function decorationsFor(doc: PmNode, nodeName: string, previous?: DecorationSet): DecorationSet {
+  const decorations: Decoration[] = []
+  doc.descendants((node, pos) => {
+    if (node.type.name !== nodeName) return true
+    const from = pos + 1
+    const spans = cache.get(node.textContent)
+    if (!spans) {
+      if (previous) decorations.push(...previous.find(from, from + node.content.size))
+      return false
+    }
+    for (const span of spans) {
+      if (span.type === "plain" || span.end <= span.start) continue
+      decorations.push(
+        Decoration.inline(from + span.start, from + span.end, { class: `gb-syntax-${span.type}` }),
+      )
+    }
+    return false
+  })
+  return DecorationSet.create(doc, decorations)
+}
+
+function missingBlocks(doc: PmNode, nodeName: string): string[] {
+  const missing = new Set<string>()
+  doc.descendants((node) => {
+    if (node.type.name !== nodeName) return true
+    const code = node.textContent
+    if (highlightable(code) && !cache.has(code)) missing.add(code)
+    return false
+  })
+  return [...missing]
+}
+
+export function gpuHighlightPlugin(nodeName: string): Plugin {
+  const key = new PluginKey<DecorationSet>("gpuHighlight")
+  return new Plugin<DecorationSet>({
+    key,
+    state: {
+      init: (_, state) => decorationsFor(state.doc, nodeName),
+      apply: (tr, previous) =>
+        tr.docChanged || tr.getMeta(key)
+          ? decorationsFor(tr.doc, nodeName, previous.map(tr.mapping, tr.doc))
+          : previous,
+    },
+    props: {
+      decorations(state) {
+        return key.getState(state)
+      },
+    },
+    view(view) {
+      let destroyed = false
+      let timer: ReturnType<typeof setTimeout> | undefined
+      const request = () => {
+        for (const code of missingBlocks(view.state.doc, nodeName)) {
+          void tokenize(code).then(() => {
+            if (!destroyed) view.dispatch(view.state.tr.setMeta(key, true))
+          })
+        }
+      }
+      request()
+      return {
+        update: (_view, previous) => {
+          if (previous.doc.eq(view.state.doc)) return
+          // One tokenize per typing pause, not per keystroke.
+          clearTimeout(timer)
+          timer = setTimeout(request, 150)
+        },
+        destroy: () => {
+          destroyed = true
+          clearTimeout(timer)
+        },
+      }
+    },
+  })
+}

package/src/editor/nodes.tsx

@@ -1,6 +1,6 @@
 import { Node, mergeAttributes } from "@tiptap/core"
-import { CodeBlockLowlight } from "@tiptap/extension-code-block-lowlight"
-import { common, createLowlight } from "lowlight"
+import { CodeBlock } from "@tiptap/extension-code-block"
+import { gpuHighlightPlugin } from "./gpu-highlight"
 import { useEffect, useMemo, useState } from "react"
 import {
   NodeViewContent,
@@ -718,7 +718,7 @@ function CodeView({ node, updateAttributes, editor }: NodeViewProps) {
   )
 }
 
-export const GbCodeBlock = CodeBlockLowlight.extend({
+export const GbCodeBlock = CodeBlock.extend({
   addAttributes() {
     return {
       ...this.parent?.(),
@@ -731,7 +731,10 @@ export const GbCodeBlock = CodeBlockLowlight.extend({
   addNodeView() {
     return ReactNodeViewRenderer(CodeView)
   },
-}).configure({ lowlight: createLowlight(common) })
+  addProseMirrorPlugins() {
+    return [...(this.parent?.() ?? []), gpuHighlightPlugin(this.name)]
+  },
+})
 
 // ---------- Inline image (GitHub badge style) ----------
 

package/src/styles.css

@@ -582,31 +582,15 @@
   color: var(--gb-code-fg);
 }
 .gb-code > pre code { background: none; padding: 0; font-size: inherit; color: inherit; }
-.gb-code > pre code .hljs-comment { font-style: italic; }
-.gb-code .hljs-keyword,
-.gb-code .hljs-selector-tag,
-.gb-code .hljs-built_in,
-.gb-code .hljs-type,
-.gb-code .hljs-tag,
-.gb-code .hljs-name { color: #c792ea; }
-.gb-code .hljs-string,
-.gb-code .hljs-attr,
-.gb-code .hljs-symbol,
-.gb-code .hljs-bullet { color: #a5d6a7; }
-.gb-code .hljs-title,
-.gb-code .hljs-section,
-.gb-code .hljs-function .hljs-title { color: #82aaff; }
-.gb-code .hljs-number,
-.gb-code .hljs-literal,
-.gb-code .hljs-variable,
-.gb-code .hljs-template-variable { color: #f78c6c; }
-.gb-code .hljs-comment,
-.gb-code .hljs-quote,
-.gb-code .hljs-meta { color: #777784; }
-.gb-code .hljs-operator,
-.gb-code .hljs-punctuation { color: #89ddff; }
-.gb-code .hljs-deletion { color: #ff6b6b; }
-.gb-code .hljs-addition { color: #69db7c; }
+/* gpu-lexer span classes (palette shared with docstream's HighlightedCode). */
+.gb-code .gb-syntax-comment { color: #6b6b78; font-style: italic; }
+.gb-code .gb-syntax-string { color: #a5d6a7; }
+.gb-code .gb-syntax-number { color: #f78c6c; }
+.gb-code .gb-syntax-keyword { color: #c792ea; }
+.gb-code .gb-syntax-type { color: #ffcb6b; }
+.gb-code .gb-syntax-function { color: #82aaff; }
+.gb-code .gb-syntax-constant { color: #f07178; }
+.gb-code .gb-syntax-operator { color: #89ddff; }
 
 /* ── Updates / changelog (mirrors .docs-updates) ────────── */
 .gb-updates { margin: 0.85em 0; border-top: 1px solid var(--gb-border); }