@peaske7/readit 0.2.0 → 0.3.0-rc.0

package/go/internal/server/htmltext.go

@@ -0,0 +1,123 @@
+package server
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	htmlTagRe = regexp.MustCompile(`<\/?([a-zA-Z][a-zA-Z0-9]*)[^>]*\/?>`)
+	entityRe  = regexp.MustCompile(`&(?:#(\d+)|#x([0-9a-fA-F]+)|(\w+));`)
+)
+
+var blockElements = map[string]bool{
+	"p": true, "div": true, "h1": true, "h2": true, "h3": true,
+	"h4": true, "h5": true, "h6": true, "pre": true, "blockquote": true,
+	"li": true, "tr": true, "br": true,
+}
+
+var namedEntities = map[string]string{
+	"amp": "&", "lt": "<", "gt": ">", "quot": `"`,
+	"apos": "'", "nbsp": "\u00a0",
+}
+
+func ExtractTextFromHTML(html string) string {
+	var b strings.Builder
+	lastPos := 0
+	prevWasBlock := false
+
+	for _, loc := range htmlTagRe.FindAllStringIndex(html, -1) {
+		text := html[lastPos:loc[0]]
+		decoded := decodeEntities(text)
+		if decoded != "" {
+			if prevWasBlock && b.Len() > 0 {
+				b.WriteByte('\n')
+			}
+			b.WriteString(decoded)
+			prevWasBlock = false
+		}
+
+		// Check if this tag is a block element
+		tag := htmlTagRe.FindStringSubmatch(html[loc[0]:loc[1]])
+		if len(tag) > 1 && blockElements[strings.ToLower(tag[1])] {
+			prevWasBlock = true
+		}
+
+		lastPos = loc[1]
+	}
+
+	// Trailing text
+	if lastPos < len(html) {
+		text := html[lastPos:]
+		decoded := decodeEntities(text)
+		if decoded != "" {
+			if prevWasBlock && b.Len() > 0 {
+				b.WriteByte('\n')
+			}
+			b.WriteString(decoded)
+		}
+	}
+
+	return b.String()
+}
+
+func decodeEntities(s string) string {
+	return entityRe.ReplaceAllStringFunc(s, func(m string) string {
+		sub := entityRe.FindStringSubmatch(m)
+		if sub[1] != "" { // &#NNN;
+			if code, err := strconv.Atoi(sub[1]); err == nil {
+				return string(rune(code))
+			}
+		}
+		if sub[2] != "" { // &#xHHH;
+			if code, err := strconv.ParseInt(sub[2], 16, 32); err == nil {
+				return string(rune(code))
+			}
+		}
+		if sub[3] != "" { // &name;
+			if r, ok := namedEntities[sub[3]]; ok {
+				return r
+			}
+		}
+		return m
+	})
+}
+
+func FindTextPosition(textContent, selectedText string, hintOffset int) (start, end int, ok bool) {
+	if selectedText == "" {
+		return 0, 0, false
+	}
+
+	var positions []int
+	searchFrom := 0
+	for {
+		idx := strings.Index(textContent[searchFrom:], selectedText)
+		if idx < 0 {
+			break
+		}
+		positions = append(positions, searchFrom+idx)
+		searchFrom += idx + 1
+	}
+
+	if len(positions) == 0 {
+		return 0, 0, false
+	}
+
+	if len(positions) == 1 {
+		return positions[0], positions[0] + len(selectedText), true
+	}
+
+	// Multiple matches - pick closest to hint
+	best := positions[0]
+	bestDist := abs(best - hintOffset)
+	for _, pos := range positions[1:] {
+		d := abs(pos - hintOffset)
+		if d < bestDist {
+			best = pos
+			bestDist = d
+		}
+	}
+
+	return best, best + len(selectedText), true
+}

package/go/internal/server/markdown.go

@@ -0,0 +1,157 @@
+package server
+
+import (
+	"bytes"
+	"fmt"
+	htmlpkg "html"
+	"regexp"
+	"strings"
+
+	chromahtml "github.com/alecthomas/chroma/v2/formatters/html"
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/yuin/goldmark"
+	highlighting "github.com/yuin/goldmark-highlighting/v2"
+	"github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer/html"
+	"github.com/yuin/goldmark/text"
+)
+
+type RenderResult struct {
+	HTML     string
+	Headings []Heading
+}
+
+type Renderer struct {
+	md       goldmark.Markdown
+	sanitize *bluemonday.Policy
+}
+
+func NewRenderer() *Renderer {
+	// Allow raw HTML through goldmark, but sanitize to strip dangerous elements.
+	p := bluemonday.UGCPolicy()
+	p.AllowAttrs("class").Globally()
+	p.AllowAttrs("id").Globally()
+	p.AllowAttrs("style").Globally()
+	p.AllowElements("details", "summary", "pre", "code", "span")
+	p.AllowAttrs("open").OnElements("details")
+	p.AllowAttrs("data-mermaid-placeholder").OnElements("div")
+
+	return &Renderer{
+		md: goldmark.New(
+			goldmark.WithExtensions(
+				extension.GFM,
+				extension.TaskList,
+				highlighting.NewHighlighting(
+					highlighting.WithStyle("onedark"),
+					highlighting.WithGuessLanguage(true),
+					highlighting.WithFormatOptions(
+						chromahtml.WithClasses(true),
+						chromahtml.WithLineNumbers(false),
+					),
+				),
+			),
+			goldmark.WithParserOptions(
+				parser.WithAutoHeadingID(),
+			),
+			goldmark.WithRendererOptions(
+				html.WithUnsafe(),
+			),
+		),
+		sanitize: p,
+	}
+}
+
+var mermaidFenceRe = regexp.MustCompile("(?m)^```mermaid\\s*\n([\\s\\S]*?)^```\\s*$")
+
+// Render converts markdown source to HTML and extracts headings.
+// Mermaid fenced blocks are extracted before rendering so chroma doesn't
+// consume them, then re-injected as <pre><code class="language-mermaid">.
+func (r *Renderer) Render(source []byte) (RenderResult, error) {
+	content := string(source)
+
+	var mermaidBlocks []string
+	processed := mermaidFenceRe.ReplaceAllStringFunc(content, func(match string) string {
+		sub := mermaidFenceRe.FindStringSubmatch(match)
+		code := strings.TrimSpace(sub[1])
+		idx := len(mermaidBlocks)
+		mermaidBlocks = append(mermaidBlocks, code)
+		return fmt.Sprintf(`<div data-mermaid-placeholder="%d"></div>`, idx)
+	})
+
+	processedBytes := []byte(processed)
+
+	// Parse into AST so we can extract headings with correct IDs
+	reader := text.NewReader(processedBytes)
+	doc := r.md.Parser().Parse(reader)
+
+	// Extract headings from the AST (IDs match rendered HTML exactly)
+	headings := extractHeadingsFromAST(doc, processedBytes)
+
+	// Render the AST to HTML
+	var buf bytes.Buffer
+	if err := r.md.Renderer().Render(&buf, processedBytes, doc); err != nil {
+		return RenderResult{}, fmt.Errorf("goldmark render: %w", err)
+	}
+	output := buf.String()
+
+	output = r.sanitize.Sanitize(output)
+
+	// Restore mermaid blocks (after sanitization since they use escaped content)
+	for i, code := range mermaidBlocks {
+		placeholder := fmt.Sprintf(`<div data-mermaid-placeholder="%d"></div>`, i)
+		replacement := fmt.Sprintf(`<pre><code class="language-mermaid">%s</code></pre>`, htmlpkg.EscapeString(code))
+		output = strings.Replace(output, placeholder, replacement, 1)
+	}
+
+	return RenderResult{HTML: output, Headings: headings}, nil
+}
+
+// extractHeadingsFromAST walks the goldmark AST to collect heading nodes.
+// This produces IDs that match the rendered HTML exactly (via WithAutoHeadingID).
+func extractHeadingsFromAST(doc ast.Node, source []byte) []Heading {
+	var headings []Heading
+
+	_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
+		if !entering || n.Kind() != ast.KindHeading {
+			return ast.WalkContinue, nil
+		}
+		h := n.(*ast.Heading)
+
+		var id string
+		if raw, ok := h.AttributeString("id"); ok {
+			if b, isByte := raw.([]byte); isByte {
+				id = string(b)
+			}
+		}
+
+		// Collect plain text from all inline children recursively
+		var textBuf bytes.Buffer
+		collectText(h, source, &textBuf)
+
+		headings = append(headings, Heading{
+			ID:    id,
+			Text:  strings.TrimSpace(textBuf.String()),
+			Level: h.Level,
+		})
+
+		return ast.WalkSkipChildren, nil
+	})
+
+	return headings
+}
+
+// collectText recursively collects plain text from AST nodes.
+func collectText(n ast.Node, source []byte, buf *bytes.Buffer) {
+	for c := n.FirstChild(); c != nil; c = c.NextSibling() {
+		if t, ok := c.(*ast.Text); ok {
+			buf.Write(t.Segment.Value(source))
+			if t.SoftLineBreak() {
+				buf.WriteByte(' ')
+			}
+		} else {
+			collectText(c, source, buf)
+		}
+	}
+}

package/go/internal/server/markdown_bench_test.go

@@ -0,0 +1,42 @@
+package server
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func generateMarkdown(lines int) []byte {
+	var b strings.Builder
+	b.WriteString("# Document Title\n\n")
+	for i := range lines {
+		switch i % 10 {
+		case 0:
+			fmt.Fprintf(&b, "## Section %d\n\n", i/10)
+		case 5:
+			b.WriteString("```go\nfunc hello() {\n\tfmt.Println(\"world\")\n}\n```\n\n")
+		default:
+			fmt.Fprintf(&b, "This is paragraph %d with some **bold** and *italic* text. ", i)
+			b.WriteString("It contains [links](https://example.com) and `inline code`.\n\n")
+		}
+	}
+	return []byte(b.String())
+}
+
+func BenchmarkRender1000Lines(b *testing.B) {
+	r := NewRenderer()
+	src := generateMarkdown(1000)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		r.Render(src)
+	}
+}
+
+func BenchmarkRender3000Lines(b *testing.B) {
+	r := NewRenderer()
+	src := generateMarkdown(3000)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		r.Render(src)
+	}
+}

package/go/internal/server/markdown_test.go

@@ -0,0 +1,79 @@
+package server
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestRenderBasicMarkdown(t *testing.T) {
+	r := NewRenderer()
+	result, err := r.Render([]byte("# Hello\n\nSome **bold** text."))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.Contains(result.HTML, "<h1") {
+		t.Error("expected <h1> in output")
+	}
+	if !strings.Contains(result.HTML, "<strong>bold</strong>") {
+		t.Error("expected <strong> in output")
+	}
+}
+
+func TestRenderCodeBlock(t *testing.T) {
+	r := NewRenderer()
+	src := "```go\nfmt.Println(\"hello\")\n```"
+	result, err := r.Render([]byte(src))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(result.HTML, "chroma") {
+		t.Error("expected chroma class in highlighted code")
+	}
+}
+
+func TestRenderMermaidPassthrough(t *testing.T) {
+	r := NewRenderer()
+	src := "```mermaid\ngraph TD\n  A --> B\n```"
+	result, err := r.Render([]byte(src))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Mermaid blocks should pass through as code blocks for client-side rendering
+	if !strings.Contains(result.HTML, "language-mermaid") {
+		t.Error("expected language-mermaid class in output")
+	}
+}
+
+func TestRenderGFMTable(t *testing.T) {
+	r := NewRenderer()
+	src := "| A | B |\n|---|---|\n| 1 | 2 |"
+	result, err := r.Render([]byte(src))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.Contains(result.HTML, "<table>") {
+		t.Error("expected <table> in GFM table output")
+	}
+}
+
+func TestRenderHeadings(t *testing.T) {
+	r := NewRenderer()
+	result, err := r.Render([]byte("# Title\n\n## Section\n\n### Sub"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(result.Headings) != 3 {
+		t.Fatalf("expected 3 headings, got %d", len(result.Headings))
+	}
+
+	if result.Headings[0].Level != 1 || result.Headings[0].Text != "Title" {
+		t.Errorf("heading 0: got level=%d text=%q", result.Headings[0].Level, result.Headings[0].Text)
+	}
+	if result.Headings[1].Level != 2 || result.Headings[1].Text != "Section" {
+		t.Errorf("heading 1: got level=%d text=%q", result.Headings[1].Level, result.Headings[1].Text)
+	}
+}