@shortlink-org/portolan 0.1.0

package/plugins/gen-markdown/llms.go

@@ -0,0 +1,302 @@
+package main
+
+import (
+	"strings"
+
+	"github.com/shortlink-org/portolan/catalog"
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// The site for a language model, in the shape llmstxt.org asks for.
+//
+// llms.txt is the index page again as flat lists: one H1, one blockquote that
+// says what the estate is, then a link and a line of context per entity. A
+// reader that fetches pages on demand starts here.
+//
+// llms-full.txt is every page of the site in one file, for a reader that has a
+// context window and not a filesystem. It is assembled from the pages already
+// rendered, not rendered again, so it cannot disagree with them.
+//
+// Neither takes the estate's public URL: the links are the same relative
+// paths every other page uses, correct wherever the docs directory is mounted.
+const (
+	llmsPage     = "llms.txt"
+	llmsFullPage = "llms-full.txt"
+)
+
+// renderLlms runs last. Everything the builder holds at that point is the
+// site, and llms-full.txt is that, in that order.
+func (s *site) renderLlms() {
+	pages := append([]plugin.File(nil), s.b.Files...)
+
+	s.b.file(llmsPage, s.llmsIndex())
+	s.b.file(llmsFullPage, s.llmsFull(pages))
+}
+
+func (s *site) llmsIndex() string {
+	const self = llmsPage
+
+	var b strings.Builder
+	b.WriteString("# " + s.title() + "\n\n")
+	b.WriteString("> " + s.llmsSummary() + "\n\n")
+	b.WriteString(s.stamp() + "\n")
+	b.WriteString("Every link below is a markdown page beside this file. " +
+		"[" + llmsFullPage + "](" + llmsFullPage + ") is all of those pages in one file, " +
+		"in the order the site lists them.\n")
+
+	var contexts, services []string
+	for i := range s.cat.Contexts {
+		ctx := &s.cat.Contexts[i]
+		contexts = append(contexts, entry(s.ref(self, ctx.ID, ctx.Name), string(ctx.Classification), firstLine(ctx.Summary)))
+
+		for j := range ctx.Services {
+			svc := &ctx.Services[j]
+			names := make([]string, 0, len(svc.Aggregates))
+			for k := range svc.Aggregates {
+				names = append(names, svc.Aggregates[k].Name)
+			}
+			aggregates := ""
+			if len(names) > 0 {
+				aggregates = "Aggregates: " + strings.Join(names, ", ")
+			}
+			services = append(services, entry(s.ref(self, svc.ID, svc.Name), "in "+ctx.Name, aggregates))
+		}
+	}
+	section(&b, "Contexts", list(contexts))
+
+	// Straight after the contexts and before anything that uses the words. A
+	// model reading this estate cold has the same problem a person has: every
+	// page below is written in a vocabulary it has not been told yet.
+	var glossaries []string
+	for i := range s.cat.Contexts {
+		ctx := &s.cat.Contexts[i]
+		terms := s.termsOf[ctx.ID]
+		if len(terms) == 0 {
+			continue
+		}
+		names := make([]string, 0, len(terms))
+		for _, term := range terms {
+			names = append(names, term.Name)
+		}
+		glossaries = append(glossaries, entry(
+			link("Glossary of "+ctx.Name, self, s.glossaryPath(ctx)),
+			plural(len(terms), "term"),
+			strings.Join(names, ", "),
+		))
+	}
+	section(&b, "Glossaries", list(glossaries))
+
+	section(&b, "Services", list(services))
+
+	// The pages a model has most to learn from: the shape of the data, the
+	// commands and the events of the estate all live here, and nowhere else
+	// on the site is the readme of an aggregate printed in full.
+	var aggregates []string
+	for i := range s.cat.Contexts {
+		ctx := &s.cat.Contexts[i]
+		for j := range ctx.Services {
+			svc := &ctx.Services[j]
+			for k := range svc.Aggregates {
+				agg := &svc.Aggregates[k]
+				commands, queries := 0, 0
+				for _, op := range agg.Operations {
+					if op.Kind == catalog.OperationQuery {
+						queries++
+					} else {
+						commands++
+					}
+				}
+				shape := []string{plural(commands, "command")}
+				if queries > 0 {
+					shape = append(shape, plural(queries, "query", "queries"))
+				}
+				shape = append(shape, plural(len(agg.Events), "event"))
+				aggregates = append(aggregates, entry(
+					s.ref(self, agg.ID, agg.Name),
+					"in "+s.ref(self, svc.ID, svc.Name),
+					"root "+code(agg.Root),
+					strings.Join(shape, ", "),
+					firstSentence(body(agg.Readme, agg.Name)),
+				))
+			}
+		}
+	}
+	section(&b, "Aggregates", list(aggregates))
+
+	externals := make([]string, 0, len(s.cat.Externals))
+	for i := range s.cat.Externals {
+		ext := &s.cat.Externals[i]
+		interfaces := make([]string, 0, len(ext.Provides))
+		for j := range ext.Provides {
+			interfaces = append(interfaces, ext.Provides[j].ID)
+		}
+		externals = append(externals, entry(s.ref(self, ext.ID, orDefault(ext.Name, ext.ID)),
+			"outside the estate", "Answers on "+strings.Join(interfaces, ", "), firstLine(ext.Summary)))
+	}
+	section(&b, "Outside the estate", list(externals))
+
+	modules := make([]string, 0, len(s.cat.Modules))
+	for i := range s.cat.Modules {
+		module := &s.cat.Modules[i]
+		modules = append(modules, entry(
+			s.ref(self, module.ID, orDefault(module.Name, module.ID)),
+			"schema module", "published by "+s.ref(self, module.Owner, module.Owner),
+			strings.Join(module.Packages, ", "),
+		))
+	}
+	section(&b, "Schema modules", list(modules))
+
+	flows := make([]string, 0, len(s.cat.Flows))
+	for i := range s.cat.Flows {
+		flow := &s.cat.Flows[i]
+		owner := ""
+		if flow.Owner != "" {
+			owner = "owned by " + s.ref(self, flow.Owner, flow.Owner)
+		}
+		flows = append(flows, entry(s.ref(self, flow.ID, flow.Name), owner, firstLine(flow.Summary)))
+	}
+	section(&b, "Flows", list(flows))
+
+	adrs := make([]string, 0, len(s.cat.Adrs))
+	for i := range s.cat.Adrs {
+		adr := &s.cat.Adrs[i]
+		var when []string
+		if adr.Status != "" {
+			when = append(when, string(adr.Status))
+		}
+		if adr.Date != "" {
+			when = append(when, adr.Date)
+		}
+		title := adr.Title
+		if len(when) > 0 {
+			title += " (" + strings.Join(when, ", ") + ")"
+		}
+		adrs = append(adrs, entry(s.ref(self, adr.ID, adr.ID), title))
+	}
+	section(&b, "Decisions", list(adrs))
+
+	// What llmstxt.org calls Optional: pages a reader short on context can
+	// skip. The shape of a shared type or a table matters once a page above
+	// has named it, and not before.
+	var optional []string
+	if len(s.cat.Defs) > 0 {
+		optional = append(optional, entry(link("Shared types", self, typesPage),
+			"types named by more than one aggregate, event or message"))
+	}
+	for i := range s.cat.Stores {
+		store := &s.cat.Stores[i]
+		note := string(store.Kind) + " store"
+		if owner, ok := s.services[store.Owner]; ok {
+			note += " of " + s.ref(self, owner.ID, owner.Name)
+		}
+		schema := plural(len(store.Tables), "table")
+		if len(store.Keyspaces) > 0 {
+			schema = plural(len(store.Keyspaces), "key pattern")
+		}
+		optional = append(optional, entry(s.ref(self, store.ID, store.Name), note, schema))
+	}
+	section(&b, "Optional", list(optional))
+
+	return b.String()
+}
+
+// llmsSummary is the blockquote: what kind of thing this is, and how much of
+// it there is.
+func (s *site) llmsSummary() string {
+	services, aggregates := 0, 0
+	for i := range s.cat.Contexts {
+		services += len(s.cat.Contexts[i].Services)
+		for j := range s.cat.Contexts[i].Services {
+			aggregates += len(s.cat.Contexts[i].Services[j].Aggregates)
+		}
+	}
+
+	parts := []string{
+		plural(len(s.cat.Contexts), "bounded context"),
+		plural(services, "service"),
+		plural(aggregates, "aggregate"),
+		plural(len(s.cat.Stores), "store"),
+	}
+	if len(s.cat.Modules) > 0 {
+		parts = append(parts, plural(len(s.cat.Modules), "schema module"))
+	}
+	// Named only when there is one: an estate that calls nobody outside has
+	// nothing to count, and "0 systems outside" reads as a claim about the
+	// world rather than about the catalog.
+	if len(s.cat.Externals) > 0 {
+		parts = append(parts, plural(len(s.cat.Externals), "system outside the estate", "systems outside the estate"))
+	}
+	parts = append(parts,
+		plural(len(s.cat.Flows), "flow"),
+		plural(len(s.cat.Adrs), "decision record"),
+	)
+
+	return "Architecture catalog of " + s.title() + ": " + strings.Join(parts[:len(parts)-1], ", ") +
+		" and " + parts[len(parts)-1] + ", read from the code, the specs and the traces of the estate."
+}
+
+func (s *site) llmsFull(pages []plugin.File) string {
+	var b strings.Builder
+	b.WriteString("*" + llmsFullPage + ": every page of " + s.title() + ", one after another. " +
+		"A page begins with a comment naming its file, and the links inside it are " +
+		"relative to that file's directory.*\n\n")
+	b.WriteString(s.stamp())
+
+	for _, page := range pages {
+		b.WriteString("\n<!-- " + page.Name + " -->\n\n")
+		b.WriteString(strings.TrimRight(page.Contents, "\n") + "\n")
+	}
+
+	return b.String()
+}
+
+// entry is one line of an llms.txt list: the link, then whatever notes are
+// not empty, sentence by sentence.
+func entry(link string, notes ...string) string {
+	kept := make([]string, 0, len(notes))
+	for _, note := range notes {
+		note = strings.TrimSpace(note)
+		if note == "" {
+			continue
+		}
+		kept = append(kept, strings.TrimSuffix(note, "."))
+	}
+	if len(kept) == 0 {
+		return "- " + link
+	}
+
+	return "- " + link + ": " + strings.Join(kept, ". ")
+}
+
+// firstSentence is the opening of a readme as one line: the first paragraph
+// unwrapped, cut at the end of its first sentence. A readme is wrapped at
+// some column by whoever wrote it, so the first line alone stops mid-phrase.
+func firstSentence(md string) string {
+	md = strings.TrimSpace(md)
+	if md == "" {
+		return ""
+	}
+	para := md
+	if i := strings.Index(md, "\n\n"); i >= 0 {
+		para = md[:i]
+	}
+	if strings.HasPrefix(para, "#") {
+		return ""
+	}
+	para = strings.Join(strings.Fields(para), " ")
+	for i := 0; i < len(para); i++ {
+		if para[i] == '.' && (i+1 == len(para) || para[i+1] == ' ') {
+			return para[:i]
+		}
+	}
+
+	return para
+}
+
+func list(items []string) string {
+	if len(items) == 0 {
+		return ""
+	}
+
+	return strings.Join(items, "\n") + "\n"
+}

package/plugins/gen-markdown/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+func main() {
+	if err := run(os.Stdin, os.Stdout); err != nil {
+		fmt.Fprintln(os.Stderr, "portolan-gen-markdown:", err)
+		os.Exit(1)
+	}
+}
+
+func run(stdin io.Reader, stdout io.Writer) error {
+	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
+		return render(req, opts), nil
+	})
+}

package/plugins/gen-markdown/markdown.go

@@ -0,0 +1,254 @@
+package main
+
+import (
+	"path"
+	"strings"
+)
+
+// Markdown primitives. Small on purpose: a template engine would put the shape
+// of a page somewhere other than the code that decides what goes on it.
+
+// table renders a GFM table, or nothing at all when there are no rows. An
+// empty table with a header is a heading that promises content and delivers a
+// horizontal rule.
+func table(headers []string, rows [][]string) string {
+	if len(rows) == 0 {
+		return ""
+	}
+
+	// A column nobody filled in is dropped. The same renderer draws a table of
+	// columns with lineage and a table of columns without any, and a column of
+	// nine dashes is a question the reader keeps re-asking.
+	keep := make([]int, 0, len(headers))
+	for i := range headers {
+		// The first column identifies the row, so it stays even when empty.
+		if i == 0 || anyFilled(rows, i) {
+			keep = append(keep, i)
+		}
+	}
+
+	var b strings.Builder
+	head := make([]string, len(keep))
+	for i, column := range keep {
+		head[i] = headers[column]
+	}
+	b.WriteString("| " + strings.Join(head, " | ") + " |\n")
+	b.WriteString("|" + strings.Repeat(" --- |", len(keep)) + "\n")
+
+	for _, row := range rows {
+		cells := make([]string, len(keep))
+		for i, column := range keep {
+			if column < len(row) {
+				cells[i] = cell(row[column])
+			} else {
+				cells[i] = cell("")
+			}
+		}
+		b.WriteString("| " + strings.Join(cells, " | ") + " |\n")
+	}
+
+	return b.String()
+}
+
+func anyFilled(rows [][]string, column int) bool {
+	for _, row := range rows {
+		if column < len(row) && strings.TrimSpace(row[column]) != "" {
+			return true
+		}
+	}
+
+	return false
+}
+
+// cell makes a string safe to sit between two pipes. A doc comment routinely
+// contains both a newline and a pipe, and either one silently ends the row.
+func cell(s string) string {
+	s = strings.ReplaceAll(s, "|", "\\|")
+	s = strings.ReplaceAll(s, "\r\n", "\n")
+	s = strings.ReplaceAll(s, "\n", " ")
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return "—"
+	}
+
+	return s
+}
+
+func code(s string) string {
+	if s == "" {
+		return ""
+	}
+	longest, run := 0, 0
+	for _, r := range s {
+		if r == '`' {
+			run++
+			if run > longest {
+				longest = run
+			}
+		} else {
+			run = 0
+		}
+	}
+	marks := strings.Repeat("`", longest+1)
+	if longest > 0 || strings.HasPrefix(s, " ") || strings.HasSuffix(s, " ") {
+		return marks + " " + s + " " + marks
+	}
+	return marks + s + marks
+}
+
+func fence(lang, body string) string {
+	return "```" + lang + "\n" + strings.TrimRight(body, "\n") + "\n```\n"
+}
+
+// link is a relative markdown link from one generated page to another. The
+// paths are the ones the host will write, so the link is correct wherever the
+// docs directory is mounted.
+func link(text, from, to string) string {
+	return "[" + text + "](" + rel(from, to) + ")"
+}
+
+// rel is filepath.Rel for slash paths, between two FILES: the result is
+// relative to the directory the first one sits in.
+//
+// It is written out rather than taken from path/filepath because a plugin
+// compiled for wasip1 and one compiled for the host must agree, and
+// filepath.Rel agrees with whatever separator it was built for.
+func rel(from, to string) string {
+	fromDir := strings.Split(path.Dir(from), "/")
+	if path.Dir(from) == "." {
+		fromDir = nil
+	}
+
+	toParts := strings.Split(to, "/")
+
+	common := 0
+	for common < len(fromDir) && common < len(toParts)-1 && fromDir[common] == toParts[common] {
+		common++
+	}
+
+	var out []string
+	for range fromDir[common:] {
+		out = append(out, "..")
+	}
+	out = append(out, toParts[common:]...)
+
+	result := strings.Join(out, "/")
+	if result == "" {
+		return path.Base(to)
+	}
+
+	return result
+}
+
+// shiftHeadings pushes every ATX heading down by `by` levels, leaving fenced
+// code alone.
+//
+// Readmes and ADR bodies are whole documents that start at `#`. Pasting one
+// under a generated `#` gives a page with two top-level headings, which every
+// table-of-contents generator then reads as two documents.
+func shiftHeadings(md string, by int) string {
+	if md == "" {
+		return ""
+	}
+
+	lines := strings.Split(strings.ReplaceAll(md, "\r\n", "\n"), "\n")
+	fenced := false
+	for i, line := range lines {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") {
+			fenced = !fenced
+
+			continue
+		}
+		if fenced || !strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		level := 0
+		for level < len(line) && line[level] == '#' {
+			level++
+		}
+		// A run of hashes with no space after it is not a heading.
+		if level >= len(line) || line[level] != ' ' {
+			continue
+		}
+		// Markdown has no h7. Anything that would go past six stays where it is,
+		// because a heading rendered as literal hashes is worse than a deep one.
+		if level+by > 6 {
+			continue
+		}
+
+		lines[i] = strings.Repeat("#", by) + line
+	}
+
+	return strings.Join(lines, "\n")
+}
+
+// section appends a titled block, and nothing at all when the block is empty.
+// Every page here is a list of optional sections, so this is what keeps a
+// service with no consumers from growing a "Consumes" heading over a blank.
+func section(b *strings.Builder, title, body string) {
+	if strings.TrimSpace(body) == "" {
+		return
+	}
+
+	b.WriteString("\n## " + title + "\n\n")
+	b.WriteString(strings.TrimRight(body, "\n") + "\n")
+}
+
+func subsection(b *strings.Builder, title, body string) {
+	if strings.TrimSpace(body) == "" {
+		return
+	}
+
+	b.WriteString("\n### " + title + "\n\n")
+	b.WriteString(strings.TrimRight(body, "\n") + "\n")
+}
+
+// defList renders a page's metadata.
+//
+// It is a bullet list rather than a two-column table because a table needs a
+// header row, and the header over an id and a path is two empty cells that
+// every renderer draws a box around.
+func defList(rows [][]string) string {
+	var b strings.Builder
+	for _, row := range rows {
+		if len(row) < 2 || strings.TrimSpace(row[1]) == "" {
+			continue
+		}
+		b.WriteString("- **" + row[0] + ":** " + strings.TrimSpace(row[1]) + "\n")
+	}
+
+	return b.String()
+}
+
+// body prepares a readme or an ADR body to sit under a generated `# Title`.
+//
+// A document that opens with its own title repeating the page's gets that line
+// dropped, and everything below it is already at the right depth. One that
+// starts lower is pushed down a level so its sections nest under the page
+// instead of beside it.
+func body(md, title string) string {
+	md = strings.TrimSpace(md)
+	if md == "" {
+		return ""
+	}
+
+	lines := strings.Split(strings.ReplaceAll(md, "\r\n", "\n"), "\n")
+	for i, line := range lines {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		if heading, ok := strings.CutPrefix(line, "# "); ok && sameTitle(heading, title) {
+			return strings.TrimSpace(strings.Join(lines[i+1:], "\n"))
+		}
+
+		break
+	}
+
+	return shiftHeadings(md, 1)
+}
+
+func sameTitle(a, b string) bool {
+	return strings.EqualFold(strings.TrimSpace(a), strings.TrimSpace(b))
+}

package/plugins/gen-markdown/markdown_test.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestRel(t *testing.T) {
+	cases := []struct{ from, to, want string }{
+		{"README.md", "types.md", "types.md"},
+		{"README.md", "shop/oms/README.md", "shop/oms/README.md"},
+		{"shop/oms/README.md", "shop/oms/aggregates/order.md", "aggregates/order.md"},
+		{"shop/oms/aggregates/order.md", "types.md", "../../../types.md"},
+		{"shop/oms/aggregates/order.md", "shop/pricing/README.md", "../../pricing/README.md"},
+		{"flows/checkout.md", "shop/oms/README.md", "../shop/oms/README.md"},
+		{"adr/org.0001.md", "adr/org.0002.md", "org.0002.md"},
+	}
+
+	for _, c := range cases {
+		if got := rel(c.from, c.to); got != c.want {
+			t.Errorf("rel(%q, %q) = %q, want %q", c.from, c.to, got, c.want)
+		}
+	}
+}
+
+func TestShiftHeadingsLeavesCodeAlone(t *testing.T) {
+	in := "# Title\n\n```sh\n# not a heading\n```\n\n## Section\n"
+	want := "## Title\n\n```sh\n# not a heading\n```\n\n### Section\n"
+
+	if got := shiftHeadings(in, 1); got != want {
+		t.Errorf("shiftHeadings:\n got: %q\nwant: %q", got, want)
+	}
+}
+
+func TestShiftHeadingsStopsAtSix(t *testing.T) {
+	if got := shiftHeadings("###### deep", 1); got != "###### deep" {
+		t.Errorf("a sixth-level heading must stay put, got %q", got)
+	}
+	if got := shiftHeadings("#hashtag", 1); got != "#hashtag" {
+		t.Errorf("a hash with no space is not a heading, got %q", got)
+	}
+}
+
+// A readme opening with the page's own title would give the page two h1s, so
+// that line is dropped and the rest keeps its depth.
+func TestBodyDropsRepeatedTitle(t *testing.T) {
+	got := body("# Basket\n\n## Lifetime\n\nShort.", "Basket")
+	if got != "## Lifetime\n\nShort." {
+		t.Errorf("body kept the repeated title: %q", got)
+	}
+
+	got = body("Just prose.\n\n# Later heading", "Basket")
+	if got != "Just prose.\n\n## Later heading" {
+		t.Errorf("body should shift a document that does not open with the title: %q", got)
+	}
+}
+
+func TestCellSurvivesPipesAndNewlines(t *testing.T) {
+	if got := cell("a | b\nc"); got != "a \\| b c" {
+		t.Errorf("cell = %q", got)
+	}
+	if got := cell("  "); got != "—" {
+		t.Errorf("an empty cell should be a dash, got %q", got)
+	}
+}
+
+func TestMermaidTextEscapesHashBeforeSemicolons(t *testing.T) {
+	// The escape for # ends in a semicolon, so the order of the two
+	// replacements is the whole of this function's correctness.
+	if got := mermaidText("markPaid #1; twice"); got != "markPaid #35;1, twice" {
+		t.Errorf("mermaidText = %q", got)
+	}
+}
+
+func TestAnchor(t *testing.T) {
+	if got := anchor("Gateway Ref"); got != "gateway-ref" {
+		t.Errorf("anchor = %q", got)
+	}
+}
+
+func TestCodeUsesALongerFenceAroundBackticks(t *testing.T) {
+	if got := code("run `gen`"); got != "`` run `gen` ``" {
+		t.Fatalf("code() = %q", got)
+	}
+}
+
+func FuzzMarkdownPrimitives(f *testing.F) {
+	f.Add("a | b\nnext `tick`")
+	f.Add("обычный текст")
+	f.Fuzz(func(t *testing.T, value string) {
+		got := cell(value)
+		if strings.ContainsAny(got, "\r\n") {
+			t.Fatalf("cell kept a line break: %q", got)
+		}
+		wrapped := code(value)
+		if value != "" && !strings.HasPrefix(wrapped, "`") {
+			t.Fatalf("code is not fenced: %q", wrapped)
+		}
+	})
+}