@shortlink-org/portolan 0.1.0

package/plugins/extract-proto/lex.go

@@ -0,0 +1,285 @@
+package main
+
+// The tokens of a .proto file, and the comments attached to them.
+//
+// Comments are not thrown away. A proto's leading comments are the only
+// documentation it carries - a descriptor drops them unless SourceCodeInfo is
+// asked for and carried around - and `Field.doc` in the catalog would come back
+// empty without them. So the lexer keeps them, attaches each block to the token
+// it precedes, and files each trailing one under the line it sits on.
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type kind int
+
+const (
+	tokEOF kind = iota
+	tokIdent
+	tokString
+	tokNumber
+	tokPunct
+)
+
+type token struct {
+	kind kind
+	text string
+	line int
+
+	// doc is the comment block directly above this token, already cleaned.
+	// Empty unless this token opens a declaration that was commented.
+	doc string
+}
+
+func (t token) is(text string) bool {
+	return (t.kind == tokPunct || t.kind == tokIdent) && t.text == text
+}
+
+// lexed is a file's tokens plus the comments that did not lead a declaration.
+type lexed struct {
+	tokens []token
+
+	// trailing is the comment sitting at the end of a line, by line number.
+	// A field documented to its right rather than above is common enough that
+	// dropping it would lose real prose.
+	trailing map[int]string
+}
+
+type lexer struct {
+	src  string
+	pos  int
+	line int
+
+	pending  []string // comment block being accumulated
+	lastEnd  int      // line the last accumulated comment ended on
+	trailing map[int]string
+	lastTok  int // line of the last non-comment token, for trailing detection
+}
+
+func lex(src string) (lexed, error) {
+	l := &lexer{src: src, line: 1, trailing: map[int]string{}, lastTok: -1}
+	out := lexed{trailing: l.trailing}
+
+	for {
+		tok, err := l.next()
+		if err != nil {
+			return lexed{}, err
+		}
+		out.tokens = append(out.tokens, tok)
+		if tok.kind == tokEOF {
+			return out, nil
+		}
+	}
+}
+
+func (l *lexer) next() (token, error) {
+	for {
+		l.skipSpace()
+		if l.pos >= len(l.src) {
+			return token{kind: tokEOF, line: l.line}, nil
+		}
+
+		if l.src[l.pos] == '/' && l.pos+1 < len(l.src) {
+			switch l.src[l.pos+1] {
+			case '/':
+				l.lineComment()
+
+				continue
+			case '*':
+				if err := l.blockComment(); err != nil {
+					return token{}, err
+				}
+
+				continue
+			}
+		}
+
+		break
+	}
+
+	start := l.line
+	c := l.src[l.pos]
+
+	switch {
+	case c == '"' || c == '\'':
+		text, err := l.str()
+		if err != nil {
+			return token{}, err
+		}
+
+		return l.emit(token{kind: tokString, text: text, line: start}), nil
+
+	case isIdentStart(c):
+		begin := l.pos
+		for l.pos < len(l.src) && isIdentPart(l.src[l.pos]) {
+			l.pos++
+		}
+
+		return l.emit(token{kind: tokIdent, text: l.src[begin:l.pos], line: start}), nil
+
+	case c >= '0' && c <= '9':
+		begin := l.pos
+		for l.pos < len(l.src) && isNumberPart(l.src[l.pos]) {
+			l.pos++
+		}
+
+		return l.emit(token{kind: tokNumber, text: l.src[begin:l.pos], line: start}), nil
+
+	default:
+		l.pos++
+
+		return l.emit(token{kind: tokPunct, text: string(c), line: start}), nil
+	}
+}
+
+// emit hands the accumulated comment block to a token, but only when the block
+// ends on the line above it. A comment separated by a blank line was about
+// whatever came before, or about nothing, and attaching it would put a
+// paragraph on a declaration that never claimed it.
+func (l *lexer) emit(t token) token {
+	if len(l.pending) > 0 {
+		if l.lastEnd == t.line-1 {
+			t.doc = strings.Join(l.pending, "\n")
+		}
+		l.pending = nil
+	}
+	l.lastTok = t.line
+
+	return t
+}
+
+func (l *lexer) lineComment() {
+	l.pos += 2
+	begin := l.pos
+	for l.pos < len(l.src) && l.src[l.pos] != '\n' {
+		l.pos++
+	}
+	text := strings.TrimSpace(l.src[begin:l.pos])
+
+	// A comment on the same line as the last token trails it rather than
+	// leading whatever comes next.
+	if l.line == l.lastTok {
+		if prev, ok := l.trailing[l.line]; ok {
+			l.trailing[l.line] = prev + " " + text
+		} else {
+			l.trailing[l.line] = text
+		}
+
+		return
+	}
+
+	if l.lastEnd != l.line-1 {
+		l.pending = nil
+	}
+	l.pending = append(l.pending, text)
+	l.lastEnd = l.line
+}
+
+func (l *lexer) blockComment() error {
+	startLine := l.line
+	l.pos += 2
+	begin := l.pos
+	for {
+		if l.pos+1 >= len(l.src) {
+			return fmt.Errorf("line %d: block comment is never closed", startLine)
+		}
+		if l.src[l.pos] == '*' && l.src[l.pos+1] == '/' {
+			break
+		}
+		if l.src[l.pos] == '\n' {
+			l.line++
+		}
+		l.pos++
+	}
+	body := l.src[begin:l.pos]
+	l.pos += 2
+
+	var lines []string
+	for _, raw := range strings.Split(body, "\n") {
+		line := strings.TrimSpace(raw)
+		line = strings.TrimPrefix(line, "*")
+		lines = append(lines, strings.TrimSpace(line))
+	}
+	for len(lines) > 0 && lines[0] == "" {
+		lines = lines[1:]
+	}
+	for len(lines) > 0 && lines[len(lines)-1] == "" {
+		lines = lines[:len(lines)-1]
+	}
+
+	if l.lastEnd != startLine-1 {
+		l.pending = nil
+	}
+	l.pending = append(l.pending, lines...)
+	l.lastEnd = l.line
+
+	return nil
+}
+
+func (l *lexer) str() (string, error) {
+	quote := l.src[l.pos]
+	startLine := l.line
+	l.pos++
+	var b strings.Builder
+	for {
+		if l.pos >= len(l.src) {
+			return "", fmt.Errorf("line %d: string is never closed", startLine)
+		}
+		c := l.src[l.pos]
+		if c == quote {
+			l.pos++
+
+			return b.String(), nil
+		}
+		if c == '\n' {
+			return "", fmt.Errorf("line %d: string is never closed", startLine)
+		}
+		if c == '\\' && l.pos+1 < len(l.src) {
+			// The escape is kept as written. Nothing downstream interprets a
+			// proto string literal; they are option values and default values,
+			// and both are shown rather than evaluated.
+			b.WriteByte(c)
+			l.pos++
+			b.WriteByte(l.src[l.pos])
+			l.pos++
+
+			continue
+		}
+		b.WriteByte(c)
+		l.pos++
+	}
+}
+
+func (l *lexer) skipSpace() {
+	for l.pos < len(l.src) {
+		c := l.src[l.pos]
+		if c == '\n' {
+			l.line++
+			l.pos++
+
+			continue
+		}
+		r, size := utf8.DecodeRuneInString(l.src[l.pos:])
+		if !unicode.IsSpace(r) {
+			return
+		}
+		l.pos += size
+	}
+}
+
+func isIdentStart(c byte) bool {
+	return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
+func isIdentPart(c byte) bool {
+	return isIdentStart(c) || (c >= '0' && c <= '9')
+}
+
+func isNumberPart(c byte) bool {
+	return (c >= '0' && c <= '9') || c == '.' || c == 'x' || c == 'X' ||
+		(c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || c == '-' || c == '+'
+}

package/plugins/extract-proto/main.go

@@ -0,0 +1,100 @@
+// Package main is portolan-extract-proto: .proto files in, a catalog fragment
+// out.
+//
+// It is a `process` plugin because it walks the input tree, and it would be a
+// wasm one the day the host preopens `Input.Root`. Nothing in it reads a clock,
+// opens a socket or looks at the environment, and nothing should: fetching a
+// module from a registry belongs to portolan-fetch-bsr, and keeping the two
+// apart is what lets this one be replayed byte-for-byte from a checkout.
+//
+// Wired into portolan.json for the order contract oms publishes; the shape of
+// a step, with a vendored copy as well:
+//
+//	{
+//	  "plugins": [
+//	    { "name": "proto", "process": { "command": "go", "args": ["run", "./plugins/extract-proto"] } }
+//	  ],
+//	  "extract": [
+//	    {
+//	      "plugin": "proto",
+//	      "in": "examples/shop",
+//	      "out": "examples/shop/portolan",
+//	      "options": {
+//	        "context": "shop",
+//	        "service": "oms",
+//	        "paths": ["proto"],
+//	        "vendored": ["internal/infrastructure/pricing"],
+//	        "peers": { "shop.v1": "shop.pricing" },
+//	        "out": "proto.json"
+//	      }
+//	    }
+//	  ]
+//	}
+//
+// Declared AFTER any fetch-bsr step that writes into the same tree: extract
+// steps run in list order, so a fetch declared first has already written its
+// protos and its lock by the time this reads them.
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// Options are what the manifest tells the extractor: the things a .proto does
+// not say about the estate it belongs to.
+type Options struct {
+	Context string `json:"context"`
+	Service string `json:"service"`
+
+	// Paths are directories of protos this service PUBLISHES, relative to the
+	// input root. They become what it provides.
+	Paths []string `json:"paths,omitempty"`
+
+	// Vendored are directories of client copies this service CONSUMES. They
+	// become what it calls.
+	//
+	// An explicit list rather than a glob: which copies a service is allowed to
+	// keep is an architectural boundary, and a boundary spelled out in the
+	// manifest is one a reviewer can see change.
+	Vendored []string `json:"vendored,omitempty"`
+
+	// Peers maps a proto package to the service id that answers on it.
+	//
+	// The only way a consumer-side extractor can name a peer. Without an entry
+	// the call keeps the raw package name, which is exactly what `RpcCall.peer`
+	// documents - guessing a service id from a package would be inventing an
+	// edge between two services.
+	Peers map[string]string `json:"peers,omitempty"`
+
+	// Modules maps a directory, relative to the input root, to the module id
+	// its protos belong to. Only needed where there is no lock file to say.
+	Modules map[string]string `json:"modules,omitempty"`
+
+	// Defs is "shared" (the default) or "off". Shared promotes a message the
+	// author put in another file to a catalog.defs entry, on the reasoning that
+	// an import is their own signal the type is not local.
+	Defs string `json:"defs,omitempty"`
+
+	Out string `json:"out,omitempty"`
+}
+
+func main() {
+	if err := run(os.Stdin, os.Stdout); err != nil {
+		fmt.Fprintln(os.Stderr, "portolan-extract-proto:", err)
+		os.Exit(1)
+	}
+}
+
+func run(stdin io.Reader, stdout io.Writer) error {
+	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
+		if req.Input.Root == "" {
+			return plugin.Response{}, fmt.Errorf("no input root: an extractor has nothing to read")
+		}
+
+		return extract(req.Input, opts)
+	})
+}

package/plugins/extract-proto/module.go

@@ -0,0 +1,120 @@
+package main
+
+// Where a set of .proto files came from.
+//
+// Two answers are possible and both are honest. A directory written into by
+// `fetch-bsr` has a `bsr.lock.json` beside it saying which module and which
+// commit; a directory of protos the estate simply keeps has neither, and gets a
+// `local:` id so nobody mistakes the path for something they could go and
+// fetch.
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/shortlink-org/portolan/catalog"
+)
+
+// lockFile is the shape fetch-bsr writes. Read here rather than shared, because
+// the two plugins are separate programs on purpose: one may run without the
+// other ever having existed.
+type lockFile struct {
+	Modules []lockModule `json:"modules"`
+}
+
+type lockModule struct {
+	Module string      `json:"module"`
+	Commit string      `json:"commit"`
+	Digest string      `json:"digest"`
+	Deps   []string    `json:"deps,omitempty"`
+	Files  []lockEntry `json:"files"`
+}
+
+type lockEntry struct {
+	Path string `json:"path"`
+}
+
+const lockName = "bsr.lock.json"
+
+// readLock loads the lock beside a directory of protos, if there is one.
+func readLock(dir string) (*lockFile, error) {
+	raw, err := os.ReadFile(filepath.Join(dir, lockName))
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var lock lockFile
+	if err := json.Unmarshal(raw, &lock); err != nil {
+		return nil, err
+	}
+
+	return &lock, nil
+}
+
+// moduleFor describes the set of protos under one directory.
+//
+// packages and files come from what was actually parsed rather than from the
+// lock, because the lock says what was fetched and this says what portolan
+// could read - and when they disagree, the second is the one a reader needs.
+func moduleFor(id, dir string, files []*File) catalog.ProtoModule {
+	packages := map[string]bool{}
+	paths := make([]string, 0, len(files))
+	for _, file := range files {
+		if file.Package != "" {
+			packages[file.Package] = true
+		}
+		paths = append(paths, relativeTo(dir, file.Path))
+	}
+
+	module := catalog.ProtoModule{
+		ID:       id,
+		Slug:     moduleSlug(id),
+		Name:     moduleName(id),
+		Registry: registryOf(id),
+		Packages: sortedKeys(packages),
+		Files:    sortedCopy(paths),
+		Source:   filepath.ToSlash(dir),
+	}
+
+	return module
+}
+
+func relativeTo(dir, path string) string {
+	rel, err := filepath.Rel(dir, path)
+	if err != nil {
+		return filepath.ToSlash(path)
+	}
+
+	return filepath.ToSlash(rel)
+}
+
+// sortedKeys and sortedCopy exist because the fragment is committed and
+// compared by `gen:check`. A map iterated in Go's order writes a different file
+// every run, which would turn every build into a diff.
+func sortedKeys(set map[string]bool) []string {
+	out := make([]string, 0, len(set))
+	for k := range set {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+
+	return out
+}
+
+func sortedCopy(in []string) []string {
+	out := append([]string(nil), in...)
+	sort.Strings(out)
+
+	return out
+}
+
+// trimSlash keeps a directory option comparable however it was written.
+func trimSlash(dir string) string {
+	return strings.TrimSuffix(filepath.ToSlash(dir), "/")
+}

package/plugins/extract-proto/options.schema.json

@@ -0,0 +1,51 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["context", "service"],
+  "properties": {
+    "context": {
+      "type": "string",
+      "description": "Slug of the bounded context the service belongs to. A .proto knows its package, not which context owns it.",
+      "examples": ["shop"]
+    },
+    "service": {
+      "type": "string",
+      "description": "Slug of the service that publishes or calls these protos.",
+      "examples": ["oms"]
+    },
+    "paths": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Directories of protos this service publishes, relative to the input root. They become what it provides. A directory fetch-bsr wrote carries its lock, and the lock says which module the protos came from.",
+      "examples": [["vendor/proto/shortlink-org/portolan-shop-order"]]
+    },
+    "vendored": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Directories of client copies this service consumes, relative to the input root. They become what it calls. Listed rather than globbed: which copies a service may keep is a boundary a reviewer should see change.",
+      "examples": [["src/infrastructure/payments/proto"]]
+    },
+    "peers": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Which service answers for a vendored copy, as the proto package to its service id. A package with no line here is called as unknown and its steps stay unresolved.",
+      "examples": [{ "payments.v1": "payments.ledger" }]
+    },
+    "modules": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Which module a directory's protos belong to, as the directory relative to the input root to the module id. Only needed where there is no lock beside them to say.",
+      "examples": [{ "src/infrastructure/payments/proto": "buf.build/shortlink-org/portolan-payments-payment" }]
+    },
+    "defs": {
+      "enum": ["shared", "off"],
+      "description": "Whether a message imported from another file is promoted to a shared type, on the reasoning that the import is the author's own signal it is not local.",
+      "default": "shared"
+    },
+    "out": {
+      "type": "string",
+      "description": "Name of the fragment file, inside the step's output directory.",
+      "default": "proto.json"
+    }
+  }
+}