@shortlink-org/portolan 0.1.0

package/plugins/extract-proto/parse_test.go

@@ -0,0 +1,307 @@
+package main
+
+// The parser, on its own. Nothing here knows the catalog exists: a proto is
+// read into an AST, and whether that AST becomes a service or a message is
+// somebody else's question.
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func parseFile(t *testing.T, name string) (*File, []Note) {
+	t.Helper()
+
+	src, err := os.ReadFile("testdata/" + name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	file, notes, err := Parse("testdata/"+name, string(src))
+	if err != nil {
+		t.Fatalf("parsing %s: %v", name, err)
+	}
+
+	return file, notes
+}
+
+func orders(t *testing.T) *File {
+	t.Helper()
+	file, notes := parseFile(t, "orders.proto")
+	if len(notes) > 0 {
+		t.Errorf("a well-formed proto3 file produced notes: %v", notes)
+	}
+
+	return file
+}
+
+func TestFileHeader(t *testing.T) {
+	file := orders(t)
+
+	if file.Syntax != "proto3" {
+		t.Errorf("syntax: %q", file.Syntax)
+	}
+	if file.Package != "shop.v1" {
+		t.Errorf("package: %q", file.Package)
+	}
+	if len(file.Imports) != 2 {
+		t.Fatalf("imports: %v", file.Imports)
+	}
+	if file.Imports[0].Path != "shop/v1/money.proto" || file.Imports[0].Public {
+		t.Errorf("plain import: %+v", file.Imports[0])
+	}
+	if !file.Imports[1].Public {
+		t.Errorf("public import was not marked public: %+v", file.Imports[1])
+	}
+	if len(file.Options) != 1 || file.Options[0].Name != "go_package" {
+		t.Errorf("options: %+v", file.Options)
+	}
+}
+
+// All four streaming forms, because a bidi method drawn as unary is a lie about
+// how the two ends are coupled.
+func TestStreaming(t *testing.T) {
+	svc := orders(t).Services[0]
+
+	want := map[string][2]bool{
+		"PlaceOrder":   {false, false},
+		"GetOrder":     {false, false},
+		"WatchOrder":   {false, true},
+		"ImportOrders": {true, false},
+		"Sync":         {true, true},
+		"CancelOrder":  {false, false},
+	}
+
+	if len(svc.Methods) != len(want) {
+		t.Fatalf("methods: %d, want %d", len(svc.Methods), len(want))
+	}
+	for _, m := range svc.Methods {
+		got := [2]bool{m.ClientStreaming, m.ServerStreaming}
+		if got != want[m.Name] {
+			t.Errorf("%s streams %v, want %v", m.Name, got, want[m.Name])
+		}
+	}
+}
+
+func TestMethodShapes(t *testing.T) {
+	svc := orders(t).Services[0]
+
+	place := svc.Methods[0]
+	if place.Request != "PlaceOrderRequest" || place.Response != "PlaceOrderResponse" {
+		t.Errorf("PlaceOrder: %q -> %q", place.Request, place.Response)
+	}
+	watch := svc.Methods[2]
+	if watch.Response != "OrderEvent" {
+		t.Errorf("a streamed response must name the message, not the stream: %q", watch.Response)
+	}
+}
+
+// A method option body must not swallow the methods after it.
+func TestMethodOptionBody(t *testing.T) {
+	svc := orders(t).Services[0]
+
+	cancel := svc.Methods[5]
+	if cancel.Name != "CancelOrder" {
+		t.Fatalf("last method: %q", cancel.Name)
+	}
+	if !cancel.Deprecated {
+		t.Error("a method with `option deprecated = true` is not marked deprecated")
+	}
+}
+
+// Comments are the only documentation a proto carries, and they arrive three
+// different ways.
+func TestDocComments(t *testing.T) {
+	file := orders(t)
+	svc := file.Services[0]
+
+	if !strings.HasPrefix(svc.Doc, "The order interface.") {
+		t.Errorf("block comment above a service: %q", svc.Doc)
+	}
+	if svc.Methods[0].Doc != "Place a new order. The response carries the id the shop assigned." {
+		t.Errorf("line comment above a method: %q", svc.Methods[0].Doc)
+	}
+	if svc.Methods[1].Doc != "Read one order back." {
+		t.Errorf("trailing comment on a method: %q", svc.Methods[1].Doc)
+	}
+
+	place := messageNamed(t, file, "PlaceOrderRequest")
+	if place.Fields[0].Doc != "Who is buying." {
+		t.Errorf("comment above a field: %q", place.Fields[0].Doc)
+	}
+	// The field after a documented one must not inherit its comment.
+	if place.Fields[1].Doc != "" {
+		t.Errorf("an undocumented field borrowed a comment: %q", place.Fields[1].Doc)
+	}
+}
+
+// Types are kept as written, because a reader comparing the page against the
+// file it came from should recognise what they are looking at.
+func TestFieldTypesAsWritten(t *testing.T) {
+	place := messageNamed(t, orders(t), "PlaceOrderRequest")
+
+	want := []struct{ name, typ, label string }{
+		{"customer_id", "string", ""},
+		{"items", "LineItem", "repeated"},
+		{"coupon", "string", "optional"},
+		{"metadata", "map<string, string>", ""},
+		{"total", "Money", ""},
+	}
+
+	if len(place.Fields) != len(want) {
+		t.Fatalf("fields: %d, want %d", len(place.Fields), len(want))
+	}
+	for i, w := range want {
+		f := place.Fields[i]
+		if f.Name != w.name || f.Type != w.typ || f.Label != w.label {
+			t.Errorf("field %d: %s %s %s", i, f.Label, f.Type, f.Name)
+		}
+		if f.Number != i+1 {
+			t.Errorf("%s is field %d, want %d", f.Name, f.Number, i+1)
+		}
+	}
+	if len(place.Reserved) != 3 {
+		t.Errorf("reserved: %v", place.Reserved)
+	}
+}
+
+// A oneof constrains which field may be set; it is not a separate place a field
+// lives. So its members are in the message's field list AND named by the oneof.
+func TestOneofMembersAreFields(t *testing.T) {
+	order := messageNamed(t, orders(t), "Order")
+
+	if len(order.Oneofs) != 1 || order.Oneofs[0].Name != "settlement" {
+		t.Fatalf("oneofs: %+v", order.Oneofs)
+	}
+	if len(order.Oneofs[0].Fields) != 2 {
+		t.Fatalf("oneof members: %+v", order.Oneofs[0].Fields)
+	}
+
+	byName := map[string]*Field{}
+	for _, f := range order.Fields {
+		byName[f.Name] = f
+	}
+	if byName["paid"] == nil || byName["paid"].Oneof != "settlement" {
+		t.Error("a oneof member is missing from the message's fields, or does not name its oneof")
+	}
+	if byName["ship_to"] == nil || byName["ship_to"].Oneof != "" {
+		t.Error("a plain field claims to be in a oneof")
+	}
+	// The fields declared after the oneof must still be there.
+	if byName["channel"] == nil {
+		t.Error("a field declared after a oneof was swallowed")
+	}
+}
+
+func TestNestedTypes(t *testing.T) {
+	order := messageNamed(t, orders(t), "Order")
+
+	if len(order.Messages) != 1 || order.Messages[0].Name != "Address" {
+		t.Fatalf("nested messages: %+v", order.Messages)
+	}
+	if !order.Messages[0].Fields[1].Deprecated {
+		t.Error("a field with [deprecated = true] is not marked deprecated")
+	}
+	if len(order.Enums) != 1 || order.Enums[0].Name != "Channel" {
+		t.Fatalf("nested enums: %+v", order.Enums)
+	}
+}
+
+func TestTopLevelEnum(t *testing.T) {
+	file := orders(t)
+
+	if len(file.Enums) != 1 {
+		t.Fatalf("enums: %+v", file.Enums)
+	}
+	status := file.Enums[0]
+	if status.Doc != "How far along an order is." {
+		t.Errorf("enum doc: %q", status.Doc)
+	}
+	if len(status.Values) != 2 || status.Values[1].Number != 1 {
+		t.Fatalf("values: %+v", status.Values)
+	}
+	if status.Values[1].Doc != "The shop has it." {
+		t.Errorf("trailing comment on an enum value: %q", status.Values[1].Doc)
+	}
+}
+
+// The tolerance rule: a construct the parser declines to model is NAMED and
+// read past, never dropped silently and never fatal. A vendored copy that has
+// been narrowed is the normal case, not the exception.
+func TestSkippedConstructsAreNamed(t *testing.T) {
+	file, notes := parseFile(t, "awkward.proto")
+
+	joined := strings.Join(messages(notes), "\n")
+	for _, want := range []string{"extend", "group", "extensions"} {
+		if !strings.Contains(joined, want) {
+			t.Errorf("%q was skipped without saying so:\n%s", want, joined)
+		}
+	}
+
+	// Everything around the skipped constructs still has to be read.
+	if len(file.Services) != 1 || len(file.Services[0].Methods) != 1 {
+		t.Errorf("a service after an unreadable construct was lost: %+v", file.Services)
+	}
+	old := messageNamed(t, file, "Old")
+	byName := map[string]*Field{}
+	for _, f := range old.Fields {
+		byName[f.Name] = f
+	}
+	if byName["name"] == nil || byName["name"].Label != "required" {
+		t.Error("a proto2 required field was not read")
+	}
+	if byName["count"] == nil || byName["count"].Default != "3" {
+		t.Errorf("a default value was not read: %+v", byName["count"])
+	}
+	if byName["kept"] == nil {
+		t.Error("the field after a group was swallowed with it")
+	}
+}
+
+// Past a truncated file the parser is not reading proto any more, it is
+// guessing - so it says so rather than returning half a file.
+func TestTruncatedFile(t *testing.T) {
+	_, _, err := Parse("broken.proto", "message Foo {\n  string a = 1;\n  // and then\n")
+	if err == nil {
+		t.Skip("an unterminated block is recoverable; only an unterminated token is not")
+	}
+}
+
+func TestUnterminatedStringIsAnError(t *testing.T) {
+	_, _, err := Parse("broken.proto", "option go_package = \"never closed;\n")
+	if err == nil {
+		t.Fatal("an unterminated string parsed cleanly")
+	}
+	if !strings.Contains(err.Error(), "broken.proto") {
+		t.Errorf("the error does not name the file: %v", err)
+	}
+}
+
+func TestUnterminatedBlockCommentIsAnError(t *testing.T) {
+	_, _, err := Parse("broken.proto", "/* never closed\nmessage Foo {}\n")
+	if err == nil {
+		t.Fatal("an unterminated block comment parsed cleanly")
+	}
+}
+
+func messageNamed(t *testing.T, file *File, name string) *Message {
+	t.Helper()
+
+	for _, m := range file.Messages {
+		if m.Name == name {
+			return m
+		}
+	}
+	t.Fatalf("no message %q", name)
+
+	return nil
+}
+
+func messages(notes []Note) []string {
+	out := make([]string, 0, len(notes))
+	for _, n := range notes {
+		out = append(out, n.Message)
+	}
+
+	return out
+}

package/plugins/extract-proto/provides.go

@@ -0,0 +1,236 @@
+package main
+
+// What a service ANSWERS, read from the protos it publishes.
+//
+// One rpc service becomes one catalog interface; one rpc method becomes one
+// catalog method, carrying the two shapes it moves and how it streams. The
+// messages listed alongside are the ones REACHABLE from those methods, walked
+// transitively - the same rule extract-openapi applies to schemas, for the same
+// reason its comment gives: a reader who has to follow three links to find out
+// what comes back has been given a worse document.
+
+import (
+	"path/filepath"
+	"sort"
+	"strconv"
+
+	"github.com/shortlink-org/portolan/catalog"
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// shared collects the messages promoted to catalog.defs, by their bare name.
+type shared struct {
+	defs map[string]catalog.TypeDef
+	// from records which fully-qualified type each def key was taken from, so a
+	// second message with the same short name can be spotted rather than
+	// silently overwriting the first.
+	from map[string]string
+	on   bool
+}
+
+func newShared(on bool) *shared {
+	return &shared{defs: map[string]catalog.TypeDef{}, from: map[string]string{}, on: on}
+}
+
+// interfaces turns every rpc service in the owned files into a catalog entry.
+func interfaces(ix *Index, files []*File, moduleID string, sh *shared, b *plugin.Builder) []catalog.RpcService {
+	var out []catalog.RpcService
+
+	for _, file := range files {
+		for _, svc := range file.Services {
+			id := interfaceID(file.Package, svc.Name)
+
+			provided := catalog.RpcService{
+				ID:      id,
+				Methods: make([]catalog.RpcMethod, 0, len(svc.Methods)),
+				Source:  sourceRef(file.Path, svc.Line),
+				Module:  moduleID,
+			}
+
+			// The messages this interface moves, in the order they are first
+			// reached: the request and response of each method, then whatever
+			// those refer to.
+			var queue []string
+			seen := map[string]bool{}
+
+			for _, m := range svc.Methods {
+				method := catalog.RpcMethod{
+					Name:       m.Name,
+					Doc:        m.Doc,
+					Deprecated: m.Deprecated,
+					Streaming:  streamingOf(m),
+				}
+
+				method.Request, method.RequestRef = shapeRef(ix, file, m.Request, sh, &queue, seen, id, b)
+				method.Response, method.ResponseRef = shapeRef(ix, file, m.Response, sh, &queue, seen, id, b)
+
+				provided.Methods = append(provided.Methods, method)
+			}
+
+			provided.Messages, provided.Enums = messagesFrom(ix, queue, seen, sh, id, b)
+			out = append(out, provided)
+		}
+	}
+
+	// Sorted, because the fragment is committed and compared by gen:check and
+	// the file walk that produced these is only as ordered as the filesystem.
+	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
+
+	return out
+}
+
+// streamingOf maps a method's two stream flags onto the catalog's one value.
+func streamingOf(m *Method) catalog.Streaming {
+	switch {
+	case m.ClientStreaming && m.ServerStreaming:
+		return catalog.StreamingBidi
+	case m.ClientStreaming:
+		return catalog.StreamingClient
+	case m.ServerStreaming:
+		return catalog.StreamingServer
+	default:
+		// Unary. Empty rather than a fourth constant, because unary is the case
+		// not worth writing down and the catalog says so by leaving it out.
+		return ""
+	}
+}
+
+// shapeRef names a method's request or response and queues it for listing.
+//
+// It returns the name as a reader says it and the defs key, when the type is
+// one the author shared through an import.
+func shapeRef(ix *Index, from *File, name string, sh *shared, queue *[]string, seen map[string]bool, where string, b *plugin.Builder) (string, string) {
+	if name == "" {
+		return "", ""
+	}
+
+	fqn, ok := ix.Resolve(from.Package, name)
+	if !ok {
+		// Not a failure. A narrowed vendored copy imports files nobody
+		// vendored beside it, which is the whole reason this reader is tolerant.
+		b.Warn(where, name+" is not declared in the protos read here; it is named but not described")
+
+		return shortName(name), ""
+	}
+
+	if !seen[fqn] {
+		seen[fqn] = true
+		*queue = append(*queue, fqn)
+	}
+
+	// Shared by the same rule as a field's type: the author put it in another
+	// file, which is their own signal that it is not local to this interface.
+	ref := ""
+	if sh.on && ix.FileOf(fqn) != from {
+		ref = sh.promote(ix, fqn, where, b)
+	}
+
+	return shortName(fqn), ref
+}
+
+// messagesFrom walks the queue, describing each message and adding whatever its
+// fields refer to, until nothing new is reached. An enum a field names is
+// reached the same way and listed beside the messages: a consumer decoding
+// the field sees a number, and the enum is what the number means.
+func messagesFrom(ix *Index, queue []string, seen map[string]bool, sh *shared, where string, b *plugin.Builder) ([]catalog.RpcMessage, []catalog.RpcEnum) {
+	var out []catalog.RpcMessage
+	var enums []catalog.RpcEnum
+
+	for i := 0; i < len(queue); i++ {
+		fqn := queue[i]
+		msg := ix.Message(fqn)
+		if msg == nil {
+			if e := ix.enums[fqn]; e != nil {
+				enums = append(enums, enumOf(fqn, e))
+			}
+			// Otherwise something only an import declares: not a message,
+			// not a set, and not for this list.
+			continue
+		}
+
+		fields := make([]catalog.Field, 0, len(msg.Fields))
+		for _, f := range msg.Fields {
+			ref := ix.resolveType(fqn, f)
+			if ref.Unresolved != "" {
+				b.Warn(where, ref.Unresolved+" is not declared in the protos read here; "+
+					shortName(fqn)+"."+f.Name+" is listed with the type as written")
+			}
+
+			field := catalog.Field{Name: f.Name, Type: ref.Written, Doc: f.Doc}
+
+			if ref.FQN != "" {
+				if !seen[ref.FQN] {
+					seen[ref.FQN] = true
+					queue = append(queue, ref.FQN)
+				}
+				// A type is SHARED when the author put it in another file - an
+				// import is their own signal that it is not local to this
+				// message. No reference counting, no heuristic.
+				if sh.on && ix.FileOf(ref.FQN) != ix.FileOf(fqn) {
+					field.Ref = sh.promote(ix, ref.FQN, where, b)
+				}
+			}
+
+			fields = append(fields, field)
+		}
+
+		out = append(out, catalog.RpcMessage{Name: shortName(fqn), Fields: fields})
+	}
+
+	return out, enums
+}
+
+// enumOf is an enum as the catalog carries it: the values in declaration
+// order with the numbers the wire uses. Named by its bare name, the way a
+// nested message is: a reader says `Channel`, and the file says where.
+func enumOf(fqn string, e *Enum) catalog.RpcEnum {
+	values := make([]catalog.RpcEnumValue, 0, len(e.Values))
+	for _, v := range e.Values {
+		values = append(values, catalog.RpcEnumValue{Name: v.Name, Number: v.Number, Doc: v.Doc})
+	}
+
+	return catalog.RpcEnum{Name: shortName(fqn), Doc: e.Doc, Values: values}
+}
+
+// promote records a message as a shared type and returns its defs key.
+//
+// The key is the message's BARE name - `Money`, not `shop.v1.Money`. That is
+// what src/merge.ts argues for at length: bare keys are deliberate, so that two
+// sources meaning different things by `Money` produce a visible conflict
+// instead of two quietly coexisting entries. Fully-qualified keys would hide
+// exactly the problem the merge exists to expose.
+func (sh *shared) promote(ix *Index, fqn, where string, b *plugin.Builder) string {
+	key := shortName(fqn)
+
+	if previous, ok := sh.from[key]; ok {
+		if previous != fqn {
+			b.Warn(where, "two different types are both called "+key+" ("+previous+" and "+fqn+
+				"); the first one read is the shared type")
+		}
+
+		return key
+	}
+
+	msg := ix.Message(fqn)
+	if msg == nil {
+		return ""
+	}
+
+	fields := make([]catalog.Field, 0, len(msg.Fields))
+	for _, f := range msg.Fields {
+		ref := ix.resolveType(fqn, f)
+		fields = append(fields, catalog.Field{Name: f.Name, Type: ref.Written, Doc: f.Doc})
+	}
+
+	sh.from[key] = fqn
+	sh.defs[key] = catalog.TypeDef{Fields: fields}
+
+	return key
+}
+
+// sourceRef is where a declaration lives, as a reader would type it:
+// `proto/shop/v1/orders.proto:12`. Already the format `RpcService.source`
+// documents.
+func sourceRef(path string, line int) string {
+	return filepath.ToSlash(path) + ":" + strconv.Itoa(line)
+}