@shortlink-org/portolan 0.1.0 → 0.2.1

package/plugins/extract-proto/parse.go

@@ -1,720 +0,0 @@
-package main
-
-// A tolerant recursive-descent parser for .proto.
-//
-// TOLERANT is the whole design, not a shortcut. docs/adr/org.0001.md has
-// consumers keeping NARROWED vendored copies - "a consumer may narrow a message
-// to the fields it actually uses" - and a narrowed copy routinely imports a file
-// that was not vendored beside it. A compiler refuses to produce anything for
-// that input; the point of reading vendored copies is to describe files that do
-// not build standalone. So an unrecognised construct is skipped to the end of
-// its block and NAMED in a Note, never dropped and never fatal.
-//
-// The two things that are fatal are a file that cannot be tokenised and a file
-// that ends mid-declaration, because past either of those the parser is not
-// reading proto any more, it is guessing.
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-)
-
-type parser struct {
-	toks  []token
-	at    int
-	path  string
-	notes []Note
-}
-
-// Parse reads one .proto. The returned notes name every construct it declined
-// to model; an error means the file could not be read at all.
-func Parse(path, src string) (*File, []Note, error) {
-	out, err := lex(src)
-	if err != nil {
-		return nil, nil, fmt.Errorf("%s: %w", path, err)
-	}
-
-	p := &parser{toks: out.tokens, path: path}
-	file := &File{Path: path, Doc: p.peek().doc}
-
-	for !p.done() {
-		if p.peek().is(";") { // a stray semicolon between declarations is legal
-			p.take()
-
-			continue
-		}
-
-		tok := p.peek()
-		if tok.kind != tokIdent {
-			p.note(tok.line, fmt.Sprintf("unexpected %q at the top level; skipped", tok.text))
-			p.skipDeclaration()
-
-			continue
-		}
-
-		switch tok.text {
-		case "syntax", "edition":
-			p.take()
-			if !p.expect("=") {
-				continue
-			}
-			if v := p.take(); v.kind == tokString {
-				file.Syntax = v.text
-			}
-			p.expect(";")
-		case "package":
-			p.take()
-			file.Package = p.qualified()
-			p.expect(";")
-		case "import":
-			file.Imports = append(file.Imports, p.parseImport())
-		case "option":
-			if o, ok := p.parseOption(); ok {
-				file.Options = append(file.Options, o)
-			}
-		case "message":
-			if m := p.parseMessage(); m != nil {
-				file.Messages = append(file.Messages, m)
-			}
-		case "enum":
-			if e := p.parseEnum(); e != nil {
-				file.Enums = append(file.Enums, e)
-			}
-		case "service":
-			if s := p.parseService(); s != nil {
-				file.Services = append(file.Services, s)
-			}
-		case "extend":
-			// Modelled by nothing in the catalog: an extension adds fields to
-			// someone else's message, and the catalog has no way to say that.
-			p.note(tok.line, "extend is not read; the fields it adds are not listed")
-			p.take()
-			p.qualified()
-			p.skipBlock()
-		default:
-			p.note(tok.line, fmt.Sprintf("%q is not a declaration this parser reads; skipped", tok.text))
-			p.skipDeclaration()
-		}
-	}
-
-	// Trailing comments are attached last, by line, so a field documented to
-	// its right reads the same as one documented above.
-	attachTrailing(file, out.trailing)
-
-	return file, p.notes, nil
-}
-
-func (p *parser) parseImport() Import {
-	tok := p.take() // "import"
-	imp := Import{Line: tok.line}
-	switch {
-	case p.peek().is("public"):
-		p.take()
-		imp.Public = true
-	case p.peek().is("weak"):
-		p.take()
-		imp.Weak = true
-	}
-	if v := p.peek(); v.kind == tokString {
-		imp.Path = p.take().text
-	}
-	p.expect(";")
-
-	return imp
-}
-
-func (p *parser) parseOption() (Option, bool) {
-	tok := p.take() // "option"
-	name := p.optionName()
-	if !p.expect("=") {
-		return Option{}, false
-	}
-	value := p.optionValue()
-	p.expect(";")
-
-	return Option{Name: name, Value: value, Line: tok.line}, true
-}
-
-// optionName reads a name that may be parenthesised and dotted:
-// `go_package`, `(google.api.http).post`.
-func (p *parser) optionName() string {
-	var b strings.Builder
-	for {
-		switch {
-		case p.peek().is("("):
-			p.take()
-			b.WriteString("(")
-			b.WriteString(p.qualified())
-			if p.peek().is(")") {
-				p.take()
-			}
-			b.WriteString(")")
-		case p.peek().kind == tokIdent:
-			b.WriteString(p.take().text)
-		case p.peek().is("."):
-			p.take()
-			b.WriteString(".")
-		default:
-			return b.String()
-		}
-		if !p.peek().is(".") {
-			return b.String()
-		}
-	}
-}
-
-// optionValue reads a scalar value, or skips an aggregate `{ ... }` body.
-func (p *parser) optionValue() string {
-	tok := p.peek()
-	if tok.is("{") {
-		// A custom option body is arbitrary and modelled by nothing here.
-		p.note(tok.line, "an aggregate option value is not read")
-		p.skipBlock()
-
-		return ""
-	}
-	if tok.is("[") {
-		p.skipBrackets()
-
-		return ""
-	}
-
-	var b strings.Builder
-	for !p.done() && !p.peek().is(";") && !p.peek().is(",") && !p.peek().is("]") {
-		t := p.take()
-		if t.kind == tokString {
-			b.WriteString(t.text)
-
-			continue
-		}
-		b.WriteString(t.text)
-	}
-
-	return b.String()
-}
-
-func (p *parser) parseMessage() *Message {
-	tok := p.take() // "message"
-	name := p.take()
-	if name.kind != tokIdent {
-		p.note(tok.line, "message has no name; skipped")
-		p.skipDeclaration()
-
-		return nil
-	}
-
-	msg := &Message{Name: name.text, Doc: tok.doc, Line: tok.line}
-	if !p.expect("{") {
-		return msg
-	}
-	p.messageBody(msg, "")
-
-	return msg
-}
-
-// messageBody reads declarations up to the closing brace, which it consumes.
-// oneof is the name of the oneof enclosing these fields, empty at the top.
-func (p *parser) messageBody(msg *Message, oneof string) {
-	for !p.done() && !p.peek().is("}") {
-		tok := p.peek()
-
-		if tok.is(";") {
-			p.take()
-
-			continue
-		}
-
-		if tok.kind != tokIdent {
-			p.note(tok.line, fmt.Sprintf("unexpected %q in message %q; skipped", tok.text, msg.Name))
-			p.skipDeclaration()
-
-			continue
-		}
-
-		switch tok.text {
-		case "message":
-			if nested := p.parseMessage(); nested != nil {
-				msg.Messages = append(msg.Messages, nested)
-			}
-		case "enum":
-			if e := p.parseEnum(); e != nil {
-				msg.Enums = append(msg.Enums, e)
-			}
-		case "option":
-			p.parseOption()
-		case "reserved":
-			msg.Reserved = append(msg.Reserved, p.parseReserved()...)
-		case "extensions":
-			p.note(tok.line, "extensions ranges are not read")
-			p.skipDeclaration()
-		case "extend":
-			p.note(tok.line, "extend is not read; the fields it adds are not listed")
-			p.take()
-			p.qualified()
-			p.skipBlock()
-		case "oneof":
-			p.parseOneof(msg)
-		case "group":
-			p.note(tok.line, "a proto2 group is not read; its fields are not listed")
-			p.skipDeclaration()
-		default:
-			if f := p.parseField(oneof); f != nil {
-				msg.Fields = append(msg.Fields, f)
-			}
-		}
-	}
-	p.expect("}")
-}
-
-func (p *parser) parseOneof(msg *Message) {
-	tok := p.take() // "oneof"
-	name := p.take()
-	if name.kind != tokIdent {
-		p.note(tok.line, "oneof has no name; skipped")
-		p.skipDeclaration()
-
-		return
-	}
-	one := &Oneof{Name: name.text, Doc: tok.doc, Line: tok.line}
-	if !p.expect("{") {
-		return
-	}
-
-	// A oneof body is a message body with one field kind excluded, and reusing
-	// the same reader is what keeps a oneof member and a plain field the same
-	// shape - which is the whole reason `Message.Fields` holds both.
-	before := len(msg.Fields)
-	p.messageBody(msg, one.Name)
-	one.Fields = append(one.Fields, msg.Fields[before:]...)
-	msg.Oneofs = append(msg.Oneofs, one)
-}
-
-func (p *parser) parseField(oneof string) *Field {
-	tok := p.peek()
-	f := &Field{Doc: tok.doc, Line: tok.line, Oneof: oneof}
-
-	switch tok.text {
-	case "repeated", "optional", "required":
-		f.Label = p.take().text
-	}
-
-	if p.peek().is("group") {
-		p.note(p.peek().line, "a proto2 group is not read; its fields are not listed")
-		p.skipDeclaration()
-
-		return nil
-	}
-
-	f.Type = p.typeName()
-	if f.Type == "" {
-		p.note(tok.line, "a field with no type; skipped")
-		p.skipDeclaration()
-
-		return nil
-	}
-
-	name := p.take()
-	if name.kind != tokIdent {
-		p.note(tok.line, fmt.Sprintf("a %s field with no name; skipped", f.Type))
-		p.skipDeclaration()
-
-		return nil
-	}
-	f.Name = name.text
-
-	if p.expect("=") {
-		if n := p.take(); n.kind == tokNumber {
-			f.Number, _ = strconv.Atoi(n.text)
-		}
-	}
-
-	if p.peek().is("[") {
-		p.fieldOptions(f)
-	}
-	p.expect(";")
-
-	return f
-}
-
-// fieldOptions reads `[deprecated = true, default = "x"]`. Only the two the
-// catalog can say anything about are kept; the rest are read past so they
-// cannot desynchronise the parser.
-func (p *parser) fieldOptions(f *Field) {
-	p.take() // "["
-	for !p.done() && !p.peek().is("]") {
-		name := p.optionName()
-		if !p.expect("=") {
-			break
-		}
-		value := p.optionValue()
-		switch name {
-		case "deprecated":
-			f.Deprecated = value == "true"
-		case "default":
-			f.Default = value
-		}
-		if p.peek().is(",") {
-			p.take()
-		}
-	}
-	p.expect("]")
-}
-
-func (p *parser) parseReserved() []string {
-	p.take() // "reserved"
-	var out []string
-	for !p.done() && !p.peek().is(";") {
-		t := p.take()
-		if t.kind == tokString || t.kind == tokNumber {
-			out = append(out, t.text)
-		}
-	}
-	p.expect(";")
-
-	return out
-}
-
-func (p *parser) parseEnum() *Enum {
-	tok := p.take() // "enum"
-	name := p.take()
-	if name.kind != tokIdent {
-		p.note(tok.line, "enum has no name; skipped")
-		p.skipDeclaration()
-
-		return nil
-	}
-
-	e := &Enum{Name: name.text, Doc: tok.doc, Line: tok.line}
-	if !p.expect("{") {
-		return e
-	}
-
-	for !p.done() && !p.peek().is("}") {
-		t := p.peek()
-		if t.is(";") {
-			p.take()
-
-			continue
-		}
-		if t.is("option") {
-			p.parseOption()
-
-			continue
-		}
-		if t.is("reserved") {
-			p.parseReserved()
-
-			continue
-		}
-		if t.kind != tokIdent {
-			p.note(t.line, fmt.Sprintf("unexpected %q in enum %q; skipped", t.text, e.Name))
-			p.skipDeclaration()
-
-			continue
-		}
-
-		value := &EnumValue{Name: p.take().text, Doc: t.doc, Line: t.line}
-		if p.expect("=") {
-			if n := p.take(); n.kind == tokNumber {
-				value.Number, _ = strconv.Atoi(n.text)
-			}
-		}
-		if p.peek().is("[") {
-			p.skipBrackets()
-		}
-		p.expect(";")
-		e.Values = append(e.Values, value)
-	}
-	p.expect("}")
-
-	return e
-}
-
-func (p *parser) parseService() *Service {
-	tok := p.take() // "service"
-	name := p.take()
-	if name.kind != tokIdent {
-		p.note(tok.line, "service has no name; skipped")
-		p.skipDeclaration()
-
-		return nil
-	}
-
-	svc := &Service{Name: name.text, Doc: tok.doc, Line: tok.line}
-	if !p.expect("{") {
-		return svc
-	}
-
-	for !p.done() && !p.peek().is("}") {
-		t := p.peek()
-		switch {
-		case t.is(";"):
-			p.take()
-		case t.is("option"):
-			p.parseOption()
-		case t.is("rpc"):
-			if m := p.parseMethod(); m != nil {
-				svc.Methods = append(svc.Methods, m)
-			}
-		default:
-			p.note(t.line, fmt.Sprintf("unexpected %q in service %q; skipped", t.text, svc.Name))
-			p.skipDeclaration()
-		}
-	}
-	p.expect("}")
-
-	return svc
-}
-
-func (p *parser) parseMethod() *Method {
-	tok := p.take() // "rpc"
-	name := p.take()
-	if name.kind != tokIdent {
-		p.note(tok.line, "rpc has no name; skipped")
-		p.skipDeclaration()
-
-		return nil
-	}
-
-	m := &Method{Name: name.text, Doc: tok.doc, Line: tok.line}
-	m.ClientStreaming, m.Request = p.methodType()
-	if !p.peek().is("returns") {
-		p.note(tok.line, fmt.Sprintf("rpc %q has no returns clause; response unknown", m.Name))
-	} else {
-		p.take()
-	}
-	m.ServerStreaming, m.Response = p.methodType()
-
-	// The body, when there is one, holds method options - `deprecated` being
-	// the only one the catalog can carry.
-	if p.peek().is("{") {
-		p.take()
-		for !p.done() && !p.peek().is("}") {
-			if p.peek().is("option") {
-				o, ok := p.parseOption()
-				if ok && o.Name == "deprecated" && o.Value == "true" {
-					m.Deprecated = true
-				}
-
-				continue
-			}
-			p.take()
-		}
-		p.expect("}")
-	} else {
-		p.expect(";")
-	}
-
-	return m
-}
-
-// methodType reads `(stream shop.v1.Req)` and says whether it streamed.
-func (p *parser) methodType() (bool, string) {
-	if !p.expect("(") {
-		return false, ""
-	}
-	stream := false
-	if p.peek().is("stream") {
-		p.take()
-		stream = true
-	}
-	name := p.qualified()
-	p.expect(")")
-
-	return stream, name
-}
-
-// typeName reads a field's type as written, including a map's two arguments.
-func (p *parser) typeName() string {
-	if p.peek().is("map") {
-		p.take()
-		if !p.peek().is("<") {
-			return "map"
-		}
-		p.take()
-		key := p.qualified()
-		p.expect(",")
-		value := p.typeName()
-		p.expect(">")
-
-		return "map<" + key + ", " + value + ">"
-	}
-
-	return p.qualified()
-}
-
-// qualified reads `shop.v1.Money`, or `.shop.v1.Money` with its leading dot
-// kept - proto's spelling for "this is absolute, do not search enclosing
-// scopes", which the resolver needs and a reader recognises.
-func (p *parser) qualified() string {
-	var b strings.Builder
-	if p.peek().is(".") {
-		p.take()
-		b.WriteString(".")
-	}
-	if p.peek().kind != tokIdent {
-		return b.String()
-	}
-	b.WriteString(p.take().text)
-	for p.peek().is(".") {
-		p.take()
-		b.WriteString(".")
-		if p.peek().kind != tokIdent {
-			break
-		}
-		b.WriteString(p.take().text)
-	}
-
-	return b.String()
-}
-
-// --- the tolerance machinery -----------------------------------------------
-
-func (p *parser) done() bool { return p.peek().kind == tokEOF }
-
-func (p *parser) peek() token {
-	if p.at >= len(p.toks) {
-		return token{kind: tokEOF}
-	}
-
-	return p.toks[p.at]
-}
-
-func (p *parser) take() token {
-	t := p.peek()
-	if p.at < len(p.toks) {
-		p.at++
-	}
-
-	return t
-}
-
-func (p *parser) expect(text string) bool {
-	if p.peek().is(text) {
-		p.take()
-
-		return true
-	}
-	p.note(p.peek().line, fmt.Sprintf("expected %q, found %q", text, p.peek().text))
-
-	return false
-}
-
-// skipDeclaration reads past whatever is here: to the semicolon that ends it,
-// or past the balanced block it opens, whichever comes first.
-func (p *parser) skipDeclaration() {
-	for !p.done() {
-		switch {
-		case p.peek().is(";"):
-			p.take()
-
-			return
-		case p.peek().is("{"):
-			p.skipBlock()
-
-			return
-		case p.peek().is("}"):
-			return
-		default:
-			p.take()
-		}
-	}
-}
-
-func (p *parser) skipBlock() {
-	if !p.peek().is("{") {
-		return
-	}
-	depth := 0
-	for !p.done() {
-		switch {
-		case p.peek().is("{"):
-			depth++
-		case p.peek().is("}"):
-			depth--
-		}
-		p.take()
-		if depth == 0 {
-			return
-		}
-	}
-}
-
-func (p *parser) skipBrackets() {
-	if !p.peek().is("[") {
-		return
-	}
-	depth := 0
-	for !p.done() {
-		switch {
-		case p.peek().is("["):
-			depth++
-		case p.peek().is("]"):
-			depth--
-		}
-		p.take()
-		if depth == 0 {
-			return
-		}
-	}
-}
-
-func (p *parser) note(line int, message string) {
-	p.notes = append(p.notes, Note{Line: line, Message: message})
-}
-
-// attachTrailing gives a declaration the comment sitting at the end of its
-// line, when nothing was written above it. Above wins: an author who wrote
-// both meant the block as the documentation and the trailing note as an aside.
-func attachTrailing(file *File, trailing map[int]string) {
-	if len(trailing) == 0 {
-		return
-	}
-
-	var doMessage func(m *Message)
-	doMessage = func(m *Message) {
-		if m.Doc == "" {
-			m.Doc = trailing[m.Line]
-		}
-		for _, f := range m.Fields {
-			if f.Doc == "" {
-				f.Doc = trailing[f.Line]
-			}
-		}
-		for _, e := range m.Enums {
-			doEnum(e, trailing)
-		}
-		for _, nested := range m.Messages {
-			doMessage(nested)
-		}
-	}
-
-	for _, m := range file.Messages {
-		doMessage(m)
-	}
-	for _, e := range file.Enums {
-		doEnum(e, trailing)
-	}
-	for _, s := range file.Services {
-		if s.Doc == "" {
-			s.Doc = trailing[s.Line]
-		}
-		for _, m := range s.Methods {
-			if m.Doc == "" {
-				m.Doc = trailing[m.Line]
-			}
-		}
-	}
-}
-
-func doEnum(e *Enum, trailing map[int]string) {
-	if e.Doc == "" {
-		e.Doc = trailing[e.Line]
-	}
-	for _, v := range e.Values {
-		if v.Doc == "" {
-			v.Doc = trailing[v.Line]
-		}
-	}
-}