@shortlink-org/portolan 0.1.0 → 0.2.0

package/plugins/extract-project/main.go

@@ -1,41 +0,0 @@
-// Package main is portolan-extract-project: a repository component in, a
-// neutral catalog fragment out. It deliberately makes no DDD assumptions.
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-type Options struct {
-	Group        string `json:"group,omitempty"`
-	GroupName    string `json:"groupName,omitempty"`
-	GroupSummary string `json:"groupSummary,omitempty"`
-	GroupKind    string `json:"groupKind,omitempty"`
-
-	Component     string   `json:"component,omitempty"`
-	ComponentName string   `json:"componentName,omitempty"`
-	ComponentKind string   `json:"componentKind,omitempty"`
-	Technologies  []string `json:"technologies,omitempty"`
-	Repo          string   `json:"repo,omitempty"`
-	Out           string   `json:"out,omitempty"`
-}
-
-func main() {
-	if err := run(os.Stdin, os.Stdout); err != nil {
-		fmt.Fprintln(os.Stderr, "portolan-extract-project:", err)
-		os.Exit(1)
-	}
-}
-
-func run(stdin io.Reader, stdout io.Writer) error {
-	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
-		if req.Input.Root == "" {
-			return plugin.Response{}, fmt.Errorf("no input root: an extractor has nothing to read")
-		}
-		return extract(req.Input, opts)
-	})
-}

package/plugins/extract-proto/ast.go

@@ -1,125 +0,0 @@
-package main
-
-// What a .proto file says, as it says it.
-//
-// Types are kept AS WRITTEN - `repeated Money`, `map<string, LineItem>`, the
-// `optional` keyword, the author's declaration order. The catalog already
-// takes this stance for schemas ("the db type as declared - uuid, timestamptz,
-// jsonb - never normalised"), and a reader comparing a page against the file it
-// came from should recognise what they are looking at.
-
-// File is one parsed .proto.
-type File struct {
-	// Path is the file as a reader would type it, relative to the repo root.
-	Path string
-
-	// Doc is the comment at the very top of the file, above `syntax`.
-	//
-	// On a vendored copy this is where the header lives that says which module
-	// it came from - docs/adr/org.0001.md requires one - so it is worth keeping
-	// apart from the documentation of any declaration inside.
-	Doc string
-
-	Syntax   string // "proto2", "proto3", "editions", or "" when unstated
-	Package  string
-	Imports  []Import
-	Options  []Option
-	Messages []*Message
-	Enums    []*Enum
-	Services []*Service
-}
-
-type Import struct {
-	Path   string
-	Public bool
-	Weak   bool
-	Line   int
-}
-
-type Option struct {
-	Name  string
-	Value string
-	Line  int
-}
-
-type Message struct {
-	Name string
-	Doc  string
-	Line int
-
-	// Fields holds every field of the message, including the members of its
-	// oneofs, in declaration order. A oneof is a constraint on which of them may
-	// be set, not a separate place a field lives, so a reader asking "what is in
-	// this message" gets one list.
-	Fields   []*Field
-	Oneofs   []*Oneof
-	Messages []*Message // nested
-	Enums    []*Enum
-	Reserved []string
-}
-
-type Field struct {
-	Name string
-	Doc  string
-	Line int
-
-	// Type as written: "string", "Money", "shop.v1.Money", "map<string, Money>".
-	Type   string
-	Label  string // "", "repeated", "optional", "required"
-	Number int
-
-	// Oneof names the oneof this field belongs to, empty when it is not in one.
-	Oneof      string
-	Deprecated bool
-	Default    string
-}
-
-type Oneof struct {
-	Name   string
-	Doc    string
-	Line   int
-	Fields []*Field
-}
-
-type Enum struct {
-	Name   string
-	Doc    string
-	Line   int
-	Values []*EnumValue
-}
-
-type EnumValue struct {
-	Name   string
-	Doc    string
-	Number int
-	Line   int
-}
-
-type Service struct {
-	Name    string
-	Doc     string
-	Line    int
-	Methods []*Method
-}
-
-type Method struct {
-	Name string
-	Doc  string
-	Line int
-
-	Request         string
-	Response        string
-	ClientStreaming bool
-	ServerStreaming bool
-	Deprecated      bool
-}
-
-// Note is something the parser declined to model, named rather than dropped.
-//
-// A construct the parser skips silently is a construct nobody knows is missing.
-// Every one of these becomes a plugin diagnostic, so an estate using `extend`
-// or a proto2 `group` finds out that portolan read past it.
-type Note struct {
-	Line    int
-	Message string
-}

package/plugins/extract-proto/consumes.go

@@ -1,77 +0,0 @@
-package main
-
-// What a service CALLS, read from the client copies it vendors.
-//
-// docs/adr/org.0001.md puts a narrowed copy of the producer's proto in the
-// consumer's infrastructure layer, with a header comment saying where it came
-// from. That copy is a statement about what this service intends to call, and
-// reading it is how a call edge gets into the catalog from the calling end.
-
-import (
-	"sort"
-
-	"github.com/shortlink-org/portolan/catalog"
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-// calls turns every method in the vendored files into a catalog call.
-//
-// STATUS IS ALWAYS "declared", NEVER "verified". Three reasons, and the test
-// that holds this is deliberate:
-//
-//  1. `problems()` treats status as authored, not derived.
-//  2. Verification is a property of the UNION, and every extractor runs before
-//     one exists - `plugin.Request.Catalog` is the zero value for an extractor.
-//  3. In the shipped catalog `verified` already means something a proto cannot
-//     supply: it sits next to a note naming the integration test that covers
-//     the call end to end.
-//
-// Reading a .proto proves the call was WRITTEN DOWN, which is what `declared`
-// means. Whether it is exercised is not a question this file can answer.
-func calls(files []*File, peers map[string]string, moduleID string, b *plugin.Builder) []catalog.RpcCall {
-	var out []catalog.RpcCall
-
-	for _, file := range files {
-		peer, named := peers[file.Package]
-		if !named {
-			// The raw package name, which is exactly what `RpcCall.peer`
-			// documents: "service id if resolved, else raw name". Guessing a
-			// service id from a proto package would be inventing an edge.
-			peer = file.Package
-		}
-
-		for _, svc := range file.Services {
-			iface := interfaceID(file.Package, svc.Name)
-			if !named {
-				b.Warn(iface, "no peer is mapped for proto package "+file.Package+
-					"; the call is listed against the package name")
-			}
-
-			for _, m := range svc.Methods {
-				out = append(out, catalog.RpcCall{
-					ID:     callID(iface, m.Name),
-					Peer:   peer,
-					Status: catalog.StatusDeclared,
-					Source: sourceRef(file.Path, m.Line),
-					Note:   file.headerNote(),
-					Module: moduleID,
-				})
-			}
-		}
-	}
-
-	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
-
-	return out
-}
-
-// headerNote is the comment at the top of a vendored copy.
-//
-// The ADR already requires that header - "the header comment on every vendored
-// copy says so" - so surfacing it costs nothing and makes the boundary
-// self-documenting on the page rather than only in the file. It sits above
-// `syntax` rather than above a declaration, which is why the parser keeps it
-// on the file instead of on whatever happens to come first.
-func (f *File) headerNote() string {
-	return f.Doc
-}

package/plugins/extract-proto/describe.go

@@ -1,19 +0,0 @@
-package main
-
-import (
-	_ "embed"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-//go:embed options.schema.json
-var optionsSchema []byte
-
-func descriptor() plugin.Descriptor {
-	return plugin.Descriptor{
-		Name:    "extract-proto",
-		Summary: "Reads .proto files and answers with what a service provides from the modules it publishes and what it calls from the copies it vendors.",
-		Phases:  []string{plugin.PhaseExtract},
-		Options: optionsSchema,
-	}
-}

package/plugins/extract-proto/extract.go

@@ -1,293 +0,0 @@
-package main
-
-// Protos in, a catalog fragment out.
-//
-// This extractor describes ONE aspect of a service - what it answers and what
-// it calls - and nothing else. The aggregates come from a different extractor
-// reading a different part of the tree, and the two meet in the merge. Neither
-// knows the other exists, which is the whole reason each of them stays small.
-//
-// It reads no clock, opens no socket and looks at no environment. Fetching a
-// module from a registry is the other plugin's job, and keeping the two apart
-// is what lets this one be replayed byte-for-byte from a checkout.
-//
-// A note on proto event schemas: a message named `OrderPlaced` in an
-// `events.proto` is the WIRE SCHEMA OF a domain event, not the event. It stays
-// an RpcMessage rather than becoming a catalog Event, because `Event.id` is
-// `<service>.<aggregate>.<Name>` and this extractor knows the package, not the
-// aggregate. A guess would either collide with the event extract-go already
-// emits from the domain package, or create a ghost under an invented aggregate
-// - and since aggregates merge whole-object by id, that ghost would sit beside
-// the real one forever.
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	"github.com/shortlink-org/portolan/catalog"
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-func extract(in plugin.Input, opts Options) (plugin.Response, error) {
-	b := &plugin.Builder{}
-	root := in.Root
-
-	if opts.Context == "" {
-		opts.Context = filepath.Base(root)
-	}
-	if opts.Service == "" {
-		opts.Service = filepath.Base(root)
-	}
-	serviceID := opts.Context + "." + opts.Service
-
-	owned, err := readDirs(root, opts.Paths, b)
-	if err != nil {
-		return plugin.Response{}, err
-	}
-	vendored, err := readDirs(root, opts.Vendored, b)
-	if err != nil {
-		return plugin.Response{}, err
-	}
-
-	if len(owned) == 0 && len(vendored) == 0 {
-		b.Warn(serviceID, "no .proto files were found under "+root)
-	}
-
-	// One index over everything read. A vendored copy may well refer to a type
-	// the owned protos declare, and refusing to look would report an unresolved
-	// name the tree can plainly answer.
-	all := append(append([]*File{}, filesOf(owned)...), filesOf(vendored)...)
-	ix := NewIndex(all)
-
-	sh := newShared(opts.Defs != "off")
-
-	var provides []catalog.RpcService
-	var consumes []catalog.RpcCall
-	var modules []catalog.ProtoModule
-	moduleIDs := map[string]bool{}
-
-	for _, dir := range owned {
-		module := describe(root, dir, opts, b)
-		if module.ID != "" && !moduleIDs[module.ID] {
-			// Owned: this service publishes it, so the estate knows the owner.
-			module.Owner = serviceID
-			modules = append(modules, module)
-			moduleIDs[module.ID] = true
-		}
-		provides = append(provides, interfaces(ix, dir.files, module.ID, sh, b)...)
-	}
-
-	for _, dir := range vendored {
-		module := describe(root, dir, opts, b)
-		if module.ID != "" && !moduleIDs[module.ID] {
-			// Vendored: somebody else publishes it. Owner is left empty rather
-			// than guessed - a module whose publisher is outside the estate is
-			// the ordinary case, not a defect.
-			modules = append(modules, module)
-			moduleIDs[module.ID] = true
-		}
-		consumes = append(consumes, calls(dir.files, opts.Peers, module.ID, b)...)
-	}
-
-	sort.Slice(modules, func(i, j int) bool { return modules[i].ID < modules[j].ID })
-
-	service := catalog.Service{
-		ID:   serviceID,
-		Slug: opts.Service,
-		// Named by whichever source knows the name. This one describes what the
-		// service answers and calls, not what it is called, and a fragment that
-		// filled these in from a proto package would be inventing.
-		Provides:   nonNilProvides(provides),
-		Consumes:   nonNilCalls(consumes),
-		Aggregates: []catalog.Aggregate{},
-		Modules:    sortedCopy(keysOf(moduleIDs)),
-	}
-
-	fragment := catalog.Catalog{
-		GeneratedAt: in.GeneratedAt,
-		Commit:      in.Commit,
-		Contexts: []catalog.BoundedContext{{
-			ID:       opts.Context,
-			Slug:     opts.Context,
-			Services: []catalog.Service{service},
-		}},
-		Defs:    sh.defs,
-		Flows:   []catalog.Flow{},
-		Adrs:    []catalog.Adr{},
-		Modules: modules,
-	}
-
-	encoded, err := json.MarshalIndent(fragment, "", "  ")
-	if err != nil {
-		return plugin.Response{}, err
-	}
-
-	b.File(firstNonEmpty(opts.Out, "proto.json"), string(encoded)+"\n")
-
-	return b.Response(), nil
-}
-
-// dir is one directory of protos and everything parsed out of it.
-type dir struct {
-	path  string
-	files []*File
-	lock  *lockFile
-}
-
-func filesOf(dirs []dir) []*File {
-	var out []*File
-	for _, d := range dirs {
-		out = append(out, d.files...)
-	}
-
-	return out
-}
-
-// readDirs walks each directory for .proto files and parses them.
-//
-// Files are visited in sorted order and parsed in that order, because the
-// fragment is committed and compared by `gen:check`: a filesystem walk that
-// came back differently would rewrite the file for no reason.
-func readDirs(root string, paths []string, b *plugin.Builder) ([]dir, error) {
-	var out []dir
-
-	for _, rel := range paths {
-		at := filepath.Join(root, filepath.FromSlash(rel))
-		info, err := os.Stat(at)
-		if os.IsNotExist(err) {
-			b.Warn(rel, "no such directory: "+filepath.ToSlash(at))
-
-			continue
-		}
-		if err != nil {
-			return nil, err
-		}
-		if !info.IsDir() {
-			return nil, fmt.Errorf("%s is not a directory", filepath.ToSlash(at))
-		}
-
-		var found []string
-		err = filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if !info.IsDir() && strings.HasSuffix(path, ".proto") {
-				found = append(found, path)
-			}
-
-			return nil
-		})
-		if err != nil {
-			return nil, err
-		}
-		sort.Strings(found)
-
-		d := dir{path: trimSlash(at)}
-		for _, path := range found {
-			src, err := os.ReadFile(path)
-			if err != nil {
-				return nil, err
-			}
-			file, notes, err := Parse(filepath.ToSlash(path), string(src))
-			if err != nil {
-				// The file could not be tokenised. Past that the parser is not
-				// reading proto any more, so it is reported and skipped rather
-				// than half-described.
-				b.Warn(filepath.ToSlash(path), err.Error())
-
-				continue
-			}
-			for _, note := range notes {
-				b.Warn(fmt.Sprintf("%s:%d", filepath.ToSlash(path), note.Line), note.Message)
-			}
-			d.files = append(d.files, file)
-		}
-
-		lock, err := readLock(at)
-		if err != nil {
-			b.Warn(rel, "the lock beside these protos could not be read: "+err.Error())
-		}
-		d.lock = lock
-
-		out = append(out, d)
-	}
-
-	return out, nil
-}
-
-// describe names the module a directory of protos belongs to.
-//
-// A lock written by fetch-bsr wins, because it is the only thing that knows the
-// commit. Failing that the manifest may say, and failing that the directory
-// gets a `local:` id so nobody mistakes the path for something fetchable.
-func describe(root string, d dir, opts Options, b *plugin.Builder) catalog.ProtoModule {
-	rel := relativeTo(root, d.path)
-
-	id := ""
-	var lock *lockModule
-	if d.lock != nil && len(d.lock.Modules) > 0 {
-		lock = &d.lock.Modules[0]
-		id = lock.Module
-		if len(d.lock.Modules) > 1 {
-			b.Warn(rel, "the lock names more than one module for this directory; the first is used")
-		}
-	}
-	if id == "" {
-		id = opts.Modules[rel]
-	}
-	if id == "" {
-		id = localModuleID(relativeTo(root, d.path))
-		b.Warn(rel, "these protos are not pinned to a published module; listed as "+id)
-	}
-
-	module := moduleFor(id, d.path, d.files)
-	module.Source = filepath.ToSlash(d.path)
-	if lock != nil {
-		module.Commit = lock.Commit
-		module.Digest = lock.Digest
-		module.Deps = sortedCopy(lock.Deps)
-	}
-
-	return module
-}
-
-func keysOf(set map[string]bool) []string {
-	out := make([]string, 0, len(set))
-	for k := range set {
-		out = append(out, k)
-	}
-
-	return out
-}
-
-// A fragment states what it knows and nothing else, but a list the schema calls
-// required has to be present even when it is empty.
-func nonNilProvides(in []catalog.RpcService) []catalog.RpcService {
-	if in == nil {
-		return []catalog.RpcService{}
-	}
-
-	return in
-}
-
-func nonNilCalls(in []catalog.RpcCall) []catalog.RpcCall {
-	if in == nil {
-		return []catalog.RpcCall{}
-	}
-
-	return in
-}
-
-func firstNonEmpty(values ...string) string {
-	for _, v := range values {
-		if v != "" {
-			return v
-		}
-	}
-
-	return ""
-}