@shortlink-org/portolan 0.1.0

package/internal/goscan/goscan_test.go

@@ -0,0 +1,227 @@
+package goscan
+
+import (
+	"go/ast"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func write(t *testing.T, root, name, contents string) {
+	t.Helper()
+	full := filepath.Join(root, name)
+	if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(full, []byte(contents), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func tree(t *testing.T) *Tree {
+	t.Helper()
+	root := t.TempDir()
+	write(t, root, "go.mod", "module example.com/m\n\ngo 1.22\n")
+	write(t, root, "main.go", `package main
+
+import (
+	other "example.com/m/sub"
+	"example.com/m/vendor/lib"
+)
+
+const (
+	A = "a"
+	B = A
+	C = other.D
+	Inherited
+	Self1 = Self2
+	Self2 = Self1
+	Foreign = lib.Default
+	Number = 3
+)
+
+type Args struct {
+	ID     string `+"`json:\"id\"`"+`
+	Secret string `+"`json:\"-\"`"+`
+	Plain  int
+	hidden bool
+}
+`)
+	write(t, root, "sub/b.go", "package sub\n\nconst D = \"d\"\n")
+	write(t, root, "sub/b_test.go", "package sub\n\nconst Test = \"never\"\n")
+	write(t, root, "sub/gen.gen.go", "package sub\n\nconst Generated = \"never\"\n")
+	write(t, root, "vendor/lib/lib.go", "package lib\n\nconst Default = \"never\"\n")
+	write(t, root, ".hidden/h.go", "package hidden\n\nconst Hidden = \"never\"\n")
+	write(t, root, "node_modules/x/x.go", "package x\n\nconst X = \"never\"\n")
+
+	out, err := Read(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+func TestReadKeepsTheSourceAndLeavesTheRest(t *testing.T) {
+	out := tree(t)
+
+	var names []string
+	for _, file := range out.Files {
+		names = append(names, file.Name+" "+file.Pkg)
+	}
+	want := []string{"main.go example.com/m", "sub/b.go example.com/m/sub"}
+	if !reflect.DeepEqual(names, want) {
+		t.Errorf("files: got %v, want %v", names, want)
+	}
+	if out.Module != "example.com/m" {
+		t.Errorf("module: got %q", out.Module)
+	}
+	for _, key := range []string{"example.com/m/sub.Test", "example.com/m/sub.Generated", "example.com/m/vendor/lib.Default", "example.com/m/.hidden.Hidden"} {
+		if _, found := out.Constants[key]; found {
+			t.Errorf("%s was read", key)
+		}
+	}
+}
+
+func TestImportsAreByLocalName(t *testing.T) {
+	out := tree(t)
+	got := out.Files[0].Imports
+	want := map[string]string{"other": "example.com/m/sub", "lib": "example.com/m/vendor/lib"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("imports: got %v, want %v", got, want)
+	}
+}
+
+func TestADottedImportSegmentIsKnownByItsPackageName(t *testing.T) {
+	root := t.TempDir()
+	write(t, root, "go.mod", "module example.com/m\n")
+	write(t, root, "main.go", "package main\n\nimport (\n\t\"github.com/nats-io/nats.go\"\n\t\"gopkg.in/yaml.v3\"\n\t\"github.com/nats-io/nats.go/jetstream\"\n)\n")
+	out, err := Read(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := out.Files[0].Imports
+	want := map[string]string{
+		"nats.go":   "github.com/nats-io/nats.go",
+		"nats":      "github.com/nats-io/nats.go",
+		"yaml.v3":   "gopkg.in/yaml.v3",
+		"yaml":      "gopkg.in/yaml.v3",
+		"jetstream": "github.com/nats-io/nats.go/jetstream",
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("imports: got %v, want %v", got, want)
+	}
+}
+
+func TestConstantsResolveThroughEachOther(t *testing.T) {
+	out := tree(t)
+	cases := map[string]string{
+		"example.com/m.A":         "a",
+		"example.com/m.B":         "a",
+		"example.com/m.C":         "d",
+		"example.com/m.Inherited": "d",
+		"example.com/m.Self1":     "",
+		"example.com/m.Foreign":   "",
+		"example.com/m.Number":    "",
+		"example.com/m.Missing":   "",
+	}
+	for key, want := range cases {
+		if got := out.ConstantString(key, nil); got != want {
+			t.Errorf("%s: got %q, want %q", key, got, want)
+		}
+	}
+}
+
+func TestForeignAnswersWhatTheTreeCannot(t *testing.T) {
+	out := tree(t)
+	out.Foreign = func(importPath, name string) (string, bool) {
+		if importPath == "example.com/m/vendor/lib" && name == "Default" {
+			return "default", true
+		}
+		return "", false
+	}
+	if got := out.ConstantString("example.com/m.Foreign", nil); got != "default" {
+		t.Errorf("got %q, want %q", got, "default")
+	}
+}
+
+func TestTypeKey(t *testing.T) {
+	out := tree(t)
+	file := out.Files[0]
+	parse := func(src string) ast.Expr {
+		expr, err := parseExpr(src)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return expr
+	}
+	cases := map[string]string{
+		"Args":             "example.com/m.Args",
+		"*Args":            "example.com/m.Args",
+		"string":           "string",
+		"other.Thing":      "example.com/m/sub.Thing",
+		"[]other.Thing":    "[]example.com/m/sub.Thing",
+		"map[string]Args":  "map[string]example.com/m.Args",
+		"interface{ M() }": "interface",
+		"func()":           "",
+		"a.b.C":            "",
+	}
+	for src, want := range cases {
+		if got := out.TypeKey(parse(src), file); got != want {
+			t.Errorf("%s: got %q, want %q", src, got, want)
+		}
+	}
+	if got := out.TypeKey(nil, file); got != "" {
+		t.Errorf("nil: got %q", got)
+	}
+}
+
+func TestFieldsOfFollowTheJSONTag(t *testing.T) {
+	out := tree(t)
+	var body *ast.StructType
+	ast.Inspect(out.Files[0].Node, func(node ast.Node) bool {
+		if spec, ok := node.(*ast.TypeSpec); ok && spec.Name.Name == "Args" {
+			body = spec.Type.(*ast.StructType)
+		}
+		return body == nil
+	})
+	got := out.FieldsOf(body)
+	want := []string{"id string", "Plain int", "hidden bool"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestAtIsRelativeToTheRoot(t *testing.T) {
+	out := tree(t)
+	file := out.Files[1]
+	at := out.At(file.Node.Decls[0].Pos())
+	if at.String() != "sub/b.go:3" {
+		t.Errorf("got %q", at.String())
+	}
+	if (Source{File: "x.go"}).String() != "x.go" {
+		t.Error("a source with no line prints its line")
+	}
+}
+
+func TestNames(t *testing.T) {
+	if got := LastSegment("example.com/m.Args"); got != "Args" {
+		t.Errorf("LastSegment: %q", got)
+	}
+	if got := LastSegment("plain"); got != "plain" {
+		t.Errorf("LastSegment plain: %q", got)
+	}
+	if got := Slug("  Send Welcome_Mail!! "); got != "send-welcome-mail" {
+		t.Errorf("Slug: %q", got)
+	}
+	if got := Title("send-welcome_mail.v1"); got != "Send Welcome Mail V1" {
+		t.Errorf("Title: %q", got)
+	}
+	if got := FirstNonEmpty(" ", "", "x", "y"); got != "x" {
+		t.Errorf("FirstNonEmpty: %q", got)
+	}
+	if got := strings.TrimSpace(FirstNonEmpty(" ", "")); got != "" {
+		t.Errorf("FirstNonEmpty empty: %q", got)
+	}
+}

package/internal/goscan/names.go

@@ -0,0 +1,52 @@
+package goscan
+
+import "strings"
+
+// LastSegment is what follows the last dot: the name off a type key, the
+// event off a wire name.
+func LastSegment(value string) string {
+	if at := strings.LastIndex(value, "."); at >= 0 {
+		return value[at+1:]
+	}
+	return value
+}
+
+// Slug is a value as an identifier in a URL or a catalog id: lower case,
+// runs of anything else collapsed to one dash, none at the ends.
+func Slug(value string) string {
+	value = strings.ToLower(strings.TrimSpace(value))
+	var out strings.Builder
+	dash := false
+	for _, r := range value {
+		if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
+			out.WriteRune(r)
+			dash = false
+		} else if out.Len() > 0 && !dash {
+			out.WriteByte('-')
+			dash = true
+		}
+	}
+	return strings.TrimSuffix(out.String(), "-")
+}
+
+// Title is a slug or an identifier as words: split on dash, underscore and
+// dot, each capitalised.
+func Title(value string) string {
+	parts := strings.FieldsFunc(value, func(r rune) bool { return r == '-' || r == '_' || r == '.' })
+	for i := range parts {
+		if parts[i] != "" {
+			parts[i] = strings.ToUpper(parts[i][:1]) + parts[i][1:]
+		}
+	}
+	return strings.Join(parts, " ")
+}
+
+// FirstNonEmpty is the first value that is more than whitespace.
+func FirstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if strings.TrimSpace(value) != "" {
+			return value
+		}
+	}
+	return ""
+}

package/internal/goscan/parse_test.go

@@ -0,0 +1,11 @@
+package goscan
+
+import (
+	"go/ast"
+	"go/parser"
+)
+
+// parseExpr is go/parser's, named here so a test reads as a table.
+func parseExpr(src string) (ast.Expr, error) {
+	return parser.ParseExpr(src)
+}

package/internal/goscan/source.go

@@ -0,0 +1,36 @@
+package goscan
+
+import (
+	"fmt"
+	"go/ast"
+)
+
+// Source is a place in the tree, the way the catalog cites one: the file
+// relative to the root, and the line when there is one.
+type Source struct {
+	File string
+	Line int
+}
+
+func (s Source) String() string {
+	if s.Line == 0 {
+		return s.File
+	}
+	return fmt.Sprintf("%s:%d", s.File, s.Line)
+}
+
+// File is one parsed file: its path under the root, the import path of its
+// package, its imports by local name, and the syntax.
+type File struct {
+	Name    string
+	Pkg     string
+	Imports map[string]string
+	Node    *ast.File
+}
+
+// ConstExpr is a constant as it was declared: the expression, and the file it
+// is in, since the expression's names mean what that file's imports say.
+type ConstExpr struct {
+	Expr ast.Expr
+	File *File
+}

package/internal/goscan/tree.go

@@ -0,0 +1,155 @@
+// Package goscan reads a Go source tree the way the extractors need it: every
+// non-test, non-generated file parsed once, each with the import path of its
+// package and its imports by local name, and the string constants of the tree
+// indexed so that a topic, a queue or a subject named by a constant resolves to
+// the literal behind it.
+//
+// It is the part of a Go extractor that has nothing to do with what the
+// extractor is looking for. River and Watermill read the tree this way, and the
+// next Go extractor should not have to copy it a third time.
+package goscan
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Tree is one Go module's source, read once.
+type Tree struct {
+	Root   string
+	Module string
+	Fset   *token.FileSet
+	// Files in path order, so that whatever is read off them comes out in the
+	// same order every run.
+	Files []*File
+	// Constants is every constant of the tree by "<import path>.<Name>", as
+	// the expression that declared it, so a constant defined through another
+	// is followed rather than copied.
+	Constants map[string]ConstExpr
+	// Foreign is what an identifier from a package outside the tree is worth,
+	// when the extractor knows: river.QueueDefault is "default", and nothing
+	// in the tree says so. Nil when there is nothing to say.
+	Foreign func(importPath, name string) (string, bool)
+}
+
+// Read parses the tree under root. Test files, generated files, vendored
+// code and dot-directories are left out: they are not what the service
+// says about itself.
+func Read(root string) (*Tree, error) {
+	t := &Tree{Root: root, Module: ModulePath(root), Fset: token.NewFileSet(), Constants: map[string]ConstExpr{}}
+
+	var names []string
+	err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			if name != root && strings.HasPrefix(entry.Name(), ".") {
+				return filepath.SkipDir
+			}
+			switch entry.Name() {
+			case ".git", ".portolan", "node_modules", "vendor":
+				if name != root {
+					return filepath.SkipDir
+				}
+			}
+			return nil
+		}
+		base := entry.Name()
+		if !strings.HasSuffix(base, ".go") || strings.HasSuffix(base, "_test.go") || strings.HasSuffix(base, ".gen.go") || strings.HasSuffix(base, "_generated.go") {
+			return nil
+		}
+		names = append(names, name)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		node, err := parser.ParseFile(t.Fset, name, nil, parser.ParseComments)
+		if err != nil {
+			return nil, fmt.Errorf("parse %s: %w", name, err)
+		}
+		rel, _ := filepath.Rel(root, name)
+		t.Files = append(t.Files, &File{
+			Name:    filepath.ToSlash(rel),
+			Pkg:     t.PackagePath(filepath.Dir(name)),
+			Imports: ImportsOf(node),
+			Node:    node,
+		})
+	}
+	t.indexConstants()
+
+	return t, nil
+}
+
+// PackagePath is the import path of the package in dir: the module path
+// with the directory under it.
+func (t *Tree) PackagePath(dir string) string {
+	rel, err := filepath.Rel(t.Root, dir)
+	if err != nil || rel == "." {
+		return t.Module
+	}
+	return strings.TrimSuffix(t.Module, "/") + "/" + filepath.ToSlash(rel)
+}
+
+// At is where a node is, relative to the root.
+func (t *Tree) At(pos token.Pos) Source {
+	position := t.Fset.Position(pos)
+	rel, err := filepath.Rel(t.Root, position.Filename)
+	if err != nil {
+		rel = position.Filename
+	}
+	return Source{File: filepath.ToSlash(rel), Line: position.Line}
+}
+
+// ModulePath is what go.mod under root calls the module; the cleaned root
+// when there is no go.mod to ask.
+func ModulePath(root string) string {
+	data, err := os.ReadFile(filepath.Join(root, "go.mod"))
+	if err == nil {
+		for _, line := range strings.Split(string(data), "\n") {
+			if value, ok := strings.CutPrefix(strings.TrimSpace(line), "module "); ok {
+				return strings.TrimSpace(value)
+			}
+		}
+	}
+	return filepath.ToSlash(filepath.Clean(root))
+}
+
+// ImportsOf is a file's imports by the name the file uses for them: the alias
+// when there is one, the last path segment when there is not. Blank and dot
+// imports keep the segment, since nothing in the file refers to them by name.
+//
+// A segment with a dot in it - `nats.go`, `yaml.v3` - is not a Go identifier,
+// and the package behind it is called by what comes before the dot; that name
+// is recorded too, since it is the one the file uses.
+func ImportsOf(node *ast.File) map[string]string {
+	out := map[string]string{}
+	for _, spec := range node.Imports {
+		value, err := strconv.Unquote(spec.Path.Value)
+		if err != nil {
+			continue
+		}
+		name := path.Base(value)
+		if spec.Name != nil && spec.Name.Name != "_" && spec.Name.Name != "." {
+			name = spec.Name.Name
+		}
+		out[name] = value
+		if short, _, dotted := strings.Cut(name, "."); dotted && short != "" {
+			out[short] = value
+		}
+	}
+	return out
+}

package/internal/goscan/types.go

@@ -0,0 +1,99 @@
+package goscan
+
+import (
+	"bytes"
+	"go/ast"
+	"go/printer"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+// Unwrap takes the parentheses, the address-of and the pointer off an
+// expression, down to the thing itself.
+func Unwrap(expr ast.Expr) ast.Expr {
+	for {
+		switch value := expr.(type) {
+		case *ast.ParenExpr:
+			expr = value.X
+		case *ast.UnaryExpr:
+			expr = value.X
+		case *ast.StarExpr:
+			expr = value.X
+		default:
+			return expr
+		}
+	}
+}
+
+// TypeKey names a type the way the tree indexes types: "<import path>.<Name>"
+// for a named type, whether written bare or through an import; the builtin's
+// own name; "[]T" and "map[K]V" over their parts; "interface" for any
+// interface literal. Empty for a type this reader does not name.
+func (t *Tree) TypeKey(expr ast.Expr, file *File) string {
+	if expr == nil {
+		return ""
+	}
+	switch value := Unwrap(expr).(type) {
+	case *ast.Ident:
+		if IsBuiltin(value.Name) {
+			return value.Name
+		}
+		return file.Pkg + "." + value.Name
+	case *ast.SelectorExpr:
+		if base, ok := value.X.(*ast.Ident); ok {
+			return file.Imports[base.Name] + "." + value.Sel.Name
+		}
+	case *ast.ArrayType:
+		return "[]" + t.TypeKey(value.Elt, file)
+	case *ast.MapType:
+		return "map[" + t.TypeKey(value.Key, file) + "]" + t.TypeKey(value.Value, file)
+	case *ast.InterfaceType:
+		return "interface"
+	}
+	return ""
+}
+
+// IsBuiltin is whether a bare identifier is one of the language's own types.
+func IsBuiltin(name string) bool {
+	switch name {
+	case "string", "bool", "byte", "rune", "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64", "float32", "float64", "error", "any":
+		return true
+	}
+	return false
+}
+
+// FieldsOf lists a struct's named fields as "name type", the name being what
+// the json tag says when it says anything, and a field tagged "-" left out:
+// the fields as they travel, not as the code calls them.
+func (t *Tree) FieldsOf(body *ast.StructType) []string {
+	var out []string
+	for _, field := range body.Fields.List {
+		if len(field.Names) == 0 {
+			continue
+		}
+		typeName := t.PrintNode(field.Type)
+		for _, ident := range field.Names {
+			name := ident.Name
+			if field.Tag != nil {
+				tagText, _ := strconv.Unquote(field.Tag.Value)
+				jsonName := strings.Split(reflect.StructTag(tagText).Get("json"), ",")[0]
+				if jsonName == "-" {
+					continue
+				}
+				if jsonName != "" {
+					name = jsonName
+				}
+			}
+			out = append(out, name+" "+typeName)
+		}
+	}
+	return out
+}
+
+// PrintNode is a node as source text.
+func (t *Tree) PrintNode(node ast.Node) string {
+	var out bytes.Buffer
+	_ = printer.Fprint(&out, t.Fset, node)
+	return out.String()
+}