@shortlink-org/portolan 0.1.0

package/plugins/extract-openapi/spec.go

@@ -0,0 +1,350 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// The document is read as a node tree rather than into structs.
+//
+// Two reasons, and both are about the reader of the output. A YAML mapping
+// unmarshalled into a Go map loses its order, and the order paths and schemas
+// are written in is the author's - alphabetising it would be a worse document
+// than the one they wrote. And a path item holds keys that are not verbs, and a
+// schema holds keywords this does not model; walking the nodes means ignoring
+// them rather than failing on them.
+
+// verbs, in the order a reader expects to meet them rather than the order they
+// happen to appear.
+var verbs = []string{"get", "post", "put", "patch", "delete", "head", "options", "trace"}
+
+type document struct {
+	root  *yaml.Node
+	path  string
+	cache map[string]*document
+}
+
+func load(path string) (*document, error) {
+	cache := map[string]*document{}
+	return loadCached(path, cache)
+}
+
+func loadCached(path string, cache map[string]*document) (*document, error) {
+	path = filepath.Clean(path)
+	if held := cache[path]; held != nil {
+		return held, nil
+	}
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var root yaml.Node
+	if err := yaml.Unmarshal(contents, &root); err != nil {
+		return nil, err
+	}
+
+	// A document node wraps the mapping.
+	node := &root
+	if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
+		node = node.Content[0]
+	}
+
+	doc := &document{root: node, path: path, cache: cache}
+	cache[path] = doc
+	return doc, nil
+}
+
+// entry is one key/value pair of a mapping, in document order.
+type entry struct {
+	key   string
+	value *yaml.Node
+}
+
+func entries(node *yaml.Node) []entry {
+	if node == nil || node.Kind != yaml.MappingNode {
+		return nil
+	}
+
+	out := make([]entry, 0, len(node.Content)/2)
+	for i := 0; i+1 < len(node.Content); i += 2 {
+		out = append(out, entry{key: node.Content[i].Value, value: node.Content[i+1]})
+	}
+
+	return out
+}
+
+// child follows a path of mapping keys, and answers nil for anything missing.
+func child(node *yaml.Node, keys ...string) *yaml.Node {
+	for _, key := range keys {
+		found := false
+		for _, e := range entries(node) {
+			if e.key == key {
+				node = e.value
+				found = true
+
+				break
+			}
+		}
+		if !found {
+			return nil
+		}
+	}
+
+	return node
+}
+
+func text(node *yaml.Node) string {
+	if node == nil {
+		return ""
+	}
+
+	return strings.TrimSpace(node.Value)
+}
+
+func list(node *yaml.Node) []string {
+	if node == nil || node.Kind != yaml.SequenceNode {
+		return nil
+	}
+
+	out := make([]string, 0, len(node.Content))
+	for _, item := range node.Content {
+		out = append(out, item.Value)
+	}
+
+	return out
+}
+
+// Where a document keeps its named schemas: under components in OpenAPI 3, and
+// under definitions in Swagger 2.0. A reference is a pointer to one of these
+// two places, and the name is what follows.
+var schemaHomes = []string{"components/schemas/", "definitions/"}
+
+type schemaRef struct {
+	doc  *document
+	node *yaml.Node
+	name string
+	key  string
+}
+
+// schemaRefs collects every component schema a subtree names, in the order they
+// are met.
+//
+// A $ref that points at another component - a shared response, a parameter - is
+// followed rather than ignored, because that is how a spec says "the same error
+// body as everywhere else", and a reader who is not shown it is being told this
+// endpoint returns nothing on failure.
+func (d *document) schemaRefs(node *yaml.Node, into *[]schemaRef, seen, visited map[string]bool) {
+	if node == nil {
+		return
+	}
+
+	switch node.Kind {
+	case yaml.MappingNode:
+		for _, e := range entries(node) {
+			if e.key != "$ref" {
+				d.schemaRefs(e.value, into, seen, visited)
+
+				continue
+			}
+
+			ref := text(e.value)
+			targetDoc, target, pointer, ok := d.resolve(ref)
+			if !ok {
+				continue
+			}
+			key := targetDoc.path + "#" + pointer
+			if name, schema := schemaName(pointer); schema {
+				if !seen[key] {
+					seen[key] = true
+					*into = append(*into, schemaRef{doc: targetDoc, node: target, name: name, key: key})
+				}
+				continue
+			}
+			if visited[key] {
+				continue
+			}
+			visited[key] = true
+			targetDoc.schemaRefs(target, into, seen, visited)
+		}
+	case yaml.SequenceNode:
+		for _, item := range node.Content {
+			d.schemaRefs(item, into, seen, visited)
+		}
+	}
+}
+
+// shapes are what an operation sends and what it answers with, by the names the
+// document gives them.
+//
+// A body behind a $ref has a name a reader can look up in Messages; an inline
+// schema has none, and inventing one would put a shape in the catalog the
+// document does not have. A success with no body at all is answered by its
+// status: 204 is a real answer, and saying nothing for it reads as a call that
+// never came back.
+func (d *document) shapes(item, operation *yaml.Node) (string, string) {
+	request := d.bodyName(child(operation, "requestBody"))
+	if request == "" {
+		request = d.bodyParameter(operation, item)
+	}
+
+	code, body := "", (*yaml.Node)(nil)
+	for _, e := range entries(child(operation, "responses")) {
+		status, err := strconv.Atoi(e.key)
+		if err != nil || status < 200 || status > 299 {
+			continue
+		}
+		if code == "" || e.key < code {
+			code, body = e.key, e.value
+		}
+	}
+	if body == nil {
+		return request, ""
+	}
+	if name := d.bodyName(body); name != "" {
+		return request, name
+	}
+
+	return request, code
+}
+
+// bodyName is the component schema a request or response body names, following
+// one $ref to a shared component on the way. A list of a schema is that schema
+// with brackets: the message is the item, and the reader wants to see there are
+// several.
+//
+// OpenAPI 3 puts the schema under a media type; Swagger 2.0 puts it straight on
+// the response. Both are read, so a document generated by a tool that still
+// writes 2.0 - swag, for one - names its shapes like any other.
+func (d *document) bodyName(node *yaml.Node) string {
+	bodyDoc, body := d.follow(node)
+	for _, e := range entries(child(body, "content")) {
+		mediaDoc, media := bodyDoc.follow(e.value)
+		schema := child(media, "schema")
+		if name := mediaDoc.schemaTypeName(schema); name != "" {
+			return name
+		}
+	}
+	if schema := child(body, "schema"); schema != nil {
+		return bodyDoc.schemaTypeName(schema)
+	}
+
+	return ""
+}
+
+// bodyParameter is the Swagger 2.0 request body: a parameter with `in: body`,
+// on the operation or inherited from its path item, whose schema is the shape.
+// The operation's own list is read first, because it is the one that wins when
+// both declare one.
+func (d *document) bodyParameter(operation, item *yaml.Node) string {
+	for _, owner := range []*yaml.Node{operation, item} {
+		for _, param := range itemsOf(child(owner, "parameters")) {
+			paramDoc, param := d.follow(param)
+			if text(child(param, "in")) != "body" {
+				continue
+			}
+			return paramDoc.schemaTypeName(child(param, "schema"))
+		}
+	}
+
+	return ""
+}
+
+func (d *document) schemaTypeName(node *yaml.Node) string {
+	if _, _, pointer, ok := d.resolve(text(child(node, "$ref"))); ok {
+		if name, schema := schemaName(pointer); schema {
+			return name
+		}
+	}
+	if text(child(node, "type")) == "array" {
+		if name := d.schemaTypeName(child(node, "items")); name != "" {
+			return name + "[]"
+		}
+	}
+	for _, keyword := range []string{"oneOf", "anyOf"} {
+		var names []string
+		for _, branch := range itemsOf(child(node, keyword)) {
+			if name := d.schemaTypeName(branch); name != "" {
+				names = append(names, name)
+			}
+		}
+		if len(names) > 0 {
+			return strings.Join(unique(names), " | ")
+		}
+	}
+	return ""
+}
+
+// follow resolves chains of local or relative-file component references.
+func (d *document) follow(node *yaml.Node) (*document, *yaml.Node) {
+	seen := map[string]bool{}
+	for {
+		ref := text(child(node, "$ref"))
+		nextDoc, next, pointer, ok := d.resolve(ref)
+		if !ok {
+			return d, node
+		}
+		key := nextDoc.path + "#" + pointer
+		if seen[key] {
+			return d, node
+		}
+		seen[key] = true
+		d, node = nextDoc, next
+	}
+}
+
+// resolve follows JSON Pointer refs in this document or in a relative YAML/
+// JSON file. Remote refs are intentionally not fetched by an extractor.
+func (d *document) resolve(ref string) (*document, *yaml.Node, string, bool) {
+	if ref == "" || strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+		return nil, nil, "", false
+	}
+	file, fragment, hasFragment := strings.Cut(ref, "#")
+	if !hasFragment {
+		return nil, nil, "", false
+	}
+	target := d
+	if file != "" {
+		loaded, err := loadCached(filepath.Join(filepath.Dir(d.path), filepath.FromSlash(file)), d.cache)
+		if err != nil {
+			return nil, nil, "", false
+		}
+		target = loaded
+	}
+	pointer := strings.TrimPrefix(fragment, "/")
+	if pointer == "" {
+		return target, target.root, pointer, true
+	}
+	parts := strings.Split(pointer, "/")
+	for i := range parts {
+		parts[i] = strings.ReplaceAll(strings.ReplaceAll(parts[i], "~1", "/"), "~0", "~")
+	}
+	node := child(target.root, parts...)
+	return target, node, pointer, node != nil
+}
+
+func schemaName(pointer string) (string, bool) {
+	for _, home := range schemaHomes {
+		name, ok := strings.CutPrefix(pointer, home)
+		if !ok || name == "" || strings.Contains(name, "/") {
+			continue
+		}
+		return strings.ReplaceAll(strings.ReplaceAll(name, "~1", "/"), "~0", "~"), true
+	}
+	return "", false
+}
+
+// refName is the name a reference spells even when it points at nothing: a
+// document that names a schema it never defines still tells the reader what the
+// author called it.
+func refName(ref string) (string, bool) {
+	pointer, ok := strings.CutPrefix(ref, "#/")
+	if !ok {
+		return "", false
+	}
+	return schemaName(pointer)
+}

package/plugins/extract-project/README.md

@@ -0,0 +1,22 @@
+# extract-project
+
+The neutral baseline extractor. It describes a repository as one component in
+one architecture group without requiring a domain model or a particular source
+layout.
+
+It reads only files already present in the checkout: `README.md`, language
+package manifests, `Dockerfile`, Compose and Helm markers, and the runner files
+(`Makefile`, `justfile`, `Taskfile.yml`, `package.json` scripts, poe and pdm
+tasks in `pyproject.toml`, `pom.xml`, the Gradle script, cargo aliases and
+xtask subcommands), whose entries become the component's commands. It does not execute
+the project, resolve dependencies, import application code or write into the
+input tree. Its response contains a catalog fragment named `project.json`;
+the host decides where that file is written.
+
+By default the group is a `system`. A repository with an executable or a
+container definition becomes an `application`; otherwise it becomes a
+`library`. Options can state more precise roles such as `team`, `worker`, `job`
+or `cli`, and `technologies` when the manifests sit above the root, as for one
+component of several in a repository. Language-specific domain extractors and
+contract extractors can be run beside it: fragment merging fills in
+aggregates, interfaces, channels and stores without duplicating the component.

package/plugins/extract-project/describe.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	_ "embed"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+//go:embed options.schema.json
+var optionsSchema []byte
+
+func descriptor() plugin.Descriptor {
+	return plugin.Descriptor{
+		Name:    "extract-project",
+		Summary: "Reads repository metadata and describes a group and component without assuming DDD or a particular language.",
+		Phases:  []string{plugin.PhaseExtract},
+		Options: optionsSchema,
+	}
+}

package/plugins/extract-project/describe_test.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/shortlink-org/portolan/plugin/schematest"
+)
+
+func TestOptionsSchemaMatchesStruct(t *testing.T) {
+	schematest.Check(t, optionsSchema, Options{})
+}

package/plugins/extract-project/extract.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/shortlink-org/portolan/catalog"
+	"github.com/shortlink-org/portolan/internal/commands"
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+var packageRepository = regexp.MustCompile(`(?m)"repository"\s*:\s*(?:"([^"]+)"|\{[^}]*"url"\s*:\s*"([^"]+)")`)
+var cargoName = regexp.MustCompile(`(?m)^name\s*=\s*"([^"]+)"`)
+
+func extract(in plugin.Input, opts Options) (plugin.Response, error) {
+	root := in.Root
+	base := slug(filepath.Base(root))
+	group := firstNonEmpty(slug(opts.Group), base, "project")
+	component := firstNonEmpty(slug(opts.Component), base, "component")
+	readme := read(filepath.Join(root, "README.md"))
+	b := &plugin.Builder{}
+	cmds, warnings := commands.Read(root)
+	for _, warning := range warnings {
+		b.Warn(root, warning)
+	}
+
+	fragment := catalog.Catalog{
+		GeneratedAt: in.GeneratedAt,
+		Commit:      in.Commit,
+		Contexts: []catalog.BoundedContext{{
+			ID:      group,
+			Slug:    group,
+			Name:    firstNonEmpty(opts.GroupName, title(group)),
+			Summary: opts.GroupSummary,
+			Kind:    catalog.GroupKind(firstNonEmpty(opts.GroupKind, string(catalog.GroupKindSystem))),
+			Services: []catalog.Service{{
+				ID:           group + "." + component,
+				Slug:         component,
+				Name:         firstNonEmpty(opts.ComponentName, markdownTitle(readme), title(component)),
+				Repo:         firstNonEmpty(opts.Repo, repository(root)),
+				Path:         filepath.ToSlash(root),
+				Readme:       readme,
+				Kind:         catalog.ComponentKind(firstNonEmpty(opts.ComponentKind, inferredKind(root))),
+				Technologies: stated(opts.Technologies, technologies(root)),
+				Provides:     []catalog.RpcService{},
+				Consumes:     []catalog.RpcCall{},
+				Aggregates:   []catalog.Aggregate{},
+				Commands:     cmds,
+			}},
+		}},
+		Defs:  map[string]catalog.TypeDef{},
+		Flows: []catalog.Flow{},
+		Adrs:  []catalog.Adr{},
+	}
+
+	encoded, err := json.MarshalIndent(fragment, "", "  ")
+	if err != nil {
+		return plugin.Response{}, err
+	}
+	b.File(firstNonEmpty(opts.Out, "project.json"), string(encoded)+"\n")
+	return b.Response(), nil
+}
+
+func repository(root string) string {
+	for _, line := range strings.Split(read(filepath.Join(root, "go.mod")), "\n") {
+		if value, ok := strings.CutPrefix(strings.TrimSpace(line), "module "); ok {
+			return strings.TrimSpace(value)
+		}
+	}
+	if match := packageRepository.FindStringSubmatch(read(filepath.Join(root, "package.json"))); len(match) > 0 {
+		return strings.TrimSuffix(firstNonEmpty(match[1], match[2]), ".git")
+	}
+	if match := cargoName.FindStringSubmatch(read(filepath.Join(root, "Cargo.toml"))); len(match) > 0 {
+		return match[1]
+	}
+	return ""
+}
+
+func inferredKind(root string) string {
+	if exists(filepath.Join(root, "cmd")) || exists(filepath.Join(root, "Dockerfile")) || exists(filepath.Join(root, "docker-compose.yml")) || exists(filepath.Join(root, "docker-compose.yaml")) {
+		return string(catalog.ComponentKindApplication)
+	}
+	return string(catalog.ComponentKindLibrary)
+}
+
+func technologies(root string) []string {
+	goMod := read(filepath.Join(root, "go.mod"))
+	packageJSON := read(filepath.Join(root, "package.json"))
+	cargo := read(filepath.Join(root, "Cargo.toml"))
+	markers := []struct {
+		name string
+		seen bool
+	}{
+		{"Go", goMod != ""},
+		{"Node.js", packageJSON != ""},
+		{"Rust", cargo != ""},
+		{"Java", exists(filepath.Join(root, "pom.xml")) || exists(filepath.Join(root, "build.gradle")) || exists(filepath.Join(root, "build.gradle.kts"))},
+		{"Python", exists(filepath.Join(root, "pyproject.toml")) || exists(filepath.Join(root, "requirements.txt")) || exists(filepath.Join(root, "manage.py"))},
+		{"Gin", strings.Contains(goMod, "github.com/gin-gonic/gin")},
+		{"Kafka", strings.Contains(goMod, "segmentio/kafka-go") || strings.Contains(goMod, "Shopify/sarama") || strings.Contains(goMod, "IBM/sarama") || strings.Contains(goMod, "ThreeDotsLabs/watermill-kafka")},
+		{"PostgreSQL", strings.Contains(goMod, "jackc/pgx") || strings.Contains(goMod, "lib/pq")},
+		{"Redis", strings.Contains(goMod, "go-redis/redis") || strings.Contains(goMod, "redis/go-redis")},
+		{"OpenTelemetry", strings.Contains(goMod, "go.opentelemetry.io/otel")},
+		{"River", strings.Contains(goMod, "riverqueue/river")},
+		{"Watermill", strings.Contains(goMod, "github.com/ThreeDotsLabs/watermill")},
+		{"Docker", exists(filepath.Join(root, "Dockerfile")) || exists(filepath.Join(root, "docker-compose.yml")) || exists(filepath.Join(root, "docker-compose.yaml"))},
+		{"Helm", hasHelm(root)},
+	}
+	out := make([]string, 0, len(markers))
+	for _, marker := range markers {
+		if marker.seen {
+			out = append(out, marker.name)
+		}
+	}
+	return out
+}
+
+func hasHelm(root string) bool {
+	if exists(filepath.Join(root, "Chart.yaml")) || exists(filepath.Join(root, "charts")) || exists(filepath.Join(root, "helm")) {
+		return true
+	}
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		return false
+	}
+	for _, entry := range entries {
+		if entry.IsDir() && strings.HasPrefix(strings.ToLower(entry.Name()), ".helm") {
+			return true
+		}
+	}
+	return false
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func read(path string) string {
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(contents))
+}
+
+func markdownTitle(md string) string {
+	fence := ""
+	for _, line := range strings.Split(md, "\n") {
+		trimmed := strings.TrimSpace(line)
+		marker := ""
+		if strings.HasPrefix(trimmed, "```") {
+			marker = "```"
+		} else if strings.HasPrefix(trimmed, "~~~") {
+			marker = "~~~"
+		}
+		if marker != "" {
+			if fence == "" {
+				fence = marker
+			} else if fence == marker {
+				fence = ""
+			}
+			continue
+		}
+		if fence != "" {
+			continue
+		}
+		if value, ok := strings.CutPrefix(trimmed, "# "); ok {
+			return strings.TrimSpace(value)
+		}
+	}
+	return ""
+}
+
+func slug(value string) string {
+	value = strings.ToLower(strings.TrimSpace(value))
+	var out strings.Builder
+	dash := false
+	for _, r := range value {
+		if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
+			out.WriteRune(r)
+			dash = false
+		} else if out.Len() > 0 && !dash {
+			out.WriteByte('-')
+			dash = true
+		}
+	}
+	return strings.TrimSuffix(out.String(), "-")
+}
+
+func title(value string) string {
+	parts := strings.FieldsFunc(value, func(r rune) bool { return r == '-' || r == '_' })
+	for i := range parts {
+		if parts[i] != "" {
+			parts[i] = strings.ToUpper(parts[i][:1]) + parts[i][1:]
+		}
+	}
+	return strings.Join(parts, " ")
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if strings.TrimSpace(value) != "" {
+			return value
+		}
+	}
+	return ""
+}
+
+// stated prefers what the manifest says to what the tree implies: a component
+// whose manifests sit above its root, one of several in a repository, has
+// nothing at its root for technologies to read.
+func stated(told, inferred []string) []string {
+	if len(told) > 0 {
+		return told
+	}
+	return inferred
+}