@shortlink-org/portolan 0.1.0

package/plugins/pyplugin/source.py

@@ -0,0 +1,285 @@
+"""The tree, read as syntax.
+
+The project is never imported: `django.setup()` runs the code it finds, reads
+the environment and opens sockets, and an extractor that does any of those is
+no longer a pure function of the tree. Everything here is `ast`, and names are
+resolved by import and by file, the way `extract-ts` resolves them without a
+type checker.
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+from dataclasses import dataclass, field
+from typing import Dict, Iterator, List, Optional, Tuple
+
+# Directories that hold no claim about the model: someone else's code, a build,
+# a history of the schema rather than the schema, and the tests.
+SKIP = {
+    ".git",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".tox",
+    ".venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "env",
+    "migrations",
+    "node_modules",
+    "site-packages",
+    "static",
+    "templates",
+    "tests",
+    "venv",
+}
+
+
+@dataclass
+class Import:
+    """A name a module can see, and where it came from."""
+
+    module: str  # dotted, as the project spells it: "invoices.events"
+    name: str  # the name in that module, "*" for `import x`
+    line: int = 0
+
+
+@dataclass
+class Module:
+    path: str  # absolute
+    rel: str  # relative to the working directory, forward slashes
+    dotted: str  # "invoices.models"
+    tree: ast.Module
+    imports: Dict[str, Import] = field(default_factory=dict)
+
+    @property
+    def package(self) -> str:
+        return self.dotted.rsplit(".", 1)[0] if "." in self.dotted else ""
+
+    def where(self, node: ast.AST) -> str:
+        return "%s:%d" % (self.rel, getattr(node, "lineno", 0))
+
+    def classes(self) -> List[ast.ClassDef]:
+        return [n for n in self.tree.body if isinstance(n, ast.ClassDef)]
+
+    def functions(self) -> List[ast.FunctionDef]:
+        return [n for n in self.tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))]
+
+
+class Project:
+    """Every module under the source root, by its dotted name."""
+
+    def __init__(self, root: str, source: str, rel):
+        self.root = root
+        self.source = source
+        self.rel = rel
+        self.modules: Dict[str, Module] = {}
+        self.broken: List[Tuple[str, str]] = []
+        self._load()
+
+    def _load(self) -> None:
+        for path in walk(self.source):
+            dotted = dotted_name(self.source, path)
+            try:
+                tree = ast.parse(read(path), filename=path)
+            except SyntaxError as err:
+                self.broken.append((self.rel(path), str(err)))
+                continue
+            module = Module(path=path, rel=self.rel(path), dotted=dotted, tree=tree)
+            module.imports = read_imports(module)
+            self.modules[dotted] = module
+
+    def module(self, dotted: str) -> Optional[Module]:
+        hit = self.modules.get(dotted)
+        if hit is not None:
+            return hit
+        return self.modules.get(dotted + ".__init__")
+
+    def under(self, package: str) -> List[Module]:
+        """Every module of a package, the package's own `__init__` included, in
+        path order so the fragment does not move when a directory is re-read."""
+        prefix = package + "."
+        out = [m for name, m in self.modules.items() if name == package or name.startswith(prefix)]
+        return sorted(out, key=lambda m: m.rel)
+
+    def resolve(self, module: Module, name: str) -> Optional[Tuple[Module, str]]:
+        """The module a name in this one came from, and what it is called
+        there. A name defined here resolves to here."""
+        imported = module.imports.get(name)
+        if imported is None:
+            if any(node_name(n) == name for n in module.tree.body):
+                return module, name
+            return None
+        target = self.module(imported.module)
+        if target is None:
+            return None
+        if imported.name == "*":
+            return target, name
+        return target, imported.name
+
+
+def walk(root: str) -> Iterator[str]:
+    """Every .py file under a directory, in a stable order."""
+    for base, dirs, files in os.walk(root):
+        dirs[:] = sorted(d for d in dirs if d not in SKIP and not d.startswith("."))
+        for name in sorted(files):
+            if name.endswith(".py"):
+                yield os.path.join(base, name)
+
+
+def read(path: str) -> str:
+    with open(path, "r", encoding="utf-8") as handle:
+        return handle.read()
+
+
+def dotted_name(source: str, path: str) -> str:
+    rel = os.path.relpath(path, source)
+    rel = rel.replace(os.sep, "/")
+    if rel.endswith(".py"):
+        rel = rel[: -len(".py")]
+    return rel.replace("/", ".")
+
+
+def read_imports(module: Module) -> Dict[str, Import]:
+    """What each name in a module refers to. A relative import is resolved
+    against the module's own package, which is how `from .events import X`
+    finds the file beside it."""
+    out: Dict[str, Import] = {}
+    for node in ast.walk(module.tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                out[alias.asname or alias.name.split(".")[0]] = Import(alias.name, "*", node.lineno)
+        elif isinstance(node, ast.ImportFrom):
+            base = node.module or ""
+            if node.level:
+                parts = module.dotted.split(".")
+                # A package's __init__ is one level shallower than its path says.
+                if parts and parts[-1] == "__init__":
+                    parts = parts[:-1]
+                parts = parts[: len(parts) - node.level]
+                base = ".".join([p for p in parts if p] + ([base] if base else []))
+            for alias in node.names:
+                out[alias.asname or alias.name] = Import(base, alias.name, node.lineno)
+    return out
+
+
+def node_name(node: ast.AST) -> str:
+    if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+        return node.name
+    if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+        return node.targets[0].id
+    if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
+        return node.target.id
+    return ""
+
+
+def dotted(node: ast.AST) -> str:
+    """`models.CharField` for an attribute chain, `Invoice` for a name, "" for
+    anything else."""
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        base = dotted(node.value)
+        return base + "." + node.attr if base else node.attr
+    if isinstance(node, ast.Call):
+        return dotted(node.func)
+    return ""
+
+
+def const_str(node: Optional[ast.AST]) -> str:
+    if isinstance(node, ast.Constant) and isinstance(node.value, str):
+        return node.value
+    return ""
+
+
+def keyword(call: ast.Call, name: str) -> Optional[ast.AST]:
+    for kw in call.keywords:
+        if kw.arg == name:
+            return kw.value
+    return None
+
+
+def keyword_str(call: ast.Call, name: str) -> str:
+    return const_str(keyword(call, name))
+
+
+def keyword_bool(call: ast.Call, name: str) -> bool:
+    value = keyword(call, name)
+    return isinstance(value, ast.Constant) and value.value is True
+
+
+def keyword_int(call: ast.Call, name: str) -> Optional[int]:
+    value = keyword(call, name)
+    if isinstance(value, ast.Constant) and isinstance(value.value, int) and not isinstance(value.value, bool):
+        return value.value
+    return None
+
+
+def doc(node: ast.AST) -> str:
+    """The docstring, first paragraph only: the rest is for whoever opens the
+    file, and a page wants the sentence."""
+    if not isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+        return ""
+    text = ast.get_docstring(node, clean=True) or ""
+    return text.split("\n\n")[0].strip().replace("\n", " ")
+
+
+def type_of(node: Optional[ast.AST]) -> str:
+    """An annotation as written. `Optional[Money]` stays `Optional[Money]`: the
+    catalog keeps types the way the author declared them."""
+    if node is None:
+        return ""
+    try:
+        return ast.unparse(node)  # type: ignore[attr-defined]
+    except Exception:
+        return dotted(node)
+
+
+def bases(node: ast.ClassDef) -> List[str]:
+    return [dotted(b) for b in node.bases]
+
+
+def decorators(node: ast.AST) -> List[ast.AST]:
+    return list(getattr(node, "decorator_list", []) or [])
+
+
+def decorator_named(node: ast.AST, *names: str) -> Optional[ast.AST]:
+    """The decorator whose last segment is one of these: `@action`, `@receiver`,
+    `@transition`, however it was imported."""
+    for dec in decorators(node):
+        if dotted(dec).split(".")[-1] in names:
+            return dec
+    return None
+
+
+def assigned(node: ast.AST) -> List[Tuple[str, ast.AST, ast.AST]]:
+    """(name, value, node) for each simple assignment in a class or module body."""
+    out = []
+    for stmt in getattr(node, "body", []) or []:
+        if isinstance(stmt, ast.Assign):
+            for target in stmt.targets:
+                if isinstance(target, ast.Name):
+                    out.append((target.id, stmt.value, stmt))
+        elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) and stmt.value is not None:
+            out.append((stmt.target.id, stmt.value, stmt))
+    return out
+
+
+def methods(node: ast.ClassDef) -> List[ast.AST]:
+    return [n for n in node.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))]
+
+
+def method(node: ast.ClassDef, name: str) -> Optional[ast.AST]:
+    for m in methods(node):
+        if m.name == name:
+            return m
+    return None
+
+
+def inner_class(node: ast.ClassDef, name: str) -> Optional[ast.ClassDef]:
+    for stmt in node.body:
+        if isinstance(stmt, ast.ClassDef) and stmt.name == name:
+            return stmt
+    return None

package/plugins/verify-codeowners/describe.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	_ "embed"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+//go:embed options.schema.json
+var optionsSchema []byte
+
+func descriptor() plugin.Descriptor {
+	return plugin.Descriptor{
+		Name:    "verify-codeowners",
+		Summary: "Reads the repository's CODEOWNERS against the catalog: who to ask about each service, which services nobody owns, and which rules own nothing.",
+		Phases:  []string{plugin.PhaseVerify},
+		Options: optionsSchema,
+	}
+}

package/plugins/verify-codeowners/describe_test.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/shortlink-org/portolan/plugin/schematest"
+)
+
+func TestOptionsSchemaMatchesStruct(t *testing.T) {
+	schematest.Check(t, optionsSchema, Options{})
+}

package/plugins/verify-codeowners/main.go

@@ -0,0 +1,75 @@
+// Command verify-codeowners says who to ask about each service, by reading the
+// CODEOWNERS file the repository already keeps.
+//
+// Ownership is the question a catalog is opened for and the one it answered
+// worst: `owner` on a flow or a store means the bounded context, not a team,
+// so "who do I ask about shop.oms" had no answer anywhere on the page. The
+// answer was already written down - every repository with review rules has it,
+// in a file the forge itself enforces - and typing it a second time into the
+// manifest would make the copy the thing that goes stale.
+//
+// So it is read, never written, and never resolved. A handle is what the file
+// says and what a reviewer types; turning `@acme/oms-team` into the people in
+// it is a call to a forge's API, which needs a credential this deliberately
+// does not have and would answer differently tomorrow.
+//
+// It is a verifier rather than an extractor because a rule is a path and only
+// the merged catalog knows where each service is. It earns the name twice
+// over: a service no rule matches, and a rule that matches no service, are
+// both reported - the second being the failure a CODEOWNERS file has and can
+// never report about itself, a team believing it owns something it does not.
+//
+// Nothing here is forge-specific. GitHub, GitLab and Gitea read the same file
+// from the same three places; where GitLab's sections change whose rule wins,
+// the file is read the flatter way and said so out loud.
+//
+//	{
+//	  "plugins": [
+//	    { "name": "codeowners", "process": { "command": "go", "args": ["run", "./plugins/verify-codeowners"] } }
+//	  ],
+//	  "verify": [
+//	    { "plugin": "codeowners", "in": ".github", "out": "data", "options": { "out": "owners.json" } }
+//	  ]
+//	}
+//
+// Point `in` at the directory the file is in rather than at the repository
+// root. The host dates a fragment from the last commit to touch the step's
+// input, and the subject of this one is the CODEOWNERS file: rooted at the
+// repository, it would be restamped by every commit ever made.
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// Options are what the manifest tells the verifier.
+type Options struct {
+	// File is the CODEOWNERS to read, relative to the input root. Left out,
+	// the three places a forge looks: CODEOWNERS, .github/CODEOWNERS and
+	// docs/CODEOWNERS, in that order.
+	File string `json:"file,omitempty"`
+
+	// Out names the fragment file.
+	Out string `json:"out,omitempty"`
+}
+
+func main() {
+	if err := run(os.Stdin, os.Stdout); err != nil {
+		fmt.Fprintln(os.Stderr, "portolan-verify-codeowners:", err)
+		os.Exit(1)
+	}
+}
+
+func run(stdin io.Reader, stdout io.Writer) error {
+	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
+		if req.Input.Root == "" {
+			return plugin.Response{}, fmt.Errorf("no input root: there is nowhere to look for a CODEOWNERS")
+		}
+
+		return verify(req, opts)
+	})
+}

package/plugins/verify-codeowners/match.go

@@ -0,0 +1,85 @@
+package main
+
+// Whether a rule owns a directory.
+//
+// CODEOWNERS patterns are gitignore patterns, and the question asked here is
+// narrower than the one gitignore answers: not "does this pattern match this
+// file" but "does this pattern own this SERVICE", where a service is a
+// directory. A rule owns a directory when it matches the directory itself or
+// anything above it - `services/` owns `services/oms` - and does not when it
+// matches only something inside it: `services/oms/internal` is a rule about
+// part of the service, and reading it as ownership of the whole would hand a
+// team a page they never asked for.
+//
+// The two shapes a pattern comes in are the whole of the rule:
+//
+//   - anchored, when it has a `/` anywhere but the end - `services/oms`,
+//     `/docs` - and is matched against the path from the repository root;
+//   - floating, when it is a single segment - `*.go`, `internal` - and matches
+//     that segment at any depth, which is how gitignore has always read one.
+//
+// `*` and `?` stay inside a segment, `**` spans any number of them.
+
+import (
+	"path"
+	"strings"
+)
+
+// owns reports whether pattern owns the directory at target, both written
+// repository-relative and with forward slashes.
+func owns(pattern, target string) bool {
+	pattern = strings.TrimSpace(pattern)
+	target = strings.Trim(target, "/")
+	if pattern == "" || target == "" {
+		return false
+	}
+
+	// A pattern that is only a `/*` or `*` owns everything at that level.
+	anchored := strings.HasPrefix(pattern, "/") || strings.Contains(strings.TrimSuffix(pattern, "/"), "/")
+	pattern = strings.Trim(pattern, "/")
+	if pattern == "" {
+		return true // "/" - the whole repository.
+	}
+
+	segments := strings.Split(target, "/")
+	if !anchored {
+		// Any segment matching means the directory it names is owned, and so
+		// is everything under it - including this one.
+		for _, segment := range segments {
+			if ok, _ := path.Match(pattern, segment); ok {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	return matchPrefix(strings.Split(pattern, "/"), segments)
+}
+
+// matchPrefix reports whether the pattern matches the target or a prefix of
+// it. A pattern longer than the target names something inside the directory
+// and owns no more of it than that.
+func matchPrefix(pattern, target []string) bool {
+	if len(pattern) == 0 {
+		return true // Every segment of the pattern matched; the rest is inside it.
+	}
+	if pattern[0] == "**" {
+		// Zero or more segments: try here, then one deeper.
+		for i := 0; i <= len(target); i++ {
+			if matchPrefix(pattern[1:], target[i:]) {
+				return true
+			}
+		}
+
+		return false
+	}
+	if len(target) == 0 {
+		return false
+	}
+	if ok, _ := path.Match(pattern[0], target[0]); !ok {
+		return false
+	}
+
+	return matchPrefix(pattern[1:], target[1:])
+}

package/plugins/verify-codeowners/match_test.go

@@ -0,0 +1,47 @@
+package main
+
+import "testing"
+
+// The rule the whole plugin turns on: a pattern owns a directory when it names
+// the directory or anything above it, and does not when it names only
+// something inside it.
+func TestOwns(t *testing.T) {
+	cases := []struct {
+		pattern string
+		target  string
+		want    bool
+		why     string
+	}{
+		{"examples/shop/oms", "examples/shop/oms", true, "the directory itself"},
+		{"examples/shop", "examples/shop/oms", true, "a directory above it"},
+		{"examples/", "examples/shop/oms", true, "a trailing slash is still a directory"},
+		{"/examples", "examples/shop/oms", true, "anchored at the root"},
+		{"*", "examples/shop/oms", true, "everything"},
+		{"/", "examples/shop/oms", true, "the whole repository"},
+
+		{"examples/shop/oms/internal", "examples/shop/oms", false, "a rule about part of the service is not a rule about the service"},
+		{"examples/auth", "examples/shop/oms", false, "a sibling"},
+		{"docs", "examples/shop/oms", false, "somewhere else entirely"},
+
+		// Floating patterns match at any depth, which is what gitignore has
+		// always meant by a pattern with no slash in it.
+		{"oms", "examples/shop/oms", true, "a single segment matches at any depth"},
+		{"shop", "examples/shop/oms", true, "including a segment above"},
+		{"om", "examples/shop/oms", false, "and matches whole segments, not prefixes"},
+
+		{"examples/*/oms", "examples/shop/oms", true, "a star stays inside its segment"},
+		{"examples/*", "examples/shop/oms", true, "and matches the segment above"},
+		{"examples/*/oms", "examples/shop/cart/oms", false, "so it does not cross a slash"},
+		{"examples/**/oms", "examples/shop/cart/oms", true, "two stars do"},
+		{"examples/**", "examples/shop/oms", true, "two stars also match nothing at all"},
+
+		{"", "examples/shop/oms", false, "an empty pattern owns nothing"},
+		{"examples", "", false, "and nothing is owned by no pattern"},
+	}
+
+	for _, c := range cases {
+		if got := owns(c.pattern, c.target); got != c.want {
+			t.Errorf("owns(%q, %q) = %v, want %v - %s", c.pattern, c.target, got, c.want, c.why)
+		}
+	}
+}

package/plugins/verify-codeowners/options.schema.json

@@ -0,0 +1,15 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "file": {
+      "type": "string",
+      "description": "The CODEOWNERS to read, relative to the input root. Left out, the three places a forge looks, in order: CODEOWNERS, .github/CODEOWNERS, docs/CODEOWNERS. Its patterns are matched against each service's path as the catalog holds it, so they are repository-relative wherever the file itself sits.",
+      "examples": ["CODEOWNERS", ".github/CODEOWNERS"]
+    },
+    "out": {
+      "type": "string",
+      "description": "Name of the fragment file, inside the step's output directory. Left out, owners.json."
+    }
+  }
+}