@shortlink-org/portolan 0.1.0

package/plugins/extract-proto/resolve.go

@@ -0,0 +1,222 @@
+package main
+
+// Turning a name as written into the thing it names.
+//
+// A proto reference is relative to where it was written: `Money` inside
+// `shop.v1.Order` might mean `shop.v1.Order.Money`, `shop.v1.Money`, or
+// `Money` at the root, and proto searches outwards in that order. A leading dot
+// means the search is skipped and the name is already absolute.
+//
+// A name that resolves to nothing is NOT an error here. A narrowed vendored
+// copy importing a file nobody vendored beside it is the normal case - see the
+// header of parse.go - and the answer is to say the type is unresolved and
+// carry on, not to refuse to describe the file.
+
+import (
+	"sort"
+	"strings"
+)
+
+// scalars are proto's built-in types, which never resolve to a declaration.
+var scalars = map[string]bool{
+	"double": true, "float": true, "int32": true, "int64": true,
+	"uint32": true, "uint64": true, "sint32": true, "sint64": true,
+	"fixed32": true, "fixed64": true, "sfixed32": true, "sfixed64": true,
+	"bool": true, "string": true, "bytes": true,
+}
+
+func isScalar(name string) bool { return scalars[name] }
+
+// Index is every declaration in a set of files, by fully-qualified name.
+type Index struct {
+	files []*File
+
+	messages map[string]*Message
+	enums    map[string]*Enum
+	fileOf   map[string]*File
+	fqnOf    map[*Message]string
+}
+
+// NewIndex walks the files once and records every message and enum in them,
+// nested ones included, under the name the rest of the world would use.
+func NewIndex(files []*File) *Index {
+	ix := &Index{
+		files:    files,
+		messages: map[string]*Message{},
+		enums:    map[string]*Enum{},
+		fileOf:   map[string]*File{},
+		fqnOf:    map[*Message]string{},
+	}
+
+	for _, file := range files {
+		for _, m := range file.Messages {
+			ix.addMessage(file, file.Package, m)
+		}
+		for _, e := range file.Enums {
+			ix.addEnum(file, join(file.Package, e.Name), e)
+		}
+	}
+
+	return ix
+}
+
+func (ix *Index) addMessage(file *File, scope string, m *Message) {
+	fqn := join(scope, m.Name)
+	ix.messages[fqn] = m
+	ix.fileOf[fqn] = file
+	ix.fqnOf[m] = fqn
+
+	for _, nested := range m.Messages {
+		ix.addMessage(file, fqn, nested)
+	}
+	for _, e := range m.Enums {
+		ix.addEnum(file, join(fqn, e.Name), e)
+	}
+}
+
+func (ix *Index) addEnum(file *File, fqn string, e *Enum) {
+	ix.enums[fqn] = e
+	ix.fileOf[fqn] = file
+}
+
+// Resolve finds what `name`, written inside `scope`, refers to.
+//
+// scope is a fully-qualified name - a package, or a message inside one. The
+// search walks outwards from it, which is what proto itself does.
+func (ix *Index) Resolve(scope, name string) (string, bool) {
+	if name == "" || isScalar(name) {
+		return "", false
+	}
+
+	if strings.HasPrefix(name, ".") {
+		fqn := strings.TrimPrefix(name, ".")
+
+		return fqn, ix.declared(fqn)
+	}
+
+	parts := strings.Split(scope, ".")
+	for i := len(parts); i >= 0; i-- {
+		fqn := join(strings.Join(parts[:i], "."), name)
+		if ix.declared(fqn) {
+			return fqn, true
+		}
+	}
+
+	return "", false
+}
+
+func (ix *Index) declared(fqn string) bool {
+	_, message := ix.messages[fqn]
+	_, enum := ix.enums[fqn]
+
+	return message || enum
+}
+
+func (ix *Index) Message(fqn string) *Message { return ix.messages[fqn] }
+func (ix *Index) Enum(fqn string) *Enum       { return ix.enums[fqn] }
+func (ix *Index) FileOf(fqn string) *File     { return ix.fileOf[fqn] }
+
+// FQN is the name a message is known by, once it has been indexed.
+func (ix *Index) FQN(m *Message) string { return ix.fqnOf[m] }
+
+// Names is every declared name, sorted. Determinism is not a nicety here: the
+// fragment these end up in is committed and compared by `gen:check`, so a map
+// iterated in Go's order would produce a different file every run.
+func (ix *Index) Names() []string {
+	out := make([]string, 0, len(ix.messages)+len(ix.enums))
+	for fqn := range ix.messages {
+		out = append(out, fqn)
+	}
+	for fqn := range ix.enums {
+		out = append(out, fqn)
+	}
+	sort.Strings(out)
+
+	return out
+}
+
+// TypeRef is one field's type, resolved as far as it could be.
+type TypeRef struct {
+	// Written is the type as the author typed it, rendered the way the catalog
+	// renders types: `[]LineItem`, `map[string]Money`, `Money`.
+	Written string
+
+	// FQN is what the named type resolves to, empty for a scalar or for a name
+	// nothing in the tree declares.
+	FQN string
+
+	// Unresolved is the name that could not be found, empty when there was
+	// nothing to find or the search succeeded. It is a diagnostic, not a
+	// failure: a narrowed vendored copy is expected to have some.
+	Unresolved string
+}
+
+// resolveType renders a field's type and resolves the name inside it.
+//
+// A map is rendered `map[K]V` and resolved on its value: the key of a proto map
+// is always a scalar, and the value is the half that can name a message.
+func (ix *Index) resolveType(scope string, f *Field) TypeRef {
+	if key, value, ok := mapParts(f.Type); ok {
+		inner := ix.resolveType(scope, &Field{Type: value})
+
+		return TypeRef{
+			Written:    "map[" + key + "]" + inner.Written,
+			FQN:        inner.FQN,
+			Unresolved: inner.Unresolved,
+		}
+	}
+
+	ref := TypeRef{Written: shortName(f.Type)}
+	if f.Label == "repeated" {
+		ref.Written = "[]" + ref.Written
+	}
+
+	if isScalar(f.Type) {
+		return ref
+	}
+
+	if fqn, ok := ix.Resolve(scope, f.Type); ok {
+		ref.FQN = fqn
+	} else {
+		ref.Unresolved = f.Type
+	}
+
+	return ref
+}
+
+// mapParts splits `map<string, Money>` into its two arguments.
+func mapParts(typ string) (string, string, bool) {
+	if !strings.HasPrefix(typ, "map<") || !strings.HasSuffix(typ, ">") {
+		return "", "", false
+	}
+	inner := typ[len("map<") : len(typ)-1]
+	comma := strings.Index(inner, ",")
+	if comma < 0 {
+		return "", "", false
+	}
+
+	return strings.TrimSpace(inner[:comma]), strings.TrimSpace(inner[comma+1:]), true
+}
+
+// shortName is the last segment of a dotted name.
+//
+// The catalog spells a type the way a reader says it - `Money`, not
+// `shop.v1.Money` - and its `defs` keys are deliberately bare for the same
+// reason: two sources meaning different things by `Money` should collide
+// visibly rather than coexist as two quietly different entries.
+func shortName(name string) string {
+	name = strings.TrimPrefix(name, ".")
+	if i := strings.LastIndex(name, "."); i >= 0 {
+		return name[i+1:]
+	}
+
+	return name
+}
+
+func join(scope, name string) string {
+	if scope == "" {
+		return name
+	}
+
+	return scope + "." + name
+}

package/plugins/extract-proto/resolve_test.go

@@ -0,0 +1,119 @@
+package main
+
+// Name resolution, on its own. A proto reference means whatever the innermost
+// enclosing scope that declares it means, and a name nothing declares is a
+// diagnostic rather than a failure.
+
+import "testing"
+
+func indexed(t *testing.T) *Index {
+	t.Helper()
+
+	orders, _ := parseFile(t, "orders.proto")
+	money, _ := parseFile(t, "money.proto")
+
+	return NewIndex([]*File{orders, money})
+}
+
+func TestResolvesAcrossFiles(t *testing.T) {
+	ix := indexed(t)
+
+	fqn, ok := ix.Resolve("shop.v1", "Money")
+	if !ok || fqn != "shop.v1.Money" {
+		t.Errorf("Money from shop.v1: %q %v", fqn, ok)
+	}
+	if ix.Message(fqn) == nil {
+		t.Error("the resolved name does not lead back to the message")
+	}
+}
+
+// Proto searches outwards from where the name was written, so the nested type
+// wins over anything of the same name further out.
+func TestResolvesInnermostFirst(t *testing.T) {
+	ix := indexed(t)
+
+	fqn, ok := ix.Resolve("shop.v1.Order", "Address")
+	if !ok || fqn != "shop.v1.Order.Address" {
+		t.Errorf("Address from inside Order: %q %v", fqn, ok)
+	}
+
+	// The same name written one scope out finds nothing, which is the point.
+	if _, ok := ix.Resolve("shop.v1", "Address"); ok {
+		t.Error("a nested type resolved from outside the message that nests it")
+	}
+}
+
+func TestLeadingDotIsAbsolute(t *testing.T) {
+	ix := indexed(t)
+
+	fqn, ok := ix.Resolve("shop.v1.Order", ".shop.v1.Money")
+	if !ok || fqn != "shop.v1.Money" {
+		t.Errorf("absolute name: %q %v", fqn, ok)
+	}
+}
+
+func TestNestedEnumIsIndexed(t *testing.T) {
+	ix := indexed(t)
+
+	fqn, ok := ix.Resolve("shop.v1.Order", "Channel")
+	if !ok || fqn != "shop.v1.Order.Channel" {
+		t.Errorf("nested enum: %q %v", fqn, ok)
+	}
+	if ix.Enum(fqn) == nil {
+		t.Error("the resolved name does not lead back to the enum")
+	}
+}
+
+// The whole reason this parser is tolerant: a narrowed vendored copy imports
+// files nobody vendored beside it, and it must still be describable.
+func TestUnresolvedIsReportedNotFatal(t *testing.T) {
+	ix := indexed(t)
+
+	ref := ix.resolveType("shop.v1", &Field{Type: "Nowhere"})
+	if ref.Unresolved != "Nowhere" {
+		t.Errorf("an unknown type was not reported: %+v", ref)
+	}
+	if ref.Written != "Nowhere" {
+		t.Errorf("an unknown type must still be shown as written: %q", ref.Written)
+	}
+}
+
+// Types are rendered the way the catalog renders types, and resolved on the
+// half that can name a message.
+func TestTypeRendering(t *testing.T) {
+	ix := indexed(t)
+
+	cases := []struct {
+		field        Field
+		written, fqn string
+	}{
+		{Field{Type: "string"}, "string", ""},
+		{Field{Type: "Money"}, "Money", "shop.v1.Money"},
+		{Field{Type: "shop.v1.Money"}, "Money", "shop.v1.Money"},
+		{Field{Type: "LineItem", Label: "repeated"}, "[]LineItem", "shop.v1.LineItem"},
+		{Field{Type: "map<string, string>"}, "map[string]string", ""},
+		{Field{Type: "map<string, Money>"}, "map[string]Money", "shop.v1.Money"},
+	}
+
+	for _, c := range cases {
+		got := ix.resolveType("shop.v1", &c.field)
+		if got.Written != c.written || got.FQN != c.fqn {
+			t.Errorf("%s: %q -> %q (want %q -> %q)", c.field.Type, got.Written, got.FQN, c.written, c.fqn)
+		}
+	}
+}
+
+// The fragment is committed and compared by gen:check, so anything derived from
+// a map has to come out in the same order every run.
+func TestNamesAreSorted(t *testing.T) {
+	names := indexed(t).Names()
+
+	for i := 1; i < len(names); i++ {
+		if names[i-1] > names[i] {
+			t.Fatalf("names are not sorted: %q before %q", names[i-1], names[i])
+		}
+	}
+	if len(names) == 0 {
+		t.Fatal("nothing was indexed")
+	}
+}

package/plugins/extract-python-kafka/README.md

@@ -0,0 +1,36 @@
+# Python Kafka extractor
+
+`extract-python-kafka` reads Kafka client construction and calls from Python
+syntax. It does not import or execute the project and it does not require a
+Django, Flask, or other framework layout.
+
+Supported clients:
+
+- `confluent_kafka.Producer.produce` and `Consumer.subscribe`;
+- `kafka.KafkaProducer.send` and constructor-based `KafkaConsumer` topics;
+- `aiokafka.AIOKafkaProducer.send` / `send_and_wait` and constructor-based
+  `AIOKafkaConsumer` topics.
+
+Literal topics, module constants, imported constants, environment defaults and
+optional `settings.NAME` references are followed. Local producer factories are
+followed to a supported constructor, so a singleton wrapper does not hide the
+client. Topics that remain dynamic are reported and no channel is guessed.
+
+The output uses generic `message` channels rather than domain events. It keeps
+source-backed broker, security mode, idempotence, retry, compression, client
+id, consumer group, key, headers and serializer facts. Authentication values
+are never emitted. Partitions, replication and retention are broker-side facts
+and remain explicitly unknown unless another catalog source declares them.
+
+```json
+{
+  "plugin": "python-kafka",
+  "in": "checkout",
+  "out": ".portolan/fragments/checkout-kafka",
+  "options": {
+    "context": "shop",
+    "service": "checkout",
+    "settings": "config.settings"
+  }
+}
+```

package/plugins/extract-python-kafka/extract.py

@@ -0,0 +1,211 @@
+"""Python Kafka client calls in, generic message channels and flows out."""
+
+from __future__ import annotations
+
+import json
+import os
+from collections import defaultdict
+from typing import Any, Dict, List
+
+import catalog
+import kafka
+from names import slug, title
+from options import Options
+from protocol import Builder, File, Input
+from source import Project
+
+MESSAGE = "message"
+
+
+def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
+    cwd = cwd or os.getcwd()
+    root = os.path.abspath(os.path.join(cwd, input_.root))
+
+    def rel(path: str) -> str:
+        return os.path.relpath(path, cwd).replace(os.sep, "/")
+
+    source = os.path.normpath(os.path.join(root, opts.source or "."))
+    context = opts.context or os.path.basename(root)
+    service = opts.service or os.path.basename(root)
+    svc_id = context + "." + service
+    project = Project(root, source, rel)
+    for path, message in project.broken:
+        b.warn(path, "cannot be parsed, so nothing in it is read: %s" % message)
+
+    found = kafka.scan(project, opts.settings)
+    if not found.publishes and not found.subscriptions:
+        b.warn(svc_id, "no supported Python Kafka producer or consumer calls under %s" % rel(source))
+    for where, expr in sorted(set(found.dynamic_topics)):
+        b.warn(where, "Kafka topic is dynamic%s, so no channel is invented" % ((": `%s`" % expr) if expr else ""))
+    clients = [item.client for item in found.publishes] + [item.client for item in found.subscriptions]
+    for source_line in sorted({client.source for client in clients if "brokers" not in client.config}):
+        b.warn(source_line or svc_id, "Kafka broker endpoints are resolved at runtime, so none are claimed in the catalog")
+
+    sends = defaultdict(list)
+    receives = defaultdict(list)
+    for item in found.publishes:
+        sends[item.topic].append(item)
+    for item in found.subscriptions:
+        receives[item.topic].append(item)
+    for topic in sorted(set(sends) - set(receives)):
+        b.warn(svc_id, "publishes Kafka topic `%s`, but no consumer for it is proven in this source tree" % topic)
+
+    channels = [channel(topic, sends[topic], receives[topic]) for topic in sorted(set(sends) | set(receives))]
+    flows = []
+    for topic in sorted(sends):
+        for index, item in enumerate(sends[topic], 1):
+            flows.append(publish_flow(context, service, svc_id, item, index))
+    for topic in sorted(receives):
+        received_message = sends[topic][0].message if len({item.message for item in sends[topic]}) == 1 else "message"
+        for index, item in enumerate(receives[topic], 1):
+            flows.append(receive_flow(context, service, svc_id, item, received_message, index))
+    flows.sort(key=lambda flow: flow["slug"])
+
+    fragment: Dict[str, Any] = {
+        "generatedAt": input_.generated_at,
+        "commit": input_.commit,
+        "contexts": [
+            {
+                "id": context,
+                "slug": context,
+                "name": "",
+                "summary": "",
+                "services": [
+                    {
+                        "id": svc_id,
+                        "slug": service,
+                        "name": "",
+                        "repo": "",
+                        "path": "",
+                        "readme": "",
+                        "provides": [],
+                        "consumes": [],
+                        "aggregates": [],
+                        "channels": channels,
+                    }
+                ],
+            }
+        ],
+        "defs": {},
+        "flows": flows,
+        "adrs": [],
+    }
+    b.files.append(File(name=opts.out or "kafka.json", contents=json.dumps(fragment, indent=2, ensure_ascii=False) + "\n"))
+
+
+def channel(topic: str, sends: List[kafka.Publish], receives: List[kafka.Subscription]) -> Dict[str, Any]:
+    messages = []
+    seen = set()
+    for item in sends:
+        key = (item.message, "send")
+        if key not in seen:
+            messages.append(catalog.message(item.message, title(item.message), message_doc(item), "send"))
+            seen.add(key)
+    for item in receives:
+        # Kafka subscriptions dispatch records, and source often does not prove
+        # their schema. Match a single proven producer name; otherwise stay
+        # honest and call the record a message.
+        name = sends[0].message if len({sent.message for sent in sends}) == 1 else "message"
+        key = (name, "receive")
+        if key not in seen:
+            messages.append(catalog.message(name, title(name), consumer_doc(item), "receive"))
+            seen.add(key)
+    clients = []
+    for item in list(sends) + list(receives):
+        note = kafka.config_note(item.client)
+        if note not in clients:
+            clients.append(note)
+    doc = "Kafka message stream. " + ". ".join(clients) + ". Broker-side partitions, replication and retention are not declared by client code."
+    source = (sends[0].line if sends else receives[0].line) if sends or receives else ""
+    return catalog.channel(topic, MESSAGE, "Kafka · " + topic, doc, messages, source)
+
+
+def message_doc(item: kafka.Publish) -> str:
+    bits = ["Published with %s" % item.client.library]
+    if item.message_expression:
+        bits.append("payload `%s`" % item.message_expression)
+    if item.key:
+        bits.append("key `%s`" % item.key)
+    if item.headers:
+        bits.append("headers `%s`" % item.headers)
+    return "; ".join(bits) + "."
+
+
+def consumer_doc(item: kafka.Subscription) -> str:
+    group = item.client.config.get("consumer group")
+    suffix = "; consumer group `%s`" % group if group else ""
+    return "Subscribed with %s%s." % (item.client.library, suffix)
+
+
+def handoff(topic: str, message: str, direction: str) -> Dict[str, str]:
+    return {
+        "kind": "message",
+        "transport": "kafka",
+        "channel": topic,
+        "message": message,
+        "direction": direction,
+    }
+
+
+def publish_flow(context: str, service: str, svc_id: str, item: kafka.Publish, index: int) -> Dict[str, Any]:
+    flow_slug = slug("%s-kafka-publish-%s-%s-%d" % (service, item.topic, item.message, index))
+    broker = "kafka-" + slug(item.topic)
+    return catalog.flow(
+        "flow." + flow_slug,
+        flow_slug,
+        "Publish %s" % title(item.message),
+        "Publishes `%s` to Kafka topic `%s`." % (item.message, item.topic),
+        item.module.rel,
+        context,
+        [
+            catalog.participant(svc_id, "service", context),
+            catalog.participant(broker, "broker", None, "Kafka · " + item.topic),
+        ],
+        [
+            catalog.step(
+                "publish",
+                svc_id,
+                broker,
+                "call",
+                "publish " + item.message,
+                catalog.DECLARED,
+                note=kafka.config_note(item.client),
+                line=item.line,
+                handoff=handoff(item.topic, item.message, "send"),
+            )
+        ],
+        trigger={"kind": "unproven", "label": "source caller not proven", "confidence": "low"},
+        entrypoint=item.entrypoint,
+    )
+
+
+def receive_flow(context: str, service: str, svc_id: str, item: kafka.Subscription, message: str, index: int) -> Dict[str, Any]:
+    flow_slug = slug("%s-kafka-consume-%s-%d" % (service, item.topic, index))
+    broker = "kafka-" + slug(item.topic)
+    return catalog.flow(
+        "flow." + flow_slug,
+        flow_slug,
+        "Consume from %s" % item.topic,
+        "Consumes Kafka messages from `%s`." % item.topic,
+        item.module.rel,
+        context,
+        [
+            catalog.participant(broker, "broker", None, "Kafka · " + item.topic),
+            catalog.participant(svc_id, "service", context),
+        ],
+        [
+            catalog.step(
+                "receive",
+                broker,
+                svc_id,
+                "call",
+                "receive message",
+                catalog.DECLARED,
+                note=consumer_doc(item),
+                line=item.line,
+                handoff=handoff(item.topic, message, "receive"),
+            )
+        ],
+        trigger={"kind": "message", "label": "Kafka · " + item.topic, "confidence": "high"},
+        entrypoint=item.entrypoint,
+    )

package/plugins/extract-python-kafka/extract_test.py

@@ -0,0 +1,68 @@
+"""Framework-neutral Kafka extraction contracts."""
+
+import json
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
+
+from extract import extract  # noqa: E402
+from options import Options  # noqa: E402
+from protocol import Builder, Input  # noqa: E402
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
+
+
+class PythonKafka(unittest.TestCase):
+    def setUp(self):
+        fixture = os.path.join(HERE, "testdata", "service")
+        builder = Builder()
+        extract(
+            Input(root=os.path.relpath(fixture, ROOT), commit="abc1234", generated_at="2026-09-08T00:00:00Z"),
+            Options.of({"context": "shop", "service": "orders", "settings": "config.settings"}),
+            builder,
+            cwd=ROOT,
+        )
+        self.warnings = builder.warnings
+        self.contents = builder.files[0].contents
+        self.fragment = json.loads(self.contents)
+        self.service = self.fragment["contexts"][0]["services"][0]
+
+    def test_three_standard_clients_form_message_channels(self):
+        channels = {item["address"]: item for item in self.service["channels"]}
+        self.assertEqual(sorted(channels), ["audit.records", "orders.created", "payments.accepted"])
+        self.assertTrue(all(item["kind"] == "message" for item in channels.values()))
+        self.assertEqual(
+            [message["direction"] for message in channels["orders.created"]["messages"]],
+            ["send", "receive"],
+        )
+        self.assertIn("consumer group `orders-indexer`", channels["orders.created"]["messages"][1]["doc"])
+
+    def test_client_configuration_is_kept_but_credentials_never_are(self):
+        orders = next(item for item in self.service["channels"] if item["address"] == "orders.created")
+        self.assertIn("brokers: kafka:9092", orders["doc"])
+        self.assertIn("idempotence: enabled", orders["doc"])
+        self.assertIn("security protocol: SASL_SSL", orders["doc"])
+        self.assertNotIn("must-not-leak", self.contents)
+        self.assertNotIn("sasl.password", self.contents)
+        self.assertNotIn("sasl.username", self.contents)
+
+    def test_publish_and_receive_flows_have_kafka_handoffs_and_source(self):
+        steps = [step for flow in self.fragment["flows"] for step in flow["steps"]]
+        handoffs = [step["handoff"] for step in steps]
+        self.assertTrue(any(item == {"kind": "message", "transport": "kafka", "channel": "payments.accepted", "message": "payment", "direction": "send"} for item in handoffs))
+        self.assertTrue(any(item == {"kind": "message", "transport": "kafka", "channel": "payments.accepted", "message": "payment", "direction": "receive"} for item in handoffs))
+        self.assertTrue(any(item["channel"] == "orders.created" and item["direction"] == "receive" for item in handoffs))
+        self.assertTrue(all(step["line"].endswith(tuple(str(n) for n in range(1, 100))) for step in steps))
+
+    def test_a_dynamic_topic_is_reported_and_not_invented(self):
+        self.assertEqual(len(self.warnings), 1)
+        self.assertIn("Kafka topic is dynamic", self.warnings[0].message)
+        self.assertIn("runtime_topic", self.warnings[0].message)
+
+
+if __name__ == "__main__":
+    unittest.main()