@shortlink-org/portolan 0.2.3 → 0.3.0

package/plugins/extract-http-clients/main.go

@@ -0,0 +1,41 @@
+// Package extracthttpclients is portolan-extract-http-clients: outbound HTTP and SOAP calls
+// in a Go repository in, source-backed dependencies and flows out.
+package extracthttpclients
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+type Options struct {
+	Context   string            `json:"context"`
+	Service   string            `json:"service"`
+	Peers     map[string]string `json:"peers,omitempty"`
+	Externals map[string]string `json:"externals,omitempty"`
+	Out       string            `json:"out,omitempty"`
+}
+
+// Serve answers one request on stdin with one response on stdout. The
+// process entrypoint and the wasm dispatcher in plugins/cmd/portolan-go both
+// call it.
+func Serve(stdin io.Reader, stdout io.Writer) error {
+	return run(stdin, stdout)
+}
+
+func run(stdin io.Reader, stdout io.Writer) error {
+	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
+		if req.Input.Root == "" {
+			return plugin.Response{}, fmt.Errorf("no input root: an extractor has nothing to read")
+		}
+		if opts.Context == "" {
+			opts.Context = slug(filepath.Base(req.Input.Root))
+		}
+		if opts.Service == "" {
+			opts.Service = slug(filepath.Base(req.Input.Root))
+		}
+		return extract(req.Input, opts)
+	})
+}

package/plugins/extract-python-kafka/README.md

@@ -22,6 +22,12 @@ id, consumer group, key, headers and serializer facts. Authentication values
 are never emitted. Partitions, replication and retention are broker-side facts
 and remain explicitly unknown unless another catalog source declares them.
 
+MessagePack serializers and deserializers declared through `msgpack` or
+`messagepack` calls are normalized to `encoding: "msgpack"` on the affected
+channel message. This includes constructor callbacks and payloads packed
+directly before a publish; Portolan records the wire format without importing
+or executing the codec.
+
 ```json
 {
   "plugin": "python-kafka",

package/plugins/extract-python-kafka/extract.py

@@ -62,8 +62,6 @@ def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
     flows.sort(key=lambda flow: flow["slug"])
 
     fragment: Dict[str, Any] = {
-        "generatedAt": input_.generated_at,
-        "commit": input_.commit,
         "contexts": [
             {
                 "id": context,
@@ -99,7 +97,7 @@ def channel(topic: str, sends: List[kafka.Publish], receives: List[kafka.Subscri
     for item in sends:
         key = (item.message, "send")
         if key not in seen:
-            messages.append(catalog.message(item.message, title(item.message), message_doc(item), "send"))
+            messages.append(catalog.message(item.message, title(item.message), message_doc(item), "send", item.encoding))
             seen.add(key)
     for item in receives:
         # Kafka subscriptions dispatch records, and source often does not prove
@@ -108,7 +106,7 @@ def channel(topic: str, sends: List[kafka.Publish], receives: List[kafka.Subscri
         name = sends[0].message if len({sent.message for sent in sends}) == 1 else "message"
         key = (name, "receive")
         if key not in seen:
-            messages.append(catalog.message(name, title(name), consumer_doc(item), "receive"))
+            messages.append(catalog.message(name, title(name), consumer_doc(item), "receive", item.client.encoding))
             seen.add(key)
     clients = []
     for item in list(sends) + list(receives):

package/plugins/extract-python-kafka/extract_test.py

@@ -1,5 +1,6 @@
 """Framework-neutral Kafka extraction contracts."""
 
+import ast
 import json
 import os
 import sys
@@ -9,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
 
 from extract import extract  # noqa: E402
+import kafka  # noqa: E402
 from options import Options  # noqa: E402
 from protocol import Builder, Input  # noqa: E402
 
@@ -21,7 +23,7 @@ class PythonKafka(unittest.TestCase):
         fixture = os.path.join(HERE, "testdata", "service")
         builder = Builder()
         extract(
-            Input(root=os.path.relpath(fixture, ROOT), commit="abc1234", generated_at="2026-09-08T00:00:00Z"),
+            Input(root=os.path.relpath(fixture, ROOT)),
             Options.of({"context": "shop", "service": "orders", "settings": "config.settings"}),
             builder,
             cwd=ROOT,
@@ -33,7 +35,7 @@ class PythonKafka(unittest.TestCase):
 
     def test_three_standard_clients_form_message_channels(self):
         channels = {item["address"]: item for item in self.service["channels"]}
-        self.assertEqual(sorted(channels), ["audit.records", "orders.created", "payments.accepted"])
+        self.assertEqual(sorted(channels), ["audit.records", "inventory.snapshots", "orders.created", "payments.accepted"])
         self.assertTrue(all(item["kind"] == "message" for item in channels.values()))
         self.assertEqual(
             [message["direction"] for message in channels["orders.created"]["messages"]],
@@ -50,6 +52,20 @@ class PythonKafka(unittest.TestCase):
         self.assertNotIn("sasl.password", self.contents)
         self.assertNotIn("sasl.username", self.contents)
 
+    def test_messagepack_serializer_and_deserializer_are_machine_readable(self):
+        snapshots = next(item for item in self.service["channels"] if item["address"] == "inventory.snapshots")
+        self.assertEqual([message["encoding"] for message in snapshots["messages"]], ["msgpack", "msgpack"])
+        self.assertIn("value serializer: lambda value: msgpack.packb(value)", snapshots["doc"])
+        self.assertIn("value deserializer: msgpack.unpackb", snapshots["doc"])
+
+    def test_direct_messagepack_call_keeps_the_payload_name(self):
+        call = ast.parse("msgpack.packb(snapshot)", mode="eval").body
+        self.assertEqual(kafka.payload_name(call), "snapshot")
+        self.assertEqual(kafka.payload_encoding(call), "msgpack")
+        packer = ast.parse("msgpack.Packer()", mode="eval").body
+        method = ast.parse("packer.pack(snapshot)", mode="eval").body
+        self.assertEqual(kafka.payload_encoding(method, {"packer": ("expr", packer)}), "msgpack")
+
     def test_publish_and_receive_flows_have_kafka_handoffs_and_source(self):
         steps = [step for flow in self.fragment["flows"] for step in flow["steps"]]
         handoffs = [step["handoff"] for step in steps]

package/plugins/extract-ts/extract.test.ts

@@ -8,7 +8,7 @@ const ROOT = "plugins/extract-ts/testdata/cart";
 const options = { context: "shop", service: "cart", store: "pg", peers: { "auth.v1": "auth.auth", "shop.v1": "shop.pricing" } };
 
 function run() {
-  return extract({ root: ROOT, commit: "abc1234", generatedAt: "2026-09-04T00:00:00Z" }, options);
+  return extract({ root: ROOT }, options);
 }
 
 function fragment() {
@@ -200,7 +200,7 @@ describe("the JavaScript twin", () => {
     );
 
   it("reads to the fragment the TypeScript tree reads to", () => {
-    const twin = extract({ root: JS, commit: "abc1234", generatedAt: "2026-09-04T00:00:00Z" }, options);
+    const twin = extract({ root: JS }, options);
     const golden = JSON.parse(readFileSync(`${ROOT}/expected.json`, "utf8"));
     expect(sameShape(JSON.parse(twin.files[0]!.contents))).toEqual(sameShape(golden));
     expect(twin.warnings.map((d) => d.ref)).toEqual(["getBasket"]);

package/plugins/extract-ts/extract.ts

@@ -5,6 +5,9 @@
 import { existsSync, readFileSync, realpathSync } from "node:fs";
 import { basename, join, relative, resolve } from "node:path";
 import type { Catalog, Service } from "../../src/catalog.ts";
+
+/** What an extractor writes: a catalog less the stamps, which are the history's to give (portolan.0010). */
+type Fragment = Omit<Catalog, "generatedAt" | "commit">;
 import { readAggregates, type WarningSink } from "./domain.ts";
 import { operationOf, readUseCases } from "./operations.ts";
 import { readBindings } from "./wiring.ts";
@@ -39,8 +42,6 @@ export interface Options {
 export interface Input {
   root: string;
   output?: string;
-  commit: string;
-  generatedAt: string;
 }
 
 export interface Warning {
@@ -151,9 +152,7 @@ export function extract(input: Input, opts: Options, cwd = process.cwd()): Respo
     for (const e of source.errors) b.warn(rel(source.path), `${rel(e.at)}: ${e.message}; the file is read only up to here`);
   }
 
-  const fragment: Catalog = {
-    generatedAt: input.generatedAt,
-    commit: input.commit,
+  const fragment: Fragment = {
     contexts: [
       {
         id: context,

package/plugins/extract-ts/graphql.test.ts

@@ -13,7 +13,7 @@ const ROOT = "plugins/extract-ts/testdata/storefront";
 const options = { context: "storefront", service: "bff", graphql: "src/schema", peers: { "auth.v1": "auth.auth" } };
 
 function run() {
-  return extract({ root: ROOT, commit: "abc1234", generatedAt: "2026-09-05T00:00:00Z" }, options);
+  return extract({ root: ROOT }, options);
 }
 
 function fragment() {

package/plugins/openapi/ids.go

@@ -0,0 +1,261 @@
+// Package openapi is what two extractors agree on about an OpenAPI document:
+// how its interfaces are named in the catalog, and which operation answers on
+// which route. The server side reads the document to say what a service
+// provides; the client side reads the copy vendored beside a generated client
+// to say what a service calls. Both have to spell `auth.v1.Sessions/login` the
+// same way, or the call would never resolve to the method, and one place is
+// how they do.
+package openapi
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// APIID is the document's title and major version: `auth` 1.0.0 gives
+// `auth.v1`. It prefixes every interface the document declares.
+func APIID(title, version string) string {
+	name := title
+	if name == "" {
+		name = "api"
+	}
+	name = strings.ReplaceAll(strings.ToLower(name), " ", "-")
+
+	if major, _, ok := strings.Cut(version, "."); ok && major != "" {
+		return name + ".v" + major
+	}
+
+	return name
+}
+
+// DocumentAPIID is the id a document says it goes by in the estate, or the one
+// built from its title and version when it says nothing.
+//
+// `x-portolan-api` in `info` is for a copy vendored from outside the estate:
+// Stripe's document is titled "Stripe API" and versioned "2026-08-26.dahlia",
+// and `stripe-api.v2026-08-26` on every arrow would be the document's words
+// where the estate wants its own. The copy is the consumer's translation
+// boundary already, so it is the one place the estate's name may be written,
+// and both sides of a call read it from there rather than from two manifests.
+func DocumentAPIID(declared, title, version string) string {
+	if declared = strings.TrimSpace(declared); declared != "" {
+		return declared
+	}
+
+	return APIID(title, version)
+}
+
+// ExternalID is the id a system outside the estate goes by when nothing but
+// its own document names it: the document's title as a slug, with a trailing
+// "API" dropped because the system is the thing, not its interface. "Gordian
+// Flights & Ancillaries API" is gordian-flights-ancillaries; "Stripe API" is
+// stripe. Both sides of a call - the extractor that reads the vendored copy
+// for what the system answers on, and the one that reads the client beside it
+// for what this service calls - derive it here, so neither needs the manifest
+// to say it.
+func ExternalID(title string) string {
+	var b strings.Builder
+	dash := false
+	for _, r := range strings.ToLower(strings.TrimSpace(title)) {
+		switch {
+		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
+			b.WriteRune(r)
+			dash = false
+		case !dash && b.Len() > 0:
+			b.WriteByte('-')
+			dash = true
+		}
+	}
+	id := strings.TrimRight(b.String(), "-")
+	if trimmed := strings.TrimSuffix(id, "-api"); trimmed != "" {
+		id = trimmed
+	}
+
+	return id
+}
+
+// Title is the human form of a tag: users becomes Users, price_list becomes
+// PriceList, because it sits in an id beside a proto-shaped service name.
+func Title(name string) string {
+	var b strings.Builder
+	for _, word := range strings.FieldsFunc(name, func(r rune) bool { return r == '_' || r == '-' || r == ' ' }) {
+		runes := []rune(word)
+		if runes[0] >= 'a' && runes[0] <= 'z' {
+			runes[0] = runes[0] - 'a' + 'A'
+		}
+		b.WriteString(string(runes))
+	}
+
+	return b.String()
+}
+
+// InterfaceID is the contract declared by the OpenAPI document. Tags organise
+// operations inside that contract; they do not create separate interfaces.
+func InterfaceID(api, _ string) string { return api }
+
+// Verbs, in the order the paths object lists them by convention.
+var Verbs = []string{"get", "put", "post", "delete", "options", "head", "patch", "trace"}
+
+// Operation is one route of the document and the name it goes by.
+type Operation struct {
+	// ID is the operationId, or `VERB /path` when the document has none.
+	ID string
+	// Tag is the first tag, retained as document metadata. It does not change
+	// the identity of the contract.
+	Tag string
+	// Verb is upper case: POST.
+	Verb string
+	// Path is the template as written: /v1/users/{userId}.
+	Path string
+}
+
+// Interface is the id of the interface the operation belongs to.
+func (o Operation) Interface(api string) string { return InterfaceID(api, o.Tag) }
+
+// CallID is what a call to this operation is known by in the catalog.
+func (o Operation) CallID(api string) string { return o.Interface(api) + "/" + o.ID }
+
+// Spec is the part of a document these extractors read.
+type Spec struct {
+	API        string
+	Operations []Operation
+
+	// Title, Description and DocsURL are what the document says about the
+	// system it describes, for naming that system when the manifest does not.
+	Title       string
+	Description string
+	DocsURL     string
+}
+
+// Read loads a document and lists its operations. The api id may be
+// overridden by whoever calls, since the manifest may name one.
+func Read(path string) (*Spec, error) {
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var root yaml.Node
+	if err := yaml.Unmarshal(contents, &root); err != nil {
+		return nil, fmt.Errorf("%s: %w", path, err)
+	}
+	node := &root
+	if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
+		node = node.Content[0]
+	}
+
+	info := child(node, "info")
+	spec := &Spec{
+		API:         DocumentAPIID(text(child(info, "x-portolan-api")), text(child(info, "title")), text(child(info, "version"))),
+		Title:       text(child(info, "title")),
+		Description: text(child(info, "description")),
+		DocsURL:     text(child(node, "externalDocs", "url")),
+	}
+	for _, p := range entries(child(node, "paths")) {
+		for _, verb := range Verbs {
+			operation := child(p.value, verb)
+			if operation == nil {
+				continue
+			}
+			id := text(child(operation, "operationId"))
+			if id == "" {
+				id = strings.ToUpper(verb) + " " + p.key
+			}
+			tags := list(child(operation, "tags"))
+			tag := ""
+			if len(tags) > 0 {
+				tag = tags[0]
+			}
+			spec.Operations = append(spec.Operations, Operation{ID: id, Tag: tag, Verb: strings.ToUpper(verb), Path: p.key})
+		}
+	}
+
+	return spec, nil
+}
+
+// Find looks up an operation by verb and route. Both sides may spell a path
+// parameter differently - `{userId}` in the document, `%s` in a generated
+// client - so parameters are compared by position, not by name.
+func (s *Spec) Find(verb, path string) (Operation, bool) {
+	want := strings.ToUpper(verb) + " " + shape(path)
+	for _, op := range s.Operations {
+		if op.Verb+" "+shape(op.Path) == want {
+			return op, true
+		}
+	}
+
+	return Operation{}, false
+}
+
+// shape replaces every parameter, however spelled, with one marker.
+func shape(path string) string {
+	var b strings.Builder
+	for _, segment := range strings.Split(strings.TrimSuffix(path, "/"), "/") {
+		if strings.HasPrefix(segment, "{") || segment == "%s" || segment == "%v" || segment == "%d" {
+			b.WriteString("/*")
+
+			continue
+		}
+		b.WriteString("/" + segment)
+	}
+
+	return b.String()
+}
+
+type entry struct {
+	key   string
+	value *yaml.Node
+}
+
+func entries(node *yaml.Node) []entry {
+	if node == nil || node.Kind != yaml.MappingNode {
+		return nil
+	}
+	var out []entry
+	for i := 0; i+1 < len(node.Content); i += 2 {
+		out = append(out, entry{key: node.Content[i].Value, value: node.Content[i+1]})
+	}
+
+	return out
+}
+
+func child(node *yaml.Node, keys ...string) *yaml.Node {
+	for _, key := range keys {
+		if node == nil || node.Kind != yaml.MappingNode {
+			return nil
+		}
+		var next *yaml.Node
+		for i := 0; i+1 < len(node.Content); i += 2 {
+			if node.Content[i].Value == key {
+				next = node.Content[i+1]
+
+				break
+			}
+		}
+		node = next
+	}
+
+	return node
+}
+
+func text(node *yaml.Node) string {
+	if node == nil || node.Kind != yaml.ScalarNode {
+		return ""
+	}
+
+	return strings.TrimSpace(node.Value)
+}
+
+func list(node *yaml.Node) []string {
+	if node == nil || node.Kind != yaml.SequenceNode {
+		return nil
+	}
+	var out []string
+	for _, item := range node.Content {
+		out = append(out, text(item))
+	}
+
+	return out
+}

package/plugins/openapi/ids_test.go

@@ -0,0 +1,98 @@
+package openapi
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestIDs(t *testing.T) {
+	if got := APIID("Auth", "1.2.3"); got != "auth.v1" {
+		t.Errorf("APIID = %q", got)
+	}
+	if got := APIID("", ""); got != "api" {
+		t.Errorf("APIID of nothing = %q", got)
+	}
+	if got := Title("price_list"); got != "PriceList" {
+		t.Errorf("Title = %q", got)
+	}
+	if got := InterfaceID("auth.v1", "sessions"); got != "auth.v1" {
+		t.Errorf("InterfaceID = %q", got)
+	}
+	if got := InterfaceID("auth.v1", ""); got != "auth.v1" {
+		t.Errorf("InterfaceID without a tag = %q", got)
+	}
+	// A vendored copy may say what the estate calls it; a document that says
+	// nothing is named the way it always was.
+	if got := DocumentAPIID(" stripe.v1 ", "Stripe API", "2026-08-26.dahlia"); got != "stripe.v1" {
+		t.Errorf("DocumentAPIID with x-portolan-api = %q", got)
+	}
+	if got := DocumentAPIID("", "Stripe API", "2026-08-26.dahlia"); got != "stripe-api.v2026-08-26" {
+		t.Errorf("DocumentAPIID without = %q", got)
+	}
+}
+
+// A system outside the estate is named by its document's title, and the
+// title's "API" is the interface, not the system.
+func TestExternalID(t *testing.T) {
+	for title, want := range map[string]string{
+		"Gordian Flights & Ancillaries API": "gordian-flights-ancillaries",
+		"Stripe API":                        "stripe",
+		"  Partner   REST API ":             "partner-rest",
+		"API":                               "api",
+		"":                                  "",
+		"Amadeus (Self-Service) v2":         "amadeus-self-service-v2",
+	} {
+		if got := ExternalID(title); got != want {
+			t.Errorf("ExternalID(%q) = %q, want %q", title, got, want)
+		}
+	}
+}
+
+const doc = `openapi: 3.0.3
+info:
+  title: auth
+  version: 1.0.0
+paths:
+  /v1/sessions:
+    post:
+      operationId: login
+      tags: [sessions]
+  /v1/users/{userId}:
+    get:
+      operationId: getUser
+      tags: [users]
+  /v1/health:
+    get: {}
+`
+
+func TestReadAndFind(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "openapi.yaml")
+	if err := os.WriteFile(path, []byte(doc), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	spec, err := Read(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if spec.API != "auth.v1" || len(spec.Operations) != 3 {
+		t.Fatalf("spec = %+v", spec)
+	}
+
+	login, ok := spec.Find("post", "/v1/sessions")
+	if !ok || login.CallID(spec.API) != "auth.v1/login" {
+		t.Errorf("login = %+v %v", login, ok)
+	}
+	// A generated client spells the parameter as a format verb.
+	user, ok := spec.Find("GET", "/v1/users/%s")
+	if !ok || user.CallID(spec.API) != "auth.v1/getUser" {
+		t.Errorf("getUser = %+v %v", user, ok)
+	}
+	health, ok := spec.Find("GET", "/v1/health")
+	if !ok || health.CallID(spec.API) != "auth.v1/GET /v1/health" {
+		t.Errorf("an operation without an id or tag = %+v %v", health, ok)
+	}
+	if _, ok := spec.Find("DELETE", "/v1/sessions"); ok {
+		t.Error("a verb the document does not declare was found")
+	}
+}

package/plugins/pyplugin/catalog.py

@@ -81,13 +81,24 @@ def channel(address: str, kind: str, title: str, doc: str, messages: List[Dict[s
     return out
 
 
-def message(name: str, title: str, doc: str, direction: str) -> Dict[str, Any]:
+def message(
+    name: str,
+    title: str,
+    doc: str,
+    direction: str,
+    encoding: str = "",
+    content_type: str = "",
+) -> Dict[str, Any]:
     out: Dict[str, Any] = {"name": name}
     if title:
         out["title"] = title
     if doc:
         out["doc"] = doc
     out["direction"] = direction
+    if encoding:
+        out["encoding"] = encoding
+    if content_type:
+        out["contentType"] = content_type
     return out