@shortlink-org/portolan 0.2.4 → 0.3.0

package/plugins/extract-http-clients/main.go

@@ -0,0 +1,41 @@
+// Package extracthttpclients is portolan-extract-http-clients: outbound HTTP and SOAP calls
+// in a Go repository in, source-backed dependencies and flows out.
+package extracthttpclients
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+type Options struct {
+	Context   string            `json:"context"`
+	Service   string            `json:"service"`
+	Peers     map[string]string `json:"peers,omitempty"`
+	Externals map[string]string `json:"externals,omitempty"`
+	Out       string            `json:"out,omitempty"`
+}
+
+// Serve answers one request on stdin with one response on stdout. The
+// process entrypoint and the wasm dispatcher in plugins/cmd/portolan-go both
+// call it.
+func Serve(stdin io.Reader, stdout io.Writer) error {
+	return run(stdin, stdout)
+}
+
+func run(stdin io.Reader, stdout io.Writer) error {
+	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
+		if req.Input.Root == "" {
+			return plugin.Response{}, fmt.Errorf("no input root: an extractor has nothing to read")
+		}
+		if opts.Context == "" {
+			opts.Context = slug(filepath.Base(req.Input.Root))
+		}
+		if opts.Service == "" {
+			opts.Service = slug(filepath.Base(req.Input.Root))
+		}
+		return extract(req.Input, opts)
+	})
+}

package/plugins/extract-python-kafka/extract.py

@@ -62,8 +62,6 @@ def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
     flows.sort(key=lambda flow: flow["slug"])
 
     fragment: Dict[str, Any] = {
-        "generatedAt": input_.generated_at,
-        "commit": input_.commit,
         "contexts": [
             {
                 "id": context,

package/plugins/extract-python-kafka/extract_test.py

@@ -23,7 +23,7 @@ class PythonKafka(unittest.TestCase):
         fixture = os.path.join(HERE, "testdata", "service")
         builder = Builder()
         extract(
-            Input(root=os.path.relpath(fixture, ROOT), commit="abc1234", generated_at="2026-09-08T00:00:00Z"),
+            Input(root=os.path.relpath(fixture, ROOT)),
             Options.of({"context": "shop", "service": "orders", "settings": "config.settings"}),
             builder,
             cwd=ROOT,

package/plugins/extract-ts/extract.test.ts

@@ -8,7 +8,7 @@ const ROOT = "plugins/extract-ts/testdata/cart";
 const options = { context: "shop", service: "cart", store: "pg", peers: { "auth.v1": "auth.auth", "shop.v1": "shop.pricing" } };
 
 function run() {
-  return extract({ root: ROOT, commit: "abc1234", generatedAt: "2026-09-04T00:00:00Z" }, options);
+  return extract({ root: ROOT }, options);
 }
 
 function fragment() {
@@ -200,7 +200,7 @@ describe("the JavaScript twin", () => {
     );
 
   it("reads to the fragment the TypeScript tree reads to", () => {
-    const twin = extract({ root: JS, commit: "abc1234", generatedAt: "2026-09-04T00:00:00Z" }, options);
+    const twin = extract({ root: JS }, options);
     const golden = JSON.parse(readFileSync(`${ROOT}/expected.json`, "utf8"));
     expect(sameShape(JSON.parse(twin.files[0]!.contents))).toEqual(sameShape(golden));
     expect(twin.warnings.map((d) => d.ref)).toEqual(["getBasket"]);

package/plugins/extract-ts/extract.ts

@@ -5,6 +5,9 @@
 import { existsSync, readFileSync, realpathSync } from "node:fs";
 import { basename, join, relative, resolve } from "node:path";
 import type { Catalog, Service } from "../../src/catalog.ts";
+
+/** What an extractor writes: a catalog less the stamps, which are the history's to give (portolan.0010). */
+type Fragment = Omit<Catalog, "generatedAt" | "commit">;
 import { readAggregates, type WarningSink } from "./domain.ts";
 import { operationOf, readUseCases } from "./operations.ts";
 import { readBindings } from "./wiring.ts";
@@ -39,8 +42,6 @@ export interface Options {
 export interface Input {
   root: string;
   output?: string;
-  commit: string;
-  generatedAt: string;
 }
 
 export interface Warning {
@@ -151,9 +152,7 @@ export function extract(input: Input, opts: Options, cwd = process.cwd()): Respo
     for (const e of source.errors) b.warn(rel(source.path), `${rel(e.at)}: ${e.message}; the file is read only up to here`);
   }
 
-  const fragment: Catalog = {
-    generatedAt: input.generatedAt,
-    commit: input.commit,
+  const fragment: Fragment = {
     contexts: [
       {
         id: context,

package/plugins/extract-ts/graphql.test.ts

@@ -13,7 +13,7 @@ const ROOT = "plugins/extract-ts/testdata/storefront";
 const options = { context: "storefront", service: "bff", graphql: "src/schema", peers: { "auth.v1": "auth.auth" } };
 
 function run() {
-  return extract({ root: ROOT, commit: "abc1234", generatedAt: "2026-09-05T00:00:00Z" }, options);
+  return extract({ root: ROOT }, options);
 }
 
 function fragment() {

package/plugins/openapi/ids.go

@@ -0,0 +1,261 @@
+// Package openapi is what two extractors agree on about an OpenAPI document:
+// how its interfaces are named in the catalog, and which operation answers on
+// which route. The server side reads the document to say what a service
+// provides; the client side reads the copy vendored beside a generated client
+// to say what a service calls. Both have to spell `auth.v1.Sessions/login` the
+// same way, or the call would never resolve to the method, and one place is
+// how they do.
+package openapi
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// APIID is the document's title and major version: `auth` 1.0.0 gives
+// `auth.v1`. It prefixes every interface the document declares.
+func APIID(title, version string) string {
+	name := title
+	if name == "" {
+		name = "api"
+	}
+	name = strings.ReplaceAll(strings.ToLower(name), " ", "-")
+
+	if major, _, ok := strings.Cut(version, "."); ok && major != "" {
+		return name + ".v" + major
+	}
+
+	return name
+}
+
+// DocumentAPIID is the id a document says it goes by in the estate, or the one
+// built from its title and version when it says nothing.
+//
+// `x-portolan-api` in `info` is for a copy vendored from outside the estate:
+// Stripe's document is titled "Stripe API" and versioned "2026-08-26.dahlia",
+// and `stripe-api.v2026-08-26` on every arrow would be the document's words
+// where the estate wants its own. The copy is the consumer's translation
+// boundary already, so it is the one place the estate's name may be written,
+// and both sides of a call read it from there rather than from two manifests.
+func DocumentAPIID(declared, title, version string) string {
+	if declared = strings.TrimSpace(declared); declared != "" {
+		return declared
+	}
+
+	return APIID(title, version)
+}
+
+// ExternalID is the id a system outside the estate goes by when nothing but
+// its own document names it: the document's title as a slug, with a trailing
+// "API" dropped because the system is the thing, not its interface. "Gordian
+// Flights & Ancillaries API" is gordian-flights-ancillaries; "Stripe API" is
+// stripe. Both sides of a call - the extractor that reads the vendored copy
+// for what the system answers on, and the one that reads the client beside it
+// for what this service calls - derive it here, so neither needs the manifest
+// to say it.
+func ExternalID(title string) string {
+	var b strings.Builder
+	dash := false
+	for _, r := range strings.ToLower(strings.TrimSpace(title)) {
+		switch {
+		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
+			b.WriteRune(r)
+			dash = false
+		case !dash && b.Len() > 0:
+			b.WriteByte('-')
+			dash = true
+		}
+	}
+	id := strings.TrimRight(b.String(), "-")
+	if trimmed := strings.TrimSuffix(id, "-api"); trimmed != "" {
+		id = trimmed
+	}
+
+	return id
+}
+
+// Title is the human form of a tag: users becomes Users, price_list becomes
+// PriceList, because it sits in an id beside a proto-shaped service name.
+func Title(name string) string {
+	var b strings.Builder
+	for _, word := range strings.FieldsFunc(name, func(r rune) bool { return r == '_' || r == '-' || r == ' ' }) {
+		runes := []rune(word)
+		if runes[0] >= 'a' && runes[0] <= 'z' {
+			runes[0] = runes[0] - 'a' + 'A'
+		}
+		b.WriteString(string(runes))
+	}
+
+	return b.String()
+}
+
+// InterfaceID is the contract declared by the OpenAPI document. Tags organise
+// operations inside that contract; they do not create separate interfaces.
+func InterfaceID(api, _ string) string { return api }
+
+// Verbs, in the order the paths object lists them by convention.
+var Verbs = []string{"get", "put", "post", "delete", "options", "head", "patch", "trace"}
+
+// Operation is one route of the document and the name it goes by.
+type Operation struct {
+	// ID is the operationId, or `VERB /path` when the document has none.
+	ID string
+	// Tag is the first tag, retained as document metadata. It does not change
+	// the identity of the contract.
+	Tag string
+	// Verb is upper case: POST.
+	Verb string
+	// Path is the template as written: /v1/users/{userId}.
+	Path string
+}
+
+// Interface is the id of the interface the operation belongs to.
+func (o Operation) Interface(api string) string { return InterfaceID(api, o.Tag) }
+
+// CallID is what a call to this operation is known by in the catalog.
+func (o Operation) CallID(api string) string { return o.Interface(api) + "/" + o.ID }
+
+// Spec is the part of a document these extractors read.
+type Spec struct {
+	API        string
+	Operations []Operation
+
+	// Title, Description and DocsURL are what the document says about the
+	// system it describes, for naming that system when the manifest does not.
+	Title       string
+	Description string
+	DocsURL     string
+}
+
+// Read loads a document and lists its operations. The api id may be
+// overridden by whoever calls, since the manifest may name one.
+func Read(path string) (*Spec, error) {
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var root yaml.Node
+	if err := yaml.Unmarshal(contents, &root); err != nil {
+		return nil, fmt.Errorf("%s: %w", path, err)
+	}
+	node := &root
+	if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
+		node = node.Content[0]
+	}
+
+	info := child(node, "info")
+	spec := &Spec{
+		API:         DocumentAPIID(text(child(info, "x-portolan-api")), text(child(info, "title")), text(child(info, "version"))),
+		Title:       text(child(info, "title")),
+		Description: text(child(info, "description")),
+		DocsURL:     text(child(node, "externalDocs", "url")),
+	}
+	for _, p := range entries(child(node, "paths")) {
+		for _, verb := range Verbs {
+			operation := child(p.value, verb)
+			if operation == nil {
+				continue
+			}
+			id := text(child(operation, "operationId"))
+			if id == "" {
+				id = strings.ToUpper(verb) + " " + p.key
+			}
+			tags := list(child(operation, "tags"))
+			tag := ""
+			if len(tags) > 0 {
+				tag = tags[0]
+			}
+			spec.Operations = append(spec.Operations, Operation{ID: id, Tag: tag, Verb: strings.ToUpper(verb), Path: p.key})
+		}
+	}
+
+	return spec, nil
+}
+
+// Find looks up an operation by verb and route. Both sides may spell a path
+// parameter differently - `{userId}` in the document, `%s` in a generated
+// client - so parameters are compared by position, not by name.
+func (s *Spec) Find(verb, path string) (Operation, bool) {
+	want := strings.ToUpper(verb) + " " + shape(path)
+	for _, op := range s.Operations {
+		if op.Verb+" "+shape(op.Path) == want {
+			return op, true
+		}
+	}
+
+	return Operation{}, false
+}
+
+// shape replaces every parameter, however spelled, with one marker.
+func shape(path string) string {
+	var b strings.Builder
+	for _, segment := range strings.Split(strings.TrimSuffix(path, "/"), "/") {
+		if strings.HasPrefix(segment, "{") || segment == "%s" || segment == "%v" || segment == "%d" {
+			b.WriteString("/*")
+
+			continue
+		}
+		b.WriteString("/" + segment)
+	}
+
+	return b.String()
+}
+
+type entry struct {
+	key   string
+	value *yaml.Node
+}
+
+func entries(node *yaml.Node) []entry {
+	if node == nil || node.Kind != yaml.MappingNode {
+		return nil
+	}
+	var out []entry
+	for i := 0; i+1 < len(node.Content); i += 2 {
+		out = append(out, entry{key: node.Content[i].Value, value: node.Content[i+1]})
+	}
+
+	return out
+}
+
+func child(node *yaml.Node, keys ...string) *yaml.Node {
+	for _, key := range keys {
+		if node == nil || node.Kind != yaml.MappingNode {
+			return nil
+		}
+		var next *yaml.Node
+		for i := 0; i+1 < len(node.Content); i += 2 {
+			if node.Content[i].Value == key {
+				next = node.Content[i+1]
+
+				break
+			}
+		}
+		node = next
+	}
+
+	return node
+}
+
+func text(node *yaml.Node) string {
+	if node == nil || node.Kind != yaml.ScalarNode {
+		return ""
+	}
+
+	return strings.TrimSpace(node.Value)
+}
+
+func list(node *yaml.Node) []string {
+	if node == nil || node.Kind != yaml.SequenceNode {
+		return nil
+	}
+	var out []string
+	for _, item := range node.Content {
+		out = append(out, text(item))
+	}
+
+	return out
+}

package/plugins/openapi/ids_test.go

@@ -0,0 +1,98 @@
+package openapi
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestIDs(t *testing.T) {
+	if got := APIID("Auth", "1.2.3"); got != "auth.v1" {
+		t.Errorf("APIID = %q", got)
+	}
+	if got := APIID("", ""); got != "api" {
+		t.Errorf("APIID of nothing = %q", got)
+	}
+	if got := Title("price_list"); got != "PriceList" {
+		t.Errorf("Title = %q", got)
+	}
+	if got := InterfaceID("auth.v1", "sessions"); got != "auth.v1" {
+		t.Errorf("InterfaceID = %q", got)
+	}
+	if got := InterfaceID("auth.v1", ""); got != "auth.v1" {
+		t.Errorf("InterfaceID without a tag = %q", got)
+	}
+	// A vendored copy may say what the estate calls it; a document that says
+	// nothing is named the way it always was.
+	if got := DocumentAPIID(" stripe.v1 ", "Stripe API", "2026-08-26.dahlia"); got != "stripe.v1" {
+		t.Errorf("DocumentAPIID with x-portolan-api = %q", got)
+	}
+	if got := DocumentAPIID("", "Stripe API", "2026-08-26.dahlia"); got != "stripe-api.v2026-08-26" {
+		t.Errorf("DocumentAPIID without = %q", got)
+	}
+}
+
+// A system outside the estate is named by its document's title, and the
+// title's "API" is the interface, not the system.
+func TestExternalID(t *testing.T) {
+	for title, want := range map[string]string{
+		"Gordian Flights & Ancillaries API": "gordian-flights-ancillaries",
+		"Stripe API":                        "stripe",
+		"  Partner   REST API ":             "partner-rest",
+		"API":                               "api",
+		"":                                  "",
+		"Amadeus (Self-Service) v2":         "amadeus-self-service-v2",
+	} {
+		if got := ExternalID(title); got != want {
+			t.Errorf("ExternalID(%q) = %q, want %q", title, got, want)
+		}
+	}
+}
+
+const doc = `openapi: 3.0.3
+info:
+  title: auth
+  version: 1.0.0
+paths:
+  /v1/sessions:
+    post:
+      operationId: login
+      tags: [sessions]
+  /v1/users/{userId}:
+    get:
+      operationId: getUser
+      tags: [users]
+  /v1/health:
+    get: {}
+`
+
+func TestReadAndFind(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "openapi.yaml")
+	if err := os.WriteFile(path, []byte(doc), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	spec, err := Read(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if spec.API != "auth.v1" || len(spec.Operations) != 3 {
+		t.Fatalf("spec = %+v", spec)
+	}
+
+	login, ok := spec.Find("post", "/v1/sessions")
+	if !ok || login.CallID(spec.API) != "auth.v1/login" {
+		t.Errorf("login = %+v %v", login, ok)
+	}
+	// A generated client spells the parameter as a format verb.
+	user, ok := spec.Find("GET", "/v1/users/%s")
+	if !ok || user.CallID(spec.API) != "auth.v1/getUser" {
+		t.Errorf("getUser = %+v %v", user, ok)
+	}
+	health, ok := spec.Find("GET", "/v1/health")
+	if !ok || health.CallID(spec.API) != "auth.v1/GET /v1/health" {
+		t.Errorf("an operation without an id or tag = %+v %v", health, ok)
+	}
+	if _, ok := spec.Find("DELETE", "/v1/sessions"); ok {
+		t.Error("a verb the document does not declare was found")
+	}
+}

package/plugins/pyplugin/protocol.py

@@ -17,12 +17,10 @@ from typing import Any, Dict, List
 
 @dataclass
 class Input:
-    """Where the source is, and the stamp the host put on this run."""
+    """Where the source is."""
 
     root: str = ""
     output: str = ""
-    commit: str = ""
-    generated_at: str = ""
 
     @staticmethod
     def of(raw: Any) -> "Input":
@@ -30,8 +28,6 @@ class Input:
         return Input(
             root=raw.get("root", ""),
             output=raw.get("output", ""),
-            commit=raw.get("commit", ""),
-            generated_at=raw.get("generatedAt", ""),
         )
 
 

package/portolan.json

@@ -167,8 +167,9 @@
     },
     {
       "name": "http-clients",
-      "wasm": {
-        "url": "file://plugins/portolan-go.wasm"
+      "process": {
+        "command": "go",
+        "args": ["run", "./plugins/cmd/portolan-http-clients"]
       }
     },
     {

package/schema/portolan.schema.json

@@ -44,6 +44,14 @@
       },
       "description": "The source projects that make up the estate. A project gives repeated pipeline inputs one name for the generated site's Settings page; estate-wide inputs such as flows need no project."
     },
+    "warningPolicies": {
+      "type": "array",
+      "maxItems": 100,
+      "items": {
+        "$ref": "#/$defs/warningPolicy"
+      },
+      "description": "CEL policies for reviewed extraction limitations. Expressions are type-checked when the manifest is read and suppression always requires a reason."
+    },
     "plugins": {
       "type": "array",
       "items": {
@@ -795,6 +803,32 @@
     }
   },
   "$defs": {
+    "warningPolicy": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "when",
+        "action",
+        "reason"
+      ],
+      "properties": {
+        "when": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 1000,
+          "description": "Boolean CEL expression over plugin, rule, severity, project, phase, ref, message and count."
+        },
+        "action": {
+          "const": "suppress",
+          "description": "Suppress matching diagnostics from the active view while retaining them in the report."
+        },
+        "reason": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Why this limitation is consciously accepted."
+        }
+      }
+    },
     "catalogProfile": {
       "type": "object",
       "additionalProperties": false,

package/scripts/README.md

@@ -1,8 +1,8 @@
 # Host
 
 The side of portolan that runs. `npm run gen` reads the manifest, runs every
-plugin it names, stamps and writes what comes back, merges the sources into
-one estate and validates the union. Nothing here draws anything.
+plugin it names, writes what comes back, merges the sources into one estate
+and validates the union. Nothing here draws anything.
 
 ## What it does
 
@@ -10,21 +10,25 @@ one estate and validates the union. Nothing here draws anything.
   the schema `npm run schema` composes out of what every declared plugin says
   it can be told.
 - Runs three phases in order, extract, verify, generate, and hands each
-  plugin one JSON message: the input root, output directory and stamp for an
+  plugin one JSON message: the input root and output directory for an
   extractor, the merged catalog for a verifier or a generator, the step's
   options unread. The output path lets a fragment point at a companion file
   returned in the same response without letting the plugin write it itself.
-- Stamps a fragment from the last commit that touched its input, never from a
-  clock, so a committed fragment changes exactly when its subject does
-  (portolan.0002).
+- Writes no provenance into a fragment. When a source last changed, and in
+  which commit, is read from the history of the checkout wherever the catalog
+  is read - here, by the LikeC4 generator, by the architecture diff and by
+  the site as Vite builds it - so a fragment is content and nothing else, and
+  regenerating from the same sources changes no byte (portolan.0010).
 - Writes the files a plugin named, refuses a name that climbs out of the
   step's output directory, keeps a listing per step of what it wrote, and
   deletes what stopped being generated (portolan.0001).
 - Merges every source the manifest's globs find and validates the union with
   the code the site runs: `src/merge.ts`, `src/catalog.ts`, `src/enrich.ts`.
 - In `--check` mode writes nothing and fails on the first file that differs
-  from disk. Either way it leaves `.portolan/build-report.json` for the
-  Settings page.
+  from disk, saying where the file first differs and what changed among the
+  step's inputs since its output was last committed - the history is the
+  record of the last generation, so nothing else has to be. Either way it
+  leaves `.portolan/build-report.json` for the Settings page.
 
 ## What it does not do
 
@@ -47,7 +51,9 @@ one estate and validates the union. Nothing here draws anything.
   that report put on the pull request as one comment kept current;
   `forge-release.mjs`, the same report as one section of a release's notes;
   `forge.mjs`, what those two share - which forge the CI is, and how to talk
-  to it; `vendor-lock.mjs`,
-  the commit a fetched copy is of; `site-docs.mjs`, generated documentation
-  put into the built site; `local-api.mjs`, what the dev server answers the
-  site with.
+  to it; `history.mjs`, what the checkout's history says about every file,
+  for a plugin that asks and for the provenance of every source;
+  `provenance.mjs`, that provenance handed to the site as one virtual module;
+  `output-diff.mjs`, where a generated file first differs from what the
+  generator produces; `site-docs.mjs`, generated documentation put into the
+  built site; `local-api.mjs`, what the dev server answers the site with.

package/scripts/catalog-sources.mjs

@@ -14,6 +14,7 @@ import { validateCatalog } from "../src/catalog.ts";
 import { filterCatalogForProfile } from "../src/catalog-profile.ts";
 import { enrichCatalog } from "../src/enrich.ts";
 import { mergeCatalogs } from "../src/merge.ts";
+import { stampsFor } from "./history.mjs";
 import { readManifest } from "./manifest.mjs";
 
 /**
@@ -48,10 +49,15 @@ export async function loadCatalog(manifestPath = "portolan.json", { exclude = []
     );
   }
 
+  // A source is dated by the history, not by itself (portolan.0010): the
+  // commit that last changed the file, and its date, read here and never
+  // written into the file.
+  const stamps = stampsFor(process.cwd(), paths);
   const merged = mergeCatalogs(
     paths.map((path) => ({
       path,
       catalog: JSON.parse(readFileSync(path, "utf8")),
+      stamp: stamps.get(path),
     })),
   );
   if (paths.length === 0) {