@shortlink-org/portolan 0.1.0

package/plugins/fetch-csr/main.go

@@ -0,0 +1,122 @@
+// Package main is portolan-fetch-csr: a Confluent Schema Registry on one side,
+// schema files committed to this repository on the other.
+//
+// It is the second plugin here that opens a socket, and it is split from
+// portolan-extract-csr for the reason portolan-fetch-bsr is split from
+// portolan-extract-proto. Extraction stays a pure function of the tree;
+// fetching is the step that can fail, need a credential, or come back with
+// something different than it did yesterday. Keeping them apart is what lets
+// CI verify an estate whose schemas live in a registry it never talks to.
+//
+// The fetched schemas are the plugin's Response.Files, not a side effect. The
+// host writes them, so they get a manifest entry, are compared by `gen:check`
+// like any other generated file, and are removed when the step stops naming
+// them. Refreshing a pin then produces ONE pull request holding the version
+// bump, the schema diff, the lock diff and the fragment diff - which is the
+// review worth having, and the one a registry UI cannot give you.
+//
+// A registered version is immutable: subject `orders-value` at version 3 is
+// the same bytes today and next year, and re-registering a changed schema
+// makes version 4. That is the same promise a BSR commit makes, and it is the
+// only reason replaying from disk is equivalent to fetching again. So a
+// subject is PINNED to a version here, exactly as a module is pinned to a
+// commit there.
+//
+// The shape of the two steps, fetch and then read:
+//
+//	{
+//	  "plugins": [
+//	    { "name": "csr",         "process": { "command": "go", "args": ["run", "./plugins/fetch-csr"] } },
+//	    { "name": "csr-schemas", "process": { "command": "go", "args": ["run", "./plugins/extract-csr"] } }
+//	  ],
+//	  "extract": [
+//	    {
+//	      "plugin": "csr",
+//	      "in": "examples/shop/oms",
+//	      "out": "examples/shop/oms/vendor/schemas",
+//	      "options": {
+//	        "registry": "https://psrc-00000.eu-central-1.aws.confluent.cloud",
+//	        "cache": "examples/shop/oms/vendor/schemas",
+//	        "subjects": [{ "subject": "shop.oms.order-value", "version": 3 }]
+//	      }
+//	    },
+//	    {
+//	      "plugin": "csr-schemas",
+//	      "in": "examples/shop/oms",
+//	      "out": "examples/shop/oms/portolan",
+//	      "options": {
+//	        "context": "shop",
+//	        "service": "oms",
+//	        "paths": ["vendor/schemas"],
+//	        "out": "schemas.json"
+//	      }
+//	    }
+//	  ]
+//	}
+//
+// The csr step is declared FIRST: extract steps run in list order, so its
+// schemas and locks are on disk by the time the reader looks for them.
+//
+// In CI, set PORTOLAN_OFFLINE=1. The step then replays the committed copies,
+// checks them against their locks and emits an identical file list, so a
+// fork's pull request needs no secret and the registry being down cannot turn
+// the tree red.
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// Options are what the manifest tells the fetcher.
+//
+// No credential, ever. portolan.json is committed; the API key comes from the
+// environment, and only auth.go looks at it. The registry URL is not a
+// credential - it is a fact about the estate, the same kind of fact as a
+// module's registry-global name - so it is written down here.
+type Options struct {
+	// Registry is the base URL of the schema registry, without a trailing
+	// path: "https://psrc-00000.eu-central-1.aws.confluent.cloud", or
+	// "http://localhost:8081" for one running beside the tests.
+	Registry string `json:"registry"`
+
+	Subjects []Subject `json:"subjects"`
+
+	// Cache is where the last run's files were written, repo-relative, and
+	// should be the same path as the step's `out`.
+	//
+	// Stated twice because a plugin is never told where its output goes. That
+	// is the whole shape of the protocol - a plugin returns files and the host
+	// decides what to do with them - and not an oversight to work around.
+	Cache string `json:"cache"`
+}
+
+// Subject is one pinned registration.
+type Subject struct {
+	// Subject is the registry's own name for it: "shop.oms.order-value" under
+	// TopicNameStrategy, "shop.oms.OrderPlaced" under RecordNameStrategy.
+	Subject string `json:"subject"`
+
+	// Version pins it. A registered version is immutable, so a pinned
+	// download is byte-reproducible - which is the only reason replaying from
+	// disk is equivalent to fetching again. Leaving it out resolves `latest`
+	// and is warned about; offline it is refused, because there is nothing to
+	// replay against.
+	Version int `json:"version,omitempty"`
+}
+
+func main() {
+	if err := run(os.Stdin, os.Stdout); err != nil {
+		fmt.Fprintln(os.Stderr, "portolan-fetch-csr:", err)
+		os.Exit(1)
+	}
+}
+
+func run(stdin io.Reader, stdout io.Writer) error {
+	return plugin.Serve(stdin, stdout, descriptor(), func(_ plugin.Request, opts Options) (plugin.Response, error) {
+		return fetch(opts)
+	})
+}

package/plugins/fetch-csr/options.schema.json

@@ -0,0 +1,42 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["registry", "subjects", "cache"],
+  "properties": {
+    "registry": {
+      "type": "string",
+      "description": "Base URL of the schema registry. The credential for it is never written here: it comes from the environment, so this file stays committable.",
+      "examples": [
+        "http://localhost:8081",
+        "https://psrc-00000.eu-central-1.aws.confluent.cloud"
+      ]
+    },
+    "subjects": {
+      "type": "array",
+      "minItems": 1,
+      "description": "The subjects to fetch, each pinned to a version. A subject a fetched schema references is fetched too, pinned by the reference itself, and needs no entry here.",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["subject"],
+        "properties": {
+          "subject": {
+            "type": "string",
+            "description": "The registry's own name for the registration: \"<topic>-value\" under TopicNameStrategy, the record's full name under RecordNameStrategy.",
+            "examples": ["shop.oms.order-value", "shop.oms.OrderPlaced"]
+          },
+          "version": {
+            "type": "integer",
+            "minimum": 1,
+            "description": "The version to fetch. A registered version is immutable, so a pinned fetch is byte-reproducible, which is what lets an offline run replay it from the tree. Left out, `latest` is resolved online and warned about, and refused offline."
+          }
+        }
+      }
+    },
+    "cache": {
+      "type": "string",
+      "description": "Where the last run's files were written, repo-relative: the same path as the step's out. An offline run replays what is there and checks it against the lock beside it.",
+      "examples": ["examples/shop/oms/vendor/schemas"]
+    }
+  }
+}

package/plugins/fetch-csr/registry.go

@@ -0,0 +1,209 @@
+package main
+
+// The registry's REST API, and only the two calls this plugin makes.
+//
+// A schema registry has a large surface - compatibility levels, modes, global
+// configuration, deletion. None of it belongs here. This plugin reads
+// registrations and writes nothing, which is what makes it safe to point at a
+// production registry from a laptop.
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Accept names the registry's own media type first and plain JSON second, so
+// the same client works against Confluent's registry and the several
+// API-compatible ones that only answer application/json.
+const accept = "application/vnd.schemaregistry.v1+json, application/json"
+
+type client struct {
+	base string
+	auth string
+	http *http.Client
+}
+
+func newClient(base, auth string) *client {
+	return &client{
+		base: strings.TrimRight(base, "/"),
+		auth: auth,
+		// A registry that has stopped answering must fail the step rather than
+		// hang a build until somebody notices.
+		http: &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+// registration is one version of one subject, as the registry answers it.
+type registration struct {
+	Subject string `json:"subject"`
+	Version int    `json:"version"`
+
+	// ID is the registry-global schema id - the four bytes a Kafka message
+	// carries in its wire header. It identifies the SCHEMA, not the
+	// registration: two subjects registering identical bytes share one id.
+	ID int `json:"id"`
+
+	// GUID is the newer, wider identifier for the same thing. Registries that
+	// predate it leave it out, so it is never required.
+	GUID string `json:"guid,omitempty"`
+
+	// SchemaType is "AVRO", "PROTOBUF" or "JSON". Absent means AVRO: the
+	// registry only started saying so when it stopped being the only option.
+	SchemaType string `json:"schemaType,omitempty"`
+
+	// Schema is the schema itself, as text. For AVRO and JSON that text is
+	// itself JSON, which is why it arrives escaped inside a JSON field.
+	Schema string `json:"schema"`
+
+	References []Reference `json:"references,omitempty"`
+}
+
+// Reference is one subject a schema depends on, pinned by the referring schema
+// rather than by the manifest. That is what makes a reference safe to follow
+// without asking: it names a version, and a version is immutable.
+type Reference struct {
+	// Name is how the referring schema spells it - an Avro full name, a
+	// protobuf import path, a JSON Schema `$ref` URL.
+	Name    string `json:"name"`
+	Subject string `json:"subject"`
+	Version int    `json:"version"`
+}
+
+// apiError is the registry's own error shape. Read rather than ignored,
+// because "Subject not found" and "Version not found" are the two mistakes a
+// manifest actually makes, and the HTTP status alone says neither.
+type apiError struct {
+	Code    int    `json:"error_code"`
+	Message string `json:"message"`
+}
+
+// version fetches one registration. A zero version asks for `latest`.
+func (c *client) version(subject string, version int) (*registration, error) {
+	which := "latest"
+	if version > 0 {
+		which = strconv.Itoa(version)
+	}
+
+	// Escaped, because a subject under RecordNameStrategy is a full name and
+	// may hold anything the producer's language allows in one.
+	at := c.base + "/subjects/" + url.PathEscape(subject) + "/versions/" + which
+
+	var got registration
+	if err := c.get(at, &got); err != nil {
+		return nil, fmt.Errorf("%s at version %s: %w", subject, which, err)
+	}
+	if strings.TrimSpace(got.Schema) == "" {
+		return nil, fmt.Errorf("%s at version %s: the registry answered with no schema", subject, which)
+	}
+	// A registry that answers `latest` says which version that was; one that
+	// does not has told us nothing we could pin.
+	if got.Version <= 0 {
+		return nil, fmt.Errorf("%s at version %s: the registry did not say which version it answered with", subject, which)
+	}
+
+	return &got, nil
+}
+
+func (c *client) get(at string, into any) error {
+	req, err := http.NewRequest(http.MethodGet, at, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Accept", accept)
+	if c.auth != "" {
+		req.Header.Set("Authorization", c.auth)
+	}
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		var fault apiError
+		if json.Unmarshal(body, &fault) == nil && fault.Message != "" {
+			return fmt.Errorf("the registry answered %s: %s (error_code %d)",
+				resp.Status, fault.Message, fault.Code)
+		}
+
+		return fmt.Errorf("the registry answered %s", resp.Status)
+	}
+
+	if err := json.Unmarshal(body, into); err != nil {
+		return fmt.Errorf("the registry's answer is not the shape this plugin reads: %w", err)
+	}
+
+	return nil
+}
+
+// schemaType normalizes the type. The empty answer is AVRO by the registry's
+// own rule, and normalizing it here means nothing downstream has to know that.
+func schemaType(declared string) string {
+	if strings.TrimSpace(declared) == "" {
+		return "AVRO"
+	}
+
+	return strings.ToUpper(strings.TrimSpace(declared))
+}
+
+// extensionFor is what the vendored file is called, so an editor opens it with
+// the right syntax and a reviewer reads a diff rather than a wall of escapes.
+func extensionFor(kind string) (string, error) {
+	switch schemaType(kind) {
+	case "AVRO":
+		return ".avsc", nil
+	case "PROTOBUF":
+		return ".proto", nil
+	case "JSON":
+		return ".json", nil
+	}
+
+	return "", fmt.Errorf("unknown schema type %q", kind)
+}
+
+// body is the schema as it should be written to the tree.
+//
+// AVRO and JSON arrive as JSON minified onto one line, which makes a version
+// bump a one-line diff nobody can read. Indenting it is done with json.Indent,
+// which reformats the bytes without reordering them, so the file still says
+// what the registry said, in the order it said it. PROTOBUF arrives as source
+// and is left exactly as it is.
+//
+// The digest is taken over the bytes as written, so the reformatting is inside
+// what the lock checks rather than something that has to be redone to verify.
+func body(kind, schema string) string {
+	if schemaType(kind) == "PROTOBUF" {
+		return ensureNewline(schema)
+	}
+
+	var pretty bytes.Buffer
+	if err := json.Indent(&pretty, []byte(schema), "", "  "); err != nil {
+		// Not an error: a registry that answered with something that is not
+		// JSON has told us something, and the honest thing is to keep it so
+		// the reader can see what arrived.
+		return ensureNewline(schema)
+	}
+
+	return ensureNewline(pretty.String())
+}
+
+func ensureNewline(s string) string {
+	if strings.HasSuffix(s, "\n") {
+		return s
+	}
+
+	return s + "\n"
+}

package/plugins/fetch-git/cache.go

@@ -0,0 +1,65 @@
+package main
+
+// Replaying a fetch from what is already on disk.
+//
+// The cache is not a second copy of anything: it IS the tree. The files this
+// plugin fetched were written by the host through the ordinary Response.Files
+// path, so they are committed, reviewable, and removed when the step stops
+// naming them. Reading them back and checking them against the lock is what
+// lets a build that never opens a socket produce byte-identical output.
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// cached is one repository's files, read back from the tree.
+type cached struct {
+	lock  LockRepo
+	files map[string][]byte
+}
+
+// replay reads a repository's committed copy and checks it against its lock.
+//
+// An incomplete or edited cache is reported rather than patched over. The two
+// failures it distinguishes matter: nothing on disk means there is nothing to
+// fall back to, and bytes that no longer match their digest mean somebody
+// edited a vendored copy - the drift docs/adr/org.0001.md wants visible.
+func replay(dir string) (*cached, error) {
+	at := filepath.Join(dir, LockName)
+	raw, err := os.ReadFile(at)
+	if os.IsNotExist(err) {
+		return nil, fmt.Errorf("no %s in %s", LockName, filepath.ToSlash(dir))
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var lock Lock
+	if err := json.Unmarshal(raw, &lock); err != nil {
+		return nil, fmt.Errorf("%s: %w", filepath.ToSlash(at), err)
+	}
+	if len(lock.Repos) != 1 {
+		return nil, fmt.Errorf("%s names %d repositories; expected exactly one", filepath.ToSlash(at), len(lock.Repos))
+	}
+
+	entry := lock.Repos[0]
+	files := make(map[string][]byte, len(entry.Files))
+	for _, want := range entry.Files {
+		content, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(want.Path)))
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("%s is in the lock but not on disk", want.Path)
+		}
+		if err != nil {
+			return nil, err
+		}
+		if got := digestOf(content); got != want.SHA256 {
+			return nil, fmt.Errorf("%s does not match its digest; the vendored copy was edited by hand", want.Path)
+		}
+		files[want.Path] = content
+	}
+
+	return &cached{lock: entry, files: files}, nil
+}

package/plugins/fetch-git/describe.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	_ "embed"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+//go:embed options.schema.json
+var optionsSchema []byte
+
+func descriptor() plugin.Descriptor {
+	return plugin.Descriptor{
+		Name:    "fetch-git",
+		Summary: "Fetches pinned directories of another git repository into the tree, with a lock, so extractors can read a service that lives elsewhere.",
+		Phases:  []string{plugin.PhaseExtract},
+		Options: optionsSchema,
+	}
+}

package/plugins/fetch-git/describe_test.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/shortlink-org/portolan/plugin/schematest"
+)
+
+func TestOptionsSchemaMatchesStruct(t *testing.T) {
+	schematest.Check(t, optionsSchema, Options{})
+}

package/plugins/fetch-git/fetch.go

@@ -0,0 +1,181 @@
+package main
+
+// Deciding whether to fetch, and what to do when the fetch cannot happen.
+//
+// The rules below are the design, not error handling, and they are the rules
+// fetch-bsr lives by. The one that matters most is the third: a failed fetch
+// with no cache is a NON-ZERO EXIT, never a short file list. The host deletes
+// files a step stops naming, and dropping a vendored service because a laptop
+// went offline is worse than a red build.
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// fetch reads nothing out of the tree being described. Its `cache` is
+// repo-relative, like every other path in the manifest, and everything else
+// it needs is in the options.
+func fetch(opts Options) (plugin.Response, error) {
+	b := &plugin.Builder{}
+
+	if len(opts.Repos) == 0 {
+		return plugin.Response{}, fmt.Errorf("no repositories to fetch: name at least one in the manifest")
+	}
+	if opts.Cache == "" {
+		return plugin.Response{}, fmt.Errorf(
+			"no cache directory: set `cache` to the same path as the step's `out`, " +
+				"so an offline run can replay what the last online one wrote")
+	}
+
+	// Sorted, so the file list and the diagnostics come out the same way every
+	// run whatever order the manifest happens to list them in.
+	wanted := append([]Repo(nil), opts.Repos...)
+	sort.Slice(wanted, func(i, j int) bool { return wanted[i].Repo < wanted[j].Repo })
+
+	skip := offline()
+
+	for _, want := range wanted {
+		url, dir, err := splitRepo(want.Repo)
+		if err != nil {
+			return plugin.Response{}, err
+		}
+		at := filepath.Join(filepath.FromSlash(opts.Cache), filepath.FromSlash(dir))
+
+		if skip {
+			if err := emitCached(b, dir, at, want, "offline"); err != nil {
+				return plugin.Response{}, err
+			}
+
+			continue
+		}
+
+		commit, files, err := live(url, want, b)
+		if err != nil {
+			// Rule 2: the tree still holds a good copy, so the output is
+			// unchanged and `--check` stays clean.
+			if cacheErr := emitCached(b, dir, at, want, err.Error()); cacheErr != nil {
+				// Rule 3: nothing to fall back to.
+				return plugin.Response{}, fmt.Errorf(
+					"%s could not be fetched (%w) and there is no usable copy in the tree (%v)",
+					want.Repo, err, cacheErr)
+			}
+
+			continue
+		}
+
+		if err := emitFetched(b, dir, want, commit, files); err != nil {
+			return plugin.Response{}, err
+		}
+	}
+
+	return b.Response(), nil
+}
+
+// live fetches one repository, pinning it first if the manifest did not.
+func live(url string, want Repo, b *plugin.Builder) (string, map[string][]byte, error) {
+	commit := want.Commit
+	if commit == "" {
+		// Rule: pin, or every run is a lottery. Resolving a ref still works,
+		// but it is said out loud, because two runs a day apart would then
+		// produce two different trees from one manifest.
+		resolved, err := resolve(url, want.Ref)
+		if err != nil {
+			return "", nil, err
+		}
+		b.Warn(want.Repo, fmt.Sprintf(
+			"is not pinned; %q resolved to %s. Pin it in portolan.json or every run is a lottery.",
+			firstNonEmpty(want.Ref, "HEAD"), resolved))
+		commit = resolved
+	}
+
+	files, err := download(url, commit, want.Paths)
+	if err != nil {
+		return "", nil, err
+	}
+	if len(files) == 0 {
+		return "", nil, fmt.Errorf("%s at %s holds no files under %s", want.Repo, commit, strings.Join(want.Paths, ", "))
+	}
+
+	return commit, files, nil
+}
+
+// emitFetched writes what came back, plus the lock that makes it checkable.
+func emitFetched(b *plugin.Builder, dir string, want Repo, commit string, files map[string][]byte) error {
+	entry := LockRepo{Repo: want.Repo, Commit: commit, Paths: append([]string(nil), want.Paths...)}
+
+	paths := make([]string, 0, len(files))
+	for at := range files {
+		paths = append(paths, at)
+	}
+	sort.Strings(paths)
+
+	for _, at := range paths {
+		content := files[at]
+		b.File(path.Join(dir, at), string(content))
+		entry.Files = append(entry.Files, LockEntry{Path: at, SHA256: digestOf(content), Size: len(content)})
+	}
+
+	lock, err := Lock{Repos: []LockRepo{entry}}.encode()
+	if err != nil {
+		return err
+	}
+	b.File(path.Join(dir, LockName), lock)
+
+	return emitPin(b, dir, want.Repo, commit)
+}
+
+// emitCached re-emits the committed copy unchanged.
+//
+// Byte-identical to what the last online run wrote, which is the whole point:
+// `gen:check` sees no drift, so CI can verify an estate whose services live in
+// repositories it never clones.
+func emitCached(b *plugin.Builder, dir, at string, want Repo, why string) error {
+	if want.Commit == "" {
+		return fmt.Errorf("%s is not pinned to a commit, so there is nothing to replay", want.Repo)
+	}
+
+	held, err := replay(at)
+	if err != nil {
+		return err
+	}
+	if held.lock.Commit != want.Commit {
+		return fmt.Errorf("%s holds commit %s but the manifest pins %s; fetch it", want.Repo, held.lock.Commit, want.Commit)
+	}
+
+	for _, entry := range held.lock.Files {
+		b.File(path.Join(dir, entry.Path), string(held.files[entry.Path]))
+	}
+
+	lock, err := Lock{Repos: []LockRepo{held.lock}}.encode()
+	if err != nil {
+		return err
+	}
+	b.File(path.Join(dir, LockName), lock)
+
+	// From the LOCK's commit, not the manifest's. They are equal by the check
+	// above, and taking it from the copy is what keeps the fragment describing
+	// what is actually on disk rather than what was asked for.
+	if err := emitPin(b, dir, want.Repo, held.lock.Commit); err != nil {
+		return err
+	}
+
+	b.Warn(want.Repo, "not fetched ("+why+"); the copy committed in this repository is used unchanged")
+
+	return nil
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if v != "" {
+			return v
+		}
+	}
+
+	return ""
+}