@shortlink-org/portolan 0.1.0

package/plugins/extract-csr/jsonschema.go

@@ -0,0 +1,343 @@
+package main
+
+// JSON Schema, read for the same two things Avro is read for.
+//
+// The one thing that has to be careful here is ORDER. An Avro record lists its
+// fields in an array, so reading them into Go keeps the order the author wrote
+// them in. A JSON Schema lists them as the keys of an object, and decoding an
+// object into a map loses that order - which would put the fields on the page
+// in a different arrangement every run and rewrite the fragment each time. So
+// the properties are walked with a token decoder instead.
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/shortlink-org/portolan/catalog"
+)
+
+// jsonSchema reads one .json into the shapes it declares, the same way avro
+// does: the name of the shape at the top, and every named shape inside it.
+//
+// fallback names the top-level shape when the document does not name itself,
+// which most of them do not: a JSON Schema's `title` is optional and a
+// registered one usually leans on the subject to say what it is.
+func jsonSchema(raw []byte, fallback string) (string, map[string]catalog.TypeDef, error) {
+	var node jsonNode
+	if err := json.Unmarshal(raw, &node); err != nil {
+		return "", nil, fmt.Errorf("not a JSON Schema: %w", err)
+	}
+
+	name := firstNonEmpty(node.Title, fallback)
+	r := &jsonReader{defs: map[string]catalog.TypeDef{}, prefix: name}
+
+	// The named shapes a document keeps to one side are read first, so a `$ref`
+	// pointing at one resolves to a def rather than to a bare label.
+	//
+	// They are filed under the document's own name. `definitions` is local to
+	// the document by definition, and `defs` in the catalog is not: two
+	// services that both keep a "Carrier" to one side, meaning different
+	// things, would otherwise collide in the merge and be reported as a
+	// disagreement neither of them is having.
+	for _, held := range []map[string]json.RawMessage{node.Definitions, node.Defs} {
+		for _, key := range keysInOrder(held) {
+			var nested jsonNode
+			if err := json.Unmarshal(held[key], &nested); err != nil {
+				continue
+			}
+			r.shape(nested, r.qualify(key))
+		}
+	}
+
+	r.shape(node, name)
+
+	return name, r.defs, nil
+}
+
+// jsonNode is the part of a JSON Schema this reads. Everything it leaves out -
+// every keyword that constrains a value rather than naming or shaping it - is
+// the registry's business, not the catalog's.
+type jsonNode struct {
+	Title       string          `json:"title"`
+	Description string          `json:"description"`
+	Type        json.RawMessage `json:"type"`
+	Format      string          `json:"format"`
+	Ref         string          `json:"$ref"`
+	Items       *jsonNode       `json:"items"`
+	Required    []string        `json:"required"`
+	OneOf       []jsonNode      `json:"oneOf"`
+	AnyOf       []jsonNode      `json:"anyOf"`
+
+	Properties  json.RawMessage            `json:"properties"`
+	Definitions map[string]json.RawMessage `json:"definitions"`
+	Defs        map[string]json.RawMessage `json:"$defs"`
+}
+
+type jsonReader struct {
+	defs map[string]catalog.TypeDef
+	open map[string]bool
+
+	// prefix is the document's own shape name, which is what makes a
+	// document-local definition globally addressable.
+	prefix string
+}
+
+// qualify files a document-local name under the document.
+func (r *jsonReader) qualify(name string) string {
+	if r.prefix == "" || name == "" {
+		return name
+	}
+
+	return r.prefix + "." + name
+}
+
+// declared says whether a shape is one this reader wrote down, including one
+// still being read further up the stack.
+func (r *jsonReader) declared(name string) bool {
+	if r.open[name] {
+		return true
+	}
+	_, written := r.defs[name]
+
+	return written
+}
+
+// shape writes down one object's fields, under the name given, and returns
+// whether it wrote anything. A schema with no properties is not a shape.
+func (r *jsonReader) shape(node jsonNode, name string) bool {
+	if name == "" || len(node.Properties) == 0 {
+		return false
+	}
+	if r.open[name] {
+		return true
+	}
+	if _, written := r.defs[name]; written {
+		return true
+	}
+
+	if r.open == nil {
+		r.open = map[string]bool{}
+	}
+	r.open[name] = true
+	defer delete(r.open, name)
+
+	required := map[string]bool{}
+	for _, key := range node.Required {
+		required[key] = true
+	}
+
+	// Written before the fields are read, so a schema that names itself inside
+	// itself finds an entry rather than recursing forever.
+	r.defs[name] = catalog.TypeDef{Fields: []catalog.Field{}}
+
+	fields := []catalog.Field{}
+	for _, key := range propertiesInOrder(node.Properties) {
+		var property jsonNode
+		if err := json.Unmarshal(key.value, &property); err != nil {
+			continue
+		}
+
+		label, ref := r.typeOf(property, name+"."+key.name)
+		if !required[key.name] && !strings.HasSuffix(label, "?") {
+			label += "?"
+		}
+
+		fields = append(fields, catalog.Field{
+			Name: key.name,
+			Type: label,
+			Doc:  strings.TrimSpace(property.Description),
+			Ref:  ref,
+		})
+	}
+
+	r.defs[name] = catalog.TypeDef{Fields: fields}
+
+	return true
+}
+
+// typeOf renders one property, and returns the defs key when it is an object
+// this reader wrote down.
+//
+// path is what an anonymous nested object is called: a schema that inlines an
+// address under `shipping` has named it, just not with a `title`, and
+// "OrderPlaced.shipping" is the name it gave it.
+func (r *jsonReader) typeOf(node jsonNode, path string) (label, ref string) {
+	if node.Ref != "" {
+		// The label stays the name the document uses - "Carrier" is what a
+		// reader of the schema sees - while the ref carries the qualified key
+		// the catalog files it under. The two are allowed to differ, and here
+		// they have to.
+		name := refName(node.Ref)
+		if key := r.qualify(name); r.declared(key) {
+			return name, key
+		}
+
+		return name, ""
+	}
+
+	if members := append(append([]jsonNode{}, node.OneOf...), node.AnyOf...); len(members) > 0 {
+		return r.choice(members, path), ""
+	}
+
+	kinds, nullable := typeNames(node.Type)
+
+	if node.Items != nil {
+		items, _ := r.typeOf(*node.Items, path)
+
+		return suffix(items+"[]", nullable), ""
+	}
+
+	if len(node.Properties) > 0 {
+		name := firstNonEmpty(node.Title, path)
+		if r.shape(node, name) {
+			return suffix(name, nullable), name
+		}
+	}
+
+	// A format says what the string MEANS, and it is what belongs on a page:
+	// "date-time" says more than the "string" underneath it.
+	if node.Format != "" {
+		return suffix(node.Format, nullable), ""
+	}
+	if len(kinds) == 0 {
+		return suffix(firstNonEmpty(node.Title, "any"), nullable), ""
+	}
+
+	return suffix(strings.Join(kinds, "|"), nullable), ""
+}
+
+// choice renders oneOf/anyOf. A member that is only `{"type":"null"}` is the
+// document saying the value is optional, which is a "?" rather than a member.
+func (r *jsonReader) choice(members []jsonNode, path string) string {
+	var (
+		labels   []string
+		nullable bool
+	)
+
+	for _, member := range members {
+		kinds, memberNull := typeNames(member.Type)
+		if memberNull || (len(kinds) == 1 && kinds[0] == "null") {
+			nullable = true
+			if len(kinds) <= 1 && member.Ref == "" && len(member.Properties) == 0 {
+				continue
+			}
+		}
+
+		label, _ := r.typeOf(member, path)
+		labels = append(labels, strings.TrimSuffix(label, "?"))
+	}
+
+	if len(labels) == 0 {
+		return "null"
+	}
+
+	return suffix(strings.Join(labels, "|"), nullable)
+}
+
+// typeNames reads `type`, which is either a string or an array of them, and
+// pulls "null" out of the array into the nullability it actually means.
+func typeNames(raw json.RawMessage) (kinds []string, nullable bool) {
+	if len(raw) == 0 {
+		return nil, false
+	}
+
+	var one string
+	if json.Unmarshal(raw, &one) == nil {
+		if one == "null" {
+			return nil, true
+		}
+
+		return []string{one}, false
+	}
+
+	var many []string
+	if json.Unmarshal(raw, &many) != nil {
+		return nil, false
+	}
+
+	for _, kind := range many {
+		if kind == "null" {
+			nullable = true
+
+			continue
+		}
+		kinds = append(kinds, kind)
+	}
+
+	return kinds, nullable
+}
+
+func suffix(label string, nullable bool) string {
+	if nullable && !strings.HasSuffix(label, "?") {
+		return label + "?"
+	}
+
+	return label
+}
+
+// refName is the last segment of a `$ref`, which is the name the definition is
+// filed under. A `$ref` reaching outside the document names something this
+// extractor has never seen, and the last segment is still the best label for
+// it.
+func refName(ref string) string {
+	at := strings.LastIndex(ref, "/")
+	if at < 0 {
+		return ref
+	}
+
+	return ref[at+1:]
+}
+
+type property struct {
+	name  string
+	value json.RawMessage
+}
+
+// propertiesInOrder walks an object with a token decoder, which is the only
+// way to keep the order its keys were written in.
+func propertiesInOrder(raw json.RawMessage) []property {
+	if len(raw) == 0 {
+		return nil
+	}
+
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	if _, err := decoder.Token(); err != nil { // the opening brace
+		return nil
+	}
+
+	var out []property
+	for decoder.More() {
+		key, err := decoder.Token()
+		if err != nil {
+			return out
+		}
+		name, isString := key.(string)
+		if !isString {
+			return out
+		}
+
+		var value json.RawMessage
+		if err := decoder.Decode(&value); err != nil {
+			return out
+		}
+		out = append(out, property{name: name, value: value})
+	}
+
+	return out
+}
+
+// keysInOrder is the same problem for `definitions`, which is decoded into a
+// map because nothing about it is ordered on the page - only sorted, so the
+// fragment comes out the same way every run.
+func keysInOrder(held map[string]json.RawMessage) []string {
+	names := make([]string, 0, len(held))
+	for name := range held {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	return names
+}

package/plugins/extract-csr/lock.go

@@ -0,0 +1,25 @@
+package main
+
+// The lock, as this side reads it.
+//
+// A narrower copy of what portolan-fetch-csr writes, on purpose: this plugin
+// needs the subject, the version and which file holds the schema, and reading
+// a shape it does not use would be a second place to keep the writer's
+// decisions in step. The two are held together by the fields' names, which is
+// the same contract every fragment in this repository is held to.
+type Lock struct {
+	Registry string        `json:"registry"`
+	Subjects []LockSubject `json:"subjects"`
+}
+
+type LockSubject struct {
+	Subject    string      `json:"subject"`
+	Version    int         `json:"version"`
+	ID         int         `json:"id"`
+	SchemaType string      `json:"schemaType"`
+	Files      []LockEntry `json:"files"`
+}
+
+type LockEntry struct {
+	Path string `json:"path"`
+}

package/plugins/extract-csr/main.go

@@ -0,0 +1,84 @@
+// Package main is portolan-extract-csr: schemas vendored out of a Confluent
+// Schema Registry in, a catalog fragment out.
+//
+// It reads what portolan-fetch-csr committed - a directory per subject, each
+// holding one schema and the `csr.lock.json` that says which registration it
+// is - and answers with the two facts a registry knows that nothing else in
+// the estate does: which topics carry which messages, and what those messages
+// are shaped like, field by field.
+//
+// It opens nothing. That is the point of the split: portolan-fetch-csr owns
+// the socket and the credential, this owns the reading, and CI runs this over
+// a tree it can verify without a registry existing at all.
+//
+// What it does NOT do is invent events. An Event in the catalog belongs to an
+// aggregate, and a registry has no idea which aggregate raises what - it holds
+// schemas, not domains. So the shapes land in `defs`, where a shared shape
+// belongs, and the topics land in the service's channels, the same place an
+// AsyncAPI document's do. The domain extractor says an aggregate raises
+// OrderPlaced and calls it `shop.oms.OrderPlaced` on the wire; this says a
+// schema by that name is registered against topic `shop.oms.order` and has
+// these fields. Neither knows the other exists, and the pages hold the two
+// against each other.
+//
+// PROTOBUF subjects are named but not shaped. Reading protobuf properly is
+// portolan-extract-proto's whole job, and a second, worse parser here would
+// be a second answer to one question. Point extract-proto at the vendored
+// directory for those.
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// Options are what the manifest tells the extractor: the things a registry
+// does not say about the estate its schemas belong to.
+type Options struct {
+	Context string `json:"context,omitempty"`
+	Service string `json:"service,omitempty"`
+
+	// Paths narrows the search to the directories fetch-csr wrote into,
+	// relative to the input root. Left out, the whole root is walked for
+	// locks.
+	Paths []string `json:"paths,omitempty"`
+
+	// Strategy is how a subject name relates to a topic and a record. It is
+	// the producer's choice of SubjectNameStrategy, and nothing in the
+	// registry's answer says which one was used - the same string is a topic
+	// plus a suffix under one and a record's full name under another.
+	Strategy string `json:"strategy,omitempty"`
+
+	// Direction is which way the channels this service registers schemas for
+	// travel, from this service's side. A registry records no producer and no
+	// consumer, so this cannot be read and has to be told; "send" is the
+	// default because the service that registers a schema is usually the one
+	// putting it on the bus.
+	Direction string `json:"direction,omitempty"`
+
+	// Subjects overrides Direction for named subjects, for the service that
+	// both publishes and listens.
+	Subjects map[string]string `json:"subjects,omitempty"`
+
+	Out string `json:"out,omitempty"`
+}
+
+func main() {
+	if err := run(os.Stdin, os.Stdout); err != nil {
+		fmt.Fprintln(os.Stderr, "portolan-extract-csr:", err)
+		os.Exit(1)
+	}
+}
+
+func run(stdin io.Reader, stdout io.Writer) error {
+	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
+		if req.Input.Root == "" {
+			return plugin.Response{}, fmt.Errorf("no input root: an extractor has nothing to read")
+		}
+
+		return extract(req.Input, opts)
+	})
+}

package/plugins/extract-csr/options.schema.json

@@ -0,0 +1,43 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "context": {
+      "type": "string",
+      "description": "Slug of the bounded context that owns the service these schemas belong to. Defaults to the name of the input directory.",
+      "examples": ["shop"]
+    },
+    "service": {
+      "type": "string",
+      "description": "Slug of the service that registers them. Defaults to the name of the input directory.",
+      "examples": ["oms"]
+    },
+    "paths": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Directories fetch-csr vendored into, relative to the input root. Left out, the whole root is walked for locks.",
+      "examples": [["vendor/schemas"]]
+    },
+    "strategy": {
+      "enum": ["topic", "record", "topic-record"],
+      "default": "topic",
+      "description": "How a subject name relates to a topic and a record - the producer's SubjectNameStrategy. Nothing in the registry's answer says which one was used, so it is told here. `topic` reads \"<topic>-value\" and \"<topic>-key\"; `record` reads the whole subject as the record's full name and yields no topic; `topic-record` reads \"<topic>-<record full name>\"."
+    },
+    "direction": {
+      "enum": ["send", "receive"],
+      "default": "send",
+      "description": "Which way these channels travel, from this service's side. A registry records no producer and no consumer, so this cannot be read and has to be told."
+    },
+    "subjects": {
+      "type": "object",
+      "additionalProperties": { "enum": ["send", "receive"] },
+      "description": "Direction per subject, for a service that both publishes and listens. Overrides `direction`.",
+      "examples": [{ "shop.oms.order-value": "send", "payments.ledger.payment-value": "receive" }]
+    },
+    "out": {
+      "type": "string",
+      "description": "Name of the fragment file, inside the step's output directory.",
+      "default": "schemas.json"
+    }
+  }
+}

package/plugins/extract-csr/subject.go

@@ -0,0 +1,102 @@
+package main
+
+// Reading a subject name.
+//
+// A registry stores subjects. A subject is whatever the producer's serializer
+// decided to call the registration, and the rule it used - the
+// SubjectNameStrategy - is not recorded anywhere in the answer. The same
+// string "shop.oms.order-value" is a topic plus a suffix under one strategy
+// and a record's full name under another, and nothing but the manifest can
+// say which.
+//
+// So the strategy is told, and everything here is the consequence of being
+// told it.
+
+import "strings"
+
+// Strategies, by the names Confluent's own serializers go by, shortened.
+const (
+	StrategyTopic       = "topic"        // TopicNameStrategy
+	StrategyRecord      = "record"       // RecordNameStrategy
+	StrategyTopicRecord = "topic-record" // TopicRecordNameStrategy
+)
+
+// registration is what a subject name resolves to.
+type registration struct {
+	// Topic is the channel address the schema is registered against, empty
+	// under RecordNameStrategy - which deliberately says nothing about topics,
+	// so a record can be reused across many.
+	Topic string
+
+	// Record is the schema's full name, and it is the name a message on the
+	// bus goes by - the same string an event's Wire.Name carries.
+	Record string
+
+	// Key is set for the "-key" half of a topic. A key is not a message on the
+	// channel; it is part of every message on it. So its shape is kept and its
+	// name is not put on the channel.
+	Key bool
+}
+
+// resolve reads a subject name into what it registers.
+//
+// declared is the record's full name as the schema itself gives it, which is
+// the only reliable way to split a TopicRecordNameStrategy subject: the
+// separator is a hyphen and both halves may contain one, so the suffix is
+// matched rather than searched for. Under the other two strategies it is a
+// fallback for a schema that names itself nothing.
+func resolve(subject, declared, strategy string) registration {
+	switch strategy {
+	case StrategyRecord:
+		return registration{Record: firstNonEmpty(declared, subject)}
+
+	case StrategyTopicRecord:
+		if declared != "" && strings.HasSuffix(subject, "-"+declared) {
+			return registration{
+				Topic:  strings.TrimSuffix(subject, "-"+declared),
+				Record: declared,
+			}
+		}
+		// The schema did not name itself, or named itself something the
+		// subject does not end with. Splitting at the first hyphen is the only
+		// thing left, and it is right for the ordinary "orders-com.acme.Order".
+		topic, record, found := strings.Cut(subject, "-")
+		if !found {
+			return registration{Record: firstNonEmpty(declared, subject)}
+		}
+
+		return registration{Topic: topic, Record: firstNonEmpty(declared, record)}
+
+	default: // StrategyTopic
+		if topic, found := strings.CutSuffix(subject, "-value"); found {
+			return registration{Topic: topic, Record: firstNonEmpty(declared, subject)}
+		}
+		if topic, found := strings.CutSuffix(subject, "-key"); found {
+			return registration{Topic: topic, Record: firstNonEmpty(declared, subject), Key: true}
+		}
+
+		// A subject under this strategy always ends in one of the two, so one
+		// that does not is a subject registered some other way. It still names
+		// a shape; it just cannot name a topic.
+		return registration{Record: firstNonEmpty(declared, subject)}
+	}
+}
+
+func strategyOf(declared string) string {
+	switch declared {
+	case StrategyRecord, StrategyTopicRecord, StrategyTopic:
+		return declared
+	}
+
+	return StrategyTopic
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if strings.TrimSpace(value) != "" {
+			return value
+		}
+	}
+
+	return ""
+}