@shortlink-org/portolan 0.1.0 → 0.2.1

package/plugins/extract-csr/jsonschema.go

@@ -1,343 +0,0 @@
-package main
-
-// JSON Schema, read for the same two things Avro is read for.
-//
-// The one thing that has to be careful here is ORDER. An Avro record lists its
-// fields in an array, so reading them into Go keeps the order the author wrote
-// them in. A JSON Schema lists them as the keys of an object, and decoding an
-// object into a map loses that order - which would put the fields on the page
-// in a different arrangement every run and rewrite the fragment each time. So
-// the properties are walked with a token decoder instead.
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/shortlink-org/portolan/catalog"
-)
-
-// jsonSchema reads one .json into the shapes it declares, the same way avro
-// does: the name of the shape at the top, and every named shape inside it.
-//
-// fallback names the top-level shape when the document does not name itself,
-// which most of them do not: a JSON Schema's `title` is optional and a
-// registered one usually leans on the subject to say what it is.
-func jsonSchema(raw []byte, fallback string) (string, map[string]catalog.TypeDef, error) {
-	var node jsonNode
-	if err := json.Unmarshal(raw, &node); err != nil {
-		return "", nil, fmt.Errorf("not a JSON Schema: %w", err)
-	}
-
-	name := firstNonEmpty(node.Title, fallback)
-	r := &jsonReader{defs: map[string]catalog.TypeDef{}, prefix: name}
-
-	// The named shapes a document keeps to one side are read first, so a `$ref`
-	// pointing at one resolves to a def rather than to a bare label.
-	//
-	// They are filed under the document's own name. `definitions` is local to
-	// the document by definition, and `defs` in the catalog is not: two
-	// services that both keep a "Carrier" to one side, meaning different
-	// things, would otherwise collide in the merge and be reported as a
-	// disagreement neither of them is having.
-	for _, held := range []map[string]json.RawMessage{node.Definitions, node.Defs} {
-		for _, key := range keysInOrder(held) {
-			var nested jsonNode
-			if err := json.Unmarshal(held[key], &nested); err != nil {
-				continue
-			}
-			r.shape(nested, r.qualify(key))
-		}
-	}
-
-	r.shape(node, name)
-
-	return name, r.defs, nil
-}
-
-// jsonNode is the part of a JSON Schema this reads. Everything it leaves out -
-// every keyword that constrains a value rather than naming or shaping it - is
-// the registry's business, not the catalog's.
-type jsonNode struct {
-	Title       string          `json:"title"`
-	Description string          `json:"description"`
-	Type        json.RawMessage `json:"type"`
-	Format      string          `json:"format"`
-	Ref         string          `json:"$ref"`
-	Items       *jsonNode       `json:"items"`
-	Required    []string        `json:"required"`
-	OneOf       []jsonNode      `json:"oneOf"`
-	AnyOf       []jsonNode      `json:"anyOf"`
-
-	Properties  json.RawMessage            `json:"properties"`
-	Definitions map[string]json.RawMessage `json:"definitions"`
-	Defs        map[string]json.RawMessage `json:"$defs"`
-}
-
-type jsonReader struct {
-	defs map[string]catalog.TypeDef
-	open map[string]bool
-
-	// prefix is the document's own shape name, which is what makes a
-	// document-local definition globally addressable.
-	prefix string
-}
-
-// qualify files a document-local name under the document.
-func (r *jsonReader) qualify(name string) string {
-	if r.prefix == "" || name == "" {
-		return name
-	}
-
-	return r.prefix + "." + name
-}
-
-// declared says whether a shape is one this reader wrote down, including one
-// still being read further up the stack.
-func (r *jsonReader) declared(name string) bool {
-	if r.open[name] {
-		return true
-	}
-	_, written := r.defs[name]
-
-	return written
-}
-
-// shape writes down one object's fields, under the name given, and returns
-// whether it wrote anything. A schema with no properties is not a shape.
-func (r *jsonReader) shape(node jsonNode, name string) bool {
-	if name == "" || len(node.Properties) == 0 {
-		return false
-	}
-	if r.open[name] {
-		return true
-	}
-	if _, written := r.defs[name]; written {
-		return true
-	}
-
-	if r.open == nil {
-		r.open = map[string]bool{}
-	}
-	r.open[name] = true
-	defer delete(r.open, name)
-
-	required := map[string]bool{}
-	for _, key := range node.Required {
-		required[key] = true
-	}
-
-	// Written before the fields are read, so a schema that names itself inside
-	// itself finds an entry rather than recursing forever.
-	r.defs[name] = catalog.TypeDef{Fields: []catalog.Field{}}
-
-	fields := []catalog.Field{}
-	for _, key := range propertiesInOrder(node.Properties) {
-		var property jsonNode
-		if err := json.Unmarshal(key.value, &property); err != nil {
-			continue
-		}
-
-		label, ref := r.typeOf(property, name+"."+key.name)
-		if !required[key.name] && !strings.HasSuffix(label, "?") {
-			label += "?"
-		}
-
-		fields = append(fields, catalog.Field{
-			Name: key.name,
-			Type: label,
-			Doc:  strings.TrimSpace(property.Description),
-			Ref:  ref,
-		})
-	}
-
-	r.defs[name] = catalog.TypeDef{Fields: fields}
-
-	return true
-}
-
-// typeOf renders one property, and returns the defs key when it is an object
-// this reader wrote down.
-//
-// path is what an anonymous nested object is called: a schema that inlines an
-// address under `shipping` has named it, just not with a `title`, and
-// "OrderPlaced.shipping" is the name it gave it.
-func (r *jsonReader) typeOf(node jsonNode, path string) (label, ref string) {
-	if node.Ref != "" {
-		// The label stays the name the document uses - "Carrier" is what a
-		// reader of the schema sees - while the ref carries the qualified key
-		// the catalog files it under. The two are allowed to differ, and here
-		// they have to.
-		name := refName(node.Ref)
-		if key := r.qualify(name); r.declared(key) {
-			return name, key
-		}
-
-		return name, ""
-	}
-
-	if members := append(append([]jsonNode{}, node.OneOf...), node.AnyOf...); len(members) > 0 {
-		return r.choice(members, path), ""
-	}
-
-	kinds, nullable := typeNames(node.Type)
-
-	if node.Items != nil {
-		items, _ := r.typeOf(*node.Items, path)
-
-		return suffix(items+"[]", nullable), ""
-	}
-
-	if len(node.Properties) > 0 {
-		name := firstNonEmpty(node.Title, path)
-		if r.shape(node, name) {
-			return suffix(name, nullable), name
-		}
-	}
-
-	// A format says what the string MEANS, and it is what belongs on a page:
-	// "date-time" says more than the "string" underneath it.
-	if node.Format != "" {
-		return suffix(node.Format, nullable), ""
-	}
-	if len(kinds) == 0 {
-		return suffix(firstNonEmpty(node.Title, "any"), nullable), ""
-	}
-
-	return suffix(strings.Join(kinds, "|"), nullable), ""
-}
-
-// choice renders oneOf/anyOf. A member that is only `{"type":"null"}` is the
-// document saying the value is optional, which is a "?" rather than a member.
-func (r *jsonReader) choice(members []jsonNode, path string) string {
-	var (
-		labels   []string
-		nullable bool
-	)
-
-	for _, member := range members {
-		kinds, memberNull := typeNames(member.Type)
-		if memberNull || (len(kinds) == 1 && kinds[0] == "null") {
-			nullable = true
-			if len(kinds) <= 1 && member.Ref == "" && len(member.Properties) == 0 {
-				continue
-			}
-		}
-
-		label, _ := r.typeOf(member, path)
-		labels = append(labels, strings.TrimSuffix(label, "?"))
-	}
-
-	if len(labels) == 0 {
-		return "null"
-	}
-
-	return suffix(strings.Join(labels, "|"), nullable)
-}
-
-// typeNames reads `type`, which is either a string or an array of them, and
-// pulls "null" out of the array into the nullability it actually means.
-func typeNames(raw json.RawMessage) (kinds []string, nullable bool) {
-	if len(raw) == 0 {
-		return nil, false
-	}
-
-	var one string
-	if json.Unmarshal(raw, &one) == nil {
-		if one == "null" {
-			return nil, true
-		}
-
-		return []string{one}, false
-	}
-
-	var many []string
-	if json.Unmarshal(raw, &many) != nil {
-		return nil, false
-	}
-
-	for _, kind := range many {
-		if kind == "null" {
-			nullable = true
-
-			continue
-		}
-		kinds = append(kinds, kind)
-	}
-
-	return kinds, nullable
-}
-
-func suffix(label string, nullable bool) string {
-	if nullable && !strings.HasSuffix(label, "?") {
-		return label + "?"
-	}
-
-	return label
-}
-
-// refName is the last segment of a `$ref`, which is the name the definition is
-// filed under. A `$ref` reaching outside the document names something this
-// extractor has never seen, and the last segment is still the best label for
-// it.
-func refName(ref string) string {
-	at := strings.LastIndex(ref, "/")
-	if at < 0 {
-		return ref
-	}
-
-	return ref[at+1:]
-}
-
-type property struct {
-	name  string
-	value json.RawMessage
-}
-
-// propertiesInOrder walks an object with a token decoder, which is the only
-// way to keep the order its keys were written in.
-func propertiesInOrder(raw json.RawMessage) []property {
-	if len(raw) == 0 {
-		return nil
-	}
-
-	decoder := json.NewDecoder(bytes.NewReader(raw))
-	if _, err := decoder.Token(); err != nil { // the opening brace
-		return nil
-	}
-
-	var out []property
-	for decoder.More() {
-		key, err := decoder.Token()
-		if err != nil {
-			return out
-		}
-		name, isString := key.(string)
-		if !isString {
-			return out
-		}
-
-		var value json.RawMessage
-		if err := decoder.Decode(&value); err != nil {
-			return out
-		}
-		out = append(out, property{name: name, value: value})
-	}
-
-	return out
-}
-
-// keysInOrder is the same problem for `definitions`, which is decoded into a
-// map because nothing about it is ordered on the page - only sorted, so the
-// fragment comes out the same way every run.
-func keysInOrder(held map[string]json.RawMessage) []string {
-	names := make([]string, 0, len(held))
-	for name := range held {
-		names = append(names, name)
-	}
-	sort.Strings(names)
-
-	return names
-}

package/plugins/extract-csr/lock.go

@@ -1,25 +0,0 @@
-package main
-
-// The lock, as this side reads it.
-//
-// A narrower copy of what portolan-fetch-csr writes, on purpose: this plugin
-// needs the subject, the version and which file holds the schema, and reading
-// a shape it does not use would be a second place to keep the writer's
-// decisions in step. The two are held together by the fields' names, which is
-// the same contract every fragment in this repository is held to.
-type Lock struct {
-	Registry string        `json:"registry"`
-	Subjects []LockSubject `json:"subjects"`
-}
-
-type LockSubject struct {
-	Subject    string      `json:"subject"`
-	Version    int         `json:"version"`
-	ID         int         `json:"id"`
-	SchemaType string      `json:"schemaType"`
-	Files      []LockEntry `json:"files"`
-}
-
-type LockEntry struct {
-	Path string `json:"path"`
-}

package/plugins/extract-csr/main.go

@@ -1,84 +0,0 @@
-// Package main is portolan-extract-csr: schemas vendored out of a Confluent
-// Schema Registry in, a catalog fragment out.
-//
-// It reads what portolan-fetch-csr committed - a directory per subject, each
-// holding one schema and the `csr.lock.json` that says which registration it
-// is - and answers with the two facts a registry knows that nothing else in
-// the estate does: which topics carry which messages, and what those messages
-// are shaped like, field by field.
-//
-// It opens nothing. That is the point of the split: portolan-fetch-csr owns
-// the socket and the credential, this owns the reading, and CI runs this over
-// a tree it can verify without a registry existing at all.
-//
-// What it does NOT do is invent events. An Event in the catalog belongs to an
-// aggregate, and a registry has no idea which aggregate raises what - it holds
-// schemas, not domains. So the shapes land in `defs`, where a shared shape
-// belongs, and the topics land in the service's channels, the same place an
-// AsyncAPI document's do. The domain extractor says an aggregate raises
-// OrderPlaced and calls it `shop.oms.OrderPlaced` on the wire; this says a
-// schema by that name is registered against topic `shop.oms.order` and has
-// these fields. Neither knows the other exists, and the pages hold the two
-// against each other.
-//
-// PROTOBUF subjects are named but not shaped. Reading protobuf properly is
-// portolan-extract-proto's whole job, and a second, worse parser here would
-// be a second answer to one question. Point extract-proto at the vendored
-// directory for those.
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-// Options are what the manifest tells the extractor: the things a registry
-// does not say about the estate its schemas belong to.
-type Options struct {
-	Context string `json:"context,omitempty"`
-	Service string `json:"service,omitempty"`
-
-	// Paths narrows the search to the directories fetch-csr wrote into,
-	// relative to the input root. Left out, the whole root is walked for
-	// locks.
-	Paths []string `json:"paths,omitempty"`
-
-	// Strategy is how a subject name relates to a topic and a record. It is
-	// the producer's choice of SubjectNameStrategy, and nothing in the
-	// registry's answer says which one was used - the same string is a topic
-	// plus a suffix under one and a record's full name under another.
-	Strategy string `json:"strategy,omitempty"`
-
-	// Direction is which way the channels this service registers schemas for
-	// travel, from this service's side. A registry records no producer and no
-	// consumer, so this cannot be read and has to be told; "send" is the
-	// default because the service that registers a schema is usually the one
-	// putting it on the bus.
-	Direction string `json:"direction,omitempty"`
-
-	// Subjects overrides Direction for named subjects, for the service that
-	// both publishes and listens.
-	Subjects map[string]string `json:"subjects,omitempty"`
-
-	Out string `json:"out,omitempty"`
-}
-
-func main() {
-	if err := run(os.Stdin, os.Stdout); err != nil {
-		fmt.Fprintln(os.Stderr, "portolan-extract-csr:", err)
-		os.Exit(1)
-	}
-}
-
-func run(stdin io.Reader, stdout io.Writer) error {
-	return plugin.Serve(stdin, stdout, descriptor(), func(req plugin.Request, opts Options) (plugin.Response, error) {
-		if req.Input.Root == "" {
-			return plugin.Response{}, fmt.Errorf("no input root: an extractor has nothing to read")
-		}
-
-		return extract(req.Input, opts)
-	})
-}

package/plugins/extract-csr/subject.go

@@ -1,102 +0,0 @@
-package main
-
-// Reading a subject name.
-//
-// A registry stores subjects. A subject is whatever the producer's serializer
-// decided to call the registration, and the rule it used - the
-// SubjectNameStrategy - is not recorded anywhere in the answer. The same
-// string "shop.oms.order-value" is a topic plus a suffix under one strategy
-// and a record's full name under another, and nothing but the manifest can
-// say which.
-//
-// So the strategy is told, and everything here is the consequence of being
-// told it.
-
-import "strings"
-
-// Strategies, by the names Confluent's own serializers go by, shortened.
-const (
-	StrategyTopic       = "topic"        // TopicNameStrategy
-	StrategyRecord      = "record"       // RecordNameStrategy
-	StrategyTopicRecord = "topic-record" // TopicRecordNameStrategy
-)
-
-// registration is what a subject name resolves to.
-type registration struct {
-	// Topic is the channel address the schema is registered against, empty
-	// under RecordNameStrategy - which deliberately says nothing about topics,
-	// so a record can be reused across many.
-	Topic string
-
-	// Record is the schema's full name, and it is the name a message on the
-	// bus goes by - the same string an event's Wire.Name carries.
-	Record string
-
-	// Key is set for the "-key" half of a topic. A key is not a message on the
-	// channel; it is part of every message on it. So its shape is kept and its
-	// name is not put on the channel.
-	Key bool
-}
-
-// resolve reads a subject name into what it registers.
-//
-// declared is the record's full name as the schema itself gives it, which is
-// the only reliable way to split a TopicRecordNameStrategy subject: the
-// separator is a hyphen and both halves may contain one, so the suffix is
-// matched rather than searched for. Under the other two strategies it is a
-// fallback for a schema that names itself nothing.
-func resolve(subject, declared, strategy string) registration {
-	switch strategy {
-	case StrategyRecord:
-		return registration{Record: firstNonEmpty(declared, subject)}
-
-	case StrategyTopicRecord:
-		if declared != "" && strings.HasSuffix(subject, "-"+declared) {
-			return registration{
-				Topic:  strings.TrimSuffix(subject, "-"+declared),
-				Record: declared,
-			}
-		}
-		// The schema did not name itself, or named itself something the
-		// subject does not end with. Splitting at the first hyphen is the only
-		// thing left, and it is right for the ordinary "orders-com.acme.Order".
-		topic, record, found := strings.Cut(subject, "-")
-		if !found {
-			return registration{Record: firstNonEmpty(declared, subject)}
-		}
-
-		return registration{Topic: topic, Record: firstNonEmpty(declared, record)}
-
-	default: // StrategyTopic
-		if topic, found := strings.CutSuffix(subject, "-value"); found {
-			return registration{Topic: topic, Record: firstNonEmpty(declared, subject)}
-		}
-		if topic, found := strings.CutSuffix(subject, "-key"); found {
-			return registration{Topic: topic, Record: firstNonEmpty(declared, subject), Key: true}
-		}
-
-		// A subject under this strategy always ends in one of the two, so one
-		// that does not is a subject registered some other way. It still names
-		// a shape; it just cannot name a topic.
-		return registration{Record: firstNonEmpty(declared, subject)}
-	}
-}
-
-func strategyOf(declared string) string {
-	switch declared {
-	case StrategyRecord, StrategyTopicRecord, StrategyTopic:
-		return declared
-	}
-
-	return StrategyTopic
-}
-
-func firstNonEmpty(values ...string) string {
-	for _, value := range values {
-		if strings.TrimSpace(value) != "" {
-			return value
-		}
-	}
-
-	return ""
-}

package/plugins/extract-flows/describe.go

@@ -1,19 +0,0 @@
-package main
-
-import (
-	_ "embed"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-//go:embed options.schema.json
-var optionsSchema []byte
-
-func descriptor() plugin.Descriptor {
-	return plugin.Descriptor{
-		Name:    "extract-flows",
-		Summary: "Reads flows written by hand as *.flow.md - one line per hop, frames closed by end - into a catalog fragment.",
-		Phases:  []string{plugin.PhaseExtract},
-		Options: optionsSchema,
-	}
-}

package/plugins/extract-flows/describe_test.go

@@ -1,11 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/shortlink-org/portolan/plugin/schematest"
-)
-
-func TestOptionsSchemaMatchesStruct(t *testing.T) {
-	schematest.Check(t, optionsSchema, Options{})
-}

package/plugins/extract-flows/extract.go

@@ -1,91 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"errors"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	"github.com/shortlink-org/portolan/catalog"
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-// extract reads every flow file the options name and answers with one
-// fragment holding the flows. A file that does not parse fails the whole run,
-// the way a compiler would: a flow written by hand is written to be read, and
-// one silently left out of the catalog is the kind of missing nobody notices.
-func extract(in plugin.Input, opts Options) (plugin.Response, error) {
-	b := &plugin.Builder{}
-
-	patterns := opts.Files
-	if len(patterns) == 0 {
-		patterns = []string{"*.flow.md"}
-	}
-	var files []string
-	seen := map[string]bool{}
-	for _, pattern := range patterns {
-		matches, err := filepath.Glob(filepath.Join(in.Root, pattern))
-		if err != nil {
-			return plugin.Response{}, err
-		}
-		for _, match := range matches {
-			if !seen[match] {
-				seen[match] = true
-				files = append(files, match)
-			}
-		}
-	}
-	sort.Strings(files)
-	if len(files) == 0 {
-		b.Warn(in.Root, "no flow files matched "+strings.Join(patterns, ", ")+"; the fragment holds no flows")
-	}
-
-	flows := []catalog.Flow{}
-	var problems []string
-	slugs := map[string]string{}
-	for _, file := range files {
-		src, err := os.ReadFile(file)
-		if err != nil {
-			return plugin.Response{}, err
-		}
-		rel := filepath.ToSlash(file)
-		flow, errs := parseFlow(rel, string(src))
-		if len(errs) > 0 {
-			problems = append(problems, errs...)
-
-			continue
-		}
-		if other, taken := slugs[flow.Slug]; taken {
-			problems = append(problems, rel+": flow "+flow.Slug+" is already declared in "+other)
-
-			continue
-		}
-		slugs[flow.Slug] = rel
-		flows = append(flows, flow)
-	}
-	if len(problems) > 0 {
-		return plugin.Response{}, errors.New(strings.Join(problems, "\n"))
-	}
-
-	fragment := catalog.Catalog{
-		GeneratedAt: in.GeneratedAt,
-		Commit:      in.Commit,
-		Contexts:    []catalog.BoundedContext{},
-		Defs:        map[string]catalog.TypeDef{},
-		Flows:       flows,
-		Adrs:        []catalog.Adr{},
-	}
-	encoded, err := json.MarshalIndent(fragment, "", "  ")
-	if err != nil {
-		return plugin.Response{}, err
-	}
-	out := opts.Out
-	if out == "" {
-		out = "flows.json"
-	}
-	b.File(out, string(encoded)+"\n")
-
-	return b.Response(), nil
-}