@shortlink-org/portolan 0.1.0 → 0.2.0

package/plugins/fetch-csr/auth.go

@@ -1,95 +0,0 @@
-package main
-
-// The only file that reads the environment.
-//
-// The plugin protocol says a plugin has "no ambient state to read, no
-// environment". That is about FACTS: nothing about the estate may come from
-// anywhere but the request, or a fragment stops being reproducible from the
-// tree it describes.
-//
-// A credential is not a fact about the estate. It is how the transport is
-// allowed to open a socket - it changes whether the fetch succeeds, never what
-// the fetch says. Isolating it in one file that nothing else calls is what
-// keeps that claim checkable.
-//
-// It is also why this plugin can never be wasm. It needs a socket and a
-// secret, and `process` exists for exactly that trade.
-
-import (
-	"encoding/base64"
-	"os"
-	"strings"
-)
-
-// The names Confluent's own tools use, longest first so the specific one wins
-// over a shell that exports both.
-const (
-	KeyEnv    = "CONFLUENT_SCHEMA_REGISTRY_API_KEY"
-	SecretEnv = "CONFLUENT_SCHEMA_REGISTRY_API_SECRET"
-
-	// Short aliases, for a self-hosted registry nobody would call Confluent.
-	ShortKeyEnv    = "CSR_API_KEY"
-	ShortSecretEnv = "CSR_API_SECRET"
-
-	// TokenEnv is a whole Authorization value, for a registry fronted by
-	// something that wants a bearer token rather than an API key pair.
-	TokenEnv = "CSR_TOKEN"
-)
-
-// authorization is the header value to send, or "" for none.
-//
-// None is a normal outcome, not a failure: a registry inside a private network
-// is commonly open to it, and the local one the tests run against certainly is.
-func authorization() string {
-	if raw := env(TokenEnv); raw != "" {
-		// Passed through when it already names its scheme, so the same
-		// variable serves "Bearer ..." and an opaque token a proxy expects
-		// bare. Guessing on the caller's behalf is how a working token starts
-		// failing after somebody adds a prefix.
-		if strings.Contains(raw, " ") {
-			return raw
-		}
-
-		return "Bearer " + raw
-	}
-
-	key := first(env(KeyEnv), env(ShortKeyEnv))
-	secret := first(env(SecretEnv), env(ShortSecretEnv))
-	if key == "" || secret == "" {
-		return ""
-	}
-
-	return "Basic " + base64.StdEncoding.EncodeToString([]byte(key+":"+secret))
-}
-
-func env(name string) string { return strings.TrimSpace(os.Getenv(name)) }
-
-func first(values ...string) string {
-	for _, value := range values {
-		if value != "" {
-			return value
-		}
-	}
-
-	return ""
-}
-
-// OfflineEnv turns the fetch off. Explicit, rather than inferred from a socket
-// timeout: CI sets it in one line, and a build that would otherwise depend on
-// the registry being up cannot silently start doing so again.
-const OfflineEnv = "PORTOLAN_OFFLINE"
-
-func offline() bool {
-	if env(OfflineEnv) != "" {
-		return true
-	}
-
-	// CI verifies what was committed. It never needs to reach the registry, and
-	// a fork's pull request has no secret to reach it with.
-	switch strings.ToLower(env("CI")) {
-	case "", "0", "false":
-		return false
-	default:
-		return true
-	}
-}

package/plugins/fetch-csr/cache.go

@@ -1,69 +0,0 @@
-package main
-
-// Replaying a fetch from what is already on disk.
-//
-// The cache is not a second copy of anything: it IS the tree. The schemas this
-// plugin fetched were written by the host through the ordinary Response.Files
-// path, so they are committed, reviewable, and GC'd when the step stops naming
-// them. Reading them back and checking them against the lock is what lets a
-// build that never opens a socket produce byte-identical output.
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-)
-
-// cached is one subject's files, read back from the tree.
-type cached struct {
-	registry string
-	lock     LockSubject
-	files    map[string][]byte
-}
-
-// replay reads a subject's committed copy and checks it against its lock.
-//
-// An incomplete or edited cache is reported rather than patched over. The two
-// failures it distinguishes matter: nothing on disk means there is nothing to
-// fall back to, and bytes that no longer match their digest mean somebody
-// edited a vendored copy - the drift docs/adr/org.0001.md wants visible.
-func replay(dir string) (*cached, error) {
-	at := filepath.Join(dir, LockName)
-
-	raw, err := os.ReadFile(at)
-	if os.IsNotExist(err) {
-		return nil, fmt.Errorf("no %s in %s", LockName, filepath.ToSlash(dir))
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	var lock Lock
-	if err := json.Unmarshal(raw, &lock); err != nil {
-		return nil, fmt.Errorf("%s: %w", filepath.ToSlash(at), err)
-	}
-	if len(lock.Subjects) != 1 {
-		return nil, fmt.Errorf("%s names %d subjects; expected exactly one",
-			filepath.ToSlash(at), len(lock.Subjects))
-	}
-
-	entry := lock.Subjects[0]
-	files := make(map[string][]byte, len(entry.Files))
-
-	for _, want := range entry.Files {
-		content, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(want.Path)))
-		if os.IsNotExist(err) {
-			return nil, fmt.Errorf("%s is in the lock but not on disk", want.Path)
-		}
-		if err != nil {
-			return nil, err
-		}
-		if got := digestOf(content); got != want.SHA256 {
-			return nil, fmt.Errorf("%s does not match its digest; the vendored copy was edited by hand", want.Path)
-		}
-		files[want.Path] = content
-	}
-
-	return &cached{registry: lock.Registry, lock: entry, files: files}, nil
-}

package/plugins/fetch-csr/describe.go

@@ -1,19 +0,0 @@
-package main
-
-import (
-	_ "embed"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-//go:embed options.schema.json
-var optionsSchema []byte
-
-func descriptor() plugin.Descriptor {
-	return plugin.Descriptor{
-		Name:    "fetch-csr",
-		Summary: "Fetches pinned subject versions from a Confluent Schema Registry into the tree, with a lock beside each, so a schema published to a registry can be read without talking to it.",
-		Phases:  []string{plugin.PhaseExtract},
-		Options: optionsSchema,
-	}
-}

package/plugins/fetch-csr/describe_test.go

@@ -1,11 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/shortlink-org/portolan/plugin/schematest"
-)
-
-func TestOptionsSchemaMatchesStruct(t *testing.T) {
-	schematest.Check(t, optionsSchema, Options{})
-}

package/plugins/fetch-csr/fetch.go

@@ -1,254 +0,0 @@
-package main
-
-// Deciding whether to fetch, and what to do when the fetch cannot happen.
-//
-// The rules below are the design, not error handling. The one that matters
-// most is the third: a failed fetch with no cache is a NON-ZERO EXIT, never a
-// short file list. The host deletes files a step stops naming, and dropping a
-// repository's vendored schemas because a laptop went offline is worse than a
-// red build.
-
-import (
-	"fmt"
-	"path"
-	"path/filepath"
-	"sort"
-	"strconv"
-	"strings"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-// fetch takes no plugin.Input: it reads nothing out of the tree being
-// described. Its `cache` is repo-relative, like every other path in the
-// manifest, and everything else it needs is in the options.
-func fetch(opts Options) (plugin.Response, error) {
-	b := &plugin.Builder{}
-
-	if strings.TrimSpace(opts.Registry) == "" {
-		return plugin.Response{}, fmt.Errorf("no registry: name the schema registry's base URL in the manifest")
-	}
-	if len(opts.Subjects) == 0 {
-		return plugin.Response{}, fmt.Errorf("no subjects to fetch: name at least one in the manifest")
-	}
-	if opts.Cache == "" {
-		return plugin.Response{}, fmt.Errorf(
-			"no cache directory: set `cache` to the same path as the step's `out`, " +
-				"so an offline run can replay what the last online one wrote")
-	}
-
-	// Sorted, so the file list and the diagnostics come out the same way every
-	// run whatever order the manifest happens to list them in.
-	queue := append([]Subject(nil), opts.Subjects...)
-	sort.Slice(queue, func(i, j int) bool { return queue[i].Subject < queue[j].Subject })
-
-	c := newClient(opts.Registry, authorization())
-	skip := offline()
-
-	// Two subjects that slug to the same directory would overwrite each other
-	// silently, and one subject wanted at two versions cannot be held in a
-	// tree that keeps one version per subject. Both are refused by name rather
-	// than resolved: the manifest is what has to change.
-	claimed := map[string]Subject{}
-	done := map[string]bool{}
-
-	for len(queue) > 0 {
-		want := queue[0]
-		queue = queue[1:]
-
-		if want.Subject == "" {
-			return plugin.Response{}, fmt.Errorf("a subject entry names no subject")
-		}
-		if held, taken := claimed[want.Subject]; taken {
-			if held.Version != want.Version {
-				return plugin.Response{}, fmt.Errorf(
-					"%s is wanted at version %s and at version %s; the tree keeps one version per subject, so pin one",
-					want.Subject, pin(held.Version), pin(want.Version))
-			}
-
-			continue
-		}
-
-		dir := slugOf(want.Subject)
-		if done[dir] {
-			// Not caught by `claimed`, which is keyed by subject: two
-			// different subjects can slug into one directory.
-			return plugin.Response{}, fmt.Errorf(
-				"%s and another subject both vendor into %s; the tree cannot hold both",
-				want.Subject, dir)
-		}
-		claimed[want.Subject] = want
-		done[dir] = true
-
-		at := filepath.Join(filepath.FromSlash(opts.Cache), filepath.FromSlash(dir))
-
-		var (
-			refs []Reference
-			err  error
-		)
-
-		if skip {
-			refs, err = emitCached(b, opts.Registry, dir, at, want, "offline")
-			if err != nil {
-				return plugin.Response{}, err
-			}
-		} else if refs, err = live(c, b, opts.Registry, dir, want); err != nil {
-			// Rule 2: the tree still holds a good copy, so the output is
-			// unchanged and `--check` stays clean.
-			cachedRefs, cacheErr := emitCached(b, opts.Registry, dir, at, want, err.Error())
-			if cacheErr != nil {
-				// Rule 3: nothing to fall back to.
-				return plugin.Response{}, fmt.Errorf(
-					"%s could not be fetched (%w) and there is no usable copy in the tree (%v)",
-					want.Subject, err, cacheErr)
-			}
-			refs = cachedRefs
-		}
-
-		// A referenced subject is pinned by the schema that references it, so
-		// following one needs no permission from the manifest and adds no
-		// lottery: the version is part of the bytes we already have.
-		following := make([]Subject, 0, len(refs))
-		for _, ref := range refs {
-			if ref.Subject == "" || claimed[ref.Subject].Subject != "" {
-				continue
-			}
-			following = append(following, Subject{Subject: ref.Subject, Version: ref.Version})
-		}
-		sort.Slice(following, func(i, j int) bool { return following[i].Subject < following[j].Subject })
-		queue = append(queue, following...)
-	}
-
-	return b.Response(), nil
-}
-
-// live fetches one subject, pinning it first if the manifest did not.
-func live(c *client, b *plugin.Builder, registry, dir string, want Subject) ([]Reference, error) {
-	got, err := c.version(want.Subject, want.Version)
-	if err != nil {
-		return nil, err
-	}
-
-	if want.Version == 0 {
-		// Rule: pin, or every run is a lottery. Resolving `latest` still
-		// works, but it is said out loud, because two runs a day apart would
-		// then produce two different trees from one manifest.
-		b.Warn(want.Subject, fmt.Sprintf(
-			"is not pinned; \"latest\" resolved to version %d. Pin it in portolan.json or every run is a lottery.",
-			got.Version))
-	}
-
-	extension, err := extensionFor(got.SchemaType)
-	if err != nil {
-		return nil, fmt.Errorf("%s at version %d: %w", want.Subject, got.Version, err)
-	}
-
-	name := "v" + strconv.Itoa(got.Version) + extension
-	content := body(got.SchemaType, got.Schema)
-
-	entry := LockSubject{
-		Subject:    want.Subject,
-		Version:    got.Version,
-		ID:         got.ID,
-		GUID:       got.GUID,
-		SchemaType: schemaType(got.SchemaType),
-		References: got.References,
-		Files: []LockEntry{{
-			Path:   name,
-			SHA256: digestOf([]byte(content)),
-			Size:   len(content),
-		}},
-	}
-
-	b.File(path.Join(dir, name), content)
-	if err := emitLock(b, registry, dir, entry); err != nil {
-		return nil, err
-	}
-
-	return entry.References, nil
-}
-
-// emitCached re-emits the committed copy unchanged.
-//
-// Byte-identical to what the last online run wrote, which is the whole point:
-// `gen:check` sees no drift, so CI can verify an estate whose schemas live in a
-// registry it never talks to.
-func emitCached(b *plugin.Builder, registry, dir, at string, want Subject, why string) ([]Reference, error) {
-	if want.Version == 0 {
-		return nil, fmt.Errorf("%s is not pinned to a version, so there is nothing to replay", want.Subject)
-	}
-
-	held, err := replay(at)
-	if err != nil {
-		return nil, err
-	}
-	if held.lock.Version != want.Version {
-		return nil, fmt.Errorf("%s holds version %d but the manifest pins %d; fetch it",
-			want.Subject, held.lock.Version, want.Version)
-	}
-	if held.lock.Subject != want.Subject {
-		return nil, fmt.Errorf("%s vendors into %s, which holds %s",
-			want.Subject, filepath.ToSlash(at), held.lock.Subject)
-	}
-	if held.registry != "" && held.registry != registry {
-		return nil, fmt.Errorf("%s was fetched from %s but the manifest names %s; fetch it again",
-			want.Subject, held.registry, registry)
-	}
-
-	for _, entry := range held.lock.Files {
-		b.File(path.Join(dir, entry.Path), string(held.files[entry.Path]))
-	}
-	if err := emitLock(b, registry, dir, held.lock); err != nil {
-		return nil, err
-	}
-
-	b.Warn(want.Subject, "not fetched ("+why+"); the copy committed in this repository is used unchanged")
-
-	return held.lock.References, nil
-}
-
-func emitLock(b *plugin.Builder, registry, dir string, entry LockSubject) error {
-	lock, err := Lock{Registry: registry, Subjects: []LockSubject{entry}}.encode()
-	if err != nil {
-		return err
-	}
-	b.File(path.Join(dir, LockName), lock)
-
-	return nil
-}
-
-// slugOf is the directory a subject vendors into.
-//
-// A subject name is whatever the producer registered, and under
-// RecordNameStrategy that is a language's full name; a directory is not. So
-// everything outside a safe set becomes an underscore, and the true subject is
-// written in the lock beside the file rather than inferred back from the path.
-func slugOf(subject string) string {
-	var out strings.Builder
-
-	for _, r := range subject {
-		switch {
-		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
-			out.WriteRune(r)
-		case r == '.' || r == '-' || r == '_':
-			out.WriteRune(r)
-		default:
-			out.WriteByte('_')
-		}
-	}
-
-	slug := strings.Trim(out.String(), ".")
-	if slug == "" {
-		return "_"
-	}
-
-	return slug
-}
-
-func pin(version int) string {
-	if version == 0 {
-		return "latest"
-	}
-
-	return strconv.Itoa(version)
-}