@shortlink-org/portolan 0.1.0 → 0.2.0

package/plugins/fetch-csr/fetch_test.go

@@ -1,494 +0,0 @@
-package main
-
-// The fetcher, against a recorded registry. NEVER against a live one: a test
-// that reaches a Confluent endpoint would fail on a train, and the whole point
-// of this plugin's design is that the build does not depend on a registry
-// being up.
-
-import (
-	"encoding/base64"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"testing"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-const (
-	valueSubject = "shop.oms.order-value"
-	partySubject = "shop.oms.Party"
-)
-
-// The value schema references the party one, which is how a registry says a
-// record is shared between subjects.
-var (
-	orderSchema = `{"type":"record","namespace":"shop.oms","name":"OrderPlaced","fields":[{"name":"order_id","type":"string"},{"name":"buyer","type":"shop.oms.Party"}]}`
-	partySchema = `{"type":"record","namespace":"shop.oms","name":"Party","fields":[{"name":"id","type":"string"}]}`
-)
-
-// registry serves the one call this plugin makes, from recorded shapes.
-func registry(t *testing.T, seen *[]string) *httptest.Server {
-	t.Helper()
-
-	answers := map[string]registration{
-		"/subjects/shop.oms.order-value/versions/3": {
-			Subject: valueSubject, Version: 3, ID: 100021, GUID: "8f0d",
-			SchemaType: "AVRO", Schema: orderSchema,
-			References: []Reference{{Name: "shop.oms.Party", Subject: partySubject, Version: 1}},
-		},
-		"/subjects/shop.oms.order-value/versions/latest": {
-			Subject: valueSubject, Version: 3, ID: 100021, GUID: "8f0d",
-			SchemaType: "AVRO", Schema: orderSchema,
-			References: []Reference{{Name: "shop.oms.Party", Subject: partySubject, Version: 1}},
-		},
-		"/subjects/shop.oms.Party/versions/1": {
-			Subject: partySubject, Version: 1, ID: 100014,
-			SchemaType: "AVRO", Schema: partySchema,
-		},
-	}
-
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if seen != nil {
-			*seen = append(*seen, r.Header.Get("Authorization"))
-		}
-
-		w.Header().Set("Content-Type", "application/json")
-
-		answer, known := answers[r.URL.Path]
-		if !known {
-			w.WriteHeader(http.StatusNotFound)
-			_, _ = w.Write([]byte(`{"error_code":40401,"message":"Subject '` + r.URL.Path + `' not found."}`))
-
-			return
-		}
-
-		_ = json.NewEncoder(w).Encode(answer)
-	}))
-}
-
-func options(server *httptest.Server, cache string, subjects ...Subject) Options {
-	return Options{Registry: server.URL, Cache: cache, Subjects: subjects}
-}
-
-// names is the file list, sorted, so a test can say what a run produced
-// without depending on the order the queue happened to drain in.
-func names(resp plugin.Response) []string {
-	var out []string
-	for _, f := range resp.Files {
-		out = append(out, f.Name)
-	}
-	sort.Strings(out)
-
-	return out
-}
-
-func contents(t *testing.T, resp plugin.Response, name string) string {
-	t.Helper()
-
-	for _, f := range resp.Files {
-		if f.Name == name {
-			return f.Contents
-		}
-	}
-	t.Fatalf("no file named %s in %v", name, names(resp))
-
-	return ""
-}
-
-// write is the host's half: it puts a response into a tree, so the next run
-// can replay from it the way a real one replays from what was committed.
-func write(t *testing.T, root string, resp plugin.Response) {
-	t.Helper()
-
-	for _, f := range resp.Files {
-		at := filepath.Join(root, filepath.FromSlash(f.Name))
-		if err := os.MkdirAll(filepath.Dir(at), 0o755); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.WriteFile(at, []byte(f.Contents), 0o644); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func offlineOff(t *testing.T) {
-	t.Helper()
-	t.Setenv(OfflineEnv, "")
-	t.Setenv("CI", "")
-}
-
-func TestFetchWritesSchemaAndLock(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-	defer server.Close()
-
-	cache := t.TempDir()
-	resp, err := fetch(options(server, cache, Subject{Subject: valueSubject, Version: 3}))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want := []string{
-		"shop.oms.Party/csr.lock.json",
-		"shop.oms.Party/v1.avsc",
-		"shop.oms.order-value/csr.lock.json",
-		"shop.oms.order-value/v3.avsc",
-	}
-	if got := names(resp); !equal(got, want) {
-		t.Errorf("files:\n got %v\nwant %v", got, want)
-	}
-
-	// The reference was followed without the manifest naming it: the referring
-	// schema pinned it, which is the whole argument for following one at all.
-	if got := contents(t, resp, "shop.oms.Party/v1.avsc"); !strings.Contains(got, `"name": "Party"`) {
-		t.Errorf("the referenced schema is not the one the registry served:\n%s", got)
-	}
-
-	// Indented rather than the one line the registry answers with, so a
-	// version bump is a diff a person can read.
-	schema := contents(t, resp, "shop.oms.order-value/v3.avsc")
-	if !strings.Contains(schema, "\n  \"type\": \"record\"") {
-		t.Errorf("the schema was not indented:\n%s", schema)
-	}
-	if !strings.HasSuffix(schema, "\n") {
-		t.Error("the schema does not end in a newline")
-	}
-	// json.Indent reformats without reordering, so the file still says what
-	// the registry said in the order it said it.
-	if strings.Index(schema, `"namespace"`) > strings.Index(schema, `"name"`) {
-		t.Errorf("the schema's keys were reordered:\n%s", schema)
-	}
-
-	var lock Lock
-	if err := json.Unmarshal([]byte(contents(t, resp, "shop.oms.order-value/csr.lock.json")), &lock); err != nil {
-		t.Fatal(err)
-	}
-	if lock.Registry != server.URL {
-		t.Errorf("lock names registry %q, want %q", lock.Registry, server.URL)
-	}
-	if len(lock.Subjects) != 1 {
-		t.Fatalf("lock names %d subjects, want 1", len(lock.Subjects))
-	}
-
-	entry := lock.Subjects[0]
-	if entry.Subject != valueSubject || entry.Version != 3 || entry.ID != 100021 {
-		t.Errorf("lock entry: %+v", entry)
-	}
-	if entry.SchemaType != "AVRO" {
-		t.Errorf("schema type %q, want AVRO", entry.SchemaType)
-	}
-	if len(entry.References) != 1 || entry.References[0].Subject != partySubject {
-		t.Errorf("references: %+v", entry.References)
-	}
-	if len(entry.Files) != 1 || entry.Files[0].Path != "v3.avsc" {
-		t.Fatalf("files: %+v", entry.Files)
-	}
-	// The digest is over the bytes as written, so verifying needs no
-	// reformatting of anything.
-	if entry.Files[0].SHA256 != digestOf([]byte(schema)) {
-		t.Error("the digest is not over the bytes that were written")
-	}
-	if entry.Files[0].Size != len(schema) {
-		t.Errorf("size %d, want %d", entry.Files[0].Size, len(schema))
-	}
-}
-
-func TestAbsentSchemaTypeIsAvro(t *testing.T) {
-	offlineOff(t)
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		// A registry that predates the field says nothing, and that silence
-		// means Avro rather than "unknown".
-		_, _ = w.Write([]byte(`{"subject":"legacy-value","version":1,"id":7,"schema":` +
-			mustQuote(partySchema) + `}`))
-	}))
-	defer server.Close()
-
-	resp, err := fetch(options(server, t.TempDir(), Subject{Subject: "legacy-value", Version: 1}))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got := names(resp); !equal(got, []string{"legacy-value/csr.lock.json", "legacy-value/v1.avsc"}) {
-		t.Errorf("files: %v", got)
-	}
-}
-
-func TestUnpinnedResolvesLatestAndSaysSo(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-	defer server.Close()
-
-	resp, err := fetch(options(server, t.TempDir(), Subject{Subject: valueSubject}))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got := contents(t, resp, "shop.oms.order-value/v3.avsc"); got == "" {
-		t.Error("latest resolved to nothing")
-	}
-
-	warnings := resp.Warnings()
-	if len(warnings) != 1 || !strings.Contains(warnings[0].Message, "not pinned") {
-		t.Errorf("an unpinned subject was not reported: %+v", warnings)
-	}
-}
-
-func TestOfflineReplaysWhatWasCommitted(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-
-	cache := t.TempDir()
-	opts := options(server, cache, Subject{Subject: valueSubject, Version: 3})
-
-	first, err := fetch(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	write(t, cache, first)
-
-	// The registry is gone AND the plugin is told not to look, which is CI.
-	server.Close()
-	t.Setenv(OfflineEnv, "1")
-
-	second, err := fetch(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Byte-identical, or `gen:check` would go red on a build that changed
-	// nothing.
-	if !sameFiles(first, second) {
-		t.Errorf("the offline run differs:\n online %v\noffline %v", names(first), names(second))
-	}
-	// The reference was followed from the lock rather than from the registry.
-	if got := contents(t, second, "shop.oms.Party/v1.avsc"); got == "" {
-		t.Error("the referenced subject was not replayed")
-	}
-}
-
-func TestOfflineRefusesAnUnpinnedSubject(t *testing.T) {
-	t.Setenv(OfflineEnv, "1")
-	server := registry(t, nil)
-	defer server.Close()
-
-	_, err := fetch(options(server, t.TempDir(), Subject{Subject: valueSubject}))
-	if err == nil || !strings.Contains(err.Error(), "not pinned") {
-		t.Errorf("an unpinned subject was replayed offline: %v", err)
-	}
-}
-
-func TestAFailedFetchFallsBackToTheTree(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-
-	cache := t.TempDir()
-	opts := options(server, cache, Subject{Subject: valueSubject, Version: 3})
-
-	first, err := fetch(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	write(t, cache, first)
-
-	// Rule 2: the registry is down, the tree still holds a good copy, so the
-	// output is unchanged.
-	server.Close()
-
-	second, err := fetch(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !sameFiles(first, second) {
-		t.Error("a fetch that failed over to the tree produced different files")
-	}
-	if len(second.Warnings()) == 0 {
-		t.Error("falling back to the tree was not reported")
-	}
-}
-
-func TestAFailedFetchWithNoTreeIsAnError(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-	server.Close()
-
-	// Rule 3: never a short file list. The host deletes what a step stops
-	// naming, and a laptop going offline must not empty the vendored schemas.
-	_, err := fetch(options(server, t.TempDir(), Subject{Subject: valueSubject, Version: 3}))
-	if err == nil {
-		t.Fatal("a fetch with nothing to fall back to succeeded")
-	}
-	if !strings.Contains(err.Error(), "no usable copy") {
-		t.Errorf("the error does not say the tree is empty: %v", err)
-	}
-}
-
-func TestAnEditedVendoredCopyIsReported(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-
-	cache := t.TempDir()
-	opts := options(server, cache, Subject{Subject: valueSubject, Version: 3})
-
-	first, err := fetch(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	write(t, cache, first)
-
-	at := filepath.Join(cache, "shop.oms.order-value", "v3.avsc")
-	if err := os.WriteFile(at, []byte("{}\n"), 0o644); err != nil {
-		t.Fatal(err)
-	}
-
-	server.Close()
-	t.Setenv(OfflineEnv, "1")
-
-	_, err = fetch(opts)
-	if err == nil || !strings.Contains(err.Error(), "edited by hand") {
-		t.Errorf("an edited vendored copy was accepted: %v", err)
-	}
-}
-
-func TestAMovedPinIsReportedRatherThanReplayed(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-
-	cache := t.TempDir()
-	first, err := fetch(options(server, cache, Subject{Subject: valueSubject, Version: 3}))
-	if err != nil {
-		t.Fatal(err)
-	}
-	write(t, cache, first)
-
-	server.Close()
-	t.Setenv(OfflineEnv, "1")
-
-	_, err = fetch(options(server, cache, Subject{Subject: valueSubject, Version: 4}))
-	if err == nil || !strings.Contains(err.Error(), "holds version 3") {
-		t.Errorf("a moved pin was replayed from the old version: %v", err)
-	}
-}
-
-func TestOneSubjectAtTwoVersionsIsRefused(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-	defer server.Close()
-
-	_, err := fetch(options(server, t.TempDir(),
-		Subject{Subject: valueSubject, Version: 3},
-		Subject{Subject: valueSubject, Version: 2},
-	))
-	if err == nil || !strings.Contains(err.Error(), "one version per subject") {
-		t.Errorf("two versions of one subject were fetched into one directory: %v", err)
-	}
-}
-
-func TestAMissingSubjectSaysWhatTheRegistrySaid(t *testing.T) {
-	offlineOff(t)
-	server := registry(t, nil)
-	defer server.Close()
-
-	_, err := fetch(options(server, t.TempDir(), Subject{Subject: "nope-value", Version: 1}))
-	if err == nil || !strings.Contains(err.Error(), "not found") {
-		t.Errorf("the registry's own message was lost: %v", err)
-	}
-}
-
-func TestTheCredentialComesFromTheEnvironment(t *testing.T) {
-	offlineOff(t)
-
-	var seen []string
-	server := registry(t, &seen)
-	defer server.Close()
-
-	t.Setenv(TokenEnv, "")
-	t.Setenv(ShortKeyEnv, "KEY")
-	t.Setenv(ShortSecretEnv, "SECRET")
-	t.Setenv(KeyEnv, "")
-	t.Setenv(SecretEnv, "")
-
-	if _, err := fetch(options(server, t.TempDir(), Subject{Subject: valueSubject, Version: 3})); err != nil {
-		t.Fatal(err)
-	}
-
-	want := "Basic " + base64.StdEncoding.EncodeToString([]byte("KEY:SECRET"))
-	for _, got := range seen {
-		if got != want {
-			t.Fatalf("Authorization %q, want %q", got, want)
-		}
-	}
-	if len(seen) == 0 {
-		t.Fatal("the registry was never called")
-	}
-}
-
-func TestABareTokenIsGivenAScheme(t *testing.T) {
-	t.Setenv(TokenEnv, "abc123")
-	if got := authorization(); got != "Bearer abc123" {
-		t.Errorf("authorization %q", got)
-	}
-
-	// One that already names its scheme is passed through: guessing on the
-	// caller's behalf is how a working token starts failing.
-	t.Setenv(TokenEnv, "Negotiate abc123")
-	if got := authorization(); got != "Negotiate abc123" {
-		t.Errorf("authorization %q", got)
-	}
-}
-
-func TestSlugOfKeepsTheSubjectReadable(t *testing.T) {
-	for subject, want := range map[string]string{
-		"shop.oms.order-value": "shop.oms.order-value",
-		"shop.oms.OrderPlaced": "shop.oms.OrderPlaced",
-		"a/b:c value":          "a_b_c_value",
-		"...":                  "_",
-	} {
-		if got := slugOf(subject); got != want {
-			t.Errorf("slugOf(%q) = %q, want %q", subject, got, want)
-		}
-	}
-}
-
-func equal(a, b []string) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-
-	return true
-}
-
-func sameFiles(a, b plugin.Response) bool {
-	if len(a.Files) != len(b.Files) {
-		return false
-	}
-
-	held := map[string]string{}
-	for _, f := range a.Files {
-		held[f.Name] = f.Contents
-	}
-	for _, f := range b.Files {
-		if held[f.Name] != f.Contents {
-			return false
-		}
-	}
-
-	return true
-}
-
-func mustQuote(s string) string {
-	out, err := json.Marshal(s)
-	if err != nil {
-		panic(err)
-	}
-
-	return string(out)
-}

package/plugins/fetch-csr/lock.go

@@ -1,95 +0,0 @@
-package main
-
-// What was fetched, so it can be checked without fetching again.
-//
-// One lock per subject directory rather than one for the whole step, so a
-// vendored subject is self-describing: the schema, the registry it came from,
-// the version it was pinned to and the subjects it references sit together,
-// and the reader finds the lock by looking beside the file it is already
-// reading.
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
-	"sort"
-)
-
-// LockName is the file written beside each subject's schema.
-const LockName = "csr.lock.json"
-
-type Lock struct {
-	// Registry is where these bytes came from. A module's registry-global name
-	// carries its registry; a subject's name does not, so it is written down
-	// here or the vendored copy cannot say where it belongs.
-	Registry string `json:"registry"`
-
-	Subjects []LockSubject `json:"subjects"`
-}
-
-type LockSubject struct {
-	Subject string `json:"subject"`
-
-	// Version is what was actually fetched. A registered version is immutable,
-	// so a pinned download is byte-reproducible and this is what makes
-	// replaying from disk equivalent to fetching again.
-	Version int `json:"version"`
-
-	// ID is the registry-global schema id, and GUID the wider identifier for
-	// the same schema. Neither is needed to replay; both are what a message on
-	// the wire actually carries, so an incident that has a schema id and
-	// nothing else can be traced back to this file.
-	ID   int    `json:"id"`
-	GUID string `json:"guid,omitempty"`
-
-	SchemaType string `json:"schemaType"`
-
-	// References are the subjects this schema depends on, pinned by the schema
-	// itself. Each is fetched into its own directory, so this list is also how
-	// an offline replay knows what else to read.
-	References []Reference `json:"references,omitempty"`
-
-	Files []LockEntry `json:"files"`
-}
-
-type LockEntry struct {
-	Path string `json:"path"`
-
-	// SHA256 is over the bytes as written. It is what turns a cache into
-	// something checkable: a vendored copy someone edited by hand stops
-	// matching, which is exactly the drift docs/adr/org.0001.md wants visible.
-	SHA256 string `json:"sha256"`
-	Size   int    `json:"size"`
-}
-
-func digestOf(content []byte) string {
-	sum := sha256.Sum256(content)
-
-	return hex.EncodeToString(sum[:])
-}
-
-// encode writes the lock the way every generated file in this repository is
-// written: sorted, indented, newline-terminated. It is committed and compared
-// by `gen:check`, so an unstable order would rewrite it on every run.
-//
-// References are sorted too. The registry answers them in registration order,
-// which is stable for a pinned version, but they are a lookup from name to
-// subject rather than a sequence, and sorting says so.
-func (l Lock) encode() (string, error) {
-	for i := range l.Subjects {
-		sort.Slice(l.Subjects[i].Files, func(a, b int) bool {
-			return l.Subjects[i].Files[a].Path < l.Subjects[i].Files[b].Path
-		})
-		sort.Slice(l.Subjects[i].References, func(a, b int) bool {
-			return l.Subjects[i].References[a].Name < l.Subjects[i].References[b].Name
-		})
-	}
-	sort.Slice(l.Subjects, func(a, b int) bool { return l.Subjects[a].Subject < l.Subjects[b].Subject })
-
-	out, err := json.MarshalIndent(l, "", "  ")
-	if err != nil {
-		return "", err
-	}
-
-	return string(out) + "\n", nil
-}