@shortlink-org/portolan 0.1.0

package/plugins/fetch-bsr/auth.go

@@ -0,0 +1,114 @@
+package main
+
+// The only file that reads the environment.
+//
+// The plugin protocol says a plugin has "no ambient state to read, no
+// environment". That is about FACTS: nothing about the estate may come from
+// anywhere but the request, or a fragment stops being reproducible from the
+// tree it describes.
+//
+// A credential is not a fact about the estate. It is how the transport is
+// allowed to open a socket, the same way a git credential is - it changes
+// whether the fetch succeeds, never what the fetch says. Isolating it in one
+// file that nothing else calls is what keeps that claim checkable.
+//
+// It is also why this plugin can never be wasm. It needs a socket and a
+// secret, and `process` exists for exactly that trade.
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// TokenEnv is where `buf registry login` is usually mirrored in CI.
+const TokenEnv = "BUF_TOKEN"
+
+// token finds a credential for a registry, or returns none.
+//
+// None is a normal outcome, not a failure: public modules download
+// anonymously, and an estate whose schemas are public never needs a token.
+func token(registry string) string {
+	if t := strings.TrimSpace(os.Getenv(TokenEnv)); t != "" {
+		return t
+	}
+
+	return netrcToken(registry)
+}
+
+// netrcToken reads the password `buf registry login` wrote.
+func netrcToken(registry string) string {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return ""
+	}
+
+	for _, name := range []string{".netrc", "_netrc"} {
+		if t := readNetrc(filepath.Join(home, name), registry); t != "" {
+			return t
+		}
+	}
+
+	return ""
+}
+
+// readNetrc walks a netrc for the machine asked about.
+//
+// Deliberately small: netrc is a stream of words, and the only entry that
+// matters here is the password belonging to one machine.
+func readNetrc(path, machine string) string {
+	file, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer file.Close()
+
+	words := bufio.NewScanner(file)
+	words.Split(bufio.ScanWords)
+
+	current := ""
+	for words.Scan() {
+		switch words.Text() {
+		case "machine":
+			if !words.Scan() {
+				return ""
+			}
+			current = words.Text()
+		case "default":
+			current = machine
+		case "password":
+			if !words.Scan() {
+				return ""
+			}
+			if current == machine {
+				return words.Text()
+			}
+		case "login", "account":
+			// Read past the value so it cannot be mistaken for a keyword.
+			words.Scan()
+		}
+	}
+
+	return ""
+}
+
+// OfflineEnv turns the fetch off. Explicit, rather than inferred from a socket
+// timeout: CI sets it in one line, and a build that would otherwise depend on
+// the registry being up cannot silently start doing so again.
+const OfflineEnv = "PORTOLAN_OFFLINE"
+
+func offline() bool {
+	if strings.TrimSpace(os.Getenv(OfflineEnv)) != "" {
+		return true
+	}
+
+	// CI verifies what was committed. It never needs to reach the registry, and
+	// a fork's pull request has no secret to reach it with.
+	switch strings.ToLower(strings.TrimSpace(os.Getenv("CI"))) {
+	case "", "0", "false":
+		return false
+	default:
+		return true
+	}
+}

package/plugins/fetch-bsr/bsr.go

@@ -0,0 +1,240 @@
+package main
+
+// The only file that knows the registry's wire.
+//
+// The BSR's public API is Connect, and a Connect unary call is an ordinary
+// HTTP POST carrying the request message as JSON - so this needs net/http and
+// encoding/json and nothing else. No grpc dependency, no protobuf runtime, in
+// a go.mod that has two narrow dependencies.
+//
+// The message shapes below are buf/registry/module/v1, and the JSON names are
+// protojson's: lowerCamelCase, `bytes` as base64. Everything the wire knows is
+// in this file on purpose - if buf changes the shape, or the older
+// buf.alpha.registry.v1alpha1 endpoints have to be fallen back to, this is the
+// only file that moves.
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+const (
+	downloadMethod = "/buf.registry.module.v1.DownloadService/Download"
+	commitsMethod  = "/buf.registry.module.v1.CommitService/GetCommits"
+
+	// The Connect protocol asks a client to say which version it speaks.
+	connectVersionHeader = "Connect-Protocol-Version"
+	connectVersion       = "1"
+)
+
+// resourceRef names a module and which version of it is wanted.
+type resourceRef struct {
+	Name *refName `json:"name,omitempty"`
+	ID   string   `json:"id,omitempty"`
+}
+
+type refName struct {
+	Owner  string `json:"owner"`
+	Module string `json:"module"`
+	// Ref is untyped: a commit id, a label, a tag. A pinned module always sends
+	// a commit id, which is the only form that makes a download reproducible.
+	Ref string `json:"ref,omitempty"`
+}
+
+type downloadRequest struct {
+	Values []downloadValue `json:"values"`
+}
+
+type downloadValue struct {
+	ResourceRef resourceRef `json:"resourceRef"`
+	// Only .proto: the docs and licence a module also carries are not something
+	// portolan reads, and fetching them would put files in the tree that
+	// nothing explains.
+	FileTypes []string `json:"fileTypes,omitempty"`
+	Paths     []string `json:"paths,omitempty"`
+	// Without this, narrowing to a path the module does not have is an error
+	// rather than an empty answer.
+	PathsAllowNotExist bool `json:"pathsAllowNotExist,omitempty"`
+}
+
+type downloadResponse struct {
+	Contents []struct {
+		Commit wireCommit `json:"commit"`
+		Files  []struct {
+			Path string `json:"path"`
+			// bytes on the wire, so base64 in JSON.
+			Content []byte `json:"content"`
+		} `json:"files"`
+	} `json:"contents"`
+}
+
+type wireCommit struct {
+	ID     string `json:"id"`
+	Digest struct {
+		Type  string `json:"type"`
+		Value []byte `json:"value"`
+	} `json:"digest"`
+}
+
+// display renders a digest the way buf writes one: `b5:<hex>`.
+func (c wireCommit) display() string {
+	if len(c.Digest.Value) == 0 {
+		return ""
+	}
+	kind := strings.ToLower(strings.TrimPrefix(c.Digest.Type, "DIGEST_TYPE_"))
+	if kind == "" || kind == "unspecified" {
+		kind = "b5"
+	}
+
+	return kind + ":" + hex.EncodeToString(c.Digest.Value)
+}
+
+type getCommitsRequest struct {
+	ResourceRefs []resourceRef `json:"resourceRefs"`
+}
+
+type getCommitsResponse struct {
+	Commits []wireCommit `json:"commits"`
+}
+
+// client talks to one registry.
+type client struct {
+	http  *http.Client
+	base  string
+	token string
+}
+
+func newClient(registry, token string) *client {
+	return &client{
+		// A timeout rather than none: a plugin that hangs holds up the whole
+		// generate run, and the host has no way to interrupt it.
+		http:  &http.Client{Timeout: 60 * time.Second},
+		base:  baseURL(registry),
+		token: token,
+	}
+}
+
+// baseURL lets a registry be given with a scheme.
+//
+// buf.build is https and needs no help. A self-hosted BSR behind a plain-http
+// internal address does, and so does a test server - which is the same need,
+// and worth meeting once rather than threading a seam through the plugin.
+func baseURL(registry string) string {
+	if strings.HasPrefix(registry, "http://") || strings.HasPrefix(registry, "https://") {
+		return strings.TrimSuffix(registry, "/")
+	}
+
+	return "https://" + registry
+}
+
+// download fetches a module's .proto files at a given ref.
+func (c *client) download(owner, module, ref string, paths []string) (wireCommit, map[string][]byte, error) {
+	req := downloadRequest{Values: []downloadValue{{
+		ResourceRef:        resourceRef{Name: &refName{Owner: owner, Module: module, Ref: ref}},
+		FileTypes:          []string{"FILE_TYPE_PROTO"},
+		Paths:              paths,
+		PathsAllowNotExist: len(paths) > 0,
+	}}}
+
+	var resp downloadResponse
+	if err := c.call(downloadMethod, req, &resp); err != nil {
+		return wireCommit{}, nil, err
+	}
+	if len(resp.Contents) == 0 {
+		return wireCommit{}, nil, fmt.Errorf("%s/%s: the registry returned no content", owner, module)
+	}
+
+	content := resp.Contents[0]
+	files := make(map[string][]byte, len(content.Files))
+	for _, f := range content.Files {
+		files[f.Path] = f.Content
+	}
+
+	return content.Commit, files, nil
+}
+
+// resolve turns a label into the commit it currently points at.
+func (c *client) resolve(owner, module, ref string) (wireCommit, error) {
+	req := getCommitsRequest{ResourceRefs: []resourceRef{
+		{Name: &refName{Owner: owner, Module: module, Ref: ref}},
+	}}
+
+	var resp getCommitsResponse
+	if err := c.call(commitsMethod, req, &resp); err != nil {
+		return wireCommit{}, err
+	}
+	if len(resp.Commits) == 0 {
+		return wireCommit{}, fmt.Errorf("%s/%s: no commit for %q", owner, module, ref)
+	}
+
+	return resp.Commits[0], nil
+}
+
+func (c *client) call(method string, in, out any) error {
+	body, err := json.Marshal(in)
+	if err != nil {
+		return err
+	}
+
+	url := c.base + method
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set(connectVersionHeader, connectVersion)
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	}
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, 256<<20))
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s: %s", method, connectError(resp.StatusCode, raw))
+	}
+
+	return json.Unmarshal(raw, out)
+}
+
+// connectError reads the error body Connect sends, falling back to the status
+// when the body is not one.
+func connectError(status int, body []byte) string {
+	var wire struct {
+		Code    string `json:"code"`
+		Message string `json:"message"`
+	}
+	if err := json.Unmarshal(body, &wire); err == nil && wire.Message != "" {
+		if wire.Code != "" {
+			return wire.Code + ": " + wire.Message
+		}
+
+		return wire.Message
+	}
+
+	return fmt.Sprintf("http %d", status)
+}
+
+// splitModule breaks `buf.build/acme/shop` into its three parts.
+func splitModule(name string) (registry, owner, module string, err error) {
+	parts := strings.Split(strings.TrimSpace(name), "/")
+	if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" {
+		return "", "", "", fmt.Errorf("%q is not a module name; expected <registry>/<owner>/<module>", name)
+	}
+
+	return parts[0], parts[1], parts[2], nil
+}

package/plugins/fetch-bsr/cache.go

@@ -0,0 +1,66 @@
+package main
+
+// Replaying a fetch from what is already on disk.
+//
+// The cache is not a second copy of anything: it IS the tree. The protos this
+// plugin fetched were written by the host through the ordinary Response.Files
+// path, so they are committed, reviewable, and GC'd when the step stops naming
+// them. Reading them back and checking them against the lock is what lets a
+// build that never opens a socket produce byte-identical output.
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// cached is one module's files, read back from the tree.
+type cached struct {
+	lock  LockModule
+	files map[string][]byte
+}
+
+// replay reads a module's committed copy and checks it against its lock.
+//
+// An incomplete or edited cache is reported rather than patched over. The two
+// failures it distinguishes matter: nothing on disk means there is nothing to
+// fall back to, and bytes that no longer match their digest mean somebody
+// edited a vendored copy - the drift docs/adr/org.0001.md wants visible.
+func replay(dir string) (*cached, error) {
+	raw, err := os.ReadFile(filepath.Join(dir, LockName))
+	if os.IsNotExist(err) {
+		return nil, fmt.Errorf("no %s in %s", LockName, filepath.ToSlash(dir))
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var lock Lock
+	if err := jsonUnmarshal(raw, &lock); err != nil {
+		return nil, fmt.Errorf("%s: %w", filepath.ToSlash(filepath.Join(dir, LockName)), err)
+	}
+	if len(lock.Modules) != 1 {
+		return nil, fmt.Errorf("%s names %d modules; expected exactly one",
+			filepath.ToSlash(filepath.Join(dir, LockName)), len(lock.Modules))
+	}
+
+	entry := lock.Modules[0]
+	files := make(map[string][]byte, len(entry.Files))
+
+	for _, want := range entry.Files {
+		at := filepath.Join(dir, filepath.FromSlash(want.Path))
+		content, err := os.ReadFile(at)
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("%s is in the lock but not on disk", want.Path)
+		}
+		if err != nil {
+			return nil, err
+		}
+		if got := digestOf(content); got != want.SHA256 {
+			return nil, fmt.Errorf("%s does not match its digest; the vendored copy was edited by hand", want.Path)
+		}
+		files[want.Path] = content
+	}
+
+	return &cached{lock: entry, files: files}, nil
+}

package/plugins/fetch-bsr/describe.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	_ "embed"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+//go:embed options.schema.json
+var optionsSchema []byte
+
+func descriptor() plugin.Descriptor {
+	return plugin.Descriptor{
+		Name:    "fetch-bsr",
+		Summary: "Fetches pinned modules from a Buf Schema Registry into the tree, with a lock beside each, so the proto extractor can read a published contract.",
+		Phases:  []string{plugin.PhaseExtract},
+		Options: optionsSchema,
+	}
+}

package/plugins/fetch-bsr/fetch.go

@@ -0,0 +1,192 @@
+package main
+
+// Deciding whether to fetch, and what to do when the fetch cannot happen.
+//
+// The rules below are the design, not error handling. The one that matters
+// most is the third: a failed fetch with no cache is a NON-ZERO EXIT, never a
+// short file list. The host deletes files a step stops naming, and dropping a
+// repository's vendored protos because a laptop went offline is worse than a
+// red build.
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"path/filepath"
+	"sort"
+
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+func jsonUnmarshal(raw []byte, into any) error { return json.Unmarshal(raw, into) }
+
+// fetch takes no plugin.Input: it reads nothing out of the tree being
+// described. Its `cache` is repo-relative, like every other path in the
+// manifest, and everything else it needs is in the options.
+func fetch(opts Options) (plugin.Response, error) {
+	b := &plugin.Builder{}
+
+	if len(opts.Modules) == 0 {
+		return plugin.Response{}, fmt.Errorf("no modules to fetch: name at least one in the manifest")
+	}
+	if opts.Cache == "" {
+		return plugin.Response{}, fmt.Errorf(
+			"no cache directory: set `cache` to the same path as the step's `out`, " +
+				"so an offline run can replay what the last online one wrote")
+	}
+
+	// Sorted, so the file list and the diagnostics come out the same way every
+	// run whatever order the manifest happens to list them in.
+	wanted := append([]Module(nil), opts.Modules...)
+	sort.Slice(wanted, func(i, j int) bool { return wanted[i].Module < wanted[j].Module })
+
+	skip := offline()
+
+	for _, want := range wanted {
+		registry, owner, module, err := splitModule(want.Module)
+		if err != nil {
+			return plugin.Response{}, err
+		}
+		if opts.Registry != "" {
+			registry = opts.Registry
+		}
+
+		// Every module lives in its own directory with its own lock, so a
+		// vendored module is self-describing and extract-proto finds the lock
+		// by looking beside the files it is already reading.
+		dir := path.Join(owner, module)
+		at := filepath.Join(filepath.FromSlash(opts.Cache), filepath.FromSlash(dir))
+
+		if skip {
+			if err := emitCached(b, dir, at, want, "offline"); err != nil {
+				return plugin.Response{}, err
+			}
+
+			continue
+		}
+
+		commit, files, err := live(registry, owner, module, want, b)
+		if err != nil {
+			// Rule 2: the tree still holds a good copy, so the output is
+			// unchanged and `--check` stays clean.
+			if cacheErr := emitCached(b, dir, at, want, err.Error()); cacheErr != nil {
+				// Rule 3: nothing to fall back to.
+				return plugin.Response{}, fmt.Errorf(
+					"%s could not be fetched (%w) and there is no usable copy in the tree (%v)",
+					want.Module, err, cacheErr)
+			}
+
+			continue
+		}
+
+		if err := emitFetched(b, dir, want.Module, commit, files); err != nil {
+			return plugin.Response{}, err
+		}
+	}
+
+	return b.Response(), nil
+}
+
+// live fetches one module, pinning it first if the manifest did not.
+func live(registry, owner, module string, want Module, b *plugin.Builder) (wireCommit, map[string][]byte, error) {
+	c := newClient(registry, token(registry))
+
+	ref := want.Commit
+	if ref == "" {
+		// Rule: pin, or every run is a lottery. Resolving a label still works,
+		// but it is said out loud, because two runs a day apart would then
+		// produce two different trees from one manifest.
+		ref = firstNonEmpty(want.Ref, "main")
+		resolved, err := c.resolve(owner, module, ref)
+		if err != nil {
+			return wireCommit{}, nil, err
+		}
+		b.Warn(want.Module, fmt.Sprintf(
+			"is not pinned; %q resolved to %s. Pin it in portolan.json or every run is a lottery.",
+			ref, resolved.ID))
+		ref = resolved.ID
+	}
+
+	commit, files, err := c.download(owner, module, ref, want.Paths)
+	if err != nil {
+		return wireCommit{}, nil, err
+	}
+	if len(files) == 0 {
+		return wireCommit{}, nil, fmt.Errorf("%s at %s holds no .proto files", want.Module, ref)
+	}
+
+	return commit, files, nil
+}
+
+// emitFetched writes what came back, plus the lock that makes it checkable.
+func emitFetched(b *plugin.Builder, dir, name string, commit wireCommit, files map[string][]byte) error {
+	entry := LockModule{Module: name, Commit: commit.ID, Digest: commit.display()}
+
+	paths := make([]string, 0, len(files))
+	for at := range files {
+		paths = append(paths, at)
+	}
+	sort.Strings(paths)
+
+	for _, at := range paths {
+		content := files[at]
+		b.File(path.Join(dir, at), string(content))
+		entry.Files = append(entry.Files, LockEntry{
+			Path:   at,
+			SHA256: digestOf(content),
+			Size:   len(content),
+		})
+	}
+
+	lock, err := Lock{Modules: []LockModule{entry}}.encode()
+	if err != nil {
+		return err
+	}
+	b.File(path.Join(dir, LockName), lock)
+
+	return nil
+}
+
+// emitCached re-emits the committed copy unchanged.
+//
+// Byte-identical to what the last online run wrote, which is the whole point:
+// `gen:check` sees no drift, so CI can verify an estate whose schemas live in a
+// registry it never talks to.
+func emitCached(b *plugin.Builder, dir, at string, want Module, why string) error {
+	if want.Commit == "" {
+		return fmt.Errorf("%s is not pinned to a commit, so there is nothing to replay", want.Module)
+	}
+
+	held, err := replay(at)
+	if err != nil {
+		return err
+	}
+	if held.lock.Commit != want.Commit {
+		return fmt.Errorf("%s holds commit %s but the manifest pins %s; fetch it",
+			want.Module, held.lock.Commit, want.Commit)
+	}
+
+	for _, entry := range held.lock.Files {
+		b.File(path.Join(dir, entry.Path), string(held.files[entry.Path]))
+	}
+
+	lock, err := Lock{Modules: []LockModule{held.lock}}.encode()
+	if err != nil {
+		return err
+	}
+	b.File(path.Join(dir, LockName), lock)
+
+	b.Warn(want.Module, "not fetched ("+why+"); the copy committed in this repository is used unchanged")
+
+	return nil
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if v != "" {
+			return v
+		}
+	}
+
+	return ""
+}