@shortlink-org/portolan 0.1.0 → 0.2.0

package/plugins/fetch-bsr/bsr.go

@@ -1,240 +0,0 @@
-package main
-
-// The only file that knows the registry's wire.
-//
-// The BSR's public API is Connect, and a Connect unary call is an ordinary
-// HTTP POST carrying the request message as JSON - so this needs net/http and
-// encoding/json and nothing else. No grpc dependency, no protobuf runtime, in
-// a go.mod that has two narrow dependencies.
-//
-// The message shapes below are buf/registry/module/v1, and the JSON names are
-// protojson's: lowerCamelCase, `bytes` as base64. Everything the wire knows is
-// in this file on purpose - if buf changes the shape, or the older
-// buf.alpha.registry.v1alpha1 endpoints have to be fallen back to, this is the
-// only file that moves.
-
-import (
-	"bytes"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-	"time"
-)
-
-const (
-	downloadMethod = "/buf.registry.module.v1.DownloadService/Download"
-	commitsMethod  = "/buf.registry.module.v1.CommitService/GetCommits"
-
-	// The Connect protocol asks a client to say which version it speaks.
-	connectVersionHeader = "Connect-Protocol-Version"
-	connectVersion       = "1"
-)
-
-// resourceRef names a module and which version of it is wanted.
-type resourceRef struct {
-	Name *refName `json:"name,omitempty"`
-	ID   string   `json:"id,omitempty"`
-}
-
-type refName struct {
-	Owner  string `json:"owner"`
-	Module string `json:"module"`
-	// Ref is untyped: a commit id, a label, a tag. A pinned module always sends
-	// a commit id, which is the only form that makes a download reproducible.
-	Ref string `json:"ref,omitempty"`
-}
-
-type downloadRequest struct {
-	Values []downloadValue `json:"values"`
-}
-
-type downloadValue struct {
-	ResourceRef resourceRef `json:"resourceRef"`
-	// Only .proto: the docs and licence a module also carries are not something
-	// portolan reads, and fetching them would put files in the tree that
-	// nothing explains.
-	FileTypes []string `json:"fileTypes,omitempty"`
-	Paths     []string `json:"paths,omitempty"`
-	// Without this, narrowing to a path the module does not have is an error
-	// rather than an empty answer.
-	PathsAllowNotExist bool `json:"pathsAllowNotExist,omitempty"`
-}
-
-type downloadResponse struct {
-	Contents []struct {
-		Commit wireCommit `json:"commit"`
-		Files  []struct {
-			Path string `json:"path"`
-			// bytes on the wire, so base64 in JSON.
-			Content []byte `json:"content"`
-		} `json:"files"`
-	} `json:"contents"`
-}
-
-type wireCommit struct {
-	ID     string `json:"id"`
-	Digest struct {
-		Type  string `json:"type"`
-		Value []byte `json:"value"`
-	} `json:"digest"`
-}
-
-// display renders a digest the way buf writes one: `b5:<hex>`.
-func (c wireCommit) display() string {
-	if len(c.Digest.Value) == 0 {
-		return ""
-	}
-	kind := strings.ToLower(strings.TrimPrefix(c.Digest.Type, "DIGEST_TYPE_"))
-	if kind == "" || kind == "unspecified" {
-		kind = "b5"
-	}
-
-	return kind + ":" + hex.EncodeToString(c.Digest.Value)
-}
-
-type getCommitsRequest struct {
-	ResourceRefs []resourceRef `json:"resourceRefs"`
-}
-
-type getCommitsResponse struct {
-	Commits []wireCommit `json:"commits"`
-}
-
-// client talks to one registry.
-type client struct {
-	http  *http.Client
-	base  string
-	token string
-}
-
-func newClient(registry, token string) *client {
-	return &client{
-		// A timeout rather than none: a plugin that hangs holds up the whole
-		// generate run, and the host has no way to interrupt it.
-		http:  &http.Client{Timeout: 60 * time.Second},
-		base:  baseURL(registry),
-		token: token,
-	}
-}
-
-// baseURL lets a registry be given with a scheme.
-//
-// buf.build is https and needs no help. A self-hosted BSR behind a plain-http
-// internal address does, and so does a test server - which is the same need,
-// and worth meeting once rather than threading a seam through the plugin.
-func baseURL(registry string) string {
-	if strings.HasPrefix(registry, "http://") || strings.HasPrefix(registry, "https://") {
-		return strings.TrimSuffix(registry, "/")
-	}
-
-	return "https://" + registry
-}
-
-// download fetches a module's .proto files at a given ref.
-func (c *client) download(owner, module, ref string, paths []string) (wireCommit, map[string][]byte, error) {
-	req := downloadRequest{Values: []downloadValue{{
-		ResourceRef:        resourceRef{Name: &refName{Owner: owner, Module: module, Ref: ref}},
-		FileTypes:          []string{"FILE_TYPE_PROTO"},
-		Paths:              paths,
-		PathsAllowNotExist: len(paths) > 0,
-	}}}
-
-	var resp downloadResponse
-	if err := c.call(downloadMethod, req, &resp); err != nil {
-		return wireCommit{}, nil, err
-	}
-	if len(resp.Contents) == 0 {
-		return wireCommit{}, nil, fmt.Errorf("%s/%s: the registry returned no content", owner, module)
-	}
-
-	content := resp.Contents[0]
-	files := make(map[string][]byte, len(content.Files))
-	for _, f := range content.Files {
-		files[f.Path] = f.Content
-	}
-
-	return content.Commit, files, nil
-}
-
-// resolve turns a label into the commit it currently points at.
-func (c *client) resolve(owner, module, ref string) (wireCommit, error) {
-	req := getCommitsRequest{ResourceRefs: []resourceRef{
-		{Name: &refName{Owner: owner, Module: module, Ref: ref}},
-	}}
-
-	var resp getCommitsResponse
-	if err := c.call(commitsMethod, req, &resp); err != nil {
-		return wireCommit{}, err
-	}
-	if len(resp.Commits) == 0 {
-		return wireCommit{}, fmt.Errorf("%s/%s: no commit for %q", owner, module, ref)
-	}
-
-	return resp.Commits[0], nil
-}
-
-func (c *client) call(method string, in, out any) error {
-	body, err := json.Marshal(in)
-	if err != nil {
-		return err
-	}
-
-	url := c.base + method
-	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set(connectVersionHeader, connectVersion)
-	if c.token != "" {
-		req.Header.Set("Authorization", "Bearer "+c.token)
-	}
-
-	resp, err := c.http.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	raw, err := io.ReadAll(io.LimitReader(resp.Body, 256<<20))
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("%s: %s", method, connectError(resp.StatusCode, raw))
-	}
-
-	return json.Unmarshal(raw, out)
-}
-
-// connectError reads the error body Connect sends, falling back to the status
-// when the body is not one.
-func connectError(status int, body []byte) string {
-	var wire struct {
-		Code    string `json:"code"`
-		Message string `json:"message"`
-	}
-	if err := json.Unmarshal(body, &wire); err == nil && wire.Message != "" {
-		if wire.Code != "" {
-			return wire.Code + ": " + wire.Message
-		}
-
-		return wire.Message
-	}
-
-	return fmt.Sprintf("http %d", status)
-}
-
-// splitModule breaks `buf.build/acme/shop` into its three parts.
-func splitModule(name string) (registry, owner, module string, err error) {
-	parts := strings.Split(strings.TrimSpace(name), "/")
-	if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" {
-		return "", "", "", fmt.Errorf("%q is not a module name; expected <registry>/<owner>/<module>", name)
-	}
-
-	return parts[0], parts[1], parts[2], nil
-}

package/plugins/fetch-bsr/cache.go

@@ -1,66 +0,0 @@
-package main
-
-// Replaying a fetch from what is already on disk.
-//
-// The cache is not a second copy of anything: it IS the tree. The protos this
-// plugin fetched were written by the host through the ordinary Response.Files
-// path, so they are committed, reviewable, and GC'd when the step stops naming
-// them. Reading them back and checking them against the lock is what lets a
-// build that never opens a socket produce byte-identical output.
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-)
-
-// cached is one module's files, read back from the tree.
-type cached struct {
-	lock  LockModule
-	files map[string][]byte
-}
-
-// replay reads a module's committed copy and checks it against its lock.
-//
-// An incomplete or edited cache is reported rather than patched over. The two
-// failures it distinguishes matter: nothing on disk means there is nothing to
-// fall back to, and bytes that no longer match their digest mean somebody
-// edited a vendored copy - the drift docs/adr/org.0001.md wants visible.
-func replay(dir string) (*cached, error) {
-	raw, err := os.ReadFile(filepath.Join(dir, LockName))
-	if os.IsNotExist(err) {
-		return nil, fmt.Errorf("no %s in %s", LockName, filepath.ToSlash(dir))
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	var lock Lock
-	if err := jsonUnmarshal(raw, &lock); err != nil {
-		return nil, fmt.Errorf("%s: %w", filepath.ToSlash(filepath.Join(dir, LockName)), err)
-	}
-	if len(lock.Modules) != 1 {
-		return nil, fmt.Errorf("%s names %d modules; expected exactly one",
-			filepath.ToSlash(filepath.Join(dir, LockName)), len(lock.Modules))
-	}
-
-	entry := lock.Modules[0]
-	files := make(map[string][]byte, len(entry.Files))
-
-	for _, want := range entry.Files {
-		at := filepath.Join(dir, filepath.FromSlash(want.Path))
-		content, err := os.ReadFile(at)
-		if os.IsNotExist(err) {
-			return nil, fmt.Errorf("%s is in the lock but not on disk", want.Path)
-		}
-		if err != nil {
-			return nil, err
-		}
-		if got := digestOf(content); got != want.SHA256 {
-			return nil, fmt.Errorf("%s does not match its digest; the vendored copy was edited by hand", want.Path)
-		}
-		files[want.Path] = content
-	}
-
-	return &cached{lock: entry, files: files}, nil
-}

package/plugins/fetch-bsr/describe.go

@@ -1,19 +0,0 @@
-package main
-
-import (
-	_ "embed"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-//go:embed options.schema.json
-var optionsSchema []byte
-
-func descriptor() plugin.Descriptor {
-	return plugin.Descriptor{
-		Name:    "fetch-bsr",
-		Summary: "Fetches pinned modules from a Buf Schema Registry into the tree, with a lock beside each, so the proto extractor can read a published contract.",
-		Phases:  []string{plugin.PhaseExtract},
-		Options: optionsSchema,
-	}
-}

package/plugins/fetch-bsr/fetch.go

@@ -1,192 +0,0 @@
-package main
-
-// Deciding whether to fetch, and what to do when the fetch cannot happen.
-//
-// The rules below are the design, not error handling. The one that matters
-// most is the third: a failed fetch with no cache is a NON-ZERO EXIT, never a
-// short file list. The host deletes files a step stops naming, and dropping a
-// repository's vendored protos because a laptop went offline is worse than a
-// red build.
-
-import (
-	"encoding/json"
-	"fmt"
-	"path"
-	"path/filepath"
-	"sort"
-
-	"github.com/shortlink-org/portolan/plugin"
-)
-
-func jsonUnmarshal(raw []byte, into any) error { return json.Unmarshal(raw, into) }
-
-// fetch takes no plugin.Input: it reads nothing out of the tree being
-// described. Its `cache` is repo-relative, like every other path in the
-// manifest, and everything else it needs is in the options.
-func fetch(opts Options) (plugin.Response, error) {
-	b := &plugin.Builder{}
-
-	if len(opts.Modules) == 0 {
-		return plugin.Response{}, fmt.Errorf("no modules to fetch: name at least one in the manifest")
-	}
-	if opts.Cache == "" {
-		return plugin.Response{}, fmt.Errorf(
-			"no cache directory: set `cache` to the same path as the step's `out`, " +
-				"so an offline run can replay what the last online one wrote")
-	}
-
-	// Sorted, so the file list and the diagnostics come out the same way every
-	// run whatever order the manifest happens to list them in.
-	wanted := append([]Module(nil), opts.Modules...)
-	sort.Slice(wanted, func(i, j int) bool { return wanted[i].Module < wanted[j].Module })
-
-	skip := offline()
-
-	for _, want := range wanted {
-		registry, owner, module, err := splitModule(want.Module)
-		if err != nil {
-			return plugin.Response{}, err
-		}
-		if opts.Registry != "" {
-			registry = opts.Registry
-		}
-
-		// Every module lives in its own directory with its own lock, so a
-		// vendored module is self-describing and extract-proto finds the lock
-		// by looking beside the files it is already reading.
-		dir := path.Join(owner, module)
-		at := filepath.Join(filepath.FromSlash(opts.Cache), filepath.FromSlash(dir))
-
-		if skip {
-			if err := emitCached(b, dir, at, want, "offline"); err != nil {
-				return plugin.Response{}, err
-			}
-
-			continue
-		}
-
-		commit, files, err := live(registry, owner, module, want, b)
-		if err != nil {
-			// Rule 2: the tree still holds a good copy, so the output is
-			// unchanged and `--check` stays clean.
-			if cacheErr := emitCached(b, dir, at, want, err.Error()); cacheErr != nil {
-				// Rule 3: nothing to fall back to.
-				return plugin.Response{}, fmt.Errorf(
-					"%s could not be fetched (%w) and there is no usable copy in the tree (%v)",
-					want.Module, err, cacheErr)
-			}
-
-			continue
-		}
-
-		if err := emitFetched(b, dir, want.Module, commit, files); err != nil {
-			return plugin.Response{}, err
-		}
-	}
-
-	return b.Response(), nil
-}
-
-// live fetches one module, pinning it first if the manifest did not.
-func live(registry, owner, module string, want Module, b *plugin.Builder) (wireCommit, map[string][]byte, error) {
-	c := newClient(registry, token(registry))
-
-	ref := want.Commit
-	if ref == "" {
-		// Rule: pin, or every run is a lottery. Resolving a label still works,
-		// but it is said out loud, because two runs a day apart would then
-		// produce two different trees from one manifest.
-		ref = firstNonEmpty(want.Ref, "main")
-		resolved, err := c.resolve(owner, module, ref)
-		if err != nil {
-			return wireCommit{}, nil, err
-		}
-		b.Warn(want.Module, fmt.Sprintf(
-			"is not pinned; %q resolved to %s. Pin it in portolan.json or every run is a lottery.",
-			ref, resolved.ID))
-		ref = resolved.ID
-	}
-
-	commit, files, err := c.download(owner, module, ref, want.Paths)
-	if err != nil {
-		return wireCommit{}, nil, err
-	}
-	if len(files) == 0 {
-		return wireCommit{}, nil, fmt.Errorf("%s at %s holds no .proto files", want.Module, ref)
-	}
-
-	return commit, files, nil
-}
-
-// emitFetched writes what came back, plus the lock that makes it checkable.
-func emitFetched(b *plugin.Builder, dir, name string, commit wireCommit, files map[string][]byte) error {
-	entry := LockModule{Module: name, Commit: commit.ID, Digest: commit.display()}
-
-	paths := make([]string, 0, len(files))
-	for at := range files {
-		paths = append(paths, at)
-	}
-	sort.Strings(paths)
-
-	for _, at := range paths {
-		content := files[at]
-		b.File(path.Join(dir, at), string(content))
-		entry.Files = append(entry.Files, LockEntry{
-			Path:   at,
-			SHA256: digestOf(content),
-			Size:   len(content),
-		})
-	}
-
-	lock, err := Lock{Modules: []LockModule{entry}}.encode()
-	if err != nil {
-		return err
-	}
-	b.File(path.Join(dir, LockName), lock)
-
-	return nil
-}
-
-// emitCached re-emits the committed copy unchanged.
-//
-// Byte-identical to what the last online run wrote, which is the whole point:
-// `gen:check` sees no drift, so CI can verify an estate whose schemas live in a
-// registry it never talks to.
-func emitCached(b *plugin.Builder, dir, at string, want Module, why string) error {
-	if want.Commit == "" {
-		return fmt.Errorf("%s is not pinned to a commit, so there is nothing to replay", want.Module)
-	}
-
-	held, err := replay(at)
-	if err != nil {
-		return err
-	}
-	if held.lock.Commit != want.Commit {
-		return fmt.Errorf("%s holds commit %s but the manifest pins %s; fetch it",
-			want.Module, held.lock.Commit, want.Commit)
-	}
-
-	for _, entry := range held.lock.Files {
-		b.File(path.Join(dir, entry.Path), string(held.files[entry.Path]))
-	}
-
-	lock, err := Lock{Modules: []LockModule{held.lock}}.encode()
-	if err != nil {
-		return err
-	}
-	b.File(path.Join(dir, LockName), lock)
-
-	b.Warn(want.Module, "not fetched ("+why+"); the copy committed in this repository is used unchanged")
-
-	return nil
-}
-
-func firstNonEmpty(values ...string) string {
-	for _, v := range values {
-		if v != "" {
-			return v
-		}
-	}
-
-	return ""
-}