@shortlink-org/portolan 0.1.0

package/plugins/verify-otel/match.go

@@ -0,0 +1,336 @@
+package main
+
+import (
+	"strings"
+
+	"github.com/shortlink-org/portolan/catalog"
+)
+
+// lookup is the catalog, indexed for the questions a span raises: whose span
+// is this, which operation answers on this route, which event travels under
+// this name, who provides this rpc, where does this service keep its state.
+type lookup struct {
+	opts Options
+
+	contexts   map[string]*catalog.BoundedContext
+	services   map[string]*catalog.Service
+	bySlug     map[string]*catalog.Service // unique slugs only
+	events     map[string]*catalog.Event
+	eventOwner map[string]*catalog.Service
+	aggregate  map[string]*catalog.Aggregate // by event id
+	byName     map[string][]string           // event name -> ids
+	byWire     map[string][]string           // declared wire name -> ids
+	providers  map[string]*catalog.Service   // interface id -> provider
+	rpcIDs     map[string]bool
+	stores     map[string][]*catalog.Store // owner -> stores
+	openings   map[string]*catalog.Flow    // opening key -> declared flow
+}
+
+func newLookup(cat *catalog.Catalog, opts Options) *lookup {
+	l := &lookup{
+		opts:       opts,
+		contexts:   map[string]*catalog.BoundedContext{},
+		services:   map[string]*catalog.Service{},
+		bySlug:     map[string]*catalog.Service{},
+		events:     map[string]*catalog.Event{},
+		eventOwner: map[string]*catalog.Service{},
+		aggregate:  map[string]*catalog.Aggregate{},
+		byName:     map[string][]string{},
+		byWire:     map[string][]string{},
+		providers:  map[string]*catalog.Service{},
+		rpcIDs:     map[string]bool{},
+		stores:     map[string][]*catalog.Store{},
+		openings:   map[string]*catalog.Flow{},
+	}
+
+	slugs := map[string]int{}
+	for i := range cat.Contexts {
+		ctx := &cat.Contexts[i]
+		l.contexts[ctx.ID] = ctx
+		for j := range ctx.Services {
+			svc := &ctx.Services[j]
+			l.services[svc.ID] = svc
+			slugs[svc.Slug]++
+			l.bySlug[svc.Slug] = svc
+			for k := range svc.Provides {
+				provided := &svc.Provides[k]
+				l.providers[provided.ID] = svc
+				for _, method := range provided.Methods {
+					l.rpcIDs[provided.ID+"/"+method.Name] = true
+				}
+			}
+			for _, call := range svc.Consumes {
+				l.rpcIDs[call.ID] = true
+			}
+			for k := range svc.Aggregates {
+				agg := &svc.Aggregates[k]
+				for m := range agg.Events {
+					ev := &agg.Events[m]
+					l.events[ev.ID] = ev
+					l.eventOwner[ev.ID] = svc
+					l.aggregate[ev.ID] = agg
+					l.byName[ev.Name] = append(l.byName[ev.Name], ev.ID)
+					if ev.Wire != nil {
+						l.byWire[ev.Wire.Name] = append(l.byWire[ev.Wire.Name], ev.ID)
+					}
+				}
+			}
+		}
+	}
+	for slug, n := range slugs {
+		if n > 1 {
+			delete(l.bySlug, slug)
+		}
+	}
+	for i := range cat.Stores {
+		store := &cat.Stores[i]
+		l.stores[store.Owner] = append(l.stores[store.Owner], store)
+	}
+	for i := range cat.Flows {
+		flow := &cat.Flows[i]
+		if key := declaredOpening(flow); key != "" {
+			if _, taken := l.openings[key]; !taken {
+				l.openings[key] = flow
+			}
+		}
+	}
+
+	return l
+}
+
+// service reads a resource's service.name back to the catalog: the manifest's
+// word first, then the one service whose slug it is.
+func (l *lookup) service(name string) *catalog.Service {
+	if id, ok := l.opts.Services[name]; ok {
+		return l.services[id]
+	}
+	if svc, ok := l.services[name]; ok {
+		return svc
+	}
+
+	return l.bySlug[name]
+}
+
+// operation reads a server span's route back to the operation the service's
+// interface declares on it, by the verb and path template the OpenAPI
+// extractor recorded.
+func (l *lookup) operation(svc *catalog.Service, method, route string) (string, bool) {
+	for _, provided := range svc.Provides {
+		for _, m := range provided.Methods {
+			if m.HTTP != nil && m.HTTP.Method == method && samePath(m.HTTP.Path, route) {
+				return m.Name, true
+			}
+		}
+	}
+
+	return "", false
+}
+
+func samePath(a, b string) bool {
+	return shapeOf(a) == shapeOf(b)
+}
+
+// shapeOf replaces every segment that is a parameter - `{userId}` in a
+// document - or looks like a value where the document has one - with a
+// marker, so a request's concrete path matches its template by position.
+// isParam says whether a route segment stands for a value: `{userId}` as an
+// OpenAPI document writes it, `:userId` as Fastify and Express report the
+// route they matched.
+func isParam(seg string) bool {
+	return (strings.HasPrefix(seg, "{") && strings.HasSuffix(seg, "}")) || (len(seg) > 1 && seg[0] == ':')
+}
+
+func shapeOf(path string) string {
+	path = strings.TrimSuffix(path, "/")
+	segments := strings.Split(path, "/")
+	for i, seg := range segments {
+		if isParam(seg) {
+			segments[i] = "*"
+		}
+	}
+
+	return strings.Join(segments, "/")
+}
+
+// httpProvider finds the service whose interface answers on a verb and a
+// concrete path, matching a template's parameters to the path's segments.
+func (l *lookup) httpProvider(method, path string) (*catalog.Service, string, string) {
+	for _, svc := range l.services {
+		for _, provided := range svc.Provides {
+			for _, m := range provided.Methods {
+				if m.HTTP != nil && m.HTTP.Method == strings.ToUpper(method) && routeMatches(m.HTTP.Path, path) {
+					return svc, m.Name, provided.ID
+				}
+			}
+		}
+	}
+
+	return nil, "", ""
+}
+
+// routeMatches says whether a concrete path fits a template, segment by
+// segment, a `{param}` standing for any one segment.
+func routeMatches(template, path string) bool {
+	t := strings.Split(strings.TrimSuffix(template, "/"), "/")
+	p := strings.Split(strings.TrimSuffix(path, "/"), "/")
+	if len(t) != len(p) {
+		return false
+	}
+	for i := range t {
+		if isParam(t[i]) {
+			continue
+		}
+		if t[i] != p[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// event reads a wire name back to an event id. The manifest's word first;
+// then the one event that declares this very name as its wire; then, for a
+// producer, the one event of its own with that name as its last segment;
+// then, for a consumer of somebody else's event, the one event anywhere with
+// that name. Two events sharing a name is an ambiguity the manifest has to
+// settle.
+func (l *lookup) event(producer *catalog.Service, wire string) (string, bool) {
+	if id, ok := l.opts.Events[wire]; ok {
+		_, known := l.events[id]
+
+		return id, known
+	}
+	if ids := l.byWire[wire]; len(ids) == 1 {
+		return ids[0], true
+	}
+	name := wire
+	if i := strings.LastIndex(wire, "."); i >= 0 {
+		name = wire[i+1:]
+	}
+
+	candidates := l.byName[name]
+	if producer != nil {
+		var own []string
+		for _, id := range candidates {
+			if l.eventOwner[id] == producer {
+				own = append(own, id)
+			}
+		}
+		if len(own) == 1 {
+			return own[0], true
+		}
+		if len(own) > 1 {
+			return "", false
+		}
+	}
+	if len(candidates) == 1 {
+		return candidates[0], true
+	}
+
+	return "", false
+}
+
+// storeLane is where a service's database calls land: its one store, on the
+// lane the code extractor names it by, or the service's own lane when it has
+// none or several - saying so once beats inventing a database.
+func (l *lookup) storeLane(svc *catalog.Service) (catalog.Participant, bool) {
+	stores := l.stores[svc.ID]
+	if len(stores) != 1 {
+		return catalog.Participant{}, false
+	}
+	context := contextOf(svc.ID)
+
+	return catalog.Participant{
+		ID:      svc.Slug + "-" + stores[0].Slug,
+		Kind:    catalog.ParticipantStore,
+		Context: &context,
+	}, true
+}
+
+func (l *lookup) serviceLane(svc *catalog.Service) catalog.Participant {
+	context := contextOf(svc.ID)
+
+	return catalog.Participant{ID: svc.ID, Kind: catalog.ParticipantService, Context: &context}
+}
+
+func contextOf(serviceID string) string {
+	context, _, _ := strings.Cut(serviceID, ".")
+
+	return context
+}
+
+// declaredOpening is the key a flow can be recognised by from its first step:
+// the operation somebody calls in on, or the event that arrives. A flow that
+// opens any other way - a store call, a self-message - is not one a trace can
+// be matched to from its root, and gets no key.
+func declaredOpening(flow *catalog.Flow) string {
+	first := firstStep(flow.Steps)
+	if first == nil {
+		return ""
+	}
+	brokers := map[string]bool{}
+	for _, p := range flow.Participants {
+		if p.Kind == catalog.ParticipantBroker {
+			brokers[p.ID] = true
+		}
+	}
+	switch {
+	case first.Kind == catalog.StepRPC && first.Ref == "" && first.From != first.To:
+		return openingKey(catalog.StepRPC, first.To, first.Label)
+	case first.Kind == catalog.StepEvent && first.Ref != "" && brokers[first.From]:
+		return openingKey(catalog.StepEvent, first.To, first.Ref)
+	}
+
+	return ""
+}
+
+func openingKey(kind catalog.StepKind, service, what string) string {
+	return string(kind) + "|" + service + "|" + what
+}
+
+func firstStep(nodes catalog.FlowNodes) *catalog.Step {
+	for _, node := range nodes {
+		switch x := node.(type) {
+		case *catalog.Step:
+			return x
+		case *catalog.Parallel:
+			for _, branch := range x.Branches {
+				if s := firstStep(branch); s != nil {
+					return s
+				}
+			}
+		case *catalog.Alt:
+			for _, branch := range x.Branches {
+				if s := firstStep(branch.Steps); s != nil {
+					return s
+				}
+			}
+		case *catalog.Loop:
+			if s := firstStep(x.Steps); s != nil {
+				return s
+			}
+		}
+	}
+
+	return nil
+}
+
+// walkSteps visits every step, frames included, in flow order.
+func walkSteps(nodes catalog.FlowNodes, visit func(*catalog.Step)) {
+	for _, node := range nodes {
+		switch x := node.(type) {
+		case *catalog.Step:
+			visit(x)
+		case *catalog.Parallel:
+			for _, branch := range x.Branches {
+				walkSteps(branch, visit)
+			}
+		case *catalog.Alt:
+			for _, branch := range x.Branches {
+				walkSteps(branch.Steps, visit)
+			}
+		case *catalog.Loop:
+			walkSteps(x.Steps, visit)
+		}
+	}
+}

package/plugins/verify-otel/options.schema.json

@@ -0,0 +1,30 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["traces"],
+  "properties": {
+    "traces": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string" },
+      "description": "The recordings to read, as globs relative to the input root. OTLP JSON, one batch per file or one per line.",
+      "examples": [["telemetry/traces.jsonl"]]
+    },
+    "services": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Which catalog service a resource's service.name belongs to, for an estate where the two differ. Without a line here a name is matched to the one service whose slug it is.",
+      "examples": [{ "auth-api": "auth.auth" }]
+    },
+    "events": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Which catalog event a wire name belongs to, for an estate where the event does not declare its wire name and the last segment of the event.name attribute is not the event's name in the model.",
+      "examples": [{ "auth.PasswordChanged": "auth.auth.PasswordChanged" }]
+    },
+    "out": {
+      "type": "string",
+      "description": "Name of the fragment file, inside the step's output directory."
+    }
+  }
+}

package/plugins/verify-otel/otlp.go

@@ -0,0 +1,200 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// A span, flattened out of the OTLP shape into the few things a hop is read
+// from. Attribute values are kept as strings: everything matched on is a
+// name, and a number that mattered would be a number in a string here.
+type span struct {
+	traceID  string
+	spanID   string
+	parentID string
+	name     string
+	kind     int
+	service  string
+	start    uint64
+	attrs    map[string]string
+	file     string
+}
+
+const (
+	kindInternal = 1
+	kindServer   = 2
+	kindClient   = 3
+	kindProducer = 4
+	kindConsumer = 5
+)
+
+// readTraces reads every file the globs name under root, in path order, so
+// that the fragment does not depend on the order a directory happens to list.
+func readTraces(root string, patterns []string) ([]span, []string, error) {
+	var files []string
+	seen := map[string]bool{}
+	for _, pattern := range patterns {
+		matches, err := filepath.Glob(filepath.Join(root, pattern))
+		if err != nil {
+			return nil, nil, fmt.Errorf("traces %q: %w", pattern, err)
+		}
+		for _, match := range matches {
+			if !seen[match] {
+				seen[match] = true
+				files = append(files, match)
+			}
+		}
+	}
+	sort.Strings(files)
+
+	var spans []span
+	var names []string
+	for _, file := range files {
+		data, err := os.ReadFile(file)
+		if err != nil {
+			return nil, nil, err
+		}
+		rel, err := filepath.Rel(root, file)
+		if err != nil {
+			rel = file
+		}
+		rel = filepath.ToSlash(rel)
+		parsed, err := parseOTLP(data, rel)
+		if err != nil {
+			return nil, nil, fmt.Errorf("%s: %w", rel, err)
+		}
+		spans = append(spans, parsed...)
+		names = append(names, rel)
+	}
+
+	return spans, names, nil
+}
+
+type otlpBatch struct {
+	ResourceSpans []struct {
+		Resource struct {
+			Attributes []otlpAttr `json:"attributes"`
+		} `json:"resource"`
+		ScopeSpans []struct {
+			Spans []otlpSpan `json:"spans"`
+		} `json:"scopeSpans"`
+	} `json:"resourceSpans"`
+}
+
+type otlpSpan struct {
+	TraceID           string          `json:"traceId"`
+	SpanID            string          `json:"spanId"`
+	ParentSpanID      string          `json:"parentSpanId"`
+	Name              string          `json:"name"`
+	Kind              json.RawMessage `json:"kind"`
+	StartTimeUnixNano string          `json:"startTimeUnixNano"`
+	Attributes        []otlpAttr      `json:"attributes"`
+}
+
+type otlpAttr struct {
+	Key   string                     `json:"key"`
+	Value map[string]json.RawMessage `json:"value"`
+}
+
+// parseOTLP reads one JSON value, or one per line: a collector's file
+// exporter writes the second, an export saved from a viewer the first, and
+// the decoder does not care which.
+func parseOTLP(data []byte, file string) ([]span, error) {
+	dec := json.NewDecoder(bytes.NewReader(data))
+	var out []span
+	for {
+		var batch otlpBatch
+		err := dec.Decode(&batch)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		for _, rs := range batch.ResourceSpans {
+			service := attrs(rs.Resource.Attributes)["service.name"]
+			for _, ss := range rs.ScopeSpans {
+				for _, s := range ss.Spans {
+					start, _ := strconv.ParseUint(s.StartTimeUnixNano, 10, 64)
+					out = append(out, span{
+						traceID:  s.TraceID,
+						spanID:   s.SpanID,
+						parentID: s.ParentSpanID,
+						name:     s.Name,
+						kind:     spanKind(s.Kind),
+						service:  service,
+						start:    start,
+						attrs:    attrs(s.Attributes),
+						file:     file,
+					})
+				}
+			}
+		}
+	}
+
+	return out, nil
+}
+
+func attrs(list []otlpAttr) map[string]string {
+	out := map[string]string{}
+	for _, a := range list {
+		if v, ok := attrString(a.Value); ok {
+			out[a.Key] = v
+		}
+	}
+
+	return out
+}
+
+// attrString reads the one-of an OTLP value is: a string, or a number or bool
+// spelled as one. Arrays and maps are not names and are left out.
+func attrString(value map[string]json.RawMessage) (string, bool) {
+	for _, key := range []string{"stringValue", "intValue", "boolValue", "doubleValue"} {
+		raw, ok := value[key]
+		if !ok {
+			continue
+		}
+		var s string
+		if err := json.Unmarshal(raw, &s); err == nil {
+			return s, true
+		}
+		// A number or a bool, unquoted.
+		return strings.TrimSpace(string(raw)), true
+	}
+
+	return "", false
+}
+
+// spanKind reads the enum however it was written: protojson gives the number,
+// some exporters the name.
+func spanKind(raw json.RawMessage) int {
+	var n int
+	if err := json.Unmarshal(raw, &n); err == nil {
+		return n
+	}
+	var s string
+	if err := json.Unmarshal(raw, &s); err == nil {
+		switch strings.TrimPrefix(s, "SPAN_KIND_") {
+		case "INTERNAL":
+			return kindInternal
+		case "SERVER":
+			return kindServer
+		case "CLIENT":
+			return kindClient
+		case "PRODUCER":
+			return kindProducer
+		case "CONSUMER":
+			return kindConsumer
+		}
+	}
+
+	return 0
+}