@shortlink-org/portolan 0.1.0

package/plugins/extract-sql/projection_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+// The note names the table it stands above, and only that one. A bare slug is
+// an aggregate of the store's owner; a dotted name is somebody's full id.
+func TestAggregateNoteBindsToTheNextCreateTable(t *testing.T) {
+	got := readProjected(`
+-- Planned stops, rebuilt from RoutePlanned.
+-- aggregate: shop.oms.order
+CREATE TABLE route_stops (
+    route_id text NOT NULL
+);
+
+-- aggregate: price_list
+CREATE TABLE IF NOT EXISTS board.prices (
+    id text PRIMARY KEY
+);
+
+CREATE TABLE seen (
+    id text PRIMARY KEY
+);`, "shop.delivery")
+
+	want := map[string]string{
+		"route_stops": "shop.oms.order",
+		"prices":      "shop.delivery.price-list",
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("projected = %v, want %v", got, want)
+	}
+}
+
+// A note that a statement gets in front of was not about the table after it,
+// and a note nothing follows says nothing.
+func TestAggregateNoteDoesNotReachPastAStatement(t *testing.T) {
+	got := readProjected(`
+-- aggregate: shop.oms.order
+CREATE INDEX route_stops_by_route ON route_stops (route_id);
+CREATE TABLE route_stops_copy (
+    route_id text NOT NULL
+);
+-- aggregate: shop.oms.order`, "shop.delivery")
+
+	if len(got) != 0 {
+		t.Errorf("no table should carry the note, got %v", got)
+	}
+}

package/plugins/extract-sql/store.go

@@ -0,0 +1,420 @@
+package main
+
+import (
+	"go/parser"
+	"go/token"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/shortlink-org/portolan/catalog"
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// readStore builds one store out of every migration in the infrastructure
+// layer: the repository packages, and the projector packages beside them.
+//
+// One store, not one per aggregate: the migrations are numbered inside their
+// own package and each keeps its own schema_migrations table, but they are all
+// applied to the same database. The numbering is about who waits for whom, not
+// about where the rows live.
+func readStore(root string, layout storageLayout, storeID, owner string, b *plugin.Builder) ([]catalog.Table, []catalog.View) {
+	tables := []catalog.Table{}
+	views := []catalog.View{}
+
+	for _, repository := range layout.repositories {
+		aggregate := repository.name
+		dir := repository.migrations
+
+		state, copies, _, ok := readMigrations(root, dir, storeID, owner, b)
+		if !ok {
+			// A repository package with no migrations of its own is normal:
+			// not every adapter keeps rows.
+			continue
+		}
+
+		// The directory names the aggregate, and an id spells it the way every
+		// extractor spells one: price_list is price-list.
+		aggregateID := owner + "." + slug(aggregate)
+		// Which column carries which field, read from the statements that
+		// write the rows rather than from the column names.
+		mapped := readMaps(root, repository.dir, aggregate, b)
+		for _, more := range []map[string]map[string]string{
+			readMapsTS(root, repository.dir, aggregate, b),
+			readMapsRust(root, repository.dir, aggregate, b),
+			readMapsJava(root, repository.dir, aggregate, b),
+		} {
+			for table, columns := range more {
+				if _, ok := mapped[table]; !ok {
+					mapped[table] = columns
+				}
+			}
+		}
+
+		first := true
+		for _, relation := range state.relations {
+			table := finishTable(relation, state, storeID)
+			// An outbox holds messages on their way out, not the
+			// aggregate: it is created beside the aggregate because the
+			// repository writes both in one transaction, and that is all
+			// the layout says about it.
+			if isOutbox(lastNamePart(table.Name)) {
+				table.Role = catalog.TableRoleOutbox
+				tables = append(tables, table)
+
+				continue
+			}
+
+			// The layout is the claim: these rows exist because this
+			// aggregate exists, and its schema lives beside the code that
+			// reads it.
+			table.Persists = &catalog.Persists{Aggregate: aggregateID}
+
+			mapping := forTable(mapped, table.Name)
+			for i := range table.Columns {
+				if field, ok := mapping[table.Columns[i].Name]; ok {
+					table.Columns[i].Maps = field
+				}
+			}
+			applyCopies(&table, copies)
+
+			// The first table an aggregate creates holds the aggregate
+			// itself; anything it creates afterwards hangs off it.
+			if first {
+				table.Role = catalog.TableRoleAggregateRoot
+				first = false
+			} else {
+				table.Role = catalog.TableRoleChild
+			}
+
+			tables = append(tables, table)
+		}
+		for _, declared := range state.views {
+			views = append(views, declared.asCatalog(storeID, declared.source, tables))
+		}
+	}
+
+	// A projector package holds one projection: rows assembled from events,
+	// kept by a subscriber that writes and nothing else. The layout says the
+	// role. It does not say whose rows these are a picture of - a projection
+	// of another service's aggregate lives here too, and its name is not in
+	// any directory of this tree - so the link is taken from the migration
+	// when it is written there (`-- aggregate:`) and left out when it is not.
+	for _, projector := range layout.projectors {
+		dir := projector.migrations
+
+		state, copies, projected, ok := readMigrations(root, dir, storeID, owner, b)
+		if !ok {
+			continue
+		}
+
+		for _, relation := range state.relations {
+			table := finishTable(relation, state, storeID)
+			table.Role = catalog.TableRoleProjection
+			if aggregate := forTable(projected, table.Name); aggregate != "" {
+				table.Persists = &catalog.Persists{Aggregate: aggregate}
+			}
+			// No `maps`: the upsert in a projector writes what an event
+			// carries, and an event's field is not a field of the aggregate
+			// the way a repository's insert argument is. Where a value came
+			// from is said by `-- from:`, which is read.
+			applyCopies(&table, copies)
+
+			tables = append(tables, table)
+		}
+		for _, declared := range state.views {
+			views = append(views, declared.asCatalog(storeID, declared.source, tables))
+		}
+	}
+
+	// A view is read where it is written, which may be before the tables it
+	// reads are: the second pass is what lets a column take its type from the
+	// column it comes from wherever that was created.
+	for i := range views {
+		views[i] = resolveView(views[i], storeID, tables)
+	}
+
+	return tables, views
+}
+
+// readMigrations applies every up migration under dir, in name order, and
+// reads the comments the grammar drops: which columns are copies (`-- from:`)
+// and which aggregate a table is a picture of (`-- aggregate:`). ok is false
+// when there is no such directory.
+func readMigrations(root, dir, storeID, owner string, b *plugin.Builder) (*ddlState, map[string]map[string][]string, map[string]string, bool) {
+	files, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(dir)))
+	if err != nil {
+		return nil, nil, nil, false
+	}
+
+	names := make([]string, 0, len(files))
+	for _, file := range files {
+		// Only the up direction describes the schema. A down migration
+		// says how to lose it; a migration with no direction in its name
+		// only goes up.
+		if !file.IsDir() && strings.HasSuffix(file.Name(), ".sql") && !strings.HasSuffix(file.Name(), ".down.sql") {
+			names = append(names, file.Name())
+		}
+	}
+	// Applied in name order, so read in name order: a later migration is
+	// allowed to know about an earlier one.
+	sort.Strings(names)
+
+	state := newDDLState()
+	copies := map[string]map[string][]string{}
+	projected := map[string]string{}
+
+	for _, name := range names {
+		source := path.Join(dir, name)
+
+		sql, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(source)))
+		if err != nil {
+			b.Warn(storeID, source+" could not be read: "+err.Error())
+
+			continue
+		}
+
+		for table, columns := range readCopies(string(sql), storeID) {
+			if copies[table] == nil {
+				copies[table] = map[string][]string{}
+			}
+			for column, from := range columns {
+				copies[table][column] = from
+			}
+		}
+		for table, aggregate := range readProjected(string(sql), owner) {
+			projected[table] = aggregate
+		}
+		unread, err := state.apply(string(sql), source)
+		if err != nil {
+			b.Warn(storeID, "could not parse "+source+": "+err.Error())
+
+			continue
+		}
+		for _, note := range unread {
+			b.Warn(storeID, source+": "+note)
+		}
+	}
+
+	return state, copies, projected, true
+}
+
+// finishTable gives a relation its place in the catalog: an id under the
+// store, its indexes, and its column types spelled the way the migration
+// spelled them.
+func finishTable(relation relation, state *ddlState, storeID string) catalog.Table {
+	table := relation.table
+	table.ID = storeID + "." + table.Name
+	table.Indexes = relation.indexes
+	for i := range table.Columns {
+		table.Columns[i].Type = state.renderType(table.Columns[i].Type)
+	}
+
+	return table
+}
+
+// applyCopies writes onto each column where its value came from, when the
+// migration says. A table cannot show a copy the way a view shows a select,
+// so the copy is declared beside the column it lands in.
+func applyCopies(table *catalog.Table, copies map[string]map[string][]string) {
+	copying := forTable(copies, table.Name)
+	for i := range table.Columns {
+		if from, ok := copying[table.Columns[i].Name]; ok {
+			table.Columns[i].From = from
+		}
+	}
+}
+
+// forTable looks a table up by the name the migration used, then by its
+// unqualified name: a comment says `orders`, the grammar says `sales.orders`.
+func forTable[T any](byName map[string]T, name string) T {
+	if held, ok := byName[name]; ok {
+		return held
+	}
+
+	return byName[lastNamePart(name)]
+}
+
+// resolveView fills in what a view could not know when it was read: the type,
+// nullability and mapping of every column that comes from exactly one place.
+func resolveView(held catalog.View, storeID string, tables []catalog.Table) catalog.View {
+	for i := range held.Columns {
+		column := &held.Columns[i]
+		if column.Type != "" || len(column.From) != 1 {
+			continue
+		}
+		at := strings.LastIndex(column.From[0], ".")
+		if at < 0 {
+			continue
+		}
+		if source := columnOf(tables, column.From[0][:at], column.From[0][at+1:]); source != nil {
+			column.Type = source.Type
+			column.Nullable = source.Nullable
+			column.Maps = source.Maps
+		}
+	}
+	if held.Persists == nil {
+		persists := ""
+		for _, read := range held.Reads {
+			table := tableOf(tables, read)
+			if table == nil || table.Persists == nil {
+				continue
+			}
+			if persists == "" {
+				persists = table.Persists.Aggregate
+
+				continue
+			}
+			if persists != table.Persists.Aggregate {
+				persists = ""
+
+				break
+			}
+		}
+		if persists != "" {
+			held.Persists = &catalog.Persists{Aggregate: persists}
+		}
+	}
+
+	return held
+}
+
+// isOutbox says whether a table is the outbox by its name, which is the one
+// convention every outbox library shares.
+func isOutbox(name string) bool {
+	return name == "outbox" || strings.HasSuffix(name, "_outbox")
+}
+
+// resolveForeignKeys turns the table names the grammar gives into table ids the
+// catalog can follow. A key pointing outside this store keeps the raw name and
+// is reported: it crosses a boundary this extractor cannot see the far side of.
+func resolveForeignKeys(storeID string, tables []catalog.Table, b *plugin.Builder) {
+	known := map[string]bool{}
+	for i := range tables {
+		known[tables[i].Name] = true
+	}
+
+	for i := range tables {
+		for j := range tables[i].Columns {
+			fk := tables[i].Columns[j].FK
+			if fk == nil {
+				continue
+			}
+			if !known[fk.Table] {
+				b.Warn(tables[i].ID, "column "+tables[i].Columns[j].Name+" references "+fk.Table+", which no migration here creates")
+
+				continue
+			}
+			fk.Table = storeID + "." + fk.Table
+		}
+	}
+}
+
+// foreignSchemas reports migrations applied from outside this module.
+//
+// A service can bring a table with a dependency - an outbox published by an
+// SDK, say - and its DDL is then in that module rather than in this tree. The
+// table is real, the catalog cannot describe it, and silence would read as
+// "there is no such table".
+//
+// What is checked is the IMPORT PATH, never the name a file refers to the
+// package by. `userrepo.Migrations` and `sdkoutbox.Migrations` look equally
+// foreign as identifiers, and only the path says that one of them is the
+// repository package next door.
+func foreignSchemas(root, module string, b *plugin.Builder, storeID string) {
+	if module == "" {
+		return
+	}
+
+	reported := map[string]bool{}
+	fset := token.NewFileSet()
+
+	_ = filepath.WalkDir(filepath.Join(root, "internal"), func(p string, entry os.DirEntry, err error) error {
+		if err != nil || entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") {
+			return nil
+		}
+		if strings.HasSuffix(entry.Name(), "_test.go") {
+			// A test standing up its own schema says nothing about what the
+			// service deploys.
+			return nil
+		}
+
+		file, err := parser.ParseFile(fset, p, nil, parser.ImportsOnly|parser.SkipObjectResolution)
+		if err != nil {
+			return nil
+		}
+
+		paths := map[string]string{}
+		for _, spec := range file.Imports {
+			importPath := strings.Trim(spec.Path.Value, `"`)
+			name := path.Base(importPath)
+			if spec.Name != nil {
+				name = spec.Name.Name
+			}
+			paths[name] = importPath
+		}
+
+		// Imports only got parsed above, so the bodies are gone; the file is
+		// read again for the one selector that matters.
+		source, err := os.ReadFile(p)
+		if err != nil {
+			return nil
+		}
+
+		for _, match := range migrationsRef.FindAllStringSubmatch(string(source), -1) {
+			importPath, known := paths[match[1]]
+			if !known || reported[importPath] {
+				continue
+			}
+			if strings.HasPrefix(importPath, module) {
+				continue
+			}
+
+			reported[importPath] = true
+			b.Warn(storeID, "migrations are applied from "+importPath+", whose schema is not in this tree; the tables it creates are missing from this store")
+		}
+
+		return nil
+	})
+}
+
+// migrationsRef finds `<pkg>.Migrations`, which is how a migration set is
+// handed to the migrator. A regular expression rather than a walk of the
+// bodies: this produces a diagnostic and never a fact, and the day it misses
+// one nothing in the catalog is wrong - there is only one fewer warning.
+var migrationsRef = regexp.MustCompile(`(\w+)\.Migrations\b`)
+
+// modulePath is the module line of go.mod: what "inside this tree" means.
+func modulePath(root string) string {
+	source, err := os.ReadFile(filepath.Join(root, "go.mod"))
+	if err != nil {
+		return ""
+	}
+
+	for _, line := range strings.Split(string(source), "\n") {
+		if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "module "); ok {
+			return strings.TrimSpace(rest)
+		}
+	}
+
+	return ""
+}
+
+func subdirs(root, rel string) []string {
+	entries, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(rel)))
+	if err != nil {
+		return nil
+	}
+
+	var out []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			out = append(out, entry.Name())
+		}
+	}
+
+	return out
+}

package/plugins/extract-sql/store_test.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/shortlink-org/portolan/catalog"
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+func fragment(t *testing.T) (catalog.Catalog, []plugin.Warning) {
+	t.Helper()
+
+	resp := extract(
+		plugin.Input{Root: "../../examples/auth", Commit: "abc1234", GeneratedAt: "2026-01-01T00:00:00Z"},
+		Options{Context: "auth", Service: "auth", Store: "pg", Name: "Auth database"},
+	)
+	if len(resp.Files) != 1 {
+		t.Fatalf("expected one fragment, got %d files", len(resp.Files))
+	}
+
+	var out catalog.Catalog
+	if err := json.Unmarshal([]byte(resp.Files[0].Contents), &out); err != nil {
+		t.Fatal(err)
+	}
+
+	return out, resp.Warnings()
+}
+
+// One store, not one per aggregate: the migrations are numbered inside their
+// own package but they are all applied to the same database.
+func TestOneStoreHoldsEveryAggregatesTables(t *testing.T) {
+	cat, _ := fragment(t)
+
+	if len(cat.Stores) != 1 {
+		t.Fatalf("expected one store, got %d", len(cat.Stores))
+	}
+
+	store := cat.Stores[0]
+	if store.ID != "auth.auth.pg" || store.Owner != "auth.auth" {
+		t.Errorf("store = %s owned by %s", store.ID, store.Owner)
+	}
+
+	persists := map[string]string{}
+	for _, table := range store.Tables {
+		persists[table.Name] = table.Persists.Aggregate
+	}
+	if persists["users"] != "auth.auth.user" || persists["sessions"] != "auth.auth.session" || persists["lockouts"] != "auth.auth.lockout" {
+		t.Errorf("tables are linked to the wrong aggregates: %v", persists)
+	}
+	if store.Source != "../../examples/auth/internal/*/infrastructure/repository" {
+		t.Errorf("feature repository source = %q", store.Source)
+	}
+}
+
+// The service half of the fragment says only which store it touches. What it
+// is called and what it holds belong to the extractors that read those.
+func TestFragmentClaimsOnlyTheLink(t *testing.T) {
+	cat, _ := fragment(t)
+
+	service := cat.Contexts[0].Services[0]
+	if len(service.Stores) != 1 || service.Stores[0] != "auth.auth.pg" {
+		t.Errorf("stores = %v", service.Stores)
+	}
+	if service.Name != "" || service.Readme != "" || len(service.Aggregates) != 0 {
+		t.Errorf("the sql fragment should not describe the service: %+v", service)
+	}
+}
+
+// A table brought by a dependency is real, and its DDL is not here. Silence
+// would read as "there is no such table".
+func TestSchemaAppliedFromOutsideTheTreeIsReported(t *testing.T) {
+	_, diagnostics := fragment(t)
+
+	var found string
+	for _, d := range diagnostics {
+		if strings.Contains(d.Message, "not in this tree") {
+			found = d.Message
+		}
+	}
+
+	if !strings.Contains(found, "go-sdk/outbox") {
+		t.Errorf("the outbox schema should be reported by import path, got %q", found)
+	}
+	// The repository packages next door are imported under aliases that look
+	// just as foreign as the SDK's. Only the path tells them apart.
+	for _, d := range diagnostics {
+		if strings.Contains(d.Message, "repo") && strings.Contains(d.Message, "not in this tree") {
+			t.Errorf("a package from this module was called foreign: %q", d.Message)
+		}
+	}
+}
+
+// writeTree lays out a service under a temporary root: one file per path.
+func writeTree(t *testing.T, files map[string]string) string {
+	t.Helper()
+
+	root := t.TempDir()
+	for rel, contents := range files {
+		full := filepath.Join(root, filepath.FromSlash(rel))
+		if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(full, []byte(contents), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	return root
+}
+
+// A projector package is the layout's word for a projection: every table it
+// creates has that role. Whose rows it pictures is not in the layout, so it is
+// taken from the migration when written there and left out when not - a
+// projection of another service's aggregate has no directory here to be named
+// by. The repository packages beside it are read exactly as before.
+func TestProjectorPackagesAreProjections(t *testing.T) {
+	root := writeTree(t, map[string]string{
+		"internal/infrastructure/repository/route/migrations/0001_routes.sql": `
+CREATE TABLE routes (
+    id      text PRIMARY KEY,
+    version bigint NOT NULL
+);
+CREATE TABLE route_legs (
+    route_id text NOT NULL REFERENCES routes (id),
+    seq      integer NOT NULL
+);`,
+		"internal/infrastructure/projector/route_board/migrations/0001_route_board.sql": `
+-- Stops with the address to print, rebuilt from OrderPlaced.
+-- aggregate: shop.oms.order
+CREATE TABLE route_board (
+    order_id     text PRIMARY KEY,
+    -- from: shop.oms.pg.orders.ship_to
+    address      text NOT NULL,
+    projected_at timestamptz NOT NULL
+);
+
+CREATE TABLE route_board_cursor (
+    topic    text PRIMARY KEY,
+    -- from: routes.version
+    position bigint NOT NULL
+);`,
+	})
+
+	resp := extract(plugin.Input{Root: root}, Options{Context: "shop", Service: "delivery"})
+	if len(resp.Files) != 1 {
+		t.Fatalf("expected one fragment, got %d files", len(resp.Files))
+	}
+	var cat catalog.Catalog
+	if err := json.Unmarshal([]byte(resp.Files[0].Contents), &cat); err != nil {
+		t.Fatal(err)
+	}
+	if len(cat.Stores) != 1 {
+		t.Fatalf("expected one store, got %d", len(cat.Stores))
+	}
+
+	byName := map[string]catalog.Table{}
+	for _, table := range cat.Stores[0].Tables {
+		byName[table.Name] = table
+	}
+	if len(byName) != 4 {
+		t.Fatalf("expected four tables, got %v", byName)
+	}
+
+	// The repository side is unchanged.
+	if byName["routes"].Role != catalog.TableRoleAggregateRoot || byName["routes"].Persists.Aggregate != "shop.delivery.route" {
+		t.Errorf("routes = %+v", byName["routes"])
+	}
+	if byName["route_legs"].Role != catalog.TableRoleChild || byName["route_legs"].Persists.Aggregate != "shop.delivery.route" {
+		t.Errorf("route_legs = %+v", byName["route_legs"])
+	}
+
+	board := byName["route_board"]
+	if board.ID != "shop.delivery.pg.route_board" || board.Role != catalog.TableRoleProjection {
+		t.Errorf("route_board = %+v", board)
+	}
+	if board.Persists == nil || board.Persists.Aggregate != "shop.oms.order" {
+		t.Errorf("the migration says whose picture this is: %+v", board.Persists)
+	}
+	for _, column := range board.Columns {
+		if column.Name != "address" {
+			continue
+		}
+		if len(column.From) != 1 || column.From[0] != "shop.oms.pg.orders.ship_to" {
+			t.Errorf("address.from = %v", column.From)
+		}
+		if column.Maps != "" {
+			t.Errorf("a projection column maps to no aggregate field, got %q", column.Maps)
+		}
+	}
+
+	cursor := byName["route_board_cursor"]
+	if cursor.Role != catalog.TableRoleProjection {
+		t.Errorf("every table a projector creates is a projection, got %q", cursor.Role)
+	}
+	if cursor.Persists != nil {
+		t.Errorf("no note, no link: %+v", cursor.Persists)
+	}
+	for _, column := range cursor.Columns {
+		if column.Name == "position" && (len(column.From) != 1 || column.From[0] != "shop.delivery.pg.routes.version") {
+			t.Errorf("a copy inside the store gets the store put back on, got %v", column.From)
+		}
+	}
+
+	for _, d := range resp.Warnings() {
+		if strings.Contains(d.Message, "no migrations") {
+			t.Errorf("the store has tables, yet: %q", d.Message)
+		}
+	}
+}
+
+// A service with projectors and no repositories still keeps state.
+func TestAStoreOfOnlyProjectionsIsStillAStore(t *testing.T) {
+	root := writeTree(t, map[string]string{
+		"internal/infrastructure/projector/user_directory/migrations/0001_user_directory.sql": `
+CREATE TABLE user_directory (
+    user_id text PRIMARY KEY,
+    email   text NOT NULL
+);`,
+	})
+
+	resp := extract(plugin.Input{Root: root}, Options{Context: "shop", Service: "bff"})
+	var cat catalog.Catalog
+	if err := json.Unmarshal([]byte(resp.Files[0].Contents), &cat); err != nil {
+		t.Fatal(err)
+	}
+	if len(cat.Stores) != 1 || len(cat.Stores[0].Tables) != 1 || cat.Stores[0].Tables[0].Role != catalog.TableRoleProjection {
+		t.Errorf("stores = %+v", cat.Stores)
+	}
+}