@shortlink-org/portolan 0.1.0

package/plugins/extract-sql/maps_ts.go

@@ -0,0 +1,398 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/pgplex/pgparser/nodes"
+	pgsql "github.com/pgplex/pgparser/parser"
+	"github.com/shortlink-org/portolan/plugin"
+)
+
+// The same fact read out of a repository written in TypeScript, where the
+// statement and its arguments sit side by side in one call:
+//
+//	client.query(
+//	  "INSERT INTO baskets (id, token, customer_id, touched_at) VALUES ($1, $2, $3, $4)",
+//	  [basket.id, basket.token, basket.customerId, basket.touchedAt.toISOString()],
+//	)
+//
+// There is no parser for the language here, and none is needed for this
+// shape: the statement is a string literal, the arguments are an array
+// literal right after it, and each argument is a property chain from the
+// aggregate - possibly wrapped in one conversion. Anything else is left
+// unmapped, for the same reason as in the Go reader: a blank is a reader
+// looking the column up, a wrong one is a reader believing something untrue.
+func readMapsTS(root, repositoryDir, aggregate string, b *plugin.Builder) map[string]map[string]string {
+	out := map[string]map[string]string{}
+	dir := mapSourceDir(root, repositoryDir, aggregate)
+	entries, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(dir)))
+	if err != nil {
+		return out
+	}
+	root_ := title(aggregate)
+
+	for _, entry := range entries {
+		name := entry.Name()
+		if entry.IsDir() || !strings.HasSuffix(name, ".ts") || strings.HasSuffix(name, ".test.ts") || strings.HasSuffix(name, ".d.ts") {
+			continue
+		}
+		source, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(dir), name))
+		if err != nil {
+			continue
+		}
+		roots := rootVarsTS(string(source), root_)
+		aliases := loopAliasesTS(string(source))
+		for _, stmt := range insertCalls(string(source)) {
+			table, columns, values := parseInsert(stmt.sql)
+			if table == "" {
+				continue
+			}
+			mapped := out[table]
+			if mapped == nil {
+				mapped = map[string]string{}
+				out[table] = mapped
+			}
+			numbers := placeholderNumbers(stmt.sql)
+			if numbers == nil || countParams(values) != len(numbers) {
+				if countParams(values) > 0 {
+					b.Warn(table, "the insert into "+table+" has placeholders this reader cannot number; its columns are left unmapped")
+				}
+				continue
+			}
+			seen := 0
+			for i, column := range columns {
+				if i >= len(values) {
+					break
+				}
+				if _, ok := values[i].(*nodes.ParamRef); !ok {
+					continue
+				}
+				number := numbers[seen]
+				seen++
+				if number < 1 || number > len(stmt.args) {
+					b.Warn(table, "the insert refers to $"+itoa(number)+" but the call passes "+itoa(len(stmt.args))+" values; "+column+" is left unmapped")
+					continue
+				}
+				field := fieldOfTS(stmt.args[number-1], roots, aliases)
+				if field == "" {
+					continue
+				}
+				mapped[column] = root_ + "." + field
+			}
+		}
+	}
+
+	return out
+}
+
+// tsInsert is one statement and the array of arguments written after it.
+type tsInsert struct {
+	sql  string
+	args []string
+}
+
+// insertCalls walks the source one character at a time, because a string
+// literal can only be found by reading from its opening quote to its closing
+// one: a pattern that skips ahead to the next quote it likes will start in
+// one literal and end in another, and read the code between them as SQL.
+// Comments are stepped over the same way. A literal that holds an INSERT and
+// is followed by a comma and an array is a statement with its arguments.
+func insertCalls(source string) []tsInsert {
+	var out []tsInsert
+	for i := 0; i < len(source); i++ {
+		c := source[i]
+		switch {
+		case c == '/' && i+1 < len(source) && source[i+1] == '/':
+			for i < len(source) && source[i] != '\n' {
+				i++
+			}
+		case c == '/' && i+1 < len(source) && source[i+1] == '*':
+			end := strings.Index(source[i+2:], "*/")
+			if end < 0 {
+				return out
+			}
+			i += 2 + end + 1
+		case c == '"' || c == '\'' || c == '`':
+			end := closingQuote(source, i)
+			if end < 0 {
+				return out
+			}
+			sql := source[i+1 : end]
+			i = end
+			j := skipSpace(source, end+1)
+			if j >= len(source) || source[j] != ',' {
+				continue
+			}
+			j = skipSpace(source, j+1)
+			if j >= len(source) || source[j] != '[' || !strings.Contains(strings.ToUpper(sql), "INSERT INTO") {
+				continue
+			}
+			close := matchingBracket(source, j)
+			if close < 0 {
+				continue
+			}
+			out = append(out, tsInsert{sql: sql, args: splitTopLevel(source[j+1 : close])})
+			i = close
+		}
+	}
+
+	return out
+}
+
+// closingQuote finds the quote that closes the one at open, stepping over
+// escapes.
+func closingQuote(source string, open int) int {
+	quote := source[open]
+	for i := open + 1; i < len(source); i++ {
+		switch source[i] {
+		case '\\':
+			i++
+		case quote:
+			return i
+		}
+	}
+
+	return -1
+}
+
+func skipSpace(source string, i int) int {
+	for i < len(source) && (source[i] == ' ' || source[i] == '\t' || source[i] == '\n' || source[i] == '\r') {
+		i++
+	}
+
+	return i
+}
+
+// matchingBracket finds the `]` that closes the `[` at open, stepping over
+// nested brackets, parentheses, braces and string literals.
+func matchingBracket(source string, open int) int {
+	depth := 0
+	var quote byte
+	for i := open; i < len(source); i++ {
+		c := source[i]
+		if quote != 0 {
+			if c == '\\' {
+				i++
+			} else if c == quote {
+				quote = 0
+			}
+			continue
+		}
+		switch c {
+		case '"', '\'', '`':
+			quote = c
+		case '[', '(', '{':
+			depth++
+		case ']', ')', '}':
+			depth--
+			if depth == 0 {
+				return i
+			}
+		}
+	}
+
+	return -1
+}
+
+// splitTopLevel splits an argument list on the commas that are not inside a
+// call, an object, an array or a string.
+func splitTopLevel(list string) []string {
+	var out []string
+	depth := 0
+	var quote byte
+	start := 0
+	for i := 0; i < len(list); i++ {
+		c := list[i]
+		if quote != 0 {
+			if c == '\\' {
+				i++
+			} else if c == quote {
+				quote = 0
+			}
+			continue
+		}
+		switch c {
+		case '"', '\'', '`':
+			quote = c
+		case '[', '(', '{':
+			depth++
+		case ']', ')', '}':
+			depth--
+		case ',':
+			if depth == 0 {
+				out = append(out, strings.TrimSpace(list[start:i]))
+				start = i + 1
+			}
+		}
+	}
+	if last := strings.TrimSpace(list[start:]); last != "" {
+		out = append(out, last)
+	}
+
+	return out
+}
+
+var tsChain = regexp.MustCompile(`^[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*)+$`)
+
+// fieldOfTS follows an argument back to the field it carries, the way fieldOf
+// does for Go: `basket.customerId` is customerId, `basket.touchedAt.toISOString()`
+// is touchedAt, `nullable(basket.checkedOutAt)` is checkedOutAt. A bare name,
+// a literal, a call that makes its own value - `randomUUID()`, `new Date()` -
+// and a call over several values carry no field.
+func fieldOfTS(expr string, roots map[string]bool, aliases map[string]string) string {
+	expr = strings.TrimSpace(expr)
+	// `basket.customerId ?? null` writes the field with a default for its
+	// absence; the column still carries the field.
+	if at := strings.LastIndex(expr, "??"); at >= 0 && tsLiteral.MatchString(strings.TrimSpace(expr[at+2:])) {
+		expr = strings.TrimSpace(expr[:at])
+	}
+	for strings.HasSuffix(expr, ")") {
+		open := openingParen(expr)
+		if open <= 0 {
+			return ""
+		}
+		callee, inner := expr[:open], expr[open+1:len(expr)-1]
+		args := splitTopLevel(inner)
+		switch {
+		case tsChain.MatchString(callee) && (len(args) == 0 || tsChain.MatchString(strings.TrimSuffix(callee[:strings.LastIndex(callee, ".")], "?"))):
+			// A method on the value: the value is the chain before it. With
+			// no arguments it is a conversion of the value - toISOString() -
+			// and with some it is still the value's own method when what it
+			// stands on is itself a chain: basket.items.slice(1) walks the
+			// items, where JSON.stringify(x) stands on a bare name and is
+			// the conversion read below.
+			expr = callee[:strings.LastIndex(callee, ".")]
+			expr = strings.TrimSuffix(expr, "?")
+		case len(args) == 1:
+			// A conversion wrapped round the value.
+			expr = strings.TrimSpace(args[0])
+		default:
+			return ""
+		}
+	}
+	if !tsChain.MatchString(expr) {
+		return ""
+	}
+	parts := strings.Split(strings.ReplaceAll(expr, "?.", "."), ".")
+	if over, ok := aliases[parts[0]]; ok {
+		// The variable of a loop stands for one element of what it walks:
+		// `for (const item of basket.items)` makes item.sku items.sku. What
+		// is walked is read as an argument is, so a slice of the items is
+		// still the items. The alias is forgotten for the rest of the read,
+		// so that a loop written over its own name cannot stand for itself.
+		rest := make(map[string]string, len(aliases))
+		for name, chain := range aliases {
+			if name != parts[0] {
+				rest[name] = chain
+			}
+		}
+		head := fieldOfTS(over, roots, rest)
+		if head == "" {
+			return ""
+		}
+
+		return strings.Join(append([]string{head}, parts[1:]...), ".")
+	}
+	if roots != nil && !roots[parts[0]] {
+		return ""
+	}
+
+	return strings.Join(parts[1:], ".")
+}
+
+// tsForOf is the head of a for-of loop, up to what it walks:
+// `for (const item of `. What follows runs to the parenthesis that closes
+// the loop's own, found by matching rather than by pattern because a slice
+// or a filter carries parentheses of its own.
+var tsForOf = regexp.MustCompile(`for\s*\(\s*(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s+of\s+`)
+
+// loopAliasesTS reads every for-of loop of the file: the variable, and the
+// expression it walks.
+func loopAliasesTS(source string) map[string]string {
+	var aliases map[string]string
+	for _, m := range tsForOf.FindAllStringSubmatchIndex(source, -1) {
+		open := strings.IndexByte(source[m[0]:], '(') + m[0]
+		close := matchingBracket(source, open)
+		if close < 0 || close < m[1] {
+			continue
+		}
+		if aliases == nil {
+			aliases = map[string]string{}
+		}
+		aliases[source[m[2]:m[3]]] = strings.TrimSpace(source[m[1]:close])
+	}
+
+	return aliases
+}
+
+// tsLiteral is what may stand after `??` without changing which field the
+// argument carries: nothing, or a constant.
+var tsLiteral = regexp.MustCompile(`^(null|undefined|true|false|-?\d+(\.\d+)?|"[^"]*"|'[^']*')$`)
+
+// tsParam is a name declared with the type after it: a parameter, a field, a
+// variable. `route: Route` is the aggregate wherever it appears in the file.
+var tsParam = regexp.MustCompile(`\b([A-Za-z_$][\w$]*)\??\s*:\s*([A-Za-z_$][\w$]*)\b`)
+
+// rootVarsTS names the variables the file declares with the root's type, or
+// nil when it declares none. Inside save(route: Route, ...) the aggregate is
+// route and nothing else: the loop variable over its stops carries a stop's
+// fields, and a column mapped from it onto the root would be a claim about
+// the wrong type. A file that never names the type - a repository over a row
+// type of its own - is read as before, any bare name standing for the root.
+func rootVarsTS(source, root string) map[string]bool {
+	var roots map[string]bool
+	for _, m := range tsParam.FindAllStringSubmatch(source, -1) {
+		if m[2] != root {
+			continue
+		}
+		if roots == nil {
+			roots = map[string]bool{}
+		}
+		roots[m[1]] = true
+	}
+
+	return roots
+}
+
+// openingParen finds the `(` that the closing paren at the end of expr pairs
+// with.
+func openingParen(expr string) int {
+	depth := 0
+	for i := len(expr) - 1; i >= 0; i-- {
+		switch expr[i] {
+		case ')':
+			depth++
+		case '(':
+			depth--
+			if depth == 0 {
+				return i
+			}
+		}
+	}
+
+	return -1
+}
+
+// parseInsert reads the table, its column list and its values out of one
+// INSERT statement.
+func parseInsert(sql string) (string, []string, []nodes.Node) {
+	tree, err := pgsql.Parse(sql)
+	if err != nil || len(tree.Items) == 0 {
+		return "", nil, nil
+	}
+	insert, ok := tree.Items[0].(*nodes.InsertStmt)
+	if !ok || insert.Relation == nil {
+		return "", nil, nil
+	}
+	var columns []string
+	for _, item := range items(insert.Cols) {
+		if target, ok := item.(*nodes.ResTarget); ok {
+			columns = append(columns, target.Name)
+		}
+	}
+
+	return relationName(insert.Relation), columns, insertValues(insert)
+}

package/plugins/extract-sql/maps_ts_test.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"testing"
+)
+
+// The same rule as for Go, read off text: the chain from the aggregate, one
+// conversion allowed round it, a method on the value stripped.
+func TestFieldOfTSFollowsTheChainAndOneWrapper(t *testing.T) {
+	cases := map[string]string{
+		"basket.id":                      "id",
+		"basket.customerId":              "customerId",
+		"basket.touchedAt.toISOString()": "touchedAt",
+		"basket.unitPrice?.amountMinor":  "unitPrice.amountMinor",
+		"nullable(basket.checkedOutAt)":  "checkedOutAt",
+		"String(basket.version)":         "version",
+		"JSON.stringify(toWire(event))":  "",
+		"randomUUID()":                   "",
+		"new Date()":                     "",
+		"combine(basket.a, basket.b)":    "",
+		"TOPIC":                          "",
+		"'literal'":                      "",
+		"basket.items.slice(1)":          "items",
+		"JSON.stringify(basket.shipTo)":  "shipTo",
+		"basket.customerId ?? null":      "customerId",
+		"basket.currency?.code ?? null":  "currency.code",
+		"basket.version ?? 0":            "version",
+		"basket.a ?? basket.b":           "",
+	}
+	for in, want := range cases {
+		if got := fieldOfTS(in, nil, nil); got != want {
+			t.Errorf("fieldOfTS(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+// Once the file says which name is the aggregate, only that name maps: the
+// item of a loop over its lines is not the basket.
+func TestFieldOfTSReadsOnlyTheRootWhenTheFileNamesIt(t *testing.T) {
+	roots := rootVarsTS("async save(basket: Basket, ...events: BasketEvent[]) { for (const item of basket.items) {} }", "Basket")
+	if !roots["basket"] || roots["events"] || roots["item"] {
+		t.Fatalf("roots = %v", roots)
+	}
+	cases := map[string]string{
+		"basket.id":     "id",
+		"item.sku":      "",
+		"item.quantity": "",
+	}
+	for in, want := range cases {
+		if got := fieldOfTS(in, roots, nil); got != want {
+			t.Errorf("fieldOfTS(%q) = %q, want %q", in, got, want)
+		}
+	}
+	if rootVarsTS("const rows = await pool.query<Row>(sql)", "Basket") != nil {
+		t.Error("a file that never names the root should leave every name open")
+	}
+}
+
+// The variable of a for-of loop stands for one element of what it walks, so
+// a column written from it carries a field of the elements of a field.
+func TestFieldOfTSFollowsALoopVariable(t *testing.T) {
+	aliases := loopAliasesTS(`for (const item of basket.items) { for (const event of events) {} } for (const self of self.items) {}
+for (const scan of basket.scans.slice(stored.rows[0]?.n ?? 0)) {}`)
+	if aliases["item"] != "basket.items" || aliases["event"] != "events" || aliases["self"] != "self.items" || aliases["scan"] != "basket.scans.slice(stored.rows[0]?.n ?? 0)" {
+		t.Fatalf("aliases = %v", aliases)
+	}
+	roots := map[string]bool{"basket": true}
+	cases := map[string]string{
+		"item.sku":                     "items.sku",
+		"item.unitPrice.currency.code": "items.unitPrice.currency.code",
+		"item.touchedAt.toISOString()": "items.touchedAt",
+		"scan.parcelId":                "scans.parcelId",
+		"event.name":                   "",
+		"self.name":                    "",
+	}
+	for in, want := range cases {
+		if got := fieldOfTS(in, roots, aliases); got != want {
+			t.Errorf("fieldOfTS(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestMapsTSAgainstTheRealService(t *testing.T) {
+	b := builderFor(t)
+	mapped := readMapsTS("../../examples/shop/cart", "src/infrastructure/repository/basket", "basket", b)
+
+	want := map[string]map[string]string{
+		"baskets": {
+			"id":          "Basket.id",
+			"token":       "Basket.token",
+			"customer_id": "Basket.customerId",
+			"currency":    "Basket.currency.code",
+			"status":      "Basket.status",
+			"touched_at":  "Basket.touchedAt",
+		},
+		"basket_items": {
+			"basket_id":        "Basket.id",
+			"sku":              "Basket.items.sku",
+			"quantity":         "Basket.items.quantity",
+			"unit_price_minor": "Basket.items.unitPrice.amountMinor",
+			"currency":         "Basket.items.unitPrice.currency.code",
+		},
+	}
+	for table, columns := range want {
+		for column, field := range columns {
+			if got := mapped[table][column]; got != field {
+				t.Errorf("%s.%s -> %q, want %q", table, column, got, field)
+			}
+		}
+		if len(mapped[table]) != len(columns) {
+			t.Errorf("%s: mapped %v, want exactly %v", table, mapped[table], columns)
+		}
+	}
+}
+
+func TestInsertCallsPairAStatementWithItsArray(t *testing.T) {
+	source := `
+await client.query("BEGIN"); // a literal before, which must not swallow what follows
+await client.query(
+  "INSERT INTO baskets (id, token, version) VALUES ($1, $2, 1)",
+  [basket.id, basket.token],
+);
+const rows = await this.pool.query<Row>("SELECT * FROM baskets WHERE id = $1", [id]);
+await client.query("INSERT INTO outbox (uuid, payload) VALUES ($1, $2)", [randomUUID(), JSON.stringify({ a: [1, 2] })]);
+`
+	calls := insertCalls(source)
+	if len(calls) != 2 {
+		t.Fatalf("found %d inserts, want 2", len(calls))
+	}
+	if len(calls[0].args) != 2 || calls[0].args[1] != "basket.token" {
+		t.Fatalf("first insert args = %q", calls[0].args)
+	}
+	if len(calls[1].args) != 2 || calls[1].args[1] != "JSON.stringify({ a: [1, 2] })" {
+		t.Fatalf("second insert args = %q", calls[1].args)
+	}
+}

package/plugins/extract-sql/options.schema.json

@@ -0,0 +1,53 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "context": {
+      "type": "string",
+      "description": "Slug of the bounded context that owns the service these migrations belong to. Defaults to the name of the input directory.",
+      "examples": ["auth"]
+    },
+    "service": {
+      "type": "string",
+      "description": "Slug of the service that owns the database. A store belongs to exactly one service. Defaults to the name of the input directory.",
+      "examples": ["auth"]
+    },
+    "store": {
+      "type": "string",
+      "description": "Slug of the database these migrations build. One service can keep state in more than one, and a slug is how the two are told apart. Give the go-domain extractor the same one.",
+      "default": "pg"
+    },
+    "name": {
+      "type": "string",
+      "description": "What the database is called on the page.",
+      "examples": ["Auth database"]
+    },
+    "kind": {
+      "enum": [
+        "postgres",
+        "mysql",
+        "sqlite",
+        "redis",
+        "mongodb",
+        "clickhouse",
+        "s3",
+        "other"
+      ],
+      "description": "What kind of store it is.",
+      "default": "postgres"
+    },
+    "repositories": {
+      "type": "string",
+      "description": "Optional adapter root relative to the input root: either one package per aggregate with migrations below each package, or one feature repository package with migrations directly below it. When omitted, Go layouts are discovered under both internal/infrastructure/repository/<aggregate> and internal/<aggregate>/infrastructure/repository. The migrations and repository writes determine which aggregate and fields the tables persist."
+    },
+    "projectors": {
+      "type": "string",
+      "description": "Optional projector root relative to the input root, using the same collection-or-feature-package convention as repositories. When omitted, horizontal and feature-sliced Go projector packages are discovered. Every table created there gets the role projection. Its aggregate is read from a `-- aggregate: <id>` line above CREATE TABLE; columns copied from elsewhere carry `-- from:`."
+    },
+    "out": {
+      "type": "string",
+      "description": "Name of the fragment file, inside the step's output directory.",
+      "default": "stores.json"
+    }
+  }
+}

package/plugins/extract-sql/projection.go

@@ -0,0 +1,70 @@
+package main
+
+// Whose rows a projection is a picture of, when the migration says.
+//
+// A repository package names its aggregate by where it sits: the directory is
+// the aggregate, and the first table it creates is the root. A projector
+// package cannot say it that way. The projection is named after the question
+// it answers, not after an aggregate, and the aggregate it is built from may
+// belong to another service whose name appears nowhere in this tree.
+//
+// So it is declared, in the same form a copied column already uses, above the
+// CREATE TABLE it speaks for:
+//
+//	-- Planned stops, rebuilt from RoutePlanned.
+//	-- aggregate: shop.oms.order
+//	CREATE TABLE route_stops (
+//
+// A bare slug (`route`) is an aggregate of the service that owns the store and
+// is spelled the way the repository layout would spell it: `price_list` is
+// `price-list`. Anything with a dot in it is a full id and is taken as
+// written. Other comment lines may sit between the note and the statement; a
+// statement may not, and a note that reaches nothing is dropped.
+
+import (
+	"regexp"
+	"strings"
+)
+
+var aggregateComment = regexp.MustCompile(`(?i)^\s*--\s*aggregate:\s*(.+?)\s*$`)
+
+// readProjected reads the `-- aggregate:` lines of one migration, as
+// unqualified table name -> aggregate id.
+func readProjected(sql, owner string) map[string]string {
+	out := map[string]string{}
+	pending := ""
+
+	for _, line := range strings.Split(sql, "\n") {
+		if found := aggregateComment.FindStringSubmatch(line); found != nil {
+			pending = aggregateID(found[1], owner)
+
+			continue
+		}
+		if found := createTable.FindStringSubmatch(line); found != nil {
+			if pending != "" {
+				out[unqualified(found[1])] = pending
+			}
+			pending = ""
+
+			continue
+		}
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "--") {
+			continue
+		}
+		// Anything else is a statement, and the note was not about it.
+		pending = ""
+	}
+
+	return out
+}
+
+// aggregateID spells the note as an id: a bare slug belongs to the owner of the
+// store, a dotted name is already one.
+func aggregateID(note, owner string) string {
+	if strings.Contains(note, ".") {
+		return note
+	}
+
+	return owner + "." + slug(note)
+}