@shortlink-org/portolan 0.1.0

package/plugins/extract-django/lifecycle_test.py

@@ -0,0 +1,76 @@
+"""The table is the claim, whichever of the two ways it is written down."""
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
+
+import apps as apps_module  # noqa: E402
+import domain  # noqa: E402
+import events as events_module  # noqa: E402
+import lifecycle  # noqa: E402
+from protocol import Builder  # noqa: E402
+from source import Project  # noqa: E402
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
+
+
+def read(fixture: str, service: str):
+    root = os.path.join(HERE, "testdata", fixture)
+    b = Builder()
+    project = Project(root, root, lambda path: os.path.relpath(path, ROOT).replace(os.sep, "/"))
+    aggregates = domain.read_aggregates(project, apps_module.discover(project, []), "shop." + service, {}, b)
+    registry = {}
+    for agg in aggregates:
+        _, found = events_module.read_events(agg, service, b)
+        registry.update(found)
+    return aggregates, registry, b
+
+
+class Transitions(unittest.TestCase):
+    def test_a_table_and_the_methods_held_to_it(self):
+        aggregates, registry, b = read("billing", "billing")
+        life = lifecycle.read(aggregates[0], registry, b)
+        self.assertEqual(life["states"], ["draft", "issued", "paid", "void"])
+        self.assertEqual(
+            [(t["from"], t["to"], t["on"]) for t in life["transitions"]],
+            [("draft", "issued", "issue"), ("issued", "paid", "pay"), ("draft", "void", "void"), ("issued", "void", "void")],
+        )
+
+    def test_the_event_a_mover_hands_back_is_what_it_publishes(self):
+        aggregates, registry, b = read("billing", "billing")
+        life = lifecycle.read(aggregates[0], registry, b)
+        emits = {t["on"]: t.get("emits", "") for t in life["transitions"]}
+        self.assertEqual(emits["issue"], "shop.billing.invoice.InvoiceIssued")
+        self.assertEqual(emits["pay"], "shop.billing.invoice.InvoicePaid")
+        self.assertEqual(emits["void"], "")
+
+    def test_a_decorator_is_a_table_written_one_edge_at_a_time(self):
+        aggregates, registry, b = read("fsm", "delivery")
+        life = lifecycle.read(aggregates[0], registry, b)
+        self.assertEqual(life["states"], ["planned", "held", "dispatched", "delivered"])
+        self.assertEqual(
+            [(t["from"], t["to"], t["on"]) for t in life["transitions"]],
+            [
+                ("planned", "held", "hold"),
+                ("planned", "dispatched", "dispatch"),
+                ("held", "dispatched", "dispatch"),
+                ("dispatched", "delivered", "deliver"),
+            ],
+        )
+
+    def test_a_table_and_its_methods_that_have_drifted_apart_are_reported(self):
+        aggregates, registry, b = read("drift", "support")
+        life = lifecycle.read(aggregates[0], registry, b)
+        messages = [d.message for d in b.warnings]
+        self.assertIn("archive moves to 'archived', which the table does not list", messages)
+        self.assertIn("the table has closed -> open and no method of Ticket makes it", messages)
+        # Reported, and still drawn: the state a method reaches is a state.
+        self.assertIn("archived", life["states"])
+
+
+if __name__ == "__main__":
+    unittest.main()

package/plugins/extract-django/main.py

@@ -0,0 +1,66 @@
+"""portolan-extract-django: a Django service in, a catalog fragment out.
+
+The same protocol as every other extractor: one JSON request on stdin, one JSON
+response on stdout, and a `describe` that answers with what the plugin is and
+what it can be told. Python 3.9 or newer, and nothing but the standard library:
+a plugin that needed the project's own dependencies installed could not be run
+over somebody else's checkout.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
+
+from extract import extract  # noqa: E402
+from options import Options  # noqa: E402
+from protocol import Builder, Input  # noqa: E402
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+DESCRIPTOR = {
+    "name": "extract-django",
+    "summary": "Reads a Django service by its applications - models, events, services, DRF views, receivers, clients - into a catalog fragment, and its models into the store they are the schema of.",
+    "phases": ["extract"],
+}
+
+
+def descriptor():
+    with open(os.path.join(HERE, "options.schema.json"), "r", encoding="utf-8") as handle:
+        out = dict(DESCRIPTOR)
+        out["options"] = json.load(handle)
+        return out
+
+
+def serve(raw: str) -> str:
+    request = json.loads(raw)
+    version = request.get("portolanVersion", "")
+    if version and version != "0.1.0":
+        raise ValueError("unsupported portolan protocol %r (plugin supports 0.1.0)" % version)
+    if request.get("kind") == "describe":
+        return json.dumps({"files": [], "describe": descriptor()})
+    input_ = Input.of(request.get("input"))
+    if not input_.root:
+        raise ValueError("no input root: an extractor has nothing to read")
+    builder = Builder()
+    extract(input_, Options.of(request.get("options")), builder)
+    for warning in builder.warnings:
+        sys.stderr.write("warning: %s%s\n" % ((warning.ref + ": ") if warning.ref else "", warning.message))
+    return json.dumps(builder.response(), indent=2)
+
+
+def main() -> int:
+    try:
+        sys.stdout.write(serve(sys.stdin.read()))
+    except Exception as err:  # the host reads a non-zero exit and the message
+        sys.stderr.write("portolan-extract-django: %s\n" % err)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/plugins/extract-django/openapi.py

@@ -0,0 +1,229 @@
+"""The document a client is vendored beside, and the names it gives.
+
+The same rules as `plugins/openapi/ids.go`, spelled a second time so that a
+Django caller and the service that answers spell one call the same way:
+`auth.v1.Sessions/validateSession` on both sides, or the call would never
+resolve to the method.
+
+The reader underneath is a small one. A document is a mapping of mappings with
+the odd sequence, and the parts read here - the title, the version, the paths,
+each operation's id and first tag - need no more than that. Anything it cannot
+make sense of is left out rather than guessed at, and a client whose document
+answers nothing is reported.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+VERBS = ["get", "put", "post", "delete", "options", "head", "patch", "trace"]
+
+
+def api_id(title: str, version: str) -> str:
+    """The document's title and major version: `auth` 1.0.0 gives `auth.v1`."""
+    name = (title or "api").lower().replace(" ", "-")
+    major = version.split(".")[0] if version else ""
+    return name + ".v" + major if major else name
+
+
+def title_of(name: str) -> str:
+    """users becomes Users, price_list becomes PriceList: it sits in an id
+    beside a proto-shaped service name."""
+    out = ""
+    for word in re.split(r"[_\-\s]+", name):
+        if word:
+            out += word[0].upper() + word[1:]
+    return out
+
+
+def interface_id(api: str, tag: str) -> str:
+    return api + "." + title_of(tag) if tag else api
+
+
+@dataclass
+class Operation:
+    id: str
+    tag: str
+    verb: str
+    path: str
+
+    def interface(self, api: str) -> str:
+        return interface_id(api, self.tag)
+
+    def call_id(self, api: str) -> str:
+        return self.interface(api) + "/" + self.id
+
+
+@dataclass
+class Spec:
+    api: str
+    path: str
+    operations: List[Operation] = field(default_factory=list)
+
+    def find(self, verb: str, route: str) -> Optional[Operation]:
+        """Both sides may spell a parameter differently - `{invoiceId}` in the
+        document, `{id}` or an f-string hole in the code - so parameters are
+        compared by position, not by name."""
+        want = verb.upper() + " " + shape(route)
+        for op in self.operations:
+            if op.verb + " " + shape(op.path) == want:
+                return op
+        return None
+
+
+def shape(path: str) -> str:
+    out = ""
+    for segment in path.rstrip("/").split("/"):
+        if segment.startswith("{") or segment in ("%s", "%v", "%d", "*"):
+            out += "/*"
+        else:
+            out += "/" + segment
+    return out
+
+
+def read(path: str) -> Optional[Spec]:
+    with open(path, "r", encoding="utf-8") as handle:
+        text = handle.read()
+    if path.endswith(".json"):
+        try:
+            doc = json.loads(text)
+        except ValueError:
+            return None
+    else:
+        doc = parse(text)
+    if not isinstance(doc, dict):
+        return None
+    info = doc.get("info") or {}
+    spec = Spec(api=api_id(str(info.get("title", "")), str(info.get("version", ""))), path=path)
+    paths = doc.get("paths") or {}
+    if not isinstance(paths, dict):
+        return spec
+    for route, entry in paths.items():
+        if not isinstance(entry, dict):
+            continue
+        for verb in VERBS:
+            operation = entry.get(verb)
+            if not isinstance(operation, dict):
+                continue
+            tags = operation.get("tags") or []
+            tag = str(tags[0]) if isinstance(tags, list) and tags else ""
+            spec.operations.append(
+                Operation(id=str(operation.get("operationId") or (verb.upper() + " " + route)), tag=tag, verb=verb.upper(), path=route)
+            )
+    return spec
+
+
+def beside(directory: str) -> Optional[str]:
+    """The document vendored next to a client: any .yaml, .yml or .json in the
+    directory, the first by name so two of them do not make the fragment move."""
+    if not os.path.isdir(directory):
+        return None
+    for name in sorted(os.listdir(directory)):
+        if name.endswith((".yaml", ".yml", ".json")) and os.path.isfile(os.path.join(directory, name)):
+            return os.path.join(directory, name)
+    return None
+
+
+# --- the small reader -------------------------------------------------------
+
+
+def parse(text: str) -> Any:
+    lines = []
+    for raw in text.split("\n"):
+        if not raw.strip() or raw.lstrip().startswith("#"):
+            continue
+        lines.append((len(raw) - len(raw.lstrip(" ")), raw.strip()))
+    return Reader(lines).block(lines[0][0] if lines else 0)
+
+
+class Reader:
+    """A cursor over (indent, text) lines. Every branch advances it, so a
+    document it cannot make sense of ends the read rather than hanging it."""
+
+    def __init__(self, lines):
+        self.lines = lines
+        self.at = 0
+
+    def peek(self):
+        return self.lines[self.at] if self.at < len(self.lines) else None
+
+    def block(self, indent: int) -> Any:
+        node = self.peek()
+        if node is None or node[0] < indent:
+            return ""
+        return self.sequence(indent) if node[1].startswith("- ") else self.mapping(indent)
+
+    def mapping(self, indent: int) -> Dict[str, Any]:
+        out: Dict[str, Any] = {}
+        while True:
+            node = self.peek()
+            if node is None or node[0] < indent:
+                break
+            own, line = node
+            if own > indent:  # deeper than anything here claims: not ours to read
+                self.at += 1
+                continue
+            if line.startswith("- "):
+                break
+            key, sep, rest = line.partition(":")
+            self.at += 1
+            if not sep:
+                continue
+            key = str(scalar(key.strip()))
+            rest = rest.strip()
+            if rest in ("|", ">", "|-", ">-", "|+", ">+"):
+                self.skip(own)
+                out[key] = ""
+            elif rest:
+                out[key] = scalar(rest)
+            else:
+                nxt = self.peek()
+                if nxt is not None and nxt[0] > own:
+                    out[key] = self.block(nxt[0])
+                elif nxt is not None and nxt[0] == own and nxt[1].startswith("- "):
+                    out[key] = self.sequence(own)
+                else:
+                    out[key] = ""
+        return out
+
+    def sequence(self, indent: int) -> List[Any]:
+        out: List[Any] = []
+        while True:
+            node = self.peek()
+            if node is None or node[0] != indent or not node[1].startswith("- "):
+                break
+            self.at += 1
+            rest = node[1][2:].strip()
+            if ":" in rest and not rest.startswith(("[", "{", chr(34), "'")):
+                key, _, value = rest.partition(":")
+                out.append({str(scalar(key.strip())): scalar(value.strip())})
+                self.skip(indent)
+            elif rest:
+                out.append(scalar(rest))
+            else:
+                nxt = self.peek()
+                out.append(self.block(nxt[0]) if nxt is not None and nxt[0] > indent else "")
+        return out
+
+    def skip(self, indent: int) -> None:
+        """Everything nested under the line just read."""
+        while True:
+            node = self.peek()
+            if node is None or node[0] <= indent:
+                return
+            self.at += 1
+
+
+def scalar(text: str) -> Any:
+    text = text.strip()
+    if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'":
+        return text[1:-1]
+    if text.startswith("[") and text.endswith("]"):
+        return [scalar(item) for item in text[1:-1].split(",") if item.strip()]
+    if text.startswith("{") and text.endswith("}"):
+        return {}
+    return text

package/plugins/extract-django/openapi_test.py

@@ -0,0 +1,60 @@
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
+
+import openapi  # noqa: E402
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+DOCUMENT = os.path.join(HERE, "testdata", "billing", "invoices", "clients", "pricing", "openapi.yaml")
+
+
+class Ids(unittest.TestCase):
+    """The ids have to be the ones plugins/openapi/ids.go makes, or a call
+    would never resolve to the method that answers it."""
+
+    def test_api_is_the_title_and_the_major_version(self):
+        self.assertEqual(openapi.api_id("auth", "1.0.0"), "auth.v1")
+        self.assertEqual(openapi.api_id("Price List", "2.1.0"), "price-list.v2")
+        self.assertEqual(openapi.api_id("", ""), "api")
+
+    def test_an_interface_is_the_api_and_the_first_tag(self):
+        self.assertEqual(openapi.interface_id("auth.v1", "sessions"), "auth.v1.Sessions")
+        self.assertEqual(openapi.interface_id("auth.v1", "price_list"), "auth.v1.PriceList")
+        self.assertEqual(openapi.interface_id("auth.v1", ""), "auth.v1")
+
+    def test_a_parameter_is_compared_by_position(self):
+        # However either side spells the hole, and however the leading slash
+        # falls out of the split - the Go side spells it the same way.
+        self.assertEqual(openapi.shape("/v1/users/{userId}"), openapi.shape("/v1/users/{}"))
+        self.assertNotEqual(openapi.shape("/v1/users/{userId}"), openapi.shape("/v1/users/me"))
+
+
+class Read(unittest.TestCase):
+    def test_reads_the_document_a_client_is_vendored_beside(self):
+        spec = openapi.read(DOCUMENT)
+        self.assertEqual(spec.api, "pricing.v1")
+        self.assertEqual([o.id for o in spec.operations], ["createQuote"])
+        found = spec.find("POST", "/v1/quotes")
+        self.assertIsNotNone(found)
+        self.assertEqual(found.call_id(spec.api), "pricing.v1.Quotes/createQuote")
+
+    def test_a_route_the_document_does_not_declare_finds_nothing(self):
+        spec = openapi.read(DOCUMENT)
+        self.assertIsNone(spec.find("GET", "/v1/quotes"))
+        self.assertIsNone(spec.find("POST", "/v1/quotes/{}/lines"))
+
+    def test_an_operation_with_no_id_is_known_by_its_route(self):
+        spec = openapi.parse('info:\n  title: x\n  version: "1.0"\npaths:\n  /health:\n    get:\n      responses:\n        "200":\n          description: fine\n')
+        self.assertEqual(spec["info"]["title"], "x")
+        self.assertIn("/health", spec["paths"])
+
+    def test_beside_finds_the_document_in_the_client_directory(self):
+        self.assertEqual(openapi.beside(os.path.dirname(DOCUMENT)), DOCUMENT)
+        self.assertIsNone(openapi.beside(os.path.join(HERE, "testdata", "billing", "config")))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/plugins/extract-django/operations.py

@@ -0,0 +1,96 @@
+"""What somebody can ask the service to do.
+
+`<app>/services.py` - the module a Django project keeps its use cases in once
+they outgrow the view - holds one function per scenario, and each is an
+operation of the application's aggregate. It is a command when it writes and a
+query when it does not, which is read off the calls it makes rather than off
+its name.
+"""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+import catalog
+from apps import App
+from domain import Aggregate
+from ids import camel
+from source import Module, doc, dotted
+
+# The same core port-write vocabulary as the Go, TS, Rust and Java extractors.
+PORT_WRITES = {
+    "save",
+    "delete",
+    "create",
+    "update",
+    "publish",
+    "remove",
+    "insert",
+    "upsert",
+}
+
+# Django adds explicit ORM and signal mutations; these are framework syntax,
+# not a competing definition of command/query.
+FRAMEWORK_WRITES = {
+    "get_or_create",
+    "update_or_create",
+    "bulk_create",
+    "bulk_update",
+    "add",
+    "set",
+    "send",
+    "send_robust",
+}
+WRITES = PORT_WRITES | FRAMEWORK_WRITES
+
+
+@dataclass
+class UseCase:
+    name: str  # issue_invoice
+    id: str  # IssueInvoice
+    node: ast.AST
+    module: Module
+    app: App
+    doc: str
+    kind: str
+
+    @property
+    def key(self) -> str:
+        return self.module.dotted + "." + self.name
+
+
+def writes(node: ast.AST) -> bool:
+    for child in ast.walk(node):
+        if isinstance(child, ast.Call):
+            name = dotted(child.func)
+            if name.split(".")[-1] in WRITES:
+                return True
+    return False
+
+
+def read_use_cases(agg: Aggregate, b) -> List[UseCase]:
+    out = []
+    for module in agg.app.package("services"):
+        for node in module.functions():
+            if node.name.startswith("_"):
+                continue
+            out.append(
+                UseCase(
+                    name=node.name,
+                    id=camel(node.name),
+                    node=node,
+                    module=module,
+                    app=agg.app,
+                    doc=doc(node),
+                    kind="command" if writes(node) else "query",
+                )
+            )
+    if not out:
+        b.warn(agg.id, "no services module under %s: the aggregate has no operations, only whatever the views do inline" % agg.app.rel)
+    return sorted(out, key=lambda u: u.id)
+
+
+def operation(use_case: UseCase, exposed_by: Optional[List[str]] = None) -> Dict[str, object]:
+    return catalog.operation(use_case.id, use_case.kind, use_case.doc, exposed_by)

package/plugins/extract-django/options.py

@@ -0,0 +1,61 @@
+"""What the manifest tells the extractor: the things a Django project does
+not say about the estate it belongs to."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List
+
+from protocol import read_options
+
+
+@dataclass
+class Options:
+    context: str = ""
+    context_name: str = ""
+    context_summary: str = ""
+    classification: str = ""
+    service: str = ""
+    service_name: str = ""
+    repo: str = ""
+    store: str = ""
+    store_name: str = ""
+    store_kind: str = ""  # "" reads DATABASES["default"]["ENGINE"] off the settings
+    apps: List[str] = field(default_factory=list)
+    aggregates: Dict[str, str] = field(default_factory=dict)
+    peers: Dict[str, str] = field(default_factory=dict)
+    events: Dict[str, str] = field(default_factory=dict)
+    flow_wrappers: Dict[str, Any] = field(default_factory=dict)
+    source: str = "."
+    settings: str = ""
+    out: str = "domain.json"
+    stores_out: str = "stores.json"
+    openapi_out: str = "openapi.inferred.yaml"
+
+    # The option as the manifest spells it, against the field that holds it.
+    KEYS = {
+        "context": "context",
+        "contextName": "context_name",
+        "contextSummary": "context_summary",
+        "classification": "classification",
+        "service": "service",
+        "serviceName": "service_name",
+        "repo": "repo",
+        "store": "store",
+        "storeName": "store_name",
+        "storeKind": "store_kind",
+        "apps": "apps",
+        "aggregates": "aggregates",
+        "peers": "peers",
+        "events": "events",
+        "flowWrappers": "flow_wrappers",
+        "source": "source",
+        "settings": "settings",
+        "out": "out",
+        "storesOut": "stores_out",
+        "openapiOut": "openapi_out",
+    }
+
+    @staticmethod
+    def of(raw: Any) -> "Options":
+        return read_options(Options(), Options.KEYS, raw)