@shortlink-org/portolan 0.1.0

package/plugins/extract-django/flows_test.py

@@ -0,0 +1,338 @@
+"""Project-call traversal used to turn Django handlers into useful flows."""
+
+import ast
+import os
+import sys
+import unittest
+from types import SimpleNamespace
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
+
+import flows  # noqa: E402
+from domain import ModelDef  # noqa: E402
+from source import Module, node_name, read_imports  # noqa: E402
+
+
+def module(name, source):
+    found = Module(path="/" + name + ".py", rel=name + ".py", dotted=name, tree=ast.parse(source))
+    found.imports = read_imports(found)
+    return found
+
+
+class Project:
+    def __init__(self, modules):
+        self.modules = {item.dotted: item for item in modules}
+
+    def module(self, name):
+        return self.modules.get(name) or self.modules.get(name + ".__init__")
+
+    def resolve(self, owner, name):
+        imported = owner.imports.get(name)
+        if imported is None:
+            if any(node_name(node) == name for node in owner.tree.body):
+                return owner, name
+            return None
+        target = self.module(imported.module)
+        if target is None:
+            return None
+        return target, name if imported.name == "*" else imported.name
+
+
+class Warnings:
+    def warn(self, *_args):
+        pass
+
+
+class FlowCalls(unittest.TestCase):
+    def test_self_and_imported_helpers_reach_http_and_cycles_are_cut(self):
+        views = module(
+            "app.views",
+            """
+from app.helpers import send
+
+class Tools:
+    @staticmethod
+    def deliver():
+        return send()
+
+class BaseView:
+    @classmethod
+    def dispatch(cls):
+        return Tools.deliver()
+
+class StatusView(BaseView):
+    def get(self, request):
+        return self.forward()
+
+    def forward(self):
+        return super().dispatch()
+""",
+        )
+        helpers = module(
+            "app.helpers",
+            """
+import os
+from requests import post
+
+BASE = os.environ.get("PEER_URL", "https://peer.example")
+
+def send():
+    post(BASE + "/v1/status")
+    again()
+
+def again():
+    send()
+""",
+        )
+        project = Project([views, helpers])
+        reader = flows.FlowReader(
+            flows.Options(context="app", svc_id="app.web", service="web", store=""),
+            project,
+            [],
+            [],
+            [],
+            [],
+            {},
+            lambda path: path,
+            Warnings(),
+        )
+        view = next(node for node in views.classes() if node.name == "StatusView")
+        handler = next(node for node in view.body if getattr(node, "name", "") == "get")
+        endpoint = SimpleNamespace(id="status_get", module=views, node=handler, view="StatusView", doc="", use_cases=[])
+
+        flow = reader.endpoint_flow(endpoint)
+
+        self.assertEqual([step["label"] for step in flow["steps"]], ["status_get", "POST /v1/status"])
+        self.assertEqual(flow["steps"][1]["status"], "unresolved")
+        self.assertEqual(flow["steps"][1]["line"], "app.helpers.py:8")
+        self.assertEqual([lane["label"] for lane in flow["participants"] if lane.get("label")], ["peer.example"])
+
+    def test_one_sided_condition_keeps_an_explicit_otherwise_branch(self):
+        views = module(
+            "app.views",
+            """
+from requests import get
+
+class StatusView:
+    def get(self, request):
+        if request.ready:
+            get("https://peer.example/ready")
+""",
+        )
+        project = Project([views])
+        reader = flows.FlowReader(
+            flows.Options(context="app", svc_id="app.web", service="web", store=""),
+            project,
+            [],
+            [],
+            [],
+            [],
+            {},
+            lambda path: path,
+            Warnings(),
+        )
+        view = views.classes()[0]
+        handler = next(node for node in view.body if getattr(node, "name", "") == "get")
+        endpoint = SimpleNamespace(id="status_get", module=views, node=handler, view="StatusView", doc="", use_cases=[])
+
+        flow = reader.endpoint_flow(endpoint)
+
+        alternative = flow["steps"][1]
+        self.assertEqual([branch["title"] for branch in alternative["branches"]], ["request.ready", "otherwise"])
+        self.assertEqual(alternative["branches"][1]["steps"], [])
+
+    def test_routed_models_are_visible_without_an_aggregate(self):
+        models = module(
+            "app.models",
+            """
+class Widget:
+    pass
+""",
+        )
+        views = module(
+            "app.views",
+            """
+from .models import Widget
+
+class WidgetView:
+    def get(self, request):
+        return Widget.objects.get(id=request.id)
+""",
+        )
+        app = SimpleNamespace(dotted="app")
+        model = ModelDef(name="Widget", node=models.classes()[0], module=models, app=app)
+        project = Project([models, views])
+        reader = flows.FlowReader(
+            flows.Options(context="app", svc_id="app.web", service="web", store=""),
+            project,
+            [],
+            [model],
+            [],
+            [],
+            {},
+            lambda path: path,
+            Warnings(),
+        )
+        view = views.classes()[0]
+        handler = next(node for node in view.body if getattr(node, "name", "") == "get")
+        endpoint = SimpleNamespace(id="widget_get", module=views, node=handler, view="WidgetView", doc="", use_cases=[])
+
+        flow = reader.endpoint_flow(endpoint)
+
+        self.assertEqual([step["label"] for step in flow["steps"]], ["widget_get", "Widget.objects.get"])
+
+    def test_configured_wrappers_are_semantic_and_technical_ones_are_counted(self):
+        views = module(
+            "app.views",
+            """
+from app.helpers import record_log, send_mail
+
+class StatusView:
+    def post(self, request):
+        record_log("started")
+        send_mail(request.email)
+        record_log("finished")
+""",
+        )
+        helpers = module(
+            "app.helpers",
+            """
+from requests import post
+
+def record_log(message):
+    post("https://logger.example/logs", json={"message": message})
+
+def send_mail(email):
+    post("https://mailer.example/messages", json={"email": email})
+""",
+        )
+        reader = flows.FlowReader(
+            flows.Options(
+                context="app",
+                svc_id="app.web",
+                service="web",
+                store="",
+                flow_wrappers={
+                    "app.helpers.record_log": {"label": "Record application log", "technical": True},
+                    "app.helpers.send_mail": {"label": "Send email", "target": "messaging.mailer"},
+                },
+            ),
+            Project([views, helpers]),
+            [],
+            [],
+            [],
+            [],
+            {},
+            lambda path: path,
+            Warnings(),
+        )
+        view = views.classes()[0]
+        handler = next(node for node in view.body if getattr(node, "name", "") == "post")
+        endpoint = SimpleNamespace(id="status_post", module=views, node=handler, view="StatusView", doc="", use_cases=[])
+
+        flow = reader.endpoint_flow(endpoint)
+
+        self.assertEqual(
+            [step["label"] for step in flow["steps"]],
+            ["status_post", "Record application log ×2", "Send email"],
+        )
+        self.assertEqual(flow["steps"][1]["from"], "app.web")
+        self.assertEqual(flow["steps"][1]["to"], "app.web")
+        self.assertIn("2 call sites are collapsed", flow["steps"][1]["note"])
+        self.assertEqual(flow["steps"][2]["to"], "messaging.mailer")
+        self.assertNotIn("logger.example", [lane.get("label") for lane in flow["participants"]])
+
+    def test_task_body_is_a_source_backed_continuation(self):
+        tasks = module(
+            "app.tasks",
+            """
+from celery import shared_task
+from requests import post
+
+class Refresher:
+    def run(self):
+        for item in self.items():
+            pass
+
+    def items(self):
+        return post("https://peer.example/refresh")
+
+@shared_task(name="app.refresh")
+def refresh():
+    refresher = Refresher()
+    refresher.run()
+""",
+        )
+        reader = flows.FlowReader(
+            flows.Options(context="app", svc_id="app.web", service="web", store=""),
+            Project([tasks]),
+            [],
+            [],
+            [],
+            [],
+            {},
+            lambda path: path,
+            Warnings(),
+        )
+
+        found = reader.task_flows()
+
+        self.assertEqual(len(found), 1)
+        self.assertEqual(found[0]["entrypoint"], "python:app.tasks:refresh")
+        self.assertEqual([step["label"] for step in found[0]["steps"]], ["POST /refresh"])
+
+    def test_kafka_factory_inside_a_task_is_a_message_handoff(self):
+        settings = module(
+            "app.settings",
+            """
+import os
+TOPIC = os.environ.get("TOPIC", "records.changed")
+""",
+        )
+        tasks = module(
+            "app.tasks",
+            """
+from celery import shared_task
+from confluent_kafka import Producer
+from django.conf import settings
+
+class Factory:
+    @classmethod
+    def get(cls):
+        config = {"bootstrap.servers": "kafka:9092", "enable.idempotence": True}
+        return Producer(config)
+
+def publish(payload):
+    producer = Factory.get()
+    producer.produce(topic=settings.TOPIC, value=payload, headers=[("schema", b"v1")])
+
+@shared_task(name="app.deliver")
+def deliver(payload):
+    publish(payload)
+""",
+        )
+        reader = flows.FlowReader(
+            flows.Options(context="app", svc_id="app.web", service="web", store="", settings="app.settings"),
+            Project([tasks, settings]),
+            [],
+            [],
+            [],
+            [],
+            {},
+            lambda path: path,
+            Warnings(),
+        )
+
+        found = reader.task_flows()
+
+        self.assertEqual(len(found), 1)
+        step = found[0]["steps"][0]
+        self.assertEqual(step["to"], "kafka-records-changed")
+        self.assertEqual(step["handoff"], {"kind": "message", "transport": "kafka", "channel": "records.changed", "message": "payload", "direction": "send"})
+        self.assertIn("idempotence: enabled", step["note"])
+        self.assertEqual(step["line"], "app.tasks.py:14")
+
+
+if __name__ == "__main__":
+    unittest.main()

package/plugins/extract-django/ids.py

@@ -0,0 +1,42 @@
+"""How a name in the source becomes an id in the catalog.
+
+The spelling itself lives in `pyplugin/names.py`, shared with every Python
+plugin; what is here is the Django half - the plural an application is named
+in, and the ids the fragment composes.
+"""
+
+from __future__ import annotations
+
+from names import camel, slug, title  # noqa: F401
+
+
+def pascal(name: str) -> str:
+    """A directory name in PascalCase, which is what its root model is called."""
+    return camel(name)
+
+
+def singular(name: str) -> str:
+    """invoices -> invoice: a Django app is named for many of the thing it holds."""
+    if name.endswith("ies") and len(name) > 3:
+        return name[:-3] + "y"
+    if name.endswith("sses") or name.endswith("shes") or name.endswith("ches") or name.endswith("xes"):
+        return name[:-2]
+    if name.endswith("s") and not name.endswith("ss"):
+        return name[:-1]
+    return name
+
+
+def service_id(context: str, service: str) -> str:
+    return context + "." + service
+
+
+def aggregate_id(service: str, aggregate: str) -> str:
+    return service + "." + aggregate
+
+
+def block_id(aggregate: str, block: str) -> str:
+    return aggregate + "." + block
+
+
+def event_id(aggregate: str, name: str) -> str:
+    return aggregate + "." + name

package/plugins/extract-django/ids_test.py

@@ -0,0 +1,30 @@
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
+
+import ids  # noqa: E402
+
+
+class Ids(unittest.TestCase):
+    def test_slug_is_the_one_extract_go_makes(self):
+        self.assertEqual(ids.slug("PriceList"), "price-list")
+        self.assertEqual(ids.slug("Address"), "address")
+        self.assertEqual(ids.slug("ID"), "id")
+        self.assertEqual(ids.slug("email.Address"), "email-address")
+        self.assertEqual(ids.slug("InvoiceLine"), "invoice-line")
+
+    def test_camel_and_title(self):
+        self.assertEqual(ids.camel("issue_invoice"), "IssueInvoice")
+        self.assertEqual(ids.title("price_list"), "Price List")
+
+    def test_an_application_is_named_for_many_of_the_thing_it_holds(self):
+        self.assertEqual(ids.singular("invoices"), "invoice")
+        self.assertEqual(ids.singular("policies"), "policy")
+        self.assertEqual(ids.singular("billing"), "billing")
+
+
+if __name__ == "__main__":
+    unittest.main()

package/plugins/extract-django/lifecycle.py

@@ -0,0 +1,222 @@
+"""The root's states, read off the table the code keeps.
+
+Never off the branches of the methods: the table is the claim and the methods
+are held to it. In Django the table is either a `TRANSITIONS` mapping on the
+model, beside the `TextChoices` that names the states, or the `@transition`
+decorators of django-fsm, which are a table written one edge at a time. An edge
+nothing makes, a move into a state the table lacks, and a status assigned
+anywhere but a mover are each reported.
+"""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass, field as dc_field
+from typing import Dict, List, Optional, Tuple
+
+import catalog
+from domain import Aggregate, ModelDef
+from source import assigned, bases, const_str, dotted, inner_class, keyword, methods
+
+
+@dataclass
+class Move:
+    to: str
+    on: str
+    emits: str
+    source: str
+    sources: List[str] = dc_field(default_factory=list)  # the states it may be made from, when the decorator says
+
+
+def choices_of(model: ModelDef, name: str) -> Dict[str, str]:
+    """A TextChoices class, as member name to the value stored in the column."""
+    node = inner_class(model.node, name)
+    if node is None:
+        for other in model.module.classes():
+            if other.name == name:
+                node = other
+                break
+    if node is None or not any(b.split(".")[-1].endswith("Choices") for b in bases(node)):
+        return {}
+    out = {}
+    for member, value, _ in assigned(node):
+        if isinstance(value, ast.Constant) and isinstance(value.value, str):
+            out[member] = value.value
+        elif isinstance(value, ast.Tuple) and value.elts and isinstance(value.elts[0], ast.Constant):
+            out[member] = value.elts[0].value
+    return out
+
+
+def state_of(node: ast.AST, choices: Dict[str, Dict[str, str]]) -> str:
+    """`Status.DRAFT` or `"draft"`, either way the value in the column."""
+    literal = const_str(node)
+    if literal:
+        return literal
+    name = dotted(node)
+    if "." in name:
+        holder, member = name.rsplit(".", 1)
+        table = choices.get(holder.split(".")[-1], {})
+        if member in table:
+            return table[member]
+    return ""
+
+
+def status_field(model: ModelDef) -> Optional[str]:
+    """The field whose choices are the states. `status` by convention, or the
+    one field whose choices name a Choices class."""
+    for f in model.fields:
+        value = keyword(f.call, "choices")
+        if value is not None and dotted(value):
+            return f.name
+    return "status" if model.field("status") else None
+
+
+def choice_tables(model: ModelDef) -> Dict[str, Dict[str, str]]:
+    out = {}
+    for node in list(model.node.body) + list(model.module.tree.body):
+        if isinstance(node, ast.ClassDef) and any(b.split(".")[-1].endswith("Choices") for b in bases(node)):
+            out[node.name] = choices_of(model, node.name)
+    return out
+
+
+def declared_table(model: ModelDef, choices: Dict[str, Dict[str, str]]) -> Dict[str, List[str]]:
+    """`TRANSITIONS = {Status.DRAFT: [Status.ISSUED], ...}`, in the order written."""
+    for name, value, _ in assigned(model.node):
+        if name != "TRANSITIONS" or not isinstance(value, ast.Dict):
+            continue
+        table: Dict[str, List[str]] = {}
+        for key, targets in zip(value.keys, value.values):
+            state = state_of(key, choices)
+            if not state:
+                continue
+            listed = []
+            if isinstance(targets, (ast.List, ast.Tuple, ast.Set)):
+                for item in targets.elts:
+                    to = state_of(item, choices)
+                    if to:
+                        listed.append(to)
+            table[state] = listed
+        return table
+    return {}
+
+
+def emits_of(node: ast.AST, events: Dict[str, object]) -> str:
+    """The event a mover hands back, as its return annotation names it, by the
+    id the catalog knows it under."""
+    annotation = getattr(node, "returns", None)
+    if annotation is None:
+        return ""
+    found = ""
+    for name in ast.walk(annotation):
+        if isinstance(name, ast.Name) and name.id in events:
+            found = events[name.id].id
+        elif isinstance(name, ast.Constant) and isinstance(name.value, str) and name.value in events:
+            found = events[name.value].id
+    return found
+
+
+def movers(model: ModelDef, status: str, choices: Dict[str, Dict[str, str]], events: Dict[str, object]) -> Tuple[List[Move], List[Tuple[str, str]]]:
+    """Every method that moves the root: the ones assigning the status field,
+    and the ones django-fsm decorates. Also what they assign that no table
+    knows, which the caller reports."""
+    out: List[Move] = []
+    loose: List[Tuple[str, str]] = []
+    for node in methods(model.node):
+        if node.name.startswith("_"):
+            continue
+        emits = emits_of(node, events)
+        where = model.module.where(node)
+        decorator = None
+        for dec in getattr(node, "decorator_list", []):
+            if dotted(dec).split(".")[-1] == "transition" and isinstance(dec, ast.Call):
+                decorator = dec
+        if decorator is not None:
+            target = state_of(keyword(decorator, "target"), choices)
+            source = keyword(decorator, "source")
+            froms = []
+            if isinstance(source, (ast.List, ast.Tuple, ast.Set)):
+                froms = [state_of(item, choices) for item in source.elts]
+            elif source is not None:
+                one = state_of(source, choices)
+                froms = [one] if one else []
+            if target:
+                out.append(Move(to=target, on=node.name, emits=emits, source=where, sources=[f for f in froms if f]))
+            continue
+        for assignment in ast.walk(node):
+            if not isinstance(assignment, ast.Assign):
+                continue
+            for target in assignment.targets:
+                if not isinstance(target, ast.Attribute) or target.attr != status:
+                    continue
+                if dotted(target.value) != "self":
+                    continue
+                to = state_of(assignment.value, choices)
+                if to:
+                    out.append(Move(to=to, on=node.name, emits=emits, source=model.module.where(assignment)))
+                else:
+                    loose.append((node.name, model.module.where(assignment)))
+    return out, loose
+
+
+def read(agg: Aggregate, events: Dict[str, object], b) -> Optional[Dict[str, object]]:
+    model = agg.root
+    status = status_field(model)
+    if status is None:
+        return None
+    choices = choice_tables(model)
+    table = declared_table(model, choices)
+    moves, loose = movers(model, status, choices, events)
+
+    if not table:
+        if not moves:
+            return None
+        # django-fsm writes the table one edge at a time; a model that only
+        # assigns its status has no table at all, and a lifecycle read off the
+        # assignments would be the branches talking.
+        if not any(m.sources for m in moves):
+            b.warn(agg.id, "%s moves its %s but declares no TRANSITIONS: the table is the claim, the methods are held to it" % (model.name, status))
+            return None
+        table = {}
+        for move in moves:
+            for source in move.sources:
+                table.setdefault(source, [])
+                if move.to not in table[source]:
+                    table[source].append(move.to)
+            table.setdefault(move.to, [])
+
+    states = list(table.keys())
+    # The order the Choices class writes them in is the order a reader expects,
+    # and the first is where a new root starts.
+    declared = next(iter(choices.values()), {})
+    if declared:
+        ordered = [value for value in declared.values() if value in table or any(value in tos for tos in table.values())]
+        for state in states:
+            if state not in ordered:
+                ordered.append(state)
+        states = ordered
+    for move in moves:
+        if move.to not in states:
+            states.append(move.to)
+            b.warn(agg.id, "%s moves to %r, which the table does not list" % (move.on, move.to))
+
+    transitions = []
+    made = set()
+    for move in moves:
+        froms = [state for state, tos in table.items() if move.to in tos]
+        if move.sources:
+            froms = [f for f in move.sources if f in states] or move.sources
+        if not froms:
+            continue
+        for state in froms:
+            transitions.append(catalog.transition(state, move.to, move.on, move.emits, move.source))
+            made.add((state, move.to))
+    for state, tos in table.items():
+        for to in tos:
+            if (state, to) not in made:
+                b.warn(agg.id, "the table has %s -> %s and no method of %s makes it" % (state, to, model.name))
+    for name, where in loose:
+        b.warn(agg.id, "%s assigns %s to something the states do not name (%s)" % (name, status, where))
+
+    if not transitions:
+        return None
+    return {"states": states, "transitions": transitions}