@shortlink-org/portolan 0.1.0

package/plugins/extract-django/options.schema.json

@@ -0,0 +1,126 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "context": {
+      "type": "string",
+      "description": "Slug of the bounded context this service belongs to. Defaults to the name of the input directory.",
+      "examples": ["shop"]
+    },
+    "contextName": {
+      "type": "string",
+      "description": "What the context is called on the page. Defaults to the slug, titled."
+    },
+    "contextSummary": {
+      "type": "string",
+      "description": "A sentence or two on what the context is responsible for."
+    },
+    "classification": {
+      "enum": ["core", "supporting", "generic"],
+      "description": "Whether the context is core to the business, supports it, or is generic and could be bought."
+    },
+    "service": {
+      "type": "string",
+      "description": "Slug of the service. Defaults to the name of the input directory.",
+      "examples": ["billing"]
+    },
+    "serviceName": {
+      "type": "string",
+      "description": "What the service is called on the page. Defaults to the title of its README, and failing that to the slug, titled."
+    },
+    "repo": {
+      "type": "string",
+      "description": "Where the source lives. Defaults to the repository url in pyproject.toml.",
+      "examples": ["github.com/shortlink-org/portolan"]
+    },
+    "store": {
+      "type": "string",
+      "description": "Slug of the database this service keeps its state in. Defaults from the kind of DATABASES['default'] when that setting is statically readable. The models are read as its schema, and a call into the ORM lands on its lane.",
+      "examples": ["pg"]
+    },
+    "storeName": {
+      "type": "string",
+      "description": "What the database is called on the page. Defaults to the service, titled, and the word database."
+    },
+    "storeKind": {
+      "enum": ["postgres", "mysql", "sqlite", "redis", "mongodb", "clickhouse", "s3", "other"],
+      "description": "Which database the models are the schema of. Read off DATABASES['default']['ENGINE'] in the settings module when left out; given here, it wins, and a disagreement with the settings is reported. It picks the column types a field becomes; only postgres has a mapping, and a kind without one keeps the field's own name and says so."
+    },
+    "apps": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "The Django applications to read, as import packages, for a project that keeps them somewhere a models module would not be found. Every package holding one is read when this is left out.",
+      "examples": [["billing.invoices"]]
+    },
+    "aggregates": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Which model is the root of an application's aggregate, as the application to the model. Only needed where the application is not named after its root - `invoices` finds `Invoice` on its own.",
+      "examples": [{ "invoices": "Invoice" }]
+    },
+    "peers": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Which service answers for what this one calls, as the api id of the document vendored beside the client to the callee's service id. A client with no line here is called as unknown and its steps stay unresolved.",
+      "examples": [{ "pricing.v1": "shop.pricing" }]
+    },
+    "events": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Which aggregate another service's events belong to, as the module a receiver imports the signal from to the aggregate id, so a policy reacting to a foreign event resolves.",
+      "examples": [{ "payments.events": "payments.ledger.payment" }]
+    },
+    "flowWrappers": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "label": {
+            "type": "string",
+            "description": "Business-facing label shown instead of expanding the wrapper's implementation. Defaults to the callable name, titled."
+          },
+          "target": {
+            "type": "string",
+            "description": "Service id the wrapper calls. Omit for a semantic operation inside this service."
+          },
+          "technical": {
+            "type": "boolean",
+            "default": false,
+            "description": "Collapse every call to this wrapper in one flow into one counted step."
+          }
+        }
+      },
+      "description": "Project callables that form a meaningful flow boundary, keyed as module.function or module.Class.method. Their internals are replaced by one semantic step; technical wrappers are also deduplicated per flow.",
+      "examples": [{
+        "util.logger.send_log": { "label": "Record application log", "technical": true },
+        "mailer.client.send": { "label": "Send email", "target": "messaging.mailer" }
+      }]
+    },
+    "source": {
+      "type": "string",
+      "description": "The directory the applications are looked for in, relative to the input root.",
+      "default": "."
+    },
+    "settings": {
+      "type": "string",
+      "description": "The Django settings module Celery is configured from, as an import spells it, so an enqueue lands on the queue the routes give it. Defaults to what manage.py sets DJANGO_SETTINGS_MODULE to.",
+      "examples": ["config.settings"]
+    },
+    "out": {
+      "type": "string",
+      "description": "Name of the fragment file, inside the step's output directory.",
+      "default": "domain.json"
+    },
+    "storesOut": {
+      "type": "string",
+      "description": "Name of the fragment holding the database the models describe, inside the step's output directory. Written when store is set or inferred from DATABASES['default'].",
+      "default": "stores.json"
+    },
+    "openapiOut": {
+      "type": "string",
+      "description": "Name of the inferred OpenAPI 3.1 document, inside the step's output directory. Written when URLConf exposes HTTP operations.",
+      "default": "openapi.inferred.yaml"
+    }
+  }
+}

package/plugins/extract-django/routing.py

@@ -0,0 +1,214 @@
+"""Django URLConf, read without importing the project.
+
+The runtime route table is assembled from ``path``/``re_path``, nested
+``include`` calls and DRF routers.  Importing it would execute settings and
+application code, so this module follows the same declarations through the
+AST instead.  The result is deliberately the wire fact only: target view,
+HTTP path and the URLConf line that declared it.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set, Tuple
+
+from celery_conf import follow, settings_module_name, str_value
+from source import Module, Project, const_str, dotted, keyword
+
+
+@dataclass(frozen=True)
+class Route:
+    module: str
+    view: str
+    path: str
+    source: str
+    name: str = ""
+    router: bool = False
+    basename: str = ""
+    parameters: Tuple[Tuple[str, str], ...] = ()
+
+
+@dataclass
+class Routes:
+    root: Optional[Module]
+    entries: List[Route]
+    runtime_schema: str = ""
+
+    def for_view(self, module: str, view: str) -> List[Route]:
+        return [route for route in self.entries if route.module == module and route.view == view]
+
+
+def read(project: Project, settings: str = "") -> Routes:
+    root = root_urlconf(project, settings)
+    if root is None:
+        return Routes(None, [])
+    entries: List[Route] = []
+    runtime_schema = ""
+    visit(project, root, "", (), entries, set())
+    for node in ast.walk(root.tree):
+        if isinstance(node, ast.Call) and dotted(node.func).split(".")[-1] in ("get_schema_view", "SpectacularAPIView"):
+            runtime_schema = root.where(node)
+            break
+    return Routes(root, entries, runtime_schema)
+
+
+def root_urlconf(project: Project, settings: str = "") -> Optional[Module]:
+    settings_name = settings or settings_module_name(project)
+    module = project.module(settings_name) if settings_name else None
+    if module is not None:
+        for node in module.tree.body:
+            if not isinstance(node, (ast.Assign, ast.AnnAssign)):
+                continue
+            targets = node.targets if isinstance(node, ast.Assign) else [node.target]
+            if not any(isinstance(target, ast.Name) and target.id == "ROOT_URLCONF" for target in targets):
+                continue
+            value = node.value
+            name = str_value(follow(value, module), module)
+            if name and project.module(name) is not None:
+                return project.module(name)
+        package = settings_name.rsplit(".", 1)[0] if "." in settings_name else ""
+        if package and project.module(package + ".urls") is not None:
+            return project.module(package + ".urls")
+
+    # Conventional fallback for small projects whose settings are assembled
+    # dynamically: prefer a URLConf beside manage.py's package, otherwise the
+    # only non-application URLConf in the tree.
+    candidates = [m for m in project.modules.values() if m.dotted.endswith(".urls") or m.dotted == "urls"]
+    return candidates[0] if len(candidates) == 1 else None
+
+
+def visit(project: Project, module: Module, prefix: str, prefix_parameters: Tuple[Tuple[str, str], ...], out: List[Route], seen: Set[Tuple[str, str]]) -> None:
+    key = (module.dotted, prefix)
+    if key in seen:
+        return
+    seen.add(key)
+
+    for node in ast.walk(module.tree):
+        if not isinstance(node, ast.Call) or dotted(node.func).split(".")[-1] not in ("path", "re_path", "url"):
+            continue
+        if len(node.args) < 2:
+            continue
+        # An empty string is a route too: `path("", include("invoices.urls"))`
+        # mounts an application at the root, and `path("", index)` is the
+        # root itself.  Only a pattern that is not a literal is skipped.
+        first = node.args[0]
+        if not (isinstance(first, ast.Constant) and isinstance(first.value, str)):
+            continue
+        fragment = first.value
+        regex = dotted(node.func).split(".")[-1] != "path"
+        path = join_path(prefix, fragment, regex)
+        parameters = prefix_parameters + route_parameters(fragment, regex)
+        target = node.args[1]
+        if isinstance(target, ast.Call) and dotted(target.func).split(".")[-1] == "include":
+            included = include_module(project, module, target)
+            if included is not None:
+                visit(project, included, path, parameters, out, seen)
+            continue
+        resolved = view_target(project, module, target)
+        if resolved is None:
+            continue
+        target_module, view = resolved
+        out.append(Route(target_module, view, path, module.where(node), const_str(keyword(node, "name")), parameters=parameters))
+
+    # ``urlpatterns = router.urls`` contributes routes without a path() call.
+    # Each registration is mounted at every prefix by which this URLConf was
+    # reached from the root.
+    for node in ast.walk(module.tree):
+        if not isinstance(node, ast.Call) or dotted(node.func).split(".")[-1] != "register" or len(node.args) < 2:
+            continue
+        route_prefix = const_str(node.args[0]).strip("^/$")
+        resolved = view_target(project, module, node.args[1])
+        if not route_prefix or resolved is None:
+            continue
+        target_module, view = resolved
+        basename = const_str(keyword(node, "basename")) or route_prefix.replace("/", "_")
+        out.append(Route(target_module, view, join_path(prefix, route_prefix + "/"), module.where(node), router=True, basename=basename, parameters=prefix_parameters))
+
+
+def route_parameters(fragment: str, regex: bool = False) -> Tuple[Tuple[str, str], ...]:
+    if regex:
+        return tuple((match.group(1), "str") for match in re.finditer(r"\(\?P<([A-Za-z_][A-Za-z0-9_]*)>[^)]+\)", fragment))
+    return tuple(
+        (match.group(2), match.group(1) or "str")
+        for match in re.finditer(r"<(?:(str|int|slug|uuid|path):)?([A-Za-z_][A-Za-z0-9_]*)>", fragment)
+    )
+
+
+def include_module(project: Project, module: Module, call: ast.Call) -> Optional[Module]:
+    if not call.args:
+        return None
+    value = call.args[0]
+    if isinstance(value, (ast.Tuple, ast.List)) and value.elts:
+        value = value.elts[0]
+    name = const_str(value)
+    if name:
+        return project.module(name)
+    resolved = imported_name(project, module, dotted(value))
+    return project.module(resolved) if resolved else None
+
+
+def view_target(project: Project, module: Module, node: ast.AST) -> Optional[Tuple[str, str]]:
+    # Decorators and ``SomeView().as_view()`` wrap the same target.  Peeling
+    # calls is safe here because only names are read; nothing is executed.
+    while isinstance(node, ast.Call):
+        name = dotted(node.func)
+        if name.endswith(".as_view"):
+            node = node.func.value  # type: ignore[union-attr]
+            continue
+        if node.args and name.split(".")[-1] in ("csrf_exempt", "login_required", "permission_required"):
+            node = node.args[0]
+            continue
+        if isinstance(node.func, ast.Attribute) and node.func.attr == "as_view":
+            node = node.func.value
+            continue
+        break
+    name = dotted(node)
+    if not name:
+        return None
+    parts = name.split(".")
+    resolved = imported_name(project, module, name)
+    if resolved:
+        parts = resolved.split(".")
+        for end in range(len(parts) - 1, 0, -1):
+            target_module = ".".join(parts[:end])
+            if project.module(target_module) is not None:
+                return target_module, ".".join(parts[end:])
+    if len(parts) == 1:
+        return module.dotted, parts[0]
+    return None
+
+
+def imported_name(project: Project, module: Module, name: str) -> str:
+    parts = name.split(".")
+    imported = module.imports.get(parts[0])
+    if imported is None:
+        return name if project.module(".".join(parts[:-1])) is not None else ""
+    base = imported.module
+    if imported.name != "*":
+        base = ".".join(part for part in (base, imported.name) if part)
+    rest = parts[1:]
+    # ``from avia.views import booking`` names the submodule
+    # ``avia.views.booking``; ``from .views import View`` names the class in
+    # ``app.views``.  Prefer the longest prefix that is an actual module.
+    candidates = []
+    combined = [part for part in base.split(".") if part] + rest
+    for end in range(len(combined), 0, -1):
+        prefix = ".".join(combined[:end])
+        if project.module(prefix) is not None:
+            candidates.append(prefix + ("." + ".".join(combined[end:]) if combined[end:] else ""))
+    return candidates[0] if candidates else ".".join(combined)
+
+
+def join_path(prefix: str, fragment: str, regex: bool = False) -> str:
+    value = fragment
+    if regex:
+        value = value.removeprefix("^").removesuffix("$")
+        value = re.sub(r"\(\?P<([A-Za-z_][A-Za-z0-9_]*)>[^)]+\)", r"{\1}", value)
+        value = value.replace("\\/", "/")
+        value = re.sub(r"\\[AbZ]", "", value)
+    value = re.sub(r"<(?:(?:str|int|slug|uuid|path):)?([A-Za-z_][A-Za-z0-9_]*)>", r"{\1}", value)
+    joined = "/".join(part.strip("/") for part in (prefix, value) if part.strip("/"))
+    trailing = value.endswith("/")
+    return "/" + joined + ("/" if trailing and joined else "")

package/plugins/extract-django/routing_test.py

@@ -0,0 +1,198 @@
+import os
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(1, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pyplugin"))
+
+import apps  # noqa: E402
+import routing  # noqa: E402
+import serializers  # noqa: E402
+import transport  # noqa: E402
+from extract import http_contracts, openapi_document, routed_applications  # noqa: E402
+from protocol import Builder  # noqa: E402
+from source import Project  # noqa: E402
+
+
+class StaticRoutes(unittest.TestCase):
+    def setUp(self):
+        self.holder = tempfile.TemporaryDirectory(prefix="portolan-django-routes-")
+        root = self.holder.name
+        files = {
+            "manage.py": 'os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")\n',
+            "config/settings.py": 'ROOT_URLCONF = "config.urls"\n',
+            "config/urls.py": '''
+from django.urls import include, path
+from drf_yasg.views import get_schema_view
+schema_view = get_schema_view()
+urlpatterns = [
+    path("api/v2/", include("orders.urls")),
+    path("health/", include("health.urls")),
+    path("", include("home.urls")),
+    path("swagger/", schema_view.with_ui("swagger")),
+]
+''',
+            "home/urls.py": '''
+from django.urls import path
+from .views import Home
+urlpatterns = [path("", Home.as_view()), path("about/", Home.as_view())]
+''',
+            "home/views.py": '''
+from rest_framework.views import APIView
+class Home(APIView):
+    def get(self, request):
+        return None
+''',
+            "health/urls.py": '''
+from django.urls import path
+from .views import Ready
+urlpatterns = [path("ready/", Ready.as_view())]
+''',
+            "health/views.py": '''
+from rest_framework.views import APIView
+class Ready(APIView):
+    def get(self, request):
+        return None
+''',
+            "orders/models.py": '''
+from django.db import models
+class Order(models.Model):
+    active = models.BooleanField(default=True)
+''',
+            "orders/serializers.py": '''
+from rest_framework import serializers
+from .models import Order
+class OrderSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Order
+        fields = ("active",)
+''',
+            "orders/views.py": '''
+from rest_framework import generics, viewsets
+from rest_framework.views import APIView
+from .models import Order
+from .serializers import OrderSerializer
+
+class OrderList(generics.ListCreateAPIView):
+    queryset = Order.objects.filter(active=True)
+    serializer_class = OrderSerializer
+
+class OrderDetail(APIView):
+    def get(self, request):
+        """Read an order."""
+        pass
+    def delete(self, request):
+        pass
+
+class OrderViewSet(viewsets.ModelViewSet):
+    serializer_class = OrderSerializer
+
+class Health:
+    def get(request):
+        pass
+
+class Maintenance:
+    @classmethod
+    def fetch(cls, request):
+        return None
+''',
+            "orders/urls.py": '''
+from django.urls import path, re_path
+from rest_framework.routers import DefaultRouter
+from .views import Health, Maintenance, OrderDetail, OrderList, OrderViewSet
+
+router = DefaultRouter()
+router.register("orders", OrderViewSet, basename="order")
+urlpatterns = [
+    path("manual/", OrderList.as_view(), name="order-list"),
+    path("manual/<uuid:pk>/", OrderDetail.as_view(), name="order-detail"),
+    re_path(r"^health/(?P<region>[^/]+)/$", Health.get),
+    path("maintenance/fetch", Maintenance.fetch),
+] + router.urls
+''',
+        }
+        for name, contents in files.items():
+            path = os.path.join(root, name)
+            directory = os.path.dirname(path)
+            if directory:
+                os.makedirs(directory, exist_ok=True)
+            with open(path, "w", encoding="utf-8") as handle:
+                handle.write(contents)
+        self.project = Project(root, root, lambda path: os.path.relpath(path, root).replace(os.sep, "/"))
+
+    def tearDown(self):
+        self.holder.cleanup()
+
+    def test_nested_includes_direct_views_regexes_and_runtime_swagger(self):
+        routes = routing.read(self.project)
+        self.assertEqual(routes.root.dotted, "config.urls")
+        self.assertEqual(routes.runtime_schema, "config/urls.py:4")
+        paths = {(route.view, route.path) for route in routes.entries}
+        self.assertIn(("OrderList", "/api/v2/manual/"), paths)
+        self.assertIn(("OrderDetail", "/api/v2/manual/{pk}/"), paths)
+        self.assertIn(("Health.get", "/api/v2/health/{region}/"), paths)
+        self.assertIn(("OrderViewSet", "/api/v2/orders/"), paths)
+        self.assertIn(("Ready", "/health/ready/"), paths)
+        # A root mount is the commonest way to include an application, and the
+        # empty prefix must not be read as "no route".
+        self.assertIn(("Home", "/"), paths)
+        self.assertIn(("Home", "/about/"), paths)
+        detail = next(route for route in routes.entries if route.view == "OrderDetail")
+        self.assertEqual(dict(detail.parameters), {"pk": "uuid"})
+
+    def test_drf_generic_and_router_methods_become_http_operations(self):
+        routes = routing.read(self.project)
+        app = apps.discover(self.project, ["orders"])[0]
+        endpoints = transport.read_endpoints(app, Builder(), routes)
+        http = {(item.verb, item.path) for item in endpoints}
+        self.assertIn(("GET", "/api/v2/manual/"), http)
+        self.assertIn(("POST", "/api/v2/manual/"), http)
+        self.assertIn(("GET", "/api/v2/manual/{pk}/"), http)
+        self.assertIn(("DELETE", "/api/v2/manual/{pk}/"), http)
+        self.assertIn(("GET", "/api/v2/orders/"), http)
+        self.assertIn(("POST", "/api/v2/orders/"), http)
+        self.assertIn(("GET", "/api/v2/orders/{id}/"), http)
+        self.assertIn(("PATCH", "/api/v2/orders/{id}/"), http)
+        self.assertIn(("GET", "/api/v2/health/{region}/"), http)
+        flow_only = next(item for item in endpoints if item.view == "Maintenance" and item.action == "fetch")
+        self.assertEqual((flow_only.verb, flow_only.path), ("", "/api/v2/maintenance/fetch"))
+        detail = next(item for item in endpoints if item.verb == "GET" and item.path.endswith("manual/{pk}/"))
+        self.assertEqual(detail.path_parameters, {"pk": "uuid"})
+        spec = openapi_document([(app, item) for item in endpoints], "Orders", Builder())
+        parameter = spec["paths"]["/api/v2/manual/{pk}/"]["get"]["parameters"][0]
+        self.assertEqual(parameter["schema"], {"type": "string", "format": "uuid"})
+
+    def test_queryset_and_serializer_metadata_resolve_the_inherited_action_model(self):
+        routes = routing.read(self.project)
+        app = apps.discover(self.project, ["orders"])[0]
+        endpoints = transport.read_endpoints(app, Builder(), routes)
+        registry = serializers.read(self.project, [app])
+        list_endpoint = next(item for item in endpoints if item.view == "OrderList" and item.action == "list")
+        create_endpoint = next(item for item in endpoints if item.view == "OrderViewSet" and item.action == "create")
+        self.assertEqual(registry.model_for_endpoint(list_endpoint).name, "Order")
+        self.assertEqual(registry.model_for_endpoint(create_endpoint).name, "Order")
+
+    def test_stateless_routed_app_still_contributes_an_http_interface(self):
+        routes = routing.read(self.project)
+        model_apps = apps.discover(self.project, [])
+        endpoint_apps = routed_applications(self.project, model_apps, routes)
+        self.assertEqual([app.dotted for app in endpoint_apps], ["health", "home", "orders"])
+        endpoints = []
+        for app in endpoint_apps:
+            endpoints += [(app, endpoint) for endpoint in transport.read_endpoints(app, Builder(), routes)]
+        contracts = http_contracts(endpoints, "shop.billing", "billing/portolan/openapi.inferred.yaml")
+        health = [contract for contract in contracts if contract["id"] == "shop.billing.health"][0]
+        self.assertEqual(health["methods"], [{"name": "health_ready_get", "http": {"method": "GET", "path": "/health/ready/"}}])
+        home = [contract for contract in contracts if contract["id"] == "shop.billing.home"][0]
+        self.assertEqual(
+            home["methods"],
+            [
+                {"name": "about_get", "http": {"method": "GET", "path": "/about/"}},
+                {"name": "home_get", "http": {"method": "GET", "path": "/"}},
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()