@shortlink-org/portolan 0.2.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. package/README.md +174 -6
  2. package/catalog/enum_test.go +46 -0
  3. package/catalog/evidence_test.go +35 -0
  4. package/catalog/model.go +1213 -0
  5. package/catalog/roundtrip_test.go +210 -0
  6. package/catalog/via_test.go +38 -0
  7. package/cli/init.test.mjs +6 -1
  8. package/cli/portolan.mjs +14 -1
  9. package/cli/portolan.test.mjs +49 -0
  10. package/go.mod +21 -0
  11. package/go.sum +34 -0
  12. package/internal/gocall/README.md +19 -0
  13. package/internal/gocall/analyze.go +189 -0
  14. package/internal/gocall/analyze_test.go +107 -0
  15. package/internal/gohttp/analyze.go +2562 -0
  16. package/internal/gohttp/destination.go +373 -0
  17. package/internal/gohttp/endpoints.go +1067 -0
  18. package/internal/gohttp/roots.go +320 -0
  19. package/internal/gohttp/typed.go +96 -0
  20. package/internal/goscan/constants.go +85 -0
  21. package/internal/goscan/goscan_test.go +227 -0
  22. package/internal/goscan/index.go +629 -0
  23. package/internal/goscan/index_test.go +66 -0
  24. package/internal/goscan/names.go +52 -0
  25. package/internal/goscan/parse_test.go +11 -0
  26. package/internal/goscan/source.go +37 -0
  27. package/internal/goscan/tree.go +284 -0
  28. package/internal/goscan/types.go +99 -0
  29. package/internal/wsdl/ids.go +127 -0
  30. package/internal/wsdl/ids_test.go +21 -0
  31. package/internal/wsdl/model.go +70 -0
  32. package/internal/wsdl/parse.go +949 -0
  33. package/internal/wsdl/parse_test.go +170 -0
  34. package/package.json +28 -15
  35. package/plugin/describe.go +156 -0
  36. package/plugin/describe_test.go +114 -0
  37. package/plugin/protocol.go +141 -0
  38. package/plugin/schematest/schematest.go +126 -0
  39. package/plugins/README.md +404 -50
  40. package/plugins/cmd/portolan-http-clients/main.go +19 -0
  41. package/plugins/extract-argocd/options.schema.json +44 -0
  42. package/plugins/extract-celery/extract.py +0 -2
  43. package/plugins/extract-celery/extract_test.py +1 -1
  44. package/plugins/extract-celery/main.py +1 -0
  45. package/plugins/extract-csharp-ddd/README.md +213 -0
  46. package/plugins/extract-csharp-ddd/options.schema.json +55 -0
  47. package/plugins/extract-django/README.md +52 -19
  48. package/plugins/extract-django/choices.py +47 -0
  49. package/plugins/extract-django/domain.py +30 -18
  50. package/plugins/extract-django/extract.py +21 -7
  51. package/plugins/extract-django/extract_test.py +68 -2
  52. package/plugins/extract-django/lifecycle.py +4 -28
  53. package/plugins/extract-django/main.py +1 -0
  54. package/plugins/extract-django/operations.py +1 -1
  55. package/plugins/extract-django/routing_test.py +109 -1
  56. package/plugins/extract-django/rules.py +154 -0
  57. package/plugins/extract-django/rules_test.py +158 -0
  58. package/plugins/extract-django/store.py +1 -1
  59. package/plugins/extract-django/transport.py +101 -55
  60. package/plugins/extract-django/verbs.py +241 -0
  61. package/plugins/extract-go/README.md +47 -0
  62. package/plugins/extract-go-sqs/options.schema.json +21 -0
  63. package/plugins/extract-http-clients/describe.go +20 -0
  64. package/plugins/extract-http-clients/describe_test.go +11 -0
  65. package/plugins/extract-http-clients/extract.go +740 -0
  66. package/plugins/extract-http-clients/extract_test.go +1561 -0
  67. package/plugins/extract-http-clients/main.go +41 -0
  68. package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
  69. package/plugins/extract-java/build/org/portolan/extract/Main.class +0 -0
  70. package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
  71. package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
  72. package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
  73. package/plugins/extract-k8s/options.schema.json +32 -0
  74. package/plugins/extract-laravel/Cargo.lock +962 -0
  75. package/plugins/extract-laravel/Cargo.toml +20 -0
  76. package/plugins/extract-laravel/README.md +200 -0
  77. package/plugins/extract-laravel/options.schema.json +68 -0
  78. package/plugins/extract-laravel/rustfmt.toml +1 -0
  79. package/plugins/extract-php-ddd/Cargo.lock +972 -0
  80. package/plugins/extract-php-ddd/Cargo.toml +22 -0
  81. package/plugins/extract-php-ddd/README.md +141 -0
  82. package/plugins/extract-php-ddd/options.schema.json +50 -0
  83. package/plugins/extract-php-ddd/rustfmt.toml +1 -0
  84. package/plugins/extract-python-kafka/extract.py +0 -2
  85. package/plugins/extract-python-kafka/extract_test.py +1 -1
  86. package/plugins/extract-python-kafka/main.py +1 -0
  87. package/plugins/extract-terraform/options.schema.json +26 -0
  88. package/plugins/extract-ts/extract.test.ts +2 -2
  89. package/plugins/extract-ts/extract.ts +4 -5
  90. package/plugins/extract-ts/graphql.test.ts +1 -1
  91. package/plugins/extract-ts/main.ts +1 -0
  92. package/plugins/openapi/ids.go +261 -0
  93. package/plugins/openapi/ids_test.go +98 -0
  94. package/plugins/phpscan/Cargo.lock +953 -0
  95. package/plugins/phpscan/Cargo.toml +19 -0
  96. package/plugins/phpscan/rustfmt.toml +1 -0
  97. package/plugins/portolan-go.wasm +0 -0
  98. package/plugins/pyplugin/catalog.py +24 -1
  99. package/plugins/pyplugin/protocol.py +1 -5
  100. package/plugins/verify-otel/options.schema.json +12 -0
  101. package/portolan.json +132 -4
  102. package/rules/builtin.json +218 -0
  103. package/schema/portolan.schema.json +905 -4
  104. package/scripts/README.md +21 -13
  105. package/scripts/catalog-sources.mjs +6 -0
  106. package/scripts/delivery-presets.mjs +57 -14
  107. package/scripts/diff.mjs +5 -1
  108. package/scripts/django-aggregates.test.mjs +58 -0
  109. package/scripts/gen-likec4.mjs +150 -17
  110. package/scripts/gen-likec4.test.mjs +96 -0
  111. package/scripts/gen.mjs +148 -118
  112. package/scripts/gitops-example.test.mjs +108 -0
  113. package/scripts/go-discovery.test.mjs +30 -0
  114. package/scripts/history.mjs +186 -3
  115. package/scripts/history.test.mjs +1 -1
  116. package/scripts/host-plugins/fetch-argocd.mjs +338 -0
  117. package/scripts/host-plugins/fetch-argocd.options.json +49 -0
  118. package/scripts/host-plugins/fetch-argocd.test.mjs +274 -0
  119. package/scripts/host-plugins/fetch-bsr.mjs +1 -0
  120. package/scripts/host-plugins/fetch-csr.mjs +1 -0
  121. package/scripts/host-plugins/fetch-git.mjs +78 -21
  122. package/scripts/host-plugins/fetch-git.test.mjs +62 -8
  123. package/scripts/host-plugins/fetch-k8s.mjs +263 -0
  124. package/scripts/host-plugins/fetch-k8s.options.json +50 -0
  125. package/scripts/host-plugins/fetch-k8s.test.mjs +259 -0
  126. package/scripts/host-plugins/k8s-topology.mjs +183 -0
  127. package/scripts/json-format.mjs +192 -0
  128. package/scripts/json-format.test.mjs +97 -0
  129. package/scripts/local-api.mjs +287 -12
  130. package/scripts/local-api.test.mjs +150 -5
  131. package/scripts/local-discovery.mjs +94 -9
  132. package/scripts/manifest.mjs +14 -3
  133. package/scripts/manifest.test.mjs +24 -0
  134. package/scripts/output-diff.mjs +94 -0
  135. package/scripts/output-diff.test.mjs +36 -0
  136. package/scripts/package-smoke.mjs +62 -4
  137. package/scripts/plugin-host.mjs +45 -3
  138. package/scripts/plugin-host.test.mjs +9 -0
  139. package/scripts/plugin-wasm-worker.mjs +4 -1
  140. package/scripts/plugins-fresh.mjs +97 -0
  141. package/scripts/plugins-fresh.test.mjs +64 -0
  142. package/scripts/provenance.mjs +72 -0
  143. package/scripts/provenance.test.mjs +149 -0
  144. package/scripts/run-builtin.mjs +62 -7
  145. package/scripts/schema.mjs +167 -19
  146. package/scripts/trace-trials.mjs +176 -0
  147. package/scripts/trace-trials.test.mjs +142 -0
  148. package/scripts/warning-policy.mjs +167 -0
  149. package/scripts/warning-policy.test.mjs +93 -0
  150. package/src/app/Breadcrumbs.test.ts +3 -0
  151. package/src/app/Breadcrumbs.tsx +3 -0
  152. package/src/app/CatalogApp.tsx +2 -0
  153. package/src/app/Sidebar.tsx +3 -3
  154. package/src/app/SidebarFooter.tsx +20 -4
  155. package/src/catalog-docs.test.ts +64 -0
  156. package/src/catalog-docs.ts +35 -0
  157. package/src/catalog-error.test.ts +15 -0
  158. package/src/catalog-index.ts +25 -0
  159. package/src/catalog-model.ts +293 -5
  160. package/src/catalog-validation.ts +113 -2
  161. package/src/catalog.test.ts +40 -1
  162. package/src/chat/Starter.tsx +5 -11
  163. package/src/chat/tools.test.ts +27 -0
  164. package/src/chat/tools.ts +5 -9
  165. package/src/components/CatalogStamp.tsx +10 -8
  166. package/src/components/ChannelRows.test.tsx +76 -1
  167. package/src/components/ChannelRows.tsx +223 -60
  168. package/src/components/DeploymentRows.tsx +131 -0
  169. package/src/components/DocsLinks.test.tsx +27 -0
  170. package/src/components/DocsLinks.tsx +56 -0
  171. package/src/components/FieldTree.tsx +5 -2
  172. package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
  173. package/src/components/HTTPDestinationEvidence.tsx +31 -0
  174. package/src/components/Integrations.tsx +1 -1
  175. package/src/components/MachineDocs.tsx +6 -5
  176. package/src/components/MethodRows.tsx +9 -2
  177. package/src/components/PluginIcon.tsx +77 -0
  178. package/src/components/ProblemRow.tsx +173 -153
  179. package/src/components/RelationEvidence.test.tsx +14 -0
  180. package/src/components/RelationEvidence.tsx +53 -0
  181. package/src/components/RuleMarks.tsx +22 -0
  182. package/src/components/ShapeRows.tsx +24 -20
  183. package/src/data.ts +46 -7
  184. package/src/enrich.test.ts +459 -4
  185. package/src/enrich.ts +308 -7
  186. package/src/er/ErCanvas.tsx +218 -12
  187. package/src/er/GroupNode.tsx +57 -0
  188. package/src/er/StoreHeader.tsx +1 -0
  189. package/src/er/layout.test.ts +85 -2
  190. package/src/er/layout.ts +140 -5
  191. package/src/er/spec.test.ts +17 -0
  192. package/src/er/spec.ts +23 -10
  193. package/src/flow/Recordings.test.tsx +52 -0
  194. package/src/flow/Recordings.tsx +236 -0
  195. package/src/flow/StepDetail.tsx +59 -0
  196. package/src/flow/TraceTrial.tsx +419 -0
  197. package/src/flow/evidence.test.ts +16 -0
  198. package/src/flow/evidence.ts +34 -0
  199. package/src/flow/examples.test.ts +33 -0
  200. package/src/flow/examples.ts +37 -0
  201. package/src/flow/outline.test.ts +28 -0
  202. package/src/flow/outline.ts +7 -2
  203. package/src/flow/trace-trial-resume.test.ts +42 -0
  204. package/src/flow/trace-trial-resume.ts +74 -0
  205. package/src/graph/elk.ts +78 -0
  206. package/src/index.css +68 -0
  207. package/src/landing/DraggableReveal.tsx +3 -2
  208. package/src/landing/EvidencePipeline.tsx +105 -0
  209. package/src/landing/LandingPage.tsx +17 -70
  210. package/src/landing/ProductTour.tsx +6 -6
  211. package/src/lib/all-problems.ts +27 -17
  212. package/src/lib/catalog-diff.ts +1 -1
  213. package/src/lib/centrality.test.ts +251 -0
  214. package/src/lib/centrality.ts +232 -0
  215. package/src/lib/confluence.test.ts +37 -0
  216. package/src/lib/confluence.ts +41 -0
  217. package/src/lib/context-color.ts +1 -1
  218. package/src/lib/deployment-drift.ts +22 -0
  219. package/src/lib/derive.ts +43 -69
  220. package/src/lib/django-aggregates.d.mts +9 -0
  221. package/src/lib/django-aggregates.mjs +36 -0
  222. package/src/lib/django-aggregates.test.ts +29 -0
  223. package/src/lib/django-aggregates.ts +5 -0
  224. package/src/lib/environments.test.ts +83 -0
  225. package/src/lib/environments.ts +57 -0
  226. package/src/lib/integration-url.test.ts +30 -0
  227. package/src/lib/integration-url.ts +63 -0
  228. package/src/lib/kafka-ui.ts +3 -45
  229. package/src/lib/local-api.ts +116 -4
  230. package/src/lib/notion.ts +13 -0
  231. package/src/lib/plugin-index.json +2765 -0
  232. package/src/lib/plugins.test.ts +68 -0
  233. package/src/lib/plugins.ts +264 -0
  234. package/src/lib/problem-flows.test.ts +61 -0
  235. package/src/lib/problem-flows.ts +78 -0
  236. package/src/lib/problem-rules-cel.d.mts +43 -0
  237. package/src/lib/problem-rules-cel.mjs +407 -0
  238. package/src/lib/problem-rules.test.ts +287 -0
  239. package/src/lib/problem-rules.ts +271 -0
  240. package/src/lib/problem-subjects.ts +737 -0
  241. package/src/lib/rule-entries.ts +39 -0
  242. package/src/lib/{data-problems.test.ts → rules-data.test.ts} +18 -16
  243. package/src/lib/rules-deploy.test.ts +133 -0
  244. package/src/lib/{problems.test.ts → rules-edges.test.ts} +8 -3
  245. package/src/lib/{proto-problems.test.ts → rules-proto.test.ts} +9 -25
  246. package/src/lib/{wire-problems.test.ts → rules-wire.test.ts} +13 -11
  247. package/src/lib/rules.test.ts +51 -0
  248. package/src/lib/rules.ts +86 -0
  249. package/src/lib/setup-info.test.ts +17 -0
  250. package/src/lib/setup-info.ts +58 -0
  251. package/src/lib/shape.test.ts +32 -0
  252. package/src/lib/shape.ts +30 -6
  253. package/src/lib/tech.ts +16 -0
  254. package/src/lib/trace-project.test.ts +34 -0
  255. package/src/lib/trace-project.ts +50 -0
  256. package/src/lib/use-problems.ts +23 -0
  257. package/src/lib/warnings.test.ts +63 -0
  258. package/src/lib/warnings.ts +260 -0
  259. package/src/likec4/ids.test.ts +6 -2
  260. package/src/likec4/ids.ts +43 -0
  261. package/src/main.tsx +23 -0
  262. package/src/map/ContextMapGraph.tsx +76 -32
  263. package/src/merge-deployments.test.ts +127 -0
  264. package/src/merge.test.ts +82 -0
  265. package/src/merge.ts +189 -18
  266. package/src/pages/AggregatePage.tsx +65 -14
  267. package/src/pages/ContextMap.tsx +45 -3
  268. package/src/pages/ContextPage.tsx +8 -5
  269. package/src/pages/EventPage.tsx +15 -5
  270. package/src/pages/FlowDetail.tsx +23 -2
  271. package/src/pages/GraphPage.tsx +40 -3
  272. package/src/pages/Overview.tsx +152 -12
  273. package/src/pages/PluginIndex.tsx +190 -0
  274. package/src/pages/Problems.tsx +396 -128
  275. package/src/pages/ServicePage.tsx +62 -5
  276. package/src/pages/Settings.tsx +217 -43
  277. package/src/pages/settings/AboutSettings.tsx +8 -1
  278. package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
  279. package/src/pages/settings/IntegrationsSettings.tsx +63 -17
  280. package/src/pages/settings/RecordingSettings.tsx +138 -0
  281. package/src/pages/settings/RulesSettings.tsx +825 -0
  282. package/src/routes.test.ts +9 -0
  283. package/src/routes.ts +22 -1
  284. package/src/selection/DetailPanel.tsx +15 -0
  285. package/src/virtual-provenance.d.ts +11 -0
  286. package/vite.config.ts +5 -0
  287. package/scripts/vendor-lock.mjs +0 -58
  288. package/scripts/vendor-lock.test.mjs +0 -69
  289. package/src/lib/data-problems.ts +0 -314
  290. package/src/lib/proto-problems.ts +0 -237
  291. package/src/lib/wire-problems.ts +0 -342
@@ -74,7 +74,7 @@ def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
74
74
  endpoint_apps = routed_applications(project, applications, route_table)
75
75
  endpoints = []
76
76
  for app in endpoint_apps:
77
- endpoints += [(app, endpoint) for endpoint in transport.read_endpoints(app, b, route_table)]
77
+ endpoints += [(app, endpoint) for endpoint in transport.read_endpoints(app, b, route_table, project)]
78
78
  serializer_registry = serializers_module.read(project, endpoint_apps)
79
79
  auth_registry = auth_module.Registry(project, opts.settings, b)
80
80
 
@@ -169,8 +169,6 @@ def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
169
169
  }
170
170
 
171
171
  fragment = {
172
- "generatedAt": input_.generated_at,
173
- "commit": input_.commit,
174
172
  "contexts": [
175
173
  {
176
174
  "id": context,
@@ -223,8 +221,6 @@ def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
223
221
  )
224
222
  store_id = "%s.%s" % (svc_id, effective_store)
225
223
  stores_fragment = {
226
- "generatedAt": input_.generated_at,
227
- "commit": input_.commit,
228
224
  "contexts": [
229
225
  {
230
226
  "id": context,
@@ -296,7 +292,7 @@ def http_contracts(endpoints, svc_id: str, source: str) -> List[Dict[str, Any]]:
296
292
  grouped: Dict[str, List[Any]] = {}
297
293
  apps: Dict[str, Any] = {}
298
294
  for app, endpoint in endpoints:
299
- if not endpoint.verb or not endpoint.path:
295
+ if not endpoint.path:
300
296
  continue
301
297
  grouped.setdefault(app.dotted, []).append(endpoint)
302
298
  apps[app.dotted] = app
@@ -313,6 +309,10 @@ def http_contracts(endpoints, svc_id: str, source: str) -> List[Dict[str, Any]]:
313
309
  method = {"name": name}
314
310
  if endpoint.doc:
315
311
  method["doc"] = endpoint.doc
312
+ # A mounted route whose verb no declaration proves keeps its path
313
+ # with the method empty: the route is a fact of the URLConf, the
314
+ # verb is explicitly unknown, and the merge will not match an
315
+ # outbound call against it until somebody declares it.
316
316
  method["http"] = {"method": endpoint.verb, "path": endpoint.path}
317
317
  methods.append(method)
318
318
  if not methods:
@@ -332,7 +332,21 @@ def openapi_document(endpoints, service_name: str, b: Builder, serializer_regist
332
332
  tags = set()
333
333
  operation_ids = set()
334
334
  for app, endpoint in sorted(endpoints, key=lambda item: (item[1].path, item[1].verb, item[0].label, item[1].id)):
335
- if not endpoint.path or not endpoint.verb:
335
+ if not endpoint.path:
336
+ continue
337
+ if not endpoint.verb:
338
+ # The path is mounted; which verb answers there is not written
339
+ # down. A path item without operations says exactly that, where
340
+ # inventing a GET would be read as a fact.
341
+ path_item = paths.setdefault(endpoint.path, {})
342
+ path_item.setdefault("summary", title(endpoint.action))
343
+ path_item.setdefault(
344
+ "description",
345
+ "Mounted in URLConf by %s, but no HTTP verb is declared in source; no operation is inferred." % endpoint.route_source,
346
+ )
347
+ path_item.setdefault("x-portolan-inferred", True)
348
+ path_item.setdefault("x-portolan-source", endpoint.route_source)
349
+ path_item.setdefault("x-portolan-verb", "unknown")
336
350
  continue
337
351
  method = endpoint.verb.lower()
338
352
  path_item = paths.setdefault(endpoint.path, {})
@@ -10,6 +10,7 @@ import ast
10
10
  import json
11
11
  import os
12
12
  import sys
13
+ import tempfile
13
14
  import unittest
14
15
 
15
16
  sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -35,7 +36,7 @@ OPTIONS = {
35
36
  def run(options):
36
37
  b = Builder()
37
38
  extract(
38
- Input(root=FIXTURE, output=FIXTURE + "/portolan", commit="abc1234", generated_at="2026-09-05T00:00:00Z"),
39
+ Input(root=FIXTURE, output=FIXTURE + "/portolan"),
39
40
  Options.of(options),
40
41
  b,
41
42
  cwd=ROOT,
@@ -193,11 +194,63 @@ class Fragment(unittest.TestCase):
193
194
  self.assertIn("ledger_ledgerentry", tables)
194
195
  self.assertIn("ledger_auditrecord", tables)
195
196
  self.assertNotIn("ledger_ledgerentryproxy", tables)
196
- self.assertNotIn("persists", tables["ledger_ledgerentry"])
197
+ self.assertEqual(tables["ledger_ledgerentry"]["persists"], {"aggregate": "shop.billing.models-ledger", "block": "shop.billing.models-ledger.ledger-entry"})
197
198
  self.assertNotIn("role", tables["ledger_ledgerentry"])
198
199
  service = json.loads(self.files["domain.json"])["contexts"][0]["services"][0]
199
200
  self.assertFalse(any(aggregate["slug"] == "ledger-entry" for aggregate in service["aggregates"]))
200
201
 
202
+ def test_ambiguous_apps_keep_all_concrete_models_without_requiring_a_root(self):
203
+ service = json.loads(self.files["domain.json"])["contexts"][0]["services"][0]
204
+ group = next(a for a in service["aggregates"] if a["slug"] == "models-ledger")
205
+ self.assertEqual(group["kind"], "model-group")
206
+ self.assertEqual(group["root"], "")
207
+ self.assertEqual([m["name"] for m in group["entities"]], ["LedgerEntry", "AuditRecord"])
208
+ self.assertNotIn("lifecycle", group)
209
+ self.assertFalse(any("models to choose from" in w.message or (w.ref == group["id"] and "no services module" in w.message) for w in self.warnings))
210
+
211
+ def test_rootless_group_keeps_app_operations_and_events_once(self):
212
+ with tempfile.TemporaryDirectory() as root:
213
+ app = os.path.join(root, "records")
214
+ os.mkdir(app)
215
+ for name, source in {
216
+ "models.py": "from django.db import models\nclass Alpha(models.Model):\n name = models.CharField(max_length=30)\nclass Beta(models.Model):\n owner = models.ForeignKey(Alpha, on_delete=models.CASCADE)\n",
217
+ "services.py": "def list_records():\n return []\n",
218
+ "events.py": 'from dataclasses import dataclass\n@dataclass\nclass RecordsChanged:\n name = "records.Changed"\n record_id: int\n',
219
+ }.items():
220
+ with open(os.path.join(app, name), "w") as handle:
221
+ handle.write(source)
222
+ builder = Builder()
223
+ extract(Input(root=root), Options.of({"context": "shop", "service": "records"}), builder, cwd=ROOT)
224
+ fragment = json.loads(next(f.contents for f in builder.files if f.name == "domain.json"))
225
+ groups = fragment["contexts"][0]["services"][0]["aggregates"]
226
+ self.assertEqual(len(groups), 1)
227
+ self.assertEqual(groups[0]["kind"], "model-group")
228
+ self.assertEqual(len(groups[0]["entities"]), 2)
229
+ self.assertEqual([op["id"] for op in groups[0]["operations"]], ["ListRecords"])
230
+ self.assertEqual([event["name"] for event in groups[0]["events"]], ["RecordsChanged"])
231
+
232
+ def test_explicit_root_replaces_the_group_without_duplicating_models(self):
233
+ files, warnings = run(dict(OPTIONS, aggregates={"ledger": "LedgerEntry"}))
234
+ self.assertFalse(any("models to choose from" in w.message for w in warnings))
235
+ service = json.loads(files["domain.json"])["contexts"][0]["services"][0]
236
+ aggregate = next(a for a in service["aggregates"] if a["slug"] == "ledger-entry")
237
+ self.assertEqual(aggregate["entities"][0]["name"], "LedgerEntry")
238
+ self.assertNotIn("kind", aggregate)
239
+ self.assertFalse(any(a["slug"] == "models-ledger" for a in service["aggregates"]))
240
+ store = json.loads(files["stores.json"])["stores"][0]
241
+ table = next(t for t in store["tables"] if t["name"] == "ledger_ledgerentry")
242
+ self.assertEqual(table["role"], "aggregate-root")
243
+
244
+ def test_invalid_or_proxy_root_offers_a_replacement(self):
245
+ for root in ["Missing", "LedgerEntryProxy", "RecordBase"]:
246
+ files, warnings = run(dict(OPTIONS, aggregates={"ledger": root}))
247
+ service = json.loads(files["domain.json"])["contexts"][0]["services"][0]
248
+ group = next(a for a in service["aggregates"] if a["slug"] == "models-ledger")
249
+ self.assertEqual(len(group["entities"]), 2)
250
+ warning = next(w for w in warnings if "aggregates names" in w.message)
251
+ candidates = json.loads(warning.message.split("; aggregate candidates: ")[1])
252
+ self.assertEqual([m["name"] for m in candidates["models"]], ["AuditRecord", "LedgerEntry"])
253
+
201
254
  def test_abstract_fields_custom_postgres_fields_and_unresolved_relations_are_described(self):
202
255
  store = json.loads(self.files["stores.json"])["stores"][0]
203
256
  table = next(table for table in store["tables"] if table["name"] == "ledger_ledgerentry")
@@ -250,6 +303,19 @@ class Reading(unittest.TestCase):
250
303
  self.assertEqual([b["name"] for b in self.aggregate["entities"]], ["Invoice", "InvoiceLine"])
251
304
  self.assertEqual([b["name"] for b in self.aggregate["valueObjects"]], ["Money"])
252
305
 
306
+ def test_a_model_field_states_its_rules_and_whether_it_must_be_given(self):
307
+ invoice = next(b for b in self.aggregate["entities"] if b["name"] == "Invoice")
308
+ fields = {f["name"]: f for f in invoice["fields"]}
309
+ self.assertEqual(fields["number"].get("rules"), [{"name": "max_len", "value": "32"}, {"name": "unique"}])
310
+ self.assertNotIn("required", fields["number"]) # null=True: the model lets it be absent
311
+ self.assertEqual(fields["currency"], {"name": "currency", "type": "CharField", "doc": "", "required": True, "rules": [{"name": "max_len", "value": "3"}]})
312
+ self.assertEqual(
313
+ fields["status"].get("rules"),
314
+ [{"name": "max_len", "value": "16"}, {"name": "in", "value": "draft, issued, paid, void"}],
315
+ )
316
+ self.assertNotIn("required", fields["status"]) # a default fills it
317
+ self.assertEqual(fields["id"].get("rules"), [{"name": "format", "value": "uuid"}])
318
+
253
319
  def test_a_service_function_is_an_operation_and_a_write_makes_it_a_command(self):
254
320
  kinds = {o["id"]: o["kind"] for o in self.aggregate["operations"]}
255
321
  self.assertEqual(kinds["IssueInvoice"], "command")
@@ -15,8 +15,9 @@ from dataclasses import dataclass, field as dc_field
15
15
  from typing import Dict, List, Optional, Tuple
16
16
 
17
17
  import catalog
18
+ from choices import choice_tables
18
19
  from domain import Aggregate, ModelDef
19
- from source import assigned, bases, const_str, dotted, inner_class, keyword, methods
20
+ from source import assigned, const_str, dotted, keyword, methods
20
21
 
21
22
 
22
23
  @dataclass
@@ -28,25 +29,6 @@ class Move:
28
29
  sources: List[str] = dc_field(default_factory=list) # the states it may be made from, when the decorator says
29
30
 
30
31
 
31
- def choices_of(model: ModelDef, name: str) -> Dict[str, str]:
32
- """A TextChoices class, as member name to the value stored in the column."""
33
- node = inner_class(model.node, name)
34
- if node is None:
35
- for other in model.module.classes():
36
- if other.name == name:
37
- node = other
38
- break
39
- if node is None or not any(b.split(".")[-1].endswith("Choices") for b in bases(node)):
40
- return {}
41
- out = {}
42
- for member, value, _ in assigned(node):
43
- if isinstance(value, ast.Constant) and isinstance(value.value, str):
44
- out[member] = value.value
45
- elif isinstance(value, ast.Tuple) and value.elts and isinstance(value.elts[0], ast.Constant):
46
- out[member] = value.elts[0].value
47
- return out
48
-
49
-
50
32
  def state_of(node: ast.AST, choices: Dict[str, Dict[str, str]]) -> str:
51
33
  """`Status.DRAFT` or `"draft"`, either way the value in the column."""
52
34
  literal = const_str(node)
@@ -71,14 +53,6 @@ def status_field(model: ModelDef) -> Optional[str]:
71
53
  return "status" if model.field("status") else None
72
54
 
73
55
 
74
- def choice_tables(model: ModelDef) -> Dict[str, Dict[str, str]]:
75
- out = {}
76
- for node in list(model.node.body) + list(model.module.tree.body):
77
- if isinstance(node, ast.ClassDef) and any(b.split(".")[-1].endswith("Choices") for b in bases(node)):
78
- out[node.name] = choices_of(model, node.name)
79
- return out
80
-
81
-
82
56
  def declared_table(model: ModelDef, choices: Dict[str, Dict[str, str]]) -> Dict[str, List[str]]:
83
57
  """`TRANSITIONS = {Status.DRAFT: [Status.ISSUED], ...}`, in the order written."""
84
58
  for name, value, _ in assigned(model.node):
@@ -160,6 +134,8 @@ def movers(model: ModelDef, status: str, choices: Dict[str, Dict[str, str]], eve
160
134
 
161
135
  def read(agg: Aggregate, events: Dict[str, object], b) -> Optional[Dict[str, object]]:
162
136
  model = agg.root
137
+ if model is None:
138
+ return None
163
139
  status = status_field(model)
164
140
  if status is None:
165
141
  return None
@@ -25,6 +25,7 @@ HERE = os.path.dirname(os.path.abspath(__file__))
25
25
  DESCRIPTOR = {
26
26
  "name": "extract-django",
27
27
  "summary": "Reads a Django service by its applications - models, events, services, DRF views, receivers, clients - into a catalog fragment, and its models into the store they are the schema of.",
28
+ "category": "code",
28
29
  "phases": ["extract"],
29
30
  }
30
31
 
@@ -87,7 +87,7 @@ def read_use_cases(agg: Aggregate, b) -> List[UseCase]:
87
87
  kind="command" if writes(node) else "query",
88
88
  )
89
89
  )
90
- if not out:
90
+ if not out and agg.root is not None:
91
91
  b.warn(agg.id, "no services module under %s: the aggregate has no operations, only whatever the views do inline" % agg.app.rel)
92
92
  return sorted(out, key=lambda u: u.id)
93
93
 
@@ -96,11 +96,70 @@ class Maintenance:
96
96
  @classmethod
97
97
  def fetch(cls, request):
98
98
  return None
99
+
100
+ from django.http import HttpResponseNotAllowed
101
+ from django.utils.decorators import method_decorator
102
+ from django.views.decorators.http import require_GET, require_http_methods
103
+ from rest_framework.decorators import api_view
104
+ from .helpers import ensure_post, guarded
105
+
106
+ class Planet:
107
+ @classmethod
108
+ @require_http_methods(["POST"])
109
+ def fetch(cls, request):
110
+ return None
111
+
112
+ @classmethod
113
+ @ensure_post
114
+ def refresh(cls, request):
115
+ return None
116
+
117
+ @classmethod
118
+ def reindex(cls, request):
119
+ if request.method != "PUT":
120
+ return HttpResponseNotAllowed(["PUT"])
121
+ return None
122
+
123
+ @classmethod
124
+ def status(cls, request):
125
+ return guarded(request, cls._status)
126
+
127
+ @classmethod
128
+ def _status(cls):
129
+ return None
130
+
131
+ @method_decorator(require_GET, name="dispatch")
132
+ class Reports:
133
+ @classmethod
134
+ def summary(cls, request):
135
+ return None
136
+
137
+ class Exports:
138
+ http_method_names = ["patch", "options", "head"]
139
+
140
+ @classmethod
141
+ def run(cls, request):
142
+ return None
143
+
144
+ @api_view(["GET", "POST"])
145
+ def toggle(request):
146
+ return None
147
+ ''',
148
+ "orders/helpers.py": '''
149
+ from django.views.decorators.http import require_POST
150
+
151
+ def ensure_post(view):
152
+ return require_POST(view)
153
+
154
+ def guarded(request, handler):
155
+ if request.method.lower() in ("delete", "patch"):
156
+ return handler()
157
+ return None
99
158
  ''',
100
159
  "orders/urls.py": '''
101
160
  from django.urls import path, re_path
102
161
  from rest_framework.routers import DefaultRouter
103
- from .views import Health, Maintenance, OrderDetail, OrderList, OrderViewSet
162
+ from .views import Exports, Health, Maintenance, OrderDetail, OrderList, OrderViewSet, Planet, Reports, toggle
104
163
 
105
164
  router = DefaultRouter()
106
165
  router.register("orders", OrderViewSet, basename="order")
@@ -109,6 +168,13 @@ urlpatterns = [
109
168
  path("manual/<uuid:pk>/", OrderDetail.as_view(), name="order-detail"),
110
169
  re_path(r"^health/(?P<region>[^/]+)/$", Health.get),
111
170
  path("maintenance/fetch", Maintenance.fetch),
171
+ path("planet/fetch", Planet.fetch),
172
+ path("planet/refresh", Planet.refresh),
173
+ path("planet/reindex", Planet.reindex),
174
+ path("planet/status", Planet.status),
175
+ path("reports/summary", Reports.summary),
176
+ path("exports/run", Exports.run),
177
+ path("toggle/", toggle),
112
178
  ] + router.urls
113
179
  ''',
114
180
  }
@@ -163,6 +229,48 @@ urlpatterns = [
163
229
  parameter = spec["paths"]["/api/v2/manual/{pk}/"]["get"]["parameters"][0]
164
230
  self.assertEqual(parameter["schema"], {"type": "string", "format": "uuid"})
165
231
 
232
+ def test_a_mounted_method_takes_its_verb_from_what_the_code_declares(self):
233
+ routes = routing.read(self.project)
234
+ app = apps.discover(self.project, ["orders"])[0]
235
+ b = Builder()
236
+ endpoints = transport.read_endpoints(app, b, routes, self.project)
237
+ verbs = {}
238
+ for item in endpoints:
239
+ verbs.setdefault(item.path, set()).add(item.verb)
240
+ # Each tier of evidence, from the handler outwards.
241
+ self.assertEqual(verbs["/api/v2/planet/fetch"], {"POST"}) # @require_http_methods on the handler
242
+ self.assertEqual(verbs["/api/v2/reports/summary"], {"GET"}) # @method_decorator(require_GET, name="dispatch") on the class
243
+ self.assertEqual(verbs["/api/v2/exports/run"], {"PATCH"}) # http_method_names, less HEAD and OPTIONS
244
+ self.assertEqual(verbs["/api/v2/planet/reindex"], {"PUT"}) # a branch on request.method
245
+ self.assertEqual(verbs["/api/v2/planet/refresh"], {"POST"}) # a project decorator that applies require_POST
246
+ self.assertEqual(verbs["/api/v2/planet/status"], {"DELETE", "PATCH"}) # a wrapper the handler hands request to
247
+ self.assertEqual(verbs["/api/v2/toggle/"], {"GET", "POST"}) # every method @api_view lists, not the first
248
+ fetch = next(item for item in endpoints if item.path == "/api/v2/planet/fetch")
249
+ self.assertTrue(fetch.verb_source.startswith("decorator at orders/views.py:"), fetch.verb_source)
250
+ status = {item.id: item.verb_source for item in endpoints if item.path == "/api/v2/planet/status"}
251
+ self.assertEqual(sorted(status), ["api_v2_planet_status", "api_v2_planet_status_patch"])
252
+ self.assertTrue(all(source.startswith("wrapper guarded at orders/helpers.py:") for source in status.values()), status)
253
+
254
+ # No tier speaks for Maintenance.fetch: the verb is unknown, and the
255
+ # route stays in the model saying so instead of disappearing.
256
+ self.assertEqual(verbs["/api/v2/maintenance/fetch"], {""})
257
+ self.assertEqual(
258
+ [w.message.split(";")[0] for w in b.warnings if w.ref == "orders/urls.py:12"],
259
+ ["Maintenance.fetch is mounted as an HTTP view, but no HTTP verb is declared"],
260
+ )
261
+ pairs = [(app, item) for item in endpoints]
262
+ contracts = http_contracts(pairs, "shop.orders", "orders/portolan/openapi.inferred.yaml")
263
+ methods = {method["name"]: method["http"] for method in contracts[0]["methods"]}
264
+ self.assertEqual(methods["api_v2_maintenance_fetch"], {"method": "", "path": "/api/v2/maintenance/fetch"})
265
+ self.assertEqual(methods["api_v2_planet_status_patch"], {"method": "PATCH", "path": "/api/v2/planet/status"})
266
+ spec = openapi_document(pairs, "Orders", Builder())
267
+ unknown = spec["paths"]["/api/v2/maintenance/fetch"]
268
+ self.assertEqual(unknown["x-portolan-verb"], "unknown")
269
+ self.assertEqual(unknown["x-portolan-source"], "orders/urls.py:12")
270
+ self.assertFalse({"get", "post", "put", "patch", "delete"} & set(unknown))
271
+ self.assertIn("post", spec["paths"]["/api/v2/planet/fetch"])
272
+ self.assertEqual(sorted(spec["paths"]["/api/v2/planet/status"]), ["delete", "patch"])
273
+
166
274
  def test_queryset_and_serializer_metadata_resolve_the_inherited_action_model(self):
167
275
  routes = routing.read(self.project)
168
276
  app = apps.discover(self.project, ["orders"])[0]
@@ -0,0 +1,154 @@
1
+ """What a model field says a value must satisfy, in the catalog's words.
2
+
3
+ A Django model states its rules in the field call: `CharField(max_length=3,
4
+ unique=True, choices=Status.choices, validators=[MinValueValidator(0)])`. The
5
+ catalog has one vocabulary for these across every source (portolan.0015), so
6
+ `max_length` is `max_len` here as `minLength` is in an OpenAPI document and
7
+ `min_len` in a proto, and the page says all three the same way.
8
+
9
+ The model is the source of truth for `required`: a field must be given when a
10
+ row is made unless the class fills it in (`default`, `auto_now`, an auto id),
11
+ lets it be empty (`blank`) or lets it be absent (`null`). What a serializer or
12
+ a form adds on top is that layer's word, not the model's, and is not read here.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import ast
18
+ from typing import Dict, List, Optional
19
+
20
+ import catalog
21
+ from choices import choice_tables
22
+ from source import assigned, dotted, keyword, keyword_bool
23
+
24
+ # The field classes whose name is itself a rule on the value.
25
+ FORMATS = {
26
+ "EmailField": "email",
27
+ "URLField": "uri",
28
+ "UUIDField": "uuid",
29
+ "GenericIPAddressField": "ip",
30
+ }
31
+ NON_NEGATIVE = {"PositiveIntegerField", "PositiveBigIntegerField", "PositiveSmallIntegerField"}
32
+
33
+ # Never required: the database numbers an auto id, a bool is False when not
34
+ # given, and a many-to-many is a table of its own with no row to be missing.
35
+ NEVER_REQUIRED = {"AutoField", "BigAutoField", "SmallAutoField", "BooleanField", "ManyToManyField"}
36
+
37
+ # Options under which the model itself fills or excuses the value.
38
+ FILLED_OR_EXCUSED = ("blank", "null", "auto_now", "auto_now_add")
39
+
40
+ VALIDATORS = {
41
+ "MinValueValidator": "gte",
42
+ "MaxValueValidator": "lte",
43
+ "MinLengthValidator": "min_len",
44
+ "MaxLengthValidator": "max_len",
45
+ "RegexValidator": "pattern",
46
+ }
47
+
48
+
49
+ def rules_of(f, model) -> Dict[str, object]:
50
+ """`required` and `rules` for `catalog.field`, as keyword arguments; an
51
+ empty dict when the field says nothing about its value."""
52
+ out: Dict[str, object] = {}
53
+ if required(f):
54
+ out["required"] = True
55
+ found = rules(f, model)
56
+ if found:
57
+ out["rules"] = found
58
+ return out
59
+
60
+
61
+ def required(f) -> bool:
62
+ if f.kind in NEVER_REQUIRED:
63
+ return False
64
+ if any(keyword_bool(f.call, option) for option in FILLED_OR_EXCUSED):
65
+ return False
66
+ return keyword(f.call, "default") is None
67
+
68
+
69
+ def rules(f, model) -> List[Dict[str, str]]:
70
+ """The rules in the order the source states them: what the field class
71
+ says first, then the options as written."""
72
+ out = []
73
+ if f.kind in FORMATS:
74
+ out.append(catalog.rule("format", FORMATS[f.kind]))
75
+ if f.kind in NON_NEGATIVE:
76
+ out.append(catalog.rule("gte", "0"))
77
+ for kw in f.call.keywords:
78
+ if kw.arg == "max_length":
79
+ length = f.integer("max_length")
80
+ if length is not None:
81
+ out.append(catalog.rule("max_len", str(length)))
82
+ elif kw.arg == "unique" and keyword_bool(f.call, "unique"):
83
+ out.append(catalog.rule("unique"))
84
+ elif kw.arg == "choices":
85
+ values = choice_values(kw.value, model)
86
+ if values:
87
+ out.append(catalog.rule("in", ", ".join(values)))
88
+ elif kw.arg == "validators":
89
+ out.extend(validator_rules(kw.value))
90
+ return out
91
+
92
+
93
+ def choice_values(node: ast.AST, model, depth: int = 0) -> List[str]:
94
+ """The values a `choices=` option allows, in the order it lists them.
95
+
96
+ `Status.choices` reads the Choices class; a literal list of pairs or of
97
+ bare values reads the value of each pair, descending into a group's own
98
+ list; a name reads the module-level constant it is assigned. Anything
99
+ else - a call, an import the module does not define - yields nothing, and
100
+ nothing is what the field then says, rather than an empty `in`.
101
+ """
102
+ if depth > 3:
103
+ return []
104
+ if isinstance(node, ast.Attribute) and node.attr == "choices":
105
+ table = choice_tables(model).get(dotted(node.value).split(".")[-1], {})
106
+ return list(table.values())
107
+ if isinstance(node, (ast.List, ast.Tuple)):
108
+ out: List[str] = []
109
+ for item in node.elts:
110
+ if isinstance(item, ast.Constant) and not isinstance(item.value, bool):
111
+ out.append(str(item.value))
112
+ elif isinstance(item, (ast.List, ast.Tuple)) and item.elts:
113
+ first = item.elts[0]
114
+ second = item.elts[1] if len(item.elts) > 1 else None
115
+ if isinstance(second, (ast.List, ast.Tuple)):
116
+ out.extend(choice_values(second, model, depth + 1)) # a group: (label, [pairs])
117
+ elif isinstance(first, ast.Constant) and not isinstance(first.value, bool):
118
+ out.append(str(first.value))
119
+ elif isinstance(first, ast.Attribute):
120
+ value = choice_member(first, model)
121
+ if value:
122
+ out.append(value)
123
+ return out
124
+ if isinstance(node, ast.Name):
125
+ for name, value, _ in assigned(model.module.tree):
126
+ if name == node.id:
127
+ return choice_values(value, model, depth + 1)
128
+ return []
129
+
130
+
131
+ def choice_member(node: ast.Attribute, model) -> Optional[str]:
132
+ """`Status.DRAFT` in a literal list of pairs: the member's stored value."""
133
+ table = choice_tables(model).get(dotted(node.value).split(".")[-1], {})
134
+ return table.get(node.attr)
135
+
136
+
137
+ def validator_rules(node: ast.AST) -> List[Dict[str, str]]:
138
+ """`validators=[MinValueValidator(0), RegexValidator(r"^[A-Z]+$")]`: the
139
+ validators the catalog has words for, with the bound they were given. A
140
+ validator built from a name the module computes, or a regex meant to be
141
+ NOT matched, is left out rather than guessed."""
142
+ if not isinstance(node, (ast.List, ast.Tuple)):
143
+ return []
144
+ out = []
145
+ for item in node.elts:
146
+ if not isinstance(item, ast.Call):
147
+ continue
148
+ rule = VALIDATORS.get(dotted(item.func).split(".")[-1])
149
+ if rule is None or keyword_bool(item, "inverse_match"):
150
+ continue
151
+ given = item.args[0] if item.args else keyword(item, "limit_value") or keyword(item, "regex")
152
+ if isinstance(given, ast.Constant) and not isinstance(given.value, bool):
153
+ out.append(catalog.rule(rule, str(given.value)))
154
+ return out