@shortlink-org/portolan 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -6
- package/catalog/model.go +147 -0
- package/catalog/roundtrip_test.go +7 -0
- package/cli/portolan.mjs +6 -1
- package/go.mod +7 -0
- package/go.sum +20 -6
- package/package.json +10 -9
- package/plugin/describe.go +38 -0
- package/plugins/README.md +296 -5
- package/plugins/extract-argocd/options.schema.json +44 -0
- package/plugins/extract-celery/main.py +1 -0
- package/plugins/extract-csharp-ddd/README.md +213 -0
- package/plugins/extract-csharp-ddd/options.schema.json +55 -0
- package/plugins/extract-django/README.md +13 -2
- package/plugins/extract-django/choices.py +47 -0
- package/plugins/extract-django/domain.py +2 -1
- package/plugins/extract-django/extract_test.py +13 -0
- package/plugins/extract-django/lifecycle.py +2 -28
- package/plugins/extract-django/main.py +1 -0
- package/plugins/extract-django/rules.py +154 -0
- package/plugins/extract-django/rules_test.py +158 -0
- package/plugins/extract-go-sqs/options.schema.json +21 -0
- package/plugins/extract-http-clients/describe.go +5 -4
- package/plugins/extract-java/build/org/portolan/extract/Main.class +0 -0
- package/plugins/extract-k8s/options.schema.json +32 -0
- package/plugins/extract-laravel/Cargo.lock +962 -0
- package/plugins/extract-laravel/Cargo.toml +20 -0
- package/plugins/extract-laravel/README.md +200 -0
- package/plugins/extract-laravel/options.schema.json +68 -0
- package/plugins/extract-laravel/rustfmt.toml +1 -0
- package/plugins/extract-php-ddd/Cargo.lock +972 -0
- package/plugins/extract-php-ddd/Cargo.toml +22 -0
- package/plugins/extract-php-ddd/README.md +141 -0
- package/plugins/extract-php-ddd/options.schema.json +50 -0
- package/plugins/extract-php-ddd/rustfmt.toml +1 -0
- package/plugins/extract-python-kafka/main.py +1 -0
- package/plugins/extract-terraform/options.schema.json +26 -0
- package/plugins/extract-ts/main.ts +1 -0
- package/plugins/phpscan/Cargo.lock +953 -0
- package/plugins/phpscan/Cargo.toml +19 -0
- package/plugins/phpscan/rustfmt.toml +1 -0
- package/plugins/portolan-go.wasm +0 -0
- package/plugins/pyplugin/catalog.py +24 -1
- package/plugins/verify-otel/options.schema.json +12 -0
- package/portolan.json +130 -3
- package/rules/builtin.json +218 -0
- package/schema/portolan.schema.json +880 -13
- package/scripts/README.md +3 -1
- package/scripts/delivery-presets.mjs +36 -3
- package/scripts/gen-likec4.mjs +149 -16
- package/scripts/gen-likec4.test.mjs +96 -0
- package/scripts/gen.mjs +30 -3
- package/scripts/gitops-example.test.mjs +108 -0
- package/scripts/host-plugins/fetch-argocd.mjs +338 -0
- package/scripts/host-plugins/fetch-argocd.options.json +49 -0
- package/scripts/host-plugins/fetch-argocd.test.mjs +274 -0
- package/scripts/host-plugins/fetch-bsr.mjs +1 -0
- package/scripts/host-plugins/fetch-csr.mjs +1 -0
- package/scripts/host-plugins/fetch-git.mjs +1 -0
- package/scripts/host-plugins/fetch-k8s.mjs +263 -0
- package/scripts/host-plugins/fetch-k8s.options.json +50 -0
- package/scripts/host-plugins/fetch-k8s.test.mjs +259 -0
- package/scripts/host-plugins/k8s-topology.mjs +183 -0
- package/scripts/json-format.mjs +192 -0
- package/scripts/json-format.test.mjs +97 -0
- package/scripts/local-api.mjs +216 -8
- package/scripts/local-api.test.mjs +88 -2
- package/scripts/local-discovery.mjs +12 -0
- package/scripts/manifest.mjs +10 -1
- package/scripts/plugin-host.mjs +23 -1
- package/scripts/plugins-fresh.mjs +97 -0
- package/scripts/plugins-fresh.test.mjs +64 -0
- package/scripts/run-builtin.mjs +26 -5
- package/scripts/schema.mjs +138 -19
- package/scripts/trace-trials.mjs +176 -0
- package/scripts/trace-trials.test.mjs +142 -0
- package/scripts/warning-policy.mjs +12 -6
- package/scripts/warning-policy.test.mjs +40 -3
- package/src/app/Breadcrumbs.test.ts +3 -0
- package/src/app/Breadcrumbs.tsx +3 -0
- package/src/app/CatalogApp.tsx +2 -0
- package/src/app/SidebarFooter.tsx +20 -4
- package/src/catalog-index.ts +25 -0
- package/src/catalog-model.ts +246 -1
- package/src/catalog-validation.ts +104 -2
- package/src/catalog.test.ts +40 -1
- package/src/components/ChannelRows.test.tsx +76 -1
- package/src/components/ChannelRows.tsx +223 -60
- package/src/components/DeploymentRows.tsx +131 -0
- package/src/components/DocsLinks.test.tsx +27 -0
- package/src/components/DocsLinks.tsx +56 -0
- package/src/components/FieldTree.tsx +5 -2
- package/src/components/HTTPDestinationEvidence.tsx +1 -1
- package/src/components/PluginIcon.tsx +77 -0
- package/src/components/ProblemRow.tsx +173 -153
- package/src/components/RuleMarks.tsx +22 -0
- package/src/components/ShapeRows.tsx +24 -20
- package/src/data.ts +21 -0
- package/src/enrich.test.ts +127 -3
- package/src/enrich.ts +104 -6
- package/src/er/ErCanvas.tsx +218 -12
- package/src/er/GroupNode.tsx +57 -0
- package/src/er/StoreHeader.tsx +1 -0
- package/src/er/layout.test.ts +85 -2
- package/src/er/layout.ts +140 -5
- package/src/er/spec.test.ts +17 -0
- package/src/er/spec.ts +23 -10
- package/src/flow/Recordings.test.tsx +52 -0
- package/src/flow/Recordings.tsx +236 -0
- package/src/flow/StepDetail.tsx +53 -0
- package/src/flow/TraceTrial.tsx +419 -0
- package/src/flow/examples.test.ts +33 -0
- package/src/flow/examples.ts +37 -0
- package/src/flow/outline.test.ts +28 -0
- package/src/flow/outline.ts +7 -2
- package/src/flow/trace-trial-resume.test.ts +42 -0
- package/src/flow/trace-trial-resume.ts +74 -0
- package/src/graph/elk.ts +78 -0
- package/src/index.css +24 -0
- package/src/landing/LandingPage.tsx +15 -11
- package/src/landing/ProductTour.tsx +6 -6
- package/src/lib/all-problems.ts +27 -17
- package/src/lib/centrality.test.ts +251 -0
- package/src/lib/centrality.ts +232 -0
- package/src/lib/confluence.test.ts +37 -0
- package/src/lib/confluence.ts +41 -0
- package/src/lib/context-color.ts +1 -1
- package/src/lib/deployment-drift.ts +22 -0
- package/src/lib/derive.ts +43 -69
- package/src/lib/environments.test.ts +83 -0
- package/src/lib/environments.ts +57 -0
- package/src/lib/integration-url.test.ts +30 -0
- package/src/lib/integration-url.ts +63 -0
- package/src/lib/kafka-ui.ts +3 -45
- package/src/lib/local-api.ts +96 -2
- package/src/lib/notion.ts +13 -0
- package/src/lib/plugin-index.json +2765 -0
- package/src/lib/plugins.test.ts +68 -0
- package/src/lib/plugins.ts +264 -0
- package/src/lib/problem-flows.test.ts +61 -0
- package/src/lib/problem-flows.ts +78 -0
- package/src/lib/problem-rules-cel.d.mts +43 -0
- package/src/lib/problem-rules-cel.mjs +407 -0
- package/src/lib/problem-rules.test.ts +287 -0
- package/src/lib/problem-rules.ts +271 -0
- package/src/lib/problem-subjects.ts +737 -0
- package/src/lib/rule-entries.ts +39 -0
- package/src/lib/{data-problems.test.ts → rules-data.test.ts} +18 -16
- package/src/lib/rules-deploy.test.ts +133 -0
- package/src/lib/{problems.test.ts → rules-edges.test.ts} +8 -3
- package/src/lib/{proto-problems.test.ts → rules-proto.test.ts} +9 -25
- package/src/lib/{wire-problems.test.ts → rules-wire.test.ts} +13 -11
- package/src/lib/rules.test.ts +51 -0
- package/src/lib/rules.ts +86 -0
- package/src/lib/shape.test.ts +32 -0
- package/src/lib/shape.ts +30 -6
- package/src/lib/tech.ts +16 -0
- package/src/lib/trace-project.test.ts +34 -0
- package/src/lib/trace-project.ts +50 -0
- package/src/lib/use-problems.ts +23 -0
- package/src/lib/warnings.test.ts +9 -0
- package/src/lib/warnings.ts +10 -10
- package/src/likec4/ids.test.ts +6 -2
- package/src/likec4/ids.ts +43 -0
- package/src/main.tsx +23 -0
- package/src/merge-deployments.test.ts +127 -0
- package/src/merge.test.ts +82 -0
- package/src/merge.ts +173 -9
- package/src/pages/AggregatePage.tsx +57 -7
- package/src/pages/ContextMap.tsx +45 -3
- package/src/pages/ContextPage.tsx +2 -0
- package/src/pages/EventPage.tsx +15 -5
- package/src/pages/FlowDetail.tsx +23 -2
- package/src/pages/GraphPage.tsx +40 -3
- package/src/pages/Overview.tsx +152 -12
- package/src/pages/PluginIndex.tsx +190 -0
- package/src/pages/Problems.tsx +396 -128
- package/src/pages/ServicePage.tsx +58 -2
- package/src/pages/Settings.tsx +28 -2
- package/src/pages/settings/AboutSettings.tsx +8 -1
- package/src/pages/settings/IntegrationsSettings.tsx +63 -17
- package/src/pages/settings/RecordingSettings.tsx +138 -0
- package/src/pages/settings/RulesSettings.tsx +825 -0
- package/src/routes.test.ts +9 -0
- package/src/routes.ts +22 -1
- package/src/lib/data-problems.ts +0 -314
- package/src/lib/proto-problems.ts +0 -237
- package/src/lib/wire-problems.ts +0 -342
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "object",
|
|
3
|
+
"additionalProperties": false,
|
|
4
|
+
"properties": {
|
|
5
|
+
"modules": {
|
|
6
|
+
"type": "string",
|
|
7
|
+
"description": "The directory the modules are in, relative to the input root: one directory per module, `Domain`, `Application`, `Infrastructure` and `IntegrationEvents` under it. Each module is a bounded context with one service.",
|
|
8
|
+
"default": "src/Modules"
|
|
9
|
+
},
|
|
10
|
+
"api": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"description": "The directory the HTTP host is in, relative to the input root. Controllers under a `Modules/<Module>` directory beneath it are that module's HTTP edge.",
|
|
13
|
+
"default": "src/API"
|
|
14
|
+
},
|
|
15
|
+
"database": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"description": "The directory the database project is in, relative to the input root: `Structure/<schema>/Tables/*.sql` and `Structure/<schema>/Views/*.sql` beneath it, one schema per module.",
|
|
18
|
+
"default": "src/Database"
|
|
19
|
+
},
|
|
20
|
+
"repo": {
|
|
21
|
+
"type": "string",
|
|
22
|
+
"description": "Where the source lives, as `host/owner/name`. Nothing in a .NET tree says, so the manifest does.",
|
|
23
|
+
"examples": ["github.com/kgrzybek/modular-monolith-with-ddd"]
|
|
24
|
+
},
|
|
25
|
+
"bus": {
|
|
26
|
+
"type": "string",
|
|
27
|
+
"description": "The address of the bus integration events cross modules on. The code names the class, `InMemoryEventBus`, and nothing else, so the manifest says.",
|
|
28
|
+
"default": "integration-events"
|
|
29
|
+
},
|
|
30
|
+
"storeKind": {
|
|
31
|
+
"enum": ["postgres", "mysql", "sqlite", "other"],
|
|
32
|
+
"description": "What the schemas live in. SQL Server is `other`.",
|
|
33
|
+
"default": "other"
|
|
34
|
+
},
|
|
35
|
+
"classification": {
|
|
36
|
+
"enum": ["core", "supporting", "generic"],
|
|
37
|
+
"description": "Whether the modules are core to the business, support it, or are generic and could be bought. One value for every module the tree declares."
|
|
38
|
+
},
|
|
39
|
+
"out": {
|
|
40
|
+
"type": "string",
|
|
41
|
+
"description": "Name of the fragment file, inside the step's output directory.",
|
|
42
|
+
"default": "domain.json"
|
|
43
|
+
},
|
|
44
|
+
"openapiOut": {
|
|
45
|
+
"type": "string",
|
|
46
|
+
"description": "Name of the inferred OpenAPI 3.1 document, inside the step's output directory, one per module with controllers; `{service}` is replaced with the module's slug.",
|
|
47
|
+
"default": "openapi.{service}.yaml"
|
|
48
|
+
},
|
|
49
|
+
"storesOut": {
|
|
50
|
+
"type": "string",
|
|
51
|
+
"description": "Name of the store fragment, inside the step's output directory. Written when the database project declares a schema a module maps to.",
|
|
52
|
+
"default": "stores.json"
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -61,8 +61,19 @@ the matching extraction step. Regenerate to apply it; other options are kept.
|
|
|
61
61
|
**Field.** Each model attribute assigned a field, with the type as written:
|
|
62
62
|
`CharField`, `DateTimeField`, and a relation as `ForeignKey[Invoice]`. The doc
|
|
63
63
|
is the field's own `help_text`, which is the one place a Django model already
|
|
64
|
-
writes down what a column means.
|
|
65
|
-
|
|
64
|
+
writes down what a column means. What the field call says a value must
|
|
65
|
+
satisfy is read into the catalog's own words (portolan.0015): `max_length`
|
|
66
|
+
is `max_len`, `unique=True` is `unique`, `choices` is `in` with the stored
|
|
67
|
+
values — read off a `Choices` class, a literal list of pairs or a module-level
|
|
68
|
+
constant, and left out when they come from a call — an `EmailField`,
|
|
69
|
+
`URLField` or `UUIDField` is `format`, a `PositiveIntegerField` is `gte 0`,
|
|
70
|
+
and `MinValueValidator`, `MaxValueValidator`, `MinLengthValidator`,
|
|
71
|
+
`MaxLengthValidator` and `RegexValidator` are the bounds they were given. The
|
|
72
|
+
model is the source of truth for `required`: a field must be given unless
|
|
73
|
+
the model fills it (`default`, `auto_now`, an auto id) or excuses it (`blank`,
|
|
74
|
+
`null`); what a serializer adds is that layer's word and is not read here.
|
|
75
|
+
Value objects are the frozen dataclasses in `values.py`, their fields the
|
|
76
|
+
annotations as written.
|
|
66
77
|
|
|
67
78
|
**Event.** `events.py`, in either of the two ways a Django project says it. A
|
|
68
79
|
dataclass with `name = "billing.InvoiceIssued"` is an event, its payload the
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""The `Choices` classes a model keeps, read as the values the column holds.
|
|
2
|
+
|
|
3
|
+
`class Status(models.TextChoices): DRAFT = "draft", "Draft"` says two things:
|
|
4
|
+
the states a lifecycle moves between, and the values a `status` field may
|
|
5
|
+
take. Both readers want the same table, member name to stored value, and
|
|
6
|
+
neither wants the label.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import ast
|
|
12
|
+
from typing import Dict
|
|
13
|
+
|
|
14
|
+
from source import assigned, bases, inner_class
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def is_choices(node: ast.AST) -> bool:
|
|
18
|
+
return isinstance(node, ast.ClassDef) and any(b.split(".")[-1].endswith("Choices") for b in bases(node))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def choices_of(model, name: str) -> Dict[str, str]:
|
|
22
|
+
"""A Choices class, as member name to the value stored in the column. Looked
|
|
23
|
+
up inside the model first, then among the module's own classes."""
|
|
24
|
+
node = inner_class(model.node, name)
|
|
25
|
+
if node is None:
|
|
26
|
+
for other in model.module.classes():
|
|
27
|
+
if other.name == name:
|
|
28
|
+
node = other
|
|
29
|
+
break
|
|
30
|
+
if node is None or not is_choices(node):
|
|
31
|
+
return {}
|
|
32
|
+
out = {}
|
|
33
|
+
for member, value, _ in assigned(node):
|
|
34
|
+
if isinstance(value, ast.Constant) and not isinstance(value.value, bool):
|
|
35
|
+
out[member] = str(value.value)
|
|
36
|
+
elif isinstance(value, ast.Tuple) and value.elts and isinstance(value.elts[0], ast.Constant):
|
|
37
|
+
out[member] = str(value.elts[0].value)
|
|
38
|
+
return out
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def choice_tables(model) -> Dict[str, Dict[str, str]]:
|
|
42
|
+
"""Every Choices class in reach of the model, by class name."""
|
|
43
|
+
out = {}
|
|
44
|
+
for node in list(model.node.body) + list(model.module.tree.body):
|
|
45
|
+
if is_choices(node):
|
|
46
|
+
out[node.name] = choices_of(model, node.name)
|
|
47
|
+
return out
|
|
@@ -16,6 +16,7 @@ from typing import Dict, List, Optional
|
|
|
16
16
|
import catalog
|
|
17
17
|
from apps import App
|
|
18
18
|
from ids import aggregate_id, block_id, pascal, singular, slug
|
|
19
|
+
from rules import rules_of
|
|
19
20
|
from source import (
|
|
20
21
|
Module,
|
|
21
22
|
Project,
|
|
@@ -315,7 +316,7 @@ def read_aggregates(
|
|
|
315
316
|
slug(model.name),
|
|
316
317
|
model.name,
|
|
317
318
|
doc(model.node),
|
|
318
|
-
[catalog.field(f.name, f.type(), f.help()) for f in model.fields],
|
|
319
|
+
[catalog.field(f.name, f.type(), f.help(), **rules_of(f, model)) for f in model.fields],
|
|
319
320
|
)
|
|
320
321
|
)
|
|
321
322
|
for module, node in value_objects(app):
|
|
@@ -303,6 +303,19 @@ class Reading(unittest.TestCase):
|
|
|
303
303
|
self.assertEqual([b["name"] for b in self.aggregate["entities"]], ["Invoice", "InvoiceLine"])
|
|
304
304
|
self.assertEqual([b["name"] for b in self.aggregate["valueObjects"]], ["Money"])
|
|
305
305
|
|
|
306
|
+
def test_a_model_field_states_its_rules_and_whether_it_must_be_given(self):
|
|
307
|
+
invoice = next(b for b in self.aggregate["entities"] if b["name"] == "Invoice")
|
|
308
|
+
fields = {f["name"]: f for f in invoice["fields"]}
|
|
309
|
+
self.assertEqual(fields["number"].get("rules"), [{"name": "max_len", "value": "32"}, {"name": "unique"}])
|
|
310
|
+
self.assertNotIn("required", fields["number"]) # null=True: the model lets it be absent
|
|
311
|
+
self.assertEqual(fields["currency"], {"name": "currency", "type": "CharField", "doc": "", "required": True, "rules": [{"name": "max_len", "value": "3"}]})
|
|
312
|
+
self.assertEqual(
|
|
313
|
+
fields["status"].get("rules"),
|
|
314
|
+
[{"name": "max_len", "value": "16"}, {"name": "in", "value": "draft, issued, paid, void"}],
|
|
315
|
+
)
|
|
316
|
+
self.assertNotIn("required", fields["status"]) # a default fills it
|
|
317
|
+
self.assertEqual(fields["id"].get("rules"), [{"name": "format", "value": "uuid"}])
|
|
318
|
+
|
|
306
319
|
def test_a_service_function_is_an_operation_and_a_write_makes_it_a_command(self):
|
|
307
320
|
kinds = {o["id"]: o["kind"] for o in self.aggregate["operations"]}
|
|
308
321
|
self.assertEqual(kinds["IssueInvoice"], "command")
|
|
@@ -15,8 +15,9 @@ from dataclasses import dataclass, field as dc_field
|
|
|
15
15
|
from typing import Dict, List, Optional, Tuple
|
|
16
16
|
|
|
17
17
|
import catalog
|
|
18
|
+
from choices import choice_tables
|
|
18
19
|
from domain import Aggregate, ModelDef
|
|
19
|
-
from source import assigned,
|
|
20
|
+
from source import assigned, const_str, dotted, keyword, methods
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
@dataclass
|
|
@@ -28,25 +29,6 @@ class Move:
|
|
|
28
29
|
sources: List[str] = dc_field(default_factory=list) # the states it may be made from, when the decorator says
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
def choices_of(model: ModelDef, name: str) -> Dict[str, str]:
|
|
32
|
-
"""A TextChoices class, as member name to the value stored in the column."""
|
|
33
|
-
node = inner_class(model.node, name)
|
|
34
|
-
if node is None:
|
|
35
|
-
for other in model.module.classes():
|
|
36
|
-
if other.name == name:
|
|
37
|
-
node = other
|
|
38
|
-
break
|
|
39
|
-
if node is None or not any(b.split(".")[-1].endswith("Choices") for b in bases(node)):
|
|
40
|
-
return {}
|
|
41
|
-
out = {}
|
|
42
|
-
for member, value, _ in assigned(node):
|
|
43
|
-
if isinstance(value, ast.Constant) and isinstance(value.value, str):
|
|
44
|
-
out[member] = value.value
|
|
45
|
-
elif isinstance(value, ast.Tuple) and value.elts and isinstance(value.elts[0], ast.Constant):
|
|
46
|
-
out[member] = value.elts[0].value
|
|
47
|
-
return out
|
|
48
|
-
|
|
49
|
-
|
|
50
32
|
def state_of(node: ast.AST, choices: Dict[str, Dict[str, str]]) -> str:
|
|
51
33
|
"""`Status.DRAFT` or `"draft"`, either way the value in the column."""
|
|
52
34
|
literal = const_str(node)
|
|
@@ -71,14 +53,6 @@ def status_field(model: ModelDef) -> Optional[str]:
|
|
|
71
53
|
return "status" if model.field("status") else None
|
|
72
54
|
|
|
73
55
|
|
|
74
|
-
def choice_tables(model: ModelDef) -> Dict[str, Dict[str, str]]:
|
|
75
|
-
out = {}
|
|
76
|
-
for node in list(model.node.body) + list(model.module.tree.body):
|
|
77
|
-
if isinstance(node, ast.ClassDef) and any(b.split(".")[-1].endswith("Choices") for b in bases(node)):
|
|
78
|
-
out[node.name] = choices_of(model, node.name)
|
|
79
|
-
return out
|
|
80
|
-
|
|
81
|
-
|
|
82
56
|
def declared_table(model: ModelDef, choices: Dict[str, Dict[str, str]]) -> Dict[str, List[str]]:
|
|
83
57
|
"""`TRANSITIONS = {Status.DRAFT: [Status.ISSUED], ...}`, in the order written."""
|
|
84
58
|
for name, value, _ in assigned(model.node):
|
|
@@ -25,6 +25,7 @@ HERE = os.path.dirname(os.path.abspath(__file__))
|
|
|
25
25
|
DESCRIPTOR = {
|
|
26
26
|
"name": "extract-django",
|
|
27
27
|
"summary": "Reads a Django service by its applications - models, events, services, DRF views, receivers, clients - into a catalog fragment, and its models into the store they are the schema of.",
|
|
28
|
+
"category": "code",
|
|
28
29
|
"phases": ["extract"],
|
|
29
30
|
}
|
|
30
31
|
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""What a model field says a value must satisfy, in the catalog's words.
|
|
2
|
+
|
|
3
|
+
A Django model states its rules in the field call: `CharField(max_length=3,
|
|
4
|
+
unique=True, choices=Status.choices, validators=[MinValueValidator(0)])`. The
|
|
5
|
+
catalog has one vocabulary for these across every source (portolan.0015), so
|
|
6
|
+
`max_length` is `max_len` here as `minLength` is in an OpenAPI document and
|
|
7
|
+
`min_len` in a proto, and the page says all three the same way.
|
|
8
|
+
|
|
9
|
+
The model is the source of truth for `required`: a field must be given when a
|
|
10
|
+
row is made unless the class fills it in (`default`, `auto_now`, an auto id),
|
|
11
|
+
lets it be empty (`blank`) or lets it be absent (`null`). What a serializer or
|
|
12
|
+
a form adds on top is that layer's word, not the model's, and is not read here.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import ast
|
|
18
|
+
from typing import Dict, List, Optional
|
|
19
|
+
|
|
20
|
+
import catalog
|
|
21
|
+
from choices import choice_tables
|
|
22
|
+
from source import assigned, dotted, keyword, keyword_bool
|
|
23
|
+
|
|
24
|
+
# The field classes whose name is itself a rule on the value.
|
|
25
|
+
FORMATS = {
|
|
26
|
+
"EmailField": "email",
|
|
27
|
+
"URLField": "uri",
|
|
28
|
+
"UUIDField": "uuid",
|
|
29
|
+
"GenericIPAddressField": "ip",
|
|
30
|
+
}
|
|
31
|
+
NON_NEGATIVE = {"PositiveIntegerField", "PositiveBigIntegerField", "PositiveSmallIntegerField"}
|
|
32
|
+
|
|
33
|
+
# Never required: the database numbers an auto id, a bool is False when not
|
|
34
|
+
# given, and a many-to-many is a table of its own with no row to be missing.
|
|
35
|
+
NEVER_REQUIRED = {"AutoField", "BigAutoField", "SmallAutoField", "BooleanField", "ManyToManyField"}
|
|
36
|
+
|
|
37
|
+
# Options under which the model itself fills or excuses the value.
|
|
38
|
+
FILLED_OR_EXCUSED = ("blank", "null", "auto_now", "auto_now_add")
|
|
39
|
+
|
|
40
|
+
VALIDATORS = {
|
|
41
|
+
"MinValueValidator": "gte",
|
|
42
|
+
"MaxValueValidator": "lte",
|
|
43
|
+
"MinLengthValidator": "min_len",
|
|
44
|
+
"MaxLengthValidator": "max_len",
|
|
45
|
+
"RegexValidator": "pattern",
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def rules_of(f, model) -> Dict[str, object]:
|
|
50
|
+
"""`required` and `rules` for `catalog.field`, as keyword arguments; an
|
|
51
|
+
empty dict when the field says nothing about its value."""
|
|
52
|
+
out: Dict[str, object] = {}
|
|
53
|
+
if required(f):
|
|
54
|
+
out["required"] = True
|
|
55
|
+
found = rules(f, model)
|
|
56
|
+
if found:
|
|
57
|
+
out["rules"] = found
|
|
58
|
+
return out
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def required(f) -> bool:
|
|
62
|
+
if f.kind in NEVER_REQUIRED:
|
|
63
|
+
return False
|
|
64
|
+
if any(keyword_bool(f.call, option) for option in FILLED_OR_EXCUSED):
|
|
65
|
+
return False
|
|
66
|
+
return keyword(f.call, "default") is None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def rules(f, model) -> List[Dict[str, str]]:
|
|
70
|
+
"""The rules in the order the source states them: what the field class
|
|
71
|
+
says first, then the options as written."""
|
|
72
|
+
out = []
|
|
73
|
+
if f.kind in FORMATS:
|
|
74
|
+
out.append(catalog.rule("format", FORMATS[f.kind]))
|
|
75
|
+
if f.kind in NON_NEGATIVE:
|
|
76
|
+
out.append(catalog.rule("gte", "0"))
|
|
77
|
+
for kw in f.call.keywords:
|
|
78
|
+
if kw.arg == "max_length":
|
|
79
|
+
length = f.integer("max_length")
|
|
80
|
+
if length is not None:
|
|
81
|
+
out.append(catalog.rule("max_len", str(length)))
|
|
82
|
+
elif kw.arg == "unique" and keyword_bool(f.call, "unique"):
|
|
83
|
+
out.append(catalog.rule("unique"))
|
|
84
|
+
elif kw.arg == "choices":
|
|
85
|
+
values = choice_values(kw.value, model)
|
|
86
|
+
if values:
|
|
87
|
+
out.append(catalog.rule("in", ", ".join(values)))
|
|
88
|
+
elif kw.arg == "validators":
|
|
89
|
+
out.extend(validator_rules(kw.value))
|
|
90
|
+
return out
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def choice_values(node: ast.AST, model, depth: int = 0) -> List[str]:
|
|
94
|
+
"""The values a `choices=` option allows, in the order it lists them.
|
|
95
|
+
|
|
96
|
+
`Status.choices` reads the Choices class; a literal list of pairs or of
|
|
97
|
+
bare values reads the value of each pair, descending into a group's own
|
|
98
|
+
list; a name reads the module-level constant it is assigned. Anything
|
|
99
|
+
else - a call, an import the module does not define - yields nothing, and
|
|
100
|
+
nothing is what the field then says, rather than an empty `in`.
|
|
101
|
+
"""
|
|
102
|
+
if depth > 3:
|
|
103
|
+
return []
|
|
104
|
+
if isinstance(node, ast.Attribute) and node.attr == "choices":
|
|
105
|
+
table = choice_tables(model).get(dotted(node.value).split(".")[-1], {})
|
|
106
|
+
return list(table.values())
|
|
107
|
+
if isinstance(node, (ast.List, ast.Tuple)):
|
|
108
|
+
out: List[str] = []
|
|
109
|
+
for item in node.elts:
|
|
110
|
+
if isinstance(item, ast.Constant) and not isinstance(item.value, bool):
|
|
111
|
+
out.append(str(item.value))
|
|
112
|
+
elif isinstance(item, (ast.List, ast.Tuple)) and item.elts:
|
|
113
|
+
first = item.elts[0]
|
|
114
|
+
second = item.elts[1] if len(item.elts) > 1 else None
|
|
115
|
+
if isinstance(second, (ast.List, ast.Tuple)):
|
|
116
|
+
out.extend(choice_values(second, model, depth + 1)) # a group: (label, [pairs])
|
|
117
|
+
elif isinstance(first, ast.Constant) and not isinstance(first.value, bool):
|
|
118
|
+
out.append(str(first.value))
|
|
119
|
+
elif isinstance(first, ast.Attribute):
|
|
120
|
+
value = choice_member(first, model)
|
|
121
|
+
if value:
|
|
122
|
+
out.append(value)
|
|
123
|
+
return out
|
|
124
|
+
if isinstance(node, ast.Name):
|
|
125
|
+
for name, value, _ in assigned(model.module.tree):
|
|
126
|
+
if name == node.id:
|
|
127
|
+
return choice_values(value, model, depth + 1)
|
|
128
|
+
return []
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def choice_member(node: ast.Attribute, model) -> Optional[str]:
|
|
132
|
+
"""`Status.DRAFT` in a literal list of pairs: the member's stored value."""
|
|
133
|
+
table = choice_tables(model).get(dotted(node.value).split(".")[-1], {})
|
|
134
|
+
return table.get(node.attr)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def validator_rules(node: ast.AST) -> List[Dict[str, str]]:
|
|
138
|
+
"""`validators=[MinValueValidator(0), RegexValidator(r"^[A-Z]+$")]`: the
|
|
139
|
+
validators the catalog has words for, with the bound they were given. A
|
|
140
|
+
validator built from a name the module computes, or a regex meant to be
|
|
141
|
+
NOT matched, is left out rather than guessed."""
|
|
142
|
+
if not isinstance(node, (ast.List, ast.Tuple)):
|
|
143
|
+
return []
|
|
144
|
+
out = []
|
|
145
|
+
for item in node.elts:
|
|
146
|
+
if not isinstance(item, ast.Call):
|
|
147
|
+
continue
|
|
148
|
+
rule = VALIDATORS.get(dotted(item.func).split(".")[-1])
|
|
149
|
+
if rule is None or keyword_bool(item, "inverse_match"):
|
|
150
|
+
continue
|
|
151
|
+
given = item.args[0] if item.args else keyword(item, "limit_value") or keyword(item, "regex")
|
|
152
|
+
if isinstance(given, ast.Constant) and not isinstance(given.value, bool):
|
|
153
|
+
out.append(catalog.rule(rule, str(given.value)))
|
|
154
|
+
return out
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""The rules a model field states, read one option at a time."""
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import unittest
|
|
5
|
+
|
|
6
|
+
from domain import FieldDef
|
|
7
|
+
from rules import rules_of
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Module:
|
|
11
|
+
"""The slice of a Module the rules reader touches: the tree and its classes."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, tree: ast.Module):
|
|
14
|
+
self.tree = tree
|
|
15
|
+
|
|
16
|
+
def classes(self):
|
|
17
|
+
return [node for node in self.tree.body if isinstance(node, ast.ClassDef)]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Model:
|
|
21
|
+
def __init__(self, node: ast.ClassDef, module: Module):
|
|
22
|
+
self.node = node
|
|
23
|
+
self.module = module
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def fields(source: str):
|
|
27
|
+
"""Every field of the last class in `source`, as the extractor reads it,
|
|
28
|
+
with the integer options it resolves ahead of time."""
|
|
29
|
+
tree = ast.parse(source)
|
|
30
|
+
node = [n for n in tree.body if isinstance(n, ast.ClassDef)][-1]
|
|
31
|
+
model = Model(node, Module(tree))
|
|
32
|
+
out = {}
|
|
33
|
+
for stmt in node.body:
|
|
34
|
+
if isinstance(stmt, ast.Assign) and isinstance(stmt.value, ast.Call):
|
|
35
|
+
name = stmt.targets[0].id
|
|
36
|
+
call = stmt.value
|
|
37
|
+
kind = call.func.attr if isinstance(call.func, ast.Attribute) else call.func.id
|
|
38
|
+
integers = {}
|
|
39
|
+
for kw in call.keywords:
|
|
40
|
+
if kw.arg == "max_length" and isinstance(kw.value, ast.Constant):
|
|
41
|
+
integers["max_length"] = kw.value.value
|
|
42
|
+
out[name] = rules_of(FieldDef(name=name, kind=kind, call=call, node=stmt, integers=integers), model)
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Options(unittest.TestCase):
|
|
47
|
+
def test_the_options_become_rules_in_the_order_written(self):
|
|
48
|
+
got = fields(
|
|
49
|
+
"class Invoice(models.Model):\n"
|
|
50
|
+
" number = models.CharField(max_length=32, unique=True, null=True)\n"
|
|
51
|
+
" currency = models.CharField(unique=True, max_length=3)\n"
|
|
52
|
+
)
|
|
53
|
+
self.assertEqual(got["number"], {"rules": [{"name": "max_len", "value": "32"}, {"name": "unique"}]})
|
|
54
|
+
self.assertEqual(got["currency"], {"required": True, "rules": [{"name": "unique"}, {"name": "max_len", "value": "3"}]})
|
|
55
|
+
|
|
56
|
+
def test_the_field_class_speaks_first(self):
|
|
57
|
+
got = fields(
|
|
58
|
+
"class Line(models.Model):\n"
|
|
59
|
+
" email = models.EmailField(max_length=254)\n"
|
|
60
|
+
" quantity = models.PositiveIntegerField()\n"
|
|
61
|
+
" site = models.URLField(blank=True)\n"
|
|
62
|
+
" owner = models.UUIDField(default=uuid.uuid4)\n"
|
|
63
|
+
)
|
|
64
|
+
self.assertEqual(got["email"]["rules"], [{"name": "format", "value": "email"}, {"name": "max_len", "value": "254"}])
|
|
65
|
+
self.assertEqual(got["quantity"], {"required": True, "rules": [{"name": "gte", "value": "0"}]})
|
|
66
|
+
self.assertEqual(got["site"], {"rules": [{"name": "format", "value": "uri"}]})
|
|
67
|
+
self.assertEqual(got["owner"], {"rules": [{"name": "format", "value": "uuid"}]})
|
|
68
|
+
|
|
69
|
+
def test_validators_carry_their_bound(self):
|
|
70
|
+
got = fields(
|
|
71
|
+
"class Line(models.Model):\n"
|
|
72
|
+
" rate = models.DecimalField(validators=[MinValueValidator(0), MaxValueValidator(limit_value=1)])\n"
|
|
73
|
+
" code = models.CharField(validators=[RegexValidator(r'^[A-Z]{3}$'), RegexValidator('x', inverse_match=True), validators.MinLengthValidator(2)])\n"
|
|
74
|
+
" other = models.CharField(validators=[custom(), MinValueValidator(LIMIT)])\n"
|
|
75
|
+
)
|
|
76
|
+
self.assertEqual(got["rate"]["rules"], [{"name": "gte", "value": "0"}, {"name": "lte", "value": "1"}])
|
|
77
|
+
self.assertEqual(got["code"]["rules"], [{"name": "pattern", "value": "^[A-Z]{3}$"}, {"name": "min_len", "value": "2"}])
|
|
78
|
+
self.assertEqual(got["other"], {"required": True})
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Choices(unittest.TestCase):
|
|
82
|
+
def test_a_choices_class_is_read_by_its_stored_values(self):
|
|
83
|
+
got = fields(
|
|
84
|
+
"class Invoice(models.Model):\n"
|
|
85
|
+
" class Status(models.TextChoices):\n"
|
|
86
|
+
" DRAFT = 'draft', 'Draft'\n"
|
|
87
|
+
" PAID = 'paid', 'Paid'\n"
|
|
88
|
+
" status = models.CharField(max_length=16, choices=Status.choices, default=Status.DRAFT)\n"
|
|
89
|
+
)
|
|
90
|
+
self.assertEqual(got["status"], {"rules": [{"name": "max_len", "value": "16"}, {"name": "in", "value": "draft, paid"}]})
|
|
91
|
+
|
|
92
|
+
def test_a_module_level_class_and_an_integer_choices_read_the_same(self):
|
|
93
|
+
got = fields(
|
|
94
|
+
"class Priority(models.IntegerChoices):\n"
|
|
95
|
+
" LOW = 1\n"
|
|
96
|
+
" HIGH = 2, 'High'\n"
|
|
97
|
+
"class Ticket(models.Model):\n"
|
|
98
|
+
" priority = models.IntegerField(choices=Priority.choices)\n"
|
|
99
|
+
)
|
|
100
|
+
self.assertEqual(got["priority"], {"required": True, "rules": [{"name": "in", "value": "1, 2"}]})
|
|
101
|
+
|
|
102
|
+
def test_literal_lists_bare_values_and_groups(self):
|
|
103
|
+
got = fields(
|
|
104
|
+
"SIZES = [('s', 'Small'), ('m', 'Medium')]\n"
|
|
105
|
+
"class Shirt(models.Model):\n"
|
|
106
|
+
" size = models.CharField(choices=SIZES)\n"
|
|
107
|
+
" fit = models.CharField(choices=['slim', 'regular'])\n"
|
|
108
|
+
" colour = models.CharField(choices=[('Warm', [('red', 'Red')]), ('Cold', [('blue', 'Blue')])])\n"
|
|
109
|
+
" made = models.CharField(choices=load_choices())\n"
|
|
110
|
+
)
|
|
111
|
+
self.assertEqual(got["size"]["rules"], [{"name": "in", "value": "s, m"}])
|
|
112
|
+
self.assertEqual(got["fit"]["rules"], [{"name": "in", "value": "slim, regular"}])
|
|
113
|
+
self.assertEqual(got["colour"]["rules"], [{"name": "in", "value": "red, blue"}])
|
|
114
|
+
self.assertEqual(got["made"], {"required": True})
|
|
115
|
+
|
|
116
|
+
def test_a_pair_naming_a_member_reads_its_value(self):
|
|
117
|
+
got = fields(
|
|
118
|
+
"class Kind(models.TextChoices):\n"
|
|
119
|
+
" ORDER = 'order', 'Order'\n"
|
|
120
|
+
"class Invoice(models.Model):\n"
|
|
121
|
+
" kind = models.CharField(choices=[(Kind.ORDER, 'An order')])\n"
|
|
122
|
+
)
|
|
123
|
+
self.assertEqual(got["kind"]["rules"], [{"name": "in", "value": "order"}])
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class Required(unittest.TestCase):
|
|
127
|
+
def test_the_model_fills_or_excuses_the_value(self):
|
|
128
|
+
got = fields(
|
|
129
|
+
"class Invoice(models.Model):\n"
|
|
130
|
+
" id = models.BigAutoField(primary_key=True)\n"
|
|
131
|
+
" order_id = models.UUIDField()\n"
|
|
132
|
+
" number = models.CharField(max_length=32, null=True)\n"
|
|
133
|
+
" note = models.TextField(blank=True)\n"
|
|
134
|
+
" created = models.DateTimeField(auto_now_add=True)\n"
|
|
135
|
+
" status = models.CharField(max_length=8, default='draft')\n"
|
|
136
|
+
" paid = models.BooleanField()\n"
|
|
137
|
+
" tags = models.ManyToManyField('Tag')\n"
|
|
138
|
+
" invoice = models.ForeignKey('Invoice', on_delete=models.CASCADE)\n"
|
|
139
|
+
)
|
|
140
|
+
required = {name: value.get("required", False) for name, value in got.items()}
|
|
141
|
+
self.assertEqual(
|
|
142
|
+
required,
|
|
143
|
+
{
|
|
144
|
+
"id": False,
|
|
145
|
+
"order_id": True,
|
|
146
|
+
"number": False,
|
|
147
|
+
"note": False,
|
|
148
|
+
"created": False,
|
|
149
|
+
"status": False,
|
|
150
|
+
"paid": False,
|
|
151
|
+
"tags": False,
|
|
152
|
+
"invoice": True,
|
|
153
|
+
},
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
if __name__ == "__main__":
|
|
158
|
+
unittest.main()
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "object",
|
|
3
|
+
"additionalProperties": false,
|
|
4
|
+
"properties": {
|
|
5
|
+
"context": {
|
|
6
|
+
"type": "string",
|
|
7
|
+
"description": "Slug of the bounded context this service belongs to. Defaults to the name of the input directory.",
|
|
8
|
+
"examples": ["shop"]
|
|
9
|
+
},
|
|
10
|
+
"service": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"description": "Slug of the service. Defaults to the name of the input directory.",
|
|
13
|
+
"examples": ["fulfillment"]
|
|
14
|
+
},
|
|
15
|
+
"out": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"description": "Name of the fragment file, inside the step's output directory.",
|
|
18
|
+
"default": "sqs.json"
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -11,9 +11,10 @@ var optionsSchema []byte
|
|
|
11
11
|
|
|
12
12
|
func descriptor() plugin.Descriptor {
|
|
13
13
|
return plugin.Descriptor{
|
|
14
|
-
Name:
|
|
15
|
-
Summary:
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
Name: "extract-http-clients",
|
|
15
|
+
Summary: "Reads outbound HTTP/SOAP calls and joins Go routes, handlers, factories, and providers into source-backed flows.",
|
|
16
|
+
Category: plugin.CategoryCode,
|
|
17
|
+
Phases: []string{plugin.PhaseExtract},
|
|
18
|
+
Options: optionsSchema,
|
|
18
19
|
}
|
|
19
20
|
}
|
|
Binary file
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "object",
|
|
3
|
+
"additionalProperties": false,
|
|
4
|
+
"properties": {
|
|
5
|
+
"context": {
|
|
6
|
+
"type": "string",
|
|
7
|
+
"description": "Slug of the bounded context this service belongs to. Defaults to the name of the input directory.",
|
|
8
|
+
"examples": ["shop"]
|
|
9
|
+
},
|
|
10
|
+
"service": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"description": "Slug of the service. Defaults to the name of the input directory. The workload is found by this name: a Deployment, StatefulSet, DaemonSet, Job or CronJob called so, or labelled so with `app` or `app.kubernetes.io/name`; when the tree holds exactly one workload, that one.",
|
|
13
|
+
"examples": ["pricing"]
|
|
14
|
+
},
|
|
15
|
+
"paths": {
|
|
16
|
+
"type": "array",
|
|
17
|
+
"items": { "type": "string" },
|
|
18
|
+
"description": "Directories under the input root that hold the manifests, so a repository's other YAML - compose files, CI, buf - is not walked. Left out, the whole root is read and anything without `apiVersion` and `kind` is passed over.",
|
|
19
|
+
"examples": [["deploy/k8s"]]
|
|
20
|
+
},
|
|
21
|
+
"namespace": {
|
|
22
|
+
"type": "string",
|
|
23
|
+
"description": "Read only the objects in this namespace. An object that names no namespace is read either way. Left out, every namespace.",
|
|
24
|
+
"examples": ["shop"]
|
|
25
|
+
},
|
|
26
|
+
"out": {
|
|
27
|
+
"type": "string",
|
|
28
|
+
"description": "Name of the fragment file, inside the step's output directory.",
|
|
29
|
+
"default": "k8s.json"
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|