@shortlink-org/portolan 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +23 -0
  2. package/catalog/enum_test.go +46 -0
  3. package/catalog/evidence_test.go +35 -0
  4. package/catalog/model.go +1066 -0
  5. package/catalog/roundtrip_test.go +203 -0
  6. package/catalog/via_test.go +38 -0
  7. package/cli/init.test.mjs +6 -1
  8. package/cli/portolan.mjs +8 -0
  9. package/cli/portolan.test.mjs +49 -0
  10. package/go.mod +14 -0
  11. package/go.sum +20 -0
  12. package/internal/gocall/README.md +19 -0
  13. package/internal/gocall/analyze.go +189 -0
  14. package/internal/gocall/analyze_test.go +107 -0
  15. package/internal/gohttp/analyze.go +2562 -0
  16. package/internal/gohttp/destination.go +373 -0
  17. package/internal/gohttp/endpoints.go +1067 -0
  18. package/internal/gohttp/roots.go +320 -0
  19. package/internal/gohttp/typed.go +96 -0
  20. package/internal/goscan/constants.go +85 -0
  21. package/internal/goscan/goscan_test.go +227 -0
  22. package/internal/goscan/index.go +629 -0
  23. package/internal/goscan/index_test.go +66 -0
  24. package/internal/goscan/names.go +52 -0
  25. package/internal/goscan/parse_test.go +11 -0
  26. package/internal/goscan/source.go +37 -0
  27. package/internal/goscan/tree.go +284 -0
  28. package/internal/goscan/types.go +99 -0
  29. package/internal/wsdl/ids.go +127 -0
  30. package/internal/wsdl/ids_test.go +21 -0
  31. package/internal/wsdl/model.go +70 -0
  32. package/internal/wsdl/parse.go +949 -0
  33. package/internal/wsdl/parse_test.go +170 -0
  34. package/package.json +22 -10
  35. package/plugin/describe.go +118 -0
  36. package/plugin/describe_test.go +114 -0
  37. package/plugin/protocol.go +141 -0
  38. package/plugin/schematest/schematest.go +126 -0
  39. package/plugins/README.md +114 -46
  40. package/plugins/cmd/portolan-http-clients/main.go +19 -0
  41. package/plugins/extract-celery/extract.py +0 -2
  42. package/plugins/extract-celery/extract_test.py +1 -1
  43. package/plugins/extract-django/README.md +39 -17
  44. package/plugins/extract-django/domain.py +28 -17
  45. package/plugins/extract-django/extract.py +21 -7
  46. package/plugins/extract-django/extract_test.py +55 -2
  47. package/plugins/extract-django/lifecycle.py +2 -0
  48. package/plugins/extract-django/operations.py +1 -1
  49. package/plugins/extract-django/routing_test.py +109 -1
  50. package/plugins/extract-django/store.py +1 -1
  51. package/plugins/extract-django/transport.py +101 -55
  52. package/plugins/extract-django/verbs.py +241 -0
  53. package/plugins/extract-go/README.md +47 -0
  54. package/plugins/extract-http-clients/describe.go +19 -0
  55. package/plugins/extract-http-clients/describe_test.go +11 -0
  56. package/plugins/extract-http-clients/extract.go +740 -0
  57. package/plugins/extract-http-clients/extract_test.go +1561 -0
  58. package/plugins/extract-http-clients/main.go +41 -0
  59. package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
  60. package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
  61. package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
  62. package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
  63. package/plugins/extract-python-kafka/README.md +6 -0
  64. package/plugins/extract-python-kafka/extract.py +2 -4
  65. package/plugins/extract-python-kafka/extract_test.py +18 -2
  66. package/plugins/extract-ts/extract.test.ts +2 -2
  67. package/plugins/extract-ts/extract.ts +4 -5
  68. package/plugins/extract-ts/graphql.test.ts +1 -1
  69. package/plugins/openapi/ids.go +261 -0
  70. package/plugins/openapi/ids_test.go +98 -0
  71. package/plugins/portolan-go.wasm +0 -0
  72. package/plugins/pyplugin/catalog.py +12 -1
  73. package/plugins/pyplugin/kafka.py +74 -3
  74. package/plugins/pyplugin/protocol.py +1 -5
  75. package/portolan.json +3 -2
  76. package/schema/portolan.schema.json +34 -0
  77. package/scripts/README.md +18 -12
  78. package/scripts/catalog-sources.mjs +6 -0
  79. package/scripts/delivery-presets.mjs +21 -11
  80. package/scripts/diff.mjs +5 -1
  81. package/scripts/django-aggregates.test.mjs +58 -0
  82. package/scripts/gen-likec4.mjs +79 -21
  83. package/scripts/gen-likec4.test.mjs +25 -2
  84. package/scripts/gen.mjs +118 -115
  85. package/scripts/go-discovery.test.mjs +30 -0
  86. package/scripts/history.mjs +186 -3
  87. package/scripts/history.test.mjs +1 -1
  88. package/scripts/host-plugins/fetch-git.mjs +77 -21
  89. package/scripts/host-plugins/fetch-git.test.mjs +62 -8
  90. package/scripts/local-api.mjs +71 -4
  91. package/scripts/local-api.test.mjs +63 -4
  92. package/scripts/local-discovery.mjs +82 -9
  93. package/scripts/manifest.mjs +5 -3
  94. package/scripts/manifest.test.mjs +24 -0
  95. package/scripts/output-diff.mjs +94 -0
  96. package/scripts/output-diff.test.mjs +36 -0
  97. package/scripts/package-smoke.mjs +62 -4
  98. package/scripts/plugin-host.mjs +22 -2
  99. package/scripts/plugin-host.test.mjs +9 -0
  100. package/scripts/plugin-wasm-worker.mjs +4 -1
  101. package/scripts/provenance.mjs +72 -0
  102. package/scripts/provenance.test.mjs +149 -0
  103. package/scripts/run-builtin.mjs +39 -5
  104. package/scripts/schema.mjs +29 -0
  105. package/scripts/warning-policy.mjs +161 -0
  106. package/scripts/warning-policy.test.mjs +56 -0
  107. package/src/app/Breadcrumbs.test.ts +4 -0
  108. package/src/app/Breadcrumbs.tsx +1 -0
  109. package/src/app/Sidebar.tsx +3 -3
  110. package/src/catalog-docs.test.ts +64 -0
  111. package/src/catalog-docs.ts +35 -0
  112. package/src/catalog-error.test.ts +15 -0
  113. package/src/catalog-model.ts +70 -6
  114. package/src/catalog-stores.test.ts +17 -0
  115. package/src/catalog-validation.ts +52 -2
  116. package/src/catalog.test.ts +13 -2
  117. package/src/chat/Starter.tsx +5 -11
  118. package/src/chat/tools.test.ts +27 -0
  119. package/src/chat/tools.ts +5 -9
  120. package/src/components/CatalogStamp.tsx +10 -8
  121. package/src/components/ChannelRows.messagepack.test.tsx +28 -0
  122. package/src/components/ChannelRows.test.tsx +54 -0
  123. package/src/components/ChannelRows.tsx +57 -10
  124. package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
  125. package/src/components/HTTPDestinationEvidence.tsx +31 -0
  126. package/src/components/Integrations.tsx +1 -1
  127. package/src/components/LifecycleDiagram.tsx +28 -12
  128. package/src/components/MachineDocs.tsx +6 -5
  129. package/src/components/MethodRows.tsx +9 -2
  130. package/src/components/ProblemRow.tsx +4 -0
  131. package/src/components/RelationEvidence.test.tsx +14 -0
  132. package/src/components/RelationEvidence.tsx +53 -0
  133. package/src/components/WhatLinksHere.tsx +6 -4
  134. package/src/data.ts +25 -7
  135. package/src/enrich.test.ts +336 -6
  136. package/src/enrich.ts +206 -3
  137. package/src/flow/StepDetail.tsx +104 -54
  138. package/src/flow/answers.test.ts +18 -1
  139. package/src/flow/answers.ts +37 -8
  140. package/src/flow/evidence.test.ts +16 -0
  141. package/src/flow/evidence.ts +34 -0
  142. package/src/index.css +44 -0
  143. package/src/landing/DraggableReveal.tsx +3 -2
  144. package/src/landing/EvidencePipeline.tsx +105 -0
  145. package/src/landing/LandingPage.tsx +2 -59
  146. package/src/lib/backlinks.test.ts +16 -1
  147. package/src/lib/backlinks.ts +20 -0
  148. package/src/lib/catalog-diff.test.ts +18 -0
  149. package/src/lib/catalog-diff.ts +20 -2
  150. package/src/lib/derive.ts +1 -0
  151. package/src/lib/django-aggregates.d.mts +9 -0
  152. package/src/lib/django-aggregates.mjs +36 -0
  153. package/src/lib/django-aggregates.test.ts +29 -0
  154. package/src/lib/django-aggregates.ts +5 -0
  155. package/src/lib/kafka-ui.test.ts +87 -0
  156. package/src/lib/kafka-ui.ts +105 -0
  157. package/src/lib/local-api.ts +20 -2
  158. package/src/lib/setup-info.test.ts +17 -0
  159. package/src/lib/setup-info.ts +58 -0
  160. package/src/lib/warnings.test.ts +54 -0
  161. package/src/lib/warnings.ts +260 -0
  162. package/src/lib/wire-problems.test.ts +21 -0
  163. package/src/lib/wire-problems.ts +62 -1
  164. package/src/likec4/FlowView.tsx +2 -6
  165. package/src/likec4/flow-edges.test.ts +64 -1
  166. package/src/likec4/flow-edges.ts +43 -7
  167. package/src/likec4/view-index.ts +8 -2
  168. package/src/map/ContextMapGraph.tsx +76 -32
  169. package/src/merge.test.ts +23 -0
  170. package/src/merge.ts +33 -10
  171. package/src/pages/AggregatePage.tsx +8 -7
  172. package/src/pages/CatalogFailure.tsx +2 -2
  173. package/src/pages/ContextPage.tsx +6 -5
  174. package/src/pages/ServicePage.tsx +4 -3
  175. package/src/pages/Settings.tsx +200 -44
  176. package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
  177. package/src/pages/settings/IntegrationsSettings.tsx +117 -0
  178. package/src/routes.test.ts +2 -0
  179. package/src/routes.ts +2 -1
  180. package/src/selection/DetailPanel.tsx +61 -1
  181. package/src/virtual-provenance.d.ts +11 -0
  182. package/vite.config.ts +5 -0
  183. package/scripts/vendor-lock.mjs +0 -58
  184. package/scripts/vendor-lock.test.mjs +0 -69
@@ -23,6 +23,8 @@ class Spec:
23
23
  CONSTRUCTORS = {
24
24
  "confluent_kafka.Producer": Spec("confluent-kafka", "producer"),
25
25
  "confluent_kafka.Consumer": Spec("confluent-kafka", "consumer"),
26
+ "confluent_kafka.SerializingProducer": Spec("confluent-kafka", "producer"),
27
+ "confluent_kafka.DeserializingConsumer": Spec("confluent-kafka", "consumer"),
26
28
  "kafka.KafkaProducer": Spec("kafka-python", "producer"),
27
29
  "kafka.KafkaConsumer": Spec("kafka-python", "consumer"),
28
30
  "aiokafka.AIOKafkaProducer": Spec("aiokafka", "producer"),
@@ -62,6 +64,23 @@ CONFIG_KEYS = {
62
64
  "enable_auto_commit": "auto commit",
63
65
  "key_serializer": "key serializer",
64
66
  "value_serializer": "value serializer",
67
+ "key_deserializer": "key deserializer",
68
+ "value_deserializer": "value deserializer",
69
+ "key.serializer": "key serializer",
70
+ "value.serializer": "value serializer",
71
+ "key.deserializer": "key deserializer",
72
+ "value.deserializer": "value deserializer",
73
+ }
74
+
75
+ SERDE_KEYS = {
76
+ "key_serializer",
77
+ "value_serializer",
78
+ "key_deserializer",
79
+ "value_deserializer",
80
+ "key.serializer",
81
+ "value.serializer",
82
+ "key.deserializer",
83
+ "value.deserializer",
65
84
  }
66
85
 
67
86
 
@@ -73,6 +92,10 @@ class Client:
73
92
  source: str = ""
74
93
  constructor_topics: List[str] = field(default_factory=list)
75
94
 
95
+ @property
96
+ def encoding(self) -> str:
97
+ return encoding_of_text(" ".join(str(self.config.get(key, "")) for key in ("value serializer", "value deserializer")))
98
+
76
99
 
77
100
  @dataclass
78
101
  class Publish:
@@ -86,6 +109,7 @@ class Publish:
86
109
  message_expression: str
87
110
  key: str = ""
88
111
  headers: str = ""
112
+ encoding: str = ""
89
113
 
90
114
  @property
91
115
  def line(self) -> str:
@@ -181,6 +205,8 @@ def value(
181
205
  for key_node, val_node in zip(node.keys, node.values):
182
206
  key = value(project, module, key_node, settings, variables, depth + 1)
183
207
  val = value(project, module, val_node, settings, variables, depth + 1)
208
+ if val is None and isinstance(key, str) and key in SERDE_KEYS:
209
+ val = serialization_expression(module, val_node, variables)
184
210
  if not isinstance(key, str) or val is None:
185
211
  continue
186
212
  out[key] = val
@@ -239,14 +265,21 @@ def _config_values(project: Project, module: Module, call: ast.Call, settings: s
239
265
  first = value(project, module, call.args[0], settings, variables)
240
266
  if isinstance(first, dict):
241
267
  raw.update(first)
268
+ # Callable serializer values are not safe scalar values, but their
269
+ # syntax is still the evidence needed to identify MessagePack.
270
+ if isinstance(call.args[0], ast.Dict):
271
+ for key_node, val_node in zip(call.args[0].keys, call.args[0].values):
272
+ key = value(project, module, key_node, settings, variables)
273
+ if isinstance(key, str) and key in SERDE_KEYS and key not in raw:
274
+ raw[key] = serialization_expression(module, val_node, variables)
242
275
  for kw in call.keywords:
243
276
  if not kw.arg:
244
277
  continue
245
278
  resolved = value(project, module, kw.value, settings, variables)
246
279
  if resolved is not None:
247
280
  raw[kw.arg] = resolved
248
- elif kw.arg in ("key_serializer", "value_serializer"):
249
- raw[kw.arg] = expression(kw.value)
281
+ elif kw.arg in SERDE_KEYS:
282
+ raw[kw.arg] = serialization_expression(module, kw.value, variables)
250
283
  out: Dict[str, Any] = {}
251
284
  for key, val in raw.items():
252
285
  normalized = CONFIG_KEYS.get(key)
@@ -377,7 +410,7 @@ def payload_name(node: Optional[ast.AST], variables: Optional[Dict[str, Tuple[st
377
410
  return node.id
378
411
  if isinstance(node, ast.Call):
379
412
  last = dotted(node.func).split(".")[-1]
380
- if last in ("dumps", "dump", "encode", "serialize", "SerializeToString", "asdict", "dict", "model_dump") and node.args:
413
+ if last in ("dumps", "dump", "encode", "serialize", "SerializeToString", "asdict", "dict", "model_dump", "pack", "packb") and node.args:
381
414
  return payload_name(node.args[0], variables, depth + 1)
382
415
  return last or "message"
383
416
  if isinstance(node, ast.Attribute):
@@ -387,6 +420,43 @@ def payload_name(node: Optional[ast.AST], variables: Optional[Dict[str, Tuple[st
387
420
  return "message"
388
421
 
389
422
 
423
+ def encoding_of_text(text: str) -> str:
424
+ lowered = text.lower()
425
+ if "msgpack" in lowered or "messagepack" in lowered:
426
+ return "msgpack"
427
+ return ""
428
+
429
+
430
+ def serialization_expression(module: Optional[Module], node: Optional[ast.AST], variables: Optional[Dict[str, Tuple[str, object]]] = None) -> str:
431
+ if node is None:
432
+ return ""
433
+ parts = [expression(node)]
434
+ if module is not None:
435
+ parts.append(external_name(module, dotted(node.func) if isinstance(node, ast.Call) else dotted(node)))
436
+ owner = node.func.value if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) else (node.value if isinstance(node, ast.Attribute) else None)
437
+ if isinstance(owner, ast.Name):
438
+ bound = (variables or {}).get(owner.id)
439
+ if bound is not None and bound[0] == "expr" and isinstance(bound[1], ast.AST):
440
+ parts.append(expression(bound[1]))
441
+ return " ".join(filter(None, parts))
442
+
443
+
444
+ def payload_encoding(
445
+ node: Optional[ast.AST],
446
+ variables: Optional[Dict[str, Tuple[str, object]]] = None,
447
+ depth: int = 0,
448
+ module: Optional[Module] = None,
449
+ ) -> str:
450
+ if node is None or depth > 6:
451
+ return ""
452
+ variables = variables or {}
453
+ if isinstance(node, ast.Name):
454
+ bound = variables.get(node.id)
455
+ if bound is not None and bound[0] == "expr" and isinstance(bound[1], ast.AST):
456
+ return payload_encoding(bound[1], variables, depth + 1, module)
457
+ return encoding_of_text(serialization_expression(module, node, variables))
458
+
459
+
390
460
  def _text(node: Optional[ast.AST]) -> str:
391
461
  text = expression(node)
392
462
  return text if len(text) <= 100 else text[:97] + "..."
@@ -541,6 +611,7 @@ class Scanner:
541
611
  _text(payload),
542
612
  _text(keyword(call, "key")),
543
613
  _text(keyword(call, "headers")),
614
+ payload_encoding(payload, env, module=module) or client.encoding,
544
615
  )
545
616
  )
546
617
 
@@ -17,12 +17,10 @@ from typing import Any, Dict, List
17
17
 
18
18
  @dataclass
19
19
  class Input:
20
- """Where the source is, and the stamp the host put on this run."""
20
+ """Where the source is."""
21
21
 
22
22
  root: str = ""
23
23
  output: str = ""
24
- commit: str = ""
25
- generated_at: str = ""
26
24
 
27
25
  @staticmethod
28
26
  def of(raw: Any) -> "Input":
@@ -30,8 +28,6 @@ class Input:
30
28
  return Input(
31
29
  root=raw.get("root", ""),
32
30
  output=raw.get("output", ""),
33
- commit=raw.get("commit", ""),
34
- generated_at=raw.get("generatedAt", ""),
35
31
  )
36
32
 
37
33
 
package/portolan.json CHANGED
@@ -167,8 +167,9 @@
167
167
  },
168
168
  {
169
169
  "name": "http-clients",
170
- "wasm": {
171
- "url": "file://plugins/portolan-go.wasm"
170
+ "process": {
171
+ "command": "go",
172
+ "args": ["run", "./plugins/cmd/portolan-http-clients"]
172
173
  }
173
174
  },
174
175
  {
@@ -44,6 +44,14 @@
44
44
  },
45
45
  "description": "The source projects that make up the estate. A project gives repeated pipeline inputs one name for the generated site's Settings page; estate-wide inputs such as flows need no project."
46
46
  },
47
+ "warningPolicies": {
48
+ "type": "array",
49
+ "maxItems": 100,
50
+ "items": {
51
+ "$ref": "#/$defs/warningPolicy"
52
+ },
53
+ "description": "CEL policies for reviewed extraction limitations. Expressions are type-checked when the manifest is read and suppression always requires a reason."
54
+ },
47
55
  "plugins": {
48
56
  "type": "array",
49
57
  "items": {
@@ -795,6 +803,32 @@
795
803
  }
796
804
  },
797
805
  "$defs": {
806
+ "warningPolicy": {
807
+ "type": "object",
808
+ "additionalProperties": false,
809
+ "required": [
810
+ "when",
811
+ "action",
812
+ "reason"
813
+ ],
814
+ "properties": {
815
+ "when": {
816
+ "type": "string",
817
+ "minLength": 1,
818
+ "maxLength": 1000,
819
+ "description": "Boolean CEL expression over plugin, rule, severity, project, phase, ref, message and count."
820
+ },
821
+ "action": {
822
+ "const": "suppress",
823
+ "description": "Suppress matching diagnostics from the active view while retaining them in the report."
824
+ },
825
+ "reason": {
826
+ "type": "string",
827
+ "minLength": 1,
828
+ "description": "Why this limitation is consciously accepted."
829
+ }
830
+ }
831
+ },
798
832
  "catalogProfile": {
799
833
  "type": "object",
800
834
  "additionalProperties": false,
package/scripts/README.md CHANGED
@@ -1,8 +1,8 @@
1
1
  # Host
2
2
 
3
3
  The side of portolan that runs. `npm run gen` reads the manifest, runs every
4
- plugin it names, stamps and writes what comes back, merges the sources into
5
- one estate and validates the union. Nothing here draws anything.
4
+ plugin it names, writes what comes back, merges the sources into one estate
5
+ and validates the union. Nothing here draws anything.
6
6
 
7
7
  ## What it does
8
8
 
@@ -10,21 +10,25 @@ one estate and validates the union. Nothing here draws anything.
10
10
  the schema `npm run schema` composes out of what every declared plugin says
11
11
  it can be told.
12
12
  - Runs three phases in order, extract, verify, generate, and hands each
13
- plugin one JSON message: the input root, output directory and stamp for an
13
+ plugin one JSON message: the input root and output directory for an
14
14
  extractor, the merged catalog for a verifier or a generator, the step's
15
15
  options unread. The output path lets a fragment point at a companion file
16
16
  returned in the same response without letting the plugin write it itself.
17
- - Stamps a fragment from the last commit that touched its input, never from a
18
- clock, so a committed fragment changes exactly when its subject does
19
- (portolan.0002).
17
+ - Writes no provenance into a fragment. When a source last changed, and in
18
+ which commit, is read from the history of the checkout wherever the catalog
19
+ is read - here, by the LikeC4 generator, by the architecture diff and by
20
+ the site as Vite builds it - so a fragment is content and nothing else, and
21
+ regenerating from the same sources changes no byte (portolan.0010).
20
22
  - Writes the files a plugin named, refuses a name that climbs out of the
21
23
  step's output directory, keeps a listing per step of what it wrote, and
22
24
  deletes what stopped being generated (portolan.0001).
23
25
  - Merges every source the manifest's globs find and validates the union with
24
26
  the code the site runs: `src/merge.ts`, `src/catalog.ts`, `src/enrich.ts`.
25
27
  - In `--check` mode writes nothing and fails on the first file that differs
26
- from disk. Either way it leaves `.portolan/build-report.json` for the
27
- Settings page.
28
+ from disk, saying where the file first differs and what changed among the
29
+ step's inputs since its output was last committed - the history is the
30
+ record of the last generation, so nothing else has to be. Either way it
31
+ leaves `.portolan/build-report.json` for the Settings page.
28
32
 
29
33
  ## What it does not do
30
34
 
@@ -47,7 +51,9 @@ one estate and validates the union. Nothing here draws anything.
47
51
  that report put on the pull request as one comment kept current;
48
52
  `forge-release.mjs`, the same report as one section of a release's notes;
49
53
  `forge.mjs`, what those two share - which forge the CI is, and how to talk
50
- to it; `vendor-lock.mjs`,
51
- the commit a fetched copy is of; `site-docs.mjs`, generated documentation
52
- put into the built site; `local-api.mjs`, what the dev server answers the
53
- site with.
54
+ to it; `history.mjs`, what the checkout's history says about every file,
55
+ for a plugin that asks and for the provenance of every source;
56
+ `provenance.mjs`, that provenance handed to the site as one virtual module;
57
+ `output-diff.mjs`, where a generated file first differs from what the
58
+ generator produces; `site-docs.mjs`, generated documentation put into the
59
+ built site; `local-api.mjs`, what the dev server answers the site with.
@@ -14,6 +14,7 @@ import { validateCatalog } from "../src/catalog.ts";
14
14
  import { filterCatalogForProfile } from "../src/catalog-profile.ts";
15
15
  import { enrichCatalog } from "../src/enrich.ts";
16
16
  import { mergeCatalogs } from "../src/merge.ts";
17
+ import { stampsFor } from "./history.mjs";
17
18
  import { readManifest } from "./manifest.mjs";
18
19
 
19
20
  /**
@@ -48,10 +49,15 @@ export async function loadCatalog(manifestPath = "portolan.json", { exclude = []
48
49
  );
49
50
  }
50
51
 
52
+ // A source is dated by the history, not by itself (portolan.0010): the
53
+ // commit that last changed the file, and its date, read here and never
54
+ // written into the file.
55
+ const stamps = stampsFor(process.cwd(), paths);
51
56
  const merged = mergeCatalogs(
52
57
  paths.map((path) => ({
53
58
  path,
54
59
  catalog: JSON.parse(readFileSync(path, "utf8")),
60
+ stamp: stamps.get(path),
55
61
  })),
56
62
  );
57
63
  if (paths.length === 0) {
@@ -23,7 +23,10 @@ const GITLAB_START = "# >>> Portolan delivery preset >>>";
23
23
  const GITLAB_END = "# <<< Portolan delivery preset <<<";
24
24
  const PROVIDERS = new Set(["github", "gitlab"]);
25
25
  const FEATURE_IDS = ["check", "diff", "sarif", "pages"];
26
- const DEFAULT_FEATURES = ["check", "pages"];
26
+ const DEFAULT_FEATURES = {
27
+ github: ["check", "pages"],
28
+ gitlab: ["pages"],
29
+ };
27
30
  const FEATURE_DETAILS = {
28
31
  check: {
29
32
  label: "Architecture check",
@@ -225,7 +228,9 @@ ${upload}`;
225
228
 
226
229
  function gitlabCheck() {
227
230
  return `"portolan:check":
228
- stage: .pre
231
+ stage: test
232
+ tags:
233
+ - runner-type:docker
229
234
  image:
230
235
  name: ghcr.io/shortlink-org/portolan:${VERSION}
231
236
  entrypoint: [""]
@@ -242,7 +247,9 @@ function gitlabCheck() {
242
247
  function gitlabDiff({ pages }) {
243
248
  const site = pages ? ' --site "$CI_PAGES_URL"' : "";
244
249
  return `"portolan:review":
245
- stage: .pre
250
+ stage: test
251
+ tags:
252
+ - runner-type:docker
246
253
  image:
247
254
  name: ghcr.io/shortlink-org/portolan:${VERSION}
248
255
  entrypoint: [""]
@@ -259,8 +266,10 @@ function gitlabDiff({ pages }) {
259
266
  }
260
267
 
261
268
  function gitlabPages() {
262
- return `"portolan:pages":
263
- stage: .post
269
+ return `pages:
270
+ stage: deploy
271
+ tags:
272
+ - runner-type:docker
264
273
  image:
265
274
  name: ghcr.io/shortlink-org/portolan:${VERSION}
266
275
  entrypoint: [""]
@@ -268,9 +277,10 @@ function gitlabPages() {
268
277
  GIT_DEPTH: "0"
269
278
  script:
270
279
  - BASE_PATH="$(node -p 'new URL(process.env.CI_PAGES_URL).pathname.replace(/\\/?$/, "/") || "/"')"
271
- - portolan build --output dist --base "$BASE_PATH"
272
- pages:
273
- publish: dist
280
+ - portolan build --output public --base "$BASE_PATH"
281
+ artifacts:
282
+ paths:
283
+ - public
274
284
  rules:
275
285
  - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
276
286
  `;
@@ -299,7 +309,7 @@ function mergeGitlab(existing, features) {
299
309
  .replace(/^\n+|\n+$/g, "");
300
310
  return { content: content ? `${content}\n` : "" };
301
311
  }
302
- if (/^[ \t]*["']?portolan:(?:check|review|pages)["']?\s*:/m.test(existing)) {
312
+ if (/^(?:["']?portolan:(?:check|review|pages)["']?|["']?pages["']?)\s*:/m.test(existing)) {
303
313
  return { conflict: "This pipeline already declares a Portolan job outside the managed region." };
304
314
  }
305
315
  const block = gitlabBlock(features);
@@ -421,14 +431,14 @@ function installedFeatures(workspace, provider) {
421
431
  );
422
432
  if (/^[ \t]*["']?portolan:check["']?\s*:/m.test(managed)) selected.add("check");
423
433
  if (/^[ \t]*["']?portolan:review["']?\s*:/m.test(managed)) selected.add("diff");
424
- if (/^[ \t]*["']?portolan:pages["']?\s*:/m.test(managed)) selected.add("pages");
434
+ if (/^(?:["']?portolan:pages["']?|["']?pages["']?)\s*:/m.test(managed)) selected.add("pages");
425
435
  return selected;
426
436
  }
427
437
 
428
438
  function selectedFeatures(workspace, request, provider) {
429
439
  const installed = request.features == null ? installedFeatures(workspace, provider) : null;
430
440
  const raw = request.features == null
431
- ? installed.size > 0 ? [...installed] : DEFAULT_FEATURES
441
+ ? installed.size > 0 ? [...installed] : DEFAULT_FEATURES[provider]
432
442
  : Array.isArray(request.features)
433
443
  ? request.features
434
444
  : String(request.features).split(",").filter(Boolean);
package/scripts/diff.mjs CHANGED
@@ -157,8 +157,12 @@ function catalogAt(ref) {
157
157
  throw new Error(ref + " holds no catalog sources matching " + JSON.stringify(manifest.sources));
158
158
  }
159
159
 
160
+ // The base is one commit, and a commit is its own provenance: every source
161
+ // there is dated by it (portolan.0010), whatever an older fragment says.
162
+ const [commit = "", generatedAt = ""] = git(["log", "-1", "--format=%h %cI", ref]).split(" ");
163
+ const stamp = { commit, generatedAt };
160
164
  const merged = mergeCatalogs(
161
- paths.map((path) => ({ path, catalog: JSON.parse(git(["show", ref + ":" + path])) })),
165
+ paths.map((path) => ({ path, catalog: JSON.parse(git(["show", ref + ":" + path])), stamp })),
162
166
  );
163
167
 
164
168
  // Enriched before it is compared, exactly as the app and the generators see
@@ -0,0 +1,58 @@
1
+ import { createHash } from "node:crypto";
2
+ import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+ import { afterEach, describe, expect, it } from "vitest";
6
+ import { djangoAggregateProposals, saveDjangoAggregates } from "./local-api.mjs";
7
+
8
+ const roots = [];
9
+ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
10
+ function fixture() {
11
+ const root = mkdtempSync(join(tmpdir(), "portolan-aggregates-")); roots.push(root);
12
+ const manifest = { sources: ["data/*.json"], extract: ["one", "two"].map((name) => ({ plugin: "django-domain", in: name, out: `data/${name}`, options: { context: name, service: name, aggregates: { existing: "Existing" }, apps: ["billing.records"] } })) };
13
+ const text = JSON.stringify(manifest);
14
+ writeFileSync(join(root, "portolan.json"), text);
15
+ const message = `billing/records: no model called Records, and 2 models to choose from: name the root in the aggregates option; aggregate candidates: ${JSON.stringify({ app: "billing.records", models: [{ name: "Entry", path: "billing/records/models.py", line: 1 }, { name: "Audit", path: "billing/records/models.py", line: 10 }] })}`;
16
+ const report = { status: "ok", manifestSha256: createHash("sha256").update(text).digest("hex"), steps: manifest.extract.map((step) => ({ phase: "extract", plugin: step.plugin, input: step.in, output: step.out, warnings: [message] })) };
17
+ mkdirSync(join(root, ".portolan")); writeFileSync(join(root, ".portolan/build-report.json"), JSON.stringify(report));
18
+ return { root, manifest, report };
19
+ }
20
+
21
+ describe("saving Django aggregate roots", () => {
22
+ it("targets the precise step and dotted app, preserving unrelated options", () => {
23
+ const { root, manifest } = fixture();
24
+ const proposals = djangoAggregateProposals(root);
25
+ expect(proposals.stale).toBe(false);
26
+ expect(proposals.proposals).toHaveLength(2);
27
+ saveDjangoAggregates(root, { revision: proposals.revision, selections: [{ id: "1:billing.records", model: "Entry" }] });
28
+ const saved = JSON.parse(readFileSync(join(root, "portolan.json"), "utf8"));
29
+ expect(saved.extract[0]).toEqual(manifest.extract[0]);
30
+ expect(saved.extract[1].options).toEqual({ ...manifest.extract[1].options, aggregates: { existing: "Existing", "billing.records": "Entry" } });
31
+ expect(djangoAggregateProposals(root).stale).toBe(true);
32
+ });
33
+
34
+ it("saves a batch atomically and rejects forged, duplicate, or stale choices", () => {
35
+ const { root } = fixture();
36
+ const { revision } = djangoAggregateProposals(root);
37
+ const original = readFileSync(join(root, "portolan.json"), "utf8");
38
+ for (const selections of [[], [{ id: "0:billing.records", model: "Proxy" }], [{ id: "no-such-step", model: "Entry" }], [{ id: "0:billing.records", model: "Entry" }, { id: "1:billing.records", model: "Invalid" }], [{ id: "0:billing.records", model: "Entry" }, { id: "0:billing.records", model: "Audit" }]]) {
39
+ expect(() => saveDjangoAggregates(root, { revision, selections })).toThrow();
40
+ expect(readFileSync(join(root, "portolan.json"), "utf8")).toBe(original);
41
+ }
42
+ const selections = [{ id: "0:billing.records", model: "Entry" }, { id: "1:billing.records", model: "Audit" }];
43
+ expect(() => saveDjangoAggregates(root, { revision: "old", selections })).toThrow(/changed/);
44
+ expect(saveDjangoAggregates(root, { revision, selections })).toEqual({ saved: 2 });
45
+ expect(() => saveDjangoAggregates(root, { revision, selections })).toThrow(/changed/);
46
+ });
47
+
48
+ it("requires fresh report evidence and refuses ambiguous step bindings", () => {
49
+ const { root, manifest, report } = fixture();
50
+ manifest.extract.push(manifest.extract[0]);
51
+ writeFileSync(join(root, "portolan.json"), JSON.stringify(manifest));
52
+ expect(djangoAggregateProposals(root).proposals.map((p) => p.id)).toEqual(["1:billing.records"]);
53
+ expect(djangoAggregateProposals(root).stale).toBe(true);
54
+ report.steps = [];
55
+ writeFileSync(join(root, ".portolan/build-report.json"), JSON.stringify(report));
56
+ expect(djangoAggregateProposals(root).proposals).toEqual([]);
57
+ });
58
+ });
@@ -278,7 +278,7 @@ for (const context of catalog.contexts) {
278
278
  model.push(` style { color ${contextColorName(context.id)} }`);
279
279
  for (const aggregate of service.aggregates) {
280
280
  model.push(
281
- ` ${safeId(aggregate.slug)} = aggregate ${q(aggregate.name)} {`,
281
+ ` ${safeId(aggregate.slug)} = aggregate ${q(aggregate.kind === "model-group" ? `${aggregate.name} (model group)` : aggregate.name)} {`,
282
282
  );
283
283
  for (const event of aggregate.events) {
284
284
  const latest = event.versions[event.versions.length - 1];
@@ -590,10 +590,11 @@ function containerPredicates(pairs, carried, indent) {
590
590
  * in a `break` frame, which is what a sequence diagram calls a branch that
591
591
  * leaves the flow rather than rejoining it.
592
592
  */
593
- // --- what comes back from a call (mirrors src/flow/answers.ts) -------------
594
- // Standalone calls keep the contract answer on the request label. Composition
595
- // turns a proven synchronous return into an explicit response step, so the
596
- // request then keeps only its own label and the return gets a dashed arrow.
593
+ // --- request and response edges (mirrors src/flow/answers.ts) --------------
594
+ // A unary RPC is two messages, not one long edge label. When composition has
595
+ // already materialised a response step, that step is the return. Otherwise we
596
+ // draw the contract response: immediately for a nested call, and at the end of
597
+ // the flow for the actor request that opened it.
597
598
  const methodOf = new Map();
598
599
  const serviceById = new Map();
599
600
  for (const context of catalog.contexts) {
@@ -615,29 +616,43 @@ for (const external of catalog.externals ?? []) {
615
616
  }
616
617
  }
617
618
 
618
- function answerOf(step) {
619
- if (step.kind !== "rpc") return "";
620
- if (step.ref) return methodOf.get(step.ref)?.response ?? "";
619
+ function contractOf(step) {
620
+ if (step.kind !== "rpc") return null;
621
+ if (step.ref) return methodOf.get(step.ref) ?? null;
621
622
  const service = serviceById.get(step.to);
622
- if (!service || !step.label) return "";
623
+ if (!service || !step.label) return null;
623
624
  for (const provided of service.provides) {
624
625
  const found = provided.methods.find((m) => m.name === step.label);
625
- if (found) return found.response ?? "";
626
+ if (found) return found;
626
627
  }
627
- return "";
628
+ return null;
629
+ }
630
+
631
+ function emitSyntheticResponse(out, indent, node, response) {
632
+ out.push(
633
+ `${indent}${participantRef(node.to)} -> ${participantRef(node.from)} ${q(response)} {`,
634
+ );
635
+ out.push(
636
+ `${indent} color ${node.status} line dashed head ${KIND_HEAD.response}`,
637
+ );
638
+ out.push(`${indent}}`);
628
639
  }
629
640
 
630
- function emitSteps(nodes, out, indent, replied) {
641
+ function emitSteps(nodes, out, indent, replied, deferredResponseId) {
631
642
  for (const node of nodes) {
632
643
  if (node.type === "step") {
633
- const answer = replied.has(node.id) ? "" : answerOf(node);
644
+ const contract = contractOf(node);
645
+ const request = contract?.request ?? "";
646
+ const response = replied.has(node.id) ? "" : (contract?.response ?? "");
634
647
  const storeLabel =
635
648
  node.storeAccess?.operation && node.storeAccess?.keyspace
636
649
  ? `${node.storeAccess.operation.toUpperCase()} ${node.storeAccess.keyspace}`
637
650
  : "";
638
651
  const label =
639
- (storeLabel || node.label || node.ref || node.kind) +
640
- (answer ? ` ${answer}` : "");
652
+ storeLabel ||
653
+ (node.kind === "rpc" && request
654
+ ? request
655
+ : node.label || node.ref || node.kind);
641
656
  const attrs = [
642
657
  `color ${node.http?.outcome === "error" ? "response_error" : node.status}`,
643
658
  `line ${node.kind === "response" ? "dashed" : "solid"}`,
@@ -656,18 +671,21 @@ function emitSteps(nodes, out, indent, replied) {
656
671
  if (notes.length > 0)
657
672
  out.push(`${indent} notes ${q(notes.join(" — "))}`);
658
673
  out.push(`${indent}}`);
674
+ if (response && node.id !== deferredResponseId) {
675
+ emitSyntheticResponse(out, indent, node, response);
676
+ }
659
677
  continue;
660
678
  }
661
679
  if (node.type === "parallel") {
662
680
  out.push(`${indent}par ${node.title ? `${q(node.title)} ` : ""}{`);
663
681
  for (const branch of node.branches)
664
- emitSteps(branch, out, `${indent} `, replied);
682
+ emitSteps(branch, out, `${indent} `, replied, deferredResponseId);
665
683
  out.push(`${indent}}`);
666
684
  continue;
667
685
  }
668
686
  if (node.type === "loop") {
669
687
  out.push(`${indent}loop ${q(node.title)} {`);
670
- emitSteps(node.steps, out, `${indent} `, replied);
688
+ emitSteps(node.steps, out, `${indent} `, replied, deferredResponseId);
671
689
  out.push(`${indent}}`);
672
690
  continue;
673
691
  }
@@ -678,10 +696,22 @@ function emitSteps(nodes, out, indent, replied) {
678
696
  out.push(`${indent} ${keyword} ${q(branch.title)} {`);
679
697
  if (branch.terminal) {
680
698
  out.push(`${indent} break 'ends the flow' {`);
681
- emitSteps(branch.steps, out, `${indent} `, replied);
699
+ emitSteps(
700
+ branch.steps,
701
+ out,
702
+ `${indent} `,
703
+ replied,
704
+ deferredResponseId,
705
+ );
682
706
  out.push(`${indent} }`);
683
707
  } else {
684
- emitSteps(branch.steps, out, `${indent} `, replied);
708
+ emitSteps(
709
+ branch.steps,
710
+ out,
711
+ `${indent} `,
712
+ replied,
713
+ deferredResponseId,
714
+ );
685
715
  }
686
716
  out.push(`${indent} }`);
687
717
  });
@@ -961,16 +991,34 @@ views.push("");
961
991
  for (const flow of catalog.flows) {
962
992
  const contexts = new Map(flow.participants.map((p) => [p.id, p.context]));
963
993
  const contextOf = (id) => contexts.get(id) ?? null;
994
+ const actorIds = new Set(
995
+ flow.participants.filter((p) => p.kind === "actor").map((p) => p.id),
996
+ );
964
997
  const replied = new Set();
965
998
  walkFlowSteps(flow.steps, (step) => {
966
999
  if (step.kind === "response" && step.replyTo) replied.add(step.replyTo);
967
1000
  });
1001
+ let deferredResponse = null;
1002
+ walkFlowSteps(flow.steps, (step) => {
1003
+ if (deferredResponse || step.kind !== "rpc" || !actorIds.has(step.from))
1004
+ return;
1005
+ const response = replied.has(step.id) ? "" : (contractOf(step)?.response ?? "");
1006
+ if (response) deferredResponse = { step, response };
1007
+ });
968
1008
 
969
1009
  views.push(` dynamic view ${flowViewId(flow)} {`);
970
1010
  views.push(` title ${q(flow.name)}`);
971
1011
  views.push(` description ${q(flow.summary)}`);
972
1012
  const body = [];
973
- emitSteps(flow.steps, body, " ", replied);
1013
+ emitSteps(flow.steps, body, " ", replied, deferredResponse?.step.id);
1014
+ if (deferredResponse) {
1015
+ emitSyntheticResponse(
1016
+ body,
1017
+ " ",
1018
+ deferredResponse.step,
1019
+ deferredResponse.response,
1020
+ );
1021
+ }
974
1022
  views.push(...body);
975
1023
  views.push(" }");
976
1024
  views.push("");
@@ -979,7 +1027,17 @@ for (const flow of catalog.flows) {
979
1027
  views.push(` dynamic view ${flowCrossViewId(flow)} {`);
980
1028
  views.push(` title ${q(`${flow.name} — crossings only`)}`);
981
1029
  const crossBody = [];
982
- emitSteps(cross, crossBody, " ", replied);
1030
+ const crossStepIds = new Set();
1031
+ walkFlowSteps(cross, (step) => crossStepIds.add(step.id));
1032
+ emitSteps(cross, crossBody, " ", replied, deferredResponse?.step.id);
1033
+ if (deferredResponse && crossStepIds.has(deferredResponse.step.id)) {
1034
+ emitSyntheticResponse(
1035
+ crossBody,
1036
+ " ",
1037
+ deferredResponse.step,
1038
+ deferredResponse.response,
1039
+ );
1040
+ }
983
1041
  if (crossBody.length === 0) {
984
1042
  // A flow with no crossing at all still needs a renderable view.
985
1043
  const first = flow.participants[0];