@shortlink-org/portolan 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +23 -0
  2. package/catalog/enum_test.go +46 -0
  3. package/catalog/evidence_test.go +35 -0
  4. package/catalog/model.go +1066 -0
  5. package/catalog/roundtrip_test.go +203 -0
  6. package/catalog/via_test.go +38 -0
  7. package/cli/init.test.mjs +6 -1
  8. package/cli/portolan.mjs +8 -0
  9. package/cli/portolan.test.mjs +49 -0
  10. package/go.mod +14 -0
  11. package/go.sum +20 -0
  12. package/internal/gocall/README.md +19 -0
  13. package/internal/gocall/analyze.go +189 -0
  14. package/internal/gocall/analyze_test.go +107 -0
  15. package/internal/gohttp/analyze.go +2562 -0
  16. package/internal/gohttp/destination.go +373 -0
  17. package/internal/gohttp/endpoints.go +1067 -0
  18. package/internal/gohttp/roots.go +320 -0
  19. package/internal/gohttp/typed.go +96 -0
  20. package/internal/goscan/constants.go +85 -0
  21. package/internal/goscan/goscan_test.go +227 -0
  22. package/internal/goscan/index.go +629 -0
  23. package/internal/goscan/index_test.go +66 -0
  24. package/internal/goscan/names.go +52 -0
  25. package/internal/goscan/parse_test.go +11 -0
  26. package/internal/goscan/source.go +37 -0
  27. package/internal/goscan/tree.go +284 -0
  28. package/internal/goscan/types.go +99 -0
  29. package/internal/wsdl/ids.go +127 -0
  30. package/internal/wsdl/ids_test.go +21 -0
  31. package/internal/wsdl/model.go +70 -0
  32. package/internal/wsdl/parse.go +949 -0
  33. package/internal/wsdl/parse_test.go +170 -0
  34. package/package.json +22 -10
  35. package/plugin/describe.go +118 -0
  36. package/plugin/describe_test.go +114 -0
  37. package/plugin/protocol.go +141 -0
  38. package/plugin/schematest/schematest.go +126 -0
  39. package/plugins/README.md +114 -46
  40. package/plugins/cmd/portolan-http-clients/main.go +19 -0
  41. package/plugins/extract-celery/extract.py +0 -2
  42. package/plugins/extract-celery/extract_test.py +1 -1
  43. package/plugins/extract-django/README.md +39 -17
  44. package/plugins/extract-django/domain.py +28 -17
  45. package/plugins/extract-django/extract.py +21 -7
  46. package/plugins/extract-django/extract_test.py +55 -2
  47. package/plugins/extract-django/lifecycle.py +2 -0
  48. package/plugins/extract-django/operations.py +1 -1
  49. package/plugins/extract-django/routing_test.py +109 -1
  50. package/plugins/extract-django/store.py +1 -1
  51. package/plugins/extract-django/transport.py +101 -55
  52. package/plugins/extract-django/verbs.py +241 -0
  53. package/plugins/extract-go/README.md +47 -0
  54. package/plugins/extract-http-clients/describe.go +19 -0
  55. package/plugins/extract-http-clients/describe_test.go +11 -0
  56. package/plugins/extract-http-clients/extract.go +740 -0
  57. package/plugins/extract-http-clients/extract_test.go +1561 -0
  58. package/plugins/extract-http-clients/main.go +41 -0
  59. package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
  60. package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
  61. package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
  62. package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
  63. package/plugins/extract-python-kafka/README.md +6 -0
  64. package/plugins/extract-python-kafka/extract.py +2 -4
  65. package/plugins/extract-python-kafka/extract_test.py +18 -2
  66. package/plugins/extract-ts/extract.test.ts +2 -2
  67. package/plugins/extract-ts/extract.ts +4 -5
  68. package/plugins/extract-ts/graphql.test.ts +1 -1
  69. package/plugins/openapi/ids.go +261 -0
  70. package/plugins/openapi/ids_test.go +98 -0
  71. package/plugins/portolan-go.wasm +0 -0
  72. package/plugins/pyplugin/catalog.py +12 -1
  73. package/plugins/pyplugin/kafka.py +74 -3
  74. package/plugins/pyplugin/protocol.py +1 -5
  75. package/portolan.json +3 -2
  76. package/schema/portolan.schema.json +34 -0
  77. package/scripts/README.md +18 -12
  78. package/scripts/catalog-sources.mjs +6 -0
  79. package/scripts/delivery-presets.mjs +21 -11
  80. package/scripts/diff.mjs +5 -1
  81. package/scripts/django-aggregates.test.mjs +58 -0
  82. package/scripts/gen-likec4.mjs +79 -21
  83. package/scripts/gen-likec4.test.mjs +25 -2
  84. package/scripts/gen.mjs +118 -115
  85. package/scripts/go-discovery.test.mjs +30 -0
  86. package/scripts/history.mjs +186 -3
  87. package/scripts/history.test.mjs +1 -1
  88. package/scripts/host-plugins/fetch-git.mjs +77 -21
  89. package/scripts/host-plugins/fetch-git.test.mjs +62 -8
  90. package/scripts/local-api.mjs +71 -4
  91. package/scripts/local-api.test.mjs +63 -4
  92. package/scripts/local-discovery.mjs +82 -9
  93. package/scripts/manifest.mjs +5 -3
  94. package/scripts/manifest.test.mjs +24 -0
  95. package/scripts/output-diff.mjs +94 -0
  96. package/scripts/output-diff.test.mjs +36 -0
  97. package/scripts/package-smoke.mjs +62 -4
  98. package/scripts/plugin-host.mjs +22 -2
  99. package/scripts/plugin-host.test.mjs +9 -0
  100. package/scripts/plugin-wasm-worker.mjs +4 -1
  101. package/scripts/provenance.mjs +72 -0
  102. package/scripts/provenance.test.mjs +149 -0
  103. package/scripts/run-builtin.mjs +39 -5
  104. package/scripts/schema.mjs +29 -0
  105. package/scripts/warning-policy.mjs +161 -0
  106. package/scripts/warning-policy.test.mjs +56 -0
  107. package/src/app/Breadcrumbs.test.ts +4 -0
  108. package/src/app/Breadcrumbs.tsx +1 -0
  109. package/src/app/Sidebar.tsx +3 -3
  110. package/src/catalog-docs.test.ts +64 -0
  111. package/src/catalog-docs.ts +35 -0
  112. package/src/catalog-error.test.ts +15 -0
  113. package/src/catalog-model.ts +70 -6
  114. package/src/catalog-stores.test.ts +17 -0
  115. package/src/catalog-validation.ts +52 -2
  116. package/src/catalog.test.ts +13 -2
  117. package/src/chat/Starter.tsx +5 -11
  118. package/src/chat/tools.test.ts +27 -0
  119. package/src/chat/tools.ts +5 -9
  120. package/src/components/CatalogStamp.tsx +10 -8
  121. package/src/components/ChannelRows.messagepack.test.tsx +28 -0
  122. package/src/components/ChannelRows.test.tsx +54 -0
  123. package/src/components/ChannelRows.tsx +57 -10
  124. package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
  125. package/src/components/HTTPDestinationEvidence.tsx +31 -0
  126. package/src/components/Integrations.tsx +1 -1
  127. package/src/components/LifecycleDiagram.tsx +28 -12
  128. package/src/components/MachineDocs.tsx +6 -5
  129. package/src/components/MethodRows.tsx +9 -2
  130. package/src/components/ProblemRow.tsx +4 -0
  131. package/src/components/RelationEvidence.test.tsx +14 -0
  132. package/src/components/RelationEvidence.tsx +53 -0
  133. package/src/components/WhatLinksHere.tsx +6 -4
  134. package/src/data.ts +25 -7
  135. package/src/enrich.test.ts +336 -6
  136. package/src/enrich.ts +206 -3
  137. package/src/flow/StepDetail.tsx +104 -54
  138. package/src/flow/answers.test.ts +18 -1
  139. package/src/flow/answers.ts +37 -8
  140. package/src/flow/evidence.test.ts +16 -0
  141. package/src/flow/evidence.ts +34 -0
  142. package/src/index.css +44 -0
  143. package/src/landing/DraggableReveal.tsx +3 -2
  144. package/src/landing/EvidencePipeline.tsx +105 -0
  145. package/src/landing/LandingPage.tsx +2 -59
  146. package/src/lib/backlinks.test.ts +16 -1
  147. package/src/lib/backlinks.ts +20 -0
  148. package/src/lib/catalog-diff.test.ts +18 -0
  149. package/src/lib/catalog-diff.ts +20 -2
  150. package/src/lib/derive.ts +1 -0
  151. package/src/lib/django-aggregates.d.mts +9 -0
  152. package/src/lib/django-aggregates.mjs +36 -0
  153. package/src/lib/django-aggregates.test.ts +29 -0
  154. package/src/lib/django-aggregates.ts +5 -0
  155. package/src/lib/kafka-ui.test.ts +87 -0
  156. package/src/lib/kafka-ui.ts +105 -0
  157. package/src/lib/local-api.ts +20 -2
  158. package/src/lib/setup-info.test.ts +17 -0
  159. package/src/lib/setup-info.ts +58 -0
  160. package/src/lib/warnings.test.ts +54 -0
  161. package/src/lib/warnings.ts +260 -0
  162. package/src/lib/wire-problems.test.ts +21 -0
  163. package/src/lib/wire-problems.ts +62 -1
  164. package/src/likec4/FlowView.tsx +2 -6
  165. package/src/likec4/flow-edges.test.ts +64 -1
  166. package/src/likec4/flow-edges.ts +43 -7
  167. package/src/likec4/view-index.ts +8 -2
  168. package/src/map/ContextMapGraph.tsx +76 -32
  169. package/src/merge.test.ts +23 -0
  170. package/src/merge.ts +33 -10
  171. package/src/pages/AggregatePage.tsx +8 -7
  172. package/src/pages/CatalogFailure.tsx +2 -2
  173. package/src/pages/ContextPage.tsx +6 -5
  174. package/src/pages/ServicePage.tsx +4 -3
  175. package/src/pages/Settings.tsx +200 -44
  176. package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
  177. package/src/pages/settings/IntegrationsSettings.tsx +117 -0
  178. package/src/routes.test.ts +2 -0
  179. package/src/routes.ts +2 -1
  180. package/src/selection/DetailPanel.tsx +61 -1
  181. package/src/virtual-provenance.d.ts +11 -0
  182. package/vite.config.ts +5 -0
  183. package/scripts/vendor-lock.mjs +0 -58
  184. package/scripts/vendor-lock.test.mjs +0 -69
@@ -0,0 +1,126 @@
1
+ // Package schematest checks that a plugin's options schema still describes its
2
+ // options struct.
3
+ //
4
+ // The schema is written by hand rather than reflected out of the struct,
5
+ // because what makes it worth having is the prose: which slug a store expects,
6
+ // whose proto package a peer answers for. None of that survives reflection.
7
+ // What reflection is good for is the half nobody would notice going stale - a
8
+ // field renamed, a field added, a schema left behind - so that is all this
9
+ // does.
10
+ package schematest
11
+
12
+ import (
13
+ "encoding/json"
14
+ "reflect"
15
+ "sort"
16
+ "strings"
17
+ "testing"
18
+ )
19
+
20
+ type document struct {
21
+ Type string `json:"type"`
22
+ AdditionalProperties *bool `json:"additionalProperties"`
23
+ Properties map[string]json.RawMessage `json:"properties"`
24
+ Required []string `json:"required"`
25
+ }
26
+
27
+ // Check reports every way the schema and the struct have come apart.
28
+ //
29
+ // options is a zero value of the plugin's options type; schema is the embedded
30
+ // document it hands back when asked to describe itself.
31
+ func Check(t *testing.T, schema []byte, options any) {
32
+ t.Helper()
33
+
34
+ var doc document
35
+ if err := json.Unmarshal(schema, &doc); err != nil {
36
+ t.Fatalf("options schema is not JSON: %v", err)
37
+ }
38
+
39
+ if doc.Type != "object" {
40
+ t.Errorf("options schema has type %q, want object", doc.Type)
41
+ }
42
+
43
+ // Without this an unknown key validates, and the misspelled option - the
44
+ // one error this whole mechanism exists to catch - goes through silently.
45
+ if doc.AdditionalProperties == nil || *doc.AdditionalProperties {
46
+ t.Error("options schema must set additionalProperties: false")
47
+ }
48
+
49
+ described := keys(doc.Properties)
50
+ declared := fields(reflect.TypeOf(options))
51
+
52
+ for _, name := range missing(declared, described) {
53
+ t.Errorf("option %q is in the struct but not in the schema", name)
54
+ }
55
+ for _, name := range missing(described, declared) {
56
+ t.Errorf("option %q is in the schema but not in the struct", name)
57
+ }
58
+
59
+ for _, name := range doc.Required {
60
+ if _, ok := doc.Properties[name]; !ok {
61
+ t.Errorf("option %q is required but not described", name)
62
+ }
63
+ }
64
+
65
+ // A property with nothing to say is a property the manifest schema will
66
+ // offer with no hint of what it means, which is where this started.
67
+ for name, raw := range doc.Properties {
68
+ var property struct {
69
+ Description string `json:"description"`
70
+ }
71
+ if err := json.Unmarshal(raw, &property); err != nil {
72
+ t.Errorf("option %q: %v", name, err)
73
+
74
+ continue
75
+ }
76
+ if strings.TrimSpace(property.Description) == "" {
77
+ t.Errorf("option %q has no description", name)
78
+ }
79
+ }
80
+ }
81
+
82
+ // fields are the json names of an options struct, which is the only shape this
83
+ // takes: a plugin's options are a flat object read out of the manifest.
84
+ func fields(t reflect.Type) []string {
85
+ var names []string
86
+
87
+ for i := 0; i < t.NumField(); i++ {
88
+ tag := t.Field(i).Tag.Get("json")
89
+ name, _, _ := strings.Cut(tag, ",")
90
+ if name == "" || name == "-" {
91
+ continue
92
+ }
93
+ names = append(names, name)
94
+ }
95
+
96
+ sort.Strings(names)
97
+
98
+ return names
99
+ }
100
+
101
+ func keys(m map[string]json.RawMessage) []string {
102
+ names := make([]string, 0, len(m))
103
+ for name := range m {
104
+ names = append(names, name)
105
+ }
106
+ sort.Strings(names)
107
+
108
+ return names
109
+ }
110
+
111
+ // missing is everything in a that is not in b.
112
+ func missing(a, b []string) []string {
113
+ have := make(map[string]bool, len(b))
114
+ for _, name := range b {
115
+ have[name] = true
116
+ }
117
+
118
+ var gone []string
119
+ for _, name := range a {
120
+ if !have[name] {
121
+ gone = append(gone, name)
122
+ }
123
+ }
124
+
125
+ return gone
126
+ }
package/plugins/README.md CHANGED
@@ -54,7 +54,10 @@ Three obligations, and they are the whole of it:
54
54
  unsafe filename or invalid response fails the run. Non-fatal extraction
55
55
  notes go to stderr; there is no advisory response property a caller may
56
56
  accidentally ignore. A note that opens with `warning: ` is kept beside the
57
- step in `.portolan/build-report.json` and listed on the Settings page.
57
+ step in `.portolan/build-report.json`. The Settings page classifies and
58
+ groups those notes by plugin, stable rule and severity. Typed CEL policies
59
+ in `portolan.json` can suppress reviewed diagnostics only with an explicit
60
+ reason; the host evaluates them and records the decision in the build report.
58
61
 
59
62
  The repository enforces those obligations with schema/field coverage tests,
60
63
  byte-for-byte permutation tests, generated-link and anchor checks, Mermaid
@@ -78,12 +81,14 @@ either be rendered or be explicitly acknowledged by the relevant exporter.
78
81
  `go:embed` and returned in the descriptor. `schematest.Check` in a test keeps
79
82
  it from drifting from the options struct: a field renamed on one side and not
80
83
  the other fails, and so does an option with no description.
81
- 3. Build it. A built-in Go plugin is a library package with
84
+ 3. Build it. A sandboxed built-in Go plugin is a library package with
82
85
  `Serve(io.Reader, io.Writer) error`; add it to the map in
83
86
  `plugins/cmd/portolan-go/main.go`, and `plugins:build` in `package.json`
84
87
  puts it in `plugins/portolan-go.wasm` with the rest (a test keeps the map
85
- and `portolan.json` in step). A plugin of your own is its own module:
86
- `GOOS=wasip1 GOARCH=wasm go build`.
88
+ and `portolan.json` in step). A built-in that genuinely needs the Go
89
+ toolchain gets a small command under `plugins/cmd/` and is declared as a
90
+ `process`; `http-clients` is the example. A plugin of your own is its own
91
+ module: `GOOS=wasip1 GOARCH=wasm go build`.
87
92
  4. Declare it in `portolan.json`, under `plugins` (how to run it) and
88
93
  `generate` (what to run it on), then run `npm run schema` so the manifest
89
94
  schema learns its options.
@@ -179,10 +184,23 @@ the original collection layout for TypeScript, Rust, Java, or custom trees and
179
184
  may also point directly at one feature repository.
180
185
 
181
186
  `extract-river` is another independent enrichment for Go repositories. It
182
- joins a job argument's `Kind()` to `Client.Insert`/`InsertTx`, the selected
183
- queue, `Worker[Args].Work`, and `river.AddWorker`. The result is a work-queue
184
- channel plus a two-hop enqueue/dispatch flow, with payload fields and source
185
- lines. It does not need aggregates and does not treat a job as a domain event.
187
+ joins a job argument's `Kind()` to `Client.Insert`/`InsertTx` and the
188
+ `InsertMany*` batches, the selected queue, `Worker[Args].Work`, and
189
+ `river.AddWorker`. The result is a work-queue channel plus a two-hop
190
+ enqueue/dispatch flow, with payload fields and source lines. It does not need
191
+ aggregates and does not treat a job as a domain event. The job type is
192
+ followed through locals, constructors' results and parameters up to the
193
+ callers, so an `Enqueue(ctx, args river.JobArgs, opts)` wrapper is read once
194
+ per concrete job passed to it, and a worker registered as
195
+ `river.AddWorker(workers, NewSendWorker(deps))` is registered. The queue is
196
+ the `InsertOpts.Queue` at the call (a literal, a constant, a config `default:`
197
+ tag, or what the callers pass), else the args type's own `InsertOpts()`, else
198
+ `default`. What no Insert in the tree feeds is not dropped: a registered
199
+ worker's queue is the one its type chooses or the one queue `river.Config`
200
+ lists, and with several and no choice the worker is kept as a flow whose
201
+ receive is unresolved. The diagnostics keep the two cases apart - an Insert
202
+ whose job type or queue the reader could not follow is said at the call, a
203
+ worker with no Insert in the tree is said at the worker.
186
204
 
187
205
  `extract-redis` finds runtime construction of go-redis, rueidis and redigo
188
206
  clients in non-test Go source. That source evidence adds a service-owned Redis
@@ -193,16 +211,27 @@ key patterns with their read/write/delete use, TTL, value type and source. The
193
211
  patterns remain Redis keyspaces rather than being presented as SQL tables.
194
212
 
195
213
  `extract-watermill` reads Watermill `Router.AddHandler` and
196
- `AddNoPublisherHandler` registrations. It resolves literal and constant topics,
197
- plus defaults on env-config structs, follows direct `Publisher.Publish` calls
198
- and one-hop publishing helpers, and traces JSON marshal/unmarshal values back to
199
- their Go payload structs. It carries enclosing `if`/`else` conditions and early
200
- returns through the handler's control flow: proven alternatives become one
201
- catalog `alt`; publications whose relationship cannot be proven remain separate
202
- possible routes. The channels merge normally with AsyncAPI declarations by address. Generic
203
- `NewEventHandler[T]` and `NewCommandHandler[T]` registrations on Watermill CQRS
204
- processors are also extracted; fixed topic generators and the standard
205
- event/command-name generator form are resolved from source.
214
+ `AddNoPublisherHandler` registrations, and `Subscriber.Subscribe` calls made
215
+ without a router. It resolves literal and constant topics - typed constants
216
+ through `string(...)`, constants concatenated with `+`, single-return
217
+ helpers - plus defaults on env-config structs, nested (`cfg.Kafka.Orders`) or
218
+ not; a topic, a name or a handler that a registering function takes as a
219
+ parameter is filled in once per caller, so a `handle(r, name, topic, fn)`
220
+ wrapper yields one handler per call. The handler value may be a function
221
+ literal, a function or method of the tree (`h.OnOrder`, `handlers.OnShipped`)
222
+ or a constructor returning a literal. It follows direct `Publisher.Publish`
223
+ calls and one-hop publishing helpers, records what the service publishes
224
+ outside any handler as a `send` on the topic, and traces JSON marshal/unmarshal
225
+ values back to their Go payload structs. It carries enclosing `if`/`else`
226
+ conditions and early returns through the handler's control flow: proven
227
+ alternatives become one catalog `alt`; publications whose relationship cannot
228
+ be proven remain separate possible routes. A handler whose topic does not
229
+ resolve is kept, with its receive step unresolved and no channel claimed, and
230
+ named in a diagnostic. The channels merge normally with AsyncAPI declarations
231
+ by address. Generic `NewEventHandler[T]` and `NewCommandHandler[T]`
232
+ registrations on Watermill CQRS processors are also extracted; fixed topic
233
+ generators and the standard event/command-name generator form are resolved
234
+ from source.
206
235
 
207
236
  `extract-go-nats` reads nats.go and JetStream calls into the subjects a service
208
237
  listens on and publishes to. A call is known by the type it is made on -
@@ -260,16 +289,19 @@ literals, direct assignments, or setters. A standalone flow says whether no
260
289
  source caller exists or callers exist but no inbound/asynchronous root was
261
290
  proved, so the UI exposes the missing evidence instead of implying a complete
262
291
  business path.
263
- When a provider branch still ends before its transport, the extractor loads
264
- the module with `go/packages`, builds SSA, and uses `x/tools` VTA to resolve
292
+ When a provider branch still ends before its transport, the extractor runs as
293
+ a workspace-local native sidecar, loads the module with `go/packages`, builds
294
+ SSA, and uses `x/tools` VTA to resolve
265
295
  calls through interface parameters, function values, return values, and
266
296
  interface-typed struct fields. Typed edges are followed only after a concrete
267
297
  factory branch is selected: applying context-insensitive VTA to a shared
268
298
  dispatcher would otherwise attach every request implementation to every
269
299
  endpoint. Factory conditions and HTTP/SOAP meaning continue to come from the
270
- source extractor. Module loading is read-only and bounded; unavailable private
271
- dependencies, type errors, or a timeout produce a warning and retain the
272
- syntax-only result rather than failing generation.
300
+ source extractor. The sidecar is built from the Portolan package through the
301
+ project's Go toolchain and runs with the project as its working directory.
302
+ Module loading is read-only and bounded; unavailable private dependencies,
303
+ type errors, or a timeout produce a warning and retain the syntax-only result
304
+ rather than failing generation.
273
305
  Routes without a provider factory are also joined to their outbound calls,
274
306
  including handlers invoked from closures and methods on locally constructed
275
307
  values. A Swagger `@Router` annotation is medium-confidence root evidence for
@@ -454,9 +486,10 @@ far side.
454
486
  The demo estate's org-wide and context-wide records live in `data/adr`, and are
455
487
  read by a step that points at that directory with a glob of its own. Root
456
488
  `docs/` is where `gen-markdown` writes, so nothing hand-written can live there.
457
- `in` is the directory of records rather than `data` itself: a step's fragment is
458
- only left out of its own stamp when the output is *inside* the input root, and
459
- `in: data` with `out: data` would be stamped from the file it writes.
489
+ `in` is the directory of records rather than `data` itself: what `gen --check`
490
+ lists as changed inputs is the root less the output directories inside it, and
491
+ `in: data` with `out: data` would list the file the step writes among the
492
+ reasons it was rewritten.
460
493
 
461
494
  ```json
462
495
  {
@@ -775,9 +808,9 @@ enforces: a team that owns a directory is a team that gets the pull request.
775
808
  ```
776
809
 
777
810
  Point `in` at the directory the file is in, not at the repository root: the
778
- host dates a fragment from the last commit to touch the step's input, and the
779
- subject of this one is the `CODEOWNERS` file. Rooted at the repository, it
780
- would be restamped by every commit ever made. Left with no `file`, the three
811
+ subject of this step is the `CODEOWNERS` file, and `gen --check` explains a
812
+ changed fragment by what moved under the step's input. Rooted at the
813
+ repository, every commit ever made would be named. Left with no `file`, the three
781
814
  places a forge looks are tried in order - `CODEOWNERS`, `.github/CODEOWNERS`,
782
815
  `docs/CODEOWNERS` - and a `file` that names something absent fails the run,
783
816
  because answering "nobody owns anything" to a typo is only noticed a month
@@ -822,21 +855,23 @@ the forge looks.
822
855
  `wasm` is the default and should stay that way. The module gets no network, no
823
856
  environment and no way to start a process. A generator gets no filesystem
824
857
  either. An extract or verify step gets the workspace preopened as `/`
825
- (portolan.0006), which is how the built-in Go extractors read a tree without a
826
- Go toolchain on the machine: every one of them, and the three generators, is
827
- the single module `plugins/portolan-go.wasm`, which answers to the plugin name
828
- the host passes as `argv[0]`. WASI preopens read-write, so an extractor is
858
+ (portolan.0006), which is how syntax-only built-in Go extractors read a tree
859
+ without a Go toolchain on the machine: those extractors and the three generators
860
+ are the single module `plugins/portolan-go.wasm`, which answers to the plugin
861
+ name the host passes as `argv[0]`. WASI preopens read-write, so an extractor is
829
862
  trusted not to write the tree it reads, the same trust a process plugin has
830
863
  today; a `sha256` pins that trust to a build.
831
864
 
832
- `process` is the escape hatch for a plugin that needs a toolchain: the Rust,
833
- Java, Python and TypeScript extractors run in their own runtimes, and
834
- `fetch-bsr` still talks to its registry from Go. It gets the same protocol
865
+ `process` is the escape hatch for a plugin that needs a toolchain: the typed Go
866
+ HTTP client analyzer and the Rust, Java, Python and TypeScript extractors run in
867
+ their own runtimes. It gets the same protocol
835
868
  and none of the sandbox, which is the trade being made and the reason it is
836
869
  not the default. It declares `command` and an `args` array; the host never
837
- feeds a command string through a shell. A built-in Go plugin that still runs
838
- as a process is the same code reached as
839
- `go run ./plugins/cmd/portolan-go <name>`.
870
+ feeds a command string through a shell. For a built-in declared as `go run`,
871
+ the adapter compiles a workspace-local native sidecar from the package and
872
+ then runs it with the scanned workspace as its current directory. This keeps
873
+ module resolution in Portolan's shipped source and project loading in the
874
+ project being analyzed.
840
875
 
841
876
  `host` is for Portolan's own code that needs what only the host has - a git
842
877
  binary, a socket - and so runs inside the host process (portolan.0008):
@@ -859,12 +894,18 @@ written.
859
894
  `fetch-git` is `fetch-bsr` for a repository rather than a registry, and it
860
895
  lives by the same four rules. A pin is a repository, a commit and the paths
861
896
  actually read; the step fetches exactly those directories at exactly that
862
- commit and hands them back as files, so the host writes them into the tree
897
+ commit and hands their text files back, so the host writes them into the tree
863
898
  beside a `git.lock.json` naming the commit and the digest of every file. The
864
899
  paths inside the copy are the repository's own, which is the point: the
865
900
  extract step that follows points its `in` at the vendored service and reads
866
901
  it exactly as it would read that service's checkout.
867
902
 
903
+ Binary files are not part of the extractor input. Known binary extensions are
904
+ discarded before their blobs are read; files without a known extension are
905
+ sniffed for NUL bytes and invalid UTF-8 before the response is assembled. The
906
+ lock records only their path, byte size and skip reason, never Base64 content,
907
+ so online fetch and offline replay produce the same bounded text-only copy.
908
+
868
909
  It runs inside the host (`scripts/host-plugins/fetch-git.mjs`,
869
910
  portolan.0008) rather than as a module, because it needs a git binary and a
870
911
  socket, and a manifest names it with `host` rather than `wasm` or `process`.
@@ -919,11 +960,12 @@ Nothing else can say it. A service says which repository it lives in, and an
919
960
  extractor reads a directory as a pure function of what is on disk; neither has
920
961
  any idea which commit somebody fetched. Without that line, every source path
921
962
  of every vendored service is dead text on the page - the file and the line are
922
- known, and there is nowhere to send a reader - and every fragment read out of
923
- the copy is stamped with the commit that VENDORED it, so the service looks
924
- fresh whenever the fetch is re-run and unchanged when its own repository
925
- moves. With it, `sourceHref` links the line at the commit it was read at and
926
- `stampFor` dates the fragment from the code rather than the vendoring.
963
+ known, and there is nowhere to send a reader. With it, `sourceHref` links the
964
+ line at the commit it was read at. The fragments read out of the copy are
965
+ dated, like every other source, by the commit of THIS repository that last
966
+ changed them (portolan.0010) - which is the fetch that brought the change in,
967
+ and is what "how fresh is this" means to the estate that vendored it; the pin
968
+ is what says which upstream commit that was.
927
969
 
928
970
  One more line is needed for the app itself: `SOURCE_GLOBS` in `src/data.ts`,
929
971
  where the same patterns are written out a second time because
@@ -1020,7 +1062,8 @@ workflow already does.
1020
1062
 
1021
1063
  Four rules govern what happens when a fetch does not:
1022
1064
 
1023
- 1. Fetch succeeded → the fetched files and a regenerated lock.
1065
+ 1. Fetch succeeded → the fetched text files, binary-skip metadata and a
1066
+ regenerated lock.
1024
1067
  2. Skipped or failed, cache complete and matching its digests → the cached files
1025
1068
  byte-identically, plus a warning. Output unchanged, so `--check` stays clean.
1026
1069
  3. Failed **and** no usable cache → a non-zero exit, never a short file list. The
@@ -1142,6 +1185,11 @@ live in another estate — would be the wrong end of the trade.
1142
1185
  service declares — the address the broker knows, and each message on it with the
1143
1186
  direction it travels. What it does **not** answer with is events.
1144
1187
 
1188
+ A message's `contentType`, or the document's `defaultContentType` when the
1189
+ message leaves it out, is kept as the exact declared media type. MessagePack
1190
+ media types are additionally normalized to `encoding: msgpack`, so source and
1191
+ contract extractors can be compared without depending on one MIME spelling.
1192
+
1145
1193
  That looks like a gap and is a boundary. `Event.id` is
1146
1194
  `<service>.<aggregate>.<Name>`, and an AsyncAPI document knows the message on the
1147
1195
  wire, not the aggregate that raised it. An extractor that guessed would either
@@ -1197,3 +1245,23 @@ it, the call before the decorator before `task_routes` before the default,
1197
1245
  and `transaction.on_commit(...)` around an enqueue is a note on the step,
1198
1246
  which is the one fact about *when* a message leaves that the code states
1199
1247
  plainly.
1248
+
1249
+ ### HTTP destination provenance
1250
+
1251
+ `extract-http-clients` emits structured `destination` evidence on HTTP consumes
1252
+ and flow steps: the request call site and endpoint expression, local path,
1253
+ base URL expression, config field and environment variable, source locations,
1254
+ and a full path when source proves the join. Functional options that assign a
1255
+ URL field to a returned client are followed through adapter constructors.
1256
+ Bindings belong to the adapter's client field; conflicting construction sites
1257
+ remain unresolved instead of sharing another client's settings.
1258
+
1259
+ A Go `default` struct tag is recorded as `config-default`, not as the effective
1260
+ runtime configuration. For example, `WithBaseURL(cfg.SettingAddr)` with
1261
+ `default:"http://localhost:8000/settings"` and a concatenated
1262
+ `/get-admin-settings` yields `/settings/get-admin-settings`. The merge matches
1263
+ that full path exactly and never falls back to a suffix if it is absent.
1264
+ Legacy suffix matches remain explicitly labeled `unique-suffix` in the evidence
1265
+ shown in RPC step details. Later runtime URL modifiers are listed separately;
1266
+ the recovered full path describes the static join. Unreadable options may
1267
+ leave only the local route available.
@@ -0,0 +1,19 @@
1
+ // Command portolan-http-clients is the native sidecar for typed Go call-graph
2
+ // analysis. It intentionally does not compile for WASI: go/packages needs the
3
+ // project Go toolchain to load build tags and dependencies before SSA/VTA can
4
+ // resolve interface calls.
5
+ package main
6
+
7
+ import (
8
+ "fmt"
9
+ "os"
10
+
11
+ extracthttpclients "github.com/shortlink-org/portolan/plugins/extract-http-clients"
12
+ )
13
+
14
+ func main() {
15
+ if err := extracthttpclients.Serve(os.Stdin, os.Stdout); err != nil {
16
+ fmt.Fprintln(os.Stderr, "portolan-http-clients:", err)
17
+ os.Exit(1)
18
+ }
19
+ }
@@ -155,8 +155,6 @@ def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
155
155
  flows.sort(key=lambda f: f["slug"])
156
156
 
157
157
  fragment: Dict[str, Any] = {
158
- "generatedAt": input_.generated_at,
159
- "commit": input_.commit,
160
158
  "contexts": [
161
159
  {
162
160
  "id": context,
@@ -27,7 +27,7 @@ ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
27
27
  def run(fixture, options):
28
28
  b = Builder()
29
29
  extract(
30
- Input(root=os.path.relpath(os.path.join(HERE, "testdata", fixture), ROOT), commit="abc1234", generated_at="2026-09-05T00:00:00Z"),
30
+ Input(root=os.path.relpath(os.path.join(HERE, "testdata", fixture), ROOT)),
31
31
  Options.of(options),
32
32
  b,
33
33
  cwd=ROOT,
@@ -39,15 +39,24 @@ resolves them without a type checker.
39
39
 
40
40
  ## What becomes what
41
41
 
42
- **Aggregate.** One per application. The root is the model named after the
43
- application `invoices` holds `Invoice` or the only model there is, or the
44
- one `aggregates` names in the manifest; an application with several models and
45
- no such name is reported and skipped, because guessing which of them the others
46
- hang off is how an aggregate boundary gets drawn wrong and stays wrong. The
47
- aggregate is named and slugged after the root, so a Django `invoices` and a Go
48
- `invoice` package land on the same id. Every concrete model of the application
49
- is an entity, the root included and first; abstract models are not. The
50
- readme is `README.md` in the application, or the root's docstring.
42
+ **Models and aggregates.** Every concrete, non-proxy model is visible. When
43
+ an application has an explicitly configured root, a model named after the
44
+ application (`invoices` `Invoice`), or just one concrete model, it retains
45
+ its existing aggregate representation and stable root-based id.
46
+
47
+ Otherwise the application is emitted as a `kind: "model-group"` entry with
48
+ an empty `root` and an id ending in `models-<application-package>`. All of its
49
+ concrete models appear once, alongside the application's value objects,
50
+ operations and events. This is a source grouping, not a claim that the models
51
+ share a transactional boundary. Its tables link to the model blocks without
52
+ being labelled aggregate roots or children. No root-selection warning is
53
+ emitted, and no configuration is required to browse the models.
54
+
55
+ `options.aggregates` remains an optional refinement: selecting a root replaces
56
+ the source group with an aggregate. A configured root that no longer exists
57
+ produces a warning with concrete candidates, while retaining every model in
58
+ the group. Settings can help repair that configuration and save the choice to
59
+ the matching extraction step. Regenerate to apply it; other options are kept.
51
60
 
52
61
  **Field.** Each model attribute assigned a field, with the type as written:
53
62
  `CharField`, `DateTimeField`, and a relation as `ForeignKey[Invoice]`. The doc
@@ -162,9 +171,24 @@ observable effects. Traversal is bounded and cycles are cut by symbol, so a
162
171
  recursive helper cannot make extraction recursive. ORM models from routed
163
172
  applications remain visible here even when the application has no unambiguous
164
173
  aggregate root. A URLConf may directly mount an arbitrarily named method of a
165
- plain class, such as `Planet.fetch`; that is still an HTTP flow root. When no
166
- decorator or conventional handler name proves one verb, the flow is retained
167
- but the route is omitted from inferred OpenAPI rather than guessed. An
174
+ plain class, such as `Planet.fetch`, or a plain function; that is still an
175
+ HTTP flow root, and its verb is read off what the code declares, in the order
176
+ a reviewer would trust it: a decorator on the handler (`@action(methods=…)`,
177
+ `@api_view`, `@require_http_methods([…])`, `@require_GET`, `@require_POST`,
178
+ `@require_safe`, also through `method_decorator`), the same decorators on the
179
+ class (`@method_decorator(…, name="dispatch")`), the class's
180
+ `http_method_names` (less `HEAD`, `OPTIONS` and `TRACE`, which every route
181
+ answers), a branch on `request.method` in the handler body, and last a project
182
+ wrapper — a decorator or a function the handler hands `request` to — whose own
183
+ body does one of those, followed a bounded number of levels deep. The first
184
+ tier that speaks decides; a declaration listing several verbs makes one
185
+ endpoint per verb, `planet_status` and `planet_status_patch`. When none of
186
+ them speaks, the verb is not guessed: the route stays in `provides` with an
187
+ empty `http.method`, the inferred OpenAPI document keeps the path as an item
188
+ with no operations and `x-portolan-verb: unknown`, the flow is retained, and
189
+ a diagnostic names the route. The merge never matches an outbound call
190
+ against a route whose verb is unknown, so the link waits for a declaration
191
+ rather than being confirmed by the path alone. An
168
192
  inherited DRF generic action has no local handler body, so
169
193
  its framework behaviour is reconstructed instead: list/retrieve read the
170
194
  model, create/update validate through the selected serializer and persist it,
@@ -267,8 +291,7 @@ The primary store is inferred from `DATABASES["default"]` when that setting is
267
291
  statically readable; `store` remains the override for settings assembled at
268
292
  runtime or for a catalog that needs a different stable slug. Every concrete,
269
293
  non-proxy model becomes a table even when its application has several possible
270
- aggregate roots: ambiguity limits the domain model, not the persistence
271
- schema. Fields from abstract model bases are copied into those tables
294
+ aggregate roots: both the model group and persistence schema stay visible. Fields from abstract model bases are copied into those tables
272
295
  regardless of which model module sorts first. `DEFAULT_AUTO_FIELD` supplies
273
296
  implicit primary-key types. PostgreSQL `ArrayField` nesting and
274
297
  `MultiSelectField` storage are rendered as their database types; `to_field`
@@ -323,9 +346,8 @@ Everything else means what it means for `extract-ts`.
323
346
 
324
347
  These cases do not become facts in the fragment:
325
348
 
326
- - a domain aggregate for an application with no models, or with several and no
327
- root among them named after it (its statically resolvable HTTP routes are
328
- still included);
349
+ - a domain aggregate for an application with no models (its statically
350
+ resolvable HTTP routes are still included);
329
351
  - a model that declares no fields;
330
352
  - an events module holding a class with no wire name, and a signal with no
331
353
  payload;
@@ -1,15 +1,15 @@
1
1
  """`models.py` read as the model.
2
2
 
3
- A Django application is one aggregate: the root is the model named after the
4
- application, the other models in it are its entities, and the frozen
5
- dataclasses in `values.py` are its value objects. Nothing is annotated for the
6
- catalog - the application is the claim, the same way a directory is the claim
7
- in `extract-ts`.
3
+ A Django application groups its models. When a root is known, the group is
4
+ an aggregate; otherwise it remains a model-group with no asserted boundary.
5
+ Frozen dataclasses in `values.py` are its value objects. Ambiguity never hides
6
+ the concrete models or requires configuration to browse them.
8
7
  """
9
8
 
10
9
  from __future__ import annotations
11
10
 
12
11
  import ast
12
+ import json
13
13
  from dataclasses import dataclass, field as dc_field
14
14
  from typing import Dict, List, Optional
15
15
 
@@ -90,7 +90,7 @@ class Aggregate:
90
90
  """One application, and the catalog object being built from it."""
91
91
 
92
92
  app: App
93
- root: ModelDef
93
+ root: Optional[ModelDef]
94
94
  models: List[ModelDef]
95
95
  aggregate: Dict[str, object]
96
96
 
@@ -257,7 +257,7 @@ def root_of(app: App, models: List[ModelDef], named: Dict[str, str], b) -> Optio
257
257
  for m in concrete:
258
258
  if m.name == want:
259
259
  return m
260
- b.warn(app.rel, "aggregates names %s for %s, and no model there is called that" % (want, app.label))
260
+ b.warn(app.rel, "aggregates names %s for %s, and no model there is called that%s" % (want, app.label, aggregate_candidates(app, concrete)))
261
261
  return None
262
262
  for candidate in (pascal(singular(app.label)), pascal(app.label)):
263
263
  for m in concrete:
@@ -268,13 +268,20 @@ def root_of(app: App, models: List[ModelDef], named: Dict[str, str], b) -> Optio
268
268
  if not concrete:
269
269
  b.warn(app.rel, "no models in this application: nothing here to be an aggregate")
270
270
  return None
271
- b.warn(
272
- app.rel,
273
- "no model called %s, and %d models to choose from: name the root in the aggregates option" % (pascal(singular(app.label)), len(concrete)),
274
- )
275
271
  return None
276
272
 
277
273
 
274
+ def aggregate_candidates(app: App, models: List[ModelDef]) -> str:
275
+ """Keep candidate evidence in the warning-only plugin protocol."""
276
+ return "; aggregate candidates: " + json.dumps({
277
+ "app": app.dotted,
278
+ "models": [
279
+ {"name": model.name, "path": model.module.rel, "line": model.node.lineno}
280
+ for model in sorted(models, key=lambda model: (model.name, model.module.rel))
281
+ ],
282
+ }, ensure_ascii=True)
283
+
284
+
278
285
  def read_aggregates(
279
286
  project: Project,
280
287
  applications: List[App],
@@ -289,14 +296,18 @@ def read_aggregates(
289
296
  if models is None:
290
297
  models = read_models(app)
291
298
  root = root_of(app, models, named, b)
292
- if root is None:
293
- continue
294
299
  concrete = [m for m in models if m.concrete]
295
- ordered = [root] + [m for m in concrete if m is not root]
296
- agg_slug = slug(root.name)
300
+ if not concrete:
301
+ continue
302
+ ordered = [root] + [m for m in concrete if m is not root] if root else concrete
303
+ # An application is a source grouping, not proof of a transactional
304
+ # boundary. Keep every model visible without inventing such a boundary.
305
+ agg_slug = slug(root.name) if root else "models-" + slug(app.dotted.replace(".", "-"))
297
306
  agg_id = aggregate_id(svc_id, agg_slug)
298
- readme = app.readme or (ast.get_docstring(root.node, clean=True) or "").strip()
299
- obj = catalog.aggregate(agg_id, agg_slug, root.name, readme, root.name)
307
+ readme = app.readme or ((ast.get_docstring(root.node, clean=True) or "").strip() if root else "")
308
+ obj = catalog.aggregate(agg_id, agg_slug, root.name if root else app.dotted, readme, root.name if root else "")
309
+ if root is None:
310
+ obj["kind"] = "model-group"
300
311
  for model in ordered:
301
312
  obj["entities"].append(
302
313
  catalog.block(