@shortlink-org/portolan 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/catalog/enum_test.go +46 -0
- package/catalog/evidence_test.go +35 -0
- package/catalog/model.go +1066 -0
- package/catalog/roundtrip_test.go +203 -0
- package/catalog/via_test.go +38 -0
- package/cli/init.test.mjs +6 -1
- package/cli/portolan.mjs +8 -0
- package/cli/portolan.test.mjs +49 -0
- package/go.mod +14 -0
- package/go.sum +20 -0
- package/internal/gocall/README.md +19 -0
- package/internal/gocall/analyze.go +189 -0
- package/internal/gocall/analyze_test.go +107 -0
- package/internal/gohttp/analyze.go +2562 -0
- package/internal/gohttp/destination.go +373 -0
- package/internal/gohttp/endpoints.go +1067 -0
- package/internal/gohttp/roots.go +320 -0
- package/internal/gohttp/typed.go +96 -0
- package/internal/goscan/constants.go +85 -0
- package/internal/goscan/goscan_test.go +227 -0
- package/internal/goscan/index.go +629 -0
- package/internal/goscan/index_test.go +66 -0
- package/internal/goscan/names.go +52 -0
- package/internal/goscan/parse_test.go +11 -0
- package/internal/goscan/source.go +37 -0
- package/internal/goscan/tree.go +284 -0
- package/internal/goscan/types.go +99 -0
- package/internal/wsdl/ids.go +127 -0
- package/internal/wsdl/ids_test.go +21 -0
- package/internal/wsdl/model.go +70 -0
- package/internal/wsdl/parse.go +949 -0
- package/internal/wsdl/parse_test.go +170 -0
- package/package.json +22 -10
- package/plugin/describe.go +118 -0
- package/plugin/describe_test.go +114 -0
- package/plugin/protocol.go +141 -0
- package/plugin/schematest/schematest.go +126 -0
- package/plugins/README.md +114 -46
- package/plugins/cmd/portolan-http-clients/main.go +19 -0
- package/plugins/extract-celery/extract.py +0 -2
- package/plugins/extract-celery/extract_test.py +1 -1
- package/plugins/extract-django/README.md +39 -17
- package/plugins/extract-django/domain.py +28 -17
- package/plugins/extract-django/extract.py +21 -7
- package/plugins/extract-django/extract_test.py +55 -2
- package/plugins/extract-django/lifecycle.py +2 -0
- package/plugins/extract-django/operations.py +1 -1
- package/plugins/extract-django/routing_test.py +109 -1
- package/plugins/extract-django/store.py +1 -1
- package/plugins/extract-django/transport.py +101 -55
- package/plugins/extract-django/verbs.py +241 -0
- package/plugins/extract-go/README.md +47 -0
- package/plugins/extract-http-clients/describe.go +19 -0
- package/plugins/extract-http-clients/describe_test.go +11 -0
- package/plugins/extract-http-clients/extract.go +740 -0
- package/plugins/extract-http-clients/extract_test.go +1561 -0
- package/plugins/extract-http-clients/main.go +41 -0
- package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
- package/plugins/extract-python-kafka/README.md +6 -0
- package/plugins/extract-python-kafka/extract.py +2 -4
- package/plugins/extract-python-kafka/extract_test.py +18 -2
- package/plugins/extract-ts/extract.test.ts +2 -2
- package/plugins/extract-ts/extract.ts +4 -5
- package/plugins/extract-ts/graphql.test.ts +1 -1
- package/plugins/openapi/ids.go +261 -0
- package/plugins/openapi/ids_test.go +98 -0
- package/plugins/portolan-go.wasm +0 -0
- package/plugins/pyplugin/catalog.py +12 -1
- package/plugins/pyplugin/kafka.py +74 -3
- package/plugins/pyplugin/protocol.py +1 -5
- package/portolan.json +3 -2
- package/schema/portolan.schema.json +34 -0
- package/scripts/README.md +18 -12
- package/scripts/catalog-sources.mjs +6 -0
- package/scripts/delivery-presets.mjs +21 -11
- package/scripts/diff.mjs +5 -1
- package/scripts/django-aggregates.test.mjs +58 -0
- package/scripts/gen-likec4.mjs +79 -21
- package/scripts/gen-likec4.test.mjs +25 -2
- package/scripts/gen.mjs +118 -115
- package/scripts/go-discovery.test.mjs +30 -0
- package/scripts/history.mjs +186 -3
- package/scripts/history.test.mjs +1 -1
- package/scripts/host-plugins/fetch-git.mjs +77 -21
- package/scripts/host-plugins/fetch-git.test.mjs +62 -8
- package/scripts/local-api.mjs +71 -4
- package/scripts/local-api.test.mjs +63 -4
- package/scripts/local-discovery.mjs +82 -9
- package/scripts/manifest.mjs +5 -3
- package/scripts/manifest.test.mjs +24 -0
- package/scripts/output-diff.mjs +94 -0
- package/scripts/output-diff.test.mjs +36 -0
- package/scripts/package-smoke.mjs +62 -4
- package/scripts/plugin-host.mjs +22 -2
- package/scripts/plugin-host.test.mjs +9 -0
- package/scripts/plugin-wasm-worker.mjs +4 -1
- package/scripts/provenance.mjs +72 -0
- package/scripts/provenance.test.mjs +149 -0
- package/scripts/run-builtin.mjs +39 -5
- package/scripts/schema.mjs +29 -0
- package/scripts/warning-policy.mjs +161 -0
- package/scripts/warning-policy.test.mjs +56 -0
- package/src/app/Breadcrumbs.test.ts +4 -0
- package/src/app/Breadcrumbs.tsx +1 -0
- package/src/app/Sidebar.tsx +3 -3
- package/src/catalog-docs.test.ts +64 -0
- package/src/catalog-docs.ts +35 -0
- package/src/catalog-error.test.ts +15 -0
- package/src/catalog-model.ts +70 -6
- package/src/catalog-stores.test.ts +17 -0
- package/src/catalog-validation.ts +52 -2
- package/src/catalog.test.ts +13 -2
- package/src/chat/Starter.tsx +5 -11
- package/src/chat/tools.test.ts +27 -0
- package/src/chat/tools.ts +5 -9
- package/src/components/CatalogStamp.tsx +10 -8
- package/src/components/ChannelRows.messagepack.test.tsx +28 -0
- package/src/components/ChannelRows.test.tsx +54 -0
- package/src/components/ChannelRows.tsx +57 -10
- package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
- package/src/components/HTTPDestinationEvidence.tsx +31 -0
- package/src/components/Integrations.tsx +1 -1
- package/src/components/LifecycleDiagram.tsx +28 -12
- package/src/components/MachineDocs.tsx +6 -5
- package/src/components/MethodRows.tsx +9 -2
- package/src/components/ProblemRow.tsx +4 -0
- package/src/components/RelationEvidence.test.tsx +14 -0
- package/src/components/RelationEvidence.tsx +53 -0
- package/src/components/WhatLinksHere.tsx +6 -4
- package/src/data.ts +25 -7
- package/src/enrich.test.ts +336 -6
- package/src/enrich.ts +206 -3
- package/src/flow/StepDetail.tsx +104 -54
- package/src/flow/answers.test.ts +18 -1
- package/src/flow/answers.ts +37 -8
- package/src/flow/evidence.test.ts +16 -0
- package/src/flow/evidence.ts +34 -0
- package/src/index.css +44 -0
- package/src/landing/DraggableReveal.tsx +3 -2
- package/src/landing/EvidencePipeline.tsx +105 -0
- package/src/landing/LandingPage.tsx +2 -59
- package/src/lib/backlinks.test.ts +16 -1
- package/src/lib/backlinks.ts +20 -0
- package/src/lib/catalog-diff.test.ts +18 -0
- package/src/lib/catalog-diff.ts +20 -2
- package/src/lib/derive.ts +1 -0
- package/src/lib/django-aggregates.d.mts +9 -0
- package/src/lib/django-aggregates.mjs +36 -0
- package/src/lib/django-aggregates.test.ts +29 -0
- package/src/lib/django-aggregates.ts +5 -0
- package/src/lib/kafka-ui.test.ts +87 -0
- package/src/lib/kafka-ui.ts +105 -0
- package/src/lib/local-api.ts +20 -2
- package/src/lib/setup-info.test.ts +17 -0
- package/src/lib/setup-info.ts +58 -0
- package/src/lib/warnings.test.ts +54 -0
- package/src/lib/warnings.ts +260 -0
- package/src/lib/wire-problems.test.ts +21 -0
- package/src/lib/wire-problems.ts +62 -1
- package/src/likec4/FlowView.tsx +2 -6
- package/src/likec4/flow-edges.test.ts +64 -1
- package/src/likec4/flow-edges.ts +43 -7
- package/src/likec4/view-index.ts +8 -2
- package/src/map/ContextMapGraph.tsx +76 -32
- package/src/merge.test.ts +23 -0
- package/src/merge.ts +33 -10
- package/src/pages/AggregatePage.tsx +8 -7
- package/src/pages/CatalogFailure.tsx +2 -2
- package/src/pages/ContextPage.tsx +6 -5
- package/src/pages/ServicePage.tsx +4 -3
- package/src/pages/Settings.tsx +200 -44
- package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
- package/src/pages/settings/IntegrationsSettings.tsx +117 -0
- package/src/routes.test.ts +2 -0
- package/src/routes.ts +2 -1
- package/src/selection/DetailPanel.tsx +61 -1
- package/src/virtual-provenance.d.ts +11 -0
- package/vite.config.ts +5 -0
- package/scripts/vendor-lock.mjs +0 -58
- package/scripts/vendor-lock.test.mjs +0 -69
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// Package schematest checks that a plugin's options schema still describes its
|
|
2
|
+
// options struct.
|
|
3
|
+
//
|
|
4
|
+
// The schema is written by hand rather than reflected out of the struct,
|
|
5
|
+
// because what makes it worth having is the prose: which slug a store expects,
|
|
6
|
+
// whose proto package a peer answers for. None of that survives reflection.
|
|
7
|
+
// What reflection is good for is the half nobody would notice going stale - a
|
|
8
|
+
// field renamed, a field added, a schema left behind - so that is all this
|
|
9
|
+
// does.
|
|
10
|
+
package schematest
|
|
11
|
+
|
|
12
|
+
import (
|
|
13
|
+
"encoding/json"
|
|
14
|
+
"reflect"
|
|
15
|
+
"sort"
|
|
16
|
+
"strings"
|
|
17
|
+
"testing"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
type document struct {
|
|
21
|
+
Type string `json:"type"`
|
|
22
|
+
AdditionalProperties *bool `json:"additionalProperties"`
|
|
23
|
+
Properties map[string]json.RawMessage `json:"properties"`
|
|
24
|
+
Required []string `json:"required"`
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Check reports every way the schema and the struct have come apart.
|
|
28
|
+
//
|
|
29
|
+
// options is a zero value of the plugin's options type; schema is the embedded
|
|
30
|
+
// document it hands back when asked to describe itself.
|
|
31
|
+
func Check(t *testing.T, schema []byte, options any) {
|
|
32
|
+
t.Helper()
|
|
33
|
+
|
|
34
|
+
var doc document
|
|
35
|
+
if err := json.Unmarshal(schema, &doc); err != nil {
|
|
36
|
+
t.Fatalf("options schema is not JSON: %v", err)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if doc.Type != "object" {
|
|
40
|
+
t.Errorf("options schema has type %q, want object", doc.Type)
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Without this an unknown key validates, and the misspelled option - the
|
|
44
|
+
// one error this whole mechanism exists to catch - goes through silently.
|
|
45
|
+
if doc.AdditionalProperties == nil || *doc.AdditionalProperties {
|
|
46
|
+
t.Error("options schema must set additionalProperties: false")
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
described := keys(doc.Properties)
|
|
50
|
+
declared := fields(reflect.TypeOf(options))
|
|
51
|
+
|
|
52
|
+
for _, name := range missing(declared, described) {
|
|
53
|
+
t.Errorf("option %q is in the struct but not in the schema", name)
|
|
54
|
+
}
|
|
55
|
+
for _, name := range missing(described, declared) {
|
|
56
|
+
t.Errorf("option %q is in the schema but not in the struct", name)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
for _, name := range doc.Required {
|
|
60
|
+
if _, ok := doc.Properties[name]; !ok {
|
|
61
|
+
t.Errorf("option %q is required but not described", name)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// A property with nothing to say is a property the manifest schema will
|
|
66
|
+
// offer with no hint of what it means, which is where this started.
|
|
67
|
+
for name, raw := range doc.Properties {
|
|
68
|
+
var property struct {
|
|
69
|
+
Description string `json:"description"`
|
|
70
|
+
}
|
|
71
|
+
if err := json.Unmarshal(raw, &property); err != nil {
|
|
72
|
+
t.Errorf("option %q: %v", name, err)
|
|
73
|
+
|
|
74
|
+
continue
|
|
75
|
+
}
|
|
76
|
+
if strings.TrimSpace(property.Description) == "" {
|
|
77
|
+
t.Errorf("option %q has no description", name)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// fields are the json names of an options struct, which is the only shape this
|
|
83
|
+
// takes: a plugin's options are a flat object read out of the manifest.
|
|
84
|
+
func fields(t reflect.Type) []string {
|
|
85
|
+
var names []string
|
|
86
|
+
|
|
87
|
+
for i := 0; i < t.NumField(); i++ {
|
|
88
|
+
tag := t.Field(i).Tag.Get("json")
|
|
89
|
+
name, _, _ := strings.Cut(tag, ",")
|
|
90
|
+
if name == "" || name == "-" {
|
|
91
|
+
continue
|
|
92
|
+
}
|
|
93
|
+
names = append(names, name)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
sort.Strings(names)
|
|
97
|
+
|
|
98
|
+
return names
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
func keys(m map[string]json.RawMessage) []string {
|
|
102
|
+
names := make([]string, 0, len(m))
|
|
103
|
+
for name := range m {
|
|
104
|
+
names = append(names, name)
|
|
105
|
+
}
|
|
106
|
+
sort.Strings(names)
|
|
107
|
+
|
|
108
|
+
return names
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// missing is everything in a that is not in b.
|
|
112
|
+
func missing(a, b []string) []string {
|
|
113
|
+
have := make(map[string]bool, len(b))
|
|
114
|
+
for _, name := range b {
|
|
115
|
+
have[name] = true
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
var gone []string
|
|
119
|
+
for _, name := range a {
|
|
120
|
+
if !have[name] {
|
|
121
|
+
gone = append(gone, name)
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return gone
|
|
126
|
+
}
|
package/plugins/README.md
CHANGED
|
@@ -54,7 +54,10 @@ Three obligations, and they are the whole of it:
|
|
|
54
54
|
unsafe filename or invalid response fails the run. Non-fatal extraction
|
|
55
55
|
notes go to stderr; there is no advisory response property a caller may
|
|
56
56
|
accidentally ignore. A note that opens with `warning: ` is kept beside the
|
|
57
|
-
step in `.portolan/build-report.json
|
|
57
|
+
step in `.portolan/build-report.json`. The Settings page classifies and
|
|
58
|
+
groups those notes by plugin, stable rule and severity. Typed CEL policies
|
|
59
|
+
in `portolan.json` can suppress reviewed diagnostics only with an explicit
|
|
60
|
+
reason; the host evaluates them and records the decision in the build report.
|
|
58
61
|
|
|
59
62
|
The repository enforces those obligations with schema/field coverage tests,
|
|
60
63
|
byte-for-byte permutation tests, generated-link and anchor checks, Mermaid
|
|
@@ -78,12 +81,14 @@ either be rendered or be explicitly acknowledged by the relevant exporter.
|
|
|
78
81
|
`go:embed` and returned in the descriptor. `schematest.Check` in a test keeps
|
|
79
82
|
it from drifting from the options struct: a field renamed on one side and not
|
|
80
83
|
the other fails, and so does an option with no description.
|
|
81
|
-
3. Build it. A built-in Go plugin is a library package with
|
|
84
|
+
3. Build it. A sandboxed built-in Go plugin is a library package with
|
|
82
85
|
`Serve(io.Reader, io.Writer) error`; add it to the map in
|
|
83
86
|
`plugins/cmd/portolan-go/main.go`, and `plugins:build` in `package.json`
|
|
84
87
|
puts it in `plugins/portolan-go.wasm` with the rest (a test keeps the map
|
|
85
|
-
and `portolan.json` in step). A
|
|
86
|
-
`
|
|
88
|
+
and `portolan.json` in step). A built-in that genuinely needs the Go
|
|
89
|
+
toolchain gets a small command under `plugins/cmd/` and is declared as a
|
|
90
|
+
`process`; `http-clients` is the example. A plugin of your own is its own
|
|
91
|
+
module: `GOOS=wasip1 GOARCH=wasm go build`.
|
|
87
92
|
4. Declare it in `portolan.json`, under `plugins` (how to run it) and
|
|
88
93
|
`generate` (what to run it on), then run `npm run schema` so the manifest
|
|
89
94
|
schema learns its options.
|
|
@@ -179,10 +184,23 @@ the original collection layout for TypeScript, Rust, Java, or custom trees and
|
|
|
179
184
|
may also point directly at one feature repository.
|
|
180
185
|
|
|
181
186
|
`extract-river` is another independent enrichment for Go repositories. It
|
|
182
|
-
joins a job argument's `Kind()` to `Client.Insert`/`InsertTx
|
|
183
|
-
queue, `Worker[Args].Work`, and
|
|
184
|
-
channel plus a two-hop
|
|
185
|
-
|
|
187
|
+
joins a job argument's `Kind()` to `Client.Insert`/`InsertTx` and the
|
|
188
|
+
`InsertMany*` batches, the selected queue, `Worker[Args].Work`, and
|
|
189
|
+
`river.AddWorker`. The result is a work-queue channel plus a two-hop
|
|
190
|
+
enqueue/dispatch flow, with payload fields and source lines. It does not need
|
|
191
|
+
aggregates and does not treat a job as a domain event. The job type is
|
|
192
|
+
followed through locals, constructors' results and parameters up to the
|
|
193
|
+
callers, so an `Enqueue(ctx, args river.JobArgs, opts)` wrapper is read once
|
|
194
|
+
per concrete job passed to it, and a worker registered as
|
|
195
|
+
`river.AddWorker(workers, NewSendWorker(deps))` is registered. The queue is
|
|
196
|
+
the `InsertOpts.Queue` at the call (a literal, a constant, a config `default:`
|
|
197
|
+
tag, or what the callers pass), else the args type's own `InsertOpts()`, else
|
|
198
|
+
`default`. What no Insert in the tree feeds is not dropped: a registered
|
|
199
|
+
worker's queue is the one its type chooses or the one queue `river.Config`
|
|
200
|
+
lists, and with several and no choice the worker is kept as a flow whose
|
|
201
|
+
receive is unresolved. The diagnostics keep the two cases apart - an Insert
|
|
202
|
+
whose job type or queue the reader could not follow is said at the call, a
|
|
203
|
+
worker with no Insert in the tree is said at the worker.
|
|
186
204
|
|
|
187
205
|
`extract-redis` finds runtime construction of go-redis, rueidis and redigo
|
|
188
206
|
clients in non-test Go source. That source evidence adds a service-owned Redis
|
|
@@ -193,16 +211,27 @@ key patterns with their read/write/delete use, TTL, value type and source. The
|
|
|
193
211
|
patterns remain Redis keyspaces rather than being presented as SQL tables.
|
|
194
212
|
|
|
195
213
|
`extract-watermill` reads Watermill `Router.AddHandler` and
|
|
196
|
-
`AddNoPublisherHandler` registrations
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
214
|
+
`AddNoPublisherHandler` registrations, and `Subscriber.Subscribe` calls made
|
|
215
|
+
without a router. It resolves literal and constant topics - typed constants
|
|
216
|
+
through `string(...)`, constants concatenated with `+`, single-return
|
|
217
|
+
helpers - plus defaults on env-config structs, nested (`cfg.Kafka.Orders`) or
|
|
218
|
+
not; a topic, a name or a handler that a registering function takes as a
|
|
219
|
+
parameter is filled in once per caller, so a `handle(r, name, topic, fn)`
|
|
220
|
+
wrapper yields one handler per call. The handler value may be a function
|
|
221
|
+
literal, a function or method of the tree (`h.OnOrder`, `handlers.OnShipped`)
|
|
222
|
+
or a constructor returning a literal. It follows direct `Publisher.Publish`
|
|
223
|
+
calls and one-hop publishing helpers, records what the service publishes
|
|
224
|
+
outside any handler as a `send` on the topic, and traces JSON marshal/unmarshal
|
|
225
|
+
values back to their Go payload structs. It carries enclosing `if`/`else`
|
|
226
|
+
conditions and early returns through the handler's control flow: proven
|
|
227
|
+
alternatives become one catalog `alt`; publications whose relationship cannot
|
|
228
|
+
be proven remain separate possible routes. A handler whose topic does not
|
|
229
|
+
resolve is kept, with its receive step unresolved and no channel claimed, and
|
|
230
|
+
named in a diagnostic. The channels merge normally with AsyncAPI declarations
|
|
231
|
+
by address. Generic `NewEventHandler[T]` and `NewCommandHandler[T]`
|
|
232
|
+
registrations on Watermill CQRS processors are also extracted; fixed topic
|
|
233
|
+
generators and the standard event/command-name generator form are resolved
|
|
234
|
+
from source.
|
|
206
235
|
|
|
207
236
|
`extract-go-nats` reads nats.go and JetStream calls into the subjects a service
|
|
208
237
|
listens on and publishes to. A call is known by the type it is made on -
|
|
@@ -260,16 +289,19 @@ literals, direct assignments, or setters. A standalone flow says whether no
|
|
|
260
289
|
source caller exists or callers exist but no inbound/asynchronous root was
|
|
261
290
|
proved, so the UI exposes the missing evidence instead of implying a complete
|
|
262
291
|
business path.
|
|
263
|
-
When a provider branch still ends before its transport, the extractor
|
|
264
|
-
the module with `go/packages`, builds
|
|
292
|
+
When a provider branch still ends before its transport, the extractor runs as
|
|
293
|
+
a workspace-local native sidecar, loads the module with `go/packages`, builds
|
|
294
|
+
SSA, and uses `x/tools` VTA to resolve
|
|
265
295
|
calls through interface parameters, function values, return values, and
|
|
266
296
|
interface-typed struct fields. Typed edges are followed only after a concrete
|
|
267
297
|
factory branch is selected: applying context-insensitive VTA to a shared
|
|
268
298
|
dispatcher would otherwise attach every request implementation to every
|
|
269
299
|
endpoint. Factory conditions and HTTP/SOAP meaning continue to come from the
|
|
270
|
-
source extractor.
|
|
271
|
-
|
|
272
|
-
|
|
300
|
+
source extractor. The sidecar is built from the Portolan package through the
|
|
301
|
+
project's Go toolchain and runs with the project as its working directory.
|
|
302
|
+
Module loading is read-only and bounded; unavailable private dependencies,
|
|
303
|
+
type errors, or a timeout produce a warning and retain the syntax-only result
|
|
304
|
+
rather than failing generation.
|
|
273
305
|
Routes without a provider factory are also joined to their outbound calls,
|
|
274
306
|
including handlers invoked from closures and methods on locally constructed
|
|
275
307
|
values. A Swagger `@Router` annotation is medium-confidence root evidence for
|
|
@@ -454,9 +486,10 @@ far side.
|
|
|
454
486
|
The demo estate's org-wide and context-wide records live in `data/adr`, and are
|
|
455
487
|
read by a step that points at that directory with a glob of its own. Root
|
|
456
488
|
`docs/` is where `gen-markdown` writes, so nothing hand-written can live there.
|
|
457
|
-
`in` is the directory of records rather than `data` itself:
|
|
458
|
-
|
|
459
|
-
`in: data` with `out: data` would
|
|
489
|
+
`in` is the directory of records rather than `data` itself: what `gen --check`
|
|
490
|
+
lists as changed inputs is the root less the output directories inside it, and
|
|
491
|
+
`in: data` with `out: data` would list the file the step writes among the
|
|
492
|
+
reasons it was rewritten.
|
|
460
493
|
|
|
461
494
|
```json
|
|
462
495
|
{
|
|
@@ -775,9 +808,9 @@ enforces: a team that owns a directory is a team that gets the pull request.
|
|
|
775
808
|
```
|
|
776
809
|
|
|
777
810
|
Point `in` at the directory the file is in, not at the repository root: the
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
811
|
+
subject of this step is the `CODEOWNERS` file, and `gen --check` explains a
|
|
812
|
+
changed fragment by what moved under the step's input. Rooted at the
|
|
813
|
+
repository, every commit ever made would be named. Left with no `file`, the three
|
|
781
814
|
places a forge looks are tried in order - `CODEOWNERS`, `.github/CODEOWNERS`,
|
|
782
815
|
`docs/CODEOWNERS` - and a `file` that names something absent fails the run,
|
|
783
816
|
because answering "nobody owns anything" to a typo is only noticed a month
|
|
@@ -822,21 +855,23 @@ the forge looks.
|
|
|
822
855
|
`wasm` is the default and should stay that way. The module gets no network, no
|
|
823
856
|
environment and no way to start a process. A generator gets no filesystem
|
|
824
857
|
either. An extract or verify step gets the workspace preopened as `/`
|
|
825
|
-
(portolan.0006), which is how
|
|
826
|
-
Go toolchain on the machine:
|
|
827
|
-
the single module `plugins/portolan-go.wasm`, which answers to the plugin
|
|
828
|
-
the host passes as `argv[0]`. WASI preopens read-write, so an extractor is
|
|
858
|
+
(portolan.0006), which is how syntax-only built-in Go extractors read a tree
|
|
859
|
+
without a Go toolchain on the machine: those extractors and the three generators
|
|
860
|
+
are the single module `plugins/portolan-go.wasm`, which answers to the plugin
|
|
861
|
+
name the host passes as `argv[0]`. WASI preopens read-write, so an extractor is
|
|
829
862
|
trusted not to write the tree it reads, the same trust a process plugin has
|
|
830
863
|
today; a `sha256` pins that trust to a build.
|
|
831
864
|
|
|
832
|
-
`process` is the escape hatch for a plugin that needs a toolchain: the
|
|
833
|
-
Java, Python and TypeScript extractors run in
|
|
834
|
-
|
|
865
|
+
`process` is the escape hatch for a plugin that needs a toolchain: the typed Go
|
|
866
|
+
HTTP client analyzer and the Rust, Java, Python and TypeScript extractors run in
|
|
867
|
+
their own runtimes. It gets the same protocol
|
|
835
868
|
and none of the sandbox, which is the trade being made and the reason it is
|
|
836
869
|
not the default. It declares `command` and an `args` array; the host never
|
|
837
|
-
feeds a command string through a shell.
|
|
838
|
-
|
|
839
|
-
|
|
870
|
+
feeds a command string through a shell. For a built-in declared as `go run`,
|
|
871
|
+
the adapter compiles a workspace-local native sidecar from the package and
|
|
872
|
+
then runs it with the scanned workspace as its current directory. This keeps
|
|
873
|
+
module resolution in Portolan's shipped source and project loading in the
|
|
874
|
+
project being analyzed.
|
|
840
875
|
|
|
841
876
|
`host` is for Portolan's own code that needs what only the host has - a git
|
|
842
877
|
binary, a socket - and so runs inside the host process (portolan.0008):
|
|
@@ -859,12 +894,18 @@ written.
|
|
|
859
894
|
`fetch-git` is `fetch-bsr` for a repository rather than a registry, and it
|
|
860
895
|
lives by the same four rules. A pin is a repository, a commit and the paths
|
|
861
896
|
actually read; the step fetches exactly those directories at exactly that
|
|
862
|
-
commit and hands
|
|
897
|
+
commit and hands their text files back, so the host writes them into the tree
|
|
863
898
|
beside a `git.lock.json` naming the commit and the digest of every file. The
|
|
864
899
|
paths inside the copy are the repository's own, which is the point: the
|
|
865
900
|
extract step that follows points its `in` at the vendored service and reads
|
|
866
901
|
it exactly as it would read that service's checkout.
|
|
867
902
|
|
|
903
|
+
Binary files are not part of the extractor input. Known binary extensions are
|
|
904
|
+
discarded before their blobs are read; files without a known extension are
|
|
905
|
+
sniffed for NUL bytes and invalid UTF-8 before the response is assembled. The
|
|
906
|
+
lock records only their path, byte size and skip reason, never Base64 content,
|
|
907
|
+
so online fetch and offline replay produce the same bounded text-only copy.
|
|
908
|
+
|
|
868
909
|
It runs inside the host (`scripts/host-plugins/fetch-git.mjs`,
|
|
869
910
|
portolan.0008) rather than as a module, because it needs a git binary and a
|
|
870
911
|
socket, and a manifest names it with `host` rather than `wasm` or `process`.
|
|
@@ -919,11 +960,12 @@ Nothing else can say it. A service says which repository it lives in, and an
|
|
|
919
960
|
extractor reads a directory as a pure function of what is on disk; neither has
|
|
920
961
|
any idea which commit somebody fetched. Without that line, every source path
|
|
921
962
|
of every vendored service is dead text on the page - the file and the line are
|
|
922
|
-
known, and there is nowhere to send a reader
|
|
923
|
-
the
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
963
|
+
known, and there is nowhere to send a reader. With it, `sourceHref` links the
|
|
964
|
+
line at the commit it was read at. The fragments read out of the copy are
|
|
965
|
+
dated, like every other source, by the commit of THIS repository that last
|
|
966
|
+
changed them (portolan.0010) - which is the fetch that brought the change in,
|
|
967
|
+
and is what "how fresh is this" means to the estate that vendored it; the pin
|
|
968
|
+
is what says which upstream commit that was.
|
|
927
969
|
|
|
928
970
|
One more line is needed for the app itself: `SOURCE_GLOBS` in `src/data.ts`,
|
|
929
971
|
where the same patterns are written out a second time because
|
|
@@ -1020,7 +1062,8 @@ workflow already does.
|
|
|
1020
1062
|
|
|
1021
1063
|
Four rules govern what happens when a fetch does not:
|
|
1022
1064
|
|
|
1023
|
-
1. Fetch succeeded → the fetched files and a
|
|
1065
|
+
1. Fetch succeeded → the fetched text files, binary-skip metadata and a
|
|
1066
|
+
regenerated lock.
|
|
1024
1067
|
2. Skipped or failed, cache complete and matching its digests → the cached files
|
|
1025
1068
|
byte-identically, plus a warning. Output unchanged, so `--check` stays clean.
|
|
1026
1069
|
3. Failed **and** no usable cache → a non-zero exit, never a short file list. The
|
|
@@ -1142,6 +1185,11 @@ live in another estate — would be the wrong end of the trade.
|
|
|
1142
1185
|
service declares — the address the broker knows, and each message on it with the
|
|
1143
1186
|
direction it travels. What it does **not** answer with is events.
|
|
1144
1187
|
|
|
1188
|
+
A message's `contentType`, or the document's `defaultContentType` when the
|
|
1189
|
+
message leaves it out, is kept as the exact declared media type. MessagePack
|
|
1190
|
+
media types are additionally normalized to `encoding: msgpack`, so source and
|
|
1191
|
+
contract extractors can be compared without depending on one MIME spelling.
|
|
1192
|
+
|
|
1145
1193
|
That looks like a gap and is a boundary. `Event.id` is
|
|
1146
1194
|
`<service>.<aggregate>.<Name>`, and an AsyncAPI document knows the message on the
|
|
1147
1195
|
wire, not the aggregate that raised it. An extractor that guessed would either
|
|
@@ -1197,3 +1245,23 @@ it, the call before the decorator before `task_routes` before the default,
|
|
|
1197
1245
|
and `transaction.on_commit(...)` around an enqueue is a note on the step,
|
|
1198
1246
|
which is the one fact about *when* a message leaves that the code states
|
|
1199
1247
|
plainly.
|
|
1248
|
+
|
|
1249
|
+
### HTTP destination provenance
|
|
1250
|
+
|
|
1251
|
+
`extract-http-clients` emits structured `destination` evidence on HTTP consumes
|
|
1252
|
+
and flow steps: the request call site and endpoint expression, local path,
|
|
1253
|
+
base URL expression, config field and environment variable, source locations,
|
|
1254
|
+
and a full path when source proves the join. Functional options that assign a
|
|
1255
|
+
URL field to a returned client are followed through adapter constructors.
|
|
1256
|
+
Bindings belong to the adapter's client field; conflicting construction sites
|
|
1257
|
+
remain unresolved instead of sharing another client's settings.
|
|
1258
|
+
|
|
1259
|
+
A Go `default` struct tag is recorded as `config-default`, not as the effective
|
|
1260
|
+
runtime configuration. For example, `WithBaseURL(cfg.SettingAddr)` with
|
|
1261
|
+
`default:"http://localhost:8000/settings"` and a concatenated
|
|
1262
|
+
`/get-admin-settings` yields `/settings/get-admin-settings`. The merge matches
|
|
1263
|
+
that full path exactly and never falls back to a suffix if it is absent.
|
|
1264
|
+
Legacy suffix matches remain explicitly labeled `unique-suffix` in the evidence
|
|
1265
|
+
shown in RPC step details. Later runtime URL modifiers are listed separately;
|
|
1266
|
+
the recovered full path describes the static join. Unreadable options may
|
|
1267
|
+
leave only the local route available.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Command portolan-http-clients is the native sidecar for typed Go call-graph
|
|
2
|
+
// analysis. It intentionally does not compile for WASI: go/packages needs the
|
|
3
|
+
// project Go toolchain to load build tags and dependencies before SSA/VTA can
|
|
4
|
+
// resolve interface calls.
|
|
5
|
+
package main
|
|
6
|
+
|
|
7
|
+
import (
|
|
8
|
+
"fmt"
|
|
9
|
+
"os"
|
|
10
|
+
|
|
11
|
+
extracthttpclients "github.com/shortlink-org/portolan/plugins/extract-http-clients"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
func main() {
|
|
15
|
+
if err := extracthttpclients.Serve(os.Stdin, os.Stdout); err != nil {
|
|
16
|
+
fmt.Fprintln(os.Stderr, "portolan-http-clients:", err)
|
|
17
|
+
os.Exit(1)
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -155,8 +155,6 @@ def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
|
|
|
155
155
|
flows.sort(key=lambda f: f["slug"])
|
|
156
156
|
|
|
157
157
|
fragment: Dict[str, Any] = {
|
|
158
|
-
"generatedAt": input_.generated_at,
|
|
159
|
-
"commit": input_.commit,
|
|
160
158
|
"contexts": [
|
|
161
159
|
{
|
|
162
160
|
"id": context,
|
|
@@ -27,7 +27,7 @@ ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
|
27
27
|
def run(fixture, options):
|
|
28
28
|
b = Builder()
|
|
29
29
|
extract(
|
|
30
|
-
Input(root=os.path.relpath(os.path.join(HERE, "testdata", fixture), ROOT)
|
|
30
|
+
Input(root=os.path.relpath(os.path.join(HERE, "testdata", fixture), ROOT)),
|
|
31
31
|
Options.of(options),
|
|
32
32
|
b,
|
|
33
33
|
cwd=ROOT,
|
|
@@ -39,15 +39,24 @@ resolves them without a type checker.
|
|
|
39
39
|
|
|
40
40
|
## What becomes what
|
|
41
41
|
|
|
42
|
-
**
|
|
43
|
-
application
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
`
|
|
49
|
-
|
|
50
|
-
|
|
42
|
+
**Models and aggregates.** Every concrete, non-proxy model is visible. When
|
|
43
|
+
an application has an explicitly configured root, a model named after the
|
|
44
|
+
application (`invoices` → `Invoice`), or just one concrete model, it retains
|
|
45
|
+
its existing aggregate representation and stable root-based id.
|
|
46
|
+
|
|
47
|
+
Otherwise the application is emitted as a `kind: "model-group"` entry with
|
|
48
|
+
an empty `root` and an id ending in `models-<application-package>`. All of its
|
|
49
|
+
concrete models appear once, alongside the application's value objects,
|
|
50
|
+
operations and events. This is a source grouping, not a claim that the models
|
|
51
|
+
share a transactional boundary. Its tables link to the model blocks without
|
|
52
|
+
being labelled aggregate roots or children. No root-selection warning is
|
|
53
|
+
emitted, and no configuration is required to browse the models.
|
|
54
|
+
|
|
55
|
+
`options.aggregates` remains an optional refinement: selecting a root replaces
|
|
56
|
+
the source group with an aggregate. A configured root that no longer exists
|
|
57
|
+
produces a warning with concrete candidates, while retaining every model in
|
|
58
|
+
the group. Settings can help repair that configuration and save the choice to
|
|
59
|
+
the matching extraction step. Regenerate to apply it; other options are kept.
|
|
51
60
|
|
|
52
61
|
**Field.** Each model attribute assigned a field, with the type as written:
|
|
53
62
|
`CharField`, `DateTimeField`, and a relation as `ForeignKey[Invoice]`. The doc
|
|
@@ -162,9 +171,24 @@ observable effects. Traversal is bounded and cycles are cut by symbol, so a
|
|
|
162
171
|
recursive helper cannot make extraction recursive. ORM models from routed
|
|
163
172
|
applications remain visible here even when the application has no unambiguous
|
|
164
173
|
aggregate root. A URLConf may directly mount an arbitrarily named method of a
|
|
165
|
-
plain class, such as `Planet.fetch
|
|
166
|
-
|
|
167
|
-
|
|
174
|
+
plain class, such as `Planet.fetch`, or a plain function; that is still an
|
|
175
|
+
HTTP flow root, and its verb is read off what the code declares, in the order
|
|
176
|
+
a reviewer would trust it: a decorator on the handler (`@action(methods=…)`,
|
|
177
|
+
`@api_view`, `@require_http_methods([…])`, `@require_GET`, `@require_POST`,
|
|
178
|
+
`@require_safe`, also through `method_decorator`), the same decorators on the
|
|
179
|
+
class (`@method_decorator(…, name="dispatch")`), the class's
|
|
180
|
+
`http_method_names` (less `HEAD`, `OPTIONS` and `TRACE`, which every route
|
|
181
|
+
answers), a branch on `request.method` in the handler body, and last a project
|
|
182
|
+
wrapper — a decorator or a function the handler hands `request` to — whose own
|
|
183
|
+
body does one of those, followed a bounded number of levels deep. The first
|
|
184
|
+
tier that speaks decides; a declaration listing several verbs makes one
|
|
185
|
+
endpoint per verb, `planet_status` and `planet_status_patch`. When none of
|
|
186
|
+
them speaks, the verb is not guessed: the route stays in `provides` with an
|
|
187
|
+
empty `http.method`, the inferred OpenAPI document keeps the path as an item
|
|
188
|
+
with no operations and `x-portolan-verb: unknown`, the flow is retained, and
|
|
189
|
+
a diagnostic names the route. The merge never matches an outbound call
|
|
190
|
+
against a route whose verb is unknown, so the link waits for a declaration
|
|
191
|
+
rather than being confirmed by the path alone. An
|
|
168
192
|
inherited DRF generic action has no local handler body, so
|
|
169
193
|
its framework behaviour is reconstructed instead: list/retrieve read the
|
|
170
194
|
model, create/update validate through the selected serializer and persist it,
|
|
@@ -267,8 +291,7 @@ The primary store is inferred from `DATABASES["default"]` when that setting is
|
|
|
267
291
|
statically readable; `store` remains the override for settings assembled at
|
|
268
292
|
runtime or for a catalog that needs a different stable slug. Every concrete,
|
|
269
293
|
non-proxy model becomes a table even when its application has several possible
|
|
270
|
-
aggregate roots:
|
|
271
|
-
schema. Fields from abstract model bases are copied into those tables
|
|
294
|
+
aggregate roots: both the model group and persistence schema stay visible. Fields from abstract model bases are copied into those tables
|
|
272
295
|
regardless of which model module sorts first. `DEFAULT_AUTO_FIELD` supplies
|
|
273
296
|
implicit primary-key types. PostgreSQL `ArrayField` nesting and
|
|
274
297
|
`MultiSelectField` storage are rendered as their database types; `to_field`
|
|
@@ -323,9 +346,8 @@ Everything else means what it means for `extract-ts`.
|
|
|
323
346
|
|
|
324
347
|
These cases do not become facts in the fragment:
|
|
325
348
|
|
|
326
|
-
- a domain aggregate for an application with no models
|
|
327
|
-
|
|
328
|
-
still included);
|
|
349
|
+
- a domain aggregate for an application with no models (its statically
|
|
350
|
+
resolvable HTTP routes are still included);
|
|
329
351
|
- a model that declares no fields;
|
|
330
352
|
- an events module holding a class with no wire name, and a signal with no
|
|
331
353
|
payload;
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
"""`models.py` read as the model.
|
|
2
2
|
|
|
3
|
-
A Django application
|
|
4
|
-
|
|
5
|
-
dataclasses in `values.py` are its value objects.
|
|
6
|
-
|
|
7
|
-
in `extract-ts`.
|
|
3
|
+
A Django application groups its models. When a root is known, the group is
|
|
4
|
+
an aggregate; otherwise it remains a model-group with no asserted boundary.
|
|
5
|
+
Frozen dataclasses in `values.py` are its value objects. Ambiguity never hides
|
|
6
|
+
the concrete models or requires configuration to browse them.
|
|
8
7
|
"""
|
|
9
8
|
|
|
10
9
|
from __future__ import annotations
|
|
11
10
|
|
|
12
11
|
import ast
|
|
12
|
+
import json
|
|
13
13
|
from dataclasses import dataclass, field as dc_field
|
|
14
14
|
from typing import Dict, List, Optional
|
|
15
15
|
|
|
@@ -90,7 +90,7 @@ class Aggregate:
|
|
|
90
90
|
"""One application, and the catalog object being built from it."""
|
|
91
91
|
|
|
92
92
|
app: App
|
|
93
|
-
root: ModelDef
|
|
93
|
+
root: Optional[ModelDef]
|
|
94
94
|
models: List[ModelDef]
|
|
95
95
|
aggregate: Dict[str, object]
|
|
96
96
|
|
|
@@ -257,7 +257,7 @@ def root_of(app: App, models: List[ModelDef], named: Dict[str, str], b) -> Optio
|
|
|
257
257
|
for m in concrete:
|
|
258
258
|
if m.name == want:
|
|
259
259
|
return m
|
|
260
|
-
b.warn(app.rel, "aggregates names %s for %s, and no model there is called that" % (want, app.label))
|
|
260
|
+
b.warn(app.rel, "aggregates names %s for %s, and no model there is called that%s" % (want, app.label, aggregate_candidates(app, concrete)))
|
|
261
261
|
return None
|
|
262
262
|
for candidate in (pascal(singular(app.label)), pascal(app.label)):
|
|
263
263
|
for m in concrete:
|
|
@@ -268,13 +268,20 @@ def root_of(app: App, models: List[ModelDef], named: Dict[str, str], b) -> Optio
|
|
|
268
268
|
if not concrete:
|
|
269
269
|
b.warn(app.rel, "no models in this application: nothing here to be an aggregate")
|
|
270
270
|
return None
|
|
271
|
-
b.warn(
|
|
272
|
-
app.rel,
|
|
273
|
-
"no model called %s, and %d models to choose from: name the root in the aggregates option" % (pascal(singular(app.label)), len(concrete)),
|
|
274
|
-
)
|
|
275
271
|
return None
|
|
276
272
|
|
|
277
273
|
|
|
274
|
+
def aggregate_candidates(app: App, models: List[ModelDef]) -> str:
|
|
275
|
+
"""Keep candidate evidence in the warning-only plugin protocol."""
|
|
276
|
+
return "; aggregate candidates: " + json.dumps({
|
|
277
|
+
"app": app.dotted,
|
|
278
|
+
"models": [
|
|
279
|
+
{"name": model.name, "path": model.module.rel, "line": model.node.lineno}
|
|
280
|
+
for model in sorted(models, key=lambda model: (model.name, model.module.rel))
|
|
281
|
+
],
|
|
282
|
+
}, ensure_ascii=True)
|
|
283
|
+
|
|
284
|
+
|
|
278
285
|
def read_aggregates(
|
|
279
286
|
project: Project,
|
|
280
287
|
applications: List[App],
|
|
@@ -289,14 +296,18 @@ def read_aggregates(
|
|
|
289
296
|
if models is None:
|
|
290
297
|
models = read_models(app)
|
|
291
298
|
root = root_of(app, models, named, b)
|
|
292
|
-
if root is None:
|
|
293
|
-
continue
|
|
294
299
|
concrete = [m for m in models if m.concrete]
|
|
295
|
-
|
|
296
|
-
|
|
300
|
+
if not concrete:
|
|
301
|
+
continue
|
|
302
|
+
ordered = [root] + [m for m in concrete if m is not root] if root else concrete
|
|
303
|
+
# An application is a source grouping, not proof of a transactional
|
|
304
|
+
# boundary. Keep every model visible without inventing such a boundary.
|
|
305
|
+
agg_slug = slug(root.name) if root else "models-" + slug(app.dotted.replace(".", "-"))
|
|
297
306
|
agg_id = aggregate_id(svc_id, agg_slug)
|
|
298
|
-
readme = app.readme or (ast.get_docstring(root.node, clean=True) or "").strip()
|
|
299
|
-
obj = catalog.aggregate(agg_id, agg_slug, root.name, readme, root.name)
|
|
307
|
+
readme = app.readme or ((ast.get_docstring(root.node, clean=True) or "").strip() if root else "")
|
|
308
|
+
obj = catalog.aggregate(agg_id, agg_slug, root.name if root else app.dotted, readme, root.name if root else "")
|
|
309
|
+
if root is None:
|
|
310
|
+
obj["kind"] = "model-group"
|
|
300
311
|
for model in ordered:
|
|
301
312
|
obj["entities"].append(
|
|
302
313
|
catalog.block(
|