@shortlink-org/portolan 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +387 -0
- package/catalog/enum_test.go +46 -0
- package/catalog/model.go +932 -0
- package/catalog/roundtrip_test.go +185 -0
- package/catalog/via_test.go +38 -0
- package/cli/portolan.mjs +368 -0
- package/go.mod +14 -0
- package/go.sum +14 -0
- package/index.html +13 -0
- package/internal/commands/cargo.go +154 -0
- package/internal/commands/commands.go +143 -0
- package/internal/commands/commands_test.go +323 -0
- package/internal/commands/gradle.go +77 -0
- package/internal/commands/justfile.go +82 -0
- package/internal/commands/makefile.go +106 -0
- package/internal/commands/maven.go +93 -0
- package/internal/commands/packagejson.go +159 -0
- package/internal/commands/pyproject.go +169 -0
- package/internal/commands/taskfile.go +101 -0
- package/internal/commands/testdata/estate/.cargo/config.toml +7 -0
- package/internal/commands/testdata/estate/Makefile +33 -0
- package/internal/commands/testdata/estate/Taskfile.yml +21 -0
- package/internal/commands/testdata/estate/build.gradle.kts +21 -0
- package/internal/commands/testdata/estate/justfile +20 -0
- package/internal/commands/testdata/estate/package.json +13 -0
- package/internal/commands/testdata/estate/pom.xml +29 -0
- package/internal/commands/testdata/estate/pyproject.toml +27 -0
- package/internal/commands/testdata/estate/xtask/src/main.rs +27 -0
- package/internal/commands/testdata/golden/commands.json +305 -0
- package/internal/gohttp/analyze.go +2500 -0
- package/internal/gohttp/endpoints.go +1067 -0
- package/internal/gohttp/roots.go +320 -0
- package/internal/gohttp/typed.go +143 -0
- package/internal/goscan/constants.go +85 -0
- package/internal/goscan/goscan_test.go +227 -0
- package/internal/goscan/names.go +52 -0
- package/internal/goscan/parse_test.go +11 -0
- package/internal/goscan/source.go +36 -0
- package/internal/goscan/tree.go +155 -0
- package/internal/goscan/types.go +99 -0
- package/internal/wsdl/ids.go +127 -0
- package/internal/wsdl/ids_test.go +21 -0
- package/internal/wsdl/model.go +62 -0
- package/internal/wsdl/parse.go +920 -0
- package/internal/wsdl/parse_test.go +133 -0
- package/package.json +117 -0
- package/plugin/describe.go +107 -0
- package/plugin/describe_test.go +114 -0
- package/plugin/protocol.go +117 -0
- package/plugin/schematest/schematest.go +126 -0
- package/plugins/README.md +1160 -0
- package/plugins/extract-adr/describe.go +19 -0
- package/plugins/extract-adr/describe_test.go +11 -0
- package/plugins/extract-adr/extract.go +153 -0
- package/plugins/extract-adr/extract_test.go +350 -0
- package/plugins/extract-adr/history.go +99 -0
- package/plugins/extract-adr/main.go +65 -0
- package/plugins/extract-adr/options.schema.json +26 -0
- package/plugins/extract-adr/parse.go +642 -0
- package/plugins/extract-adr/parse_test.go +405 -0
- package/plugins/extract-asyncapi/describe.go +19 -0
- package/plugins/extract-asyncapi/describe_test.go +11 -0
- package/plugins/extract-asyncapi/extract.go +315 -0
- package/plugins/extract-asyncapi/extract_test.go +197 -0
- package/plugins/extract-asyncapi/main.go +48 -0
- package/plugins/extract-asyncapi/options.schema.json +26 -0
- package/plugins/extract-asyncapi/spec.go +151 -0
- package/plugins/extract-celery/README.md +169 -0
- package/plugins/extract-celery/extract.py +346 -0
- package/plugins/extract-celery/extract_test.py +192 -0
- package/plugins/extract-celery/main.py +66 -0
- package/plugins/extract-celery/options.py +31 -0
- package/plugins/extract-celery/options.schema.json +31 -0
- package/plugins/extract-celery/producers.py +156 -0
- package/plugins/extract-celery/routes_test.py +106 -0
- package/plugins/extract-commands/describe.go +19 -0
- package/plugins/extract-commands/describe_test.go +11 -0
- package/plugins/extract-commands/extract.go +79 -0
- package/plugins/extract-commands/extract_test.go +92 -0
- package/plugins/extract-commands/main.go +55 -0
- package/plugins/extract-commands/options.schema.json +20 -0
- package/plugins/extract-csr/avro.go +258 -0
- package/plugins/extract-csr/describe.go +19 -0
- package/plugins/extract-csr/describe_test.go +11 -0
- package/plugins/extract-csr/extract.go +338 -0
- package/plugins/extract-csr/extract_test.go +374 -0
- package/plugins/extract-csr/jsonschema.go +343 -0
- package/plugins/extract-csr/lock.go +25 -0
- package/plugins/extract-csr/main.go +84 -0
- package/plugins/extract-csr/options.schema.json +43 -0
- package/plugins/extract-csr/subject.go +102 -0
- package/plugins/extract-django/README.md +386 -0
- package/plugins/extract-django/apps.py +100 -0
- package/plugins/extract-django/auth.py +302 -0
- package/plugins/extract-django/auth_test.py +188 -0
- package/plugins/extract-django/clients.py +121 -0
- package/plugins/extract-django/contracts.py +458 -0
- package/plugins/extract-django/database.py +119 -0
- package/plugins/extract-django/domain.py +318 -0
- package/plugins/extract-django/events.py +86 -0
- package/plugins/extract-django/extract.py +601 -0
- package/plugins/extract-django/extract_test.py +397 -0
- package/plugins/extract-django/flows.py +1278 -0
- package/plugins/extract-django/flows_test.py +338 -0
- package/plugins/extract-django/ids.py +42 -0
- package/plugins/extract-django/ids_test.py +30 -0
- package/plugins/extract-django/lifecycle.py +222 -0
- package/plugins/extract-django/lifecycle_test.py +76 -0
- package/plugins/extract-django/main.py +66 -0
- package/plugins/extract-django/openapi.py +229 -0
- package/plugins/extract-django/openapi_test.py +60 -0
- package/plugins/extract-django/operations.py +96 -0
- package/plugins/extract-django/options.py +61 -0
- package/plugins/extract-django/options.schema.json +126 -0
- package/plugins/extract-django/routing.py +214 -0
- package/plugins/extract-django/routing_test.py +198 -0
- package/plugins/extract-django/serializers.py +558 -0
- package/plugins/extract-django/store.py +434 -0
- package/plugins/extract-django/transport.py +242 -0
- package/plugins/extract-flows/describe.go +19 -0
- package/plugins/extract-flows/describe_test.go +11 -0
- package/plugins/extract-flows/extract.go +91 -0
- package/plugins/extract-flows/main.go +45 -0
- package/plugins/extract-flows/options.schema.json +16 -0
- package/plugins/extract-flows/parse.go +593 -0
- package/plugins/extract-flows/parse_test.go +204 -0
- package/plugins/extract-glossary/describe.go +19 -0
- package/plugins/extract-glossary/describe_test.go +11 -0
- package/plugins/extract-glossary/extract.go +115 -0
- package/plugins/extract-glossary/extract_test.go +220 -0
- package/plugins/extract-glossary/main.go +59 -0
- package/plugins/extract-glossary/options.schema.json +21 -0
- package/plugins/extract-glossary/parse.go +214 -0
- package/plugins/extract-glossary/parse_test.go +203 -0
- package/plugins/extract-go/aggregate.go +214 -0
- package/plugins/extract-go/client.go +409 -0
- package/plugins/extract-go/client_test.go +266 -0
- package/plugins/extract-go/describe.go +19 -0
- package/plugins/extract-go/describe_test.go +11 -0
- package/plugins/extract-go/enum.go +195 -0
- package/plugins/extract-go/enum_test.go +82 -0
- package/plugins/extract-go/event.go +99 -0
- package/plugins/extract-go/extract.go +191 -0
- package/plugins/extract-go/extract_test.go +261 -0
- package/plugins/extract-go/flow.go +1441 -0
- package/plugins/extract-go/flow_test.go +609 -0
- package/plugins/extract-go/httpclient.go +174 -0
- package/plugins/extract-go/httpclient_test.go +296 -0
- package/plugins/extract-go/ids.go +92 -0
- package/plugins/extract-go/layout.go +225 -0
- package/plugins/extract-go/layout_test.go +125 -0
- package/plugins/extract-go/lifecycle.go +324 -0
- package/plugins/extract-go/lifecycle_test.go +108 -0
- package/plugins/extract-go/main.go +79 -0
- package/plugins/extract-go/operation.go +157 -0
- package/plugins/extract-go/options.schema.json +66 -0
- package/plugins/extract-go/source.go +316 -0
- package/plugins/extract-go/transport.go +321 -0
- package/plugins/extract-go/transport_test.go +145 -0
- package/plugins/extract-go/wiring.go +434 -0
- package/plugins/extract-go-nats/describe.go +19 -0
- package/plugins/extract-go-nats/describe_test.go +11 -0
- package/plugins/extract-go-nats/extract.go +177 -0
- package/plugins/extract-go-nats/extract_test.go +389 -0
- package/plugins/extract-go-nats/index.go +379 -0
- package/plugins/extract-go-nats/main.go +42 -0
- package/plugins/extract-go-nats/options.schema.json +21 -0
- package/plugins/extract-go-nats/resolve.go +161 -0
- package/plugins/extract-go-nats/sites.go +224 -0
- package/plugins/extract-graphql/describe.go +19 -0
- package/plugins/extract-graphql/describe_test.go +11 -0
- package/plugins/extract-graphql/extract.go +433 -0
- package/plugins/extract-graphql/extract_test.go +256 -0
- package/plugins/extract-graphql/ids.go +49 -0
- package/plugins/extract-graphql/lex.go +237 -0
- package/plugins/extract-graphql/main.go +51 -0
- package/plugins/extract-graphql/options.schema.json +31 -0
- package/plugins/extract-graphql/parse.go +621 -0
- package/plugins/extract-graphql/parse_test.go +122 -0
- package/plugins/extract-http-clients/describe.go +19 -0
- package/plugins/extract-http-clients/describe_test.go +11 -0
- package/plugins/extract-http-clients/extract.go +705 -0
- package/plugins/extract-http-clients/extract_test.go +1263 -0
- package/plugins/extract-http-clients/main.go +42 -0
- package/plugins/extract-http-clients/options.schema.json +33 -0
- package/plugins/extract-java/README.md +190 -0
- package/plugins/extract-java/build/org/portolan/extract/Catalog.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Clients$1.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Clients$2.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Clients$Call.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Clients$Client.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Clients$Route.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Clients.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Domain$Aggregate.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Domain.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Events$Found.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Events.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Flows$1.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Flows$Binding.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Flows$Draft.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Flows$Frame.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Flows$Options.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Flows.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Ids.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Json$Reader.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Json.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Lifecycle$1.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Lifecycle$Move.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Lifecycle.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Main.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/OpenApi$Operation.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/OpenApi$Spec.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/OpenApi.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Operations$1.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Operations$UseCase.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Operations.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Proto$Rpc.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Proto$Service.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Proto.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Source$Project.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Source$Unit.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Source.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Transport$Endpoint.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Transport.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Yaml$Line.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Yaml$Reader.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Yaml.class +0 -0
- package/plugins/extract-java/options.schema.json +70 -0
- package/plugins/extract-openapi/describe.go +19 -0
- package/plugins/extract-openapi/describe_test.go +11 -0
- package/plugins/extract-openapi/discover.go +243 -0
- package/plugins/extract-openapi/extract.go +526 -0
- package/plugins/extract-openapi/extract_test.go +545 -0
- package/plugins/extract-openapi/main.go +75 -0
- package/plugins/extract-openapi/options.schema.json +62 -0
- package/plugins/extract-openapi/spec.go +350 -0
- package/plugins/extract-project/README.md +22 -0
- package/plugins/extract-project/describe.go +19 -0
- package/plugins/extract-project/describe_test.go +11 -0
- package/plugins/extract-project/extract.go +221 -0
- package/plugins/extract-project/extract_test.go +109 -0
- package/plugins/extract-project/main.go +41 -0
- package/plugins/extract-project/options.schema.json +48 -0
- package/plugins/extract-proto/ast.go +125 -0
- package/plugins/extract-proto/consumes.go +77 -0
- package/plugins/extract-proto/describe.go +19 -0
- package/plugins/extract-proto/extract.go +293 -0
- package/plugins/extract-proto/extract_test.go +459 -0
- package/plugins/extract-proto/ids.go +89 -0
- package/plugins/extract-proto/ids_test.go +57 -0
- package/plugins/extract-proto/lex.go +285 -0
- package/plugins/extract-proto/main.go +100 -0
- package/plugins/extract-proto/module.go +120 -0
- package/plugins/extract-proto/options.schema.json +51 -0
- package/plugins/extract-proto/parse.go +720 -0
- package/plugins/extract-proto/parse_test.go +307 -0
- package/plugins/extract-proto/provides.go +236 -0
- package/plugins/extract-proto/resolve.go +222 -0
- package/plugins/extract-proto/resolve_test.go +119 -0
- package/plugins/extract-python-kafka/README.md +36 -0
- package/plugins/extract-python-kafka/extract.py +211 -0
- package/plugins/extract-python-kafka/extract_test.py +68 -0
- package/plugins/extract-python-kafka/main.py +59 -0
- package/plugins/extract-python-kafka/options.py +29 -0
- package/plugins/extract-python-kafka/options.schema.json +31 -0
- package/plugins/extract-redis/README.md +11 -0
- package/plugins/extract-redis/describe.go +19 -0
- package/plugins/extract-redis/describe_test.go +11 -0
- package/plugins/extract-redis/extract.go +183 -0
- package/plugins/extract-redis/extract_test.go +168 -0
- package/plugins/extract-redis/keyspaces.go +469 -0
- package/plugins/extract-redis/main.go +44 -0
- package/plugins/extract-redis/options.schema.json +30 -0
- package/plugins/extract-river/describe.go +19 -0
- package/plugins/extract-river/describe_test.go +11 -0
- package/plugins/extract-river/extract.go +507 -0
- package/plugins/extract-river/extract_test.go +132 -0
- package/plugins/extract-river/main.go +42 -0
- package/plugins/extract-river/options.schema.json +21 -0
- package/plugins/extract-rust/Cargo.lock +133 -0
- package/plugins/extract-rust/Cargo.toml +23 -0
- package/plugins/extract-rust/README.md +204 -0
- package/plugins/extract-rust/options.schema.json +65 -0
- package/plugins/extract-rust/rustfmt.toml +1 -0
- package/plugins/extract-sql/ddl.go +893 -0
- package/plugins/extract-sql/ddl_test.go +401 -0
- package/plugins/extract-sql/describe.go +19 -0
- package/plugins/extract-sql/describe_test.go +11 -0
- package/plugins/extract-sql/layout.go +221 -0
- package/plugins/extract-sql/layout_test.go +55 -0
- package/plugins/extract-sql/lineage.go +107 -0
- package/plugins/extract-sql/main.go +141 -0
- package/plugins/extract-sql/maps.go +564 -0
- package/plugins/extract-sql/maps_java.go +117 -0
- package/plugins/extract-sql/maps_rust.go +333 -0
- package/plugins/extract-sql/maps_rust_test.go +70 -0
- package/plugins/extract-sql/maps_test.go +204 -0
- package/plugins/extract-sql/maps_ts.go +398 -0
- package/plugins/extract-sql/maps_ts_test.go +136 -0
- package/plugins/extract-sql/options.schema.json +53 -0
- package/plugins/extract-sql/projection.go +70 -0
- package/plugins/extract-sql/projection_test.go +50 -0
- package/plugins/extract-sql/store.go +420 -0
- package/plugins/extract-sql/store_test.go +233 -0
- package/plugins/extract-sql/view.go +295 -0
- package/plugins/extract-ts/README.md +306 -0
- package/plugins/extract-ts/ast.test.ts +178 -0
- package/plugins/extract-ts/ast.ts +648 -0
- package/plugins/extract-ts/clients.ts +185 -0
- package/plugins/extract-ts/domain.ts +164 -0
- package/plugins/extract-ts/extract.test.ts +208 -0
- package/plugins/extract-ts/extract.ts +200 -0
- package/plugins/extract-ts/flows.ts +783 -0
- package/plugins/extract-ts/graphql.test.ts +73 -0
- package/plugins/extract-ts/graphql.ts +181 -0
- package/plugins/extract-ts/ids.test.ts +21 -0
- package/plugins/extract-ts/ids.ts +52 -0
- package/plugins/extract-ts/lifecycle.ts +138 -0
- package/plugins/extract-ts/main.ts +48 -0
- package/plugins/extract-ts/openapi.test.ts +64 -0
- package/plugins/extract-ts/openapi.ts +99 -0
- package/plugins/extract-ts/operations.ts +93 -0
- package/plugins/extract-ts/options.schema.json +96 -0
- package/plugins/extract-ts/source.test.ts +174 -0
- package/plugins/extract-ts/source.ts +465 -0
- package/plugins/extract-ts/transport.ts +147 -0
- package/plugins/extract-ts/tsconfig.json +13 -0
- package/plugins/extract-ts/wiring.ts +190 -0
- package/plugins/extract-watermill/describe.go +19 -0
- package/plugins/extract-watermill/describe_test.go +11 -0
- package/plugins/extract-watermill/extract.go +1228 -0
- package/plugins/extract-watermill/extract_test.go +234 -0
- package/plugins/extract-watermill/main.go +41 -0
- package/plugins/extract-watermill/options.schema.json +21 -0
- package/plugins/extract-wsdl/describe.go +19 -0
- package/plugins/extract-wsdl/describe_test.go +11 -0
- package/plugins/extract-wsdl/extract.go +148 -0
- package/plugins/extract-wsdl/extract_test.go +45 -0
- package/plugins/extract-wsdl/main.go +53 -0
- package/plugins/extract-wsdl/options.schema.json +59 -0
- package/plugins/fetch-bsr/auth.go +114 -0
- package/plugins/fetch-bsr/bsr.go +240 -0
- package/plugins/fetch-bsr/cache.go +66 -0
- package/plugins/fetch-bsr/describe.go +19 -0
- package/plugins/fetch-bsr/fetch.go +192 -0
- package/plugins/fetch-bsr/fetch_test.go +465 -0
- package/plugins/fetch-bsr/lock.go +71 -0
- package/plugins/fetch-bsr/main.go +117 -0
- package/plugins/fetch-bsr/options.schema.json +42 -0
- package/plugins/fetch-csr/auth.go +95 -0
- package/plugins/fetch-csr/cache.go +69 -0
- package/plugins/fetch-csr/describe.go +19 -0
- package/plugins/fetch-csr/describe_test.go +11 -0
- package/plugins/fetch-csr/fetch.go +254 -0
- package/plugins/fetch-csr/fetch_test.go +494 -0
- package/plugins/fetch-csr/lock.go +95 -0
- package/plugins/fetch-csr/main.go +122 -0
- package/plugins/fetch-csr/options.schema.json +42 -0
- package/plugins/fetch-csr/registry.go +209 -0
- package/plugins/fetch-git/cache.go +65 -0
- package/plugins/fetch-git/describe.go +19 -0
- package/plugins/fetch-git/describe_test.go +11 -0
- package/plugins/fetch-git/fetch.go +181 -0
- package/plugins/fetch-git/fetch_test.go +332 -0
- package/plugins/fetch-git/git.go +169 -0
- package/plugins/fetch-git/lock.go +72 -0
- package/plugins/fetch-git/main.go +127 -0
- package/plugins/fetch-git/offline.go +39 -0
- package/plugins/fetch-git/options.schema.json +42 -0
- package/plugins/fetch-git/pin.go +90 -0
- package/plugins/fetch-git/pin_test.go +74 -0
- package/plugins/gen-backstage/describe.go +17 -0
- package/plugins/gen-backstage/main.go +22 -0
- package/plugins/gen-backstage/options.schema.json +30 -0
- package/plugins/gen-backstage/plugin.go +473 -0
- package/plugins/gen-backstage/plugin_test.go +145 -0
- package/plugins/gen-backstage.wasm +0 -0
- package/plugins/gen-markdown/adr.go +162 -0
- package/plugins/gen-markdown/aggregate.go +362 -0
- package/plugins/gen-markdown/canonical.go +93 -0
- package/plugins/gen-markdown/context.go +90 -0
- package/plugins/gen-markdown/coverage_test.go +89 -0
- package/plugins/gen-markdown/describe.go +19 -0
- package/plugins/gen-markdown/describe_test.go +11 -0
- package/plugins/gen-markdown/external.go +71 -0
- package/plugins/gen-markdown/flow.go +277 -0
- package/plugins/gen-markdown/glossary.go +76 -0
- package/plugins/gen-markdown/glossary_test.go +148 -0
- package/plugins/gen-markdown/llms.go +302 -0
- package/plugins/gen-markdown/main.go +22 -0
- package/plugins/gen-markdown/markdown.go +254 -0
- package/plugins/gen-markdown/markdown_test.go +100 -0
- package/plugins/gen-markdown/module.go +107 -0
- package/plugins/gen-markdown/options.schema.json +16 -0
- package/plugins/gen-markdown/plugin.go +39 -0
- package/plugins/gen-markdown/quality_test.go +176 -0
- package/plugins/gen-markdown/redis_test.go +38 -0
- package/plugins/gen-markdown/render.go +344 -0
- package/plugins/gen-markdown/render_test.go +186 -0
- package/plugins/gen-markdown/service.go +456 -0
- package/plugins/gen-markdown/source.go +177 -0
- package/plugins/gen-markdown/store.go +201 -0
- package/plugins/gen-markdown.wasm +0 -0
- package/plugins/gen-mermaid/describe.go +17 -0
- package/plugins/gen-mermaid/main.go +22 -0
- package/plugins/gen-mermaid/options.schema.json +10 -0
- package/plugins/gen-mermaid/plugin.go +104 -0
- package/plugins/gen-mermaid/plugin_test.go +33 -0
- package/plugins/gen-mermaid.wasm +0 -0
- package/plugins/openapi/ids.go +261 -0
- package/plugins/openapi/ids_test.go +98 -0
- package/plugins/pyplugin/catalog.py +209 -0
- package/plugins/pyplugin/celery_conf.py +436 -0
- package/plugins/pyplugin/celery_tasks.py +76 -0
- package/plugins/pyplugin/kafka.py +570 -0
- package/plugins/pyplugin/names.py +38 -0
- package/plugins/pyplugin/protocol.py +79 -0
- package/plugins/pyplugin/source.py +285 -0
- package/plugins/verify-codeowners/describe.go +19 -0
- package/plugins/verify-codeowners/describe_test.go +11 -0
- package/plugins/verify-codeowners/main.go +75 -0
- package/plugins/verify-codeowners/match.go +85 -0
- package/plugins/verify-codeowners/match_test.go +47 -0
- package/plugins/verify-codeowners/options.schema.json +15 -0
- package/plugins/verify-codeowners/owners.go +164 -0
- package/plugins/verify-codeowners/owners_test.go +225 -0
- package/plugins/verify-codeowners/parse.go +90 -0
- package/plugins/verify-codeowners/parse_test.go +62 -0
- package/plugins/verify-otel/describe.go +19 -0
- package/plugins/verify-otel/describe_test.go +11 -0
- package/plugins/verify-otel/main.go +53 -0
- package/plugins/verify-otel/match.go +336 -0
- package/plugins/verify-otel/options.schema.json +30 -0
- package/plugins/verify-otel/otlp.go +200 -0
- package/plugins/verify-otel/verify.go +734 -0
- package/plugins/verify-otel/verify_test.go +460 -0
- package/portolan.json +1073 -0
- package/public/favicon.svg +12 -0
- package/schema/portolan.schema.json +2558 -0
- package/scripts/README.md +53 -0
- package/scripts/build-report.mjs +60 -0
- package/scripts/build-report.test.mjs +44 -0
- package/scripts/builtin-plugins.mjs +49 -0
- package/scripts/catalog-sources.mjs +67 -0
- package/scripts/diff.mjs +315 -0
- package/scripts/diff.test.mjs +151 -0
- package/scripts/forge-comment.mjs +144 -0
- package/scripts/forge-comment.test.mjs +151 -0
- package/scripts/forge-release.mjs +138 -0
- package/scripts/forge-release.test.mjs +106 -0
- package/scripts/forge.mjs +61 -0
- package/scripts/gen-likec4.mjs +854 -0
- package/scripts/gen-likec4.test.mjs +258 -0
- package/scripts/gen.mjs +603 -0
- package/scripts/generated-docs.test.mjs +66 -0
- package/scripts/local-api.mjs +1192 -0
- package/scripts/local-api.test.mjs +387 -0
- package/scripts/manifest.mjs +214 -0
- package/scripts/manifest.test.mjs +148 -0
- package/scripts/output-path.mjs +79 -0
- package/scripts/output-path.test.mjs +45 -0
- package/scripts/package-smoke.mjs +37 -0
- package/scripts/plugin-host.mjs +419 -0
- package/scripts/plugin-host.test.mjs +109 -0
- package/scripts/plugin-wasm-worker.mjs +34 -0
- package/scripts/run-builtin.mjs +81 -0
- package/scripts/schema.mjs +357 -0
- package/scripts/site-docs.mjs +295 -0
- package/scripts/site-docs.test.mjs +181 -0
- package/scripts/vendor-lock.mjs +58 -0
- package/scripts/vendor-lock.test.mjs +69 -0
- package/src/README.md +38 -0
- package/src/app/App.tsx +448 -0
- package/src/app/BranchPicker.test.ts +46 -0
- package/src/app/BranchPicker.tsx +153 -0
- package/src/app/Breadcrumbs.test.ts +38 -0
- package/src/app/Breadcrumbs.tsx +223 -0
- package/src/app/BuildStamp.tsx +146 -0
- package/src/app/CommandPalette.tsx +292 -0
- package/src/app/HashScroll.tsx +45 -0
- package/src/app/Sidebar.tsx +1742 -0
- package/src/app/TopBar.tsx +393 -0
- package/src/app/density.tsx +64 -0
- package/src/app/forge-access.test.ts +27 -0
- package/src/app/forge-access.tsx +80 -0
- package/src/app/panels.tsx +141 -0
- package/src/app/pins.tsx +242 -0
- package/src/app/query-client.ts +24 -0
- package/src/app/responsive.ts +55 -0
- package/src/app/shortcuts.test.ts +39 -0
- package/src/app/shortcuts.tsx +282 -0
- package/src/app/theme.tsx +51 -0
- package/src/app/title.test.ts +14 -0
- package/src/app/title.ts +30 -0
- package/src/app/toast.tsx +59 -0
- package/src/app/tree-keys.test.ts +80 -0
- package/src/app/tree-keys.ts +87 -0
- package/src/app/ui-store.ts +52 -0
- package/src/catalog-error.test.ts +105 -0
- package/src/catalog-profile.test.ts +59 -0
- package/src/catalog-profile.ts +101 -0
- package/src/catalog-stores.test.ts +431 -0
- package/src/catalog.test.ts +1215 -0
- package/src/catalog.ts +3125 -0
- package/src/chat/Cards.tsx +408 -0
- package/src/chat/ChatPanel.tsx +79 -0
- package/src/chat/ChatSettings.tsx +219 -0
- package/src/chat/Composer.tsx +99 -0
- package/src/chat/Conversation.tsx +281 -0
- package/src/chat/Header.tsx +46 -0
- package/src/chat/Message.tsx +49 -0
- package/src/chat/Notice.tsx +195 -0
- package/src/chat/Starter.tsx +67 -0
- package/src/chat/Steps.tsx +45 -0
- package/src/chat/Waiting.tsx +87 -0
- package/src/chat/flags.test.ts +59 -0
- package/src/chat/flags.ts +155 -0
- package/src/chat/linkify.test.ts +30 -0
- package/src/chat/linkify.ts +45 -0
- package/src/chat/prefs.ts +56 -0
- package/src/chat/prompt.test.ts +43 -0
- package/src/chat/prompt.ts +157 -0
- package/src/chat/store.ts +35 -0
- package/src/chat/tools.ts +58 -0
- package/src/chat/transport.ts +70 -0
- package/src/components/AdrRow.tsx +29 -0
- package/src/components/AnchorLink.tsx +67 -0
- package/src/components/ApiReference.tsx +134 -0
- package/src/components/AsyncApiReference.tsx +120 -0
- package/src/components/CatalogStamp.tsx +155 -0
- package/src/components/ChannelRows.tsx +132 -0
- package/src/components/CommandRows.tsx +139 -0
- package/src/components/DiagramSkeleton.tsx +71 -0
- package/src/components/EditorLink.tsx +85 -0
- package/src/components/FieldTree.tsx +305 -0
- package/src/components/FlowTrigger.tsx +34 -0
- package/src/components/Ident.tsx +89 -0
- package/src/components/Integrations.tsx +243 -0
- package/src/components/LifecycleDiagram.tsx +110 -0
- package/src/components/MachineDocs.tsx +42 -0
- package/src/components/Markdown.tsx +173 -0
- package/src/components/MarkdownTable.tsx +206 -0
- package/src/components/Mermaid.tsx +211 -0
- package/src/components/MethodRows.tsx +309 -0
- package/src/components/Overlay.tsx +125 -0
- package/src/components/PageHeader.tsx +189 -0
- package/src/components/RowActions.tsx +97 -0
- package/src/components/SchemaDocument.tsx +89 -0
- package/src/components/Select.tsx +126 -0
- package/src/components/ShapeRows.tsx +118 -0
- package/src/components/SourceDoc.tsx +82 -0
- package/src/components/SourcePreview.tsx +441 -0
- package/src/components/TabRow.tsx +182 -0
- package/src/components/TechIcon.tsx +34 -0
- package/src/components/Toc.tsx +132 -0
- package/src/components/WhatLinksHere.tsx +273 -0
- package/src/components/WsdlReference.tsx +84 -0
- package/src/components/ddd-icons.tsx +107 -0
- package/src/components/kind.tsx +157 -0
- package/src/components/logo.tsx +61 -0
- package/src/components/primitives.tsx +264 -0
- package/src/data.test.ts +18 -0
- package/src/data.ts +202 -0
- package/src/enrich.test.ts +1150 -0
- package/src/enrich.ts +672 -0
- package/src/er/ErCanvas.tsx +533 -0
- package/src/er/RedisSchema.tsx +74 -0
- package/src/er/RelationCard.tsx +267 -0
- package/src/er/StoreHeader.tsx +155 -0
- package/src/er/TableNode.tsx +84 -0
- package/src/er/ViewNode.tsx +83 -0
- package/src/er/layout.test.ts +115 -0
- package/src/er/layout.ts +77 -0
- package/src/er/lineage.test.ts +140 -0
- package/src/er/lineage.ts +107 -0
- package/src/er/markers.tsx +79 -0
- package/src/er/spec.test.ts +344 -0
- package/src/er/spec.ts +426 -0
- package/src/flow/ChainList.tsx +162 -0
- package/src/flow/FlowTable.tsx +123 -0
- package/src/flow/FlowToolbar.tsx +372 -0
- package/src/flow/StepDetail.tsx +380 -0
- package/src/flow/StepRail.tsx +428 -0
- package/src/flow/answers.test.ts +50 -0
- package/src/flow/answers.ts +58 -0
- package/src/flow/chain.test.ts +383 -0
- package/src/flow/chain.ts +353 -0
- package/src/flow/chapters.test.ts +185 -0
- package/src/flow/chapters.ts +225 -0
- package/src/flow/continues.test.ts +102 -0
- package/src/flow/continues.ts +65 -0
- package/src/flow/cross-context.test.ts +97 -0
- package/src/flow/cross-context.ts +35 -0
- package/src/flow/mermaid.test.ts +22 -0
- package/src/flow/mermaid.ts +141 -0
- package/src/flow/outline.test.ts +212 -0
- package/src/flow/outline.ts +170 -0
- package/src/flow/paths.test.ts +129 -0
- package/src/flow/paths.ts +147 -0
- package/src/flow/prefs.test.ts +80 -0
- package/src/flow/prefs.ts +104 -0
- package/src/graph/DependencyGraph.tsx +283 -0
- package/src/graph/DependencyNodes.tsx +162 -0
- package/src/graph/EdgeLabel.tsx +61 -0
- package/src/graph/FocusedEventGraph.tsx +211 -0
- package/src/graph/GraphToolbar.tsx +187 -0
- package/src/graph/RoutedEdge.tsx +80 -0
- package/src/graph/dependency-layout.test.ts +360 -0
- package/src/graph/dependency-layout.ts +444 -0
- package/src/graph/elk.test.ts +70 -0
- package/src/graph/elk.ts +287 -0
- package/src/graph/nodes.tsx +97 -0
- package/src/graph/theme.ts +81 -0
- package/src/graph/useElkFlow.ts +125 -0
- package/src/index.css +1547 -0
- package/src/language/InLanguage.tsx +40 -0
- package/src/language/TermCard.tsx +132 -0
- package/src/language/cards.test.ts +114 -0
- package/src/language/cards.ts +111 -0
- package/src/lib/admonition.test.ts +49 -0
- package/src/lib/admonition.ts +48 -0
- package/src/lib/adr.test.ts +131 -0
- package/src/lib/adr.ts +68 -0
- package/src/lib/api.test.ts +122 -0
- package/src/lib/api.ts +135 -0
- package/src/lib/backlinks.test.ts +330 -0
- package/src/lib/backlinks.ts +567 -0
- package/src/lib/branch-compare.ts +16 -0
- package/src/lib/build-info.test.ts +76 -0
- package/src/lib/build-info.ts +80 -0
- package/src/lib/catalog-diff.test.ts +275 -0
- package/src/lib/catalog-diff.ts +655 -0
- package/src/lib/clipboard.ts +43 -0
- package/src/lib/comparison-memory.test.ts +40 -0
- package/src/lib/comparison-memory.ts +39 -0
- package/src/lib/context-color.ts +27 -0
- package/src/lib/context-map.test.ts +148 -0
- package/src/lib/context-map.ts +399 -0
- package/src/lib/data-model.test.ts +245 -0
- package/src/lib/data-model.ts +399 -0
- package/src/lib/data-problems.test.ts +178 -0
- package/src/lib/data-problems.ts +314 -0
- package/src/lib/derive.test.ts +251 -0
- package/src/lib/derive.ts +515 -0
- package/src/lib/editor-link.test.ts +125 -0
- package/src/lib/editor-link.ts +96 -0
- package/src/lib/editor-prefs.ts +27 -0
- package/src/lib/event-graph.test.ts +237 -0
- package/src/lib/event-graph.ts +327 -0
- package/src/lib/export-canvas.ts +37 -0
- package/src/lib/export-file.test.ts +30 -0
- package/src/lib/export-file.ts +20 -0
- package/src/lib/flow-tree.test.ts +177 -0
- package/src/lib/flow-tree.ts +160 -0
- package/src/lib/forge-refs.test.ts +50 -0
- package/src/lib/forge-refs.ts +60 -0
- package/src/lib/format.test.ts +35 -0
- package/src/lib/format.ts +64 -0
- package/src/lib/github-catalog.test.ts +301 -0
- package/src/lib/github-catalog.ts +610 -0
- package/src/lib/integrations.test.ts +119 -0
- package/src/lib/integrations.ts +133 -0
- package/src/lib/kinds.test.ts +91 -0
- package/src/lib/kinds.ts +204 -0
- package/src/lib/lifecycle.test.ts +93 -0
- package/src/lib/lifecycle.ts +165 -0
- package/src/lib/local-api.ts +194 -0
- package/src/lib/motion-features.ts +11 -0
- package/src/lib/motion.test.ts +120 -0
- package/src/lib/motion.tsx +269 -0
- package/src/lib/palette-recent.test.ts +80 -0
- package/src/lib/palette-recent.ts +90 -0
- package/src/lib/palette.test.ts +365 -0
- package/src/lib/palette.ts +593 -0
- package/src/lib/pins.test.ts +139 -0
- package/src/lib/pins.ts +130 -0
- package/src/lib/problems.test.ts +112 -0
- package/src/lib/project-preview.ts +5 -0
- package/src/lib/proto-problems.test.ts +121 -0
- package/src/lib/proto-problems.ts +65 -0
- package/src/lib/queries.test.ts +113 -0
- package/src/lib/queries.ts +131 -0
- package/src/lib/registry.test.ts +219 -0
- package/src/lib/registry.ts +208 -0
- package/src/lib/scenarios.ts +547 -0
- package/src/lib/setup-info.test.ts +199 -0
- package/src/lib/setup-info.ts +415 -0
- package/src/lib/shape.test.ts +289 -0
- package/src/lib/shape.ts +454 -0
- package/src/lib/sidebar-prefs.test.ts +32 -0
- package/src/lib/sidebar-prefs.ts +75 -0
- package/src/lib/source-code.test.ts +159 -0
- package/src/lib/source-code.ts +286 -0
- package/src/lib/source-doc.test.ts +131 -0
- package/src/lib/source-doc.ts +103 -0
- package/src/lib/source-highlight.test.ts +29 -0
- package/src/lib/source-highlight.ts +24 -0
- package/src/lib/source-link.test.ts +153 -0
- package/src/lib/source-link.ts +201 -0
- package/src/lib/spec-files.test.ts +24 -0
- package/src/lib/spec-files.ts +23 -0
- package/src/lib/table-directive.test.ts +139 -0
- package/src/lib/table-directive.ts +106 -0
- package/src/lib/tech.test.ts +120 -0
- package/src/lib/tech.ts +421 -0
- package/src/lib/terms.test.ts +67 -0
- package/src/lib/terms.ts +179 -0
- package/src/lib/wire-problems.test.ts +263 -0
- package/src/lib/wire-problems.ts +281 -0
- package/src/likec4/C4View.tsx +55 -0
- package/src/likec4/CanvasBridge.tsx +187 -0
- package/src/likec4/FlowView.tsx +235 -0
- package/src/likec4/InteractiveView.tsx +144 -0
- package/src/likec4/canvas-viewport.test.ts +149 -0
- package/src/likec4/canvas-viewport.ts +135 -0
- package/src/likec4/flow-edges.test.ts +51 -0
- package/src/likec4/flow-edges.ts +89 -0
- package/src/likec4/frame-theme.test.ts +76 -0
- package/src/likec4/frame-theme.ts +101 -0
- package/src/likec4/highlight-css.test.ts +49 -0
- package/src/likec4/highlight-css.ts +52 -0
- package/src/likec4/ids.test.ts +151 -0
- package/src/likec4/ids.ts +115 -0
- package/src/likec4/levels.tsx +70 -0
- package/src/likec4/mapping.test.ts +66 -0
- package/src/likec4/mapping.ts +54 -0
- package/src/likec4/reserved.json +141 -0
- package/src/likec4/view-index.ts +78 -0
- package/src/main.tsx +13 -0
- package/src/map/ContextMapGraph.tsx +274 -0
- package/src/map/ContextNode.tsx +70 -0
- package/src/map/FloatingEdge.tsx +86 -0
- package/src/merge.test.ts +1038 -0
- package/src/merge.ts +1004 -0
- package/src/pages/AdrDetail.tsx +242 -0
- package/src/pages/AdrIndex.tsx +114 -0
- package/src/pages/AggregatePage.tsx +727 -0
- package/src/pages/BlockPage.tsx +478 -0
- package/src/pages/CatalogFailure.tsx +111 -0
- package/src/pages/Changes.tsx +313 -0
- package/src/pages/ContextMap.tsx +534 -0
- package/src/pages/ContextPage.tsx +346 -0
- package/src/pages/EnumPage.tsx +282 -0
- package/src/pages/EventPage.tsx +686 -0
- package/src/pages/ExternalPage.tsx +204 -0
- package/src/pages/FlowDetail.tsx +663 -0
- package/src/pages/FlowIndex.tsx +303 -0
- package/src/pages/GraphPage.tsx +217 -0
- package/src/pages/Language.tsx +257 -0
- package/src/pages/ModulePage.tsx +366 -0
- package/src/pages/NotFound.tsx +54 -0
- package/src/pages/Overview.tsx +344 -0
- package/src/pages/Problems.tsx +360 -0
- package/src/pages/RegistryIndex.tsx +118 -0
- package/src/pages/ServicePage.tsx +746 -0
- package/src/pages/Settings.tsx +975 -0
- package/src/pages/StorePage.tsx +114 -0
- package/src/peek/PeekCard.test.ts +35 -0
- package/src/peek/PeekCard.tsx +118 -0
- package/src/peek/PeekLayer.tsx +147 -0
- package/src/peek/model.test.ts +156 -0
- package/src/peek/model.ts +253 -0
- package/src/routes.test.ts +222 -0
- package/src/routes.ts +449 -0
- package/src/selection/DetailPanel.tsx +1407 -0
- package/src/selection/hash.test.ts +73 -0
- package/src/selection/hash.ts +58 -0
- package/src/selection/model.test.ts +199 -0
- package/src/selection/model.ts +473 -0
- package/src/selection/pages.test.ts +115 -0
- package/src/selection/pages.ts +268 -0
- package/src/selection/store.test.ts +62 -0
- package/src/selection/store.ts +46 -0
- package/src/selection/sync.tsx +99 -0
- package/src/table/ColumnsMenu.tsx +72 -0
- package/src/table/DataTable.tsx +780 -0
- package/src/table/Facets.tsx +54 -0
- package/src/table/GroupMenu.tsx +69 -0
- package/src/table/Toolbar.tsx +190 -0
- package/src/table/cells.tsx +117 -0
- package/src/table/compare.test.ts +117 -0
- package/src/table/compare.ts +196 -0
- package/src/table/export.test.ts +88 -0
- package/src/table/export.ts +66 -0
- package/src/table/facet-groups.test.ts +74 -0
- package/src/table/facet-groups.ts +56 -0
- package/src/table/grouping.test.ts +84 -0
- package/src/table/grouping.ts +82 -0
- package/src/table/infer.test.ts +57 -0
- package/src/table/infer.ts +50 -0
- package/src/table/persist.test.ts +76 -0
- package/src/table/persist.ts +112 -0
- package/src/table/sort-url.test.ts +75 -0
- package/src/table/sort-url.ts +52 -0
- package/src/table/types.ts +80 -0
- package/src/table/useTableState.ts +220 -0
- package/src/test-catalog.ts +7 -0
- package/src/testing/estate/data__catalog.json +2847 -0
- package/src/testing/estate/data__flows.json +1724 -0
- package/src/testing/estate/examples__auth__portolan__api.json +212 -0
- package/src/testing/estate/examples__auth__portolan__domain.json +1204 -0
- package/src/testing/estate/examples__auth__portolan__glossary.json +226 -0
- package/src/testing/estate/examples__auth__portolan__observed.json +757 -0
- package/src/testing/estate/examples__auth__portolan__stores.json +195 -0
- package/src/testing/estate/examples__shop__cart__portolan__api.json +234 -0
- package/src/testing/estate/examples__shop__cart__portolan__commands.json +86 -0
- package/src/testing/estate/examples__shop__cart__portolan__domain.json +966 -0
- package/src/testing/estate/examples__shop__cart__portolan__observed.json +498 -0
- package/src/testing/estate/examples__shop__cart__portolan__stores.json +208 -0
- package/src/testing/estate/examples__shop__oms__portolan__glossary.json +98 -0
- package/src/testing/estate/examples__shop__oms__portolan__proto.json +186 -0
- package/src/testing/estate.ts +49 -0
- package/src/trail/Trail.tsx +81 -0
- package/src/trail/model.test.ts +157 -0
- package/src/trail/model.ts +189 -0
- package/src/trail/record.tsx +32 -0
- package/src/trail/store.test.ts +66 -0
- package/src/trail/store.ts +102 -0
- package/tsconfig.json +35 -0
- package/vite.config.ts +171 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
package main
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"os"
|
|
5
|
+
"strings"
|
|
6
|
+
|
|
7
|
+
"gopkg.in/yaml.v3"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
// The document is read as a node tree rather than into structs, for the reason
|
|
11
|
+
// the OpenAPI extractor gives beside its own reader: a mapping unmarshalled
|
|
12
|
+
// into a Go map loses the order its author wrote, and a channel object holds
|
|
13
|
+
// keys this does not model, which walking the nodes ignores rather than fails
|
|
14
|
+
// on.
|
|
15
|
+
//
|
|
16
|
+
// Two versions have to be read here, and they disagree about a word. In 3.x an
|
|
17
|
+
// operation says `action: send` or `action: receive`, from the application's
|
|
18
|
+
// side, and that is the end of it. In 2.x a channel has `publish` and
|
|
19
|
+
// `subscribe`, and both are written from the *client's* side: `publish` is what
|
|
20
|
+
// somebody else publishes to the application, and `subscribe` is what the
|
|
21
|
+
// application produces for somebody else to subscribe to. Reading 2.x the
|
|
22
|
+
// obvious way gets every arrow backwards, which is the confusion 3.x was
|
|
23
|
+
// changed to end.
|
|
24
|
+
|
|
25
|
+
type document struct {
|
|
26
|
+
root *yaml.Node
|
|
27
|
+
path string
|
|
28
|
+
// major is the major version of the document: 2 or 3.
|
|
29
|
+
major int
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
func load(path string) (*document, error) {
|
|
33
|
+
contents, err := os.ReadFile(path)
|
|
34
|
+
if err != nil {
|
|
35
|
+
return nil, err
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
var root yaml.Node
|
|
39
|
+
if err := yaml.Unmarshal(contents, &root); err != nil {
|
|
40
|
+
return nil, err
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
node := &root
|
|
44
|
+
if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
|
|
45
|
+
node = node.Content[0]
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
doc := &document{root: node, path: path, major: 3}
|
|
49
|
+
if version := text(child(node, "asyncapi")); strings.HasPrefix(version, "2.") {
|
|
50
|
+
doc.major = 2
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return doc, nil
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// entry is one key/value pair of a mapping, in document order.
|
|
57
|
+
type entry struct {
|
|
58
|
+
key string
|
|
59
|
+
value *yaml.Node
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
func entries(node *yaml.Node) []entry {
|
|
63
|
+
if node == nil || node.Kind != yaml.MappingNode {
|
|
64
|
+
return nil
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
out := make([]entry, 0, len(node.Content)/2)
|
|
68
|
+
for i := 0; i+1 < len(node.Content); i += 2 {
|
|
69
|
+
out = append(out, entry{key: node.Content[i].Value, value: node.Content[i+1]})
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return out
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// child follows a path of mapping keys, and answers nil for anything missing.
|
|
76
|
+
func child(node *yaml.Node, keys ...string) *yaml.Node {
|
|
77
|
+
for _, key := range keys {
|
|
78
|
+
found := false
|
|
79
|
+
for _, e := range entries(node) {
|
|
80
|
+
if e.key == key {
|
|
81
|
+
node = e.value
|
|
82
|
+
found = true
|
|
83
|
+
|
|
84
|
+
break
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if !found {
|
|
88
|
+
return nil
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return node
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
func text(node *yaml.Node) string {
|
|
96
|
+
if node == nil {
|
|
97
|
+
return ""
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return strings.TrimSpace(node.Value)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// items reads a sequence, or the single node written where a sequence was
|
|
104
|
+
// allowed. A document that names one message where it could have named several
|
|
105
|
+
// is the common case, and both spellings mean the same thing.
|
|
106
|
+
func items(node *yaml.Node) []*yaml.Node {
|
|
107
|
+
if node == nil {
|
|
108
|
+
return nil
|
|
109
|
+
}
|
|
110
|
+
if node.Kind != yaml.SequenceNode {
|
|
111
|
+
return []*yaml.Node{node}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return node.Content
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// deref follows a local $ref, and answers the node itself when there is none.
|
|
118
|
+
//
|
|
119
|
+
// Only local refs are followed. A document that reaches into another file is
|
|
120
|
+
// describing something this extractor cannot see, and guessing at a name from
|
|
121
|
+
// the last segment of a path would be inventing one.
|
|
122
|
+
func (d *document) deref(node *yaml.Node) (*yaml.Node, string) {
|
|
123
|
+
ref := text(child(node, "$ref"))
|
|
124
|
+
if ref == "" {
|
|
125
|
+
return node, ""
|
|
126
|
+
}
|
|
127
|
+
if !strings.HasPrefix(ref, "#/") {
|
|
128
|
+
return nil, ref
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
var keys []string
|
|
132
|
+
for _, segment := range strings.Split(strings.TrimPrefix(ref, "#/"), "/") {
|
|
133
|
+
// JSON Pointer escapes, which a channel address containing a slash
|
|
134
|
+
// needs: `shop~1cart` is one key, not two.
|
|
135
|
+
segment = strings.ReplaceAll(segment, "~1", "/")
|
|
136
|
+
keys = append(keys, strings.ReplaceAll(segment, "~0", "~"))
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return child(d.root, keys...), ref
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// refKey is the last segment of a ref, which is the key the thing it points at
|
|
143
|
+
// is filed under.
|
|
144
|
+
func refKey(ref string) string {
|
|
145
|
+
at := strings.LastIndex(ref, "/")
|
|
146
|
+
if at < 0 {
|
|
147
|
+
return ref
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return strings.ReplaceAll(strings.ReplaceAll(ref[at+1:], "~1", "/"), "~0", "~")
|
|
151
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# extract-celery
|
|
2
|
+
|
|
3
|
+
A tree with Celery in it in, work queues and job flows out. The Python twin of
|
|
4
|
+
`extract-river`: it does not need aggregates, does not treat a task as a
|
|
5
|
+
domain event, and answers with a channel of kind `job` per queue and a two-hop
|
|
6
|
+
flow per task — the call that enqueues it, and the worker that runs it.
|
|
7
|
+
|
|
8
|
+
Written in Python and run as a process plugin, `python3 plugins/extract-celery/main.py`.
|
|
9
|
+
Python 3.9 or newer and nothing but the standard library; the protocol, the
|
|
10
|
+
tree reader and the fragment shapes are `plugins/pyplugin`, shared with
|
|
11
|
+
`extract-django` — and so are the task and configuration readers,
|
|
12
|
+
`celery_tasks.py` and `celery_conf.py`, which is how an enqueue inside a
|
|
13
|
+
Django flow lands on the same `celery-<queue>` participant this plugin's flow
|
|
14
|
+
draws. **The project is never imported**, and neither is Celery:
|
|
15
|
+
everything is read through `ast`, and names are resolved by import and by
|
|
16
|
+
file.
|
|
17
|
+
|
|
18
|
+
## What it reads
|
|
19
|
+
|
|
20
|
+
**A task** is a module-level function decorated `@shared_task`, `@app.task`
|
|
21
|
+
or `@<anything>.task`, with or without arguments. Its wire name is the
|
|
22
|
+
`name=` it was given, or what Celery composes when it was not: the module's
|
|
23
|
+
dotted path and the function's name, `invoices.tasks.send_invoice_email`. A
|
|
24
|
+
`queue=` on the decorator is where it goes unless a call says otherwise. The
|
|
25
|
+
doc is the docstring, first paragraph. A class-based task is not read.
|
|
26
|
+
|
|
27
|
+
**An enqueue** is `t.delay(...)`, `t.apply_async(...)`, or either through a
|
|
28
|
+
signature, `t.s(...).apply_async()`. A `chain`, `group` or `chord` holds
|
|
29
|
+
several signatures; each is recorded as an enqueue, noted with the canvas it
|
|
30
|
+
sits in, and the order the canvas imposes is not drawn. `app.send_task("name")`
|
|
31
|
+
enqueues by wire name, for a task that may live in another tree.
|
|
32
|
+
`transaction.on_commit(lambda: t.delay(...))` — and the `partial` form — is
|
|
33
|
+
the same enqueue with the note that it waits for the commit, which is the one
|
|
34
|
+
fact about *when* a message leaves that the code states plainly. A
|
|
35
|
+
`countdown=` or an `eta=` is noted too. Every module of the tree is searched:
|
|
36
|
+
services, views, receivers, other tasks.
|
|
37
|
+
|
|
38
|
+
**A schedule** is what the clock sets off: an entry of `beat_schedule` —
|
|
39
|
+
`CELERY_BEAT_SCHEDULE` in the settings, `app.conf.beat_schedule`, the legacy
|
|
40
|
+
`CELERYBEAT_SCHEDULE` — or a call of `add_periodic_task(schedule, task.s(),
|
|
41
|
+
name=...)`, wherever it is made, usually under `@app.on_after_configure.connect`.
|
|
42
|
+
The entry names the task on the wire, the call names it through its
|
|
43
|
+
signature, and `options={"queue": ...}` or a `queue=` on the call places it
|
|
44
|
+
before the task's own queue. The schedule object is read into a sentence:
|
|
45
|
+
`crontab(minute=0, hour="*/3")` is `cron 0 */3 * * *` in crontab's own field
|
|
46
|
+
order with `*` for what is not said, a number of seconds or a `timedelta` is
|
|
47
|
+
`every 6h`, `solar("sunset", …)` is `solar sunset`; anything else is kept as
|
|
48
|
+
written and said to be unreadable. Beat is a producer like any other, so a
|
|
49
|
+
task only the clock enqueues is not reported as one nothing enqueues. A
|
|
50
|
+
project with `django_celery_beat` installed keeps its schedule in the
|
|
51
|
+
database, which is said once and not read.
|
|
52
|
+
|
|
53
|
+
**The queue** is decided the way Celery decides it: the `queue=` at the call,
|
|
54
|
+
then the decorator's, then `task_routes` — an exact name before a glob, in the
|
|
55
|
+
order written — then `task_default_queue`, and past all of those Celery's own
|
|
56
|
+
`celery`. Each message says which rule placed it.
|
|
57
|
+
|
|
58
|
+
**The configuration** is read from two places, the settings first and the app
|
|
59
|
+
module over them, which is the order Celery applies them in. The app module
|
|
60
|
+
is wherever `X = Celery(...)` is assigned; its `config_from_object(...,
|
|
61
|
+
namespace="CELERY")` names the prefix the settings are read under, and its
|
|
62
|
+
`X.conf.update(...)` and `X.conf.key = ...` are read as written. The settings
|
|
63
|
+
module is what `manage.py` sets `DJANGO_SETTINGS_MODULE` to, or the `settings`
|
|
64
|
+
option. Three keys matter — `task_routes`, `task_default_queue`, `broker_url`
|
|
65
|
+
— and the pre-4.0 spellings of them are accepted. A value is read as far as
|
|
66
|
+
syntax carries it: a literal, the default of `os.environ.get("X", default)`,
|
|
67
|
+
a module-level constant followed once. A router function is not run, and is
|
|
68
|
+
said not to be.
|
|
69
|
+
|
|
70
|
+
## What comes out
|
|
71
|
+
|
|
72
|
+
One `channel` per queue, `kind: "job"`, `address` the queue, one message per
|
|
73
|
+
task on it: `send` when this tree enqueues it, `receive` when this tree
|
|
74
|
+
declares it. The doc of the `receive` says which rule placed it. The kind is
|
|
75
|
+
what lets many callers enqueue one task without the merge calling them rival
|
|
76
|
+
publishers, and what keeps a task out of the Problems page's search for a
|
|
77
|
+
domain event with the same wire name.
|
|
78
|
+
|
|
79
|
+
One `flow` per task that is both enqueued and declared here: `service →
|
|
80
|
+
broker : call enqueue <task>` at the first call in path order, with the
|
|
81
|
+
others named in the note, then `broker → service : call <task>` at the
|
|
82
|
+
function. The broker participant is `celery-<queue>`. Every step is
|
|
83
|
+
`declared`. Both steps carry the exact Celery queue and task wire name as a
|
|
84
|
+
job handoff. This lets a Django endpoint flow that calls `.delay()` or
|
|
85
|
+
`.apply_async()` continue at the corresponding worker step without repeating
|
|
86
|
+
the enqueue.
|
|
87
|
+
|
|
88
|
+
The worker step also carries the task function's source entrypoint. When the
|
|
89
|
+
Django extractor reads the same tree, enrichment appends the task body's
|
|
90
|
+
database, event and API effects after that receive step. This produces one
|
|
91
|
+
continuous synchronous-to-asynchronous scenario while keeping Celery as the
|
|
92
|
+
owner of queue and routing facts.
|
|
93
|
+
|
|
94
|
+
One more `flow` per task the clock sets off: `celery-beat → broker : call
|
|
95
|
+
enqueue <task>` at the entry, then the same worker step, with the trigger
|
|
96
|
+
`scheduled` and the schedule as its label. A task the code also enqueues
|
|
97
|
+
keeps that flow and gets this one beside it, told apart by `-beat` in the
|
|
98
|
+
slug — what happens when the clock fires and what happens when a request does
|
|
99
|
+
are two different questions, and a reader asks them separately.
|
|
100
|
+
|
|
101
|
+
## What it does not read
|
|
102
|
+
|
|
103
|
+
Named here rather than left to be discovered: **a schedule kept in the
|
|
104
|
+
database** (`django_celery_beat`), **task bodies when no Django extractor is
|
|
105
|
+
paired with this plugin**, **retries, rate
|
|
106
|
+
limits and time limits**, **the result backend**, **task
|
|
107
|
+
priority**, a **router function** in `task_routes`, a **regex** route, and a
|
|
108
|
+
**worker's `-Q`** — which queues a worker actually consumes is a fact about a
|
|
109
|
+
deployment, not the tree.
|
|
110
|
+
|
|
111
|
+
## Options
|
|
112
|
+
|
|
113
|
+
```json
|
|
114
|
+
{
|
|
115
|
+
"context": "shop",
|
|
116
|
+
"service": "billing",
|
|
117
|
+
"source": ".",
|
|
118
|
+
"settings": "config.settings",
|
|
119
|
+
"out": "celery.json"
|
|
120
|
+
}
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`source` is the directory the tasks and their callers are looked for in, the
|
|
124
|
+
input root unless said otherwise. `settings` is only needed where `manage.py`
|
|
125
|
+
does not name the settings module.
|
|
126
|
+
|
|
127
|
+
## Extraction limits
|
|
128
|
+
|
|
129
|
+
These cases do not become facts in the fragment, and each is reported:
|
|
130
|
+
|
|
131
|
+
- a tree with no task in it;
|
|
132
|
+
- a task nothing in the tree enqueues — it is still a `receive`, on the queue
|
|
133
|
+
its route gives it;
|
|
134
|
+
- an enqueue of a name that does not resolve to a task declared here;
|
|
135
|
+
- a task sent by name that no function here declares — a `send` with no
|
|
136
|
+
handler, and no flow;
|
|
137
|
+
- a `task_routes` pattern that matches no task;
|
|
138
|
+
- `task_routes` that is not a literal mapping;
|
|
139
|
+
- a settings module named and not found;
|
|
140
|
+
- a file that does not parse, by path.
|
|
141
|
+
|
|
142
|
+
## Manifest
|
|
143
|
+
|
|
144
|
+
```json
|
|
145
|
+
{
|
|
146
|
+
"plugins": [{ "name": "celery", "process": { "command": "python3", "args": ["plugins/extract-celery/main.py"] } }],
|
|
147
|
+
"extract": [
|
|
148
|
+
{
|
|
149
|
+
"plugin": "celery",
|
|
150
|
+
"in": "examples/shop/billing",
|
|
151
|
+
"out": "examples/shop/billing/portolan",
|
|
152
|
+
"options": { "context": "shop", "service": "billing", "out": "celery.json" }
|
|
153
|
+
}
|
|
154
|
+
]
|
|
155
|
+
}
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## Tests
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
python3 -m unittest discover -s plugins/extract-celery -t plugins/extract-celery -p "*_test.py"
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
The reader is held to fixtures under `testdata/`: `billing`, a Django project
|
|
165
|
+
with each shape this reads present once — the three ways a queue is chosen,
|
|
166
|
+
`on_commit` in both forms, a signature, a countdown — against the golden
|
|
167
|
+
`expected.json`; and `drift`, a tree where every diagnostic the reader is
|
|
168
|
+
meant to raise is raised once. `UPDATE_GOLDEN=1` writes the goldens again
|
|
169
|
+
after a deliberate change, and the diff is the review.
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
"""A tree with Celery in it in, one fragment out: the work queues the service
|
|
2
|
+
sends on and works from, and one flow per task that is both enqueued and
|
|
3
|
+
declared here.
|
|
4
|
+
|
|
5
|
+
A fragment, not a catalog: it carries one context and one service with
|
|
6
|
+
nothing on it but channels, and is merged with what the domain extractor said
|
|
7
|
+
about the same service before anything validates it. A task is not a domain
|
|
8
|
+
event and is not written as one; the queue is a channel of kind `job`, which
|
|
9
|
+
is what lets many callers enqueue the same task without the merge calling
|
|
10
|
+
them rival publishers.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
18
|
+
|
|
19
|
+
import catalog
|
|
20
|
+
import celery_conf as conf
|
|
21
|
+
import celery_conf as routes
|
|
22
|
+
from names import slug, title
|
|
23
|
+
from options import Options
|
|
24
|
+
from producers import Producer, read_producers, resolve
|
|
25
|
+
from protocol import Builder, File, Input
|
|
26
|
+
from source import Project
|
|
27
|
+
from celery_tasks import Task, index, read_tasks
|
|
28
|
+
|
|
29
|
+
JOB = "job"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Queue:
|
|
33
|
+
"""One queue as the fragment sees it: the tasks sent on it and the tasks
|
|
34
|
+
worked from it."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, address: str):
|
|
37
|
+
self.address = address
|
|
38
|
+
self.sends: Dict[str, List[Producer]] = {} # wire name -> its producers, in path order
|
|
39
|
+
self.works: Dict[str, Task] = {} # wire name -> the task declared here
|
|
40
|
+
self.how: Dict[str, str] = {} # wire name -> which rule routed it
|
|
41
|
+
self.schedules: Dict[str, List[conf.Schedule]] = {} # wire name -> what the clock sets off
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def extract(input_: Input, opts: Options, b: Builder, cwd: str = "") -> None:
|
|
45
|
+
cwd = cwd or os.getcwd()
|
|
46
|
+
root = os.path.abspath(os.path.join(cwd, input_.root))
|
|
47
|
+
|
|
48
|
+
def rel(path: str) -> str:
|
|
49
|
+
return os.path.relpath(path, cwd).replace(os.sep, "/")
|
|
50
|
+
|
|
51
|
+
source = os.path.normpath(os.path.join(root, opts.source or "."))
|
|
52
|
+
context = opts.context or os.path.basename(root)
|
|
53
|
+
service = opts.service or os.path.basename(root)
|
|
54
|
+
svc_id = context + "." + service
|
|
55
|
+
|
|
56
|
+
project = Project(root, source, rel)
|
|
57
|
+
for path, message in project.broken:
|
|
58
|
+
b.warn(path, "cannot be parsed, so nothing in it is read: %s" % message)
|
|
59
|
+
|
|
60
|
+
tasks = read_tasks(project)
|
|
61
|
+
if not tasks:
|
|
62
|
+
b.warn(svc_id, "no Celery tasks under %s: a function decorated @shared_task or @app.task is what this reads" % rel(source))
|
|
63
|
+
by_key, by_name = index(tasks)
|
|
64
|
+
|
|
65
|
+
cfg = conf.read_config(project, opts.settings)
|
|
66
|
+
if cfg.settings and not cfg.settings_found:
|
|
67
|
+
b.warn(svc_id, "settings module `%s` not found: only what the app module configures itself is read, and a task nothing places lands on `%s`" % (cfg.settings, conf.DEFAULT_QUEUE))
|
|
68
|
+
elif not cfg.settings and not cfg.apps:
|
|
69
|
+
b.warn(svc_id, "no Celery app and no settings module found, so a task with no queue of its own lands on `%s`" % conf.DEFAULT_QUEUE)
|
|
70
|
+
for where in cfg.opaque:
|
|
71
|
+
b.warn(where, "task_routes is not a literal mapping, so the routes are not read: this reader does not run a router")
|
|
72
|
+
for pattern in routes.unmatched(cfg, [task.name for task in tasks]):
|
|
73
|
+
b.warn(svc_id, "task_routes pattern `%s` matches no task in this tree" % pattern)
|
|
74
|
+
|
|
75
|
+
queues: Dict[str, Queue] = {}
|
|
76
|
+
|
|
77
|
+
def queue(address: str) -> Queue:
|
|
78
|
+
if address not in queues:
|
|
79
|
+
queues[address] = Queue(address)
|
|
80
|
+
return queues[address]
|
|
81
|
+
|
|
82
|
+
enqueued = set()
|
|
83
|
+
for producer in read_producers(project):
|
|
84
|
+
task: Optional[Task] = None
|
|
85
|
+
if producer.wire:
|
|
86
|
+
task = by_name.get(producer.wire)
|
|
87
|
+
wire = producer.wire
|
|
88
|
+
else:
|
|
89
|
+
task = by_key.get(producer.key) if producer.key else None
|
|
90
|
+
if task is None:
|
|
91
|
+
# An unresolved name that only one task answers to is that task.
|
|
92
|
+
same = [t for t in tasks if t.short == producer.label]
|
|
93
|
+
task = same[0] if len(same) == 1 else None
|
|
94
|
+
if task is None:
|
|
95
|
+
b.warn(producer.line, "`%s` is enqueued, but does not resolve to a task declared in this tree" % producer.label)
|
|
96
|
+
continue
|
|
97
|
+
wire = task.name
|
|
98
|
+
address, how = routes.queue_for(wire, producer.queue, task.queue if task else "", cfg)
|
|
99
|
+
q = queue(address)
|
|
100
|
+
q.sends.setdefault(wire, []).append(producer)
|
|
101
|
+
q.how.setdefault(wire, how)
|
|
102
|
+
if task is not None:
|
|
103
|
+
q.works[wire] = task
|
|
104
|
+
enqueued.add(task.key)
|
|
105
|
+
else:
|
|
106
|
+
b.warn(producer.line, "`%s` is sent as a task no function in this tree declares; recorded as a send with no handler here" % wire)
|
|
107
|
+
|
|
108
|
+
# Beat is one more producer: the clock enqueues the task, and the entry
|
|
109
|
+
# says which task, when, and on which queue.
|
|
110
|
+
if cfg.beat_in_database:
|
|
111
|
+
b.warn(svc_id, "%s is installed, so the schedules live in the database and are not read from this tree" % conf.BEAT_IN_DATABASE)
|
|
112
|
+
for where in cfg.opaque_schedules:
|
|
113
|
+
b.warn(where, "beat_schedule is not a literal mapping of entries, so what the clock sets off is not read")
|
|
114
|
+
for entry in cfg.schedules:
|
|
115
|
+
task = by_name.get(entry.task) if entry.task else None
|
|
116
|
+
if task is None and entry.ref:
|
|
117
|
+
key = resolve(project, entry.module, entry.ref)
|
|
118
|
+
task = by_key.get(key) if key else None
|
|
119
|
+
if task is None:
|
|
120
|
+
same = [t for t in tasks if t.short == entry.ref.split(".")[-1]]
|
|
121
|
+
task = same[0] if len(same) == 1 else None
|
|
122
|
+
if task is None:
|
|
123
|
+
b.warn(entry.where, "beat entry `%s` schedules `%s`, which no task in this tree declares" % (entry.name, entry.task or entry.ref))
|
|
124
|
+
continue
|
|
125
|
+
if entry.opaque:
|
|
126
|
+
b.warn(entry.where, "beat entry `%s` has a schedule this reader cannot read as syntax: `%s`" % (entry.name, entry.when))
|
|
127
|
+
address, how = routes.queue_for(task.name, entry.queue, task.queue, cfg)
|
|
128
|
+
q = queue(address)
|
|
129
|
+
q.works[task.name] = task
|
|
130
|
+
q.how.setdefault(task.name, how)
|
|
131
|
+
q.schedules.setdefault(task.name, []).append(entry)
|
|
132
|
+
enqueued.add(task.key)
|
|
133
|
+
|
|
134
|
+
for task in tasks:
|
|
135
|
+
if task.key in enqueued:
|
|
136
|
+
continue
|
|
137
|
+
b.warn(task.line, "task `%s` is declared, but nothing in this tree enqueues it" % task.name)
|
|
138
|
+
address, how = routes.queue_for(task.name, "", task.queue, cfg)
|
|
139
|
+
q = queue(address)
|
|
140
|
+
q.works[task.name] = task
|
|
141
|
+
q.how.setdefault(task.name, how)
|
|
142
|
+
|
|
143
|
+
channels = [channel_of(queues[address], cfg) for address in sorted(queues)]
|
|
144
|
+
flows = []
|
|
145
|
+
for address in sorted(queues):
|
|
146
|
+
q = queues[address]
|
|
147
|
+
for wire in sorted(set(q.sends) | set(q.schedules)):
|
|
148
|
+
task = q.works.get(wire)
|
|
149
|
+
if task is None:
|
|
150
|
+
continue
|
|
151
|
+
if wire in q.sends:
|
|
152
|
+
flows.append(flow_of(svc_id, context, service, q, wire, task, queues))
|
|
153
|
+
if wire in q.schedules:
|
|
154
|
+
flows.append(beat_flow_of(svc_id, context, service, q, wire, task, queues))
|
|
155
|
+
flows.sort(key=lambda f: f["slug"])
|
|
156
|
+
|
|
157
|
+
fragment: Dict[str, Any] = {
|
|
158
|
+
"generatedAt": input_.generated_at,
|
|
159
|
+
"commit": input_.commit,
|
|
160
|
+
"contexts": [
|
|
161
|
+
{
|
|
162
|
+
"id": context,
|
|
163
|
+
"slug": context,
|
|
164
|
+
"name": "",
|
|
165
|
+
"summary": "",
|
|
166
|
+
"services": [
|
|
167
|
+
{
|
|
168
|
+
"id": svc_id,
|
|
169
|
+
"slug": service,
|
|
170
|
+
"name": "",
|
|
171
|
+
"repo": "",
|
|
172
|
+
"path": "",
|
|
173
|
+
"readme": "",
|
|
174
|
+
"provides": [],
|
|
175
|
+
"consumes": [],
|
|
176
|
+
"aggregates": [],
|
|
177
|
+
"channels": channels,
|
|
178
|
+
}
|
|
179
|
+
],
|
|
180
|
+
}
|
|
181
|
+
],
|
|
182
|
+
"defs": {},
|
|
183
|
+
"flows": flows,
|
|
184
|
+
"adrs": [],
|
|
185
|
+
}
|
|
186
|
+
b.files.append(File(name=opts.out or "celery.json", contents=json.dumps(fragment, indent=2, ensure_ascii=False) + "\n"))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def channel_of(q: Queue, cfg: conf.Config) -> Dict[str, Any]:
|
|
190
|
+
messages = []
|
|
191
|
+
source = ""
|
|
192
|
+
for wire in sorted(set(q.sends) | set(q.works)):
|
|
193
|
+
task = q.works.get(wire)
|
|
194
|
+
producers = q.sends.get(wire, [])
|
|
195
|
+
schedules = q.schedules.get(wire, [])
|
|
196
|
+
doc = task.doc if task else ""
|
|
197
|
+
if producers:
|
|
198
|
+
messages.append(catalog.message(wire, task.short if task else wire, doc, "send"))
|
|
199
|
+
source = source or producers[0].line
|
|
200
|
+
elif schedules:
|
|
201
|
+
messages.append(catalog.message(wire, task.short if task else wire, ("Sent by Celery beat, %s. " % when_of(schedules) + doc).strip(), "send"))
|
|
202
|
+
source = source or schedules[0].where
|
|
203
|
+
if task:
|
|
204
|
+
handled = "Worked by `%s`, %s." % (task.short, q.how[wire])
|
|
205
|
+
if schedules:
|
|
206
|
+
handled += " Set off by Celery beat, %s." % when_of(schedules)
|
|
207
|
+
if not producers and not schedules:
|
|
208
|
+
handled += " Nothing in this tree enqueues it."
|
|
209
|
+
messages.append(catalog.message(wire, task.short, (handled + " " + doc).strip(), "receive"))
|
|
210
|
+
source = source or task.line
|
|
211
|
+
doc = "Tasks enqueued and worked through Celery"
|
|
212
|
+
doc += " over %s." % cfg.broker_scheme if cfg.broker_scheme else "."
|
|
213
|
+
return catalog.channel(q.address, JOB, "Celery · " + q.address, doc, messages, source)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def flow_of(svc_id: str, context: str, service: str, q: Queue, wire: str, task: Task, queues: Dict[str, Queue]) -> Dict[str, Any]:
|
|
217
|
+
slugged = slug(service + "-celery-" + task.short)
|
|
218
|
+
# The same task sent on two queues is two flows, told apart by the queue.
|
|
219
|
+
if sum(1 for other in queues.values() if wire in other.sends and wire in other.works) > 1:
|
|
220
|
+
slugged += "-" + slug(q.address)
|
|
221
|
+
# No dot in the id: a participant is a root of the model, and a dotted id
|
|
222
|
+
# would read as something nested under a `celery` nobody declared.
|
|
223
|
+
broker = "celery-" + slug(q.address)
|
|
224
|
+
producers = q.sends[wire]
|
|
225
|
+
first = producers[0]
|
|
226
|
+
notes = list(first.notes)
|
|
227
|
+
enqueue_note = ("Enqueued " + ", ".join(notes) + ". " if notes else "") + task.doc
|
|
228
|
+
if len(producers) > 1:
|
|
229
|
+
enqueue_note = (enqueue_note + " Also enqueued at %s." % ", ".join(p.line for p in producers[1:])).strip()
|
|
230
|
+
|
|
231
|
+
def handoff(direction: str) -> Dict[str, str]:
|
|
232
|
+
return {
|
|
233
|
+
"kind": "job",
|
|
234
|
+
"transport": "celery",
|
|
235
|
+
"channel": q.address,
|
|
236
|
+
"message": wire,
|
|
237
|
+
"direction": direction,
|
|
238
|
+
}
|
|
239
|
+
return catalog.flow(
|
|
240
|
+
"flow." + slugged,
|
|
241
|
+
slugged,
|
|
242
|
+
title(task.short) + " task",
|
|
243
|
+
"Celery task `%s` is enqueued on `%s` and worked by `%s`." % (wire, q.address, task.short),
|
|
244
|
+
first.module.rel,
|
|
245
|
+
context,
|
|
246
|
+
[
|
|
247
|
+
catalog.participant(svc_id, "service", context),
|
|
248
|
+
catalog.participant(broker, "broker", None, "Celery · " + q.address),
|
|
249
|
+
],
|
|
250
|
+
[
|
|
251
|
+
catalog.step(
|
|
252
|
+
"enqueue",
|
|
253
|
+
svc_id,
|
|
254
|
+
broker,
|
|
255
|
+
"call",
|
|
256
|
+
"enqueue " + task.short,
|
|
257
|
+
catalog.DECLARED,
|
|
258
|
+
note=enqueue_note.strip(),
|
|
259
|
+
line=first.line,
|
|
260
|
+
handoff=handoff("send"),
|
|
261
|
+
),
|
|
262
|
+
catalog.step(
|
|
263
|
+
"work",
|
|
264
|
+
broker,
|
|
265
|
+
svc_id,
|
|
266
|
+
"call",
|
|
267
|
+
task.short,
|
|
268
|
+
catalog.DECLARED,
|
|
269
|
+
note="Celery hands `%s` to the worker consuming `%s`, %s." % (wire, q.address, q.how[wire]),
|
|
270
|
+
line=task.line,
|
|
271
|
+
continues_at=task.entrypoint,
|
|
272
|
+
handoff=handoff("receive"),
|
|
273
|
+
),
|
|
274
|
+
],
|
|
275
|
+
trigger={"kind": "job", "label": "Celery · " + q.address, "confidence": "high"},
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def when_of(schedules: List[conf.Schedule]) -> str:
|
|
280
|
+
"""`cron 0 9 * * * (remind-unpaid-invoices)`, one per entry."""
|
|
281
|
+
return ", ".join("%s (`%s`)" % (entry.when, entry.name) for entry in schedules)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def beat_flow_of(svc_id: str, context: str, service: str, q: Queue, wire: str, task: Task, queues: Dict[str, Queue]) -> Dict[str, Any]:
|
|
285
|
+
"""The flow the clock opens: beat enqueues the task, and the worker runs
|
|
286
|
+
it. A task the code also enqueues keeps that flow and gets this one beside
|
|
287
|
+
it, told apart by `-beat`, because what happens when the clock fires and
|
|
288
|
+
what happens when a request does are two different questions."""
|
|
289
|
+
slugged = slug(service + "-celery-" + task.short)
|
|
290
|
+
if any(wire in other.sends for other in queues.values()):
|
|
291
|
+
slugged += "-beat"
|
|
292
|
+
if sum(1 for other in queues.values() if wire in other.schedules and wire in other.works) > 1:
|
|
293
|
+
slugged += "-" + slug(q.address)
|
|
294
|
+
broker = "celery-" + slug(q.address)
|
|
295
|
+
schedules = q.schedules[wire]
|
|
296
|
+
first = schedules[0]
|
|
297
|
+
|
|
298
|
+
def handoff(direction: str) -> Dict[str, str]:
|
|
299
|
+
return {
|
|
300
|
+
"kind": "job",
|
|
301
|
+
"transport": "celery",
|
|
302
|
+
"channel": q.address,
|
|
303
|
+
"message": wire,
|
|
304
|
+
"direction": direction,
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
enqueue_note = "Scheduled %s. " % when_of(schedules) + task.doc
|
|
308
|
+
return catalog.flow(
|
|
309
|
+
"flow." + slugged,
|
|
310
|
+
slugged,
|
|
311
|
+
title(task.short) + " task, on schedule",
|
|
312
|
+
"Celery beat enqueues `%s` on `%s` %s, and `%s` works it." % (wire, q.address, ", ".join(e.when for e in schedules), task.short),
|
|
313
|
+
first.module.rel,
|
|
314
|
+
context,
|
|
315
|
+
[
|
|
316
|
+
catalog.participant("celery-beat", "actor", None, "Celery beat"),
|
|
317
|
+
catalog.participant(svc_id, "service", context),
|
|
318
|
+
catalog.participant(broker, "broker", None, "Celery · " + q.address),
|
|
319
|
+
],
|
|
320
|
+
[
|
|
321
|
+
catalog.step(
|
|
322
|
+
"enqueue",
|
|
323
|
+
"celery-beat",
|
|
324
|
+
broker,
|
|
325
|
+
"call",
|
|
326
|
+
"enqueue " + task.short,
|
|
327
|
+
catalog.DECLARED,
|
|
328
|
+
note=enqueue_note.strip(),
|
|
329
|
+
line=first.where,
|
|
330
|
+
handoff=handoff("send"),
|
|
331
|
+
),
|
|
332
|
+
catalog.step(
|
|
333
|
+
"work",
|
|
334
|
+
broker,
|
|
335
|
+
svc_id,
|
|
336
|
+
"call",
|
|
337
|
+
task.short,
|
|
338
|
+
catalog.DECLARED,
|
|
339
|
+
note="Celery hands `%s` to the worker consuming `%s`, %s." % (wire, q.address, q.how[wire]),
|
|
340
|
+
line=task.line,
|
|
341
|
+
continues_at=task.entrypoint,
|
|
342
|
+
handoff=handoff("receive"),
|
|
343
|
+
),
|
|
344
|
+
],
|
|
345
|
+
trigger={"kind": "scheduled", "label": ", ".join(e.when for e in schedules), "confidence": "high"},
|
|
346
|
+
)
|