@shortlink-org/portolan 0.2.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. package/README.md +174 -6
  2. package/catalog/enum_test.go +46 -0
  3. package/catalog/evidence_test.go +35 -0
  4. package/catalog/model.go +1213 -0
  5. package/catalog/roundtrip_test.go +210 -0
  6. package/catalog/via_test.go +38 -0
  7. package/cli/init.test.mjs +6 -1
  8. package/cli/portolan.mjs +14 -1
  9. package/cli/portolan.test.mjs +49 -0
  10. package/go.mod +21 -0
  11. package/go.sum +34 -0
  12. package/internal/gocall/README.md +19 -0
  13. package/internal/gocall/analyze.go +189 -0
  14. package/internal/gocall/analyze_test.go +107 -0
  15. package/internal/gohttp/analyze.go +2562 -0
  16. package/internal/gohttp/destination.go +373 -0
  17. package/internal/gohttp/endpoints.go +1067 -0
  18. package/internal/gohttp/roots.go +320 -0
  19. package/internal/gohttp/typed.go +96 -0
  20. package/internal/goscan/constants.go +85 -0
  21. package/internal/goscan/goscan_test.go +227 -0
  22. package/internal/goscan/index.go +629 -0
  23. package/internal/goscan/index_test.go +66 -0
  24. package/internal/goscan/names.go +52 -0
  25. package/internal/goscan/parse_test.go +11 -0
  26. package/internal/goscan/source.go +37 -0
  27. package/internal/goscan/tree.go +284 -0
  28. package/internal/goscan/types.go +99 -0
  29. package/internal/wsdl/ids.go +127 -0
  30. package/internal/wsdl/ids_test.go +21 -0
  31. package/internal/wsdl/model.go +70 -0
  32. package/internal/wsdl/parse.go +949 -0
  33. package/internal/wsdl/parse_test.go +170 -0
  34. package/package.json +28 -15
  35. package/plugin/describe.go +156 -0
  36. package/plugin/describe_test.go +114 -0
  37. package/plugin/protocol.go +141 -0
  38. package/plugin/schematest/schematest.go +126 -0
  39. package/plugins/README.md +404 -50
  40. package/plugins/cmd/portolan-http-clients/main.go +19 -0
  41. package/plugins/extract-argocd/options.schema.json +44 -0
  42. package/plugins/extract-celery/extract.py +0 -2
  43. package/plugins/extract-celery/extract_test.py +1 -1
  44. package/plugins/extract-celery/main.py +1 -0
  45. package/plugins/extract-csharp-ddd/README.md +213 -0
  46. package/plugins/extract-csharp-ddd/options.schema.json +55 -0
  47. package/plugins/extract-django/README.md +52 -19
  48. package/plugins/extract-django/choices.py +47 -0
  49. package/plugins/extract-django/domain.py +30 -18
  50. package/plugins/extract-django/extract.py +21 -7
  51. package/plugins/extract-django/extract_test.py +68 -2
  52. package/plugins/extract-django/lifecycle.py +4 -28
  53. package/plugins/extract-django/main.py +1 -0
  54. package/plugins/extract-django/operations.py +1 -1
  55. package/plugins/extract-django/routing_test.py +109 -1
  56. package/plugins/extract-django/rules.py +154 -0
  57. package/plugins/extract-django/rules_test.py +158 -0
  58. package/plugins/extract-django/store.py +1 -1
  59. package/plugins/extract-django/transport.py +101 -55
  60. package/plugins/extract-django/verbs.py +241 -0
  61. package/plugins/extract-go/README.md +47 -0
  62. package/plugins/extract-go-sqs/options.schema.json +21 -0
  63. package/plugins/extract-http-clients/describe.go +20 -0
  64. package/plugins/extract-http-clients/describe_test.go +11 -0
  65. package/plugins/extract-http-clients/extract.go +740 -0
  66. package/plugins/extract-http-clients/extract_test.go +1561 -0
  67. package/plugins/extract-http-clients/main.go +41 -0
  68. package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
  69. package/plugins/extract-java/build/org/portolan/extract/Main.class +0 -0
  70. package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
  71. package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
  72. package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
  73. package/plugins/extract-k8s/options.schema.json +32 -0
  74. package/plugins/extract-laravel/Cargo.lock +962 -0
  75. package/plugins/extract-laravel/Cargo.toml +20 -0
  76. package/plugins/extract-laravel/README.md +200 -0
  77. package/plugins/extract-laravel/options.schema.json +68 -0
  78. package/plugins/extract-laravel/rustfmt.toml +1 -0
  79. package/plugins/extract-php-ddd/Cargo.lock +972 -0
  80. package/plugins/extract-php-ddd/Cargo.toml +22 -0
  81. package/plugins/extract-php-ddd/README.md +141 -0
  82. package/plugins/extract-php-ddd/options.schema.json +50 -0
  83. package/plugins/extract-php-ddd/rustfmt.toml +1 -0
  84. package/plugins/extract-python-kafka/extract.py +0 -2
  85. package/plugins/extract-python-kafka/extract_test.py +1 -1
  86. package/plugins/extract-python-kafka/main.py +1 -0
  87. package/plugins/extract-terraform/options.schema.json +26 -0
  88. package/plugins/extract-ts/extract.test.ts +2 -2
  89. package/plugins/extract-ts/extract.ts +4 -5
  90. package/plugins/extract-ts/graphql.test.ts +1 -1
  91. package/plugins/extract-ts/main.ts +1 -0
  92. package/plugins/openapi/ids.go +261 -0
  93. package/plugins/openapi/ids_test.go +98 -0
  94. package/plugins/phpscan/Cargo.lock +953 -0
  95. package/plugins/phpscan/Cargo.toml +19 -0
  96. package/plugins/phpscan/rustfmt.toml +1 -0
  97. package/plugins/portolan-go.wasm +0 -0
  98. package/plugins/pyplugin/catalog.py +24 -1
  99. package/plugins/pyplugin/protocol.py +1 -5
  100. package/plugins/verify-otel/options.schema.json +12 -0
  101. package/portolan.json +132 -4
  102. package/rules/builtin.json +218 -0
  103. package/schema/portolan.schema.json +905 -4
  104. package/scripts/README.md +21 -13
  105. package/scripts/catalog-sources.mjs +6 -0
  106. package/scripts/delivery-presets.mjs +57 -14
  107. package/scripts/diff.mjs +5 -1
  108. package/scripts/django-aggregates.test.mjs +58 -0
  109. package/scripts/gen-likec4.mjs +150 -17
  110. package/scripts/gen-likec4.test.mjs +96 -0
  111. package/scripts/gen.mjs +148 -118
  112. package/scripts/gitops-example.test.mjs +108 -0
  113. package/scripts/go-discovery.test.mjs +30 -0
  114. package/scripts/history.mjs +186 -3
  115. package/scripts/history.test.mjs +1 -1
  116. package/scripts/host-plugins/fetch-argocd.mjs +338 -0
  117. package/scripts/host-plugins/fetch-argocd.options.json +49 -0
  118. package/scripts/host-plugins/fetch-argocd.test.mjs +274 -0
  119. package/scripts/host-plugins/fetch-bsr.mjs +1 -0
  120. package/scripts/host-plugins/fetch-csr.mjs +1 -0
  121. package/scripts/host-plugins/fetch-git.mjs +78 -21
  122. package/scripts/host-plugins/fetch-git.test.mjs +62 -8
  123. package/scripts/host-plugins/fetch-k8s.mjs +263 -0
  124. package/scripts/host-plugins/fetch-k8s.options.json +50 -0
  125. package/scripts/host-plugins/fetch-k8s.test.mjs +259 -0
  126. package/scripts/host-plugins/k8s-topology.mjs +183 -0
  127. package/scripts/json-format.mjs +192 -0
  128. package/scripts/json-format.test.mjs +97 -0
  129. package/scripts/local-api.mjs +287 -12
  130. package/scripts/local-api.test.mjs +150 -5
  131. package/scripts/local-discovery.mjs +94 -9
  132. package/scripts/manifest.mjs +14 -3
  133. package/scripts/manifest.test.mjs +24 -0
  134. package/scripts/output-diff.mjs +94 -0
  135. package/scripts/output-diff.test.mjs +36 -0
  136. package/scripts/package-smoke.mjs +62 -4
  137. package/scripts/plugin-host.mjs +45 -3
  138. package/scripts/plugin-host.test.mjs +9 -0
  139. package/scripts/plugin-wasm-worker.mjs +4 -1
  140. package/scripts/plugins-fresh.mjs +97 -0
  141. package/scripts/plugins-fresh.test.mjs +64 -0
  142. package/scripts/provenance.mjs +72 -0
  143. package/scripts/provenance.test.mjs +149 -0
  144. package/scripts/run-builtin.mjs +62 -7
  145. package/scripts/schema.mjs +167 -19
  146. package/scripts/trace-trials.mjs +176 -0
  147. package/scripts/trace-trials.test.mjs +142 -0
  148. package/scripts/warning-policy.mjs +167 -0
  149. package/scripts/warning-policy.test.mjs +93 -0
  150. package/src/app/Breadcrumbs.test.ts +3 -0
  151. package/src/app/Breadcrumbs.tsx +3 -0
  152. package/src/app/CatalogApp.tsx +2 -0
  153. package/src/app/Sidebar.tsx +3 -3
  154. package/src/app/SidebarFooter.tsx +20 -4
  155. package/src/catalog-docs.test.ts +64 -0
  156. package/src/catalog-docs.ts +35 -0
  157. package/src/catalog-error.test.ts +15 -0
  158. package/src/catalog-index.ts +25 -0
  159. package/src/catalog-model.ts +293 -5
  160. package/src/catalog-validation.ts +113 -2
  161. package/src/catalog.test.ts +40 -1
  162. package/src/chat/Starter.tsx +5 -11
  163. package/src/chat/tools.test.ts +27 -0
  164. package/src/chat/tools.ts +5 -9
  165. package/src/components/CatalogStamp.tsx +10 -8
  166. package/src/components/ChannelRows.test.tsx +76 -1
  167. package/src/components/ChannelRows.tsx +223 -60
  168. package/src/components/DeploymentRows.tsx +131 -0
  169. package/src/components/DocsLinks.test.tsx +27 -0
  170. package/src/components/DocsLinks.tsx +56 -0
  171. package/src/components/FieldTree.tsx +5 -2
  172. package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
  173. package/src/components/HTTPDestinationEvidence.tsx +31 -0
  174. package/src/components/Integrations.tsx +1 -1
  175. package/src/components/MachineDocs.tsx +6 -5
  176. package/src/components/MethodRows.tsx +9 -2
  177. package/src/components/PluginIcon.tsx +77 -0
  178. package/src/components/ProblemRow.tsx +173 -153
  179. package/src/components/RelationEvidence.test.tsx +14 -0
  180. package/src/components/RelationEvidence.tsx +53 -0
  181. package/src/components/RuleMarks.tsx +22 -0
  182. package/src/components/ShapeRows.tsx +24 -20
  183. package/src/data.ts +46 -7
  184. package/src/enrich.test.ts +459 -4
  185. package/src/enrich.ts +308 -7
  186. package/src/er/ErCanvas.tsx +218 -12
  187. package/src/er/GroupNode.tsx +57 -0
  188. package/src/er/StoreHeader.tsx +1 -0
  189. package/src/er/layout.test.ts +85 -2
  190. package/src/er/layout.ts +140 -5
  191. package/src/er/spec.test.ts +17 -0
  192. package/src/er/spec.ts +23 -10
  193. package/src/flow/Recordings.test.tsx +52 -0
  194. package/src/flow/Recordings.tsx +236 -0
  195. package/src/flow/StepDetail.tsx +59 -0
  196. package/src/flow/TraceTrial.tsx +419 -0
  197. package/src/flow/evidence.test.ts +16 -0
  198. package/src/flow/evidence.ts +34 -0
  199. package/src/flow/examples.test.ts +33 -0
  200. package/src/flow/examples.ts +37 -0
  201. package/src/flow/outline.test.ts +28 -0
  202. package/src/flow/outline.ts +7 -2
  203. package/src/flow/trace-trial-resume.test.ts +42 -0
  204. package/src/flow/trace-trial-resume.ts +74 -0
  205. package/src/graph/elk.ts +78 -0
  206. package/src/index.css +68 -0
  207. package/src/landing/DraggableReveal.tsx +3 -2
  208. package/src/landing/EvidencePipeline.tsx +105 -0
  209. package/src/landing/LandingPage.tsx +17 -70
  210. package/src/landing/ProductTour.tsx +6 -6
  211. package/src/lib/all-problems.ts +27 -17
  212. package/src/lib/catalog-diff.ts +1 -1
  213. package/src/lib/centrality.test.ts +251 -0
  214. package/src/lib/centrality.ts +232 -0
  215. package/src/lib/confluence.test.ts +37 -0
  216. package/src/lib/confluence.ts +41 -0
  217. package/src/lib/context-color.ts +1 -1
  218. package/src/lib/deployment-drift.ts +22 -0
  219. package/src/lib/derive.ts +43 -69
  220. package/src/lib/django-aggregates.d.mts +9 -0
  221. package/src/lib/django-aggregates.mjs +36 -0
  222. package/src/lib/django-aggregates.test.ts +29 -0
  223. package/src/lib/django-aggregates.ts +5 -0
  224. package/src/lib/environments.test.ts +83 -0
  225. package/src/lib/environments.ts +57 -0
  226. package/src/lib/integration-url.test.ts +30 -0
  227. package/src/lib/integration-url.ts +63 -0
  228. package/src/lib/kafka-ui.ts +3 -45
  229. package/src/lib/local-api.ts +116 -4
  230. package/src/lib/notion.ts +13 -0
  231. package/src/lib/plugin-index.json +2765 -0
  232. package/src/lib/plugins.test.ts +68 -0
  233. package/src/lib/plugins.ts +264 -0
  234. package/src/lib/problem-flows.test.ts +61 -0
  235. package/src/lib/problem-flows.ts +78 -0
  236. package/src/lib/problem-rules-cel.d.mts +43 -0
  237. package/src/lib/problem-rules-cel.mjs +407 -0
  238. package/src/lib/problem-rules.test.ts +287 -0
  239. package/src/lib/problem-rules.ts +271 -0
  240. package/src/lib/problem-subjects.ts +737 -0
  241. package/src/lib/rule-entries.ts +39 -0
  242. package/src/lib/{data-problems.test.ts → rules-data.test.ts} +18 -16
  243. package/src/lib/rules-deploy.test.ts +133 -0
  244. package/src/lib/{problems.test.ts → rules-edges.test.ts} +8 -3
  245. package/src/lib/{proto-problems.test.ts → rules-proto.test.ts} +9 -25
  246. package/src/lib/{wire-problems.test.ts → rules-wire.test.ts} +13 -11
  247. package/src/lib/rules.test.ts +51 -0
  248. package/src/lib/rules.ts +86 -0
  249. package/src/lib/setup-info.test.ts +17 -0
  250. package/src/lib/setup-info.ts +58 -0
  251. package/src/lib/shape.test.ts +32 -0
  252. package/src/lib/shape.ts +30 -6
  253. package/src/lib/tech.ts +16 -0
  254. package/src/lib/trace-project.test.ts +34 -0
  255. package/src/lib/trace-project.ts +50 -0
  256. package/src/lib/use-problems.ts +23 -0
  257. package/src/lib/warnings.test.ts +63 -0
  258. package/src/lib/warnings.ts +260 -0
  259. package/src/likec4/ids.test.ts +6 -2
  260. package/src/likec4/ids.ts +43 -0
  261. package/src/main.tsx +23 -0
  262. package/src/map/ContextMapGraph.tsx +76 -32
  263. package/src/merge-deployments.test.ts +127 -0
  264. package/src/merge.test.ts +82 -0
  265. package/src/merge.ts +189 -18
  266. package/src/pages/AggregatePage.tsx +65 -14
  267. package/src/pages/ContextMap.tsx +45 -3
  268. package/src/pages/ContextPage.tsx +8 -5
  269. package/src/pages/EventPage.tsx +15 -5
  270. package/src/pages/FlowDetail.tsx +23 -2
  271. package/src/pages/GraphPage.tsx +40 -3
  272. package/src/pages/Overview.tsx +152 -12
  273. package/src/pages/PluginIndex.tsx +190 -0
  274. package/src/pages/Problems.tsx +396 -128
  275. package/src/pages/ServicePage.tsx +62 -5
  276. package/src/pages/Settings.tsx +217 -43
  277. package/src/pages/settings/AboutSettings.tsx +8 -1
  278. package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
  279. package/src/pages/settings/IntegrationsSettings.tsx +63 -17
  280. package/src/pages/settings/RecordingSettings.tsx +138 -0
  281. package/src/pages/settings/RulesSettings.tsx +825 -0
  282. package/src/routes.test.ts +9 -0
  283. package/src/routes.ts +22 -1
  284. package/src/selection/DetailPanel.tsx +15 -0
  285. package/src/virtual-provenance.d.ts +11 -0
  286. package/vite.config.ts +5 -0
  287. package/scripts/vendor-lock.mjs +0 -58
  288. package/scripts/vendor-lock.test.mjs +0 -69
  289. package/src/lib/data-problems.ts +0 -314
  290. package/src/lib/proto-problems.ts +0 -237
  291. package/src/lib/wire-problems.ts +0 -342
package/scripts/gen.mjs CHANGED
@@ -15,9 +15,8 @@ import {
15
15
  rmSync,
16
16
  statSync,
17
17
  } from "node:fs";
18
- import { execFileSync } from "node:child_process";
19
18
  import { createHash } from "node:crypto";
20
- import { join, relative, resolve } from "node:path";
19
+ import { isAbsolute, join, relative } from "node:path";
21
20
 
22
21
  import {
23
22
  addBuildStep,
@@ -26,16 +25,18 @@ import {
26
25
  writeBuildReport,
27
26
  } from "./build-report.mjs";
28
27
  import { loadCatalog } from "./catalog-sources.mjs";
29
- import { historyFor } from "./history.mjs";
28
+ import { changedSince, fileAt, historyFor, lastCommitTouching } from "./history.mjs";
30
29
  import { loadManifest, stepKeys } from "./manifest.mjs";
31
30
  import { describePlugin, runPlugin } from "./plugin-host.mjs";
32
- import { vendoredCommit } from "./vendor-lock.mjs";
31
+ import { explainChange } from "./output-diff.mjs";
33
32
  import {
34
33
  removeOutputFile,
35
34
  safeOutputPath,
36
35
  writeOutputFile,
37
36
  } from "./output-path.mjs";
38
37
  import { builtinPlugin } from "./builtin-plugins.mjs";
38
+ import { diagnoseWarnings } from "./warning-policy.mjs";
39
+ import { likec4Sources } from "./gen-likec4.mjs";
39
40
 
40
41
  const PORTOLAN_VERSION = "0.1.0";
41
42
  const EVENTS = process.env.PORTOLAN_EVENTS === "1";
@@ -63,6 +64,10 @@ function event(value) {
63
64
  // in one run is drift that never happened.
64
65
  const MANIFEST = ".portolan-manifest";
65
66
 
67
+ // The one step no manifest declares: the app's own pictures, generated from
68
+ // every source, keyed by its own name in likec4/.portolan-manifest.
69
+ const LIKEC4_STEP = { plugin: "likec4", out: "likec4", key: "likec4" };
70
+
66
71
  const check = process.argv.includes("--check");
67
72
  const manifestSha256 = createHash("sha256")
68
73
  .update(readFileSync("portolan.json"))
@@ -122,9 +127,12 @@ async function generate() {
122
127
 
123
128
  // Extractors run first and write catalog fragments; only then is there a
124
129
  // catalog for anything else to read.
130
+ //
131
+ // No stamp travels with the request (portolan.0010). A fragment is content
132
+ // and nothing else; when it last changed is what the history of the
133
+ // fragment says, read wherever the catalog is read.
125
134
  for (const step of manifest.extract ?? []) {
126
135
  const plugin = pluginNamed(step.plugin);
127
- const stamp = stampFor(step.in, step.out);
128
136
  // A plugin that asks for history gets the root's, read once per checkout
129
137
  // (portolan.0007); left out when the root is not in a checkout, which the
130
138
  // plugin reports in its own words.
@@ -135,13 +143,11 @@ async function generate() {
135
143
  input: {
136
144
  root: step.in,
137
145
  output: step.out,
138
- commit: stamp.commit,
139
- generatedAt: stamp.generatedAt,
140
146
  ...(history ? { history } : {}),
141
147
  },
142
148
  options: step.options ?? {},
143
149
  }, {}, { workspace: process.cwd() }),
144
- );
150
+ inputsOf(step));
145
151
  }
146
152
 
147
153
  // Verifiers read observed evidence against the merged catalog while leaving
@@ -152,17 +158,16 @@ async function generate() {
152
158
  // carry it into the generators and into the next run.
153
159
  for (const step of manifest.verify ?? []) {
154
160
  const plugin = pluginNamed(step.plugin);
155
- const stamp = stampFor(step.in, step.out);
156
161
  const own = (previous(step.out)[keys.keyOf(step)] ?? []).map((name) => join(step.out, name));
157
- const { catalog } = await loadSources({ exclude: [...own, ...staleAll()] });
162
+ const { catalog, sources } = await loadSources({ exclude: [...own, ...staleAll()] });
158
163
  await executeStep("verify", step, `${step.plugin} ⇐ ${step.in}`, async () =>
159
164
  runPlugin(plugin, {
160
165
  portolanVersion: PORTOLAN_VERSION,
161
- input: { root: step.in, output: step.out, commit: stamp.commit, generatedAt: stamp.generatedAt },
166
+ input: { root: step.in, output: step.out },
162
167
  catalog,
163
168
  options: step.options ?? {},
164
169
  }, {}, { workspace: process.cwd() }),
165
- );
170
+ inputsOf(step, sources.map((source) => source.path)));
166
171
  }
167
172
 
168
173
  // What a dropped extract step wrote is not part of the catalog, so it is
@@ -182,18 +187,27 @@ async function generate() {
182
187
 
183
188
  for (const step of manifest.generate ?? []) {
184
189
  const plugin = pluginNamed(step.plugin);
185
- const generatedCatalog = step.catalog
186
- ? (await loadSources({ profile: step.catalog })).catalog
187
- : catalog;
190
+ const generated = step.catalog
191
+ ? await loadSources({ profile: step.catalog })
192
+ : { catalog, sources };
188
193
  await executeStep("generate", step, `${step.plugin} → ${step.out}`, async () =>
189
194
  runPlugin(plugin, {
190
195
  portolanVersion: PORTOLAN_VERSION,
191
- catalog: generatedCatalog,
196
+ catalog: generated.catalog,
192
197
  options: step.options ?? {},
193
198
  }),
194
- );
199
+ { inputs: generated.sources.map((source) => source.path), excludes: [] });
195
200
  }
196
201
 
202
+ // The LikeC4 sources are pages of the catalog like any the generators
203
+ // write: committed, reviewed as a diff, and held to the catalog by the
204
+ // check. They used to be written only before the dev server started,
205
+ // which left a flow a verifier had just changed with a picture of the
206
+ // flow before - and a run from the page with no picture of a new flow.
207
+ await executeStep("generate", LIKEC4_STEP, "likec4 → likec4", async () => ({
208
+ files: await likec4Sources({ catalog, manifest }),
209
+ }), { inputs: sources.map((source) => source.path), excludes: [] });
210
+
197
211
  sweepAll();
198
212
  }
199
213
 
@@ -208,6 +222,17 @@ function sweepAll() {
208
222
  }
209
223
  }
210
224
 
225
+ /**
226
+ * The keys of the steps that write into a directory this run: the manifest's,
227
+ * and the likec4 step's in its own directory, which no manifest declares and
228
+ * the sweep would otherwise take for a step that was dropped.
229
+ */
230
+ function liveKeysIn(out) {
231
+ const live = new Set(keys.liveIn(out));
232
+ if (out === LIKEC4_STEP.out) live.add(LIKEC4_STEP.key);
233
+ return live;
234
+ }
235
+
211
236
  /** Every file the sweep would remove, in every directory a step writes into. */
212
237
  function staleAll() {
213
238
  const outs = new Set(
@@ -230,7 +255,7 @@ function staleAll() {
230
255
  */
231
256
  function staleIn(out) {
232
257
  const listing = previous(out);
233
- const live = keys.liveIn(out);
258
+ const live = liveKeysIn(out);
234
259
  const dead = Object.keys(listing).filter((key) => !live.has(key));
235
260
  if (dead.length === 0) return [];
236
261
 
@@ -249,13 +274,27 @@ function staleIn(out) {
249
274
  return stale;
250
275
  }
251
276
 
252
- async function executeStep(phase, step, label, work) {
277
+ /**
278
+ * Runs one step and settles its files. `reads` is what the step reads - paths,
279
+ * and the output directories among them to leave out - so that an output
280
+ * that moved can be explained by what moved among its inputs since the
281
+ * output was last committed.
282
+ */
283
+ async function executeStep(phase, step, label, work, reads) {
253
284
  const startedAt = Date.now();
254
285
  event({ type: "step-started", ordinal: report.steps.length, phase, plugin: step.plugin, input: step.in, output: step.out });
255
286
  try {
256
287
  const { files, warnings = [] } = await work();
257
- const changes = apply(files, step.out, keys.keyOf(step), check);
258
- const changed = summarise(label, files, changes, warnings);
288
+ const diagnostics = diagnoseWarnings({
289
+ plugin: step.plugin,
290
+ warnings,
291
+ policies: manifest.warningPolicies,
292
+ project: projectForStep(step),
293
+ phase,
294
+ });
295
+ const changes = apply(files, step.out, step.key ?? keys.keyOf(step), check);
296
+ const since = changes.length > 0 ? whyChanged(step, files.map((file) => join(step.out, file.name)), reads) : null;
297
+ const changed = summarise(label, files, changes, diagnostics, since);
259
298
  drifted = changed || drifted;
260
299
  const result = {
261
300
  phase,
@@ -267,10 +306,14 @@ async function executeStep(phase, step, label, work) {
267
306
  fileCount: files.length,
268
307
  changedCount: changes.length,
269
308
  changes,
309
+ // What moved among the inputs since the output was last committed, when
310
+ // something in the output moved; the Settings page can say why.
311
+ ...(since ? { since } : {}),
270
312
  files: files.map((file) => join(step.out, file.name)),
271
313
  // What the plugin could not read, in its own words. Already printed as
272
314
  // it ran; kept here so the Settings page can list it beside the step.
273
315
  warnings,
316
+ diagnostics,
274
317
  };
275
318
  addBuildStep(report, result);
276
319
  persistReport();
@@ -296,6 +339,15 @@ async function executeStep(phase, step, label, work) {
296
339
  }
297
340
  }
298
341
 
342
+ function projectForStep(step) {
343
+ if (!step.in) return "";
344
+ const input = String(step.in).replace(/\\/g, "/").replace(/\/+$/, "");
345
+ const projects = (manifest.projects ?? [])
346
+ .filter((project) => typeof project?.id === "string" && typeof project?.root === "string")
347
+ .sort((left, right) => right.root.length - left.root.length);
348
+ return projects.find((project) => input === project.root || input.startsWith(`${project.root}/`))?.id ?? "";
349
+ }
350
+
299
351
  function persistReport() {
300
352
  try {
301
353
  writeBuildReport(report);
@@ -334,9 +386,17 @@ async function needsOf(plugin) {
334
386
  return pluginNeeds.get(plugin.name);
335
387
  }
336
388
 
337
- /** Prints what a step did, and says whether it left the tree out of date. */
338
- function summarise(label, files, changes, warnings = []) {
339
- const said = warnings.length > 0 ? `, ${warnings.length} warning${warnings.length === 1 ? "" : "s"}` : "";
389
+ /**
390
+ * Prints what a step did, and says whether it left the tree out of date. A
391
+ * file that differs says where it first differs, and the step says what moved
392
+ * among its inputs since the output was last committed - or that nothing
393
+ * did, which points at the plugin.
394
+ */
395
+ function summarise(label, files, changes, diagnostics = [], since = null) {
396
+ const suppressed = diagnostics.filter((diagnostic) => diagnostic.suppressed).length;
397
+ const said = diagnostics.length > 0
398
+ ? `, ${diagnostics.length} warning${diagnostics.length === 1 ? "" : "s"}${suppressed ? ` (${suppressed} suppressed)` : ""}`
399
+ : "";
340
400
  const summary = `${label}: ${files.length} file${files.length === 1 ? "" : "s"}${said}`;
341
401
 
342
402
  if (changes.length === 0) {
@@ -347,120 +407,86 @@ function summarise(label, files, changes, warnings = []) {
347
407
 
348
408
  if (!check) {
349
409
  console.log(`${summary}, ${changes.length} written`);
410
+ if (since) console.log(` ${describeSince(since)}`);
350
411
 
351
412
  return false;
352
413
  }
353
414
 
354
415
  console.error(`${summary}, ${changes.length} out of date:`);
355
- for (const change of changes.slice(0, 20)) console.error(` ${change.kind.padEnd(8)} ${change.path}`);
416
+ for (const change of changes.slice(0, 20)) {
417
+ console.error(` ${change.kind.padEnd(8)} ${change.path}${change.reason ? ` — ${change.reason}` : ""}`);
418
+ }
356
419
  if (changes.length > 20) console.error(` ... and ${changes.length - 20} more`);
420
+ if (since) console.error(` ${describeSince(since)}`);
357
421
 
358
422
  return true;
359
423
  }
360
424
 
361
425
  /**
362
- * When the source a fragment describes last changed, and at which commit
363
- * (portolan.0002).
364
- *
365
- * The host works this out rather than the extractor, for two reasons. A plugin
366
- * that reads a clock produces a different fragment on every run, which cannot
367
- * be committed and cannot be checked; and a plugin that shells out to git is a
368
- * plugin that can never be sandboxed. Stamped from the last commit to touch the
369
- * directory, a fragment changes exactly when its subject does.
370
- *
371
- * The output is excluded from that history, and it has to be: a fragment
372
- * written beside the code it describes is inside the directory it is stamped
373
- * from, so committing one would move the stamp, which would make the fragment
374
- * out of date, which would rewrite it - and `--check` would never come back
375
- * clean two runs in a row.
426
+ * What a step reads, for saying why its output moved: its root and whatever
427
+ * else it was handed, less every output directory inside the root, because a
428
+ * fragment written beside the code it describes is not an input to itself.
376
429
  */
377
- function stampFor(root, out) {
378
- // A copy of another repository is dated by the commit it is a copy OF, which
379
- // the fetch step wrote down beside it. Read before git, and without touching
380
- // git at all: the local history of a vendored directory says when somebody
381
- // ran the fetch, which is a fact about this repository and not about the
382
- // service the fragment describes.
383
- //
384
- // Short, like every other stamp, because a stamp is read rather than
385
- // resolved. The full sha travels separately, in the pin the fetch also
386
- // wrote, where a link is built from it.
387
- const vendored = vendoredCommit(root);
388
- if (vendored) {
389
- // No date. Nothing here knows when that commit was made - the lock records
390
- // what was fetched, not when it was authored - and a date invented from
391
- // the local clock would make the merge call this the stalest source in the
392
- // estate every time it ran.
393
- return { commit: vendored.slice(0, 7), generatedAt: "" };
394
- }
395
-
396
- // The history read is the one the root lives in. A manifest pointed at a
397
- // checkout elsewhere on the disk - the service being documented, not a copy
398
- // of it vendored here - is stamped from that checkout's history, because the
399
- // fragment describes that service and not the repository the manifest sits
400
- // in. A root with no repository around it is stamped as uncommitted.
401
- const repo = repositoryOf(root);
402
- if (!repo) {
403
- return { commit: "uncommitted", generatedAt: process.env.PORTOLAN_GENERATED_AT || new Date().toISOString() };
404
- }
405
-
406
- // A shallow clone has no history to read: the one commit that was fetched has
407
- // no parent, so every path looks as though it changed there and every fragment
408
- // is stamped with the checkout rather than with its subject. That is wrong
409
- // quietly - the fragments regenerate, `--check` reports drift, and nothing
410
- // says why - so it is refused here instead.
411
- if (shallow(repo)) {
412
- fail(
413
- `${repo} is a shallow clone, where every path looks as though it changed in ` +
414
- "the single commit that was fetched, so a fragment cannot be stamped " +
415
- "with the commit it describes. Fetch the full history first " +
416
- "(git fetch --unshallow, or actions/checkout with fetch-depth: 0).",
417
- );
430
+ function inputsOf(step, extra = []) {
431
+ const excludes = [];
432
+ for (const other of allSteps()) {
433
+ if (!other.out || other.out === step.in) continue;
434
+ const inside = relative(step.in, other.out);
435
+ if (inside && !inside.startsWith("..") && !isAbsolute(inside)) excludes.push(other.out);
418
436
  }
437
+ return { inputs: [step.in, ...extra], excludes };
438
+ }
419
439
 
420
- // Only an output INSIDE the root is excluded. An output beside it, or above
421
- // it, is not in the root's history to begin with - and excluding a parent
422
- // would exclude the root itself, leaving nothing to read and a stamp that
423
- // moved on every commit.
424
- const rootInRepo = relative(repo, resolve(root)) || ".";
425
- const outInRepo = out ? relative(repo, resolve(out)) : "";
426
- const inside = out && (outInRepo === rootInRepo || outInRepo.startsWith(`${rootInRepo.replace(/\/$/, "")}/`) || rootInRepo === ".");
427
- const exclude = inside && outInRepo !== rootInRepo && !outInRepo.startsWith("..") ? [`:(exclude)${outInRepo}`] : [];
428
-
429
- for (const args of [
430
- ["log", "-1", "--format=%h %cI", "--", rootInRepo, ...exclude],
431
- ["log", "-1", "--format=%h %cI"],
432
- ]) {
433
- const answer = git(repo, args);
434
- if (!answer) continue;
435
- const [commit, generatedAt] = answer.split(" ");
436
- if (commit && generatedAt) return { commit, generatedAt };
437
- }
440
+ function allSteps() {
441
+ return [...(manifest.extract ?? []), ...(manifest.verify ?? []), ...(manifest.generate ?? [])];
442
+ }
438
443
 
439
- return { commit: "uncommitted", generatedAt: process.env.PORTOLAN_GENERATED_AT || new Date().toISOString() };
444
+ /**
445
+ * Why a step's output is not what it was: the commit that last touched the
446
+ * output is when it was last generated, and what changed among the inputs
447
+ * since then is the reason. The history is the record of the last generation
448
+ * (portolan.0010); nothing is written down to know this.
449
+ *
450
+ * The manifest counts as an input only where this step's own entry moved.
451
+ * The whole file changes whenever any step is touched, and naming it for
452
+ * every other step would explain nothing.
453
+ */
454
+ function whyChanged(step, outputs, reads) {
455
+ const committed = lastCommitTouching(process.cwd(), outputs);
456
+ if (!committed) return { committed: null, changed: [] };
457
+ const changed = changedSince(process.cwd(), committed.commit, reads.inputs, reads.excludes);
458
+ if (stepEntryChanged(committed.commit, step)) changed.push("portolan.json (this step's entry)");
459
+ return { committed, changed };
440
460
  }
441
461
 
442
- /** Runs git in a repository and answers with its trimmed output, or "" when it refused. */
443
- function git(repo, args) {
462
+ /** Whether the manifest at `commit` told this step the same thing it is told now. */
463
+ function stepEntryChanged(commit, step) {
464
+ // A step no manifest declares has no entry to have changed.
465
+ if (step.key) return false;
466
+ let then;
444
467
  try {
445
- return execFileSync("git", ["-C", repo, ...args], {
446
- encoding: "utf8",
447
- stdio: ["ignore", "pipe", "ignore"],
448
- }).trim();
468
+ then = JSON.parse(fileAt(process.cwd(), commit, "portolan.json"));
449
469
  } catch {
450
- // Not a repository, or no commit touches this path yet. git has already
451
- // said so on its own stderr, which is not this run's log.
452
- return "";
470
+ return true; // No manifest there, or not one that parses: everything about the step is new.
453
471
  }
472
+ const phase = ["extract", "verify", "generate"].find((name) => (manifest[name] ?? []).includes(step));
473
+ const before = (then[phase] ?? []).find(
474
+ (other) =>
475
+ other.plugin === step.plugin &&
476
+ (other.in ?? "") === (step.in ?? "") &&
477
+ other.out === step.out &&
478
+ (other.options?.out ?? "") === (step.options?.out ?? ""),
479
+ );
480
+ return !before || JSON.stringify(before) !== JSON.stringify(step);
454
481
  }
455
482
 
456
- /** The working tree a directory belongs to, or "" when no repository holds it. */
457
- function repositoryOf(dir) {
458
- return git(dir, ["rev-parse", "--show-toplevel"]);
459
- }
460
-
461
- /** Whether the history this runs against is truncated. */
462
- function shallow(repo) {
463
- return git(repo, ["rev-parse", "--is-shallow-repository"]) === "true";
483
+ function describeSince({ committed, changed }) {
484
+ if (!committed) return "the output has never been committed, so there is nothing to compare its inputs against";
485
+ const when = `${committed.commit} (${committed.date}), when the output was last committed`;
486
+ if (changed.length === 0) return `no input changed since ${when}; the plugin itself did`;
487
+ const shown = changed.slice(0, 6).join(", ");
488
+ const more = changed.length > 6 ? ` and ${changed.length - 6} more` : "";
489
+ return `since ${when}, ${changed.length} input${changed.length === 1 ? "" : "s"} changed: ${shown}${more}`;
464
490
  }
465
491
 
466
492
  /** Reads, merges and validates every source the manifest names. */
@@ -497,7 +523,11 @@ function apply(files, out, key, checkOnly) {
497
523
 
498
524
  if (binary ? Buffer.isBuffer(current) && current.equals(wanted) : current === wanted) continue;
499
525
 
500
- changes.push({ kind: current === null ? "added" : "changed", path: join(out, file.name) });
526
+ changes.push({
527
+ kind: current === null ? "added" : "changed",
528
+ path: join(out, file.name),
529
+ reason: explainChange(current, wanted, file.name),
530
+ });
501
531
  if (checkOnly) continue;
502
532
 
503
533
  try {
@@ -549,7 +579,7 @@ function apply(files, out, key, checkOnly) {
549
579
  function sweep(out, checkOnly) {
550
580
  const changes = [];
551
581
  const listing = previous(out);
552
- const live = keys.liveIn(out);
582
+ const live = liveKeysIn(out);
553
583
  const dead = Object.keys(listing).filter((key) => !live.has(key));
554
584
  if (dead.length === 0) return changes;
555
585
 
@@ -0,0 +1,108 @@
1
+ // The example GitOps tree, rendered the way Argo CD would render it.
2
+ //
3
+ // Every overlay under examples/gitops/envs/*/shop/* is built with kustomize
4
+ // - the binary when it is on the PATH, kubectl's copy otherwise - and what
5
+ // the environment says (the tag, the replica count, the variable) has to be
6
+ // what comes out. Then the tree is held to the snapshot in examples/argocd:
7
+ // an overlay that no Application in the snapshot deploys, or an Application
8
+ // pointing at a directory that is not here, is the two examples drifting
9
+ // apart, which is the one thing an example must not do.
10
+ import { execFileSync } from "node:child_process";
11
+ import { existsSync, readdirSync, readFileSync } from "node:fs";
12
+ import { join } from "node:path";
13
+ import { fileURLToPath } from "node:url";
14
+ import { describe, expect, it } from "vitest";
15
+
16
+ const root = fileURLToPath(new URL("..", import.meta.url));
17
+ const gitops = join(root, "examples", "gitops");
18
+
19
+ /** kustomize itself, else the copy kubectl ships; the output is the same. */
20
+ function kustomizeCommand() {
21
+ for (const candidate of ["kustomize"]) {
22
+ try {
23
+ execFileSync(candidate, ["version"], { stdio: "ignore" });
24
+ return [candidate, "build"];
25
+ } catch {
26
+ // Not on the PATH.
27
+ }
28
+ }
29
+ return ["kubectl", "kustomize"];
30
+ }
31
+
32
+ const [command, ...args] = kustomizeCommand();
33
+ const build = (dir) => execFileSync(command, [...args, dir], { encoding: "utf8" });
34
+
35
+ /** `<env>/shop/<service>` for every overlay in the tree, sorted. */
36
+ function overlays() {
37
+ const out = [];
38
+ for (const env of readdirSync(join(gitops, "envs")).sort()) {
39
+ const shop = join(gitops, "envs", env, "shop");
40
+ if (!existsSync(shop)) continue;
41
+ for (const service of readdirSync(shop).sort()) {
42
+ if (existsSync(join(shop, service, "kustomization.yaml"))) out.push({ env, service, dir: join(shop, service) });
43
+ }
44
+ }
45
+ return out;
46
+ }
47
+
48
+ describe("the example GitOps tree", () => {
49
+ it("has an overlay for cart in both environments and pricing in prod only", () => {
50
+ expect(overlays().map((o) => `${o.env}/${o.service}`)).toEqual(["prod/cart", "prod/pricing", "staging/cart"]);
51
+ });
52
+
53
+ it("renders cart in prod at the tag prod runs, three replicas, labelled with the environment", () => {
54
+ const rendered = build(join(gitops, "envs", "prod", "shop", "cart"));
55
+ expect(rendered).toContain("image: ghcr.io/shortlink-org/cart:2.1.0");
56
+ expect(rendered).toContain("replicas: 3");
57
+ expect(rendered).toContain("env: prod");
58
+ expect(rendered).toContain("app.kubernetes.io/part-of: shop");
59
+ // The base's names and namespace survive the overlay.
60
+ expect(rendered).toMatch(/kind: Ingress[\s\S]*namespace: shop/);
61
+ });
62
+
63
+ it("renders cart in staging as the release candidate, one replica, NODE_ENV restated and nothing else lost", () => {
64
+ const rendered = build(join(gitops, "envs", "staging", "shop", "cart"));
65
+ expect(rendered).toContain("image: ghcr.io/shortlink-org/cart:2.2.0-rc.1");
66
+ expect(rendered).toContain("replicas: 1");
67
+ expect(rendered).toMatch(/name: NODE_ENV\n\s+value: staging/);
68
+ // Merged by name: the other variables the base declares are still there.
69
+ expect(rendered).toMatch(/name: PRICING_ADDR\n\s+value: pricing\.shop\.svc:9090/);
70
+ expect(rendered).not.toContain("value: production");
71
+ });
72
+
73
+ it("renders pricing in prod with the longer cache and the secret untouched", () => {
74
+ const rendered = build(join(gitops, "envs", "prod", "shop", "pricing"));
75
+ expect(rendered).toContain("image: ghcr.io/shortlink-org/pricing:1.4.2");
76
+ expect(rendered).toContain("PRICE_LIST_TTL: 1h");
77
+ expect(rendered).toContain("OTEL_EXPORTER_OTLP_ENDPOINT");
78
+ expect(rendered).toContain("kind: Secret");
79
+ });
80
+
81
+ it("is what the snapshot in examples/argocd was recorded from", () => {
82
+ const snapshot = JSON.parse(readFileSync(join(root, "examples", "argocd", "argocd.apps.json"), "utf8"));
83
+ const placed = snapshot.deployments.filter((d) => d.project === "shop");
84
+ // One Application per overlay, named as the ApplicationSet names it,
85
+ // pointing at the overlay, labelled onto the service.
86
+ expect(placed.map((d) => [d.name, d.environment, d.path, d.service]).sort()).toEqual(
87
+ overlays().map((o) => [`shop-${o.service}-${o.env}`, o.env, `examples/gitops/envs/${o.env}/shop/${o.service}`, `shop.${o.service}`]).sort(),
88
+ );
89
+ // The image the snapshot says runs is the image the overlay renders.
90
+ for (const deployment of placed) {
91
+ const rendered = build(join(root, deployment.path));
92
+ for (const image of deployment.images ?? []) expect(rendered).toContain(`image: ${image}`);
93
+ }
94
+ // The clusters the environments name are the ones the snapshot places in.
95
+ for (const { env } of overlays()) {
96
+ const cluster = readFileSync(join(gitops, "envs", env, "cluster.yaml"), "utf8").match(/^\s+name: (.+)$/m)?.[1];
97
+ for (const deployment of placed.filter((d) => d.environment === env)) expect(deployment.cluster).toBe(cluster);
98
+ }
99
+ });
100
+
101
+ it("names every Application in the shop ApplicationSet with the labels the fetcher places by", () => {
102
+ const appset = readFileSync(join(gitops, "appsets", "shop.yaml"), "utf8");
103
+ expect(appset).toContain("app.kubernetes.io/part-of: shop");
104
+ expect(appset).toContain('app.kubernetes.io/name: "{{.path.basename}}"');
105
+ expect(appset).toContain('name: "shop-{{.path.basename}}-{{.env}}"');
106
+ expect(appset).toContain("path: examples/gitops/envs/*/cluster.yaml");
107
+ });
108
+ });
@@ -0,0 +1,30 @@
1
+ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { dirname, join } from "node:path";
4
+ import { afterEach, expect, it } from "vitest";
5
+ import { discoverProject } from "./local-discovery.mjs";
6
+
7
+ const roots = [];
8
+ afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
9
+ function discover(files) {
10
+ const root = mkdtempSync(join(tmpdir(), "portolan-go-discovery-"));
11
+ roots.push(root);
12
+ for (const [name, text] of Object.entries({ "go.mod": "module example.com/app\n", ...files })) {
13
+ const path = join(root, name);
14
+ mkdirSync(dirname(path), { recursive: true });
15
+ writeFileSync(path, text);
16
+ }
17
+ return discoverProject(root, ".").detections.map((item) => item.plugin);
18
+ }
19
+ it("offers Go flows and SQL for a service without any DDD directories", () => {
20
+ expect(discover({
21
+ "web/routes.go": 'package web\nimport "net/http"\nfunc Register(m *http.ServeMux) { m.HandleFunc("POST /orders", create) }',
22
+ "store/migrations/001.sql": "CREATE TABLE orders (id uuid PRIMARY KEY);",
23
+ })).toEqual(expect.arrayContaining(["go-domain", "sql"]));
24
+ });
25
+ it("does not offer Go domain extraction for unregistered helpers or generated handlers", () => {
26
+ for (const source of [
27
+ 'package web\nimport "net/http"\nfunc Help(w http.ResponseWriter, r *http.Request) {}',
28
+ '// Code generated by fixture. DO NOT EDIT.\npackage web\nimport "net/http"\nfunc Register() { http.HandleFunc("/x", help) }',
29
+ ]) expect(discover({ "web/routes.go": source })).not.toContain("go-domain");
30
+ });