@shortlink-org/portolan 0.2.4 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +23 -0
  2. package/catalog/enum_test.go +46 -0
  3. package/catalog/evidence_test.go +35 -0
  4. package/catalog/model.go +1066 -0
  5. package/catalog/roundtrip_test.go +203 -0
  6. package/catalog/via_test.go +38 -0
  7. package/cli/init.test.mjs +6 -1
  8. package/cli/portolan.mjs +8 -0
  9. package/cli/portolan.test.mjs +49 -0
  10. package/go.mod +14 -0
  11. package/go.sum +20 -0
  12. package/internal/gocall/README.md +19 -0
  13. package/internal/gocall/analyze.go +189 -0
  14. package/internal/gocall/analyze_test.go +107 -0
  15. package/internal/gohttp/analyze.go +2562 -0
  16. package/internal/gohttp/destination.go +373 -0
  17. package/internal/gohttp/endpoints.go +1067 -0
  18. package/internal/gohttp/roots.go +320 -0
  19. package/internal/gohttp/typed.go +96 -0
  20. package/internal/goscan/constants.go +85 -0
  21. package/internal/goscan/goscan_test.go +227 -0
  22. package/internal/goscan/index.go +629 -0
  23. package/internal/goscan/index_test.go +66 -0
  24. package/internal/goscan/names.go +52 -0
  25. package/internal/goscan/parse_test.go +11 -0
  26. package/internal/goscan/source.go +37 -0
  27. package/internal/goscan/tree.go +284 -0
  28. package/internal/goscan/types.go +99 -0
  29. package/internal/wsdl/ids.go +127 -0
  30. package/internal/wsdl/ids_test.go +21 -0
  31. package/internal/wsdl/model.go +70 -0
  32. package/internal/wsdl/parse.go +949 -0
  33. package/internal/wsdl/parse_test.go +170 -0
  34. package/package.json +22 -10
  35. package/plugin/describe.go +118 -0
  36. package/plugin/describe_test.go +114 -0
  37. package/plugin/protocol.go +141 -0
  38. package/plugin/schematest/schematest.go +126 -0
  39. package/plugins/README.md +109 -46
  40. package/plugins/cmd/portolan-http-clients/main.go +19 -0
  41. package/plugins/extract-celery/extract.py +0 -2
  42. package/plugins/extract-celery/extract_test.py +1 -1
  43. package/plugins/extract-django/README.md +39 -17
  44. package/plugins/extract-django/domain.py +28 -17
  45. package/plugins/extract-django/extract.py +21 -7
  46. package/plugins/extract-django/extract_test.py +55 -2
  47. package/plugins/extract-django/lifecycle.py +2 -0
  48. package/plugins/extract-django/operations.py +1 -1
  49. package/plugins/extract-django/routing_test.py +109 -1
  50. package/plugins/extract-django/store.py +1 -1
  51. package/plugins/extract-django/transport.py +101 -55
  52. package/plugins/extract-django/verbs.py +241 -0
  53. package/plugins/extract-go/README.md +47 -0
  54. package/plugins/extract-http-clients/describe.go +19 -0
  55. package/plugins/extract-http-clients/describe_test.go +11 -0
  56. package/plugins/extract-http-clients/extract.go +740 -0
  57. package/plugins/extract-http-clients/extract_test.go +1561 -0
  58. package/plugins/extract-http-clients/main.go +41 -0
  59. package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
  60. package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
  61. package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
  62. package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
  63. package/plugins/extract-python-kafka/extract.py +0 -2
  64. package/plugins/extract-python-kafka/extract_test.py +1 -1
  65. package/plugins/extract-ts/extract.test.ts +2 -2
  66. package/plugins/extract-ts/extract.ts +4 -5
  67. package/plugins/extract-ts/graphql.test.ts +1 -1
  68. package/plugins/openapi/ids.go +261 -0
  69. package/plugins/openapi/ids_test.go +98 -0
  70. package/plugins/portolan-go.wasm +0 -0
  71. package/plugins/pyplugin/protocol.py +1 -5
  72. package/portolan.json +3 -2
  73. package/schema/portolan.schema.json +34 -0
  74. package/scripts/README.md +18 -12
  75. package/scripts/catalog-sources.mjs +6 -0
  76. package/scripts/delivery-presets.mjs +21 -11
  77. package/scripts/diff.mjs +5 -1
  78. package/scripts/django-aggregates.test.mjs +58 -0
  79. package/scripts/gen-likec4.mjs +1 -1
  80. package/scripts/gen.mjs +118 -115
  81. package/scripts/go-discovery.test.mjs +30 -0
  82. package/scripts/history.mjs +186 -3
  83. package/scripts/history.test.mjs +1 -1
  84. package/scripts/host-plugins/fetch-git.mjs +77 -21
  85. package/scripts/host-plugins/fetch-git.test.mjs +62 -8
  86. package/scripts/local-api.mjs +71 -4
  87. package/scripts/local-api.test.mjs +63 -4
  88. package/scripts/local-discovery.mjs +82 -9
  89. package/scripts/manifest.mjs +5 -3
  90. package/scripts/manifest.test.mjs +24 -0
  91. package/scripts/output-diff.mjs +94 -0
  92. package/scripts/output-diff.test.mjs +36 -0
  93. package/scripts/package-smoke.mjs +62 -4
  94. package/scripts/plugin-host.mjs +22 -2
  95. package/scripts/plugin-host.test.mjs +9 -0
  96. package/scripts/plugin-wasm-worker.mjs +4 -1
  97. package/scripts/provenance.mjs +72 -0
  98. package/scripts/provenance.test.mjs +149 -0
  99. package/scripts/run-builtin.mjs +39 -5
  100. package/scripts/schema.mjs +29 -0
  101. package/scripts/warning-policy.mjs +161 -0
  102. package/scripts/warning-policy.test.mjs +56 -0
  103. package/src/app/Sidebar.tsx +3 -3
  104. package/src/catalog-docs.test.ts +64 -0
  105. package/src/catalog-docs.ts +35 -0
  106. package/src/catalog-error.test.ts +15 -0
  107. package/src/catalog-model.ts +48 -5
  108. package/src/catalog-validation.ts +9 -0
  109. package/src/chat/Starter.tsx +5 -11
  110. package/src/chat/tools.test.ts +27 -0
  111. package/src/chat/tools.ts +5 -9
  112. package/src/components/CatalogStamp.tsx +10 -8
  113. package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
  114. package/src/components/HTTPDestinationEvidence.tsx +31 -0
  115. package/src/components/Integrations.tsx +1 -1
  116. package/src/components/MachineDocs.tsx +6 -5
  117. package/src/components/MethodRows.tsx +9 -2
  118. package/src/components/RelationEvidence.test.tsx +14 -0
  119. package/src/components/RelationEvidence.tsx +53 -0
  120. package/src/data.ts +25 -7
  121. package/src/enrich.test.ts +332 -1
  122. package/src/enrich.ts +206 -3
  123. package/src/flow/StepDetail.tsx +6 -0
  124. package/src/flow/evidence.test.ts +16 -0
  125. package/src/flow/evidence.ts +34 -0
  126. package/src/index.css +44 -0
  127. package/src/landing/DraggableReveal.tsx +3 -2
  128. package/src/landing/EvidencePipeline.tsx +105 -0
  129. package/src/landing/LandingPage.tsx +2 -59
  130. package/src/lib/catalog-diff.ts +1 -1
  131. package/src/lib/django-aggregates.d.mts +9 -0
  132. package/src/lib/django-aggregates.mjs +36 -0
  133. package/src/lib/django-aggregates.test.ts +29 -0
  134. package/src/lib/django-aggregates.ts +5 -0
  135. package/src/lib/local-api.ts +20 -2
  136. package/src/lib/setup-info.test.ts +17 -0
  137. package/src/lib/setup-info.ts +58 -0
  138. package/src/lib/warnings.test.ts +54 -0
  139. package/src/lib/warnings.ts +260 -0
  140. package/src/map/ContextMapGraph.tsx +76 -32
  141. package/src/merge.ts +16 -9
  142. package/src/pages/AggregatePage.tsx +8 -7
  143. package/src/pages/ContextPage.tsx +6 -5
  144. package/src/pages/ServicePage.tsx +4 -3
  145. package/src/pages/Settings.tsx +189 -41
  146. package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
  147. package/src/selection/DetailPanel.tsx +15 -0
  148. package/src/virtual-provenance.d.ts +11 -0
  149. package/vite.config.ts +5 -0
  150. package/scripts/vendor-lock.mjs +0 -58
  151. package/scripts/vendor-lock.test.mjs +0 -69
@@ -23,7 +23,10 @@ const GITLAB_START = "# >>> Portolan delivery preset >>>";
23
23
  const GITLAB_END = "# <<< Portolan delivery preset <<<";
24
24
  const PROVIDERS = new Set(["github", "gitlab"]);
25
25
  const FEATURE_IDS = ["check", "diff", "sarif", "pages"];
26
- const DEFAULT_FEATURES = ["check", "pages"];
26
+ const DEFAULT_FEATURES = {
27
+ github: ["check", "pages"],
28
+ gitlab: ["pages"],
29
+ };
27
30
  const FEATURE_DETAILS = {
28
31
  check: {
29
32
  label: "Architecture check",
@@ -225,7 +228,9 @@ ${upload}`;
225
228
 
226
229
  function gitlabCheck() {
227
230
  return `"portolan:check":
228
- stage: .pre
231
+ stage: test
232
+ tags:
233
+ - runner-type:docker
229
234
  image:
230
235
  name: ghcr.io/shortlink-org/portolan:${VERSION}
231
236
  entrypoint: [""]
@@ -242,7 +247,9 @@ function gitlabCheck() {
242
247
  function gitlabDiff({ pages }) {
243
248
  const site = pages ? ' --site "$CI_PAGES_URL"' : "";
244
249
  return `"portolan:review":
245
- stage: .pre
250
+ stage: test
251
+ tags:
252
+ - runner-type:docker
246
253
  image:
247
254
  name: ghcr.io/shortlink-org/portolan:${VERSION}
248
255
  entrypoint: [""]
@@ -259,8 +266,10 @@ function gitlabDiff({ pages }) {
259
266
  }
260
267
 
261
268
  function gitlabPages() {
262
- return `"portolan:pages":
263
- stage: .post
269
+ return `pages:
270
+ stage: deploy
271
+ tags:
272
+ - runner-type:docker
264
273
  image:
265
274
  name: ghcr.io/shortlink-org/portolan:${VERSION}
266
275
  entrypoint: [""]
@@ -268,9 +277,10 @@ function gitlabPages() {
268
277
  GIT_DEPTH: "0"
269
278
  script:
270
279
  - BASE_PATH="$(node -p 'new URL(process.env.CI_PAGES_URL).pathname.replace(/\\/?$/, "/") || "/"')"
271
- - portolan build --output dist --base "$BASE_PATH"
272
- pages:
273
- publish: dist
280
+ - portolan build --output public --base "$BASE_PATH"
281
+ artifacts:
282
+ paths:
283
+ - public
274
284
  rules:
275
285
  - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
276
286
  `;
@@ -299,7 +309,7 @@ function mergeGitlab(existing, features) {
299
309
  .replace(/^\n+|\n+$/g, "");
300
310
  return { content: content ? `${content}\n` : "" };
301
311
  }
302
- if (/^[ \t]*["']?portolan:(?:check|review|pages)["']?\s*:/m.test(existing)) {
312
+ if (/^(?:["']?portolan:(?:check|review|pages)["']?|["']?pages["']?)\s*:/m.test(existing)) {
303
313
  return { conflict: "This pipeline already declares a Portolan job outside the managed region." };
304
314
  }
305
315
  const block = gitlabBlock(features);
@@ -421,14 +431,14 @@ function installedFeatures(workspace, provider) {
421
431
  );
422
432
  if (/^[ \t]*["']?portolan:check["']?\s*:/m.test(managed)) selected.add("check");
423
433
  if (/^[ \t]*["']?portolan:review["']?\s*:/m.test(managed)) selected.add("diff");
424
- if (/^[ \t]*["']?portolan:pages["']?\s*:/m.test(managed)) selected.add("pages");
434
+ if (/^(?:["']?portolan:pages["']?|["']?pages["']?)\s*:/m.test(managed)) selected.add("pages");
425
435
  return selected;
426
436
  }
427
437
 
428
438
  function selectedFeatures(workspace, request, provider) {
429
439
  const installed = request.features == null ? installedFeatures(workspace, provider) : null;
430
440
  const raw = request.features == null
431
- ? installed.size > 0 ? [...installed] : DEFAULT_FEATURES
441
+ ? installed.size > 0 ? [...installed] : DEFAULT_FEATURES[provider]
432
442
  : Array.isArray(request.features)
433
443
  ? request.features
434
444
  : String(request.features).split(",").filter(Boolean);
package/scripts/diff.mjs CHANGED
@@ -157,8 +157,12 @@ function catalogAt(ref) {
157
157
  throw new Error(ref + " holds no catalog sources matching " + JSON.stringify(manifest.sources));
158
158
  }
159
159
 
160
+ // The base is one commit, and a commit is its own provenance: every source
161
+ // there is dated by it (portolan.0010), whatever an older fragment says.
162
+ const [commit = "", generatedAt = ""] = git(["log", "-1", "--format=%h %cI", ref]).split(" ");
163
+ const stamp = { commit, generatedAt };
160
164
  const merged = mergeCatalogs(
161
- paths.map((path) => ({ path, catalog: JSON.parse(git(["show", ref + ":" + path])) })),
165
+ paths.map((path) => ({ path, catalog: JSON.parse(git(["show", ref + ":" + path])), stamp })),
162
166
  );
163
167
 
164
168
  // Enriched before it is compared, exactly as the app and the generators see
@@ -0,0 +1,58 @@
1
+ import { createHash } from "node:crypto";
2
+ import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+ import { afterEach, describe, expect, it } from "vitest";
6
+ import { djangoAggregateProposals, saveDjangoAggregates } from "./local-api.mjs";
7
+
8
+ const roots = [];
9
+ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
10
+ function fixture() {
11
+ const root = mkdtempSync(join(tmpdir(), "portolan-aggregates-")); roots.push(root);
12
+ const manifest = { sources: ["data/*.json"], extract: ["one", "two"].map((name) => ({ plugin: "django-domain", in: name, out: `data/${name}`, options: { context: name, service: name, aggregates: { existing: "Existing" }, apps: ["billing.records"] } })) };
13
+ const text = JSON.stringify(manifest);
14
+ writeFileSync(join(root, "portolan.json"), text);
15
+ const message = `billing/records: no model called Records, and 2 models to choose from: name the root in the aggregates option; aggregate candidates: ${JSON.stringify({ app: "billing.records", models: [{ name: "Entry", path: "billing/records/models.py", line: 1 }, { name: "Audit", path: "billing/records/models.py", line: 10 }] })}`;
16
+ const report = { status: "ok", manifestSha256: createHash("sha256").update(text).digest("hex"), steps: manifest.extract.map((step) => ({ phase: "extract", plugin: step.plugin, input: step.in, output: step.out, warnings: [message] })) };
17
+ mkdirSync(join(root, ".portolan")); writeFileSync(join(root, ".portolan/build-report.json"), JSON.stringify(report));
18
+ return { root, manifest, report };
19
+ }
20
+
21
+ describe("saving Django aggregate roots", () => {
22
+ it("targets the precise step and dotted app, preserving unrelated options", () => {
23
+ const { root, manifest } = fixture();
24
+ const proposals = djangoAggregateProposals(root);
25
+ expect(proposals.stale).toBe(false);
26
+ expect(proposals.proposals).toHaveLength(2);
27
+ saveDjangoAggregates(root, { revision: proposals.revision, selections: [{ id: "1:billing.records", model: "Entry" }] });
28
+ const saved = JSON.parse(readFileSync(join(root, "portolan.json"), "utf8"));
29
+ expect(saved.extract[0]).toEqual(manifest.extract[0]);
30
+ expect(saved.extract[1].options).toEqual({ ...manifest.extract[1].options, aggregates: { existing: "Existing", "billing.records": "Entry" } });
31
+ expect(djangoAggregateProposals(root).stale).toBe(true);
32
+ });
33
+
34
+ it("saves a batch atomically and rejects forged, duplicate, or stale choices", () => {
35
+ const { root } = fixture();
36
+ const { revision } = djangoAggregateProposals(root);
37
+ const original = readFileSync(join(root, "portolan.json"), "utf8");
38
+ for (const selections of [[], [{ id: "0:billing.records", model: "Proxy" }], [{ id: "no-such-step", model: "Entry" }], [{ id: "0:billing.records", model: "Entry" }, { id: "1:billing.records", model: "Invalid" }], [{ id: "0:billing.records", model: "Entry" }, { id: "0:billing.records", model: "Audit" }]]) {
39
+ expect(() => saveDjangoAggregates(root, { revision, selections })).toThrow();
40
+ expect(readFileSync(join(root, "portolan.json"), "utf8")).toBe(original);
41
+ }
42
+ const selections = [{ id: "0:billing.records", model: "Entry" }, { id: "1:billing.records", model: "Audit" }];
43
+ expect(() => saveDjangoAggregates(root, { revision: "old", selections })).toThrow(/changed/);
44
+ expect(saveDjangoAggregates(root, { revision, selections })).toEqual({ saved: 2 });
45
+ expect(() => saveDjangoAggregates(root, { revision, selections })).toThrow(/changed/);
46
+ });
47
+
48
+ it("requires fresh report evidence and refuses ambiguous step bindings", () => {
49
+ const { root, manifest, report } = fixture();
50
+ manifest.extract.push(manifest.extract[0]);
51
+ writeFileSync(join(root, "portolan.json"), JSON.stringify(manifest));
52
+ expect(djangoAggregateProposals(root).proposals.map((p) => p.id)).toEqual(["1:billing.records"]);
53
+ expect(djangoAggregateProposals(root).stale).toBe(true);
54
+ report.steps = [];
55
+ writeFileSync(join(root, ".portolan/build-report.json"), JSON.stringify(report));
56
+ expect(djangoAggregateProposals(root).proposals).toEqual([]);
57
+ });
58
+ });
@@ -278,7 +278,7 @@ for (const context of catalog.contexts) {
278
278
  model.push(` style { color ${contextColorName(context.id)} }`);
279
279
  for (const aggregate of service.aggregates) {
280
280
  model.push(
281
- ` ${safeId(aggregate.slug)} = aggregate ${q(aggregate.name)} {`,
281
+ ` ${safeId(aggregate.slug)} = aggregate ${q(aggregate.kind === "model-group" ? `${aggregate.name} (model group)` : aggregate.name)} {`,
282
282
  );
283
283
  for (const event of aggregate.events) {
284
284
  const latest = event.versions[event.versions.length - 1];
package/scripts/gen.mjs CHANGED
@@ -15,9 +15,8 @@ import {
15
15
  rmSync,
16
16
  statSync,
17
17
  } from "node:fs";
18
- import { execFileSync } from "node:child_process";
19
18
  import { createHash } from "node:crypto";
20
- import { join, relative, resolve } from "node:path";
19
+ import { isAbsolute, join, relative } from "node:path";
21
20
 
22
21
  import {
23
22
  addBuildStep,
@@ -26,16 +25,17 @@ import {
26
25
  writeBuildReport,
27
26
  } from "./build-report.mjs";
28
27
  import { loadCatalog } from "./catalog-sources.mjs";
29
- import { historyFor } from "./history.mjs";
28
+ import { changedSince, fileAt, historyFor, lastCommitTouching } from "./history.mjs";
30
29
  import { loadManifest, stepKeys } from "./manifest.mjs";
31
30
  import { describePlugin, runPlugin } from "./plugin-host.mjs";
32
- import { vendoredCommit } from "./vendor-lock.mjs";
31
+ import { explainChange } from "./output-diff.mjs";
33
32
  import {
34
33
  removeOutputFile,
35
34
  safeOutputPath,
36
35
  writeOutputFile,
37
36
  } from "./output-path.mjs";
38
37
  import { builtinPlugin } from "./builtin-plugins.mjs";
38
+ import { diagnoseWarnings } from "./warning-policy.mjs";
39
39
 
40
40
  const PORTOLAN_VERSION = "0.1.0";
41
41
  const EVENTS = process.env.PORTOLAN_EVENTS === "1";
@@ -122,9 +122,12 @@ async function generate() {
122
122
 
123
123
  // Extractors run first and write catalog fragments; only then is there a
124
124
  // catalog for anything else to read.
125
+ //
126
+ // No stamp travels with the request (portolan.0010). A fragment is content
127
+ // and nothing else; when it last changed is what the history of the
128
+ // fragment says, read wherever the catalog is read.
125
129
  for (const step of manifest.extract ?? []) {
126
130
  const plugin = pluginNamed(step.plugin);
127
- const stamp = stampFor(step.in, step.out);
128
131
  // A plugin that asks for history gets the root's, read once per checkout
129
132
  // (portolan.0007); left out when the root is not in a checkout, which the
130
133
  // plugin reports in its own words.
@@ -135,13 +138,11 @@ async function generate() {
135
138
  input: {
136
139
  root: step.in,
137
140
  output: step.out,
138
- commit: stamp.commit,
139
- generatedAt: stamp.generatedAt,
140
141
  ...(history ? { history } : {}),
141
142
  },
142
143
  options: step.options ?? {},
143
144
  }, {}, { workspace: process.cwd() }),
144
- );
145
+ inputsOf(step));
145
146
  }
146
147
 
147
148
  // Verifiers read observed evidence against the merged catalog while leaving
@@ -152,17 +153,16 @@ async function generate() {
152
153
  // carry it into the generators and into the next run.
153
154
  for (const step of manifest.verify ?? []) {
154
155
  const plugin = pluginNamed(step.plugin);
155
- const stamp = stampFor(step.in, step.out);
156
156
  const own = (previous(step.out)[keys.keyOf(step)] ?? []).map((name) => join(step.out, name));
157
- const { catalog } = await loadSources({ exclude: [...own, ...staleAll()] });
157
+ const { catalog, sources } = await loadSources({ exclude: [...own, ...staleAll()] });
158
158
  await executeStep("verify", step, `${step.plugin} ⇐ ${step.in}`, async () =>
159
159
  runPlugin(plugin, {
160
160
  portolanVersion: PORTOLAN_VERSION,
161
- input: { root: step.in, output: step.out, commit: stamp.commit, generatedAt: stamp.generatedAt },
161
+ input: { root: step.in, output: step.out },
162
162
  catalog,
163
163
  options: step.options ?? {},
164
164
  }, {}, { workspace: process.cwd() }),
165
- );
165
+ inputsOf(step, sources.map((source) => source.path)));
166
166
  }
167
167
 
168
168
  // What a dropped extract step wrote is not part of the catalog, so it is
@@ -182,16 +182,16 @@ async function generate() {
182
182
 
183
183
  for (const step of manifest.generate ?? []) {
184
184
  const plugin = pluginNamed(step.plugin);
185
- const generatedCatalog = step.catalog
186
- ? (await loadSources({ profile: step.catalog })).catalog
187
- : catalog;
185
+ const generated = step.catalog
186
+ ? await loadSources({ profile: step.catalog })
187
+ : { catalog, sources };
188
188
  await executeStep("generate", step, `${step.plugin} → ${step.out}`, async () =>
189
189
  runPlugin(plugin, {
190
190
  portolanVersion: PORTOLAN_VERSION,
191
- catalog: generatedCatalog,
191
+ catalog: generated.catalog,
192
192
  options: step.options ?? {},
193
193
  }),
194
- );
194
+ { inputs: generated.sources.map((source) => source.path), excludes: [] });
195
195
  }
196
196
 
197
197
  sweepAll();
@@ -249,13 +249,27 @@ function staleIn(out) {
249
249
  return stale;
250
250
  }
251
251
 
252
- async function executeStep(phase, step, label, work) {
252
+ /**
253
+ * Runs one step and settles its files. `reads` is what the step reads - paths,
254
+ * and the output directories among them to leave out - so that an output
255
+ * that moved can be explained by what moved among its inputs since the
256
+ * output was last committed.
257
+ */
258
+ async function executeStep(phase, step, label, work, reads) {
253
259
  const startedAt = Date.now();
254
260
  event({ type: "step-started", ordinal: report.steps.length, phase, plugin: step.plugin, input: step.in, output: step.out });
255
261
  try {
256
262
  const { files, warnings = [] } = await work();
263
+ const diagnostics = diagnoseWarnings({
264
+ plugin: step.plugin,
265
+ warnings,
266
+ policies: manifest.warningPolicies,
267
+ project: projectForStep(step),
268
+ phase,
269
+ });
257
270
  const changes = apply(files, step.out, keys.keyOf(step), check);
258
- const changed = summarise(label, files, changes, warnings);
271
+ const since = changes.length > 0 ? whyChanged(step, files.map((file) => join(step.out, file.name)), reads) : null;
272
+ const changed = summarise(label, files, changes, diagnostics, since);
259
273
  drifted = changed || drifted;
260
274
  const result = {
261
275
  phase,
@@ -267,10 +281,14 @@ async function executeStep(phase, step, label, work) {
267
281
  fileCount: files.length,
268
282
  changedCount: changes.length,
269
283
  changes,
284
+ // What moved among the inputs since the output was last committed, when
285
+ // something in the output moved; the Settings page can say why.
286
+ ...(since ? { since } : {}),
270
287
  files: files.map((file) => join(step.out, file.name)),
271
288
  // What the plugin could not read, in its own words. Already printed as
272
289
  // it ran; kept here so the Settings page can list it beside the step.
273
290
  warnings,
291
+ diagnostics,
274
292
  };
275
293
  addBuildStep(report, result);
276
294
  persistReport();
@@ -296,6 +314,15 @@ async function executeStep(phase, step, label, work) {
296
314
  }
297
315
  }
298
316
 
317
+ function projectForStep(step) {
318
+ if (!step.in) return "";
319
+ const input = String(step.in).replace(/\\/g, "/").replace(/\/+$/, "");
320
+ const projects = (manifest.projects ?? [])
321
+ .filter((project) => typeof project?.id === "string" && typeof project?.root === "string")
322
+ .sort((left, right) => right.root.length - left.root.length);
323
+ return projects.find((project) => input === project.root || input.startsWith(`${project.root}/`))?.id ?? "";
324
+ }
325
+
299
326
  function persistReport() {
300
327
  try {
301
328
  writeBuildReport(report);
@@ -334,9 +361,17 @@ async function needsOf(plugin) {
334
361
  return pluginNeeds.get(plugin.name);
335
362
  }
336
363
 
337
- /** Prints what a step did, and says whether it left the tree out of date. */
338
- function summarise(label, files, changes, warnings = []) {
339
- const said = warnings.length > 0 ? `, ${warnings.length} warning${warnings.length === 1 ? "" : "s"}` : "";
364
+ /**
365
+ * Prints what a step did, and says whether it left the tree out of date. A
366
+ * file that differs says where it first differs, and the step says what moved
367
+ * among its inputs since the output was last committed - or that nothing
368
+ * did, which points at the plugin.
369
+ */
370
+ function summarise(label, files, changes, diagnostics = [], since = null) {
371
+ const suppressed = diagnostics.filter((diagnostic) => diagnostic.suppressed).length;
372
+ const said = diagnostics.length > 0
373
+ ? `, ${diagnostics.length} warning${diagnostics.length === 1 ? "" : "s"}${suppressed ? ` (${suppressed} suppressed)` : ""}`
374
+ : "";
340
375
  const summary = `${label}: ${files.length} file${files.length === 1 ? "" : "s"}${said}`;
341
376
 
342
377
  if (changes.length === 0) {
@@ -347,120 +382,84 @@ function summarise(label, files, changes, warnings = []) {
347
382
 
348
383
  if (!check) {
349
384
  console.log(`${summary}, ${changes.length} written`);
385
+ if (since) console.log(` ${describeSince(since)}`);
350
386
 
351
387
  return false;
352
388
  }
353
389
 
354
390
  console.error(`${summary}, ${changes.length} out of date:`);
355
- for (const change of changes.slice(0, 20)) console.error(` ${change.kind.padEnd(8)} ${change.path}`);
391
+ for (const change of changes.slice(0, 20)) {
392
+ console.error(` ${change.kind.padEnd(8)} ${change.path}${change.reason ? ` — ${change.reason}` : ""}`);
393
+ }
356
394
  if (changes.length > 20) console.error(` ... and ${changes.length - 20} more`);
395
+ if (since) console.error(` ${describeSince(since)}`);
357
396
 
358
397
  return true;
359
398
  }
360
399
 
361
400
  /**
362
- * When the source a fragment describes last changed, and at which commit
363
- * (portolan.0002).
364
- *
365
- * The host works this out rather than the extractor, for two reasons. A plugin
366
- * that reads a clock produces a different fragment on every run, which cannot
367
- * be committed and cannot be checked; and a plugin that shells out to git is a
368
- * plugin that can never be sandboxed. Stamped from the last commit to touch the
369
- * directory, a fragment changes exactly when its subject does.
370
- *
371
- * The output is excluded from that history, and it has to be: a fragment
372
- * written beside the code it describes is inside the directory it is stamped
373
- * from, so committing one would move the stamp, which would make the fragment
374
- * out of date, which would rewrite it - and `--check` would never come back
375
- * clean two runs in a row.
401
+ * What a step reads, for saying why its output moved: its root and whatever
402
+ * else it was handed, less every output directory inside the root, because a
403
+ * fragment written beside the code it describes is not an input to itself.
376
404
  */
377
- function stampFor(root, out) {
378
- // A copy of another repository is dated by the commit it is a copy OF, which
379
- // the fetch step wrote down beside it. Read before git, and without touching
380
- // git at all: the local history of a vendored directory says when somebody
381
- // ran the fetch, which is a fact about this repository and not about the
382
- // service the fragment describes.
383
- //
384
- // Short, like every other stamp, because a stamp is read rather than
385
- // resolved. The full sha travels separately, in the pin the fetch also
386
- // wrote, where a link is built from it.
387
- const vendored = vendoredCommit(root);
388
- if (vendored) {
389
- // No date. Nothing here knows when that commit was made - the lock records
390
- // what was fetched, not when it was authored - and a date invented from
391
- // the local clock would make the merge call this the stalest source in the
392
- // estate every time it ran.
393
- return { commit: vendored.slice(0, 7), generatedAt: "" };
394
- }
395
-
396
- // The history read is the one the root lives in. A manifest pointed at a
397
- // checkout elsewhere on the disk - the service being documented, not a copy
398
- // of it vendored here - is stamped from that checkout's history, because the
399
- // fragment describes that service and not the repository the manifest sits
400
- // in. A root with no repository around it is stamped as uncommitted.
401
- const repo = repositoryOf(root);
402
- if (!repo) {
403
- return { commit: "uncommitted", generatedAt: process.env.PORTOLAN_GENERATED_AT || new Date().toISOString() };
404
- }
405
-
406
- // A shallow clone has no history to read: the one commit that was fetched has
407
- // no parent, so every path looks as though it changed there and every fragment
408
- // is stamped with the checkout rather than with its subject. That is wrong
409
- // quietly - the fragments regenerate, `--check` reports drift, and nothing
410
- // says why - so it is refused here instead.
411
- if (shallow(repo)) {
412
- fail(
413
- `${repo} is a shallow clone, where every path looks as though it changed in ` +
414
- "the single commit that was fetched, so a fragment cannot be stamped " +
415
- "with the commit it describes. Fetch the full history first " +
416
- "(git fetch --unshallow, or actions/checkout with fetch-depth: 0).",
417
- );
405
+ function inputsOf(step, extra = []) {
406
+ const excludes = [];
407
+ for (const other of allSteps()) {
408
+ if (!other.out || other.out === step.in) continue;
409
+ const inside = relative(step.in, other.out);
410
+ if (inside && !inside.startsWith("..") && !isAbsolute(inside)) excludes.push(other.out);
418
411
  }
412
+ return { inputs: [step.in, ...extra], excludes };
413
+ }
419
414
 
420
- // Only an output INSIDE the root is excluded. An output beside it, or above
421
- // it, is not in the root's history to begin with - and excluding a parent
422
- // would exclude the root itself, leaving nothing to read and a stamp that
423
- // moved on every commit.
424
- const rootInRepo = relative(repo, resolve(root)) || ".";
425
- const outInRepo = out ? relative(repo, resolve(out)) : "";
426
- const inside = out && (outInRepo === rootInRepo || outInRepo.startsWith(`${rootInRepo.replace(/\/$/, "")}/`) || rootInRepo === ".");
427
- const exclude = inside && outInRepo !== rootInRepo && !outInRepo.startsWith("..") ? [`:(exclude)${outInRepo}`] : [];
428
-
429
- for (const args of [
430
- ["log", "-1", "--format=%h %cI", "--", rootInRepo, ...exclude],
431
- ["log", "-1", "--format=%h %cI"],
432
- ]) {
433
- const answer = git(repo, args);
434
- if (!answer) continue;
435
- const [commit, generatedAt] = answer.split(" ");
436
- if (commit && generatedAt) return { commit, generatedAt };
437
- }
415
+ function allSteps() {
416
+ return [...(manifest.extract ?? []), ...(manifest.verify ?? []), ...(manifest.generate ?? [])];
417
+ }
438
418
 
439
- return { commit: "uncommitted", generatedAt: process.env.PORTOLAN_GENERATED_AT || new Date().toISOString() };
419
+ /**
420
+ * Why a step's output is not what it was: the commit that last touched the
421
+ * output is when it was last generated, and what changed among the inputs
422
+ * since then is the reason. The history is the record of the last generation
423
+ * (portolan.0010); nothing is written down to know this.
424
+ *
425
+ * The manifest counts as an input only where this step's own entry moved.
426
+ * The whole file changes whenever any step is touched, and naming it for
427
+ * every other step would explain nothing.
428
+ */
429
+ function whyChanged(step, outputs, reads) {
430
+ const committed = lastCommitTouching(process.cwd(), outputs);
431
+ if (!committed) return { committed: null, changed: [] };
432
+ const changed = changedSince(process.cwd(), committed.commit, reads.inputs, reads.excludes);
433
+ if (stepEntryChanged(committed.commit, step)) changed.push("portolan.json (this step's entry)");
434
+ return { committed, changed };
440
435
  }
441
436
 
442
- /** Runs git in a repository and answers with its trimmed output, or "" when it refused. */
443
- function git(repo, args) {
437
+ /** Whether the manifest at `commit` told this step the same thing it is told now. */
438
+ function stepEntryChanged(commit, step) {
439
+ let then;
444
440
  try {
445
- return execFileSync("git", ["-C", repo, ...args], {
446
- encoding: "utf8",
447
- stdio: ["ignore", "pipe", "ignore"],
448
- }).trim();
441
+ then = JSON.parse(fileAt(process.cwd(), commit, "portolan.json"));
449
442
  } catch {
450
- // Not a repository, or no commit touches this path yet. git has already
451
- // said so on its own stderr, which is not this run's log.
452
- return "";
443
+ return true; // No manifest there, or not one that parses: everything about the step is new.
453
444
  }
445
+ const phase = ["extract", "verify", "generate"].find((name) => (manifest[name] ?? []).includes(step));
446
+ const before = (then[phase] ?? []).find(
447
+ (other) =>
448
+ other.plugin === step.plugin &&
449
+ (other.in ?? "") === (step.in ?? "") &&
450
+ other.out === step.out &&
451
+ (other.options?.out ?? "") === (step.options?.out ?? ""),
452
+ );
453
+ return !before || JSON.stringify(before) !== JSON.stringify(step);
454
454
  }
455
455
 
456
- /** The working tree a directory belongs to, or "" when no repository holds it. */
457
- function repositoryOf(dir) {
458
- return git(dir, ["rev-parse", "--show-toplevel"]);
459
- }
460
-
461
- /** Whether the history this runs against is truncated. */
462
- function shallow(repo) {
463
- return git(repo, ["rev-parse", "--is-shallow-repository"]) === "true";
456
+ function describeSince({ committed, changed }) {
457
+ if (!committed) return "the output has never been committed, so there is nothing to compare its inputs against";
458
+ const when = `${committed.commit} (${committed.date}), when the output was last committed`;
459
+ if (changed.length === 0) return `no input changed since ${when}; the plugin itself did`;
460
+ const shown = changed.slice(0, 6).join(", ");
461
+ const more = changed.length > 6 ? ` and ${changed.length - 6} more` : "";
462
+ return `since ${when}, ${changed.length} input${changed.length === 1 ? "" : "s"} changed: ${shown}${more}`;
464
463
  }
465
464
 
466
465
  /** Reads, merges and validates every source the manifest names. */
@@ -497,7 +496,11 @@ function apply(files, out, key, checkOnly) {
497
496
 
498
497
  if (binary ? Buffer.isBuffer(current) && current.equals(wanted) : current === wanted) continue;
499
498
 
500
- changes.push({ kind: current === null ? "added" : "changed", path: join(out, file.name) });
499
+ changes.push({
500
+ kind: current === null ? "added" : "changed",
501
+ path: join(out, file.name),
502
+ reason: explainChange(current, wanted, file.name),
503
+ });
501
504
  if (checkOnly) continue;
502
505
 
503
506
  try {
@@ -0,0 +1,30 @@
1
+ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { dirname, join } from "node:path";
4
+ import { afterEach, expect, it } from "vitest";
5
+ import { discoverProject } from "./local-discovery.mjs";
6
+
7
+ const roots = [];
8
+ afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
9
+ function discover(files) {
10
+ const root = mkdtempSync(join(tmpdir(), "portolan-go-discovery-"));
11
+ roots.push(root);
12
+ for (const [name, text] of Object.entries({ "go.mod": "module example.com/app\n", ...files })) {
13
+ const path = join(root, name);
14
+ mkdirSync(dirname(path), { recursive: true });
15
+ writeFileSync(path, text);
16
+ }
17
+ return discoverProject(root, ".").detections.map((item) => item.plugin);
18
+ }
19
+ it("offers Go flows and SQL for a service without any DDD directories", () => {
20
+ expect(discover({
21
+ "web/routes.go": 'package web\nimport "net/http"\nfunc Register(m *http.ServeMux) { m.HandleFunc("POST /orders", create) }',
22
+ "store/migrations/001.sql": "CREATE TABLE orders (id uuid PRIMARY KEY);",
23
+ })).toEqual(expect.arrayContaining(["go-domain", "sql"]));
24
+ });
25
+ it("does not offer Go domain extraction for unregistered helpers or generated handlers", () => {
26
+ for (const source of [
27
+ 'package web\nimport "net/http"\nfunc Help(w http.ResponseWriter, r *http.Request) {}',
28
+ '// Code generated by fixture. DO NOT EDIT.\npackage web\nimport "net/http"\nfunc Register() { http.HandleFunc("/x", help) }',
29
+ ]) expect(discover({ "web/routes.go": source })).not.toContain("go-domain");
30
+ });