@shortlink-org/portolan 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +23 -0
  2. package/catalog/enum_test.go +46 -0
  3. package/catalog/evidence_test.go +35 -0
  4. package/catalog/model.go +1066 -0
  5. package/catalog/roundtrip_test.go +203 -0
  6. package/catalog/via_test.go +38 -0
  7. package/cli/init.test.mjs +6 -1
  8. package/cli/portolan.mjs +8 -0
  9. package/cli/portolan.test.mjs +49 -0
  10. package/go.mod +14 -0
  11. package/go.sum +20 -0
  12. package/internal/gocall/README.md +19 -0
  13. package/internal/gocall/analyze.go +189 -0
  14. package/internal/gocall/analyze_test.go +107 -0
  15. package/internal/gohttp/analyze.go +2562 -0
  16. package/internal/gohttp/destination.go +373 -0
  17. package/internal/gohttp/endpoints.go +1067 -0
  18. package/internal/gohttp/roots.go +320 -0
  19. package/internal/gohttp/typed.go +96 -0
  20. package/internal/goscan/constants.go +85 -0
  21. package/internal/goscan/goscan_test.go +227 -0
  22. package/internal/goscan/index.go +629 -0
  23. package/internal/goscan/index_test.go +66 -0
  24. package/internal/goscan/names.go +52 -0
  25. package/internal/goscan/parse_test.go +11 -0
  26. package/internal/goscan/source.go +37 -0
  27. package/internal/goscan/tree.go +284 -0
  28. package/internal/goscan/types.go +99 -0
  29. package/internal/wsdl/ids.go +127 -0
  30. package/internal/wsdl/ids_test.go +21 -0
  31. package/internal/wsdl/model.go +70 -0
  32. package/internal/wsdl/parse.go +949 -0
  33. package/internal/wsdl/parse_test.go +170 -0
  34. package/package.json +22 -10
  35. package/plugin/describe.go +118 -0
  36. package/plugin/describe_test.go +114 -0
  37. package/plugin/protocol.go +141 -0
  38. package/plugin/schematest/schematest.go +126 -0
  39. package/plugins/README.md +114 -46
  40. package/plugins/cmd/portolan-http-clients/main.go +19 -0
  41. package/plugins/extract-celery/extract.py +0 -2
  42. package/plugins/extract-celery/extract_test.py +1 -1
  43. package/plugins/extract-django/README.md +39 -17
  44. package/plugins/extract-django/domain.py +28 -17
  45. package/plugins/extract-django/extract.py +21 -7
  46. package/plugins/extract-django/extract_test.py +55 -2
  47. package/plugins/extract-django/lifecycle.py +2 -0
  48. package/plugins/extract-django/operations.py +1 -1
  49. package/plugins/extract-django/routing_test.py +109 -1
  50. package/plugins/extract-django/store.py +1 -1
  51. package/plugins/extract-django/transport.py +101 -55
  52. package/plugins/extract-django/verbs.py +241 -0
  53. package/plugins/extract-go/README.md +47 -0
  54. package/plugins/extract-http-clients/describe.go +19 -0
  55. package/plugins/extract-http-clients/describe_test.go +11 -0
  56. package/plugins/extract-http-clients/extract.go +740 -0
  57. package/plugins/extract-http-clients/extract_test.go +1561 -0
  58. package/plugins/extract-http-clients/main.go +41 -0
  59. package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
  60. package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
  61. package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
  62. package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
  63. package/plugins/extract-python-kafka/README.md +6 -0
  64. package/plugins/extract-python-kafka/extract.py +2 -4
  65. package/plugins/extract-python-kafka/extract_test.py +18 -2
  66. package/plugins/extract-ts/extract.test.ts +2 -2
  67. package/plugins/extract-ts/extract.ts +4 -5
  68. package/plugins/extract-ts/graphql.test.ts +1 -1
  69. package/plugins/openapi/ids.go +261 -0
  70. package/plugins/openapi/ids_test.go +98 -0
  71. package/plugins/portolan-go.wasm +0 -0
  72. package/plugins/pyplugin/catalog.py +12 -1
  73. package/plugins/pyplugin/kafka.py +74 -3
  74. package/plugins/pyplugin/protocol.py +1 -5
  75. package/portolan.json +3 -2
  76. package/schema/portolan.schema.json +34 -0
  77. package/scripts/README.md +18 -12
  78. package/scripts/catalog-sources.mjs +6 -0
  79. package/scripts/delivery-presets.mjs +21 -11
  80. package/scripts/diff.mjs +5 -1
  81. package/scripts/django-aggregates.test.mjs +58 -0
  82. package/scripts/gen-likec4.mjs +79 -21
  83. package/scripts/gen-likec4.test.mjs +25 -2
  84. package/scripts/gen.mjs +118 -115
  85. package/scripts/go-discovery.test.mjs +30 -0
  86. package/scripts/history.mjs +186 -3
  87. package/scripts/history.test.mjs +1 -1
  88. package/scripts/host-plugins/fetch-git.mjs +77 -21
  89. package/scripts/host-plugins/fetch-git.test.mjs +62 -8
  90. package/scripts/local-api.mjs +71 -4
  91. package/scripts/local-api.test.mjs +63 -4
  92. package/scripts/local-discovery.mjs +82 -9
  93. package/scripts/manifest.mjs +5 -3
  94. package/scripts/manifest.test.mjs +24 -0
  95. package/scripts/output-diff.mjs +94 -0
  96. package/scripts/output-diff.test.mjs +36 -0
  97. package/scripts/package-smoke.mjs +62 -4
  98. package/scripts/plugin-host.mjs +22 -2
  99. package/scripts/plugin-host.test.mjs +9 -0
  100. package/scripts/plugin-wasm-worker.mjs +4 -1
  101. package/scripts/provenance.mjs +72 -0
  102. package/scripts/provenance.test.mjs +149 -0
  103. package/scripts/run-builtin.mjs +39 -5
  104. package/scripts/schema.mjs +29 -0
  105. package/scripts/warning-policy.mjs +161 -0
  106. package/scripts/warning-policy.test.mjs +56 -0
  107. package/src/app/Breadcrumbs.test.ts +4 -0
  108. package/src/app/Breadcrumbs.tsx +1 -0
  109. package/src/app/Sidebar.tsx +3 -3
  110. package/src/catalog-docs.test.ts +64 -0
  111. package/src/catalog-docs.ts +35 -0
  112. package/src/catalog-error.test.ts +15 -0
  113. package/src/catalog-model.ts +70 -6
  114. package/src/catalog-stores.test.ts +17 -0
  115. package/src/catalog-validation.ts +52 -2
  116. package/src/catalog.test.ts +13 -2
  117. package/src/chat/Starter.tsx +5 -11
  118. package/src/chat/tools.test.ts +27 -0
  119. package/src/chat/tools.ts +5 -9
  120. package/src/components/CatalogStamp.tsx +10 -8
  121. package/src/components/ChannelRows.messagepack.test.tsx +28 -0
  122. package/src/components/ChannelRows.test.tsx +54 -0
  123. package/src/components/ChannelRows.tsx +57 -10
  124. package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
  125. package/src/components/HTTPDestinationEvidence.tsx +31 -0
  126. package/src/components/Integrations.tsx +1 -1
  127. package/src/components/LifecycleDiagram.tsx +28 -12
  128. package/src/components/MachineDocs.tsx +6 -5
  129. package/src/components/MethodRows.tsx +9 -2
  130. package/src/components/ProblemRow.tsx +4 -0
  131. package/src/components/RelationEvidence.test.tsx +14 -0
  132. package/src/components/RelationEvidence.tsx +53 -0
  133. package/src/components/WhatLinksHere.tsx +6 -4
  134. package/src/data.ts +25 -7
  135. package/src/enrich.test.ts +336 -6
  136. package/src/enrich.ts +206 -3
  137. package/src/flow/StepDetail.tsx +104 -54
  138. package/src/flow/answers.test.ts +18 -1
  139. package/src/flow/answers.ts +37 -8
  140. package/src/flow/evidence.test.ts +16 -0
  141. package/src/flow/evidence.ts +34 -0
  142. package/src/index.css +44 -0
  143. package/src/landing/DraggableReveal.tsx +3 -2
  144. package/src/landing/EvidencePipeline.tsx +105 -0
  145. package/src/landing/LandingPage.tsx +2 -59
  146. package/src/lib/backlinks.test.ts +16 -1
  147. package/src/lib/backlinks.ts +20 -0
  148. package/src/lib/catalog-diff.test.ts +18 -0
  149. package/src/lib/catalog-diff.ts +20 -2
  150. package/src/lib/derive.ts +1 -0
  151. package/src/lib/django-aggregates.d.mts +9 -0
  152. package/src/lib/django-aggregates.mjs +36 -0
  153. package/src/lib/django-aggregates.test.ts +29 -0
  154. package/src/lib/django-aggregates.ts +5 -0
  155. package/src/lib/kafka-ui.test.ts +87 -0
  156. package/src/lib/kafka-ui.ts +105 -0
  157. package/src/lib/local-api.ts +20 -2
  158. package/src/lib/setup-info.test.ts +17 -0
  159. package/src/lib/setup-info.ts +58 -0
  160. package/src/lib/warnings.test.ts +54 -0
  161. package/src/lib/warnings.ts +260 -0
  162. package/src/lib/wire-problems.test.ts +21 -0
  163. package/src/lib/wire-problems.ts +62 -1
  164. package/src/likec4/FlowView.tsx +2 -6
  165. package/src/likec4/flow-edges.test.ts +64 -1
  166. package/src/likec4/flow-edges.ts +43 -7
  167. package/src/likec4/view-index.ts +8 -2
  168. package/src/map/ContextMapGraph.tsx +76 -32
  169. package/src/merge.test.ts +23 -0
  170. package/src/merge.ts +33 -10
  171. package/src/pages/AggregatePage.tsx +8 -7
  172. package/src/pages/CatalogFailure.tsx +2 -2
  173. package/src/pages/ContextPage.tsx +6 -5
  174. package/src/pages/ServicePage.tsx +4 -3
  175. package/src/pages/Settings.tsx +200 -44
  176. package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
  177. package/src/pages/settings/IntegrationsSettings.tsx +117 -0
  178. package/src/routes.test.ts +2 -0
  179. package/src/routes.ts +2 -1
  180. package/src/selection/DetailPanel.tsx +61 -1
  181. package/src/virtual-provenance.d.ts +11 -0
  182. package/vite.config.ts +5 -0
  183. package/scripts/vendor-lock.mjs +0 -58
  184. package/scripts/vendor-lock.test.mjs +0 -69
@@ -164,11 +164,34 @@ describe("the LikeC4 generator", () => {
164
164
  },
165
165
  ],
166
166
  },
167
+ {
168
+ id: "flow.get-book-contract",
169
+ slug: "get-book-contract",
170
+ name: "Get book contract",
171
+ summary: "",
172
+ owner: "demo",
173
+ participants: [
174
+ { id: "demo.api", kind: "service", context: "demo" },
175
+ { id: "demo.book", kind: "service", context: "demo" },
176
+ ],
177
+ steps: [
178
+ {
179
+ type: "step",
180
+ id: "request",
181
+ from: "demo.api",
182
+ to: "demo.book",
183
+ kind: "rpc",
184
+ ref: "book.v1.Book/Get",
185
+ label: "Get",
186
+ status: "declared",
187
+ },
188
+ ],
189
+ },
167
190
  ],
168
191
  });
169
192
 
170
193
  expect(views).toContain(
171
- "demo.api -> demo.book 'Get' {\n color declared line solid head normal",
194
+ "demo.api -> demo.book 'GetRequest' {\n color declared line solid head normal",
172
195
  );
173
196
  expect(views).toContain(
174
197
  "demo.book -> demo.api 'GetResponse' {\n color declared line dashed head normal",
@@ -177,7 +200,7 @@ describe("the LikeC4 generator", () => {
177
200
  "demo.book -> demo.api '500 · Error' {\n color response_error line dashed head normal",
178
201
  );
179
202
  expect(spec).toContain("color response_error #b7646b");
180
- expect(views).not.toContain("Get → GetResponse");
203
+ expect(views).not.toContain("GetRequest → GetResponse");
181
204
  });
182
205
 
183
206
  it("treats dots in a root participant id as data, not containment", () => {
package/scripts/gen.mjs CHANGED
@@ -15,9 +15,8 @@ import {
15
15
  rmSync,
16
16
  statSync,
17
17
  } from "node:fs";
18
- import { execFileSync } from "node:child_process";
19
18
  import { createHash } from "node:crypto";
20
- import { join, relative, resolve } from "node:path";
19
+ import { isAbsolute, join, relative } from "node:path";
21
20
 
22
21
  import {
23
22
  addBuildStep,
@@ -26,16 +25,17 @@ import {
26
25
  writeBuildReport,
27
26
  } from "./build-report.mjs";
28
27
  import { loadCatalog } from "./catalog-sources.mjs";
29
- import { historyFor } from "./history.mjs";
28
+ import { changedSince, fileAt, historyFor, lastCommitTouching } from "./history.mjs";
30
29
  import { loadManifest, stepKeys } from "./manifest.mjs";
31
30
  import { describePlugin, runPlugin } from "./plugin-host.mjs";
32
- import { vendoredCommit } from "./vendor-lock.mjs";
31
+ import { explainChange } from "./output-diff.mjs";
33
32
  import {
34
33
  removeOutputFile,
35
34
  safeOutputPath,
36
35
  writeOutputFile,
37
36
  } from "./output-path.mjs";
38
37
  import { builtinPlugin } from "./builtin-plugins.mjs";
38
+ import { diagnoseWarnings } from "./warning-policy.mjs";
39
39
 
40
40
  const PORTOLAN_VERSION = "0.1.0";
41
41
  const EVENTS = process.env.PORTOLAN_EVENTS === "1";
@@ -122,9 +122,12 @@ async function generate() {
122
122
 
123
123
  // Extractors run first and write catalog fragments; only then is there a
124
124
  // catalog for anything else to read.
125
+ //
126
+ // No stamp travels with the request (portolan.0010). A fragment is content
127
+ // and nothing else; when it last changed is what the history of the
128
+ // fragment says, read wherever the catalog is read.
125
129
  for (const step of manifest.extract ?? []) {
126
130
  const plugin = pluginNamed(step.plugin);
127
- const stamp = stampFor(step.in, step.out);
128
131
  // A plugin that asks for history gets the root's, read once per checkout
129
132
  // (portolan.0007); left out when the root is not in a checkout, which the
130
133
  // plugin reports in its own words.
@@ -135,13 +138,11 @@ async function generate() {
135
138
  input: {
136
139
  root: step.in,
137
140
  output: step.out,
138
- commit: stamp.commit,
139
- generatedAt: stamp.generatedAt,
140
141
  ...(history ? { history } : {}),
141
142
  },
142
143
  options: step.options ?? {},
143
144
  }, {}, { workspace: process.cwd() }),
144
- );
145
+ inputsOf(step));
145
146
  }
146
147
 
147
148
  // Verifiers read observed evidence against the merged catalog while leaving
@@ -152,17 +153,16 @@ async function generate() {
152
153
  // carry it into the generators and into the next run.
153
154
  for (const step of manifest.verify ?? []) {
154
155
  const plugin = pluginNamed(step.plugin);
155
- const stamp = stampFor(step.in, step.out);
156
156
  const own = (previous(step.out)[keys.keyOf(step)] ?? []).map((name) => join(step.out, name));
157
- const { catalog } = await loadSources({ exclude: [...own, ...staleAll()] });
157
+ const { catalog, sources } = await loadSources({ exclude: [...own, ...staleAll()] });
158
158
  await executeStep("verify", step, `${step.plugin} ⇐ ${step.in}`, async () =>
159
159
  runPlugin(plugin, {
160
160
  portolanVersion: PORTOLAN_VERSION,
161
- input: { root: step.in, output: step.out, commit: stamp.commit, generatedAt: stamp.generatedAt },
161
+ input: { root: step.in, output: step.out },
162
162
  catalog,
163
163
  options: step.options ?? {},
164
164
  }, {}, { workspace: process.cwd() }),
165
- );
165
+ inputsOf(step, sources.map((source) => source.path)));
166
166
  }
167
167
 
168
168
  // What a dropped extract step wrote is not part of the catalog, so it is
@@ -182,16 +182,16 @@ async function generate() {
182
182
 
183
183
  for (const step of manifest.generate ?? []) {
184
184
  const plugin = pluginNamed(step.plugin);
185
- const generatedCatalog = step.catalog
186
- ? (await loadSources({ profile: step.catalog })).catalog
187
- : catalog;
185
+ const generated = step.catalog
186
+ ? await loadSources({ profile: step.catalog })
187
+ : { catalog, sources };
188
188
  await executeStep("generate", step, `${step.plugin} → ${step.out}`, async () =>
189
189
  runPlugin(plugin, {
190
190
  portolanVersion: PORTOLAN_VERSION,
191
- catalog: generatedCatalog,
191
+ catalog: generated.catalog,
192
192
  options: step.options ?? {},
193
193
  }),
194
- );
194
+ { inputs: generated.sources.map((source) => source.path), excludes: [] });
195
195
  }
196
196
 
197
197
  sweepAll();
@@ -249,13 +249,27 @@ function staleIn(out) {
249
249
  return stale;
250
250
  }
251
251
 
252
- async function executeStep(phase, step, label, work) {
252
+ /**
253
+ * Runs one step and settles its files. `reads` is what the step reads - paths,
254
+ * and the output directories among them to leave out - so that an output
255
+ * that moved can be explained by what moved among its inputs since the
256
+ * output was last committed.
257
+ */
258
+ async function executeStep(phase, step, label, work, reads) {
253
259
  const startedAt = Date.now();
254
260
  event({ type: "step-started", ordinal: report.steps.length, phase, plugin: step.plugin, input: step.in, output: step.out });
255
261
  try {
256
262
  const { files, warnings = [] } = await work();
263
+ const diagnostics = diagnoseWarnings({
264
+ plugin: step.plugin,
265
+ warnings,
266
+ policies: manifest.warningPolicies,
267
+ project: projectForStep(step),
268
+ phase,
269
+ });
257
270
  const changes = apply(files, step.out, keys.keyOf(step), check);
258
- const changed = summarise(label, files, changes, warnings);
271
+ const since = changes.length > 0 ? whyChanged(step, files.map((file) => join(step.out, file.name)), reads) : null;
272
+ const changed = summarise(label, files, changes, diagnostics, since);
259
273
  drifted = changed || drifted;
260
274
  const result = {
261
275
  phase,
@@ -267,10 +281,14 @@ async function executeStep(phase, step, label, work) {
267
281
  fileCount: files.length,
268
282
  changedCount: changes.length,
269
283
  changes,
284
+ // What moved among the inputs since the output was last committed, when
285
+ // something in the output moved; the Settings page can say why.
286
+ ...(since ? { since } : {}),
270
287
  files: files.map((file) => join(step.out, file.name)),
271
288
  // What the plugin could not read, in its own words. Already printed as
272
289
  // it ran; kept here so the Settings page can list it beside the step.
273
290
  warnings,
291
+ diagnostics,
274
292
  };
275
293
  addBuildStep(report, result);
276
294
  persistReport();
@@ -296,6 +314,15 @@ async function executeStep(phase, step, label, work) {
296
314
  }
297
315
  }
298
316
 
317
+ function projectForStep(step) {
318
+ if (!step.in) return "";
319
+ const input = String(step.in).replace(/\\/g, "/").replace(/\/+$/, "");
320
+ const projects = (manifest.projects ?? [])
321
+ .filter((project) => typeof project?.id === "string" && typeof project?.root === "string")
322
+ .sort((left, right) => right.root.length - left.root.length);
323
+ return projects.find((project) => input === project.root || input.startsWith(`${project.root}/`))?.id ?? "";
324
+ }
325
+
299
326
  function persistReport() {
300
327
  try {
301
328
  writeBuildReport(report);
@@ -334,9 +361,17 @@ async function needsOf(plugin) {
334
361
  return pluginNeeds.get(plugin.name);
335
362
  }
336
363
 
337
- /** Prints what a step did, and says whether it left the tree out of date. */
338
- function summarise(label, files, changes, warnings = []) {
339
- const said = warnings.length > 0 ? `, ${warnings.length} warning${warnings.length === 1 ? "" : "s"}` : "";
364
+ /**
365
+ * Prints what a step did, and says whether it left the tree out of date. A
366
+ * file that differs says where it first differs, and the step says what moved
367
+ * among its inputs since the output was last committed - or that nothing
368
+ * did, which points at the plugin.
369
+ */
370
+ function summarise(label, files, changes, diagnostics = [], since = null) {
371
+ const suppressed = diagnostics.filter((diagnostic) => diagnostic.suppressed).length;
372
+ const said = diagnostics.length > 0
373
+ ? `, ${diagnostics.length} warning${diagnostics.length === 1 ? "" : "s"}${suppressed ? ` (${suppressed} suppressed)` : ""}`
374
+ : "";
340
375
  const summary = `${label}: ${files.length} file${files.length === 1 ? "" : "s"}${said}`;
341
376
 
342
377
  if (changes.length === 0) {
@@ -347,120 +382,84 @@ function summarise(label, files, changes, warnings = []) {
347
382
 
348
383
  if (!check) {
349
384
  console.log(`${summary}, ${changes.length} written`);
385
+ if (since) console.log(` ${describeSince(since)}`);
350
386
 
351
387
  return false;
352
388
  }
353
389
 
354
390
  console.error(`${summary}, ${changes.length} out of date:`);
355
- for (const change of changes.slice(0, 20)) console.error(` ${change.kind.padEnd(8)} ${change.path}`);
391
+ for (const change of changes.slice(0, 20)) {
392
+ console.error(` ${change.kind.padEnd(8)} ${change.path}${change.reason ? ` — ${change.reason}` : ""}`);
393
+ }
356
394
  if (changes.length > 20) console.error(` ... and ${changes.length - 20} more`);
395
+ if (since) console.error(` ${describeSince(since)}`);
357
396
 
358
397
  return true;
359
398
  }
360
399
 
361
400
  /**
362
- * When the source a fragment describes last changed, and at which commit
363
- * (portolan.0002).
364
- *
365
- * The host works this out rather than the extractor, for two reasons. A plugin
366
- * that reads a clock produces a different fragment on every run, which cannot
367
- * be committed and cannot be checked; and a plugin that shells out to git is a
368
- * plugin that can never be sandboxed. Stamped from the last commit to touch the
369
- * directory, a fragment changes exactly when its subject does.
370
- *
371
- * The output is excluded from that history, and it has to be: a fragment
372
- * written beside the code it describes is inside the directory it is stamped
373
- * from, so committing one would move the stamp, which would make the fragment
374
- * out of date, which would rewrite it - and `--check` would never come back
375
- * clean two runs in a row.
401
+ * What a step reads, for saying why its output moved: its root and whatever
402
+ * else it was handed, less every output directory inside the root, because a
403
+ * fragment written beside the code it describes is not an input to itself.
376
404
  */
377
- function stampFor(root, out) {
378
- // A copy of another repository is dated by the commit it is a copy OF, which
379
- // the fetch step wrote down beside it. Read before git, and without touching
380
- // git at all: the local history of a vendored directory says when somebody
381
- // ran the fetch, which is a fact about this repository and not about the
382
- // service the fragment describes.
383
- //
384
- // Short, like every other stamp, because a stamp is read rather than
385
- // resolved. The full sha travels separately, in the pin the fetch also
386
- // wrote, where a link is built from it.
387
- const vendored = vendoredCommit(root);
388
- if (vendored) {
389
- // No date. Nothing here knows when that commit was made - the lock records
390
- // what was fetched, not when it was authored - and a date invented from
391
- // the local clock would make the merge call this the stalest source in the
392
- // estate every time it ran.
393
- return { commit: vendored.slice(0, 7), generatedAt: "" };
394
- }
395
-
396
- // The history read is the one the root lives in. A manifest pointed at a
397
- // checkout elsewhere on the disk - the service being documented, not a copy
398
- // of it vendored here - is stamped from that checkout's history, because the
399
- // fragment describes that service and not the repository the manifest sits
400
- // in. A root with no repository around it is stamped as uncommitted.
401
- const repo = repositoryOf(root);
402
- if (!repo) {
403
- return { commit: "uncommitted", generatedAt: process.env.PORTOLAN_GENERATED_AT || new Date().toISOString() };
404
- }
405
-
406
- // A shallow clone has no history to read: the one commit that was fetched has
407
- // no parent, so every path looks as though it changed there and every fragment
408
- // is stamped with the checkout rather than with its subject. That is wrong
409
- // quietly - the fragments regenerate, `--check` reports drift, and nothing
410
- // says why - so it is refused here instead.
411
- if (shallow(repo)) {
412
- fail(
413
- `${repo} is a shallow clone, where every path looks as though it changed in ` +
414
- "the single commit that was fetched, so a fragment cannot be stamped " +
415
- "with the commit it describes. Fetch the full history first " +
416
- "(git fetch --unshallow, or actions/checkout with fetch-depth: 0).",
417
- );
405
+ function inputsOf(step, extra = []) {
406
+ const excludes = [];
407
+ for (const other of allSteps()) {
408
+ if (!other.out || other.out === step.in) continue;
409
+ const inside = relative(step.in, other.out);
410
+ if (inside && !inside.startsWith("..") && !isAbsolute(inside)) excludes.push(other.out);
418
411
  }
412
+ return { inputs: [step.in, ...extra], excludes };
413
+ }
419
414
 
420
- // Only an output INSIDE the root is excluded. An output beside it, or above
421
- // it, is not in the root's history to begin with - and excluding a parent
422
- // would exclude the root itself, leaving nothing to read and a stamp that
423
- // moved on every commit.
424
- const rootInRepo = relative(repo, resolve(root)) || ".";
425
- const outInRepo = out ? relative(repo, resolve(out)) : "";
426
- const inside = out && (outInRepo === rootInRepo || outInRepo.startsWith(`${rootInRepo.replace(/\/$/, "")}/`) || rootInRepo === ".");
427
- const exclude = inside && outInRepo !== rootInRepo && !outInRepo.startsWith("..") ? [`:(exclude)${outInRepo}`] : [];
428
-
429
- for (const args of [
430
- ["log", "-1", "--format=%h %cI", "--", rootInRepo, ...exclude],
431
- ["log", "-1", "--format=%h %cI"],
432
- ]) {
433
- const answer = git(repo, args);
434
- if (!answer) continue;
435
- const [commit, generatedAt] = answer.split(" ");
436
- if (commit && generatedAt) return { commit, generatedAt };
437
- }
415
+ function allSteps() {
416
+ return [...(manifest.extract ?? []), ...(manifest.verify ?? []), ...(manifest.generate ?? [])];
417
+ }
438
418
 
439
- return { commit: "uncommitted", generatedAt: process.env.PORTOLAN_GENERATED_AT || new Date().toISOString() };
419
+ /**
420
+ * Why a step's output is not what it was: the commit that last touched the
421
+ * output is when it was last generated, and what changed among the inputs
422
+ * since then is the reason. The history is the record of the last generation
423
+ * (portolan.0010); nothing is written down to know this.
424
+ *
425
+ * The manifest counts as an input only where this step's own entry moved.
426
+ * The whole file changes whenever any step is touched, and naming it for
427
+ * every other step would explain nothing.
428
+ */
429
+ function whyChanged(step, outputs, reads) {
430
+ const committed = lastCommitTouching(process.cwd(), outputs);
431
+ if (!committed) return { committed: null, changed: [] };
432
+ const changed = changedSince(process.cwd(), committed.commit, reads.inputs, reads.excludes);
433
+ if (stepEntryChanged(committed.commit, step)) changed.push("portolan.json (this step's entry)");
434
+ return { committed, changed };
440
435
  }
441
436
 
442
- /** Runs git in a repository and answers with its trimmed output, or "" when it refused. */
443
- function git(repo, args) {
437
+ /** Whether the manifest at `commit` told this step the same thing it is told now. */
438
+ function stepEntryChanged(commit, step) {
439
+ let then;
444
440
  try {
445
- return execFileSync("git", ["-C", repo, ...args], {
446
- encoding: "utf8",
447
- stdio: ["ignore", "pipe", "ignore"],
448
- }).trim();
441
+ then = JSON.parse(fileAt(process.cwd(), commit, "portolan.json"));
449
442
  } catch {
450
- // Not a repository, or no commit touches this path yet. git has already
451
- // said so on its own stderr, which is not this run's log.
452
- return "";
443
+ return true; // No manifest there, or not one that parses: everything about the step is new.
453
444
  }
445
+ const phase = ["extract", "verify", "generate"].find((name) => (manifest[name] ?? []).includes(step));
446
+ const before = (then[phase] ?? []).find(
447
+ (other) =>
448
+ other.plugin === step.plugin &&
449
+ (other.in ?? "") === (step.in ?? "") &&
450
+ other.out === step.out &&
451
+ (other.options?.out ?? "") === (step.options?.out ?? ""),
452
+ );
453
+ return !before || JSON.stringify(before) !== JSON.stringify(step);
454
454
  }
455
455
 
456
- /** The working tree a directory belongs to, or "" when no repository holds it. */
457
- function repositoryOf(dir) {
458
- return git(dir, ["rev-parse", "--show-toplevel"]);
459
- }
460
-
461
- /** Whether the history this runs against is truncated. */
462
- function shallow(repo) {
463
- return git(repo, ["rev-parse", "--is-shallow-repository"]) === "true";
456
+ function describeSince({ committed, changed }) {
457
+ if (!committed) return "the output has never been committed, so there is nothing to compare its inputs against";
458
+ const when = `${committed.commit} (${committed.date}), when the output was last committed`;
459
+ if (changed.length === 0) return `no input changed since ${when}; the plugin itself did`;
460
+ const shown = changed.slice(0, 6).join(", ");
461
+ const more = changed.length > 6 ? ` and ${changed.length - 6} more` : "";
462
+ return `since ${when}, ${changed.length} input${changed.length === 1 ? "" : "s"} changed: ${shown}${more}`;
464
463
  }
465
464
 
466
465
  /** Reads, merges and validates every source the manifest names. */
@@ -497,7 +496,11 @@ function apply(files, out, key, checkOnly) {
497
496
 
498
497
  if (binary ? Buffer.isBuffer(current) && current.equals(wanted) : current === wanted) continue;
499
498
 
500
- changes.push({ kind: current === null ? "added" : "changed", path: join(out, file.name) });
499
+ changes.push({
500
+ kind: current === null ? "added" : "changed",
501
+ path: join(out, file.name),
502
+ reason: explainChange(current, wanted, file.name),
503
+ });
501
504
  if (checkOnly) continue;
502
505
 
503
506
  try {
@@ -0,0 +1,30 @@
1
+ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { dirname, join } from "node:path";
4
+ import { afterEach, expect, it } from "vitest";
5
+ import { discoverProject } from "./local-discovery.mjs";
6
+
7
+ const roots = [];
8
+ afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
9
+ function discover(files) {
10
+ const root = mkdtempSync(join(tmpdir(), "portolan-go-discovery-"));
11
+ roots.push(root);
12
+ for (const [name, text] of Object.entries({ "go.mod": "module example.com/app\n", ...files })) {
13
+ const path = join(root, name);
14
+ mkdirSync(dirname(path), { recursive: true });
15
+ writeFileSync(path, text);
16
+ }
17
+ return discoverProject(root, ".").detections.map((item) => item.plugin);
18
+ }
19
+ it("offers Go flows and SQL for a service without any DDD directories", () => {
20
+ expect(discover({
21
+ "web/routes.go": 'package web\nimport "net/http"\nfunc Register(m *http.ServeMux) { m.HandleFunc("POST /orders", create) }',
22
+ "store/migrations/001.sql": "CREATE TABLE orders (id uuid PRIMARY KEY);",
23
+ })).toEqual(expect.arrayContaining(["go-domain", "sql"]));
24
+ });
25
+ it("does not offer Go domain extraction for unregistered helpers or generated handlers", () => {
26
+ for (const source of [
27
+ 'package web\nimport "net/http"\nfunc Help(w http.ResponseWriter, r *http.Request) {}',
28
+ '// Code generated by fixture. DO NOT EDIT.\npackage web\nimport "net/http"\nfunc Register() { http.HandleFunc("/x", help) }',
29
+ ]) expect(discover({ "web/routes.go": source })).not.toContain("go-domain");
30
+ });
@@ -1,5 +1,6 @@
1
1
  // When each file was first committed and last changed, read by the host for a
2
- // plugin that asks (portolan.0007).
2
+ // plugin that asks (portolan.0007), and for the provenance of every catalog
3
+ // source (portolan.0010).
3
4
  //
4
5
  // One `git log` over the whole checkout, walked oldest first with renames
5
6
  // followed, gives every path its first commit and its last. A plugin used to
@@ -8,8 +9,8 @@
8
9
  // reads it once and hands each step the part under its root.
9
10
 
10
11
  import { execFileSync } from "node:child_process";
11
- import { realpathSync } from "node:fs";
12
- import { relative, resolve, sep } from "node:path";
12
+ import { realpathSync, statSync } from "node:fs";
13
+ import { dirname, relative, resolve, sep } from "node:path";
13
14
 
14
15
  const RECORD = "\x1e";
15
16
  const FIELD = "\x1f";
@@ -17,6 +18,188 @@ const FIELD = "\x1f";
17
18
  /** Per repository, per process: the history does not change during a run. */
18
19
  const cache = new Map();
19
20
 
21
+ /** Repositories already told they are shallow; it is said once. */
22
+ const warnedShallow = new Set();
23
+
24
+ /**
25
+ * When each of `paths` last changed, as the history says: the commit that
26
+ * last touched the file, and that commit's date.
27
+ *
28
+ * This is where a source's provenance comes from (portolan.0010). It is read
29
+ * off the checkout that holds the file and never written into the file, so a
30
+ * fragment is content and nothing else, and a regeneration that changes no
31
+ * fact changes no byte. The stamp names the commit the change landed in,
32
+ * which a stamp written into the file never could: that one was the parent
33
+ * at best, and it took a second commit to catch up.
34
+ *
35
+ * A file with uncommitted changes, or one git does not track, is
36
+ * `uncommitted` and dated by its mtime - an observation of the working tree,
37
+ * shown and never committed. A path outside any repository is the same. A
38
+ * shallow clone makes every path look as though it changed in the one commit
39
+ * that was fetched; that is said once, and the stamps are what the truncated
40
+ * history says, which is at least the checkout.
41
+ *
42
+ * @param {string} workspace
43
+ * @param {string[]} paths relative to `workspace`, forward slashes
44
+ * @returns {Map<string, {commit: string, generatedAt: string}>}
45
+ */
46
+ export function stampsFor(workspace, paths) {
47
+ const workspaceReal = realpathSync(resolve(workspace));
48
+ const stamps = new Map();
49
+ const repoOfDir = new Map();
50
+ const byRepo = new Map();
51
+
52
+ for (const path of paths) {
53
+ const absolute = resolve(workspaceReal, path);
54
+ const dir = dirname(absolute);
55
+ if (!repoOfDir.has(dir)) repoOfDir.set(dir, repositoryOf(dir));
56
+ const repo = repoOfDir.get(dir);
57
+ if (!repo) {
58
+ stamps.set(path, uncommitted(absolute));
59
+ continue;
60
+ }
61
+ if (!byRepo.has(repo)) byRepo.set(repo, []);
62
+ byRepo.get(repo).push({ path, absolute, key: toPosix(relative(repo, absolute)) });
63
+ }
64
+
65
+ for (const [repo, files] of byRepo) {
66
+ warnIfShallow(repo);
67
+ const history = fileHistory(repo);
68
+ const dirty = dirtyIn(repo);
69
+ for (const file of files) {
70
+ const entry = history.get(file.key);
71
+ const last = entry?.revised ?? entry?.created;
72
+ stamps.set(
73
+ file.path,
74
+ !last || dirty.has(file.key)
75
+ ? uncommitted(file.absolute)
76
+ : { commit: last.commit.slice(0, 7), generatedAt: last.date },
77
+ );
78
+ }
79
+ }
80
+
81
+ return stamps;
82
+ }
83
+
84
+ /**
85
+ * The commit that last touched any of `paths`, and its date, or null when
86
+ * none of them has been committed. For a step's output this is when it was
87
+ * last generated: the history remembers it, so nothing has to write it down
88
+ * anywhere else (portolan.0010).
89
+ *
90
+ * @param {string} workspace
91
+ * @param {string[]} paths relative to `workspace`
92
+ * @returns {{commit: string, date: string} | null}
93
+ */
94
+ export function lastCommitTouching(workspace, paths) {
95
+ const { repo, keys } = inRepo(workspace, paths);
96
+ if (!repo || keys.length === 0) return null;
97
+ const answer = git(repo, ["log", "-1", "--format=%H %cI", "--", ...keys]);
98
+ if (!answer) return null;
99
+ const [sha, date] = answer.split(" ");
100
+ // Seven characters, the width every stamp has, so the two read alike.
101
+ return sha && date ? { commit: sha.slice(0, 7), date } : null;
102
+ }
103
+
104
+ /**
105
+ * Which files under `paths` differ from what `commit` held, working tree
106
+ * included: modified, added, deleted and untracked, less anything under
107
+ * `excludes`. Paths in and out are relative to `workspace`.
108
+ *
109
+ * @param {string} workspace
110
+ * @param {string} commit
111
+ * @param {string[]} paths
112
+ * @param {string[]} [excludes]
113
+ * @returns {string[]}
114
+ */
115
+ export function changedSince(workspace, commit, paths, excludes = []) {
116
+ const { repo, keys, toWorkspace } = inRepo(workspace, paths);
117
+ if (!repo || keys.length === 0) return [];
118
+ const spec = [...keys, ...inRepo(workspace, excludes).keys.map((key) => `:(exclude)${key}`)];
119
+ const tracked = git(repo, ["diff", "--name-only", commit, "--", ...spec]);
120
+ const untracked = git(repo, ["ls-files", "--others", "--exclude-standard", "--", ...spec]);
121
+ const changed = new Set();
122
+ for (const line of `${tracked}\n${untracked}`.split("\n")) {
123
+ if (line) changed.add(toWorkspace(line));
124
+ }
125
+ return [...changed].sort();
126
+ }
127
+
128
+ /**
129
+ * The text of `path` as `commit` held it, or "" when the commit did not hold
130
+ * it or there is no history to ask.
131
+ *
132
+ * @param {string} workspace
133
+ * @param {string} commit
134
+ * @param {string} path relative to `workspace`
135
+ * @returns {string}
136
+ */
137
+ export function fileAt(workspace, commit, path) {
138
+ const { repo, keys } = inRepo(workspace, [path]);
139
+ if (!repo || keys.length === 0) return "";
140
+ return git(repo, ["show", `${commit}:${keys[0]}`]);
141
+ }
142
+
143
+ /** The paths as the workspace's repository spells them, and the way back. */
144
+ function inRepo(workspace, paths) {
145
+ const workspaceReal = realpathSync(resolve(workspace));
146
+ const repo = repositoryOf(workspaceReal);
147
+ if (!repo) return { repo: "", keys: [], toWorkspace: (line) => line };
148
+ const keys = [];
149
+ for (const path of paths) {
150
+ const key = toPosix(relative(repo, resolve(workspaceReal, path)));
151
+ if (key.startsWith("..")) continue;
152
+ keys.push(key || ".");
153
+ }
154
+ return {
155
+ repo,
156
+ keys,
157
+ toWorkspace: (line) => toPosix(relative(workspaceReal, resolve(repo, line))),
158
+ };
159
+ }
160
+
161
+ function uncommitted(absolute) {
162
+ let generatedAt = "";
163
+ try {
164
+ generatedAt = statSync(absolute).mtime.toISOString();
165
+ } catch {
166
+ // Gone, or never there: undated rather than dated by a guess.
167
+ }
168
+ return { commit: "uncommitted", generatedAt };
169
+ }
170
+
171
+ /** Every path with a change the history does not hold yet, relative to the repository. */
172
+ function dirtyIn(repo) {
173
+ const dirty = new Set();
174
+ // Read raw: an entry begins with two status letters, and a space is one of
175
+ // them, which the trimming reader would eat off the first line.
176
+ let status = "";
177
+ try {
178
+ status = execFileSync("git", ["-C", repo, "status", "--porcelain", "-z", "--no-renames", "--untracked-files=all"], {
179
+ encoding: "utf8",
180
+ stdio: ["ignore", "pipe", "ignore"],
181
+ maxBuffer: 64 * 1024 * 1024,
182
+ });
183
+ } catch {
184
+ return dirty;
185
+ }
186
+ for (const entry of status.split("\0")) {
187
+ if (entry.length > 3) dirty.add(entry.slice(3));
188
+ }
189
+ return dirty;
190
+ }
191
+
192
+ function warnIfShallow(repo) {
193
+ if (warnedShallow.has(repo)) return;
194
+ warnedShallow.add(repo);
195
+ if (git(repo, ["rev-parse", "--is-shallow-repository"]) !== "true") return;
196
+ console.warn(
197
+ `${repo} is a shallow clone, where every source looks as though it last changed in ` +
198
+ "the one commit that was fetched. Fetch the full history (git fetch --unshallow, " +
199
+ "or actions/checkout with fetch-depth: 0) for the provenance to say anything.",
200
+ );
201
+ }
202
+
20
203
  /**
21
204
  * History for every file under `root`, keyed by the path a plugin would name
22
205
  * it - relative to `workspace`, forward slashes. `undefined` when the root is
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
4
4
  import { join } from "node:path";
5
5
  import { afterEach, describe, expect, it } from "vitest";
6
6
 
7
- import { fileHistory, forgetHistory, historyFor, parseLog } from "./history.mjs";
7
+ import { changedSince, fileHistory, forgetHistory, historyFor, lastCommitTouching, parseLog, stampsFor } from "./history.mjs";
8
8
 
9
9
  const created = [];
10
10
  afterEach(() => {