@shortlink-org/portolan 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/catalog/enum_test.go +46 -0
- package/catalog/evidence_test.go +35 -0
- package/catalog/model.go +1066 -0
- package/catalog/roundtrip_test.go +203 -0
- package/catalog/via_test.go +38 -0
- package/cli/init.test.mjs +6 -1
- package/cli/portolan.mjs +8 -0
- package/cli/portolan.test.mjs +49 -0
- package/go.mod +14 -0
- package/go.sum +20 -0
- package/internal/gocall/README.md +19 -0
- package/internal/gocall/analyze.go +189 -0
- package/internal/gocall/analyze_test.go +107 -0
- package/internal/gohttp/analyze.go +2562 -0
- package/internal/gohttp/destination.go +373 -0
- package/internal/gohttp/endpoints.go +1067 -0
- package/internal/gohttp/roots.go +320 -0
- package/internal/gohttp/typed.go +96 -0
- package/internal/goscan/constants.go +85 -0
- package/internal/goscan/goscan_test.go +227 -0
- package/internal/goscan/index.go +629 -0
- package/internal/goscan/index_test.go +66 -0
- package/internal/goscan/names.go +52 -0
- package/internal/goscan/parse_test.go +11 -0
- package/internal/goscan/source.go +37 -0
- package/internal/goscan/tree.go +284 -0
- package/internal/goscan/types.go +99 -0
- package/internal/wsdl/ids.go +127 -0
- package/internal/wsdl/ids_test.go +21 -0
- package/internal/wsdl/model.go +70 -0
- package/internal/wsdl/parse.go +949 -0
- package/internal/wsdl/parse_test.go +170 -0
- package/package.json +22 -10
- package/plugin/describe.go +118 -0
- package/plugin/describe_test.go +114 -0
- package/plugin/protocol.go +141 -0
- package/plugin/schematest/schematest.go +126 -0
- package/plugins/README.md +109 -46
- package/plugins/cmd/portolan-http-clients/main.go +19 -0
- package/plugins/extract-celery/extract.py +0 -2
- package/plugins/extract-celery/extract_test.py +1 -1
- package/plugins/extract-django/README.md +39 -17
- package/plugins/extract-django/domain.py +28 -17
- package/plugins/extract-django/extract.py +21 -7
- package/plugins/extract-django/extract_test.py +55 -2
- package/plugins/extract-django/lifecycle.py +2 -0
- package/plugins/extract-django/operations.py +1 -1
- package/plugins/extract-django/routing_test.py +109 -1
- package/plugins/extract-django/store.py +1 -1
- package/plugins/extract-django/transport.py +101 -55
- package/plugins/extract-django/verbs.py +241 -0
- package/plugins/extract-go/README.md +47 -0
- package/plugins/extract-http-clients/describe.go +19 -0
- package/plugins/extract-http-clients/describe_test.go +11 -0
- package/plugins/extract-http-clients/extract.go +740 -0
- package/plugins/extract-http-clients/extract_test.go +1561 -0
- package/plugins/extract-http-clients/main.go +41 -0
- package/plugins/extract-java/build/org/portolan/extract/Extract.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Builder.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Input.class +0 -0
- package/plugins/extract-java/build/org/portolan/extract/Protocol$Options.class +0 -0
- package/plugins/extract-python-kafka/extract.py +0 -2
- package/plugins/extract-python-kafka/extract_test.py +1 -1
- package/plugins/extract-ts/extract.test.ts +2 -2
- package/plugins/extract-ts/extract.ts +4 -5
- package/plugins/extract-ts/graphql.test.ts +1 -1
- package/plugins/openapi/ids.go +261 -0
- package/plugins/openapi/ids_test.go +98 -0
- package/plugins/portolan-go.wasm +0 -0
- package/plugins/pyplugin/protocol.py +1 -5
- package/portolan.json +3 -2
- package/schema/portolan.schema.json +34 -0
- package/scripts/README.md +18 -12
- package/scripts/catalog-sources.mjs +6 -0
- package/scripts/delivery-presets.mjs +21 -11
- package/scripts/diff.mjs +5 -1
- package/scripts/django-aggregates.test.mjs +58 -0
- package/scripts/gen-likec4.mjs +1 -1
- package/scripts/gen.mjs +118 -115
- package/scripts/go-discovery.test.mjs +30 -0
- package/scripts/history.mjs +186 -3
- package/scripts/history.test.mjs +1 -1
- package/scripts/host-plugins/fetch-git.mjs +77 -21
- package/scripts/host-plugins/fetch-git.test.mjs +62 -8
- package/scripts/local-api.mjs +71 -4
- package/scripts/local-api.test.mjs +63 -4
- package/scripts/local-discovery.mjs +82 -9
- package/scripts/manifest.mjs +5 -3
- package/scripts/manifest.test.mjs +24 -0
- package/scripts/output-diff.mjs +94 -0
- package/scripts/output-diff.test.mjs +36 -0
- package/scripts/package-smoke.mjs +62 -4
- package/scripts/plugin-host.mjs +22 -2
- package/scripts/plugin-host.test.mjs +9 -0
- package/scripts/plugin-wasm-worker.mjs +4 -1
- package/scripts/provenance.mjs +72 -0
- package/scripts/provenance.test.mjs +149 -0
- package/scripts/run-builtin.mjs +39 -5
- package/scripts/schema.mjs +29 -0
- package/scripts/warning-policy.mjs +161 -0
- package/scripts/warning-policy.test.mjs +56 -0
- package/src/app/Sidebar.tsx +3 -3
- package/src/catalog-docs.test.ts +64 -0
- package/src/catalog-docs.ts +35 -0
- package/src/catalog-error.test.ts +15 -0
- package/src/catalog-model.ts +48 -5
- package/src/catalog-validation.ts +9 -0
- package/src/chat/Starter.tsx +5 -11
- package/src/chat/tools.test.ts +27 -0
- package/src/chat/tools.ts +5 -9
- package/src/components/CatalogStamp.tsx +10 -8
- package/src/components/HTTPDestinationEvidence.test.tsx +23 -0
- package/src/components/HTTPDestinationEvidence.tsx +31 -0
- package/src/components/Integrations.tsx +1 -1
- package/src/components/MachineDocs.tsx +6 -5
- package/src/components/MethodRows.tsx +9 -2
- package/src/components/RelationEvidence.test.tsx +14 -0
- package/src/components/RelationEvidence.tsx +53 -0
- package/src/data.ts +25 -7
- package/src/enrich.test.ts +332 -1
- package/src/enrich.ts +206 -3
- package/src/flow/StepDetail.tsx +6 -0
- package/src/flow/evidence.test.ts +16 -0
- package/src/flow/evidence.ts +34 -0
- package/src/index.css +44 -0
- package/src/landing/DraggableReveal.tsx +3 -2
- package/src/landing/EvidencePipeline.tsx +105 -0
- package/src/landing/LandingPage.tsx +2 -59
- package/src/lib/catalog-diff.ts +1 -1
- package/src/lib/django-aggregates.d.mts +9 -0
- package/src/lib/django-aggregates.mjs +36 -0
- package/src/lib/django-aggregates.test.ts +29 -0
- package/src/lib/django-aggregates.ts +5 -0
- package/src/lib/local-api.ts +20 -2
- package/src/lib/setup-info.test.ts +17 -0
- package/src/lib/setup-info.ts +58 -0
- package/src/lib/warnings.test.ts +54 -0
- package/src/lib/warnings.ts +260 -0
- package/src/map/ContextMapGraph.tsx +76 -32
- package/src/merge.ts +16 -9
- package/src/pages/AggregatePage.tsx +8 -7
- package/src/pages/ContextPage.tsx +6 -5
- package/src/pages/ServicePage.tsx +4 -3
- package/src/pages/Settings.tsx +189 -41
- package/src/pages/settings/DjangoAggregateChoices.tsx +79 -0
- package/src/selection/DetailPanel.tsx +15 -0
- package/src/virtual-provenance.d.ts +11 -0
- package/vite.config.ts +5 -0
- package/scripts/vendor-lock.mjs +0 -58
- package/scripts/vendor-lock.test.mjs +0 -69
|
@@ -23,7 +23,10 @@ const GITLAB_START = "# >>> Portolan delivery preset >>>";
|
|
|
23
23
|
const GITLAB_END = "# <<< Portolan delivery preset <<<";
|
|
24
24
|
const PROVIDERS = new Set(["github", "gitlab"]);
|
|
25
25
|
const FEATURE_IDS = ["check", "diff", "sarif", "pages"];
|
|
26
|
-
const DEFAULT_FEATURES =
|
|
26
|
+
const DEFAULT_FEATURES = {
|
|
27
|
+
github: ["check", "pages"],
|
|
28
|
+
gitlab: ["pages"],
|
|
29
|
+
};
|
|
27
30
|
const FEATURE_DETAILS = {
|
|
28
31
|
check: {
|
|
29
32
|
label: "Architecture check",
|
|
@@ -225,7 +228,9 @@ ${upload}`;
|
|
|
225
228
|
|
|
226
229
|
function gitlabCheck() {
|
|
227
230
|
return `"portolan:check":
|
|
228
|
-
stage:
|
|
231
|
+
stage: test
|
|
232
|
+
tags:
|
|
233
|
+
- runner-type:docker
|
|
229
234
|
image:
|
|
230
235
|
name: ghcr.io/shortlink-org/portolan:${VERSION}
|
|
231
236
|
entrypoint: [""]
|
|
@@ -242,7 +247,9 @@ function gitlabCheck() {
|
|
|
242
247
|
function gitlabDiff({ pages }) {
|
|
243
248
|
const site = pages ? ' --site "$CI_PAGES_URL"' : "";
|
|
244
249
|
return `"portolan:review":
|
|
245
|
-
stage:
|
|
250
|
+
stage: test
|
|
251
|
+
tags:
|
|
252
|
+
- runner-type:docker
|
|
246
253
|
image:
|
|
247
254
|
name: ghcr.io/shortlink-org/portolan:${VERSION}
|
|
248
255
|
entrypoint: [""]
|
|
@@ -259,8 +266,10 @@ function gitlabDiff({ pages }) {
|
|
|
259
266
|
}
|
|
260
267
|
|
|
261
268
|
function gitlabPages() {
|
|
262
|
-
return `
|
|
263
|
-
stage:
|
|
269
|
+
return `pages:
|
|
270
|
+
stage: deploy
|
|
271
|
+
tags:
|
|
272
|
+
- runner-type:docker
|
|
264
273
|
image:
|
|
265
274
|
name: ghcr.io/shortlink-org/portolan:${VERSION}
|
|
266
275
|
entrypoint: [""]
|
|
@@ -268,9 +277,10 @@ function gitlabPages() {
|
|
|
268
277
|
GIT_DEPTH: "0"
|
|
269
278
|
script:
|
|
270
279
|
- BASE_PATH="$(node -p 'new URL(process.env.CI_PAGES_URL).pathname.replace(/\\/?$/, "/") || "/"')"
|
|
271
|
-
- portolan build --output
|
|
272
|
-
|
|
273
|
-
|
|
280
|
+
- portolan build --output public --base "$BASE_PATH"
|
|
281
|
+
artifacts:
|
|
282
|
+
paths:
|
|
283
|
+
- public
|
|
274
284
|
rules:
|
|
275
285
|
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
|
276
286
|
`;
|
|
@@ -299,7 +309,7 @@ function mergeGitlab(existing, features) {
|
|
|
299
309
|
.replace(/^\n+|\n+$/g, "");
|
|
300
310
|
return { content: content ? `${content}\n` : "" };
|
|
301
311
|
}
|
|
302
|
-
if (/^[
|
|
312
|
+
if (/^(?:["']?portolan:(?:check|review|pages)["']?|["']?pages["']?)\s*:/m.test(existing)) {
|
|
303
313
|
return { conflict: "This pipeline already declares a Portolan job outside the managed region." };
|
|
304
314
|
}
|
|
305
315
|
const block = gitlabBlock(features);
|
|
@@ -421,14 +431,14 @@ function installedFeatures(workspace, provider) {
|
|
|
421
431
|
);
|
|
422
432
|
if (/^[ \t]*["']?portolan:check["']?\s*:/m.test(managed)) selected.add("check");
|
|
423
433
|
if (/^[ \t]*["']?portolan:review["']?\s*:/m.test(managed)) selected.add("diff");
|
|
424
|
-
if (/^[
|
|
434
|
+
if (/^(?:["']?portolan:pages["']?|["']?pages["']?)\s*:/m.test(managed)) selected.add("pages");
|
|
425
435
|
return selected;
|
|
426
436
|
}
|
|
427
437
|
|
|
428
438
|
function selectedFeatures(workspace, request, provider) {
|
|
429
439
|
const installed = request.features == null ? installedFeatures(workspace, provider) : null;
|
|
430
440
|
const raw = request.features == null
|
|
431
|
-
? installed.size > 0 ? [...installed] : DEFAULT_FEATURES
|
|
441
|
+
? installed.size > 0 ? [...installed] : DEFAULT_FEATURES[provider]
|
|
432
442
|
: Array.isArray(request.features)
|
|
433
443
|
? request.features
|
|
434
444
|
: String(request.features).split(",").filter(Boolean);
|
package/scripts/diff.mjs
CHANGED
|
@@ -157,8 +157,12 @@ function catalogAt(ref) {
|
|
|
157
157
|
throw new Error(ref + " holds no catalog sources matching " + JSON.stringify(manifest.sources));
|
|
158
158
|
}
|
|
159
159
|
|
|
160
|
+
// The base is one commit, and a commit is its own provenance: every source
|
|
161
|
+
// there is dated by it (portolan.0010), whatever an older fragment says.
|
|
162
|
+
const [commit = "", generatedAt = ""] = git(["log", "-1", "--format=%h %cI", ref]).split(" ");
|
|
163
|
+
const stamp = { commit, generatedAt };
|
|
160
164
|
const merged = mergeCatalogs(
|
|
161
|
-
paths.map((path) => ({ path, catalog: JSON.parse(git(["show", ref + ":" + path])) })),
|
|
165
|
+
paths.map((path) => ({ path, catalog: JSON.parse(git(["show", ref + ":" + path])), stamp })),
|
|
162
166
|
);
|
|
163
167
|
|
|
164
168
|
// Enriched before it is compared, exactly as the app and the generators see
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
6
|
+
import { djangoAggregateProposals, saveDjangoAggregates } from "./local-api.mjs";
|
|
7
|
+
|
|
8
|
+
const roots = [];
|
|
9
|
+
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
|
10
|
+
function fixture() {
|
|
11
|
+
const root = mkdtempSync(join(tmpdir(), "portolan-aggregates-")); roots.push(root);
|
|
12
|
+
const manifest = { sources: ["data/*.json"], extract: ["one", "two"].map((name) => ({ plugin: "django-domain", in: name, out: `data/${name}`, options: { context: name, service: name, aggregates: { existing: "Existing" }, apps: ["billing.records"] } })) };
|
|
13
|
+
const text = JSON.stringify(manifest);
|
|
14
|
+
writeFileSync(join(root, "portolan.json"), text);
|
|
15
|
+
const message = `billing/records: no model called Records, and 2 models to choose from: name the root in the aggregates option; aggregate candidates: ${JSON.stringify({ app: "billing.records", models: [{ name: "Entry", path: "billing/records/models.py", line: 1 }, { name: "Audit", path: "billing/records/models.py", line: 10 }] })}`;
|
|
16
|
+
const report = { status: "ok", manifestSha256: createHash("sha256").update(text).digest("hex"), steps: manifest.extract.map((step) => ({ phase: "extract", plugin: step.plugin, input: step.in, output: step.out, warnings: [message] })) };
|
|
17
|
+
mkdirSync(join(root, ".portolan")); writeFileSync(join(root, ".portolan/build-report.json"), JSON.stringify(report));
|
|
18
|
+
return { root, manifest, report };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
describe("saving Django aggregate roots", () => {
|
|
22
|
+
it("targets the precise step and dotted app, preserving unrelated options", () => {
|
|
23
|
+
const { root, manifest } = fixture();
|
|
24
|
+
const proposals = djangoAggregateProposals(root);
|
|
25
|
+
expect(proposals.stale).toBe(false);
|
|
26
|
+
expect(proposals.proposals).toHaveLength(2);
|
|
27
|
+
saveDjangoAggregates(root, { revision: proposals.revision, selections: [{ id: "1:billing.records", model: "Entry" }] });
|
|
28
|
+
const saved = JSON.parse(readFileSync(join(root, "portolan.json"), "utf8"));
|
|
29
|
+
expect(saved.extract[0]).toEqual(manifest.extract[0]);
|
|
30
|
+
expect(saved.extract[1].options).toEqual({ ...manifest.extract[1].options, aggregates: { existing: "Existing", "billing.records": "Entry" } });
|
|
31
|
+
expect(djangoAggregateProposals(root).stale).toBe(true);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("saves a batch atomically and rejects forged, duplicate, or stale choices", () => {
|
|
35
|
+
const { root } = fixture();
|
|
36
|
+
const { revision } = djangoAggregateProposals(root);
|
|
37
|
+
const original = readFileSync(join(root, "portolan.json"), "utf8");
|
|
38
|
+
for (const selections of [[], [{ id: "0:billing.records", model: "Proxy" }], [{ id: "no-such-step", model: "Entry" }], [{ id: "0:billing.records", model: "Entry" }, { id: "1:billing.records", model: "Invalid" }], [{ id: "0:billing.records", model: "Entry" }, { id: "0:billing.records", model: "Audit" }]]) {
|
|
39
|
+
expect(() => saveDjangoAggregates(root, { revision, selections })).toThrow();
|
|
40
|
+
expect(readFileSync(join(root, "portolan.json"), "utf8")).toBe(original);
|
|
41
|
+
}
|
|
42
|
+
const selections = [{ id: "0:billing.records", model: "Entry" }, { id: "1:billing.records", model: "Audit" }];
|
|
43
|
+
expect(() => saveDjangoAggregates(root, { revision: "old", selections })).toThrow(/changed/);
|
|
44
|
+
expect(saveDjangoAggregates(root, { revision, selections })).toEqual({ saved: 2 });
|
|
45
|
+
expect(() => saveDjangoAggregates(root, { revision, selections })).toThrow(/changed/);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("requires fresh report evidence and refuses ambiguous step bindings", () => {
|
|
49
|
+
const { root, manifest, report } = fixture();
|
|
50
|
+
manifest.extract.push(manifest.extract[0]);
|
|
51
|
+
writeFileSync(join(root, "portolan.json"), JSON.stringify(manifest));
|
|
52
|
+
expect(djangoAggregateProposals(root).proposals.map((p) => p.id)).toEqual(["1:billing.records"]);
|
|
53
|
+
expect(djangoAggregateProposals(root).stale).toBe(true);
|
|
54
|
+
report.steps = [];
|
|
55
|
+
writeFileSync(join(root, ".portolan/build-report.json"), JSON.stringify(report));
|
|
56
|
+
expect(djangoAggregateProposals(root).proposals).toEqual([]);
|
|
57
|
+
});
|
|
58
|
+
});
|
package/scripts/gen-likec4.mjs
CHANGED
|
@@ -278,7 +278,7 @@ for (const context of catalog.contexts) {
|
|
|
278
278
|
model.push(` style { color ${contextColorName(context.id)} }`);
|
|
279
279
|
for (const aggregate of service.aggregates) {
|
|
280
280
|
model.push(
|
|
281
|
-
` ${safeId(aggregate.slug)} = aggregate ${q(aggregate.name)} {`,
|
|
281
|
+
` ${safeId(aggregate.slug)} = aggregate ${q(aggregate.kind === "model-group" ? `${aggregate.name} (model group)` : aggregate.name)} {`,
|
|
282
282
|
);
|
|
283
283
|
for (const event of aggregate.events) {
|
|
284
284
|
const latest = event.versions[event.versions.length - 1];
|
package/scripts/gen.mjs
CHANGED
|
@@ -15,9 +15,8 @@ import {
|
|
|
15
15
|
rmSync,
|
|
16
16
|
statSync,
|
|
17
17
|
} from "node:fs";
|
|
18
|
-
import { execFileSync } from "node:child_process";
|
|
19
18
|
import { createHash } from "node:crypto";
|
|
20
|
-
import { join, relative
|
|
19
|
+
import { isAbsolute, join, relative } from "node:path";
|
|
21
20
|
|
|
22
21
|
import {
|
|
23
22
|
addBuildStep,
|
|
@@ -26,16 +25,17 @@ import {
|
|
|
26
25
|
writeBuildReport,
|
|
27
26
|
} from "./build-report.mjs";
|
|
28
27
|
import { loadCatalog } from "./catalog-sources.mjs";
|
|
29
|
-
import { historyFor } from "./history.mjs";
|
|
28
|
+
import { changedSince, fileAt, historyFor, lastCommitTouching } from "./history.mjs";
|
|
30
29
|
import { loadManifest, stepKeys } from "./manifest.mjs";
|
|
31
30
|
import { describePlugin, runPlugin } from "./plugin-host.mjs";
|
|
32
|
-
import {
|
|
31
|
+
import { explainChange } from "./output-diff.mjs";
|
|
33
32
|
import {
|
|
34
33
|
removeOutputFile,
|
|
35
34
|
safeOutputPath,
|
|
36
35
|
writeOutputFile,
|
|
37
36
|
} from "./output-path.mjs";
|
|
38
37
|
import { builtinPlugin } from "./builtin-plugins.mjs";
|
|
38
|
+
import { diagnoseWarnings } from "./warning-policy.mjs";
|
|
39
39
|
|
|
40
40
|
const PORTOLAN_VERSION = "0.1.0";
|
|
41
41
|
const EVENTS = process.env.PORTOLAN_EVENTS === "1";
|
|
@@ -122,9 +122,12 @@ async function generate() {
|
|
|
122
122
|
|
|
123
123
|
// Extractors run first and write catalog fragments; only then is there a
|
|
124
124
|
// catalog for anything else to read.
|
|
125
|
+
//
|
|
126
|
+
// No stamp travels with the request (portolan.0010). A fragment is content
|
|
127
|
+
// and nothing else; when it last changed is what the history of the
|
|
128
|
+
// fragment says, read wherever the catalog is read.
|
|
125
129
|
for (const step of manifest.extract ?? []) {
|
|
126
130
|
const plugin = pluginNamed(step.plugin);
|
|
127
|
-
const stamp = stampFor(step.in, step.out);
|
|
128
131
|
// A plugin that asks for history gets the root's, read once per checkout
|
|
129
132
|
// (portolan.0007); left out when the root is not in a checkout, which the
|
|
130
133
|
// plugin reports in its own words.
|
|
@@ -135,13 +138,11 @@ async function generate() {
|
|
|
135
138
|
input: {
|
|
136
139
|
root: step.in,
|
|
137
140
|
output: step.out,
|
|
138
|
-
commit: stamp.commit,
|
|
139
|
-
generatedAt: stamp.generatedAt,
|
|
140
141
|
...(history ? { history } : {}),
|
|
141
142
|
},
|
|
142
143
|
options: step.options ?? {},
|
|
143
144
|
}, {}, { workspace: process.cwd() }),
|
|
144
|
-
);
|
|
145
|
+
inputsOf(step));
|
|
145
146
|
}
|
|
146
147
|
|
|
147
148
|
// Verifiers read observed evidence against the merged catalog while leaving
|
|
@@ -152,17 +153,16 @@ async function generate() {
|
|
|
152
153
|
// carry it into the generators and into the next run.
|
|
153
154
|
for (const step of manifest.verify ?? []) {
|
|
154
155
|
const plugin = pluginNamed(step.plugin);
|
|
155
|
-
const stamp = stampFor(step.in, step.out);
|
|
156
156
|
const own = (previous(step.out)[keys.keyOf(step)] ?? []).map((name) => join(step.out, name));
|
|
157
|
-
const { catalog } = await loadSources({ exclude: [...own, ...staleAll()] });
|
|
157
|
+
const { catalog, sources } = await loadSources({ exclude: [...own, ...staleAll()] });
|
|
158
158
|
await executeStep("verify", step, `${step.plugin} ⇐ ${step.in}`, async () =>
|
|
159
159
|
runPlugin(plugin, {
|
|
160
160
|
portolanVersion: PORTOLAN_VERSION,
|
|
161
|
-
input: { root: step.in, output: step.out
|
|
161
|
+
input: { root: step.in, output: step.out },
|
|
162
162
|
catalog,
|
|
163
163
|
options: step.options ?? {},
|
|
164
164
|
}, {}, { workspace: process.cwd() }),
|
|
165
|
-
);
|
|
165
|
+
inputsOf(step, sources.map((source) => source.path)));
|
|
166
166
|
}
|
|
167
167
|
|
|
168
168
|
// What a dropped extract step wrote is not part of the catalog, so it is
|
|
@@ -182,16 +182,16 @@ async function generate() {
|
|
|
182
182
|
|
|
183
183
|
for (const step of manifest.generate ?? []) {
|
|
184
184
|
const plugin = pluginNamed(step.plugin);
|
|
185
|
-
const
|
|
186
|
-
?
|
|
187
|
-
: catalog;
|
|
185
|
+
const generated = step.catalog
|
|
186
|
+
? await loadSources({ profile: step.catalog })
|
|
187
|
+
: { catalog, sources };
|
|
188
188
|
await executeStep("generate", step, `${step.plugin} → ${step.out}`, async () =>
|
|
189
189
|
runPlugin(plugin, {
|
|
190
190
|
portolanVersion: PORTOLAN_VERSION,
|
|
191
|
-
catalog:
|
|
191
|
+
catalog: generated.catalog,
|
|
192
192
|
options: step.options ?? {},
|
|
193
193
|
}),
|
|
194
|
-
);
|
|
194
|
+
{ inputs: generated.sources.map((source) => source.path), excludes: [] });
|
|
195
195
|
}
|
|
196
196
|
|
|
197
197
|
sweepAll();
|
|
@@ -249,13 +249,27 @@ function staleIn(out) {
|
|
|
249
249
|
return stale;
|
|
250
250
|
}
|
|
251
251
|
|
|
252
|
-
|
|
252
|
+
/**
|
|
253
|
+
* Runs one step and settles its files. `reads` is what the step reads - paths,
|
|
254
|
+
* and the output directories among them to leave out - so that an output
|
|
255
|
+
* that moved can be explained by what moved among its inputs since the
|
|
256
|
+
* output was last committed.
|
|
257
|
+
*/
|
|
258
|
+
async function executeStep(phase, step, label, work, reads) {
|
|
253
259
|
const startedAt = Date.now();
|
|
254
260
|
event({ type: "step-started", ordinal: report.steps.length, phase, plugin: step.plugin, input: step.in, output: step.out });
|
|
255
261
|
try {
|
|
256
262
|
const { files, warnings = [] } = await work();
|
|
263
|
+
const diagnostics = diagnoseWarnings({
|
|
264
|
+
plugin: step.plugin,
|
|
265
|
+
warnings,
|
|
266
|
+
policies: manifest.warningPolicies,
|
|
267
|
+
project: projectForStep(step),
|
|
268
|
+
phase,
|
|
269
|
+
});
|
|
257
270
|
const changes = apply(files, step.out, keys.keyOf(step), check);
|
|
258
|
-
const
|
|
271
|
+
const since = changes.length > 0 ? whyChanged(step, files.map((file) => join(step.out, file.name)), reads) : null;
|
|
272
|
+
const changed = summarise(label, files, changes, diagnostics, since);
|
|
259
273
|
drifted = changed || drifted;
|
|
260
274
|
const result = {
|
|
261
275
|
phase,
|
|
@@ -267,10 +281,14 @@ async function executeStep(phase, step, label, work) {
|
|
|
267
281
|
fileCount: files.length,
|
|
268
282
|
changedCount: changes.length,
|
|
269
283
|
changes,
|
|
284
|
+
// What moved among the inputs since the output was last committed, when
|
|
285
|
+
// something in the output moved; the Settings page can say why.
|
|
286
|
+
...(since ? { since } : {}),
|
|
270
287
|
files: files.map((file) => join(step.out, file.name)),
|
|
271
288
|
// What the plugin could not read, in its own words. Already printed as
|
|
272
289
|
// it ran; kept here so the Settings page can list it beside the step.
|
|
273
290
|
warnings,
|
|
291
|
+
diagnostics,
|
|
274
292
|
};
|
|
275
293
|
addBuildStep(report, result);
|
|
276
294
|
persistReport();
|
|
@@ -296,6 +314,15 @@ async function executeStep(phase, step, label, work) {
|
|
|
296
314
|
}
|
|
297
315
|
}
|
|
298
316
|
|
|
317
|
+
function projectForStep(step) {
|
|
318
|
+
if (!step.in) return "";
|
|
319
|
+
const input = String(step.in).replace(/\\/g, "/").replace(/\/+$/, "");
|
|
320
|
+
const projects = (manifest.projects ?? [])
|
|
321
|
+
.filter((project) => typeof project?.id === "string" && typeof project?.root === "string")
|
|
322
|
+
.sort((left, right) => right.root.length - left.root.length);
|
|
323
|
+
return projects.find((project) => input === project.root || input.startsWith(`${project.root}/`))?.id ?? "";
|
|
324
|
+
}
|
|
325
|
+
|
|
299
326
|
function persistReport() {
|
|
300
327
|
try {
|
|
301
328
|
writeBuildReport(report);
|
|
@@ -334,9 +361,17 @@ async function needsOf(plugin) {
|
|
|
334
361
|
return pluginNeeds.get(plugin.name);
|
|
335
362
|
}
|
|
336
363
|
|
|
337
|
-
/**
|
|
338
|
-
|
|
339
|
-
|
|
364
|
+
/**
|
|
365
|
+
* Prints what a step did, and says whether it left the tree out of date. A
|
|
366
|
+
* file that differs says where it first differs, and the step says what moved
|
|
367
|
+
* among its inputs since the output was last committed - or that nothing
|
|
368
|
+
* did, which points at the plugin.
|
|
369
|
+
*/
|
|
370
|
+
function summarise(label, files, changes, diagnostics = [], since = null) {
|
|
371
|
+
const suppressed = diagnostics.filter((diagnostic) => diagnostic.suppressed).length;
|
|
372
|
+
const said = diagnostics.length > 0
|
|
373
|
+
? `, ${diagnostics.length} warning${diagnostics.length === 1 ? "" : "s"}${suppressed ? ` (${suppressed} suppressed)` : ""}`
|
|
374
|
+
: "";
|
|
340
375
|
const summary = `${label}: ${files.length} file${files.length === 1 ? "" : "s"}${said}`;
|
|
341
376
|
|
|
342
377
|
if (changes.length === 0) {
|
|
@@ -347,120 +382,84 @@ function summarise(label, files, changes, warnings = []) {
|
|
|
347
382
|
|
|
348
383
|
if (!check) {
|
|
349
384
|
console.log(`${summary}, ${changes.length} written`);
|
|
385
|
+
if (since) console.log(` ${describeSince(since)}`);
|
|
350
386
|
|
|
351
387
|
return false;
|
|
352
388
|
}
|
|
353
389
|
|
|
354
390
|
console.error(`${summary}, ${changes.length} out of date:`);
|
|
355
|
-
for (const change of changes.slice(0, 20))
|
|
391
|
+
for (const change of changes.slice(0, 20)) {
|
|
392
|
+
console.error(` ${change.kind.padEnd(8)} ${change.path}${change.reason ? ` — ${change.reason}` : ""}`);
|
|
393
|
+
}
|
|
356
394
|
if (changes.length > 20) console.error(` ... and ${changes.length - 20} more`);
|
|
395
|
+
if (since) console.error(` ${describeSince(since)}`);
|
|
357
396
|
|
|
358
397
|
return true;
|
|
359
398
|
}
|
|
360
399
|
|
|
361
400
|
/**
|
|
362
|
-
*
|
|
363
|
-
*
|
|
364
|
-
*
|
|
365
|
-
* The host works this out rather than the extractor, for two reasons. A plugin
|
|
366
|
-
* that reads a clock produces a different fragment on every run, which cannot
|
|
367
|
-
* be committed and cannot be checked; and a plugin that shells out to git is a
|
|
368
|
-
* plugin that can never be sandboxed. Stamped from the last commit to touch the
|
|
369
|
-
* directory, a fragment changes exactly when its subject does.
|
|
370
|
-
*
|
|
371
|
-
* The output is excluded from that history, and it has to be: a fragment
|
|
372
|
-
* written beside the code it describes is inside the directory it is stamped
|
|
373
|
-
* from, so committing one would move the stamp, which would make the fragment
|
|
374
|
-
* out of date, which would rewrite it - and `--check` would never come back
|
|
375
|
-
* clean two runs in a row.
|
|
401
|
+
* What a step reads, for saying why its output moved: its root and whatever
|
|
402
|
+
* else it was handed, less every output directory inside the root, because a
|
|
403
|
+
* fragment written beside the code it describes is not an input to itself.
|
|
376
404
|
*/
|
|
377
|
-
function
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
//
|
|
384
|
-
// Short, like every other stamp, because a stamp is read rather than
|
|
385
|
-
// resolved. The full sha travels separately, in the pin the fetch also
|
|
386
|
-
// wrote, where a link is built from it.
|
|
387
|
-
const vendored = vendoredCommit(root);
|
|
388
|
-
if (vendored) {
|
|
389
|
-
// No date. Nothing here knows when that commit was made - the lock records
|
|
390
|
-
// what was fetched, not when it was authored - and a date invented from
|
|
391
|
-
// the local clock would make the merge call this the stalest source in the
|
|
392
|
-
// estate every time it ran.
|
|
393
|
-
return { commit: vendored.slice(0, 7), generatedAt: "" };
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
// The history read is the one the root lives in. A manifest pointed at a
|
|
397
|
-
// checkout elsewhere on the disk - the service being documented, not a copy
|
|
398
|
-
// of it vendored here - is stamped from that checkout's history, because the
|
|
399
|
-
// fragment describes that service and not the repository the manifest sits
|
|
400
|
-
// in. A root with no repository around it is stamped as uncommitted.
|
|
401
|
-
const repo = repositoryOf(root);
|
|
402
|
-
if (!repo) {
|
|
403
|
-
return { commit: "uncommitted", generatedAt: process.env.PORTOLAN_GENERATED_AT || new Date().toISOString() };
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
// A shallow clone has no history to read: the one commit that was fetched has
|
|
407
|
-
// no parent, so every path looks as though it changed there and every fragment
|
|
408
|
-
// is stamped with the checkout rather than with its subject. That is wrong
|
|
409
|
-
// quietly - the fragments regenerate, `--check` reports drift, and nothing
|
|
410
|
-
// says why - so it is refused here instead.
|
|
411
|
-
if (shallow(repo)) {
|
|
412
|
-
fail(
|
|
413
|
-
`${repo} is a shallow clone, where every path looks as though it changed in ` +
|
|
414
|
-
"the single commit that was fetched, so a fragment cannot be stamped " +
|
|
415
|
-
"with the commit it describes. Fetch the full history first " +
|
|
416
|
-
"(git fetch --unshallow, or actions/checkout with fetch-depth: 0).",
|
|
417
|
-
);
|
|
405
|
+
function inputsOf(step, extra = []) {
|
|
406
|
+
const excludes = [];
|
|
407
|
+
for (const other of allSteps()) {
|
|
408
|
+
if (!other.out || other.out === step.in) continue;
|
|
409
|
+
const inside = relative(step.in, other.out);
|
|
410
|
+
if (inside && !inside.startsWith("..") && !isAbsolute(inside)) excludes.push(other.out);
|
|
418
411
|
}
|
|
412
|
+
return { inputs: [step.in, ...extra], excludes };
|
|
413
|
+
}
|
|
419
414
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
// moved on every commit.
|
|
424
|
-
const rootInRepo = relative(repo, resolve(root)) || ".";
|
|
425
|
-
const outInRepo = out ? relative(repo, resolve(out)) : "";
|
|
426
|
-
const inside = out && (outInRepo === rootInRepo || outInRepo.startsWith(`${rootInRepo.replace(/\/$/, "")}/`) || rootInRepo === ".");
|
|
427
|
-
const exclude = inside && outInRepo !== rootInRepo && !outInRepo.startsWith("..") ? [`:(exclude)${outInRepo}`] : [];
|
|
428
|
-
|
|
429
|
-
for (const args of [
|
|
430
|
-
["log", "-1", "--format=%h %cI", "--", rootInRepo, ...exclude],
|
|
431
|
-
["log", "-1", "--format=%h %cI"],
|
|
432
|
-
]) {
|
|
433
|
-
const answer = git(repo, args);
|
|
434
|
-
if (!answer) continue;
|
|
435
|
-
const [commit, generatedAt] = answer.split(" ");
|
|
436
|
-
if (commit && generatedAt) return { commit, generatedAt };
|
|
437
|
-
}
|
|
415
|
+
function allSteps() {
|
|
416
|
+
return [...(manifest.extract ?? []), ...(manifest.verify ?? []), ...(manifest.generate ?? [])];
|
|
417
|
+
}
|
|
438
418
|
|
|
439
|
-
|
|
419
|
+
/**
|
|
420
|
+
* Why a step's output is not what it was: the commit that last touched the
|
|
421
|
+
* output is when it was last generated, and what changed among the inputs
|
|
422
|
+
* since then is the reason. The history is the record of the last generation
|
|
423
|
+
* (portolan.0010); nothing is written down to know this.
|
|
424
|
+
*
|
|
425
|
+
* The manifest counts as an input only where this step's own entry moved.
|
|
426
|
+
* The whole file changes whenever any step is touched, and naming it for
|
|
427
|
+
* every other step would explain nothing.
|
|
428
|
+
*/
|
|
429
|
+
function whyChanged(step, outputs, reads) {
|
|
430
|
+
const committed = lastCommitTouching(process.cwd(), outputs);
|
|
431
|
+
if (!committed) return { committed: null, changed: [] };
|
|
432
|
+
const changed = changedSince(process.cwd(), committed.commit, reads.inputs, reads.excludes);
|
|
433
|
+
if (stepEntryChanged(committed.commit, step)) changed.push("portolan.json (this step's entry)");
|
|
434
|
+
return { committed, changed };
|
|
440
435
|
}
|
|
441
436
|
|
|
442
|
-
/**
|
|
443
|
-
function
|
|
437
|
+
/** Whether the manifest at `commit` told this step the same thing it is told now. */
|
|
438
|
+
function stepEntryChanged(commit, step) {
|
|
439
|
+
let then;
|
|
444
440
|
try {
|
|
445
|
-
|
|
446
|
-
encoding: "utf8",
|
|
447
|
-
stdio: ["ignore", "pipe", "ignore"],
|
|
448
|
-
}).trim();
|
|
441
|
+
then = JSON.parse(fileAt(process.cwd(), commit, "portolan.json"));
|
|
449
442
|
} catch {
|
|
450
|
-
//
|
|
451
|
-
// said so on its own stderr, which is not this run's log.
|
|
452
|
-
return "";
|
|
443
|
+
return true; // No manifest there, or not one that parses: everything about the step is new.
|
|
453
444
|
}
|
|
445
|
+
const phase = ["extract", "verify", "generate"].find((name) => (manifest[name] ?? []).includes(step));
|
|
446
|
+
const before = (then[phase] ?? []).find(
|
|
447
|
+
(other) =>
|
|
448
|
+
other.plugin === step.plugin &&
|
|
449
|
+
(other.in ?? "") === (step.in ?? "") &&
|
|
450
|
+
other.out === step.out &&
|
|
451
|
+
(other.options?.out ?? "") === (step.options?.out ?? ""),
|
|
452
|
+
);
|
|
453
|
+
return !before || JSON.stringify(before) !== JSON.stringify(step);
|
|
454
454
|
}
|
|
455
455
|
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
return git(repo, ["rev-parse", "--is-shallow-repository"]) === "true";
|
|
456
|
+
function describeSince({ committed, changed }) {
|
|
457
|
+
if (!committed) return "the output has never been committed, so there is nothing to compare its inputs against";
|
|
458
|
+
const when = `${committed.commit} (${committed.date}), when the output was last committed`;
|
|
459
|
+
if (changed.length === 0) return `no input changed since ${when}; the plugin itself did`;
|
|
460
|
+
const shown = changed.slice(0, 6).join(", ");
|
|
461
|
+
const more = changed.length > 6 ? ` and ${changed.length - 6} more` : "";
|
|
462
|
+
return `since ${when}, ${changed.length} input${changed.length === 1 ? "" : "s"} changed: ${shown}${more}`;
|
|
464
463
|
}
|
|
465
464
|
|
|
466
465
|
/** Reads, merges and validates every source the manifest names. */
|
|
@@ -497,7 +496,11 @@ function apply(files, out, key, checkOnly) {
|
|
|
497
496
|
|
|
498
497
|
if (binary ? Buffer.isBuffer(current) && current.equals(wanted) : current === wanted) continue;
|
|
499
498
|
|
|
500
|
-
changes.push({
|
|
499
|
+
changes.push({
|
|
500
|
+
kind: current === null ? "added" : "changed",
|
|
501
|
+
path: join(out, file.name),
|
|
502
|
+
reason: explainChange(current, wanted, file.name),
|
|
503
|
+
});
|
|
501
504
|
if (checkOnly) continue;
|
|
502
505
|
|
|
503
506
|
try {
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { afterEach, expect, it } from "vitest";
|
|
5
|
+
import { discoverProject } from "./local-discovery.mjs";
|
|
6
|
+
|
|
7
|
+
const roots = [];
|
|
8
|
+
afterEach(() => roots.splice(0).forEach((root) => rmSync(root, { recursive: true, force: true })));
|
|
9
|
+
function discover(files) {
|
|
10
|
+
const root = mkdtempSync(join(tmpdir(), "portolan-go-discovery-"));
|
|
11
|
+
roots.push(root);
|
|
12
|
+
for (const [name, text] of Object.entries({ "go.mod": "module example.com/app\n", ...files })) {
|
|
13
|
+
const path = join(root, name);
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
15
|
+
writeFileSync(path, text);
|
|
16
|
+
}
|
|
17
|
+
return discoverProject(root, ".").detections.map((item) => item.plugin);
|
|
18
|
+
}
|
|
19
|
+
it("offers Go flows and SQL for a service without any DDD directories", () => {
|
|
20
|
+
expect(discover({
|
|
21
|
+
"web/routes.go": 'package web\nimport "net/http"\nfunc Register(m *http.ServeMux) { m.HandleFunc("POST /orders", create) }',
|
|
22
|
+
"store/migrations/001.sql": "CREATE TABLE orders (id uuid PRIMARY KEY);",
|
|
23
|
+
})).toEqual(expect.arrayContaining(["go-domain", "sql"]));
|
|
24
|
+
});
|
|
25
|
+
it("does not offer Go domain extraction for unregistered helpers or generated handlers", () => {
|
|
26
|
+
for (const source of [
|
|
27
|
+
'package web\nimport "net/http"\nfunc Help(w http.ResponseWriter, r *http.Request) {}',
|
|
28
|
+
'// Code generated by fixture. DO NOT EDIT.\npackage web\nimport "net/http"\nfunc Register() { http.HandleFunc("/x", help) }',
|
|
29
|
+
]) expect(discover({ "web/routes.go": source })).not.toContain("go-domain");
|
|
30
|
+
});
|