@arc-e-tect/api-only-publisher 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.adoc +501 -7
- package/package.json +1 -1
- package/src/build-error.js +7 -0
- package/src/channels.js +11 -5
- package/src/cli.js +22 -1
- package/src/handoff.js +43 -0
- package/src/index.js +1 -0
- package/src/init-config.js +2 -2
- package/src/init.js +3 -3
- package/src/nuget.js +5 -4
- package/src/pipeline.js +35 -13
- package/src/toolchain.js +123 -0
package/README.adoc
CHANGED
|
@@ -6,10 +6,11 @@
|
|
|
6
6
|
:source-highlighter: rouge
|
|
7
7
|
// The released version of this component: its snippets use it. The release
|
|
8
8
|
// workflow updates it; do not change it by hand.
|
|
9
|
-
:api-only-publisher-version: 0.
|
|
9
|
+
:api-only-publisher-version: 0.12.0
|
|
10
10
|
|
|
11
11
|
image:https://github.com/Arc-E-Tect/SoftwareEngineeringDoneRight-API/actions/workflows/nvd-cache-refresh.yml/badge.svg[Vulnerability Scan,link=https://github.com/Arc-E-Tect/SoftwareEngineeringDoneRight-API/actions/workflows/nvd-cache-refresh.yml]
|
|
12
12
|
image:https://img.shields.io/npm/v/@arc-e-tect/api-only-publisher[npm,link=https://www.npmjs.com/package/@arc-e-tect/api-only-publisher]
|
|
13
|
+
image:https://img.shields.io/docker/v/arcetect/api-only-publisher?sort=semver&label=Docker%20Hub[Docker Hub,link=https://hub.docker.com/r/arcetect/api-only-publisher]
|
|
13
14
|
image:https://img.shields.io/github/v/release/Arc-E-Tect/SoftwareEngineeringDoneRight-API?filter=api-only-publisher-v*[GitHub release,link=https://github.com/Arc-E-Tect/SoftwareEngineeringDoneRight-API/releases?q=api-only-publisher]
|
|
14
15
|
image:https://img.shields.io/badge/Node-22.14%2B-339933?logo=node.js[Node 22.14+,link=https://nodejs.org]
|
|
15
16
|
image:https://img.shields.io/badge/License-MIT-blue[MIT,link=LICENSE]
|
|
@@ -63,6 +64,8 @@ To set it up for your situation, start at link:../docs/README.adoc[the documenta
|
|
|
63
64
|
Nothing else.
|
|
64
65
|
The bundlers, https://redocly.com/docs/cli/[Redocly CLI ↗] and https://www.asyncapi.com/tools/cli[AsyncAPI CLI ↗], are fetched by `npx` at the versions `apionly.yaml` pins, so they need no installation, and the only runtime dependency is `yaml`.
|
|
65
66
|
|
|
67
|
+
Or Docker instead of Node.js: the image holds the Publisher and its bundlers, and downloads nothing when it runs, as <<docker>> describes.
|
|
68
|
+
|
|
66
69
|
[#installation]
|
|
67
70
|
== Installation
|
|
68
71
|
|
|
@@ -131,6 +134,22 @@ A published contract should change only when its fragments or its version change
|
|
|
131
134
|
The bundlers are a separate matter.
|
|
132
135
|
However the Publisher itself is started, it fetches them with `npx`, at the versions `toolchain` in `apionly.yaml` pins.
|
|
133
136
|
|
|
137
|
+
[#as-a-docker-image]
|
|
138
|
+
=== As a Docker image
|
|
139
|
+
|
|
140
|
+
The fit for a CI pipeline that may run only vetted, mirrored artefacts.
|
|
141
|
+
|
|
142
|
+
[source,console,subs="attributes+"]
|
|
143
|
+
----
|
|
144
|
+
docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/work" arcetect/api-only-publisher:{api-only-publisher-version} build
|
|
145
|
+
----
|
|
146
|
+
|
|
147
|
+
* One artefact, with one digest, holds the Publisher *and* its bundlers, so a pipeline pins, verifies, scans and mirrors everything it runs at once.
|
|
148
|
+
* It downloads nothing when it runs, so `apionly.yaml` must pin the bundler versions the image carries.
|
|
149
|
+
* The pin is the image's digest, in each pipeline that runs it; nothing in the library records it.
|
|
150
|
+
|
|
151
|
+
<<docker>> describes it in full.
|
|
152
|
+
|
|
134
153
|
[#commands]
|
|
135
154
|
== Commands
|
|
136
155
|
|
|
@@ -176,6 +195,7 @@ A target whose closure is untouched needs no new version, however much else in t
|
|
|
176
195
|
`--target` and `--channel` may be repeated.
|
|
177
196
|
`-C <dir>` runs as if started in `<dir>`; without it the configuration is found by walking up from the current directory.
|
|
178
197
|
`-q` or `--quiet` reports only errors, which is what a script that reads the output wants.
|
|
198
|
+
`--version`, on its own, prints the Publisher's version.
|
|
179
199
|
`--pre-release <ids>` appends pre-release identifiers to each target's version, as <<versions,Versions §>> describes.
|
|
180
200
|
|
|
181
201
|
[source,console]
|
|
@@ -651,6 +671,28 @@ What happens when the destination already has the version:
|
|
|
651
671
|
A NuGet feed that cannot return the package it has, so there is nothing to compare, fails as a different one would.
|
|
652
672
|
|===
|
|
653
673
|
|
|
674
|
+
[#handoff]
|
|
675
|
+
=== The hand-off: what a `publish` run wrote
|
|
676
|
+
|
|
677
|
+
Every `publish` run lists each file its channels wrote locally, with its SHA-256, in two files in the packages directory -- `build.packages`, `build/packages` by default, or `--out`:
|
|
678
|
+
|
|
679
|
+
[cols="1,3", options="header"]
|
|
680
|
+
|===
|
|
681
|
+
| File | Content
|
|
682
|
+
|
|
683
|
+
| `packages.json`
|
|
684
|
+
| `schemaVersion`, and `files`: for each file, its `path`, `sha256`, `target`, `version` and `channel`.
|
|
685
|
+
|
|
686
|
+
| `packages.sha256`
|
|
687
|
+
| The same files in the format `sha256sum -c` checks.
|
|
688
|
+
|===
|
|
689
|
+
|
|
690
|
+
Paths are relative to the library root, with forward slashes, so `sha256sum -c build/packages/packages.sha256` checks them from there.
|
|
691
|
+
A run replaces both files.
|
|
692
|
+
A channel in a remote mode writes nothing locally, so it adds nothing.
|
|
693
|
+
|
|
694
|
+
The hand-off is for a later step that publishes those files itself -- the second of <<docker-two-steps,the image's two steps §>> -- so that it ships exactly the bytes the Publisher produced, and nothing else.
|
|
695
|
+
|
|
654
696
|
[#channel-text]
|
|
655
697
|
=== What a package says about itself
|
|
656
698
|
|
|
@@ -797,8 +839,8 @@ lint:
|
|
|
797
839
|
openapi: warn # a request body or response with no example: error, warn (the default) or off
|
|
798
840
|
|
|
799
841
|
toolchain:
|
|
800
|
-
redocly: "@redocly/cli@2.
|
|
801
|
-
asyncapi: "@asyncapi/cli@6.0
|
|
842
|
+
redocly: "@redocly/cli@2.55.0"
|
|
843
|
+
asyncapi: "@asyncapi/cli@6.2.0"
|
|
802
844
|
|
|
803
845
|
channels:
|
|
804
846
|
file:
|
|
@@ -863,6 +905,24 @@ A bundle root moved into or out of `bundles/` needs its relative `$ref`s adjuste
|
|
|
863
905
|
Unpinned, the documents this tool produces could change because an upstream release happened, with nothing in the library having changed.
|
|
864
906
|
An upgrade should be a deliberate commit that moves the pin, reviewed with whatever it changes in the published documents.
|
|
865
907
|
|
|
908
|
+
Two environment variables decide where a pinned bundler comes from.
|
|
909
|
+
Neither is set outside the Docker image, which is then exactly the `npx` behaviour above.
|
|
910
|
+
|
|
911
|
+
[cols="1,3", options="header"]
|
|
912
|
+
|===
|
|
913
|
+
| Variable | Effect
|
|
914
|
+
|
|
915
|
+
| `API_ONLY_PUBLISHER_TOOLCHAIN_DIR`
|
|
916
|
+
| A directory holding installed bundlers, as `npm install --prefix <dir>` leaves it.
|
|
917
|
+
A bundler installed there at exactly the pinned version runs from there, without `npx`.
|
|
918
|
+
|
|
919
|
+
| `API_ONLY_PUBLISHER_TOOLCHAIN_DOWNLOAD`
|
|
920
|
+
| `allow`, the default, fetches any other pin with `npx`.
|
|
921
|
+
`never` refuses to: `build` and `lint` fail before doing anything, naming the versions the directory holds.
|
|
922
|
+
|===
|
|
923
|
+
|
|
924
|
+
The Docker image sets both, with `never`, <<docker-toolchain>>.
|
|
925
|
+
|
|
866
926
|
[#behaviour]
|
|
867
927
|
== Behaviour worth knowing about
|
|
868
928
|
|
|
@@ -988,13 +1048,13 @@ So there is nothing to hoist and nothing to select: every fragment is stamped, a
|
|
|
988
1048
|
----
|
|
989
1049
|
channels:
|
|
990
1050
|
auditV1:
|
|
991
|
-
x-fragment-path: asyncapi/channels/
|
|
992
|
-
address:
|
|
1051
|
+
x-fragment-path: asyncapi/channels/apionly/useraccount/AuditV1.yaml
|
|
1052
|
+
address: apionly.useraccount.audit.v1
|
|
993
1053
|
messages:
|
|
994
1054
|
registrationInitiatedMessage:
|
|
995
|
-
x-fragment-path: asyncapi/messages/
|
|
1055
|
+
x-fragment-path: asyncapi/messages/apionly/useraccount/RegistrationInitiatedMessageV1.yaml
|
|
996
1056
|
payload:
|
|
997
|
-
x-fragment-path: asyncapi/components/
|
|
1057
|
+
x-fragment-path: asyncapi/components/apionly/useraccount/schemas/RegistrationInitiatedEventV1.yaml
|
|
998
1058
|
properties:
|
|
999
1059
|
username:
|
|
1000
1060
|
x-fragment-path: openapi/components/common/schemas/UsernameV1.yaml
|
|
@@ -1046,6 +1106,440 @@ A producer has no business knowing the directory layout of the projects that con
|
|
|
1046
1106
|
The `distribution` block copies built documents into such a layout, so that a library can adopt this tool before its consumers adopt the API-Only Subscriber.
|
|
1047
1107
|
Once they subscribe, each consuming project declares where its own copy lands, and the block goes away.
|
|
1048
1108
|
|
|
1109
|
+
[#docker]
|
|
1110
|
+
== The Docker image
|
|
1111
|
+
|
|
1112
|
+
The Publisher is also a Docker image, for CI pipelines that may run only what has been vetted and mirrored into the organisation's own registry.
|
|
1113
|
+
|
|
1114
|
+
[cols="1,3"]
|
|
1115
|
+
|===
|
|
1116
|
+
| Docker Hub | `docker.io/arcetect/api-only-publisher`
|
|
1117
|
+
| GHCR | `ghcr.io/arc-e-tect/api-only-publisher`
|
|
1118
|
+
| Platforms | `linux/amd64` and `linux/arm64`, as one multi-platform image
|
|
1119
|
+
|===
|
|
1120
|
+
|
|
1121
|
+
Vetting the npm package does not vet what it runs: it fetches Redocly CLI and AsyncAPI CLI with `npx` when it runs.
|
|
1122
|
+
The image is one artefact, with one digest, holding one Publisher version *and* its toolchain, and it is *sealed*: it downloads nothing, and contacts nothing, when it runs.
|
|
1123
|
+
It can be pulled by digest, its signature verified, its SBOM and provenance read, scanned, and mirrored, and then a pipeline knows exactly what it runs.
|
|
1124
|
+
|
|
1125
|
+
Both registries serve the same image, with the same digest: Docker Hub gets a copy of what GHCR was given, never a rebuild.
|
|
1126
|
+
|
|
1127
|
+
link:../docs/guides/publishing-from-ci/overview.adoc[Publishing from CI with the image →] is the guide, with prompts that set up the pipeline in your own repository.
|
|
1128
|
+
|
|
1129
|
+
[#docker-two-steps]
|
|
1130
|
+
=== Two steps: package in the image, publish with the platform's tools
|
|
1131
|
+
|
|
1132
|
+
. *In the image*, the Publisher builds, lints and packages bundles into files, with the local modes of its channels: `file`, `maven` and `nuget` to a directory, and `npm` with `publish: false`.
|
|
1133
|
+
It needs no network and no credential.
|
|
1134
|
+
. *In the pipeline*, a following job publishes those files to the organisation's registry with that platform's own tools and credentials: `npm publish`, `mvn deploy:deploy-file`, `dotnet nuget push`, a release upload.
|
|
1135
|
+
|
|
1136
|
+
Every `publish` run writes a hand-off, <<handoff,The hand-off §>>, listing each file it wrote with its SHA-256.
|
|
1137
|
+
The publishing job checks the files against it, and publishes exactly those bytes: the bytes in the registry are the bytes the vetted tool produced.
|
|
1138
|
+
|
|
1139
|
+
Credentials never enter the image.
|
|
1140
|
+
|
|
1141
|
+
[IMPORTANT]
|
|
1142
|
+
.What the image does not do yet
|
|
1143
|
+
====
|
|
1144
|
+
The image does not publish to a remote registry itself.
|
|
1145
|
+
The channels' remote modes -- `maven` or `nuget` to a URL, `npm` with publishing, `github-release` -- may work from inside it, but they are neither tested nor supported there.
|
|
1146
|
+
Publish in a second step, as <<docker-pipelines>> shows.
|
|
1147
|
+
====
|
|
1148
|
+
|
|
1149
|
+
[#docker-tags]
|
|
1150
|
+
=== Tags, and why to pin by digest
|
|
1151
|
+
|
|
1152
|
+
[cols="1,3", options="header"]
|
|
1153
|
+
|===
|
|
1154
|
+
| Tag | Points at
|
|
1155
|
+
|
|
1156
|
+
| `{api-only-publisher-version}`
|
|
1157
|
+
| That version, and nothing else, for good.
|
|
1158
|
+
Once pushed, an exact-version tag never points at another digest: a re-publish of a version reuses the image it already has, and never rebuilds it.
|
|
1159
|
+
|
|
1160
|
+
| `0.11`, `0`, `latest`
|
|
1161
|
+
| The highest released version.
|
|
1162
|
+
They move with each new highest release, never for a re-publish of an older version, and never for a pre-release.
|
|
1163
|
+
|
|
1164
|
+
| `1.0.0-rc.1`, …
|
|
1165
|
+
| A pre-release gets only its exact tag.
|
|
1166
|
+
|===
|
|
1167
|
+
|
|
1168
|
+
Pin by digest, as `arcetect/api-only-publisher@sha256:<digest>`.
|
|
1169
|
+
A digest names the bytes a vetting team approved, in any registry the image is mirrored to; a tag is only a name, and a mirror may be configured to let it move.
|
|
1170
|
+
|
|
1171
|
+
Find a version's digest with:
|
|
1172
|
+
|
|
1173
|
+
[source,console,subs="attributes+"]
|
|
1174
|
+
----
|
|
1175
|
+
docker buildx imagetools inspect arcetect/api-only-publisher:{api-only-publisher-version} --format '{{ .Manifest.Digest }}'
|
|
1176
|
+
----
|
|
1177
|
+
|
|
1178
|
+
The GitHub Release of each version lists it too, with both references.
|
|
1179
|
+
|
|
1180
|
+
[#docker-vetting]
|
|
1181
|
+
=== For a vetting team
|
|
1182
|
+
|
|
1183
|
+
[#docker-contents]
|
|
1184
|
+
==== What is inside
|
|
1185
|
+
|
|
1186
|
+
[cols="1,3", options="header"]
|
|
1187
|
+
|===
|
|
1188
|
+
| Part | Detail
|
|
1189
|
+
|
|
1190
|
+
| Base
|
|
1191
|
+
| `node:24-trixie-slim`, the official Node.js 24 image on Debian 13, pinned by digest.
|
|
1192
|
+
|
|
1193
|
+
| Added packages
|
|
1194
|
+
| `git` and `ca-certificates`; `tar` and `npm` come with the base.
|
|
1195
|
+
|
|
1196
|
+
| The Publisher
|
|
1197
|
+
| The npm package `@arc-e-tect/api-only-publisher` at the image's version: the release workflow installs the tarball npm serves for that version.
|
|
1198
|
+
|
|
1199
|
+
| The toolchain
|
|
1200
|
+
| Redocly CLI and AsyncAPI CLI, at the versions `init` pins, installed from a lockfile into `/opt/api-only-publisher/toolchain`.
|
|
1201
|
+
|
|
1202
|
+
| User
|
|
1203
|
+
| UID `10001`, GID `0`: not root.
|
|
1204
|
+
Any other UID works too, <<docker-user>>.
|
|
1205
|
+
|===
|
|
1206
|
+
|
|
1207
|
+
The labels say so without running anything:
|
|
1208
|
+
|
|
1209
|
+
[source,console,subs="attributes+"]
|
|
1210
|
+
----
|
|
1211
|
+
docker pull arcetect/api-only-publisher:{api-only-publisher-version}
|
|
1212
|
+
docker image inspect arcetect/api-only-publisher:{api-only-publisher-version} --format '{{ json .Config.Labels }}'
|
|
1213
|
+
----
|
|
1214
|
+
|
|
1215
|
+
Besides the OCI annotations -- `source`, `revision`, `version`, `created`, `licenses` and the rest -- two labels name the toolchain:
|
|
1216
|
+
|
|
1217
|
+
[cols="2,1", options="header"]
|
|
1218
|
+
|===
|
|
1219
|
+
| Label | Example
|
|
1220
|
+
| `com.arc-e-tect.api-only-publisher.toolchain.redocly` | `@redocly/cli@2.55.0`
|
|
1221
|
+
| `com.arc-e-tect.api-only-publisher.toolchain.asyncapi` | `@asyncapi/cli@6.2.0`
|
|
1222
|
+
|===
|
|
1223
|
+
|
|
1224
|
+
[#docker-sbom]
|
|
1225
|
+
==== SBOM and provenance
|
|
1226
|
+
|
|
1227
|
+
Each image carries an SPDX SBOM and SLSA provenance as attestations, per platform.
|
|
1228
|
+
The SBOM lists every package in the image: the Debian packages, and the npm packages of the Publisher and the toolchain with their whole dependency trees.
|
|
1229
|
+
The provenance names this repository, the release workflow and the commit.
|
|
1230
|
+
|
|
1231
|
+
[source,console,subs="attributes+"]
|
|
1232
|
+
----
|
|
1233
|
+
docker buildx imagetools inspect arcetect/api-only-publisher:{api-only-publisher-version} --format '{{ json .SBOM }}'
|
|
1234
|
+
docker buildx imagetools inspect arcetect/api-only-publisher:{api-only-publisher-version} --format '{{ json .Provenance }}'
|
|
1235
|
+
----
|
|
1236
|
+
|
|
1237
|
+
The GitHub Release of each version has both platforms' SBOMs attached, as `api-only-publisher-image-<version>-linux-<arch>.spdx.json`.
|
|
1238
|
+
|
|
1239
|
+
[#docker-signature]
|
|
1240
|
+
==== Verifying the signature
|
|
1241
|
+
|
|
1242
|
+
Each image is signed with https://docs.sigstore.dev/[Sigstore cosign ↗], keyless: the signing certificate is issued to the release workflow's GitHub OIDC identity, so no signing key exists to be stolen.
|
|
1243
|
+
Verify against exactly that identity:
|
|
1244
|
+
|
|
1245
|
+
[source,console,subs="attributes+"]
|
|
1246
|
+
----
|
|
1247
|
+
cosign verify \
|
|
1248
|
+
--certificate-identity https://github.com/Arc-E-Tect/SoftwareEngineeringDoneRight-API/.github/workflows/api-only-publisher-image.yml@refs/heads/main \
|
|
1249
|
+
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
|
1250
|
+
arcetect/api-only-publisher@sha256:<digest>
|
|
1251
|
+
----
|
|
1252
|
+
|
|
1253
|
+
The same command, with `ghcr.io/arc-e-tect/api-only-publisher`, verifies GHCR's copy.
|
|
1254
|
+
The identity is the image workflow, at `main`, which the release workflow calls.
|
|
1255
|
+
|
|
1256
|
+
[#docker-scanning]
|
|
1257
|
+
==== Scanning
|
|
1258
|
+
|
|
1259
|
+
Each release is scanned with https://trivy.dev/[Trivy ↗], every platform, every severity, and the reports are attached to its GitHub Release as `api-only-publisher-image-<version>-linux-<arch>.trivy.json` and `.txt`.
|
|
1260
|
+
|
|
1261
|
+
A *CRITICAL* finding that has a fix blocks the release from reaching Docker Hub.
|
|
1262
|
+
The exceptions are listed in link:.trivyignore.yaml[`.trivyignore.yaml` →], each with its reason, the path it applies to, and the date it expires.
|
|
1263
|
+
Findings without a fix, which every Debian base has, are reported and do not block.
|
|
1264
|
+
|
|
1265
|
+
Scan it yourself as your policy requires, for example:
|
|
1266
|
+
|
|
1267
|
+
[source,console]
|
|
1268
|
+
----
|
|
1269
|
+
trivy image --ignorefile .trivyignore.yaml arcetect/api-only-publisher@sha256:<digest>
|
|
1270
|
+
----
|
|
1271
|
+
|
|
1272
|
+
[#docker-mirroring]
|
|
1273
|
+
==== Mirroring into an internal registry
|
|
1274
|
+
|
|
1275
|
+
Copy the image with everything attached to it -- both platforms, their SBOM and provenance attestations, and the signature -- with https://oras.land/[the ORAS CLI ↗]:
|
|
1276
|
+
|
|
1277
|
+
[source,console,subs="attributes+"]
|
|
1278
|
+
----
|
|
1279
|
+
oras cp -r \
|
|
1280
|
+
docker.io/arcetect/api-only-publisher@sha256:<digest> \
|
|
1281
|
+
registry.example.com/mirror/arcetect/api-only-publisher:{api-only-publisher-version}
|
|
1282
|
+
----
|
|
1283
|
+
|
|
1284
|
+
`-r` copies the image's referrers too, which is where cosign stores the signature: an OCI referrer, a Sigstore bundle, not a `sha256-<digest>.sig` tag.
|
|
1285
|
+
`skopeo copy --all` and `crane copy` keep the platforms and the attestations, but not a referrer, so a copy made with them has no signature to verify.
|
|
1286
|
+
The mirror must support the OCI 1.1 referrers API, as Docker Hub, GHCR and the CNCF Distribution registry, version 3, do; check your registry's documentation for its own.
|
|
1287
|
+
|
|
1288
|
+
The digest does not change in the copy.
|
|
1289
|
+
Verify the mirror's copy as the original, with the mirror's name:
|
|
1290
|
+
|
|
1291
|
+
[source,console]
|
|
1292
|
+
----
|
|
1293
|
+
cosign verify \
|
|
1294
|
+
--certificate-identity https://github.com/Arc-E-Tect/SoftwareEngineeringDoneRight-API/.github/workflows/api-only-publisher-image.yml@refs/heads/main \
|
|
1295
|
+
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
|
1296
|
+
registry.example.com/mirror/arcetect/api-only-publisher@sha256:<digest>
|
|
1297
|
+
----
|
|
1298
|
+
|
|
1299
|
+
Pipelines then pin the same digest, with the mirror's name.
|
|
1300
|
+
|
|
1301
|
+
[#docker-pipelines]
|
|
1302
|
+
=== For a pipeline author
|
|
1303
|
+
|
|
1304
|
+
[#docker-run]
|
|
1305
|
+
==== Every command, as `docker run`
|
|
1306
|
+
|
|
1307
|
+
Run from the library's root, which is mounted at `/work`, the image's working directory:
|
|
1308
|
+
|
|
1309
|
+
[source,console,subs="attributes+"]
|
|
1310
|
+
----
|
|
1311
|
+
docker run --rm --network none --user "$(id -u):$(id -g)" -v "$PWD:/work" \
|
|
1312
|
+
arcetect/api-only-publisher@sha256:<digest> init --yes --openapi --asyncapi .
|
|
1313
|
+
docker run --rm --network none --user "$(id -u):$(id -g)" -v "$PWD:/work" \
|
|
1314
|
+
arcetect/api-only-publisher@sha256:<digest> build
|
|
1315
|
+
docker run --rm --network none --user "$(id -u):$(id -g)" -v "$PWD:/work" \
|
|
1316
|
+
arcetect/api-only-publisher@sha256:<digest> publish --target orders --target payments
|
|
1317
|
+
----
|
|
1318
|
+
|
|
1319
|
+
The image's arguments are the Publisher's: every command in <<commands>> works this way, and so do `--help` and `--version`.
|
|
1320
|
+
Anything else runs as given, `sh -c '…'` included.
|
|
1321
|
+
|
|
1322
|
+
`--network none` is not needed, and costs nothing: nothing in the image uses the network.
|
|
1323
|
+
|
|
1324
|
+
[#docker-user]
|
|
1325
|
+
==== The user, file ownership and git
|
|
1326
|
+
|
|
1327
|
+
`--user "$(id -u):$(id -g)"` runs the Publisher as you, so the files it writes into `/work` are yours.
|
|
1328
|
+
Any UID works, with or without an entry in the image's `/etc/passwd`: `HOME` is `/tmp`, which any user can write.
|
|
1329
|
+
|
|
1330
|
+
`git` trusts a mounted repository whoever owns it; without that, git would refuse a repository owned by another user as "dubious ownership".
|
|
1331
|
+
The library does not have to be a git repository, but `pack` and `publish` read the commit time and the `origin` remote from it when it is.
|
|
1332
|
+
|
|
1333
|
+
[#docker-read-only]
|
|
1334
|
+
==== A read-only root filesystem
|
|
1335
|
+
|
|
1336
|
+
The image runs with `--read-only`, as hardened platforms enforce, given a writable `/tmp`:
|
|
1337
|
+
|
|
1338
|
+
[source,console]
|
|
1339
|
+
----
|
|
1340
|
+
docker run --rm --read-only --tmpfs /tmp --user "$(id -u):$(id -g)" -v "$PWD:/work" \
|
|
1341
|
+
arcetect/api-only-publisher@sha256:<digest> build
|
|
1342
|
+
----
|
|
1343
|
+
|
|
1344
|
+
[#docker-toolchain]
|
|
1345
|
+
==== The toolchain rule, and the opt-in
|
|
1346
|
+
|
|
1347
|
+
The image runs its own toolchain, and only that: `apionly.yaml` must pin the versions the image carries, the ones its `com.arc-e-tect.api-only-publisher.toolchain.*` labels name, and `init` writes.
|
|
1348
|
+
A library pinning another version fails at once, before anything is built and without reaching for the network:
|
|
1349
|
+
|
|
1350
|
+
----
|
|
1351
|
+
Error: apionly.yaml's toolchain.redocly pins @redocly/cli@2.51.0, and API_ONLY_PUBLISHER_TOOLCHAIN_DOWNLOAD=never forbids downloading it.
|
|
1352
|
+
This installation carries @asyncapi/cli@6.2.0 and @redocly/cli@2.55.0.
|
|
1353
|
+
Set toolchain.redocly to "@redocly/cli@2.55.0" in apionly.yaml, or use the api-only-publisher image whose com.arc-e-tect.api-only-publisher.toolchain.redocly label is @redocly/cli@2.51.0.
|
|
1354
|
+
To allow the download instead, run with API_ONLY_PUBLISHER_TOOLCHAIN_DOWNLOAD=allow.
|
|
1355
|
+
----
|
|
1356
|
+
|
|
1357
|
+
Align `toolchain` in `apionly.yaml`, or use the image version that carries the pinned toolchain.
|
|
1358
|
+
|
|
1359
|
+
`API_ONLY_PUBLISHER_TOOLCHAIN_DOWNLOAD=allow` lets the Publisher fetch a pin the image does not carry with `npx`, as it does outside Docker.
|
|
1360
|
+
It is off in the image, and exists for someone who knowingly accepts a download; the image is then no longer sealed.
|
|
1361
|
+
With a read-only root, `/tmp` must allow what `npx` fetches to run:
|
|
1362
|
+
|
|
1363
|
+
[source,console]
|
|
1364
|
+
----
|
|
1365
|
+
docker run --rm --read-only --tmpfs /tmp:exec -e API_ONLY_PUBLISHER_TOOLCHAIN_DOWNLOAD=allow \
|
|
1366
|
+
--user "$(id -u):$(id -g)" -v "$PWD:/work" arcetect/api-only-publisher@sha256:<digest> build
|
|
1367
|
+
----
|
|
1368
|
+
|
|
1369
|
+
<<pinning-the-bundlers>> describes the variables.
|
|
1370
|
+
|
|
1371
|
+
[#docker-network]
|
|
1372
|
+
==== Nothing phones home
|
|
1373
|
+
|
|
1374
|
+
The image makes no outbound connection: it sets, in its environment, what switches off each tool's own:
|
|
1375
|
+
|
|
1376
|
+
[cols="1,2", options="header"]
|
|
1377
|
+
|===
|
|
1378
|
+
| Tool | Switched off with
|
|
1379
|
+
| npm | `NPM_CONFIG_UPDATE_NOTIFIER=false`, `NPM_CONFIG_FUND=false`, `NPM_CONFIG_AUDIT=false`
|
|
1380
|
+
| Redocly CLI | `REDOCLY_TELEMETRY=off`, `REDOCLY_SUPPRESS_UPDATE_NOTICE=true`
|
|
1381
|
+
| AsyncAPI CLI | `ASYNCAPI_METRICS_CONFIG_PATH`, naming a file with analytics disabled, and `ASYNCAPI_SKIP_NEW_VERSION_CHECK=true`
|
|
1382
|
+
|===
|
|
1383
|
+
|
|
1384
|
+
This is tested, not assumed: the whole smoke test was run on a network whose every packet was captured, and the image sent none.
|
|
1385
|
+
|
|
1386
|
+
[#docker-ci-job]
|
|
1387
|
+
==== As a CI job image
|
|
1388
|
+
|
|
1389
|
+
The image is also a job image, for platforms that run a job's own shell commands inside it.
|
|
1390
|
+
Its entrypoint runs the Publisher when the first argument is one of its commands or a flag, and anything else as given, so a platform that starts the image with a shell needs no entrypoint override.
|
|
1391
|
+
|
|
1392
|
+
GitHub Actions replaces the entrypoint of a `container:` job, and runs its steps as the image's user by default, which cannot write the workspace GitHub mounts; run the job as the runner's user, UID 1001:
|
|
1393
|
+
|
|
1394
|
+
[source,yaml]
|
|
1395
|
+
----
|
|
1396
|
+
include::pipelines/github-actions-container-job.yml[]
|
|
1397
|
+
----
|
|
1398
|
+
|
|
1399
|
+
A GitLab CI job uses it as it stands:
|
|
1400
|
+
|
|
1401
|
+
[source,yaml]
|
|
1402
|
+
----
|
|
1403
|
+
package:
|
|
1404
|
+
image: docker.io/arcetect/api-only-publisher@sha256:<digest>
|
|
1405
|
+
script:
|
|
1406
|
+
- api-only-publisher build
|
|
1407
|
+
----
|
|
1408
|
+
|
|
1409
|
+
[#docker-github-actions]
|
|
1410
|
+
==== Reference pipeline: GitHub Actions
|
|
1411
|
+
|
|
1412
|
+
Two jobs: `package` runs the image and hands the files on, `publish` checks them against the hand-off and publishes them to npm, Maven and NuGet on GitHub Packages, and to a GitHub Release, with the job's `GITHUB_TOKEN`.
|
|
1413
|
+
Copy it into `.github/workflows/`, then set the digest, the targets and the trigger.
|
|
1414
|
+
The container-job variant above replaces its `package` job.
|
|
1415
|
+
|
|
1416
|
+
It is exercised end to end, both variants, by a workflow in this repository that publishes to scratch packages and resolves each one with its ecosystem's own client and with the API-Only Subscriber.
|
|
1417
|
+
|
|
1418
|
+
[source,yaml]
|
|
1419
|
+
----
|
|
1420
|
+
include::pipelines/github-actions.yml[]
|
|
1421
|
+
----
|
|
1422
|
+
|
|
1423
|
+
What the two steps lose, and how the pipeline makes up for it:
|
|
1424
|
+
|
|
1425
|
+
* *Idempotence.* A channel's remote mode leaves a version the registry has alone when it is identical, and refuses it when it differs; `npm publish` or `dotnet nuget push` of an existing version fails instead.
|
|
1426
|
+
Each publishing step checks for the version first, compares the bytes, and publishes only what is missing, so a re-run is safe and a real conflict still fails.
|
|
1427
|
+
* *Maven checksums and metadata.* The local Maven layout holds the artifact, its POM and its manifest, and no `.md5`, `.sha1` or `maven-metadata.xml`.
|
|
1428
|
+
`mvn deploy:deploy-file` writes those when it uploads, so the repository ends up as a remote `maven` channel would leave it, and Gradle resolves it.
|
|
1429
|
+
|
|
1430
|
+
[#docker-gitlab]
|
|
1431
|
+
==== Reference pipeline: GitLab CI
|
|
1432
|
+
|
|
1433
|
+
[WARNING]
|
|
1434
|
+
====
|
|
1435
|
+
*Not exercised.*
|
|
1436
|
+
This pipeline is validated against GitLab's published CI schema only; no GitLab pipeline has run it yet.
|
|
1437
|
+
====
|
|
1438
|
+
|
|
1439
|
+
The same two steps: `package` runs in the image, and one `publish:*` job per registry publishes to the project's package registry with `CI_JOB_TOKEN`: npm, Maven and NuGet, and the archives to the generic package registry.
|
|
1440
|
+
|
|
1441
|
+
[source,yaml]
|
|
1442
|
+
----
|
|
1443
|
+
include::pipelines/gitlab-ci.yml[]
|
|
1444
|
+
----
|
|
1445
|
+
|
|
1446
|
+
[#docker-mirror-in-pipelines]
|
|
1447
|
+
==== Using a mirrored image
|
|
1448
|
+
|
|
1449
|
+
Replace the image reference with the mirror's, keeping the digest: `registry.example.com/mirror/arcetect/api-only-publisher@sha256:<digest>`.
|
|
1450
|
+
Both reference pipelines name the image in one place: `PUBLISHER_IMAGE`, and the container job's `image`.
|
|
1451
|
+
|
|
1452
|
+
[#docker-maintainers]
|
|
1453
|
+
=== For maintainers
|
|
1454
|
+
|
|
1455
|
+
[#docker-release]
|
|
1456
|
+
==== How an image is released
|
|
1457
|
+
|
|
1458
|
+
The release workflow calls link:../.github/workflows/api-only-publisher-image.yml[`api-only-publisher-image.yml` →] after the npm upload:
|
|
1459
|
+
|
|
1460
|
+
. *Plan.* The tag rules, link:docker/image-tags.js[`docker/image-tags.js` →]: whether GHCR already has the version, and which floating tags move.
|
|
1461
|
+
. *Build*, each platform on a native runner, from the tarball npm serves for the version, pushed to GHCR by digest with its SBOM and provenance.
|
|
1462
|
+
. *Tag and sign on GHCR*: the multi-platform image, its exact-version tag, and a keyless signature.
|
|
1463
|
+
. *Smoke test* on both platforms, from an anonymous pull by digest, which proves the package is public: link:docker/smoke-test.sh[`docker/smoke-test.sh` →].
|
|
1464
|
+
. *Scan*, reported in full, blocking on a fixable critical finding.
|
|
1465
|
+
. *Promote*: the same digest copied to Docker Hub, its floating tags moved on both registries, the digest checked on Docker Hub, and signed and verified there.
|
|
1466
|
+
. *Describe*: the Docker Hub short description and overview, from link:docker/DOCKERHUB.md[`docker/DOCKERHUB.md` →].
|
|
1467
|
+
. *Release assets*: the SBOMs and scan reports attached to the GitHub Release, and the image listed in its notes.
|
|
1468
|
+
|
|
1469
|
+
A failure in any of these never undoes the npm release; the workflow summary shows it.
|
|
1470
|
+
A branch that changes what the image is built from builds it for `linux/amd64` and smoke-tests it, without pushing.
|
|
1471
|
+
|
|
1472
|
+
[#docker-secrets]
|
|
1473
|
+
==== Secrets and variables
|
|
1474
|
+
|
|
1475
|
+
[cols="1,1,3", options="header"]
|
|
1476
|
+
|===
|
|
1477
|
+
| Name | Kind | Scope and use
|
|
1478
|
+
|
|
1479
|
+
| `GITHUB_TOKEN`
|
|
1480
|
+
| Automatic
|
|
1481
|
+
| `packages: write` to push to GHCR, `contents: write` to attach release assets, `id-token: write` for cosign's keyless signature. No stored secret.
|
|
1482
|
+
|
|
1483
|
+
| `DOCKERHUB_NAMESPACE`
|
|
1484
|
+
| Variable
|
|
1485
|
+
| The Docker Hub namespace, `arcetect`: the image is `<namespace>/api-only-publisher`. A variable, so the image can move to an organisation without editing a workflow.
|
|
1486
|
+
|
|
1487
|
+
| `DOCKERHUB_USERNAME`
|
|
1488
|
+
| Variable
|
|
1489
|
+
| The Docker Hub account the tokens belong to.
|
|
1490
|
+
|
|
1491
|
+
| `DOCKERHUB_TOKEN`
|
|
1492
|
+
| Secret
|
|
1493
|
+
| A Docker Hub personal access token with *Read & Write* scope: pushes the image and its tags.
|
|
1494
|
+
|
|
1495
|
+
| `DOCKERHUB_DESCRIPTION_TOKEN`
|
|
1496
|
+
| Secret
|
|
1497
|
+
| A Docker Hub personal access token with *Read, Write, Delete* scope, the scope Docker Hub requires to change a repository's description. Only the description job holds it.
|
|
1498
|
+
|
|
1499
|
+
| `E2E_CLEANUP_TOKEN`
|
|
1500
|
+
| Secret, optional
|
|
1501
|
+
| A classic GitHub personal access token with `read:packages` and `delete:packages`, with which the end-to-end workflow deletes its scratch package versions. Without it, it lists them for deletion by hand.
|
|
1502
|
+
|===
|
|
1503
|
+
|
|
1504
|
+
The repository's Actions policy allows only GitHub-owned actions, verified creators and listed ones, and requires every action pinned by SHA.
|
|
1505
|
+
`peter-evans/dockerhub-description`, which updates the description, is on that list; a workflow using an action that is not fails to start, with no message saying why.
|
|
1506
|
+
|
|
1507
|
+
[#docker-visibility]
|
|
1508
|
+
==== GHCR visibility
|
|
1509
|
+
|
|
1510
|
+
GitHub documents that a package starts private and does not inherit its repository's visibility, which can be changed only by hand.
|
|
1511
|
+
In practice, the scratch package the dry run pushed, linked to this public repository through the image's `source` annotation, was public from its first push.
|
|
1512
|
+
|
|
1513
|
+
So the first release may need nothing; the smoke test finds out, since it pulls the image anonymously.
|
|
1514
|
+
If that pull fails, its message names this step: open the package's settings, https://github.com/users/Arc-E-Tect/packages/container/api-only-publisher/settings[api-only-publisher ↗], choose *Change visibility* under *Danger Zone*, make it *Public*, and re-run the failed jobs.
|
|
1515
|
+
|
|
1516
|
+
[#docker-republish]
|
|
1517
|
+
==== Re-publishing
|
|
1518
|
+
|
|
1519
|
+
Run the release workflow by hand with `republish_version` set to a released version, when its npm upload or its image failed.
|
|
1520
|
+
A version npm already has is not uploaded again.
|
|
1521
|
+
If GHCR has the image, it is reused, not rebuilt, and Docker Hub is given that same digest, or left alone if it has it; an exact-version tag is never pointed at another digest.
|
|
1522
|
+
The floating tags move only if the version is the highest released.
|
|
1523
|
+
|
|
1524
|
+
[#docker-verifying]
|
|
1525
|
+
==== Running the smoke test and the end-to-end workflow
|
|
1526
|
+
|
|
1527
|
+
The smoke test runs an image exactly as the release does, with nothing but Docker:
|
|
1528
|
+
|
|
1529
|
+
[source,console]
|
|
1530
|
+
----
|
|
1531
|
+
cd api-only-publisher
|
|
1532
|
+
docker/build-context.sh 0.0.0-local
|
|
1533
|
+
docker build -t api-only-publisher:local --build-arg VERSION=0.0.0-local .
|
|
1534
|
+
docker/smoke-test.sh api-only-publisher:local 0.0.0-local
|
|
1535
|
+
----
|
|
1536
|
+
|
|
1537
|
+
It runs every step with no network, a read-only root and your own user: the version, the help, the toolchain labels against what is installed and against `init`'s pins, a library with two targets built, linted and packaged to every local channel in one run, ownership, byte-identical packaging twice, the hand-off, a foreign toolchain pin failing at once, the shell mode, and a build as an arbitrary UID.
|
|
1538
|
+
|
|
1539
|
+
link:../.github/workflows/api-only-publisher-image-e2e.yml[`api-only-publisher-image-e2e.yml` →] runs the GitHub Actions reference pipeline for real, by hand only.
|
|
1540
|
+
Give it an image by digest; it publishes version `0.<run>.<attempt>` of two scratch targets, `apionly-e2e-alpha` and `apionly-e2e-beta`, to npm, Maven and NuGet on GitHub Packages and to a scratch release, publishes again to prove a re-run is safe, resolves each package with its own client and with the Subscriber, and removes what it published.
|
|
1541
|
+
The image workflow, dispatched by hand, is the release's dry run: its defaults are the scratch names `ghcr.io/arc-e-tect/api-only-publisher-scratch` and `arcetect/api-only-publisher-scratch`, and it builds from the branch's source.
|
|
1542
|
+
|
|
1049
1543
|
[#adapting]
|
|
1050
1544
|
== Adapting it to another project
|
|
1051
1545
|
|
package/package.json
CHANGED
package/src/channels.js
CHANGED
|
@@ -49,7 +49,7 @@ function publishFile(archive, manifest, options, log) {
|
|
|
49
49
|
fs.writeFileSync(path.join(targetDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
|
|
50
50
|
|
|
51
51
|
log(`-- Published ${manifest.target} ${manifest.version} to ${archiveDest}`);
|
|
52
|
-
return { location: archiveDest };
|
|
52
|
+
return { location: archiveDest, files: [archiveDest, path.join(targetDir, "manifest.json")] };
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
function pom(groupId, artifactId, version, packaging, description = `API description documents for ${artifactId}.`) {
|
|
@@ -104,14 +104,20 @@ function publishMaven(archive, manifest, options, log) {
|
|
|
104
104
|
const base = `${artifactId}-${version}`;
|
|
105
105
|
const artifactDest = path.join(dir, `${base}.${extension}`);
|
|
106
106
|
fs.copyFileSync(archive, artifactDest);
|
|
107
|
-
|
|
107
|
+
const pomDest = path.join(dir, `${base}.pom`);
|
|
108
|
+
fs.writeFileSync(pomDest, pom(groupId, artifactId, version, extension, pomDescription(options, manifest)));
|
|
108
109
|
|
|
109
110
|
// The manifest travels beside the artifact as well as inside it, so a
|
|
110
111
|
// consumer can read provenance without unpacking anything.
|
|
111
|
-
|
|
112
|
+
const manifestDest = path.join(dir, `${base}-manifest.json`);
|
|
113
|
+
fs.writeFileSync(manifestDest, JSON.stringify(manifest, null, 2) + "\n");
|
|
112
114
|
|
|
113
115
|
log(`-- Published ${groupId}:${artifactId}:${version} to ${repository}`);
|
|
114
|
-
return {
|
|
116
|
+
return {
|
|
117
|
+
location: artifactDest,
|
|
118
|
+
coordinates: `${groupId}:${artifactId}:${version}@${extension}`,
|
|
119
|
+
files: [artifactDest, pomDest, manifestDest],
|
|
120
|
+
};
|
|
115
121
|
}
|
|
116
122
|
|
|
117
123
|
/**
|
|
@@ -161,7 +167,7 @@ function publishNpm(archive, manifest, options, log) {
|
|
|
161
167
|
{ cwd: workDir, encoding: "utf8" }).trim().split("\n").pop();
|
|
162
168
|
const location = path.join(outDir, packed);
|
|
163
169
|
log(`-- Packed npm ${name}@${version} (${distTag}) to ${location}`);
|
|
164
|
-
return { location, distTag, name };
|
|
170
|
+
return { location, distTag, name, files: [location] };
|
|
165
171
|
}
|
|
166
172
|
|
|
167
173
|
// Idempotent: a version the registry already has is left alone when it is the
|
package/src/cli.js
CHANGED
|
@@ -20,6 +20,8 @@ const { publish, ChannelError } = require("./channels");
|
|
|
20
20
|
const { VersionError: PolicyError, describe } = require("./version-policy");
|
|
21
21
|
const { versionOf, BundleVersionError } = require("./bundle-version");
|
|
22
22
|
const { unreferenced } = require("./unreferenced");
|
|
23
|
+
const { requireTools } = require("./toolchain");
|
|
24
|
+
const { writeHandoff } = require("./handoff");
|
|
23
25
|
|
|
24
26
|
const USAGE = `api-only-publisher -- build and distribute API description documents
|
|
25
27
|
|
|
@@ -67,6 +69,7 @@ Options:
|
|
|
67
69
|
-C <dir> Run as if started in <dir>.
|
|
68
70
|
-q, --quiet Only report errors.
|
|
69
71
|
-h, --help Show this help.
|
|
72
|
+
--version On its own: print the Publisher's version.
|
|
70
73
|
|
|
71
74
|
What gets built, and where each document goes, is declared in apionly.yaml.
|
|
72
75
|
A published target's version is read from its version file:
|
|
@@ -97,6 +100,11 @@ function parseArgs(argv) {
|
|
|
97
100
|
case "--target": options.targets.push(next()); break;
|
|
98
101
|
case "--pre-release": options.preRelease = next(); break;
|
|
99
102
|
case "--version":
|
|
103
|
+
// Alone, it asks for the Publisher's own version, as any tool's does.
|
|
104
|
+
if (argv.length === 1) {
|
|
105
|
+
options.version = true;
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
100
108
|
throw new ConfigError(
|
|
101
109
|
"--version is no longer accepted: each published target's version is read from its version " +
|
|
102
110
|
"file, <target>.bundle.properties beside its bundle root. Pass --pre-release <ids> to cut a " +
|
|
@@ -283,6 +291,10 @@ async function main(argv, io = {}) {
|
|
|
283
291
|
const { options, positional } = parseArgs(argv);
|
|
284
292
|
const command = positional[0];
|
|
285
293
|
|
|
294
|
+
if (options.version) {
|
|
295
|
+
process.stdout.write(`${require("../package.json").version}\n`);
|
|
296
|
+
return 0;
|
|
297
|
+
}
|
|
286
298
|
if (options.help || !command) {
|
|
287
299
|
process.stdout.write(USAGE);
|
|
288
300
|
return 0;
|
|
@@ -334,6 +346,8 @@ async function main(argv, io = {}) {
|
|
|
334
346
|
// in dist/. Every selected document is linted even when one fails, so
|
|
335
347
|
// one run reports every failure rather than only the first.
|
|
336
348
|
const { lint } = require("./pipeline");
|
|
349
|
+
requireTools(config, ["openapi", "asyncapi"].filter((kind) =>
|
|
350
|
+
config.targetsFor(kind).some((target) => !targets || targets.includes(target))));
|
|
337
351
|
const failures = [];
|
|
338
352
|
let linted = 0;
|
|
339
353
|
for (const kind of ["openapi", "asyncapi"]) {
|
|
@@ -441,6 +455,7 @@ async function main(argv, io = {}) {
|
|
|
441
455
|
const versions = versionsOf(config, shipped, options.preRelease);
|
|
442
456
|
const closures = forTargets(config, undefined, shipped);
|
|
443
457
|
let published = 0;
|
|
458
|
+
const handoff = [];
|
|
444
459
|
for (const target of Object.keys(config.targets)) {
|
|
445
460
|
if (targets && !targets.includes(target)) continue;
|
|
446
461
|
if (!config.isPublished(target)) continue;
|
|
@@ -453,11 +468,17 @@ async function main(argv, io = {}) {
|
|
|
453
468
|
});
|
|
454
469
|
for (const name of names) {
|
|
455
470
|
// A remote channel returns a promise; a local one does not.
|
|
456
|
-
await publish(archive, manifest, name, { ...(configured[name] || {}), baseDir: config.root }, log);
|
|
471
|
+
const result = await publish(archive, manifest, name, { ...(configured[name] || {}), baseDir: config.root }, log);
|
|
472
|
+
for (const file of result.files || []) {
|
|
473
|
+
handoff.push({ file, target, version: manifest.version, channel: name });
|
|
474
|
+
}
|
|
457
475
|
published += 1;
|
|
458
476
|
}
|
|
459
477
|
}
|
|
478
|
+
// What a later step publishes byte for byte; README.adoc#handoff.
|
|
479
|
+
const { sha256: listing } = writeHandoff(config.root, outDir, handoff);
|
|
460
480
|
log(`\nPublished ${published} artifact(s).`);
|
|
481
|
+
log(`Listed ${handoff.length} file(s) in ${path.relative(config.root, listing)}.`);
|
|
461
482
|
return 0;
|
|
462
483
|
}
|
|
463
484
|
case "split": {
|
package/src/handoff.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// The hand-off: what a publish run left on disk for a later step to ship.
|
|
4
|
+
//
|
|
5
|
+
// In the two-step model the Publisher -- typically in its Docker image -- packages
|
|
6
|
+
// bundles into files, and a following pipeline job publishes those files with the
|
|
7
|
+
// registry's own tools. That job must ship exactly the bytes the Publisher wrote,
|
|
8
|
+
// and nothing else, so every publish run lists each file a local channel wrote,
|
|
9
|
+
// with its SHA-256, in two forms: packages.json for a script to read, and
|
|
10
|
+
// packages.sha256 for `sha256sum -c`. Paths are relative to the library root, so
|
|
11
|
+
// the check runs from there.
|
|
12
|
+
|
|
13
|
+
const crypto = require("crypto");
|
|
14
|
+
const fs = require("fs");
|
|
15
|
+
const path = require("path");
|
|
16
|
+
|
|
17
|
+
const HANDOFF_JSON = "packages.json";
|
|
18
|
+
const HANDOFF_SHA256 = "packages.sha256";
|
|
19
|
+
const HANDOFF_SCHEMA_VERSION = 1;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Writes packages.json and packages.sha256 into `outDir`, replacing any from an
|
|
23
|
+
* earlier run, listing `entries` -- `{ file, target, version, channel }`, `file`
|
|
24
|
+
* absolute -- sorted by path.
|
|
25
|
+
*/
|
|
26
|
+
function writeHandoff(root, outDir, entries) {
|
|
27
|
+
const files = entries
|
|
28
|
+
.map(({ file, target, version, channel }) => ({
|
|
29
|
+
path: path.relative(root, file).split(path.sep).join("/"),
|
|
30
|
+
sha256: crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"),
|
|
31
|
+
target, version, channel,
|
|
32
|
+
}))
|
|
33
|
+
.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
34
|
+
|
|
35
|
+
fs.mkdirSync(outDir, { recursive: true });
|
|
36
|
+
const json = path.join(outDir, HANDOFF_JSON);
|
|
37
|
+
const sha256 = path.join(outDir, HANDOFF_SHA256);
|
|
38
|
+
fs.writeFileSync(json, JSON.stringify({ schemaVersion: HANDOFF_SCHEMA_VERSION, files }, null, 2) + "\n");
|
|
39
|
+
fs.writeFileSync(sha256, files.map((f) => `${f.sha256} ${f.path}\n`).join(""));
|
|
40
|
+
return { json, sha256 };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
module.exports = { writeHandoff, HANDOFF_JSON, HANDOFF_SHA256, HANDOFF_SCHEMA_VERSION };
|
package/src/index.js
CHANGED
package/src/init-config.js
CHANGED
|
@@ -17,7 +17,7 @@ const YAML = require("yaml");
|
|
|
17
17
|
|
|
18
18
|
/** Where a kind's tool is pinned in `toolchain`; the two kinds do not share a key name. */
|
|
19
19
|
const TOOLCHAIN_KEY = { openapi: "redocly", asyncapi: "asyncapi" };
|
|
20
|
-
const TOOLCHAIN_PIN = { openapi: "@redocly/cli@2.
|
|
20
|
+
const TOOLCHAIN_PIN = { openapi: "@redocly/cli@2.55.0", asyncapi: "@asyncapi/cli@6.2.0" };
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* The slots scaffold(v) would fill for one kind, in the order config(v) declares them,
|
|
@@ -132,4 +132,4 @@ function addMissingConfig(text, v, portfolio) {
|
|
|
132
132
|
return added.length === 0 ? { text, added } : { text: doc.toString(), added };
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
-
module.exports = { addMissingConfig };
|
|
135
|
+
module.exports = { addMissingConfig, TOOLCHAIN_PIN };
|
package/src/init.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
const fs = require("fs");
|
|
16
16
|
const path = require("path");
|
|
17
17
|
const YAML = require("yaml");
|
|
18
|
-
const { addMissingConfig } = require("./init-config");
|
|
18
|
+
const { addMissingConfig, TOOLCHAIN_PIN } = require("./init-config");
|
|
19
19
|
|
|
20
20
|
/** Every value the scaffold is written from, as it is when nobody chooses otherwise. */
|
|
21
21
|
const DEFAULTS = Object.freeze({
|
|
@@ -66,8 +66,8 @@ function config(v) {
|
|
|
66
66
|
"",
|
|
67
67
|
"toolchain:",
|
|
68
68
|
);
|
|
69
|
-
if (openapi) lines.push(
|
|
70
|
-
if (asyncapi) lines.push(
|
|
69
|
+
if (openapi) lines.push(` redocly: "${TOOLCHAIN_PIN.openapi}"`);
|
|
70
|
+
if (asyncapi) lines.push(` asyncapi: "${TOOLCHAIN_PIN.asyncapi}"`);
|
|
71
71
|
lines.push("", "targets:", ` ${v.target}:`);
|
|
72
72
|
if (openapi) lines.push(" openapi:", ` bundle: bundles/${v.target}_openapi_structure.yaml`);
|
|
73
73
|
if (asyncapi) lines.push(" asyncapi:", ` bundle: bundles/${v.target}_asyncapi_structure.yaml`);
|
package/src/nuget.js
CHANGED
|
@@ -160,12 +160,13 @@ function publishLocal(archive, manifest, options, id, log) {
|
|
|
160
160
|
|
|
161
161
|
const file = path.join(dir, `${lowerId}.${lowerVersion}.nupkg`);
|
|
162
162
|
fs.writeFileSync(file, bytes);
|
|
163
|
-
|
|
164
|
-
fs.writeFileSync(
|
|
165
|
-
fs.writeFileSync(
|
|
163
|
+
const written = [file, `${file}.sha512`, path.join(dir, `${lowerId}.nuspec`), path.join(dir, "manifest.json")];
|
|
164
|
+
fs.writeFileSync(written[1], crypto.createHash("sha512").update(bytes).digest("base64"));
|
|
165
|
+
fs.writeFileSync(written[2], spec);
|
|
166
|
+
fs.writeFileSync(written[3], JSON.stringify(manifest, null, 2) + "\n");
|
|
166
167
|
|
|
167
168
|
log(`-- Published NuGet ${id} ${manifest.version} to ${feed}`);
|
|
168
|
-
return { location: file, id, version: manifest.version, prerelease: isPrerelease(manifest.version) };
|
|
169
|
+
return { location: file, id, version: manifest.version, prerelease: isPrerelease(manifest.version), files: written };
|
|
169
170
|
}
|
|
170
171
|
|
|
171
172
|
/** The resources a feed's service index lists. */
|
package/src/pipeline.js
CHANGED
|
@@ -14,8 +14,8 @@ const { stampFiles, componentPaths, strayPaths, fragmentStamps, unresolvedStamps
|
|
|
14
14
|
const {
|
|
15
15
|
asyncapiOperationsWithoutExamples, openapiOperationsWithoutExamples, asyncapiMessage, openapiMessage,
|
|
16
16
|
} = require("./examples");
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
const { BuildError } = require("./build-error");
|
|
18
|
+
const toolchain = require("./toolchain");
|
|
19
19
|
|
|
20
20
|
function run(command, args, { quiet, reportFile = null } = {}) {
|
|
21
21
|
try {
|
|
@@ -38,6 +38,23 @@ function run(command, args, { quiet, reportFile = null } = {}) {
|
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Copy a directory tree, symlinks as symlinks.
|
|
43
|
+
*
|
|
44
|
+
* Not fs.cpSync: its native implementation fails with EACCES on Docker Desktop's
|
|
45
|
+
* bind mounts, which is where the Publisher's image stages a mounted library.
|
|
46
|
+
*/
|
|
47
|
+
function copyTree(from, to) {
|
|
48
|
+
fs.mkdirSync(to, { recursive: true });
|
|
49
|
+
for (const entry of fs.readdirSync(from, { withFileTypes: true })) {
|
|
50
|
+
const source = path.join(from, entry.name);
|
|
51
|
+
const dest = path.join(to, entry.name);
|
|
52
|
+
if (entry.isDirectory()) copyTree(source, dest);
|
|
53
|
+
else if (entry.isSymbolicLink()) fs.symlinkSync(fs.readlinkSync(source), dest);
|
|
54
|
+
else fs.copyFileSync(source, dest);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
41
58
|
/**
|
|
42
59
|
* Copy the whole source root to the staging directory for one specification
|
|
43
60
|
* type.
|
|
@@ -52,8 +69,7 @@ function stage(config, kind, log) {
|
|
|
52
69
|
const to = config.stagingRoot(kind);
|
|
53
70
|
log(`-- Staging ${path.relative(config.root, from)} -> ${path.relative(config.root, to)}`);
|
|
54
71
|
fs.rmSync(to, { recursive: true, force: true });
|
|
55
|
-
|
|
56
|
-
fs.cpSync(from, to, { recursive: true });
|
|
72
|
+
copyTree(from, to);
|
|
57
73
|
return to;
|
|
58
74
|
}
|
|
59
75
|
|
|
@@ -94,8 +110,9 @@ function bundle(config, target, kind, outFile, log, { stagingRoot } = {}) {
|
|
|
94
110
|
}
|
|
95
111
|
fs.mkdirSync(path.dirname(outFile), { recursive: true });
|
|
96
112
|
log(`-- Bundling ${path.basename(source)}`);
|
|
97
|
-
const tool =
|
|
98
|
-
|
|
113
|
+
const tool = config.tool(toolchain.TOOL_OF_KIND[kind]);
|
|
114
|
+
const runner = toolchain.resolve(toolchain.TOOL_OF_KIND[kind], tool);
|
|
115
|
+
run(runner.command, [...runner.args, "bundle", source, "--output", outFile], { quiet: true });
|
|
99
116
|
if (!fs.existsSync(outFile)) {
|
|
100
117
|
throw new BuildError(`target '${target}': ${tool} produced no output at ${outFile}`);
|
|
101
118
|
}
|
|
@@ -121,7 +138,7 @@ function bundleWithFragmentPaths(config, target, kind, outFile, log) {
|
|
|
121
138
|
|
|
122
139
|
const pass = (name, only) => {
|
|
123
140
|
const root = path.join(scratch, name);
|
|
124
|
-
|
|
141
|
+
copyTree(config.stagingRoot(kind), root);
|
|
125
142
|
try {
|
|
126
143
|
stampFiles(root, { only });
|
|
127
144
|
} catch (error) {
|
|
@@ -170,7 +187,7 @@ function bundleInlinedWithFragmentPaths(config, target, kind, outFile, log) {
|
|
|
170
187
|
const scratch = config.fragmentPathStaging(target);
|
|
171
188
|
fs.rmSync(scratch, { recursive: true, force: true });
|
|
172
189
|
const root = path.join(scratch, "stamp");
|
|
173
|
-
|
|
190
|
+
copyTree(config.stagingRoot(kind), root);
|
|
174
191
|
|
|
175
192
|
const bundleRoot = path.relative(root, config.bundleRootPath(target, kind, root)).split(path.sep).join("/");
|
|
176
193
|
try {
|
|
@@ -199,12 +216,12 @@ function bundleInlinedWithFragmentPaths(config, target, kind, outFile, log) {
|
|
|
199
216
|
}
|
|
200
217
|
|
|
201
218
|
function lint(config, kind, file, log, { report = false, reportFile = null } = {}) {
|
|
202
|
-
const
|
|
219
|
+
const runner = toolchain.resolve(toolchain.TOOL_OF_KIND[kind], config.tool(toolchain.TOOL_OF_KIND[kind]));
|
|
203
220
|
const args = kind === "openapi"
|
|
204
|
-
? ["
|
|
205
|
-
: ["
|
|
221
|
+
? ["lint"].concat(config.lintConfig("openapi") ? ["--config", config.lintConfig("openapi")] : []).concat([file])
|
|
222
|
+
: ["validate", file];
|
|
206
223
|
log(`-- Validating ${path.basename(file)}`);
|
|
207
|
-
const output = run(
|
|
224
|
+
const output = run(runner.command, [...runner.args, ...args], { quiet: true, reportFile });
|
|
208
225
|
if (report && output.trim()) log(output.trimEnd());
|
|
209
226
|
}
|
|
210
227
|
|
|
@@ -289,6 +306,11 @@ function prepare(config, { kinds = ["openapi", "asyncapi"], log = () => {} } = {
|
|
|
289
306
|
* @returns {Array<{target, kind, file, distributed}>}
|
|
290
307
|
*/
|
|
291
308
|
function build(config, { targets, versionOf = () => null, kinds = ["openapi", "asyncapi"], log = () => {} } = {}) {
|
|
309
|
+
// Every tool this build needs is found before anything is staged, so a toolchain
|
|
310
|
+
// the installation cannot run fails the build before it has done any work.
|
|
311
|
+
const needed = kinds.filter((kind) => config.targetsFor(kind).some((t) => !targets || targets.includes(t)));
|
|
312
|
+
toolchain.requireTools(config, needed);
|
|
313
|
+
|
|
292
314
|
const results = [];
|
|
293
315
|
for (const kind of kinds) {
|
|
294
316
|
const all = config.targetsFor(kind).filter((t) => !targets || targets.includes(t));
|
|
@@ -349,5 +371,5 @@ function build(config, { targets, versionOf = () => null, kinds = ["openapi", "a
|
|
|
349
371
|
}
|
|
350
372
|
|
|
351
373
|
module.exports = {
|
|
352
|
-
build, prepare, stage, substituteTree, bundle, bundleWithFragmentPaths, lint, checkExamples, distribute, BuildError,
|
|
374
|
+
build, prepare, stage, copyTree, substituteTree, bundle, bundleWithFragmentPaths, lint, checkExamples, distribute, BuildError,
|
|
353
375
|
};
|
package/src/toolchain.js
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Finding the bundler and validator a build runs.
|
|
4
|
+
//
|
|
5
|
+
// Out of the box each tool runs through `npx --yes <pin>`, which downloads the
|
|
6
|
+
// pinned version when it is not cached. An installation that already carries the
|
|
7
|
+
// tools -- the Docker image -- names where they are, and a tool installed there at
|
|
8
|
+
// exactly the pinned version runs from its own bin instead. An installation that
|
|
9
|
+
// must not download anything at all says so, and a pin it does not carry then
|
|
10
|
+
// fails before anything runs, rather than reaching for the network.
|
|
11
|
+
//
|
|
12
|
+
// Both are environment variables, so the Publisher outside the image behaves as
|
|
13
|
+
// it always has: neither is set there.
|
|
14
|
+
|
|
15
|
+
const fs = require("fs");
|
|
16
|
+
const path = require("path");
|
|
17
|
+
|
|
18
|
+
const { BuildError } = require("./build-error");
|
|
19
|
+
|
|
20
|
+
/** Where the installed toolchain is: a directory `npm install --prefix` filled. */
|
|
21
|
+
const DIR_ENV = "API_ONLY_PUBLISHER_TOOLCHAIN_DIR";
|
|
22
|
+
/** `allow`, the default, falls back to npx; `never` refuses to. */
|
|
23
|
+
const DOWNLOAD_ENV = "API_ONLY_PUBLISHER_TOOLCHAIN_DOWNLOAD";
|
|
24
|
+
/** The image label naming the pin each tool is installed at, per toolchain key. */
|
|
25
|
+
const LABEL_PREFIX = "com.arc-e-tect.api-only-publisher.toolchain.";
|
|
26
|
+
|
|
27
|
+
/** Which toolchain key each kind of document is built with. */
|
|
28
|
+
const TOOL_OF_KIND = { openapi: "redocly", asyncapi: "asyncapi" };
|
|
29
|
+
|
|
30
|
+
class ToolchainError extends BuildError {}
|
|
31
|
+
|
|
32
|
+
/** A pin such as `@redocly/cli@2.52.0` as its package name and version; a bare name has none. */
|
|
33
|
+
function parseSpec(spec) {
|
|
34
|
+
const at = spec.lastIndexOf("@");
|
|
35
|
+
if (at <= 0) return { name: spec, version: null };
|
|
36
|
+
return { name: spec.slice(0, at), version: spec.slice(at + 1) };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function readPackage(file) {
|
|
40
|
+
try {
|
|
41
|
+
return JSON.parse(fs.readFileSync(file, "utf8"));
|
|
42
|
+
} catch {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function packageDir(dir, name) {
|
|
48
|
+
return path.join(dir, "node_modules", ...name.split("/"));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Every tool the toolchain directory declares, as `name@installed-version`, in declaration order. */
|
|
52
|
+
function installed(dir) {
|
|
53
|
+
if (!dir) return [];
|
|
54
|
+
const manifest = readPackage(path.join(dir, "package.json"));
|
|
55
|
+
if (!manifest) return [];
|
|
56
|
+
return Object.keys(manifest.dependencies || {}).map((name) => {
|
|
57
|
+
const own = readPackage(path.join(packageDir(dir, name), "package.json"));
|
|
58
|
+
return `${name}@${own ? own.version : "(missing)"}`;
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// The bin to run: the only one, or the one named after the package or its scope --
|
|
63
|
+
// Redocly CLI declares `openapi` and `redocly`, and `redocly` is the tool.
|
|
64
|
+
function binOf(own, spec) {
|
|
65
|
+
if (typeof own.bin === "string") return own.bin;
|
|
66
|
+
const bins = Object.keys(own.bin || {});
|
|
67
|
+
if (bins.length === 1) return own.bin[bins[0]];
|
|
68
|
+
const scope = own.name.startsWith("@") ? own.name.slice(1).split("/")[0] : null;
|
|
69
|
+
const unscoped = own.name.split("/").pop();
|
|
70
|
+
const named = [unscoped, scope].find((candidate) => candidate && own.bin[candidate]);
|
|
71
|
+
if (named) return own.bin[named];
|
|
72
|
+
throw new ToolchainError(`${spec} declares several bins (${bins.join(", ")}), and none is named after it`);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function downloadMode(env) {
|
|
76
|
+
const mode = env[DOWNLOAD_ENV] === undefined || env[DOWNLOAD_ENV] === "" ? "allow" : env[DOWNLOAD_ENV];
|
|
77
|
+
if (mode !== "allow" && mode !== "never") {
|
|
78
|
+
throw new ToolchainError(`${DOWNLOAD_ENV} must be allow or never, not ${JSON.stringify(mode)}`);
|
|
79
|
+
}
|
|
80
|
+
return mode;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function refusal(key, spec, dir, carried) {
|
|
84
|
+
const { name } = parseSpec(spec);
|
|
85
|
+
const same = carried.find((entry) => parseSpec(entry).name === name);
|
|
86
|
+
const lines = [`apionly.yaml's toolchain.${key} pins ${spec}, and ${DOWNLOAD_ENV}=never forbids downloading it.`];
|
|
87
|
+
if (carried.length === 0) {
|
|
88
|
+
lines.push(`This installation carries no toolchain${dir ? ` in ${dir}` : `: ${DIR_ENV} is not set`}.`);
|
|
89
|
+
} else {
|
|
90
|
+
const list = carried.length === 1 ? carried[0] : `${carried.slice(0, -1).join(", ")} and ${carried[carried.length - 1]}`;
|
|
91
|
+
lines.push(`This installation carries ${list}.`);
|
|
92
|
+
}
|
|
93
|
+
const align = same ? `Set toolchain.${key} to "${same}" in apionly.yaml, or use` : "Use";
|
|
94
|
+
lines.push(`${align} the api-only-publisher image whose ${LABEL_PREFIX}${key} label is ${spec}.`);
|
|
95
|
+
lines.push(`To allow the download instead, run with ${DOWNLOAD_ENV}=allow.`);
|
|
96
|
+
return new ToolchainError(lines.join("\n"));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* How to run the tool pinned as `spec` under toolchain key `key`: a command and
|
|
101
|
+
* the arguments that precede the tool's own.
|
|
102
|
+
*/
|
|
103
|
+
function resolve(key, spec, env = process.env) {
|
|
104
|
+
const mode = downloadMode(env);
|
|
105
|
+
const dir = env[DIR_ENV] || null;
|
|
106
|
+
const { name, version } = parseSpec(spec);
|
|
107
|
+
if (dir && version) {
|
|
108
|
+
const root = packageDir(dir, name);
|
|
109
|
+
const own = readPackage(path.join(root, "package.json"));
|
|
110
|
+
if (own && own.version === version) return { command: path.join(root, binOf(own, spec)), args: [] };
|
|
111
|
+
}
|
|
112
|
+
if (mode === "never") throw refusal(key, spec, dir, installed(dir));
|
|
113
|
+
return { command: "npx", args: ["--yes", spec] };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Resolves the tool of every kind in `kinds`, so a run that cannot have them fails before it starts. */
|
|
117
|
+
function requireTools(config, kinds, env = process.env) {
|
|
118
|
+
for (const kind of kinds) resolve(TOOL_OF_KIND[kind], config.tool(TOOL_OF_KIND[kind]), env);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
module.exports = {
|
|
122
|
+
parseSpec, installed, resolve, requireTools, ToolchainError, DIR_ENV, DOWNLOAD_ENV, LABEL_PREFIX, TOOL_OF_KIND,
|
|
123
|
+
};
|