@voxgig/sdkgen 4.14.0 → 4.16.0

package/src/cmp/PublishWorkflow.ts

@@ -25,6 +25,15 @@ import { packageName, packageVersion, repoInfo } from '../helpers/packageMeta'
 // deliberate act by someone who can see the result, which is what a release
 // should be.
 //
+// IT TAGS WHAT IT PUBLISHED. A release that reaches the registry but leaves
+// no ref behind cannot be answered later: `@voxgig-sdk/github-sdk@0.0.3` went
+// out by dispatch and the repository had only v0.0.1 and v0.0.2, so nothing
+// in git said which tree the published tarball came from, and nothing
+// downstream could pin the SDK by tag. npm's trusted publisher is bound to
+// ONE workflow file, so the tag has to be cut inside this one -- and a ref
+// pushed with GITHUB_TOKEN starts no further workflow run, so it cannot be
+// delegated to a tag-triggered publisher either.
+//
 // THE WORKFLOW FILE NAME IS PART OF THE TRUST CONFIGURATION. npm binds a
 // trusted publisher to the repository AND the workflow filename, so renaming
 // this file silently breaks publishing until the npm side is updated to
@@ -65,7 +74,7 @@ const PublishWorkflow = cmp(function PublishWorkflow(props: any) {
     Folder({ name: 'workflows' }, () => {
       for (const target of npmTargets) {
         File({ name: `publish-${target.name}.yml` }, () => {
-          Content(publishWorkflow(model, target))
+          Content(publishWorkflow(model, target, npmTargets))
         })
       }
     })
@@ -84,8 +93,40 @@ const PublishWorkflow = cmp(function PublishWorkflow(props: any) {
 })
 
 
-function publishWorkflow(model: any, target: any): string {
+// WHICH TAG A TARGET'S RELEASE CARRIES.
+//
+// A repo here publishes more than one npm package -- `ts/` and `js/` are
+// separate packages from separate folders, on a lockstep version -- so a bare
+// `v<version>` cut by each would be two targets racing for one tag name, and
+// the loser would fail a release that had already published.
+//
+// The convention is the toolchain's own. @voxgig/apidef tags `v<version>` for
+// its npm package and `go/v<version>` for its Go module; the generated root
+// Makefile tags `<target>/v<version>` for every port. So: the ecosystem's
+// PRIMARY npm target owns the bare tag, and any other npm target is prefixed
+// with its own name.
+//
+// Resolved by comparing package names rather than hardcoding 'ts', so that a
+// project that renamed or replaced its primary target still gets one bare tag
+// -- and if none matches, every target is prefixed, which is wrong in no way
+// that loses a release.
+function isPrimaryNpm(model: any, target: any, npmTargets: any[]): boolean {
+  if (1 === npmTargets.length) {
+    return true
+  }
+  return packageName(model, target.name) === packageName(model, 'npm')
+}
+
+
+function releaseTag(model: any, target: any, npmTargets: any[]): string {
+  return isPrimaryNpm(model, target, npmTargets) ?
+    'v$VERSION' : `${target.name}/v$VERSION`
+}
+
+
+function publishWorkflow(model: any, target: any, npmTargets: any[]): string {
   const name = target.name
+  const tag = releaseTag(model, target, npmTargets)
   // BY TARGET, NOT BY ECOSYSTEM. `packageName(model, 'npm')` resolves the
   // ecosystem's PRIMARY target — ts — so every npm target's workflow named
   // the ts package: publish-js.yml claimed `@voxgig-sdk/github-sdk` while
@@ -117,14 +158,25 @@ function publishWorkflow(model: any, target: any): string {
 # version by two mechanisms. Releasing is an act someone performs and
 # watches.
 #
-# TWO JOBS, BECAUSE THEY NEED DIFFERENT PRIVILEGES. A dependency lifecycle
+# THREE JOBS, BECAUSE THEY NEED DIFFERENT PRIVILEGES. A dependency lifecycle
 # script can ask the runner for any OIDC token its job is permitted to mint,
 # so a job that both installs dependencies and holds \`id-token: write\` can be
-# made to publish before its own gates finish.
+# made to publish before its own gates finish. The same reasoning keeps
+# \`contents: write\` out of both: checkout persists its token into the git
+# config for the whole job.
 #
 #   verify   contents: read, nothing else. Installs, builds and tests.
 #   publish  id-token: write. Installs no dependencies and runs no project
 #            code; it packs what is already in the tree.
+#   tag      contents: write. Runs git and nothing else.
+#
+# THE TAG IS CUT HERE BECAUSE IT CANNOT BE CUT ANYWHERE ELSE. npm binds the
+# trusted publisher to ONE workflow file, so whatever must accompany a publish
+# belongs inside it; and a ref pushed with GITHUB_TOKEN starts no further
+# workflow run, so "tag, and let a tag-triggered publisher fire" does not
+# work. A release that publishes but leaves no ref cannot be answered later:
+# nothing in git says which tree the tarball came from, and nothing
+# downstream can pin this SDK by tag.
 
 name: publish-${name}
 
@@ -236,6 +288,55 @@ jobs:
         if: steps.registry.outputs.published == 'false'
         working-directory: ${name}
         run: npm publish --access public
+
+  # Runs git and nothing else. No install, no project code — so the
+  # repository-write credential is never in scope while third-party code runs.
+  #
+  # It runs even when the publish step SKIPPED because the version was already
+  # on the registry: that is what makes re-dispatching after a partial release
+  # finish the job rather than leave a published version permanently untagged.
+  tag:
+    name: tag
+    needs: [verify, publish]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    # The ONLY job that may write to the repository, and the only one that
+    # runs no project code.
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          # Tags are needed to tell "already tagged" from "not tagged yet".
+          fetch-depth: 0
+
+      - name: Tag the release
+        env:
+          VERSION: \${{ needs.verify.outputs.version }}
+        run: |
+          set -euo pipefail
+          TAG="${tag}"
+          HEAD_SHA="\$(git rev-parse HEAD)"
+
+          # EXISTENCE IS NOT ENOUGH. \`--verify\` only asks whether the name
+          # resolves; a tag created between the verify job's guard and this
+          # checkout could point anywhere. Treating that as idempotent would
+          # leave a green run with a published artifact from THIS commit and a
+          # release tag on a different one.
+          AT="\$(git rev-parse -q --verify "refs/tags/\$TAG^{commit}" 2>/dev/null || true)"
+          if [ -n "\$AT" ]; then
+            if [ "\$AT" != "\$HEAD_SHA" ]; then
+              echo "::error::\$TAG exists on \$AT, not this commit (\$HEAD_SHA)"
+              exit 1
+            fi
+            echo "\$TAG already points here — nothing to do" >> "\$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+          git tag "\$TAG"
+          git push origin "refs/tags/\$TAG"
+          echo "pushed \$TAG" >> "\$GITHUB_STEP_SUMMARY"
 `
 }
 
@@ -255,6 +356,14 @@ function publishingDoc(model: any, targets: any[], repoUrl: string): string {
     `| \`${t.name}/\` | ${packageName(model, t.name)} | ` +
     `\`.github/workflows/publish-${t.name}.yml\` |`).join('\n')
 
+  // WHICH TAG EACH TARGET CUTS. Worth a table rather than a sentence: the
+  // primary npm target owns the bare `v<version>` and every other one is
+  // prefixed, so with more than one npm target the answer differs per row.
+  const tagRows = '| target | tag |\n|---|---|\n' + targets.map((t: any) =>
+    `| \`${t.name}/\` | \`` +
+    releaseTag(model, t, targets).replace('$VERSION', '<version>') +
+    '\` |').join('\n')
+
   return `# Publishing
 
 GENERATED by @voxgig/sdkgen — regenerated on every \`npm run generate\`.
@@ -292,6 +401,26 @@ It is a DISPATCH rather than a tag push because the root \`Makefile\` also
 publishes, with a vault-injected token. A tag-triggered workflow racing it
 would publish one version by two mechanisms.
 
+## The release is tagged for you
+
+The workflow cuts the git tag itself, after a successful publish, so a
+released version always has a ref naming the tree it came from. Nothing to
+do by hand.
+
+${tagRows}
+
+Re-dispatching a version that is already on the registry skips the publish
+and still cuts a missing tag, so a partial release is finished by running it
+again rather than repaired by hand. If the tag already exists on a DIFFERENT
+commit the run fails rather than moving it: that combination means the
+published artifact and the tag disagree, and only a person should decide
+which is wrong.
+
+The tag job is the only one that may write to the repository, and it runs
+git and nothing else. \`contents: write\` never shares a job with project
+code, for the same reason \`id-token: write\` does not — \`checkout\` persists
+its token into the git config for the whole job.
+
 ## One-time set-up
 
 npm has to be told which repository and which workflow file may publish this
@@ -318,7 +447,7 @@ package goes out by hand from an authenticated machine:
 
 After that, register the publisher and every later release is a dispatch.
 
-## Why the workflow has two jobs
+## Why the workflow has three jobs
 
 A dependency lifecycle script can ask the runner for any OIDC token the job
 it runs in is permitted to mint. A job that both installs dependencies and
@@ -326,8 +455,14 @@ holds \`id-token: write\` can therefore be made to publish as this package
 before its own gates finish.
 
 So \`verify\` installs, builds and tests under \`contents: read\` and holds no
-credential, and \`publish\` holds \`id-token: write\` while installing nothing
-and running no project code.
+credential; \`publish\` holds \`id-token: write\` while installing nothing and
+running no project code; and \`tag\` holds \`contents: write\` while running
+git and nothing else.
+
+The tag cannot be moved out into its own workflow: npm binds the trusted
+publisher to ONE workflow filename, so anything that must accompany a publish
+has to live inside this file. Nor can it be left to a tag-triggered
+publisher — a ref pushed with \`GITHUB_TOKEN\` starts no further workflow run.
 `
 }
 

package/src/sdkgen.ts

@@ -181,6 +181,32 @@ type SdkGenOptions = {
   }
 
   dryrun?: boolean
+
+  // WHERE AN OUT-OF-TREE ITEM GOES, DECIDED AT GENERATE TIME RATHER THAN IN
+  // THE MODEL. Keyed by item name:
+  //
+  //   external: { 'seneca-provider': { path: '../..', sdkrel: '.sdksrc/acme-sdk', enclosing: true } }
+  //
+  // `output: path` in the model is a fact about ONE developer's checkout
+  // layout, and it is committed. The same model generated from a different
+  // layout is the same model — a different INVOCATION of it, not a second
+  // truth to encode — so the second layout belongs here and not in a second
+  // committed value that the two layouts then fight over.
+  //
+  // The case that forced it: a provider repo that carries a tagged checkout
+  // of its SDK in a subfolder and regenerates itself from it. The SDK's own
+  // committed model cannot describe that, because the SDK does not know it
+  // has been cloned into someone else's repo.
+  external?: Record<string, ExternalOverride>
+}
+
+
+// See SdkGenOptions.external. `enclosing` is separate from `path` on
+// purpose — see checkExternalFolders.
+type ExternalOverride = {
+  path?: string
+  sdkrel?: string
+  enclosing?: boolean
 }
 
 
@@ -317,12 +343,14 @@ function SdkGen(opts: SdkGenOptions) {
     // so resolve it ONCE: every destination is compared against it, and a
     // comparison between a relative and an absolute path is meaningless.
     const root = Path.resolve(folder)
+
+    const externalOverride = resolveExternalOverride(opts, log)
     // Snapshot the decision before preflight. In particular, do not check a
     // missing optional destination once for safety and AGAIN before writing:
     // if it appeared between those checks, the pass could write into content
     // that was never ownership-validated.
     const external: ExternalPlan[] =
-      externalItems(model, root, ['target', 'docs'])
+      externalItems(model, root, ['target', 'docs'], externalOverride)
         .map((ext) => ({ ...ext, skip: externalSkipReason(ext, fs) }))
 
     // Before ANY file is written, in-tree included: a destination that turns
@@ -700,6 +728,11 @@ type ExternalSpec = {
   target: any
   folder: string
   active: boolean
+
+  // The caller asked, at generate time, to write into a folder that CONTAINS
+  // the SDK project. Never read from the model — see SdkGenOptions.external
+  // and checkExternalFolders.
+  enclosing: boolean
 }
 
 
@@ -721,6 +754,74 @@ type ExternalPlan = ExternalSpec & {
 // An INACTIVE target is still listed: it must be taken out of the in-tree
 // model (see withoutExternal) so that switching it off does not silently
 // relocate it into `<sdk-repo>/<target>/`. The generate loop skips it.
+// The generate-time override of where out-of-tree items are written.
+//
+// Two ways in, merged, the environment winning:
+//
+//   opts.external     the caller holds the config — a build script it owns.
+//   SDKGEN_EXTERNAL   JSON, for driving a checkout the caller does NOT own.
+//
+// The environment matters more than it looks. The case this exists for is a
+// repository that carries a tagged checkout of its SDK and regenerates
+// itself from it: the script doing that owns neither the SDK's model nor its
+// `.sdk/build/sdkgen.js`, so any route that requires editing a file inside
+// the checkout means patching someone else's repo on every clone — and the
+// point of the exercise was that the clone is disposable.
+//
+// ONE JSON VARIABLE, not one variable per item per field. Item names carry
+// hyphens (`seneca-provider`), so a
+// `SDKGEN_EXTERNAL_SENECA_PROVIDER_PATH` scheme needs a name mangling with
+// no inverse: `a-b` and `a_b` collide, and nothing can tell which was meant.
+function resolveExternalOverride(
+  opts: SdkGenOptions, log: any,
+): Record<string, ExternalOverride> {
+  const declared = opts.external || {}
+  const raw = process.env.SDKGEN_EXTERNAL
+
+  if (null == raw || '' === raw.trim()) {
+    return declared
+  }
+
+  let parsed: any
+
+  try {
+    parsed = JSON.parse(raw)
+  }
+  catch (err: any) {
+    throw new SdkGenError(
+      'SDKGEN_EXTERNAL is not valid JSON: ' + (err?.message || err) +
+      '\n  Expected an object keyed by item name, for example:' +
+      '\n    {"seneca-provider":{"path":"../..","sdkrel":".sdksrc/acme-sdk","enclosing":true}}' +
+      '\n  Got: ' + raw.slice(0, 200))
+  }
+
+  if (null == parsed || 'object' !== typeof parsed || Array.isArray(parsed)) {
+    throw new SdkGenError(
+      'SDKGEN_EXTERNAL must be a JSON OBJECT keyed by item name, for example:' +
+      '\n    {"seneca-provider":{"path":"../..","sdkrel":".sdksrc/acme-sdk","enclosing":true}}' +
+      '\n  Got: ' + raw.slice(0, 200))
+  }
+
+  const merged: Record<string, ExternalOverride> = { ...declared }
+
+  for (const name of Object.keys(parsed)) {
+    merged[name] = { ...(declared[name] || {}), ...(parsed[name] || {}) }
+  }
+
+  // LOUDLY. Generation writing somewhere other than the model says is
+  // exactly the thing whose only previous trace was one INFO line naming a
+  // resolved folder, and that cost a silent green run that emitted nothing.
+  const named = Object.keys(parsed).sort()
+  log.info({
+    point: 'external-override',
+    items: named.join(','),
+    note: 'SDKGEN_EXTERNAL overrides the output of: ' + named.join(', '),
+  })
+
+  return merged
+}
+
+
 // Every item, of every kind, that generates OUTSIDE the SDK repo.
 //
 // Keyed by kind because `docs` needs exactly this and a second copy of it
@@ -733,19 +834,48 @@ type ExternalPlan = ExternalSpec & {
 // other, in whatever order the passes happen to run.
 function externalItems(
   model: any, folder: string, kinds: string[],
+  override: Record<string, ExternalOverride>,
 ): ExternalSpec[] {
   return kinds.flatMap((kind: string) => {
     const items = model?.main?.[KIT]?.[kind] || {}
 
     return Object.keys(items).sort()
-      .map((name: string) => ({ kind, name, target: items[name] }))
+      .map((name: string) => {
+        // Keyed by NAME, across kinds: a docs item and a target that share a
+        // name share an override. They already share an output folder if both
+        // are pointed at one (the claim map refuses that), so a name that
+        // means two things is a problem before it reaches here.
+        const ov = override[name] || {}
+
+        // A SHALLOW CLONE, so the override never writes back into the model.
+        // The in-tree pass still reads it, and `target add` still round-trips
+        // it to disk; an override is for THIS RUN. Nothing compares these by
+        // identity — withoutExternal keys by kind:name.
+        const output = { ...(items[name]?.output || {}) }
+
+        if (null != ov.path && '' !== ov.path) {
+          output.path = ov.path
+        }
+        if (null != ov.sdkrel && '' !== ov.sdkrel) {
+          output.sdkrel = ov.sdkrel
+        }
+
+        return { kind, name, target: { ...items[name], output }, ov }
+      })
+      // AFTER the override, so it can send an item out of tree that the
+      // model generates in tree. The machinery needs nothing else for that:
+      // withoutExternal takes it out of the in-tree pass by kind:name, and
+      // every destination guard below runs on it either way.
       .filter((t: any) => {
-        const path = t.target?.output?.path
+        const path = t.target.output.path
         return null != path && '' !== path
       })
       .map((t: any) => ({
-        ...t,
+        kind: t.kind,
+        name: t.name,
+        target: t.target,
         folder: Path.resolve(folder, String(t.target.output.path)),
+        enclosing: true === t.ov.enclosing,
         active: false !== t.target.active,
       }))
   })
@@ -848,11 +978,31 @@ function checkExternalFolders(external: ExternalPlan[], root: string, fs: any) {
         ext.name + '/.')
     }
 
-    if (folderContains(ext.folder, root)) {
+    // A DESTINATION THAT CONTAINS THE PROJECT is what `..` produces, and it
+    // is refused — UNLESS the caller said at generate time that it meant it.
+    //
+    // The legitimate case is a repository that carries a tagged checkout of
+    // its SDK in a subfolder and regenerates itself from it: the SDK then
+    // sits INSIDE its own output folder, and `output: path` resolves to an
+    // ancestor. Generation writes the files its components declare and
+    // prunes nothing, so the checkout survives its own run.
+    //
+    // `enclosing` is a separate flag from `path` on purpose, and only the
+    // override can set it — never the model. Overriding a path is one
+    // decision; writing over the directory holding the project is a second,
+    // much worse thing to get wrong, and it fails silently: a typo'd `..`
+    // fabricates a package tree over an unrelated repo, overwriting its
+    // package.json, README, LICENSE and CI in place. Saying it twice is the
+    // cost of keeping the typo caught for everyone who did not ask.
+    if (folderContains(ext.folder, root) && !ext.enclosing) {
       throw new SdkGenError(
         'External output path contains the SDK project.\n  ' + where +
         '\n  Generation would write this package over the directory holding ' +
-        'the SDK project itself.')
+        'the SDK project itself.' +
+        '\n  If that is deliberate — the SDK is checked out INSIDE its own ' +
+        'output folder, and regenerates it — say so at generate time with ' +
+        '`enclosing: true` in the ' + ext.name + ' entry of SDKGEN_EXTERNAL ' +
+        'or the `external` build option. It cannot be set in the model.')
     }
 
     // ACROSS KINDS, not within one: `docs` and `target` are separate
@@ -876,6 +1026,20 @@ function checkExternalFolders(external: ExternalPlan[], root: string, fs: any) {
 
     if (true === ext.target.output.adopt) continue
 
+    // AN ENCLOSING DESTINATION ALWAYS HOLDS CONTENT: the SDK project itself
+    // is inside it, along with whatever else the repository carries. The
+    // emptiness check can therefore only ever say "yes, it has content", so
+    // it carries no information here — and requiring `output: adopt` on top
+    // of the opt-in would put the layout back in the SDK's COMMITTED model,
+    // which is the coupling the generate-time override exists to break. An
+    // SDK cloned into a repository it regenerates cannot have anticipated
+    // being cloned there.
+    //
+    // The containment check above is what guards this case, and it is
+    // stricter: it refuses unless the caller named this item and said
+    // `enclosing` for it.
+    if (ext.enclosing) continue
+
     const entries: string[] = fs.readdirSync(ext.folder)
       .map((entry: any) => String(entry))
 
@@ -936,6 +1100,17 @@ function externalSdkRel(ext: ExternalSpec, root: string, log: any): string {
   // earlier one is a directory ABOVE it, which nothing in the model declares.
   const named = derived.split('/').filter((seg) => '..' !== seg)
 
+  // NOT WHEN THE OUTPUT ENCLOSES THE PROJECT. There the walk back descends
+  // rather than ascends — `.sdksrc/acme-sdk` names the subfolder holding the
+  // checkout and then the checkout, both INSIDE the output folder and both
+  // chosen by whoever asked for this layout. Nothing is above anything, so
+  // the warning's complaint (directories the model does not declare) is
+  // false, and its advice (declare `output: sdkrel`) would put one layout's
+  // path into the other's committed model.
+  if (ext.enclosing) {
+    return derived
+  }
+
   if (1 < named.length) {
     log.warn({
       point: 'external-sdkrel-derived', target: ext.name, sdkrel: derived,