@voxgig/sdkgen 3.4.6 → 3.5.0

package/src/action/feature.ts

@@ -27,6 +27,12 @@ import { SdkGenError } from '../utility'
 
 import { findFeatureSources } from '../helpers/featureSource'
 
+import { templateReplacements } from '../helpers/stdrep'
+
+import { copyOpts } from '../helpers/junk'
+
+import { resolveKind, kindModel } from './kind'
+
 
 import {
   UpdateIndex,
@@ -41,6 +47,13 @@ const CMD_MAP: any = {
 
 const BASE = 'node_modules/@voxgig/sdkgen'
 
+// The `.sdk` folder a bundled feature comes from — the value recorded as its
+// provenance. Still hardcoded, like the path above: giving `feature add` the
+// ref grammar `target add` already has is the next step, and this becomes
+// whatever the ref resolved to.
+const SDKFOLDER = BASE + '/project/.sdk'
+
+
 
 async function action_feature(args: string[], actx: ActionContext): Promise<ActionResult> {
 
@@ -89,6 +102,9 @@ async function feature_add(features: string[], actx: ActionContext): Promise<Act
     control: {
       dryrun: !!actx.opts.dryrun
     },
+    // Per-call for the same reason, and covering the same accident: see
+    // helpers/junk.
+    cmp: copyOpts(),
   }
 
   opts.log.info({
@@ -123,28 +139,56 @@ const FeatureRoot = cmp(function FeatureRoot(props: any) {
   const target = model.main[KIT].target
 
   Project({}, () => {
+    // The names as INSTALLED, which is what the index must list. A ref is
+    // not a name: `feature add @acme/sdkgen-iot/circuitbreaker` installs
+    // `circuitbreaker`, and writing the raw ref into feature-index.aontu
+    // would produce an include of a file that does not exist.
+    const fnames: string[] = []
+
     each(features, (n) => {
-      const fname = n.val$
+      const fref = n.val$
       // TODO: validate feature is a-z0-9-_. only
 
+      // Resolution, the alias policy, and the bare-name fallback to what the
+      // model records are the same for every kind — see action/kind.
+      //
+      // A name that no longer resolves must not abort the RUN: `target add`
+      // re-runs this action for every active feature, so one feature whose
+      // source has moved (package uninstalled, checkout relocated) would
+      // otherwise stop every other feature — including `test`, which every
+      // generated target needs — from being copied at all.
+      let source: any
+      try {
+        source = resolveKind(fref, 'feature', ctx$)
+      }
+      catch (err: any) {
+        if (err instanceof SdkGenError) {
+          throw err
+        }
+
+        log.warn({
+          point: 'feature-source-unresolved', feature: fref,
+          err: err.message,
+          note: fref + ': cannot find its source (' + err.message +
+            '); skipping, the already-copied files are left alone'
+        })
+        return
+      }
+
+      const fname = source.name
+      fnames.push(fname)
+
       log.info({
         point: 'feature-build',
         feature: fname,
-        note: fname
+        note: fname + (fname === fref ? '' : ' ref:' + fref)
       })
 
 
-      Folder({ name: 'model/feature' }, () => {
-        Copy({
-          // TODO: these paths needs to be parameterised
-          from: BASE + '/project/.sdk/model/feature/' + fname + '.aontu',
-          exclude: true
-        })
-        File({ name: 'feature-index.aontu' }, () => UpdateIndex({
-          content: ctx$.meta.content.feature_index,
-          names: features,
-        }))
-      })
+      Folder({ name: 'model/feature' }, () => kindModel({
+        ctx$, kind: 'feature', source, names: fnames,
+        content: ctx$.meta.content.feature_index,
+      }))
 
       // Bring in the feature's source for every target already in the model.
       // Where that source lives is language-specific — `src/feature/<name>/`
@@ -154,10 +198,51 @@ const FeatureRoot = cmp(function FeatureRoot(props: any) {
       // `src/feature/<name>` meant `feature add` silently added nothing for
       // every target that keeps feature source elsewhere.
       each(target, (t) => {
-        const sdkfolder = t.base || Path.join(BASE, 'project/.sdk')
-        const tmfolder = Path.join(sdkfolder, 'tm', t.name)
+        // The target's OWN tree, under the name it has in ITS source — an
+        // aliased target's templates live at `tm/<origname>`, so searching
+        // `tm/<t.name>` missed them entirely.
+        const sdkfolder = t.base || SDKFOLDER
+        const torigname = t.origname || t.name
+        const owntm = Path.join(sdkfolder, 'tm', torigname)
+
+        // Two places a feature's per-target source can live, in order:
+        // the FEATURE package's own overlay for this target, then the
+        // target's own tree. A feature shipped by one package for a target
+        // shipped by another has nowhere else to put it.
+        //
+        // First hit wins at DISCOVERY rather than by copy order: jostraca
+        // writes last-write-wins, so copying both would silently invert the
+        // precedence.
+        const featuretm = Path.join(source.folder, 'tm', torigname)
+        const overlay = featuretm === owntm ? [] :
+          findFeatureSources(fs, featuretm, [fname])
+
+        const own = 0 < overlay.length ?
+          findFeatureSources(fs, owntm, [fname]) : []
+
+        // Both trees carrying source for one feature means two packages claim
+        // the same name. The overlay wins, but the target's own files were
+        // already copied by `target add` and Copy never removes, so the
+        // project is left holding BOTH — duplicate symbols, or stale feature
+        // code that still runs. Detecting the name collision at add time is
+        // what actually fixes this (it belongs with manifest validation); say
+        // so loudly until then, rather than leaving a silent hybrid.
+        if (0 < own.length) {
+          log.warn({
+            point: 'feature-source-shadowed', feature: fname, target: t.name,
+            overlay: featuretm, own: owntm,
+            note: fname + ': both ' + featuretm + ' and ' + owntm +
+              ' provide source for target ' + t.name +
+              '; the overlay is used, but the files already copied from the ' +
+              "target's own tree are NOT removed — check tm/" + t.name +
+              ' for a mix of the two'
+          })
+        }
+
+        const sources = 0 < overlay.length ? overlay :
+          findFeatureSources(fs, owntm, [fname])
 
-        const sources = findFeatureSources(fs, tmfolder, [fname])
+        const tmfolder = 0 < overlay.length ? featuretm : owntm
 
         if (0 === sources.length) {
           log.warn({
@@ -176,7 +261,14 @@ const FeatureRoot = cmp(function FeatureRoot(props: any) {
           Folder({ name: 'tm/' + t.name + '/' + dest }, () => {
             Copy({
               from: Path.join(tmfolder, source.path),
-              exclude: true
+              // The SAME map `target add` writes `tm/<t>` with. Without it
+              // this copy laid RAW template text over files the target add
+              // had already substituted, so `ProjectName` / `PROJECTVERSION`
+              // survived into the project depending only on which action
+              // wrote the file last — the writer/writer disagreement
+              // helpers/stdrep.ts exists to prevent, in the one place that
+              // did not share the map.
+              replace: templateReplacements(model, t.name),
             })
           })
         }

package/src/action/kind.ts

@@ -0,0 +1,375 @@
+// KINDS: the things an `add` can install.
+//
+// `target` and `feature` are two of them, `docs` and others are meant to
+// follow (docs/design/sdkgen-packages.md §8). They were two hand-written
+// pipelines that had already drifted apart — only one took path refs, only
+// one applied a replace map, only one recorded provenance — and a third kind
+// would have been a third copy of the same drift.
+//
+// What every kind shares is declared here and executed once:
+//
+//   - the ref grammar and resolution, including the fallback to what the
+//     model already records for a bare name;
+//   - whether the kind may be installed under a different name (`~alias`);
+//   - the definition file: `model/<kind>/<name>.aontu`, stamped with
+//     provenance and landing under the INSTALLED name;
+//   - the include list: `model/<kind>/<kind>-index.aontu`.
+//
+// What a kind adds on top — a target's component and template trees, a
+// feature's per-target source fan-out — stays in that kind's own action. Those
+// are genuinely different work, not the same work with different strings, and
+// pretending otherwise would buy generality nobody can use.
+
+import Path from 'node:path'
+
+import { File, Copy, Content, template } from 'jostraca'
+
+import { KIT } from '../types'
+
+import { SdkGenError } from '../utility'
+
+import { provenanceReplace } from '../helpers/stdrep'
+
+import { resolveSource } from './resolve'
+import type { Source } from './resolve'
+
+import { UpdateIndex } from './action'
+
+
+type KindDef = {
+  // The kind's name, which is also its model subdirectory and the prefix of
+  // its index file.
+  name: string
+
+  // May it be installed under a different name? Targets yes — that is how a
+  // project gets two Go modules from one target. Features no: a feature's
+  // name is part of the generated `options.feature.<name>` config key and of
+  // the hook wiring in every target, so renaming one at install time is real
+  // work with no customer.
+  alias: boolean
+
+  // Rewrite the definition's own text when it is installed under a different
+  // name (targets rewrite their `main: kit: target: <name>:` key). Only
+  // reached when `alias` is true.
+  rename?: (src: string, origname: string, name: string) => string
+
+  // Is the copied definition PROJECT-OWNED once written? An alias exists to
+  // be differentiated, so its model file is created and then never
+  // overwritten; everything else is toolchain-owned and resyncs.
+  ownedWhenAliased?: boolean
+
+  // Everything BESIDES the definition file that an item of this kind must
+  // have in its package, `{name}`-templated and relative to the `.sdk`
+  // folder. `validateManifest` checks these, which is how a manifest claiming
+  // a target it does not ship fails at `package add` rather than partway
+  // through installing it.
+  //
+  // A feature has none: its definition is the whole of it. Its per-target
+  // SOURCE is deliberately not required — a feature package ships overlays
+  // only for the targets it supports, and `targetsSupported` is where that
+  // coverage is declared (design §7).
+  trees?: TreeDef[]
+}
+
+
+// One tree an item of this kind owns, `{name}`-templated and relative to the
+// `.sdk` folder.
+type TreeDef = {
+  path: string
+
+  // What the copy substitutes on the way in — and therefore what doctor must
+  // re-apply before comparing, or every templated file reads as edited.
+  //   none     — copied verbatim, so a byte compare is the truth
+  //   template — through jostraca's template() with ProjectName et al
+  replace: 'none' | 'template'
+
+  // Must it be there for the item to be installable? A missing REQUIRED tree
+  // fails manifest validation before anything is written; a missing optional
+  // one is simply not copied.
+  required: boolean
+}
+
+
+// Rewrite the ITEM KEY in a copied model file, for an aliased install.
+//
+// Parameterised by kind because `main: kit: docs: <n>:` is the same rewrite
+// with a different word, and a second copy of this regex is exactly the
+// same-rule-written-twice defect the registry exists to prevent.
+//
+// Two forms are in use across the shipped models — bare (`target: go:`) and
+// quoted (`target: 'go-cli':`) — and two paths carry the key: the item
+// block itself (`main: kit: target: <t>:`) and the per-target feature-deps
+// slot every target model declares
+// (`main: kit: feature: &: target: <t>: deps: &:`). Both belong to the
+// installed item, so both move. Matching on `<kind>: ` rather than on the
+// bare name is what keeps the rewrite off the item's own values — `ext: go`
+// and `module: name: '$$name$$'` must not change.
+//
+// ONE regex, with the quote optional and captured, rather than two entries in
+// jostraca's `replace` map: that map canonicalises each key into a regex
+// group NAME, and the bare and quoted spellings of the same key reduce to the
+// same name — so one silently overwrote the other and `go-cli~cli2` came out
+// as the BARE `target: cli2:`, losing the quoting a hyphenated key needs.
+//
+// The alias may also NEED quoting when the origin did not: aontu rejects a
+// bare key containing a hyphen (`unexpected character(s): -`), so
+// `target add go~go-alt` emitting the origin's unquoted style produced a
+// model that could not compile at all. Quote when the origin was quoted OR
+// the alias is not a bare identifier.
+const BARE_KEY_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/
+
+function aliasModelKey(kind: string) {
+  return function aliasModelText(
+    src: string, origname: string, name: string,
+  ): string {
+    const mustQuote = !BARE_KEY_RE.test(name)
+
+    return src.replace(
+      new RegExp(kind + ":(\\s*)('?)" + escapeRe(origname) + "\\2:", 'g'),
+      (_m: string, gap: string, quote: string) => {
+        const q = ('' !== quote || mustQuote) ? "'" : ''
+        return kind + ':' + gap + q + name + q + ':'
+      })
+  }
+}
+
+
+function escapeRe(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+
+const KINDS: Record<string, KindDef> = Object.assign(Object.create(null), {
+  target: {
+    name: 'target', alias: true, ownedWhenAliased: true,
+    rename: aliasModelKey('target'),
+    // Components are dispatched by the convention `cmp/<t>/Main_<t>`, and the
+    // template tree is what `target add` copies — a target missing either is
+    // not installable, however complete its model file looks.
+    trees: [
+      { path: 'src/cmp/{name}', replace: 'none', required: true },
+      { path: 'tm/{name}', replace: 'template', required: true },
+    ],
+  },
+
+  feature: { name: 'feature', alias: false },
+
+  // DOCS — the third kind. See docs/design/sdkgen-packages.md §20.
+  //
+  // Its trees are NESTED under the kind name (`src/cmp/docs/<n>`, not
+  // `src/cmp/<n>`) so a docs item and a target may share a name without
+  // sharing a directory. Everything that composes those paths takes them
+  // from here.
+  //
+  // `tm/docs/{name}` is NOT required: a docs item whose every emitted byte
+  // depends on the API — a catalogue entry, a config file — legitimately
+  // ships no template tree, while a static site needs one. So it is
+  // copy-if-present, and `package check` does not demand it.
+  docs: {
+    name: 'docs', alias: true, ownedWhenAliased: true,
+    rename: aliasModelKey('docs'),
+    trees: [
+      { path: 'src/cmp/docs/{name}', replace: 'none', required: true },
+      { path: 'tm/docs/{name}', replace: 'template', required: false },
+    ],
+  },
+})
+
+
+// The trees a kind's add copies, with `{name}` resolved. ONE composition of
+// these paths, read by the add, by doctor's drift walk and (through
+// `requires`) by manifest validation — three readers that must agree about
+// where an item's content lives, and previously would each have spelled it.
+function kindTrees(kind: string, name: string): TreeDef[] {
+  return (kindDef(kind).trees ?? []).map((t: TreeDef) => ({
+    ...t,
+    path: t.path.split('{name}').join(name),
+  }))
+}
+
+
+function kindDef(kind: string): KindDef {
+  const def = KINDS[kind]
+  if (null == def) {
+    throw new SdkGenError('Unknown kind: ' + kind)
+  }
+  return def
+}
+
+
+// Resolve one ref for a kind, applying that kind's alias policy.
+//
+// A BARE name resolves against what the model already RECORDS. Without that,
+// provenance would be write-only: the add actions re-run with the model's own
+// keys (`circuitbreaker`, not the ref it was installed from), a bare name
+// falls back to the bundled scaffold — whose `.sdk` folder exists, so
+// resolution SUCCEEDS — and the copy then throws on a definition that is not
+// there. An explicit ref always wins: that is how something is moved to a new
+// source.
+function resolveKind(ref: string, kind: string, ctx$: any): Source {
+  const def = kindDef(kind)
+  const model = ctx$.model
+
+  const declared: any = model?.main?.[KIT]?.[kind]?.[ref]
+  const recorded = (isBare(ref) && recordedRef(declared, ref)) || ref
+
+  const source = resolveSource(recorded, kind, ctx$)
+
+  // Asked of the RESOLVER rather than by re-reading the ref. `~` separates an
+  // alias only in the last segment, and a check that looked for one anywhere
+  // rejected every ref whose PATH contains a tilde — a Windows 8.3 short name
+  // like `C:\Users\RUNNER~1\...` is one. Parsing the ref in two places is what
+  // let that defect come back after it was fixed.
+  if (!def.alias && source.name !== source.origname) {
+    throw new SdkGenError(
+      capitalise(kind) + ' aliasing is not supported: ' + ref +
+      '\n  A ' + kind + ' name is part of the generated config ' +
+      '(options.' + kind + '.<name>) and of the hook wiring in every target, ' +
+      'so it cannot be renamed at install time.')
+  }
+
+  // The DEFINITION has to be there, not just the folder that would hold it.
+  //
+  // Resolution stops at the `.sdk` folder, and a bare name resolves to the
+  // bundled scaffold — whose folder always exists — so a name with no
+  // definition anywhere resolved "successfully" and then failed much later
+  // inside jostraca's Copy, as a shape validation error naming a path the
+  // caller never wrote.
+  //
+  // Where that bites is the fan-out: `target add` re-runs `feature add` for
+  // every feature key in the model, and a key with no definition installed
+  // aborted the ENTIRE target add — no templates, no components, nothing —
+  // over one feature. A plain Error, not an SdkGenError, so the fan-out's
+  // per-item guard skips it with a warning; the add path for an explicit ref
+  // does not catch, so `feature add nosuch` still fails, now saying what is
+  // actually missing.
+  if (!ctx$.fs().existsSync(source.model)) {
+    throw new Error(
+      capitalise(kind) + ' definition not found: ' + source.model)
+  }
+
+  return source
+}
+
+
+// Emit the kind's definition file and its index entry.
+//
+// Called inside a `Folder({ name: 'model/<kind>' })`, so both land together.
+// `names` is every INSTALLED name seen so far in this run: the index File is
+// re-rendered per item and the last render wins, so each render has to carry
+// all of them.
+function kindModel(props: {
+  ctx$: any,
+  kind: string,
+  source: Source,
+  names: string[],
+  content: string,
+}) {
+  const { ctx$, kind, source, names, content } = props
+  const def = kindDef(kind)
+  const fs = ctx$.fs()
+  const log = ctx$.log
+
+  const aliased = source.name !== source.origname
+
+  const replace = provenanceReplace({
+    base: source.base,
+    origname: source.origname,
+    name: source.name,
+    package: source.package,
+  })
+
+  if (aliased) {
+    // The copy lands under the INSTALLED name AND declares it. Left alone it
+    // kept the origin basename (jostraca defaults a single-file Copy's
+    // destination to the source's), so the index named a file that does not
+    // exist — which fails the whole model compile, not just the alias.
+    //
+    // `exclude: true` — CREATE, never overwrite, for a kind whose aliased
+    // definition is project-owned: an alias exists to be differentiated,
+    // which is why doctor reports its diffs as informational and why
+    // add-a-target tells the project to edit it.
+    const owned = true === def.ownedWhenAliased
+
+    if (owned) {
+      const dest = Path.join(
+        ctx$.folder ?? '.', 'model', kind, source.name + '.aontu')
+
+      if (fs.existsSync(dest)) {
+        log.info({
+          point: kind + '-alias-model-kept', [kind]: source.name, file: dest,
+          note: source.name + ': keeping the existing aliased ' + kind +
+            ' model (project-owned — an alias is differentiated by editing it)'
+        })
+      }
+    }
+
+    const src = fs.readFileSync(source.model, 'utf8')
+    const text = null == def.rename ? src :
+      def.rename(src, source.origname, source.name)
+
+    File({ name: source.name + '.aontu', exclude: owned }, () =>
+      Content(template(text, ctx$.model, { replace })))
+  }
+  else {
+    Copy({ from: source.model, replace })
+  }
+
+  File({ name: def.name + '-index.aontu' }, () => UpdateIndex({
+    content,
+    names,
+  }))
+}
+
+
+// The ref that reinstalls what the model already records — `base` says which
+// `.sdk` folder, `origname` says what it is called there.
+//
+// The ALIAS has to be carried back through. Rebuilding only `<base>/../<orig>`
+// resolves to the ORIGIN name, so `target add go2` (after installing
+// `go~go2`) would refresh and index a new `go` target and leave `go2` stale —
+// losing the differentiated identity the alias exists for, and the
+// project-owned model file with it.
+//
+// ONE definition, used by the add actions and by doctor. This reconstruction
+// was written twice and the two copies had already diverged on exactly this
+// point, which is the drift the kind spine exists to end.
+function recordedRef(declared: any, name: string): string | undefined {
+  if (null == declared?.base || '' === declared.base) {
+    return undefined
+  }
+
+  const origname = declared.origname || name
+
+  return Path.join(declared.base, '..', origname) +
+    (origname === name ? '' : '~' + name)
+}
+
+
+// A bare NAME, as opposed to a ref that locates a source.
+function isBare(ref: string): boolean {
+  return !ref.includes('/') && !ref.includes(Path.sep)
+}
+
+
+function capitalise(s: string): string {
+  return s.charAt(0).toUpperCase() + s.slice(1)
+}
+
+
+export type {
+  KindDef,
+  TreeDef,
+}
+
+export {
+  KINDS,
+  recordedRef,
+  aliasModelKey,
+  kindTrees,
+  escapeRe,
+  kindDef,
+  resolveKind,
+  kindModel,
+  isBare,
+}