@voxgig/sdkgen 3.4.7 → 3.5.0

package/src/cmp/ExternalDocs.ts

@@ -0,0 +1,54 @@
+// The Root for a DOCS item that generates outside the SDK repo
+// (`main: kit: docs: <n>: output: path`).
+//
+// Out of tree is the NORMAL case for a docs item, not the exception it is for
+// a target: a documentation site usually has its own repo, and in-tree it
+// would collide with the SDK repo's own README and CI workflows.
+//
+// The two facts that make this a separate `generate()` pass rather than a
+// folder are the ones `ExternalTarget` documents at length: jostraca's
+// `FileHandler.validName` refuses a `..` segment in a folder name (the guard
+// that keeps generation inside the tree it was pointed at, not an oversight
+// to route around), and the output root is `jopts.folder` PER CALL — so
+// writing somewhere else means another call.
+//
+// No Folder wraps the item: the destination IS the site, so its files go at
+// the root of the output path.
+
+import { cmp, names, Project } from 'jostraca'
+
+import { KIT } from '../types'
+
+import { DocsItem, prepareModel } from './Docs'
+
+
+const ExternalDocs = cmp(function ExternalDocs(props: any) {
+  const { model, item, cmpfolder, sdkrelpath } = props
+  const ctx$ = props.ctx$
+
+  // Components live in the PROJECT, not in the repo being written to. This
+  // pass has retargeted jostraca's output folder, which is what requirePath
+  // otherwise resolves against — see utility.resolvePath. Losing this line
+  // makes the pass look for components under the DESTINATION.
+  ctx$.cmpfolder = cmpfolder
+
+  // The path from the destination back to the SDK project, for a site that
+  // sits beside the SDK in a known layout and needs to name it.
+  ctx$.sdkrelpath = sdkrelpath
+
+  // The same preamble the in-tree pass performs — one definition, both
+  // callers, because a project whose ONLY docs item is out of tree must
+  // generate exactly as one with both.
+  prepareModel(model, ctx$)
+
+  Project({}, () => {
+    names(item, item.name ?? item.key$)
+
+    DocsItem({ item })
+  })
+})
+
+
+export {
+  ExternalDocs
+}

package/src/helpers/definition.ts

@@ -0,0 +1,82 @@
+// WHERE A KIND'S DEFINITION FILES LIVE.
+//
+// `model/<kind>/<name>.aontu`, with `model/<kind>/<kind>-index.aontu` as the
+// include list beside them. One line of path-building — and it was written out
+// longhand in three places (the resolver, the feature catalogue, doctor), each
+// with its own idea of which files in that directory count.
+//
+// That is the same shape of defect this workstream has now fixed four times:
+// the same rule expressed twice, drifting. The manifest makes it four callers,
+// so it becomes one function first.
+//
+// The EXCLUSIONS matter as much as the path. A `model/<kind>/` directory holds
+// the index file (a list of includes, not a definition) and, in the shipped
+// scaffold, a `README.md`. Neither is an item, and a caller that forgets one
+// invents a `feature-index` feature.
+
+import Path from 'node:path'
+
+import { isJunk } from './junk'
+
+
+// The definition file for one item.
+function definitionPath(sdkfolder: string, kind: string, name: string): string {
+  return Path.join(sdkfolder, 'model', kind, name + '.aontu')
+}
+
+
+// The directory holding a kind's definitions.
+function definitionFolder(sdkfolder: string, kind: string): string {
+  return Path.join(sdkfolder, 'model', kind)
+}
+
+
+// The include list beside them.
+function indexName(kind: string): string {
+  return kind + '-index.aontu'
+}
+
+
+// Every item of `kind` a `.sdk` folder DEFINES, sorted. Sorted because it
+// feeds an exact-set comparison against a manifest and, through
+// `availableFeatures`, the feature-trim catalogue — both of which are compared
+// as text, so a readdir order that varies by filesystem would vary the result.
+//
+// An absent directory is not an error: a package providing only features has
+// no `model/target/`, and that is the normal shape of a feature package.
+function definitionNames(fs: any, sdkfolder: string, kind: string): string[] {
+  const dir = definitionFolder(sdkfolder, kind)
+  const index = indexName(kind)
+
+  // `existsSync` is true for a REGULAR FILE and for a directory this process
+  // cannot read, so guarding on it and then calling `readdirSync` let a bare
+  // `ENOTDIR`/`EACCES` escape to callers that have no catch — including
+  // `featureCatalogue`, which runs on every ordinary `target add`, so the
+  // whole add aborted with an errno instead of a diagnostic. Not-a-readable-
+  // directory is the same answer as not-there: this kind defines nothing
+  // here.
+  let entries: string[]
+  try {
+    entries = fs.readdirSync(dir)
+  }
+  catch (err: any) {
+    return []
+  }
+
+  // The `.aontu` suffix is not enough on its own: an emacs lock link is named
+  // `.#target.aontu` and a merge leaves `target.aontu.orig`, so an editor open
+  // in the wrong window invents an item called `.#target`, which then fails to
+  // resolve everywhere it is named. See helpers/junk.
+  return entries
+    .filter((n: string) => n.endsWith('.aontu') && index !== n && !isJunk(n))
+    .map((n: string) => n.replace(/\.aontu$/, ''))
+    .sort()
+}
+
+
+export {
+  definitionPath,
+  definitionFolder,
+  definitionNames,
+  indexName,
+}

package/src/helpers/featureSource.ts

@@ -37,6 +37,10 @@ import Path from 'node:path'
 
 import { KIT, getModelPath } from '@voxgig/apidef'
 
+import { definitionNames } from './definition'
+
+import { isJunk } from './junk'
+
 
 // One feature's source within a target template tree.
 type FeatureSource = {
@@ -52,12 +56,25 @@ type FeatureSource = {
 }
 
 
+// One entry inside a feature container, before the catalogue has had its say.
+type FeatureEntry = FeatureSource & {
+  // Does it follow a per-feature NAMING convention? See `featureShaped`.
+  shaped: boolean
+}
+
+
 // Directory name that marks a feature container. Kept exact (not a substring
 // match) so `utility/feature_add.go` and `test/feature_test.go` — which are
 // shared machinery, not per-feature source — are never treated as features.
 const FEATURE_DIR = 'feature'
 
 
+// The always-present foundation every other feature builds on. It has no
+// model file, so it is in no catalogue, and it must never be trimmed or
+// reported as an unrecognised stray — every target ships one.
+const BASE_FEATURE = 'base'
+
+
 // Derive the feature name an entry inside a feature directory belongs to.
 // Returns the lowercased name, which the caller then checks against the
 // available set.
@@ -88,25 +105,51 @@ function featureOf(entry: string, folder: boolean): string {
 // Feature names this generator can supply, read from the scaffold's
 // `model/feature/*.aontu`. This is the authoritative catalogue: a name not in
 // it is not a feature, so nothing outside it is ever excluded.
+//
+// LOWERCASED, unlike `definitionNames`, because these are matched against
+// names DERIVED FROM FILENAMES (`RetryFeature.swift` -> `retry`), and the
+// languages disagree about case. The manifest compares definition names as
+// written, so the two callers cannot share the lowercasing.
 function availableFeatures(fs: any, sdkfolder: string): string[] {
-  const dir = Path.join(sdkfolder, 'model', 'feature')
+  return definitionNames(fs, sdkfolder, 'feature')
+    .map((n: string) => n.toLowerCase())
+    .sort()
+}
+
 
-  if (!fs.existsSync(dir)) {
-    return []
+// Does the entry LOOK like per-feature source, whatever it is called?
+//
+// The naming conventions carry the intent: `<name>_feature.<ext>`,
+// `<Name>Feature.<ext>`, and a directory named for the feature. A bare
+// `<name>.<ext>` does NOT — `retry.rs` and `support.rs` are written the same
+// way, so shape cannot separate rust's feature source from rust's shared
+// machinery, and only the catalogue can. That is the deliberate blind spot of
+// `package check`'s unrecognised-source finding: it reports what it can prove
+// looks like a feature, and stays quiet where it would have to guess.
+function featureShaped(entry: string, folder: boolean): boolean {
+  if (folder) {
+    return true
   }
 
-  return fs.readdirSync(dir)
-    .filter((n: string) => n.endsWith('.aontu') && 'feature-index.aontu' !== n)
-    .map((n: string) => n.replace(/\.aontu$/, '').toLowerCase())
-    .sort()
+  const stem = entry.replace(/\.[^.]+$/, '')
+
+  return /_feature$/i.test(stem) || /Feature$/.test(stem)
 }
 
 
-// Walk a target's template tree and return every per-feature source entry.
-// `tmfolder` is the target template root (`<sdk>/tm/<lang>`).
-function findFeatureSources(fs: any, tmfolder: string, available: string[]): FeatureSource[] {
-  const known = new Set(available)
-  const found: FeatureSource[] = []
+// Every entry inside a feature container of a target's template tree —
+// whether or not its derived name is a feature this generator knows.
+//
+// `known` decides DESCENT, not membership: a directory that names a known
+// feature is the whole feature, so the walk stops there, and one that does
+// not is walked through (`src/feature/base/` holds `BaseFeature.ts`, and the
+// container rule below then ignores it because its parent is not `feature`).
+// The unknown entries are what `package check` reports and what
+// `findFeatureSources` drops; both need the same walk, so there is one.
+function findFeatureEntries(
+  fs: any, tmfolder: string, known: Set<string>,
+): FeatureEntry[] {
+  const found: FeatureEntry[] = []
 
   if (!fs.existsSync(tmfolder)) {
     return found
@@ -118,6 +161,14 @@ function findFeatureSources(fs: any, tmfolder: string, available: string[]): Fea
     const entries = fs.readdirSync(abs).sort()
 
     for (const entry of entries) {
+      // A `__pycache__` inside `src/feature/` is not a feature — but every
+      // rule here derives a feature NAME from an entry name, so without this
+      // it becomes one: reported by `package check` as an unknown feature, and
+      // trimmed (or not) as if it were source. See helpers/junk.
+      if (isJunk(entry)) {
+        continue
+      }
+
       const entryrel = '' === rel ? entry : rel + '/' + entry
       const folder = fs.statSync(Path.join(tmfolder, entryrel)).isDirectory()
 
@@ -126,8 +177,12 @@ function findFeatureSources(fs: any, tmfolder: string, available: string[]): Fea
       // looking for more.
       if (FEATURE_DIR === Path.basename(rel)) {
         const name = featureOf(entry, folder)
+
+        found.push({
+          name, path: entryrel, folder, shaped: featureShaped(entry, folder),
+        })
+
         if (known.has(name)) {
-          found.push({ name, path: entryrel, folder })
           continue
         }
       }
@@ -144,6 +199,17 @@ function findFeatureSources(fs: any, tmfolder: string, available: string[]): Fea
 }
 
 
+// Walk a target's template tree and return every per-feature source entry.
+// `tmfolder` is the target template root (`<sdk>/tm/<lang>`).
+function findFeatureSources(fs: any, tmfolder: string, available: string[]): FeatureSource[] {
+  const known = new Set(available)
+
+  return findFeatureEntries(fs, tmfolder, known)
+    .filter((e: FeatureEntry) => known.has(e.name))
+    .map(({ name, path, folder }: FeatureEntry) => ({ name, path, folder }))
+}
+
+
 // Turn discovered sources into path patterns for a jostraca `Copy` exclude.
 //
 // jostraca tests each candidate against the path built up during its walk,
@@ -196,11 +262,15 @@ function srcFeatureExcludes(model: any): RegExp[] {
 
 export type {
   FeatureSource,
+  FeatureEntry,
 }
 
 export {
+  BASE_FEATURE,
   featureOf,
+  featureShaped,
   availableFeatures,
+  findFeatureEntries,
   findFeatureSources,
   featureExcludes,
   fullsetExcludes,

package/src/helpers/junk.ts

@@ -0,0 +1,168 @@
+// WHAT NEVER COUNTS AS CONTENT.
+//
+// Every tree sdkgen reads — the scaffold's `tm/<lang>` and `src/cmp/<lang>`, an
+// external package's `.sdk`, a project's own copies of both — is a DIRECTORY ON
+// A DEVELOPER'S MACHINE, and a developer's machine drops things in directories:
+// `.DS_Store` from the Finder, `__pycache__` from running a python module,
+// `node_modules` from an `npm install` in the wrong shell, a `~` backup from an
+// editor. None of it is authored, none of it is tracked (it is all gitignored),
+// and all of it is invisible in `git status` — so the first thing that notices
+// is whatever compares two trees.
+//
+// It bit as: a stray `tm/py/utility/__pycache__/make_options.cpython-314.pyc`,
+// created in August by running that module in place, copied verbatim by
+// `target add py` into the add output, and reported by the golden-manifest test
+// as toolchain drift. The copy is the real problem — the test only saw it. Left
+// alone, that `.pyc` ships into every consumer's SDK repo, compiled against
+// whatever python the maintainer happened to have.
+//
+// The rule is one list, in one place, applied by everything that walks a tree:
+// the WRITER (jostraca's Copy, through `cmp.Copy.ignore`, plus the two walks
+// target-add does itself) and every READER that compares against what the
+// writer produced (doctor's drift walk, the feature-source scan, the definition
+// listing). A writer and a reader that disagree about what counts report the
+// whole tree as a fork — the same discipline `templateReplacements` and
+// `aliasCmpName` are held to, and for the same reason.
+//
+// MEMBERSHIP IS A PROMISE that no template will ever legitimately carry that
+// name, so this list holds only what a TOOL creates and a person does not: no
+// `build`, `dist`, `bin` or `out` (a template may well ship one), and
+// emphatically no `target` — sdkgen's own model directory is called that, and
+// it is also where cargo and maven put their output. `test/junk.test.ts` checks
+// the list against every name in the shipped scaffold, so an over-broad pattern
+// fails there rather than by silently dropping a file from someone's SDK.
+
+// All of these are matched against the BARE NAME of a directory entry, not its
+// path — the same way jostraca's `cmp.Copy.ignore` is applied, so that naming a
+// directory prunes its whole subtree without every caller having to spell out
+// what is inside it.
+
+
+// Droppings that are not output of anything: an editor, the Finder, or a
+// version control system left them, and they occupy no space in the meaning of
+// the tree. Kept separate from BUILD because "would we copy this" and "is this
+// directory empty" are different questions, and only this half answers both —
+// a destination holding `node_modules` is not empty by any reading.
+const NOISE: RegExp[] = [
+  // Editors and merges. `~` duplicates jostraca's built-in `IGNORED_RE`,
+  // deliberately: that one governs its Copy walk alone, and the walks sdkgen
+  // does itself (the prune, the feature scan) have to agree with it.
+  /~$/,                 // editor backup
+  /^\.#/,               // emacs lock link
+  /^#.*#$/,             // emacs autosave
+  /\.sw[a-p]$/,         // vim swap
+  /\.(bak|orig|rej)$/,  // backups, and what a conflicted merge leaves behind
+
+  // Operating systems.
+  /^\.DS_Store$/,
+  /^__MACOSX$/,
+  /^Thumbs\.db$/i,
+  /^desktop\.ini$/i,
+
+  // Version control. A nested checkout inside a template tree is never the
+  // payload, and copying one produces a project git cannot make sense of.
+  /^\.git$/,
+  /^\.svn$/,
+  /^\.hg$/,
+  /^\.idea$/,           // not `.vscode` — an SDK may ship editor settings
+]
+
+
+// Toolchain output, per language. Each of these appears exactly when someone
+// runs that language's tools inside a template tree, which is the normal way to
+// check that a template actually works — and none of them is ever the payload.
+const BUILD: RegExp[] = [
+  /^node_modules$/,
+  /^__pycache__$/,
+  /\.py[co]$/,
+  /^\.(pytest|mypy|ruff)_cache$/,
+  /^\.tox$/,
+  /^\.venv$/,
+  /^\.gradle$/,
+  /\.class$/,
+  /^\.dart_tool$/,
+  /^\.?zig-cache$/,
+  /^zig-out$/,
+  /^_build$/,           // mix, dune
+  /^\.elixir_ls$/,
+  /^\.cpcache$/,        // clojure
+  /^\.stack-work$/,
+  /^dist-newstyle$/,    // cabal
+  /^\.cargo$/,
+]
+
+
+const JUNK: RegExp[] = [...NOISE, ...BUILD]
+
+
+// Is this directory entry something no tree operation should see?
+function isJunk(name: string): boolean {
+  return matches(name, JUNK)
+}
+
+
+// Weaker: does this entry leave a directory as good as empty? Toolchain output
+// does NOT — it is junk to copy and content to find.
+function isNoise(name: string): boolean {
+  return matches(name, NOISE)
+}
+
+
+function matches(name: string, patterns: RegExp[]): boolean {
+  for (const re of patterns) {
+    // These are module-level regexes reused across every entry of every tree.
+    // None carries `g` or `y` today, but a `test` on a stateful regex resumes
+    // from the previous match and would skip every other file — the exact bug
+    // jostraca's own `excluded()` carries a comment about.
+    re.lastIndex = 0
+
+    if (re.test(name)) {
+      return true
+    }
+  }
+
+  return false
+}
+
+
+// The jostraca `cmp` option block, which is how the WRITER side of the rule is
+// applied: jostraca tests `cmp.Copy.ignore` against the bare name of every
+// entry it walks, directories included, and prunes what matches.
+//
+// Passed PER CALL by each action, not left to the Jostraca instance — the same
+// discipline `control.dryrun` is held to, and for a related reason: the actions
+// take whatever instance their caller hands them (`actx.jostraca`), and a
+// consumer that builds its own would otherwise silently lose the filter. A
+// per-call value is merged last and wins.
+//
+// `/~$/` is first, and must stay first while any supported jostraca is at or
+// below 0.31.0. That merge takes this list over its own default of `[/~$/]`
+// with `deep()`, which recursed INTO index 0 — two RegExps are both objects, so
+// it iterated the incoming one's enumerable properties, of which a RegExp has
+// none, and kept the default. Whatever sat at index 0 was silently discarded.
+// Leading with the value already there makes that a no-op instead of a hole,
+// and jostraca's built-in `IGNORED_RE` covers `~` anyway.
+//
+// FIXED UPSTREAM (jostraca `deep` now replaces a custom-constructor value
+// rather than walking into it), so this is compatibility, not a live
+// workaround: harmless on a fixed jostraca, load-bearing on an older one.
+//
+// A FRESH object each call: `deep` mutates its base, and jostraca's merge order
+// leaves these values reachable from options it keeps.
+function copyOpts(): { Copy: { ignore: RegExp[] } } {
+  return {
+    Copy: {
+      ignore: [/~$/, ...JUNK]
+    }
+  }
+}
+
+
+export {
+  JUNK,
+  NOISE,
+  BUILD,
+  isJunk,
+  isNoise,
+  copyOpts,
+}