@voxgig/sdkgen 3.4.6 → 3.5.0

package/src/sdkgen.ts

@@ -5,7 +5,6 @@ import Path from 'node:path'
 
 import { prettyPino, Pino } from '@voxgig/util'
 
-import { Jsonic } from '@tabnas/jsonic'
 import * as JostracaModule from 'jostraca'
 import { Aontu } from 'aontu'
 
@@ -25,6 +24,8 @@ import { SdkGenError, requirePath, isAuthActive, resolveAuthPrefix,
 
 import { Main } from './cmp/Main'
 import { ExternalTarget } from './cmp/ExternalTarget'
+import { Docs, DocsItem } from './cmp/Docs'
+import { ExternalDocs } from './cmp/ExternalDocs'
 
 import { KIT } from '@voxgig/apidef'
 import { Deploy } from './cmp/Deploy'
@@ -73,6 +74,19 @@ import {
   srcFeatureExcludes,
 } from './helpers/featureSource'
 import type { FeatureSource } from './helpers/featureSource'
+import {
+  definitionPath,
+  definitionFolder,
+  definitionNames,
+} from './helpers/definition'
+import { isNoise, copyOpts } from './helpers/junk'
+import {
+  MANIFEST,
+  manifestPath,
+  readManifest,
+  validateManifest,
+} from './helpers/manifest'
+import type { Manifest, ManifestRead } from './helpers/manifest'
 import {
   packageName,
   installCommand,
@@ -117,6 +131,19 @@ import {
 } from './action/doctor'
 import type { DoctorReport } from './action/doctor'
 
+import {
+  action_package,
+  package_add,
+  package_update,
+} from './action/package'
+
+import { cmd_package_check } from './action/check'
+
+import { docs_add } from './action/docs'
+
+// The verbs, built from the kind registry — see action/dispatch.
+import { ACTION_MAP, actionNames, needsModel } from './action/dispatch'
+
 
 
 // TODO: use shape
@@ -149,13 +176,6 @@ type SdkGenOptions = {
 const { Jostraca } = JostracaModule
 
 
-const ACTION_MAP: any = {
-  target: action_target,
-  feature: action_feature,
-  doctor: action_doctor,
-}
-
-
 const dlog = getdlog('sdkgen', __filename)
 
 
@@ -188,7 +208,12 @@ function SdkGen(opts: SdkGenOptions) {
         write: true,
         merge: false
       }
-    }
+    },
+    // No copied tree carries a maintainer's `.DS_Store` or `__pycache__` into
+    // an SDK. Set here AND per call (see helpers/junk) — the actions run on
+    // whatever instance their caller supplies, so neither placement covers
+    // every path on its own.
+    cmp: copyOpts()
   }
 
   const jostraca = Jostraca(jopts)
@@ -229,6 +254,9 @@ function SdkGen(opts: SdkGenOptions) {
       control: {
         dryrun: !!opts.dryrun
       },
+      // Per-call for the same reason as `control` above: an option the
+      // instance carries is not reliably the one a generate sees.
+      cmp: copyOpts(),
     }
 
     // Targets that write OUTSIDE the SDK repo (`output: path`) are generated
@@ -240,7 +268,7 @@ function SdkGen(opts: SdkGenOptions) {
     // so resolve it ONCE: every destination is compared against it, and a
     // comparison between a relative and an absolute path is meaningless.
     const root = Path.resolve(folder)
-    const external = externalTargets(model, root)
+    const external = externalItems(model, root, ['target', 'docs'])
 
     // Before ANY file is written, in-tree included: a destination that turns
     // out to be wrong must abort the whole generation, not leave half of it
@@ -248,15 +276,85 @@ function SdkGen(opts: SdkGenOptions) {
     checkExternalFolders(external, root, fs)
 
     const jres = await jostraca.generate(
-      jopts, () => Root({ model: 0 === external.length ? model : withoutTargets(model, external) }))
+      jopts, () => Root({ model: 0 === external.length ? model : withoutExternal(model, external) }))
 
     showChanges(jopts.log, 'generate-result', jres, Path.dirname(process.cwd()))
 
+    // IN-TREE DOCS, in their own pass over the same root.
+    //
+    // The alternative is one `Kinds()` call in the consumer's `Root.ts` —
+    // which create-sdkgen writes once and never revisits, so every project
+    // scaffolded before the docs kind existed would have to be hand-edited
+    // before `docs add` did anything. A second pass costs one more changes
+    // report and makes the kind work in projects that predate it, which is
+    // the whole point of putting the destinations in packages.
+    //
+    // Items with `output: path` are excluded here and generated by the loop
+    // below, exactly as out-of-tree targets are.
+    const intreedocs = Object.keys(model?.main?.[KIT]?.docs ?? {})
+      .filter((name: string) => {
+        const item = model.main[KIT].docs[name]
+        const path = item?.output?.path
+        return false !== item?.active && (null == path || '' === path)
+      })
+
+    // AN IN-TREE DOCS ITEM AND A TARGET CANNOT SHARE A NAME.
+    //
+    // `docs` and `target` are separate namespaces — deliberately, so an item
+    // of each may be called `api` — and their `.sdk` trees are nested apart
+    // (`src/cmp/docs/<n>` vs `src/cmp/<n>`), so installing both is fine.
+    //
+    // Their GENERATED destinations are not separate. Both render into
+    // `<sdk-repo>/<name>/`, the docs pass runs second, and jostraca is
+    // last-write-wins — so a docs item named `go` would quietly overwrite the
+    // Go SDK's README, and whatever else the two have in common, leaving a
+    // corrupt SDK and a green run.
+    //
+    // Refused rather than reordered, and refused BEFORE the docs pass writes
+    // anything: this is the in-tree half of the rule `checkExternalFolders`
+    // enforces for out-of-tree destinations, and the fix is the same either
+    // way — give one of them `output: path`, or rename it.
+    const targets: any = model?.main?.[KIT]?.target ?? {}
+
+    for (const name of intreedocs) {
+      const target = targets[name]
+
+      if (null == target || false === target.active) {
+        continue
+      }
+
+      const tpath = target.output?.path
+
+      if (null != tpath && '' !== tpath) {
+        continue
+      }
+
+      throw new SdkGenError(
+        'Docs item "' + name + '" and target "' + name + '" would both ' +
+        'generate into <sdk-project>/' + name + '/.\n' +
+        '  They are separate kinds, so sharing a name is legal — but they ' +
+        'share one output folder, and the docs pass runs second, so it ' +
+        'would overwrite the SDK.\n' +
+        '  Give one of them `output: path`, or rename it.')
+    }
+
+    if (0 < intreedocs.length) {
+      log.info({
+        point: 'generate-docs-start', docs: intreedocs,
+        note: intreedocs.join(', ')
+      })
+
+      const dres = await jostraca.generate(jopts, () => Docs({ model }))
+
+      showChanges(jopts.log, 'generate-docs-result', dres,
+        Path.dirname(process.cwd()))
+    }
+
     for (const ext of external) {
       // `active: false` is the project's only lever to stop the generator
       // writing into a repo it does not own, so it has to be honoured HERE —
       // the consumer Root iterates targets raw and does not check it. The
-      // target is still removed from the in-tree model above (withoutTargets
+      // target is still removed from the in-tree model above (withoutExternal
       // takes every `output: path` target, active or not), so switching one
       // off generates it nowhere rather than relocating it into
       // `<sdk-repo>/<target>/`.
@@ -273,15 +371,23 @@ function SdkGen(opts: SdkGenOptions) {
         note: ext.name + ' -> ' + ext.folder
       })
 
-      const eres = await jostraca.generate(
+      const sdkrelpath = externalSdkRel(ext, root, log)
+
+      const eres = 'docs' === ext.kind ?
+        await jostraca.generate(
+          { ...jopts, folder: ext.folder },
+          () => ExternalDocs({
+            model, item: ext.target, cmpfolder: folder, sdkrelpath,
+          })) :
+        await jostraca.generate(
         { ...jopts, folder: ext.folder },
         () => ExternalTarget({
           model, target: ext.target, cmpfolder: folder,
-          // How to walk BACK to the SDK project from the destination. A
-          // target generating out of tree usually sits beside the SDK in a
+          // How to walk BACK to the SDK project from the destination. An
+          // item generating out of tree usually sits beside the SDK in a
           // known layout, and its own docs, scripts and live tests need to
           // name that path.
-          sdkrelpath: externalSdkRel(ext, root, log),
+          sdkrelpath,
         }))
 
       showChanges(jopts.log, 'generate-result', eres, Path.dirname(process.cwd()))
@@ -300,26 +406,47 @@ function SdkGen(opts: SdkGenOptions) {
   }
 
 
-  async function action(args: string[]): Promise<any> {
-    const pargs = args.map(arg => Jsonic(arg))
-
+  // Action arguments are PATHS AND NAMES, and they reach the actions as the
+  // raw strings the shell gave us.
+  //
+  // They used to be mapped through `Jsonic(arg)` first, which parses each one
+  // as relaxed JSON — so a Windows absolute ref arrived as an OBJECT:
+  // `Jsonic('C:\\pkg\\go')` is `{ C: '\\pkg\\go' }`, and every downstream path
+  // join then missed. (`ts,py` also became `['ts','py']`, which is why
+  // parseAddNames still carries a non-string branch; it splits on commas
+  // itself, so nothing is lost by handing it strings.)
+  //
+  // Nothing here wants structured arguments: every action takes names and
+  // refs. Parsing them was pure loss.
+  //
+  // `flags` is the second parameter because `--only` and `--alias` are
+  // arguments to ONE command, not generator configuration: passing them
+  // through `SdkGen({…})` like `debug`/`dryrun` would make a later
+  // `action()` call on the same instance silently inherit them.
+  async function action(
+    args: string[], flags?: Record<string, any>,
+  ): Promise<any> {
     const actname = args[0]
     const actionFunc = ACTION_MAP[actname]
 
     if (null == actionFunc) {
-      throw new SdkGenError('Unknown action: ' + actname)
+      throw new SdkGenError(
+        'Unknown action: ' + actname +
+        ' (expected: ' + actionNames().join(', ') + ')')
     }
 
-    const ctx = resolveActionContext()
+    const ctx = resolveActionContext(flags, needsModel(args))
 
-    return await actionFunc(pargs, ctx)
+    return await actionFunc(args, ctx)
   }
 
 
-  function resolveActionContext(): ActionContext {
+  function resolveActionContext(
+    flags?: Record<string, any>, wantmodel?: boolean,
+  ): ActionContext {
 
     // TODO: use AsyncLocalStorage to avoid reloading model
-    const { model, url } = resolveModel()
+    const { model, url } = resolveModel(false !== wantmodel)
 
     const ctx: ActionContext = {
       fs: () => fs,
@@ -329,16 +456,25 @@ function SdkGen(opts: SdkGenOptions) {
       url,
       jostraca,
       opts,
+      flags: flags ?? {},
     }
 
     return ctx
   }
 
 
-  function resolveModel() {
+  function resolveModel(wanted: boolean) {
     const path = './model/sdk.aontu'
     const errs: any[] = []
 
+    // A verb that does not act on a project (see `needsModel`) is run where
+    // there is no project model, so its absence is not an error there. Its
+    // PRESENCE still is compiled — an author checking a package from inside a
+    // project should get the same model every other verb gets.
+    if (!wanted && !fs.existsSync(path)) {
+      return { model: { main: {} } as any, url: path }
+    }
+
     if (null == aontu) {
       aontu = new Aontu()
     }
@@ -388,6 +524,47 @@ function SdkGen(opts: SdkGenOptions) {
     }
   }
 
+  // The third kind. sdkgen ships no docs items, so every ref here is a path
+  // into a package — see action/docs.
+  const docs = {
+    add: async (items: string[]): Promise<ActionResult> => {
+      const ctx = resolveActionContext()
+      return docs_add(items, ctx)
+    }
+  }
+
+  // The whole-package verbs. `flags` mirrors the CLI's `--only` / `--alias`,
+  // for the same reason they are `action`'s second parameter rather than
+  // constructor options: they are arguments to one call.
+  const packages = {
+    add: async (
+      refs: string[], flags?: Record<string, any>,
+    ): Promise<ActionResult> => {
+      const ctx = resolveActionContext(flags)
+      return package_add(refs, ctx)
+    },
+
+    list: async (): Promise<ActionResult> => {
+      const ctx = resolveActionContext()
+      return action_package(['package', 'list'], ctx)
+    },
+
+    update: async (
+      names: string[], flags?: Record<string, any>,
+    ): Promise<ActionResult> => {
+      const ctx = resolveActionContext(flags)
+      return package_update(names, ctx)
+    },
+
+    // Validate a package being AUTHORED — see action/check. The only verb
+    // here that does not need a project model, because it does not act on a
+    // project; `false` says so to `resolveActionContext`.
+    check: async (refs?: string[]): Promise<ActionResult> => {
+      const ctx = resolveActionContext(undefined, false)
+      return cmd_package_check(['package', 'check', ...(refs ?? [])], ctx)
+    },
+  }
+
   // Has this project's `.sdk/` drifted from the scaffold? See action/doctor.
   const check = async (): Promise<ActionResult> => {
     const ctx = resolveActionContext()
@@ -401,8 +578,13 @@ function SdkGen(opts: SdkGenOptions) {
     generate,
     action,
     check,
+    docs,
     target,
     feature,
+
+    // `package` is a reserved word in a strict-mode object shorthand, so the
+    // local binding is `packages` and the PUBLIC name matches the CLI verb.
+    package: packages,
   }
 
 }
@@ -437,6 +619,9 @@ SdkGen.makeBuild = async function(opts: SdkGenOptions) {
 
 
 type ExternalSpec = {
+  // Which kind's collection this item came from — `target` or `docs`. The
+  // out-of-tree rules are the same for both; only the dispatch differs.
+  kind: string
   name: string
   target: any
   folder: string
@@ -453,22 +638,36 @@ type ExternalSpec = {
 // directory the command happened to be run from.
 //
 // An INACTIVE target is still listed: it must be taken out of the in-tree
-// model (see withoutTargets) so that switching it off does not silently
+// model (see withoutExternal) so that switching it off does not silently
 // relocate it into `<sdk-repo>/<target>/`. The generate loop skips it.
-function externalTargets(model: any, folder: string): ExternalSpec[] {
-  const targets = model?.main?.[KIT]?.target || {}
-
-  return Object.keys(targets).sort()
-    .map((name: string) => ({ name, target: targets[name] }))
-    .filter((t: any) => {
-      const path = t.target?.output?.path
-      return null != path && '' !== path
-    })
-    .map((t: any) => ({
-      ...t,
-      folder: Path.resolve(folder, String(t.target.output.path)),
-      active: false !== t.target.active,
-    }))
+// Every item, of every kind, that generates OUTSIDE the SDK repo.
+//
+// Keyed by kind because `docs` needs exactly this and a second copy of it
+// would drift: the out-of-tree rules — a destination must be outside the
+// project, unclaimed, and either empty or previously generated into — are
+// properties of WRITING SOMEWHERE ELSE, not of being a target.
+//
+// The claim map spanning kinds is the load-bearing part: a docs item and a
+// target pointed at the same folder would otherwise silently overwrite each
+// other, in whatever order the passes happen to run.
+function externalItems(
+  model: any, folder: string, kinds: string[],
+): ExternalSpec[] {
+  return kinds.flatMap((kind: string) => {
+    const items = model?.main?.[KIT]?.[kind] || {}
+
+    return Object.keys(items).sort()
+      .map((name: string) => ({ kind, name, target: items[name] }))
+      .filter((t: any) => {
+        const path = t.target?.output?.path
+        return null != path && '' !== path
+      })
+      .map((t: any) => ({
+        ...t,
+        folder: Path.resolve(folder, String(t.target.output.path)),
+        active: false !== t.target.active,
+      }))
+  })
 }
 
 
@@ -480,9 +679,12 @@ function externalTargets(model: any, folder: string): ExternalSpec[] {
 const EXTERNAL_MARKER = '.jostraca'
 
 // Entries that do not count as content when deciding whether a destination
-// is empty. A `git init` (or a clone of an empty repo) leaves only `.git`,
-// and that is exactly the destination a FIRST generation is aimed at.
-const EXTERNAL_EMPTY = ['.git', '.DS_Store']
+// is empty. A `git init` (or a clone of an empty repo) leaves only `.git`, and
+// that is exactly the destination a FIRST generation is aimed at — as is one
+// the maintainer opened in the Finder on the way to declaring it.
+//
+// `isNoise`, not `isJunk`: a destination holding build output is not empty,
+// whether or not sdkgen would ever copy such a thing. See helpers/junk.
 
 
 // Refuse a destination the project cannot have meant.
@@ -517,39 +719,44 @@ function checkExternalFolders(external: ExternalSpec[], root: string, fs: any) {
     // path valid.
     if (!ext.active) continue
 
-    const where = 'Target "' + ext.name + '" has output path "' +
+    const label = ext.kind.charAt(0).toUpperCase() + ext.kind.slice(1)
+
+    const where = label + ' "' + ext.name + '" has output path "' +
       ext.target.output.path + '", which resolves to: ' + ext.folder +
       '\n  (SDK project: ' + root + ')'
 
     if (ext.folder === root || folderContains(root, ext.folder)) {
       throw new SdkGenError(
-        'External target output path is inside the SDK project.\n  ' + where +
-        '\n  A target generating into the SDK project must leave `output: ' +
+        'External output path is inside the SDK project.\n  ' + where +
+        '\n  An item generating into the SDK project must leave `output: ' +
         'path` unset — it is then generated in-tree, as <sdk-project>/' +
         ext.name + '/.')
     }
 
     if (folderContains(ext.folder, root)) {
       throw new SdkGenError(
-        'External target output path contains the SDK project.\n  ' + where +
+        'External output path contains the SDK project.\n  ' + where +
         '\n  Generation would write this package over the directory holding ' +
         'the SDK project itself.')
     }
 
+    // ACROSS KINDS, not within one: `docs` and `target` are separate
+    // namespaces, so an item of each may legitimately be called `api` — and
+    // pointing both at one folder is the mistake this catches.
     if (null != claimed[ext.folder]) {
       throw new SdkGenError(
-        'External target output path is already claimed by target "' +
-        claimed[ext.folder] + '".\n  ' + where +
-        '\n  Two targets generating into the same folder overwrite each ' +
-        'other, in target-name order.')
+        'External output path is already claimed by ' +
+        claimed[ext.folder] + '.\n  ' + where +
+        '\n  Two items generating into the same folder overwrite each ' +
+        'other, in whatever order the passes run.')
     }
-    claimed[ext.folder] = ext.name
+    claimed[ext.folder] = ext.kind + ' "' + ext.name + '"'
 
     if (!fs.existsSync(ext.folder)) continue
 
     if (!fs.statSync(ext.folder).isDirectory()) {
       throw new SdkGenError(
-        'External target output path is not a folder.\n  ' + where)
+        'External output path is not a folder.\n  ' + where)
     }
 
     if (true === ext.target.output.adopt) continue
@@ -559,7 +766,7 @@ function checkExternalFolders(external: ExternalSpec[], root: string, fs: any) {
 
     if (entries.includes(EXTERNAL_MARKER)) continue
 
-    const content = entries.filter((entry) => !EXTERNAL_EMPTY.includes(entry))
+    const content = entries.filter((entry) => !isNoise(entry))
 
     if (0 < content.length) {
       throw new SdkGenError(
@@ -635,23 +842,32 @@ function externalSdkRel(ext: ExternalSpec, root: string, log: any): string {
 // A shallow clone down to `target` only — the model is large, entities and
 // features are shared with the external pass, and a deep copy would both
 // cost and quietly break identity comparisons.
-function withoutTargets(model: any, external: { name: string }[]): any {
-  const drop = new Set(external.map((e) => e.name))
-  const targets = model?.main?.[KIT]?.target || {}
-
-  const kept: any = {}
-  for (const name of Object.keys(targets)) {
-    if (!drop.has(name)) {
-      kept[name] = targets[name]
+// The model the IN-TREE pass sees: every out-of-tree item removed, whatever
+// its kind. Rendering one in both passes would ALSO write it into
+// `<sdk-repo>/<name>/`, which is the folder the item asked not to use.
+function withoutExternal(
+  model: any, external: { kind: string, name: string }[],
+): any {
+  const drop = new Set(external.map((e) => e.kind + ':' + e.name))
+
+  const kit: any = { ...model.main[KIT] }
+
+  for (const kind of new Set(external.map((e) => e.kind))) {
+    const items = model?.main?.[KIT]?.[kind] || {}
+    const kept: any = {}
+
+    for (const name of Object.keys(items)) {
+      if (!drop.has(kind + ':' + name)) {
+        kept[name] = items[name]
+      }
     }
+
+    kit[kind] = kept
   }
 
   return {
     ...model,
-    main: {
-      ...model.main,
-      [KIT]: { ...model.main[KIT], target: kept },
-    },
+    main: { ...model.main, [KIT]: kit },
   }
 }
 
@@ -698,6 +914,8 @@ export type {
   FeatureSource,
   DoctorReport,
   RegisterOptions,
+  Manifest,
+  ManifestRead,
 }
 
 export type {
@@ -838,6 +1056,16 @@ export {
   featureExcludes,
   fullsetExcludes,
   srcFeatureExcludes,
+
+  definitionPath,
+  definitionFolder,
+  definitionNames,
+
+  MANIFEST,
+  manifestPath,
+  readManifest,
+  validateManifest,
+
   jsProp,
   jsOptProp,
   jsKey,

package/src/types.ts

@@ -64,7 +64,7 @@ type ModelTarget = NameCases & {
   // the SDK repo — see cmp/ExternalTarget and
   // docs/explanation/out-of-tree-targets.
   //
-  // Typed rather than left to the index signature because `externalTargets()`
+  // Typed rather than left to the index signature because `externalItems()`
   // decides from these keys whether to write OUTSIDE the repo, and a
   // destination path read off a bare `any` is one a rename can silently
   // change to undefined.
@@ -121,12 +121,41 @@ type ActionContext = {
   model: SdkModel,
   url: string,
   opts: any,
-  jostraca: any
+  jostraca: any,
+
+  // PER-INVOCATION action arguments — `--only`, `--alias` — as opposed to
+  // generator configuration. `debug` and `dryrun` reach actions through the
+  // `SdkGen({…})` constructor because they describe the generator; these
+  // describe one command, so smuggling them through the constructor would
+  // make a second `action()` call on the same instance inherit them.
+  flags?: Record<string, any>
+
+  // How `package update` fetches a new version. Injectable so tests do not
+  // shell out, and so a caller with its own dependency management (a
+  // monorepo, a vendored checkout) can supply one. Defaults to npm.
+  fetchPackage?: (pkgname: string, actx: ActionContext) => Promise<void>
 }
 
 
+// An action either WROTE something or REPORTED something, and the two kinds
+// of verb return different things: `target add` returns jostraca's result,
+// while `doctor` and `package check` return a report the CLI turns into an
+// exit code. Both optional, because neither verb has anything to say about
+// the other's field — a check that invented an empty `jres` would be lying
+// about having generated nothing.
 type ActionResult = {
-  jres: JostracaResult
+  jres?: JostracaResult
+  report?: ActionReport
+}
+
+
+// What a CHECKING verb returns. `ok` is the contract — `bin/voxgig-sdkgen`
+// exits non-zero on `false` — and `summary` is the one line it prints, so the
+// binary never has to know what any particular verb's findings look like.
+type ActionReport = {
+  ok: boolean
+  summary?: string
+  [key: string]: any
 }
 
 
@@ -138,6 +167,7 @@ export {
 export type {
   ActionContext,
   ActionResult,
+  ActionReport,
   SdkModel,
   ModelKit,
   ModelTarget,

package/src/utility.ts

@@ -133,6 +133,23 @@ export {
 //
 // The reader side is unchanged either way: make_config returns the same map,
 // so nothing downstream can tell which representation it got.
+//
+// For two targets the literal is not merely expensive but IMPOSSIBLE past a
+// point, which is what fixes the threshold rather than leaving it a taste
+// question:
+//
+//   haskell  GHC 9.4.7 refuses a large static structure outright -
+//            "sorry! (unimplemented feature or known bug) ... Trying to
+//            allocate more than 129024 bytes ... Suggestion: read data from a
+//            file instead of having large static data structures in code"
+//            (GHC issue 4505). Measured: the CV literal compiles at 828 KB of
+//            model and fails at 1.4 MB, so 256 KB clears it by more than 3x.
+//
+//   clojure  a string literal is a constant-pool UTF-8 entry capped at 65,535
+//            bytes, so the DATA constant has to be chunked - see cljStringChunks.
+//
+// Anything above the threshold therefore takes the data path in every target,
+// and the languages with a hard ceiling are the ones with the most margin.
 const CONFIG_DATA_THRESHOLD = 256 * 1024