@voxgig/sdkgen 4.17.5 → 4.18.0

package/project/.sdk/tm/ts/src/feature/validate/ValidateFeature.ts

@@ -0,0 +1,298 @@
+import type { Context, FeatureOptions } from '../../types'
+import type { ProjectNameSDK } from '../../ProjectNameSDK'
+
+import { ENTITYSPEC } from '../../Schema'
+
+import { BaseFeature } from '../base/BaseFeature'
+
+
+// Payload validation against the model's own field types.
+//
+// The specs are NOT written here and not written in the model either: every
+// entity field already carries a canonical type sentinel (`$STRING`,
+// `$INTEGER`, the `$ONE` union for an OpenAPI multi-type), which is the same
+// vocabulary struct.validate speaks. The generator maps them once
+// (helpers/canonSpec) and emits `ENTITYSPEC` beside this file, so a field
+// whose type changes in the API spec changes what this feature enforces with
+// no edit anywhere.
+//
+// WHAT IS CHECKED
+//   outbound (PreSpec)  the payload the caller asked to send, against
+//                       `spec.op[<opname>]` — the operation's request shape,
+//                       which is the SAME partiality policy that decides what
+//                       the generated `<Name>CreateData` type requires.
+//   inbound  (PreDone)  each record the operation returned, against
+//                       `spec.data` — the entity's own field types.
+//
+// WHAT IS NOT. The model carries no array element types, no nested object
+// schemas, no enums, formats or bounds (see canonSpec's note), so this checks
+// the shape the model knows and nothing more. It is a guard against the
+// mistakes the model CAN see — a number where a string belongs, a required
+// field left out, a misspelled key under `strict` — not a substitute for the
+// server's own validation.
+class ValidateFeature extends BaseFeature {
+  version = '0.0.1'
+  name = 'validate'
+  active = true
+
+  _client?: ProjectNameSDK
+  _options: any = {}
+  _spec: Record<string, any> = {}
+
+  _request = true
+  _response = false
+  _mode = 'throw'
+
+
+  init(ctx: Context, options: FeatureOptions): void | Promise<any> {
+    this._client = ctx.client
+    this._options = options || {}
+    this.active = (options as any).active
+
+    // DEFAULTS ARE APPLIED HERE, not by the option spec. The model's
+    // `config.options` documents them and types them; it does not inject
+    // them, because each feature entry in the spec is optional and struct
+    // fills in nothing through an optional union. So every feature resolves
+    // its own — and a `mode` left undefined here once meant `'throw' !==
+    // undefined`, which silently turned every rejection into a no-op.
+    this._request = false !== this._options.request
+    this._response = true === this._options.response
+
+    // FAIL CLOSED. Only the exact string 'report' selects report mode, so a
+    // typo (`mode: 'thow'`) still rejects rather than silently turning
+    // enforcement off — the failure nobody would notice. The option spec
+    // rejects the typo outright; this is what happens if it ever does not.
+    this._mode = 'report' === this._options.mode ? 'report' : 'throw'
+
+    // `strict` is applied ONCE, here, by rebuilding the spec tree without the
+    // `$OPEN` markers — rather than per call, which would clone a spec for
+    // every request an SDK ever makes.
+    this._spec = true === this._options.strict ? close(ENTITYSPEC) : ENTITYSPEC
+  }
+
+
+  // Outbound. `makeSpec` short-circuits on an `ctx.out.spec` that is already
+  // set, so assigning the error here rejects the operation before the request
+  // is built — the same seam rbac uses one stage earlier.
+  PreSpec(this: any, ctx: any) {
+    if (!this.active || !this._request) {
+      return
+    }
+
+    const opname = (ctx.op && ctx.op.name) || ''
+    const spec = this._entitySpec(ctx)
+    const opspec = spec && spec.op ? spec.op[opname] : null
+
+    if (null == opspec) {
+      return
+    }
+
+    const errs = this._check(ctx, this._payload(ctx, opname), opspec, 'request')
+    if (0 === errs.length || 'report' === this._mode) {
+      return
+    }
+
+    const err = ctx.error('validate_failed',
+      'Invalid ' + opname + ' request for entity "' + entname(ctx) + '": ' +
+      errs.join('; '))
+    ctx.out.spec = err
+    return err
+  }
+
+
+  // Inbound. PreDone rather than PreResult: the records are extracted from
+  // the response body by `makeResult`, which runs between the two, so at
+  // PreResult there is nothing to check but the envelope.
+  //
+  // HOOK ORDER MATTERS HERE, and the default order is not the one you want.
+  // PreDone hooks fire in feature ADD order, which defaults to `test` first
+  // and then names sorted — and `validate` sorts last, after `audit`,
+  // `cost`, `debug`, `metrics` and `telemetry`. Those observers therefore
+  // record the operation as a success before this hook has looked at it.
+  // Activating features as an ORDERED ARRAY fixes it:
+  //
+  //   feature: [{ name: 'validate', active: true, response: true },
+  //             { name: 'metrics', active: true }]
+  //
+  // What this feature can fix from here, it does: the result is marked
+  // failed and its records are cleared, so the entity absorbs nothing.
+  PreDone(this: any, ctx: any) {
+    if (!this.active || !this._response) {
+      return
+    }
+
+    const spec = this._entitySpec(ctx)
+    if (null == spec || null == spec.data) {
+      return
+    }
+
+    const resdata = ctx.result && ctx.result.resdata
+    if (null == resdata) {
+      return
+    }
+
+    // A list op returns many records and a load returns one; both are checked
+    // against the same record spec, because they are the same entity.
+    const records = Array.isArray(resdata) ? resdata : [resdata]
+    const errs: string[] = []
+    for (const record of records) {
+      if (null == record) {
+        continue
+      }
+
+      // A NON-OBJECT IS A FAILURE, not something to skip. A load that
+      // answered `42` where the entity's spec wants a record used to pass
+      // this feature silently, which is the one outcome a validator must
+      // never produce. struct rejects it with the field it could not find.
+      for (const e of this._check(ctx, unwrap(record), spec.data, 'response')) {
+        errs.push(e)
+      }
+    }
+
+    if (0 === errs.length || 'report' === this._mode) {
+      return
+    }
+
+    const err = ctx.error('validate_failed',
+      'Invalid response for entity "' + entname(ctx) + '": ' + errs.join('; '))
+
+    // BOTH, and `ok` is the load-bearing half: `done` returns `resdata`
+    // whenever `result.ok` is true and never looks at `err`, so setting the
+    // error alone handed the caller the very records that failed the spec.
+    ctx.result.ok = false
+    ctx.result.err = err
+
+    // AND THE DATA GOES. The load/update fragments copy `result.resdata`
+    // into the entity's own state on any non-null value, BEFORE `done`
+    // raises — so rejecting the operation while leaving the records in place
+    // left the caller holding an entity populated from a payload this
+    // feature had just declared invalid. Clearing it is the only half of
+    // that this feature owns; see the note on hook order below.
+    ctx.result.resdata = undefined
+
+    return err
+  }
+
+
+  // The payload an operation is about to send.
+  //
+  // TWO SLOTS, AND THE OP PICKS. A body op (create/update/patch) carries the
+  // caller's argument in `reqdata` over the entity's `data`; a match op
+  // (load/list/remove) carries it in `reqmatch` over `match`. That is what
+  // the Entity*Op fragments pass to makeContext and what makePoint reads
+  // (`getprop(ctx, 'req' + op.input)`) — so reading `reqdata` for every op
+  // checked a `load({ id })` against the entity's STALE stored match and
+  // rejected it for the id the caller had just supplied.
+  _payload(this: any, ctx: any, opname: string): Record<string, any> {
+    const body = 'create' === opname || 'update' === opname || 'patch' === opname
+
+    const base = body ? ctx.data : ctx.match
+    const req = body ? ctx.reqdata : ctx.reqmatch
+
+    const out: Record<string, any> = { ...(base || {}), ...(req || {}) }
+
+    // `$action` SELECTS A CUSTOM ENDPOINT; it is not a field of the record.
+    // makePoint reads it off this same argument and the request transformer
+    // drops it before the body is built, so a spec built from the API's own
+    // fields will never name it — and under `strict` every custom-action
+    // call would be rejected for the one key that made it reachable.
+    delete out.$action
+
+    return out
+  }
+
+
+  _entitySpec(this: any, ctx: any): any {
+    return this._spec[entname(ctx)]
+  }
+
+
+  // One validate call. Errors are COLLECTED, never thrown: struct throws on
+  // the first failure unless given an `errs` array, and a caller fixing a
+  // payload wants every problem with it, not the first one.
+  _check(this: any, ctx: any, data: any, spec: any, direction: string): string[] {
+    const struct = ctx.utility.struct
+    const errs: string[] = []
+
+    try {
+      struct.validate(data, spec, { errs })
+    }
+    catch (e: any) {
+      // A spec this port cannot run at all (rather than a payload that fails
+      // it) must not take the operation down with it: report it like any
+      // other failure and let `mode` decide.
+      errs.push(e && e.message ? e.message : String(e))
+    }
+
+    if (0 < errs.length && 'function' === typeof this._options.onInvalid) {
+      try {
+        this._options.onInvalid({
+          entity: entname(ctx),
+          op: (ctx.op && ctx.op.name) || '',
+          direction,
+          errs,
+          data,
+        })
+      }
+      catch (_e) { }
+    }
+
+    return errs
+  }
+}
+
+
+// A RESULT RECORD AS DATA.
+//
+// `makeResult` turns every record of a LIST into an entity instance
+// (`entity.make()` then `ent.data(entry)`), so what reaches PreDone for a
+// list is wrappers, not records — and a wrapper checked against a field spec
+// fails on every required field while its actual data goes unchecked. A load
+// returns the record itself, so this has to handle both.
+function unwrap(record: any): any {
+  if (null != record && 'function' === typeof record.data) {
+    const data = record.data()
+    if (null != data) {
+      return data
+    }
+  }
+  return record
+}
+
+
+function entname(ctx: any): string {
+  return (ctx.entity && ctx.entity.name) || (ctx.op && ctx.op.entity) || ''
+}
+
+
+// The spec tree with every `$OPEN` marker removed, so an undeclared key is an
+// error rather than a pass. Rebuilt rather than mutated: ENTITYSPEC is a
+// module constant shared by every client in the process.
+function close(node: any): any {
+  if (Array.isArray(node)) {
+    return node.map((n: any) => close(n))
+  }
+
+  if (null == node || 'object' !== typeof node) {
+    return node
+  }
+
+  const out: Record<string, any> = {}
+  for (const key of Object.keys(node)) {
+    if (OPEN === key) {
+      continue
+    }
+    out[key] = close(node[key])
+  }
+
+  return out
+}
+
+
+// Built rather than written, so the backticks cannot be lost in an edit.
+const OPEN = String.fromCharCode(96) + '$OPEN' + String.fromCharCode(96)
+
+
+export {
+  ValidateFeature
+}

package/project/.sdk/tm/ts/src/utility/MakeOptionsUtility.ts

@@ -1,5 +1,6 @@
 
 import { Context } from '../types'
+import { OPTSPEC } from '../Schema'
 
 
 function makeOptions(ctx: Context) {
@@ -54,67 +55,19 @@ function makeOptions(ctx: Context) {
   let config = ctx.config || {}
   let cfgopts = config.options || {}
 
-  // Standard SDK option values.
-  const optspec = {
-    apikey: '',
-    secret: '',
-    base: 'http://localhost:8000',
-    prefix: '',
-    suffix: '',
-    auth: {
-      prefix: '',
-      basic: false
-    },
-    headers: {
-      '`$CHILD`': '`$STRING`'
-    },
-    allow: {
-      method: 'GET,PUT,POST,PATCH,DELETE,OPTIONS',
-      op: 'create,update,load,list,remove,command,direct,graphql'
-    },
-    entity: {
-      '`$CHILD`': {
-        '`$OPEN`': true,
-        active: false,
-        alias: {}
-      }
-    },
-    feature: {
-      '`$CHILD`': {
-        '`$OPEN`': true,
-        active: false,
-      }
-    },
-    utility: {},
-    // Feature INSTANCES supplied at construction (the station adopt
-    // path): consumed by the constructor's featureAdd loop, so they are
-    // class instances, not data - `$ANY` accepts them verbatim. Without
-    // this entry the seam is dead: the constructor reads
-    // options.extend, but validate rejected the key.
-    extend: '`$ANY`' as any,
-    system: {
-      fetch: undefined as any
-    },
-    test: {
-      active: false,
-      entity: {
-        '`$OPEN`': true,
-      }
-    },
-    clean: {
-      keys: 'key,token,id'
-    },
-    // Server-variable values for a templated base URL (OpenAPI server
-    // variables): `{name}` placeholders in `base` are substituted from
-    // this map at construction. Spec defaults arrive via the generated
-    // Config; user values override them.
-    server: {
-      '`$CHILD`': ''
-    }
-  }
-
-  // JavaScript specific option values.
-  optspec.system.fetch = opts.system?.fetch || global.fetch
+  // THE OPTION SPEC IS GENERATED, NOT WRITTEN HERE.
+  //
+  // `Spec.OPTSPEC` is built from the model: `main.kit.optspec` for the
+  // standard options, plus one entry per feature this target carries, taken
+  // from that feature's own `config.options` / `config.optspec`. Editing this
+  // file to add an option would put it back where it was — one of twenty
+  // hand-maintained copies of a schema nothing cross-checked — so add it to
+  // the model instead and every ported target validates it.
+  //
+  // NOT MUTATED. It is a module-level constant shared by every client this
+  // process constructs; the platform default below is applied to the RESULT,
+  // never to the spec.
+  const optspec = OPTSPEC
 
   // Clone the config side before merging: `config` is a module-level
   // singleton in ts/js, and merge would otherwise use its nested maps as
@@ -124,6 +77,16 @@ function makeOptions(ctx: Context) {
 
   opts = validate(opts, optspec)
 
+  // The platform fetch, supplied AFTER validate rather than as a spec
+  // default. `system.fetch` is declared `$ANY`, which passes a caller's own
+  // fetch through untouched but inserts nothing when the key is absent — and
+  // the spec is shared, so writing the default into it (as this did while the
+  // spec was a per-call literal) would hand one client's fetch to the next.
+  opts.system = opts.system || {}
+  if (null == opts.system.fetch) {
+    opts.system.fetch = global.fetch
+  }
+
   // Restore the suppression the optspec default would otherwise erase.
   if (authSuppressed) {
     opts.auth = null

package/project/.sdk/tm/ts/test/feature/secrets/Secrets.test.ts

@@ -377,16 +377,45 @@ describe('secrets', () => {
     // Before any op, nothing has been resolved.
     assert.equal(sdk.options().apikey, '')
 
-    const name = names[0]
-    const accessor = name.charAt(0).toUpperCase() + name.slice(1)
+    // AN OP THAT EXISTS, not `list` on whichever entity comes first.
+    //
+    // A generated TS entity carries only the ops its model DECLARES, so
+    // `list` is absent from an entity declaring `create` alone — and
+    // calling it throws before the PreSpec hook can run, failing the
+    // assertion below for a reason that has nothing to do with secrets.
+    // univec-sdk hit exactly that: its first entity declares `create`
+    // only, so this test failed on every run while PreSpec worked
+    // perfectly. Any real op exercises the hook; `list` is not special.
+    //
+    // This file is a TEMPLATE, so no project's op names are known here
+    // either. Discover the first callable one rather than assuming.
+    let ran = false
+    for (const one of names) {
+      const acc = one.charAt(0).toUpperCase() + one.slice(1)
+      const ent = (sdk as any)[acc]?.()
+      if (null == ent) {
+        continue
+      }
+      const opname = ['list', 'load', 'create']
+        .find((o: string) => 'function' === typeof (ent as any)[o])
+      if (null == opname) {
+        continue
+      }
+      // The op itself may fail (no seeded data, no live API) — irrelevant
+      // here. What matters is that the awaited PreSpec hook ran and the
+      // credential reached the live options before the spec was built.
+      try {
+        await (ent as any)[opname]({})
+      }
+      catch (_err) { }
+      ran = true
+      break
+    }
 
-    // The op itself may fail (no seeded data, no live API) — irrelevant
-    // here. What matters is that the awaited PreSpec hook ran and the
-    // credential reached the live options before the spec was built.
-    try {
-      await sdk[accessor]().list()
+    if (!ran) {
+      // Entities, but not one callable op between them: no PreSpec path.
+      return
     }
-    catch (_err) { }
 
     assert.equal(sdk.options().apikey, 'ENVKEY02',
       'the entity op did not resolve the secret through PreSpec')

package/project/sdkgen-package.json

@@ -3,7 +3,7 @@
     "package": 1
   },
   "name": "@voxgig/sdkgen",
-  "version": "4.17.5",
+  "version": "4.18.0",
   "provides": {
     "target": [
       "c",
@@ -49,7 +49,8 @@
       "streaming",
       "telemetry",
       "test",
-      "timeout"
+      "timeout",
+      "validate"
     ]
   }
 }

package/src/cmp/AgentGuideContent.ts

@@ -167,6 +167,36 @@ npm run generate
 Note: the \`voxgig-sdkgen\` CLI only *scaffolds* (\`target add\` /
 \`feature add\`). Generation itself runs via \`npm run generate\` (backed by
 \`@voxgig/model\`) — there is no \`generate\` CLI subcommand.
+
+### Two silent failure modes
+
+Generation has two ways of going wrong that **nothing reports**. Neither
+breaks a build or a test, so the only symptom is a tree that disagrees with
+the model — which is easy to commit past.
+
+**\`voxgig-model --no-config\` writes a REDUCED model.** The
+\`.model-config\` build is what registers the generator actions, and an SDK
+project loads \`apidef\` and \`sdkgen\` through exactly that mechanism:
+
+\`\`\`
+sys: model: action: { apidef: load: 'build/apidef.js', sdkgen: load: 'build/sdkgen.js' }
+sys: model: order: action: 'apidef,sdkgen'
+\`\`\`
+
+\`--no-config\` skips it, so those actions never run — and the model build
+still *writes* the model file, now missing whatever they contribute (the
+name case variants, and whole subtrees). A reduced model is a valid model,
+so nothing downstream complains. To inspect the model layer **without side
+effects**, use \`npm run dry-generate\` (\`-y\`, writes nothing). Never
+\`--no-config\` in anything whose output might be committed.
+
+**Regeneration never DELETES.** A file the generator has stopped emitting
+stays in the tree, and \`git status\` is silent because it is committed and
+unchanged. Narrowing a feature's plugin selection, or dropping a target, can
+leave whole modules behind that nothing references and no test covers.
+
+To find either, the target trees must be deleted and regenerated — a
+regeneration in place cannot see stale output at all.
 `
 }
 

package/src/cmp/FeatureDocs.ts

@@ -20,6 +20,13 @@ type FeatureDoc = {
   transport: string
   wraps: boolean
   options: Array<{ name: string, value: string }>
+
+  // Options the feature accepts but does not default: callbacks, injected
+  // clocks, values the caller either supplies or does not. Declared in
+  // `config.optspec` because a DEFAULTS map cannot describe an option that
+  // has no default — which is why the tables built from `config.options`
+  // alone were incomplete, and had to say so.
+  extras: Array<{ name: string, type: string }>
 }
 
 
@@ -33,6 +40,27 @@ function isWrapping(feat: any): boolean {
 }
 
 
+// A struct.validate sentinel as a reader's word for it: '`$FUNCTION`' ->
+// 'function'. A union renders its members, so netsim's latency reads
+// 'number | map'. Anything unrecognised renders verbatim rather than being
+// dropped — a doc table that silently omits an option is the failure this
+// whole path exists to fix.
+function sentinelName(v: any): string {
+  if (Array.isArray(v)) {
+    const members = v.slice(1).map((m: any) => sentinelName(m))
+      .filter((m: string) => 'one' !== m)
+    return 0 === members.length ? 'any' : members.join(' | ')
+  }
+
+  if ('string' !== typeof v) {
+    return 'any'
+  }
+
+  const bare = v.replace(/[`$]/g, '').trim().toLowerCase()
+  return '' === bare ? 'any' : bare
+}
+
+
 function renderValue(v: any): string {
   if (null == v) { return '' }
   if (Array.isArray(v)) { return '[' + v.map((x) => renderValue(x)).join(', ') + ']' }
@@ -63,6 +91,15 @@ function featureDocs(model: any, target?: any): FeatureDoc[] {
         name: k,
         value: renderValue(opts[k]),
       }))
+
+      const extra = (f.config && f.config.optspec) || {}
+      const extras = Object.keys(extra)
+        // A name in both is documented by its DEFAULT; `config.optspec`
+        // only sharpened its type (netsim's `latency`, a number that is
+        // also a { min, max } map), and a reader wants the default.
+        .filter((k) => null == opts[k])
+        .sort()
+        .map((k) => ({ name: k, type: sentinelName(extra[k]) }))
       return {
         name: f.name,
         Name: f.Name || f.name,
@@ -70,6 +107,7 @@ function featureDocs(model: any, target?: any): FeatureDoc[] {
         transport: f.transport || 'none',
         wraps: isWrapping(f),
         options,
+        extras,
       }
     })
     .sort((a: FeatureDoc, b: FeatureDoc) => a.name.localeCompare(b.name))
@@ -100,6 +138,7 @@ function honoursActivationOrder(target: any): boolean {
 export {
   featureDocs,
   renderValue,
+  sentinelName,
   honoursActivationOrder,
 }
 

package/src/cmp/ReadmeRefFeatures.ts

@@ -78,13 +78,14 @@ ${f.title}.
 
 `)
 
-    // THE TABLE IS THE MODEL'S OPTIONS, AND THE MODEL IS INCOMPLETE. Several
-    // features accept runtime-only options the model never declares — cost
-    // reads `actor` and `sink`, audit reads `sink` — so a table derived from
-    // `config.options` cannot list them. Deriving from a second hand-written
-    // list would reintroduce exactly the drift this module exists to avoid,
-    // so the honest fix is upstream: declare them in the feature model. Until
-    // then, say so rather than imply the table is exhaustive.
+    // THE TABLE IS THE MODEL'S OPTIONS, and the model now carries both
+    // halves of them. It used to carry one: `config.options` is a DEFAULTS
+    // map, so the options with no default — cost's `sink`, audit's `sink`,
+    // every injected clock — could not appear in it, and this table had to
+    // admit it was not exhaustive. `config.optspec` declares those with a
+    // type instead of a default, and they are listed below the defaults.
+    // Same declaration the generated option spec validates against, so a
+    // documented option and a validated option cannot be different sets.
 
 
     if (0 < f.options.length) {
@@ -98,18 +99,28 @@ ${f.title}.
       Content(`
 `)
     }
-    else {
+    else if (0 === f.extras.length) {
       Content(`\`active\` only — this feature takes no further options.
 
 `)
     }
 
-    Content(`Options above are those the model carries a default for. A feature may
-also accept callback options — a \`sink\` to receive each record, for
-instance — which have no default and are covered in the full feature
-reference.
+    if (0 < f.extras.length) {
+      Content(`| Option | Type |
+|---|---|
+`)
+      for (const o of f.extras) {
+        Content(`| \`${o.name}\` | ${o.type} |
+`)
+      }
+      Content(`
+These take no default: the feature behaves one way when you supply them and
+another when you do not.
+
+`)
+    }
 
-**Usage**
+    Content(`**Usage**
 
 Set \`feature.${f.name}.active\` to true in the client options${
   0 < f.options.length ?

package/src/helpers/applicability.ts

@@ -105,6 +105,14 @@ function targetFeatures(model: any, target: any): Record<string, any> {
 const TAGS = [
   // A vendored sekreto port lives in this target's feature container.
   'sekreto',
+
+  // The target emits the generated `Schema` module: the model's option spec
+  // and, when a feature asks for them, per-entity struct.validate specs.
+  // `validate` needs this, and the porting order is the reason it is a tag
+  // rather than an assumption — every target vendors a struct with
+  // `validate` in it, but only a target whose Main emits Schema has
+  // anything for that function to check against.
+  'schema',
 ]