@voxgig/sdkgen 3.4.2 → 3.4.5

package/project/.sdk/src/cmp/go/Config_go.ts

@@ -10,6 +10,7 @@ import {
   cmp,
   each,
   isAuthActive,
+  isConfigData,
   resolveAuthPrefix,
   serverVariables,
 } from '@voxgig/sdkgen'
@@ -63,8 +64,131 @@ const Config = cmp(async function Config(props: any) {
     : ''
 
   // Config is now in core/ package
+  // The same config as an OBJECT. Built regardless of which representation
+  // wins, because the JSON is what the threshold is measured on - the emitted
+  // source size varies by language, the model does not.
+  const entityDefs: any = {}
+  each(entity, (e: any) => {
+    entityDefs[e.name] = clean({
+      fields: e.fields,
+      name: e.name,
+      op: e.op,
+      relations: e.relations,
+    }, true)
+  })
+
+  const featureDefs: any = {}
+  each(feature, (f: any) => {
+    featureDefs[f.name] = f.config || {}
+  })
+
+  const entityStubs: any = {}
+  each(entity, (e: any) => {
+    entityStubs[e.name] = {}
+  })
+
+  const optionsDef: any = { base: baseUrl }
+  if (0 < svars.length) {
+    optionsDef.server = svars.reduce(
+      (a: any, v: any) => (a[v.name] = v.dflt, a), {})
+  }
+  if (authActive) {
+    optionsDef.auth = { prefix: authPrefix }
+  }
+  optionsDef.headers = headers
+  optionsDef.entity = entityStubs
+
+  // Key order here is the order `each` produced, which is sorted, so the
+  // emitted JSON is byte-stable across runs like the literal it replaces.
+  const configDef = {
+    main: { name: model.const.Name },
+    feature: featureDefs,
+    options: optionsDef,
+    entity: entityDefs,
+  }
+  const configJson = JSON.stringify(configDef)
+
+  // `auto` decides by size; 'data'/'literal' pin it for this SDK.
+  let configReprSetting = 'auto'
+  try {
+    configReprSetting = getModelPath(model, `main.${KIT}.config.repr`) || 'auto'
+  } catch (_e) { }
+  const asData = isConfigData(configJson, configReprSetting)
+
   File({ name: 'config.' + target.ext }, () => {
 
+    // ABOVE THE THRESHOLD: emit the model as DATA.
+    //
+    // A composite literal makes the compiler walk every node of the model;
+    // a string constant is one token. On the real gitlab model that is 30.8 s
+    // and 2.49 GB of compiler memory versus 0.34 s and 0.06 GB, and a binary
+    // 2.1x smaller. MakeConfig still returns the same map, so nothing
+    // downstream can tell which representation it got.
+    //
+    // JSON.stringify output is a valid Go interpreted string literal: JSON
+    // escapes are a subset of Go's, and Go source is UTF-8 so non-ASCII needs
+    // no escaping. A raw (backtick) literal could NOT be used - the model
+    // contains backticks in values like `$STRING`.
+    if (asData) {
+      Content(`package core
+
+import (
+	"encoding/json"
+	"math"
+	"sync"
+)
+
+// The API model, emitted as data rather than as a composite literal: see
+// sdkgen rung L1. Parsed by MakeConfig, and parsed once by SharedConfig.
+const configJSON = ${JSON.stringify(configJson)}
+
+// json.Unmarshal decodes EVERY JSON number as float64, but the literal
+// representation emits an integer token as an untyped constant that lands in
+// map[string]any as an int. MakeConfig is public API and consumers type-assert
+// against it, so the two representations must not disagree about the type of
+// a whole number just because the model crossed a size threshold.
+//
+// Whole values become int; anything fractional, or too large to be exact in a
+// float64, stays float64 - which is what the literal branch does too.
+func configNormalise(val any) any {
+	switch v := val.(type) {
+	case map[string]any:
+		for k, c := range v {
+			v[k] = configNormalise(c)
+		}
+		return v
+	case []any:
+		for i, c := range v {
+			v[i] = configNormalise(c)
+		}
+		return v
+	case float64:
+		if v == math.Trunc(v) && math.Abs(v) <= 1<<53 {
+			return int(v)
+		}
+		return v
+	}
+	return val
+}
+
+// MakeConfig parses a fresh, fully materialised config map. Every call
+// re-parses, so prefer SharedConfig unless you need a private copy you
+// intend to mutate.
+func MakeConfig() map[string]any {
+	var out map[string]any
+	if err := json.Unmarshal([]byte(configJSON), &out); err != nil {
+		// Unreachable: the constant is generated by json.Marshal's
+		// counterpart and never edited by hand. Panic rather than return a
+		// silently empty config, which would fail far from the cause.
+		panic("${model.const.Name}: embedded config is not valid JSON: " + err.Error())
+	}
+	out, _ = configNormalise(out).(map[string]any)
+	return out
+}
+`)
+    }
+    else {
+
     Content(`package core
 
 import (
@@ -113,7 +237,10 @@ ${serverBlock}${authBlock}			"headers": ${formatGoMap(headers, 3)},
       }, true), a), {}), 2)},
 	}
 }
+`)
+    }
 
+    Content(`
 var (
 	sharedConfigOnce sync.Once
 	sharedConfigVal  map[string]any

package/project/.sdk/src/cmp/ts/Config_ts.ts

@@ -11,6 +11,7 @@ import {
   each,
   indent,
   isAuthActive,
+  isConfigData,
   resolveAuthPrefix,
   serverVariables,
 } from '@voxgig/sdkgen'
@@ -62,13 +63,100 @@ const Config = cmp(async function Config(props: any) {
     svars.map((v: any) => `      ${JSON.stringify(v.name)}: ${JSON.stringify(v.dflt)},\n`).join('') +
     '    },\n\n    '
 
+  // Read the base URL here rather than leaving it to a `$$...$$` stdrep
+  // placeholder in the fragment. stdrep can only substitute a path the model
+  // actually has: a model with no `info.servers` left the placeholder itself in
+  // the generated source, so `options.base` came out as the literal string
+  // '$main.kit.info.servers.0.url$'. Reading it explicitly yields '' in that
+  // case, which is what every other target already emits, and is identical to
+  // the old output whenever the model does define a server.
+  let baseUrl = ''
+  try {
+    baseUrl = getModelPath(model, `main.${KIT}.info.servers.0.url`)
+  } catch (_e) { }
+
+  // The same config as an OBJECT, so it can be measured and, above the
+  // threshold, emitted as data instead of as class field literals. Mirrors the
+  // literal branch below exactly - feature config is NOT cleaned here, because
+  // it is not cleaned there either.
+  const entityDefs: any = {}
+  each(entity, (e: any) => {
+    entityDefs[e.name] = clean({
+      fields: e.fields,
+      name: e.name,
+      op: e.op,
+      relations: e.relations,
+    }, true)
+  })
+
+  const featureDefs: any = {}
+  each(feature, (f: any) => {
+    featureDefs[f.name] = f.config
+  })
+
+  const entityStubs: any = {}
+  each(entity, (e: any) => {
+    entityStubs[e.name] = {}
+  })
+
+  const optionsDef: any = { base: baseUrl }
+  if (0 < svars.length) {
+    optionsDef.server = svars.reduce(
+      (a: any, v: any) => (a[v.name] = v.dflt, a), {})
+  }
+  if (authActive) {
+    optionsDef.auth = { prefix: authPrefix }
+  }
+  optionsDef.headers = headers
+  optionsDef.entity = entityStubs
+
+  const configDef = {
+    main: { name: model.const.Name },
+    feature: featureDefs,
+    options: optionsDef,
+    entity: entityDefs,
+  }
+  const configJson = JSON.stringify(configDef)
+
+  // `auto` decides by size; 'data'/'literal' pin it for this SDK.
+  let configReprSetting = 'auto'
+  try {
+    configReprSetting = getModelPath(model, `main.${KIT}.config.repr`) || 'auto'
+  } catch (_e) { }
+
   File({ name: 'Config.' + target.ext }, () => {
 
+    if (isConfigData(configJson, configReprSetting)) {
+      Fragment({
+        from: ff + 'Config.data.fragment.ts',
+
+        replace: {
+
+          '// #ImportFeatures': () => each(feature, (f: any) => {
+            Line(`import { ${nom(f, 'Name')}Feature } from ` +
+              `'./feature/${f.name}/${nom(f, 'Name')}Feature'`)
+          }),
+
+          '// #FeatureClasses': () => each(feature, (f: any) => {
+            Line(` ${f.name}: ${nom(f, 'Name')}Feature,`)
+          }),
+
+          // A JS string literal, so the JSON survives verbatim. JSON.stringify
+          // escapes the quotes and backslashes the model contains (values like
+          // `$STRING` carry backticks, which a template literal could not).
+          "'CONFIGJSON'": JSON.stringify(configJson),
+        }
+      })
+      return
+    }
+
     Fragment({
       from: ff + 'Config.fragment.ts',
 
       replace: {
 
+        "'BASEURL'": JSON.stringify(baseUrl),
+
         "'SERVERBLOCK'": serverBlock,
 
         "'AUTHBLOCK'": authBlock,

package/project/.sdk/src/cmp/ts/fragment/Config.data.fragment.ts

@@ -0,0 +1,50 @@
+
+import { BaseFeature } from './feature/base/BaseFeature'
+// #ImportFeatures
+
+
+const FEATURE_CLASS: Record<string, typeof BaseFeature> = {
+  // #FeatureClasses
+}
+
+
+// THE API MODEL, EMBEDDED AS DATA (sdkgen rung L1).
+//
+// The literal form of this file declares the whole model as nested object
+// literals on the class. For a large API that is megabytes of source that the
+// TypeScript compiler must parse and infer a type for on every build, and that
+// the JavaScript engine must build node by node on every module load.
+//
+// As a single string constant it is one token to the compiler, and V8's JSON
+// parser builds the object far faster than the equivalent literal - JSON.parse
+// is the well-worn trick for exactly this shape of data.
+//
+// Emitted only above a size threshold, or when `main.kit.config.repr` pins it:
+// for a small model the literal is smaller, loads no slower, and is far easier
+// to read when debugging.
+const CONFIG_DATA = 'CONFIGJSON'
+
+
+class Config {
+
+  makeFeature(this: any, fn: string) {
+    const fc = FEATURE_CLASS[fn]
+    const fi = new fc()
+    // TODO: errors etc
+    return fi
+  }
+
+}
+
+
+// Parsed ONCE, at module load, exactly like the literal form was built once.
+//
+// The parsed data is assigned onto an instance rather than replacing it, so
+// `config.main` / `.feature` / `.options` / `.entity` read exactly as they did
+// when they were field initialisers, and `makeFeature` stays where it was on
+// the prototype. Callers cannot tell which representation they were given.
+const config: any = Object.assign(new Config(), JSON.parse(CONFIG_DATA))
+
+export {
+  config
+}

package/project/.sdk/src/cmp/ts/fragment/Config.fragment.ts

@@ -29,7 +29,7 @@ class Config {
 
 
   options = {
-    base: '$$main.kit.info.servers.0.url$$',
+    base: 'BASEURL',
 
     'SERVERBLOCK''AUTHBLOCK'headers: 'HEADERS',
 

package/src/action/target.ts

@@ -207,6 +207,22 @@ const TargetRoot = cmp(function TargetRoot(props: any) {
       // language puts feature source somewhere different.
       const trim = trimFeatures(ctx$, tfolder, torigname, tname, features)
 
+      // Copy only ADDS and overwrites — it never removes, and it never even
+      // looks at a file the trim excludes. So a template that this SDK should
+      // NOT have lived on at whatever revision it was first copied at, and
+      // kept being generated from.
+      //
+      // That is how 30 cedar repos ended up with tm/go/test/feature_test.go
+      // still declaring the nine fh* harness helpers after upstream moved them
+      // into feature_harness_test.go: feature_test.go is feature-source, so it
+      // is trimmed for an SDK without those features, so Copy skipped it, so
+      // the pre-move revision survived every `target add go` those repos ever
+      // ran. Generation then emitted it alongside the new harness and the go
+      // package failed to compile — "fhHasFeature redeclared in this block".
+      //
+      // The invariant this restores: tm/<target> == source tree MINUS trim.
+      pruneStaleTemplates(ctx$, tfolder + '/tm/' + torigname, 'tm/' + tname, trim)
+
       Folder({ name: 'tm/' + tname }, () => {
         Copy({
           from: tfolder + '/tm/' + torigname,
@@ -234,6 +250,105 @@ const TargetRoot = cmp(function TargetRoot(props: any) {
 // read. Trimming a target whose templates are not ready for it produces a
 // project that does not build, so an unreadable declaration must fail safe
 // rather than fail tidy.
+// Bring the consumer's `tm/<target>` back to the invariant that Copy alone
+// cannot maintain: it must contain EXACTLY the source tree minus the files
+// this SDK's feature set trims away.
+//
+// Copy adds and overwrites. It does not remove, and it does not touch an
+// excluded file at all — so both of these persist silently forever:
+//
+//   - a template the toolchain has RETIRED (absent from the source tree);
+//   - a template this SDK should not have (present in source, but trimmed),
+//     frozen at whatever revision it was first copied at.
+//
+// The second is the one that bit: tm/go/test/feature_test.go is feature
+// source, so it is trimmed for an SDK without those features, so it was never
+// refreshed after upstream moved the fh* harness helpers out of it.
+//
+// tm/ is toolchain-owned — the scaffold rewrites it on every add-target, and
+// model/guide/guide.aontu is the one file a user owns (merged separately by
+// create-sdkgen) — so removing what the toolchain says should not be there is
+// consistent with how the rest of that tree is already treated.
+function pruneStaleTemplates(
+  ctx$: any,
+  fromDir: string,
+  toRel: string,
+  trim: RegExp[],
+) {
+  const { log } = ctx$
+  const fs = ctx$.fs()
+  const folder = ctx$.folder ?? '.'
+  const destDir = Path.join(folder, toRel)
+
+  const listRel = (root: string): string[] => {
+    const out: string[] = []
+    const walk = (dir: string, rel: string) => {
+      let entries: any[]
+      try {
+        entries = fs.readdirSync(dir, { withFileTypes: true })
+      }
+      catch (e: any) {
+        return
+      }
+      for (const ent of entries) {
+        const child = Path.join(dir, ent.name)
+        const childRel = '' === rel ? ent.name : rel + '/' + ent.name
+        if (ent.isDirectory()) {
+          walk(child, childRel)
+        }
+        else {
+          out.push(childRel)
+        }
+      }
+    }
+    walk(root, '')
+    return out
+  }
+
+  const sourceFiles = listRel(fromDir)
+
+  // An unreadable source tree must not be read as "everything is stale" — that
+  // would empty the destination.
+  if (0 === sourceFiles.length) {
+    return
+  }
+
+  // What SHOULD be present: source, minus anything the trim excludes. The trim
+  // patterns are matched against the source-relative path, the same way Copy
+  // applies them.
+  const trimmed = (rel: string) => trim.some((re) => re.test(rel))
+  const want = new Set(sourceFiles.filter((rel) => !trimmed(rel)))
+
+  const stale = listRel(destDir).filter((rel) => !want.has(rel))
+  if (0 === stale.length) {
+    return
+  }
+
+  const removed: string[] = []
+  for (const rel of stale) {
+    try {
+      fs.unlinkSync(Path.join(destDir, rel))
+      removed.push(rel)
+    }
+    catch (e: any) {
+      log.warn({
+        point: 'target-template-prune', target: toRel, file: rel,
+        note: 'could not remove stale template ' + rel + ': ' + e.message
+      })
+    }
+  }
+
+  if (0 < removed.length) {
+    log.info({
+      point: 'target-template-prune', target: toRel, count: removed.length,
+      files: removed,
+      note: toRel + ': removed ' + removed.length +
+        ' stale template(s) the toolchain no longer provides for this SDK'
+    })
+  }
+}
+
+
 function trimFeatures(
   ctx$: any,
   tfolder: string,

package/src/cmp/ReadmeTop.ts

@@ -577,7 +577,16 @@ The OpenAPI spec(s) this SDK was generated from are kept in the
 
 `)
     if (upstreamUrl) {
-      Content(`- Upstream API: [${upstreamUrl}](${upstreamUrl})
+      // A per-tenant server URL carries an OpenAPI server variable —
+      // `https://{instance}.dreamapply.com/api` — and is the right thing to
+      // SHOW, because it tells the reader the host is theirs to fill in. It is
+      // not a thing to LINK: the braces are not a resolvable address, so a
+      // markdown link renders as clickable and dead. Show it as code instead.
+      const templated = /[{}]/.test(upstreamUrl)
+      Content(templated
+        ? `- Upstream API: \`${upstreamUrl}\`
+`
+        : `- Upstream API: [${upstreamUrl}](${upstreamUrl})
 `)
     }
     if (docsUrl && docsUrl !== upstreamUrl) {

package/src/sdkgen.ts

@@ -19,7 +19,8 @@ import type {
   ActionResult,
 } from './types'
 
-import { SdkGenError, requirePath, isAuthActive, resolveAuthPrefix } from './utility'
+import { SdkGenError, requirePath, isAuthActive, resolveAuthPrefix,
+  CONFIG_DATA_THRESHOLD, CONFIG_REPR_VALUES, isConfigData, configRepr } from './utility'
 
 import { Main } from './cmp/Main'
 import { ExternalTarget } from './cmp/ExternalTarget'
@@ -776,6 +777,10 @@ export {
   requirePath,
   isAuthActive,
   resolveAuthPrefix,
+  CONFIG_DATA_THRESHOLD,
+  CONFIG_REPR_VALUES,
+  isConfigData,
+  configRepr,
 
   // Scaffold components need this to fail a generation with an actionable
   // message rather than a bare Error (py-data guards on its sibling `py`).

package/src/utility.ts

@@ -96,4 +96,73 @@ export {
   isAuthActive,
   resolveAuthPrefix,
   SdkGenError,
+  CONFIG_DATA_THRESHOLD,
+  CONFIG_REPR_VALUES,
+  isConfigData,
+  configRepr,
+}
+
+
+// CONFIG REPRESENTATION (design rung L1, threshold from design Q7).
+//
+// Above a size threshold the API model is emitted as DATA - a JSON string
+// constant parsed once - rather than as a composite literal. Below it the
+// literal stays, because for a small model the literal is smaller, simpler,
+// faster to load and far easier to debug, and a symbol table would be pure
+// complexity.
+//
+// The threshold is on the JSON, not the emitted source, because the emitted
+// source size varies by language while the model does not. It is measured in
+// UTF-8 BYTES rather than string length: `.length` counts UTF-16 code units,
+// so a CJK-heavy model would read as roughly a third of its real size and
+// stay on the expensive literal path well past the point where it hurts.
+//
+// Measured on the real gitlab model (923.5 KB of JSON), Go, cold cache,
+// recompiling only the config package:
+//
+//                       composite literal   JSON string constant
+//   compile+link wall        30.80 s              0.34 s     91x faster
+//   peak compiler RSS         2.49 GB             0.06 GB    39x less
+//   binary                    7.44 MB             3.51 MB    2.1x smaller
+//
+// The reader side is unchanged either way: make_config returns the same map,
+// so nothing downstream can tell which representation it got.
+const CONFIG_DATA_THRESHOLD = 256 * 1024
+
+
+// Should this model be emitted as data rather than as a literal?
+//
+// `repr` is the per-SDK override from `main.kit.config.repr`: 'auto' (the
+// default) decides by size, 'data' and 'literal' pin it. The override is what
+// lets a small fixture exercise the data path - by size alone no test model
+// comes near the threshold, so the branch every large SDK depends on would
+// never be generated, compiled or run in CI.
+const CONFIG_REPR_VALUES = ['auto', 'data', 'literal']
+
+function isConfigData(configJson: string, repr?: string): boolean {
+  // An unknown value is REJECTED, not ignored. The aontu declaration
+  // documents the closed set but does not enforce it here, and silently
+  // treating `repr: 'date'` as `auto` would quietly restore the compile cost
+  // this exists to remove - the failure mode being a slow build nobody
+  // connects to a typo.
+  if (null != repr && '' !== repr && !CONFIG_REPR_VALUES.includes(repr)) {
+    throw new SdkGenError(
+      'sdkgen: main.kit.config.repr must be one of ' +
+      CONFIG_REPR_VALUES.join(', ') + ' (got: ' + repr + ')', {})
+  }
+  if ('data' === repr) {
+    return true
+  }
+  if ('literal' === repr) {
+    return false
+  }
+  return CONFIG_DATA_THRESHOLD < Buffer.byteLength(configJson, 'utf8')
+}
+
+
+// The chosen representation, as a word - for generation logs and for the
+// per-SDK reporting the fleet regen needs, so a model crossing the threshold
+// is visible rather than showing up as an unexplained whole-file diff.
+function configRepr(configJson: string, repr?: string): string {
+  return isConfigData(configJson, repr) ? 'data' : 'literal'
 }