@voxgig/create-sdkgen 0.26.0 → 0.28.0

package/src/project/standard/CreateRoot.ts

@@ -16,6 +16,7 @@ import {
 
 
 import { ModelSdk } from './ModelSdk'
+import { migrateToAontu, scaffoldFiles } from './migrate'
 
 
 const GITIGNORE_TOP = `# Local config / secrets
@@ -61,15 +62,16 @@ log/
 `
 
 
-const PROJECT_FILE = 'project.aon'
+const PROJECT_FILE = 'project.aontu'
 const PROJECT_STUB = `# Project overlay — YOURS. The scaffold creates this file once and never
-# overwrites it, unlike every other file it writes.
+# overwrites it.
 #
-# Everything else under model/ is toolchain-derived and is deliberately
-# regenerated so that toolchain fixes propagate. Put anything here that is a
-# decision about THIS project rather than a fact about the API.
+# Most of model/ is toolchain-derived and is deliberately regenerated so that
+# toolchain fixes propagate; this file, the guide, and the target, feature and
+# edition indexes are kept. Put anything here that is a decision about THIS
+# project rather than a fact about the API.
 #
-# Included LAST by sdk.aon, after target/target-index.aon, because a key
+# Included LAST by sdk.aontu, after target/target-index.aontu, because a key
 # under main.kit.target.<t> can only refine a target that has already been
 # defined. Declared earlier, the model build fails with "Cannot unify value:
 # nil with value: string / key ext value was: nil", which names nothing that
@@ -89,27 +91,18 @@ const PROJECT_STUB = `# Project overlay — YOURS. The scaffold creates this fil
 `
 
 
-const GUIDE_FILE = 'guide.aon'
+const GUIDE_FILE = 'guide.aontu'
 const GUIDE_REL = ['model', 'guide', GUIDE_FILE]
 
+const INDEX_KINDS = ['target', 'feature', 'edition']
+const indexFile = (kind: string) => kind + '-index.aontu'
 
-function migrateOverlay(fs: any, dir: string, name: string): void {
-  const next = Path.join(dir, name + '.aon')
-  const prev = Path.join(dir, name + '.aontu')
-  if (fs.existsSync(next) || !fs.existsSync(prev)) {
-    return
-  }
-  try {
-    const src = String(fs.readFileSync(prev))
-      .replace(/(@['"][^'"]+)\.aontu(['"])/g, '$1.aon$2')
-    fs.writeFileSync(next, src)
-    fs.unlinkSync(prev)
-  }
-  catch (_err: any) {
-    // A failed migration must not fail the scaffold: the worst case is the
-    // template being written fresh, which is what would have happened anyway.
-  }
-}
+// Written back from the project rather than from the template.
+const KEPT = [
+  ['model', PROJECT_FILE],
+  GUIDE_REL,
+  ...INDEX_KINDS.map((kind: string) => ['model', kind, indexFile(kind)]),
+].map((rel: string[]) => rel.join('/'))
 
 
 function mergeGuide(existing: string | null, template: string): string {
@@ -168,10 +161,12 @@ const CreateRoot = cmp(function CreateRoot(props: any) {
 
 
     const guideExclude = [spec.sdk_folder, ...GUIDE_REL].join('/')
+    const indexExclude = INDEX_KINDS.map((kind: string) =>
+      [spec.sdk_folder, 'model', kind, indexFile(kind)].join('/'))
 
     Copy({
       from,
-      exclude: [/\.fragment\./, guideExclude, /^\.sdk\/admin\/.*\.sh$/]
+      exclude: [/\.fragment\./, guideExclude, ...indexExclude, /^\.sdk\/admin\/.*\.sh$/]
     })
 
     File({ name: '.gitignore' }, () => {
@@ -183,6 +178,16 @@ const CreateRoot = cmp(function CreateRoot(props: any) {
     spec.def = projdef
 
     Folder({ name: spec.sdk_folder }, () => {
+      // Before anything reads the project's own files, so a project from the
+      // .aon era is read under the names it now has.
+      if (!spec.dryrun) {
+        const sdk = Path.join(folder, spec.sdk_folder)
+        migrateToAontu(fs, sdk, scaffoldFiles(fs, Path.join(from, spec.sdk_folder), KEPT),
+          (file: string, note: string) => ctx$.log.warn({
+            point: 'migrate-aontu', file: Path.join(sdk, file), note: file + ': ' + note,
+          }))
+      }
+
       Folder({ name: 'admin' }, () => {
         const admin = Path.join(from, spec.sdk_folder, 'admin')
         for (const name of fs.readdirSync(admin).filter((name: string) => name.endsWith('.sh')).sort()) {
@@ -214,10 +219,6 @@ const CreateRoot = cmp(function CreateRoot(props: any) {
         const projectPath =
           Path.join(folder, spec.sdk_folder, 'model', PROJECT_FILE)
 
-        // Same hazard, worse symptom: this file carries the release version,
-        // so an ignored project.aontu silently resets every generated manifest
-        // to the sdkgen default 0.0.1 — the exact bug fixed earlier this week.
-        migrateOverlay(fs, Path.dirname(projectPath), 'project')
         const existingProject =
           fs.existsSync(projectPath) ? fs.readFileSync(projectPath, 'utf8') : null
 
@@ -225,6 +226,20 @@ const CreateRoot = cmp(function CreateRoot(props: any) {
           Content(null == existingProject ? PROJECT_STUB : existingProject)
         })
 
+        // `target add`, `feature add` and docgen register their items here, and
+        // docgen bootstraps only once, so a reset index would lose them for good.
+        for (const kind of INDEX_KINDS) {
+          Folder({ name: kind }, () => {
+            const name = indexFile(kind)
+            const indexPath = Path.join(folder, spec.sdk_folder, 'model', kind, name)
+            File({ name }, () => {
+              Content(fs.existsSync(indexPath) ?
+                fs.readFileSync(indexPath, 'utf8') :
+                fs.readFileSync(Path.join(from, spec.sdk_folder, 'model', kind, name), 'utf8'))
+            })
+          })
+        }
+
         // Re-emit the guide the Copy skipped, merged over whatever is already
         // there. On a fresh scaffold there is no existing file and this writes
         // the template unchanged; on a re-scaffold the user's overlay survives.
@@ -232,10 +247,6 @@ const CreateRoot = cmp(function CreateRoot(props: any) {
           const guideTemplate =
             fs.readFileSync(Path.join(from, spec.sdk_folder, ...GUIDE_REL), 'utf8')
           const guidePath = Path.join(folder, spec.sdk_folder, ...GUIDE_REL)
-
-          // Before the merge looks for it — otherwise a project whose guide is
-          // still named .aontu reads as having no overlay at all.
-          migrateOverlay(fs, Path.dirname(guidePath), 'guide')
           const existingGuide =
             fs.existsSync(guidePath) ? fs.readFileSync(guidePath, 'utf8') : null
 

package/src/project/standard/ModelSdk.ts

@@ -17,9 +17,9 @@ const ModelSdk = cmp(function ModelSdk(props: any) {
   const from =
     Path.resolve(Path.join(__dirname, '..', '..', '..', 'project', 'standard'))
 
-  File({ name: 'sdk.aon' }, () => {
+  File({ name: 'sdk.aontu' }, () => {
     Fragment({
-      from: Path.join(from, spec.sdk_folder, 'model', 'sdk.fragment.aon'),
+      from: Path.join(from, spec.sdk_folder, 'model', 'sdk.fragment.aontu'),
       replace: {
         ProjectName: spec.name,
         NAME: spec.name,

package/src/project/standard/migrate.ts

@@ -0,0 +1,172 @@
+import Path from 'node:path'
+
+
+const TREES = ['model', 'test']
+
+const LEGACY = '.aon'
+const CURRENT = '.aontu'
+
+const TOOLCHAIN_RE = /^@voxgig\//
+const APIDEF_WRITES_RE = /^model\/guide\/[^/]*base-guide\.aontu$/
+
+// Comments and strings are matched whole, so an include is only ever an `@`
+// outside both: a `.aon` named in a comment or held as data is not rewritten.
+const TOKEN_RE =
+  /#[^\n]*|@([ \t]*)(["'`])([^"'`\n]*)\2|"(?:\\.|[^"\\\n])*"|'(?:\\.|[^'\\\n])*'|`(?:\\.|[^`\\])*`/g
+
+
+type Warn = (file: string, note: string) => void
+
+type Scaffold = {
+  kept: Set<string>
+  replaced: Set<string>
+}
+
+
+function rewriteIncludes(src: string, rename: (path: string) => boolean): string {
+  return src.replace(TOKEN_RE, (token: string, space: string, quote: string, path: string) =>
+    null != quote && path.endsWith(LEGACY) && rename(path) ?
+      '@' + space + quote + path.slice(0, -LEGACY.length) + CURRENT + quote : token)
+}
+
+
+function legacyIncludes(src: string): string[] {
+  return [...src.matchAll(TOKEN_RE)]
+    .filter((m) => null != m[2] && m[3].endsWith(LEGACY))
+    .map((m) => m[3])
+}
+
+
+function modelFiles(fs: any, sdk: string, ext: string): string[] {
+  const found: string[] = []
+  const walk = (rel: string) => {
+    let entries: any[]
+    try {
+      entries = fs.readdirSync(Path.join(sdk, rel), { withFileTypes: true })
+    }
+    catch (_err: any) {
+      return
+    }
+    for (const entry of entries) {
+      const child = rel + '/' + entry.name
+      if (entry.isDirectory()) {
+        walk(child)
+      }
+      else if (entry.isFile() && entry.name.endsWith(ext)) {
+        found.push(child)
+      }
+    }
+  }
+  TREES.forEach(walk)
+  return found.sort()
+}
+
+
+function scaffoldFiles(fs: any, templateSdk: string, kept: string[]): Scaffold {
+  const keep = new Set(kept)
+  const written = modelFiles(fs, templateSdk, CURRENT)
+    .map((rel: string) => rel.replace('.fragment.', '.'))
+  return { kept: keep, replaced: new Set(written.filter((rel) => !keep.has(rel))) }
+}
+
+
+// A failed step must not fail the scaffold: the file keeps the name it had.
+function attempt(step: () => void): void {
+  try {
+    step()
+  }
+  catch (_err: any) {
+  }
+}
+
+
+// Written beside the target and renamed over it, so a write that fails part
+// way leaves the file it would have replaced whole.
+function writeWhole(fs: any, path: string, data: any): void {
+  const tmp = path + '.migrating'
+  try {
+    fs.writeFileSync(tmp, data)
+    fs.renameSync(tmp, path)
+  }
+  catch (err: any) {
+    attempt(() => fs.unlinkSync(tmp))
+    throw err
+  }
+}
+
+
+function migrateFile(fs: any, prev: string, next: string): void {
+  if (fs.existsSync(next) || !fs.existsSync(prev)) {
+    return
+  }
+  attempt(() => {
+    writeWhole(fs, next, fs.readFileSync(prev))
+    fs.unlinkSync(prev)
+  })
+}
+
+
+function includeRenamer(fs: any, sdk: string, scaffold: Scaffold, from: string) {
+  const dir = Path.dirname(Path.join(sdk, from))
+  return (path: string): boolean => {
+    if (TOOLCHAIN_RE.test(path)) {
+      return true
+    }
+    if (path.startsWith('@')) {
+      return false
+    }
+    const twin = Path.resolve(dir, path.slice(0, -LEGACY.length) + CURRENT)
+    const rel = Path.relative(sdk, twin).split(Path.sep).join('/')
+    return fs.existsSync(twin) || scaffold.kept.has(rel) || scaffold.replaced.has(rel) ||
+      APIDEF_WRITES_RE.test(rel)
+  }
+}
+
+
+// Renames first and rewrites after, so an include changes only once the file
+// it names is on disk as .aontu.
+function migrateToAontu(fs: any, sdk: string, scaffold: Scaffold, warn: Warn): void {
+  const legacy = modelFiles(fs, sdk, LEGACY)
+  const twin = (rel: string) => rel.slice(0, -LEGACY.length) + CURRENT
+  const abs = (rel: string) => Path.join(sdk, rel)
+
+  for (const rel of legacy.filter((rel) => !scaffold.replaced.has(twin(rel)))) {
+    migrateFile(fs, abs(rel), abs(twin(rel)))
+  }
+
+  const sources = modelFiles(fs, sdk, CURRENT).filter((rel) => !scaffold.replaced.has(rel))
+  for (const rel of sources) {
+    attempt(() => {
+      const src = String(fs.readFileSync(abs(rel)))
+      const out = rewriteIncludes(src, includeRenamer(fs, sdk, scaffold, rel))
+      if (out !== src) {
+        writeWhole(fs, abs(rel), out)
+      }
+    })
+  }
+
+  for (const rel of legacy.filter((rel) => scaffold.replaced.has(twin(rel)))) {
+    attempt(() => fs.unlinkSync(abs(rel)))
+  }
+
+  for (const rel of modelFiles(fs, sdk, LEGACY)) {
+    warn(rel, fs.existsSync(abs(twin(rel))) ?
+      'left in place: ' + twin(rel) + ' exists, and is the file aontu reads' :
+      'could not be migrated to ' + twin(rel))
+  }
+  for (const rel of sources) {
+    attempt(() => {
+      for (const path of legacyIncludes(String(fs.readFileSync(abs(rel))))) {
+        warn(rel, 'includes ' + path + ', which aontu refuses: rename that file ' +
+          'to .aontu and update the include')
+      }
+    })
+  }
+}
+
+
+export {
+  migrateToAontu,
+  rewriteIncludes,
+  scaffoldFiles,
+}

package/project/standard/.sdk/model/guide/guide.aon

@@ -1,8 +0,0 @@
-# Guide
-
-@"@voxgig/apidef/model/guide.aon"
-
-@"./base-guide.aon"
-
-
-

package/project/standard/.sdk/model/sdk.fragment.aon

@@ -1,39 +0,0 @@
-
-# Model definitions.
-@"@voxgig/apidef/model/apidef.aon"
-@"@voxgig/sdkgen/model/sdkgen.aon"
-
-
-name: 'NAME'
-origin: 'voxgig-sdk'
-def: 'DEF'
-
-
-# API Information
-@"api/api-info.aon"
-
-
-# SDK Configuration.
-@"./config.aon"
-
-
-# SDK Entities
-@"entity/entity-index.aon"
-
-
-# SDK Features.
-@"feature/feature-index.aon"
-
-
-# SDK Targets.
-@"target/target-index.aon"
-
-
-# Operation workflows.
-@"flow/flow-index.aon"
-
-
-@"./project.aon"
-
-# Documentation editions. Settings belong in project.aon.
-@"edition/edition-index.aon"

package/project/standard/.sdk/test/.model-config/model-config.aon

@@ -1,4 +0,0 @@
-
-@"@voxgig/model/model/.model-config/model-config.aon"
-
-sys: model: builders: {}

package/project/standard/.sdk/test/primary/primary-test-index.aon

@@ -1,30 +0,0 @@
-primary: clean: @"./clean.aon"
-primary: done: @"./done.aon"
-primary: featureAdd: @"./featureAdd.aon"
-primary: featureHook: @"./featureHook.aon"
-primary: featureInit: @"./featureInit.aon"
-primary: fetcher: @"./fetcher.aon"
-primary: makeContext: @"./makeContext.aon"
-primary: makeError: @"./makeError.aon"
-primary: makeFetchDef: @"./makeFetchDef.aon"
-primary: makeOptions: @"./makeOptions.aon"
-primary: makeRequest: @"./makeRequest.aon"
-primary: makeResponse: @"./makeResponse.aon"
-primary: makeResult: @"./makeResult.aon"
-primary: makeSpec: @"./makeSpec.aon"
-primary: makePoint: @"./makePoint.aon"
-primary: makeUrl: @"./makeUrl.aon"
-primary: operator: @"./operator.aon"
-primary: param: @"./param.aon"
-primary: prepareAuth: @"./prepareAuth.aon"
-primary: prepareBody: @"./prepareBody.aon"
-primary: prepareHeaders: @"./prepareHeaders.aon"
-primary: prepareMethod: @"./prepareMethod.aon"
-primary: prepareParams: @"./prepareParams.aon"
-primary: preparePath: @"./preparePath.aon"
-primary: prepareQuery: @"./prepareQuery.aon"
-primary: resultBasic: @"./resultBasic.aon"
-primary: resultBody: @"./resultBody.aon"
-primary: resultHeaders: @"./resultHeaders.aon"
-primary: transformRequest: @"./transformRequest.aon"
-primary: transformResponse: @"./transformResponse.aon"

package/project/standard/.sdk/test/struct/test.aon

@@ -1,26 +0,0 @@
-
-struct: &: {
-  name: key()
-  set: []
-}
-
-struct: minor: @"./minor.aon"
-struct: getpath: @"./getpath.aon"
-struct: inject: @"./inject.aon"
-struct: merge: @"./merge.aon"
-struct: transform: @"./transform.aon"
-struct: walk: @"./walk.aon"
-struct: validate: @"./validate.aon"
-struct: select: @"./select.aon"
-struct: nullsem: @"./nullsem.aon"
-
-primary: check: {
-  DEF: {
-    client:a:test:options:foo:1
-  }
-  basic: set: [
-    { ctx:meta:bar:BAR0, out:zed:ZED_BAR0 }
-    { ctx:meta:bar:BAR1, client:a, out:zed:ZED1_BAR1 }
-  ]
-}
-

package/project/standard/.sdk/test/test.aon

@@ -1,4 +0,0 @@
-
-@"struct/test.aon"
-@"primary/primary-test-index.aon"
-@"feature/feature-test-index.aon"