@voxgig/model 10.0.1 → 10.1.0

package/src/build.ts

@@ -0,0 +1,254 @@
+/* Copyright © 2021-2024 Voxgig Ltd, MIT License. */
+
+
+import { Aontu } from 'aontu'
+
+
+import type {
+  Build,
+  BuildResult,
+  BuildContext,
+  BuildSpec,
+  RunSpec,
+  Log,
+  ProducerDef
+} from './types'
+
+
+class BuildImpl implements Build {
+  id
+  base
+  path
+  opts: any
+  pdef: ProducerDef[]
+  spec: BuildSpec
+  model: any
+  use = {}
+  errs: any[] = []
+  ctx: BuildContext
+  log: Log
+  fs: any
+  dryrun: boolean
+  args: any
+  aontu: Aontu
+  deps: any
+
+  // Signature of the last successful generate: path -> mtimeMs (-1 = missing).
+  // When every tracked file still matches, resolveModel() reuses this.model.
+  cacheSig: Map<string, number> | null = null
+
+
+  constructor(spec: BuildSpec, log: Log) {
+    this.id = String(Math.random()).substring(3, 9)
+    this.log = log
+
+    this.spec = spec
+
+    this.dryrun = !!spec.dryrun
+    this.args = spec.buildargs
+    this.fs = spec.fs
+    this.base = null == spec.base ? '' : spec.base
+    this.path = null == spec.path ? '' : spec.path
+    this.opts = {}
+    this.ctx = { step: 'pre', watch: false, state: {} }
+
+    if (null != spec.base) {
+      this.opts.base = spec.base
+      this.opts.path = spec.path
+    }
+
+    if (null != spec.require) {
+      this.opts.require = spec.require
+    }
+
+    this.pdef = spec.res || []
+
+    Object.assign(this.use, spec.use || {})
+
+    this.deps = {}
+    this.aontu = new Aontu()
+
+    // Aontu comments are `#` only. The npm engine's jsonic parser also
+    // enables `//` and `/* */` by default, which the Go engine does not
+    // support — disable them so both implementations reject the same sources.
+    this.aontu.lang.jsonic.options({
+      comment: { def: { slash: null, multi: null } }
+    })
+  }
+
+
+  async run(rspec: RunSpec): Promise<BuildResult> {
+    let hasErr = false
+    let runlog = []
+
+    // Reset per-run error state. The BuildImpl is reused across watch
+    // rebuilds, so without this a single failure would stick to every
+    // later build. Reassign (don't clear in place) so a previously
+    // returned BuildResult keeps its own errors.
+    this.errs = []
+
+    this.ctx = { step: 'pre', state: {}, watch: rspec.watch }
+    const plog: any[] = []
+
+    if (!hasErr) {
+      runlog.push('model:initial')
+      hasErr = await this.resolveModel()
+    }
+
+    let forceReload = false
+
+    if (!hasErr) {
+      for (let producer of this.pdef) {
+        try {
+          runlog.push('producer:pre:' + producer.build.name)
+          let pr = await producer.build(this, this.ctx)
+          forceReload = forceReload || pr.reload
+          plog.push(pr)
+          if (!pr.ok) {
+            hasErr = true
+            break
+          }
+        }
+        catch (err: any) {
+          hasErr = true
+          this.errs.push(err)
+          break
+        }
+      }
+    }
+
+    // Only reload when a pre-producer actually modified model sources
+    // (signalled via pr.reload). Previously this always ran on success.
+    const reload = forceReload && !hasErr
+
+    if (reload) {
+      runlog.push('model:full')
+      hasErr = await this.resolveModel()
+    }
+
+    if (!hasErr) {
+      this.ctx.step = 'post'
+
+      for (let producer of this.pdef) {
+        try {
+          runlog.push('producer:post:' + producer.build.name)
+          let pr = await producer.build(this, this.ctx)
+          plog.push(pr)
+          if (!pr.ok) {
+            hasErr = true
+            break
+          }
+        }
+        catch (err: any) {
+          hasErr = true
+          this.errs.push(err)
+          break
+        }
+      }
+
+    }
+
+    const br: BuildResult =
+    {
+      // TODO: remove need for this
+      build: () => this,
+
+      ok: !hasErr,
+      producers: plog,
+      errs: this.errs,
+      runlog
+    }
+
+    return br
+  }
+
+
+  async resolveModel() {
+    if (this.model && this.cacheSig && this.cacheHit()) {
+      return false
+    }
+
+    let hasErr = false
+
+    let src: string = ''
+    if (!hasErr) {
+      try {
+        src = this.fs.readFileSync(this.path, 'utf8')
+      }
+      catch (err: any) {
+        hasErr = true
+        this.errs.push(err)
+      }
+    }
+
+    if (!hasErr) {
+      // Collect this generation's errors into a fresh array. The option key
+      // is `err` (not `errs`) — that is what aontu reads, and providing it
+      // puts aontu in collect mode so model errors are gathered rather than
+      // thrown. A per-call array avoids leaking errors into later builds.
+      const modelErrs: any[] = []
+      this.opts.err = modelErrs
+      this.opts.deps = this.deps
+      this.opts.fs = this.fs
+
+      try {
+        this.model = this.aontu.generate(src, this.opts)
+      }
+      catch (err: any) {
+        // collect mode normally prevents throws, but guard the rare cases
+        // (e.g. unresolved imports) so they surface as build errors.
+        modelErrs.push(err)
+      }
+
+      if (0 < modelErrs.length) {
+        hasErr = true
+        this.errs.push(...modelErrs)
+      }
+    }
+
+    this.cacheSig = hasErr ? null : this.snapshotSig()
+
+    return hasErr
+  }
+
+
+  // Collect mtimeMs for the root file and every file aontu recorded as a dep.
+  snapshotSig(): Map<string, number> {
+    const sig = new Map<string, number>()
+    sig.set(this.path, mtime(this.fs, this.path))
+    for (const parent of Object.keys(this.deps)) {
+      for (const child of Object.keys(this.deps[parent])) {
+        if (!sig.has(child)) sig.set(child, mtime(this.fs, child))
+      }
+    }
+    return sig
+  }
+
+
+  cacheHit(): boolean {
+    if (!this.cacheSig) return false
+    for (const [path, prev] of this.cacheSig) {
+      if (mtime(this.fs, path) !== prev) return false
+    }
+    return true
+  }
+}
+
+
+function mtime(fs: any, path: string): number {
+  try { return fs.statSync(path).mtimeMs }
+  catch { return -1 }
+}
+
+
+function makeBuild(spec: BuildSpec, log: Log) {
+  return new BuildImpl(spec, log)
+}
+
+export {
+  makeBuild,
+  BuildSpec,
+}
+
+
+

package/src/config.ts

@@ -0,0 +1,49 @@
+/* Copyright © 2021-2025 Voxgig Ltd, MIT License. */
+
+
+import type { BuildResult, BuildSpec, Log } from './types'
+
+import { Watch } from './watch'
+
+
+
+class Config {
+  build: BuildSpec
+  watch: Watch
+  log: Log
+
+  constructor(spec: BuildSpec, log: Log) {
+    this.log = log
+
+    this.build = {
+      path: spec.path,
+      base: spec.base,
+      res: [
+        ...(spec.res || [])
+      ],
+      require: spec.require,
+      log: this.log,
+      fs: spec.fs
+    }
+
+    this.watch = new Watch(this.build, this.log)
+  }
+
+  async run(watch: boolean): Promise<BuildResult> {
+    return this.watch.run('config', watch, '<config>')
+  }
+
+  async start(initial: boolean = true) {
+    return this.watch.start(initial)
+  }
+
+  async stop() {
+    return this.watch.stop()
+  }
+}
+
+
+export { Config, BuildSpec }
+
+
+

package/src/init.ts

@@ -0,0 +1,65 @@
+/* Copyright © 2021-2025 Voxgig Ltd, MIT License. */
+
+import Path from 'node:path'
+
+
+const STARTER_MODEL = `# Voxgig model. Edit this file, then build it:
+#   voxgig-model model/model.aon
+#
+# Models are unified .aon - add types, defaults, references, imports.
+# Tutorial: https://github.com/voxgig/model/blob/main/docs/tutorial.md
+
+name: 'my-model'
+`
+
+const STARTER_CONFIG = `# Model configuration. Declare build actions and their order here.
+#
+# Example (TypeScript loads the module; Go binds the name to a
+# registered action func):
+#   sys: model: action: { example: load: 'build/example' }
+#   sys: model: order: action: 'example'
+
+sys: model: action: {}
+sys: model: order: action: *''
+`
+
+
+type InitResult = {
+  created: string[]
+  skipped: string[]
+}
+
+
+// Scaffold a starter model and config under <dir>/model. Existing files are
+// left untouched.
+function initModel(dir: string, fs: any): InitResult {
+  const d = dir || '.'
+  const files: [string, string][] = [
+    [Path.join(d, 'model', 'model.aon'), STARTER_MODEL],
+    [Path.join(d, 'model', '.model-config', 'model-config.aon'), STARTER_CONFIG],
+  ]
+
+  const created: string[] = []
+  const skipped: string[] = []
+
+  for (const [p, content] of files) {
+    if (fs.existsSync(p)) {
+      skipped.push(p)
+      continue
+    }
+    fs.mkdirSync(Path.dirname(p), { recursive: true })
+    fs.writeFileSync(p, content)
+    created.push(p)
+  }
+
+  return { created, skipped }
+}
+
+
+export {
+  initModel,
+}
+
+export type {
+  InitResult,
+}

package/src/model.ts

@@ -0,0 +1,330 @@
+/* Copyright © 2021-2025 Voxgig Ltd, MIT License. */
+
+
+import * as NodeFs from 'node:fs'
+
+import { memfs as MemFs } from 'memfs'
+
+import { prettyPino } from '@voxgig/util'
+
+import type {
+  Build,
+  BuildResult,
+  ProducerDef,
+  BuildContext,
+  BuildSpec,
+  ModelSpec,
+  Log,
+  FST,
+  ProducerResult,
+} from './types'
+
+
+import { Config } from './config'
+import { Watch } from './watch'
+
+import { model_producer } from './producer/model'
+import { local_producer } from './producer/local'
+import { msg_producer } from './producer/msg'
+
+import { initModel } from './init'
+
+
+class Model {
+  config?: Config
+  build: BuildSpec
+  watch: Watch
+
+  trigger_model = false
+
+  log: Log
+  fs: any
+
+  constructor(mspec: ModelSpec) {
+    const self = this
+
+    this.fs = { ...(mspec.fs || NodeFs) }
+
+    if (mspec.dryrun) {
+      makeReadOnly(this.fs)
+    }
+
+    const pino = prettyPino('model', mspec as any)
+
+    this.log = pino.child({ cmp: 'model' })
+
+    this.log.info({ point: 'model-init' })
+    if (this.log.isLevelEnabled('debug')) {
+      this.log.debug({
+        point: 'model-spec', mspec, note: '\n' +
+          JSON.stringify({ ...mspec, src: '<NOT-SHOWN>' }, null, 2)
+            .replace(/"/g, '')
+            .replaceAll(process.cwd(), '.')
+      })
+    }
+
+    // Config is a special Watch to handle model config. It is optional: when
+    // mspec.config is false, the .model-config/ build is skipped entirely and
+    // the model runs on its own (see run/start below).
+    const useConfig = false !== mspec.config
+
+    this.config = !useConfig ? undefined : makeConfig(mspec, this.log, this.fs, {
+      path: '/',
+      build: async function trigger_model(build: Build, ctx: BuildContext) {
+        let pres: ProducerResult = {
+          ok: false, name: 'config', step: '', active: true, reload: false, errs: [], runlog: []
+        }
+
+        if ('post' !== ctx.step) {
+          pres.ok = true
+          return pres
+        }
+
+
+        if (self.trigger_model) {
+
+          // TODO: better design
+          // Point the config's last result at the current build so the model
+          // producer reads fresh config state. It must be a thunk to satisfy
+          // BuildResult.build's `() => Build` contract (consumers call it).
+          const lastConfig = self.build.use?.config?.watch?.last
+          if (lastConfig) {
+            lastConfig.build = () => build
+          }
+
+          const br = await self.watch.run('model', true)
+          pres.ok = br.ok
+          pres.errs = br.errs
+        }
+        else {
+          self.trigger_model = true
+          pres.ok = true
+        }
+
+        if (ctx.watch) {
+          const watchmap = build.model?.sys?.model?.watch
+
+          if (watchmap) {
+            Object.keys(watchmap).forEach((file: string) => {
+              self.watch.add(file)
+            })
+          }
+        }
+
+        return pres
+      }
+    })
+
+    // The actual model.
+    this.build = {
+      path: mspec.path,
+      base: mspec.base,
+      debug: mspec.debug,
+      dryrun: mspec.dryrun,
+      buildargs: mspec.buildargs,
+      use: self.config ? { config: self.config } : {},
+      res: [
+        // Validates message declarations (pre phase), so an inconsistent
+        // main.msg fails the build before any output is written.
+        {
+          path: '/',
+          build: msg_producer
+        },
+        {
+          path: '/',
+          build: model_producer
+        },
+        {
+          path: '/',
+          build: local_producer
+        }
+      ],
+      require: mspec.require,
+      log: this.log,
+      fs: this.fs,
+      watch: mspec.watch,
+    }
+
+    this.watch = new Watch(self.build, this.log)
+  }
+
+
+  // Run once. With config enabled, the config build runs first and triggers
+  // the model build; without it, the model build runs directly.
+  async run(): Promise<BuildResult> {
+    this.trigger_model = false
+    if (!this.config) {
+      return this.watch.run('model', false, '<start>')
+    }
+    const br = await this.config.run(false)
+    return br.ok ? this.watch.run('model', false, '<start>') : br
+  }
+
+
+  // Start watching for file changes. Runs an initial build, then watches
+  // both the model files and (when enabled) the config files for ongoing
+  // changes.
+  async start() {
+    this.trigger_model = false
+    if (!this.config) {
+      return this.watch.start()
+    }
+    const br = await this.config.run(true)
+    if (!br.ok) {
+      return br
+    }
+    // Watch config files too. The initial config build is already done
+    // above, so start without forcing another one; a later config change
+    // rebuilds the config and re-triggers the model build.
+    this.config.start(false)
+    return this.watch.start()
+  }
+
+
+  async stop() {
+    // start() also spins up a config-file watcher; stop both so no
+    // chokidar handle is left open keeping the process alive.
+    await this.config?.stop()
+    return this.watch.stop()
+  }
+}
+
+
+function makeConfig(mspec: ModelSpec, log: Log, fs: any, trigger_model_build: ProducerDef) {
+  let cbase = mspec.base + '/.model-config'
+  let cpath = cbase + '/model-config.aon'
+
+  // MIGRATE A LEGACY .aontu CONFIG RATHER THAN WRITING OVER IT. This block
+  // CREATES the config when it finds none, so looking only for `.aon` in a
+  // project that has a `model-config.aontu` would not read the old file — it
+  // would decide there is no config and write a fresh default beside it,
+  // silently discarding whatever the project had declared there.
+  const legacycpath = cbase + '/model-config.aontu'
+  if (!fs.existsSync(cpath) && fs.existsSync(legacycpath)) {
+    // NOT A VERBATIM COPY. This package's OWN config moved to .aon in v10, so
+    // a legacy config's import of it names a file that no longer ships and the
+    // migrated config fails to resolve - `aontu/multisource_not_found:
+    // @voxgig/model/model/.model-config/model-config.aontu` - on the first
+    // build after upgrading. Renaming the file without retargeting that import
+    // just moves the breakage.
+    //
+    // Only THIS package's import is retargeted, and only where it is an
+    // IMPORT. Two things are deliberately left alone:
+    //
+    //   - a project's own `.aontu` imports, which still name real files on
+    //     its disk that nothing here renamed;
+    //   - this same pathname held as ordinary string DATA (a note, a
+    //     compatibility path in action metadata).
+    //
+    // Hence the match is anchored to aontu's `@"..."` import syntax, closing
+    // quote included, rather than to the bare pathname. Both are declarations
+    // the migration exists to preserve, and silently editing one during a
+    // one-time migration is precisely the failure this whole block guards
+    // against.
+    const legacy = fs.readFileSync(legacycpath, 'utf8')
+    fs.writeFileSync(cpath, legacy.replace(
+      /@(\s*)"(@voxgig\/model\/[^"]*model-config)\.aontu"/g, '@$1"$2.aon"'))
+    try { fs.unlinkSync(legacycpath) } catch (_err: any) { }
+  }
+
+  if (!fs.existsSync(cpath)) {
+    fs.mkdirSync(cbase, { recursive: true })
+    fs.writeFileSync(cpath, `
+@"@voxgig/model/model/.model-config/model-config.aon"
+
+sys: model: action: {}
+`)
+  }
+
+  let cspec: BuildSpec = {
+    name: 'config',
+    path: cpath,
+    base: cbase,
+    debug: mspec.debug,
+    res: [
+
+      // Generate full config model and save as a file.
+      {
+        path: '/',
+        build: model_producer
+      },
+
+      // Trigger main model build.
+      trigger_model_build
+    ],
+    require: mspec.require,
+    log,
+    fs,
+  }
+
+  return new Config(cspec, log)
+}
+
+
+function makeReadOnly(fsm: FST) {
+
+  // NOTE: NOT COMPLETE!
+  // Just for internal use,
+  const writers = [
+    'writeFile',
+    'writeFileSync',
+    'appendFile',
+    'appendFileSync',
+    'chmod',
+    'chmodSync',
+    'chown',
+    'chownSync',
+    'cp',
+    'cpSync',
+    'createWriteStream',
+    'mkdir',
+    'mkdirSync',
+    'rename',
+    'renameSync',
+    'rm',
+    'rmSync',
+    'rmdir',
+    'rmdirSync',
+    'symlink',
+    'symlinkSync',
+    'truncate',
+    'truncateSync',
+    'unlink',
+    'unlinkSync',
+    'write',
+    'writev',
+  ]
+
+  const { fs } = MemFs({ [process.cwd()]: {} })
+
+  for (let w of writers) {
+    if ((fs as any)[w]) {
+      (fsm as any)[w] = (fs as any)[w].bind(fs)
+    }
+  }
+
+  // Also redirect the promise-based writers. fsm.promises is shared by
+  // reference with the real fs module, so replace it with a copy rather
+  // than mutating the caller's fs.
+  const memPromises = (fs as any).promises
+  if ((fsm as any).promises && memPromises) {
+    const promises: any = { ...(fsm as any).promises }
+    for (let w of writers) {
+      if ('function' === typeof memPromises[w]) {
+        promises[w] = memPromises[w].bind(memPromises)
+      }
+    }
+    ;(fsm as any).promises = promises
+  }
+
+  return fsm
+}
+
+
+export {
+  Model,
+  BuildSpec,
+  initModel,
+}
+
+