@voxgig/model 10.0.1 → 10.1.0

package/src/producer/local.ts

@@ -0,0 +1,116 @@
+
+import Path from 'path'
+
+import type { Build, Producer, BuildContext, ProducerResult } from '../types'
+
+const ORDERING_SPLIT_RE = /\s*,+\s*/
+
+// Runs any producers local to the repo.
+const local_producer: Producer = async (build: Build, ctx: BuildContext) => {
+
+  ctx.state.local = (ctx.state.local || {})
+  let actionDefs = ctx.state.local.actionDefs
+
+  if (null == actionDefs) {
+    actionDefs = ctx.state.local.actionDefs = []
+
+    // TODO: need to provide project root via build
+    let root = Path.resolve(build.path, '..', '..')
+
+    // TODO: build should do this
+    // Config is optional: with no .model-config build linked in, there are no
+    // declared actions and the model runs on its own.
+    let configBuildResult = build.use.config?.watch?.last
+    let configBuild = configBuildResult?.build()
+    let config = configBuild?.model || {}
+
+    let actions = config.sys?.model?.action ||
+      // NOTE: backwards compat
+      config.sys?.model?.builders ||
+      {}
+
+    let ordering = config.sys?.model?.order?.action
+    ordering = null == ordering ? Object.keys(actions) :
+      ordering.split(ORDERING_SPLIT_RE).filter((n: string) => null != n && '' != n)
+
+    // load actions
+    for (let name of ordering) {
+      let actiondef = actions[name]
+
+      if (null == actiondef) {
+        throw new Error(
+          'Unknown model action "' + name +
+          '" referenced in sys.model.order.action')
+      }
+
+      if (null == actiondef.load) {
+        throw new Error(
+          'Model action "' + name + '" is missing a "load" path')
+      }
+
+      let actionpath = Path.join(root, actiondef.load)
+
+      let action = require(actionpath)
+
+      if (action instanceof Promise) {
+        action = await action
+      }
+
+      const step = action.step || 'post'
+
+      actionDefs.push({ name, actiondef, action, step })
+    }
+  }
+
+  const runActionDefs = actionDefs.filter((ad: any) => ctx.step === ad.step || 'all' === ad.step)
+
+  build.log.info({
+    point: ctx.step + '-actions', step: ctx.step, actions: runActionDefs,
+    note: runActionDefs.map((ad: any) => ad.name).join(';')
+  })
+
+  let ok = true
+  let areslog = []
+  let reload = false
+
+  for (let actionDef of runActionDefs) {
+    try {
+      // TODO: this call signature needs to be well-defined as it is an external interface
+      let ares = await actionDef.action(build.model, build, ctx)
+      ok = ok && (null == ares || !!ares.ok)
+      reload = reload || ares?.reload
+
+      areslog.push(ares)
+
+      if (!ok) { break }
+    }
+    catch (err: any) {
+      if (!err.__logged__) {
+        build.log.error({
+          point: ctx.step + '-action', step: ctx.step, action: actionDef,
+          note: actionDef.name,
+          err
+        })
+        err.__logged__ = true
+      }
+      throw err
+    }
+  }
+
+  let pr: ProducerResult = {
+    ok,
+    reload,
+    name: 'local',
+    step: ctx.step,
+    active: true,
+    errs: [],
+    runlog: []
+  }
+
+  return pr
+}
+
+
+export {
+  local_producer
+}

package/src/producer/model.ts

@@ -0,0 +1,122 @@
+
+import Path from 'path'
+
+import type { Build, Producer, BuildContext, ProducerResult } from '../types'
+
+
+// Serialize the model to two-space-indented JSON with object keys in strictly
+// lexical (UTF-8 byte) order, byte-for-byte identical to the Go
+// implementation's encoding/json. JSON.stringify cannot express this: JS
+// objects iterate integer-like keys ("9", "10") in numeric order ahead of the
+// other keys regardless of insertion order, so the order must be imposed
+// during serialization, and the default JS string sort compares UTF-16 code
+// units, which disagrees with Go's byte order for astral-plane keys. Arrays
+// keep their order. Values only producer mutation can introduce mirror
+// JSON.stringify: undefined, functions, and symbols are dropped from objects
+// and become null in arrays (sparse holes too), and toJSON results are fed
+// back through the canonical serializer.
+function jsonify(value: any, indent: string): string {
+  if (Array.isArray(value)) {
+    if (0 === value.length) {
+      return '[]'
+    }
+    const inner = indent + '  '
+    // Array.from visits holes (as undefined); map/join would skip them.
+    return '[\n' +
+      Array.from(value, (item) => inner + jsonify(item, inner)).join(',\n') +
+      '\n' + indent + ']'
+  }
+
+  if (null != value && 'object' === typeof value) {
+    if ('function' === typeof value.toJSON) {
+      return jsonify(value.toJSON(), indent)
+    }
+    const keys = Object.keys(value)
+      .filter((key) => {
+        const item = value[key]
+        return undefined !== item &&
+          'function' !== typeof item && 'symbol' !== typeof item
+      })
+      .sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))
+    if (0 === keys.length) {
+      return '{}'
+    }
+    const inner = indent + '  '
+    return '{\n' +
+      keys.map((key) =>
+        inner + jstr(key) + ': ' + jsonify(value[key], inner))
+        .join(',\n') +
+      '\n' + indent + '}'
+  }
+
+  const scalar = jstr(value)
+  return undefined === scalar ? 'null' : scalar
+}
+
+
+// JSON.stringify, plus the U+2028/U+2029 escapes Go's encoding/json always
+// applies even with HTML escaping off. JSON.stringify emits the separators
+// literally (both forms are valid JSON), so escape them here for byte parity.
+function jstr(value: any): string | undefined {
+  const out = JSON.stringify(value)
+  return undefined === out ? undefined :
+    out.replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029')
+}
+
+
+// Builds the main model file, after unification.
+const model_producer: Producer = async (build: Build, ctx: BuildContext) => {
+  let pr: ProducerResult = {
+    ok: true,
+    name: 'model',
+    reload: false,
+    step: ctx.step,
+    active: true,
+    errs: [],
+    runlog: []
+  }
+
+  if ('post' !== ctx.step) {
+    return pr
+  }
+
+  let json = jsonify(build.model, '')
+
+  let filename = Path.basename(build.path)
+  let filenameparts = filename.match(/^(.*)\.[^.]+$/)
+  if (filenameparts) {
+    filename = filenameparts[1]
+  }
+
+  let file = build.opts.base + '/' + filename + '.json'
+
+  // Skip write when output is unchanged — avoids mtime churn that would
+  // invalidate caches (here and in downstream watchers).
+  let existing: string | undefined
+  try { existing = build.fs.readFileSync(file, 'utf8') } catch { }
+
+  if (existing === json) {
+    build.log.debug({
+      point: 'write-model-skip',
+      path: file,
+      note: file.replace(process.cwd(), '.') + ' (unchanged)'
+    })
+    return pr
+  }
+
+  build.log.info({
+    point: 'write-model',
+    path: file,
+    note: file.replace(process.cwd(), '.')
+  })
+
+  build.fs.mkdirSync(Path.dirname(file), { recursive: true })
+  build.fs.writeFileSync(file, json)
+
+
+  return pr
+}
+
+export {
+  model_producer
+}

package/src/producer/msg.ts

@@ -0,0 +1,207 @@
+/* Copyright © 2026 Voxgig Ltd, MIT License. */
+
+import type { Build, Producer, BuildContext, ProducerResult } from '../types'
+
+
+// Message declarations live in `main.msg` and come in two shapes.
+//
+// The legacy shape nests the pattern pairs, so the pattern is the path down
+// to the definition, and the definition sits at whatever depth that reaches:
+//
+//   aim: web: { on: todo: { save: item: { '$': { file: './web_save_item' } } } }
+//
+// The declared shape is flat - one entry per message, keyed by the message
+// name, with the pattern as data and the definition at a known depth:
+//
+//   save_item: { pat: [ {aim: web}, {save: item} ], doc: "Save a todo item" }
+//
+// The two are told apart by `pat`: a definition is a map holding a `pat`
+// LIST, and a legacy chain node never holds one, because every value in a
+// chain node is a map - either the next pattern level or the '$' leaf. So the
+// discriminator holds even for a legacy pattern pair that happens to be
+// spelled `pat:`. Anything without a `pat` list is left alone entirely, which
+// is what lets the two shapes coexist while models migrate message by
+// message.
+//
+// Only the declared shape is checked here. The checks are the two things the
+// flat shape makes checkable and the nested one did not:
+//
+//   1. the entry key agrees with the last pattern pair - the key names the
+//      action file, a convention the legacy shape got implicitly from the
+//      chain's leaf, and which becomes a real consistency check once the key
+//      is written out by hand;
+//   2. no two messages declare the same pattern - previously impossible to
+//      state twice, because the pattern WAS the path.
+
+
+// Report a problem against the message it belongs to.
+function msgerr(name: string, why: string): string {
+  return 'model msg "' + name + '": ' + why
+}
+
+
+function isObj(val: any): boolean {
+  return null != val && 'object' === typeof val && !Array.isArray(val)
+}
+
+
+// A message definition declares its pattern as a list; a chain node never does.
+function isMsgDef(val: any): boolean {
+  return isObj(val) && Array.isArray(val.pat)
+}
+
+
+// Sort in UTF-8 byte order, matching Go's sort.Strings, so both
+// implementations report the same problems in the same order. The default JS
+// string sort compares UTF-16 code units, which disagrees for astral-plane
+// names (see the model producer's jsonify, which sorts keys the same way).
+function sortNames(names: string[]): string[] {
+  return names.sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))
+}
+
+
+// Validate the declared-shape message entries in main.msg, returning one
+// message per problem found (empty when the model is valid, which includes a
+// model with no messages at all, or only legacy chains).
+function checkMsg(model: any): string[] {
+  const problems: string[] = []
+
+  const msg = model?.main?.msg
+  if (!isObj(msg)) {
+    return problems
+  }
+
+  // Canonical pattern -> the message that claimed it first.
+  const seen: { [canon: string]: string } = {}
+
+  for (const name of sortNames(Object.keys(msg))) {
+    const def = msg[name]
+
+    // Legacy chain node (or not a map at all): not this check's business.
+    if (!isMsgDef(def)) {
+      continue
+    }
+
+    const pat: any[] = def.pat
+
+    if (0 === pat.length) {
+      problems.push(msgerr(name, 'pat declares no pattern pairs'))
+      continue
+    }
+
+    // Reduce the pattern to its pairs, stopping at the first malformed one -
+    // the rest of the checks read the pairs, so there is nothing further to
+    // say about this message until its pattern is well-formed.
+    //
+    // Two renderings: `pairs` reads well in a message, and `canon` identifies
+    // the pattern. They cannot be the same string, because `key:value` joined
+    // by commas is ambiguous once a key or value contains a delimiter -
+    // [{a: "b,c:d"}] and [{a: b}, {c: d}] would both render `a:b,c:d` and the
+    // second would be rejected as a duplicate of the first. Quoting each part
+    // removes the ambiguity: a delimiter inside a part is escaped, so only
+    // genuinely equal patterns produce equal keys.
+    const pairs: string[] = []
+    const canon: string[] = []
+    let last: string[] | undefined
+
+    for (let pI = 0; pI < pat.length; pI++) {
+      const pair = pat[pI]
+      const keys = isObj(pair) ? Object.keys(pair) : []
+
+      if (1 !== keys.length) {
+        problems.push(msgerr(name, 'pat pair ' + pI +
+          ' is not a single key:value pair'))
+        last = undefined
+        break
+      }
+
+      const key = keys[0]
+      const val = pair[key]
+
+      if ('string' !== typeof val) {
+        problems.push(msgerr(name, 'pat pair ' + pI + ' (' + key +
+          ') value is not a string'))
+        last = undefined
+        break
+      }
+
+      pairs.push(key + ':' + val)
+      canon.push(JSON.stringify(key) + ':' + JSON.stringify(val))
+      last = [key, val]
+    }
+
+    if (null == last) {
+      continue
+    }
+
+    // The entry key names the action file, so it must agree with the last
+    // pattern pair.
+    const expected = last[0] + '_' + last[1]
+    if (name !== expected) {
+      problems.push(msgerr(name, 'key does not match last pat pair ' +
+        last[0] + ':' + last[1] + ' (expected "' + expected + '")'))
+    }
+
+    const canonKey = canon.join(',')
+    if (null == seen[canonKey]) {
+      seen[canonKey] = name
+    }
+    else {
+      problems.push(msgerr(name, 'pat [' + pairs.join(',') +
+        '] is already declared by "' + seen[canonKey] + '"'))
+    }
+  }
+
+  return problems
+}
+
+
+// Checks the message declarations before anything is written.
+//
+// This runs in BOTH phases, and must: a `pre` action can rewrite model source
+// and request a reload, and the build re-resolves the model AFTER the pre
+// phase has finished (see BuildImpl.run). A pre-only check would then have
+// validated a model that no longer exists, and the model producer would write
+// the regenerated one unchecked. Checking again in post closes that window -
+// this producer is first in the pipeline, so it still runs ahead of the model
+// producer, and a build whose model went bad during a reload fails with
+// nothing written.
+const msg_producer: Producer = async (build: Build, ctx: BuildContext) => {
+  const pr: ProducerResult = {
+    ok: true,
+    name: 'msg',
+    reload: false,
+    step: ctx.step,
+    active: true,
+    errs: [],
+    runlog: []
+  }
+
+  const problems = checkMsg(build.model)
+
+  if (0 < problems.length) {
+    pr.ok = false
+    pr.errs = problems.map((problem) => new Error(problem))
+
+    // Add them to the build too. BuildImpl.run collects the errors a producer
+    // THROWS, but not the ones it returns, so a returned error would
+    // otherwise be missing from the BuildResult. (The Go port needs no such
+    // push: its runProducer merges a failed producer's Errs itself. Both
+    // implementations end up with the same errors on the build.)
+    build.errs.push(...pr.errs)
+
+    build.log.error({
+      point: 'msg-invalid',
+      count: problems.length,
+      note: problems.join('; ')
+    })
+  }
+
+  return pr
+}
+
+
+export {
+  msg_producer,
+  checkMsg,
+}

package/src/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "tsBuildInfoFile": "../dist/tsconfig.tsbuildinfo",
+    "esModuleInterop": true,
+    "module": "nodenext",
+    "noEmitOnError": true,
+    "outDir":"../dist",
+    "rootDir":".",
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "sourceMap": true,
+    "strict": true,
+    "target": "es2021",
+    "declaration": true,
+    "declarationDir": "../dist"
+  }
+}
+

package/src/types.ts

@@ -0,0 +1,162 @@
+/* Copyright © 2021-2024 Voxgig Ltd, MIT License. */
+
+import Fs from 'node:fs'
+
+import Pino from 'pino'
+
+
+import type { Aontu } from 'aontu'
+
+
+type FST = typeof Fs
+
+type Log = ReturnType<typeof Pino>
+
+interface Build {
+  id: string
+  base: string
+  path: string
+  opts: { [key: string]: any }
+  pdef: ProducerDef[]
+  spec: BuildSpec
+  model: any
+  use: { [name: string]: any }
+  errs: any[]
+  ctx: BuildContext
+  deps: any
+  run: (rspec: RunSpec) => Promise<BuildResult>
+  log: Log
+  fs: FST
+  dryrun: boolean
+  args: any
+  aontu: Aontu
+
+}
+
+
+interface BuildResult {
+  ok: boolean
+  builder?: string
+  path?: string
+  producers?: ProducerResult[]
+  step?: string
+  errs: any[]
+  runlog: string[]
+  build?: () => Build
+}
+
+
+interface BuildContext {
+  step: 'pre' | 'post'
+  watch: boolean,
+  state: Record<string, any>
+}
+
+
+interface BuildSpec {
+  path?: string
+  base?: string
+  res?: ProducerDef[]
+  require?: any
+  use?: { [name: string]: any }
+  log?: Log
+  idle?: number
+  name?: string
+  debug?: boolean | string
+  dryrun?: boolean,
+  buildargs?: any,
+  watch?: {
+    mod?: boolean // file modification
+    add?: boolean // file addition
+    rem?: boolean // file deletion
+  }
+  fs: FST
+}
+
+
+interface ProducerDef {
+  path: string
+  build: Producer
+}
+
+type Producer = (
+  build: Build,
+  ctx: BuildContext,
+) => Promise<ProducerResult>
+
+
+interface ProducerResult {
+  ok: boolean
+  name: string
+  active: boolean
+  errs: any[]
+  runlog: string[]
+  step: string
+  reload: boolean
+}
+
+
+type Run = {
+  canon: string
+  path: string
+  start: number
+  end: number
+  result?: BuildResult
+}
+
+
+type RunSpec = {
+  watch: boolean
+}
+
+type Canon = {
+  path: string
+  isFolder: boolean
+  when: number
+}
+
+
+type ChangeItem = {
+  path: string
+  when: number
+}
+
+interface ModelSpec {
+  path?: string
+  base?: string
+  require?: any
+  log?: Log
+  idle?: number
+  debug?: boolean | string
+  dryrun?: boolean
+  buildargs?: any
+  fs?: any
+  // Resolve a .model-config/model-config.aon (auto-created when missing)
+  // that declares the build actions and their order. Defaults to true; set
+  // false to skip the config entirely and run the model on its own.
+  config?: boolean
+  watch?: {
+    mod?: boolean // file modification
+    add?: boolean // file addition
+    rem?: boolean // file deletion
+  }
+}
+
+
+
+export type {
+  Build,
+  BuildResult,
+  ProducerDef,
+  Producer,
+  ProducerResult,
+  BuildContext,
+  BuildSpec,
+  Log,
+  Run,
+  Canon,
+  ChangeItem,
+  RunSpec,
+  ModelSpec,
+  FST,
+}