@voxgig/sdkgen 4.32.1 → 4.34.0

package/src/action/doctor.ts

@@ -37,6 +37,9 @@ import { findFeatureSources } from '../helpers/featureSource'
 
 import { isJunk } from '../helpers/junk'
 
+import { readCopies, untouched, itemKey } from './copies'
+import type { CopyRecord } from './copies'
+
 
 const IGNORED_RE = /(~|-jostraca-off)$/
 
@@ -97,10 +100,41 @@ type DoctorReport = {
   // it is what an alias is FOR.
   aliasedDiff: string[]
 
+  // Differs from its source, but the copy record shows it untouched since an
+  // add wrote it: the source moved on, so refreshing it discards nothing.
+  outdated: string[]
+
+  // The forked and edited findings the copy record has no entry for, so
+  // nothing shows whether the project changed them.
+  unrecorded: string[]
+
+  // Per item with findings, keyed `<kind>/<name>`.
+  byItem: Record<string, ItemDrift>
+
   ok: boolean
 }
 
 
+type ItemDrift = {
+  // The package the item resolves to now, which is what refreshes it.
+  package?: string
+
+  // The version its copies came from, and the version installed now.
+  from?: string
+  to?: string
+
+  outdated: number
+
+  // Forked, edited, missing and stale files.
+  changed: number
+}
+
+
+type Category = 'forked' | 'edited' | 'missing' | 'stale'
+
+type Found = (category: Category, label: string) => void
+
+
 const CMD_MAP: any = {
   check: cmd_doctor_check,
   prune: cmd_doctor_prune,
@@ -243,9 +277,12 @@ async function doctor(
   const report: DoctorReport = {
     forked: [], edited: [], stale: [], missing: [], additive: [],
     superseded: [], unwired: [], orphanModel: [],
-    resyncPending: [], aliasedDiff: [], ok: true,
+    resyncPending: [], aliasedDiff: [],
+    outdated: [], unrecorded: [], byItem: {}, ok: true,
   }
 
+  const copies = readCopies(fs, root)
+
   report.superseded = supersededFiles(actx)
 
   report.orphanModel = orphanModelFiles(actx)
@@ -285,12 +322,14 @@ async function doctor(
         continue
       }
 
+      const found = noteDrift(actx, report, copies, kind, source)
+
       if ('target' === kind) {
-        checkTarget(actx, source, report, selected)
+        checkTarget(actx, source, report, found, selected)
       }
 
       if ('edition' === kind) {
-        checkEdition(actx, source, report)
+        checkEdition(actx, source, report, found)
       }
 
       // Only an ACTIVE feature has source copied out; an inactive one's
@@ -298,10 +337,10 @@ async function doctor(
       if ('feature' === kind &&
         (false !== (model as any)?.main?.[KIT]?.feature?.[name]?.active ||
           true === selected?.includes(name))) {
-        checkFeatureSource(actx, source, targets, report)
+        checkFeatureSource(actx, source, targets, found)
       }
 
-      checkItemModel(actx, kind, source, report)
+      checkItemModel(actx, kind, source, report, found)
     }
   }
 
@@ -316,7 +355,7 @@ async function doctor(
   // authoritative either way.
   report.ok = 0 === report.forked.length + report.edited.length +
     report.stale.length + report.missing.length + report.superseded.length +
-    report.orphanModel.length
+    report.orphanModel.length + report.outdated.length
 
   for (const [kind, note] of [
     ['forked', 'FORKED (will be reverted by `target add`)'],
@@ -329,12 +368,24 @@ async function doctor(
     ['orphanModel', 'ORPHAN MODEL FILE (on disk, included by nothing, read by nobody)'],
     ['resyncPending', 'RESYNC PENDING (predates provenance; `target add` updates it)'],
     ['aliasedDiff', 'aliased model differs from its origin (project-owned, not drift)'],
+    ['outdated', 'OUTDATED (unchanged since it was copied, but its source has moved on)'],
   ] as [keyof DoctorReport, string][]) {
     for (const file of (report[kind] as string[])) {
       log.info({ point: 'doctor-finding', kind, file, note: note + ': ' + file })
     }
   }
 
+  for (const item of Object.keys(report.byItem).sort()) {
+    const drift = report.byItem[item]
+    if (0 < drift.outdated) {
+      log.info({
+        point: 'doctor-outdated', item, ...drift,
+        note: outdatedNote(item, drift) + ' - `voxgig-sdkgen ' +
+          refreshCommand(item, drift.package) + '` refreshes them'
+      })
+    }
+  }
+
   log.info({
     point: 'doctor-end',
     ok: report.ok,
@@ -348,11 +399,13 @@ async function doctor(
     orphanModel: report.orphanModel.length,
     resyncPending: report.resyncPending.length,
     aliasedDiff: report.aliasedDiff.length,
+    outdated: report.outdated.length,
     note: report.ok ?
       ('.sdk matches the scaffold (' + report.additive.length + ' additive)') :
       ('.sdk has drifted: ' + report.forked.length + ' forked, ' +
         report.edited.length + ' edited, ' + report.stale.length + ' stale, ' +
-        report.missing.length + ' missing')
+        report.missing.length + ' missing, ' + report.outdated.length +
+        ' outdated')
   })
 
   return { report }
@@ -412,7 +465,7 @@ function resolveDeclared(
 
 
 function checkTarget(
-  actx: ActionContext, resolved: Source, report: DoctorReport,
+  actx: ActionContext, resolved: Source, report: DoctorReport, found: Found,
   selected?: string[],
 ) {
   const tname = resolved.name
@@ -466,7 +519,7 @@ function checkTarget(
     { log: quietLog(actx.log), fs: () => fs, folder: root, model },
     tfolder, torigname, tname, features)
 
-  compareTrees(actx, report, trees, {
+  compareTrees(actx, report, found, trees, {
     excludes,
     foreign: (kind: string) => 'edited' === kind ?
       foreignFeatureSource(actx, resolved) : [],
@@ -477,6 +530,7 @@ function checkTarget(
 function compareTrees(
   actx: ActionContext,
   report: DoctorReport,
+  found: Found,
   trees: TreeCompare[],
   opts?: {
     excludes?: RegExp[],
@@ -518,7 +572,7 @@ function compareTrees(
 
     for (const rel of expected) {
       if (!actualSet.has(rel)) {
-        report.missing.push(label + rel)
+        found('missing', label + rel)
         continue
       }
 
@@ -535,7 +589,7 @@ function compareTrees(
 
       if (differs(fs, from, Path.join(tree.project, rel),
         model, tree.replace, undefined, tree.rewrite)) {
-        report[tree.kind].push(label + rel)
+        found(tree.kind, label + rel)
       }
     }
 
@@ -553,7 +607,7 @@ function compareTrees(
         report.additive.push(label + rel)
       }
       else {
-        report.stale.push(label + rel)
+        found('stale', label + rel)
       }
     }
   }
@@ -561,7 +615,7 @@ function compareTrees(
 
 
 function checkEdition(
-  actx: ActionContext, resolved: Source, report: DoctorReport,
+  actx: ActionContext, resolved: Source, report: DoctorReport, found: Found,
 ) {
   const fs = actx.fs()
   const root = actx.folder
@@ -593,7 +647,7 @@ function checkEdition(
     } as TreeCompare]
   })
 
-  compareTrees(actx, report, trees)
+  compareTrees(actx, report, found, trees)
 }
 
 
@@ -677,7 +731,7 @@ function checkFeatureSource(
   actx: ActionContext,
   feature: Source,
   targets: Map<string, Source>,
-  report: DoctorReport,
+  found: Found,
 ) {
   const fs = actx.fs()
   const root = actx.folder
@@ -689,14 +743,14 @@ function checkFeatureSource(
       const label = 'tm/' + tname + '/' + rel
 
       if (!fs.existsSync(project)) {
-        report.missing.push(label)
+        found('missing', label)
         continue
       }
 
       // The same map `feature add` copies with — see helpers/stdrep. A
       // different one here would report every substituted file as edited.
       if (differs(fs, from, project, model, templateReplacements(model, tname))) {
-        report.edited.push(label)
+        found('edited', label)
       }
     }
   }
@@ -705,6 +759,7 @@ function checkFeatureSource(
 
 function checkItemModel(
   actx: ActionContext, kind: string, source: Source, report: DoctorReport,
+  found: Found,
 ) {
   const name = source.name
   const origname = source.origname
@@ -723,7 +778,7 @@ function checkItemModel(
   const label = 'model/' + kind + '/' + Path.basename(project)
 
   if (!fs.existsSync(project)) {
-    report.missing.push(label)
+    found('missing', label)
     return
   }
 
@@ -746,7 +801,92 @@ function checkItemModel(
     return
   }
 
-  report.forked.push(label)
+  found('forked', label)
+}
+
+
+// Files every finding through one place: a forked or edited copy the record
+// shows untouched is outdated instead, and every item keeps a tally.
+function noteDrift(
+  actx: ActionContext, report: DoctorReport, copies: CopyRecord,
+  kind: string, source: Source,
+): Found {
+  const fs = actx.fs()
+  const item = itemKey(kind, source.name)
+  const copied = copies.items[item]
+
+  return (category: Category, label: string) => {
+    let into: Category | 'outdated' = category
+
+    if ('forked' === category || 'edited' === category) {
+      const same = untouched(fs, actx.folder, copies, label)
+
+      if (true === same) {
+        into = 'outdated'
+      }
+      else if (undefined === same) {
+        report.unrecorded.push(label)
+      }
+    }
+
+    report[into].push(label)
+
+    const drift = report.byItem[item] = report.byItem[item] ?? {
+      ...pkgOf(source.package ?? copied?.package),
+      ...(null == copied?.version ? {} : { from: copied.version }),
+      ...(null == source.version ? {} : { to: source.version }),
+      outdated: 0,
+      changed: 0,
+    }
+
+    if ('outdated' === into) {
+      drift.outdated++
+    }
+    else {
+      drift.changed++
+    }
+  }
+}
+
+
+function pkgOf(name?: string): { package?: string } {
+  return null == name ? {} : { package: name }
+}
+
+
+// The command that brings an item's copies up to the installed source. The
+// package form checks first and refuses to overwrite a local edit.
+function refreshCommand(item: string, pkg?: string): string {
+  if (null != pkg) {
+    return 'package update ' + pkg + ' --no-fetch'
+  }
+
+  const [kind, ...name] = item.split('/')
+  return kind + ' add ' + name.join('/')
+}
+
+
+function outdatedNote(item: string, drift: ItemDrift): string {
+  return item.replace('/', ' ') + ': ' + drift.outdated +
+    ' file(s) are unchanged copies' + sinceCopied(drift.package, [drift])
+}
+
+
+// Names both versions only when they differ: a source that changed under the
+// same version (a linked checkout, say) is still a source that moved on.
+function sinceCopied(pkg: string | undefined, drift: ItemDrift[]): string {
+  const froms = Array.from(new Set(drift.map((d: ItemDrift) => d.from)))
+  const to = drift[0].to
+  const from = 1 === froms.length ? froms[0] : undefined
+  const name = pkg ?? 'their source'
+
+  if (null != from && null != to && from !== to) {
+    return ' from ' + name + ' ' + from + ', but ' + name + ' ' + to +
+      ' is installed'
+  }
+
+  return ', but ' + name + (null == to ? '' : ' ' + to) +
+    ' has changed since they were copied'
 }
 
 
@@ -868,13 +1008,19 @@ function renderPair(
   const rawsrc = fs.readFileSync(scaffoldPath, 'utf8')
   const src = null == rewrite ? rawsrc : rewrite(rawsrc)
 
+  // git may rewrite line endings on checkout (core.autocrlf).
   return {
-    expected: template(src, model, { replace }),
-    actual: fs.readFileSync(projectPath, 'utf8'),
+    expected: lfEndings(template(src, model, { replace })),
+    actual: lfEndings(fs.readFileSync(projectPath, 'utf8')),
   }
 }
 
 
+function lfEndings(text: string): string {
+  return text.replace(/\r\n/g, '\n')
+}
+
+
 function quietLog(log: any): any {
   const noop = () => { }
   const quiet: any = { info: noop, debug: noop, warn: log.warn.bind(log), error: noop, trace: noop, fatal: noop }
@@ -883,12 +1029,92 @@ function quietLog(log: any): any {
 }
 
 
+function silentLog(): any {
+  const noop = () => { }
+  const silent: any = { info: noop, debug: noop, warn: noop, error: noop, trace: noop, fatal: noop }
+  silent.child = () => silent
+  return silent
+}
+
+
+// What `generate` checks before it reads the copies: the active items only,
+// and a warning per refresh command rather than a failure, since a
+// difference may be a deliberate edit.
+async function copyCheck(actx: ActionContext): Promise<string[]> {
+  const active = (kind: string, name: string) =>
+    false !== kindCollection(actx.model, kind)?.[name]?.active
+
+  const res: any = await doctor({ ...actx, log: silentLog() }, active)
+
+  return copyWarnings(res.report)
+}
+
+
+function copyWarnings(report: DoctorReport): string[] {
+  const groups = new Map<string, { items: string[], files: number, drift: ItemDrift[] }>()
+
+  const add = (key: string, item: string, files: number, drift: ItemDrift) => {
+    const group = groups.get(key) ?? { items: [], files: 0, drift: [] }
+    group.items.push(item.replace('/', ' '))
+    group.files += files
+    group.drift.push(drift)
+    groups.set(key, group)
+  }
+
+  for (const item of Object.keys(report.byItem).sort()) {
+    const drift = report.byItem[item]
+    const command = refreshCommand(item, drift.package)
+
+    if (0 < drift.outdated) {
+      add('outdated\n' + command, item, drift.outdated, drift)
+    }
+
+    if (0 < drift.changed) {
+      add('changed\n' + command, item, drift.changed, drift)
+    }
+  }
+
+  const lines: string[] = []
+
+  for (const [key, group] of groups) {
+    const [what, command] = key.split('\n')
+    const pkg = group.drift[0].package
+    const to = group.drift[0].to
+    const installed = null == pkg ? 'their source' :
+      (pkg + (null == to ? '' : ' ' + to))
+    const refresh = '`npx voxgig-sdkgen ' + command + '`'
+
+    if ('outdated' === what) {
+      lines.push(group.items.join(', ') + ': ' + group.files +
+        ' file(s) in .sdk are unchanged copies' +
+        sinceCopied(pkg, group.drift) + ', so this run generates from ' +
+        'older templates. Run ' + refresh + ' to refresh them.')
+    }
+    else {
+      lines.push(group.items.join(', ') + ': ' + group.files +
+        ' file(s) in .sdk differ from what ' + installed + ' installs: ' +
+        'changed in this project, or copied before sdkgen recorded its ' +
+        'copies. `npx voxgig-sdkgen doctor` lists them; ' + refresh +
+        ' refreshes them' +
+        (null == pkg ? ', discarding any local edit.' :
+          ', and refuses before it overwrites a local edit.'))
+    }
+  }
+
+  return lines
+}
+
+
 export type {
   DoctorReport,
   DoctorScope,
+  ItemDrift,
 }
 
 export {
   action_doctor,
   doctor,
+  copyCheck,
+  copyWarnings,
+  refreshCommand,
 }

package/src/action/edition.ts

@@ -17,6 +17,9 @@ import { resolveKind, kindModel, kindIndex, kindTrees } from './kind'
 import type { TreeDef } from './kind'
 
 import { registerInstalled } from './resolve'
+import type { Source } from './resolve'
+
+import { recordCopies } from './copies'
 
 import { aliasCmpTree, pruneStaleTemplates } from './target'
 
@@ -107,8 +110,12 @@ async function edition_add(
   // Later items in the command read this in-memory registration.
   registerInstalled('edition', edition, actx)
 
+  const installed: Source[] = []
+
   const jres = await jostraca.generate(opts, () =>
-    EditionRoot({ edition, actx }))
+    EditionRoot({ edition, actx, installed }))
+
+  recordCopies(actx, jres, 'edition', installed)
 
   return { jres }
 }
@@ -139,7 +146,7 @@ function preflight(edition: string[], actx: ActionContext) {
 
 
 const EditionRoot = cmp(function EditionRoot(props: any) {
-  const { ctx$, edition } = props
+  const { ctx$, edition, installed } = props
   const { log } = ctx$
 
   Project({}, () => {
@@ -153,6 +160,7 @@ const EditionRoot = cmp(function EditionRoot(props: any) {
       const source = resolveKind(dref, 'edition', ctx$)
 
       dnames.push(source.name)
+      installed?.push(source)
 
       log.info({
         point: 'edition-name', edition: source.name, folder: source.folder, ref: dref,

package/src/action/feature.ts

@@ -32,6 +32,9 @@ import { templateReplacements } from '../helpers/stdrep'
 import { copyOpts } from '../helpers/junk'
 
 import { resolveKind, kindModel, kindIndex } from './kind'
+import type { Source } from './resolve'
+
+import { recordCopies } from './copies'
 
 
 import {
@@ -111,10 +114,15 @@ async function feature_add(features: string[], actx: ActionContext): Promise<Act
     note: (actx.opts.dryrun ? '** DRY RUN **' : '')
   })
 
-  const jres = await jostraca.generate(opts, () => FeatureRoot({ features }))
+  const installed: Source[] = []
+
+  const jres = await jostraca.generate(opts, () =>
+    FeatureRoot({ features, installed }))
 
   showChanges(opts.log, 'feature-result', jres)
 
+  recordCopies(actx, jres, 'feature', installed)
+
   if (actx.opts.dryrun) {
     showDryrun(opts.log, 'feature-result', jres, actx.folder)
   }
@@ -131,7 +139,7 @@ async function feature_add(features: string[], actx: ActionContext): Promise<Act
 
 
 const FeatureRoot = cmp(function FeatureRoot(props: any) {
-  const { ctx$, features } = props
+  const { ctx$, features, installed } = props
   const { model, log } = ctx$
 
   const fs = ctx$.fs()
@@ -167,6 +175,7 @@ const FeatureRoot = cmp(function FeatureRoot(props: any) {
 
       const fname = source.name
       fnames.push(fname)
+      installed?.push(source)
 
       log.info({
         point: 'feature-build',

package/src/action/package.ts

@@ -29,6 +29,8 @@ import { cmd_package_check } from './check'
 
 import { doctor } from './doctor'
 
+import { COPIES } from './copies'
+
 import { resolveSource, registerInstalled, nameConflict } from './resolve'
 import type { Source } from './resolve'
 
@@ -639,6 +641,16 @@ async function preCheck(
 
   const report = res.report
 
+  // Unchanged since an add wrote them, so refreshing them loses nothing.
+  if (0 < report.outdated.length) {
+    actx.log.info({
+      point: 'package-update-outdated', package: pkgname,
+      files: report.outdated.length,
+      note: pkgname + ': ' + report.outdated.length + ' outdated file(s) ' +
+        'will be refreshed; ' + COPIES + ' shows nothing in them was changed'
+    })
+  }
+
   const changed = [...report.forked, ...report.edited]
 
   if (0 === changed.length) {
@@ -654,19 +666,28 @@ async function preCheck(
     return
   }
 
+  const unrecorded = new Set<string>(report.unrecorded)
+  const edited = changed.filter((file: string) => !unrecorded.has(file))
+  const unknown = changed.filter((file: string) => unrecorded.has(file))
+
   throw new SdkGenError(
     pkgname + ': ' + changed.length + ' file(s) differ from the installed ' +
     'source, so updating would overwrite them:\n  ' + changed.join('\n  ') +
-    '\n\n  This means one of two things, and nothing recorded in the project ' +
-    'tells them apart:' +
-    '\n    - they are LOCAL EDITS, and `--force` will discard them;' +
-    '\n    - or ' + pkgname + ' was already updated out of band (an ' +
-    '`npm update` in another shell), in which case they are merely STALE ' +
-    'and nothing is at risk.' +
-    '\n\n  If you did not update it: copy anything you want to keep into ' +
-    '.sdk/model/, then re-run with --force.' +
-    '\n  If you did: reinstall the version you had, re-run this command, ' +
-    'and it will check against the right source.')
+    (0 === edited.length ? '' :
+      '\n\n  ' + edited.length + ' of them changed in this project after ' +
+      'an add wrote them (' + COPIES + ' records what it wrote), so updating ' +
+      'would discard those edits.') +
+    (0 === unknown.length ? '' :
+      '\n\n  ' + unknown.length + ' of them predate ' + COPIES + ', so ' +
+      'nothing tells these two apart:' +
+      '\n    - they are LOCAL EDITS, and `--force` will discard them;' +
+      '\n    - or ' + pkgname + ' was already updated out of band (an ' +
+      '`npm update` in another shell), in which case they are merely STALE ' +
+      'and nothing is at risk.' +
+      '\n  If you updated it and changed nothing, --force is safe. To check ' +
+      'first, reinstall the version you had and re-run this command.') +
+    '\n\n  Copy anything you want to keep into .sdk/model/, then re-run ' +
+    'with --force.')
 }
 
 

package/src/action/remove.ts

@@ -29,6 +29,8 @@ import { removeIndexEntries } from './action'
 
 import { doctor } from './doctor'
 
+import { forgetCopies } from './copies'
+
 
 // Everything one `remove` would touch, decided before anything is written.
 type RemovePlan = {
@@ -121,6 +123,8 @@ async function kind_remove(
     }
 
     removed.push(...applyRemove(plan, actx, dryrun))
+
+    forgetCopies(actx, kind, plan.name)
   }
 
   log.info({

package/src/action/resolve.ts

@@ -30,6 +30,9 @@ type Source = {
   // The sdkgen package that provided it, when the source has a manifest.
   // Undefined for a bare `.sdk`-shaped folder, which stays legal.
   package?: string
+
+  // That package's version, read from the same manifest.
+  version?: string
 }
 
 
@@ -113,6 +116,8 @@ function resolveSource(ref: string, kind: string, ctx$: any): Source {
   const nroot = Path.normalize(root)
   const rootslash = nroot.endsWith(Path.sep) ? nroot : nroot + Path.sep
 
+  const owner = sourcePackage(fs, folder, kind, origname, ctx$)
+
   return {
     name,
     origname,
@@ -125,14 +130,17 @@ function resolveSource(ref: string, kind: string, ctx$: any): Source {
     // separator path that some readers handle and others do not.
     model: definitionPathAny(fs, folder, kind, origname),
 
-    package: sourcePackage(fs, folder, kind, origname, ctx$),
+    ...(null == owner ? {} : {
+      package: owner.name,
+      ...(null == owner.version ? {} : { version: owner.version }),
+    }),
   }
 }
 
 
 function sourcePackage(
   fs: any, folder: string, kind: string, origname: string, ctx$: any,
-): string | undefined {
+): { name: string, version?: string } | undefined {
   const read = readManifest(fs, folder)
 
   if (null != read.err) {
@@ -164,7 +172,12 @@ function sourcePackage(
     return undefined
   }
 
-  return read.manifest.name
+  const version = read.manifest.version
+
+  return {
+    name: read.manifest.name,
+    ...('string' === typeof version && '' !== version ? { version } : {}),
+  }
 }