@voxgig/sdkgen 4.33.0 → 4.34.0

package/src/action/copies.ts

@@ -0,0 +1,219 @@
+import Path from 'node:path'
+
+import { createHash } from 'node:crypto'
+
+import type { ActionContext } from '../types'
+
+import {
+  definitionFileName, definitionFolder, indexName,
+} from '../helpers/definition'
+
+import { KINDS } from './kind'
+
+import type { Source } from './resolve'
+
+
+// What every add last wrote into `.sdk`, by fingerprint, and the version of
+// the package each item came from. It is the only record that tells a copy
+// the project changed from one its source has since moved past.
+const COPIES = 'sdkgen-copies.json'
+
+const ABOUT = 'Written by voxgig-sdkgen on every add: what it copied into ' +
+  '.sdk, so that doctor, generate and package update can tell an outdated ' +
+  'copy from a local edit. Commit it; do not edit it.'
+
+
+type CopyItem = {
+  package?: string
+  version?: string
+}
+
+type CopyRecord = {
+  items: Record<string, CopyItem>
+  files: Record<string, string>
+}
+
+
+function copiesPath(folder: string): string {
+  return Path.join(folder, COPIES)
+}
+
+
+function itemKey(kind: string, name: string): string {
+  return kind + '/' + name
+}
+
+
+// CRLF and LF copies of one file share a fingerprint, as doctor compares
+// them. latin1 maps each byte to one character, so nothing else changes.
+function fingerprint(content: any): string {
+  const bytes = Buffer.isBuffer(content) ? content : Buffer.from(String(content))
+  const lf = Buffer.from(bytes.toString('latin1').replace(/\r\n/g, '\n'), 'latin1')
+  return createHash('sha256').update(lf).digest('hex').slice(0, 16)
+}
+
+
+// An unreadable record is treated as absent: every copy is then unrecorded,
+// which is exactly the state of a project from before the record existed.
+function readCopies(fs: any, folder: string): CopyRecord {
+  const empty: CopyRecord = { items: {}, files: {} }
+  const path = copiesPath(folder)
+
+  if (!fs.existsSync(path)) {
+    return empty
+  }
+
+  try {
+    const read = JSON.parse(String(fs.readFileSync(path, 'utf8')))
+    return {
+      items: { ...(read?.items ?? {}) },
+      files: { ...(read?.files ?? {}) },
+    }
+  }
+  catch (err: any) {
+    return empty
+  }
+}
+
+
+function writeCopies(fs: any, folder: string, record: CopyRecord) {
+  const sorted = (obj: Record<string, any>) => Object.fromEntries(
+    Object.keys(obj).sort().map((key: string) => [key, obj[key]]))
+
+  fs.writeFileSync(copiesPath(folder), JSON.stringify({
+    about: ABOUT,
+    items: sorted(record.items),
+    files: sorted(record.files),
+  }, null, 2) + '\n')
+}
+
+
+function provenanceOf(source: Source): CopyItem {
+  return {
+    ...(null == source.package ? {} : { package: source.package }),
+    ...(null == source.version ? {} : { version: source.version }),
+  }
+}
+
+
+// The trees and definition files an add owns. An index is excluded: every
+// add rewrites it and no source holds a copy to compare it with.
+const TREE_ROOTS = Array.from(new Set(Object.values(KINDS)
+  .flatMap((kind: any) => (kind.trees ?? [])
+    .map((tree: any) => String(tree.path).split('{name}')[0]))))
+
+function isCopy(rel: string): boolean {
+  if (TREE_ROOTS.some((root: string) => rel.startsWith(root))) {
+    return true
+  }
+
+  const dir = Path.posix.dirname(rel)
+  const base = Path.posix.basename(rel)
+
+  return Object.keys(KINDS).some((kind: string) =>
+    dir === definitionFolder('', kind).split(Path.sep).join('/') &&
+    base !== indexName(kind) &&
+    base.endsWith(definitionFileName('')))
+}
+
+
+function relativeTo(folder: string, abs: string): string | undefined {
+  const rel = Path.relative(Path.resolve(folder), Path.resolve(abs))
+    .split(Path.sep).join('/')
+
+  return ('' === rel || rel.startsWith('../') || Path.isAbsolute(rel)) ?
+    undefined : rel
+}
+
+
+// Called by each add with what its own jostraca run wrote. A dry run writes
+// nothing, so it records nothing.
+function recordCopies(
+  actx: ActionContext, jres: any, kind: string, sources: Source[],
+) {
+  if (true === actx.opts?.dryrun || null == jres?.files) {
+    return
+  }
+
+  const fs = actx.fs()
+  const folder = actx.folder
+  const record = readCopies(fs, folder)
+
+  const files: string[] = [
+    ...(jres.files.written ?? []), ...(jres.files.unchanged ?? []),
+  ]
+
+  for (const abs of files) {
+    const rel = relativeTo(folder, abs)
+
+    if (null != rel && isCopy(rel) && fs.existsSync(abs)) {
+      record.files[rel] = fingerprint(fs.readFileSync(abs))
+    }
+  }
+
+  for (const source of sources) {
+    record.items[itemKey(kind, source.name)] = provenanceOf(source)
+  }
+
+  pruneMissing(fs, folder, record)
+  writeCopies(fs, folder, record)
+}
+
+
+// After a remove: the item goes, with every file entry nothing holds now.
+function forgetCopies(actx: ActionContext, kind: string, name: string) {
+  const fs = actx.fs()
+  const folder = actx.folder
+
+  if (true === actx.opts?.dryrun || !fs.existsSync(copiesPath(folder))) {
+    return
+  }
+
+  const record = readCopies(fs, folder)
+  delete record.items[itemKey(kind, name)]
+  pruneMissing(fs, folder, record)
+  writeCopies(fs, folder, record)
+}
+
+
+function pruneMissing(fs: any, folder: string, record: CopyRecord) {
+  for (const rel of Object.keys(record.files)) {
+    if (!fs.existsSync(Path.join(folder, ...rel.split('/')))) {
+      delete record.files[rel]
+    }
+  }
+}
+
+
+// Was this project file left exactly as an add wrote it?
+function untouched(
+  fs: any, folder: string, record: CopyRecord, rel: string,
+): boolean | undefined {
+  const recorded = record.files[rel]
+
+  if (null == recorded) {
+    return undefined
+  }
+
+  const abs = Path.join(folder, ...rel.split('/'))
+
+  return fs.existsSync(abs) && recorded === fingerprint(fs.readFileSync(abs))
+}
+
+
+export type {
+  CopyItem,
+  CopyRecord,
+}
+
+export {
+  COPIES,
+  copiesPath,
+  itemKey,
+  fingerprint,
+  readCopies,
+  recordCopies,
+  forgetCopies,
+  untouched,
+  isCopy,
+}

package/src/action/doctor.ts

@@ -37,6 +37,9 @@ import { findFeatureSources } from '../helpers/featureSource'
 
 import { isJunk } from '../helpers/junk'
 
+import { readCopies, untouched, itemKey } from './copies'
+import type { CopyRecord } from './copies'
+
 
 const IGNORED_RE = /(~|-jostraca-off)$/
 
@@ -97,10 +100,41 @@ type DoctorReport = {
   // it is what an alias is FOR.
   aliasedDiff: string[]
 
+  // Differs from its source, but the copy record shows it untouched since an
+  // add wrote it: the source moved on, so refreshing it discards nothing.
+  outdated: string[]
+
+  // The forked and edited findings the copy record has no entry for, so
+  // nothing shows whether the project changed them.
+  unrecorded: string[]
+
+  // Per item with findings, keyed `<kind>/<name>`.
+  byItem: Record<string, ItemDrift>
+
   ok: boolean
 }
 
 
+type ItemDrift = {
+  // The package the item resolves to now, which is what refreshes it.
+  package?: string
+
+  // The version its copies came from, and the version installed now.
+  from?: string
+  to?: string
+
+  outdated: number
+
+  // Forked, edited, missing and stale files.
+  changed: number
+}
+
+
+type Category = 'forked' | 'edited' | 'missing' | 'stale'
+
+type Found = (category: Category, label: string) => void
+
+
 const CMD_MAP: any = {
   check: cmd_doctor_check,
   prune: cmd_doctor_prune,
@@ -243,9 +277,12 @@ async function doctor(
   const report: DoctorReport = {
     forked: [], edited: [], stale: [], missing: [], additive: [],
     superseded: [], unwired: [], orphanModel: [],
-    resyncPending: [], aliasedDiff: [], ok: true,
+    resyncPending: [], aliasedDiff: [],
+    outdated: [], unrecorded: [], byItem: {}, ok: true,
   }
 
+  const copies = readCopies(fs, root)
+
   report.superseded = supersededFiles(actx)
 
   report.orphanModel = orphanModelFiles(actx)
@@ -285,12 +322,14 @@ async function doctor(
         continue
       }
 
+      const found = noteDrift(actx, report, copies, kind, source)
+
       if ('target' === kind) {
-        checkTarget(actx, source, report, selected)
+        checkTarget(actx, source, report, found, selected)
       }
 
       if ('edition' === kind) {
-        checkEdition(actx, source, report)
+        checkEdition(actx, source, report, found)
       }
 
       // Only an ACTIVE feature has source copied out; an inactive one's
@@ -298,10 +337,10 @@ async function doctor(
       if ('feature' === kind &&
         (false !== (model as any)?.main?.[KIT]?.feature?.[name]?.active ||
           true === selected?.includes(name))) {
-        checkFeatureSource(actx, source, targets, report)
+        checkFeatureSource(actx, source, targets, found)
       }
 
-      checkItemModel(actx, kind, source, report)
+      checkItemModel(actx, kind, source, report, found)
     }
   }
 
@@ -316,7 +355,7 @@ async function doctor(
   // authoritative either way.
   report.ok = 0 === report.forked.length + report.edited.length +
     report.stale.length + report.missing.length + report.superseded.length +
-    report.orphanModel.length
+    report.orphanModel.length + report.outdated.length
 
   for (const [kind, note] of [
     ['forked', 'FORKED (will be reverted by `target add`)'],
@@ -329,12 +368,24 @@ async function doctor(
     ['orphanModel', 'ORPHAN MODEL FILE (on disk, included by nothing, read by nobody)'],
     ['resyncPending', 'RESYNC PENDING (predates provenance; `target add` updates it)'],
     ['aliasedDiff', 'aliased model differs from its origin (project-owned, not drift)'],
+    ['outdated', 'OUTDATED (unchanged since it was copied, but its source has moved on)'],
   ] as [keyof DoctorReport, string][]) {
     for (const file of (report[kind] as string[])) {
       log.info({ point: 'doctor-finding', kind, file, note: note + ': ' + file })
     }
   }
 
+  for (const item of Object.keys(report.byItem).sort()) {
+    const drift = report.byItem[item]
+    if (0 < drift.outdated) {
+      log.info({
+        point: 'doctor-outdated', item, ...drift,
+        note: outdatedNote(item, drift) + ' - `voxgig-sdkgen ' +
+          refreshCommand(item, drift.package) + '` refreshes them'
+      })
+    }
+  }
+
   log.info({
     point: 'doctor-end',
     ok: report.ok,
@@ -348,11 +399,13 @@ async function doctor(
     orphanModel: report.orphanModel.length,
     resyncPending: report.resyncPending.length,
     aliasedDiff: report.aliasedDiff.length,
+    outdated: report.outdated.length,
     note: report.ok ?
       ('.sdk matches the scaffold (' + report.additive.length + ' additive)') :
       ('.sdk has drifted: ' + report.forked.length + ' forked, ' +
         report.edited.length + ' edited, ' + report.stale.length + ' stale, ' +
-        report.missing.length + ' missing')
+        report.missing.length + ' missing, ' + report.outdated.length +
+        ' outdated')
   })
 
   return { report }
@@ -412,7 +465,7 @@ function resolveDeclared(
 
 
 function checkTarget(
-  actx: ActionContext, resolved: Source, report: DoctorReport,
+  actx: ActionContext, resolved: Source, report: DoctorReport, found: Found,
   selected?: string[],
 ) {
   const tname = resolved.name
@@ -466,7 +519,7 @@ function checkTarget(
     { log: quietLog(actx.log), fs: () => fs, folder: root, model },
     tfolder, torigname, tname, features)
 
-  compareTrees(actx, report, trees, {
+  compareTrees(actx, report, found, trees, {
     excludes,
     foreign: (kind: string) => 'edited' === kind ?
       foreignFeatureSource(actx, resolved) : [],
@@ -477,6 +530,7 @@ function checkTarget(
 function compareTrees(
   actx: ActionContext,
   report: DoctorReport,
+  found: Found,
   trees: TreeCompare[],
   opts?: {
     excludes?: RegExp[],
@@ -518,7 +572,7 @@ function compareTrees(
 
     for (const rel of expected) {
       if (!actualSet.has(rel)) {
-        report.missing.push(label + rel)
+        found('missing', label + rel)
         continue
       }
 
@@ -535,7 +589,7 @@ function compareTrees(
 
       if (differs(fs, from, Path.join(tree.project, rel),
         model, tree.replace, undefined, tree.rewrite)) {
-        report[tree.kind].push(label + rel)
+        found(tree.kind, label + rel)
       }
     }
 
@@ -553,7 +607,7 @@ function compareTrees(
         report.additive.push(label + rel)
       }
       else {
-        report.stale.push(label + rel)
+        found('stale', label + rel)
       }
     }
   }
@@ -561,7 +615,7 @@ function compareTrees(
 
 
 function checkEdition(
-  actx: ActionContext, resolved: Source, report: DoctorReport,
+  actx: ActionContext, resolved: Source, report: DoctorReport, found: Found,
 ) {
   const fs = actx.fs()
   const root = actx.folder
@@ -593,7 +647,7 @@ function checkEdition(
     } as TreeCompare]
   })
 
-  compareTrees(actx, report, trees)
+  compareTrees(actx, report, found, trees)
 }
 
 
@@ -677,7 +731,7 @@ function checkFeatureSource(
   actx: ActionContext,
   feature: Source,
   targets: Map<string, Source>,
-  report: DoctorReport,
+  found: Found,
 ) {
   const fs = actx.fs()
   const root = actx.folder
@@ -689,14 +743,14 @@ function checkFeatureSource(
       const label = 'tm/' + tname + '/' + rel
 
       if (!fs.existsSync(project)) {
-        report.missing.push(label)
+        found('missing', label)
         continue
       }
 
       // The same map `feature add` copies with — see helpers/stdrep. A
       // different one here would report every substituted file as edited.
       if (differs(fs, from, project, model, templateReplacements(model, tname))) {
-        report.edited.push(label)
+        found('edited', label)
       }
     }
   }
@@ -705,6 +759,7 @@ function checkFeatureSource(
 
 function checkItemModel(
   actx: ActionContext, kind: string, source: Source, report: DoctorReport,
+  found: Found,
 ) {
   const name = source.name
   const origname = source.origname
@@ -723,7 +778,7 @@ function checkItemModel(
   const label = 'model/' + kind + '/' + Path.basename(project)
 
   if (!fs.existsSync(project)) {
-    report.missing.push(label)
+    found('missing', label)
     return
   }
 
@@ -746,7 +801,92 @@ function checkItemModel(
     return
   }
 
-  report.forked.push(label)
+  found('forked', label)
+}
+
+
+// Files every finding through one place: a forked or edited copy the record
+// shows untouched is outdated instead, and every item keeps a tally.
+function noteDrift(
+  actx: ActionContext, report: DoctorReport, copies: CopyRecord,
+  kind: string, source: Source,
+): Found {
+  const fs = actx.fs()
+  const item = itemKey(kind, source.name)
+  const copied = copies.items[item]
+
+  return (category: Category, label: string) => {
+    let into: Category | 'outdated' = category
+
+    if ('forked' === category || 'edited' === category) {
+      const same = untouched(fs, actx.folder, copies, label)
+
+      if (true === same) {
+        into = 'outdated'
+      }
+      else if (undefined === same) {
+        report.unrecorded.push(label)
+      }
+    }
+
+    report[into].push(label)
+
+    const drift = report.byItem[item] = report.byItem[item] ?? {
+      ...pkgOf(source.package ?? copied?.package),
+      ...(null == copied?.version ? {} : { from: copied.version }),
+      ...(null == source.version ? {} : { to: source.version }),
+      outdated: 0,
+      changed: 0,
+    }
+
+    if ('outdated' === into) {
+      drift.outdated++
+    }
+    else {
+      drift.changed++
+    }
+  }
+}
+
+
+function pkgOf(name?: string): { package?: string } {
+  return null == name ? {} : { package: name }
+}
+
+
+// The command that brings an item's copies up to the installed source. The
+// package form checks first and refuses to overwrite a local edit.
+function refreshCommand(item: string, pkg?: string): string {
+  if (null != pkg) {
+    return 'package update ' + pkg + ' --no-fetch'
+  }
+
+  const [kind, ...name] = item.split('/')
+  return kind + ' add ' + name.join('/')
+}
+
+
+function outdatedNote(item: string, drift: ItemDrift): string {
+  return item.replace('/', ' ') + ': ' + drift.outdated +
+    ' file(s) are unchanged copies' + sinceCopied(drift.package, [drift])
+}
+
+
+// Names both versions only when they differ: a source that changed under the
+// same version (a linked checkout, say) is still a source that moved on.
+function sinceCopied(pkg: string | undefined, drift: ItemDrift[]): string {
+  const froms = Array.from(new Set(drift.map((d: ItemDrift) => d.from)))
+  const to = drift[0].to
+  const from = 1 === froms.length ? froms[0] : undefined
+  const name = pkg ?? 'their source'
+
+  if (null != from && null != to && from !== to) {
+    return ' from ' + name + ' ' + from + ', but ' + name + ' ' + to +
+      ' is installed'
+  }
+
+  return ', but ' + name + (null == to ? '' : ' ' + to) +
+    ' has changed since they were copied'
 }
 
 
@@ -868,13 +1008,19 @@ function renderPair(
   const rawsrc = fs.readFileSync(scaffoldPath, 'utf8')
   const src = null == rewrite ? rawsrc : rewrite(rawsrc)
 
+  // git may rewrite line endings on checkout (core.autocrlf).
   return {
-    expected: template(src, model, { replace }),
-    actual: fs.readFileSync(projectPath, 'utf8'),
+    expected: lfEndings(template(src, model, { replace })),
+    actual: lfEndings(fs.readFileSync(projectPath, 'utf8')),
   }
 }
 
 
+function lfEndings(text: string): string {
+  return text.replace(/\r\n/g, '\n')
+}
+
+
 function quietLog(log: any): any {
   const noop = () => { }
   const quiet: any = { info: noop, debug: noop, warn: log.warn.bind(log), error: noop, trace: noop, fatal: noop }
@@ -883,12 +1029,92 @@ function quietLog(log: any): any {
 }
 
 
+function silentLog(): any {
+  const noop = () => { }
+  const silent: any = { info: noop, debug: noop, warn: noop, error: noop, trace: noop, fatal: noop }
+  silent.child = () => silent
+  return silent
+}
+
+
+// What `generate` checks before it reads the copies: the active items only,
+// and a warning per refresh command rather than a failure, since a
+// difference may be a deliberate edit.
+async function copyCheck(actx: ActionContext): Promise<string[]> {
+  const active = (kind: string, name: string) =>
+    false !== kindCollection(actx.model, kind)?.[name]?.active
+
+  const res: any = await doctor({ ...actx, log: silentLog() }, active)
+
+  return copyWarnings(res.report)
+}
+
+
+function copyWarnings(report: DoctorReport): string[] {
+  const groups = new Map<string, { items: string[], files: number, drift: ItemDrift[] }>()
+
+  const add = (key: string, item: string, files: number, drift: ItemDrift) => {
+    const group = groups.get(key) ?? { items: [], files: 0, drift: [] }
+    group.items.push(item.replace('/', ' '))
+    group.files += files
+    group.drift.push(drift)
+    groups.set(key, group)
+  }
+
+  for (const item of Object.keys(report.byItem).sort()) {
+    const drift = report.byItem[item]
+    const command = refreshCommand(item, drift.package)
+
+    if (0 < drift.outdated) {
+      add('outdated\n' + command, item, drift.outdated, drift)
+    }
+
+    if (0 < drift.changed) {
+      add('changed\n' + command, item, drift.changed, drift)
+    }
+  }
+
+  const lines: string[] = []
+
+  for (const [key, group] of groups) {
+    const [what, command] = key.split('\n')
+    const pkg = group.drift[0].package
+    const to = group.drift[0].to
+    const installed = null == pkg ? 'their source' :
+      (pkg + (null == to ? '' : ' ' + to))
+    const refresh = '`npx voxgig-sdkgen ' + command + '`'
+
+    if ('outdated' === what) {
+      lines.push(group.items.join(', ') + ': ' + group.files +
+        ' file(s) in .sdk are unchanged copies' +
+        sinceCopied(pkg, group.drift) + ', so this run generates from ' +
+        'older templates. Run ' + refresh + ' to refresh them.')
+    }
+    else {
+      lines.push(group.items.join(', ') + ': ' + group.files +
+        ' file(s) in .sdk differ from what ' + installed + ' installs: ' +
+        'changed in this project, or copied before sdkgen recorded its ' +
+        'copies. `npx voxgig-sdkgen doctor` lists them; ' + refresh +
+        ' refreshes them' +
+        (null == pkg ? ', discarding any local edit.' :
+          ', and refuses before it overwrites a local edit.'))
+    }
+  }
+
+  return lines
+}
+
+
 export type {
   DoctorReport,
   DoctorScope,
+  ItemDrift,
 }
 
 export {
   action_doctor,
   doctor,
+  copyCheck,
+  copyWarnings,
+  refreshCommand,
 }